From 81857f16420c6db2781596cb663eaafe211d896f Mon Sep 17 00:00:00 2001 From: Chris McCord Date: Tue, 21 Jul 2026 14:52:49 +0000 Subject: [PATCH 01/16] Expose local membership generation monitor --- CHANGELOG.md | 2 ++ lib/group.ex | 20 ++++++++++++++++++++ test/group_test.exs | 15 +++++++++++++++ 3 files changed, 37 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e01b07f..dbdc0dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,6 @@ ## Unreleased +- Add `Group.monitor_generation/1` so long-lived registration owners can + terminate and re-register when the local membership ETS generation is lost. - **Breaking**: `Group.disconnect/3` now discards the complete local view of each departed cluster — remote entries included, and monitors receive `:unregistered`/`:left` events for them — instead of removing only locally owned rows. Reconnecting resyncs through the normal diff --git a/lib/group.ex b/lib/group.ex index 27ce1eb..6a4c6b2 100644 --- a/lib/group.ex +++ b/lib/group.ex @@ -233,6 +233,26 @@ defmodule Group do def start_link(opts), do: Group.Supervisor.start_link(opts) + @doc """ + Monitors the process that owns the local Group membership generation. + + All local registry and process-group entries are stored in ETS tables owned + by this process. If it exits, those entries no longer exist even when their + owner processes remain alive. Long-lived owners can use this monitor to + terminate and re-register against the next Group generation. + + A generation that exits between lookup and monitor creation still produces + the normal immediate `:DOWN` message for the returned monitor reference. + + Returns `{:ok, pid, monitor_ref}` or `{:error, :not_running}`. + """ + def monitor_generation(name) when is_atom(name) do + case GenServer.whereis(Data.data_name(name)) do + pid when is_pid(pid) -> {:ok, pid, Process.monitor(pid)} + nil -> {:error, :not_running} + end + end + # =========================================================================== # Cluster Management (Node <-> Cluster) # =========================================================================== diff --git a/test/group_test.exs b/test/group_test.exs index d0c1c22..6bc6b80 100644 --- a/test/group_test.exs +++ b/test/group_test.exs @@ -49,6 +49,21 @@ defmodule GroupTest do end end + describe "monitor_generation/1" do + test "notifies long-lived owners when local membership storage exits", %{name: name} do + assert {:ok, generation_pid, monitor_ref} = Group.monitor_generation(name) + + Process.exit(generation_pid, :kill) + + assert_receive {:DOWN, ^monitor_ref, :process, ^generation_pid, :killed} + end + + test "returns not_running for an unknown Group" do + name = :"missing_group_#{System.unique_integer([:positive])}" + assert {:error, :not_running} = Group.monitor_generation(name) + end + end + describe "join/3 and leave/2" do test "joined process appears in members/2", %{name: name} do key = "chat/room/#{System.unique_integer([:positive])}" From fe2cfc940a74b452608a735ec3c041d533c15124 Mon Sep 17 00:00:00 2001 From: Chris McCord Date: Fri, 31 Jul 2026 05:31:15 +0000 Subject: [PATCH 02/16] Add nonblocking anti-entropy replication --- CHANGELOG.md | 32 +- README.md | 138 +- lib/group.ex | 62 +- lib/group/replica.ex | 3234 ++++++++++++++++----- lib/group/replica/data.ex | 1009 ++++++- lib/group/replica/protocol.ex | 30 + lib/group/replica/transport.ex | 97 + lib/group/supervisor.ex | 66 +- priv/bench/README.md | 66 +- priv/bench/lib/group_bench/distributed.ex | 253 +- priv/bench/lib/group_bench/local.ex | 121 +- priv/bench/lib/group_bench/replica.ex | 173 +- test/README.md | 32 +- test/distributed_test.exs | 1539 +++++++++- test/group_test.exs | 180 +- test/replica_adversarial_test.exs | 358 +++ test/support/test_cluster.ex | 273 ++ test/support/test_replica_transport.ex | 140 + 18 files changed, 6793 insertions(+), 1010 deletions(-) create mode 100644 lib/group/replica/protocol.ex create mode 100644 lib/group/replica/transport.ex create mode 100644 test/replica_adversarial_test.exs create mode 100644 test/support/test_replica_transport.ex diff --git a/CHANGELOG.md b/CHANGELOG.md index dbdc0dc..22721e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,23 @@ ## Unreleased +- Replace replica state sends/snapshots with per-origin, generation- and + cluster-epoch-fenced streams: sequenced deltas repair gaps from a bounded + oplog and fall back to exact origin snapshots after pruning. Replica data now + uses a pluggable nonblocking transport (dist Erlang by default via + `send_nosuspend`), while dist Erlang remains the control plane. Nonblocking + control heartbeats lease peer state, requesting a fresh authoritative hello + on generation or epoch-revision changes, so a stopped Group on a connected + VM cannot leave permanent registry or membership rows. Reconnects also sweep + superseded per-shard receive cursors and reconstruct epochless PG rows, so + reordered cluster controls cannot strand live rows from an older epoch. Full + epoch authority is installed once by shard 0; matching data shards exchange + constant-size lane hellos and retain shard-to-shard transport ordering. + Authority capture is serialized with epoch activation, and exact versus + incrementally observed revisions are tracked separately so a concurrent + partial snapshot cannot be mistaken for complete authority. +- Registry authority is retained per origin separately from the visible + winner. Conflict callbacks select the winner; Group now records and + propagates an authoritative loser delete, and each owner node terminates only + its own losing process. This also applies to custom conflict callbacks. - Add `Group.monitor_generation/1` so long-lived registration owners can terminate and re-register when the local membership ETS generation is lost. - **Breaking**: `Group.disconnect/3` now discards the complete local view of each departed @@ -6,14 +25,13 @@ them — instead of removing only locally owned rows. Reconnecting resyncs through the normal snapshot exchange. `connect`/`disconnect` also raise `ArgumentError` for non-binary cluster names instead of silently tolerating them. -- The built-in registry conflict resolver now consistently includes the winner's metadata in - the losing process's `{:group_registry_conflict, key, winner_meta}` exit reason. Custom - `resolve_registry_conflict` callbacks remain responsible for any process exits they require. +- The registry conflict resolver now consistently includes the winner's metadata in + the losing process's `{:group_registry_conflict, key, winner_meta}` exit reason. - **Breaking**: `Group.dispatch/4` remote sends and process-DOWN replication are now - non-suspending and never auto-connect — on a busy or disconnected distribution link the - message is dropped, the link is force-disconnected, and bounded reconnect retries begin (the - same policy replication lanes have used since 0.1.8). Previously dispatch could block the - caller and initiate new connections. + non-suspending and never auto-connect. Busy dispatch drops still force a disconnect and + bounded reconnect retry; replica frames are dropped and repaired by anti-entropy without + disturbing the dist-Erlang control connection. Previously dispatch could block the caller + and initiate new connections. - Configured function-form `extract_meta` callbacks are now applied on reads and lifecycle events (previously they were silently ignored and full metadata was exposed), and invalid `:extract_meta` values raise `ArgumentError` at startup. diff --git a/README.md b/README.md index c3d8fce..0943994 100644 --- a/README.md +++ b/README.md @@ -212,14 +212,16 @@ delivers an event with `previous_meta` set to the old value. All operations are **eventually consistent**: - Writes (`register`, `join`, etc.) return immediately after updating local ETS. -- Changes replicate to other nodes asynchronously over Erlang distribution. +- Changes replicate asynchronously over a configurable, nonblocking replica + transport. Erlang distribution remains the membership/control plane. - During network partitions, nodes may have divergent views. -- When partitions heal, state is re-synced via `cluster_state` messages. +- When connectivity returns, per-origin stream heads repair missing sequence + ranges from a bounded oplog; a lag beyond the retained prefix falls back to + an exact snapshot of that origin's shard/cluster slice. - Registry conflicts (same key registered on two nodes during a partition) can be resolved with a configurable `resolve_registry_conflict` callback. The - built-in resolver kills the losing process with - `{:group_registry_conflict, key, winner_meta}`; custom resolvers control any - process exits themselves. + callback selects a winner; each origin retires and terminates only its own + losing process with `{:group_registry_conflict, key, winner_meta}`. ## Configuration @@ -238,7 +240,11 @@ All operations are **eventually consistent**: replicated_sender_flush_interval: 5, busy_dist_retry_attempts: 300, busy_dist_retry_interval: 1_000, - replicated_pg_receiver_local_request_quota: 8 + replicated_pg_receiver_local_request_quota: 8, + replica_transport: Group.Replica.Transport.Distribution, + replicated_oplog_max_entries: 65_536, + replicated_anti_entropy_interval: 1_000, + replicated_peer_lease_timeout: 15_000 } ``` @@ -257,9 +263,10 @@ All operations are **eventually consistent**: - **`resolve_registry_conflict`** — `{module, function, extra_args}` callback invoked as `apply(mod, fun, [name, key, {pid1, meta1, time1}, {pid2, meta2, time2} | extra_args])`. Called when partition healing or concurrent registration finds the same key - registered on two nodes. Must return the winning pid and is responsible for - any process exits it requires. Runs synchronously inside the shard GenServer — - must return quickly and never block. + registered on two nodes. Must return the winning pid (or neither pid to + reject both). Group records an authoritative delete and terminates a losing + owner only on that owner's local node. The callback runs synchronously inside + the shard GenServer, so it must return quickly and never block. - **`extract_meta`** — `{module, function, args}` or `fun(meta)` applied to metadata on reads and lifecycle events. Useful for stripping internal fields. - **`replicated_pg_receiver_buffer_size`** — max buffered replicated PG @@ -275,18 +282,35 @@ All operations are **eventually consistent**: - **`replicated_sender_flush_interval`** — max outbound buffer age in milliseconds. Defaults to 5. - **`busy_dist_retry_attempts`** — reconnect attempts after a non-suspending - remote send reports a busy link. Defaults to 300. -- **`busy_dist_retry_interval`** — milliseconds between busy-link reconnect - attempts. Defaults to 1,000. -- **`replicated_pg_receiver_local_request_quota`** — local PG requests drained - after each replicated receiver turn. Defaults to 8. + remote dispatch reports a busy dist link. Defaults to 300. Replica transport + frames are simply dropped and repaired instead of forcing a disconnect. +- **`busy_dist_retry_interval`** — milliseconds between dispatch busy-link + reconnect attempts. Defaults to 1,000. +- **`replicated_pg_receiver_local_request_quota`** — legacy-named quota for + queued local shard requests drained per fairness turn while replica data or + cluster controls are busy. Defaults to 8. +- **`replica_transport`** — a module implementing + `Group.Replica.Transport`, or `{module, opts}`. The default adapter uses + `:erlang.send_nosuspend/3`; adapters must return promptly with `:ok`, `:busy`, + or `:disconnected`. Dropped and busy frames are repaired by anti-entropy. +- **`replicated_oplog_max_entries`** — maximum retained replica records per + shard across all local streams. Defaults to 65,536. Pruning never waits for + peer acknowledgements; a peer behind the retained floor receives an exact + snapshot. +- **`replicated_anti_entropy_interval`** — interval in milliseconds for stream + head advertisements and nonblocking control heartbeats. Defaults to 1,000. +- **`replicated_peer_lease_timeout`** — time without a dist-Erlang control + heartbeat before state owned by that Group peer is purged. Defaults to 15,000 + and must exceed the anti-entropy interval. Probes continue after expiry so a + Group restart on a still-connected VM recovers automatically. ## Architecture ``` Group.Supervisor (:"my_app_group_sup") -├── Group.Replica.Data — owns ETS tables and serializes membership writes -├── Group.PeerReconnect — bounded recovery after busy distribution links +├── optional transport child — sideband adapter listener/pool +├── Group.Replica.Data — owns ETS, journal, generations, and epochs +├── Group.PeerReconnect — bounded recovery after busy remote dispatch ├── Group.Replica.Supervisor — supervises N shard GenServers │ ├── Group.Replica (shard 0) │ ├── Group.Replica (shard 1) @@ -310,7 +334,7 @@ contention for unrelated keys. ### ETS Tables -Each shard owns 4 ETS tables: +Each shard has materialized read indexes plus authority/recovery indexes: | Table | Type | Key | Purpose | |---|---|---|---| @@ -319,6 +343,12 @@ Each shard owns 4 ETS tables: | `pg_by_key` | `:ordered_set` | `{cluster, key, pid}` | Group membership lookup | | `pg_by_pid` | `:ordered_set` | `{pid, cluster, key}` | Reverse index for death cleanup | +Registry claim tables retain one authoritative claim per origin independently +of the visible winner. Stream metadata, oplog, append-order, and receive-cursor +tables support crash replay and gap repair. Keeping claims separate from the +single visible `reg_by_key` projection prevents a losing-but-still-live remote +claim from being forgotten before its owner emits an authoritative delete. + Plus 3 shared tables: - `cluster_nodes` (`:bag`, cluster→nodes) @@ -340,29 +370,66 @@ nodes. This handshake: 1. Validates that shard counts match (raises on mismatch). 2. Exchanges cluster membership lists. -3. Each shard sends its locally owned registry and group slice in a - `cluster_state` message for each shared cluster. - -This is how a new node catches up to the existing cluster state. +3. Shard 0 exchanges protocol version, origin generation, and one complete + active named-cluster epoch snapshot per node. Matching data shards exchange + only constant-size lane/transport descriptors tied to that authority + revision. + +Constant-size heartbeats renew the peer lease. If an origin generation or +cluster-epoch revision changes, the receiver requests a fresh authoritative +hello; if heartbeats stop, lease expiry purges that origin's complete local +view and discovery probes allow it to rejoin later. + +Incremental cluster open/close controls are generation fenced, receiver +batched, and installed by shard 0 into one node-wide authority table. The +highest observed revision keeps heartbeats constant-size during a burst; after +the burst becomes quiet, one authoritative hello closes any gaps left by +dropped or reordered controls. Per-shard view rows record only constant-size +lane readiness; they do not copy the epoch map. Snapshot capture is serialized +with local epoch activation, so its revision and epoch rows are one coherent +point-in-time value. The highest observed incremental revision is tracked +separately and can never promote a partial view to exact authority. Discovery +hints never mutate membership on their own. Authority installation fans a +local fence to every lane, which sweeps only that lane's retained receive +streams. Because PG rows intentionally do not carry protocol epochs, a +superseded origin/cluster slice is cleared and its current cursor reset so the +next head reconstructs it from retained deltas or an exact snapshot. + +Replica state itself does not travel on the control plane. Once the hello is +fenced, stream-head exchange on the replica transport catches the peer up. ### Replication -After the initial sync, steady-state changes propagate through separate sender -and receiver batching lanes: - -- local writes enqueue outbound registry or PG replication in shard-local sender - buffers -- sender flushes group those ops by target node and send one - `replicate_registry_batch` or `replicate_pg_batch` message per remote node -- remote shards buffer those replicated registry / PG ops receiver-side, apply - them in FIFO order with bulk ETS operations, then take a bounded fairness turn - before yielding back to the mailbox - -The sender flush timer is mainly a fallback for idle periods. Outbound buffers -also flush immediately when they hit the configured size, when a new enqueue +Every local mutation is first appended to a stream identified by +`{group, origin_node, origin_generation, shard, cluster, cluster_epoch}` and a +strictly increasing sequence number. It is then applied to the materialized +ETS view and batched into one delta frame per target. Process-death registry +and PG removals can share one record and retain their one-event-batch behavior. + +Receivers advance a cursor only across a contiguous sequence prefix. A gap +requests the missing suffix. Repeated head advertisements recover a dropped +tail even when no later write occurs. If the requested sequence is older than +the bounded oplog floor, the origin sends an exact snapshot of only its own +registry claims and PG memberships; absence from that snapshot is a delete. + +There are no leaders, quorum acknowledgements, tombstones, or known-membership +retention barriers. Oplog memory is bounded locally and independently of slow +peers. Deletes are normal ordered records while retained, and exact snapshots +close gaps after pruning. + +The sender flush timer is mainly a fallback for idle periods. The unified +outbound buffer also flushes immediately when it hits the configured size, when a new enqueue finds the buffer already past its flush interval, and before control or routing work such as cluster connect/disconnect or peer-protocol handling. +Transport ordering is not required for correctness: each shard serializes +writes, each stream numbers them, and receivers reject gaps and duplicates. +Per-shard ordered delivery is still a useful fast path. Cross-stream order is +not a correctness dependency; cluster epochs reject data racing a disconnect +or reconnect, and generation fencing rejects data from a restarted origin. +An alternative sideband adapter authenticates the peer as a dist-Erlang node +and calls `Group.Replica.Transport.deliver/4` locally. + ### Named Cluster TTL Leases Named-cluster TTLs are a local way to reduce replication fanout to nodes that @@ -383,7 +450,8 @@ no longer care about a cluster. Shards monitor all registered/joined processes. On `DOWN`, the shard: 1. Removes entries from both the primary and reverse-index ETS tables. -2. Groups removed entries by peer and sends one non-suspending process-down batch per peer. +2. Appends authoritative unregister/leave mutations before deleting the rows, + then sends one non-suspending sequenced delta batch per peer. 3. Fires `:unregistered` / `:left` events to local monitors. ### Node Disconnect diff --git a/lib/group.ex b/lib/group.ex index 6a4c6b2..f125efd 100644 --- a/lib/group.ex +++ b/lib/group.ex @@ -10,15 +10,16 @@ defmodule Group do ## Consistency Model - All operations are **eventually consistent**. The built-in replication layer uses - Erlang distribution to propagate state across nodes, which means: + All operations are **eventually consistent**. Erlang distribution remains + the membership/control plane; replica state uses a configurable nonblocking + transport with sequenced anti-entropy streams. This means: - Writes (register, join, etc.) return immediately after local update - - Other nodes receive updates asynchronously via Erlang distribution + - Other nodes receive updates asynchronously over the replica transport - During network partitions, nodes may have divergent views - When partitions heal, conflicts are resolved. The built-in resolver kills - the losing process with `{:group_registry_conflict, key, winner_meta}`; - custom resolvers control any process exits themselves + each losing origin records an authoritative delete and terminates only its + own local process with `{:group_registry_conflict, key, winner_meta}` ## Clusters @@ -195,10 +196,9 @@ defmodule Group do - `:resolve_registry_conflict` — `{module, function, extra_args}` callback invoked when two nodes hold the same registry key (partition heal or concurrent registration). Called as `apply(module, function, [name, key, {pid1, meta1, time1}, {pid2, meta2, time2} | extra_args])`. - Must return the winner pid and is responsible for any process exits it requires. When - no callback is configured, the built-in resolver kills the loser with - `{:group_registry_conflict, key, winner_meta}`. **Important:** This callback runs - synchronously inside the shard GenServer — it must + Must return the winner pid (or neither contender to reject both). Group records + an authoritative delete and terminates a losing process only on its owner node. + **Important:** This callback runs synchronously inside the shard GenServer — it must return quickly and never block. Any information needed for the decision should be carried in the registration metadata, not fetched at resolution time. - `:extract_meta` — `{module, function, args}` or a one-argument function to @@ -218,13 +218,26 @@ defmodule Group do buffer replicated outbound ops before flushing during idle periods. Sender buffers also flush on size, overdue enqueue, and control/routing barriers (default: `5`) - - `:busy_dist_retry_attempts` — max reconnect attempts after a shard hits - `send_nosuspend == false` to a remote node and forces a disconnect + - `:busy_dist_retry_attempts` — max reconnect attempts after a remote + `Group.dispatch/4` send reports a busy dist link and forces a disconnect (default: `300`) - - `:busy_dist_retry_interval` — interval in milliseconds between reconnect - attempts after a busy-dist disconnect (default: `1_000`) - - `:replicated_pg_receiver_local_request_quota` — max queued local PG shard requests - drained after each replicated PG flush before yielding (default: `8`) + - `:busy_dist_retry_interval` — interval in milliseconds between dispatch + busy-link reconnect attempts (default: `1_000`) + - `:replicated_pg_receiver_local_request_quota` — legacy-named quota for queued + local shard requests drained in each fairness turn, including while replica + data or cluster controls are busy (default: `8`) + - `:replica_transport` — replica data transport module or `{module, opts}` tuple. + Defaults to `Group.Replica.Transport.Distribution`. The transport must be + nonblocking and may return `:busy`; anti-entropy repairs dropped frames. + - `:replicated_oplog_max_entries` — maximum retained replica records per shard + before old prefixes are pruned and lagging peers require a snapshot + (default: `65_536`) + - `:replicated_anti_entropy_interval` — milliseconds between repeated stream + head advertisements (default: `1_000`) + - `:replicated_peer_lease_timeout` — milliseconds without a dist-Erlang + replica heartbeat before remote Group state is purged (default: `15_000`). + Must be greater than `:replicated_anti_entropy_interval`. Discovery probes + an expired peer so a restarted Group can recover automatically. """ def child_spec(opts) do name = Keyword.fetch!(opts, :name) @@ -1094,6 +1107,7 @@ defmodule Group do @doc false def connect_clusters(name, clusters, timeout) when is_atom(name) and is_list(clusters) and is_integer(timeout) do + _epochs = Data.activate_local_clusters(name, clusters) Data.add_cluster_node(name, clusters, node()) notify_shard = :rand.uniform(get_config(name).num_shards) - 1 @@ -1108,17 +1122,25 @@ defmodule Group do @doc false def disconnect_clusters(name, clusters, timeout) when is_atom(name) and is_list(clusters) and is_integer(timeout) do + _epochs = Data.deactivate_local_clusters(name, clusters) Data.remove_cluster_node(name, clusters, node()) num_shards = get_config(name).num_shards shard_names = for i <- 0..(num_shards - 1), do: Replica.shard_name(name, i) - Replica.local_request_all( - shard_names, - {:cluster_disconnect, clusters}, - timeout - ) + result = + Replica.local_request_all( + shard_names, + {:cluster_disconnect, clusters}, + timeout + ) + + # Keep the remote routing rows through the shard barrier so any buffered + # records are dispatched before the cluster-close control message. Once + # every shard has crossed the barrier, no cluster rows may remain locally. + Data.remove_clusters(name, clusters) + result end # =========================================================================== diff --git a/lib/group/replica.ex b/lib/group/replica.ex index 933c795..95bb6b6 100644 --- a/lib/group/replica.ex +++ b/lib/group/replica.ex @@ -6,428 +6,124 @@ defmodule Group.Replica do @process_down_batch_size 32 @replicated_pg_receiver_flush_timer :flush_replicated_pg_receiver_buffer @replicated_registry_receiver_flush_timer :flush_replicated_registry_receiver_buffer - @replicated_pg_broadcast_flush_timer :flush_replicated_pg_broadcast_buffer - @replicated_registry_broadcast_flush_timer :flush_replicated_registry_broadcast_buffer + @replica_broadcast_flush_timer :flush_replica_broadcast_buffer + @anti_entropy_timer :group_replica_anti_entropy @local_request_tag :group_local_request @local_reply_tag :group_local_reply + @protocol_version Group.Replica.Protocol.version() _archdoc = ~S""" - Sharded GenServer: peer discovery, replication, monitoring, conflict resolution. - - One per shard. Registered as :"#{name}_replica_#{shard_index}". - - ## Message Protocol - - | Message | Direction | Purpose | - |------------------------------------------------------------|----------------|----------------------------------| - | `{:peer_connect, pid, shard, num_shards, clusters}` | A→B (per-shard)| Establish peer relationship | - | `{:peer_connect_ack, pid, shard, num_shards, clusters}` | B→A (per-shard)| Acknowledge peer | - | `{:cluster_state, cluster, reg_data, pg_data}` | both | Per-cluster data snapshot | - | `{:replicate_registry_batch, ops}` | broadcast | Propagate batched registry ops | - | `{:replicate_pg_batch, ops}` | broadcast | Propagate batched PG ops | - | `{:cluster_connect, clusters, pid}` | S→remote S | Node joining named clusters | - | `{:cluster_connect_ack, clusters, pid, cluster_data}` | S→remote S | Ack + bundled shard data | - | `{:cluster_disconnect, clusters, pid}` | shard 0→remote | Node leaving named clusters | - | `{:send_cluster_data, clusters, target_node}` | local fan-out | Notify siblings: send shard data | - | `{:group_dispatch, pids, message}` | caller→remote | Per-node fan-out for dispatch | - - ## Protocol Flows - - ### 1. Peer Discovery (nodeup or init) - - Triggered by `nodeup` or `init`. Each shard independently discovers its - counterpart on the remote node. Both sides exchange cluster lists, then send - per-cluster `cluster_state` snapshots for shared clusters (always includes - nil). Merge applies data; conflicts with local entries go through - `resolve_conflict`. - - Node A shard i Node B shard i - ──────────── ──────────── - │ │ - │ {:peer_connect, pid, i, N, clusters} │ - │──────────────────────────────────────>│ - │ │── add A to nil cluster ETS - │ │── compute shared clusters - │ │── add A to shared named clusters - │ │── monitor A's shard pid - │ │ - │ {:peer_connect_ack, pid, i, N, clusters} - │<──────────────────────────────────────│ - │── add B to nil cluster ETS │ - │── compute shared clusters │ {:cluster_state, C, reg, pg} - │── add B to shared named clusters │──────────────────────────────>│ - │── monitor B's shard pid │ (one per shared cluster) │ - │ │ │ - │ {:cluster_state, C, reg, pg} │ │ - │──────────────────────────────>│ │ │ - │ (one per shared cluster) │ │ │ - │ │ │ │ - ▼ ▼ ▼ ▼ - merge_remote_cluster_data merge_remote_cluster_data - ├─ no conflict: insert ├─ no conflict: insert - ├─ local vs remote: resolve_conflict (default kills loser; re-broadcast winner) - └─ both remote: timestamp wins └─ both remote: timestamp wins - - ### 2. Steady-State Replication - - After peer discovery, local writes enqueue outbound registry or PG replication - in separate sender buffers. Flushes group those ops by target node and send - one `{:replicate_registry_batch, ops}` or `{:replicate_pg_batch, ops}` per - remote node. The nil cluster uses `remote_shards` (per-shard map); named - clusters use `cluster_nodes` ETS. Reads (`lookup`, `members`) go directly to - ETS — no GenServer involved. - - Node A shard i Node B shard i - ──────────── ──────────── - │ │ - Group.register(name, key, meta) │ - │── ETS insert (by_key + by_pid) │ - │── monitor pid │ - │── enqueue sender-side registry op │ - │ │ - │ {:replicate_registry_batch, [ ... ]} - │──────────────────────────────────────>│ - │ │── enqueue receiver-side - │ │ registry ops - │ │── flush in FIFO order - │ │ ├─ nil: insert - │ │ ├─ same pid: update - │ │ ├─ local conflict: - │ │ │ resolve_conflict() - │ │ └─ both remote: - │ │ timestamp wins - │ │ - Group.join(name, group, meta) │ - │── ETS insert (by_key + by_pid) │ - │── enqueue sender-side PG op │ - │ │ - │ {:replicate_pg_batch, [ ... ]} - │──────────────────────────────────────>│ - │ │── enqueue receiver-side PG - │ │ ops - │ │── flush in FIFO order - │ │ (no overwrite conflict - │ │ for PG) - │ │ - Group.dispatch(name, group, msg) │ - │── send directly to local pids │ - │── group remote pids by node │ - │── hash self() to pick shard j │ - │ │ - │ {:group_dispatch, pids, msg} Node B shard j - │────────────────────────────────────────>│ - │ │── send msg to each local pid - │ │ - - Dispatch groups remote PG members by node and sends one - `:group_dispatch` message per remote node, reducing cross-node - messages from O(members) to O(nodes). The target shard is chosen - by hashing the caller's pid (`phash2(self(), num_shards)`), so - back-to-back dispatches from the same caller always route through - the same shard, preserving per-sender message ordering. - - ### 3. Named Cluster Connect (random shard S + fan-out) - - `Group.connect/2` adds local node to ETS, picks random shard S, and sends - one GenServer.call. Shard S notifies remote shard S, which acks with bundled - data and fans out to siblings. Randomizing S load-balances across shards - when many concurrent connects happen. - - Node A Node B - ────── ────── - Group.connect(name, "game") - │── ETS: add self to "game" - │── pick random shard S - │ - Shard S Shard S - ─────── ─────── - │ │ - │ {:cluster_connect, ["game"], pid} │ - │──────────────────────────────────────>│ - │ │── ETS: add A to "game" - │ │── bundle shard S data - │ │ - │ {:cluster_connect_ack, ["game"], pid, [{cluster, reg, pg}]} - │<──────────────────────────────────────│ - │ │ - │ Shard S sends to siblings: - │ {:send_cluster_data, ["game"], A} - │ │ - │ Shards 0..N (except S): - │ │── {:cluster_state, "game", reg, pg} - │ │──────────────────────────────>│ - │ │ (to matching A shard) │ - │ │ │ - │── merge bundled ack data │ - │── ETS: add B to "game" │ - │── send shard S cluster_state ────────>│ - │── fan out to siblings: │ - │ {:send_cluster_data, ["game"], B} │ - │ │ - Shards 0..N (except S): │ - │── {:cluster_state, "game", reg, pg} │ - │──────────────────────────────────────>│ - │ (to matching B shard) │ - - ### 4. Named Cluster Disconnect (all shards local + shard 0 broadcast) - - `Group.disconnect/2` removes local node from ETS, then calls ALL local shards - to purge their entries. Only shard 0 broadcasts to remote shard 0, which fans - out to siblings for per-shard purge. - - Node A Node B - ────── ────── - Group.disconnect(name, "game") - │── ETS: remove self from "game" - │ - Shards 0..N (all called): - │── purge own entries for "game"+A - │── dispatch :unregistered/:left events - │ - Shard 0 only: - │ {:cluster_disconnect, ["game"], pid} - │──────────────────────────────────────>│ Shard 0 - │ │── ETS: remove A from "game" - │ │── fan out to siblings: - │ │ {:cluster_disconnect, ["game"], pid} - │ │ - │ Shards 0..N: - │ │── purge entries for "game"+A - │ │── dispatch events - - ### 5. Partition Heal (peer discovery re-runs + conflict resolution) - - When a partition heals, `nodeup` triggers peer discovery on both sides. - Both exchange `cluster_state` snapshots. Registry key conflicts where the - existing entry is local go through `resolve_conflict` — the same path used - for live contention. The default resolver kills the loser process. - - The tiebreaker must be deterministic regardless of which node is resolving. - The default uses timestamp comparison, with pid ordering as a tiebreaker - when timestamps are equal (`pid2 > pid1`). Erlang pids have a total order - (by node name then id), so this produces the same winner on all nodes. - Using a perspective-dependent tiebreaker (e.g. "remote wins on ties") would - cause mutual kill — both nodes pick the other's pid, both processes die. - - Node A Node B - ────── ────── - (partition: A and B both register key K) - A has: {K, pid_a, time_a, local} B has: {K, pid_b, time_b, local} - │ │ - ─────── partition heals (nodeup) ──────────── - │ │ - │ peer_connect / peer_connect_ack │ - │<─────────────────────────────────────>│ - │ │ - │ {:cluster_state, nil, [{K, pid_b, ...}], []} - │<──────────────────────────────────────│ - │ │ - │ {:cluster_state, nil, [{K, pid_a, ...}], []} - │──────────────────────────────────────>│ - │ │ - merge: K exists locally merge: K exists locally - resolve_conflict( resolve_conflict( - local={pid_a, time_a}, local={pid_b, time_b}, - remote={pid_b, time_b}) remote={pid_a, time_a}) - │ │ - (assuming time_b > time_a): (assuming time_b > time_a): - pid_b wins (remote) pid_b wins (local) - ├─ kill pid_a ├─ kill pid_a (cross-node, idempotent) - ├─ delete pid_a entry ├─ re-insert pid_b with new timestamp - ├─ insert pid_b └─ re-broadcast pid_b - ├─ demonitor pid_a in a registry batch - ├─ dispatch :unregistered(pid_a) │──────────────────────────>│ - │ │ (arrives as same-pid │ - │ │ update — harmless) │ - - ### 6. Nodedown / Process Death Cleanup - - `nodedown` purges all remote node data. Local process `DOWN` purges the pid's - entries and broadcasts unregister/leave to cluster members. - - Node A Node B dies - ────── ────────── - │ X - {:nodedown, B} │ - │ │ - All shards (each independently): │ - │── purge_cluster_node(B) │ - │ (remove B from all cluster_nodes │ - │ and node_clusters — idempotent, │ - │ guards against late peer_connect) │ - │ │ - │── purge_node(shard, B) │ - │ (scan by_key for node==B, │ - │ delete from both by_key + by_pid) │ - │── dispatch :unregistered/:left events │ - │── remove B from remote_shards │ - - ────────────────────────────────────────────── - - Local process dies Node B - ────────────────── ────── - {:DOWN, mref, :process, pid, reason} │ - │ │ - Owning shard: │ - │── delete_all_for_pid(shard, pid) │ - │ (scan by_pid, delete from by_key, │ - │ match_delete from by_pid) │ - │── enqueue unregister/leave ops │ - │ into replicated sender batches ──────>│── delete if pid matches - │── demonitor pid │ - │── dispatch events │ - - ## Cluster Membership Tracking - - The nil cluster is tracked in ETS (cluster_nodes table), maintained by the - peer_connect protocol. Nodes are added on peer discovery and removed on - nodedown/shard death. This allows Group.nodes/1 to return actual Group peers - rather than all Erlang nodes. - - ## Sharding - - Each key is routed to a shard via `:erlang.phash2({cluster, key}, num_shards)`. - Including `cluster` in the hash input means the same key string in different - clusters may land on different shards — this is intentional so named-cluster - operations don't create false contention with nil-cluster operations. - - `phash2` produces near-uniform distribution across shards for diverse keyspaces. - With 10K distinct keys across 2–8 shards, observed deviation from perfect - uniformity is <2%. In practice, real workloads with varied key prefixes will - see balanced shard load. - - **Hot keys:** A single extremely popular key (e.g. a chat room - with thousands of joins/leaves) always hashes to one shard, so all *writes* - for that key serialize through that shard's GenServer. However, *reads* — - `Group.lookup/3` and `Group.members/3` — go directly to ETS and bypass the - GenServer entirely. Since reads typically dominate, a hot key's impact on - overall throughput is limited to write-heavy scenarios. Adding more shards - does not help a single hot key (it still lands on one shard), but it does - reduce contention between unrelated keys. - - Shard counts must match across all nodes in a cluster. The peer_connect - handshake validates `num_shards` and raises on mismatch, since a disagreement - would route the same key to different shards on different nodes, breaking - replication consistency. - - ## Conflict Resolution is Synchronous - - The `:resolve_registry_conflict` callback runs synchronously inside the shard - GenServer's `handle_info` (during replicated registry apply or - `merge_remote_cluster_data`). - This is intentional: the resolver's return value determines ETS mutations (delete - loser entry, insert winner, demonitor evicted local pid, re-broadcast winner) that - must happen atomically within a single `handle_info` turn. Making the resolver async - would open a window where another replicated registry update, `DOWN`, or `cluster_state` - for the same key could race with the pending resolution, corrupting the dual-index - ETS tables. - - Consequence: a blocking resolver stalls the **entire shard** — no registrations, - joins, replication, or cleanup can proceed on that shard until the callback returns. - Callers must ensure their resolver returns quickly. Any information needed for the - decision (e.g. priority, version, creation time) should be carried in the - registration metadata, not fetched at resolution time. - - ## Monitor Event Delivery - - Lifecycle events (`:registered`, `:unregistered`, `:joined`, `:left`) are delivered - to `Group.monitor/3` subscribers in a batched diff of `{:group, events, info}` tuples. - Each GenServer handler invocation is a natural batch boundary: - - - **Single local operations** (register, join, leave, unregister): build one - event, deliver one tuple with one event per matching subscriber. - - **Buffered replicated operations** (batched replicated registry and PG - ops): receiver shards may accumulate several ops before flushing, then - deliver one tuple per subscriber containing the ordered events from that - flush. - - **Bulk operations** (nodedown, process DOWN, cluster_disconnect, cluster_state - merge): accumulate events into a local variable, then deliver one tuple per - subscriber containing all matching events from that handler turn. - - Events are built by `build_event/6`, accumulated in reverse via prepend, and - flushed by `notify_monitors/2` which reverses once, resolves only the monitor - keys that can match each event (`:all`, `{:exact, key}`, and the key's - slash-terminated prefixes), caches those lookups per batch, and sends one - `{:group, events, %{name: name}}` per subscriber. Both functions are private to - this module. - - `resolve_conflict/5` returns `{state, event_or_nil}` so callers can accumulate - the event. `merge_remote_cluster_data/5` threads `{state, events}` through its - reduce, generating `:registered`/`:joined` events for new entries and conflict - events for existing ones. This means `cluster_state` merges (peer discovery, - partition heal, `Group.connect`) produce batched diffs with all new entries. - `build_purged_events/5` takes an events accumulator and prepends purged-entry - events to it. - - ## Replicated Sender Buffering - - Local writes stage outbound replicated registry and PG operations in separate - sender buffers. On flush, those ops are grouped by target node so one shard - send can carry many logical replication updates. - - Sender-side buffering is not timer-only. A sender buffer flushes when: - - - the lane reaches `replicated_sender_buffer_size` - - a new enqueue notices the lane is already older than - `replicated_sender_flush_interval` - - a control or topology path crosses the sender barrier (`cluster_connect`, - `cluster_disconnect`, peer protocol, `cluster_state`, `DOWN`, `nodedown`, - process-down cleanup, explicit mailbox barriers) - - terminate runs - - The timer is therefore a fallback for idle periods, not the only flush - trigger. A very long single GenServer callback can still delay all of these - flush paths until that callback returns; batching and fairness only apply - between mailbox turns. - - Remote shard sends use `send_nosuspend(..., [:noconnect])`. If a send returns - `false`, Group treats that as a degraded link: it drops the unsent message, - force-disconnects the Erlang node, and enters a bounded reconnect loop for - that node. Recovery only starts from this explicit busy-send path; ordinary - `nodedown` events do not start reconnect retries on their own. - - ## Replicated Receiver Fairness - - Receiver-side batching solves the apply-cost problem for both replicated PG - and replicated registry traffic, but a hot stream in either lane can still - monopolize the shard if every completed replication turn is immediately - followed by another one. - - To keep local latency-sensitive writes from sitting behind an unbounded remote - backlog, the shard gives a bounded local request turn after each completed - replicated PG or replicated registry apply turn: - - - one bounded replicated lane turn (PG or registry) - - then drain any already-waiting cluster/protocol messages - - then drain up to `replicated_pg_receiver_local_request_quota` local PG - `join` / `leave` requests, or one local non-PG request - - then yield back to the GenServer loop - - Contiguous local PG `join` / `leave` requests from that bounded local turn are - staged against an in-memory view and applied with bulk ETS operations, while - replicated registry flushes are staged against an in-memory view per - `{cluster, key}` and applied with bulk ETS operations. Other local request - types still execute sequentially in FIFO order. - - Local callers use an explicit request/reply lane (`send` + monitor + tagged - reply) rather than `GenServer.call/3`, so the replica can selectively receive - one local request turn without reaching into `'$gen_call'` internals. - - The fairness model ensures ordering is preserved where correctness matters: - - - all public local shard calls get protection from replicated PG and registry - backlog, but earlier cluster/protocol messages still run first, avoiding - stale ordering around disconnect, peer discovery, and cluster sync - - FIFO is preserved within the local lane because the selective receive matches - a single broad `@local_request_tag` shape and therefore takes the oldest - queued local request in the mailbox - - local PG batching does not reorder within that local lane; it only batches - contiguous `join` / `leave` messages already collected in FIFO order + Sharded control process for local writes, replica transport, anti-entropy, + process monitoring, and registry conflict projection. + + There is one process per shard, registered as + :"#{name}_replica_#{shard_index}". Reads bypass it and use the materialized + ETS indexes owned by Group.Replica.Data. + + ## Authority and identity + + Each locally owned mutation belongs to one stream: + + {group, origin_node, origin_generation, shard, cluster, cluster_epoch} + + The shard assigns a strictly increasing sequence number and appends the record + to its write-ahead oplog before changing materialized ETS. Data owns the + journal, so a shard crash replays any appended-but-unapplied record before + rebuilding local process monitors. + + A registry's authoritative claims are stored per origin separately from its + single visible winner. Conflict selection folds claims in a stable order. + When a local claim loses, only its owner node appends the authoritative + unregister and terminates the local process. Retaining hidden remote claims + until their origin deletes them prevents a later winner change from orphaning + or permanently forgetting a live claim. + + PG memberships need no winner projection: the origin stream owns exactly the + rows whose member processes live on that origin node. + + ## Wire protocol + + Dist Erlang remains the control plane: + + - peer_connect / peer_connect_ack discover matching shards and clusters. + - shard 0 exchanges replica_hello authority containing the origin generation + and complete active named-cluster epoch set exactly once per node. + - matching nonzero shards exchange constant-size replica_lane_hello messages + containing their transport descriptor and the authority revision they use. + - replica_cluster_open / replica_cluster_close fence named-cluster lifetimes. + - constant-size periodic heartbeats provide a bounded peer lease without + creating remote process monitors. A generation/epoch-revision mismatch + requests a fresh authoritative hello. + + Replica state uses the configured Group.Replica.Transport: + + - heads advertises {stream, retained_floor, head}. + - delta_batch carries one or more contiguous stream runs. + - need requests the receiver's next missing sequence. + - snapshot exactly replaces one origin's registry claims and PG slice when + the requested prefix has already been pruned. + + Every stream field is validated against the authenticated source node and + current generation/epoch. An old generation, a closed epoch, a wrong shard, + or a transitive claim for another node's pid is rejected. Control/data + reordering is safe: early frames are ignored and repeated heads repair them; + late frames fail their generation or epoch fence. + + ## Bounded recovery + + The oplog is bounded per shard, not by peer acknowledgements. A dropped tail + is found by periodic heads. A gap inside the retained range is repaired with + bounded delta batches. A gap below the retained floor receives the existing + full-sync primitive, narrowed to an exact origin/shard/cluster snapshot. + Absence from that snapshot is deletion, so no tombstones are required. + + There is no leader, quorum, retention ACK, or requirement to know all members. + A slow or disconnected peer cannot pin memory. When it returns it repairs from + deltas when possible and a snapshot otherwise. + + ## Nonblocking transport and ordering + + All cross-node control messages use :erlang.send_nosuspend/3 with :noconnect. + The default replica adapter does the same. Transport callbacks return :ok, + :busy, or :disconnected; failure drops the frame and anti-entropy repairs it. + Replica shards never remotely monitor or exit member processes. + + The transport need not order frames for correctness. The local shard + serializes writes, sequence numbers establish per-stream order, and receivers + reject duplicates and gaps. TCP shard-to-shard ordering remains the efficient + fast path. No semantic operation spans clusters, so cross-stream ordering is + unnecessary; generation and cluster-epoch fences cover lifecycle races. + + ## Batching and fairness + + Local writes share one outbound sender buffer so registry and PG mutations + retain mailbox order. Flushes group records by target and stream. Size, age, + control/routing barriers, and idle timers bound the delay. + + Incremental cluster controls are generation fenced, receiver batched, and + installed by shard 0 into the node-wide authority table. Their observed + revision suppresses full-hello storms during bursts; a quiet authoritative + hello repairs any missing or reordered controls. Snapshot capture is + serialized with epoch activation, and the last exact revision is distinct + from the highest incrementally observed revision. Shard-local lane readiness + is separate from shared authority, so no epoch map is copied per shard. + + Incoming PG mutations retain the bulk receiver lane. Contiguous registry + records in one stream run are projected together and emit one monitor event + batch. A mixed process-down record applies maximal same-domain segments in + wire order and emits one combined batch. + + After replicated work, the shard takes a bounded local-request turn before + yielding. FIFO is preserved within the local request lane, while protocol and + cluster barriers flush earlier buffered state first. + + Receive-only handlers for the previous direct batch/snapshot messages remain + for rolling compatibility and tests; protocol v1 never emits them. """ require Logger - alias Group.Replica.Data + alias Group.Replica.{Data, Protocol} defstruct [ :name, @@ -440,24 +136,30 @@ defmodule Group.Replica do :replicated_sender_buffer_size, :replicated_sender_flush_interval, :replicated_pg_receiver_local_request_quota, + :replicated_oplog_max_entries, + :replicated_anti_entropy_interval, + :replicated_peer_lease_timeout, + :replica_transport, + :replica_transport_opts, + :anti_entropy_ref, :pending_replicated_pg_started_at, :pending_replicated_pg_flush_ref, :pending_replicated_registry_started_at, :pending_replicated_registry_flush_ref, - :pending_replicated_pg_broadcast_started_at, - :pending_replicated_pg_broadcast_flush_ref, - :pending_replicated_registry_broadcast_started_at, - :pending_replicated_registry_broadcast_flush_ref, + :pending_replica_broadcast_started_at, + :pending_replica_broadcast_flush_ref, pending_replicated_pg_len: 0, pending_replicated_pg_ops: [], pending_replicated_registry_len: 0, pending_replicated_registry_ops: [], - pending_replicated_pg_broadcast_len: 0, - pending_replicated_pg_broadcast_ops: [], - pending_replicated_registry_broadcast_len: 0, - pending_replicated_registry_broadcast_ops: [], + pending_replica_broadcast_len: 0, + pending_replica_broadcast_ops: [], remote_shards: %{}, - monitors: %{} + peer_last_seen: %{}, + cluster_control_dirty: %{}, + authority_dirty_notified: MapSet.new(), + monitors: %{}, + peer_transports: %{} ] def start_link(opts) do @@ -536,9 +238,20 @@ defmodule Group.Replica do replicated_sender_buffer_size: config.replicated_sender_buffer_size, replicated_sender_flush_interval: config.replicated_sender_flush_interval, replicated_pg_receiver_local_request_quota: - config.replicated_pg_receiver_local_request_quota + config.replicated_pg_receiver_local_request_quota, + replicated_oplog_max_entries: config.replicated_oplog_max_entries, + replicated_anti_entropy_interval: config.replicated_anti_entropy_interval, + replicated_peer_lease_timeout: config.replicated_peer_lease_timeout, + replica_transport: elem(config.replica_transport, 0), + replica_transport_opts: elem(config.replica_transport, 1) } + state = schedule_anti_entropy(state) + + # Complete any write-ahead record left unapplied by a shard crash, then + # rebuild local process monitors from the surviving materialized tables. + state = replay_local_journal(state) + # Rebuild monitors from any surviving ETS data (after shard crash/restart) state = rebuild_monitors(state) @@ -595,11 +308,21 @@ defmodule Group.Replica do # Cluster connect/disconnect (broadcast to all shards, rare operation) # ===================================================================== + def handle_call({:cluster_connect, _, _} = request, _from, state) do + {reply, state} = process_local_request(state, request) + {:reply, reply, state} + end + def handle_call({:cluster_connect, _} = request, _from, state) do {reply, state} = process_local_request(state, request) {:reply, reply, state} end + def handle_call({:cluster_disconnect, _, _} = request, _from, state) do + {reply, state} = process_local_request(state, request) + {:reply, reply, state} + end + def handle_call({:cluster_disconnect, _} = request, _from, state) do {reply, state} = process_local_request(state, request) {:reply, reply, state} @@ -628,6 +351,428 @@ defmodule Group.Replica do {:noreply, state} end + def handle_info( + {:replica_hello, remote_pid, version, generation, epoch_revision, cluster_epochs, + transport_id, transport_descriptor}, + %{shard_index: 0} = state + ) do + state = flush_pending_replicated_message_barrier(state) + remote_node = node(remote_pid) + + known_generation = Data.remote_generation(state.name, remote_node) + observed_revision = Data.remote_cluster_epoch_observed_revision(state.name, remote_node) + authoritative_revision = Data.remote_cluster_epoch_revision(state.name, remote_node) + + exact_revision = Data.remote_cluster_epoch_exact_revision(state.name, remote_node) + + stale_revision? = + known_generation == generation and + Enum.any?([observed_revision, authoritative_revision], fn + revision when is_integer(revision) -> epoch_revision < revision + _ -> false + end) + + cond do + version != Protocol.version() or transport_id != state.replica_transport.id() -> + Logger.error( + "#{log_prefix_shard(state)} incompatible replica protocol/transport from #{inspect(remote_node)}" + ) + + {:noreply, state} + + stale_revision? -> + {:noreply, state} + + known_generation == generation and exact_revision == epoch_revision -> + # Requests and heartbeats may race while one large authority snapshot + # is being installed. Once this exact revision is present, another + # identical hello is only a lease/descriptor refresh; reinstalling its + # full epoch set would serialize every shard behind redundant ETS work. + state = notify_replica_transport_peer_up(state, remote_node, transport_descriptor) + + state = %{ + state + | remote_shards: Map.put(state.remote_shards, remote_node, remote_pid), + peer_last_seen: Map.put(state.peer_last_seen, remote_node, monotonic_millis()), + cluster_control_dirty: Map.delete(state.cluster_control_dirty, remote_node), + peer_transports: + Map.put(state.peer_transports, remote_node, {transport_id, transport_descriptor}) + } + + {:noreply, state} + + true -> + {:noreply, + install_replica_authority( + state, + remote_pid, + generation, + epoch_revision, + cluster_epochs, + transport_id, + transport_descriptor + )} + end + end + + def handle_info( + {:replica_hello, remote_pid, _version, _generation, _epoch_revision, _cluster_epochs, + _transport_id, _transport_descriptor}, + state + ) do + # Full authority is installed only by shard 0. A full hello delivered to a + # data lane cannot be used as its lane identity because remote_pid belongs + # to the remote control shard, not this matching shard. + {:noreply, request_replica_authority(state, node(remote_pid))} + end + + def handle_info( + {:replica_lane_hello, remote_pid, version, generation, epoch_revision, transport_id, + transport_descriptor}, + state + ) do + state = flush_pending_replicated_message_barrier(state) + remote_node = node(remote_pid) + + if version == Protocol.version() and transport_id == state.replica_transport.id() do + if function_exported?(state.replica_transport, :peer_up, 4) do + :ok = + state.replica_transport.peer_up( + state.name, + remote_node, + transport_descriptor, + state.replica_transport_opts + ) + end + + state = %{ + state + | remote_shards: Map.put(state.remote_shards, remote_node, remote_pid), + peer_transports: + Map.put(state.peer_transports, remote_node, {transport_id, transport_descriptor}) + } + + if replica_authority_current?(state, remote_node, generation, epoch_revision) do + :ok = + Data.put_remote_view_info( + state.name, + state.shard_index, + remote_node, + generation, + Data.remote_cluster_epoch_exact_revision(state.name, remote_node), + epoch_revision + ) + + state = + state + |> purge_remote_streams_outside_authority(remote_node) + |> touch_replica_peer(remote_node) + |> Map.update!(:cluster_control_dirty, &Map.delete(&1, remote_node)) + |> send_replica_heads(remote_node) + + {:noreply, state} + else + {:noreply, request_replica_authority(state, remote_node)} + end + else + Logger.error( + "#{log_prefix_shard(state)} incompatible replica protocol/transport from #{inspect(remote_node)}" + ) + + {:noreply, state} + end + end + + def handle_info( + {:replica_authority_installed_local, remote_node, generation, epoch_revision, + old_generation, stale_epochs}, + state + ) do + state = flush_pending_replicated_message_barrier(state) + + if replica_authority_current?(state, remote_node, generation, epoch_revision) do + state = maybe_purge_remote_generation(state, remote_node, old_generation, generation) + + state = + if old_generation == generation do + state + |> purge_closed_remote_epochs(remote_node, stale_epochs) + |> purge_remote_streams_outside_authority(remote_node) + else + state + end + + state = %{ + state + | cluster_control_dirty: Map.delete(state.cluster_control_dirty, remote_node), + authority_dirty_notified: MapSet.delete(state.authority_dirty_notified, remote_node) + } + + state = + if Map.has_key?(state.remote_shards, remote_node) do + state + |> touch_replica_peer(remote_node) + |> send_replica_heads(remote_node) + else + state + end + + {:noreply, state} + else + {:noreply, state} + end + end + + def handle_info({:replica_authority_removed_local, remote_node}, state) do + state = flush_pending_replicated_message_barrier(state) + {:noreply, expire_replica_peer(state, remote_node)} + end + + def handle_info( + {:replica_cluster_open, remote_pid, generation, revision, epochs}, + state + ) do + state = flush_pending_replicated_message_barrier(state) + remote_node = node(remote_pid) + + controls = + collect_replica_cluster_controls( + :replica_cluster_open, + remote_pid, + generation, + [{revision, epochs}], + state.replicated_sender_buffer_size - 1 + ) + + case accepted_replica_cluster_epochs(state, remote_node, generation, controls) do + {:accept, observed_revision, epochs} -> + stale = + Data.put_remote_cluster_epochs( + state.name, + state.shard_index, + remote_node, + observed_revision, + epochs + ) + + shared = + Enum.filter(epochs, fn {cluster, _epoch} -> + node() in Data.cluster_nodes(state.name, cluster) + end) + + Data.add_cluster_node(state.name, Enum.map(shared, &elem(&1, 0)), remote_node) + + fan_out_to_siblings( + state, + {:replica_cluster_open_control_local, remote_node, generation, observed_revision, + epochs, stale, Enum.map(shared, &elem(&1, 0))} + ) + + state = + state + |> mark_authority_dirty(remote_node) + |> purge_closed_remote_epochs(remote_node, stale) + |> purge_superseded_remote_streams(remote_node, epochs) + + state = send_replica_heads(state, remote_node, Enum.map(shared, &elem(&1, 0))) + {:noreply, take_one_local_request_turn(state)} + + :stale -> + {:noreply, take_one_local_request_turn(state)} + + :refresh -> + {:noreply, + state + |> request_replica_authority(remote_node) + |> take_one_local_request_turn()} + end + end + + def handle_info({:replica_authority_dirty_local, remote_node}, %{shard_index: 0} = state) do + {:noreply, mark_cluster_control_dirty(state, remote_node)} + end + + def handle_info({:replica_authority_dirty_local, remote_node}, state) do + send(shard_name(state.name, 0), {:replica_authority_dirty_local, remote_node}) + {:noreply, state} + end + + def handle_info( + {:replica_cluster_open_control_local, remote_node, generation, revision, epochs, stale, + shared}, + state + ) do + state = flush_pending_replicated_message_barrier(state) + + state = + if replica_authority_current?(state, remote_node, generation, revision) do + state + |> purge_closed_remote_epochs(remote_node, stale) + |> purge_superseded_remote_streams(remote_node, epochs) + |> send_replica_heads(remote_node, shared) + else + state + end + + {:noreply, take_one_local_request_turn(state)} + end + + def handle_info({:replica_cluster_stale_epochs_local, remote_node, stale}, state) do + state = flush_pending_replicated_message_barrier(state) + state = purge_closed_remote_epochs(state, remote_node, stale) + {:noreply, send_replica_heads(state, remote_node)} + end + + def handle_info( + {:replica_cluster_close, remote_pid, generation, revision, epochs}, + %{shard_index: 0} = state + ) do + state = flush_pending_replicated_message_barrier(state) + remote_node = node(remote_pid) + + controls = + collect_replica_cluster_controls( + :replica_cluster_close, + remote_pid, + generation, + [{revision, epochs}], + state.replicated_sender_buffer_size - 1 + ) + + case accepted_replica_cluster_epochs(state, remote_node, generation, controls) do + {:accept, observed_revision, epochs} -> + closed = + Data.close_remote_cluster_epochs( + state.name, + 0, + remote_node, + observed_revision, + epochs + ) + + if state.shard_index == 0 do + Data.remove_cluster_node(state.name, Enum.map(closed, &elem(&1, 0)), remote_node) + end + + fan_out_to_siblings( + state, + {:replica_cluster_close_control_local, remote_node, generation, observed_revision, + closed} + ) + + state = + state + |> mark_cluster_control_dirty(remote_node) + |> purge_closed_remote_epochs(remote_node, closed) + + {:noreply, take_one_local_request_turn(state)} + + :stale -> + {:noreply, take_one_local_request_turn(state)} + + :refresh -> + {:noreply, + state + |> request_replica_authority(remote_node) + |> take_one_local_request_turn()} + end + end + + def handle_info({:replica_cluster_close, remote_pid, _generation, _revision, _epochs}, state) do + {:noreply, request_replica_authority(state, node(remote_pid))} + end + + def handle_info( + {:replica_cluster_close_control_local, remote_node, generation, revision, closed}, + state + ) do + state = flush_pending_replicated_message_barrier(state) + + state = + if replica_authority_current?(state, remote_node, generation, revision) do + purge_closed_remote_epochs(state, remote_node, closed) + else + state + end + + {:noreply, take_one_local_request_turn(state)} + end + + def handle_info({:replica_cluster_close_local, remote_node, closed}, state) do + state = flush_pending_replicated_message_barrier(state) + + :ok = + Data.forget_remote_cluster_epochs(state.name, state.shard_index, remote_node, closed) + + {:noreply, purge_closed_remote_epochs(state, remote_node, closed)} + end + + def handle_info( + {:replica_heartbeat, remote_pid, version, generation, epoch_revision}, + state + ) do + remote_node = node(remote_pid) + + state = + if version == Protocol.version() and + replica_authority_current?(state, remote_node, generation, epoch_revision) do + :ok = + Data.put_remote_view_info( + state.name, + state.shard_index, + remote_node, + generation, + Data.remote_cluster_epoch_exact_revision(state.name, remote_node), + epoch_revision + ) + + state + |> put_remote_shard(remote_node, remote_pid) + |> touch_replica_peer(remote_node) + else + request_replica_authority(state, remote_node) + end + + {:noreply, state} + end + + def handle_info({:replica_hello_request, remote_pid}, state) do + if state.shard_index == 0 do + {:noreply, send_replica_hello(state, node(remote_pid))} + else + send(shard_name(state.name, 0), {:replica_hello_request, remote_pid}) + {:noreply, state} + end + end + + def handle_info({:group_replica_frame, remote_pid, frame}, state) when is_pid(remote_pid) do + remote_node = node(remote_pid) + state = handle_replica_frame(state, remote_node, frame) + {:noreply, take_priority_turn(state)} + end + + def handle_info({:group_replica_frame, remote_node, frame}, state) when is_atom(remote_node) do + state = handle_replica_frame(state, remote_node, frame) + {:noreply, take_priority_turn(state)} + end + + def handle_info({@anti_entropy_timer, ref}, state) do + state = + if state.anti_entropy_ref == ref do + state + |> expire_stale_replica_peers() + |> probe_replica_peers() + |> request_quiet_cluster_hellos() + |> broadcast_replica_heartbeats() + |> broadcast_replica_heads() + |> schedule_anti_entropy() + else + state + end + + {:noreply, state} + end + def handle_info({@local_request_tag, caller_pid, ref, request}, state) when is_pid(caller_pid) and is_reference(ref) do {:noreply, process_local_request_turn(state, [{{:send, caller_pid, ref}, request}])} @@ -655,21 +800,17 @@ defmodule Group.Replica do %{name: name, shard_index: shard} = state remote_node = node(remote_pid) - # Compute shared clusters and add the remote node to nil plus every shared - # named cluster in one serialized membership mutation. + # Compute shared clusters for diagnostics only. The generation-fenced hello + # is the sole authority that mutates peer and cluster membership. Keeping + # discovery hints side-effect free prevents a delayed pre-restart + # peer_connect from permanently re-adding stale cluster rows. my_clusters = Data.my_clusters(name) shared = compute_shared_clusters(my_clusters, remote_clusters) - Data.add_cluster_node(name, [nil | Enum.reject(shared, &is_nil/1)], remote_node) - - already_known = Map.has_key?(state.remote_shards, remote_node) - state = - if already_known do - state - else - Process.monitor(remote_pid) - %{state | remote_shards: Map.put(state.remote_shards, remote_node, remote_pid)} - end + # Replica peers are addressed by registered `{name, node}` and established + # only by replica_hello. Do not remotely monitor the shard PID: creating a + # remote monitor itself emits a distribution signal and may suspend on a + # busy dist connection. # Send ack with our cluster list send_to_peer( @@ -678,14 +819,12 @@ defmodule Group.Replica do {:peer_connect_ack, self(), shard, state.num_shards, my_clusters} ) + send_replica_hello(state, remote_node) + log_once(state, fn -> "#{log_prefix(state)} peer_connect from #{remote_node} (#{length(shared)} shared clusters)" end) - # Send cluster_state for all shared clusters in one pass (single table scan - # instead of one scan per cluster — O(N) vs O(C×N)) - send_cluster_states(state, shared, remote_node) - {:noreply, state} end @@ -709,28 +848,16 @@ defmodule Group.Replica do %{name: name} = state remote_node = node(remote_pid) - # Compute shared clusters and add the remote node to nil plus every shared - # named cluster in one serialized membership mutation. + # Discovery acknowledgements are hints only; replica_hello is the sole + # generation-fenced authority for peer and cluster membership. my_clusters = Data.my_clusters(name) shared = compute_shared_clusters(my_clusters, remote_clusters) - Data.add_cluster_node(name, [nil | Enum.reject(shared, &is_nil/1)], remote_node) - - already_known = Map.has_key?(state.remote_shards, remote_node) - - state = - if already_known do - state - else - Process.monitor(remote_pid) - %{state | remote_shards: Map.put(state.remote_shards, remote_node, remote_pid)} - end log_once(state, fn -> "#{log_prefix(state)} peer_connect_ack from #{remote_node} (#{length(shared)} shared clusters)" end) - # Send cluster_state for all shared clusters in one pass - send_cluster_states(state, shared, remote_node) + send_replica_hello(state, remote_node) {:noreply, state} end @@ -787,20 +914,9 @@ defmodule Group.Replica do if shared != [] do Data.add_cluster_node(name, shared, remote_node) - # Bundle this shard's cluster data directly into the ack (one cross-node - # message instead of ack + N separate cluster_state messages) - {reg_by_cluster, pg_by_cluster} = - Data.local_data_by_cluster(name, state.shard_index, shared) - - cluster_data = - for cluster <- shared do - reg_data = Map.get(reg_by_cluster, cluster, []) - pg_data = Map.get(pg_by_cluster, cluster, []) - {cluster, reg_data, pg_data} - end - - send_to_peer(state, remote_node, {:cluster_connect_ack, shared, self(), cluster_data}) - fan_out_to_siblings(state, {:send_cluster_data, shared, remote_node}) + # Membership is a control-plane handshake. Replica state follows on the + # data transport via heads/deltas (or an exact snapshot fallback). + send_to_peer(state, remote_node, {:cluster_connect_ack, shared, self(), []}) end {:noreply, state} @@ -821,7 +937,8 @@ defmodule Group.Replica do if active != [] do Data.add_cluster_node(name, active, remote_node) - # Merge the data bundled in the ack + # The empty data list is the v1 contract. Retain merge support for a + # rolling peer that still bundles legacy cluster data. {new_state, events} = Enum.reduce(cluster_data, {state, []}, fn {cluster, reg_data, pg_data}, {acc_state, acc_events} -> @@ -832,9 +949,7 @@ defmodule Group.Replica do end end) - send_cluster_states(new_state, active, remote_node) - fan_out_to_siblings(new_state, {:send_cluster_data, active, remote_node}) - {new_state, events} + {send_replica_heads(new_state, remote_node), events} else {state, []} end @@ -860,10 +975,24 @@ defmodule Group.Replica do fan_out_to_siblings(state, {:cluster_disconnect, clusters, remote_pid}) end - events = - Enum.reduce(clusters, [], fn cluster, acc -> + {state, events} = + Enum.reduce(clusters, {state, []}, fn cluster, {outer_state, acc} -> + affected_keys = + Data.purge_registry_claims_for_cluster(name, shard, cluster, remote_node) + {purged_reg, purged_pg} = purge_cluster_entries(name, shard, cluster, remote_node) - build_purged_events(name, purged_reg, purged_pg, :cluster_disconnect, acc) + + acc = build_purged_events(name, purged_reg, purged_pg, :cluster_disconnect, acc) + + Enum.reduce(affected_keys, {outer_state, acc}, fn key, {inner_state, inner_events} -> + reconcile_registry_projection( + inner_state, + cluster, + key, + :cluster_disconnect, + inner_events + ) + end) end) notify_monitors(name, events) @@ -900,14 +1029,36 @@ defmodule Group.Replica do # Purge all data from the dead node {purged_reg, purged_pg} = Data.purge_node(name, shard, dead_node) + affected_claims = Data.purge_registry_claims_for_origin(name, shard, dead_node) log_once(state, fn -> "#{log_prefix(state)} nodedown #{dead_node} (purged #{length(purged_reg)} reg, #{length(purged_pg)} pg entries)" end) events = build_purged_events(name, purged_reg, purged_pg, :nodedown) + + {state, events} = + Enum.reduce(affected_claims, {state, events}, fn {cluster, key}, {acc, inner_events} -> + reconcile_registry_projection(acc, cluster, key, :nodedown, inner_events) + end) + notify_monitors(name, events) - state = %{state | remote_shards: Map.delete(state.remote_shards, dead_node)} + + state = %{ + state + | remote_shards: Map.delete(state.remote_shards, dead_node), + peer_last_seen: Map.delete(state.peer_last_seen, dead_node), + authority_dirty_notified: MapSet.delete(state.authority_dirty_notified, dead_node) + } + + Data.delete_replica_cursors_for_origin(name, shard, dead_node) + Data.delete_remote_replica_info(name, shard, dead_node) + + if function_exported?(state.replica_transport, :peer_down, 3) do + :ok = state.replica_transport.peer_down(name, dead_node, state.replica_transport_opts) + end + + state = %{state | peer_transports: Map.delete(state.peer_transports, dead_node)} {:noreply, state} end @@ -926,12 +1077,25 @@ defmodule Group.Replica do # Unconditional (not gated on shard 0) — same reasoning as nodedown handler. Data.purge_cluster_node(name, remote_node) {purged_reg, purged_pg} = Data.purge_node(name, shard, remote_node) + affected_claims = Data.purge_registry_claims_for_origin(name, shard, remote_node) log_verbose(state, fn -> "#{log_prefix_shard(state)} remote_shard_down #{remote_node} (purged #{length(purged_reg)} reg, #{length(purged_pg)} pg)" end) events = build_purged_events(name, purged_reg, purged_pg, {:nodedown, remote_node}) + + {state, events} = + Enum.reduce(affected_claims, {state, events}, fn {cluster, key}, {acc, inner_events} -> + reconcile_registry_projection( + acc, + cluster, + key, + {:nodedown, remote_node}, + inner_events + ) + end) + notify_monitors(name, events) state = %{state | remote_shards: Map.delete(state.remote_shards, remote_node)} state = %{state | monitors: Map.delete(state.monitors, pid)} @@ -947,13 +1111,26 @@ defmodule Group.Replica do pids = Enum.map(downs, &elem(&1, 0)) reason_by_pid = Map.new(downs) + {visible_reg, pending_pg} = Data.entries_for_pids(name, shard, pids) + + claimed_reg = + Data.local_registry_claims_by_pids(name, shard, pids) + |> Enum.map(fn {pid, cluster, key, meta, _generation, _epoch} -> + {pid, cluster, key, meta} + end) + + pending_reg = Enum.uniq(visible_reg ++ claimed_reg) + + sequenced_downs = + append_process_down_records(state, reason_by_pid, pending_reg, pending_pg) + {purged_reg, purged_pg} = Data.delete_all_for_pids(name, shard, pids) log_verbose(state, fn -> "#{log_prefix_shard(state)} process_down_batch pids=#{length(downs)} (#{length(purged_reg) + length(purged_pg)} entries cleaned)" end) - broadcast_process_down_batch(state, reason_by_pid, purged_reg, purged_pg) + state = finish_process_down_records(state, sequenced_downs) events = build_process_down_events(name, purged_reg, purged_pg, reason_by_pid) notify_monitors(name, events) state = %{state | monitors: Map.drop(monitors, pids)} @@ -1024,21 +1201,10 @@ defmodule Group.Replica do {:noreply, state} end - def handle_info({@replicated_pg_broadcast_flush_timer, flush_ref}, state) do - state = - if state.pending_replicated_pg_broadcast_flush_ref == flush_ref do - flush_pending_replicated_pg_broadcast(state) - else - state - end - - {:noreply, state} - end - - def handle_info({@replicated_registry_broadcast_flush_timer, flush_ref}, state) do + def handle_info({@replica_broadcast_flush_timer, flush_ref}, state) do state = - if state.pending_replicated_registry_broadcast_flush_ref == flush_ref do - flush_pending_replicated_registry_broadcast(state) + if state.pending_replica_broadcast_flush_ref == flush_ref do + flush_pending_replica_broadcast(state) else state end @@ -1180,16 +1346,10 @@ defmodule Group.Replica do defp process_local_request_turn( state, - [{_reply_to, request} | _] = initial_messages + initial_messages ) do remaining = - case local_request_domain(request) do - :pg -> - max(state.replicated_pg_receiver_local_request_quota - length(initial_messages), 0) - - :other -> - 0 - end + max(state.replicated_pg_receiver_local_request_quota - length(initial_messages), 0) messages = collect_local_request_messages(initial_messages, remaining) process_local_request_messages(state, messages) @@ -1285,8 +1445,14 @@ defmodule Group.Replica do {:cluster_connect, clusters} -> do_cluster_connect(state, clusters) + {:cluster_connect, clusters, epochs} -> + do_cluster_connect(state, clusters, epochs) + {:cluster_disconnect, clusters} -> do_cluster_disconnect(state, clusters) + + {:cluster_disconnect, clusters, epochs} -> + do_cluster_disconnect(state, clusters, epochs) end end @@ -1334,60 +1500,14 @@ defmodule Group.Replica do |> flush_pending_replicated_barrier() end - defp flush_pending_replicated_sender_barrier( - %{ - pending_replicated_pg_broadcast_len: 0, - pending_replicated_registry_broadcast_len: 0 - } = state - ), - do: state + defp flush_pending_replicated_sender_barrier(%{pending_replica_broadcast_len: 0} = state), + do: state - defp flush_pending_replicated_sender_barrier( - %{pending_replicated_pg_broadcast_len: 0, pending_replicated_registry_broadcast_len: len} = - state - ) - when len > 0, - do: flush_pending_replicated_registry_broadcast(state) + defp flush_pending_replicated_sender_barrier(state), do: flush_pending_replica_broadcast(state) - defp flush_pending_replicated_sender_barrier( - %{pending_replicated_pg_broadcast_len: len, pending_replicated_registry_broadcast_len: 0} = - state - ) - when len > 0, - do: flush_pending_replicated_pg_broadcast(state) - - defp flush_pending_replicated_sender_barrier(state) do - if state.pending_replicated_pg_broadcast_started_at <= - state.pending_replicated_registry_broadcast_started_at do - state - |> flush_pending_replicated_pg_broadcast() - |> flush_pending_replicated_registry_broadcast() - else - state - |> flush_pending_replicated_registry_broadcast() - |> flush_pending_replicated_pg_broadcast() - end - end - - defp flush_pending_replicated_pg_broadcast_barrier( - %{pending_replicated_pg_broadcast_len: 0} = state - ), - do: state - - defp flush_pending_replicated_pg_broadcast_barrier(state), - do: flush_pending_replicated_pg_broadcast(state) - - defp flush_pending_replicated_registry_broadcast_barrier( - %{pending_replicated_registry_broadcast_len: 0} = state - ), - do: state - - defp flush_pending_replicated_registry_broadcast_barrier(state), - do: flush_pending_replicated_registry_broadcast(state) - - defp process_pg_local_request_batch(state, messages) do - %{name: name, shard_index: shard} = state - local_node = node() + defp process_pg_local_request_batch(state, messages) do + %{name: name, shard_index: shard} = state + local_node = node() {entries, replies, events, broadcasts, new_monitors, maybe_demonitor_pids} = Enum.reduce( @@ -1475,11 +1595,21 @@ defmodule Group.Replica do end ) + sequenced_broadcasts = + broadcasts + |> Enum.reverse() + |> Enum.map(&append_local_replica_record(state, &1)) + {insert_entries, delete_entries} = pg_batch_diff(entries) Data.pg_delete_many(name, shard, delete_entries) Data.pg_insert_many(name, shard, insert_entries) state = finalize_local_batch_monitors(state, new_monitors, maybe_demonitor_pids) - state = send_local_batch_broadcasts(state, broadcasts) + + state = + Enum.reduce(sequenced_broadcasts, state, fn record, acc -> + finish_local_replica_record(acc, record, :pg) + end) + notify_monitors(name, events) reply_local_requests(replies) state @@ -1604,19 +1734,63 @@ defmodule Group.Replica do end defp enqueue_broadcast_op(state, {:register, _cluster, _key, _pid, _meta, _time, _node} = op), - do: enqueue_replicated_registry_broadcast(state, op) + do: sequence_and_enqueue_broadcast(state, op, :registry) defp enqueue_broadcast_op(state, {:unregister, _cluster, _key, _pid, _meta, _reason} = op), - do: enqueue_replicated_registry_broadcast(state, op) + do: sequence_and_enqueue_broadcast(state, op, :registry) defp enqueue_broadcast_op( state, {:join, _cluster, _key, _pid, _meta, _time, _reason, _node} = op ), - do: enqueue_replicated_pg_broadcast(state, op) + do: sequence_and_enqueue_broadcast(state, op, :pg) defp enqueue_broadcast_op(state, {:leave, _cluster, _key, _pid, _meta, _reason} = op), - do: enqueue_replicated_pg_broadcast(state, op) + do: sequence_and_enqueue_broadcast(state, op, :pg) + + defp sequence_and_enqueue_broadcast(state, op, domain) do + record = append_local_replica_record(state, op) + finish_local_replica_record(state, record, domain) + end + + defp append_local_replica_record(state, op) do + cluster = Protocol.op_cluster(op) + + case Data.local_stream_id(state.name, state.shard_index, cluster) do + nil -> + nil + + stream_id -> + {seq, mutations} = + Data.append_replica_record(state.name, state.shard_index, stream_id, [op]) + + {:sequenced, stream_id, seq, mutations} + end + end + + defp finish_local_replica_record(state, nil, _domain), do: state + + defp finish_local_replica_record( + state, + {:sequenced, stream_id, seq, mutations} = sequenced, + domain + ) do + apply_registry_claim_mutations(state, stream_id, seq, mutations) + + :ok = Data.mark_local_replica_applied(state.name, state.shard_index, stream_id, seq) + + :ok = + Data.prune_replica_oplog( + state.name, + state.shard_index, + state.replicated_oplog_max_entries + ) + + case domain do + :registry -> enqueue_replicated_registry_broadcast(state, sequenced) + :pg -> enqueue_replicated_pg_broadcast(state, sequenced) + end + end defp reply_local_requests(replies) do Enum.each(Enum.reverse(replies), fn {reply_to, reply} -> @@ -1634,6 +1808,8 @@ defmodule Group.Replica do case Data.registry_lookup(name, shard, cluster, key) do nil -> time = System.system_time() + op = {:register, cluster, key, pid, meta, time, node(pid)} + record = append_local_replica_record(state, op) mref = monitor_pid(state, pid) Data.registry_insert(name, shard, cluster, key, pid, meta, time, node(pid)) @@ -1641,11 +1817,7 @@ defmodule Group.Replica do "#{log_prefix_shard(state)} register key=#{inspect(key)} pid=#{inspect(pid)} cluster=#{inspect(cluster)}" end) - state = - enqueue_replicated_registry_broadcast( - state, - {:register, cluster, key, pid, meta, time, node(pid)} - ) + state = finish_local_replica_record(state, record, :registry) state = put_monitor(state, pid, mref) @@ -1660,17 +1832,15 @@ defmodule Group.Replica do {^pid, old_meta, _time, _node} -> time = System.system_time() + op = {:register, cluster, key, pid, meta, time, node(pid)} + record = append_local_replica_record(state, op) Data.registry_insert(name, shard, cluster, key, pid, meta, time, node(pid)) log_verbose(state, fn -> "#{log_prefix_shard(state)} re-register key=#{inspect(key)} pid=#{inspect(pid)} cluster=#{inspect(cluster)}" end) - state = - enqueue_replicated_registry_broadcast( - state, - {:register, cluster, key, pid, meta, time, node(pid)} - ) + state = finish_local_replica_record(state, record, :registry) event = build_event(name, :registered, key, pid, meta, %{ @@ -1691,6 +1861,8 @@ defmodule Group.Replica do case Data.registry_lookup(name, shard, cluster, key) do {pid, meta, _time, entry_node} when entry_node == node() -> + op = {:unregister, cluster, key, pid, meta, :unregister} + record = append_local_replica_record(state, op) Data.registry_delete(name, shard, cluster, key, pid) state = maybe_demonitor_pid(state, name, shard, pid) @@ -1698,11 +1870,7 @@ defmodule Group.Replica do "#{log_prefix_shard(state)} unregister key=#{inspect(key)} pid=#{inspect(pid)} cluster=#{inspect(cluster)}" end) - state = - enqueue_replicated_registry_broadcast( - state, - {:unregister, cluster, key, pid, meta, :unregister} - ) + state = finish_local_replica_record(state, record, :registry) event = build_event(name, :unregistered, key, pid, meta, %{ @@ -1727,6 +1895,8 @@ defmodule Group.Replica do case Data.pg_lookup(name, shard, cluster, key, pid) do nil -> time = System.system_time() + op = {:join, cluster, key, pid, meta, time, :join, node(pid)} + record = append_local_replica_record(state, op) mref = monitor_pid(state, pid) Data.pg_insert(name, shard, cluster, key, pid, meta, time, node(pid)) @@ -1734,11 +1904,7 @@ defmodule Group.Replica do "#{log_prefix_shard(state)} join key=#{inspect(key)} pid=#{inspect(pid)} cluster=#{inspect(cluster)}" end) - state = - enqueue_replicated_pg_broadcast( - state, - {:join, cluster, key, pid, meta, time, :join, node(pid)} - ) + state = finish_local_replica_record(state, record, :pg) state = put_monitor(state, pid, mref) @@ -1753,17 +1919,15 @@ defmodule Group.Replica do {old_meta, _time, _node} -> time = System.system_time() + op = {:join, cluster, key, pid, meta, time, :update, node(pid)} + record = append_local_replica_record(state, op) Data.pg_insert(name, shard, cluster, key, pid, meta, time, node(pid)) log_verbose(state, fn -> "#{log_prefix_shard(state)} re-join key=#{inspect(key)} pid=#{inspect(pid)} cluster=#{inspect(cluster)}" end) - state = - enqueue_replicated_pg_broadcast( - state, - {:join, cluster, key, pid, meta, time, :update, node(pid)} - ) + state = finish_local_replica_record(state, record, :pg) event = build_event(name, :joined, key, pid, meta, %{previous_meta: old_meta, cluster: cluster}) @@ -1781,6 +1945,8 @@ defmodule Group.Replica do {{:error, :not_in_group}, state} {meta, _time, _node} -> + op = {:leave, cluster, key, pid, meta, :leave} + record = append_local_replica_record(state, op) Data.pg_delete(name, shard, cluster, key, pid) state = maybe_demonitor_pid(state, name, shard, pid) @@ -1788,11 +1954,7 @@ defmodule Group.Replica do "#{log_prefix_shard(state)} leave key=#{inspect(key)} pid=#{inspect(pid)} cluster=#{inspect(cluster)}" end) - state = - enqueue_replicated_pg_broadcast( - state, - {:leave, cluster, key, pid, meta, :leave} - ) + state = finish_local_replica_record(state, record, :pg) event = build_event(name, :left, key, pid, meta, %{reason: :leave, cluster: cluster}) notify_monitors(name, [event]) @@ -1800,7 +1962,15 @@ defmodule Group.Replica do end end - defp do_cluster_connect(state, clusters) do + defp do_cluster_connect(state, clusters), + do: + do_cluster_connect( + state, + clusters, + Enum.map(clusters, &{&1, Data.local_cluster_epoch(state.name, &1)}) + ) + + defp do_cluster_connect(state, clusters, epochs) do state = flush_pending_replicated_sender_barrier(state) %{name: name} = state @@ -1811,13 +1981,33 @@ defmodule Group.Replica do peers = Data.cluster_nodes(name, nil) -- [node()] for target_node <- peers do - send_remote_shard_message(state, target_node, {:cluster_connect, clusters, self()}) + shared = + Enum.filter(clusters, fn cluster -> + not is_nil(Data.remote_cluster_epoch(name, target_node, cluster)) + end) + + Data.add_cluster_node(name, shared, target_node) + + send_remote_shard_message( + state, + target_node, + {:replica_cluster_open, self(), Data.generation(name), + Data.local_cluster_epoch_revision(name), epochs} + ) end {:ok, state} end - defp do_cluster_disconnect(state, clusters) do + defp do_cluster_disconnect(state, clusters), + do: + do_cluster_disconnect( + state, + clusters, + Enum.map(clusters, &{&1, Data.closed_local_cluster_epoch(state.name, &1)}) + ) + + defp do_cluster_disconnect(state, clusters, epochs) do state = flush_pending_replicated_sender_barrier(state) %{name: name, shard_index: shard} = state @@ -1827,7 +2017,9 @@ defmodule Group.Replica do {events, local_pids} = Enum.reduce(clusters, {[], MapSet.new()}, fn cluster, {events, local_pids} -> - {purged_reg, purged_pg} = purge_cluster_entries(name, shard, cluster, :all) + affected_keys = Data.purge_registry_claims_for_cluster(name, shard, cluster) + purged_reg = Data.delete_registry_keys(name, shard, cluster, affected_keys) + purged_pg = Data.delete_pg_cluster(name, shard, cluster) local_pids = Enum.reduce(purged_reg ++ purged_pg, local_pids, fn @@ -1847,10 +2039,28 @@ defmodule Group.Replica do maybe_demonitor_pid(acc, name, shard, pid) end) + # Disconnect purges every origin's materialized rows for these clusters. + # Forget their receive cursors as well: if this node later reconnects while + # a remote origin kept the same epoch, its advertised head must rebuild the + # rows instead of being mistaken for data we still retain. + :ok = Data.delete_replica_cursors_for_clusters(name, shard, clusters) + if shard == 0 do - broadcast_to_peers(state, {:cluster_disconnect, clusters, self()}) + broadcast_to_peers( + state, + {:replica_cluster_close, self(), Data.generation(name), + Data.local_cluster_epoch_revision(name), epochs} + ) end + Enum.each(epochs, fn + {cluster, epoch} when not is_nil(epoch) -> + Data.drop_local_stream(name, shard, cluster, epoch) + + _ -> + :ok + end) + notify_monitors(name, events) {:ok, state} end @@ -1919,66 +2129,35 @@ defmodule Group.Replica do end defp enqueue_replicated_pg_broadcast(state, op), - do: enqueue_replicated_pg_broadcasts(state, [op]) - - defp enqueue_replicated_pg_broadcasts(state, ops) do - state = flush_pending_replicated_registry_broadcast_barrier(state) - now = System.monotonic_time(:millisecond) - ops_len = length(ops) - - state = - case state.pending_replicated_pg_broadcast_len do - 0 -> - %{state | pending_replicated_pg_broadcast_started_at: now} - |> schedule_replicated_pg_broadcast_flush() - |> Map.put(:pending_replicated_pg_broadcast_ops, Enum.reverse(ops)) - |> Map.put(:pending_replicated_pg_broadcast_len, ops_len) - - len -> - %{ - state - | pending_replicated_pg_broadcast_ops: - Enum.reverse(ops, state.pending_replicated_pg_broadcast_ops), - pending_replicated_pg_broadcast_len: len + ops_len - } - end - - if state.pending_replicated_pg_broadcast_len >= state.replicated_sender_buffer_size or - pending_replicated_pg_broadcast_due?(state, now) do - flush_pending_replicated_pg_broadcast(state) - else - state - end - end + do: enqueue_replica_broadcasts(state, [op]) defp enqueue_replicated_registry_broadcast(state, op), - do: enqueue_replicated_registry_broadcasts(state, [op]) + do: enqueue_replica_broadcasts(state, [op]) - defp enqueue_replicated_registry_broadcasts(state, ops) do - state = flush_pending_replicated_pg_broadcast_barrier(state) + defp enqueue_replica_broadcasts(state, ops) do now = System.monotonic_time(:millisecond) ops_len = length(ops) state = - case state.pending_replicated_registry_broadcast_len do + case state.pending_replica_broadcast_len do 0 -> - %{state | pending_replicated_registry_broadcast_started_at: now} - |> schedule_replicated_registry_broadcast_flush() - |> Map.put(:pending_replicated_registry_broadcast_ops, Enum.reverse(ops)) - |> Map.put(:pending_replicated_registry_broadcast_len, ops_len) + %{state | pending_replica_broadcast_started_at: now} + |> schedule_replica_broadcast_flush() + |> Map.put(:pending_replica_broadcast_ops, Enum.reverse(ops)) + |> Map.put(:pending_replica_broadcast_len, ops_len) len -> %{ state - | pending_replicated_registry_broadcast_ops: - Enum.reverse(ops, state.pending_replicated_registry_broadcast_ops), - pending_replicated_registry_broadcast_len: len + ops_len + | pending_replica_broadcast_ops: + Enum.reverse(ops, state.pending_replica_broadcast_ops), + pending_replica_broadcast_len: len + ops_len } end - if state.pending_replicated_registry_broadcast_len >= state.replicated_sender_buffer_size or - pending_replicated_registry_broadcast_due?(state, now) do - flush_pending_replicated_registry_broadcast(state) + if state.pending_replica_broadcast_len >= state.replicated_sender_buffer_size or + pending_replica_broadcast_due?(state, now) do + flush_pending_replica_broadcast(state) else state end @@ -2025,6 +2204,55 @@ defmodule Group.Replica do state = process_inline_priority_message(state, msg) take_priority_control_turn(state) + {:replica_hello, _remote_pid, _version, _generation, _epoch_revision, _cluster_epochs, + _transport_id, _descriptor} = msg -> + state = process_inline_priority_message(state, msg) + take_priority_control_turn(state) + + {:replica_lane_hello, _remote_pid, _version, _generation, _epoch_revision, _transport_id, + _descriptor} = msg -> + state = process_inline_priority_message(state, msg) + take_priority_control_turn(state) + + {:replica_authority_installed_local, _remote_node, _generation, _epoch_revision, + _old_generation, _stale_epochs} = msg -> + state = process_inline_priority_message(state, msg) + take_priority_control_turn(state) + + {:replica_authority_removed_local, _remote_node} = msg -> + state = process_inline_priority_message(state, msg) + take_priority_control_turn(state) + + {:replica_authority_dirty_local, _remote_node} = msg -> + state = process_inline_priority_message(state, msg) + take_priority_control_turn(state) + + {:replica_cluster_open_control_local, _remote_node, _generation, _revision, _epochs, _stale, + _shared} = msg -> + state = process_inline_priority_message(state, msg) + take_priority_control_turn(state) + + {:replica_cluster_close_control_local, _remote_node, _generation, _revision, _closed} = + msg -> + state = process_inline_priority_message(state, msg) + take_priority_control_turn(state) + + {:replica_heartbeat, _remote_pid, _version, _generation, _epoch_revision} = msg -> + state = process_inline_priority_message(state, msg) + take_priority_control_turn(state) + + {:replica_hello_request, _remote_pid} = msg -> + state = process_inline_priority_message(state, msg) + take_priority_control_turn(state) + + {:replica_cluster_open, _remote_pid, _generation, _revision, _epochs} = msg -> + state = process_inline_priority_message(state, msg) + take_priority_control_turn(state) + + {:replica_cluster_close, _remote_pid, _generation, _revision, _epochs} = msg -> + state = process_inline_priority_message(state, msg) + take_priority_control_turn(state) + {:send_cluster_data, _clusters, _target_node} = msg -> state = process_inline_priority_message(state, msg) take_priority_control_turn(state) @@ -2098,38 +2326,20 @@ defmodule Group.Replica do %{state | pending_replicated_registry_flush_ref: flush_ref} end - defp schedule_replicated_pg_broadcast_flush(%{replicated_sender_flush_interval: 0} = state) do - %{state | pending_replicated_pg_broadcast_flush_ref: nil} - end - - defp schedule_replicated_pg_broadcast_flush(state) do - flush_ref = make_ref() - - Process.send_after( - self(), - {@replicated_pg_broadcast_flush_timer, flush_ref}, - state.replicated_sender_flush_interval - ) - - %{state | pending_replicated_pg_broadcast_flush_ref: flush_ref} - end - - defp schedule_replicated_registry_broadcast_flush( - %{replicated_sender_flush_interval: 0} = state - ) do - %{state | pending_replicated_registry_broadcast_flush_ref: nil} + defp schedule_replica_broadcast_flush(%{replicated_sender_flush_interval: 0} = state) do + %{state | pending_replica_broadcast_flush_ref: nil} end - defp schedule_replicated_registry_broadcast_flush(state) do + defp schedule_replica_broadcast_flush(state) do flush_ref = make_ref() Process.send_after( self(), - {@replicated_registry_broadcast_flush_timer, flush_ref}, + {@replica_broadcast_flush_timer, flush_ref}, state.replicated_sender_flush_interval ) - %{state | pending_replicated_registry_broadcast_flush_ref: flush_ref} + %{state | pending_replica_broadcast_flush_ref: flush_ref} end defp pending_replicated_pg_due?(%{pending_replicated_pg_len: 0}, _now), do: false @@ -2147,24 +2357,12 @@ defmodule Group.Replica do state.replicated_registry_receiver_flush_interval end - defp pending_replicated_pg_broadcast_due?(%{pending_replicated_pg_broadcast_len: 0}, _now), + defp pending_replica_broadcast_due?(%{pending_replica_broadcast_len: 0}, _now), do: false - defp pending_replicated_pg_broadcast_due?(state, now) do - state.replicated_sender_flush_interval == 0 or - now - state.pending_replicated_pg_broadcast_started_at >= - state.replicated_sender_flush_interval - end - - defp pending_replicated_registry_broadcast_due?( - %{pending_replicated_registry_broadcast_len: 0}, - _now - ), - do: false - - defp pending_replicated_registry_broadcast_due?(state, now) do + defp pending_replica_broadcast_due?(state, now) do state.replicated_sender_flush_interval == 0 or - now - state.pending_replicated_registry_broadcast_started_at >= + now - state.pending_replica_broadcast_started_at >= state.replicated_sender_flush_interval end @@ -2222,47 +2420,24 @@ defmodule Group.Replica do } end - defp flush_pending_replicated_pg_broadcast(%{pending_replicated_pg_broadcast_len: 0} = state), + defp flush_pending_replica_broadcast(%{pending_replica_broadcast_len: 0} = state), do: state - defp flush_pending_replicated_pg_broadcast(state) do - ops = Enum.reverse(state.pending_replicated_pg_broadcast_ops) - - log_verbose(state, fn -> - "#{log_prefix_shard(state)} flush_replicated_pg_broadcast_buffer ops=#{length(ops)}" - end) - - send_replicated_pg_batches(state, ops) - - %{ - state - | pending_replicated_pg_broadcast_ops: [], - pending_replicated_pg_broadcast_len: 0, - pending_replicated_pg_broadcast_started_at: nil, - pending_replicated_pg_broadcast_flush_ref: nil - } - end - - defp flush_pending_replicated_registry_broadcast( - %{pending_replicated_registry_broadcast_len: 0} = state - ), - do: state - - defp flush_pending_replicated_registry_broadcast(state) do - ops = Enum.reverse(state.pending_replicated_registry_broadcast_ops) + defp flush_pending_replica_broadcast(state) do + ops = Enum.reverse(state.pending_replica_broadcast_ops) log_verbose(state, fn -> - "#{log_prefix_shard(state)} flush_replicated_registry_broadcast_buffer ops=#{length(ops)}" + "#{log_prefix_shard(state)} flush_replica_broadcast_buffer ops=#{length(ops)}" end) - send_replicated_registry_batches(state, ops) + send_replicated_batches(state, ops) %{ state - | pending_replicated_registry_broadcast_ops: [], - pending_replicated_registry_broadcast_len: 0, - pending_replicated_registry_broadcast_started_at: nil, - pending_replicated_registry_broadcast_flush_ref: nil + | pending_replica_broadcast_ops: [], + pending_replica_broadcast_len: 0, + pending_replica_broadcast_started_at: nil, + pending_replica_broadcast_flush_ref: nil } end @@ -2381,22 +2556,28 @@ defmodule Group.Replica do member = {cluster, key, pid} {initial, current} = replicated_pg_entry(entries, name, shard, member) - previous_meta = - case {current, reason} do - {{old_meta, _old_time, _old_node}, :update} -> old_meta - _ -> nil - end + case current do + {^meta, ^time, ^entry_node} -> + {entries, events} - event = - build_event(name, :joined, key, pid, meta, %{ - previous_meta: previous_meta, - cluster: cluster - }) + _ -> + previous_meta = + case {current, reason} do + {{old_meta, _old_time, _old_node}, :update} -> old_meta + _ -> nil + end + + event = + build_event(name, :joined, key, pid, meta, %{ + previous_meta: previous_meta, + cluster: cluster + }) - updated_entries = - Map.put(entries, member, {initial, {meta, time, entry_node}}) + updated_entries = + Map.put(entries, member, {initial, {meta, time, entry_node}}) - {updated_entries, [event | events]} + {updated_entries, [event | events]} + end {:leave, cluster, key, pid, meta, reason}, {entries, events} -> member = {cluster, key, pid} @@ -2477,20 +2658,33 @@ defmodule Group.Replica do end) end - defp send_replicated_pg_batches(state, ops) do + defp send_replicated_batches(state, ops) do ops - |> group_broadcast_ops_by_target(state, &pg_op_cluster/1) + |> group_broadcast_ops_by_target(state, &sequenced_op_cluster/1) |> Enum.each(fn {target_node, target_ops} -> - send_remote_shard_message(state, target_node, {:replicate_pg_batch, target_ops}) + send_replica_delta_batch(state, target_node, target_ops) end) end - defp send_replicated_registry_batches(state, ops) do - ops - |> group_broadcast_ops_by_target(state, ®istry_op_cluster/1) - |> Enum.each(fn {target_node, target_ops} -> - send_remote_shard_message(state, target_node, {:replicate_registry_batch, target_ops}) - end) + defp send_replica_delta_batch(state, target_node, sequenced_ops) do + runs = + sequenced_ops + |> Enum.group_by(fn {:sequenced, stream_id, _seq, _mutations} -> stream_id end) + |> Enum.map(fn {stream_id, records} -> + records = + records + |> Enum.map(fn {:sequenced, ^stream_id, seq, mutations} -> {seq, mutations} end) + |> Enum.sort_by(&elem(&1, 0)) + + {first_seq, _mutations} = hd(records) + + {_floor, head, _applied} = + Data.replica_stream_head(state.name, state.shard_index, stream_id) + + {stream_id, first_seq, records, head} + end) + + try_send_replica_frame(state, target_node, {:delta_batch, Protocol.version(), runs}) end defp group_broadcast_ops_by_target(ops, state, cluster_fun) do @@ -2521,7 +2715,7 @@ defmodule Group.Replica do target_nodes = case cluster do nil -> - for {target_node, _pid} <- state.remote_shards, do: target_node + for {target_node, _last_seen} <- state.peer_last_seen, do: target_node _cluster -> for target_node <- Data.cluster_nodes(state.name, cluster), @@ -2538,11 +2732,19 @@ defmodule Group.Replica do defp pg_op_cluster({:leave, cluster, _key, _pid, _meta, _reason}), do: cluster + defp pg_op_cluster({:sequenced, _stream_id, _seq, [op | _]}), do: pg_op_cluster(op) + defp registry_op_cluster({:register, cluster, _key, _pid, _meta, _time, _entry_node}), do: cluster defp registry_op_cluster({:unregister, cluster, _key, _pid, _meta, _reason}), do: cluster + defp registry_op_cluster({:sequenced, _stream_id, _seq, [op | _]}), + do: registry_op_cluster(op) + + defp sequenced_op_cluster({:sequenced, stream_id, _seq, _mutations}), + do: Protocol.stream_cluster(stream_id) + defp replicated_op_for_active_cluster?(name, op, cluster_fun) when is_function(cluster_fun, 1) do case cluster_fun.(op) do @@ -2557,7 +2759,7 @@ defmodule Group.Replica do defp broadcast_to_peers(state, message) do for {target_node, _pid} <- state.remote_shards do - send_remote_shard_message(state, target_node, message) + send_remote_control_message(state, target_node, message) end end @@ -2569,82 +2771,1183 @@ defmodule Group.Replica do :ok false -> - Group.PeerReconnect.busy_link(state.name, target_node) - :ok + :busy end end - defp broadcast_process_down_batch(state, reason_by_pid, reg_entries, pg_entries) do - messages = - Enum.reduce(reg_entries, %{}, fn {pid, cluster, key, meta}, acc -> - accumulate_process_down_entry( - acc, - process_down_targets(state, cluster), - {:reg, pid, cluster, key, meta, Map.fetch!(reason_by_pid, pid)} - ) - end) - |> then(fn acc -> - Enum.reduce(pg_entries, acc, fn {pid, cluster, key, meta}, inner -> - accumulate_process_down_entry( - inner, - process_down_targets(state, cluster), - {:pg, pid, cluster, key, meta, Map.fetch!(reason_by_pid, pid)} - ) - end) - end) + defp send_remote_control_message(state, target_node, message) do + control_name = shard_name(state.name, 0) - Enum.each(messages, fn {target_node, {reg_entries, pg_entries}} -> - send_remote_shard_message( - state, - target_node, - {:replicate_process_down_batch, Enum.reverse(reg_entries), Enum.reverse(pg_entries)} - ) - end) + case :erlang.send_nosuspend({control_name, target_node}, message, [:noconnect]) do + true -> :ok + false -> :busy + end end - defp process_down_targets(state, nil) do - for {target_node, _pid} <- state.remote_shards, do: target_node - end + defp try_send_replica_frame(state, target_node, frame) do + case state.replica_transport.try_send( + state.name, + target_node, + state.shard_index, + frame, + state.replica_transport_opts + ) do + :ok -> :ok + :busy -> :ok + :disconnected -> :ok + end - defp process_down_targets(%{name: name}, cluster) do - for target_node <- Data.cluster_nodes(name, cluster), target_node != node(), do: target_node + state end - defp accumulate_process_down_entry(acc, target_nodes, {:reg, pid, cluster, key, meta, reason}) do - Enum.reduce(target_nodes, acc, fn target_node, inner -> - Map.update( - inner, - target_node, - {[{pid, cluster, key, meta, reason}], []}, - fn {reg_entries, pg_entries} -> - {[{pid, cluster, key, meta, reason} | reg_entries], pg_entries} - end + defp install_replica_authority( + state, + remote_pid, + generation, + epoch_revision, + cluster_epochs, + transport_id, + transport_descriptor + ) do + remote_node = node(remote_pid) + + shared = + cluster_epochs + |> Enum.map(&elem(&1, 0)) + |> compute_shared_clusters(Data.my_clusters(state.name)) + + previous_shared = Data.clusters_for_node(state.name, remote_node) -- [nil] + shared_set = MapSet.new(shared) + departed = Enum.reject(previous_shared, &MapSet.member?(shared_set, &1)) + Data.remove_cluster_node(state.name, departed, remote_node) + + Data.add_cluster_node( + state.name, + [nil | Enum.reject(shared, &is_nil/1)], + remote_node + ) + + {old_generation, stale_epochs} = + Data.put_remote_replica_info( + state.name, + 0, + remote_node, + generation, + epoch_revision, + cluster_epochs ) - end) + + state = maybe_purge_remote_generation(state, remote_node, old_generation, generation) + state = notify_replica_transport_peer_up(state, remote_node, transport_descriptor) + + state = + if old_generation == generation do + state + |> purge_closed_remote_epochs(remote_node, stale_epochs) + |> purge_remote_streams_outside_authority(remote_node) + else + state + end + + state = %{ + state + | remote_shards: Map.put(state.remote_shards, remote_node, remote_pid), + peer_last_seen: Map.put(state.peer_last_seen, remote_node, monotonic_millis()), + cluster_control_dirty: Map.delete(state.cluster_control_dirty, remote_node), + peer_transports: + Map.put(state.peer_transports, remote_node, {transport_id, transport_descriptor}) + } + + fan_out_to_siblings( + state, + {:replica_authority_installed_local, remote_node, generation, epoch_revision, + old_generation, stale_epochs} + ) + + send_replica_heads(state, remote_node) end - defp accumulate_process_down_entry(acc, target_nodes, {:pg, pid, cluster, key, meta, reason}) do - Enum.reduce(target_nodes, acc, fn target_node, inner -> - Map.update( - inner, - target_node, - {[], [{pid, cluster, key, meta, reason}]}, - fn {reg_entries, pg_entries} -> - {reg_entries, [{pid, cluster, key, meta, reason} | pg_entries]} - end - ) - end) + defp notify_replica_transport_peer_up(state, remote_node, transport_descriptor) do + if function_exported?(state.replica_transport, :peer_up, 4) do + :ok = + state.replica_transport.peer_up( + state.name, + remote_node, + transport_descriptor, + state.replica_transport_opts + ) + end + + state end - defp collect_local_process_downs(acc, monitors, 0), do: {Enum.reverse(acc), monitors} + defp send_replica_hello(state, target_node) do + descriptor = state.replica_transport.descriptor(state.name, state.replica_transport_opts) - defp collect_local_process_downs(acc, monitors, remaining) do - receive do - {:DOWN, _mref, :process, pid, reason} when is_map_key(monitors, pid) -> - collect_local_process_downs([{pid, reason} | acc], monitors, remaining - 1) - after - 0 -> - {Enum.reverse(acc), monitors} + if state.shard_index == 0 do + {generation, epoch_revision, cluster_epochs} = + Data.local_replica_authority(state.name) + + send_remote_control_message( + state, + target_node, + {:replica_hello, self(), Protocol.version(), generation, epoch_revision, cluster_epochs, + state.replica_transport.id(), descriptor} + ) + else + send_remote_shard_message( + state, + target_node, + {:replica_lane_hello, self(), Protocol.version(), Data.generation(state.name), + Data.local_cluster_epoch_revision(state.name), state.replica_transport.id(), descriptor} + ) + end + + state + end + + defp request_replica_authority(state, remote_node) do + send_remote_control_message(state, remote_node, {:replica_hello_request, self()}) + state + end + + defp replica_authority_current?(state, remote_node, generation, epoch_revision) do + Data.remote_generation(state.name, remote_node) == generation and + Data.remote_cluster_epoch_observed_revision(state.name, remote_node) == epoch_revision + end + + defp schedule_anti_entropy(state) do + ref = make_ref() + + Process.send_after( + self(), + {@anti_entropy_timer, ref}, + state.replicated_anti_entropy_interval + ) + + %{state | anti_entropy_ref: ref} + end + + defp monotonic_millis, do: System.monotonic_time(:millisecond) + + defp put_remote_shard(state, remote_node, remote_pid) do + %{state | remote_shards: Map.put(state.remote_shards, remote_node, remote_pid)} + end + + defp touch_replica_peer(state, remote_node) do + %{state | peer_last_seen: Map.put(state.peer_last_seen, remote_node, monotonic_millis())} + end + + defp collect_replica_cluster_controls(_tag, _remote_pid, _generation, acc, 0), + do: Enum.reverse(acc) + + defp collect_replica_cluster_controls(tag, remote_pid, generation, acc, remaining) do + receive do + {^tag, ^remote_pid, ^generation, revision, epochs} -> + collect_replica_cluster_controls( + tag, + remote_pid, + generation, + [{revision, epochs} | acc], + remaining - 1 + ) + after + 0 -> Enum.reverse(acc) + end + end + + defp accepted_replica_cluster_epochs(state, remote_node, generation, controls) do + case Data.remote_view_generation(state.name, state.shard_index, remote_node) do + ^generation -> + authoritative_revision = + Data.remote_view_cluster_epoch_revision( + state.name, + state.shard_index, + remote_node + ) + + accepted = + Enum.filter(controls, fn {revision, _epochs} -> + is_nil(authoritative_revision) or revision > authoritative_revision + end) + + case accepted do + [] -> + :stale + + accepted -> + accepted = Enum.sort_by(accepted, &elem(&1, 0)) + observed_revision = accepted |> List.last() |> elem(0) + + epochs = + accepted + |> Enum.flat_map(&elem(&1, 1)) + |> Map.new() + |> Map.to_list() + + {:accept, observed_revision, epochs} + end + + _other_generation -> + :refresh + end + end + + defp mark_cluster_control_dirty(state, remote_node) do + %{ + state + | cluster_control_dirty: + Map.put(state.cluster_control_dirty, remote_node, monotonic_millis()) + } + end + + defp mark_authority_dirty(%{shard_index: 0} = state, remote_node) do + mark_cluster_control_dirty(state, remote_node) + end + + defp mark_authority_dirty(state, remote_node) do + if MapSet.member?(state.authority_dirty_notified, remote_node) do + state + else + send(shard_name(state.name, 0), {:replica_authority_dirty_local, remote_node}) + + %{ + state + | authority_dirty_notified: MapSet.put(state.authority_dirty_notified, remote_node) + } + end + end + + defp request_quiet_cluster_hellos(state) do + now = monotonic_millis() + + dirty = + Enum.reduce(state.cluster_control_dirty, %{}, fn {remote_node, last_activity}, acc -> + if now - last_activity >= state.replicated_anti_entropy_interval do + request_replica_authority(state, remote_node) + Map.put(acc, remote_node, now) + else + Map.put(acc, remote_node, last_activity) + end + end) + + %{state | cluster_control_dirty: dirty} + end + + defp broadcast_replica_heartbeats(state) do + Enum.reduce(state.remote_shards, state, fn {target_node, _pid}, acc -> + send_remote_shard_message( + acc, + target_node, + {:replica_heartbeat, self(), Protocol.version(), Data.generation(acc.name), + Data.local_cluster_epoch_revision(acc.name)} + ) + + acc + end) + end + + defp probe_replica_peers(state) do + Enum.each(Node.list(), fn remote_node -> + unless Map.has_key?(state.remote_shards, remote_node) do + send_remote_shard_message( + state, + remote_node, + {:peer_connect, self(), state.shard_index, state.num_shards, + Data.my_clusters(state.name)} + ) + end + end) + + state + end + + defp broadcast_replica_heads(state) do + Enum.reduce(state.peer_last_seen, state, fn {target_node, _last_seen}, acc -> + send_replica_heads(acc, target_node) + end) + end + + defp expire_stale_replica_peers(state) do + now = monotonic_millis() + + Enum.reduce(state.peer_last_seen, state, fn {remote_node, last_seen}, acc -> + if now - last_seen > acc.replicated_peer_lease_timeout do + expire_replica_peer(acc, remote_node) + else + acc + end + end) + end + + defp expire_replica_peer(state, remote_node) do + %{name: name, shard_index: shard} = state + + if shard == 0 do + Data.purge_cluster_node(name, remote_node) + end + + {purged_reg, purged_pg} = Data.purge_node(name, shard, remote_node) + affected_claims = Data.purge_registry_claims_for_origin(name, shard, remote_node) + events = build_purged_events(name, purged_reg, purged_pg, :peer_lease_expired) + + {state, events} = + Enum.reduce(affected_claims, {state, events}, fn {cluster, key}, {acc, inner_events} -> + reconcile_registry_projection(acc, cluster, key, :peer_lease_expired, inner_events) + end) + + notify_monitors(name, events) + Data.delete_replica_cursors_for_origin(name, shard, remote_node) + Data.delete_remote_replica_info(name, shard, remote_node) + + if shard == 0 do + fan_out_to_siblings(state, {:replica_authority_removed_local, remote_node}) + end + + if function_exported?(state.replica_transport, :peer_down, 3) do + :ok = state.replica_transport.peer_down(name, remote_node, state.replica_transport_opts) + end + + %{ + state + | remote_shards: Map.delete(state.remote_shards, remote_node), + peer_last_seen: Map.delete(state.peer_last_seen, remote_node), + cluster_control_dirty: Map.delete(state.cluster_control_dirty, remote_node), + authority_dirty_notified: MapSet.delete(state.authority_dirty_notified, remote_node), + peer_transports: Map.delete(state.peer_transports, remote_node) + } + end + + defp send_replica_heads(state, target_node) do + send_replica_heads(state, target_node, :all) + end + + defp send_replica_heads(state, target_node, clusters) do + heads = replica_heads_for_clusters(state, target_node, clusters) + + if heads == [] do + state + else + try_send_replica_frame(state, target_node, {:heads, Protocol.version(), heads}) + end + end + + defp replica_heads_for_clusters(state, target_node, :all) do + state.name + |> Data.replica_stream_heads(state.shard_index) + |> Enum.filter(fn {stream_id, _floor, _head} -> + replica_stream_target?(state, stream_id, target_node) + end) + end + + defp replica_heads_for_clusters(state, target_node, clusters) when is_list(clusters) do + Enum.flat_map(clusters, fn cluster -> + case Data.local_stream_id(state.name, state.shard_index, cluster) do + nil -> + [] + + stream_id -> + {floor, head, _applied} = + Data.replica_stream_head(state.name, state.shard_index, stream_id) + + if head > 0 and replica_stream_target?(state, stream_id, target_node) do + [{stream_id, floor, head}] + else + [] + end + end + end) + end + + defp replica_stream_target?(state, stream_id, target_node) do + Protocol.stream_name(stream_id) == state.name and + Protocol.stream_origin(stream_id) == node() and + Protocol.stream_shard(stream_id) == state.shard_index and + Protocol.stream_generation(stream_id) == Data.generation(state.name) and + Protocol.stream_epoch(stream_id) == + Data.local_cluster_epoch(state.name, Protocol.stream_cluster(stream_id)) and + case Protocol.stream_cluster(stream_id) do + nil -> Map.has_key?(state.peer_last_seen, target_node) + cluster -> target_node in Data.cluster_nodes(state.name, cluster) + end + end + + defp valid_remote_stream?(state, source_node, stream_id) do + cluster = Protocol.stream_cluster(stream_id) + + Protocol.stream_name(stream_id) == state.name and + Protocol.stream_origin(stream_id) == source_node and + Protocol.stream_shard(stream_id) == state.shard_index and + Protocol.stream_generation(stream_id) == Data.remote_generation(state.name, source_node) and + Protocol.stream_epoch(stream_id) == + Data.remote_cluster_epoch(state.name, source_node, cluster) and + (is_nil(cluster) or cluster_member?(state.name, cluster)) + end + + defp handle_replica_frame(state, source_node, {:heads, version, heads}) + when version == @protocol_version do + needs = + Enum.flat_map(heads, fn {stream_id, _floor, head} -> + if valid_remote_stream?(state, source_node, stream_id) do + cursor = Data.replica_cursor(state.name, state.shard_index, stream_id) + if head > cursor, do: [{stream_id, cursor + 1}], else: [] + else + [] + end + end) + + needs + |> Enum.chunk_every(state.replicated_sender_buffer_size) + |> Enum.reduce(state, fn chunk, acc -> + try_send_replica_frame(acc, source_node, {:needs, Protocol.version(), chunk}) + end) + end + + defp handle_replica_frame(state, source_node, {:delta_batch, version, runs}) + when version == @protocol_version do + state = flush_pending_replicated_sender_barrier(state) + + Enum.reduce(runs, state, fn {stream_id, _first_seq, records, advertised_head}, acc -> + apply_replica_delta_run(acc, source_node, stream_id, records, advertised_head) + end) + end + + defp handle_replica_frame(state, source_node, {:need, version, stream_id, next_seq}) + when version == @protocol_version do + if Protocol.stream_origin(stream_id) == node() and + Protocol.stream_shard(stream_id) == state.shard_index and + replica_stream_target?(state, stream_id, source_node) do + send_replica_repair(state, source_node, stream_id, next_seq) + else + state + end + end + + defp handle_replica_frame(state, source_node, {:needs, version, needs}) + when version == @protocol_version do + send_replica_repairs(state, source_node, needs) + end + + defp handle_replica_frame( + state, + source_node, + {:snapshot, version, stream_id, snapshot_seq, reg_data, pg_data} + ) + when version == @protocol_version do + if valid_remote_stream?(state, source_node, stream_id) and + snapshot_seq >= Data.replica_cursor(state.name, state.shard_index, stream_id) do + state = flush_pending_replicated_barrier(state) + cluster = Protocol.stream_cluster(stream_id) + + reg_data = + Enum.filter(reg_data, fn {key, pid, _meta, _time} -> + node(pid) == source_node and + shard_index_for(cluster, key, state.num_shards) == state.shard_index + end) + + affected_registry_keys = + Data.replace_registry_claims_for_stream( + state.name, + state.shard_index, + stream_id, + snapshot_seq, + reg_data + ) + + {state, events} = + Enum.reduce(affected_registry_keys, {state, []}, fn key, {acc, inner_events} -> + reconcile_registry_projection(acc, cluster, key, :reconcile, inner_events) + end) + + events = replace_remote_pg_snapshot(state, source_node, cluster, pg_data, events) + :ok = Data.put_replica_cursor(state.name, state.shard_index, stream_id, snapshot_seq) + notify_monitors(state.name, events) + state + else + state + end + end + + defp handle_replica_frame(state, _source_node, _frame), do: state + + defp apply_replica_delta_run(state, source_node, stream_id, records, advertised_head) do + if valid_remote_stream?(state, source_node, stream_id) do + cursor = Data.replica_cursor(state.name, state.shard_index, stream_id) + records = Enum.drop_while(records, fn {seq, _mutations} -> seq <= cursor end) + + case records do + [] -> + state + + [{first_seq, _mutations} | _] when first_seq > cursor + 1 -> + request_replica_need(state, source_node, stream_id, cursor + 1) + + _ -> + {contiguous, _next_seq} = take_contiguous_replica_records(records, cursor + 1, []) + + {accepted, rejected} = + Enum.split_while(contiguous, fn {_seq, mutations} -> + valid_replica_mutations?(stream_id, mutations) + end) + + if rejected != [] do + Logger.error( + "#{log_prefix_shard(state)} rejected replica record with invalid origin/cluster authority from #{inspect(source_node)}" + ) + end + + state = + apply_received_replica_records(state, stream_id, accepted) + |> flush_pending_replicated_barrier() + + case List.last(accepted) do + nil -> + if rejected == [] do + state + else + request_replica_need(state, source_node, stream_id, cursor + 1) + end + + {last_seq, _mutations} -> + :ok = Data.put_replica_cursor(state.name, state.shard_index, stream_id, last_seq) + + if last_seq < advertised_head or length(accepted) < length(records) do + request_replica_need(state, source_node, stream_id, last_seq + 1) + else + state + end + end + end + else + state + end + end + + defp take_contiguous_replica_records([], next_seq, acc), do: {Enum.reverse(acc), next_seq} + + defp take_contiguous_replica_records([{seq, mutations} | rest], seq, acc) do + take_contiguous_replica_records(rest, seq + 1, [{seq, mutations} | acc]) + end + + defp take_contiguous_replica_records(_records, next_seq, acc), + do: {Enum.reverse(acc), next_seq} + + defp valid_replica_mutations?(stream_id, mutations) do + origin = Protocol.stream_origin(stream_id) + cluster = Protocol.stream_cluster(stream_id) + + mutations != [] and Enum.all?(mutations, &valid_replica_mutation?(&1, cluster, origin)) + end + + defp valid_replica_mutation?( + {:register, cluster, _key, pid, _meta, _time, entry_node}, + cluster, + origin + ), + do: node(pid) == origin and entry_node == origin + + defp valid_replica_mutation?( + {:unregister, cluster, _key, pid, _meta, _reason}, + cluster, + origin + ), + do: node(pid) == origin + + defp valid_replica_mutation?( + {:join, cluster, _key, pid, _meta, _time, _reason, entry_node}, + cluster, + origin + ), + do: node(pid) == origin and entry_node == origin + + defp valid_replica_mutation?({:leave, cluster, _key, pid, _meta, _reason}, cluster, origin), + do: node(pid) == origin + + defp valid_replica_mutation?(_mutation, _cluster, _origin), do: false + + defp apply_received_replica_records(state, stream_id, records) do + records + |> Enum.chunk_by(fn {_seq, mutations} -> replica_record_domain(mutations) end) + |> Enum.reduce(state, fn records, acc -> + case records |> hd() |> elem(1) |> replica_record_domain() do + :registry -> + acc = flush_pending_replicated_pg_barrier(acc) + + {acc, events} = + Enum.reduce(records, {acc, []}, fn {seq, mutations}, {inner, events} -> + {inner, record_events} = + apply_received_registry_claims(inner, stream_id, seq, mutations) + + {inner, record_events ++ events} + end) + + notify_monitors(acc.name, events) + acc + + _pg_or_mixed -> + Enum.reduce(records, acc, fn {seq, mutations}, inner -> + apply_received_replica_record(inner, stream_id, seq, mutations) + end) + end + end) + end + + defp replica_record_domain(mutations) do + case mutations |> Enum.map(&replica_mutation_domain/1) |> Enum.uniq() do + [domain] -> domain + [] -> :empty + _ -> :mixed + end + end + + defp enqueue_received_replica_mutation(state, {:register, _, _, _, _, _, _} = op) do + {state, _flushed?} = enqueue_replicated_registry_ops(state, [op]) + state + end + + defp enqueue_received_replica_mutation(state, {:unregister, _, _, _, _, _} = op) do + {state, _flushed?} = enqueue_replicated_registry_ops(state, [op]) + state + end + + defp enqueue_received_replica_mutation(state, {:join, _, _, _, _, _, _, _} = op) do + {state, _flushed?} = enqueue_replicated_pg_ops(state, [op]) + state + end + + defp enqueue_received_replica_mutation(state, {:leave, _, _, _, _, _} = op) do + {state, _flushed?} = enqueue_replicated_pg_ops(state, [op]) + state + end + + defp apply_received_replica_record(state, stream_id, seq, mutations) do + domains = mutations |> Enum.map(&replica_mutation_domain/1) |> Enum.uniq() + + if length(domains) > 1 do + apply_received_mixed_replica_record(state, stream_id, seq, mutations) + else + apply_received_homogeneous_replica_record(state, stream_id, seq, mutations, domains) + end + end + + defp apply_received_homogeneous_replica_record( + state, + stream_id, + seq, + mutations, + [:registry] + ) do + state = flush_pending_replicated_pg_barrier(state) + {state, _events} = apply_received_registry_claims(state, stream_id, seq, mutations) + state + end + + defp apply_received_homogeneous_replica_record(state, _stream_id, _seq, mutations, [:pg]) do + Enum.reduce(mutations, state, fn op, acc -> + enqueue_received_replica_mutation(acc, op) + end) + end + + defp apply_received_homogeneous_replica_record(state, _stream_id, _seq, [], []), do: state + + # Process death may remove registry and PG rows in one authoritative record. + # Apply its maximal same-domain segments in wire order and emit one monitor + # batch, preserving the existing process-down batching contract. + defp apply_received_mixed_replica_record(state, stream_id, seq, mutations) do + state = flush_pending_replicated_barrier(state) + + {state, events} = + mutations + |> Enum.chunk_by(&replica_mutation_domain/1) + |> Enum.reduce({state, []}, fn segment, {acc, events} -> + case replica_mutation_domain(hd(segment)) do + :registry -> + {acc, segment_events} = apply_received_registry_claims(acc, stream_id, seq, segment) + {acc, segment_events ++ events} + + :pg -> + {insert_entries, delete_entries, segment_events} = + apply_replicated_pg_ops(acc.name, acc.shard_index, segment) + + Data.pg_delete_many(acc.name, acc.shard_index, delete_entries) + Data.pg_insert_many(acc.name, acc.shard_index, insert_entries) + {acc, segment_events ++ events} + end + end) + + notify_monitors(state.name, events) + state + end + + defp replica_mutation_domain({:register, _, _, _, _, _, _}), do: :registry + defp replica_mutation_domain({:unregister, _, _, _, _, _}), do: :registry + defp replica_mutation_domain({:join, _, _, _, _, _, _, _}), do: :pg + defp replica_mutation_domain({:leave, _, _, _, _, _}), do: :pg + + defp apply_received_registry_claims(state, stream_id, seq, ops) do + keys = + Enum.map(ops, fn + {:register, _cluster, key, pid, meta, time, _entry_node} -> + Data.put_registry_claim( + state.name, + state.shard_index, + stream_id, + seq, + key, + pid, + meta, + time + ) + + key + + {:unregister, _cluster, key, pid, _meta, _reason} -> + Data.delete_registry_claim(state.name, state.shard_index, stream_id, seq, key, pid) + key + end) + |> Enum.uniq() + + cluster = Protocol.stream_cluster(stream_id) + + Enum.reduce(keys, {state, []}, fn key, {acc, events} -> + reconcile_registry_projection(acc, cluster, key, :reconcile, events) + end) + end + + defp send_replica_repair(state, target_node, stream_id, next_seq) do + send_replica_repairs(state, target_node, [{stream_id, next_seq}]) + end + + defp request_replica_need(state, target_node, stream_id, next_seq) do + try_send_replica_frame( + state, + target_node, + {:needs, Protocol.version(), [{stream_id, next_seq}]} + ) + end + + defp send_replica_repairs(state, target_node, needs) do + {state, runs} = + Enum.reduce(needs, {state, []}, fn {stream_id, next_seq}, {acc, runs} -> + if Protocol.stream_origin(stream_id) == node() and + Protocol.stream_shard(stream_id) == acc.shard_index and + replica_stream_target?(acc, stream_id, target_node) do + case replica_repair(acc, target_node, stream_id, next_seq) do + {:run, run} -> {acc, [run | runs]} + {:state, acc} -> {acc, runs} + end + else + {acc, runs} + end + end) + + case runs do + [] -> + state + + runs -> + try_send_replica_frame( + state, + target_node, + {:delta_batch, Protocol.version(), Enum.reverse(runs)} + ) + end + end + + defp replica_repair(state, target_node, stream_id, next_seq) do + {floor, head, _applied} = + Data.replica_stream_head(state.name, state.shard_index, stream_id) + + cond do + next_seq > head -> + {:state, state} + + next_seq >= floor -> + records = + Data.replica_records( + state.name, + state.shard_index, + stream_id, + next_seq, + state.replicated_sender_buffer_size + ) + + case records do + [] -> + {:state, send_replica_snapshot(state, target_node, stream_id, head)} + + [{first_seq, _} | _] -> + {:run, {stream_id, first_seq, records, head}} + end + + true -> + {:state, send_replica_snapshot(state, target_node, stream_id, head)} + end + end + + defp send_replica_snapshot(state, target_node, stream_id, head) do + cluster = Protocol.stream_cluster(stream_id) + + {_reg_by_cluster, pg_by_cluster} = + Data.local_data_by_cluster(state.name, state.shard_index, [cluster]) + + reg_data = Data.registry_claims_for_stream(state.name, state.shard_index, stream_id) + + try_send_replica_frame( + state, + target_node, + {:snapshot, Protocol.version(), stream_id, head, reg_data, + Map.get(pg_by_cluster, cluster, [])} + ) + end + + defp replace_remote_pg_snapshot(state, source_node, cluster, pg_data, events) do + current = + state.name + |> Data.pg_entries_for_origin(state.shard_index, cluster, source_node) + |> Map.new(fn {key, pid, meta, time} -> {{key, pid}, {meta, time}} end) + + desired = + pg_data + |> Enum.filter(fn {key, pid, _meta, _time} -> + node(pid) == source_node and + shard_index_for(cluster, key, state.num_shards) == state.shard_index + end) + |> Map.new(fn {key, pid, meta, time} -> {{key, pid}, {meta, time}} end) + + {inserts, deletes, events} = + current + |> Map.keys() + |> Kernel.++(Map.keys(desired)) + |> Enum.uniq() + |> Enum.reduce({[], [], events}, fn {key, pid}, {inserts, deletes, acc} -> + case {Map.get(current, {key, pid}), Map.get(desired, {key, pid})} do + {same, same} -> + {inserts, deletes, acc} + + {{old_meta, _old_time}, nil} -> + event = + build_event(state.name, :left, key, pid, old_meta, %{ + reason: :reconcile, + cluster: cluster + }) + + {inserts, [{cluster, key, pid} | deletes], [event | acc]} + + {nil, {meta, time}} -> + event = build_event(state.name, :joined, key, pid, meta, %{cluster: cluster}) + + {[{cluster, key, pid, meta, time, source_node} | inserts], deletes, [event | acc]} + + {{old_meta, _old_time}, {meta, time}} -> + event = + if old_meta == meta do + nil + else + build_event(state.name, :joined, key, pid, meta, %{ + previous_meta: old_meta, + cluster: cluster + }) + end + + acc = if event, do: [event | acc], else: acc + {[{cluster, key, pid, meta, time, source_node} | inserts], deletes, acc} + end + end) + + Data.pg_delete_many(state.name, state.shard_index, deletes) + Data.pg_insert_many(state.name, state.shard_index, inserts) + events + end + + defp maybe_purge_remote_generation(state, _remote_node, nil, _generation), do: state + + defp maybe_purge_remote_generation(state, _remote_node, generation, generation), do: state + + defp maybe_purge_remote_generation(state, remote_node, _old_generation, _generation) do + {_reg, _pg} = Data.purge_node(state.name, state.shard_index, remote_node) + + affected = + Data.purge_registry_claims_for_origin( + state.name, + state.shard_index, + remote_node + ) + + {state, events} = + Enum.reduce(affected, {state, []}, fn {cluster, key}, {acc, inner_events} -> + reconcile_registry_projection(acc, cluster, key, :nodedown, inner_events) + end) + + notify_monitors(state.name, events) + Data.delete_replica_cursors_for_origin(state.name, state.shard_index, remote_node) + state + end + + defp purge_closed_remote_epochs(state, _remote_node, []), do: state + + defp purge_closed_remote_epochs(state, remote_node, cluster_epochs) do + generation = Data.remote_generation(state.name, remote_node) + + stream_ids = + Enum.map(cluster_epochs, fn {cluster, epoch} -> + Protocol.stream_id( + state.name, + remote_node, + generation, + state.shard_index, + cluster, + epoch + ) + end) + + affected_keys = + Data.purge_registry_claims_for_streams( + state.name, + state.shard_index, + stream_ids + ) + + Enum.each(stream_ids, fn stream_id -> + :ok = Data.delete_replica_cursor(state.name, state.shard_index, stream_id) + end) + + clusters = cluster_epochs |> Enum.map(&elem(&1, 0)) |> Enum.uniq() + + purged_pg = + Data.delete_pg_for_origin_clusters( + state.name, + state.shard_index, + clusters, + remote_node + ) + + events = build_purged_events(state.name, [], purged_pg, :cluster_disconnect, []) + + {state, events} = + Enum.reduce(affected_keys, {state, events}, fn {cluster, key}, + {inner_state, inner_events} -> + reconcile_registry_projection( + inner_state, + cluster, + key, + :cluster_disconnect, + inner_events + ) + end) + + notify_monitors(state.name, events) + state + end + + defp purge_superseded_remote_streams(state, remote_node, current_epochs) do + generation = Data.remote_generation(state.name, remote_node) + + current_epochs = Map.new(current_epochs) + + superseded = + Enum.flat_map(current_epochs, fn {cluster, current_epoch} -> + current_stream = + Protocol.stream_id( + state.name, + remote_node, + generation, + state.shard_index, + cluster, + current_epoch + ) + + state.name + |> Data.replica_cursor_streams_for_origin_cluster( + state.shard_index, + remote_node, + cluster + ) + |> Enum.reject(&(&1 == current_stream)) + end) + + purge_superseded_remote_streams(state, remote_node, current_epochs, superseded) + end + + defp purge_remote_streams_outside_authority(state, remote_node) do + generation = Data.remote_generation(state.name, remote_node) + + streams = + Data.replica_cursor_streams_for_origin( + state.name, + state.shard_index, + remote_node + ) + + # A shard only needs authority for clusters for which it has retained + # receive state. This keeps local fanout proportional to actual shard data, + # rather than rebuilding the node-wide epoch map in every lane. + current_epochs = + streams + |> Enum.map(&Protocol.stream_cluster/1) + |> Enum.uniq() + |> Map.new(fn cluster -> + {cluster, Data.remote_cluster_epoch(state.name, remote_node, cluster)} + end) + + superseded = + Enum.reject(streams, fn stream_id -> + Protocol.stream_generation(stream_id) == generation and + Map.get(current_epochs, Protocol.stream_cluster(stream_id)) == + Protocol.stream_epoch(stream_id) + end) + + purge_superseded_remote_streams(state, remote_node, current_epochs, superseded) + end + + defp purge_superseded_remote_streams(state, _remote_node, _current_epochs, []), do: state + + defp purge_superseded_remote_streams( + state, + remote_node, + current_epochs, + superseded + ) do + generation = Data.remote_generation(state.name, remote_node) + + superseded + |> Enum.group_by(&Protocol.stream_cluster/1) + |> Enum.reduce(state, fn {cluster, cluster_streams}, acc -> + affected_keys = + Data.purge_registry_claims_for_streams( + state.name, + state.shard_index, + cluster_streams + ) + + Enum.each(cluster_streams, fn stream_id -> + :ok = Data.delete_replica_cursor(state.name, state.shard_index, stream_id) + end) + + # PG rows do not carry their stream epoch. Remove the origin/cluster + # slice and reset the current cursor so its exact state is rebuilt by + # the next head advertisement (delta when retained, snapshot after + # pruning). Registry claims do carry epochs and are removed narrowly. + purged_pg = + Data.delete_pg_for_origin_clusters( + state.name, + state.shard_index, + [cluster], + remote_node + ) + + case Map.get(current_epochs, cluster) do + nil -> + :ok + + current_epoch -> + current_stream = + Protocol.stream_id( + state.name, + remote_node, + generation, + state.shard_index, + cluster, + current_epoch + ) + + :ok = Data.delete_replica_cursor(state.name, state.shard_index, current_stream) + end + + events = build_purged_events(state.name, [], purged_pg, :cluster_disconnect, []) + + {acc, events} = + Enum.reduce(affected_keys, {acc, events}, fn {affected_cluster, key}, + {inner, inner_events} -> + reconcile_registry_projection( + inner, + affected_cluster, + key, + :cluster_disconnect, + inner_events + ) + end) + + notify_monitors(state.name, events) + acc + end) + end + + defp append_process_down_records(state, reason_by_pid, reg_entries, pg_entries) do + mutations_by_cluster = + Enum.reduce(reg_entries, %{}, fn {pid, cluster, key, meta}, acc -> + op = {:unregister, cluster, key, pid, meta, Map.fetch!(reason_by_pid, pid)} + Map.update(acc, cluster, [op], &[op | &1]) + end) + |> then(fn acc -> + Enum.reduce(pg_entries, acc, fn {pid, cluster, key, meta}, inner -> + op = {:leave, cluster, key, pid, meta, Map.fetch!(reason_by_pid, pid)} + Map.update(inner, cluster, [op], &[op | &1]) + end) + end) + + Enum.flat_map(mutations_by_cluster, fn {cluster, mutations} -> + case Data.local_stream_id(state.name, state.shard_index, cluster) do + nil -> + [] + + stream_id -> + {seq, mutations} = + Data.append_replica_record( + state.name, + state.shard_index, + stream_id, + Enum.reverse(mutations) + ) + + [{:sequenced, stream_id, seq, mutations}] + end + end) + end + + defp finish_process_down_records(state, records) do + Enum.each(records, fn {:sequenced, stream_id, seq, mutations} -> + apply_registry_claim_mutations(state, stream_id, seq, mutations) + :ok = Data.mark_local_replica_applied(state.name, state.shard_index, stream_id, seq) + end) + + :ok = + Data.prune_replica_oplog( + state.name, + state.shard_index, + state.replicated_oplog_max_entries + ) + + records + |> Enum.reduce(%{}, fn {:sequenced, _stream_id, _seq, [op | _]} = record, acc -> + cluster = Protocol.op_cluster(op) + + Enum.reduce(process_down_targets(state, cluster), acc, fn target_node, inner -> + Map.update(inner, target_node, [record], &[record | &1]) + end) + end) + |> Enum.reduce(state, fn {target_node, target_records}, acc -> + send_replica_delta_batch(acc, target_node, Enum.reverse(target_records)) + end) + end + + defp process_down_targets(state, nil) do + for {target_node, _last_seen} <- state.peer_last_seen, do: target_node + end + + defp process_down_targets(%{name: name}, cluster) do + for target_node <- Data.cluster_nodes(name, cluster), target_node != node(), do: target_node + end + + defp collect_local_process_downs(acc, monitors, 0), do: {Enum.reverse(acc), monitors} + + defp collect_local_process_downs(acc, monitors, remaining) do + receive do + {:DOWN, _mref, :process, pid, reason} when is_map_key(monitors, pid) -> + collect_local_process_downs([{pid, reason} | acc], monitors, remaining - 1) + after + 0 -> + {Enum.reverse(acc), monitors} end end @@ -2768,6 +4071,111 @@ defmodule Group.Replica do %{state | monitors: monitors} end + defp replay_local_journal(state) do + state.name + |> Data.local_replica_unapplied(state.shard_index) + |> Enum.each(fn {stream_id, seq, mutations} -> + if current_local_stream?(state, stream_id) do + apply_registry_claim_mutations(state, stream_id, seq, mutations) + Enum.each(mutations, &replay_local_mutation(state, &1)) + end + + :ok = Data.mark_local_replica_applied(state.name, state.shard_index, stream_id, seq) + end) + + :ok = + Data.prune_replica_oplog( + state.name, + state.shard_index, + state.replicated_oplog_max_entries + ) + + state + end + + defp current_local_stream?(state, stream_id) do + cluster = Protocol.stream_cluster(stream_id) + + Protocol.stream_name(stream_id) == state.name and + Protocol.stream_origin(stream_id) == node() and + Protocol.stream_generation(stream_id) == Data.generation(state.name) and + Protocol.stream_shard(stream_id) == state.shard_index and + Protocol.stream_epoch(stream_id) == Data.local_cluster_epoch(state.name, cluster) + end + + defp apply_registry_claim_mutations(state, stream_id, seq, mutations) do + Enum.each(mutations, fn + {:register, _cluster, key, pid, meta, time, _entry_node} -> + Data.put_registry_claim( + state.name, + state.shard_index, + stream_id, + seq, + key, + pid, + meta, + time + ) + + {:unregister, _cluster, key, pid, _meta, _reason} -> + Data.delete_registry_claim(state.name, state.shard_index, stream_id, seq, key, pid) + + _pg_mutation -> + :ok + end) + + :ok + end + + defp replay_local_mutation(state, {:register, cluster, key, pid, meta, time, entry_node}) do + Data.registry_insert( + state.name, + state.shard_index, + cluster, + key, + pid, + meta, + time, + entry_node + ) + end + + defp replay_local_mutation(state, {:unregister, cluster, key, pid, meta, reason}) do + Data.registry_delete_matching_many( + state.name, + state.shard_index, + [{pid, cluster, key, meta, reason}] + ) + + :ok + end + + defp replay_local_mutation( + state, + {:join, cluster, key, pid, meta, time, _reason, entry_node} + ) do + Data.pg_insert( + state.name, + state.shard_index, + cluster, + key, + pid, + meta, + time, + entry_node + ) + end + + defp replay_local_mutation(state, {:leave, cluster, key, pid, meta, reason}) do + Data.pg_delete_matching_many( + state.name, + state.shard_index, + [{pid, cluster, key, meta, reason}] + ) + + :ok + end + defp cluster_member?(name, cluster) do node() in Data.cluster_nodes(name, cluster) end @@ -2882,6 +4290,7 @@ defmodule Group.Replica do cond do winner_pid == remote_pid -> + exit_local_conflict_loser(local_pid, key, remote_meta) time = System.system_time() event = @@ -2893,7 +4302,10 @@ defmodule Group.Replica do { Map.put(entries, entry, {initial, {remote_pid, remote_meta, time, node(remote_pid)}}), [event | events], - broadcasts, + [ + {:unregister, cluster, key, local_pid, local_meta, :resolve_conflict} + | broadcasts + ], MapSet.put(maybe_demonitor_pids, local_pid) } @@ -2908,6 +4320,8 @@ defmodule Group.Replica do } true -> + exit_local_conflict_loser(local_pid, key, nil) + event = build_event(state.name, :unregistered, key, local_pid, local_meta, %{ reason: :resolve_conflict, @@ -2923,6 +4337,214 @@ defmodule Group.Replica do end end + defp reconcile_registry_projection(state, cluster, key, reason, events) do + claims = Data.registry_claims(state.name, state.shard_index, cluster, key) + winner = select_registry_claim_winner(state, cluster, key, claims) + + {state, retired?} = retire_local_registry_losers(state, cluster, key, claims, winner) + + winner = + if retired? do + state.name + |> Data.registry_claims(state.shard_index, cluster, key) + |> then(&select_registry_claim_winner(state, cluster, key, &1)) + else + winner + end + + current = Data.registry_lookup(state.name, state.shard_index, cluster, key) + + projection_reason = if retired?, do: :resolve_conflict, else: reason + + project_registry_winner( + state, + cluster, + key, + current, + winner, + projection_reason, + events + ) + end + + defp select_registry_claim_winner(_state, _cluster, _key, []), do: nil + defp select_registry_claim_winner(_state, _cluster, _key, [claim]), do: claim + + defp select_registry_claim_winner(state, cluster, key, claims) do + claims = + Enum.sort_by(claims, fn {pid, _meta, time, origin_node, generation, epoch, _seq} -> + {time, pid, origin_node, generation, epoch} + end) + + Enum.reduce_while(tl(claims), hd(claims), fn claim, winner -> + {winner_pid, winner_meta, winner_time, _origin, _generation, _epoch, _seq} = winner + {pid, meta, time, _origin, _generation, _epoch, _seq} = claim + + selected = + resolve_conflict_winner( + state, + cluster, + key, + {winner_pid, winner_meta, winner_time}, + {pid, meta, time} + ) + + cond do + selected == winner_pid -> {:cont, winner} + selected == pid -> {:cont, claim} + true -> {:halt, nil} + end + end) + end + + defp retire_local_registry_losers(state, cluster, key, claims, winner) do + winner_pid = if winner, do: elem(winner, 0), else: nil + + local_losers = + Enum.filter(claims, fn {pid, _meta, _time, origin_node, _generation, _epoch, _seq} -> + origin_node == node() and pid != winner_pid + end) + + state = + Enum.reduce(local_losers, state, fn + {pid, meta, _time, _origin_node, _generation, _epoch, _seq}, acc -> + op = {:unregister, cluster, key, pid, meta, :resolve_conflict} + record = append_local_replica_record(acc, op) + acc = finish_local_replica_record(acc, record, :registry) + winner_meta = if winner, do: elem(winner, 1), else: nil + exit_local_conflict_loser(pid, key, winner_meta) + acc + end) + + {state, local_losers != []} + end + + defp project_registry_winner(state, _cluster, _key, nil, nil, _reason, events), + do: {state, events} + + defp project_registry_winner( + state, + cluster, + key, + nil, + {pid, meta, time, origin_node, _generation, _epoch, _seq}, + _reason, + events + ) do + Data.registry_insert( + state.name, + state.shard_index, + cluster, + key, + pid, + meta, + time, + origin_node + ) + + event = build_event(state.name, :registered, key, pid, meta, %{cluster: cluster}) + {state, [event | events]} + end + + defp project_registry_winner( + state, + cluster, + key, + {pid, old_meta, old_time, old_node}, + {pid, meta, time, origin_node, _generation, _epoch, _seq}, + _reason, + events + ) do + if old_meta == meta and old_time == time and old_node == origin_node do + {state, events} + else + Data.registry_insert( + state.name, + state.shard_index, + cluster, + key, + pid, + meta, + time, + origin_node + ) + + event = + build_event(state.name, :registered, key, pid, meta, %{ + previous_meta: old_meta, + cluster: cluster + }) + + {state, [event | events]} + end + end + + defp project_registry_winner( + state, + cluster, + key, + {old_pid, old_meta, _old_time, old_node}, + nil, + reason, + events + ) do + Data.registry_delete(state.name, state.shard_index, cluster, key, old_pid) + + state = + if old_node == node() do + maybe_demonitor_pid(state, state.name, state.shard_index, old_pid) + else + state + end + + event = + build_event(state.name, :unregistered, key, old_pid, old_meta, %{ + reason: reason, + cluster: cluster + }) + + {state, [event | events]} + end + + defp project_registry_winner( + state, + cluster, + key, + {old_pid, old_meta, _old_time, old_node}, + {pid, meta, time, origin_node, _generation, _epoch, _seq}, + reason, + events + ) do + Data.registry_delete(state.name, state.shard_index, cluster, key, old_pid) + + state = + if old_node == node() do + maybe_demonitor_pid(state, state.name, state.shard_index, old_pid) + else + state + end + + Data.registry_insert( + state.name, + state.shard_index, + cluster, + key, + pid, + meta, + time, + origin_node + ) + + unregistered = + build_event(state.name, :unregistered, key, old_pid, old_meta, %{ + reason: reason, + cluster: cluster + }) + + registered = build_event(state.name, :registered, key, pid, meta, %{cluster: cluster}) + {state, [registered, unregistered | events]} + end + defp resolve_conflict( state, cluster, @@ -2943,6 +4565,7 @@ defmodule Group.Replica do cond do winner_pid == remote_pid -> + exit_local_conflict_loser(local_pid, key, remote_meta) # Remote wins — replace local entry Data.registry_delete(name, shard, cluster, key, local_pid) state = maybe_demonitor_pid(state, name, shard, local_pid) @@ -2968,6 +4591,12 @@ defmodule Group.Replica do cluster: cluster }) + state = + enqueue_broadcast_op( + state, + {:unregister, cluster, key, local_pid, local_meta, :resolve_conflict} + ) + {state, event} winner_pid == local_pid -> @@ -2986,7 +4615,7 @@ defmodule Group.Replica do ) state = - enqueue_replicated_registry_broadcast( + enqueue_broadcast_op( state, {:register, cluster, key, local_pid, local_meta, time, node(local_pid)} ) @@ -2994,12 +4623,13 @@ defmodule Group.Replica do {state, nil} true -> + exit_local_conflict_loser(local_pid, key, nil) # Neither wins — remove both Data.registry_delete(name, shard, cluster, key, local_pid) state = maybe_demonitor_pid(state, name, shard, local_pid) state = - enqueue_replicated_registry_broadcast( + enqueue_broadcast_op( state, {:unregister, cluster, key, local_pid, local_meta, :resolve_conflict} ) @@ -3055,7 +4685,7 @@ defmodule Group.Replica do # causing mutual kill (both processes die, key becomes unregistered). # Erlang pids have a total order (by node name then id), so pid comparison # gives a consistent tiebreaker across all nodes. - {winner_pid, winner_meta, loser_pid} = + {winner_pid, _winner_meta, _loser_pid} = if time2 > time1 or (time2 == time1 and pid2 > pid1) do {pid2, meta2, pid1} else @@ -3067,12 +4697,18 @@ defmodule Group.Replica do "pid1=#{inspect(pid1)}, pid2=#{inspect(pid2)}, picking #{inspect(winner_pid)} as winner" end) - Process.exit(loser_pid, {:group_registry_conflict, key, winner_meta}) winner_pid end - # Gather local data for all shared clusters in ONE table scan (instead of C scans) - # and send per-cluster cluster_state messages. One O(N) scan vs C × O(N) scans. + defp exit_local_conflict_loser(pid, key, winner_meta) when node(pid) == node() do + Process.exit(pid, {:group_registry_conflict, key, winner_meta}) + :ok + end + + defp exit_local_conflict_loser(_pid, _key, _winner_meta), do: :ok + + # Legacy receive-only compatibility: gather local data for all requested + # clusters in one scan before emitting the old cluster_state messages. defp send_cluster_states(state, clusters, target_node) do %{name: name, shard_index: shard} = state {reg_by_cluster, pg_by_cluster} = Data.local_data_by_cluster(name, shard, clusters) diff --git a/lib/group/replica/data.ex b/lib/group/replica/data.ex index edfd9a0..e6e4c81 100644 --- a/lib/group/replica/data.ex +++ b/lib/group/replica/data.ex @@ -10,8 +10,9 @@ defmodule Group.Replica.Data do ## ETS Table Layout - Each shard owns 4 tables. There are also 3 shared tables per Group instance: - 2 for cluster membership and 1 for local named-cluster TTL leases. + Each shard has materialized registry/PG indexes, authoritative registry-claim + indexes, and replica stream/oplog/cursor tables. Shared tables hold cluster + membership, local named-cluster TTL leases, generations, and cluster epochs. ### reg_by_key — `:set`, keyed by `{cluster, key}` @@ -139,8 +140,8 @@ defmodule Group.Replica.Data do All tables are `:public` with `read_concurrency: true`. Reads happen directly from any process (the Replica GenServer, Group API callers, etc.). Writes are serialized through the Replica GenServer for each shard, ensuring consistent paired updates to both the - by_key and by_pid tables. The Data GenServer itself only owns the tables (for crash - survival via rest_for_one) — it handles no messages after init. + by_key and by_pid tables. The Data GenServer owns the tables (for shard-crash survival + via rest_for_one) and serializes cross-shard generation, epoch, and cluster-node changes. """ def start_link(opts) do @@ -151,6 +152,371 @@ defmodule Group.Replica.Data do def data_name(name), do: :"#{name}_data" + # ===================================================================== + # Replica generations, epochs, journal, and cursors + # ===================================================================== + + def generation(name) do + :ets.lookup_element(replication_meta_table(name), :generation, 2) + end + + def local_cluster_epoch_revision(name) do + :ets.lookup_element(replication_meta_table(name), :cluster_epoch_revision, 2) + end + + def local_cluster_epoch(name, nil), do: generation(name) + + def local_cluster_epoch(name, cluster) do + case :ets.lookup(local_cluster_epochs_table(name), cluster) do + [{^cluster, epoch}] -> epoch + [] -> nil + end + end + + def local_cluster_epochs(name) do + [{nil, generation(name)} | :ets.tab2list(local_cluster_epochs_table(name))] + end + + def local_replica_authority(name) do + GenServer.call(data_name(name), :local_replica_authority, :infinity) + end + + def closed_local_cluster_epoch(name, cluster) do + case :ets.lookup(closed_local_cluster_epochs_table(name), cluster) do + [{^cluster, epoch}] -> epoch + [] -> nil + end + end + + def remote_generation(name, remote_node) do + case :ets.lookup(replication_meta_table(name), {:remote_generation, remote_node}) do + [{{:remote_generation, ^remote_node}, generation}] -> generation + [] -> nil + end + end + + def remote_cluster_epoch(name, remote_node, nil), do: remote_generation(name, remote_node) + + def remote_cluster_epoch(name, remote_node, cluster) do + case :ets.lookup(remote_cluster_epochs_table(name), {remote_node, cluster}) do + [{{^remote_node, ^cluster}, epoch}] -> epoch + [] -> nil + end + end + + def remote_cluster_epoch_revision(name, remote_node) do + case :ets.lookup(replication_meta_table(name), {:remote_epoch_revision, remote_node}) do + [{{:remote_epoch_revision, ^remote_node}, revision}] -> revision + [] -> nil + end + end + + def remote_cluster_epoch_exact_revision(name, remote_node) do + case :ets.lookup(replication_meta_table(name), {:remote_epoch_exact, remote_node}) do + [{{:remote_epoch_exact, ^remote_node}, revision}] -> revision + [] -> nil + end + end + + def remote_cluster_epoch_observed_revision(name, remote_node) do + case :ets.lookup(replication_meta_table(name), {:remote_epoch_observed, remote_node}) do + [{{:remote_epoch_observed, ^remote_node}, revision}] -> revision + [] -> nil + end + end + + @doc false + def remote_authority_install_count(name, remote_node) do + case :ets.lookup(replication_meta_table(name), {:remote_authority_installs, remote_node}) do + [{{:remote_authority_installs, ^remote_node}, count}] -> count + [] -> 0 + end + end + + def remote_view_generation(name, shard, remote_node) do + case :ets.lookup(replication_meta_table(name), {:remote_view_info, shard, remote_node}) do + [{{:remote_view_info, ^shard, ^remote_node}, generation, _revision, _observed}] -> + generation + + [] -> + nil + end + end + + def remote_view_cluster_epoch_revision(name, shard, remote_node) do + case :ets.lookup(replication_meta_table(name), {:remote_view_info, shard, remote_node}) do + [{{:remote_view_info, ^shard, ^remote_node}, _generation, revision, _observed}] -> + revision + + [] -> + nil + end + end + + def remote_view_observed_revision(name, shard, remote_node) do + case :ets.lookup(replication_meta_table(name), {:remote_view_info, shard, remote_node}) do + [{{:remote_view_info, ^shard, ^remote_node}, _generation, _revision, observed}] -> + observed + + [] -> + nil + end + end + + def put_remote_replica_info(name, shard, remote_node, generation, epoch_revision, epochs) do + GenServer.call( + data_name(name), + {:put_remote_replica_info, shard, remote_node, generation, epoch_revision, epochs}, + :infinity + ) + end + + def put_remote_view_info(name, shard, remote_node, generation, authoritative, observed) do + GenServer.call( + data_name(name), + {:put_remote_view_info, shard, remote_node, generation, authoritative, observed}, + :infinity + ) + end + + def put_remote_cluster_epochs(name, shard, remote_node, revision, epochs) do + GenServer.call( + data_name(name), + {:put_remote_cluster_epochs, shard, remote_node, revision, epochs}, + :infinity + ) + end + + def close_remote_cluster_epochs(name, shard, remote_node, revision, epochs) do + GenServer.call( + data_name(name), + {:close_remote_cluster_epochs, shard, remote_node, revision, epochs}, + :infinity + ) + end + + def forget_remote_cluster_epochs(name, shard, remote_node, epochs) do + GenServer.call( + data_name(name), + {:forget_remote_cluster_epochs, shard, remote_node, epochs}, + :infinity + ) + end + + def delete_remote_replica_info(name, shard, remote_node) do + GenServer.call( + data_name(name), + {:delete_remote_replica_info, shard, remote_node}, + :infinity + ) + end + + def activate_local_clusters(name, clusters) do + GenServer.call(data_name(name), {:activate_local_clusters, clusters}, :infinity) + end + + def deactivate_local_clusters(name, clusters) do + GenServer.call(data_name(name), {:deactivate_local_clusters, clusters}, :infinity) + end + + def local_stream_id(name, shard, cluster) do + case local_cluster_epoch(name, cluster) do + nil -> + nil + + epoch -> + Group.Replica.Protocol.stream_id( + name, + node(), + generation(name), + shard, + cluster, + epoch + ) + end + end + + def append_replica_record(name, shard, stream_id, mutations) when is_list(mutations) do + stream_table = replica_stream_meta_table(name, shard) + + head = + :ets.update_counter( + stream_table, + stream_id, + {2, 1}, + {stream_id, 0, 1, 0} + ) + + append_id = + :ets.update_counter( + replication_meta_table(name), + {:append_counter, shard}, + {2, 1}, + {{:append_counter, shard}, 0} + ) + + :ets.insert(replica_oplog_table(name, shard), {{stream_id, head}, append_id, mutations}) + :ets.insert(replica_oplog_order_table(name, shard), {append_id, stream_id, head}) + {head, mutations} + end + + def mark_local_replica_applied(name, shard, stream_id, seq) do + :ets.update_element(replica_stream_meta_table(name, shard), stream_id, {4, seq}) + :ok + end + + def local_replica_unapplied(name, shard) do + replica_stream_meta_table(name, shard) + |> :ets.tab2list() + |> Enum.flat_map(fn {stream_id, head, _floor, applied} -> + if applied < head do + replica_records(name, shard, stream_id, applied + 1, head - applied) + |> Enum.map(fn {seq, mutations} -> {stream_id, seq, mutations} end) + else + [] + end + end) + end + + def replica_stream_heads(name, shard) do + :ets.tab2list(replica_stream_meta_table(name, shard)) + |> Enum.map(fn {stream_id, head, floor, _applied} -> {stream_id, floor, head} end) + end + + def replica_stream_head(name, shard, stream_id) do + case :ets.lookup(replica_stream_meta_table(name, shard), stream_id) do + [{^stream_id, head, floor, applied}] -> {floor, head, applied} + [] -> {1, 0, 0} + end + end + + def replica_records(_name, _shard, _stream_id, _from_seq, limit) when limit <= 0, do: [] + + def replica_records(name, shard, stream_id, from_seq, limit) do + table = replica_oplog_table(name, shard) + + table + |> :ets.select( + [ + {{{stream_id, :"$1"}, :_, :"$2"}, [{:>=, :"$1", from_seq}], [{{:"$1", :"$2"}}]} + ], + limit + ) + |> case do + :"$end_of_table" -> [] + {records, _continuation} -> records + end + end + + def prune_replica_oplog(name, shard, max_entries) do + order_table = replica_oplog_order_table(name, shard) + do_prune_replica_oplog(name, shard, order_table, :ets.info(order_table, :size) - max_entries) + end + + defp do_prune_replica_oplog(_name, _shard, _order_table, remaining) when remaining <= 0, + do: :ok + + defp do_prune_replica_oplog(name, shard, order_table, remaining) do + case :ets.first(order_table) do + :"$end_of_table" -> + :ok + + append_id -> + [{^append_id, stream_id, seq}] = :ets.lookup(order_table, append_id) + {_floor, _head, applied} = replica_stream_head(name, shard, stream_id) + + if seq <= applied do + :ets.delete(order_table, append_id) + :ets.delete(replica_oplog_table(name, shard), {stream_id, seq}) + + case :ets.lookup(replica_stream_meta_table(name, shard), stream_id) do + [{^stream_id, head, floor, local_applied}] -> + :ets.insert( + replica_stream_meta_table(name, shard), + {stream_id, head, max(floor, seq + 1), local_applied} + ) + + [] -> + :ok + end + + do_prune_replica_oplog(name, shard, order_table, remaining - 1) + else + :ok + end + end + end + + def replica_cursor(name, shard, stream_id) do + case :ets.lookup(replica_cursor_table(name, shard), stream_id) do + [{^stream_id, seq}] -> seq + [] -> 0 + end + end + + def replica_cursor_streams_for_origin_cluster(name, shard, origin_node, cluster) do + :ets.select(replica_cursor_table(name, shard), [ + {{{name, origin_node, :"$1", shard, cluster, :"$2"}, :_}, [], + [{{name, origin_node, :"$1", shard, cluster, :"$2"}}]} + ]) + end + + def replica_cursor_streams_for_origin(name, shard, origin_node) do + :ets.select(replica_cursor_table(name, shard), [ + {{{name, origin_node, :"$1", shard, :"$2", :"$3"}, :_}, [], + [{{name, origin_node, :"$1", shard, :"$2", :"$3"}}]} + ]) + end + + def put_replica_cursor(name, shard, stream_id, seq) do + :ets.insert(replica_cursor_table(name, shard), {stream_id, seq}) + :ok + end + + def delete_replica_cursors_for_origin(name, shard, origin_node) do + :ets.select_delete(replica_cursor_table(name, shard), [ + {{{name, origin_node, :_, shard, :_, :_}, :_}, [], [true]} + ]) + + :ok + end + + def delete_replica_cursors_for_clusters(_name, _shard, []), do: :ok + + def delete_replica_cursors_for_clusters(name, shard, clusters) do + match_specs = + Enum.map(clusters, fn cluster -> + {{{name, :_, :_, shard, cluster, :_}, :_}, [], [true]} + end) + + :ets.select_delete(replica_cursor_table(name, shard), match_specs) + :ok + end + + def delete_replica_cursor(name, shard, stream_id) do + :ets.delete(replica_cursor_table(name, shard), stream_id) + :ok + end + + def drop_local_stream(name, shard, cluster, epoch) do + stream_id = + Group.Replica.Protocol.stream_id(name, node(), generation(name), shard, cluster, epoch) + + append_rows = + :ets.select(replica_oplog_table(name, shard), [ + {{{stream_id, :"$1"}, :"$2", :_}, [], [{{:"$1", :"$2"}}]} + ]) + + Enum.each(append_rows, fn {seq, append_id} -> + :ets.delete(replica_oplog_table(name, shard), {stream_id, seq}) + :ets.delete(replica_oplog_order_table(name, shard), append_id) + end) + + :ets.delete(replica_stream_meta_table(name, shard), stream_id) + + :ok + end + # ===================================================================== # Registry operations # ===================================================================== @@ -248,6 +614,224 @@ defmodule Group.Replica.Data do ]) end + # ===================================================================== + # Authoritative registry claims + # ===================================================================== + + def put_registry_claim(name, shard, stream_id, seq, key, pid, meta, time) do + cluster = Group.Replica.Protocol.stream_cluster(stream_id) + origin_node = Group.Replica.Protocol.stream_origin(stream_id) + generation = Group.Replica.Protocol.stream_generation(stream_id) + epoch = Group.Replica.Protocol.stream_epoch(stream_id) + claim_key = {cluster, key, origin_node, generation, epoch} + by_key = reg_claim_by_key_table(name, shard) + + case :ets.lookup(by_key, claim_key) do + [{^claim_key, _old_pid, _old_meta, _old_time, old_seq}] when old_seq >= seq -> + :ok + + [{^claim_key, old_pid, _old_meta, _old_time, _old_seq}] -> + :ets.delete( + reg_claim_by_pid_table(name, shard), + {old_pid, cluster, key, origin_node, generation, epoch} + ) + + insert_registry_claim(name, shard, claim_key, seq, pid, meta, time) + + [] -> + insert_registry_claim(name, shard, claim_key, seq, pid, meta, time) + end + end + + defp insert_registry_claim(name, shard, claim_key, seq, pid, meta, time) do + {cluster, key, origin_node, generation, epoch} = claim_key + :ets.insert(reg_claim_by_key_table(name, shard), {claim_key, pid, meta, time, seq}) + + :ets.insert( + reg_claim_by_pid_table(name, shard), + {{pid, cluster, key, origin_node, generation, epoch}, meta, time, seq} + ) + + :ok + end + + def delete_registry_claim(name, shard, stream_id, seq, key, pid) do + cluster = Group.Replica.Protocol.stream_cluster(stream_id) + origin_node = Group.Replica.Protocol.stream_origin(stream_id) + generation = Group.Replica.Protocol.stream_generation(stream_id) + epoch = Group.Replica.Protocol.stream_epoch(stream_id) + claim_key = {cluster, key, origin_node, generation, epoch} + + case :ets.lookup(reg_claim_by_key_table(name, shard), claim_key) do + [{^claim_key, ^pid, _meta, _time, old_seq}] when old_seq <= seq -> + :ets.delete(reg_claim_by_key_table(name, shard), claim_key) + + :ets.delete( + reg_claim_by_pid_table(name, shard), + {pid, cluster, key, origin_node, generation, epoch} + ) + + :ok + + _ -> + :ok + end + end + + def registry_claims(name, shard, cluster, key) do + :ets.select(reg_claim_by_key_table(name, shard), [ + {{{cluster, key, :"$1", :"$2", :"$3"}, :"$4", :"$5", :"$6", :"$7"}, [], + [{{:"$4", :"$5", :"$6", :"$1", :"$2", :"$3", :"$7"}}]} + ]) + end + + def registry_claims_for_stream(name, shard, stream_id) do + cluster = Group.Replica.Protocol.stream_cluster(stream_id) + origin_node = Group.Replica.Protocol.stream_origin(stream_id) + generation = Group.Replica.Protocol.stream_generation(stream_id) + epoch = Group.Replica.Protocol.stream_epoch(stream_id) + + :ets.select(reg_claim_by_key_table(name, shard), [ + {{{cluster, :"$1", origin_node, generation, epoch}, :"$2", :"$3", :"$4", :_}, [], + [{{:"$1", :"$2", :"$3", :"$4"}}]} + ]) + end + + def replace_registry_claims_for_stream(name, shard, stream_id, snapshot_seq, claims) do + cluster = Group.Replica.Protocol.stream_cluster(stream_id) + origin_node = Group.Replica.Protocol.stream_origin(stream_id) + generation = Group.Replica.Protocol.stream_generation(stream_id) + epoch = Group.Replica.Protocol.stream_epoch(stream_id) + existing = registry_claims_for_stream(name, shard, stream_id) + + Enum.each(existing, fn {key, pid, _meta, _time} -> + :ets.delete( + reg_claim_by_key_table(name, shard), + {cluster, key, origin_node, generation, epoch} + ) + + :ets.delete( + reg_claim_by_pid_table(name, shard), + {pid, cluster, key, origin_node, generation, epoch} + ) + end) + + Enum.each(claims, fn {key, pid, meta, time} -> + put_registry_claim(name, shard, stream_id, snapshot_seq, key, pid, meta, time) + end) + + Enum.uniq(Enum.map(existing, &elem(&1, 0)) ++ Enum.map(claims, &elem(&1, 0))) + end + + def purge_registry_claims_for_origin(name, shard, origin_node) do + claims = + :ets.select(reg_claim_by_key_table(name, shard), [ + {{{:"$1", :"$2", origin_node, :"$3", :"$4"}, :"$5", :"$6", :"$7", :_}, [], + [{{:"$1", :"$2", :"$5", :"$6", :"$7", :"$3", :"$4"}}]} + ]) + + Enum.each(claims, fn {cluster, key, pid, _meta, _time, generation, epoch} -> + :ets.delete( + reg_claim_by_key_table(name, shard), + {cluster, key, origin_node, generation, epoch} + ) + + :ets.delete( + reg_claim_by_pid_table(name, shard), + {pid, cluster, key, origin_node, generation, epoch} + ) + end) + + Enum.uniq(Enum.map(claims, fn {cluster, key, _, _, _, _, _} -> {cluster, key} end)) + end + + def purge_registry_claims_for_streams(_name, _shard, []), do: [] + + def purge_registry_claims_for_streams(name, shard, stream_ids) do + streams = + MapSet.new(stream_ids, fn stream_id -> + { + Group.Replica.Protocol.stream_cluster(stream_id), + Group.Replica.Protocol.stream_origin(stream_id), + Group.Replica.Protocol.stream_generation(stream_id), + Group.Replica.Protocol.stream_epoch(stream_id) + } + end) + + claims = + :ets.tab2list(reg_claim_by_key_table(name, shard)) + |> Enum.filter(fn {{cluster, _key, origin, generation, epoch}, _pid, _meta, _time, _seq} -> + MapSet.member?(streams, {cluster, origin, generation, epoch}) + end) + + Enum.each(claims, fn {{cluster, key, origin, generation, epoch}, pid, _meta, _time, _seq} -> + :ets.delete( + reg_claim_by_key_table(name, shard), + {cluster, key, origin, generation, epoch} + ) + + :ets.delete( + reg_claim_by_pid_table(name, shard), + {pid, cluster, key, origin, generation, epoch} + ) + end) + + claims + |> Enum.map(fn {{cluster, key, _origin, _generation, _epoch}, _pid, _meta, _time, _seq} -> + {cluster, key} + end) + |> Enum.uniq() + end + + def purge_registry_claims_for_cluster(name, shard, cluster, origin_node \\ :all) + + def purge_registry_claims_for_cluster(name, shard, cluster, :all) do + claims = + :ets.select(reg_claim_by_key_table(name, shard), [ + {{{cluster, :"$1", :"$2", :"$3", :"$4"}, :"$5", :"$6", :"$7", :_}, [], + [{{:"$1", :"$5", :"$6", :"$7", :"$2", :"$3", :"$4"}}]} + ]) + + delete_registry_claim_rows(name, shard, cluster, claims) + end + + def purge_registry_claims_for_cluster(name, shard, cluster, origin_node) do + claims = + :ets.select(reg_claim_by_key_table(name, shard), [ + {{{cluster, :"$1", origin_node, :"$2", :"$3"}, :"$4", :"$5", :"$6", :_}, [], + [{{:"$1", :"$4", :"$5", :"$6", origin_node, :"$2", :"$3"}}]} + ]) + + delete_registry_claim_rows(name, shard, cluster, claims) + end + + defp delete_registry_claim_rows(name, shard, cluster, claims) do + Enum.each(claims, fn {key, pid, _meta, _time, claim_origin, generation, epoch} -> + :ets.delete( + reg_claim_by_key_table(name, shard), + {cluster, key, claim_origin, generation, epoch} + ) + + :ets.delete( + reg_claim_by_pid_table(name, shard), + {pid, cluster, key, claim_origin, generation, epoch} + ) + end) + + Enum.uniq(Enum.map(claims, &elem(&1, 0))) + end + + def local_registry_claims_by_pids(name, shard, pids) do + local_node = node() + + Enum.flat_map(Enum.uniq(pids), fn pid -> + :ets.select(reg_claim_by_pid_table(name, shard), [ + {{{pid, :"$1", :"$2", local_node, :"$3", :"$4"}, :"$5", :"$6", :_}, [], + [{{pid, :"$1", :"$2", :"$5", :"$3", :"$4"}}]} + ]) + end) + end + # ===================================================================== # Process group operations # ===================================================================== @@ -423,6 +1007,17 @@ defmodule Group.Replica.Data do process DOWN cleanup. Returns lean `{pid, cluster, key, meta}` tuples for dispatch/event building. """ + def entries_for_pids(_name, _shard, []), do: {[], []} + + def entries_for_pids(name, shard, pids) do + pids = Enum.uniq(pids) + + { + select_entries_for_pids(reg_by_pid_table(name, shard), pids), + select_entries_for_pids(pg_by_pid_table(name, shard), pids) + } + end + def delete_all_for_pids(_name, _shard, []), do: {[], []} def delete_all_for_pids(name, shard, pids) do @@ -621,6 +1216,72 @@ defmodule Group.Replica.Data do {reg_by_cluster, pg_by_cluster} end + def pg_entries_for_origin(name, shard, cluster, origin_node) do + :ets.select(pg_by_key_table(name, shard), [ + {{{cluster, :"$1", :"$2"}, :"$3", :"$4", origin_node}, [], [{{:"$1", :"$2", :"$3", :"$4"}}]} + ]) + end + + def delete_pg_for_origin_cluster(name, shard, cluster, origin_node) do + entries = pg_entries_for_origin(name, shard, cluster, origin_node) + + Enum.each(entries, fn {key, pid, _meta, _time} -> + :ets.delete(pg_by_key_table(name, shard), {cluster, key, pid}) + :ets.delete(pg_by_pid_table(name, shard), {pid, cluster, key}) + end) + + Enum.map(entries, fn {key, pid, meta, time} -> {cluster, key, pid, meta, time} end) + end + + def delete_pg_for_origin_clusters(_name, _shard, [], _origin_node), do: [] + + def delete_pg_for_origin_clusters(name, shard, clusters, origin_node) do + cluster_set = MapSet.new(clusters) + + entries = + :ets.select(pg_by_key_table(name, shard), [ + {{{:"$1", :"$2", :"$3"}, :"$4", :"$5", origin_node}, [], + [{{:"$1", :"$2", :"$3", :"$4", :"$5"}}]} + ]) + |> Enum.filter(fn {cluster, _key, _pid, _meta, _time} -> + MapSet.member?(cluster_set, cluster) + end) + + Enum.each(entries, fn {cluster, key, pid, _meta, _time} -> + :ets.delete(pg_by_key_table(name, shard), {cluster, key, pid}) + :ets.delete(pg_by_pid_table(name, shard), {pid, cluster, key}) + end) + + entries + end + + def delete_registry_keys(name, shard, cluster, keys) do + Enum.flat_map(keys, fn key -> + case registry_lookup(name, shard, cluster, key) do + {pid, meta, time, _entry_node} -> + registry_delete(name, shard, cluster, key, pid) + [{cluster, key, pid, meta, time}] + + nil -> + [] + end + end) + end + + def delete_pg_cluster(name, shard, cluster) do + entries = + :ets.select(pg_by_key_table(name, shard), [ + {{{cluster, :"$1", :"$2"}, :"$3", :"$4", :_}, [], [{{:"$1", :"$2", :"$3", :"$4"}}]} + ]) + + Enum.each(entries, fn {key, pid, _meta, _time} -> + :ets.delete(pg_by_key_table(name, shard), {cluster, key, pid}) + :ets.delete(pg_by_pid_table(name, shard), {pid, cluster, key}) + end) + + Enum.map(entries, fn {key, pid, meta, time} -> {cluster, key, pid, meta, time} end) + end + def purge_node(name, shard, dead_node) do reg_table = reg_by_key_table(name, shard) reg_pid_table = reg_by_pid_table(name, shard) @@ -786,6 +1447,10 @@ defmodule Group.Replica.Data do GenServer.call(data_name(name), {:remove_cluster_node, clusters, node}, :infinity) end + def remove_clusters(name, clusters) when is_list(clusters) do + GenServer.call(data_name(name), {:remove_clusters, clusters}, :infinity) + end + def all_clusters(name) do table = cluster_nodes_table(name) :ets.select(table, [{{:"$1", :_}, [], [:"$1"]}]) |> Enum.uniq() @@ -796,6 +1461,10 @@ defmodule Group.Replica.Data do :ets.lookup(table, node()) |> Enum.map(&elem(&1, 1)) end + def clusters_for_node(name, target_node) do + :ets.lookup(node_clusters_table(name), target_node) |> Enum.map(&elem(&1, 1)) + end + def purge_cluster_node(name, dead_node) do GenServer.call(data_name(name), {:purge_cluster_node, dead_node}, :infinity) end @@ -841,11 +1510,21 @@ defmodule Group.Replica.Data do def reg_by_key_table(name, shard), do: :"#{name}_s#{shard}_reg_by_key" def reg_by_pid_table(name, shard), do: :"#{name}_s#{shard}_reg_by_pid" + def reg_claim_by_key_table(name, shard), do: :"#{name}_s#{shard}_reg_claim_by_key" + def reg_claim_by_pid_table(name, shard), do: :"#{name}_s#{shard}_reg_claim_by_pid" def pg_by_key_table(name, shard), do: :"#{name}_s#{shard}_pg_by_key" def pg_by_pid_table(name, shard), do: :"#{name}_s#{shard}_pg_by_pid" def cluster_nodes_table(name), do: :"#{name}_cluster_nodes" def node_clusters_table(name), do: :"#{name}_node_clusters" def cluster_leases_table(name), do: :"#{name}_cluster_leases" + def replication_meta_table(name), do: :"#{name}_replication_meta" + def local_cluster_epochs_table(name), do: :"#{name}_local_cluster_epochs" + def closed_local_cluster_epochs_table(name), do: :"#{name}_closed_local_cluster_epochs" + def remote_cluster_epochs_table(name), do: :"#{name}_remote_cluster_epochs" + def replica_stream_meta_table(name, shard), do: :"#{name}_s#{shard}_replica_stream_meta" + def replica_oplog_table(name, shard), do: :"#{name}_s#{shard}_replica_oplog" + def replica_oplog_order_table(name, shard), do: :"#{name}_s#{shard}_replica_oplog_order" + def replica_cursor_table(name, shard), do: :"#{name}_s#{shard}_replica_cursor" # ===================================================================== # GenServer callbacks @@ -867,6 +1546,22 @@ defmodule Group.Replica.Data do {:reply, :ok, state} end + def handle_call({:remove_clusters, clusters}, _from, state) do + Enum.each(clusters, fn cluster -> + nodes = cluster_nodes(state.name, cluster) + :ets.delete(cluster_nodes_table(state.name), cluster) + + Enum.each(nodes, fn cluster_node -> + :ets.delete_object( + node_clusters_table(state.name), + {cluster_node, cluster} + ) + end) + end) + + {:reply, :ok, state} + end + def handle_call({:purge_cluster_node, dead_node}, _from, state) do # Scan the forward index directly so this also repairs a one-sided row left # by an interrupted or older dual-index mutation. @@ -878,14 +1573,267 @@ defmodule Group.Replica.Data do {:reply, :ok, state} end + def handle_call({:activate_local_clusters, clusters}, _from, state) do + if clusters != [] do + :ets.update_counter( + replication_meta_table(state.name), + :cluster_epoch_revision, + {2, 1}, + {:cluster_epoch_revision, 0} + ) + end + + epochs = + Enum.map(clusters, fn cluster -> + epoch = + case :ets.lookup(local_cluster_epochs_table(state.name), cluster) do + [{^cluster, existing}] -> existing + [] -> make_ref() + end + + :ets.insert(local_cluster_epochs_table(state.name), {cluster, epoch}) + :ets.delete(closed_local_cluster_epochs_table(state.name), cluster) + {cluster, epoch} + end) + + {:reply, epochs, state} + end + + def handle_call(:local_replica_authority, _from, state) do + generation = generation(state.name) + revision = local_cluster_epoch_revision(state.name) + epochs = [{nil, generation} | :ets.tab2list(local_cluster_epochs_table(state.name))] + {:reply, {generation, revision, epochs}, state} + end + + def handle_call({:deactivate_local_clusters, clusters}, _from, state) do + if clusters != [] do + :ets.update_counter( + replication_meta_table(state.name), + :cluster_epoch_revision, + {2, 1}, + {:cluster_epoch_revision, 0} + ) + end + + epochs = + Enum.map(clusters, fn cluster -> + epoch = local_cluster_epoch(state.name, cluster) + :ets.delete(local_cluster_epochs_table(state.name), cluster) + if epoch, do: :ets.insert(closed_local_cluster_epochs_table(state.name), {cluster, epoch}) + {cluster, epoch} + end) + + {:reply, epochs, state} + end + + def handle_call( + {:put_remote_replica_info, shard, remote_node, generation, epoch_revision, epochs}, + _from, + state + ) do + # The epoch snapshot is node-wide authority, not shard-local replica data. + # Only shard 0 sends it, and Data serializes the one exact replacement for + # every local replica lane. Keeping the argument in the API makes the + # control-owner invariant explicit and catches accidental reintroduction of + # one full copy per shard. + 0 = shard + seen_generation = remote_generation(state.name, remote_node) + current_epochs = Map.new(epochs) + + stale_epochs = + if seen_generation == generation do + for {{^remote_node, cluster}, epoch} <- + :ets.match_object( + remote_cluster_epochs_table(state.name), + {{remote_node, :_}, :_} + ), + not is_nil(cluster), + Map.get(current_epochs, cluster) != epoch, + do: {cluster, epoch} + else + [] + end + + # A hello is a complete epoch snapshot. The replica handler fences older + # revisions before this call, so replace the shared view rather than merely + # adding rows; otherwise a dropped close control could leave a cluster epoch + # permanently valid after the heartbeat-driven repair. + :ets.select_delete(remote_cluster_epochs_table(state.name), [ + {{{remote_node, :_}, :_}, [], [true]} + ]) + + :ets.insert( + replication_meta_table(state.name), + {{:remote_generation, remote_node}, generation} + ) + + :ets.insert( + replication_meta_table(state.name), + {{:remote_epoch_revision, remote_node}, epoch_revision} + ) + + :ets.insert( + replication_meta_table(state.name), + {{:remote_epoch_exact, remote_node}, epoch_revision} + ) + + :ets.insert( + replication_meta_table(state.name), + {{:remote_epoch_observed, remote_node}, epoch_revision} + ) + + :ets.update_counter( + replication_meta_table(state.name), + {:remote_authority_installs, remote_node}, + {2, 1}, + {{:remote_authority_installs, remote_node}, 0} + ) + + for view_shard <- 0..(state.num_shards - 1) do + :ets.insert( + replication_meta_table(state.name), + {{:remote_view_info, view_shard, remote_node}, generation, epoch_revision, epoch_revision} + ) + end + + rows = + for {cluster, epoch} <- epochs, not is_nil(cluster), do: {{remote_node, cluster}, epoch} + + :ets.insert(remote_cluster_epochs_table(state.name), rows) + + {:reply, {seen_generation, stale_epochs}, state} + end + + def handle_call( + {:put_remote_view_info, shard, remote_node, generation, authoritative, observed}, + _from, + state + ) do + :ets.insert( + replication_meta_table(state.name), + {{:remote_view_info, shard, remote_node}, generation, authoritative, observed} + ) + + {:reply, :ok, state} + end + + def handle_call( + {:put_remote_cluster_epochs, shard, remote_node, revision, epochs}, + _from, + state + ) do + _ = shard + observe_remote_cluster_revision(state.name, remote_node, revision, state.num_shards) + + stale_epochs = + Enum.flat_map(epochs, fn {cluster, epoch} -> + case remote_cluster_epoch(state.name, remote_node, cluster) do + old_epoch when not is_nil(old_epoch) and old_epoch != epoch -> [{cluster, old_epoch}] + _ -> [] + end + end) + + rows = + for {cluster, epoch} <- epochs, + not is_nil(cluster), + do: {{remote_node, cluster}, epoch} + + :ets.insert(remote_cluster_epochs_table(state.name), rows) + + current_revision = remote_cluster_epoch_revision(state.name, remote_node) + + :ets.insert( + replication_meta_table(state.name), + {{:remote_epoch_revision, remote_node}, max(current_revision || revision, revision)} + ) + + {:reply, stale_epochs, state} + end + + def handle_call( + {:close_remote_cluster_epochs, shard, remote_node, revision, epochs}, + _from, + state + ) do + 0 = shard + observe_remote_cluster_revision(state.name, remote_node, revision, state.num_shards) + + closed = + Enum.filter(epochs, fn {cluster, epoch} -> + remote_cluster_epoch(state.name, remote_node, cluster) == epoch + end) + + Enum.each(epochs, fn {cluster, epoch} -> + case :ets.lookup(remote_cluster_epochs_table(state.name), {remote_node, cluster}) do + [{{^remote_node, ^cluster}, ^epoch}] -> + :ets.delete(remote_cluster_epochs_table(state.name), {remote_node, cluster}) + + _ -> + :ok + end + end) + + current_revision = remote_cluster_epoch_revision(state.name, remote_node) + + :ets.insert( + replication_meta_table(state.name), + {{:remote_epoch_revision, remote_node}, max(current_revision || revision, revision)} + ) + + {:reply, closed, state} + end + + def handle_call( + {:forget_remote_cluster_epochs, shard, remote_node, epochs}, + _from, + state + ) do + # Kept for the rolling-compatibility receive path. The node-wide authority + # table is intentionally not mutated by a shard-local purge. + _ = {shard, remote_node, epochs} + {:reply, :ok, state} + end + + def handle_call({:delete_remote_replica_info, shard, remote_node}, _from, state) do + :ets.delete( + replication_meta_table(state.name), + {:remote_view_info, shard, remote_node} + ) + + if shard == 0 do + :ets.delete(replication_meta_table(state.name), {:remote_generation, remote_node}) + :ets.delete(replication_meta_table(state.name), {:remote_epoch_revision, remote_node}) + :ets.delete(replication_meta_table(state.name), {:remote_epoch_exact, remote_node}) + :ets.delete(replication_meta_table(state.name), {:remote_epoch_observed, remote_node}) + :ets.delete(replication_meta_table(state.name), {:remote_authority_installs, remote_node}) + + if state.num_shards > 1 do + for view_shard <- 1..(state.num_shards - 1) do + :ets.delete( + replication_meta_table(state.name), + {:remote_view_info, view_shard, remote_node} + ) + end + end + + :ets.select_delete(remote_cluster_epochs_table(state.name), [ + {{{remote_node, :_}, :_}, [], [true]} + ]) + end + + {:reply, :ok, state} + end + @impl true def init({name, num_shards}) do # ETS performance options: # - read_concurrency: splits table into read-optimized segments (less lock contention) # - decentralized_counters: reduces contention on table size counter (OTP 23+) - # Note: write_concurrency is intentionally omitted — the sharded GenServer already - # serializes writes per shard, so ETS write locking is never contended. Adding - # write_concurrency adds overhead (~30-40% on serial benchmarks) without benefit. + # Per-shard tables omit write_concurrency because each shard GenServer serializes + # their writes. The shared replication metadata table is different: every shard + # atomically advances its own {:append_counter, shard} object there, so it needs + # concurrent writes without weakening ETS's single-object atomicity. set_opts = [ :set, :public, @@ -894,6 +1842,8 @@ defmodule Group.Replica.Data do decentralized_counters: true ] + shared_meta_opts = Keyword.put(set_opts, :write_concurrency, :auto) + ordered_set_opts = [ :ordered_set, :public, @@ -913,17 +1863,62 @@ defmodule Group.Replica.Data do for shard <- 0..(num_shards - 1) do :ets.new(reg_by_key_table(name, shard), set_opts) :ets.new(reg_by_pid_table(name, shard), ordered_set_opts) + :ets.new(reg_claim_by_key_table(name, shard), ordered_set_opts) + :ets.new(reg_claim_by_pid_table(name, shard), ordered_set_opts) :ets.new(pg_by_key_table(name, shard), ordered_set_opts) :ets.new(pg_by_pid_table(name, shard), ordered_set_opts) + :ets.new(replica_stream_meta_table(name, shard), set_opts) + :ets.new(replica_oplog_table(name, shard), ordered_set_opts) + :ets.new(replica_oplog_order_table(name, shard), ordered_set_opts) + :ets.new(replica_cursor_table(name, shard), set_opts) end :ets.new(cluster_nodes_table(name), bag_opts) :ets.new(node_clusters_table(name), bag_opts) :ets.new(cluster_leases_table(name), set_opts) + :ets.new(replication_meta_table(name), shared_meta_opts) + :ets.new(local_cluster_epochs_table(name), set_opts) + :ets.new(closed_local_cluster_epochs_table(name), set_opts) + :ets.new(remote_cluster_epochs_table(name), set_opts) + :ets.insert(replication_meta_table(name), {:generation, make_ref()}) + :ets.insert(replication_meta_table(name), {:cluster_epoch_revision, 0}) {:ok, %{name: name, num_shards: num_shards}} end + defp observe_remote_cluster_revision(name, remote_node, revision, num_shards) do + key = {:remote_epoch_observed, remote_node} + + case :ets.lookup(replication_meta_table(name), key) do + [{^key, current}] when current >= revision -> :ok + _ -> :ets.insert(replication_meta_table(name), {key, revision}) + end + + for shard <- 0..(num_shards - 1) do + view_key = {:remote_view_info, shard, remote_node} + + case :ets.lookup(replication_meta_table(name), view_key) do + [{^view_key, _generation, _authoritative, observed}] when observed >= revision -> + :ok + + [{^view_key, generation, authoritative, _observed}] -> + :ets.insert( + replication_meta_table(name), + {view_key, generation, authoritative, revision} + ) + + [] -> + :ets.insert( + replication_meta_table(name), + {view_key, remote_generation(name, remote_node), + remote_cluster_epoch_revision(name, remote_node), revision} + ) + end + end + + :ok + end + defp select(table, match_spec, :infinity), do: :ets.select(table, match_spec) defp select(_table, _match_spec, 0), do: [] diff --git a/lib/group/replica/protocol.ex b/lib/group/replica/protocol.ex new file mode 100644 index 0000000..fe88e24 --- /dev/null +++ b/lib/group/replica/protocol.ex @@ -0,0 +1,30 @@ +defmodule Group.Replica.Protocol do + @moduledoc false + + @version 1 + + def version, do: @version + + def stream_id(name, origin_node, origin_generation, shard, cluster, cluster_epoch) do + {name, origin_node, origin_generation, shard, cluster, cluster_epoch} + end + + def stream_name({name, _origin_node, _generation, _shard, _cluster, _epoch}), do: name + + def stream_origin({_name, origin_node, _generation, _shard, _cluster, _epoch}), + do: origin_node + + def stream_generation({_name, _origin_node, generation, _shard, _cluster, _epoch}), + do: generation + + def stream_shard({_name, _origin_node, _generation, shard, _cluster, _epoch}), do: shard + + def stream_cluster({_name, _origin_node, _generation, _shard, cluster, _epoch}), do: cluster + + def stream_epoch({_name, _origin_node, _generation, _shard, _cluster, epoch}), do: epoch + + def op_cluster({:register, cluster, _key, _pid, _meta, _time, _node}), do: cluster + def op_cluster({:unregister, cluster, _key, _pid, _meta, _reason}), do: cluster + def op_cluster({:join, cluster, _key, _pid, _meta, _time, _reason, _node}), do: cluster + def op_cluster({:leave, cluster, _key, _pid, _meta, _reason}), do: cluster +end diff --git a/lib/group/replica/transport.ex b/lib/group/replica/transport.ex new file mode 100644 index 0000000..8b23595 --- /dev/null +++ b/lib/group/replica/transport.ex @@ -0,0 +1,97 @@ +defmodule Group.Replica.Transport do + @moduledoc """ + Transport contract for Group replica data. + + Implementations must return promptly and must never wait for socket or remote + mailbox backpressure. This applies to `try_send/5` and the optional lifecycle + callbacks. Returning `:busy` or `:disconnected` is safe: replica anti-entropy + will retransmit the missing state. + + Erlang distribution remains Group's control plane and supplies the stable + node identity used here. A sideband adapter can use its `descriptor/2` in the + control hello to exchange endpoints, authenticate the connection as that + node, and pass inbound frames to `deliver/4`. + + Adapters do not need to preserve ordering. Group serializes writes per shard + and sequences each origin/generation/shard/cluster/epoch stream; receivers + discard duplicates and request gaps. Per-shard ordered delivery avoids repair + traffic and is therefore the preferred fast path. + """ + + @type frame :: term() + @type send_result :: :ok | :busy | :disconnected + + @callback id() :: term() + @callback descriptor(group :: atom(), opts :: keyword()) :: term() + @callback try_send( + group :: atom(), + target_node :: node(), + shard :: non_neg_integer(), + frame(), + opts :: keyword() + ) :: send_result() + + @callback child_spec(keyword()) :: Supervisor.child_spec() | :ignore + @callback peer_up(group :: atom(), node(), descriptor :: term(), opts :: keyword()) :: :ok + @callback peer_down(group :: atom(), node(), opts :: keyword()) :: :ok + + @optional_callbacks child_spec: 1, peer_up: 4, peer_down: 3 + + @doc """ + Delivers a frame received by a transport adapter to the local replica shard. + + `source_node` must come from the adapter's authenticated peer identity, never + from untrusted frame contents. Delivery is a local mailbox operation; stream + generation, epoch, group, shard, and origin are validated by the replica. + """ + def deliver(group, source_node, shard, frame) + when is_atom(group) and is_atom(source_node) and is_integer(shard) and shard >= 0 do + send(Group.Replica.shard_name(group, shard), {:group_replica_frame, source_node, frame}) + :ok + end + + def normalize(module) when is_atom(module), do: {module, []} + def normalize({module, opts}) when is_atom(module) and is_list(opts), do: {module, opts} + + def normalize(other) do + raise ArgumentError, + "expected :replica_transport to be a module or {module, opts}, got: #{inspect(other)}" + end + + def validate!({module, _opts} = transport) do + Code.ensure_loaded!(module) + + for {function, arity} <- [id: 0, descriptor: 2, try_send: 5] do + unless function_exported?(module, function, arity) do + raise ArgumentError, + "replica transport #{inspect(module)} must implement #{function}/#{arity}" + end + end + + transport + end +end + +defmodule Group.Replica.Transport.Distribution do + @moduledoc false + @behaviour Group.Replica.Transport + + alias Group.Replica + + @impl true + def id, do: :erlang_distribution + + @impl true + def descriptor(_group, _opts), do: :erlang_distribution + + @impl true + def try_send(group, target_node, shard, frame, _opts) do + destination = {Replica.shard_name(group, shard), target_node} + message = {:group_replica_frame, node(), frame} + + case :erlang.send_nosuspend(destination, message, [:noconnect]) do + true -> :ok + false -> :busy + end + end +end diff --git a/lib/group/supervisor.ex b/lib/group/supervisor.ex index e4e15b2..0bbd773 100644 --- a/lib/group/supervisor.ex +++ b/lib/group/supervisor.ex @@ -42,6 +42,27 @@ defmodule Group.Supervisor do replicated_pg_receiver_local_request_quota = positive_integer_opt(opts, :replicated_pg_receiver_local_request_quota, 8) + replica_transport = + opts + |> Keyword.get(:replica_transport, Group.Replica.Transport.Distribution) + |> Group.Replica.Transport.normalize() + |> Group.Replica.Transport.validate!() + + replicated_oplog_max_entries = + positive_integer_opt(opts, :replicated_oplog_max_entries, 65_536) + + replicated_anti_entropy_interval = + positive_integer_opt(opts, :replicated_anti_entropy_interval, 1_000) + + replicated_peer_lease_timeout = + positive_integer_opt(opts, :replicated_peer_lease_timeout, 15_000) + + if replicated_peer_lease_timeout <= replicated_anti_entropy_interval do + raise ArgumentError, + ":replicated_peer_lease_timeout must be greater than " <> + ":replicated_anti_entropy_interval" + end + # persistent_term config — must be set before children start (Replica reads it) config = %{ num_shards: num_shards, @@ -54,7 +75,11 @@ defmodule Group.Supervisor do replicated_sender_flush_interval: replicated_sender_flush_interval, busy_dist_retry_attempts: busy_dist_retry_attempts, busy_dist_retry_interval: busy_dist_retry_interval, - replicated_pg_receiver_local_request_quota: replicated_pg_receiver_local_request_quota + replicated_pg_receiver_local_request_quota: replicated_pg_receiver_local_request_quota, + replica_transport: replica_transport, + replicated_oplog_max_entries: replicated_oplog_max_entries, + replicated_anti_entropy_interval: replicated_anti_entropy_interval, + replicated_peer_lease_timeout: replicated_peer_lease_timeout } config = if extract_meta, do: Map.put(config, :extract_meta, extract_meta), else: config @@ -66,18 +91,33 @@ defmodule Group.Supervisor do :persistent_term.put({Group, name}, config) - children = [ - {Group.Replica.Data, name: name, num_shards: num_shards}, - { - Group.PeerReconnect, - name: name, - busy_dist_retry_attempts: busy_dist_retry_attempts, - busy_dist_retry_interval: busy_dist_retry_interval - }, - {Group.Replica.Supervisor, name: name, num_shards: num_shards}, - {Registry, keys: :duplicate, name: Group.registry_name(name)}, - {Group.ClusterLease, name: name, num_shards: num_shards} - ] + transport_children = + case replica_transport do + {module, transport_opts} -> + if function_exported?(module, :child_spec, 1) do + case module.child_spec([name: name, num_shards: num_shards] ++ transport_opts) do + :ignore -> [] + child_spec -> [child_spec] + end + else + [] + end + end + + children = + transport_children ++ + [ + {Group.Replica.Data, name: name, num_shards: num_shards}, + { + Group.PeerReconnect, + name: name, + busy_dist_retry_attempts: busy_dist_retry_attempts, + busy_dist_retry_interval: busy_dist_retry_interval + }, + {Group.Replica.Supervisor, name: name, num_shards: num_shards}, + {Registry, keys: :duplicate, name: Group.registry_name(name)}, + {Group.ClusterLease, name: name, num_shards: num_shards} + ] Supervisor.init(children, strategy: :rest_for_one) end diff --git a/priv/bench/README.md b/priv/bench/README.md index a51e3a8..6278aa5 100644 --- a/priv/bench/README.md +++ b/priv/bench/README.md @@ -7,7 +7,7 @@ separate BEAM VMs. ## Running ```bash -cd priv/group/priv/bench +cd priv/bench mix deps.get ``` @@ -30,10 +30,19 @@ Uses 3 separate BEAM VMs (coordinator + 2 replicas) as OS processes: The script compiles once, starts both replicas in the background, then launches the coordinator. Replicas are killed automatically on exit. +To isolate the 10,000-cluster lifecycle scenario: + +```bash +./run_distributed.sh --shards 4 \ + --coordinator-expr 'GroupBench.Distributed.run_many_clusters_only(shards: 4)' +``` + ## Local Scenarios All local benchmarks run for both the default (nil) cluster and a named cluster (`"game"`) to verify there's no performance difference between the two paths. +Each spawned process cohort is stopped after its measurement and before the +next case; cohort teardown is outside the measured interval. ### 1. Lookup throughput @@ -52,8 +61,10 @@ Slower than lookup because each call copies a 100-element list out of ETS. ### 3. Register throughput (shard scaling) Measures concurrent `Group.register/4` calls — each of 10K spawned processes -registers itself in parallel. Varies shard count (1, 2, 4, schedulers_online) -to show how write throughput scales with sharding. +registers itself in parallel. Uses the library default of 8 shards for the +non-scaling scenarios and a fixed 1, 2, 4, 8, 16, 32, 64 shard sweep. The +fixed sweep keeps results comparable across machines and avoids treating BEAM +scheduler count as a shard-count recommendation. ### 4. Register/unregister cycle @@ -84,7 +95,8 @@ The core distributed measurement. Registers a key on replica1, then spin-polls `Group.lookup` on replica2 until it appears. Repeats 1,000 times. Reports p50/p99/max latency covering the full path: GenServer call on replica1, -Erlang distribution message, GenServer cast on replica2, ETS insert. +write-ahead append, nonblocking replica transport, receiver application, and +ETS projection on replica2. ### 2. Bulk sync (new peer catches up) @@ -92,8 +104,10 @@ Measures how fast a new node catches up to an existing peer's state. Registers N keys on replica1 (1K and 10K), then starts Group on replica2 and polls until all N entries are visible. -Group sends all data in a single `cluster_state` message on peer discovery, so -this is bounded by serialization + network, not per-key round-trips. +Group advertises stream heads on peer discovery. A new peer requests the +missing range; if the bounded oplog no longer contains the prefix, Group sends +an exact per-origin snapshot. The measurement therefore covers the normal +catch-up decision as well as serialization and network transfer. ### 3. Concurrent cross-node writes @@ -115,8 +129,8 @@ compared to the default nil cluster. The critical distributed cleanup path. Registers 1K and 5K processes on replica1, kills them all, then measures how long until replica2 sees zero -entries. Exercises: local DOWN handler → `replicate_unregister` broadcast → -remote ETS cleanup. +entries. Exercises: local DOWN handler → authoritative sequenced unregister +records → nonblocking delta batch → remote ETS cleanup. This scenario catches O(N²) message amplification bugs where remote nodes redundantly monitor pids and re-broadcast cleanup messages. @@ -136,10 +150,42 @@ convergence on replica2 via the `replicate_leave` path. All members hash to the same shard (single key), making this the worst case for shard contention during bulk cleanup. +### 8. Many-cluster lifecycle + +Connects 10K named clusters, registers one process in each, forces peer +re-discovery, then disconnects and verifies cleanup. This exposes control-plane +message amplification and epoch-fence costs. + +The connect phase reports local `Group.connect/2` completion separately from +full remote control convergence. Full convergence checks the reverse cluster +index on both nodes for all 10K named clusters plus the default cluster; seeing +only the last submitted cluster is insufficient because controls may be +reordered or repaired asynchronously. On generation/epoch-aware builds the +barrier also requires every replica shard to hold the source's current control +revision and all 10K remote epoch rows. Registration starts only after this +barrier, so its result does not inherit unfinished connect work. + +Registration reports local completion, the remote count at that handoff, and +the remaining data-convergence tail separately. Re-discovery similarly splits +restart, local reconnect, control convergence, and data convergence; disconnect +splits local completion from remote cleanup. + +### 9. Busy application convergence + +Runs registry and PG churn across 50 clusters and 40K initial pids. Reports +application throughput and then verifies the replicas agree exactly. A fast +wall-clock result is not considered successful unless convergence completes. + +### 10. Local writes under replicated PG pressure + +Floods one receiver shard with remote membership updates while measuring local +register and join calls at increasing concurrency. This checks that bounded +replica turns preserve local control-plane progress. + ## Architecture ``` -priv/group/priv/bench/ +priv/bench/ ├── mix.exs # depends on :group via path: "../../" ├── run_distributed.sh # starts 3 VMs, cleans up on exit ├── README.md @@ -147,7 +193,7 @@ priv/group/priv/bench/ │ ├── group_bench.ex # CLI entry — dispatches local/distributed │ ├── group_bench/ │ │ ├── local.ex # 6 local benchmarks -│ │ ├── distributed.ex # coordinator: connects + drives 7 benchmarks +│ │ ├── distributed.ex # coordinator: connects + drives 10 benchmarks │ │ ├── replica.ex # helpers called by coordinator via :erpc │ │ └── helpers.ex # timing, formatting, percentile math ``` diff --git a/priv/bench/lib/group_bench/distributed.ex b/priv/bench/lib/group_bench/distributed.ex index 68369d2..66b601f 100644 --- a/priv/bench/lib/group_bench/distributed.ex +++ b/priv/bench/lib/group_bench/distributed.ex @@ -52,6 +52,21 @@ defmodule GroupBench.Distributed do IO.puts("\n Done.\n") end + def run_many_clusters_only(opts \\ []) do + shards = Keyword.get(opts, :shards, 4) + Process.put(:bench_shards, shards) + + header("Distributed Many-Clusters Benchmark") + IO.puts(" coordinator: #{node()}") + IO.puts(" shards: #{shards}") + IO.puts(" schedulers: #{System.schedulers_online()}") + + connect_replicas() + bench_many_clusters(@replicas) + + IO.puts("\n Done.\n") + end + # ── Connection ──────────────────────────────────────────────────────── defp connect_replicas do @@ -428,7 +443,7 @@ defmodule GroupBench.Distributed do start_group_on(r2) wait_for_peer_discovery(replicas) - {connect_us, _} = + {local_connect_us, _} = :timer.tc(fn -> t1 = Task.async(fn -> @@ -453,48 +468,84 @@ defmodule GroupBench.Distributed do end) Task.await_many([t1, t2], 120_000) - - # Wait for convergence — both nodes see each other in the last cluster - poll_until( - fn -> - n1 = :erpc.call(r1, Group, :nodes, [@name, "#{prefix}#{num_clusters}"]) - n2 = :erpc.call(r2, Group, :nodes, [@name, "#{prefix}#{num_clusters}"]) - length(n1) >= 1 and length(n2) >= 1 - end, - 60_000 - ) end) - IO.puts(" connect: #{format_number(div(connect_us, 1000))} ms") - IO.puts(" clusters/sec: #{format_number(round(num_clusters * 1_000_000 / connect_us))}") + # Local completion only means both nodes accepted all Group.connect calls. + # Controls may be reordered or repaired asynchronously, so seeing one + # sentinel cluster cannot prove that the other 9,999 have converged. + expected_cluster_count = num_clusters + 1 + + {control_convergence_us, _} = + try do + :timer.tc(fn -> + wait_for_cluster_control_convergence(r1, r2, expected_cluster_count) + end) + rescue + error -> + r1_status = :erpc.call(r1, GroupBench.Replica, :cluster_control_status, [@name, r2]) + r2_status = :erpc.call(r2, GroupBench.Replica, :cluster_control_status, [@name, r1]) + + reraise RuntimeError, + [ + message: + "#{Exception.message(error)}; r1=#{inspect(r1_status)} " <> + "r2=#{inspect(r2_status)}" + ], + __STACKTRACE__ + end + + connect_total_us = local_connect_us + control_convergence_us + + IO.puts(" local calls: #{format_number(div(local_connect_us, 1000))} ms") + IO.puts(" control convergence: #{format_number(div(control_convergence_us, 1000))} ms") + IO.puts(" end-to-end connect: #{format_number(div(connect_total_us, 1000))} ms") + + IO.puts( + " local clusters/sec: #{format_number(round(num_clusters * 1_000_000 / local_connect_us))}" + ) + + IO.puts( + " converged clusters/sec: #{format_number(round(num_clusters * 1_000_000 / connect_total_us))}" + ) # -- 8b. Register 1 key per cluster -- subheader("register across #{format_number(num_clusters)} clusters") - {reg_us, pids} = + {local_reg_us, pids} = :timer.tc(fn -> - pids = - :erpc.call( - r1, - GroupBench.Replica, - :bulk_register_per_cluster, - [@name, num_clusters, prefix], - 120_000 - ) + :erpc.call( + r1, + GroupBench.Replica, + :bulk_register_per_cluster, + [@name, num_clusters, prefix], + 120_000 + ) + end) + + remote_count_at_handoff = + :erpc.call(r2, GroupBench.Replica, :total_registry_count, [@name]) + {reg_convergence_us, _} = + :timer.tc(fn -> poll_until( fn -> :erpc.call(r2, GroupBench.Replica, :total_registry_count, [@name]) >= num_clusters end, 60_000 ) - - pids end) - IO.puts(" register+converge: #{format_number(div(reg_us, 1000))} ms") - IO.puts(" ops/sec: #{format_number(round(num_clusters * 1_000_000 / reg_us))}") + reg_total_us = local_reg_us + reg_convergence_us + + IO.puts(" local registration: #{format_number(div(local_reg_us, 1000))} ms") + IO.puts(" remote at handoff: #{format_number(remote_count_at_handoff)} / 10,000") + IO.puts(" data convergence: #{format_number(div(reg_convergence_us, 1000))} ms") + IO.puts(" register end-to-end: #{format_number(div(reg_total_us, 1000))} ms") + + IO.puts( + " converged ops/sec: #{format_number(round(num_clusters * 1_000_000 / reg_total_us))}" + ) # -- 8c. Peer re-discovery with many clusters -- @@ -504,28 +555,75 @@ defmodule GroupBench.Distributed do stop_group_on(r2) Process.sleep(500) - {rediscovery_us, _} = + {restart_us, _} = :timer.tc(fn -> start_group_on(r2) wait_for_peer_discovery(replicas) + end) + {local_reconnect_us, _} = + :timer.tc(fn -> :erpc.call(r2, GroupBench.Replica, :bulk_connect, [@name, num_clusters, prefix], 120_000) + end) - poll_until( - fn -> - :erpc.call(r2, GroupBench.Replica, :total_registry_count, [@name]) >= num_clusters - end, - 120_000 - ) + {reconnect_control_us, _} = + :timer.tc(fn -> + wait_for_cluster_control_convergence(r1, r2, expected_cluster_count) end) - IO.puts(" re-discovery: #{format_number(div(rediscovery_us, 1000))} ms") + {rediscovery_data_us, _} = + try do + :timer.tc(fn -> + poll_until( + fn -> + :erpc.call(r2, GroupBench.Replica, :total_registry_count, [@name]) >= num_clusters + end, + 120_000 + ) + end) + rescue + error -> + counts = + :erpc.call(r2, GroupBench.Replica, :registry_counts_by_shard, [@name]) + + diagnostics = + :erpc.call(r2, GroupBench.Replica, :replica_process_diagnostics, [@name]) + + r1_revision = + :erpc.call(r1, Group.Replica.Data, :local_cluster_epoch_revision, [@name]) + + r2_remote_revision = + :erpc.call(r2, Group.Replica.Data, :remote_cluster_epoch_revision, [@name, r1]) + + r1_remote_revision = + :erpc.call(r1, Group.Replica.Data, :remote_cluster_epoch_revision, [@name, r2]) + + reraise RuntimeError, + [ + message: + "#{Exception.message(error)}; rediscovery_counts=#{inspect(counts)} " <> + "r1_revision=#{r1_revision} " <> + "r2_remote_revision=#{inspect(r2_remote_revision)} " <> + "r1_remote_revision=#{inspect(r1_remote_revision)} " <> + "replica=#{inspect(diagnostics)}" + ], + __STACKTRACE__ + end + + rediscovery_total_us = + restart_us + local_reconnect_us + reconnect_control_us + rediscovery_data_us + + IO.puts(" restart + discovery: #{format_number(div(restart_us, 1000))} ms") + IO.puts(" local reconnect: #{format_number(div(local_reconnect_us, 1000))} ms") + IO.puts(" control convergence: #{format_number(div(reconnect_control_us, 1000))} ms") + IO.puts(" data convergence: #{format_number(div(rediscovery_data_us, 1000))} ms") + IO.puts(" re-discovery total: #{format_number(div(rediscovery_total_us, 1000))} ms") # -- 8d. Disconnect cleanup -- subheader("disconnect #{format_number(num_clusters)} clusters") - {disconnect_us, _} = + {local_disconnect_us, _} = :timer.tc(fn -> :erpc.call( r1, @@ -534,20 +632,58 @@ defmodule GroupBench.Distributed do [@name, num_clusters, prefix], 120_000 ) + end) + {cleanup_us, _} = + :timer.tc(fn -> # Wait for r2 to see r1's entries cleaned from all clusters - poll_until( - fn -> - :erpc.call(r2, GroupBench.Replica, :total_registry_count, [@name]) == 0 - end, - 60_000 - ) + try do + poll_until( + fn -> + :erpc.call(r2, GroupBench.Replica, :total_registry_count, [@name]) == 0 + end, + 60_000 + ) + rescue + error -> + count = :erpc.call(r2, GroupBench.Replica, :total_registry_count, [@name]) + sample = :erpc.call(r2, GroupBench.Replica, :registry_sample, [@name]) + + local_revision = + :erpc.call(r1, Group.Replica.Data, :local_cluster_epoch_revision, [@name]) + + remote_revision = + :erpc.call(r2, Group.Replica.Data, :remote_cluster_epoch_revision, [@name, r1]) + + observed_revision = + :erpc.call( + r2, + Group.Replica.Data, + :remote_cluster_epoch_observed_revision, + [@name, r1] + ) + + reraise RuntimeError, + [ + message: + "#{Exception.message(error)}; remaining=#{count} " <> + "local_revision=#{local_revision} " <> + "remote_revision=#{inspect(remote_revision)} " <> + "observed_revision=#{inspect(observed_revision)} " <> + "sample=#{inspect(sample)}" + ], + __STACKTRACE__ + end end) - IO.puts(" disconnect+cleanup: #{format_number(div(disconnect_us, 1000))} ms") + disconnect_total_us = local_disconnect_us + cleanup_us + + IO.puts(" local disconnect: #{format_number(div(local_disconnect_us, 1000))} ms") + IO.puts(" remote cleanup: #{format_number(div(cleanup_us, 1000))} ms") + IO.puts(" disconnect total: #{format_number(div(disconnect_total_us, 1000))} ms") IO.puts( - " clusters/sec: #{format_number(round(num_clusters * 1_000_000 / disconnect_us))}" + " converged clusters/sec: #{format_number(round(num_clusters * 1_000_000 / disconnect_total_us))}" ) # Kill leftover processes @@ -555,6 +691,37 @@ defmodule GroupBench.Distributed do stop_groups(replicas) end + defp wait_for_cluster_control_convergence(r1, r2, expected_cluster_count) do + r1_revision = + :erpc.call(r1, GroupBench.Replica, :cluster_control_revision, [@name]) + + r2_revision = + :erpc.call(r2, GroupBench.Replica, :cluster_control_revision, [@name]) + + poll_until( + fn -> + r1_converged? = + :erpc.call( + r1, + GroupBench.Replica, + :cluster_control_converged?, + [@name, r2, expected_cluster_count, r2_revision] + ) + + r2_converged? = + :erpc.call( + r2, + GroupBench.Replica, + :cluster_control_converged?, + [@name, r1, expected_cluster_count, r1_revision] + ) + + r1_converged? and r2_converged? + end, + 120_000 + ) + end + # ── 9. Busy app simulation ────────────────────────────────────────── defp bench_busy_app([r1, r2] = replicas) do diff --git a/priv/bench/lib/group_bench/local.ex b/priv/bench/lib/group_bench/local.ex index dad6cf1..667bae3 100644 --- a/priv/bench/lib/group_bench/local.ex +++ b/priv/bench/lib/group_bench/local.ex @@ -6,11 +6,14 @@ defmodule GroupBench.Local do import GroupBench.Helpers @name :bench - @default_shards System.schedulers_online() + @default_shards 8 + @shard_counts [1, 2, 4, 8, 16, 32, 64] def run do header("Local Benchmarks") IO.puts(" schedulers_online: #{System.schedulers_online()}") + IO.puts(" default_shards: #{@default_shards}") + IO.puts(" shard_sweep: #{Enum.join(@shard_counts, ", ")}") bench_lookup() bench_members() @@ -36,21 +39,26 @@ defmodule GroupBench.Local do measure_count = 100_000 # Each process registers itself - register_from_spawned_processes(key_count, fn i -> - Group.register(@name, "key-#{i}", %{i: i}, cluster_opts(cluster_opt)) - end) + pids = + register_from_spawned_processes(key_count, fn i -> + Group.register(@name, "key-#{i}", %{i: i}, cluster_opts(cluster_opt)) + end) - # warmup - warmup(1_000, fn -> Group.lookup(@name, "key-1", cluster_opts(cluster_opt)) end) + try do + # warmup + warmup(1_000, fn -> Group.lookup(@name, "key-1", cluster_opts(cluster_opt)) end) - # measure - samples = - collect_samples(measure_count, fn -> - i = :rand.uniform(key_count) - Group.lookup(@name, "key-#{i}", cluster_opts(cluster_opt)) - end) + # measure + samples = + collect_samples(measure_count, fn -> + i = :rand.uniform(key_count) + Group.lookup(@name, "key-#{i}", cluster_opts(cluster_opt)) + end) - report_latency("Group.lookup/3", samples) + report_latency("Group.lookup/3", samples) + after + stop_spawned_processes(pids) + end end) end end @@ -72,20 +80,25 @@ defmodule GroupBench.Local do total = group_count * members_per_group # Each process joins a group - register_from_spawned_processes(total, fn i -> - gi = rem(i - 1, group_count) + 1 - Group.join(@name, "group-#{gi}", %{}, cluster_opts(cluster_opt)) - end) + pids = + register_from_spawned_processes(total, fn i -> + gi = rem(i - 1, group_count) + 1 + Group.join(@name, "group-#{gi}", %{}, cluster_opts(cluster_opt)) + end) - warmup(1_000, fn -> Group.members(@name, "group-1", cluster_opts(cluster_opt)) end) + try do + warmup(1_000, fn -> Group.members(@name, "group-1", cluster_opts(cluster_opt)) end) - samples = - collect_samples(measure_count, fn -> - gi = :rand.uniform(group_count) - Group.members(@name, "group-#{gi}", cluster_opts(cluster_opt)) - end) + samples = + collect_samples(measure_count, fn -> + gi = :rand.uniform(group_count) + Group.members(@name, "group-#{gi}", cluster_opts(cluster_opt)) + end) - report_latency("Group.members/3", samples) + report_latency("Group.members/3", samples) + after + stop_spawned_processes(pids) + end end) end end @@ -96,23 +109,26 @@ defmodule GroupBench.Local do header("3. Register Throughput (shard scaling)") n = 10_000 - shard_counts = Enum.uniq([1, 2, 4, @default_shards]) for {cluster_label, cluster_opt} <- clusters() do subheader("cluster: #{cluster_label}") - for shards <- shard_counts do + for shards <- @shard_counts do with_group([name: @name, shards: shards], fn -> maybe_connect_cluster(cluster_opt) - {wall_us, _} = + {wall_us, pids} = time_us(fn -> register_from_spawned_processes(n, fn i -> Group.register(@name, "reg-#{i}", %{}, cluster_opts(cluster_opt)) end) end) - report_throughput("shards=#{shards}", n, wall_us) + try do + report_throughput("shards=#{shards}", n, wall_us) + after + stop_spawned_processes(pids) + end end) end end @@ -151,23 +167,26 @@ defmodule GroupBench.Local do header("5. Join Throughput (shard scaling)") n = 10_000 - shard_counts = Enum.uniq([1, 2, 4, @default_shards]) for {cluster_label, cluster_opt} <- clusters() do subheader("cluster: #{cluster_label}") - for shards <- shard_counts do + for shards <- @shard_counts do with_group([name: @name, shards: shards], fn -> maybe_connect_cluster(cluster_opt) - {wall_us, _} = + {wall_us, pids} = time_us(fn -> register_from_spawned_processes(n, fn i -> Group.join(@name, "join-group-#{rem(i, 100)}", %{}, cluster_opts(cluster_opt)) end) end) - report_throughput("shards=#{shards}", n, wall_us) + try do + report_throughput("shards=#{shards}", n, wall_us) + after + stop_spawned_processes(pids) + end end) end end @@ -186,18 +205,25 @@ defmodule GroupBench.Local do with_group([name: @name, shards: @default_shards], fn -> maybe_connect_cluster(cluster_opt) :ok = Group.monitor(@name, :all, cluster_opts(cluster_opt)) + drain_stale_group_events() - {wall_us, _} = + {wall_us, pids} = time_us(fn -> - register_from_spawned_processes(n, fn i -> - Group.register(@name, "mon-#{i}", %{}, cluster_opts(cluster_opt)) - end) + pids = + register_from_spawned_processes(n, fn i -> + Group.register(@name, "mon-#{i}", %{}, cluster_opts(cluster_opt)) + end) # drain all N events drain_events(n) + pids end) - report_throughput("events (register → receive)", n, wall_us) + try do + report_throughput("events (register → receive)", n, wall_us) + after + stop_spawned_processes(pids) + end end) end end @@ -243,6 +269,19 @@ defmodule GroupBench.Local do pids end + defp stop_spawned_processes(pids) do + refs = Enum.map(pids, &{&1, Process.monitor(&1)}) + Enum.each(pids, &Process.exit(&1, :kill)) + + Enum.each(refs, fn {pid, ref} -> + receive do + {:DOWN, ^ref, :process, ^pid, _reason} -> :ok + after + 5_000 -> raise "Timed out stopping #{inspect(pid)}" + end + end) + end + defp drain_events(0), do: :ok defp drain_events(remaining) do @@ -254,4 +293,12 @@ defmodule GroupBench.Local do 5_000 -> IO.puts(" WARNING: timed out waiting for events, #{remaining} remaining") end end + + defp drain_stale_group_events do + receive do + {:group, _events, _info} -> drain_stale_group_events() + after + 0 -> :ok + end + end end diff --git a/priv/bench/lib/group_bench/replica.ex b/priv/bench/lib/group_bench/replica.ex index 22df3c0..0bace05 100644 --- a/priv/bench/lib/group_bench/replica.ex +++ b/priv/bench/lib/group_bench/replica.ex @@ -73,6 +73,25 @@ defmodule GroupBench.Replica do end) end + @doc false + def registry_counts_by_shard(name) do + for shard <- 0..(Group.get_config(name).num_shards - 1) do + {shard, :ets.info(Group.Replica.Data.reg_by_key_table(name, shard), :size)} + end + end + + def registry_sample(name, limit \\ 10) do + shards = Group.get_config(name).num_shards + + 0..(shards - 1) + |> Enum.flat_map(fn shard -> + Group.Replica.Data.reg_by_key_table(name, shard) + |> :ets.tab2list() + |> Enum.take(limit) + end) + |> Enum.take(limit) + end + @doc """ Registers a single key from a spawned process. Returns after registration. """ @@ -577,23 +596,78 @@ defmodule GroupBench.Replica do pids = Enum.map(1..n, fn i -> spawn(fn -> - :ok = Group.register(name, "key", %{}, cluster: "#{prefix}#{i}") - send(parent, {:done, self()}) - Process.sleep(:infinity) + try do + :ok = Group.register(name, "key", %{}, cluster: "#{prefix}#{i}") + send(parent, {:done, self()}) + Process.sleep(:infinity) + catch + kind, reason -> + send( + parent, + {:failed, self(), kind, {reason, replica_process_diagnostics(name)}, + __STACKTRACE__} + ) + end end) end) Enum.each(pids, fn pid -> receive do - {:done, ^pid} -> :ok + {:done, ^pid} -> + :ok + + {:failed, ^pid, kind, reason, stacktrace} -> + :erlang.raise(kind, reason, stacktrace) after - 60_000 -> raise "Timed out waiting for bulk_register_per_cluster" + 60_000 -> + awaited_pid = + {pid, + Process.info(pid, [ + :status, + :current_function, + :message_queue_len, + :reductions + ])} + + registry_count = total_registry_count(name) + + raise "Timed out waiting for bulk_register_per_cluster: " <> + "replica=#{inspect(replica_process_diagnostics(name))} " <> + "awaited=#{inspect(awaited_pid)} registry_count=#{registry_count}" end end) pids end + @doc false + def replica_process_diagnostics(name) do + shards = + for shard <- 0..(Group.get_config(name).num_shards - 1) do + shard_pid = Process.whereis(Group.Replica.shard_name(name, shard)) + + {shard, + Process.info(shard_pid, [ + :status, + :current_function, + :message_queue_len, + :reductions + ])} + end + + data_pid = Process.whereis(Group.Replica.Data.data_name(name)) + + data = + Process.info(data_pid, [ + :status, + :current_function, + :message_queue_len, + :reductions + ]) + + %{shards: shards, data: data} + end + @doc """ Returns the number of clusters this node is a member of (via reverse index). """ @@ -601,6 +675,95 @@ defmodule GroupBench.Replica do length(Group.Replica.Data.my_clusters(name)) end + @doc """ + Returns the number of clusters currently associated with `target_node`. + + The many-cluster benchmark uses the reverse index instead of checking one + sentinel cluster: replica control messages may be reordered, so observing + the last submitted cluster does not prove that every earlier cluster has + converged. + """ + def cluster_count_for_node(name, target_node) do + name + |> Group.Replica.Data.node_clusters_table() + |> :ets.lookup(target_node) + |> length() + end + + @doc false + def cluster_control_revision(name) do + if function_exported?(Group.Replica.Data, :local_cluster_epoch_revision, 1) do + apply(Group.Replica.Data, :local_cluster_epoch_revision, [name]) + else + :legacy + end + end + + @doc false + def cluster_control_converged?(name, target_node, expected_cluster_count, source_revision) do + membership_converged? = + cluster_count_for_node(name, target_node) >= expected_cluster_count + + if source_revision == :legacy or + not function_exported?(Group.Replica.Data, :remote_view_generation, 3) do + membership_converged? + else + shards = Group.get_config(name).num_shards + data = Group.Replica.Data + generation = apply(data, :remote_generation, [name, target_node]) + revision = apply(data, :remote_cluster_epoch_revision, [name, target_node]) + epoch_table = apply(data, :remote_cluster_epochs_table, [name]) + + epoch_count = + :ets.select_count(epoch_table, [ + {{{target_node, :_}, :_}, [], [true]} + ]) + + shard_views_converged? = + Enum.all?(0..(shards - 1), fn shard -> + apply(data, :remote_view_generation, [name, shard, target_node]) == generation and + apply(data, :remote_view_cluster_epoch_revision, [name, shard, target_node]) == + source_revision and + apply(data, :remote_view_observed_revision, [name, shard, target_node]) == + source_revision + end) + + membership_converged? and not is_nil(generation) and revision == source_revision and + epoch_count >= expected_cluster_count - 1 and shard_views_converged? + end + end + + @doc false + def cluster_control_status(name, target_node) do + data = Group.Replica.Data + shards = Group.get_config(name).num_shards + epoch_table = data.remote_cluster_epochs_table(name) + + %{ + local_revision: data.local_cluster_epoch_revision(name), + local_epoch_count: :ets.info(data.local_cluster_epochs_table(name), :size), + membership_count: cluster_count_for_node(name, target_node), + remote_generation: data.remote_generation(name, target_node), + remote_revision: data.remote_cluster_epoch_revision(name, target_node), + remote_exact_revision: data.remote_cluster_epoch_exact_revision(name, target_node), + remote_observed: data.remote_cluster_epoch_observed_revision(name, target_node), + remote_epoch_count: + :ets.select_count(epoch_table, [ + {{{target_node, :_}, :_}, [], [true]} + ]), + authority_installs: data.remote_authority_install_count(name, target_node), + views: + for( + shard <- 0..(shards - 1), + do: + {shard, data.remote_view_generation(name, shard, target_node), + data.remote_view_cluster_epoch_revision(name, shard, target_node), + data.remote_view_observed_revision(name, shard, target_node)} + ), + replica: replica_process_diagnostics(name) + } + end + @doc """ Simulates a busy app worker: registers, joins groups, does lookups and dispatches, then some processes die and re-register. Returns final pids. diff --git a/test/README.md b/test/README.md index 58a6d51..6dc3dac 100644 --- a/test/README.md +++ b/test/README.md @@ -6,6 +6,7 @@ mix test # all tests mix test test/group_test.exs # local only mix test test/distributed_test.exs # distributed only +mix test test/replica_adversarial_test.exs # seeded transport chaos ``` ## Test files @@ -13,7 +14,8 @@ mix test test/distributed_test.exs # distributed only | File | What it tests | |------|---------------| | `group_test.exs` | Single-node: register/unregister, join/leave, members, monitor/demonitor, named clusters, concurrent operations | -| `distributed_test.exs` | Multi-node: replication, peer discovery, node disconnect cleanup, partition healing, conflict resolution, event ordering, rolling restarts | +| `distributed_test.exs` | Multi-node: replication, peer discovery, node disconnect cleanup, partition healing, conflict resolution, event ordering, rolling restarts, and adversarial replica-transport loss/busy/snapshot recovery | +| `replica_adversarial_test.exs` | Reproducible mixed-operation state machines: drops, busy returns, duplication, reordering, bounded delay, oplog pruning, conflicts, owner death, and named-cluster epoch churn, followed by exact convergence/dead-owner/internal-index checks | ## How distribution works @@ -194,6 +196,34 @@ TestCluster.start_group( The resolver uses "most recent wins" — keeps the registration with the higher timestamp. +### Replica transport fault injection + +`Group.TestReplicaTransport` implements the production transport behaviour but +can return `:busy`, drop selected frame types, duplicate or delay frames, and +capture frames for explicit stale-generation/epoch replay. Its `{:chaos, opts}` +mode is deterministic for a given frame, which makes failures reproducible. + +The distributed anti-entropy tests cover dropped creates and deletes, cursor +gaps, globally pruned multi-stream oplogs, exact snapshot fallback, malformed +authority, stale frame replay, lease expiry on a live VM, and multi-shard +generation recovery. They also restart a suspended data lane after deliberately +losing its cluster-close fence and require the lane to sweep the stale registry +and PG slices from shared authority. Authority topology tests suspend every +receiver shard and inspect the queued protocol: only shard 0 may receive/install +the full epoch snapshot, nonzero shards receive constant-size lane hellos, and +incremental opens stay on their matching shard. Separate tests suspend a +backlogged authority shard while other replica lanes continue converging and +deliver data before authority to prove rejection does not advance the cursor +and the same frame applies after authority repair. Concurrent snapshot tests +require every advertised revision to contain exactly that many unique named +epochs, and heartbeat tests prove observed revisions cannot advance the exact +authority marker. + +`Group.TestCluster.assert_replica_consistent/1` checks the +public dual indexes plus registry claim authority, oplog/order equivalence, and +contiguous retained stream ranges. Seeded tests additionally require every PID +retained as authority to still be alive after convergence. + ## Typical test patterns ### Basic replication test diff --git a/test/distributed_test.exs b/test/distributed_test.exs index e62a4d9..bf34406 100644 --- a/test/distributed_test.exs +++ b/test/distributed_test.exs @@ -555,10 +555,10 @@ defmodule Group.DistributedTest do timeout: 10_000 ) - # Custom resolvers own process lifecycle decisions. This resolver only - # picks a registry winner, so neither owner is terminated by Group. - assert TestCluster.rpc!(node_a, Process, :alive?, [pid_a]) + # The resolver selects the winner; each origin is responsible for + # retiring (and terminating) only its own losing owner. assert TestCluster.rpc!(node_b, Process, :alive?, [pid_b]) + refute TestCluster.rpc!(node_a, Process, :alive?, [pid_a]) end end @@ -1025,6 +1025,8 @@ defmodule Group.DistributedTest do cluster = "game" registry_key = "remote/registry" pg_key = "remote/pg" + retained_registry_key = "remote/registry/retained" + retained_pg_key = "remote/pg/retained" start_group_on_peers(peers, name: name, shards: 2) @@ -1045,13 +1047,47 @@ defmodule Group.DistributedTest do pg_pid = TestCluster.spawn_join(node_b, name, pg_key, %{from: :b}, cluster: cluster) + retained_registry_pid = + TestCluster.spawn_register_in_cluster( + node_b, + name, + retained_registry_key, + %{from: :b, retained: true}, + cluster + ) + + retained_pg_pid = + TestCluster.spawn_join( + node_b, + name, + retained_pg_key, + %{from: :b, retained: true}, + cluster: cluster + ) + TestCluster.assert_eventually(fn -> TestCluster.rpc!(node_a, Group, :lookup, [ name, registry_key, [cluster: cluster] ]) != nil and - TestCluster.rpc!(node_a, Group, :members, [name, pg_key, [cluster: cluster]]) != [] + TestCluster.rpc!(node_a, Group, :members, [name, pg_key, [cluster: cluster]]) != [] and + match?( + {^retained_registry_pid, _}, + TestCluster.rpc!(node_a, Group, :lookup, [ + name, + retained_registry_key, + [cluster: cluster] + ]) + ) and + match?( + [{^retained_pg_pid, _}], + TestCluster.rpc!(node_a, Group, :members, [ + name, + retained_pg_key, + [cluster: cluster] + ]) + ) end) assert :ok = TestCluster.rpc!(node_a, Group, :disconnect, [name, cluster]) @@ -1088,6 +1124,25 @@ defmodule Group.DistributedTest do ]) == nil assert TestCluster.rpc!(node_a, Group, :members, [name, pg_key, [cluster: cluster]]) == [] + + TestCluster.assert_eventually(fn -> + match?( + {^retained_registry_pid, %{from: :b, retained: true}}, + TestCluster.rpc!(node_a, Group, :lookup, [ + name, + retained_registry_key, + [cluster: cluster] + ]) + ) and + match?( + [{^retained_pg_pid, %{from: :b, retained: true}}], + TestCluster.rpc!(node_a, Group, :members, [ + name, + retained_pg_key, + [cluster: cluster] + ]) + ) + end) end test "local join does not overtake an earlier remote cluster_disconnect after replicated PG flush" do @@ -1778,21 +1833,30 @@ defmodule Group.DistributedTest do messages = TestCluster.shard_messages(node_b, name, 0) case Enum.filter(messages, fn - {:replicate_registry_batch, _ops} -> true - {:cluster_disconnect, ["game"], _remote_pid} -> true - _ -> false + {:group_replica_frame, _source, {:delta_batch, 1, _runs}} -> + true + + {:replica_cluster_close, _remote_pid, _generation, _revision, [{"game", _epoch}]} -> + true + + _ -> + false end) do [ - {:replicate_registry_batch, ops}, - {:cluster_disconnect, ["game"], _remote_pid} + {:group_replica_frame, _source, {:delta_batch, 1, runs}}, + {:replica_cluster_close, _remote_pid, _generation, _revision, [{"game", _epoch}]} ] -> - Enum.any?(ops, fn - {:register, "game", ^key, reg_pid, %{v: 1}, _time, _entry_node} - when reg_pid == pid -> - true - - _ -> - false + Enum.any?(runs, fn {_stream_id, _first_seq, records, _head} -> + Enum.any?(records, fn {_seq, mutations} -> + Enum.any?(mutations, fn + {:register, "game", ^key, reg_pid, %{v: 1}, _time, _entry_node} + when reg_pid == pid -> + true + + _ -> + false + end) + end) end) _ -> @@ -2354,18 +2418,55 @@ defmodule Group.DistributedTest do TestCluster.rpc!(node_a, Group, :disconnect, [name, dropped_cluster]) # A's registrations in org/10 should be gone everywhere - TestCluster.assert_eventually( - fn -> - Enum.all?(nodes, fn check_node -> - TestCluster.rpc!(check_node, Group, :lookup, [ - name, - "user/0_1", - [cluster: dropped_cluster] - ]) == nil - end) - end, - timeout: 10_000 - ) + try do + TestCluster.assert_eventually( + fn -> + Enum.all?(nodes, fn check_node -> + TestCluster.rpc!(check_node, Group, :lookup, [ + name, + "user/0_1", + [cluster: dropped_cluster] + ]) == nil + end) + end, + timeout: 10_000 + ) + rescue + error -> + diagnostics = + for check_node <- nodes do + lookup = + TestCluster.rpc!(check_node, Group, :lookup, [ + name, + "user/0_1", + [cluster: dropped_cluster] + ]) + + protocol = + TestCluster.rpc!( + check_node, + Group.TestCluster, + :replica_protocol_state, + [name] + ) + |> Enum.map(fn %{shard: shard, cursors: cursors} -> + relevant = + Enum.filter(cursors, fn {stream_id, _seq} -> + Group.Replica.Protocol.stream_origin(stream_id) == node_a and + Group.Replica.Protocol.stream_cluster(stream_id) == dropped_cluster + end) + + {shard, relevant} + end) + + {check_node, lookup, protocol} + end + + flunk( + "cluster disconnect convergence failed: #{Exception.message(error)} " <> + "diagnostics=#{inspect(diagnostics, limit: :infinity)}" + ) + end # B and C still see each other's org/10 data TestCluster.assert_eventually(fn -> @@ -3834,6 +3935,1388 @@ defmodule Group.DistributedTest do end end + describe "replica transport anti-entropy" do + @tag timeout: 60_000 + test "dropped tails and dropped deletes converge without orphaning registry or PG rows" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_drop_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 1_000 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_b in TestCluster.rpc!(node_a, Group, :nodes, [name]) + end) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + reg_key = "anti-entropy/dropped-register" + pg_key = "anti-entropy/dropped-join" + reg_pid = TestCluster.spawn_register(node_a, name, reg_key, %{owner: :a}) + pg_pid = TestCluster.spawn_join(node_a, name, pg_key, %{owner: :a}) + Process.sleep(100) + assert TestCluster.rpc!(node_b, Group, :lookup, [name, reg_key]) == nil + assert TestCluster.rpc!(node_b, Group, :members, [name, pg_key]) == [] + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :pass]) + + TestCluster.assert_eventually(fn -> + match?({^reg_pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, reg_key])) and + match?([{^pg_pid, _}], TestCluster.rpc!(node_b, Group, :members, [name, pg_key])) + end) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + TestCluster.rpc!(node_a, Process, :exit, [reg_pid, :kill]) + TestCluster.rpc!(node_a, Process, :exit, [pg_pid, :kill]) + TestCluster.flush_shards(node_a, name) + + assert match?({^reg_pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, reg_key])) + assert match?([{^pg_pid, _}], TestCluster.rpc!(node_b, Group, :members, [name, pg_key])) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :pass]) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(node_b, Group, :lookup, [name, reg_key]) == nil and + TestCluster.rpc!(node_b, Group, :members, [name, pg_key]) == [] + end) + end + + @tag timeout: 60_000 + test "busy sends recover from the advertised stream head" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_busy_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 1_000 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_b in TestCluster.rpc!(node_a, Group, :nodes, [name]) + end) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :busy]) + key = "anti-entropy/busy" + pid = TestCluster.spawn_register(node_a, name, key, %{}) + Process.sleep(100) + assert TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :pass]) + + TestCluster.assert_eventually(fn -> + match?({^pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + end + + @tag timeout: 60_000 + test "a pruned gap falls back to an exact origin snapshot and removes stale rows" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_snapshot_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 1_000, + replicated_oplog_max_entries: 2 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_b in TestCluster.rpc!(node_a, Group, :nodes, [name]) + end) + + stale_reg_key = "anti-entropy/snapshot/stale-reg" + stale_pg_key = "anti-entropy/snapshot/stale-pg" + stale_reg_pid = TestCluster.spawn_register(node_a, name, stale_reg_key, %{}) + stale_pg_pid = TestCluster.spawn_join(node_a, name, stale_pg_key, %{}) + + TestCluster.assert_eventually(fn -> + match?( + {^stale_reg_pid, _}, + TestCluster.rpc!(node_b, Group, :lookup, [name, stale_reg_key]) + ) and + match?( + [{^stale_pg_pid, _}], + TestCluster.rpc!(node_b, Group, :members, [name, stale_pg_key]) + ) + end) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + TestCluster.rpc!(node_a, Process, :exit, [stale_reg_pid, :kill]) + TestCluster.rpc!(node_a, Process, :exit, [stale_pg_pid, :kill]) + + fresh_keys = for i <- 1..6, do: "anti-entropy/snapshot/fresh-#{i}" + fresh_pids = for key <- fresh_keys, do: TestCluster.spawn_register(node_a, name, key, %{}) + TestCluster.flush_shards(node_a, name) + + [{_stream_id, floor, head}] = + TestCluster.rpc!(node_a, Group.Replica.Data, :replica_stream_heads, [name, 0]) + + assert floor > 3 + assert head >= 10 + + assert match?( + {^stale_reg_pid, _}, + TestCluster.rpc!(node_b, Group, :lookup, [name, stale_reg_key]) + ) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :pass]) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(node_b, Group, :lookup, [name, stale_reg_key]) == nil and + TestCluster.rpc!(node_b, Group, :members, [name, stale_pg_key]) == [] and + Enum.zip(fresh_keys, fresh_pids) + |> Enum.all?(fn {key, pid} -> + match?({^pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + end) + end + + @tag timeout: 60_000 + test "the control lease removes a stopped Group on a still-connected node and probes recovery" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_lease_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 150 + ] + + start_group_on_peers(peers, opts) + key = "anti-entropy/lease/stale" + stale_pid = TestCluster.spawn_register(node_a, name, key, %{}) + + TestCluster.assert_eventually(fn -> + match?({^stale_pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + + supervisor = TestCluster.rpc!(node_a, Process, :whereis, [:"#{name}_group_sup"]) + :ok = TestCluster.rpc!(node_a, Supervisor, :stop, [supervisor, :normal, 5_000]) + assert TestCluster.rpc!(node_b, Node, :ping, [node_a]) == :pong + + TestCluster.assert_eventually( + fn -> + TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil and + node_a not in TestCluster.rpc!(node_b, Group, :nodes, [name]) + end, + timeout: 5_000 + ) + + {:ok, _pid} = TestCluster.start_group(node_a, opts) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name]) + end) + + recovered_key = "anti-entropy/lease/recovered" + recovered_pid = TestCluster.spawn_register(node_a, name, recovered_key, %{}) + + TestCluster.assert_eventually(fn -> + match?( + {^recovered_pid, _}, + TestCluster.rpc!(node_b, Group, :lookup, [name, recovered_key]) + ) + end) + end + + @tag timeout: 60_000 + test "a restarted data lane sweeps a cluster close missed while it was down" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_lane_restart_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 2, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 1_000 + ] + + start_group_on_peers(peers, opts) + :ok = TestCluster.rpc!(node_a, Group, :connect, [name, "game"]) + :ok = TestCluster.rpc!(node_b, Group, :connect, [name, "game"]) + + reg_key = + Enum.find(1..1_000, fn suffix -> + Group.Replica.shard_index_for("game", "lane-reg-#{suffix}", 2) == 1 + end) + |> then(&"lane-reg-#{&1}") + + pg_key = + Enum.find(1..1_000, fn suffix -> + Group.Replica.shard_index_for("game", "lane-pg-#{suffix}", 2) == 1 + end) + |> then(&"lane-pg-#{&1}") + + reg_pid = TestCluster.spawn_register_in_cluster(node_a, name, reg_key, %{}, "game") + pg_pid = TestCluster.spawn_join(node_a, name, pg_key, %{}, cluster: "game") + + TestCluster.assert_eventually(fn -> + match?( + {^reg_pid, _}, + TestCluster.rpc!(node_b, Group, :lookup, [name, reg_key, [cluster: "game"]]) + ) and + match?( + [{^pg_pid, _}], + TestCluster.rpc!(node_b, Group, :members, [name, pg_key, [cluster: "game"]]) + ) + end) + + old_lane = TestCluster.rpc!(node_b, Process, :whereis, [shard_name(name, 1)]) + :ok = TestCluster.rpc!(node_b, :sys, :suspend, [old_lane]) + + :ok = TestCluster.rpc!(node_a, Group, :disconnect, [name, "game"]) + + TestCluster.assert_eventually(fn -> + is_nil( + TestCluster.rpc!(node_b, Group.Replica.Data, :remote_cluster_epoch, [ + name, + node_a, + "game" + ]) + ) + end) + + control = TestCluster.rpc!(node_b, Process, :whereis, [shard_name(name, 0)]) + _state = TestCluster.rpc!(node_b, :sys, :get_state, [control]) + + messages = TestCluster.rpc!(node_b, Process, :info, [old_lane, :messages]) + + assert {:messages, queued} = messages + + assert Enum.any?(queued, fn + {:replica_cluster_close_control_local, ^node_a, _generation, _revision, + [{"game", _epoch}]} -> + true + + _ -> + false + end) + + true = TestCluster.rpc!(node_b, Process, :exit, [old_lane, :kill]) + + TestCluster.assert_eventually(fn -> + case TestCluster.rpc!(node_b, Process, :whereis, [shard_name(name, 1)]) do + lane when is_pid(lane) -> lane != old_lane + _ -> false + end + end) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(node_b, Group, :lookup, [name, reg_key, [cluster: "game"]]) == nil and + TestCluster.rpc!(node_b, Group, :members, [name, pg_key, [cluster: "game"]]) == [] + end) + + assert :ok = + TestCluster.rpc!(node_b, Group.TestCluster, :assert_replica_consistent, [name]) + end + + @tag timeout: 60_000 + test "full authority is installed once on shard zero while incremental control stays sharded" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_authority_topology_#{System.unique_integer([:positive])}" + shards = 3 + + opts = [ + name: name, + shards: shards, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name]) + end) + + b_lanes = + for shard <- 0..(shards - 1) do + {shard, TestCluster.rpc!(node_b, Process, :whereis, [shard_name(name, shard)])} + end + + Enum.each(b_lanes, fn {_shard, pid} -> + :ok = TestCluster.rpc!(node_b, :sys, :suspend, [pid]) + end) + + installs_before = + TestCluster.rpc!( + node_b, + Group.Replica.Data, + :remote_authority_install_count, + [name, node_a] + ) + + [_authority_epoch] = + TestCluster.rpc!(node_a, Group.Replica.Data, :activate_local_clusters, [ + name, + ["authority-new"] + ]) + + remote_clusters = TestCluster.rpc!(node_b, Group.Replica.Data, :my_clusters, [name]) + + Enum.each(b_lanes, fn {shard, b_pid} -> + TestCluster.rpc!(node_a, :erlang, :send, [ + shard_name(name, shard), + {:peer_connect_ack, b_pid, shard, shards, remote_clusters} + ]) + end) + + TestCluster.assert_eventually(fn -> + Enum.all?(b_lanes, fn {shard, pid} -> + {:messages, messages} = TestCluster.rpc!(node_b, Process, :info, [pid, :messages]) + + if shard == 0 do + Enum.any?(messages, fn + {:replica_hello, remote_pid, _version, _generation, _revision, _epochs, _transport, + _descriptor} -> + node(remote_pid) == node_a + + _ -> + false + end) + else + Enum.any?(messages, fn + {:replica_lane_hello, remote_pid, _version, _generation, _revision, _transport, + _descriptor} -> + node(remote_pid) == node_a + + _ -> + false + end) + end + end) + end) + + mailboxes = + Map.new(b_lanes, fn {shard, pid} -> + {:messages, messages} = TestCluster.rpc!(node_b, Process, :info, [pid, :messages]) + {shard, messages} + end) + + full_authority_lanes = + for {shard, messages} <- mailboxes, + Enum.any?(messages, fn + {:replica_hello, remote_pid, _version, _generation, _revision, _epochs, _transport, + _descriptor} -> + node(remote_pid) == node_a + + _ -> + false + end), + do: shard + + assert full_authority_lanes == [0] + + assert Enum.any?(mailboxes[0], fn + {:replica_hello, remote_pid, _version, _generation, revision, epochs, _transport, + _descriptor} -> + node(remote_pid) == node_a and + revision == Enum.count(epochs, &(not is_nil(elem(&1, 0)))) + + _ -> + false + end) + + for shard <- 1..(shards - 1) do + assert Enum.any?(mailboxes[shard], fn + {:replica_lane_hello, remote_pid, _version, _generation, _revision, _transport, + _descriptor} -> + node(remote_pid) == node_a + + _ -> + false + end) + end + + [epoch] = + TestCluster.rpc!(node_a, Group.Replica.Data, :activate_local_clusters, [ + name, + ["lane-open"] + ]) + + TestCluster.rpc!(node_a, Group.Replica.Data, :add_cluster_node, [ + name, + ["lane-open"], + node_a + ]) + + assert :ok = + TestCluster.rpc!(node_a, Group.Replica, :local_request, [ + shard_name(name, 2), + {:cluster_connect, ["lane-open"], [epoch]}, + 5_000 + ]) + + TestCluster.assert_eventually(fn -> + {:messages, messages} = + TestCluster.rpc!(node_b, Process, :info, [Map.fetch!(Map.new(b_lanes), 2), :messages]) + + Enum.any?(messages, fn + {:replica_cluster_open, remote_pid, _generation, _revision, [^epoch]} -> + node(remote_pid) == node_a + + _ -> + false + end) + end) + + {:messages, control_messages} = + TestCluster.rpc!(node_b, Process, :info, [Map.fetch!(Map.new(b_lanes), 0), :messages]) + + refute Enum.any?(control_messages, fn + {:replica_cluster_open, remote_pid, _generation, _revision, [^epoch]} -> + node(remote_pid) == node_a + + _ -> + false + end) + + {0, b_control} = List.keyfind!(b_lanes, 0, 0) + :ok = TestCluster.rpc!(node_b, :sys, :resume, [b_control]) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!( + node_b, + Group.Replica.Data, + :remote_authority_install_count, + [name, node_a] + ) == installs_before + 1 + end) + + duplicate_full = + Enum.find(mailboxes[0], fn + {:replica_hello, remote_pid, _version, _generation, _revision, _epochs, _transport, + _descriptor} -> + node(remote_pid) == node_a + + _ -> + false + end) + + {:replica_hello, _remote_pid, _version, _generation, full_revision, _epochs, _transport, + _descriptor} = duplicate_full + + Enum.each(1..128, fn _ -> send(b_control, duplicate_full) end) + drain_ref = make_ref() + send(b_control, {:group_dispatch, [self()], {:authority_duplicates_drained, drain_ref}}) + assert_receive {:authority_duplicates_drained, ^drain_ref}, 5_000 + + assert TestCluster.rpc!( + node_b, + Group.Replica.Data, + :remote_authority_install_count, + [name, node_a] + ) == installs_before + 1 + + Enum.each(b_lanes, fn + {0, _pid} -> :ok + {_shard, pid} -> :ok = TestCluster.rpc!(node_b, :sys, :resume, [pid]) + end) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(node_b, Group.Replica.Data, :remote_cluster_epoch, [ + name, + node_a, + "lane-open" + ]) == elem(epoch, 1) + end) + + latest_revision = + TestCluster.rpc!(node_a, Group.Replica.Data, :local_cluster_epoch_revision, [name]) + + a_lane = TestCluster.rpc!(node_a, Process, :whereis, [shard_name(name, 2)]) + + TestCluster.rpc!(node_b, :erlang, :send, [ + shard_name(name, 2), + {:replica_heartbeat, a_lane, Group.Replica.Protocol.version(), + TestCluster.rpc!(node_a, Group.Replica.Data, :generation, [name]), latest_revision} + ]) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(node_b, Group.Replica.Data, :remote_view_observed_revision, [ + name, + 2, + node_a + ]) == latest_revision + end) + + assert TestCluster.rpc!( + node_b, + Group.Replica.Data, + :remote_cluster_epoch_exact_revision, + [name, node_a] + ) == full_revision + + assert TestCluster.rpc!( + node_b, + Group.Replica.Data, + :remote_view_cluster_epoch_revision, + [name, 2, node_a] + ) == full_revision + end + + @tag timeout: 60_000 + test "a backlogged authority shard cannot block independent replica lanes" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_authority_backlog_#{System.unique_integer([:positive])}" + shards = 3 + + opts = [ + name: name, + shards: shards, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name]) + end) + + b_control = TestCluster.rpc!(node_b, Process, :whereis, [shard_name(name, 0)]) + a_control = TestCluster.rpc!(node_a, Process, :whereis, [shard_name(name, 0)]) + :ok = TestCluster.rpc!(node_b, :sys, :suspend, [b_control]) + + TestCluster.rpc!(node_a, :erlang, :send, [ + shard_name(name, 0), + {:peer_connect_ack, b_control, 0, shards, [nil]} + ]) + + TestCluster.assert_eventually(fn -> + {:messages, messages} = + TestCluster.rpc!(node_b, Process, :info, [b_control, :messages]) + + Enum.any?(messages, fn + {:replica_hello, ^a_control, _version, _generation, _revision, _epochs, _transport, + _descriptor} -> + true + + _ -> + false + end) + end) + + [reg_key] = keys_for_shard(nil, "authority-backlog/reg", shards, 1, 1) + [pg_key] = keys_for_shard(nil, "authority-backlog/pg", shards, 2, 1) + reg_pid = TestCluster.spawn_register(node_a, name, reg_key, %{lane: 1}) + pg_pid = TestCluster.spawn_join(node_a, name, pg_key, %{lane: 2}) + + TestCluster.assert_eventually(fn -> + match?({^reg_pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, reg_key])) and + match?([{^pg_pid, _}], TestCluster.rpc!(node_b, Group, :members, [name, pg_key])) + end) + + :ok = TestCluster.rpc!(node_b, :sys, :resume, [b_control]) + end + + @tag timeout: 60_000 + test "data arriving without authority is rejected without cursor advance and repairs later" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_authority_before_data_#{System.unique_integer([:positive])}" + shards = 2 + + opts = [ + name: name, + shards: shards, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name]) + end) + + :ok = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [ + name, + {:capture_drop, [:delta_batch]} + ]) + + [key] = keys_for_shard(nil, "authority-before-data", shards, 1, 1) + pid = TestCluster.spawn_register(node_a, name, key, %{valid: true}) + TestCluster.flush_shards(node_a, name) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :captured, [name]) != [] + end) + + {_target, 1, frame} = + node_a + |> TestCluster.rpc!(Group.TestReplicaTransport, :captured, [name]) + |> Enum.find(fn {_target, shard, _frame} -> shard == 1 end) + + stream_id = TestCluster.rpc!(node_a, Group.Replica.Data, :local_stream_id, [name, 1, nil]) + generation = TestCluster.rpc!(node_a, Group.Replica.Data, :generation, [name]) + + revision = + TestCluster.rpc!(node_a, Group.Replica.Data, :local_cluster_epoch_revision, [name]) + + :ok = + TestCluster.rpc!(node_b, Group.Replica.Data, :delete_remote_replica_info, [ + name, + 0, + node_a + ]) + + :ok = + TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + name, + node_a, + 1, + frame + ]) + + TestCluster.flush_shards(node_b, name) + assert TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + + assert TestCluster.rpc!(node_b, Group.Replica.Data, :replica_cursor, [ + name, + 1, + stream_id + ]) == 0 + + a_lane = TestCluster.rpc!(node_a, Process, :whereis, [shard_name(name, 1)]) + + TestCluster.rpc!(node_b, :erlang, :send, [ + shard_name(name, 1), + {:replica_heartbeat, a_lane, Group.Replica.Protocol.version(), generation, revision} + ]) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(node_b, Group.Replica.Data, :remote_generation, [name, node_a]) == + generation + end) + + :ok = + TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + name, + node_a, + 1, + frame + ]) + + TestCluster.assert_eventually(fn -> + match?({^pid, %{valid: true}}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + + assert :ok = + TestCluster.rpc!(node_b, Group.TestCluster, :assert_replica_consistent, [name]) + end + + @tag timeout: 60_000 + test "a delayed cluster close from an older epoch revision cannot undo a reconnect" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_epoch_fence_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(peers, opts) + TestCluster.rpc!(node_a, Group, :connect, [name, "game"]) + TestCluster.rpc!(node_b, Group, :connect, [name, "game"]) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name, "game"]) + end) + + generation = TestCluster.rpc!(node_a, Group.Replica.Data, :generation, [name]) + + old_revision = + TestCluster.rpc!(node_a, Group.Replica.Data, :local_cluster_epoch_revision, [name]) + + old_epoch = + TestCluster.rpc!(node_a, Group.Replica.Data, :local_cluster_epoch, [name, "game"]) + + TestCluster.rpc!(node_a, Group, :disconnect, [name, "game"]) + TestCluster.rpc!(node_a, Group, :connect, [name, "game"]) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name, "game"]) + end) + + current_revision = + TestCluster.rpc!(node_a, Group.Replica.Data, :local_cluster_epoch_revision, [name]) + + assert current_revision > old_revision + + key = "anti-entropy/epoch-fence/current" + pid = TestCluster.spawn_register_in_cluster(node_a, name, key, %{}, "game") + + TestCluster.assert_eventually(fn -> + match?( + {^pid, _}, + TestCluster.rpc!(node_b, Group, :lookup, [name, key, [cluster: "game"]]) + ) + end) + + remote_pid = TestCluster.rpc!(node_a, Process, :whereis, [shard_name(name, 0)]) + + TestCluster.rpc!(node_a, :erlang, :send, [ + {shard_name(name, 0), node_b}, + {:replica_cluster_close, remote_pid, generation, old_revision, [{"game", old_epoch}]} + ]) + + Process.sleep(100) + TestCluster.flush_shards(node_b, name) + + assert node_a in TestCluster.rpc!(node_b, Group, :nodes, [name, "game"]) + + assert match?( + {^pid, _}, + TestCluster.rpc!(node_b, Group, :lookup, [name, key, [cluster: "game"]]) + ) + end + + @tag timeout: 60_000 + test "duplicate and reordered replica frames are idempotent and emit one lifecycle event" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_duplicate_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 1_000 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_b in TestCluster.rpc!(node_a, Group, :nodes, [name]) + end) + + forwarder = TestCluster.spawn_monitor_forwarder(node_b, name, :all, self()) + assert_receive {:monitor_ready, ^forwarder}, 5_000 + + :ok = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [ + name, + {:chaos, [drop_every: 0, duplicate_every: 2, max_delay: 75]} + ]) + + key = "anti-entropy/duplicate" + pid = TestCluster.spawn_register(node_a, name, key, %{version: 1}) + + assert_receive {:got_event, %Group.Event{type: :registered, key: ^key, pid: ^pid}}, 5_000 + + TestCluster.assert_eventually(fn -> + match?({^pid, %{version: 1}}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + + Process.sleep(250) + + refute_received {:got_event, %Group.Event{type: :registered, key: ^key, pid: ^pid}} + + assert :ok = + TestCluster.rpc!(node_b, Group.TestCluster, :assert_replica_consistent, [name]) + end + + @tag timeout: 60_000 + test "captured data from old origin generations and cluster epochs cannot resurrect rows" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_stale_frames_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 150 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_b in TestCluster.rpc!(node_a, Group, :nodes, [name]) + end) + + :ok = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [ + name, + {:capture_drop, [:delta_batch]} + ]) + + generation_key = "anti-entropy/stale-generation" + old_generation_pid = TestCluster.spawn_register(node_a, name, generation_key, %{old: true}) + TestCluster.flush_shards(node_a, name) + + generation_frames = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :captured, [name]) + + assert generation_frames != [] + assert TestCluster.rpc!(node_b, Group, :lookup, [name, generation_key]) == nil + + supervisor = TestCluster.rpc!(node_a, Process, :whereis, [:"#{name}_group_sup"]) + :ok = TestCluster.rpc!(node_a, Supervisor, :stop, [supervisor, :normal, 5_000]) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :pass]) + + TestCluster.assert_eventually( + fn -> node_a not in TestCluster.rpc!(node_b, Group, :nodes, [name]) end, + timeout: 5_000 + ) + + {:ok, _pid} = TestCluster.start_group(node_a, opts) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name]) + end) + + Enum.each(generation_frames, fn {_target, shard, frame} -> + :ok = + TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + name, + node_a, + shard, + frame + ]) + end) + + TestCluster.flush_shards(node_b, name) + assert TestCluster.rpc!(node_b, Group, :lookup, [name, generation_key]) == nil + + new_generation_pid = + TestCluster.spawn_register(node_a, name, generation_key, %{old: false}) + + TestCluster.assert_eventually(fn -> + match?( + {^new_generation_pid, %{old: false}}, + TestCluster.rpc!(node_b, Group, :lookup, [name, generation_key]) + ) + end) + + TestCluster.rpc!(node_a, Group, :connect, [name, "game"]) + TestCluster.rpc!(node_b, Group, :connect, [name, "game"]) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name, "game"]) + end) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :clear_captured, [name]) + + :ok = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [ + name, + {:capture_drop, [:delta_batch]} + ]) + + epoch_key = "anti-entropy/stale-epoch" + + old_epoch_pid = + TestCluster.spawn_register_in_cluster(node_a, name, epoch_key, %{old: true}, "game") + + TestCluster.flush_shards(node_a, name) + epoch_frames = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :captured, [name]) + assert epoch_frames != [] + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :pass]) + TestCluster.rpc!(node_a, Group, :disconnect, [name, "game"]) + TestCluster.rpc!(node_a, Group, :connect, [name, "game"]) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name, "game"]) + end) + + Enum.each(epoch_frames, fn {_target, shard, frame} -> + :ok = + TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + name, + node_a, + shard, + frame + ]) + end) + + TestCluster.flush_shards(node_b, name) + + assert TestCluster.rpc!(node_b, Group, :lookup, [ + name, + epoch_key, + [cluster: "game"] + ]) == nil + + new_epoch_pid = + TestCluster.spawn_register_in_cluster(node_a, name, epoch_key, %{old: false}, "game") + + TestCluster.assert_eventually(fn -> + match?( + {^new_epoch_pid, %{old: false}}, + TestCluster.rpc!(node_b, Group, :lookup, [ + name, + epoch_key, + [cluster: "game"] + ]) + ) + end) + + TestCluster.rpc!(node_a, Process, :exit, [old_generation_pid, :kill]) + TestCluster.rpc!(node_a, Process, :exit, [old_epoch_pid, :kill]) + + assert :ok = + TestCluster.rpc!(node_b, Group.TestCluster, :assert_replica_consistent, [name]) + end + + @tag timeout: 60_000 + test "global shard pruning repairs a cold stream without sending or deleting other origins" do + peers = TestCluster.start_peers(3) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}, {_, node_c}] = peers + name = :"anti_entropy_multistream_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 1_000, + replicated_oplog_max_entries: 4 + ] + + start_group_on_peers(peers, opts) + + for target <- [node_a, node_b, node_c] do + TestCluster.assert_eventually(fn -> + length(TestCluster.rpc!(target, Group, :nodes, [name])) == 2 + end) + + TestCluster.rpc!(target, Group, :connect, [name, ["cold", "hot"]]) + end + + TestCluster.assert_eventually( + fn -> + length(TestCluster.rpc!(node_c, Group, :nodes, [name, "cold"])) == 3 and + length(TestCluster.rpc!(node_c, Group, :nodes, [name, "hot"])) == 3 + end, + timeout: 10_000 + ) + + b_key = "anti-entropy/other-origin" + b_group = "anti-entropy/other-origin-pg" + b_pid = TestCluster.spawn_register(node_b, name, b_key, %{origin: :b}) + b_pg_pid = TestCluster.spawn_join(node_b, name, b_group, %{origin: :b}) + + stale_key = "anti-entropy/cold/stale" + stale_group = "anti-entropy/cold/stale-pg" + + stale_pid = + TestCluster.spawn_register_in_cluster(node_a, name, stale_key, %{origin: :a}, "cold") + + stale_pg_pid = + TestCluster.spawn_join_in_cluster(node_a, name, stale_group, %{origin: :a}, "cold") + + TestCluster.assert_eventually(fn -> + match?({^b_pid, _}, TestCluster.rpc!(node_c, Group, :lookup, [name, b_key])) and + match?([{^b_pg_pid, _}], TestCluster.rpc!(node_c, Group, :members, [name, b_group])) and + match?( + {^stale_pid, _}, + TestCluster.rpc!(node_c, Group, :lookup, [name, stale_key, [cluster: "cold"]]) + ) and + match?( + [{^stale_pg_pid, _}], + TestCluster.rpc!(node_c, Group, :members, [name, stale_group, [cluster: "cold"]]) + ) + end) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + TestCluster.rpc!(node_a, Process, :exit, [stale_pid, :kill]) + TestCluster.rpc!(node_a, Process, :exit, [stale_pg_pid, :kill]) + + fresh = + for i <- 1..12 do + key = "anti-entropy/hot/#{i}" + + {key, TestCluster.spawn_register_in_cluster(node_a, name, key, %{i: i}, "hot")} + end + + TestCluster.flush_shards(node_a, name) + + {_stream_id, floor, _head} = + node_a + |> TestCluster.rpc!(Group.Replica.Data, :replica_stream_heads, [name, 0]) + |> Enum.find(fn {stream_id, _floor, _head} -> + Group.Replica.Protocol.stream_cluster(stream_id) == "cold" + end) + + assert floor > 2 + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :pass]) + + TestCluster.assert_eventually( + fn -> + TestCluster.rpc!(node_c, Group, :lookup, [name, stale_key, [cluster: "cold"]]) == + nil and + TestCluster.rpc!(node_c, Group, :members, [ + name, + stale_group, + [cluster: "cold"] + ]) == [] and + match?({^b_pid, _}, TestCluster.rpc!(node_c, Group, :lookup, [name, b_key])) and + match?( + [{^b_pg_pid, _}], + TestCluster.rpc!(node_c, Group, :members, [name, b_group]) + ) and + Enum.all?(fresh, fn {key, pid} -> + match?( + {^pid, _}, + TestCluster.rpc!(node_c, Group, :lookup, [name, key, [cluster: "hot"]]) + ) + end) + end, + timeout: 10_000 + ) + + assert :ok = + TestCluster.rpc!(node_c, Group.TestCluster, :assert_replica_consistent, [name]) + end + + @tag timeout: 60_000 + test "invalid authority and a misattributed frame cannot poison a stream cursor" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_authority_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 1_000 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_b in TestCluster.rpc!(node_a, Group, :nodes, [name]) + end) + + stream_id = + TestCluster.rpc!(node_a, Group.Replica.Data, :local_stream_id, [name, 0, nil]) + + foreign_pid = + TestCluster.spawn_register(node_b, name, "anti-entropy/authority/foreign-owner", %{}) + + :ok = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [ + name, + {:capture_drop, [:delta_batch]} + ]) + + key = "anti-entropy/authority/legitimate" + legitimate_pid = TestCluster.spawn_register(node_a, name, key, %{valid: true}) + TestCluster.flush_shards(node_a, name) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :captured, [name]) != [] + end) + + [{_target, 0, legitimate_frame} | _] = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :captured, [name]) + + invalid_key = "anti-entropy/authority/forged" + + invalid_frame = + {:delta_batch, Group.Replica.Protocol.version(), + [ + {stream_id, 1, + [ + {1, + [ + {:register, nil, invalid_key, foreign_pid, %{forged: true}, System.system_time(), + node_b} + ]} + ], 1} + ]} + + :ok = + TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + name, + node_a, + 0, + invalid_frame + ]) + + TestCluster.flush_shards(node_b, name) + assert TestCluster.rpc!(node_b, Group, :lookup, [name, invalid_key]) == nil + + assert TestCluster.rpc!(node_b, Group.Replica.Data, :replica_cursor, [ + name, + 0, + stream_id + ]) == 0 + + :ok = + TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + name, + node_b, + 0, + legitimate_frame + ]) + + TestCluster.flush_shards(node_b, name) + assert TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + + assert TestCluster.rpc!(node_b, Group.Replica.Data, :replica_cursor, [ + name, + 0, + stream_id + ]) == 0 + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :pass]) + + TestCluster.assert_eventually(fn -> + match?( + {^legitimate_pid, %{valid: true}}, + TestCluster.rpc!(node_b, Group, :lookup, [name, key]) + ) + end) + + assert :ok = + TestCluster.rpc!(node_b, Group.TestCluster, :assert_replica_consistent, [name]) + end + + @tag timeout: 60_000 + test "lease expiry purges every shard before accepting a fresh generation" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_multishard_lease_#{System.unique_integer([:positive])}" + shards = 3 + + opts = [ + name: name, + shards: shards, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 2, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 150 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_b in TestCluster.rpc!(node_a, Group, :nodes, [name]) + end) + + stale = + for shard <- 0..(shards - 1) do + [reg_key] = keys_for_shard(nil, "anti-entropy/lease/reg/s#{shard}", shards, shard, 1) + [pg_key] = keys_for_shard(nil, "anti-entropy/lease/pg/s#{shard}", shards, shard, 1) + + { + reg_key, + TestCluster.spawn_register(node_a, name, reg_key, %{generation: :old, shard: shard}), + pg_key, + TestCluster.spawn_join(node_a, name, pg_key, %{generation: :old, shard: shard}) + } + end + + TestCluster.assert_eventually( + fn -> + Enum.all?(stale, fn {reg_key, reg_pid, pg_key, pg_pid} -> + match?({^reg_pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, reg_key])) and + match?([{^pg_pid, _}], TestCluster.rpc!(node_b, Group, :members, [name, pg_key])) + end) + end, + timeout: 10_000 + ) + + old_generation = TestCluster.rpc!(node_a, Group.Replica.Data, :generation, [name]) + supervisor = TestCluster.rpc!(node_a, Process, :whereis, [:"#{name}_group_sup"]) + :ok = TestCluster.rpc!(node_a, Supervisor, :stop, [supervisor, :normal, 5_000]) + assert TestCluster.rpc!(node_b, Node, :ping, [node_a]) == :pong + + TestCluster.assert_eventually( + fn -> + node_a not in TestCluster.rpc!(node_b, Group, :nodes, [name]) and + Enum.all?(stale, fn {reg_key, _reg_pid, pg_key, _pg_pid} -> + TestCluster.rpc!(node_b, Group, :lookup, [name, reg_key]) == nil and + TestCluster.rpc!(node_b, Group, :members, [name, pg_key]) == [] + end) + end, + timeout: 5_000 + ) + + assert :ok = + TestCluster.rpc!(node_b, Group.TestCluster, :assert_replica_consistent, [name]) + + {:ok, _pid} = TestCluster.start_group(node_a, opts) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name]) + end) + + new_generation = TestCluster.rpc!(node_a, Group.Replica.Data, :generation, [name]) + refute new_generation == old_generation + + fresh = + for shard <- 0..(shards - 1) do + [reg_key] = keys_for_shard(nil, "anti-entropy/fresh/reg/s#{shard}", shards, shard, 1) + [pg_key] = keys_for_shard(nil, "anti-entropy/fresh/pg/s#{shard}", shards, shard, 1) + + { + reg_key, + TestCluster.spawn_register(node_a, name, reg_key, %{generation: :new, shard: shard}), + pg_key, + TestCluster.spawn_join(node_a, name, pg_key, %{generation: :new, shard: shard}) + } + end + + TestCluster.assert_eventually( + fn -> + Enum.all?(fresh, fn {reg_key, reg_pid, pg_key, pg_pid} -> + match?({^reg_pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, reg_key])) and + match?([{^pg_pid, _}], TestCluster.rpc!(node_b, Group, :members, [name, pg_key])) + end) + end, + timeout: 10_000 + ) + + assert :ok = + TestCluster.rpc!(node_b, Group.TestCluster, :assert_replica_consistent, [name]) + end + + @tag timeout: 120_000 + test "concurrent many-cluster controls converge every revision before replica writes" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_many_controls_#{System.unique_integer([:positive])}" + clusters = for i <- 1..512, do: "tenant/#{i}" + + opts = [ + name: name, + shards: 4, + replicated_sender_buffer_size: 8, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 2_000 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_b in TestCluster.rpc!(node_a, Group, :nodes, [name]) + end) + + tasks = + for node <- [node_a, node_b] do + Task.async(fn -> TestCluster.connect_many_concurrently(node, name, clusters) end) + end + + assert [:ok, :ok] = Task.await_many(tasks, 60_000) + + TestCluster.assert_eventually( + fn -> + Enum.all?([node_a, node_b], fn node -> + expected = MapSet.new([nil | clusters]) + + actual = + TestCluster.rpc!(node, Group.Replica.Data, :my_clusters, [name]) + |> MapSet.new() + + MapSet.subset?(expected, actual) + end) and + Enum.all?(clusters, fn cluster -> + length(TestCluster.rpc!(node_a, Group, :nodes, [name, cluster])) == 2 and + length(TestCluster.rpc!(node_b, Group, :nodes, [name, cluster])) == 2 + end) + end, + timeout: 30_000, + interval: 100 + ) + + entries = + TestCluster.spawn_register_many_clusters( + node_a, + name, + clusters, + "anti-entropy/many-controls" + ) + + TestCluster.assert_eventually( + fn -> + TestCluster.rpc!(node_b, Group.TestCluster, :registry_entries_present?, [name, entries]) + end, + timeout: 30_000, + interval: 100 + ) + + for node <- [node_a, node_b] do + assert :ok = + TestCluster.rpc!(node, Group.TestCluster, :assert_replica_consistent, [name]) + end + end + end + # Helpers for event assertion tests defp flush_events do diff --git a/test/group_test.exs b/test/group_test.exs index 6bc6b80..7de75ae 100644 --- a/test/group_test.exs +++ b/test/group_test.exs @@ -2288,7 +2288,12 @@ defmodule GroupTest do assert_receive {:replicated_registry_buffer_flushed, ^shard_name}, 1_000 :sys.replace_state(shard_pid, fn state -> - %{state | remote_shards: Map.put(state.remote_shards, node(), self())} + %{ + state + | remote_shards: Map.put(state.remote_shards, node(), self()), + peer_last_seen: + Map.put(state.peer_last_seen, node(), System.monotonic_time(:millisecond)) + } end) :erlang.trace(shard_pid, true, [:call]) @@ -2306,10 +2311,16 @@ defmodule GroupTest do {:erlang, :send_nosuspend, [ {^shard_name, ^local_node}, - {:replicate_process_down_batch, _reg_entries, _pg_entries}, + {:group_replica_frame, ^local_node, {:delta_batch, 1, runs}}, [:noconnect] ]}}, 1_000 + + assert Enum.any?(runs, fn {_stream_id, _first_seq, records, _head} -> + Enum.any?(records, fn {_seq, mutations} -> + {:unregister, nil, key, owner, %{}, :killed} in mutations + end) + end) end test "process death batches multiple :left events for same-shard keys", %{name: name} do @@ -2694,7 +2705,7 @@ defmodule GroupTest do Group.Replica.Data.registry_lookup_by_pid(name, 0, new_pid) end - test "custom conflict resolver controls the losing registry owner's lifecycle" do + test "custom conflict resolver selects winner and Group terminates only the local loser" do key = "replicated-registry/custom-loser/#{System.unique_integer([:positive])}" name = @@ -2739,8 +2750,9 @@ defmodule GroupTest do Group.lookup(name, key) == {remote_pid, %{owner: :remote}} end) - refute_receive {:DOWN, ^owner_ref, :process, ^local_owner, _reason}, 50 - assert Process.alive?(local_owner) + assert_receive {:DOWN, ^owner_ref, :process, ^local_owner, + {:group_registry_conflict, ^key, %{owner: :remote}}}, + 1_000 end test "batched remote conflict keeps the staged local winner when later unregister arrives" do @@ -2869,6 +2881,156 @@ defmodule GroupTest do end end + describe "replica write-ahead journal" do + test "concurrent shards retain independent append order", %{name: name} do + named_cluster = "journal/append-order" + operations_per_shard = 100 + :ok = Group.connect(name, named_cluster) + + parent = self() + + owners = + for shard <- 0..3 do + nil_keys = + keys_for_shard(nil, "journal/append-order/nil/#{shard}", 4, shard, 50) + + named_keys = + keys_for_shard( + named_cluster, + "journal/append-order/named/#{shard}", + 4, + shard, + 50 + ) + + spawn(fn -> + nil_keys + |> Enum.zip(named_keys) + |> Enum.each(fn {nil_key, named_key} -> + :ok = Group.register(name, nil_key, %{}) + :ok = Group.register(name, named_key, %{}, cluster: named_cluster) + end) + + send(parent, {:append_order_complete, shard, self()}) + Process.sleep(:infinity) + end) + end + + on_exit(fn -> Enum.each(owners, &kill_if_alive/1) end) + + Enum.with_index(owners) + |> Enum.each(fn {owner, shard} -> + assert_receive {:append_order_complete, ^shard, ^owner}, 10_000 + end) + + metadata = Group.Replica.Data.replication_meta_table(name) + assert :ets.info(metadata, :write_concurrency) == :auto + + for shard <- 0..3 do + assert [{{:append_counter, shard}, operations_per_shard}] == + :ets.lookup(metadata, {:append_counter, shard}) + + order_rows = + name + |> Group.Replica.Data.replica_oplog_order_table(shard) + |> :ets.tab2list() + |> Enum.sort() + + assert Enum.map(order_rows, &elem(&1, 0)) == + Enum.to_list(1..operations_per_shard) + + assert Enum.all?(order_rows, fn {_append_id, stream_id, _seq} -> + Group.Replica.Protocol.stream_shard(stream_id) == shard + end) + + oplog_rows = + name + |> Group.Replica.Data.replica_oplog_table(shard) + |> :ets.tab2list() + |> MapSet.new(fn {{stream_id, seq}, append_id, _mutations} -> + {append_id, stream_id, seq} + end) + + assert MapSet.new(order_rows) == oplog_rows + + order_rows + |> Enum.group_by(fn {_append_id, stream_id, _seq} -> stream_id end) + |> Enum.each(fn {_stream_id, rows} -> + assert Enum.map(rows, &elem(&1, 2)) == Enum.to_list(1..50) + end) + end + end + + test "a shard restart replays an appended mixed record and later cleans its owner" do + name = start_single_shard_group(replicated_oplog_max_entries: 16) + key = "journal/replay/#{System.unique_integer([:positive])}" + owner = spawn(fn -> Process.sleep(:infinity) end) + on_exit(fn -> kill_if_alive(owner) end) + + stream_id = Group.Replica.Data.local_stream_id(name, 0, nil) + time = System.system_time() + + {seq, _mutations} = + Group.Replica.Data.append_replica_record(name, 0, stream_id, [ + {:register, nil, key, owner, %{kind: :registry}, time, node()}, + {:join, nil, key, owner, %{kind: :pg}, time, :join, node()} + ]) + + old_shard = Process.whereis(Group.Replica.shard_name(name, 0)) + Process.exit(old_shard, :kill) + + Group.TestCluster.assert_eventually(fn -> + new_shard = Process.whereis(Group.Replica.shard_name(name, 0)) + + is_pid(new_shard) and new_shard != old_shard and + Group.lookup(name, key) == {owner, %{kind: :registry}} and + Group.members(name, key) == [{owner, %{kind: :pg}}] + end) + + assert {_floor, ^seq, ^seq} = + Group.Replica.Data.replica_stream_head(name, 0, stream_id) + + Process.exit(owner, :kill) + + Group.TestCluster.assert_eventually(fn -> + Group.lookup(name, key) == nil and Group.members(name, key) == [] + end) + end + end + + describe "replica authority snapshots" do + test "revision and epoch rows remain coherent during concurrent activation", %{name: name} do + clusters = for i <- 1..1_000, do: "authority/#{i}" + + activators = + clusters + |> Enum.chunk_every(125) + |> Enum.map(fn chunk -> + Task.async(fn -> + Enum.each(chunk, fn cluster -> + [{^cluster, _epoch}] = + Group.Replica.Data.activate_local_clusters(name, [cluster]) + end) + end) + end) + + for _ <- 1..200 do + {generation, revision, epochs} = + Group.Replica.Data.local_replica_authority(name) + + assert {nil, generation} in epochs + assert revision == Enum.count(epochs, &(not is_nil(elem(&1, 0)))) + end + + Task.await_many(activators, 10_000) + + {generation, 1_000, epochs} = Group.Replica.Data.local_replica_authority(name) + assert {nil, generation} in epochs + assert length(epochs) == 1_001 + assert Map.new(epochs) |> map_size() == 1_001 + end + end + defp start_single_shard_group(opts \\ []) do name = :"test_timeout_group_#{System.unique_integer([:positive])}" opts = Keyword.merge([name: name, shards: 1, log: false], opts) @@ -2876,6 +3038,14 @@ defmodule GroupTest do name end + defp keys_for_shard(cluster, prefix, num_shards, shard, count) do + 1 + |> Stream.iterate(&(&1 + 1)) + |> Stream.map(&"#{prefix}/#{&1}") + |> Stream.filter(&(Group.Replica.shard_index_for(cluster, &1, num_shards) == shard)) + |> Enum.take(count) + end + defp suspend_only_shard(name) do shard = Group.Replica.shard_name(name, 0) :ok = :sys.suspend(shard) diff --git a/test/replica_adversarial_test.exs b/test/replica_adversarial_test.exs new file mode 100644 index 0000000..6b1437a --- /dev/null +++ b/test/replica_adversarial_test.exs @@ -0,0 +1,358 @@ +defmodule Group.ReplicaAdversarialTest do + use ExUnit.Case + + @moduletag :capture_log + @moduletag timeout: 120_000 + + alias Group.TestCluster + + @seeds [10_007, 20_011, 40_009] + @clusters ["red", "blue"] + + for seed <- @seeds do + @seed seed + @tag chaos_seed: seed + + test "seeded mixed-operation transport chaos converges without zombies (seed #{@seed})" do + seed = @seed + :rand.seed(:exsss, {seed, seed * 3 + 1, seed * 7 + 2}) + + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"replica_chaos_#{seed}_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 3, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 2, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 2_000, + replicated_oplog_max_entries: 12 + ] + + for {_peer, node} <- peers do + {:ok, _pid} = TestCluster.start_group(node, opts) + :ok = TestCluster.rpc!(node, Group, :connect, [name, @clusters]) + end + + TestCluster.assert_eventually( + fn -> + Enum.all?([node_a, node_b], fn node -> + length(TestCluster.rpc!(node, Group, :nodes, [name])) == 1 and + Enum.all?(@clusters, fn cluster -> + length(TestCluster.rpc!(node, Group, :nodes, [name, cluster])) == 2 + end) + end) + end, + timeout: 10_000 + ) + + :ok = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [ + name, + {:chaos, [drop_every: 5, duplicate_every: 7, max_delay: 40]} + ]) + + :ok = + TestCluster.rpc!(node_b, Group.TestReplicaTransport, :set_mode, [ + name, + {:chaos, [drop_every: 7, duplicate_every: 5, max_delay: 55]} + ]) + + initial = %{ + active: %{node_a => MapSet.new(@clusters), node_b => MapSet.new(@clusters)}, + counter: 0, + pg_keys: MapSet.new(), + pids: [], + reg_keys: MapSet.new(), + trace: [] + } + + state = + Enum.reduce(1..72, initial, fn step, state -> + apply_random_operation(state, step, seed, name, node_a, node_b) + end) + + for node <- [node_a, node_b] do + :ok = TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [name, :pass]) + :ok = TestCluster.rpc!(node, Group, :connect, [name, @clusters]) + end + + assert_converges(name, node_a, node_b, state) + + # Let delayed frames from the chaos phase arrive, then prove they are + # duplicates/stale rather than a source of resurrection. + Process.sleep(150) + TestCluster.flush_shards(node_a, name) + TestCluster.flush_shards(node_b, name) + assert_converges(name, node_a, node_b, state) + + for node <- [node_a, node_b] do + assert :ok = + TestCluster.rpc!(node, Group.TestCluster, :assert_replica_consistent, [name]) + end + + TestCluster.assert_eventually( + fn -> retained_owners_alive?(name, [node_a, node_b]) end, + timeout: 15_000 + ) + end + end + + defp apply_random_operation(state, step, seed, name, node_a, node_b) do + nodes = [node_a, node_b] + + case :rand.uniform(12) do + choice when choice in 1..3 -> + add_registration(state, step, seed, name, random(nodes)) + + choice when choice in 4..6 -> + add_membership(state, step, seed, name, random(nodes)) + + 7 -> + kill_random_owner(state) + + 8 -> + add_registry_conflict(state, step, seed, name, node_a, node_b) + + 9 -> + toggle_cluster(state, step, name, random(nodes), random(@clusters)) + + 10 -> + change_transport_mode(state, step, name, random(nodes)) + + _ -> + Enum.reduce(1..3, state, fn offset, acc -> + add_registration(acc, step * 10 + offset, seed, name, random(nodes)) + end) + end + end + + defp add_registration(state, step, seed, name, origin) do + cluster = random([nil | MapSet.to_list(state.active[origin])]) + {key, state} = next_key(state, seed, "reg", step) + meta = %{seed: seed, step: step, origin: origin} + + pid = + if cluster do + TestCluster.spawn_register_in_cluster(origin, name, key, meta, cluster) + else + TestCluster.spawn_register(origin, name, key, meta) + end + + state + |> Map.update!(:pids, &[{origin, pid} | &1]) + |> Map.update!(:reg_keys, &MapSet.put(&1, {cluster, key})) + |> trace({:register, origin, cluster, key, pid}) + end + + defp add_membership(state, step, seed, name, origin) do + cluster = random([nil | MapSet.to_list(state.active[origin])]) + {key, state} = next_key(state, seed, "pg", step) + meta = %{seed: seed, step: step, origin: origin} + + pid = + if cluster do + TestCluster.spawn_join_in_cluster(origin, name, key, meta, cluster) + else + TestCluster.spawn_join(origin, name, key, meta) + end + + state + |> Map.update!(:pids, &[{origin, pid} | &1]) + |> Map.update!(:pg_keys, &MapSet.put(&1, {cluster, key})) + |> trace({:join, origin, cluster, key, pid}) + end + + defp kill_random_owner(%{pids: []} = state), do: trace(state, :kill_noop) + + defp kill_random_owner(state) do + {origin, pid} = random(state.pids) + TestCluster.rpc!(origin, Process, :exit, [pid, :kill]) + + state + |> Map.update!(:pids, &List.delete(&1, {origin, pid})) + |> trace({:kill, origin, pid}) + end + + defp add_registry_conflict(state, step, seed, name, node_a, node_b) do + {key, state} = next_key(state, seed, "conflict", step) + + # Establish the competing claims before either origin can observe the + # other. Drain any already-scheduled delayed frame first; otherwise this + # operation nondeterministically degenerates into a local :taken result. + for node <- [node_a, node_b] do + :ok = TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [name, :drop]) + end + + Process.sleep(75) + pid_a = TestCluster.spawn_register(node_a, name, key, %{side: :a, seed: seed}) + pid_b = TestCluster.spawn_register(node_b, name, key, %{side: :b, seed: seed}) + + :ok = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [ + name, + {:chaos, [drop_every: 5, duplicate_every: 7, max_delay: 40]} + ]) + + :ok = + TestCluster.rpc!(node_b, Group.TestReplicaTransport, :set_mode, [ + name, + {:chaos, [drop_every: 7, duplicate_every: 5, max_delay: 55]} + ]) + + state + |> Map.update!(:pids, &[{node_a, pid_a}, {node_b, pid_b} | &1]) + |> Map.update!(:reg_keys, &MapSet.put(&1, {nil, key})) + |> trace({:conflict, key, pid_a, pid_b}) + end + + defp toggle_cluster(state, step, name, origin, cluster) do + if MapSet.member?(state.active[origin], cluster) do + :ok = TestCluster.rpc!(origin, Group, :disconnect, [name, cluster]) + + state + |> put_in([:active, origin], MapSet.delete(state.active[origin], cluster)) + |> trace({:disconnect, step, origin, cluster}) + else + :ok = TestCluster.rpc!(origin, Group, :connect, [name, cluster]) + + state + |> put_in([:active, origin], MapSet.put(state.active[origin], cluster)) + |> trace({:connect, step, origin, cluster}) + end + end + + defp change_transport_mode(state, step, name, origin) do + mode = + random([ + :drop, + :busy, + {:chaos, [drop_every: 4, duplicate_every: 3, max_delay: 65]}, + {:chaos, [drop_every: 9, duplicate_every: 2, max_delay: 30]} + ]) + + :ok = TestCluster.rpc!(origin, Group.TestReplicaTransport, :set_mode, [name, mode]) + trace(state, {:transport, step, origin, mode}) + end + + defp next_key(state, seed, kind, step) do + counter = state.counter + 1 + {"chaos/#{seed}/#{kind}/#{step}/#{counter}", %{state | counter: counter}} + end + + defp assert_converges(name, node_a, node_b, state) do + TestCluster.assert_eventually( + fn -> + length(TestCluster.rpc!(node_a, Group, :nodes, [name])) == 1 and + length(TestCluster.rpc!(node_b, Group, :nodes, [name])) == 1 and + Enum.all?(@clusters, fn cluster -> + length(TestCluster.rpc!(node_a, Group, :nodes, [name, cluster])) == 2 and + length(TestCluster.rpc!(node_b, Group, :nodes, [name, cluster])) == 2 + end) and + registry_equal?(name, node_a, node_b, state.reg_keys) and + memberships_equal?(name, node_a, node_b, state.pg_keys) + end, + timeout: 20_000, + interval: 75 + ) + rescue + error -> + flunk( + "chaos convergence failed: #{Exception.message(error)}\n" <> + "differences=#{inspect(convergence_differences(name, node_a, node_b, state), limit: :infinity)}\n" <> + "recent operations=#{inspect(Enum.take(state.trace, 20), limit: :infinity)}" + ) + end + + defp convergence_differences(name, node_a, node_b, state) do + registry = + state.reg_keys + |> Enum.flat_map(fn {cluster, key} -> + args = [name, key, cluster_opts(cluster)] + value_a = TestCluster.rpc!(node_a, Group, :lookup, args) + value_b = TestCluster.rpc!(node_b, Group, :lookup, args) + if value_a == value_b, do: [], else: [{:registry, cluster, key, value_a, value_b}] + end) + |> Enum.take(10) + + pg = + state.pg_keys + |> Enum.flat_map(fn {cluster, key} -> + args = [name, key, cluster_opts(cluster)] + value_a = TestCluster.rpc!(node_a, Group, :members, args) |> Enum.sort() + value_b = TestCluster.rpc!(node_b, Group, :members, args) |> Enum.sort() + if value_a == value_b, do: [], else: [{:pg, cluster, key, value_a, value_b}] + end) + |> Enum.take(10) + + nodes = + for cluster <- [nil | @clusters] do + value_a = group_nodes(node_a, name, cluster) + value_b = group_nodes(node_b, name, cluster) + {cluster, value_a, value_b} + end + + cluster_trace = + state.trace + |> Enum.filter(fn + {:connect, _step, _origin, _cluster} -> true + {:disconnect, _step, _origin, _cluster} -> true + _ -> false + end) + + protocol = + for node <- [node_a, node_b] do + {node, TestCluster.rpc!(node, Group.TestCluster, :replica_protocol_state, [name])} + end + + [ + nodes: nodes, + registry: registry, + pg: pg, + cluster_trace: cluster_trace, + protocol: protocol + ] + end + + defp group_nodes(node, name, nil), do: TestCluster.rpc!(node, Group, :nodes, [name]) + + defp group_nodes(node, name, cluster), + do: TestCluster.rpc!(node, Group, :nodes, [name, cluster]) + + defp registry_equal?(name, node_a, node_b, keys) do + Enum.all?(keys, fn {cluster, key} -> + args = [name, key, cluster_opts(cluster)] + + TestCluster.rpc!(node_a, Group, :lookup, args) == + TestCluster.rpc!(node_b, Group, :lookup, args) + end) + end + + defp memberships_equal?(name, node_a, node_b, keys) do + Enum.all?(keys, fn {cluster, key} -> + args = [name, key, cluster_opts(cluster)] + members_a = TestCluster.rpc!(node_a, Group, :members, args) |> Enum.sort() + members_b = TestCluster.rpc!(node_b, Group, :members, args) |> Enum.sort() + members_a == members_b + end) + end + + defp retained_owners_alive?(name, nodes) do + nodes + |> Enum.flat_map(fn node -> + TestCluster.rpc!(node, Group.TestCluster, :replica_owner_pids, [name]) + end) + |> Enum.uniq() + |> Enum.all?(fn pid -> TestCluster.rpc!(node(pid), Process, :alive?, [pid]) end) + end + + defp cluster_opts(nil), do: [] + defp cluster_opts(cluster), do: [cluster: cluster] + + defp trace(state, operation), do: Map.update!(state, :trace, &[operation | &1]) + defp random(values), do: Enum.at(values, :rand.uniform(length(values)) - 1) +end diff --git a/test/support/test_cluster.ex b/test/support/test_cluster.ex index f9b8c1e..ebffc86 100644 --- a/test/support/test_cluster.ex +++ b/test/support/test_cluster.ex @@ -299,6 +299,92 @@ defmodule Group.TestCluster do end) end + @doc "Spawn a process on a remote node that joins in a named cluster and sleeps." + def spawn_join_in_cluster(node, name, key, meta, cluster) do + :erpc.call(node, fn -> + parent = self() + + pid = + spawn(fn -> + :ok = Group.join(name, key, meta, cluster: cluster) + send(parent, {:joined, self()}) + Process.sleep(:infinity) + end) + + receive do + {:joined, ^pid} -> pid + after + 5000 -> raise "spawn_join_in_cluster timed out" + end + end) + end + + @doc "Connects every cluster through an independent concurrent caller." + def connect_many_concurrently(node, name, clusters) do + :erpc.call(node, __MODULE__, :do_connect_many_concurrently, [name, clusters], 60_000) + end + + @doc false + def do_connect_many_concurrently(name, clusters) do + clusters + |> Task.async_stream( + fn cluster -> Group.connect(name, cluster) end, + max_concurrency: 64, + ordered: false, + timeout: 30_000 + ) + |> Enum.each(fn {:ok, :ok} -> :ok end) + + :ok + end + + @doc "Spawns one long-lived registration owner in each named cluster." + def spawn_register_many_clusters(node, name, clusters, key_prefix) do + :erpc.call( + node, + __MODULE__, + :do_spawn_register_many_clusters, + [name, clusters, key_prefix], + 60_000 + ) + end + + @doc false + def do_spawn_register_many_clusters(name, clusters, key_prefix) do + parent = self() + + entries = + Enum.map(clusters, fn cluster -> + key = "#{key_prefix}/#{cluster}" + + pid = + spawn(fn -> + :ok = Group.register(name, key, %{cluster: cluster}, cluster: cluster) + send(parent, {:registered_many, self()}) + Process.sleep(:infinity) + end) + + {cluster, key, pid} + end) + + Enum.each(entries, fn {_cluster, _key, pid} -> + receive do + {:registered_many, ^pid} -> :ok + after + 30_000 -> raise "spawn_register_many_clusters timed out" + end + end) + + entries + end + + @doc false + def registry_entries_present?(name, entries) do + Enum.all?(entries, fn {cluster, key, pid} -> + match?({^pid, %{cluster: ^cluster}}, Group.lookup(name, key, cluster: cluster)) + end) + end + @doc "Monitor nodedown events from a remote node, forwarding to caller" def monitor_nodes_on(node, target_pid) do :erpc.call(node, fn -> @@ -553,6 +639,193 @@ defmodule Group.TestCluster do :ok end + @doc """ + Asserts the replica-only authority and journal invariants in addition to the + public dual-index invariants checked by `assert_ets_consistent/1`. + + This is intended for quiescent convergence points in adversarial tests. + """ + def assert_replica_consistent(name) do + :ok = assert_ets_consistent(name) + num_shards = Group.get_config(name).num_shards + + for shard <- 0..(num_shards - 1) do + assert_registry_claim_indexes(name, shard) + assert_registry_projection_has_authority(name, shard) + assert_oplog_indexes(name, shard) + assert_replica_cursor_authority(name, shard) + end + + :ok + end + + @doc """ + Returns every PID currently retained as replica authority or visible PG state. + + Adversarial tests use this at a quiescent convergence point to prove that no + dead owner remains hidden behind otherwise-consistent dual indexes. + """ + def replica_owner_pids(name) do + num_shards = Group.get_config(name).num_shards + + 0..(num_shards - 1) + |> Enum.flat_map(fn shard -> + registry_pids = + Group.Replica.Data.reg_claim_by_key_table(name, shard) + |> :ets.tab2list() + |> Enum.map(fn {{_cluster, _key, _origin, _generation, _epoch}, pid, _meta, _time, _seq} -> + pid + end) + + pg_pids = + Group.Replica.Data.pg_by_key_table(name, shard) + |> :ets.tab2list() + |> Enum.map(fn {{_cluster, _key, pid}, _meta, _time, _node} -> pid end) + + registry_pids ++ pg_pids + end) + |> Enum.uniq() + end + + @doc false + def replica_protocol_state(name) do + num_shards = Group.get_config(name).num_shards + + for shard <- 0..(num_shards - 1) do + %{ + shard: shard, + heads: Group.Replica.Data.replica_stream_heads(name, shard), + cursors: + Group.Replica.Data.replica_cursor_table(name, shard) + |> :ets.tab2list() + |> Enum.sort() + } + end + end + + defp assert_registry_claim_indexes(name, shard) do + by_key = Group.Replica.Data.reg_claim_by_key_table(name, shard) + by_pid = Group.Replica.Data.reg_claim_by_pid_table(name, shard) + + key_set = + :ets.tab2list(by_key) + |> MapSet.new(fn + {{cluster, key, origin, generation, epoch}, pid, meta, time, seq} -> + {cluster, key, pid, meta, time, origin, generation, epoch, seq} + end) + + pid_set = + :ets.tab2list(by_pid) + |> MapSet.new(fn + {{pid, cluster, key, origin, generation, epoch}, meta, time, seq} -> + {cluster, key, pid, meta, time, origin, generation, epoch, seq} + end) + + if key_set != pid_set do + raise "registry claim index inconsistency in #{name} shard #{shard}: " <> + "by_key_only=#{inspect(MapSet.difference(key_set, pid_set) |> MapSet.to_list())} " <> + "by_pid_only=#{inspect(MapSet.difference(pid_set, key_set) |> MapSet.to_list())}" + end + + case Enum.find(key_set, fn {_cluster, _key, pid, _meta, _time, origin, _gen, _epoch, _seq} -> + node(pid) != origin + end) do + nil -> :ok + invalid -> raise "registry claim has invalid origin authority: #{inspect(invalid)}" + end + end + + defp assert_registry_projection_has_authority(name, shard) do + claims = + Group.Replica.Data.reg_claim_by_key_table(name, shard) + |> :ets.tab2list() + |> MapSet.new(fn + {{cluster, key, origin, _generation, _epoch}, pid, meta, time, _seq} -> + {cluster, key, pid, meta, time, origin} + end) + + visible = + Group.Replica.Data.reg_by_key_table(name, shard) + |> :ets.tab2list() + |> MapSet.new(fn {{cluster, key}, pid, meta, time, origin} -> + {cluster, key, pid, meta, time, origin} + end) + + missing_authority = MapSet.difference(visible, claims) + + if MapSet.size(missing_authority) > 0 do + raise "visible registry rows without an authoritative claim in #{name} shard #{shard}: " <> + inspect(MapSet.to_list(missing_authority)) + end + end + + defp assert_oplog_indexes(name, shard) do + oplog = + Group.Replica.Data.replica_oplog_table(name, shard) + |> :ets.tab2list() + |> MapSet.new(fn {{stream_id, seq}, append_id, _mutations} -> + {append_id, stream_id, seq} + end) + + order = + Group.Replica.Data.replica_oplog_order_table(name, shard) + |> :ets.tab2list() + |> MapSet.new() + + if oplog != order do + raise "oplog/order index inconsistency in #{name} shard #{shard}: " <> + "oplog_only=#{inspect(MapSet.difference(oplog, order) |> MapSet.to_list())} " <> + "order_only=#{inspect(MapSet.difference(order, oplog) |> MapSet.to_list())}" + end + + Group.Replica.Data.replica_stream_meta_table(name, shard) + |> :ets.tab2list() + |> Enum.each(fn {stream_id, head, floor, applied} -> + unless floor >= 1 and floor <= head + 1 and applied >= 0 and applied <= head do + raise "invalid stream bounds in #{name} shard #{shard}: " <> + inspect({stream_id, head, floor, applied}) + end + + retained = + oplog + |> Enum.filter(fn {_append_id, row_stream, _seq} -> row_stream == stream_id end) + |> Enum.map(&elem(&1, 2)) + |> Enum.sort() + + expected = if floor <= head, do: Enum.to_list(floor..head), else: [] + + if retained != expected do + raise "non-contiguous retained oplog in #{name} shard #{shard}: " <> + "stream=#{inspect(stream_id)} retained=#{inspect(retained)} " <> + "expected=#{inspect(expected)}" + end + end) + end + + defp assert_replica_cursor_authority(name, shard) do + Group.Replica.Data.replica_cursor_table(name, shard) + |> :ets.tab2list() + |> Enum.each(fn {stream_id, seq} -> + origin = Group.Replica.Protocol.stream_origin(stream_id) + generation = Group.Replica.Protocol.stream_generation(stream_id) + cluster = Group.Replica.Protocol.stream_cluster(stream_id) + epoch = Group.Replica.Protocol.stream_epoch(stream_id) + + valid? = + Group.Replica.Protocol.stream_name(stream_id) == name and + Group.Replica.Protocol.stream_shard(stream_id) == shard and + origin != node() and + generation == Group.Replica.Data.remote_generation(name, origin) and + epoch == Group.Replica.Data.remote_cluster_epoch(name, origin, cluster) and + seq >= 0 + + unless valid? do + raise "replica cursor is not fenced by current authority in #{name} shard #{shard}: " <> + inspect({stream_id, seq}) + end + end) + end + @doc "Wait for a condition to become true, with retries" def assert_eventually(fun, opts \\ []) do timeout = Keyword.get(opts, :timeout, 2000) diff --git a/test/support/test_replica_transport.ex b/test/support/test_replica_transport.ex new file mode 100644 index 0000000..5eef3d0 --- /dev/null +++ b/test/support/test_replica_transport.ex @@ -0,0 +1,140 @@ +defmodule Group.TestReplicaTransport do + @moduledoc false + @behaviour Group.Replica.Transport + + @impl true + def id, do: :group_test_transport + + @impl true + def descriptor(_group, _opts), do: :group_test_transport + + @simple_modes [:pass, :drop, :busy, :duplicate] + + def set_mode(group, mode) + when mode in @simple_modes or + (is_tuple(mode) and tuple_size(mode) == 2 and + elem(mode, 0) in [:drop_types, :duplicate_types, :capture_drop, :capture_pass]) or + (is_tuple(mode) and tuple_size(mode) == 3 and elem(mode, 0) == :delay_types) or + (is_tuple(mode) and tuple_size(mode) == 2 and elem(mode, 0) == :chaos) do + :persistent_term.put({__MODULE__, group}, mode) + :ok + end + + def captured(group) do + :persistent_term.get({__MODULE__, group, :captured}, []) |> Enum.reverse() + end + + def clear_captured(group) do + :persistent_term.erase({__MODULE__, group, :captured}) + :ok + end + + def clear(group) do + :persistent_term.erase({__MODULE__, group}) + :persistent_term.erase({__MODULE__, group, :captured}) + :ok + end + + @impl true + def try_send(group, target_node, shard, frame, _opts) do + case :persistent_term.get({__MODULE__, group}, :pass) do + :drop -> + :ok + + :busy -> + :busy + + :duplicate -> + deliver(group, target_node, shard, frame) + deliver(group, target_node, shard, frame) + + {:drop_types, types} -> + if frame_type(frame) in types, do: :ok, else: deliver(group, target_node, shard, frame) + + {:duplicate_types, types} -> + if frame_type(frame) in types do + deliver(group, target_node, shard, frame) + deliver(group, target_node, shard, frame) + else + deliver(group, target_node, shard, frame) + end + + {:delay_types, delays, default_delay} -> + delay = Map.get(delays, frame_type(frame), default_delay) + delayed_deliver(group, target_node, shard, frame, delay) + + {:capture_drop, types} -> + if frame_type(frame) in types, do: capture(group, target_node, shard, frame) + :ok + + {:capture_pass, types} -> + if frame_type(frame) in types, do: capture(group, target_node, shard, frame) + deliver(group, target_node, shard, frame) + + {:chaos, opts} -> + chaos_deliver(group, target_node, shard, frame, opts) + + :pass -> + deliver(group, target_node, shard, frame) + end + end + + defp chaos_deliver(group, target_node, shard, frame, opts) do + hash = :erlang.phash2({target_node, shard, frame}, 1_000_003) + drop_every = Keyword.get(opts, :drop_every, 0) + duplicate_every = Keyword.get(opts, :duplicate_every, 0) + max_delay = Keyword.get(opts, :max_delay, 0) + + cond do + drop_every > 0 and rem(hash, drop_every) == 0 -> + :ok + + duplicate_every > 0 and rem(hash, duplicate_every) == 0 -> + delay = if max_delay > 0, do: rem(hash, max_delay + 1), else: 0 + delayed_deliver(group, target_node, shard, frame, delay) + delayed_deliver(group, target_node, shard, frame, max(max_delay - delay, 0)) + + true -> + delay = if max_delay > 0, do: rem(hash, max_delay + 1), else: 0 + delayed_deliver(group, target_node, shard, frame, delay) + end + end + + defp delayed_deliver(group, target_node, shard, frame, delay) when delay <= 0, + do: deliver(group, target_node, shard, frame) + + defp delayed_deliver(group, target_node, shard, frame, delay) do + source_node = node() + + spawn(fn -> + receive do + after + delay -> deliver(group, target_node, shard, frame, source_node) + end + end) + + :ok + end + + defp capture(group, target_node, shard, frame) do + key = {__MODULE__, group, :captured} + captured = :persistent_term.get(key, []) + :persistent_term.put(key, [{target_node, shard, frame} | captured]) + end + + defp deliver(group, target_node, shard, frame), + do: deliver(group, target_node, shard, frame, node()) + + defp deliver(group, target_node, shard, frame, source_node) do + destination = {Group.Replica.shard_name(group, shard), target_node} + message = {:group_replica_frame, source_node, frame} + + case :erlang.send_nosuspend(destination, message, [:noconnect]) do + true -> :ok + false -> :busy + end + end + + defp frame_type(frame) when is_tuple(frame), do: elem(frame, 0) + defp frame_type(_frame), do: :unknown +end From d1773a0021ef46a3a56832c50aa287f0422e18e5 Mon Sep 17 00:00:00 2001 From: Chris McCord Date: Fri, 31 Jul 2026 07:54:44 +0000 Subject: [PATCH 03/16] Harden anti-entropy recovery and add TCP transport --- .gitignore | 3 + README.md | 47 +- lib/group.ex | 1 + lib/group/replica.ex | 121 +++-- lib/group/replica/data.ex | 287 ++++++++-- lib/group/replica/transport/tcp.ex | 448 ++++++++++++++++ mix.exs | 3 +- mix.lock | 1 + test/README.md | 44 +- test/distributed_test.exs | 495 ++++++++++++++++++ test/formal/GroupAntiEntropy.cfg | 17 + test/formal/GroupAntiEntropy.tla | 481 +++++++++++++++++ test/formal/README.md | 44 ++ test/formal/check.sh | 19 + test/group_test.exs | 185 +++++++ test/mutation/README.md | 24 + test/mutation/run.exs | 422 +++++++++++++++ test/replica_model_property_test.exs | 381 ++++++++++++++ test/support/controlled_replica_transport.ex | 49 ++ test/support/model_conflict_resolver.ex | 15 + test/support/replica_lifecycle_model.ex | 223 ++++++++ test/support/replica_model_scheduler.ex | 521 +++++++++++++++++++ test/support/test_cluster.ex | 84 ++- 23 files changed, 3829 insertions(+), 86 deletions(-) create mode 100644 lib/group/replica/transport/tcp.ex create mode 100644 test/formal/GroupAntiEntropy.cfg create mode 100644 test/formal/GroupAntiEntropy.tla create mode 100644 test/formal/README.md create mode 100755 test/formal/check.sh create mode 100644 test/mutation/README.md create mode 100644 test/mutation/run.exs create mode 100644 test/replica_model_property_test.exs create mode 100644 test/support/controlled_replica_transport.ex create mode 100644 test/support/model_conflict_resolver.ex create mode 100644 test/support/replica_lifecycle_model.ex create mode 100644 test/support/replica_model_scheduler.ex diff --git a/.gitignore b/.gitignore index caee45d..8328a5b 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,9 @@ doc/ # Temporary files, for example, from tests. tmp/ +# TLC's default state directory when the formal model is run by hand. +/states/ + # If the VM crashes, it generates a dump, let's ignore it too. erl_crash.dump diff --git a/README.md b/README.md index 0943994..c9d351e 100644 --- a/README.md +++ b/README.md @@ -293,6 +293,9 @@ All operations are **eventually consistent**: `Group.Replica.Transport`, or `{module, opts}`. The default adapter uses `:erlang.send_nosuspend/3`; adapters must return promptly with `:ok`, `:busy`, or `:disconnected`. Dropped and busy frames are repaired by anti-entropy. + `Group.Replica.Transport.TCP` is an included sideband adapter with bounded + per-peer writer queues; its socket owners are separate processes, so socket + backpressure cannot block a Group shard. - **`replicated_oplog_max_entries`** — maximum retained replica records per shard across all local streams. Defaults to 65,536. Pruning never waits for peer acknowledgements; a peer behind the retained floor receives an exact @@ -391,9 +394,13 @@ point-in-time value. The highest observed incremental revision is tracked separately and can never promote a partial view to exact authority. Discovery hints never mutate membership on their own. Authority installation fans a local fence to every lane, which sweeps only that lane's retained receive -streams. Because PG rows intentionally do not carry protocol epochs, a -superseded origin/cluster slice is cleared and its current cursor reset so the -next head reconstructs it from retained deltas or an exact snapshot. +streams. Shared authority may become visible before that fanout reaches a lane, +but the lane's constant-size view is not marked installed until its purge +finishes; data validation requires that marker. A heartbeat or lane hello can +confirm an installed view but cannot promote a pending one. Because PG rows +intentionally do not carry protocol epochs, a superseded origin/cluster slice +is cleared and its current cursor reset so the next head reconstructs it from +retained deltas or an exact snapshot. Replica state itself does not travel on the control plane. Once the hello is fenced, stream-head exchange on the replica transport catches the peer up. @@ -412,10 +419,13 @@ tail even when no later write occurs. If the requested sequence is older than the bounded oplog floor, the origin sends an exact snapshot of only its own registry claims and PG memberships; absence from that snapshot is a delete. -There are no leaders, quorum acknowledgements, tombstones, or known-membership -retention barriers. Oplog memory is bounded locally and independently of slow -peers. Deletes are normal ordered records while retained, and exact snapshots -close gaps after pruning. +There are no leaders, quorum acknowledgements, per-entry replicated tombstones, +or known-membership retention barriers. Oplog memory is bounded locally and +independently of slow peers. Deletes are normal ordered records while retained, +and exact snapshots close gaps after pruning. Named-cluster close uses only a +temporary local shard-completion barrier; the final shard removes it and all +routing rows, including after a caller timeout or shard restart. Reconnect +waits for that barrier so a prior close cannot erase newly accepted writes. The sender flush timer is mainly a fallback for idle periods. The unified outbound buffer also flushes immediately when it hits the configured size, when a new enqueue @@ -430,6 +440,26 @@ or reconnect, and generation fencing rejects data from a restarted origin. An alternative sideband adapter authenticates the peer as a dist-Erlang node and calls `Group.Replica.Transport.deliver/4` locally. +For example, replica data can use the included sideband TCP adapter while +authority and membership remain on dist Erlang: + +```elixir +replica_transport: + {Group.Replica.Transport.TCP, + [ + ip: {0, 0, 0, 0}, + advertised_ip: {10, 0, 1, 12}, + port: 44_321, + max_queue: 1_024 + ]} +``` + +Each node advertises its own reachable address. TCP frames are capability +authenticated by the dist-Erlang hello but are not encrypted, so use a trusted +network or place the connection behind TLS. The adapter deliberately has no +control/data ordering relationship; the generation/epoch lane barrier and +stream sequence checks supply correctness. + ### Named Cluster TTL Leases Named-cluster TTLs are a local way to reduce replication fanout to nodes that @@ -466,7 +496,8 @@ mix test ``` See [`test/README.md`](test/README.md) for details on the distributed test -infrastructure. +infrastructure, shrinkable StreamData lifecycle-model tests, and the bounded +TLA+ anti-entropy model. ## Benchmarks diff --git a/lib/group.ex b/lib/group.ex index f125efd..449e4fa 100644 --- a/lib/group.ex +++ b/lib/group.ex @@ -1107,6 +1107,7 @@ defmodule Group do @doc false def connect_clusters(name, clusters, timeout) when is_atom(name) and is_list(clusters) and is_integer(timeout) do + timeout = Data.await_closed_local_clusters(name, clusters, timeout) _epochs = Data.activate_local_clusters(name, clusters) Data.add_cluster_node(name, clusters, node()) diff --git a/lib/group/replica.ex b/lib/group/replica.ex index 95bb6b6..58ce632 100644 --- a/lib/group/replica.ex +++ b/lib/group/replica.ex @@ -248,9 +248,17 @@ defmodule Group.Replica do state = schedule_anti_entropy(state) - # Complete any write-ahead record left unapplied by a shard crash, then - # rebuild local process monitors from the surviving materialized tables. + # Repair any interrupted multi-table journal/index mutation, complete + # write-ahead records left unapplied by a shard crash, then rebuild local + # process monitors from the surviving materialized tables. + :ok = Data.repair_local_replica_journal(name, shard_index) state = replay_local_journal(state) + :ok = Data.repair_shard_indexes(name, shard_index) + + completed_clusters = + Data.mark_closed_cluster_shard(name, Data.closed_local_clusters(name), shard_index) + + if completed_clusters != [], do: Data.remove_clusters(name, completed_clusters) # Rebuild monitors from any surviving ETS data (after shard crash/restart) state = rebuild_monitors(state) @@ -452,27 +460,25 @@ defmodule Group.Replica do Map.put(state.peer_transports, remote_node, {transport_id, transport_descriptor}) } - if replica_authority_current?(state, remote_node, generation, epoch_revision) do - :ok = - Data.put_remote_view_info( - state.name, - state.shard_index, - remote_node, - generation, - Data.remote_cluster_epoch_exact_revision(state.name, remote_node), - epoch_revision - ) + cond do + replica_authority_current?(state, remote_node, generation, epoch_revision) and + replica_view_current?(state, remote_node) -> + state = + state + |> purge_remote_streams_outside_authority(remote_node) + |> touch_replica_peer(remote_node) + |> Map.update!(:cluster_control_dirty, &Map.delete(&1, remote_node)) + |> send_replica_heads(remote_node) - state = - state - |> purge_remote_streams_outside_authority(remote_node) - |> touch_replica_peer(remote_node) - |> Map.update!(:cluster_control_dirty, &Map.delete(&1, remote_node)) - |> send_replica_heads(remote_node) + {:noreply, state} - {:noreply, state} - else - {:noreply, request_replica_authority(state, remote_node)} + replica_authority_current?(state, remote_node, generation, epoch_revision) -> + # Shared authority arrived first; its local fanout is already the + # ordered marker that will purge and install this lane's view. + {:noreply, state} + + true -> + {:noreply, request_replica_authority(state, remote_node)} end else Logger.error( @@ -502,6 +508,8 @@ defmodule Group.Replica do state end + :ok = install_replica_view(state, remote_node, generation) + state = %{ state | cluster_control_dirty: Map.delete(state.cluster_control_dirty, remote_node), @@ -574,6 +582,7 @@ defmodule Group.Replica do |> purge_closed_remote_epochs(remote_node, stale) |> purge_superseded_remote_streams(remote_node, epochs) + :ok = install_replica_view(state, remote_node, generation) state = send_replica_heads(state, remote_node, Enum.map(shared, &elem(&1, 0))) {:noreply, take_one_local_request_turn(state)} @@ -606,10 +615,13 @@ defmodule Group.Replica do state = if replica_authority_current?(state, remote_node, generation, revision) do - state - |> purge_closed_remote_epochs(remote_node, stale) - |> purge_superseded_remote_streams(remote_node, epochs) - |> send_replica_heads(remote_node, shared) + state = + state + |> purge_closed_remote_epochs(remote_node, stale) + |> purge_superseded_remote_streams(remote_node, epochs) + + :ok = install_replica_view(state, remote_node, generation) + send_replica_heads(state, remote_node, shared) else state end @@ -665,6 +677,7 @@ defmodule Group.Replica do |> mark_cluster_control_dirty(remote_node) |> purge_closed_remote_epochs(remote_node, closed) + :ok = install_replica_view(state, remote_node, generation) {:noreply, take_one_local_request_turn(state)} :stale -> @@ -690,7 +703,9 @@ defmodule Group.Replica do state = if replica_authority_current?(state, remote_node, generation, revision) do - purge_closed_remote_epochs(state, remote_node, closed) + state = purge_closed_remote_epochs(state, remote_node, closed) + :ok = install_replica_view(state, remote_node, generation) + state else state end @@ -714,23 +729,20 @@ defmodule Group.Replica do remote_node = node(remote_pid) state = - if version == Protocol.version() and - replica_authority_current?(state, remote_node, generation, epoch_revision) do - :ok = - Data.put_remote_view_info( - state.name, - state.shard_index, - remote_node, - generation, - Data.remote_cluster_epoch_exact_revision(state.name, remote_node), - epoch_revision - ) + cond do + version == Protocol.version() and + replica_authority_current?(state, remote_node, generation, epoch_revision) and + replica_view_current?(state, remote_node) -> + state + |> put_remote_shard(remote_node, remote_pid) + |> touch_replica_peer(remote_node) - state - |> put_remote_shard(remote_node, remote_pid) - |> touch_replica_peer(remote_node) - else - request_replica_authority(state, remote_node) + version == Protocol.version() and + replica_authority_current?(state, remote_node, generation, epoch_revision) -> + state + + true -> + request_replica_authority(state, remote_node) end {:noreply, state} @@ -2061,6 +2073,8 @@ defmodule Group.Replica do :ok end) + completed_clusters = Data.mark_closed_cluster_shard(name, clusters, shard) + if completed_clusters != [], do: Data.remove_clusters(name, completed_clusters) notify_monitors(name, events) {:ok, state} end @@ -2849,6 +2863,8 @@ defmodule Group.Replica do state end + :ok = install_replica_view(state, remote_node, generation) + state = %{ state | remote_shards: Map.put(state.remote_shards, remote_node, remote_pid), @@ -2916,6 +2932,26 @@ defmodule Group.Replica do Data.remote_cluster_epoch_observed_revision(state.name, remote_node) == epoch_revision end + defp replica_view_current?(state, remote_node) do + Data.remote_view_generation(state.name, state.shard_index, remote_node) == + Data.remote_generation(state.name, remote_node) and + Data.remote_view_cluster_epoch_revision(state.name, state.shard_index, remote_node) == + Data.remote_cluster_epoch_exact_revision(state.name, remote_node) and + Data.remote_view_observed_revision(state.name, state.shard_index, remote_node) == + Data.remote_cluster_epoch_observed_revision(state.name, remote_node) + end + + defp install_replica_view(state, remote_node, generation) do + Data.put_remote_view_info( + state.name, + state.shard_index, + remote_node, + generation, + Data.remote_cluster_epoch_exact_revision(state.name, remote_node), + Data.remote_cluster_epoch_observed_revision(state.name, remote_node) + ) + end + defp schedule_anti_entropy(state) do ref = make_ref() @@ -3178,6 +3214,7 @@ defmodule Group.Replica do Protocol.stream_name(stream_id) == state.name and Protocol.stream_origin(stream_id) == source_node and Protocol.stream_shard(stream_id) == state.shard_index and + replica_view_current?(state, source_node) and Protocol.stream_generation(stream_id) == Data.remote_generation(state.name, source_node) and Protocol.stream_epoch(stream_id) == Data.remote_cluster_epoch(state.name, source_node, cluster) and diff --git a/lib/group/replica/data.ex b/lib/group/replica/data.ex index e6e4c81..94f803a 100644 --- a/lib/group/replica/data.ex +++ b/lib/group/replica/data.ex @@ -2,6 +2,8 @@ defmodule Group.Replica.Data do @moduledoc false use GenServer + alias Group.Replica.Protocol + _archdoc = """ GenServer that owns ETS tables for all shards. @@ -183,11 +185,23 @@ defmodule Group.Replica.Data do def closed_local_cluster_epoch(name, cluster) do case :ets.lookup(closed_local_cluster_epochs_table(name), cluster) do - [{^cluster, epoch}] -> epoch + [{^cluster, epoch, _pending_shards}] -> epoch [] -> nil end end + def closed_local_clusters(name) do + closed_local_cluster_epochs_table(name) + |> :ets.tab2list() + |> Enum.map(&elem(&1, 0)) + end + + def await_closed_local_clusters(name, clusters, timeout) + when is_list(clusters) and is_integer(timeout) and timeout >= 0 do + started_at = System.monotonic_time(:millisecond) + await_closed_local_clusters(name, clusters, timeout, started_at) + end + def remote_generation(name, remote_node) do case :ets.lookup(replication_meta_table(name), {:remote_generation, remote_node}) do [{{:remote_generation, ^remote_node}, generation}] -> generation @@ -319,6 +333,10 @@ defmodule Group.Replica.Data do GenServer.call(data_name(name), {:deactivate_local_clusters, clusters}, :infinity) end + def mark_closed_cluster_shard(name, clusters, shard) do + GenServer.call(data_name(name), {:mark_closed_cluster_shard, clusters, shard}, :infinity) + end + def local_stream_id(name, shard, cluster) do case local_cluster_epoch(name, cluster) do nil -> @@ -378,11 +396,209 @@ defmodule Group.Replica.Data do end) end + @doc false + def repair_local_replica_journal(name, shard) do + stream_table = replica_stream_meta_table(name, shard) + oplog_table = replica_oplog_table(name, shard) + order_table = replica_oplog_order_table(name, shard) + + stream_table + |> :ets.tab2list() + |> Enum.each(fn {stream_id, head, floor, applied} -> + if current_local_stream?(name, shard, stream_id) do + present = + oplog_table + |> :ets.select([ + {{{stream_id, :"$1"}, :_, :_}, [], [:"$1"]} + ]) + |> MapSet.new() + + repaired_floor = + floor + |> missing_applied_sequences(applied, present) + |> case do + [] -> floor + missing -> Enum.max(missing) + 1 + end + + repaired_head = contiguous_unapplied_head(applied, head, present) + + if repaired_head < head do + :ets.select_delete(oplog_table, [ + {{{stream_id, :"$1"}, :_, :_}, [{:>, :"$1", repaired_head}], [true]} + ]) + end + + repaired_floor = min(repaired_floor, repaired_head + 1) + repaired_applied = min(applied, repaired_head) + + :ets.insert( + stream_table, + {stream_id, repaired_head, repaired_floor, repaired_applied} + ) + else + drop_local_stream( + name, + shard, + Protocol.stream_cluster(stream_id), + Protocol.stream_epoch(stream_id) + ) + end + end) + + :ets.delete_all_objects(order_table) + + oplog_table + |> :ets.tab2list() + |> Enum.each(fn {{stream_id, seq}, append_id, _mutations} -> + :ets.insert(order_table, {append_id, stream_id, seq}) + end) + + :ok + end + + @doc false + def repair_shard_indexes(name, shard) do + purge_inactive_cluster_rows(name, shard) + rebuild_registry_reverse_index(name, shard) + rebuild_registry_claim_reverse_index(name, shard) + rebuild_pg_reverse_index(name, shard) + :ok + end + def replica_stream_heads(name, shard) do :ets.tab2list(replica_stream_meta_table(name, shard)) |> Enum.map(fn {stream_id, head, floor, _applied} -> {stream_id, floor, head} end) end + defp missing_applied_sequences(floor, applied, _present) when floor > applied, do: [] + + defp missing_applied_sequences(floor, applied, present) do + Enum.reject(floor..applied, &MapSet.member?(present, &1)) + end + + defp contiguous_unapplied_head(applied, head, _present) when applied >= head, do: head + + defp contiguous_unapplied_head(applied, head, present) do + Enum.reduce_while((applied + 1)..head, applied, fn seq, _last -> + if MapSet.member?(present, seq), do: {:cont, seq}, else: {:halt, seq - 1} + end) + end + + defp current_local_stream?(name, shard, stream_id) do + cluster = Protocol.stream_cluster(stream_id) + + Protocol.stream_name(stream_id) == name and + Protocol.stream_origin(stream_id) == node() and + Protocol.stream_generation(stream_id) == generation(name) and + Protocol.stream_shard(stream_id) == shard and + Protocol.stream_epoch(stream_id) == local_cluster_epoch(name, cluster) + end + + defp await_closed_local_clusters(name, clusters, timeout, started_at) do + pending? = + Enum.any?(clusters, fn cluster -> + not is_nil(closed_local_cluster_epoch(name, cluster)) + end) + + elapsed = System.monotonic_time(:millisecond) - started_at + + cond do + not pending? -> + max(timeout - elapsed, 0) + + elapsed >= timeout -> + exit( + {:timeout, + {GenServer, :call, + [data_name(name), {:await_closed_local_clusters, clusters}, timeout]}} + ) + + true -> + receive do + after + min(10, timeout - elapsed) -> + await_closed_local_clusters(name, clusters, timeout, started_at) + end + end + end + + defp rebuild_registry_reverse_index(name, shard) do + reverse = reg_by_pid_table(name, shard) + :ets.delete_all_objects(reverse) + + reg_by_key_table(name, shard) + |> :ets.tab2list() + |> Enum.each(fn {{cluster, key}, pid, meta, time, entry_node} -> + :ets.insert(reverse, {{pid, cluster, key}, meta, time, entry_node}) + end) + end + + defp rebuild_registry_claim_reverse_index(name, shard) do + reverse = reg_claim_by_pid_table(name, shard) + :ets.delete_all_objects(reverse) + + reg_claim_by_key_table(name, shard) + |> :ets.tab2list() + |> Enum.each(fn {{cluster, key, origin, generation, epoch}, pid, meta, time, seq} -> + :ets.insert( + reverse, + {{pid, cluster, key, origin, generation, epoch}, meta, time, seq} + ) + end) + end + + defp rebuild_pg_reverse_index(name, shard) do + reverse = pg_by_pid_table(name, shard) + :ets.delete_all_objects(reverse) + + pg_by_key_table(name, shard) + |> :ets.tab2list() + |> Enum.each(fn {{cluster, key, pid}, meta, time, entry_node} -> + :ets.insert(reverse, {{pid, cluster, key}, meta, time, entry_node}) + end) + end + + defp purge_inactive_cluster_rows(name, shard) do + clusters = + Enum.concat([ + Enum.map(:ets.tab2list(reg_by_key_table(name, shard)), fn + {{cluster, _key}, _pid, _meta, _time, _entry_node} -> cluster + end), + Enum.map(:ets.tab2list(reg_claim_by_key_table(name, shard)), fn + {{cluster, _key, _origin, _generation, _epoch}, _pid, _meta, _time, _seq} -> + cluster + end), + Enum.map(:ets.tab2list(pg_by_key_table(name, shard)), fn + {{cluster, _key, _pid}, _meta, _time, _entry_node} -> cluster + end), + Enum.map(:ets.tab2list(replica_cursor_table(name, shard)), fn {stream_id, _seq} -> + Protocol.stream_cluster(stream_id) + end) + ]) + |> Enum.reject(&is_nil/1) + |> Enum.uniq() + |> Enum.filter(&is_nil(local_cluster_epoch(name, &1))) + + Enum.each(clusters, fn cluster -> + :ets.select_delete(reg_by_key_table(name, shard), [ + {{{cluster, :_}, :_, :_, :_, :_}, [], [true]} + ]) + + :ets.select_delete(reg_claim_by_key_table(name, shard), [ + {{{cluster, :_, :_, :_, :_}, :_, :_, :_, :_}, [], [true]} + ]) + + :ets.select_delete(pg_by_key_table(name, shard), [ + {{{cluster, :_, :_}, :_, :_, :_}, [], [true]} + ]) + end) + + :ok = delete_replica_cursors_for_clusters(name, shard, clusters) + if clusters != [], do: remove_clusters(name, clusters) + :ok + end + def replica_stream_head(name, shard, stream_id) do case :ets.lookup(replica_stream_meta_table(name, shard), stream_id) do [{^stream_id, head, floor, applied}] -> {floor, head, applied} @@ -1620,13 +1836,49 @@ defmodule Group.Replica.Data do Enum.map(clusters, fn cluster -> epoch = local_cluster_epoch(state.name, cluster) :ets.delete(local_cluster_epochs_table(state.name), cluster) - if epoch, do: :ets.insert(closed_local_cluster_epochs_table(state.name), {cluster, epoch}) + + if epoch do + pending_shards = MapSet.new(0..(state.num_shards - 1)) + + :ets.insert( + closed_local_cluster_epochs_table(state.name), + {cluster, epoch, pending_shards} + ) + end + {cluster, epoch} end) {:reply, epochs, state} end + def handle_call({:mark_closed_cluster_shard, clusters, shard}, _from, state) do + completed = + Enum.reduce(clusters, [], fn cluster, acc -> + case :ets.lookup(closed_local_cluster_epochs_table(state.name), cluster) do + [{^cluster, epoch, pending_shards}] -> + pending_shards = MapSet.delete(pending_shards, shard) + + if MapSet.size(pending_shards) == 0 do + :ets.delete(closed_local_cluster_epochs_table(state.name), cluster) + [cluster | acc] + else + :ets.insert( + closed_local_cluster_epochs_table(state.name), + {cluster, epoch, pending_shards} + ) + + acc + end + + [] -> + acc + end + end) + + {:reply, Enum.reverse(completed), state} + end + def handle_call( {:put_remote_replica_info, shard, remote_node, generation, epoch_revision, epochs}, _from, @@ -1690,13 +1942,6 @@ defmodule Group.Replica.Data do {{:remote_authority_installs, remote_node}, 0} ) - for view_shard <- 0..(state.num_shards - 1) do - :ets.insert( - replication_meta_table(state.name), - {{:remote_view_info, view_shard, remote_node}, generation, epoch_revision, epoch_revision} - ) - end - rows = for {cluster, epoch} <- epochs, not is_nil(cluster), do: {{remote_node, cluster}, epoch} @@ -1886,7 +2131,7 @@ defmodule Group.Replica.Data do {:ok, %{name: name, num_shards: num_shards}} end - defp observe_remote_cluster_revision(name, remote_node, revision, num_shards) do + defp observe_remote_cluster_revision(name, remote_node, revision, _num_shards) do key = {:remote_epoch_observed, remote_node} case :ets.lookup(replication_meta_table(name), key) do @@ -1894,28 +2139,6 @@ defmodule Group.Replica.Data do _ -> :ets.insert(replication_meta_table(name), {key, revision}) end - for shard <- 0..(num_shards - 1) do - view_key = {:remote_view_info, shard, remote_node} - - case :ets.lookup(replication_meta_table(name), view_key) do - [{^view_key, _generation, _authoritative, observed}] when observed >= revision -> - :ok - - [{^view_key, generation, authoritative, _observed}] -> - :ets.insert( - replication_meta_table(name), - {view_key, generation, authoritative, revision} - ) - - [] -> - :ets.insert( - replication_meta_table(name), - {view_key, remote_generation(name, remote_node), - remote_cluster_epoch_revision(name, remote_node), revision} - ) - end - end - :ok end diff --git a/lib/group/replica/transport/tcp.ex b/lib/group/replica/transport/tcp.ex new file mode 100644 index 0000000..1914ac6 --- /dev/null +++ b/lib/group/replica/transport/tcp.ex @@ -0,0 +1,448 @@ +defmodule Group.Replica.Transport.TCP do + @moduledoc """ + Sideband TCP transport for replica data. + + Erlang distribution still carries Group discovery and authority controls. + Replica frames use independent TCP connections, so there is no ordering + relationship between a control message and its data lane. + + `try_send/5` never writes a socket. It reserves one slot in a bounded + per-peer queue and sends to a dedicated writer process. The writer may block + up to `:send_timeout` without blocking a Group shard. A full queue returns + `:busy`; a missing connection returns `:disconnected`. + + The endpoint capability in the dist-Erlang hello prevents an unrelated + socket client from injecting frames. This transport is intended for trusted + cluster networks; it does not encrypt traffic. Put it behind a private + network or a TLS/WebSocket tunnel when confidentiality is required. + + ## Options + + * `:ip` - listen address, default `{127, 0, 0, 1}` + * `:advertised_ip` - address placed in the hello, defaults to `:ip` + * `:port` - listen port, default `0` (ephemeral) + * `:max_queue` - maximum queued frames per peer, default `1_024` + * `:connect_timeout` - outbound connect timeout in milliseconds, default `1_000` + * `:send_timeout` - writer socket send timeout in milliseconds, default `1_000` + * `:reconnect_interval` - retry delay in milliseconds, default `50` + """ + + use GenServer + + @behaviour Group.Replica.Transport + + @impl true + def id, do: :group_sideband_tcp_v1 + + @impl true + def child_spec(opts) do + name = Keyword.fetch!(opts, :name) + + %{ + id: {__MODULE__, name}, + start: {__MODULE__, :start_link, [opts]}, + type: :worker, + restart: :permanent, + shutdown: 5_000 + } + end + + def start_link(opts) do + name = Keyword.fetch!(opts, :name) + GenServer.start_link(__MODULE__, opts, name: server_name(name)) + end + + @impl true + def descriptor(group, _opts) do + :persistent_term.get({__MODULE__, group, :descriptor}) + end + + @impl true + def try_send(group, target_node, shard, frame, _opts) do + case :ets.lookup(route_table(group), target_node) do + [{^target_node, writer, queued, max_queue}] -> + if :atomics.add_get(queued, 1, 1) <= max_queue do + if :erlang.send_nosuspend(writer, {:replica_frame, shard, frame}) do + :ok + else + :atomics.sub(queued, 1, 1) + :busy + end + else + :atomics.sub(queued, 1, 1) + :busy + end + + [] -> + :disconnected + end + end + + @impl true + def peer_up(group, remote_node, descriptor, _opts) do + send_manager(group, {:peer_up, remote_node, descriptor}) + end + + @impl true + def peer_down(group, remote_node, _opts) do + send_manager(group, {:peer_down, remote_node}) + end + + @doc false + def disconnect_peer(group, remote_node) do + GenServer.call(server_name(group), {:disable_peer, remote_node}) + end + + @doc false + def reconnect_peer(group, remote_node) do + GenServer.call(server_name(group), {:enable_peer, remote_node}) + end + + @doc false + def connected?(group, remote_node) do + :ets.member(route_table(group), remote_node) + end + + @doc false + def status(group) do + GenServer.call(server_name(group), :status) + end + + @impl true + def init(opts) do + Process.flag(:trap_exit, true) + group = Keyword.fetch!(opts, :name) + ip = Keyword.get(opts, :ip, {127, 0, 0, 1}) + advertised_ip = Keyword.get(opts, :advertised_ip, ip) + port = Keyword.get(opts, :port, 0) + + {:ok, listener} = + :gen_tcp.listen(port, [ + :binary, + packet: 4, + active: false, + reuseaddr: true, + ip: ip + ]) + + {:ok, {_listen_ip, listen_port}} = :inet.sockname(listener) + capability = :erlang.term_to_binary({node(), make_ref(), System.unique_integer()}) + descriptor = {:group_sideband_tcp_v1, advertised_ip, listen_port, capability} + :persistent_term.put({__MODULE__, group, :descriptor}, descriptor) + + :ets.new(route_table(group), [ + :named_table, + :public, + :set, + read_concurrency: true, + write_concurrency: true + ]) + + manager = self() + acceptor = spawn_link(fn -> accept_loop(listener, manager, group, capability) end) + + {:ok, + %{ + group: group, + listener: listener, + acceptor: acceptor, + peers: %{}, + writers: %{}, + inbound: %{}, + disabled: MapSet.new(), + max_queue: Keyword.get(opts, :max_queue, 1_024), + connect_timeout: Keyword.get(opts, :connect_timeout, 1_000), + send_timeout: Keyword.get(opts, :send_timeout, 1_000), + reconnect_interval: Keyword.get(opts, :reconnect_interval, 50) + }} + end + + @impl true + def handle_info({:peer_up, remote_node, descriptor}, state) do + state = %{state | peers: Map.put(state.peers, remote_node, descriptor)} + + state = + if MapSet.member?(state.disabled, remote_node) do + state + else + ensure_writer(state, remote_node) + end + + {:noreply, state} + end + + def handle_info({:peer_down, remote_node}, state) do + {:noreply, drop_peer(state, remote_node, true)} + end + + def handle_info({:writer_ready, remote_node, writer, queued}, state) do + if Map.get(state.writers, remote_node) == writer and + not MapSet.member?(state.disabled, remote_node) do + :ets.insert( + route_table(state.group), + {remote_node, writer, queued, state.max_queue} + ) + end + + {:noreply, state} + end + + def handle_info({:writer_failed, remote_node, writer}, state) do + {:noreply, writer_failed(state, remote_node, writer)} + end + + def handle_info({:reader_ready, source_node, reader}, state) do + {:noreply, %{state | inbound: Map.put(state.inbound, source_node, reader)}} + end + + def handle_info({:reconnect, remote_node}, state) do + {:noreply, ensure_writer(state, remote_node)} + end + + def handle_info({:EXIT, pid, _reason}, %{acceptor: pid} = state) do + {:stop, :acceptor_stopped, state} + end + + def handle_info({:EXIT, writer, _reason}, state) do + case Enum.find(state.writers, fn {_node, pid} -> pid == writer end) do + {remote_node, ^writer} -> + {:noreply, writer_failed(state, remote_node, writer)} + + nil -> + {:noreply, state} + end + end + + @impl true + def handle_call({:disable_peer, remote_node}, _from, state) do + state = %{state | disabled: MapSet.put(state.disabled, remote_node)} + {:reply, :ok, drop_writer(state, remote_node)} + end + + def handle_call({:enable_peer, remote_node}, _from, state) do + state = %{state | disabled: MapSet.delete(state.disabled, remote_node)} + {:reply, :ok, ensure_writer(state, remote_node)} + end + + def handle_call(:status, _from, state) do + {:reply, + %{ + peers: Map.keys(state.peers), + writers: Map.keys(state.writers), + connected: :ets.tab2list(route_table(state.group)) |> Enum.map(&elem(&1, 0)), + inbound: state.inbound, + disabled: MapSet.to_list(state.disabled) + }, state} + end + + @impl true + def terminate(_reason, state) do + :persistent_term.erase({__MODULE__, state.group, :descriptor}) + :gen_tcp.close(state.listener) + :ok + end + + defp send_manager(group, message) do + case Process.whereis(server_name(group)) do + nil -> + :ok + + pid -> + _ = :erlang.send_nosuspend(pid, message) + :ok + end + end + + defp ensure_writer(state, remote_node) do + cond do + MapSet.member?(state.disabled, remote_node) -> + state + + Map.has_key?(state.writers, remote_node) -> + state + + descriptor = Map.get(state.peers, remote_node) -> + manager = self() + + writer = + spawn_link(fn -> + writer_connect( + manager, + state.group, + remote_node, + descriptor, + state.connect_timeout, + state.send_timeout + ) + end) + + %{state | writers: Map.put(state.writers, remote_node, writer)} + + true -> + state + end + end + + defp writer_failed(state, remote_node, writer) do + if Map.get(state.writers, remote_node) == writer do + :ets.delete(route_table(state.group), remote_node) + state = %{state | writers: Map.delete(state.writers, remote_node)} + + if Map.has_key?(state.peers, remote_node) and + not MapSet.member?(state.disabled, remote_node) do + Process.send_after(self(), {:reconnect, remote_node}, state.reconnect_interval) + end + + state + else + state + end + end + + defp drop_peer(state, remote_node, remove_descriptor?) do + state = drop_writer(state, remote_node) + + if remove_descriptor? do + %{state | peers: Map.delete(state.peers, remote_node)} + else + state + end + end + + defp drop_writer(state, remote_node) do + :ets.delete(route_table(state.group), remote_node) + + case Map.pop(state.writers, remote_node) do + {nil, writers} -> + %{state | writers: writers} + + {writer, writers} -> + Process.exit(writer, :shutdown) + %{state | writers: writers} + end + end + + defp writer_connect( + manager, + group, + remote_node, + {:group_sideband_tcp_v1, host, port, capability}, + connect_timeout, + send_timeout + ) do + opts = [ + :binary, + packet: 4, + active: false, + send_timeout: send_timeout, + send_timeout_close: true + ] + + case :gen_tcp.connect(host, port, opts, connect_timeout) do + {:ok, socket} -> + case :gen_tcp.send(socket, :erlang.term_to_binary({:hello, group, node(), capability})) do + :ok -> + queued = :atomics.new(1, signed: false) + send(manager, {:writer_ready, remote_node, self(), queued}) + writer_loop(socket, manager, remote_node, queued) + + {:error, _reason} -> + :gen_tcp.close(socket) + send(manager, {:writer_failed, remote_node, self()}) + end + + {:error, _reason} -> + send(manager, {:writer_failed, remote_node, self()}) + end + end + + defp writer_connect(manager, _group, remote_node, _descriptor, _connect_timeout, _send_timeout) do + send(manager, {:writer_failed, remote_node, self()}) + end + + defp writer_loop(socket, manager, remote_node, queued) do + receive do + {:replica_frame, shard, frame} -> + result = :gen_tcp.send(socket, :erlang.term_to_binary({shard, frame})) + :atomics.sub(queued, 1, 1) + + case result do + :ok -> + writer_loop(socket, manager, remote_node, queued) + + {:error, _reason} -> + :gen_tcp.close(socket) + send(manager, {:writer_failed, remote_node, self()}) + end + end + end + + defp accept_loop(listener, manager, group, capability) do + case :gen_tcp.accept(listener) do + {:ok, socket} -> + reader = + spawn(fn -> + receive do + {:accepted_socket, accepted} -> + reader_handshake(accepted, manager, group, capability) + end + end) + + :ok = :gen_tcp.controlling_process(socket, reader) + send(reader, {:accepted_socket, socket}) + accept_loop(listener, manager, group, capability) + + {:error, :closed} -> + :ok + + {:error, _reason} -> + send(manager, {:EXIT, self(), :accept_failed}) + end + end + + defp reader_handshake(socket, manager, group, capability) do + with {:ok, payload} <- :gen_tcp.recv(socket, 0), + {:ok, {:hello, ^group, source_node, ^capability}} <- decode(payload), + true <- is_atom(source_node) do + send(manager, {:reader_ready, source_node, self()}) + reader_loop(socket, group, source_node) + else + _ -> :gen_tcp.close(socket) + end + end + + defp reader_loop(socket, group, source_node) do + case :gen_tcp.recv(socket, 0) do + {:ok, payload} -> + case decode_authenticated_frame(payload) do + {:ok, {shard, frame}} when is_integer(shard) and shard >= 0 -> + :ok = Group.Replica.Transport.deliver(group, source_node, shard, frame) + reader_loop(socket, group, source_node) + + _ -> + :gen_tcp.close(socket) + end + + {:error, _reason} -> + :ok + end + end + + defp decode(payload) do + {:ok, :erlang.binary_to_term(payload, [:safe])} + rescue + ArgumentError -> :error + end + + # The capability handshake above establishes the same trusted-cluster + # boundary as Erlang distribution. Replica metadata is an arbitrary BEAM + # term and may legitimately contain atoms not yet loaded on this node. + defp decode_authenticated_frame(payload) do + {:ok, :erlang.binary_to_term(payload)} + rescue + ArgumentError -> :error + end + + defp server_name(group), do: :"#{group}_replica_tcp_transport" + defp route_table(group), do: :"#{group}_replica_tcp_routes" +end diff --git a/mix.exs b/mix.exs index c618678..44fb164 100644 --- a/mix.exs +++ b/mix.exs @@ -34,7 +34,8 @@ defmodule Group.MixProject do defp deps do [ - {:ex_doc, "~> 0.30", only: :dev, runtime: false} + {:ex_doc, "~> 0.30", only: :dev, runtime: false}, + {:stream_data, "~> 1.4", only: :test} ] end diff --git a/mix.lock b/mix.lock index 0c19687..90b786f 100644 --- a/mix.lock +++ b/mix.lock @@ -5,4 +5,5 @@ "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, "makeup_erlang": {:hex, :makeup_erlang, "1.0.3", "4252d5d4098da7415c390e847c814bad3764c94a814a0b4245176215615e1035", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "953297c02582a33411ac6208f2c6e55f0e870df7f80da724ed613f10e6706afd"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, + "stream_data": {:hex, :stream_data, "1.4.0", "026f929db613aabea6208012ae9b8970d3fd5f88b3bdf26831bc536f98c42036", [:mix], [], "hexpm", "2b0ee3a340dcce1c8cf6302a763ee757d1e01c54d6e16d9069062509d68b1dc9"}, } diff --git a/test/README.md b/test/README.md index 6dc3dac..65c12aa 100644 --- a/test/README.md +++ b/test/README.md @@ -7,6 +7,7 @@ mix test # all tests mix test test/group_test.exs # local only mix test test/distributed_test.exs # distributed only mix test test/replica_adversarial_test.exs # seeded transport chaos +mix test test/replica_model_property_test.exs # shrinkable model-based histories ``` ## Test files @@ -16,6 +17,32 @@ mix test test/replica_adversarial_test.exs # seeded transport chaos | `group_test.exs` | Single-node: register/unregister, join/leave, members, monitor/demonitor, named clusters, concurrent operations | | `distributed_test.exs` | Multi-node: replication, peer discovery, node disconnect cleanup, partition healing, conflict resolution, event ordering, rolling restarts, and adversarial replica-transport loss/busy/snapshot recovery | | `replica_adversarial_test.exs` | Reproducible mixed-operation state machines: drops, busy returns, duplication, reordering, bounded delay, oplog pruning, conflicts, owner death, and named-cluster epoch churn, followed by exact convergence/dead-owner/internal-index checks | +| `replica_model_property_test.exs` | StreamData-generated and shrunk owner histories against an independent lifecycle oracle and scheduler-controlled replica transport | + +## Model-based and formal checks + +`replica_model_property_test.exs` runs real Group instances on three peer VMs. +The controlled transport queues each replica frame so generated commands can +deliver, duplicate, drop, reorder, or strand it. After the bounded-fault +prefix, the test enables fair delivery and compares every tracked registry and +PG key against an independent application-level lifecycle oracle. It also +requires internal replica indexes to be consistent, every retained owner to be +alive, and registry conflict losers to be dead. Restart, pruning, and named +cluster histories retain independent C-owned state while A recovers, so repair +cannot pass merely by making one origin and one receiver agree. + +StreamData reports the ExUnit seed and shrinks a failure to its smallest command +history. Local defaults are intentionally quick. Increase the budgets without +changing the generator: + +```bash +GROUP_MODEL_RUNS=1000 GROUP_MODEL_COMMANDS=100 \ + mix test test/replica_model_property_test.exs +``` + +The independent TLA+ model and TLC configuration live in `test/formal/`. +See [`formal/README.md`](formal/README.md) for its checked invariants, finite +model bounds, and run command. ## How distribution works @@ -203,6 +230,12 @@ can return `:busy`, drop selected frame types, duplicate or delay frames, and capture frames for explicit stale-generation/epoch replay. Its `{:chaos, opts}` mode is deterministic for a given frame, which makes failures reproducible. +`Group.ControlledReplicaTransport` is the model-test transport. It queues frames +at the test process without scheduling timers; `Group.ReplicaModelScheduler` +then owns the exact delivery schedule. These roles are separate so the existing +timing-oriented regressions retain their original mechanics while property +failures can be replayed and shrunk exactly. + The distributed anti-entropy tests cover dropped creates and deletes, cursor gaps, globally pruned multi-stream oplogs, exact snapshot fallback, malformed authority, stale frame replay, lease expiry on a live VM, and multi-shard @@ -217,13 +250,22 @@ deliver data before authority to prove rejection does not advance the cursor and the same frame applies after authority repair. Concurrent snapshot tests require every advertised revision to contain exactly that many unique named epochs, and heartbeat tests prove observed revisions cannot advance the exact -authority marker. +authority marker. Crash-window tests interrupt journal, dual-index, receive +cursor, and named-cluster close updates, then require startup repair to remove +every invisible row and temporary close barrier. A three-node sideband TCP test +disconnects one origin's real socket, prunes its oplog, reconnects it, and +requires snapshot recovery without changing the third node's independent +registry or PG state. `Group.TestCluster.assert_replica_consistent/1` checks the public dual indexes plus registry claim authority, oplog/order equivalence, and contiguous retained stream ranges. Seeded tests additionally require every PID retained as authority to still be alive after convergence. +The isolated mutation runner in `test/mutation/` disables individual protocol +guards and repair steps only in copied checkouts. See +[`mutation/README.md`](mutation/README.md) for the command and artifact format. + ## Typical test patterns ### Basic replication test diff --git a/test/distributed_test.exs b/test/distributed_test.exs index bf34406..e43d1c4 100644 --- a/test/distributed_test.exs +++ b/test/distributed_test.exs @@ -5208,6 +5208,14 @@ defmodule Group.DistributedTest do assert :ok = TestCluster.rpc!(node_b, Group.TestCluster, :assert_replica_consistent, [name]) + assert :ok = + TestCluster.rpc!( + node_b, + Group.TestCluster, + :assert_replica_origin_purged, + [name, node_a] + ) + {:ok, _pid} = TestCluster.start_group(node_a, opts) TestCluster.assert_eventually(fn -> @@ -5315,6 +5323,493 @@ defmodule Group.DistributedTest do TestCluster.rpc!(node, Group.TestCluster, :assert_replica_consistent, [name]) end end + + @tag timeout: 60_000 + test "mixed-generation data is rejected even when its epoch matches current authority" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_generation_guard_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name]) + end) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + key = "anti-entropy/mixed-generation" + meta = %{forged: true} + pid = TestCluster.spawn_register(node_a, name, key, meta) + TestCluster.flush_shards(node_a, name) + + current_generation = + TestCluster.rpc!(node_a, Group.Replica.Data, :generation, [name]) + + stream_id = + Group.Replica.Protocol.stream_id( + name, + node_a, + make_ref(), + 0, + nil, + current_generation + ) + + mutation = {:register, nil, key, pid, meta, System.monotonic_time(), node_a} + + :ok = + TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + name, + node_a, + 0, + {:delta_batch, Group.Replica.Protocol.version(), [{stream_id, 1, [{1, [mutation]}], 1}]} + ]) + + TestCluster.flush_shards(node_b, name) + assert TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + + assert TestCluster.rpc!(node_b, Group.Replica.Data, :replica_cursor, [name, 0, stream_id]) == + 0 + end + + @tag timeout: 60_000 + test "an out-of-order delta cannot advance the cursor across a missing sequence" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_gap_guard_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_b in TestCluster.rpc!(node_a, Group, :nodes, [name]) + end) + + :ok = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [ + name, + {:capture_drop, [:delta_batch]} + ]) + + first_key = "anti-entropy/gap/first" + second_key = "anti-entropy/gap/second" + group_key = "anti-entropy/gap/group" + first_pid = TestCluster.spawn_register(node_a, name, first_key, %{seq: 1}) + second_pid = TestCluster.spawn_register(node_a, name, second_key, %{seq: 2}) + member_pid = TestCluster.spawn_join(node_a, name, group_key, %{seq: 3}) + TestCluster.flush_shards(node_a, name) + + captured = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :captured, [name]) + + frames_by_first_seq = + Map.new(captured, fn + {_target, shard, + {:delta_batch, _version, [{_stream_id, first_seq, _records, _head}]} = frame} -> + {first_seq, {shard, frame}} + end) + + {shard, {:delta_batch, _version, [{stream_id, 2, _records, _head}]} = second_frame} = + Map.fetch!(frames_by_first_seq, 2) + + :ok = + TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + name, + node_a, + shard, + second_frame + ]) + + TestCluster.flush_shards(node_b, name) + + assert TestCluster.rpc!(node_b, Group.Replica.Data, :replica_cursor, [ + name, + shard, + stream_id + ]) == 0 + + assert TestCluster.rpc!(node_b, Group, :lookup, [name, second_key]) == nil + + {^shard, first_frame} = Map.fetch!(frames_by_first_seq, 1) + {^shard, third_frame} = Map.fetch!(frames_by_first_seq, 3) + + for frame <- [first_frame, second_frame, third_frame] do + :ok = + TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + name, + node_a, + shard, + frame + ]) + end + + TestCluster.flush_shards(node_b, name) + + assert match?( + {^first_pid, %{seq: 1}}, + TestCluster.rpc!(node_b, Group, :lookup, [name, first_key]) + ) + + assert match?( + {^second_pid, %{seq: 2}}, + TestCluster.rpc!(node_b, Group, :lookup, [name, second_key]) + ) + + assert match?( + [{^member_pid, %{seq: 3}}], + TestCluster.rpc!(node_b, Group, :members, [name, group_key]) + ) + + # Model a receiver crash after all materialized ETS writes but before + # its cursor write. Replaying the accepted prefix must be exactly + # idempotent for both registry claims and PG membership. + :ok = + TestCluster.rpc!(node_b, Group.Replica.Data, :put_replica_cursor, [ + name, + shard, + stream_id, + 0 + ]) + + for frame <- [first_frame, second_frame, third_frame] do + :ok = + TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + name, + node_a, + shard, + frame + ]) + end + + TestCluster.flush_shards(node_b, name) + + assert TestCluster.rpc!(node_b, Group.Replica.Data, :replica_cursor, [ + name, + shard, + stream_id + ]) == 3 + + assert match?( + {^first_pid, %{seq: 1}}, + TestCluster.rpc!(node_b, Group, :lookup, [name, first_key]) + ) + + assert match?( + {^second_pid, %{seq: 2}}, + TestCluster.rpc!(node_b, Group, :lookup, [name, second_key]) + ) + + assert match?( + [{^member_pid, %{seq: 3}}], + TestCluster.rpc!(node_b, Group, :members, [name, group_key]) + ) + + assert :ok = + TestCluster.rpc!(node_b, Group.TestCluster, :assert_replica_consistent, [name]) + end + + @tag timeout: 60_000 + test "a new full authority generation purges nonzero shards before any lane-down signal" do + peers = TestCluster.start_peers(2) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}] = peers + name = :"anti_entropy_authority_fanout_#{System.unique_integer([:positive])}" + shards = 2 + + opts = [ + name: name, + shards: shards, + replica_transport: Group.TestReplicaTransport, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name]) + end) + + key = + Enum.find(Stream.iterate(0, &(&1 + 1)), fn suffix -> + :erlang.phash2({nil, "anti-entropy/authority-fanout/#{suffix}"}, shards) == 1 + end) + |> then(&"anti-entropy/authority-fanout/#{&1}") + + pid = TestCluster.spawn_register(node_a, name, key, %{generation: :old}) + + TestCluster.assert_eventually(fn -> + match?({^pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + + forwarder = TestCluster.spawn_monitor_forwarder(node_b, name, :all, self()) + assert_receive {:monitor_ready, ^forwarder}, 5_000 + + a_control = TestCluster.rpc!(node_a, Process, :whereis, [shard_name(name, 0)]) + b_control = TestCluster.rpc!(node_b, Process, :whereis, [shard_name(name, 0)]) + b_lane = TestCluster.rpc!(node_b, Process, :whereis, [shard_name(name, 1)]) + new_generation = make_ref() + + new_key = + Enum.find(Stream.iterate(0, &(&1 + 1)), fn suffix -> + :erlang.phash2({nil, "anti-entropy/authority-fanout/new/#{suffix}"}, shards) == 1 + end) + |> then(&"anti-entropy/authority-fanout/new/#{&1}") + + stream_id = + Group.Replica.Protocol.stream_id( + name, + node_a, + new_generation, + 1, + nil, + new_generation + ) + + new_mutation = + {:register, nil, new_key, pid, %{generation: :new}, System.system_time(), node_a} + + new_frame = + {:delta_batch, Group.Replica.Protocol.version(), + [{stream_id, 1, [{1, [new_mutation]}], 1}]} + + :ok = TestCluster.rpc!(node_b, :sys, :suspend, [b_lane]) + + # Queue new-generation lane data before the control shard can enqueue + # its local authority marker. Shared authority will be current by the + # time this frame runs, but shard 1 must still reject it until its own + # old-generation purge has completed. + :ok = + TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + name, + node_a, + 1, + new_frame + ]) + + send( + b_control, + {:replica_hello, a_control, Group.Replica.Protocol.version(), new_generation, 0, + [{nil, new_generation}], Group.TestReplicaTransport.id(), + Group.TestReplicaTransport.descriptor(name, [])} + ) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(node_b, Group.Replica.Data, :remote_generation, [name, node_a]) == + new_generation + end) + + :ok = TestCluster.rpc!(node_b, :sys, :resume, [b_lane]) + TestCluster.flush_shards(node_b, name) + + assert TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + assert TestCluster.rpc!(node_b, Group, :lookup, [name, new_key]) == nil + refute_receive {:got_event, %Group.Event{key: ^new_key}}, 100 + + assert TestCluster.rpc!(node_b, Group.Replica.Data, :replica_cursor, [ + name, + 1, + stream_id + ]) == 0 + + :ok = + TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + name, + node_a, + 1, + new_frame + ]) + + TestCluster.flush_shards(node_b, name) + + assert TestCluster.rpc!(node_b, Group, :lookup, [name, new_key]) == + {pid, %{generation: :new}} + + assert :ok = TestCluster.rpc!(node_b, Group.TestCluster, :assert_replica_consistent, [name]) + end + + @tag timeout: 60_000 + test "three nodes recover a pruned origin over sideband TCP without disturbing another origin" do + peers = TestCluster.start_peers(3) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}, {_, node_c}] = peers + name = :"anti_entropy_sideband_tcp_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 2, + replica_transport: + {Group.Replica.Transport.TCP, + [ + max_queue: 16, + connect_timeout: 250, + send_timeout: 250, + reconnect_interval: 10 + ]}, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 1_000, + replicated_oplog_max_entries: 2 + ] + + start_group_on_peers(peers, opts) + + nodes = [node_a, node_b, node_c] + + TestCluster.assert_eventually( + fn -> + Enum.all?(nodes, fn source -> + Enum.all?(nodes -- [source], fn target -> + TestCluster.rpc!( + source, + Group.Replica.Transport.TCP, + :connected?, + [name, target] + ) + end) + end) + end, + timeout: 10_000 + ) + + stale_key = "sideband/a/stale" + fresh_key = "sideband/a/fresh" + c_key = "sideband/c/independent" + c_group = "sideband/c/group" + + stale_pid = TestCluster.spawn_register(node_a, name, stale_key, %{origin: :a}) + + c_pid = + TestCluster.spawn_register_and_join( + node_c, + name, + c_key, + %{origin: :c}, + c_group, + %{origin: :c} + ) + + TestCluster.assert_eventually(fn -> + Enum.all?(nodes, fn receiver -> + match?( + {^stale_pid, %{origin: :a}}, + TestCluster.rpc!(receiver, Group, :lookup, [name, stale_key]) + ) and + match?( + {^c_pid, %{origin: :c}}, + TestCluster.rpc!(receiver, Group, :lookup, [name, c_key]) + ) and + match?( + [{^c_pid, %{origin: :c}}], + TestCluster.rpc!(receiver, Group, :members, [name, c_group]) + ) + end) + end) + + :ok = + TestCluster.rpc!(node_a, Group.Replica.Transport.TCP, :disconnect_peer, [name, node_b]) + + old_reader = + TestCluster.rpc!(node_b, Group.Replica.Transport.TCP, :status, [name]) + |> get_in([:inbound, node_a]) + + refute TestCluster.rpc!( + node_a, + Group.Replica.Transport.TCP, + :connected?, + [name, node_b] + ) + + true = TestCluster.rpc!(node_a, Process, :exit, [stale_pid, :kill]) + + for i <- 1..6 do + churn_key = "sideband/a/churn/#{i}" + churn_pid = TestCluster.spawn_register(node_a, name, churn_key, %{i: i}) + true = TestCluster.rpc!(node_a, Process, :exit, [churn_pid, :kill]) + end + + fresh_pid = TestCluster.spawn_register(node_a, name, fresh_key, %{origin: :a, fresh: true}) + TestCluster.flush_shards(node_a, name) + + assert match?( + {^stale_pid, %{origin: :a}}, + TestCluster.rpc!(node_b, Group, :lookup, [name, stale_key]) + ) + + assert TestCluster.rpc!(node_b, Group, :lookup, [name, fresh_key]) == nil + + assert match?( + {^c_pid, %{origin: :c}}, + TestCluster.rpc!(node_b, Group, :lookup, [name, c_key]) + ) + + :ok = + TestCluster.rpc!(node_a, Group.Replica.Transport.TCP, :reconnect_peer, [name, node_b]) + + TestCluster.assert_eventually(fn -> + new_reader = + TestCluster.rpc!(node_b, Group.Replica.Transport.TCP, :status, [name]) + |> get_in([:inbound, node_a]) + + is_pid(new_reader) and new_reader != old_reader + end) + + TestCluster.assert_eventually( + fn -> + TestCluster.rpc!( + node_a, + Group.Replica.Transport.TCP, + :connected?, + [name, node_b] + ) and + TestCluster.rpc!(node_b, Group, :lookup, [name, stale_key]) == nil and + match?( + {^fresh_pid, %{origin: :a, fresh: true}}, + TestCluster.rpc!(node_b, Group, :lookup, [name, fresh_key]) + ) and + match?( + {^c_pid, %{origin: :c}}, + TestCluster.rpc!(node_b, Group, :lookup, [name, c_key]) + ) and + match?( + [{^c_pid, %{origin: :c}}], + TestCluster.rpc!(node_b, Group, :members, [name, c_group]) + ) + end, + timeout: 10_000 + ) + + for receiver <- nodes do + assert :ok = + TestCluster.rpc!( + receiver, + Group.TestCluster, + :assert_replica_consistent, + [name] + ) + end + end end # Helpers for event assertion tests diff --git a/test/formal/GroupAntiEntropy.cfg b/test/formal/GroupAntiEntropy.cfg new file mode 100644 index 0000000..566fae5 --- /dev/null +++ b/test/formal/GroupAntiEntropy.cfg @@ -0,0 +1,17 @@ +SPECIFICATION Spec + +CONSTANTS + Nodes = {n1, n2, n3} + Origins = {n1} + Keys = {k1} + MaxSeq = 2 + OplogBound = 1 + MaxMessages = 1 + +INVARIANTS + TypeOK + BoundedJournal + CurrentReplicaIsAStreamPrefix + +PROPERTY + HealedConvergence diff --git a/test/formal/GroupAntiEntropy.tla b/test/formal/GroupAntiEntropy.tla new file mode 100644 index 0000000..96a3238 --- /dev/null +++ b/test/formal/GroupAntiEntropy.tla @@ -0,0 +1,481 @@ +------------------------- MODULE GroupAntiEntropy ------------------------- +EXTENDS Integers, FiniteSets, TLC + +(* +An abstract model of Group's per-origin anti-entropy stream. + +The model intentionally does not duplicate the Elixir data structures. It +models the protocol contract: exact generation/epoch authority, sequenced +deltas, bounded retained prefixes, exact snapshot fallback, arbitrary finite +loss/reordering/duplication, and fair repair after the network heals. +*) + +CONSTANTS Nodes, Origins, Keys, MaxSeq, OplogBound, MaxMessages + +ASSUME /\ IsFiniteSet(Nodes) + /\ Cardinality(Nodes) >= 2 + /\ Origins \subseteq Nodes + /\ Cardinality(Origins) >= 1 + /\ IsFiniteSet(Keys) + /\ Cardinality(Keys) >= 1 + /\ MaxSeq >= 1 + /\ OplogBound >= 1 + /\ MaxMessages >= 1 + +Seq == 1..MaxSeq +Generations == 0..2 +Revisions == 0..4 +Epochs == 0..4 +BoolMap == [Keys -> BOOLEAN] +EmptyView == [key \in Keys |-> FALSE] +EmptyRecord == [key |-> CHOOSE key \in Keys : TRUE, value |-> FALSE] + +HelloMessages == + [kind : {"hello"}, + from : Nodes, + to : Nodes, + wireGeneration : Generations, + wireRevision : Revisions, + wireEpoch : Epochs, + wireActive : BOOLEAN] + +DeltaMessages == + [kind : {"delta"}, + from : Nodes, + to : Nodes, + wireGeneration : Generations, + wireRevision : Revisions, + wireEpoch : Epochs, + seq : Seq, + key : Keys, + value : BOOLEAN] + +SnapshotMessages == + [kind : {"snapshot"}, + from : Nodes, + to : Nodes, + wireGeneration : Generations, + wireRevision : Revisions, + wireEpoch : Epochs, + seq : 0..MaxSeq, + state : BoolMap] + +Message == HelloMessages \union DeltaMessages \union SnapshotMessages + +VARIABLES phase, + generation, + revision, + epoch, + active, + truth, + head, + floor, + history, + authorityGeneration, + authorityRevision, + authorityEpoch, + authorityActive, + cursor, + replica, + messages + +vars == + <> + +RECURSIVE Replay(_, _) +Replay(records, n) == + IF n = 0 + THEN EmptyView + ELSE [Replay(records, n - 1) EXCEPT + ![records[n].key] = records[n].value] + +CurrentAuthority(receiver, origin) == + /\ authorityGeneration[receiver][origin] = generation[origin] + /\ authorityRevision[receiver][origin] = revision[origin] + /\ authorityEpoch[receiver][origin] = epoch[origin] + /\ authorityActive[receiver][origin] = active[origin] + +PairConverged(receiver, origin) == + IF receiver = origin + THEN TRUE + ELSE + /\ CurrentAuthority(receiver, origin) + /\ IF active[origin] + THEN /\ cursor[receiver][origin] = head[origin] + /\ replica[receiver][origin] = truth[origin] + ELSE /\ cursor[receiver][origin] = 0 + /\ replica[receiver][origin] = EmptyView + +Converged == + \A receiver \in Nodes : + \A origin \in Origins : + PairConverged(receiver, origin) + +Init == + /\ phase = "faulting" + /\ generation = [node \in Nodes |-> 1] + /\ revision = [node \in Nodes |-> 0] + /\ epoch = [node \in Nodes |-> 0] + /\ active = [node \in Nodes |-> FALSE] + /\ truth = [node \in Nodes |-> EmptyView] + /\ head = [node \in Nodes |-> 0] + /\ floor = [node \in Nodes |-> 1] + /\ history = [node \in Nodes |-> + [seq \in Seq |-> EmptyRecord]] + /\ authorityGeneration = + [receiver \in Nodes |-> [origin \in Nodes |-> 0]] + /\ authorityRevision = + [receiver \in Nodes |-> [origin \in Nodes |-> 0]] + /\ authorityEpoch = + [receiver \in Nodes |-> [origin \in Nodes |-> 0]] + /\ authorityActive = + [receiver \in Nodes |-> [origin \in Nodes |-> FALSE]] + /\ cursor = + [receiver \in Nodes |-> [origin \in Nodes |-> 0]] + /\ replica = + [receiver \in Nodes |-> [origin \in Nodes |-> EmptyView]] + /\ messages = {} + +Open(origin) == + /\ phase = "faulting" + /\ ~active[origin] + /\ revision[origin] < 4 + /\ epoch[origin] < 4 + /\ active' = [active EXCEPT ![origin] = TRUE] + /\ revision' = [revision EXCEPT ![origin] = @ + 1] + /\ epoch' = [epoch EXCEPT ![origin] = @ + 1] + /\ truth' = [truth EXCEPT ![origin] = EmptyView] + /\ head' = [head EXCEPT ![origin] = 0] + /\ floor' = [floor EXCEPT ![origin] = 1] + /\ history' = [history EXCEPT + ![origin] = [seq \in Seq |-> EmptyRecord]] + /\ UNCHANGED <> + +Close(origin) == + /\ phase = "faulting" + /\ active[origin] + /\ revision[origin] < 4 + /\ active' = [active EXCEPT ![origin] = FALSE] + /\ revision' = [revision EXCEPT ![origin] = @ + 1] + /\ truth' = [truth EXCEPT ![origin] = EmptyView] + /\ head' = [head EXCEPT ![origin] = 0] + /\ floor' = [floor EXCEPT ![origin] = 1] + /\ history' = [history EXCEPT + ![origin] = [seq \in Seq |-> EmptyRecord]] + /\ UNCHANGED <> + +Restart(origin) == + /\ phase = "faulting" + /\ generation[origin] < 2 + /\ generation' = [generation EXCEPT ![origin] = @ + 1] + /\ revision' = [revision EXCEPT ![origin] = 0] + /\ epoch' = [epoch EXCEPT ![origin] = 0] + /\ active' = [active EXCEPT ![origin] = FALSE] + /\ truth' = [truth EXCEPT ![origin] = EmptyView] + /\ head' = [head EXCEPT ![origin] = 0] + /\ floor' = [floor EXCEPT ![origin] = 1] + /\ history' = [history EXCEPT + ![origin] = [seq \in Seq |-> EmptyRecord]] + /\ UNCHANGED <> + +Mutate(origin, key, value) == + /\ phase = "faulting" + /\ active[origin] + /\ head[origin] < MaxSeq + /\ LET next == head[origin] + 1 + nextFloor == IF next - floor[origin] + 1 > OplogBound + THEN floor[origin] + 1 + ELSE floor[origin] + IN /\ history' = + [history EXCEPT + ![origin][next] = [key |-> key, value |-> value]] + /\ truth' = [truth EXCEPT ![origin][key] = value] + /\ head' = [head EXCEPT ![origin] = next] + /\ floor' = [floor EXCEPT ![origin] = nextFloor] + /\ UNCHANGED <> + +SendHello(origin, receiver) == + /\ origin # receiver + /\ Cardinality(messages) < MaxMessages + /\ messages' = + messages \union + {[kind |-> "hello", + from |-> origin, + to |-> receiver, + wireGeneration |-> generation[origin], + wireRevision |-> revision[origin], + wireEpoch |-> epoch[origin], + wireActive |-> active[origin]]} + /\ UNCHANGED <> + +SendDelta(origin, receiver, seq) == + /\ origin # receiver + /\ active[origin] + /\ seq \in floor[origin]..head[origin] + /\ Cardinality(messages) < MaxMessages + /\ LET record == history[origin][seq] + IN messages' = + messages \union + {[kind |-> "delta", + from |-> origin, + to |-> receiver, + wireGeneration |-> generation[origin], + wireRevision |-> revision[origin], + wireEpoch |-> epoch[origin], + seq |-> seq, + key |-> record.key, + value |-> record.value]} + /\ UNCHANGED <> + +SendSnapshot(origin, receiver) == + /\ origin # receiver + /\ active[origin] + /\ Cardinality(messages) < MaxMessages + /\ messages' = + messages \union + {[kind |-> "snapshot", + from |-> origin, + to |-> receiver, + wireGeneration |-> generation[origin], + wireRevision |-> revision[origin], + wireEpoch |-> epoch[origin], + seq |-> head[origin], + state |-> truth[origin]]} + /\ UNCHANGED <> + +FreshHello(message) == + \/ message.wireGeneration > authorityGeneration[message.to][message.from] + \/ /\ message.wireGeneration = + authorityGeneration[message.to][message.from] + /\ message.wireRevision >= authorityRevision[message.to][message.from] + +DeliverHello(message) == + /\ message.kind = "hello" + /\ LET changed == + \/ message.wireGeneration # + authorityGeneration[message.to][message.from] + \/ message.wireRevision # + authorityRevision[message.to][message.from] + \/ message.wireEpoch # + authorityEpoch[message.to][message.from] + \/ message.wireActive # + authorityActive[message.to][message.from] + install == FreshHello(message) + IN /\ authorityGeneration' = + IF install + THEN [authorityGeneration EXCEPT + ![message.to][message.from] = message.wireGeneration] + ELSE authorityGeneration + /\ authorityRevision' = + IF install + THEN [authorityRevision EXCEPT + ![message.to][message.from] = message.wireRevision] + ELSE authorityRevision + /\ authorityEpoch' = + IF install + THEN [authorityEpoch EXCEPT + ![message.to][message.from] = message.wireEpoch] + ELSE authorityEpoch + /\ authorityActive' = + IF install + THEN [authorityActive EXCEPT + ![message.to][message.from] = message.wireActive] + ELSE authorityActive + /\ cursor' = + IF install /\ (changed \/ ~message.wireActive) + THEN [cursor EXCEPT ![message.to][message.from] = 0] + ELSE cursor + /\ replica' = + IF install /\ (changed \/ ~message.wireActive) + THEN [replica EXCEPT + ![message.to][message.from] = EmptyView] + ELSE replica + /\ UNCHANGED <> + +ValidData(message) == + /\ authorityGeneration[message.to][message.from] = message.wireGeneration + /\ authorityRevision[message.to][message.from] = message.wireRevision + /\ authorityEpoch[message.to][message.from] = message.wireEpoch + /\ authorityActive[message.to][message.from] + +DeliverDelta(message) == + /\ message.kind = "delta" + /\ IF ValidData(message) /\ + message.seq = cursor[message.to][message.from] + 1 + THEN /\ cursor' = + [cursor EXCEPT + ![message.to][message.from] = message.seq] + /\ replica' = + [replica EXCEPT + ![message.to][message.from][message.key] = message.value] + ELSE /\ UNCHANGED cursor + /\ UNCHANGED replica + /\ UNCHANGED <> + +DeliverSnapshot(message) == + /\ message.kind = "snapshot" + /\ IF ValidData(message) /\ + message.seq >= cursor[message.to][message.from] + THEN /\ cursor' = + [cursor EXCEPT + ![message.to][message.from] = message.seq] + /\ replica' = + [replica EXCEPT + ![message.to][message.from] = message.state] + ELSE /\ UNCHANGED cursor + /\ UNCHANGED replica + /\ UNCHANGED <> + +Deliver(message) == + /\ message \in messages + /\ \/ DeliverHello(message) + \/ DeliverDelta(message) + \/ DeliverSnapshot(message) + +Drop(message) == + /\ phase = "faulting" + /\ message \in messages + /\ messages' = messages \ {message} + /\ UNCHANGED <> + +Heal == + /\ phase = "faulting" + /\ phase' = "healed" + /\ UNCHANGED <> + +(* +After healing, Repair represents one fair successful hello/head/need response. +If the retained prefix no longer contains the next sequence, it performs an +exact snapshot replacement. Otherwise it applies exactly the next delta. +*) +Repair(receiver, origin) == + /\ phase = "healed" + /\ receiver # origin + /\ ~PairConverged(receiver, origin) + /\ IF ~CurrentAuthority(receiver, origin) + THEN /\ authorityGeneration' = + [authorityGeneration EXCEPT + ![receiver][origin] = generation[origin]] + /\ authorityRevision' = + [authorityRevision EXCEPT + ![receiver][origin] = revision[origin]] + /\ authorityEpoch' = + [authorityEpoch EXCEPT + ![receiver][origin] = epoch[origin]] + /\ authorityActive' = + [authorityActive EXCEPT + ![receiver][origin] = active[origin]] + /\ cursor' = [cursor EXCEPT ![receiver][origin] = 0] + /\ replica' = + [replica EXCEPT ![receiver][origin] = EmptyView] + ELSE IF ~active[origin] + THEN /\ UNCHANGED <> + /\ cursor' = [cursor EXCEPT ![receiver][origin] = 0] + /\ replica' = + [replica EXCEPT ![receiver][origin] = EmptyView] + ELSE IF cursor[receiver][origin] + 1 < floor[origin] + THEN /\ UNCHANGED <> + /\ cursor' = + [cursor EXCEPT ![receiver][origin] = head[origin]] + /\ replica' = + [replica EXCEPT ![receiver][origin] = truth[origin]] + ELSE LET next == cursor[receiver][origin] + 1 + record == history[origin][next] + IN /\ UNCHANGED <> + /\ cursor' = [cursor EXCEPT ![receiver][origin] = next] + /\ replica' = + [replica EXCEPT + ![receiver][origin][record.key] = record.value] + /\ UNCHANGED <> + +Next == + \/ \E origin \in Origins : Open(origin) + \/ \E origin \in Origins : Close(origin) + \/ \E origin \in Origins : Restart(origin) + \/ \E origin \in Origins, key \in Keys, value \in BOOLEAN : + Mutate(origin, key, value) + \/ \E origin \in Origins, receiver \in Nodes : + SendHello(origin, receiver) + \/ \E origin \in Origins, receiver \in Nodes, seq \in Seq : + SendDelta(origin, receiver, seq) + \/ \E origin \in Origins, receiver \in Nodes : + SendSnapshot(origin, receiver) + \/ \E message \in messages : Deliver(message) + \/ \E message \in messages : Drop(message) + \/ Heal + \/ \E receiver \in Nodes, origin \in Origins : + Repair(receiver, origin) + +TypeOK == + /\ phase \in {"faulting", "healed"} + /\ generation \in [Nodes -> Generations] + /\ revision \in [Nodes -> Revisions] + /\ epoch \in [Nodes -> Epochs] + /\ active \in [Nodes -> BOOLEAN] + /\ truth \in [Nodes -> BoolMap] + /\ head \in [Nodes -> 0..MaxSeq] + /\ floor \in [Nodes -> 1..(MaxSeq + 1)] + /\ history \in [Nodes -> [Seq -> [key : Keys, value : BOOLEAN]]] + /\ authorityGeneration \in [Nodes -> [Nodes -> Generations]] + /\ authorityRevision \in [Nodes -> [Nodes -> Revisions]] + /\ authorityEpoch \in [Nodes -> [Nodes -> Epochs]] + /\ authorityActive \in [Nodes -> [Nodes -> BOOLEAN]] + /\ cursor \in [Nodes -> [Nodes -> 0..MaxSeq]] + /\ replica \in [Nodes -> [Nodes -> BoolMap]] + /\ messages \subseteq Message + +BoundedJournal == + \A origin \in Origins : + /\ floor[origin] <= head[origin] + 1 + /\ head[origin] - floor[origin] + 1 <= OplogBound + +CurrentReplicaIsAStreamPrefix == + \A receiver \in Nodes : + \A origin \in Origins : + IF receiver # origin /\ + CurrentAuthority(receiver, origin) /\ + active[origin] + THEN /\ cursor[receiver][origin] <= head[origin] + /\ replica[receiver][origin] = + Replay(history[origin], cursor[receiver][origin]) + ELSE TRUE + +HealedConvergence == + phase = "healed" ~> Converged + +Spec == + /\ Init + /\ [][Next]_vars + /\ \A receiver \in Nodes : + \A origin \in Origins : + WF_vars(Repair(receiver, origin)) + +============================================================================= diff --git a/test/formal/README.md b/test/formal/README.md new file mode 100644 index 0000000..05c2c10 --- /dev/null +++ b/test/formal/README.md @@ -0,0 +1,44 @@ +# Group anti-entropy formal model + +`GroupAntiEntropy.tla` is an independent finite-state model of the replica +contract. It covers: + +- generation and named-cluster epoch fencing; +- arbitrary finite frame loss, duplication, and reordering; +- contiguous sequence application; +- bounded oplog pruning; +- exact per-origin snapshot fallback; and +- fair convergence after healing. + +The default TLC configuration uses three nodes: one origin and two independent +receivers. The origin has one key, a two-record stream, a one-record oplog, and +the system retains one arbitrary network frame. This forces delta repair, +snapshot fallback, stale-frame fencing, and independent recovery at both +receivers. The retained frame may be redelivered for duplication, while +nondeterministic sequence selection and delivery model out-of-order arrival +without paying the state-space cost of every two-frame set. + +The TLA+ protocol state is deliberately factored per origin: no transition for +one origin reads or writes another origin's stream. Checking multiple origins +in this model therefore forms a Cartesian product of the same state machine +rather than adding an interaction. Concurrent A/C authority, registry conflict +projection, and preservation of C-owned state while A recovers are instead +driven against three real BEAM nodes by `replica_model_property_test.exs`. + +Run it with Java 17 or later and a current `tla2tools.jar`: + +```bash +TLA_JAR=/path/to/tla2tools.jar test/formal/check.sh +``` + +`TLC_WORKERS` controls worker concurrency and defaults to 4. `TLA_CONFIG` can +point at an alternate finite configuration. + +TLC proves the listed invariants and liveness property for the configured +finite instance, not for arbitrary unbounded node and key sets. Larger models +should be run periodically by increasing `Nodes`, `Origins`, `Keys`, `MaxSeq`, +`OplogBound`, and `MaxMessages`. + +The checked three-node default explores 1,835,826 states, finds 490,236 +distinct states to a depth of 30, and completes in roughly 1 minute 40 seconds +on the development machine used for the validation run. diff --git a/test/formal/check.sh b/test/formal/check.sh new file mode 100755 index 0000000..1336a08 --- /dev/null +++ b/test/formal/check.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ -z "${TLA_JAR:-}" ]]; then + echo "TLA_JAR must point to tla2tools.jar" >&2 + exit 2 +fi + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +metadir="${repo_root}/tmp/tlc" +config="${TLA_CONFIG:-${repo_root}/test/formal/GroupAntiEntropy.cfg}" +mkdir -p "${metadir}" + +exec java -XX:+UseParallelGC -cp "${TLA_JAR}" tlc2.TLC \ + -cleanup \ + -metadir "${metadir}" \ + -workers "${TLC_WORKERS:-4}" \ + -config "${config}" \ + "${repo_root}/test/formal/GroupAntiEntropy.tla" diff --git a/test/group_test.exs b/test/group_test.exs index 7de75ae..572eb46 100644 --- a/test/group_test.exs +++ b/test/group_test.exs @@ -1344,6 +1344,9 @@ defmodule GroupTest do assert :ok = Group.join(name, key, %{}, cluster: cluster) assert Group.members(name, key, cluster: cluster) == [{self(), %{}}] + remote_route = :"disconnect-timeout@remote" + :ok = Group.Replica.Data.add_cluster_node(name, [cluster], remote_route) + shard_zero = Group.Replica.shard_name(name, 0) :ok = :sys.suspend(shard_zero) @@ -1358,6 +1361,48 @@ defmodule GroupTest do :sys.get_state(shard_zero) refute Group.connected?(name, cluster) assert Group.members(name, key, cluster: cluster) == [] + + wait_until(fn -> + Group.Replica.Data.closed_local_clusters(name) == [] and + Group.Replica.Data.cluster_nodes(name, cluster) == [] + end) + end + + test "reconnect waits for every old-epoch shard cleanup before admitting new writes" do + name = :"test_reconnect_barrier_#{System.unique_integer([:positive])}" + cluster = "reconnect_barrier" + key = "reconnect/barrier/#{System.unique_integer([:positive])}" + start_supervised!({Group, name: name, shards: 2, log: false}) + + assert :ok = Group.connect(name, cluster) + shard_zero = Group.Replica.shard_name(name, 0) + :ok = :sys.suspend(shard_zero) + + reconnect_caller = + try do + assert_genserver_call_timeout(fn -> + Group.disconnect(name, cluster, timeout: 10) + end) + + caller = + spawn_requester( + fn -> Group.connect(name, cluster) end, + :reconnect_barrier_result + ) + + refute_receive {:reconnect_barrier_result, ^caller, _result}, 50 + caller + after + resume_shard_if_alive(shard_zero) + end + + on_exit(fn -> kill_if_alive(reconnect_caller) end) + assert_receive {:reconnect_barrier_result, ^reconnect_caller, :ok}, 1_000 + + assert Group.Replica.Data.closed_local_clusters(name) == [] + assert :ok = Group.join(name, key, %{epoch: :new}, cluster: cluster) + assert Group.members(name, key, cluster: cluster) == [{self(), %{epoch: :new}}] + assert :ok = Group.TestCluster.assert_replica_consistent(name) end end @@ -2996,6 +3041,146 @@ defmodule GroupTest do Group.lookup(name, key) == nil and Group.members(name, key) == [] end) end + + test "a shard restart repairs an interrupted append tail and interrupted prune floor" do + name = start_single_shard_group(replicated_oplog_max_entries: 16) + stream_id = Group.Replica.Data.local_stream_id(name, 0, nil) + first_key = "journal/crash-window/first/#{System.unique_integer([:positive])}" + second_key = "journal/crash-window/second/#{System.unique_integer([:positive])}" + third_key = "journal/crash-window/third/#{System.unique_integer([:positive])}" + + :ok = Group.register(name, first_key, %{seq: 1}) + :ok = Group.register(name, second_key, %{seq: 2}) + + oplog = Group.Replica.Data.replica_oplog_table(name, 0) + order = Group.Replica.Data.replica_oplog_order_table(name, 0) + stream_meta = Group.Replica.Data.replica_stream_meta_table(name, 0) + + [{{^stream_id, 1}, first_append_id, _mutations}] = :ets.lookup(oplog, {stream_id, 1}) + + # Pruning removes order -> record -> advances floor. Model a kill after + # the first two writes but before the floor update. + :ets.delete(order, first_append_id) + :ets.delete(oplog, {stream_id, 1}) + + # Appending advances head -> append counter -> record -> order. Model a + # kill after the head update but before the record exists. + assert 3 = :ets.update_counter(stream_meta, stream_id, {2, 1}) + + old_shard = Process.whereis(Group.Replica.shard_name(name, 0)) + Process.exit(old_shard, :kill) + + Group.TestCluster.assert_eventually(fn -> + new_shard = Process.whereis(Group.Replica.shard_name(name, 0)) + is_pid(new_shard) and new_shard != old_shard + end) + + :sys.get_state(Group.Replica.shard_name(name, 0)) + assert {2, 2, 2} = Group.Replica.Data.replica_stream_head(name, 0, stream_id) + assert :ok = Group.TestCluster.assert_replica_consistent(name) + + :ok = Group.register(name, third_key, %{seq: 3}) + assert {2, 3, 3} = Group.Replica.Data.replica_stream_head(name, 0, stream_id) + + assert Group.lookup(name, first_key) == {self(), %{seq: 1}} + assert Group.lookup(name, second_key) == {self(), %{seq: 2}} + assert Group.lookup(name, third_key) == {self(), %{seq: 3}} + assert :ok = Group.TestCluster.assert_replica_consistent(name) + end + + test "a shard restart rebuilds every one-sided materialized index" do + name = start_single_shard_group(replicated_oplog_max_entries: 16) + reg_key = "indexes/crash-window/reg/#{System.unique_integer([:positive])}" + pg_key = "indexes/crash-window/pg/#{System.unique_integer([:positive])}" + + :ok = Group.register(name, reg_key, %{kind: :registry}) + :ok = Group.join(name, pg_key, %{kind: :pg}) + + stream_id = Group.Replica.Data.local_stream_id(name, 0, nil) + generation = Group.Replica.Protocol.stream_generation(stream_id) + epoch = Group.Replica.Protocol.stream_epoch(stream_id) + + :ets.delete(Group.Replica.Data.reg_by_pid_table(name, 0), {self(), nil, reg_key}) + :ets.delete(Group.Replica.Data.pg_by_pid_table(name, 0), {self(), nil, pg_key}) + + :ets.delete( + Group.Replica.Data.reg_claim_by_pid_table(name, 0), + {self(), nil, reg_key, node(), generation, epoch} + ) + + orphan = spawn(fn -> Process.sleep(:infinity) end) + on_exit(fn -> kill_if_alive(orphan) end) + + :ets.insert( + Group.Replica.Data.reg_by_pid_table(name, 0), + {{orphan, nil, "indexes/orphan/reg"}, %{}, 0, node()} + ) + + :ets.insert( + Group.Replica.Data.pg_by_pid_table(name, 0), + {{orphan, nil, "indexes/orphan/pg"}, %{}, 0, node()} + ) + + :ets.insert( + Group.Replica.Data.reg_claim_by_pid_table(name, 0), + {{orphan, nil, "indexes/orphan/claim", node(), generation, epoch}, %{}, 0, 1} + ) + + old_shard = Process.whereis(Group.Replica.shard_name(name, 0)) + Process.exit(old_shard, :kill) + + Group.TestCluster.assert_eventually(fn -> + new_shard = Process.whereis(Group.Replica.shard_name(name, 0)) + is_pid(new_shard) and new_shard != old_shard + end) + + :sys.get_state(Group.Replica.shard_name(name, 0)) + assert Group.lookup(name, reg_key) == {self(), %{kind: :registry}} + assert Group.members(name, pg_key) == [{self(), %{kind: :pg}}] + assert Group.Replica.Data.registry_lookup_by_pid(name, 0, orphan) == [] + assert Group.Replica.Data.entries_by_pid(name, 0, orphan) == [] + assert :ok = Group.TestCluster.assert_replica_consistent(name) + end + + test "a shard restart completes an interrupted named-cluster close without retained rows" do + name = start_single_shard_group(replicated_oplog_max_entries: 16) + cluster = "close/crash-window/#{System.unique_integer([:positive])}" + reg_key = "close/crash-window/reg/#{System.unique_integer([:positive])}" + pg_key = "close/crash-window/pg/#{System.unique_integer([:positive])}" + remote_route = :"close-crash-window@remote" + + :ok = Group.connect(name, cluster) + :ok = Group.register(name, reg_key, %{kind: :registry}, cluster: cluster) + :ok = Group.join(name, pg_key, %{kind: :pg}, cluster: cluster) + :ok = Group.Replica.Data.add_cluster_node(name, [cluster], remote_route) + + stream_id = Group.Replica.Data.local_stream_id(name, 0, cluster) + old_epoch = Group.Replica.Protocol.stream_epoch(stream_id) + + # Group.disconnect/3 closes authority and routing before its request + # reaches every shard. Model a shard kill in that exact window. + assert [{^cluster, ^old_epoch}] = + Group.Replica.Data.deactivate_local_clusters(name, [cluster]) + + :ok = Group.Replica.Data.remove_cluster_node(name, [cluster], node()) + assert Group.Replica.Data.closed_local_clusters(name) == [cluster] + + old_shard = Process.whereis(Group.Replica.shard_name(name, 0)) + Process.exit(old_shard, :kill) + + Group.TestCluster.assert_eventually(fn -> + new_shard = Process.whereis(Group.Replica.shard_name(name, 0)) + + is_pid(new_shard) and new_shard != old_shard and + Group.lookup(name, reg_key, cluster: cluster) == nil and + Group.members(name, pg_key, cluster: cluster) == [] and + Group.Replica.Data.closed_local_clusters(name) == [] and + Group.Replica.Data.cluster_nodes(name, cluster) == [] + end) + + assert :ets.lookup(Group.Replica.Data.replica_stream_meta_table(name, 0), stream_id) == [] + assert :ok = Group.TestCluster.assert_replica_consistent(name) + end end describe "replica authority snapshots" do diff --git a/test/mutation/README.md b/test/mutation/README.md new file mode 100644 index 0000000..269edf8 --- /dev/null +++ b/test/mutation/README.md @@ -0,0 +1,24 @@ +# Replica mutation campaign + +This campaign calibrates the anti-entropy tests against deliberate failures of +catastrophic protocol obligations. It covers generation and epoch fencing, +contiguous sequence application, exact registry and PG snapshots, below-floor +repair, process-down sequencing, conflict-loser retirement, authority fanout, +per-lane authority installation, periodic head advertisement, interrupted +journal/index repair, and named-cluster close completion. + +The runner first verifies every unmodified regression target. It then copies +the current checkout once per mutant, changes only that copy, recompiles it, +and runs the designated real multi-node regression. A compiling mutant is +`killed` only when the regression fails. Any surviving or non-compiling mutant +makes the campaign fail. + +```bash +mix run --no-start test/mutation/run.exs + +# List or run selected mutations +mix run --no-start test/mutation/run.exs --list +mix run --no-start test/mutation/run.exs disable_below_floor_snapshot +``` + +Artifacts and complete logs are written below `tmp/mutation/`. diff --git a/test/mutation/run.exs b/test/mutation/run.exs new file mode 100644 index 0000000..60ecf6f --- /dev/null +++ b/test/mutation/run.exs @@ -0,0 +1,422 @@ +defmodule Group.MutationCampaign do + @moduledoc """ + Runs protocol mutations in isolated repository copies. + + A mutant is useful only when it compiles and its designated regression test + fails. The source checkout is never edited. + """ + + @repo Path.expand("../..", __DIR__) + @timeout_seconds "120" + + # Each entry replaces one correct production fragment with an intentionally + # faulty fragment, but only inside an isolated campaign checkout. + @mutations [ + %{ + name: "accept_old_generation", + file: "lib/group/replica.ex", + correct_source: + "Protocol.stream_generation(stream_id) == Data.remote_generation(state.name, source_node) and", + faulty_source: "true and", + test: ["test/distributed_test.exs:5328"] + }, + %{ + name: "accept_old_epoch", + file: "lib/group/replica.ex", + correct_source: """ + Protocol.stream_epoch(stream_id) == + Data.remote_cluster_epoch(state.name, source_node, cluster) and + """, + faulty_source: """ + true and + """, + test: ["test/distributed_test.exs:4774"] + }, + %{ + name: "advance_cursor_across_gap", + file: "lib/group/replica.ex", + correct_source: """ + [{first_seq, _mutations} | _] when first_seq > cursor + 1 -> + request_replica_need(state, source_node, stream_id, cursor + 1) + """, + faulty_source: """ + [{first_seq, _mutations} | _] when first_seq > cursor + 1 -> + :ok = + Data.put_replica_cursor( + state.name, + state.shard_index, + stream_id, + first_seq - 1 + ) + + apply_replica_delta_run( + state, + source_node, + stream_id, + records, + advertised_head + ) + """, + test: ["test/distributed_test.exs:5386"] + }, + %{ + name: "registry_snapshot_is_additive", + file: "lib/group/replica/data.ex", + correct_source: "existing = registry_claims_for_stream(name, shard, stream_id)", + faulty_source: "existing = []", + test: ["test/distributed_test.exs:4030"] + }, + %{ + name: "pg_snapshot_is_additive", + file: "lib/group/replica.ex", + correct_source: """ + current = + state.name + |> Data.pg_entries_for_origin(state.shard_index, cluster, source_node) + |> Map.new(fn {key, pid, meta, time} -> {{key, pid}, {meta, time}} end) + """, + faulty_source: """ + current = %{} + """, + test: ["test/distributed_test.exs:4030"] + }, + %{ + name: "disable_below_floor_snapshot", + file: "lib/group/replica.ex", + correct_source: """ + true -> + {:state, send_replica_snapshot(state, target_node, stream_id, head)} + """, + faulty_source: """ + true -> + {:state, state} + """, + test: ["test/replica_model_property_test.exs:180"] + }, + %{ + name: "do_not_sequence_process_down", + file: "lib/group/replica.ex", + correct_source: """ + sequenced_downs = + append_process_down_records(state, reason_by_pid, pending_reg, pending_pg) + """, + faulty_source: + " sequenced_downs =\n if false,\n do: append_process_down_records(state, reason_by_pid, pending_reg, pending_pg),\n else: []\n", + test: ["test/distributed_test.exs:3940"] + }, + %{ + name: "do_not_exit_conflict_loser", + file: "lib/group/replica.ex", + correct_source: """ + winner_meta = if winner, do: elem(winner, 1), else: nil + exit_local_conflict_loser(pid, key, winner_meta) + acc + """, + faulty_source: """ + _winner_meta = if winner, do: elem(winner, 1), else: nil + _ = Process.alive?(pid) + acc + """, + test: ["test/replica_model_property_test.exs:77"] + }, + %{ + name: "heartbeat_promotes_observed_authority", + file: "lib/group/replica.ex", + correct_source: + " replica_view_current?(state, remote_node) ->\n" <> + " state\n" <> + " |> put_remote_shard(remote_node, remote_pid)\n" <> + " |> touch_replica_peer(remote_node)", + faulty_source: + " replica_view_current?(state, remote_node) ->\n" <> + " :ok =\n" <> + " Data.put_remote_view_info(\n" <> + " state.name,\n" <> + " state.shard_index,\n" <> + " remote_node,\n" <> + " generation,\n" <> + " epoch_revision,\n" <> + " epoch_revision\n" <> + " )\n\n" <> + " state\n" <> + " |> put_remote_shard(remote_node, remote_pid)\n" <> + " |> touch_replica_peer(remote_node)", + test: ["test/distributed_test.exs:4247"] + }, + %{ + name: "skip_authority_fanout", + file: "lib/group/replica.ex", + correct_source: """ + fan_out_to_siblings( + state, + {:replica_authority_installed_local, remote_node, generation, epoch_revision, + old_generation, stale_epochs} + ) + """, + faulty_source: """ + :ok + """, + test: ["test/distributed_test.exs:5531"] + }, + %{ + name: "skip_generation_purge", + file: "lib/group/replica.ex", + correct_source: """ + defp maybe_purge_remote_generation(state, remote_node, _old_generation, _generation) do + {_reg, _pg} = Data.purge_node(state.name, state.shard_index, remote_node) + + affected = + Data.purge_registry_claims_for_origin( + state.name, + state.shard_index, + remote_node + ) + + {state, events} = + Enum.reduce(affected, {state, []}, fn {cluster, key}, {acc, inner_events} -> + reconcile_registry_projection(acc, cluster, key, :nodedown, inner_events) + end) + + notify_monitors(state.name, events) + Data.delete_replica_cursors_for_origin(state.name, state.shard_index, remote_node) + state + end + """, + faulty_source: """ + defp maybe_purge_remote_generation(state, _remote_node, _old_generation, _generation), + do: state + """, + test: ["test/distributed_test.exs:5531"] + }, + %{ + name: "disable_periodic_heads", + file: "lib/group/replica.ex", + correct_source: """ + defp broadcast_replica_heads(state) do + Enum.reduce(state.peer_last_seen, state, fn {target_node, _last_seen}, acc -> + send_replica_heads(acc, target_node) + end) + end + """, + faulty_source: """ + defp broadcast_replica_heads(state), do: state + """, + test: ["test/distributed_test.exs:3940"] + }, + %{ + name: "skip_journal_crash_repair", + file: "lib/group/replica.ex", + correct_source: ":ok = Data.repair_local_replica_journal(name, shard_index)", + faulty_source: ":ok", + test: ["test/group_test.exs:3045"] + }, + %{ + name: "skip_index_crash_repair", + file: "lib/group/replica.ex", + correct_source: ":ok = Data.repair_shard_indexes(name, shard_index)", + faulty_source: ":ok", + test: ["test/group_test.exs:3091"] + }, + %{ + name: "skip_inactive_cluster_repair", + file: "lib/group/replica/data.ex", + correct_source: """ + def repair_shard_indexes(name, shard) do + purge_inactive_cluster_rows(name, shard) + """, + faulty_source: """ + def repair_shard_indexes(name, shard) do + if false, do: purge_inactive_cluster_rows(name, shard) + """, + test: ["test/group_test.exs:3145"] + }, + %{ + name: "skip_closed_cluster_completion", + file: "lib/group/replica.ex", + correct_source: """ + completed_clusters = + Data.mark_closed_cluster_shard(name, Data.closed_local_clusters(name), shard_index) + """, + faulty_source: """ + completed_clusters = [] + """, + test: ["test/group_test.exs:3145"] + }, + %{ + name: "accept_shared_authority_before_lane_install", + file: "lib/group/replica.ex", + correct_source: """ + Protocol.stream_shard(stream_id) == state.shard_index and + replica_view_current?(state, source_node) and + """, + faulty_source: """ + Protocol.stream_shard(stream_id) == state.shard_index and + true and + """, + test: ["test/distributed_test.exs:5531"] + } + ] + + def run(args) do + selected = select_mutations(args) + campaign_dir = campaign_dir() + File.mkdir_p!(campaign_dir) + + IO.puts("mutation artifacts: #{campaign_dir}") + verify_baselines!(selected, campaign_dir) + + results = Enum.map(selected, &run_mutant(&1, campaign_dir)) + print_summary(results) + + if Enum.any?(results, fn {_name, status, _path} -> status != :killed end) do + System.halt(1) + end + end + + defp select_mutations(["--list"]) do + Enum.each(@mutations, &IO.puts(&1.name)) + System.halt(0) + end + + defp select_mutations([]), do: @mutations + + defp select_mutations(names) do + by_name = Map.new(@mutations, &{&1.name, &1}) + unknown = names -- Map.keys(by_name) + + if unknown != [] do + raise "unknown mutations: #{Enum.join(unknown, ", ")}" + end + + Enum.map(names, &Map.fetch!(by_name, &1)) + end + + defp verify_baselines!(mutations, campaign_dir) do + mutations + |> Enum.map(& &1.test) + |> Enum.uniq() + |> Enum.each(fn test -> + label = test |> hd() |> String.replace(~r/[^A-Za-z0-9_.-]/, "_") + log = Path.join(campaign_dir, "baseline-#{label}.log") + IO.write("baseline #{Enum.join(test, " ")} ... ") + {output, status} = run_test(@repo, test) + File.write!(log, output) + + if status == 0 do + IO.puts("pass") + else + IO.puts("FAIL") + raise "baseline failed; see #{log}" + end + end) + end + + defp run_mutant(mutation, campaign_dir) do + work = Path.join(campaign_dir, mutation.name) + File.mkdir_p!(work) + copy_checkout!(work) + + source_path = Path.join(work, mutation.file) + source = File.read!(source_path) + matches = :binary.matches(source, mutation.correct_source) + + if length(matches) != 1 do + log = Path.join(work, "mutation-error.log") + File.write!(log, "expected one match, found #{length(matches)}\n") + IO.puts("#{mutation.name}: INVALID (replacement matched #{length(matches)} times)") + {mutation.name, :invalid, work} + else + File.write!( + source_path, + String.replace(source, mutation.correct_source, mutation.faulty_source) + ) + + compile_log = Path.join(work, "compile.log") + {compile_output, compile_status} = run_mix(work, ["compile", "--warnings-as-errors"]) + File.write!(compile_log, compile_output) + + if compile_status != 0 do + IO.puts("#{mutation.name}: INVALID (does not compile)") + {mutation.name, :invalid, work} + else + test_log = Path.join(work, "test.log") + {test_output, test_status} = run_test(work, mutation.test) + File.write!(test_log, test_output) + + case test_status do + 0 -> + IO.puts("#{mutation.name}: SURVIVED") + {mutation.name, :survived, work} + + 124 -> + IO.puts("#{mutation.name}: killed (timeout)") + {mutation.name, :killed, work} + + _ -> + IO.puts("#{mutation.name}: killed") + {mutation.name, :killed, work} + end + end + end + end + + defp copy_checkout!(target) do + rsync = System.find_executable("rsync") || raise "rsync is required" + + {_output, 0} = + System.cmd( + rsync, + [ + "-a", + "--exclude=.git", + "--exclude=deps", + "--exclude=tmp", + "#{@repo}/", + "#{target}/" + ], + stderr_to_stdout: true + ) + + File.ln_s!(Path.join(@repo, "deps"), Path.join(target, "deps")) + end + + defp run_test(directory, test) do + run_with_timeout(directory, ["mix", "test" | test], + env: [{"GROUP_MODEL_RUNS", "1"}, {"GROUP_MODEL_COMMANDS", "8"}] + ) + end + + defp run_mix(directory, args) do + System.cmd("mix", args, + cd: directory, + env: [{"MIX_ENV", "test"}], + stderr_to_stdout: true + ) + end + + defp run_with_timeout(directory, command, opts) do + timeout = System.find_executable("timeout") || raise "timeout is required" + env = Keyword.fetch!(opts, :env) + + System.cmd(timeout, [@timeout_seconds | command], + cd: directory, + env: [{"MIX_ENV", "test"} | env], + stderr_to_stdout: true + ) + end + + defp campaign_dir do + stamp = Calendar.strftime(DateTime.utc_now(), "%Y%m%dT%H%M%S") + Path.join([@repo, "tmp", "mutation", "#{stamp}-#{System.unique_integer([:positive])}"]) + end + + defp print_summary(results) do + IO.puts("\nmutation summary") + + Enum.each(results, fn {name, status, path} -> + IO.puts(" #{String.pad_trailing(name, 38)} #{status} #{path}") + end) + end +end + +Group.MutationCampaign.run(System.argv()) diff --git a/test/replica_model_property_test.exs b/test/replica_model_property_test.exs new file mode 100644 index 0000000..4999c3a --- /dev/null +++ b/test/replica_model_property_test.exs @@ -0,0 +1,381 @@ +defmodule Group.ReplicaModelPropertyTest do + use ExUnit.Case, async: false + use ExUnitProperties + + alias Group.{ + ControlledReplicaTransport, + ModelConflictResolver, + ReplicaModelScheduler, + TestCluster + } + + @moduletag :capture_log + @moduletag timeout: 180_000 + + @model_runs System.get_env("GROUP_MODEL_RUNS", "12") |> String.to_integer() + @max_commands System.get_env("GROUP_MODEL_COMMANDS", "30") |> String.to_integer() + + setup_all do + peers = TestCluster.start_peers(3, schedulers: 4) + on_exit(fn -> TestCluster.stop_peers(peers) end) + + [{_, node_a}, {_, node_b}, {_, node_c}] = peers + {:ok, nodes: %{a: node_a, b: node_b, c: node_c}} + end + + property "accepted owner lifecycles converge without permanent orphans or zombies", %{ + nodes: nodes + } do + check all( + commands <- command_history(), + max_runs: @model_runs, + max_shrinking_steps: 100 + ) do + name = :"replica_model_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 2, + resolve_registry_conflict: {ModelConflictResolver, :resolve, []}, + replica_transport: {ControlledReplicaTransport, controller: self()}, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000, + replicated_oplog_max_entries: 4 + ] + + Enum.each(nodes, fn {_id, node} -> + {:ok, _pid} = TestCluster.start_group(node, opts) + end) + + scheduler = ReplicaModelScheduler.new(name, nodes, opts) + + try do + await_discovery(nodes, name) + + scheduler = + commands + |> Enum.reduce(ReplicaModelScheduler.sync(scheduler), fn command, state -> + ReplicaModelScheduler.execute(state, command) + end) + |> ReplicaModelScheduler.stabilize_and_assert!() + + assert scheduler.model != nil + after + ReplicaModelScheduler.cleanup(scheduler) + end + end + end + + property "concurrent claims converge to one live winner and permanently retire the loser", %{ + nodes: nodes + } do + check all( + key_slot <- integer(0..2), + winner <- member_of([:a, :b]), + schedule <- list_of(network_command(), min_length: 0, max_length: 20), + max_runs: max(div(@model_runs, 2), 1), + max_shrinking_steps: 100 + ) do + name = :"replica_conflict_model_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 2, + resolve_registry_conflict: {ModelConflictResolver, :resolve, []}, + replica_transport: {ControlledReplicaTransport, controller: self()}, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000, + replicated_oplog_max_entries: 2 + ] + + Enum.each(nodes, fn {_id, node} -> + {:ok, _pid} = TestCluster.start_group(node, opts) + end) + + scheduler = ReplicaModelScheduler.new(name, nodes, opts) + + try do + await_discovery(nodes, name) + + {rank_a, rank_b} = if winner == :a, do: {2, 1}, else: {1, 2} + + scheduler = + scheduler + |> ReplicaModelScheduler.sync() + |> ReplicaModelScheduler.execute({:register, 12, :c, key_slot + 10, 1}) + |> ReplicaModelScheduler.execute({:join, 12, :c, key_slot + 10, 1}) + |> ReplicaModelScheduler.execute({:claim, 10, :a, key_slot, rank_a}) + |> ReplicaModelScheduler.execute({:claim, 11, :b, key_slot, rank_b}) + + scheduler = + schedule + |> Enum.reduce(scheduler, fn command, state -> + ReplicaModelScheduler.execute(state, command) + end) + |> ReplicaModelScheduler.stabilize_and_assert!() + + assert scheduler.model != nil + after + ReplicaModelScheduler.cleanup(scheduler) + end + end + end + + property "old frames cannot survive a real Group restart and generation change", %{nodes: nodes} do + check all( + before_restart <- list_of(network_command(), max_length: 8), + after_restart <- list_of(network_command(), max_length: 12), + max_runs: transition_runs(), + max_shrinking_steps: 100 + ) do + name = :"replica_restart_model_#{System.unique_integer([:positive])}" + opts = model_opts(name, self(), shards: 2, oplog: 2) + start_groups(nodes, opts) + scheduler = ReplicaModelScheduler.new(name, nodes, opts) + + try do + await_discovery(nodes, name) + + scheduler = + scheduler + |> ReplicaModelScheduler.sync() + |> ReplicaModelScheduler.execute({:transport, :a, :pass}) + |> ReplicaModelScheduler.execute({:transport, :b, :pass}) + |> ReplicaModelScheduler.execute({:transport, :c, :pass}) + |> ReplicaModelScheduler.execute({:register, 30, :a, 0, 1}) + |> ReplicaModelScheduler.execute({:join, 30, :a, 0, 1}) + |> ReplicaModelScheduler.execute({:register, 32, :c, 1, 1}) + |> ReplicaModelScheduler.execute({:join, 32, :c, 1, 1}) + |> ReplicaModelScheduler.stabilize_and_assert!() + |> ReplicaModelScheduler.execute({:transport, :a, :capture}) + |> ReplicaModelScheduler.execute({:transport, :b, :capture}) + |> ReplicaModelScheduler.execute({:transport, :c, :capture}) + |> ReplicaModelScheduler.execute({:register, 30, :a, 0, 2}) + |> ReplicaModelScheduler.execute({:join, 30, :a, 0, 2}) + |> run_schedule(before_restart) + |> ReplicaModelScheduler.execute({:restart, :a}) + |> ReplicaModelScheduler.execute(:deliver_all) + |> ReplicaModelScheduler.execute({:register, 31, :a, 1, 2}) + |> ReplicaModelScheduler.execute({:join, 31, :a, 1, 2}) + |> run_schedule(after_restart) + |> ReplicaModelScheduler.stabilize_and_assert!() + + assert scheduler.model != nil + after + ReplicaModelScheduler.cleanup(scheduler) + end + end + end + + property "a receiver below the real oplog floor converges through an exact snapshot", %{ + nodes: nodes + } do + check all( + schedule <- list_of(network_command(), max_length: 10), + max_runs: transition_runs(), + max_shrinking_steps: 100 + ) do + name = :"replica_pruning_model_#{System.unique_integer([:positive])}" + opts = model_opts(name, self(), shards: 1, oplog: 2) + start_groups(nodes, opts) + scheduler = ReplicaModelScheduler.new(name, nodes, opts) + + try do + await_discovery(nodes, name) + + scheduler = + scheduler + |> ReplicaModelScheduler.sync() + |> ReplicaModelScheduler.execute({:transport, :a, :pass}) + |> ReplicaModelScheduler.execute({:transport, :b, :pass}) + |> ReplicaModelScheduler.execute({:transport, :c, :pass}) + |> ReplicaModelScheduler.execute({:register, 40, :a, 0, 1}) + |> ReplicaModelScheduler.execute({:join, 40, :a, 0, 1}) + |> ReplicaModelScheduler.execute({:register, 47, :c, 1, 1}) + |> ReplicaModelScheduler.execute({:join, 47, :c, 1, 1}) + |> ReplicaModelScheduler.stabilize_and_assert!() + |> ReplicaModelScheduler.execute({:transport, :a, :drop}) + |> ReplicaModelScheduler.execute({:kill, 40}) + + scheduler = + Enum.reduce(41..46, scheduler, fn owner_id, state -> + state + |> ReplicaModelScheduler.execute({:register, owner_id, :a, 0, owner_id}) + |> ReplicaModelScheduler.execute({:unregister, owner_id, 0}) + end) + + scheduler = + scheduler + |> run_schedule(schedule) + |> ReplicaModelScheduler.stabilize_and_assert!() + + assert scheduler.model != nil + after + ReplicaModelScheduler.cleanup(scheduler) + end + end + end + + property "named-cluster close and reopen fences stale epoch frames on real shards", %{ + nodes: nodes + } do + check all( + cluster_slot <- integer(0..2), + key_slot <- integer(0..2), + before_reopen <- list_of(network_command(), max_length: 8), + after_reopen <- list_of(network_command(), max_length: 12), + max_runs: transition_runs(), + max_shrinking_steps: 100 + ) do + name = :"replica_authority_model_#{System.unique_integer([:positive])}" + cluster = "model_cluster_#{cluster_slot}" + opts = model_opts(name, self(), shards: 2, oplog: 2) + start_groups(nodes, opts) + scheduler = ReplicaModelScheduler.new(name, nodes, opts) + + try do + await_discovery(nodes, name) + + scheduler = + scheduler + |> ReplicaModelScheduler.sync() + |> ReplicaModelScheduler.execute({:connect, :a, cluster}) + |> ReplicaModelScheduler.execute({:connect, :b, cluster}) + |> ReplicaModelScheduler.execute({:connect, :c, cluster}) + |> ReplicaModelScheduler.execute({:register_cluster, 50, :a, cluster, key_slot, 1}) + |> ReplicaModelScheduler.execute({:join_cluster, 50, :a, cluster, key_slot, 1}) + |> ReplicaModelScheduler.execute({ + :register_cluster, + 52, + :c, + cluster, + key_slot + 10, + 1 + }) + |> ReplicaModelScheduler.execute({ + :join_cluster, + 52, + :c, + cluster, + key_slot + 10, + 1 + }) + |> run_schedule(before_reopen) + |> ReplicaModelScheduler.execute({:disconnect, :a, cluster}) + |> ReplicaModelScheduler.execute({:connect, :a, cluster}) + |> ReplicaModelScheduler.execute(:deliver_all) + |> ReplicaModelScheduler.execute({:register_cluster, 51, :a, cluster, key_slot, 2}) + |> ReplicaModelScheduler.execute({:join_cluster, 51, :a, cluster, key_slot, 2}) + |> run_schedule(after_reopen) + |> ReplicaModelScheduler.stabilize_and_assert!() + + assert scheduler.model != nil + after + ReplicaModelScheduler.cleanup(scheduler) + end + end + end + + defp command_history do + list_of(command(), min_length: 1, max_length: @max_commands) + end + + defp command do + one_of([ + owner_command(:register), + owner_command(:join), + owner_key_command(:unregister), + owner_key_command(:leave), + map(owner_id(), &{:kill, &1}), + transport_mode_command(), + map(non_negative_integer(), &{:deliver, &1}), + map(non_negative_integer(), &{:duplicate, &1}), + map(non_negative_integer(), &{:drop, &1}), + constant(:deliver_all), + constant(:anti_entropy), + constant(:flush) + ]) + end + + defp network_command do + one_of([ + transport_mode_command(), + map(non_negative_integer(), &{:deliver, &1}), + map(non_negative_integer(), &{:duplicate, &1}), + map(non_negative_integer(), &{:drop, &1}), + constant(:anti_entropy), + constant(:flush) + ]) + end + + defp owner_command(operation) do + gen all( + owner_id <- owner_id(), + node_id <- member_of([:a, :b, :c]), + slot <- integer(0..1), + revision <- integer(0..3) + ) do + {operation, owner_id, node_id, slot, revision} + end + end + + defp owner_key_command(operation) do + gen all( + owner_id <- owner_id(), + slot <- integer(0..1) + ) do + {operation, owner_id, slot} + end + end + + defp transport_mode_command do + gen all( + node_id <- member_of([:a, :b, :c]), + mode <- member_of([:capture, :busy, :drop]) + ) do + {:transport, node_id, mode} + end + end + + defp owner_id, do: integer(0..5) + + defp transition_runs, do: max(div(@model_runs, 3), 1) + + defp run_schedule(state, commands) do + Enum.reduce(commands, state, fn command, acc -> + ReplicaModelScheduler.execute(acc, command) + end) + end + + defp model_opts(name, controller, overrides) do + [ + name: name, + shards: Keyword.fetch!(overrides, :shards), + resolve_registry_conflict: {ModelConflictResolver, :resolve, []}, + replica_transport: {ControlledReplicaTransport, controller: controller}, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000, + replicated_oplog_max_entries: Keyword.fetch!(overrides, :oplog) + ] + end + + defp start_groups(nodes, opts) do + Enum.each(nodes, fn {_id, node} -> + {:ok, _pid} = TestCluster.start_group(node, opts) + end) + end + + defp await_discovery(nodes, name) do + TestCluster.assert_eventually( + fn -> + Enum.all?(nodes, fn {_id, node} -> + length(TestCluster.rpc!(node, Group, :nodes, [name])) == map_size(nodes) - 1 + end) + end, + timeout: 10_000 + ) + end +end diff --git a/test/support/controlled_replica_transport.ex b/test/support/controlled_replica_transport.ex new file mode 100644 index 0000000..70d7d48 --- /dev/null +++ b/test/support/controlled_replica_transport.ex @@ -0,0 +1,49 @@ +defmodule Group.ControlledReplicaTransport do + @moduledoc false + @behaviour Group.Replica.Transport + + @impl true + def id, do: :group_controlled_replica_transport + + @impl true + def descriptor(_group, _opts), do: :group_controlled_replica_transport + + def set_mode(group, mode) when mode in [:capture, :pass, :busy, :drop] do + :persistent_term.put({__MODULE__, group, :mode}, mode) + :ok + end + + def clear(group) do + :persistent_term.erase({__MODULE__, group, :mode}) + :ok + end + + @impl true + def try_send(group, target_node, shard, frame, opts) do + case :persistent_term.get({__MODULE__, group, :mode}, :capture) do + :capture -> + controller = Keyword.fetch!(opts, :controller) + send(controller, {__MODULE__, :frame, group, node(), target_node, shard, frame}) + :ok + + :pass -> + deliver(group, target_node, shard, frame) + + :busy -> + :busy + + :drop -> + :ok + end + end + + defp deliver(group, target_node, shard, frame) do + destination = {Group.Replica.shard_name(group, shard), target_node} + message = {:group_replica_frame, node(), frame} + + case :erlang.send_nosuspend(destination, message, [:noconnect]) do + true -> :ok + false -> :busy + end + end +end diff --git a/test/support/model_conflict_resolver.ex b/test/support/model_conflict_resolver.ex new file mode 100644 index 0000000..6ac3502 --- /dev/null +++ b/test/support/model_conflict_resolver.ex @@ -0,0 +1,15 @@ +defmodule Group.ModelConflictResolver do + @moduledoc false + + def resolve(_name, _key, {pid1, meta1, _time1}, {pid2, meta2, _time2}) do + rank1 = Map.fetch!(meta1, :rank) + rank2 = Map.fetch!(meta2, :rank) + + cond do + rank1 > rank2 -> pid1 + rank2 > rank1 -> pid2 + pid1 > pid2 -> pid1 + true -> pid2 + end + end +end diff --git a/test/support/replica_lifecycle_model.ex b/test/support/replica_lifecycle_model.ex new file mode 100644 index 0000000..abb0dc1 --- /dev/null +++ b/test/support/replica_lifecycle_model.ex @@ -0,0 +1,223 @@ +defmodule Group.ReplicaLifecycleModel do + @moduledoc """ + Independent application-level oracle for replica convergence tests. + + This model deliberately knows nothing about Group's oplog, receive cursors, + batching, authority tables, or wire frames. It records only operations that + the public API accepted and the owner lifecycle consequences that follow. + """ + + defstruct owners: %{}, + registrations: %{}, + memberships: MapSet.new(), + seen_registration_keys: MapSet.new(), + seen_membership_keys: MapSet.new() + + @type owner_id :: non_neg_integer() + @type cluster :: term() + @type key :: binary() + @type scoped_key :: {cluster(), key()} + @type owner :: %{node: atom(), alive?: boolean()} + @type t :: %__MODULE__{ + owners: %{optional(owner_id()) => owner()}, + registrations: %{optional(scoped_key()) => %{optional(owner_id()) => map()}}, + memberships: MapSet.t({cluster(), key(), owner_id(), map()}), + seen_registration_keys: MapSet.t(scoped_key()), + seen_membership_keys: MapSet.t(scoped_key()) + } + + def new, do: %__MODULE__{} + + def owner(%__MODULE__{} = model, owner_id), do: Map.get(model.owners, owner_id) + + def put_owner(%__MODULE__{} = model, owner_id, node) do + case owner(model, owner_id) do + nil -> + put_in(model.owners[owner_id], %{node: node, alive?: true}) + + %{node: ^node} -> + model + + %{node: other_node} -> + raise ArgumentError, + "logical owner #{inspect(owner_id)} moved from #{inspect(other_node)} to #{inspect(node)}" + end + end + + def record_register(%__MODULE__{} = model, owner_id, key, meta, :ok) do + model + |> ensure_alive!(owner_id) + |> Map.update!(:seen_registration_keys, &MapSet.put(&1, key)) + |> Map.update!(:registrations, fn registrations -> + Map.update(registrations, key, %{owner_id => meta}, &Map.put(&1, owner_id, meta)) + end) + end + + def record_register(%__MODULE__{} = model, _owner_id, key, _meta, {:error, :taken}) do + Map.update!(model, :seen_registration_keys, &MapSet.put(&1, key)) + end + + def record_unregister(%__MODULE__{} = model, owner_id, key, :ok) do + registrations = + model.registrations + |> Map.update(key, %{}, &Map.delete(&1, owner_id)) + |> drop_empty_claim_sets() + + %{model | registrations: registrations} + end + + def record_unregister(%__MODULE__{} = model, _owner_id, _key, {:error, _reason}), do: model + + def record_join(%__MODULE__{} = model, owner_id, {cluster, key} = scope, meta, :ok) do + model + |> ensure_alive!(owner_id) + |> Map.update!(:seen_membership_keys, &MapSet.put(&1, scope)) + |> Map.update!(:memberships, fn memberships -> + memberships + |> Enum.reject(fn {member_cluster, member_key, member_owner, _old_meta} -> + member_cluster == cluster and member_key == key and member_owner == owner_id + end) + |> MapSet.new() + |> MapSet.put({cluster, key, owner_id, meta}) + end) + end + + def record_leave(%__MODULE__{} = model, owner_id, {cluster, key}, :ok) do + memberships = + model.memberships + |> Enum.reject(fn {member_cluster, member_key, member_owner, _meta} -> + member_cluster == cluster and member_key == key and member_owner == owner_id + end) + |> MapSet.new() + + %{model | memberships: memberships} + end + + def record_leave(%__MODULE__{} = model, _owner_id, _key, {:error, _reason}), do: model + + def kill(%__MODULE__{} = model, owner_id) do + case owner(model, owner_id) do + nil -> + model + + owner -> + registrations = + model.registrations + |> Map.new(fn {key, claims} -> {key, Map.delete(claims, owner_id)} end) + |> drop_empty_claim_sets() + + memberships = + model.memberships + |> Enum.reject(fn {_cluster, _key, member_owner, _meta} -> + member_owner == owner_id + end) + |> MapSet.new() + + %{ + model + | owners: Map.put(model.owners, owner_id, %{owner | alive?: false}), + registrations: registrations, + memberships: memberships + } + end + end + + def restart_node(%__MODULE__{} = model, node) do + remove_owned_scope(model, fn owner -> owner.node == node end, fn _scope -> true end) + end + + def disconnect_cluster(%__MODULE__{} = model, node, cluster) do + remove_owned_scope( + model, + fn owner -> owner.node == node end, + fn {entry_cluster, _key} -> entry_cluster == cluster end + ) + end + + def expected_registrations(%__MODULE__{} = model) do + Map.new(model.registrations, fn {key, claims} -> + {owner_id, meta} = + Enum.max_by(claims, fn {owner_id, meta} -> + {Map.get(meta, :rank, owner_id), owner_id} + end) + + {key, {owner_id, meta}} + end) + end + + def expected_memberships(%__MODULE__{} = model) do + model.memberships + |> Enum.group_by( + fn {cluster, key, _owner_id, _meta} -> {cluster, key} end, + fn {_cluster, _key, owner_id, meta} -> {owner_id, meta} end + ) + |> Map.new(fn {key, members} -> {key, Enum.sort(members)} end) + end + + def resolve_registry_conflicts(%__MODULE__{} = model) do + losing_owners = + model.registrations + |> Enum.flat_map(fn {_key, claims} -> + if map_size(claims) > 1 do + {winner, _meta} = + Enum.max_by(claims, fn {owner_id, meta} -> + {Map.get(meta, :rank, owner_id), owner_id} + end) + + Map.keys(claims) -- [winner] + else + [] + end + end) + |> MapSet.new() + + if MapSet.size(losing_owners) == 0 do + model + else + losing_owners + |> Enum.reduce(model, &kill(&2, &1)) + |> resolve_registry_conflicts() + end + end + + defp ensure_alive!(model, owner_id) do + case owner(model, owner_id) do + %{alive?: true} -> model + other -> raise ArgumentError, "owner #{inspect(owner_id)} is not alive: #{inspect(other)}" + end + end + + defp drop_empty_claim_sets(registrations) do + Map.reject(registrations, fn {_key, claims} -> map_size(claims) == 0 end) + end + + defp remove_owned_scope(model, owner_filter, scope_filter) do + owner_ids = + model.owners + |> Enum.filter(fn {_owner_id, owner} -> owner_filter.(owner) end) + |> MapSet.new(&elem(&1, 0)) + + registrations = + model.registrations + |> Map.new(fn {scope, claims} -> + claims = + if scope_filter.(scope) do + Map.reject(claims, fn {owner_id, _meta} -> MapSet.member?(owner_ids, owner_id) end) + else + claims + end + + {scope, claims} + end) + |> drop_empty_claim_sets() + + memberships = + model.memberships + |> Enum.reject(fn {cluster, key, owner_id, _meta} -> + scope_filter.({cluster, key}) and MapSet.member?(owner_ids, owner_id) + end) + |> MapSet.new() + + %{model | registrations: registrations, memberships: memberships} + end +end diff --git a/test/support/replica_model_scheduler.ex b/test/support/replica_model_scheduler.ex new file mode 100644 index 0000000..b698324 --- /dev/null +++ b/test/support/replica_model_scheduler.ex @@ -0,0 +1,521 @@ +defmodule Group.ReplicaModelScheduler do + @moduledoc false + + alias Group.{ControlledReplicaTransport, ReplicaLifecycleModel, TestCluster} + + defmodule Envelope do + @moduledoc false + defstruct [:id, :source, :target, :shard, :frame] + end + + defstruct [:name, :nodes, :model, :group_opts, owners: %{}, queue: [], next_frame_id: 1] + + def new(name, nodes, group_opts \\ []) do + %__MODULE__{ + name: name, + nodes: Map.new(nodes), + model: ReplicaLifecycleModel.new(), + group_opts: group_opts + } + end + + def execute(%__MODULE__{} = state, {:register, owner_id, node_id, slot, revision}) do + with_owner(state, owner_id, node_id, fn state, _pid -> + key = registration_key(owner_id, slot) + meta = %{owner: owner_id, rank: owner_id, revision: revision} + result = owner_call(state, owner_id, {:register, state.name, key, meta, []}) + + model = + ReplicaLifecycleModel.record_register(state.model, owner_id, {nil, key}, meta, result) + + sync(%{state | model: model}) + end) + end + + def execute(%__MODULE__{} = state, {:claim, owner_id, node_id, key_slot, rank}) do + with_owner(state, owner_id, node_id, fn state, _pid -> + key = "model/conflict/#{key_slot}" + meta = %{owner: owner_id, rank: rank} + result = owner_call(state, owner_id, {:register, state.name, key, meta, []}) + + model = + ReplicaLifecycleModel.record_register(state.model, owner_id, {nil, key}, meta, result) + + sync(%{state | model: model}) + end) + end + + def execute(%__MODULE__{} = state, {:unregister, owner_id, slot}) do + with_existing_owner(state, owner_id, fn state, _pid -> + key = registration_key(owner_id, slot) + result = owner_call(state, owner_id, {:unregister, state.name, key, []}) + model = ReplicaLifecycleModel.record_unregister(state.model, owner_id, {nil, key}, result) + sync(%{state | model: model}) + end) + end + + def execute(%__MODULE__{} = state, {:join, owner_id, node_id, slot, revision}) do + with_owner(state, owner_id, node_id, fn state, _pid -> + key = membership_key(owner_id, slot) + meta = %{owner: owner_id, revision: revision} + result = owner_call(state, owner_id, {:join, state.name, key, meta, []}) + model = ReplicaLifecycleModel.record_join(state.model, owner_id, {nil, key}, meta, result) + sync(%{state | model: model}) + end) + end + + def execute(%__MODULE__{} = state, {:leave, owner_id, slot}) do + with_existing_owner(state, owner_id, fn state, _pid -> + key = membership_key(owner_id, slot) + result = owner_call(state, owner_id, {:leave, state.name, key, []}) + model = ReplicaLifecycleModel.record_leave(state.model, owner_id, {nil, key}, result) + sync(%{state | model: model}) + end) + end + + def execute( + %__MODULE__{} = state, + {:register_cluster, owner_id, node_id, cluster, slot, revision} + ) do + with_owner(state, owner_id, node_id, fn state, _pid -> + key = cluster_registration_key(owner_id, slot) + meta = %{owner: owner_id, rank: owner_id, revision: revision, cluster: cluster} + opts = [cluster: cluster] + result = owner_call(state, owner_id, {:register, state.name, key, meta, opts}) + scope = {cluster, key} + model = ReplicaLifecycleModel.record_register(state.model, owner_id, scope, meta, result) + sync(%{state | model: model}) + end) + end + + def execute(%__MODULE__{} = state, {:join_cluster, owner_id, node_id, cluster, slot, revision}) do + with_owner(state, owner_id, node_id, fn state, _pid -> + key = cluster_membership_key(owner_id, slot) + meta = %{owner: owner_id, revision: revision, cluster: cluster} + opts = [cluster: cluster] + result = owner_call(state, owner_id, {:join, state.name, key, meta, opts}) + scope = {cluster, key} + model = ReplicaLifecycleModel.record_join(state.model, owner_id, scope, meta, result) + sync(%{state | model: model}) + end) + end + + def execute(%__MODULE__{} = state, {:connect, node_id, cluster}) do + node = Map.fetch!(state.nodes, node_id) + :ok = TestCluster.rpc!(node, Group, :connect, [state.name, cluster]) + sync(state) + end + + def execute(%__MODULE__{} = state, {:disconnect, node_id, cluster}) do + node = Map.fetch!(state.nodes, node_id) + :ok = TestCluster.rpc!(node, Group, :disconnect, [state.name, cluster]) + model = ReplicaLifecycleModel.disconnect_cluster(state.model, node, cluster) + sync(%{state | model: model}) + end + + def execute(%__MODULE__{} = state, {:kill, owner_id}) do + case Map.get(state.owners, owner_id) do + nil -> + state + + %{pid: pid, node: node} -> + if remote_alive?(node, pid) do + true = TestCluster.rpc!(node, Process, :exit, [pid, :kill]) + end + + state + |> Map.update!(:model, &ReplicaLifecycleModel.kill(&1, owner_id)) + |> sync() + end + end + + def execute(%__MODULE__{} = state, {:transport, node_id, mode}) do + node = Map.fetch!(state.nodes, node_id) + :ok = TestCluster.rpc!(node, ControlledReplicaTransport, :set_mode, [state.name, mode]) + state + end + + def execute(%__MODULE__{} = state, {:deliver, selector}) do + state + |> sync() + |> take_envelope(selector, fn state, envelope -> + deliver_envelope(state, envelope, 1) + end) + end + + def execute(%__MODULE__{} = state, {:duplicate, selector}) do + state + |> sync() + |> take_envelope(selector, fn state, envelope -> + deliver_envelope(state, envelope, 2) + end) + end + + def execute(%__MODULE__{} = state, {:drop, selector}) do + state + |> sync() + |> take_envelope(selector, fn state, _envelope -> state end) + end + + def execute(%__MODULE__{} = state, :deliver_all) do + state = sync(state) + envelopes = state.queue + + Enum.reduce(envelopes, %{state | queue: []}, fn envelope, acc -> + deliver_envelope(acc, envelope, 1) + end) + end + + def execute(%__MODULE__{} = state, {:restart, node_id}) do + state = sync(state) + node = Map.fetch!(state.nodes, node_id) + :ok = stop_group_local(node, state.name) + model = ReplicaLifecycleModel.restart_node(state.model, node) + {:ok, _pid} = TestCluster.start_group(node, state.group_opts) + state = %{state | model: model} + + TestCluster.assert_eventually( + fn -> + Enum.all?(state.nodes, fn {_id, peer} -> + expected = map_size(state.nodes) - 1 + length(TestCluster.rpc!(peer, Group, :nodes, [state.name])) == expected + end) + end, + timeout: 10_000, + interval: 25 + ) + + sync(state) + end + + def execute(%__MODULE__{} = state, :anti_entropy) do + Enum.each(state.nodes, fn {_id, node} -> + TestCluster.rpc!(node, __MODULE__, :trigger_anti_entropy_local, [state.name]) + end) + + sync(state) + end + + def execute(%__MODULE__{} = state, :flush), do: sync(state) + + def stabilize_and_assert!(%__MODULE__{} = state) do + state = sync(state) + + Enum.each(state.nodes, fn {_id, node} -> + :ok = TestCluster.rpc!(node, ControlledReplicaTransport, :set_mode, [state.name, :pass]) + end) + + expected = ReplicaLifecycleModel.resolve_registry_conflicts(state.model) + state = %{state | model: expected, queue: []} + + TestCluster.assert_eventually( + fn -> + pump_anti_entropy(state) + converged?(state) + end, + timeout: 15_000, + interval: 25 + ) + + Enum.each(state.nodes, fn {_id, node} -> + :ok = TestCluster.rpc!(node, TestCluster, :assert_replica_consistent, [state.name]) + end) + + assert_expected_owner_lifecycle!(state) + assert_no_dead_retained_owners!(state) + state + end + + def cleanup(%__MODULE__{} = state) do + Enum.each(state.nodes, fn {_id, node} -> + TestCluster.rpc!(node, __MODULE__, :cleanup_owners_local, [state.name]) + TestCluster.rpc!(node, ControlledReplicaTransport, :clear, [state.name]) + stop_group_local(node, state.name) + end) + + drain_transport_messages(state.name) + :ok + end + + def sync(%__MODULE__{} = state) do + Enum.each(state.nodes, fn {_id, node} -> + TestCluster.flush_shards(node, state.name) + end) + + drain(state, 2) + end + + def drain(%__MODULE__{} = state, wait_ms \\ 0) do + receive do + {ControlledReplicaTransport, :frame, group, source, target, shard, frame} + when group == state.name -> + envelope = %Envelope{ + id: state.next_frame_id, + source: source, + target: target, + shard: shard, + frame: frame + } + + drain( + %{state | queue: state.queue ++ [envelope], next_frame_id: state.next_frame_id + 1}, + wait_ms + ) + after + wait_ms -> state + end + end + + def trigger_anti_entropy_local(name) do + num_shards = Group.get_config(name).num_shards + + Enum.each(0..(num_shards - 1), fn shard -> + shard_name = Group.Replica.shard_name(name, shard) + state = :sys.get_state(shard_name) + send(shard_name, {:group_replica_anti_entropy, state.anti_entropy_ref}) + end) + + :ok + end + + def spawn_owner(name) do + pid = spawn(fn -> owner_loop() end) + key = {__MODULE__, :owners, name} + owners = :persistent_term.get(key, []) + :persistent_term.put(key, [pid | owners]) + pid + end + + def cleanup_owners_local(name) do + key = {__MODULE__, :owners, name} + + key + |> :persistent_term.get([]) + |> Enum.each(fn pid -> + if Process.alive?(pid), do: Process.exit(pid, :kill) + end) + + :persistent_term.erase(key) + :ok + end + + def call_owner(pid, operation) do + if Process.alive?(pid) do + ref = make_ref() + send(pid, {__MODULE__, :call, self(), ref, operation}) + + receive do + {__MODULE__, :reply, ^ref, result} -> result + after + 5_000 -> raise "model owner #{inspect(pid)} did not answer #{inspect(operation)}" + end + else + {:error, :owner_dead} + end + end + + defp owner_loop do + receive do + {__MODULE__, :call, caller, ref, operation} -> + result = apply_owner_operation(operation) + send(caller, {__MODULE__, :reply, ref, result}) + owner_loop() + end + end + + defp apply_owner_operation({:register, name, key, meta, opts}), + do: Group.register(name, key, meta, opts) + + defp apply_owner_operation({:unregister, name, key, opts}), + do: Group.unregister(name, key, opts) + + defp apply_owner_operation({:join, name, key, meta, opts}), + do: Group.join(name, key, meta, opts) + + defp apply_owner_operation({:leave, name, key, opts}), + do: Group.leave(name, key, opts) + + defp with_owner(state, owner_id, node_id, fun) do + case Map.get(state.owners, owner_id) do + nil -> + node = Map.fetch!(state.nodes, node_id) + pid = TestCluster.rpc!(node, __MODULE__, :spawn_owner, [state.name]) + model = ReplicaLifecycleModel.put_owner(state.model, owner_id, node) + + state = %{ + state + | owners: Map.put(state.owners, owner_id, %{node: node, pid: pid}), + model: model + } + + fun.(state, pid) + + %{node: node, pid: pid} -> + if remote_alive?(node, pid), do: fun.(state, pid), else: state + end + end + + defp with_existing_owner(state, owner_id, fun) do + case Map.get(state.owners, owner_id) do + nil -> + state + + %{node: node, pid: pid} -> + if remote_alive?(node, pid), do: fun.(state, pid), else: state + end + end + + defp owner_call(state, owner_id, operation) do + %{node: node, pid: pid} = Map.fetch!(state.owners, owner_id) + + case TestCluster.rpc!(node, __MODULE__, :call_owner, [pid, operation]) do + {:error, :owner_dead} -> + raise "model owner #{owner_id} died outside an expected lifecycle transition" + + result -> + result + end + end + + defp take_envelope(%{queue: []} = state, _selector, _fun), do: state + + defp take_envelope(state, selector, fun) do + index = rem(selector, length(state.queue)) + {envelope, queue} = List.pop_at(state.queue, index) + fun.(%{state | queue: queue}, envelope) + end + + defp deliver_envelope(state, envelope, times) do + Enum.each(1..times, fn _ -> + :ok = + TestCluster.rpc!( + envelope.target, + Group.Replica.Transport, + :deliver, + [state.name, envelope.source, envelope.shard, envelope.frame] + ) + + TestCluster.flush_shards(envelope.target, state.name) + end) + + drain(state, 2) + end + + defp pump_anti_entropy(state) do + Enum.each(state.nodes, fn {_id, node} -> + TestCluster.rpc!(node, __MODULE__, :trigger_anti_entropy_local, [state.name]) + end) + + Enum.each(1..2, fn _ -> + Enum.each(state.nodes, fn {_id, node} -> + TestCluster.flush_shards(node, state.name) + end) + end) + end + + defp converged?(state) do + expected_registrations = ReplicaLifecycleModel.expected_registrations(state.model) + expected_memberships = ReplicaLifecycleModel.expected_memberships(state.model) + pid_to_owner = Map.new(state.owners, fn {owner_id, %{pid: pid}} -> {pid, owner_id} end) + + Enum.all?(state.nodes, fn {_node_id, node} -> + registrations_match?( + node, + state.name, + state.model.seen_registration_keys, + expected_registrations, + pid_to_owner + ) and + memberships_match?( + node, + state.name, + state.model.seen_membership_keys, + expected_memberships, + pid_to_owner + ) + end) + end + + defp registrations_match?(node, name, keys, expected, pid_to_owner) do + Enum.all?(keys, fn {cluster, key} = scope -> + actual = + case TestCluster.rpc!(node, Group, :lookup, [name, key, cluster_opts(cluster)]) do + nil -> nil + {pid, meta} -> {Map.get(pid_to_owner, pid, {:unknown_pid, pid}), meta} + end + + actual == Map.get(expected, scope) + end) + end + + defp memberships_match?(node, name, keys, expected, pid_to_owner) do + Enum.all?(keys, fn {cluster, key} = scope -> + actual = + node + |> TestCluster.rpc!(Group, :members, [name, key, cluster_opts(cluster)]) + |> Enum.map(fn {pid, meta} -> {Map.get(pid_to_owner, pid, {:unknown_pid, pid}), meta} end) + |> Enum.sort() + + actual == Map.get(expected, scope, []) + end) + end + + defp assert_no_dead_retained_owners!(state) do + retained = + state.nodes + |> Enum.flat_map(fn {_id, node} -> + TestCluster.rpc!(node, TestCluster, :replica_owner_pids, [state.name]) + end) + |> Enum.uniq() + + dead = + Enum.reject(retained, fn pid -> TestCluster.rpc!(node(pid), Process, :alive?, [pid]) end) + + if dead != [], do: raise("dead owners retained after convergence: #{inspect(dead)}") + end + + defp assert_expected_owner_lifecycle!(state) do + mismatches = + Enum.flat_map(state.model.owners, fn {owner_id, %{alive?: expected_alive?}} -> + %{node: node, pid: pid} = Map.fetch!(state.owners, owner_id) + actual_alive? = remote_alive?(node, pid) + + if expected_alive? == actual_alive? do + [] + else + [{owner_id, expected_alive?, actual_alive?, pid}] + end + end) + + if mismatches != [] do + raise "owner lifecycle diverged from model: #{inspect(mismatches)}" + end + end + + defp remote_alive?(node, pid), do: TestCluster.rpc!(node, Process, :alive?, [pid]) + + defp stop_group_local(node, name) do + case TestCluster.rpc!(node, Process, :whereis, [:"#{name}_group_sup"]) do + nil -> :ok + pid -> TestCluster.rpc!(node, Supervisor, :stop, [pid, :normal, 5_000]) + end + catch + :exit, _ -> :ok + end + + defp drain_transport_messages(name) do + receive do + {ControlledReplicaTransport, :frame, ^name, _source, _target, _shard, _frame} -> + drain_transport_messages(name) + after + 0 -> :ok + end + end + + defp registration_key(owner_id, slot), do: "model/reg/#{owner_id}/#{slot}" + defp membership_key(owner_id, slot), do: "model/pg/#{owner_id}/#{slot}" + defp cluster_registration_key(owner_id, slot), do: "model/cluster/reg/#{owner_id}/#{slot}" + defp cluster_membership_key(owner_id, slot), do: "model/cluster/pg/#{owner_id}/#{slot}" + defp cluster_opts(nil), do: [] + defp cluster_opts(cluster), do: [cluster: cluster] +end diff --git a/test/support/test_cluster.ex b/test/support/test_cluster.ex index ebffc86..8f97eb7 100644 --- a/test/support/test_cluster.ex +++ b/test/support/test_cluster.ex @@ -5,16 +5,36 @@ defmodule Group.TestCluster do def start_peers(count, opts \\ []) do cookie = Keyword.get(opts, :cookie, Node.get_cookie()) code_paths = :code.get_path() + schedulers = Keyword.get(opts, :schedulers) + + scheduler_args = + if schedulers, do: [~c"+S", ~c"#{schedulers}:#{schedulers}"], else: [] args = - [~c"-setcookie", ~c"#{cookie}", ~c"-kernel", ~c"prevent_overlapping_partitions", ~c"false"] ++ + scheduler_args ++ + [ + ~c"-setcookie", + ~c"#{cookie}", + ~c"-kernel", + ~c"prevent_overlapping_partitions", + ~c"false" + ] ++ Enum.flat_map(code_paths, fn p -> [~c"-pa", p] end) for _i <- 1..count do name = :"peer#{System.unique_integer([:positive])}" + # A fixed inet_dist_listen_min/max inherited through ERL_AFLAGS makes + # every child contend for the parent VM's distribution port. Peer args + # above carry every setting the test nodes require explicitly. {:ok, pid, node} = - :peer.start(%{name: name, host: ~c"127.0.0.1", longnames: true, args: args}) + :peer.start(%{ + name: name, + host: ~c"127.0.0.1", + longnames: true, + args: args, + env: [{~c"ERL_AFLAGS", ~c""}] + }) {:ok, _} = :rpc.call(node, :application, :ensure_all_started, [:elixir]) {:ok, _} = :rpc.call(node, :application, :ensure_all_started, [:group]) @@ -659,6 +679,59 @@ defmodule Group.TestCluster do :ok end + @doc false + def assert_replica_origin_purged(name, origin) do + num_shards = Group.get_config(name).num_shards + + for shard <- 0..(num_shards - 1) do + retained_claims = + Group.Replica.Data.reg_claim_by_key_table(name, shard) + |> :ets.tab2list() + |> Enum.filter(fn {{_cluster, _key, row_origin, _generation, _epoch}, _pid, _meta, _time, + _seq} -> + row_origin == origin + end) + + retained_registry = + Group.Replica.Data.reg_by_key_table(name, shard) + |> :ets.tab2list() + |> Enum.filter(fn {_key, _pid, _meta, _time, entry_node} -> entry_node == origin end) + + retained_pg = + Group.Replica.Data.pg_by_key_table(name, shard) + |> :ets.tab2list() + |> Enum.filter(fn {_key, _meta, _time, entry_node} -> entry_node == origin end) + + retained_cursors = + Group.Replica.Data.replica_cursor_table(name, shard) + |> :ets.tab2list() + |> Enum.filter(fn {stream_id, _seq} -> + Group.Replica.Protocol.stream_origin(stream_id) == origin + end) + + retained_view = Group.Replica.Data.remote_view_generation(name, shard, origin) + + unless retained_claims == [] and retained_registry == [] and retained_pg == [] and + retained_cursors == [] and is_nil(retained_view) do + raise "replica origin was not fully purged from #{name} shard #{shard}: " <> + inspect(%{ + claims: retained_claims, + registry: retained_registry, + pg: retained_pg, + cursors: retained_cursors, + view_generation: retained_view + }) + end + end + + unless is_nil(Group.Replica.Data.remote_generation(name, origin)) and + Group.Replica.Data.clusters_for_node(name, origin) == [] do + raise "replica origin retained shared authority after purge: #{inspect(origin)}" + end + + :ok + end + @doc """ Returns every PID currently retained as replica authority or visible PG state. @@ -778,6 +851,13 @@ defmodule Group.TestCluster do "order_only=#{inspect(MapSet.difference(order, oplog) |> MapSet.to_list())}" end + max_entries = Group.get_config(name).replicated_oplog_max_entries + + if MapSet.size(order) > max_entries do + raise "oplog bound exceeded in #{name} shard #{shard}: " <> + "size=#{MapSet.size(order)} max=#{max_entries}" + end + Group.Replica.Data.replica_stream_meta_table(name, shard) |> :ets.tab2list() |> Enum.each(fn {stream_id, head, floor, applied} -> From 976bdb5b9c15ab27d9566e9ab72c10bbb6531bef Mon Sep 17 00:00:00 2001 From: Chris McCord Date: Sun, 9 Aug 2026 17:52:07 +0000 Subject: [PATCH 04/16] Chunk exact snapshots and harden transport boundaries --- CHANGELOG.md | 9 + README.md | 50 +- lib/group.ex | 5 + lib/group/replica.ex | 494 +++++++++++++++-- lib/group/replica/data.ex | 43 ++ lib/group/replica/protocol.ex | 2 +- lib/group/replica/snapshot.ex | 144 +++++ lib/group/replica/transport.ex | 23 + lib/group/replica/transport/outbox.ex | 318 +++++++++++ lib/group/replica/transport/tcp.ex | 123 +++-- lib/group/supervisor.ex | 4 + priv/bench/README.md | 8 + priv/bench/lib/group_bench/distributed.ex | 59 ++ test/README.md | 11 +- test/distributed_test.exs | 4 +- test/formal/README.md | 15 + test/formal/SnapshotAssembly.cfg | 7 + test/formal/SnapshotAssembly.tla | 181 ++++++ test/formal/check.sh | 3 +- test/group_test.exs | 2 +- test/mutation/README.md | 4 +- test/mutation/run.exs | 141 ++++- test/replica_model_property_test.exs | 17 +- test/replica_snapshot_distributed_test.exs | 608 +++++++++++++++++++++ test/replica_snapshot_test.exs | 75 +++ test/replica_transport_outbox_test.exs | 155 ++++++ test/support/test_cluster.ex | 22 + 27 files changed, 2413 insertions(+), 114 deletions(-) create mode 100644 lib/group/replica/snapshot.ex create mode 100644 lib/group/replica/transport/outbox.ex create mode 100644 test/formal/SnapshotAssembly.cfg create mode 100644 test/formal/SnapshotAssembly.tla create mode 100644 test/replica_snapshot_distributed_test.exs create mode 100644 test/replica_snapshot_test.exs create mode 100644 test/replica_transport_outbox_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 22721e4..671cf9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,13 @@ ## Unreleased +- **Breaking**: replica protocol v2 splits exact snapshots into + transport-neutral, byte-targeted chunks (`1 MiB` by default). Receivers stage + chunks in shard-owned private ETS and advance the stream cursor only after an + exact, authority-fenced assembly is complete; loss, duplication, reordering, + supersession, expiry, and shard crashes remain repairable by anti-entropy. + Single-chunk snapshots retain a direct fast path. Sideband transports can use + per-shard local outboxes for bounded batching without adding a hop to the + default dist-Erlang adapter. Late-starting replica lanes now rebuild their + view from shared exact authority when startup fanout races registration. - Replace replica state sends/snapshots with per-origin, generation- and cluster-epoch-fenced streams: sequenced deltas repair gaps from a bounded oplog and fall back to exact origin snapshots after pruning. Replica data now diff --git a/README.md b/README.md index c9d351e..00ea74c 100644 --- a/README.md +++ b/README.md @@ -243,6 +243,7 @@ All operations are **eventually consistent**: replicated_pg_receiver_local_request_quota: 8, replica_transport: Group.Replica.Transport.Distribution, replicated_oplog_max_entries: 65_536, + replicated_snapshot_chunk_target_bytes: 1_048_576, replicated_anti_entropy_interval: 1_000, replicated_peer_lease_timeout: 15_000 } @@ -293,13 +294,18 @@ All operations are **eventually consistent**: `Group.Replica.Transport`, or `{module, opts}`. The default adapter uses `:erlang.send_nosuspend/3`; adapters must return promptly with `:ok`, `:busy`, or `:disconnected`. Dropped and busy frames are repaired by anti-entropy. - `Group.Replica.Transport.TCP` is an included sideband adapter with bounded - per-peer writer queues; its socket owners are separate processes, so socket - backpressure cannot block a Group shard. + `Group.Replica.Transport.TCP` is an included sideband adapter with local + per-shard batching and bounded per-peer writer queues; its socket owners are + separate processes, so socket backpressure cannot block a Group shard. - **`replicated_oplog_max_entries`** — maximum retained replica records per shard across all local streams. Defaults to 65,536. Pruning never waits for peer acknowledgements; a peer behind the retained floor receives an exact snapshot. +- **`replicated_snapshot_chunk_target_bytes`** — target maximum encoded size + of each exact-snapshot frame. Defaults to 1 MiB and applies above every + transport, including dist Erlang. A single row larger than the target is + sent alone. Receivers stage chunks in shard-owned private ETS and replace + visible state only after the complete exact slice is present. - **`replicated_anti_entropy_interval`** — interval in milliseconds for stream head advertisements and nonblocking control heartbeats. Defaults to 1,000. - **`replicated_peer_lease_timeout`** — time without a dist-Erlang control @@ -311,7 +317,7 @@ All operations are **eventually consistent**: ``` Group.Supervisor (:"my_app_group_sup") -├── optional transport child — sideband adapter listener/pool +├── optional transport child — sideband manager and per-shard outboxes ├── Group.Replica.Data — owns ETS, journal, generations, and epochs ├── Group.PeerReconnect — bounded recovery after busy remote dispatch ├── Group.Replica.Supervisor — supervises N shard GenServers @@ -422,10 +428,14 @@ registry claims and PG memberships; absence from that snapshot is a delete. There are no leaders, quorum acknowledgements, per-entry replicated tombstones, or known-membership retention barriers. Oplog memory is bounded locally and independently of slow peers. Deletes are normal ordered records while retained, -and exact snapshots close gaps after pruning. Named-cluster close uses only a -temporary local shard-completion barrier; the final shard removes it and all -routing rows, including after a caller timeout or shard restart. Reconnect -waits for that barrier so a prior close cannot erase newly accepted writes. +and exact snapshots close gaps after pruning. Exact snapshots are split into +transport-neutral byte-bounded frames; loss, duplication, or reordering leaves +the old visible slice and cursor untouched until all chunks arrive. Incomplete +staging expires after a peer-lease interval without progress and is destroyed +automatically with its owning shard. Named-cluster close uses only a temporary +local shard-completion barrier; the final shard removes it and all routing rows, +including after a caller timeout or shard restart. Reconnect waits for that +barrier so a prior close cannot erase newly accepted writes. The sender flush timer is mainly a fallback for idle periods. The unified outbound buffer also flushes immediately when it hits the configured size, when a new enqueue @@ -450,7 +460,11 @@ replica_transport: ip: {0, 0, 0, 0}, advertised_ip: {10, 0, 1, 12}, port: 44_321, - max_queue: 1_024 + max_queue: 1_024, + outbox_batch_size: 64, + outbox_batch_bytes: 1_048_576, + outbox_flush_interval: 1, + outbox_deadline: 100 ]} ``` @@ -460,6 +474,24 @@ network or place the connection behind TLS. The adapter deliberately has no control/data ordering relationship; the generation/epoch lane barrier and stream sequence checks supply correctness. +The default distribution adapter still sends directly to the remote shard and +does not pay for a local outbox. Sideband adapters can delegate `try_send/5` to +`Group.Replica.Transport.Outbox.try_send/5` and supervise one outbox per shard +with `Group.Replica.Transport.Outbox.child_spec/1`. An outbox groups frames by +target and invokes the adapter's `send_batch/4` callback. Calls that expire or +return `:busy`/`:disconnected` are dropped without a local retry; the next +anti-entropy exchange repairs them. + +A message-oriented backend fits this callback shape by obtaining a connection +once from `init_outbox/3`, then sending each `send_batch/4` result to a +registered ingress name on the target node. Queue pressure maps to `:busy` and +a missing session maps to `:disconnected`. Ingress must attach the authenticated +connection's source node; an adapter must never trust a source node supplied +inside the payload. Exact snapshots are already bounded by Group. A transport +with a smaller maximum frame may additionally segment an encoded batch, but it +must completely reassemble that batch before calling +`Group.Replica.Transport.deliver_batch/4`. + ### Named Cluster TTL Leases Named-cluster TTLs are a local way to reduce replication fanout to nodes that diff --git a/lib/group.ex b/lib/group.ex index 449e4fa..e3dd976 100644 --- a/lib/group.ex +++ b/lib/group.ex @@ -229,9 +229,14 @@ defmodule Group do - `:replica_transport` — replica data transport module or `{module, opts}` tuple. Defaults to `Group.Replica.Transport.Distribution`. The transport must be nonblocking and may return `:busy`; anti-entropy repairs dropped frames. + Sideband transports can use `Group.Replica.Transport.Outbox` for lossy, + batched, per-shard isolation without adding a hop to the default transport. - `:replicated_oplog_max_entries` — maximum retained replica records per shard before old prefixes are pruned and lagging peers require a snapshot (default: `65_536`) + - `:replicated_snapshot_chunk_target_bytes` — target maximum encoded size of + each transport-neutral exact-snapshot chunk (default: `1_048_576`). A + single registry or membership row larger than the target remains one chunk. - `:replicated_anti_entropy_interval` — milliseconds between repeated stream head advertisements (default: `1_000`) - `:replicated_peer_lease_timeout` — milliseconds without a dist-Erlang diff --git a/lib/group/replica.ex b/lib/group/replica.ex index 58ce632..d0f888d 100644 --- a/lib/group/replica.ex +++ b/lib/group/replica.ex @@ -60,14 +60,17 @@ defmodule Group.Replica do - heads advertises {stream, retained_floor, head}. - delta_batch carries one or more contiguous stream runs. - need requests the receiver's next missing sequence. - - snapshot exactly replaces one origin's registry claims and PG slice when - the requested prefix has already been pruned. + - snapshot_chunk carries a byte-bounded part of one exact origin slice when + the requested prefix has already been pruned. Receivers stage chunks in a + private ETS table and expose nothing until every chunk is present. Every stream field is validated against the authenticated source node and current generation/epoch. An old generation, a closed epoch, a wrong shard, or a transitive claim for another node's pid is rejected. Control/data reordering is safe: early frames are ignored and repeated heads repair them; - late frames fail their generation or epoch fence. + late frames fail their generation or epoch fence. Snapshot chunks may be + lost, duplicated, reordered, or mixed across retransmissions at the same + stream head; exact row counts and set insertion prevent partial commits. ## Bounded recovery @@ -117,13 +120,13 @@ defmodule Group.Replica do yielding. FIFO is preserved within the local request lane, while protocol and cluster barriers flush earlier buffered state first. - Receive-only handlers for the previous direct batch/snapshot messages remain - for rolling compatibility and tests; protocol v1 never emits them. + Snapshot staging is owned by the receiving shard, expires after a peer lease + without progress, and disappears automatically if the shard crashes. """ require Logger - alias Group.Replica.{Data, Protocol} + alias Group.Replica.{Data, Protocol, Snapshot} defstruct [ :name, @@ -137,6 +140,7 @@ defmodule Group.Replica do :replicated_sender_flush_interval, :replicated_pg_receiver_local_request_quota, :replicated_oplog_max_entries, + :replicated_snapshot_chunk_target_bytes, :replicated_anti_entropy_interval, :replicated_peer_lease_timeout, :replica_transport, @@ -159,7 +163,8 @@ defmodule Group.Replica do cluster_control_dirty: %{}, authority_dirty_notified: MapSet.new(), monitors: %{}, - peer_transports: %{} + peer_transports: %{}, + snapshot_transfers: %{} ] def start_link(opts) do @@ -240,6 +245,7 @@ defmodule Group.Replica do replicated_pg_receiver_local_request_quota: config.replicated_pg_receiver_local_request_quota, replicated_oplog_max_entries: config.replicated_oplog_max_entries, + replicated_snapshot_chunk_target_bytes: config.replicated_snapshot_chunk_target_bytes, replicated_anti_entropy_interval: config.replicated_anti_entropy_interval, replicated_peer_lease_timeout: config.replicated_peer_lease_timeout, replica_transport: elem(config.replica_transport, 0), @@ -473,9 +479,10 @@ defmodule Group.Replica do {:noreply, state} replica_authority_current?(state, remote_node, generation, epoch_revision) -> - # Shared authority arrived first; its local fanout is already the - # ordered marker that will purge and install this lane's view. - {:noreply, state} + # The shared authority can arrive before this sibling is registered, + # so shard-zero fanout is intentionally lossy at startup. Rebuild + # this lane directly from the exact shared authority. + {:noreply, install_current_replica_lane(state, remote_node, generation)} true -> {:noreply, request_replica_authority(state, remote_node)} @@ -768,10 +775,17 @@ defmodule Group.Replica do {:noreply, take_priority_turn(state)} end + def handle_info({:group_replica_batch, remote_node, frames}, state) + when is_atom(remote_node) and is_list(frames) do + state = Enum.reduce(frames, state, &handle_replica_frame(&2, remote_node, &1)) + {:noreply, take_priority_turn(state)} + end + def handle_info({@anti_entropy_timer, ref}, state) do state = if state.anti_entropy_ref == ref do state + |> expire_stale_snapshot_transfers() |> expire_stale_replica_peers() |> probe_replica_peers() |> request_quiet_cluster_hellos() @@ -2952,6 +2966,20 @@ defmodule Group.Replica do ) end + defp install_current_replica_lane(state, remote_node, generation) do + old_generation = + Data.remote_view_generation(state.name, state.shard_index, remote_node) + + state = maybe_purge_remote_generation(state, remote_node, old_generation, generation) + state = purge_remote_streams_outside_authority(state, remote_node) + :ok = install_replica_view(state, remote_node, generation) + + state + |> touch_replica_peer(remote_node) + |> Map.update!(:cluster_control_dirty, &Map.delete(&1, remote_node)) + |> send_replica_heads(remote_node) + end + defp schedule_anti_entropy(state) do ref = make_ref() @@ -3116,7 +3144,45 @@ defmodule Group.Replica do end) end + defp expire_stale_snapshot_transfers(%{snapshot_transfers: transfers} = state) + when map_size(transfers) == 0, + do: state + + defp expire_stale_snapshot_transfers(state) do + now = monotonic_millis() + + Enum.reduce(state.snapshot_transfers, state, fn {key, transfer}, acc -> + if now - transfer.last_progress > acc.replicated_peer_lease_timeout do + discard_snapshot_transfer(acc, key) + else + acc + end + end) + end + + defp discard_snapshot_transfer(state, key) do + case Map.pop(state.snapshot_transfers, key) do + {nil, transfers} -> + %{state | snapshot_transfers: transfers} + + {transfer, transfers} -> + :ok = Snapshot.delete_staging_table(transfer.table) + %{state | snapshot_transfers: transfers} + end + end + + defp discard_snapshot_transfers_for_source(state, source_node) do + Enum.reduce(state.snapshot_transfers, state, fn + {{^source_node, _stream_id} = key, _transfer}, acc -> + discard_snapshot_transfer(acc, key) + + {_key, _transfer}, acc -> + acc + end) + end + defp expire_replica_peer(state, remote_node) do + state = discard_snapshot_transfers_for_source(state, remote_node) %{name: name, shard_index: shard} = state if shard == 0 do @@ -3268,44 +3334,279 @@ defmodule Group.Replica do defp handle_replica_frame( state, source_node, - {:snapshot, version, stream_id, snapshot_seq, reg_data, pg_data} + {:snapshot_chunk, version, stream_id, snapshot_seq, chunk_index, chunk_count, + registry_count, pg_count, reg_data, pg_data} ) - when version == @protocol_version do - if valid_remote_stream?(state, source_node, stream_id) and - snapshot_seq >= Data.replica_cursor(state.name, state.shard_index, stream_id) do - state = flush_pending_replicated_barrier(state) - cluster = Protocol.stream_cluster(stream_id) + when version == @protocol_version and is_integer(snapshot_seq) and snapshot_seq >= 0 and + is_integer(chunk_index) and is_integer(chunk_count) and + is_integer(registry_count) and is_integer(pg_count) and is_list(reg_data) and + is_list(pg_data) do + if valid_snapshot_stream?(state, source_node, stream_id, snapshot_seq) and + valid_snapshot_manifest?( + chunk_index, + chunk_count, + registry_count, + pg_count, + reg_data, + pg_data + ) and + valid_snapshot_rows?(state, source_node, stream_id, reg_data, pg_data) do + if chunk_count == 1 and registry_count == length(reg_data) and + pg_count == length(pg_data) do + apply_complete_snapshot_rows( + state, + source_node, + stream_id, + snapshot_seq, + reg_data, + pg_data + ) + else + stage_replica_snapshot_chunk( + state, + source_node, + stream_id, + snapshot_seq, + chunk_index, + chunk_count, + registry_count, + pg_count, + reg_data, + pg_data + ) + end + else + state + end + end + + defp handle_replica_frame(state, _source_node, _frame), do: state + + defp valid_snapshot_stream?(state, source_node, stream_id, snapshot_seq) do + valid_remote_stream?(state, source_node, stream_id) and + snapshot_seq > Data.replica_cursor(state.name, state.shard_index, stream_id) + end - reg_data = - Enum.filter(reg_data, fn {key, pid, _meta, _time} -> + defp valid_snapshot_manifest?( + chunk_index, + chunk_count, + registry_count, + pg_count, + reg_data, + pg_data + ) do + total_count = registry_count + pg_count + chunk_row_count = length(reg_data) + length(pg_data) + + chunk_count > 0 and chunk_index > 0 and chunk_index <= chunk_count and + registry_count >= 0 and pg_count >= 0 and + chunk_count <= max(total_count, 1) and + ((total_count == 0 and chunk_count == 1 and chunk_row_count == 0) or + (total_count > 0 and chunk_row_count > 0)) + end + + defp valid_snapshot_rows?(state, source_node, stream_id, reg_data, pg_data) do + cluster = Protocol.stream_cluster(stream_id) + + Enum.all?(reg_data, fn + {key, pid, _meta, _time} when is_pid(pid) -> + node(pid) == source_node and + shard_index_for(cluster, key, state.num_shards) == state.shard_index + + _other -> + false + end) and + Enum.all?(pg_data, fn + {key, pid, _meta, _time} when is_pid(pid) -> node(pid) == source_node and shard_index_for(cluster, key, state.num_shards) == state.shard_index - end) - affected_registry_keys = - Data.replace_registry_claims_for_stream( - state.name, - state.shard_index, - stream_id, - snapshot_seq, - reg_data - ) + _other -> + false + end) + end - {state, events} = - Enum.reduce(affected_registry_keys, {state, []}, fn key, {acc, inner_events} -> - reconcile_registry_projection(acc, cluster, key, :reconcile, inner_events) - end) + defp stage_replica_snapshot_chunk( + state, + source_node, + stream_id, + snapshot_seq, + chunk_index, + chunk_count, + registry_count, + pg_count, + reg_data, + pg_data + ) do + key = {source_node, stream_id} + manifest = {chunk_count, registry_count, pg_count} - events = replace_remote_pg_snapshot(state, source_node, cluster, pg_data, events) - :ok = Data.put_replica_cursor(state.name, state.shard_index, stream_id, snapshot_seq) - notify_monitors(state.name, events) - state + case snapshot_transfer(state, key, snapshot_seq, manifest) do + {:ignore, state} -> + state + + {:ok, state, transfer} -> + cond do + MapSet.member?(transfer.received, chunk_index) -> + state + + transfer.registry_seen + length(reg_data) > registry_count or + transfer.pg_seen + length(pg_data) > pg_count -> + discard_snapshot_transfer(state, key) + + true -> + case Snapshot.stage_rows(transfer.table, chunk_index, reg_data, pg_data) do + :ok -> + transfer = %{ + transfer + | received: MapSet.put(transfer.received, chunk_index), + registry_seen: transfer.registry_seen + length(reg_data), + pg_seen: transfer.pg_seen + length(pg_data), + last_progress: monotonic_millis() + } + + state = put_snapshot_transfer(state, key, transfer) + maybe_commit_snapshot_transfer(state, key, source_node, stream_id) + + {:error, :duplicate_row} -> + discard_snapshot_transfer(state, key) + end + end + end + end + + defp snapshot_transfer(state, key, snapshot_seq, manifest) do + case Map.get(state.snapshot_transfers, key) do + nil -> + {:ok, state, new_snapshot_transfer(snapshot_seq, manifest)} + + %{snapshot_seq: existing_seq} when existing_seq > snapshot_seq -> + {:ignore, state} + + %{snapshot_seq: existing_seq} when existing_seq < snapshot_seq -> + state = discard_snapshot_transfer(state, key) + {:ok, state, new_snapshot_transfer(snapshot_seq, manifest)} + + %{manifest: ^manifest} = transfer -> + {:ok, state, transfer} + + _conflicting_transfer -> + {:ignore, discard_snapshot_transfer(state, key)} + end + end + + defp new_snapshot_transfer(snapshot_seq, {chunk_count, registry_count, pg_count} = manifest) do + %{ + snapshot_seq: snapshot_seq, + manifest: manifest, + chunk_count: chunk_count, + registry_count: registry_count, + pg_count: pg_count, + registry_seen: 0, + pg_seen: 0, + received: MapSet.new(), + last_progress: monotonic_millis(), + table: Snapshot.new_staging_table() + } + end + + defp put_snapshot_transfer(state, key, transfer) do + %{state | snapshot_transfers: Map.put(state.snapshot_transfers, key, transfer)} + end + + defp maybe_commit_snapshot_transfer(state, key, source_node, stream_id) do + transfer = Map.fetch!(state.snapshot_transfers, key) + + if MapSet.size(transfer.received) == transfer.chunk_count do + if transfer.registry_seen == transfer.registry_count and + transfer.pg_seen == transfer.pg_count do + commit_snapshot_transfer(state, key, source_node, stream_id, transfer) + else + discard_snapshot_transfer(state, key) + end else state end end - defp handle_replica_frame(state, _source_node, _frame), do: state + defp commit_snapshot_transfer(state, key, source_node, stream_id, transfer) do + state = + if valid_snapshot_stream?(state, source_node, stream_id, transfer.snapshot_seq) do + state = flush_pending_replicated_barrier(state) + cluster = Protocol.stream_cluster(stream_id) + + affected_registry_keys = + Data.replace_registry_claims_for_stream_from_staging( + state.name, + state.shard_index, + stream_id, + transfer.snapshot_seq, + transfer.table, + transfer.chunk_count + ) + + {state, events} = + Enum.reduce(affected_registry_keys, {state, []}, fn key, {acc, inner_events} -> + reconcile_registry_projection(acc, cluster, key, :reconcile, inner_events) + end) + + events = + replace_remote_pg_snapshot_from_staging( + state, + source_node, + cluster, + transfer.table, + transfer.chunk_count, + events + ) + + :ok = + Data.put_replica_cursor( + state.name, + state.shard_index, + stream_id, + transfer.snapshot_seq + ) + + notify_monitors(state.name, events) + state + else + state + end + + discard_snapshot_transfer(state, key) + end + + defp apply_complete_snapshot_rows( + state, + source_node, + stream_id, + snapshot_seq, + reg_data, + pg_data + ) do + state = flush_pending_replicated_barrier(state) + cluster = Protocol.stream_cluster(stream_id) + + affected_registry_keys = + Data.replace_registry_claims_for_stream( + state.name, + state.shard_index, + stream_id, + snapshot_seq, + reg_data + ) + + {state, events} = + Enum.reduce(affected_registry_keys, {state, []}, fn key, {acc, inner_events} -> + reconcile_registry_projection(acc, cluster, key, :reconcile, inner_events) + end) + + events = replace_remote_pg_snapshot_rows(state, source_node, cluster, pg_data, events) + :ok = Data.put_replica_cursor(state.name, state.shard_index, stream_id, snapshot_seq) + notify_monitors(state.name, events) + state + end defp apply_replica_delta_run(state, source_node, stream_id, records, advertised_head) do if valid_remote_stream?(state, source_node, stream_id) do @@ -3624,33 +3925,117 @@ defmodule Group.Replica do defp send_replica_snapshot(state, target_node, stream_id, head) do cluster = Protocol.stream_cluster(stream_id) + reg_data = Data.registry_claims_for_stream(state.name, state.shard_index, stream_id) + pg_data = Data.pg_entries_for_origin(state.name, state.shard_index, cluster, node()) - {_reg_by_cluster, pg_by_cluster} = - Data.local_data_by_cluster(state.name, state.shard_index, [cluster]) + envelope_bytes = + Snapshot.frame_envelope_bytes(stream_id, head, length(reg_data), length(pg_data)) - reg_data = Data.registry_claims_for_stream(state.name, state.shard_index, stream_id) + snapshot = + Snapshot.chunk_rows( + reg_data, + pg_data, + state.replicated_snapshot_chunk_target_bytes, + envelope_bytes + ) - try_send_replica_frame( - state, - target_node, - {:snapshot, Protocol.version(), stream_id, head, reg_data, - Map.get(pg_by_cluster, cluster, [])} - ) + chunk_count = length(snapshot.chunks) + + snapshot.chunks + |> Enum.with_index(1) + |> Enum.reduce(state, fn {{reg_chunk, pg_chunk}, chunk_index}, acc -> + try_send_replica_frame( + acc, + target_node, + {:snapshot_chunk, Protocol.version(), stream_id, head, chunk_index, chunk_count, + snapshot.registry_count, snapshot.pg_count, reg_chunk, pg_chunk} + ) + end) + end + + defp replace_remote_pg_snapshot_from_staging( + state, + source_node, + cluster, + staging_table, + chunk_count, + events + ) do + current = Data.pg_entries_for_origin(state.name, state.shard_index, cluster, source_node) + + events = + Snapshot.fold_pg(staging_table, chunk_count, events, fn {key, pid, meta, time}, acc -> + case Data.pg_lookup(state.name, state.shard_index, cluster, key, pid) do + nil -> + :ok = + Data.pg_insert( + state.name, + state.shard_index, + cluster, + key, + pid, + meta, + time, + source_node + ) + + [build_event(state.name, :joined, key, pid, meta, %{cluster: cluster}) | acc] + + {^meta, ^time, ^source_node} -> + acc + + {old_meta, _old_time, ^source_node} -> + :ok = + Data.pg_insert( + state.name, + state.shard_index, + cluster, + key, + pid, + meta, + time, + source_node + ) + + if old_meta == meta do + acc + else + [ + build_event(state.name, :joined, key, pid, meta, %{ + previous_meta: old_meta, + cluster: cluster + }) + | acc + ] + end + end + end) + + Enum.reduce(current, events, fn {key, pid, old_meta, _old_time}, acc -> + if Snapshot.member_pg?(staging_table, key, pid) do + acc + else + :ok = Data.pg_delete(state.name, state.shard_index, cluster, key, pid) + + event = + build_event(state.name, :left, key, pid, old_meta, %{ + reason: :reconcile, + cluster: cluster + }) + + [event | acc] + end + end) end - defp replace_remote_pg_snapshot(state, source_node, cluster, pg_data, events) do + defp replace_remote_pg_snapshot_rows(state, source_node, cluster, pg_data, events) do current = state.name |> Data.pg_entries_for_origin(state.shard_index, cluster, source_node) |> Map.new(fn {key, pid, meta, time} -> {{key, pid}, {meta, time}} end) desired = - pg_data - |> Enum.filter(fn {key, pid, _meta, _time} -> - node(pid) == source_node and - shard_index_for(cluster, key, state.num_shards) == state.shard_index - end) - |> Map.new(fn {key, pid, meta, time} -> {{key, pid}, {meta, time}} end) + Map.new(pg_data, fn {key, pid, meta, time} -> {{key, pid}, {meta, time}} end) {inserts, deletes, events} = current @@ -4029,7 +4414,10 @@ defmodule Group.Replica do %{name: name, shard_index: shard_index, num_shards: num_shards} = state for i <- 0..(num_shards - 1), i != shard_index do - send(shard_name(name, i), message) + case Process.whereis(shard_name(name, i)) do + pid when is_pid(pid) -> send(pid, message) + nil -> :ok + end end end diff --git a/lib/group/replica/data.ex b/lib/group/replica/data.ex index 94f803a..7511fc7 100644 --- a/lib/group/replica/data.ex +++ b/lib/group/replica/data.ex @@ -939,6 +939,49 @@ defmodule Group.Replica.Data do Enum.uniq(Enum.map(existing, &elem(&1, 0)) ++ Enum.map(claims, &elem(&1, 0))) end + def replace_registry_claims_for_stream_from_staging( + name, + shard, + stream_id, + snapshot_seq, + staging_table, + chunk_count + ) do + cluster = Group.Replica.Protocol.stream_cluster(stream_id) + origin_node = Group.Replica.Protocol.stream_origin(stream_id) + generation = Group.Replica.Protocol.stream_generation(stream_id) + epoch = Group.Replica.Protocol.stream_epoch(stream_id) + existing = registry_claims_for_stream(name, shard, stream_id) + + keys = + Enum.reduce(existing, MapSet.new(), fn {key, pid, _meta, _time}, keys -> + :ets.delete( + reg_claim_by_key_table(name, shard), + {cluster, key, origin_node, generation, epoch} + ) + + :ets.delete( + reg_claim_by_pid_table(name, shard), + {pid, cluster, key, origin_node, generation, epoch} + ) + + MapSet.put(keys, key) + end) + + keys = + Group.Replica.Snapshot.fold_registry( + staging_table, + chunk_count, + keys, + fn {key, pid, meta, time}, keys -> + put_registry_claim(name, shard, stream_id, snapshot_seq, key, pid, meta, time) + MapSet.put(keys, key) + end + ) + + MapSet.to_list(keys) + end + def purge_registry_claims_for_origin(name, shard, origin_node) do claims = :ets.select(reg_claim_by_key_table(name, shard), [ diff --git a/lib/group/replica/protocol.ex b/lib/group/replica/protocol.ex index fe88e24..a79d983 100644 --- a/lib/group/replica/protocol.ex +++ b/lib/group/replica/protocol.ex @@ -1,7 +1,7 @@ defmodule Group.Replica.Protocol do @moduledoc false - @version 1 + @version 2 def version, do: @version diff --git a/lib/group/replica/snapshot.ex b/lib/group/replica/snapshot.ex new file mode 100644 index 0000000..13dc29a --- /dev/null +++ b/lib/group/replica/snapshot.ex @@ -0,0 +1,144 @@ +defmodule Group.Replica.Snapshot do + @moduledoc false + + # The target is for the complete snapshot frame, not just its rows. Per-row + # external sizes conservatively include an extra ETF version byte, and this + # reserve covers the frame tuple, stream identity, both list headers, and + # integer fields. A single entry larger than the target remains one chunk. + @default_envelope_reserve 512 + @max_compact_chunk_count 2_147_483_647 + + def chunk_rows(registry_rows, pg_rows, target_bytes) + when is_list(registry_rows) and is_list(pg_rows) and is_integer(target_bytes) and + target_bytes > 0 do + chunk_rows(registry_rows, pg_rows, target_bytes, @default_envelope_reserve) + end + + def chunk_rows(registry_rows, pg_rows, target_bytes, envelope_bytes) + when is_list(registry_rows) and is_list(pg_rows) and is_integer(target_bytes) and + target_bytes > 0 and is_integer(envelope_bytes) and envelope_bytes >= 0 do + registry_rows = Enum.sort_by(registry_rows, fn {key, _pid, _meta, _time} -> key end) + pg_rows = Enum.sort_by(pg_rows, fn {key, pid, _meta, _time} -> {key, pid} end) + payload_target = max(target_bytes - envelope_bytes, 1) + + acc = %{chunks: [], registry: [], pg: [], bytes: 0, count: 0} + acc = Enum.reduce(registry_rows, acc, &add_row(&2, :registry, &1, payload_target)) + acc = Enum.reduce(pg_rows, acc, &add_row(&2, :pg, &1, payload_target)) + + chunks = + acc + |> flush_chunk() + |> Map.fetch!(:chunks) + |> Enum.reverse() + |> case do + [] -> [{[], []}] + chunks -> chunks + end + + %{ + registry_count: length(registry_rows), + pg_count: length(pg_rows), + chunks: chunks + } + end + + def frame_envelope_bytes(stream_id, snapshot_seq, registry_count, pg_count) do + # A non-empty ETF list adds a five-byte LIST_EXT header relative to an + # empty list. Reserve that once for each domain. The large chunk integers + # ensure every practical index/count uses no more space than this envelope. + :erlang.external_size( + {:snapshot_chunk, Group.Replica.Protocol.version(), stream_id, snapshot_seq, + @max_compact_chunk_count, @max_compact_chunk_count, registry_count, pg_count, [], []} + ) + 10 + end + + def new_staging_table do + :ets.new(__MODULE__, [:set, :private]) + end + + def delete_staging_table(table) do + try do + :ets.delete(table) + rescue + ArgumentError -> true + end + + :ok + end + + def stage_rows(table, chunk_index, registry_rows, pg_rows) do + objects = + Enum.map(registry_rows, &staging_object(:registry, &1)) ++ + Enum.map(pg_rows, &staging_object(:pg, &1)) + + size_before = :ets.info(table, :size) + + if :ets.insert_new(table, objects) and + :ets.info(table, :size) - size_before == length(objects) do + true = :ets.insert_new(table, {{:chunk, chunk_index}, registry_rows, pg_rows}) + :ok + else + {:error, :duplicate_row} + end + end + + def fold_registry(table, chunk_count, acc, fun) when is_function(fun, 2) do + Enum.reduce(1..chunk_count, acc, fn chunk_index, inner -> + {registry_rows, _pg_rows} = fetch_chunk(table, chunk_index) + Enum.reduce(registry_rows, inner, fun) + end) + end + + def fold_pg(table, chunk_count, acc, fun) when is_function(fun, 2) do + Enum.reduce(1..chunk_count, acc, fn chunk_index, inner -> + {_registry_rows, pg_rows} = fetch_chunk(table, chunk_index) + Enum.reduce(pg_rows, inner, fun) + end) + end + + def member_pg?(table, key, pid), do: :ets.member(table, {:pg, key, pid}) + + defp add_row(%{count: count, bytes: bytes} = acc, domain, row, target) do + row_bytes = :erlang.external_size(row) + + acc = + if count > 0 and bytes + row_bytes > target do + flush_chunk(acc) + else + acc + end + + case domain do + :registry -> + %{ + acc + | registry: [row | acc.registry], + bytes: acc.bytes + row_bytes, + count: acc.count + 1 + } + + :pg -> + %{acc | pg: [row | acc.pg], bytes: acc.bytes + row_bytes, count: acc.count + 1} + end + end + + defp flush_chunk(%{count: 0} = acc), do: acc + + defp flush_chunk(acc) do + chunk = {Enum.reverse(acc.registry), Enum.reverse(acc.pg)} + + %{acc | chunks: [chunk | acc.chunks], registry: [], pg: [], bytes: 0, count: 0} + end + + defp staging_object(:registry, {key, _pid, _meta, _time}), + do: {{:registry, key}} + + defp staging_object(:pg, {key, pid, _meta, _time}), + do: {{:pg, key, pid}} + + defp fetch_chunk(table, chunk_index) do + case :ets.lookup(table, {:chunk, chunk_index}) do + [{{:chunk, ^chunk_index}, registry_rows, pg_rows}] -> {registry_rows, pg_rows} + end + end +end diff --git a/lib/group/replica/transport.ex b/lib/group/replica/transport.ex index 8b23595..033acd6 100644 --- a/lib/group/replica/transport.ex +++ b/lib/group/replica/transport.ex @@ -16,6 +16,11 @@ defmodule Group.Replica.Transport do and sequences each origin/generation/shard/cluster/epoch stream; receivers discard duplicates and request gaps. Per-shard ordered delivery avoids repair traffic and is therefore the preferred fast path. + + A sideband implementation can delegate `try_send/5` to + `Group.Replica.Transport.Outbox.try_send/5`. That adds one local send only for + the configured sideband transport; the default distribution adapter retains + its direct remote `:erlang.send_nosuspend/3` path. """ @type frame :: term() @@ -50,6 +55,24 @@ defmodule Group.Replica.Transport do :ok end + @doc """ + Delivers a complete batch received from one authenticated peer. + + A finite-frame transport may segment the encoded batch on the wire, but it + must authenticate the peer and reassemble every segment before calling this + function. Group never observes or applies a partial batch. + """ + def deliver_batch(group, source_node, shard, frames) + when is_atom(group) and is_atom(source_node) and is_integer(shard) and shard >= 0 and + is_list(frames) do + send( + Group.Replica.shard_name(group, shard), + {:group_replica_batch, source_node, frames} + ) + + :ok + end + def normalize(module) when is_atom(module), do: {module, []} def normalize({module, opts}) when is_atom(module) and is_list(opts), do: {module, opts} diff --git a/lib/group/replica/transport/outbox.ex b/lib/group/replica/transport/outbox.ex new file mode 100644 index 0000000..aee2ba6 --- /dev/null +++ b/lib/group/replica/transport/outbox.ex @@ -0,0 +1,318 @@ +defmodule Group.Replica.Transport.Outbox do + @moduledoc """ + Lossy per-shard outboxes for sideband replica transports. + + This module is an implementation helper, not a replacement for + `Group.Replica.Transport`. Distribution can continue sending directly with + `:erlang.send_nosuspend/3`. A sideband adapter delegates `try_send/5` to + `try_send/5`, which performs only a local `send/2` to the matching shard + outbox. + + Each outbox batches frames by target node outside the Group shard. Expired + frames and batches rejected by the backend are deliberately dropped; + anti-entropy repairs them. Backends may perform bounded blocking work in + `send_batch/4` because they run in the outbox rather than a Group process. + + A backend using this helper implements: + + @behaviour Group.Replica.Transport.Outbox + + def init_outbox(group, shard, opts), do: {:ok, backend_state} + + def send_batch(target_node, frames, deadline, backend_state) do + # Return promptly once `deadline` has passed. It is safe to drop. + {:ok, backend_state} + end + + The backend is responsible for authenticated ingress and must pass only + complete logical frames to `Group.Replica.Transport.deliver_batch/4`. + + ## Options + + * `:outbox_batch_size` - maximum logical frames collected per flush, + default `64` + * `:outbox_batch_bytes` - approximate external-term bytes collected per + flush, default `1_048_576` + * `:outbox_flush_interval` - maximum batching delay in milliseconds, + default `1` + * `:outbox_deadline` - maximum useful residence time for an outbound frame + in milliseconds, default `100` + + The deadline bounds stale work, not mailbox memory. A backend must also put a + finite bound on every socket enqueue or write it performs. Exact snapshot + frames are independently bounded by `:replicated_snapshot_chunk_target_bytes`. + Other logical frames or a whole batch may still exceed `:outbox_batch_bytes`; + a transport with a smaller finite frame size must segment and completely + reassemble those batches before local delivery. + """ + + @type frame :: Group.Replica.Transport.frame() + @type send_result :: Group.Replica.Transport.send_result() + @type backend_state :: term() + + @callback init_outbox(group :: atom(), shard :: non_neg_integer(), opts :: keyword()) :: + {:ok, backend_state()} + + @callback send_batch( + target_node :: node(), + frames :: [frame()], + deadline :: integer(), + backend_state() + ) :: {send_result(), backend_state()} + + @default_deadline 100 + + @doc """ + Returns a supervisor child specification for one outbox per Group shard. + + `:name`, `:num_shards`, and `:backend` are required. All options are passed + unchanged to `backend.init_outbox/3`. + """ + def child_spec(opts) do + group = Keyword.fetch!(opts, :name) + + %{ + id: {__MODULE__, group}, + start: {Group.Replica.Transport.Outbox.Supervisor, :start_link, [opts]}, + type: :supervisor, + restart: :permanent, + shutdown: :infinity + } + end + + @doc """ + Enqueues a frame into its local shard outbox. + + This performs no backend or socket operation. `:ok` only means the message + was sent to the current local outbox PID; the outbox may later drop it on + expiry or backpressure. A concurrently terminating outbox can also lose an + accepted message, which anti-entropy repairs. + """ + def try_send(group, target_node, shard, frame, opts) + when is_atom(group) and is_atom(target_node) and is_integer(shard) and shard >= 0 and + is_list(opts) do + case Process.whereis(name(group, shard)) do + pid when is_pid(pid) -> + deadline = monotonic_ms() + deadline(opts) + send(pid, {:group_replica_outbox_send, target_node, deadline, frame}) + :ok + + nil -> + :disconnected + end + end + + @doc false + def name(group, shard), do: :"#{group}_replica_transport_outbox_#{shard}" + + @doc false + def monotonic_ms, do: System.monotonic_time(:millisecond) + + defp deadline(opts) do + case Keyword.get(opts, :outbox_deadline, @default_deadline) do + value when is_integer(value) and value > 0 -> + value + + other -> + raise ArgumentError, "expected :outbox_deadline to be positive, got: #{inspect(other)}" + end + end +end + +defmodule Group.Replica.Transport.Outbox.Supervisor do + @moduledoc false + use Supervisor + + def start_link(opts), do: Supervisor.start_link(__MODULE__, opts) + + @impl true + def init(opts) do + group = Keyword.fetch!(opts, :name) + num_shards = Keyword.fetch!(opts, :num_shards) + backend = Keyword.fetch!(opts, :backend) + + Code.ensure_loaded!(backend) + + for {function, arity} <- [init_outbox: 3, send_batch: 4] do + unless function_exported?(backend, function, arity) do + raise ArgumentError, + "outbox backend #{inspect(backend)} must implement #{function}/#{arity}" + end + end + + children = + for shard <- 0..(num_shards - 1) do + %{ + id: {Group.Replica.Transport.Outbox.Worker, group, shard}, + start: {Group.Replica.Transport.Outbox.Worker, :start_link, [opts, shard]}, + restart: :permanent, + shutdown: 5_000 + } + end + + Supervisor.init(children, strategy: :one_for_one) + end +end + +defmodule Group.Replica.Transport.Outbox.Worker do + @moduledoc false + use GenServer + + alias Group.Replica.Transport.Outbox + + @default_batch_size 64 + @default_batch_bytes 1_048_576 + @default_flush_interval 1 + + def start_link(opts, shard) do + group = Keyword.fetch!(opts, :name) + GenServer.start_link(__MODULE__, {opts, shard}, name: Outbox.name(group, shard)) + end + + @impl true + def init({opts, shard}) do + group = Keyword.fetch!(opts, :name) + backend = Keyword.fetch!(opts, :backend) + {:ok, backend_state} = backend.init_outbox(group, shard, opts) + + {:ok, + %{ + group: group, + shard: shard, + backend: backend, + backend_state: backend_state, + batch_size: positive_opt(opts, :outbox_batch_size, @default_batch_size), + batch_bytes: positive_opt(opts, :outbox_batch_bytes, @default_batch_bytes), + flush_interval: non_negative_opt(opts, :outbox_flush_interval, @default_flush_interval), + pending: [], + pending_count: 0, + pending_bytes: 0, + flush_ref: nil + }} + end + + @impl true + def handle_info({:group_replica_outbox_send, target_node, deadline, frame}, state) + when is_atom(target_node) and is_integer(deadline) do + if deadline <= Outbox.monotonic_ms() do + {:noreply, state} + else + bytes = :erlang.external_size({target_node, frame}) + + state = + if state.pending_count > 0 and + (state.pending_count + 1 > state.batch_size or + state.pending_bytes + bytes > state.batch_bytes) do + flush(state) + else + state + end + + state = enqueue(state, target_node, deadline, frame, bytes) + + if state.pending_count >= state.batch_size or state.pending_bytes >= state.batch_bytes do + {:noreply, flush(state)} + else + {:noreply, schedule_flush(state)} + end + end + end + + def handle_info({:group_replica_outbox_flush, ref}, %{flush_ref: ref} = state) do + {:noreply, flush(%{state | flush_ref: nil})} + end + + def handle_info({:group_replica_outbox_flush, _stale_ref}, state), do: {:noreply, state} + def handle_info(_message, state), do: {:noreply, state} + + defp enqueue(state, target_node, deadline, frame, bytes) do + entry = {target_node, deadline, frame} + + %{ + state + | pending: [entry | state.pending], + pending_count: state.pending_count + 1, + pending_bytes: state.pending_bytes + bytes + } + end + + defp schedule_flush(%{pending_count: 0} = state), do: state + defp schedule_flush(%{flush_ref: ref} = state) when is_reference(ref), do: state + + defp schedule_flush(state) do + ref = make_ref() + Process.send_after(self(), {:group_replica_outbox_flush, ref}, state.flush_interval) + %{state | flush_ref: ref} + end + + defp flush(%{pending_count: 0} = state), do: cancel_flush(state) + + defp flush(state) do + state = cancel_flush(state) + now = Outbox.monotonic_ms() + + batches = + state.pending + |> Enum.reverse() + |> Enum.reject(fn {_target_node, deadline, _frame} -> deadline <= now end) + |> Enum.group_by(fn {target_node, _deadline, _frame} -> target_node end) + + backend_state = + Enum.reduce(batches, state.backend_state, fn {target_node, entries}, backend_state -> + frames = Enum.map(entries, fn {_target_node, _deadline, frame} -> frame end) + + deadline = + entries + |> Enum.map(fn {_target_node, deadline, _frame} -> deadline end) + |> Enum.min() + + if deadline <= Outbox.monotonic_ms() do + backend_state + else + case state.backend.send_batch(target_node, frames, deadline, backend_state) do + {result, next_backend_state} when result in [:ok, :busy, :disconnected] -> + next_backend_state + + other -> + raise "invalid #{inspect(state.backend)}.send_batch/4 return: #{inspect(other)}" + end + end + end) + + %{ + state + | backend_state: backend_state, + pending: [], + pending_count: 0, + pending_bytes: 0 + } + end + + defp cancel_flush(%{flush_ref: nil} = state), do: state + + defp cancel_flush(state) do + Process.cancel_timer(state.flush_ref) + %{state | flush_ref: nil} + end + + defp positive_opt(opts, key, default) do + case Keyword.get(opts, key, default) do + value when is_integer(value) and value > 0 -> + value + + other -> + raise ArgumentError, "expected #{inspect(key)} to be positive, got: #{inspect(other)}" + end + end + + defp non_negative_opt(opts, key, default) do + case Keyword.get(opts, key, default) do + value when is_integer(value) and value >= 0 -> + value + + other -> + raise ArgumentError, + "expected #{inspect(key)} to be non-negative, got: #{inspect(other)}" + end + end +end diff --git a/lib/group/replica/transport/tcp.ex b/lib/group/replica/transport/tcp.ex index 1914ac6..0cedaa8 100644 --- a/lib/group/replica/transport/tcp.ex +++ b/lib/group/replica/transport/tcp.ex @@ -6,10 +6,11 @@ defmodule Group.Replica.Transport.TCP do Replica frames use independent TCP connections, so there is no ordering relationship between a control message and its data lane. - `try_send/5` never writes a socket. It reserves one slot in a bounded - per-peer queue and sends to a dedicated writer process. The writer may block - up to `:send_timeout` without blocking a Group shard. A full queue returns - `:busy`; a missing connection returns `:disconnected`. + `try_send/5` only sends to a local per-shard outbox. The outbox batches + frames and forwards each target batch to a bounded per-peer writer queue. + The writer may block up to `:send_timeout` without blocking a Group shard. + Expired, busy, and disconnected batches are dropped and repaired by + anti-entropy. The endpoint capability in the dist-Erlang hello prevents an unrelated socket client from injecting frames. This transport is intended for trusted @@ -21,18 +22,23 @@ defmodule Group.Replica.Transport.TCP do * `:ip` - listen address, default `{127, 0, 0, 1}` * `:advertised_ip` - address placed in the hello, defaults to `:ip` * `:port` - listen port, default `0` (ephemeral) - * `:max_queue` - maximum queued frames per peer, default `1_024` + * `:max_queue` - maximum queued batches per peer, default `1_024` * `:connect_timeout` - outbound connect timeout in milliseconds, default `1_000` * `:send_timeout` - writer socket send timeout in milliseconds, default `1_000` * `:reconnect_interval` - retry delay in milliseconds, default `50` + + See `Group.Replica.Transport.Outbox` for batching and deadline options. """ use GenServer @behaviour Group.Replica.Transport + @behaviour Group.Replica.Transport.Outbox + + alias Group.Replica.Transport.Outbox @impl true - def id, do: :group_sideband_tcp_v1 + def id, do: :group_sideband_tcp_v2 @impl true def child_spec(opts) do @@ -40,10 +46,10 @@ defmodule Group.Replica.Transport.TCP do %{ id: {__MODULE__, name}, - start: {__MODULE__, :start_link, [opts]}, - type: :worker, + start: {Group.Replica.Transport.TCP.Supervisor, :start_link, [opts]}, + type: :supervisor, restart: :permanent, - shutdown: 5_000 + shutdown: :infinity } end @@ -58,24 +64,34 @@ defmodule Group.Replica.Transport.TCP do end @impl true - def try_send(group, target_node, shard, frame, _opts) do - case :ets.lookup(route_table(group), target_node) do - [{^target_node, writer, queued, max_queue}] -> - if :atomics.add_get(queued, 1, 1) <= max_queue do - if :erlang.send_nosuspend(writer, {:replica_frame, shard, frame}) do - :ok - else - :atomics.sub(queued, 1, 1) - :busy - end - else - :atomics.sub(queued, 1, 1) - :busy + def try_send(group, target_node, shard, frame, opts), + do: Outbox.try_send(group, target_node, shard, frame, opts) + + @impl Group.Replica.Transport.Outbox + def init_outbox(group, shard, _opts), do: {:ok, %{group: group, shard: shard}} + + @impl Group.Replica.Transport.Outbox + def send_batch(target_node, frames, deadline, %{group: group, shard: shard} = state) do + result = + try do + case :ets.lookup(route_table(group), target_node) do + [{^target_node, writer, queued, max_queue}] -> + if :atomics.add_get(queued, 1, 1) <= max_queue do + send(writer, {:replica_batch, deadline, shard, frames}) + :ok + else + :atomics.sub(queued, 1, 1) + :busy + end + + [] -> + :disconnected end + rescue + ArgumentError -> :disconnected + end - [] -> - :disconnected - end + {result, state} end @impl true @@ -126,8 +142,9 @@ defmodule Group.Replica.Transport.TCP do ]) {:ok, {_listen_ip, listen_port}} = :inet.sockname(listener) + capability = :erlang.term_to_binary({node(), make_ref(), System.unique_integer()}) - descriptor = {:group_sideband_tcp_v1, advertised_ip, listen_port, capability} + descriptor = {:group_sideband_tcp_v2, advertised_ip, listen_port, capability} :persistent_term.put({__MODULE__, group, :descriptor}, descriptor) :ets.new(route_table(group), [ @@ -248,7 +265,7 @@ defmodule Group.Replica.Transport.TCP do :ok pid -> - _ = :erlang.send_nosuspend(pid, message) + send(pid, message) :ok end end @@ -326,7 +343,7 @@ defmodule Group.Replica.Transport.TCP do manager, group, remote_node, - {:group_sideband_tcp_v1, host, port, capability}, + {:group_sideband_tcp_v2, host, port, capability}, connect_timeout, send_timeout ) do @@ -362,12 +379,18 @@ defmodule Group.Replica.Transport.TCP do defp writer_loop(socket, manager, remote_node, queued) do receive do - {:replica_frame, shard, frame} -> - result = :gen_tcp.send(socket, :erlang.term_to_binary({shard, frame})) + {:replica_batch, deadline, shard, frames} -> + result = + if deadline <= Outbox.monotonic_ms() do + :expired + else + :gen_tcp.send(socket, :erlang.term_to_binary({:batch, shard, frames})) + end + :atomics.sub(queued, 1, 1) case result do - :ok -> + result when result in [:ok, :expired] -> writer_loop(socket, manager, remote_node, queued) {:error, _reason} -> @@ -415,8 +438,9 @@ defmodule Group.Replica.Transport.TCP do case :gen_tcp.recv(socket, 0) do {:ok, payload} -> case decode_authenticated_frame(payload) do - {:ok, {shard, frame}} when is_integer(shard) and shard >= 0 -> - :ok = Group.Replica.Transport.deliver(group, source_node, shard, frame) + {:ok, {:batch, shard, frames}} + when is_integer(shard) and shard >= 0 and is_list(frames) -> + :ok = Group.Replica.Transport.deliver_batch(group, source_node, shard, frames) reader_loop(socket, group, source_node) _ -> @@ -446,3 +470,36 @@ defmodule Group.Replica.Transport.TCP do defp server_name(group), do: :"#{group}_replica_tcp_transport" defp route_table(group), do: :"#{group}_replica_tcp_routes" end + +defmodule Group.Replica.Transport.TCP.Supervisor do + @moduledoc false + use Supervisor + + alias Group.Replica.Transport.{Outbox, TCP} + + def start_link(opts), do: Supervisor.start_link(__MODULE__, opts) + + @impl true + def init(opts) do + group = Keyword.fetch!(opts, :name) + + manager = %{ + id: {TCP, group, :manager}, + start: {TCP, :start_link, [opts]}, + type: :worker, + restart: :transient, + shutdown: 5_000, + significant: true + } + + outboxes = + opts + |> Keyword.put(:backend, TCP) + |> Outbox.child_spec() + + Supervisor.init([manager, outboxes], + strategy: :rest_for_one, + auto_shutdown: :any_significant + ) + end +end diff --git a/lib/group/supervisor.ex b/lib/group/supervisor.ex index 0bbd773..df2e97d 100644 --- a/lib/group/supervisor.ex +++ b/lib/group/supervisor.ex @@ -51,6 +51,9 @@ defmodule Group.Supervisor do replicated_oplog_max_entries = positive_integer_opt(opts, :replicated_oplog_max_entries, 65_536) + replicated_snapshot_chunk_target_bytes = + positive_integer_opt(opts, :replicated_snapshot_chunk_target_bytes, 1_048_576) + replicated_anti_entropy_interval = positive_integer_opt(opts, :replicated_anti_entropy_interval, 1_000) @@ -78,6 +81,7 @@ defmodule Group.Supervisor do replicated_pg_receiver_local_request_quota: replicated_pg_receiver_local_request_quota, replica_transport: replica_transport, replicated_oplog_max_entries: replicated_oplog_max_entries, + replicated_snapshot_chunk_target_bytes: replicated_snapshot_chunk_target_bytes, replicated_anti_entropy_interval: replicated_anti_entropy_interval, replicated_peer_lease_timeout: replicated_peer_lease_timeout } diff --git a/priv/bench/README.md b/priv/bench/README.md index 6278aa5..679b01e 100644 --- a/priv/bench/README.md +++ b/priv/bench/README.md @@ -37,6 +37,14 @@ To isolate the 10,000-cluster lifecycle scenario: --coordinator-expr 'GroupBench.Distributed.run_many_clusters_only(shards: 4)' ``` +To measure the exact-snapshot fallback independently (one shard models one +busy lane of a much larger sharded deployment): + +```bash +./run_distributed.sh --shards 1 \ + --coordinator-expr 'GroupBench.Distributed.run_snapshot_sync_only(shards: 1, entries: 50000)' +``` + ## Local Scenarios All local benchmarks run for both the default (nil) cluster and a named cluster diff --git a/priv/bench/lib/group_bench/distributed.ex b/priv/bench/lib/group_bench/distributed.ex index 66b601f..9d3b34f 100644 --- a/priv/bench/lib/group_bench/distributed.ex +++ b/priv/bench/lib/group_bench/distributed.ex @@ -67,6 +67,23 @@ defmodule GroupBench.Distributed do IO.puts("\n Done.\n") end + def run_snapshot_sync_only(opts \\ []) do + shards = Keyword.get(opts, :shards, 1) + entries = Keyword.get(opts, :entries, 50_000) + Process.put(:bench_shards, shards) + + header("Distributed Exact-Snapshot Benchmark") + IO.puts(" coordinator: #{node()}") + IO.puts(" shards: #{shards}") + IO.puts(" entries: #{format_number(entries)}") + IO.puts(" schedulers: #{System.schedulers_online()}") + + connect_replicas() + bench_snapshot_sync(@replicas, entries) + + IO.puts("\n Done.\n") + end + # ── Connection ──────────────────────────────────────────────────────── defp connect_replicas do @@ -208,6 +225,48 @@ defmodule GroupBench.Distributed do end end + defp bench_snapshot_sync([r1, r2] = replicas, key_count) do + header("Exact Snapshot (receiver below oplog floor)") + subheader("#{format_number(key_count)} keys") + + start_group_on(r1, + replicated_oplog_max_entries: 64, + replicated_anti_entropy_interval: 100, + replicated_peer_lease_timeout: 15_000 + ) + + :erpc.call( + r1, + GroupBench.Replica, + :bulk_register, + [@name, key_count, "snapshot-"], + 180_000 + ) + + {sync_us, _} = + :timer.tc(fn -> + start_group_on(r2, + replicated_oplog_max_entries: 64, + replicated_anti_entropy_interval: 100, + replicated_peer_lease_timeout: 15_000 + ) + + poll_until( + fn -> + :erpc.call(r2, GroupBench.Replica, :total_registry_count, [@name]) >= key_count + end, + 120_000 + ) + end) + + rate = if sync_us > 0, do: round(key_count * 1_000_000 / sync_us), else: 0 + + IO.puts(" sync time: #{format_number(div(sync_us, 1000))} ms") + IO.puts(" keys/sec: #{format_number(rate)}") + + stop_groups(replicas) + end + # ── 3. Concurrent cross-node writes ────────────────────────────────── defp bench_concurrent_cross_node([r1, r2] = replicas) do diff --git a/test/README.md b/test/README.md index 65c12aa..16ce634 100644 --- a/test/README.md +++ b/test/README.md @@ -18,6 +18,8 @@ mix test test/replica_model_property_test.exs # shrinkable model-based histories | `distributed_test.exs` | Multi-node: replication, peer discovery, node disconnect cleanup, partition healing, conflict resolution, event ordering, rolling restarts, and adversarial replica-transport loss/busy/snapshot recovery | | `replica_adversarial_test.exs` | Reproducible mixed-operation state machines: drops, busy returns, duplication, reordering, bounded delay, oplog pruning, conflicts, owner death, and named-cluster epoch churn, followed by exact convergence/dead-owner/internal-index checks | | `replica_model_property_test.exs` | StreamData-generated and shrunk owner histories against an independent lifecycle oracle and scheduler-controlled replica transport | +| `replica_snapshot_test.exs` | Pure byte partitioning and set-valued private-ETS snapshot staging | +| `replica_snapshot_distributed_test.exs` | Real-node exact-snapshot loss, reorder, duplicate, conflicting retransmission, supersession, authority fencing, expiry, and shard-crash recovery | ## Model-based and formal checks @@ -29,7 +31,9 @@ PG key against an independent application-level lifecycle oracle. It also requires internal replica indexes to be consistent, every retained owner to be alive, and registry conflict losers to be dead. Restart, pruning, and named cluster histories retain independent C-owned state while A recovers, so repair -cannot pass merely by making one origin and one receiver agree. +cannot pass merely by making one origin and one receiver agree. Model groups +use a deliberately tiny snapshot target so pruning recovery traverses the real +multi-chunk assembly path. StreamData reports the ExUnit seed and shrinks a failure to its smallest command history. Local defaults are intentionally quick. Increase the budgets without @@ -257,6 +261,11 @@ disconnects one origin's real socket, prunes its oplog, reconnects it, and requires snapshot recovery without changing the third node's independent registry or PG state. +`replica_transport_outbox_test.exs` proves that a blocked sideband backend +cannot delay the Group-facing local send, frames expire behind that backend, +busy batches are not retried locally, and batching preserves per-target order. +The real three-node TCP recovery test runs through the same outbox path. + `Group.TestCluster.assert_replica_consistent/1` checks the public dual indexes plus registry claim authority, oplog/order equivalence, and contiguous retained stream ranges. Seeded tests additionally require every PID diff --git a/test/distributed_test.exs b/test/distributed_test.exs index e43d1c4..3a7c2d7 100644 --- a/test/distributed_test.exs +++ b/test/distributed_test.exs @@ -1833,7 +1833,7 @@ defmodule Group.DistributedTest do messages = TestCluster.shard_messages(node_b, name, 0) case Enum.filter(messages, fn - {:group_replica_frame, _source, {:delta_batch, 1, _runs}} -> + {:group_replica_frame, _source, {:delta_batch, _version, _runs}} -> true {:replica_cluster_close, _remote_pid, _generation, _revision, [{"game", _epoch}]} -> @@ -1843,7 +1843,7 @@ defmodule Group.DistributedTest do false end) do [ - {:group_replica_frame, _source, {:delta_batch, 1, runs}}, + {:group_replica_frame, _source, {:delta_batch, _version, runs}}, {:replica_cluster_close, _remote_pid, _generation, _revision, [{"game", _epoch}]} ] -> Enum.any?(runs, fn {_stream_id, _first_seq, records, _head} -> diff --git a/test/formal/README.md b/test/formal/README.md index 05c2c10..7cb743e 100644 --- a/test/formal/README.md +++ b/test/formal/README.md @@ -10,6 +10,13 @@ contract. It covers: - exact per-origin snapshot fallback; and - fair convergence after healing. +`SnapshotAssembly.tla` separately models the non-atomic wire delivery of an +exact snapshot. It explores arbitrary chunk loss, duplication, reordering, +newer-snapshot supersession, authority epoch changes, staging expiry, and +receiver crashes. Its invariants require visible data and the cursor to remain +at a previously committed exact state until every chunk of one valid snapshot +is present; stale or mixed partial state can never become visible. + The default TLC configuration uses three nodes: one origin and two independent receivers. The origin has one key, a two-record stream, a one-record oplog, and the system retains one arbitrary network frame. This forces delta repair, @@ -29,6 +36,11 @@ Run it with Java 17 or later and a current `tla2tools.jar`: ```bash TLA_JAR=/path/to/tla2tools.jar test/formal/check.sh + +TLA_JAR=/path/to/tla2tools.jar \ + TLA_SPEC="$PWD/test/formal/SnapshotAssembly.tla" \ + TLA_CONFIG="$PWD/test/formal/SnapshotAssembly.cfg" \ + test/formal/check.sh ``` `TLC_WORKERS` controls worker concurrency and defaults to 4. `TLA_CONFIG` can @@ -42,3 +54,6 @@ should be run periodically by increasing `Nodes`, `Origins`, `Keys`, `MaxSeq`, The checked three-node default explores 1,835,826 states, finds 490,236 distinct states to a depth of 30, and completes in roughly 1 minute 40 seconds on the development machine used for the validation run. + +The snapshot-assembly model explores 15,681 states, finds 1,088 distinct states +to a depth of 13, and completes in under a second on the same class of machine. diff --git a/test/formal/SnapshotAssembly.cfg b/test/formal/SnapshotAssembly.cfg new file mode 100644 index 0000000..38a4dba --- /dev/null +++ b/test/formal/SnapshotAssembly.cfg @@ -0,0 +1,7 @@ +SPECIFICATION Spec + +INVARIANTS + TypeOK + VisibleIsAnExactCommittedSnapshot + StagingNeverLeaksIntoVisible + StagingBelongsToOneSnapshot diff --git a/test/formal/SnapshotAssembly.tla b/test/formal/SnapshotAssembly.tla new file mode 100644 index 0000000..3d6c18d --- /dev/null +++ b/test/formal/SnapshotAssembly.tla @@ -0,0 +1,181 @@ +------------------------- MODULE SnapshotAssembly ------------------------- +EXTENDS Integers, FiniteSets, TLC + +(* +Finite model of the exact-snapshot chunk assembly boundary. It deliberately +models two snapshots in one authority epoch plus a new-epoch snapshot so TLC +can explore loss, duplication, reordering, supersession, stale final chunks, +expiry, and receiver crashes independently of the larger anti-entropy model. +*) + +Snapshots == {1, 2, 3} +Chunks == {1, 2} +Rows == {"a", "b", "c", "d"} + +SnapshotEpoch(snapshot) == + CASE snapshot = 0 -> 0 + [] snapshot = 1 -> 1 + [] snapshot = 2 -> 1 + [] snapshot = 3 -> 2 + +SnapshotSeq(snapshot) == + CASE snapshot = 0 -> 0 + [] snapshot = 1 -> 1 + [] snapshot = 2 -> 2 + [] snapshot = 3 -> 1 + +SnapshotRows(snapshot) == + CASE snapshot = 1 -> {"a", "b"} + [] snapshot = 2 -> {"c", "d"} + [] snapshot = 3 -> {"a", "d"} + +ChunkRows(snapshot, chunk) == + CASE snapshot = 1 /\ chunk = 1 -> {"a"} + [] snapshot = 1 /\ chunk = 2 -> {"b"} + [] snapshot = 2 /\ chunk = 1 -> {"c"} + [] snapshot = 2 /\ chunk = 2 -> {"d"} + [] snapshot = 3 /\ chunk = 1 -> {"a"} + [] snapshot = 3 /\ chunk = 2 -> {"d"} + +Message == [snapshot : Snapshots, chunk : Chunks] + +VARIABLES authorityEpoch, + cursor, + visible, + stagedSnapshot, + stagedChunks, + stagedRows, + messages + +vars == + <> + +Init == + /\ authorityEpoch = 1 + /\ cursor = 0 + /\ visible = {} + /\ stagedSnapshot = 0 + /\ stagedChunks = {} + /\ stagedRows = {} + /\ messages = {} + +Send(snapshot, chunk) == + /\ messages' = messages \union + {[snapshot |-> snapshot, chunk |-> chunk]} + /\ UNCHANGED <> + +Valid(message) == + /\ SnapshotEpoch(message.snapshot) = authorityEpoch + /\ SnapshotSeq(message.snapshot) > cursor + +StartsNewAssembly(message) == + /\ Valid(message) + /\ \/ stagedSnapshot = 0 + \/ SnapshotEpoch(stagedSnapshot) # authorityEpoch + \/ SnapshotSeq(message.snapshot) > SnapshotSeq(stagedSnapshot) + +StartAssembly(message) == + /\ StartsNewAssembly(message) + /\ stagedSnapshot' = message.snapshot + /\ stagedChunks' = {message.chunk} + /\ stagedRows' = ChunkRows(message.snapshot, message.chunk) + /\ UNCHANGED <> + +ContinueAssembly(message) == + /\ Valid(message) + /\ stagedSnapshot = message.snapshot + /\ LET nextChunks == stagedChunks \union {message.chunk} + nextRows == stagedRows \union + ChunkRows(message.snapshot, message.chunk) + IN IF nextChunks = Chunks + THEN /\ cursor' = SnapshotSeq(message.snapshot) + /\ visible' = SnapshotRows(message.snapshot) + /\ stagedSnapshot' = 0 + /\ stagedChunks' = {} + /\ stagedRows' = {} + ELSE /\ UNCHANGED <> + /\ stagedChunks' = nextChunks + /\ stagedRows' = nextRows + /\ UNCHANGED <> + +IgnoreChunk(message) == + /\ ~StartsNewAssembly(message) + /\ ~(/\ Valid(message) + /\ stagedSnapshot = message.snapshot) + /\ UNCHANGED vars + +Deliver(message) == + /\ message \in messages + /\ \/ StartAssembly(message) + \/ ContinueAssembly(message) + \/ IgnoreChunk(message) + +Drop(message) == + /\ message \in messages + /\ messages' = messages \ {message} + /\ UNCHANGED <> + +InstallNewAuthority == + /\ authorityEpoch = 1 + /\ authorityEpoch' = 2 + /\ cursor' = 0 + /\ visible' = {} + (* The implementation may retain invisible old staging until expiry. *) + /\ UNCHANGED <> + +ExpireStaging == + /\ stagedSnapshot # 0 + /\ stagedSnapshot' = 0 + /\ stagedChunks' = {} + /\ stagedRows' = {} + /\ UNCHANGED <> + +CrashReceiver == + /\ stagedSnapshot # 0 + /\ stagedSnapshot' = 0 + /\ stagedChunks' = {} + /\ stagedRows' = {} + /\ UNCHANGED <> + +Next == + \/ \E snapshot \in Snapshots, chunk \in Chunks : Send(snapshot, chunk) + \/ \E message \in messages : Deliver(message) + \/ \E message \in messages : Drop(message) + \/ InstallNewAuthority + \/ ExpireStaging + \/ CrashReceiver + +TypeOK == + /\ authorityEpoch \in {1, 2} + /\ cursor \in 0..2 + /\ visible \subseteq Rows + /\ stagedSnapshot \in {0} \union Snapshots + /\ stagedChunks \subseteq Chunks + /\ stagedRows \subseteq Rows + /\ messages \subseteq Message + +VisibleIsAnExactCommittedSnapshot == + \/ /\ authorityEpoch = 1 + /\ \/ /\ cursor = 0 /\ visible = {} + \/ /\ cursor = 1 /\ visible = SnapshotRows(1) + \/ /\ cursor = 2 /\ visible = SnapshotRows(2) + \/ /\ authorityEpoch = 2 + /\ \/ /\ cursor = 0 /\ visible = {} + \/ /\ cursor = 1 /\ visible = SnapshotRows(3) + +StagingNeverLeaksIntoVisible == + stagedSnapshot # 0 /\ stagedChunks # Chunks => + VisibleIsAnExactCommittedSnapshot + +StagingBelongsToOneSnapshot == + stagedSnapshot # 0 => + /\ stagedRows = + UNION {ChunkRows(stagedSnapshot, chunk) : chunk \in stagedChunks} + /\ stagedChunks # Chunks + +Spec == Init /\ [][Next]_vars + +============================================================================= diff --git a/test/formal/check.sh b/test/formal/check.sh index 1336a08..1f5b1c5 100755 --- a/test/formal/check.sh +++ b/test/formal/check.sh @@ -9,6 +9,7 @@ fi repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" metadir="${repo_root}/tmp/tlc" config="${TLA_CONFIG:-${repo_root}/test/formal/GroupAntiEntropy.cfg}" +spec="${TLA_SPEC:-${repo_root}/test/formal/GroupAntiEntropy.tla}" mkdir -p "${metadir}" exec java -XX:+UseParallelGC -cp "${TLA_JAR}" tlc2.TLC \ @@ -16,4 +17,4 @@ exec java -XX:+UseParallelGC -cp "${TLA_JAR}" tlc2.TLC \ -metadir "${metadir}" \ -workers "${TLC_WORKERS:-4}" \ -config "${config}" \ - "${repo_root}/test/formal/GroupAntiEntropy.tla" + "${spec}" diff --git a/test/group_test.exs b/test/group_test.exs index 572eb46..f6356a5 100644 --- a/test/group_test.exs +++ b/test/group_test.exs @@ -2356,7 +2356,7 @@ defmodule GroupTest do {:erlang, :send_nosuspend, [ {^shard_name, ^local_node}, - {:group_replica_frame, ^local_node, {:delta_batch, 1, runs}}, + {:group_replica_frame, ^local_node, {:delta_batch, _version, runs}}, [:noconnect] ]}}, 1_000 diff --git a/test/mutation/README.md b/test/mutation/README.md index 269edf8..fa9e50e 100644 --- a/test/mutation/README.md +++ b/test/mutation/README.md @@ -5,7 +5,9 @@ catastrophic protocol obligations. It covers generation and epoch fencing, contiguous sequence application, exact registry and PG snapshots, below-floor repair, process-down sequencing, conflict-loser retirement, authority fanout, per-lane authority installation, periodic head advertisement, interrupted -journal/index repair, and named-cluster close completion. +journal/index repair, and named-cluster close completion. Snapshot calibration +also covers incomplete commit, conflicting retransmission rows, newer-snapshot +supersession, stale-authority fencing, and staging expiry. The runner first verifies every unmodified regression target. It then copies the current checkout once per mutant, changes only that copy, recompiles it, diff --git a/test/mutation/run.exs b/test/mutation/run.exs index 60ecf6f..6efefbe 100644 --- a/test/mutation/run.exs +++ b/test/mutation/run.exs @@ -62,23 +62,120 @@ defmodule Group.MutationCampaign do %{ name: "registry_snapshot_is_additive", file: "lib/group/replica/data.ex", - correct_source: "existing = registry_claims_for_stream(name, shard, stream_id)", - faulty_source: "existing = []", - test: ["test/distributed_test.exs:4030"] + correct_source: "Enum.reduce(existing, MapSet.new(), fn {key, pid, _meta, _time}, keys ->", + faulty_source: + "Enum.reduce(Enum.take(existing, 0), MapSet.new(), fn {key, pid, _meta, _time}, keys ->", + test: ["test/replica_snapshot_distributed_test.exs:16"] }, %{ name: "pg_snapshot_is_additive", file: "lib/group/replica.ex", + correct_source: "Enum.reduce(current, events, fn {key, pid, old_meta, _old_time}, acc ->", + faulty_source: + "Enum.reduce(Enum.take(current, 0), events, fn {key, pid, old_meta, _old_time}, acc ->", + test: ["test/replica_snapshot_distributed_test.exs:16"] + }, + %{ + name: "single_chunk_registry_snapshot_is_additive", + file: "lib/group/replica/data.ex", + correct_source: "Enum.each(existing, fn {key, pid, _meta, _time} ->", + faulty_source: "Enum.each(Enum.take(existing, 0), fn {key, pid, _meta, _time} ->", + test: ["test/distributed_test.exs:4030"] + }, + %{ + name: "single_chunk_pg_snapshot_is_additive", + file: "lib/group/replica.ex", + correct_source: " current\n |> Map.keys()\n", + faulty_source: " %{}\n |> Map.keys()\n", + test: ["test/distributed_test.exs:4030"] + }, + %{ + name: "commit_incomplete_snapshot", + file: "lib/group/replica.ex", correct_source: """ - current = - state.name - |> Data.pg_entries_for_origin(state.shard_index, cluster, source_node) - |> Map.new(fn {key, pid, meta, time} -> {{key, pid}, {meta, time}} end) + if MapSet.size(transfer.received) == transfer.chunk_count do + if transfer.registry_seen == transfer.registry_count and + transfer.pg_seen == transfer.pg_count do + commit_snapshot_transfer(state, key, source_node, stream_id, transfer) + else + discard_snapshot_transfer(state, key) + end + else + state + end """, faulty_source: """ - current = %{} + if MapSet.size(transfer.received) >= 1 do + commit_snapshot_transfer(state, key, source_node, stream_id, transfer) + else + state + end """, - test: ["test/distributed_test.exs:4030"] + test: ["test/replica_snapshot_distributed_test.exs:16"] + }, + %{ + name: "allow_duplicate_snapshot_rows", + file: "lib/group/replica/snapshot.ex", + correct_source: """ + if :ets.insert_new(table, objects) and + :ets.info(table, :size) - size_before == length(objects) do + """, + faulty_source: """ + if :ets.insert(table, objects) and size_before >= 0 do + """, + test: ["test/replica_snapshot_distributed_test.exs:164"] + }, + %{ + name: "do_not_supersede_partial_snapshot", + file: "lib/group/replica.ex", + correct_source: """ + %{snapshot_seq: existing_seq} when existing_seq < snapshot_seq -> + state = discard_snapshot_transfer(state, key) + {:ok, state, new_snapshot_transfer(snapshot_seq, manifest)} + """, + faulty_source: """ + %{snapshot_seq: existing_seq} when existing_seq < snapshot_seq -> + _ = existing_seq + {:ignore, state} + """, + test: ["test/replica_snapshot_distributed_test.exs:107"] + }, + %{ + name: "accept_stale_snapshot_authority", + file: "lib/group/replica.ex", + correct_source: """ + defp valid_snapshot_stream?(state, source_node, stream_id, snapshot_seq) do + valid_remote_stream?(state, source_node, stream_id) and + snapshot_seq > Data.replica_cursor(state.name, state.shard_index, stream_id) + end + """, + faulty_source: """ + defp valid_snapshot_stream?(state, _source_node, stream_id, snapshot_seq) do + snapshot_seq > Data.replica_cursor(state.name, state.shard_index, stream_id) + end + """, + test: ["test/replica_snapshot_distributed_test.exs:202"] + }, + %{ + name: "disable_snapshot_staging_expiry", + file: "lib/group/replica.ex", + correct_source: """ + if now - transfer.last_progress > acc.replicated_peer_lease_timeout do + discard_snapshot_transfer(acc, key) + else + acc + end + """, + faulty_source: """ + _ = {now, transfer} + + if Process.get(:force_snapshot_staging_expiry, false) do + discard_snapshot_transfer(acc, key) + else + acc + end + """, + test: ["test/replica_snapshot_distributed_test.exs:202"] }, %{ name: "disable_below_floor_snapshot", @@ -158,6 +255,32 @@ defmodule Group.MutationCampaign do """, test: ["test/distributed_test.exs:5531"] }, + %{ + name: "assume_authority_fanout_reaches_late_lane", + file: "lib/group/replica.ex", + correct_source: """ + defp install_current_replica_lane(state, remote_node, generation) do + old_generation = + Data.remote_view_generation(state.name, state.shard_index, remote_node) + + state = maybe_purge_remote_generation(state, remote_node, old_generation, generation) + state = purge_remote_streams_outside_authority(state, remote_node) + :ok = install_replica_view(state, remote_node, generation) + + state + |> touch_replica_peer(remote_node) + |> Map.update!(:cluster_control_dirty, &Map.delete(&1, remote_node)) + |> send_replica_heads(remote_node) + end + """, + faulty_source: """ + defp install_current_replica_lane(state, remote_node, generation) do + _ = {remote_node, generation} + state + end + """, + test: ["test/replica_snapshot_distributed_test.exs:329"] + }, %{ name: "skip_generation_purge", file: "lib/group/replica.ex", diff --git a/test/replica_model_property_test.exs b/test/replica_model_property_test.exs index 4999c3a..e4175d2 100644 --- a/test/replica_model_property_test.exs +++ b/test/replica_model_property_test.exs @@ -41,6 +41,7 @@ defmodule Group.ReplicaModelPropertyTest do replicated_sender_buffer_size: 1, replicated_anti_entropy_interval: 60_000, replicated_peer_lease_timeout: 120_000, + replicated_snapshot_chunk_target_bytes: 256, replicated_oplog_max_entries: 4 ] @@ -87,6 +88,7 @@ defmodule Group.ReplicaModelPropertyTest do replicated_sender_buffer_size: 1, replicated_anti_entropy_interval: 60_000, replicated_peer_lease_timeout: 120_000, + replicated_snapshot_chunk_target_bytes: 256, replicated_oplog_max_entries: 2 ] @@ -201,9 +203,17 @@ defmodule Group.ReplicaModelPropertyTest do scheduler = Enum.reduce(41..46, scheduler, fn owner_id, state -> - state - |> ReplicaModelScheduler.execute({:register, owner_id, :a, 0, owner_id}) - |> ReplicaModelScheduler.execute({:unregister, owner_id, 0}) + state = + ReplicaModelScheduler.execute( + state, + {:register, owner_id, :a, 0, owner_id} + ) + + if rem(owner_id, 2) == 0 do + ReplicaModelScheduler.execute(state, {:unregister, owner_id, 0}) + else + state + end end) scheduler = @@ -358,6 +368,7 @@ defmodule Group.ReplicaModelPropertyTest do replicated_sender_buffer_size: 1, replicated_anti_entropy_interval: 60_000, replicated_peer_lease_timeout: 120_000, + replicated_snapshot_chunk_target_bytes: 256, replicated_oplog_max_entries: Keyword.fetch!(overrides, :oplog) ] end diff --git a/test/replica_snapshot_distributed_test.exs b/test/replica_snapshot_distributed_test.exs new file mode 100644 index 0000000..a517770 --- /dev/null +++ b/test/replica_snapshot_distributed_test.exs @@ -0,0 +1,608 @@ +defmodule Group.ReplicaSnapshotDistributedTest do + use ExUnit.Case, async: false + + @moduletag :capture_log + @moduletag timeout: 120_000 + + alias Group.TestCluster + + setup_all do + peers = TestCluster.start_peers(2, schedulers: 4) + on_exit(fn -> TestCluster.stop_peers(peers) end) + [{_, node_a}, {_, node_b}] = peers + {:ok, node_a: node_a, node_b: node_b} + end + + test "loss, reordering, and duplication expose nothing until exact commit", context do + %{name: name, node_a: node_a, node_b: node_b} = start_pair(context) + + stale_reg_key = "snapshot/stale-reg" + stale_pg_key = "snapshot/stale-pg" + stale_reg_pid = TestCluster.spawn_register(node_a, name, stale_reg_key, %{stale: true}) + stale_pg_pid = TestCluster.spawn_join(node_a, name, stale_pg_key, %{stale: true}) + + TestCluster.assert_eventually(fn -> + match?({^stale_reg_pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, stale_reg_key])) and + match?( + [{^stale_pg_pid, _}], + TestCluster.rpc!(node_b, Group, :members, [name, stale_pg_key]) + ) + end) + + stream_id = local_stream(node_a, name, nil) + old_cursor = replica_cursor(node_b, name, stream_id) + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + TestCluster.rpc!(node_a, Process, :exit, [stale_reg_pid, :kill]) + TestCluster.rpc!(node_a, Process, :exit, [stale_pg_pid, :kill]) + + fresh_reg = + for index <- 1..8 do + key = "snapshot/fresh-reg/#{index}" + + {key, + TestCluster.spawn_register(node_a, name, key, %{ + payload: String.duplicate("r", 160) + })} + end + + fresh_pg_key = "snapshot/fresh-pg" + + fresh_pg = + for _index <- 1..8 do + TestCluster.spawn_join(node_a, name, fresh_pg_key, %{ + payload: String.duplicate("p", 160) + }) + end + + TestCluster.flush_shards(node_a, name) + frames = capture_snapshot(node_a, node_b, name, stream_id, old_cursor + 1) + assert length(frames) > 2 + [missing | delivered] = frames + + deliver_frames(node_b, node_a, name, Enum.reverse(delivered)) + deliver_frames(node_b, node_a, name, [List.last(delivered)]) + TestCluster.flush_shards(node_b, name) + + assert replica_cursor(node_b, name, stream_id) == old_cursor + + assert match?( + {^stale_reg_pid, _}, + TestCluster.rpc!(node_b, Group, :lookup, [name, stale_reg_key]) + ) + + assert match?( + [{^stale_pg_pid, _}], + TestCluster.rpc!(node_b, Group, :members, [name, stale_pg_key]) + ) + + assert Enum.all?(fresh_reg, fn {key, _pid} -> + TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + end) + + assert TestCluster.rpc!(node_b, Group, :members, [name, fresh_pg_key]) == [] + + deliver_frames(node_b, node_a, name, [missing]) + TestCluster.flush_shards(node_b, name) + snapshot_seq = elem(missing, 3) + + assert replica_cursor(node_b, name, stream_id) == snapshot_seq + assert TestCluster.rpc!(node_b, Group, :lookup, [name, stale_reg_key]) == nil + assert TestCluster.rpc!(node_b, Group, :members, [name, stale_pg_key]) == [] + + assert Enum.all?(fresh_reg, fn {key, pid} -> + match?({^pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + + assert MapSet.new( + Enum.map( + TestCluster.rpc!(node_b, Group, :members, [name, fresh_pg_key]), + &elem(&1, 0) + ) + ) == + MapSet.new(fresh_pg) + + assert snapshot_transfer_count(node_b, name) == 0 + end + + test "a newer exact snapshot supersedes an incomplete older one", context do + %{name: name, node_a: node_a, node_b: node_b} = start_pair(context) + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + old_entries = + for index <- 1..8 do + key = "snapshot/superseded/#{index}" + + {key, + TestCluster.spawn_register(node_a, name, key, %{ + payload: String.duplicate("o", 160) + })} + end + + TestCluster.flush_shards(node_a, name) + stream_id = local_stream(node_a, name, nil) + older = capture_snapshot(node_a, node_b, name, stream_id, 1) + assert length(older) > 1 + + Enum.each(old_entries, fn {_key, pid} -> + TestCluster.rpc!(node_a, Process, :exit, [pid, :kill]) + end) + + fresh_entries = + for index <- 1..8 do + key = "snapshot/replacement/#{index}" + + {key, + TestCluster.spawn_register(node_a, name, key, %{ + payload: String.duplicate("n", 160) + })} + end + + TestCluster.flush_shards(node_a, name) + newer = capture_snapshot(node_a, node_b, name, stream_id, 1) + assert elem(hd(newer), 3) > elem(hd(older), 3) + + deliver_frames(node_b, node_a, name, [hd(older)]) + assert snapshot_transfer_count(node_b, name) == 1 + + deliver_frames(node_b, node_a, name, Enum.reverse(newer)) + deliver_frames(node_b, node_a, name, tl(older)) + TestCluster.flush_shards(node_b, name) + + assert replica_cursor(node_b, name, stream_id) == elem(hd(newer), 3) + + assert Enum.all?(old_entries, fn {key, _pid} -> + TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + end) + + assert Enum.all?(fresh_entries, fn {key, pid} -> + match?({^pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + + assert snapshot_transfer_count(node_b, name) == 0 + end + + test "conflicting retransmission chunks cannot manufacture an exact snapshot", context do + %{name: name, node_a: node_a, node_b: node_b} = start_pair(context) + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + entries = + for index <- 1..8 do + key = "snapshot/conflicting/#{index}" + + {key, + TestCluster.spawn_register(node_a, name, key, %{ + payload: String.duplicate("m", 160) + })} + end + + TestCluster.flush_shards(node_a, name) + stream_id = local_stream(node_a, name, nil) + [first, second | rest] = frames = capture_snapshot(node_a, node_b, name, stream_id, 1) + [first_row | _] = elem(first, 8) + [_second_row | second_tail] = elem(second, 8) + conflicting_second = put_elem(second, 8, [first_row | second_tail]) + + deliver_frames(node_b, node_a, name, [first, conflicting_second | rest]) + + assert replica_cursor(node_b, name, stream_id) == 0 + + assert Enum.all?(entries, fn {key, _pid} -> + TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + end) + + deliver_frames(node_b, node_a, name, frames) + + assert replica_cursor(node_b, name, stream_id) == elem(first, 3) + + assert Enum.all?(entries, fn {key, pid} -> + match?({^pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + end + + test "an authority epoch change fences partial chunks and their staging expires", context do + cluster = "snapshot-epoch" + + %{name: name, node_a: node_a, node_b: node_b} = + start_pair(context, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 250 + ) + + :ok = TestCluster.rpc!(node_a, Group, :connect, [name, cluster]) + :ok = TestCluster.rpc!(node_b, Group, :connect, [name, cluster]) + + TestCluster.assert_eventually(fn -> + length(TestCluster.rpc!(node_a, Group, :nodes, [name, cluster])) == 2 and + length(TestCluster.rpc!(node_b, Group, :nodes, [name, cluster])) == 2 + end) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + old_entries = + for index <- 1..8 do + key = "snapshot/epoch/#{index}" + + {key, + TestCluster.spawn_register_in_cluster( + node_a, + name, + key, + %{payload: String.duplicate("e", 160)}, + cluster + )} + end + + TestCluster.flush_shards(node_a, name) + old_stream = local_stream(node_a, name, cluster) + old_epoch = Group.Replica.Protocol.stream_epoch(old_stream) + frames = capture_snapshot(node_a, node_b, name, old_stream, 1) + assert length(frames) > 1 + {partial, [last]} = Enum.split(frames, -1) + deliver_frames(node_b, node_a, name, partial) + assert snapshot_transfer_count(node_b, name) == 1 + + :ok = TestCluster.rpc!(node_a, Group, :disconnect, [name, cluster]) + :ok = TestCluster.rpc!(node_a, Group, :connect, [name, cluster]) + + TestCluster.assert_eventually(fn -> + epoch = + TestCluster.rpc!(node_b, Group.Replica.Data, :remote_cluster_epoch, [ + name, + node_a, + cluster + ]) + + not is_nil(epoch) and epoch != old_epoch + end) + + deliver_frames(node_b, node_a, name, [last]) + TestCluster.flush_shards(node_b, name) + + assert Enum.all?(old_entries, fn {key, _pid} -> + TestCluster.rpc!(node_b, Group, :lookup, [name, key, [cluster: cluster]]) == nil + end) + + assert replica_cursor(node_b, name, old_stream) == 0 + + TestCluster.assert_eventually( + fn -> snapshot_transfer_count(node_b, name) == 0 end, + timeout: 2_000, + interval: 25 + ) + end + + test "an origin restart fences a partial old generation and commits the new exact snapshot", + context do + %{name: name, node_a: node_a, node_b: node_b, opts: opts} = + start_pair(context, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 2_000 + ) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + old_entries = + for index <- 1..8 do + key = "snapshot/restart/old/#{index}" + + {key, + TestCluster.spawn_register(node_a, name, key, %{ + payload: String.duplicate("o", 160) + })} + end + + TestCluster.flush_shards(node_a, name) + old_stream = local_stream(node_a, name, nil) + old_frames = capture_snapshot(node_a, node_b, name, old_stream, 1) + assert length(old_frames) > 1 + + {old_partial, _old_tail} = Enum.split(old_frames, -1) + deliver_frames(node_b, node_a, name, old_partial) + assert snapshot_transfer_count(node_b, name) == 1 + assert replica_cursor(node_b, name, old_stream) == 0 + + supervisor = TestCluster.rpc!(node_a, Process, :whereis, [:"#{name}_group_sup"]) + :ok = TestCluster.rpc!(node_a, Supervisor, :stop, [supervisor, :normal, 5_000]) + {:ok, _pid} = TestCluster.start_group(node_a, opts) + + new_stream = local_stream(node_a, name, nil) + refute new_stream == old_stream + new_generation = Group.Replica.Protocol.stream_generation(new_stream) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(node_b, Group.Replica.Data, :remote_generation, [name, node_a]) == + new_generation + end) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + new_entries = + for index <- 1..8 do + key = "snapshot/restart/new/#{index}" + + {key, + TestCluster.spawn_register(node_a, name, key, %{ + payload: String.duplicate("n", 160) + })} + end + + TestCluster.flush_shards(node_a, name) + new_frames = capture_snapshot(node_a, node_b, name, new_stream, 1) + assert length(new_frames) > 1 + + deliver_frames(node_b, node_a, name, Enum.reverse(new_frames)) + new_snapshot_seq = new_frames |> hd() |> elem(3) + + assert replica_cursor(node_b, name, new_stream) == new_snapshot_seq + + assert Enum.all?(new_entries, fn {key, pid} -> + match?({^pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + + assert Enum.all?(old_entries, fn {key, _pid} -> + TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + end) + + # Replaying every old-generation chunk after the new exact commit must not + # resurrect the old slice or advance its cursor. + deliver_frames(node_b, node_a, name, old_frames) + assert replica_cursor(node_b, name, old_stream) == 0 + assert replica_cursor(node_b, name, new_stream) == new_snapshot_seq + + assert Enum.all?(new_entries, fn {key, pid} -> + match?({^pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + + assert Enum.all?(old_entries, fn {key, _pid} -> + TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + end) + + TestCluster.assert_eventually( + fn -> snapshot_transfer_count(node_b, name) == 0 end, + timeout: 5_000, + interval: 25 + ) + end + + test "a receiver shard crash destroys partial staging and anti-entropy rebuilds exactly", + context do + %{name: name, node_a: node_a, node_b: node_b} = + start_pair(context, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 1_000 + ) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + entries = + for index <- 1..8 do + key = "snapshot/crash/#{index}" + + {key, + TestCluster.spawn_register(node_a, name, key, %{ + payload: String.duplicate("c", 160) + })} + end + + TestCluster.flush_shards(node_a, name) + stream_id = local_stream(node_a, name, nil) + frames = capture_snapshot(node_a, node_b, name, stream_id, 1) + assert length(frames) > 1 + deliver_frames(node_b, node_a, name, [hd(frames)]) + + {old_shard, staging_info_after_crash} = + TestCluster.rpc!(node_b, TestCluster, :kill_shard_with_snapshot_staging, [name, 0]) + + assert staging_info_after_crash == :undefined + + TestCluster.assert_eventually(fn -> + case TestCluster.rpc!(node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) do + pid when is_pid(pid) -> pid != old_shard + nil -> false + end + end) + + assert snapshot_transfer_count(node_b, name) == 0 + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :pass]) + + TestCluster.assert_eventually( + fn -> + Enum.all?(entries, fn {key, pid} -> + match?({^pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + end, + timeout: 10_000, + interval: 25 + ) + + assert :ok = TestCluster.rpc!(node_b, Group.TestCluster, :assert_replica_consistent, [name]) + end + + test "authority fanout tolerates a sibling that is not registered", context do + name = :"authority_startup_fanout_#{System.unique_integer([:positive])}" + + opts = [ + name: name, + shards: 2, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + for node <- [context.node_a, context.node_b] do + {:ok, _pid} = TestCluster.start_group(node, opts) + end + + TestCluster.assert_eventually(fn -> + context.node_b in TestCluster.rpc!(context.node_a, Group, :nodes, [name]) + end) + + replica_supervisor = + TestCluster.rpc!(context.node_b, Process, :whereis, [:"#{name}_replica_sup"]) + + :ok = TestCluster.rpc!(context.node_b, :sys, :suspend, [replica_supervisor]) + + on_exit(fn -> + TestCluster.rpc!(context.node_b, Group.TestCluster, :resume_if_alive, [replica_supervisor]) + end) + + sibling = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + sibling_monitor = Process.monitor(sibling) + TestCluster.rpc!(context.node_b, Process, :exit, [sibling, :kill]) + assert_receive {:DOWN, ^sibling_monitor, :process, ^sibling, :killed}, 5_000 + + assert TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) == + nil + + source_control = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + target_control = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + :ok = TestCluster.rpc!(context.node_a, Group, :connect, [name, "late-lane-authority"]) + + {generation, revision, epochs} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_replica_authority, [name]) + + control_monitor = Process.monitor(target_control) + + send( + target_control, + {:replica_hello, source_control, Group.Replica.Protocol.version(), generation, revision, + epochs, Group.Replica.Transport.Distribution.id(), + Group.Replica.Transport.Distribution.descriptor(name, [])} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_control]) + refute_receive {:DOWN, ^control_monitor, :process, ^target_control, _reason}, 250 + assert TestCluster.rpc!(context.node_b, Process, :alive?, [target_control]) + + :ok = TestCluster.rpc!(context.node_b, :sys, :resume, [replica_supervisor]) + + TestCluster.assert_eventually(fn -> + sibling = + TestCluster.rpc!(context.node_b, Process, :whereis, [ + Group.Replica.shard_name(name, 1) + ]) + + view_generation = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_view_generation, [ + name, + 1, + context.node_a + ]) + + view_exact_revision = + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_view_cluster_epoch_revision, + [name, 1, context.node_a] + ) + + exact_revision = + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_exact_revision, + [name, context.node_a] + ) + + is_pid(sibling) and view_generation == generation and + view_exact_revision == exact_revision + end) + end + + defp start_pair(context, extra_opts \\ []) do + name = :"snapshot_chunks_#{System.unique_integer([:positive])}" + + opts = + Keyword.merge( + [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 1, + replicated_oplog_max_entries: 2, + replicated_snapshot_chunk_target_bytes: 700, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ], + extra_opts + ) + + for node <- [context.node_a, context.node_b] do + {:ok, _pid} = TestCluster.start_group(node, opts) + end + + TestCluster.assert_eventually(fn -> + context.node_b in TestCluster.rpc!(context.node_a, Group, :nodes, [name]) and + context.node_a in TestCluster.rpc!(context.node_b, Group, :nodes, [name]) + end) + + %{name: name, node_a: context.node_a, node_b: context.node_b, opts: opts} + end + + defp capture_snapshot(node_a, node_b, name, stream_id, next_seq) do + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :clear_captured, [name]) + + :ok = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [ + name, + {:capture_drop, [:snapshot_chunk]} + ]) + + :ok = + TestCluster.rpc!(node_a, Group.Replica.Transport, :deliver, [ + name, + node_b, + 0, + {:needs, Group.Replica.Protocol.version(), [{stream_id, next_seq}]} + ]) + + TestCluster.flush_shards(node_a, name) + + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :captured, [name]) + |> Enum.flat_map(fn + {^node_b, 0, + {:snapshot_chunk, _version, ^stream_id, _seq, _index, _count, _reg_count, _pg_count, _reg, + _pg} = frame} -> + [frame] + + _other -> + [] + end) + |> Enum.sort_by(&elem(&1, 4)) + end + + defp deliver_frames(node_b, node_a, name, frames) do + Enum.each(frames, fn frame -> + :ok = + TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + name, + node_a, + 0, + frame + ]) + end) + + TestCluster.flush_shards(node_b, name) + end + + defp local_stream(node, name, cluster) do + TestCluster.rpc!(node, Group.Replica.Data, :local_stream_id, [name, 0, cluster]) + end + + defp replica_cursor(node, name, stream_id) do + TestCluster.rpc!(node, Group.Replica.Data, :replica_cursor, [name, 0, stream_id]) + end + + defp snapshot_transfer_count(node, name) do + TestCluster.rpc!(node, :erlang, :map_size, [ + TestCluster.rpc!(node, :sys, :get_state, [Group.Replica.shard_name(name, 0)]).snapshot_transfers + ]) + end +end diff --git a/test/replica_snapshot_test.exs b/test/replica_snapshot_test.exs new file mode 100644 index 0000000..d87700f --- /dev/null +++ b/test/replica_snapshot_test.exs @@ -0,0 +1,75 @@ +defmodule Group.ReplicaSnapshotTest do + use ExUnit.Case, async: true + + alias Group.Replica.Snapshot + + test "partitions a complete exact slice into byte-bounded deterministic chunks" do + pid = self() + metadata = %{payload: String.duplicate("x", 96)} + + registry_rows = + for index <- 1..80 do + {"registry/#{index}", pid, metadata, index} + end + + pg_rows = + for index <- 1..80 do + {"pg/#{index}", pid, metadata, index} + end + + target = 2_048 + stream_id = {:group, node(), make_ref(), 0, nil, make_ref()} + envelope = Snapshot.frame_envelope_bytes(stream_id, 123, 80, 80) + + snapshot = + Snapshot.chunk_rows(Enum.reverse(registry_rows), Enum.reverse(pg_rows), target, envelope) + + assert snapshot.registry_count == 80 + assert snapshot.pg_count == 80 + assert length(snapshot.chunks) > 1 + + assert snapshot.chunks == + Snapshot.chunk_rows(registry_rows, pg_rows, target, envelope).chunks + + assert snapshot.chunks |> Enum.flat_map(&elem(&1, 0)) |> MapSet.new() == + MapSet.new(registry_rows) + + assert snapshot.chunks |> Enum.flat_map(&elem(&1, 1)) |> MapSet.new() == + MapSet.new(pg_rows) + + chunk_count = length(snapshot.chunks) + + Enum.with_index(snapshot.chunks, 1) + |> Enum.each(fn {{registry, pg}, index} -> + frame = + {:snapshot_chunk, Group.Replica.Protocol.version(), stream_id, 123, index, chunk_count, + snapshot.registry_count, snapshot.pg_count, registry, pg} + + assert :erlang.external_size(frame) <= target + end) + end + + test "represents an empty exact slice and permits one intrinsically oversized row" do + assert Snapshot.chunk_rows([], [], 1_024).chunks == [{[], []}] + + row = {"large", self(), String.duplicate("x", 4_096), 1} + snapshot = Snapshot.chunk_rows([row], [], 1_024) + assert snapshot.chunks == [{[row], []}] + end + + test "staging is set-valued across chunks and remains private to its owner" do + table = Snapshot.new_staging_table() + registry = {"registry", self(), %{v: 1}, 1} + pg = {"pg", self(), %{v: 2}, 2} + + assert :ok = Snapshot.stage_rows(table, 1, [registry], [pg]) + assert {:error, :duplicate_row} = Snapshot.stage_rows(table, 2, [registry], []) + + assert Snapshot.fold_registry(table, 1, [], fn row, acc -> [row | acc] end) == [registry] + assert Snapshot.fold_pg(table, 1, [], fn row, acc -> [row | acc] end) == [pg] + assert Snapshot.member_pg?(table, "pg", self()) + + assert :ok = Snapshot.delete_staging_table(table) + assert :ets.info(table) == :undefined + end +end diff --git a/test/replica_transport_outbox_test.exs b/test/replica_transport_outbox_test.exs new file mode 100644 index 0000000..329ed96 --- /dev/null +++ b/test/replica_transport_outbox_test.exs @@ -0,0 +1,155 @@ +defmodule Group.ReplicaTransportOutboxTest do + use ExUnit.Case, async: true + + alias Group.Replica.Transport.Outbox + + defmodule Backend do + @behaviour Outbox + + @impl true + def init_outbox(group, shard, opts) do + {:ok, + %{ + controller: Keyword.fetch!(opts, :controller), + group: group, + shard: shard, + result: Keyword.get(opts, :backend_result, :ok), + sleep: Keyword.get(opts, :backend_sleep, 0) + }} + end + + @impl true + def send_batch(target_node, frames, deadline, state) do + send( + state.controller, + {:outbox_batch, state.group, state.shard, target_node, frames, deadline} + ) + + if state.sleep > 0, do: Process.sleep(state.sleep) + {state.result, state} + end + end + + test "batches frames per target while preserving per-target order" do + group = unique_group(:batch) + target_a = :"outbox-a@test" + target_b = :"outbox-b@test" + + start_outboxes(group, + outbox_batch_size: 3, + outbox_flush_interval: 1_000 + ) + + assert :ok = Outbox.try_send(group, target_a, 0, {:frame, 1}, outbox_deadline: 1_000) + assert :ok = Outbox.try_send(group, target_b, 0, {:frame, 2}, outbox_deadline: 1_000) + assert :ok = Outbox.try_send(group, target_a, 0, {:frame, 3}, outbox_deadline: 1_000) + + batches = + for _ <- 1..2, into: %{} do + assert_receive {:outbox_batch, ^group, 0, target, frames, deadline}, 1_000 + assert deadline > Outbox.monotonic_ms() + {target, frames} + end + + assert batches == %{ + target_a => [{:frame, 1}, {:frame, 3}], + target_b => [{:frame, 2}] + } + end + + test "a blocked backend never blocks the Group-facing local send" do + group = unique_group(:blocked) + target = :"outbox-blocked@test" + + start_outboxes(group, + outbox_batch_size: 1, + backend_sleep: 200 + ) + + assert :ok = Outbox.try_send(group, target, 0, :first, outbox_deadline: 1_000) + assert_receive {:outbox_batch, ^group, 0, ^target, [:first], _deadline}, 1_000 + + caller = self() + + spawn(fn -> + result = Outbox.try_send(group, target, 0, :expires_behind_backend, outbox_deadline: 10) + send(caller, {:try_send_returned, result}) + end) + + assert_receive {:try_send_returned, :ok}, 100 + refute_receive {:outbox_batch, ^group, 0, ^target, [:expires_behind_backend], _deadline}, 300 + end + + test "expired frames and backend backpressure are dropped without local retries" do + expired_group = unique_group(:expired) + target = :"outbox-expired@test" + + start_outboxes(expired_group, + outbox_flush_interval: 50 + ) + + assert :ok = + Outbox.try_send(expired_group, target, 0, :expired, outbox_deadline: 5) + + refute_receive {:outbox_batch, ^expired_group, 0, ^target, [:expired], _deadline}, 100 + + busy_group = unique_group(:busy) + + start_outboxes(busy_group, + outbox_batch_size: 1, + backend_result: :busy + ) + + assert :ok = Outbox.try_send(busy_group, target, 0, :busy, outbox_deadline: 1_000) + assert_receive {:outbox_batch, ^busy_group, 0, ^target, [:busy], _deadline}, 1_000 + refute_receive {:outbox_batch, ^busy_group, 0, ^target, [:busy], _deadline}, 100 + end + + test "complete inbound batches use one authenticated local delivery" do + group = unique_group(:deliver) + source_node = :"outbox-source@test" + parent = self() + shard_name = Group.Replica.shard_name(group, 0) + + receiver = + spawn(fn -> + Process.register(self(), shard_name) + send(parent, :receiver_ready) + + receive do + message -> send(parent, {:receiver_message, message}) + end + end) + + assert_receive :receiver_ready + + assert :ok = + Group.Replica.Transport.deliver_batch( + group, + source_node, + 0, + [{:heads, 1, []}, {:needs, 1, []}] + ) + + assert_receive {:receiver_message, + {:group_replica_batch, ^source_node, [{:heads, 1, []}, {:needs, 1, []}]}} + + refute Process.alive?(receiver) + end + + defp start_outboxes(group, opts) do + base = [ + name: group, + num_shards: 1, + backend: Backend, + controller: self(), + outbox_deadline: 100 + ] + + start_supervised!(Outbox.child_spec(Keyword.merge(base, opts))) + end + + defp unique_group(suffix) do + :"outbox_#{suffix}_#{System.unique_integer([:positive])}" + end +end diff --git a/test/support/test_cluster.ex b/test/support/test_cluster.ex index 8f97eb7..80cd37e 100644 --- a/test/support/test_cluster.ex +++ b/test/support/test_cluster.ex @@ -54,6 +54,12 @@ defmodule Group.TestCluster do end) end + @doc false + def resume_if_alive(pid) when is_pid(pid) do + if Process.alive?(pid), do: :sys.resume(pid) + :ok + end + @doc "Call a function on a remote node, raise on badrpc" def rpc!(node, mod, fun, args) do case :rpc.call(node, mod, fun, args) do @@ -503,6 +509,22 @@ defmodule Group.TestCluster do end) end + @doc false + def kill_shard_with_snapshot_staging(name, shard_index) do + shard = Process.whereis(Group.Replica.shard_name(name, shard_index)) + state = :sys.get_state(shard) + {_key, transfer} = Enum.at(state.snapshot_transfers, 0) + monitor = Process.monitor(shard) + Process.exit(shard, :kill) + + receive do + {:DOWN, ^monitor, :process, ^shard, :killed} -> + {shard, :ets.info(transfer.table)} + after + 5_000 -> raise "snapshot staging owner did not terminate" + end + end + @doc "Returns the current message_queue_len for a shard on a remote node." def shard_message_queue_len(node, name, shard) do :erpc.call(node, __MODULE__, :do_shard_message_queue_len, [name, shard]) From bc5bf48f1ada2c57696ef84e25806b7d2537d371 Mon Sep 17 00:00:00 2001 From: Chris McCord Date: Tue, 11 Aug 2026 04:22:31 +0000 Subject: [PATCH 05/16] test: add anti-entropy release qualification --- CHANGELOG.md | 6 + CLAUDE.md | 391 ++--- README.md | 53 +- lib/group.ex | 12 +- lib/group/replica/data.ex | 46 +- lib/group/replica/transport.ex | 10 +- mix.exs | 16 + test/README.md | 25 +- test/formal/GroupAntiEntropyExtended.cfg | 17 + test/formal/PeerEviction.cfg | 15 + test/formal/PeerEviction.tla | 207 +++ test/formal/README.md | 26 +- test/formal/check_matrix.sh | 22 + test/jepsen/.gitignore | 5 + test/jepsen/Dockerfile.node | 22 + test/jepsen/README.md | 149 ++ test/jepsen/campaign.sh | 67 + test/jepsen/checker.sh | 6 + test/jepsen/docker-compose.yml | 49 + test/jepsen/entrypoint.sh | 10 + test/jepsen/lein.sh | 20 + test/jepsen/node.exs | 1376 ++++++++++++++++++ test/jepsen/project.clj | 8 + test/jepsen/qualify.sh | 55 + test/jepsen/run.sh | 41 + test/jepsen/src/group/jepsen/client.clj | 132 ++ test/jepsen/src/group/jepsen/core.clj | 187 +++ test/jepsen/src/group/jepsen/db.clj | 28 + test/jepsen/src/group/jepsen/docker.clj | 121 ++ test/jepsen/src/group/jepsen/model.clj | 219 +++ test/jepsen/src/group/jepsen/nemesis.clj | 166 +++ test/jepsen/test/group/jepsen/model_test.clj | 208 +++ test/replica_adversarial_test.exs | 114 +- 33 files changed, 3578 insertions(+), 251 deletions(-) create mode 100644 test/formal/GroupAntiEntropyExtended.cfg create mode 100644 test/formal/PeerEviction.cfg create mode 100644 test/formal/PeerEviction.tla create mode 100755 test/formal/check_matrix.sh create mode 100644 test/jepsen/.gitignore create mode 100644 test/jepsen/Dockerfile.node create mode 100644 test/jepsen/README.md create mode 100755 test/jepsen/campaign.sh create mode 100755 test/jepsen/checker.sh create mode 100644 test/jepsen/docker-compose.yml create mode 100755 test/jepsen/entrypoint.sh create mode 100755 test/jepsen/lein.sh create mode 100644 test/jepsen/node.exs create mode 100644 test/jepsen/project.clj create mode 100755 test/jepsen/qualify.sh create mode 100755 test/jepsen/run.sh create mode 100644 test/jepsen/src/group/jepsen/client.clj create mode 100644 test/jepsen/src/group/jepsen/core.clj create mode 100644 test/jepsen/src/group/jepsen/db.clj create mode 100644 test/jepsen/src/group/jepsen/docker.clj create mode 100644 test/jepsen/src/group/jepsen/model.clj create mode 100644 test/jepsen/src/group/jepsen/nemesis.clj create mode 100644 test/jepsen/test/group/jepsen/model_test.clj diff --git a/CHANGELOG.md b/CHANGELOG.md index 671cf9d..2f41d3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,10 @@ ## Unreleased +- Add layered anti-entropy qualification: three-node StreamData lifecycle + models, seeded adversarial transport histories, TLA+ models for convergence, + chunk assembly, and permanent peer eviction, plus a Docker-backed Jepsen + oracle across distribution, sideband TCP, and lossy/reordering transports. + `mix test` is the every-PR ExUnit/property/checker gate and `mix test.soak` + runs the six-profile nightly/release campaign. - **Breaking**: replica protocol v2 splits exact snapshots into transport-neutral, byte-targeted chunks (`1 MiB` by default). Receivers stage chunks in shard-owned private ETS and advance the stream cursor only after an diff --git a/CLAUDE.md b/CLAUDE.md index 29fb3e3..9c326c2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,208 +1,261 @@ -# Group — CLAUDE.md +# Group — Maintainer Architecture Guide -## What is Group +## What Group Is -Distributed process registry + process groups + lifecycle monitoring + isolated subclusters. +Group is an eventually consistent distributed process registry, process-group +service, lifecycle monitor, and named-subcluster layer. Erlang distribution is +the membership and authority control plane. Replica state moves over a +configurable, nonblocking data transport and converges through sequenced +anti-entropy streams. ## Project Structure ``` lib/ - group.ex — Public API: register, join, members, monitor, dispatch, connect/disconnect - group/event.ex — %Group.Event{} struct - group/supervisor.ex — Top-level supervisor (rest_for_one: Data → PeerReconnect → Replica.Supervisor → Registry → ClusterLease) - group/cluster_lease.ex — Local named-cluster TTL sweeper - group/peer_reconnect.ex — Bounded reconnect loop after busy distribution links - group/replica/ - data.ex — GenServer that owns ETS tables and serializes shared membership mutations. - supervisor.ex — one_for_one supervisor for Replica shards - group/replica.ex — Sharded GenServer: replication, peer discovery, conflict resolution, monitoring - group/application.ex — Empty app supervisor (Group instances are started by consumers) + group.ex — public API and configuration docs + group/supervisor.ex — rest_for_one instance supervisor + group/cluster_lease.ex — local named-cluster TTL policy + group/peer_reconnect.ex — bounded retry for busy dispatch links + group/replica.ex — sharded writes, control, AE, projection + group/replica/data.ex — ETS owner, journal, authority, indexes + group/replica/protocol.ex — stream identity and mutation helpers + group/replica/snapshot.ex — byte-targeted snapshot chunks/staging + group/replica/transport.ex — replica transport contract + dist adapter + group/replica/transport/outbox.ex — optional lossy sideband outboxes + group/replica/transport/tcp.ex — included sideband TCP adapter test/ - test_helper.exs — Ensures EPMD/distribution are running; disables prevent_overlapping_partitions - group_test.exs — Local tests (async: true) - distributed_test.exs — Multi-node tests using OTP :peer - support/ - test_cluster.ex — Peer node helpers (start_peers, spawn_register, spawn_join, etc.) - test_conflict_resolver.ex — Custom resolver for tests -priv/bench/ — Benchmarks (run_local.sh, run_distributed.sh) + replica_model_property_test.exs — shrinkable real-node lifecycle model + replica_adversarial_test.exs — seeded three-node transport chaos + replica_snapshot_* — chunk/assembly failure coverage + formal/ — TLC protocol, assembly, eviction models + jepsen/ — independent-VM lifecycle oracle/campaign +priv/bench/ — local and distributed benchmark project ``` -## Running Tests +## Required Gates ```bash -mix test # all tests -mix test test/group_test.exs # local only -mix test test/distributed_test.exs # distributed only -``` - -Tests require `elixirc_paths(:test)` includes `test/support/`. Distributed tests use OTP `:peer` module for real Erlang nodes. +mix test # every-PR ExUnit/property/chaos/pure-checker gate +mix test.soak # nightly/release six-profile Jepsen campaign -## Running Benchmarks - -```bash -cd priv/bench && ./run_local.sh -cd priv/bench && ./run_distributed.sh +# Focused development +mix test test/group_test.exs +mix test test/distributed_test.exs +mix test test/replica_model_property_test.exs ``` -## Architecture +`mix test` accepts normal Mix test paths/options and then runs the pure Jepsen +checker qualification. `mix test.soak` runs that gate first, then twenty +five-minute histories for distribution/TCP/chaos × mixed/permanent scenarios. +See `test/README.md`, `test/formal/README.md`, and `test/jepsen/README.md`. -### Supervision Tree +## Supervision and Ownership ``` Group.Supervisor (rest_for_one) -├── Group.Replica.Data — owns all ETS tables; serializes membership mutations -├── Group.PeerReconnect — bounded reconnects after busy-link disconnects -├── Group.Replica.Supervisor — one_for_one, N shard GenServers -│ ├── Replica shard 0 -│ ├── Replica shard 1 +├── optional sideband transport manager + per-shard outboxes +├── Group.Replica.Data +├── Group.PeerReconnect +├── Group.Replica.Supervisor +│ ├── Group.Replica shard 0 +│ ├── Group.Replica shard 1 │ └── ... -├── Registry (Elixir) — :duplicate, for monitor subscriptions -└── Group.ClusterLease — local named-cluster TTL sweeper +├── Registry — local monitor subscriptions +└── Group.ClusterLease — local named-cluster TTL sweeper ``` -`rest_for_one` means: if Data dies, every later child restarts. If PeerReconnect dies, the Replica supervisor and monitor Registry restart. If Replica.Supervisor dies, Registry and ClusterLease restart. Replica shards rebuild local process monitors from surviving ETS after restart. - -### Sharding - -`phash2({cluster, key}, num_shards)` routes to shard. Default 8 shards. Must match across all nodes (validated on peer_connect). Including cluster in hash avoids false contention between default and named cluster operations. - -### ETS Tables (per shard × 4 + 3 shared) - -| Table | Type | Key | Tuple | -|-------|------|-----|-------| -| `reg_by_key` | `:set` | `{cluster, key}` | `{{cluster, key}, pid, meta, time, node}` | -| `reg_by_pid` | `:ordered_set` | `{pid, cluster, key}` | `{{pid, cluster, key}, meta, time, node}` | -| `pg_by_key` | `:ordered_set` | `{cluster, key, pid}` | `{{cluster, key, pid}, meta, time, node}` | -| `pg_by_pid` | `:ordered_set` | `{pid, cluster, key}` | `{{pid, cluster, key}, meta, time, node}` | -| `cluster_nodes` | `:bag` | cluster | `{cluster, node}` | -| `node_clusters` | `:bag` | node | `{node, cluster}` | -| `cluster_leases` | `:set` | cluster | `{cluster, ttl_ms, expires_at}` | - -**Why ordered_set for by_pid tables**: Contiguous range scans for `entries_by_pid` and `delete_all_for_pid` (process death cleanup). Also enables efficient existence checks with `select(..., 1)`. - -**Why ordered_set for pg_by_key**: `pg_members/4` scans `{cluster, key, *}` as a contiguous range — O(members in group), not O(table). - -**Why set for reg_by_key**: Only needs direct lookup/delete by `{cluster, key}` — O(1). - -All tables: `:public`, `read_concurrency: true`, `decentralized_counters: true`. Per-key registry/PG writes serialize through the owning Replica shard; shared cluster-membership dual-index mutations serialize through `Group.Replica.Data`. No table enables `write_concurrency`. - -### Reads vs Writes - -- **Reads** (`lookup`, `members`, `local_registry_count`) go directly to ETS — no GenServer involved -- **Writes** (`register`, `join`, `leave`, `unregister`) go through the shard's - local request lane (`send` + monitor + tagged reply), not `GenServer.call` -- **Replication** arrives as `handle_info` messages on shard GenServers - -### Config - -Stored in `persistent_term` keyed by `{Group, name}`. Includes shard/buffering/reconnect settings plus `log`, and optionally `extract_meta` and `resolve_registry_conflict`. +`Group.Replica.Data` owns every ETS table. A shard restart therefore preserves +the tables, repairs interrupted index/journal work, replays appended-but-not- +applied local mutations, and rebuilds monitors for locally owned processes. +The optional transport precedes Data so losing its manager restarts the whole +instance and cannot leave stale transport sessions attached to retained state. -## Key Protocols +Reads use public ETS directly. Writes route through +`:erlang.phash2({cluster, key}, num_shards)` to a local shard request lane. +Shard counts must match across peers. -### Peer Discovery (per-shard, on nodeup/init) +## ETS State -1. Each shard sends `{:peer_connect, pid, shard, num_shards, clusters}` to its counterpart -2. Receiver adds sender to nil cluster ETS, computes shared clusters, sends `{:peer_connect_ack, ...}` -3. Both sides send `{:cluster_state, cluster, reg_data, pg_data}` for each shared cluster -4. `merge_remote_cluster_data` applies data: new entries insert, conflicts go through `resolve_conflict` +Per shard: -### Replication (steady state) +| Table | Key | Role | +|---|---|---| +| `reg_by_key` | `{cluster, key}` | visible registry winner | +| `reg_by_pid` | `{pid, cluster, key}` | visible reverse index | +| `reg_claim_by_key` | `{cluster, key, origin, generation, epoch}` | authoritative claims | +| `reg_claim_by_pid` | `{pid, cluster, key, origin, generation, epoch}` | claim reverse index | +| `pg_by_key` | `{cluster, key, pid}` | visible PG membership | +| `pg_by_pid` | `{pid, cluster, key}` | PG reverse index | +| `replica_stream_meta` | stream id | head, retained floor, journal position | +| `replica_oplog` | `{stream, sequence}` | retained mutation record | +| `replica_oplog_order` | append id | shard-wide pruning order | +| `replica_cursor` | stream id | highest contiguous sequence applied | -After discovery, writes replicate in two stages: -- nil cluster: uses `state.remote_shards` map -- Named clusters: uses `cluster_nodes` ETS table -- Sender batches: `replicate_registry_batch`, `replicate_pg_batch` -- Receiver buffers registry and PG lanes separately, bulk-applies ETS writes, - then takes a bounded fairness turn for local work -- Remote shard sends use `send_nosuspend(..., [:noconnect])`; a `false` result - force-disconnects that node and enters bounded reconnect retries for that - peer only +Shared tables hold cluster/node indexes, local TTL leases, local and remote +cluster epochs, closed-cluster barriers, origin generations, exact/observed +authority revisions, installed lane views, and journal metadata. -### Conflict Resolution +Per-shard tables omit `write_concurrency` because one shard serializes their +writes. The shared replication metadata table uses +`write_concurrency: :auto`: every shard atomically updates only its own +`{:append_counter, shard}` object. Cross-shard arrival order has no semantic +meaning; an append id exists only to bound that shard's oplog across streams. -`resolve_conflict/5` and the batched equivalent handle all registry key conflicts (live replicated registry ops and partition-heal `merge_remote_cluster_data`). +## Authority and Stream Identity -- Default resolver: most recent timestamp wins; pid ordering tiebreaker on equal timestamps -- **Tiebreaker MUST be deterministic across all nodes**: `time2 > time1 or (time2 == time1 and pid2 > pid1)`. Using `>=` causes mutual kill. -- Custom: `resolve_registry_conflict: {mod, func, extra_args}` option; the callback owns any process exits -- The default resolver kills the loser with `{:group_registry_conflict, key, winner_meta}` -- Runs synchronously inside shard GenServer — must return quickly +Every mutation belongs to: -### Named Cluster Connect/Disconnect - -- `connect/2`: adds to ETS, picks random shard S, S notifies remote S, remote acks with bundled data + fans out to siblings -- `disconnect/2`: removes local membership, calls ALL local shards to purge the complete replicated cluster view, and has shard 0 broadcast to remotes -- `connect(..., ttl: ms)`: still checks `cluster_nodes` first, so an already-connected - cluster stays an ETS-fast noop and does not refresh the TTL -- TTL rows are local policy only; they do not change `cluster_nodes` / - `node_clusters` semantics -- On TTL expiry, `Group.ClusterLease` disconnects only if the local node has no - cluster-scoped monitors, no local registrations, and no local PG memberships - in that cluster. Otherwise it extends the lease by one TTL interval. - -### Nodedown / Process Death - -- `nodedown`: every shard calls `purge_cluster_node` (unconditional — not gated on shard 0) then `purge_node` -- Process DOWN: `delete_all_for_pids` + one non-suspending `replicate_process_down_batch` per target peer -- Remote shard DOWN (monitored pid): treated like nodedown for that node - -### Monitor Events - -Events delivered as `{:group, [%Group.Event{}, ...], %{name: name}}`. Batched per handler turn: -- Single ops: one event per message -- Bulk ops (nodedown, process DOWN, cluster_state merge): all events in one message - -Patterns: `:all`, `{:exact, key}`, `{:prefix, "prefix/"}` - -### Prefix Queries - -`Group.members(name, "prefix/")` scans ALL shards (can't hash prefix to one shard). Uses ETS range guards: -```elixir -{:andalso, {:>=, :"$1", prefix}, {:<, :"$1", next_binary_prefix(prefix)}} ``` -where `next_binary_prefix` increments the last byte of the prefix string. - -Keys ending with `"/"` are rejected by `validate_key!/1` in register/unregister/join/leave — trailing slash is reserved for prefix queries. - -### Dispatch - -`dispatch/4` sends to all members (registry + PG). Groups remote PG members by node and sends one non-suspending `{:group_dispatch, pids, message}` per remote node (O(nodes), not O(members)). The target shard is chosen by `phash2(self(), num_shards)` for per-sender ordering. - -`dispatch_local/4` skips cross-node messaging. - -## ETS Match Spec Patterns - -- Use `{:==, :"$N", value}` for runtime variables in guards (e.g., filtering by node) -- **NOT** `{:const, value}` — it's invalid in ETS match specs -- Literal Elixir variables interpolate directly into match pattern tuple positions as exact-match filters -- For result bodies, runtime values can't be embedded directly — use `Enum.map` post-select - -## Distributed Test Patterns +{group, origin_node, origin_generation, shard, cluster, cluster_epoch} +``` -- `Group.TestCluster.start_peers(N)` starts N real Erlang nodes via OTP `:peer` -- All helpers (`spawn_register`, `spawn_join`, `spawn_monitor_forwarder`) use `:erpc.call` with compiled modules from `test/support/` -- `Node.spawn` with anonymous functions won't work — remote node needs the defining module's beam file -- `assert_eventually/2` polls with retries for async replication -- `flush_shards/2` sends a mailbox barrier through each shard so buffered sender - and receiver replication work is flushed too -- `assert_ets_consistent/1` verifies dual-index tables match -- Partition tests use 3 nodes (isolate 1 from other 2). 2-node partitions are unreliable because the test node bridges them. -- `Supervisor.start_link` links to caller — in RPC context, must `Process.unlink(pid)` or supervisor dies on RPC return -- `test_helper.exs`: starts EPMD when needed, calls `Node.start/2`, sets the cookie, and disables `prevent_overlapping_partitions` +The origin appends a strictly increasing sequence before applying the +materialized change. Generation fences a restarted Group instance. A +named-cluster epoch fences close/reopen. The nil cluster uses the origin +generation as its epoch. + +Shard 0 installs one exact node-wide authority snapshot: + +- origin generation; +- complete active cluster→epoch set; +- exact authority revision; and +- authenticated transport descriptor. + +Other shards exchange constant-size lane hellos. Shared authority is not enough +to accept data: each shard records an installed lane view only after it has +purged streams outside that authority. Exact and incrementally observed +revisions are distinct; heartbeats and partial cluster-control bursts can never +promote an incomplete epoch set to exact authority. + +`peer_connect` and `peer_connect_ack` are discovery hints, not authority. +The old `cluster_state` handler is receive-only rolling compatibility; new +replica recovery must use heads/deltas/exact snapshots and must not add new +dependencies on additive full-state merge. + +## Anti-Entropy + +Replica data frames are: + +- `heads`: stream, retained floor, and head; +- `delta_batch`: one or more contiguous stream runs; +- `need`: the receiver's next missing sequence; and +- `snapshot_chunk`: one byte-targeted part of an exact origin slice. + +A receiver advances its cursor only through a contiguous prefix. Duplicates are +idempotent; gaps request the missing suffix. Periodic heads recover a dropped +tail even if no later write occurs. If the requested sequence is below the +bounded oplog floor, the origin sends an exact snapshot containing only its own +claims and memberships for that shard/cluster stream. Absence is deletion. + +The oplog is bounded per shard and never waits for acknowledgements. There are +no leaders, quorums, replicated per-entry tombstones, known-member lists, or +retention barriers. A slow peer cannot pin memory. + +Snapshots are transport-neutral and byte-targeted (1 MiB by default). A single +oversized row is one chunk. Multi-chunk receivers stage rows in shard-owned +private ETS and preserve the old visible slice/cursor until the complete, +authority-valid manifest is present. Loss, duplication, reordering, +supersession, stale authority, expiry, and shard crash must leave no partial +visible state. Staging expires after one peer-lease interval without progress. + +## Nonblocking Transport + +All cross-node Group control sends use +`:erlang.send_nosuspend(..., [:noconnect])`. The default distribution replica +adapter sends directly the same way and adds no local hop. `:busy` and +`:disconnected` mean “drop this frame”; periodic anti-entropy repairs it. + +A sideband adapter may use one local `Group.Replica.Transport.Outbox` per +shard. Outboxes batch by peer, impose deadlines, and run bounded socket work +outside Group shards. Queue overflow, expiry, or socket backpressure drops the +batch. The included TCP adapter adds bounded per-peer writer queues and +capability-authenticated ingress while distribution still authenticates node +identity and carries authority. TCP is not encrypted. + +Transport ordering is not required for correctness. Per-shard ordered delivery +is a fast path; stream sequences reject duplicate/out-of-order data, and +generation/epoch/lane fences handle control/data reordering. Ingress must derive +`source_node` from the authenticated connection, never from payload data, and +must reassemble any transport segmentation before `deliver_batch/4`. + +## Registry Projection and Process Ownership + +Registry claims remain authoritative per origin even when hidden by another +origin's visible winner. Conflict resolution folds claims deterministically. +The configured callback chooses a pid (or neither), but Group owns lifecycle +effects: each losing origin appends its own authoritative unregister and exits +only its local process with +`{:group_registry_conflict, key, winner_meta}`. + +A node never monitors or exits another node's member processes. Local shards +monitor locally owned registration/PG pids. Local `DOWN` appends ordered +unregister/leave mutations before deletion and replication. Remote owner death +arrives through those records; `nodedown` or lease expiry is the fallback for +an origin that cannot emit them. + +## Peer and Cluster Lifecycle + +- Dist-Erlang `nodedown` immediately purges that node's visible rows, claims, + cursors, authority, cluster routing, and sideband session on every shard. +- Constant-size control heartbeats cover the case where the Erlang node remains + connected but its Group instance disappears. Peer-lease expiry performs the + same complete purge. +- Discovery probes continue after expiry. A returning current/new generation + is fenced, installs authority per lane, and reconstructs through deltas or an + exact snapshot. +- Incremental named-cluster open/close controls are generation fenced and + batched. A quiet exact hello repairs dropped/reordered control messages. +- Local cluster close uses a temporary all-shard completion barrier. The last + shard removes routing/epoch rows; restart repair completes abandoned closes, + and reconnect waits so an old close cannot erase new writes. + +TTL leases are local policy only. On expiry, Group disconnects a named cluster +only when no local registrations, PG memberships, or cluster monitors remain. + +## Batching and Fairness + +Registry and PG mutations share one outbound sender buffer so local mailbox +order is retained. It flushes on size, age, idle timer, and before control or +routing barriers. Receivers apply contiguous runs in bulk and emit lifecycle +events in operation batches. After replicated work, each shard takes a bounded +FIFO local-request turn to prevent replica pressure from starving callers. + +## Distributed Test Rules + +- Use three peers for partition/recovery tests. Two nodes cannot cover an + independent survivor while an origin and receiver disagree. +- Remote helpers must be compiled under `test/support/`; call them with MFA + through `:erpc`. +- Unlink supervisors started inside RPC helpers. +- `flush_shards/2` is only a mailbox/barrier aid; convergence assertions must + still wait for AE and inspect exact public/internal state. +- Always assert dead owners are absent, retained owners are alive, claims and + projections agree, cursors are contiguous, no partial snapshot remains, and + retired origins have no rows. ## Critical Invariants -1. **purge_cluster_node is unconditional** — every shard calls it on nodedown/DOWN, not just shard 0. Late peer_connect on non-zero shard can re-add dead node after shard 0 cleaned it. -2. **Dispatch :unregistered for evicted local pid** in "remote wins" branch of resolve_conflict — monitors need to see the eviction. -3. **merge_remote_cluster_data uses Enum.reduce** (not `for`) to thread `{state, events}` through, since resolve_conflict modifies `state.monitors`. -4. **Additive merge only** — cluster_state merge inserts but never deletes. Local named-cluster disconnect therefore purges the complete local cluster view, and replicated batches are membership-gated at apply time. -5. **PG tables have no overwrite conflicts** — `pg_by_key` key includes pid: `{cluster, key, pid}`. -6. **Named-cluster data is membership-gated** — both `cluster_state` and buffered replicated operations reject data for clusters the local node has left. +1. A cursor never advances across a gap or before a full exact snapshot commits. +2. Exact snapshots replace one origin slice; they are never additive merges. +3. Authority requires generation, exact epoch revision, and installed lane + readiness. Observed heartbeats/controls are not exact authority. +4. A stale generation, epoch, lane, shard, transitive pid, or unauthenticated + source is rejected before applying replica data. +5. Registry claims are retained per origin until that origin deletes them or is + retired; the visible winner is reconstructible from remaining claims. +6. Only an owner node monitors, retires, or exits its member processes. +7. Oplog pruning is local and bounded; lagging peers use exact snapshot repair. +8. `nodedown` and peer-lease expiry purge every public and internal reference + to the retired origin. Remote shard death cannot leave state permanently; + lease expiry or fenced rediscovery completes cleanup/recovery. +9. Snapshot staging is private, all-or-nothing, authority fenced, and expiring. +10. Local append/journal repair makes an appended mutation either replayable or + durably applied after a shard crash. +11. Cross-node control and replica calls never block a Group shard. +12. Cluster close completion survives caller timeout and shard restart. ## Logging -- `log:` option: `:info` (default), `false` (routine logs disabled), `:verbose` (all shards) -- `log/2` (normal), `log_verbose/2` (verbose only), `log_once/2` (shard 0 only) use `Logger.info`; `:verbose` is Group's own flag, not a Logger level -- Registry conflicts are unconditional `Logger.error` events; busy distribution links are unconditional `Logger.warning` events -- Runtime change: `Group.log_level(name, level)` +`log: :info | :verbose | false` controls routine Group logs and can be changed +with `Group.log_level/2`. Registry conflicts remain errors and busy dispatch +links remain warnings regardless of the routine level. diff --git a/README.md b/README.md index 00ea74c..8a00c32 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,8 @@ lifecycle monitoring, and isolated subclusters for Elixir. No external dependenc where only connected nodes participate. - **Sharded writes** — writes fan out across N GenServer shards to reduce contention. Reads go directly to ETS. +- **Nonblocking anti-entropy** — replica sends never wait on a remote socket; + sequenced deltas, bounded oplogs, and exact snapshots repair dropped work. ## Installation @@ -218,6 +220,9 @@ All operations are **eventually consistent**: - When connectivity returns, per-origin stream heads repair missing sequence ranges from a bounded oplog; a lag beyond the retained prefix falls back to an exact snapshot of that origin's shard/cluster slice. +- A dist-Erlang `nodedown` removes that node's view immediately. If the Erlang + node remains connected but its Group instance stops responding, a bounded + control-plane lease removes the same state and later discovery can rebuild it. - Registry conflicts (same key registered on two nodes during a partition) can be resolved with a configurable `resolve_registry_conflict` callback. The callback selects a winner; each origin retires and terminates only its own @@ -349,8 +354,14 @@ Each shard has materialized read indexes plus authority/recovery indexes: |---|---|---|---| | `reg_by_key` | `:set` | `{cluster, key}` | Registry lookup — O(1) | | `reg_by_pid` | `:ordered_set` | `{pid, cluster, key}` | Reverse index for death cleanup | +| `reg_claim_by_key` | `:ordered_set` | `{cluster, key, origin, generation, epoch}` | One authoritative registry claim per origin | +| `reg_claim_by_pid` | `:ordered_set` | `{pid, cluster, key, origin, generation, epoch}` | Reverse claim index for owner death and repair | | `pg_by_key` | `:ordered_set` | `{cluster, key, pid}` | Group membership lookup | | `pg_by_pid` | `:ordered_set` | `{pid, cluster, key}` | Reverse index for death cleanup | +| `replica_stream_meta` | `:set` | `stream_id` | Local stream head, retained floor, and applied journal position | +| `replica_oplog` | `:ordered_set` | `{stream_id, sequence}` | Bounded sequenced mutation records | +| `replica_oplog_order` | `:ordered_set` | `append_id` | Shard-wide pruning order across streams | +| `replica_cursor` | `:set` | `stream_id` | Highest contiguous remote sequence applied | Registry claim tables retain one authoritative claim per origin independently of the visible winner. Stream metadata, oplog, append-order, and receive-cursor @@ -358,15 +369,23 @@ tables support crash replay and gap repair. Keeping claims separate from the single visible `reg_by_key` projection prevents a losing-but-still-live remote claim from being forgotten before its owner emits an authoritative delete. -Plus 3 shared tables: +The node also has shared control/authority tables: - `cluster_nodes` (`:bag`, cluster→nodes) - `node_clusters` (`:bag`, node→clusters) - `cluster_leases` (`:set`, cluster→`{ttl_ms, expires_at}`) for local `connect(..., ttl: ms)` policy - -`cluster_nodes` / `node_clusters` remain the authoritative cluster-membership -tables. `cluster_leases` is only local lease metadata used by the sweeper. +- `replication_meta` (`:set`) for the local generation, authority revisions, + per-lane installed views, journal metadata, and one atomic append counter per + shard +- `local_cluster_epochs` and `closed_local_cluster_epochs` (`:set`) for active + and closing local named-cluster lifetimes +- `remote_cluster_epochs` (`:set`) for exact generation-fenced remote authority + +`cluster_nodes` / `node_clusters` are the routing projection read by APIs and +replication fanout. Generation-fenced local/remote epoch tables are the +authority used to install that projection. `cluster_leases` is only local +policy metadata used by the sweeper. `Group.Replica.Data` owns all tables and is supervised with `rest_for_one` so tables survive shard crashes. @@ -438,9 +457,10 @@ including after a caller timeout or shard restart. Reconnect waits for that barrier so a prior close cannot erase newly accepted writes. The sender flush timer is mainly a fallback for idle periods. The unified -outbound buffer also flushes immediately when it hits the configured size, when a new enqueue -finds the buffer already past its flush interval, and before control or -routing work such as cluster connect/disconnect or peer-protocol handling. +outbound buffer also flushes immediately when it hits the configured size, +when a new enqueue finds the buffer already past its flush interval, and before +control or routing work such as cluster connect/disconnect or peer-protocol +handling. Transport ordering is not required for correctness: each shard serializes writes, each stream numbers them, and receivers reject gaps and duplicates. @@ -509,27 +529,34 @@ no longer care about a cluster. ### Process Death Cleanup -Shards monitor all registered/joined processes. On `DOWN`, the shard: +Shards monitor only locally owned registered/joined processes. A node never +monitors or exits another node's member processes. On a local owner `DOWN`, the +shard: 1. Removes entries from both the primary and reverse-index ETS tables. 2. Appends authoritative unregister/leave mutations before deleting the rows, then sends one non-suspending sequenced delta batch per peer. 3. Fires `:unregistered` / `:left` events to local monitors. -### Node Disconnect +### Peer Removal and Recovery On `nodedown`, each shard purges all entries owned by the disconnected node -from its ETS tables and fires events for each removed entry. +from its ETS tables, claims, cursors, and authority indexes and fires events for +each removed entry. If dist Erlang stays connected but a Group instance or its +control lane disappears, heartbeat lease expiry performs the same complete +purge. Discovery probes continue after expiry; a returning instance announces +a new or current generation and anti-entropy reconstructs its live state. ## Testing ```bash mix test +mix test.soak # nightly/release qualification ``` -See [`test/README.md`](test/README.md) for details on the distributed test -infrastructure, shrinkable StreamData lifecycle-model tests, and the bounded -TLA+ anti-entropy model. +See [`test/README.md`](test/README.md) for the every-PR gate, shrinkable +StreamData lifecycle-model tests, bounded TLA+ models, and the nightly +three-node Jepsen transport/lifecycle campaign. ## Benchmarks diff --git a/lib/group.ex b/lib/group.ex index e3dd976..0e949e8 100644 --- a/lib/group.ex +++ b/lib/group.ex @@ -17,9 +17,11 @@ defmodule Group do - Writes (register, join, etc.) return immediately after local update - Other nodes receive updates asynchronously over the replica transport - During network partitions, nodes may have divergent views - - When partitions heal, conflicts are resolved. The built-in resolver kills - each losing origin records an authoritative delete and terminates only its - own local process with `{:group_registry_conflict, key, winner_meta}` + - When connectivity heals, stream gaps are repaired from a bounded oplog or + an exact per-origin snapshot; dropped replica sends are therefore safe + - Registry conflicts resolve deterministically. Each losing origin records an + authoritative delete and terminates only its own local process with + `{:group_registry_conflict, key, winner_meta}` ## Clusters @@ -170,6 +172,10 @@ defmodule Group do - **Memberships** are stored in replicated, sharded ETS indexes and are automatically cleaned up when member processes die. + + - **Process ownership is local**: a shard monitors and exits only processes + owned by its own node. Remote lifecycle changes arrive as sequenced replica + records or are removed by `nodedown`/peer-lease expiry. """ alias Group.Replica diff --git a/lib/group/replica/data.ex b/lib/group/replica/data.ex index 7511fc7..43e4d47 100644 --- a/lib/group/replica/data.ex +++ b/lib/group/replica/data.ex @@ -56,6 +56,24 @@ defmodule Group.Replica.Data do `entries_by_pid` can select all entries for a pid as a contiguous range scan. Also used by `maybe_demonitor` to check if a pid has any remaining entries (select with limit 1). + ### reg_claim_by_key / reg_claim_by_pid — authoritative registry claims + + {{cluster, key, origin, generation, epoch}, pid, meta, time, sequence} + {{pid, cluster, key, origin, generation, epoch}, meta, time, sequence} + + The visible `reg_by_key` row is only a deterministic projection. These tables retain one + independently versioned claim per origin so a losing-but-live claim is not forgotten. + Only its owner stream can delete it; conflict resolution then recomputes the visible winner. + + ### replica_stream_meta / replica_oplog / replica_oplog_order / replica_cursor + + Local streams are keyed by `{group, origin, generation, shard, cluster, epoch}`. + `replica_stream_meta` records each stream's head, retained floor, and applied journal + position. `replica_oplog` stores `{stream, sequence}` mutation records while + `replica_oplog_order` gives them one shard-wide append order for bounded pruning. + `replica_cursor` records only the highest contiguous sequence applied from each remote + stream. A gap below the retained floor is repaired by exact per-origin snapshot replacement. + ### cluster_nodes — `:bag`, keyed by cluster name {cluster, node} @@ -74,9 +92,9 @@ defmodule Group.Replica.Data do targeted deletes from both tables — O(clusters for node) instead of O(total entries). Both tables are shared across all shards. Used for the default cluster (nil) and named - clusters. The nil cluster is maintained by the peer_connect protocol — nodes are added on - peer discovery and removed on nodedown/shard death. `Group.nodes/1` reads nil cluster - from cluster_nodes. + clusters. Peer-connect messages are discovery hints; shard 0's generation-fenced exact + authority installs membership. `nodedown`, shard death, or peer-lease expiry removes it. + `Group.nodes/1` reads the nil cluster from cluster_nodes. ### cluster_leases — `:set`, keyed by cluster name @@ -94,6 +112,15 @@ defmodule Group.Replica.Data do Keeping leases separate avoids adding policy state to the hot cluster-membership lookups used by `Group.connect/3`, peer discovery, and replication fanout. + ### replication_meta and epoch tables + + `replication_meta` holds the local origin generation, exact and observed authority + revisions, per-shard installed remote views, journal metadata, and one append counter per + shard. `local_cluster_epochs` and `closed_local_cluster_epochs` fence local named-cluster + lifetimes; `remote_cluster_epochs` is the exact node-wide authority installed by shard 0. + Exact and merely observed revisions are separate so a partial control burst cannot be + promoted to authoritative membership. + ## Match Spec Patterns All match specs use `{:==, :"$N", value}` guards to filter on runtime values (e.g. node @@ -104,10 +131,12 @@ defmodule Group.Replica.Data do ## Bulk Operations & Their Costs - `purge_node/3`: Full table scan via `ets.select` filtering by node, then individual - deletes. O(table size) for the scan, but this only runs on nodedown — rare path. + deletes. O(table size) for the scan, but this only runs on nodedown, remote shard death, + or peer-lease expiry — rare paths. - `local_data_by_cluster/3`: Full table scan filtering by `node() == local_node`, - grouped by cluster. Only runs during discovery/sync protocol. + grouped by cluster. Retained only for the legacy receive-only cluster-state + compatibility path; current recovery uses anti-entropy streams. - `registry_count`, `pg_count`, `pg_count_by_prefix`, `local_registry_count`, `local_pg_count`, `local_registry_present?`, `local_pg_present?`: Uses @@ -127,8 +156,11 @@ defmodule Group.Replica.Data do local processes that registered before the crash would be orphaned — nobody would monitor them, and their ETS entries would persist forever if they later died. - Remote data doesn't need this protection — the discovery protocol re-syncs everything - from remote nodes on restart. Only local process entries need the ETS scan. + Only locally owned member processes are monitored. Remote owner death arrives as a + sequenced delete from that owner; nodedown or peer-lease expiry removes the complete + remote origin if it cannot report the delete. Periodic anti-entropy then rebuilds a + returning origin from retained deltas or an exact snapshot. No node monitors or exits + another node's member processes. The `state.monitors` map also deduplicates: a pid registered under multiple keys in the same shard gets one monitor, not one per key. diff --git a/lib/group/replica/transport.ex b/lib/group/replica/transport.ex index 033acd6..b7f46c1 100644 --- a/lib/group/replica/transport.ex +++ b/lib/group/replica/transport.ex @@ -96,7 +96,15 @@ defmodule Group.Replica.Transport do end defmodule Group.Replica.Transport.Distribution do - @moduledoc false + @moduledoc """ + Default nonblocking replica transport over Erlang distribution. + + Frames are sent directly to the matching remote shard with + `:erlang.send_nosuspend/3` and `:noconnect`, so the caller never waits for a + busy distribution socket and never initiates a connection. A busy or absent + link returns `:busy`; Group drops that frame and repairs it through periodic + anti-entropy. + """ @behaviour Group.Replica.Transport alias Group.Replica diff --git a/mix.exs b/mix.exs index 44fb164..6adb847 100644 --- a/mix.exs +++ b/mix.exs @@ -10,8 +10,10 @@ defmodule Group.MixProject do version: @version, elixir: "~> 1.19", elixirc_paths: elixirc_paths(Mix.env()), + test_ignore_filters: [~r"^test/jepsen/", ~r"^test/mutation/"], start_permanent: Mix.env() == :prod, deps: deps(), + aliases: aliases(), package: package(), docs: docs(), name: "Group", @@ -32,6 +34,10 @@ defmodule Group.MixProject do ] end + def cli do + [preferred_envs: ["test.soak": :test]] + end + defp deps do [ {:ex_doc, "~> 0.30", only: :dev, runtime: false}, @@ -55,4 +61,14 @@ defmodule Group.MixProject do source_ref: "v#{@version}" ] end + + defp aliases do + [ + test: ["test", "cmd test/jepsen/checker.sh"], + "test.soak": [ + "test", + "cmd env GROUP_JEPSEN_SKIP_CHECKER=1 test/jepsen/campaign.sh" + ] + ] + end end diff --git a/test/README.md b/test/README.md index 16ce634..224f250 100644 --- a/test/README.md +++ b/test/README.md @@ -3,20 +3,29 @@ ## Running tests ```bash -mix test # all tests +mix test # every-PR ExUnit/property/chaos/checker gate +mix test.soak # nightly six-profile Jepsen campaign mix test test/group_test.exs # local only mix test test/distributed_test.exs # distributed only mix test test/replica_adversarial_test.exs # seeded transport chaos mix test test/replica_model_property_test.exs # shrinkable model-based histories +test/jepsen/run.sh # one OS-partition/restart Jepsen model test ``` +`mix test` preserves normal Mix test arguments while always running the pure +Jepsen lifecycle-checker qualification after ExUnit. It does not require +Docker. `mix test.soak` first runs that complete PR gate, then runs the +distribution/TCP/chaos × mixed/permanent Jepsen campaign. The soak defaults to +20 five-minute fault histories per combination and is intended for nightly and +release qualification rather than individual edits. + ## Test files | File | What it tests | |------|---------------| | `group_test.exs` | Single-node: register/unregister, join/leave, members, monitor/demonitor, named clusters, concurrent operations | | `distributed_test.exs` | Multi-node: replication, peer discovery, node disconnect cleanup, partition healing, conflict resolution, event ordering, rolling restarts, and adversarial replica-transport loss/busy/snapshot recovery | -| `replica_adversarial_test.exs` | Reproducible mixed-operation state machines: drops, busy returns, duplication, reordering, bounded delay, oplog pruning, conflicts, owner death, and named-cluster epoch churn, followed by exact convergence/dead-owner/internal-index checks | +| `replica_adversarial_test.exs` | Reproducible three-node mixed-operation state machines: drops, busy returns, duplication, reordering, bounded delay, oplog pruning, conflicts, owner death, and named-cluster epoch churn, followed by exact convergence/dead-owner/internal-index checks | | `replica_model_property_test.exs` | StreamData-generated and shrunk owner histories against an independent lifecycle oracle and scheduler-controlled replica transport | | `replica_snapshot_test.exs` | Pure byte partitioning and set-valued private-ETS snapshot staging | | `replica_snapshot_distributed_test.exs` | Real-node exact-snapshot loss, reorder, duplicate, conflicting retransmission, supersession, authority fencing, expiry, and shard-crash recovery | @@ -48,6 +57,18 @@ The independent TLA+ model and TLC configuration live in `test/formal/`. See [`formal/README.md`](formal/README.md) for its checked invariants, finite model bounds, and run command. +The Docker-backed Jepsen harness lives in [`jepsen/`](jepsen/). It drives +three independent BEAM containers through concurrent, multi-entry owner +lifecycles, named-cluster epoch churn, directed/full partitions, transport +session resets, and VM restarts. The same workload runs over distribution, +real sideband TCP, and a lossy/duplicating/reordering transport. After healing, +its independent oracle checks exact public views and the internal registry, +PG, claim, cluster, cursor, oplog, snapshot-staging, and retired-origin +invariants. Its permanent-retirement scenario proves eviction even when a peer +never returns. `test/jepsen/campaign.sh` runs the full profile/scenario matrix; +`test/jepsen/qualify.sh` mutation-tests the implementation and proves that the +live checker rejects injected faults. + ## How distribution works The test node starts as a named Erlang node in `test_helper.exs`: diff --git a/test/formal/GroupAntiEntropyExtended.cfg b/test/formal/GroupAntiEntropyExtended.cfg new file mode 100644 index 0000000..25d4ff5 --- /dev/null +++ b/test/formal/GroupAntiEntropyExtended.cfg @@ -0,0 +1,17 @@ +SPECIFICATION Spec + +CONSTANTS + Nodes = {n1, n2, n3} + Origins = {n1} + Keys = {k1, k2} + MaxSeq = 3 + OplogBound = 2 + MaxMessages = 1 + +INVARIANTS + TypeOK + BoundedJournal + CurrentReplicaIsAStreamPrefix + +PROPERTY + HealedConvergence diff --git a/test/formal/PeerEviction.cfg b/test/formal/PeerEviction.cfg new file mode 100644 index 0000000..3a3017a --- /dev/null +++ b/test/formal/PeerEviction.cfg @@ -0,0 +1,15 @@ +SPECIFICATION Spec + +CONSTANTS + Receivers = {r1, r2} + Keys = {k1, k2} + MaxGeneration = 2 + MaxMessages = 2 + +INVARIANTS + TypeOK + VisibleRowsMatchInstalledGeneration + NoRowsWithoutAuthority + +PROPERTY + HealedConvergence diff --git a/test/formal/PeerEviction.tla b/test/formal/PeerEviction.tla new file mode 100644 index 0000000..35b98ad --- /dev/null +++ b/test/formal/PeerEviction.tla @@ -0,0 +1,207 @@ +--------------------------- MODULE PeerEviction --------------------------- +EXTENDS Integers, FiniteSets, TLC + +(* +Finite model of the peer lease and reincarnation boundary. An origin may +disappear permanently or restart at a larger generation while arbitrary old +hello and snapshot messages remain in the network. A receiver may expire its +lease at any point. Once the faulting prefix ends, fair repair must either +install the current generation exactly or erase the permanently absent peer. +*) + +CONSTANTS Receivers, Keys, MaxGeneration, MaxMessages + +ASSUME /\ IsFiniteSet(Receivers) + /\ Cardinality(Receivers) >= 1 + /\ IsFiniteSet(Keys) + /\ Cardinality(Keys) >= 1 + /\ MaxGeneration >= 2 + /\ MaxMessages >= 1 + +Generations == 1..MaxGeneration +Token == Generations \X Keys + +HelloMessage == + [kind : {"hello"}, to : Receivers, wireGeneration : Generations, + wireUp : BOOLEAN] + +SnapshotMessage == + [kind : {"snapshot"}, to : Receivers, wireGeneration : Generations, + rows : SUBSET Token] + +Message == HelloMessage \union SnapshotMessage + +VARIABLES phase, up, generation, truth, authorityGeneration, authorityUp, view, messages + +vars == + <> + +Init == + /\ phase = "faulting" + /\ up = TRUE + /\ generation = 1 + /\ truth = {} + /\ authorityGeneration = [receiver \in Receivers |-> 0] + /\ authorityUp = [receiver \in Receivers |-> FALSE] + /\ view = [receiver \in Receivers |-> {}] + /\ messages = {} + +Write(key, present) == + /\ phase = "faulting" + /\ up + /\ IF present + THEN truth' = truth \union {<>} + ELSE truth' = truth \ {<>} + /\ UNCHANGED + <> + +Crash == + /\ phase = "faulting" + /\ up + /\ up' = FALSE + /\ truth' = {} + /\ UNCHANGED + <> + +Restart == + /\ phase = "faulting" + /\ ~up + /\ generation < MaxGeneration + /\ up' = TRUE + /\ generation' = generation + 1 + /\ truth' = {} + /\ UNCHANGED <> + +SendHello(receiver) == + /\ phase = "faulting" + /\ Cardinality(messages) < MaxMessages + /\ messages' = messages \union + {[kind |-> "hello", to |-> receiver, + wireGeneration |-> generation, wireUp |-> up]} + /\ UNCHANGED + <> + +SendSnapshot(receiver) == + /\ phase = "faulting" + /\ up + /\ Cardinality(messages) < MaxMessages + /\ messages' = messages \union + {[kind |-> "snapshot", to |-> receiver, + wireGeneration |-> generation, rows |-> truth]} + /\ UNCHANGED + <> + +DeliverHello(message) == + /\ message.kind = "hello" + /\ IF message.wireGeneration >= authorityGeneration[message.to] + THEN /\ authorityGeneration' = + [authorityGeneration EXCEPT ![message.to] = message.wireGeneration] + /\ authorityUp' = [authorityUp EXCEPT ![message.to] = message.wireUp] + /\ view' = + IF message.wireGeneration # authorityGeneration[message.to] + \/ ~message.wireUp + THEN [view EXCEPT ![message.to] = {}] + ELSE view + ELSE /\ UNCHANGED authorityGeneration + /\ UNCHANGED authorityUp + /\ UNCHANGED view + /\ UNCHANGED <> + +DeliverSnapshot(message) == + /\ message.kind = "snapshot" + /\ IF message.wireGeneration = authorityGeneration[message.to] + /\ authorityUp[message.to] + THEN view' = [view EXCEPT ![message.to] = message.rows] + ELSE UNCHANGED view + /\ UNCHANGED + <> + +Deliver(message) == + /\ phase = "faulting" + /\ message \in messages + /\ \/ DeliverHello(message) + \/ DeliverSnapshot(message) + +Drop(message) == + /\ phase = "faulting" + /\ message \in messages + /\ messages' = messages \ {message} + /\ UNCHANGED + <> + +ExpireLease(receiver) == + /\ phase = "faulting" + /\ authorityGeneration' = [authorityGeneration EXCEPT ![receiver] = 0] + /\ authorityUp' = [authorityUp EXCEPT ![receiver] = FALSE] + /\ view' = [view EXCEPT ![receiver] = {}] + /\ UNCHANGED <> + +Heal == + /\ phase = "faulting" + /\ phase' = "healed" + /\ UNCHANGED + <> + +Repair(receiver) == + /\ phase = "healed" + /\ IF up + THEN /\ authorityGeneration' = + [authorityGeneration EXCEPT ![receiver] = generation] + /\ authorityUp' = [authorityUp EXCEPT ![receiver] = TRUE] + /\ view' = [view EXCEPT ![receiver] = truth] + ELSE /\ authorityGeneration' = + [authorityGeneration EXCEPT ![receiver] = 0] + /\ authorityUp' = [authorityUp EXCEPT ![receiver] = FALSE] + /\ view' = [view EXCEPT ![receiver] = {}] + /\ UNCHANGED <> + +Next == + \/ \E key \in Keys, present \in BOOLEAN : Write(key, present) + \/ Crash + \/ Restart + \/ \E receiver \in Receivers : SendHello(receiver) + \/ \E receiver \in Receivers : SendSnapshot(receiver) + \/ \E message \in messages : Deliver(message) + \/ \E message \in messages : Drop(message) + \/ \E receiver \in Receivers : ExpireLease(receiver) + \/ Heal + \/ \E receiver \in Receivers : Repair(receiver) + +TypeOK == + /\ phase \in {"faulting", "healed"} + /\ up \in BOOLEAN + /\ generation \in Generations + /\ truth \subseteq Token + /\ authorityGeneration \in [Receivers -> 0..MaxGeneration] + /\ authorityUp \in [Receivers -> BOOLEAN] + /\ view \in [Receivers -> SUBSET Token] + /\ messages \subseteq Message + +VisibleRowsMatchInstalledGeneration == + \A receiver \in Receivers : + \A token \in view[receiver] : token[1] = authorityGeneration[receiver] + +NoRowsWithoutAuthority == + \A receiver \in Receivers : + (authorityGeneration[receiver] = 0 \/ ~authorityUp[receiver]) + => view[receiver] = {} + +Converged == + \A receiver \in Receivers : + IF up + THEN /\ authorityGeneration[receiver] = generation + /\ authorityUp[receiver] + /\ view[receiver] = truth + ELSE /\ authorityGeneration[receiver] = 0 + /\ ~authorityUp[receiver] + /\ view[receiver] = {} + +HealedConvergence == phase = "healed" ~> Converged + +Spec == + /\ Init + /\ [][Next]_vars + /\ \A receiver \in Receivers : WF_vars(Repair(receiver)) + +============================================================================= diff --git a/test/formal/README.md b/test/formal/README.md index 7cb743e..d34b5ea 100644 --- a/test/formal/README.md +++ b/test/formal/README.md @@ -17,6 +17,14 @@ receiver crashes. Its invariants require visible data and the cursor to remain at a previously committed exact state until every chunk of one valid snapshot is present; stale or mixed partial state can never become visible. +`PeerEviction.tla` isolates the lifecycle boundary for a peer which never +returns and for a later process using the same node name with a fresh +generation. During its finite faulty prefix it retains and reorders stale +hello and snapshot messages while leases expire. Authority consists of both a +generation and an active bit, so an inactive hello fences even a same-epoch +snapshot. After healing, fair repair must either install only the current +generation or erase every row and authority reference for the absent peer. + The default TLC configuration uses three nodes: one origin and two independent receivers. The origin has one key, a two-record stream, a one-record oplog, and the system retains one arbitrary network frame. This forces delta repair, @@ -41,6 +49,12 @@ TLA_JAR=/path/to/tla2tools.jar \ TLA_SPEC="$PWD/test/formal/SnapshotAssembly.tla" \ TLA_CONFIG="$PWD/test/formal/SnapshotAssembly.cfg" \ test/formal/check.sh + +# Run all default models +TLA_JAR=/path/to/tla2tools.jar test/formal/check_matrix.sh + +# Also run the larger two-key, three-sequence anti-entropy state space +TLA_JAR=/path/to/tla2tools.jar TLA_EXTENDED=1 test/formal/check_matrix.sh ``` `TLC_WORKERS` controls worker concurrency and defaults to 4. `TLA_CONFIG` can @@ -49,7 +63,9 @@ point at an alternate finite configuration. TLC proves the listed invariants and liveness property for the configured finite instance, not for arbitrary unbounded node and key sets. Larger models should be run periodically by increasing `Nodes`, `Origins`, `Keys`, `MaxSeq`, -`OplogBound`, and `MaxMessages`. +`OplogBound`, and `MaxMessages`. `check_matrix.sh` runs the protocol, snapshot +assembly, and peer-eviction models; set `TLA_EXTENDED=1` for the larger +anti-entropy configuration. The checked three-node default explores 1,835,826 states, finds 490,236 distinct states to a depth of 30, and completes in roughly 1 minute 40 seconds @@ -57,3 +73,11 @@ on the development machine used for the validation run. The snapshot-assembly model explores 15,681 states, finds 1,088 distinct states to a depth of 13, and completes in under a second on the same class of machine. + +The peer-eviction model explores 1,527,116 states, finds 238,120 distinct +states to a depth of 26, and completes in roughly 20 seconds on the development +machine used for validation. + +The extended two-key, three-sequence model explores 127,557,634 states, finds +32,238,304 distinct states to a depth of 34, and completes in roughly two hours +on the development machine used for validation. diff --git a/test/formal/check_matrix.sh b/test/formal/check_matrix.sh new file mode 100755 index 0000000..cd60e2e --- /dev/null +++ b/test/formal/check_matrix.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +: "${TLA_JAR:?TLA_JAR must point to tla2tools.jar}" + +run_check() { + local spec="$1" + local config="$2" + + TLA_SPEC="${script_dir}/${spec}.tla" \ + TLA_CONFIG="${script_dir}/${config}.cfg" \ + "${script_dir}/check.sh" +} + +run_check GroupAntiEntropy GroupAntiEntropy +run_check SnapshotAssembly SnapshotAssembly +run_check PeerEviction PeerEviction + +if [[ "${TLA_EXTENDED:-0}" == "1" ]]; then + run_check GroupAntiEntropy GroupAntiEntropyExtended +fi diff --git a/test/jepsen/.gitignore b/test/jepsen/.gitignore new file mode 100644 index 0000000..acbda9e --- /dev/null +++ b/test/jepsen/.gitignore @@ -0,0 +1,5 @@ +.cache/ +.lein-failures +.nrepl-port +store/ +target/ diff --git a/test/jepsen/Dockerfile.node b/test/jepsen/Dockerfile.node new file mode 100644 index 0000000..08a5f41 --- /dev/null +++ b/test/jepsen/Dockerfile.node @@ -0,0 +1,22 @@ +FROM elixir:1.19.5-otp-28-slim + +RUN apt-get update \ + && apt-get install --yes --no-install-recommends ca-certificates iptables procps \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /group + +ENV MIX_ENV=prod + +COPY mix.exs mix.lock ./ +COPY lib ./lib +COPY test/jepsen/node.exs ./test/jepsen/node.exs + +RUN mix local.hex --force \ + && mix deps.get --only prod \ + && mix compile + +COPY test/jepsen/entrypoint.sh /usr/local/bin/group-jepsen-node +RUN chmod +x /usr/local/bin/group-jepsen-node + +ENTRYPOINT ["/usr/local/bin/group-jepsen-node"] diff --git a/test/jepsen/README.md b/test/jepsen/README.md new file mode 100644 index 0000000..d543df9 --- /dev/null +++ b/test/jepsen/README.md @@ -0,0 +1,149 @@ +# Group Jepsen model test + +This harness drives real Group instances in three independent BEAM VMs and +checks their quiescent state against a separate process-lifecycle oracle. It +tests failures outside the deterministic in-VM scheduler used by the +StreamData suite. + +Each node has eight independent client drivers, so operations on different +owners can overlap on the same node. Owners can hold multiple registry and PG +entries in the root, `red`, and `blue` cluster epochs. During the bounded-fault +prefix Jepsen: + +- registers, unregisters, joins, leaves, and kills real owner processes; +- closes and recreates named-cluster epochs while old traffic is stranded; +- creates a three-node registry conflict and requires every loser to die; +- partitions the full Erlang mesh, or only selected directed replica lanes; +- exercises isolation, all-way partition, and asymmetric one-way loss; +- resets transport sessions and kills/restarts complete BEAM nodes; and +- uses a 16-entry oplog and 1 KiB snapshot target so repair crosses pruning + and multi-chunk exact-snapshot paths. + +The replica lane is selectable without changing the workload or checker: + +- `distribution` delegates to Group's production Erlang-distribution adapter; +- `tcp` uses Group's production sideband TCP adapter while Erlang distribution + remains the control plane; and +- `chaos` is a local per-shard outbox which deterministically drops, + duplicates, delays, and reorders replica frames. + +After faults stop, every surviving node reconnects and the harness takes two +terminal snapshots. The independent checker requires: + +- exact, identical public registry and PG views on every survivor; +- every live owner claim to be visible, and no dead owner token to remain; +- deterministic resolution of registry conflicts with no unexpected owner + deaths; +- the expected peer set, including complete removal of a permanently retired + node; +- consistent registry, PG, cluster, claim, cursor, oplog, and remote-authority + indexes inside every shard; +- no staged partial snapshot and no retained data for a retired origin; +- coverage of delta batches, snapshot fallback, multi-chunk assembly, and + registry conflict termination; +- two identical quiescent observations per node; and +- every acknowledged Group operation below the configured latency ceiling. + +The oracle lives in owner processes outside Group's ETS and replica indexes. +Unexpected deaths and low-volume qualification evidence such as registry +conflict termination are also written to container-local logs which survive a +BEAM restart. A restart changes the boot component of new owner tokens, so a +stale generation, delayed delta, partial snapshot, or orphaned registry/PG row +cannot masquerade as a current owner. Every history explicitly restarts one +node after the deterministic conflict prelude, proving the checker does not +mistake restart-sensitive instrumentation for missing protocol coverage. + +## Requirements + +- Docker with Compose v2 +- Java 21 or newer +- `curl` + +The runner downloads Leiningen 2.12.0 into the ignored `.cache/` directory and +uses Jepsen 0.3.13. It installs nothing globally. + +## Run + +From the repository root: + +```bash +test/jepsen/run.sh +``` + +The default is a 60-second mixed-lifecycle run over three nodes and six client +workers. Normal Jepsen options and Group-specific options can be supplied: + +```bash +test/jepsen/run.sh test \ + --no-ssh \ + --nodes n1,n2,n3 \ + --concurrency 3n \ + --time-limit 300 \ + --test-count 10 \ + --key-count 32 \ + --owner-count 128 \ + --fault-interval 2 \ + --recovery-time 15 \ + --transport tcp \ + --scenario permanent \ + --max-operation-latency-ms 2000 +``` + +`--transport` accepts `distribution`, `tcp`, or `chaos`. `--scenario mixed` +restarts every transiently killed node; `--scenario permanent` finally retires +`n1` and proves that `n2` and `n3` converge after purging all of its public and +internal state. + +Run the sustained matrix over all transport and lifecycle combinations with: + +```bash +test/jepsen/campaign.sh +``` + +Its defaults are 20 five-minute runs for each of six combinations. The +`GROUP_JEPSEN_CAMPAIGN_*` environment variables in the script control duration, +count, concurrency, keys, owners, and recovery time. + +Run mutation qualification plus live positive- and negative-checker tests: + +```bash +test/jepsen/qualify.sh +``` + +This must reject production mutations which remove generation fencing, gap +detection, exact snapshot replacement, complete snapshot assembly, periodic +repair, or retirement purging. It then verifies that a healthy live history is +accepted and deliberately injected owner-death and internal-index corruption +are rejected. + +Results and histories are written below `test/jepsen/store/`. Containers are +removed after a run. Set `GROUP_JEPSEN_KEEP_CONTAINERS=1` to retain them and +inspect logs with: + +```bash +docker compose -f test/jepsen/docker-compose.yml logs +``` + +The pure checker qualification tests do not require Docker: + +```bash +test/jepsen/checker.sh +``` + +At the repository root, `mix test` runs this pure checker after the complete +ExUnit, StreamData, and deterministic-chaos suite. `mix test.soak` runs that +same PR gate followed by `campaign.sh`. + +## Scope + +This checker verifies Group's eventual lifecycle contract, not +linearizability. While communication is unavailable, each side may serve its +local view and accept new owners. The requirement begins after the explicitly +bounded fault prefix: surviving peers must then always resolve to the exact +same lifecycle view, while a peer which never returns must be completely +evicted after its lease expires. + +The formal models prove the protocol for finite state spaces; StreamData +generates and shrinks scheduler-controlled histories inside real Group nodes; +this harness tests OS sockets, independent VMs, VM death, and real transport +adapters. None of these layers alone is treated as a proof of the others. diff --git a/test/jepsen/campaign.sh b/test/jepsen/campaign.sh new file mode 100755 index 0000000..63ffe54 --- /dev/null +++ b/test/jepsen/campaign.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo_dir="$(cd "${script_dir}/../.." && pwd)" + +time_limit="${GROUP_JEPSEN_CAMPAIGN_TIME:-300}" +test_count="${GROUP_JEPSEN_CAMPAIGN_COUNT:-20}" +concurrency="${GROUP_JEPSEN_CAMPAIGN_CONCURRENCY:-4n}" +owner_count="${GROUP_JEPSEN_CAMPAIGN_OWNERS:-128}" +key_count="${GROUP_JEPSEN_CAMPAIGN_KEYS:-32}" +recovery_time="${GROUP_JEPSEN_CAMPAIGN_RECOVERY:-15}" +artifact_dir="${GROUP_JEPSEN_CAMPAIGN_ARTIFACT_DIR:-}" + +cd "${repo_dir}" + +if [[ "${GROUP_JEPSEN_SKIP_CHECKER:-0}" != "1" ]]; then + "${script_dir}/checker.sh" +fi + +export GROUP_JEPSEN_SKIP_CHECKER=1 +mkdir -p "${script_dir}/.cache" + +if [[ -z "${artifact_dir}" ]]; then + artifact_dir="$(mktemp -d "${script_dir}/.cache/campaign.XXXXXX")" +else + mkdir -p "${artifact_dir}" +fi + +echo "Jepsen campaign artifacts: ${artifact_dir}" + +for transport in distribution tcp chaos; do + for scenario in mixed permanent; do + log="${artifact_dir}/${transport}-${scenario}.log" + echo "Starting ${transport}/${scenario}: ${test_count} histories x ${time_limit}s" + + if "${script_dir}/run.sh" test \ + --no-ssh \ + --nodes n1,n2,n3 \ + --concurrency "${concurrency}" \ + --time-limit "${time_limit}" \ + --test-count "${test_count}" \ + --key-count "${key_count}" \ + --owner-count "${owner_count}" \ + --fault-interval 2 \ + --recovery-time "${recovery_time}" \ + --transport "${transport}" \ + --scenario "${scenario}" >"${log}" 2>&1; then + valid_count="$(rg -c "Everything looks good" "${log}" || true)" + + if [[ "${valid_count}" != "${test_count}" ]]; then + echo "Failed ${transport}/${scenario}: expected ${test_count} valid histories, found ${valid_count:-0}" >&2 + tail -200 "${log}" >&2 + exit 1 + fi + + echo "Passed ${transport}/${scenario}: ${valid_count}/${test_count} valid histories" + else + status=$? + echo "Failed ${transport}/${scenario}; tail of ${log}:" >&2 + tail -200 "${log}" >&2 + exit "${status}" + fi + done +done + +echo "Jepsen campaign passed: ${artifact_dir}" diff --git a/test/jepsen/checker.sh b/test/jepsen/checker.sh new file mode 100755 index 0000000..336063f --- /dev/null +++ b/test/jepsen/checker.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +exec "${script_dir}/lein.sh" test diff --git a/test/jepsen/docker-compose.yml b/test/jepsen/docker-compose.yml new file mode 100644 index 0000000..e93b5bd --- /dev/null +++ b/test/jepsen/docker-compose.yml @@ -0,0 +1,49 @@ +services: + n1: + build: + context: ../.. + dockerfile: test/jepsen/Dockerfile.node + container_name: group-jepsen-n1 + hostname: n1 + cap_add: [NET_ADMIN] + environment: + GROUP_JEPSEN_NODE: n1 + GROUP_JEPSEN_PORT: 9080 + GROUP_JEPSEN_PEERS: group@n1,group@n2,group@n3 + GROUP_JEPSEN_TRANSPORT: ${GROUP_JEPSEN_TRANSPORT:-distribution} + ports: ["19081:9080"] + networks: [group] + + n2: + build: + context: ../.. + dockerfile: test/jepsen/Dockerfile.node + container_name: group-jepsen-n2 + hostname: n2 + cap_add: [NET_ADMIN] + environment: + GROUP_JEPSEN_NODE: n2 + GROUP_JEPSEN_PORT: 9080 + GROUP_JEPSEN_PEERS: group@n1,group@n2,group@n3 + GROUP_JEPSEN_TRANSPORT: ${GROUP_JEPSEN_TRANSPORT:-distribution} + ports: ["19082:9080"] + networks: [group] + + n3: + build: + context: ../.. + dockerfile: test/jepsen/Dockerfile.node + container_name: group-jepsen-n3 + hostname: n3 + cap_add: [NET_ADMIN] + environment: + GROUP_JEPSEN_NODE: n3 + GROUP_JEPSEN_PORT: 9080 + GROUP_JEPSEN_PEERS: group@n1,group@n2,group@n3 + GROUP_JEPSEN_TRANSPORT: ${GROUP_JEPSEN_TRANSPORT:-distribution} + ports: ["19083:9080"] + networks: [group] + +networks: + group: + name: group-jepsen-net diff --git a/test/jepsen/entrypoint.sh b/test/jepsen/entrypoint.sh new file mode 100755 index 0000000..8911254 --- /dev/null +++ b/test/jepsen/entrypoint.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${GROUP_JEPSEN_NODE:?GROUP_JEPSEN_NODE is required}" + +exec elixir \ + --sname group \ + --cookie group_jepsen \ + --erl "-kernel net_ticktime 2" \ + -S mix run --no-compile --no-deps-check test/jepsen/node.exs diff --git a/test/jepsen/lein.sh b/test/jepsen/lein.sh new file mode 100755 index 0000000..5c70bef --- /dev/null +++ b/test/jepsen/lein.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cache_dir="${script_dir}/.cache" +lein="${cache_dir}/lein" + +mkdir -p "${cache_dir}" + +if [[ ! -x "${lein}" ]]; then + curl --fail --silent --show-error --location \ + https://raw.githubusercontent.com/technomancy/leiningen/2.12.0/bin/lein \ + --output "${lein}" + chmod +x "${lein}" +fi + +export LEIN_HOME="${cache_dir}/lein-home" +cd "${script_dir}" + +exec "${lein}" "$@" diff --git a/test/jepsen/node.exs b/test/jepsen/node.exs new file mode 100644 index 0000000..76817fb --- /dev/null +++ b/test/jepsen/node.exs @@ -0,0 +1,1376 @@ +defmodule Group.Jepsen.Transport.Stats do + @moduledoc false + use GenServer + + @table :group_jepsen_transport_stats + @gate :group_jepsen_transport_gate + @persistent_event_log "/tmp/group-jepsen-persistent-events" + + def start_link(_opts), do: GenServer.start_link(__MODULE__, :ok, name: __MODULE__) + + def increment(event, amount \\ 1) do + :ets.update_counter(@table, event, {2, amount}, {event, 0}) + :ok + rescue + ArgumentError -> :ok + end + + def increment_persistent(event, amount \\ 1) do + increment(event, amount) + File.write!(@persistent_event_log, "#{event}\t#{amount}\n", [:append]) + :ok + end + + def observe_max(event, value) when is_integer(value) and value >= 0 do + current = :ets.update_counter(@table, event, {2, 0}, {event, 0}) + if value > current, do: :ets.insert(@table, {event, value}) + :ok + rescue + ArgumentError -> :ok + end + + def snapshot do + persisted = persistent_events() + + @table + |> :ets.tab2list() + |> Map.new() + |> Map.merge(persisted, fn _event, current, durable -> max(current, durable) end) + end + + def block(target_node), do: :ets.insert(@gate, {target_node}) + def unblock(target_node), do: :ets.delete(@gate, target_node) + def heal, do: :ets.delete_all_objects(@gate) + def blocked?(target_node), do: :ets.member(@gate, target_node) + + @impl true + def init(:ok) do + _table = :ets.new(@table, [:named_table, :public, :set, write_concurrency: true]) + _gate = :ets.new(@gate, [:named_table, :public, :set, read_concurrency: true]) + {:ok, %{}} + end + + defp persistent_events do + case File.read(@persistent_event_log) do + {:ok, contents} -> + contents + |> String.split("\n", trim: true) + |> Enum.reduce(%{}, fn line, events -> + case String.split(line, "\t", parts: 2) do + [event, amount] -> + Map.update( + events, + String.to_existing_atom(event), + String.to_integer(amount), + &(&1 + String.to_integer(amount)) + ) + + _invalid -> + events + end + end) + + {:error, :enoent} -> + %{} + end + end +end + +defmodule Group.Jepsen.Transport.Common do + @moduledoc false + alias Group.Jepsen.Transport.Stats + + def try_send(delegate, group, target_node, shard, frame, opts) do + record(frame) + + if Stats.blocked?(target_node) do + Stats.increment(:logical_drop) + :ok + else + result = delegate.try_send(group, target_node, shard, frame, opts) + Stats.increment(transport_result(result)) + observe_outbox(group, shard) + result + end + end + + def record({:snapshot_chunk, _version, _stream, _seq, _index, chunk_count, _, _, _, _}) do + Stats.increment(:snapshot_chunk) + + if chunk_count > 1 do + Stats.increment(:multi_chunk_snapshot_chunk) + end + end + + def record({:delta_batch, _version, _runs}), do: Stats.increment(:delta_batch) + def record(_frame), do: Stats.increment(:other_frame) + + defp transport_result(:ok), do: :transport_ok + defp transport_result(:busy), do: :transport_busy + defp transport_result(:disconnected), do: :transport_disconnected + + defp observe_outbox(group, shard) do + case Process.whereis(Group.Replica.Transport.Outbox.name(group, shard)) do + pid when is_pid(pid) -> + case Process.info(pid, :message_queue_len) do + {:message_queue_len, length} -> Stats.observe_max(:outbox_mailbox_peak, length) + _ -> :ok + end + + nil -> + :ok + end + end +end + +defmodule Group.Jepsen.Transport.Distribution do + @moduledoc false + @behaviour Group.Replica.Transport + + alias Group.Jepsen.Transport.{Common, Stats} + alias Group.Replica.Transport.Distribution, as: Delegate + + @impl true + def id, do: Delegate.id() + + @impl true + def descriptor(group, opts), do: Delegate.descriptor(group, opts) + + @impl true + def child_spec(opts), do: {Stats, opts} + + @impl true + def try_send(group, target_node, shard, frame, opts) do + Common.try_send(Delegate, group, target_node, shard, frame, opts) + end +end + +defmodule Group.Jepsen.Transport.TCP do + @moduledoc false + @behaviour Group.Replica.Transport + + alias Group.Jepsen.Transport.Common + alias Group.Replica.Transport.TCP, as: Delegate + + @impl true + def id, do: Delegate.id() + + @impl true + def descriptor(group, opts), do: Delegate.descriptor(group, opts) + + @impl true + def child_spec(opts) do + %{ + id: {__MODULE__, Keyword.fetch!(opts, :name)}, + start: {Group.Jepsen.Transport.TCP.Supervisor, :start_link, [opts]}, + type: :supervisor + } + end + + @impl true + def try_send(group, target_node, shard, frame, opts) do + Common.try_send(Delegate, group, target_node, shard, frame, opts) + end + + @impl true + def peer_up(group, remote_node, descriptor, opts), + do: Delegate.peer_up(group, remote_node, descriptor, opts) + + @impl true + def peer_down(group, remote_node, opts), do: Delegate.peer_down(group, remote_node, opts) +end + +defmodule Group.Jepsen.Transport.TCP.Supervisor do + @moduledoc false + use Supervisor + + def start_link(opts), do: Supervisor.start_link(__MODULE__, opts) + + @impl true + def init(opts) do + children = [ + {Group.Jepsen.Transport.Stats, opts}, + Group.Replica.Transport.TCP.child_spec(opts) + ] + + Supervisor.init(children, strategy: :one_for_one) + end +end + +defmodule Group.Jepsen.Transport.Chaos do + @moduledoc false + @behaviour Group.Replica.Transport + + alias Group.Jepsen.Transport.{Common, Stats} + + @impl true + def id, do: :group_jepsen_unordered_v1 + + @impl true + def descriptor(_group, _opts), do: :group_jepsen_unordered_v1 + + @impl true + def child_spec(opts) do + %{ + id: {__MODULE__, Keyword.fetch!(opts, :name)}, + start: {Group.Jepsen.Transport.Chaos.Supervisor, :start_link, [opts]}, + type: :supervisor + } + end + + @impl true + def try_send(group, target_node, shard, frame, _opts) do + Common.record(frame) + + if Stats.blocked?(target_node) do + Stats.increment(:logical_drop) + :ok + else + case Process.whereis(worker_name(group, shard)) do + pid when is_pid(pid) -> + send(pid, {:send, target_node, frame}) + :ok + + nil -> + :disconnected + end + end + end + + def worker_name(group, shard), do: :"#{group}_jepsen_chaos_#{shard}" +end + +defmodule Group.Jepsen.Transport.Chaos.Supervisor do + @moduledoc false + use Supervisor + + def start_link(opts), do: Supervisor.start_link(__MODULE__, opts) + + @impl true + def init(opts) do + group = Keyword.fetch!(opts, :name) + num_shards = Keyword.fetch!(opts, :num_shards) + + workers = + for shard <- 0..(num_shards - 1) do + %{ + id: {Group.Jepsen.Transport.Chaos.Worker, shard}, + start: {Group.Jepsen.Transport.Chaos.Worker, :start_link, [group, shard]} + } + end + + Supervisor.init([{Group.Jepsen.Transport.Stats, opts} | workers], strategy: :one_for_one) + end +end + +defmodule Group.Jepsen.Transport.Chaos.Worker do + @moduledoc false + use GenServer + + alias Group.Jepsen.Transport.{Chaos, Stats} + + def start_link(group, shard) do + GenServer.start_link(__MODULE__, {group, shard}, name: Chaos.worker_name(group, shard)) + end + + @impl true + def init({group, shard}), do: {:ok, %{group: group, shard: shard, counter: 0}} + + @impl true + def handle_info({:send, target_node, frame}, state) do + counter = state.counter + 1 + next = %{state | counter: counter} + + if rem(counter, 5) == 0 do + Stats.increment(:chaos_drop) + else + delay = rem(counter * 17, 41) + Process.send_after(self(), {:deliver, target_node, frame}, delay) + + if rem(counter, 7) == 0 do + Stats.increment(:chaos_duplicate) + Process.send_after(self(), {:deliver, target_node, frame}, rem(delay + 19, 47)) + end + + if delay > 0, do: Stats.increment(:chaos_delay) + end + + {:noreply, next} + end + + def handle_info({:deliver, target_node, frame}, state) do + destination = {Group.Replica.shard_name(state.group, state.shard), target_node} + message = {:group_replica_frame, node(), frame} + + case :erlang.send_nosuspend(destination, message, [:noconnect]) do + true -> Stats.increment(:chaos_delivered) + false -> Stats.increment(:chaos_busy) + end + + {:noreply, state} + end +end + +defmodule Group.Jepsen.Transport.Control do + @moduledoc false + + alias Group.Jepsen.Transport.Stats + + def profile do + case System.get_env("GROUP_JEPSEN_TRANSPORT", "distribution") do + "distribution" -> :distribution + "tcp" -> :tcp + "chaos" -> :chaos + other -> raise "unknown GROUP_JEPSEN_TRANSPORT #{inspect(other)}" + end + end + + def transport(node_id) do + case profile() do + :distribution -> + Group.Jepsen.Transport.Distribution + + :chaos -> + Group.Jepsen.Transport.Chaos + + :tcp -> + {:ok, advertised_ip} = :inet.getaddr(String.to_charlist(node_id), :inet) + + {Group.Jepsen.Transport.TCP, + [ + ip: {0, 0, 0, 0}, + advertised_ip: advertised_ip, + port: 10_000, + max_queue: 32, + connect_timeout: 100, + send_timeout: 100, + reconnect_interval: 25, + outbox_batch_size: 16, + outbox_batch_bytes: 65_536, + outbox_flush_interval: 1, + outbox_deadline: 100 + ]} + end + end + + def block(target_node) do + Stats.block(target_node) + maybe_disconnect(target_node) + :ok + end + + def unblock(target_node) do + Stats.unblock(target_node) + maybe_reconnect(target_node) + :ok + end + + def heal(peer_nodes) do + Stats.heal() + Enum.each(peer_nodes, &maybe_reconnect/1) + :ok + end + + def reset(target_node) do + maybe_disconnect(target_node) + Process.sleep(10) + maybe_reconnect(target_node) + :ok + end + + defp maybe_disconnect(target_node) do + if profile() == :tcp do + Group.Replica.Transport.TCP.disconnect_peer(:jepsen_group, target_node) + end + catch + :exit, _ -> :ok + end + + defp maybe_reconnect(target_node) do + if profile() == :tcp do + Group.Replica.Transport.TCP.reconnect_peer(:jepsen_group, target_node) + end + catch + :exit, _ -> :ok + end +end + +defmodule Group.Jepsen.ConflictResolver do + @moduledoc false + + def resolve(_name, _key, {pid1, meta1, _time1}, {pid2, meta2, _time2}) do + if rank(meta1) >= rank(meta2), do: pid1, else: pid2 + end + + defp rank(%{revision: revision, token: token}), do: {revision, token} + defp rank(_meta), do: {-1, ""} +end + +defmodule Group.Jepsen.Owner do + @moduledoc false + use GenServer + + def start(token), do: GenServer.start(__MODULE__, token) + + @impl true + def init(token), do: {:ok, %{token: token, registrations: %{}, memberships: %{}}} + + @impl true + def handle_call({:mutate, :register, cluster, key, revision}, _from, state) do + meta = %{token: state.token, revision: revision} + + case safe_group_call(fn -> + Group.register(:jepsen_group, registry_key(key), meta, cluster_opts(cluster)) + end) do + :ok -> + entry = %{cluster: cluster, key: key, revision: revision} + state = put_in(state.registrations[{cluster, key}], entry) + {:reply, {:ok, snapshot(state)}, state} + + {:error, reason} -> + {:reply, {:error, reason, snapshot(state)}, state} + end + end + + def handle_call({:mutate, :unregister, cluster, key, _revision}, _from, state) do + owner_key = {cluster, key} + + if Map.has_key?(state.registrations, owner_key) do + case safe_group_call(fn -> + Group.unregister(:jepsen_group, registry_key(key), cluster_opts(cluster)) + end) do + :ok -> + state = %{state | registrations: Map.delete(state.registrations, owner_key)} + {:reply, {:ok, snapshot(state)}, state} + + {:error, reason} -> + {:reply, {:error, reason, snapshot(state)}, state} + end + else + {:reply, {:error, :not_owned, snapshot(state)}, state} + end + end + + def handle_call({:mutate, :join, cluster, key, revision}, _from, state) do + meta = %{token: state.token, revision: revision} + + case safe_group_call(fn -> + Group.join(:jepsen_group, pg_key(key), meta, cluster_opts(cluster)) + end) do + :ok -> + entry = %{cluster: cluster, key: key, revision: revision} + state = put_in(state.memberships[{cluster, key}], entry) + {:reply, {:ok, snapshot(state)}, state} + + {:error, reason} -> + {:reply, {:error, reason, snapshot(state)}, state} + end + end + + def handle_call({:mutate, :leave, cluster, key, _revision}, _from, state) do + owner_key = {cluster, key} + + if Map.has_key?(state.memberships, owner_key) do + case safe_group_call(fn -> + Group.leave(:jepsen_group, pg_key(key), cluster_opts(cluster)) + end) do + :ok -> + state = %{state | memberships: Map.delete(state.memberships, owner_key)} + {:reply, {:ok, snapshot(state)}, state} + + {:error, reason} -> + {:reply, {:error, reason, snapshot(state)}, state} + end + else + {:reply, {:error, :not_owned, snapshot(state)}, state} + end + end + + def handle_call({:drop_cluster, cluster}, _from, state) do + registrations = drop_cluster(state.registrations, cluster) + memberships = drop_cluster(state.memberships, cluster) + state = %{state | registrations: registrations, memberships: memberships} + {:reply, :ok, state} + end + + def handle_call(:snapshot, _from, state), do: {:reply, snapshot(state), state} + + defp drop_cluster(entries, cluster) do + entries + |> Enum.reject(fn {{entry_cluster, _key}, _entry} -> entry_cluster == cluster end) + |> Map.new() + end + + defp snapshot(state) do + %{ + token: state.token, + registrations: state.registrations |> Map.values() |> sort_entries(), + memberships: state.memberships |> Map.values() |> sort_entries() + } + end + + defp sort_entries(entries), do: Enum.sort_by(entries, &{&1.cluster || "", &1.key}) + + defp safe_group_call(fun) do + fun.() + rescue + exception -> {:error, {:exception, Exception.message(exception)}} + catch + kind, reason -> {:error, {kind, reason}} + end + + defp cluster_opts(nil), do: [] + defp cluster_opts(cluster), do: [cluster: cluster] + defp registry_key(key), do: "jepsen/registry/#{key}" + defp pg_key(key), do: "jepsen/pg/#{key}" +end + +defmodule Group.Jepsen.Driver do + @moduledoc false + use GenServer + + alias Group.Jepsen.Transport.Stats + + @driver_count 8 + @unexpected_death_log "/tmp/group-jepsen-unexpected-deaths" + + def start_link(opts) do + index = Keyword.fetch!(opts, :index) + GenServer.start_link(__MODULE__, opts, name: name(index)) + end + + def child_specs(opts) do + for index <- 0..(@driver_count - 1) do + %{ + id: {__MODULE__, index}, + start: {__MODULE__, :start_link, [Keyword.put(opts, :index, index)]} + } + end + end + + def mutate(operation, logical_owner, cluster, key, revision) do + started_at = System.monotonic_time(:microsecond) + + response = + GenServer.call( + driver(logical_owner), + {:mutate, operation, logical_owner, cluster, key, revision}, + 10_000 + ) + + Map.put(response, :latency_us, System.monotonic_time(:microsecond) - started_at) + end + + def kill(logical_owner), do: GenServer.call(driver(logical_owner), {:kill, logical_owner}) + + def drop_cluster(cluster) do + Enum.each(names(), &GenServer.call(&1, {:drop_cluster, cluster}, 10_000)) + :ok + end + + def owner_snapshots do + names() + |> Enum.flat_map(&GenServer.call(&1, :owner_snapshots, 30_000)) + |> Enum.sort_by(& &1.token) + end + + def unexpected_deaths do + in_memory = Enum.flat_map(names(), &GenServer.call(&1, :unexpected_deaths, 30_000)) + + (in_memory ++ persisted_unexpected_deaths()) + |> Enum.uniq() + |> Enum.sort_by(& &1.token) + end + + @impl true + def init(opts) do + {:ok, + %{ + node_id: Keyword.fetch!(opts, :node_id), + boot_id: Keyword.fetch!(opts, :boot_id), + owners: %{}, + monitors: %{}, + incarnations: %{}, + unexpected_deaths: [] + }} + end + + @impl true + def handle_call({:mutate, operation, logical_owner, cluster, key, revision}, _from, state) do + {pid, state} = owner(state, logical_owner) + + try do + case GenServer.call(pid, {:mutate, operation, cluster, key, revision}, 8_000) do + {:ok, owner_state} -> + {:reply, %{status: :ok, owner: owner_state}, + put_owner_state(state, logical_owner, pid, owner_state)} + + {:error, reason, owner_state} -> + {:reply, %{status: :fail, error: inspect(reason), owner: owner_state}, + put_owner_state(state, logical_owner, pid, owner_state)} + end + catch + :exit, reason -> + {:reply, %{status: :unknown, error: inspect(reason)}, state} + end + end + + def handle_call({:kill, logical_owner}, _from, state) do + case Map.get(state.owners, logical_owner) do + nil -> + {:reply, %{status: :ok, killed: nil}, state} + + {pid, token, monitor_ref, _owner_state} -> + if Process.alive?(pid) do + Process.exit(pid, :kill) + Process.demonitor(monitor_ref, [:flush]) + + {:reply, %{status: :ok, killed: token}, + %{ + state + | owners: Map.delete(state.owners, logical_owner), + monitors: Map.delete(state.monitors, monitor_ref) + }} + else + {:reply, %{status: :unknown, error: "owner already dead"}, state} + end + end + end + + def handle_call({:drop_cluster, cluster}, _from, state) do + Enum.each(state.owners, fn {_logical_owner, {pid, _token, _monitor, _owner_state}} -> + if Process.alive?(pid), do: GenServer.call(pid, {:drop_cluster, cluster}, 10_000) + end) + + owners = + Map.new(state.owners, fn {logical_owner, {pid, token, monitor, _owner_state}} -> + owner_state = if Process.alive?(pid), do: GenServer.call(pid, :snapshot), else: nil + {logical_owner, {pid, token, monitor, owner_state}} + end) + + {:reply, :ok, %{state | owners: owners}} + end + + def handle_call(:owner_snapshots, _from, state) do + owners = + state.owners + |> Enum.flat_map(fn + {_logical_owner, {_pid, _token, _monitor_ref, nil}} -> [] + {_logical_owner, {_pid, _token, _monitor_ref, owner_state}} -> [owner_state] + end) + + {:reply, owners, state} + end + + def handle_call(:unexpected_deaths, _from, state) do + {:reply, state.unexpected_deaths, state} + end + + @impl true + def handle_info({:DOWN, monitor_ref, :process, _pid, reason}, state) do + case Map.pop(state.monitors, monitor_ref) do + {nil, monitors} -> + {:noreply, %{state | monitors: monitors}} + + {{logical_owner, token}, monitors} -> + owners = + case Map.get(state.owners, logical_owner) do + {_pid, ^token, ^monitor_ref, _owner_state} -> Map.delete(state.owners, logical_owner) + _newer_incarnation -> state.owners + end + + unexpected_deaths = + if match?({:group_registry_conflict, _key, _winner_meta}, reason) do + Stats.increment_persistent(:registry_conflict_death) + state.unexpected_deaths + else + death = %{token: token, reason: inspect(reason)} + :ok = persist_unexpected_death(death) + [death | state.unexpected_deaths] + end + + {:noreply, + %{state | owners: owners, monitors: monitors, unexpected_deaths: unexpected_deaths}} + end + end + + defp owner(state, logical_owner) do + case Map.get(state.owners, logical_owner) do + {pid, _token, _monitor_ref, _owner_state} when is_pid(pid) -> + if Process.alive?(pid), do: {pid, state}, else: start_owner(state, logical_owner) + + nil -> + start_owner(state, logical_owner) + end + end + + defp start_owner(state, logical_owner) do + incarnation = Map.get(state.incarnations, logical_owner, 0) + 1 + token = "#{state.node_id}/#{state.boot_id}/#{logical_owner}/#{incarnation}" + {:ok, pid} = Group.Jepsen.Owner.start(token) + monitor_ref = Process.monitor(pid) + owner_state = %{token: token, registrations: [], memberships: []} + + state = %{ + state + | owners: Map.put(state.owners, logical_owner, {pid, token, monitor_ref, owner_state}), + monitors: Map.put(state.monitors, monitor_ref, {logical_owner, token}), + incarnations: Map.put(state.incarnations, logical_owner, incarnation) + } + + {pid, state} + end + + defp put_owner_state(state, logical_owner, pid, owner_state) do + owners = + case Map.get(state.owners, logical_owner) do + {^pid, token, monitor_ref, _old_owner_state} -> + Map.put(state.owners, logical_owner, {pid, token, monitor_ref, owner_state}) + + _replaced_owner -> + state.owners + end + + %{state | owners: owners} + end + + defp names, do: Enum.map(0..(@driver_count - 1), &name/1) + defp driver(logical_owner), do: name(:erlang.phash2(logical_owner, @driver_count)) + defp name(index), do: :"group_jepsen_driver_#{index}" + + defp persist_unexpected_death(%{token: token, reason: reason}) do + File.write(@unexpected_death_log, token <> "\t" <> reason <> "\n", [:append]) + end + + defp persisted_unexpected_deaths do + case File.read(@unexpected_death_log) do + {:ok, contents} -> + contents + |> String.split("\n", trim: true) + |> Enum.flat_map(fn line -> + case String.split(line, "\t", parts: 2) do + [token, reason] -> [%{token: token, reason: reason}] + _invalid -> [] + end + end) + + {:error, :enoent} -> + [] + + {:error, reason} -> + [%{token: "ORACLE-READ-FAILURE", reason: inspect(reason)}] + end + end +end + +defmodule Group.Jepsen.Driver.Supervisor do + @moduledoc false + use Supervisor + + def start_link(opts), do: Supervisor.start_link(__MODULE__, opts, name: __MODULE__) + + @impl true + def init(opts), + do: Supervisor.init(Group.Jepsen.Driver.child_specs(opts), strategy: :one_for_one) +end + +defmodule Group.Jepsen.Cluster do + @moduledoc false + use GenServer + + def start_link(clusters), do: GenServer.start_link(__MODULE__, clusters, name: __MODULE__) + def connect(cluster), do: GenServer.call(__MODULE__, {:connect, cluster}, 60_000) + def disconnect(cluster), do: GenServer.call(__MODULE__, {:disconnect, cluster}, 60_000) + def connect_all, do: GenServer.call(__MODULE__, :connect_all, 60_000) + + @impl true + def init(clusters) do + :ok = Group.connect(:jepsen_group, clusters) + {:ok, %{clusters: clusters}} + end + + @impl true + def handle_call({:connect, cluster}, _from, state) do + result = Group.connect(:jepsen_group, cluster) + {:reply, result(result), state} + end + + def handle_call({:disconnect, cluster}, _from, state) do + result = Group.disconnect(:jepsen_group, cluster) + if result == :ok, do: Group.Jepsen.Driver.drop_cluster(cluster) + {:reply, result(result), state} + end + + def handle_call(:connect_all, _from, state) do + result = Group.connect(:jepsen_group, state.clusters) + {:reply, result(result), state} + end + + defp result(:ok), do: %{status: :ok} + defp result({:error, reason}), do: %{status: :fail, error: inspect(reason)} +end + +defmodule Group.Jepsen.Invariant do + @moduledoc false + + alias Group.Replica.{Data, Protocol} + + def snapshot(retired_nodes) do + config = Group.get_config(:jepsen_group) + shards = 0..(config.num_shards - 1) + + errors = + check("dual indexes", &assert_dual_indexes/0) ++ + check("registry claims", &assert_registry_claims/0) ++ + check("oplog", &assert_oplogs/0) ++ + check("cursor authority", &assert_cursor_authority/0) ++ + check("retired origins", fn -> assert_retired_origins(retired_nodes) end) + + staging_count = + Enum.reduce(shards, 0, fn shard, total -> + state = :sys.get_state(Group.Replica.shard_name(:jepsen_group, shard)) + total + map_size(state.snapshot_transfers) + end) + + oplog_entries = + Enum.reduce(shards, 0, fn shard, total -> + total + :ets.info(Data.replica_oplog_order_table(:jepsen_group, shard), :size) + end) + + %{ + healthy: errors == [] and staging_count == 0, + errors: errors, + snapshot_staging_count: staging_count, + oplog_entries: oplog_entries, + oplog_max_entries_per_shard: config.replicated_oplog_max_entries, + shard_mailbox_max: + mailbox_max(Enum.map(shards, &Group.Replica.shard_name(:jepsen_group, &1))), + outbox_mailbox_max: + mailbox_max(Enum.map(shards, &Group.Replica.Transport.Outbox.name(:jepsen_group, &1))), + total_memory_bytes: :erlang.memory(:total) + } + rescue + exception -> + %{ + healthy: false, + errors: ["invariant snapshot failed: #{Exception.message(exception)}"], + snapshot_staging_count: -1 + } + end + + defp check(label, fun) do + fun.() + [] + rescue + exception -> ["#{label}: #{Exception.message(exception)}"] + catch + kind, reason -> ["#{label}: #{inspect({kind, reason})}"] + end + + defp assert_dual_indexes do + shards(fn shard -> + reg_key = + Data.reg_by_key_table(:jepsen_group, shard) + |> :ets.tab2list() + |> MapSet.new(fn {{cluster, key}, pid, meta, time, origin} -> + {cluster, key, pid, meta, time, origin} + end) + + reg_pid = + Data.reg_by_pid_table(:jepsen_group, shard) + |> :ets.tab2list() + |> MapSet.new(fn {{pid, cluster, key}, meta, time, origin} -> + {cluster, key, pid, meta, time, origin} + end) + + pg_key = + Data.pg_by_key_table(:jepsen_group, shard) + |> :ets.tab2list() + |> MapSet.new(fn {{cluster, key, pid}, meta, time, origin} -> + {cluster, key, pid, meta, time, origin} + end) + + pg_pid = + Data.pg_by_pid_table(:jepsen_group, shard) + |> :ets.tab2list() + |> MapSet.new(fn {{pid, cluster, key}, meta, time, origin} -> + {cluster, key, pid, meta, time, origin} + end) + + assert_equal!(reg_key, reg_pid, "registry dual indexes shard #{shard}") + assert_equal!(pg_key, pg_pid, "PG dual indexes shard #{shard}") + end) + + cluster_nodes = + Data.cluster_nodes_table(:jepsen_group) + |> :ets.tab2list() + |> MapSet.new(fn {cluster, origin} -> {cluster, origin} end) + + node_clusters = + Data.node_clusters_table(:jepsen_group) + |> :ets.tab2list() + |> MapSet.new(fn {origin, cluster} -> {cluster, origin} end) + + assert_equal!(cluster_nodes, node_clusters, "cluster dual indexes") + end + + defp assert_registry_claims do + shards(fn shard -> + by_key = + Data.reg_claim_by_key_table(:jepsen_group, shard) + |> :ets.tab2list() + |> MapSet.new(fn + {{cluster, key, origin, generation, epoch}, pid, meta, time, seq} -> + {cluster, key, pid, meta, time, origin, generation, epoch, seq} + end) + + by_pid = + Data.reg_claim_by_pid_table(:jepsen_group, shard) + |> :ets.tab2list() + |> MapSet.new(fn + {{pid, cluster, key, origin, generation, epoch}, meta, time, seq} -> + {cluster, key, pid, meta, time, origin, generation, epoch, seq} + end) + + assert_equal!(by_key, by_pid, "registry claim indexes shard #{shard}") + + invalid_origin = + Enum.find(by_key, fn {_cluster, _key, pid, _meta, _time, origin, _gen, _epoch, _seq} -> + node(pid) != origin + end) + + if invalid_origin, do: raise("claim with invalid PID origin #{inspect(invalid_origin)}") + + authority = + MapSet.new(by_key, fn {cluster, key, pid, meta, time, origin, _gen, _epoch, _seq} -> + {cluster, key, pid, meta, time, origin} + end) + + visible = + Data.reg_by_key_table(:jepsen_group, shard) + |> :ets.tab2list() + |> MapSet.new(fn {{cluster, key}, pid, meta, time, origin} -> + {cluster, key, pid, meta, time, origin} + end) + + missing = MapSet.difference(visible, authority) + + if MapSet.size(missing) > 0, + do: raise("visible registry rows lack claims #{inspect(missing)}") + end) + end + + defp assert_oplogs do + max_entries = Group.get_config(:jepsen_group).replicated_oplog_max_entries + + shards(fn shard -> + oplog = + Data.replica_oplog_table(:jepsen_group, shard) + |> :ets.tab2list() + |> MapSet.new(fn {{stream, seq}, append_id, _mutations} -> {append_id, stream, seq} end) + + order = + Data.replica_oplog_order_table(:jepsen_group, shard) |> :ets.tab2list() |> MapSet.new() + + assert_equal!(oplog, order, "oplog/order shard #{shard}") + + if MapSet.size(order) > max_entries do + raise "oplog bound exceeded shard #{shard}: #{MapSet.size(order)} > #{max_entries}" + end + + Data.replica_stream_meta_table(:jepsen_group, shard) + |> :ets.tab2list() + |> Enum.each(fn {stream, head, floor, applied} -> + unless floor >= 1 and floor <= head + 1 and applied >= 0 and applied <= head do + raise "invalid stream bounds #{inspect({stream, head, floor, applied})}" + end + + retained = + oplog + |> Enum.filter(fn {_append, row_stream, _seq} -> row_stream == stream end) + |> Enum.map(&elem(&1, 2)) + |> Enum.sort() + + expected = if floor <= head, do: Enum.to_list(floor..head), else: [] + if retained != expected, do: raise("non-contiguous oplog #{inspect(stream)}") + end) + end) + end + + defp assert_cursor_authority do + shards(fn shard -> + Data.replica_cursor_table(:jepsen_group, shard) + |> :ets.tab2list() + |> Enum.each(fn {stream, seq} -> + origin = Protocol.stream_origin(stream) + cluster = Protocol.stream_cluster(stream) + + valid? = + Protocol.stream_name(stream) == :jepsen_group and + Protocol.stream_shard(stream) == shard and + origin != node() and + Protocol.stream_generation(stream) == Data.remote_generation(:jepsen_group, origin) and + Protocol.stream_epoch(stream) == + Data.remote_cluster_epoch(:jepsen_group, origin, cluster) and seq >= 0 + + unless valid?, do: raise("cursor lacks current authority #{inspect({stream, seq})}") + end) + end) + end + + defp assert_retired_origins(retired_nodes) do + Enum.each(retired_nodes, fn origin -> + shards(fn shard -> + claims = + Data.reg_claim_by_key_table(:jepsen_group, shard) + |> :ets.tab2list() + |> Enum.filter(fn {{_cluster, _key, row_origin, _gen, _epoch}, _, _, _, _} -> + row_origin == origin + end) + + registry = + Data.reg_by_key_table(:jepsen_group, shard) + |> :ets.tab2list() + |> Enum.filter(fn {_key, _pid, _meta, _time, row_origin} -> row_origin == origin end) + + pg = + Data.pg_by_key_table(:jepsen_group, shard) + |> :ets.tab2list() + |> Enum.filter(fn {_key, _meta, _time, row_origin} -> row_origin == origin end) + + cursors = + Data.replica_cursor_table(:jepsen_group, shard) + |> :ets.tab2list() + |> Enum.filter(fn {stream, _seq} -> Protocol.stream_origin(stream) == origin end) + + view = Data.remote_view_generation(:jepsen_group, shard, origin) + + unless claims == [] and registry == [] and pg == [] and cursors == [] and is_nil(view) do + raise "retired origin retained on shard #{shard}: #{inspect(origin)}" + end + end) + + unless is_nil(Data.remote_generation(:jepsen_group, origin)) and + Data.clusters_for_node(:jepsen_group, origin) == [] do + raise "retired origin retained shared authority: #{inspect(origin)}" + end + end) + end + + defp shards(fun) do + num_shards = Group.get_config(:jepsen_group).num_shards + Enum.each(0..(num_shards - 1), fun) + end + + defp assert_equal!(left, right, label) do + if left != right do + raise "#{label}: left-only=#{inspect(MapSet.difference(left, right))} " <> + "right-only=#{inspect(MapSet.difference(right, left))}" + end + end + + defp mailbox_max(names) do + names + |> Enum.map(fn name -> + case Process.whereis(name) do + pid when is_pid(pid) -> + case Process.info(pid, :message_queue_len) do + {:message_queue_len, length} -> length + _ -> 0 + end + + nil -> + 0 + end + end) + |> Enum.max(fn -> 0 end) + end +end + +defmodule Group.Jepsen.Snapshot do + @moduledoc false + + def capture(node_id, boot_id, key_count, clusters, retired_nodes) do + owners = Group.Jepsen.Driver.owner_snapshots() + + registry = + Map.new([nil | clusters], fn cluster -> + values = + Map.new(0..(key_count - 1), fn key -> + value = + case Group.lookup(:jepsen_group, registry_key(key), cluster_opts(cluster)) do + nil -> nil + {_pid, %{token: token}} -> token + {_pid, other} -> "INVALID:#{inspect(other)}" + end + + {key, value} + end) + + {cluster_name(cluster), values} + end) + + pg = + Map.new([nil | clusters], fn cluster -> + values = + Map.new(0..(key_count - 1), fn key -> + tokens = + :jepsen_group + |> Group.members(pg_key(key), cluster_opts(cluster)) + |> Enum.map(fn + {_pid, %{token: token}} -> token + {_pid, other} -> "INVALID:#{inspect(other)}" + end) + |> Enum.sort() + + {key, tokens} + end) + + {cluster_name(cluster), values} + end) + + %{ + status: :ok, + snapshot: %{ + node: node_id, + boot: boot_id, + peers: Group.nodes(:jepsen_group) |> Enum.map(&Atom.to_string/1) |> Enum.sort(), + owners: owners, + unexpected_deaths: Group.Jepsen.Driver.unexpected_deaths(), + transport_events: Group.Jepsen.Transport.Stats.snapshot(), + transport_profile: Group.Jepsen.Transport.Control.profile(), + internal: Group.Jepsen.Invariant.snapshot(retired_nodes), + registry: registry, + pg: pg + } + } + end + + defp cluster_name(nil), do: "root" + defp cluster_name(cluster), do: cluster + defp cluster_opts(nil), do: [] + defp cluster_opts(cluster), do: [cluster: cluster] + defp registry_key(key), do: "jepsen/registry/#{key}" + defp pg_key(key), do: "jepsen/pg/#{key}" +end + +defmodule Group.Jepsen.EDN do + @moduledoc false + + def encode(nil), do: "nil" + def encode(true), do: "true" + def encode(false), do: "false" + def encode(value) when is_integer(value), do: Integer.to_string(value) + def encode(value) when is_binary(value), do: inspect(value) + + def encode(value) when is_atom(value) do + ":" <> (value |> Atom.to_string() |> String.replace("_", "-")) + end + + def encode(value) when is_list(value) do + "[" <> Enum.map_join(value, " ", &encode/1) <> "]" + end + + def encode(%MapSet{} = value) do + "#" <> "{" <> (value |> Enum.sort() |> Enum.map_join(" ", &encode/1)) <> "}" + end + + def encode(value) when is_map(value) do + encoded = + value + |> Enum.map(fn {key, inner} -> {encode(key), encode(inner)} end) + |> Enum.sort_by(&elem(&1, 0)) + |> Enum.map_join(" ", fn {key, inner} -> key <> " " <> inner end) + + "{" <> encoded <> "}" + end +end + +defmodule Group.Jepsen.Wire do + @moduledoc false + + def serve(port, context) do + {:ok, listener} = + :gen_tcp.listen(port, [:binary, packet: 4, active: false, reuseaddr: true]) + + accept(listener, context) + end + + defp accept(listener, context) do + {:ok, socket} = :gen_tcp.accept(listener) + spawn(fn -> connection(socket, context) end) + accept(listener, context) + end + + defp connection(socket, context) do + case :gen_tcp.recv(socket, 0) do + {:ok, payload} -> + response = payload |> command(context) |> Group.Jepsen.EDN.encode() + :ok = :gen_tcp.send(socket, response) + connection(socket, context) + + {:error, _reason} -> + :gen_tcp.close(socket) + end + end + + defp command(payload, context) do + case String.split(payload, "\t") do + ["ping"] -> + %{status: :ok} + + ["ready", expected] -> + expected = String.to_integer(expected) + + if length(Group.nodes(:jepsen_group)) == expected - 1 do + %{status: :ok} + else + %{status: :retry, peers: length(Group.nodes(:jepsen_group))} + end + + ["mutate", operation, logical_owner, cluster, key, revision] -> + Group.Jepsen.Driver.mutate( + String.to_existing_atom(operation), + logical_owner, + parse_cluster(cluster), + String.to_integer(key), + String.to_integer(revision) + ) + + ["kill", logical_owner] -> + Group.Jepsen.Driver.kill(logical_owner) + + ["cluster", "connect", cluster] -> + Group.Jepsen.Cluster.connect(cluster) + + ["cluster", "disconnect", cluster] -> + Group.Jepsen.Cluster.disconnect(cluster) + + ["cluster", "connect-all"] -> + Group.Jepsen.Cluster.connect_all() + + ["transport", "block", target] -> + :ok = Group.Jepsen.Transport.Control.block(String.to_atom(target)) + %{status: :ok} + + ["transport", "unblock", target] -> + :ok = Group.Jepsen.Transport.Control.unblock(String.to_atom(target)) + %{status: :ok} + + ["transport", "heal"] -> + :ok = Group.Jepsen.Transport.Control.heal(context.peers) + %{status: :ok} + + ["transport", "reset", target] -> + :ok = Group.Jepsen.Transport.Control.reset(String.to_atom(target)) + %{status: :ok} + + ["snapshot", key_count, clusters, retired] -> + Group.Jepsen.Snapshot.capture( + context.node_id, + context.boot_id, + String.to_integer(key_count), + parse_list(clusters), + retired |> parse_list() |> Enum.map(&String.to_atom/1) + ) + + ["corrupt", mode] -> + corrupt(mode) + + other -> + %{status: :fail, error: "unknown command #{inspect(other)}"} + end + rescue + exception -> %{status: :unknown, error: Exception.message(exception)} + catch + kind, reason -> %{status: :unknown, error: inspect({kind, reason})} + end + + defp corrupt("unexpected-death") do + File.write!( + "/tmp/group-jepsen-unexpected-deaths", + "oracle-self-test\t:injected\n", + [:append] + ) + + %{status: :ok} + end + + defp corrupt("internal-index") do + table = Group.Replica.Data.reg_by_pid_table(:jepsen_group, 0) + :ets.insert(table, {{self(), nil, "jepsen/registry/corrupt"}, %{}, 0, node()}) + %{status: :ok} + end + + defp corrupt(other), do: %{status: :fail, error: "unknown corruption #{inspect(other)}"} + defp parse_cluster("root"), do: nil + defp parse_cluster(cluster), do: cluster + defp parse_list(""), do: [] + defp parse_list(value), do: String.split(value, ",", trim: true) +end + +defmodule Group.Jepsen.Main do + @moduledoc false + @clusters ["red", "blue"] + + def run(argv) do + {opts, _rest, []} = + OptionParser.parse(argv, + strict: [node: :string, port: :integer, peers: :string] + ) + + node_id = Keyword.get(opts, :node) || System.fetch_env!("GROUP_JEPSEN_NODE") + + port = + Keyword.get(opts, :port) || + System.get_env("GROUP_JEPSEN_PORT", "9080") |> String.to_integer() + + peers = + (Keyword.get(opts, :peers) || + System.get_env("GROUP_JEPSEN_PEERS", "group@n1,group@n2,group@n3")) + |> String.split(",", trim: true) + |> Enum.map(&String.to_atom/1) + + {:ok, _apps} = Application.ensure_all_started(:group) + + {:ok, group} = + Group.start_link( + name: :jepsen_group, + shards: 4, + log: false, + resolve_registry_conflict: {Group.Jepsen.ConflictResolver, :resolve, []}, + replica_transport: Group.Jepsen.Transport.Control.transport(node_id), + replicated_sender_buffer_size: 1, + replicated_oplog_max_entries: 16, + replicated_snapshot_chunk_target_bytes: 1_024, + replicated_anti_entropy_interval: 50, + replicated_peer_lease_timeout: 750 + ) + + Process.unlink(group) + boot_id = Base.encode16(:crypto.strong_rand_bytes(8), case: :lower) + + {:ok, _drivers} = + Group.Jepsen.Driver.Supervisor.start_link(node_id: node_id, boot_id: boot_id) + + {:ok, _cluster} = Group.Jepsen.Cluster.start_link(@clusters) + spawn_link(fn -> reconnect_loop(peers) end) + + Group.Jepsen.Wire.serve(port, %{ + node_id: node_id, + boot_id: boot_id, + peers: Enum.reject(peers, &(&1 == node())) + }) + end + + defp reconnect_loop(peers) do + Enum.each(peers, fn peer -> + if peer != node(), do: Node.connect(peer) + end) + + Process.sleep(100) + reconnect_loop(peers) + end +end + +Group.Jepsen.Main.run(System.argv()) diff --git a/test/jepsen/project.clj b/test/jepsen/project.clj new file mode 100644 index 0000000..05284a5 --- /dev/null +++ b/test/jepsen/project.clj @@ -0,0 +1,8 @@ +(defproject group-jepsen "0.1.0-SNAPSHOT" + :description "Jepsen lifecycle and convergence tests for Group" + :url "https://github.com/phoenixframework/group" + :license {:name "MIT"} + :dependencies [[org.clojure/clojure "1.12.4"] + [jepsen "0.3.13"]] + :main group.jepsen.core + :jvm-opts ["-Xmx4g" "-Djava.awt.headless=true" "-server"]) diff --git a/test/jepsen/qualify.sh b/test/jepsen/qualify.sh new file mode 100755 index 0000000..5de63c1 --- /dev/null +++ b/test/jepsen/qualify.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo_dir="$(cd "${script_dir}/../.." && pwd)" +artifact_dir="$(mktemp -d "${script_dir}/.cache/qualification.XXXXXX")" + +cd "${repo_dir}" + +mix run test/mutation/run.exs \ + accept_old_generation \ + advance_cursor_across_gap \ + registry_snapshot_is_additive \ + commit_incomplete_snapshot \ + disable_periodic_heads \ + skip_generation_purge + +run_jepsen() { + local expectation="$1" + local corruption="$2" + local log="${artifact_dir}/${expectation}-${corruption}.log" + local status=0 + + set +e + "${script_dir}/run.sh" test \ + --no-ssh \ + --nodes n1,n2,n3 \ + --concurrency 2n \ + --time-limit 6 \ + --fault-interval 1 \ + --recovery-time 5 \ + --transport distribution \ + --scenario mixed \ + --corruption "${corruption}" >"${log}" 2>&1 + status=$? + set -e + + if [[ "${expectation}" == "pass" ]] && [[ "${status}" -ne 0 ]]; then + echo "healthy Jepsen baseline failed; see ${log}" >&2 + return 1 + fi + + if [[ "${expectation}" == "fail" ]] && [[ "${status}" -eq 0 ]]; then + echo "Jepsen checker accepted corruption ${corruption}; see ${log}" >&2 + return 1 + fi + + echo "${expectation}: ${corruption} (${log})" +} + +run_jepsen pass none +run_jepsen fail unexpected-death +run_jepsen fail internal-index + +echo "mutation and live checker qualification passed" diff --git a/test/jepsen/run.sh b/test/jepsen/run.sh new file mode 100755 index 0000000..9ee7f33 --- /dev/null +++ b/test/jepsen/run.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +compose_file="${script_dir}/docker-compose.yml" + +if [[ "${GROUP_JEPSEN_SKIP_CHECKER:-0}" != "1" ]]; then + "${script_dir}/checker.sh" +fi + +cleanup() { + if [[ "${GROUP_JEPSEN_KEEP_CONTAINERS:-0}" != "1" ]]; then + docker compose --file "${compose_file}" down --volumes >/dev/null + fi +} + +trap cleanup EXIT + +transport="${GROUP_JEPSEN_TRANSPORT:-distribution}" +args=("$@") + +for ((index = 0; index < ${#args[@]}; index++)); do + if [[ "${args[index]}" == "--transport" ]] && ((index + 1 < ${#args[@]})); then + transport="${args[index + 1]}" + fi +done + +export GROUP_JEPSEN_TRANSPORT="${transport}" + +docker compose --file "${compose_file}" up --detach --build --force-recreate + +if [[ "$#" -eq 0 ]]; then + set -- test \ + --no-ssh \ + --nodes n1,n2,n3 \ + --concurrency 2n \ + --time-limit 60 \ + --transport "${transport}" +fi + +"${script_dir}/lein.sh" run -- "$@" diff --git a/test/jepsen/src/group/jepsen/client.clj b/test/jepsen/src/group/jepsen/client.clj new file mode 100644 index 0000000..e6de357 --- /dev/null +++ b/test/jepsen/src/group/jepsen/client.clj @@ -0,0 +1,132 @@ +(ns group.jepsen.client + (:require [clojure.edn :as edn] + [clojure.string :as str] + [group.jepsen.docker :as docker] + [jepsen.client :as client]) + (:import (java.io DataInputStream DataOutputStream) + (java.net InetSocketAddress Socket) + (java.nio.charset StandardCharsets))) + +(defn request! + ([node fields] (request! node fields 3000)) + ([node fields timeout-ms] + (with-open [socket (Socket.)] + (let [^String host "127.0.0.1"] + (.connect socket (InetSocketAddress. host (int (docker/port node))) 1000)) + (.setSoTimeout socket timeout-ms) + (let [payload (.getBytes (str/join "\t" fields) StandardCharsets/UTF_8) + out (DataOutputStream. (.getOutputStream socket)) + in (DataInputStream. (.getInputStream socket))] + (.writeInt out (alength payload)) + (.write out payload) + (.flush out) + (let [length (.readInt in) + response (byte-array length)] + (.readFully in response) + (edn/read-string (String. response StandardCharsets/UTF_8))))))) + +(defn wait-ready! + ([node expected] (wait-ready! node expected 30000)) + ([node expected timeout-ms] + (let [deadline (+ (System/currentTimeMillis) timeout-ms)] + (loop [] + (let [response (try + (request! node ["ready" (str expected)] 1000) + (catch Exception _ nil))] + (cond + (= :ok (:status response)) true + (< (System/currentTimeMillis) deadline) + (do (Thread/sleep 100) (recur)) + :else + (throw (ex-info "Group node did not become ready" + {:node node, :last-response response})))))))) + +(defn wait-listening! + ([node] (wait-listening! node 15000)) + ([node timeout-ms] + (let [deadline (+ (System/currentTimeMillis) timeout-ms)] + (loop [] + (if (try + (= :ok (:status (request! node ["ping"] 1000))) + (catch Exception _ false)) + true + (if (< (System/currentTimeMillis) deadline) + (do (Thread/sleep 100) (recur)) + (throw (ex-info "Group node did not start listening" {:node node})))))))) + +(defn cluster-field [cluster] + (if (nil? cluster) "root" cluster)) + +(defn command [op] + (let [logical-owner (str (or (get-in op [:value :owner]) (:process op))) + cluster (cluster-field (get-in op [:value :cluster]))] + (case (:f op) + :register + ["mutate" "register" logical-owner cluster + (str (get-in op [:value :key])) + (str (get-in op [:value :revision]))] + + :unregister + ["mutate" "unregister" logical-owner cluster + (str (get-in op [:value :key])) "0"] + + :join + ["mutate" "join" logical-owner cluster + (str (get-in op [:value :key])) + (str (get-in op [:value :revision]))] + + :leave + ["mutate" "leave" logical-owner cluster + (str (get-in op [:value :key])) "0"] + + :kill + ["kill" logical-owner] + + :cluster-connect + ["cluster" "connect" (get-in op [:value :cluster])] + + :cluster-disconnect + ["cluster" "disconnect" (get-in op [:value :cluster])] + + :connect-all + ["cluster" "connect-all"] + + :corrupt + ["corrupt" (name (get-in op [:value :mode]))] + + :snapshot + ["snapshot" + (str (get-in op [:value :key-count])) + (str/join "," (get-in op [:value :clusters])) + (->> (get-in op [:value :retired-nodes]) + (map #(str "group@" (name %))) + (str/join ","))]))) + +(defrecord GroupClient [node] + client/Client + (open! [this _test node] (assoc this :node (name node))) + (setup! [this _test] this) + + (invoke! [_this _test op] + (let [target (name (or (get-in op [:value :target]) node))] + (try + (let [response (request! target (command op) 10000) + value (if (= :snapshot (:f op)) + (:snapshot response) + {:request (:value op), :response response, :node target})] + (case (:status response) + :ok (assoc op :type :ok, :value value) + :fail (assoc op :type :fail, :value value, :error (:error response)) + (assoc op :type :info, :value value, :error (:error response)))) + (catch Exception exception + (assoc op :type :info + :error {:class (str (class exception)) + :message (.getMessage exception)}))))) + + (teardown! [this _test] this) + (close! [_this _test]) + + client/Reusable + (reusable? [_this _test] true)) + +(defn client [], (GroupClient. nil)) diff --git a/test/jepsen/src/group/jepsen/core.clj b/test/jepsen/src/group/jepsen/core.clj new file mode 100644 index 0000000..2db2c8c --- /dev/null +++ b/test/jepsen/src/group/jepsen/core.clj @@ -0,0 +1,187 @@ +(ns group.jepsen.core + (:gen-class) + (:require [group.jepsen.client :as group-client] + [group.jepsen.db :as group-db] + [group.jepsen.model :as model] + [group.jepsen.nemesis :as group-nemesis] + [jepsen.cli :as cli] + [jepsen.generator :as gen] + [jepsen.os :as os] + [jepsen.tests :as tests])) + +(def clusters ["red" "blue"]) + +(defn mutation [revision key-count owner-count] + (let [key (rand-int key-count) + owner (rand-int owner-count) + cluster (rand-nth [nil nil nil "red" "blue"]) + rev #(swap! revision inc)] + (case (long (rand-int 16)) + 0 {:f :kill, :value {:owner owner}} + 1 {:f :unregister, :value {:owner owner, :cluster cluster, :key key}} + 2 {:f :leave, :value {:owner owner, :cluster cluster, :key key}} + 3 {:f :cluster-disconnect, :value {:cluster (rand-nth clusters)}} + 4 {:f :cluster-connect, :value {:cluster (rand-nth clusters)}} + 5 {:f :join, :value {:owner owner, :cluster cluster, :key key, :revision (rev)}} + 6 {:f :join, :value {:owner owner, :cluster cluster, :key key, :revision (rev)}} + 7 {:f :join, :value {:owner owner, :cluster cluster, :key key, :revision (rev)}} + 8 {:f :join, :value {:owner owner, :cluster cluster, :key key, :revision (rev)}} + {:f :register, :value {:owner owner, :cluster cluster, :key key, :revision (rev)}}))) + +(defn fault-cycle [fault-interval] + (cycle [(gen/sleep fault-interval) + {:type :info, :f :replica-partition-start} + (gen/sleep fault-interval) + {:type :info, :f :replica-reset} + (gen/sleep fault-interval) + {:type :info, :f :kill-node} + (gen/sleep fault-interval) + {:type :info, :f :restart-node} + (gen/sleep fault-interval) + {:type :info, :f :replica-partition-stop} + (gen/sleep fault-interval) + {:type :info, :f :partition-start} + (gen/sleep fault-interval) + {:type :info, :f :partition-stop}])) + +(defn targeted [target f value] + {:f f, :value (assoc value :target target)}) + +(defn prelude [] + [(gen/log "Opening a three-way replica partition for deterministic conflict and epoch fencing") + (gen/nemesis {:type :info, :f :replica-partition-start, :value {:shape :all}}) + (gen/sleep 0.25) + (gen/clients + (gen/once + (targeted :n1 :register + {:owner "epoch-old", :cluster "red", :key 0, :revision 1000}))) + (gen/clients + (gen/once (targeted :n1 :cluster-disconnect {:cluster "red"}))) + (gen/clients + (gen/once (targeted :n1 :cluster-connect {:cluster "red"}))) + (gen/clients + (gen/once + (targeted :n1 :register + {:owner "epoch-new", :cluster "red", :key 0, :revision 1001}))) + (gen/log "Creating three independent claims for one root registry key") + (gen/clients + [(targeted :n1 :register {:owner "triple-n1", :cluster nil, :key 0, :revision 2001}) + (targeted :n2 :register {:owner "triple-n2", :cluster nil, :key 0, :revision 2002}) + (targeted :n3 :register {:owner "triple-n3", :cluster nil, :key 0, :revision 2003})]) + (gen/sleep 0.25) + (gen/nemesis {:type :info, :f :replica-partition-stop}) + (gen/sleep 1) + (gen/log "Restarting n2 after conflict resolution to prove durable qualification evidence") + (gen/nemesis {:type :info, :f :kill-node, :value {:node :n2}}) + (gen/nemesis {:type :info, :f :restart-node}) + (gen/sleep 1)]) + +(defn terminal-snapshot [opts] + {:f :snapshot + :value {:key-count (:key-count opts) + :clusters clusters + :retired-nodes (:retired-nodes opts)}}) + +(defn snapshot-round [opts] + (let [read (terminal-snapshot opts) + permanent? (= "permanent" (:scenario opts))] + (gen/clients + (gen/each-thread + (if permanent? + (gen/once read) + (gen/until-ok (repeat read))))))) + +(defn terminal-phases [opts] + (let [permanent? (= "permanent" (:scenario opts)) + corruption (keyword (:corruption opts))] + (cond-> + [(gen/log "Healing every fault and restarting transiently killed nodes") + (gen/nemesis {:type :info, :f :restart-node}) + (gen/nemesis {:type :info, :f :partition-stop}) + (gen/nemesis {:type :info, :f :replica-partition-stop}) + (gen/clients + (gen/each-thread + (gen/until-ok (repeat {:f :connect-all, :value {}})))) + (gen/sleep (:recovery-time opts))] + permanent? + (conj (gen/log "Retiring n1 permanently and waiting for complete eviction") + (gen/nemesis {:type :info, :f :retire-node, :value {:node :n1}}) + (gen/sleep (:recovery-time opts))) + + (not= :none corruption) + (conj (gen/log "Injecting a checker-qualification corruption") + (gen/clients + (gen/once + (targeted (first (:terminal-nodes opts)) :corrupt {:mode corruption})))) + + true + (conj (gen/log "Collecting first terminal model snapshot") + (snapshot-round opts) + (gen/sleep 1) + (gen/log "Collecting stable terminal model snapshot") + (snapshot-round opts))))) + +(defn workload [opts] + (let [revision (atom 3000) + active (->> (repeatedly #(mutation revision (:key-count opts) (:owner-count opts))) + (gen/stagger 0.005) + (gen/nemesis (fault-cycle (:fault-interval opts))) + (gen/time-limit (:time-limit opts)))] + (apply gen/phases (concat (prelude) [active] (terminal-phases opts))))) + +(defn group-test [opts] + (let [db (group-db/db) + permanent? (= "permanent" (:scenario opts)) + terminal-nodes (if permanent? (vec (rest (:nodes opts))) (:nodes opts)) + retired-nodes (if permanent? [(first (:nodes opts))] []) + opts (assoc opts + :clusters clusters + :terminal-nodes terminal-nodes + :retired-nodes retired-nodes)] + (merge tests/noop-test + opts + {:name (str "group lifecycle convergence (" (:transport opts) "/" + (:scenario opts) ")") + :os os/noop + :db db + :client (group-client/client) + :nemesis (group-nemesis/nemesis db) + :pure-generators true + :generator (workload opts) + :checker (model/checker)}))) + +(def cli-options + [[nil "--key-count NUMBER" "Number of keys in each cluster and data type" + :default 8 + :parse-fn #(Long/parseLong %) + :validate [pos? "Must be positive"]] + [nil "--fault-interval SECONDS" "Seconds between fault transitions" + :default 2 + :parse-fn #(Double/parseDouble %) + :validate [pos? "Must be positive"]] + [nil "--owner-count NUMBER" "Logical owner slots per node" + :default 32 + :parse-fn #(Long/parseLong %) + :validate [pos? "Must be positive"]] + [nil "--recovery-time SECONDS" "Fault-free convergence time before checking" + :default 8 + :parse-fn #(Long/parseLong %) + :validate [pos? "Must be positive"]] + [nil "--transport PROFILE" "Replica transport: distribution, tcp, or chaos" + :default "distribution" + :validate [#{"distribution" "tcp" "chaos"} "Unsupported transport"]] + [nil "--scenario SCENARIO" "Lifecycle scenario: mixed or permanent" + :default "mixed" + :validate [#{"mixed" "permanent"} "Unsupported scenario"]] + [nil "--corruption MODE" "Checker qualification: none, unexpected-death, internal-index" + :default "none" + :validate [#{"none" "unexpected-death" "internal-index"} "Unsupported corruption"]] + [nil "--max-operation-latency-ms MILLIS" "Maximum acknowledged Group call latency" + :default 2000 + :parse-fn #(Long/parseLong %) + :validate [pos? "Must be positive"]]]) + +(defn -main [& args] + (cli/run! + (cli/single-test-cmd {:test-fn group-test, :opt-spec cli-options}) + args)) diff --git a/test/jepsen/src/group/jepsen/db.clj b/test/jepsen/src/group/jepsen/db.clj new file mode 100644 index 0000000..702b41c --- /dev/null +++ b/test/jepsen/src/group/jepsen/db.clj @@ -0,0 +1,28 @@ +(ns group.jepsen.db + (:require [group.jepsen.client :as group-client] + [group.jepsen.docker :as docker] + [jepsen.db :as db])) + +(defrecord DockerDB [] + db/DB + (setup! [_this test node] + (docker/heal! (:nodes test)) + (docker/restart! node) + (docker/reset-oracle! node) + (group-client/wait-ready! node (count (:nodes test)))) + + (teardown! [_this test _node] + (docker/heal! (:nodes test))) + + db/Kill + (kill! [_this _test node] + (docker/stop! node)) + + (start! [_this _test node] + (docker/start! node) + (group-client/wait-listening! node)) + + db/LogFiles + (log-files [_this _test _node] [])) + +(defn db [], (DockerDB.)) diff --git a/test/jepsen/src/group/jepsen/docker.clj b/test/jepsen/src/group/jepsen/docker.clj new file mode 100644 index 0000000..6fc987f --- /dev/null +++ b/test/jepsen/src/group/jepsen/docker.clj @@ -0,0 +1,121 @@ +(ns group.jepsen.docker + (:require [clojure.java.shell :as shell] + [clojure.string :as str])) + +(def containers + {"n1" "group-jepsen-n1" + "n2" "group-jepsen-n2" + "n3" "group-jepsen-n3"}) + +(def ports + {"n1" 19081 + "n2" 19082 + "n3" 19083}) + +(def full-chain "GROUP_JEPSEN_FULL") +(def replica-chain "GROUP_JEPSEN_REPLICA") +(def replica-port 10000) + +(defn container [node] + (or (get containers (name node)) + (throw (ex-info "unknown Jepsen node" {:node node})))) + +(defn port [node] + (or (get ports (name node)) + (throw (ex-info "unknown Jepsen node" {:node node})))) + +(defn shell! + [& args] + (let [{:keys [exit out err]} (apply shell/sh args)] + (when-not (zero? exit) + (throw (ex-info "command failed" + {:command args, :exit exit, :out out, :err err}))) + (str/trim out))) + +(defn docker! + [& args] + (apply shell! "docker" args)) + +(defn running? [node] + (= "true" + (try + (docker! "inspect" "--format" "{{.State.Running}}" (container node)) + (catch Exception _ "false")))) + +(defn start! [node] + (docker! "start" (container node))) + +(defn stop! [node] + (when (running? node) + (docker! "stop" "--time" "0" (container node)))) + +(defn restart! [node] + (docker! "restart" "--time" "0" (container node))) + +(defn ip [node] + (docker! "inspect" + "--format" + "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}" + (container node))) + +(defn exec-sh! + [node script] + (docker! "exec" (container node) "sh" "-c" script)) + +(defn reset-oracle! [node] + (exec-sh! node + "rm -f /tmp/group-jepsen-unexpected-deaths /tmp/group-jepsen-persistent-events")) + +(defn ensure-firewall-chain! [node chain] + (exec-sh! + node + (str "iptables -N " chain " 2>/dev/null || true; " + "iptables -C INPUT -j " chain " 2>/dev/null || " + "iptables -I INPUT 1 -j " chain "; " + "iptables -C OUTPUT -j " chain " 2>/dev/null || " + "iptables -I OUTPUT 1 -j " chain))) + +(defn flush-chain! [node chain] + (when (running? node) + (ensure-firewall-chain! node chain) + (exec-sh! node (str "iptables -F " chain)))) + +(defn heal-full! [nodes] + (doseq [node nodes] + (flush-chain! node full-chain))) + +(defn heal-replica! [nodes] + (doseq [node nodes] + (flush-chain! node replica-chain))) + +(defn heal! [nodes] + (heal-full! nodes) + (heal-replica! nodes)) + +(defn isolate! + "Cuts one node off from every other DB node while preserving client traffic." + [nodes isolated] + (heal-full! nodes) + (let [ips (into {} (map (juxt identity ip) nodes))] + (doseq [node nodes + peer nodes + :when (and (not= node peer) + (or (= node isolated) (= peer isolated)))] + (exec-sh! + node + (str "iptables -A " full-chain + " -d " (get ips peer) " -j DROP; " + "iptables -A " full-chain + " -s " (get ips peer) " -j DROP"))))) + +(defn partition-replica! + "Drops only sideband TCP packets for the supplied directed node pairs." + [nodes edges] + (heal-replica! nodes) + (let [ips (into {} (map (juxt identity ip) nodes))] + (doseq [[source target] edges] + (exec-sh! + source + (str "iptables -A " replica-chain + " -p tcp -d " (get ips target) + " --dport " replica-port " -j DROP"))))) diff --git a/test/jepsen/src/group/jepsen/model.clj b/test/jepsen/src/group/jepsen/model.clj new file mode 100644 index 0000000..b0e782a --- /dev/null +++ b/test/jepsen/src/group/jepsen/model.clj @@ -0,0 +1,219 @@ +(ns group.jepsen.model + (:require [clojure.set :as set] + [jepsen.checker :as checker] + [jepsen.history :as history])) + +(defn successful-snapshots [history] + (->> history + (remove history/invoke?) + (filter #(and (= :snapshot (:f %)) (= :ok (:type %)))) + (sort-by :index))) + +(defn snapshots-by-node [history] + (reduce (fn [snapshots op] + (update snapshots (get-in op [:value :node]) (fnil conj []) (:value op))) + {} + (successful-snapshots history))) + +(defn latest-snapshots [history] + (->> (successful-snapshots history) + (reduce (fn [snapshots op] + (assoc snapshots (get-in op [:value :node]) (:value op))) + {}))) + +(defn spaces [test] + (cons "root" (:clusters test))) + +(defn empty-view [test empty-value] + (into {} + (for [cluster (spaces test)] + [cluster (zipmap (range (:key-count test)) (repeat empty-value))]))) + +(defn owner-entries [owner field] + (or (get owner field) [])) + +(defn expected-state [test snapshots] + (let [owners (->> snapshots vals (mapcat :owners) (map (juxt :token identity)) (into {})) + registry-candidates + (reduce (fn [by-key [_ owner]] + (reduce (fn [entries {:keys [cluster key]}] + (update entries [(or cluster "root") key] + (fnil conj #{}) (:token owner))) + by-key + (owner-entries owner :registrations))) + {} + owners) + conflicts (into {} (filter (comp #(< 1 %) count val) registry-candidates)) + registry + (reduce (fn [view [[cluster key] tokens]] + (assoc-in view [cluster key] (first tokens))) + (empty-view test nil) + registry-candidates) + pg + (reduce (fn [view [_ owner]] + (reduce (fn [entries {:keys [cluster key]}] + (update-in entries [(or cluster "root") key] + (fnil conj #{}) (:token owner))) + view + (owner-entries owner :memberships))) + (empty-view test #{}) + owners)] + {:owners owners, :registry registry, :pg pg, :conflicts conflicts})) + +(defn normalize-view [test snapshot] + {:registry + (into {} + (for [cluster (spaces test)] + [cluster + (into {} + (for [key (range (:key-count test))] + [key (get-in snapshot [:registry cluster key])]))])) + :pg + (into {} + (for [cluster (spaces test)] + [cluster + (into {} + (for [key (range (:key-count test))] + [key (set (get-in snapshot [:pg cluster key]))]))]))}) + +(defn stable-internal [snapshot] + (select-keys (:internal snapshot) + [:healthy :errors :snapshot-staging-count :oplog-entries])) + +(defn snapshot-fingerprint [test snapshot] + {:owners (set (:owners snapshot)) + :peers (set (:peers snapshot)) + :unexpected-deaths (set (:unexpected-deaths snapshot)) + :view (normalize-view test snapshot) + :internal (stable-internal snapshot)}) + +(defn operation-latencies [history] + (->> history + (remove history/invoke?) + (keep #(get-in % [:value :response :latency-us])))) + +(defn analyze [test history] + (let [observations (snapshots-by-node history) + snapshots (latest-snapshots history) + required-nodes (set (map name (or (:terminal-nodes test) (:nodes test)))) + relevant-observations (select-keys observations required-nodes) + relevant-snapshots (select-keys snapshots required-nodes) + missing-nodes (set/difference required-nodes (set (keys relevant-snapshots))) + minimum-observations (get test :terminal-snapshots-per-node 2) + insufficient-observations + (into {} + (keep (fn [node] + (let [observation-count (count (get relevant-observations node))] + (when (< observation-count minimum-observations) + [node observation-count])))) + required-nodes) + unstable-observations + (into {} + (keep (fn [[node node-observations]] + (let [fingerprints + (set (map #(snapshot-fingerprint test %) + node-observations))] + (when (< 1 (count fingerprints)) + [node fingerprints])))) + relevant-observations) + expected (expected-state test relevant-snapshots) + expected-view (select-keys expected [:registry :pg]) + views (into {} (map (fn [[node snapshot]] + [node (normalize-view test snapshot)])) + relevant-snapshots) + mismatches (into {} (remove (comp #(= expected-view %) val) views)) + peer-mismatches + (into {} + (keep (fn [[node snapshot]] + (let [expected-peers (->> required-nodes + (remove #(= node %)) + (map #(str "group@" %)) + set) + actual-peers (set (:peers snapshot))] + (when (not= expected-peers actual-peers) + [node {:expected expected-peers, :actual actual-peers}])))) + relevant-snapshots) + transport-events + (reduce #(merge-with + %1 %2) + {} + (map #(or (:transport-events %) {}) (vals relevant-snapshots))) + required-transport-events + (get test :required-transport-events + #{:delta-batch :snapshot-chunk :multi-chunk-snapshot-chunk + :registry-conflict-death}) + missing-transport-events + (set (remove #(pos? (get transport-events % 0)) required-transport-events)) + expected-profile (keyword (:transport test)) + transport-profile-mismatches + (into {} + (keep (fn [[node snapshot]] + (when (not= expected-profile (:transport-profile snapshot)) + [node (:transport-profile snapshot)]))) + relevant-snapshots) + internal-errors + (into {} + (keep (fn [[node snapshot]] + (let [internal (:internal snapshot)] + (when (or (not= true (:healthy internal)) + (seq (:errors internal)) + (not= 0 (:snapshot-staging-count internal))) + [node internal])))) + relevant-snapshots) + unexpected-deaths (->> relevant-snapshots vals (mapcat :unexpected-deaths) set) + live-tokens (set (keys (:owners expected))) + actual-tokens (->> views + vals + (mapcat (fn [{:keys [registry pg]}] + (concat + (->> registry vals (mapcat vals) (remove nil?)) + (->> pg vals (mapcat vals) (mapcat identity))))) + set) + expected-tokens + (set/union + (->> (:registry expected) vals (mapcat vals) (remove nil?) set) + (->> (:pg expected) vals (mapcat vals) (mapcat identity) set)) + orphaned (set/difference actual-tokens live-tokens) + missing-live (set/difference expected-tokens actual-tokens) + latencies (operation-latencies history) + max-latency-us (if (seq latencies) (apply max latencies) 0) + latency-limit-us (* 1000 (get test :max-operation-latency-ms 2000)) + latency-violation? (> max-latency-us latency-limit-us) + valid? (and (empty? missing-nodes) + (empty? insufficient-observations) + (empty? unstable-observations) + (empty? peer-mismatches) + (empty? missing-transport-events) + (empty? transport-profile-mismatches) + (empty? internal-errors) + (empty? (:conflicts expected)) + (empty? mismatches) + (empty? unexpected-deaths) + (empty? orphaned) + (empty? missing-live) + (not latency-violation?))] + {:valid? valid? + :snapshots (set (keys relevant-snapshots)) + :missing-nodes missing-nodes + :insufficient-terminal-observations insufficient-observations + :unstable-terminal-observations unstable-observations + :peer-mismatches peer-mismatches + :transport-events transport-events + :missing-transport-events missing-transport-events + :transport-profile-mismatches transport-profile-mismatches + :internal-invariant-errors internal-errors + :max-group-operation-latency-ms (/ max-latency-us 1000.0) + :group-operation-latency-limit-ms (/ latency-limit-us 1000.0) + :live-owner-count (count live-tokens) + :live-registry-conflicts (:conflicts expected) + :mismatched-views mismatches + :unexpected-owner-deaths unexpected-deaths + :orphaned-owner-tokens orphaned + :missing-live-owner-tokens missing-live + :expected expected-view})) + +(defrecord LifecycleChecker [] + checker/Checker + (check [_this test history _opts] + (analyze test history))) + +(defn checker [], (LifecycleChecker.)) diff --git a/test/jepsen/src/group/jepsen/nemesis.clj b/test/jepsen/src/group/jepsen/nemesis.clj new file mode 100644 index 0000000..e11c40f --- /dev/null +++ b/test/jepsen/src/group/jepsen/nemesis.clj @@ -0,0 +1,166 @@ +(ns group.jepsen.nemesis + (:require [group.jepsen.client :as group-client] + [group.jepsen.docker :as docker] + [jepsen.db :as db] + [jepsen.nemesis :as nemesis])) + +(defn ordered-pairs [nodes] + (for [source nodes, target nodes :when (not= source target)] [source target])) + +(defn partition-shape [nodes requested] + (let [nodes (vec nodes) + shape (or requested (rand-nth [:isolate :all :asymmetric]))] + (case shape + :all {:shape :all, :edges (vec (ordered-pairs nodes))} + :asymmetric + (let [source (rand-nth nodes) + target (rand-nth (vec (remove #(= source %) nodes)))] + {:shape :asymmetric, :edges [[source target]]}) + :isolate + (let [isolated (rand-nth nodes)] + {:shape :isolate + :isolated isolated + :edges (vec (filter (fn [[source target]] + (or (= source isolated) (= target isolated))) + (ordered-pairs nodes)))})))) + +(defn logical-block! [edges] + (doseq [[source target] edges] + (when (docker/running? source) + (group-client/request! source ["transport" "block" (str "group@" (name target))])))) + +(defn logical-heal! [nodes] + (doseq [node nodes] + (when (docker/running? node) + (try + (group-client/request! node ["transport" "heal"]) + (catch Exception _ nil))))) + +(defrecord PartitionNemesis [isolated] + nemesis/Nemesis + (setup! [this test] + (docker/heal-full! (:nodes test)) + this) + + (invoke! [_this test op] + (case (:f op) + :start + (if @isolated + (assoc op :type :info, :value {:already-isolated @isolated}) + (let [node (rand-nth (vec (:nodes test)))] + (docker/isolate! (:nodes test) node) + (reset! isolated node) + (assoc op :type :info, :value {:isolated node}))) + + :stop + (do + (docker/heal-full! (:nodes test)) + (let [node @isolated] + (reset! isolated nil) + (assoc op :type :info, :value {:healed node}))))) + + (teardown! [_this test] + (docker/heal-full! (:nodes test)))) + +(defrecord ReplicaNemesis [active] + nemesis/Nemesis + (setup! [this test] + (docker/heal-replica! (:nodes test)) + (logical-heal! (:nodes test)) + this) + + (invoke! [_this test op] + (case (:f op) + :start + (if @active + (assoc op :type :info, :value {:already-active @active}) + (let [requested (get-in op [:value :shape]) + fault (partition-shape (:nodes test) requested)] + (if (= "tcp" (:transport test)) + (docker/partition-replica! (:nodes test) (:edges fault)) + (logical-block! (:edges fault))) + (reset! active fault) + (assoc op :type :info, :value fault))) + + :stop + (do + (docker/heal-replica! (:nodes test)) + (logical-heal! (:nodes test)) + (let [fault @active] + (reset! active nil) + (assoc op :type :info, :value {:healed fault}))) + + :reset + (let [nodes (vec (filter docker/running? (:nodes test))) + source (when (seq nodes) (rand-nth nodes)) + targets (when source (vec (remove #(= source %) nodes))) + target (when (seq targets) (rand-nth targets))] + (when (and source target) + (group-client/request! + source + ["transport" "reset" (str "group@" (name target))])) + (assoc op :type :info, :value {:source source, :target target})))) + + (teardown! [_this test] + (docker/heal-replica! (:nodes test)) + (logical-heal! (:nodes test)))) + +(defrecord ProcessNemesis [db killed] + nemesis/Nemesis + (setup! [this _test] this) + + (invoke! [_this test op] + (case (:f op) + :start + (if @killed + (assoc op :type :info, :value {:already-killed @killed}) + (let [node (or (get-in op [:value :node]) + (rand-nth (vec (:nodes test))))] + (db/kill! db test node) + (reset! killed node) + (assoc op :type :info, :value {:killed node}))) + + :stop + (if-let [node @killed] + (do + (db/start! db test node) + (reset! killed nil) + (assoc op :type :info, :value {:restarted node})) + (assoc op :type :info, :value {:restarted nil})))) + + (teardown! [_this test] + (when-let [node @killed] + (db/start! db test node) + (reset! killed nil)))) + +(defrecord RetirementNemesis [db retired] + nemesis/Nemesis + (setup! [this _test] this) + + (invoke! [_this test op] + (let [node (or (get-in op [:value :node]) (first (:nodes test)))] + (when (and (nil? @retired) (docker/running? node)) + (db/kill! db test node) + (reset! retired node)) + (assoc op :type :info, :value {:retired @retired}))) + + (teardown! [_this test] + (when-let [node @retired] + (db/start! db test node) + (reset! retired nil)))) + +(defn nemesis [db] + (nemesis/compose + {{:partition-start :start, :partition-stop :stop} + (PartitionNemesis. (atom nil)) + + {:replica-partition-start :start, + :replica-partition-stop :stop, + :replica-reset :reset} + (ReplicaNemesis. (atom nil)) + + {:kill-node :start, :restart-node :stop} + (ProcessNemesis. db (atom nil)) + + {:retire-node :retire} + (RetirementNemesis. db (atom nil))})) diff --git a/test/jepsen/test/group/jepsen/model_test.clj b/test/jepsen/test/group/jepsen/model_test.clj new file mode 100644 index 0000000..241fb15 --- /dev/null +++ b/test/jepsen/test/group/jepsen/model_test.clj @@ -0,0 +1,208 @@ +(ns group.jepsen.model-test + (:require [clojure.test :refer :all] + [group.jepsen.model :as model])) + +(def test-map + {:nodes ["n1" "n2" "n3"] + :terminal-nodes ["n1" "n2" "n3"] + :key-count 2 + :clusters ["red"] + :transport "distribution" + :terminal-snapshots-per-node 1 + :required-transport-events #{}}) + +(defn peers-for [node nodes] + (->> nodes + (remove #(= node %)) + (map #(str "group@" %)) + sort + vec)) + +(defn public-view [root red] + {"root" root, "red" red}) + +(defn healthy-internal [] + {:healthy true + :errors [] + :snapshot-staging-count 0 + :oplog-entries 2}) + +(defn snapshot-op + ([index node owners registry pg] + (snapshot-op index node (:terminal-nodes test-map) owners registry pg)) + ([index node nodes owners registry pg] + {:index index + :process index + :type :ok + :f :snapshot + :value {:node node + :peers (peers-for node nodes) + :owners owners + :unexpected-deaths [] + :transport-events {} + :transport-profile :distribution + :internal (healthy-internal) + :registry registry + :pg pg}})) + +(defn owner [token registrations memberships] + {:token token, :registrations registrations, :memberships memberships}) + +(defn registration [cluster key revision] + {:cluster cluster, :key key, :revision revision}) + +(defn membership [cluster key revision] + {:cluster cluster, :key key, :revision revision}) + +(defn empty-registry [] + (public-view {0 nil, 1 nil} {0 nil, 1 nil})) + +(defn empty-pg [] + (public-view {0 [], 1 []} {0 [], 1 []})) + +(defn with-unexpected-death [op token] + (assoc-in op [:value :unexpected-deaths] [{:token token, :reason ":boom"}])) + +(deftest accepts-an-exact-converged-multi-cluster-view + (let [owners [(owner "a" [(registration nil 0 1) (registration "red" 1 2)] []) + (owner "b" [] [(membership nil 1 2) (membership "red" 0 3)])] + registry (public-view {0 "a", 1 nil} {0 nil, 1 "a"}) + pg (public-view {0 [], 1 ["b"]} {0 ["b"], 1 []}) + history [(snapshot-op 1 "n1" owners registry pg) + (snapshot-op 2 "n2" [] registry pg) + (snapshot-op 3 "n3" [] registry pg)]] + (is (:valid? (model/analyze test-map history))))) + +(deftest does-not-require-an-owner-without-group-intent + (let [idle-owner (owner "idle" [] []) + history [(snapshot-op 1 "n1" [idle-owner] (empty-registry) (empty-pg)) + (snapshot-op 2 "n2" [] (empty-registry) (empty-pg)) + (snapshot-op 3 "n3" [] (empty-registry) (empty-pg))]] + (is (:valid? (model/analyze test-map history))))) + +(deftest rejects-an-incomplete-terminal-observation + (let [history [(snapshot-op 1 "n1" [] (empty-registry) (empty-pg)) + (snapshot-op 2 "n2" [] (empty-registry) (empty-pg))] + result (model/analyze test-map history)] + (is (false? (:valid? result))) + (is (= #{"n3"} (:missing-nodes result))))) + +(deftest accepts-a-permanently-retired-node-and-requires-its-absence + (let [survivors ["n2" "n3"] + permanent-test (assoc test-map :terminal-nodes survivors) + history [(snapshot-op 1 "n2" survivors [] (empty-registry) (empty-pg)) + (snapshot-op 2 "n3" survivors [] (empty-registry) (empty-pg))]] + (is (:valid? (model/analyze permanent-test history))))) + +(deftest rejects-zombies-missing-live-owners-and-divergence + (let [live (owner "live" [(registration nil 0 1)] []) + stale-registry (assoc-in (empty-registry) ["root" 0] "dead") + result (model/analyze + test-map + [(snapshot-op 1 "n1" [live] stale-registry (empty-pg)) + (snapshot-op 2 "n2" [] (empty-registry) (empty-pg)) + (snapshot-op 3 "n3" [] (empty-registry) (empty-pg))])] + (is (false? (:valid? result))) + (is (= #{"dead"} (:orphaned-owner-tokens result))) + (is (= #{"live"} (:missing-live-owner-tokens result))) + (is (seq (:mismatched-views result))))) + +(deftest rejects-a-live-unresolved-registry-conflict + (let [owners [(owner "a" [(registration nil 0 1)] []) + (owner "b" [(registration nil 0 2)] [])] + registry (assoc-in (empty-registry) ["root" 0] "b") + history [(snapshot-op 1 "n1" owners registry (empty-pg)) + (snapshot-op 2 "n2" [] registry (empty-pg)) + (snapshot-op 3 "n3" [] registry (empty-pg))] + result (model/analyze test-map history)] + (is (false? (:valid? result))) + (is (= {["root" 0] #{"a" "b"}} (:live-registry-conflicts result))))) + +(deftest rejects-an-unexpected-owner-death-even-after-cleanup + (let [result (model/analyze + test-map + [(with-unexpected-death + (snapshot-op 1 "n1" [] (empty-registry) (empty-pg)) + "lost-owner") + (snapshot-op 2 "n2" [] (empty-registry) (empty-pg)) + (snapshot-op 3 "n3" [] (empty-registry) (empty-pg))])] + (is (false? (:valid? result))) + (is (= #{{:token "lost-owner", :reason ":boom"}} + (:unexpected-owner-deaths result))))) + +(deftest rejects-terminal-state-which-keeps-changing + (let [stale-registry (assoc-in (empty-registry) ["root" 0] "stale") + history [(snapshot-op 1 "n1" [] stale-registry (empty-pg)) + (snapshot-op 2 "n2" [] (empty-registry) (empty-pg)) + (snapshot-op 3 "n3" [] (empty-registry) (empty-pg)) + (snapshot-op 4 "n1" [] (empty-registry) (empty-pg)) + (snapshot-op 5 "n2" [] (empty-registry) (empty-pg)) + (snapshot-op 6 "n3" [] (empty-registry) (empty-pg))] + result (model/analyze + (assoc test-map :terminal-snapshots-per-node 2) + history)] + (is (false? (:valid? result))) + (is (contains? (:unstable-terminal-observations result) "n1")))) + +(deftest rejects-a-node-without-all-control-plane-peers + (let [history [(assoc-in (snapshot-op 1 "n1" [] (empty-registry) (empty-pg)) + [:value :peers] + ["group@n2"]) + (snapshot-op 2 "n2" [] (empty-registry) (empty-pg)) + (snapshot-op 3 "n3" [] (empty-registry) (empty-pg))] + result (model/analyze test-map history)] + (is (false? (:valid? result))) + (is (= #{"group@n2" "group@n3"} + (get-in result [:peer-mismatches "n1" :expected]))))) + +(deftest rejects-a-run-which-did-not-exercise-required-repair-paths + (let [history [(snapshot-op 1 "n1" [] (empty-registry) (empty-pg)) + (snapshot-op 2 "n2" [] (empty-registry) (empty-pg)) + (snapshot-op 3 "n3" [] (empty-registry) (empty-pg))] + result (model/analyze + (assoc test-map + :required-transport-events + #{:delta-batch :snapshot-chunk :multi-chunk-snapshot-chunk}) + history)] + (is (false? (:valid? result))) + (is (= #{:delta-batch :snapshot-chunk :multi-chunk-snapshot-chunk} + (:missing-transport-events result))))) + +(deftest rejects-internal-corruption-or-leftover-snapshot-staging + (let [bad (-> (snapshot-op 1 "n1" [] (empty-registry) (empty-pg)) + (assoc-in [:value :internal :healthy] false) + (assoc-in [:value :internal :snapshot-staging-count] 1) + (assoc-in [:value :internal :errors] ["broken index"])) + result (model/analyze + test-map + [bad + (snapshot-op 2 "n2" [] (empty-registry) (empty-pg)) + (snapshot-op 3 "n3" [] (empty-registry) (empty-pg))])] + (is (false? (:valid? result))) + (is (= 1 (get-in result [:internal-invariant-errors "n1" + :snapshot-staging-count]))))) + +(deftest rejects-the-wrong-transport-profile + (let [wrong (assoc-in (snapshot-op 1 "n1" [] (empty-registry) (empty-pg)) + [:value :transport-profile] + :tcp) + result (model/analyze + test-map + [wrong + (snapshot-op 2 "n2" [] (empty-registry) (empty-pg)) + (snapshot-op 3 "n3" [] (empty-registry) (empty-pg))])] + (is (false? (:valid? result))) + (is (= {"n1" :tcp} (:transport-profile-mismatches result))))) + +(deftest rejects-a-blocking-group-operation + (let [base [(snapshot-op 1 "n1" [] (empty-registry) (empty-pg)) + (snapshot-op 2 "n2" [] (empty-registry) (empty-pg)) + (snapshot-op 3 "n3" [] (empty-registry) (empty-pg))] + slow {:index 4 + :process 0 + :type :ok + :f :register + :value {:response {:latency-us 2500000}}} + result (model/analyze test-map (conj base slow))] + (is (false? (:valid? result))) + (is (= 2500.0 (:max-group-operation-latency-ms result))))) diff --git a/test/replica_adversarial_test.exs b/test/replica_adversarial_test.exs index 6b1437a..9bcfa4c 100644 --- a/test/replica_adversarial_test.exs +++ b/test/replica_adversarial_test.exs @@ -13,14 +13,14 @@ defmodule Group.ReplicaAdversarialTest do @seed seed @tag chaos_seed: seed - test "seeded mixed-operation transport chaos converges without zombies (seed #{@seed})" do + test "seeded three-node transport chaos converges without zombies (seed #{@seed})" do seed = @seed :rand.seed(:exsss, {seed, seed * 3 + 1, seed * 7 + 2}) - peers = TestCluster.start_peers(2) + peers = TestCluster.start_peers(3) on_exit(fn -> TestCluster.stop_peers(peers) end) - [{_, node_a}, {_, node_b}] = peers + nodes = Enum.map(peers, &elem(&1, 1)) name = :"replica_chaos_#{seed}_#{System.unique_integer([:positive])}" opts = [ @@ -40,30 +40,29 @@ defmodule Group.ReplicaAdversarialTest do TestCluster.assert_eventually( fn -> - Enum.all?([node_a, node_b], fn node -> - length(TestCluster.rpc!(node, Group, :nodes, [name])) == 1 and + Enum.all?(nodes, fn node -> + length(TestCluster.rpc!(node, Group, :nodes, [name])) == length(nodes) - 1 and Enum.all?(@clusters, fn cluster -> - length(TestCluster.rpc!(node, Group, :nodes, [name, cluster])) == 2 + length(TestCluster.rpc!(node, Group, :nodes, [name, cluster])) == length(nodes) end) end) end, timeout: 10_000 ) - :ok = - TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [ - name, - {:chaos, [drop_every: 5, duplicate_every: 7, max_delay: 40]} - ]) + chaos_modes = [ + {:chaos, [drop_every: 5, duplicate_every: 7, max_delay: 40]}, + {:chaos, [drop_every: 7, duplicate_every: 5, max_delay: 55]}, + {:chaos, [drop_every: 4, duplicate_every: 9, max_delay: 70]} + ] - :ok = - TestCluster.rpc!(node_b, Group.TestReplicaTransport, :set_mode, [ - name, - {:chaos, [drop_every: 7, duplicate_every: 5, max_delay: 55]} - ]) + Enum.zip(nodes, chaos_modes) + |> Enum.each(fn {node, mode} -> + :ok = TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [name, mode]) + end) initial = %{ - active: %{node_a => MapSet.new(@clusters), node_b => MapSet.new(@clusters)}, + active: Map.new(nodes, &{&1, MapSet.new(@clusters)}), counter: 0, pg_keys: MapSet.new(), pids: [], @@ -73,37 +72,36 @@ defmodule Group.ReplicaAdversarialTest do state = Enum.reduce(1..72, initial, fn step, state -> - apply_random_operation(state, step, seed, name, node_a, node_b) + apply_random_operation(state, step, seed, name, nodes) end) - for node <- [node_a, node_b] do + for node <- nodes do :ok = TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [name, :pass]) :ok = TestCluster.rpc!(node, Group, :connect, [name, @clusters]) end - assert_converges(name, node_a, node_b, state) + assert_converges(name, nodes, state) # Let delayed frames from the chaos phase arrive, then prove they are # duplicates/stale rather than a source of resurrection. Process.sleep(150) - TestCluster.flush_shards(node_a, name) - TestCluster.flush_shards(node_b, name) - assert_converges(name, node_a, node_b, state) + Enum.each(nodes, &TestCluster.flush_shards(&1, name)) + assert_converges(name, nodes, state) - for node <- [node_a, node_b] do + for node <- nodes do assert :ok = TestCluster.rpc!(node, Group.TestCluster, :assert_replica_consistent, [name]) end TestCluster.assert_eventually( - fn -> retained_owners_alive?(name, [node_a, node_b]) end, + fn -> retained_owners_alive?(name, nodes) end, timeout: 15_000 ) end end - defp apply_random_operation(state, step, seed, name, node_a, node_b) do - nodes = [node_a, node_b] + defp apply_random_operation(state, step, seed, name, nodes) do + [node_a, node_b | _rest] = nodes case :rand.uniform(12) do choice when choice in 1..3 -> @@ -244,17 +242,19 @@ defmodule Group.ReplicaAdversarialTest do {"chaos/#{seed}/#{kind}/#{step}/#{counter}", %{state | counter: counter}} end - defp assert_converges(name, node_a, node_b, state) do + defp assert_converges(name, nodes, state) do TestCluster.assert_eventually( fn -> - length(TestCluster.rpc!(node_a, Group, :nodes, [name])) == 1 and - length(TestCluster.rpc!(node_b, Group, :nodes, [name])) == 1 and + Enum.all?(nodes, fn node -> + length(TestCluster.rpc!(node, Group, :nodes, [name])) == length(nodes) - 1 + end) and Enum.all?(@clusters, fn cluster -> - length(TestCluster.rpc!(node_a, Group, :nodes, [name, cluster])) == 2 and - length(TestCluster.rpc!(node_b, Group, :nodes, [name, cluster])) == 2 + Enum.all?(nodes, fn node -> + length(TestCluster.rpc!(node, Group, :nodes, [name, cluster])) == length(nodes) + end) end) and - registry_equal?(name, node_a, node_b, state.reg_keys) and - memberships_equal?(name, node_a, node_b, state.pg_keys) + registry_equal?(name, nodes, state.reg_keys) and + memberships_equal?(name, nodes, state.pg_keys) end, timeout: 20_000, interval: 75 @@ -263,19 +263,21 @@ defmodule Group.ReplicaAdversarialTest do error -> flunk( "chaos convergence failed: #{Exception.message(error)}\n" <> - "differences=#{inspect(convergence_differences(name, node_a, node_b, state), limit: :infinity)}\n" <> + "differences=#{inspect(convergence_differences(name, nodes, state), limit: :infinity)}\n" <> "recent operations=#{inspect(Enum.take(state.trace, 20), limit: :infinity)}" ) end - defp convergence_differences(name, node_a, node_b, state) do + defp convergence_differences(name, nodes, state) do registry = state.reg_keys |> Enum.flat_map(fn {cluster, key} -> args = [name, key, cluster_opts(cluster)] - value_a = TestCluster.rpc!(node_a, Group, :lookup, args) - value_b = TestCluster.rpc!(node_b, Group, :lookup, args) - if value_a == value_b, do: [], else: [{:registry, cluster, key, value_a, value_b}] + values = Map.new(nodes, &{&1, TestCluster.rpc!(&1, Group, :lookup, args)}) + + if values |> Map.values() |> Enum.uniq() |> length() == 1, + do: [], + else: [{:registry, cluster, key, values}] end) |> Enum.take(10) @@ -283,17 +285,17 @@ defmodule Group.ReplicaAdversarialTest do state.pg_keys |> Enum.flat_map(fn {cluster, key} -> args = [name, key, cluster_opts(cluster)] - value_a = TestCluster.rpc!(node_a, Group, :members, args) |> Enum.sort() - value_b = TestCluster.rpc!(node_b, Group, :members, args) |> Enum.sort() - if value_a == value_b, do: [], else: [{:pg, cluster, key, value_a, value_b}] + values = Map.new(nodes, &{&1, TestCluster.rpc!(&1, Group, :members, args) |> Enum.sort()}) + + if values |> Map.values() |> Enum.uniq() |> length() == 1, + do: [], + else: [{:pg, cluster, key, values}] end) |> Enum.take(10) - nodes = + topology = for cluster <- [nil | @clusters] do - value_a = group_nodes(node_a, name, cluster) - value_b = group_nodes(node_b, name, cluster) - {cluster, value_a, value_b} + {cluster, Map.new(nodes, &{&1, group_nodes(&1, name, cluster)})} end cluster_trace = @@ -305,12 +307,12 @@ defmodule Group.ReplicaAdversarialTest do end) protocol = - for node <- [node_a, node_b] do + for node <- nodes do {node, TestCluster.rpc!(node, Group.TestCluster, :replica_protocol_state, [name])} end [ - nodes: nodes, + nodes: topology, registry: registry, pg: pg, cluster_trace: cluster_trace, @@ -323,21 +325,23 @@ defmodule Group.ReplicaAdversarialTest do defp group_nodes(node, name, cluster), do: TestCluster.rpc!(node, Group, :nodes, [name, cluster]) - defp registry_equal?(name, node_a, node_b, keys) do + defp registry_equal?(name, nodes, keys) do Enum.all?(keys, fn {cluster, key} -> args = [name, key, cluster_opts(cluster)] - TestCluster.rpc!(node_a, Group, :lookup, args) == - TestCluster.rpc!(node_b, Group, :lookup, args) + nodes |> Enum.map(&TestCluster.rpc!(&1, Group, :lookup, args)) |> Enum.uniq() |> length() == + 1 end) end - defp memberships_equal?(name, node_a, node_b, keys) do + defp memberships_equal?(name, nodes, keys) do Enum.all?(keys, fn {cluster, key} -> args = [name, key, cluster_opts(cluster)] - members_a = TestCluster.rpc!(node_a, Group, :members, args) |> Enum.sort() - members_b = TestCluster.rpc!(node_b, Group, :members, args) |> Enum.sort() - members_a == members_b + + nodes + |> Enum.map(&(TestCluster.rpc!(&1, Group, :members, args) |> Enum.sort())) + |> Enum.uniq() + |> length() == 1 end) end From 1e85161f302089ad4d4178a7c137e91daba0262d Mon Sep 17 00:00:00 2001 From: Chris McCord Date: Tue, 11 Aug 2026 12:52:35 +0000 Subject: [PATCH 06/16] Refine replica transport boundary --- CHANGELOG.md | 6 +- CLAUDE.md | 9 ++- README.md | 33 +++++----- lib/group.ex | 2 +- lib/group/replica.ex | 55 ++++++++-------- lib/group/replica/transport.ex | 66 +++++++++++--------- lib/group/replica/transport/outbox.ex | 64 +++++++++---------- lib/group/replica/transport/tcp.ex | 26 ++++---- test/README.md | 17 ++--- test/distributed_test.exs | 26 ++++---- test/jepsen/README.md | 2 +- test/jepsen/node.exs | 32 +++++----- test/replica_snapshot_distributed_test.exs | 4 +- test/replica_transport_outbox_test.exs | 38 +++++------ test/support/controlled_replica_transport.ex | 10 +-- test/support/replica_model_scheduler.ex | 18 +++--- test/support/test_replica_transport.ex | 66 ++++++++++---------- 17 files changed, 245 insertions(+), 229 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f41d3d..45bec76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ oracle across distribution, sideband TCP, and lossy/reordering transports. `mix test` is the every-PR ExUnit/property/checker gate and `mix test.soak` runs the six-profile nightly/release campaign. +- **Breaking**: the replica transport boundary now names logical direction + rather than implementation mechanics: adapters implement `outgoing/5`, + sideband adapters use `Group.Replica.Transport.Outbox.push/5`, and receiving + adapters call `incoming/4` or `incoming_batch/4`. - **Breaking**: replica protocol v2 splits exact snapshots into transport-neutral, byte-targeted chunks (`1 MiB` by default). Receivers stage chunks in shard-owned private ETS and advance the stream cursor only after an @@ -44,7 +48,7 @@ the losing process's `{:group_registry_conflict, key, winner_meta}` exit reason. - **Breaking**: `Group.dispatch/4` remote sends and process-DOWN replication are now non-suspending and never auto-connect. Busy dispatch drops still force a disconnect and - bounded reconnect retry; replica frames are dropped and repaired by anti-entropy without + bounded reconnect retry; replica messages are dropped and repaired by anti-entropy without disturbing the dist-Erlang control connection. Previously dispatch could block the caller and initiate new connections. - Configured function-form `extract_meta` callbacks are now applied on reads and lifecycle diff --git a/CLAUDE.md b/CLAUDE.md index 9c326c2..15ba7ad 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -134,7 +134,7 @@ dependencies on additive full-state merge. ## Anti-Entropy -Replica data frames are: +Replica data messages are: - `heads`: stream, retained floor, and head; - `delta_batch`: one or more contiguous stream runs; @@ -163,7 +163,7 @@ visible state. Staging expires after one peer-lease interval without progress. All cross-node Group control sends use `:erlang.send_nosuspend(..., [:noconnect])`. The default distribution replica adapter sends directly the same way and adds no local hop. `:busy` and -`:disconnected` mean “drop this frame”; periodic anti-entropy repairs it. +`:disconnected` mean “drop this message”; periodic anti-entropy repairs it. A sideband adapter may use one local `Group.Replica.Transport.Outbox` per shard. Outboxes batch by peer, impose deadlines, and run bounded socket work @@ -174,9 +174,8 @@ identity and carries authority. TCP is not encrypted. Transport ordering is not required for correctness. Per-shard ordered delivery is a fast path; stream sequences reject duplicate/out-of-order data, and -generation/epoch/lane fences handle control/data reordering. Ingress must derive -`source_node` from the authenticated connection, never from payload data, and -must reassemble any transport segmentation before `deliver_batch/4`. +generation/epoch/lane fences handle control/data reordering. A transport must +reassemble any transport segmentation before `incoming_batch/4`. ## Registry Projection and Process Ownership diff --git a/README.md b/README.md index 8a00c32..51ac159 100644 --- a/README.md +++ b/README.md @@ -289,7 +289,7 @@ All operations are **eventually consistent**: milliseconds. Defaults to 5. - **`busy_dist_retry_attempts`** — reconnect attempts after a non-suspending remote dispatch reports a busy dist link. Defaults to 300. Replica transport - frames are simply dropped and repaired instead of forcing a disconnect. + messages are simply dropped and repaired instead of forcing a disconnect. - **`busy_dist_retry_interval`** — milliseconds between dispatch busy-link reconnect attempts. Defaults to 1,000. - **`replicated_pg_receiver_local_request_quota`** — legacy-named quota for @@ -298,7 +298,7 @@ All operations are **eventually consistent**: - **`replica_transport`** — a module implementing `Group.Replica.Transport`, or `{module, opts}`. The default adapter uses `:erlang.send_nosuspend/3`; adapters must return promptly with `:ok`, `:busy`, - or `:disconnected`. Dropped and busy frames are repaired by anti-entropy. + or `:disconnected`. Dropped and busy messages are repaired by anti-entropy. `Group.Replica.Transport.TCP` is an included sideband adapter with local per-shard batching and bounded per-peer writer queues; its socket owners are separate processes, so socket backpressure cannot block a Group shard. @@ -307,7 +307,7 @@ All operations are **eventually consistent**: peer acknowledgements; a peer behind the retained floor receives an exact snapshot. - **`replicated_snapshot_chunk_target_bytes`** — target maximum encoded size - of each exact-snapshot frame. Defaults to 1 MiB and applies above every + of each exact-snapshot message. Defaults to 1 MiB and applies above every transport, including dist Erlang. A single row larger than the target is sent alone. Receivers stage chunks in shard-owned private ETS and replace visible state only after the complete exact slice is present. @@ -435,7 +435,7 @@ fenced, stream-head exchange on the replica transport catches the peer up. Every local mutation is first appended to a stream identified by `{group, origin_node, origin_generation, shard, cluster, cluster_epoch}` and a strictly increasing sequence number. It is then applied to the materialized -ETS view and batched into one delta frame per target. Process-death registry +ETS view and batched into one delta message per target. Process-death registry and PG removals can share one record and retain their one-event-batch behavior. Receivers advance a cursor only across a contiguous sequence prefix. A gap @@ -448,7 +448,7 @@ There are no leaders, quorum acknowledgements, per-entry replicated tombstones, or known-membership retention barriers. Oplog memory is bounded locally and independently of slow peers. Deletes are normal ordered records while retained, and exact snapshots close gaps after pruning. Exact snapshots are split into -transport-neutral byte-bounded frames; loss, duplication, or reordering leaves +transport-neutral byte-bounded messages; loss, duplication, or reordering leaves the old visible slice and cursor untouched until all chunks arrive. Incomplete staging expires after a peer-lease interval without progress and is destroyed automatically with its owning shard. Named-cluster close uses only a temporary @@ -467,8 +467,8 @@ writes, each stream numbers them, and receivers reject gaps and duplicates. Per-shard ordered delivery is still a useful fast path. Cross-stream order is not a correctness dependency; cluster epochs reject data racing a disconnect or reconnect, and generation fencing rejects data from a restarted origin. -An alternative sideband adapter authenticates the peer as a dist-Erlang node -and calls `Group.Replica.Transport.deliver/4` locally. +An alternative sideband adapter passes incoming messages to +`Group.Replica.Transport.incoming/4` locally. For example, replica data can use the included sideband TCP adapter while authority and membership remain on dist Erlang: @@ -495,22 +495,21 @@ control/data ordering relationship; the generation/epoch lane barrier and stream sequence checks supply correctness. The default distribution adapter still sends directly to the remote shard and -does not pay for a local outbox. Sideband adapters can delegate `try_send/5` to -`Group.Replica.Transport.Outbox.try_send/5` and supervise one outbox per shard -with `Group.Replica.Transport.Outbox.child_spec/1`. An outbox groups frames by +does not pay for a local outbox. Sideband adapters can delegate `outgoing/5` to +`Group.Replica.Transport.Outbox.push/5` and supervise one outbox per shard +with `Group.Replica.Transport.Outbox.child_spec/1`. An outbox groups messages by target and invokes the adapter's `send_batch/4` callback. Calls that expire or return `:busy`/`:disconnected` are dropped without a local retry; the next anti-entropy exchange repairs them. A message-oriented backend fits this callback shape by obtaining a connection once from `init_outbox/3`, then sending each `send_batch/4` result to a -registered ingress name on the target node. Queue pressure maps to `:busy` and -a missing session maps to `:disconnected`. Ingress must attach the authenticated -connection's source node; an adapter must never trust a source node supplied -inside the payload. Exact snapshots are already bounded by Group. A transport -with a smaller maximum frame may additionally segment an encoded batch, but it -must completely reassemble that batch before calling -`Group.Replica.Transport.deliver_batch/4`. +registered incoming name on the target node. Queue pressure maps to `:busy` and +a missing session maps to `:disconnected`. The adapter passes the trusted peer's +source node alongside each message. Exact snapshots are already bounded by +Group. A transport with a smaller maximum frame may additionally segment an +encoded batch, but it must completely reassemble that batch before calling +`Group.Replica.Transport.incoming_batch/4`. ### Named Cluster TTL Leases diff --git a/lib/group.ex b/lib/group.ex index 0e949e8..17a3df6 100644 --- a/lib/group.ex +++ b/lib/group.ex @@ -234,7 +234,7 @@ defmodule Group do data or cluster controls are busy (default: `8`) - `:replica_transport` — replica data transport module or `{module, opts}` tuple. Defaults to `Group.Replica.Transport.Distribution`. The transport must be - nonblocking and may return `:busy`; anti-entropy repairs dropped frames. + nonblocking and may return `:busy`; anti-entropy repairs dropped messages. Sideband transports can use `Group.Replica.Transport.Outbox` for lossy, batched, per-shard isolation without adding a hop to the default transport. - `:replicated_oplog_max_entries` — maximum retained replica records per shard diff --git a/lib/group/replica.ex b/lib/group/replica.ex index d0f888d..d70b462 100644 --- a/lib/group/replica.ex +++ b/lib/group/replica.ex @@ -64,11 +64,11 @@ defmodule Group.Replica do the requested prefix has already been pruned. Receivers stage chunks in a private ETS table and expose nothing until every chunk is present. - Every stream field is validated against the authenticated source node and + Every stream field is validated against the source node and current generation/epoch. An old generation, a closed epoch, a wrong shard, or a transitive claim for another node's pid is rejected. Control/data - reordering is safe: early frames are ignored and repeated heads repair them; - late frames fail their generation or epoch fence. Snapshot chunks may be + reordering is safe: early messages are ignored and repeated heads repair them; + late messages fail their generation or epoch fence. Snapshot chunks may be lost, duplicated, reordered, or mixed across retransmissions at the same stream head; exact row counts and set insertion prevent partial commits. @@ -88,10 +88,10 @@ defmodule Group.Replica do All cross-node control messages use :erlang.send_nosuspend/3 with :noconnect. The default replica adapter does the same. Transport callbacks return :ok, - :busy, or :disconnected; failure drops the frame and anti-entropy repairs it. + :busy, or :disconnected; failure drops the message and anti-entropy repairs it. Replica shards never remotely monitor or exit member processes. - The transport need not order frames for correctness. The local shard + The transport need not order messages for correctness. The local shard serializes writes, sequence numbers establish per-stream order, and receivers reject duplicates and gaps. TCP shard-to-shard ordering remains the efficient fast path. No semantic operation spans clusters, so cross-stream ordering is @@ -764,20 +764,21 @@ defmodule Group.Replica do end end - def handle_info({:group_replica_frame, remote_pid, frame}, state) when is_pid(remote_pid) do + def handle_info({:group_replica_frame, remote_pid, message}, state) when is_pid(remote_pid) do remote_node = node(remote_pid) - state = handle_replica_frame(state, remote_node, frame) + state = handle_replica_message(state, remote_node, message) {:noreply, take_priority_turn(state)} end - def handle_info({:group_replica_frame, remote_node, frame}, state) when is_atom(remote_node) do - state = handle_replica_frame(state, remote_node, frame) + def handle_info({:group_replica_frame, remote_node, message}, state) + when is_atom(remote_node) do + state = handle_replica_message(state, remote_node, message) {:noreply, take_priority_turn(state)} end - def handle_info({:group_replica_batch, remote_node, frames}, state) - when is_atom(remote_node) and is_list(frames) do - state = Enum.reduce(frames, state, &handle_replica_frame(&2, remote_node, &1)) + def handle_info({:group_replica_batch, remote_node, messages}, state) + when is_atom(remote_node) and is_list(messages) do + state = Enum.reduce(messages, state, &handle_replica_message(&2, remote_node, &1)) {:noreply, take_priority_turn(state)} end @@ -2712,7 +2713,7 @@ defmodule Group.Replica do {stream_id, first_seq, records, head} end) - try_send_replica_frame(state, target_node, {:delta_batch, Protocol.version(), runs}) + outgoing_replica_message(state, target_node, {:delta_batch, Protocol.version(), runs}) end defp group_broadcast_ops_by_target(ops, state, cluster_fun) do @@ -2812,12 +2813,12 @@ defmodule Group.Replica do end end - defp try_send_replica_frame(state, target_node, frame) do - case state.replica_transport.try_send( + defp outgoing_replica_message(state, target_node, message) do + case state.replica_transport.outgoing( state.name, target_node, state.shard_index, - frame, + message, state.replica_transport_opts ) do :ok -> :ok @@ -3230,7 +3231,7 @@ defmodule Group.Replica do if heads == [] do state else - try_send_replica_frame(state, target_node, {:heads, Protocol.version(), heads}) + outgoing_replica_message(state, target_node, {:heads, Protocol.version(), heads}) end end @@ -3287,7 +3288,7 @@ defmodule Group.Replica do (is_nil(cluster) or cluster_member?(state.name, cluster)) end - defp handle_replica_frame(state, source_node, {:heads, version, heads}) + defp handle_replica_message(state, source_node, {:heads, version, heads}) when version == @protocol_version do needs = Enum.flat_map(heads, fn {stream_id, _floor, head} -> @@ -3302,11 +3303,11 @@ defmodule Group.Replica do needs |> Enum.chunk_every(state.replicated_sender_buffer_size) |> Enum.reduce(state, fn chunk, acc -> - try_send_replica_frame(acc, source_node, {:needs, Protocol.version(), chunk}) + outgoing_replica_message(acc, source_node, {:needs, Protocol.version(), chunk}) end) end - defp handle_replica_frame(state, source_node, {:delta_batch, version, runs}) + defp handle_replica_message(state, source_node, {:delta_batch, version, runs}) when version == @protocol_version do state = flush_pending_replicated_sender_barrier(state) @@ -3315,7 +3316,7 @@ defmodule Group.Replica do end) end - defp handle_replica_frame(state, source_node, {:need, version, stream_id, next_seq}) + defp handle_replica_message(state, source_node, {:need, version, stream_id, next_seq}) when version == @protocol_version do if Protocol.stream_origin(stream_id) == node() and Protocol.stream_shard(stream_id) == state.shard_index and @@ -3326,12 +3327,12 @@ defmodule Group.Replica do end end - defp handle_replica_frame(state, source_node, {:needs, version, needs}) + defp handle_replica_message(state, source_node, {:needs, version, needs}) when version == @protocol_version do send_replica_repairs(state, source_node, needs) end - defp handle_replica_frame( + defp handle_replica_message( state, source_node, {:snapshot_chunk, version, stream_id, snapshot_seq, chunk_index, chunk_count, @@ -3380,7 +3381,7 @@ defmodule Group.Replica do end end - defp handle_replica_frame(state, _source_node, _frame), do: state + defp handle_replica_message(state, _source_node, _message), do: state defp valid_snapshot_stream?(state, source_node, stream_id, snapshot_seq) do valid_remote_stream?(state, source_node, stream_id) and @@ -3857,7 +3858,7 @@ defmodule Group.Replica do end defp request_replica_need(state, target_node, stream_id, next_seq) do - try_send_replica_frame( + outgoing_replica_message( state, target_node, {:needs, Protocol.version(), [{stream_id, next_seq}]} @@ -3884,7 +3885,7 @@ defmodule Group.Replica do state runs -> - try_send_replica_frame( + outgoing_replica_message( state, target_node, {:delta_batch, Protocol.version(), Enum.reverse(runs)} @@ -3944,7 +3945,7 @@ defmodule Group.Replica do snapshot.chunks |> Enum.with_index(1) |> Enum.reduce(state, fn {{reg_chunk, pg_chunk}, chunk_index}, acc -> - try_send_replica_frame( + outgoing_replica_message( acc, target_node, {:snapshot_chunk, Protocol.version(), stream_id, head, chunk_index, chunk_count, diff --git a/lib/group/replica/transport.ex b/lib/group/replica/transport.ex index b7f46c1..46bb532 100644 --- a/lib/group/replica/transport.ex +++ b/lib/group/replica/transport.ex @@ -3,38 +3,45 @@ defmodule Group.Replica.Transport do Transport contract for Group replica data. Implementations must return promptly and must never wait for socket or remote - mailbox backpressure. This applies to `try_send/5` and the optional lifecycle + mailbox backpressure. This applies to `outgoing/5` and the optional lifecycle callbacks. Returning `:busy` or `:disconnected` is safe: replica anti-entropy will retransmit the missing state. Erlang distribution remains Group's control plane and supplies the stable node identity used here. A sideband adapter can use its `descriptor/2` in the - control hello to exchange endpoints, authenticate the connection as that - node, and pass inbound frames to `deliver/4`. + control hello to exchange endpoints and pass incoming messages to + `incoming/4` or `incoming_batch/4`. Adapters do not need to preserve ordering. Group serializes writes per shard and sequences each origin/generation/shard/cluster/epoch stream; receivers discard duplicates and request gaps. Per-shard ordered delivery avoids repair traffic and is therefore the preferred fast path. - A sideband implementation can delegate `try_send/5` to - `Group.Replica.Transport.Outbox.try_send/5`. That adds one local send only for + A sideband implementation can delegate `outgoing/5` to + `Group.Replica.Transport.Outbox.push/5`. That adds one local send only for the configured sideband transport; the default distribution adapter retains its direct remote `:erlang.send_nosuspend/3` path. """ - @type frame :: term() - @type send_result :: :ok | :busy | :disconnected + @type message :: term() + @type outgoing_result :: :ok | :busy | :disconnected @callback id() :: term() @callback descriptor(group :: atom(), opts :: keyword()) :: term() - @callback try_send( + @doc """ + Called when Group has an outgoing replica message for another node. + + This callback must return promptly and must never wait for transport or + remote backpressure. `:ok` means the transport took responsibility for the + message, not that the remote shard received it. + """ + @callback outgoing( group :: atom(), target_node :: node(), shard :: non_neg_integer(), - frame(), + message(), opts :: keyword() - ) :: send_result() + ) :: outgoing_result() @callback child_spec(keyword()) :: Supervisor.child_spec() | :ignore @callback peer_up(group :: atom(), node(), descriptor :: term(), opts :: keyword()) :: :ok @@ -43,31 +50,34 @@ defmodule Group.Replica.Transport do @optional_callbacks child_spec: 1, peer_up: 4, peer_down: 3 @doc """ - Delivers a frame received by a transport adapter to the local replica shard. + Passes an incoming replica message to the corresponding local shard. - `source_node` must come from the adapter's authenticated peer identity, never - from untrusted frame contents. Delivery is a local mailbox operation; stream - generation, epoch, group, shard, and origin are validated by the replica. + This is a local mailbox operation. Stream generation, epoch, group, shard, + and origin are validated by the replica. """ - def deliver(group, source_node, shard, frame) + def incoming(group, source_node, shard, message) when is_atom(group) and is_atom(source_node) and is_integer(shard) and shard >= 0 do - send(Group.Replica.shard_name(group, shard), {:group_replica_frame, source_node, frame}) + send( + Group.Replica.shard_name(group, shard), + {:group_replica_frame, source_node, message} + ) + :ok end @doc """ - Delivers a complete batch received from one authenticated peer. + Passes a complete incoming batch to the corresponding local shard. - A finite-frame transport may segment the encoded batch on the wire, but it - must authenticate the peer and reassemble every segment before calling this - function. Group never observes or applies a partial batch. + A finite-message transport may segment the encoded batch on the wire, but it + must reassemble every segment before calling this function. Group never + observes or applies a partial batch. """ - def deliver_batch(group, source_node, shard, frames) + def incoming_batch(group, source_node, shard, messages) when is_atom(group) and is_atom(source_node) and is_integer(shard) and shard >= 0 and - is_list(frames) do + is_list(messages) do send( Group.Replica.shard_name(group, shard), - {:group_replica_batch, source_node, frames} + {:group_replica_batch, source_node, messages} ) :ok @@ -84,7 +94,7 @@ defmodule Group.Replica.Transport do def validate!({module, _opts} = transport) do Code.ensure_loaded!(module) - for {function, arity} <- [id: 0, descriptor: 2, try_send: 5] do + for {function, arity} <- [id: 0, descriptor: 2, outgoing: 5] do unless function_exported?(module, function, arity) do raise ArgumentError, "replica transport #{inspect(module)} must implement #{function}/#{arity}" @@ -99,10 +109,10 @@ defmodule Group.Replica.Transport.Distribution do @moduledoc """ Default nonblocking replica transport over Erlang distribution. - Frames are sent directly to the matching remote shard with + Messages are sent directly to the matching remote shard with `:erlang.send_nosuspend/3` and `:noconnect`, so the caller never waits for a busy distribution socket and never initiates a connection. A busy or absent - link returns `:busy`; Group drops that frame and repairs it through periodic + link returns `:busy`; Group drops that message and repairs it through periodic anti-entropy. """ @behaviour Group.Replica.Transport @@ -116,9 +126,9 @@ defmodule Group.Replica.Transport.Distribution do def descriptor(_group, _opts), do: :erlang_distribution @impl true - def try_send(group, target_node, shard, frame, _opts) do + def outgoing(group, target_node, shard, replica_message, _opts) do destination = {Replica.shard_name(group, shard), target_node} - message = {:group_replica_frame, node(), frame} + message = {:group_replica_frame, node(), replica_message} case :erlang.send_nosuspend(destination, message, [:noconnect]) do true -> :ok diff --git a/lib/group/replica/transport/outbox.ex b/lib/group/replica/transport/outbox.ex index aee2ba6..43c23b3 100644 --- a/lib/group/replica/transport/outbox.ex +++ b/lib/group/replica/transport/outbox.ex @@ -4,12 +4,12 @@ defmodule Group.Replica.Transport.Outbox do This module is an implementation helper, not a replacement for `Group.Replica.Transport`. Distribution can continue sending directly with - `:erlang.send_nosuspend/3`. A sideband adapter delegates `try_send/5` to - `try_send/5`, which performs only a local `send/2` to the matching shard + `:erlang.send_nosuspend/3`. A sideband adapter delegates `outgoing/5` to + `push/5`, which performs only a local `send/2` to the matching shard outbox. - Each outbox batches frames by target node outside the Group shard. Expired - frames and batches rejected by the backend are deliberately dropped; + Each outbox batches messages by target node outside the Group shard. Expired + messages and batches rejected by the backend are deliberately dropped; anti-entropy repairs them. Backends may perform bounded blocking work in `send_batch/4` because they run in the outbox rather than a Group process. @@ -19,35 +19,35 @@ defmodule Group.Replica.Transport.Outbox do def init_outbox(group, shard, opts), do: {:ok, backend_state} - def send_batch(target_node, frames, deadline, backend_state) do + def send_batch(target_node, messages, deadline, backend_state) do # Return promptly once `deadline` has passed. It is safe to drop. {:ok, backend_state} end - The backend is responsible for authenticated ingress and must pass only - complete logical frames to `Group.Replica.Transport.deliver_batch/4`. + The backend must pass only complete logical messages to + `Group.Replica.Transport.incoming_batch/4`. ## Options - * `:outbox_batch_size` - maximum logical frames collected per flush, + * `:outbox_batch_size` - maximum logical messages collected per flush, default `64` * `:outbox_batch_bytes` - approximate external-term bytes collected per flush, default `1_048_576` * `:outbox_flush_interval` - maximum batching delay in milliseconds, default `1` - * `:outbox_deadline` - maximum useful residence time for an outbound frame + * `:outbox_deadline` - maximum useful residence time for an outgoing message in milliseconds, default `100` The deadline bounds stale work, not mailbox memory. A backend must also put a finite bound on every socket enqueue or write it performs. Exact snapshot - frames are independently bounded by `:replicated_snapshot_chunk_target_bytes`. - Other logical frames or a whole batch may still exceed `:outbox_batch_bytes`; + messages are independently bounded by `:replicated_snapshot_chunk_target_bytes`. + Other logical messages or a whole batch may still exceed `:outbox_batch_bytes`; a transport with a smaller finite frame size must segment and completely reassemble those batches before local delivery. """ - @type frame :: Group.Replica.Transport.frame() - @type send_result :: Group.Replica.Transport.send_result() + @type message :: Group.Replica.Transport.message() + @type outgoing_result :: Group.Replica.Transport.outgoing_result() @type backend_state :: term() @callback init_outbox(group :: atom(), shard :: non_neg_integer(), opts :: keyword()) :: @@ -55,10 +55,10 @@ defmodule Group.Replica.Transport.Outbox do @callback send_batch( target_node :: node(), - frames :: [frame()], + messages :: [message()], deadline :: integer(), backend_state() - ) :: {send_result(), backend_state()} + ) :: {outgoing_result(), backend_state()} @default_deadline 100 @@ -81,20 +81,20 @@ defmodule Group.Replica.Transport.Outbox do end @doc """ - Enqueues a frame into its local shard outbox. + Pushes a replica message into its shard's local outbox. - This performs no backend or socket operation. `:ok` only means the message - was sent to the current local outbox PID; the outbox may later drop it on - expiry or backpressure. A concurrently terminating outbox can also lose an - accepted message, which anti-entropy repairs. + This operation is local and nonblocking and performs no backend or socket + work. `:ok` only means the outbox was available; the message may later expire + or be dropped under transport pressure. A concurrently terminating outbox + can also lose an accepted message, which anti-entropy repairs. """ - def try_send(group, target_node, shard, frame, opts) + def push(group, target_node, shard, message, opts) when is_atom(group) and is_atom(target_node) and is_integer(shard) and shard >= 0 and is_list(opts) do case Process.whereis(name(group, shard)) do pid when is_pid(pid) -> deadline = monotonic_ms() + deadline(opts) - send(pid, {:group_replica_outbox_send, target_node, deadline, frame}) + send(pid, {:group_replica_outbox_push, target_node, deadline, message}) :ok nil -> @@ -192,12 +192,12 @@ defmodule Group.Replica.Transport.Outbox.Worker do end @impl true - def handle_info({:group_replica_outbox_send, target_node, deadline, frame}, state) + def handle_info({:group_replica_outbox_push, target_node, deadline, message}, state) when is_atom(target_node) and is_integer(deadline) do if deadline <= Outbox.monotonic_ms() do {:noreply, state} else - bytes = :erlang.external_size({target_node, frame}) + bytes = :erlang.external_size({target_node, message}) state = if state.pending_count > 0 and @@ -208,7 +208,7 @@ defmodule Group.Replica.Transport.Outbox.Worker do state end - state = enqueue(state, target_node, deadline, frame, bytes) + state = put_pending(state, target_node, deadline, message, bytes) if state.pending_count >= state.batch_size or state.pending_bytes >= state.batch_bytes do {:noreply, flush(state)} @@ -225,8 +225,8 @@ defmodule Group.Replica.Transport.Outbox.Worker do def handle_info({:group_replica_outbox_flush, _stale_ref}, state), do: {:noreply, state} def handle_info(_message, state), do: {:noreply, state} - defp enqueue(state, target_node, deadline, frame, bytes) do - entry = {target_node, deadline, frame} + defp put_pending(state, target_node, deadline, message, bytes) do + entry = {target_node, deadline, message} %{ state @@ -254,22 +254,22 @@ defmodule Group.Replica.Transport.Outbox.Worker do batches = state.pending |> Enum.reverse() - |> Enum.reject(fn {_target_node, deadline, _frame} -> deadline <= now end) - |> Enum.group_by(fn {target_node, _deadline, _frame} -> target_node end) + |> Enum.reject(fn {_target_node, deadline, _message} -> deadline <= now end) + |> Enum.group_by(fn {target_node, _deadline, _message} -> target_node end) backend_state = Enum.reduce(batches, state.backend_state, fn {target_node, entries}, backend_state -> - frames = Enum.map(entries, fn {_target_node, _deadline, frame} -> frame end) + messages = Enum.map(entries, fn {_target_node, _deadline, message} -> message end) deadline = entries - |> Enum.map(fn {_target_node, deadline, _frame} -> deadline end) + |> Enum.map(fn {_target_node, deadline, _message} -> deadline end) |> Enum.min() if deadline <= Outbox.monotonic_ms() do backend_state else - case state.backend.send_batch(target_node, frames, deadline, backend_state) do + case state.backend.send_batch(target_node, messages, deadline, backend_state) do {result, next_backend_state} when result in [:ok, :busy, :disconnected] -> next_backend_state diff --git a/lib/group/replica/transport/tcp.ex b/lib/group/replica/transport/tcp.ex index 0cedaa8..d074099 100644 --- a/lib/group/replica/transport/tcp.ex +++ b/lib/group/replica/transport/tcp.ex @@ -3,17 +3,17 @@ defmodule Group.Replica.Transport.TCP do Sideband TCP transport for replica data. Erlang distribution still carries Group discovery and authority controls. - Replica frames use independent TCP connections, so there is no ordering + Replica messages use independent TCP connections, so there is no ordering relationship between a control message and its data lane. - `try_send/5` only sends to a local per-shard outbox. The outbox batches - frames and forwards each target batch to a bounded per-peer writer queue. + `outgoing/5` only pushes to a local per-shard outbox. The outbox batches + messages and forwards each target batch to a bounded per-peer writer queue. The writer may block up to `:send_timeout` without blocking a Group shard. Expired, busy, and disconnected batches are dropped and repaired by anti-entropy. The endpoint capability in the dist-Erlang hello prevents an unrelated - socket client from injecting frames. This transport is intended for trusted + socket client from injecting messages. This transport is intended for trusted cluster networks; it does not encrypt traffic. Put it behind a private network or a TLS/WebSocket tunnel when confidentiality is required. @@ -64,20 +64,20 @@ defmodule Group.Replica.Transport.TCP do end @impl true - def try_send(group, target_node, shard, frame, opts), - do: Outbox.try_send(group, target_node, shard, frame, opts) + def outgoing(group, target_node, shard, message, opts), + do: Outbox.push(group, target_node, shard, message, opts) @impl Group.Replica.Transport.Outbox def init_outbox(group, shard, _opts), do: {:ok, %{group: group, shard: shard}} @impl Group.Replica.Transport.Outbox - def send_batch(target_node, frames, deadline, %{group: group, shard: shard} = state) do + def send_batch(target_node, messages, deadline, %{group: group, shard: shard} = state) do result = try do case :ets.lookup(route_table(group), target_node) do [{^target_node, writer, queued, max_queue}] -> if :atomics.add_get(queued, 1, 1) <= max_queue do - send(writer, {:replica_batch, deadline, shard, frames}) + send(writer, {:replica_batch, deadline, shard, messages}) :ok else :atomics.sub(queued, 1, 1) @@ -379,12 +379,12 @@ defmodule Group.Replica.Transport.TCP do defp writer_loop(socket, manager, remote_node, queued) do receive do - {:replica_batch, deadline, shard, frames} -> + {:replica_batch, deadline, shard, messages} -> result = if deadline <= Outbox.monotonic_ms() do :expired else - :gen_tcp.send(socket, :erlang.term_to_binary({:batch, shard, frames})) + :gen_tcp.send(socket, :erlang.term_to_binary({:batch, shard, messages})) end :atomics.sub(queued, 1, 1) @@ -438,9 +438,9 @@ defmodule Group.Replica.Transport.TCP do case :gen_tcp.recv(socket, 0) do {:ok, payload} -> case decode_authenticated_frame(payload) do - {:ok, {:batch, shard, frames}} - when is_integer(shard) and shard >= 0 and is_list(frames) -> - :ok = Group.Replica.Transport.deliver_batch(group, source_node, shard, frames) + {:ok, {:batch, shard, messages}} + when is_integer(shard) and shard >= 0 and is_list(messages) -> + :ok = Group.Replica.Transport.incoming_batch(group, source_node, shard, messages) reader_loop(socket, group, source_node) _ -> diff --git a/test/README.md b/test/README.md index 224f250..ffb2986 100644 --- a/test/README.md +++ b/test/README.md @@ -33,7 +33,7 @@ release qualification rather than individual edits. ## Model-based and formal checks `replica_model_property_test.exs` runs real Group instances on three peer VMs. -The controlled transport queues each replica frame so generated commands can +The controlled transport queues each replica message so generated commands can deliver, duplicate, drop, reorder, or strand it. After the bounded-fault prefix, the test enables fair delivery and compares every tracked registry and PG key against an independent application-level lifecycle oracle. It also @@ -251,11 +251,12 @@ timestamp. ### Replica transport fault injection `Group.TestReplicaTransport` implements the production transport behaviour but -can return `:busy`, drop selected frame types, duplicate or delay frames, and -capture frames for explicit stale-generation/epoch replay. Its `{:chaos, opts}` -mode is deterministic for a given frame, which makes failures reproducible. +can return `:busy`, drop selected message types, duplicate or delay messages, +and capture messages for explicit stale-generation/epoch replay. Its +`{:chaos, opts}` mode is deterministic for a given message, which makes failures +reproducible. -`Group.ControlledReplicaTransport` is the model-test transport. It queues frames +`Group.ControlledReplicaTransport` is the model-test transport. It queues messages at the test process without scheduling timers; `Group.ReplicaModelScheduler` then owns the exact delivery schedule. These roles are separate so the existing timing-oriented regressions retain their original mechanics while property @@ -263,7 +264,7 @@ failures can be replayed and shrunk exactly. The distributed anti-entropy tests cover dropped creates and deletes, cursor gaps, globally pruned multi-stream oplogs, exact snapshot fallback, malformed -authority, stale frame replay, lease expiry on a live VM, and multi-shard +authority, stale message replay, lease expiry on a live VM, and multi-shard generation recovery. They also restart a suspended data lane after deliberately losing its cluster-close fence and require the lane to sweep the stale registry and PG slices from shared authority. Authority topology tests suspend every @@ -272,7 +273,7 @@ the full epoch snapshot, nonzero shards receive constant-size lane hellos, and incremental opens stay on their matching shard. Separate tests suspend a backlogged authority shard while other replica lanes continue converging and deliver data before authority to prove rejection does not advance the cursor -and the same frame applies after authority repair. Concurrent snapshot tests +and the same message applies after authority repair. Concurrent snapshot tests require every advertised revision to contain exactly that many unique named epochs, and heartbeat tests prove observed revisions cannot advance the exact authority marker. Crash-window tests interrupt journal, dual-index, receive @@ -283,7 +284,7 @@ requires snapshot recovery without changing the third node's independent registry or PG state. `replica_transport_outbox_test.exs` proves that a blocked sideband backend -cannot delay the Group-facing local send, frames expire behind that backend, +cannot delay the Group-facing local push, messages expire behind that backend, busy batches are not retried locally, and batching preserves per-target order. The real three-node TCP recovery test runs through the same outbox path. diff --git a/test/distributed_test.exs b/test/distributed_test.exs index 3a7c2d7..ebe8278 100644 --- a/test/distributed_test.exs +++ b/test/distributed_test.exs @@ -4606,7 +4606,7 @@ defmodule Group.DistributedTest do ]) :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ name, node_a, 1, @@ -4635,7 +4635,7 @@ defmodule Group.DistributedTest do end) :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ name, node_a, 1, @@ -4722,7 +4722,7 @@ defmodule Group.DistributedTest do end @tag timeout: 60_000 - test "duplicate and reordered replica frames are idempotent and emit one lifecycle event" do + test "duplicate and reordered replica messages are idempotent and emit one lifecycle event" do peers = TestCluster.start_peers(2) on_exit(fn -> TestCluster.stop_peers(peers) end) @@ -4827,7 +4827,7 @@ defmodule Group.DistributedTest do Enum.each(generation_frames, fn {_target, shard, frame} -> :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ name, node_a, shard, @@ -4882,7 +4882,7 @@ defmodule Group.DistributedTest do Enum.each(epoch_frames, fn {_target, shard, frame} -> :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ name, node_a, shard, @@ -5096,7 +5096,7 @@ defmodule Group.DistributedTest do ]} :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ name, node_a, 0, @@ -5113,7 +5113,7 @@ defmodule Group.DistributedTest do ]) == 0 :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ name, node_b, 0, @@ -5368,7 +5368,7 @@ defmodule Group.DistributedTest do mutation = {:register, nil, key, pid, meta, System.monotonic_time(), node_a} :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ name, node_a, 0, @@ -5432,7 +5432,7 @@ defmodule Group.DistributedTest do Map.fetch!(frames_by_first_seq, 2) :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ name, node_a, shard, @@ -5454,7 +5454,7 @@ defmodule Group.DistributedTest do for frame <- [first_frame, second_frame, third_frame] do :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ name, node_a, shard, @@ -5492,7 +5492,7 @@ defmodule Group.DistributedTest do for frame <- [first_frame, second_frame, third_frame] do :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ name, node_a, shard, @@ -5600,7 +5600,7 @@ defmodule Group.DistributedTest do # time this frame runs, but shard 1 must still reject it until its own # old-generation purge has completed. :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ name, node_a, 1, @@ -5633,7 +5633,7 @@ defmodule Group.DistributedTest do ]) == 0 :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ name, node_a, 1, diff --git a/test/jepsen/README.md b/test/jepsen/README.md index d543df9..aeba2ce 100644 --- a/test/jepsen/README.md +++ b/test/jepsen/README.md @@ -25,7 +25,7 @@ The replica lane is selectable without changing the workload or checker: - `tcp` uses Group's production sideband TCP adapter while Erlang distribution remains the control plane; and - `chaos` is a local per-shard outbox which deterministically drops, - duplicates, delays, and reorders replica frames. + duplicates, delays, and reorders replica messages. After faults stop, every surviving node reconnects and the harness takes two terminal snapshots. The independent checker requires: diff --git a/test/jepsen/node.exs b/test/jepsen/node.exs index 76817fb..c72774b 100644 --- a/test/jepsen/node.exs +++ b/test/jepsen/node.exs @@ -80,14 +80,14 @@ defmodule Group.Jepsen.Transport.Common do @moduledoc false alias Group.Jepsen.Transport.Stats - def try_send(delegate, group, target_node, shard, frame, opts) do - record(frame) + def outgoing(delegate, group, target_node, shard, message, opts) do + record(message) if Stats.blocked?(target_node) do Stats.increment(:logical_drop) :ok else - result = delegate.try_send(group, target_node, shard, frame, opts) + result = delegate.outgoing(group, target_node, shard, message, opts) Stats.increment(transport_result(result)) observe_outbox(group, shard) result @@ -103,7 +103,7 @@ defmodule Group.Jepsen.Transport.Common do end def record({:delta_batch, _version, _runs}), do: Stats.increment(:delta_batch) - def record(_frame), do: Stats.increment(:other_frame) + def record(_message), do: Stats.increment(:other_message) defp transport_result(:ok), do: :transport_ok defp transport_result(:busy), do: :transport_busy @@ -140,8 +140,8 @@ defmodule Group.Jepsen.Transport.Distribution do def child_spec(opts), do: {Stats, opts} @impl true - def try_send(group, target_node, shard, frame, opts) do - Common.try_send(Delegate, group, target_node, shard, frame, opts) + def outgoing(group, target_node, shard, message, opts) do + Common.outgoing(Delegate, group, target_node, shard, message, opts) end end @@ -168,8 +168,8 @@ defmodule Group.Jepsen.Transport.TCP do end @impl true - def try_send(group, target_node, shard, frame, opts) do - Common.try_send(Delegate, group, target_node, shard, frame, opts) + def outgoing(group, target_node, shard, message, opts) do + Common.outgoing(Delegate, group, target_node, shard, message, opts) end @impl true @@ -219,8 +219,8 @@ defmodule Group.Jepsen.Transport.Chaos do end @impl true - def try_send(group, target_node, shard, frame, _opts) do - Common.record(frame) + def outgoing(group, target_node, shard, message, _opts) do + Common.record(message) if Stats.blocked?(target_node) do Stats.increment(:logical_drop) @@ -228,7 +228,7 @@ defmodule Group.Jepsen.Transport.Chaos do else case Process.whereis(worker_name(group, shard)) do pid when is_pid(pid) -> - send(pid, {:send, target_node, frame}) + send(pid, {:outgoing, target_node, message}) :ok nil -> @@ -277,7 +277,7 @@ defmodule Group.Jepsen.Transport.Chaos.Worker do def init({group, shard}), do: {:ok, %{group: group, shard: shard, counter: 0}} @impl true - def handle_info({:send, target_node, frame}, state) do + def handle_info({:outgoing, target_node, message}, state) do counter = state.counter + 1 next = %{state | counter: counter} @@ -285,11 +285,11 @@ defmodule Group.Jepsen.Transport.Chaos.Worker do Stats.increment(:chaos_drop) else delay = rem(counter * 17, 41) - Process.send_after(self(), {:deliver, target_node, frame}, delay) + Process.send_after(self(), {:forward, target_node, message}, delay) if rem(counter, 7) == 0 do Stats.increment(:chaos_duplicate) - Process.send_after(self(), {:deliver, target_node, frame}, rem(delay + 19, 47)) + Process.send_after(self(), {:forward, target_node, message}, rem(delay + 19, 47)) end if delay > 0, do: Stats.increment(:chaos_delay) @@ -298,9 +298,9 @@ defmodule Group.Jepsen.Transport.Chaos.Worker do {:noreply, next} end - def handle_info({:deliver, target_node, frame}, state) do + def handle_info({:forward, target_node, replica_message}, state) do destination = {Group.Replica.shard_name(state.group, state.shard), target_node} - message = {:group_replica_frame, node(), frame} + message = {:group_replica_frame, node(), replica_message} case :erlang.send_nosuspend(destination, message, [:noconnect]) do true -> Stats.increment(:chaos_delivered) diff --git a/test/replica_snapshot_distributed_test.exs b/test/replica_snapshot_distributed_test.exs index a517770..89804b8 100644 --- a/test/replica_snapshot_distributed_test.exs +++ b/test/replica_snapshot_distributed_test.exs @@ -556,7 +556,7 @@ defmodule Group.ReplicaSnapshotDistributedTest do ]) :ok = - TestCluster.rpc!(node_a, Group.Replica.Transport, :deliver, [ + TestCluster.rpc!(node_a, Group.Replica.Transport, :incoming, [ name, node_b, 0, @@ -581,7 +581,7 @@ defmodule Group.ReplicaSnapshotDistributedTest do defp deliver_frames(node_b, node_a, name, frames) do Enum.each(frames, fn frame -> :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :deliver, [ + TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ name, node_a, 0, diff --git a/test/replica_transport_outbox_test.exs b/test/replica_transport_outbox_test.exs index 329ed96..63c3a4e 100644 --- a/test/replica_transport_outbox_test.exs +++ b/test/replica_transport_outbox_test.exs @@ -19,10 +19,10 @@ defmodule Group.ReplicaTransportOutboxTest do end @impl true - def send_batch(target_node, frames, deadline, state) do + def send_batch(target_node, messages, deadline, state) do send( state.controller, - {:outbox_batch, state.group, state.shard, target_node, frames, deadline} + {:outbox_batch, state.group, state.shard, target_node, messages, deadline} ) if state.sleep > 0, do: Process.sleep(state.sleep) @@ -30,7 +30,7 @@ defmodule Group.ReplicaTransportOutboxTest do end end - test "batches frames per target while preserving per-target order" do + test "batches messages per target while preserving per-target order" do group = unique_group(:batch) target_a = :"outbox-a@test" target_b = :"outbox-b@test" @@ -40,20 +40,20 @@ defmodule Group.ReplicaTransportOutboxTest do outbox_flush_interval: 1_000 ) - assert :ok = Outbox.try_send(group, target_a, 0, {:frame, 1}, outbox_deadline: 1_000) - assert :ok = Outbox.try_send(group, target_b, 0, {:frame, 2}, outbox_deadline: 1_000) - assert :ok = Outbox.try_send(group, target_a, 0, {:frame, 3}, outbox_deadline: 1_000) + assert :ok = Outbox.push(group, target_a, 0, {:message, 1}, outbox_deadline: 1_000) + assert :ok = Outbox.push(group, target_b, 0, {:message, 2}, outbox_deadline: 1_000) + assert :ok = Outbox.push(group, target_a, 0, {:message, 3}, outbox_deadline: 1_000) batches = for _ <- 1..2, into: %{} do - assert_receive {:outbox_batch, ^group, 0, target, frames, deadline}, 1_000 + assert_receive {:outbox_batch, ^group, 0, target, messages, deadline}, 1_000 assert deadline > Outbox.monotonic_ms() - {target, frames} + {target, messages} end assert batches == %{ - target_a => [{:frame, 1}, {:frame, 3}], - target_b => [{:frame, 2}] + target_a => [{:message, 1}, {:message, 3}], + target_b => [{:message, 2}] } end @@ -66,21 +66,21 @@ defmodule Group.ReplicaTransportOutboxTest do backend_sleep: 200 ) - assert :ok = Outbox.try_send(group, target, 0, :first, outbox_deadline: 1_000) + assert :ok = Outbox.push(group, target, 0, :first, outbox_deadline: 1_000) assert_receive {:outbox_batch, ^group, 0, ^target, [:first], _deadline}, 1_000 caller = self() spawn(fn -> - result = Outbox.try_send(group, target, 0, :expires_behind_backend, outbox_deadline: 10) - send(caller, {:try_send_returned, result}) + result = Outbox.push(group, target, 0, :expires_behind_backend, outbox_deadline: 10) + send(caller, {:push_returned, result}) end) - assert_receive {:try_send_returned, :ok}, 100 + assert_receive {:push_returned, :ok}, 100 refute_receive {:outbox_batch, ^group, 0, ^target, [:expires_behind_backend], _deadline}, 300 end - test "expired frames and backend backpressure are dropped without local retries" do + test "expired messages and backend backpressure are dropped without local retries" do expired_group = unique_group(:expired) target = :"outbox-expired@test" @@ -89,7 +89,7 @@ defmodule Group.ReplicaTransportOutboxTest do ) assert :ok = - Outbox.try_send(expired_group, target, 0, :expired, outbox_deadline: 5) + Outbox.push(expired_group, target, 0, :expired, outbox_deadline: 5) refute_receive {:outbox_batch, ^expired_group, 0, ^target, [:expired], _deadline}, 100 @@ -100,12 +100,12 @@ defmodule Group.ReplicaTransportOutboxTest do backend_result: :busy ) - assert :ok = Outbox.try_send(busy_group, target, 0, :busy, outbox_deadline: 1_000) + assert :ok = Outbox.push(busy_group, target, 0, :busy, outbox_deadline: 1_000) assert_receive {:outbox_batch, ^busy_group, 0, ^target, [:busy], _deadline}, 1_000 refute_receive {:outbox_batch, ^busy_group, 0, ^target, [:busy], _deadline}, 100 end - test "complete inbound batches use one authenticated local delivery" do + test "complete incoming batches use one local mailbox operation" do group = unique_group(:deliver) source_node = :"outbox-source@test" parent = self() @@ -124,7 +124,7 @@ defmodule Group.ReplicaTransportOutboxTest do assert_receive :receiver_ready assert :ok = - Group.Replica.Transport.deliver_batch( + Group.Replica.Transport.incoming_batch( group, source_node, 0, diff --git a/test/support/controlled_replica_transport.ex b/test/support/controlled_replica_transport.ex index 70d7d48..9658457 100644 --- a/test/support/controlled_replica_transport.ex +++ b/test/support/controlled_replica_transport.ex @@ -19,15 +19,15 @@ defmodule Group.ControlledReplicaTransport do end @impl true - def try_send(group, target_node, shard, frame, opts) do + def outgoing(group, target_node, shard, message, opts) do case :persistent_term.get({__MODULE__, group, :mode}, :capture) do :capture -> controller = Keyword.fetch!(opts, :controller) - send(controller, {__MODULE__, :frame, group, node(), target_node, shard, frame}) + send(controller, {__MODULE__, :message, group, node(), target_node, shard, message}) :ok :pass -> - deliver(group, target_node, shard, frame) + forward(group, target_node, shard, message) :busy -> :busy @@ -37,9 +37,9 @@ defmodule Group.ControlledReplicaTransport do end end - defp deliver(group, target_node, shard, frame) do + defp forward(group, target_node, shard, replica_message) do destination = {Group.Replica.shard_name(group, shard), target_node} - message = {:group_replica_frame, node(), frame} + message = {:group_replica_frame, node(), replica_message} case :erlang.send_nosuspend(destination, message, [:noconnect]) do true -> :ok diff --git a/test/support/replica_model_scheduler.ex b/test/support/replica_model_scheduler.ex index b698324..12955db 100644 --- a/test/support/replica_model_scheduler.ex +++ b/test/support/replica_model_scheduler.ex @@ -5,10 +5,10 @@ defmodule Group.ReplicaModelScheduler do defmodule Envelope do @moduledoc false - defstruct [:id, :source, :target, :shard, :frame] + defstruct [:id, :source, :target, :shard, :message] end - defstruct [:name, :nodes, :model, :group_opts, owners: %{}, queue: [], next_frame_id: 1] + defstruct [:name, :nodes, :model, :group_opts, owners: %{}, queue: [], next_message_id: 1] def new(name, nodes, group_opts \\ []) do %__MODULE__{ @@ -247,18 +247,18 @@ defmodule Group.ReplicaModelScheduler do def drain(%__MODULE__{} = state, wait_ms \\ 0) do receive do - {ControlledReplicaTransport, :frame, group, source, target, shard, frame} + {ControlledReplicaTransport, :message, group, source, target, shard, message} when group == state.name -> envelope = %Envelope{ - id: state.next_frame_id, + id: state.next_message_id, source: source, target: target, shard: shard, - frame: frame + message: message } drain( - %{state | queue: state.queue ++ [envelope], next_frame_id: state.next_frame_id + 1}, + %{state | queue: state.queue ++ [envelope], next_message_id: state.next_message_id + 1}, wait_ms ) after @@ -391,8 +391,8 @@ defmodule Group.ReplicaModelScheduler do TestCluster.rpc!( envelope.target, Group.Replica.Transport, - :deliver, - [state.name, envelope.source, envelope.shard, envelope.frame] + :incoming, + [state.name, envelope.source, envelope.shard, envelope.message] ) TestCluster.flush_shards(envelope.target, state.name) @@ -505,7 +505,7 @@ defmodule Group.ReplicaModelScheduler do defp drain_transport_messages(name) do receive do - {ControlledReplicaTransport, :frame, ^name, _source, _target, _shard, _frame} -> + {ControlledReplicaTransport, :message, ^name, _source, _target, _shard, _message} -> drain_transport_messages(name) after 0 -> :ok diff --git a/test/support/test_replica_transport.ex b/test/support/test_replica_transport.ex index 5eef3d0..4efc7b2 100644 --- a/test/support/test_replica_transport.ex +++ b/test/support/test_replica_transport.ex @@ -36,7 +36,7 @@ defmodule Group.TestReplicaTransport do end @impl true - def try_send(group, target_node, shard, frame, _opts) do + def outgoing(group, target_node, shard, message, _opts) do case :persistent_term.get({__MODULE__, group}, :pass) do :drop -> :ok @@ -45,42 +45,44 @@ defmodule Group.TestReplicaTransport do :busy :duplicate -> - deliver(group, target_node, shard, frame) - deliver(group, target_node, shard, frame) + forward(group, target_node, shard, message) + forward(group, target_node, shard, message) {:drop_types, types} -> - if frame_type(frame) in types, do: :ok, else: deliver(group, target_node, shard, frame) + if message_type(message) in types, + do: :ok, + else: forward(group, target_node, shard, message) {:duplicate_types, types} -> - if frame_type(frame) in types do - deliver(group, target_node, shard, frame) - deliver(group, target_node, shard, frame) + if message_type(message) in types do + forward(group, target_node, shard, message) + forward(group, target_node, shard, message) else - deliver(group, target_node, shard, frame) + forward(group, target_node, shard, message) end {:delay_types, delays, default_delay} -> - delay = Map.get(delays, frame_type(frame), default_delay) - delayed_deliver(group, target_node, shard, frame, delay) + delay = Map.get(delays, message_type(message), default_delay) + delayed_forward(group, target_node, shard, message, delay) {:capture_drop, types} -> - if frame_type(frame) in types, do: capture(group, target_node, shard, frame) + if message_type(message) in types, do: capture(group, target_node, shard, message) :ok {:capture_pass, types} -> - if frame_type(frame) in types, do: capture(group, target_node, shard, frame) - deliver(group, target_node, shard, frame) + if message_type(message) in types, do: capture(group, target_node, shard, message) + forward(group, target_node, shard, message) {:chaos, opts} -> - chaos_deliver(group, target_node, shard, frame, opts) + chaos_forward(group, target_node, shard, message, opts) :pass -> - deliver(group, target_node, shard, frame) + forward(group, target_node, shard, message) end end - defp chaos_deliver(group, target_node, shard, frame, opts) do - hash = :erlang.phash2({target_node, shard, frame}, 1_000_003) + defp chaos_forward(group, target_node, shard, message, opts) do + hash = :erlang.phash2({target_node, shard, message}, 1_000_003) drop_every = Keyword.get(opts, :drop_every, 0) duplicate_every = Keyword.get(opts, :duplicate_every, 0) max_delay = Keyword.get(opts, :max_delay, 0) @@ -91,43 +93,43 @@ defmodule Group.TestReplicaTransport do duplicate_every > 0 and rem(hash, duplicate_every) == 0 -> delay = if max_delay > 0, do: rem(hash, max_delay + 1), else: 0 - delayed_deliver(group, target_node, shard, frame, delay) - delayed_deliver(group, target_node, shard, frame, max(max_delay - delay, 0)) + delayed_forward(group, target_node, shard, message, delay) + delayed_forward(group, target_node, shard, message, max(max_delay - delay, 0)) true -> delay = if max_delay > 0, do: rem(hash, max_delay + 1), else: 0 - delayed_deliver(group, target_node, shard, frame, delay) + delayed_forward(group, target_node, shard, message, delay) end end - defp delayed_deliver(group, target_node, shard, frame, delay) when delay <= 0, - do: deliver(group, target_node, shard, frame) + defp delayed_forward(group, target_node, shard, message, delay) when delay <= 0, + do: forward(group, target_node, shard, message) - defp delayed_deliver(group, target_node, shard, frame, delay) do + defp delayed_forward(group, target_node, shard, message, delay) do source_node = node() spawn(fn -> receive do after - delay -> deliver(group, target_node, shard, frame, source_node) + delay -> forward(group, target_node, shard, message, source_node) end end) :ok end - defp capture(group, target_node, shard, frame) do + defp capture(group, target_node, shard, message) do key = {__MODULE__, group, :captured} captured = :persistent_term.get(key, []) - :persistent_term.put(key, [{target_node, shard, frame} | captured]) + :persistent_term.put(key, [{target_node, shard, message} | captured]) end - defp deliver(group, target_node, shard, frame), - do: deliver(group, target_node, shard, frame, node()) + defp forward(group, target_node, shard, message), + do: forward(group, target_node, shard, message, node()) - defp deliver(group, target_node, shard, frame, source_node) do + defp forward(group, target_node, shard, replica_message, source_node) do destination = {Group.Replica.shard_name(group, shard), target_node} - message = {:group_replica_frame, source_node, frame} + message = {:group_replica_frame, source_node, replica_message} case :erlang.send_nosuspend(destination, message, [:noconnect]) do true -> :ok @@ -135,6 +137,6 @@ defmodule Group.TestReplicaTransport do end end - defp frame_type(frame) when is_tuple(frame), do: elem(frame, 0) - defp frame_type(_frame), do: :unknown + defp message_type(message) when is_tuple(message), do: elem(message, 0) + defp message_type(_message), do: :unknown end From 1f81c498151633ce1a3694a6f4be62fa48ad6807 Mon Sep 17 00:00:00 2001 From: Chris McCord Date: Wed, 12 Aug 2026 18:21:09 +0000 Subject: [PATCH 07/16] Move replica transports to public namespace --- CHANGELOG.md | 16 ++- CLAUDE.md | 26 ++-- README.md | 56 ++++----- lib/group.ex | 4 +- lib/group/replica.ex | 116 +++++++++--------- lib/group/replica/data.ex | 72 +++++------ lib/group/replica/snapshot.ex | 2 +- .../replica/{protocol.ex => wire_protocol.ex} | 2 +- lib/group/supervisor.ex | 6 +- lib/group/{replica => }/transport.ex | 48 ++------ lib/group/transport/dist_erl.ex | 32 +++++ lib/group/{replica => }/transport/outbox.ex | 24 ++-- test/README.md | 5 +- test/distributed_test.exs | 63 +++++----- test/group_test.exs | 8 +- test/jepsen/Dockerfile.node | 1 + test/jepsen/README.md | 2 +- test/jepsen/node.exs | 39 +++--- test/mutation/run.exs | 8 +- test/replica_snapshot_distributed_test.exs | 15 ++- test/replica_snapshot_test.exs | 4 +- test/replica_transport_outbox_test.exs | 4 +- test/support/controlled_replica_transport.ex | 2 +- test/support/replica_model_scheduler.ex | 2 +- test/support/test_cluster.ex | 14 +-- test/support/test_replica_transport.ex | 2 +- .../support/test_tcp_transport.ex | 58 +++------ 27 files changed, 304 insertions(+), 327 deletions(-) rename lib/group/replica/{protocol.ex => wire_protocol.ex} (96%) rename lib/group/{replica => }/transport.ex (73%) create mode 100644 lib/group/transport/dist_erl.ex rename lib/group/{replica => }/transport/outbox.ex (92%) rename lib/group/replica/transport/tcp.ex => test/support/test_tcp_transport.ex (84%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45bec76..47fdd90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,13 +2,21 @@ - Add layered anti-entropy qualification: three-node StreamData lifecycle models, seeded adversarial transport histories, TLA+ models for convergence, chunk assembly, and permanent peer eviction, plus a Docker-backed Jepsen - oracle across distribution, sideband TCP, and lossy/reordering transports. + oracle across distribution, a test-only sideband TCP lane, and + lossy/reordering transports. `mix test` is the every-PR ExUnit/property/checker gate and `mix test.soak` runs the six-profile nightly/release campaign. -- **Breaking**: the replica transport boundary now names logical direction +- **Breaking**: move the replica transport API from + `Group.Replica.Transport.*` to `Group.Transport.*`; the default adapter is + now `Group.Transport.DistErl`. The boundary also names logical direction rather than implementation mechanics: adapters implement `outgoing/5`, - sideband adapters use `Group.Replica.Transport.Outbox.push/5`, and receiving - adapters call `incoming/4` or `incoming_batch/4`. + sideband adapters use `Group.Transport.Outbox.push/5`, and receiving adapters + call `Group.Transport.incoming/4` or `incoming_batch/4`. No compatibility + aliases are provided. +- Rename the internal replica wire helper from `Group.Replica.Protocol` to + `Group.Replica.WireProtocol` to avoid overloading Elixir protocol terminology. + The standalone TCP adapter is retained only as hidden test infrastructure; + Group ships the transport contract, dist-Erlang adapter, and outbox helper. - **Breaking**: replica protocol v2 splits exact snapshots into transport-neutral, byte-targeted chunks (`1 MiB` by default). Receivers stage chunks in shard-owned private ETS and advance the stream cursor only after an diff --git a/CLAUDE.md b/CLAUDE.md index 15ba7ad..fe5f774 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -18,12 +18,13 @@ lib/ group/peer_reconnect.ex — bounded retry for busy dispatch links group/replica.ex — sharded writes, control, AE, projection group/replica/data.ex — ETS owner, journal, authority, indexes - group/replica/protocol.ex — stream identity and mutation helpers + group/replica/wire_protocol.ex — wire version, stream identity, mutations group/replica/snapshot.ex — byte-targeted snapshot chunks/staging - group/replica/transport.ex — replica transport contract + dist adapter - group/replica/transport/outbox.ex — optional lossy sideband outboxes - group/replica/transport/tcp.ex — included sideband TCP adapter + group/transport.ex — replica transport contract + group/transport/dist_erl.ex — default dist-Erlang adapter + group/transport/outbox.ex — optional lossy sideband outboxes test/ + support/test_tcp_transport.ex — test-only independent-socket adapter replica_model_property_test.exs — shrinkable real-node lifecycle model replica_adversarial_test.exs — seeded three-node transport chaos replica_snapshot_* — chunk/assembly failure coverage @@ -165,12 +166,11 @@ All cross-node Group control sends use adapter sends directly the same way and adds no local hop. `:busy` and `:disconnected` mean “drop this message”; periodic anti-entropy repairs it. -A sideband adapter may use one local `Group.Replica.Transport.Outbox` per -shard. Outboxes batch by peer, impose deadlines, and run bounded socket work -outside Group shards. Queue overflow, expiry, or socket backpressure drops the -batch. The included TCP adapter adds bounded per-peer writer queues and -capability-authenticated ingress while distribution still authenticates node -identity and carries authority. TCP is not encrypted. +A sideband adapter may use one local `Group.Transport.Outbox` per shard. +Outboxes batch by peer, impose deadlines, and run bounded socket work outside +Group shards. Queue overflow, expiry, or socket backpressure drops the batch. +The test suite's hidden TCP adapter exercises a genuinely independent socket +lane; it is validation infrastructure, not a supported production transport. Transport ordering is not required for correctness. Per-shard ordered delivery is a fast path; stream sequences reject duplicate/out-of-order data, and @@ -238,8 +238,10 @@ FIFO local-request turn to prevent replica pressure from starving callers. 2. Exact snapshots replace one origin slice; they are never additive merges. 3. Authority requires generation, exact epoch revision, and installed lane readiness. Observed heartbeats/controls are not exact authority. -4. A stale generation, epoch, lane, shard, transitive pid, or unauthenticated - source is rejected before applying replica data. +4. A stale generation, epoch, lane, shard, transitive pid, or stream whose + origin differs from the transport-reported source is rejected before + applying replica data. Group trusts the adapter's source identity; + authenticating a sideband peer belongs to that transport. 5. Registry claims are retained per origin until that origin deletes them or is retired; the visible winner is reconstructible from remaining claims. 6. Only an owner node monitors, retires, or exits its member processes. diff --git a/README.md b/README.md index 51ac159..43c6cb4 100644 --- a/README.md +++ b/README.md @@ -246,7 +246,7 @@ All operations are **eventually consistent**: busy_dist_retry_attempts: 300, busy_dist_retry_interval: 1_000, replicated_pg_receiver_local_request_quota: 8, - replica_transport: Group.Replica.Transport.Distribution, + replica_transport: Group.Transport.DistErl, replicated_oplog_max_entries: 65_536, replicated_snapshot_chunk_target_bytes: 1_048_576, replicated_anti_entropy_interval: 1_000, @@ -296,12 +296,12 @@ All operations are **eventually consistent**: queued local shard requests drained per fairness turn while replica data or cluster controls are busy. Defaults to 8. - **`replica_transport`** — a module implementing - `Group.Replica.Transport`, or `{module, opts}`. The default adapter uses - `:erlang.send_nosuspend/3`; adapters must return promptly with `:ok`, `:busy`, - or `:disconnected`. Dropped and busy messages are repaired by anti-entropy. - `Group.Replica.Transport.TCP` is an included sideband adapter with local - per-shard batching and bounded per-peer writer queues; its socket owners are - separate processes, so socket backpressure cannot block a Group shard. + `Group.Transport`, or `{module, opts}`. The default + `Group.Transport.DistErl` adapter uses `:erlang.send_nosuspend/3`; adapters + must return promptly with `:ok`, `:busy`, or `:disconnected`. Dropped and busy + messages are repaired by anti-entropy. Sideband implementations can use + `Group.Transport.Outbox` to move bounded batching and socket work outside the + Group shard. - **`replicated_oplog_max_entries`** — maximum retained replica records per shard across all local streams. Defaults to 65,536. Pruning never waits for peer acknowledgements; a peer behind the retained floor receives an exact @@ -468,36 +468,20 @@ Per-shard ordered delivery is still a useful fast path. Cross-stream order is not a correctness dependency; cluster epochs reject data racing a disconnect or reconnect, and generation fencing rejects data from a restarted origin. An alternative sideband adapter passes incoming messages to -`Group.Replica.Transport.incoming/4` locally. - -For example, replica data can use the included sideband TCP adapter while +`Group.Transport.incoming/4` locally. Configure a custom adapter while authority and membership remain on dist Erlang: ```elixir replica_transport: - {Group.Replica.Transport.TCP, - [ - ip: {0, 0, 0, 0}, - advertised_ip: {10, 0, 1, 12}, - port: 44_321, - max_queue: 1_024, - outbox_batch_size: 64, - outbox_batch_bytes: 1_048_576, - outbox_flush_interval: 1, - outbox_deadline: 100 - ]} + {MyApp.GroupTransport, + [outbox_batch_size: 64, outbox_batch_bytes: 1_048_576, + outbox_flush_interval: 1, outbox_deadline: 100]} ``` -Each node advertises its own reachable address. TCP frames are capability -authenticated by the dist-Erlang hello but are not encrypted, so use a trusted -network or place the connection behind TLS. The adapter deliberately has no -control/data ordering relationship; the generation/epoch lane barrier and -stream sequence checks supply correctness. - -The default distribution adapter still sends directly to the remote shard and +The default `Group.Transport.DistErl` adapter sends directly to the remote shard and does not pay for a local outbox. Sideband adapters can delegate `outgoing/5` to -`Group.Replica.Transport.Outbox.push/5` and supervise one outbox per shard -with `Group.Replica.Transport.Outbox.child_spec/1`. An outbox groups messages by +`Group.Transport.Outbox.push/5` and supervise one outbox per shard +with `Group.Transport.Outbox.child_spec/1`. An outbox groups messages by target and invokes the adapter's `send_batch/4` callback. Calls that expire or return `:busy`/`:disconnected` are dropped without a local retry; the next anti-entropy exchange repairs them. @@ -505,11 +489,13 @@ anti-entropy exchange repairs them. A message-oriented backend fits this callback shape by obtaining a connection once from `init_outbox/3`, then sending each `send_batch/4` result to a registered incoming name on the target node. Queue pressure maps to `:busy` and -a missing session maps to `:disconnected`. The adapter passes the trusted peer's -source node alongside each message. Exact snapshots are already bounded by -Group. A transport with a smaller maximum frame may additionally segment an -encoded batch, but it must completely reassemble that batch before calling -`Group.Replica.Transport.incoming_batch/4`. +a missing session maps to `:disconnected`. The adapter passes its trusted peer +identity as the source node; Group verifies that stream origins and member pids +match that identity but does not authenticate the sideband connection itself. +Exact snapshots are already bounded by Group. A transport with a smaller +maximum frame may additionally segment an encoded batch, but it must completely +reassemble that batch before calling +`Group.Transport.incoming_batch/4`. ### Named Cluster TTL Leases diff --git a/lib/group.ex b/lib/group.ex index 17a3df6..284b8da 100644 --- a/lib/group.ex +++ b/lib/group.ex @@ -233,9 +233,9 @@ defmodule Group do local shard requests drained in each fairness turn, including while replica data or cluster controls are busy (default: `8`) - `:replica_transport` — replica data transport module or `{module, opts}` tuple. - Defaults to `Group.Replica.Transport.Distribution`. The transport must be + Defaults to `Group.Transport.DistErl`. The transport must be nonblocking and may return `:busy`; anti-entropy repairs dropped messages. - Sideband transports can use `Group.Replica.Transport.Outbox` for lossy, + Sideband transports can use `Group.Transport.Outbox` for lossy, batched, per-shard isolation without adding a hop to the default transport. - `:replicated_oplog_max_entries` — maximum retained replica records per shard before old prefixes are pruned and lagging peers require a snapshot diff --git a/lib/group/replica.ex b/lib/group/replica.ex index d70b462..39ea943 100644 --- a/lib/group/replica.ex +++ b/lib/group/replica.ex @@ -10,7 +10,7 @@ defmodule Group.Replica do @anti_entropy_timer :group_replica_anti_entropy @local_request_tag :group_local_request @local_reply_tag :group_local_reply - @protocol_version Group.Replica.Protocol.version() + @protocol_version Group.Replica.WireProtocol.version() _archdoc = ~S""" Sharded control process for local writes, replica transport, anti-entropy, @@ -55,7 +55,7 @@ defmodule Group.Replica do creating remote process monitors. A generation/epoch-revision mismatch requests a fresh authoritative hello. - Replica state uses the configured Group.Replica.Transport: + Replica state uses the configured Group.Transport: - heads advertises {stream, retained_floor, head}. - delta_batch carries one or more contiguous stream runs. @@ -126,7 +126,7 @@ defmodule Group.Replica do require Logger - alias Group.Replica.{Data, Protocol, Snapshot} + alias Group.Replica.{Data, Snapshot, WireProtocol} defstruct [ :name, @@ -387,7 +387,7 @@ defmodule Group.Replica do end) cond do - version != Protocol.version() or transport_id != state.replica_transport.id() -> + version != WireProtocol.version() or transport_id != state.replica_transport.id() -> Logger.error( "#{log_prefix_shard(state)} incompatible replica protocol/transport from #{inspect(remote_node)}" ) @@ -448,7 +448,7 @@ defmodule Group.Replica do state = flush_pending_replicated_message_barrier(state) remote_node = node(remote_pid) - if version == Protocol.version() and transport_id == state.replica_transport.id() do + if version == WireProtocol.version() and transport_id == state.replica_transport.id() do if function_exported?(state.replica_transport, :peer_up, 4) do :ok = state.replica_transport.peer_up( @@ -737,14 +737,14 @@ defmodule Group.Replica do state = cond do - version == Protocol.version() and + version == WireProtocol.version() and replica_authority_current?(state, remote_node, generation, epoch_revision) and replica_view_current?(state, remote_node) -> state |> put_remote_shard(remote_node, remote_pid) |> touch_replica_peer(remote_node) - version == Protocol.version() and + version == WireProtocol.version() and replica_authority_current?(state, remote_node, generation, epoch_revision) -> state @@ -1781,7 +1781,7 @@ defmodule Group.Replica do end defp append_local_replica_record(state, op) do - cluster = Protocol.op_cluster(op) + cluster = WireProtocol.op_cluster(op) case Data.local_stream_id(state.name, state.shard_index, cluster) do nil -> @@ -2713,7 +2713,7 @@ defmodule Group.Replica do {stream_id, first_seq, records, head} end) - outgoing_replica_message(state, target_node, {:delta_batch, Protocol.version(), runs}) + outgoing_replica_message(state, target_node, {:delta_batch, WireProtocol.version(), runs}) end defp group_broadcast_ops_by_target(ops, state, cluster_fun) do @@ -2772,7 +2772,7 @@ defmodule Group.Replica do do: registry_op_cluster(op) defp sequenced_op_cluster({:sequenced, stream_id, _seq, _mutations}), - do: Protocol.stream_cluster(stream_id) + do: WireProtocol.stream_cluster(stream_id) defp replicated_op_for_active_cluster?(name, op, cluster_fun) when is_function(cluster_fun, 1) do @@ -2922,14 +2922,14 @@ defmodule Group.Replica do send_remote_control_message( state, target_node, - {:replica_hello, self(), Protocol.version(), generation, epoch_revision, cluster_epochs, - state.replica_transport.id(), descriptor} + {:replica_hello, self(), WireProtocol.version(), generation, epoch_revision, + cluster_epochs, state.replica_transport.id(), descriptor} ) else send_remote_shard_message( state, target_node, - {:replica_lane_hello, self(), Protocol.version(), Data.generation(state.name), + {:replica_lane_hello, self(), WireProtocol.version(), Data.generation(state.name), Data.local_cluster_epoch_revision(state.name), state.replica_transport.id(), descriptor} ) end @@ -3104,7 +3104,7 @@ defmodule Group.Replica do send_remote_shard_message( acc, target_node, - {:replica_heartbeat, self(), Protocol.version(), Data.generation(acc.name), + {:replica_heartbeat, self(), WireProtocol.version(), Data.generation(acc.name), Data.local_cluster_epoch_revision(acc.name)} ) @@ -3231,7 +3231,7 @@ defmodule Group.Replica do if heads == [] do state else - outgoing_replica_message(state, target_node, {:heads, Protocol.version(), heads}) + outgoing_replica_message(state, target_node, {:heads, WireProtocol.version(), heads}) end end @@ -3263,27 +3263,27 @@ defmodule Group.Replica do end defp replica_stream_target?(state, stream_id, target_node) do - Protocol.stream_name(stream_id) == state.name and - Protocol.stream_origin(stream_id) == node() and - Protocol.stream_shard(stream_id) == state.shard_index and - Protocol.stream_generation(stream_id) == Data.generation(state.name) and - Protocol.stream_epoch(stream_id) == - Data.local_cluster_epoch(state.name, Protocol.stream_cluster(stream_id)) and - case Protocol.stream_cluster(stream_id) do + WireProtocol.stream_name(stream_id) == state.name and + WireProtocol.stream_origin(stream_id) == node() and + WireProtocol.stream_shard(stream_id) == state.shard_index and + WireProtocol.stream_generation(stream_id) == Data.generation(state.name) and + WireProtocol.stream_epoch(stream_id) == + Data.local_cluster_epoch(state.name, WireProtocol.stream_cluster(stream_id)) and + case WireProtocol.stream_cluster(stream_id) do nil -> Map.has_key?(state.peer_last_seen, target_node) cluster -> target_node in Data.cluster_nodes(state.name, cluster) end end defp valid_remote_stream?(state, source_node, stream_id) do - cluster = Protocol.stream_cluster(stream_id) + cluster = WireProtocol.stream_cluster(stream_id) - Protocol.stream_name(stream_id) == state.name and - Protocol.stream_origin(stream_id) == source_node and - Protocol.stream_shard(stream_id) == state.shard_index and + WireProtocol.stream_name(stream_id) == state.name and + WireProtocol.stream_origin(stream_id) == source_node and + WireProtocol.stream_shard(stream_id) == state.shard_index and replica_view_current?(state, source_node) and - Protocol.stream_generation(stream_id) == Data.remote_generation(state.name, source_node) and - Protocol.stream_epoch(stream_id) == + WireProtocol.stream_generation(stream_id) == Data.remote_generation(state.name, source_node) and + WireProtocol.stream_epoch(stream_id) == Data.remote_cluster_epoch(state.name, source_node, cluster) and (is_nil(cluster) or cluster_member?(state.name, cluster)) end @@ -3303,7 +3303,7 @@ defmodule Group.Replica do needs |> Enum.chunk_every(state.replicated_sender_buffer_size) |> Enum.reduce(state, fn chunk, acc -> - outgoing_replica_message(acc, source_node, {:needs, Protocol.version(), chunk}) + outgoing_replica_message(acc, source_node, {:needs, WireProtocol.version(), chunk}) end) end @@ -3318,8 +3318,8 @@ defmodule Group.Replica do defp handle_replica_message(state, source_node, {:need, version, stream_id, next_seq}) when version == @protocol_version do - if Protocol.stream_origin(stream_id) == node() and - Protocol.stream_shard(stream_id) == state.shard_index and + if WireProtocol.stream_origin(stream_id) == node() and + WireProtocol.stream_shard(stream_id) == state.shard_index and replica_stream_target?(state, stream_id, source_node) do send_replica_repair(state, source_node, stream_id, next_seq) else @@ -3407,7 +3407,7 @@ defmodule Group.Replica do end defp valid_snapshot_rows?(state, source_node, stream_id, reg_data, pg_data) do - cluster = Protocol.stream_cluster(stream_id) + cluster = WireProtocol.stream_cluster(stream_id) Enum.all?(reg_data, fn {key, pid, _meta, _time} when is_pid(pid) -> @@ -3534,7 +3534,7 @@ defmodule Group.Replica do state = if valid_snapshot_stream?(state, source_node, stream_id, transfer.snapshot_seq) do state = flush_pending_replicated_barrier(state) - cluster = Protocol.stream_cluster(stream_id) + cluster = WireProtocol.stream_cluster(stream_id) affected_registry_keys = Data.replace_registry_claims_for_stream_from_staging( @@ -3587,7 +3587,7 @@ defmodule Group.Replica do pg_data ) do state = flush_pending_replicated_barrier(state) - cluster = Protocol.stream_cluster(stream_id) + cluster = WireProtocol.stream_cluster(stream_id) affected_registry_keys = Data.replace_registry_claims_for_stream( @@ -3672,8 +3672,8 @@ defmodule Group.Replica do do: {Enum.reverse(acc), next_seq} defp valid_replica_mutations?(stream_id, mutations) do - origin = Protocol.stream_origin(stream_id) - cluster = Protocol.stream_cluster(stream_id) + origin = WireProtocol.stream_origin(stream_id) + cluster = WireProtocol.stream_cluster(stream_id) mutations != [] and Enum.all?(mutations, &valid_replica_mutation?(&1, cluster, origin)) end @@ -3846,7 +3846,7 @@ defmodule Group.Replica do end) |> Enum.uniq() - cluster = Protocol.stream_cluster(stream_id) + cluster = WireProtocol.stream_cluster(stream_id) Enum.reduce(keys, {state, []}, fn key, {acc, events} -> reconcile_registry_projection(acc, cluster, key, :reconcile, events) @@ -3861,15 +3861,15 @@ defmodule Group.Replica do outgoing_replica_message( state, target_node, - {:needs, Protocol.version(), [{stream_id, next_seq}]} + {:needs, WireProtocol.version(), [{stream_id, next_seq}]} ) end defp send_replica_repairs(state, target_node, needs) do {state, runs} = Enum.reduce(needs, {state, []}, fn {stream_id, next_seq}, {acc, runs} -> - if Protocol.stream_origin(stream_id) == node() and - Protocol.stream_shard(stream_id) == acc.shard_index and + if WireProtocol.stream_origin(stream_id) == node() and + WireProtocol.stream_shard(stream_id) == acc.shard_index and replica_stream_target?(acc, stream_id, target_node) do case replica_repair(acc, target_node, stream_id, next_seq) do {:run, run} -> {acc, [run | runs]} @@ -3888,7 +3888,7 @@ defmodule Group.Replica do outgoing_replica_message( state, target_node, - {:delta_batch, Protocol.version(), Enum.reverse(runs)} + {:delta_batch, WireProtocol.version(), Enum.reverse(runs)} ) end end @@ -3925,7 +3925,7 @@ defmodule Group.Replica do end defp send_replica_snapshot(state, target_node, stream_id, head) do - cluster = Protocol.stream_cluster(stream_id) + cluster = WireProtocol.stream_cluster(stream_id) reg_data = Data.registry_claims_for_stream(state.name, state.shard_index, stream_id) pg_data = Data.pg_entries_for_origin(state.name, state.shard_index, cluster, node()) @@ -3948,7 +3948,7 @@ defmodule Group.Replica do outgoing_replica_message( acc, target_node, - {:snapshot_chunk, Protocol.version(), stream_id, head, chunk_index, chunk_count, + {:snapshot_chunk, WireProtocol.version(), stream_id, head, chunk_index, chunk_count, snapshot.registry_count, snapshot.pg_count, reg_chunk, pg_chunk} ) end) @@ -4114,7 +4114,7 @@ defmodule Group.Replica do stream_ids = Enum.map(cluster_epochs, fn {cluster, epoch} -> - Protocol.stream_id( + WireProtocol.stream_id( state.name, remote_node, generation, @@ -4171,7 +4171,7 @@ defmodule Group.Replica do superseded = Enum.flat_map(current_epochs, fn {cluster, current_epoch} -> current_stream = - Protocol.stream_id( + WireProtocol.stream_id( state.name, remote_node, generation, @@ -4207,7 +4207,7 @@ defmodule Group.Replica do # rather than rebuilding the node-wide epoch map in every lane. current_epochs = streams - |> Enum.map(&Protocol.stream_cluster/1) + |> Enum.map(&WireProtocol.stream_cluster/1) |> Enum.uniq() |> Map.new(fn cluster -> {cluster, Data.remote_cluster_epoch(state.name, remote_node, cluster)} @@ -4215,9 +4215,9 @@ defmodule Group.Replica do superseded = Enum.reject(streams, fn stream_id -> - Protocol.stream_generation(stream_id) == generation and - Map.get(current_epochs, Protocol.stream_cluster(stream_id)) == - Protocol.stream_epoch(stream_id) + WireProtocol.stream_generation(stream_id) == generation and + Map.get(current_epochs, WireProtocol.stream_cluster(stream_id)) == + WireProtocol.stream_epoch(stream_id) end) purge_superseded_remote_streams(state, remote_node, current_epochs, superseded) @@ -4234,7 +4234,7 @@ defmodule Group.Replica do generation = Data.remote_generation(state.name, remote_node) superseded - |> Enum.group_by(&Protocol.stream_cluster/1) + |> Enum.group_by(&WireProtocol.stream_cluster/1) |> Enum.reduce(state, fn {cluster, cluster_streams}, acc -> affected_keys = Data.purge_registry_claims_for_streams( @@ -4265,7 +4265,7 @@ defmodule Group.Replica do current_epoch -> current_stream = - Protocol.stream_id( + WireProtocol.stream_id( state.name, remote_node, generation, @@ -4343,7 +4343,7 @@ defmodule Group.Replica do records |> Enum.reduce(%{}, fn {:sequenced, _stream_id, _seq, [op | _]} = record, acc -> - cluster = Protocol.op_cluster(op) + cluster = WireProtocol.op_cluster(op) Enum.reduce(process_down_targets(state, cluster), acc, fn target_node, inner -> Map.update(inner, target_node, [record], &[record | &1]) @@ -4520,13 +4520,13 @@ defmodule Group.Replica do end defp current_local_stream?(state, stream_id) do - cluster = Protocol.stream_cluster(stream_id) + cluster = WireProtocol.stream_cluster(stream_id) - Protocol.stream_name(stream_id) == state.name and - Protocol.stream_origin(stream_id) == node() and - Protocol.stream_generation(stream_id) == Data.generation(state.name) and - Protocol.stream_shard(stream_id) == state.shard_index and - Protocol.stream_epoch(stream_id) == Data.local_cluster_epoch(state.name, cluster) + WireProtocol.stream_name(stream_id) == state.name and + WireProtocol.stream_origin(stream_id) == node() and + WireProtocol.stream_generation(stream_id) == Data.generation(state.name) and + WireProtocol.stream_shard(stream_id) == state.shard_index and + WireProtocol.stream_epoch(stream_id) == Data.local_cluster_epoch(state.name, cluster) end defp apply_registry_claim_mutations(state, stream_id, seq, mutations) do diff --git a/lib/group/replica/data.ex b/lib/group/replica/data.ex index 43e4d47..3f26250 100644 --- a/lib/group/replica/data.ex +++ b/lib/group/replica/data.ex @@ -2,7 +2,7 @@ defmodule Group.Replica.Data do @moduledoc false use GenServer - alias Group.Replica.Protocol + alias Group.Replica.WireProtocol _archdoc = """ GenServer that owns ETS tables for all shards. @@ -375,7 +375,7 @@ defmodule Group.Replica.Data do nil epoch -> - Group.Replica.Protocol.stream_id( + Group.Replica.WireProtocol.stream_id( name, node(), generation(name), @@ -472,8 +472,8 @@ defmodule Group.Replica.Data do drop_local_stream( name, shard, - Protocol.stream_cluster(stream_id), - Protocol.stream_epoch(stream_id) + WireProtocol.stream_cluster(stream_id), + WireProtocol.stream_epoch(stream_id) ) end end) @@ -518,13 +518,13 @@ defmodule Group.Replica.Data do end defp current_local_stream?(name, shard, stream_id) do - cluster = Protocol.stream_cluster(stream_id) + cluster = WireProtocol.stream_cluster(stream_id) - Protocol.stream_name(stream_id) == name and - Protocol.stream_origin(stream_id) == node() and - Protocol.stream_generation(stream_id) == generation(name) and - Protocol.stream_shard(stream_id) == shard and - Protocol.stream_epoch(stream_id) == local_cluster_epoch(name, cluster) + WireProtocol.stream_name(stream_id) == name and + WireProtocol.stream_origin(stream_id) == node() and + WireProtocol.stream_generation(stream_id) == generation(name) and + WireProtocol.stream_shard(stream_id) == shard and + WireProtocol.stream_epoch(stream_id) == local_cluster_epoch(name, cluster) end defp await_closed_local_clusters(name, clusters, timeout, started_at) do @@ -605,7 +605,7 @@ defmodule Group.Replica.Data do {{cluster, _key, _pid}, _meta, _time, _entry_node} -> cluster end), Enum.map(:ets.tab2list(replica_cursor_table(name, shard)), fn {stream_id, _seq} -> - Protocol.stream_cluster(stream_id) + WireProtocol.stream_cluster(stream_id) end) ]) |> Enum.reject(&is_nil/1) @@ -748,7 +748,7 @@ defmodule Group.Replica.Data do def drop_local_stream(name, shard, cluster, epoch) do stream_id = - Group.Replica.Protocol.stream_id(name, node(), generation(name), shard, cluster, epoch) + Group.Replica.WireProtocol.stream_id(name, node(), generation(name), shard, cluster, epoch) append_rows = :ets.select(replica_oplog_table(name, shard), [ @@ -867,10 +867,10 @@ defmodule Group.Replica.Data do # ===================================================================== def put_registry_claim(name, shard, stream_id, seq, key, pid, meta, time) do - cluster = Group.Replica.Protocol.stream_cluster(stream_id) - origin_node = Group.Replica.Protocol.stream_origin(stream_id) - generation = Group.Replica.Protocol.stream_generation(stream_id) - epoch = Group.Replica.Protocol.stream_epoch(stream_id) + cluster = Group.Replica.WireProtocol.stream_cluster(stream_id) + origin_node = Group.Replica.WireProtocol.stream_origin(stream_id) + generation = Group.Replica.WireProtocol.stream_generation(stream_id) + epoch = Group.Replica.WireProtocol.stream_epoch(stream_id) claim_key = {cluster, key, origin_node, generation, epoch} by_key = reg_claim_by_key_table(name, shard) @@ -904,10 +904,10 @@ defmodule Group.Replica.Data do end def delete_registry_claim(name, shard, stream_id, seq, key, pid) do - cluster = Group.Replica.Protocol.stream_cluster(stream_id) - origin_node = Group.Replica.Protocol.stream_origin(stream_id) - generation = Group.Replica.Protocol.stream_generation(stream_id) - epoch = Group.Replica.Protocol.stream_epoch(stream_id) + cluster = Group.Replica.WireProtocol.stream_cluster(stream_id) + origin_node = Group.Replica.WireProtocol.stream_origin(stream_id) + generation = Group.Replica.WireProtocol.stream_generation(stream_id) + epoch = Group.Replica.WireProtocol.stream_epoch(stream_id) claim_key = {cluster, key, origin_node, generation, epoch} case :ets.lookup(reg_claim_by_key_table(name, shard), claim_key) do @@ -934,10 +934,10 @@ defmodule Group.Replica.Data do end def registry_claims_for_stream(name, shard, stream_id) do - cluster = Group.Replica.Protocol.stream_cluster(stream_id) - origin_node = Group.Replica.Protocol.stream_origin(stream_id) - generation = Group.Replica.Protocol.stream_generation(stream_id) - epoch = Group.Replica.Protocol.stream_epoch(stream_id) + cluster = Group.Replica.WireProtocol.stream_cluster(stream_id) + origin_node = Group.Replica.WireProtocol.stream_origin(stream_id) + generation = Group.Replica.WireProtocol.stream_generation(stream_id) + epoch = Group.Replica.WireProtocol.stream_epoch(stream_id) :ets.select(reg_claim_by_key_table(name, shard), [ {{{cluster, :"$1", origin_node, generation, epoch}, :"$2", :"$3", :"$4", :_}, [], @@ -946,10 +946,10 @@ defmodule Group.Replica.Data do end def replace_registry_claims_for_stream(name, shard, stream_id, snapshot_seq, claims) do - cluster = Group.Replica.Protocol.stream_cluster(stream_id) - origin_node = Group.Replica.Protocol.stream_origin(stream_id) - generation = Group.Replica.Protocol.stream_generation(stream_id) - epoch = Group.Replica.Protocol.stream_epoch(stream_id) + cluster = Group.Replica.WireProtocol.stream_cluster(stream_id) + origin_node = Group.Replica.WireProtocol.stream_origin(stream_id) + generation = Group.Replica.WireProtocol.stream_generation(stream_id) + epoch = Group.Replica.WireProtocol.stream_epoch(stream_id) existing = registry_claims_for_stream(name, shard, stream_id) Enum.each(existing, fn {key, pid, _meta, _time} -> @@ -979,10 +979,10 @@ defmodule Group.Replica.Data do staging_table, chunk_count ) do - cluster = Group.Replica.Protocol.stream_cluster(stream_id) - origin_node = Group.Replica.Protocol.stream_origin(stream_id) - generation = Group.Replica.Protocol.stream_generation(stream_id) - epoch = Group.Replica.Protocol.stream_epoch(stream_id) + cluster = Group.Replica.WireProtocol.stream_cluster(stream_id) + origin_node = Group.Replica.WireProtocol.stream_origin(stream_id) + generation = Group.Replica.WireProtocol.stream_generation(stream_id) + epoch = Group.Replica.WireProtocol.stream_epoch(stream_id) existing = registry_claims_for_stream(name, shard, stream_id) keys = @@ -1042,10 +1042,10 @@ defmodule Group.Replica.Data do streams = MapSet.new(stream_ids, fn stream_id -> { - Group.Replica.Protocol.stream_cluster(stream_id), - Group.Replica.Protocol.stream_origin(stream_id), - Group.Replica.Protocol.stream_generation(stream_id), - Group.Replica.Protocol.stream_epoch(stream_id) + Group.Replica.WireProtocol.stream_cluster(stream_id), + Group.Replica.WireProtocol.stream_origin(stream_id), + Group.Replica.WireProtocol.stream_generation(stream_id), + Group.Replica.WireProtocol.stream_epoch(stream_id) } end) diff --git a/lib/group/replica/snapshot.ex b/lib/group/replica/snapshot.ex index 13dc29a..8287407 100644 --- a/lib/group/replica/snapshot.ex +++ b/lib/group/replica/snapshot.ex @@ -47,7 +47,7 @@ defmodule Group.Replica.Snapshot do # empty list. Reserve that once for each domain. The large chunk integers # ensure every practical index/count uses no more space than this envelope. :erlang.external_size( - {:snapshot_chunk, Group.Replica.Protocol.version(), stream_id, snapshot_seq, + {:snapshot_chunk, Group.Replica.WireProtocol.version(), stream_id, snapshot_seq, @max_compact_chunk_count, @max_compact_chunk_count, registry_count, pg_count, [], []} ) + 10 end diff --git a/lib/group/replica/protocol.ex b/lib/group/replica/wire_protocol.ex similarity index 96% rename from lib/group/replica/protocol.ex rename to lib/group/replica/wire_protocol.ex index a79d983..b8e1908 100644 --- a/lib/group/replica/protocol.ex +++ b/lib/group/replica/wire_protocol.ex @@ -1,4 +1,4 @@ -defmodule Group.Replica.Protocol do +defmodule Group.Replica.WireProtocol do @moduledoc false @version 2 diff --git a/lib/group/supervisor.ex b/lib/group/supervisor.ex index df2e97d..3c66484 100644 --- a/lib/group/supervisor.ex +++ b/lib/group/supervisor.ex @@ -44,9 +44,9 @@ defmodule Group.Supervisor do replica_transport = opts - |> Keyword.get(:replica_transport, Group.Replica.Transport.Distribution) - |> Group.Replica.Transport.normalize() - |> Group.Replica.Transport.validate!() + |> Keyword.get(:replica_transport, Group.Transport.DistErl) + |> Group.Transport.normalize() + |> Group.Transport.validate!() replicated_oplog_max_entries = positive_integer_opt(opts, :replicated_oplog_max_entries, 65_536) diff --git a/lib/group/replica/transport.ex b/lib/group/transport.ex similarity index 73% rename from lib/group/replica/transport.ex rename to lib/group/transport.ex index 46bb532..30eba2d 100644 --- a/lib/group/replica/transport.ex +++ b/lib/group/transport.ex @@ -1,4 +1,4 @@ -defmodule Group.Replica.Transport do +defmodule Group.Transport do @moduledoc """ Transport contract for Group replica data. @@ -10,7 +10,9 @@ defmodule Group.Replica.Transport do Erlang distribution remains Group's control plane and supplies the stable node identity used here. A sideband adapter can use its `descriptor/2` in the control hello to exchange endpoints and pass incoming messages to - `incoming/4` or `incoming_batch/4`. + `incoming/4` or `incoming_batch/4`. Group trusts the `source_node` supplied + by the adapter and validates stream origins and member pids against it; peer + authentication, when needed, belongs to the transport. Adapters do not need to preserve ordering. Group serializes writes per shard and sequences each origin/generation/shard/cluster/epoch stream; receivers @@ -18,7 +20,7 @@ defmodule Group.Replica.Transport do traffic and is therefore the preferred fast path. A sideband implementation can delegate `outgoing/5` to - `Group.Replica.Transport.Outbox.push/5`. That adds one local send only for + `Group.Transport.Outbox.push/5`. That adds one local send only for the configured sideband transport; the default distribution adapter retains its direct remote `:erlang.send_nosuspend/3` path. """ @@ -52,8 +54,9 @@ defmodule Group.Replica.Transport do @doc """ Passes an incoming replica message to the corresponding local shard. - This is a local mailbox operation. Stream generation, epoch, group, shard, - and origin are validated by the replica. + This is a local mailbox operation. `source_node` is the trusted peer identity + established by the adapter. Stream generation, epoch, group, shard, origin, + and member-pid ownership are validated by the replica. """ def incoming(group, source_node, shard, message) when is_atom(group) and is_atom(source_node) and is_integer(shard) and shard >= 0 do @@ -70,7 +73,8 @@ defmodule Group.Replica.Transport do A finite-message transport may segment the encoded batch on the wire, but it must reassemble every segment before calling this function. Group never - observes or applies a partial batch. + observes or applies a partial batch. `source_node` has the same trusted-peer + meaning as in `incoming/4`. """ def incoming_batch(group, source_node, shard, messages) when is_atom(group) and is_atom(source_node) and is_integer(shard) and shard >= 0 and @@ -104,35 +108,3 @@ defmodule Group.Replica.Transport do transport end end - -defmodule Group.Replica.Transport.Distribution do - @moduledoc """ - Default nonblocking replica transport over Erlang distribution. - - Messages are sent directly to the matching remote shard with - `:erlang.send_nosuspend/3` and `:noconnect`, so the caller never waits for a - busy distribution socket and never initiates a connection. A busy or absent - link returns `:busy`; Group drops that message and repairs it through periodic - anti-entropy. - """ - @behaviour Group.Replica.Transport - - alias Group.Replica - - @impl true - def id, do: :erlang_distribution - - @impl true - def descriptor(_group, _opts), do: :erlang_distribution - - @impl true - def outgoing(group, target_node, shard, replica_message, _opts) do - destination = {Replica.shard_name(group, shard), target_node} - message = {:group_replica_frame, node(), replica_message} - - case :erlang.send_nosuspend(destination, message, [:noconnect]) do - true -> :ok - false -> :busy - end - end -end diff --git a/lib/group/transport/dist_erl.ex b/lib/group/transport/dist_erl.ex new file mode 100644 index 0000000..9812fb5 --- /dev/null +++ b/lib/group/transport/dist_erl.ex @@ -0,0 +1,32 @@ +defmodule Group.Transport.DistErl do + @moduledoc """ + Default nonblocking replica transport over Erlang distribution. + + Messages are sent directly to the matching remote shard with + `:erlang.send_nosuspend/3` and `:noconnect`, so the caller never waits for a + busy distribution socket and never initiates a connection. A busy or absent + link returns `:busy`; Group drops that message and repairs it through periodic + anti-entropy. + """ + + @behaviour Group.Transport + + alias Group.Replica + + @impl true + def id, do: :erlang_distribution + + @impl true + def descriptor(_group, _opts), do: :erlang_distribution + + @impl true + def outgoing(group, target_node, shard, replica_message, _opts) do + destination = {Replica.shard_name(group, shard), target_node} + message = {:group_replica_frame, node(), replica_message} + + case :erlang.send_nosuspend(destination, message, [:noconnect]) do + true -> :ok + false -> :busy + end + end +end diff --git a/lib/group/replica/transport/outbox.ex b/lib/group/transport/outbox.ex similarity index 92% rename from lib/group/replica/transport/outbox.ex rename to lib/group/transport/outbox.ex index 43c23b3..4b523f1 100644 --- a/lib/group/replica/transport/outbox.ex +++ b/lib/group/transport/outbox.ex @@ -1,9 +1,9 @@ -defmodule Group.Replica.Transport.Outbox do +defmodule Group.Transport.Outbox do @moduledoc """ Lossy per-shard outboxes for sideband replica transports. This module is an implementation helper, not a replacement for - `Group.Replica.Transport`. Distribution can continue sending directly with + `Group.Transport`. Distribution can continue sending directly with `:erlang.send_nosuspend/3`. A sideband adapter delegates `outgoing/5` to `push/5`, which performs only a local `send/2` to the matching shard outbox. @@ -15,7 +15,7 @@ defmodule Group.Replica.Transport.Outbox do A backend using this helper implements: - @behaviour Group.Replica.Transport.Outbox + @behaviour Group.Transport.Outbox def init_outbox(group, shard, opts), do: {:ok, backend_state} @@ -25,7 +25,7 @@ defmodule Group.Replica.Transport.Outbox do end The backend must pass only complete logical messages to - `Group.Replica.Transport.incoming_batch/4`. + `Group.Transport.incoming_batch/4`. ## Options @@ -46,8 +46,8 @@ defmodule Group.Replica.Transport.Outbox do reassemble those batches before local delivery. """ - @type message :: Group.Replica.Transport.message() - @type outgoing_result :: Group.Replica.Transport.outgoing_result() + @type message :: Group.Transport.message() + @type outgoing_result :: Group.Transport.outgoing_result() @type backend_state :: term() @callback init_outbox(group :: atom(), shard :: non_neg_integer(), opts :: keyword()) :: @@ -73,7 +73,7 @@ defmodule Group.Replica.Transport.Outbox do %{ id: {__MODULE__, group}, - start: {Group.Replica.Transport.Outbox.Supervisor, :start_link, [opts]}, + start: {Group.Transport.Outbox.Supervisor, :start_link, [opts]}, type: :supervisor, restart: :permanent, shutdown: :infinity @@ -119,7 +119,7 @@ defmodule Group.Replica.Transport.Outbox do end end -defmodule Group.Replica.Transport.Outbox.Supervisor do +defmodule Group.Transport.Outbox.Supervisor do @moduledoc false use Supervisor @@ -143,8 +143,8 @@ defmodule Group.Replica.Transport.Outbox.Supervisor do children = for shard <- 0..(num_shards - 1) do %{ - id: {Group.Replica.Transport.Outbox.Worker, group, shard}, - start: {Group.Replica.Transport.Outbox.Worker, :start_link, [opts, shard]}, + id: {Group.Transport.Outbox.Worker, group, shard}, + start: {Group.Transport.Outbox.Worker, :start_link, [opts, shard]}, restart: :permanent, shutdown: 5_000 } @@ -154,11 +154,11 @@ defmodule Group.Replica.Transport.Outbox.Supervisor do end end -defmodule Group.Replica.Transport.Outbox.Worker do +defmodule Group.Transport.Outbox.Worker do @moduledoc false use GenServer - alias Group.Replica.Transport.Outbox + alias Group.Transport.Outbox @default_batch_size 64 @default_batch_bytes 1_048_576 diff --git a/test/README.md b/test/README.md index ffb2986..198a961 100644 --- a/test/README.md +++ b/test/README.md @@ -61,7 +61,8 @@ The Docker-backed Jepsen harness lives in [`jepsen/`](jepsen/). It drives three independent BEAM containers through concurrent, multi-entry owner lifecycles, named-cluster epoch churn, directed/full partitions, transport session resets, and VM restarts. The same workload runs over distribution, -real sideband TCP, and a lossy/duplicating/reordering transport. After healing, +a test-only real sideband TCP lane, and a lossy/duplicating/reordering +transport. After healing, its independent oracle checks exact public views and the internal registry, PG, claim, cluster, cursor, oplog, snapshot-staging, and retired-origin invariants. Its permanent-retirement scenario proves eviction even when a peer @@ -278,7 +279,7 @@ require every advertised revision to contain exactly that many unique named epochs, and heartbeat tests prove observed revisions cannot advance the exact authority marker. Crash-window tests interrupt journal, dual-index, receive cursor, and named-cluster close updates, then require startup repair to remove -every invisible row and temporary close barrier. A three-node sideband TCP test +every invisible row and temporary close barrier. A three-node test-only TCP test disconnects one origin's real socket, prunes its oplog, reconnects it, and requires snapshot recovery without changing the third node's independent registry or PG state. diff --git a/test/distributed_test.exs b/test/distributed_test.exs index ebe8278..ed8cf1b 100644 --- a/test/distributed_test.exs +++ b/test/distributed_test.exs @@ -2452,8 +2452,8 @@ defmodule Group.DistributedTest do |> Enum.map(fn %{shard: shard, cursors: cursors} -> relevant = Enum.filter(cursors, fn {stream_id, _seq} -> - Group.Replica.Protocol.stream_origin(stream_id) == node_a and - Group.Replica.Protocol.stream_cluster(stream_id) == dropped_cluster + Group.Replica.WireProtocol.stream_origin(stream_id) == node_a and + Group.Replica.WireProtocol.stream_cluster(stream_id) == dropped_cluster end) {shard, relevant} @@ -4464,7 +4464,7 @@ defmodule Group.DistributedTest do TestCluster.rpc!(node_b, :erlang, :send, [ shard_name(name, 2), - {:replica_heartbeat, a_lane, Group.Replica.Protocol.version(), + {:replica_heartbeat, a_lane, Group.Replica.WireProtocol.version(), TestCluster.rpc!(node_a, Group.Replica.Data, :generation, [name]), latest_revision} ]) @@ -4606,7 +4606,7 @@ defmodule Group.DistributedTest do ]) :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ name, node_a, 1, @@ -4626,7 +4626,7 @@ defmodule Group.DistributedTest do TestCluster.rpc!(node_b, :erlang, :send, [ shard_name(name, 1), - {:replica_heartbeat, a_lane, Group.Replica.Protocol.version(), generation, revision} + {:replica_heartbeat, a_lane, Group.Replica.WireProtocol.version(), generation, revision} ]) TestCluster.assert_eventually(fn -> @@ -4635,7 +4635,7 @@ defmodule Group.DistributedTest do end) :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ name, node_a, 1, @@ -4827,7 +4827,7 @@ defmodule Group.DistributedTest do Enum.each(generation_frames, fn {_target, shard, frame} -> :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ name, node_a, shard, @@ -4882,7 +4882,7 @@ defmodule Group.DistributedTest do Enum.each(epoch_frames, fn {_target, shard, frame} -> :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ name, node_a, shard, @@ -4999,7 +4999,7 @@ defmodule Group.DistributedTest do node_a |> TestCluster.rpc!(Group.Replica.Data, :replica_stream_heads, [name, 0]) |> Enum.find(fn {stream_id, _floor, _head} -> - Group.Replica.Protocol.stream_cluster(stream_id) == "cold" + Group.Replica.WireProtocol.stream_cluster(stream_id) == "cold" end) assert floor > 2 @@ -5083,7 +5083,7 @@ defmodule Group.DistributedTest do invalid_key = "anti-entropy/authority/forged" invalid_frame = - {:delta_batch, Group.Replica.Protocol.version(), + {:delta_batch, Group.Replica.WireProtocol.version(), [ {stream_id, 1, [ @@ -5096,7 +5096,7 @@ defmodule Group.DistributedTest do ]} :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ name, node_a, 0, @@ -5113,7 +5113,7 @@ defmodule Group.DistributedTest do ]) == 0 :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ name, node_b, 0, @@ -5356,7 +5356,7 @@ defmodule Group.DistributedTest do TestCluster.rpc!(node_a, Group.Replica.Data, :generation, [name]) stream_id = - Group.Replica.Protocol.stream_id( + Group.Replica.WireProtocol.stream_id( name, node_a, make_ref(), @@ -5368,11 +5368,12 @@ defmodule Group.DistributedTest do mutation = {:register, nil, key, pid, meta, System.monotonic_time(), node_a} :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ name, node_a, 0, - {:delta_batch, Group.Replica.Protocol.version(), [{stream_id, 1, [{1, [mutation]}], 1}]} + {:delta_batch, Group.Replica.WireProtocol.version(), + [{stream_id, 1, [{1, [mutation]}], 1}]} ]) TestCluster.flush_shards(node_b, name) @@ -5432,7 +5433,7 @@ defmodule Group.DistributedTest do Map.fetch!(frames_by_first_seq, 2) :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ name, node_a, shard, @@ -5454,7 +5455,7 @@ defmodule Group.DistributedTest do for frame <- [first_frame, second_frame, third_frame] do :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ name, node_a, shard, @@ -5492,7 +5493,7 @@ defmodule Group.DistributedTest do for frame <- [first_frame, second_frame, third_frame] do :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ name, node_a, shard, @@ -5577,7 +5578,7 @@ defmodule Group.DistributedTest do |> then(&"anti-entropy/authority-fanout/new/#{&1}") stream_id = - Group.Replica.Protocol.stream_id( + Group.Replica.WireProtocol.stream_id( name, node_a, new_generation, @@ -5590,7 +5591,7 @@ defmodule Group.DistributedTest do {:register, nil, new_key, pid, %{generation: :new}, System.system_time(), node_a} new_frame = - {:delta_batch, Group.Replica.Protocol.version(), + {:delta_batch, Group.Replica.WireProtocol.version(), [{stream_id, 1, [{1, [new_mutation]}], 1}]} :ok = TestCluster.rpc!(node_b, :sys, :suspend, [b_lane]) @@ -5600,7 +5601,7 @@ defmodule Group.DistributedTest do # time this frame runs, but shard 1 must still reject it until its own # old-generation purge has completed. :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ name, node_a, 1, @@ -5609,7 +5610,7 @@ defmodule Group.DistributedTest do send( b_control, - {:replica_hello, a_control, Group.Replica.Protocol.version(), new_generation, 0, + {:replica_hello, a_control, Group.Replica.WireProtocol.version(), new_generation, 0, [{nil, new_generation}], Group.TestReplicaTransport.id(), Group.TestReplicaTransport.descriptor(name, [])} ) @@ -5633,7 +5634,7 @@ defmodule Group.DistributedTest do ]) == 0 :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ name, node_a, 1, @@ -5660,7 +5661,7 @@ defmodule Group.DistributedTest do name: name, shards: 2, replica_transport: - {Group.Replica.Transport.TCP, + {Group.TestTCPTransport, [ max_queue: 16, connect_timeout: 250, @@ -5683,7 +5684,7 @@ defmodule Group.DistributedTest do Enum.all?(nodes -- [source], fn target -> TestCluster.rpc!( source, - Group.Replica.Transport.TCP, + Group.TestTCPTransport, :connected?, [name, target] ) @@ -5728,15 +5729,15 @@ defmodule Group.DistributedTest do end) :ok = - TestCluster.rpc!(node_a, Group.Replica.Transport.TCP, :disconnect_peer, [name, node_b]) + TestCluster.rpc!(node_a, Group.TestTCPTransport, :disconnect_peer, [name, node_b]) old_reader = - TestCluster.rpc!(node_b, Group.Replica.Transport.TCP, :status, [name]) + TestCluster.rpc!(node_b, Group.TestTCPTransport, :status, [name]) |> get_in([:inbound, node_a]) refute TestCluster.rpc!( node_a, - Group.Replica.Transport.TCP, + Group.TestTCPTransport, :connected?, [name, node_b] ) @@ -5765,11 +5766,11 @@ defmodule Group.DistributedTest do ) :ok = - TestCluster.rpc!(node_a, Group.Replica.Transport.TCP, :reconnect_peer, [name, node_b]) + TestCluster.rpc!(node_a, Group.TestTCPTransport, :reconnect_peer, [name, node_b]) TestCluster.assert_eventually(fn -> new_reader = - TestCluster.rpc!(node_b, Group.Replica.Transport.TCP, :status, [name]) + TestCluster.rpc!(node_b, Group.TestTCPTransport, :status, [name]) |> get_in([:inbound, node_a]) is_pid(new_reader) and new_reader != old_reader @@ -5779,7 +5780,7 @@ defmodule Group.DistributedTest do fn -> TestCluster.rpc!( node_a, - Group.Replica.Transport.TCP, + Group.TestTCPTransport, :connected?, [name, node_b] ) and diff --git a/test/group_test.exs b/test/group_test.exs index f6356a5..25a5f68 100644 --- a/test/group_test.exs +++ b/test/group_test.exs @@ -2985,7 +2985,7 @@ defmodule GroupTest do Enum.to_list(1..operations_per_shard) assert Enum.all?(order_rows, fn {_append_id, stream_id, _seq} -> - Group.Replica.Protocol.stream_shard(stream_id) == shard + Group.Replica.WireProtocol.stream_shard(stream_id) == shard end) oplog_rows = @@ -3097,8 +3097,8 @@ defmodule GroupTest do :ok = Group.join(name, pg_key, %{kind: :pg}) stream_id = Group.Replica.Data.local_stream_id(name, 0, nil) - generation = Group.Replica.Protocol.stream_generation(stream_id) - epoch = Group.Replica.Protocol.stream_epoch(stream_id) + generation = Group.Replica.WireProtocol.stream_generation(stream_id) + epoch = Group.Replica.WireProtocol.stream_epoch(stream_id) :ets.delete(Group.Replica.Data.reg_by_pid_table(name, 0), {self(), nil, reg_key}) :ets.delete(Group.Replica.Data.pg_by_pid_table(name, 0), {self(), nil, pg_key}) @@ -3155,7 +3155,7 @@ defmodule GroupTest do :ok = Group.Replica.Data.add_cluster_node(name, [cluster], remote_route) stream_id = Group.Replica.Data.local_stream_id(name, 0, cluster) - old_epoch = Group.Replica.Protocol.stream_epoch(stream_id) + old_epoch = Group.Replica.WireProtocol.stream_epoch(stream_id) # Group.disconnect/3 closes authority and routing before its request # reaches every shard. Model a shard kill in that exact window. diff --git a/test/jepsen/Dockerfile.node b/test/jepsen/Dockerfile.node index 08a5f41..5f1107e 100644 --- a/test/jepsen/Dockerfile.node +++ b/test/jepsen/Dockerfile.node @@ -11,6 +11,7 @@ ENV MIX_ENV=prod COPY mix.exs mix.lock ./ COPY lib ./lib COPY test/jepsen/node.exs ./test/jepsen/node.exs +COPY test/support/test_tcp_transport.ex ./test/support/test_tcp_transport.ex RUN mix local.hex --force \ && mix deps.get --only prod \ diff --git a/test/jepsen/README.md b/test/jepsen/README.md index aeba2ce..42b13cc 100644 --- a/test/jepsen/README.md +++ b/test/jepsen/README.md @@ -22,7 +22,7 @@ prefix Jepsen: The replica lane is selectable without changing the workload or checker: - `distribution` delegates to Group's production Erlang-distribution adapter; -- `tcp` uses Group's production sideband TCP adapter while Erlang distribution +- `tcp` uses Group's hidden test-only TCP adapter while Erlang distribution remains the control plane; and - `chaos` is a local per-shard outbox which deterministically drops, duplicates, delays, and reorders replica messages. diff --git a/test/jepsen/node.exs b/test/jepsen/node.exs index c72774b..c2ef0eb 100644 --- a/test/jepsen/node.exs +++ b/test/jepsen/node.exs @@ -1,3 +1,5 @@ +Code.require_file("../support/test_tcp_transport.ex", __DIR__) + defmodule Group.Jepsen.Transport.Stats do @moduledoc false use GenServer @@ -110,7 +112,7 @@ defmodule Group.Jepsen.Transport.Common do defp transport_result(:disconnected), do: :transport_disconnected defp observe_outbox(group, shard) do - case Process.whereis(Group.Replica.Transport.Outbox.name(group, shard)) do + case Process.whereis(Group.Transport.Outbox.name(group, shard)) do pid when is_pid(pid) -> case Process.info(pid, :message_queue_len) do {:message_queue_len, length} -> Stats.observe_max(:outbox_mailbox_peak, length) @@ -125,10 +127,10 @@ end defmodule Group.Jepsen.Transport.Distribution do @moduledoc false - @behaviour Group.Replica.Transport + @behaviour Group.Transport alias Group.Jepsen.Transport.{Common, Stats} - alias Group.Replica.Transport.Distribution, as: Delegate + alias Group.Transport.DistErl, as: Delegate @impl true def id, do: Delegate.id() @@ -147,10 +149,10 @@ end defmodule Group.Jepsen.Transport.TCP do @moduledoc false - @behaviour Group.Replica.Transport + @behaviour Group.Transport alias Group.Jepsen.Transport.Common - alias Group.Replica.Transport.TCP, as: Delegate + alias Group.TestTCPTransport, as: Delegate @impl true def id, do: Delegate.id() @@ -190,7 +192,7 @@ defmodule Group.Jepsen.Transport.TCP.Supervisor do def init(opts) do children = [ {Group.Jepsen.Transport.Stats, opts}, - Group.Replica.Transport.TCP.child_spec(opts) + Group.TestTCPTransport.child_spec(opts) ] Supervisor.init(children, strategy: :one_for_one) @@ -199,7 +201,7 @@ end defmodule Group.Jepsen.Transport.Chaos do @moduledoc false - @behaviour Group.Replica.Transport + @behaviour Group.Transport alias Group.Jepsen.Transport.{Common, Stats} @@ -380,7 +382,7 @@ defmodule Group.Jepsen.Transport.Control do defp maybe_disconnect(target_node) do if profile() == :tcp do - Group.Replica.Transport.TCP.disconnect_peer(:jepsen_group, target_node) + Group.TestTCPTransport.disconnect_peer(:jepsen_group, target_node) end catch :exit, _ -> :ok @@ -388,7 +390,7 @@ defmodule Group.Jepsen.Transport.Control do defp maybe_reconnect(target_node) do if profile() == :tcp do - Group.Replica.Transport.TCP.reconnect_peer(:jepsen_group, target_node) + Group.TestTCPTransport.reconnect_peer(:jepsen_group, target_node) end catch :exit, _ -> :ok @@ -813,7 +815,7 @@ end defmodule Group.Jepsen.Invariant do @moduledoc false - alias Group.Replica.{Data, Protocol} + alias Group.Replica.{Data, WireProtocol} def snapshot(retired_nodes) do config = Group.get_config(:jepsen_group) @@ -846,7 +848,7 @@ defmodule Group.Jepsen.Invariant do shard_mailbox_max: mailbox_max(Enum.map(shards, &Group.Replica.shard_name(:jepsen_group, &1))), outbox_mailbox_max: - mailbox_max(Enum.map(shards, &Group.Replica.Transport.Outbox.name(:jepsen_group, &1))), + mailbox_max(Enum.map(shards, &Group.Transport.Outbox.name(:jepsen_group, &1))), total_memory_bytes: :erlang.memory(:total) } rescue @@ -1002,15 +1004,16 @@ defmodule Group.Jepsen.Invariant do Data.replica_cursor_table(:jepsen_group, shard) |> :ets.tab2list() |> Enum.each(fn {stream, seq} -> - origin = Protocol.stream_origin(stream) - cluster = Protocol.stream_cluster(stream) + origin = WireProtocol.stream_origin(stream) + cluster = WireProtocol.stream_cluster(stream) valid? = - Protocol.stream_name(stream) == :jepsen_group and - Protocol.stream_shard(stream) == shard and + WireProtocol.stream_name(stream) == :jepsen_group and + WireProtocol.stream_shard(stream) == shard and origin != node() and - Protocol.stream_generation(stream) == Data.remote_generation(:jepsen_group, origin) and - Protocol.stream_epoch(stream) == + WireProtocol.stream_generation(stream) == + Data.remote_generation(:jepsen_group, origin) and + WireProtocol.stream_epoch(stream) == Data.remote_cluster_epoch(:jepsen_group, origin, cluster) and seq >= 0 unless valid?, do: raise("cursor lacks current authority #{inspect({stream, seq})}") @@ -1041,7 +1044,7 @@ defmodule Group.Jepsen.Invariant do cursors = Data.replica_cursor_table(:jepsen_group, shard) |> :ets.tab2list() - |> Enum.filter(fn {stream, _seq} -> Protocol.stream_origin(stream) == origin end) + |> Enum.filter(fn {stream, _seq} -> WireProtocol.stream_origin(stream) == origin end) view = Data.remote_view_generation(:jepsen_group, shard, origin) diff --git a/test/mutation/run.exs b/test/mutation/run.exs index 6efefbe..19c9f51 100644 --- a/test/mutation/run.exs +++ b/test/mutation/run.exs @@ -16,7 +16,7 @@ defmodule Group.MutationCampaign do name: "accept_old_generation", file: "lib/group/replica.ex", correct_source: - "Protocol.stream_generation(stream_id) == Data.remote_generation(state.name, source_node) and", + "WireProtocol.stream_generation(stream_id) == Data.remote_generation(state.name, source_node) and", faulty_source: "true and", test: ["test/distributed_test.exs:5328"] }, @@ -24,7 +24,7 @@ defmodule Group.MutationCampaign do name: "accept_old_epoch", file: "lib/group/replica.ex", correct_source: """ - Protocol.stream_epoch(stream_id) == + WireProtocol.stream_epoch(stream_id) == Data.remote_cluster_epoch(state.name, source_node, cluster) and """, faulty_source: """ @@ -369,11 +369,11 @@ defmodule Group.MutationCampaign do name: "accept_shared_authority_before_lane_install", file: "lib/group/replica.ex", correct_source: """ - Protocol.stream_shard(stream_id) == state.shard_index and + WireProtocol.stream_shard(stream_id) == state.shard_index and replica_view_current?(state, source_node) and """, faulty_source: """ - Protocol.stream_shard(stream_id) == state.shard_index and + WireProtocol.stream_shard(stream_id) == state.shard_index and true and """, test: ["test/distributed_test.exs:5531"] diff --git a/test/replica_snapshot_distributed_test.exs b/test/replica_snapshot_distributed_test.exs index 89804b8..9987963 100644 --- a/test/replica_snapshot_distributed_test.exs +++ b/test/replica_snapshot_distributed_test.exs @@ -234,7 +234,7 @@ defmodule Group.ReplicaSnapshotDistributedTest do TestCluster.flush_shards(node_a, name) old_stream = local_stream(node_a, name, cluster) - old_epoch = Group.Replica.Protocol.stream_epoch(old_stream) + old_epoch = Group.Replica.WireProtocol.stream_epoch(old_stream) frames = capture_snapshot(node_a, node_b, name, old_stream, 1) assert length(frames) > 1 {partial, [last]} = Enum.split(frames, -1) @@ -307,7 +307,7 @@ defmodule Group.ReplicaSnapshotDistributedTest do new_stream = local_stream(node_a, name, nil) refute new_stream == old_stream - new_generation = Group.Replica.Protocol.stream_generation(new_stream) + new_generation = Group.Replica.WireProtocol.stream_generation(new_stream) TestCluster.assert_eventually(fn -> TestCluster.rpc!(node_b, Group.Replica.Data, :remote_generation, [name, node_a]) == @@ -471,9 +471,8 @@ defmodule Group.ReplicaSnapshotDistributedTest do send( target_control, - {:replica_hello, source_control, Group.Replica.Protocol.version(), generation, revision, - epochs, Group.Replica.Transport.Distribution.id(), - Group.Replica.Transport.Distribution.descriptor(name, [])} + {:replica_hello, source_control, Group.Replica.WireProtocol.version(), generation, revision, + epochs, Group.Transport.DistErl.id(), Group.Transport.DistErl.descriptor(name, [])} ) _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_control]) @@ -556,11 +555,11 @@ defmodule Group.ReplicaSnapshotDistributedTest do ]) :ok = - TestCluster.rpc!(node_a, Group.Replica.Transport, :incoming, [ + TestCluster.rpc!(node_a, Group.Transport, :incoming, [ name, node_b, 0, - {:needs, Group.Replica.Protocol.version(), [{stream_id, next_seq}]} + {:needs, Group.Replica.WireProtocol.version(), [{stream_id, next_seq}]} ]) TestCluster.flush_shards(node_a, name) @@ -581,7 +580,7 @@ defmodule Group.ReplicaSnapshotDistributedTest do defp deliver_frames(node_b, node_a, name, frames) do Enum.each(frames, fn frame -> :ok = - TestCluster.rpc!(node_b, Group.Replica.Transport, :incoming, [ + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ name, node_a, 0, diff --git a/test/replica_snapshot_test.exs b/test/replica_snapshot_test.exs index d87700f..2728870 100644 --- a/test/replica_snapshot_test.exs +++ b/test/replica_snapshot_test.exs @@ -42,8 +42,8 @@ defmodule Group.ReplicaSnapshotTest do Enum.with_index(snapshot.chunks, 1) |> Enum.each(fn {{registry, pg}, index} -> frame = - {:snapshot_chunk, Group.Replica.Protocol.version(), stream_id, 123, index, chunk_count, - snapshot.registry_count, snapshot.pg_count, registry, pg} + {:snapshot_chunk, Group.Replica.WireProtocol.version(), stream_id, 123, index, + chunk_count, snapshot.registry_count, snapshot.pg_count, registry, pg} assert :erlang.external_size(frame) <= target end) diff --git a/test/replica_transport_outbox_test.exs b/test/replica_transport_outbox_test.exs index 63c3a4e..c2739ea 100644 --- a/test/replica_transport_outbox_test.exs +++ b/test/replica_transport_outbox_test.exs @@ -1,7 +1,7 @@ defmodule Group.ReplicaTransportOutboxTest do use ExUnit.Case, async: true - alias Group.Replica.Transport.Outbox + alias Group.Transport.Outbox defmodule Backend do @behaviour Outbox @@ -124,7 +124,7 @@ defmodule Group.ReplicaTransportOutboxTest do assert_receive :receiver_ready assert :ok = - Group.Replica.Transport.incoming_batch( + Group.Transport.incoming_batch( group, source_node, 0, diff --git a/test/support/controlled_replica_transport.ex b/test/support/controlled_replica_transport.ex index 9658457..aecef9b 100644 --- a/test/support/controlled_replica_transport.ex +++ b/test/support/controlled_replica_transport.ex @@ -1,6 +1,6 @@ defmodule Group.ControlledReplicaTransport do @moduledoc false - @behaviour Group.Replica.Transport + @behaviour Group.Transport @impl true def id, do: :group_controlled_replica_transport diff --git a/test/support/replica_model_scheduler.ex b/test/support/replica_model_scheduler.ex index 12955db..fc766ff 100644 --- a/test/support/replica_model_scheduler.ex +++ b/test/support/replica_model_scheduler.ex @@ -390,7 +390,7 @@ defmodule Group.ReplicaModelScheduler do :ok = TestCluster.rpc!( envelope.target, - Group.Replica.Transport, + Group.Transport, :incoming, [state.name, envelope.source, envelope.shard, envelope.message] ) diff --git a/test/support/test_cluster.ex b/test/support/test_cluster.ex index 80cd37e..ab11a61 100644 --- a/test/support/test_cluster.ex +++ b/test/support/test_cluster.ex @@ -728,7 +728,7 @@ defmodule Group.TestCluster do Group.Replica.Data.replica_cursor_table(name, shard) |> :ets.tab2list() |> Enum.filter(fn {stream_id, _seq} -> - Group.Replica.Protocol.stream_origin(stream_id) == origin + Group.Replica.WireProtocol.stream_origin(stream_id) == origin end) retained_view = Group.Replica.Data.remote_view_generation(name, shard, origin) @@ -908,14 +908,14 @@ defmodule Group.TestCluster do Group.Replica.Data.replica_cursor_table(name, shard) |> :ets.tab2list() |> Enum.each(fn {stream_id, seq} -> - origin = Group.Replica.Protocol.stream_origin(stream_id) - generation = Group.Replica.Protocol.stream_generation(stream_id) - cluster = Group.Replica.Protocol.stream_cluster(stream_id) - epoch = Group.Replica.Protocol.stream_epoch(stream_id) + origin = Group.Replica.WireProtocol.stream_origin(stream_id) + generation = Group.Replica.WireProtocol.stream_generation(stream_id) + cluster = Group.Replica.WireProtocol.stream_cluster(stream_id) + epoch = Group.Replica.WireProtocol.stream_epoch(stream_id) valid? = - Group.Replica.Protocol.stream_name(stream_id) == name and - Group.Replica.Protocol.stream_shard(stream_id) == shard and + Group.Replica.WireProtocol.stream_name(stream_id) == name and + Group.Replica.WireProtocol.stream_shard(stream_id) == shard and origin != node() and generation == Group.Replica.Data.remote_generation(name, origin) and epoch == Group.Replica.Data.remote_cluster_epoch(name, origin, cluster) and diff --git a/test/support/test_replica_transport.ex b/test/support/test_replica_transport.ex index 4efc7b2..f5ff194 100644 --- a/test/support/test_replica_transport.ex +++ b/test/support/test_replica_transport.ex @@ -1,6 +1,6 @@ defmodule Group.TestReplicaTransport do @moduledoc false - @behaviour Group.Replica.Transport + @behaviour Group.Transport @impl true def id, do: :group_test_transport diff --git a/lib/group/replica/transport/tcp.ex b/test/support/test_tcp_transport.ex similarity index 84% rename from lib/group/replica/transport/tcp.ex rename to test/support/test_tcp_transport.ex index d074099..39dbce2 100644 --- a/lib/group/replica/transport/tcp.ex +++ b/test/support/test_tcp_transport.ex @@ -1,44 +1,15 @@ -defmodule Group.Replica.Transport.TCP do - @moduledoc """ - Sideband TCP transport for replica data. - - Erlang distribution still carries Group discovery and authority controls. - Replica messages use independent TCP connections, so there is no ordering - relationship between a control message and its data lane. - - `outgoing/5` only pushes to a local per-shard outbox. The outbox batches - messages and forwards each target batch to a bounded per-peer writer queue. - The writer may block up to `:send_timeout` without blocking a Group shard. - Expired, busy, and disconnected batches are dropped and repaired by - anti-entropy. - - The endpoint capability in the dist-Erlang hello prevents an unrelated - socket client from injecting messages. This transport is intended for trusted - cluster networks; it does not encrypt traffic. Put it behind a private - network or a TLS/WebSocket tunnel when confidentiality is required. - - ## Options - - * `:ip` - listen address, default `{127, 0, 0, 1}` - * `:advertised_ip` - address placed in the hello, defaults to `:ip` - * `:port` - listen port, default `0` (ephemeral) - * `:max_queue` - maximum queued batches per peer, default `1_024` - * `:connect_timeout` - outbound connect timeout in milliseconds, default `1_000` - * `:send_timeout` - writer socket send timeout in milliseconds, default `1_000` - * `:reconnect_interval` - retry delay in milliseconds, default `50` - - See `Group.Replica.Transport.Outbox` for batching and deadline options. - """ +defmodule Group.TestTCPTransport do + @moduledoc false use GenServer - @behaviour Group.Replica.Transport - @behaviour Group.Replica.Transport.Outbox + @behaviour Group.Transport + @behaviour Group.Transport.Outbox - alias Group.Replica.Transport.Outbox + alias Group.Transport.Outbox @impl true - def id, do: :group_sideband_tcp_v2 + def id, do: :group_test_sideband_tcp_v2 @impl true def child_spec(opts) do @@ -46,7 +17,7 @@ defmodule Group.Replica.Transport.TCP do %{ id: {__MODULE__, name}, - start: {Group.Replica.Transport.TCP.Supervisor, :start_link, [opts]}, + start: {Group.TestTCPTransport.Supervisor, :start_link, [opts]}, type: :supervisor, restart: :permanent, shutdown: :infinity @@ -67,10 +38,10 @@ defmodule Group.Replica.Transport.TCP do def outgoing(group, target_node, shard, message, opts), do: Outbox.push(group, target_node, shard, message, opts) - @impl Group.Replica.Transport.Outbox + @impl Group.Transport.Outbox def init_outbox(group, shard, _opts), do: {:ok, %{group: group, shard: shard}} - @impl Group.Replica.Transport.Outbox + @impl Group.Transport.Outbox def send_batch(target_node, messages, deadline, %{group: group, shard: shard} = state) do result = try do @@ -144,7 +115,7 @@ defmodule Group.Replica.Transport.TCP do {:ok, {_listen_ip, listen_port}} = :inet.sockname(listener) capability = :erlang.term_to_binary({node(), make_ref(), System.unique_integer()}) - descriptor = {:group_sideband_tcp_v2, advertised_ip, listen_port, capability} + descriptor = {:group_test_sideband_tcp_v2, advertised_ip, listen_port, capability} :persistent_term.put({__MODULE__, group, :descriptor}, descriptor) :ets.new(route_table(group), [ @@ -343,7 +314,7 @@ defmodule Group.Replica.Transport.TCP do manager, group, remote_node, - {:group_sideband_tcp_v2, host, port, capability}, + {:group_test_sideband_tcp_v2, host, port, capability}, connect_timeout, send_timeout ) do @@ -440,7 +411,7 @@ defmodule Group.Replica.Transport.TCP do case decode_authenticated_frame(payload) do {:ok, {:batch, shard, messages}} when is_integer(shard) and shard >= 0 and is_list(messages) -> - :ok = Group.Replica.Transport.incoming_batch(group, source_node, shard, messages) + :ok = Group.Transport.incoming_batch(group, source_node, shard, messages) reader_loop(socket, group, source_node) _ -> @@ -471,11 +442,12 @@ defmodule Group.Replica.Transport.TCP do defp route_table(group), do: :"#{group}_replica_tcp_routes" end -defmodule Group.Replica.Transport.TCP.Supervisor do +defmodule Group.TestTCPTransport.Supervisor do @moduledoc false use Supervisor - alias Group.Replica.Transport.{Outbox, TCP} + alias Group.TestTCPTransport, as: TCP + alias Group.Transport.Outbox def start_link(opts), do: Supervisor.start_link(__MODULE__, opts) From de8ebd00123a2f02862ad6d0fcb7996952566dc9 Mon Sep 17 00:00:00 2001 From: Chris McCord Date: Thu, 13 Aug 2026 13:22:40 +0000 Subject: [PATCH 08/16] Fix anti-entropy authority and transport edge cases --- README.md | 11 +- lib/group/replica.ex | 136 ++++++- lib/group/transport.ex | 56 ++- lib/group/transport/outbox.ex | 52 ++- test/README.md | 5 +- test/anti_entropy_fault_regression_test.exs | 429 ++++++++++++++++++++ test/distributed_test.exs | 18 + test/group_test.exs | 5 +- test/jepsen/node.exs | 7 +- test/mutation/run.exs | 3 +- test/replica_snapshot_distributed_test.exs | 73 ++++ test/replica_transport_outbox_test.exs | 27 ++ test/support/test_cluster.ex | 53 ++- test/support/test_tcp_transport.ex | 45 +- 14 files changed, 866 insertions(+), 54 deletions(-) create mode 100644 test/anti_entropy_fault_regression_test.exs diff --git a/README.md b/README.md index 43c6cb4..ab58877 100644 --- a/README.md +++ b/README.md @@ -451,7 +451,9 @@ and exact snapshots close gaps after pruning. Exact snapshots are split into transport-neutral byte-bounded messages; loss, duplication, or reordering leaves the old visible slice and cursor untouched until all chunks arrive. Incomplete staging expires after a peer-lease interval without progress and is destroyed -automatically with its owning shard. Named-cluster close uses only a temporary +automatically with its owning shard. Rejected first chunks, nodedown, +generation replacement, and retired epochs destroy matching staging +immediately. Named-cluster close uses only a temporary local shard-completion barrier; the final shard removes it and all routing rows, including after a caller timeout or shard restart. Reconnect waits for that barrier so a prior close cannot erase newly accepted writes. @@ -486,6 +488,13 @@ target and invokes the adapter's `send_batch/4` callback. Calls that expire or return `:busy`/`:disconnected` are dropped without a local retry; the next anti-entropy exchange repairs them. +The optional `peer_up/5` and `peer_down/4` callbacks report one shard lane at a +time. A sideband adapter that shares a single node connection must retain it +while any reported lane remains live and release it after the last lane goes +down. `incoming/4` and `incoming_batch/4` return `:disconnected` and drop when +their local shard is restarting; an ingress reader must treat that as an +expected lossy delivery outcome. + A message-oriented backend fits this callback shape by obtaining a connection once from `init_outbox/3`, then sending each `send_batch/4` result to a registered incoming name on the target node. Queue pressure maps to `:busy` and diff --git a/lib/group/replica.ex b/lib/group/replica.ex index 39ea943..6a7bc88 100644 --- a/lib/group/replica.ex +++ b/lib/group/replica.ex @@ -36,7 +36,10 @@ defmodule Group.Replica do When a local claim loses, only its owner node appends the authoritative unregister and terminates the local process. Retaining hidden remote claims until their origin deletes them prevents a later winner change from orphaning - or permanently forgetting a live claim. + or permanently forgetting a live claim. Local unregister and process-DOWN + paths always re-project every affected key after deleting their claims. On a + shard restart, the complete claim/key union is projected again, closing the + crash window between journal application and the materialized winner write. PG memberships need no winner projection: the origin stream owns exactly the rows whose member processes live on that origin node. @@ -71,6 +74,9 @@ defmodule Group.Replica do late messages fail their generation or epoch fence. Snapshot chunks may be lost, duplicated, reordered, or mixed across retransmissions at the same stream head; exact row counts and set insertion prevent partial commits. + Rejected first chunks destroy their staging table immediately. Node loss, + generation replacement, and retired cluster streams discard matching partial + assemblies immediately rather than waiting for their inactivity deadline. ## Bounded recovery @@ -91,6 +97,10 @@ defmodule Group.Replica do :busy, or :disconnected; failure drops the message and anti-entropy repairs it. Replica shards never remotely monitor or exit member processes. + Optional peer lifecycle callbacks are per shard lane. A sideband adapter that + shares one node connection across shards keeps that route until its final + live lane goes down; one flapping shard cannot disconnect healthy lanes. + The transport need not order messages for correctness. The local shard serializes writes, sequence numbers establish per-stream order, and receivers reject duplicates and gaps. TCP shard-to-shard ordering remains the efficient @@ -413,6 +423,13 @@ defmodule Group.Replica do Map.put(state.peer_transports, remote_node, {transport_id, transport_descriptor}) } + state = + if replica_view_current?(state, remote_node) do + state + else + install_current_replica_lane(state, remote_node, generation) + end + {:noreply, state} true -> @@ -449,11 +466,12 @@ defmodule Group.Replica do remote_node = node(remote_pid) if version == WireProtocol.version() and transport_id == state.replica_transport.id() do - if function_exported?(state.replica_transport, :peer_up, 4) do + if function_exported?(state.replica_transport, :peer_up, 5) do :ok = state.replica_transport.peer_up( state.name, remote_node, + state.shard_index, transport_descriptor, state.replica_transport_opts ) @@ -1045,6 +1063,7 @@ defmodule Group.Replica do def handle_info({:nodedown, dead_node}, state) do state = flush_pending_replicated_message_barrier(state) + state = discard_snapshot_transfers_for_source(state, dead_node) %{name: name, shard_index: shard} = state # Remove cluster memberships from shared tables. Every shard calls this @@ -1081,8 +1100,14 @@ defmodule Group.Replica do Data.delete_replica_cursors_for_origin(name, shard, dead_node) Data.delete_remote_replica_info(name, shard, dead_node) - if function_exported?(state.replica_transport, :peer_down, 3) do - :ok = state.replica_transport.peer_down(name, dead_node, state.replica_transport_opts) + if function_exported?(state.replica_transport, :peer_down, 4) do + :ok = + state.replica_transport.peer_down( + name, + dead_node, + shard, + state.replica_transport_opts + ) end state = %{state | peer_transports: Map.delete(state.peer_transports, dead_node)} @@ -1100,6 +1125,8 @@ defmodule Group.Replica do remote_node = node(pid) if remote_node != node() and Map.get(state.remote_shards, remote_node) == pid do + state = discard_snapshot_transfers_for_source(state, remote_node) + # Remote shard process died — purge its cluster memberships and node data. # Unconditional (not gated on shard 0) — same reasoning as nodedown handler. Data.purge_cluster_node(name, remote_node) @@ -1126,6 +1153,17 @@ defmodule Group.Replica do notify_monitors(name, events) state = %{state | remote_shards: Map.delete(state.remote_shards, remote_node)} state = %{state | monitors: Map.delete(state.monitors, pid)} + + if function_exported?(state.replica_transport, :peer_down, 4) do + :ok = + state.replica_transport.peer_down( + name, + remote_node, + shard, + state.replica_transport_opts + ) + end + {:noreply, state} else if Map.has_key?(state.monitors, pid) do @@ -1158,7 +1196,19 @@ defmodule Group.Replica do end) state = finish_process_down_records(state, sequenced_downs) - events = build_process_down_events(name, purged_reg, purged_pg, reason_by_pid) + + affected_registry_keys = + pending_reg + |> Enum.map(fn {_pid, cluster, key, _meta} -> {cluster, key} end) + |> Enum.uniq() + + {state, projection_events} = + reconcile_registry_keys(state, affected_registry_keys, reason, []) + + events = + projection_events ++ + build_process_down_events(name, purged_reg, purged_pg, reason_by_pid) + notify_monitors(name, events) state = %{state | monitors: Map.drop(monitors, pids)} {:noreply, state} @@ -1905,7 +1955,10 @@ defmodule Group.Replica do cluster: cluster }) - notify_monitors(name, [event]) + {state, projection_events} = + reconcile_registry_keys(state, [{cluster, key}], :unregister, []) + + notify_monitors(name, projection_events ++ [event]) {:ok, state} nil -> @@ -2044,9 +2097,8 @@ defmodule Group.Replica do {events, local_pids} = Enum.reduce(clusters, {[], MapSet.new()}, fn cluster, {events, local_pids} -> - affected_keys = Data.purge_registry_claims_for_cluster(name, shard, cluster) - purged_reg = Data.delete_registry_keys(name, shard, cluster, affected_keys) - purged_pg = Data.delete_pg_cluster(name, shard, cluster) + _affected_keys = Data.purge_registry_claims_for_cluster(name, shard, cluster) + {purged_reg, purged_pg} = purge_cluster_entries(name, shard, cluster, :all) local_pids = Enum.reduce(purged_reg ++ purged_pg, local_pids, fn @@ -2899,11 +2951,12 @@ defmodule Group.Replica do end defp notify_replica_transport_peer_up(state, remote_node, transport_descriptor) do - if function_exported?(state.replica_transport, :peer_up, 4) do + if function_exported?(state.replica_transport, :peer_up, 5) do :ok = state.replica_transport.peer_up( state.name, remote_node, + state.shard_index, transport_descriptor, state.replica_transport_opts ) @@ -3182,6 +3235,19 @@ defmodule Group.Replica do end) end + defp discard_snapshot_transfers_for_streams(state, stream_ids) do + stream_ids = MapSet.new(stream_ids) + + Enum.reduce(state.snapshot_transfers, state, fn + {{_source_node, stream_id} = key, _transfer}, acc -> + if MapSet.member?(stream_ids, stream_id) do + discard_snapshot_transfer(acc, key) + else + acc + end + end) + end + defp expire_replica_peer(state, remote_node) do state = discard_snapshot_transfers_for_source(state, remote_node) %{name: name, shard_index: shard} = state @@ -3207,8 +3273,14 @@ defmodule Group.Replica do fan_out_to_siblings(state, {:replica_authority_removed_local, remote_node}) end - if function_exported?(state.replica_transport, :peer_down, 3) do - :ok = state.replica_transport.peer_down(name, remote_node, state.replica_transport_opts) + if function_exported?(state.replica_transport, :peer_down, 4) do + :ok = + state.replica_transport.peer_down( + name, + remote_node, + shard, + state.replica_transport_opts + ) end %{ @@ -3479,14 +3551,16 @@ defmodule Group.Replica do defp snapshot_transfer(state, key, snapshot_seq, manifest) do case Map.get(state.snapshot_transfers, key) do nil -> - {:ok, state, new_snapshot_transfer(snapshot_seq, manifest)} + transfer = new_snapshot_transfer(snapshot_seq, manifest) + {:ok, put_snapshot_transfer(state, key, transfer), transfer} %{snapshot_seq: existing_seq} when existing_seq > snapshot_seq -> {:ignore, state} %{snapshot_seq: existing_seq} when existing_seq < snapshot_seq -> state = discard_snapshot_transfer(state, key) - {:ok, state, new_snapshot_transfer(snapshot_seq, manifest)} + transfer = new_snapshot_transfer(snapshot_seq, manifest) + {:ok, put_snapshot_transfer(state, key, transfer), transfer} %{manifest: ^manifest} = transfer -> {:ok, state, transfer} @@ -4088,6 +4162,7 @@ defmodule Group.Replica do defp maybe_purge_remote_generation(state, _remote_node, generation, generation), do: state defp maybe_purge_remote_generation(state, remote_node, _old_generation, _generation) do + state = discard_snapshot_transfers_for_source(state, remote_node) {_reg, _pg} = Data.purge_node(state.name, state.shard_index, remote_node) affected = @@ -4124,6 +4199,8 @@ defmodule Group.Replica do ) end) + state = discard_snapshot_transfers_for_streams(state, stream_ids) + affected_keys = Data.purge_registry_claims_for_streams( state.name, @@ -4231,6 +4308,7 @@ defmodule Group.Replica do current_epochs, superseded ) do + state = discard_snapshot_transfers_for_streams(state, superseded) generation = Data.remote_generation(state.name, remote_node) superseded @@ -4516,9 +4594,33 @@ defmodule Group.Replica do state.replicated_oplog_max_entries ) + {state, _events} = rebuild_registry_projections(state) state end + defp rebuild_registry_projections(state) do + claim_keys = + state.name + |> Data.reg_claim_by_key_table(state.shard_index) + |> :ets.select([ + {{{:"$1", :"$2", :_, :_, :_}, :_, :_, :_, :_}, [], [{{:"$1", :"$2"}}]} + ]) + + visible_keys = + state.name + |> Data.reg_by_key_table(state.shard_index) + |> :ets.select([ + {{{:"$1", :"$2"}, :_, :_, :_, :_}, [], [{{:"$1", :"$2"}}]} + ]) + + reconcile_registry_keys( + state, + Enum.uniq(claim_keys ++ visible_keys), + :journal_replay, + [] + ) + end + defp current_local_stream?(state, stream_id) do cluster = WireProtocol.stream_cluster(stream_id) @@ -4793,6 +4895,12 @@ defmodule Group.Replica do ) end + defp reconcile_registry_keys(state, keys, reason, events) do + Enum.reduce(keys, {state, events}, fn {cluster, key}, {acc, inner_events} -> + reconcile_registry_projection(acc, cluster, key, reason, inner_events) + end) + end + defp select_registry_claim_winner(_state, _cluster, _key, []), do: nil defp select_registry_claim_winner(_state, _cluster, _key, [claim]), do: claim diff --git a/lib/group/transport.ex b/lib/group/transport.ex index 30eba2d..8390879 100644 --- a/lib/group/transport.ex +++ b/lib/group/transport.ex @@ -46,10 +46,30 @@ defmodule Group.Transport do ) :: outgoing_result() @callback child_spec(keyword()) :: Supervisor.child_spec() | :ignore - @callback peer_up(group :: atom(), node(), descriptor :: term(), opts :: keyword()) :: :ok - @callback peer_down(group :: atom(), node(), opts :: keyword()) :: :ok - @optional_callbacks child_spec: 1, peer_up: 4, peer_down: 3 + @doc """ + Reports that one shard lane can address a peer through this transport. + + A transport sharing one node-level connection across lanes should retain the + route until `peer_down/4` has retired every shard previously reported up. + """ + @callback peer_up( + group :: atom(), + node(), + shard :: non_neg_integer(), + descriptor :: term(), + opts :: keyword() + ) :: :ok + + @doc "Reports that one shard lane no longer has live peer authority." + @callback peer_down( + group :: atom(), + node(), + shard :: non_neg_integer(), + opts :: keyword() + ) :: :ok + + @optional_callbacks child_spec: 1, peer_up: 5, peer_down: 4 @doc """ Passes an incoming replica message to the corresponding local shard. @@ -57,15 +77,20 @@ defmodule Group.Transport do This is a local mailbox operation. `source_node` is the trusted peer identity established by the adapter. Stream generation, epoch, group, shard, origin, and member-pid ownership are validated by the replica. + + Returns `:disconnected` and drops the message if that shard is not currently + registered, for example while its supervisor is restarting. """ def incoming(group, source_node, shard, message) when is_atom(group) and is_atom(source_node) and is_integer(shard) and shard >= 0 do - send( - Group.Replica.shard_name(group, shard), - {:group_replica_frame, source_node, message} - ) + case Process.whereis(Group.Replica.shard_name(group, shard)) do + pid when is_pid(pid) -> + send(pid, {:group_replica_frame, source_node, message}) + :ok - :ok + nil -> + :disconnected + end end @doc """ @@ -75,16 +100,21 @@ defmodule Group.Transport do must reassemble every segment before calling this function. Group never observes or applies a partial batch. `source_node` has the same trusted-peer meaning as in `incoming/4`. + + Like `incoming/4`, this returns `:disconnected` if the destination shard is + unavailable. """ def incoming_batch(group, source_node, shard, messages) when is_atom(group) and is_atom(source_node) and is_integer(shard) and shard >= 0 and is_list(messages) do - send( - Group.Replica.shard_name(group, shard), - {:group_replica_batch, source_node, messages} - ) + case Process.whereis(Group.Replica.shard_name(group, shard)) do + pid when is_pid(pid) -> + send(pid, {:group_replica_batch, source_node, messages}) + :ok - :ok + nil -> + :disconnected + end end def normalize(module) when is_atom(module), do: {module, []} diff --git a/lib/group/transport/outbox.ex b/lib/group/transport/outbox.ex index 4b523f1..567a6e0 100644 --- a/lib/group/transport/outbox.ex +++ b/lib/group/transport/outbox.ex @@ -108,6 +108,15 @@ defmodule Group.Transport.Outbox do @doc false def monotonic_ms, do: System.monotonic_time(:millisecond) + @doc false + def validate_options!(opts) when is_list(opts) do + deadline(opts) + positive_opt(opts, :outbox_batch_size, 64) + positive_opt(opts, :outbox_batch_bytes, 1_048_576) + non_negative_opt(opts, :outbox_flush_interval, 1) + :ok + end + defp deadline(opts) do case Keyword.get(opts, :outbox_deadline, @default_deadline) do value when is_integer(value) and value > 0 -> @@ -117,6 +126,27 @@ defmodule Group.Transport.Outbox do raise ArgumentError, "expected :outbox_deadline to be positive, got: #{inspect(other)}" end end + + defp positive_opt(opts, key, default) do + case Keyword.get(opts, key, default) do + value when is_integer(value) and value > 0 -> + value + + other -> + raise ArgumentError, "expected #{inspect(key)} to be positive, got: #{inspect(other)}" + end + end + + defp non_negative_opt(opts, key, default) do + case Keyword.get(opts, key, default) do + value when is_integer(value) and value >= 0 -> + value + + other -> + raise ArgumentError, + "expected #{inspect(key)} to be non-negative, got: #{inspect(other)}" + end + end end defmodule Group.Transport.Outbox.Supervisor do @@ -127,6 +157,7 @@ defmodule Group.Transport.Outbox.Supervisor do @impl true def init(opts) do + :ok = Group.Transport.Outbox.validate_options!(opts) group = Keyword.fetch!(opts, :name) num_shards = Keyword.fetch!(opts, :num_shards) backend = Keyword.fetch!(opts, :backend) @@ -171,6 +202,7 @@ defmodule Group.Transport.Outbox.Worker do @impl true def init({opts, shard}) do + :ok = Outbox.validate_options!(opts) group = Keyword.fetch!(opts, :name) backend = Keyword.fetch!(opts, :backend) {:ok, backend_state} = backend.init_outbox(group, shard, opts) @@ -218,7 +250,10 @@ defmodule Group.Transport.Outbox.Worker do end end - def handle_info({:group_replica_outbox_flush, ref}, %{flush_ref: ref} = state) do + def handle_info( + {:group_replica_outbox_flush, ref}, + %{flush_ref: {ref, _timer_ref}} = state + ) do {:noreply, flush(%{state | flush_ref: nil})} end @@ -237,12 +272,17 @@ defmodule Group.Transport.Outbox.Worker do end defp schedule_flush(%{pending_count: 0} = state), do: state - defp schedule_flush(%{flush_ref: ref} = state) when is_reference(ref), do: state + + defp schedule_flush(%{flush_ref: {_ref, timer_ref}} = state) when is_reference(timer_ref), + do: state defp schedule_flush(state) do ref = make_ref() - Process.send_after(self(), {:group_replica_outbox_flush, ref}, state.flush_interval) - %{state | flush_ref: ref} + + timer_ref = + Process.send_after(self(), {:group_replica_outbox_flush, ref}, state.flush_interval) + + %{state | flush_ref: {ref, timer_ref}} end defp flush(%{pending_count: 0} = state), do: cancel_flush(state) @@ -290,8 +330,8 @@ defmodule Group.Transport.Outbox.Worker do defp cancel_flush(%{flush_ref: nil} = state), do: state - defp cancel_flush(state) do - Process.cancel_timer(state.flush_ref) + defp cancel_flush(%{flush_ref: {_ref, timer_ref}} = state) do + Process.cancel_timer(timer_ref) %{state | flush_ref: nil} end diff --git a/test/README.md b/test/README.md index 198a961..394eff7 100644 --- a/test/README.md +++ b/test/README.md @@ -25,6 +25,7 @@ release qualification rather than individual edits. |------|---------------| | `group_test.exs` | Single-node: register/unregister, join/leave, members, monitor/demonitor, named clusters, concurrent operations | | `distributed_test.exs` | Multi-node: replication, peer discovery, node disconnect cleanup, partition healing, conflict resolution, event ordering, rolling restarts, and adversarial replica-transport loss/busy/snapshot recovery | +| `anti_entropy_fault_regression_test.exs` | Three-node regressions for hidden-winner projection, crash-journal replay, claimless cluster cleanup, shard-zero view repair, and shard-scoped sideband lifecycle | | `replica_adversarial_test.exs` | Reproducible three-node mixed-operation state machines: drops, busy returns, duplication, reordering, bounded delay, oplog pruning, conflicts, owner death, and named-cluster epoch churn, followed by exact convergence/dead-owner/internal-index checks | | `replica_model_property_test.exs` | StreamData-generated and shrunk owner histories against an independent lifecycle oracle and scheduler-controlled replica transport | | `replica_snapshot_test.exs` | Pure byte partitioning and set-valued private-ETS snapshot staging | @@ -286,7 +287,9 @@ registry or PG state. `replica_transport_outbox_test.exs` proves that a blocked sideband backend cannot delay the Group-facing local push, messages expire behind that backend, -busy batches are not retried locally, and batching preserves per-target order. +busy batches are not retried locally, batching preserves per-target order, +invalid deadlines fail at boot, and ingress drops rather than raising while a +destination shard is absent. The real three-node TCP recovery test runs through the same outbox path. `Group.TestCluster.assert_replica_consistent/1` checks the diff --git a/test/anti_entropy_fault_regression_test.exs b/test/anti_entropy_fault_regression_test.exs new file mode 100644 index 0000000..fb17b4f --- /dev/null +++ b/test/anti_entropy_fault_regression_test.exs @@ -0,0 +1,429 @@ +defmodule Group.AntiEntropyFaultRegressionTest do + use ExUnit.Case, async: false + + @moduletag :capture_log + @moduletag timeout: 120_000 + + alias Group.TestCluster + + setup_all do + peers = TestCluster.start_peers(3, schedulers: 4) + on_exit(fn -> TestCluster.stop_peers(peers) end) + [{_, node_a}, {_, node_b}, {_, node_c}] = peers + {:ok, peers: peers, node_a: node_a, node_b: node_b, node_c: node_c} + end + + test "local unregister promotes a retained remote claim after its delta is pruned", context do + %{name: name, pid_a: pid_a, pid_b: pid_b, stream_b: stream_b} = + establish_hidden_remote_claim(context, :unregister) + + {floor, head, _applied} = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :replica_stream_head, [ + name, + 0, + stream_b + ]) + + assert floor > 1 + + assert TestCluster.rpc!(context.node_a, Group.Replica.Data, :replica_cursor, [ + name, + 0, + stream_b + ]) == head + + assert :ok = TestCluster.unregister_owner(pid_a) + TestCluster.flush_shards(context.node_a, name) + + assert TestCluster.rpc!(context.node_a, Group.Replica.Data, :registry_claims, [ + name, + 0, + nil, + "hidden/unregister" + ]) + |> Enum.map(&elem(&1, 0)) == [pid_b] + + TestCluster.assert_eventually( + fn -> + match?( + {^pid_b, %{rank: 1}}, + TestCluster.rpc!(context.node_a, Group, :lookup, [name, "hidden/unregister"]) + ) + end, + timeout: 500, + interval: 25 + ) + + for node <- [context.node_a, context.node_b, context.node_c] do + assert :ok = TestCluster.rpc!(node, TestCluster, :assert_replica_consistent, [name]) + end + end + + test "journal replay promotes a retained remote claim before marking the delete applied", + context do + %{name: name, pid_a: pid_a, pid_b: pid_b} = + establish_hidden_remote_claim(context, :journal) + + stream_a = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_stream_id, [name, 0, nil]) + + {seq, _mutations} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :append_replica_record, [ + name, + 0, + stream_a, + [{:unregister, nil, "hidden/journal", pid_a, %{rank: 2}, :injected_crash}] + ]) + + assert [{^stream_a, ^seq, _}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_replica_unapplied, [ + name, + 0 + ]) + + old_shard = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + true = TestCluster.rpc!(context.node_a, Process, :exit, [old_shard, :kill]) + + TestCluster.assert_eventually(fn -> + case TestCluster.rpc!(context.node_a, Process, :whereis, [ + Group.Replica.shard_name(name, 0) + ]) do + pid when is_pid(pid) -> pid != old_shard + nil -> false + end + end) + + TestCluster.flush_shards(context.node_a, name) + + claims = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :registry_claims, [ + name, + 0, + nil, + "hidden/journal" + ]) + + journal_state = %{ + claims: claims, + head: + TestCluster.rpc!(context.node_a, Group.Replica.Data, :replica_stream_head, [ + name, + 0, + stream_a + ]), + unapplied: + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_replica_unapplied, [ + name, + 0 + ]) + } + + assert Enum.map(claims, &elem(&1, 0)) == [pid_b], inspect(journal_state) + + assert {^pid_b, %{rank: 1}} = + TestCluster.rpc!(context.node_a, Group, :lookup, [name, "hidden/journal"]) + + assert [] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_replica_unapplied, [ + name, + 0 + ]) + + assert :ok = + TestCluster.rpc!(context.node_a, TestCluster, :assert_replica_consistent, [name]) + end + + test "local process DOWN promotes a retained remote claim", context do + %{name: name, pid_a: pid_a, pid_b: pid_b} = + establish_hidden_remote_claim(context, :process_down) + + true = TestCluster.rpc!(context.node_a, Process, :exit, [pid_a, :kill]) + TestCluster.flush_shards(context.node_a, name) + + assert [^pid_b] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :registry_claims, [ + name, + 0, + nil, + "hidden/process_down" + ]) + |> Enum.map(&elem(&1, 0)) + + assert {^pid_b, %{rank: 1}} = + TestCluster.rpc!(context.node_a, Group, :lookup, [name, "hidden/process_down"]) + + assert :ok = + TestCluster.rpc!(context.node_a, TestCluster, :assert_replica_consistent, [name]) + end + + test "local cluster disconnect removes a claimless legacy registry row", context do + name = unique_name(:claimless_disconnect) + cluster = "legacy-slice" + + opts = [ + name: name, + shards: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + {:ok, _pid} = TestCluster.start_group(context.node_a, opts) + :ok = TestCluster.rpc!(context.node_a, Group, :connect, [name, cluster]) + + owner = + TestCluster.spawn_register_in_cluster( + context.node_a, + name, + "legacy/claimless", + %{legacy: true}, + cluster + ) + + assert ["legacy/claimless"] = + TestCluster.rpc!( + context.node_a, + Group.Replica.Data, + :purge_registry_claims_for_cluster, + [name, 0, cluster] + ) + + assert {^owner, %{legacy: true}} = + TestCluster.rpc!(context.node_a, Group, :lookup, [ + name, + "legacy/claimless", + [cluster: cluster] + ]) + + :ok = TestCluster.rpc!(context.node_a, Group, :disconnect, [name, cluster]) + + assert nil == + TestCluster.rpc!(context.node_a, Group, :lookup, [ + name, + "legacy/claimless", + [cluster: cluster] + ]) + end + + test "expiring one sideband lane keeps the shared node route while other lanes are live", + context do + name = unique_name(:sideband_lane) + + opts = [ + name: name, + shards: 3, + replica_transport: + {Group.TestTCPTransport, + [connect_timeout: 250, send_timeout: 250, reconnect_interval: 10]}, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + TestCluster.assert_eventually( + fn -> + TestCluster.rpc!(context.node_b, Group.TestTCPTransport, :connected?, [ + name, + context.node_a + ]) + end, + timeout: 10_000 + ) + + lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + send(lane, {:replica_authority_removed_local, context.node_a}) + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [lane]) + Process.sleep(100) + + assert TestCluster.rpc!(context.node_b, Group.TestTCPTransport, :connected?, [ + name, + context.node_a + ]) + + status = TestCluster.rpc!(context.node_b, Group.TestTCPTransport, :status, [name]) + assert context.node_a in status.peers + + for shard <- [0, 2] do + replica = + TestCluster.rpc!(context.node_b, Process, :whereis, [ + Group.Replica.shard_name(name, shard) + ]) + + send(replica, {:replica_authority_removed_local, context.node_a}) + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [replica]) + end + + TestCluster.assert_eventually(fn -> + not TestCluster.rpc!(context.node_b, Group.TestTCPTransport, :connected?, [ + name, + context.node_a + ]) and + context.node_a not in TestCluster.rpc!(context.node_b, Group.TestTCPTransport, :status, [ + name + ]).peers + end) + end + + test "an exact shard-zero hello repairs a missing local authority view", context do + name = unique_name(:control_view_repair) + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + {generation, revision, epochs} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_replica_authority, [name]) + + assert generation == + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_view_generation, [ + name, + 0, + context.node_a + ]) + + :ok = + TestCluster.rpc!(context.node_b, TestCluster, :delete_remote_view_info, [ + name, + 0, + context.node_a + ]) + + assert nil == + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_view_generation, [ + name, + 0, + context.node_a + ]) + + assert generation == + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_generation, [ + name, + context.node_a + ]) + + source = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + target = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + send( + target, + {:replica_hello, source, Group.Replica.WireProtocol.version(), generation, revision, epochs, + Group.TestReplicaTransport.id(), Group.TestReplicaTransport.descriptor(name, [])} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target]) + + assert generation == + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_view_generation, [ + name, + 0, + context.node_a + ]) + + assert revision == + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_view_cluster_epoch_revision, + [name, 0, context.node_a] + ) + end + + defp establish_hidden_remote_claim(context, suffix) do + name = unique_name(suffix) + key = "hidden/#{suffix}" + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + resolve_registry_conflict: {Group.ModelConflictResolver, :resolve, []}, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 5_000, + replicated_oplog_max_entries: 2 + ] + + start_group_on_peers(context.peers, opts) + + for node <- [context.node_a, context.node_b, context.node_c] do + :ok = TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [name, :drop]) + end + + pid_a = TestCluster.spawn_register(context.node_a, name, key, %{rank: 2}) + pid_b = TestCluster.spawn_register(context.node_b, name, key, %{rank: 1}) + :ok = TestCluster.rpc!(context.node_b, Group.TestReplicaTransport, :set_mode, [name, :pass]) + + TestCluster.assert_eventually(fn -> + claims = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :registry_claims, [ + name, + 0, + nil, + key + ]) + + match?({^pid_a, %{rank: 2}}, TestCluster.rpc!(context.node_a, Group, :lookup, [name, key])) and + MapSet.new(Enum.map(claims, &elem(&1, 0))) == MapSet.new([pid_a, pid_b]) + end) + + for index <- 1..4 do + churn = + TestCluster.spawn_register(context.node_b, name, "#{key}/churn/#{index}", %{rank: 1}) + + true = TestCluster.rpc!(context.node_b, Process, :exit, [churn, :kill]) + end + + TestCluster.flush_shards(context.node_b, name) + + stream_b = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :local_stream_id, [name, 0, nil]) + + TestCluster.assert_eventually(fn -> + {_floor, head, _applied} = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :replica_stream_head, [ + name, + 0, + stream_b + ]) + + TestCluster.rpc!(context.node_a, Group.Replica.Data, :replica_cursor, [ + name, + 0, + stream_b + ]) == head + end) + + %{name: name, pid_a: pid_a, pid_b: pid_b, stream_b: stream_b} + end + + defp start_group_on_peers(peers, opts) do + Enum.each(peers, fn {_peer, node} -> + {:ok, _pid} = TestCluster.start_group(node, opts) + end) + + nodes = Enum.map(peers, &elem(&1, 1)) + + TestCluster.assert_eventually(fn -> + Enum.all?(nodes, fn node -> + Enum.sort(TestCluster.rpc!(node, Group, :nodes, [Keyword.fetch!(opts, :name)])) == + Enum.sort(nodes -- [node]) + end) + end) + end + + defp unique_name(suffix) do + :"ae_fault_#{suffix}_#{System.unique_integer([:positive])}" + end +end diff --git a/test/distributed_test.exs b/test/distributed_test.exs index ed8cf1b..ae8d3c7 100644 --- a/test/distributed_test.exs +++ b/test/distributed_test.exs @@ -4666,6 +4666,24 @@ defmodule Group.DistributedTest do ] start_group_on_peers(peers, opts) + + TestCluster.assert_eventually(fn -> + not is_nil( + TestCluster.rpc!(node_a, Group.Replica.Data, :remote_view_generation, [ + name, + 0, + node_b + ]) + ) and + not is_nil( + TestCluster.rpc!(node_b, Group.Replica.Data, :remote_view_generation, [ + name, + 0, + node_a + ]) + ) + end) + TestCluster.rpc!(node_a, Group, :connect, [name, "game"]) TestCluster.rpc!(node_b, Group, :connect, [name, "game"]) diff --git a/test/group_test.exs b/test/group_test.exs index 25a5f68..3f1b1ac 100644 --- a/test/group_test.exs +++ b/test/group_test.exs @@ -2675,7 +2675,7 @@ defmodule GroupTest do assert Group.lookup(name, key2) == {pid2, %{v: 2}} end - test "terminate flushes buffered replicated registry ops before shard restart" do + test "restart purges a flushed legacy registry row that has no authoritative claim" do name = start_single_shard_group( replicated_registry_receiver_buffer_size: 32, @@ -2706,7 +2706,8 @@ defmodule GroupTest do end end) - assert Group.lookup(name, key) == {pid, %{v: 1}} + assert Group.lookup(name, key) == nil + assert :ok = Group.TestCluster.assert_replica_consistent(name) end test "replaces stale remote registry owner and clears the old by-pid entry" do diff --git a/test/jepsen/node.exs b/test/jepsen/node.exs index c2ef0eb..e02e0ac 100644 --- a/test/jepsen/node.exs +++ b/test/jepsen/node.exs @@ -175,11 +175,12 @@ defmodule Group.Jepsen.Transport.TCP do end @impl true - def peer_up(group, remote_node, descriptor, opts), - do: Delegate.peer_up(group, remote_node, descriptor, opts) + def peer_up(group, remote_node, shard, descriptor, opts), + do: Delegate.peer_up(group, remote_node, shard, descriptor, opts) @impl true - def peer_down(group, remote_node, opts), do: Delegate.peer_down(group, remote_node, opts) + def peer_down(group, remote_node, shard, opts), + do: Delegate.peer_down(group, remote_node, shard, opts) end defmodule Group.Jepsen.Transport.TCP.Supervisor do diff --git a/test/mutation/run.exs b/test/mutation/run.exs index 19c9f51..60a89c3 100644 --- a/test/mutation/run.exs +++ b/test/mutation/run.exs @@ -131,7 +131,8 @@ defmodule Group.MutationCampaign do correct_source: """ %{snapshot_seq: existing_seq} when existing_seq < snapshot_seq -> state = discard_snapshot_transfer(state, key) - {:ok, state, new_snapshot_transfer(snapshot_seq, manifest)} + transfer = new_snapshot_transfer(snapshot_seq, manifest) + {:ok, put_snapshot_transfer(state, key, transfer), transfer} """, faulty_source: """ %{snapshot_seq: existing_seq} when existing_seq < snapshot_seq -> diff --git a/test/replica_snapshot_distributed_test.exs b/test/replica_snapshot_distributed_test.exs index 9987963..b966b78 100644 --- a/test/replica_snapshot_distributed_test.exs +++ b/test/replica_snapshot_distributed_test.exs @@ -199,6 +199,72 @@ defmodule Group.ReplicaSnapshotDistributedTest do end) end + test "rejected first chunks do not leak their private staging tables", context do + %{name: name, node_a: node_a, node_b: node_b} = start_pair(context) + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + for index <- 1..4 do + TestCluster.spawn_register(node_a, name, "snapshot/rejected/#{index}", %{ + payload: String.duplicate("x", 160) + }) + end + + TestCluster.flush_shards(node_a, name) + stream_id = local_stream(node_a, name, nil) + frames = capture_snapshot(node_a, node_b, name, stream_id, 1) + + rows = + frames + |> Enum.flat_map(&elem(&1, 8)) + |> Enum.take(2) + + assert [first_row, second_row] = rows + {:snapshot_chunk, version, ^stream_id, snapshot_seq, _, _, _, _, _, _} = hd(frames) + assert snapshot_staging_tables(node_b, name) == [] + + overflow = + {:snapshot_chunk, version, stream_id, snapshot_seq, 1, 2, 1, 1, [first_row, second_row], []} + + deliver_frames(node_b, node_a, name, [overflow]) + assert snapshot_transfer_count(node_b, name) == 0 + assert snapshot_staging_tables(node_b, name) == [] + + duplicate = + {:snapshot_chunk, version, stream_id, snapshot_seq, 1, 2, 2, 0, [first_row, first_row], []} + + deliver_frames(node_b, node_a, name, [duplicate]) + assert snapshot_transfer_count(node_b, name) == 0 + assert snapshot_staging_tables(node_b, name) == [] + end + + test "nodedown immediately destroys partial staging owned by the retired source", context do + %{name: name, node_a: node_a, node_b: node_b} = start_pair(context) + on_exit(fn -> TestCluster.reconnect_nodes(node_a, node_b) end) + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + for index <- 1..4 do + TestCluster.spawn_register(node_a, name, "snapshot/nodedown/#{index}", %{ + payload: String.duplicate("n", 160) + }) + end + + TestCluster.flush_shards(node_a, name) + stream_id = local_stream(node_a, name, nil) + frames = capture_snapshot(node_a, node_b, name, stream_id, 1) + assert length(frames) > 1 + deliver_frames(node_b, node_a, name, [hd(frames)]) + assert snapshot_transfer_count(node_b, name) == 1 + + TestCluster.disconnect_nodes(node_a, node_b) + + TestCluster.assert_eventually(fn -> + node_a not in TestCluster.rpc!(node_b, Group, :nodes, [name]) + end) + + assert snapshot_transfer_count(node_b, name) == 0 + assert snapshot_staging_tables(node_b, name) == [] + end + test "an authority epoch change fences partial chunks and their staging expires", context do cluster = "snapshot-epoch" @@ -314,6 +380,9 @@ defmodule Group.ReplicaSnapshotDistributedTest do new_generation end) + assert snapshot_transfer_count(node_b, name) == 0 + assert snapshot_staging_tables(node_b, name) == [] + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) new_entries = @@ -604,4 +673,8 @@ defmodule Group.ReplicaSnapshotDistributedTest do TestCluster.rpc!(node, :sys, :get_state, [Group.Replica.shard_name(name, 0)]).snapshot_transfers ]) end + + defp snapshot_staging_tables(node, name) do + TestCluster.rpc!(node, TestCluster, :snapshot_staging_tables, [name, 0]) + end end diff --git a/test/replica_transport_outbox_test.exs b/test/replica_transport_outbox_test.exs index c2739ea..64aadea 100644 --- a/test/replica_transport_outbox_test.exs +++ b/test/replica_transport_outbox_test.exs @@ -137,6 +137,33 @@ defmodule Group.ReplicaTransportOutboxTest do refute Process.alive?(receiver) end + test "incoming messages are dropped when the destination shard is unavailable" do + group = unique_group(:missing_ingress) + source_node = :"missing-ingress@test" + + assert :disconnected = Group.Transport.incoming(group, source_node, 0, {:heads, 1, []}) + + assert :disconnected = + Group.Transport.incoming_batch(group, source_node, 0, [ + {:heads, 1, []}, + {:needs, 1, []} + ]) + end + + test "invalid outbox deadlines are rejected when the outbox supervisor boots" do + group = unique_group(:invalid_deadline) + + assert_raise ArgumentError, ~r/:outbox_deadline/, fn -> + Group.Transport.Outbox.Supervisor.init( + name: group, + num_shards: 1, + backend: Backend, + controller: self(), + outbox_deadline: 0 + ) + end + end + defp start_outboxes(group, opts) do base = [ name: group, diff --git a/test/support/test_cluster.ex b/test/support/test_cluster.ex index ab11a61..5c5990d 100644 --- a/test/support/test_cluster.ex +++ b/test/support/test_cluster.ex @@ -113,7 +113,7 @@ defmodule Group.TestCluster do spawn(fn -> :ok = Group.register(name, key, meta) send(parent, {:registered, self()}) - Process.sleep(:infinity) + registration_owner_loop(name, key) end) receive do @@ -132,6 +132,26 @@ defmodule Group.TestCluster do end) end + @doc false + def unregister_owner(pid) when is_pid(pid) do + send(pid, {:unregister, self()}) + + receive do + {:unregistered, ^pid, result} -> result + after + 5_000 -> raise "unregister_owner timed out" + end + end + + defp registration_owner_loop(name, key) do + receive do + {:unregister, reply_to} when is_pid(reply_to) -> + result = Group.unregister(name, key) + send(reply_to, {:unregistered, self(), result}) + registration_owner_loop(name, key) + end + end + @doc "Spawn a process on a remote node that joins and sleeps forever. Waits for the join to complete before returning." def spawn_join(node, name, key, meta, opts \\ []) do @@ -525,6 +545,24 @@ defmodule Group.TestCluster do end end + @doc false + def snapshot_staging_tables(name, shard_index) do + owner = Process.whereis(Group.Replica.shard_name(name, shard_index)) + + :ets.all() + |> Enum.filter(fn table -> :ets.info(table, :owner) == owner end) + end + + @doc false + def delete_remote_view_info(name, shard_index, remote_node) do + :ets.delete( + Group.Replica.Data.replication_meta_table(name), + {:remote_view_info, shard_index, remote_node} + ) + + :ok + end + @doc "Returns the current message_queue_len for a shard on a remote node." def shard_message_queue_len(node, name, shard) do :erpc.call(node, __MODULE__, :do_shard_message_queue_len, [name, shard]) @@ -848,10 +886,23 @@ defmodule Group.TestCluster do missing_authority = MapSet.difference(visible, claims) + claimed_keys = + MapSet.new(claims, fn {cluster, key, _pid, _meta, _time, _origin} -> {cluster, key} end) + + visible_keys = + MapSet.new(visible, fn {cluster, key, _pid, _meta, _time, _origin} -> {cluster, key} end) + + missing_projection = MapSet.difference(claimed_keys, visible_keys) + if MapSet.size(missing_authority) > 0 do raise "visible registry rows without an authoritative claim in #{name} shard #{shard}: " <> inspect(MapSet.to_list(missing_authority)) end + + if MapSet.size(missing_projection) > 0 do + raise "authoritative registry claims without a visible projection in #{name} shard #{shard}: " <> + inspect(MapSet.to_list(missing_projection)) + end end defp assert_oplog_indexes(name, shard) do diff --git a/test/support/test_tcp_transport.ex b/test/support/test_tcp_transport.ex index 39dbce2..82e8ca7 100644 --- a/test/support/test_tcp_transport.ex +++ b/test/support/test_tcp_transport.ex @@ -66,13 +66,13 @@ defmodule Group.TestTCPTransport do end @impl true - def peer_up(group, remote_node, descriptor, _opts) do - send_manager(group, {:peer_up, remote_node, descriptor}) + def peer_up(group, remote_node, shard, descriptor, _opts) do + send_manager(group, {:peer_up, remote_node, shard, descriptor}) end @impl true - def peer_down(group, remote_node, _opts) do - send_manager(group, {:peer_down, remote_node}) + def peer_down(group, remote_node, shard, _opts) do + send_manager(group, {:peer_down, remote_node, shard}) end @doc false @@ -146,8 +146,14 @@ defmodule Group.TestTCPTransport do end @impl true - def handle_info({:peer_up, remote_node, descriptor}, state) do - state = %{state | peers: Map.put(state.peers, remote_node, descriptor)} + def handle_info({:peer_up, remote_node, shard, descriptor}, state) do + peer = + state.peers + |> Map.get(remote_node, %{descriptor: descriptor, lanes: MapSet.new()}) + |> Map.put(:descriptor, descriptor) + |> Map.update!(:lanes, &MapSet.put(&1, shard)) + + state = %{state | peers: Map.put(state.peers, remote_node, peer)} state = if MapSet.member?(state.disabled, remote_node) do @@ -159,8 +165,21 @@ defmodule Group.TestTCPTransport do {:noreply, state} end - def handle_info({:peer_down, remote_node}, state) do - {:noreply, drop_peer(state, remote_node, true)} + def handle_info({:peer_down, remote_node, shard}, state) do + case Map.get(state.peers, remote_node) do + nil -> + {:noreply, state} + + peer -> + lanes = MapSet.delete(peer.lanes, shard) + + if MapSet.size(lanes) == 0 do + {:noreply, drop_peer(state, remote_node, true)} + else + peer = %{peer | lanes: lanes} + {:noreply, %{state | peers: Map.put(state.peers, remote_node, peer)}} + end + end end def handle_info({:writer_ready, remote_node, writer, queued}, state) do @@ -249,7 +268,7 @@ defmodule Group.TestTCPTransport do Map.has_key?(state.writers, remote_node) -> state - descriptor = Map.get(state.peers, remote_node) -> + peer = Map.get(state.peers, remote_node) -> manager = self() writer = @@ -258,7 +277,7 @@ defmodule Group.TestTCPTransport do manager, state.group, remote_node, - descriptor, + peer.descriptor, state.connect_timeout, state.send_timeout ) @@ -411,8 +430,10 @@ defmodule Group.TestTCPTransport do case decode_authenticated_frame(payload) do {:ok, {:batch, shard, messages}} when is_integer(shard) and shard >= 0 and is_list(messages) -> - :ok = Group.Transport.incoming_batch(group, source_node, shard, messages) - reader_loop(socket, group, source_node) + case Group.Transport.incoming_batch(group, source_node, shard, messages) do + result when result in [:ok, :disconnected] -> + reader_loop(socket, group, source_node) + end _ -> :gen_tcp.close(socket) From 28cd76a3e7dc6339c8460848f28772a765733b68 Mon Sep 17 00:00:00 2001 From: Chris McCord Date: Fri, 14 Aug 2026 14:43:16 +0000 Subject: [PATCH 09/16] Harden anti-entropy correctness and recovery --- CLAUDE.md | 82 +- README.md | 75 +- lib/group.ex | 79 +- lib/group/replica.ex | 2556 ++++++------ lib/group/replica/data.ex | 1135 ++++-- lib/group/replica/wire_protocol.ex | 28 + lib/group/transport.ex | 13 +- lib/group/transport/outbox.ex | 60 +- test/README.md | 50 +- test/anti_entropy_fault_regression_test.exs | 4027 ++++++++++++++++++- test/distributed_test.exs | 185 +- test/formal/AuthorityHint.cfg | 17 + test/formal/AuthorityHint.tla | 249 ++ test/formal/AuthorityProjection.cfg | 10 + test/formal/AuthorityProjection.tla | 117 + test/formal/README.md | 27 +- test/formal/check_matrix.sh | 2 + test/group_test.exs | 1366 +++---- test/jepsen/node.exs | 4 +- test/mutation/README.md | 20 + test/mutation/run.exs | 719 +++- test/replica_adversarial_test.exs | 18 +- test/replica_snapshot_distributed_test.exs | 177 +- test/replica_transport_outbox_test.exs | 35 + test/support/cyclic_conflict_resolver.ex | 19 + test/support/model_conflict_resolver.ex | 12 +- test/support/pausing_conflict_resolver.ex | 19 + test/support/test_cluster.ex | 303 +- test/support/test_conflict_resolver.ex | 7 +- test/support/test_replica_transport.ex | 24 + 30 files changed, 8726 insertions(+), 2709 deletions(-) create mode 100644 test/formal/AuthorityHint.cfg create mode 100644 test/formal/AuthorityHint.tla create mode 100644 test/formal/AuthorityProjection.cfg create mode 100644 test/formal/AuthorityProjection.tla create mode 100644 test/support/cyclic_conflict_resolver.ex create mode 100644 test/support/pausing_conflict_resolver.ex diff --git a/CLAUDE.md b/CLAUDE.md index fe5f774..c20b86d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -120,18 +120,34 @@ Shard 0 installs one exact node-wide authority snapshot: - origin generation; - complete active cluster→epoch set; - exact authority revision; and -- authenticated transport descriptor. +- opaque transport descriptor. + +The exact authority and its shared-cluster forward/reverse indexes are one +serialized Data mutation. A concurrent local connect is ordered wholly before +or after that mutation, so exact authority cannot exist without the matching +replica route indefinitely. +Local activation projects its epoch, self route, and all already-exact remote +routes at that same boundary. Local deactivation removes admission and queues +old-epoch cleanup to every shard before replying; callers and timed-out aliases +are not lifecycle coordinators. Other shards exchange constant-size lane hellos. Shared authority is not enough to accept data: each shard records an installed lane view only after it has purged streams outside that authority. Exact and incrementally observed -revisions are distinct; heartbeats and partial cluster-control bursts can never -promote an incomplete epoch set to exact authority. +revisions are distinct from the complete applied revision. A persisted +`{generation, revision}` hint atomically fences every lane when any heartbeat +observes newer authority. Incremental controls compare-and-install their +expected revision against the applied/observed/hinted state in one Data turn, +so a raced heartbeat cannot publish partial authority. Hints refine only a +peer with prior exact authority; after retirement, delayed heartbeats or lane +hellos cannot recreate authority, a route, or a lease. Only the exact +dist-Erlang hello reintroduces the peer. If a lane hello precedes exact +authority during discovery, authority fanout immediately re-probes that lane; +the rejected hello is never retained as a route. `peer_connect` and `peer_connect_ack` are discovery hints, not authority. -The old `cluster_state` handler is receive-only rolling compatibility; new -replica recovery must use heads/deltas/exact snapshots and must not add new -dependencies on additive full-state merge. +Unsequenced legacy state messages are ignored; all replica recovery uses +heads, contiguous deltas, or exact snapshots. ## Anti-Entropy @@ -158,6 +174,9 @@ private ETS and preserve the old visible slice/cursor until the complete, authority-valid manifest is present. Loss, duplication, reordering, supersession, stale authority, expiry, and shard crash must leave no partial visible state. Staging expires after one peer-lease interval without progress. +At most one sender worker per shard captures rows off the control process and +sends only if the stream identity and fully-applied head remain unchanged after +the scan; otherwise anti-entropy retries. ## Nonblocking Transport @@ -167,8 +186,9 @@ adapter sends directly the same way and adds no local hop. `:busy` and `:disconnected` mean “drop this message”; periodic anti-entropy repairs it. A sideband adapter may use one local `Group.Transport.Outbox` per shard. -Outboxes batch by peer, impose deadlines, and run bounded socket work outside -Group shards. Queue overflow, expiry, or socket backpressure drops the batch. +Outboxes batch by peer, bound admitted messages, impose deadlines, and run +bounded socket work outside Group shards. Queue overflow, expiry, or socket +backpressure drops the batch. The test suite's hidden TCP adapter exercises a genuinely independent socket lane; it is validation infrastructure, not a supported production transport. @@ -180,10 +200,10 @@ reassemble any transport segmentation before `incoming_batch/4`. ## Registry Projection and Process Ownership Registry claims remain authoritative per origin even when hidden by another -origin's visible winner. Conflict resolution folds claims deterministically. -The configured callback chooses a pid (or neither), but Group owns lifecycle -effects: each losing origin appends its own authoritative unregister and exits -only its local process with +origin's visible winner. The configured callback deterministically ranks each +claim independently; Group selects the maximum `{rank, pid}` so the result is +associative and commutative. Group owns lifecycle effects: each losing origin +appends its own authoritative unregister and exits only its local process with `{:group_registry_conflict, key, winner_meta}`. A node never monitors or exits another node's member processes. Local shards @@ -199,11 +219,24 @@ an origin that cannot emit them. - Constant-size control heartbeats cover the case where the Erlang node remains connected but its Group instance disappears. Peer-lease expiry performs the same complete purge. +- After a receiver shard restart, retained authority/view metadata reconstructs + leases for origins whose ETS rows survived, so a permanently disappeared + Group is still evicted without scanning registry or PG data. +- Each lane deletes its own persisted view only after purging its rows and + cursors; shard 0 never erases a sibling's restart breadcrumb. If shard 0 + restarts with `observed_revision != exact_revision`, it also reconstructs the + pending exact-hello repair obligation. Persisted hints reconstruct the same + bounded lease if a lane crashes after fencing authority but before updating + its in-memory deadline. - Discovery probes continue after expiry. A returning current/new generation is fenced, installs authority per lane, and reconstructs through deltas or an - exact snapshot. + exact snapshot. Delayed cleanup rechecks shared authority before deleting + routes, and a lane hello is not reported `peer_up` until authority admits it. - Incremental named-cluster open/close controls are generation fenced and - batched. A quiet exact hello repairs dropped/reordered control messages. + batched, but shard 0 is their only node-wide authority writer; controls + arriving on other lanes are forwarded locally. Revisions must be contiguous. + A gap fences every lane and requests an exact hello instead of applying a + partial authority set. - Local cluster close uses a temporary all-shard completion barrier. The last shard removes routing/epoch rows; restart repair completes abandoned closes, and reconnect waits so an old close cannot erase new writes. @@ -236,19 +269,30 @@ FIFO local-request turn to prevent replica pressure from starving callers. 1. A cursor never advances across a gap or before a full exact snapshot commits. 2. Exact snapshots replace one origin slice; they are never additive merges. -3. Authority requires generation, exact epoch revision, and installed lane - readiness. Observed heartbeats/controls are not exact authority. +3. Authority requires generation, complete applied epoch revision, a matching + persisted observation hint, and installed lane readiness. Observed + heartbeats alone are not authority; the last exact snapshot revision remains + separate while a contiguous incremental update is applied. Exact + authority and its shared-cluster routing projection are installed atomically. + Local activation is projected atomically; deactivation cleanup is durable + before its initiating caller can disappear. 4. A stale generation, epoch, lane, shard, transitive pid, or stream whose origin differs from the transport-reported source is rejected before applying replica data. Group trusts the adapter's source identity; authenticating a sideband peer belongs to that transport. 5. Registry claims are retained per origin until that origin deletes them or is - retired; the visible winner is reconstructible from remaining claims. + retired; the visible winner is reconstructible from remaining claims. A + remote winner is serially revalidated against shared and lane authority + immediately before any local losing owner is retired. Keys reconciled while + a retained claim is fenced by an authority gap are reprojected when that + lane installs the exact view; a current cursor must not strand an older + visible winner. 6. Only an owner node monitors, retires, or exits its member processes. 7. Oplog pruning is local and bounded; lagging peers use exact snapshot repair. 8. `nodedown` and peer-lease expiry purge every public and internal reference - to the retired origin. Remote shard death cannot leave state permanently; - lease expiry or fenced rediscovery completes cleanup/recovery. + to the retired origin. A lane retains its persisted restart breadcrumb until + its own rows/cursors are gone. Remote shard death cannot leave state + permanently; lease expiry or fenced rediscovery completes cleanup/recovery. 9. Snapshot staging is private, all-or-nothing, authority fenced, and expiring. 10. Local append/journal repair makes an appended mutation either replayable or durably applied after a shard crash. diff --git a/README.md b/README.md index ab58877..856edf1 100644 --- a/README.md +++ b/README.md @@ -267,12 +267,13 @@ All operations are **eventually consistent**: `Logger.error` events and busy distribution links remain `Logger.warning` events. The level can be changed at runtime with `Group.log_level/2`. - **`resolve_registry_conflict`** — `{module, function, extra_args}` callback - invoked as `apply(mod, fun, [name, key, {pid1, meta1, time1}, {pid2, meta2, time2} | extra_args])`. - Called when partition healing or concurrent registration finds the same key - registered on two nodes. Must return the winning pid (or neither pid to - reject both). Group records an authoritative delete and terminates a losing - owner only on that owner's local node. The callback runs synchronously inside - the shard GenServer, so it must return quickly and never block. + invoked once per competing claim as + `apply(mod, fun, [name, key, {pid, meta, time} | extra_args])`. It must return + a deterministic Erlang term used as the claim's rank. Group chooses the + maximum `{rank, pid}`, making the winner independent of delivery order and + grouping. Group records an authoritative delete and terminates a losing owner + only on that owner's local node. The callback runs synchronously inside the + shard GenServer, so it must return quickly and never block. - **`extract_meta`** — `{module, function, args}` or `fun(meta)` applied to metadata on reads and lifecycle events. Useful for stripping internal fields. - **`replicated_pg_receiver_buffer_size`** — max buffered replicated PG @@ -413,12 +414,26 @@ batched, and installed by shard 0 into one node-wide authority table. The highest observed revision keeps heartbeats constant-size during a burst; after the burst becomes quiet, one authoritative hello closes any gaps left by dropped or reordered controls. Per-shard view rows record only constant-size -lane readiness; they do not copy the epoch map. Snapshot capture is serialized -with local epoch activation, so its revision and epoch rows are one coherent -point-in-time value. The highest observed incremental revision is tracked -separately and can never promote a partial view to exact authority. Discovery -hints never mutate membership on their own. Authority installation fans a -local fence to every lane, which sweeps only that lane's retained receive +lane readiness; they do not copy the epoch map. The highest observed +incremental revision, complete applied revision, and last exact revision are +tracked separately. Data installs a contiguous incremental batch only if its +expected generation/revision still matches the applied authority, observation, +and persisted hint in the same serialized callback; a raced heartbeat rejects +the whole batch. A persisted `{generation, revision}` hint fences every lane +when any heartbeat observes newer authority. It can refine only an already +known peer: after complete retirement, delayed heartbeats and lane hellos cannot +recreate authority, a transport route, or a lease. Only an exact dist-Erlang +hello reintroduces the peer. Discovery hints never mutate membership on their +own. +The exact authority and its shared-cluster forward/reverse index rows are +replaced in one serialized Data operation. This closes the race where a local +cluster connect and a remote exact install could each miss the other's state +and permanently omit a valid replica route. Local activation likewise installs +its epoch, self route, and already-exact remote routes in one Data turn. Local +deactivation removes admission and queues idempotent old-epoch cleanup on every +shard before returning; API timeout or caller death cannot strand rows, routes, +or a close barrier. Authority installation then fans a local fence to every +lane, which sweeps only that lane's retained receive streams. Shared authority may become visible before that fanout reaches a lane, but the lane's constant-size view is not marked installed until its purge finishes; data validation requires that marker. A heartbeat or lane hello can @@ -427,6 +442,22 @@ intentionally do not carry protocol epochs, a superseded origin/cluster slice is cleared and its current cursor reset so the next head reconstructs it from retained deltas or an exact snapshot. +If a receiver shard restarts while Data retains remote rows, it reconstructs +lease candidates from constant-size authority/view metadata rather than +scanning registry or PG entries. A live Group refreshes through discovery; a +Group that never returns is purged after the normal bounded lease timeout. +Each lane removes that persisted view only after purging its own rows and +cursors, so shard 0 cannot erase a suspended sibling's restart breadcrumb. +Shard 0 also reconstructs any pending applied/observed-versus-exact authority +repair. Persisted hints seed the same bounded retirement lease after a lane +restart, and final route cleanup rechecks that no newer exact authority or hint +was installed while an older cleanup caller was delayed. +Registry conflicts reconciled during that temporary authority gap retain a +bounded set of affected keys in the receiving lane. Installing the exact view +reprojects those keys before normal processing resumes, so retained current +claims cannot be hidden forever behind a stale visible winner and authority +repair never requires scanning every claim in the shard. + Replica state itself does not travel on the control plane. Once the hello is fenced, stream-head exchange on the replica transport catches the peer up. @@ -477,7 +508,8 @@ authority and membership remain on dist Erlang: replica_transport: {MyApp.GroupTransport, [outbox_batch_size: 64, outbox_batch_bytes: 1_048_576, - outbox_flush_interval: 1, outbox_deadline: 100]} + outbox_flush_interval: 1, outbox_deadline: 100, + outbox_max_messages: 1_024]} ``` The default `Group.Transport.DistErl` adapter sends directly to the remote shard and @@ -488,19 +520,28 @@ target and invokes the adapter's `send_batch/4` callback. Calls that expire or return `:busy`/`:disconnected` are dropped without a local retry; the next anti-entropy exchange repairs them. +Exact-snapshot row capture runs in at most one off-shard worker per shard. The +worker sends only when the stream identity and fully-applied head are unchanged +after its scans. A concurrent write invalidates the capture and periodic +anti-entropy retries, keeping million-row scans off the Group control process. + The optional `peer_up/5` and `peer_down/4` callbacks report one shard lane at a time. A sideband adapter that shares a single node connection must retain it while any reported lane remains live and release it after the last lane goes down. `incoming/4` and `incoming_batch/4` return `:disconnected` and drop when their local shard is restarting; an ingress reader must treat that as an -expected lossy delivery outcome. +expected lossy delivery outcome. Group invokes `peer_up/5` and records an +outbound lane only after exact/current authority admits it; a delayed lane hello +after peer retirement remains a side-effect-free request for exact authority. +When a hello legitimately outruns first-time exact authority, authority fanout +immediately re-probes that shard without retaining a speculative route or +waiting for the periodic anti-entropy interval. A message-oriented backend fits this callback shape by obtaining a connection once from `init_outbox/3`, then sending each `send_batch/4` result to a registered incoming name on the target node. Queue pressure maps to `:busy` and -a missing session maps to `:disconnected`. The adapter passes its trusted peer -identity as the source node; Group verifies that stream origins and member pids -match that identity but does not authenticate the sideband connection itself. +a missing session maps to `:disconnected`. The adapter passes its peer node as +`source_node`; Group verifies that stream origins and member pids match it. Exact snapshots are already bounded by Group. A transport with a smaller maximum frame may additionally segment an encoded batch, but it must completely reassemble that batch before calling diff --git a/lib/group.ex b/lib/group.ex index 284b8da..ae703d8 100644 --- a/lib/group.ex +++ b/lib/group.ex @@ -176,6 +176,11 @@ defmodule Group do - **Process ownership is local**: a shard monitors and exits only processes owned by its own node. Remote lifecycle changes arrive as sequenced replica records or are removed by `nodedown`/peer-lease expiry. + + - **Replica recovery is bounded and leaderless**: per-origin sequence gaps use + retained deltas and then exact origin-slice snapshots after oplog pruning. + Persisted generation/revision hints fence every lane across authority races; + only an exact dist-Erlang hello can reintroduce a fully retired peer. """ alias Group.Replica @@ -200,10 +205,13 @@ defmodule Group do replication messages. `false` disables routine info/verbose logs; registry conflicts and busy distribution links still emit error/warning logs. - `:resolve_registry_conflict` — `{module, function, extra_args}` callback invoked when - two nodes hold the same registry key (partition heal or concurrent registration). - Called as `apply(module, function, [name, key, {pid1, meta1, time1}, {pid2, meta2, time2} | extra_args])`. - Must return the winner pid (or neither contender to reject both). Group records - an authoritative delete and terminates a losing process only on its owner node. + multiple origins hold the same registry key after partition healing or concurrent + registration. Called once per claim as + `apply(module, function, [name, key, {pid, meta, time} | extra_args])` and must + return a deterministic Erlang term used as that claim's rank. Group chooses the + maximum `{rank, pid}` so every node obtains the same winner regardless of claim + arrival or grouping. It records an authoritative delete and terminates a losing + process only on its owner node. **Important:** This callback runs synchronously inside the shard GenServer — it must return quickly and never block. Any information needed for the decision should be carried in the registration metadata, not fetched at resolution time. @@ -463,10 +471,14 @@ defmodule Group do when is_atom(name) and is_binary(key) and is_map(meta) and is_list(opts) do validate_key!(key) cluster = Keyword.get(opts, :cluster) - validate_cluster_connected!(name, cluster) + epoch = validate_cluster_connected!(name, cluster) shard = Replica.shard_for(name, cluster, key) - Replica.local_request(shard, {:register, cluster, key, self(), meta}, call_timeout(opts)) + Replica.local_request( + shard, + {:register, cluster, epoch, key, self(), meta}, + call_timeout(opts) + ) end @doc """ @@ -497,10 +509,10 @@ defmodule Group do when is_atom(name) and is_binary(key) and is_list(opts) do validate_key!(key) cluster = Keyword.get(opts, :cluster) - validate_cluster_connected!(name, cluster) + epoch = validate_cluster_connected!(name, cluster) shard = Replica.shard_for(name, cluster, key) - Replica.local_request(shard, {:unregister, cluster, key}, call_timeout(opts)) + Replica.local_request(shard, {:unregister, cluster, epoch, key}, call_timeout(opts)) end @doc """ @@ -654,10 +666,14 @@ defmodule Group do is_list(opts) do validate_key!(group) cluster = Keyword.get(opts, :cluster) - validate_cluster_connected!(name, cluster) + epoch = validate_cluster_connected!(name, cluster) shard = Replica.shard_for(name, cluster, group) - Replica.local_request(shard, {:join, cluster, group, self(), meta}, call_timeout(opts)) + Replica.local_request( + shard, + {:join, cluster, epoch, group, self(), meta}, + call_timeout(opts) + ) end @doc """ @@ -685,10 +701,10 @@ defmodule Group do when is_atom(name) and is_binary(group) and is_list(opts) do validate_key!(group) cluster = Keyword.get(opts, :cluster) - validate_cluster_connected!(name, cluster) + epoch = validate_cluster_connected!(name, cluster) shard = Replica.shard_for(name, cluster, group) - Replica.local_request(shard, {:leave, cluster, group, self()}, call_timeout(opts)) + Replica.local_request(shard, {:leave, cluster, epoch, group, self()}, call_timeout(opts)) end # =========================================================================== @@ -1119,14 +1135,13 @@ defmodule Group do def connect_clusters(name, clusters, timeout) when is_atom(name) and is_list(clusters) and is_integer(timeout) do timeout = Data.await_closed_local_clusters(name, clusters, timeout) - _epochs = Data.activate_local_clusters(name, clusters) - Data.add_cluster_node(name, clusters, node()) + epochs = Data.activate_local_clusters_durable(name, clusters) notify_shard = :rand.uniform(get_config(name).num_shards) - 1 Replica.local_request( Replica.shard_name(name, notify_shard), - {:cluster_connect, clusters}, + {:cluster_connect, clusters, epochs}, timeout ) end @@ -1134,25 +1149,9 @@ defmodule Group do @doc false def disconnect_clusters(name, clusters, timeout) when is_atom(name) and is_list(clusters) and is_integer(timeout) do - _epochs = Data.deactivate_local_clusters(name, clusters) - Data.remove_cluster_node(name, clusters, node()) - - num_shards = get_config(name).num_shards - - shard_names = for i <- 0..(num_shards - 1), do: Replica.shard_name(name, i) - - result = - Replica.local_request_all( - shard_names, - {:cluster_disconnect, clusters}, - timeout - ) - - # Keep the remote routing rows through the shard barrier so any buffered - # records are dispatched before the cluster-close control message. Once - # every shard has crossed the barrier, no cluster rows may remain locally. - Data.remove_clusters(name, clusters) - result + _epochs = Data.deactivate_local_clusters_durable(name, clusters) + _remaining_timeout = Data.await_closed_local_clusters(name, clusters, timeout) + :ok end # =========================================================================== @@ -1224,12 +1223,16 @@ defmodule Group do |> List.flatten() end - defp validate_cluster_connected!(_name, nil), do: :ok + defp validate_cluster_connected!(name, nil), do: Data.generation(name) defp validate_cluster_connected!(name, cluster) do - unless node() in Data.cluster_nodes(name, cluster) do - raise ArgumentError, - "not connected to cluster #{inspect(cluster)}. Call Group.connect(#{inspect(name)}, #{inspect(cluster)}) first" + case Data.local_cluster_epoch(name, cluster) do + nil -> + raise ArgumentError, + "not connected to cluster #{inspect(cluster)}. Call Group.connect(#{inspect(name)}, #{inspect(cluster)}) first" + + epoch -> + epoch end end diff --git a/lib/group/replica.ex b/lib/group/replica.ex index 6a7bc88..63df82c 100644 --- a/lib/group/replica.ex +++ b/lib/group/replica.ex @@ -11,6 +11,8 @@ defmodule Group.Replica do @local_request_tag :group_local_request @local_reply_tag :group_local_reply @protocol_version Group.Replica.WireProtocol.version() + @priority_control_quota 64 + @incoming_batch_quota 64 _archdoc = ~S""" Sharded control process for local writes, replica transport, anti-entropy, @@ -32,14 +34,20 @@ defmodule Group.Replica do rebuilding local process monitors. A registry's authoritative claims are stored per origin separately from its - single visible winner. Conflict selection folds claims in a stable order. + single visible winner. Conflict selection computes one deterministic rank per + claim and chooses the maximum `{rank, pid}`, so selection is associative, + commutative, and independent of delivery order. When a local claim loses, only its owner node appends the authoritative - unregister and terminates the local process. Retaining hidden remote claims - until their origin deletes them prevents a later winner change from orphaning - or permanently forgetting a live claim. Local unregister and process-DOWN - paths always re-project every affected key after deleting their claims. On a - shard restart, the complete claim/key union is projected again, closing the - crash window between journal application and the materialized winner write. + unregister and terminates the local process. Immediately before that + irreversible step, Data serially revalidates the remote winner against the + node-wide authority and this lane's installed view; an authority change + restarts selection without killing either owner. Retaining hidden remote + claims until their origin deletes them prevents a later winner change from + orphaning or permanently forgetting a live claim. Local unregister and + process-DOWN paths always re-project every affected key after deleting their + claims. On a shard restart, the complete claim/key union is projected again, + closing the crash window between journal application and the materialized + winner write. PG memberships need no winner projection: the origin stream owns exactly the rows whose member processes live on that origin node. @@ -58,6 +66,28 @@ defmodule Group.Replica do creating remote process monitors. A generation/epoch-revision mismatch requests a fresh authoritative hello. + Data installs an exact remote authority and its shared-cluster dual-index + projection in one serialized operation. A concurrent local cluster connect + therefore observes either the old authority and is covered by the install, + or the new authority and projects the peer itself; neither ordering can leave + exact authority permanently disconnected from replica routing. + Local activation uses the same Data serialization point to install its epoch, + self route, and every already-exact remote route. Local deactivation removes + admission and enqueues idempotent cleanup to every shard before Data replies. + The public API still waits for the shard barrier, but caller survival is not a + correctness dependency; shard restart repair consumes the same close marker. + + One persisted `{generation, revision}` authority hint is the cross-lane + fence. A heartbeat or lane hello may advance it only for a peer that already + has exact authority; Data invalidates every installed lane view in the same + turn. A delayed hint after complete retirement is discovery only: it cannot + recreate authority, a transport route, or an unbounded lease. Only the + dist-Erlang exact hello reintroduces that peer. Contiguous incremental cluster + controls compare their expected generation/revision against the applied + authority, observed revision, and hint inside one Data callback. A raced + heartbeat therefore rejects the complete incremental write instead of + installing a partial epoch set. + Replica state uses the configured Group.Transport: - heads advertises {stream, retained_floor, head}. @@ -77,6 +107,8 @@ defmodule Group.Replica do Rejected first chunks destroy their staging table immediately. Node loss, generation replacement, and retired cluster streams discard matching partial assemblies immediately rather than waiting for their inactivity deadline. + Unsequenced legacy state messages and malformed replica frames are rejected + before they can mutate a cursor or materialized row. ## Bounded recovery @@ -99,7 +131,12 @@ defmodule Group.Replica do Optional peer lifecycle callbacks are per shard lane. A sideband adapter that shares one node connection across shards keeps that route until its final - live lane goes down; one flapping shard cannot disconnect healthy lanes. + live lane goes down; one flapping shard cannot disconnect healthy lanes. A + lane hello calls `peer_up` and enters `remote_shards` only after exact/current + authority admits that lane, so delayed post-retirement hellos cannot strand + an unleased transport route. If a lane hello arrives before exact authority, + authority fanout immediately repeats shard-local discovery instead of waiting + for the periodic anti-entropy probe. The transport need not order messages for correctness. The local shard serializes writes, sequence numbers establish per-stream order, and receivers @@ -114,12 +151,30 @@ defmodule Group.Replica do control/routing barriers, and idle timers bound the delay. Incremental cluster controls are generation fenced, receiver batched, and - installed by shard 0 into the node-wide authority table. Their observed - revision suppresses full-hello storms during bursts; a quiet authoritative - hello repairs any missing or reordered controls. Snapshot capture is - serialized with epoch activation, and the last exact revision is distinct - from the highest incrementally observed revision. Shard-local lane readiness - is separate from shared authority, so no epoch map is copied per shard. + installed only by shard 0 into the node-wide authority table; controls that + arrive on another lane are forwarded locally to that single owner. Revisions + must be contiguous. A gap records the highest observation, fences every lane, + and requests an exact hello instead of applying a partial authority set. The + last exact revision, complete applied revision, and highest observed revision + are distinct. The Data owner compare-and-installs an incremental batch only + when its expected revision still matches the applied revision and persisted + hint. Shard-local lane readiness is separate from shared authority, so no + epoch map is copied per shard. On shard restart, constant-size retained + authority/view metadata seeds leases for origins whose rows survived in Data; + a disappeared Group is still retired after the normal bounded timeout. Each + lane deletes its own view only after purging its rows/cursors, so shard 0 + cannot erase a sibling's restart breadcrumb. Shard 0 also reconstructs an + exact-hello obligation whenever the persisted observed authority revision is + newer than the last exact revision. If a registry conflict is reconciled + while one retained claim is temporarily fenced by such an authority gap, the + lane remembers only that key and reprojects it when the exact view is + installed. This avoids a shard-wide claim scan while ensuring a current + cursor can never strand an old visible winner. + + Exact-snapshot row capture runs in at most one off-shard worker per shard. It + sends only if the local stream identity and fully-applied head are unchanged + after both row scans; overlapping writes discard the capture and periodic + anti-entropy retries. This keeps million-row scans off the control process. Incoming PG mutations retain the bulk receiver lane. Contiguous registry records in one stream run are projected together and emit one monitor event @@ -127,11 +182,16 @@ defmodule Group.Replica do wire order and emits one combined batch. After replicated work, the shard takes a bounded local-request turn before - yielding. FIFO is preserved within the local request lane, while protocol and - cluster barriers flush earlier buffered state first. + yielding. Priority-control recursion is also quota-bounded. FIFO is preserved + within the local request lane, while protocol and cluster barriers flush + earlier buffered state first. Snapshot staging is owned by the receiving shard, expires after a peer lease - without progress, and disappears automatically if the shard crashes. + without progress, and disappears automatically if the shard crashes. Commit + first replaces the numeric receive cursor with a durable + `{:snapshot_installing, sequence}` marker. Startup repair sees that marker, + removes any partially replaced registry/PG slice, and clears the cursor so + anti-entropy requests the exact state again. """ require Logger @@ -172,9 +232,11 @@ defmodule Group.Replica do peer_last_seen: %{}, cluster_control_dirty: %{}, authority_dirty_notified: MapSet.new(), + pending_registry_reprojections: %{}, monitors: %{}, - peer_transports: %{}, - snapshot_transfers: %{} + snapshot_transfers: %{}, + snapshot_send: nil, + snapshot_send_offsets: %{} ] def start_link(opts) do @@ -224,6 +286,16 @@ defmodule Group.Replica do :ok end + @doc false + def local_cast(shard_name, request) when is_atom(shard_name) or is_pid(shard_name) do + case GenServer.whereis(shard_name) do + nil -> :ok + pid -> send(pid, {@local_request_tag, :noreply, request}) + end + + :ok + end + # ===================================================================== # GenServer callbacks # ===================================================================== @@ -270,15 +342,51 @@ defmodule Group.Replica do :ok = Data.repair_local_replica_journal(name, shard_index) state = replay_local_journal(state) :ok = Data.repair_shard_indexes(name, shard_index) + {state, _events} = rebuild_registry_projections(state) - completed_clusters = - Data.mark_closed_cluster_shard(name, Data.closed_local_clusters(name), shard_index) - - if completed_clusters != [], do: Data.remove_clusters(name, completed_clusters) + _completed_clusters = + Data.mark_closed_cluster_shard( + name, + Data.closed_local_cluster_epochs(name), + shard_index + ) # Rebuild monitors from any surviving ETS data (after shard crash/restart) state = rebuild_monitors(state) + # A shard can die after replica rows are materialized but before it handles + # the peer's retirement. The ETS owner survives a shard restart, while the + # in-memory lease map does not. Reconstruct every retained origin as a lease + # candidate: live Groups answer the normal discovery probe below and refresh + # the lease; permanently disappeared Groups are purged when it expires. + restarted_at = monotonic_millis() + retained_origins = Data.retained_replica_origins(name, shard_index) + + cluster_control_dirty = + if shard_index == 0 do + Enum.reduce(retained_origins, %{}, fn origin, dirty -> + exact = Data.remote_cluster_epoch_exact_revision(name, origin) + observed = Data.remote_cluster_epoch_observed_revision(name, origin) + known_generation = Data.remote_generation(name, origin) + hint = Data.remote_replica_authority_hint(name, origin) + + if (not is_nil(observed) and observed != exact) or + (not is_nil(hint) and elem(hint, 0) != known_generation) do + Map.put(dirty, origin, restarted_at) + else + dirty + end + end) + else + %{} + end + + state = %{ + state + | peer_last_seen: Map.new(retained_origins, &{&1, restarted_at}), + cluster_control_dirty: cluster_control_dirty + } + log_once(state, fn -> "#{log_prefix(state)} started (shards=#{num_shards})" end) # Discover peers on all known nodes @@ -304,12 +412,12 @@ defmodule Group.Replica do # ===================================================================== @impl true - def handle_call({:register, _, _, _, _} = request, _from, state) do + def handle_call({:register, _, _, _, _, _} = request, _from, state) do {reply, state} = process_local_request(state, request) {:reply, reply, state} end - def handle_call({:unregister, _, _} = request, _from, state) do + def handle_call({:unregister, _, _, _} = request, _from, state) do {reply, state} = process_local_request(state, request) {:reply, reply, state} end @@ -318,12 +426,12 @@ defmodule Group.Replica do # Process group calls # ===================================================================== - def handle_call({:join, _, _, _, _} = request, _from, state) do + def handle_call({:join, _, _, _, _, _} = request, _from, state) do {reply, state} = process_local_request(state, request) {:reply, reply, state} end - def handle_call({:leave, _, _, _} = request, _from, state) do + def handle_call({:leave, _, _, _, _} = request, _from, state) do {reply, state} = process_local_request(state, request) {:reply, reply, state} end @@ -357,24 +465,6 @@ defmodule Group.Replica do # ===================================================================== @impl true - def handle_info({:replicate_registry_batch, ops}, state) do - state = flush_pending_replicated_sender_barrier(state) - - {state, flushed?} = enqueue_replicated_registry_ops(state, ops) - - state = if flushed?, do: take_priority_turn(state), else: state - {:noreply, state} - end - - def handle_info({:replicate_pg_batch, ops}, state) do - state = flush_pending_replicated_sender_barrier(state) - - {state, flushed?} = enqueue_replicated_pg_ops(state, ops) - - state = if flushed?, do: take_priority_turn(state), else: state - {:noreply, state} - end - def handle_info( {:replica_hello, remote_pid, version, generation, epoch_revision, cluster_epochs, transport_id, transport_descriptor}, @@ -384,11 +474,22 @@ defmodule Group.Replica do remote_node = node(remote_pid) known_generation = Data.remote_generation(state.name, remote_node) + + hinted_generation = + case Data.remote_replica_authority_hint(state.name, remote_node) do + {hinted_generation, _revision} -> hinted_generation + nil -> known_generation + end + observed_revision = Data.remote_cluster_epoch_observed_revision(state.name, remote_node) authoritative_revision = Data.remote_cluster_epoch_revision(state.name, remote_node) exact_revision = Data.remote_cluster_epoch_exact_revision(state.name, remote_node) + stale_generation? = + not is_nil(hinted_generation) and hinted_generation != generation and + not WireProtocol.generation_newer?(generation, hinted_generation) + stale_revision? = known_generation == generation and Enum.any?([observed_revision, authoritative_revision], fn @@ -397,14 +498,15 @@ defmodule Group.Replica do end) cond do - version != WireProtocol.version() or transport_id != state.replica_transport.id() -> + version != WireProtocol.version() or transport_id != state.replica_transport.id() or + not WireProtocol.valid_generation?(generation) -> Logger.error( "#{log_prefix_shard(state)} incompatible replica protocol/transport from #{inspect(remote_node)}" ) {:noreply, state} - stale_revision? -> + stale_generation? or stale_revision? -> {:noreply, state} known_generation == generation and exact_revision == epoch_revision -> @@ -412,17 +514,6 @@ defmodule Group.Replica do # is being installed. Once this exact revision is present, another # identical hello is only a lease/descriptor refresh; reinstalling its # full epoch set would serialize every shard behind redundant ETS work. - state = notify_replica_transport_peer_up(state, remote_node, transport_descriptor) - - state = %{ - state - | remote_shards: Map.put(state.remote_shards, remote_node, remote_pid), - peer_last_seen: Map.put(state.peer_last_seen, remote_node, monotonic_millis()), - cluster_control_dirty: Map.delete(state.cluster_control_dirty, remote_node), - peer_transports: - Map.put(state.peer_transports, remote_node, {transport_id, transport_descriptor}) - } - state = if replica_view_current?(state, remote_node) do state @@ -430,7 +521,23 @@ defmodule Group.Replica do install_current_replica_lane(state, remote_node, generation) end - {:noreply, state} + if replica_authority_current?(state, remote_node, generation, epoch_revision) and + replica_view_current?(state, remote_node) do + state = notify_replica_transport_peer_up(state, remote_node, transport_descriptor) + + {:noreply, + %{ + state + | remote_shards: Map.put(state.remote_shards, remote_node, remote_pid), + peer_last_seen: Map.put(state.peer_last_seen, remote_node, monotonic_millis()), + cluster_control_dirty: Map.delete(state.cluster_control_dirty, remote_node) + }} + else + {:noreply, + state + |> mark_cluster_control_dirty(remote_node) + |> request_replica_authority(remote_node)} + end true -> {:noreply, @@ -466,29 +573,15 @@ defmodule Group.Replica do remote_node = node(remote_pid) if version == WireProtocol.version() and transport_id == state.replica_transport.id() do - if function_exported?(state.replica_transport, :peer_up, 5) do - :ok = - state.replica_transport.peer_up( - state.name, - remote_node, - state.shard_index, - transport_descriptor, - state.replica_transport_opts - ) - end - - state = %{ - state - | remote_shards: Map.put(state.remote_shards, remote_node, remote_pid), - peer_transports: - Map.put(state.peer_transports, remote_node, {transport_id, transport_descriptor}) - } + state = observe_replica_authority_hint(state, remote_node, generation, epoch_revision) cond do replica_authority_current?(state, remote_node, generation, epoch_revision) and replica_view_current?(state, remote_node) -> state = state + |> notify_replica_transport_peer_up(remote_node, transport_descriptor) + |> put_remote_shard(remote_node, remote_pid) |> purge_remote_streams_outside_authority(remote_node) |> touch_replica_peer(remote_node) |> Map.update!(:cluster_control_dirty, &Map.delete(&1, remote_node)) @@ -496,13 +589,20 @@ defmodule Group.Replica do {:noreply, state} - replica_authority_current?(state, remote_node, generation, epoch_revision) -> + replica_exact_authority_current?(state, remote_node, generation, epoch_revision) -> # The shared authority can arrive before this sibling is registered, # so shard-zero fanout is intentionally lossy at startup. Rebuild # this lane directly from the exact shared authority. - {:noreply, install_current_replica_lane(state, remote_node, generation)} + {:noreply, + state + |> notify_replica_transport_peer_up(remote_node, transport_descriptor) + |> put_remote_shard(remote_node, remote_pid) + |> install_current_replica_lane(remote_node, generation)} true -> + # A lane hello is only a hint until node-wide exact authority exists. + # In particular, a delayed hello after retirement must not recreate + # an unleased route that can live forever and suppress rediscovery. {:noreply, request_replica_authority(state, remote_node)} end else @@ -533,7 +633,7 @@ defmodule Group.Replica do state end - :ok = install_replica_view(state, remote_node, generation) + state = install_replica_view(state, remote_node, generation) state = %{ state @@ -547,6 +647,18 @@ defmodule Group.Replica do |> touch_replica_peer(remote_node) |> send_replica_heads(remote_node) else + # A lane hello can legitimately outrun shard zero's exact authority. + # The hello is not retained as a route, because a delayed hello after + # retirement must not recreate an unleased peer. Once exact authority + # reaches this lane, repeat shard-local discovery immediately instead + # of waiting for the next anti-entropy probe. + send_remote_shard_message( + state, + remote_node, + {:peer_connect, self(), state.shard_index, state.num_shards, + Data.my_clusters(state.name)} + ) + state end @@ -556,69 +668,97 @@ defmodule Group.Replica do end end - def handle_info({:replica_authority_removed_local, remote_node}, state) do - state = flush_pending_replicated_message_barrier(state) - {:noreply, expire_replica_peer(state, remote_node)} - end - def handle_info( - {:replica_cluster_open, remote_pid, generation, revision, epochs}, + {:replica_cluster_open, remote_pid, generation, revision, epochs} = control, state ) do state = flush_pending_replicated_message_barrier(state) - remote_node = node(remote_pid) - - controls = - collect_replica_cluster_controls( - :replica_cluster_open, - remote_pid, - generation, - [{revision, epochs}], - state.replicated_sender_buffer_size - 1 - ) - case accepted_replica_cluster_epochs(state, remote_node, generation, controls) do - {:accept, observed_revision, epochs} -> - stale = - Data.put_remote_cluster_epochs( - state.name, - state.shard_index, - remote_node, - observed_revision, - epochs - ) + if state.shard_index == 0 do + remote_node = node(remote_pid) - shared = - Enum.filter(epochs, fn {cluster, _epoch} -> - node() in Data.cluster_nodes(state.name, cluster) - end) + controls = + collect_replica_cluster_controls( + :replica_cluster_open, + remote_pid, + generation, + [{revision, epochs}], + state.replicated_sender_buffer_size - 1 + ) - Data.add_cluster_node(state.name, Enum.map(shared, &elem(&1, 0)), remote_node) + case accepted_replica_cluster_epochs(state, remote_node, generation, controls) do + {:accept, expected_revision, observed_revision, epochs} -> + case Data.put_remote_cluster_epochs( + state.name, + state.shard_index, + remote_node, + generation, + expected_revision, + observed_revision, + epochs + ) do + {:ok, stale} -> + shared = + Enum.filter(epochs, fn {cluster, _epoch} -> + node() in Data.cluster_nodes(state.name, cluster) + end) + + Data.add_cluster_node(state.name, Enum.map(shared, &elem(&1, 0)), remote_node) + + fan_out_to_siblings( + state, + {:replica_cluster_open_control_local, remote_node, generation, observed_revision, + epochs, stale, Enum.map(shared, &elem(&1, 0))} + ) - fan_out_to_siblings( - state, - {:replica_cluster_open_control_local, remote_node, generation, observed_revision, - epochs, stale, Enum.map(shared, &elem(&1, 0))} - ) + state = + state + |> mark_authority_dirty(remote_node) + |> purge_closed_remote_epochs(remote_node, stale) + |> purge_superseded_remote_streams(remote_node, epochs) + |> purge_remote_streams_outside_authority(remote_node) + + state = install_replica_view(state, remote_node, generation) + state = send_replica_heads(state, remote_node, Enum.map(shared, &elem(&1, 0))) + {:noreply, take_one_local_request_turn(state)} + + :stale -> + {:noreply, + state + |> mark_authority_dirty(remote_node) + |> request_replica_authority(remote_node) + |> take_one_local_request_turn()} + end - state = - state - |> mark_authority_dirty(remote_node) - |> purge_closed_remote_epochs(remote_node, stale) - |> purge_superseded_remote_streams(remote_node, epochs) + :stale -> + {:noreply, take_one_local_request_turn(state)} - :ok = install_replica_view(state, remote_node, generation) - state = send_replica_heads(state, remote_node, Enum.map(shared, &elem(&1, 0))) - {:noreply, take_one_local_request_turn(state)} + :refresh -> + {:noreply, + state + |> request_replica_authority(remote_node) + |> take_one_local_request_turn()} - :stale -> - {:noreply, take_one_local_request_turn(state)} + {:gap, observed_revision} -> + :ok = + Data.observe_remote_cluster_epoch_revision( + state.name, + remote_node, + observed_revision + ) - :refresh -> - {:noreply, - state - |> request_replica_authority(remote_node) - |> take_one_local_request_turn()} + {:noreply, + state + |> mark_authority_dirty(remote_node) + |> request_replica_authority(remote_node) + |> take_one_local_request_turn()} + end + else + # Incremental authority is node-wide and therefore has one local owner. + # Forward a control that arrived on another lane instead of racing its + # check/update against shard 0. + _ = send_local_control_message(state, control) + {:noreply, take_one_local_request_turn(state)} end end @@ -627,7 +767,9 @@ defmodule Group.Replica do end def handle_info({:replica_authority_dirty_local, remote_node}, state) do - send(shard_name(state.name, 0), {:replica_authority_dirty_local, remote_node}) + _ = + send_local_control_message(state, {:replica_authority_dirty_local, remote_node}) + {:noreply, state} end @@ -644,9 +786,11 @@ defmodule Group.Replica do state |> purge_closed_remote_epochs(remote_node, stale) |> purge_superseded_remote_streams(remote_node, epochs) + |> purge_remote_streams_outside_authority(remote_node) - :ok = install_replica_view(state, remote_node, generation) - send_replica_heads(state, remote_node, shared) + state + |> install_replica_view(remote_node, generation) + |> send_replica_heads(remote_node, shared) else state end @@ -654,12 +798,6 @@ defmodule Group.Replica do {:noreply, take_one_local_request_turn(state)} end - def handle_info({:replica_cluster_stale_epochs_local, remote_node, stale}, state) do - state = flush_pending_replicated_message_barrier(state) - state = purge_closed_remote_epochs(state, remote_node, stale) - {:noreply, send_replica_heads(state, remote_node)} - end - def handle_info( {:replica_cluster_close, remote_pid, generation, revision, epochs}, %{shard_index: 0} = state @@ -677,33 +815,43 @@ defmodule Group.Replica do ) case accepted_replica_cluster_epochs(state, remote_node, generation, controls) do - {:accept, observed_revision, epochs} -> - closed = - Data.close_remote_cluster_epochs( - state.name, - 0, - remote_node, - observed_revision, - epochs - ) - - if state.shard_index == 0 do - Data.remove_cluster_node(state.name, Enum.map(closed, &elem(&1, 0)), remote_node) - end - - fan_out_to_siblings( - state, - {:replica_cluster_close_control_local, remote_node, generation, observed_revision, - closed} - ) + {:accept, expected_revision, observed_revision, epochs} -> + case Data.close_remote_cluster_epochs( + state.name, + 0, + remote_node, + generation, + expected_revision, + observed_revision, + epochs + ) do + {:ok, closed} -> + if state.shard_index == 0 do + Data.remove_cluster_node(state.name, Enum.map(closed, &elem(&1, 0)), remote_node) + end - state = - state - |> mark_cluster_control_dirty(remote_node) - |> purge_closed_remote_epochs(remote_node, closed) + fan_out_to_siblings( + state, + {:replica_cluster_close_control_local, remote_node, generation, observed_revision, + closed} + ) - :ok = install_replica_view(state, remote_node, generation) - {:noreply, take_one_local_request_turn(state)} + state = + state + |> mark_cluster_control_dirty(remote_node) + |> purge_closed_remote_epochs(remote_node, closed) + |> purge_remote_streams_outside_authority(remote_node) + + state = install_replica_view(state, remote_node, generation) + {:noreply, take_one_local_request_turn(state)} + + :stale -> + {:noreply, + state + |> mark_authority_dirty(remote_node) + |> request_replica_authority(remote_node) + |> take_one_local_request_turn()} + end :stale -> {:noreply, take_one_local_request_turn(state)} @@ -713,6 +861,20 @@ defmodule Group.Replica do state |> request_replica_authority(remote_node) |> take_one_local_request_turn()} + + {:gap, observed_revision} -> + :ok = + Data.observe_remote_cluster_epoch_revision( + state.name, + remote_node, + observed_revision + ) + + {:noreply, + state + |> mark_cluster_control_dirty(remote_node) + |> request_replica_authority(remote_node) + |> take_one_local_request_turn()} end end @@ -728,9 +890,12 @@ defmodule Group.Replica do state = if replica_authority_current?(state, remote_node, generation, revision) do - state = purge_closed_remote_epochs(state, remote_node, closed) - :ok = install_replica_view(state, remote_node, generation) - state + state = + state + |> purge_closed_remote_epochs(remote_node, closed) + |> purge_remote_streams_outside_authority(remote_node) + + install_replica_view(state, remote_node, generation) else state end @@ -738,31 +903,34 @@ defmodule Group.Replica do {:noreply, take_one_local_request_turn(state)} end - def handle_info({:replica_cluster_close_local, remote_node, closed}, state) do - state = flush_pending_replicated_message_barrier(state) - - :ok = - Data.forget_remote_cluster_epochs(state.name, state.shard_index, remote_node, closed) - - {:noreply, purge_closed_remote_epochs(state, remote_node, closed)} - end - def handle_info( - {:replica_heartbeat, remote_pid, version, generation, epoch_revision}, + {:replica_heartbeat, remote_pid, version, generation, epoch_revision, transport_id, + transport_descriptor}, state ) do remote_node = node(remote_pid) + compatible? = + version == WireProtocol.version() and transport_id == state.replica_transport.id() + + state = + if compatible? do + observe_replica_authority_hint(state, remote_node, generation, epoch_revision) + else + state + end + state = cond do - version == WireProtocol.version() and + compatible? and replica_authority_current?(state, remote_node, generation, epoch_revision) and replica_view_current?(state, remote_node) -> state + |> notify_replica_transport_peer_up(remote_node, transport_descriptor) |> put_remote_shard(remote_node, remote_pid) |> touch_replica_peer(remote_node) - version == WireProtocol.version() and + compatible? and replica_authority_current?(state, remote_node, generation, epoch_revision) -> state @@ -777,7 +945,7 @@ defmodule Group.Replica do if state.shard_index == 0 do {:noreply, send_replica_hello(state, node(remote_pid))} else - send(shard_name(state.name, 0), {:replica_hello_request, remote_pid}) + _ = send_local_control_message(state, {:replica_hello_request, remote_pid}) {:noreply, state} end end @@ -796,10 +964,50 @@ defmodule Group.Replica do def handle_info({:group_replica_batch, remote_node, messages}, state) when is_atom(remote_node) and is_list(messages) do - state = Enum.reduce(messages, state, &handle_replica_message(&2, remote_node, &1)) + {turn, remaining} = Enum.split(messages, @incoming_batch_quota) + state = Enum.reduce(turn, state, &handle_replica_message(&2, remote_node, &1)) + + if remaining != [] do + send(self(), {:group_replica_batch, remote_node, remaining}) + end + {:noreply, take_priority_turn(state)} end + def handle_info( + {:replica_snapshot_send_complete, token, worker, snapshot_key, result}, + %{snapshot_send: {worker, token, snapshot_key}} = state + ) do + offsets = + case result do + :complete -> + Map.delete(state.snapshot_send_offsets, snapshot_key) + + {:resume, chunk_index} -> + if current_snapshot_send?(state, snapshot_key) do + Map.put(state.snapshot_send_offsets, snapshot_key, chunk_index) + else + Map.delete(state.snapshot_send_offsets, snapshot_key) + end + + :retry -> + if current_snapshot_send?(state, snapshot_key) do + state.snapshot_send_offsets + else + Map.delete(state.snapshot_send_offsets, snapshot_key) + end + end + + {:noreply, %{state | snapshot_send: nil, snapshot_send_offsets: offsets}} + end + + def handle_info( + {:replica_snapshot_send_complete, _token, _worker, _snapshot_key, _result}, + state + ) do + {:noreply, state} + end + def handle_info({@anti_entropy_timer, ref}, state) do state = if state.anti_entropy_ref == ref do @@ -827,6 +1035,10 @@ defmodule Group.Replica do {:noreply, process_local_request_turn(state, [{{:alias, alias_ref}, request}])} end + def handle_info({@local_request_tag, :noreply, request}, state) do + {:noreply, process_local_request_turn(state, [{:noreply, request}])} + end + # ===================================================================== # Peer discovery protocol # ===================================================================== @@ -913,337 +1125,142 @@ defmodule Group.Replica do end # ===================================================================== - # Cluster state (unified handler for peer discovery + cluster join) + # Node up/down # ===================================================================== - def handle_info({:cluster_state, cluster, reg_data, pg_data}, state) do + def handle_info({:nodeup, remote_node}, state) do state = flush_pending_replicated_message_barrier(state) - %{name: name} = state - - # Guard: skip merge for named clusters we're not a member of - if cluster_member?(name, cluster) do - log_once(state, fn -> - "#{log_prefix(state)} cluster_state cluster=#{inspect(cluster)} (#{length(reg_data)} reg, #{length(pg_data)} pg entries)" - end) + %{shard_index: shard, name: name} = state - log_verbose(state, fn -> - "#{log_prefix_shard(state)} merging cluster=#{inspect(cluster)} (#{length(reg_data)} reg, #{length(pg_data)} pg entries)" - end) + send_remote_shard_message( + state, + remote_node, + {:peer_connect, self(), shard, state.num_shards, Data.my_clusters(name)} + ) - {state, events} = merge_remote_cluster_data(state, cluster, reg_data, pg_data) - notify_monitors(name, events) - {:noreply, state} - else - {:noreply, state} - end + {:noreply, state} end - # ===================================================================== - # Cluster connect/disconnect from remote - # ===================================================================== - - def handle_info({:cluster_connect, clusters, remote_pid}, state) do + def handle_info({:nodedown, dead_node}, state) do state = flush_pending_replicated_message_barrier(state) - %{name: name} = state - remote_node = node(remote_pid) + state = discard_snapshot_transfers_for_source(state, dead_node) + state = discard_snapshot_send_offsets_for_target(state, dead_node) + state = discard_pending_registry_reprojections(state, dead_node) + %{name: name, shard_index: shard} = state - shared = - Enum.filter(clusters, fn c -> - node() in Data.cluster_nodes(name, c) - end) + # Cursor absence is the durable restart marker for an incomplete or + # retiring remote stream. Clear it before touching materialized rows so a + # shard crash at any later purge step finishes that retirement on restart. + Data.delete_replica_cursors_for_origin(name, shard, dead_node) + + # Remove cluster memberships from shared tables. Every shard calls this + # unconditionally (not just shard 0) to handle the race where a non-zero + # shard processes a late peer_connect from the dead node (re-adding it to + # cluster_nodes) AFTER shard 0's nodedown already cleaned it. Since :bag + # delete_object is idempotent, redundant calls from multiple shards are safe. + Data.purge_cluster_node(name, dead_node) + + # Purge all data from the dead node + {purged_reg, purged_pg} = Data.purge_node(name, shard, dead_node) + affected_claims = Data.purge_registry_claims_for_origin(name, shard, dead_node) log_once(state, fn -> - "#{log_prefix(state)} #{remote_node} cluster_connect #{inspect(shared)} (#{length(shared)}/#{length(clusters)} shared)" + "#{log_prefix(state)} nodedown #{dead_node} (purged #{length(purged_reg)} reg, #{length(purged_pg)} pg entries)" end) - if shared != [] do - Data.add_cluster_node(name, shared, remote_node) + events = build_purged_events(name, purged_reg, purged_pg, :nodedown) - # Membership is a control-plane handshake. Replica state follows on the - # data transport via heads/deltas (or an exact snapshot fallback). - send_to_peer(state, remote_node, {:cluster_connect_ack, shared, self(), []}) - end + {state, events} = + Enum.reduce(affected_claims, {state, events}, fn {cluster, key}, {acc, inner_events} -> + reconcile_registry_projection(acc, cluster, key, :nodedown, inner_events) + end) - {:noreply, state} - end + notify_monitors(name, events) - def handle_info({:cluster_connect_ack, clusters, remote_pid, cluster_data}, state) do - state = flush_pending_replicated_message_barrier(state) - %{name: name} = state - remote_node = node(remote_pid) + state = %{ + state + | remote_shards: Map.delete(state.remote_shards, dead_node), + peer_last_seen: Map.delete(state.peer_last_seen, dead_node), + cluster_control_dirty: Map.delete(state.cluster_control_dirty, dead_node), + authority_dirty_notified: MapSet.delete(state.authority_dirty_notified, dead_node) + } - # Guard: skip if remote node went down (nodedown race) or if we left the - # cluster (connect+disconnect race). Without these, a delayed ack would - # re-add a dead/irrelevant node to cluster_nodes permanently. - {state, events} = - if Map.has_key?(state.remote_shards, remote_node) do - active = Enum.filter(clusters, fn c -> node() in Data.cluster_nodes(name, c) end) - - if active != [] do - Data.add_cluster_node(name, active, remote_node) - - # The empty data list is the v1 contract. Retain merge support for a - # rolling peer that still bundles legacy cluster data. - {new_state, events} = - Enum.reduce(cluster_data, {state, []}, fn {cluster, reg_data, pg_data}, - {acc_state, acc_events} -> - if cluster in active and (reg_data != [] or pg_data != []) do - merge_remote_cluster_data(acc_state, cluster, reg_data, pg_data, acc_events) - else - {acc_state, acc_events} - end - end) + Data.delete_remote_replica_info(name, shard, dead_node) - {send_replica_heads(new_state, remote_node), events} - else - {state, []} - end - else - {state, []} - end + if function_exported?(state.replica_transport, :peer_down, 4) do + :ok = + state.replica_transport.peer_down( + name, + dead_node, + shard, + state.replica_transport_opts + ) + end - notify_monitors(name, events) - {:noreply, state} - end - - def handle_info({:cluster_disconnect, clusters, remote_pid}, state) do - state = flush_pending_replicated_message_barrier(state) - %{name: name, shard_index: shard} = state - remote_node = node(remote_pid) - - log_once(state, fn -> - "#{log_prefix(state)} #{remote_node} cluster_disconnect #{inspect(clusters)}" - end) - - if shard == 0 do - Data.remove_cluster_node(name, clusters, remote_node) - fan_out_to_siblings(state, {:cluster_disconnect, clusters, remote_pid}) - end - - {state, events} = - Enum.reduce(clusters, {state, []}, fn cluster, {outer_state, acc} -> - affected_keys = - Data.purge_registry_claims_for_cluster(name, shard, cluster, remote_node) - - {purged_reg, purged_pg} = purge_cluster_entries(name, shard, cluster, remote_node) - - acc = build_purged_events(name, purged_reg, purged_pg, :cluster_disconnect, acc) - - Enum.reduce(affected_keys, {outer_state, acc}, fn key, {inner_state, inner_events} -> - reconcile_registry_projection( - inner_state, - cluster, - key, - :cluster_disconnect, - inner_events - ) - end) - end) - - notify_monitors(name, events) {:noreply, state} end # ===================================================================== - # Node up/down + # Process DOWN # ===================================================================== - def handle_info({:nodeup, remote_node}, state) do - state = flush_pending_replicated_message_barrier(state) - %{shard_index: shard, name: name} = state - - send_remote_shard_message( - state, - remote_node, - {:peer_connect, self(), shard, state.num_shards, Data.my_clusters(name)} - ) - - {:noreply, state} - end - - def handle_info({:nodedown, dead_node}, state) do + def handle_info({:DOWN, _mref, :process, pid, reason}, state) do state = flush_pending_replicated_message_barrier(state) - state = discard_snapshot_transfers_for_source(state, dead_node) %{name: name, shard_index: shard} = state - # Remove cluster memberships from shared tables. Every shard calls this - # unconditionally (not just shard 0) to handle the race where a non-zero - # shard processes a late peer_connect from the dead node (re-adding it to - # cluster_nodes) AFTER shard 0's nodedown already cleaned it. Since :bag - # delete_object is idempotent, redundant calls from multiple shards are safe. - Data.purge_cluster_node(name, dead_node) - - # Purge all data from the dead node - {purged_reg, purged_pg} = Data.purge_node(name, shard, dead_node) - affected_claims = Data.purge_registry_claims_for_origin(name, shard, dead_node) - - log_once(state, fn -> - "#{log_prefix(state)} nodedown #{dead_node} (purged #{length(purged_reg)} reg, #{length(purged_pg)} pg entries)" - end) - - events = build_purged_events(name, purged_reg, purged_pg, :nodedown) - - {state, events} = - Enum.reduce(affected_claims, {state, events}, fn {cluster, key}, {acc, inner_events} -> - reconcile_registry_projection(acc, cluster, key, :nodedown, inner_events) - end) - - notify_monitors(name, events) - - state = %{ - state - | remote_shards: Map.delete(state.remote_shards, dead_node), - peer_last_seen: Map.delete(state.peer_last_seen, dead_node), - authority_dirty_notified: MapSet.delete(state.authority_dirty_notified, dead_node) - } - - Data.delete_replica_cursors_for_origin(name, shard, dead_node) - Data.delete_remote_replica_info(name, shard, dead_node) - - if function_exported?(state.replica_transport, :peer_down, 4) do - :ok = - state.replica_transport.peer_down( - name, - dead_node, - shard, - state.replica_transport_opts + # Replica shards intentionally never create remote process monitors: their + # liveness is generation/lease fenced and remote monitoring can itself + # suspend on a busy distribution connection. Consequently every genuine + # DOWN handled here belongs to a locally owned registry/PG process. + if Map.has_key?(state.monitors, pid) do + {downs, monitors} = + collect_local_process_downs( + [{pid, reason}], + state.monitors, + @process_down_batch_size - 1 ) - end - - state = %{state | peer_transports: Map.delete(state.peer_transports, dead_node)} - {:noreply, state} - end - # ===================================================================== - # Process DOWN - # ===================================================================== + pids = Enum.map(downs, &elem(&1, 0)) + reason_by_pid = Map.new(downs) + {visible_reg, pending_pg} = Data.entries_for_pids(name, shard, pids) - def handle_info({:DOWN, _mref, :process, pid, reason}, state) do - state = flush_pending_replicated_message_barrier(state) - %{name: name, shard_index: shard} = state + claimed_reg = + Data.local_registry_claims_by_pids(name, shard, pids) + |> Enum.map(fn {pid, cluster, key, meta, _generation, _epoch} -> + {pid, cluster, key, meta} + end) - remote_node = node(pid) + pending_reg = Enum.uniq(visible_reg ++ claimed_reg) - if remote_node != node() and Map.get(state.remote_shards, remote_node) == pid do - state = discard_snapshot_transfers_for_source(state, remote_node) + sequenced_downs = + append_process_down_records(state, reason_by_pid, pending_reg, pending_pg) - # Remote shard process died — purge its cluster memberships and node data. - # Unconditional (not gated on shard 0) — same reasoning as nodedown handler. - Data.purge_cluster_node(name, remote_node) - {purged_reg, purged_pg} = Data.purge_node(name, shard, remote_node) - affected_claims = Data.purge_registry_claims_for_origin(name, shard, remote_node) + {purged_reg, purged_pg} = Data.delete_all_for_pids(name, shard, pids) log_verbose(state, fn -> - "#{log_prefix_shard(state)} remote_shard_down #{remote_node} (purged #{length(purged_reg)} reg, #{length(purged_pg)} pg)" + "#{log_prefix_shard(state)} process_down_batch pids=#{length(downs)} (#{length(purged_reg) + length(purged_pg)} entries cleaned)" end) - events = build_purged_events(name, purged_reg, purged_pg, {:nodedown, remote_node}) + state = finish_process_down_records(state, sequenced_downs) - {state, events} = - Enum.reduce(affected_claims, {state, events}, fn {cluster, key}, {acc, inner_events} -> - reconcile_registry_projection( - acc, - cluster, - key, - {:nodedown, remote_node}, - inner_events - ) - end) + affected_registry_keys = + pending_reg + |> Enum.map(fn {_pid, cluster, key, _meta} -> {cluster, key} end) + |> Enum.uniq() - notify_monitors(name, events) - state = %{state | remote_shards: Map.delete(state.remote_shards, remote_node)} - state = %{state | monitors: Map.delete(state.monitors, pid)} + {state, projection_events} = + reconcile_registry_keys(state, affected_registry_keys, reason, []) - if function_exported?(state.replica_transport, :peer_down, 4) do - :ok = - state.replica_transport.peer_down( - name, - remote_node, - shard, - state.replica_transport_opts - ) - end + events = + projection_events ++ + build_process_down_events(name, purged_reg, purged_pg, reason_by_pid) + notify_monitors(name, events) + state = %{state | monitors: Map.drop(monitors, pids)} {:noreply, state} else - if Map.has_key?(state.monitors, pid) do - {downs, monitors} = - collect_local_process_downs( - [{pid, reason}], - state.monitors, - @process_down_batch_size - 1 - ) - - pids = Enum.map(downs, &elem(&1, 0)) - reason_by_pid = Map.new(downs) - {visible_reg, pending_pg} = Data.entries_for_pids(name, shard, pids) - - claimed_reg = - Data.local_registry_claims_by_pids(name, shard, pids) - |> Enum.map(fn {pid, cluster, key, meta, _generation, _epoch} -> - {pid, cluster, key, meta} - end) - - pending_reg = Enum.uniq(visible_reg ++ claimed_reg) - - sequenced_downs = - append_process_down_records(state, reason_by_pid, pending_reg, pending_pg) - - {purged_reg, purged_pg} = Data.delete_all_for_pids(name, shard, pids) - - log_verbose(state, fn -> - "#{log_prefix_shard(state)} process_down_batch pids=#{length(downs)} (#{length(purged_reg) + length(purged_pg)} entries cleaned)" - end) - - state = finish_process_down_records(state, sequenced_downs) - - affected_registry_keys = - pending_reg - |> Enum.map(fn {_pid, cluster, key, _meta} -> {cluster, key} end) - |> Enum.uniq() - - {state, projection_events} = - reconcile_registry_keys(state, affected_registry_keys, reason, []) - - events = - projection_events ++ - build_process_down_events(name, purged_reg, purged_pg, reason_by_pid) - - notify_monitors(name, events) - state = %{state | monitors: Map.drop(monitors, pids)} - {:noreply, state} - else - {:noreply, state} - end - end - end - - def handle_info({:replicate_process_down_batch, reg_entries, pg_entries}, state) do - state = flush_pending_replicated_message_barrier(state) - %{name: name, shard_index: shard} = state - - log_verbose(state, fn -> - "#{log_prefix_shard(state)} replicate_process_down_batch (#{length(reg_entries)} reg, #{length(pg_entries)} pg)" - end) - - deleted_reg = Data.registry_delete_matching_many(name, shard, reg_entries) - deleted_pg = Data.pg_delete_matching_many(name, shard, pg_entries) - events = build_process_down_batch_events(name, deleted_reg, deleted_pg) - notify_monitors(name, events) - {:noreply, state} - end - - def handle_info({:send_cluster_data, clusters, target_node}, state) do - state = flush_pending_replicated_message_barrier(state) - %{name: name} = state - - active = Enum.filter(clusters, fn c -> node() in Data.cluster_nodes(name, c) end) - - if active != [] do - send_cluster_states(state, active, target_node) + {:noreply, state} end - - {:noreply, state} end def handle_info({:group_dispatch, pids, message}, state) do @@ -1421,6 +1438,8 @@ defmodule Group.Replica do :ok end + defp reply_local_request(:noreply, _reply), do: :ok + defp process_local_request_turn( state, initial_messages @@ -1442,6 +1461,9 @@ defmodule Group.Replica do {@local_request_tag, alias_ref, request} when is_reference(alias_ref) -> collect_local_request_messages([{{:alias, alias_ref}, request} | acc], remaining - 1) + + {@local_request_tag, :noreply, request} -> + collect_local_request_messages([{:noreply, request} | acc], remaining - 1) after 0 -> Enum.reverse(acc) @@ -1506,30 +1528,70 @@ defmodule Group.Replica do end defp process_local_request_without_barrier(state, request) do + with :ok <- validate_local_mutation_epoch(state, request) do + case request do + {:register, cluster, _epoch, key, pid, meta} -> + do_register(state, cluster, key, pid, meta) + + {:unregister, cluster, _epoch, key} -> + do_unregister(state, cluster, key) + + {:join, cluster, _epoch, key, pid, meta} -> + do_join(state, cluster, key, pid, meta) + + {:leave, cluster, _epoch, key, pid} -> + do_leave(state, cluster, key, pid) + + {:cluster_connect, clusters} -> + do_cluster_connect(state, clusters) + + {:cluster_connect, clusters, epochs} -> + do_cluster_connect(state, clusters, epochs) + + {:cluster_disconnect, clusters} -> + do_cluster_disconnect(state, clusters) + + {:cluster_disconnect, clusters, epochs} -> + do_cluster_disconnect(state, clusters, epochs) + + _ -> + {{:error, :invalid_local_request}, state} + end + else + {:error, reason} -> {{:error, reason}, state} + end + end + + defp validate_local_mutation_epoch(state, request) do case request do - {:register, cluster, key, pid, meta} -> - do_register(state, cluster, key, pid, meta) + {op, cluster, epoch, _key, _pid, _meta} when op in [:register, :join] -> + validate_local_mutation_epoch(state, cluster, epoch) - {:unregister, cluster, key} -> - do_unregister(state, cluster, key) + {op, cluster, epoch, _key} when op == :unregister -> + validate_local_mutation_epoch(state, cluster, epoch) - {:join, cluster, key, pid, meta} -> - do_join(state, cluster, key, pid, meta) + {op, cluster, epoch, _key, _pid} when op == :leave -> + validate_local_mutation_epoch(state, cluster, epoch) - {:leave, cluster, key, pid} -> - do_leave(state, cluster, key, pid) + {op, _cluster, _key, _pid, _meta} when op in [:register, :join] -> + {:error, :stale_cluster_epoch} - {:cluster_connect, clusters} -> - do_cluster_connect(state, clusters) + {op, _cluster, _key} when op == :unregister -> + {:error, :stale_cluster_epoch} - {:cluster_connect, clusters, epochs} -> - do_cluster_connect(state, clusters, epochs) + {op, _cluster, _key, _pid} when op == :leave -> + {:error, :stale_cluster_epoch} - {:cluster_disconnect, clusters} -> - do_cluster_disconnect(state, clusters) + _ -> + :ok + end + end - {:cluster_disconnect, clusters, epochs} -> - do_cluster_disconnect(state, clusters, epochs) + defp validate_local_mutation_epoch(state, cluster, epoch) do + if Data.local_cluster_epoch(state.name, cluster) == epoch do + :ok + else + {:error, :stale_cluster_epoch} end end @@ -1592,82 +1654,90 @@ defmodule Group.Replica do {%{}, [], [], [], %{}, MapSet.new()}, fn {reply_to, request}, {entries, replies, events, broadcasts, new_monitors, maybe_demonitor_pids} -> - case request do - {:join, cluster, key, pid, meta} -> - member = {cluster, key, pid} - {initial, current} = local_pg_batch_entry(entries, name, shard, member) - - case current do - nil -> - time = System.system_time() - new_monitors = ensure_local_batch_monitor(state, new_monitors, pid) - - { - Map.put(entries, member, {initial, {meta, time, local_node}}), - [{reply_to, :ok} | replies], - [ - build_event(name, :joined, key, pid, meta, %{ - previous_meta: nil, - cluster: cluster - }) - | events - ], - [ - {:join, cluster, key, pid, meta, time, :join, node(pid)} | broadcasts - ], - new_monitors, - maybe_demonitor_pids - } - - {old_meta, _time, _node} when old_meta == meta -> - {entries, [{reply_to, :ok} | replies], events, broadcasts, new_monitors, - maybe_demonitor_pids} - - {old_meta, _time, _node} -> - time = System.system_time() - - { - Map.put(entries, member, {initial, {meta, time, local_node}}), - [{reply_to, :ok} | replies], - [ - build_event(name, :joined, key, pid, meta, %{ - previous_meta: old_meta, - cluster: cluster - }) - | events - ], - [ - {:join, cluster, key, pid, meta, time, :update, node(pid)} | broadcasts - ], - new_monitors, - maybe_demonitor_pids - } - end + if validate_local_mutation_epoch(state, request) != :ok do + {entries, [{reply_to, {:error, :stale_cluster_epoch}} | replies], events, broadcasts, + new_monitors, maybe_demonitor_pids} + else + case request do + {:join, cluster, _epoch, key, pid, meta} -> + member = {cluster, key, pid} + {initial, current} = local_pg_batch_entry(entries, name, shard, member) + + case current do + nil -> + time = System.system_time() + new_monitors = ensure_local_batch_monitor(state, new_monitors, pid) + + { + Map.put(entries, member, {initial, {meta, time, local_node}}), + [{reply_to, :ok} | replies], + [ + build_event(name, :joined, key, pid, meta, %{ + previous_meta: nil, + cluster: cluster + }) + | events + ], + [ + {:join, cluster, key, pid, meta, time, :join, node(pid)} | broadcasts + ], + new_monitors, + maybe_demonitor_pids + } + + {old_meta, _time, _node} when old_meta == meta -> + {entries, [{reply_to, :ok} | replies], events, broadcasts, new_monitors, + maybe_demonitor_pids} + + {old_meta, _time, _node} -> + time = System.system_time() + + { + Map.put(entries, member, {initial, {meta, time, local_node}}), + [{reply_to, :ok} | replies], + [ + build_event(name, :joined, key, pid, meta, %{ + previous_meta: old_meta, + cluster: cluster + }) + | events + ], + [ + {:join, cluster, key, pid, meta, time, :update, node(pid)} | broadcasts + ], + new_monitors, + maybe_demonitor_pids + } + end - {:leave, cluster, key, pid} -> - member = {cluster, key, pid} - {initial, current} = local_pg_batch_entry(entries, name, shard, member) - - case current do - nil -> - {entries, [{reply_to, {:error, :not_in_group}} | replies], events, broadcasts, - new_monitors, maybe_demonitor_pids} - - {meta, _time, _node} -> - { - Map.put(entries, member, {initial, nil}), - [{reply_to, :ok} | replies], - [ - build_event(name, :left, key, pid, meta, %{reason: :leave, cluster: cluster}) - | events - ], - [ - {:leave, cluster, key, pid, meta, :leave} | broadcasts - ], - new_monitors, - MapSet.put(maybe_demonitor_pids, pid) - } - end + {:leave, cluster, _epoch, key, pid} -> + member = {cluster, key, pid} + {initial, current} = local_pg_batch_entry(entries, name, shard, member) + + case current do + nil -> + {entries, [{reply_to, {:error, :not_in_group}} | replies], events, broadcasts, + new_monitors, maybe_demonitor_pids} + + {meta, _time, _node} -> + { + Map.put(entries, member, {initial, nil}), + [{reply_to, :ok} | replies], + [ + build_event(name, :left, key, pid, meta, %{ + reason: :leave, + cluster: cluster + }) + | events + ], + [ + {:leave, cluster, key, pid, meta, :leave} | broadcasts + ], + new_monitors, + MapSet.put(maybe_demonitor_pids, pid) + } + end + end end end ) @@ -1875,8 +1945,8 @@ defmodule Group.Replica do end) end - defp local_request_domain({:join, _cluster, _key, _pid, _meta}), do: :pg - defp local_request_domain({:leave, _cluster, _key, _pid}), do: :pg + defp local_request_domain({:join, _cluster, _epoch, _key, _pid, _meta}), do: :pg + defp local_request_domain({:leave, _cluster, _epoch, _key, _pid}), do: :pg defp local_request_domain(_request), do: :other defp do_register(state, cluster, key, pid, meta) do @@ -2061,13 +2131,6 @@ defmodule Group.Replica do peers = Data.cluster_nodes(name, nil) -- [node()] for target_node <- peers do - shared = - Enum.filter(clusters, fn cluster -> - not is_nil(Data.remote_cluster_epoch(name, target_node, cluster)) - end) - - Data.add_cluster_node(name, shared, target_node) - send_remote_shard_message( state, target_node, @@ -2087,9 +2150,55 @@ defmodule Group.Replica do Enum.map(clusters, &{&1, Data.closed_local_cluster_epoch(state.name, &1)}) ) - defp do_cluster_disconnect(state, clusters, epochs) do + defp do_cluster_disconnect(state, _clusters, epochs) do + epochs = + Enum.filter(epochs, fn + {cluster, epoch} when not is_nil(epoch) -> + Data.closed_local_cluster_pending?( + state.name, + cluster, + epoch, + state.shard_index + ) + + _ -> + false + end) + + case epochs do + [] -> {:ok, state} + epochs -> do_cluster_disconnect_epochs(state, epochs) + end + end + + defp do_cluster_disconnect_epochs(state, epochs) do state = flush_pending_replicated_sender_barrier(state) %{name: name, shard_index: shard} = state + clusters = Enum.map(epochs, &elem(&1, 0)) + + closed_streams = + Enum.flat_map(epochs, fn + {cluster, epoch} when not is_nil(epoch) -> + [ + WireProtocol.stream_id( + name, + node(), + Data.generation(name), + shard, + cluster, + epoch + ) + ] + + _ -> + [] + end) + + state = discard_snapshot_send_offsets_for_streams(state, closed_streams) + + # See the nodedown ordering note: restart repair rejects remote rows whose + # receive cursor was cleared before this cluster-wide purge. + :ok = Data.delete_replica_cursors_for_clusters(name, shard, clusters) log_once(state, fn -> "#{log_prefix(state)} cluster_disconnect #{inspect(clusters)}" @@ -2122,8 +2231,6 @@ defmodule Group.Replica do # Forget their receive cursors as well: if this node later reconnects while # a remote origin kept the same epoch, its advertised head must rebuild the # rows instead of being mistaken for data we still retain. - :ok = Data.delete_replica_cursors_for_clusters(name, shard, clusters) - if shard == 0 do broadcast_to_peers( state, @@ -2140,8 +2247,7 @@ defmodule Group.Replica do :ok end) - completed_clusters = Data.mark_closed_cluster_shard(name, clusters, shard) - if completed_clusters != [], do: Data.remove_clusters(name, completed_clusters) + _completed_clusters = Data.mark_closed_cluster_shard(name, epochs, shard) notify_monitors(name, events) {:ok, state} end @@ -2257,102 +2363,80 @@ defmodule Group.Replica do |> take_one_local_request_turn() end - defp take_priority_control_turn(state) do + defp take_priority_control_turn(state), + do: take_priority_control_turn(state, @priority_control_quota) + + defp take_priority_control_turn(state, 0), do: state + + defp take_priority_control_turn(state, remaining) do receive do {:peer_connect, _remote_pid, _remote_shard_index, _remote_num_shards, _remote_clusters} = msg -> state = process_inline_priority_message(state, msg) - take_priority_control_turn(state) + take_priority_control_turn(state, remaining - 1) {:peer_connect_ack, _remote_pid, _remote_shard_index, _remote_num_shards, _remote_clusters} = msg -> state = process_inline_priority_message(state, msg) - take_priority_control_turn(state) - - {:cluster_connect, _clusters, _remote_pid} = msg -> - state = process_inline_priority_message(state, msg) - take_priority_control_turn(state) - - {:cluster_connect_ack, _clusters, _remote_pid, _cluster_data} = msg -> - state = process_inline_priority_message(state, msg) - take_priority_control_turn(state) - - {:cluster_disconnect, _clusters, _remote_pid} = msg -> - state = process_inline_priority_message(state, msg) - take_priority_control_turn(state) - - {:cluster_state, _cluster, _reg_data, _pg_data} = msg -> - state = process_inline_priority_message(state, msg) - take_priority_control_turn(state) + take_priority_control_turn(state, remaining - 1) {:replica_hello, _remote_pid, _version, _generation, _epoch_revision, _cluster_epochs, _transport_id, _descriptor} = msg -> state = process_inline_priority_message(state, msg) - take_priority_control_turn(state) + take_priority_control_turn(state, remaining - 1) {:replica_lane_hello, _remote_pid, _version, _generation, _epoch_revision, _transport_id, _descriptor} = msg -> state = process_inline_priority_message(state, msg) - take_priority_control_turn(state) + take_priority_control_turn(state, remaining - 1) {:replica_authority_installed_local, _remote_node, _generation, _epoch_revision, _old_generation, _stale_epochs} = msg -> state = process_inline_priority_message(state, msg) - take_priority_control_turn(state) - - {:replica_authority_removed_local, _remote_node} = msg -> - state = process_inline_priority_message(state, msg) - take_priority_control_turn(state) + take_priority_control_turn(state, remaining - 1) {:replica_authority_dirty_local, _remote_node} = msg -> state = process_inline_priority_message(state, msg) - take_priority_control_turn(state) + take_priority_control_turn(state, remaining - 1) {:replica_cluster_open_control_local, _remote_node, _generation, _revision, _epochs, _stale, _shared} = msg -> state = process_inline_priority_message(state, msg) - take_priority_control_turn(state) + take_priority_control_turn(state, remaining - 1) {:replica_cluster_close_control_local, _remote_node, _generation, _revision, _closed} = msg -> state = process_inline_priority_message(state, msg) - take_priority_control_turn(state) + take_priority_control_turn(state, remaining - 1) - {:replica_heartbeat, _remote_pid, _version, _generation, _epoch_revision} = msg -> + {:replica_heartbeat, _remote_pid, _version, _generation, _epoch_revision, _transport_id, + _transport_descriptor} = msg -> state = process_inline_priority_message(state, msg) - take_priority_control_turn(state) + take_priority_control_turn(state, remaining - 1) {:replica_hello_request, _remote_pid} = msg -> state = process_inline_priority_message(state, msg) - take_priority_control_turn(state) + take_priority_control_turn(state, remaining - 1) {:replica_cluster_open, _remote_pid, _generation, _revision, _epochs} = msg -> state = process_inline_priority_message(state, msg) - take_priority_control_turn(state) + take_priority_control_turn(state, remaining - 1) {:replica_cluster_close, _remote_pid, _generation, _revision, _epochs} = msg -> state = process_inline_priority_message(state, msg) - take_priority_control_turn(state) - - {:send_cluster_data, _clusters, _target_node} = msg -> - state = process_inline_priority_message(state, msg) - take_priority_control_turn(state) - - {:replicate_process_down_batch, _reg_entries, _pg_entries} = msg -> - state = process_inline_priority_message(state, msg) - take_priority_control_turn(state) + take_priority_control_turn(state, remaining - 1) {:DOWN, _mref, :process, _pid, _reason} = msg -> state = process_inline_priority_message(state, msg) - take_priority_control_turn(state) + take_priority_control_turn(state, remaining - 1) {:nodeup, _remote_node} = msg -> state = process_inline_priority_message(state, msg) - take_priority_control_turn(state) + take_priority_control_turn(state, remaining - 1) {:nodedown, _remote_node} = msg -> state = process_inline_priority_message(state, msg) - take_priority_control_turn(state) + take_priority_control_turn(state, remaining - 1) after 0 -> state @@ -2367,6 +2451,9 @@ defmodule Group.Replica do {@local_request_tag, alias_ref, request} when is_reference(alias_ref) -> process_local_request_turn(state, [{{:alias, alias_ref}, request}]) + + {@local_request_tag, :noreply, request} -> + process_local_request_turn(state, [{:noreply, request}]) after 0 -> state @@ -2887,37 +2974,48 @@ defmodule Group.Replica do generation, epoch_revision, cluster_epochs, - transport_id, + _transport_id, transport_descriptor ) do remote_node = node(remote_pid) - shared = - cluster_epochs - |> Enum.map(&elem(&1, 0)) - |> compute_shared_clusters(Data.my_clusters(state.name)) - - previous_shared = Data.clusters_for_node(state.name, remote_node) -- [nil] - shared_set = MapSet.new(shared) - departed = Enum.reject(previous_shared, &MapSet.member?(shared_set, &1)) - Data.remove_cluster_node(state.name, departed, remote_node) - - Data.add_cluster_node( - state.name, - [nil | Enum.reject(shared, &is_nil/1)], - remote_node - ) + case Data.put_remote_replica_info( + state.name, + 0, + remote_node, + generation, + epoch_revision, + cluster_epochs + ) do + :stale -> + state + |> mark_cluster_control_dirty(remote_node) + |> request_replica_authority(remote_node) - {old_generation, stale_epochs} = - Data.put_remote_replica_info( - state.name, - 0, - remote_node, - generation, - epoch_revision, - cluster_epochs - ) + {old_generation, stale_epochs} -> + finish_replica_authority_install( + state, + remote_pid, + remote_node, + generation, + epoch_revision, + transport_descriptor, + old_generation, + stale_epochs + ) + end + end + defp finish_replica_authority_install( + state, + remote_pid, + remote_node, + generation, + epoch_revision, + transport_descriptor, + old_generation, + stale_epochs + ) do state = maybe_purge_remote_generation(state, remote_node, old_generation, generation) state = notify_replica_transport_peer_up(state, remote_node, transport_descriptor) @@ -2930,24 +3028,29 @@ defmodule Group.Replica do state end - :ok = install_replica_view(state, remote_node, generation) - - state = %{ - state - | remote_shards: Map.put(state.remote_shards, remote_node, remote_pid), - peer_last_seen: Map.put(state.peer_last_seen, remote_node, monotonic_millis()), - cluster_control_dirty: Map.delete(state.cluster_control_dirty, remote_node), - peer_transports: - Map.put(state.peer_transports, remote_node, {transport_id, transport_descriptor}) - } + state = install_replica_view(state, remote_node, generation) - fan_out_to_siblings( - state, - {:replica_authority_installed_local, remote_node, generation, epoch_revision, - old_generation, stale_epochs} - ) + if replica_authority_current?(state, remote_node, generation, epoch_revision) and + replica_view_current?(state, remote_node) do + state = %{ + state + | remote_shards: Map.put(state.remote_shards, remote_node, remote_pid), + peer_last_seen: Map.put(state.peer_last_seen, remote_node, monotonic_millis()), + cluster_control_dirty: Map.delete(state.cluster_control_dirty, remote_node) + } + + fan_out_to_siblings( + state, + {:replica_authority_installed_local, remote_node, generation, epoch_revision, + old_generation, stale_epochs} + ) - send_replica_heads(state, remote_node) + send_replica_heads(state, remote_node) + else + state + |> mark_cluster_control_dirty(remote_node) + |> request_replica_authority(remote_node) + end end defp notify_replica_transport_peer_up(state, remote_node, transport_descriptor) do @@ -2997,27 +3100,69 @@ defmodule Group.Replica do defp replica_authority_current?(state, remote_node, generation, epoch_revision) do Data.remote_generation(state.name, remote_node) == generation and - Data.remote_cluster_epoch_observed_revision(state.name, remote_node) == epoch_revision + Data.remote_cluster_epoch_observed_revision(state.name, remote_node) == epoch_revision and + Data.remote_replica_authority_hint(state.name, remote_node) == + {generation, epoch_revision} + end + + defp replica_exact_authority_current?(state, remote_node, generation, epoch_revision) do + replica_authority_current?(state, remote_node, generation, epoch_revision) and + Data.remote_cluster_epoch_exact_revision(state.name, remote_node) == epoch_revision end defp replica_view_current?(state, remote_node) do - Data.remote_view_generation(state.name, state.shard_index, remote_node) == - Data.remote_generation(state.name, remote_node) and + known_generation = Data.remote_generation(state.name, remote_node) + observed_revision = Data.remote_cluster_epoch_observed_revision(state.name, remote_node) + + Data.remote_replica_authority_hint(state.name, remote_node) == + {known_generation, observed_revision} and + Data.remote_cluster_epoch_revision(state.name, remote_node) == observed_revision and + Data.remote_view_generation(state.name, state.shard_index, remote_node) == + known_generation and Data.remote_view_cluster_epoch_revision(state.name, state.shard_index, remote_node) == Data.remote_cluster_epoch_exact_revision(state.name, remote_node) and Data.remote_view_observed_revision(state.name, state.shard_index, remote_node) == - Data.remote_cluster_epoch_observed_revision(state.name, remote_node) + observed_revision + end + + defp observe_replica_authority_hint(state, remote_node, generation, epoch_revision) do + if WireProtocol.valid_generation?(generation) and is_integer(epoch_revision) and + epoch_revision >= 0 and + Data.observe_remote_replica_hint( + state.name, + remote_node, + generation, + epoch_revision + ) do + # A heartbeat or lane hello can outrun a dropped authority control. Data + # has already fenced every affected lane; retain an exact-hello retry + # obligation independently of this one best-effort request. + state + |> ensure_replica_peer_retirement_deadline(remote_node) + |> mark_authority_dirty(remote_node) + |> request_replica_authority(remote_node) + else + state + end end defp install_replica_view(state, remote_node, generation) do - Data.put_remote_view_info( - state.name, - state.shard_index, - remote_node, - generation, - Data.remote_cluster_epoch_exact_revision(state.name, remote_node), - Data.remote_cluster_epoch_observed_revision(state.name, remote_node) - ) + case Data.put_remote_view_info( + state.name, + state.shard_index, + remote_node, + generation, + Data.remote_cluster_epoch_exact_revision(state.name, remote_node), + Data.remote_cluster_epoch_observed_revision(state.name, remote_node) + ) do + :ok -> + reproject_pending_registry_keys(state, remote_node) + + :stale -> + state + |> mark_authority_dirty(remote_node) + |> request_replica_authority(remote_node) + end end defp install_current_replica_lane(state, remote_node, generation) do @@ -3026,12 +3171,16 @@ defmodule Group.Replica do state = maybe_purge_remote_generation(state, remote_node, old_generation, generation) state = purge_remote_streams_outside_authority(state, remote_node) - :ok = install_replica_view(state, remote_node, generation) + state = install_replica_view(state, remote_node, generation) - state - |> touch_replica_peer(remote_node) - |> Map.update!(:cluster_control_dirty, &Map.delete(&1, remote_node)) - |> send_replica_heads(remote_node) + if replica_view_current?(state, remote_node) do + state + |> touch_replica_peer(remote_node) + |> Map.update!(:cluster_control_dirty, &Map.delete(&1, remote_node)) + |> send_replica_heads(remote_node) + else + state + end end defp schedule_anti_entropy(state) do @@ -3056,6 +3205,13 @@ defmodule Group.Replica do %{state | peer_last_seen: Map.put(state.peer_last_seen, remote_node, monotonic_millis())} end + defp ensure_replica_peer_retirement_deadline(state, remote_node) do + %{ + state + | peer_last_seen: Map.put_new(state.peer_last_seen, remote_node, monotonic_millis()) + } + end + defp collect_replica_cluster_controls(_tag, _remote_pid, _generation, acc, 0), do: Enum.reverse(acc) @@ -3075,42 +3231,55 @@ defmodule Group.Replica do end defp accepted_replica_cluster_epochs(state, remote_node, generation, controls) do - case Data.remote_view_generation(state.name, state.shard_index, remote_node) do - ^generation -> - authoritative_revision = - Data.remote_view_cluster_epoch_revision( - state.name, - state.shard_index, - remote_node - ) + case { + Data.remote_replica_authority_hint(state.name, remote_node), + Data.remote_view_generation(state.name, state.shard_index, remote_node) + } do + {{^generation, _hinted_revision}, ^generation} -> + expected_revision = + Data.remote_cluster_epoch_observed_revision(state.name, remote_node) accepted = Enum.filter(controls, fn {revision, _epochs} -> - is_nil(authoritative_revision) or revision > authoritative_revision + is_nil(expected_revision) or revision > expected_revision end) + |> Enum.sort_by(&elem(&1, 0)) + |> Enum.uniq_by(&elem(&1, 0)) case accepted do [] -> :stale accepted -> - accepted = Enum.sort_by(accepted, &elem(&1, 0)) - observed_revision = accepted |> List.last() |> elem(0) + next_revision = (expected_revision || -1) + 1 + + if contiguous_cluster_controls?(accepted, next_revision) do + observed_revision = accepted |> List.last() |> elem(0) - epochs = - accepted - |> Enum.flat_map(&elem(&1, 1)) - |> Map.new() - |> Map.to_list() + epochs = + accepted + |> Enum.flat_map(&elem(&1, 1)) + |> Map.new() + |> Map.to_list() - {:accept, observed_revision, epochs} + {:accept, expected_revision, observed_revision, epochs} + else + {:gap, accepted |> List.last() |> elem(0)} + end end - _other_generation -> + _other_authority -> :refresh end end + defp contiguous_cluster_controls?(controls, expected_revision) do + Enum.reduce_while(controls, expected_revision, fn + {^expected_revision, _epochs}, ^expected_revision -> {:cont, expected_revision + 1} + _control, _expected -> {:halt, false} + end) != false + end + defp mark_cluster_control_dirty(state, remote_node) do %{ state @@ -3127,12 +3296,16 @@ defmodule Group.Replica do if MapSet.member?(state.authority_dirty_notified, remote_node) do state else - send(shard_name(state.name, 0), {:replica_authority_dirty_local, remote_node}) + case send_local_control_message(state, {:replica_authority_dirty_local, remote_node}) do + :ok -> + %{ + state + | authority_dirty_notified: MapSet.put(state.authority_dirty_notified, remote_node) + } - %{ - state - | authority_dirty_notified: MapSet.put(state.authority_dirty_notified, remote_node) - } + :disconnected -> + state + end end end @@ -3141,11 +3314,17 @@ defmodule Group.Replica do dirty = Enum.reduce(state.cluster_control_dirty, %{}, fn {remote_node, last_activity}, acc -> - if now - last_activity >= state.replicated_anti_entropy_interval do - request_replica_authority(state, remote_node) - Map.put(acc, remote_node, now) - else - Map.put(acc, remote_node, last_activity) + cond do + is_nil(Data.remote_generation(state.name, remote_node)) and + is_nil(Data.remote_replica_authority_hint(state.name, remote_node)) -> + acc + + now - last_activity >= state.replicated_anti_entropy_interval -> + request_replica_authority(state, remote_node) + Map.put(acc, remote_node, now) + + true -> + Map.put(acc, remote_node, last_activity) end end) @@ -3153,12 +3332,15 @@ defmodule Group.Replica do end defp broadcast_replica_heartbeats(state) do + transport_id = state.replica_transport.id() + descriptor = state.replica_transport.descriptor(state.name, state.replica_transport_opts) + Enum.reduce(state.remote_shards, state, fn {target_node, _pid}, acc -> send_remote_shard_message( acc, target_node, {:replica_heartbeat, self(), WireProtocol.version(), Data.generation(acc.name), - Data.local_cluster_epoch_revision(acc.name)} + Data.local_cluster_epoch_revision(acc.name), transport_id, descriptor} ) acc @@ -3248,13 +3430,35 @@ defmodule Group.Replica do end) end + defp discard_snapshot_send_offsets_for_target(state, target_node) do + offsets = + Map.reject(state.snapshot_send_offsets, fn + {{^target_node, _stream_id, _head}, _chunk_index} -> true + {_key, _chunk_index} -> false + end) + + %{state | snapshot_send_offsets: offsets} + end + + defp discard_snapshot_send_offsets_for_streams(state, stream_ids) do + stream_ids = MapSet.new(stream_ids) + + offsets = + Map.reject(state.snapshot_send_offsets, fn + {{_target_node, stream_id, _head}, _chunk_index} -> + MapSet.member?(stream_ids, stream_id) + end) + + %{state | snapshot_send_offsets: offsets} + end + defp expire_replica_peer(state, remote_node) do state = discard_snapshot_transfers_for_source(state, remote_node) + state = discard_snapshot_send_offsets_for_target(state, remote_node) + state = discard_pending_registry_reprojections(state, remote_node) %{name: name, shard_index: shard} = state - if shard == 0 do - Data.purge_cluster_node(name, remote_node) - end + Data.delete_replica_cursors_for_origin(name, shard, remote_node) {purged_reg, purged_pg} = Data.purge_node(name, shard, remote_node) affected_claims = Data.purge_registry_claims_for_origin(name, shard, remote_node) @@ -3266,12 +3470,9 @@ defmodule Group.Replica do end) notify_monitors(name, events) - Data.delete_replica_cursors_for_origin(name, shard, remote_node) - Data.delete_remote_replica_info(name, shard, remote_node) - if shard == 0 do - fan_out_to_siblings(state, {:replica_authority_removed_local, remote_node}) - end + retirement = Data.expire_remote_replica_lane(name, shard, remote_node) + if retirement == :node_retired, do: Data.purge_cluster_node(name, remote_node) if function_exported?(state.replica_transport, :peer_down, 4) do :ok = @@ -3288,8 +3489,7 @@ defmodule Group.Replica do | remote_shards: Map.delete(state.remote_shards, remote_node), peer_last_seen: Map.delete(state.peer_last_seen, remote_node), cluster_control_dirty: Map.delete(state.cluster_control_dirty, remote_node), - authority_dirty_notified: MapSet.delete(state.authority_dirty_notified, remote_node), - peer_transports: Map.delete(state.peer_transports, remote_node) + authority_dirty_notified: MapSet.delete(state.authority_dirty_notified, remote_node) } end @@ -3335,7 +3535,8 @@ defmodule Group.Replica do end defp replica_stream_target?(state, stream_id, target_node) do - WireProtocol.stream_name(stream_id) == state.name and + WireProtocol.valid_stream_id?(stream_id) and + WireProtocol.stream_name(stream_id) == state.name and WireProtocol.stream_origin(stream_id) == node() and WireProtocol.stream_shard(stream_id) == state.shard_index and WireProtocol.stream_generation(stream_id) == Data.generation(state.name) and @@ -3348,49 +3549,49 @@ defmodule Group.Replica do end defp valid_remote_stream?(state, source_node, stream_id) do - cluster = WireProtocol.stream_cluster(stream_id) - - WireProtocol.stream_name(stream_id) == state.name and - WireProtocol.stream_origin(stream_id) == source_node and - WireProtocol.stream_shard(stream_id) == state.shard_index and - replica_view_current?(state, source_node) and - WireProtocol.stream_generation(stream_id) == Data.remote_generation(state.name, source_node) and - WireProtocol.stream_epoch(stream_id) == - Data.remote_cluster_epoch(state.name, source_node, cluster) and - (is_nil(cluster) or cluster_member?(state.name, cluster)) + if WireProtocol.valid_stream_id?(stream_id) do + cluster = WireProtocol.stream_cluster(stream_id) + + WireProtocol.stream_name(stream_id) == state.name and + WireProtocol.stream_origin(stream_id) == source_node and + WireProtocol.stream_shard(stream_id) == state.shard_index and + replica_view_current?(state, source_node) and + WireProtocol.stream_generation(stream_id) == + Data.remote_generation(state.name, source_node) and + WireProtocol.stream_epoch(stream_id) == + Data.remote_cluster_epoch(state.name, source_node, cluster) and + (is_nil(cluster) or cluster_member?(state.name, cluster)) + else + false + end end defp handle_replica_message(state, source_node, {:heads, version, heads}) - when version == @protocol_version do - needs = - Enum.flat_map(heads, fn {stream_id, _floor, head} -> - if valid_remote_stream?(state, source_node, stream_id) do - cursor = Data.replica_cursor(state.name, state.shard_index, stream_id) - if head > cursor, do: [{stream_id, cursor + 1}], else: [] - else - [] - end - end) - - needs - |> Enum.chunk_every(state.replicated_sender_buffer_size) - |> Enum.reduce(state, fn chunk, acc -> - outgoing_replica_message(acc, source_node, {:needs, WireProtocol.version(), chunk}) - end) + when version == @protocol_version and is_list(heads) do + if Enum.all?(heads, &valid_replica_head?/1) do + handle_replica_heads(state, source_node, heads) + else + state + end end defp handle_replica_message(state, source_node, {:delta_batch, version, runs}) - when version == @protocol_version do - state = flush_pending_replicated_sender_barrier(state) + when version == @protocol_version and is_list(runs) do + if Enum.all?(runs, &valid_replica_delta_run?/1) do + state = flush_pending_replicated_sender_barrier(state) - Enum.reduce(runs, state, fn {stream_id, _first_seq, records, advertised_head}, acc -> - apply_replica_delta_run(acc, source_node, stream_id, records, advertised_head) - end) + Enum.reduce(runs, state, fn {stream_id, _first_seq, records, advertised_head}, acc -> + apply_replica_delta_run(acc, source_node, stream_id, records, advertised_head) + end) + else + state + end end defp handle_replica_message(state, source_node, {:need, version, stream_id, next_seq}) - when version == @protocol_version do - if WireProtocol.stream_origin(stream_id) == node() and + when version == @protocol_version and is_integer(next_seq) and next_seq > 0 do + if WireProtocol.valid_stream_id?(stream_id) and + WireProtocol.stream_origin(stream_id) == node() and WireProtocol.stream_shard(stream_id) == state.shard_index and replica_stream_target?(state, stream_id, source_node) do send_replica_repair(state, source_node, stream_id, next_seq) @@ -3400,8 +3601,12 @@ defmodule Group.Replica do end defp handle_replica_message(state, source_node, {:needs, version, needs}) - when version == @protocol_version do - send_replica_repairs(state, source_node, needs) + when version == @protocol_version and is_list(needs) do + if Enum.all?(needs, &valid_replica_need?/1) do + send_replica_repairs(state, source_node, needs) + else + state + end end defp handle_replica_message( @@ -3455,6 +3660,24 @@ defmodule Group.Replica do defp handle_replica_message(state, _source_node, _message), do: state + defp handle_replica_heads(state, source_node, heads) do + needs = + Enum.flat_map(heads, fn {stream_id, _floor, head} -> + if valid_remote_stream?(state, source_node, stream_id) do + cursor = Data.replica_cursor(state.name, state.shard_index, stream_id) + if head > cursor, do: [{stream_id, cursor + 1}], else: [] + else + [] + end + end) + + needs + |> Enum.chunk_every(state.replicated_sender_buffer_size) + |> Enum.reduce(state, fn chunk, acc -> + outgoing_replica_message(acc, source_node, {:needs, WireProtocol.version(), chunk}) + end) + end + defp valid_snapshot_stream?(state, source_node, stream_id, snapshot_seq) do valid_remote_stream?(state, source_node, stream_id) and snapshot_seq > Data.replica_cursor(state.name, state.shard_index, stream_id) @@ -3482,7 +3705,8 @@ defmodule Group.Replica do cluster = WireProtocol.stream_cluster(stream_id) Enum.all?(reg_data, fn - {key, pid, _meta, _time} when is_pid(pid) -> + {key, pid, meta, time} + when is_binary(key) and is_pid(pid) and is_map(meta) and is_integer(time) -> node(pid) == source_node and shard_index_for(cluster, key, state.num_shards) == state.shard_index @@ -3490,13 +3714,58 @@ defmodule Group.Replica do false end) and Enum.all?(pg_data, fn - {key, pid, _meta, _time} when is_pid(pid) -> + {key, pid, meta, time} + when is_binary(key) and is_pid(pid) and is_map(meta) and is_integer(time) -> node(pid) == source_node and shard_index_for(cluster, key, state.num_shards) == state.shard_index _other -> false - end) + end) and unique_snapshot_rows?(reg_data, pg_data) + end + + defp unique_snapshot_rows?(reg_data, pg_data) do + reg_data + |> MapSet.new(fn {key, _pid, _meta, _time} -> key end) + |> MapSet.size() == length(reg_data) and + pg_data + |> MapSet.new(fn {key, pid, _meta, _time} -> {key, pid} end) + |> MapSet.size() == length(pg_data) + end + + defp valid_replica_head?({stream_id, floor, head}) do + WireProtocol.valid_stream_id?(stream_id) and is_integer(floor) and floor >= 1 and + is_integer(head) and head >= 0 and floor <= head + 1 + end + + defp valid_replica_head?(_head), do: false + + defp valid_replica_need?({stream_id, next_seq}) do + WireProtocol.valid_stream_id?(stream_id) and is_integer(next_seq) and next_seq > 0 + end + + defp valid_replica_need?(_need), do: false + + defp valid_replica_delta_run?({stream_id, first_seq, records, advertised_head}) + when is_integer(first_seq) and first_seq > 0 and is_list(records) and records != [] and + is_integer(advertised_head) and advertised_head >= first_seq do + WireProtocol.valid_stream_id?(stream_id) and + valid_replica_record_sequence?(records, first_seq, advertised_head) + end + + defp valid_replica_delta_run?(_run), do: false + + defp valid_replica_record_sequence?(records, first_seq, advertised_head) do + case Enum.reduce_while(records, first_seq, fn + {seq, mutations}, expected when seq == expected and is_list(mutations) -> + {:cont, expected + 1} + + _record, _expected -> + {:halt, :invalid} + end) do + next_seq when is_integer(next_seq) -> next_seq - 1 <= advertised_head + :invalid -> false + end end defp stage_replica_snapshot_chunk( @@ -3610,6 +3879,14 @@ defmodule Group.Replica do state = flush_pending_replicated_barrier(state) cluster = WireProtocol.stream_cluster(stream_id) + :ok = + Data.begin_replica_snapshot_install( + state.name, + state.shard_index, + stream_id, + transfer.snapshot_seq + ) + affected_registry_keys = Data.replace_registry_claims_for_stream_from_staging( state.name, @@ -3663,6 +3940,14 @@ defmodule Group.Replica do state = flush_pending_replicated_barrier(state) cluster = WireProtocol.stream_cluster(stream_id) + :ok = + Data.begin_replica_snapshot_install( + state.name, + state.shard_index, + stream_id, + snapshot_seq + ) + affected_registry_keys = Data.replace_registry_claims_for_stream( state.name, @@ -3700,7 +3985,7 @@ defmodule Group.Replica do {accepted, rejected} = Enum.split_while(contiguous, fn {_seq, mutations} -> - valid_replica_mutations?(stream_id, mutations) + valid_replica_mutations?(state, stream_id, mutations) end) if rejected != [] do @@ -3710,7 +3995,12 @@ defmodule Group.Replica do end state = - apply_received_replica_records(state, stream_id, accepted) + if accepted == [] do + state + else + :ok = Data.ensure_replica_cursor(state.name, state.shard_index, stream_id) + apply_received_replica_records(state, stream_id, accepted) + end |> flush_pending_replicated_barrier() case List.last(accepted) do @@ -3745,38 +4035,68 @@ defmodule Group.Replica do defp take_contiguous_replica_records(_records, next_seq, acc), do: {Enum.reverse(acc), next_seq} - defp valid_replica_mutations?(stream_id, mutations) do + defp valid_replica_mutations?(state, stream_id, mutations) do origin = WireProtocol.stream_origin(stream_id) cluster = WireProtocol.stream_cluster(stream_id) - mutations != [] and Enum.all?(mutations, &valid_replica_mutation?(&1, cluster, origin)) + mutations != [] and + Enum.all?( + mutations, + &valid_replica_mutation?( + &1, + cluster, + origin, + state.shard_index, + state.num_shards + ) + ) end defp valid_replica_mutation?( - {:register, cluster, _key, pid, _meta, _time, entry_node}, + {:register, cluster, key, pid, meta, time, entry_node}, cluster, - origin - ), - do: node(pid) == origin and entry_node == origin + origin, + shard, + num_shards + ) + when is_binary(key) and is_pid(pid) and is_map(meta) and is_integer(time), + do: + node(pid) == origin and entry_node == origin and + shard_index_for(cluster, key, num_shards) == shard defp valid_replica_mutation?( - {:unregister, cluster, _key, pid, _meta, _reason}, + {:unregister, cluster, key, pid, meta, _reason}, cluster, - origin - ), - do: node(pid) == origin + origin, + shard, + num_shards + ) + when is_binary(key) and is_pid(pid) and is_map(meta), + do: node(pid) == origin and shard_index_for(cluster, key, num_shards) == shard defp valid_replica_mutation?( - {:join, cluster, _key, pid, _meta, _time, _reason, entry_node}, + {:join, cluster, key, pid, meta, time, _reason, entry_node}, cluster, - origin - ), - do: node(pid) == origin and entry_node == origin + origin, + shard, + num_shards + ) + when is_binary(key) and is_pid(pid) and is_map(meta) and is_integer(time), + do: + node(pid) == origin and entry_node == origin and + shard_index_for(cluster, key, num_shards) == shard - defp valid_replica_mutation?({:leave, cluster, _key, pid, _meta, _reason}, cluster, origin), - do: node(pid) == origin + defp valid_replica_mutation?( + {:leave, cluster, key, pid, meta, _reason}, + cluster, + origin, + shard, + num_shards + ) + when is_binary(key) and is_pid(pid) and is_map(meta), + do: node(pid) == origin and shard_index_for(cluster, key, num_shards) == shard - defp valid_replica_mutation?(_mutation, _cluster, _origin), do: false + defp valid_replica_mutation?(_mutation, _cluster, _origin, _shard, _num_shards), do: false defp apply_received_replica_records(state, stream_id, records) do records @@ -3999,33 +4319,131 @@ defmodule Group.Replica do end defp send_replica_snapshot(state, target_node, stream_id, head) do + case state.snapshot_send do + {worker, _token, _snapshot_key} when is_pid(worker) -> + if Process.alive?(worker), + do: state, + else: start_replica_snapshot_send(state, target_node, stream_id, head) + + nil -> + start_replica_snapshot_send(state, target_node, stream_id, head) + end + end + + defp start_replica_snapshot_send(state, target_node, stream_id, head) do + owner = self() + token = make_ref() + snapshot_key = {target_node, stream_id, head} + + offsets = + state.snapshot_send_offsets + |> Enum.reject(fn + {{^target_node, ^stream_id, other_head}, _chunk_index} -> other_head != head + {_other_key, _chunk_index} -> false + end) + |> Map.new() + + start_index = Map.get(offsets, snapshot_key, 1) + + snapshot_context = %{ + name: state.name, + shard_index: state.shard_index, + replicated_snapshot_chunk_target_bytes: state.replicated_snapshot_chunk_target_bytes, + replica_transport: state.replica_transport, + replica_transport_opts: state.replica_transport_opts + } + + worker = + spawn(fn -> + result = + try do + capture_and_send_replica_snapshot( + snapshot_context, + target_node, + stream_id, + head, + start_index + ) + catch + kind, reason -> + Logger.error( + "#{log_prefix_shard(snapshot_context)} snapshot capture failed: " <> + Exception.format_banner(kind, reason) + ) + + :retry + end + + send( + owner, + {:replica_snapshot_send_complete, token, self(), snapshot_key, result} + ) + end) + + %{state | snapshot_send: {worker, token, snapshot_key}, snapshot_send_offsets: offsets} + end + + defp capture_and_send_replica_snapshot(state, target_node, stream_id, head, start_index) do cluster = WireProtocol.stream_cluster(stream_id) reg_data = Data.registry_claims_for_stream(state.name, state.shard_index, stream_id) pg_data = Data.pg_entries_for_origin(state.name, state.shard_index, cluster, node()) - envelope_bytes = - Snapshot.frame_envelope_bytes(stream_id, head, length(reg_data), length(pg_data)) + {_floor, current_head, applied} = + Data.replica_stream_head(state.name, state.shard_index, stream_id) - snapshot = - Snapshot.chunk_rows( - reg_data, - pg_data, - state.replicated_snapshot_chunk_target_bytes, - envelope_bytes - ) + # Appending a mutation advances the head before materializing its table + # changes. Therefore an unchanged, fully-applied head after both scans + # proves these rows are one exact state at `head`; an overlapping write + # makes the capture disposable and the receiver will ask again. + if Data.local_stream_id(state.name, state.shard_index, cluster) == stream_id and + current_head == head and applied == head and + target_node in Data.cluster_nodes(state.name, cluster) do + envelope_bytes = + Snapshot.frame_envelope_bytes(stream_id, head, length(reg_data), length(pg_data)) + + snapshot = + Snapshot.chunk_rows( + reg_data, + pg_data, + state.replicated_snapshot_chunk_target_bytes, + envelope_bytes + ) - chunk_count = length(snapshot.chunks) + chunk_count = length(snapshot.chunks) + + snapshot.chunks + |> Enum.with_index(1) + |> Enum.drop(start_index - 1) + |> Enum.reduce_while(:complete, fn {{reg_chunk, pg_chunk}, chunk_index}, _acc -> + message = + {:snapshot_chunk, WireProtocol.version(), stream_id, head, chunk_index, chunk_count, + snapshot.registry_count, snapshot.pg_count, reg_chunk, pg_chunk} + + case state.replica_transport.outgoing( + state.name, + target_node, + state.shard_index, + message, + state.replica_transport_opts + ) do + :ok -> {:cont, :complete} + result when result in [:busy, :disconnected] -> {:halt, {:resume, chunk_index}} + end + end) + else + :complete + end + end - snapshot.chunks - |> Enum.with_index(1) - |> Enum.reduce(state, fn {{reg_chunk, pg_chunk}, chunk_index}, acc -> - outgoing_replica_message( - acc, - target_node, - {:snapshot_chunk, WireProtocol.version(), stream_id, head, chunk_index, chunk_count, - snapshot.registry_count, snapshot.pg_count, reg_chunk, pg_chunk} - ) - end) + defp current_snapshot_send?(state, {target_node, stream_id, head}) do + cluster = WireProtocol.stream_cluster(stream_id) + + {_floor, current_head, applied} = + Data.replica_stream_head(state.name, state.shard_index, stream_id) + + Data.local_stream_id(state.name, state.shard_index, cluster) == stream_id and + current_head == head and applied == head and + target_node in Data.cluster_nodes(state.name, cluster) end defp replace_remote_pg_snapshot_from_staging( @@ -4163,6 +4581,8 @@ defmodule Group.Replica do defp maybe_purge_remote_generation(state, remote_node, _old_generation, _generation) do state = discard_snapshot_transfers_for_source(state, remote_node) + state = discard_pending_registry_reprojections(state, remote_node) + Data.delete_replica_cursors_for_origin(state.name, state.shard_index, remote_node) {_reg, _pg} = Data.purge_node(state.name, state.shard_index, remote_node) affected = @@ -4178,7 +4598,6 @@ defmodule Group.Replica do end) notify_monitors(state.name, events) - Data.delete_replica_cursors_for_origin(state.name, state.shard_index, remote_node) state end @@ -4201,6 +4620,10 @@ defmodule Group.Replica do state = discard_snapshot_transfers_for_streams(state, stream_ids) + Enum.each(stream_ids, fn stream_id -> + :ok = Data.delete_replica_cursor(state.name, state.shard_index, stream_id) + end) + affected_keys = Data.purge_registry_claims_for_streams( state.name, @@ -4208,10 +4631,6 @@ defmodule Group.Replica do stream_ids ) - Enum.each(stream_ids, fn stream_id -> - :ok = Data.delete_replica_cursor(state.name, state.shard_index, stream_id) - end) - clusters = cluster_epochs |> Enum.map(&elem(&1, 0)) |> Enum.uniq() purged_pg = @@ -4237,6 +4656,7 @@ defmodule Group.Replica do end) notify_monitors(state.name, events) + state end @@ -4314,29 +4734,10 @@ defmodule Group.Replica do superseded |> Enum.group_by(&WireProtocol.stream_cluster/1) |> Enum.reduce(state, fn {cluster, cluster_streams}, acc -> - affected_keys = - Data.purge_registry_claims_for_streams( - state.name, - state.shard_index, - cluster_streams - ) - Enum.each(cluster_streams, fn stream_id -> :ok = Data.delete_replica_cursor(state.name, state.shard_index, stream_id) end) - # PG rows do not carry their stream epoch. Remove the origin/cluster - # slice and reset the current cursor so its exact state is rebuilt by - # the next head advertisement (delta when retained, snapshot after - # pruning). Registry claims do carry epochs and are removed narrowly. - purged_pg = - Data.delete_pg_for_origin_clusters( - state.name, - state.shard_index, - [cluster], - remote_node - ) - case Map.get(current_epochs, cluster) do nil -> :ok @@ -4355,6 +4756,25 @@ defmodule Group.Replica do :ok = Data.delete_replica_cursor(state.name, state.shard_index, current_stream) end + affected_keys = + Data.purge_registry_claims_for_streams( + state.name, + state.shard_index, + cluster_streams + ) + + # PG rows do not carry their stream epoch. Remove the origin/cluster + # slice and reset the current cursor so its exact state is rebuilt by + # the next head advertisement (delta when retained, snapshot after + # pruning). Registry claims do carry epochs and are removed narrowly. + purged_pg = + Data.delete_pg_for_origin_clusters( + state.name, + state.shard_index, + [cluster], + remote_node + ) + events = build_purged_events(state.name, [], purged_pg, :cluster_disconnect, []) {acc, events} = @@ -4370,6 +4790,7 @@ defmodule Group.Replica do end) notify_monitors(state.name, events) + acc end) end @@ -4475,20 +4896,6 @@ defmodule Group.Replica do end) end - defp build_process_down_batch_events(name, reg_entries, pg_entries) do - events = - Enum.reduce(reg_entries, [], fn {pid, cluster, key, meta, reason}, acc -> - [ - build_event(name, :unregistered, key, pid, meta, %{reason: reason, cluster: cluster}) - | acc - ] - end) - - Enum.reduce(pg_entries, events, fn {pid, cluster, key, meta, reason}, acc -> - [build_event(name, :left, key, pid, meta, %{reason: reason, cluster: cluster}) | acc] - end) - end - defp fan_out_to_siblings(state, message) do %{name: name, shard_index: shard_index, num_shards: num_shards} = state @@ -4500,6 +4907,17 @@ defmodule Group.Replica do end end + defp send_local_control_message(state, message) do + case Process.whereis(shard_name(state.name, 0)) do + pid when is_pid(pid) -> + send(pid, message) + :ok + + nil -> + :disconnected + end + end + defp monitor_pid(state, pid) do Map.get(state.monitors, pid) || Process.monitor(pid) end @@ -4594,7 +5012,6 @@ defmodule Group.Replica do state.replicated_oplog_max_entries ) - {state, _events} = rebuild_registry_projections(state) state end @@ -4708,91 +5125,6 @@ defmodule Group.Replica do node() in Data.cluster_nodes(name, cluster) end - # Additive merge: inserts new entries and resolves conflicts, but does not - # delete local entries missing from the incoming snapshot. This is safe because - # Erlang dist uses TCP — either all replicate_* messages arrive in order (no - # stale entries) or the connection dies and nodedown purges everything before - # cluster_state can arrive. There is no case where stale entries survive into - # the merge. - defp merge_remote_cluster_data(state, cluster, reg_data, pg_data, events \\ []) do - %{name: name, shard_index: shard, num_shards: num_shards} = state - - # Registry merge uses Enum.reduce to thread state + events, because - # resolve_conflict modifies state.monitors (demonitor evicted local pids) - # and may produce an event. - {state, events} = - Enum.reduce(reg_data, {state, events}, fn {key, pid, meta, time}, {acc_state, acc_events} -> - if shard_index_for(cluster, key, num_shards) != shard do - {acc_state, acc_events} - else - case Data.registry_lookup(name, shard, cluster, key) do - nil -> - Data.registry_insert(name, shard, cluster, key, pid, meta, time, node(pid)) - event = build_event(name, :registered, key, pid, meta, %{cluster: cluster}) - {acc_state, [event | acc_events]} - - {^pid, _meta, existing_time, _node} -> - # Same pid (bounceback or metadata update) — apply if newer - if time > existing_time do - Data.registry_insert(name, shard, cluster, key, pid, meta, time, node(pid)) - end - - {acc_state, acc_events} - - {existing_pid, existing_meta, existing_time, existing_node} - when existing_node == node() -> - # Local entry vs incoming remote — use full conflict resolution - # (runs the configured resolver and re-broadcasts the winner) - {new_state, event} = - resolve_conflict( - acc_state, - cluster, - key, - {existing_pid, existing_meta, existing_time}, - {pid, meta, time} - ) - - acc_events = if event, do: [event | acc_events], else: acc_events - {new_state, acc_events} - - {existing_pid, _meta, existing_time, _node} when time > existing_time -> - # Both remote — keep the more recent one - if existing_pid != pid do - Data.registry_delete(name, shard, cluster, key, existing_pid) - end - - Data.registry_insert(name, shard, cluster, key, pid, meta, time, node(pid)) - {acc_state, acc_events} - - _ -> - {acc_state, acc_events} - end - end - end) - - events = - Enum.reduce(pg_data, events, fn {key, pid, meta, time}, acc_events -> - if shard_index_for(cluster, key, num_shards) != shard do - acc_events - else - case Data.pg_lookup(name, shard, cluster, key, pid) do - nil -> - Data.pg_insert(name, shard, cluster, key, pid, meta, time, node(pid)) - [build_event(name, :joined, key, pid, meta, %{cluster: cluster}) | acc_events] - - {_meta, existing_time, _node} when time > existing_time -> - Data.pg_insert(name, shard, cluster, key, pid, meta, time, node(pid)) - acc_events - - _ -> - acc_events - end - end - end) - - {state, events} - end - defp resolve_replicated_registry_conflict( state, cluster, @@ -4867,32 +5199,38 @@ defmodule Group.Replica do defp reconcile_registry_projection(state, cluster, key, reason, events) do claims = Data.registry_claims(state.name, state.shard_index, cluster, key) + state = remember_pending_registry_reprojections(state, cluster, key, claims) winner = select_registry_claim_winner(state, cluster, key, claims) - {state, retired?} = retire_local_registry_losers(state, cluster, key, claims, winner) + {state, retirement} = retire_local_registry_losers(state, cluster, key, claims, winner) - winner = - if retired? do - state.name - |> Data.registry_claims(state.shard_index, cluster, key) - |> then(&select_registry_claim_winner(state, cluster, key, &1)) - else - winner - end + case retirement do + :authority_changed -> + reconcile_registry_projection(state, cluster, key, reason, events) - current = Data.registry_lookup(state.name, state.shard_index, cluster, key) + retired? -> + winner = + if retired? do + state.name + |> Data.registry_claims(state.shard_index, cluster, key) + |> then(&select_registry_claim_winner(state, cluster, key, &1)) + else + winner + end - projection_reason = if retired?, do: :resolve_conflict, else: reason + current = Data.registry_lookup(state.name, state.shard_index, cluster, key) + projection_reason = if retired?, do: :resolve_conflict, else: reason - project_registry_winner( - state, - cluster, - key, - current, - winner, - projection_reason, - events - ) + project_registry_winner( + state, + cluster, + key, + current, + winner, + projection_reason, + events + ) + end end defp reconcile_registry_keys(state, keys, reason, events) do @@ -4901,36 +5239,76 @@ defmodule Group.Replica do end) end - defp select_registry_claim_winner(_state, _cluster, _key, []), do: nil - defp select_registry_claim_winner(_state, _cluster, _key, [claim]), do: claim + defp remember_pending_registry_reprojections(state, _cluster, _key, []), do: state + defp remember_pending_registry_reprojections(state, _cluster, _key, [_claim]), do: state - defp select_registry_claim_winner(state, cluster, key, claims) do - claims = - Enum.sort_by(claims, fn {pid, _meta, time, origin_node, generation, epoch, _seq} -> - {time, pid, origin_node, generation, epoch} - end) + defp remember_pending_registry_reprojections(state, cluster, key, claims) do + Enum.reduce(claims, state, fn + {_pid, _meta, _time, origin, generation, epoch, _seq}, acc when origin != node() -> + local_cluster_active? = + is_nil(cluster) or not is_nil(Data.local_cluster_epoch(acc.name, cluster)) - Enum.reduce_while(tl(claims), hd(claims), fn claim, winner -> - {winner_pid, winner_meta, winner_time, _origin, _generation, _epoch, _seq} = winner - {pid, meta, time, _origin, _generation, _epoch, _seq} = claim + waiting_for_lane_view? = + local_cluster_active? and generation == Data.remote_generation(acc.name, origin) and + epoch == Data.remote_cluster_epoch(acc.name, origin, cluster) and + not replica_view_current?(acc, origin) - selected = - resolve_conflict_winner( - state, - cluster, - key, - {winner_pid, winner_meta, winner_time}, - {pid, meta, time} - ) + if waiting_for_lane_view? do + pending = + Map.update( + acc.pending_registry_reprojections, + origin, + MapSet.new([{cluster, key}]), + &MapSet.put(&1, {cluster, key}) + ) - cond do - selected == winner_pid -> {:cont, winner} - selected == pid -> {:cont, claim} - true -> {:halt, nil} - end + %{acc | pending_registry_reprojections: pending} + else + acc + end + + _local_claim, acc -> + acc end) end + defp reproject_pending_registry_keys(state, remote_node) do + case Map.pop(state.pending_registry_reprojections, remote_node) do + {nil, _pending} -> + state + + {keys, pending} -> + state = %{state | pending_registry_reprojections: pending} + {state, events} = reconcile_registry_keys(state, keys, :reconcile, []) + notify_monitors(state.name, events) + state + end + end + + defp discard_pending_registry_reprojections(state, remote_node) do + %{ + state + | pending_registry_reprojections: + Map.delete(state.pending_registry_reprojections, remote_node) + } + end + + defp select_registry_claim_winner(_state, _cluster, _key, []), do: nil + defp select_registry_claim_winner(_state, _cluster, _key, [claim]), do: claim + + defp select_registry_claim_winner(%{name: name} = state, cluster, key, claims) do + resolver = Map.get(Group.get_config(name), :resolve_registry_conflict) + + claims + |> Enum.filter(®istry_claim_current?(state, cluster, &1)) + |> Enum.max_by( + fn {pid, meta, time, _origin, _generation, _epoch, _seq} -> + registry_conflict_order_key(name, key, {pid, meta, time}, resolver) + end, + fn -> nil end + ) + end + defp retire_local_registry_losers(state, cluster, key, claims, winner) do winner_pid = if winner, do: elem(winner, 0), else: nil @@ -4938,19 +5316,75 @@ defmodule Group.Replica do Enum.filter(claims, fn {pid, _meta, _time, origin_node, _generation, _epoch, _seq} -> origin_node == node() and pid != winner_pid end) + |> Enum.filter(®istry_claim_current?(state, cluster, &1)) - state = - Enum.reduce(local_losers, state, fn - {pid, meta, _time, _origin_node, _generation, _epoch, _seq}, acc -> - op = {:unregister, cluster, key, pid, meta, :resolve_conflict} - record = append_local_replica_record(acc, op) - acc = finish_local_replica_record(acc, record, :registry) - winner_meta = if winner, do: elem(winner, 1), else: nil - exit_local_conflict_loser(pid, key, winner_meta) - acc - end) + cond do + local_losers == [] -> + {state, false} + + registry_winner_authoritative?(state, cluster, winner) -> + state = + Enum.reduce(local_losers, state, fn + {pid, meta, _time, _origin_node, _generation, _epoch, _seq}, acc -> + op = {:unregister, cluster, key, pid, meta, :resolve_conflict} + record = append_local_replica_record(acc, op) + acc = finish_local_replica_record(acc, record, :registry) + winner_meta = if winner, do: elem(winner, 1), else: nil + exit_local_conflict_loser(pid, key, winner_meta) + acc + end) + + {state, true} - {state, local_losers != []} + true -> + # Authority changed after winner selection. Nothing irreversible has + # happened yet; select again against the now-current authority. + {state, :authority_changed} + end + end + + defp registry_claim_current?( + state, + cluster, + {_pid, _meta, _time, origin, generation, epoch, _seq} + ) do + local_cluster_active? = + is_nil(cluster) or not is_nil(Data.local_cluster_epoch(state.name, cluster)) + + local_cluster_active? and + if origin == node() do + generation == Data.generation(state.name) and + epoch == Data.local_cluster_epoch(state.name, cluster) + else + replica_view_current?(state, origin) and + generation == Data.remote_generation(state.name, origin) and + epoch == Data.remote_cluster_epoch(state.name, origin, cluster) + end + end + + defp registry_winner_authoritative?(_state, _cluster, nil), do: true + + defp registry_winner_authoritative?( + _state, + _cluster, + {_pid, _meta, _time, origin, _generation, _epoch, _seq} + ) + when origin == node(), + do: true + + defp registry_winner_authoritative?( + state, + cluster, + {_pid, _meta, _time, origin, generation, epoch, _seq} + ) do + Data.remote_registry_claim_authoritative?( + state.name, + state.shard_index, + origin, + generation, + cluster, + epoch + ) end defp project_registry_winner(state, _cluster, _key, nil, nil, _reason, events), @@ -5079,159 +5513,49 @@ defmodule Group.Replica do {state, [registered, unregistered | events]} end - defp resolve_conflict( - state, - cluster, - key, - {local_pid, local_meta, local_time}, - {remote_pid, remote_meta, remote_time} - ) do - %{name: name, shard_index: shard} = state - - winner_pid = - resolve_conflict_winner( - state, - cluster, - key, - {local_pid, local_meta, local_time}, - {remote_pid, remote_meta, remote_time} - ) - - cond do - winner_pid == remote_pid -> - exit_local_conflict_loser(local_pid, key, remote_meta) - # Remote wins — replace local entry - Data.registry_delete(name, shard, cluster, key, local_pid) - state = maybe_demonitor_pid(state, name, shard, local_pid) - time = System.system_time() - - Data.registry_insert( - name, - shard, - cluster, - key, - remote_pid, - remote_meta, - time, - node(remote_pid) - ) - - # Dispatch lifecycle events so monitors see the eviction. - # The :registered event for remote_pid will arrive via the winner's - # re-broadcast registry op, so we only dispatch :unregistered here. - event = - build_event(name, :unregistered, key, local_pid, local_meta, %{ - reason: :resolve_conflict, - cluster: cluster - }) - - state = - enqueue_broadcast_op( - state, - {:unregister, cluster, key, local_pid, local_meta, :resolve_conflict} - ) - - {state, event} - - winner_pid == local_pid -> - # Local wins — re-broadcast to override remote - time = System.system_time() - - Data.registry_insert( - name, - shard, - cluster, - key, - local_pid, - local_meta, - time, - node(local_pid) - ) - - state = - enqueue_broadcast_op( - state, - {:register, cluster, key, local_pid, local_meta, time, node(local_pid)} - ) - - {state, nil} - - true -> - exit_local_conflict_loser(local_pid, key, nil) - # Neither wins — remove both - Data.registry_delete(name, shard, cluster, key, local_pid) - state = maybe_demonitor_pid(state, name, shard, local_pid) - - state = - enqueue_broadcast_op( - state, - {:unregister, cluster, key, local_pid, local_meta, :resolve_conflict} - ) - - event = - build_event(name, :unregistered, key, local_pid, local_meta, %{ - reason: :resolve_conflict, - cluster: cluster - }) - - {state, event} - end - end - defp resolve_conflict_winner( %{name: name}, - cluster, + _cluster, key, {local_pid, local_meta, local_time}, {remote_pid, remote_meta, remote_time} ) do config = Group.get_config(name) + resolver = Map.get(config, :resolve_registry_conflict) - case Map.get(config, :resolve_registry_conflict) do - nil -> - default_resolve_conflict( - name, - cluster, - key, - {local_pid, local_meta, local_time}, - {remote_pid, remote_meta, remote_time} - ) + local = {local_pid, local_meta, local_time} + remote = {remote_pid, remote_meta, remote_time} - {mod, func, extra_args} -> - apply(mod, func, [ - name, - key, - {local_pid, local_meta, local_time}, - {remote_pid, remote_meta, remote_time} | extra_args - ]) + winner = + if registry_conflict_order_key(name, key, remote, resolver) > + registry_conflict_order_key(name, key, local, resolver) do + remote_pid + else + local_pid + end + + if is_nil(resolver) do + log_default_registry_conflict(key, local_pid, remote_pid, winner) end + + winner end - defp default_resolve_conflict( - _name, - _cluster, - key, - {pid1, meta1, time1}, - {pid2, meta2, time2} - ) do - # Tiebreaker must be deterministic regardless of which node is resolving. - # Using `>=` would pick the remote on BOTH nodes when timestamps are equal, - # causing mutual kill (both processes die, key becomes unregistered). - # Erlang pids have a total order (by node name then id), so pid comparison - # gives a consistent tiebreaker across all nodes. - {winner_pid, _winner_meta, _loser_pid} = - if time2 > time1 or (time2 == time1 and pid2 > pid1) do - {pid2, meta2, pid1} - else - {pid1, meta1, pid2} - end + defp registry_conflict_order_key(_name, _key, {pid, _meta, time}, nil), do: {time, pid} + + defp registry_conflict_order_key(name, key, {pid, _meta, _time} = claim, { + mod, + func, + extra_args + }) do + {apply(mod, func, [name, key, claim | extra_args]), pid} + end + defp log_default_registry_conflict(key, pid1, pid2, winner_pid) do Logger.error(fn -> "#{inspect(__MODULE__)}: registry conflict detected: key=#{inspect(key)}, " <> "pid1=#{inspect(pid1)}, pid2=#{inspect(pid2)}, picking #{inspect(winner_pid)} as winner" end) - - winner_pid end defp exit_local_conflict_loser(pid, key, winner_meta) when node(pid) == node() do @@ -5241,26 +5565,6 @@ defmodule Group.Replica do defp exit_local_conflict_loser(_pid, _key, _winner_meta), do: :ok - # Legacy receive-only compatibility: gather local data for all requested - # clusters in one scan before emitting the old cluster_state messages. - defp send_cluster_states(state, clusters, target_node) do - %{name: name, shard_index: shard} = state - {reg_by_cluster, pg_by_cluster} = Data.local_data_by_cluster(name, shard, clusters) - - log_verbose(state, fn -> - "#{log_prefix_shard(state)} sending cluster_states to #{target_node} (#{length(clusters)} clusters)" - end) - - for cluster <- clusters do - reg_data = Map.get(reg_by_cluster, cluster, []) - pg_data = Map.get(pg_by_cluster, cluster, []) - - if reg_data != [] or pg_data != [] do - send_to_peer(state, target_node, {:cluster_state, cluster, reg_data, pg_data}) - end - end - end - defp compute_shared_clusters(my_clusters, remote_clusters) do my_set = MapSet.new(my_clusters) remote_set = MapSet.new(remote_clusters) diff --git a/lib/group/replica/data.ex b/lib/group/replica/data.ex index 3f26250..221531f 100644 --- a/lib/group/replica/data.ex +++ b/lib/group/replica/data.ex @@ -72,7 +72,12 @@ defmodule Group.Replica.Data do position. `replica_oplog` stores `{stream, sequence}` mutation records while `replica_oplog_order` gives them one shard-wide append order for bounded pruning. `replica_cursor` records only the highest contiguous sequence applied from each remote - stream. A gap below the retained floor is repaired by exact per-origin snapshot replacement. + stream. During exact replacement it temporarily stores + `{:snapshot_installing, snapshot_sequence}`; startup repair treats that marker as an + interrupted transaction, purges the partial origin slice, and removes the cursor so the + sender retransmits. Cursor absence is also the durable retirement marker: every peer, + generation, epoch, and local-cluster purge clears cursors before deleting rows. A gap below + the retained floor is repaired by exact per-origin snapshot replacement. ### cluster_nodes — `:bag`, keyed by cluster name @@ -93,8 +98,10 @@ defmodule Group.Replica.Data do Both tables are shared across all shards. Used for the default cluster (nil) and named clusters. Peer-connect messages are discovery hints; shard 0's generation-fenced exact - authority installs membership. `nodedown`, shard death, or peer-lease expiry removes it. - `Group.nodes/1` reads the nil cluster from cluster_nodes. + authority installs membership. Exact authority and both membership indexes are replaced + in one Data GenServer turn, closing the local-connect/remote-install race without a scan + outside the remote authority's cluster set. `nodedown`, shard death, or peer-lease expiry + removes it. `Group.nodes/1` reads the nil cluster from cluster_nodes. ### cluster_leases — `:set`, keyed by cluster name @@ -114,12 +121,25 @@ defmodule Group.Replica.Data do ### replication_meta and epoch tables - `replication_meta` holds the local origin generation, exact and observed authority - revisions, per-shard installed remote views, journal metadata, and one append counter per - shard. `local_cluster_epochs` and `closed_local_cluster_epochs` fence local named-cluster - lifetimes; `remote_cluster_epochs` is the exact node-wide authority installed by shard 0. - Exact and merely observed revisions are separate so a partial control burst cannot be - promoted to authoritative membership. + `replication_meta` holds the local origin generation, last exact, complete applied, and + highest observed authority revisions, a persisted `{generation, revision}` authority hint, + per-shard installed remote views, journal metadata, and one append counter per shard. + `local_cluster_epochs` and `closed_local_cluster_epochs` fence local named-cluster + lifetimes; `remote_cluster_epochs` is the node-wide authority installed by shard 0. + The three revision roles are separate so a partial control burst cannot be promoted to + authoritative membership. Contiguous incremental controls compare-and-install against the + current generation, applied revision, observed revision, and hint in this GenServer turn; + a concurrent heartbeat makes the whole update stale. A newer hint atomically fences every + lane view, but cannot be created after exact authority has been retired; only a later exact + hello can reintroduce the peer. Irreversible registry conflict retirement is + revalidated through this GenServer, serializing the decision with node-wide generation, + epoch, observed-revision, installed-lane, and local-cluster changes. + Local cluster activation also projects self and already-authoritative remote routes in + this serialized turn. Deactivation removes local admission and queues explicit old-epoch + cleanup on every shard before replying, so a caller exit cannot strand rows or a close + barrier; a shard that restarts before handling the message repairs from the marker. + Final peer-route cleanup rechecks that both exact authority and its hint are absent, so a + delayed retirement caller cannot erase a rediscovered generation's routes. ## Match Spec Patterns @@ -134,10 +154,6 @@ defmodule Group.Replica.Data do deletes. O(table size) for the scan, but this only runs on nodedown, remote shard death, or peer-lease expiry — rare paths. - - `local_data_by_cluster/3`: Full table scan filtering by `node() == local_node`, - grouped by cluster. Retained only for the legacy receive-only cluster-state - compatibility path; current recovery uses anti-entropy streams. - - `registry_count`, `pg_count`, `pg_count_by_prefix`, `local_registry_count`, `local_pg_count`, `local_registry_present?`, `local_pg_present?`: Uses `ets.select_count`. Full scan but returns only a count/existence signal @@ -228,6 +244,19 @@ defmodule Group.Replica.Data do |> Enum.map(&elem(&1, 0)) end + def closed_local_cluster_epochs(name) do + closed_local_cluster_epochs_table(name) + |> :ets.tab2list() + |> Enum.map(fn {cluster, epoch, _pending_shards} -> {cluster, epoch} end) + end + + def closed_local_cluster_pending?(name, cluster, epoch, shard) do + case :ets.lookup(closed_local_cluster_epochs_table(name), cluster) do + [{^cluster, ^epoch, pending_shards}] -> MapSet.member?(pending_shards, shard) + _ -> false + end + end + def await_closed_local_clusters(name, clusters, timeout) when is_list(clusters) and is_integer(timeout) and timeout >= 0 do started_at = System.monotonic_time(:millisecond) @@ -271,6 +300,16 @@ defmodule Group.Replica.Data do end end + def remote_replica_authority_hint(name, remote_node) do + case :ets.lookup(replication_meta_table(name), {:remote_authority_hint, remote_node}) do + [{{:remote_authority_hint, ^remote_node}, generation, revision}] -> + {generation, revision} + + [] -> + nil + end + end + @doc false def remote_authority_install_count(name, remote_node) do case :ets.lookup(replication_meta_table(name), {:remote_authority_installs, remote_node}) do @@ -325,26 +364,67 @@ defmodule Group.Replica.Data do ) end - def put_remote_cluster_epochs(name, shard, remote_node, revision, epochs) do + def put_remote_cluster_epochs( + name, + shard, + remote_node, + generation, + expected_revision, + revision, + epochs + ) do GenServer.call( data_name(name), - {:put_remote_cluster_epochs, shard, remote_node, revision, epochs}, + {:put_remote_cluster_epochs, shard, remote_node, generation, expected_revision, revision, + epochs}, :infinity ) end - def close_remote_cluster_epochs(name, shard, remote_node, revision, epochs) do + def observe_remote_cluster_epoch_revision(name, remote_node, revision) do GenServer.call( data_name(name), - {:close_remote_cluster_epochs, shard, remote_node, revision, epochs}, + {:observe_remote_cluster_epoch_revision, remote_node, revision}, :infinity ) end - def forget_remote_cluster_epochs(name, shard, remote_node, epochs) do + def observe_remote_replica_hint(name, remote_node, generation, revision) do GenServer.call( data_name(name), - {:forget_remote_cluster_epochs, shard, remote_node, epochs}, + {:observe_remote_replica_hint, remote_node, generation, revision}, + :infinity + ) + end + + def remote_registry_claim_authoritative?( + name, + shard, + remote_node, + generation, + cluster, + epoch + ) do + GenServer.call( + data_name(name), + {:remote_registry_claim_authoritative, shard, remote_node, generation, cluster, epoch}, + :infinity + ) + end + + def close_remote_cluster_epochs( + name, + shard, + remote_node, + generation, + expected_revision, + revision, + epochs + ) do + GenServer.call( + data_name(name), + {:close_remote_cluster_epochs, shard, remote_node, generation, expected_revision, revision, + epochs}, :infinity ) end @@ -357,16 +437,36 @@ defmodule Group.Replica.Data do ) end + def expire_remote_replica_lane(name, shard, remote_node) do + GenServer.call( + data_name(name), + {:expire_remote_replica_lane, shard, remote_node}, + :infinity + ) + end + def activate_local_clusters(name, clusters) do GenServer.call(data_name(name), {:activate_local_clusters, clusters}, :infinity) end + def activate_local_clusters_durable(name, clusters) do + GenServer.call(data_name(name), {:activate_local_clusters_durable, clusters}, :infinity) + end + def deactivate_local_clusters(name, clusters) do GenServer.call(data_name(name), {:deactivate_local_clusters, clusters}, :infinity) end - def mark_closed_cluster_shard(name, clusters, shard) do - GenServer.call(data_name(name), {:mark_closed_cluster_shard, clusters, shard}, :infinity) + def deactivate_local_clusters_durable(name, clusters) do + GenServer.call(data_name(name), {:deactivate_local_clusters_durable, clusters}, :infinity) + end + + def mark_closed_cluster_shard(name, cluster_epochs, shard) do + GenServer.call( + data_name(name), + {:mark_closed_cluster_shard, cluster_epochs, shard}, + :infinity + ) end def local_stream_id(name, shard, cluster) do @@ -491,7 +591,8 @@ defmodule Group.Replica.Data do @doc false def repair_shard_indexes(name, shard) do - purge_inactive_cluster_rows(name, shard) + repair_interrupted_snapshot_installs(name, shard) + repair_primary_replica_rows(name, shard) rebuild_registry_reverse_index(name, shard) rebuild_registry_claim_reverse_index(name, shard) rebuild_pg_reverse_index(name, shard) @@ -527,6 +628,37 @@ defmodule Group.Replica.Data do WireProtocol.stream_epoch(stream_id) == local_cluster_epoch(name, cluster) end + defp repair_interrupted_snapshot_installs(name, shard) do + streams = + :ets.select(replica_cursor_table(name, shard), [ + {{:"$1", {:snapshot_installing, :_}}, [], [:"$1"]} + ]) + + {streams, malformed} = + Enum.split_with(streams, fn stream_id -> + WireProtocol.valid_stream_id?(stream_id) and + WireProtocol.stream_name(stream_id) == name and + WireProtocol.stream_shard(stream_id) == shard and + WireProtocol.stream_origin(stream_id) != node() + end) + + Enum.each(malformed, &:ets.delete(replica_cursor_table(name, shard), &1)) + + if streams != [] do + _affected_keys = purge_registry_claims_for_streams(name, shard, streams) + + streams + |> Enum.group_by(&WireProtocol.stream_origin/1, &WireProtocol.stream_cluster/1) + |> Enum.each(fn {origin, clusters} -> + delete_pg_for_origin_clusters(name, shard, Enum.uniq(clusters), origin) + end) + + Enum.each(streams, &:ets.delete(replica_cursor_table(name, shard), &1)) + end + + :ok + end + defp await_closed_local_clusters(name, clusters, timeout, started_at) do pending? = Enum.any?(clusters, fn cluster -> @@ -591,46 +723,179 @@ defmodule Group.Replica.Data do end) end - defp purge_inactive_cluster_rows(name, shard) do - clusters = - Enum.concat([ - Enum.map(:ets.tab2list(reg_by_key_table(name, shard)), fn - {{cluster, _key}, _pid, _meta, _time, _entry_node} -> cluster - end), - Enum.map(:ets.tab2list(reg_claim_by_key_table(name, shard)), fn - {{cluster, _key, _origin, _generation, _epoch}, _pid, _meta, _time, _seq} -> - cluster - end), - Enum.map(:ets.tab2list(pg_by_key_table(name, shard)), fn - {{cluster, _key, _pid}, _meta, _time, _entry_node} -> cluster - end), - Enum.map(:ets.tab2list(replica_cursor_table(name, shard)), fn {stream_id, _seq} -> - WireProtocol.stream_cluster(stream_id) - end) - ]) - |> Enum.reject(&is_nil/1) - |> Enum.uniq() - |> Enum.filter(&is_nil(local_cluster_epoch(name, &1))) + # A shard can crash between writes to its materialized rows and receive + # cursor, or while retiring an epoch across multiple ETS tables. Recover from + # the primary tables themselves: stale claims carry their complete stream + # authority, while a remote PG row is retained only when the current stream + # has a cursor (including the sequence-zero admission marker). This pass also + # replaces the old multi-million-element cluster list with one fixed-table + # traversal and O(number of inactive clusters) accumulator memory. + defp repair_primary_replica_rows(name, shard) do + inactive_clusters = MapSet.new() + + inactive_clusters = + repair_ets_table( + reg_by_key_table(name, shard), + inactive_clusters, + fn {{cluster, key}, _pid, _meta, _time, _entry_node}, inactive -> + if active_local_cluster?(name, cluster) do + inactive + else + :ets.delete(reg_by_key_table(name, shard), {cluster, key}) + remember_inactive_cluster(name, inactive, cluster) + end + end + ) - Enum.each(clusters, fn cluster -> - :ets.select_delete(reg_by_key_table(name, shard), [ - {{{cluster, :_}, :_, :_, :_, :_}, [], [true]} - ]) + inactive_clusters = + repair_ets_table( + reg_claim_by_key_table(name, shard), + inactive_clusters, + fn {{cluster, key, origin, claim_generation, epoch}, _pid, _meta, _time, _seq}, + inactive -> + if active_local_cluster?(name, cluster) and + valid_claim_authority?( + name, + shard, + cluster, + origin, + claim_generation, + epoch + ) do + inactive + else + :ets.delete( + reg_claim_by_key_table(name, shard), + {cluster, key, origin, claim_generation, epoch} + ) + + remember_inactive_cluster(name, inactive, cluster) + end + end + ) - :ets.select_delete(reg_claim_by_key_table(name, shard), [ - {{{cluster, :_, :_, :_, :_}, :_, :_, :_, :_}, [], [true]} - ]) + inactive_clusters = + repair_ets_table( + pg_by_key_table(name, shard), + inactive_clusters, + fn {{cluster, key, pid}, _meta, _time, entry_node}, inactive -> + valid? = + active_local_cluster?(name, cluster) and node(pid) == entry_node and + (entry_node == node() or + valid_remote_pg_authority?(name, shard, cluster, entry_node)) + + if valid? do + inactive + else + :ets.delete(pg_by_key_table(name, shard), {cluster, key, pid}) + remember_inactive_cluster(name, inactive, cluster) + end + end + ) - :ets.select_delete(pg_by_key_table(name, shard), [ - {{{cluster, :_, :_}, :_, :_, :_}, [], [true]} - ]) - end) + inactive_clusters = + repair_ets_table( + replica_cursor_table(name, shard), + inactive_clusters, + fn {stream_id, _cursor}, inactive -> + cluster = + if WireProtocol.valid_stream_id?(stream_id), + do: WireProtocol.stream_cluster(stream_id) + + if valid_remote_cursor_authority?(name, shard, stream_id) do + inactive + else + :ets.delete(replica_cursor_table(name, shard), stream_id) + remember_inactive_cluster(name, inactive, cluster) + end + end + ) - :ok = delete_replica_cursors_for_clusters(name, shard, clusters) - if clusters != [], do: remove_clusters(name, clusters) + inactive_clusters = MapSet.to_list(inactive_clusters) + if inactive_clusters != [], do: remove_clusters(name, inactive_clusters) :ok end + defp repair_ets_table(table, acc, fun) do + :ets.safe_fixtable(table, true) + + try do + repair_ets_table(table, :ets.first(table), acc, fun) + after + :ets.safe_fixtable(table, false) + end + end + + defp repair_ets_table(_table, :"$end_of_table", acc, _fun), do: acc + + defp repair_ets_table(table, key, acc, fun) do + next_key = :ets.next(table, key) + + acc = + case :ets.lookup(table, key) do + [object] -> fun.(object, acc) + [] -> acc + end + + repair_ets_table(table, next_key, acc, fun) + end + + defp active_local_cluster?(_name, nil), do: true + defp active_local_cluster?(name, cluster), do: not is_nil(local_cluster_epoch(name, cluster)) + + defp remember_inactive_cluster(_name, inactive, nil), do: inactive + + defp remember_inactive_cluster(name, inactive, cluster) do + if active_local_cluster?(name, cluster), do: inactive, else: MapSet.put(inactive, cluster) + end + + defp valid_claim_authority?(name, _shard, cluster, origin, claim_generation, epoch) + when origin == node() do + claim_generation == generation(name) and epoch == local_cluster_epoch(name, cluster) + end + + defp valid_claim_authority?(name, shard, cluster, origin, claim_generation, epoch) do + if claim_generation == remote_generation(name, origin) and + epoch == remote_cluster_epoch(name, origin, cluster) do + stream_id = + WireProtocol.stream_id(name, origin, claim_generation, shard, cluster, epoch) + + :ets.member(replica_cursor_table(name, shard), stream_id) + else + false + end + end + + defp valid_remote_pg_authority?(name, shard, cluster, origin) do + remote_generation = remote_generation(name, origin) + remote_epoch = remote_cluster_epoch(name, origin, cluster) + + if WireProtocol.valid_generation?(remote_generation) and not is_nil(remote_epoch) do + stream_id = + WireProtocol.stream_id(name, origin, remote_generation, shard, cluster, remote_epoch) + + :ets.member(replica_cursor_table(name, shard), stream_id) + else + false + end + end + + defp valid_remote_cursor_authority?(name, shard, stream_id) do + WireProtocol.valid_stream_id?(stream_id) and + WireProtocol.stream_name(stream_id) == name and + WireProtocol.stream_shard(stream_id) == shard and + WireProtocol.stream_origin(stream_id) != node() and + active_local_cluster?(name, WireProtocol.stream_cluster(stream_id)) and + WireProtocol.stream_generation(stream_id) == + remote_generation(name, WireProtocol.stream_origin(stream_id)) and + WireProtocol.stream_epoch(stream_id) == + remote_cluster_epoch( + name, + WireProtocol.stream_origin(stream_id), + WireProtocol.stream_cluster(stream_id) + ) + end + def replica_stream_head(name, shard, stream_id) do case :ets.lookup(replica_stream_meta_table(name, shard), stream_id) do [{^stream_id, head, floor, applied}] -> {floor, head, applied} @@ -697,7 +962,8 @@ defmodule Group.Replica.Data do def replica_cursor(name, shard, stream_id) do case :ets.lookup(replica_cursor_table(name, shard), stream_id) do - [{^stream_id, seq}] -> seq + [{^stream_id, seq}] when is_integer(seq) -> seq + [{^stream_id, {:snapshot_installing, _snapshot_seq}}] -> 0 [] -> 0 end end @@ -721,6 +987,20 @@ defmodule Group.Replica.Data do :ok end + def ensure_replica_cursor(name, shard, stream_id) do + :ets.insert_new(replica_cursor_table(name, shard), {stream_id, 0}) + :ok + end + + def begin_replica_snapshot_install(name, shard, stream_id, snapshot_seq) do + :ets.insert( + replica_cursor_table(name, shard), + {stream_id, {:snapshot_installing, snapshot_seq}} + ) + + :ok + end + def delete_replica_cursors_for_origin(name, shard, origin_node) do :ets.select_delete(replica_cursor_table(name, shard), [ {{{name, origin_node, :_, shard, :_, :_}, :_}, [], [true]} @@ -746,6 +1026,33 @@ defmodule Group.Replica.Data do :ok end + @doc false + def retained_replica_origins(name, shard) do + # Accepted replica data is always fenced by this persisted lane view. Every + # retirement path destroys data and cursors before deleting the view, so a + # shard crash cannot leave valid data without this restart index. Scanning + # it is O(known peers * shards), never O(registry + PG cardinality). + match_specs = [ + {{{:remote_view_info, shard, :"$1"}, :_, :_, :_}, [], [:"$1"]}, + # A hint is the durable fence left before the observing lane records its + # in-memory lease deadline. Every restarting lane must recognize it so a + # crash in that window cannot strand the peer forever. + {{{:remote_authority_hint, :"$1"}, :_, :_}, [], [:"$1"]} + ] + + match_specs = + if shard == 0 do + [{{{:remote_generation, :"$1"}, :_}, [], [:"$1"]} | match_specs] + else + match_specs + end + + replication_meta_table(name) + |> :ets.select(match_specs) + |> Enum.reject(&(&1 == node())) + |> Enum.uniq() + end + def drop_local_stream(name, shard, cluster, epoch) do stream_id = Group.Replica.WireProtocol.stream_id(name, node(), generation(name), shard, cluster, epoch) @@ -1039,22 +1346,20 @@ defmodule Group.Replica.Data do def purge_registry_claims_for_streams(_name, _shard, []), do: [] def purge_registry_claims_for_streams(name, shard, stream_ids) do - streams = - MapSet.new(stream_ids, fn stream_id -> - { - Group.Replica.WireProtocol.stream_cluster(stream_id), - Group.Replica.WireProtocol.stream_origin(stream_id), - Group.Replica.WireProtocol.stream_generation(stream_id), - Group.Replica.WireProtocol.stream_epoch(stream_id) - } - end) + # Select only matching claims into memory. Epoch churn is rare, but the + # complete claim table may contain millions of unrelated rows. + match_specs = + Enum.map(stream_ids, fn stream_id -> + cluster = Group.Replica.WireProtocol.stream_cluster(stream_id) + origin = Group.Replica.WireProtocol.stream_origin(stream_id) + generation = Group.Replica.WireProtocol.stream_generation(stream_id) + epoch = Group.Replica.WireProtocol.stream_epoch(stream_id) - claims = - :ets.tab2list(reg_claim_by_key_table(name, shard)) - |> Enum.filter(fn {{cluster, _key, origin, generation, epoch}, _pid, _meta, _time, _seq} -> - MapSet.member?(streams, {cluster, origin, generation, epoch}) + {{{cluster, :"$1", origin, generation, epoch}, :"$2", :"$3", :"$4", :"$5"}, [], [:"$_"]} end) + claims = :ets.select(reg_claim_by_key_table(name, shard), match_specs) + Enum.each(claims, fn {{cluster, key, origin, generation, epoch}, pid, _meta, _time, _seq} -> :ets.delete( reg_claim_by_key_table(name, shard), @@ -1480,33 +1785,6 @@ defmodule Group.Replica.Data do ) end - def local_data_by_cluster(name, shard, clusters) do - cluster_set = MapSet.new(clusters) - local_node = node() - - reg_table = reg_by_key_table(name, shard) - - reg_by_cluster = - :ets.select(reg_table, [ - {{{:"$1", :"$2"}, :"$3", :"$4", :"$5", :"$6"}, [{:==, :"$6", local_node}], - [{{:"$1", :"$2", :"$3", :"$4", :"$5"}}]} - ]) - |> Enum.filter(fn {cluster, _, _, _, _} -> MapSet.member?(cluster_set, cluster) end) - |> Enum.group_by(&elem(&1, 0), fn {_, key, pid, meta, time} -> {key, pid, meta, time} end) - - pg_table = pg_by_key_table(name, shard) - - pg_by_cluster = - :ets.select(pg_table, [ - {{{:"$1", :"$2", :"$3"}, :"$4", :"$5", :"$6"}, [{:==, :"$6", local_node}], - [{{:"$1", :"$2", :"$3", :"$4", :"$5"}}]} - ]) - |> Enum.filter(fn {cluster, _, _, _, _} -> MapSet.member?(cluster_set, cluster) end) - |> Enum.group_by(&elem(&1, 0), fn {_, key, pid, meta, time} -> {key, pid, meta, time} end) - - {reg_by_cluster, pg_by_cluster} - end - def pg_entries_for_origin(name, shard, cluster, origin_node) do :ets.select(pg_by_key_table(name, shard), [ {{{cluster, :"$1", :"$2"}, :"$3", :"$4", origin_node}, [], [{{:"$1", :"$2", :"$3", :"$4"}}]} @@ -1821,50 +2099,98 @@ defmodule Group.Replica.Data do # GenServer callbacks # ===================================================================== - @impl true - def handle_call({:add_cluster_node, clusters, node}, _from, state) do - :ets.insert(cluster_nodes_table(state.name), Enum.map(clusters, &{&1, node})) - :ets.insert(node_clusters_table(state.name), Enum.map(clusters, &{node, &1})) - {:reply, :ok, state} + defp replace_remote_cluster_projection(name, remote_node, remote_epochs) do + local_clusters = + local_cluster_epochs_table(name) + |> :ets.select([{{:"$1", :_}, [], [:"$1"]}]) + |> MapSet.new() + |> MapSet.put(nil) + + shared_clusters = + remote_epochs + |> Map.keys() + |> Enum.filter(&MapSet.member?(local_clusters, &1)) + |> MapSet.new() + + previous_clusters = MapSet.new(clusters_for_node(name, remote_node)) + + previous_clusters + |> MapSet.difference(shared_clusters) + |> Enum.each(fn cluster -> + :ets.delete_object(cluster_nodes_table(name), {cluster, remote_node}) + :ets.delete_object(node_clusters_table(name), {remote_node, cluster}) + end) + + rows = MapSet.to_list(shared_clusters) + :ets.insert(cluster_nodes_table(name), Enum.map(rows, &{&1, remote_node})) + :ets.insert(node_clusters_table(name), Enum.map(rows, &{remote_node, &1})) + :ok end - def handle_call({:remove_cluster_node, clusters, node}, _from, state) do + defp insert_cluster_nodes(name, clusters, target_node) do + :ets.insert(cluster_nodes_table(name), Enum.map(clusters, &{&1, target_node})) + :ets.insert(node_clusters_table(name), Enum.map(clusters, &{target_node, &1})) + :ok + end + + defp delete_cluster_nodes(name, clusters, target_node) do Enum.each(clusters, fn cluster -> - :ets.delete_object(cluster_nodes_table(state.name), {cluster, node}) - :ets.delete_object(node_clusters_table(state.name), {node, cluster}) + :ets.delete_object(cluster_nodes_table(name), {cluster, target_node}) + :ets.delete_object(node_clusters_table(name), {target_node, cluster}) end) - {:reply, :ok, state} + :ok end - def handle_call({:remove_clusters, clusters}, _from, state) do + defp delete_cluster_routes(name, clusters) do Enum.each(clusters, fn cluster -> - nodes = cluster_nodes(state.name, cluster) - :ets.delete(cluster_nodes_table(state.name), cluster) + nodes = cluster_nodes(name, cluster) + :ets.delete(cluster_nodes_table(name), cluster) Enum.each(nodes, fn cluster_node -> - :ets.delete_object( - node_clusters_table(state.name), - {cluster_node, cluster} - ) + :ets.delete_object(node_clusters_table(name), {cluster_node, cluster}) end) end) - {:reply, :ok, state} + :ok end - def handle_call({:purge_cluster_node, dead_node}, _from, state) do + defp delete_peer_routes(name, remote_node) do # Scan the forward index directly so this also repairs a one-sided row left # by an interrupted or older dual-index mutation. - :ets.select_delete(cluster_nodes_table(state.name), [ - {{:_, dead_node}, [], [true]} + :ets.select_delete(cluster_nodes_table(name), [ + {{:_, remote_node}, [], [true]} ]) - :ets.delete(node_clusters_table(state.name), dead_node) - {:reply, :ok, state} + :ets.delete(node_clusters_table(name), remote_node) + :ok end - def handle_call({:activate_local_clusters, clusters}, _from, state) do + defp project_activated_local_clusters(name, clusters) do + :ok = insert_cluster_nodes(name, clusters, node()) + + name + |> cluster_nodes(nil) + |> Enum.reject(&(&1 == node())) + |> Enum.each(fn remote_node -> + shared = + Enum.filter(clusters, fn cluster -> + not is_nil(remote_cluster_epoch(name, remote_node, cluster)) + end) + + :ok = insert_cluster_nodes(name, shared, remote_node) + end) + + :ok + end + + defp cast_cluster_lifecycle(name, shards, request) do + Enum.each(shards, fn shard -> + :ok = Group.Replica.local_cast(Group.Replica.shard_name(name, shard), request) + end) + end + + defp activate_local_clusters(state, clusters, durable?) do if clusters != [] do :ets.update_counter( replication_meta_table(state.name), @@ -1887,17 +2213,12 @@ defmodule Group.Replica.Data do {cluster, epoch} end) - {:reply, epochs, state} - end + if durable?, do: project_activated_local_clusters(state.name, clusters) - def handle_call(:local_replica_authority, _from, state) do - generation = generation(state.name) - revision = local_cluster_epoch_revision(state.name) - epochs = [{nil, generation} | :ets.tab2list(local_cluster_epochs_table(state.name))] - {:reply, {generation, revision, epochs}, state} + {epochs, state} end - def handle_call({:deactivate_local_clusters, clusters}, _from, state) do + defp deactivate_local_clusters(state, clusters, durable?) do if clusters != [] do :ets.update_counter( replication_meta_table(state.name), @@ -1924,29 +2245,106 @@ defmodule Group.Replica.Data do {cluster, epoch} end) + if durable? do + :ok = delete_cluster_nodes(state.name, clusters, node()) + + cast_cluster_lifecycle( + state.name, + 0..(state.num_shards - 1), + {:cluster_disconnect, clusters, epochs} + ) + end + + {epochs, state} + end + + @impl true + def handle_call({:add_cluster_node, clusters, node}, _from, state) do + :ok = insert_cluster_nodes(state.name, clusters, node) + {:reply, :ok, state} + end + + def handle_call({:remove_cluster_node, clusters, node}, _from, state) do + :ok = delete_cluster_nodes(state.name, clusters, node) + {:reply, :ok, state} + end + + def handle_call({:remove_clusters, clusters}, _from, state) do + # Startup repair discovers inactive clusters outside the Data process. A + # reconnect may install a new epoch before this serialized cleanup runs; + # recheck authority here so stale repair work cannot erase the new routes. + inactive_clusters = + Enum.filter(clusters, &is_nil(local_cluster_epoch(state.name, &1))) + + :ok = delete_cluster_routes(state.name, inactive_clusters) + + {:reply, :ok, state} + end + + def handle_call({:purge_cluster_node, dead_node}, _from, state) do + # Nodedown/lease callers may resume after a newer exact authority has + # already reinstalled this peer. Recheck the serialized authority fence so + # stale cleanup cannot erase routes belonging to the new incarnation. + if is_nil(remote_generation(state.name, dead_node)) and + is_nil(remote_replica_authority_hint(state.name, dead_node)) do + :ok = delete_peer_routes(state.name, dead_node) + end + + {:reply, :ok, state} + end + + def handle_call({:activate_local_clusters, clusters}, _from, state) do + {epochs, state} = activate_local_clusters(state, clusters, false) {:reply, epochs, state} end - def handle_call({:mark_closed_cluster_shard, clusters, shard}, _from, state) do + def handle_call({:activate_local_clusters_durable, clusters}, _from, state) do + {epochs, state} = activate_local_clusters(state, clusters, true) + {:reply, epochs, state} + end + + def handle_call(:local_replica_authority, _from, state) do + generation = generation(state.name) + revision = local_cluster_epoch_revision(state.name) + epochs = [{nil, generation} | :ets.tab2list(local_cluster_epochs_table(state.name))] + {:reply, {generation, revision, epochs}, state} + end + + def handle_call({:deactivate_local_clusters, clusters}, _from, state) do + {epochs, state} = deactivate_local_clusters(state, clusters, false) + {:reply, epochs, state} + end + + def handle_call({:deactivate_local_clusters_durable, clusters}, _from, state) do + {epochs, state} = deactivate_local_clusters(state, clusters, true) + {:reply, epochs, state} + end + + def handle_call({:mark_closed_cluster_shard, cluster_epochs, shard}, _from, state) do completed = - Enum.reduce(clusters, [], fn cluster, acc -> + Enum.reduce(cluster_epochs, [], fn {cluster, request_epoch}, acc -> case :ets.lookup(closed_local_cluster_epochs_table(state.name), cluster) do - [{^cluster, epoch, pending_shards}] -> + [{^cluster, ^request_epoch, pending_shards}] -> pending_shards = MapSet.delete(pending_shards, shard) if MapSet.size(pending_shards) == 0 do + # The close marker is the durable recovery obligation. Remove + # every route before deleting it so a caller and the final shard + # can both disappear immediately after this acknowledgement + # without leaving an unowned cluster membership behind. + :ok = delete_cluster_routes(state.name, [cluster]) :ets.delete(closed_local_cluster_epochs_table(state.name), cluster) [cluster | acc] else :ets.insert( closed_local_cluster_epochs_table(state.name), - {cluster, epoch, pending_shards} + {cluster, request_epoch, pending_shards} ) acc end - [] -> + _stale_or_completed -> acc end end) @@ -1966,63 +2364,82 @@ defmodule Group.Replica.Data do # one full copy per shard. 0 = shard seen_generation = remote_generation(state.name, remote_node) - current_epochs = Map.new(epochs) - - stale_epochs = - if seen_generation == generation do - for {{^remote_node, cluster}, epoch} <- - :ets.match_object( - remote_cluster_epochs_table(state.name), - {{remote_node, :_}, :_} - ), - not is_nil(cluster), - Map.get(current_epochs, cluster) != epoch, - do: {cluster, epoch} - else - [] - end - # A hello is a complete epoch snapshot. The replica handler fences older - # revisions before this call, so replace the shared view rather than merely - # adding rows; otherwise a dropped close control could leave a cluster epoch - # permanently valid after the heartbeat-driven repair. - :ets.select_delete(remote_cluster_epochs_table(state.name), [ - {{{remote_node, :_}, :_}, [], [true]} - ]) + stale? = + stale_remote_authority_install?( + state.name, + remote_node, + generation, + epoch_revision + ) - :ets.insert( - replication_meta_table(state.name), - {{:remote_generation, remote_node}, generation} - ) + if stale? do + {:reply, :stale, state} + else + current_epochs = Map.new(epochs) + + stale_epochs = + if seen_generation == generation do + for {{^remote_node, cluster}, epoch} <- + :ets.match_object( + remote_cluster_epochs_table(state.name), + {{remote_node, :_}, :_} + ), + not is_nil(cluster), + Map.get(current_epochs, cluster) != epoch, + do: {cluster, epoch} + else + [] + end - :ets.insert( - replication_meta_table(state.name), - {{:remote_epoch_revision, remote_node}, epoch_revision} - ) + # A hello is a complete epoch snapshot. The replica handler fences older + # revisions before this call, so replace the shared view rather than merely + # adding rows; otherwise a dropped close control could leave a cluster epoch + # permanently valid after the heartbeat-driven repair. + :ets.select_delete(remote_cluster_epochs_table(state.name), [ + {{{remote_node, :_}, :_}, [], [true]} + ]) - :ets.insert( - replication_meta_table(state.name), - {{:remote_epoch_exact, remote_node}, epoch_revision} - ) + :ets.insert( + replication_meta_table(state.name), + {{:remote_generation, remote_node}, generation} + ) - :ets.insert( - replication_meta_table(state.name), - {{:remote_epoch_observed, remote_node}, epoch_revision} - ) + :ets.insert( + replication_meta_table(state.name), + {{:remote_epoch_revision, remote_node}, epoch_revision} + ) - :ets.update_counter( - replication_meta_table(state.name), - {:remote_authority_installs, remote_node}, - {2, 1}, - {{:remote_authority_installs, remote_node}, 0} - ) + :ets.insert( + replication_meta_table(state.name), + {{:remote_epoch_exact, remote_node}, epoch_revision} + ) + + :ets.insert( + replication_meta_table(state.name), + {{:remote_epoch_observed, remote_node}, epoch_revision} + ) - rows = - for {cluster, epoch} <- epochs, not is_nil(cluster), do: {{remote_node, cluster}, epoch} + put_remote_authority_hint(state.name, remote_node, generation, epoch_revision) - :ets.insert(remote_cluster_epochs_table(state.name), rows) + replace_remote_cluster_projection(state.name, remote_node, current_epochs) - {:reply, {seen_generation, stale_epochs}, state} + :ets.update_counter( + replication_meta_table(state.name), + {:remote_authority_installs, remote_node}, + {2, 1}, + {{:remote_authority_installs, remote_node}, 0} + ) + + rows = + for {cluster, epoch} <- epochs, + not is_nil(cluster), + do: {{remote_node, cluster}, epoch} + + :ets.insert(remote_cluster_epochs_table(state.name), rows) + + {:reply, {seen_generation, stale_epochs}, state} + end end def handle_call( @@ -2030,89 +2447,202 @@ defmodule Group.Replica.Data do _from, state ) do - :ets.insert( - replication_meta_table(state.name), - {{:remote_view_info, shard, remote_node}, generation, authoritative, observed} - ) + if remote_generation(state.name, remote_node) == generation and + remote_cluster_epoch_exact_revision(state.name, remote_node) == authoritative and + remote_cluster_epoch_revision(state.name, remote_node) == observed and + remote_cluster_epoch_observed_revision(state.name, remote_node) == observed and + remote_replica_authority_hint(state.name, remote_node) == {generation, observed} do + :ets.insert( + replication_meta_table(state.name), + {{:remote_view_info, shard, remote_node}, generation, authoritative, observed} + ) - {:reply, :ok, state} + {:reply, :ok, state} + else + {:reply, :stale, state} + end end def handle_call( - {:put_remote_cluster_epochs, shard, remote_node, revision, epochs}, + {:put_remote_cluster_epochs, shard, remote_node, generation, expected_revision, revision, + epochs}, _from, state ) do - _ = shard - observe_remote_cluster_revision(state.name, remote_node, revision, state.num_shards) - - stale_epochs = - Enum.flat_map(epochs, fn {cluster, epoch} -> - case remote_cluster_epoch(state.name, remote_node, cluster) do - old_epoch when not is_nil(old_epoch) and old_epoch != epoch -> [{cluster, old_epoch}] - _ -> [] - end - end) + 0 = shard - rows = - for {cluster, epoch} <- epochs, - not is_nil(cluster), - do: {{remote_node, cluster}, epoch} + if incremental_authority_installable?( + state.name, + remote_node, + generation, + expected_revision, + revision + ) do + observe_remote_cluster_revision(state.name, remote_node, revision, state.num_shards) + + stale_epochs = + Enum.flat_map(epochs, fn {cluster, epoch} -> + case remote_cluster_epoch(state.name, remote_node, cluster) do + old_epoch when not is_nil(old_epoch) and old_epoch != epoch -> [{cluster, old_epoch}] + _ -> [] + end + end) - :ets.insert(remote_cluster_epochs_table(state.name), rows) + rows = + for {cluster, epoch} <- epochs, + not is_nil(cluster), + do: {{remote_node, cluster}, epoch} - current_revision = remote_cluster_epoch_revision(state.name, remote_node) + :ets.insert(remote_cluster_epochs_table(state.name), rows) - :ets.insert( - replication_meta_table(state.name), - {{:remote_epoch_revision, remote_node}, max(current_revision || revision, revision)} - ) + :ets.insert( + replication_meta_table(state.name), + {{:remote_epoch_revision, remote_node}, revision} + ) - {:reply, stale_epochs, state} + {:reply, {:ok, stale_epochs}, state} + else + {:reply, :stale, state} + end end def handle_call( - {:close_remote_cluster_epochs, shard, remote_node, revision, epochs}, + {:remote_registry_claim_authoritative, shard, remote_node, generation, cluster, epoch}, _from, state ) do - 0 = shard - observe_remote_cluster_revision(state.name, remote_node, revision, state.num_shards) + exact = remote_cluster_epoch_exact_revision(state.name, remote_node) + observed = remote_cluster_epoch_observed_revision(state.name, remote_node) + + current_view? = + case :ets.lookup( + replication_meta_table(state.name), + {:remote_view_info, shard, remote_node} + ) do + [{{:remote_view_info, ^shard, ^remote_node}, ^generation, ^exact, ^observed}] -> true + _ -> false + end - closed = - Enum.filter(epochs, fn {cluster, epoch} -> - remote_cluster_epoch(state.name, remote_node, cluster) == epoch - end) + authoritative? = + current_view? and remote_generation(state.name, remote_node) == generation and + remote_cluster_epoch_revision(state.name, remote_node) == observed and + remote_replica_authority_hint(state.name, remote_node) == {generation, observed} and + remote_cluster_epoch(state.name, remote_node, cluster) == epoch and + (is_nil(cluster) or active_local_cluster?(state.name, cluster)) - Enum.each(epochs, fn {cluster, epoch} -> - case :ets.lookup(remote_cluster_epochs_table(state.name), {remote_node, cluster}) do - [{{^remote_node, ^cluster}, ^epoch}] -> - :ets.delete(remote_cluster_epochs_table(state.name), {remote_node, cluster}) + {:reply, authoritative?, state} + end - _ -> - :ok - end - end) + def handle_call( + {:observe_remote_cluster_epoch_revision, remote_node, revision}, + _from, + state + ) do + :ok = observe_remote_cluster_revision(state.name, remote_node, revision, state.num_shards) + {:reply, :ok, state} + end - current_revision = remote_cluster_epoch_revision(state.name, remote_node) + def handle_call( + {:observe_remote_replica_hint, remote_node, generation, revision}, + _from, + state + ) do + known_generation = remote_generation(state.name, remote_node) + + {hint_generation, hint_revision} = + remote_replica_authority_hint(state.name, remote_node) || + {known_generation, remote_cluster_epoch_observed_revision(state.name, remote_node)} + + fenced? = + cond do + hint_generation == generation and + (is_nil(hint_revision) or revision > hint_revision) -> + put_remote_authority_hint(state.name, remote_node, generation, revision) + + if known_generation == generation do + :ok = + observe_remote_cluster_revision( + state.name, + remote_node, + revision, + state.num_shards + ) + end - :ets.insert( - replication_meta_table(state.name), - {{:remote_epoch_revision, remote_node}, max(current_revision || revision, revision)} - ) + true + + not is_nil(hint_generation) and + WireProtocol.generation_newer?(generation, hint_generation) -> + # This one shared row is the cross-lane fence. Public readers include + # it in replica_view_current?/2, so no lane can admit the prior + # generation after this insert even while the per-lane breadcrumbs + # below are being updated. + put_remote_authority_hint(state.name, remote_node, generation, revision) + + # Preserve each lane's old generation as the later exact install's + # purge key, but make its authority revisions impossible to match. + # This fences every shard in the same Data turn even when only one + # sideband lane observes the restarted origin first. + state.name + |> replication_meta_table() + |> :ets.match_object({{:remote_view_info, :_, remote_node}, :_, :_, :_}) + |> Enum.each(fn {key, lane_generation, _exact, _observed} -> + :ets.insert( + replication_meta_table(state.name), + {key, lane_generation, nil, nil} + ) + end) + + true + + true -> + false + end - {:reply, closed, state} + {:reply, fenced?, state} end def handle_call( - {:forget_remote_cluster_epochs, shard, remote_node, epochs}, + {:close_remote_cluster_epochs, shard, remote_node, generation, expected_revision, + revision, epochs}, _from, state ) do - # Kept for the rolling-compatibility receive path. The node-wide authority - # table is intentionally not mutated by a shard-local purge. - _ = {shard, remote_node, epochs} - {:reply, :ok, state} + 0 = shard + + if incremental_authority_installable?( + state.name, + remote_node, + generation, + expected_revision, + revision + ) do + observe_remote_cluster_revision(state.name, remote_node, revision, state.num_shards) + + closed = + Enum.filter(epochs, fn {cluster, epoch} -> + remote_cluster_epoch(state.name, remote_node, cluster) == epoch + end) + + Enum.each(epochs, fn {cluster, epoch} -> + case :ets.lookup(remote_cluster_epochs_table(state.name), {remote_node, cluster}) do + [{{^remote_node, ^cluster}, ^epoch}] -> + :ets.delete(remote_cluster_epochs_table(state.name), {remote_node, cluster}) + + _ -> + :ok + end + end) + + :ets.insert( + replication_meta_table(state.name), + {{:remote_epoch_revision, remote_node}, revision} + ) + + {:reply, {:ok, closed}, state} + else + {:reply, :stale, state} + end end def handle_call({:delete_remote_replica_info, shard, remote_node}, _from, state) do @@ -2122,29 +2652,34 @@ defmodule Group.Replica.Data do ) if shard == 0 do - :ets.delete(replication_meta_table(state.name), {:remote_generation, remote_node}) - :ets.delete(replication_meta_table(state.name), {:remote_epoch_revision, remote_node}) - :ets.delete(replication_meta_table(state.name), {:remote_epoch_exact, remote_node}) - :ets.delete(replication_meta_table(state.name), {:remote_epoch_observed, remote_node}) - :ets.delete(replication_meta_table(state.name), {:remote_authority_installs, remote_node}) - - if state.num_shards > 1 do - for view_shard <- 1..(state.num_shards - 1) do - :ets.delete( - replication_meta_table(state.name), - {:remote_view_info, view_shard, remote_node} - ) - end - end - - :ets.select_delete(remote_cluster_epochs_table(state.name), [ - {{{remote_node, :_}, :_}, [], [true]} - ]) + delete_remote_authority(state.name, remote_node) end {:reply, :ok, state} end + def handle_call({:expire_remote_replica_lane, shard, remote_node}, _from, state) do + :ets.delete( + replication_meta_table(state.name), + {:remote_view_info, shard, remote_node} + ) + + remaining_lanes = + :ets.select_count(replication_meta_table(state.name), [ + {{{:remote_view_info, :_, remote_node}, :_, :_, :_}, [], [true]} + ]) + + result = + if remaining_lanes == 0 do + delete_remote_authority(state.name, remote_node) + :node_retired + else + :lane_retired + end + + {:reply, result, state} + end + @impl true def init({name, num_shards}) do # ETS performance options: @@ -2200,7 +2735,7 @@ defmodule Group.Replica.Data do :ets.new(local_cluster_epochs_table(name), set_opts) :ets.new(closed_local_cluster_epochs_table(name), set_opts) :ets.new(remote_cluster_epochs_table(name), set_opts) - :ets.insert(replication_meta_table(name), {:generation, make_ref()}) + :ets.insert(replication_meta_table(name), {:generation, WireProtocol.new_generation()}) :ets.insert(replication_meta_table(name), {:cluster_epoch_revision, 0}) {:ok, %{name: name, num_shards: num_shards}} @@ -2214,6 +2749,102 @@ defmodule Group.Replica.Data do _ -> :ets.insert(replication_meta_table(name), {key, revision}) end + case remote_generation(name, remote_node) do + generation when not is_nil(generation) -> + put_remote_authority_hint(name, remote_node, generation, revision) + + nil -> + :ok + end + + :ok + end + + defp put_remote_authority_hint(name, remote_node, generation, revision) do + key = {:remote_authority_hint, remote_node} + + install? = + case remote_replica_authority_hint(name, remote_node) do + nil -> + true + + {^generation, current_revision} -> + revision > current_revision + + {current_generation, _current_revision} -> + WireProtocol.generation_newer?(generation, current_generation) + end + + if install? do + :ets.insert(replication_meta_table(name), {key, generation, revision}) + end + + :ok + end + + defp stale_remote_authority_install?(name, remote_node, generation, revision) do + known_generation = remote_generation(name, remote_node) + authoritative_revision = remote_cluster_epoch_revision(name, remote_node) + observed_revision = remote_cluster_epoch_observed_revision(name, remote_node) + + hinted_stale? = + case remote_replica_authority_hint(name, remote_node) do + nil -> + false + + {^generation, hinted_revision} -> + revision < hinted_revision + + {hinted_generation, _hinted_revision} -> + not WireProtocol.generation_newer?(generation, hinted_generation) + end + + known_stale? = + not is_nil(known_generation) and known_generation != generation and + not WireProtocol.generation_newer?(generation, known_generation) + + revision_stale? = + known_generation == generation and + Enum.any?([authoritative_revision, observed_revision], fn + current when is_integer(current) -> revision < current + _ -> false + end) + + hinted_stale? or known_stale? or revision_stale? + end + + defp incremental_authority_installable?( + name, + remote_node, + generation, + expected_revision, + revision + ) do + is_integer(expected_revision) and is_integer(revision) and revision > expected_revision and + remote_generation(name, remote_node) == generation and + remote_cluster_epoch_revision(name, remote_node) == expected_revision and + remote_cluster_epoch_observed_revision(name, remote_node) == expected_revision and + remote_replica_authority_hint(name, remote_node) == {generation, expected_revision} + end + + defp delete_remote_authority(name, remote_node) do + # Fence every public-ETS reader first. The remaining mutations execute in + # this same Data callback, so a caller cannot observe completion between + # them; if Data itself dies, it takes every owned table with it. Removing + # the generation before routing therefore closes the read interleaving + # where a sibling lane could still accept data during retirement. + :ets.delete(replication_meta_table(name), {:remote_generation, remote_node}) + :ok = delete_peer_routes(name, remote_node) + :ets.delete(replication_meta_table(name), {:remote_epoch_revision, remote_node}) + :ets.delete(replication_meta_table(name), {:remote_epoch_exact, remote_node}) + :ets.delete(replication_meta_table(name), {:remote_epoch_observed, remote_node}) + :ets.delete(replication_meta_table(name), {:remote_authority_hint, remote_node}) + :ets.delete(replication_meta_table(name), {:remote_authority_installs, remote_node}) + + :ets.select_delete(remote_cluster_epochs_table(name), [ + {{{remote_node, :_}, :_}, [], [true]} + ]) + :ok end diff --git a/lib/group/replica/wire_protocol.ex b/lib/group/replica/wire_protocol.ex index b8e1908..3bb4aeb 100644 --- a/lib/group/replica/wire_protocol.ex +++ b/lib/group/replica/wire_protocol.ex @@ -5,10 +5,38 @@ defmodule Group.Replica.WireProtocol do def version, do: @version + # The counter orders Group incarnations created within one BEAM. The ref + # keeps the identity globally unique across BEAM restarts; Erlang + # distribution supplies nodedown before a restarted VM can install a new + # authority, so ordering is only required within one VM lifetime. + def new_generation do + {System.unique_integer([:monotonic, :positive]), make_ref()} + end + + def valid_generation?({counter, identity}), + do: is_integer(counter) and counter > 0 and is_reference(identity) + + def valid_generation?(_generation), do: false + + def generation_newer?({new_counter, _new_identity}, {old_counter, _old_identity}) + when is_integer(new_counter) and is_integer(old_counter), + do: new_counter > old_counter + + def generation_newer?(_new_generation, _old_generation), do: false + def stream_id(name, origin_node, origin_generation, shard, cluster, cluster_epoch) do {name, origin_node, origin_generation, shard, cluster, cluster_epoch} end + def valid_stream_id?({name, origin_node, origin_generation, shard, cluster, cluster_epoch}) do + is_atom(name) and is_atom(origin_node) and valid_generation?(origin_generation) and + is_integer(shard) and shard >= 0 and + ((is_nil(cluster) and cluster_epoch == origin_generation) or + (is_binary(cluster) and is_reference(cluster_epoch))) + end + + def valid_stream_id?(_stream_id), do: false + def stream_name({name, _origin_node, _generation, _shard, _cluster, _epoch}), do: name def stream_origin({_name, origin_node, _generation, _shard, _cluster, _epoch}), diff --git a/lib/group/transport.ex b/lib/group/transport.ex index 8390879..dd1164e 100644 --- a/lib/group/transport.ex +++ b/lib/group/transport.ex @@ -10,9 +10,8 @@ defmodule Group.Transport do Erlang distribution remains Group's control plane and supplies the stable node identity used here. A sideband adapter can use its `descriptor/2` in the control hello to exchange endpoints and pass incoming messages to - `incoming/4` or `incoming_batch/4`. Group trusts the `source_node` supplied - by the adapter and validates stream origins and member pids against it; peer - authentication, when needed, belongs to the transport. + `incoming/4` or `incoming_batch/4`. `source_node` is routing metadata supplied + by the adapter; Group uses it to validate stream origins and member pids. Adapters do not need to preserve ordering. Group serializes writes per shard and sequences each origin/generation/shard/cluster/epoch stream; receivers @@ -74,9 +73,9 @@ defmodule Group.Transport do @doc """ Passes an incoming replica message to the corresponding local shard. - This is a local mailbox operation. `source_node` is the trusted peer identity - established by the adapter. Stream generation, epoch, group, shard, origin, - and member-pid ownership are validated by the replica. + This is a local mailbox operation. `source_node` identifies the peer whose + replica lane supplied the message. Stream generation, epoch, group, shard, + origin, and member-pid ownership are validated by the replica. Returns `:disconnected` and drops the message if that shard is not currently registered, for example while its supervisor is restarting. @@ -98,7 +97,7 @@ defmodule Group.Transport do A finite-message transport may segment the encoded batch on the wire, but it must reassemble every segment before calling this function. Group never - observes or applies a partial batch. `source_node` has the same trusted-peer + observes or applies a partial batch. `source_node` has the same routing meaning as in `incoming/4`. Like `incoming/4`, this returns `:disconnected` if the destination shard is diff --git a/lib/group/transport/outbox.ex b/lib/group/transport/outbox.ex index 567a6e0..4bd8d55 100644 --- a/lib/group/transport/outbox.ex +++ b/lib/group/transport/outbox.ex @@ -37,8 +37,10 @@ defmodule Group.Transport.Outbox do default `1` * `:outbox_deadline` - maximum useful residence time for an outgoing message in milliseconds, default `100` + * `:outbox_max_messages` - maximum accepted messages across the local + mailbox, pending batch, and backend send, default `1_024` - The deadline bounds stale work, not mailbox memory. A backend must also put a + Admission bounds mailbox growth by message count. A backend must also put a finite bound on every socket enqueue or write it performs. Exact snapshot messages are independently bounded by `:replicated_snapshot_chunk_target_bytes`. Other logical messages or a whole batch may still exceed `:outbox_batch_bytes`; @@ -91,11 +93,21 @@ defmodule Group.Transport.Outbox do def push(group, target_node, shard, message, opts) when is_atom(group) and is_atom(target_node) and is_integer(shard) and shard >= 0 and is_list(opts) do - case Process.whereis(name(group, shard)) do - pid when is_pid(pid) -> - deadline = monotonic_ms() + deadline(opts) - send(pid, {:group_replica_outbox_push, target_node, deadline, message}) - :ok + expires_at = monotonic_ms() + deadline(opts) + + case :persistent_term.get(admission_key(group, shard), nil) do + {pid, admission, max_messages} when is_pid(pid) -> + if Process.whereis(name(group, shard)) == pid do + if :atomics.add_get(admission, 1, 1) <= max_messages do + send(pid, {:group_replica_outbox_push, target_node, expires_at, message}) + :ok + else + :atomics.sub(admission, 1, 1) + :busy + end + else + :disconnected + end nil -> :disconnected @@ -105,6 +117,9 @@ defmodule Group.Transport.Outbox do @doc false def name(group, shard), do: :"#{group}_replica_transport_outbox_#{shard}" + @doc false + def admission_key(group, shard), do: {__MODULE__, group, shard, :admission} + @doc false def monotonic_ms, do: System.monotonic_time(:millisecond) @@ -113,6 +128,7 @@ defmodule Group.Transport.Outbox do deadline(opts) positive_opt(opts, :outbox_batch_size, 64) positive_opt(opts, :outbox_batch_bytes, 1_048_576) + positive_opt(opts, :outbox_max_messages, 1_024) non_negative_opt(opts, :outbox_flush_interval, 1) :ok end @@ -194,6 +210,7 @@ defmodule Group.Transport.Outbox.Worker do @default_batch_size 64 @default_batch_bytes 1_048_576 @default_flush_interval 1 + @default_max_messages 1_024 def start_link(opts, shard) do group = Keyword.fetch!(opts, :name) @@ -206,6 +223,9 @@ defmodule Group.Transport.Outbox.Worker do group = Keyword.fetch!(opts, :name) backend = Keyword.fetch!(opts, :backend) {:ok, backend_state} = backend.init_outbox(group, shard, opts) + admission = :atomics.new(1, signed: false) + max_messages = positive_opt(opts, :outbox_max_messages, @default_max_messages) + :persistent_term.put(Outbox.admission_key(group, shard), {self(), admission, max_messages}) {:ok, %{ @@ -213,6 +233,7 @@ defmodule Group.Transport.Outbox.Worker do shard: shard, backend: backend, backend_state: backend_state, + admission: admission, batch_size: positive_opt(opts, :outbox_batch_size, @default_batch_size), batch_bytes: positive_opt(opts, :outbox_batch_bytes, @default_batch_bytes), flush_interval: non_negative_opt(opts, :outbox_flush_interval, @default_flush_interval), @@ -223,11 +244,26 @@ defmodule Group.Transport.Outbox.Worker do }} end + @impl true + def terminate(_reason, state) do + key = Outbox.admission_key(state.group, state.shard) + + case :persistent_term.get(key, nil) do + {pid, _admission, _max_messages} when pid == self() -> + :persistent_term.erase(key) + + _ -> + :ok + end + + :ok + end + @impl true def handle_info({:group_replica_outbox_push, target_node, deadline, message}, state) when is_atom(target_node) and is_integer(deadline) do if deadline <= Outbox.monotonic_ms() do - {:noreply, state} + {:noreply, release_admission(state, 1)} else bytes = :erlang.external_size({target_node, message}) @@ -288,6 +324,7 @@ defmodule Group.Transport.Outbox.Worker do defp flush(%{pending_count: 0} = state), do: cancel_flush(state) defp flush(state) do + released = state.pending_count state = cancel_flush(state) now = Outbox.monotonic_ms() @@ -319,13 +356,20 @@ defmodule Group.Transport.Outbox.Worker do end end) - %{ + state = %{ state | backend_state: backend_state, pending: [], pending_count: 0, pending_bytes: 0 } + + release_admission(state, released) + end + + defp release_admission(state, count) do + :atomics.sub(state.admission, 1, count) + state end defp cancel_flush(%{flush_ref: nil} = state), do: state diff --git a/test/README.md b/test/README.md index 394eff7..d69c195 100644 --- a/test/README.md +++ b/test/README.md @@ -25,7 +25,7 @@ release qualification rather than individual edits. |------|---------------| | `group_test.exs` | Single-node: register/unregister, join/leave, members, monitor/demonitor, named clusters, concurrent operations | | `distributed_test.exs` | Multi-node: replication, peer discovery, node disconnect cleanup, partition healing, conflict resolution, event ordering, rolling restarts, and adversarial replica-transport loss/busy/snapshot recovery | -| `anti_entropy_fault_regression_test.exs` | Three-node regressions for hidden-winner projection, crash-journal replay, claimless cluster cleanup, shard-zero view repair, and shard-scoped sideband lifecycle | +| `anti_entropy_fault_regression_test.exs` | Three-node regressions for hidden-winner projection, receiver restart eviction, nodedown/lease lane retirement, authority gaps and cross-lane races, in-flight conflict fencing, crash-journal replay, cursorless/interrupted snapshot repair, malformed ingress, and sideband rediscovery | | `replica_adversarial_test.exs` | Reproducible three-node mixed-operation state machines: drops, busy returns, duplication, reordering, bounded delay, oplog pruning, conflicts, owner death, and named-cluster epoch churn, followed by exact convergence/dead-owner/internal-index checks | | `replica_model_property_test.exs` | StreamData-generated and shrunk owner histories against an independent lifecycle oracle and scheduler-controlled replica transport | | `replica_snapshot_test.exs` | Pure byte partitioning and set-valued private-ETS snapshot staging | @@ -247,8 +247,9 @@ TestCluster.start_group( ) ``` -The resolver uses "most recent wins" — keeps the registration with the higher -timestamp. +The resolver ranks each claim by timestamp. Group chooses the maximum +`{rank, pid}`, so every peer reaches the same winner regardless of claim arrival +order. ### Replica transport fault injection @@ -272,7 +273,8 @@ losing its cluster-close fence and require the lane to sweep the stale registry and PG slices from shared authority. Authority topology tests suspend every receiver shard and inspect the queued protocol: only shard 0 may receive/install the full epoch snapshot, nonzero shards receive constant-size lane hellos, and -incremental opens stay on their matching shard. Separate tests suspend a +incremental controls arriving on any lane are serialized through shard 0. +Separate tests suspend a backlogged authority shard while other replica lanes continue converging and deliver data before authority to prove rejection does not advance the cursor and the same message applies after authority repair. Concurrent snapshot tests @@ -285,17 +287,49 @@ disconnects one origin's real socket, prunes its oplog, reconnects it, and requires snapshot recovery without changing the third node's independent registry or PG state. +Authority races also replace authority while conflict resolution is paused, +remove the local shard-zero owner, and deliver `nodedown` while a sibling lane +is suspended. The assertions require that stale remote winners cannot terminate +local owners, that retained conflict claims are reprojected after an authority +gap closes even when their stream cursor is already current, and that each lane +keeps its own restart breadcrumb until its rows and cursor are gone. +The same conflict-gap schedule is held open until lease expiry to prove a source +that never returns leaves no deferred projection key, claim, cursor, or visible +remote winner behind. A separate nodedown regression proves the immediate +retirement path cannot strand that deferred state after removing its lease. +Newer-generation and same-generation heartbeat schedules assert the persisted +hint fences every lane before exact repair, rejects delayed exact/incremental +authority, and reconstructs its retirement deadline after a lane crash. Once +the peer is fully retired, replayed heartbeats and lane hellos must create no +hint, lease, or outbound route. A separate compare-and-install regression races +a newer hint across an older incremental control and requires no epoch row or +lane view to become authoritative. +The inverse ordering is forced independently: a first-time lane hello is +processed while shard zero is suspended, then exact authority must trigger an +immediate shard-local re-probe without first admitting the speculative route. +Exact-authority regression also activates a local cluster alongside a remote +epoch install and requires the authority plus both cluster indexes to become +visible as one operation; the high-volume control test repeats the projection +check over three nodes before admitting replica writes. +Interrupted lifecycle tests suspend the notification shard, stop after the +durable activation/deactivation mutation, and require activation routing to be +complete immediately and close cleanup to finish after the shard resumes. This +proves neither path relies on the API caller remaining alive. + `replica_transport_outbox_test.exs` proves that a blocked sideband backend cannot delay the Group-facing local push, messages expire behind that backend, busy batches are not retried locally, batching preserves per-target order, invalid deadlines fail at boot, and ingress drops rather than raising while a -destination shard is absent. +destination shard is absent. Admission is bounded across the local mailbox, +pending batch, and backend send. The real three-node TCP recovery test runs through the same outbox path. `Group.TestCluster.assert_replica_consistent/1` checks the -public dual indexes plus registry claim authority, oplog/order equivalence, and -contiguous retained stream ranges. Seeded tests additionally require every PID -retained as authority to still be alive after convergence. +public dual indexes plus exact deterministic registry projection, row/shard +placement, lane authority revisions, registry/PG origin authority, +oplog/order equivalence, and contiguous retained stream ranges. Seeded tests +additionally require every PID retained as authority to still be alive after +convergence. The isolated mutation runner in `test/mutation/` disables individual protocol guards and repair steps only in copied checkouts. See diff --git a/test/anti_entropy_fault_regression_test.exs b/test/anti_entropy_fault_regression_test.exs index fb17b4f..68c41eb 100644 --- a/test/anti_entropy_fault_regression_test.exs +++ b/test/anti_entropy_fault_regression_test.exs @@ -13,263 +13,3805 @@ defmodule Group.AntiEntropyFaultRegressionTest do {:ok, peers: peers, node_a: node_a, node_b: node_b, node_c: node_c} end - test "local unregister promotes a retained remote claim after its delta is pruned", context do - %{name: name, pid_a: pid_a, pid_b: pid_b, stream_b: stream_b} = - establish_hidden_remote_claim(context, :unregister) + test "a receiver shard restart purges a permanently disappeared Group", context do + name = unique_name(:receiver_restart_permanent_loss) - {floor, head, _applied} = - TestCluster.rpc!(context.node_b, Group.Replica.Data, :replica_stream_head, [ + opts = [ + name: name, + shards: 2, + replicated_sender_buffer_size: 1, + replicated_pg_receiver_buffer_size: 1, + replicated_registry_receiver_buffer_size: 1, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 150 + ] + + start_group_on_peers(context.peers, opts) + + registry_key = "receiver-restart/registry" + pg_key = "receiver-restart/pg" + + owner = + TestCluster.spawn_register_and_join( + context.node_a, name, - 0, - stream_b - ]) + registry_key, + %{owner: :a}, + pg_key, + %{owner: :a} + ) - assert floor > 1 + for receiver <- [context.node_b, context.node_c] do + TestCluster.assert_eventually(fn -> + match?( + {^owner, %{owner: :a}}, + TestCluster.rpc!(receiver, Group, :lookup, [name, registry_key]) + ) and + match?( + [{^owner, %{owner: :a}}], + TestCluster.rpc!(receiver, Group, :members, [name, pg_key]) + ) + end) + end - assert TestCluster.rpc!(context.node_a, Group.Replica.Data, :replica_cursor, [ - name, - 0, - stream_b - ]) == head + old_receiver = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) - assert :ok = TestCluster.unregister_owner(pid_a) - TestCluster.flush_shards(context.node_a, name) + :ok = TestCluster.rpc!(context.node_b, :sys, :suspend, [old_receiver]) - assert TestCluster.rpc!(context.node_a, Group.Replica.Data, :registry_claims, [ - name, - 0, - nil, - "hidden/unregister" - ]) - |> Enum.map(&elem(&1, 0)) == [pid_b] + # A's BEAM intentionally remains connected. Only its Group disappears, so + # B must learn retirement from the lease it misses while its shard is down. + supervisor = TestCluster.rpc!(context.node_a, Process, :whereis, [:"#{name}_group_sup"]) + :ok = TestCluster.rpc!(context.node_a, Supervisor, :stop, [supervisor, :normal, 5_000]) TestCluster.assert_eventually( fn -> - match?( - {^pid_b, %{rank: 1}}, - TestCluster.rpc!(context.node_a, Group, :lookup, [name, "hidden/unregister"]) - ) + TestCluster.rpc!(context.node_c, Group, :lookup, [name, registry_key]) == nil and + TestCluster.rpc!(context.node_c, Group, :members, [name, pg_key]) == [] and + context.node_a not in TestCluster.rpc!(context.node_c, Group, :nodes, [name]) end, - timeout: 500, + timeout: 5_000 + ) + + Process.sleep(250) + true = TestCluster.rpc!(context.node_b, Process, :exit, [old_receiver, :kill]) + + TestCluster.assert_eventually(fn -> + case TestCluster.rpc!(context.node_b, Process, :whereis, [ + Group.Replica.shard_name(name, 0) + ]) do + pid when is_pid(pid) -> pid != old_receiver + _ -> false + end + end) + + # No peer is allowed to resurrect A or supply a later delete. Correctness + # therefore depends entirely on restart-time authority reconciliation. + TestCluster.assert_eventually( + fn -> + TestCluster.rpc!(context.node_b, Group, :lookup, [name, registry_key]) == nil and + TestCluster.rpc!(context.node_b, Group, :members, [name, pg_key]) == [] and + context.node_a not in TestCluster.rpc!(context.node_b, Group, :nodes, [name]) + end, + timeout: 1_000, interval: 25 ) - for node <- [context.node_a, context.node_b, context.node_c] do - assert :ok = TestCluster.rpc!(node, TestCluster, :assert_replica_consistent, [name]) - end + assert :ok = + TestCluster.rpc!(context.node_b, TestCluster, :assert_replica_origin_purged, [ + name, + context.node_a + ]) + + assert :ok = + TestCluster.rpc!(context.node_b, TestCluster, :assert_replica_consistent, [name]) end - test "journal replay promotes a retained remote claim before marking the delete applied", + test "a sibling restart cannot lose the retirement lease after shard zero expires a peer", context do - %{name: name, pid_a: pid_a, pid_b: pid_b} = - establish_hidden_remote_claim(context, :journal) + name = unique_name(:sibling_restart_during_peer_expiry) - stream_a = - TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_stream_id, [name, 0, nil]) + opts = [ + name: name, + shards: 2, + replicated_sender_buffer_size: 1, + replicated_pg_receiver_buffer_size: 1, + replicated_registry_receiver_buffer_size: 1, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 150 + ] - {seq, _mutations} = - TestCluster.rpc!(context.node_a, Group.Replica.Data, :append_replica_record, [ + start_group_on_peers(context.peers, opts) + + key = + 1..1_000 + |> Enum.map(&"sibling-expiry/#{&1}") + |> Enum.find(&(Group.Replica.shard_index_for(nil, &1, 2) == 1)) + + owner = + TestCluster.spawn_register_and_join( + context.node_a, name, - 0, - stream_a, - [{:unregister, nil, "hidden/journal", pid_a, %{rank: 2}, :injected_crash}] - ]) + key, + %{owner: :a}, + key, + %{owner: :a} + ) - assert [{^stream_a, ^seq, _}] = - TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_replica_unapplied, [ + for receiver <- [context.node_b, context.node_c] do + TestCluster.assert_eventually(fn -> + match?({^owner, %{owner: :a}}, TestCluster.rpc!(receiver, Group, :lookup, [name, key])) and + match?( + [{^owner, %{owner: :a}}], + TestCluster.rpc!(receiver, Group, :members, [name, key]) + ) + end) + end + + old_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + :ok = TestCluster.rpc!(context.node_b, :sys, :suspend, [old_lane]) + + supervisor = TestCluster.rpc!(context.node_a, Process, :whereis, [:"#{name}_group_sup"]) + :ok = TestCluster.rpc!(context.node_a, Supervisor, :stop, [supervisor, :normal, 5_000]) + + # Shard zero retires only its lane while the suspended sibling remains a + # live restart breadcrumb. Shared authority must survive until that final + # lane performs its own bounded retirement. + TestCluster.assert_eventually( + fn -> + is_nil( + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_view_generation, [ + name, + 0, + context.node_a + ]) + ) + end, + timeout: 5_000 + ) + + refute is_nil( + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_generation, [ name, - 0 + context.node_a ]) + ) - old_shard = - TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + # The lane view is its restart breadcrumb. Shard zero must not erase it on + # behalf of a lane that has not purged its own retained rows and cursor. + refute is_nil( + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_view_generation, + [name, 1, context.node_a] + ) + ) - true = TestCluster.rpc!(context.node_a, Process, :exit, [old_shard, :kill]) + assert {^owner, %{owner: :a}} = + TestCluster.rpc!(context.node_b, Group, :lookup, [name, key]) + + true = TestCluster.rpc!(context.node_b, Process, :exit, [old_lane, :kill]) TestCluster.assert_eventually(fn -> - case TestCluster.rpc!(context.node_a, Process, :whereis, [ - Group.Replica.shard_name(name, 0) + case TestCluster.rpc!(context.node_b, Process, :whereis, [ + Group.Replica.shard_name(name, 1) ]) do - pid when is_pid(pid) -> pid != old_shard - nil -> false + pid when is_pid(pid) -> pid != old_lane + _ -> false end end) - TestCluster.flush_shards(context.node_a, name) + TestCluster.assert_eventually( + fn -> + TestCluster.rpc!(context.node_b, Group, :lookup, [name, key]) == nil and + TestCluster.rpc!(context.node_b, Group, :members, [name, key]) == [] + end, + timeout: 1_000, + interval: 25 + ) - claims = - TestCluster.rpc!(context.node_a, Group.Replica.Data, :registry_claims, [ + # Restart repair can remove invalid rows before the reconstructed lease + # retires its final constant-size view breadcrumb. + TestCluster.assert_eventually( + fn -> + is_nil( + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_view_generation, + [name, 1, context.node_a] + ) + ) + end, + timeout: 1_000, + interval: 25 + ) + + assert :ok = + TestCluster.rpc!(context.node_b, TestCluster, :assert_replica_origin_purged, [ + name, + context.node_a + ]) + end + + test "nodedown on shard zero preserves a suspended sibling's restart breadcrumb", context do + name = unique_name(:sibling_restart_during_nodedown) + + opts = [ + name: name, + shards: 2, + replicated_sender_buffer_size: 1, + replicated_pg_receiver_buffer_size: 1, + replicated_registry_receiver_buffer_size: 1, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 150 + ] + + start_group_on_peers(context.peers, opts) + + key = + 1..1_000 + |> Enum.map(&"sibling-nodedown/#{&1}") + |> Enum.find(&(Group.Replica.shard_index_for(nil, &1, 2) == 1)) + + owner = + TestCluster.spawn_register_and_join( + context.node_a, name, - 0, - nil, - "hidden/journal" - ]) + key, + %{owner: :a}, + key, + %{owner: :a} + ) - journal_state = %{ - claims: claims, - head: - TestCluster.rpc!(context.node_a, Group.Replica.Data, :replica_stream_head, [ - name, - 0, - stream_a - ]), - unapplied: - TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_replica_unapplied, [ + TestCluster.assert_eventually(fn -> + match?({^owner, _}, TestCluster.rpc!(context.node_b, Group, :lookup, [name, key])) and + match?([{^owner, _}], TestCluster.rpc!(context.node_b, Group, :members, [name, key])) + end) + + old_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + :ok = TestCluster.rpc!(context.node_b, :sys, :suspend, [old_lane]) + + on_exit(fn -> + TestCluster.reconnect_nodes(context.node_b, context.node_a) + end) + + TestCluster.disconnect_nodes(context.node_b, context.node_a) + + TestCluster.assert_eventually(fn -> + is_nil( + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_generation, [ name, - 0 + context.node_a ]) - } + ) + end) + + # Shard zero owns node-wide authority, but it must not erase another lane's + # durable view before that lane has purged its own rows and cursor. + refute is_nil( + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_view_generation, + [name, 1, context.node_a] + ) + ) + + assert {^owner, %{owner: :a}} = + TestCluster.rpc!(context.node_b, Group, :lookup, [name, key]) + + true = TestCluster.rpc!(context.node_b, Process, :exit, [old_lane, :kill]) + + TestCluster.assert_eventually(fn -> + case TestCluster.rpc!(context.node_b, Process, :whereis, [ + Group.Replica.shard_name(name, 1) + ]) do + pid when is_pid(pid) -> pid != old_lane + _ -> false + end + end) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(context.node_b, Group, :lookup, [name, key]) == nil and + TestCluster.rpc!(context.node_b, Group, :members, [name, key]) == [] + end) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :registry_claims, [ + name, + 1, + nil, + key + ]) == [] + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :pg_entries_for_origin, [ + name, + 1, + nil, + context.node_a + ]) == [] + + assert TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :replica_cursor_streams_for_origin, + [name, 1, context.node_a] + ) == [] + + assert :ok = + TestCluster.rpc!(context.node_b, TestCluster, :assert_replica_consistent, [name]) + end + + test "shard zero restart retains the obligation to repair partially observed authority", + context do + name = unique_name(:authority_dirty_restart) + cluster = "authority-dirty/restart" + + opts = [ + name: name, + shards: 1, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + exact_before = + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_exact_revision, + [name, context.node_a] + ) + + :ok = TestCluster.rpc!(context.node_a, Group, :connect, [name, cluster]) + + TestCluster.assert_eventually(fn -> + observed = + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_observed_revision, + [name, context.node_a] + ) + + exact = + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_exact_revision, + [name, context.node_a] + ) + + is_integer(observed) and observed > exact_before and exact == exact_before + end) + + source = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + :ok = TestCluster.rpc!(context.node_a, :sys, :suspend, [source]) + + on_exit(fn -> + _ = TestCluster.resume_shard_if_alive(context.node_a, name, 0) + end) + + old_receiver = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + true = TestCluster.rpc!(context.node_b, Process, :exit, [old_receiver, :kill]) + + TestCluster.assert_eventually(fn -> + case TestCluster.rpc!(context.node_b, Process, :whereis, [ + Group.Replica.shard_name(name, 0) + ]) do + pid when is_pid(pid) -> pid != old_receiver + _ -> false + end + end) + + receiver = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + state = TestCluster.rpc!(context.node_b, :sys, :get_state, [receiver]) + assert Map.has_key?(state.cluster_control_dirty, context.node_a) + + :ok = TestCluster.rpc!(context.node_a, :sys, :resume, [source]) + + TestCluster.assert_eventually( + fn -> + exact = + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_exact_revision, + [name, context.node_a] + ) + + observed = + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_observed_revision, + [name, context.node_a] + ) + + exact == observed and exact > exact_before + end, + timeout: 5_000 + ) + end + + test "malformed replica frames are rejected without restarting the receiving shard", context do + name = unique_name(:malformed_replica_frames) + + opts = [ + name: name, + shards: 1, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 5_000 + ] + + start_group_on_peers(context.peers, opts) + + receiver = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + stream_id = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_stream_id, [name, 0, nil]) + + version = Group.Replica.WireProtocol.version() + + frames = [ + {:heads, version, [:not_a_head]}, + {:delta_batch, version, [:not_a_delta_run]}, + {:need, version, :not_a_stream, 1}, + {:needs, version, [:not_a_need]}, + {:snapshot_chunk, version, :not_a_stream, 1, 1, 1, 0, 0, [], []}, + {:delta_batch, version, + [ + {stream_id, 1, + [{1, [{:register, nil, "malformed/pid", :not_a_pid, %{}, 0, context.node_a}]}], 1} + ]} + ] + + for frame <- frames do + assert :ok = + TestCluster.rpc!(context.node_b, Group.Transport, :incoming, [ + name, + context.node_a, + 0, + frame + ]) + + Process.sleep(25) + + assert TestCluster.rpc!(context.node_b, Process, :alive?, [receiver]), + "receiver restarted after malformed frame: #{inspect(frame)}" + end + + assert receiver == + TestCluster.rpc!(context.node_b, Process, :whereis, [ + Group.Replica.shard_name(name, 0) + ]) + + assert nil == + TestCluster.rpc!(context.node_b, Group, :lookup, [name, "malformed/pid"]) + + assert :ok = + TestCluster.rpc!(context.node_b, TestCluster, :assert_replica_consistent, [name]) + end + + test "a delta cannot place registry or PG rows on the wrong shard", context do + name = unique_name(:wrong_shard_delta) + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 1, + replicated_pg_receiver_buffer_size: 1, + replicated_registry_receiver_buffer_size: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + wrong_key = + 1..1_000 + |> Enum.map(&"wrong-shard/#{&1}") + |> Enum.find(&(Group.Replica.shard_index_for(nil, &1, 2) == 1)) + + source_key = + 1..1_000 + |> Enum.map(&"source-shard/#{&1}") + |> Enum.find(&(Group.Replica.shard_index_for(nil, &1, 2) == 0)) + + :ok = + TestCluster.rpc!(context.node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + owner = TestCluster.spawn_register(context.node_a, name, source_key, %{owner: :a}) + + stream_id = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_stream_id, [name, 0, nil]) + + frame = + {:delta_batch, Group.Replica.WireProtocol.version(), + [ + {stream_id, 1, + [ + {1, + [ + {:register, nil, wrong_key, owner, %{owner: :a}, 1, context.node_a}, + {:join, nil, wrong_key, owner, %{owner: :a}, 1, :join, context.node_a} + ]} + ], 1} + ]} + + assert :ok = + TestCluster.rpc!(context.node_b, Group.Transport, :incoming, [ + name, + context.node_a, + 0, + frame + ]) + + TestCluster.flush_shards(context.node_b, name) + + assert nil == TestCluster.rpc!(context.node_b, Group, :lookup, [name, wrong_key]) + assert [] == TestCluster.rpc!(context.node_b, Group, :members, [name, wrong_key]) + + assert 0 == + TestCluster.rpc!(context.node_b, Group.Replica.Data, :replica_cursor, [ + name, + 0, + stream_id + ]) + + assert :ok = + TestCluster.rpc!(context.node_b, TestCluster, :assert_replica_consistent, [name]) + end + + test "full snapshot capture never runs on the replica shard", context do + name = unique_name(:snapshot_capture_isolation) + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 1, + replicated_oplog_max_entries: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + for index <- 1..4 do + owner = TestCluster.spawn_register(context.node_a, name, "snapshot/isolation/#{index}", %{}) + true = TestCluster.rpc!(context.node_a, Process, :exit, [owner, :kill]) + end + + TestCluster.flush_shards(context.node_a, name) + + stream_id = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_stream_id, [name, 0, nil]) + + {floor, _head, _applied} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :replica_stream_head, [ + name, + 0, + stream_id + ]) + + assert floor > 1 + + shard = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + tracer = TestCluster.spawn_trace_forwarder(context.node_a, self()) + + assert 1 == + TestCluster.rpc!(context.node_a, :erlang, :trace_pattern, [ + {Group.Replica.Data, :registry_claims_for_stream, 3}, + true, + [:local] + ]) + + assert 1 == + TestCluster.rpc!(context.node_a, :erlang, :trace, [ + shard, + true, + [:call, :set_on_spawn, {:tracer, tracer}] + ]) + + on_exit(fn -> + TestCluster.rpc!(context.node_a, :erlang, :trace, [shard, false, [:all]]) + + TestCluster.rpc!(context.node_a, :erlang, :trace_pattern, [ + {Group.Replica.Data, :registry_claims_for_stream, 3}, + false, + [:local] + ]) + end) + + assert :ok = + TestCluster.rpc!(context.node_a, Group.Transport, :incoming, [ + name, + context.node_b, + 0, + {:needs, Group.Replica.WireProtocol.version(), [{stream_id, 1}]} + ]) + + assert_receive {:forwarded_trace, + {:trace, capture_pid, :call, + {Group.Replica.Data, :registry_claims_for_stream, [^name, 0, ^stream_id]}}}, + 1_000 + + refute capture_pid == shard + assert TestCluster.rpc!(context.node_a, Process, :alive?, [shard]) + end + + test "local unregister promotes a retained remote claim after its delta is pruned", context do + %{name: name, pid_a: pid_a, pid_b: pid_b, stream_b: stream_b} = + establish_hidden_remote_claim(context, :unregister) + + {floor, head, _applied} = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :replica_stream_head, [ + name, + 0, + stream_b + ]) + + assert floor > 1 + + assert TestCluster.rpc!(context.node_a, Group.Replica.Data, :replica_cursor, [ + name, + 0, + stream_b + ]) == head + + assert :ok = TestCluster.unregister_owner(pid_a) + TestCluster.flush_shards(context.node_a, name) + + assert TestCluster.rpc!(context.node_a, Group.Replica.Data, :registry_claims, [ + name, + 0, + nil, + "hidden/unregister" + ]) + |> Enum.map(&elem(&1, 0)) == [pid_b] + + TestCluster.assert_eventually( + fn -> + match?( + {^pid_b, %{rank: 1}}, + TestCluster.rpc!(context.node_a, Group, :lookup, [name, "hidden/unregister"]) + ) + end, + timeout: 500, + interval: 25 + ) + + for node <- [context.node_a, context.node_b, context.node_c] do + assert :ok = TestCluster.rpc!(node, TestCluster, :assert_replica_consistent, [name]) + end + end + + test "journal replay promotes a retained remote claim before marking the delete applied", + context do + %{name: name, pid_a: pid_a, pid_b: pid_b} = + establish_hidden_remote_claim(context, :journal) + + stream_a = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_stream_id, [name, 0, nil]) + + {seq, _mutations} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :append_replica_record, [ + name, + 0, + stream_a, + [{:unregister, nil, "hidden/journal", pid_a, %{rank: 2}, :injected_crash}] + ]) + + assert [{^stream_a, ^seq, _}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_replica_unapplied, [ + name, + 0 + ]) + + old_shard = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + true = TestCluster.rpc!(context.node_a, Process, :exit, [old_shard, :kill]) + + TestCluster.assert_eventually(fn -> + case TestCluster.rpc!(context.node_a, Process, :whereis, [ + Group.Replica.shard_name(name, 0) + ]) do + pid when is_pid(pid) -> pid != old_shard + nil -> false + end + end) + + TestCluster.flush_shards(context.node_a, name) + + claims = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :registry_claims, [ + name, + 0, + nil, + "hidden/journal" + ]) + + journal_state = %{ + claims: claims, + head: + TestCluster.rpc!(context.node_a, Group.Replica.Data, :replica_stream_head, [ + name, + 0, + stream_a + ]), + unapplied: + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_replica_unapplied, [ + name, + 0 + ]) + } + + assert Enum.map(claims, &elem(&1, 0)) == [pid_b], inspect(journal_state) + + assert {^pid_b, %{rank: 1}} = + TestCluster.rpc!(context.node_a, Group, :lookup, [name, "hidden/journal"]) + + assert [] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_replica_unapplied, [ + name, + 0 + ]) + + assert :ok = + TestCluster.rpc!(context.node_a, TestCluster, :assert_replica_consistent, [name]) + end + + test "local process DOWN promotes a retained remote claim", context do + %{name: name, pid_a: pid_a, pid_b: pid_b} = + establish_hidden_remote_claim(context, :process_down) + + true = TestCluster.rpc!(context.node_a, Process, :exit, [pid_a, :kill]) + TestCluster.flush_shards(context.node_a, name) + + assert [^pid_b] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :registry_claims, [ + name, + 0, + nil, + "hidden/process_down" + ]) + |> Enum.map(&elem(&1, 0)) + + assert {^pid_b, %{rank: 1}} = + TestCluster.rpc!(context.node_a, Group, :lookup, [name, "hidden/process_down"]) + + assert :ok = + TestCluster.rpc!(context.node_a, TestCluster, :assert_replica_consistent, [name]) + end + + test "local cluster disconnect removes a claimless legacy registry row", context do + name = unique_name(:claimless_disconnect) + cluster = "legacy-slice" + + opts = [ + name: name, + shards: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + {:ok, _pid} = TestCluster.start_group(context.node_a, opts) + :ok = TestCluster.rpc!(context.node_a, Group, :connect, [name, cluster]) + + owner = + TestCluster.spawn_register_in_cluster( + context.node_a, + name, + "legacy/claimless", + %{legacy: true}, + cluster + ) + + assert ["legacy/claimless"] = + TestCluster.rpc!( + context.node_a, + Group.Replica.Data, + :purge_registry_claims_for_cluster, + [name, 0, cluster] + ) + + assert {^owner, %{legacy: true}} = + TestCluster.rpc!(context.node_a, Group, :lookup, [ + name, + "legacy/claimless", + [cluster: cluster] + ]) + + :ok = TestCluster.rpc!(context.node_a, Group, :disconnect, [name, cluster]) + + assert nil == + TestCluster.rpc!(context.node_a, Group, :lookup, [ + name, + "legacy/claimless", + [cluster: cluster] + ]) + end + + test "expiring one sideband lane keeps the shared node route while other lanes are live", + context do + name = unique_name(:sideband_lane) + + opts = [ + name: name, + shards: 3, + replica_transport: + {Group.TestTCPTransport, + [connect_timeout: 250, send_timeout: 250, reconnect_interval: 10]}, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + TestCluster.assert_eventually( + fn -> + TestCluster.rpc!(context.node_b, Group.TestTCPTransport, :connected?, [ + name, + context.node_a + ]) + end, + timeout: 10_000 + ) + + source_lanes = + for shard <- 0..2 do + lane = + TestCluster.rpc!(context.node_a, Process, :whereis, [ + Group.Replica.shard_name(name, shard) + ]) + + :ok = TestCluster.rpc!(context.node_a, :sys, :suspend, [lane]) + lane + end + + on_exit(fn -> + Enum.each(source_lanes, fn lane -> + TestCluster.rpc!(context.node_a, TestCluster, :resume_if_alive, [lane]) + end) + end) + + :ok = + TestCluster.rpc!(context.node_b, TestCluster, :expire_replica_lane, [ + name, + 0, + context.node_a + ]) + + Process.sleep(100) + + assert TestCluster.rpc!(context.node_b, Group.TestTCPTransport, :connected?, [ + name, + context.node_a + ]) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_generation, [ + name, + context.node_a + ]) == + TestCluster.rpc!(context.node_a, Group.Replica.Data, :generation, [name]) + + for live_shard <- [1, 2] do + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_view_generation, [ + name, + live_shard, + context.node_a + ]) != nil + end + + status = TestCluster.rpc!(context.node_b, Group.TestTCPTransport, :status, [name]) + assert context.node_a in status.peers + + for shard <- [1, 2] do + :ok = + TestCluster.rpc!(context.node_b, TestCluster, :expire_replica_lane, [ + name, + shard, + context.node_a + ]) + end + + TestCluster.assert_eventually(fn -> + not TestCluster.rpc!(context.node_b, Group.TestTCPTransport, :connected?, [ + name, + context.node_a + ]) and + context.node_a not in TestCluster.rpc!(context.node_b, Group.TestTCPTransport, :status, [ + name + ]).peers + end) + end + + test "a restarted sideband manager rediscovers live peers without nodeup", context do + name = unique_name(:sideband_manager_restart) + + opts = [ + name: name, + shards: 2, + replica_transport: + {Group.TestTCPTransport, + [connect_timeout: 250, send_timeout: 250, reconnect_interval: 10]}, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 1_000 + ] + + start_group_on_peers(context.peers, opts) + + TestCluster.assert_eventually( + fn -> + TestCluster.rpc!(context.node_b, Group.TestTCPTransport, :connected?, [ + name, + context.node_a + ]) + end, + timeout: 10_000 + ) + + seed = TestCluster.spawn_register(context.node_a, name, "manager-restart/seed", %{}) + + TestCluster.assert_eventually(fn -> + match?( + {^seed, %{}}, + TestCluster.rpc!(context.node_b, Group, :lookup, [name, "manager-restart/seed"]) + ) + end) + + manager_name = :"#{name}_replica_tcp_transport" + old_manager = TestCluster.rpc!(context.node_b, Process, :whereis, [manager_name]) + true = TestCluster.rpc!(context.node_b, Process, :exit, [old_manager, :kill]) + + TestCluster.assert_eventually(fn -> + case TestCluster.rpc!(context.node_b, Process, :whereis, [manager_name]) do + manager when is_pid(manager) -> manager != old_manager + _ -> false + end + end) + + # Group and every replica lane stayed alive; there is deliberately no + # nodeup or authority-generation change to trigger rediscovery. + fresh = TestCluster.spawn_register(context.node_a, name, "manager-restart/fresh", %{}) + + TestCluster.assert_eventually( + fn -> + TestCluster.rpc!(context.node_b, Group.TestTCPTransport, :connected?, [ + name, + context.node_a + ]) and + match?( + {^fresh, %{}}, + TestCluster.rpc!(context.node_b, Group, :lookup, [name, "manager-restart/fresh"]) + ) + end, + timeout: 5_000, + interval: 25 + ) + + for node <- [context.node_a, context.node_b, context.node_c] do + assert :ok = TestCluster.rpc!(node, TestCluster, :assert_replica_consistent, [name]) + end + end + + test "three-origin conflict resolution cannot retire every owner", context do + name = unique_name(:three_origin_conflict) + key = "three-origin/conflict" + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + resolve_registry_conflict: {Group.CyclicConflictResolver, :resolve, []}, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + nodes = [context.node_a, context.node_b, context.node_c] + + for node <- nodes do + :ok = + TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [ + name, + {:capture_drop, [:delta_batch]} + ]) + end + + owner_a = TestCluster.spawn_register(context.node_a, name, key, %{rank: :a}) + owner_b = TestCluster.spawn_register(context.node_b, name, key, %{rank: :b}) + owner_c = TestCluster.spawn_register(context.node_c, name, key, %{rank: :c}) + TestCluster.flush_shards(context.node_a, name) + TestCluster.flush_shards(context.node_b, name) + TestCluster.flush_shards(context.node_c, name) + + # Deliver one edge of the cyclic preference to each owner. With the old + # pairwise callback A loses to C, B loses to A, and C loses to B before any + # node has observed all three claims. + for {source, target} <- [ + {context.node_c, context.node_a}, + {context.node_a, context.node_b}, + {context.node_b, context.node_c} + ] do + {_target, 0, frame} = + source + |> TestCluster.rpc!(Group.TestReplicaTransport, :captured, [name]) + |> Enum.find(fn {captured_target, shard, frame} -> + captured_target == target and shard == 0 and elem(frame, 0) == :delta_batch + end) + + :ok = TestCluster.rpc!(target, Group.Transport, :incoming, [name, source, 0, frame]) + end + + for node <- nodes, do: TestCluster.flush_shards(node, name) + + for node <- nodes do + :ok = TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [name, :pass]) + end + + TestCluster.assert_eventually(fn -> + alive = + Enum.filter([owner_a, owner_b, owner_c], fn owner -> + TestCluster.rpc!(node(owner), Process, :alive?, [owner]) + end) + + case alive do + [winner] -> + Enum.all?(nodes, fn receiver -> + match?({^winner, _}, TestCluster.rpc!(receiver, Group, :lookup, [name, key])) + end) + + _ -> + false + end + end) + + for node <- nodes do + assert :ok = TestCluster.rpc!(node, TestCluster, :assert_replica_consistent, [name]) + end + end + + test "an exact shard-zero hello repairs a missing local authority view", context do + name = unique_name(:control_view_repair) + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + {generation, revision, epochs} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_replica_authority, [name]) + + assert generation == + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_view_generation, [ + name, + 0, + context.node_a + ]) + + :ok = + TestCluster.rpc!(context.node_b, TestCluster, :delete_remote_view_info, [ + name, + 0, + context.node_a + ]) + + assert nil == + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_view_generation, [ + name, + 0, + context.node_a + ]) + + assert generation == + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_generation, [ + name, + context.node_a + ]) + + source = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + target = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + send( + target, + {:replica_hello, source, Group.Replica.WireProtocol.version(), generation, revision, epochs, + Group.TestReplicaTransport.id(), Group.TestReplicaTransport.descriptor(name, [])} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target]) + + assert generation == + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_view_generation, [ + name, + 0, + context.node_a + ]) + + assert revision == + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_view_cluster_epoch_revision, + [name, 0, context.node_a] + ) + end + + test "shared authority cannot admit a delta before its receiving lane is installed", context do + name = unique_name(:lane_view_fence) + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + key = + 1..1_000 + |> Enum.map(&"lane-view-fence/#{&1}") + |> Enum.find(&(Group.Replica.shard_index_for(nil, &1, 2) == 1)) + + :ok = + TestCluster.rpc!(context.node_a, Group.TestReplicaTransport, :set_mode, [ + name, + {:capture_drop, [:delta_batch]} + ]) + + pid = TestCluster.spawn_register(context.node_a, name, key, %{lane: 1}) + TestCluster.flush_shards(context.node_a, name) + + {stream_id, frame} = + TestCluster.rpc!(context.node_a, Group.TestReplicaTransport, :captured, [name]) + |> Enum.find_value(fn + {target, 1, {:delta_batch, _version, [{stream_id, _first_seq, _records, _head}]} = frame} + when target == context.node_b -> + {stream_id, frame} + + _other -> + nil + end) + + generation = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_generation, [ + name, + context.node_a + ]) + + assert generation == Group.Replica.WireProtocol.stream_generation(stream_id) + + :ok = + TestCluster.rpc!(context.node_b, TestCluster, :delete_remote_view_info, [ + name, + 1, + context.node_a + ]) + + assert nil == + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_view_generation, [ + name, + 1, + context.node_a + ]) + + cursor_before = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :replica_cursor, [ + name, + 1, + stream_id + ]) + + :ok = + TestCluster.rpc!(context.node_b, Group.Transport, :incoming, [ + name, + context.node_a, + 1, + frame + ]) + + TestCluster.flush_shards(context.node_b, name) + + assert cursor_before == + TestCluster.rpc!(context.node_b, Group.Replica.Data, :replica_cursor, [ + name, + 1, + stream_id + ]) + + assert TestCluster.rpc!(context.node_b, Group, :lookup, [name, key]) == nil + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :registry_claims, [ + name, + 1, + nil, + key + ]) == [] + + # A matching lane hello installs the per-shard fence. The exact same + # previously rejected frame is then admissible and advances contiguously. + source_lane = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + target_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + revision = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_cluster_epoch_revision, [name]) + + send( + target_lane, + {:replica_lane_hello, source_lane, Group.Replica.WireProtocol.version(), generation, + revision, Group.TestReplicaTransport.id(), Group.TestReplicaTransport.descriptor(name, [])} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_lane]) + + assert generation == + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_view_generation, [ + name, + 1, + context.node_a + ]) + + :ok = + TestCluster.rpc!(context.node_b, Group.Transport, :incoming, [ + name, + context.node_a, + 1, + frame + ]) + + TestCluster.flush_shards(context.node_b, name) + + assert match?( + {^pid, %{lane: 1}}, + TestCluster.rpc!(context.node_b, Group, :lookup, [name, key]) + ) + + assert :ok = + TestCluster.rpc!(context.node_b, TestCluster, :assert_replica_consistent, [name]) + end + + test "a missing local authority owner does not crash a healthy replica lane", context do + name = unique_name(:missing_local_authority_owner) + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + source_lane = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + target_control = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + target_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + replica_supervisor = + TestCluster.rpc!(context.node_b, Process, :whereis, [:"#{name}_replica_sup"]) + + :ok = TestCluster.rpc!(context.node_b, :sys, :suspend, [replica_supervisor]) + + on_exit(fn -> + _ = TestCluster.rpc!(context.node_b, TestCluster, :resume_if_alive, [replica_supervisor]) + end) + + monitor = Process.monitor(target_control) + true = TestCluster.rpc!(context.node_b, Process, :exit, [target_control, :kill]) + assert_receive {:DOWN, ^monitor, :process, ^target_control, :killed}, 5_000 + + assert TestCluster.rpc!(context.node_b, Process, :whereis, [ + Group.Replica.shard_name(name, 0) + ]) == nil + + {generation, revision, [{nil, base_epoch}]} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_replica_authority, [name]) + + send( + target_lane, + {:replica_cluster_open, source_lane, generation, revision, [{nil, base_epoch}]} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_lane]) + assert TestCluster.rpc!(context.node_b, Process, :alive?, [target_lane]) + + :ok = TestCluster.rpc!(context.node_b, :sys, :resume, [replica_supervisor]) + end + + test "incremental authority from different lanes cannot race across a revision gap", context do + name = unique_name(:cross_lane_authority_gap) + cluster = "cross-lane-authority-gap" + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + :ok = TestCluster.rpc!(context.node_b, Group, :connect, [name, cluster]) + + [{^cluster, old_epoch}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :activate_local_clusters, [ + name, + [cluster] + ]) + + old_revision = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_cluster_epoch_revision, [name]) + + [{^cluster, ^old_epoch}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :deactivate_local_clusters, [ + name, + [cluster] + ]) + + [{^cluster, current_epoch}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :activate_local_clusters, [ + name, + [cluster] + ]) + + generation = TestCluster.rpc!(context.node_a, Group.Replica.Data, :generation, [name]) + + current_revision = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_cluster_epoch_revision, [name]) + + assert current_revision > old_revision + + source_control = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + source_lane = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + target_control = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + target_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + data_owner = + TestCluster.rpc!(context.node_b, Process, :whereis, [ + Group.Replica.Data.data_name(name) + ]) + + :ok = TestCluster.rpc!(context.node_a, :sys, :suspend, [source_control]) + :ok = TestCluster.rpc!(context.node_b, :sys, :suspend, [data_owner]) + + on_exit(fn -> + _ = + TestCluster.rpc!(context.node_a, TestCluster, :resume_if_alive, [source_control]) + + _ = TestCluster.rpc!(context.node_b, TestCluster, :resume_if_alive, [data_owner]) + end) + + # Shard 0 sees revision 3 first, detects that revisions 1 and 2 are absent, + # and blocks while recording the gap in the shared Data owner. + send( + target_control, + {:replica_cluster_open, source_control, generation, current_revision, + [{cluster, current_epoch}]} + ) + + TestCluster.assert_eventually(fn -> + {:messages, messages} = + TestCluster.rpc!(context.node_b, Process, :info, [data_owner, :messages]) + + Enum.any?(messages, fn + {:"$gen_call", _from, {:observe_remote_cluster_epoch_revision, source, revision}} -> + source == context.node_a and revision == current_revision + + _ -> + false + end) + end) + + # Before the shared gap fence is visible, another lane inspects revision 0 + # and queues an older revision-1 update behind it. A lane must not be able + # to install that stale epoch after shard 0 records the newer gap. + send( + target_lane, + {:replica_cluster_open, source_lane, generation, old_revision, [{cluster, old_epoch}]} + ) + + TestCluster.assert_eventually(fn -> + {:messages, messages} = + TestCluster.rpc!(context.node_b, Process, :info, [target_control, :messages]) + + Enum.any?(messages, fn + {:replica_cluster_open, ^source_lane, ^generation, ^old_revision, + [{^cluster, ^old_epoch}]} -> + true + + _ -> + false + end) + end) + + :ok = TestCluster.rpc!(context.node_b, :sys, :resume, [data_owner]) + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_control]) + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_lane]) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_cluster_epoch, [ + name, + context.node_a, + cluster + ]) == nil + + assert TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_observed_revision, + [name, context.node_a] + ) == current_revision + + refute TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_view_observed_revision, + [name, 1, context.node_a] + ) == current_revision + + {^generation, ^current_revision, authority} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_replica_authority, [name]) + + send( + target_control, + {:replica_hello, source_control, Group.Replica.WireProtocol.version(), generation, + current_revision, authority, Group.TestReplicaTransport.id(), + Group.TestReplicaTransport.descriptor(name, [])} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_control]) + :ok = TestCluster.rpc!(context.node_a, :sys, :resume, [source_control]) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_cluster_epoch, [ + name, + context.node_a, + cluster + ]) == current_epoch + + assert :ok = TestCluster.rpc!(context.node_b, TestCluster, :assert_replica_consistent, [name]) + end + + test "authority replacement fences an in-flight stale registry conflict", context do + name = unique_name(:inflight_stale_conflict) + cluster = "inflight-stale-conflict" + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + resolve_registry_conflict: {Group.PausingConflictResolver, :resolve, [self()]}, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + TestCluster.assert_eventually(fn -> + generation = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_generation, [ + name, + context.node_a + ]) + + observed = + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_observed_revision, + [name, context.node_a] + ) + + not is_nil(generation) and + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_view_generation, [ + name, + 1, + context.node_a + ]) == generation and + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_view_observed_revision, + [name, 1, context.node_a] + ) == observed + end) + + for node <- [context.node_a, context.node_b, context.node_c] do + :ok = TestCluster.rpc!(node, Group, :connect, [name, cluster]) + end + + TestCluster.assert_eventually(fn -> + not is_nil( + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_cluster_epoch, [ + name, + context.node_a, + cluster + ]) + ) + end) + + for node <- [context.node_a, context.node_b, context.node_c] do + :ok = TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [name, :drop]) + end + + key = + 1..1_000 + |> Enum.map(&"inflight-stale-conflict/key/#{&1}") + |> Enum.find(&(Group.Replica.shard_index_for(cluster, &1, 2) == 1)) + + local_owner = + TestCluster.spawn_register_in_cluster( + context.node_b, + name, + key, + %{rank: 1}, + cluster + ) + + remote_owner = + TestCluster.spawn_register_in_cluster( + context.node_a, + name, + key, + %{rank: 2, pause: true}, + cluster + ) + + generation = TestCluster.rpc!(context.node_a, Group.Replica.Data, :generation, [name]) + + epoch = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_cluster_epoch, [ + name, + cluster + ]) + + stream_id = + Group.Replica.WireProtocol.stream_id( + name, + context.node_a, + generation, + 1, + cluster, + epoch + ) + + [{^key, ^remote_owner, remote_meta, remote_time}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :registry_claims_for_stream, [ + name, + 1, + stream_id + ]) + + {_floor, head, applied} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :replica_stream_head, [ + name, + 1, + stream_id + ]) + + assert head == applied + + record = + {head, + [ + {:register, cluster, key, remote_owner, remote_meta, remote_time, context.node_a} + ]} + + :ok = + TestCluster.rpc!(context.node_b, Group.Transport, :incoming, [ + name, + context.node_a, + 1, + {:delta_batch, Group.Replica.WireProtocol.version(), [{stream_id, head, [record], head}]} + ]) + + assert_receive {:conflict_resolver_waiting, resolver, ref, ^key, ^remote_owner}, 5_000 + + on_exit(fn -> send(resolver, {:continue_conflict_resolution, ref}) end) + + [{^cluster, ^epoch}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :deactivate_local_clusters, [ + name, + [cluster] + ]) + + close_revision = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_cluster_epoch_revision, [name]) + + source_control = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + target_control = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + send( + target_control, + {:replica_cluster_close, source_control, generation, close_revision, [{cluster, epoch}]} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_control]) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_cluster_epoch, [ + name, + context.node_a, + cluster + ]) == nil + + send(resolver, {:continue_conflict_resolution, ref}) + TestCluster.flush_shards(context.node_b, name) + + assert TestCluster.rpc!(context.node_b, Process, :alive?, [local_owner]) + + assert match?( + {^local_owner, %{rank: 1}}, + TestCluster.rpc!(context.node_b, Group, :lookup, [ + name, + key, + [cluster: cluster] + ]) + ) + + assert :ok = TestCluster.rpc!(context.node_b, TestCluster, :assert_replica_consistent, [name]) + end + + test "restoring lane authority reprojects a conflict retained during the authority gap", + context do + name = unique_name(:authority_restore_reprojects_conflict) + authority_bump_cluster = "authority-restore-bump" + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + resolve_registry_conflict: {Group.PausingConflictResolver, :resolve, [self()]}, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + TestCluster.assert_eventually(fn -> + generation = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_generation, [ + name, + context.node_a + ]) + + observed = + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_observed_revision, + [name, context.node_a] + ) + + source_generation = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :remote_generation, [ + name, + context.node_b + ]) + + not is_nil(generation) and not is_nil(source_generation) and + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_view_generation, [ + name, + 1, + context.node_a + ]) == generation and + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_view_observed_revision, + [name, 1, context.node_a] + ) == observed + end) + + for node <- [context.node_a, context.node_b, context.node_c] do + :ok = TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [name, :drop]) + end + + key = + 1..1_000 + |> Enum.map(&"authority-restore-conflict/#{&1}") + |> Enum.find(&(Group.Replica.shard_index_for(nil, &1, 2) == 1)) + + local_owner = TestCluster.spawn_register(context.node_b, name, key, %{rank: 1}) + + remote_owner = + TestCluster.spawn_register(context.node_a, name, key, %{rank: 2, pause: true}) + + stream_id = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_stream_id, [name, 1, nil]) + + [{^key, ^remote_owner, remote_meta, remote_time}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :registry_claims_for_stream, [ + name, + 1, + stream_id + ]) + + {_floor, head, applied} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :replica_stream_head, [ + name, + 1, + stream_id + ]) + + assert head == applied + + record = + {head, + [ + {:register, nil, key, remote_owner, remote_meta, remote_time, context.node_a} + ]} + + :ok = + TestCluster.rpc!(context.node_b, Group.Transport, :incoming, [ + name, + context.node_a, + 1, + {:delta_batch, Group.Replica.WireProtocol.version(), [{stream_id, head, [record], head}]} + ]) + + assert_receive {:conflict_resolver_waiting, resolver, ref, ^key, ^remote_owner}, 5_000 + on_exit(fn -> send(resolver, {:continue_conflict_resolution, ref}) end) + + :ok = TestCluster.rpc!(context.node_a, Group, :connect, [name, authority_bump_cluster]) + + generation = TestCluster.rpc!(context.node_a, Group.Replica.Data, :generation, [name]) + + bumped_revision = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_cluster_epoch_revision, [name]) + + bump_epoch = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_cluster_epoch, [ + name, + authority_bump_cluster + ]) + + source_control = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + target_control = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + send( + target_control, + {:replica_cluster_open, source_control, generation, bumped_revision, + [{authority_bump_cluster, bump_epoch}]} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_control]) + + assert TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_observed_revision, + [name, context.node_a] + ) == bumped_revision + + refute TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_view_observed_revision, + [name, 1, context.node_a] + ) == bumped_revision + + send(resolver, {:continue_conflict_resolution, ref}) + + assert_receive {:conflict_resolver_waiting, ^resolver, repair_ref, ^key, ^remote_owner}, 5_000 + send(resolver, {:continue_conflict_resolution, repair_ref}) + + TestCluster.flush_shards(context.node_b, name) + + assert match?( + {^remote_owner, %{rank: 2, pause: true}}, + TestCluster.rpc!(context.node_b, Group, :lookup, [name, key]) + ) + + refute TestCluster.rpc!(context.node_b, Process, :alive?, [local_owner]) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :replica_cursor, [ + name, + 1, + stream_id + ]) == head + + consistency = + Task.async(fn -> + TestCluster.rpc!(context.node_b, TestCluster, :assert_replica_consistent, [name]) + end) + + assert_receive {:conflict_resolver_waiting, consistency_resolver, consistency_ref, ^key, + ^remote_owner}, + 5_000 + + send(consistency_resolver, {:continue_conflict_resolution, consistency_ref}) + assert :ok = Task.await(consistency, 5_000) + end + + test "lease expiry discards conflict reprojection state when the source never returns", + context do + name = unique_name(:authority_gap_expiry) + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + resolve_registry_conflict: {Group.PausingConflictResolver, :resolve, [self()]}, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 500 + ] + + start_group_on_peers(context.peers, opts) + + TestCluster.assert_eventually(fn -> + not is_nil( + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_generation, [ + name, + context.node_a + ]) + ) + end) + + for node <- [context.node_a, context.node_b, context.node_c] do + :ok = TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [name, :drop]) + end + + source_control = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + :ok = TestCluster.rpc!(context.node_a, :sys, :suspend, [source_control]) + + on_exit(fn -> + TestCluster.rpc!(context.node_a, TestCluster, :resume_if_alive, [source_control]) + end) + + key = + 1..1_000 + |> Enum.map(&"authority-gap-expiry/#{&1}") + |> Enum.find(&(Group.Replica.shard_index_for(nil, &1, 2) == 1)) + + local_owner = TestCluster.spawn_register(context.node_b, name, key, %{rank: 1}) + remote_owner = TestCluster.spawn_register(context.node_a, name, key, %{rank: 2, pause: true}) + + stream_id = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_stream_id, [name, 1, nil]) + + [{^key, ^remote_owner, remote_meta, remote_time}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :registry_claims_for_stream, [ + name, + 1, + stream_id + ]) + + {_floor, head, applied} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :replica_stream_head, [ + name, + 1, + stream_id + ]) + + assert head == applied + + :ok = + TestCluster.rpc!(context.node_b, Group.Transport, :incoming, [ + name, + context.node_a, + 1, + {:delta_batch, Group.Replica.WireProtocol.version(), + [ + {stream_id, head, + [ + {head, + [ + {:register, nil, key, remote_owner, remote_meta, remote_time, context.node_a} + ]} + ], head} + ]} + ]) + + assert_receive {:conflict_resolver_waiting, resolver, ref, ^key, ^remote_owner}, 5_000 + on_exit(fn -> send(resolver, {:continue_conflict_resolution, ref}) end) + + initial_revision = + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_observed_revision, + [ + name, + context.node_a + ] + ) + + [{"authority-gap-expiry-1", _epoch1}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :activate_local_clusters, [ + name, + ["authority-gap-expiry-1"] + ]) + + [{"authority-gap-expiry-2", epoch2}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :activate_local_clusters, [ + name, + ["authority-gap-expiry-2"] + ]) + + gap_revision = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_cluster_epoch_revision, [name]) + + assert gap_revision >= initial_revision + 2 + + generation = TestCluster.rpc!(context.node_a, Group.Replica.Data, :generation, [name]) + + target_control = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + send( + target_control, + {:replica_cluster_open, source_control, generation, gap_revision, + [{"authority-gap-expiry-2", epoch2}]} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_control]) + + assert TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_observed_revision, + [name, context.node_a] + ) == gap_revision + + send(resolver, {:continue_conflict_resolution, ref}) + + target_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + TestCluster.assert_eventually(fn -> + state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_lane]) + + state.pending_registry_reprojections + |> Map.get(context.node_a, MapSet.new()) + |> MapSet.member?({nil, key}) + end) + + TestCluster.assert_eventually( + fn -> + state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_lane]) + + not Map.has_key?(state.pending_registry_reprojections, context.node_a) and + TestCluster.rpc!(context.node_b, Group.Replica.Data, :registry_claims, [ + name, + 1, + nil, + key + ]) + |> Enum.all?(&(elem(&1, 3) != context.node_a)) and + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :replica_cursor_streams_for_origin, + [name, 1, context.node_a] + ) == [] + end, + timeout: 10_000 + ) + + assert TestCluster.rpc!(context.node_b, Process, :alive?, [local_owner]) + + assert match?( + {^local_owner, %{rank: 1}}, + TestCluster.rpc!(context.node_b, Group, :lookup, [name, key]) + ) + end + + test "a newer heartbeat fences old-epoch data before exact authority arrives", context do + name = unique_name(:heartbeat_authority_fence) + cluster = "heartbeat-authority-fence" + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + resolve_registry_conflict: {Group.ModelConflictResolver, :resolve, []}, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + for node <- [context.node_a, context.node_b, context.node_c] do + :ok = TestCluster.rpc!(node, Group, :connect, [name, cluster]) + end + + TestCluster.assert_eventually(fn -> + not is_nil( + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_cluster_epoch, [ + name, + context.node_a, + cluster + ]) + ) + end) + + for node <- [context.node_a, context.node_b, context.node_c] do + :ok = TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [name, :drop]) + end + + key = + 1..1_000 + |> Enum.map(&"heartbeat-authority-fence/key/#{&1}") + |> Enum.find(&(Group.Replica.shard_index_for(cluster, &1, 2) == 1)) + + local_owner = + TestCluster.spawn_register_in_cluster(context.node_b, name, key, %{rank: 1}, cluster) + + stale_owner = + TestCluster.spawn_register_in_cluster(context.node_a, name, key, %{rank: 2}, cluster) + + generation = TestCluster.rpc!(context.node_a, Group.Replica.Data, :generation, [name]) + + old_epoch = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_cluster_epoch, [ + name, + cluster + ]) + + old_stream = + Group.Replica.WireProtocol.stream_id( + name, + context.node_a, + generation, + 1, + cluster, + old_epoch + ) + + [{^key, ^stale_owner, stale_meta, stale_time}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :registry_claims_for_stream, [ + name, + 1, + old_stream + ]) + + {_floor, old_head, old_head} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :replica_stream_head, [ + name, + 1, + old_stream + ]) + + source_control = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + source_lane = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + target_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + :ok = TestCluster.rpc!(context.node_a, :sys, :suspend, [source_control]) + + on_exit(fn -> + TestCluster.rpc!(context.node_a, TestCluster, :resume_if_alive, [source_control]) + end) + + # Change only the source's durable authority. The incremental close/open + # controls are intentionally absent, while the matching data lane exposes + # the newer revision through its normal heartbeat. + [{^cluster, ^old_epoch}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :deactivate_local_clusters, [ + name, + [cluster] + ]) + + [{^cluster, new_epoch}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :activate_local_clusters, [ + name, + [cluster] + ]) + + refute new_epoch == old_epoch + + new_revision = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_cluster_epoch_revision, [name]) + + send( + target_lane, + {:replica_heartbeat, source_lane, Group.Replica.WireProtocol.version(), generation, + new_revision, Group.TestReplicaTransport.id(), + Group.TestReplicaTransport.descriptor(name, [])} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_lane]) + + assert TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_observed_revision, + [name, context.node_a] + ) == new_revision + + old_record = + {old_head, + [ + {:register, cluster, key, stale_owner, stale_meta, stale_time, context.node_a} + ]} + + :ok = + TestCluster.rpc!(context.node_b, Group.Transport, :incoming, [ + name, + context.node_a, + 1, + {:delta_batch, Group.Replica.WireProtocol.version(), + [{old_stream, old_head, [old_record], old_head}]} + ]) + + TestCluster.flush_shards(context.node_b, name) + + assert TestCluster.rpc!(context.node_b, Process, :alive?, [local_owner]) + + assert match?( + {^local_owner, %{rank: 1}}, + TestCluster.rpc!(context.node_b, Group, :lookup, [ + name, + key, + [cluster: cluster] + ]) + ) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :replica_cursor, [ + name, + 1, + old_stream + ]) == 0 + end + + test "a newer-generation heartbeat fences prior-generation data before exact authority", + context do + name = unique_name(:heartbeat_generation_fence) + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + resolve_registry_conflict: {Group.ModelConflictResolver, :resolve, []}, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + for node <- [context.node_a, context.node_b, context.node_c] do + :ok = TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [name, :drop]) + end + + key = + 1..1_000 + |> Enum.map(&"heartbeat-generation-fence/key/#{&1}") + |> Enum.find(&(Group.Replica.shard_index_for(nil, &1, 2) == 1)) + + local_owner = TestCluster.spawn_register(context.node_b, name, key, %{rank: 1}) + stale_owner = TestCluster.spawn_register(context.node_a, name, key, %{rank: 2}) + stale_cluster = "heartbeat-generation-fence/stale-cluster" + + [{^stale_cluster, stale_epoch}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :activate_local_clusters, [ + name, + [stale_cluster] + ]) + + old_stream = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_stream_id, [name, 1, nil]) + + old_generation = Group.Replica.WireProtocol.stream_generation(old_stream) + + {^old_generation, old_revision, old_authority} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_replica_authority, [name]) + + [{^key, ^stale_owner, stale_meta, stale_time}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :registry_claims_for_stream, [ + name, + 1, + old_stream + ]) + + {_floor, old_head, old_head} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :replica_stream_head, [ + name, + 1, + old_stream + ]) + + source_control = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + source_lane = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + target_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + target_control = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + :ok = TestCluster.rpc!(context.node_a, :sys, :suspend, [source_control]) + + on_exit(fn -> + TestCluster.rpc!(context.node_a, TestCluster, :resume_if_alive, [source_control]) + end) + + new_generation = + TestCluster.rpc!(context.node_a, Group.Replica.WireProtocol, :new_generation, []) + + assert Group.Replica.WireProtocol.generation_newer?(new_generation, old_generation) + + send( + target_lane, + {:replica_heartbeat, source_lane, Group.Replica.WireProtocol.version(), new_generation, 0, + Group.TestReplicaTransport.id(), Group.TestReplicaTransport.descriptor(name, [])} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_lane]) + + refute TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_view_cluster_epoch_revision, + [ + name, + 1, + context.node_a + ] + ) == + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_exact_revision, + [name, context.node_a] + ) + + send( + target_control, + {:replica_cluster_open, source_control, old_generation, old_revision, + [{stale_cluster, stale_epoch}]} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_control]) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_cluster_epoch, [ + name, + context.node_a, + stale_cluster + ]) == nil + + assert :stale = + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :put_remote_replica_info, + [ + name, + 0, + context.node_a, + old_generation, + old_revision, + old_authority + ] + ) + + assert :stale = + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :put_remote_view_info, + [name, 0, context.node_a, old_generation, old_revision, old_revision] + ) + + # The newer generation is only a hint until its exact authority arrives, + # but it must still fence a delayed exact hello from the prior generation. + send( + target_control, + {:replica_hello, source_control, Group.Replica.WireProtocol.version(), old_generation, + old_revision, old_authority, Group.TestReplicaTransport.id(), + Group.TestReplicaTransport.descriptor(name, [])} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_control]) + + refute TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_view_cluster_epoch_revision, + [name, 0, context.node_a] + ) == + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_exact_revision, + [name, context.node_a] + ) + + old_record = + {old_head, + [ + {:register, nil, key, stale_owner, stale_meta, stale_time, context.node_a} + ]} + + :ok = + TestCluster.rpc!(context.node_b, Group.Transport, :incoming, [ + name, + context.node_a, + 1, + {:delta_batch, Group.Replica.WireProtocol.version(), + [{old_stream, old_head, [old_record], old_head}]} + ]) + + TestCluster.flush_shards(context.node_b, name) + + assert TestCluster.rpc!(context.node_b, Process, :alive?, [local_owner]) + + assert match?( + {^local_owner, %{rank: 1}}, + TestCluster.rpc!(context.node_b, Group, :lookup, [name, key]) + ) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :replica_cursor, [ + name, + 1, + old_stream + ]) == 0 + end + + test "a newer lane hello cannot reinstall a view from stale exact authority", context do + name = unique_name(:lane_hello_exact_fence) + bump_cluster = "lane-hello-exact-fence/bump" + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + for node <- [context.node_a, context.node_b, context.node_c] do + :ok = TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [name, :drop]) + end + + key = + 1..1_000 + |> Enum.map(&"lane-hello-exact-fence/member/#{&1}") + |> Enum.find(&(Group.Replica.shard_index_for(nil, &1, 2) == 1)) + + owner = TestCluster.spawn_join(context.node_a, name, key, %{owner: :a}) + + stream = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_stream_id, [name, 1, nil]) + + generation = Group.Replica.WireProtocol.stream_generation(stream) + + {_floor, head, head} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :replica_stream_head, [ + name, + 1, + stream + ]) + + [{^head, mutations}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :replica_records, [ + name, + 1, + stream, + head, + 1 + ]) + + source_control = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + source_lane = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + target_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + :ok = TestCluster.rpc!(context.node_a, :sys, :suspend, [source_control]) + + on_exit(fn -> + TestCluster.rpc!(context.node_a, TestCluster, :resume_if_alive, [source_control]) + end) + + [{^bump_cluster, _epoch}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :activate_local_clusters, [ + name, + [bump_cluster] + ]) + + revision = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_cluster_epoch_revision, [name]) + + send( + target_lane, + {:replica_lane_hello, source_lane, Group.Replica.WireProtocol.version(), generation, + revision, Group.TestReplicaTransport.id(), Group.TestReplicaTransport.descriptor(name, [])} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_lane]) + + refute TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_view_observed_revision, + [name, 1, context.node_a] + ) == revision + + :ok = + TestCluster.rpc!(context.node_b, Group.Transport, :incoming, [ + name, + context.node_a, + 1, + {:delta_batch, Group.Replica.WireProtocol.version(), + [{stream, head, [{head, mutations}], head}]} + ]) + + TestCluster.flush_shards(context.node_b, name) + + assert TestCluster.rpc!(context.node_b, Group, :members, [name, key]) == [] + assert TestCluster.rpc!(context.node_a, Process, :alive?, [owner]) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :replica_cursor, [ + name, + 1, + stream + ]) == 0 + end + + test "a delayed incremental control cannot roll observed authority backward", context do + name = unique_name(:incremental_authority_rollback) + cluster = "incremental-authority-rollback" + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + resolve_registry_conflict: {Group.ModelConflictResolver, :resolve, []}, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + for node <- [context.node_a, context.node_b, context.node_c] do + :ok = TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [name, :drop]) + end + + :ok = TestCluster.rpc!(context.node_b, Group, :connect, [name, cluster]) + + [{^cluster, old_epoch}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :activate_local_clusters, [ + name, + [cluster] + ]) + + old_revision = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_cluster_epoch_revision, [name]) + + [{^cluster, ^old_epoch}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :deactivate_local_clusters, [ + name, + [cluster] + ]) + + [{^cluster, current_epoch}] = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :activate_local_clusters, [ + name, + [cluster] + ]) + + refute current_epoch == old_epoch + + :ok = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :add_cluster_node, [ + name, + [cluster], + context.node_a + ]) + + generation = TestCluster.rpc!(context.node_a, Group.Replica.Data, :generation, [name]) + + current_revision = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_cluster_epoch_revision, [name]) + + assert current_revision > old_revision + + source_control = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + source_data_lane = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + target_control = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + target_data_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + # Keep the exact-authority request caused by the intentional revision gap + # queued at its source until this test has exercised the delayed control and + # stale data frame. The Group control plane intentionally bypasses the test + # replica transport, so transport :drop alone cannot create this window. + :ok = TestCluster.rpc!(context.node_a, :sys, :suspend, [source_control]) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_cluster_epoch, [ + name, + context.node_a, + cluster + ]) == nil + + initial_observed_revision = + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_observed_revision, + [name, context.node_a] + ) + + assert is_integer(initial_observed_revision) + assert initial_observed_revision < old_revision + + send( + target_data_lane, + {:replica_cluster_open, source_data_lane, generation, current_revision, + [{cluster, current_epoch}]} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_data_lane]) + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_control]) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_cluster_epoch, [ + name, + context.node_a, + cluster + ]) == nil + + assert TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_observed_revision, + [name, context.node_a] + ) == current_revision + + # The old open was delayed behind the newer open. Accepting it can make an + # obsolete stream authoritative long enough to retire a valid local owner; + # the later exact hello can purge the stale claim but cannot resurrect that + # killed owner. + send( + target_control, + {:replica_cluster_open, source_control, generation, old_revision, [{cluster, old_epoch}]} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_control]) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_cluster_epoch, [ + name, + context.node_a, + cluster + ]) == nil + + key = + 1..1_000 + |> Enum.map(&"incremental-authority-rollback/key/#{&1}") + |> Enum.find(&(Group.Replica.shard_index_for(cluster, &1, 2) == 0)) + + local_owner = + TestCluster.spawn_register_in_cluster( + context.node_b, + name, + key, + %{rank: 1}, + cluster + ) + + stale_owner = TestCluster.spawn_monitor_forwarder(context.node_a, name, :all, self()) + assert_receive {:monitor_ready, ^stale_owner}, 5_000 + + stale_stream = + Group.Replica.WireProtocol.stream_id( + name, + context.node_a, + generation, + 0, + cluster, + old_epoch + ) + + stale_record = + {1, + [ + {:register, cluster, key, stale_owner, %{rank: 2}, System.system_time(), context.node_a} + ]} + + :ok = + TestCluster.rpc!(context.node_b, Group.Transport, :incoming, [ + name, + context.node_a, + 0, + {:delta_batch, Group.Replica.WireProtocol.version(), + [{stale_stream, 1, [stale_record], 1}]} + ]) + + TestCluster.flush_shards(context.node_b, name) + + {^generation, ^current_revision, current_authority} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_replica_authority, [name]) + + send( + target_control, + {:replica_hello, source_control, Group.Replica.WireProtocol.version(), generation, + current_revision, current_authority, Group.TestReplicaTransport.id(), + Group.TestReplicaTransport.descriptor(name, [])} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_control]) + + :ok = TestCluster.rpc!(context.node_a, :sys, :resume, [source_control]) + + assert TestCluster.rpc!(context.node_b, Process, :alive?, [local_owner]) + + assert match?( + {^local_owner, %{rank: 1}}, + TestCluster.rpc!(context.node_b, Group, :lookup, [ + name, + key, + [cluster: cluster] + ]) + ) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_cluster_epoch, [ + name, + context.node_a, + cluster + ]) == current_epoch + + assert :ok = TestCluster.rpc!(context.node_b, TestCluster, :assert_replica_consistent, [name]) + end + + test "a receiver restart forgets cursors for a locally deactivated cluster", context do + name = unique_name(:inactive_cluster_cursor_restart) + cluster = "inactive-cluster-cursor-restart" + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + for node <- [context.node_a, context.node_b, context.node_c] do + :ok = TestCluster.rpc!(node, Group, :connect, [name, cluster]) + end + + TestCluster.assert_eventually( + fn -> + Enum.all?([context.node_a, context.node_b, context.node_c], fn node -> + length(TestCluster.rpc!(node, Group, :nodes, [name, cluster])) == 3 + end) + end, + timeout: 30_000, + interval: 50 + ) + + key = + 1..1_000 + |> Enum.map(&"inactive-cluster-cursor/#{&1}") + |> Enum.find(&(Group.Replica.shard_index_for(cluster, &1, 2) == 1)) + + registry_owner = + TestCluster.spawn_register_in_cluster( + context.node_a, + name, + key, + %{kind: :registry}, + cluster + ) + + pg_owner = + TestCluster.spawn_join_in_cluster( + context.node_a, + name, + key, + %{kind: :pg}, + cluster + ) + + TestCluster.assert_eventually(fn -> + match?( + {^registry_owner, %{kind: :registry}}, + TestCluster.rpc!(context.node_b, Group, :lookup, [name, key, [cluster: cluster]]) + ) and + match?( + [{^pg_owner, %{kind: :pg}}], + TestCluster.rpc!(context.node_b, Group, :members, [name, key, [cluster: cluster]]) + ) + end) + + stream_id = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_stream_id, [ + name, + 1, + cluster + ]) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :replica_cursor, [ + name, + 1, + stream_id + ]) > 0 + + # This is the durable first half of Group.disconnect. Kill the lane before + # its normal disconnect request can erase receive cursors and rows. + [{^cluster, _local_epoch}] = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :deactivate_local_clusters, [ + name, + [cluster] + ]) + + old_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + monitor = Process.monitor(old_lane) + true = TestCluster.rpc!(context.node_b, Process, :exit, [old_lane, :kill]) + assert_receive {:DOWN, ^monitor, :process, ^old_lane, :killed}, 5_000 + + TestCluster.assert_eventually(fn -> + case TestCluster.rpc!(context.node_b, Process, :whereis, [ + Group.Replica.shard_name(name, 1) + ]) do + pid when is_pid(pid) -> pid != old_lane + _ -> false + end + end) + + assert TestCluster.rpc!(context.node_b, :ets, :lookup, [ + Group.Replica.Data.replica_cursor_table(name, 1), + stream_id + ]) == [] + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :registry_lookup, [ + name, + 1, + cluster, + key + ]) == nil + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :pg_lookup, [ + name, + 1, + cluster, + key, + pg_owner + ]) == nil + + assert :ok = TestCluster.rpc!(context.node_b, TestCluster, :assert_replica_consistent, [name]) + end + + test "a receiver restart removes cursorless rows whose named-stream authority closed", + context do + name = unique_name(:cursorless_closed_stream_restart) + cluster = "cursorless-restart" + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + replicated_sender_buffer_size: 1, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 5_000 + ] + + start_group_on_peers(context.peers, opts) + + for node <- [context.node_a, context.node_b, context.node_c] do + :ok = TestCluster.rpc!(node, Group, :connect, [name, cluster]) + end + + TestCluster.assert_eventually( + fn -> + Enum.all?([context.node_a, context.node_b, context.node_c], fn node -> + length(TestCluster.rpc!(node, Group, :nodes, [name, cluster])) == 3 + end) + end, + timeout: 30_000, + interval: 50 + ) + + key = + 1..1_000 + |> Enum.map(&"cursorless-restart/#{&1}") + |> Enum.find(&(Group.Replica.shard_index_for(cluster, &1, 2) == 1)) + + owner = + TestCluster.spawn_register_in_cluster( + context.node_a, + name, + key, + %{kind: :registry}, + cluster + ) + + member = + TestCluster.spawn_join_in_cluster( + context.node_a, + name, + key, + %{kind: :pg}, + cluster + ) + + for receiver <- [context.node_b, context.node_c] do + TestCluster.assert_eventually(fn -> + match?( + {^owner, %{kind: :registry}}, + TestCluster.rpc!(receiver, Group, :lookup, [name, key, [cluster: cluster]]) + ) and + match?( + [{^member, %{kind: :pg}}], + TestCluster.rpc!(receiver, Group, :members, [name, key, [cluster: cluster]]) + ) + end) + end + + stream_id = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_stream_id, [ + name, + 1, + cluster + ]) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :replica_cursor, [ + name, + 1, + stream_id + ]) > 0 + + # Model either receiver crash window: rows were materialized before the + # cursor was recorded, or authority cleanup deleted the cursor before all + # rows. Restart repair must never depend solely on that missing breadcrumb. + :ok = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :delete_replica_cursor, [ + name, + 1, + stream_id + ]) + + replica_supervisor = + TestCluster.rpc!(context.node_b, Process, :whereis, [:"#{name}_replica_sup"]) + + old_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + :ok = TestCluster.rpc!(context.node_b, :sys, :suspend, [replica_supervisor]) + + on_exit(fn -> + TestCluster.rpc!(context.node_b, TestCluster, :resume_if_alive, [replica_supervisor]) + end) + + lane_monitor = Process.monitor(old_lane) + true = TestCluster.rpc!(context.node_b, Process, :exit, [old_lane, :kill]) + assert_receive {:DOWN, ^lane_monitor, :process, ^old_lane, :killed}, 5_000 + + :ok = TestCluster.rpc!(context.node_a, Group, :disconnect, [name, cluster]) + + exact_revision = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_cluster_epoch_revision, [name]) + + TestCluster.assert_eventually( + fn -> + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_exact_revision, + [name, context.node_a] + ) == exact_revision and + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_cluster_epoch, [ + name, + context.node_a, + cluster + ]) == nil + end, + timeout: 5_000, + interval: 25 + ) + + :ok = TestCluster.rpc!(context.node_b, :sys, :resume, [replica_supervisor]) + + TestCluster.assert_eventually(fn -> + case TestCluster.rpc!(context.node_b, Process, :whereis, [ + Group.Replica.shard_name(name, 1) + ]) do + pid when is_pid(pid) -> pid != old_lane + _ -> false + end + end) + + TestCluster.assert_eventually( + fn -> + TestCluster.rpc!(context.node_b, Group, :lookup, [name, key, [cluster: cluster]]) == nil and + TestCluster.rpc!(context.node_b, Group, :members, [name, key, [cluster: cluster]]) == [] + end, + timeout: 2_000, + interval: 25 + ) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :registry_claims, [ + name, + 1, + cluster, + key + ]) == [] + + assert :ok = + TestCluster.rpc!(context.node_b, TestCluster, :assert_replica_consistent, [name]) + end + + test "a receiver restart rejects cursorless rows under otherwise current authority", context do + name = unique_name(:cursorless_current_authority_restart) + + opts = [ + name: name, + shards: 1, + replica_transport: Group.TestReplicaTransport, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + key = "cursorless-current-authority/key" + registry_owner = TestCluster.spawn_register(context.node_a, name, key, %{kind: :registry}) + pg_owner = TestCluster.spawn_join(context.node_a, name, key, %{kind: :pg}) + + for receiver <- [context.node_b, context.node_c] do + TestCluster.assert_eventually(fn -> + match?( + {^registry_owner, %{kind: :registry}}, + TestCluster.rpc!(receiver, Group, :lookup, [name, key]) + ) and + match?( + [{^pg_owner, %{kind: :pg}}], + TestCluster.rpc!(receiver, Group, :members, [name, key]) + ) + end) + end + + for node <- [context.node_a, context.node_b, context.node_c] do + :ok = TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [name, :drop]) + end + + stream_id = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_stream_id, [name, 0, nil]) + + :ok = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :delete_replica_cursor, [ + name, + 0, + stream_id + ]) + + old_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + monitor = Process.monitor(old_lane) + true = TestCluster.rpc!(context.node_b, Process, :exit, [old_lane, :kill]) + assert_receive {:DOWN, ^monitor, :process, ^old_lane, :killed}, 5_000 + + TestCluster.assert_eventually(fn -> + case TestCluster.rpc!(context.node_b, Process, :whereis, [ + Group.Replica.shard_name(name, 0) + ]) do + pid when is_pid(pid) -> pid != old_lane + _ -> false + end + end) + + new_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [new_lane]) + + assert TestCluster.rpc!(context.node_b, Group, :lookup, [name, key]) == nil + assert TestCluster.rpc!(context.node_b, Group, :members, [name, key]) == [] + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :registry_claims, [ + name, + 0, + nil, + key + ]) == [] + + for node <- [context.node_a, context.node_b, context.node_c] do + :ok = TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [name, :pass]) + end + + {floor, head, _applied} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :replica_stream_head, [ + name, + 0, + stream_id + ]) + + :ok = + TestCluster.rpc!(context.node_b, Group.Transport, :incoming, [ + name, + context.node_a, + 0, + {:heads, Group.Replica.WireProtocol.version(), [{stream_id, floor, head}]} + ]) + + TestCluster.assert_eventually( + fn -> + match?( + {^registry_owner, %{kind: :registry}}, + TestCluster.rpc!(context.node_b, Group, :lookup, [name, key]) + ) and + match?( + [{^pg_owner, %{kind: :pg}}], + TestCluster.rpc!(context.node_b, Group, :members, [name, key]) + ) + end, + timeout: 10_000, + interval: 25 + ) + + assert :ok = TestCluster.rpc!(context.node_b, TestCluster, :assert_replica_consistent, [name]) + end + + test "a receiver restart never projects a partially installed exact snapshot", context do + name = unique_name(:partial_snapshot_install_restart) + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + replicated_oplog_max_entries: 1, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + for node <- [context.node_a, context.node_b, context.node_c] do + :ok = TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [name, :drop]) + end + + registry_key = + 1..1_000 + |> Enum.map(&"partial-install/registry/#{&1}") + |> Enum.find(&(Group.Replica.shard_index_for(nil, &1, 2) == 1)) + + pg_key = + 1..1_000 + |> Enum.map(&"partial-install/pg/#{&1}") + |> Enum.find(&(Group.Replica.shard_index_for(nil, &1, 2) == 1)) + + registry_owner = + TestCluster.spawn_register(context.node_a, name, registry_key, %{kind: :registry}) + + pg_owner = TestCluster.spawn_join(context.node_a, name, pg_key, %{kind: :pg}) + TestCluster.flush_shards(context.node_a, name) + + stream_id = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_stream_id, [name, 1, nil]) + + {floor, head, applied} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :replica_stream_head, [ + name, + 1, + stream_id + ]) + + assert floor > 1 + assert applied == head + + # Model the exact crash boundary in both row domains: the receiver has + # written only a subset of the exact image after recording its sequence-0 + # admission marker, but dies before publishing the snapshot cursor. These + # are the same public ETS writes used by the commit path. + :ok = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :begin_replica_snapshot_install, [ + name, + 1, + stream_id, + head + ]) + + :ok = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :put_registry_claim, [ + name, + 1, + stream_id, + head, + registry_key, + registry_owner, + %{kind: :registry}, + System.system_time() + ]) + + :ok = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :pg_insert, [ + name, + 1, + nil, + pg_key, + pg_owner, + %{kind: :pg}, + System.system_time(), + context.node_a + ]) + + replica_supervisor = + TestCluster.rpc!(context.node_b, Process, :whereis, [:"#{name}_replica_sup"]) + + old_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + :ok = TestCluster.rpc!(context.node_b, :sys, :suspend, [replica_supervisor]) + + on_exit(fn -> + TestCluster.rpc!(context.node_b, TestCluster, :resume_if_alive, [replica_supervisor]) + end) + + monitor = Process.monitor(old_lane) + true = TestCluster.rpc!(context.node_b, Process, :exit, [old_lane, :kill]) + assert_receive {:DOWN, ^monitor, :process, ^old_lane, :killed}, 5_000 + + :ok = TestCluster.rpc!(context.node_b, :sys, :resume, [replica_supervisor]) + + TestCluster.assert_eventually(fn -> + case TestCluster.rpc!(context.node_b, Process, :whereis, [ + Group.Replica.shard_name(name, 1) + ]) do + pid when is_pid(pid) -> pid != old_lane + _ -> false + end + end) + + assert TestCluster.rpc!(context.node_b, Group, :lookup, [name, registry_key]) == nil + assert TestCluster.rpc!(context.node_b, Group, :members, [name, pg_key]) == [] + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :registry_claims, [ + name, + 1, + nil, + registry_key + ]) == [] + + cursor_table = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :replica_cursor_table, [name, 1]) + + refute TestCluster.rpc!(context.node_b, :ets, :member, [cursor_table, stream_id]) + assert :ok = TestCluster.rpc!(context.node_b, TestCluster, :assert_replica_consistent, [name]) + end + + test "a delayed prior-generation hello cannot roll authority backward", context do + name = unique_name(:generation_rollback) + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] + + start_group_on_peers(context.peers, opts) + + {old_generation, old_revision, old_epochs} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_replica_authority, [name]) + + old_control = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + supervisor = TestCluster.rpc!(context.node_a, Process, :whereis, [:"#{name}_group_sup"]) + :ok = TestCluster.rpc!(context.node_a, Supervisor, :stop, [supervisor, :normal, 5_000]) + {:ok, _pid} = TestCluster.start_group(context.node_a, opts) + + new_generation = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :generation, [name]) + + refute new_generation == old_generation + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_generation, [ + name, + context.node_a + ]) == new_generation + end) + + target = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + send( + target, + {:replica_hello, old_control, Group.Replica.WireProtocol.version(), old_generation, + old_revision, old_epochs, Group.TestReplicaTransport.id(), + Group.TestReplicaTransport.descriptor(name, [])} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target]) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_generation, [ + name, + context.node_a + ]) == new_generation + end + + test "an unresolved authority hint is retired and cannot recreate a retired peer", context do + name = unique_name(:delayed_hint_retirement) + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 150 + ] + + start_group_on_peers(context.peers, opts) + + old_generation = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :generation, [name]) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_generation, [ + name, + context.node_a + ]) == old_generation + end) + + source_control = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + source_lane = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + target_control = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + target_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + for source <- [source_control, source_lane] do + :ok = TestCluster.rpc!(context.node_a, :sys, :suspend, [source]) + end + + on_exit(fn -> + for source <- [source_control, source_lane] do + TestCluster.rpc!(context.node_a, TestCluster, :resume_if_alive, [source]) + end + end) + + new_generation = + TestCluster.rpc!(context.node_a, Group.Replica.WireProtocol, :new_generation, []) + + assert Group.Replica.WireProtocol.generation_newer?(new_generation, old_generation) + + send( + target_lane, + {:replica_heartbeat, source_lane, Group.Replica.WireProtocol.version(), new_generation, 0, + Group.TestReplicaTransport.id(), Group.TestReplicaTransport.descriptor(name, [])} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_lane]) + + assert TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_replica_authority_hint, + [name, context.node_a] + ) == {new_generation, 0} + + TestCluster.assert_eventually( + fn -> + hint = + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_replica_authority_hint, + [name, context.node_a] + ) + + control_state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_control]) + lane_state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_lane]) + + is_nil(hint) and + not Map.has_key?(control_state.cluster_control_dirty, context.node_a) and + not Map.has_key?(lane_state.peer_last_seen, context.node_a) + end, + timeout: 3_000, + interval: 25 + ) + + # A dirty notification can have been sent locally before the final lane + # retired but reach shard zero afterward. The next sweep must forget that + # now-authority-less repair obligation instead of retaining one map entry + # forever for the departed node. + send(target_control, {:replica_authority_dirty_local, context.node_a}) + control_state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_control]) + assert Map.has_key?(control_state.cluster_control_dirty, context.node_a) + + send(target_control, {:group_replica_anti_entropy, control_state.anti_entropy_ref}) + control_state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_control]) + refute Map.has_key?(control_state.cluster_control_dirty, context.node_a) + + # Once lease retirement removes exact authority, even a delayed heartbeat + # from that incarnation is only a discovery prompt. It cannot recreate a + # hint, route, or new lease without the dist-Erlang exact hello. + send( + target_lane, + {:replica_heartbeat, source_lane, Group.Replica.WireProtocol.version(), new_generation, 0, + Group.TestReplicaTransport.id(), Group.TestReplicaTransport.descriptor(name, [])} + ) + + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_lane]) + + assert TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_replica_authority_hint, + [name, context.node_a] + ) == nil + + refute Map.has_key?( + TestCluster.rpc!(context.node_b, :sys, :get_state, [target_lane]).peer_last_seen, + context.node_a + ) + + send( + target_lane, + {:replica_lane_hello, source_lane, Group.Replica.WireProtocol.version(), new_generation, 0, + Group.TestReplicaTransport.id(), Group.TestReplicaTransport.descriptor(name, [])} + ) + + lane_state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_lane]) + + assert TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_replica_authority_hint, + [name, context.node_a] + ) == nil + + refute Map.has_key?(lane_state.remote_shards, context.node_a) + refute Map.has_key?(lane_state.peer_last_seen, context.node_a) + end + + test "restart authority repair runs before stale claims can retire a local owner", context do + name = unique_name(:repair_before_projection) + cluster = "repair-before-projection" + + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + resolve_registry_conflict: {Group.ModelConflictResolver, :resolve, []}, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 5_000 + ] + + start_group_on_peers(context.peers, opts) + + for node <- [context.node_a, context.node_b, context.node_c] do + :ok = TestCluster.rpc!(node, Group, :connect, [name, cluster]) + end + + TestCluster.assert_eventually(fn -> + Enum.all?([context.node_a, context.node_b, context.node_c], fn node -> + length(TestCluster.rpc!(node, Group, :nodes, [name, cluster])) == 3 + end) + end) + + for node <- [context.node_a, context.node_b, context.node_c] do + :ok = TestCluster.rpc!(node, Group.TestReplicaTransport, :set_mode, [name, :drop]) + end + + key = + 1..1_000 + |> Enum.map(&"repair-before-projection/#{&1}") + |> Enum.find(&(Group.Replica.shard_index_for(cluster, &1, 2) == 1)) + + local_owner = + TestCluster.spawn_register_in_cluster( + context.node_b, + name, + key, + %{rank: 1}, + cluster + ) + + stale_remote_owner = + TestCluster.spawn_register_in_cluster( + context.node_a, + name, + key, + %{rank: 2}, + cluster + ) + + TestCluster.flush_shards(context.node_a, name) + + stale_stream = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_stream_id, [ + name, + 1, + cluster + ]) + + :ok = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :put_registry_claim, [ + name, + 1, + stale_stream, + 1, + key, + stale_remote_owner, + %{rank: 2}, + System.system_time() + ]) + + assert match?( + {^local_owner, %{rank: 1}}, + TestCluster.rpc!(context.node_b, Group, :lookup, [name, key, [cluster: cluster]]) + ) + + replica_supervisor = + TestCluster.rpc!(context.node_b, Process, :whereis, [:"#{name}_replica_sup"]) + + old_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + :ok = TestCluster.rpc!(context.node_b, :sys, :suspend, [replica_supervisor]) + + on_exit(fn -> + TestCluster.rpc!(context.node_b, TestCluster, :resume_if_alive, [replica_supervisor]) + end) + + monitor = Process.monitor(old_lane) + true = TestCluster.rpc!(context.node_b, Process, :exit, [old_lane, :kill]) + assert_receive {:DOWN, ^monitor, :process, ^old_lane, :killed}, 5_000 + + :ok = TestCluster.rpc!(context.node_a, Group, :disconnect, [name, cluster]) + + exact_revision = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_cluster_epoch_revision, [name]) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!( + context.node_b, + Group.Replica.Data, + :remote_cluster_epoch_exact_revision, + [name, context.node_a] + ) == exact_revision and + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_cluster_epoch, [ + name, + context.node_a, + cluster + ]) == nil + end) + + :ok = TestCluster.rpc!(context.node_b, :sys, :resume, [replica_supervisor]) + + TestCluster.assert_eventually(fn -> + case TestCluster.rpc!(context.node_b, Process, :whereis, [ + Group.Replica.shard_name(name, 1) + ]) do + pid when is_pid(pid) -> pid != old_lane + _ -> false + end + end) - assert Enum.map(claims, &elem(&1, 0)) == [pid_b], inspect(journal_state) + assert TestCluster.rpc!(context.node_b, Process, :alive?, [local_owner]) - assert {^pid_b, %{rank: 1}} = - TestCluster.rpc!(context.node_a, Group, :lookup, [name, "hidden/journal"]) + assert match?( + {^local_owner, %{rank: 1}}, + TestCluster.rpc!(context.node_b, Group, :lookup, [name, key, [cluster: cluster]]) + ) - assert [] = - TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_replica_unapplied, [ - name, - 0 - ]) + claims = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :registry_claims, [ + name, + 1, + cluster, + key + ]) - assert :ok = - TestCluster.rpc!(context.node_a, TestCluster, :assert_replica_consistent, [name]) + assert Enum.map(claims, &elem(&1, 0)) == [local_owner] + assert :ok = TestCluster.rpc!(context.node_b, TestCluster, :assert_replica_consistent, [name]) end - test "local process DOWN promotes a retained remote claim", context do - %{name: name, pid_a: pid_a, pid_b: pid_b} = - establish_hidden_remote_claim(context, :process_down) + test "nodedown discards deferred registry reprojection state", context do + name = unique_name(:nodedown_deferred_projection) - true = TestCluster.rpc!(context.node_a, Process, :exit, [pid_a, :kill]) - TestCluster.flush_shards(context.node_a, name) + opts = [ + name: name, + shards: 2, + replica_transport: Group.TestReplicaTransport, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ] - assert [^pid_b] = - TestCluster.rpc!(context.node_a, Group.Replica.Data, :registry_claims, [ - name, - 0, - nil, - "hidden/process_down" - ]) - |> Enum.map(&elem(&1, 0)) + start_group_on_peers(context.peers, opts) - assert {^pid_b, %{rank: 1}} = - TestCluster.rpc!(context.node_a, Group, :lookup, [name, "hidden/process_down"]) + target_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) - assert :ok = - TestCluster.rpc!(context.node_a, TestCluster, :assert_replica_consistent, [name]) + key = "nodedown/deferred-projection" + + :ok = + TestCluster.rpc!(context.node_b, TestCluster, :put_pending_registry_reprojection, [ + target_lane, + context.node_a, + nil, + key + ]) + + send(target_lane, {:nodedown, context.node_a}) + + TestCluster.assert_eventually(fn -> + state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_lane]) + not Map.has_key?(state.pending_registry_reprojections, context.node_a) + end) + + assert TestCluster.rpc!(context.node_b, Process, :alive?, [target_lane]) end - test "local cluster disconnect removes a claimless legacy registry row", context do - name = unique_name(:claimless_disconnect) - cluster = "legacy-slice" + test "exact authority installation atomically projects shared clusters", context do + name = unique_name(:atomic_authority_projection) + cluster = "authority/install-race" opts = [ name: name, - shards: 1, + shards: 2, replicated_anti_entropy_interval: 60_000, replicated_peer_lease_timeout: 120_000 ] - {:ok, _pid} = TestCluster.start_group(context.node_a, opts) - :ok = TestCluster.rpc!(context.node_a, Group, :connect, [name, cluster]) + start_group_on_peers(context.peers, opts) - owner = - TestCluster.spawn_register_in_cluster( - context.node_a, - name, - "legacy/claimless", - %{legacy: true}, - cluster - ) + TestCluster.assert_eventually(fn -> + Enum.sort(TestCluster.rpc!(context.node_b, Group, :nodes, [name])) == + Enum.sort([context.node_a, context.node_c]) + end) - assert ["legacy/claimless"] = - TestCluster.rpc!( - context.node_a, - Group.Replica.Data, - :purge_registry_claims_for_cluster, - [name, 0, cluster] - ) + local_epoch = make_ref() - assert {^owner, %{legacy: true}} = - TestCluster.rpc!(context.node_a, Group, :lookup, [ - name, - "legacy/claimless", - [cluster: cluster] - ]) + local_epochs_table = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :local_cluster_epochs_table, [name]) - :ok = TestCluster.rpc!(context.node_a, Group, :disconnect, [name, cluster]) + true = + TestCluster.rpc!(context.node_b, :ets, :insert, [ + local_epochs_table, + {cluster, local_epoch} + ]) - assert nil == - TestCluster.rpc!(context.node_a, Group, :lookup, [ - name, - "legacy/claimless", - [cluster: cluster] - ]) + :ok = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :add_cluster_node, [ + name, + [cluster], + context.node_b + ]) + + generation = TestCluster.rpc!(context.node_a, Group.Replica.Data, :generation, [name]) + revision = 1 + remote_epoch = make_ref() + epochs = [{nil, generation}, {cluster, remote_epoch}] + + {_old_generation, _stale_epochs} = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :put_remote_replica_info, [ + name, + 0, + context.node_a, + generation, + revision, + epochs + ]) + + assert Enum.sort(TestCluster.rpc!(context.node_b, Group, :nodes, [name, cluster])) == + Enum.sort([context.node_a, context.node_b]) end - test "expiring one sideband lane keeps the shared node route while other lanes are live", - context do - name = unique_name(:sideband_lane) + test "local cluster activation survives its caller before shard notification", context do + name = unique_name(:interrupted_cluster_activation) + cluster = "cluster/activation-interrupted" opts = [ name: name, - shards: 3, - replica_transport: - {Group.TestTCPTransport, - [connect_timeout: 250, send_timeout: 250, reconnect_interval: 10]}, - replicated_anti_entropy_interval: 60_000, - replicated_peer_lease_timeout: 120_000 + shards: 2, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 5_000 ] start_group_on_peers(context.peers, opts) + :ok = TestCluster.rpc!(context.node_a, Group, :connect, [name, cluster]) - TestCluster.assert_eventually( - fn -> - TestCluster.rpc!(context.node_b, Group.TestTCPTransport, :connected?, [ + TestCluster.assert_eventually(fn -> + not is_nil( + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_cluster_epoch, [ name, - context.node_a + context.node_a, + cluster ]) - end, - timeout: 10_000 - ) - - lane = - TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) - - send(lane, {:replica_authority_removed_local, context.node_a}) - _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [lane]) - Process.sleep(100) + ) + end) - assert TestCluster.rpc!(context.node_b, Group.TestTCPTransport, :connected?, [ - name, - context.node_a - ]) + control = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) - status = TestCluster.rpc!(context.node_b, Group.TestTCPTransport, :status, [name]) - assert context.node_a in status.peers + :ok = TestCluster.rpc!(context.node_b, :sys, :suspend, [control]) - for shard <- [0, 2] do - replica = - TestCluster.rpc!(context.node_b, Process, :whereis, [ - Group.Replica.shard_name(name, shard) + try do + # This is the durable first step of Group.connect/3. Simulate its caller + # disappearing while the queued shard notification cannot run. + [_epoch] = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :activate_local_clusters_durable, [ + name, + [cluster] ]) - send(replica, {:replica_authority_removed_local, context.node_a}) - _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [replica]) + assert Enum.sort(TestCluster.rpc!(context.node_b, Group, :nodes, [name, cluster])) == + Enum.sort([context.node_a, context.node_b]) + after + :ok = TestCluster.rpc!(context.node_b, TestCluster, :resume_if_alive, [control]) end TestCluster.assert_eventually(fn -> - not TestCluster.rpc!(context.node_b, Group.TestTCPTransport, :connected?, [ - name, - context.node_a - ]) and - context.node_a not in TestCluster.rpc!(context.node_b, Group.TestTCPTransport, :status, [ - name - ]).peers + context.node_b in TestCluster.rpc!(context.node_a, Group, :nodes, [name, cluster]) end) end - test "an exact shard-zero hello repairs a missing local authority view", context do - name = unique_name(:control_view_repair) + test "local cluster deactivation survives its caller before shard cleanup", context do + name = unique_name(:interrupted_cluster_deactivation) + cluster = "cluster/deactivation-interrupted" + reg_key = "deactivation/interrupted/registry" + pg_key = "deactivation/interrupted/pg" + opts = [name: name, shards: 2] + + start_group_on_peers(context.peers, opts) + :ok = TestCluster.rpc!(context.node_b, Group, :connect, [name, cluster]) + + owner = + TestCluster.spawn_register_and_join( + context.node_b, + name, + reg_key, + %{kind: :registry}, + pg_key, + %{kind: :pg}, + cluster: cluster + ) + + assert {^owner, %{kind: :registry}} = + TestCluster.rpc!(context.node_b, Group, :lookup, [ + name, + reg_key, + [cluster: cluster] + ]) + + assert [{^owner, %{kind: :pg}}] = + TestCluster.rpc!(context.node_b, Group, :members, [ + name, + pg_key, + [cluster: cluster] + ]) + + lanes = + for shard <- 0..1 do + lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [ + Group.Replica.shard_name(name, shard) + ]) + + :ok = TestCluster.rpc!(context.node_b, :sys, :suspend, [lane]) + lane + end + + try do + # This is the durable first step of Group.disconnect/3. Simulate its + # caller disappearing while no shard can process the queued cleanup. + [_epoch] = + TestCluster.rpc!(context.node_b, Group.Replica.Data, :deactivate_local_clusters_durable, [ + name, + [cluster] + ]) + + refute context.node_b in TestCluster.rpc!(context.node_b, Group, :nodes, [name, cluster]) + after + Enum.each(lanes, fn lane -> + :ok = TestCluster.rpc!(context.node_b, TestCluster, :resume_if_alive, [lane]) + end) + end + + TestCluster.assert_eventually( + fn -> + TestCluster.rpc!(context.node_b, Group, :lookup, [ + name, + reg_key, + [cluster: cluster] + ]) == nil and + TestCluster.rpc!(context.node_b, Group, :members, [ + name, + pg_key, + [cluster: cluster] + ]) == [] and + TestCluster.rpc!(context.node_b, Group, :nodes, [name, cluster]) == [] and + TestCluster.rpc!(context.node_b, Group.Replica.Data, :closed_local_clusters, [name]) == + [] + end, + timeout: 5_000 + ) + end + + test "exact authority immediately re-probes a lane whose hello arrived first", context do + name = unique_name(:authority_reprobes_early_lane) opts = [ name: name, @@ -279,66 +3821,107 @@ defmodule Group.AntiEntropyFaultRegressionTest do replicated_peer_lease_timeout: 120_000 ] - start_group_on_peers(context.peers, opts) + {:ok, _pid} = TestCluster.start_group(context.node_b, opts) + {:ok, _pid} = TestCluster.start_group(context.node_c, opts) - {generation, revision, epochs} = - TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_replica_authority, [name]) + TestCluster.assert_eventually(fn -> + context.node_c in TestCluster.rpc!(context.node_b, Group, :nodes, [name]) + end) - assert generation == - TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_view_generation, [ - name, - 0, - context.node_a - ]) + target_control = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) - :ok = - TestCluster.rpc!(context.node_b, TestCluster, :delete_remote_view_info, [ - name, - 0, - context.node_a - ]) + target_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) - assert nil == - TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_view_generation, [ - name, - 0, - context.node_a - ]) + :ok = TestCluster.rpc!(context.node_b, :sys, :suspend, [target_control]) - assert generation == - TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_generation, [ - name, - context.node_a - ]) + on_exit(fn -> + TestCluster.rpc!(context.node_b, TestCluster, :resume_if_alive, [target_control]) + end) - source = + {:ok, _pid} = TestCluster.start_group(context.node_a, opts) + + source_control = TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) - target = - TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + source_lane = + TestCluster.rpc!(context.node_a, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + {generation, revision, _epochs} = + TestCluster.rpc!(context.node_a, Group.Replica.Data, :local_replica_authority, [name]) + + :ok = TestCluster.rpc!(context.node_a, :sys, :suspend, [source_lane]) + + on_exit(fn -> + TestCluster.rpc!(context.node_a, TestCluster, :resume_if_alive, [source_lane]) + end) + + # Drain any discovery traffic emitted while A was starting, then force the + # lane hello to be processed while shard zero is still suspended. + Process.sleep(100) + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_lane]) send( - target, - {:replica_hello, source, Group.Replica.WireProtocol.version(), generation, revision, epochs, - Group.TestReplicaTransport.id(), Group.TestReplicaTransport.descriptor(name, [])} + target_lane, + {:replica_lane_hello, source_lane, Group.Replica.WireProtocol.version(), generation, + revision, Group.TestReplicaTransport.id(), Group.TestReplicaTransport.descriptor(name, [])} ) - _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target]) + target_lane_state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_lane]) + refute Map.has_key?(target_lane_state.remote_shards, context.node_a) - assert generation == - TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_view_generation, [ - name, - 0, - context.node_a - ]) + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_generation, [ + name, + context.node_a + ]) == nil - assert revision == - TestCluster.rpc!( - context.node_b, - Group.Replica.Data, - :remote_view_cluster_epoch_revision, - [name, 0, context.node_a] - ) + TestCluster.assert_eventually(fn -> + {:messages, messages} = + TestCluster.rpc!(context.node_b, Process, :info, [target_control, :messages]) + + Enum.any?(messages, fn + {:replica_hello, ^source_control, _version, ^generation, ^revision, _epochs, _transport, + _descriptor} -> + true + + _message -> + false + end) + end) + + :ok = TestCluster.rpc!(context.node_b, :sys, :resume, [target_control]) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_view_generation, [ + name, + 1, + context.node_a + ]) == generation + end) + + # The source lane is suspended, so the only way this peer_connect can be + # present is the immediate post-authority re-probe from the target lane. + TestCluster.assert_eventually(fn -> + {:messages, messages} = + TestCluster.rpc!(context.node_a, Process, :info, [source_lane, :messages]) + + Enum.any?(messages, fn + {:peer_connect, ^target_lane, 1, 2, _clusters} -> true + _message -> false + end) + end) + + :ok = TestCluster.rpc!(context.node_a, :sys, :resume, [source_lane]) + + TestCluster.assert_eventually(fn -> + lane_state = TestCluster.rpc!(context.node_b, :sys, :get_state, [target_lane]) + + Map.has_key?(lane_state.remote_shards, context.node_a) and + Map.has_key?(lane_state.peer_last_seen, context.node_a) + end) + + assert :ok = TestCluster.rpc!(context.node_b, TestCluster, :assert_replica_consistent, [name]) end defp establish_hidden_remote_claim(context, suffix) do diff --git a/test/distributed_test.exs b/test/distributed_test.exs index ae8d3c7..3072d0a 100644 --- a/test/distributed_test.exs +++ b/test/distributed_test.exs @@ -2354,14 +2354,27 @@ defmodule Group.DistributedTest do # --- Phase 4: verify all data converged on all nodes --- # Check registrations: every node should see every registration - for {_reg_node, cluster, key} <- reg_pids do + for {reg_node, cluster, key} <- reg_pids do TestCluster.assert_eventually( fn -> Enum.all?(nodes, fn check_node -> TestCluster.rpc!(check_node, Group, :lookup, [name, key, [cluster: cluster]]) != nil end) end, - timeout: 10_000 + timeout: 10_000, + diagnostic: fn -> + Map.new(nodes, fn check_node -> + state = + TestCluster.rpc!( + check_node, + Group.TestCluster, + :replica_registry_replication_state, + [name, cluster, key, reg_node] + ) + + {check_node, state} + end) + end ) end @@ -3702,11 +3715,11 @@ defmodule Group.DistributedTest do assert Enum.map(left_events, & &1.key) |> Enum.sort() == Enum.sort(keys) end - test "partition heal (cluster_state merge) batches new entries into one message" do - peers = TestCluster.start_peers(2) + test "partition-heal delta catch-up batches same-stream entries into one message" do + peers = TestCluster.start_peers(3) on_exit(fn -> TestCluster.stop_peers(peers) end) - [{_, node_a}, {_, node_b}] = peers + [{_, node_a}, {_, node_b}, {_, node_c}] = peers num_shards = 4 name = :"batch_heal_#{System.unique_integer([:positive])}" opts = [name: name, shards: num_shards] @@ -3714,7 +3727,18 @@ defmodule Group.DistributedTest do start_group_on_peers(peers, opts) TestCluster.assert_eventually(fn -> - TestCluster.rpc!(node_a, Group, :nodes, [name]) == [node_b] + Enum.sort(TestCluster.rpc!(node_a, Group, :nodes, [name])) == + Enum.sort([node_b, node_c]) + end) + + survivor_key = "heal_batch/survivor" + survivor = TestCluster.spawn_register(node_c, name, survivor_key, %{owner: :c}) + + TestCluster.assert_eventually(fn -> + match?( + {^survivor, %{owner: :c}}, + TestCluster.rpc!(node_b, Group, :lookup, [name, survivor_key]) + ) end) # Partition @@ -3740,7 +3764,7 @@ defmodule Group.DistributedTest do forwarder = TestCluster.spawn_batch_forwarder(node_b, name, :all, self()) assert_receive {:monitor_ready, ^forwarder}, 1000 - # Reconnect — cluster_state exchange merges all 3 entries in one handler turn + # Reconnect — one contiguous delta run applies all 3 entries in one turn. TestCluster.reconnect_nodes(node_a, node_b) # All 3 :registered events should arrive in a single batch @@ -3748,6 +3772,9 @@ defmodule Group.DistributedTest do reg_events = Enum.filter(events, &(&1.type == :registered)) assert length(reg_events) == 3 assert Enum.map(reg_events, & &1.key) |> Enum.sort() == Enum.sort(keys) + + assert {^survivor, %{owner: :c}} = + TestCluster.rpc!(node_b, Group, :lookup, [name, survivor_key]) end test "Group.connect on existing cluster batches incoming data" do @@ -3784,7 +3811,7 @@ defmodule Group.DistributedTest do assert_receive {:monitor_ready, ^forwarder}, 1000 - # A connects — receives cluster_state with all 3 entries from B + # A connects — stream heads trigger sequenced catch-up for all 3 entries. TestCluster.rpc!(node_a, Group, :connect, [name, "game"]) # All 3 :joined events should arrive in a single batch @@ -4244,7 +4271,7 @@ defmodule Group.DistributedTest do end @tag timeout: 60_000 - test "full authority is installed once on shard zero while incremental control stays sharded" do + test "full and incremental authority install once through shard zero" do peers = TestCluster.start_peers(2) on_exit(fn -> TestCluster.stop_peers(peers) end) @@ -4465,7 +4492,8 @@ defmodule Group.DistributedTest do TestCluster.rpc!(node_b, :erlang, :send, [ shard_name(name, 2), {:replica_heartbeat, a_lane, Group.Replica.WireProtocol.version(), - TestCluster.rpc!(node_a, Group.Replica.Data, :generation, [name]), latest_revision} + TestCluster.rpc!(node_a, Group.Replica.Data, :generation, [name]), latest_revision, + Group.Transport.DistErl.id(), Group.Transport.DistErl.descriptor(name, [])} ]) TestCluster.assert_eventually(fn -> @@ -4513,6 +4541,25 @@ defmodule Group.DistributedTest do node_a in TestCluster.rpc!(node_b, Group, :nodes, [name]) end) + a_generation = + TestCluster.rpc!(node_a, Group.Replica.Data, :generation, [name]) + + Enum.each([1, 2], fn shard_index -> + b_lane = TestCluster.rpc!(node_b, Process, :whereis, [shard_name(name, shard_index)]) + + TestCluster.assert_eventually(fn -> + b_lane_state = TestCluster.rpc!(node_b, :sys, :get_state, [b_lane]) + + Map.has_key?(b_lane_state.peer_last_seen, node_a) and + TestCluster.rpc!( + node_b, + Group.Replica.Data, + :remote_view_generation, + [name, shard_index, node_a] + ) == a_generation + end) + end) + b_control = TestCluster.rpc!(node_b, Process, :whereis, [shard_name(name, 0)]) a_control = TestCluster.rpc!(node_a, Process, :whereis, [shard_name(name, 0)]) :ok = TestCluster.rpc!(node_b, :sys, :suspend, [b_control]) @@ -4573,6 +4620,21 @@ defmodule Group.DistributedTest do node_a in TestCluster.rpc!(node_b, Group, :nodes, [name]) end) + a_lane = TestCluster.rpc!(node_a, Process, :whereis, [shard_name(name, 1)]) + + TestCluster.assert_eventually(fn -> + lane_state = TestCluster.rpc!(node_a, :sys, :get_state, [a_lane]) + + Map.has_key?(lane_state.peer_last_seen, node_b) and + not is_nil( + TestCluster.rpc!(node_a, Group.Replica.Data, :remote_view_generation, [ + name, + 1, + node_b + ]) + ) + end) + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [ name, @@ -4622,11 +4684,10 @@ defmodule Group.DistributedTest do stream_id ]) == 0 - a_lane = TestCluster.rpc!(node_a, Process, :whereis, [shard_name(name, 1)]) - TestCluster.rpc!(node_b, :erlang, :send, [ shard_name(name, 1), - {:replica_heartbeat, a_lane, Group.Replica.WireProtocol.version(), generation, revision} + {:replica_heartbeat, a_lane, Group.Replica.WireProtocol.version(), generation, revision, + Group.TestReplicaTransport.id(), Group.TestReplicaTransport.descriptor(name, [])} ]) TestCluster.assert_eventually(fn -> @@ -4856,6 +4917,42 @@ defmodule Group.DistributedTest do TestCluster.flush_shards(node_b, name) assert TestCluster.rpc!(node_b, Group, :lookup, [name, generation_key]) == nil + # The global stream epoch normally equals its generation, so a replayed + # old frame is rejected by both guards. Rewrite only the epoch to the + # current one to prove that the generation fence independently rejects a + # semantically impossible mixed-generation stream instead of advancing + # its cursor or materializing its rows. + current_generation = + TestCluster.rpc!(node_b, Group.Replica.Data, :remote_generation, [name, node_a]) + + mixed_generation_frames = + Enum.map(generation_frames, fn {target, shard, {:delta_batch, version, runs}} -> + runs = + Enum.map(runs, fn + {{stream_name, origin, old_generation, stream_shard, nil, _old_epoch}, first_seq, + records, advertised_head} -> + stream_id = + {stream_name, origin, old_generation, stream_shard, nil, current_generation} + + {stream_id, first_seq, records, advertised_head} + end) + + {target, shard, {:delta_batch, version, runs}} + end) + + Enum.each(mixed_generation_frames, fn {_target, shard, frame} -> + :ok = + TestCluster.rpc!(node_b, Group.Transport, :incoming, [ + name, + node_a, + shard, + frame + ]) + end) + + TestCluster.flush_shards(node_b, name) + assert TestCluster.rpc!(node_b, Group, :lookup, [name, generation_key]) == nil + new_generation_pid = TestCluster.spawn_register(node_a, name, generation_key, %{old: false}) @@ -5272,10 +5369,11 @@ defmodule Group.DistributedTest do @tag timeout: 120_000 test "concurrent many-cluster controls converge every revision before replica writes" do - peers = TestCluster.start_peers(2) + peers = TestCluster.start_peers(3) on_exit(fn -> TestCluster.stop_peers(peers) end) - [{_, node_a}, {_, node_b}] = peers + [{_, node_a}, {_, node_b}, {_, node_c}] = peers + nodes = [node_a, node_b, node_c] name = :"anti_entropy_many_controls_#{System.unique_integer([:positive])}" clusters = for i <- 1..512, do: "tenant/#{i}" @@ -5294,15 +5392,15 @@ defmodule Group.DistributedTest do end) tasks = - for node <- [node_a, node_b] do + for node <- nodes do Task.async(fn -> TestCluster.connect_many_concurrently(node, name, clusters) end) end - assert [:ok, :ok] = Task.await_many(tasks, 60_000) + assert [:ok, :ok, :ok] = Task.await_many(tasks, 60_000) TestCluster.assert_eventually( fn -> - Enum.all?([node_a, node_b], fn node -> + Enum.all?(nodes, fn node -> expected = MapSet.new([nil | clusters]) actual = @@ -5312,8 +5410,9 @@ defmodule Group.DistributedTest do MapSet.subset?(expected, actual) end) and Enum.all?(clusters, fn cluster -> - length(TestCluster.rpc!(node_a, Group, :nodes, [name, cluster])) == 2 and - length(TestCluster.rpc!(node_b, Group, :nodes, [name, cluster])) == 2 + Enum.all?(nodes, fn node -> + length(TestCluster.rpc!(node, Group, :nodes, [name, cluster])) == 3 + end) end) end, timeout: 30_000, @@ -5330,7 +5429,12 @@ defmodule Group.DistributedTest do TestCluster.assert_eventually( fn -> - TestCluster.rpc!(node_b, Group.TestCluster, :registry_entries_present?, [name, entries]) + Enum.all?([node_b, node_c], fn node -> + TestCluster.rpc!(node, Group.TestCluster, :registry_entries_present?, [ + name, + entries + ]) + end) end, timeout: 30_000, interval: 100 @@ -5364,26 +5468,43 @@ defmodule Group.DistributedTest do node_a in TestCluster.rpc!(node_b, Group, :nodes, [name]) end) + TestCluster.rpc!(node_a, Group, :connect, [name, "game"]) + TestCluster.rpc!(node_b, Group, :connect, [name, "game"]) + + TestCluster.assert_eventually(fn -> + node_a in TestCluster.rpc!(node_b, Group, :nodes, [name, "game"]) + end) + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) key = "anti-entropy/mixed-generation" meta = %{forged: true} - pid = TestCluster.spawn_register(node_a, name, key, meta) + pid = TestCluster.spawn_register_in_cluster(node_a, name, key, meta, "game") TestCluster.flush_shards(node_a, name) current_generation = TestCluster.rpc!(node_a, Group.Replica.Data, :generation, [name]) + current_epoch = + TestCluster.rpc!(node_a, Group.Replica.Data, :local_cluster_epoch, [name, "game"]) + + forged_generation = + current_generation + |> elem(0) + |> Kernel.-(1) + |> max(1) + |> then(&{&1, make_ref()}) + stream_id = Group.Replica.WireProtocol.stream_id( name, node_a, - make_ref(), + forged_generation, 0, - nil, - current_generation + "game", + current_epoch ) - mutation = {:register, nil, key, pid, meta, System.monotonic_time(), node_a} + mutation = {:register, "game", key, pid, meta, System.monotonic_time(), node_a} :ok = TestCluster.rpc!(node_b, Group.Transport, :incoming, [ @@ -5395,7 +5516,8 @@ defmodule Group.DistributedTest do ]) TestCluster.flush_shards(node_b, name) - assert TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + + assert TestCluster.rpc!(node_b, Group, :lookup, [name, key, [cluster: "game"]]) == nil assert TestCluster.rpc!(node_b, Group.Replica.Data, :replica_cursor, [name, 0, stream_id]) == 0 @@ -5587,7 +5709,14 @@ defmodule Group.DistributedTest do a_control = TestCluster.rpc!(node_a, Process, :whereis, [shard_name(name, 0)]) b_control = TestCluster.rpc!(node_b, Process, :whereis, [shard_name(name, 0)]) b_lane = TestCluster.rpc!(node_b, Process, :whereis, [shard_name(name, 1)]) - new_generation = make_ref() + + {old_generation_counter, _old_generation_identity} = + TestCluster.rpc!(node_a, Group.Replica.Data, :generation, [name]) + + # Generation counters are ordered only within the origin BEAM. Construct + # the injected restart generation relative to node A's actual authority; + # using this test runner's independent counter makes the test flaky. + new_generation = {old_generation_counter + 1, make_ref()} new_key = Enum.find(Stream.iterate(0, &(&1 + 1)), fn suffix -> diff --git a/test/formal/AuthorityHint.cfg b/test/formal/AuthorityHint.cfg new file mode 100644 index 0000000..b9d2036 --- /dev/null +++ b/test/formal/AuthorityHint.cfg @@ -0,0 +1,17 @@ +SPECIFICATION Spec + +CONSTANTS + MaxGeneration = 2 + MaxRevision = 2 + +INVARIANTS + TypeOK + EnabledLaneMatchesExactAuthority + ExactAuthorityOwnsItsRoute + NoLaneWithoutExactAuthority + HintRequiresPriorExactAuthority + AppliedAuthorityRespectsItsFence + +PROPERTIES + UnresolvedHintIsBounded + DelayedCleanupCompletes diff --git a/test/formal/AuthorityHint.tla b/test/formal/AuthorityHint.tla new file mode 100644 index 0000000..5f851b1 --- /dev/null +++ b/test/formal/AuthorityHint.tla @@ -0,0 +1,249 @@ +---------------------------- MODULE AuthorityHint ---------------------------- +EXTENDS Integers, TLC + +(* +Finite model of the shared authority-hint fence. A heartbeat on any replica +lane may observe a newer generation/revision before shard zero receives its +exact hello. The hint must disable every old lane, reject delayed old exact +and lane-view installs, retain a bounded repair/lease obligation, and allow a +stale cleanup caller to finish without deleting a rediscovered peer's route. +*) + +CONSTANTS MaxGeneration, MaxRevision + +ASSUME /\ MaxGeneration >= 2 + /\ MaxRevision >= 1 + +Generations == 1..MaxGeneration +Revisions == 0..MaxRevision + +VARIABLES exactGeneration, + exactRevision, + appliedRevision, + hintedGeneration, + hintedRevision, + viewGeneration, + viewExactRevision, + viewObservedRevision, + laneEnabled, + repairPending, + leasePending, + routeGeneration, + cleanupPending + +vars == + <> + +Init == + /\ exactGeneration = 1 + /\ exactRevision = 0 + /\ appliedRevision = 0 + /\ hintedGeneration = 1 + /\ hintedRevision = 0 + /\ viewGeneration = 1 + /\ viewExactRevision = 0 + /\ viewObservedRevision = 0 + /\ laneEnabled = TRUE + /\ repairPending = FALSE + /\ leasePending = TRUE + /\ routeGeneration = 1 + /\ cleanupPending = FALSE + +Newer(generation, revision, currentGeneration, currentRevision) == + \/ generation > currentGeneration + \/ /\ generation = currentGeneration + /\ revision > currentRevision + +NewerOrEqual(generation, revision, currentGeneration, currentRevision) == + \/ Newer(generation, revision, currentGeneration, currentRevision) + \/ /\ generation = currentGeneration + /\ revision = currentRevision + +ObserveHint(generation, revision) == + /\ exactGeneration > 0 + /\ hintedGeneration > 0 + /\ generation \in Generations + /\ revision \in Revisions + /\ Newer(generation, revision, hintedGeneration, hintedRevision) + /\ hintedGeneration' = generation + /\ hintedRevision' = revision + /\ laneEnabled' = FALSE + /\ repairPending' = TRUE + /\ leasePending' = TRUE + /\ UNCHANGED + <> + +InstallExact(generation, revision) == + /\ generation \in Generations + /\ revision \in Revisions + /\ NewerOrEqual(generation, revision, hintedGeneration, hintedRevision) + /\ \/ exactGeneration = 0 + \/ NewerOrEqual(generation, revision, exactGeneration, exactRevision) + /\ exactGeneration' = generation + /\ exactRevision' = revision + /\ appliedRevision' = revision + /\ hintedGeneration' = generation + /\ hintedRevision' = revision + /\ routeGeneration' = generation + /\ laneEnabled' = FALSE + /\ repairPending' = TRUE + /\ leasePending' = TRUE + /\ UNCHANGED + <> + +(* +Shard zero may install a contiguous incremental authority change without +moving the last exact-snapshot revision. The expected revision is compared to +the shared applied revision and hint in the same serialized transition; a +heartbeat racing it forward therefore disables this action instead of allowing +partial epoch rows to become authoritative. +*) +InstallIncremental(expected, revision) == + /\ exactGeneration > 0 + /\ expected \in Revisions + /\ revision \in Revisions + /\ revision = expected + 1 + /\ appliedRevision = expected + /\ hintedGeneration = exactGeneration + /\ hintedRevision = expected + /\ appliedRevision' = revision + /\ hintedRevision' = revision + /\ laneEnabled' = FALSE + /\ repairPending' = TRUE + /\ leasePending' = TRUE + /\ UNCHANGED + <> + +InstallLaneView == + /\ exactGeneration > 0 + /\ exactGeneration = hintedGeneration + /\ appliedRevision = hintedRevision + /\ viewGeneration' = exactGeneration + /\ viewExactRevision' = exactRevision + /\ viewObservedRevision' = hintedRevision + /\ laneEnabled' = TRUE + /\ repairPending' = FALSE + /\ leasePending' = TRUE + /\ UNCHANGED + <> + +QueueCleanup == + /\ ~cleanupPending + /\ cleanupPending' = TRUE + /\ UNCHANGED + <> + +RunCleanup == + /\ cleanupPending + /\ cleanupPending' = FALSE + /\ routeGeneration' = + IF exactGeneration = 0 /\ hintedGeneration = 0 + THEN 0 + ELSE routeGeneration + /\ UNCHANGED + <> + +RetireUnresolvedHint == + /\ repairPending + /\ leasePending + /\ exactGeneration' = 0 + /\ exactRevision' = 0 + /\ appliedRevision' = 0 + /\ hintedGeneration' = 0 + /\ hintedRevision' = 0 + /\ viewGeneration' = 0 + /\ viewExactRevision' = 0 + /\ viewObservedRevision' = 0 + /\ laneEnabled' = FALSE + /\ repairPending' = FALSE + /\ leasePending' = FALSE + /\ routeGeneration' = 0 + /\ UNCHANGED cleanupPending + +DropPeer == + /\ exactGeneration' = 0 + /\ exactRevision' = 0 + /\ appliedRevision' = 0 + /\ hintedGeneration' = 0 + /\ hintedRevision' = 0 + /\ viewGeneration' = 0 + /\ viewExactRevision' = 0 + /\ viewObservedRevision' = 0 + /\ laneEnabled' = FALSE + /\ repairPending' = FALSE + /\ leasePending' = FALSE + /\ routeGeneration' = 0 + /\ UNCHANGED cleanupPending + +Next == + \/ \E generation \in Generations, revision \in Revisions : + ObserveHint(generation, revision) + \/ \E generation \in Generations, revision \in Revisions : + InstallExact(generation, revision) + \/ \E expected \in Revisions, revision \in Revisions : + InstallIncremental(expected, revision) + \/ InstallLaneView + \/ QueueCleanup + \/ RunCleanup + \/ RetireUnresolvedHint + \/ DropPeer + +TypeOK == + /\ exactGeneration \in 0..MaxGeneration + /\ exactRevision \in Revisions + /\ appliedRevision \in Revisions + /\ hintedGeneration \in 0..MaxGeneration + /\ hintedRevision \in Revisions + /\ viewGeneration \in 0..MaxGeneration + /\ viewExactRevision \in Revisions + /\ viewObservedRevision \in Revisions + /\ laneEnabled \in BOOLEAN + /\ repairPending \in BOOLEAN + /\ leasePending \in BOOLEAN + /\ routeGeneration \in 0..MaxGeneration + /\ cleanupPending \in BOOLEAN + +EnabledLaneMatchesExactAuthority == + laneEnabled => + /\ exactGeneration > 0 + /\ viewGeneration = exactGeneration + /\ viewExactRevision = exactRevision + /\ viewObservedRevision = hintedRevision + /\ hintedGeneration = exactGeneration + /\ hintedRevision = appliedRevision + +ExactAuthorityOwnsItsRoute == + exactGeneration > 0 => routeGeneration = exactGeneration + +NoLaneWithoutExactAuthority == + exactGeneration = 0 => ~laneEnabled + +HintRequiresPriorExactAuthority == + hintedGeneration > 0 => exactGeneration > 0 + +AppliedAuthorityRespectsItsFence == + /\ (exactGeneration > 0 => exactRevision <= appliedRevision) + /\ (hintedGeneration = exactGeneration => appliedRevision <= hintedRevision) + +UnresolvedHintIsBounded == repairPending ~> ~repairPending + +DelayedCleanupCompletes == cleanupPending ~> ~cleanupPending + +Spec == + /\ Init + /\ [][Next]_vars + /\ WF_vars(RetireUnresolvedHint) + /\ WF_vars(RunCleanup) + +============================================================================= diff --git a/test/formal/AuthorityProjection.cfg b/test/formal/AuthorityProjection.cfg new file mode 100644 index 0000000..519f335 --- /dev/null +++ b/test/formal/AuthorityProjection.cfg @@ -0,0 +1,10 @@ +SPECIFICATION Spec + +INVARIANTS + TypeOK + SelfProjectionIsExact + SettledRemoteProjectionIsExact + InactiveRowsAreBeingCleaned + +PROPERTY + CleanupSurvivesCaller diff --git a/test/formal/AuthorityProjection.tla b/test/formal/AuthorityProjection.tla new file mode 100644 index 0000000..bacb93b --- /dev/null +++ b/test/formal/AuthorityProjection.tla @@ -0,0 +1,117 @@ +------------------------ MODULE AuthorityProjection ------------------------ +EXTENDS TLC + +(* +Finite model of the serialized boundary between named-cluster authority and +its routing/materialized state. The API caller may disappear after either +durable lifecycle mutation. Activation must already project every exact remote +authority; deactivation must leave cleanup enabled independently of the caller. +*) + +VARIABLES localActive, + selfRoute, + remoteExact, + remoteRoute, + rows, + cleanupPending, + callerAlive + +vars == + <> + +Init == + /\ localActive = FALSE + /\ selfRoute = FALSE + /\ remoteExact = FALSE + /\ remoteRoute = FALSE + /\ rows = FALSE + /\ cleanupPending = FALSE + /\ callerAlive = TRUE + +InstallRemote(exact) == + /\ remoteExact' = exact + /\ remoteRoute' = localActive /\ exact + /\ UNCHANGED + <> + +Activate == + /\ callerAlive = TRUE + /\ localActive = FALSE + /\ cleanupPending = FALSE + /\ localActive' = TRUE + /\ selfRoute' = TRUE + /\ remoteRoute' = remoteExact + /\ UNCHANGED <> + +Write == + /\ callerAlive = TRUE + /\ localActive = TRUE + /\ rows' = TRUE + /\ UNCHANGED + <> + +Deactivate == + /\ callerAlive = TRUE + /\ localActive = TRUE + /\ localActive' = FALSE + /\ selfRoute' = FALSE + /\ cleanupPending' = TRUE + /\ UNCHANGED <> + +Cleanup == + /\ cleanupPending = TRUE + /\ cleanupPending' = FALSE + /\ remoteRoute' = FALSE + /\ rows' = FALSE + /\ UNCHANGED <> + +CrashCaller == + /\ callerAlive = TRUE + /\ callerAlive' = FALSE + /\ UNCHANGED + <> + +NewCaller == + /\ callerAlive = FALSE + /\ callerAlive' = TRUE + /\ UNCHANGED + <> + +Next == + \/ \E exact \in BOOLEAN : InstallRemote(exact) + \/ Activate + \/ Write + \/ Deactivate + \/ Cleanup + \/ CrashCaller + \/ NewCaller + +TypeOK == + /\ localActive \in BOOLEAN + /\ selfRoute \in BOOLEAN + /\ remoteExact \in BOOLEAN + /\ remoteRoute \in BOOLEAN + /\ rows \in BOOLEAN + /\ cleanupPending \in BOOLEAN + /\ callerAlive \in BOOLEAN + +SelfProjectionIsExact == selfRoute = localActive + +SettledRemoteProjectionIsExact == + cleanupPending = FALSE => remoteRoute = (localActive /\ remoteExact) + +InactiveRowsAreBeingCleaned == + (localActive = FALSE /\ rows = TRUE) => cleanupPending = TRUE + +CleanupSurvivesCaller == cleanupPending = TRUE ~> cleanupPending = FALSE + +Spec == + /\ Init + /\ [][Next]_vars + /\ WF_vars(Cleanup) + +============================================================================= diff --git a/test/formal/README.md b/test/formal/README.md index d34b5ea..0626af3 100644 --- a/test/formal/README.md +++ b/test/formal/README.md @@ -25,6 +25,20 @@ generation and an active bit, so an inactive hello fences even a same-epoch snapshot. After healing, fair repair must either install only the current generation or erase every row and authority reference for the absent peer. +`AuthorityProjection.tla` models concurrent exact remote installs, local named +cluster activation/deactivation, materialized rows, and a lifecycle caller that +may disappear after the durable mutation. Its safety invariants require local +activation and shared routing to project authority consistently; its liveness +property requires queued close cleanup to finish without the original caller. + +`AuthorityHint.tla` models the cross-lane fence created when a heartbeat or lane +hello observes a newer authority before the exact hello arrives. It checks that +delayed old exact/view installs cannot re-enable a lane, unresolved hints retain +a bounded lease/repair obligation, contiguous incremental authority is installed +only from the currently hinted/applied revision, unknown post-retirement hints +cannot establish authority, and delayed retirement cleanup cannot erase a +rediscovered generation's route. + The default TLC configuration uses three nodes: one origin and two independent receivers. The origin has one key, a two-record stream, a one-record oplog, and the system retains one arbitrary network frame. This forces delta repair, @@ -64,8 +78,8 @@ TLC proves the listed invariants and liveness property for the configured finite instance, not for arbitrary unbounded node and key sets. Larger models should be run periodically by increasing `Nodes`, `Origins`, `Keys`, `MaxSeq`, `OplogBound`, and `MaxMessages`. `check_matrix.sh` runs the protocol, snapshot -assembly, and peer-eviction models; set `TLA_EXTENDED=1` for the larger -anti-entropy configuration. +assembly, peer-eviction, authority-projection, and authority-hint models; set +`TLA_EXTENDED=1` for the larger anti-entropy configuration. The checked three-node default explores 1,835,826 states, finds 490,236 distinct states to a depth of 30, and completes in roughly 1 minute 40 seconds @@ -78,6 +92,15 @@ The peer-eviction model explores 1,527,116 states, finds 238,120 distinct states to a depth of 26, and completes in roughly 20 seconds on the development machine used for validation. +The authority-projection model explores 71 states, finds 24 distinct states to +a depth of 6, and completes in under a second. Its small state space is +intentional: it exhaustively crosses the two authority directions, lifecycle +caller loss/replacement, writes, and independently fair close cleanup. + +The authority-hint model explores 2,989 states, finds 428 distinct states to a +depth of 9, and completes in roughly one second. It separates the last exact +revision from the complete applied revision and highest persisted hint. + The extended two-key, three-sequence model explores 127,557,634 states, finds 32,238,304 distinct states to a depth of 34, and completes in roughly two hours on the development machine used for validation. diff --git a/test/formal/check_matrix.sh b/test/formal/check_matrix.sh index cd60e2e..19521e9 100755 --- a/test/formal/check_matrix.sh +++ b/test/formal/check_matrix.sh @@ -16,6 +16,8 @@ run_check() { run_check GroupAntiEntropy GroupAntiEntropy run_check SnapshotAssembly SnapshotAssembly run_check PeerEviction PeerEviction +run_check AuthorityProjection AuthorityProjection +run_check AuthorityHint AuthorityHint if [[ "${TLA_EXTENDED:-0}" == "1" ]]; then run_check GroupAntiEntropy GroupAntiEntropyExtended diff --git a/test/group_test.exs b/test/group_test.exs index 3f1b1ac..c0679a1 100644 --- a/test/group_test.exs +++ b/test/group_test.exs @@ -2,22 +2,6 @@ defmodule GroupTest.ExtractMeta do def strip(meta), do: Map.take(meta, [:public]) end -defmodule GroupTest.ResolveRegistryConflict do - def pick( - _name, - _key, - {local_pid, _local_meta, _local_time}, - {remote_pid, _remote_meta, _remote_time}, - winner - ) do - case winner do - :local -> local_pid - :remote -> remote_pid - :none -> :none - end - end -end - defmodule GroupTest do use ExUnit.Case, async: true @@ -49,6 +33,52 @@ defmodule GroupTest do end end + describe "replica ingress fairness" do + test "an oversized incoming batch yields to an already queued local write", %{name: name} do + key = + 1..1_000 + |> Enum.map(&"ingress-fairness/#{&1}") + |> Enum.find(&(Group.Replica.shard_index_for(nil, &1, 4) == 0)) + + shard = Process.whereis(Group.Replica.shard_name(name, 0)) + :ok = :sys.suspend(shard) + + on_exit(fn -> + if Process.alive?(shard), do: :erlang.trace(shard, false, [:call]) + :erlang.trace_pattern({Group.Replica, :handle_replica_message, 3}, false, [:local]) + Group.TestCluster.resume_if_alive(shard) + end) + + parent = self() + owner = spawn(fn -> replica_ingress_fairness_owner(parent) end) + + :erlang.trace(shard, true, [:call, {:tracer, owner}]) + :erlang.trace_pattern({Group.Replica, :handle_replica_message, 3}, true, [:local]) + + source_node = :"ingress-source@test" + batch_size = 65 + messages = List.duplicate({:malformed_replica_message, make_ref()}, batch_size) + + assert :ok = Group.Transport.incoming_batch(name, source_node, 0, messages) + + epoch = Group.Replica.Data.local_cluster_epoch(name, nil) + send(owner, {:write, shard, {:register, nil, epoch, key, owner, %{kind: :local}}}) + + wait_until(fn -> + match?( + {:message_queue_len, length} when length >= 2, + Process.info(shard, :message_queue_len) + ) + end) + + :ok = :sys.resume(shard) + + assert_receive {:local_write_finished, ^owner, :ok, calls_before_local}, 5_000 + assert calls_before_local < batch_size + assert match?({^owner, %{kind: :local}}, Group.lookup(name, key)) + end + end + describe "monitor_generation/1" do test "notifies long-lived owners when local membership storage exits", %{name: name} do assert {:ok, generation_pid, monitor_ref} = Group.monitor_generation(name) @@ -198,6 +228,42 @@ defmodule GroupTest do end end + describe "named-cluster mutation fencing" do + test "a shard rejects registry and PG writes after their cluster epoch retires", %{name: name} do + cluster = "retired/#{System.unique_integer([:positive])}" + registry_key = "retired/registry" + pg_key = "retired/pg" + + assert :ok = Group.connect(name, cluster) + old_epoch = Group.Replica.Data.local_cluster_epoch(name, cluster) + assert :ok = Group.disconnect(name, cluster) + assert :ok = Group.connect(name, cluster) + refute Group.Replica.Data.local_cluster_epoch(name, cluster) == old_epoch + + registry_shard = Group.Replica.shard_for(name, cluster, registry_key) + pg_shard = Group.Replica.shard_for(name, cluster, pg_key) + + registry_result = + Group.Replica.local_request( + registry_shard, + {:register, cluster, old_epoch, registry_key, self(), %{stale: true}}, + 5_000 + ) + + pg_result = + Group.Replica.local_request( + pg_shard, + {:join, cluster, old_epoch, pg_key, self(), %{stale: true}}, + 5_000 + ) + + assert registry_result == {:error, :stale_cluster_epoch} + assert pg_result == {:error, :stale_cluster_epoch} + assert Group.lookup(name, registry_key, cluster: cluster) == nil + assert Group.members(name, pg_key, cluster: cluster) == [] + end + end + describe "register/unregister" do test "register makes process discoverable via lookup", %{name: name} do key = "user/#{System.unique_integer([:positive])}" @@ -767,9 +833,20 @@ defmodule GroupTest do disconnect_queued? = Enum.any?(messages, fn - {:group_local_request, _alias, {:cluster_disconnect, [^cluster]}} -> true - {:group_local_request, _caller, _ref, {:cluster_disconnect, [^cluster]}} -> true - _ -> false + {:group_local_request, _alias, {:cluster_disconnect, [^cluster]}} -> + true + + {:group_local_request, _alias, {:cluster_disconnect, [^cluster], _epochs}} -> + true + + {:group_local_request, _caller, _ref, {:cluster_disconnect, [^cluster]}} -> + true + + {:group_local_request, _caller, _ref, {:cluster_disconnect, [^cluster], _epochs}} -> + true + + _ -> + false end) not Group.connected?(name, cluster) and disconnect_queued? @@ -1404,296 +1481,75 @@ defmodule GroupTest do assert Group.members(name, key, cluster: cluster) == [{self(), %{epoch: :new}}] assert :ok = Group.TestCluster.assert_replica_consistent(name) end - end - - describe "local request fairness" do - test "local register gets a turn ahead of replicated registry backlog" do - name = - start_single_shard_group( - replicated_registry_receiver_buffer_size: 1, - replicated_registry_receiver_flush_interval: 60_000 - ) - - shard = suspend_only_shard(name) - remote_pid = spawn_forever() - local_key = "fair/register/local/#{System.unique_integer([:positive])}" - backlog_prefix = "fair/register/backlog/#{System.unique_integer([:positive])}" - - on_exit(fn -> - kill_if_alive(remote_pid) - end) - - enqueue_replicated_registry_backlog(shard, backlog_prefix, remote_pid, 1_000) - - caller = - spawn_requester( - fn -> - Group.register(name, local_key, %{local: true}) - end, - :local_register_result - ) - - on_exit(fn -> kill_if_alive(caller) end) - Process.sleep(20) - :ok = :sys.resume(shard) - - assert_receive {:local_register_result, ^caller, :ok}, 1_000 - assert shard_message_queue_len(shard) > 0 - assert Group.lookup(name, local_key) == {caller, %{local: true}} - wait_until(fn -> shard_message_queue_len(shard) == 0 end, 5_000) - end - - test "local join gets a turn ahead of replicated registry backlog" do - name = - start_single_shard_group( - replicated_registry_receiver_buffer_size: 1, - replicated_registry_receiver_flush_interval: 60_000 - ) - - shard = suspend_only_shard(name) - remote_pid = spawn_forever() - local_key = "fair/join-registry/local/#{System.unique_integer([:positive])}" - backlog_prefix = "fair/join-registry/backlog/#{System.unique_integer([:positive])}" - - on_exit(fn -> - kill_if_alive(remote_pid) - end) - - enqueue_replicated_registry_backlog(shard, backlog_prefix, remote_pid, 1_000) - - caller = - spawn_requester( - fn -> - Group.join(name, local_key, %{local: true}) - end, - :local_join_result - ) - - on_exit(fn -> kill_if_alive(caller) end) - Process.sleep(20) - :ok = :sys.resume(shard) - - assert_receive {:local_join_result, ^caller, :ok}, 1_000 - assert shard_message_queue_len(shard) > 0 - assert Group.members(name, local_key) == [{caller, %{local: true}}] - wait_until(fn -> shard_message_queue_len(shard) == 0 end, 5_000) - end - - test "local join gets a turn ahead of replicated PG backlog" do - name = - start_single_shard_group( - replicated_pg_receiver_buffer_size: 1, - replicated_pg_receiver_flush_interval: 60_000 - ) - - shard = suspend_only_shard(name) - remote_pid = spawn_forever() - local_key = "fair/join/local/#{System.unique_integer([:positive])}" - backlog_prefix = "fair/join/backlog/#{System.unique_integer([:positive])}" - - on_exit(fn -> - kill_if_alive(remote_pid) - end) - - enqueue_replicated_pg_backlog(shard, backlog_prefix, remote_pid, 1_000) - - caller = - spawn_requester( - fn -> - Group.join(name, local_key, %{local: true}) - end, - :local_join_result - ) - - on_exit(fn -> kill_if_alive(caller) end) - Process.sleep(20) - :ok = :sys.resume(shard) - - assert_receive {:local_join_result, ^caller, :ok}, 1_000 - assert shard_message_queue_len(shard) > 0 - assert Group.members(name, local_key) == [{caller, %{local: true}}] - wait_until(fn -> shard_message_queue_len(shard) == 0 end, 5_000) - end - - test "local connect and disconnect each get a turn ahead of replicated PG backlog" do - name = - start_single_shard_group( - replicated_pg_receiver_buffer_size: 1, - replicated_pg_receiver_flush_interval: 60_000 - ) - - shard = suspend_only_shard(name) - remote_pid = spawn_forever() - cluster = "fair/connect/#{System.unique_integer([:positive])}" - connect_prefix = "fair/connect/backlog/#{System.unique_integer([:positive])}" - - on_exit(fn -> - kill_if_alive(remote_pid) - end) - - enqueue_replicated_pg_backlog(shard, connect_prefix, remote_pid, 1_000) - - connect_caller = - spawn_requester( - fn -> - Group.connect(name, cluster) - end, - :local_connect_result - ) - - on_exit(fn -> kill_if_alive(connect_caller) end) - Process.sleep(20) - :ok = :sys.resume(shard) - - assert_receive {:local_connect_result, ^connect_caller, :ok}, 1_000 - assert shard_message_queue_len(shard) > 0 - assert Group.connected?(name, cluster) - wait_until(fn -> shard_message_queue_len(shard) == 0 end, 5_000) - - :ok = :sys.suspend(shard) - - disconnect_prefix = "fair/disconnect/backlog/#{System.unique_integer([:positive])}" - - enqueue_replicated_pg_backlog(shard, disconnect_prefix, remote_pid, 1_000) - - disconnect_caller = - spawn_requester( - fn -> - Group.disconnect(name, cluster) - end, - :local_disconnect_result - ) - - on_exit(fn -> kill_if_alive(disconnect_caller) end) - Process.sleep(20) - :ok = :sys.resume(shard) - - assert_receive {:local_disconnect_result, ^disconnect_caller, :ok}, 1_000 - assert shard_message_queue_len(shard) > 0 - refute Group.connected?(name, cluster) - wait_until(fn -> shard_message_queue_len(shard) == 0 end, 5_000) - end - - test "fairness preserves FIFO within the local request lane" do - name = - start_single_shard_group( - replicated_pg_receiver_buffer_size: 1, - replicated_pg_receiver_flush_interval: 60_000 - ) - - shard = suspend_only_shard(name) - remote_pid = spawn_forever() - backlog_prefix = "fair/fifo/backlog/#{System.unique_integer([:positive])}" - key1 = "fair/fifo/local/#{System.unique_integer([:positive])}/1" - key2 = "fair/fifo/local/#{System.unique_integer([:positive])}/2" - - on_exit(fn -> - kill_if_alive(remote_pid) - end) - - enqueue_replicated_pg_backlog(shard, backlog_prefix, remote_pid, 1_000) - - caller1 = - spawn_requester( - fn -> - Group.join(name, key1, %{order: 1}) - end, - :fifo_result - ) - Process.sleep(20) - - caller2 = - spawn_requester( - fn -> - Group.join(name, key2, %{order: 2}) - end, - :fifo_result - ) - - on_exit(fn -> - kill_if_alive(caller1) - kill_if_alive(caller2) - end) + test "a delayed duplicate disconnect cannot purge a reconnected epoch" do + name = start_single_shard_group() + cluster = "duplicate-disconnect/#{System.unique_integer([:positive])}" + reg_key = "duplicate-disconnect/registry/#{System.unique_integer([:positive])}" + pg_key = "duplicate-disconnect/pg/#{System.unique_integer([:positive])}" - Process.sleep(20) - :ok = :sys.resume(shard) + :ok = Group.connect(name, cluster) + old_epoch = Group.Replica.Data.local_cluster_epoch(name, cluster) + :ok = Group.disconnect(name, cluster) + :ok = Group.connect(name, cluster) + :ok = Group.register(name, reg_key, %{epoch: :new}, cluster: cluster) + :ok = Group.join(name, pg_key, %{epoch: :new}, cluster: cluster) + + # A concurrent second disconnect that observed the already-closed local + # epoch can queue either an unfenced request or the completed old epoch. + # Model both arriving only after reconnect. + for stale_epoch <- [nil, old_epoch] do + assert :ok = + Group.Replica.local_request( + Group.Replica.shard_name(name, 0), + {:cluster_disconnect, [cluster], [{cluster, stale_epoch}]}, + 5_000 + ) + + assert Group.connected?(name, cluster) + assert Group.lookup(name, reg_key, cluster: cluster) == {self(), %{epoch: :new}} + assert Group.members(name, pg_key, cluster: cluster) == [{self(), %{epoch: :new}}] + end - assert_receive {:fifo_result, ^caller1, :ok}, 1_000 - assert_receive {:fifo_result, ^caller2, :ok}, 1_000 - assert shard_message_queue_len(shard) > 0 - assert Group.members(name, key1) == [{caller1, %{order: 1}}] - assert Group.members(name, key2) == [{caller2, %{order: 2}}] - wait_until(fn -> shard_message_queue_len(shard) == 0 end, 5_000) + assert :ok = Group.TestCluster.assert_replica_consistent(name) end + end - test "configurable local fairness quota drains multiple local requests before older non-local work" do - name = - start_single_shard_group( - replicated_pg_receiver_buffer_size: 1, - replicated_pg_receiver_flush_interval: 60_000, - replicated_pg_receiver_local_request_quota: 2 - ) - + describe "local request fairness" do + test "a control flood yields to a queued local request after bounded work" do + name = start_single_shard_group() shard = suspend_only_shard(name) - remote_pid = spawn_forever() - remote_key1 = "fair/quota/remote/#{System.unique_integer([:positive])}/1" - remote_key2 = "fair/quota/remote/#{System.unique_integer([:positive])}/2" - local_key1 = "fair/quota/local/#{System.unique_integer([:positive])}/1" - local_key2 = "fair/quota/local/#{System.unique_integer([:positive])}/2" - local_key3 = "fair/quota/local/#{System.unique_integer([:positive])}/3" - - on_exit(fn -> - kill_if_alive(remote_pid) - end) - - send(shard, replicated_pg_join(nil, remote_key1, remote_pid, %{remote: 1}, :join)) - send(shard, replicated_pg_join(nil, remote_key2, remote_pid, %{remote: 2}, :join)) - send(shard, {:group_dispatch, [self()], {:quota_marker, shard}}) + remote_node = :"missing-control-peer@nohost" + key = "fair/control/local/#{System.unique_integer([:positive])}" - caller1 = - spawn_requester( - fn -> - Group.join(name, local_key1, %{order: 1}) - end, - :quota_result - ) + send( + shard, + {:group_replica_frame, remote_node, {:heads, Group.Replica.WireProtocol.version(), []}} + ) - caller2 = - spawn_requester( - fn -> - Group.join(name, local_key2, %{order: 2}) - end, - :quota_result - ) + for _ <- 1..10_000 do + send(shard, {:nodedown, remote_node}) + end - caller3 = - spawn_requester( - fn -> - Group.join(name, local_key3, %{order: 3}) - end, - :quota_result - ) + ref = make_ref() + epoch = Group.Replica.Data.generation(name) - on_exit(fn -> - kill_if_alive(caller1) - kill_if_alive(caller2) - kill_if_alive(caller3) - end) + send( + shard, + {:group_local_request, self(), ref, {:register, nil, epoch, key, self(), %{local: true}}} + ) - wait_until(fn -> shard_message_queue_len(shard) >= 6 end, 1_000) + wait_until(fn -> shard_message_queue_len(shard) >= 10_002 end, 2_000) :ok = :sys.resume(shard) - assert_receive {:quota_result, ^caller1, :ok}, 1_000 - assert_receive {:quota_result, ^caller2, :ok}, 1_000 - assert_receive {:quota_result, ^caller3, :ok}, 1_000 - assert_receive {:quota_marker, ^shard}, 1_000 + assert_receive {:group_local_reply, ^ref, :ok}, 100 + assert Group.lookup(name, key) == {self(), %{local: true}} - assert Group.members(name, remote_key1) == [{remote_pid, %{remote: 1}}] - assert Group.members(name, remote_key2) == [{remote_pid, %{remote: 2}}] - assert Group.members(name, local_key1) == [{caller1, %{order: 1}}] - assert Group.members(name, local_key2) == [{caller2, %{order: 2}}] - assert Group.members(name, local_key3) == [{caller3, %{order: 3}}] + # The assertion has already proved bounded yielding. Drop the synthetic + # duplicate-control tail instead of charging every PR for draining it. + old_shard = Process.whereis(shard) + Process.exit(old_shard, :kill) + wait_until(fn -> is_pid(Process.whereis(shard)) and Process.whereis(shard) != old_shard end) end test "local PG batching applies mixed join and leave requests correctly" do @@ -2421,524 +2277,56 @@ defmodule GroupTest do end describe "replicated PG receiver buffering" do - test "flushes buffered replicated joins when buffer size is reached" do - name = - start_single_shard_group( - replicated_pg_receiver_buffer_size: 2, - replicated_pg_receiver_flush_interval: 60_000 - ) - + test "legacy unsequenced ingress cannot materialize or delete rows" do + name = start_single_shard_group() shard = Group.Replica.shard_name(name, 0) - key1 = "replicated/size/#{System.unique_integer([:positive])}/1" - key2 = "replicated/size/#{System.unique_integer([:positive])}/2" - pid1 = spawn_forever() - pid2 = spawn_forever() + registry_key = "legacy-ingress/registry/#{System.unique_integer([:positive])}" + pg_key = "legacy-ingress/pg/#{System.unique_integer([:positive])}" + cluster_state_key = "legacy-ingress/cluster-state/#{System.unique_integer([:positive])}" + retained_key = "legacy-ingress/retained/#{System.unique_integer([:positive])}" + owner = spawn_forever() - on_exit(fn -> - kill_if_alive(pid1) - kill_if_alive(pid2) - end) + on_exit(fn -> kill_if_alive(owner) end) - :ok = Group.monitor(name, :all) + send(shard, replicated_register(nil, registry_key, owner, %{legacy: true}, :register)) + send(shard, replicated_pg_join(nil, pg_key, owner, %{legacy: true}, :join)) + send(shard, {:cluster_state, nil, [{cluster_state_key, owner, %{}, 1}], []}) + + :ok = Group.register(name, retained_key, %{retained: true}) - send(shard, replicated_pg_join(nil, key1, pid1, %{v: 1}, :join)) + # Pre-AE cluster lifecycle messages were unsequenced and unfenced. A + # delayed copy must not purge current rows while leaving their stream + # cursor advanced. + send(shard, {:cluster_disconnect, [nil], self()}) - assert Group.members(name, key1) == [] - refute_receive {:group, _events, _info}, 50 + send( + shard, + {:replicate_process_down_batch, + [{self(), nil, retained_key, %{retained: true}, :legacy_delete}], []} + ) - send(shard, replicated_pg_join(nil, key2, pid2, %{v: 2}, :join)) + _state = :sys.get_state(shard) - assert_receive {:group, events, _}, 1000 - assert Enum.map(events, & &1.key) == [key1, key2] - assert Enum.map(events, & &1.type) == [:joined, :joined] - assert Group.members(name, key1) == [{pid1, %{v: 1}}] - assert Group.members(name, key2) == [{pid2, %{v: 2}}] + assert Group.lookup(name, registry_key) == nil + assert Group.members(name, pg_key) == [] + assert Group.lookup(name, cluster_state_key) == nil + assert Group.lookup(name, retained_key) == {self(), %{retained: true}} + assert :ok = Group.TestCluster.assert_replica_consistent(name) end + end - test "barrier messages flush buffered replicated ops in order" do - name = - start_single_shard_group( - replicated_pg_receiver_buffer_size: 32, - replicated_pg_receiver_flush_interval: 60_000 - ) + describe "replica write-ahead journal" do + test "concurrent shards retain independent append order", %{name: name} do + named_cluster = "journal/append-order" + operations_per_shard = 100 + :ok = Group.connect(name, named_cluster) - shard = Group.Replica.shard_name(name, 0) - key = "replicated/barrier/#{System.unique_integer([:positive])}" - pid = spawn_forever() + parent = self() - on_exit(fn -> kill_if_alive(pid) end) - - :ok = Group.monitor(name, :all) - - send(shard, replicated_pg_join(nil, key, pid, %{v: 1}, :join)) - send(shard, replicated_pg_join(nil, key, pid, %{v: 2}, :update)) - send(shard, replicated_pg_leave(nil, key, pid, %{v: 2}, :leave)) - - assert Group.members(name, key) == [] - flush_replicated_pg_barrier(shard) - - assert_receive {:group, events, _}, 1000 - - assert [ - %Group.Event{ - type: :joined, - key: ^key, - pid: ^pid, - meta: %{v: 1}, - previous_meta: nil - }, - %Group.Event{ - type: :joined, - key: ^key, - pid: ^pid, - meta: %{v: 2}, - previous_meta: %{v: 1} - }, - %Group.Event{type: :left, key: ^key, pid: ^pid, meta: %{v: 2}, reason: :leave} - ] = events - - assert_receive {:replicated_pg_buffer_flushed, ^shard}, 1000 - assert Group.members(name, key) == [] - end - - test "batchable traffic can flush an overdue buffer without relying on the timer" do - name = - start_single_shard_group( - replicated_pg_receiver_buffer_size: 32, - replicated_pg_receiver_flush_interval: 1_000 - ) - - shard = Group.Replica.shard_name(name, 0) - key1 = "replicated/due/#{System.unique_integer([:positive])}/1" - key2 = "replicated/due/#{System.unique_integer([:positive])}/2" - pid1 = spawn_forever() - pid2 = spawn_forever() - - on_exit(fn -> - kill_if_alive(pid1) - kill_if_alive(pid2) - end) - - :ok = Group.monitor(name, :all) - - send(shard, replicated_pg_join(nil, key1, pid1, %{v: 1}, :join)) - Process.sleep(10) - - :sys.replace_state(shard, fn state -> - %{ - state - | pending_replicated_pg_started_at: System.monotonic_time(:millisecond) - 5_000, - pending_replicated_pg_flush_ref: nil - } - end) - - send(shard, replicated_pg_join(nil, key2, pid2, %{v: 2}, :join)) - - assert_receive {:group, events, _}, 1000 - assert Enum.map(events, & &1.key) == [key1, key2] - - state = :sys.get_state(shard) - assert state.pending_replicated_pg_len == 0 - assert Group.members(name, key1) == [{pid1, %{v: 1}}] - assert Group.members(name, key2) == [{pid2, %{v: 2}}] - end - - test "terminate flushes buffered replicated ops before shard restart" do - name = - start_single_shard_group( - replicated_pg_receiver_buffer_size: 32, - replicated_pg_receiver_flush_interval: 60_000 - ) - - shard = Group.Replica.shard_name(name, 0) - shard_pid = Process.whereis(shard) - key = "replicated/terminate/#{System.unique_integer([:positive])}" - pid = spawn_forever() - - on_exit(fn -> kill_if_alive(pid) end) - - :ok = Group.monitor(name, :all) - - send(shard, replicated_pg_join(nil, key, pid, %{v: 1}, :join)) - assert Group.members(name, key) == [] - - ref = Process.monitor(shard_pid) - :ok = GenServer.stop(shard_pid, :shutdown) - assert_receive {:DOWN, ^ref, :process, ^shard_pid, :shutdown}, 1000 - assert_receive {:group, [%Group.Event{type: :joined, key: ^key, pid: ^pid}], _}, 1000 - - wait_until(fn -> - case Process.whereis(shard) do - nil -> false - new_pid -> new_pid != shard_pid - end - end) - - assert Group.members(name, key) == [{pid, %{v: 1}}] - end - end - - describe "replicated registry receiver buffering" do - test "barrier messages flush buffered replicated register and unregister ops in order" do - name = - start_single_shard_group( - replicated_registry_receiver_buffer_size: 32, - replicated_registry_receiver_flush_interval: 60_000 - ) - - shard = Group.Replica.shard_name(name, 0) - key = "replicated-registry/barrier/#{System.unique_integer([:positive])}" - pid = spawn_forever() - time1 = System.system_time() - time2 = time1 + 1 - - on_exit(fn -> kill_if_alive(pid) end) - - :ok = Group.monitor(name, :all) - - send(shard, replicated_register(nil, key, pid, %{v: 1}, :register, time1)) - send(shard, replicated_register(nil, key, pid, %{v: 2}, :update, time2)) - send(shard, replicated_unregister(nil, key, pid, %{v: 2}, :unregister)) - - assert Group.lookup(name, key) == nil - flush_replicated_registry_barrier(shard) - - assert_receive {:group, events, _}, 1_000 - - assert [ - %Group.Event{ - type: :registered, - key: ^key, - pid: ^pid, - meta: %{v: 1}, - previous_meta: nil - }, - %Group.Event{ - type: :registered, - key: ^key, - pid: ^pid, - meta: %{v: 2}, - previous_meta: %{v: 1} - }, - %Group.Event{ - type: :unregistered, - key: ^key, - pid: ^pid, - meta: %{v: 2}, - reason: :unregister - } - ] = events - - assert_receive {:replicated_registry_buffer_flushed, ^shard}, 1_000 - assert Group.lookup(name, key) == nil - end - - test "batchable registry traffic can flush an overdue buffer without relying on the timer" do - name = - start_single_shard_group( - replicated_registry_receiver_buffer_size: 32, - replicated_registry_receiver_flush_interval: 1_000 - ) - - shard = Group.Replica.shard_name(name, 0) - key1 = "replicated-registry/due/#{System.unique_integer([:positive])}/1" - key2 = "replicated-registry/due/#{System.unique_integer([:positive])}/2" - pid1 = spawn_forever() - pid2 = spawn_forever() - - on_exit(fn -> - kill_if_alive(pid1) - kill_if_alive(pid2) - end) - - :ok = Group.monitor(name, :all) - - send(shard, replicated_register(nil, key1, pid1, %{v: 1}, :register)) - Process.sleep(10) - - :sys.replace_state(shard, fn state -> - %{ - state - | pending_replicated_registry_started_at: System.monotonic_time(:millisecond) - 5_000, - pending_replicated_registry_flush_ref: nil - } - end) - - send(shard, replicated_register(nil, key2, pid2, %{v: 2}, :register)) - - assert_receive {:group, events, _}, 1_000 - assert Enum.map(events, & &1.key) == [key1, key2] - - state = :sys.get_state(shard) - assert state.pending_replicated_registry_len == 0 - assert Group.lookup(name, key1) == {pid1, %{v: 1}} - assert Group.lookup(name, key2) == {pid2, %{v: 2}} - end - - test "restart purges a flushed legacy registry row that has no authoritative claim" do - name = - start_single_shard_group( - replicated_registry_receiver_buffer_size: 32, - replicated_registry_receiver_flush_interval: 60_000 - ) - - shard = Group.Replica.shard_name(name, 0) - shard_pid = Process.whereis(shard) - key = "replicated-registry/terminate/#{System.unique_integer([:positive])}" - pid = spawn_forever() - - on_exit(fn -> kill_if_alive(pid) end) - - :ok = Group.monitor(name, :all) - - send(shard, replicated_register(nil, key, pid, %{v: 1}, :register)) - assert Group.lookup(name, key) == nil - - ref = Process.monitor(shard_pid) - :ok = GenServer.stop(shard_pid, :shutdown) - assert_receive {:DOWN, ^ref, :process, ^shard_pid, :shutdown}, 1_000 - assert_receive {:group, [%Group.Event{type: :registered, key: ^key, pid: ^pid}], _}, 1_000 - - wait_until(fn -> - case Process.whereis(shard) do - nil -> false - new_pid -> new_pid != shard_pid - end - end) - - assert Group.lookup(name, key) == nil - assert :ok = Group.TestCluster.assert_replica_consistent(name) - end - - test "replaces stale remote registry owner and clears the old by-pid entry" do - name = - start_single_shard_group( - replicated_registry_receiver_buffer_size: 32, - replicated_registry_receiver_flush_interval: 60_000 - ) - - shard = Group.Replica.shard_name(name, 0) - key = "replicated-registry/replace/#{System.unique_integer([:positive])}" - old_pid = spawn_forever() - new_pid = spawn_forever() - time1 = System.system_time() - time2 = time1 + 1 - - on_exit(fn -> - kill_if_alive(old_pid) - kill_if_alive(new_pid) - end) - - Group.Replica.Data.registry_insert( - name, - 0, - nil, - key, - old_pid, - %{v: 1}, - time1, - :"remote_a@127.0.0.1" - ) - - send(shard, replicated_register(nil, key, new_pid, %{v: 2}, :register, time2)) - flush_replicated_registry_barrier(shard) - - assert_receive {:replicated_registry_buffer_flushed, ^shard}, 1_000 - assert Group.lookup(name, key) == {new_pid, %{v: 2}} - assert Group.Replica.Data.registry_lookup_by_pid(name, 0, old_pid) == [] - - assert [{nil, ^key, %{v: 2}, ^time2, _entry_node}] = - Group.Replica.Data.registry_lookup_by_pid(name, 0, new_pid) - end - - test "custom conflict resolver selects winner and Group terminates only the local loser" do - key = "replicated-registry/custom-loser/#{System.unique_integer([:positive])}" - - name = - start_single_shard_group( - replicated_registry_receiver_buffer_size: 1, - resolve_registry_conflict: {GroupTest.ResolveRegistryConflict, :pick, [:remote]} - ) - - parent = self() - - local_owner = - spawn(fn -> - :ok = Group.register(name, key, %{owner: :local}) - send(parent, {:custom_conflict_owner_ready, self()}) - Process.sleep(:infinity) - end) - - remote_pid = spawn_forever() - - on_exit(fn -> - kill_if_alive(local_owner) - kill_if_alive(remote_pid) - end) - - assert_receive {:custom_conflict_owner_ready, ^local_owner}, 1_000 - owner_ref = Process.monitor(local_owner) - shard = Group.Replica.shard_name(name, 0) - - send( - shard, - replicated_register( - nil, - key, - remote_pid, - %{owner: :remote}, - :register, - System.system_time() - ) - ) - - wait_until(fn -> - Group.lookup(name, key) == {remote_pid, %{owner: :remote}} - end) - - assert_receive {:DOWN, ^owner_ref, :process, ^local_owner, - {:group_registry_conflict, ^key, %{owner: :remote}}}, - 1_000 - end - - test "batched remote conflict keeps the staged local winner when later unregister arrives" do - key = "replicated-registry/conflict-local/#{System.unique_integer([:positive])}" - - name = - start_single_shard_group( - replicated_registry_receiver_buffer_size: 32, - replicated_registry_receiver_flush_interval: 60_000, - resolve_registry_conflict: {GroupTest.ResolveRegistryConflict, :pick, [:local]} - ) - - parent = self() - - local_owner = - spawn(fn -> - :ok = Group.register(name, key, %{owner: :local}) - send(parent, {:local_registry_owner_ready, self()}) - Process.sleep(:infinity) - end) - - remote_pid = spawn_forever() - - on_exit(fn -> - kill_if_alive(local_owner) - kill_if_alive(remote_pid) - end) - - assert_receive {:local_registry_owner_ready, ^local_owner}, 1_000 - assert Group.lookup(name, key) == {local_owner, %{owner: :local}} - - :ok = Group.monitor(name, :all) - shard = Group.Replica.shard_name(name, 0) - - send( - shard, - replicated_register( - nil, - key, - remote_pid, - %{owner: :remote}, - :register, - System.system_time() - ) - ) - - send(shard, replicated_unregister(nil, key, remote_pid, %{owner: :remote}, :unregister)) - flush_replicated_registry_barrier(shard) - - assert_receive {:replicated_registry_buffer_flushed, ^shard}, 1_000 - refute_receive {:group, _events, _info}, 50 - assert Group.lookup(name, key) == {local_owner, %{owner: :local}} - assert Group.Replica.Data.registry_lookup_by_pid(name, 0, remote_pid) == [] - end - - test "batched remote conflict removes the staged remote winner when later unregister arrives" do - key = "replicated-registry/conflict-remote/#{System.unique_integer([:positive])}" - - name = - start_single_shard_group( - replicated_registry_receiver_buffer_size: 32, - replicated_registry_receiver_flush_interval: 60_000, - resolve_registry_conflict: {GroupTest.ResolveRegistryConflict, :pick, [:remote]} - ) - - parent = self() - - local_owner = - spawn(fn -> - :ok = Group.register(name, key, %{owner: :local}) - send(parent, {:remote_registry_owner_ready, self()}) - Process.sleep(:infinity) - end) - - remote_pid = spawn_forever() - - on_exit(fn -> - kill_if_alive(local_owner) - kill_if_alive(remote_pid) - end) - - assert_receive {:remote_registry_owner_ready, ^local_owner}, 1_000 - assert Group.lookup(name, key) == {local_owner, %{owner: :local}} - - :ok = Group.monitor(name, :all) - shard = Group.Replica.shard_name(name, 0) - - send( - shard, - replicated_register( - nil, - key, - remote_pid, - %{owner: :remote}, - :register, - System.system_time() - ) - ) - - send(shard, replicated_unregister(nil, key, remote_pid, %{owner: :remote}, :unregister)) - flush_replicated_registry_barrier(shard) - - assert_receive {:group, events, _}, 1_000 - - assert [ - %Group.Event{ - type: :unregistered, - key: ^key, - pid: ^local_owner, - meta: %{owner: :local}, - reason: :resolve_conflict - }, - %Group.Event{ - type: :unregistered, - key: ^key, - pid: ^remote_pid, - meta: %{owner: :remote}, - reason: :unregister - } - ] = events - - assert_receive {:replicated_registry_buffer_flushed, ^shard}, 1_000 - assert Group.lookup(name, key) == nil - assert Group.Replica.Data.registry_lookup_by_pid(name, 0, local_owner) == [] - assert Group.Replica.Data.registry_lookup_by_pid(name, 0, remote_pid) == [] - end - end - - describe "replica write-ahead journal" do - test "concurrent shards retain independent append order", %{name: name} do - named_cluster = "journal/append-order" - operations_per_shard = 100 - :ok = Group.connect(name, named_cluster) - - parent = self() - - owners = - for shard <- 0..3 do - nil_keys = - keys_for_shard(nil, "journal/append-order/nil/#{shard}", 4, shard, 50) + owners = + for shard <- 0..3 do + nil_keys = + keys_for_shard(nil, "journal/append-order/nil/#{shard}", 4, shard, 50) named_keys = keys_for_shard( @@ -3157,16 +2545,17 @@ defmodule GroupTest do stream_id = Group.Replica.Data.local_stream_id(name, 0, cluster) old_epoch = Group.Replica.WireProtocol.stream_epoch(stream_id) + old_shard = Process.whereis(Group.Replica.shard_name(name, 0)) + :ok = :sys.suspend(old_shard) # Group.disconnect/3 closes authority and routing before its request - # reaches every shard. Model a shard kill in that exact window. + # reaches every shard. Hold the durable cleanup message in the shard's + # mailbox, then model a kill in that exact window. assert [{^cluster, ^old_epoch}] = Group.Replica.Data.deactivate_local_clusters(name, [cluster]) - :ok = Group.Replica.Data.remove_cluster_node(name, [cluster], node()) assert Group.Replica.Data.closed_local_clusters(name) == [cluster] - old_shard = Process.whereis(Group.Replica.shard_name(name, 0)) Process.exit(old_shard, :kill) Group.TestCluster.assert_eventually(fn -> @@ -3182,9 +2571,326 @@ defmodule GroupTest do assert :ets.lookup(Group.Replica.Data.replica_stream_meta_table(name, 0), stream_id) == [] assert :ok = Group.TestCluster.assert_replica_consistent(name) end + + test "the last close acknowledgement atomically removes all cluster routing" do + name = start_single_shard_group() + cluster = "close/terminal-ack/#{System.unique_integer([:positive])}" + remote_route = :"close-terminal-ack@remote" + + :ok = Group.connect(name, cluster) + :ok = Group.Replica.Data.add_cluster_node(name, [cluster], remote_route) + + shard = Process.whereis(Group.Replica.shard_name(name, 0)) + :ok = :sys.suspend(shard) + + on_exit(fn -> Group.TestCluster.resume_if_alive(shard) end) + + # Model the durable deactivation caller and the final shard both dying + # immediately after the final acknowledgement. Once the close marker is + # gone there is no later recovery hook, so routing must already be gone. + assert [{^cluster, epoch}] = + Group.Replica.Data.deactivate_local_clusters_durable(name, [cluster]) + + assert is_reference(epoch) + + assert [] = + Group.Replica.Data.mark_closed_cluster_shard(name, [{cluster, make_ref()}], 0) + + assert Group.Replica.Data.closed_local_clusters(name) == [cluster] + + assert [^cluster] = + Group.Replica.Data.mark_closed_cluster_shard(name, [{cluster, epoch}], 0) + + assert Group.Replica.Data.closed_local_clusters(name) == [] + assert Group.Replica.Data.cluster_nodes(name, cluster) == [] + assert Group.Replica.Data.clusters_for_node(name, remote_route) == [] + end + + test "delayed restart cleanup cannot remove a reactivated cluster's routes" do + name = start_single_shard_group() + cluster = "close/reactivated-cleanup/#{System.unique_integer([:positive])}" + remote_route = :"close-reactivated-cleanup@remote" + + :ok = Group.connect(name, cluster) + :ok = Group.Replica.Data.add_cluster_node(name, [cluster], remote_route) + + # Model repair_primary_replica_rows/2 observing this cluster while it was + # inactive, followed by a concurrent reconnect completing before repair's + # serialized routing cleanup runs. + :ok = Group.Replica.Data.remove_clusters(name, [cluster]) + + assert Enum.sort(Group.Replica.Data.cluster_nodes(name, cluster)) == + Enum.sort([node(), remote_route]) + + assert cluster in Group.Replica.Data.clusters_for_node(name, node()) + assert cluster in Group.Replica.Data.clusters_for_node(name, remote_route) + end + + test "the last expired replica lane atomically removes peer routing" do + name = start_single_shard_group() + remote_node = :"terminal-peer-retirement@remote" + generation = Group.Replica.WireProtocol.new_generation() + + assert {nil, []} = + Group.Replica.Data.put_remote_replica_info( + name, + 0, + remote_node, + generation, + 0, + [{nil, generation}] + ) + + :ok = + Group.Replica.Data.put_remote_view_info( + name, + 0, + remote_node, + generation, + 0, + 0 + ) + + assert remote_node in Group.Replica.Data.cluster_nodes(name, nil) + assert [nil] = Group.Replica.Data.clusters_for_node(name, remote_node) + + # Once this call removes the last persisted lane view, no shard restart + # can reconstruct an expiry obligation for the peer. Its routing must be + # gone before the terminal retirement is acknowledged. + assert :node_retired = + Group.Replica.Data.expire_remote_replica_lane(name, 0, remote_node) + + assert Group.Replica.Data.cluster_nodes(name, nil) |> Enum.member?(remote_node) == false + assert Group.Replica.Data.clusters_for_node(name, remote_node) == [] + assert Group.Replica.Data.remote_generation(name, remote_node) == nil + end + + test "nodedown removes a retired peer's pending authority repair" do + name = start_single_shard_group() + remote_node = :"retired-authority-repair@remote" + shard = Group.Replica.shard_name(name, 0) + + send(shard, {:replica_authority_dirty_local, remote_node}) + + assert %{cluster_control_dirty: %{^remote_node => _timestamp}} = :sys.get_state(shard) + + send(shard, {:nodedown, remote_node}) + + refute Map.has_key?(:sys.get_state(shard).cluster_control_dirty, remote_node) + end + + test "delayed peer cleanup cannot remove a rediscovered generation's routes" do + name = start_single_shard_group() + remote_node = :"rediscovered-peer-route@remote" + old_generation = Group.Replica.WireProtocol.new_generation() + + assert {nil, []} = + Group.Replica.Data.put_remote_replica_info( + name, + 0, + remote_node, + old_generation, + 0, + [{nil, old_generation}] + ) + + :ok = + Group.Replica.Data.put_remote_view_info( + name, + 0, + remote_node, + old_generation, + 0, + 0 + ) + + assert :node_retired = + Group.Replica.Data.expire_remote_replica_lane(name, 0, remote_node) + + new_generation = Group.Replica.WireProtocol.new_generation() + + assert {nil, []} = + Group.Replica.Data.put_remote_replica_info( + name, + 0, + remote_node, + new_generation, + 0, + [{nil, new_generation}] + ) + + # Model the old expiry/nodedown caller resuming only after rediscovery. + :ok = Group.Replica.Data.purge_cluster_node(name, remote_node) + + assert remote_node in Group.Replica.Data.cluster_nodes(name, nil) + assert [nil] = Group.Replica.Data.clusters_for_node(name, remote_node) + assert Group.Replica.Data.remote_generation(name, remote_node) == new_generation + end + + test "a lane restart reconstructs retirement from a persisted authority hint" do + name = :"group_hint_restart_#{System.unique_integer([:positive])}" + + start_supervised!( + {Group, + name: name, + shards: 2, + log: false, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 150} + ) + + remote_node = :"hint-restart-retirement@remote" + old_generation = Group.Replica.WireProtocol.new_generation() + + assert {nil, []} = + Group.Replica.Data.put_remote_replica_info( + name, + 0, + remote_node, + old_generation, + 0, + [{nil, old_generation}] + ) + + generation = Group.Replica.WireProtocol.new_generation() + + assert Group.Replica.Data.observe_remote_replica_hint( + name, + remote_node, + generation, + 0 + ) + + old_lane = Process.whereis(Group.Replica.shard_name(name, 1)) + monitor = Process.monitor(old_lane) + Process.exit(old_lane, :kill) + assert_receive {:DOWN, ^monitor, :process, ^old_lane, :killed}, 5_000 + + wait_until(fn -> + case Process.whereis(Group.Replica.shard_name(name, 1)) do + pid when is_pid(pid) -> pid != old_lane + _ -> false + end + end) + + wait_until( + fn -> + Group.Replica.Data.remote_replica_authority_hint(name, remote_node) == nil + end, + 2_000 + ) + + refute Map.has_key?( + :sys.get_state(Group.Replica.shard_name(name, 1)).peer_last_seen, + remote_node + ) + end end describe "replica authority snapshots" do + test "a raced authority hint prevents a partial incremental view install", %{name: name} do + remote_node = :"incremental-authority-race@remote" + generation = Group.Replica.WireProtocol.new_generation() + cluster = "incremental-authority-race/cluster" + epoch = make_ref() + + assert {nil, []} = + Group.Replica.Data.put_remote_replica_info( + name, + 0, + remote_node, + generation, + 0, + [{nil, generation}] + ) + + :ok = + Group.Replica.Data.put_remote_view_info( + name, + 0, + remote_node, + generation, + 0, + 0 + ) + + assert Group.Replica.Data.observe_remote_replica_hint( + name, + remote_node, + generation, + 2 + ) + + assert :stale = + Group.Replica.Data.put_remote_cluster_epochs( + name, + 0, + remote_node, + generation, + 0, + 1, + [{cluster, epoch}] + ) + + assert Group.Replica.Data.remote_cluster_epoch(name, remote_node, cluster) == nil + + assert :stale = + Group.Replica.Data.put_remote_view_info( + name, + 0, + remote_node, + generation, + 0, + 2 + ) + + refute Group.Replica.Data.remote_registry_claim_authoritative?( + name, + 0, + remote_node, + generation, + nil, + generation + ) + end + + test "a newer-generation hint atomically rejects an old lane view", %{name: name} do + remote_node = :"lane-view-generation-race@remote" + old_generation = Group.Replica.WireProtocol.new_generation() + + assert {nil, []} = + Group.Replica.Data.put_remote_replica_info( + name, + 0, + remote_node, + old_generation, + 0, + [{nil, old_generation}] + ) + + new_generation = Group.Replica.WireProtocol.new_generation() + assert Group.Replica.WireProtocol.generation_newer?(new_generation, old_generation) + + assert Group.Replica.Data.observe_remote_replica_hint( + name, + remote_node, + new_generation, + 0 + ) + + assert :stale = + Group.Replica.Data.put_remote_view_info( + name, + 1, + remote_node, + old_generation, + 0, + 0 + ) + + assert Group.Replica.Data.remote_view_generation(name, 1, remote_node) == nil + end + test "revision and epoch rows remain coherent during concurrent activation", %{name: name} do clusters = for i <- 1..1_000, do: "authority/#{i}" @@ -3255,34 +2961,10 @@ defmodule GroupTest do [{:join, cluster, key, pid, meta, System.system_time(), reason, node(pid)}]} end - defp replicated_pg_leave(cluster, key, pid, meta, reason) do - {:replicate_pg_batch, [{:leave, cluster, key, pid, meta, reason}]} - end - defp replicated_register(cluster, key, pid, meta, _reason, time \\ System.system_time()) do {:replicate_registry_batch, [{:register, cluster, key, pid, meta, time, node(pid)}]} end - defp replicated_unregister(cluster, key, pid, meta, reason) do - {:replicate_registry_batch, [{:unregister, cluster, key, pid, meta, reason}]} - end - - defp enqueue_replicated_pg_backlog(shard, key_prefix, pid, count) do - for i <- 1..count do - send(shard, replicated_pg_join(nil, "#{key_prefix}/#{i}", pid, %{}, :join)) - end - - :ok - end - - defp enqueue_replicated_registry_backlog(shard, key_prefix, pid, count) do - for i <- 1..count do - send(shard, replicated_register(nil, "#{key_prefix}/#{i}", pid, %{seq: i}, :register)) - end - - :ok - end - defp spawn_requester(fun, tag) do parent = self() @@ -3300,10 +2982,6 @@ defmodule GroupTest do end end - defp flush_replicated_pg_barrier(shard) do - send(shard, {:group_dispatch, [self()], {:replicated_pg_buffer_flushed, shard}}) - end - defp flush_replicated_registry_barrier(shard) do send(shard, {:group_dispatch, [self()], {:replicated_registry_buffer_flushed, shard}}) end @@ -3332,6 +3010,28 @@ defmodule GroupTest do spawn(fn -> Process.sleep(:infinity) end) end + defp replica_ingress_fairness_owner(parent) do + receive do + {:write, shard, request} -> + ref = make_ref() + send(shard, {:group_local_request, self(), ref, request}) + {reply, calls} = receive_local_write_with_trace(shard, ref, 0) + send(parent, {:local_write_finished, self(), reply, calls}) + Process.sleep(:infinity) + end + end + + defp receive_local_write_with_trace(shard, ref, calls) do + receive do + {:trace, ^shard, :call, + {Group.Replica, :handle_replica_message, [_state, _source_node, _message]}} -> + receive_local_write_with_trace(shard, ref, calls + 1) + + {:group_local_reply, ^ref, reply} -> + {reply, calls} + end + end + defp kill_if_alive(pid) do if Process.alive?(pid) do Process.exit(pid, :kill) diff --git a/test/jepsen/node.exs b/test/jepsen/node.exs index e02e0ac..efe370d 100644 --- a/test/jepsen/node.exs +++ b/test/jepsen/node.exs @@ -401,9 +401,7 @@ end defmodule Group.Jepsen.ConflictResolver do @moduledoc false - def resolve(_name, _key, {pid1, meta1, _time1}, {pid2, meta2, _time2}) do - if rank(meta1) >= rank(meta2), do: pid1, else: pid2 - end + def resolve(_name, _key, {_pid, meta, _time}), do: rank(meta) defp rank(%{revision: revision, token: token}), do: {revision, token} defp rank(_meta), do: {-1, ""} diff --git a/test/mutation/README.md b/test/mutation/README.md index fa9e50e..f616b75 100644 --- a/test/mutation/README.md +++ b/test/mutation/README.md @@ -8,12 +8,32 @@ per-lane authority installation, periodic head advertisement, interrupted journal/index repair, and named-cluster close completion. Snapshot calibration also covers incomplete commit, conflicting retransmission rows, newer-snapshot supersession, stale-authority fencing, and staging expiry. +Restart calibration covers per-lane eviction breadcrumbs and partially observed +authority, while wire calibration rejects wrong-shard rows and unsequenced +cluster lifecycle messages. The 65-mutant campaign also independently removes +the generation and epoch fences, races authority changes against local-owner +retirement, skips conflict reprojection after exact authority returns, bypasses +shard-zero authority serialization, separates exact authority from its shared +cluster projection, interrupts local cluster activation/deactivation before +shard notification, deletes terminal close/peer breadcrumbs before their route +cleanup, erases a suspended lane's nodedown breadcrumb, retains a dead peer's +authority-repair obligation, lets stale restart or duplicate-close cleanup +erase reactivated epochs, accepts the wrong epoch's close acknowledgement, +allows newer heartbeats or lane hellos to leave old lane views unfenced, retains +cursorless claims, admits incremental authority across a revision race or gap, +lets an unknown post-retirement hint recreate authority or an unleased lane +route, delays a pre-authority lane's rediscovery until the periodic probe, and +disables bounded snapshot-send/receive progress. The runner first verifies every unmodified regression target. It then copies the current checkout once per mutant, changes only that copy, recompiles it, and runs the designated real multi-node regression. A compiling mutant is `killed` only when the regression fails. Any surviving or non-compiling mutant makes the campaign fail. +Before either listing or running mutants, it also requires every production +replacement to match exactly once and every test selector to point at the test +declaration itself. Source edits therefore fail loudly instead of turning a +stale mutation into a placebo run of an adjacent test. ```bash mix run --no-start test/mutation/run.exs diff --git a/test/mutation/run.exs b/test/mutation/run.exs index 60a89c3..f514104 100644 --- a/test/mutation/run.exs +++ b/test/mutation/run.exs @@ -16,21 +16,19 @@ defmodule Group.MutationCampaign do name: "accept_old_generation", file: "lib/group/replica.ex", correct_source: - "WireProtocol.stream_generation(stream_id) == Data.remote_generation(state.name, source_node) and", - faulty_source: "true and", - test: ["test/distributed_test.exs:5328"] + " WireProtocol.stream_generation(stream_id) ==\n" <> + " Data.remote_generation(state.name, source_node) and", + faulty_source: " true and", + test: ["test/distributed_test.exs:5450"] }, %{ name: "accept_old_epoch", file: "lib/group/replica.ex", - correct_source: """ - WireProtocol.stream_epoch(stream_id) == - Data.remote_cluster_epoch(state.name, source_node, cluster) and - """, - faulty_source: """ - true and - """, - test: ["test/distributed_test.exs:4774"] + correct_source: + " WireProtocol.stream_epoch(stream_id) ==\n" <> + " Data.remote_cluster_epoch(state.name, source_node, cluster) and", + faulty_source: " true and", + test: ["test/distributed_test.exs:4853"] }, %{ name: "advance_cursor_across_gap", @@ -57,7 +55,7 @@ defmodule Group.MutationCampaign do advertised_head ) """, - test: ["test/distributed_test.exs:5386"] + test: ["test/distributed_test.exs:5527"] }, %{ name: "registry_snapshot_is_additive", @@ -80,14 +78,14 @@ defmodule Group.MutationCampaign do file: "lib/group/replica/data.ex", correct_source: "Enum.each(existing, fn {key, pid, _meta, _time} ->", faulty_source: "Enum.each(Enum.take(existing, 0), fn {key, pid, _meta, _time} ->", - test: ["test/distributed_test.exs:4030"] + test: ["test/distributed_test.exs:4057"] }, %{ name: "single_chunk_pg_snapshot_is_additive", file: "lib/group/replica.ex", correct_source: " current\n |> Map.keys()\n", faulty_source: " %{}\n |> Map.keys()\n", - test: ["test/distributed_test.exs:4030"] + test: ["test/distributed_test.exs:4057"] }, %{ name: "commit_incomplete_snapshot", @@ -155,7 +153,7 @@ defmodule Group.MutationCampaign do snapshot_seq > Data.replica_cursor(state.name, state.shard_index, stream_id) end """, - test: ["test/replica_snapshot_distributed_test.exs:202"] + test: ["test/replica_snapshot_distributed_test.exs:314"] }, %{ name: "disable_snapshot_staging_expiry", @@ -176,7 +174,7 @@ defmodule Group.MutationCampaign do acc end """, - test: ["test/replica_snapshot_distributed_test.exs:202"] + test: ["test/replica_snapshot_distributed_test.exs:240"] }, %{ name: "disable_below_floor_snapshot", @@ -189,72 +187,180 @@ defmodule Group.MutationCampaign do true -> {:state, state} """, - test: ["test/replica_model_property_test.exs:180"] + test: ["test/replica_model_property_test.exs:174"] }, %{ name: "do_not_sequence_process_down", file: "lib/group/replica.ex", - correct_source: """ - sequenced_downs = - append_process_down_records(state, reason_by_pid, pending_reg, pending_pg) - """, + correct_source: + " sequenced_downs =\n" <> + " append_process_down_records(state, reason_by_pid, pending_reg, pending_pg)\n", faulty_source: - " sequenced_downs =\n if false,\n do: append_process_down_records(state, reason_by_pid, pending_reg, pending_pg),\n else: []\n", - test: ["test/distributed_test.exs:3940"] + " sequenced_downs =\n if false,\n do: append_process_down_records(state, reason_by_pid, pending_reg, pending_pg),\n else: []\n", + test: ["test/distributed_test.exs:3967"] }, %{ name: "do_not_exit_conflict_loser", file: "lib/group/replica.ex", - correct_source: """ - winner_meta = if winner, do: elem(winner, 1), else: nil - exit_local_conflict_loser(pid, key, winner_meta) - acc - """, - faulty_source: """ - _winner_meta = if winner, do: elem(winner, 1), else: nil - _ = Process.alive?(pid) - acc - """, - test: ["test/replica_model_property_test.exs:77"] + correct_source: + " winner_meta = if winner, do: elem(winner, 1), else: nil\n" <> + " exit_local_conflict_loser(pid, key, winner_meta)\n" <> + " acc\n", + faulty_source: + " _winner_meta = if winner, do: elem(winner, 1), else: nil\n" <> + " _ = Process.alive?(pid)\n" <> + " acc\n", + test: ["test/replica_model_property_test.exs:71"] + }, + %{ + name: "heartbeat_does_not_fence_newer_authority", + file: "lib/group/replica/data.ex", + correct_source: " hint_generation == generation and\n", + faulty_source: " false and hint_generation == generation and\n", + test: ["test/anti_entropy_fault_regression_test.exs:2057"] }, %{ - name: "heartbeat_promotes_observed_authority", + name: "heartbeat_does_not_fence_newer_generation", + file: "lib/group/replica/data.ex", + correct_source: + " not is_nil(hint_generation) and\n" <> + " WireProtocol.generation_newer?(generation, hint_generation) ->\n", + faulty_source: + " not is_nil(hint_generation) and\n" <> + " WireProtocol.generation_newer?(generation, hint_generation) and\n" <> + " Process.get(:fence_newer_generation, false) ->\n", + test: ["test/anti_entropy_fault_regression_test.exs:2220"] + }, + %{ + name: "drop_new_generation_authority_hint", + file: "lib/group/replica/data.ex", + correct_source: + " # below are being updated.\n" <> + " put_remote_authority_hint(state.name, remote_node, generation, revision)", + faulty_source: + " # below are being updated.\n" <> + " _ = {state.name, remote_node, generation, revision}", + test: ["test/anti_entropy_fault_regression_test.exs:2220"] + }, + %{ + name: "accept_authority_older_than_generation_hint", + file: "lib/group/replica/data.ex", + correct_source: " hinted_stale? or known_stale? or revision_stale?", + faulty_source: + " _ = hinted_stale?\n" <> + " known_stale? or revision_stale?", + test: ["test/anti_entropy_fault_regression_test.exs:2220"] + }, + %{ + name: "install_lane_view_behind_generation_hint", + file: "lib/group/replica/data.ex", + correct_source: + " remote_replica_authority_hint(state.name, remote_node) == {generation, observed} do", + faulty_source: + " elem(remote_replica_authority_hint(state.name, remote_node), 1) == observed do", + test: ["test/group_test.exs:2857"] + }, + %{ + name: "install_incremental_after_newer_hint", + file: "lib/group/replica/data.ex", + correct_source: + " remote_cluster_epoch_observed_revision(name, remote_node) == expected_revision and\n" <> + " remote_replica_authority_hint(name, remote_node) == {generation, expected_revision}\n", + faulty_source: + " Process.get(:ignore_incremental_authority_race, true) and\n" <> + " is_tuple(remote_replica_authority_hint(name, remote_node))\n", + test: ["test/group_test.exs:2791"] + }, + %{ + name: "accept_hint_without_exact_authority", + file: "lib/group/replica/data.ex", + correct_source: + " not is_nil(hint_generation) and\n" <> + " WireProtocol.generation_newer?(generation, hint_generation) ->\n", + faulty_source: + " (is_nil(hint_generation) or\n" <> + " WireProtocol.generation_newer?(generation, hint_generation)) ->\n", + test: ["test/anti_entropy_fault_regression_test.exs:3308"] + }, + %{ + name: "admit_retired_lane_route_without_authority", + file: "lib/group/replica.ex", + correct_source: + " # A lane hello is only a hint until node-wide exact authority exists.\n" <> + " # In particular, a delayed hello after retirement must not recreate\n" <> + " # an unleased route that can live forever and suppress rediscovery.\n" <> + " {:noreply, request_replica_authority(state, remote_node)}", + faulty_source: + " state = put_remote_shard(state, remote_node, remote_pid)\n" <> + " {:noreply, request_replica_authority(state, remote_node)}", + test: ["test/anti_entropy_fault_regression_test.exs:3308"] + }, + %{ + name: "do_not_restore_hint_lease_after_lane_restart", + file: "lib/group/replica/data.ex", + correct_source: + " {{{:remote_view_info, shard, :\"$1\"}, :_, :_, :_}, [], [:\"$1\"]},\n" <> + " # A hint is the durable fence left before the observing lane records its\n" <> + " # in-memory lease deadline. Every restarting lane must recognize it so a\n" <> + " # crash in that window cannot strand the peer forever.\n" <> + " {{{:remote_authority_hint, :\"$1\"}, :_, :_}, [], [:\"$1\"]}\n", + faulty_source: " {{{:remote_view_info, shard, :\"$1\"}, :_, :_, :_}, [], [:\"$1\"]}\n", + test: ["test/group_test.exs:2730"] + }, + %{ + name: "retain_retired_authority_repair", file: "lib/group/replica.ex", correct_source: - " replica_view_current?(state, remote_node) ->\n" <> - " state\n" <> - " |> put_remote_shard(remote_node, remote_pid)\n" <> - " |> touch_replica_peer(remote_node)", + " is_nil(Data.remote_generation(state.name, remote_node)) and\n" <> + " is_nil(Data.remote_replica_authority_hint(state.name, remote_node)) ->\n" <> + " acc\n", faulty_source: - " replica_view_current?(state, remote_node) ->\n" <> - " :ok =\n" <> - " Data.put_remote_view_info(\n" <> - " state.name,\n" <> - " state.shard_index,\n" <> - " remote_node,\n" <> - " generation,\n" <> - " epoch_revision,\n" <> - " epoch_revision\n" <> - " )\n\n" <> - " state\n" <> - " |> put_remote_shard(remote_node, remote_pid)\n" <> - " |> touch_replica_peer(remote_node)", - test: ["test/distributed_test.exs:4247"] + " is_nil(Data.remote_generation(state.name, remote_node)) and\n" <> + " is_nil(Data.remote_replica_authority_hint(state.name, remote_node)) ->\n" <> + " Map.put(acc, remote_node, last_activity)\n", + test: ["test/anti_entropy_fault_regression_test.exs:3308"] }, %{ name: "skip_authority_fanout", file: "lib/group/replica.ex", correct_source: """ - fan_out_to_siblings( - state, - {:replica_authority_installed_local, remote_node, generation, epoch_revision, - old_generation, stale_epochs} - ) + fan_out_to_siblings( + state, + {:replica_authority_installed_local, remote_node, generation, epoch_revision, + old_generation, stale_epochs} + ) """, faulty_source: """ - :ok + :ok """, - test: ["test/distributed_test.exs:5531"] + test: ["test/distributed_test.exs:5672"] + }, + %{ + name: "wait_for_periodic_lane_probe_after_authority_fanout", + file: "lib/group/replica.ex", + correct_source: """ + else + # A lane hello can legitimately outrun shard zero's exact authority. + # The hello is not retained as a route, because a delayed hello after + # retirement must not recreate an unleased peer. Once exact authority + # reaches this lane, repeat shard-local discovery immediately instead + # of waiting for the next anti-entropy probe. + send_remote_shard_message( + state, + remote_node, + {:peer_connect, self(), state.shard_index, state.num_shards, + Data.my_clusters(state.name)} + ) + + state + end + """, + faulty_source: """ + else + state + end + """, + test: ["test/anti_entropy_fault_regression_test.exs:3813"] }, %{ name: "assume_authority_fanout_reaches_late_lane", @@ -266,12 +372,16 @@ defmodule Group.MutationCampaign do state = maybe_purge_remote_generation(state, remote_node, old_generation, generation) state = purge_remote_streams_outside_authority(state, remote_node) - :ok = install_replica_view(state, remote_node, generation) + state = install_replica_view(state, remote_node, generation) - state - |> touch_replica_peer(remote_node) - |> Map.update!(:cluster_control_dirty, &Map.delete(&1, remote_node)) - |> send_replica_heads(remote_node) + if replica_view_current?(state, remote_node) do + state + |> touch_replica_peer(remote_node) + |> Map.update!(:cluster_control_dirty, &Map.delete(&1, remote_node)) + |> send_replica_heads(remote_node) + else + state + end end """, faulty_source: """ @@ -280,13 +390,16 @@ defmodule Group.MutationCampaign do state end """, - test: ["test/replica_snapshot_distributed_test.exs:329"] + test: ["test/replica_snapshot_distributed_test.exs:539"] }, %{ name: "skip_generation_purge", file: "lib/group/replica.ex", correct_source: """ defp maybe_purge_remote_generation(state, remote_node, _old_generation, _generation) do + state = discard_snapshot_transfers_for_source(state, remote_node) + state = discard_pending_registry_reprojections(state, remote_node) + Data.delete_replica_cursors_for_origin(state.name, state.shard_index, remote_node) {_reg, _pg} = Data.purge_node(state.name, state.shard_index, remote_node) affected = @@ -302,7 +415,6 @@ defmodule Group.MutationCampaign do end) notify_monitors(state.name, events) - Data.delete_replica_cursors_for_origin(state.name, state.shard_index, remote_node) state end """, @@ -310,7 +422,7 @@ defmodule Group.MutationCampaign do defp maybe_purge_remote_generation(state, _remote_node, _old_generation, _generation), do: state """, - test: ["test/distributed_test.exs:5531"] + test: ["test/distributed_test.exs:5672"] }, %{ name: "disable_periodic_heads", @@ -325,64 +437,454 @@ defmodule Group.MutationCampaign do faulty_source: """ defp broadcast_replica_heads(state), do: state """, - test: ["test/distributed_test.exs:3940"] + test: ["test/distributed_test.exs:3967"] }, %{ name: "skip_journal_crash_repair", file: "lib/group/replica.ex", correct_source: ":ok = Data.repair_local_replica_journal(name, shard_index)", faulty_source: ":ok", - test: ["test/group_test.exs:3045"] + test: ["test/group_test.exs:2434"] }, %{ name: "skip_index_crash_repair", file: "lib/group/replica.ex", correct_source: ":ok = Data.repair_shard_indexes(name, shard_index)", faulty_source: ":ok", - test: ["test/group_test.exs:3091"] + test: ["test/group_test.exs:2480"] }, %{ name: "skip_inactive_cluster_repair", file: "lib/group/replica/data.ex", + correct_source: " repair_primary_replica_rows(name, shard)", + faulty_source: + " if Process.get(:run_primary_replica_repair, false),\n" <> + " do: repair_primary_replica_rows(name, shard),\n" <> + " else: :ok", + test: ["test/group_test.exs:2534"] + }, + %{ + name: "skip_closed_cluster_completion", + file: "lib/group/replica.ex", correct_source: """ - def repair_shard_indexes(name, shard) do - purge_inactive_cluster_rows(name, shard) + _completed_clusters = + Data.mark_closed_cluster_shard( + name, + Data.closed_local_cluster_epochs(name), + shard_index + ) """, faulty_source: """ - def repair_shard_indexes(name, shard) do - if false, do: purge_inactive_cluster_rows(name, shard) + _completed_clusters = [] """, - test: ["test/group_test.exs:3145"] + test: ["test/group_test.exs:2534"] }, %{ - name: "skip_closed_cluster_completion", + name: "accept_unfenced_cluster_disconnect", + file: "lib/group/replica.ex", + correct_source: + " _ ->\n" <> + " false\n" <> + " end)\n\n" <> + " case epochs do\n", + faulty_source: + " _ ->\n" <> + " Process.get(:accept_unfenced_cluster_disconnect, true)\n" <> + " end)\n\n" <> + " case epochs do\n", + test: ["test/group_test.exs:1485"] + }, + %{ + name: "accept_completed_cluster_disconnect", + file: "lib/group/replica.ex", + correct_source: + " Data.closed_local_cluster_pending?(\n" <> + " state.name,\n" <> + " cluster,\n" <> + " epoch,\n" <> + " state.shard_index\n" <> + " )", + faulty_source: + " _ = cluster\n" <> + " Process.get(:accept_completed_cluster_disconnect, true)", + test: ["test/group_test.exs:1485"] + }, + %{ + name: "acknowledge_wrong_cluster_close_epoch", + file: "lib/group/replica/data.ex", + correct_source: " [{^cluster, ^request_epoch, pending_shards}] ->\n", + faulty_source: " [{^cluster, _stored_epoch, pending_shards}] ->\n", + test: ["test/group_test.exs:2575"] + }, + %{ + name: "accept_shared_authority_before_lane_install", + file: "lib/group/replica.ex", + correct_source: + " WireProtocol.stream_shard(stream_id) == state.shard_index and\n" <> + " replica_view_current?(state, source_node) and", + faulty_source: + " WireProtocol.stream_shard(stream_id) == state.shard_index and\n" <> + " true and", + test: ["test/anti_entropy_fault_regression_test.exs:1171"] + }, + %{ + name: "apply_incremental_authority_across_revision_gap", + file: "lib/group/replica.ex", + correct_source: " if contiguous_cluster_controls?(accepted, next_revision) do", + faulty_source: + " if contiguous_cluster_controls?(accepted, next_revision) or accepted != [] do", + test: ["test/anti_entropy_fault_regression_test.exs:1363"] + }, + %{ + name: "allow_non_owner_lane_to_mutate_shared_authority", + file: "lib/group/replica.ex", + correct_source: " if state.shard_index == 0 do\n remote_node = node(remote_pid)", + faulty_source: " if true do\n remote_node = node(remote_pid)", + test: ["test/anti_entropy_fault_regression_test.exs:1363"] + }, + %{ + name: "crash_lane_when_local_authority_owner_is_missing", + file: "lib/group/replica.ex", + correct_source: " _ = send_local_control_message(state, control)", + faulty_source: " send(shard_name(state.name, 0), control)", + test: ["test/anti_entropy_fault_regression_test.exs:1310"] + }, + %{ + name: "retire_local_owner_after_remote_authority_changed", + file: "lib/group/replica.ex", + correct_source: " registry_winner_authoritative?(state, cluster, winner) ->", + faulty_source: + " Process.get(:skip_remote_registry_authority, true) or\n" <> + " registry_winner_authoritative?(state, cluster, winner) ->", + test: ["test/anti_entropy_fault_regression_test.exs:1522"] + }, + %{ + name: "skip_registry_reprojection_after_authority_restore", file: "lib/group/replica.ex", correct_source: """ - completed_clusters = - Data.mark_closed_cluster_shard(name, Data.closed_local_clusters(name), shard_index) + defp reproject_pending_registry_keys(state, remote_node) do + case Map.pop(state.pending_registry_reprojections, remote_node) do + {nil, _pending} -> + state + + {keys, pending} -> + state = %{state | pending_registry_reprojections: pending} + {state, events} = reconcile_registry_keys(state, keys, :reconcile, []) + notify_monitors(state.name, events) + state + end + end """, faulty_source: """ - completed_clusters = [] + defp reproject_pending_registry_keys(state, remote_node) do + _ = remote_node + state + end """, - test: ["test/group_test.exs:3145"] + test: ["test/anti_entropy_fault_regression_test.exs:1705"] }, %{ - name: "accept_shared_authority_before_lane_install", + name: "retain_registry_reprojection_after_peer_expiry", + file: "lib/group/replica.ex", + correct_source: """ + defp expire_replica_peer(state, remote_node) do + state = discard_snapshot_transfers_for_source(state, remote_node) + state = discard_snapshot_send_offsets_for_target(state, remote_node) + state = discard_pending_registry_reprojections(state, remote_node) + """, + faulty_source: """ + defp expire_replica_peer(state, remote_node) do + state = discard_snapshot_transfers_for_source(state, remote_node) + state = discard_snapshot_send_offsets_for_target(state, remote_node) + """, + test: ["test/anti_entropy_fault_regression_test.exs:1881"] + }, + %{ + name: "retain_registry_reprojection_after_nodedown", file: "lib/group/replica.ex", correct_source: """ - WireProtocol.stream_shard(stream_id) == state.shard_index and - replica_view_current?(state, source_node) and + def handle_info({:nodedown, dead_node}, state) do + state = flush_pending_replicated_message_barrier(state) + state = discard_snapshot_transfers_for_source(state, dead_node) + state = discard_snapshot_send_offsets_for_target(state, dead_node) + state = discard_pending_registry_reprojections(state, dead_node) """, faulty_source: """ - WireProtocol.stream_shard(stream_id) == state.shard_index and - true and + def handle_info({:nodedown, dead_node}, state) do + state = flush_pending_replicated_message_barrier(state) + state = discard_snapshot_transfers_for_source(state, dead_node) + state = discard_snapshot_send_offsets_for_target(state, dead_node) """, - test: ["test/distributed_test.exs:5531"] + test: ["test/anti_entropy_fault_regression_test.exs:3591"] + }, + %{ + name: "separate_exact_authority_from_cluster_projection", + file: "lib/group/replica/data.ex", + correct_source: + " replace_remote_cluster_projection(state.name, remote_node, current_epochs)\n", + faulty_source: + " _ = {&replace_remote_cluster_projection/3, state.name, remote_node, current_epochs}\n", + test: ["test/anti_entropy_fault_regression_test.exs:3627"] + }, + %{ + name: "separate_local_activation_from_cluster_projection", + file: "lib/group/replica/data.ex", + correct_source: + " if durable?, do: project_activated_local_clusters(state.name, clusters)\n", + faulty_source: + " _ = {durable?, &project_activated_local_clusters/2, state.name, clusters}\n", + test: ["test/anti_entropy_fault_regression_test.exs:3682"] + }, + %{ + name: "drop_durable_cluster_deactivation_cleanup", + file: "lib/group/replica/data.ex", + correct_source: + " cast_cluster_lifecycle(\n" <> + " state.name,\n" <> + " 0..(state.num_shards - 1),\n" <> + " {:cluster_disconnect, clusters, epochs}\n" <> + " )\n", + faulty_source: + " _ = {&cast_cluster_lifecycle/3, state.name, state.num_shards, clusters, epochs}\n", + test: ["test/anti_entropy_fault_regression_test.exs:3731"] + }, + %{ + name: "delete_close_marker_before_terminal_route_cleanup", + file: "lib/group/replica/data.ex", + correct_source: + " :ok = delete_cluster_routes(state.name, [cluster])\n" <> + " :ets.delete(closed_local_cluster_epochs_table(state.name), cluster)\n", + faulty_source: + " :ets.delete(closed_local_cluster_epochs_table(state.name), cluster)\n", + test: ["test/group_test.exs:2575"] + }, + %{ + name: "retire_peer_authority_before_terminal_route_cleanup", + file: "lib/group/replica/data.ex", + correct_source: + " :ets.delete(replication_meta_table(name), {:remote_generation, remote_node})\n" <> + " :ok = delete_peer_routes(name, remote_node)\n", + faulty_source: + " :ets.delete(replication_meta_table(name), {:remote_generation, remote_node})\n", + test: ["test/group_test.exs:2629"] + }, + %{ + name: "stale_peer_cleanup_removes_rediscovered_routes", + file: "lib/group/replica/data.ex", + correct_source: + " if is_nil(remote_generation(state.name, dead_node)) and\n" <> + " is_nil(remote_replica_authority_hint(state.name, dead_node)) do\n", + faulty_source: + " if Process.get(:purge_rediscovered_peer_routes, true) or\n" <> + " (is_nil(remote_generation(state.name, dead_node)) and\n" <> + " is_nil(remote_replica_authority_hint(state.name, dead_node))) do\n", + test: ["test/group_test.exs:2682"] + }, + %{ + name: "stale_restart_cleanup_removes_reactivated_routes", + file: "lib/group/replica/data.ex", + correct_source: + " Enum.filter(clusters, &is_nil(local_cluster_epoch(state.name, &1)))\n", + faulty_source: " clusters\n", + test: ["test/group_test.exs:2609"] + }, + %{ + name: "retain_authority_repair_after_nodedown", + file: "lib/group/replica.ex", + correct_source: + " cluster_control_dirty: Map.delete(state.cluster_control_dirty, dead_node),\n" <> + " authority_dirty_notified: MapSet.delete(state.authority_dirty_notified, dead_node)\n", + faulty_source: + " cluster_control_dirty: state.cluster_control_dirty,\n" <> + " authority_dirty_notified: MapSet.delete(state.authority_dirty_notified, dead_node)\n", + test: ["test/group_test.exs:2668"] + }, + %{ + name: "retain_receive_cursor_for_inactive_local_cluster", + file: "lib/group/replica/data.ex", + correct_source: + " WireProtocol.stream_origin(stream_id) != node() and\n" <> + " active_local_cluster?(name, WireProtocol.stream_cluster(stream_id)) and", + faulty_source: + " WireProtocol.stream_origin(stream_id) != node() and\n" <> + " true and", + test: ["test/anti_entropy_fault_regression_test.exs:2734"] + }, + %{ + name: "retire_shared_authority_with_live_lanes", + file: "lib/group/replica/data.ex", + correct_source: " result =\n if remaining_lanes == 0 do", + faulty_source: " _ = remaining_lanes\n\n result =\n if true do", + test: ["test/anti_entropy_fault_regression_test.exs:855"] + }, + %{ + name: "shard_zero_deletes_sibling_restart_views", + file: "lib/group/replica/data.ex", + correct_source: """ + if shard == 0 do + delete_remote_authority(state.name, remote_node) + end + """, + faulty_source: """ + if shard == 0 do + if state.num_shards > 1 do + for view_shard <- 1..(state.num_shards - 1) do + :ets.delete( + replication_meta_table(state.name), + {:remote_view_info, view_shard, remote_node} + ) + end + end + + delete_remote_authority(state.name, remote_node) + end + """, + test: ["test/anti_entropy_fault_regression_test.exs:240"] + }, + %{ + name: "forget_partial_authority_after_restart", + file: "lib/group/replica.ex", + correct_source: """ + cluster_control_dirty = + if shard_index == 0 do + Enum.reduce(retained_origins, %{}, fn origin, dirty -> + exact = Data.remote_cluster_epoch_exact_revision(name, origin) + observed = Data.remote_cluster_epoch_observed_revision(name, origin) + known_generation = Data.remote_generation(name, origin) + hint = Data.remote_replica_authority_hint(name, origin) + + if (not is_nil(observed) and observed != exact) or + (not is_nil(hint) and elem(hint, 0) != known_generation) do + Map.put(dirty, origin, restarted_at) + else + dirty + end + end) + else + %{} + end + """, + faulty_source: "cluster_control_dirty = %{}\n", + test: ["test/anti_entropy_fault_regression_test.exs:350"] + }, + %{ + name: "accept_wrong_shard_delta_rows", + file: "lib/group/replica.ex", + correct_source: + " Enum.split_while(contiguous, fn {_seq, mutations} ->\n" <> + " valid_replica_mutations?(state, stream_id, mutations)\n" <> + " end)", + faulty_source: + " Enum.split_while(contiguous, fn {_seq, mutations} ->\n" <> + " valid_replica_mutations?(%{state | num_shards: 1}, stream_id, mutations)\n" <> + " end)", + test: ["test/anti_entropy_fault_regression_test.exs:511"] + }, + %{ + name: "restore_unsequenced_cluster_disconnect", + file: "lib/group/replica.ex", + correct_source: """ + def handle_info(_msg, state) do + state = flush_pending_replicated_message_barrier(state) + {:noreply, state} + end + """, + faulty_source: """ + def handle_info({:cluster_disconnect, clusters, remote_pid}, state) do + Enum.each(clusters, fn cluster -> + Data.purge_registry_claims_for_cluster( + state.name, + state.shard_index, + cluster, + node(remote_pid) + ) + + purge_cluster_entries( + state.name, + state.shard_index, + cluster, + node(remote_pid) + ) + end) + + {:noreply, state} + end + + def handle_info(_msg, state) do + state = flush_pending_replicated_message_barrier(state) + {:noreply, state} + end + """, + test: ["test/group_test.exs:2280"] + }, + %{ + name: "skip_cursorless_restart_authority_repair", + file: "lib/group/replica/data.ex", + correct_source: " repair_primary_replica_rows(name, shard)", + faulty_source: + " if Process.get(:run_primary_replica_repair, false),\n" <> + " do: repair_primary_replica_rows(name, shard),\n" <> + " else: :ok", + test: ["test/anti_entropy_fault_regression_test.exs:3016"] + }, + %{ + name: "project_stale_claims_before_restart_repair", + file: "lib/group/replica.ex", + correct_source: """ + state = replay_local_journal(state) + :ok = Data.repair_shard_indexes(name, shard_index) + {state, _events} = rebuild_registry_projections(state) + """, + faulty_source: """ + state = replay_local_journal(state) + {state, _events} = rebuild_registry_projections(state) + :ok = Data.repair_shard_indexes(name, shard_index) + """, + test: ["test/anti_entropy_fault_regression_test.exs:3448"] + }, + %{ + name: "skip_interrupted_snapshot_install_repair", + file: "lib/group/replica/data.ex", + correct_source: " repair_interrupted_snapshot_installs(name, shard)", + faulty_source: + " if Process.get(:run_snapshot_install_repair, false),\n" <> + " do: repair_interrupted_snapshot_installs(name, shard),\n" <> + " else: :ok", + test: ["test/anti_entropy_fault_regression_test.exs:3128"] + }, + %{ + name: "retain_cursorless_remote_registry_claims", + file: "lib/group/replica/data.ex", + correct_source: + " :ets.member(replica_cursor_table(name, shard), stream_id)\n else\n false\n end\n end\n\n defp valid_remote_pg_authority?", + faulty_source: + " is_tuple(stream_id)\n else\n false\n end\n end\n\n defp valid_remote_pg_authority?", + test: ["test/anti_entropy_fault_regression_test.exs:3016"] + }, + %{ + name: "restart_snapshot_from_first_chunk_after_busy", + file: "lib/group/replica.ex", + correct_source: " start_index = Map.get(offsets, snapshot_key, 1)", + faulty_source: " _ = {offsets, snapshot_key}\n start_index = 1", + test: ["test/replica_snapshot_distributed_test.exs:635"] + }, + %{ + name: "drain_oversized_ingress_batch_without_yield", + file: "lib/group/replica.ex", + correct_source: " {turn, remaining} = Enum.split(messages, @incoming_batch_quota)", + faulty_source: " _ = @incoming_batch_quota\n turn = messages\n remaining = []", + test: ["test/group_test.exs:37"] } ] + def run(["--list"]) do + verify_mutation_definitions!(@mutations) + Enum.each(@mutations, &IO.puts(&1.name)) + end + def run(args) do selected = select_mutations(args) + verify_mutation_definitions!(selected) campaign_dir = campaign_dir() File.mkdir_p!(campaign_dir) @@ -397,11 +899,6 @@ defmodule Group.MutationCampaign do end end - defp select_mutations(["--list"]) do - Enum.each(@mutations, &IO.puts(&1.name)) - System.halt(0) - end - defp select_mutations([]), do: @mutations defp select_mutations(names) do @@ -415,6 +912,23 @@ defmodule Group.MutationCampaign do Enum.map(names, &Map.fetch!(by_name, &1)) end + defp verify_mutation_definitions!(mutations) do + invalid = + Enum.flat_map(mutations, fn mutation -> + verify_test_selectors!(mutation.test) + + source = mutation.file |> then(&Path.join(@repo, &1)) |> File.read!() + matches = :binary.matches(source, mutation.correct_source) + + if length(matches) == 1, do: [], else: [{mutation.name, length(matches)}] + end) + + if invalid != [] do + details = Enum.map_join(invalid, ", ", fn {name, count} -> "#{name}=#{count}" end) + raise "mutation definitions must match exactly one production fragment: #{details}" + end + end + defp verify_baselines!(mutations, campaign_dir) do mutations |> Enum.map(& &1.test) @@ -435,6 +949,27 @@ defmodule Group.MutationCampaign do end) end + # ExUnit accepts a line anywhere inside a test body, which can silently run + # the preceding test after source edits shift declarations. Mutation tests + # must point at the declaration itself so a stale selector is invalid rather + # than being mistaken for evidence that a mutant survived. + defp verify_test_selectors!(selectors) do + Enum.each(selectors, fn selector -> + with [_, relative, line] <- Regex.run(~r/^(.*):(\d+)$/, selector), + {line, ""} <- Integer.parse(line), + source when is_binary(source) <- File.read!(Path.join(@repo, relative)), + declaration when is_binary(declaration) <- + Enum.at(String.split(source, "\n"), line - 1), + true <- + String.starts_with?(String.trim_leading(declaration), ["test \"", "property \""]) do + :ok + else + _ -> + raise "mutation selector must name an exact test/property declaration: #{selector}" + end + end) + end + defp run_mutant(mutation, campaign_dir) do work = Path.join(campaign_dir, mutation.name) File.mkdir_p!(work) diff --git a/test/replica_adversarial_test.exs b/test/replica_adversarial_test.exs index 9bcfa4c..3b5003f 100644 --- a/test/replica_adversarial_test.exs +++ b/test/replica_adversarial_test.exs @@ -275,9 +275,21 @@ defmodule Group.ReplicaAdversarialTest do args = [name, key, cluster_opts(cluster)] values = Map.new(nodes, &{&1, TestCluster.rpc!(&1, Group, :lookup, args)}) - if values |> Map.values() |> Enum.uniq() |> length() == 1, - do: [], - else: [{:registry, cluster, key, values}] + if values |> Map.values() |> Enum.uniq() |> length() == 1 do + [] + else + replica = + Map.new(nodes, fn node -> + {node, + TestCluster.rpc!(node, Group.TestCluster, :replica_registry_key_state, [ + name, + cluster, + key + ])} + end) + + [{:registry, cluster, key, values, replica}] + end end) |> Enum.take(10) diff --git a/test/replica_snapshot_distributed_test.exs b/test/replica_snapshot_distributed_test.exs index b966b78..b04b3f2 100644 --- a/test/replica_snapshot_distributed_test.exs +++ b/test/replica_snapshot_distributed_test.exs @@ -7,10 +7,10 @@ defmodule Group.ReplicaSnapshotDistributedTest do alias Group.TestCluster setup_all do - peers = TestCluster.start_peers(2, schedulers: 4) + peers = TestCluster.start_peers(3, schedulers: 4) on_exit(fn -> TestCluster.stop_peers(peers) end) - [{_, node_a}, {_, node_b}] = peers - {:ok, node_a: node_a, node_b: node_b} + [{_, node_a}, {_, node_b}, {_, node_c}] = peers + {:ok, node_a: node_a, node_b: node_b, node_c: node_c} end test "loss, reordering, and duplication expose nothing until exact commit", context do @@ -237,6 +237,52 @@ defmodule Group.ReplicaSnapshotDistributedTest do assert snapshot_staging_tables(node_b, name) == [] end + test "an incomplete current-authority snapshot expires without touching visible state", + context do + %{name: name, node_a: node_a, node_b: node_b} = + start_pair(context, + replicated_anti_entropy_interval: 25, + replicated_peer_lease_timeout: 250 + ) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + entries = + for index <- 1..8 do + key = "snapshot/expiry/#{index}" + + {key, + TestCluster.spawn_register(node_a, name, key, %{ + payload: String.duplicate("x", 160) + })} + end + + TestCluster.flush_shards(node_a, name) + stream_id = local_stream(node_a, name, nil) + frames = capture_snapshot(node_a, node_b, name, stream_id, 1) + assert length(frames) > 1 + + deliver_frames(node_b, node_a, name, [hd(frames)]) + assert snapshot_transfer_count(node_b, name) == 1 + assert replica_cursor(node_b, name, stream_id) == 0 + + assert Enum.all?(entries, fn {key, _pid} -> + TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + end) + + TestCluster.assert_eventually( + fn -> snapshot_transfer_count(node_b, name) == 0 end, + timeout: 2_000, + interval: 25 + ) + + assert replica_cursor(node_b, name, stream_id) == 0 + + assert Enum.all?(entries, fn {key, _pid} -> + TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + end) + end + test "nodedown immediately destroys partial staging owned by the retired source", context do %{name: name, node_a: node_a, node_b: node_b} = start_pair(context) on_exit(fn -> TestCluster.reconnect_nodes(node_a, node_b) end) @@ -321,7 +367,9 @@ defmodule Group.ReplicaSnapshotDistributedTest do not is_nil(epoch) and epoch != old_epoch end) - deliver_frames(node_b, node_a, name, [last]) + # Replay the entire retired snapshot. A receiver that checks authority only + # when the transfer was first created would now stage and commit it. + deliver_frames(node_b, node_a, name, partial ++ [last]) TestCluster.flush_shards(node_b, name) assert Enum.all?(old_entries, fn {key, _pid} -> @@ -498,7 +546,7 @@ defmodule Group.ReplicaSnapshotDistributedTest do replicated_peer_lease_timeout: 120_000 ] - for node <- [context.node_a, context.node_b] do + for node <- [context.node_a, context.node_b, context.node_c] do {:ok, _pid} = TestCluster.start_group(node, opts) end @@ -584,6 +632,58 @@ defmodule Group.ReplicaSnapshotDistributedTest do end) end + test "a bounded transport makes forward progress across a snapshot larger than its window", + context do + %{name: name, node_a: node_a, node_b: node_b} = + start_pair(context, + replicated_sender_buffer_size: 1, + replicated_oplog_max_entries: 2, + replicated_snapshot_chunk_target_bytes: 700, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ) + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + entries = + for index <- 1..20 do + key = "snapshot/window/#{index}" + + {key, + TestCluster.spawn_register(node_a, name, key, %{ + payload: String.duplicate("w", 160) + })} + end + + TestCluster.flush_shards(node_a, name) + stream_id = local_stream(node_a, name, nil) + + request_snapshot_window(node_a, node_b, name, stream_id, 2) + TestCluster.flush_shards(node_b, name) + + {received, chunk_count} = snapshot_transfer_progress(node_b, name) + assert received == 2 + assert chunk_count > received + + for _attempt <- 2..ceil_div(chunk_count, 2) do + request_snapshot_window(node_a, node_b, name, stream_id, 2) + TestCluster.flush_shards(node_b, name) + end + + TestCluster.assert_eventually( + fn -> + Enum.all?(entries, fn {key, pid} -> + match?({^pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + end, + timeout: 5_000, + interval: 25 + ) + + assert snapshot_transfer_count(node_b, name) == 0 + assert :ok = TestCluster.rpc!(node_b, Group.TestCluster, :assert_replica_consistent, [name]) + end + defp start_pair(context, extra_opts \\ []) do name = :"snapshot_chunks_#{System.unique_integer([:positive])}" @@ -602,16 +702,24 @@ defmodule Group.ReplicaSnapshotDistributedTest do extra_opts ) - for node <- [context.node_a, context.node_b] do + for node <- [context.node_a, context.node_b, context.node_c] do {:ok, _pid} = TestCluster.start_group(node, opts) end TestCluster.assert_eventually(fn -> context.node_b in TestCluster.rpc!(context.node_a, Group, :nodes, [name]) and - context.node_a in TestCluster.rpc!(context.node_b, Group, :nodes, [name]) + context.node_c in TestCluster.rpc!(context.node_a, Group, :nodes, [name]) and + context.node_a in TestCluster.rpc!(context.node_b, Group, :nodes, [name]) and + context.node_a in TestCluster.rpc!(context.node_c, Group, :nodes, [name]) end) - %{name: name, node_a: context.node_a, node_b: context.node_b, opts: opts} + %{ + name: name, + node_a: context.node_a, + node_b: context.node_b, + node_c: context.node_c, + opts: opts + } end defp capture_snapshot(node_a, node_b, name, stream_id, next_seq) do @@ -631,7 +739,21 @@ defmodule Group.ReplicaSnapshotDistributedTest do {:needs, Group.Replica.WireProtocol.version(), [{stream_id, next_seq}]} ]) - TestCluster.flush_shards(node_a, name) + TestCluster.assert_eventually( + fn -> + captured = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :captured, [name]) + + snapshot_send = + TestCluster.rpc!(node_a, :sys, :get_state, [Group.Replica.shard_name(name, 0)]).snapshot_send + + Enum.any?(captured, fn + {^node_b, 0, {:snapshot_chunk, _, ^stream_id, _, _, _, _, _, _, _}} -> true + _ -> false + end) and is_nil(snapshot_send) + end, + timeout: 5_000, + interval: 10 + ) TestCluster.rpc!(node_a, Group.TestReplicaTransport, :captured, [name]) |> Enum.flat_map(fn @@ -646,6 +768,43 @@ defmodule Group.ReplicaSnapshotDistributedTest do |> Enum.sort_by(&elem(&1, 4)) end + defp request_snapshot_window(node_a, node_b, name, stream_id, limit) do + :ok = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [ + name, + {:accept_types_up_to, [:snapshot_chunk], limit} + ]) + + :ok = + TestCluster.rpc!(node_a, Group.Transport, :incoming, [ + name, + node_b, + 0, + {:needs, Group.Replica.WireProtocol.version(), [{stream_id, 1}]} + ]) + + source = TestCluster.rpc!(node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + _state = TestCluster.rpc!(node_a, :sys, :get_state, [source]) + + TestCluster.assert_eventually( + fn -> + TestCluster.rpc!(node_a, :sys, :get_state, [source]).snapshot_send == nil + end, + timeout: 5_000, + interval: 10 + ) + end + + defp snapshot_transfer_progress(node, name) do + state = + TestCluster.rpc!(node, :sys, :get_state, [Group.Replica.shard_name(name, 0)]) + + [{_key, transfer}] = Map.to_list(state.snapshot_transfers) + {MapSet.size(transfer.received), transfer.chunk_count} + end + + defp ceil_div(value, divisor), do: div(value + divisor - 1, divisor) + defp deliver_frames(node_b, node_a, name, frames) do Enum.each(frames, fn frame -> :ok = diff --git a/test/replica_transport_outbox_test.exs b/test/replica_transport_outbox_test.exs index 64aadea..bec5990 100644 --- a/test/replica_transport_outbox_test.exs +++ b/test/replica_transport_outbox_test.exs @@ -80,6 +80,31 @@ defmodule Group.ReplicaTransportOutboxTest do refute_receive {:outbox_batch, ^group, 0, ^target, [:expires_behind_backend], _deadline}, 300 end + test "bounded admission drops excess messages before the local mailbox grows" do + group = unique_group(:bounded_admission) + target = :"outbox-bounded@test" + + start_outboxes(group, + outbox_max_messages: 8, + outbox_batch_size: 8, + outbox_flush_interval: 1_000 + ) + + outbox = Process.whereis(Outbox.name(group, 0)) + :ok = :sys.suspend(outbox) + + results = + for message <- 1..32 do + Outbox.push(group, target, 0, message, outbox_deadline: 1_000) + end + + assert Enum.count(results, &(&1 == :ok)) == 8 + assert Enum.count(results, &(&1 == :busy)) == 24 + assert {:message_queue_len, 8} = Process.info(outbox, :message_queue_len) + + :ok = :sys.resume(outbox) + end + test "expired messages and backend backpressure are dropped without local retries" do expired_group = unique_group(:expired) target = :"outbox-expired@test" @@ -162,6 +187,16 @@ defmodule Group.ReplicaTransportOutboxTest do outbox_deadline: 0 ) end + + assert_raise ArgumentError, ~r/:outbox_max_messages/, fn -> + Group.Transport.Outbox.Supervisor.init( + name: group, + num_shards: 1, + backend: Backend, + controller: self(), + outbox_max_messages: 0 + ) + end end defp start_outboxes(group, opts) do diff --git a/test/support/cyclic_conflict_resolver.ex b/test/support/cyclic_conflict_resolver.ex new file mode 100644 index 0000000..ba25950 --- /dev/null +++ b/test/support/cyclic_conflict_resolver.ex @@ -0,0 +1,19 @@ +defmodule Group.CyclicConflictResolver do + @moduledoc false + + # New contract: rank one claim. Erlang term ordering plus Group's PID + # tiebreaker makes max/2 associative and independent of arrival order. + def resolve(_name, _key, {_pid, %{rank: rank}, _time}), do: rank + + # Old contract: a deterministic but cyclic pairwise choice. This exists only + # so the regression demonstrates why arbitrary pairwise resolvers are unsafe. + def resolve(_name, _key, {pid1, %{rank: rank1}, _time1}, {pid2, %{rank: rank2}, _time2}) do + if beats?(rank1, rank2), do: pid1, else: pid2 + end + + defp beats?(:a, :b), do: true + defp beats?(:b, :c), do: true + defp beats?(:c, :a), do: true + defp beats?(rank, rank), do: true + defp beats?(_left, _right), do: false +end diff --git a/test/support/model_conflict_resolver.ex b/test/support/model_conflict_resolver.ex index 6ac3502..58ed671 100644 --- a/test/support/model_conflict_resolver.ex +++ b/test/support/model_conflict_resolver.ex @@ -1,15 +1,5 @@ defmodule Group.ModelConflictResolver do @moduledoc false - def resolve(_name, _key, {pid1, meta1, _time1}, {pid2, meta2, _time2}) do - rank1 = Map.fetch!(meta1, :rank) - rank2 = Map.fetch!(meta2, :rank) - - cond do - rank1 > rank2 -> pid1 - rank2 > rank1 -> pid2 - pid1 > pid2 -> pid1 - true -> pid2 - end - end + def resolve(_name, _key, {_pid, meta, _time}), do: Map.fetch!(meta, :rank) end diff --git a/test/support/pausing_conflict_resolver.ex b/test/support/pausing_conflict_resolver.ex new file mode 100644 index 0000000..16e3cfd --- /dev/null +++ b/test/support/pausing_conflict_resolver.ex @@ -0,0 +1,19 @@ +defmodule Group.PausingConflictResolver do + @moduledoc false + + # Test-only deterministic scheduling point. The resolver still returns the + # normal total rank; a marked claim merely waits until the test releases the + # replica shard that is evaluating it. + def resolve(_name, key, {pid, %{rank: rank} = meta, _time}, controller) do + if Map.get(meta, :pause, false) do + ref = make_ref() + send(controller, {:conflict_resolver_waiting, self(), ref, key, pid}) + + receive do + {:continue_conflict_resolution, ^ref} -> :ok + end + end + + rank + end +end diff --git a/test/support/test_cluster.ex b/test/support/test_cluster.ex index 5c5990d..62836a3 100644 --- a/test/support/test_cluster.ex +++ b/test/support/test_cluster.ex @@ -60,6 +60,39 @@ defmodule Group.TestCluster do :ok end + @doc false + def expire_replica_lane(name, shard, remote_node) do + replica = Process.whereis(Group.Replica.shard_name(name, shard)) + now = System.monotonic_time(:millisecond) + + :sys.replace_state(replica, fn state -> + expired_at = now - state.replicated_peer_lease_timeout - 1 + %{state | peer_last_seen: Map.put(state.peer_last_seen, remote_node, expired_at)} + end) + + state = :sys.get_state(replica) + send(replica, {:group_replica_anti_entropy, state.anti_entropy_ref}) + _state = :sys.get_state(replica) + :ok + end + + @doc false + def put_pending_registry_reprojection(replica, remote_node, cluster, key) do + :sys.replace_state(replica, fn state -> + pending = + Map.update( + state.pending_registry_reprojections, + remote_node, + MapSet.new([{cluster, key}]), + &MapSet.put(&1, {cluster, key}) + ) + + %{state | pending_registry_reprojections: pending} + end) + + :ok + end + @doc "Call a function on a remote node, raise on badrpc" def rpc!(node, mod, fun, args) do case :rpc.call(node, mod, fun, args) do @@ -68,6 +101,19 @@ defmodule Group.TestCluster do end end + @doc false + def spawn_trace_forwarder(node, target_pid) do + :erpc.call(node, fn -> spawn(fn -> forward_trace_messages(target_pid) end) end) + end + + defp forward_trace_messages(target_pid) do + receive do + message -> + send(target_pid, {:forwarded_trace, message}) + forward_trace_messages(target_pid) + end + end + @doc "Start Group on a remote node" def start_group(node, opts) do opts = Keyword.put_new(opts, :log, false) @@ -175,14 +221,14 @@ defmodule Group.TestCluster do @doc "Spawn a process on a remote node that registers, joins, and sleeps forever. Waits for both operations to complete before returning." - def spawn_register_and_join(node, name, reg_key, reg_meta, join_key, join_meta) do + def spawn_register_and_join(node, name, reg_key, reg_meta, join_key, join_meta, opts \\ []) do :erpc.call(node, fn -> parent = self() pid = spawn(fn -> - :ok = Group.register(name, reg_key, reg_meta) - :ok = Group.join(name, join_key, join_meta) + :ok = Group.register(name, reg_key, reg_meta, opts) + :ok = Group.join(name, join_key, join_meta, opts) send(parent, {:ready, self()}) Process.sleep(:infinity) end) @@ -730,8 +776,11 @@ defmodule Group.TestCluster do num_shards = Group.get_config(name).num_shards for shard <- 0..(num_shards - 1) do + assert_rows_on_matching_shard(name, shard, num_shards) assert_registry_claim_indexes(name, shard) assert_registry_projection_has_authority(name, shard) + assert_registry_claim_authority(name, shard) + assert_pg_row_authority(name, shard) assert_oplog_indexes(name, shard) assert_replica_cursor_authority(name, shard) end @@ -739,6 +788,37 @@ defmodule Group.TestCluster do :ok end + defp assert_rows_on_matching_shard(name, shard, num_shards) do + checks = [ + {Group.Replica.Data.reg_by_key_table(name, shard), + fn + {{cluster, key}, _pid, _meta, _time, _origin} -> {cluster, key} + end}, + {Group.Replica.Data.reg_claim_by_key_table(name, shard), + fn + {{cluster, key, _origin, _generation, _epoch}, _pid, _meta, _time, _seq} -> + {cluster, key} + end}, + {Group.Replica.Data.pg_by_key_table(name, shard), + fn + {{cluster, key, _pid}, _meta, _time, _origin} -> {cluster, key} + end} + ] + + Enum.each(checks, fn {table, key_fun} -> + case Enum.find(:ets.tab2list(table), fn row -> + {cluster, key} = key_fun.(row) + Group.Replica.shard_index_for(cluster, key, num_shards) != shard + end) do + nil -> + :ok + + row -> + raise "row stored on wrong shard in #{name} shard #{shard}: #{inspect(row)}" + end + end) + end + @doc false def assert_replica_origin_purged(name, origin) do num_shards = Group.get_config(name).num_shards @@ -785,6 +865,7 @@ defmodule Group.TestCluster do end unless is_nil(Group.Replica.Data.remote_generation(name, origin)) and + is_nil(Group.Replica.Data.remote_replica_authority_hint(name, origin)) and Group.Replica.Data.clusters_for_node(name, origin) == [] do raise "replica origin retained shared authority after purge: #{inspect(origin)}" end @@ -836,6 +917,94 @@ defmodule Group.TestCluster do end end + @doc false + def replica_registry_key_state(name, cluster, key) do + num_shards = Group.get_config(name).num_shards + shard = Group.Replica.shard_index_for(cluster, key, num_shards) + + %{ + shard: shard, + projection: Group.Replica.Data.registry_lookup(name, shard, cluster, key), + claims: Group.Replica.Data.registry_claims(name, shard, cluster, key) + } + end + + @doc false + def replica_registry_replication_state(name, cluster, key, origin) do + num_shards = Group.get_config(name).num_shards + shard = Group.Replica.shard_index_for(cluster, key, num_shards) + replica = Process.whereis(Group.Replica.shard_name(name, shard)) + replica_state = :sys.get_state(replica) + local_stream_id = Group.Replica.Data.local_stream_id(name, shard, cluster) + + streams = + Group.Replica.Data.replica_cursor_streams_for_origin_cluster( + name, + shard, + origin, + cluster + ) + + peer_authority = + Node.list() + |> Map.new(fn peer -> + {peer, + %{ + generation: Group.Replica.Data.remote_generation(name, peer), + epoch: Group.Replica.Data.remote_cluster_epoch(name, peer, cluster), + revision: Group.Replica.Data.remote_cluster_epoch_revision(name, peer), + exact: Group.Replica.Data.remote_cluster_epoch_exact_revision(name, peer), + observed: Group.Replica.Data.remote_cluster_epoch_observed_revision(name, peer), + hint: Group.Replica.Data.remote_replica_authority_hint(name, peer), + lane_view: { + Group.Replica.Data.remote_view_generation(name, shard, peer), + Group.Replica.Data.remote_view_cluster_epoch_revision(name, shard, peer), + Group.Replica.Data.remote_view_observed_revision(name, shard, peer) + } + }} + end) + + %{ + key: replica_registry_key_state(name, cluster, key), + origin: origin, + cluster_nodes: Group.nodes(name, cluster), + local_generation: Group.Replica.Data.generation(name), + local_epoch: Group.Replica.Data.local_cluster_epoch(name, cluster), + local_revision: Group.Replica.Data.local_cluster_epoch_revision(name), + local_stream: + if(local_stream_id, + do: + {local_stream_id, + Group.Replica.Data.replica_stream_head(name, shard, local_stream_id)}, + else: nil + ), + peer_authority: peer_authority, + remote_generation: Group.Replica.Data.remote_generation(name, origin), + remote_epoch: Group.Replica.Data.remote_cluster_epoch(name, origin, cluster), + remote_revision: Group.Replica.Data.remote_cluster_epoch_revision(name, origin), + remote_exact_revision: Group.Replica.Data.remote_cluster_epoch_exact_revision(name, origin), + remote_observed_revision: + Group.Replica.Data.remote_cluster_epoch_observed_revision(name, origin), + remote_authority_hint: Group.Replica.Data.remote_replica_authority_hint(name, origin), + lane_view: { + Group.Replica.Data.remote_view_generation(name, shard, origin), + Group.Replica.Data.remote_view_cluster_epoch_revision(name, shard, origin), + Group.Replica.Data.remote_view_observed_revision(name, shard, origin) + }, + cursors: + Enum.map(streams, fn stream_id -> + {stream_id, Group.Replica.Data.replica_cursor(name, shard, stream_id)} + end), + remote_shards: Map.keys(replica_state.remote_shards), + peer_last_seen_nodes: Map.keys(replica_state.peer_last_seen), + peer_last_seen: Map.get(replica_state.peer_last_seen, origin), + pending_reprojection?: + replica_state.pending_registry_reprojections + |> Map.get(origin, MapSet.new()) + |> MapSet.member?({cluster, key}) + } + end + defp assert_registry_claim_indexes(name, shard) do by_key = Group.Replica.Data.reg_claim_by_key_table(name, shard) by_pid = Group.Replica.Data.reg_claim_by_pid_table(name, shard) @@ -869,12 +1038,32 @@ defmodule Group.TestCluster do end defp assert_registry_projection_has_authority(name, shard) do - claims = + claim_rows = Group.Replica.Data.reg_claim_by_key_table(name, shard) |> :ets.tab2list() - |> MapSet.new(fn - {{cluster, key, origin, _generation, _epoch}, pid, meta, time, _seq} -> - {cluster, key, pid, meta, time, origin} + + resolver = Map.get(Group.get_config(name), :resolve_registry_conflict) + + expected = + claim_rows + |> Enum.group_by(fn + {{cluster, key, _origin, _generation, _epoch}, _pid, _meta, _time, _seq} -> + {cluster, key} + end) + |> MapSet.new(fn {{cluster, key}, claims} -> + {{^cluster, ^key, origin, _generation, _epoch}, pid, meta, time, _seq} = + Enum.max_by(claims, fn + {{^cluster, ^key, _origin, _generation, _epoch}, claim_pid, claim_meta, claim_time, + _seq} -> + registry_claim_order_key( + name, + key, + {claim_pid, claim_meta, claim_time}, + resolver + ) + end) + + {cluster, key, pid, meta, time, origin} end) visible = @@ -884,25 +1073,66 @@ defmodule Group.TestCluster do {cluster, key, pid, meta, time, origin} end) - missing_authority = MapSet.difference(visible, claims) + if visible != expected do + raise "registry projection does not match the deterministic claim winner in #{name} " <> + "shard #{shard}: expected=#{inspect(MapSet.to_list(expected))} " <> + "visible=#{inspect(MapSet.to_list(visible))}" + end + end - claimed_keys = - MapSet.new(claims, fn {cluster, key, _pid, _meta, _time, _origin} -> {cluster, key} end) + defp registry_claim_order_key(_name, _key, {pid, _meta, time}, nil), do: {time, pid} - visible_keys = - MapSet.new(visible, fn {cluster, key, _pid, _meta, _time, _origin} -> {cluster, key} end) + defp registry_claim_order_key(name, key, {pid, _meta, _time} = claim, { + mod, + func, + extra_args + }) do + {apply(mod, func, [name, key, claim | extra_args]), pid} + end - missing_projection = MapSet.difference(claimed_keys, visible_keys) + defp assert_registry_claim_authority(name, shard) do + Group.Replica.Data.reg_claim_by_key_table(name, shard) + |> :ets.tab2list() + |> Enum.each(fn + {{cluster, key, origin, generation, epoch}, pid, _meta, _time, seq} = claim -> + valid? = + if origin == node() do + generation == Group.Replica.Data.generation(name) and + epoch == Group.Replica.Data.local_cluster_epoch(name, cluster) + else + generation == Group.Replica.Data.remote_generation(name, origin) and + remote_lane_current?(name, shard, origin) and + epoch == Group.Replica.Data.remote_cluster_epoch(name, origin, cluster) + end + + unless valid? and node(pid) == origin and seq > 0 do + raise "registry claim is not fenced by current authority in #{name} shard #{shard}: " <> + inspect({claim, key}) + end + end) + end - if MapSet.size(missing_authority) > 0 do - raise "visible registry rows without an authoritative claim in #{name} shard #{shard}: " <> - inspect(MapSet.to_list(missing_authority)) - end + defp assert_pg_row_authority(name, shard) do + Group.Replica.Data.pg_by_key_table(name, shard) + |> :ets.tab2list() + |> Enum.each(fn {{cluster, key, pid}, _meta, _time, origin} = row -> + valid? = + node(pid) == origin and + if origin == node() do + not is_nil(Group.Replica.Data.local_cluster_epoch(name, cluster)) + else + generation = Group.Replica.Data.remote_generation(name, origin) - if MapSet.size(missing_projection) > 0 do - raise "authoritative registry claims without a visible projection in #{name} shard #{shard}: " <> - inspect(MapSet.to_list(missing_projection)) - end + not is_nil(generation) and + remote_lane_current?(name, shard, origin) and + not is_nil(Group.Replica.Data.remote_cluster_epoch(name, origin, cluster)) + end + + unless valid? do + raise "PG row is not fenced by current authority in #{name} shard #{shard}: " <> + inspect({row, key}) + end + end) end defp assert_oplog_indexes(name, shard) do @@ -969,6 +1199,7 @@ defmodule Group.TestCluster do Group.Replica.WireProtocol.stream_shard(stream_id) == shard and origin != node() and generation == Group.Replica.Data.remote_generation(name, origin) and + remote_lane_current?(name, shard, origin) and epoch == Group.Replica.Data.remote_cluster_epoch(name, origin, cluster) and seq >= 0 @@ -979,27 +1210,49 @@ defmodule Group.TestCluster do end) end + defp remote_lane_current?(name, shard, origin) do + generation = Group.Replica.Data.remote_generation(name, origin) + observed = Group.Replica.Data.remote_cluster_epoch_observed_revision(name, origin) + + Group.Replica.Data.remote_replica_authority_hint(name, origin) == + {generation, observed} and + Group.Replica.Data.remote_cluster_epoch_revision(name, origin) == observed and + Group.Replica.Data.remote_view_generation(name, shard, origin) == generation and + Group.Replica.Data.remote_view_cluster_epoch_revision(name, shard, origin) == + Group.Replica.Data.remote_cluster_epoch_exact_revision(name, origin) and + Group.Replica.Data.remote_view_observed_revision(name, shard, origin) == + observed + end + @doc "Wait for a condition to become true, with retries" def assert_eventually(fun, opts \\ []) do timeout = Keyword.get(opts, :timeout, 2000) interval = Keyword.get(opts, :interval, 50) + diagnostic = Keyword.get(opts, :diagnostic) deadline = System.monotonic_time(:millisecond) + timeout - do_assert_eventually(fun, interval, deadline) + do_assert_eventually(fun, interval, deadline, diagnostic) end - defp do_assert_eventually(fun, interval, deadline) do + defp do_assert_eventually(fun, interval, deadline, diagnostic) do case fun.() do true -> true false -> if System.monotonic_time(:millisecond) >= deadline do - raise "assert_eventually timed out" + details = + if is_function(diagnostic, 0) do + "\ndiagnostic: " <> inspect(diagnostic.(), pretty: true, limit: :infinity) + else + "" + end + + raise "assert_eventually timed out#{details}" end Process.sleep(interval) - do_assert_eventually(fun, interval, deadline) + do_assert_eventually(fun, interval, deadline, diagnostic) end end end diff --git a/test/support/test_conflict_resolver.ex b/test/support/test_conflict_resolver.ex index ce41c34..c75e4e3 100644 --- a/test/support/test_conflict_resolver.ex +++ b/test/support/test_conflict_resolver.ex @@ -4,10 +4,5 @@ defmodule Group.TestConflictResolver do # A compiled module for conflict resolution that records calls to an ETS table. # Must be in test/support/ so it's compiled to beam and available on peer nodes. - def resolve(_name, _key, {pid1, _meta1, time1}, {pid2, _meta2, time2}) do - # Keep the one with the higher time (more recent wins). - # On equal timestamps, use pid ordering for a deterministic tiebreaker - # that doesn't depend on which node is resolving (avoids mutual kill). - if time2 > time1 or (time2 == time1 and pid2 > pid1), do: pid2, else: pid1 - end + def resolve(_name, _key, {_pid, _meta, time}), do: time end diff --git a/test/support/test_replica_transport.ex b/test/support/test_replica_transport.ex index f5ff194..3f961b0 100644 --- a/test/support/test_replica_transport.ex +++ b/test/support/test_replica_transport.ex @@ -15,8 +15,15 @@ defmodule Group.TestReplicaTransport do (is_tuple(mode) and tuple_size(mode) == 2 and elem(mode, 0) in [:drop_types, :duplicate_types, :capture_drop, :capture_pass]) or (is_tuple(mode) and tuple_size(mode) == 3 and elem(mode, 0) == :delay_types) or + (is_tuple(mode) and tuple_size(mode) == 3 and + elem(mode, 0) == :accept_types_up_to) or (is_tuple(mode) and tuple_size(mode) == 2 and elem(mode, 0) == :chaos) do :persistent_term.put({__MODULE__, group}, mode) + + if is_tuple(mode) and tuple_size(mode) == 3 and elem(mode, 0) == :accept_types_up_to do + :persistent_term.put({__MODULE__, group, :accepted}, 0) + end + :ok end @@ -32,6 +39,7 @@ defmodule Group.TestReplicaTransport do def clear(group) do :persistent_term.erase({__MODULE__, group}) :persistent_term.erase({__MODULE__, group, :captured}) + :persistent_term.erase({__MODULE__, group, :accepted}) :ok end @@ -65,6 +73,22 @@ defmodule Group.TestReplicaTransport do delay = Map.get(delays, message_type(message), default_delay) delayed_forward(group, target_node, shard, message, delay) + {:accept_types_up_to, types, limit} + when is_list(types) and is_integer(limit) and limit > 0 -> + if message_type(message) in types do + key = {__MODULE__, group, :accepted} + accepted = :persistent_term.get(key, 0) + + if accepted < limit do + :persistent_term.put(key, accepted + 1) + forward(group, target_node, shard, message) + else + :busy + end + else + forward(group, target_node, shard, message) + end + {:capture_drop, types} -> if message_type(message) in types, do: capture(group, target_node, shard, message) :ok From 401d19592d0991b0589ef47a18503fc63890e5c5 Mon Sep 17 00:00:00 2001 From: Chris McCord Date: Fri, 14 Aug 2026 20:19:00 +0000 Subject: [PATCH 10/16] Optimize anti-entropy recovery at million-row scale --- CLAUDE.md | 11 +- README.md | 7 + lib/group/replica.ex | 406 ++++++++++---------- lib/group/replica/data.ex | 313 +++++++++------ lib/group/replica/snapshot.ex | 166 +++++++- priv/bench/README.md | 16 + priv/bench/lib/group_bench/distributed.ex | 165 ++++++++ priv/bench/lib/group_bench/replica.ex | 182 +++++++++ priv/bench/mix.exs | 2 +- test/README.md | 2 +- test/anti_entropy_fault_regression_test.exs | 15 +- test/distributed_test.exs | 4 +- test/group_test.exs | 40 ++ test/mutation/README.md | 2 +- test/mutation/run.exs | 140 +++---- test/replica_snapshot_distributed_test.exs | 14 + test/replica_snapshot_test.exs | 75 ++++ 17 files changed, 1157 insertions(+), 403 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index c20b86d..a792db4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -68,6 +68,9 @@ Group.Supervisor (rest_for_one) `Group.Replica.Data` owns every ETS table. A shard restart therefore preserves the tables, repairs interrupted index/journal work, replays appended-but-not- applied local mutations, and rebuilds monitors for locally owned processes. +Restart repair streams each primary materialized table once while rebuilding +its existing reverse index; it does not allocate a whole-table list or retain +additional per-origin indexes. The optional transport precedes Data so losing its manager restarts the whole instance and cannot leave stale transport sessions attached to retained state. @@ -176,7 +179,13 @@ supersession, stale authority, expiry, and shard crash must leave no partial visible state. Staging expires after one peer-lease interval without progress. At most one sender worker per shard captures rows off the control process and sends only if the stream identity and fully-applied head remain unchanged after -the scan; otherwise anti-entropy retries. +the scan; otherwise anti-entropy retries. Sender capture and receiver staging +retain completed byte-bounded chunks in unnamed private ETS tables owned by +their worker/shard. Only one chunk is assembled or copied on a process heap at +a time. These tables are ephemeral: explicit completion/retry cleanup deletes +them, and owner death deletes them automatically. Monitor events produced by a +large exact install are staged in bounded private-ETS batches until the cursor +commits. ## Nonblocking Transport diff --git a/README.md b/README.md index 856edf1..e677dd2 100644 --- a/README.md +++ b/README.md @@ -524,6 +524,13 @@ Exact-snapshot row capture runs in at most one off-shard worker per shard. The worker sends only when the stream identity and fully-applied head are unchanged after its scans. A concurrent write invalidates the capture and periodic anti-entropy retries, keeping million-row scans off the Group control process. +The worker holds at most one byte-targeted chunk on its process heap and keeps +completed chunks in an unnamed private ETS table for that snapshot attempt. +The receiver uses the same bounded-chunk shape plus minimal row-presence markers +until exact commit. Both sides delete this ephemeral staging after completion or +retry, and owner death deletes it automatically; it is not steady-state state +or an additional authority source. Exact-install monitor events are likewise +buffered into bounded private-ETS batches before the cursor becomes visible. The optional `peer_up/5` and `peer_down/4` callbacks report one shard lane at a time. A sideband adapter that shares a single node connection must retain it diff --git a/lib/group/replica.ex b/lib/group/replica.ex index 63df82c..9bb6533 100644 --- a/lib/group/replica.ex +++ b/lib/group/replica.ex @@ -13,6 +13,7 @@ defmodule Group.Replica do @protocol_version Group.Replica.WireProtocol.version() @priority_control_quota 64 @incoming_batch_quota 64 + @snapshot_event_batch_size 512 _archdoc = ~S""" Sharded control process for local writes, replica transport, anti-entropy, @@ -174,7 +175,12 @@ defmodule Group.Replica do Exact-snapshot row capture runs in at most one off-shard worker per shard. It sends only if the local stream identity and fully-applied head are unchanged after both row scans; overlapping writes discard the capture and periodic - anti-entropy retries. This keeps million-row scans off the control process. + anti-entropy retries. The worker builds at most one byte-targeted chunk on its + heap and retains completed chunks only in an unnamed private ETS table owned + by that attempt. Receiver payload chunks and exact-install monitor-event + batches use the same ephemeral private-ETS ownership. Completion, retry, or + owner death deletes them. This keeps million-row scans and whole-snapshot + heaps off the control process without adding steady-state indexes. Incoming PG mutations retain the bulk receiver lane. Contiguous registry records in one stream run are projected together and emit one monitor event @@ -3363,8 +3369,38 @@ defmodule Group.Replica do end defp broadcast_replica_heads(state) do - Enum.reduce(state.peer_last_seen, state, fn {target_node, _last_seen}, acc -> - send_replica_heads(acc, target_node) + peers = Map.keys(state.peer_last_seen) + + heads_by_target = + state.name + |> Data.replica_stream_heads(state.shard_index) + |> Enum.reduce(%{}, fn {stream_id, _floor, _head} = head, acc -> + if current_local_replica_stream?(state, stream_id) do + targets = + case WireProtocol.stream_cluster(stream_id) do + nil -> + peers + + cluster -> + state.name + |> Data.cluster_nodes(cluster) + |> Enum.filter(&Map.has_key?(state.peer_last_seen, &1)) + end + + Enum.reduce(targets, acc, fn target_node, inner -> + Map.update(inner, target_node, [head], &[head | &1]) + end) + else + acc + end + end) + + Enum.reduce(heads_by_target, state, fn {target_node, heads}, acc -> + outgoing_replica_message( + acc, + target_node, + {:heads, WireProtocol.version(), Enum.reverse(heads)} + ) end) end @@ -3403,6 +3439,7 @@ defmodule Group.Replica do {transfer, transfers} -> :ok = Snapshot.delete_staging_table(transfer.table) + :ok = Snapshot.delete_staging_table(transfer.events) %{state | snapshot_transfers: transfers} end end @@ -3535,17 +3572,21 @@ defmodule Group.Replica do end defp replica_stream_target?(state, stream_id, target_node) do + current_local_replica_stream?(state, stream_id) and + case WireProtocol.stream_cluster(stream_id) do + nil -> Map.has_key?(state.peer_last_seen, target_node) + cluster -> target_node in Data.cluster_nodes(state.name, cluster) + end + end + + defp current_local_replica_stream?(state, stream_id) do WireProtocol.valid_stream_id?(stream_id) and WireProtocol.stream_name(stream_id) == state.name and WireProtocol.stream_origin(stream_id) == node() and WireProtocol.stream_shard(stream_id) == state.shard_index and WireProtocol.stream_generation(stream_id) == Data.generation(state.name) and WireProtocol.stream_epoch(stream_id) == - Data.local_cluster_epoch(state.name, WireProtocol.stream_cluster(stream_id)) and - case WireProtocol.stream_cluster(stream_id) do - nil -> Map.has_key?(state.peer_last_seen, target_node) - cluster -> target_node in Data.cluster_nodes(state.name, cluster) - end + Data.local_cluster_epoch(state.name, WireProtocol.stream_cluster(stream_id)) end defp valid_remote_stream?(state, source_node, stream_id) do @@ -3629,30 +3670,18 @@ defmodule Group.Replica do pg_data ) and valid_snapshot_rows?(state, source_node, stream_id, reg_data, pg_data) do - if chunk_count == 1 and registry_count == length(reg_data) and - pg_count == length(pg_data) do - apply_complete_snapshot_rows( - state, - source_node, - stream_id, - snapshot_seq, - reg_data, - pg_data - ) - else - stage_replica_snapshot_chunk( - state, - source_node, - stream_id, - snapshot_seq, - chunk_index, - chunk_count, - registry_count, - pg_count, - reg_data, - pg_data - ) - end + stage_replica_snapshot_chunk( + state, + source_node, + stream_id, + snapshot_seq, + chunk_index, + chunk_count, + registry_count, + pg_count, + reg_data, + pg_data + ) else state end @@ -3850,7 +3879,8 @@ defmodule Group.Replica do pg_seen: 0, received: MapSet.new(), last_progress: monotonic_millis(), - table: Snapshot.new_staging_table() + table: Snapshot.new_staging_table(), + events: Snapshot.new_event_table() } end @@ -3887,31 +3917,35 @@ defmodule Group.Replica do transfer.snapshot_seq ) - affected_registry_keys = + event_buffer = Snapshot.new_event_buffer(transfer.events) + + {state, event_buffer} = Data.replace_registry_claims_for_stream_from_staging( state.name, state.shard_index, stream_id, transfer.snapshot_seq, transfer.table, - transfer.chunk_count + transfer.chunk_count, + {state, event_buffer}, + fn key, {acc, buffer} -> + {acc, events} = reconcile_registry_projection(acc, cluster, key, :reconcile, []) + {acc, Snapshot.buffer_events(Enum.reverse(events), buffer)} + end ) - {state, events} = - Enum.reduce(affected_registry_keys, {state, []}, fn key, {acc, inner_events} -> - reconcile_registry_projection(acc, cluster, key, :reconcile, inner_events) - end) - - events = + event_buffer = replace_remote_pg_snapshot_from_staging( state, source_node, cluster, transfer.table, transfer.chunk_count, - events + event_buffer ) + _event_buffer = Snapshot.finish_event_buffer(event_buffer) + :ok = Data.put_replica_cursor( state.name, @@ -3920,7 +3954,7 @@ defmodule Group.Replica do transfer.snapshot_seq ) - notify_monitors(state.name, events) + notify_snapshot_events(state.name, transfer.events) state else state @@ -3929,45 +3963,6 @@ defmodule Group.Replica do discard_snapshot_transfer(state, key) end - defp apply_complete_snapshot_rows( - state, - source_node, - stream_id, - snapshot_seq, - reg_data, - pg_data - ) do - state = flush_pending_replicated_barrier(state) - cluster = WireProtocol.stream_cluster(stream_id) - - :ok = - Data.begin_replica_snapshot_install( - state.name, - state.shard_index, - stream_id, - snapshot_seq - ) - - affected_registry_keys = - Data.replace_registry_claims_for_stream( - state.name, - state.shard_index, - stream_id, - snapshot_seq, - reg_data - ) - - {state, events} = - Enum.reduce(affected_registry_keys, {state, []}, fn key, {acc, inner_events} -> - reconcile_registry_projection(acc, cluster, key, :reconcile, inner_events) - end) - - events = replace_remote_pg_snapshot_rows(state, source_node, cluster, pg_data, events) - :ok = Data.put_replica_cursor(state.name, state.shard_index, stream_id, snapshot_seq) - notify_monitors(state.name, events) - state - end - defp apply_replica_delta_run(state, source_node, stream_id, records, advertised_head) do if valid_remote_stream?(state, source_node, stream_id) do cursor = Data.replica_cursor(state.name, state.shard_index, stream_id) @@ -4385,53 +4380,87 @@ defmodule Group.Replica do defp capture_and_send_replica_snapshot(state, target_node, stream_id, head, start_index) do cluster = WireProtocol.stream_cluster(stream_id) - reg_data = Data.registry_claims_for_stream(state.name, state.shard_index, stream_id) - pg_data = Data.pg_entries_for_origin(state.name, state.shard_index, cluster, node()) - - {_floor, current_head, applied} = - Data.replica_stream_head(state.name, state.shard_index, stream_id) + capture_table = Snapshot.new_capture_table() + envelope_bytes = Snapshot.capture_envelope_bytes(stream_id, head) + + capture = + Snapshot.new_capture( + capture_table, + state.replicated_snapshot_chunk_target_bytes, + envelope_bytes + ) - # Appending a mutation advances the head before materializing its table - # changes. Therefore an unchanged, fully-applied head after both scans - # proves these rows are one exact state at `head`; an overlapping write - # makes the capture disposable and the receiver will ask again. - if Data.local_stream_id(state.name, state.shard_index, cluster) == stream_id and - current_head == head and applied == head and - target_node in Data.cluster_nodes(state.name, cluster) do - envelope_bytes = - Snapshot.frame_envelope_bytes(stream_id, head, length(reg_data), length(pg_data)) - - snapshot = - Snapshot.chunk_rows( - reg_data, - pg_data, - state.replicated_snapshot_chunk_target_bytes, - envelope_bytes + try do + capture = + Data.reduce_registry_claim_batches_for_stream( + state.name, + state.shard_index, + stream_id, + capture, + &Snapshot.capture_registry_many/2 ) - chunk_count = length(snapshot.chunks) - - snapshot.chunks - |> Enum.with_index(1) - |> Enum.drop(start_index - 1) - |> Enum.reduce_while(:complete, fn {{reg_chunk, pg_chunk}, chunk_index}, _acc -> - message = - {:snapshot_chunk, WireProtocol.version(), stream_id, head, chunk_index, chunk_count, - snapshot.registry_count, snapshot.pg_count, reg_chunk, pg_chunk} + capture = + Data.reduce_pg_entry_batches_for_origin( + state.name, + state.shard_index, + cluster, + node(), + capture, + &Snapshot.capture_pg_many/2 + ) - case state.replica_transport.outgoing( - state.name, - target_node, - state.shard_index, - message, - state.replica_transport_opts + capture = Snapshot.finish_capture(capture) + registry_count = capture.registry_count + pg_count = capture.pg_count + chunk_count = capture.chunk_count + + {_floor, current_head, applied} = + Data.replica_stream_head(state.name, state.shard_index, stream_id) + + # Appending advances the head before materializing its table changes. + # Therefore an unchanged, fully-applied head after both scans proves the + # private capture is one exact state at `head`; an overlapping write makes + # it disposable and anti-entropy retries without sending a partial view. + if Data.local_stream_id(state.name, state.shard_index, cluster) == stream_id and + current_head == head and applied == head and + target_node in Data.cluster_nodes(state.name, cluster) do + case Snapshot.reduce_capture_chunks( + capture_table, + chunk_count, + 1, + fn reg_chunk, pg_chunk, chunk_index -> + if chunk_index < start_index do + {:cont, chunk_index + 1} + else + message = + {:snapshot_chunk, WireProtocol.version(), stream_id, head, chunk_index, + chunk_count, registry_count, pg_count, reg_chunk, pg_chunk} + + case state.replica_transport.outgoing( + state.name, + target_node, + state.shard_index, + message, + state.replica_transport_opts + ) do + :ok -> + {:cont, chunk_index + 1} + + result when result in [:busy, :disconnected] -> + {:halt, {:resume, chunk_index}} + end + end + end ) do - :ok -> {:cont, :complete} - result when result in [:busy, :disconnected] -> {:halt, {:resume, chunk_index}} + {:ok, _next_index} -> :complete + {:halt, result} -> result end - end) - else - :complete + else + :complete + end + after + Snapshot.delete_staging_table(capture_table) end end @@ -4452,12 +4481,11 @@ defmodule Group.Replica do cluster, staging_table, chunk_count, - events + event_buffer ) do - current = Data.pg_entries_for_origin(state.name, state.shard_index, cluster, source_node) - - events = - Snapshot.fold_pg(staging_table, chunk_count, events, fn {key, pid, meta, time}, acc -> + event_buffer = + Snapshot.fold_pg(staging_table, chunk_count, event_buffer, fn {key, pid, meta, time}, + buffer -> case Data.pg_lookup(state.name, state.shard_index, cluster, key, pid) do nil -> :ok = @@ -4472,10 +4500,13 @@ defmodule Group.Replica do source_node ) - [build_event(state.name, :joined, key, pid, meta, %{cluster: cluster}) | acc] + Snapshot.buffer_event( + build_event(state.name, :joined, key, pid, meta, %{cluster: cluster}), + buffer + ) {^meta, ^time, ^source_node} -> - acc + buffer {old_meta, _old_time, ^source_node} -> :ok = @@ -4490,89 +4521,45 @@ defmodule Group.Replica do source_node ) - if old_meta == meta do - acc - else - [ + if old_meta != meta do + Snapshot.buffer_event( build_event(state.name, :joined, key, pid, meta, %{ previous_meta: old_meta, cluster: cluster - }) - | acc - ] + }), + buffer + ) + else + buffer end end end) - Enum.reduce(current, events, fn {key, pid, old_meta, _old_time}, acc -> - if Snapshot.member_pg?(staging_table, key, pid) do - acc - else - :ok = Data.pg_delete(state.name, state.shard_index, cluster, key, pid) - - event = - build_event(state.name, :left, key, pid, old_meta, %{ - reason: :reconcile, - cluster: cluster - }) - - [event | acc] - end - end) - end - - defp replace_remote_pg_snapshot_rows(state, source_node, cluster, pg_data, events) do - current = - state.name - |> Data.pg_entries_for_origin(state.shard_index, cluster, source_node) - |> Map.new(fn {key, pid, meta, time} -> {{key, pid}, {meta, time}} end) - - desired = - Map.new(pg_data, fn {key, pid, meta, time} -> {{key, pid}, {meta, time}} end) - - {inserts, deletes, events} = - current - |> Map.keys() - |> Kernel.++(Map.keys(desired)) - |> Enum.uniq() - |> Enum.reduce({[], [], events}, fn {key, pid}, {inserts, deletes, acc} -> - case {Map.get(current, {key, pid}), Map.get(desired, {key, pid})} do - {same, same} -> - {inserts, deletes, acc} + event_buffer = + Data.fold_pg_entries_for_origin( + state.name, + state.shard_index, + cluster, + source_node, + event_buffer, + fn {key, pid, old_meta, _old_time}, buffer -> + if Snapshot.member_pg?(staging_table, key, pid) do + buffer + else + :ok = Data.pg_delete(state.name, state.shard_index, cluster, key, pid) - {{old_meta, _old_time}, nil} -> event = build_event(state.name, :left, key, pid, old_meta, %{ reason: :reconcile, cluster: cluster }) - {inserts, [{cluster, key, pid} | deletes], [event | acc]} - - {nil, {meta, time}} -> - event = build_event(state.name, :joined, key, pid, meta, %{cluster: cluster}) - - {[{cluster, key, pid, meta, time, source_node} | inserts], deletes, [event | acc]} - - {{old_meta, _old_time}, {meta, time}} -> - event = - if old_meta == meta do - nil - else - build_event(state.name, :joined, key, pid, meta, %{ - previous_meta: old_meta, - cluster: cluster - }) - end - - acc = if event, do: [event | acc], else: acc - {[{cluster, key, pid, meta, time, source_node} | inserts], deletes, acc} + Snapshot.buffer_event(event, buffer) + end end - end) + ) - Data.pg_delete_many(state.name, state.shard_index, deletes) - Data.pg_insert_many(state.name, state.shard_index, inserts) - events + event_buffer end defp maybe_purge_remote_generation(state, _remote_node, nil, _generation), do: state @@ -5576,7 +5563,6 @@ defmodule Group.Replica do # Remote disconnects remove only data owned by the departing node. node_guard = if target == :all, do: [], else: [{:==, :"$5", target}] reg_table = Data.reg_by_key_table(name, shard) - reg_pid_table = Data.reg_by_pid_table(name, shard) purged_reg = :ets.select(reg_table, [ @@ -5586,12 +5572,10 @@ defmodule Group.Replica do |> Enum.map(fn {key, pid, meta, time} -> {cluster, key, pid, meta, time} end) for {^cluster, key, pid, _meta, _time} <- purged_reg do - :ets.delete(reg_table, {cluster, key}) - :ets.delete(reg_pid_table, {pid, cluster, key}) + Data.registry_delete(name, shard, cluster, key, pid) end pg_table = Data.pg_by_key_table(name, shard) - pg_pid_table = Data.pg_by_pid_table(name, shard) purged_pg = :ets.select(pg_table, [ @@ -5601,8 +5585,7 @@ defmodule Group.Replica do |> Enum.map(fn {key, pid, meta, time} -> {cluster, key, pid, meta, time} end) for {^cluster, key, pid, _meta, _time} <- purged_pg do - :ets.delete(pg_table, {cluster, key, pid}) - :ets.delete(pg_pid_table, {pid, cluster, key}) + Data.pg_delete(name, shard, cluster, key, pid) end {purged_reg, purged_pg} @@ -5678,6 +5661,23 @@ defmodule Group.Replica do :ok end + defp notify_snapshot_events(name, table) do + {events, _count} = + Snapshot.fold_events(table, {[], 0}, fn event, {events, count} -> + events = [event | events] + count = count + 1 + + if count >= @snapshot_event_batch_size do + notify_monitors(name, events) + {[], 0} + else + {events, count} + end + end) + + notify_monitors(name, events) + end + defp matching_subscribers(name, cluster, key, cache) do {all_subscribers, cache} = get_cached_subscribers(name, cluster, :all, cache) {exact_subscribers, cache} = get_cached_subscribers(name, cluster, {:exact, key}, cache) diff --git a/lib/group/replica/data.ex b/lib/group/replica/data.ex index 221531f..8717b48 100644 --- a/lib/group/replica/data.ex +++ b/lib/group/replica/data.ex @@ -593,9 +593,6 @@ defmodule Group.Replica.Data do def repair_shard_indexes(name, shard) do repair_interrupted_snapshot_installs(name, shard) repair_primary_replica_rows(name, shard) - rebuild_registry_reverse_index(name, shard) - rebuild_registry_claim_reverse_index(name, shard) - rebuild_pg_reverse_index(name, shard) :ok end @@ -687,58 +684,32 @@ defmodule Group.Replica.Data do end end - defp rebuild_registry_reverse_index(name, shard) do - reverse = reg_by_pid_table(name, shard) - :ets.delete_all_objects(reverse) - - reg_by_key_table(name, shard) - |> :ets.tab2list() - |> Enum.each(fn {{cluster, key}, pid, meta, time, entry_node} -> - :ets.insert(reverse, {{pid, cluster, key}, meta, time, entry_node}) - end) - end - - defp rebuild_registry_claim_reverse_index(name, shard) do - reverse = reg_claim_by_pid_table(name, shard) - :ets.delete_all_objects(reverse) - - reg_claim_by_key_table(name, shard) - |> :ets.tab2list() - |> Enum.each(fn {{cluster, key, origin, generation, epoch}, pid, meta, time, seq} -> - :ets.insert( - reverse, - {{pid, cluster, key, origin, generation, epoch}, meta, time, seq} - ) - end) - end - - defp rebuild_pg_reverse_index(name, shard) do - reverse = pg_by_pid_table(name, shard) - :ets.delete_all_objects(reverse) - - pg_by_key_table(name, shard) - |> :ets.tab2list() - |> Enum.each(fn {{cluster, key, pid}, meta, time, entry_node} -> - :ets.insert(reverse, {{pid, cluster, key}, meta, time, entry_node}) - end) - end - # A shard can crash between writes to its materialized rows and receive # cursor, or while retiring an epoch across multiple ETS tables. Recover from # the primary tables themselves: stale claims carry their complete stream # authority, while a remote PG row is retained only when the current stream # has a cursor (including the sequence-zero admission marker). This pass also # replaces the old multi-million-element cluster list with one fixed-table - # traversal and O(number of inactive clusters) accumulator memory. + # traversal and O(number of inactive clusters) accumulator memory. Reverse + # indexes are rebuilt during the same primary-table pass, avoiding both a + # second traversal and a complete `tab2list/1` heap copy per index. defp repair_primary_replica_rows(name, shard) do + reg_reverse = reg_by_pid_table(name, shard) + claim_reverse = reg_claim_by_pid_table(name, shard) + pg_reverse = pg_by_pid_table(name, shard) + :ets.delete_all_objects(reg_reverse) + :ets.delete_all_objects(claim_reverse) + :ets.delete_all_objects(pg_reverse) + inactive_clusters = MapSet.new() inactive_clusters = repair_ets_table( reg_by_key_table(name, shard), inactive_clusters, - fn {{cluster, key}, _pid, _meta, _time, _entry_node}, inactive -> + fn {{cluster, key}, pid, meta, time, entry_node}, inactive -> if active_local_cluster?(name, cluster) do + :ets.insert(reg_reverse, {{pid, cluster, key}, meta, time, entry_node}) inactive else :ets.delete(reg_by_key_table(name, shard), {cluster, key}) @@ -751,8 +722,7 @@ defmodule Group.Replica.Data do repair_ets_table( reg_claim_by_key_table(name, shard), inactive_clusters, - fn {{cluster, key, origin, claim_generation, epoch}, _pid, _meta, _time, _seq}, - inactive -> + fn {{cluster, key, origin, claim_generation, epoch}, pid, meta, time, seq}, inactive -> if active_local_cluster?(name, cluster) and valid_claim_authority?( name, @@ -762,6 +732,11 @@ defmodule Group.Replica.Data do claim_generation, epoch ) do + :ets.insert( + claim_reverse, + {{pid, cluster, key, origin, claim_generation, epoch}, meta, time, seq} + ) + inactive else :ets.delete( @@ -778,13 +753,14 @@ defmodule Group.Replica.Data do repair_ets_table( pg_by_key_table(name, shard), inactive_clusters, - fn {{cluster, key, pid}, _meta, _time, entry_node}, inactive -> + fn {{cluster, key, pid}, meta, time, entry_node}, inactive -> valid? = active_local_cluster?(name, cluster) and node(pid) == entry_node and (entry_node == node() or valid_remote_pg_authority?(name, shard, cluster, entry_node)) if valid? do + :ets.insert(pg_reverse, {{pid, cluster, key}, meta, time, entry_node}) inactive else :ets.delete(pg_by_key_table(name, shard), {cluster, key, pid}) @@ -1219,11 +1195,15 @@ defmodule Group.Replica.Data do case :ets.lookup(reg_claim_by_key_table(name, shard), claim_key) do [{^claim_key, ^pid, _meta, _time, old_seq}] when old_seq <= seq -> - :ets.delete(reg_claim_by_key_table(name, shard), claim_key) - - :ets.delete( - reg_claim_by_pid_table(name, shard), - {pid, cluster, key, origin_node, generation, epoch} + delete_registry_claim_indexes( + name, + shard, + cluster, + key, + origin_node, + generation, + epoch, + pid ) :ok @@ -1241,15 +1221,36 @@ defmodule Group.Replica.Data do end def registry_claims_for_stream(name, shard, stream_id) do + fold_registry_claims_for_stream(name, shard, stream_id, [], fn row, rows -> [row | rows] end) + |> Enum.reverse() + end + + def fold_registry_claims_for_stream(name, shard, stream_id, acc, fun) + when is_function(fun, 2) do + {table, match_spec} = registry_claim_stream_selection(name, shard, stream_id) + + fold_select_batches_fixed(table, match_spec, acc, fn rows, inner -> + Enum.reduce(rows, inner, fun) + end) + end + + def reduce_registry_claim_batches_for_stream(name, shard, stream_id, acc, fun) + when is_function(fun, 2) do + {table, match_spec} = registry_claim_stream_selection(name, shard, stream_id) + fold_select_batches(table, match_spec, acc, fun) + end + + defp registry_claim_stream_selection(name, shard, stream_id) do cluster = Group.Replica.WireProtocol.stream_cluster(stream_id) origin_node = Group.Replica.WireProtocol.stream_origin(stream_id) generation = Group.Replica.WireProtocol.stream_generation(stream_id) epoch = Group.Replica.WireProtocol.stream_epoch(stream_id) - :ets.select(reg_claim_by_key_table(name, shard), [ + { + reg_claim_by_key_table(name, shard), {{{cluster, :"$1", origin_node, generation, epoch}, :"$2", :"$3", :"$4", :_}, [], [{{:"$1", :"$2", :"$3", :"$4"}}]} - ]) + } end def replace_registry_claims_for_stream(name, shard, stream_id, snapshot_seq, claims) do @@ -1260,14 +1261,15 @@ defmodule Group.Replica.Data do existing = registry_claims_for_stream(name, shard, stream_id) Enum.each(existing, fn {key, pid, _meta, _time} -> - :ets.delete( - reg_claim_by_key_table(name, shard), - {cluster, key, origin_node, generation, epoch} - ) - - :ets.delete( - reg_claim_by_pid_table(name, shard), - {pid, cluster, key, origin_node, generation, epoch} + delete_registry_claim_indexes( + name, + shard, + cluster, + key, + origin_node, + generation, + epoch, + pid ) end) @@ -1284,41 +1286,46 @@ defmodule Group.Replica.Data do stream_id, snapshot_seq, staging_table, - chunk_count - ) do + chunk_count, + acc, + fun + ) + when is_function(fun, 2) do cluster = Group.Replica.WireProtocol.stream_cluster(stream_id) origin_node = Group.Replica.WireProtocol.stream_origin(stream_id) generation = Group.Replica.WireProtocol.stream_generation(stream_id) epoch = Group.Replica.WireProtocol.stream_epoch(stream_id) - existing = registry_claims_for_stream(name, shard, stream_id) - - keys = - Enum.reduce(existing, MapSet.new(), fn {key, pid, _meta, _time}, keys -> - :ets.delete( - reg_claim_by_key_table(name, shard), - {cluster, key, origin_node, generation, epoch} - ) - :ets.delete( - reg_claim_by_pid_table(name, shard), - {pid, cluster, key, origin_node, generation, epoch} - ) + acc = + fold_registry_claims_for_stream(name, shard, stream_id, acc, fn + {key, pid, _meta, _time}, inner -> + delete_registry_claim_indexes( + name, + shard, + cluster, + key, + origin_node, + generation, + epoch, + pid + ) - MapSet.put(keys, key) + if Group.Replica.Snapshot.member_registry?(staging_table, key) do + inner + else + fun.(key, inner) + end end) - keys = - Group.Replica.Snapshot.fold_registry( - staging_table, - chunk_count, - keys, - fn {key, pid, meta, time}, keys -> - put_registry_claim(name, shard, stream_id, snapshot_seq, key, pid, meta, time) - MapSet.put(keys, key) - end - ) - - MapSet.to_list(keys) + Group.Replica.Snapshot.fold_registry( + staging_table, + chunk_count, + acc, + fn {key, pid, meta, time}, inner -> + put_registry_claim(name, shard, stream_id, snapshot_seq, key, pid, meta, time) + fun.(key, inner) + end + ) end def purge_registry_claims_for_origin(name, shard, origin_node) do @@ -1329,14 +1336,15 @@ defmodule Group.Replica.Data do ]) Enum.each(claims, fn {cluster, key, pid, _meta, _time, generation, epoch} -> - :ets.delete( - reg_claim_by_key_table(name, shard), - {cluster, key, origin_node, generation, epoch} - ) - - :ets.delete( - reg_claim_by_pid_table(name, shard), - {pid, cluster, key, origin_node, generation, epoch} + delete_registry_claim_indexes( + name, + shard, + cluster, + key, + origin_node, + generation, + epoch, + pid ) end) @@ -1361,14 +1369,15 @@ defmodule Group.Replica.Data do claims = :ets.select(reg_claim_by_key_table(name, shard), match_specs) Enum.each(claims, fn {{cluster, key, origin, generation, epoch}, pid, _meta, _time, _seq} -> - :ets.delete( - reg_claim_by_key_table(name, shard), - {cluster, key, origin, generation, epoch} - ) - - :ets.delete( - reg_claim_by_pid_table(name, shard), - {pid, cluster, key, origin, generation, epoch} + delete_registry_claim_indexes( + name, + shard, + cluster, + key, + origin, + generation, + epoch, + pid ) end) @@ -1403,20 +1412,41 @@ defmodule Group.Replica.Data do defp delete_registry_claim_rows(name, shard, cluster, claims) do Enum.each(claims, fn {key, pid, _meta, _time, claim_origin, generation, epoch} -> - :ets.delete( - reg_claim_by_key_table(name, shard), - {cluster, key, claim_origin, generation, epoch} - ) - - :ets.delete( - reg_claim_by_pid_table(name, shard), - {pid, cluster, key, claim_origin, generation, epoch} + delete_registry_claim_indexes( + name, + shard, + cluster, + key, + claim_origin, + generation, + epoch, + pid ) end) Enum.uniq(Enum.map(claims, &elem(&1, 0))) end + defp delete_registry_claim_indexes( + name, + shard, + cluster, + key, + origin, + generation, + epoch, + pid + ) do + :ets.delete(reg_claim_by_key_table(name, shard), {cluster, key, origin, generation, epoch}) + + :ets.delete( + reg_claim_by_pid_table(name, shard), + {pid, cluster, key, origin, generation, epoch} + ) + + :ok + end + def local_registry_claims_by_pids(name, shard, pids) do local_node = node() @@ -1786,17 +1816,39 @@ defmodule Group.Replica.Data do end def pg_entries_for_origin(name, shard, cluster, origin_node) do - :ets.select(pg_by_key_table(name, shard), [ + fold_pg_entries_for_origin(name, shard, cluster, origin_node, [], fn row, rows -> + [row | rows] + end) + |> Enum.reverse() + end + + def fold_pg_entries_for_origin(name, shard, cluster, origin_node, acc, fun) + when is_function(fun, 2) do + {table, match_spec} = pg_origin_selection(name, shard, cluster, origin_node) + + fold_select_batches_fixed(table, match_spec, acc, fn rows, inner -> + Enum.reduce(rows, inner, fun) + end) + end + + def reduce_pg_entry_batches_for_origin(name, shard, cluster, origin_node, acc, fun) + when is_function(fun, 2) do + {table, match_spec} = pg_origin_selection(name, shard, cluster, origin_node) + fold_select_batches(table, match_spec, acc, fun) + end + + defp pg_origin_selection(name, shard, cluster, origin_node) do + { + pg_by_key_table(name, shard), {{{cluster, :"$1", :"$2"}, :"$3", :"$4", origin_node}, [], [{{:"$1", :"$2", :"$3", :"$4"}}]} - ]) + } end def delete_pg_for_origin_cluster(name, shard, cluster, origin_node) do entries = pg_entries_for_origin(name, shard, cluster, origin_node) Enum.each(entries, fn {key, pid, _meta, _time} -> - :ets.delete(pg_by_key_table(name, shard), {cluster, key, pid}) - :ets.delete(pg_by_pid_table(name, shard), {pid, cluster, key}) + pg_delete(name, shard, cluster, key, pid) end) Enum.map(entries, fn {key, pid, meta, time} -> {cluster, key, pid, meta, time} end) @@ -1817,8 +1869,7 @@ defmodule Group.Replica.Data do end) Enum.each(entries, fn {cluster, key, pid, _meta, _time} -> - :ets.delete(pg_by_key_table(name, shard), {cluster, key, pid}) - :ets.delete(pg_by_pid_table(name, shard), {pid, cluster, key}) + pg_delete(name, shard, cluster, key, pid) end) entries @@ -1844,8 +1895,7 @@ defmodule Group.Replica.Data do ]) Enum.each(entries, fn {key, pid, _meta, _time} -> - :ets.delete(pg_by_key_table(name, shard), {cluster, key, pid}) - :ets.delete(pg_by_pid_table(name, shard), {pid, cluster, key}) + pg_delete(name, shard, cluster, key, pid) end) Enum.map(entries, fn {key, pid, meta, time} -> {cluster, key, pid, meta, time} end) @@ -2848,6 +2898,33 @@ defmodule Group.Replica.Data do :ok end + defp fold_select_batches(table, match_spec, acc, fun) do + case :ets.select(table, [match_spec], 4_096) do + :"$end_of_table" -> acc + {matches, continuation} -> fold_select_batches(continuation, fun.(matches, acc), fun) + end + end + + defp fold_select_batches_fixed(table, match_spec, acc, fun) do + # Some consumers delete the selected origin slice while folding it. Keep + # the traversal fixed so deleting the current batch cannot move unseen + # ordered-set keys past the continuation and leave a permanent stale row. + :ets.safe_fixtable(table, true) + + try do + fold_select_batches(table, match_spec, acc, fun) + after + :ets.safe_fixtable(table, false) + end + end + + defp fold_select_batches(continuation, acc, fun) do + case :ets.select(continuation) do + :"$end_of_table" -> acc + {matches, next} -> fold_select_batches(next, fun.(matches, acc), fun) + end + end + defp select(table, match_spec, :infinity), do: :ets.select(table, match_spec) defp select(_table, _match_spec, 0), do: [] diff --git a/lib/group/replica/snapshot.ex b/lib/group/replica/snapshot.ex index 8287407..68f2aa6 100644 --- a/lib/group/replica/snapshot.ex +++ b/lib/group/replica/snapshot.ex @@ -6,7 +6,7 @@ defmodule Group.Replica.Snapshot do # reserve covers the frame tuple, stream identity, both list headers, and # integer fields. A single entry larger than the target remains one chunk. @default_envelope_reserve 512 - @max_compact_chunk_count 2_147_483_647 + @max_frame_counter 18_446_744_073_709_551_615 def chunk_rows(registry_rows, pg_rows, target_bytes) when is_list(registry_rows) and is_list(pg_rows) and is_integer(target_bytes) and @@ -48,7 +48,18 @@ defmodule Group.Replica.Snapshot do # ensure every practical index/count uses no more space than this envelope. :erlang.external_size( {:snapshot_chunk, Group.Replica.WireProtocol.version(), stream_id, snapshot_seq, - @max_compact_chunk_count, @max_compact_chunk_count, registry_count, pg_count, [], []} + @max_frame_counter, @max_frame_counter, registry_count, pg_count, [], []} + ) + 10 + end + + def capture_envelope_bytes(stream_id, snapshot_seq) do + # Capture has to choose chunk boundaries before it knows the final row and + # chunk counts. Reserving their largest practical (unsigned 64-bit) ETF + # representation makes every emitted frame no larger than the configured + # target without requiring a second pass over the captured rows. + :erlang.external_size( + {:snapshot_chunk, Group.Replica.WireProtocol.version(), stream_id, snapshot_seq, + @max_frame_counter, @max_frame_counter, @max_frame_counter, @max_frame_counter, [], []} ) + 10 end @@ -56,6 +67,14 @@ defmodule Group.Replica.Snapshot do :ets.new(__MODULE__, [:set, :private]) end + def new_capture_table do + :ets.new(__MODULE__, [:set, :private]) + end + + def new_event_table do + :ets.new(__MODULE__, [:ordered_set, :private]) + end + def delete_staging_table(table) do try do :ets.delete(table) @@ -75,8 +94,11 @@ defmodule Group.Replica.Snapshot do if :ets.insert_new(table, objects) and :ets.info(table, :size) - size_before == length(objects) do - true = :ets.insert_new(table, {{:chunk, chunk_index}, registry_rows, pg_rows}) - :ok + if :ets.insert_new(table, {{:chunk, chunk_index}, registry_rows, pg_rows}) do + :ok + else + {:error, :duplicate_row} + end else {:error, :duplicate_row} end @@ -96,7 +118,86 @@ defmodule Group.Replica.Snapshot do end) end + def new_event_buffer(table) do + %{table: table, chunk_index: 0, events: [], count: 0} + end + + def buffer_events(events, buffer) when is_list(events) do + Enum.reduce(events, buffer, &buffer_event/2) + end + + def buffer_event(event, buffer) do + buffer = + if buffer.count == 512 do + flush_event_buffer(buffer) + else + buffer + end + + %{buffer | events: [event | buffer.events], count: buffer.count + 1} + end + + def finish_event_buffer(%{count: 0} = buffer), do: buffer + def finish_event_buffer(buffer), do: flush_event_buffer(buffer) + + def fold_events(table, acc, fun) when is_function(fun, 2) do + :ets.foldl( + fn {{:events, _chunk_index}, events}, inner -> + Enum.reduce(events, inner, fun) + end, + acc, + table + ) + end + def member_pg?(table, key, pid), do: :ets.member(table, {:pg, key, pid}) + def member_registry?(table, key), do: :ets.member(table, {:registry, key}) + + def new_capture(table, target_bytes, envelope_bytes) + when is_integer(target_bytes) and target_bytes > 0 and is_integer(envelope_bytes) and + envelope_bytes >= 0 do + %{ + table: table, + payload_target: max(target_bytes - envelope_bytes, 1), + registry: [], + pg: [], + bytes: 0, + current_count: 0, + chunk_count: 0, + registry_count: 0, + pg_count: 0 + } + end + + def capture_registry_many(rows, capture) when is_list(rows) do + Enum.reduce(rows, capture, &capture_row(&2, :registry, &1)) + end + + def capture_pg_many(rows, capture) when is_list(rows) do + Enum.reduce(rows, capture, &capture_row(&2, :pg, &1)) + end + + def finish_capture(%{current_count: 0, chunk_count: 0} = capture), + do: flush_capture_chunk(capture) + + def finish_capture(%{current_count: 0} = capture), do: capture + def finish_capture(capture), do: flush_capture_chunk(capture) + + def reduce_capture_chunks(table, chunk_count, acc, fun) + when is_integer(chunk_count) and chunk_count > 0 and is_function(fun, 3) do + Enum.reduce_while(1..chunk_count, acc, fn chunk_index, inner -> + [{{:chunk, ^chunk_index}, registry, pg}] = :ets.lookup(table, {:chunk, chunk_index}) + + case fun.(registry, pg, inner) do + {:cont, next} -> {:cont, next} + {:halt, result} -> {:halt, {:halt, result}} + end + end) + |> case do + {:halt, result} -> {:halt, result} + result -> {:ok, result} + end + end defp add_row(%{count: count, bytes: bytes} = acc, domain, row, target) do row_bytes = :erlang.external_size(row) @@ -141,4 +242,61 @@ defmodule Group.Replica.Snapshot do [{{:chunk, ^chunk_index}, registry_rows, pg_rows}] -> {registry_rows, pg_rows} end end + + defp capture_row(capture, domain, row) do + row_bytes = :erlang.external_size(row) + + capture = + if capture.current_count > 0 and + capture.bytes + row_bytes > capture.payload_target do + flush_capture_chunk(capture) + else + capture + end + + case domain do + :registry -> + %{ + capture + | registry: [row | capture.registry], + bytes: capture.bytes + row_bytes, + current_count: capture.current_count + 1, + registry_count: capture.registry_count + 1 + } + + :pg -> + %{ + capture + | pg: [row | capture.pg], + bytes: capture.bytes + row_bytes, + current_count: capture.current_count + 1, + pg_count: capture.pg_count + 1 + } + end + end + + defp flush_capture_chunk(capture) do + chunk_index = capture.chunk_count + 1 + + true = + :ets.insert_new( + capture.table, + {{:chunk, chunk_index}, Enum.reverse(capture.registry), Enum.reverse(capture.pg)} + ) + + %{ + capture + | registry: [], + pg: [], + bytes: 0, + current_count: 0, + chunk_count: chunk_index + } + end + + defp flush_event_buffer(buffer) do + chunk_index = buffer.chunk_index + 1 + true = :ets.insert_new(buffer.table, {{:events, chunk_index}, Enum.reverse(buffer.events)}) + %{buffer | chunk_index: chunk_index, events: [], count: 0} + end end diff --git a/priv/bench/README.md b/priv/bench/README.md index 679b01e..07d5212 100644 --- a/priv/bench/README.md +++ b/priv/bench/README.md @@ -45,6 +45,22 @@ busy lane of a much larger sharded deployment): --coordinator-expr 'GroupBench.Distributed.run_snapshot_sync_only(shards: 1, entries: 50000)' ``` +To exercise one million rows across the recommended 32-shard scale point, +including exact catch-up, hottest-shard restart repair, memory, and permanent +peer eviction: + +```bash +./run_distributed.sh --shards 32 \ + --coordinator-expr 'GroupBench.Distributed.run_scale_recovery_only(shards: 32, entries: 1000000, mode: :registry)' + +./run_distributed.sh --shards 32 \ + --coordinator-expr 'GroupBench.Distributed.run_scale_recovery_only(shards: 32, entries: 1000000, mode: :pg_hotspot)' +``` + +`:registry` spreads a million exact claims across all shards. +`:pg_hotspot` deliberately puts a million memberships on one key/shard, which +is the worst case for snapshot installation and restart rebuilding. + ## Local Scenarios All local benchmarks run for both the default (nil) cluster and a named cluster diff --git a/priv/bench/lib/group_bench/distributed.ex b/priv/bench/lib/group_bench/distributed.ex index 9d3b34f..a99f4ba 100644 --- a/priv/bench/lib/group_bench/distributed.ex +++ b/priv/bench/lib/group_bench/distributed.ex @@ -84,6 +84,29 @@ defmodule GroupBench.Distributed do IO.puts("\n Done.\n") end + def run_scale_recovery_only(opts \\ []) do + shards = Keyword.get(opts, :shards, 32) + entries = Keyword.get(opts, :entries, 1_000_000) + mode = Keyword.get(opts, :mode, :registry) + Process.put(:bench_shards, shards) + + unless mode in [:registry, :pg_hotspot] do + raise ArgumentError, "expected :mode to be :registry or :pg_hotspot" + end + + header("Distributed Million-Row Recovery Benchmark") + IO.puts(" coordinator: #{node()}") + IO.puts(" shards: #{shards}") + IO.puts(" entries: #{format_number(entries)}") + IO.puts(" mode: #{mode}") + IO.puts(" schedulers: #{System.schedulers_online()}") + + connect_replicas() + bench_scale_recovery(@replicas, mode, entries) + + IO.puts("\n Done.\n") + end + # ── Connection ──────────────────────────────────────────────────────── defp connect_replicas do @@ -109,6 +132,148 @@ defmodule GroupBench.Distributed do end end + defp bench_scale_recovery([source, receiver] = replicas, mode, entries) do + header("Exact snapshot, shard restart, and permanent Group loss") + + group_opts = [ + replicated_oplog_max_entries: 64, + replicated_snapshot_chunk_target_bytes: 1_048_576, + replicated_anti_entropy_interval: 250, + replicated_peer_lease_timeout: 5_000 + ] + + stop_groups(replicas) + start_group_on(source, group_opts) + + {seed_us, seed_result} = + :timer.tc(fn -> + case mode do + :registry -> + :erpc.call( + source, + GroupBench.Replica, + :seed_registry_slice, + [@name, entries, "scale/registry/", 10_000], + 1_800_000 + ) + + :pg_hotspot -> + :erpc.call( + source, + GroupBench.Replica, + :seed_pg_hotspot, + [@name, entries, "scale/pg-hotspot", 10_000], + 1_800_000 + ) + end + end) + + source_memory = :erpc.call(source, GroupBench.Replica, :memory_snapshot, [@name]) + + {snapshot_us, _} = + :timer.tc(fn -> + start_group_on(receiver, group_opts) + + poll_until( + fn -> replicated_row_count(receiver, mode) == entries end, + 900_000 + ) + + :ok = + :erpc.call(receiver, GroupBench.Replica, :flush_shards, [@name], 900_000) + end) + + receiver_memory = :erpc.call(receiver, GroupBench.Replica, :memory_snapshot, [@name]) + + restart_shard = + case mode do + :registry -> + source + |> :erpc.call(GroupBench.Replica, :registry_counts_by_shard, [@name]) + |> Enum.max_by(&elem(&1, 1)) + |> elem(0) + + :pg_hotspot -> + seed_result + end + + restart = + :erpc.call( + receiver, + GroupBench.Replica, + :restart_shard, + [@name, restart_shard], + 900_000 + ) + + unless replicated_row_count(receiver, mode) == entries do + raise "row count changed across receiver shard restart" + end + + {eviction_us, _} = + :timer.tc(fn -> + stop_group_on(source) + + poll_until( + fn -> + counts = :erpc.call(receiver, GroupBench.Replica, :replica_row_counts, [@name]) + + counts.registry_rows == 0 and counts.registry_claim_rows == 0 and + counts.pg_rows == 0 and counts.replica_cursor_rows == 0 + end, + 900_000 + ) + end) + + after_eviction_memory = + :erpc.call(receiver, GroupBench.Replica, :memory_snapshot, [@name]) + + unless after_eviction_memory.registry_rows == 0 and after_eviction_memory.pg_rows == 0 and + after_eviction_memory.registry_claim_rows == 0 and + after_eviction_memory.replica_cursor_rows == 0 do + raise "retired source left visible rows, authoritative claims, or receive cursors: " <> + inspect( + Map.take(after_eviction_memory, [ + :registry_rows, + :registry_claim_rows, + :pg_rows, + :replica_cursor_rows + ]) + ) + end + + if mode == :registry and is_pid(seed_result) do + :erpc.call(source, Process, :exit, [seed_result, :kill]) + end + + result = %{ + mode: mode, + entries: entries, + shards: Process.get(:bench_shards), + seed_ms: div(seed_us, 1_000), + snapshot_ms: div(snapshot_us, 1_000), + snapshot_rows_per_second: round(entries * 1_000_000 / max(snapshot_us, 1)), + restart_shard: restart_shard, + restart_ms: div(restart.elapsed_us, 1_000), + eviction_ms: div(eviction_us, 1_000), + source_memory: source_memory, + receiver_memory: receiver_memory, + after_eviction_memory: after_eviction_memory + } + + IO.puts("\n PERF_RESULT #{inspect(result, pretty: true, limit: :infinity)}") + stop_groups(replicas) + result + end + + defp replicated_row_count(node, :registry) do + :erpc.call(node, GroupBench.Replica, :total_registry_count, [@name]) + end + + defp replicated_row_count(node, :pg_hotspot) do + :erpc.call(node, GroupBench.Replica, :total_pg_count, [@name]) + end + # ── Group lifecycle helpers (all MFA) ───────────────────────────────── defp start_group_on(node, opts \\ []) do diff --git a/priv/bench/lib/group_bench/replica.ex b/priv/bench/lib/group_bench/replica.ex index 0bace05..179d505 100644 --- a/priv/bench/lib/group_bench/replica.ex +++ b/priv/bench/lib/group_bench/replica.ex @@ -157,6 +157,188 @@ defmodule GroupBench.Replica do end) end + @doc false + def seed_registry_slice(name, count, key_prefix, batch_size \\ 10_000) do + owner = spawn(fn -> Process.sleep(:infinity) end) + shards = Group.get_config(name).num_shards + + 1..count + |> Stream.chunk_every(batch_size) + |> Enum.each(fn indexes -> + indexes + |> Enum.group_by(fn index -> + Group.Replica.shard_index_for(nil, "#{key_prefix}#{index}", shards) + end) + |> Task.async_stream( + fn {shard, shard_indexes} -> + stream_id = Group.Replica.Data.local_stream_id(name, shard, nil) + + entries = + Enum.map(shard_indexes, fn index -> + key = "#{key_prefix}#{index}" + + :ok = + Group.Replica.Data.put_registry_claim( + name, + shard, + stream_id, + 1, + key, + owner, + %{}, + index + ) + + {nil, key, owner, %{}, index, node()} + end) + + :ok = Group.Replica.Data.registry_insert_many(name, shard, entries) + end, + ordered: false, + timeout: :infinity + ) + |> Stream.run() + end) + + install_benchmark_stream_heads(name) + owner + end + + @doc false + def seed_pg_hotspot(name, count, key, batch_size \\ 10_000) do + shard = Group.Replica.shard_index_for(nil, key, Group.get_config(name).num_shards) + + 1..count + |> Stream.chunk_every(batch_size) + |> Enum.each(fn indexes -> + entries = + Enum.map(indexes, fn index -> + pid = :erlang.list_to_pid(String.to_charlist("<0.#{100_000_000 + index}.0>")) + {nil, key, pid, %{}, index, node()} + end) + + :ok = Group.Replica.Data.pg_insert_many(name, shard, entries) + end) + + install_benchmark_stream_heads(name) + shard + end + + @doc false + def restart_shard(name, shard) do + old_pid = Process.whereis(Group.Replica.shard_name(name, shard)) + started = System.monotonic_time(:microsecond) + Process.exit(old_pid, :kill) + new_pid = await_new_shard(name, shard, old_pid, 600_000) + elapsed = System.monotonic_time(:microsecond) - started + %{elapsed_us: elapsed, old_pid: old_pid, new_pid: new_pid} + end + + @doc false + def flush_shards(name) do + Enum.each(0..(Group.get_config(name).num_shards - 1), fn shard -> + :sys.get_state(Group.Replica.shard_name(name, shard), 600_000) + end) + + :ok + end + + @doc false + def memory_snapshot(name) do + shards = Group.get_config(name).num_shards + counts = replica_row_counts(name) + + table_bytes = + 0..(shards - 1) + |> Enum.flat_map(&replica_tables(name, &1)) + |> Enum.uniq() + |> Enum.reduce(0, fn table, total -> + case :ets.info(table, :memory) do + :undefined -> total + words -> total + words * :erlang.system_info(:wordsize) + end + end) + + %{ + total_bytes: :erlang.memory(:total), + process_bytes: :erlang.memory(:processes_used), + ets_bytes: :erlang.memory(:ets), + group_table_bytes: table_bytes, + registry_rows: counts.registry_rows, + registry_claim_rows: counts.registry_claim_rows, + pg_rows: counts.pg_rows, + replica_cursor_rows: counts.replica_cursor_rows + } + end + + @doc false + def replica_row_counts(name) do + %{ + registry_rows: total_registry_count(name), + registry_claim_rows: total_table_rows(name, &Group.Replica.Data.reg_claim_by_key_table/2), + pg_rows: total_pg_count(name), + replica_cursor_rows: total_table_rows(name, &Group.Replica.Data.replica_cursor_table/2) + } + end + + defp total_table_rows(name, table_fun) do + Enum.reduce(0..(Group.get_config(name).num_shards - 1), 0, fn shard, total -> + total + :ets.info(table_fun.(name, shard), :size) + end) + end + + defp install_benchmark_stream_heads(name) do + shards = Group.get_config(name).num_shards + + Enum.each(0..(shards - 1), fn shard -> + stream_id = Group.Replica.Data.local_stream_id(name, shard, nil) + table = Group.Replica.Data.replica_stream_meta_table(name, shard) + :ets.insert(table, {stream_id, 1, 2, 1}) + end) + + :ok + end + + defp await_new_shard(name, shard, old_pid, timeout) do + started = System.monotonic_time(:millisecond) + do_await_new_shard(name, shard, old_pid, timeout, started) + end + + defp do_await_new_shard(name, shard, old_pid, timeout, started) do + case Process.whereis(Group.Replica.shard_name(name, shard)) do + pid when is_pid(pid) and pid != old_pid -> + :sys.get_state(pid, timeout) + pid + + _ -> + if System.monotonic_time(:millisecond) - started >= timeout do + raise "timed out restarting shard #{shard}" + end + + Process.sleep(10) + do_await_new_shard(name, shard, old_pid, timeout, started) + end + end + + defp replica_tables(name, shard) do + data = Group.Replica.Data + + base = [ + data.reg_by_key_table(name, shard), + data.reg_by_pid_table(name, shard), + data.reg_claim_by_key_table(name, shard), + data.reg_claim_by_pid_table(name, shard), + data.pg_by_key_table(name, shard), + data.pg_by_pid_table(name, shard), + data.replica_stream_meta_table(name, shard), + data.replica_oplog_table(name, shard), + data.replica_oplog_order_table(name, shard), + data.replica_cursor_table(name, shard) + ] + + base + end + @doc """ Starts `worker_count` local processes that repeatedly re-join the same key with changing metadata, generating a sustained stream of replicated PG updates. diff --git a/priv/bench/mix.exs b/priv/bench/mix.exs index 71d6a86..fc48c0b 100644 --- a/priv/bench/mix.exs +++ b/priv/bench/mix.exs @@ -17,6 +17,6 @@ defmodule GroupBench.MixProject do end defp deps do - [{:group, path: "../../"}] + [{:group, path: System.get_env("GROUP_BENCH_GROUP_PATH", "../../")}] end end diff --git a/test/README.md b/test/README.md index d69c195..ad83318 100644 --- a/test/README.md +++ b/test/README.md @@ -28,7 +28,7 @@ release qualification rather than individual edits. | `anti_entropy_fault_regression_test.exs` | Three-node regressions for hidden-winner projection, receiver restart eviction, nodedown/lease lane retirement, authority gaps and cross-lane races, in-flight conflict fencing, crash-journal replay, cursorless/interrupted snapshot repair, malformed ingress, and sideband rediscovery | | `replica_adversarial_test.exs` | Reproducible three-node mixed-operation state machines: drops, busy returns, duplication, reordering, bounded delay, oplog pruning, conflicts, owner death, and named-cluster epoch churn, followed by exact convergence/dead-owner/internal-index checks | | `replica_model_property_test.exs` | StreamData-generated and shrunk owner histories against an independent lifecycle oracle and scheduler-controlled replica transport | -| `replica_snapshot_test.exs` | Pure byte partitioning and set-valued private-ETS snapshot staging | +| `replica_snapshot_test.exs` | Pure byte partitioning plus bounded private-ETS capture, receive staging, and event batching | | `replica_snapshot_distributed_test.exs` | Real-node exact-snapshot loss, reorder, duplicate, conflicting retransmission, supersession, authority fencing, expiry, and shard-crash recovery | ## Model-based and formal checks diff --git a/test/anti_entropy_fault_regression_test.exs b/test/anti_entropy_fault_regression_test.exs index 68c41eb..8b90600 100644 --- a/test/anti_entropy_fault_regression_test.exs +++ b/test/anti_entropy_fault_regression_test.exs @@ -620,7 +620,7 @@ defmodule Group.AntiEntropyFaultRegressionTest do assert 1 == TestCluster.rpc!(context.node_a, :erlang, :trace_pattern, [ - {Group.Replica.Data, :registry_claims_for_stream, 3}, + {Group.Replica.Data, :reduce_registry_claim_batches_for_stream, 5}, true, [:local] ]) @@ -636,7 +636,7 @@ defmodule Group.AntiEntropyFaultRegressionTest do TestCluster.rpc!(context.node_a, :erlang, :trace, [shard, false, [:all]]) TestCluster.rpc!(context.node_a, :erlang, :trace_pattern, [ - {Group.Replica.Data, :registry_claims_for_stream, 3}, + {Group.Replica.Data, :reduce_registry_claim_batches_for_stream, 5}, false, [:local] ]) @@ -652,7 +652,8 @@ defmodule Group.AntiEntropyFaultRegressionTest do assert_receive {:forwarded_trace, {:trace, capture_pid, :call, - {Group.Replica.Data, :registry_claims_for_stream, [^name, 0, ^stream_id]}}}, + {Group.Replica.Data, :reduce_registry_claim_batches_for_stream, + [^name, 0, ^stream_id, _capture, _fun]}}}, 1_000 refute capture_pid == shard @@ -2831,6 +2832,14 @@ defmodule Group.AntiEntropyFaultRegressionTest do end end) + new_lane = + TestCluster.rpc!(context.node_b, Process, :whereis, [Group.Replica.shard_name(name, 1)]) + + # A registered GenServer name only proves the replacement process exists; + # init still has to finish its retained-ETS repair before direct ETS reads + # can assert the post-restart state. + _state = TestCluster.rpc!(context.node_b, :sys, :get_state, [new_lane]) + assert TestCluster.rpc!(context.node_b, :ets, :lookup, [ Group.Replica.Data.replica_cursor_table(name, 1), stream_id diff --git a/test/distributed_test.exs b/test/distributed_test.exs index 3072d0a..0d4e2c0 100644 --- a/test/distributed_test.exs +++ b/test/distributed_test.exs @@ -4282,8 +4282,8 @@ defmodule Group.DistributedTest do opts = [ name: name, shards: shards, - replicated_anti_entropy_interval: 60_000, - replicated_peer_lease_timeout: 120_000 + replicated_anti_entropy_interval: 600_000, + replicated_peer_lease_timeout: 1_200_000 ] start_group_on_peers(peers, opts) diff --git a/test/group_test.exs b/test/group_test.exs index c0679a1..7751fba 100644 --- a/test/group_test.exs +++ b/test/group_test.exs @@ -2531,6 +2531,46 @@ defmodule GroupTest do assert :ok = Group.TestCluster.assert_replica_consistent(name) end + test "streamed exact replacement deletes every claim across select continuations" do + name = start_single_shard_group(replicated_oplog_max_entries: 16) + stream_id = Group.Replica.Data.local_stream_id(name, 0, nil) + staging = Group.Replica.Snapshot.new_staging_table() + + on_exit(fn -> Group.Replica.Snapshot.delete_staging_table(staging) end) + + Enum.each(1..10_000, fn index -> + :ok = + Group.Replica.Data.put_registry_claim( + name, + 0, + stream_id, + 1, + "snapshot/select-continuation/#{index}", + self(), + %{}, + index + ) + end) + + assert :ok = Group.Replica.Snapshot.stage_rows(staging, 1, [], []) + + assert :ok = + Group.Replica.Data.replace_registry_claims_for_stream_from_staging( + name, + 0, + stream_id, + 1, + staging, + 1, + :ok, + fn _key, :ok -> :ok end + ) + + assert Group.Replica.Data.registry_claims_for_stream(name, 0, stream_id) == [] + assert :ets.info(Group.Replica.Data.reg_claim_by_key_table(name, 0), :size) == 0 + assert :ets.info(Group.Replica.Data.reg_claim_by_pid_table(name, 0), :size) == 0 + end + test "a shard restart completes an interrupted named-cluster close without retained rows" do name = start_single_shard_group(replicated_oplog_max_entries: 16) cluster = "close/crash-window/#{System.unique_integer([:positive])}" diff --git a/test/mutation/README.md b/test/mutation/README.md index f616b75..e1a1944 100644 --- a/test/mutation/README.md +++ b/test/mutation/README.md @@ -10,7 +10,7 @@ also covers incomplete commit, conflicting retransmission rows, newer-snapshot supersession, stale-authority fencing, and staging expiry. Restart calibration covers per-lane eviction breadcrumbs and partially observed authority, while wire calibration rejects wrong-shard rows and unsequenced -cluster lifecycle messages. The 65-mutant campaign also independently removes +cluster lifecycle messages. The 64-mutant campaign also independently removes the generation and epoch fences, races authority changes against local-owner retirement, skips conflict reprojection after exact authority returns, bypasses shard-zero authority serialization, separates exact authority from its shared diff --git a/test/mutation/run.exs b/test/mutation/run.exs index f514104..9341093 100644 --- a/test/mutation/run.exs +++ b/test/mutation/run.exs @@ -60,32 +60,28 @@ defmodule Group.MutationCampaign do %{ name: "registry_snapshot_is_additive", file: "lib/group/replica/data.ex", - correct_source: "Enum.reduce(existing, MapSet.new(), fn {key, pid, _meta, _time}, keys ->", - faulty_source: - "Enum.reduce(Enum.take(existing, 0), MapSet.new(), fn {key, pid, _meta, _time}, keys ->", - test: ["test/replica_snapshot_distributed_test.exs:16"] + correct_source: """ + acc = + fold_registry_claims_for_stream(name, shard, stream_id, acc, fn + """, + faulty_source: """ + acc = + Enum.reduce([], acc, fn + """, + test: [ + "test/replica_snapshot_distributed_test.exs:16", + "test/distributed_test.exs:4057" + ] }, %{ name: "pg_snapshot_is_additive", file: "lib/group/replica.ex", - correct_source: "Enum.reduce(current, events, fn {key, pid, old_meta, _old_time}, acc ->", - faulty_source: - "Enum.reduce(Enum.take(current, 0), events, fn {key, pid, old_meta, _old_time}, acc ->", - test: ["test/replica_snapshot_distributed_test.exs:16"] - }, - %{ - name: "single_chunk_registry_snapshot_is_additive", - file: "lib/group/replica/data.ex", - correct_source: "Enum.each(existing, fn {key, pid, _meta, _time} ->", - faulty_source: "Enum.each(Enum.take(existing, 0), fn {key, pid, _meta, _time} ->", - test: ["test/distributed_test.exs:4057"] - }, - %{ - name: "single_chunk_pg_snapshot_is_additive", - file: "lib/group/replica.ex", - correct_source: " current\n |> Map.keys()\n", - faulty_source: " %{}\n |> Map.keys()\n", - test: ["test/distributed_test.exs:4057"] + correct_source: " if Snapshot.member_pg?(staging_table, key, pid) do", + faulty_source: " if Process.alive?(self()) do", + test: [ + "test/replica_snapshot_distributed_test.exs:16", + "test/distributed_test.exs:4057" + ] }, %{ name: "commit_incomplete_snapshot", @@ -111,6 +107,13 @@ defmodule Group.MutationCampaign do """, test: ["test/replica_snapshot_distributed_test.exs:16"] }, + %{ + name: "drop_final_snapshot_event_batch", + file: "lib/group/replica.ex", + correct_source: " _event_buffer = Snapshot.finish_event_buffer(event_buffer)", + faulty_source: " _event_buffer = event_buffer", + test: ["test/replica_snapshot_distributed_test.exs:16"] + }, %{ name: "allow_duplicate_snapshot_rows", file: "lib/group/replica/snapshot.ex", @@ -121,7 +124,7 @@ defmodule Group.MutationCampaign do faulty_source: """ if :ets.insert(table, objects) and size_before >= 0 do """, - test: ["test/replica_snapshot_distributed_test.exs:164"] + test: ["test/replica_snapshot_distributed_test.exs:178"] }, %{ name: "do_not_supersede_partial_snapshot", @@ -137,7 +140,7 @@ defmodule Group.MutationCampaign do _ = existing_seq {:ignore, state} """, - test: ["test/replica_snapshot_distributed_test.exs:107"] + test: ["test/replica_snapshot_distributed_test.exs:121"] }, %{ name: "accept_stale_snapshot_authority", @@ -153,7 +156,7 @@ defmodule Group.MutationCampaign do snapshot_seq > Data.replica_cursor(state.name, state.shard_index, stream_id) end """, - test: ["test/replica_snapshot_distributed_test.exs:314"] + test: ["test/replica_snapshot_distributed_test.exs:328"] }, %{ name: "disable_snapshot_staging_expiry", @@ -174,7 +177,7 @@ defmodule Group.MutationCampaign do acc end """, - test: ["test/replica_snapshot_distributed_test.exs:240"] + test: ["test/replica_snapshot_distributed_test.exs:254"] }, %{ name: "disable_below_floor_snapshot", @@ -217,7 +220,7 @@ defmodule Group.MutationCampaign do file: "lib/group/replica/data.ex", correct_source: " hint_generation == generation and\n", faulty_source: " false and hint_generation == generation and\n", - test: ["test/anti_entropy_fault_regression_test.exs:2057"] + test: ["test/anti_entropy_fault_regression_test.exs:2058"] }, %{ name: "heartbeat_does_not_fence_newer_generation", @@ -229,7 +232,7 @@ defmodule Group.MutationCampaign do " not is_nil(hint_generation) and\n" <> " WireProtocol.generation_newer?(generation, hint_generation) and\n" <> " Process.get(:fence_newer_generation, false) ->\n", - test: ["test/anti_entropy_fault_regression_test.exs:2220"] + test: ["test/anti_entropy_fault_regression_test.exs:2221"] }, %{ name: "drop_new_generation_authority_hint", @@ -240,7 +243,7 @@ defmodule Group.MutationCampaign do faulty_source: " # below are being updated.\n" <> " _ = {state.name, remote_node, generation, revision}", - test: ["test/anti_entropy_fault_regression_test.exs:2220"] + test: ["test/anti_entropy_fault_regression_test.exs:2221"] }, %{ name: "accept_authority_older_than_generation_hint", @@ -249,7 +252,7 @@ defmodule Group.MutationCampaign do faulty_source: " _ = hinted_stale?\n" <> " known_stale? or revision_stale?", - test: ["test/anti_entropy_fault_regression_test.exs:2220"] + test: ["test/anti_entropy_fault_regression_test.exs:2221"] }, %{ name: "install_lane_view_behind_generation_hint", @@ -258,7 +261,7 @@ defmodule Group.MutationCampaign do " remote_replica_authority_hint(state.name, remote_node) == {generation, observed} do", faulty_source: " elem(remote_replica_authority_hint(state.name, remote_node), 1) == observed do", - test: ["test/group_test.exs:2857"] + test: ["test/group_test.exs:2897"] }, %{ name: "install_incremental_after_newer_hint", @@ -269,7 +272,7 @@ defmodule Group.MutationCampaign do faulty_source: " Process.get(:ignore_incremental_authority_race, true) and\n" <> " is_tuple(remote_replica_authority_hint(name, remote_node))\n", - test: ["test/group_test.exs:2791"] + test: ["test/group_test.exs:2831"] }, %{ name: "accept_hint_without_exact_authority", @@ -280,7 +283,7 @@ defmodule Group.MutationCampaign do faulty_source: " (is_nil(hint_generation) or\n" <> " WireProtocol.generation_newer?(generation, hint_generation)) ->\n", - test: ["test/anti_entropy_fault_regression_test.exs:3308"] + test: ["test/anti_entropy_fault_regression_test.exs:3317"] }, %{ name: "admit_retired_lane_route_without_authority", @@ -293,7 +296,7 @@ defmodule Group.MutationCampaign do faulty_source: " state = put_remote_shard(state, remote_node, remote_pid)\n" <> " {:noreply, request_replica_authority(state, remote_node)}", - test: ["test/anti_entropy_fault_regression_test.exs:3308"] + test: ["test/anti_entropy_fault_regression_test.exs:3317"] }, %{ name: "do_not_restore_hint_lease_after_lane_restart", @@ -305,7 +308,7 @@ defmodule Group.MutationCampaign do " # crash in that window cannot strand the peer forever.\n" <> " {{{:remote_authority_hint, :\"$1\"}, :_, :_}, [], [:\"$1\"]}\n", faulty_source: " {{{:remote_view_info, shard, :\"$1\"}, :_, :_, :_}, [], [:\"$1\"]}\n", - test: ["test/group_test.exs:2730"] + test: ["test/group_test.exs:2770"] }, %{ name: "retain_retired_authority_repair", @@ -318,7 +321,7 @@ defmodule Group.MutationCampaign do " is_nil(Data.remote_generation(state.name, remote_node)) and\n" <> " is_nil(Data.remote_replica_authority_hint(state.name, remote_node)) ->\n" <> " Map.put(acc, remote_node, last_activity)\n", - test: ["test/anti_entropy_fault_regression_test.exs:3308"] + test: ["test/anti_entropy_fault_regression_test.exs:3317"] }, %{ name: "skip_authority_fanout", @@ -360,7 +363,7 @@ defmodule Group.MutationCampaign do state end """, - test: ["test/anti_entropy_fault_regression_test.exs:3813"] + test: ["test/anti_entropy_fault_regression_test.exs:3822"] }, %{ name: "assume_authority_fanout_reaches_late_lane", @@ -390,7 +393,7 @@ defmodule Group.MutationCampaign do state end """, - test: ["test/replica_snapshot_distributed_test.exs:539"] + test: ["test/replica_snapshot_distributed_test.exs:553"] }, %{ name: "skip_generation_purge", @@ -429,13 +432,12 @@ defmodule Group.MutationCampaign do file: "lib/group/replica.ex", correct_source: """ defp broadcast_replica_heads(state) do - Enum.reduce(state.peer_last_seen, state, fn {target_node, _last_seen}, acc -> - send_replica_heads(acc, target_node) - end) - end + peers = Map.keys(state.peer_last_seen) """, faulty_source: """ - defp broadcast_replica_heads(state), do: state + defp broadcast_replica_heads(state) do + _ = state.peer_last_seen + peers = [] """, test: ["test/distributed_test.exs:3967"] }, @@ -461,7 +463,7 @@ defmodule Group.MutationCampaign do " if Process.get(:run_primary_replica_repair, false),\n" <> " do: repair_primary_replica_rows(name, shard),\n" <> " else: :ok", - test: ["test/group_test.exs:2534"] + test: ["test/group_test.exs:2574"] }, %{ name: "skip_closed_cluster_completion", @@ -477,7 +479,7 @@ defmodule Group.MutationCampaign do faulty_source: """ _completed_clusters = [] """, - test: ["test/group_test.exs:2534"] + test: ["test/group_test.exs:2574"] }, %{ name: "accept_unfenced_cluster_disconnect", @@ -514,7 +516,7 @@ defmodule Group.MutationCampaign do file: "lib/group/replica/data.ex", correct_source: " [{^cluster, ^request_epoch, pending_shards}] ->\n", faulty_source: " [{^cluster, _stored_epoch, pending_shards}] ->\n", - test: ["test/group_test.exs:2575"] + test: ["test/group_test.exs:2615"] }, %{ name: "accept_shared_authority_before_lane_install", @@ -525,7 +527,7 @@ defmodule Group.MutationCampaign do faulty_source: " WireProtocol.stream_shard(stream_id) == state.shard_index and\n" <> " true and", - test: ["test/anti_entropy_fault_regression_test.exs:1171"] + test: ["test/anti_entropy_fault_regression_test.exs:1172"] }, %{ name: "apply_incremental_authority_across_revision_gap", @@ -533,21 +535,21 @@ defmodule Group.MutationCampaign do correct_source: " if contiguous_cluster_controls?(accepted, next_revision) do", faulty_source: " if contiguous_cluster_controls?(accepted, next_revision) or accepted != [] do", - test: ["test/anti_entropy_fault_regression_test.exs:1363"] + test: ["test/anti_entropy_fault_regression_test.exs:1364"] }, %{ name: "allow_non_owner_lane_to_mutate_shared_authority", file: "lib/group/replica.ex", correct_source: " if state.shard_index == 0 do\n remote_node = node(remote_pid)", faulty_source: " if true do\n remote_node = node(remote_pid)", - test: ["test/anti_entropy_fault_regression_test.exs:1363"] + test: ["test/anti_entropy_fault_regression_test.exs:1364"] }, %{ name: "crash_lane_when_local_authority_owner_is_missing", file: "lib/group/replica.ex", correct_source: " _ = send_local_control_message(state, control)", faulty_source: " send(shard_name(state.name, 0), control)", - test: ["test/anti_entropy_fault_regression_test.exs:1310"] + test: ["test/anti_entropy_fault_regression_test.exs:1311"] }, %{ name: "retire_local_owner_after_remote_authority_changed", @@ -556,7 +558,7 @@ defmodule Group.MutationCampaign do faulty_source: " Process.get(:skip_remote_registry_authority, true) or\n" <> " registry_winner_authoritative?(state, cluster, winner) ->", - test: ["test/anti_entropy_fault_regression_test.exs:1522"] + test: ["test/anti_entropy_fault_regression_test.exs:1523"] }, %{ name: "skip_registry_reprojection_after_authority_restore", @@ -581,7 +583,7 @@ defmodule Group.MutationCampaign do state end """, - test: ["test/anti_entropy_fault_regression_test.exs:1705"] + test: ["test/anti_entropy_fault_regression_test.exs:1706"] }, %{ name: "retain_registry_reprojection_after_peer_expiry", @@ -597,7 +599,7 @@ defmodule Group.MutationCampaign do state = discard_snapshot_transfers_for_source(state, remote_node) state = discard_snapshot_send_offsets_for_target(state, remote_node) """, - test: ["test/anti_entropy_fault_regression_test.exs:1881"] + test: ["test/anti_entropy_fault_regression_test.exs:1882"] }, %{ name: "retain_registry_reprojection_after_nodedown", @@ -615,7 +617,7 @@ defmodule Group.MutationCampaign do state = discard_snapshot_transfers_for_source(state, dead_node) state = discard_snapshot_send_offsets_for_target(state, dead_node) """, - test: ["test/anti_entropy_fault_regression_test.exs:3591"] + test: ["test/anti_entropy_fault_regression_test.exs:3600"] }, %{ name: "separate_exact_authority_from_cluster_projection", @@ -624,7 +626,7 @@ defmodule Group.MutationCampaign do " replace_remote_cluster_projection(state.name, remote_node, current_epochs)\n", faulty_source: " _ = {&replace_remote_cluster_projection/3, state.name, remote_node, current_epochs}\n", - test: ["test/anti_entropy_fault_regression_test.exs:3627"] + test: ["test/anti_entropy_fault_regression_test.exs:3636"] }, %{ name: "separate_local_activation_from_cluster_projection", @@ -633,7 +635,7 @@ defmodule Group.MutationCampaign do " if durable?, do: project_activated_local_clusters(state.name, clusters)\n", faulty_source: " _ = {durable?, &project_activated_local_clusters/2, state.name, clusters}\n", - test: ["test/anti_entropy_fault_regression_test.exs:3682"] + test: ["test/anti_entropy_fault_regression_test.exs:3691"] }, %{ name: "drop_durable_cluster_deactivation_cleanup", @@ -646,7 +648,7 @@ defmodule Group.MutationCampaign do " )\n", faulty_source: " _ = {&cast_cluster_lifecycle/3, state.name, state.num_shards, clusters, epochs}\n", - test: ["test/anti_entropy_fault_regression_test.exs:3731"] + test: ["test/anti_entropy_fault_regression_test.exs:3740"] }, %{ name: "delete_close_marker_before_terminal_route_cleanup", @@ -656,7 +658,7 @@ defmodule Group.MutationCampaign do " :ets.delete(closed_local_cluster_epochs_table(state.name), cluster)\n", faulty_source: " :ets.delete(closed_local_cluster_epochs_table(state.name), cluster)\n", - test: ["test/group_test.exs:2575"] + test: ["test/group_test.exs:2615"] }, %{ name: "retire_peer_authority_before_terminal_route_cleanup", @@ -666,7 +668,7 @@ defmodule Group.MutationCampaign do " :ok = delete_peer_routes(name, remote_node)\n", faulty_source: " :ets.delete(replication_meta_table(name), {:remote_generation, remote_node})\n", - test: ["test/group_test.exs:2629"] + test: ["test/group_test.exs:2669"] }, %{ name: "stale_peer_cleanup_removes_rediscovered_routes", @@ -678,7 +680,7 @@ defmodule Group.MutationCampaign do " if Process.get(:purge_rediscovered_peer_routes, true) or\n" <> " (is_nil(remote_generation(state.name, dead_node)) and\n" <> " is_nil(remote_replica_authority_hint(state.name, dead_node))) do\n", - test: ["test/group_test.exs:2682"] + test: ["test/group_test.exs:2722"] }, %{ name: "stale_restart_cleanup_removes_reactivated_routes", @@ -686,7 +688,7 @@ defmodule Group.MutationCampaign do correct_source: " Enum.filter(clusters, &is_nil(local_cluster_epoch(state.name, &1)))\n", faulty_source: " clusters\n", - test: ["test/group_test.exs:2609"] + test: ["test/group_test.exs:2649"] }, %{ name: "retain_authority_repair_after_nodedown", @@ -697,7 +699,7 @@ defmodule Group.MutationCampaign do faulty_source: " cluster_control_dirty: state.cluster_control_dirty,\n" <> " authority_dirty_notified: MapSet.delete(state.authority_dirty_notified, dead_node)\n", - test: ["test/group_test.exs:2668"] + test: ["test/group_test.exs:2708"] }, %{ name: "retain_receive_cursor_for_inactive_local_cluster", @@ -708,14 +710,14 @@ defmodule Group.MutationCampaign do faulty_source: " WireProtocol.stream_origin(stream_id) != node() and\n" <> " true and", - test: ["test/anti_entropy_fault_regression_test.exs:2734"] + test: ["test/anti_entropy_fault_regression_test.exs:2735"] }, %{ name: "retire_shared_authority_with_live_lanes", file: "lib/group/replica/data.ex", correct_source: " result =\n if remaining_lanes == 0 do", faulty_source: " _ = remaining_lanes\n\n result =\n if true do", - test: ["test/anti_entropy_fault_regression_test.exs:855"] + test: ["test/anti_entropy_fault_regression_test.exs:856"] }, %{ name: "shard_zero_deletes_sibling_restart_views", @@ -825,7 +827,7 @@ defmodule Group.MutationCampaign do " if Process.get(:run_primary_replica_repair, false),\n" <> " do: repair_primary_replica_rows(name, shard),\n" <> " else: :ok", - test: ["test/anti_entropy_fault_regression_test.exs:3016"] + test: ["test/anti_entropy_fault_regression_test.exs:3025"] }, %{ name: "project_stale_claims_before_restart_repair", @@ -840,7 +842,7 @@ defmodule Group.MutationCampaign do {state, _events} = rebuild_registry_projections(state) :ok = Data.repair_shard_indexes(name, shard_index) """, - test: ["test/anti_entropy_fault_regression_test.exs:3448"] + test: ["test/anti_entropy_fault_regression_test.exs:3457"] }, %{ name: "skip_interrupted_snapshot_install_repair", @@ -850,7 +852,7 @@ defmodule Group.MutationCampaign do " if Process.get(:run_snapshot_install_repair, false),\n" <> " do: repair_interrupted_snapshot_installs(name, shard),\n" <> " else: :ok", - test: ["test/anti_entropy_fault_regression_test.exs:3128"] + test: ["test/anti_entropy_fault_regression_test.exs:3137"] }, %{ name: "retain_cursorless_remote_registry_claims", @@ -859,14 +861,14 @@ defmodule Group.MutationCampaign do " :ets.member(replica_cursor_table(name, shard), stream_id)\n else\n false\n end\n end\n\n defp valid_remote_pg_authority?", faulty_source: " is_tuple(stream_id)\n else\n false\n end\n end\n\n defp valid_remote_pg_authority?", - test: ["test/anti_entropy_fault_regression_test.exs:3016"] + test: ["test/anti_entropy_fault_regression_test.exs:3025"] }, %{ name: "restart_snapshot_from_first_chunk_after_busy", file: "lib/group/replica.ex", correct_source: " start_index = Map.get(offsets, snapshot_key, 1)", faulty_source: " _ = {offsets, snapshot_key}\n start_index = 1", - test: ["test/replica_snapshot_distributed_test.exs:635"] + test: ["test/replica_snapshot_distributed_test.exs:649"] }, %{ name: "drain_oversized_ingress_batch_without_yield", diff --git a/test/replica_snapshot_distributed_test.exs b/test/replica_snapshot_distributed_test.exs index b04b3f2..3b32c5d 100644 --- a/test/replica_snapshot_distributed_test.exs +++ b/test/replica_snapshot_distributed_test.exs @@ -29,6 +29,9 @@ defmodule Group.ReplicaSnapshotDistributedTest do ) end) + forwarder = TestCluster.spawn_monitor_forwarder(node_b, name, :all, self()) + assert_receive {:monitor_ready, ^forwarder}, 5_000 + stream_id = local_stream(node_a, name, nil) old_cursor = replica_cursor(node_b, name, stream_id) :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) @@ -101,6 +104,17 @@ defmodule Group.ReplicaSnapshotDistributedTest do ) == MapSet.new(fresh_pg) + {fresh_reg_key, fresh_reg_pid} = hd(fresh_reg) + fresh_pg_pid = hd(fresh_pg) + + assert_receive {:got_event, + %Group.Event{type: :registered, key: ^fresh_reg_key, pid: ^fresh_reg_pid}}, + 5_000 + + assert_receive {:got_event, + %Group.Event{type: :joined, key: ^fresh_pg_key, pid: ^fresh_pg_pid}}, + 5_000 + assert snapshot_transfer_count(node_b, name) == 0 end diff --git a/test/replica_snapshot_test.exs b/test/replica_snapshot_test.exs index 2728870..52af1a6 100644 --- a/test/replica_snapshot_test.exs +++ b/test/replica_snapshot_test.exs @@ -63,6 +63,7 @@ defmodule Group.ReplicaSnapshotTest do pg = {"pg", self(), %{v: 2}, 2} assert :ok = Snapshot.stage_rows(table, 1, [registry], [pg]) + assert :ets.info(table, :size) == 3 assert {:error, :duplicate_row} = Snapshot.stage_rows(table, 2, [registry], []) assert Snapshot.fold_registry(table, 1, [], fn row, acc -> [row | acc] end) == [registry] @@ -72,4 +73,78 @@ defmodule Group.ReplicaSnapshotTest do assert :ok = Snapshot.delete_staging_table(table) assert :ets.info(table) == :undefined end + + test "capture tables emit deterministic chunks with only one bounded chunk on the heap" do + table = Snapshot.new_capture_table() + pid = self() + metadata = %{payload: String.duplicate("x", 96)} + + registry_rows = + for index <- 80..1//-1 do + {"registry/#{index}", pid, metadata, index} + end + + pg_rows = + for index <- 80..1//-1 do + {"pg/#{index}", pid, metadata, index} + end + + target = 2_048 + stream_id = {:group, node(), make_ref(), 0, nil, make_ref()} + envelope = Snapshot.capture_envelope_bytes(stream_id, 123) + + capture = Snapshot.new_capture(table, target, envelope) + capture = Snapshot.capture_registry_many(registry_rows, capture) + capture = Snapshot.capture_pg_many(pg_rows, capture) + capture = Snapshot.finish_capture(capture) + chunk_count = capture.chunk_count + + assert chunk_count > 1 + assert :ets.info(table, :size) == chunk_count + assert capture.registry == [] + assert capture.pg == [] + + assert {:ok, chunks} = + Snapshot.reduce_capture_chunks(table, chunk_count, [], fn registry, pg, acc -> + {:cont, [{registry, pg} | acc]} + end) + + chunks = Enum.reverse(chunks) + assert length(chunks) == chunk_count + + chunks + |> Enum.with_index(1) + |> Enum.each(fn {{registry, pg}, index} -> + frame = + {:snapshot_chunk, Group.Replica.WireProtocol.version(), stream_id, 123, index, + chunk_count, 80, 80, registry, pg} + + assert :erlang.external_size(frame) <= target + end) + + assert chunks |> Enum.flat_map(&elem(&1, 0)) |> MapSet.new() == MapSet.new(registry_rows) + assert chunks |> Enum.flat_map(&elem(&1, 1)) |> MapSet.new() == MapSet.new(pg_rows) + assert :ets.info(table, :size) == chunk_count + + assert :ok = Snapshot.delete_staging_table(table) + end + + test "event buffering preserves every event across bounded ETS chunks" do + table = Snapshot.new_event_table() + events = Enum.map(1..1_025, &{:event, &1}) + + buffer = Snapshot.new_event_buffer(table) + buffer = Snapshot.buffer_events(events, buffer) + buffer = Snapshot.finish_event_buffer(buffer) + + assert buffer.events == [] + assert buffer.count == 0 + assert buffer.chunk_index == 3 + assert :ets.info(table, :size) == 3 + + assert Snapshot.fold_events(table, [], fn event, acc -> [event | acc] end) + |> Enum.reverse() == events + + assert :ok = Snapshot.delete_staging_table(table) + end end From 19926d05e1f2b4e39e8eb90afecff9471414da22 Mon Sep 17 00:00:00 2001 From: Chris McCord Date: Fri, 14 Aug 2026 21:04:49 +0000 Subject: [PATCH 11/16] Stream exact snapshots in one pass --- CHANGELOG.md | 16 +- README.md | 43 ++- lib/group.ex | 3 +- lib/group/replica.ex | 408 +++++++++++++------- lib/group/replica/snapshot.ex | 291 ++++++-------- lib/group/replica/wire_protocol.ex | 2 +- test/README.md | 4 +- test/anti_entropy_fault_regression_test.exs | 3 +- test/formal/README.md | 17 +- test/formal/SnapshotAssembly.cfg | 2 +- test/formal/SnapshotAssembly.tla | 150 ++++--- test/jepsen/node.exs | 8 +- test/mutation/README.md | 5 +- test/mutation/run.exs | 175 ++++++--- test/replica_snapshot_distributed_test.exs | 317 +++++++++++++-- test/replica_snapshot_test.exs | 135 ++++--- test/support/test_cluster.ex | 4 +- test/support/test_replica_transport.ex | 32 ++ 18 files changed, 1037 insertions(+), 578 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47fdd90..4d6e920 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,13 +17,15 @@ `Group.Replica.WireProtocol` to avoid overloading Elixir protocol terminology. The standalone TCP adapter is retained only as hidden test infrastructure; Group ships the transport contract, dist-Erlang adapter, and outbox helper. -- **Breaking**: replica protocol v2 splits exact snapshots into - transport-neutral, byte-targeted chunks (`1 MiB` by default). Receivers stage - chunks in shard-owned private ETS and advance the stream cursor only after an - exact, authority-fenced assembly is complete; loss, duplication, reordering, - supersession, expiry, and shard crashes remain repairable by anti-entropy. - Single-chunk snapshots retain a direct fast path. Sideband transports can use - per-shard local outboxes for bounded batching without adding a hop to the +- **Breaking**: replica protocol v3 streams exact snapshots as provisional, + transport-neutral byte-targeted chunks (`1 MiB` by default) followed by an + independently retryable terminal manifest. The sender scans once and retains + only its current chunk; a concurrent mutation suppresses commit. Receivers + stage in reusable shard-owned private ETS and advance the cursor only after + one exact, authority-fenced assembly is complete. Chunk/commit loss, + duplication, reordering, conflicting retransmission, supersession, expiry, + and shard crashes remain repairable by anti-entropy. Sideband transports can + use per-shard local outboxes for bounded batching without adding a hop to the default dist-Erlang adapter. Late-starting replica lanes now rebuild their view from shared exact authority when startup fanout races registration. - Replace replica state sends/snapshots with per-origin, generation- and diff --git a/README.md b/README.md index e677dd2..05ce985 100644 --- a/README.md +++ b/README.md @@ -310,8 +310,9 @@ All operations are **eventually consistent**: - **`replicated_snapshot_chunk_target_bytes`** — target maximum encoded size of each exact-snapshot message. Defaults to 1 MiB and applies above every transport, including dist Erlang. A single row larger than the target is - sent alone. Receivers stage chunks in shard-owned private ETS and replace - visible state only after the complete exact slice is present. + sent alone. Receivers stage provisional chunks in shard-owned private ETS and + replace visible state only after the complete slice and its terminal manifest + are present. - **`replicated_anti_entropy_interval`** — interval in milliseconds for stream head advertisements and nonblocking control heartbeats. Defaults to 1,000. - **`replicated_peer_lease_timeout`** — time without a dist-Erlang control @@ -479,10 +480,11 @@ There are no leaders, quorum acknowledgements, per-entry replicated tombstones, or known-membership retention barriers. Oplog memory is bounded locally and independently of slow peers. Deletes are normal ordered records while retained, and exact snapshots close gaps after pruning. Exact snapshots are split into -transport-neutral byte-bounded messages; loss, duplication, or reordering leaves -the old visible slice and cursor untouched until all chunks arrive. Incomplete -staging expires after a peer-lease interval without progress and is destroyed -automatically with its owning shard. Rejected first chunks, nodedown, +transport-neutral byte-bounded provisional chunks followed by a small terminal +manifest. Loss, duplication, reordering, or receipt of every chunk without that +commit leaves the old visible slice and cursor untouched. Incomplete staging +expires after a peer-lease interval without progress and is destroyed +automatically with its owning shard. Rejected chunks or manifests, nodedown, generation replacement, and retired epochs destroy matching staging immediately. Named-cluster close uses only a temporary local shard-completion barrier; the final shard removes it and all routing rows, @@ -520,17 +522,24 @@ target and invokes the adapter's `send_batch/4` callback. Calls that expire or return `:busy`/`:disconnected` are dropped without a local retry; the next anti-entropy exchange repairs them. -Exact-snapshot row capture runs in at most one off-shard worker per shard. The -worker sends only when the stream identity and fully-applied head are unchanged -after its scans. A concurrent write invalidates the capture and periodic -anti-entropy retries, keeping million-row scans off the Group control process. -The worker holds at most one byte-targeted chunk on its process heap and keeps -completed chunks in an unnamed private ETS table for that snapshot attempt. -The receiver uses the same bounded-chunk shape plus minimal row-presence markers -until exact commit. Both sides delete this ephemeral staging after completion or -retry, and owner death deletes it automatically; it is not steady-state state -or an additional authority source. Exact-install monitor events are likewise -buffered into bounded private-ETS batches before the cursor becomes visible. +Exact-snapshot scans run in at most one off-shard worker per shard. The worker +validates the stream identity and fully-applied head both before and after one +pass over the registry and PG tables. It streams each completed chunk +immediately and retains only the current byte-targeted chunk on its heap. A +concurrent write suppresses the terminal manifest, so every provisional chunk +remains invisible and anti-entropy retries the newer head. Backpressure resumes +from the first unsent chunk; if only the manifest was backpressured, the sender +retains that small tuple and retries it without rescanning. + +The receiver necessarily retains one complete candidate in private ETS before +beginning exact replacement: absence from the committed candidate is a delete. +It also stores minimal row-presence markers to reject mixed/duplicate +assemblies and bounded monitor-event batches until the cursor becomes visible. +Transfer tables are cleared and pooled by the shard after completion or +rejection instead of being created for every retry; shard death destroys active +and pooled tables automatically. Sender memory is therefore O(chunk size), +while receiver staging is O(the exact origin slice), with neither becoming a +new authority source. The optional `peer_up/5` and `peer_down/4` callbacks report one shard lane at a time. A sideband adapter that shares a single node connection must retain it diff --git a/lib/group.ex b/lib/group.ex index ae703d8..97b5cda 100644 --- a/lib/group.ex +++ b/lib/group.ex @@ -250,7 +250,8 @@ defmodule Group do (default: `65_536`) - `:replicated_snapshot_chunk_target_bytes` — target maximum encoded size of each transport-neutral exact-snapshot chunk (default: `1_048_576`). A - single registry or membership row larger than the target remains one chunk. + single registry or membership row larger than the target remains one chunk; + a separate small terminal manifest commits the complete candidate. - `:replicated_anti_entropy_interval` — milliseconds between repeated stream head advertisements (default: `1_000`) - `:replicated_peer_lease_timeout` — milliseconds without a dist-Erlang diff --git a/lib/group/replica.ex b/lib/group/replica.ex index 9bb6533..d706a10 100644 --- a/lib/group/replica.ex +++ b/lib/group/replica.ex @@ -95,8 +95,9 @@ defmodule Group.Replica do - delta_batch carries one or more contiguous stream runs. - need requests the receiver's next missing sequence. - snapshot_chunk carries a byte-bounded part of one exact origin slice when - the requested prefix has already been pruned. Receivers stage chunks in a - private ETS table and expose nothing until every chunk is present. + the requested prefix has already been pruned. snapshot_commit carries the + independently retryable terminal row/chunk counts. Receivers expose nothing + until one valid manifest and every provisional chunk are present. Every stream field is validated against the source node and current generation/epoch. An old generation, a closed epoch, a wrong shard, @@ -104,8 +105,9 @@ defmodule Group.Replica do reordering is safe: early messages are ignored and repeated heads repair them; late messages fail their generation or epoch fence. Snapshot chunks may be lost, duplicated, reordered, or mixed across retransmissions at the same - stream head; exact row counts and set insertion prevent partial commits. - Rejected first chunks destroy their staging table immediately. Node loss, + stream head; exact row counts, set insertion, and conflicting-retransmission + checks prevent partial or mixed commits. Rejected chunks/manifests clear their + staging immediately. Node loss, generation replacement, and retired cluster streams discard matching partial assemblies immediately rather than waiting for their inactivity deadline. Unsequenced legacy state messages and malformed replica frames are rejected @@ -172,15 +174,17 @@ defmodule Group.Replica do installed. This avoids a shard-wide claim scan while ensuring a current cursor can never strand an old visible winner. - Exact-snapshot row capture runs in at most one off-shard worker per shard. It - sends only if the local stream identity and fully-applied head are unchanged - after both row scans; overlapping writes discard the capture and periodic - anti-entropy retries. The worker builds at most one byte-targeted chunk on its - heap and retains completed chunks only in an unnamed private ETS table owned - by that attempt. Receiver payload chunks and exact-install monitor-event - batches use the same ephemeral private-ETS ownership. Completion, retry, or - owner death deletes them. This keeps million-row scans and whole-snapshot - heaps off the control process without adding steady-state indexes. + Exact-snapshot scans run in at most one off-shard worker per shard. The worker + validates the local stream identity and fully-applied head before and after a + single pass, sends completed provisional chunks immediately, and retains only + one byte-targeted chunk. Overlapping writes suppress the terminal commit and + periodic anti-entropy retries the newer head. A busy chunk resumes at its + index; a busy commit retains only its small manifest. Receivers retain the + complete candidate and exact-install events in private ETS because exact + absence-as-delete replacement must start from a complete set. Cleared tables are + pooled for reuse; shard death deletes active and pooled tables. This keeps + sender memory bounded at million-to-tens-of-millions scale without putting + whole-snapshot heaps on the control process. Incoming PG mutations retain the bulk receiver lane. Contiguous registry records in one stream run are projected together and emit one monitor event @@ -241,6 +245,7 @@ defmodule Group.Replica do pending_registry_reprojections: %{}, monitors: %{}, snapshot_transfers: %{}, + snapshot_staging_pool: [], snapshot_send: nil, snapshot_send_offsets: %{} ] @@ -991,7 +996,14 @@ defmodule Group.Replica do {:resume, chunk_index} -> if current_snapshot_send?(state, snapshot_key) do - Map.put(state.snapshot_send_offsets, snapshot_key, chunk_index) + Map.put(state.snapshot_send_offsets, snapshot_key, {:chunk, chunk_index}) + else + Map.delete(state.snapshot_send_offsets, snapshot_key) + end + + {:resume_commit, manifest} -> + if current_snapshot_send?(state, snapshot_key) do + Map.put(state.snapshot_send_offsets, snapshot_key, {:commit, manifest}) else Map.delete(state.snapshot_send_offsets, snapshot_key) end @@ -3438,9 +3450,16 @@ defmodule Group.Replica do %{state | snapshot_transfers: transfers} {transfer, transfers} -> - :ok = Snapshot.delete_staging_table(transfer.table) - :ok = Snapshot.delete_staging_table(transfer.events) - %{state | snapshot_transfers: transfers} + :ok = Snapshot.clear_staging_table(transfer.table) + :ok = Snapshot.clear_staging_table(transfer.events) + + %{ + state + | snapshot_transfers: transfers, + snapshot_staging_pool: [ + {transfer.table, transfer.events} | state.snapshot_staging_pool + ] + } end end @@ -3653,22 +3672,12 @@ defmodule Group.Replica do defp handle_replica_message( state, source_node, - {:snapshot_chunk, version, stream_id, snapshot_seq, chunk_index, chunk_count, - registry_count, pg_count, reg_data, pg_data} + {:snapshot_chunk, version, stream_id, snapshot_seq, chunk_index, reg_data, pg_data} ) when version == @protocol_version and is_integer(snapshot_seq) and snapshot_seq >= 0 and - is_integer(chunk_index) and is_integer(chunk_count) and - is_integer(registry_count) and is_integer(pg_count) and is_list(reg_data) and - is_list(pg_data) do + is_integer(chunk_index) and is_list(reg_data) and is_list(pg_data) do if valid_snapshot_stream?(state, source_node, stream_id, snapshot_seq) and - valid_snapshot_manifest?( - chunk_index, - chunk_count, - registry_count, - pg_count, - reg_data, - pg_data - ) and + valid_snapshot_chunk?(chunk_index, reg_data, pg_data) and valid_snapshot_rows?(state, source_node, stream_id, reg_data, pg_data) do stage_replica_snapshot_chunk( state, @@ -3676,9 +3685,6 @@ defmodule Group.Replica do stream_id, snapshot_seq, chunk_index, - chunk_count, - registry_count, - pg_count, reg_data, pg_data ) @@ -3687,6 +3693,30 @@ defmodule Group.Replica do end end + defp handle_replica_message( + state, + source_node, + {:snapshot_commit, version, stream_id, snapshot_seq, chunk_count, registry_count, + pg_count} + ) + when version == @protocol_version and is_integer(snapshot_seq) and snapshot_seq >= 0 and + is_integer(chunk_count) and is_integer(registry_count) and is_integer(pg_count) do + if valid_snapshot_stream?(state, source_node, stream_id, snapshot_seq) and + valid_snapshot_commit_manifest?(chunk_count, registry_count, pg_count) do + stage_replica_snapshot_commit( + state, + source_node, + stream_id, + snapshot_seq, + chunk_count, + registry_count, + pg_count + ) + else + state + end + end + defp handle_replica_message(state, _source_node, _message), do: state defp handle_replica_heads(state, source_node, heads) do @@ -3712,22 +3742,17 @@ defmodule Group.Replica do snapshot_seq > Data.replica_cursor(state.name, state.shard_index, stream_id) end - defp valid_snapshot_manifest?( - chunk_index, - chunk_count, - registry_count, - pg_count, - reg_data, - pg_data - ) do - total_count = registry_count + pg_count + defp valid_snapshot_chunk?(chunk_index, reg_data, pg_data) do chunk_row_count = length(reg_data) + length(pg_data) + chunk_index > 0 and (chunk_row_count > 0 or chunk_index == 1) + end - chunk_count > 0 and chunk_index > 0 and chunk_index <= chunk_count and - registry_count >= 0 and pg_count >= 0 and + defp valid_snapshot_commit_manifest?(chunk_count, registry_count, pg_count) do + total_count = registry_count + pg_count + + chunk_count > 0 and registry_count >= 0 and pg_count >= 0 and chunk_count <= max(total_count, 1) and - ((total_count == 0 and chunk_count == 1 and chunk_row_count == 0) or - (total_count > 0 and chunk_row_count > 0)) + (total_count > 0 or chunk_count == 1) end defp valid_snapshot_rows?(state, source_node, stream_id, reg_data, pg_data) do @@ -3803,26 +3828,25 @@ defmodule Group.Replica do stream_id, snapshot_seq, chunk_index, - chunk_count, - registry_count, - pg_count, reg_data, pg_data ) do key = {source_node, stream_id} - manifest = {chunk_count, registry_count, pg_count} - case snapshot_transfer(state, key, snapshot_seq, manifest) do + case snapshot_transfer(state, key, snapshot_seq) do {:ignore, state} -> state {:ok, state, transfer} -> cond do - MapSet.member?(transfer.received, chunk_index) -> + MapSet.member?(transfer.received, chunk_index) and + Snapshot.chunk_matches?(transfer.table, chunk_index, reg_data, pg_data) -> state - transfer.registry_seen + length(reg_data) > registry_count or - transfer.pg_seen + length(pg_data) > pg_count -> + MapSet.member?(transfer.received, chunk_index) -> + discard_snapshot_transfer(state, key) + + not snapshot_chunk_within_manifest?(transfer, chunk_index, reg_data, pg_data) -> discard_snapshot_transfer(state, key) true -> @@ -3846,10 +3870,43 @@ defmodule Group.Replica do end end - defp snapshot_transfer(state, key, snapshot_seq, manifest) do + defp stage_replica_snapshot_commit( + state, + source_node, + stream_id, + snapshot_seq, + chunk_count, + registry_count, + pg_count + ) do + key = {source_node, stream_id} + manifest = {chunk_count, registry_count, pg_count} + + case snapshot_transfer(state, key, snapshot_seq) do + {:ignore, state} -> + state + + {:ok, state, %{manifest: nil} = transfer} -> + if snapshot_transfer_within_manifest?(transfer, manifest) do + transfer = %{transfer | manifest: manifest, last_progress: monotonic_millis()} + state = put_snapshot_transfer(state, key, transfer) + maybe_commit_snapshot_transfer(state, key, source_node, stream_id) + else + discard_snapshot_transfer(state, key) + end + + {:ok, state, %{manifest: ^manifest}} -> + maybe_commit_snapshot_transfer(state, key, source_node, stream_id) + + {:ok, state, _conflicting_transfer} -> + discard_snapshot_transfer(state, key) + end + end + + defp snapshot_transfer(state, key, snapshot_seq) do case Map.get(state.snapshot_transfers, key) do nil -> - transfer = new_snapshot_transfer(snapshot_seq, manifest) + {state, transfer} = new_snapshot_transfer(state, snapshot_seq) {:ok, put_snapshot_transfer(state, key, transfer), transfer} %{snapshot_seq: existing_seq} when existing_seq > snapshot_seq -> @@ -3857,31 +3914,33 @@ defmodule Group.Replica do %{snapshot_seq: existing_seq} when existing_seq < snapshot_seq -> state = discard_snapshot_transfer(state, key) - transfer = new_snapshot_transfer(snapshot_seq, manifest) + {state, transfer} = new_snapshot_transfer(state, snapshot_seq) {:ok, put_snapshot_transfer(state, key, transfer), transfer} - %{manifest: ^manifest} = transfer -> + %{snapshot_seq: ^snapshot_seq} = transfer -> {:ok, state, transfer} - - _conflicting_transfer -> - {:ignore, discard_snapshot_transfer(state, key)} end end - defp new_snapshot_transfer(snapshot_seq, {chunk_count, registry_count, pg_count} = manifest) do - %{ + defp new_snapshot_transfer(state, snapshot_seq) do + {table, events, pool} = + case state.snapshot_staging_pool do + [{table, events} | pool] -> {table, events, pool} + [] -> {Snapshot.new_staging_table(), Snapshot.new_event_table(), []} + end + + transfer = %{ snapshot_seq: snapshot_seq, - manifest: manifest, - chunk_count: chunk_count, - registry_count: registry_count, - pg_count: pg_count, + manifest: nil, registry_seen: 0, pg_seen: 0, received: MapSet.new(), last_progress: monotonic_millis(), - table: Snapshot.new_staging_table(), - events: Snapshot.new_event_table() + table: table, + events: events } + + {%{state | snapshot_staging_pool: pool}, transfer} end defp put_snapshot_transfer(state, key, transfer) do @@ -3891,19 +3950,45 @@ defmodule Group.Replica do defp maybe_commit_snapshot_transfer(state, key, source_node, stream_id) do transfer = Map.fetch!(state.snapshot_transfers, key) - if MapSet.size(transfer.received) == transfer.chunk_count do - if transfer.registry_seen == transfer.registry_count and - transfer.pg_seen == transfer.pg_count do - commit_snapshot_transfer(state, key, source_node, stream_id, transfer) - else - discard_snapshot_transfer(state, key) - end - else - state + case transfer.manifest do + {chunk_count, registry_count, pg_count} -> + if MapSet.size(transfer.received) == chunk_count and + transfer.registry_seen == registry_count and transfer.pg_seen == pg_count do + commit_snapshot_transfer(state, key, source_node, stream_id, transfer) + else + state + end + + nil -> + state end end + defp snapshot_chunk_within_manifest?(%{manifest: nil}, _chunk_index, _reg_data, _pg_data), + do: true + + defp snapshot_chunk_within_manifest?( + %{manifest: {chunk_count, registry_count, pg_count}} = transfer, + chunk_index, + reg_data, + pg_data + ) do + chunk_index <= chunk_count and + transfer.registry_seen + length(reg_data) <= registry_count and + transfer.pg_seen + length(pg_data) <= pg_count + end + + defp snapshot_transfer_within_manifest?( + transfer, + {chunk_count, registry_count, pg_count} + ) do + Enum.all?(transfer.received, &(&1 <= chunk_count)) and + transfer.registry_seen <= registry_count and transfer.pg_seen <= pg_count + end + defp commit_snapshot_transfer(state, key, source_node, stream_id, transfer) do + {chunk_count, _registry_count, _pg_count} = transfer.manifest + state = if valid_snapshot_stream?(state, source_node, stream_id, transfer.snapshot_seq) do state = flush_pending_replicated_barrier(state) @@ -3926,7 +4011,7 @@ defmodule Group.Replica do stream_id, transfer.snapshot_seq, transfer.table, - transfer.chunk_count, + chunk_count, {state, event_buffer}, fn key, {acc, buffer} -> {acc, events} = reconcile_registry_projection(acc, cluster, key, :reconcile, []) @@ -3940,7 +4025,7 @@ defmodule Group.Replica do source_node, cluster, transfer.table, - transfer.chunk_count, + chunk_count, event_buffer ) @@ -4338,7 +4423,7 @@ defmodule Group.Replica do end) |> Map.new() - start_index = Map.get(offsets, snapshot_key, 1) + resume = Map.get(offsets, snapshot_key, {:chunk, 1}) snapshot_context = %{ name: state.name, @@ -4352,12 +4437,12 @@ defmodule Group.Replica do spawn(fn -> result = try do - capture_and_send_replica_snapshot( + stream_replica_snapshot( snapshot_context, target_node, stream_id, head, - start_index + resume ) catch kind, reason -> @@ -4378,93 +4463,120 @@ defmodule Group.Replica do %{state | snapshot_send: {worker, token, snapshot_key}, snapshot_send_offsets: offsets} end - defp capture_and_send_replica_snapshot(state, target_node, stream_id, head, start_index) do + defp stream_replica_snapshot( + state, + target_node, + stream_id, + head, + {:commit, manifest} + ) do + if current_snapshot_send?(state, target_node, stream_id, head) do + send_replica_snapshot_commit(state, target_node, stream_id, head, manifest) + else + :complete + end + end + + defp stream_replica_snapshot(state, target_node, stream_id, head, {:chunk, start_index}) do + if current_snapshot_send?(state, target_node, stream_id, head) do + do_stream_replica_snapshot(state, target_node, stream_id, head, start_index) + else + :complete + end + end + + defp do_stream_replica_snapshot(state, target_node, stream_id, head, start_index) do cluster = WireProtocol.stream_cluster(stream_id) - capture_table = Snapshot.new_capture_table() - envelope_bytes = Snapshot.capture_envelope_bytes(stream_id, head) - - capture = - Snapshot.new_capture( - capture_table, - state.replicated_snapshot_chunk_target_bytes, - envelope_bytes - ) + envelope_bytes = Snapshot.stream_envelope_bytes(stream_id, head) + + emit = fn registry, pg, chunk_index -> + message = + {:snapshot_chunk, WireProtocol.version(), stream_id, head, chunk_index, registry, pg} + + case state.replica_transport.outgoing( + state.name, + target_node, + state.shard_index, + message, + state.replica_transport_opts + ) do + :ok -> :ok + result when result in [:busy, :disconnected] -> throw({:snapshot_resume, chunk_index}) + end + end try do - capture = + stream = + Snapshot.new_stream( + state.replicated_snapshot_chunk_target_bytes, + envelope_bytes, + start_index, + emit + ) + + stream = Data.reduce_registry_claim_batches_for_stream( state.name, state.shard_index, stream_id, - capture, - &Snapshot.capture_registry_many/2 + stream, + &Snapshot.stream_registry_many/2 ) - capture = + stream = Data.reduce_pg_entry_batches_for_origin( state.name, state.shard_index, cluster, node(), - capture, - &Snapshot.capture_pg_many/2 + stream, + &Snapshot.stream_pg_many/2 ) - capture = Snapshot.finish_capture(capture) - registry_count = capture.registry_count - pg_count = capture.pg_count - chunk_count = capture.chunk_count - - {_floor, current_head, applied} = - Data.replica_stream_head(state.name, state.shard_index, stream_id) - - # Appending advances the head before materializing its table changes. - # Therefore an unchanged, fully-applied head after both scans proves the - # private capture is one exact state at `head`; an overlapping write makes - # it disposable and anti-entropy retries without sending a partial view. - if Data.local_stream_id(state.name, state.shard_index, cluster) == stream_id and - current_head == head and applied == head and - target_node in Data.cluster_nodes(state.name, cluster) do - case Snapshot.reduce_capture_chunks( - capture_table, - chunk_count, - 1, - fn reg_chunk, pg_chunk, chunk_index -> - if chunk_index < start_index do - {:cont, chunk_index + 1} - else - message = - {:snapshot_chunk, WireProtocol.version(), stream_id, head, chunk_index, - chunk_count, registry_count, pg_count, reg_chunk, pg_chunk} - - case state.replica_transport.outgoing( - state.name, - target_node, - state.shard_index, - message, - state.replica_transport_opts - ) do - :ok -> - {:cont, chunk_index + 1} - - result when result in [:busy, :disconnected] -> - {:halt, {:resume, chunk_index}} - end - end - end - ) do - {:ok, _next_index} -> :complete - {:halt, result} -> result - end + stream = Snapshot.finish_stream(stream) + manifest = {stream.chunk_count, stream.registry_count, stream.pg_count} + + # Chunks are provisional. Appending advances the head before changing + # materialized rows and advances `applied` only afterward, so an + # unchanged fully-applied head proves the completed scan is one exact + # state at `head`. Only the terminal commit makes those chunks visible. + if current_snapshot_send?(state, target_node, stream_id, head) do + send_replica_snapshot_commit(state, target_node, stream_id, head, manifest) else :complete end - after - Snapshot.delete_staging_table(capture_table) + catch + {:snapshot_resume, chunk_index} -> {:resume, chunk_index} + end + end + + defp send_replica_snapshot_commit( + state, + target_node, + stream_id, + head, + {chunk_count, registry_count, pg_count} = manifest + ) do + message = + {:snapshot_commit, WireProtocol.version(), stream_id, head, chunk_count, registry_count, + pg_count} + + case state.replica_transport.outgoing( + state.name, + target_node, + state.shard_index, + message, + state.replica_transport_opts + ) do + :ok -> :complete + result when result in [:busy, :disconnected] -> {:resume_commit, manifest} end end - defp current_snapshot_send?(state, {target_node, stream_id, head}) do + defp current_snapshot_send?(state, {target_node, stream_id, head}), + do: current_snapshot_send?(state, target_node, stream_id, head) + + defp current_snapshot_send?(state, target_node, stream_id, head) do cluster = WireProtocol.stream_cluster(stream_id) {_floor, current_head, applied} = diff --git a/lib/group/replica/snapshot.ex b/lib/group/replica/snapshot.ex index 68f2aa6..5a943a2 100644 --- a/lib/group/replica/snapshot.ex +++ b/lib/group/replica/snapshot.ex @@ -1,73 +1,48 @@ defmodule Group.Replica.Snapshot do @moduledoc false - # The target is for the complete snapshot frame, not just its rows. Per-row - # external sizes conservatively include an extra ETF version byte, and this - # reserve covers the frame tuple, stream identity, both list headers, and - # integer fields. A single entry larger than the target remains one chunk. - @default_envelope_reserve 512 @max_frame_counter 18_446_744_073_709_551_615 - def chunk_rows(registry_rows, pg_rows, target_bytes) - when is_list(registry_rows) and is_list(pg_rows) and is_integer(target_bytes) and - target_bytes > 0 do - chunk_rows(registry_rows, pg_rows, target_bytes, @default_envelope_reserve) + def stream_envelope_bytes(stream_id, snapshot_seq) do + :erlang.external_size( + {:snapshot_chunk, Group.Replica.WireProtocol.version(), stream_id, snapshot_seq, + @max_frame_counter, [], []} + ) + 10 end - def chunk_rows(registry_rows, pg_rows, target_bytes, envelope_bytes) - when is_list(registry_rows) and is_list(pg_rows) and is_integer(target_bytes) and - target_bytes > 0 and is_integer(envelope_bytes) and envelope_bytes >= 0 do - registry_rows = Enum.sort_by(registry_rows, fn {key, _pid, _meta, _time} -> key end) - pg_rows = Enum.sort_by(pg_rows, fn {key, pid, _meta, _time} -> {key, pid} end) - payload_target = max(target_bytes - envelope_bytes, 1) - - acc = %{chunks: [], registry: [], pg: [], bytes: 0, count: 0} - acc = Enum.reduce(registry_rows, acc, &add_row(&2, :registry, &1, payload_target)) - acc = Enum.reduce(pg_rows, acc, &add_row(&2, :pg, &1, payload_target)) - - chunks = - acc - |> flush_chunk() - |> Map.fetch!(:chunks) - |> Enum.reverse() - |> case do - [] -> [{[], []}] - chunks -> chunks - end - + def new_stream(target_bytes, envelope_bytes, start_index, emit) + when is_integer(target_bytes) and target_bytes > 0 and is_integer(envelope_bytes) and + envelope_bytes >= 0 and is_integer(start_index) and start_index > 0 and + is_function(emit, 3) do %{ - registry_count: length(registry_rows), - pg_count: length(pg_rows), - chunks: chunks + payload_target: max(target_bytes - envelope_bytes, 1), + start_index: start_index, + emit: emit, + registry: [], + pg: [], + bytes: 0, + current_count: 0, + chunk_count: 0, + registry_count: 0, + pg_count: 0 } end - def frame_envelope_bytes(stream_id, snapshot_seq, registry_count, pg_count) do - # A non-empty ETF list adds a five-byte LIST_EXT header relative to an - # empty list. Reserve that once for each domain. The large chunk integers - # ensure every practical index/count uses no more space than this envelope. - :erlang.external_size( - {:snapshot_chunk, Group.Replica.WireProtocol.version(), stream_id, snapshot_seq, - @max_frame_counter, @max_frame_counter, registry_count, pg_count, [], []} - ) + 10 + def stream_registry_many(rows, stream) when is_list(rows) do + Enum.reduce(rows, stream, &stream_row(&2, :registry, &1)) end - def capture_envelope_bytes(stream_id, snapshot_seq) do - # Capture has to choose chunk boundaries before it knows the final row and - # chunk counts. Reserving their largest practical (unsigned 64-bit) ETF - # representation makes every emitted frame no larger than the configured - # target without requiring a second pass over the captured rows. - :erlang.external_size( - {:snapshot_chunk, Group.Replica.WireProtocol.version(), stream_id, snapshot_seq, - @max_frame_counter, @max_frame_counter, @max_frame_counter, @max_frame_counter, [], []} - ) + 10 + def stream_pg_many(rows, stream) when is_list(rows) do + Enum.reduce(rows, stream, &stream_row(&2, :pg, &1)) end - def new_staging_table do - :ets.new(__MODULE__, [:set, :private]) - end + def finish_stream(%{current_count: 0, chunk_count: 0} = stream), + do: flush_stream_chunk(stream) + + def finish_stream(%{current_count: 0} = stream), do: stream + def finish_stream(stream), do: flush_stream_chunk(stream) - def new_capture_table do + def new_staging_table do :ets.new(__MODULE__, [:set, :private]) end @@ -85,37 +60,64 @@ defmodule Group.Replica.Snapshot do :ok end + def clear_staging_table(table) do + try do + :ets.delete_all_objects(table) + rescue + ArgumentError -> true + end + + :ok + end + def stage_rows(table, chunk_index, registry_rows, pg_rows) do - objects = - Enum.map(registry_rows, &staging_object(:registry, &1)) ++ - Enum.map(pg_rows, &staging_object(:pg, &1)) + row_objects = + Enum.map(registry_rows, &staging_object(:registry, chunk_index, &1)) ++ + Enum.map(pg_rows, &staging_object(:pg, chunk_index, &1)) + objects = [{{:chunk, chunk_index}, length(registry_rows), length(pg_rows)} | row_objects] size_before = :ets.info(table, :size) if :ets.insert_new(table, objects) and :ets.info(table, :size) - size_before == length(objects) do - if :ets.insert_new(table, {{:chunk, chunk_index}, registry_rows, pg_rows}) do - :ok - else - {:error, :duplicate_row} - end + :ok else {:error, :duplicate_row} end end - def fold_registry(table, chunk_count, acc, fun) when is_function(fun, 2) do - Enum.reduce(1..chunk_count, acc, fn chunk_index, inner -> - {registry_rows, _pg_rows} = fetch_chunk(table, chunk_index) - Enum.reduce(registry_rows, inner, fun) - end) + def chunk_matches?(table, chunk_index, registry_rows, pg_rows) do + case :ets.lookup(table, {:chunk, chunk_index}) do + [{{:chunk, ^chunk_index}, registry_count, pg_count}] -> + registry_count == length(registry_rows) and pg_count == length(pg_rows) and + Enum.all?(registry_rows, &staged_registry_row?(table, chunk_index, &1)) and + Enum.all?(pg_rows, &staged_pg_row?(table, chunk_index, &1)) + + [] -> + false + end end - def fold_pg(table, chunk_count, acc, fun) when is_function(fun, 2) do - Enum.reduce(1..chunk_count, acc, fn chunk_index, inner -> - {_registry_rows, pg_rows} = fetch_chunk(table, chunk_index) - Enum.reduce(pg_rows, inner, fun) - end) + def fold_registry(table, _chunk_count, acc, fun) when is_function(fun, 2) do + fold_staged_rows( + table, + [ + {{{:registry, :"$1"}, :"$2", :"$3", :"$4", :_}, [], [{{:"$1", :"$2", :"$3", :"$4"}}]} + ], + acc, + fun + ) + end + + def fold_pg(table, _chunk_count, acc, fun) when is_function(fun, 2) do + fold_staged_rows( + table, + [ + {{{:pg, :"$1", :"$2"}, :"$3", :"$4", :_}, [], [{{:"$1", :"$2", :"$3", :"$4"}}]} + ], + acc, + fun + ) end def new_event_buffer(table) do @@ -153,139 +155,80 @@ defmodule Group.Replica.Snapshot do def member_pg?(table, key, pid), do: :ets.member(table, {:pg, key, pid}) def member_registry?(table, key), do: :ets.member(table, {:registry, key}) - def new_capture(table, target_bytes, envelope_bytes) - when is_integer(target_bytes) and target_bytes > 0 and is_integer(envelope_bytes) and - envelope_bytes >= 0 do - %{ - table: table, - payload_target: max(target_bytes - envelope_bytes, 1), - registry: [], - pg: [], - bytes: 0, - current_count: 0, - chunk_count: 0, - registry_count: 0, - pg_count: 0 - } - end + defp staging_object(:registry, chunk_index, {key, pid, meta, time}), + do: {{:registry, key}, pid, meta, time, chunk_index} - def capture_registry_many(rows, capture) when is_list(rows) do - Enum.reduce(rows, capture, &capture_row(&2, :registry, &1)) - end + defp staging_object(:pg, chunk_index, {key, pid, meta, time}), + do: {{:pg, key, pid}, meta, time, chunk_index} - def capture_pg_many(rows, capture) when is_list(rows) do - Enum.reduce(rows, capture, &capture_row(&2, :pg, &1)) + defp staged_registry_row?(table, chunk_index, {key, pid, meta, time}) do + :ets.lookup(table, {:registry, key}) == + [{{:registry, key}, pid, meta, time, chunk_index}] end - def finish_capture(%{current_count: 0, chunk_count: 0} = capture), - do: flush_capture_chunk(capture) - - def finish_capture(%{current_count: 0} = capture), do: capture - def finish_capture(capture), do: flush_capture_chunk(capture) - - def reduce_capture_chunks(table, chunk_count, acc, fun) - when is_integer(chunk_count) and chunk_count > 0 and is_function(fun, 3) do - Enum.reduce_while(1..chunk_count, acc, fn chunk_index, inner -> - [{{:chunk, ^chunk_index}, registry, pg}] = :ets.lookup(table, {:chunk, chunk_index}) - - case fun.(registry, pg, inner) do - {:cont, next} -> {:cont, next} - {:halt, result} -> {:halt, {:halt, result}} - end - end) - |> case do - {:halt, result} -> {:halt, result} - result -> {:ok, result} - end + defp staged_pg_row?(table, chunk_index, {key, pid, meta, time}) do + :ets.lookup(table, {:pg, key, pid}) == [{{:pg, key, pid}, meta, time, chunk_index}] end - defp add_row(%{count: count, bytes: bytes} = acc, domain, row, target) do - row_bytes = :erlang.external_size(row) - - acc = - if count > 0 and bytes + row_bytes > target do - flush_chunk(acc) - else - acc - end - - case domain do - :registry -> - %{ - acc - | registry: [row | acc.registry], - bytes: acc.bytes + row_bytes, - count: acc.count + 1 - } - - :pg -> - %{acc | pg: [row | acc.pg], bytes: acc.bytes + row_bytes, count: acc.count + 1} + defp fold_staged_rows(table, match_spec, acc, fun) do + case :ets.select(table, match_spec, 4_096) do + :"$end_of_table" -> acc + {rows, continuation} -> fold_staged_rows(continuation, Enum.reduce(rows, acc, fun), fun) end end - defp flush_chunk(%{count: 0} = acc), do: acc - - defp flush_chunk(acc) do - chunk = {Enum.reverse(acc.registry), Enum.reverse(acc.pg)} - - %{acc | chunks: [chunk | acc.chunks], registry: [], pg: [], bytes: 0, count: 0} - end - - defp staging_object(:registry, {key, _pid, _meta, _time}), - do: {{:registry, key}} - - defp staging_object(:pg, {key, pid, _meta, _time}), - do: {{:pg, key, pid}} - - defp fetch_chunk(table, chunk_index) do - case :ets.lookup(table, {:chunk, chunk_index}) do - [{{:chunk, ^chunk_index}, registry_rows, pg_rows}] -> {registry_rows, pg_rows} + defp fold_staged_rows(continuation, acc, fun) do + case :ets.select(continuation) do + :"$end_of_table" -> acc + {rows, next} -> fold_staged_rows(next, Enum.reduce(rows, acc, fun), fun) end end - defp capture_row(capture, domain, row) do + defp stream_row(stream, domain, row) do row_bytes = :erlang.external_size(row) - capture = - if capture.current_count > 0 and - capture.bytes + row_bytes > capture.payload_target do - flush_capture_chunk(capture) + stream = + if stream.current_count > 0 and stream.bytes + row_bytes > stream.payload_target do + flush_stream_chunk(stream) else - capture + stream end case domain do :registry -> %{ - capture - | registry: [row | capture.registry], - bytes: capture.bytes + row_bytes, - current_count: capture.current_count + 1, - registry_count: capture.registry_count + 1 + stream + | registry: [row | stream.registry], + bytes: stream.bytes + row_bytes, + current_count: stream.current_count + 1, + registry_count: stream.registry_count + 1 } :pg -> %{ - capture - | pg: [row | capture.pg], - bytes: capture.bytes + row_bytes, - current_count: capture.current_count + 1, - pg_count: capture.pg_count + 1 + stream + | pg: [row | stream.pg], + bytes: stream.bytes + row_bytes, + current_count: stream.current_count + 1, + pg_count: stream.pg_count + 1 } end end - defp flush_capture_chunk(capture) do - chunk_index = capture.chunk_count + 1 + defp flush_stream_chunk(stream) do + chunk_index = stream.chunk_count + 1 - true = - :ets.insert_new( - capture.table, - {{:chunk, chunk_index}, Enum.reverse(capture.registry), Enum.reverse(capture.pg)} - ) + if chunk_index >= stream.start_index do + :ok = + stream.emit.( + Enum.reverse(stream.registry), + Enum.reverse(stream.pg), + chunk_index + ) + end %{ - capture + stream | registry: [], pg: [], bytes: 0, diff --git a/lib/group/replica/wire_protocol.ex b/lib/group/replica/wire_protocol.ex index 3bb4aeb..8219350 100644 --- a/lib/group/replica/wire_protocol.ex +++ b/lib/group/replica/wire_protocol.ex @@ -1,7 +1,7 @@ defmodule Group.Replica.WireProtocol do @moduledoc false - @version 2 + @version 3 def version, do: @version diff --git a/test/README.md b/test/README.md index ad83318..f62538e 100644 --- a/test/README.md +++ b/test/README.md @@ -28,8 +28,8 @@ release qualification rather than individual edits. | `anti_entropy_fault_regression_test.exs` | Three-node regressions for hidden-winner projection, receiver restart eviction, nodedown/lease lane retirement, authority gaps and cross-lane races, in-flight conflict fencing, crash-journal replay, cursorless/interrupted snapshot repair, malformed ingress, and sideband rediscovery | | `replica_adversarial_test.exs` | Reproducible three-node mixed-operation state machines: drops, busy returns, duplication, reordering, bounded delay, oplog pruning, conflicts, owner death, and named-cluster epoch churn, followed by exact convergence/dead-owner/internal-index checks | | `replica_model_property_test.exs` | StreamData-generated and shrunk owner histories against an independent lifecycle oracle and scheduler-controlled replica transport | -| `replica_snapshot_test.exs` | Pure byte partitioning plus bounded private-ETS capture, receive staging, and event batching | -| `replica_snapshot_distributed_test.exs` | Real-node exact-snapshot loss, reorder, duplicate, conflicting retransmission, supersession, authority fencing, expiry, and shard-crash recovery | +| `replica_snapshot_test.exs` | Pure single-pass byte-bounded streaming, suffix resume, receive staging, and event batching | +| `replica_snapshot_distributed_test.exs` | Real-node provisional-chunk/terminal-commit loss, reorder, duplicate, conflicting retransmission/manifest, concurrent-source invalidation, supersession, authority fencing, expiry, pooled staging, and shard-crash recovery | ## Model-based and formal checks diff --git a/test/anti_entropy_fault_regression_test.exs b/test/anti_entropy_fault_regression_test.exs index 8b90600..7636dd1 100644 --- a/test/anti_entropy_fault_regression_test.exs +++ b/test/anti_entropy_fault_regression_test.exs @@ -473,7 +473,8 @@ defmodule Group.AntiEntropyFaultRegressionTest do {:delta_batch, version, [:not_a_delta_run]}, {:need, version, :not_a_stream, 1}, {:needs, version, [:not_a_need]}, - {:snapshot_chunk, version, :not_a_stream, 1, 1, 1, 0, 0, [], []}, + {:snapshot_chunk, version, :not_a_stream, 1, 1, [], []}, + {:snapshot_commit, version, :not_a_stream, 1, 1, 0, 0}, {:delta_batch, version, [ {stream_id, 1, diff --git a/test/formal/README.md b/test/formal/README.md index 0626af3..7eae990 100644 --- a/test/formal/README.md +++ b/test/formal/README.md @@ -11,11 +11,13 @@ contract. It covers: - fair convergence after healing. `SnapshotAssembly.tla` separately models the non-atomic wire delivery of an -exact snapshot. It explores arbitrary chunk loss, duplication, reordering, +exact snapshot. It explores independent provisional-chunk and terminal-commit +loss, duplication, and reordering, source invalidation before commit emission, newer-snapshot supersession, authority epoch changes, staging expiry, and receiver crashes. Its invariants require visible data and the cursor to remain -at a previously committed exact state until every chunk of one valid snapshot -is present; stale or mixed partial state can never become visible. +at a previously committed exact state until every chunk and a valid terminal +commit for one snapshot are present; stale or mixed partial state can never +become visible. `PeerEviction.tla` isolates the lifecycle boundary for a peer which never returns and for a later process using the same node name with a fresh @@ -82,11 +84,12 @@ assembly, peer-eviction, authority-projection, and authority-hint models; set `TLA_EXTENDED=1` for the larger anti-entropy configuration. The checked three-node default explores 1,835,826 states, finds 490,236 -distinct states to a depth of 30, and completes in roughly 1 minute 40 seconds -on the development machine used for the validation run. +distinct states to a depth of 30, and completes in roughly one minute on the +development machine used for the validation run. -The snapshot-assembly model explores 15,681 states, finds 1,088 distinct states -to a depth of 13, and completes in under a second on the same class of machine. +The snapshot-assembly model generates 3,305,473 states, finds 167,936 distinct +states to a depth of 23, and completes in roughly five seconds on the current +development machine. The peer-eviction model explores 1,527,116 states, finds 238,120 distinct states to a depth of 26, and completes in roughly 20 seconds on the development diff --git a/test/formal/SnapshotAssembly.cfg b/test/formal/SnapshotAssembly.cfg index 38a4dba..6c7706f 100644 --- a/test/formal/SnapshotAssembly.cfg +++ b/test/formal/SnapshotAssembly.cfg @@ -3,5 +3,5 @@ SPECIFICATION Spec INVARIANTS TypeOK VisibleIsAnExactCommittedSnapshot - StagingNeverLeaksIntoVisible StagingBelongsToOneSnapshot + NoCommitMeansNoInstall diff --git a/test/formal/SnapshotAssembly.tla b/test/formal/SnapshotAssembly.tla index 3d6c18d..ce75e21 100644 --- a/test/formal/SnapshotAssembly.tla +++ b/test/formal/SnapshotAssembly.tla @@ -2,10 +2,11 @@ EXTENDS Integers, FiniteSets, TLC (* -Finite model of the exact-snapshot chunk assembly boundary. It deliberately -models two snapshots in one authority epoch plus a new-epoch snapshot so TLC -can explore loss, duplication, reordering, supersession, stale final chunks, -expiry, and receiver crashes independently of the larger anti-entropy model. +Finite model of the exact-snapshot assembly boundary. Snapshot chunks are +provisional: a separately lossy, duplicable, and reorderable terminal commit +is required before a complete candidate can replace visible state. Two +snapshots share an authority epoch and one belongs to a replacement epoch so +TLC also explores supersession, expiry, receiver crashes, and stale messages. *) Snapshots == {1, 2, 3} @@ -37,7 +38,8 @@ ChunkRows(snapshot, chunk) == [] snapshot = 3 /\ chunk = 1 -> {"a"} [] snapshot = 3 /\ chunk = 2 -> {"d"} -Message == [snapshot : Snapshots, chunk : Chunks] +Message == + [kind : {"chunk", "commit"}, snapshot : Snapshots, chunk : {0} \union Chunks] VARIABLES authorityEpoch, cursor, @@ -45,11 +47,13 @@ VARIABLES authorityEpoch, stagedSnapshot, stagedChunks, stagedRows, + stagedCommitted, + commitAllowed, messages vars == <> + stagedRows, stagedCommitted, commitAllowed, messages>> Init == /\ authorityEpoch = 1 @@ -58,95 +62,135 @@ Init == /\ stagedSnapshot = 0 /\ stagedChunks = {} /\ stagedRows = {} + /\ stagedCommitted = FALSE + /\ commitAllowed = Snapshots /\ messages = {} -Send(snapshot, chunk) == +SendChunk(snapshot, chunk) == /\ messages' = messages \union - {[snapshot |-> snapshot, chunk |-> chunk]} + {[kind |-> "chunk", snapshot |-> snapshot, chunk |-> chunk]} /\ UNCHANGED <> + stagedChunks, stagedRows, stagedCommitted, commitAllowed>> -Valid(message) == - /\ SnapshotEpoch(message.snapshot) = authorityEpoch - /\ SnapshotSeq(message.snapshot) > cursor +SendCommit(snapshot) == + /\ snapshot \in commitAllowed + /\ messages' = messages \union + {[kind |-> "commit", snapshot |-> snapshot, chunk |-> 0]} + /\ UNCHANGED <> + +InvalidateBeforeCommit(snapshot) == + /\ snapshot \in commitAllowed + /\ commitAllowed' = commitAllowed \ {snapshot} + /\ UNCHANGED <> + +Valid(snapshot) == + /\ SnapshotEpoch(snapshot) = authorityEpoch + /\ SnapshotSeq(snapshot) > cursor -StartsNewAssembly(message) == - /\ Valid(message) +StartsNewAssembly(snapshot) == + /\ Valid(snapshot) /\ \/ stagedSnapshot = 0 \/ SnapshotEpoch(stagedSnapshot) # authorityEpoch - \/ SnapshotSeq(message.snapshot) > SnapshotSeq(stagedSnapshot) + \/ SnapshotSeq(snapshot) > SnapshotSeq(stagedSnapshot) -StartAssembly(message) == - /\ StartsNewAssembly(message) +StartChunk(message) == + /\ message.kind = "chunk" + /\ StartsNewAssembly(message.snapshot) /\ stagedSnapshot' = message.snapshot /\ stagedChunks' = {message.chunk} /\ stagedRows' = ChunkRows(message.snapshot, message.chunk) - /\ UNCHANGED <> + /\ stagedCommitted' = FALSE + /\ UNCHANGED <> + +StartCommit(message) == + /\ message.kind = "commit" + /\ StartsNewAssembly(message.snapshot) + /\ stagedSnapshot' = message.snapshot + /\ stagedChunks' = {} + /\ stagedRows' = {} + /\ stagedCommitted' = TRUE + /\ UNCHANGED <> -ContinueAssembly(message) == - /\ Valid(message) +ContinueChunk(message) == + /\ message.kind = "chunk" + /\ Valid(message.snapshot) /\ stagedSnapshot = message.snapshot /\ LET nextChunks == stagedChunks \union {message.chunk} - nextRows == stagedRows \union - ChunkRows(message.snapshot, message.chunk) - IN IF nextChunks = Chunks + nextRows == stagedRows \union ChunkRows(message.snapshot, message.chunk) + IN IF stagedCommitted /\ nextChunks = Chunks THEN /\ cursor' = SnapshotSeq(message.snapshot) /\ visible' = SnapshotRows(message.snapshot) /\ stagedSnapshot' = 0 /\ stagedChunks' = {} /\ stagedRows' = {} - ELSE /\ UNCHANGED <> + /\ stagedCommitted' = FALSE + ELSE /\ UNCHANGED <> /\ stagedChunks' = nextChunks /\ stagedRows' = nextRows - /\ UNCHANGED <> + /\ UNCHANGED <> -IgnoreChunk(message) == - /\ ~StartsNewAssembly(message) - /\ ~(/\ Valid(message) +ContinueCommit(message) == + /\ message.kind = "commit" + /\ Valid(message.snapshot) + /\ stagedSnapshot = message.snapshot + /\ IF stagedChunks = Chunks + THEN /\ cursor' = SnapshotSeq(message.snapshot) + /\ visible' = SnapshotRows(message.snapshot) + /\ stagedSnapshot' = 0 + /\ stagedChunks' = {} + /\ stagedRows' = {} + /\ stagedCommitted' = FALSE + ELSE /\ stagedCommitted' = TRUE + /\ UNCHANGED <> + /\ UNCHANGED <> + +Ignore(message) == + /\ ~StartsNewAssembly(message.snapshot) + /\ ~(/\ Valid(message.snapshot) /\ stagedSnapshot = message.snapshot) /\ UNCHANGED vars Deliver(message) == /\ message \in messages - /\ \/ StartAssembly(message) - \/ ContinueAssembly(message) - \/ IgnoreChunk(message) + /\ \/ StartChunk(message) + \/ StartCommit(message) + \/ ContinueChunk(message) + \/ ContinueCommit(message) + \/ Ignore(message) Drop(message) == /\ message \in messages /\ messages' = messages \ {message} /\ UNCHANGED <> + stagedChunks, stagedRows, stagedCommitted, commitAllowed>> InstallNewAuthority == /\ authorityEpoch = 1 /\ authorityEpoch' = 2 /\ cursor' = 0 /\ visible' = {} - (* The implementation may retain invisible old staging until expiry. *) - /\ UNCHANGED <> - -ExpireStaging == - /\ stagedSnapshot # 0 - /\ stagedSnapshot' = 0 - /\ stagedChunks' = {} - /\ stagedRows' = {} - /\ UNCHANGED <> + (* Invisible old staging may remain until expiry, but can never commit. *) + /\ UNCHANGED <> -CrashReceiver == +DiscardStaging == /\ stagedSnapshot # 0 /\ stagedSnapshot' = 0 /\ stagedChunks' = {} /\ stagedRows' = {} - /\ UNCHANGED <> + /\ stagedCommitted' = FALSE + /\ UNCHANGED <> Next == - \/ \E snapshot \in Snapshots, chunk \in Chunks : Send(snapshot, chunk) + \/ \E snapshot \in Snapshots, chunk \in Chunks : SendChunk(snapshot, chunk) + \/ \E snapshot \in Snapshots : SendCommit(snapshot) + \/ \E snapshot \in Snapshots : InvalidateBeforeCommit(snapshot) \/ \E message \in messages : Deliver(message) \/ \E message \in messages : Drop(message) \/ InstallNewAuthority - \/ ExpireStaging - \/ CrashReceiver + \/ DiscardStaging TypeOK == /\ authorityEpoch \in {1, 2} @@ -155,6 +199,8 @@ TypeOK == /\ stagedSnapshot \in {0} \union Snapshots /\ stagedChunks \subseteq Chunks /\ stagedRows \subseteq Rows + /\ stagedCommitted \in BOOLEAN + /\ commitAllowed \subseteq Snapshots /\ messages \subseteq Message VisibleIsAnExactCommittedSnapshot == @@ -166,15 +212,13 @@ VisibleIsAnExactCommittedSnapshot == /\ \/ /\ cursor = 0 /\ visible = {} \/ /\ cursor = 1 /\ visible = SnapshotRows(3) -StagingNeverLeaksIntoVisible == - stagedSnapshot # 0 /\ stagedChunks # Chunks => - VisibleIsAnExactCommittedSnapshot - StagingBelongsToOneSnapshot == stagedSnapshot # 0 => - /\ stagedRows = - UNION {ChunkRows(stagedSnapshot, chunk) : chunk \in stagedChunks} - /\ stagedChunks # Chunks + stagedRows = UNION {ChunkRows(stagedSnapshot, chunk) : chunk \in stagedChunks} + +NoCommitMeansNoInstall == + stagedSnapshot # 0 /\ ~stagedCommitted => + SnapshotSeq(stagedSnapshot) > cursor Spec == Init /\ [][Next]_vars diff --git a/test/jepsen/node.exs b/test/jepsen/node.exs index efe370d..6920da3 100644 --- a/test/jepsen/node.exs +++ b/test/jepsen/node.exs @@ -96,11 +96,15 @@ defmodule Group.Jepsen.Transport.Common do end end - def record({:snapshot_chunk, _version, _stream, _seq, _index, chunk_count, _, _, _, _}) do + def record({:snapshot_chunk, _version, _stream, _seq, _index, _registry, _pg}) do Stats.increment(:snapshot_chunk) + end + + def record({:snapshot_commit, _version, _stream, _seq, chunk_count, _, _}) do + Stats.increment(:snapshot_commit) if chunk_count > 1 do - Stats.increment(:multi_chunk_snapshot_chunk) + Stats.increment(:multi_chunk_snapshot) end end diff --git a/test/mutation/README.md b/test/mutation/README.md index e1a1944..3eb0d6d 100644 --- a/test/mutation/README.md +++ b/test/mutation/README.md @@ -6,11 +6,12 @@ contiguous sequence application, exact registry and PG snapshots, below-floor repair, process-down sequencing, conflict-loser retirement, authority fanout, per-lane authority installation, periodic head advertisement, interrupted journal/index repair, and named-cluster close completion. Snapshot calibration -also covers incomplete commit, conflicting retransmission rows, newer-snapshot +also covers missing/incomplete terminal commit, conflicting retransmission rows +and manifests, source mutation during a single-pass scan, newer-snapshot supersession, stale-authority fencing, and staging expiry. Restart calibration covers per-lane eviction breadcrumbs and partially observed authority, while wire calibration rejects wrong-shard rows and unsequenced -cluster lifecycle messages. The 64-mutant campaign also independently removes +cluster lifecycle messages. The 68-mutant campaign also independently removes the generation and epoch fences, races authority changes against local-owner retirement, skips conflict reprojection after exact authority returns, bypasses shard-zero authority serialization, separates exact authority from its shared diff --git a/test/mutation/run.exs b/test/mutation/run.exs index 9341093..18eb1e7 100644 --- a/test/mutation/run.exs +++ b/test/mutation/run.exs @@ -86,33 +86,82 @@ defmodule Group.MutationCampaign do %{ name: "commit_incomplete_snapshot", file: "lib/group/replica.ex", + correct_source: + " if MapSet.size(transfer.received) == chunk_count and\n" <> + " transfer.registry_seen == registry_count and transfer.pg_seen == pg_count do", + faulty_source: + " if MapSet.size(transfer.received) >= 1 and chunk_count >= 1 and\n" <> + " registry_count >= 0 and pg_count >= 0 do", + test: ["test/replica_snapshot_distributed_test.exs:177"] + }, + %{ + name: "commit_snapshot_without_terminal_manifest", + file: "lib/group/replica.ex", + correct_source: + " nil ->\n" <> + " state\n" <> + " end\n" <> + " end\n\n" <> + " defp snapshot_chunk_within_manifest?", + faulty_source: + " nil ->\n" <> + " chunk_count = MapSet.size(transfer.received)\n" <> + " transfer = %{transfer | manifest: {chunk_count, transfer.registry_seen, transfer.pg_seen}}\n" <> + " commit_snapshot_transfer(state, key, source_node, stream_id, transfer)\n" <> + " end\n" <> + " end\n\n" <> + " defp snapshot_chunk_within_manifest?", + test: ["test/replica_snapshot_distributed_test.exs:16"] + }, + %{ + name: "accept_conflicting_snapshot_chunk_retransmission", + file: "lib/group/replica.ex", + correct_source: + " MapSet.member?(transfer.received, chunk_index) and\n" <> + " Snapshot.chunk_matches?(transfer.table, chunk_index, reg_data, pg_data) ->", + faulty_source: + " MapSet.member?(transfer.received, chunk_index) and\n" <> + " Process.alive?(self()) ->", + test: ["test/replica_snapshot_distributed_test.exs:339"] + }, + %{ + name: "retain_conflicting_snapshot_manifest", + file: "lib/group/replica.ex", correct_source: """ - if MapSet.size(transfer.received) == transfer.chunk_count do - if transfer.registry_seen == transfer.registry_count and - transfer.pg_seen == transfer.pg_count do - commit_snapshot_transfer(state, key, source_node, stream_id, transfer) - else + {:ok, state, _conflicting_transfer} -> discard_snapshot_transfer(state, key) - end - else - state - end """, faulty_source: """ - if MapSet.size(transfer.received) >= 1 do - commit_snapshot_transfer(state, key, source_node, stream_id, transfer) - else - state - end + {:ok, state, _conflicting_transfer} -> + state """, - test: ["test/replica_snapshot_distributed_test.exs:16"] + test: ["test/replica_snapshot_distributed_test.exs:145"] + }, + %{ + name: "commit_snapshot_after_source_changes_during_scan", + file: "lib/group/replica.ex", + correct_source: + " if current_snapshot_send?(state, target_node, stream_id, head) do\n" <> + " send_replica_snapshot_commit(state, target_node, stream_id, head, manifest)\n" <> + " else\n" <> + " :complete\n" <> + " end\n" <> + " catch", + faulty_source: + " if Process.alive?(self()) do\n" <> + " send_replica_snapshot_commit(state, target_node, stream_id, head, manifest)\n" <> + " else\n" <> + " :complete\n" <> + " end\n" <> + " catch", + test: ["test/replica_snapshot_distributed_test.exs:55"] }, %{ name: "drop_final_snapshot_event_batch", file: "lib/group/replica.ex", correct_source: " _event_buffer = Snapshot.finish_event_buffer(event_buffer)", faulty_source: " _event_buffer = event_buffer", - test: ["test/replica_snapshot_distributed_test.exs:16"] + test: ["test/replica_snapshot_distributed_test.exs:177"] }, %{ name: "allow_duplicate_snapshot_rows", @@ -124,23 +173,21 @@ defmodule Group.MutationCampaign do faulty_source: """ if :ets.insert(table, objects) and size_before >= 0 do """, - test: ["test/replica_snapshot_distributed_test.exs:178"] + test: ["test/replica_snapshot_distributed_test.exs:339"] }, %{ name: "do_not_supersede_partial_snapshot", file: "lib/group/replica.ex", - correct_source: """ - %{snapshot_seq: existing_seq} when existing_seq < snapshot_seq -> - state = discard_snapshot_transfer(state, key) - transfer = new_snapshot_transfer(snapshot_seq, manifest) - {:ok, put_snapshot_transfer(state, key, transfer), transfer} - """, - faulty_source: """ - %{snapshot_seq: existing_seq} when existing_seq < snapshot_seq -> - _ = existing_seq - {:ignore, state} - """, - test: ["test/replica_snapshot_distributed_test.exs:121"] + correct_source: + " %{snapshot_seq: existing_seq} when existing_seq < snapshot_seq ->\n" <> + " state = discard_snapshot_transfer(state, key)\n" <> + " {state, transfer} = new_snapshot_transfer(state, snapshot_seq)\n" <> + " {:ok, put_snapshot_transfer(state, key, transfer), transfer}", + faulty_source: + " %{snapshot_seq: existing_seq} when existing_seq < snapshot_seq ->\n" <> + " _ = existing_seq\n" <> + " {:ignore, state}", + test: ["test/replica_snapshot_distributed_test.exs:282"] }, %{ name: "accept_stale_snapshot_authority", @@ -156,7 +203,7 @@ defmodule Group.MutationCampaign do snapshot_seq > Data.replica_cursor(state.name, state.shard_index, stream_id) end """, - test: ["test/replica_snapshot_distributed_test.exs:328"] + test: ["test/replica_snapshot_distributed_test.exs:510"] }, %{ name: "disable_snapshot_staging_expiry", @@ -177,7 +224,7 @@ defmodule Group.MutationCampaign do acc end """, - test: ["test/replica_snapshot_distributed_test.exs:254"] + test: ["test/replica_snapshot_distributed_test.exs:436"] }, %{ name: "disable_below_floor_snapshot", @@ -220,7 +267,7 @@ defmodule Group.MutationCampaign do file: "lib/group/replica/data.ex", correct_source: " hint_generation == generation and\n", faulty_source: " false and hint_generation == generation and\n", - test: ["test/anti_entropy_fault_regression_test.exs:2058"] + test: ["test/anti_entropy_fault_regression_test.exs:2059"] }, %{ name: "heartbeat_does_not_fence_newer_generation", @@ -232,7 +279,7 @@ defmodule Group.MutationCampaign do " not is_nil(hint_generation) and\n" <> " WireProtocol.generation_newer?(generation, hint_generation) and\n" <> " Process.get(:fence_newer_generation, false) ->\n", - test: ["test/anti_entropy_fault_regression_test.exs:2221"] + test: ["test/anti_entropy_fault_regression_test.exs:2222"] }, %{ name: "drop_new_generation_authority_hint", @@ -243,7 +290,7 @@ defmodule Group.MutationCampaign do faulty_source: " # below are being updated.\n" <> " _ = {state.name, remote_node, generation, revision}", - test: ["test/anti_entropy_fault_regression_test.exs:2221"] + test: ["test/anti_entropy_fault_regression_test.exs:2222"] }, %{ name: "accept_authority_older_than_generation_hint", @@ -252,7 +299,7 @@ defmodule Group.MutationCampaign do faulty_source: " _ = hinted_stale?\n" <> " known_stale? or revision_stale?", - test: ["test/anti_entropy_fault_regression_test.exs:2221"] + test: ["test/anti_entropy_fault_regression_test.exs:2222"] }, %{ name: "install_lane_view_behind_generation_hint", @@ -283,7 +330,7 @@ defmodule Group.MutationCampaign do faulty_source: " (is_nil(hint_generation) or\n" <> " WireProtocol.generation_newer?(generation, hint_generation)) ->\n", - test: ["test/anti_entropy_fault_regression_test.exs:3317"] + test: ["test/anti_entropy_fault_regression_test.exs:3318"] }, %{ name: "admit_retired_lane_route_without_authority", @@ -296,7 +343,7 @@ defmodule Group.MutationCampaign do faulty_source: " state = put_remote_shard(state, remote_node, remote_pid)\n" <> " {:noreply, request_replica_authority(state, remote_node)}", - test: ["test/anti_entropy_fault_regression_test.exs:3317"] + test: ["test/anti_entropy_fault_regression_test.exs:3318"] }, %{ name: "do_not_restore_hint_lease_after_lane_restart", @@ -321,7 +368,7 @@ defmodule Group.MutationCampaign do " is_nil(Data.remote_generation(state.name, remote_node)) and\n" <> " is_nil(Data.remote_replica_authority_hint(state.name, remote_node)) ->\n" <> " Map.put(acc, remote_node, last_activity)\n", - test: ["test/anti_entropy_fault_regression_test.exs:3317"] + test: ["test/anti_entropy_fault_regression_test.exs:3318"] }, %{ name: "skip_authority_fanout", @@ -363,7 +410,7 @@ defmodule Group.MutationCampaign do state end """, - test: ["test/anti_entropy_fault_regression_test.exs:3822"] + test: ["test/anti_entropy_fault_regression_test.exs:3823"] }, %{ name: "assume_authority_fanout_reaches_late_lane", @@ -393,7 +440,7 @@ defmodule Group.MutationCampaign do state end """, - test: ["test/replica_snapshot_distributed_test.exs:553"] + test: ["test/replica_snapshot_distributed_test.exs:680"] }, %{ name: "skip_generation_purge", @@ -527,7 +574,7 @@ defmodule Group.MutationCampaign do faulty_source: " WireProtocol.stream_shard(stream_id) == state.shard_index and\n" <> " true and", - test: ["test/anti_entropy_fault_regression_test.exs:1172"] + test: ["test/anti_entropy_fault_regression_test.exs:1173"] }, %{ name: "apply_incremental_authority_across_revision_gap", @@ -535,21 +582,21 @@ defmodule Group.MutationCampaign do correct_source: " if contiguous_cluster_controls?(accepted, next_revision) do", faulty_source: " if contiguous_cluster_controls?(accepted, next_revision) or accepted != [] do", - test: ["test/anti_entropy_fault_regression_test.exs:1364"] + test: ["test/anti_entropy_fault_regression_test.exs:1365"] }, %{ name: "allow_non_owner_lane_to_mutate_shared_authority", file: "lib/group/replica.ex", correct_source: " if state.shard_index == 0 do\n remote_node = node(remote_pid)", faulty_source: " if true do\n remote_node = node(remote_pid)", - test: ["test/anti_entropy_fault_regression_test.exs:1364"] + test: ["test/anti_entropy_fault_regression_test.exs:1365"] }, %{ name: "crash_lane_when_local_authority_owner_is_missing", file: "lib/group/replica.ex", correct_source: " _ = send_local_control_message(state, control)", faulty_source: " send(shard_name(state.name, 0), control)", - test: ["test/anti_entropy_fault_regression_test.exs:1311"] + test: ["test/anti_entropy_fault_regression_test.exs:1312"] }, %{ name: "retire_local_owner_after_remote_authority_changed", @@ -558,7 +605,7 @@ defmodule Group.MutationCampaign do faulty_source: " Process.get(:skip_remote_registry_authority, true) or\n" <> " registry_winner_authoritative?(state, cluster, winner) ->", - test: ["test/anti_entropy_fault_regression_test.exs:1523"] + test: ["test/anti_entropy_fault_regression_test.exs:1524"] }, %{ name: "skip_registry_reprojection_after_authority_restore", @@ -583,7 +630,7 @@ defmodule Group.MutationCampaign do state end """, - test: ["test/anti_entropy_fault_regression_test.exs:1706"] + test: ["test/anti_entropy_fault_regression_test.exs:1707"] }, %{ name: "retain_registry_reprojection_after_peer_expiry", @@ -599,7 +646,7 @@ defmodule Group.MutationCampaign do state = discard_snapshot_transfers_for_source(state, remote_node) state = discard_snapshot_send_offsets_for_target(state, remote_node) """, - test: ["test/anti_entropy_fault_regression_test.exs:1882"] + test: ["test/anti_entropy_fault_regression_test.exs:1883"] }, %{ name: "retain_registry_reprojection_after_nodedown", @@ -617,7 +664,7 @@ defmodule Group.MutationCampaign do state = discard_snapshot_transfers_for_source(state, dead_node) state = discard_snapshot_send_offsets_for_target(state, dead_node) """, - test: ["test/anti_entropy_fault_regression_test.exs:3600"] + test: ["test/anti_entropy_fault_regression_test.exs:3601"] }, %{ name: "separate_exact_authority_from_cluster_projection", @@ -626,7 +673,7 @@ defmodule Group.MutationCampaign do " replace_remote_cluster_projection(state.name, remote_node, current_epochs)\n", faulty_source: " _ = {&replace_remote_cluster_projection/3, state.name, remote_node, current_epochs}\n", - test: ["test/anti_entropy_fault_regression_test.exs:3636"] + test: ["test/anti_entropy_fault_regression_test.exs:3637"] }, %{ name: "separate_local_activation_from_cluster_projection", @@ -635,7 +682,7 @@ defmodule Group.MutationCampaign do " if durable?, do: project_activated_local_clusters(state.name, clusters)\n", faulty_source: " _ = {durable?, &project_activated_local_clusters/2, state.name, clusters}\n", - test: ["test/anti_entropy_fault_regression_test.exs:3691"] + test: ["test/anti_entropy_fault_regression_test.exs:3692"] }, %{ name: "drop_durable_cluster_deactivation_cleanup", @@ -648,7 +695,7 @@ defmodule Group.MutationCampaign do " )\n", faulty_source: " _ = {&cast_cluster_lifecycle/3, state.name, state.num_shards, clusters, epochs}\n", - test: ["test/anti_entropy_fault_regression_test.exs:3740"] + test: ["test/anti_entropy_fault_regression_test.exs:3741"] }, %{ name: "delete_close_marker_before_terminal_route_cleanup", @@ -710,14 +757,14 @@ defmodule Group.MutationCampaign do faulty_source: " WireProtocol.stream_origin(stream_id) != node() and\n" <> " true and", - test: ["test/anti_entropy_fault_regression_test.exs:2735"] + test: ["test/anti_entropy_fault_regression_test.exs:2736"] }, %{ name: "retire_shared_authority_with_live_lanes", file: "lib/group/replica/data.ex", correct_source: " result =\n if remaining_lanes == 0 do", faulty_source: " _ = remaining_lanes\n\n result =\n if true do", - test: ["test/anti_entropy_fault_regression_test.exs:856"] + test: ["test/anti_entropy_fault_regression_test.exs:857"] }, %{ name: "shard_zero_deletes_sibling_restart_views", @@ -780,7 +827,7 @@ defmodule Group.MutationCampaign do " Enum.split_while(contiguous, fn {_seq, mutations} ->\n" <> " valid_replica_mutations?(%{state | num_shards: 1}, stream_id, mutations)\n" <> " end)", - test: ["test/anti_entropy_fault_regression_test.exs:511"] + test: ["test/anti_entropy_fault_regression_test.exs:512"] }, %{ name: "restore_unsequenced_cluster_disconnect", @@ -827,7 +874,7 @@ defmodule Group.MutationCampaign do " if Process.get(:run_primary_replica_repair, false),\n" <> " do: repair_primary_replica_rows(name, shard),\n" <> " else: :ok", - test: ["test/anti_entropy_fault_regression_test.exs:3025"] + test: ["test/anti_entropy_fault_regression_test.exs:3026"] }, %{ name: "project_stale_claims_before_restart_repair", @@ -842,7 +889,7 @@ defmodule Group.MutationCampaign do {state, _events} = rebuild_registry_projections(state) :ok = Data.repair_shard_indexes(name, shard_index) """, - test: ["test/anti_entropy_fault_regression_test.exs:3457"] + test: ["test/anti_entropy_fault_regression_test.exs:3458"] }, %{ name: "skip_interrupted_snapshot_install_repair", @@ -852,7 +899,7 @@ defmodule Group.MutationCampaign do " if Process.get(:run_snapshot_install_repair, false),\n" <> " do: repair_interrupted_snapshot_installs(name, shard),\n" <> " else: :ok", - test: ["test/anti_entropy_fault_regression_test.exs:3137"] + test: ["test/anti_entropy_fault_regression_test.exs:3138"] }, %{ name: "retain_cursorless_remote_registry_claims", @@ -861,14 +908,20 @@ defmodule Group.MutationCampaign do " :ets.member(replica_cursor_table(name, shard), stream_id)\n else\n false\n end\n end\n\n defp valid_remote_pg_authority?", faulty_source: " is_tuple(stream_id)\n else\n false\n end\n end\n\n defp valid_remote_pg_authority?", - test: ["test/anti_entropy_fault_regression_test.exs:3025"] + test: ["test/anti_entropy_fault_regression_test.exs:3026"] }, %{ name: "restart_snapshot_from_first_chunk_after_busy", file: "lib/group/replica.ex", - correct_source: " start_index = Map.get(offsets, snapshot_key, 1)", - faulty_source: " _ = {offsets, snapshot_key}\n start_index = 1", - test: ["test/replica_snapshot_distributed_test.exs:649"] + correct_source: " resume = Map.get(offsets, snapshot_key, {:chunk, 1})", + faulty_source: """ + resume = + case Map.get(offsets, snapshot_key) do + {:commit, _manifest} = commit -> commit + _chunk_resume -> {:chunk, 1} + end + """, + test: ["test/replica_snapshot_distributed_test.exs:831"] }, %{ name: "drain_oversized_ingress_batch_without_yield", diff --git a/test/replica_snapshot_distributed_test.exs b/test/replica_snapshot_distributed_test.exs index 3b32c5d..ae2691a 100644 --- a/test/replica_snapshot_distributed_test.exs +++ b/test/replica_snapshot_distributed_test.exs @@ -13,6 +13,167 @@ defmodule Group.ReplicaSnapshotDistributedTest do {:ok, node_a: node_a, node_b: node_b, node_c: node_c} end + test "complete provisional chunks expose nothing until terminal commit", context do + %{name: name, node_a: node_a, node_b: node_b} = start_pair(context) + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + entries = + for index <- 1..8 do + key = "snapshot/terminal-commit/#{index}" + + {key, + TestCluster.spawn_register(node_a, name, key, %{ + payload: String.duplicate("c", 160) + })} + end + + TestCluster.flush_shards(node_a, name) + stream_id = local_stream(node_a, name, nil) + {chunks, commit} = capture_snapshot_with_commit(node_a, node_b, name, stream_id, 1) + assert length(chunks) > 1 + + deliver_frames(node_b, node_a, name, Enum.reverse(chunks)) + TestCluster.flush_shards(node_b, name) + + assert replica_cursor(node_b, name, stream_id) == 0 + + assert Enum.all?(entries, fn {key, _pid} -> + TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + end) + + deliver_frames(node_b, node_a, name, [commit]) + TestCluster.flush_shards(node_b, name) + snapshot_seq = elem(commit, 3) + + assert replica_cursor(node_b, name, stream_id) == snapshot_seq + + assert Enum.all?(entries, fn {key, pid} -> + match?({^pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + end + + test "a source mutation during a single-pass scan prevents terminal commit", context do + %{name: name, node_a: node_a, node_b: node_b} = start_pair(context) + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + entries = + for index <- 1..8 do + key = "snapshot/concurrent-write/#{index}" + + {key, + TestCluster.spawn_register(node_a, name, key, %{ + payload: String.duplicate("s", 160) + })} + end + + TestCluster.flush_shards(node_a, name) + stream_id = local_stream(node_a, name, nil) + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :clear_captured, [name]) + + :ok = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [ + name, + {:capture_drop_pause_once, [:snapshot_chunk, :snapshot_commit], self()} + ]) + + :ok = + TestCluster.rpc!(node_a, Group.Transport, :incoming, [ + name, + node_b, + 0, + {:need, Group.Replica.WireProtocol.version(), stream_id, 1} + ]) + + assert_receive {:replica_transport_paused, worker, ^name, :snapshot_chunk}, 5_000 + + extra_key = "snapshot/concurrent-write/after-scan-started" + extra_pid = TestCluster.spawn_register(node_a, name, extra_key, %{after_start: true}) + TestCluster.flush_shards(node_a, name) + send(worker, {:resume_replica_transport, name}) + + source = TestCluster.rpc!(node_a, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(node_a, :sys, :get_state, [source]).snapshot_send == nil + end) + + captured = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :captured, [name]) + + chunks = + Enum.flat_map(captured, fn + {^node_b, 0, {:snapshot_chunk, _, ^stream_id, _, _, _, _} = chunk} -> [chunk] + _other -> [] + end) + + assert chunks != [] + + refute Enum.any?(captured, fn + {^node_b, 0, {:snapshot_commit, _, ^stream_id, _, _, _, _}} -> true + _other -> false + end) + + deliver_frames(node_b, node_a, name, chunks) + assert replica_cursor(node_b, name, stream_id) == 0 + + assert Enum.all?(entries, fn {key, _pid} -> + TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + end) + + assert TestCluster.rpc!(node_b, Group, :lookup, [name, extra_key]) == nil + + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :pass]) + + :ok = + TestCluster.rpc!(node_a, Group.Transport, :incoming, [ + name, + node_b, + 0, + {:need, Group.Replica.WireProtocol.version(), stream_id, 1} + ]) + + TestCluster.assert_eventually(fn -> + Enum.all?(entries, fn {key, pid} -> + match?({^pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) and + match?( + {^extra_pid, _}, + TestCluster.rpc!(node_b, Group, :lookup, [name, extra_key]) + ) + end) + end + + test "conflicting terminal manifests discard the candidate instead of manufacturing commit", + context do + %{name: name, node_a: node_a, node_b: node_b} = start_pair(context) + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + entries = + for index <- 1..8 do + key = "snapshot/conflicting-commit/#{index}" + {key, TestCluster.spawn_register(node_a, name, key, %{index: index})} + end + + TestCluster.flush_shards(node_a, name) + stream_id = local_stream(node_a, name, nil) + {chunks, commit} = capture_snapshot_with_commit(node_a, node_b, name, stream_id, 1) + conflicting_commit = put_elem(commit, 5, elem(commit, 5) + 1) + + deliver_frames(node_b, node_a, name, chunks ++ [conflicting_commit]) + assert replica_cursor(node_b, name, stream_id) == 0 + assert snapshot_transfer_count(node_b, name) == 1 + + deliver_frames(node_b, node_a, name, [commit]) + assert replica_cursor(node_b, name, stream_id) == 0 + assert snapshot_transfer_count(node_b, name) == 0 + + deliver_frames(node_b, node_a, name, chunks ++ [commit]) + assert replica_cursor(node_b, name, stream_id) == elem(commit, 3) + + assert Enum.all?(entries, fn {key, pid} -> + match?({^pid, _}, TestCluster.rpc!(node_b, Group, :lookup, [name, key])) + end) + end + test "loss, reordering, and duplication expose nothing until exact commit", context do %{name: name, node_a: node_a, node_b: node_b} = start_pair(context) @@ -191,20 +352,31 @@ defmodule Group.ReplicaSnapshotDistributedTest do TestCluster.flush_shards(node_a, name) stream_id = local_stream(node_a, name, nil) - [first, second | rest] = frames = capture_snapshot(node_a, node_b, name, stream_id, 1) - [first_row | _] = elem(first, 8) - [_second_row | second_tail] = elem(second, 8) - conflicting_second = put_elem(second, 8, [first_row | second_tail]) + [first, second | rest] = capture_snapshot(node_a, node_b, name, stream_id, 1) + [first_row | first_tail] = elem(first, 5) + [second_row | second_tail] = elem(second, 5) + conflicting_first = put_elem(first, 5, [second_row | first_tail]) + conflicting_second = put_elem(second, 5, [first_row | second_tail]) - deliver_frames(node_b, node_a, name, [first, conflicting_second | rest]) + deliver_frames(node_b, node_a, name, [first, conflicting_first]) + assert replica_cursor(node_b, name, stream_id) == 0 + assert snapshot_transfer_count(node_b, name) == 0 + + assert Enum.all?(entries, fn {key, _pid} -> + TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + end) + + # A different chunk index cannot reuse an identity already staged by the + # first chunk to satisfy the terminal row count while omitting another row. + deliver_frames(node_b, node_a, name, [first, conflicting_second | rest]) assert replica_cursor(node_b, name, stream_id) == 0 assert Enum.all?(entries, fn {key, _pid} -> TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil end) - deliver_frames(node_b, node_a, name, frames) + deliver_frames(node_b, node_a, name, [first, second | rest]) assert replica_cursor(node_b, name, stream_id) == elem(first, 3) @@ -213,7 +385,7 @@ defmodule Group.ReplicaSnapshotDistributedTest do end) end - test "rejected first chunks do not leak their private staging tables", context do + test "rejected first chunks clear and reuse their private staging tables", context do %{name: name, node_a: node_a, node_b: node_b} = start_pair(context) :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) @@ -229,26 +401,36 @@ defmodule Group.ReplicaSnapshotDistributedTest do rows = frames - |> Enum.flat_map(&elem(&1, 8)) + |> Enum.filter(&(elem(&1, 0) == :snapshot_chunk)) + |> Enum.flat_map(&elem(&1, 5)) |> Enum.take(2) assert [first_row, second_row] = rows - {:snapshot_chunk, version, ^stream_id, snapshot_seq, _, _, _, _, _, _} = hd(frames) + {:snapshot_chunk, version, ^stream_id, snapshot_seq, _, _, _} = hd(frames) assert snapshot_staging_tables(node_b, name) == [] + commit = {:snapshot_commit, version, stream_id, snapshot_seq, 1, 1, 0} + deliver_frames(node_b, node_a, name, [commit]) + assert snapshot_transfer_count(node_b, name) == 1 + first_tables = snapshot_transfer_tables(node_b, name) + overflow = - {:snapshot_chunk, version, stream_id, snapshot_seq, 1, 2, 1, 1, [first_row, second_row], []} + {:snapshot_chunk, version, stream_id, snapshot_seq, 1, [first_row, second_row], []} deliver_frames(node_b, node_a, name, [overflow]) assert snapshot_transfer_count(node_b, name) == 0 assert snapshot_staging_tables(node_b, name) == [] duplicate = - {:snapshot_chunk, version, stream_id, snapshot_seq, 1, 2, 2, 0, [first_row, first_row], []} + {:snapshot_chunk, version, stream_id, snapshot_seq, 1, [first_row, first_row], []} deliver_frames(node_b, node_a, name, [duplicate]) assert snapshot_transfer_count(node_b, name) == 0 assert snapshot_staging_tables(node_b, name) == [] + + deliver_frames(node_b, node_a, name, [hd(frames)]) + assert snapshot_transfer_count(node_b, name) == 1 + assert snapshot_transfer_tables(node_b, name) == first_tables end test "an incomplete current-authority snapshot expires without touching visible state", @@ -672,17 +854,25 @@ defmodule Group.ReplicaSnapshotDistributedTest do TestCluster.flush_shards(node_a, name) stream_id = local_stream(node_a, name, nil) - request_snapshot_window(node_a, node_b, name, stream_id, 2) + {chunk_count, registry_count, pg_count} = + drive_snapshot_chunks_until_commit_pending(node_a, node_b, name, stream_id, 100) + TestCluster.flush_shards(node_b, name) + {received, manifest} = snapshot_transfer_progress(node_b, name) + assert received == chunk_count + assert manifest == nil + assert registry_count == length(entries) + assert pg_count == 0 + assert replica_cursor(node_b, name, stream_id) == 0 - {received, chunk_count} = snapshot_transfer_progress(node_b, name) - assert received == 2 - assert chunk_count > received + assert Enum.all?(entries, fn {key, _pid} -> + TestCluster.rpc!(node_b, Group, :lookup, [name, key]) == nil + end) - for _attempt <- 2..ceil_div(chunk_count, 2) do - request_snapshot_window(node_a, node_b, name, stream_id, 2) - TestCluster.flush_shards(node_b, name) - end + # The sender retained only the tiny terminal manifest after backpressure. + # Its next repair attempt sends that commit directly without rescanning or + # retransmitting the already staged chunks. + request_snapshot_window(node_a, node_b, name, stream_id, 1) TestCluster.assert_eventually( fn -> @@ -742,7 +932,7 @@ defmodule Group.ReplicaSnapshotDistributedTest do :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [ name, - {:capture_drop, [:snapshot_chunk]} + {:capture_drop, [:snapshot_chunk, :snapshot_commit]} ]) :ok = @@ -760,33 +950,51 @@ defmodule Group.ReplicaSnapshotDistributedTest do snapshot_send = TestCluster.rpc!(node_a, :sys, :get_state, [Group.Replica.shard_name(name, 0)]).snapshot_send - Enum.any?(captured, fn - {^node_b, 0, {:snapshot_chunk, _, ^stream_id, _, _, _, _, _, _, _}} -> true - _ -> false - end) and is_nil(snapshot_send) + has_chunk? = + Enum.any?(captured, fn + {^node_b, 0, {:snapshot_chunk, _, ^stream_id, _, _, _, _}} -> true + _ -> false + end) + + has_commit? = + Enum.any?(captured, fn + {^node_b, 0, {:snapshot_commit, _, ^stream_id, _, _, _, _}} -> true + _ -> false + end) + + has_chunk? and has_commit? and is_nil(snapshot_send) end, timeout: 5_000, interval: 10 ) - TestCluster.rpc!(node_a, Group.TestReplicaTransport, :captured, [name]) - |> Enum.flat_map(fn - {^node_b, 0, - {:snapshot_chunk, _version, ^stream_id, _seq, _index, _count, _reg_count, _pg_count, _reg, - _pg} = frame} -> - [frame] + messages = + TestCluster.rpc!(node_a, Group.TestReplicaTransport, :captured, [name]) + |> Enum.flat_map(fn + {^node_b, 0, {:snapshot_chunk, _, ^stream_id, _, _, _, _} = frame} -> [frame] + {^node_b, 0, {:snapshot_commit, _, ^stream_id, _, _, _, _} = frame} -> [frame] + _other -> [] + end) + + chunks = + messages + |> Enum.filter(&(elem(&1, 0) == :snapshot_chunk)) + |> Enum.sort_by(&elem(&1, 4)) + + [commit] = Enum.filter(messages, &(elem(&1, 0) == :snapshot_commit)) + chunks ++ [commit] + end - _other -> - [] - end) - |> Enum.sort_by(&elem(&1, 4)) + defp capture_snapshot_with_commit(node_a, node_b, name, stream_id, next_seq) do + frames = capture_snapshot(node_a, node_b, name, stream_id, next_seq) + {Enum.drop(frames, -1), List.last(frames)} end defp request_snapshot_window(node_a, node_b, name, stream_id, limit) do :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [ name, - {:accept_types_up_to, [:snapshot_chunk], limit} + {:accept_types_up_to, [:snapshot_chunk, :snapshot_commit], limit} ]) :ok = @@ -814,10 +1022,37 @@ defmodule Group.ReplicaSnapshotDistributedTest do TestCluster.rpc!(node, :sys, :get_state, [Group.Replica.shard_name(name, 0)]) [{_key, transfer}] = Map.to_list(state.snapshot_transfers) - {MapSet.size(transfer.received), transfer.chunk_count} + {MapSet.size(transfer.received), transfer.manifest} end - defp ceil_div(value, divisor), do: div(value + divisor - 1, divisor) + defp drive_snapshot_chunks_until_commit_pending( + node_a, + node_b, + name, + stream_id, + attempts_left + ) + when attempts_left > 0 do + request_snapshot_window(node_a, node_b, name, stream_id, 1) + TestCluster.flush_shards(node_b, name) + + state = + TestCluster.rpc!(node_a, :sys, :get_state, [Group.Replica.shard_name(name, 0)]) + + case Map.values(state.snapshot_send_offsets) do + [{:commit, manifest}] -> + manifest + + [{:chunk, _next_index}] -> + drive_snapshot_chunks_until_commit_pending( + node_a, + node_b, + name, + stream_id, + attempts_left - 1 + ) + end + end defp deliver_frames(node_b, node_a, name, frames) do Enum.each(frames, fn frame -> @@ -847,6 +1082,14 @@ defmodule Group.ReplicaSnapshotDistributedTest do ]) end + defp snapshot_transfer_tables(node, name) do + state = + TestCluster.rpc!(node, :sys, :get_state, [Group.Replica.shard_name(name, 0)]) + + [{_key, transfer}] = Map.to_list(state.snapshot_transfers) + {transfer.table, transfer.events} + end + defp snapshot_staging_tables(node, name) do TestCluster.rpc!(node, TestCluster, :snapshot_staging_tables, [name, 0]) end diff --git a/test/replica_snapshot_test.exs b/test/replica_snapshot_test.exs index 52af1a6..957b232 100644 --- a/test/replica_snapshot_test.exs +++ b/test/replica_snapshot_test.exs @@ -3,7 +3,7 @@ defmodule Group.ReplicaSnapshotTest do alias Group.Replica.Snapshot - test "partitions a complete exact slice into byte-bounded deterministic chunks" do + test "streams a complete exact slice in byte-bounded chunks without retaining prior chunks" do pid = self() metadata = %{payload: String.duplicate("x", 96)} @@ -19,42 +19,62 @@ defmodule Group.ReplicaSnapshotTest do target = 2_048 stream_id = {:group, node(), make_ref(), 0, nil, make_ref()} - envelope = Snapshot.frame_envelope_bytes(stream_id, 123, 80, 80) - - snapshot = - Snapshot.chunk_rows(Enum.reverse(registry_rows), Enum.reverse(pg_rows), target, envelope) - - assert snapshot.registry_count == 80 - assert snapshot.pg_count == 80 - assert length(snapshot.chunks) > 1 - - assert snapshot.chunks == - Snapshot.chunk_rows(registry_rows, pg_rows, target, envelope).chunks - - assert snapshot.chunks |> Enum.flat_map(&elem(&1, 0)) |> MapSet.new() == - MapSet.new(registry_rows) - - assert snapshot.chunks |> Enum.flat_map(&elem(&1, 1)) |> MapSet.new() == - MapSet.new(pg_rows) + envelope = Snapshot.stream_envelope_bytes(stream_id, 123) + owner = self() + + emit = fn registry, pg, index -> + send(owner, {:chunk, index, registry, pg}) + :ok + end + + stream = Snapshot.new_stream(target, envelope, 1, emit) + stream = Snapshot.stream_registry_many(registry_rows, stream) + stream = Snapshot.stream_pg_many(pg_rows, stream) + stream = Snapshot.finish_stream(stream) + + assert stream.registry_count == 80 + assert stream.pg_count == 80 + assert stream.chunk_count > 1 + assert stream.registry == [] + assert stream.pg == [] + + chunks = + for index <- 1..stream.chunk_count do + assert_receive {:chunk, ^index, registry, pg} + + {registry, pg} + end - chunk_count = length(snapshot.chunks) + assert chunks |> Enum.flat_map(&elem(&1, 0)) == registry_rows + assert chunks |> Enum.flat_map(&elem(&1, 1)) == pg_rows - Enum.with_index(snapshot.chunks, 1) + Enum.with_index(chunks, 1) |> Enum.each(fn {{registry, pg}, index} -> frame = - {:snapshot_chunk, Group.Replica.WireProtocol.version(), stream_id, 123, index, - chunk_count, snapshot.registry_count, snapshot.pg_count, registry, pg} + {:snapshot_chunk, Group.Replica.WireProtocol.version(), stream_id, 123, index, registry, + pg} assert :erlang.external_size(frame) <= target end) end - test "represents an empty exact slice and permits one intrinsically oversized row" do - assert Snapshot.chunk_rows([], [], 1_024).chunks == [{[], []}] + test "streams an empty exact slice and permits one intrinsically oversized row" do + owner = self() + + emit = fn registry, pg, index -> + send(owner, {:chunk, index, registry, pg}) + :ok + end + + empty = Snapshot.new_stream(1_024, 512, 1, emit) |> Snapshot.finish_stream() + assert empty.chunk_count == 1 + assert_receive {:chunk, 1, [], []} row = {"large", self(), String.duplicate("x", 4_096), 1} - snapshot = Snapshot.chunk_rows([row], [], 1_024) - assert snapshot.chunks == [{[row], []}] + large = Snapshot.new_stream(1_024, 512, 1, emit) + large = Snapshot.stream_registry_many([row], large) |> Snapshot.finish_stream() + assert large.chunk_count == 1 + assert_receive {:chunk, 1, [^row], []} end test "staging is set-valued across chunks and remains private to its owner" do @@ -74,59 +94,48 @@ defmodule Group.ReplicaSnapshotTest do assert :ets.info(table) == :undefined end - test "capture tables emit deterministic chunks with only one bounded chunk on the heap" do - table = Snapshot.new_capture_table() + test "a resumed stream recounts the exact snapshot but emits only the requested suffix" do pid = self() metadata = %{payload: String.duplicate("x", 96)} registry_rows = - for index <- 80..1//-1 do + for index <- 1..80 do {"registry/#{index}", pid, metadata, index} end - pg_rows = - for index <- 80..1//-1 do - {"pg/#{index}", pid, metadata, index} - end - target = 2_048 stream_id = {:group, node(), make_ref(), 0, nil, make_ref()} - envelope = Snapshot.capture_envelope_bytes(stream_id, 123) + envelope = Snapshot.stream_envelope_bytes(stream_id, 123) - capture = Snapshot.new_capture(table, target, envelope) - capture = Snapshot.capture_registry_many(registry_rows, capture) - capture = Snapshot.capture_pg_many(pg_rows, capture) - capture = Snapshot.finish_capture(capture) - chunk_count = capture.chunk_count + emit_all = fn _registry, _pg, index -> + send(self(), {:first_pass, index}) + :ok + end - assert chunk_count > 1 - assert :ets.info(table, :size) == chunk_count - assert capture.registry == [] - assert capture.pg == [] + first = Snapshot.new_stream(target, envelope, 1, emit_all) + first = Snapshot.stream_registry_many(registry_rows, first) |> Snapshot.finish_stream() + assert first.chunk_count > 3 - assert {:ok, chunks} = - Snapshot.reduce_capture_chunks(table, chunk_count, [], fn registry, pg, acc -> - {:cont, [{registry, pg} | acc]} - end) + for index <- 1..first.chunk_count do + assert_receive {:first_pass, ^index} + end - chunks = Enum.reverse(chunks) - assert length(chunks) == chunk_count + emit_suffix = fn registry, pg, index -> + send(self(), {:suffix, index, registry, pg}) + :ok + end - chunks - |> Enum.with_index(1) - |> Enum.each(fn {{registry, pg}, index} -> - frame = - {:snapshot_chunk, Group.Replica.WireProtocol.version(), stream_id, 123, index, - chunk_count, 80, 80, registry, pg} + resumed = Snapshot.new_stream(target, envelope, 3, emit_suffix) + resumed = Snapshot.stream_registry_many(registry_rows, resumed) |> Snapshot.finish_stream() - assert :erlang.external_size(frame) <= target - end) + assert resumed.chunk_count == first.chunk_count + assert resumed.registry_count == 80 + refute_receive {:suffix, 1, _, _} + refute_receive {:suffix, 2, _, _} - assert chunks |> Enum.flat_map(&elem(&1, 0)) |> MapSet.new() == MapSet.new(registry_rows) - assert chunks |> Enum.flat_map(&elem(&1, 1)) |> MapSet.new() == MapSet.new(pg_rows) - assert :ets.info(table, :size) == chunk_count - - assert :ok = Snapshot.delete_staging_table(table) + for index <- 3..resumed.chunk_count do + assert_receive {:suffix, ^index, _, _} + end end test "event buffering preserves every event across bounded ETS chunks" do diff --git a/test/support/test_cluster.ex b/test/support/test_cluster.ex index 62836a3..1bfc9d2 100644 --- a/test/support/test_cluster.ex +++ b/test/support/test_cluster.ex @@ -596,7 +596,9 @@ defmodule Group.TestCluster do owner = Process.whereis(Group.Replica.shard_name(name, shard_index)) :ets.all() - |> Enum.filter(fn table -> :ets.info(table, :owner) == owner end) + |> Enum.filter(fn table -> + :ets.info(table, :owner) == owner and :ets.info(table, :size) > 0 + end) end @doc false diff --git a/test/support/test_replica_transport.ex b/test/support/test_replica_transport.ex index 3f961b0..4821fc6 100644 --- a/test/support/test_replica_transport.ex +++ b/test/support/test_replica_transport.ex @@ -17,6 +17,8 @@ defmodule Group.TestReplicaTransport do (is_tuple(mode) and tuple_size(mode) == 3 and elem(mode, 0) == :delay_types) or (is_tuple(mode) and tuple_size(mode) == 3 and elem(mode, 0) == :accept_types_up_to) or + (is_tuple(mode) and tuple_size(mode) == 3 and + elem(mode, 0) == :capture_drop_pause_once) or (is_tuple(mode) and tuple_size(mode) == 2 and elem(mode, 0) == :chaos) do :persistent_term.put({__MODULE__, group}, mode) @@ -24,6 +26,10 @@ defmodule Group.TestReplicaTransport do :persistent_term.put({__MODULE__, group, :accepted}, 0) end + if is_tuple(mode) and tuple_size(mode) == 3 and elem(mode, 0) == :capture_drop_pause_once do + :persistent_term.erase({__MODULE__, group, :paused}) + end + :ok end @@ -40,6 +46,7 @@ defmodule Group.TestReplicaTransport do :persistent_term.erase({__MODULE__, group}) :persistent_term.erase({__MODULE__, group, :captured}) :persistent_term.erase({__MODULE__, group, :accepted}) + :persistent_term.erase({__MODULE__, group, :paused}) :ok end @@ -97,6 +104,14 @@ defmodule Group.TestReplicaTransport do if message_type(message) in types, do: capture(group, target_node, shard, message) forward(group, target_node, shard, message) + {:capture_drop_pause_once, types, observer} -> + if message_type(message) in types do + capture(group, target_node, shard, message) + pause_once(group, message_type(message), observer) + end + + :ok + {:chaos, opts} -> chaos_forward(group, target_node, shard, message, opts) @@ -105,6 +120,23 @@ defmodule Group.TestReplicaTransport do end end + defp pause_once(group, message_type, observer) do + key = {__MODULE__, group, :paused} + + if :persistent_term.get(key, false) do + :ok + else + :persistent_term.put(key, true) + send(observer, {:replica_transport_paused, self(), group, message_type}) + + receive do + {:resume_replica_transport, ^group} -> :ok + after + 5_000 -> :ok + end + end + end + defp chaos_forward(group, target_node, shard, message, opts) do hash = :erlang.phash2({target_node, shard, message}, 1_000_003) drop_every = Keyword.get(opts, :drop_every, 0) From 0465da83465abd1d045fccba4213a707d363f600 Mon Sep 17 00:00:00 2001 From: Chris McCord Date: Sat, 15 Aug 2026 03:02:42 +0000 Subject: [PATCH 12/16] Batch exact snapshot installation --- lib/group/replica.ex | 181 +++++++++++++++++++++++++--------- lib/group/replica/data.ex | 112 +++++++++++++++++++-- lib/group/replica/snapshot.ex | 24 +++-- 3 files changed, 257 insertions(+), 60 deletions(-) diff --git a/lib/group/replica.ex b/lib/group/replica.ex index d706a10..80ed9da 100644 --- a/lib/group/replica.ex +++ b/lib/group/replica.ex @@ -4016,6 +4016,9 @@ defmodule Group.Replica do fn key, {acc, buffer} -> {acc, events} = reconcile_registry_projection(acc, cluster, key, :reconcile, []) {acc, Snapshot.buffer_events(Enum.reverse(events), buffer)} + end, + fn claims, {acc, buffer} -> + reconcile_registry_snapshot_batch(acc, cluster, stream_id, claims, buffer) end ) @@ -4592,59 +4595,45 @@ defmodule Group.Replica do source_node, cluster, staging_table, - chunk_count, + _chunk_count, event_buffer ) do event_buffer = - Snapshot.fold_pg(staging_table, chunk_count, event_buffer, fn {key, pid, meta, time}, - buffer -> - case Data.pg_lookup(state.name, state.shard_index, cluster, key, pid) do - nil -> - :ok = - Data.pg_insert( - state.name, - state.shard_index, - cluster, - key, - pid, - meta, - time, - source_node - ) - - Snapshot.buffer_event( - build_event(state.name, :joined, key, pid, meta, %{cluster: cluster}), - buffer - ) - - {^meta, ^time, ^source_node} -> - buffer + Snapshot.reduce_pg_batches(staging_table, event_buffer, fn rows, buffer -> + {inserts, buffer} = + Enum.reduce(rows, {[], buffer}, fn {key, pid, meta, time}, {inserts, inner} -> + case Data.pg_lookup(state.name, state.shard_index, cluster, key, pid) do + nil -> + event = build_event(state.name, :joined, key, pid, meta, %{cluster: cluster}) - {old_meta, _old_time, ^source_node} -> - :ok = - Data.pg_insert( - state.name, - state.shard_index, - cluster, - key, - pid, - meta, - time, - source_node - ) + { + [{cluster, key, pid, meta, time, source_node} | inserts], + Snapshot.buffer_event(event, inner) + } - if old_meta != meta do - Snapshot.buffer_event( - build_event(state.name, :joined, key, pid, meta, %{ - previous_meta: old_meta, - cluster: cluster - }), - buffer - ) - else - buffer + {^meta, ^time, ^source_node} -> + {inserts, inner} + + {old_meta, _old_time, ^source_node} -> + inner = + if old_meta != meta do + Snapshot.buffer_event( + build_event(state.name, :joined, key, pid, meta, %{ + previous_meta: old_meta, + cluster: cluster + }), + inner + ) + else + inner + end + + {[{cluster, key, pid, meta, time, source_node} | inserts], inner} end - end + end) + + :ok = Data.pg_insert_many(state.name, state.shard_index, Enum.reverse(inserts)) + buffer end) event_buffer = @@ -5332,6 +5321,104 @@ defmodule Group.Replica do end end + defp reconcile_registry_snapshot_batch(state, cluster, stream_id, claims, event_buffer) do + source_node = WireProtocol.stream_origin(stream_id) + generation = WireProtocol.stream_generation(stream_id) + epoch = WireProtocol.stream_epoch(stream_id) + claim_table = Data.reg_claim_by_key_table(state.name, state.shard_index) + projection_table = Data.reg_by_key_table(state.name, state.shard_index) + + classified = + Enum.map(claims, fn {key, _pid, _meta, _time} = claim -> + claim_key = {cluster, key, source_node, generation, epoch} + + { + claim, + Data.registry_claim_uncontended_in_table?(claim_table, claim_key) + } + end) + + entries = + Enum.map(claims, fn {key, pid, meta, time} -> + {cluster, key, pid, meta, time, source_node} + end) + + if Enum.all?(classified, &elem(&1, 1)) and + Data.registry_insert_new_many(state.name, state.shard_index, entries) do + event_buffer = + Enum.reduce(claims, event_buffer, fn {key, pid, meta, _time}, buffer -> + event = build_event(state.name, :registered, key, pid, meta, %{cluster: cluster}) + Snapshot.buffer_event(event, buffer) + end) + + {state, event_buffer} + else + reconcile_registry_snapshot_batch_rows( + state, + cluster, + source_node, + projection_table, + classified, + event_buffer + ) + end + end + + defp reconcile_registry_snapshot_batch_rows( + state, + cluster, + source_node, + projection_table, + classified, + event_buffer + ) do + {state, event_buffer, inserts} = + Enum.reduce(classified, {state, event_buffer, []}, fn + {{key, pid, meta, time}, uncontended?}, {acc, buffer, inserts} -> + current = Data.registry_lookup_in_table(projection_table, cluster, key) + + case {uncontended?, current} do + {true, nil} -> + event = build_event(acc.name, :registered, key, pid, meta, %{cluster: cluster}) + + { + acc, + Snapshot.buffer_event(event, buffer), + [{cluster, key, pid, meta, time, source_node} | inserts] + } + + {true, {^pid, old_meta, old_time, ^source_node}} -> + if old_meta == meta and old_time == time do + {acc, buffer, inserts} + else + event = + build_event(acc.name, :registered, key, pid, meta, %{ + previous_meta: old_meta, + cluster: cluster + }) + + { + acc, + Snapshot.buffer_event(event, buffer), + [{cluster, key, pid, meta, time, source_node} | inserts] + } + end + + _ -> + :ok = + Data.registry_insert_many(acc.name, acc.shard_index, Enum.reverse(inserts)) + + {acc, events} = + reconcile_registry_projection(acc, cluster, key, :reconcile, []) + + {acc, Snapshot.buffer_events(Enum.reverse(events), buffer), []} + end + end) + + :ok = Data.registry_insert_many(state.name, state.shard_index, Enum.reverse(inserts)) + {state, event_buffer} + end + defp reconcile_registry_keys(state, keys, reason, events) do Enum.reduce(keys, {state, events}, fn {cluster, key}, {acc, inner_events} -> reconcile_registry_projection(acc, cluster, key, reason, inner_events) diff --git a/lib/group/replica/data.ex b/lib/group/replica/data.ex index 8717b48..69439bc 100644 --- a/lib/group/replica/data.ex +++ b/lib/group/replica/data.ex @@ -1083,6 +1083,32 @@ defmodule Group.Replica.Data do :ok end + @doc false + def registry_insert_new_many(_name, _shard, []), do: true + + def registry_insert_new_many(name, shard, entries) do + table = reg_by_key_table(name, shard) + table_pid = reg_by_pid_table(name, shard) + + objects = + Enum.map(entries, fn {cluster, key, pid, meta, time, node} -> + {{cluster, key}, pid, meta, time, node} + end) + + if :ets.insert_new(table, objects) do + :ets.insert( + table_pid, + Enum.map(entries, fn {cluster, key, pid, meta, time, node} -> + {{pid, cluster, key}, meta, time, node} + end) + ) + + true + else + false + end + end + def registry_delete(name, shard, cluster, key, pid) do table = reg_by_key_table(name, shard) :ets.delete(table, {cluster, key}) @@ -1116,8 +1142,11 @@ defmodule Group.Replica.Data do end def registry_lookup(name, shard, cluster, key) do - table = reg_by_key_table(name, shard) + registry_lookup_in_table(reg_by_key_table(name, shard), cluster, key) + end + @doc false + def registry_lookup_in_table(table, cluster, key) do case :ets.lookup(table, {cluster, key}) do [{{^cluster, ^key}, pid, meta, time, node}] -> {pid, meta, time, node} @@ -1186,6 +1215,33 @@ defmodule Group.Replica.Data do :ok end + @doc false + def insert_exact_registry_claims_many(_name, _shard, _stream_id, _seq, []), do: :ok + + def insert_exact_registry_claims_many(name, shard, stream_id, seq, claims) + when is_list(claims) do + cluster = Group.Replica.WireProtocol.stream_cluster(stream_id) + origin_node = Group.Replica.WireProtocol.stream_origin(stream_id) + generation = Group.Replica.WireProtocol.stream_generation(stream_id) + epoch = Group.Replica.WireProtocol.stream_epoch(stream_id) + + :ets.insert( + reg_claim_by_key_table(name, shard), + Enum.map(claims, fn {key, pid, meta, time} -> + {{cluster, key, origin_node, generation, epoch}, pid, meta, time, seq} + end) + ) + + :ets.insert( + reg_claim_by_pid_table(name, shard), + Enum.map(claims, fn {key, pid, meta, time} -> + {{pid, cluster, key, origin_node, generation, epoch}, meta, time, seq} + end) + ) + + :ok + end + def delete_registry_claim(name, shard, stream_id, seq, key, pid) do cluster = Group.Replica.WireProtocol.stream_cluster(stream_id) origin_node = Group.Replica.WireProtocol.stream_origin(stream_id) @@ -1220,6 +1276,20 @@ defmodule Group.Replica.Data do ]) end + @doc false + def registry_claim_uncontended_in_table?( + table, + {cluster, key, _origin_node, _generation, _epoch} = claim_key + ) do + not same_registry_claim_key?(:ets.prev(table, claim_key), cluster, key) and + not same_registry_claim_key?(:ets.next(table, claim_key), cluster, key) + end + + defp same_registry_claim_key?({cluster, key, _origin, _generation, _epoch}, cluster, key), + do: true + + defp same_registry_claim_key?(_claim_key, _cluster, _key), do: false + def registry_claims_for_stream(name, shard, stream_id) do fold_registry_claims_for_stream(name, shard, stream_id, [], fn row, rows -> [row | rows] end) |> Enum.reverse() @@ -1291,6 +1361,35 @@ defmodule Group.Replica.Data do fun ) when is_function(fun, 2) do + replace_registry_claims_for_stream_from_staging( + name, + shard, + stream_id, + snapshot_seq, + staging_table, + chunk_count, + acc, + fun, + fn claims, inner -> + Enum.reduce(claims, inner, fn {key, _pid, _meta, _time}, batch_inner -> + fun.(key, batch_inner) + end) + end + ) + end + + def replace_registry_claims_for_stream_from_staging( + name, + shard, + stream_id, + snapshot_seq, + staging_table, + _chunk_count, + acc, + removed_fun, + installed_batch_fun + ) + when is_function(removed_fun, 2) and is_function(installed_batch_fun, 2) do cluster = Group.Replica.WireProtocol.stream_cluster(stream_id) origin_node = Group.Replica.WireProtocol.stream_origin(stream_id) generation = Group.Replica.WireProtocol.stream_generation(stream_id) @@ -1313,17 +1412,16 @@ defmodule Group.Replica.Data do if Group.Replica.Snapshot.member_registry?(staging_table, key) do inner else - fun.(key, inner) + removed_fun.(key, inner) end end) - Group.Replica.Snapshot.fold_registry( + Group.Replica.Snapshot.reduce_registry_batches( staging_table, - chunk_count, acc, - fn {key, pid, meta, time}, inner -> - put_registry_claim(name, shard, stream_id, snapshot_seq, key, pid, meta, time) - fun.(key, inner) + fn claims, inner -> + :ok = insert_exact_registry_claims_many(name, shard, stream_id, snapshot_seq, claims) + installed_batch_fun.(claims, inner) end ) end diff --git a/lib/group/replica/snapshot.ex b/lib/group/replica/snapshot.ex index 5a943a2..95935d2 100644 --- a/lib/group/replica/snapshot.ex +++ b/lib/group/replica/snapshot.ex @@ -99,7 +99,13 @@ defmodule Group.Replica.Snapshot do end def fold_registry(table, _chunk_count, acc, fun) when is_function(fun, 2) do - fold_staged_rows( + reduce_registry_batches(table, acc, fn rows, inner -> + Enum.reduce(rows, inner, fun) + end) + end + + def reduce_registry_batches(table, acc, fun) when is_function(fun, 2) do + fold_staged_row_batches( table, [ {{{:registry, :"$1"}, :"$2", :"$3", :"$4", :_}, [], [{{:"$1", :"$2", :"$3", :"$4"}}]} @@ -110,7 +116,13 @@ defmodule Group.Replica.Snapshot do end def fold_pg(table, _chunk_count, acc, fun) when is_function(fun, 2) do - fold_staged_rows( + reduce_pg_batches(table, acc, fn rows, inner -> + Enum.reduce(rows, inner, fun) + end) + end + + def reduce_pg_batches(table, acc, fun) when is_function(fun, 2) do + fold_staged_row_batches( table, [ {{{:pg, :"$1", :"$2"}, :"$3", :"$4", :_}, [], [{{:"$1", :"$2", :"$3", :"$4"}}]} @@ -170,17 +182,17 @@ defmodule Group.Replica.Snapshot do :ets.lookup(table, {:pg, key, pid}) == [{{:pg, key, pid}, meta, time, chunk_index}] end - defp fold_staged_rows(table, match_spec, acc, fun) do + defp fold_staged_row_batches(table, match_spec, acc, fun) do case :ets.select(table, match_spec, 4_096) do :"$end_of_table" -> acc - {rows, continuation} -> fold_staged_rows(continuation, Enum.reduce(rows, acc, fun), fun) + {rows, continuation} -> fold_staged_row_batches(continuation, fun.(rows, acc), fun) end end - defp fold_staged_rows(continuation, acc, fun) do + defp fold_staged_row_batches(continuation, acc, fun) do case :ets.select(continuation) do :"$end_of_table" -> acc - {rows, next} -> fold_staged_rows(next, Enum.reduce(rows, acc, fun), fun) + {rows, next} -> fold_staged_row_batches(next, fun.(rows, acc), fun) end end From fed690ffa6b589f3dc5bcca748fd2f8cd9556384 Mon Sep 17 00:00:00 2001 From: Chris McCord Date: Sun, 16 Aug 2026 02:59:30 +0000 Subject: [PATCH 13/16] Fix Jepsen transport event requirement --- test/jepsen/src/group/jepsen/model.clj | 7 +++++-- test/jepsen/test/group/jepsen/model_test.clj | 20 ++++++++++++++++++-- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/test/jepsen/src/group/jepsen/model.clj b/test/jepsen/src/group/jepsen/model.clj index b0e782a..f1d80e9 100644 --- a/test/jepsen/src/group/jepsen/model.clj +++ b/test/jepsen/src/group/jepsen/model.clj @@ -3,6 +3,10 @@ [jepsen.checker :as checker] [jepsen.history :as history])) +(def default-required-transport-events + #{:delta-batch :snapshot-chunk :multi-chunk-snapshot + :registry-conflict-death}) + (defn successful-snapshots [history] (->> history (remove history/invoke?) @@ -139,8 +143,7 @@ (map #(or (:transport-events %) {}) (vals relevant-snapshots))) required-transport-events (get test :required-transport-events - #{:delta-batch :snapshot-chunk :multi-chunk-snapshot-chunk - :registry-conflict-death}) + default-required-transport-events) missing-transport-events (set (remove #(pos? (get transport-events % 0)) required-transport-events)) expected-profile (keyword (:transport test)) diff --git a/test/jepsen/test/group/jepsen/model_test.clj b/test/jepsen/test/group/jepsen/model_test.clj index 241fb15..80ee3d8 100644 --- a/test/jepsen/test/group/jepsen/model_test.clj +++ b/test/jepsen/test/group/jepsen/model_test.clj @@ -162,12 +162,28 @@ result (model/analyze (assoc test-map :required-transport-events - #{:delta-batch :snapshot-chunk :multi-chunk-snapshot-chunk}) + model/default-required-transport-events) history)] (is (false? (:valid? result))) - (is (= #{:delta-batch :snapshot-chunk :multi-chunk-snapshot-chunk} + (is (= model/default-required-transport-events (:missing-transport-events result))))) +(deftest accepts-the-transport-event-names-emitted-by-the-live-nodes + (let [events {:delta-batch 1 + :snapshot-chunk 2 + :multi-chunk-snapshot 1 + :registry-conflict-death 1} + history [(assoc-in (snapshot-op 1 "n1" [] (empty-registry) (empty-pg)) + [:value :transport-events] + events) + (snapshot-op 2 "n2" [] (empty-registry) (empty-pg)) + (snapshot-op 3 "n3" [] (empty-registry) (empty-pg))] + result (model/analyze + (dissoc test-map :required-transport-events) + history)] + (is (:valid? result)) + (is (empty? (:missing-transport-events result))))) + (deftest rejects-internal-corruption-or-leftover-snapshot-staging (let [bad (-> (snapshot-op 1 "n1" [] (empty-registry) (empty-pg)) (assoc-in [:value :internal :healthy] false) From e320edbda5002900885672d903d2142ac804d4b2 Mon Sep 17 00:00:00 2001 From: Chris McCord Date: Sun, 16 Aug 2026 14:05:34 +0000 Subject: [PATCH 14/16] Harden anti-entropy qualification and startup recovery --- CLAUDE.md | 9 +- README.md | 2 +- lib/group/replica.ex | 18 ++++ mix.exs | 1 + priv/bench/README.md | 10 ++ priv/bench/lib/group_bench/distributed.ex | 93 +++++++++++++++++ priv/bench/lib/group_bench/replica.ex | 40 ++++++++ priv/bench/run_distributed.sh | 7 +- test/README.md | 13 ++- test/anti_entropy_fault_regression_test.exs | 31 ++++++ test/jepsen/README.md | 13 +-- test/jepsen/campaign.sh | 13 ++- test/jepsen/docker-compose.yml | 3 + test/jepsen/node.exs | 102 +++++++++++++++++-- test/jepsen/qualify.sh | 11 +- test/jepsen/src/group/jepsen/core.clj | 9 +- test/jepsen/src/group/jepsen/docker.clj | 4 +- test/jepsen/src/group/jepsen/model.clj | 9 ++ test/jepsen/test/group/jepsen/model_test.clj | 30 ++++++ test/mutation/run.exs | 33 +++--- test/replica_snapshot_distributed_test.exs | 46 +++++++++ 21 files changed, 445 insertions(+), 52 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index a792db4..90041ce 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -37,7 +37,7 @@ priv/bench/ — local and distributed benchmark project ```bash mix test # every-PR ExUnit/property/chaos/pure-checker gate -mix test.soak # nightly/release six-profile Jepsen campaign +mix test.soak # nightly mutation/live-checker + six-profile Jepsen campaign # Focused development mix test test/group_test.exs @@ -46,8 +46,11 @@ mix test test/replica_model_property_test.exs ``` `mix test` accepts normal Mix test paths/options and then runs the pure Jepsen -checker qualification. `mix test.soak` runs that gate first, then twenty -five-minute histories for distribution/TCP/chaos × mixed/permanent scenarios. +checker qualification. `mix test.soak` runs that gate first, kills every +defined protocol mutant, proves the live checker rejects injected corruption, +then runs twenty five-minute histories for distribution/TCP/chaos × +mixed/permanent scenarios. Chaos/mixed must exercise a multi-record delta +repair; the remaining profiles retain the one-record stress configuration. See `test/README.md`, `test/formal/README.md`, and `test/jepsen/README.md`. ## Supervision and Ownership diff --git a/README.md b/README.md index 05ce985..1d08668 100644 --- a/README.md +++ b/README.md @@ -602,7 +602,7 @@ a new or current generation and anti-entropy reconstructs its live state. ```bash mix test -mix test.soak # nightly/release qualification +mix test.soak # nightly mutation/live-checker and six-profile Jepsen qualification ``` See [`test/README.md`](test/README.md) for the every-PR gate, shrinkable diff --git a/lib/group/replica.ex b/lib/group/replica.ex index 80ed9da..5ce1de7 100644 --- a/lib/group/replica.ex +++ b/lib/group/replica.ex @@ -4614,6 +4614,24 @@ defmodule Group.Replica do {^meta, ^time, ^source_node} -> {inserts, inner} + {old_meta, _old_time, old_source} when old_source != source_node -> + Logger.error( + "#{log_prefix_shard(state)} repaired PG row with impossible stored origin " <> + "#{inspect(old_source)} while installing exact snapshot from " <> + inspect(source_node) + ) + + event = + build_event(state.name, :joined, key, pid, meta, %{ + previous_meta: old_meta, + cluster: cluster + }) + + { + [{cluster, key, pid, meta, time, source_node} | inserts], + Snapshot.buffer_event(event, inner) + } + {old_meta, _old_time, ^source_node} -> inner = if old_meta != meta do diff --git a/mix.exs b/mix.exs index 6adb847..0c6d68d 100644 --- a/mix.exs +++ b/mix.exs @@ -67,6 +67,7 @@ defmodule Group.MixProject do test: ["test", "cmd test/jepsen/checker.sh"], "test.soak": [ "test", + "cmd env GROUP_JEPSEN_SKIP_CHECKER=1 test/jepsen/qualify.sh", "cmd env GROUP_JEPSEN_SKIP_CHECKER=1 test/jepsen/campaign.sh" ] ] diff --git a/priv/bench/README.md b/priv/bench/README.md index 07d5212..35ccb5a 100644 --- a/priv/bench/README.md +++ b/priv/bench/README.md @@ -61,6 +61,16 @@ peer eviction: `:pg_hotspot` deliberately puts a million memberships on one key/shard, which is the worst case for snapshot installation and restart rebuilding. +To isolate the synchronous registry projection rebuild performed by a +restarting shard, seed only the proportional load for one shard. For example, +one million total rows at 32 shards benchmarks 31,250 rows in the restarted +lane without including snapshot or network time: + +```bash +./run_distributed.sh --shards 32 \ + --coordinator-expr 'GroupBench.Distributed.run_registry_init_only(shards: 32, entries: 1000000, samples: 3)' +``` + ## Local Scenarios All local benchmarks run for both the default (nil) cluster and a named cluster diff --git a/priv/bench/lib/group_bench/distributed.ex b/priv/bench/lib/group_bench/distributed.ex index a99f4ba..3fbae03 100644 --- a/priv/bench/lib/group_bench/distributed.ex +++ b/priv/bench/lib/group_bench/distributed.ex @@ -107,6 +107,25 @@ defmodule GroupBench.Distributed do IO.puts("\n Done.\n") end + def run_registry_init_only(opts \\ []) do + shards = Keyword.get(opts, :shards, 32) + total_entries = Keyword.get(opts, :entries, 1_000_000) + samples = Keyword.get(opts, :samples, 3) + Process.put(:bench_shards, shards) + + header("Registry Projection Shard-Init Benchmark") + IO.puts(" coordinator: #{node()}") + IO.puts(" shards: #{shards}") + IO.puts(" total rows: #{format_number(total_entries)}") + IO.puts(" samples: #{samples}") + IO.puts(" schedulers: #{System.schedulers_online()}") + + connect_replicas() + result = bench_registry_init(hd(@replicas), total_entries, samples) + IO.puts("\n Done.\n") + result + end + # ── Connection ──────────────────────────────────────────────────────── defp connect_replicas do @@ -266,6 +285,80 @@ defmodule GroupBench.Distributed do result end + defp bench_registry_init(node, total_entries, samples) do + header("Synchronous projection rebuild during shard restart") + shards = Process.get(:bench_shards) + rows_per_shard = div(total_entries + shards - 1, shards) + target_shard = 0 + + stop_group_on(node) + + start_group_on(node, + replicated_anti_entropy_interval: 60_000, + replicated_peer_lease_timeout: 120_000 + ) + + {seed_us, owner} = + :timer.tc(fn -> + :erpc.call( + node, + GroupBench.Replica, + :seed_registry_shard, + [@name, target_shard, rows_per_shard, "init/registry/", 10_000], + 1_800_000 + ) + end) + + counts = :erpc.call(node, GroupBench.Replica, :registry_counts_by_shard, [@name]) + + unless List.keyfind(counts, target_shard, 0) == {target_shard, rows_per_shard} do + raise "registry init benchmark seeded the wrong target-shard row count" + end + + before_memory = :erpc.call(node, GroupBench.Replica, :memory_snapshot, [@name]) + + restart_us = + Enum.map(1..samples, fn _sample -> + result = + :erpc.call( + node, + GroupBench.Replica, + :restart_shard, + [@name, target_shard], + 900_000 + ) + + unless List.keyfind( + :erpc.call(node, GroupBench.Replica, :registry_counts_by_shard, [@name]), + target_shard, + 0 + ) == {target_shard, rows_per_shard} do + raise "registry row count changed across shard restart" + end + + result.elapsed_us + end) + |> Enum.sort() + + result = %{ + total_entries: total_entries, + shards: shards, + rows_per_shard: rows_per_shard, + samples: samples, + seed_ms: div(seed_us, 1_000), + restart_ms: Enum.map(restart_us, &Float.round(&1 / 1_000, 3)), + restart_min_ms: Float.round(hd(restart_us) / 1_000, 3), + restart_p50_ms: Float.round(Enum.at(restart_us, div(length(restart_us), 2)) / 1_000, 3), + restart_max_ms: Float.round(List.last(restart_us) / 1_000, 3), + table_memory_bytes: before_memory.group_table_bytes + } + + IO.puts("\n PERF_RESULT #{inspect(result, pretty: true, limit: :infinity)}") + :erpc.call(node, Process, :exit, [owner, :kill]) + stop_group_on(node) + result + end + defp replicated_row_count(node, :registry) do :erpc.call(node, GroupBench.Replica, :total_registry_count, [@name]) end diff --git a/priv/bench/lib/group_bench/replica.ex b/priv/bench/lib/group_bench/replica.ex index 179d505..8ba3afe 100644 --- a/priv/bench/lib/group_bench/replica.ex +++ b/priv/bench/lib/group_bench/replica.ex @@ -204,6 +204,46 @@ defmodule GroupBench.Replica do owner end + @doc false + def seed_registry_shard(name, shard, count, key_prefix, batch_size \\ 10_000) do + owner = spawn(fn -> Process.sleep(:infinity) end) + shards = Group.get_config(name).num_shards + stream_id = Group.Replica.Data.local_stream_id(name, shard, nil) + + 1 + |> Stream.iterate(&(&1 + 1)) + |> Stream.filter(fn index -> + Group.Replica.shard_index_for(nil, "#{key_prefix}#{index}", shards) == shard + end) + |> Stream.take(count) + |> Stream.chunk_every(batch_size) + |> Enum.each(fn indexes -> + entries = + Enum.map(indexes, fn index -> + key = "#{key_prefix}#{index}" + + :ok = + Group.Replica.Data.put_registry_claim( + name, + shard, + stream_id, + 1, + key, + owner, + %{}, + index + ) + + {nil, key, owner, %{}, index, node()} + end) + + :ok = Group.Replica.Data.registry_insert_many(name, shard, entries) + end) + + install_benchmark_stream_heads(name) + owner + end + @doc false def seed_pg_hotspot(name, count, key, batch_size \\ 10_000) do shard = Group.Replica.shard_index_for(nil, key, Group.get_config(name).num_shards) diff --git a/priv/bench/run_distributed.sh b/priv/bench/run_distributed.sh index 3273aef..5f42bd1 100755 --- a/priv/bench/run_distributed.sh +++ b/priv/bench/run_distributed.sh @@ -6,6 +6,7 @@ cd "$(dirname "$0")" COOKIE=bench SHARDS=8 COORDINATOR_EXPR= +BENCH_ERL_AFLAGS="${GROUP_BENCH_ERL_AFLAGS:-}" while [[ $# -gt 0 ]]; do case "$1" in @@ -24,12 +25,12 @@ mix deps.get --check 2>/dev/null || mix deps.get mix compile echo "==> Starting replica1..." -elixir --name replica1@127.0.0.1 --cookie "$COOKIE" \ +ERL_AFLAGS="$BENCH_ERL_AFLAGS" elixir --name replica1@127.0.0.1 --cookie "$COOKIE" \ -S mix run --no-halt -e "GroupBench.Replica.start()" & REPLICA1_PID=$! echo "==> Starting replica2..." -elixir --name replica2@127.0.0.1 --cookie "$COOKIE" \ +ERL_AFLAGS="$BENCH_ERL_AFLAGS" elixir --name replica2@127.0.0.1 --cookie "$COOKIE" \ -S mix run --no-halt -e "GroupBench.Replica.start()" & REPLICA2_PID=$! @@ -44,5 +45,5 @@ trap cleanup EXIT sleep 2 echo "==> Starting coordinator (shards=$SHARDS)..." -elixir --name coordinator@127.0.0.1 --cookie "$COOKIE" \ +ERL_AFLAGS="$BENCH_ERL_AFLAGS" elixir --name coordinator@127.0.0.1 --cookie "$COOKIE" \ -S mix run -e "$COORDINATOR_EXPR" diff --git a/test/README.md b/test/README.md index f62538e..b8eea83 100644 --- a/test/README.md +++ b/test/README.md @@ -14,10 +14,11 @@ test/jepsen/run.sh # one OS-partition/restart Jepsen model test `mix test` preserves normal Mix test arguments while always running the pure Jepsen lifecycle-checker qualification after ExUnit. It does not require -Docker. `mix test.soak` first runs that complete PR gate, then runs the -distribution/TCP/chaos × mixed/permanent Jepsen campaign. The soak defaults to -20 five-minute fault histories per combination and is intended for nightly and -release qualification rather than individual edits. +Docker. `mix test.soak` first runs that complete PR gate, kills every defined +protocol mutant, runs live positive/negative checker qualification, and then +runs the distribution/TCP/chaos × mixed/permanent Jepsen campaign. The soak +defaults to 20 five-minute fault histories per combination and is intended for +nightly and release qualification rather than individual edits. ## Test files @@ -69,7 +70,9 @@ PG, claim, cluster, cursor, oplog, snapshot-staging, and retired-origin invariants. Its permanent-retirement scenario proves eviction even when a peer never returns. `test/jepsen/campaign.sh` runs the full profile/scenario matrix; `test/jepsen/qualify.sh` mutation-tests the implementation and proves that the -live checker rejects injected faults. +live checker rejects injected faults. Chaos/mixed uses a larger repair window +and is invalid unless it observes a multi-record delta run; the other profiles +retain the one-record stress configuration. ## How distribution works diff --git a/test/anti_entropy_fault_regression_test.exs b/test/anti_entropy_fault_regression_test.exs index 7636dd1..bac5c47 100644 --- a/test/anti_entropy_fault_regression_test.exs +++ b/test/anti_entropy_fault_regression_test.exs @@ -3910,6 +3910,37 @@ defmodule Group.AntiEntropyFaultRegressionTest do ]) == generation end) + # Once authority is exact, a lane can still start after shard-zero's + # best-effort fanout (or lose its lane-local view on restart). Prove that + # the lane hello itself reconstructs that view; merely observing the first + # fanout would not exercise install_current_replica_lane/3. + :ok = + TestCluster.rpc!(context.node_b, TestCluster, :delete_remote_view_info, [ + name, + 1, + context.node_a + ]) + + assert TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_view_generation, [ + name, + 1, + context.node_a + ]) == nil + + send( + target_lane, + {:replica_lane_hello, source_lane, Group.Replica.WireProtocol.version(), generation, + revision, Group.TestReplicaTransport.id(), Group.TestReplicaTransport.descriptor(name, [])} + ) + + TestCluster.assert_eventually(fn -> + TestCluster.rpc!(context.node_b, Group.Replica.Data, :remote_view_generation, [ + name, + 1, + context.node_a + ]) == generation + end) + # The source lane is suspended, so the only way this peer_connect can be # present is the immediate post-authority re-probe from the target lane. TestCluster.assert_eventually(fn -> diff --git a/test/jepsen/README.md b/test/jepsen/README.md index 42b13cc..b510981 100644 --- a/test/jepsen/README.md +++ b/test/jepsen/README.md @@ -110,11 +110,9 @@ Run mutation qualification plus live positive- and negative-checker tests: test/jepsen/qualify.sh ``` -This must reject production mutations which remove generation fencing, gap -detection, exact snapshot replacement, complete snapshot assembly, periodic -repair, or retirement purging. It then verifies that a healthy live history is -accepted and deliberately injected owner-death and internal-index corruption -are rejected. +This runs every mutation defined by `test/mutation/run.exs`, then verifies that +a healthy live history is accepted and deliberately injected owner-death, +internal-index, and stranded snapshot-cursor corruption are rejected. Results and histories are written below `test/jepsen/store/`. Containers are removed after a run. Set `GROUP_JEPSEN_KEEP_CONTAINERS=1` to retain them and @@ -132,7 +130,10 @@ test/jepsen/checker.sh At the repository root, `mix test` runs this pure checker after the complete ExUnit, StreamData, and deterministic-chaos suite. `mix test.soak` runs that -same PR gate followed by `campaign.sh`. +same PR gate, the complete mutation/live-checker qualification, and then +`campaign.sh`. Chaos/mixed uses a sender/repair buffer of 32 and requires +evidence that one repaired delta run contained at least two records; all other +profiles keep the single-record stress setting. ## Scope diff --git a/test/jepsen/campaign.sh b/test/jepsen/campaign.sh index 63ffe54..28a6492 100755 --- a/test/jepsen/campaign.sh +++ b/test/jepsen/campaign.sh @@ -32,9 +32,17 @@ echo "Jepsen campaign artifacts: ${artifact_dir}" for transport in distribution tcp chaos; do for scenario in mixed permanent; do log="${artifact_dir}/${transport}-${scenario}.log" - echo "Starting ${transport}/${scenario}: ${test_count} histories x ${time_limit}s" + sender_buffer_size=1 + min_delta_run_records=1 - if "${script_dir}/run.sh" test \ + if [[ "${transport}/${scenario}" == "chaos/mixed" ]]; then + sender_buffer_size=32 + min_delta_run_records=2 + fi + + echo "Starting ${transport}/${scenario}: ${test_count} histories x ${time_limit}s (sender buffer ${sender_buffer_size})" + + if GROUP_JEPSEN_SENDER_BUFFER_SIZE="${sender_buffer_size}" "${script_dir}/run.sh" test \ --no-ssh \ --nodes n1,n2,n3 \ --concurrency "${concurrency}" \ @@ -44,6 +52,7 @@ for transport in distribution tcp chaos; do --owner-count "${owner_count}" \ --fault-interval 2 \ --recovery-time "${recovery_time}" \ + --min-delta-run-records "${min_delta_run_records}" \ --transport "${transport}" \ --scenario "${scenario}" >"${log}" 2>&1; then valid_count="$(rg -c "Everything looks good" "${log}" || true)" diff --git a/test/jepsen/docker-compose.yml b/test/jepsen/docker-compose.yml index e93b5bd..039f07a 100644 --- a/test/jepsen/docker-compose.yml +++ b/test/jepsen/docker-compose.yml @@ -11,6 +11,7 @@ services: GROUP_JEPSEN_PORT: 9080 GROUP_JEPSEN_PEERS: group@n1,group@n2,group@n3 GROUP_JEPSEN_TRANSPORT: ${GROUP_JEPSEN_TRANSPORT:-distribution} + GROUP_JEPSEN_SENDER_BUFFER_SIZE: ${GROUP_JEPSEN_SENDER_BUFFER_SIZE:-1} ports: ["19081:9080"] networks: [group] @@ -26,6 +27,7 @@ services: GROUP_JEPSEN_PORT: 9080 GROUP_JEPSEN_PEERS: group@n1,group@n2,group@n3 GROUP_JEPSEN_TRANSPORT: ${GROUP_JEPSEN_TRANSPORT:-distribution} + GROUP_JEPSEN_SENDER_BUFFER_SIZE: ${GROUP_JEPSEN_SENDER_BUFFER_SIZE:-1} ports: ["19082:9080"] networks: [group] @@ -41,6 +43,7 @@ services: GROUP_JEPSEN_PORT: 9080 GROUP_JEPSEN_PEERS: group@n1,group@n2,group@n3 GROUP_JEPSEN_TRANSPORT: ${GROUP_JEPSEN_TRANSPORT:-distribution} + GROUP_JEPSEN_SENDER_BUFFER_SIZE: ${GROUP_JEPSEN_SENDER_BUFFER_SIZE:-1} ports: ["19083:9080"] networks: [group] diff --git a/test/jepsen/node.exs b/test/jepsen/node.exs index 6920da3..424259d 100644 --- a/test/jepsen/node.exs +++ b/test/jepsen/node.exs @@ -108,7 +108,20 @@ defmodule Group.Jepsen.Transport.Common do end end - def record({:delta_batch, _version, _runs}), do: Stats.increment(:delta_batch) + def record({:delta_batch, _version, runs}) do + Stats.increment(:delta_batch) + + peak = + runs + |> Enum.map(fn + {_stream, _first_seq, records, _head} when is_list(records) -> length(records) + _invalid_run -> 0 + end) + |> Enum.max(fn -> 0 end) + + Stats.observe_max(:delta_run_records_peak, peak) + end + def record(_message), do: Stats.increment(:other_message) defp transport_result(:ok), do: :transport_ok @@ -575,7 +588,12 @@ defmodule Group.Jepsen.Driver do def owner_snapshots do names() - |> Enum.flat_map(&GenServer.call(&1, :owner_snapshots, 30_000)) + |> Enum.flat_map(fn driver -> + case GenServer.call(driver, :owner_snapshots, 30_000) do + {:ok, snapshots} -> snapshots + {:error, reason} -> raise "owner snapshot refresh failed: #{inspect(reason)}" + end + end) |> Enum.sort_by(& &1.token) end @@ -657,14 +675,41 @@ defmodule Group.Jepsen.Driver do end def handle_call(:owner_snapshots, _from, state) do - owners = - state.owners - |> Enum.flat_map(fn - {_logical_owner, {_pid, _token, _monitor_ref, nil}} -> [] - {_logical_owner, {_pid, _token, _monitor_ref, owner_state}} -> [owner_state] + result = + Enum.reduce_while(state.owners, {[], %{}}, fn + {logical_owner, {pid, token, monitor_ref, cached}}, {snapshots, acc} -> + case live_owner_snapshot(pid) do + {:ok, owner_state} -> + {:cont, + { + [owner_state | snapshots], + Map.put(acc, logical_owner, {pid, token, monitor_ref, owner_state}) + }} + + {:error, reason} -> + {:halt, {:error, {logical_owner, token, reason, cached}}} + end end) - {:reply, owners, state} + case result do + {:error, reason} -> + {:reply, {:error, reason}, state} + + {owners, refreshed} -> + {:reply, {:ok, Enum.reverse(owners)}, %{state | owners: refreshed}} + end + end + + defp live_owner_snapshot(pid) do + if Process.alive?(pid) do + try do + {:ok, GenServer.call(pid, :snapshot, 10_000)} + catch + :exit, reason -> {:error, reason} + end + else + {:error, :not_alive} + end end def handle_call(:unexpected_deaths, _from, state) do @@ -823,6 +868,7 @@ defmodule Group.Jepsen.Invariant do def snapshot(retired_nodes) do config = Group.get_config(:jepsen_group) shards = 0..(config.num_shards - 1) + maybe_inject_cursor_marker_corruption(shards) errors = check("dual indexes", &assert_dual_indexes/0) ++ @@ -863,6 +909,31 @@ defmodule Group.Jepsen.Invariant do } end + defp maybe_inject_cursor_marker_corruption(shards) do + if File.exists?("/tmp/group-jepsen-cursor-marker-corruption") do + cursor = + Enum.find_value(shards, fn shard -> + Data.replica_cursor_table(:jepsen_group, shard) + |> :ets.tab2list() + |> case do + [{stream, _cursor} | _] -> {shard, stream} + [] -> nil + end + end) + + case cursor do + {shard, stream} -> + :ets.insert( + Data.replica_cursor_table(:jepsen_group, shard), + {stream, {:snapshot_installing, 1}} + ) + + nil -> + raise "no remote replica cursor available for corruption" + end + end + end + defp check(label, fun) do fun.() [] @@ -1017,7 +1088,8 @@ defmodule Group.Jepsen.Invariant do WireProtocol.stream_generation(stream) == Data.remote_generation(:jepsen_group, origin) and WireProtocol.stream_epoch(stream) == - Data.remote_cluster_epoch(:jepsen_group, origin, cluster) and seq >= 0 + Data.remote_cluster_epoch(:jepsen_group, origin, cluster) and is_integer(seq) and + seq >= 0 unless valid?, do: raise("cursor lacks current authority #{inspect({stream, seq})}") end) @@ -1308,6 +1380,11 @@ defmodule Group.Jepsen.Wire do %{status: :ok} end + defp corrupt("cursor-marker") do + File.write!("/tmp/group-jepsen-cursor-marker-corruption", "enabled\n") + %{status: :ok} + end + defp corrupt(other), do: %{status: :fail, error: "unknown corruption #{inspect(other)}"} defp parse_cluster("root"), do: nil defp parse_cluster(cluster), do: cluster @@ -1346,7 +1423,7 @@ defmodule Group.Jepsen.Main do log: false, resolve_registry_conflict: {Group.Jepsen.ConflictResolver, :resolve, []}, replica_transport: Group.Jepsen.Transport.Control.transport(node_id), - replicated_sender_buffer_size: 1, + replicated_sender_buffer_size: positive_env!("GROUP_JEPSEN_SENDER_BUFFER_SIZE", 1), replicated_oplog_max_entries: 16, replicated_snapshot_chunk_target_bytes: 1_024, replicated_anti_entropy_interval: 50, @@ -1377,6 +1454,11 @@ defmodule Group.Jepsen.Main do Process.sleep(100) reconnect_loop(peers) end + + defp positive_env!(name, default) do + value = System.get_env(name, Integer.to_string(default)) |> String.to_integer() + if value > 0, do: value, else: raise("#{name} must be positive") + end end Group.Jepsen.Main.run(System.argv()) diff --git a/test/jepsen/qualify.sh b/test/jepsen/qualify.sh index 5de63c1..46d4770 100755 --- a/test/jepsen/qualify.sh +++ b/test/jepsen/qualify.sh @@ -7,13 +7,9 @@ artifact_dir="$(mktemp -d "${script_dir}/.cache/qualification.XXXXXX")" cd "${repo_dir}" -mix run test/mutation/run.exs \ - accept_old_generation \ - advance_cursor_across_gap \ - registry_snapshot_is_additive \ - commit_incomplete_snapshot \ - disable_periodic_heads \ - skip_generation_purge +mix run test/mutation/run.exs + +export GROUP_JEPSEN_SKIP_CHECKER=1 run_jepsen() { local expectation="$1" @@ -51,5 +47,6 @@ run_jepsen() { run_jepsen pass none run_jepsen fail unexpected-death run_jepsen fail internal-index +run_jepsen fail cursor-marker echo "mutation and live checker qualification passed" diff --git a/test/jepsen/src/group/jepsen/core.clj b/test/jepsen/src/group/jepsen/core.clj index 2db2c8c..6130aa9 100644 --- a/test/jepsen/src/group/jepsen/core.clj +++ b/test/jepsen/src/group/jepsen/core.clj @@ -173,12 +173,17 @@ [nil "--scenario SCENARIO" "Lifecycle scenario: mixed or permanent" :default "mixed" :validate [#{"mixed" "permanent"} "Unsupported scenario"]] - [nil "--corruption MODE" "Checker qualification: none, unexpected-death, internal-index" + [nil "--corruption MODE" "Checker qualification corruption mode" :default "none" - :validate [#{"none" "unexpected-death" "internal-index"} "Unsupported corruption"]] + :validate [#{"none" "unexpected-death" "internal-index" "cursor-marker"} + "Unsupported corruption"]] [nil "--max-operation-latency-ms MILLIS" "Maximum acknowledged Group call latency" :default 2000 :parse-fn #(Long/parseLong %) + :validate [pos? "Must be positive"]] + [nil "--min-delta-run-records NUMBER" "Required peak records in one repaired delta run" + :default 1 + :parse-fn #(Long/parseLong %) :validate [pos? "Must be positive"]]]) (defn -main [& args] diff --git a/test/jepsen/src/group/jepsen/docker.clj b/test/jepsen/src/group/jepsen/docker.clj index 6fc987f..da9cc6f 100644 --- a/test/jepsen/src/group/jepsen/docker.clj +++ b/test/jepsen/src/group/jepsen/docker.clj @@ -64,7 +64,9 @@ (defn reset-oracle! [node] (exec-sh! node - "rm -f /tmp/group-jepsen-unexpected-deaths /tmp/group-jepsen-persistent-events")) + (str "rm -f /tmp/group-jepsen-unexpected-deaths " + "/tmp/group-jepsen-persistent-events " + "/tmp/group-jepsen-cursor-marker-corruption"))) (defn ensure-firewall-chain! [node chain] (exec-sh! diff --git a/test/jepsen/src/group/jepsen/model.clj b/test/jepsen/src/group/jepsen/model.clj index f1d80e9..bc772aa 100644 --- a/test/jepsen/src/group/jepsen/model.clj +++ b/test/jepsen/src/group/jepsen/model.clj @@ -146,6 +146,12 @@ default-required-transport-events) missing-transport-events (set (remove #(pos? (get transport-events % 0)) required-transport-events)) + delta-run-records-peak + (reduce max 0 + (map #(get-in % [:transport-events :delta-run-records-peak] 0) + (vals relevant-snapshots))) + min-delta-run-records (get test :min-delta-run-records 0) + delta-run-coverage? (>= delta-run-records-peak min-delta-run-records) expected-profile (keyword (:transport test)) transport-profile-mismatches (into {} @@ -186,6 +192,7 @@ (empty? unstable-observations) (empty? peer-mismatches) (empty? missing-transport-events) + delta-run-coverage? (empty? transport-profile-mismatches) (empty? internal-errors) (empty? (:conflicts expected)) @@ -202,6 +209,8 @@ :peer-mismatches peer-mismatches :transport-events transport-events :missing-transport-events missing-transport-events + :delta-run-records-peak delta-run-records-peak + :min-delta-run-records min-delta-run-records :transport-profile-mismatches transport-profile-mismatches :internal-invariant-errors internal-errors :max-group-operation-latency-ms (/ max-latency-us 1000.0) diff --git a/test/jepsen/test/group/jepsen/model_test.clj b/test/jepsen/test/group/jepsen/model_test.clj index 80ee3d8..81e5263 100644 --- a/test/jepsen/test/group/jepsen/model_test.clj +++ b/test/jepsen/test/group/jepsen/model_test.clj @@ -184,6 +184,36 @@ (is (:valid? result)) (is (empty? (:missing-transport-events result))))) +(deftest rejects-a-profile-which-never-repairs-a-multi-record-delta-run + (let [single-record-events {:delta-batch 3 + :snapshot-chunk 2 + :multi-chunk-snapshot 1 + :registry-conflict-death 1 + :delta-run-records-peak 1} + with-events #(assoc-in % [:value :transport-events] single-record-events) + history [(with-events (snapshot-op 1 "n1" [] (empty-registry) (empty-pg))) + (with-events (snapshot-op 2 "n2" [] (empty-registry) (empty-pg))) + (with-events (snapshot-op 3 "n3" [] (empty-registry) (empty-pg)))] + result (model/analyze (assoc test-map :min-delta-run-records 2) history)] + (is (false? (:valid? result))) + (is (= 1 (:delta-run-records-peak result))) + (is (= 2 (:min-delta-run-records result))))) + +(deftest accepts-a-profile-which-repairs-a-multi-record-delta-run + (let [events {:delta-batch 1 + :snapshot-chunk 1 + :multi-chunk-snapshot 1 + :registry-conflict-death 1 + :delta-run-records-peak 8} + history [(assoc-in (snapshot-op 1 "n1" [] (empty-registry) (empty-pg)) + [:value :transport-events] + events) + (snapshot-op 2 "n2" [] (empty-registry) (empty-pg)) + (snapshot-op 3 "n3" [] (empty-registry) (empty-pg))] + result (model/analyze (assoc test-map :min-delta-run-records 2) history)] + (is (:valid? result)) + (is (= 8 (:delta-run-records-peak result))))) + (deftest rejects-internal-corruption-or-leftover-snapshot-staging (let [bad (-> (snapshot-op 1 "n1" [] (empty-registry) (empty-pg)) (assoc-in [:value :internal :healthy] false) diff --git a/test/mutation/run.exs b/test/mutation/run.exs index 18eb1e7..a7f9d79 100644 --- a/test/mutation/run.exs +++ b/test/mutation/run.exs @@ -92,7 +92,7 @@ defmodule Group.MutationCampaign do faulty_source: " if MapSet.size(transfer.received) >= 1 and chunk_count >= 1 and\n" <> " registry_count >= 0 and pg_count >= 0 do", - test: ["test/replica_snapshot_distributed_test.exs:177"] + test: ["test/replica_snapshot_distributed_test.exs:223"] }, %{ name: "commit_snapshot_without_terminal_manifest", @@ -122,7 +122,7 @@ defmodule Group.MutationCampaign do faulty_source: " MapSet.member?(transfer.received, chunk_index) and\n" <> " Process.alive?(self()) ->", - test: ["test/replica_snapshot_distributed_test.exs:339"] + test: ["test/replica_snapshot_distributed_test.exs:385"] }, %{ name: "retain_conflicting_snapshot_manifest", @@ -135,7 +135,7 @@ defmodule Group.MutationCampaign do {:ok, state, _conflicting_transfer} -> state """, - test: ["test/replica_snapshot_distributed_test.exs:145"] + test: ["test/replica_snapshot_distributed_test.exs:191"] }, %{ name: "commit_snapshot_after_source_changes_during_scan", @@ -154,14 +154,14 @@ defmodule Group.MutationCampaign do " :complete\n" <> " end\n" <> " catch", - test: ["test/replica_snapshot_distributed_test.exs:55"] + test: ["test/replica_snapshot_distributed_test.exs:101"] }, %{ name: "drop_final_snapshot_event_batch", file: "lib/group/replica.ex", correct_source: " _event_buffer = Snapshot.finish_event_buffer(event_buffer)", faulty_source: " _event_buffer = event_buffer", - test: ["test/replica_snapshot_distributed_test.exs:177"] + test: ["test/replica_snapshot_distributed_test.exs:223"] }, %{ name: "allow_duplicate_snapshot_rows", @@ -173,7 +173,7 @@ defmodule Group.MutationCampaign do faulty_source: """ if :ets.insert(table, objects) and size_before >= 0 do """, - test: ["test/replica_snapshot_distributed_test.exs:339"] + test: ["test/replica_snapshot_distributed_test.exs:385"] }, %{ name: "do_not_supersede_partial_snapshot", @@ -187,7 +187,7 @@ defmodule Group.MutationCampaign do " %{snapshot_seq: existing_seq} when existing_seq < snapshot_seq ->\n" <> " _ = existing_seq\n" <> " {:ignore, state}", - test: ["test/replica_snapshot_distributed_test.exs:282"] + test: ["test/replica_snapshot_distributed_test.exs:328"] }, %{ name: "accept_stale_snapshot_authority", @@ -203,7 +203,7 @@ defmodule Group.MutationCampaign do snapshot_seq > Data.replica_cursor(state.name, state.shard_index, stream_id) end """, - test: ["test/replica_snapshot_distributed_test.exs:510"] + test: ["test/replica_snapshot_distributed_test.exs:556"] }, %{ name: "disable_snapshot_staging_expiry", @@ -224,7 +224,7 @@ defmodule Group.MutationCampaign do acc end """, - test: ["test/replica_snapshot_distributed_test.exs:436"] + test: ["test/replica_snapshot_distributed_test.exs:482"] }, %{ name: "disable_below_floor_snapshot", @@ -440,7 +440,7 @@ defmodule Group.MutationCampaign do state end """, - test: ["test/replica_snapshot_distributed_test.exs:680"] + test: ["test/anti_entropy_fault_regression_test.exs:3823"] }, %{ name: "skip_generation_purge", @@ -921,7 +921,7 @@ defmodule Group.MutationCampaign do _chunk_resume -> {:chunk, 1} end """, - test: ["test/replica_snapshot_distributed_test.exs:831"] + test: ["test/replica_snapshot_distributed_test.exs:877"] }, %{ name: "drain_oversized_ingress_batch_without_yield", @@ -1085,6 +1085,11 @@ defmodule Group.MutationCampaign do "--exclude=.git", "--exclude=deps", "--exclude=tmp", + # Jepsen histories and caches are runtime artifacts. Copying them into + # every mutant can multiply a long soak's disk usage by the number of + # mutations without contributing anything to compilation or tests. + "--exclude=test/jepsen/store", + "--exclude=test/jepsen/.cache", "#{@repo}/", "#{target}/" ], @@ -1096,7 +1101,11 @@ defmodule Group.MutationCampaign do defp run_test(directory, test) do run_with_timeout(directory, ["mix", "test" | test], - env: [{"GROUP_MODEL_RUNS", "1"}, {"GROUP_MODEL_COMMANDS", "8"}] + env: [ + {"GROUP_MODEL_RUNS", "1"}, + {"GROUP_MODEL_COMMANDS", "8"}, + {"GROUP_JEPSEN_SKIP_CHECKER", "1"} + ] ) end diff --git a/test/replica_snapshot_distributed_test.exs b/test/replica_snapshot_distributed_test.exs index ae2691a..e94a736 100644 --- a/test/replica_snapshot_distributed_test.exs +++ b/test/replica_snapshot_distributed_test.exs @@ -52,6 +52,52 @@ defmodule Group.ReplicaSnapshotDistributedTest do end) end + test "exact PG install repairs an impossible conflicting stored origin", context do + %{name: name, node_a: node_a, node_b: node_b, node_c: node_c} = start_pair(context) + :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) + + key = "snapshot/pg-origin-collision" + member = TestCluster.spawn_join(node_a, name, key, %{source: true}) + + # Move the source stream floor past the membership so requesting sequence + # one necessarily exercises an exact snapshot rather than a delta. + for index <- 1..4 do + TestCluster.spawn_register(node_a, name, "snapshot/pg-origin-filler/#{index}", %{}) + end + + TestCluster.flush_shards(node_a, name) + stream_id = local_stream(node_a, name, nil) + {chunks, commit} = capture_snapshot_with_commit(node_a, node_b, name, stream_id, 1) + + :ok = + TestCluster.rpc!(node_b, Group.Replica.Data, :pg_insert_many, [ + name, + 0, + [{nil, key, member, %{corrupt: true}, 0, node_c}] + ]) + + shard = TestCluster.rpc!(node_b, Process, :whereis, [Group.Replica.shard_name(name, 0)]) + monitor = Process.monitor(shard) + + deliver_frames(node_b, node_a, name, chunks ++ [commit]) + TestCluster.flush_shards(node_b, name) + + refute_receive {:DOWN, ^monitor, :process, ^shard, _reason}, 250 + assert TestCluster.rpc!(node_b, Process, :alive?, [shard]) + assert [{^member, %{source: true}}] = TestCluster.rpc!(node_b, Group, :members, [name, key]) + + assert {%{source: true}, _time, ^node_a} = + TestCluster.rpc!(node_b, Group.Replica.Data, :pg_lookup, [ + name, + 0, + nil, + key, + member + ]) + + assert :ok = TestCluster.rpc!(node_b, Group.TestCluster, :assert_replica_consistent, [name]) + end + test "a source mutation during a single-pass scan prevents terminal commit", context do %{name: name, node_a: node_a, node_b: node_b} = start_pair(context) :ok = TestCluster.rpc!(node_a, Group.TestReplicaTransport, :set_mode, [name, :drop]) From 898d3ec1023e0f3c7dece42175454ca9bd646a0d Mon Sep 17 00:00:00 2001 From: Chris McCord Date: Mon, 17 Aug 2026 14:38:39 +0000 Subject: [PATCH 15/16] Bound anti-entropy soak execution --- mix.exs | 6 +++++- test/jepsen/campaign.sh | 10 +++++++++- test/jepsen/src/group/jepsen/core.clj | 17 ++++++++++++++--- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/mix.exs b/mix.exs index 0c6d68d..f9b9580 100644 --- a/mix.exs +++ b/mix.exs @@ -66,7 +66,11 @@ defmodule Group.MixProject do [ test: ["test", "cmd test/jepsen/checker.sh"], "test.soak": [ - "test", + # Run the PR gate in a child VM. test_helper starts distribution, and + # keeping that VM alive for the following `cmd` phases can retain a + # fixed ERL_AFLAGS distribution port and make qualification fail with + # eaddrinuse. + "cmd mix test", "cmd env GROUP_JEPSEN_SKIP_CHECKER=1 test/jepsen/qualify.sh", "cmd env GROUP_JEPSEN_SKIP_CHECKER=1 test/jepsen/campaign.sh" ] diff --git a/test/jepsen/campaign.sh b/test/jepsen/campaign.sh index 28a6492..179d4cb 100755 --- a/test/jepsen/campaign.sh +++ b/test/jepsen/campaign.sh @@ -10,6 +10,7 @@ concurrency="${GROUP_JEPSEN_CAMPAIGN_CONCURRENCY:-4n}" owner_count="${GROUP_JEPSEN_CAMPAIGN_OWNERS:-128}" key_count="${GROUP_JEPSEN_CAMPAIGN_KEYS:-32}" recovery_time="${GROUP_JEPSEN_CAMPAIGN_RECOVERY:-15}" +profile_grace="${GROUP_JEPSEN_CAMPAIGN_PROFILE_GRACE:-90}" artifact_dir="${GROUP_JEPSEN_CAMPAIGN_ARTIFACT_DIR:-}" cd "${repo_dir}" @@ -42,7 +43,14 @@ for transport in distribution tcp chaos; do echo "Starting ${transport}/${scenario}: ${test_count} histories x ${time_limit}s (sender buffer ${sender_buffer_size})" - if GROUP_JEPSEN_SENDER_BUFFER_SIZE="${sender_buffer_size}" "${script_dir}/run.sh" test \ + # Jepsen's active generator is time-limited, but setup, recovery, checker, + # and bugs in a terminal generator live outside that limit. Keep the + # nightly gate itself bounded as a final defense against hung campaigns. + profile_timeout=$((test_count * (time_limit + recovery_time + profile_grace))) + + if GROUP_JEPSEN_SENDER_BUFFER_SIZE="${sender_buffer_size}" timeout \ + --signal=TERM --kill-after=30 "${profile_timeout}" \ + "${script_dir}/run.sh" test \ --no-ssh \ --nodes n1,n2,n3 \ --concurrency "${concurrency}" \ diff --git a/test/jepsen/src/group/jepsen/core.clj b/test/jepsen/src/group/jepsen/core.clj index 6130aa9..9b737b1 100644 --- a/test/jepsen/src/group/jepsen/core.clj +++ b/test/jepsen/src/group/jepsen/core.clj @@ -91,6 +91,19 @@ (gen/once read) (gen/until-ok (repeat read))))))) +(defn recovery-connect-round [opts] + ;; A dead node can reject connections immediately. An unpaced until-ok loop + ;; then creates an unbounded history after the active workload's time limit + ;; has already elapsed. Bound terminal recovery too: failure to reconnect is + ;; evidence for an invalid history, not a reason for the harness to run + ;; forever. + (gen/clients + (gen/each-thread + (->> (repeat {:f :connect-all, :value {}}) + (gen/stagger 0.1) + gen/until-ok + (gen/time-limit (:recovery-time opts)))))) + (defn terminal-phases [opts] (let [permanent? (= "permanent" (:scenario opts)) corruption (keyword (:corruption opts))] @@ -99,9 +112,7 @@ (gen/nemesis {:type :info, :f :restart-node}) (gen/nemesis {:type :info, :f :partition-stop}) (gen/nemesis {:type :info, :f :replica-partition-stop}) - (gen/clients - (gen/each-thread - (gen/until-ok (repeat {:f :connect-all, :value {}})))) + (recovery-connect-round opts) (gen/sleep (:recovery-time opts))] permanent? (conj (gen/log "Retiring n1 permanently and waiting for complete eviction") From 2e5e7fe73209f53b397a74312d79ea4771e2c89b Mon Sep 17 00:00:00 2001 From: Chris McCord Date: Mon, 17 Aug 2026 19:40:15 +0000 Subject: [PATCH 16/16] Clarify authority hint projection semantics --- lib/group/replica/data.ex | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/group/replica/data.ex b/lib/group/replica/data.ex index 69439bc..8964999 100644 --- a/lib/group/replica/data.ex +++ b/lib/group/replica/data.ex @@ -2721,10 +2721,11 @@ defmodule Group.Replica.Data do not is_nil(hint_generation) and WireProtocol.generation_newer?(generation, hint_generation) -> - # This one shared row is the cross-lane fence. Public readers include - # it in replica_view_current?/2, so no lane can admit the prior + # This one shared row is the cross-lane admission fence. Replica lanes + # include it in replica_view_current?/2, so no lane can admit the prior # generation after this insert even while the per-lane breadcrumbs - # below are being updated. + # below are being updated. Existing materialized projections may remain + # visible until exact-authority repair or bounded lease retirement. put_remote_authority_hint(state.name, remote_node, generation, revision) # Preserve each lane's old generation as the later exact install's