From 6d810999a8e7a243d7673cc3178ff76d2da7a114 Mon Sep 17 00:00:00 2001 From: Vladimir Babin Date: Mon, 13 Jul 2026 19:16:18 +0800 Subject: [PATCH 1/5] feat(snapshot): post-import completeness/invariant checks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The payload checksum only proves the download matches what the serving node serialized; it cannot detect that the serving node serialized INCOMPLETE state. A single missing account on the serving side passes the checksum, imports cleanly, then wedges the node the moment a canonical block references that account (2026-07-13 rpc.viz.cx incident: out_of_range "viz-social-bot" at database get_account). Add three post-import checks inside the import write-lock: - per-section object-count reconciliation: header.object_counts (recorded at export from the serialized section arrays) vs the live index sizes for all multi-instance indices. Catches a lossy import. - referential integrity: every account_authority must reference an existing account (the exact invariant the wedge violated). - critical singletons present: dynamic_global_property, validator_schedule, hardfork_property. On failure FC_ASSERT aborts the import — loud and retryable: the P2P snapshot-sync path rejects the incomplete snapshot and tries another trusted peer instead of silently starting on corrupt state. --- plugins/snapshot/plugin.cpp | 110 ++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/plugins/snapshot/plugin.cpp b/plugins/snapshot/plugin.cpp index e5b9e0c63c..8f4327451c 100644 --- a/plugins/snapshot/plugin.cpp +++ b/plugins/snapshot/plugin.cpp @@ -1777,6 +1777,116 @@ void snapshot_plugin::plugin_impl::load_snapshot(const fc::path& input_path) { << " (was " << old_lib << ") for P2P sync\n"; } + // ── Post-import completeness / invariant verification (Finding A) ── + // The payload_checksum only proves the download matches what the serving + // node serialized; it cannot detect that the serving node serialized + // INCOMPLETE state. A single missing account on the serving side would + // pass the checksum, import cleanly, and then wedge the node the moment a + // canonical block references that account (the 2026-07-13 rpc.viz.cx + // incident: out_of_range "viz-social-bot" at database.cpp get_account). + // + // These checks catch a lossy import (serialized N, imported M) and gross + // referential holes regardless of cause, turning a silent bad-state + // acceptance into a loud, retryable FC_ASSERT — the P2P-sync path then + // rejects the snapshot and retries another trusted peer. + { + uint32_t invariant_failures = 0; + + // (1) Per-section object-count reconciliation. header.object_counts + // is recorded at export from the serialized section-array sizes; every + // multi-instance index is cleared before import (see clear block above), + // so the live index size must equal the recorded count. A mismatch + // means the import silently dropped objects. + auto expect_count = [&](const std::string& section, size_t actual) { + auto itr = header.object_counts.find(section); + if (itr == header.object_counts.end()) return; // section absent from this snapshot + if (itr->second != actual) { + elog(CLOG_RED "Snapshot completeness: section '${s}' count mismatch — " + "header=${h}, imported=${a}" CLOG_RESET, + ("s", section)("h", itr->second)("a", static_cast(actual))); + ++invariant_failures; + } + }; + + #define CHECK_SECTION(section, index_type) \ + expect_count(section, db.get_index().indices().size()); + + CHECK_SECTION("account", account_index) + CHECK_SECTION("account_authority", account_authority_index) + CHECK_SECTION("validator", validator_index) + CHECK_SECTION("validator_vote", validator_vote_index) + CHECK_SECTION("block_summary", block_summary_index) + CHECK_SECTION("content", content_index) + CHECK_SECTION("content_vote", content_vote_index) + CHECK_SECTION("block_post_validation", validator_confirmation_index) + CHECK_SECTION("transaction", transaction_index) + CHECK_SECTION("vesting_delegation", vesting_delegation_index) + CHECK_SECTION("vesting_delegation_expiration", vesting_delegation_expiration_index) + CHECK_SECTION("fix_vesting_delegation", fix_vesting_delegation_index) + CHECK_SECTION("withdraw_vesting_route", withdraw_vesting_route_index) + CHECK_SECTION("escrow", escrow_index) + CHECK_SECTION("proposal", proposal_index) + CHECK_SECTION("required_approval", required_approval_index) + CHECK_SECTION("committee_request", committee_request_index) + CHECK_SECTION("committee_vote", committee_vote_index) + CHECK_SECTION("invite", invite_index) + CHECK_SECTION("award_shares_expire", award_shares_expire_index) + CHECK_SECTION("paid_subscription", paid_subscription_index) + CHECK_SECTION("paid_subscribe", paid_subscribe_index) + CHECK_SECTION("validator_penalty_expire", validator_penalty_expire_index) + CHECK_SECTION("content_type", content_type_index) + CHECK_SECTION("account_metadata", account_metadata_index) + CHECK_SECTION("master_authority_history", master_authority_history_index) + CHECK_SECTION("account_recovery_request", account_recovery_request_index) + CHECK_SECTION("change_recovery_account_request", change_recovery_account_request_index) + + #undef CHECK_SECTION + + // (2) Referential integrity: every account_authority must reference an + // existing account. This is the exact invariant the wedge incident + // violated (a canonical block referenced an account absent from state). + { + uint32_t dangling = 0; + const auto& auth_idx = db.get_index().indices(); + for (auto itr = auth_idx.begin(); itr != auth_idx.end(); ++itr) { + if (db.find_account(itr->account) == nullptr) { + if (dangling < 20) + elog(CLOG_RED "Snapshot completeness: account_authority references missing " + "account '${a}'" CLOG_RESET, ("a", itr->account)); + ++dangling; + } + } + if (dangling > 0) { + elog(CLOG_RED "Snapshot completeness: ${n} account_authority record(s) reference " + "missing accounts" CLOG_RESET, ("n", dangling)); + ++invariant_failures; + } + } + + // (3) Critical singletons must be present. + if (db.find() == nullptr) { + elog(CLOG_RED "Snapshot completeness: dynamic_global_property singleton missing" CLOG_RESET); + ++invariant_failures; + } + if (db.find() == nullptr) { + elog(CLOG_RED "Snapshot completeness: validator_schedule singleton missing" CLOG_RESET); + ++invariant_failures; + } + if (db.find() == nullptr) { + elog(CLOG_RED "Snapshot completeness: hardfork_property singleton missing" CLOG_RESET); + ++invariant_failures; + } + + FC_ASSERT(invariant_failures == 0, + "Snapshot import failed post-import invariant checks (${n} failure(s)); the snapshot " + "is incomplete. Refusing to start on corrupt state — will retry another trusted peer.", + ("n", invariant_failures)); + + ilog(CLOG_ORANGE "Snapshot completeness checks passed (${a} accounts, ${w} validators)" CLOG_RESET, + ("a", db.get_index().indices().size()) + ("w", db.get_index().indices().size())); + } + ilog(CLOG_ORANGE "All objects imported successfully" CLOG_RESET); } catch (const fc::exception& e) { elog(CLOG_RED "Snapshot import failed with fc::exception: ${e}" CLOG_RESET, ("e", e.to_detail_string())); From 455a767d67ea802b3c56cf38db9522147bce6c5f Mon Sep 17 00:00:00 2001 From: Vladimir Babin Date: Mon, 13 Jul 2026 19:16:34 +0800 Subject: [PATCH 2/5] feat(p2p): self-healing wedged-behind-network watchdog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A snapshot-synced node whose state is missing an object rejects every canonical block that references it (out_of_range at apply_block). This is a rejection livelock, distinct from the #117-#120 write-lock deadlock monitors: push_block fails fast, so those monitors never fire. The divergence sits at/below LIB, so fork-switch cannot recover — only wipe + snapshot re-import does, and that was gated on empty state, so it never self-triggered. The 2026-07-13 rpc.viz.cx outage needed a manual docker compose --force-recreate after ~14h frozen. Detection (dlt_p2p_node::check_wedge_watchdog, in periodic_task): declare WEDGED when, sustained for WEDGE_CONFIRM_SEC (900s): behind > WEDGE_BEHIND_THRESHOLD (200) AND head flat AND gap-fill rejections still climbing. The triple-AND rules out a healthy syncing node (head advancing) and a partition (behind+flat but no rejections). A new monotonic _gap_rejected_total drives the "still rejecting" signal because _gap_rejected_count oscillates (resets on blacklist). The pure predicate is_wedged() is factored out for table-testing. Action (gated behind auto-resync-on-wedge, default OFF for this release so we observe the elog/marker in the wild first): elog, write a force_resync marker next to shared_memory.bin, then std::_Exit(2) (precedent: undo_all watchdog). With OFF, the watchdog only logs. Recovery (chain plugin_startup): a force_resync marker wipes state so head becomes 0, and the existing empty-state gate re-bootstraps from the trusted snapshot peer, then deletes the marker — the exact path a manual --force-recreate triggers today. The marker is removed only after the wipe, so a crash mid-wipe re-triggers recovery; it composes with the resize marker (wipe clears it too). The marker lives in the shared-memory dir (exposed via chain::plugin::get_state_dir) so it persists exactly as long as the state it condemns. --- docs/self-heal-wedge-watchdog-plan.md | 223 ++++++++++++++++++ libraries/network/dlt_p2p_node.cpp | 140 +++++++++++ .../include/graphene/network/dlt_p2p_node.hpp | 48 ++++ .../include/graphene/plugins/chain/plugin.hpp | 5 + plugins/chain/plugin.cpp | 32 +++ plugins/p2p/p2p_plugin.cpp | 23 +- 6 files changed, 470 insertions(+), 1 deletion(-) create mode 100644 docs/self-heal-wedge-watchdog-plan.md diff --git a/docs/self-heal-wedge-watchdog-plan.md b/docs/self-heal-wedge-watchdog-plan.md new file mode 100644 index 0000000000..8eaae0f5ea --- /dev/null +++ b/docs/self-heal-wedge-watchdog-plan.md @@ -0,0 +1,223 @@ +# Plan: self-healing "wedged-behind-network" watchdog + snapshot-import completeness + +Status: **DRAFT / not implemented.** Design for a fresh implementation session. +Author context: written after the 2026-07-13 rpc.viz.cx stuck-node incident. + +--- + +## 1. The incident this prevents + +rpc.viz.cx (a snapshot-synced RPC node) froze at head **81,579,861** for ~14h while the +network advanced to ~81,597,000. Root symptom in the logs: + +``` +_apply_block ] 13 out_of_range: unknown key {"name":"viz-social-bot"} database.cpp:1469 get_account +Failed to push new block next_block.block_num()=81579862 +Gap fill: block #81579862 rejected 3 times — blacklisting ... (validator key likely invalid on this fork) +``` + +The node's committed state was **missing account `viz-social-bot`**, so it could not apply +the canonical block that did a `withdraw_vesting` for that account — and every subsequent +canonical block the same way. The node interpreted the real chain as a "fork with an invalid +validator key" and blacklisted the peers serving it. Head never advanced again. + +Recovery required a **manual** `docker compose up -d --force-recreate vizd` (fresh writable +layer → snapshot re-sync). This class of outage recurs (see the 2026-06-17 and 2026-06-22 +dead-fork incidents) and always needs a human. + +### Why the existing guards did not catch it + +The `#117–#120` machinery (`push_block_appears_stalled()`, the undo_all watchdog at +`libraries/chain/database.cpp:302-334`, the push_block monitor at `database.cpp:1128`) +detects a **write-lock deadlock**: `push_block` *held* past `PUSH_BLOCK_STALL_WARN_SEC` +with `_apply_progress` not moving. Our failure is the opposite — a **rejection livelock**: +`push_block` fails *fast* every time, so the lock is never held long and the deadlock +monitor never fires. Nothing currently watches "head frozen while peers are thousands of +blocks ahead and we keep rejecting their blocks." + +### Why it is permanent, not transient + +The divergence is at/below the node's **irreversible** head (head ≈ LIB at freeze time). +Fork switching recovers only *within* the reversible window via `pop_block`; you cannot pop +below LIB. So the bad state is baked in. The p2p recovery primitives that exist — +`resync_from_lib()` / `trigger_resync()` (`libraries/network/dlt_p2p_node.cpp:2460/2490`) — +only re-hello peers and re-request blocks; they never touch state. The **only** code path +that rebuilds state (`database::wipe()` + snapshot import) is gated on **empty state** +(`sync-snapshot-from-trusted-peer` = "load snapshot on empty state") or a newer snapshot +appearing. A node with corrupt-but-non-empty state never triggers it → stuck forever. + +--- + +## 2. Root-cause findings (snapshot import), read-only static analysis + +Investigated `plugins/snapshot/plugin.cpp` + `snapshot_serializer.hpp`. Findings, with +confidence levels. **None of these is proven to be the exact cause of the single missing +account** — that needs reproduction against the actual snapshot — but they are real gaps. + +### Finding A — no import-side *completeness* verification (HIGH confidence gap) + +- Export computes a SHA-256 over the serialized `state` JSON (`plugin.cpp:1410-1414`) and + import verifies it (`plugin.cpp:1408-1416`). **This only proves the download matches what + the serving node serialized.** If the serving node serialized *incomplete* state, the + checksum still passes — garbage-in is verified as valid garbage. +- After import there is **no invariant check** — e.g. "every `account` referenced by an + `account_authority` exists", "`dgp.current_supply` reconciles with summed balances", + "critical singletons present". A missing account would sail through. + +### Finding B — type-enum fragility, guarded for exactly ONE index (HIGH confidence, but wrong signature for this incident) + +- `plugin.cpp:982-992` documents the hazard: if the chainbase `object_type` enum shifts + (types added/removed before a given object's type id), `get_index()` can resolve to the + wrong/empty index and a whole section silently serializes empty — "Snapshot will be + INCOMPLETE." Only `validator_vote` is sanity-checked (validators>0 but votes==0). +- `account_index` has **no** equivalent guard. **However**, this bug empties a *whole* + section, not a single account — so it does not explain losing just `viz-social-bot`. Worth + fixing defensively, but it is not the incident's mechanism. + +### Finding C — undo / fork-switch interaction with snapshot-established state (MEDIUM confidence — most plausible for a *single* missing account) + +- After import, `fork_db` holds only the head block (`plugin.cpp:1724`), and + snapshot-established objects have no undo history. Recent commits target exactly this seam: + `ec57277 fix(snapshot): update undo logic ... during snapshot imports`, + `3b3b9ea fix(database): prevent undo_all() stall and handle corrupted shared memory`, + `1b08a0b`/`4f3e00d` (pop_block no-progress deadlocks). +- Plausible mechanism: an account created *after* the snapshot point, then a fork switch / + `pop_block` / `undo` sequence that removes it without the canonical branch re-creating it + — leaving a permanent hole. **Needs reproduction to confirm**; do not fix blind. + +### Recommended root-cause follow-ups (ranked, not yet actioned) + +1. **Add post-import invariant checks** (Finding A) — cheap, high value, catches *any* + incomplete import regardless of cause. `plugin.cpp` after the import block (~1740). +2. **Generalize the type-enum sanity check** (Finding B) to all critical indices, or better, + persist per-section object counts in the snapshot header and assert imported counts match. +3. **Reproduce Finding C** on a test net: snapshot at block N, create an account at N+k, force + a fork switch across N+k, verify the account survives. Only then design the undo fix. + +--- + +## 3. The fix: wedged-behind-network watchdog (primary deliverable) + +Turn the permanent manual-intervention outage into automatic self-heal within ~15–20 min. +The node already tracks everything needed — this wires it together and adds a recovery trigger. + +### 3.1 Where it lives + +`libraries/network/dlt_p2p_node.cpp`, in `periodic_task()` (the fiber loop started at +`dlt_p2p_node.cpp:165`, body around `:172`). The p2p node already has, at fiber cadence: + +- `_delegate->get_head_block_num()` — our head (used at `:1186`, `:1206`, etc.) +- `_highest_seen_block_num` — network tip as seen from peers (maintained at `:1462`, `:1647`) +- `_gap_rejected_count` / `_gap_rejected_blacklist_until` — gap-fill rejection signal + (`:1747-1748`, `:2206-2207`) + +### 3.2 Detection logic (all conditions must hold, sustained) + +Add a small watchdog struct evaluated each `periodic_task()` tick: + +``` +behind = _highest_seen_block_num - our_head // how far behind the network +head_stuck = our_head has not increased since last observation +rejecting = _gap_rejected_count increased since last observation // actively refusing the chain +``` + +Declare **WEDGED** when, continuously for `WEDGE_CONFIRM_SEC`: +- `behind > WEDGE_BEHIND_THRESHOLD` (e.g. **200** blocks), AND +- `head_stuck` (our_head flat), AND +- `rejecting` (gap-fill rejections still climbing — proves it is divergence, not just a + slow/partitioned link where we'd simply have no blocks to apply). + +Proposed constants (mirror the style of `PUSH_BLOCK_STALL_WARN_SEC` in +`database.hpp:725`), as `static constexpr` on `dlt_p2p_node`: + +``` +WEDGE_BEHIND_THRESHOLD = 200 // blocks behind network tip +WEDGE_CONFIRM_SEC = 900 // 15 min sustained before acting +WEDGE_RELOG_SEC = 60 // loud re-log cadence while wedged +``` + +Rationale for the AND of all three: a healthy node that is merely *syncing* is behind but +`our_head` is *advancing* (fails `head_stuck`). A network partition leaves us behind and +stuck but with **no** rejections (fails `rejecting`) — for that we just keep retrying peers, +we must not wipe good state. Only genuine state divergence produces behind + stuck + +actively-rejecting simultaneously. + +### 3.3 Action on confirmed WEDGE + +1. `elog` a loud, unambiguous line (behind count, our_head, highest_seen, reject count). +2. Write a `force_resync` marker file into the state dir (next to `shared_memory.bin`), + containing the reason + timestamps for the post-mortem. +3. `std::_Exit(N)` with a distinct code — precedent: the undo_all watchdog already does + `std::_Exit(1)` at `database.cpp:334`. Exiting from the fiber is safe; do **not** try to + wipe+reimport in-process (that path is fragile mid-run). + +### 3.4 Recovery path (on next start) + +The snapshot plugin decides at open whether to bootstrap. Extend that decision (the +`open_from_snapshot` / "empty state" gate in `plugins/snapshot/plugin.cpp`) so that: + +> if a `force_resync` marker exists → treat state as needing re-bootstrap: run the existing +> `database::wipe()` (`database.cpp:1070`) + snapshot import from `trusted-snapshot-peer`, +> then delete the marker. + +This reuses the exact path a `--force-recreate` triggers today, so it is well-exercised. +Combined with a container `restart: unless-stopped` policy (already set in the compose file), +`std::_Exit` → restart → marker seen → wipe + snapshot-resync → back at tip. No operator, no +`docker compose` call. + +> IMPLEMENTATION NOTE — verify the exact open-time bootstrap gate before wiring the marker. +> Confirm where `open_from_snapshot` vs. open-existing is chosen and that `database::wipe()` +> is callable there. Also confirm the marker path survives container restart on *our* deploy +> (state is in the container writable layer for rpc.viz.cx; on validators it may be a +> volume — the marker must live wherever `shared_memory.bin` lives so it persists exactly as +> long as the state it condemns). + +### 3.5 Safety / edge cases + +- **False positive → needless resync.** Worst case is an unnecessary snapshot re-bootstrap + (minutes of downtime), not data loss. The triple-AND + 15-min confirm makes this rare. +- **Validators (`id`, `babin`).** They run the same binary. A wedged validator is *already* + missing blocks; auto-resync is strictly better than silent missed-block deactivation. But + a validator mid-resync must not sign — confirm block production stays disabled until the + node is fork-aligned again after import (should already hold via SYNC state, verify). +- **Snapshot-serving.** rpc.viz.cx also *serves* snapshots (`allow-snapshot-serving=true`). + Ensure a node that just wiped does not serve a half-imported snapshot (serving should be + gated on fork-aligned/normal status — verify). +- **Resize interplay.** There is already a `resize_in_progress` marker (`database.cpp:273`) + and shared-file resize logic. Make sure `force_resync` handling composes with it (don't + wipe mid-resize). + +--- + +## 4. Testing + +- **Unit-ish:** factor the wedge predicate into a pure function of + `(behind, head_advanced, rejects_advanced, elapsed)` and table-test the state machine + (syncing → not wedged; partition → not wedged; divergence → wedged after confirm). +- **Integration (testnet):** force a node onto a dead branch below LIB (import a stale/partial + snapshot), point it at healthy peers, assert: watchdog fires within `WEDGE_CONFIRM_SEC`, + marker written, process exits, restart re-bootstraps, head reaches tip. +- **Negative:** partition the node (drop peer connections) → assert it does **not** wipe. + +--- + +## 5. Rollout + +1. Land watchdog + marker recovery behind a config flag `auto-resync-on-wedge` (default + **off** for one release; observe the `elog`/marker in the wild without the `_Exit`). +2. After confirming detection has zero false positives on all three of our nodes + (rpc.viz.cx, axveer `id`, martin `babin`), flip default on. +3. Separately land the Finding A post-import invariant checks — independent, lower risk, + ships first. + +--- + +## 6. Open questions for the implementer + +- Exact location + signature of the open-time snapshot-bootstrap gate (§3.4). +- Does `_delegate->get_head_block_num()` reflect *irreversible* progress or just head? For + `head_stuck` we want head; fine. But consider also gating on LIB not advancing to avoid + firing during a legitimate deep reorg. +- Should the threshold be block-count (200) or time-based (network tip timestamp vs. our head + timestamp)? Time-based ("our head is >10 min behind wall clock per peer consensus") may be + more robust than a raw block delta. Decide during implementation. diff --git a/libraries/network/dlt_p2p_node.cpp b/libraries/network/dlt_p2p_node.cpp index 4910b60728..432f2d9b09 100644 --- a/libraries/network/dlt_p2p_node.cpp +++ b/libraries/network/dlt_p2p_node.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -34,6 +35,9 @@ constexpr uint32_t dlt_p2p_node::FORWARD_STAGNATION_SEC; constexpr uint32_t dlt_p2p_node::ISOLATION_RESET_SEC; constexpr uint32_t dlt_p2p_node::GAP_REJECT_BLACKLIST_SEC; constexpr uint32_t dlt_p2p_node::BLOCKED_IP_DURATION_SEC; +constexpr uint32_t dlt_p2p_node::WEDGE_BEHIND_THRESHOLD; +constexpr uint32_t dlt_p2p_node::WEDGE_CONFIRM_SEC; +constexpr uint32_t dlt_p2p_node::WEDGE_RELOG_SEC; // ── Construction / destruction ─────────────────────────────────────── @@ -83,6 +87,17 @@ void dlt_p2p_node::set_witness_diag_provider(std::function fn) { _witness_diag_provider = std::move(fn); } +void dlt_p2p_node::set_state_dir(const std::string& dir) { _state_dir = dir; } + +void dlt_p2p_node::set_auto_resync_on_wedge(bool enabled) { + _auto_resync_on_wedge = enabled; + if (enabled) + ilog("DLT P2P: auto-resync-on-wedge ENABLED — a confirmed network wedge will " + "write a force_resync marker and exit for supervised snapshot re-bootstrap"); + else + ilog("DLT P2P: auto-resync-on-wedge disabled — wedge watchdog will elog only (no auto-exit)"); +} + void dlt_p2p_node::block_incoming_ip(uint32_t ip, const std::string& reason) { // NAT safety: if multiple active peers share this IP (nodes behind the same NAT), // blocking the IP would kill all of them. Only block when a single peer is using @@ -1777,6 +1792,10 @@ void dlt_p2p_node::on_dlt_block_reply(peer_id peer, const dlt_block_reply_messag if (result == dlt_block_accept_result::REJECTED) { wlog(DLT_LOG_RED "Rejected block #${n} from ${ep}" DLT_LOG_RESET, ("n", block_num)("ep", state.endpoint)); + // Monotonic total (never reset) — the wedge watchdog uses this to prove + // rejections are still climbing. _gap_rejected_count oscillates + // (resets to 0 on blacklist), so it cannot be used for a "still rejecting" signal. + _gap_rejected_total++; // Track rejected blocks to prevent gap fill infinite retry loop if (block_num == _gap_rejected_block_num) { _gap_rejected_count++; @@ -2236,6 +2255,8 @@ void dlt_p2p_node::on_dlt_gap_fill_reply(peer_id peer, const dlt_gap_fill_reply& } else { wlog(DLT_LOG_RED "Gap fill: rejected block #${n} from ${ep}" DLT_LOG_RESET, ("n", block.block_num())("ep", it->second.endpoint)); + // Monotonic total (never reset) — see note at the other rejection site. + _gap_rejected_total++; // Track rejected blocks to prevent infinite retry loop if (block.block_num() == _gap_rejected_block_num) { _gap_rejected_count++; @@ -3741,6 +3762,122 @@ void dlt_p2p_node::periodic_known_peers_cleanup() { } } +void dlt_p2p_node::check_wedge_watchdog() { + if (!_delegate) return; + + uint32_t our_head = _delegate->get_head_block_num(); + if (our_head == 0) return; // not bootstrapped yet (or mid snapshot import) + + uint32_t behind = (_highest_seen_block_num > our_head) + ? (_highest_seen_block_num - our_head) : 0; + + auto now = fc::time_point::now(); + + // "Stuck behind" gate: far enough behind the network tip to be suspicious. + // If we're not this far behind, the node is healthy or merely lagging a + // little — no window. + if (behind <= WEDGE_BEHIND_THRESHOLD) { + if (_wedge_since != fc::time_point()) { + ilog(DLT_LOG_GREEN "Wedge watchdog: caught up (behind=${b} <= ${thr}), clearing wedge timer " + "(our_head=#${h})" DLT_LOG_RESET, + ("b", behind)("thr", WEDGE_BEHIND_THRESHOLD)("h", our_head)); + } + _wedge_since = fc::time_point(); + return; + } + + // Behind this tick. Open the window if not already open. + if (_wedge_since == fc::time_point()) { + _wedge_since = now; + _wedge_window_head = our_head; + _wedge_reject_baseline = _gap_rejected_total; + _wedge_last_relog = now; + elog(DLT_LOG_RED "Wedge watchdog: node is ${b} blocks behind network tip " + "(our_head=#${h}, network_tip=#${t}) — watching for a rejection livelock. " + "Will confirm over ${c}s if head stays flat and rejections keep climbing." DLT_LOG_RESET, + ("b", behind)("h", our_head)("t", _highest_seen_block_num)("c", WEDGE_CONFIRM_SEC)); + return; + } + + // Window is open. Head advancing at all means we are syncing, not wedged — + // reset and re-arm (we're still behind, so a fresh window opens next tick). + bool head_advanced = (our_head != _wedge_window_head); + if (head_advanced) { + ilog(DLT_LOG_GREEN "Wedge watchdog: head advanced (#${p} → #${h}) while behind — syncing, not wedged; " + "resetting timer" DLT_LOG_RESET, ("p", _wedge_window_head)("h", our_head)); + _wedge_since = fc::time_point(); + return; + } + + bool rejects_climbed = (_gap_rejected_total > _wedge_reject_baseline); + uint32_t elapsed_sec = static_cast((now - _wedge_since).count() / 1000000LL); + + // Loud periodic re-log while the wedge builds toward confirmation. + if ((now - _wedge_last_relog).count() >= int64_t(WEDGE_RELOG_SEC) * 1000000LL) { + _wedge_last_relog = now; + elog(DLT_LOG_RED "Wedge watchdog: head flat at #${h} for ${s}s / ${c}s — behind=${b} " + "(network_tip=#${t}), rejections ${rc} (total=${r})" DLT_LOG_RESET, + ("h", our_head)("s", elapsed_sec)("c", WEDGE_CONFIRM_SEC)("b", behind) + ("t", _highest_seen_block_num) + ("rc", rejects_climbed ? "climbing" : "flat")("r", _gap_rejected_total)); + } + + if (!is_wedged(behind, head_advanced, rejects_climbed, elapsed_sec)) return; + + // ── Confirmed wedge ────────────────────────────────────────── + elog(DLT_LOG_RED "Wedge watchdog: CONFIRMED network wedge — head frozen at #${h} for ${s}s while " + "${b} blocks behind (network_tip=#${t}), gap rejections total=${r}. State has diverged at/below " + "LIB; fork-switch cannot recover — only wipe + snapshot re-import can." DLT_LOG_RESET, + ("h", our_head)("s", elapsed_sec)("b", behind)("t", _highest_seen_block_num)("r", _gap_rejected_total)); + + if (!_auto_resync_on_wedge) { + elog(DLT_LOG_RED "Wedge watchdog: auto-resync-on-wedge is OFF — NOT exiting. Enable " + "'auto-resync-on-wedge = true' for automatic snapshot re-bootstrap. Manual recovery: " + "restart with a fresh state dir (docker compose up -d --force-recreate)." DLT_LOG_RESET); + // Keep the window open but re-arm the relog timer so we emit the CONFIRMED + // line at WEDGE_RELOG_SEC cadence rather than every 5s tick. + _wedge_last_relog = now; + return; + } + + // Write the force_resync marker next to shared_memory.bin so the supervised + // restart re-bootstraps from a trusted snapshot (see chain plugin_startup). + bool marker_written = false; + if (!_state_dir.empty()) { + std::string marker_path = _state_dir; + if (marker_path.back() != '/' && marker_path.back() != '\\') marker_path += '/'; + marker_path += "force_resync"; + try { + std::ofstream f(marker_path); + f << "reason=wedged-behind-network\n" + << "our_head=" << our_head << "\n" + << "network_tip=" << _highest_seen_block_num << "\n" + << "behind=" << behind << "\n" + << "gap_rejected_total=" << _gap_rejected_total << "\n" + << "sustained_sec=" << elapsed_sec << "\n" + << "detected_at_us=" << now.time_since_epoch().count() << "\n"; + f.flush(); + marker_written = f.good(); + } catch (...) { + marker_written = false; + } + if (marker_written) + elog(DLT_LOG_RED "Wedge watchdog: wrote force_resync marker at ${p}" DLT_LOG_RESET, ("p", marker_path)); + else + elog(DLT_LOG_RED "Wedge watchdog: FAILED to write force_resync marker at ${p}" DLT_LOG_RESET, ("p", marker_path)); + } else { + elog(DLT_LOG_RED "Wedge watchdog: no state dir configured — cannot write force_resync marker. " + "Exiting anyway; operator must re-bootstrap manually." DLT_LOG_RESET); + } + + elog(DLT_LOG_RED "Wedge watchdog: exiting process (code 2) for supervised snapshot re-bootstrap." DLT_LOG_RESET); + std::cerr << "FATAL: node wedged behind network (behind=" << behind + << ", head=" << our_head << ", tip=" << _highest_seen_block_num + << "). Wrote force_resync marker=" << (marker_written ? "yes" : "no") + << "; exiting for snapshot re-bootstrap." << std::endl; + std::_Exit(2); +} + void dlt_p2p_node::periodic_task() { if (!_dead_fibers.empty()) { std::vector> to_clean; @@ -3810,6 +3947,9 @@ void dlt_p2p_node::periodic_task() { try { request_gap_fill(); } // P36 fix: fill gaps via exchange-enabled peers catch (const fc::exception& e) { wlog("request_gap_fill: ${e}", ("e", e.to_detail_string())); } catch (const std::exception& e) { wlog("request_gap_fill: ${e}", ("e", std::string(e.what()))); } + try { check_wedge_watchdog(); } // self-heal: detect rejection-livelock wedge behind the network + catch (const fc::exception& e) { wlog("check_wedge_watchdog: ${e}", ("e", e.to_detail_string())); } + catch (const std::exception& e) { wlog("check_wedge_watchdog: ${e}", ("e", std::string(e.what()))); } try { periodic_peer_exchange(); } catch (const fc::exception& e) { wlog("periodic_peer_exchange: ${e}", ("e", e.to_detail_string())); } catch (const std::exception& e) { wlog("periodic_peer_exchange: ${e}", ("e", std::string(e.what()))); } diff --git a/libraries/network/include/graphene/network/dlt_p2p_node.hpp b/libraries/network/include/graphene/network/dlt_p2p_node.hpp index 772271e0bb..b4e512e3f1 100644 --- a/libraries/network/include/graphene/network/dlt_p2p_node.hpp +++ b/libraries/network/include/graphene/network/dlt_p2p_node.hpp @@ -116,6 +116,14 @@ class dlt_p2p_node { // both outbound (no requests sent) and inbound (empty reply returned). void set_isolated_peers(bool isolated); + // Wedged-behind-network watchdog configuration. + // _state_dir is the directory that holds shared_memory.bin — the force_resync + // marker must live there so it persists exactly as long as the state it + // condemns. auto_resync_on_wedge gates the destructive std::_Exit action: + // when OFF (default) the watchdog only elogs on a confirmed wedge. + void set_state_dir(const std::string& dir); + void set_auto_resync_on_wedge(bool enabled); + // Registers a callback that returns a compact witness-state string. // Called during FORWARD stagnation logs so the P2P layer can include // witness production state without taking a plugin dependency. @@ -282,6 +290,34 @@ class dlt_p2p_node { void periodic_task(); void block_validation_timeout(); + // ── Wedged-behind-network watchdog ─────────────────────────── + // Detects the "rejection livelock": our head frozen far behind the network + // tip while we actively reject the canonical chain — state divergence at or + // below LIB that fork-switching cannot repair (only wipe + snapshot re-import + // does). This is DISTINCT from the push_block write-lock deadlock monitors + // (#117–#120): there push_block is *held* too long; here it fails *fast* on + // every canonical block, so those monitors never fire. + void check_wedge_watchdog(); + + // Pure predicate for the wedge state machine — no I/O, table-testable. + // Aggregated over the sustained observation window: + // behind — how far our head trails the network tip (blocks) + // head_advanced — did our head move at all during the window + // rejects_climbed — did gap-fill rejections increase during the window + // elapsed_sec — how long the (behind && head-flat) condition has held + // Verdict CONFIRMED only when the node is far behind, its head never moved, + // it is still actively rejecting the chain, and this has been sustained. + // - a syncing node advances its head → head_advanced ⇒ not wedged + // - a partitioned node has no blocks/rejects → !rejects_climbed ⇒ not wedged + // - genuine divergence trips all three, and only after WEDGE_CONFIRM_SEC. + static bool is_wedged(uint32_t behind, bool head_advanced, + bool rejects_climbed, uint32_t elapsed_sec) { + return behind > WEDGE_BEHIND_THRESHOLD + && !head_advanced + && rejects_climbed + && elapsed_sec >= WEDGE_CONFIRM_SEC; + } + // ── Subnet diversity ───────────────────────────────────────── uint32_t count_peers_in_subnet(const fc::ip::address& addr) const; bool is_same_subnet(const fc::ip::address& a, const fc::ip::address& b) const; @@ -407,10 +443,22 @@ class dlt_p2p_node { // ── Gap fill rejection tracking ────────────────────────────── uint32_t _gap_rejected_block_num = 0; ///< Last block num rejected by gap fill uint32_t _gap_rejected_count = 0; ///< How many times that block was rejected + uint64_t _gap_rejected_total = 0; ///< Monotonic total rejections (never reset; wedge watchdog signal) static constexpr uint32_t GAP_REJECT_MAX_RETRIES = 3; ///< Give up after this many rejections of same block static constexpr uint32_t GAP_REJECT_BLACKLIST_SEC = 120; ///< Blacklist a permanently-rejected block for this long fc::time_point _gap_rejected_blacklist_until; ///< Don't gap-fill any block until this time + // ── Wedged-behind-network watchdog ─────────────────────────── + static constexpr uint32_t WEDGE_BEHIND_THRESHOLD = 200; ///< Blocks behind network tip to be considered "behind" + static constexpr uint32_t WEDGE_CONFIRM_SEC = 900; ///< Wedge condition must hold this long (15 min) before acting + static constexpr uint32_t WEDGE_RELOG_SEC = 60; ///< Loud re-log cadence while a wedge is building + bool _auto_resync_on_wedge = false; ///< Gate the destructive _Exit action (default OFF: elog only) + std::string _state_dir; ///< Directory holding shared_memory.bin (force_resync marker lives here) + fc::time_point _wedge_since; ///< When the (behind && head-flat) window began (unset when not behind) + uint32_t _wedge_window_head = 0; ///< our_head at window start — any change ⇒ head advanced ⇒ reset + uint64_t _wedge_reject_baseline = 0; ///< _gap_rejected_total at window start — a rise ⇒ still rejecting + fc::time_point _wedge_last_relog; ///< Last loud re-log while a wedge is building + // ── FORWARD stagnation ────────────────────────────────────── uint32_t _last_forward_head_num = 0; fc::time_point _last_forward_progress_time; ///< Last time our head advanced in FORWARD mode diff --git a/plugins/chain/include/graphene/plugins/chain/plugin.hpp b/plugins/chain/include/graphene/plugins/chain/plugin.hpp index 7aa5390d00..2fc38f1ee9 100644 --- a/plugins/chain/include/graphene/plugins/chain/plugin.hpp +++ b/plugins/chain/include/graphene/plugins/chain/plugin.hpp @@ -90,6 +90,11 @@ namespace graphene { const graphene::chain::database &db() const; + /// Directory containing shared_memory.bin (the state dir). Used by + /// the P2P wedge watchdog to place the force_resync marker next to + /// the state it condemns, so it persists exactly as long as that state. + std::string get_state_dir() const; + /// Returns true when the node is processing P2P sync blocks /// (i.e. catching up to the network head). Plugins that perform /// heavy background work (e.g. periodic snapshots) should defer diff --git a/plugins/chain/plugin.cpp b/plugins/chain/plugin.cpp index 7bb5399d6e..204d79507e 100644 --- a/plugins/chain/plugin.cpp +++ b/plugins/chain/plugin.cpp @@ -280,6 +280,10 @@ namespace chain { return my->db; } + std::string plugin::get_state_dir() const { + return my->shared_memory_dir.string(); + } + bool plugin::is_syncing() const { return my->currently_syncing.load(std::memory_order_acquire); } @@ -512,6 +516,34 @@ namespace chain { my->db.wipe(data_dir, my->shared_memory_dir, true); } + // ========== force_resync marker recovery (wedge watchdog self-heal) ========== + // The P2P wedged-behind-network watchdog (dlt_p2p_node) writes this marker + // next to shared_memory.bin and exits when it confirms the node is + // permanently stuck on a divergent fork below LIB (rejection livelock, + // 2026-07-13 rpc.viz.cx incident). Fork-switch can't recover state below + // LIB — only wipe + snapshot re-import can. Honor the marker by wiping + // state so head becomes 0; the empty-state gate later in this function then + // re-bootstraps from a trusted snapshot peer. This is the exact recovery a + // manual `docker compose up -d --force-recreate` performs, minus the operator. + { + auto force_resync_marker = my->shared_memory_dir / "force_resync"; + if (boost::filesystem::exists(force_resync_marker)) { + wlog("Detected force_resync marker (P2P wedge watchdog). Wiping state to re-bootstrap " + "from a trusted snapshot peer."); + std::cerr << " force_resync marker found — wiping state for snapshot re-bootstrap.\n"; + try { + my->db.wipe(data_dir, my->shared_memory_dir, true); + } catch (...) { + wlog("force_resync: db.wipe() threw; removing shared_memory.bin directly."); + auto shm = my->shared_memory_dir / "shared_memory.bin"; + if (boost::filesystem::exists(shm)) boost::filesystem::remove(shm); + } + // Remove the marker only after the wipe so a crash mid-wipe re-triggers recovery. + boost::filesystem::remove(force_resync_marker); + ilog("force_resync marker cleared; state wiped. Empty-state gate will re-bootstrap from snapshot."); + } + } + my->db.set_flush_interval(my->flush_interval); my->db.add_checkpoints(my->loaded_checkpoints); my->db.set_require_locking(my->check_locks); diff --git a/plugins/p2p/p2p_plugin.cpp b/plugins/p2p/p2p_plugin.cpp index 632918072c..16020ef154 100644 --- a/plugins/p2p/p2p_plugin.cpp +++ b/plugins/p2p/p2p_plugin.cpp @@ -450,6 +450,7 @@ class p2p_plugin_impl { uint32_t peer_exchange_min_uptime_sec = 600; uint32_t stats_interval_sec = 300; bool isolated_peers = false; + bool auto_resync_on_wedge = false; chain::plugin& chain; @@ -497,7 +498,13 @@ void p2p_plugin::set_program_options( ("p2p-isolated-peers", boost::program_options::bool_switch()->default_value(false), "Restrict P2P to configured seed nodes only: reject inbound connections from " "unknown IPs and suppress peer exchange. Useful for nodes that must only talk " - "to a fixed set of peers."); + "to a fixed set of peers.") + ("auto-resync-on-wedge", boost::program_options::value()->default_value(false), + "When the node detects it is permanently wedged behind the network (head frozen " + "far below the tip while it keeps rejecting the canonical chain — a state divergence " + "below LIB that fork-switching cannot repair), write a force_resync marker next to " + "shared_memory.bin and exit so a supervised restart re-bootstraps from a trusted " + "snapshot. Default false: the watchdog only logs the condition (observe before enabling)."); } void p2p_plugin::plugin_initialize(const boost::program_options::variables_map& options) { @@ -571,6 +578,9 @@ void p2p_plugin::plugin_initialize(const boost::program_options::variables_map& if (options.count("p2p-isolated-peers")) { my->isolated_peers = options.at("p2p-isolated-peers").as(); } + if (options.count("auto-resync-on-wedge")) { + my->auto_resync_on_wedge = options.at("auto-resync-on-wedge").as(); + } } void p2p_plugin::plugin_startup() { @@ -602,6 +612,17 @@ void p2p_plugin::plugin_startup() { my->node->set_stats_log_interval(my->stats_interval_sec); my->node->set_isolated_peers(my->isolated_peers); + // Wedged-behind-network watchdog: give the node the state dir so it can + // place the force_resync marker next to shared_memory.bin, and gate the + // destructive auto-exit behind the config flag (default off). + try { + my->node->set_state_dir(my->chain.get_state_dir()); + } catch (...) { + wlog("Could not resolve chain state dir for wedge watchdog marker; " + "auto-resync-on-wedge will exit without writing a marker."); + } + my->node->set_auto_resync_on_wedge(my->auto_resync_on_wedge); + // Wire up witness diagnostic provider so FORWARD stagnation logs include // production state without the network library taking a plugin dependency. my->node->set_witness_diag_provider([]() -> std::string { From 46d01a684183ba2ab5be3e866320f3f12a6796a4 Mon Sep 17 00:00:00 2001 From: Vladimir Babin Date: Tue, 14 Jul 2026 00:11:09 +0800 Subject: [PATCH 3/5] fix(snapshot): expand CHECK_SECTION macro to avoid preprocessor-in-macro-arg UB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The completeness-check block lives inside the db.with_strong_write_lock([&]{...}) lambda, and with_strong_write_lock is itself a function-like macro. The local #define/#undef CHECK_SECTION therefore sat inside that macro's argument list, which is undefined behavior — GCC silently dropped the #define, leaving CHECK_SECTION undeclared and breaking the build. Replace the macro with 28 direct check_section(...) calls via a small local lambda; identical behavior, no preprocessor directives inside the macro arg. --- plugins/snapshot/plugin.cpp | 70 ++++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 33 deletions(-) diff --git a/plugins/snapshot/plugin.cpp b/plugins/snapshot/plugin.cpp index 8f4327451c..f401290c21 100644 --- a/plugins/snapshot/plugin.cpp +++ b/plugins/snapshot/plugin.cpp @@ -1808,39 +1808,43 @@ void snapshot_plugin::plugin_impl::load_snapshot(const fc::path& input_path) { } }; - #define CHECK_SECTION(section, index_type) \ - expect_count(section, db.get_index().indices().size()); - - CHECK_SECTION("account", account_index) - CHECK_SECTION("account_authority", account_authority_index) - CHECK_SECTION("validator", validator_index) - CHECK_SECTION("validator_vote", validator_vote_index) - CHECK_SECTION("block_summary", block_summary_index) - CHECK_SECTION("content", content_index) - CHECK_SECTION("content_vote", content_vote_index) - CHECK_SECTION("block_post_validation", validator_confirmation_index) - CHECK_SECTION("transaction", transaction_index) - CHECK_SECTION("vesting_delegation", vesting_delegation_index) - CHECK_SECTION("vesting_delegation_expiration", vesting_delegation_expiration_index) - CHECK_SECTION("fix_vesting_delegation", fix_vesting_delegation_index) - CHECK_SECTION("withdraw_vesting_route", withdraw_vesting_route_index) - CHECK_SECTION("escrow", escrow_index) - CHECK_SECTION("proposal", proposal_index) - CHECK_SECTION("required_approval", required_approval_index) - CHECK_SECTION("committee_request", committee_request_index) - CHECK_SECTION("committee_vote", committee_vote_index) - CHECK_SECTION("invite", invite_index) - CHECK_SECTION("award_shares_expire", award_shares_expire_index) - CHECK_SECTION("paid_subscription", paid_subscription_index) - CHECK_SECTION("paid_subscribe", paid_subscribe_index) - CHECK_SECTION("validator_penalty_expire", validator_penalty_expire_index) - CHECK_SECTION("content_type", content_type_index) - CHECK_SECTION("account_metadata", account_metadata_index) - CHECK_SECTION("master_authority_history", master_authority_history_index) - CHECK_SECTION("account_recovery_request", account_recovery_request_index) - CHECK_SECTION("change_recovery_account_request", change_recovery_account_request_index) - - #undef CHECK_SECTION + // NB: expanded as direct calls rather than a local #define/#undef. + // This whole block is the body of the db.with_strong_write_lock([&]{...}) + // lambda, and with_strong_write_lock is itself a function-like macro — a + // preprocessor directive inside a macro argument list is undefined + // behavior (GCC silently drops the #define, leaving CHECK_SECTION + // undeclared and the build broken). + auto check_section = [&](const std::string& section, size_t actual) { + expect_count(section, actual); + }; + check_section("account", db.get_index().indices().size()); + check_section("account_authority", db.get_index().indices().size()); + check_section("validator", db.get_index().indices().size()); + check_section("validator_vote", db.get_index().indices().size()); + check_section("block_summary", db.get_index().indices().size()); + check_section("content", db.get_index().indices().size()); + check_section("content_vote", db.get_index().indices().size()); + check_section("block_post_validation", db.get_index().indices().size()); + check_section("transaction", db.get_index().indices().size()); + check_section("vesting_delegation", db.get_index().indices().size()); + check_section("vesting_delegation_expiration", db.get_index().indices().size()); + check_section("fix_vesting_delegation", db.get_index().indices().size()); + check_section("withdraw_vesting_route", db.get_index().indices().size()); + check_section("escrow", db.get_index().indices().size()); + check_section("proposal", db.get_index().indices().size()); + check_section("required_approval", db.get_index().indices().size()); + check_section("committee_request", db.get_index().indices().size()); + check_section("committee_vote", db.get_index().indices().size()); + check_section("invite", db.get_index().indices().size()); + check_section("award_shares_expire", db.get_index().indices().size()); + check_section("paid_subscription", db.get_index().indices().size()); + check_section("paid_subscribe", db.get_index().indices().size()); + check_section("validator_penalty_expire", db.get_index().indices().size()); + check_section("content_type", db.get_index().indices().size()); + check_section("account_metadata", db.get_index().indices().size()); + check_section("master_authority_history", db.get_index().indices().size()); + check_section("account_recovery_request", db.get_index().indices().size()); + check_section("change_recovery_account_request", db.get_index().indices().size()); // (2) Referential integrity: every account_authority must reference an // existing account. This is the exact invariant the wedge incident From 5b1d4cdc2ca2983b0321ac16c087602bec8631fa Mon Sep 17 00:00:00 2001 From: Vladimir Babin Date: Tue, 14 Jul 2026 11:28:57 +0800 Subject: [PATCH 4/5] =?UTF-8?q?fix(self-heal):=20address=20PR=20review=20?= =?UTF-8?q?=E2=80=94=20eclipse-safe=20tip,=20sustained-reject,=20no=20re-w?= =?UTF-8?q?edge=20loop?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses On1x review on PR #125: [major] Recovery re-import loop (plugins/chain/plugin.cpp): db.wipe() does not clear the snapshots/ dir, so on a snapshot-serving node --snapshot-auto-latest could rediscover a snapshot serialized FROM the corrupt state and re-wedge. On force_resync, drop the local snapshot_path and disable auto_recover_from_snapshot so no node-local snapshot is silently re-imported; the node comes up empty and resyncs from the network. Corrected the comments: there is no in-wire snapshot-fetch-from-peer, recovery is network resync or an operator-supplied snapshot. [major] Sticky rejects (dlt_p2p_node): _wedge_reject_baseline was captured once at window open, so a single early rejection kept rejects_climbed true for the whole 900s window (a node that rejected once then went silent would falsely confirm). Replace with recency tracking: "climbing" now means a rejection was observed within the last WEDGE_RELOG_SEC — sustained, not "happened once". [major] Unvalidated network tip (dlt_p2p_node): _highest_seen_block_num is monotonic and peer-influenced, so one bogus advertised head armed the watchdog forever (eclipse-style auto-wipe). Derive a corroborated tip = 2nd-highest head across established (SYNCING/ACTIVE) peers, requiring >= WEDGE_MIN_CORROBORATING_PEERS; a single outlier can no longer move it, and an under-corroborated node never arms. [medium] Count-reconciliation scope (plugins/snapshot/plugin.cpp): documented that header.object_counts is derived at export from the same arrays, so check (1) catches lossy IMPORT only, not export-side incompleteness; the referential check is the real net for the incident. The stronger supply-vs-balances invariant is deferred (database::validate_invariants() is declared but not defined; a hand-rolled partial accounting would risk rejecting valid snapshots). [minor] Marker remove() could throw outside the try/catch and abort plugin_startup (crash loop) — use the non-throwing error_code overload (also for the fallback shared_memory.bin remove). [minor] Dropped the trivial check_section pass-through; the 28 sites now call expect_count directly. --- libraries/network/dlt_p2p_node.cpp | 78 ++++++++++----- .../include/graphene/network/dlt_p2p_node.hpp | 15 ++- plugins/chain/plugin.cpp | 51 +++++++--- plugins/snapshot/plugin.cpp | 94 +++++++++++-------- 4 files changed, 159 insertions(+), 79 deletions(-) diff --git a/libraries/network/dlt_p2p_node.cpp b/libraries/network/dlt_p2p_node.cpp index 432f2d9b09..a729e7ddb5 100644 --- a/libraries/network/dlt_p2p_node.cpp +++ b/libraries/network/dlt_p2p_node.cpp @@ -3768,19 +3768,41 @@ void dlt_p2p_node::check_wedge_watchdog() { uint32_t our_head = _delegate->get_head_block_num(); if (our_head == 0) return; // not bootstrapped yet (or mid snapshot import) - uint32_t behind = (_highest_seen_block_num > our_head) - ? (_highest_seen_block_num - our_head) : 0; - auto now = fc::time_point::now(); - // "Stuck behind" gate: far enough behind the network tip to be suspicious. - // If we're not this far behind, the node is healthy or merely lagging a - // little — no window. + // ── Corroborated network tip (anti-eclipse) ────────────────────── + // _highest_seen_block_num is monotonic and peer-influenced: a single + // malicious/buggy peer advertising an impossibly-high head would pin + // `behind` open forever and, with the flag ON, drive an operator-invisible + // auto-wipe under an eclipse scenario. For this DESTRUCTIVE decision we + // require corroboration — use the SECOND-highest head across established + // (SYNCING/ACTIVE) peers, so no single outlier can move the tip. Fewer + // than WEDGE_MIN_CORROBORATING_PEERS established peers ⇒ we cannot + // corroborate ⇒ do not arm the watchdog (network_tip collapses to 0). + uint32_t highest_peer_head = 0, second_peer_head = 0, established_peers = 0; + for (const auto& kv : _peer_states) { + const auto& ps = kv.second; + if (ps.lifecycle_state != DLT_PEER_LIFECYCLE_SYNCING && + ps.lifecycle_state != DLT_PEER_LIFECYCLE_ACTIVE) continue; + uint32_t h = ps.peer_head_num; + if (h == 0) continue; + ++established_peers; + if (h > highest_peer_head) { second_peer_head = highest_peer_head; highest_peer_head = h; } + else if (h > second_peer_head) { second_peer_head = h; } + } + uint32_t network_tip = (established_peers >= WEDGE_MIN_CORROBORATING_PEERS) + ? second_peer_head : 0; + + uint32_t behind = (network_tip > our_head) ? (network_tip - our_head) : 0; + + // "Stuck behind" gate: far enough behind the CORROBORATED tip to be + // suspicious. An under-corroborated tip is 0, collapsing `behind` to 0 and + // tripping this branch — so an eclipsed / low-peer node never arms. if (behind <= WEDGE_BEHIND_THRESHOLD) { if (_wedge_since != fc::time_point()) { - ilog(DLT_LOG_GREEN "Wedge watchdog: caught up (behind=${b} <= ${thr}), clearing wedge timer " - "(our_head=#${h})" DLT_LOG_RESET, - ("b", behind)("thr", WEDGE_BEHIND_THRESHOLD)("h", our_head)); + ilog(DLT_LOG_GREEN "Wedge watchdog: caught up / under-corroborated (behind=${b} <= ${thr}, " + "corroborating_peers=${p}), clearing wedge timer (our_head=#${h})" DLT_LOG_RESET, + ("b", behind)("thr", WEDGE_BEHIND_THRESHOLD)("p", established_peers)("h", our_head)); } _wedge_since = fc::time_point(); return; @@ -3790,12 +3812,13 @@ void dlt_p2p_node::check_wedge_watchdog() { if (_wedge_since == fc::time_point()) { _wedge_since = now; _wedge_window_head = our_head; - _wedge_reject_baseline = _gap_rejected_total; + _wedge_reject_last_total = _gap_rejected_total; + _wedge_last_reject = fc::time_point(); // no rejection observed yet this window _wedge_last_relog = now; - elog(DLT_LOG_RED "Wedge watchdog: node is ${b} blocks behind network tip " - "(our_head=#${h}, network_tip=#${t}) — watching for a rejection livelock. " - "Will confirm over ${c}s if head stays flat and rejections keep climbing." DLT_LOG_RESET, - ("b", behind)("h", our_head)("t", _highest_seen_block_num)("c", WEDGE_CONFIRM_SEC)); + elog(DLT_LOG_RED "Wedge watchdog: node is ${b} blocks behind corroborated network tip " + "(our_head=#${h}, network_tip=#${t}, corroborating_peers=${p}) — watching for a rejection " + "livelock. Will confirm over ${c}s if head stays flat and rejections keep climbing." DLT_LOG_RESET, + ("b", behind)("h", our_head)("t", network_tip)("p", established_peers)("c", WEDGE_CONFIRM_SEC)); return; } @@ -3809,16 +3832,26 @@ void dlt_p2p_node::check_wedge_watchdog() { return; } - bool rejects_climbed = (_gap_rejected_total > _wedge_reject_baseline); + // "Rejections still climbing" = a NEW gap-fill rejection observed within the + // last WEDGE_RELOG_SEC. A once-captured baseline stays tripped for the whole + // window after a single early rejection, so a node that took one rejection + // then went silent (partial isolation) would falsely confirm. Track the last + // increment time instead, so "climbing" means SUSTAINED, not "happened once". + if (_gap_rejected_total > _wedge_reject_last_total) { + _wedge_reject_last_total = _gap_rejected_total; + _wedge_last_reject = now; + } + bool rejects_climbed = (_wedge_last_reject != fc::time_point()) + && (now - _wedge_last_reject).count() <= int64_t(WEDGE_RELOG_SEC) * 1000000LL; uint32_t elapsed_sec = static_cast((now - _wedge_since).count() / 1000000LL); // Loud periodic re-log while the wedge builds toward confirmation. if ((now - _wedge_last_relog).count() >= int64_t(WEDGE_RELOG_SEC) * 1000000LL) { _wedge_last_relog = now; elog(DLT_LOG_RED "Wedge watchdog: head flat at #${h} for ${s}s / ${c}s — behind=${b} " - "(network_tip=#${t}), rejections ${rc} (total=${r})" DLT_LOG_RESET, + "(network_tip=#${t}, corroborating_peers=${p}), rejections ${rc} (total=${r})" DLT_LOG_RESET, ("h", our_head)("s", elapsed_sec)("c", WEDGE_CONFIRM_SEC)("b", behind) - ("t", _highest_seen_block_num) + ("t", network_tip)("p", established_peers) ("rc", rejects_climbed ? "climbing" : "flat")("r", _gap_rejected_total)); } @@ -3826,9 +3859,9 @@ void dlt_p2p_node::check_wedge_watchdog() { // ── Confirmed wedge ────────────────────────────────────────── elog(DLT_LOG_RED "Wedge watchdog: CONFIRMED network wedge — head frozen at #${h} for ${s}s while " - "${b} blocks behind (network_tip=#${t}), gap rejections total=${r}. State has diverged at/below " - "LIB; fork-switch cannot recover — only wipe + snapshot re-import can." DLT_LOG_RESET, - ("h", our_head)("s", elapsed_sec)("b", behind)("t", _highest_seen_block_num)("r", _gap_rejected_total)); + "${b} blocks behind (corroborated network_tip=#${t}), gap rejections total=${r}. State has " + "diverged at/below LIB; fork-switch cannot recover — only wipe + snapshot re-import can." DLT_LOG_RESET, + ("h", our_head)("s", elapsed_sec)("b", behind)("t", network_tip)("r", _gap_rejected_total)); if (!_auto_resync_on_wedge) { elog(DLT_LOG_RED "Wedge watchdog: auto-resync-on-wedge is OFF — NOT exiting. Enable " @@ -3851,7 +3884,8 @@ void dlt_p2p_node::check_wedge_watchdog() { std::ofstream f(marker_path); f << "reason=wedged-behind-network\n" << "our_head=" << our_head << "\n" - << "network_tip=" << _highest_seen_block_num << "\n" + << "network_tip=" << network_tip << "\n" + << "highest_seen=" << _highest_seen_block_num << "\n" << "behind=" << behind << "\n" << "gap_rejected_total=" << _gap_rejected_total << "\n" << "sustained_sec=" << elapsed_sec << "\n" @@ -3872,7 +3906,7 @@ void dlt_p2p_node::check_wedge_watchdog() { elog(DLT_LOG_RED "Wedge watchdog: exiting process (code 2) for supervised snapshot re-bootstrap." DLT_LOG_RESET); std::cerr << "FATAL: node wedged behind network (behind=" << behind - << ", head=" << our_head << ", tip=" << _highest_seen_block_num + << ", head=" << our_head << ", tip=" << network_tip << "). Wrote force_resync marker=" << (marker_written ? "yes" : "no") << "; exiting for snapshot re-bootstrap." << std::endl; std::_Exit(2); diff --git a/libraries/network/include/graphene/network/dlt_p2p_node.hpp b/libraries/network/include/graphene/network/dlt_p2p_node.hpp index b4e512e3f1..14d50a149f 100644 --- a/libraries/network/include/graphene/network/dlt_p2p_node.hpp +++ b/libraries/network/include/graphene/network/dlt_p2p_node.hpp @@ -301,14 +301,17 @@ class dlt_p2p_node { // Pure predicate for the wedge state machine — no I/O, table-testable. // Aggregated over the sustained observation window: - // behind — how far our head trails the network tip (blocks) + // behind — how far our head trails the CORROBORATED network tip (blocks) // head_advanced — did our head move at all during the window - // rejects_climbed — did gap-fill rejections increase during the window + // rejects_climbed — was a gap-fill rejection observed within the last + // WEDGE_RELOG_SEC (i.e. rejections are STILL climbing now, + // not merely "happened once early in the window") // elapsed_sec — how long the (behind && head-flat) condition has held // Verdict CONFIRMED only when the node is far behind, its head never moved, // it is still actively rejecting the chain, and this has been sustained. - // - a syncing node advances its head → head_advanced ⇒ not wedged - // - a partitioned node has no blocks/rejects → !rejects_climbed ⇒ not wedged + // - a syncing node advances its head → head_advanced ⇒ not wedged + // - a partitioned/silent node stops rejecting → !rejects_climbed ⇒ not wedged + // (recency matters: one early rejection then silence must NOT confirm) // - genuine divergence trips all three, and only after WEDGE_CONFIRM_SEC. static bool is_wedged(uint32_t behind, bool head_advanced, bool rejects_climbed, uint32_t elapsed_sec) { @@ -456,8 +459,10 @@ class dlt_p2p_node { std::string _state_dir; ///< Directory holding shared_memory.bin (force_resync marker lives here) fc::time_point _wedge_since; ///< When the (behind && head-flat) window began (unset when not behind) uint32_t _wedge_window_head = 0; ///< our_head at window start — any change ⇒ head advanced ⇒ reset - uint64_t _wedge_reject_baseline = 0; ///< _gap_rejected_total at window start — a rise ⇒ still rejecting + uint64_t _wedge_reject_last_total = 0; ///< _gap_rejected_total at last observed increment (edge detect) + fc::time_point _wedge_last_reject; ///< When a rejection was last observed; "climbing" ⇒ one within WEDGE_RELOG_SEC fc::time_point _wedge_last_relog; ///< Last loud re-log while a wedge is building + static constexpr uint32_t WEDGE_MIN_CORROBORATING_PEERS = 2; ///< Network tip must be corroborated by >=N peers (anti-eclipse) // ── FORWARD stagnation ────────────────────────────────────── uint32_t _last_forward_head_num = 0; diff --git a/plugins/chain/plugin.cpp b/plugins/chain/plugin.cpp index 204d79507e..c7de3fdb3f 100644 --- a/plugins/chain/plugin.cpp +++ b/plugins/chain/plugin.cpp @@ -521,26 +521,55 @@ namespace chain { // next to shared_memory.bin and exits when it confirms the node is // permanently stuck on a divergent fork below LIB (rejection livelock, // 2026-07-13 rpc.viz.cx incident). Fork-switch can't recover state below - // LIB — only wipe + snapshot re-import can. Honor the marker by wiping - // state so head becomes 0; the empty-state gate later in this function then - // re-bootstraps from a trusted snapshot peer. This is the exact recovery a - // manual `docker compose up -d --force-recreate` performs, minus the operator. + // LIB — only wipe + fresh re-bootstrap can. Honor the marker by wiping + // state so head becomes 0, then come up on empty state. This is the exact + // recovery a manual `docker compose up -d --force-recreate` performs, minus + // the operator. { auto force_resync_marker = my->shared_memory_dir / "force_resync"; if (boost::filesystem::exists(force_resync_marker)) { - wlog("Detected force_resync marker (P2P wedge watchdog). Wiping state to re-bootstrap " - "from a trusted snapshot peer."); - std::cerr << " force_resync marker found — wiping state for snapshot re-bootstrap.\n"; + wlog("Detected force_resync marker (P2P wedge watchdog). Wiping state for a clean " + "re-bootstrap (network resync or fresh operator-supplied snapshot)."); + std::cerr << " force_resync marker found — wiping state for clean re-bootstrap.\n"; try { my->db.wipe(data_dir, my->shared_memory_dir, true); } catch (...) { wlog("force_resync: db.wipe() threw; removing shared_memory.bin directly."); auto shm = my->shared_memory_dir / "shared_memory.bin"; - if (boost::filesystem::exists(shm)) boost::filesystem::remove(shm); + if (boost::filesystem::exists(shm)) { + boost::system::error_code _shm_ec; + boost::filesystem::remove(shm, _shm_ec); + } } - // Remove the marker only after the wipe so a crash mid-wipe re-triggers recovery. - boost::filesystem::remove(force_resync_marker); - ilog("force_resync marker cleared; state wiped. Empty-state gate will re-bootstrap from snapshot."); + // A wedge is a below-LIB divergence: any snapshot THIS node created + // locally (allow-snapshot-serving) was serialized FROM the corrupt + // state, so re-importing it would just re-wedge — a crash loop. + // db.wipe() clears shared_memory.bin + block_log but NOT the + // snapshots/ dir, so --snapshot / --snapshot-auto-latest could + // rediscover that poisoned local snapshot. For this boot, drop any + // configured/auto local snapshot path and disable local snapshot + // auto-recovery so neither the snapshot-load path nor the + // schema/revision recovery paths silently re-import a node-local + // snapshot. The node then comes up on empty state and resyncs from + // the P2P network; if the deployment refreshes a trusted snapshot + // out of band on the supervised restart, that fresh file is used on + // the NEXT boot (marker already cleared). NB: there is no in-wire + // snapshot-fetch-from-peer — recovery is network resync or an + // operator/entrypoint-supplied snapshot, not a protocol download. + my->snapshot_path.clear(); + my->auto_recover_from_snapshot = false; + // Remove the marker only after the wipe so a crash mid-wipe re-triggers + // recovery. Use the non-throwing overload: this runs OUTSIDE the + // surrounding try/catch, and a throw here (file locked, permission) + // would abort plugin_startup — with the crash-safe re-trigger that + // becomes a crash loop. + boost::system::error_code _rm_ec; + boost::filesystem::remove(force_resync_marker, _rm_ec); + if (_rm_ec) + wlog("force_resync: failed to remove marker (${e}); may re-trigger next boot.", + ("e", _rm_ec.message())); + ilog("force_resync marker cleared; state wiped. Node will come up on empty state and " + "resync from the network (local snapshot auto-import disabled for this boot)."); } } diff --git a/plugins/snapshot/plugin.cpp b/plugins/snapshot/plugin.cpp index f401290c21..96ea2b101f 100644 --- a/plugins/snapshot/plugin.cpp +++ b/plugins/snapshot/plugin.cpp @@ -1785,10 +1785,24 @@ void snapshot_plugin::plugin_impl::load_snapshot(const fc::path& input_path) { // canonical block references that account (the 2026-07-13 rpc.viz.cx // incident: out_of_range "viz-social-bot" at database.cpp get_account). // - // These checks catch a lossy import (serialized N, imported M) and gross - // referential holes regardless of cause, turning a silent bad-state - // acceptance into a loud, retryable FC_ASSERT — the P2P-sync path then - // rejects the snapshot and retries another trusted peer. + // SCOPE / LIMITATION: these are import-side sanity checks, NOT a full + // state-integrity proof. Check (1) reconciles live index sizes against + // header.object_counts, so it catches a LOSSY IMPORT (serialized N, + // imported M) — but NOT export-side incompleteness, because + // header.object_counts is itself derived from the same serialized arrays + // at export: a short export records the short count and reconciles + // cleanly. The real net for the incident class (an account missing + // entirely on the serving side) is the referential check (2) below, and + // only if a dangling reference survived. A stronger value invariant + // (dgp.current_supply vs summed balances/vesting/escrow) is the proper + // export-side detector but is deferred: the chain's own + // database::validate_invariants() is currently only declared, not + // defined, and a hand-rolled partial accounting here would risk rejecting + // VALID snapshots. Tracked as follow-up. + // + // What these DO buy: a lossy import and gross referential holes become a + // loud, retryable FC_ASSERT instead of silent bad-state acceptance — the + // P2P-sync path then rejects the snapshot and retries another trusted peer. { uint32_t invariant_failures = 0; @@ -1796,7 +1810,8 @@ void snapshot_plugin::plugin_impl::load_snapshot(const fc::path& input_path) { // is recorded at export from the serialized section-array sizes; every // multi-instance index is cleared before import (see clear block above), // so the live index size must equal the recorded count. A mismatch - // means the import silently dropped objects. + // means the import silently dropped objects (lossy import only — see + // the SCOPE note above for why this cannot see export-side gaps). auto expect_count = [&](const std::string& section, size_t actual) { auto itr = header.object_counts.find(section); if (itr == header.object_counts.end()) return; // section absent from this snapshot @@ -1808,43 +1823,40 @@ void snapshot_plugin::plugin_impl::load_snapshot(const fc::path& input_path) { } }; - // NB: expanded as direct calls rather than a local #define/#undef. - // This whole block is the body of the db.with_strong_write_lock([&]{...}) - // lambda, and with_strong_write_lock is itself a function-like macro — a + // NB: direct calls rather than a local #define/#undef. This whole + // block is the body of the db.with_strong_write_lock([&]{...}) lambda, + // and with_strong_write_lock is itself a function-like macro — a // preprocessor directive inside a macro argument list is undefined - // behavior (GCC silently drops the #define, leaving CHECK_SECTION - // undeclared and the build broken). - auto check_section = [&](const std::string& section, size_t actual) { - expect_count(section, actual); - }; - check_section("account", db.get_index().indices().size()); - check_section("account_authority", db.get_index().indices().size()); - check_section("validator", db.get_index().indices().size()); - check_section("validator_vote", db.get_index().indices().size()); - check_section("block_summary", db.get_index().indices().size()); - check_section("content", db.get_index().indices().size()); - check_section("content_vote", db.get_index().indices().size()); - check_section("block_post_validation", db.get_index().indices().size()); - check_section("transaction", db.get_index().indices().size()); - check_section("vesting_delegation", db.get_index().indices().size()); - check_section("vesting_delegation_expiration", db.get_index().indices().size()); - check_section("fix_vesting_delegation", db.get_index().indices().size()); - check_section("withdraw_vesting_route", db.get_index().indices().size()); - check_section("escrow", db.get_index().indices().size()); - check_section("proposal", db.get_index().indices().size()); - check_section("required_approval", db.get_index().indices().size()); - check_section("committee_request", db.get_index().indices().size()); - check_section("committee_vote", db.get_index().indices().size()); - check_section("invite", db.get_index().indices().size()); - check_section("award_shares_expire", db.get_index().indices().size()); - check_section("paid_subscription", db.get_index().indices().size()); - check_section("paid_subscribe", db.get_index().indices().size()); - check_section("validator_penalty_expire", db.get_index().indices().size()); - check_section("content_type", db.get_index().indices().size()); - check_section("account_metadata", db.get_index().indices().size()); - check_section("master_authority_history", db.get_index().indices().size()); - check_section("account_recovery_request", db.get_index().indices().size()); - check_section("change_recovery_account_request", db.get_index().indices().size()); + // behavior (GCC silently drops the #define, leaving the section-check + // macro undeclared and the build broken). + expect_count("account", db.get_index().indices().size()); + expect_count("account_authority", db.get_index().indices().size()); + expect_count("validator", db.get_index().indices().size()); + expect_count("validator_vote", db.get_index().indices().size()); + expect_count("block_summary", db.get_index().indices().size()); + expect_count("content", db.get_index().indices().size()); + expect_count("content_vote", db.get_index().indices().size()); + expect_count("block_post_validation", db.get_index().indices().size()); + expect_count("transaction", db.get_index().indices().size()); + expect_count("vesting_delegation", db.get_index().indices().size()); + expect_count("vesting_delegation_expiration", db.get_index().indices().size()); + expect_count("fix_vesting_delegation", db.get_index().indices().size()); + expect_count("withdraw_vesting_route", db.get_index().indices().size()); + expect_count("escrow", db.get_index().indices().size()); + expect_count("proposal", db.get_index().indices().size()); + expect_count("required_approval", db.get_index().indices().size()); + expect_count("committee_request", db.get_index().indices().size()); + expect_count("committee_vote", db.get_index().indices().size()); + expect_count("invite", db.get_index().indices().size()); + expect_count("award_shares_expire", db.get_index().indices().size()); + expect_count("paid_subscription", db.get_index().indices().size()); + expect_count("paid_subscribe", db.get_index().indices().size()); + expect_count("validator_penalty_expire", db.get_index().indices().size()); + expect_count("content_type", db.get_index().indices().size()); + expect_count("account_metadata", db.get_index().indices().size()); + expect_count("master_authority_history", db.get_index().indices().size()); + expect_count("account_recovery_request", db.get_index().indices().size()); + expect_count("change_recovery_account_request", db.get_index().indices().size()); // (2) Referential integrity: every account_authority must reference an // existing account. This is the exact invariant the wedge incident From dded0d54687ad3b58cf40914df64150214e2d862 Mon Sep 17 00:00:00 2001 From: Vladimir Babin Date: Tue, 14 Jul 2026 12:03:16 +0800 Subject: [PATCH 5/5] test(p2p): is_wedged truth-table unit test + make predicate public The wedge watchdog only takes its destructive wipe+resync action when the pure is_wedged() predicate returns true, so pin its boolean contract with a regression guard. The predicate's semantics changed in the review-fix round (rejects_climbed is now recency-based, behind is corroborated-tip-based) though its signature did not. - Promote is_wedged() and its WEDGE_BEHIND_THRESHOLD / WEDGE_CONFIRM_SEC thresholds into a public: block so the test can assert boundaries symbolically. The predicate was documented as table-testable but was actually private; out-of-line constexpr defs and other .cpp references are unaffected by the access change. - Add tests/consensus_sim/scenarios/test_wedge_predicate.cpp: 8 cases covering the sole confirming combination, each dimension vetoed individually, and both boundaries (behind strictly >, elapsed inclusive >=). Wired into the existing Boost.Test runner and linked graphene::network. Runs only under -DBUILD_CONSENSUS_TESTS=ON + ctest (opt-in, as with the other consensus_sim scenarios); the current Docker CI builds only vizd/cli_wallet so it compiles the header change but not this test. --- .../include/graphene/network/dlt_p2p_node.hpp | 12 ++- tests/consensus_sim/CMakeLists.txt | 2 + .../scenarios/test_wedge_predicate.cpp | 91 +++++++++++++++++++ 3 files changed, 103 insertions(+), 2 deletions(-) create mode 100644 tests/consensus_sim/scenarios/test_wedge_predicate.cpp diff --git a/libraries/network/include/graphene/network/dlt_p2p_node.hpp b/libraries/network/include/graphene/network/dlt_p2p_node.hpp index 14d50a149f..532b0ef538 100644 --- a/libraries/network/include/graphene/network/dlt_p2p_node.hpp +++ b/libraries/network/include/graphene/network/dlt_p2p_node.hpp @@ -299,6 +299,13 @@ class dlt_p2p_node { // every canonical block, so those monitors never fire. void check_wedge_watchdog(); +public: + // Thresholds feeding the pure wedge predicate below. Public (with the + // predicate) so the truth-table unit test can assert the boundaries + // symbolically rather than hard-coding magic numbers. + static constexpr uint32_t WEDGE_BEHIND_THRESHOLD = 200; ///< Blocks behind network tip to be considered "behind" + static constexpr uint32_t WEDGE_CONFIRM_SEC = 900; ///< Wedge condition must hold this long (15 min) before acting + // Pure predicate for the wedge state machine — no I/O, table-testable. // Aggregated over the sustained observation window: // behind — how far our head trails the CORROBORATED network tip (blocks) @@ -321,6 +328,7 @@ class dlt_p2p_node { && elapsed_sec >= WEDGE_CONFIRM_SEC; } +private: // ── Subnet diversity ───────────────────────────────────────── uint32_t count_peers_in_subnet(const fc::ip::address& addr) const; bool is_same_subnet(const fc::ip::address& a, const fc::ip::address& b) const; @@ -452,8 +460,8 @@ class dlt_p2p_node { fc::time_point _gap_rejected_blacklist_until; ///< Don't gap-fill any block until this time // ── Wedged-behind-network watchdog ─────────────────────────── - static constexpr uint32_t WEDGE_BEHIND_THRESHOLD = 200; ///< Blocks behind network tip to be considered "behind" - static constexpr uint32_t WEDGE_CONFIRM_SEC = 900; ///< Wedge condition must hold this long (15 min) before acting + // (WEDGE_BEHIND_THRESHOLD / WEDGE_CONFIRM_SEC are public, declared with + // the is_wedged predicate above.) static constexpr uint32_t WEDGE_RELOG_SEC = 60; ///< Loud re-log cadence while a wedge is building bool _auto_resync_on_wedge = false; ///< Gate the destructive _Exit action (default OFF: elog only) std::string _state_dir; ///< Directory holding shared_memory.bin (force_resync marker lives here) diff --git a/tests/consensus_sim/CMakeLists.txt b/tests/consensus_sim/CMakeLists.txt index cabc7e4a4e..fa742fc535 100644 --- a/tests/consensus_sim/CMakeLists.txt +++ b/tests/consensus_sim/CMakeLists.txt @@ -56,12 +56,14 @@ set(SCENARIO_SOURCES scenarios/test_smoke_no_faults.cpp scenarios/test_determinism_replay.cpp scenarios/test_equivocation.cpp + scenarios/test_wedge_predicate.cpp ) add_executable(consensus_sim_tests ${SCENARIO_SOURCES}) target_link_libraries(consensus_sim_tests PRIVATE consensus_sim_harness + graphene::network # test_wedge_predicate.cpp: dlt_p2p_node::is_wedged Boost::unit_test_framework ) diff --git a/tests/consensus_sim/scenarios/test_wedge_predicate.cpp b/tests/consensus_sim/scenarios/test_wedge_predicate.cpp new file mode 100644 index 0000000000..e952d58504 --- /dev/null +++ b/tests/consensus_sim/scenarios/test_wedge_predicate.cpp @@ -0,0 +1,91 @@ +// Truth-table regression guard for dlt_p2p_node::is_wedged(). +// +// is_wedged() is the pure, I/O-free heart of the self-healing +// wedged-behind-network watchdog (PR #125). The watchdog only takes the +// destructive wipe+resync action when this predicate returns true, so a +// silent change to its boolean composition — or to the WEDGE_BEHIND_THRESHOLD +// / WEDGE_CONFIRM_SEC boundaries — could either wedge a healthy node or fail +// to heal a genuinely stuck one. The predicate's *semantics* changed in the +// review-fix round (rejects_climbed is now recency-based, behind is now +// corroborated-tip-based) even though its signature did not; this table pins +// the contract those inputs must satisfy. +// +// CONFIRMED only when ALL of: +// behind > WEDGE_BEHIND_THRESHOLD (far behind the corroborated network tip) +// !head_advanced (our head never moved in the window) +// rejects_climbed (still actively rejecting the chain now) +// elapsed_sec >= WEDGE_CONFIRM_SEC (sustained long enough) + +#include + +#include + +using graphene::network::dlt_p2p_node; + +namespace { +constexpr uint32_t BEHIND = dlt_p2p_node::WEDGE_BEHIND_THRESHOLD; +constexpr uint32_t CONFIRM = dlt_p2p_node::WEDGE_CONFIRM_SEC; + +// Values comfortably satisfying / violating each dimension. +constexpr uint32_t FAR_BEHIND = BEHIND + 1; // strictly greater ⇒ "behind" +constexpr uint32_t NOT_BEHIND = BEHIND; // NOT strictly greater ⇒ not behind +constexpr uint32_t LONG_ENOUGH = CONFIRM; // >= confirm ⇒ sustained +constexpr uint32_t TOO_SHORT = CONFIRM - 1; // < confirm ⇒ not yet +} // namespace + +BOOST_AUTO_TEST_SUITE(wedge_predicate) + +// The one and only combination that must confirm a wedge. +BOOST_AUTO_TEST_CASE(confirms_only_when_all_conditions_hold) { + BOOST_CHECK(dlt_p2p_node::is_wedged(FAR_BEHIND, /*head_advanced=*/false, + /*rejects_climbed=*/true, LONG_ENOUGH)); +} + +// Each single dimension flipped must veto the verdict. +BOOST_AUTO_TEST_CASE(not_wedged_when_head_advanced) { + // A syncing node advances its head — never a wedge, however far behind. + BOOST_CHECK(!dlt_p2p_node::is_wedged(FAR_BEHIND, /*head_advanced=*/true, + /*rejects_climbed=*/true, LONG_ENOUGH)); +} + +BOOST_AUTO_TEST_CASE(not_wedged_when_rejects_not_climbing) { + // A partitioned/silent node stops rejecting — recency vetoes the verdict. + BOOST_CHECK(!dlt_p2p_node::is_wedged(FAR_BEHIND, /*head_advanced=*/false, + /*rejects_climbed=*/false, LONG_ENOUGH)); +} + +BOOST_AUTO_TEST_CASE(not_wedged_when_not_far_behind) { + // At/under the threshold we are not "behind" enough to be wedged. + BOOST_CHECK(!dlt_p2p_node::is_wedged(NOT_BEHIND, /*head_advanced=*/false, + /*rejects_climbed=*/true, LONG_ENOUGH)); +} + +BOOST_AUTO_TEST_CASE(not_wedged_before_confirm_window) { + // All spatial conditions met but not yet sustained long enough. + BOOST_CHECK(!dlt_p2p_node::is_wedged(FAR_BEHIND, /*head_advanced=*/false, + /*rejects_climbed=*/true, TOO_SHORT)); +} + +// ── Boundary conditions ────────────────────────────────────────────── + +BOOST_AUTO_TEST_CASE(behind_boundary_is_strictly_greater) { + // Exactly at the threshold does NOT count (predicate uses `>`, not `>=`). + BOOST_CHECK(!dlt_p2p_node::is_wedged(BEHIND, false, true, LONG_ENOUGH)); + // One block past the threshold does. + BOOST_CHECK(dlt_p2p_node::is_wedged(BEHIND + 1, false, true, LONG_ENOUGH)); +} + +BOOST_AUTO_TEST_CASE(confirm_boundary_is_inclusive) { + // Exactly at the confirm window DOES count (predicate uses `>=`). + BOOST_CHECK(dlt_p2p_node::is_wedged(FAR_BEHIND, false, true, CONFIRM)); + // One second short does not. + BOOST_CHECK(!dlt_p2p_node::is_wedged(FAR_BEHIND, false, true, CONFIRM - 1)); +} + +// Nothing set — the trivial safe default. +BOOST_AUTO_TEST_CASE(not_wedged_when_nothing_holds) { + BOOST_CHECK(!dlt_p2p_node::is_wedged(0, /*head_advanced=*/true, + /*rejects_climbed=*/false, 0)); +} + +BOOST_AUTO_TEST_SUITE_END()