From 2a7c00c3d917fe08663640f272eadaa54496eab6 Mon Sep 17 00:00:00 2001 From: nol4lej Date: Sun, 26 Jul 2026 23:41:46 -0400 Subject: [PATCH 1/3] fix(caddy): unbuffer WS proxy and raise upgrade rate limit --- common/Caddyfile | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/common/Caddyfile b/common/Caddyfile index da44f21..71e9fc4 100644 --- a/common/Caddyfile +++ b/common/Caddyfile @@ -8,6 +8,11 @@ # browser reject the response with "multiple values '*, *'"). (rpc_upstream) { reverse_proxy localhost:9944 { + # WebSocket subscriptions are long-lived bidirectional streams. Without + # immediate flushing Caddy buffers frames and can stall/drop idle sockets, + # which shows up as WS dying after seconds and clients reconnecting in a + # loop. -1 disables buffering so frames pass through as they arrive. + flush_interval -1 header_down -Access-Control-Allow-Origin header_down -Access-Control-Allow-Methods header_down -Access-Control-Allow-Headers @@ -48,7 +53,7 @@ # WebSocket subscriptions (live blocks, mempool) are a single long-lived upgrade # request. The limiter counts each upgrade as one event, so this caps how often # an IP may (re)open a socket — NOT how long one stays open. A healthy client - # opens one socket and holds it (the SDK's ws heartbeat keeps it alive), so 10 + # opens one socket and holds it (the SDK's ws heartbeat keeps it alive), so 30 # upgrades / 60s is ample headroom for a page reload or a genuine reconnect, # while a bot that reopens a socket every few seconds trips it. Keyed on the # real client IP Cloudflare forwards; direct-origin (no CF header) falls back @@ -60,7 +65,7 @@ rate_limit { zone ws_cf { key {http.request.header.CF-Connecting-IP} - events 10 + events 30 window 60s } } @@ -70,7 +75,7 @@ rate_limit { zone ws_direct { key {remote_host} - events 10 + events 30 window 60s } } From 033b8c57b9e13f001439dcbb41d9150481ce88ea Mon Sep 17 00:00:00 2001 From: nol4lej Date: Tue, 28 Jul 2026 17:04:31 -0400 Subject: [PATCH 2/3] fix(caddy): unbuffer WS proxy and raise upgrade rate limit Disable response buffering on the RPC upstream (flush_interval -1) so WebSocket frames pass through immediately. Buffering stalled long-lived subscription sockets, which surfaced as WS connections dying after a few seconds and clients reconnecting in a loop. Raise the WS upgrade limit from 10 to 30 events / 60s on both the CF-Connecting-IP and direct-origin zones, so the reconnect storms caused by the above don't trip the limiter while the fix is validated. Also pin COMPOSE_PROJECT_NAME=testnet in the RPC .env example. Compose derives the volume prefix from the directory name, so this stack would look for `rpc_rpc-node-data` while nodes deployed from the old `docker/testnet/` layout keep their chain DB in `testnet_rpc-node-data`. Without the pin, migrating an existing RPC starts an archive resync from block 0. --- testnet/rpc/.env.example | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/testnet/rpc/.env.example b/testnet/rpc/.env.example index db55ae4..3e44de5 100644 --- a/testnet/rpc/.env.example +++ b/testnet/rpc/.env.example @@ -9,6 +9,14 @@ # must be Proxied (orange cloud) in Cloudflare. # ───────────────────────────────────────────────────────────────────────────── +# Compose derives the project name — and therefore the volume prefix — from the +# directory name, so this stack would look for `rpc_rpc-node-data`. Nodes first +# deployed from the old `docker/testnet/` layout have their chain DB in +# `testnet_rpc-node-data`; pinning the project name here reuses that volume +# instead of starting an archive resync from block 0. Keep it set on fresh +# installs too, so the volume name no longer depends on where the repo is cloned. +COMPOSE_PROJECT_NAME=testnet + # Container image (override to pin a specific tag) ORBINUM_IMAGE=ghcr.io/orbinum/node:testnet-latest From 6bab1a2299ac4915afada02b0caca697fc0ab129 Mon Sep 17 00:00:00 2001 From: nol4lej Date: Tue, 28 Jul 2026 17:27:59 -0400 Subject: [PATCH 3/3] fix(scripts): track sync-cloudflare-ufw.sh --- scripts/sync-cloudflare-ufw.sh | 56 ++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100755 scripts/sync-cloudflare-ufw.sh diff --git a/scripts/sync-cloudflare-ufw.sh b/scripts/sync-cloudflare-ufw.sh new file mode 100755 index 0000000..d03b4a3 --- /dev/null +++ b/scripts/sync-cloudflare-ufw.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Sync the ufw allow-list for ports 80/443 to Cloudflare's current edge ranges. +# +# Cloudflare's IP ranges change occasionally. If a new range is added and ufw +# doesn't allow it, legitimate Cloudflare traffic to the origin gets dropped. +# Run this from cron (monthly is plenty) to keep the rules in sync. +# +# It is idempotent: it deletes the ufw rules it previously added (tagged via the +# rule comment) and re-adds the current set, so it never accumulates stale rules. +# +# Usage: +# sudo ./sync-cloudflare-ufw.sh +# +# Cron (monthly, logs to syslog): +# 0 4 1 * * root /path/to/node-deploy/scripts/sync-cloudflare-ufw.sh 2>&1 | logger -t cf-ufw +set -euo pipefail + +PORTS=(80 443) +TAG="cf-edge" # ufw rule comment used to find/remove our own rules + +if [[ $EUID -ne 0 ]]; then + echo "Must run as root (ufw needs root)." >&2 + exit 1 +fi + +# Fetch current ranges. Fail closed: if the fetch fails, leave existing rules +# untouched rather than wiping the allow-list and locking Cloudflare out. +v4="$(curl -fsS https://www.cloudflare.com/ips-v4)" || { echo "fetch ips-v4 failed" >&2; exit 1; } +v6="$(curl -fsS https://www.cloudflare.com/ips-v6)" || { echo "fetch ips-v6 failed" >&2; exit 1; } + +if [[ -z "$v4" ]]; then + echo "empty ips-v4 response, aborting" >&2 + exit 1 +fi + +# Remove the rules we added on a previous run (matched by the TAG comment). +# ufw has no "delete by comment", so parse `ufw status numbered` and delete by +# index from the bottom up (indices shift as you delete). +mapfile -t stale < <(ufw status numbered | grep "# ${TAG}" | grep -oE '^\[[ 0-9]+\]' | tr -d '[] ' | sort -rn) +for idx in "${stale[@]}"; do + # --force skips the y/n prompt. Do NOT pipe `yes` here: when ufw exits, yes + # keeps writing to a closed pipe, gets SIGPIPE, and under `set -o pipefail` + # that aborts the whole script mid-delete (leaving the allow-list incomplete). + ufw --force delete "$idx" >/dev/null +done + +# Re-add the current ranges for each port. ponytail: empty ips-v6 just means +# no v6 rules get added — harmless on an IPv4-only host. +for ip in $v4 $v6; do + [[ -z "$ip" ]] && continue + for port in "${PORTS[@]}"; do + ufw allow from "$ip" to any port "$port" proto tcp comment "${TAG}" >/dev/null + done +done + +echo "Synced ufw ${PORTS[*]} allow-list to $(wc -w <<<"$v4 $v6") Cloudflare ranges."