feat: monitoring deployment

.gitignore

@@ -43,3 +43,6 @@ node_modules/
 
 # Pre-built binaries
 deploy_gcp/binaries/
+
+# Monitoring config (contains JWT secrets + Slack webhook)
+monitoring/monitoring.conf

deploy_gcp/seismic_deploy/cli.py

@@ -1107,6 +1107,108 @@ def sync(
     click.secho("Sync complete!", fg="green")
 
 
+# ─── update-config ────────────────────────────────────────────────────────────
+
+
+@cli.command("update-config")
+@click.argument("target")
+@click.option(
+    "--fqdn",
+    help="Fully-qualified domain name for the node (required with --nginx)",
+)
+@click.option(
+    "--nginx", "update_nginx", is_flag=True, help="Re-render and push nginx.conf"
+)
+@click.option(
+    "--supervisor",
+    "update_supervisor",
+    is_flag=True,
+    help="Re-render and push supervisor.conf",
+)
+@click.option(
+    "--bootnode-enode", help="Bootnode enode to inject when rendering supervisor.conf"
+)
+@click.option(
+    "--restart-reth",
+    is_flag=True,
+    help="After pushing supervisor.conf, restart the reth process "
+    "(required for flag changes, e.g. --ops.enable, to take effect; "
+    "briefly interrupts /rpc and /ws, and rotates reth's ops validator_id)",
+)
+@click.option(
+    "--rate-limit-rps", default=20, show_default=True, help="Nginx rate limit (req/s)"
+)
+@click.option(
+    "--rate-limit-burst", default=40, show_default=True, help="Nginx rate limit burst"
+)
+@click.option(
+    "--ssh-key",
+    "ssh_key_path",
+    type=click.Path(exists=True, path_type=Path),
+    help="Path to SSH private key",
+)
+@handle_errors
+def update_config(
+    target: str,
+    fqdn: str | None,
+    update_nginx: bool,
+    update_supervisor: bool,
+    bootnode_enode: str | None,
+    restart_reth: bool,
+    rate_limit_rps: int,
+    rate_limit_burst: int,
+    ssh_key_path: Path | None,
+):
+    """Push updated nginx and/or supervisor config to an already-deployed node.
+
+    TARGET is the SSH host of the node (e.g. ubuntu@1.2.3.4).
+
+    Unlike `deploy`, this does not touch VM/firewall/DNS state and does not
+    require a saved DeployState — it just re-renders the requested template(s)
+    from the current source tree and pushes them over SSH. Nginx reloads with
+    zero downtime; supervisor changes are not applied to the running process
+    until `--restart-reth` is also passed (or you restart it manually).
+    """
+    if not update_nginx and not update_supervisor:
+        raise click.ClickException("Pass --nginx and/or --supervisor")
+    if update_nginx and not fqdn:
+        raise click.ClickException("--fqdn is required with --nginx")
+
+    from deploy_gcp.seismic_deploy.config import DeploymentConfig
+    from deploy_gcp.seismic_deploy.ssh.connection import SSHConnection
+    from deploy_gcp.seismic_deploy.ssh.deploy import deploy_nginx, deploy_supervisor
+
+    conn = SSHConnection(target, ssh_key_path)
+
+    if update_nginx:
+        click.echo(f"Pushing nginx config to {target}...")
+        config = DeploymentConfig(
+            node_name=target,
+            record_name=fqdn.split(".", 1)[0] if fqdn else "",
+            domain_name=fqdn.split(".", 1)[1] if fqdn and "." in fqdn else fqdn or "",
+            rate_limit_rps=rate_limit_rps,
+            rate_limit_burst=rate_limit_burst,
+        )
+        deploy_nginx(conn, config)
+
+    if update_supervisor:
+        click.echo(f"Pushing supervisor config to {target}...")
+        deploy_supervisor(conn, DeploymentConfig(node_name=target), bootnode_enode)
+        if restart_reth:
+            click.echo("Restarting reth...")
+            conn.exec("sudo supervisorctl restart reth")
+            click.echo("  reth restarted")
+        else:
+            click.secho(
+                "  supervisor.conf updated, but reth was NOT restarted — "
+                "flag changes won't take effect until it is "
+                "(rerun with --restart-reth, or restart manually)",
+                fg="yellow",
+            )
+
+    click.secho("Config update complete!", fg="green")
+
+
 # ─── status ───────────────────────────────────────────────────────────────────
 
 

deploy_gcp/seismic_deploy/network/bootnode.py

@@ -71,7 +71,11 @@ def _normalize_rpc_base(host: str) -> str:
 
 
 def fetch_enode(bootnode_rpc: str) -> str:
-    """Fetch the enode URL from a bootnode via seismic_nodeInfo on reth RPC."""
+    """Fetch the enode URL from a bootnode via seismic_nodeInfo on reth RPC.
+
+    The admin namespace was removed from the public RPC (Veridise 1202),
+    so the enode is exposed via the seismic namespace instead.
+    """
     base = _normalize_rpc_base(bootnode_rpc)
     reth_url = f"{base}/rpc"
 

deploy_gcp/seismic_deploy/ssh/connection.py

@@ -127,6 +127,13 @@ def wait_for_file(
                 check=False,
             )
             if status.stdout.strip() in ("inactive", "failed"):
+                # The script may have finished (and written the marker) between
+                # the file check above and this status check — re-check before
+                # declaring failure.
+                result = self.exec(f"test -f '{filepath}'", check=False)
+                if result.returncode == 0:
+                    click.echo(f"  Remote file found: {filepath}")
+                    return
                 # Grab last 30 lines for diagnostics
                 logs = self.exec(
                     "sudo journalctl -u google-startup-scripts.service "

deploy_gcp/templates/nginx.conf

@@ -79,6 +79,16 @@ http {
             proxy_pass http://localhost:3030;
         }
 
+        # Ops signature-auth RPC (privileged: whitelist/revoke sentinel txs,
+        # validator id / admin nonce bootstrap reads). Requests are further
+        # gated by EIP-712 signature auth inside reth itself; rate limiting
+        # here is a first line of defense against abuse.
+        location /ops {
+            access_by_lua_file /usr/local/openresty/nginx/lua/rate_limit.lua;
+            rewrite ^/ops/?(.*)$ /$1 break;
+            proxy_pass http://localhost:8552;
+        }
+
         # Prometheus Summit (rate limited + JWT protected)
         location /prom-summit {
             access_by_lua_block {

deploy_gcp/templates/supervisor.conf

@@ -31,6 +31,9 @@ command=/home/ubuntu/seismic-reth/target/release/seismic-reth
     --disable-dns-discovery
     --enable-discv5-discovery
     --discovery.v5.port 30303
+    --ops.enable
+    --ops.addr 127.0.0.1
+    --ops.port 8552
 autostart=false
 autorestart=false
 startsecs=10

deploy_metadata.json

@@ -170,6 +170,86 @@
           "zone": "us-central1-a",
           "size": "c4-standard-4-lssd"
         }
+      },
+      "temp-1": {
+        "artifact": "ubuntu-minimal-2404-auto-build",
+        "public_ip": "34.141.5.238",
+        "domain": {
+          "url": "https://temp-1.seismictest.net",
+          "record": "temp-1",
+          "name": "seismictest.net"
+        },
+        "vm": {
+          "project": "testnet-477314",
+          "name": "temp-1",
+          "cloud": "gcp",
+          "zone": "europe-west3-a",
+          "size": "c4-standard-4-lssd"
+        }
+      },
+      "node-0-internal": {
+        "artifact": "ubuntu-minimal-2404-auto-build",
+        "public_ip": "136.119.180.186",
+        "domain": {
+          "url": "https://internal-0.seismictest.net",
+          "record": "internal-0",
+          "name": "seismictest.net"
+        },
+        "vm": {
+          "project": "testnet-477314",
+          "name": "node-0-internal",
+          "cloud": "gcp",
+          "zone": "us-central1-c",
+          "size": "c4-standard-4-lssd"
+        }
+      },
+      "node-1-internal": {
+        "artifact": "ubuntu-minimal-2404-auto-build",
+        "public_ip": "104.198.81.77",
+        "domain": {
+          "url": "https://internal-1.seismictest.net",
+          "record": "internal-1",
+          "name": "seismictest.net"
+        },
+        "vm": {
+          "project": "testnet-477314",
+          "name": "node-1-internal",
+          "cloud": "gcp",
+          "zone": "asia-northeast1-c",
+          "size": "c4-standard-4-lssd"
+        }
+      },
+      "node-2-internal": {
+        "artifact": "ubuntu-minimal-2404-auto-build",
+        "public_ip": "34.34.132.127",
+        "domain": {
+          "url": "https://internal-2.seismictest.net",
+          "record": "internal-2",
+          "name": "seismictest.net"
+        },
+        "vm": {
+          "project": "testnet-477314",
+          "name": "node-2-internal",
+          "cloud": "gcp",
+          "zone": "europe-west1-c",
+          "size": "c4-standard-4-lssd"
+        }
+      },
+      "node-3-internal": {
+        "artifact": "ubuntu-minimal-2404-auto-build",
+        "public_ip": "34.129.54.128",
+        "domain": {
+          "url": "https://internal-3.seismictest.net",
+          "record": "internal-3",
+          "name": "seismictest.net"
+        },
+        "vm": {
+          "project": "testnet-477314",
+          "name": "node-3-internal",
+          "cloud": "gcp",
+          "zone": "australia-southeast2-b",
+          "size": "c4-standard-4-lssd"
+        }
       }
     }
   },