fix(db): one wedged handler could freeze the whole database — three bounds

.github/workflows/deploy.yml

@@ -131,7 +131,15 @@ jobs:
     permissions:
       contents: write
     steps:
+      # fetch-depth: 0, not the default shallow clone. `git push` proves the
+      # fast-forward CLIENT-side: with depth 1 the runner does not have the
+      # commit `release` currently points at, so every push after the one that
+      # CREATED the ref was rejected with "fetch first" — release sat at the
+      # first published commit (#360) while three merges deployed past it,
+      # and the box's pull path had nothing new to converge on.
       - uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
 
       - name: Fast-forward release to this commit
         run: |
@@ -200,7 +208,14 @@ jobs:
           cp scripts/vps_apply.sh /tmp/deploy_remote.sh
 
           ssh_deploy() {
-            timeout 900 ssh \
+            # 1800s, matched to the pull unit's TimeoutStartSec. 900 was fatal
+            # in a way verify() cannot see: the pre-migration backup alone
+            # takes ~11 minutes on the 4GB box, so the session was killed
+            # MID-APPLY — migrations never ran, services never restarted —
+            # and verify() then blessed the still-running OLD deployment as
+            # green. A timeout kill is the one failure mode where "the app is
+            # healthy" says nothing about "the deploy happened".
+            timeout 1800 ssh \
               -i ~/.ssh/deploy_key -p "$SSH_PORT" \
               -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
               -o ConnectTimeout=30 -o ServerAliveInterval=15 \

ai-company-brain/specs/deploy_delivery_path.md

@@ -310,6 +310,35 @@ run this script as `ssh acb@host 'bash -s'`, and the applied script calls `sudo`
 passwordless sudo. So `User=acb` is not a workaround; running it as root was the
 deviation.
 
+⚠️ **Corrections exist on paper until they are installed.** The corrected unit
+below sat in this spec while the box kept running the `User=root` version — the
+poller failed every five minutes from its first tick until 2026-08-06, when the
+WS-26 activation found it. `.claude/` and `/etc/systemd/system/` share the same
+failure mode: not in the repo, so no PR can fix them; every correction here is
+also a hand-carried box change.
+
+**Defect 3 (2026-08-06) — the SHA can be current while the APPLY never
+happened.** The poller skips when `HEAD == origin/release`. But the push path's
+`git reset --hard` moves HEAD *before* migrations and restarts — so when the
+push-path SSH session was killed mid-apply (Defect 4b), the tree said `8d83ca10`
+while the DB had no migration 144/145 and both services still ran old code. The
+poller then reported `already current` and stood down: both delivery paths
+converged on believing a deploy that had not happened. `--force` is the manual
+escape; the durable fix would be gating the skip on the `/var/lib/acb`
+last-success marker (recorded at the END of an apply), not on git state.
+
+**Defect 4 (2026-08-06) — two `deploy.yml` failures, one green run.**
+(a) `publish-release` used the default depth-1 checkout; `git push` proves
+fast-forward client-side, so every publish after the ref-CREATING one was
+rejected with "fetch first" — `release` sat at #360 for three merges. Fixed:
+`fetch-depth: 0`. (b) `ssh_deploy`'s `timeout 900` is shorter than the
+pre-migration backup alone (~11 min on the 4GB box); the session was killed
+mid-apply and `verify()` blessed the still-healthy OLD deployment. Fixed:
+`timeout 1800`, matched to the pull unit's `TimeoutStartSec`. Residual hole,
+accepted and named: health-verify cannot distinguish "deploy succeeded" from
+"deploy never finished but yesterday's app is healthy" — only an
+identity-bearing health signal (commit SHA in `/health`) closes it.
+
 ```ini
 # /etc/systemd/system/acb-pull.service
 [Unit]

apps/services/email_ingestion/email_ingestion/inbound.py

@@ -271,7 +271,7 @@ async def _persist_message(msg: EmailMessage) -> None:
         from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
 
         engine = create_async_engine(
-            db_url, echo=False, connect_args={"timeout": _connect_timeout()}
+            db_url, echo=False, connect_args=_connect_args()
         )
         session_factory = async_sessionmaker(engine, expire_on_commit=False)
 
@@ -322,6 +322,22 @@ def _connect_timeout() -> int:
         return 10
 
 
+def _connect_args() -> dict:
+    """Connect args for this module's engines.
+
+    :func:`_connect_timeout` bounds getting IN; the server-side
+    ``idle_in_transaction_session_timeout`` bounds staying in, so a wedged
+    persist cannot hold its locks for the life of the process. Mirrors
+    ``scheduler._connect_args`` and ``gateway.db.engine_connect_args`` — see the
+    2026-08-06 write-up for why an unbounded one took the app down."""
+    return {
+        "timeout": _connect_timeout(),
+        "server_settings": {
+            "idle_in_transaction_session_timeout": "600000",  # 10 min
+        },
+    }
+
+
 # -- Lifecycle management -----------------------------------------------------
 
 

apps/services/email_ingestion/email_ingestion/scheduler.py

@@ -87,6 +87,24 @@ def _connect_timeout() -> int:
         return 10
 
 
+def _connect_args() -> dict:
+    """Connect args for every engine in this service.
+
+    :func:`_connect_timeout` bounds getting IN; the server-side
+    ``idle_in_transaction_session_timeout`` bounds staying in. A sync tick holds
+    an open session across provider calls, so a tick that wedges would otherwise
+    hold its row locks for as long as the process lives — which on 2026-08-06
+    (gateway side, same shape) queued a migration's ALTER TABLE behind it and
+    took the whole app down. Mirrors ``gateway.db.engine_connect_args``; the two
+    are separate packages and cannot share the constant."""
+    return {
+        "timeout": _connect_timeout(),
+        "server_settings": {
+            "idle_in_transaction_session_timeout": "600000",  # 10 min
+        },
+    }
+
+
 def _next_backoff(current: int, interval: int, *, failed: bool) -> int:
     """The next sleep length for a sync loop.
 
@@ -141,7 +159,7 @@ async def _sync_account(
     # the session releases it, so the missing dispose no longer leaks.
     engine = create_async_engine(
         db_url, echo=False, poolclass=NullPool,
-        connect_args={"timeout": _connect_timeout()},
+        connect_args=_connect_args(),
     )
     session_factory = async_sessionmaker(engine, expire_on_commit=False)
 
@@ -525,7 +543,7 @@ async def _get_account_sync_interval(account_id: str) -> int | None:
     """Read the current sync_interval_secs for an account."""
     db_url = _get_db_url()
     engine = create_async_engine(
-        db_url, echo=False, connect_args={"timeout": _connect_timeout()}
+        db_url, echo=False, connect_args=_connect_args()
     )
     session_factory = async_sessionmaker(engine, expire_on_commit=False)
     try:
@@ -558,7 +576,7 @@ async def start_background_sync() -> dict[str, int]:
 
         db_url = _get_db_url()
         engine = create_async_engine(
-            db_url, echo=False, connect_args={"timeout": _connect_timeout()}
+            db_url, echo=False, connect_args=_connect_args()
         )
         session_factory = async_sessionmaker(engine, expire_on_commit=False)
 

apps/services/gateway/gateway/db.py

@@ -53,20 +53,60 @@ def async_database_url() -> str:
     return db_url
 
 
+#: Ceiling on how long one of OUR sessions may sit `idle in transaction`, in ms.
+#:
+#: This is a lock-release deadline, not a performance knob. SQLAlchemy's
+#: ``AsyncSession`` opens a transaction on first ``execute()`` and holds it until
+#: commit/rollback/close, so a handler that reads a row and then awaits a slow
+#: network call is `idle in transaction` — holding an ACCESS SHARE lock — for the
+#: whole call. That is normal and fine. What is not fine is an await that never
+#: returns: on 2026-08-06 a hung LLM call pinned one such transaction for 14h44m,
+#: a migration's ``ALTER TABLE`` queued behind its lock, and because Postgres's
+#: lock queue is FIFO every later reader of that table queued behind the *waiting*
+#: ALTER. Sending mail stopped, and the pool drained behind the blocked readers.
+#:
+#: ⚠️ MUST stay comfortably above the LLM wall-clock worst case, because the email
+#: automation package legitimately awaits completions with a session open.
+#: ``acb_llm.client`` bounds one call at 3 attempts x 90s + 6s backoff ≈ 276s. At
+#: 600s a genuine retrying completion can never trip this, while a hang is capped
+#: at ten minutes instead of unbounded. Raise ``LLM_REQUEST_TIMEOUT_SECS`` and you
+#: must raise this too — that coupling is the whole reason both numbers are
+#: written down next to their reasoning.
+_IDLE_IN_TXN_TIMEOUT_MS = "600000"  # 10 minutes
+
+
+def engine_connect_args() -> dict[str, Any]:
+    """Driver-level connect args every gateway engine should share.
+
+    Two bounds, both about failing instead of hanging:
+
+    * ``timeout`` — asyncpg's CONNECT-phase ceiling, so a slow or unreachable DB
+      fails fast rather than stalling request handlers.
+    * ``idle_in_transaction_session_timeout`` — the server-side deadline above.
+      Set through asyncpg's ``server_settings`` so it rides the connection's
+      startup packet and applies to every session from this pool, with no
+      migration and no ``ALTER ROLE``. Scoping it to the app's own connections is
+      deliberate: ``pg_dump`` and the migration runner connect as the same role
+      and must NOT inherit an app-tuned deadline.
+    """
+    return {
+        "timeout": get_settings().db_connect_timeout,
+        "server_settings": {
+            "idle_in_transaction_session_timeout": _IDLE_IN_TXN_TIMEOUT_MS,
+        },
+    }
+
+
 def get_engine() -> Any:
     """The shared pooled async engine, created on first use."""
     global _ENGINE
     if _ENGINE is None:
         from sqlalchemy.ext.asyncio import create_async_engine
 
-        settings = get_settings()
         _ENGINE = create_async_engine(
             async_database_url(), echo=False, pool_pre_ping=True,
             pool_size=10, max_overflow=20, pool_recycle=1800,
-            # Bound the CONNECT phase (asyncpg's `timeout`) so a slow or
-            # unreachable DB fails fast instead of stalling request handlers —
-            # same ceiling as acb_graph's engine.
-            connect_args={"timeout": settings.db_connect_timeout},
+            connect_args=engine_connect_args(),
         )
     return _ENGINE
 

apps/services/gateway/gateway/routes/admin/_common.py

@@ -75,6 +75,7 @@ def _get_session_factory() -> Any:
             create_async_engine,
         )
 
+        from gateway.db import engine_connect_args
         settings = get_settings()
         db_url = os.environ.get("DATABASE_URL", settings.database_url)
         if "postgresql+psycopg" in db_url:
@@ -84,6 +85,7 @@ def _get_session_factory() -> Any:
         _ENGINE = create_async_engine(
             db_url, echo=False, pool_pre_ping=True,
             pool_size=5, max_overflow=10, pool_recycle=1800,
+            connect_args=engine_connect_args(),
         )
         _SESSION_FACTORY = async_sessionmaker(_ENGINE, expire_on_commit=False)
     return _SESSION_FACTORY

apps/services/gateway/gateway/routes/apps/_common.py

@@ -80,6 +80,8 @@ def _get_session_factory() -> Any:
             async_sessionmaker,
             create_async_engine,
         )
+
+        from gateway.db import engine_connect_args
         settings = get_settings()
         db_url = os.environ.get("DATABASE_URL", settings.database_url)
         if "postgresql+psycopg" in db_url:
@@ -89,7 +91,7 @@ def _get_session_factory() -> Any:
         _ENGINE = create_async_engine(
             db_url, echo=False, pool_pre_ping=True,
             pool_size=10, max_overflow=20, pool_recycle=1800,
-            connect_args={"timeout": settings.db_connect_timeout},
+            connect_args=engine_connect_args(),
         )
         _SESSION_FACTORY = async_sessionmaker(_ENGINE, expire_on_commit=False)
     return _SESSION_FACTORY

apps/services/gateway/gateway/routes/email/core.py

@@ -393,6 +393,8 @@ def _get_session_factory():
             async_sessionmaker,
             create_async_engine,
         )
+
+        from gateway.db import engine_connect_args
         settings = get_settings()
         db_url = os.environ.get("DATABASE_URL", settings.database_url)
         if "postgresql+psycopg" in db_url:
@@ -404,10 +406,11 @@ def _get_session_factory():
         _ENGINE = create_async_engine(
             db_url, echo=False, pool_pre_ping=True,
             pool_size=10, max_overflow=20, pool_recycle=1800,
-            # Bound the CONNECT phase (asyncpg's `timeout`) so a slow/unreachable
-            # DB fails fast instead of stalling request handlers — same ceiling
-            # as acb_graph's engine (settings.db_connect_timeout).
-            connect_args={"timeout": settings.db_connect_timeout},
+            # Bounds the connect phase AND how long one of our sessions may sit
+            # `idle in transaction`. This engine is the one that drained on
+            # 2026-08-06 — see gateway/db.py::engine_connect_args for why the
+            # second bound exists and why it is 10 minutes and not 5.
+            connect_args=engine_connect_args(),
         )
         _SESSION_FACTORY = async_sessionmaker(_ENGINE, expire_on_commit=False)
     return _SESSION_FACTORY

apps/services/gateway/gateway/routes/notes/core.py

@@ -152,6 +152,8 @@ def _get_session_factory():
             async_sessionmaker,
             create_async_engine,
         )
+
+        from gateway.db import engine_connect_args
         settings = get_settings()
         db_url = os.environ.get("DATABASE_URL", settings.database_url)
         if "postgresql+psycopg" in db_url:
@@ -161,7 +163,7 @@ def _get_session_factory():
         _ENGINE = create_async_engine(
             db_url, echo=False, pool_pre_ping=True,
             pool_size=5, max_overflow=10, pool_recycle=1800,
-            connect_args={"timeout": settings.db_connect_timeout},
+            connect_args=engine_connect_args(),
         )
         _SESSION_FACTORY = async_sessionmaker(_ENGINE, expire_on_commit=False)
     return _SESSION_FACTORY

apps/services/gateway/gateway/routes/whatsapp/core.py

@@ -126,6 +126,8 @@ def _get_session_factory():
             async_sessionmaker,
             create_async_engine,
         )
+
+        from gateway.db import engine_connect_args
         settings = get_settings()
         db_url = os.environ.get("DATABASE_URL", settings.database_url)
         if "postgresql+psycopg" in db_url:
@@ -135,7 +137,7 @@ def _get_session_factory():
         _ENGINE = create_async_engine(
             db_url, echo=False, pool_pre_ping=True,
             pool_size=5, max_overflow=10, pool_recycle=1800,
-            connect_args={"timeout": settings.db_connect_timeout},
+            connect_args=engine_connect_args(),
         )
         _SESSION_FACTORY = async_sessionmaker(_ENGINE, expire_on_commit=False)
     return _SESSION_FACTORY

apps/services/gateway/gateway/routes/workflows/core.py

@@ -55,6 +55,7 @@ def _get_session_factory() -> Any:
             create_async_engine,
         )
 
+        from gateway.db import engine_connect_args
         settings = get_settings()
         db_url = os.environ.get("DATABASE_URL", settings.database_url)
         if "postgresql+psycopg" in db_url:
@@ -68,7 +69,7 @@ def _get_session_factory() -> Any:
             pool_size=5,
             max_overflow=10,
             pool_recycle=1800,
-            connect_args={"timeout": settings.db_connect_timeout},
+            connect_args=engine_connect_args(),
         )
         _SESSION_FACTORY = async_sessionmaker(_ENGINE, expire_on_commit=False)
     return _SESSION_FACTORY