Dump the database before applying pending migrations

.github/workflows/publish.yaml

@@ -69,6 +69,7 @@ jobs:
 
           app=$(docker compose -p smoke ps -q --all template-app)
           mig=$(docker compose -p smoke ps -q --all template-migrations)
+          bak=$(docker compose -p smoke ps -q --all template-backup)
 
           for i in $(seq 1 60); do
             status=$(docker inspect -f '{{.State.Health.Status}}' "$app")
@@ -77,6 +78,27 @@ jobs:
             sleep 2
           done
 
+          # BACKUP_MODE defaults to required, so migrations only ran if the dump succeeded.
+          code=$(docker inspect -f '{{.State.ExitCode}}' "$bak")
+          if [ "$code" != "0" ]; then
+            echo "::error::pre-migration backup exited $code"
+            docker compose -p smoke logs
+            exit 1
+          fi
+
+          docker compose -p smoke logs template-backup | grep -q "Backup complete" || {
+            echo "::error::backup service did not report a completed dump"
+            docker compose -p smoke logs template-backup
+            exit 1
+          }
+
+          # pg_restore --list proves the dump is a readable archive, not just a non-empty file.
+          docker compose -p smoke run --rm --no-deps --entrypoint /bin/sh template-backup \
+            -euc 'f=$(ls -1t /backups/pre-migrate-*.dump | head -1); pg_restore --list "$f" >/dev/null && echo "dump verified: $f"' || {
+            echo "::error::pre-migration dump failed pg_restore --list"
+            exit 1
+          }
+
           code=$(docker inspect -f '{{.State.ExitCode}}' "$mig")
           if [ "$code" != "0" ]; then
             echo "::error::migrations exited $code in the published image"

README.md

@@ -116,6 +116,8 @@ The application uses Docker Compose for production deployments with an automated
 ### Architecture
 
 - **Database**: PostgreSQL 17 with persistent volume storage
+- **Backup**: Init container that `pg_dump`s the database before migrations run,
+  from the same image as the database
 - **Migrations**: Init container that runs database migrations before the app
   starts, from the same image as the app
 - **Application**: Next.js standalone server with optimized production build
@@ -149,6 +151,38 @@ template-migrations:
 The ordering guarantee is unchanged — the app still waits on
 `service_completed_successfully`, so it starts only after migrations exit 0.
 
+### Pre-migration backup
+
+`prisma migrate deploy` is forward-only, so a bad migration has no way back.
+`template-backup` runs `pg_dump -Fc` before `template-migrations`, using the
+same chaining the app already relies on:
+
+```
+template-db (healthy) → template-backup → template-migrations → template-app
+```
+
+It uses the `postgres:17` image rather than the app image, so `pg_dump` is
+version-matched to the server by construction and there is no client to keep in
+sync. Dumps are verified with `pg_restore --list` and moved into place only
+after passing, so a truncated file can never look like a good backup. They land
+on the `template-backups` volume, pruned to the newest `BACKUP_KEEP` (default 10).
+
+`BACKUP_MODE` controls it:
+
+| Value                | Behaviour                                              |
+| -------------------- | ------------------------------------------------------ |
+| `required` (default) | A failed dump blocks migrations — the app never starts |
+| `best-effort`        | Dumps, but migrates anyway if the dump fails           |
+| `off`                | Never dumps                                            |
+
+`required` needs no scripting to enforce: `template-migrations` waits on
+`service_completed_successfully`, so a non-zero backup stops the chain. An
+unrecognised `BACKUP_MODE` fails rather than silently downgrading.
+
+> These dumps sit on the same host and disk as `template-db`. They protect
+> against a bad migration, **not** against losing the machine. Add off-box
+> backups separately if the data warrants it.
+
 The migration tooling is installed with **npm**, not pnpm, and lands at
 `/node_modules` rather than `/app/node_modules`. Both details are load-bearing:
 pnpm's symlink farm does not survive a `COPY` between stages, and the Next.js

docker-compose.yml

@@ -19,6 +19,63 @@ services:
       start_period: 30s
       start_interval: 2s
 
+  # Pre-migration backup (production only). `migrate deploy` is forward-only, so
+  # this is the only way back from a bad migration. Same image as the database so
+  # pg_dump is version-matched by construction. BACKUP_MODE=required makes a failed
+  # dump block migrations, because template-migrations waits on this completing.
+  # NOTE: these dumps live on the same host and disk as template-db — they protect
+  # against a bad migration, not against losing the machine.
+  template-backup:
+    image: postgres:17
+    entrypoint: ["/bin/sh", "-euc"]
+    command:
+      - |
+        MODE="$${BACKUP_MODE:-required}"
+        case "$$MODE" in
+          off) echo "[backup] BACKUP_MODE=off; skipping the pre-migration backup."; exit 0 ;;
+          best-effort|required) ;;
+          *) echo "[backup] ERROR: BACKUP_MODE must be off, best-effort or required (got '$$MODE')."; exit 1 ;;
+        esac
+
+        stamp=$$(date -u +%Y%m%dT%H%M%SZ)
+        final="/backups/pre-migrate-$$stamp.dump"
+        tmp="/backups/.pre-migrate-$$stamp.dump"
+
+        # Move into place only after verifying, so a partial file never looks like a good backup.
+        fail() {
+          echo "[backup] $$1"
+          rm -f "$$tmp"
+          if [ "$$MODE" = required ]; then exit 1; fi
+          echo "[backup] continuing anyway because BACKUP_MODE=$$MODE."
+          exit 0
+        }
+
+        echo "[backup] Dumping $$PGDATABASE from $$PGHOST:$${PGPORT:-5432} to $$final ..."
+        pg_dump -Fc -f "$$tmp" || fail "pg_dump failed"
+        pg_restore --list "$$tmp" >/dev/null 2>&1 || fail "dump failed pg_restore --list verification"
+        mv "$$tmp" "$$final"
+        echo "[backup] Backup complete: $$final ($$(du -h "$$final" | cut -f1))"
+
+        ls -1t /backups/pre-migrate-*.dump 2>/dev/null | tail -n "+$$(( $${BACKUP_KEEP:-10} + 1 ))" | while read -r old; do
+          echo "[backup] Pruning old backup $$(basename "$$old")"
+          rm -f "$$old"
+        done
+    environment:
+      - PGHOST=template-db
+      - PGUSER=${DATABASE_USER}
+      - PGPASSWORD=${DATABASE_PW}
+      - PGDATABASE=${DATABASE_NAME}
+      - BACKUP_MODE=${BACKUP_MODE:-required}
+      - BACKUP_KEEP=${BACKUP_KEEP:-10}
+    volumes:
+      - template-backups:/backups
+    restart: "no"
+    depends_on:
+      template-db:
+        condition: service_healthy
+    profiles:
+      - production
+
   # Database migrations (production only). Same image as the app, different
   # command — the image ships the Prisma CLI for exactly this.
   template-migrations:
@@ -30,6 +87,8 @@ services:
     depends_on:
       template-db:
         condition: service_healthy
+      template-backup:
+        condition: service_completed_successfully
     profiles:
       - production
 
@@ -69,4 +128,6 @@ services:
       - production
 
 volumes:
-  template-db-data:
+  template-db-data:
+  # Pre-migration dumps; must outlive the one-shot backup container.
+  template-backups:

example.env

@@ -19,4 +19,8 @@ AUTH_GOOGLE_SECRET=
 RESEND_API_KEY=
 # Replace with keys generate from https://knock.app/tools/vapid-key-generator
 NEXT_PUBLIC_VAPID_PUBLIC_KEY=
-VAPID_PRIVATE_KEY=
+VAPID_PRIVATE_KEY=
+# Pre-migration pg_dump, taken before template-migrations runs.
+# off = never dump, best-effort = dump but migrate anyway on failure, required = block the migration.
+BACKUP_MODE=required
+BACKUP_KEEP=10