diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index b8e8fa5..665c6f4 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -69,6 +69,7 @@ jobs: app=$(docker compose -p smoke ps -q --all template-app) mig=$(docker compose -p smoke ps -q --all template-migrations) + bak=$(docker compose -p smoke ps -q --all template-backup) for i in $(seq 1 60); do status=$(docker inspect -f '{{.State.Health.Status}}' "$app") @@ -77,6 +78,27 @@ jobs: sleep 2 done + # BACKUP_MODE defaults to required, so migrations only ran if the dump succeeded. + code=$(docker inspect -f '{{.State.ExitCode}}' "$bak") + if [ "$code" != "0" ]; then + echo "::error::pre-migration backup exited $code" + docker compose -p smoke logs + exit 1 + fi + + docker compose -p smoke logs template-backup | grep -q "Backup complete" || { + echo "::error::backup service did not report a completed dump" + docker compose -p smoke logs template-backup + exit 1 + } + + # pg_restore --list proves the dump is a readable archive, not just a non-empty file. + docker compose -p smoke run --rm --no-deps --entrypoint /bin/sh template-backup \ + -euc 'f=$(ls -1t /backups/pre-migrate-*.dump | head -1); pg_restore --list "$f" >/dev/null && echo "dump verified: $f"' || { + echo "::error::pre-migration dump failed pg_restore --list" + exit 1 + } + code=$(docker inspect -f '{{.State.ExitCode}}' "$mig") if [ "$code" != "0" ]; then echo "::error::migrations exited $code in the published image" diff --git a/README.md b/README.md index 6b4fa63..ec9b20c 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,8 @@ The application uses Docker Compose for production deployments with an automated ### Architecture - **Database**: PostgreSQL 17 with persistent volume storage +- **Backup**: Init container that `pg_dump`s the database before migrations run, + from the same image as the database - **Migrations**: Init container that runs database migrations before the app starts, from the same image as the app - **Application**: Next.js standalone server with optimized production build @@ -149,6 +151,38 @@ template-migrations: The ordering guarantee is unchanged — the app still waits on `service_completed_successfully`, so it starts only after migrations exit 0. +### Pre-migration backup + +`prisma migrate deploy` is forward-only, so a bad migration has no way back. +`template-backup` runs `pg_dump -Fc` before `template-migrations`, using the +same chaining the app already relies on: + +``` +template-db (healthy) → template-backup → template-migrations → template-app +``` + +It uses the `postgres:17` image rather than the app image, so `pg_dump` is +version-matched to the server by construction and there is no client to keep in +sync. Dumps are verified with `pg_restore --list` and moved into place only +after passing, so a truncated file can never look like a good backup. They land +on the `template-backups` volume, pruned to the newest `BACKUP_KEEP` (default 10). + +`BACKUP_MODE` controls it: + +| Value | Behaviour | +| -------------------- | ------------------------------------------------------ | +| `required` (default) | A failed dump blocks migrations — the app never starts | +| `best-effort` | Dumps, but migrates anyway if the dump fails | +| `off` | Never dumps | + +`required` needs no scripting to enforce: `template-migrations` waits on +`service_completed_successfully`, so a non-zero backup stops the chain. An +unrecognised `BACKUP_MODE` fails rather than silently downgrading. + +> These dumps sit on the same host and disk as `template-db`. They protect +> against a bad migration, **not** against losing the machine. Add off-box +> backups separately if the data warrants it. + The migration tooling is installed with **npm**, not pnpm, and lands at `/node_modules` rather than `/app/node_modules`. Both details are load-bearing: pnpm's symlink farm does not survive a `COPY` between stages, and the Next.js diff --git a/docker-compose.yml b/docker-compose.yml index 1261251..367809f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,6 +19,63 @@ services: start_period: 30s start_interval: 2s + # Pre-migration backup (production only). `migrate deploy` is forward-only, so + # this is the only way back from a bad migration. Same image as the database so + # pg_dump is version-matched by construction. BACKUP_MODE=required makes a failed + # dump block migrations, because template-migrations waits on this completing. + # NOTE: these dumps live on the same host and disk as template-db — they protect + # against a bad migration, not against losing the machine. + template-backup: + image: postgres:17 + entrypoint: ["/bin/sh", "-euc"] + command: + - | + MODE="$${BACKUP_MODE:-required}" + case "$$MODE" in + off) echo "[backup] BACKUP_MODE=off; skipping the pre-migration backup."; exit 0 ;; + best-effort|required) ;; + *) echo "[backup] ERROR: BACKUP_MODE must be off, best-effort or required (got '$$MODE')."; exit 1 ;; + esac + + stamp=$$(date -u +%Y%m%dT%H%M%SZ) + final="/backups/pre-migrate-$$stamp.dump" + tmp="/backups/.pre-migrate-$$stamp.dump" + + # Move into place only after verifying, so a partial file never looks like a good backup. + fail() { + echo "[backup] $$1" + rm -f "$$tmp" + if [ "$$MODE" = required ]; then exit 1; fi + echo "[backup] continuing anyway because BACKUP_MODE=$$MODE." + exit 0 + } + + echo "[backup] Dumping $$PGDATABASE from $$PGHOST:$${PGPORT:-5432} to $$final ..." + pg_dump -Fc -f "$$tmp" || fail "pg_dump failed" + pg_restore --list "$$tmp" >/dev/null 2>&1 || fail "dump failed pg_restore --list verification" + mv "$$tmp" "$$final" + echo "[backup] Backup complete: $$final ($$(du -h "$$final" | cut -f1))" + + ls -1t /backups/pre-migrate-*.dump 2>/dev/null | tail -n "+$$(( $${BACKUP_KEEP:-10} + 1 ))" | while read -r old; do + echo "[backup] Pruning old backup $$(basename "$$old")" + rm -f "$$old" + done + environment: + - PGHOST=template-db + - PGUSER=${DATABASE_USER} + - PGPASSWORD=${DATABASE_PW} + - PGDATABASE=${DATABASE_NAME} + - BACKUP_MODE=${BACKUP_MODE:-required} + - BACKUP_KEEP=${BACKUP_KEEP:-10} + volumes: + - template-backups:/backups + restart: "no" + depends_on: + template-db: + condition: service_healthy + profiles: + - production + # Database migrations (production only). Same image as the app, different # command — the image ships the Prisma CLI for exactly this. template-migrations: @@ -30,6 +87,8 @@ services: depends_on: template-db: condition: service_healthy + template-backup: + condition: service_completed_successfully profiles: - production @@ -69,4 +128,6 @@ services: - production volumes: - template-db-data: \ No newline at end of file + template-db-data: + # Pre-migration dumps; must outlive the one-shot backup container. + template-backups: \ No newline at end of file diff --git a/example.env b/example.env index 531f0b6..ba3fa02 100644 --- a/example.env +++ b/example.env @@ -19,4 +19,8 @@ AUTH_GOOGLE_SECRET= RESEND_API_KEY= # Replace with keys generate from https://knock.app/tools/vapid-key-generator NEXT_PUBLIC_VAPID_PUBLIC_KEY= -VAPID_PRIVATE_KEY= \ No newline at end of file +VAPID_PRIVATE_KEY= +# Pre-migration pg_dump, taken before template-migrations runs. +# off = never dump, best-effort = dump but migrate anyway on failure, required = block the migration. +BACKUP_MODE=required +BACKUP_KEEP=10 \ No newline at end of file