From 870a71f6206149f8485e46f65c071da9aa3fb746 Mon Sep 17 00:00:00 2001 From: Tamal Saha Date: Tue, 4 Aug 2026 20:57:01 +0600 Subject: [PATCH 1/6] postgres: add migration guide for self-managed sources via remote replica Migrating a PostgreSQL that KubeDB does not manage -- a bare StatefulSet, a VM in another datacenter, a hardened install whose superuser is never shared -- into a KubeDB cluster: seed with pg_basebackup, stream until lag is zero, stop writes verifiably, cut over. Every step was executed against two live source flavours before being written down: a stock postgres:17.4 with shared superuser credentials, and a hardened source with scram auth, a replication-only user, a restricted pg_hba.conf and no "postgres" role at all (initdb run as a different superuser). Measured write gap for the cutover was 43s (bare, from server-side row timestamps) and 37s (hardened, cutover-initiated to first accepted write). The guide leads with the limitation table: self-managed sources only, matching major version, remote-config inapplicable (hand-crafted AppBinding shown), port 5432 fixed, the postgres-role-must-exist rule with its one-line fix that replicates through WAL, libc/collation mismatch with the datcollversion remedy, extensions, tablespaces, and slotless WAL retention. Notable mechanics it documents from the live runs: reading the address to allowlist out of the source's own pg_hba rejection log (the seed retries forever, so the fix needs no restart); running the warm phase with the migration user's credentials as the authSecret so health checks pass without the source's superuser password; and the authSecret rule at cutover -- keep it if its username is postgres, remove it otherwise and let the operator generate a fresh superuser secret that promotion re-keys the copied catalog to. Signed-off-by: Tamal Saha --- .../migration-yamls/pg-mig-standalone.yaml | 19 ++ .../migration-yamls/pg-mig.yaml | 25 ++ .../migration-yamls/source-appbinding.yaml | 33 ++ .../postgres/remote-replica/migration.md | 293 ++++++++++++++++++ 4 files changed, 370 insertions(+) create mode 100644 docs/guides/postgres/remote-replica/migration-yamls/pg-mig-standalone.yaml create mode 100644 docs/guides/postgres/remote-replica/migration-yamls/pg-mig.yaml create mode 100644 docs/guides/postgres/remote-replica/migration-yamls/source-appbinding.yaml create mode 100644 docs/guides/postgres/remote-replica/migration.md diff --git a/docs/guides/postgres/remote-replica/migration-yamls/pg-mig-standalone.yaml b/docs/guides/postgres/remote-replica/migration-yamls/pg-mig-standalone.yaml new file mode 100644 index 000000000..23c3b7e4d --- /dev/null +++ b/docs/guides/postgres/remote-replica/migration-yamls/pg-mig-standalone.yaml @@ -0,0 +1,19 @@ +apiVersion: kubedb.com/v1 +kind: Postgres +metadata: + name: pg-mig + namespace: demo +spec: + healthChecker: + disableWriteCheck: true + clientAuthMode: md5 + standbyMode: Hot + replicas: 1 + storage: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 10Gi # must hold the full source database plus WAL + storageType: Durable + deletionPolicy: Halt + version: "17.4" # must match the source's MAJOR version diff --git a/docs/guides/postgres/remote-replica/migration-yamls/pg-mig.yaml b/docs/guides/postgres/remote-replica/migration-yamls/pg-mig.yaml new file mode 100644 index 000000000..936a67bfc --- /dev/null +++ b/docs/guides/postgres/remote-replica/migration-yamls/pg-mig.yaml @@ -0,0 +1,25 @@ +apiVersion: kubedb.com/v1 +kind: Postgres +metadata: + name: pg-mig + namespace: demo +spec: + remoteReplica: + sourceRef: + name: source-pg # the AppBinding above, in THIS cluster + namespace: demo + authSecret: + name: source-pg-auth # during the warm phase: the migration user's credentials + healthChecker: + disableWriteCheck: true + clientAuthMode: md5 + standbyMode: Hot + replicas: 1 + storage: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 10Gi # must hold the full source database plus WAL + storageType: Durable + deletionPolicy: Halt + version: "17.4" # must match the source's MAJOR version diff --git a/docs/guides/postgres/remote-replica/migration-yamls/source-appbinding.yaml b/docs/guides/postgres/remote-replica/migration-yamls/source-appbinding.yaml new file mode 100644 index 000000000..ff8d8690b --- /dev/null +++ b/docs/guides/postgres/remote-replica/migration-yamls/source-appbinding.yaml @@ -0,0 +1,33 @@ +# Hand-crafted AppBinding for a source PostgreSQL that is NOT managed by KubeDB. +# Edit: service.name (source address), secret credentials, and sslmode. +apiVersion: v1 +kind: Secret +metadata: + name: source-pg-auth + namespace: demo +stringData: + username: migrator # the replication user the source DBA created for you + password: "" +type: kubernetes.io/basic-auth +--- +apiVersion: appcatalog.appscode.com/v1alpha1 +kind: AppBinding +metadata: + name: source-pg + namespace: demo + labels: + app.kubernetes.io/name: postgreses.kubedb.com +spec: + clientConfig: + service: + name: 10.2.0.30 # source address reachable from THIS cluster; port must be 5432 + path: / + port: 5432 + query: sslmode=disable # or verify-ca with a tlsSecret carrying the SOURCE's CA + scheme: postgresql + secret: + apiGroup: "" + kind: Secret + name: source-pg-auth + type: kubedb.com/postgres + version: "17.4" diff --git a/docs/guides/postgres/remote-replica/migration.md b/docs/guides/postgres/remote-replica/migration.md new file mode 100644 index 000000000..756c0938d --- /dev/null +++ b/docs/guides/postgres/remote-replica/migration.md @@ -0,0 +1,293 @@ +--- +title: Migrate a Self-Managed PostgreSQL into KubeDB +menu: + docs_{{ .version }}: + identifier: pg-remote-replica-migration + name: Migration from Self-Managed + parent: pg-remote-replica + weight: 40 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Migrate a Self-Managed PostgreSQL into KubeDB + +This guide migrates a **self-managed** PostgreSQL — one KubeDB does not manage: a bare +StatefulSet, a VM in another datacenter, a hardened installation whose superuser you will +never see — into a KubeDB-managed cluster with seconds of write downtime, using the remote +replica mechanism: seed with `pg_basebackup`, stream until the lag is zero, stop writes, +cut over. + +Everything below was executed end to end against two source flavours: + +- **bare**: stock `postgres:17.4`, superuser credentials shared with us +- **hardened**: `scram-sha-256`, a replication-only user, a restricted `pg_hba.conf`, and + **no `postgres` role at all** (`initdb` run as a different superuser) + +> Note: YAML files used in this tutorial are stored in [docs/guides/postgres/remote-replica/migration-yamls](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/guides/postgres/remote-replica/migration-yamls) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Limitations — read first + +| Limitation | Why | Way out | +|---|---|---| +| **Self-managed sources only** | Managed services (RDS, Cloud SQL, …) do not expose physical replication to external standbys | Logical replication (different machinery, not this guide) | +| **Major version must match** | Physical standby; WAL format is per-major | Pick the matching `version:` from the KubeDB catalog; upgrade after migration | +| **`kubectl-dba remote-config` cannot be used** | It lists the source's pods by label and `kubectl exec`s into them — a foreign source has no pods | Hand-craft the AppBinding + secret (Step 2 of this guide) | +| **The source must be reachable at port 5432** | The generated connection strings and `primary_conninfo` do not carry a port | Expose the source at `:5432` on the address you put in the AppBinding | +| **A `LOGIN`-able `postgres` role must exist in the source catalog before cutover** | Promotion connects locally *as* `postgres` to re-key passwords | One-liner on the source, replicates automatically (Step 5) | +| **libc / collation mismatch** | A glibc-built source seeded onto a musl-based image records a collation version the new libc cannot confirm; text indexes are ordered by the old libc | Post-cutover: clear `datcollversion`, `REINDEX` collation-sensitive indexes (Step 7) | +| **Extensions** | WAL replays fine, but queries touching extension objects need the `.so` present in the KubeDB image | Inventory `pg_extension` on the source first; anything not in the image blocks this method | +| **Tablespaces with absolute paths** | Paths from the source host do not exist in the container | Consolidate to the default tablespace first | +| **No replication slot for steady-state streaming** | The replica streams slotless; if it disconnects longer than the source retains WAL, it cannot resume | Set `wal_keep_size` on the source to cover your longest tolerable outage. The initial seed itself needs **no** WAL retention config: `pg_basebackup -Xs` backs it with a temporary slot it creates and drops itself | + +## Step 1: prepare the source (the source DBA does this) + +A dedicated replication user — superuser **not** required: + +```sql +CREATE ROLE migrator LOGIN REPLICATION PASSWORD ''; +``` + +Two `pg_hba.conf` lines — one for the WAL stream, one because KubeDB's monitor queries +`pg_stat_replication` over a normal connection to the `postgres` database: + +``` +host replication migrator scram-sha-256 +host postgres migrator scram-sha-256 +``` + +then `SELECT pg_reload_conf();`. + +**Don't guess ``** — NAT between the clusters decides what the source +sees. Deploy the replica first (Step 3) and read the address out of the source's own log: + +``` +FATAL: no pg_hba.conf entry for replication connection from host "10.42.0.17", user "migrator", no encryption +``` + +That `host` value is the address to allowlist. The replica retries the seed forever, so +fixing `pg_hba.conf` after the fact needs no restart of anything — the seed simply proceeds +on the next retry. + +Requirements that are defaults on modern PostgreSQL: `wal_level = replica`, +`max_wal_senders` ≥ 2 free (the `-Xs` seed briefly uses two), `listen_addresses` covering +the ingress path. + +## Step 2: hand-craft the AppBinding + +On the **destination** cluster (`remote-config` cannot generate this for a foreign source): + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: source-pg-auth + namespace: demo +stringData: + username: migrator + password: "" +type: kubernetes.io/basic-auth +--- +apiVersion: appcatalog.appscode.com/v1alpha1 +kind: AppBinding +metadata: + name: source-pg + namespace: demo + labels: + app.kubernetes.io/name: postgreses.kubedb.com +spec: + clientConfig: + service: + name: 10.2.0.30 # source address; must serve PostgreSQL on 5432 + path: / + port: 5432 + query: sslmode=disable # TLS source: verify-ca + a tlsSecret carrying the SOURCE's CA + scheme: postgresql + secret: + apiGroup: "" + kind: Secret + name: source-pg-auth + type: kubedb.com/postgres + version: "17.4" +``` + +```bash +kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/remote-replica/migration-yamls/source-appbinding.yaml +``` + +## Step 3: deploy the warm replica + +```yaml +apiVersion: kubedb.com/v1 +kind: Postgres +metadata: + name: pg-mig + namespace: demo +spec: + remoteReplica: + sourceRef: + name: source-pg + namespace: demo + authSecret: + name: source-pg-auth # yes — the migration user's credentials; see below + healthChecker: + disableWriteCheck: true + clientAuthMode: md5 + standbyMode: Hot + replicas: 1 + storage: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 10Gi + storageType: Durable + deletionPolicy: Halt + version: "17.4" +``` + +```bash +kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/remote-replica/migration-yamls/pg-mig.yaml +kubectl wait pg pg-mig -n demo --for=jsonpath='{.status.phase}'=Ready --timeout=900s +``` + +**Why `authSecret` points at the migration user:** `pg_basebackup` copies the source's +`pg_authid` wholesale, so after the seed the replica's passwords are the *source's* +passwords. KubeDB's health checker authenticates with the authSecret — against a hardened +source whose superuser password you don't have, the only credentials guaranteed to work in +the copied catalog are the migration user's. With them, the CR reports `Ready` throughout +the warm phase. (Against a bare source that shared its `postgres` password, an authSecret +with `username: postgres` works the same way.) + +The seed streams WAL concurrently with the copy (`pg_basebackup -Xs`, temporary slot, +self-cleaning), so the source needs no WAL-retention configuration for it. + +## Step 4: watch the lag + +The coordinator sidecar logs the replica's byte lag behind the source. Sampling starts at +5s; every consecutive zero-lag reading doubles the interval up to 300s, and any non-zero +reading resets it to 5s — so a catching-up replica (the phase you actually watch) is +sampled every 5 seconds: + +```bash +kubectl logs -f pg-mig-0 -n demo -c pg-coordinator | grep LagMonitor +``` + +``` +[LagMonitor] Pod pg-mig-0: lag=0 B (in sync with source); next check in 10s +[LagMonitor] Pod pg-mig-0: lag=0 B (in sync with source); next check in 20s +[LagMonitor] Pod pg-mig-0: lag=48681472 B behind source; next check in 5s +[LagMonitor] Pod pg-mig-0: lag=0 B (in sync with source); next check in 10s +``` + +## Step 5: pre-cutover checks (while streaming, zero risk) + +The replica is readable, so every check runs against live data. + +**The `postgres` role.** Promotion runs `ALTER USER postgres … PASSWORD` connecting locally +*as* `postgres`. If the source's `initdb` used another superuser name, that role does not +exist and promotion cannot complete: + +```bash +kubectl exec -n demo pg-mig-0 -c postgres -- \ + psql -U migrator -d postgres -tAc "SELECT count(*) FROM pg_roles WHERE rolname='postgres';" +``` + +If `0`, have the source DBA run — it replicates within seconds, re-check to confirm: + +```sql +CREATE ROLE postgres LOGIN SUPERUSER; +``` + +**Collation-sensitive indexes** (matters whenever source and replica differ in libc — a +glibc VM or Debian-based image onto KubeDB's musl-based image always does): + +```bash +kubectl exec -n demo pg-mig-0 -c postgres -- psql -U migrator -d postgres -tAc \ + "SELECT count(*) FROM pg_index i WHERE EXISTS (SELECT 1 FROM unnest(i.indcollation) col + WHERE col <> 0 AND col NOT IN (SELECT oid FROM pg_collation WHERE collname IN ('C','POSIX')));" +``` + +Remember the number — it is your post-cutover `REINDEX` workload (Step 7). + +**Extensions:** `SELECT extname FROM pg_extension;` on the source; anything beyond what the +KubeDB image ships must be resolved before you rely on this method. + +## Step 6: cutover + +**1. Stop application writes at the source — and verify they stopped.** Do not trust +"the app was told to disconnect"; killing a client does not necessarily kill server-side +sessions. The source's WAL position is the truth — it must be frozen: + +```bash +# run twice, 2s apart, on the source; proceed only when identical +SELECT pg_current_wal_lsn(); +``` + +**2. Wait for the replica to apply everything** (compare against the frozen LSN from step 1; +`>=`, not equality — the source still emits checkpoint WAL after quiescing): + +```bash +kubectl exec -n demo pg-mig-0 -c postgres -- psql -U migrator -d postgres -tAc \ + "SELECT pg_wal_lsn_diff(pg_last_wal_replay_lsn(), '') >= 0;" # wait for: t +``` + +**3. Promote.** Remove `spec.remoteReplica`; for `spec.authSecret`, one rule: + +- authSecret's `username` **is** `postgres` → keep it. Its password becomes the superuser + password at promotion. +- authSecret's `username` is **not** `postgres` (the hardened case) → **remove it too**. + The operator generates `-auth` with user `postgres` and a fresh password, and + promotion re-keys the copied catalog to it. You never needed the source's superuser + password at any point. + +```bash +kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/postgres/remote-replica/migration-yamls/pg-mig-standalone.yaml +kubectl delete pod pg-mig-0 -n demo +``` + +**4. Redirect writes and confirm.** The database is migrated when a write is accepted with +the final credentials: + +```bash +PGPASSWORD=$(kubectl get secret pg-mig-auth -n demo -o jsonpath='{.data.password}' | base64 -d) +kubectl exec -n demo pg-mig-0 -c postgres -- env PGPASSWORD="$PGPASSWORD" \ + psql -h 127.0.0.1 -U postgres -d postgres -c "SELECT NOT pg_is_in_recovery();" +``` + +**Measured on the runs behind this guide** (single-replica, ~1–2 GB databases, same-LAN +clusters): write gap from last replicated write to first accepted write **43 s** (bare, +measured from server-side row timestamps); cutover-initiated to first accepted write with +the operator-generated password **37 s** (hardened). The time is dominated by the pod +recreate and promotion, not by data size — the data was already there. + +## Step 7: after the cutover + +**Collation record** — on a libc-mismatched migration every connection warns +`database "postgres" has no actual collation version, but a version was recorded`, and +`ALTER DATABASE … REFRESH COLLATION VERSION` fails with `invalid collation version change` +because the new libc reports no version to refresh *to*. Clear the recorded version, and +rebuild whatever Step 5's index count found: + +```sql +UPDATE pg_database SET datcollversion = NULL WHERE datcollversion IS NOT NULL; +-- for each collation-sensitive index found in Step 5: +REINDEX INDEX ; -- or REINDEX DATABASE if the count was large +``` + +**Hygiene** — the migration user came along in the copied catalog: + +```sql +DROP OWNED BY migrator; DROP ROLE migrator; +``` + +The source's other roles and databases are all present — that is the migration payload. +From here the database is a normal KubeDB Postgres: scale it, enable TLS, attach a remote +replica of its own for DR. + +## Next Steps + +- [Remote Replica overview](/docs/guides/postgres/remote-replica/remotereplica.md) +- [Cross-Cluster DR with Bidirectional Failover](/docs/guides/postgres/remote-replica/advanced-setup.md) +- [Zero-Data-Loss Cross-Cluster Replication](/docs/guides/postgres/remote-replica/synchronous.md) From 645dadfdde1527309e53de482d1a20fe60a11272 Mon Sep 17 00:00:00 2001 From: Tamal Saha Date: Wed, 5 Aug 2026 00:23:22 +0600 Subject: [PATCH 2/6] postgres/migration: make the cutover a verified zero-loss runbook The previous Step 6 measured downtime but never proved the migration lossless: its numbers came from runs whose writers either kept writing after "stopping" (a killed kubectl exec does not kill the server-side loop) or never wrote at all. A migration that lost in-flight rows is not a successful migration. Step 6 is now gated: - stop application writes and VERIFY the stop: pg_current_wal_lsn() AND row counts frozen across two samples - freeze a source content fingerprint: order-independent, constant-memory (sum of hashtextextended over each table), so it works at any table size - after the replica reports the frozen LSN applied, the same fingerprint on the replica must be IDENTICAL, or the cutover is aborted with the replica left safely streaming - after promotion, the fingerprint scoped to pre-cutover data must reproduce the frozen value: zero rows lost, proven, not assumed Replica-side checks run over the local socket as the source's own application user -- the replication user cannot read the application's tables and the final postgres password does not exist until promotion. Executed twice against a hardened source (no postgres role, replication-only user): once while writing the runbook, once again afterwards following the published steps verbatim. Both runs zero loss, write gaps 52.22 s and 48.57 s from server-side row timestamps. Signed-off-by: Tamal Saha --- .../postgres/remote-replica/migration.md | 87 +++++++++++++++---- 1 file changed, 72 insertions(+), 15 deletions(-) diff --git a/docs/guides/postgres/remote-replica/migration.md b/docs/guides/postgres/remote-replica/migration.md index 756c0938d..3d548bba1 100644 --- a/docs/guides/postgres/remote-replica/migration.md +++ b/docs/guides/postgres/remote-replica/migration.md @@ -214,26 +214,69 @@ Remember the number — it is your post-cutover `REINDEX` workload (Step 7). **Extensions:** `SELECT extname FROM pg_extension;` on the source; anything beyond what the KubeDB image ships must be resolved before you rely on this method. -## Step 6: cutover +## Step 6: lossless cutover + +A migration is only successful if the migrated database contains **every row the source +ever acknowledged**. This runbook makes that a verified gate, not an assumption: cutover +is forbidden until a content fingerprint of the source and the replica are identical. + +Two practical notes before the steps: + +- Run the replica-side verification queries **over the local unix socket as the source's + own application user** (`admin` in this guide). That role exists in the copied catalog + and can read its own tables; the replica's `local ... trust` pg_hba line means no + password is needed. The replication user typically cannot `SELECT` from the + application's tables, and the final `postgres` password does not exist yet. +- On a libc-mismatched migration every `psql` invocation prints + `WARNING: database "postgres" has no actual collation version ...` on stderr. It is + harmless here and fixed in Step 7 — but don't let it confuse scripts that merge stderr + into stdout. **1. Stop application writes at the source — and verify they stopped.** Do not trust -"the app was told to disconnect"; killing a client does not necessarily kill server-side -sessions. The source's WAL position is the truth — it must be frozen: +"the app was told to disconnect": killing a client does not kill server-side sessions +(in our testing, a "killed" writer kept inserting for ten more minutes). The source's +WAL position and row counts are the truth — both must be frozen across two samples: ```bash -# run twice, 2s apart, on the source; proceed only when identical +# on the source, twice, 2 s apart; proceed only when BOTH are identical SELECT pg_current_wal_lsn(); +SELECT count(*) FROM writes; -- your busiest table(s) ``` -**2. Wait for the replica to apply everything** (compare against the frozen LSN from step 1; -`>=`, not equality — the source still emits checkpoint WAL after quiescing): +**2. Freeze the source fingerprint.** Order-independent, content-sensitive, and constant +memory, so it works on tables of any size (extend the pattern to every table you care +about): + +```sql +SELECT (SELECT count(*) FROM payload) + ||'|'|| (SELECT coalesce(sum(hashtextextended(id::text||data, 0)), 0) FROM payload) + ||'|'|| (SELECT count(*) FROM writes) + ||'|'|| (SELECT coalesce(sum(hashtextextended(id::text||origin||ts::text, 0)), 0) FROM writes); +``` + +Record the result — this is the value the migrated database must reproduce. + +**3. Wait for the replica to apply everything** (compare against the frozen LSN from +step 1; `>=`, not equality — the source still emits checkpoint WAL after quiescing): ```bash -kubectl exec -n demo pg-mig-0 -c postgres -- psql -U migrator -d postgres -tAc \ +kubectl exec -n demo pg-mig-0 -c postgres -- psql -U admin -d postgres -tAc \ "SELECT pg_wal_lsn_diff(pg_last_wal_replay_lsn(), '') >= 0;" # wait for: t ``` -**3. Promote.** Remove `spec.remoteReplica`; for `spec.authSecret`, one rule: +**4. THE ZERO-LOSS GATE.** Run the *same* fingerprint query on the replica, over the +socket as the application user: + +```bash +kubectl exec -n demo pg-mig-0 -c postgres -- psql -U admin -d postgres -tAc "" +``` + +- **Identical** → every acknowledged row is on the replica; proceed. +- **Different** → **do not cut over.** Nothing is lost — the replica is still streaming + and the source is intact. Find what is still moving (a second application? a cron?) + and return to step 1. + +**5. Promote.** Remove `spec.remoteReplica`; for `spec.authSecret`, one rule: - authSecret's `username` **is** `postgres` → keep it. Its password becomes the superuser password at promotion. @@ -247,19 +290,33 @@ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}} kubectl delete pod pg-mig-0 -n demo ``` -**4. Redirect writes and confirm.** The database is migrated when a write is accepted with -the final credentials: +**6. Redirect writes, then prove zero loss.** The database is migrated when a write is +accepted with the final credentials: ```bash PGPASSWORD=$(kubectl get secret pg-mig-auth -n demo -o jsonpath='{.data.password}' | base64 -d) kubectl exec -n demo pg-mig-0 -c postgres -- env PGPASSWORD="$PGPASSWORD" \ - psql -h 127.0.0.1 -U postgres -d postgres -c "SELECT NOT pg_is_in_recovery();" + psql -h 127.0.0.1 -U postgres -d postgres -c \ + "INSERT INTO writes(origin) VALUES ('first-write-after-migration') RETURNING id;" +``` + +Then re-run the fingerprint query on the migrated database, scoped to exclude +post-cutover writes — it must equal the value frozen in step 2. If your write traffic +carries server-side timestamps, the migration's write gap is computable from the data +itself, immune to clock skew between clusters: + +```sql +SELECT min(ts) FILTER (WHERE origin = 'first-write-after-migration') + - max(ts) FILTER (WHERE origin <> 'first-write-after-migration') FROM writes; ``` -**Measured on the runs behind this guide** (single-replica, ~1–2 GB databases, same-LAN -clusters): write gap from last replicated write to first accepted write **43 s** (bare, -measured from server-side row timestamps); cutover-initiated to first accepted write with -the operator-generated password **37 s** (hardened). The time is dominated by the pod +**Measured on the runs behind this guide** (hardened source, single replica, 1.2 GB, +same-LAN clusters, a writer inserting ~5 rows/s until the verified stop). This runbook +was executed twice — once while being written, and once again afterwards following the +published steps verbatim. Both runs: fingerprints identical before and after cutover — +**zero rows lost** — with a write gap from the last acknowledged source write to the +first write accepted by the migrated database of **52.22 s** and **48.57 s** +respectively, measured from server-side row timestamps. The gap is dominated by the pod recreate and promotion, not by data size — the data was already there. ## Step 7: after the cutover From 5d3c14d1534d19383fcb78cef413e889197cc4c7 Mon Sep 17 00:00:00 2001 From: Tamal Saha Date: Wed, 5 Aug 2026 10:07:55 +0600 Subject: [PATCH 3/6] migration: source port is now honored via the AppBinding The port-5432 limitation is lifted: the operator injects PRIMARY_PORT from spec.clientConfig.service.port into the remote replica containers, the init scripts use it for pg_isready/psql/pg_basebackup/primary_conninfo, and the coordinator uses it for its source client, pg_rewind source-server string and recovery basebackup. Defaults to 5432 everywhere when unset. Verified live: a replica seeded and streaming from a source exposed only on port 5433, with port=5433 visible in primary_conninfo and the lag monitor sampling over the same port. Signed-off-by: Tamal Saha --- .../remote-replica/migration-yamls/source-appbinding.yaml | 4 ++-- docs/guides/postgres/remote-replica/migration.md | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/guides/postgres/remote-replica/migration-yamls/source-appbinding.yaml b/docs/guides/postgres/remote-replica/migration-yamls/source-appbinding.yaml index ff8d8690b..3f726f1ec 100644 --- a/docs/guides/postgres/remote-replica/migration-yamls/source-appbinding.yaml +++ b/docs/guides/postgres/remote-replica/migration-yamls/source-appbinding.yaml @@ -20,9 +20,9 @@ metadata: spec: clientConfig: service: - name: 10.2.0.30 # source address reachable from THIS cluster; port must be 5432 + name: 10.2.0.30 # source address reachable from THIS cluster path: / - port: 5432 + port: 5432 # honored: set to the source's actual port query: sslmode=disable # or verify-ca with a tlsSecret carrying the SOURCE's CA scheme: postgresql secret: diff --git a/docs/guides/postgres/remote-replica/migration.md b/docs/guides/postgres/remote-replica/migration.md index 3d548bba1..0a72d8dea 100644 --- a/docs/guides/postgres/remote-replica/migration.md +++ b/docs/guides/postgres/remote-replica/migration.md @@ -35,7 +35,7 @@ Everything below was executed end to end against two source flavours: | **Self-managed sources only** | Managed services (RDS, Cloud SQL, …) do not expose physical replication to external standbys | Logical replication (different machinery, not this guide) | | **Major version must match** | Physical standby; WAL format is per-major | Pick the matching `version:` from the KubeDB catalog; upgrade after migration | | **`kubectl-dba remote-config` cannot be used** | It lists the source's pods by label and `kubectl exec`s into them — a foreign source has no pods | Hand-craft the AppBinding + secret (Step 2 of this guide) | -| **The source must be reachable at port 5432** | The generated connection strings and `primary_conninfo` do not carry a port | Expose the source at `:5432` on the address you put in the AppBinding | +| **Non-standard source ports are honored via the AppBinding** | Every source-facing connection (seed, streaming, monitor, recovery) uses `spec.clientConfig.service.port` | Set `port:` in the AppBinding (Step 2); it defaults to 5432 when unset | | **A `LOGIN`-able `postgres` role must exist in the source catalog before cutover** | Promotion connects locally *as* `postgres` to re-key passwords | One-liner on the source, replicates automatically (Step 5) | | **libc / collation mismatch** | A glibc-built source seeded onto a musl-based image records a collation version the new libc cannot confirm; text indexes are ordered by the old libc | Post-cutover: clear `datcollversion`, `REINDEX` collation-sensitive indexes (Step 7) | | **Extensions** | WAL replays fine, but queries touching extension objects need the `.so` present in the KubeDB image | Inventory `pg_extension` on the source first; anything not in the image blocks this method | @@ -100,9 +100,9 @@ metadata: spec: clientConfig: service: - name: 10.2.0.30 # source address; must serve PostgreSQL on 5432 + name: 10.2.0.30 # source address path: / - port: 5432 + port: 5432 # honored: set to the source's actual port query: sslmode=disable # TLS source: verify-ca + a tlsSecret carrying the SOURCE's CA scheme: postgresql secret: From cb31d01dbab8ea72a47d1f997f628d82e31b585d Mon Sep 17 00:00:00 2001 From: Tamal Saha Date: Wed, 5 Aug 2026 15:55:44 +0600 Subject: [PATCH 4/6] migration: require matching OS distribution; add zero-loss verification The collation-remedy content (datcollversion clearing, REINDEX of collation-sensitive indexes) is removed and replaced by a hard requirement: the source's OS/libc must match the chosen PostgresVersion image's, verified by comparing SELECT version() on both sides -- a mismatch is not supported. Re-ran the full migration against an Alpine source matching the Official Alpine image: with matching libc no collation warning appears at all, which is also documented as the tell-tale for a mismatch. Adds a "Verifying the cutover" section with the exact procedure and real outputs from a live run with a ~5 TPS writer: sentinel-file writer stop (killing a client does not stop server-side sessions), quiesce verified by a frozen WAL position across two samples, catch-up checked with >= against the frozen LSN, zero transaction loss proven by identical count/max-id/ordered-id fingerprint on both sides (2904|2904|9b409e93...), and downtime measured from server-side row timestamps: 37.63 s. Notes the normal sequence jump across promotion so it is not mistaken for loss. Also drops the explicit healthChecker.disableWriteCheck from the replica manifest -- the health checker already handles remote replicas on its own. Signed-off-by: Tamal Saha --- .../migration-yamls/pg-mig.yaml | 2 - .../postgres/remote-replica/migration.md | 120 ++++++++++++++---- 2 files changed, 93 insertions(+), 29 deletions(-) diff --git a/docs/guides/postgres/remote-replica/migration-yamls/pg-mig.yaml b/docs/guides/postgres/remote-replica/migration-yamls/pg-mig.yaml index 936a67bfc..812e7d805 100644 --- a/docs/guides/postgres/remote-replica/migration-yamls/pg-mig.yaml +++ b/docs/guides/postgres/remote-replica/migration-yamls/pg-mig.yaml @@ -10,8 +10,6 @@ spec: namespace: demo authSecret: name: source-pg-auth # during the warm phase: the migration user's credentials - healthChecker: - disableWriteCheck: true clientAuthMode: md5 standbyMode: Hot replicas: 1 diff --git a/docs/guides/postgres/remote-replica/migration.md b/docs/guides/postgres/remote-replica/migration.md index 0a72d8dea..9f1639707 100644 --- a/docs/guides/postgres/remote-replica/migration.md +++ b/docs/guides/postgres/remote-replica/migration.md @@ -37,7 +37,7 @@ Everything below was executed end to end against two source flavours: | **`kubectl-dba remote-config` cannot be used** | It lists the source's pods by label and `kubectl exec`s into them — a foreign source has no pods | Hand-craft the AppBinding + secret (Step 2 of this guide) | | **Non-standard source ports are honored via the AppBinding** | Every source-facing connection (seed, streaming, monitor, recovery) uses `spec.clientConfig.service.port` | Set `port:` in the AppBinding (Step 2); it defaults to 5432 when unset | | **A `LOGIN`-able `postgres` role must exist in the source catalog before cutover** | Promotion connects locally *as* `postgres` to re-key passwords | One-liner on the source, replicates automatically (Step 5) | -| **libc / collation mismatch** | A glibc-built source seeded onto a musl-based image records a collation version the new libc cannot confirm; text indexes are ordered by the old libc | Post-cutover: clear `datcollversion`, `REINDEX` collation-sensitive indexes (Step 7) | +| **The source's OS/libc must match the KubeDB image's** | Physical replication carries text indexes ordered by the source's libc collations; running them under a different libc (glibc↔musl) risks silent index corruption. **A distribution mismatch is not supported.** | Pick a `PostgresVersion` whose `spec.db.image` uses the same base as the source (the `Official` 17.4 image is Alpine/musl). Verify with the check in Step 5 | | **Extensions** | WAL replays fine, but queries touching extension objects need the `.so` present in the KubeDB image | Inventory `pg_extension` on the source first; anything not in the image blocks this method | | **Tablespaces with absolute paths** | Paths from the source host do not exist in the container | Consolidate to the default tablespace first | | **No replication slot for steady-state streaming** | The replica streams slotless; if it disconnects longer than the source retains WAL, it cannot resume | Set `wal_keep_size` on the source to cover your longest tolerable outage. The initial seed itself needs **no** WAL retention config: `pg_basebackup -Xs` backs it with a temporary slot it creates and drops itself | @@ -132,8 +132,6 @@ spec: namespace: demo authSecret: name: source-pg-auth # yes — the migration user's credentials; see below - healthChecker: - disableWriteCheck: true clientAuthMode: md5 standbyMode: Hot replicas: 1 @@ -200,16 +198,24 @@ If `0`, have the source DBA run — it replicates within seconds, re-check to co CREATE ROLE postgres LOGIN SUPERUSER; ``` -**Collation-sensitive indexes** (matters whenever source and replica differ in libc — a -glibc VM or Debian-based image onto KubeDB's musl-based image always does): +**Distribution / libc match** — the source and the KubeDB image must be built against the +same libc. Compare `version()` on both sides; the platform triple must match (here: +`x86_64-pc-linux-musl` on both). If they differ, stop and pick a matching `PostgresVersion` +— a mismatch is not supported: ```bash -kubectl exec -n demo pg-mig-0 -c postgres -- psql -U migrator -d postgres -tAc \ - "SELECT count(*) FROM pg_index i WHERE EXISTS (SELECT 1 FROM unnest(i.indcollation) col - WHERE col <> 0 AND col NOT IN (SELECT oid FROM pg_collation WHERE collname IN ('C','POSIX')));" +kubectl exec -n -- psql -U -d postgres -tAc "SELECT version();" +kubectl exec -n demo pg-mig-0 -c postgres -- psql -U migrator -d postgres -tAc "SELECT version();" ``` -Remember the number — it is your post-cutover `REINDEX` workload (Step 7). +``` +PostgreSQL 17.4 on x86_64-pc-linux-musl, compiled by gcc (Alpine 14.2.0) 14.2.0, 64-bit +PostgreSQL 17.4 on x86_64-pc-linux-musl, compiled by gcc (Alpine 14.2.0) 14.2.0, 64-bit +``` + +With matching distributions the replica connects without any collation-version warning — +if you see `database "postgres" has no actual collation version, but a version was +recorded` on every connection, the libc differs and this migration path does not apply. **Extensions:** `SELECT extname FROM pg_extension;` on the source; anything beyond what the KubeDB image ships must be resolved before you rely on this method. @@ -310,29 +316,89 @@ SELECT min(ts) FILTER (WHERE origin = 'first-write-after-migration') - max(ts) FILTER (WHERE origin <> 'first-write-after-migration') FROM writes; ``` -**Measured on the runs behind this guide** (hardened source, single replica, 1.2 GB, -same-LAN clusters, a writer inserting ~5 rows/s until the verified stop). This runbook -was executed twice — once while being written, and once again afterwards following the -published steps verbatim. Both runs: fingerprints identical before and after cutover — -**zero rows lost** — with a write gap from the last acknowledged source write to the -first write accepted by the migrated database of **52.22 s** and **48.57 s** -respectively, measured from server-side row timestamps. The gap is dominated by the pod -recreate and promotion, not by data size — the data was already there. +**Measured on the runs behind this guide** (single-replica, ~1–2 GB databases, same-LAN +clusters, a ~5 TPS writer running until cutover): write gap from last committed source +transaction to first accepted write on the migrated database **37.63 s**, measured from +server-side row timestamps, with all 2904 in-flight-era transactions verified present by +fingerprint. The full procedure is in the verification section below. -## Step 7: after the cutover +## Verifying the cutover: zero transaction loss and measured downtime -**Collation record** — on a libc-mismatched migration every connection warns -`database "postgres" has no actual collation version, but a version was recorded`, and -`ALTER DATABASE … REFRESH COLLATION VERSION` fails with `invalid collation version change` -because the new libc reports no version to refresh *to*. Clear the recorded version, and -rebuild whatever Step 5's index count found: +Run a writer against the source during the warm phase and prove afterwards that every +committed transaction arrived and how long writes were unavailable. Everything below is +from a live run: a ~5 TPS writer, 2904 transactions committed before cutover. -```sql -UPDATE pg_database SET datcollversion = NULL WHERE datcollversion IS NOT NULL; --- for each collation-sensitive index found in Step 5: -REINDEX INDEX ; -- or REINDEX DATABASE if the count was large +**Writer** (on the source; stopped by creating a sentinel file — never rely on killing a +client, server-side sessions survive it): + +```bash +kubectl exec -n -- bash -c 'rm -f /tmp/stopw +cat > /tmp/writer.sh <<"EOF" +#!/bin/bash +while [ ! -f /tmp/stopw ]; do + psql -U -d postgres -qc "INSERT INTO writes(origin) VALUES ('"'"'writer'"'"');" + sleep 0.2 +done +EOF +chmod +x /tmp/writer.sh; nohup /tmp/writer.sh >/tmp/writer.log 2>&1 &' +``` + +**Stop and verify quiesce** — the source's WAL position must be identical across two +samples; only then is the snapshot below the full truth: + +```bash +kubectl exec -n -- touch /tmp/stopw +# run twice, 2s apart; proceed only when both values are identical +kubectl exec -n -- psql -U -tAc "SELECT pg_current_wal_lsn();" +``` + +**Snapshot the source truth** (count, highest id, and a fingerprint over every id): + +```bash +kubectl exec -n -- psql -U -d postgres -tAc "SELECT count(*)||'|'||max(id)||'|'||md5(string_agg(id::text,',' ORDER BY id)) FROM writes;" +``` + +``` +2904|2904|9b409e931a89ac8134e47cdcb91da247 +``` + +**Wait for the replica to apply everything** (`>=` against the frozen LSN — the source +still emits checkpoint WAL after quiescing, so never compare for equality): + +```bash +kubectl exec -n demo pg-mig-0 -c postgres -- psql -U migrator -d postgres -tAc "SELECT pg_wal_lsn_diff(pg_last_wal_replay_lsn(), '') >= 0;" # wait for: t +``` + +Then cut over as in Step 6. Once the first write is accepted, compare: + +```bash +PGPASSWORD=$(kubectl get secret pg-mig-auth -n demo -o jsonpath='{.data.password}' | base64 -d) +kubectl exec -n demo pg-mig-0 -c postgres -- env PGPASSWORD="$PGPASSWORD" psql -h 127.0.0.1 -U postgres -d postgres -tAc "SELECT count(*)||'|'||max(id)||'|'||md5(string_agg(id::text,',' ORDER BY id)) FROM writes WHERE origin='writer';" +``` + +``` +2904|2904|9b409e931a89ac8134e47cdcb91da247 <- identical to the source snapshot: zero loss ``` +**Downtime, from server-side row timestamps** (skew-free — both rows were stamped by a +database clock): + +```bash +kubectl exec -n demo pg-mig-0 -c postgres -- env PGPASSWORD="$PGPASSWORD" psql -h 127.0.0.1 -U postgres -d postgres -tAc "SELECT round(extract(epoch FROM (SELECT min(ts) FROM writes WHERE origin<>'writer') + - (SELECT max(ts) FROM writes WHERE origin='writer'))::numeric,2)||' s';" +``` + +``` +37.63 s +``` + +The measured gap is dominated by the pod recreate and promotion, not by data size. Note +that the first post-cutover `id` may jump ahead (2921 in this run, after 2904): PostgreSQL +sequences advance in cached increments across a promotion. That is normal sequence +behavior, not lost rows — the fingerprint comparison above is the loss check. + +## Step 7: after the cutover + **Hygiene** — the migration user came along in the copied catalog: ```sql From ff34f6cf7363d4cfe8caca1f1ea091cf883f3fe6 Mon Sep 17 00:00:00 2001 From: Tamal Saha Date: Wed, 5 Aug 2026 15:56:13 +0600 Subject: [PATCH 5/6] migration: align the stray stderr note with the distribution-match requirement The note still described the collation warning as harmless-and-fixed-later, which contradicted the new hard requirement one section above. It is now the mismatch tell-tale pointing back to the Step 5 check. Signed-off-by: Tamal Saha --- docs/guides/postgres/remote-replica/migration.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/guides/postgres/remote-replica/migration.md b/docs/guides/postgres/remote-replica/migration.md index 9f1639707..485897641 100644 --- a/docs/guides/postgres/remote-replica/migration.md +++ b/docs/guides/postgres/remote-replica/migration.md @@ -233,10 +233,9 @@ Two practical notes before the steps: and can read its own tables; the replica's `local ... trust` pg_hba line means no password is needed. The replication user typically cannot `SELECT` from the application's tables, and the final `postgres` password does not exist yet. -- On a libc-mismatched migration every `psql` invocation prints - `WARNING: database "postgres" has no actual collation version ...` on stderr. It is - harmless here and fixed in Step 7 — but don't let it confuse scripts that merge stderr - into stdout. +- With matching distributions (Step 5) `psql` connects cleanly. If you ever see + `WARNING: database "postgres" has no actual collation version ...` the libc differs — + stop and revisit the distribution check; this path does not support a mismatch. **1. Stop application writes at the source — and verify they stopped.** Do not trust "the app was told to disconnect": killing a client does not kill server-side sessions From 68bb4038784347a4a28944083684357eba445fcb Mon Sep 17 00:00:00 2001 From: Tamal Saha Date: Sat, 8 Aug 2026 09:55:22 +0600 Subject: [PATCH 6/6] migration: drop disableWriteCheck from the promoted standalone, frame downtime as approximate The health checker's write check is valuable on a promoted primary and the remote-replica phase no longer needs the override. Present the measured write gap as a 40-70s range across repeated runs (a clean-room re-run measured 68.45s) rather than a single number. Signed-off-by: Tamal Saha --- .../migration-yamls/pg-mig-standalone.yaml | 2 -- docs/guides/postgres/remote-replica/migration.md | 10 ++++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/guides/postgres/remote-replica/migration-yamls/pg-mig-standalone.yaml b/docs/guides/postgres/remote-replica/migration-yamls/pg-mig-standalone.yaml index 23c3b7e4d..da7fe18ad 100644 --- a/docs/guides/postgres/remote-replica/migration-yamls/pg-mig-standalone.yaml +++ b/docs/guides/postgres/remote-replica/migration-yamls/pg-mig-standalone.yaml @@ -4,8 +4,6 @@ metadata: name: pg-mig namespace: demo spec: - healthChecker: - disableWriteCheck: true clientAuthMode: md5 standbyMode: Hot replicas: 1 diff --git a/docs/guides/postgres/remote-replica/migration.md b/docs/guides/postgres/remote-replica/migration.md index 485897641..e0d4c78d8 100644 --- a/docs/guides/postgres/remote-replica/migration.md +++ b/docs/guides/postgres/remote-replica/migration.md @@ -16,7 +16,7 @@ section_menu_id: guides This guide migrates a **self-managed** PostgreSQL — one KubeDB does not manage: a bare StatefulSet, a VM in another datacenter, a hardened installation whose superuser you will -never see — into a KubeDB-managed cluster with seconds of write downtime, using the remote +never see — into a KubeDB-managed cluster with roughly a minute of write downtime, using the remote replica mechanism: seed with `pg_basebackup`, stream until the lag is zero, stop writes, cut over. @@ -317,9 +317,11 @@ SELECT min(ts) FILTER (WHERE origin = 'first-write-after-migration') **Measured on the runs behind this guide** (single-replica, ~1–2 GB databases, same-LAN clusters, a ~5 TPS writer running until cutover): write gap from last committed source -transaction to first accepted write on the migrated database **37.63 s**, measured from -server-side row timestamps, with all 2904 in-flight-era transactions verified present by -fingerprint. The full procedure is in the verification section below. +transaction to first accepted write on the migrated database **roughly 40–70 s across +repeated runs** (37.63 s in the run shown below), measured from server-side row +timestamps, with all in-flight-era transactions verified present by fingerprint. Treat +the number as an approximation — it is dominated by the pod recreate and promotion, not +by data size. The full procedure is in the verification section below. ## Verifying the cutover: zero transaction loss and measured downtime