From 7a4b4d3b02f3271a47e7d988e5aeb3e62378d3c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Wed, 5 Aug 2026 19:20:32 +0300 Subject: [PATCH 1/9] K8SPG-1114: Automate deploying logical replicas --- Makefile | 2 +- .../pgv2.percona.com_perconapgclusters.yaml | 677 +++++++ .../pgv2.percona.com_perconapgclusters.yaml | 677 +++++++ ...eam.pgv2.percona.com_postgresclusters.yaml | 12 + config/rbac/cluster/role.yaml | 21 +- config/rbac/namespace/role.yaml | 21 +- deploy/bundle.yaml | 710 ++++++- deploy/cr.yaml | 26 + deploy/crd.yaml | 689 +++++++ deploy/cw-bundle.yaml | 710 ++++++- deploy/cw-rbac.yaml | 21 +- deploy/rbac.yaml | 21 +- e2e-tests/run-minikube.csv | 1 + e2e-tests/run-pr.csv | 1 + e2e-tests/run-release.csv | 1 + .../tests/logical-replica/00-assert.yaml | 24 + .../logical-replica/00-deploy-operator.yaml | 13 + .../tests/logical-replica/01-assert.yaml | 12 + .../logical-replica/01-create-cluster.yaml | 11 + .../tests/logical-replica/02-write-data.yaml | 30 + .../03-add-logical-replica.yaml | 41 + .../tests/logical-replica/03-assert.yaml | 55 + .../04-verify-seeded-data.yaml | 31 + .../logical-replica/05-stream-new-data.yaml | 38 + .../06-slot-survives-patroni.yaml | 54 + .../07-replica-is-read-only.yaml | 43 + .../tests/logical-replica/08-assert.yaml | 16 + .../logical-replica/08-create-backup.yaml | 28 + .../09-write-after-backup.yaml | 39 + .../tests/logical-replica/10-restore.yaml | 45 + .../tests/logical-replica/11-assert.yaml | 73 + .../12-remove-logical-replica.yaml | 73 + .../tests/logical-replica/13-assert.yaml | 65 + .../13-reseed-logical-replica.yaml | 33 + .../logical-replica/14-missing-database.yaml | 102 + .../99-remove-cluster-gracefully.yaml | 22 + .../logical-replica/conf/logical-replica.yaml | 17 + .../controller/postgrescluster/controller.go | 2 + internal/logicalreplica/postgres.go | 208 ++ internal/logicalreplica/postgres_test.go | 230 +++ internal/patroni/config.go | 13 +- internal/patroni/config_test.go | 88 + internal/pgbackrest/reconcile.go | 15 +- percona/controller/pgcluster/controller.go | 51 +- .../controller/pgcluster/logicalreplica.go | 1716 +++++++++++++++++ .../pgcluster/logicalreplica_envtest_test.go | 260 +++ .../pgcluster/logicalreplica_restore.go | 221 +++ .../pgcluster/logicalreplica_restore_test.go | 634 ++++++ .../pgcluster/logicalreplica_test.go | 1528 +++++++++++++++ percona/controller/pgcluster/status.go | 1 + percona/controller/pgcluster/status_test.go | 6 + percona/naming/conditions.go | 5 + percona/naming/label.go | 4 + .../v2/logicalreplica_types_test.go | 199 ++ .../v2/perconapgcluster_types.go | 271 +++ .../v2/zz_generated.deepcopy.go | 112 ++ .../v1beta1/postgrescluster_types.go | 18 + .../v1beta1/zz_generated.deepcopy.go | 20 + 58 files changed, 10015 insertions(+), 42 deletions(-) create mode 100644 e2e-tests/tests/logical-replica/00-assert.yaml create mode 100644 e2e-tests/tests/logical-replica/00-deploy-operator.yaml create mode 100644 e2e-tests/tests/logical-replica/01-assert.yaml create mode 100644 e2e-tests/tests/logical-replica/01-create-cluster.yaml create mode 100644 e2e-tests/tests/logical-replica/02-write-data.yaml create mode 100644 e2e-tests/tests/logical-replica/03-add-logical-replica.yaml create mode 100644 e2e-tests/tests/logical-replica/03-assert.yaml create mode 100644 e2e-tests/tests/logical-replica/04-verify-seeded-data.yaml create mode 100644 e2e-tests/tests/logical-replica/05-stream-new-data.yaml create mode 100644 e2e-tests/tests/logical-replica/06-slot-survives-patroni.yaml create mode 100644 e2e-tests/tests/logical-replica/07-replica-is-read-only.yaml create mode 100644 e2e-tests/tests/logical-replica/08-assert.yaml create mode 100644 e2e-tests/tests/logical-replica/08-create-backup.yaml create mode 100644 e2e-tests/tests/logical-replica/09-write-after-backup.yaml create mode 100644 e2e-tests/tests/logical-replica/10-restore.yaml create mode 100644 e2e-tests/tests/logical-replica/11-assert.yaml create mode 100644 e2e-tests/tests/logical-replica/12-remove-logical-replica.yaml create mode 100644 e2e-tests/tests/logical-replica/13-assert.yaml create mode 100644 e2e-tests/tests/logical-replica/13-reseed-logical-replica.yaml create mode 100644 e2e-tests/tests/logical-replica/14-missing-database.yaml create mode 100644 e2e-tests/tests/logical-replica/99-remove-cluster-gracefully.yaml create mode 100644 e2e-tests/tests/logical-replica/conf/logical-replica.yaml create mode 100644 internal/logicalreplica/postgres.go create mode 100644 internal/logicalreplica/postgres_test.go create mode 100644 percona/controller/pgcluster/logicalreplica.go create mode 100644 percona/controller/pgcluster/logicalreplica_envtest_test.go create mode 100644 percona/controller/pgcluster/logicalreplica_restore.go create mode 100644 percona/controller/pgcluster/logicalreplica_restore_test.go create mode 100644 percona/controller/pgcluster/logicalreplica_test.go create mode 100644 pkg/apis/pgv2.percona.com/v2/logicalreplica_types_test.go diff --git a/Makefile b/Makefile index 231663bfef..e3bb1828b2 100644 --- a/Makefile +++ b/Makefile @@ -155,8 +155,8 @@ go-fix: ## Run go fix on all packages .PHONY: generate generate: ## Generate crd, crd-docs, deepcopy functions, and rbac - $(MAKE) go-fix $(MAKE) generate-all + $(MAKE) go-fix .PHONY: generate-all generate-all: kustomize diff --git a/build/crd/percona/generated/pgv2.percona.com_perconapgclusters.yaml b/build/crd/percona/generated/pgv2.percona.com_perconapgclusters.yaml index 9de1a704ec..37a1f1ff4f 100644 --- a/build/crd/percona/generated/pgv2.percona.com_perconapgclusters.yaml +++ b/build/crd/percona/generated/pgv2.percona.com_perconapgclusters.yaml @@ -12348,6 +12348,646 @@ spec: required: - image type: object + logicalReplicas: + items: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + bootstrapMethod: + default: pgbackrest + enum: + - pgbackrest + - pg_basebackup + type: string + dataVolumeClaimSpec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + databases: + items: + maxLength: 63 + minLength: 1 + type: string + type: array + expose: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + loadBalancerClass: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + nodePort: + format: int32 + type: integer + type: + default: ClusterIP + enum: + - ClusterIP + - NodePort + - LoadBalancer + type: string + type: object + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + name: + maxLength: 20 + pattern: ^[a-z][a-z0-9-]*[a-z0-9]$ + type: string + priorityClassName: + type: string + resources: + properties: + claims: + items: + properties: + name: + type: string + request: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + required: + - dataVolumeClaimSpec + - name + type: object + type: array metadata: properties: annotations: @@ -15431,6 +16071,15 @@ spec: and is true rule: '!has(self.users) || self.postgresVersion >= 15 || self.users.all(u, !has(u.grantPublicSchemaAccess) || !u.grantPublicSchemaAccess)' + - message: spec.logicalReplicas requires spec.postgresVersion >= 17 + rule: '!has(self.logicalReplicas) || size(self.logicalReplicas) == 0 + || self.postgresVersion >= 17' + - message: spec.logicalReplicas[].bootstrapMethod must be 'pg_basebackup' + when spec.backups.enabled is false + rule: '!has(self.logicalReplicas) || size(self.logicalReplicas) == 0 + || !has(self.backups) || !has(self.backups.enabled) || self.backups.enabled + || self.logicalReplicas.all(r, has(r.bootstrapMethod) && r.bootstrapMethod + == ''pg_basebackup'')' status: properties: conditions: @@ -15475,6 +16124,34 @@ spec: items: type: string type: array + logicalReplicas: + items: + properties: + databases: + items: + type: string + type: array + invalidatedAt: + format: date-time + type: string + message: + type: string + name: + type: string + reason: + type: string + seededAt: + format: date-time + type: string + state: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map observedGeneration: format: int64 type: integer diff --git a/config/crd/bases/pgv2.percona.com_perconapgclusters.yaml b/config/crd/bases/pgv2.percona.com_perconapgclusters.yaml index e301e47b19..ca737d09da 100644 --- a/config/crd/bases/pgv2.percona.com_perconapgclusters.yaml +++ b/config/crd/bases/pgv2.percona.com_perconapgclusters.yaml @@ -12762,6 +12762,646 @@ spec: required: - image type: object + logicalReplicas: + items: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + bootstrapMethod: + default: pgbackrest + enum: + - pgbackrest + - pg_basebackup + type: string + dataVolumeClaimSpec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + databases: + items: + maxLength: 63 + minLength: 1 + type: string + type: array + expose: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + loadBalancerClass: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + nodePort: + format: int32 + type: integer + type: + default: ClusterIP + enum: + - ClusterIP + - NodePort + - LoadBalancer + type: string + type: object + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + name: + maxLength: 20 + pattern: ^[a-z][a-z0-9-]*[a-z0-9]$ + type: string + priorityClassName: + type: string + resources: + properties: + claims: + items: + properties: + name: + type: string + request: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + required: + - dataVolumeClaimSpec + - name + type: object + type: array metadata: properties: annotations: @@ -15845,6 +16485,15 @@ spec: and is true rule: '!has(self.users) || self.postgresVersion >= 15 || self.users.all(u, !has(u.grantPublicSchemaAccess) || !u.grantPublicSchemaAccess)' + - message: spec.logicalReplicas requires spec.postgresVersion >= 17 + rule: '!has(self.logicalReplicas) || size(self.logicalReplicas) == 0 + || self.postgresVersion >= 17' + - message: spec.logicalReplicas[].bootstrapMethod must be 'pg_basebackup' + when spec.backups.enabled is false + rule: '!has(self.logicalReplicas) || size(self.logicalReplicas) == 0 + || !has(self.backups) || !has(self.backups.enabled) || self.backups.enabled + || self.logicalReplicas.all(r, has(r.bootstrapMethod) && r.bootstrapMethod + == ''pg_basebackup'')' status: properties: conditions: @@ -15889,6 +16538,34 @@ spec: items: type: string type: array + logicalReplicas: + items: + properties: + databases: + items: + type: string + type: array + invalidatedAt: + format: date-time + type: string + message: + type: string + name: + type: string + reason: + type: string + seededAt: + format: date-time + type: string + state: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map observedGeneration: format: int64 type: integer diff --git a/config/crd/bases/upstream.pgv2.percona.com_postgresclusters.yaml b/config/crd/bases/upstream.pgv2.percona.com_postgresclusters.yaml index 006ee90b1a..ac57836708 100644 --- a/config/crd/bases/upstream.pgv2.percona.com_postgresclusters.yaml +++ b/config/crd/bases/upstream.pgv2.percona.com_postgresclusters.yaml @@ -10819,6 +10819,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + logicalReplicas: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map metadata: properties: annotations: diff --git a/config/rbac/cluster/role.yaml b/config/rbac/cluster/role.yaml index 462ac735b9..e3d9934558 100644 --- a/config/rbac/cluster/role.yaml +++ b/config/rbac/cluster/role.yaml @@ -8,9 +8,7 @@ rules: - '' resources: - configmaps - - pods - - secrets - - serviceaccounts + - persistentvolumeclaims - services verbs: - create @@ -18,6 +16,7 @@ rules: - get - list - patch + - update - watch - apiGroups: - '' @@ -49,20 +48,31 @@ rules: - apiGroups: - '' resources: - - persistentvolumeclaims + - pods + - secrets + - serviceaccounts verbs: - create - delete - get - list - patch - - update - watch - apiGroups: - apps resources: - deployments - replicasets + verbs: + - create + - delete + - get + - list + - patch + - watch +- apiGroups: + - apps + resources: - statefulsets verbs: - create @@ -70,6 +80,7 @@ rules: - get - list - patch + - update - watch - apiGroups: - batch diff --git a/config/rbac/namespace/role.yaml b/config/rbac/namespace/role.yaml index 5361ec1a72..0e7a695913 100644 --- a/config/rbac/namespace/role.yaml +++ b/config/rbac/namespace/role.yaml @@ -8,9 +8,7 @@ rules: - '' resources: - configmaps - - pods - - secrets - - serviceaccounts + - persistentvolumeclaims - services verbs: - create @@ -18,6 +16,7 @@ rules: - get - list - patch + - update - watch - apiGroups: - '' @@ -49,20 +48,31 @@ rules: - apiGroups: - '' resources: - - persistentvolumeclaims + - pods + - secrets + - serviceaccounts verbs: - create - delete - get - list - patch - - update - watch - apiGroups: - apps resources: - deployments - replicasets + verbs: + - create + - delete + - get + - list + - patch + - watch +- apiGroups: + - apps + resources: - statefulsets verbs: - create @@ -70,6 +80,7 @@ rules: - get - list - patch + - update - watch - apiGroups: - batch diff --git a/deploy/bundle.yaml b/deploy/bundle.yaml index 2f4574cd59..7e1c39ffdf 100644 --- a/deploy/bundle.yaml +++ b/deploy/bundle.yaml @@ -12963,6 +12963,646 @@ spec: required: - image type: object + logicalReplicas: + items: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + bootstrapMethod: + default: pgbackrest + enum: + - pgbackrest + - pg_basebackup + type: string + dataVolumeClaimSpec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + databases: + items: + maxLength: 63 + minLength: 1 + type: string + type: array + expose: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + loadBalancerClass: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + nodePort: + format: int32 + type: integer + type: + default: ClusterIP + enum: + - ClusterIP + - NodePort + - LoadBalancer + type: string + type: object + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + name: + maxLength: 20 + pattern: ^[a-z][a-z0-9-]*[a-z0-9]$ + type: string + priorityClassName: + type: string + resources: + properties: + claims: + items: + properties: + name: + type: string + request: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + required: + - dataVolumeClaimSpec + - name + type: object + type: array metadata: properties: annotations: @@ -16046,6 +16686,15 @@ spec: and is true rule: '!has(self.users) || self.postgresVersion >= 15 || self.users.all(u, !has(u.grantPublicSchemaAccess) || !u.grantPublicSchemaAccess)' + - message: spec.logicalReplicas requires spec.postgresVersion >= 17 + rule: '!has(self.logicalReplicas) || size(self.logicalReplicas) == 0 + || self.postgresVersion >= 17' + - message: spec.logicalReplicas[].bootstrapMethod must be 'pg_basebackup' + when spec.backups.enabled is false + rule: '!has(self.logicalReplicas) || size(self.logicalReplicas) == 0 + || !has(self.backups) || !has(self.backups.enabled) || self.backups.enabled + || self.logicalReplicas.all(r, has(r.bootstrapMethod) && r.bootstrapMethod + == ''pg_basebackup'')' status: properties: conditions: @@ -16090,6 +16739,34 @@ spec: items: type: string type: array + logicalReplicas: + items: + properties: + databases: + items: + type: string + type: array + invalidatedAt: + format: date-time + type: string + message: + type: string + name: + type: string + reason: + type: string + seededAt: + format: date-time + type: string + state: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map observedGeneration: format: int64 type: integer @@ -31028,6 +31705,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + logicalReplicas: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map metadata: properties: annotations: @@ -35415,9 +36104,7 @@ rules: - "" resources: - configmaps - - pods - - secrets - - serviceaccounts + - persistentvolumeclaims - services verbs: - create @@ -35425,6 +36112,7 @@ rules: - get - list - patch + - update - watch - apiGroups: - "" @@ -35456,20 +36144,31 @@ rules: - apiGroups: - "" resources: - - persistentvolumeclaims + - pods + - secrets + - serviceaccounts verbs: - create - delete - get - list - patch - - update - watch - apiGroups: - apps resources: - deployments - replicasets + verbs: + - create + - delete + - get + - list + - patch + - watch +- apiGroups: + - apps + resources: - statefulsets verbs: - create @@ -35477,6 +36176,7 @@ rules: - get - list - patch + - update - watch - apiGroups: - batch diff --git a/deploy/cr.yaml b/deploy/cr.yaml index 95c65127d5..22866790ff 100644 --- a/deploy/cr.yaml +++ b/deploy/cr.yaml @@ -376,6 +376,24 @@ spec: # - mountPath: /pgdata/dicts # readOnly: true +# logicalReplicas: +# - name: analytics +# databases: +# - cluster1 +# bootstrapMethod: pgbackrest +# dataVolumeClaimSpec: +# accessModes: +# - ReadWriteOnce +# resources: +# requests: +# storage: 1Gi +# resources: +# limits: +# cpu: 2.0 +# memory: 4Gi +# expose: +# type: LoadBalancer + proxy: pgBouncer: replicas: 3 @@ -865,6 +883,14 @@ spec: # max_worker_processes: 2 # shared_buffers: 1GB # work_mem: 2MB +# use_slots: true +# slots: +# cdc_consumer: +# type: logical +# database: myapp +# plugin: pgoutput +# external_replica: +# type: physical # createReplicaMethods: # - pgbackrest # - basebackup diff --git a/deploy/crd.yaml b/deploy/crd.yaml index d86293633a..3af3c94930 100644 --- a/deploy/crd.yaml +++ b/deploy/crd.yaml @@ -12963,6 +12963,646 @@ spec: required: - image type: object + logicalReplicas: + items: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + bootstrapMethod: + default: pgbackrest + enum: + - pgbackrest + - pg_basebackup + type: string + dataVolumeClaimSpec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + databases: + items: + maxLength: 63 + minLength: 1 + type: string + type: array + expose: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + loadBalancerClass: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + nodePort: + format: int32 + type: integer + type: + default: ClusterIP + enum: + - ClusterIP + - NodePort + - LoadBalancer + type: string + type: object + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + name: + maxLength: 20 + pattern: ^[a-z][a-z0-9-]*[a-z0-9]$ + type: string + priorityClassName: + type: string + resources: + properties: + claims: + items: + properties: + name: + type: string + request: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + required: + - dataVolumeClaimSpec + - name + type: object + type: array metadata: properties: annotations: @@ -16046,6 +16686,15 @@ spec: and is true rule: '!has(self.users) || self.postgresVersion >= 15 || self.users.all(u, !has(u.grantPublicSchemaAccess) || !u.grantPublicSchemaAccess)' + - message: spec.logicalReplicas requires spec.postgresVersion >= 17 + rule: '!has(self.logicalReplicas) || size(self.logicalReplicas) == 0 + || self.postgresVersion >= 17' + - message: spec.logicalReplicas[].bootstrapMethod must be 'pg_basebackup' + when spec.backups.enabled is false + rule: '!has(self.logicalReplicas) || size(self.logicalReplicas) == 0 + || !has(self.backups) || !has(self.backups.enabled) || self.backups.enabled + || self.logicalReplicas.all(r, has(r.bootstrapMethod) && r.bootstrapMethod + == ''pg_basebackup'')' status: properties: conditions: @@ -16090,6 +16739,34 @@ spec: items: type: string type: array + logicalReplicas: + items: + properties: + databases: + items: + type: string + type: array + invalidatedAt: + format: date-time + type: string + message: + type: string + name: + type: string + reason: + type: string + seededAt: + format: date-time + type: string + state: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map observedGeneration: format: int64 type: integer @@ -31028,6 +31705,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + logicalReplicas: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map metadata: properties: annotations: diff --git a/deploy/cw-bundle.yaml b/deploy/cw-bundle.yaml index c7334d4b87..78c90861a7 100644 --- a/deploy/cw-bundle.yaml +++ b/deploy/cw-bundle.yaml @@ -12963,6 +12963,646 @@ spec: required: - image type: object + logicalReplicas: + items: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + bootstrapMethod: + default: pgbackrest + enum: + - pgbackrest + - pg_basebackup + type: string + dataVolumeClaimSpec: + properties: + accessModes: + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + databases: + items: + maxLength: 63 + minLength: 1 + type: string + type: array + expose: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + loadBalancerClass: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + nodePort: + format: int32 + type: integer + type: + default: ClusterIP + enum: + - ClusterIP + - NodePort + - LoadBalancer + type: string + type: object + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + name: + maxLength: 20 + pattern: ^[a-z][a-z0-9-]*[a-z0-9]$ + type: string + priorityClassName: + type: string + resources: + properties: + claims: + items: + properties: + name: + type: string + request: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + required: + - dataVolumeClaimSpec + - name + type: object + type: array metadata: properties: annotations: @@ -16046,6 +16686,15 @@ spec: and is true rule: '!has(self.users) || self.postgresVersion >= 15 || self.users.all(u, !has(u.grantPublicSchemaAccess) || !u.grantPublicSchemaAccess)' + - message: spec.logicalReplicas requires spec.postgresVersion >= 17 + rule: '!has(self.logicalReplicas) || size(self.logicalReplicas) == 0 + || self.postgresVersion >= 17' + - message: spec.logicalReplicas[].bootstrapMethod must be 'pg_basebackup' + when spec.backups.enabled is false + rule: '!has(self.logicalReplicas) || size(self.logicalReplicas) == 0 + || !has(self.backups) || !has(self.backups.enabled) || self.backups.enabled + || self.logicalReplicas.all(r, has(r.bootstrapMethod) && r.bootstrapMethod + == ''pg_basebackup'')' status: properties: conditions: @@ -16090,6 +16739,34 @@ spec: items: type: string type: array + logicalReplicas: + items: + properties: + databases: + items: + type: string + type: array + invalidatedAt: + format: date-time + type: string + message: + type: string + name: + type: string + reason: + type: string + seededAt: + format: date-time + type: string + state: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map observedGeneration: format: int64 type: integer @@ -31028,6 +31705,18 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + logicalReplicas: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map metadata: properties: annotations: @@ -35415,9 +36104,7 @@ rules: - "" resources: - configmaps - - pods - - secrets - - serviceaccounts + - persistentvolumeclaims - services verbs: - create @@ -35425,6 +36112,7 @@ rules: - get - list - patch + - update - watch - apiGroups: - "" @@ -35456,20 +36144,31 @@ rules: - apiGroups: - "" resources: - - persistentvolumeclaims + - pods + - secrets + - serviceaccounts verbs: - create - delete - get - list - patch - - update - watch - apiGroups: - apps resources: - deployments - replicasets + verbs: + - create + - delete + - get + - list + - patch + - watch +- apiGroups: + - apps + resources: - statefulsets verbs: - create @@ -35477,6 +36176,7 @@ rules: - get - list - patch + - update - watch - apiGroups: - batch diff --git a/deploy/cw-rbac.yaml b/deploy/cw-rbac.yaml index 1072b497e1..c71c0cc0cd 100644 --- a/deploy/cw-rbac.yaml +++ b/deploy/cw-rbac.yaml @@ -12,9 +12,7 @@ rules: - "" resources: - configmaps - - pods - - secrets - - serviceaccounts + - persistentvolumeclaims - services verbs: - create @@ -22,6 +20,7 @@ rules: - get - list - patch + - update - watch - apiGroups: - "" @@ -53,20 +52,31 @@ rules: - apiGroups: - "" resources: - - persistentvolumeclaims + - pods + - secrets + - serviceaccounts verbs: - create - delete - get - list - patch - - update - watch - apiGroups: - apps resources: - deployments - replicasets + verbs: + - create + - delete + - get + - list + - patch + - watch +- apiGroups: + - apps + resources: - statefulsets verbs: - create @@ -74,6 +84,7 @@ rules: - get - list - patch + - update - watch - apiGroups: - batch diff --git a/deploy/rbac.yaml b/deploy/rbac.yaml index c815cb520a..2aed389efc 100644 --- a/deploy/rbac.yaml +++ b/deploy/rbac.yaml @@ -12,9 +12,7 @@ rules: - "" resources: - configmaps - - pods - - secrets - - serviceaccounts + - persistentvolumeclaims - services verbs: - create @@ -22,6 +20,7 @@ rules: - get - list - patch + - update - watch - apiGroups: - "" @@ -53,20 +52,31 @@ rules: - apiGroups: - "" resources: - - persistentvolumeclaims + - pods + - secrets + - serviceaccounts verbs: - create - delete - get - list - patch - - update - watch - apiGroups: - apps resources: - deployments - replicasets + verbs: + - create + - delete + - get + - list + - patch + - watch +- apiGroups: + - apps + resources: - statefulsets verbs: - create @@ -74,6 +84,7 @@ rules: - get - list - patch + - update - watch - apiGroups: - batch diff --git a/e2e-tests/run-minikube.csv b/e2e-tests/run-minikube.csv index 3e4c0bebd0..2e51e0315b 100644 --- a/e2e-tests/run-minikube.csv +++ b/e2e-tests/run-minikube.csv @@ -4,6 +4,7 @@ database-init-sql demand-backup finalizers init-deploy +logical-replica one-pod operator-self-healing pgbouncer diff --git a/e2e-tests/run-pr.csv b/e2e-tests/run-pr.csv index e898338f52..c74e3806da 100644 --- a/e2e-tests/run-pr.csv +++ b/e2e-tests/run-pr.csv @@ -13,6 +13,7 @@ extra-volumes finalizers init-deploy huge-pages +logical-replica major-upgrade-14-to-15 major-upgrade-15-to-16 major-upgrade-16-to-17 diff --git a/e2e-tests/run-release.csv b/e2e-tests/run-release.csv index dc8e18f537..39396ed587 100644 --- a/e2e-tests/run-release.csv +++ b/e2e-tests/run-release.csv @@ -14,6 +14,7 @@ finalizers init-deploy k8s-upgrade huge-pages +logical-replica major-upgrade-14-to-15 major-upgrade-15-to-16 major-upgrade-16-to-17 diff --git a/e2e-tests/tests/logical-replica/00-assert.yaml b/e2e-tests/tests/logical-replica/00-assert.yaml new file mode 100644 index 0000000000..ae5a062d84 --- /dev/null +++ b/e2e-tests/tests/logical-replica/00-assert.yaml @@ -0,0 +1,24 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 120 +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: perconapgclusters.pgv2.percona.com +spec: + group: pgv2.percona.com + names: + kind: PerconaPGCluster + listKind: PerconaPGClusterList + plural: perconapgclusters + singular: perconapgcluster + scope: Namespaced +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: check-operator-deploy-status +timeout: 120 +commands: + - script: kubectl assert exist-enhanced deployment percona-postgresql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1 diff --git a/e2e-tests/tests/logical-replica/00-deploy-operator.yaml b/e2e-tests/tests/logical-replica/00-deploy-operator.yaml new file mode 100644 index 0000000000..790c92569d --- /dev/null +++ b/e2e-tests/tests/logical-replica/00-deploy-operator.yaml @@ -0,0 +1,13 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - timeout: 120 + script: |- + set -o errexit + set -o xtrace + + source ../../functions + init_temp_dir # do this only in the first TestStep + + deploy_operator + deploy_client diff --git a/e2e-tests/tests/logical-replica/01-assert.yaml b/e2e-tests/tests/logical-replica/01-assert.yaml new file mode 100644 index 0000000000..e04908b051 --- /dev/null +++ b/e2e-tests/tests/logical-replica/01-assert.yaml @@ -0,0 +1,12 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + kubectl -n "${NAMESPACE}" wait --for=jsonpath='{.status.state}'=ready \ + --timeout=580s pg/logical-replica diff --git a/e2e-tests/tests/logical-replica/01-create-cluster.yaml b/e2e-tests/tests/logical-replica/01-create-cluster.yaml new file mode 100644 index 0000000000..d830cc3136 --- /dev/null +++ b/e2e-tests/tests/logical-replica/01-create-cluster.yaml @@ -0,0 +1,11 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - timeout: 30 + script: |- + set -o errexit + set -o xtrace + + source ../../functions + + get_cr | kubectl -n "${NAMESPACE}" apply -f - diff --git a/e2e-tests/tests/logical-replica/02-write-data.yaml b/e2e-tests/tests/logical-replica/02-write-data.yaml new file mode 100644 index 0000000000..b5f10a4826 --- /dev/null +++ b/e2e-tests/tests/logical-replica/02-write-data.yaml @@ -0,0 +1,30 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - timeout: 60 + script: |- + set -o errexit + set -o xtrace + + source ../../functions + + if [[ ${PG_VER} -lt 17 ]]; then + exit 0 + fi + + PASS=$(get_psql_user_pass logical-replica-pguser-postgres) + HOST=$(get_psql_user_host logical-replica-pguser-postgres) + + # Seeded before the logical replica exists, so these rows can only reach + # it through the backup the bootstrap Job restores. + run_psql \ + 'CREATE DATABASE \"logical-replica\";' \ + "postgres://postgres:${PASS}@${HOST}" + + run_psql \ + '\c logical-replica \\\ CREATE TABLE IF NOT EXISTS lr_test (id int PRIMARY KEY);' \ + "postgres://postgres:${PASS}@${HOST}" + + run_psql \ + '\c logical-replica \\\ INSERT INTO lr_test (id) VALUES (1);' \ + "postgres://postgres:${PASS}@${HOST}" diff --git a/e2e-tests/tests/logical-replica/03-add-logical-replica.yaml b/e2e-tests/tests/logical-replica/03-add-logical-replica.yaml new file mode 100644 index 0000000000..e86f68465e --- /dev/null +++ b/e2e-tests/tests/logical-replica/03-add-logical-replica.yaml @@ -0,0 +1,41 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - timeout: 30 + script: |- + set -o errexit + set -o xtrace + + source ../../functions + + if [[ ${PG_VER} -lt 17 ]]; then + exit 0 + fi + + # Both seeding methods against the same primary and the same database. + # The slot, publication and subscription names all carry the replica name, + # so the two never collide. + kubectl -n "${NAMESPACE}" patch pg logical-replica --type=merge --patch '{ + "spec": { + "logicalReplicas": [ + { + "name": "analytics", + "databases": ["logical-replica"], + "bootstrapMethod": "pgbackrest", + "dataVolumeClaimSpec": { + "accessModes": ["ReadWriteOnce"], + "resources": {"requests": {"storage": "1Gi"}} + } + }, + { + "name": "reporting", + "databases": ["logical-replica"], + "bootstrapMethod": "pg_basebackup", + "dataVolumeClaimSpec": { + "accessModes": ["ReadWriteOnce"], + "resources": {"requests": {"storage": "1Gi"}} + } + } + ] + } + }' diff --git a/e2e-tests/tests/logical-replica/03-assert.yaml b/e2e-tests/tests/logical-replica/03-assert.yaml new file mode 100644 index 0000000000..54ab45e82e --- /dev/null +++ b/e2e-tests/tests/logical-replica/03-assert.yaml @@ -0,0 +1,55 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 900 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + if [[ ${PG_VER} -lt 17 ]]; then + exit 0 + fi + + retry 30 5 bash -c ' + status=$(kubectl -n "'"${NAMESPACE}"'" get pg logical-replica \ + -o jsonpath="{.status.conditions[?(@.type==\"ReadyForLogicalReplication\")].status}") + echo "ReadyForLogicalReplication: ${status}" + [[ ${status} == "True" ]] + ' + + PASS=$(get_psql_user_pass logical-replica-pguser-postgres) + + for replica in analytics reporting; do + retry 60 10 bash -c ' + state=$(kubectl -n "'"${NAMESPACE}"'" get pg logical-replica \ + -o jsonpath="{.status.logicalReplicas[?(@.name==\"'"${replica}"'\")].state}") + echo "logical replica '"${replica}"' state: ${state}" + [[ ${state} == "ready" ]] + ' + + kubectl -n "${NAMESPACE}" wait --for=condition=Ready pod \ + -l "pgv2.percona.com/logical-replica=${replica}" --timeout=120s + + REPLICA="logical-replica-lr-${replica}.${NAMESPACE}.svc" + + worker=$(run_psql \ + '\c logical-replica \\\ SELECT count(*) FROM pg_stat_subscription WHERE pid IS NOT NULL;' \ + "postgres://postgres:${PASS}@${REPLICA}" | xargs) + + if [[ ${worker} != "1" ]]; then + echo "ERROR: expected a running apply worker on ${replica}, got: ${worker}" + kubectl -n "${NAMESPACE}" logs "statefulset/logical-replica-lr-${replica}" --tail=50 || true + exit 1 + fi + + without=$(run_psql \ + '\c logical-replica \\\ SELECT count(*) FROM pg_subscription WHERE NOT subdisableonerr;' \ + "postgres://postgres:${PASS}@${REPLICA}" | xargs) + + if [[ ${without} != "0" ]]; then + echo "ERROR: ${without} subscriptions on ${replica} are not set to disable on error" + exit 1 + fi + done diff --git a/e2e-tests/tests/logical-replica/04-verify-seeded-data.yaml b/e2e-tests/tests/logical-replica/04-verify-seeded-data.yaml new file mode 100644 index 0000000000..a6f4f8e49f --- /dev/null +++ b/e2e-tests/tests/logical-replica/04-verify-seeded-data.yaml @@ -0,0 +1,31 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - timeout: 120 + script: |- + set -o errexit + set -o xtrace + + source ../../functions + + if [[ ${PG_VER} -lt 17 ]]; then + exit 0 + fi + + PASS=$(get_psql_user_pass logical-replica-pguser-postgres) + + # The row predates both replicas, so it can only be there because the + # physical seed brought it along. This is what proves each bootstrap + # method copied the data directory correctly. + for replica in analytics reporting; do + REPLICA="logical-replica-lr-${replica}.${NAMESPACE}.svc" + + data=$(run_psql \ + '\c logical-replica \\\ SELECT id FROM lr_test ORDER BY id;' \ + "postgres://postgres:${PASS}@${REPLICA}") + + if [[ $(echo ${data} | xargs) != "1" ]]; then + echo "ERROR: expected the seeded row on ${replica}, got: ${data}" + exit 1 + fi + done diff --git a/e2e-tests/tests/logical-replica/05-stream-new-data.yaml b/e2e-tests/tests/logical-replica/05-stream-new-data.yaml new file mode 100644 index 0000000000..3878d50896 --- /dev/null +++ b/e2e-tests/tests/logical-replica/05-stream-new-data.yaml @@ -0,0 +1,38 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - timeout: 600 + script: |- + set -o errexit + set -o xtrace + + source ../../functions + + if [[ ${PG_VER} -lt 17 ]]; then + exit 0 + fi + + PASS=$(get_psql_user_pass logical-replica-pguser-postgres) + HOST=$(get_psql_user_host logical-replica-pguser-postgres) + + replica_row_count() { + local replica=$1 + local want=$2 + local got + + got=$(run_psql \ + '\c logical-replica \\\ SELECT count(*) FROM lr_test;' \ + "postgres://postgres:${PASS}@logical-replica-lr-${replica}.${NAMESPACE}.svc" | xargs) + + echo "rows on ${replica}: ${got} (want ${want})" + [[ ${got} == "${want}" ]] + } + + # Written after the conversion, so it can only arrive over the + # subscription that pg_createsubscriber set up. + run_psql \ + '\c logical-replica \\\ INSERT INTO lr_test (id) VALUES (2);' \ + "postgres://postgres:${PASS}@${HOST}" + + retry 30 5 replica_row_count analytics 2 + retry 30 5 replica_row_count reporting 2 diff --git a/e2e-tests/tests/logical-replica/06-slot-survives-patroni.yaml b/e2e-tests/tests/logical-replica/06-slot-survives-patroni.yaml new file mode 100644 index 0000000000..4dcd571495 --- /dev/null +++ b/e2e-tests/tests/logical-replica/06-slot-survives-patroni.yaml @@ -0,0 +1,54 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - timeout: 300 + script: |- + set -o errexit + set -o xtrace + + source ../../functions + + if [[ ${PG_VER} -lt 17 ]]; then + exit 0 + fi + + PASS=$(get_psql_user_pass logical-replica-pguser-postgres) + HOST=$(get_psql_user_host logical-replica-pguser-postgres) + + # run_psql interpolates the statement into a printf format string, so it + # cannot carry single quotes. plugin IS NOT NULL is what makes a slot + # logical, which is the only kind this cluster has. + slot_count() { + run_psql \ + "SELECT count(*) FROM pg_replication_slots WHERE plugin IS NOT NULL;" \ + "postgres://postgres:${PASS}@${HOST}" | xargs + } + + # One per logical replica: analytics and reporting. + before=$(slot_count) + if [[ ${before} != "2" ]]; then + echo "ERROR: expected two logical slots on the primary, got: ${before}" + exit 1 + fi + + # Regression guard for the ignore_slots wiring. Patroni drops slots it + # does not know about on every HA loop, and that is not gated on + # use_slots, which this operator leaves off by default and this cluster + # does not override. loop_wait defaults to 10s, so wait out several loops + # before re-checking. + sleep 120 + + after=$(slot_count) + if [[ ${after} != "2" ]]; then + echo "ERROR: Patroni deleted a logical replication slot (count=${after})" + exit 1 + fi + + for replica in analytics reporting; do + state=$(kubectl -n "${NAMESPACE}" get pg logical-replica \ + -o jsonpath="{.status.logicalReplicas[?(@.name==\"${replica}\")].state}") + if [[ ${state} != "ready" ]]; then + echo "ERROR: logical replica ${replica} is ${state}, expected ready" + exit 1 + fi + done diff --git a/e2e-tests/tests/logical-replica/07-replica-is-read-only.yaml b/e2e-tests/tests/logical-replica/07-replica-is-read-only.yaml new file mode 100644 index 0000000000..adf2d4b869 --- /dev/null +++ b/e2e-tests/tests/logical-replica/07-replica-is-read-only.yaml @@ -0,0 +1,43 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - timeout: 60 + script: |- + set -o errexit + set -o xtrace + + source ../../functions + + if [[ ${PG_VER} -lt 17 ]]; then + exit 0 + fi + + PASS=$(get_psql_user_pass logical-replica-pguser-postgres) + REPLICA="logical-replica-lr-analytics.${NAMESPACE}.svc" + + # Reads work. + run_psql \ + '\c logical-replica \\\ SELECT count(*) FROM lr_test;' \ + "postgres://postgres:${PASS}@${REPLICA}" + + # Writes do not: the replica defaults to read-only, because diverging it + # from the primary breaks apply on the first conflicting row. + if run_psql \ + '\c logical-replica \\\ INSERT INTO lr_test (id) VALUES (99);' \ + "postgres://postgres:${PASS}@${REPLICA}"; then + echo "ERROR: the logical replica accepted a write but should be read-only" + exit 1 + fi + + # Replication itself is unaffected by default_transaction_read_only: the + # apply worker does not go through the executor, which enforces it. + # 05-stream-new-data already proved a post-conversion row arrives; assert + # the write above did not land either. + data=$(run_psql \ + '\c logical-replica \\\ SELECT count(*) FROM lr_test;' \ + "postgres://postgres:${PASS}@${REPLICA}") + + if [[ $(echo ${data} | xargs) != "2" ]]; then + echo "ERROR: expected 2 rows on the logical replica, got: ${data}" + exit 1 + fi diff --git a/e2e-tests/tests/logical-replica/08-assert.yaml b/e2e-tests/tests/logical-replica/08-assert.yaml new file mode 100644 index 0000000000..95ddcdf2eb --- /dev/null +++ b/e2e-tests/tests/logical-replica/08-assert.yaml @@ -0,0 +1,16 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 660 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + if [[ ${PG_VER} -lt 17 ]]; then + exit 0 + fi + + kubectl -n "${NAMESPACE}" wait --for=jsonpath='{.status.state}'=Succeeded \ + --timeout=560s pg-backup/logical-replica-full-repo1 diff --git a/e2e-tests/tests/logical-replica/08-create-backup.yaml b/e2e-tests/tests/logical-replica/08-create-backup.yaml new file mode 100644 index 0000000000..9ed4b56aca --- /dev/null +++ b/e2e-tests/tests/logical-replica/08-create-backup.yaml @@ -0,0 +1,28 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - timeout: 30 + script: |- + set -o errexit + set -o xtrace + + source ../../functions + + if [[ ${PG_VER} -lt 17 ]]; then + exit 0 + fi + + # Taken while both replicas are running, so the restore below brings the + # publications back with it while the replication slots stay gone: + # pgBackRest does not back up pg_replslot. + cat </dev/null + + replicas=$(kubectl -n "${NAMESPACE}" get statefulset "logical-replica-lr-${replica}" \ + -o jsonpath='{.spec.replicas}') + if [[ ${replicas} != "0" ]]; then + echo "ERROR: an invalidated replica must stay stopped, got replicas: ${replicas}" + exit 1 + fi + done + + slots_gone() { + local got + got=$(run_psql \ + "SELECT count(*) FROM pg_replication_slots WHERE plugin IS NOT NULL;" \ + "postgres://postgres:${PASS}@${HOST}" | xargs) + + echo "logical slots on the restored primary: ${got} (want 0)" + [[ ${got} == "0" ]] + } + + retry 30 5 slots_gone diff --git a/e2e-tests/tests/logical-replica/12-remove-logical-replica.yaml b/e2e-tests/tests/logical-replica/12-remove-logical-replica.yaml new file mode 100644 index 0000000000..457adb8ab4 --- /dev/null +++ b/e2e-tests/tests/logical-replica/12-remove-logical-replica.yaml @@ -0,0 +1,73 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - timeout: 900 + script: |- + set -o errexit + set -o xtrace + + source ../../functions + + if [[ ${PG_VER} -lt 17 ]]; then + exit 0 + fi + + PASS=$(get_psql_user_pass logical-replica-pguser-postgres) + HOST=$(get_psql_user_host logical-replica-pguser-postgres) + + kubectl -n "${NAMESPACE}" patch pg logical-replica --type=json \ + --patch '[{"op": "remove", "path": "/spec/logicalReplicas"}]' + + no_slots_left() { + local got + got=$(run_psql \ + "SELECT count(*) FROM pg_replication_slots WHERE plugin IS NOT NULL;" \ + "postgres://postgres:${PASS}@${HOST}" | xargs) + + echo "logical slots left on the primary: ${got}" + [[ ${got} == "0" ]] + } + + no_publications_left() { + local got + got=$(run_psql \ + '\c logical-replica \\\ SELECT count(*) FROM pg_publication;' \ + "postgres://postgres:${PASS}@${HOST}" | xargs) + + echo "publications left on the primary: ${got}" + [[ ${got} == "0" ]] + } + + objects_gone() { + local replica + for replica in analytics reporting; do + kubectl -n "${NAMESPACE}" get statefulset "logical-replica-lr-${replica}" >/dev/null 2>&1 && return 1 + kubectl -n "${NAMESPACE}" get pvc "logical-replica-lr-${replica}-pgdata" >/dev/null 2>&1 && return 1 + done + + return 0 + } + + condition_gone() { + local got + got=$(kubectl -n "${NAMESPACE}" get pg logical-replica \ + -o jsonpath='{.status.conditions[?(@.type=="ReadyForLogicalReplication")].status}') + + echo "ReadyForLogicalReplication: '${got}'" + [[ -z ${got} ]] + } + + status_gone() { + local got + got=$(kubectl -n "${NAMESPACE}" get pg logical-replica \ + -o jsonpath='{.status.logicalReplicas}') + + echo "status.logicalReplicas: '${got}'" + [[ -z ${got} || ${got} == "[]" ]] + } + + retry 30 5 no_slots_left + retry 30 5 no_publications_left + retry 30 5 objects_gone + retry 30 5 condition_gone + retry 30 5 status_gone diff --git a/e2e-tests/tests/logical-replica/13-assert.yaml b/e2e-tests/tests/logical-replica/13-assert.yaml new file mode 100644 index 0000000000..5420d80284 --- /dev/null +++ b/e2e-tests/tests/logical-replica/13-assert.yaml @@ -0,0 +1,65 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 900 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + if [[ ${PG_VER} -lt 17 ]]; then + exit 0 + fi + + retry 60 10 bash -c ' + state=$(kubectl -n "'"${NAMESPACE}"'" get pg logical-replica \ + -o jsonpath="{.status.logicalReplicas[?(@.name==\"analytics\")].state}") + echo "logical replica analytics state: ${state}" + [[ ${state} == "ready" ]] + ' + + kubectl -n "${NAMESPACE}" wait --for=condition=Ready pod \ + -l "pgv2.percona.com/logical-replica=analytics" --timeout=120s + + PASS=$(get_psql_user_pass logical-replica-pguser-postgres) + HOST=$(get_psql_user_host logical-replica-pguser-postgres) + REPLICA="logical-replica-lr-analytics.${NAMESPACE}.svc" + + # Two rows, not three. Three would mean the old data volume survived and + # the replica merely resumed, which is the failure this covers: it would + # be serving changes the restored cluster no longer has. + replica_row_count() { + local want=$1 + local got + + got=$(run_psql \ + '\c logical-replica \\\ SELECT count(*) FROM lr_test;' \ + "postgres://postgres:${PASS}@${REPLICA}" | xargs) + + echo "rows on analytics: ${got} (want ${want})" + [[ ${got} == "${want}" ]] + } + + retry 30 5 replica_row_count 2 + + # A fresh slot on the restored primary, and no leftovers from the old ones. + one_slot() { + local got + got=$(run_psql \ + "SELECT count(*) FROM pg_replication_slots WHERE plugin IS NOT NULL;" \ + "postgres://postgres:${PASS}@${HOST}" | xargs) + + echo "logical slots on the primary: ${got} (want 1)" + [[ ${got} == "1" ]] + } + + retry 30 5 one_slot + + # Written after the re-seed, so it can only arrive over the subscription + # pg_createsubscriber has just set up. + run_psql \ + '\c logical-replica \\\ INSERT INTO lr_test (id) VALUES (4);' \ + "postgres://postgres:${PASS}@${HOST}" + + retry 30 5 replica_row_count 3 diff --git a/e2e-tests/tests/logical-replica/13-reseed-logical-replica.yaml b/e2e-tests/tests/logical-replica/13-reseed-logical-replica.yaml new file mode 100644 index 0000000000..5ffe245de9 --- /dev/null +++ b/e2e-tests/tests/logical-replica/13-reseed-logical-replica.yaml @@ -0,0 +1,33 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - timeout: 30 + script: |- + set -o errexit + set -o xtrace + + source ../../functions + + if [[ ${PG_VER} -lt 17 ]]; then + exit 0 + fi + + # The same entry as step 03, now against the restored cluster. Step 12 + # left nothing of the old incarnation behind, so this is an ordinary first + # bootstrap. One replica is enough here: both methods are already covered + # above, and this seeds from the repository the restore came from. + kubectl -n "${NAMESPACE}" patch pg logical-replica --type=merge --patch '{ + "spec": { + "logicalReplicas": [ + { + "name": "analytics", + "databases": ["logical-replica"], + "bootstrapMethod": "pgbackrest", + "dataVolumeClaimSpec": { + "accessModes": ["ReadWriteOnce"], + "resources": {"requests": {"storage": "1Gi"}} + } + } + ] + } + }' diff --git a/e2e-tests/tests/logical-replica/14-missing-database.yaml b/e2e-tests/tests/logical-replica/14-missing-database.yaml new file mode 100644 index 0000000000..df8280226a --- /dev/null +++ b/e2e-tests/tests/logical-replica/14-missing-database.yaml @@ -0,0 +1,102 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - timeout: 600 + script: |- + set -o errexit + set -o xtrace + + source ../../functions + + if [[ ${PG_VER} -lt 17 ]]; then + exit 0 + fi + + kubectl -n "${NAMESPACE}" patch pg logical-replica --type=merge --patch '{ + "spec": { + "logicalReplicas": [ + { + "name": "analytics", + "databases": ["logical-replica"], + "bootstrapMethod": "pgbackrest", + "dataVolumeClaimSpec": { + "accessModes": ["ReadWriteOnce"], + "resources": {"requests": {"storage": "1Gi"}} + } + }, + { + "name": "nodb", + "databases": ["not-created-yet"], + "bootstrapMethod": "pgbackrest", + "dataVolumeClaimSpec": { + "accessModes": ["ReadWriteOnce"], + "resources": {"requests": {"storage": "1Gi"}} + } + } + ] + } + }' + + waiting_for_databases() { + local state reason + state=$(kubectl -n "${NAMESPACE}" get pg logical-replica \ + -o jsonpath='{.status.logicalReplicas[?(@.name=="nodb")].state}') + reason=$(kubectl -n "${NAMESPACE}" get pg logical-replica \ + -o jsonpath='{.status.logicalReplicas[?(@.name=="nodb")].reason}') + + echo "logical replica nodb: state='${state}' reason='${reason}'" + [[ ${state} == "bootstrapping" && ${reason} == "WaitingForDatabases" ]] + } + + retry 30 5 waiting_for_databases + + kubectl -n "${NAMESPACE}" get pg logical-replica \ + -o jsonpath='{.status.logicalReplicas[?(@.name=="nodb")].message}' \ + | grep -q "not-created-yet" + + test -z "$(kubectl -n "${NAMESPACE}" get pg logical-replica \ + -o jsonpath='{.status.logicalReplicas[?(@.name=="nodb")].databases}')" + + test -z "$(kubectl -n "${NAMESPACE}" get job \ + -l "pgv2.percona.com/logical-replica=nodb" -o name)" + + test -z "$(kubectl -n "${NAMESPACE}" get pvc,statefulset,configmap \ + -l "pgv2.percona.com/logical-replica=nodb" -o name)" + + analytics_still_ready() { + local state + state=$(kubectl -n "${NAMESPACE}" get pg logical-replica \ + -o jsonpath='{.status.logicalReplicas[?(@.name=="analytics")].state}') + + echo "logical replica analytics state: ${state}" + [[ ${state} == "ready" ]] + } + + retry 12 5 analytics_still_ready + + kubectl -n "${NAMESPACE}" patch pg logical-replica --type=merge --patch '{ + "spec": { + "logicalReplicas": [ + { + "name": "analytics", + "databases": ["logical-replica"], + "bootstrapMethod": "pgbackrest", + "dataVolumeClaimSpec": { + "accessModes": ["ReadWriteOnce"], + "resources": {"requests": {"storage": "1Gi"}} + } + } + ] + } + }' + + status_entry_gone() { + local got + got=$(kubectl -n "${NAMESPACE}" get pg logical-replica \ + -o jsonpath='{.status.logicalReplicas[?(@.name=="nodb")].name}') + + echo "status entry for nodb: '${got}'" + [[ -z ${got} ]] + } + + retry 30 5 status_entry_gone diff --git a/e2e-tests/tests/logical-replica/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/logical-replica/99-remove-cluster-gracefully.yaml new file mode 100644 index 0000000000..8e68f7e87b --- /dev/null +++ b/e2e-tests/tests/logical-replica/99-remove-cluster-gracefully.yaml @@ -0,0 +1,22 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: +- apiVersion: pgv2.percona.com/v2 + kind: PerconaPGCluster + metadata: + name: logical-replica +- apiVersion: upstream.pgv2.percona.com/v1beta1 + kind: PostgresCluster + metadata: + name: logical-replica +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + remove_all_finalizers + check_operator_panic + destroy_operator + timeout: 60 diff --git a/e2e-tests/tests/logical-replica/conf/logical-replica.yaml b/e2e-tests/tests/logical-replica/conf/logical-replica.yaml new file mode 100644 index 0000000000..f18b8c20f6 --- /dev/null +++ b/e2e-tests/tests/logical-replica/conf/logical-replica.yaml @@ -0,0 +1,17 @@ +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGCluster +spec: + instances: + - name: instance1 + replicas: 2 + dataVolumeClaimSpec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + # This test covers both bootstrap methods and an in-place restore of the + # cluster, so it keeps the pgBackRest repository both of those read from. + proxy: + pgBouncer: + replicas: 1 diff --git a/internal/controller/postgrescluster/controller.go b/internal/controller/postgrescluster/controller.go index 36fad289df..26f58732fb 100644 --- a/internal/controller/postgrescluster/controller.go +++ b/internal/controller/postgrescluster/controller.go @@ -43,6 +43,7 @@ import ( pgbruntime "github.com/percona/percona-postgresql-operator/v2/internal/controller/runtime/pgbouncer" "github.com/percona/percona-postgresql-operator/v2/internal/initialize" "github.com/percona/percona-postgresql-operator/v2/internal/logging" + "github.com/percona/percona-postgresql-operator/v2/internal/logicalreplica" "github.com/percona/percona-postgresql-operator/v2/internal/naming" "github.com/percona/percona-postgresql-operator/v2/internal/pgaudit" "github.com/percona/percona-postgresql-operator/v2/internal/pgbackrest" @@ -282,6 +283,7 @@ func (r *Reconciler) Reconcile( pmm.PostgreSQLHBAs(cluster, &pgHBAs) pgmonitor.PostgreSQLHBAs(cluster, &pgHBAs) pgbouncer.PostgreSQL(cluster, &pgHBAs) + logicalreplica.PostgreSQLHBAs(cluster, &pgHBAs) // K8SPG-554 if cluster.Spec.TLSOnly { diff --git a/internal/logicalreplica/postgres.go b/internal/logicalreplica/postgres.go new file mode 100644 index 0000000000..f67cf04d78 --- /dev/null +++ b/internal/logicalreplica/postgres.go @@ -0,0 +1,208 @@ +// Copyright 2021 - 2026 Crunchy Data Solutions, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +package logicalreplica + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "strings" + + "github.com/percona/percona-postgresql-operator/v2/internal/postgres" + "github.com/percona/percona-postgresql-operator/v2/pkg/apis/upstream.pgv2.percona.com/v1beta1" +) + +const ( + // ReplicationUser must match v2.UserLogicalReplication. + ReplicationUser = "logicalrepl" + + OutputPlugin = "pgoutput" + + // maxIdentifierLength is PostgreSQL's NAMEDATALEN-1. + maxIdentifierLength = 63 + + // hbaAuthMethod is what tells the rules in [PostgreSQLHBAs] apart from the + // "reject" rule beside them, which spec.tlsOnly renders as "hostssl" too. + hbaAuthMethod = "scram-sha-256" + hbaOrigin = "hostssl" +) + +// Reasons reported by [PrimaryReadinessQuery], rendered by +// [PrimaryReadinessMessage]. +const ( + ReasonPrimaryInRecovery = "PrimaryInRecovery" + ReasonWALLevelNotLogical = "WALLevelNotLogical" + ReasonRestartPending = "RestartPending" + ReasonReplicationRoleNotReady = "ReplicationRoleNotReady" + ReasonReplicationHBAMissing = "ReplicationHBAMissing" +) + +// PrimaryReadinessQuery returns a query whose single column lists, comma +// separated, every prerequisite of a logical replica bootstrap that the primary +// does not yet satisfy. An empty result means it satisfies all of them. +// +// The pg_hba check is load-bearing beyond itself: patroni.DynamicConfiguration +// is the only writer of pg_hba.conf and it carries ignore_slots in the same map, +// so a rule on disk proves ignore_slots reached Patroni too. +func PrimaryReadinessQuery() string { + return fmt.Sprintf(`SELECT coalesce(pg_catalog.string_agg(c.name, ',' ORDER BY c.n), '') + FROM (VALUES + (1, %[2]s, NOT pg_catalog.pg_is_in_recovery()), + (2, %[3]s, pg_catalog.current_setting('wal_level') = 'logical'), + (3, %[4]s, NOT EXISTS ( + SELECT 1 FROM pg_catalog.pg_settings WHERE pending_restart)), + (4, %[5]s, EXISTS ( + SELECT 1 FROM pg_catalog.pg_authid + WHERE rolname = %[1]s + AND rolcanlogin AND rolreplication AND rolsuper + AND rolpassword IS NOT NULL + AND (rolvaliduntil IS NULL OR rolvaliduntil > pg_catalog.now()))), + (5, %[6]s, EXISTS ( + SELECT 1 FROM pg_catalog.pg_hba_file_rules r + WHERE r.error IS NULL AND r.type = %[7]s AND r.auth_method = %[8]s + AND %[1]s = ANY (r.user_name) AND 'replication' = ANY (r.database)) + AND EXISTS ( + SELECT 1 FROM pg_catalog.pg_hba_file_rules r + WHERE r.error IS NULL AND r.type = %[7]s AND r.auth_method = %[8]s + AND %[1]s = ANY (r.user_name) AND 'all' = ANY (r.database))) + ) AS c(n, name, satisfied) + WHERE NOT c.satisfied;`, + postgres.QuoteLiteral(ReplicationUser), + postgres.QuoteLiteral(ReasonPrimaryInRecovery), + postgres.QuoteLiteral(ReasonWALLevelNotLogical), + postgres.QuoteLiteral(ReasonRestartPending), + postgres.QuoteLiteral(ReasonReplicationRoleNotReady), + postgres.QuoteLiteral(ReasonReplicationHBAMissing), + postgres.QuoteLiteral(hbaOrigin), + postgres.QuoteLiteral(hbaAuthMethod)) +} + +// ParsePrimaryReadinessReasons reads the output of [PrimaryReadinessQuery]. A +// nil result means the primary satisfies every prerequisite. +func ParsePrimaryReadinessReasons(stdout string) []string { + var reasons []string + + for reason := range strings.SplitSeq(stdout, ",") { + if reason = strings.TrimSpace(reason); reason != "" { + reasons = append(reasons, reason) + } + } + + return reasons +} + +// PrimaryReadinessMessage renders one of the reasons above as a sentence for a +// condition message. +func PrimaryReadinessMessage(reason string) string { + switch reason { + case ReasonPrimaryInRecovery: + return "the primary is in recovery; pg_createsubscriber needs a writable publisher" + case ReasonWALLevelNotLogical: + return `"wal_level" is not "logical"; set it back via spec.patroni.dynamicConfiguration` + case ReasonRestartPending: + return "PostgreSQL has a parameter change pending a restart, which would cut off the bootstrap" + case ReasonReplicationRoleNotReady: + return "the " + ReplicationUser + " role has not been created yet" + case ReasonReplicationHBAMissing: + return "the pg_hba rules that let " + ReplicationUser + " reach the primary have not been written yet" + default: + return reason + } +} + +// Enabled returns whether the cluster has any logical replica configured. +func Enabled(inCluster *v1beta1.PostgresCluster) bool { + return inCluster != nil && len(inCluster.Spec.LogicalReplicas) > 0 +} + +// PostgreSQLHBAs provides the Postgres HBA rules that let a logical replica and +// the tools that build it reach the primary as ReplicationUser. +func PostgreSQLHBAs(inCluster *v1beta1.PostgresCluster, outHBAs *postgres.HBAs) { + if !Enabled(inCluster) { + return + } + + // Mandatory rather than Default: the default "hostssl all all all" rule is + // dropped as soon as the user supplies any pg_hba rules of their own. + outHBAs.Mandatory = append(outHBAs.Mandatory, + // PrimaryReadinessQuery matches exactly these two rules; keep them in step. + postgres.NewHBA().TLS().Users(ReplicationUser).Method(hbaAuthMethod).Replication(), + postgres.NewHBA().TLS().Users(ReplicationUser).Method(hbaAuthMethod), + // Never allow this superuser to authenticate without TLS. + postgres.NewHBA().TCP().Users(ReplicationUser).Method("reject")) +} + +// IgnoreSlotsMatchers returns the Patroni "ignore_slots" entries that keep it +// from deleting the logical replication slots backing the logical replicas. +// Patroni drops slots it does not know about regardless of "use_slots", so +// without this the slot pg_createsubscriber leaves behind dies on the next HA +// loop. +// +// The matcher deliberately carries no name: slot names are only known once the +// databases of a replica have been resolved, which happens after this +// configuration has been rendered. Matching every logical pgoutput slot is +// stable from the moment a logical replica appears in the spec. +func IgnoreSlotsMatchers(inCluster *v1beta1.PostgresCluster) []any { + if !Enabled(inCluster) { + return nil + } + + return []any{map[string]any{ + "type": "logical", + "plugin": OutputPlugin, + }} +} + +// Names of the replication objects backing database db of a logical replica. +func SlotName(replica, db string) string { + return identifier("pgo_lr_slot", replica, db) +} + +func PublicationName(replica, db string) string { + return identifier("pgo_lr_pub", replica, db) +} + +func SubscriptionName(replica, db string) string { + return identifier("pgo_lr_sub", replica, db) +} + +// DisableOnErrorSQL returns the statement that puts the subscription of database +// db on the given logical replica into "disable_on_error" mode. +// +// pg_createsubscriber has no option for this and it is a per-subscription option +// rather than a server setting. Without it a single conflicting row leaves the +// apply worker retrying forever while its slot pins WAL on the primary. +func DisableOnErrorSQL(replica, db string) string { + return fmt.Sprintf("ALTER SUBSCRIPTION %q SET (disable_on_error = true);", + SubscriptionName(replica, db)) +} + +// identifier folds the given parts to [a-z0-9_], caps the result at +// NAMEDATALEN-1, and always appends a hash so that names which sanitise or +// truncate alike stay distinct. +func identifier(prefix, replica, db string) string { + sum := sha256.Sum256([]byte(replica + "\x00" + db)) + suffix := "_" + hex.EncodeToString(sum[:])[:8] + + name := prefix + "_" + sanitize(replica) + "_" + sanitize(db) + if len(name) > maxIdentifierLength-len(suffix) { + name = name[:maxIdentifierLength-len(suffix)] + } + + return name + suffix +} + +func sanitize(s string) string { + return strings.Map(func(r rune) rune { + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9': + return r + case r >= 'A' && r <= 'Z': + return r + ('a' - 'A') + default: + return '_' + } + }, s) +} diff --git a/internal/logicalreplica/postgres_test.go b/internal/logicalreplica/postgres_test.go new file mode 100644 index 0000000000..5e1399aef2 --- /dev/null +++ b/internal/logicalreplica/postgres_test.go @@ -0,0 +1,230 @@ +// Copyright 2021 - 2026 Crunchy Data Solutions, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +package logicalreplica + +import ( + "strings" + "testing" + + pg_query "github.com/pganalyze/pg_query_go/v6" + "gotest.tools/v3/assert" + + "github.com/percona/percona-postgresql-operator/v2/internal/postgres" + "github.com/percona/percona-postgresql-operator/v2/pkg/apis/upstream.pgv2.percona.com/v1beta1" +) + +func clusterWithReplicas(names ...string) *v1beta1.PostgresCluster { + cluster := new(v1beta1.PostgresCluster) + for _, name := range names { + cluster.Spec.LogicalReplicas = append(cluster.Spec.LogicalReplicas, + v1beta1.LogicalReplicaSpec{Name: name}) + } + return cluster +} + +func TestEnabled(t *testing.T) { + assert.Assert(t, !Enabled(nil)) + assert.Assert(t, !Enabled(new(v1beta1.PostgresCluster))) + assert.Assert(t, Enabled(clusterWithReplicas("analytics"))) +} + +func TestPostgreSQLHBAs(t *testing.T) { + t.Run("no rules without logical replicas", func(t *testing.T) { + hbas := postgres.NewHBAs() + before := len(hbas.Mandatory) + + PostgreSQLHBAs(new(v1beta1.PostgresCluster), &hbas) + + assert.Equal(t, len(hbas.Mandatory), before) + }) + + t.Run("rules are mandatory so custom pg_hba cannot drop them", func(t *testing.T) { + hbas := postgres.NewHBAs() + defaults := len(hbas.Default) + + PostgreSQLHBAs(clusterWithReplicas("analytics"), &hbas) + + rendered := make([]string, 0, len(hbas.Mandatory)) + for _, hba := range hbas.Mandatory { + rendered = append(rendered, hba.String()) + } + all := strings.Join(rendered, "\n") + + // A replication connection for the replica itself and for pg_basebackup + // when it is the bootstrap method, a regular one for + // pg_createsubscriber, and a reject for anything not using TLS. + assert.Assert(t, strings.Contains(all, `hostssl replication "logicalrepl" all scram-sha-256`), all) + assert.Assert(t, strings.Contains(all, `hostssl all "logicalrepl" all scram-sha-256`), all) + assert.Assert(t, strings.Contains(all, `host all "logicalrepl" all reject`), all) + + // Nothing was added to Default, which users can displace. + assert.Equal(t, len(hbas.Default), defaults) + }) +} + +func TestPrimaryReadinessQuery(t *testing.T) { + query := PrimaryReadinessQuery() + + // The query runs on a live primary, where a syntax error would surface as an + // unreachable primary rather than as anything obviously wrong. Parse it with + // PostgreSQL's own grammar instead. + tree, err := pg_query.Parse(query) + assert.NilError(t, err) + assert.Equal(t, len(tree.GetStmts()), 1, query) + + // Every reason the parser can report has to be one the caller knows how to + // render, and each has to be reachable. + for _, reason := range []string{ + ReasonPrimaryInRecovery, ReasonWALLevelNotLogical, ReasonRestartPending, + ReasonReplicationRoleNotReady, ReasonReplicationHBAMissing, + } { + assert.Equal(t, strings.Count(query, postgres.QuoteLiteral(reason)), 1, reason) + assert.Assert(t, PrimaryReadinessMessage(reason) != reason, reason) + } + + assert.Assert(t, strings.Contains(query, postgres.QuoteLiteral(ReplicationUser)), query) +} + +// TestPrimaryReadinessQueryMatchesHBAs pins the query to the rules it looks for. +// Nothing at runtime would report a mismatch: a query that cannot find the rules +// leaves the condition False forever, and the bootstrap simply never starts. +func TestPrimaryReadinessQueryMatchesHBAs(t *testing.T) { + query := PrimaryReadinessQuery() + + // Both of the rules PostgreSQLHBAs adds for the replication user, and only + // those: the "reject" rule alongside them must not satisfy the check. + assert.Assert(t, strings.Contains(query, postgres.QuoteLiteral(hbaOrigin)), query) + assert.Assert(t, strings.Contains(query, postgres.QuoteLiteral(hbaAuthMethod)), query) + assert.Assert(t, strings.Contains(query, `'replication' = ANY (r.database)`), query) + assert.Assert(t, strings.Contains(query, `'all' = ANY (r.database)`), query) + + rendered := func(tlsOnly bool) string { + hbas := postgres.NewHBAs() + PostgreSQLHBAs(clusterWithReplicas("analytics"), &hbas) + + lines := make([]string, 0, len(hbas.Mandatory)) + for _, hba := range hbas.Mandatory { + if tlsOnly { + hba = hba.TLSOnly() + } + lines = append(lines, hba.String()) + } + return strings.Join(lines, "\n") + } + + for _, tlsOnly := range []bool{false, true} { + all := rendered(tlsOnly) + + // pg_hba_file_rules strips the quoting around the user name, and reports + // the keywords as written. + assert.Assert(t, strings.Contains(all, + hbaOrigin+` replication "`+ReplicationUser+`" all `+hbaAuthMethod), all) + assert.Assert(t, strings.Contains(all, + hbaOrigin+` all "`+ReplicationUser+`" all `+hbaAuthMethod), all) + } + + // Under spec.tlsOnly the reject rule becomes a "hostssl" record too, which is + // why the query has to match on the method and not just the type. + assert.Assert(t, strings.Contains(rendered(true), + hbaOrigin+` all "`+ReplicationUser+`" all reject`), rendered(true)) +} + +func TestParsePrimaryReadinessReasons(t *testing.T) { + // An empty result is the whole point: it means the primary is ready. + assert.Assert(t, ParsePrimaryReadinessReasons("") == nil) + assert.Assert(t, ParsePrimaryReadinessReasons(" \n ") == nil) + + assert.DeepEqual(t, ParsePrimaryReadinessReasons(" RestartPending \n"), + []string{"RestartPending"}) + + // psql -t pads its output; the order is the query's, and the caller reports + // the first as the condition reason. + assert.DeepEqual(t, ParsePrimaryReadinessReasons(" RestartPending,ReplicationHBAMissing \n"), + []string{"RestartPending", "ReplicationHBAMissing"}) + + // Unknown reasons pass through rather than being dropped, so a newer operator + // reading an older reason still says something. + assert.Equal(t, PrimaryReadinessMessage("Whatever"), "Whatever") +} + +func TestIgnoreSlotsMatchers(t *testing.T) { + assert.Assert(t, IgnoreSlotsMatchers(new(v1beta1.PostgresCluster)) == nil) + + matchers := IgnoreSlotsMatchers(clusterWithReplicas("analytics", "reporting")) + + // The matcher must not depend on the replica set or on database names: + // those are only known after the operator queries the primary, which would + // leave a window where a slot exists but Patroni does not know to skip it. + assert.Equal(t, len(matchers), 1) + matcher, ok := matchers[0].(map[string]any) + assert.Assert(t, ok) + assert.Equal(t, matcher["type"], "logical") + assert.Equal(t, matcher["plugin"], OutputPlugin) + _, hasName := matcher["name"] + assert.Assert(t, !hasName) +} + +func TestDisableOnErrorSQL(t *testing.T) { + sql := DisableOnErrorSQL("analytics", `we-ird."DB`) + + // The bootstrap Job hands this to psql, where a syntax error would surface as + // a failed bootstrap and a data volume that has to be seeded again. Parse it + // with PostgreSQL's own grammar instead. + tree, err := pg_query.Parse(sql) + assert.NilError(t, err) + assert.Equal(t, len(tree.GetStmts()), 1, sql) + + alter := tree.GetStmts()[0].GetStmt().GetAlterSubscriptionStmt() + assert.Assert(t, alter != nil, sql) + assert.Equal(t, alter.GetSubname(), SubscriptionName("analytics", `we-ird."DB`)) + + // Only the one option, and nothing that would touch the connection: the + // target reaches the publisher again the moment the replica starts, and a + // REFRESH here would try it from the Job. + assert.Equal(t, len(alter.GetOptions()), 1, sql) + assert.Equal(t, alter.GetOptions()[0].GetDefElem().GetDefname(), "disable_on_error") + + // The statement is passed to psql as one single-quoted shell word. + assert.Assert(t, !strings.Contains(sql, "'"), sql) +} + +func TestIdentifierNames(t *testing.T) { + t.Run("distinct kinds do not collide", func(t *testing.T) { + slot := SlotName("analytics", "cluster1") + pub := PublicationName("analytics", "cluster1") + sub := SubscriptionName("analytics", "cluster1") + + assert.Assert(t, slot != pub) + assert.Assert(t, slot != sub) + assert.Assert(t, pub != sub) + }) + + t.Run("deterministic", func(t *testing.T) { + assert.Equal(t, SlotName("analytics", "cluster1"), SlotName("analytics", "cluster1")) + }) + + t.Run("unsafe characters are folded", func(t *testing.T) { + name := SlotName("analytics", `we-ird."DB`) + for _, r := range name { + ok := (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '_' + assert.Assert(t, ok, "unexpected character %q in %q", r, name) + } + }) + + t.Run("bounded by NAMEDATALEN", func(t *testing.T) { + name := SlotName(strings.Repeat("r", 40), strings.Repeat("d", 63)) + assert.Assert(t, len(name) <= maxIdentifierLength, "%d: %q", len(name), name) + }) + + t.Run("names that sanitise alike stay distinct", func(t *testing.T) { + // Both fold to "a_b", so only the hash keeps them apart. + assert.Assert(t, SlotName("r", "a.b") != SlotName("r", "a-b")) + }) + + t.Run("names that truncate alike stay distinct", func(t *testing.T) { + long := strings.Repeat("d", 80) + assert.Assert(t, SlotName("r", long+"one") != SlotName("r", long+"two")) + }) +} diff --git a/internal/patroni/config.go b/internal/patroni/config.go index 848ac0cee6..6853cb45a4 100644 --- a/internal/patroni/config.go +++ b/internal/patroni/config.go @@ -8,12 +8,14 @@ import ( "fmt" "maps" "path" + "slices" "strings" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/yaml" + "github.com/percona/percona-postgresql-operator/v2/internal/logicalreplica" "github.com/percona/percona-postgresql-operator/v2/internal/naming" "github.com/percona/percona-postgresql-operator/v2/internal/postgres" "github.com/percona/percona-postgresql-operator/v2/pkg/apis/upstream.pgv2.percona.com/v1beta1" @@ -207,7 +209,9 @@ func DynamicConfiguration( // Copy the "postgresql" section before making any changes. postgresql := map[string]any{ - // TODO(cbandy): explain this. requires an archive, perhaps. + // Replicas stream from the archive rather than from a slot on the + // primary, so leaving slots off means a lagging or removed replica + // cannot pin WAL on the primary. "use_slots": false, } @@ -329,6 +333,13 @@ func DynamicConfiguration( root["standby_cluster"] = standby } + if matchers := logicalreplica.IgnoreSlotsMatchers(cluster); len(matchers) > 0 { + existing, _ := root["ignore_slots"].([]any) + // Concat rather than append: root is a shallow copy of configuration, so + // appending could write into the user's own slice. + root["ignore_slots"] = slices.Concat(existing, matchers) + } + return root } diff --git a/internal/patroni/config_test.go b/internal/patroni/config_test.go index 5cfc026544..b37b005b1f 100644 --- a/internal/patroni/config_test.go +++ b/internal/patroni/config_test.go @@ -411,6 +411,38 @@ func TestDynamicConfiguration(t *testing.T) { }, }, }, + { + name: "postgresql: use_slots enabled with permanent slots", + input: map[string]any{ + "postgresql": map[string]any{ + "use_slots": true, + }, + "slots": map[string]any{ + "my_slot": map[string]any{ + "type": "logical", + "database": "mydb", + "plugin": "pgoutput", + }, + }, + }, + expected: map[string]any{ + "loop_wait": int32(10), + "ttl": int32(30), + "postgresql": map[string]any{ + "parameters": map[string]any{}, + "pg_hba": []string{}, + "use_pg_rewind": true, + "use_slots": true, + }, + "slots": map[string]any{ + "my_slot": map[string]any{ + "type": "logical", + "database": "mydb", + "plugin": "pgoutput", + }, + }, + }, + }, { name: "postgresql.parameters: wrong-type is ignored", input: map[string]any{ @@ -1037,6 +1069,62 @@ func TestDynamicConfiguration(t *testing.T) { } } +func TestDynamicConfigurationIgnoreSlots(t *testing.T) { + t.Parallel() + + newCluster := func(replicas ...string) *v1beta1.PostgresCluster { + cluster := new(v1beta1.PostgresCluster) + cluster.Spec.PostgresVersion = 17 + for _, name := range replicas { + cluster.Spec.LogicalReplicas = append(cluster.Spec.LogicalReplicas, + v1beta1.LogicalReplicaSpec{Name: name}) + } + assert.NilError(t, cluster.Default(context.Background(), nil)) + return cluster + } + + t.Run("absent without logical replicas", func(t *testing.T) { + actual := DynamicConfiguration(newCluster(), nil, postgres.HBAs{}, postgres.Parameters{}) + + _, ok := actual["ignore_slots"] + assert.Assert(t, !ok) + }) + + t.Run("added for logical replicas", func(t *testing.T) { + actual := DynamicConfiguration(newCluster("analytics"), nil, postgres.HBAs{}, postgres.Parameters{}) + + // Without this Patroni deletes the slot pg_createsubscriber leaves on + // the primary: dropping unknown slots is not gated on use_slots, which + // this operator pins to false. + assert.DeepEqual(t, actual["ignore_slots"], []any{ + map[string]any{"type": "logical", "plugin": "pgoutput"}, + }) + }) + + t.Run("user entries are preserved", func(t *testing.T) { + input := map[string]any{ + "ignore_slots": []any{ + map[string]any{"name": "mine", "type": "physical"}, + }, + } + + actual := DynamicConfiguration(newCluster("analytics"), input, postgres.HBAs{}, postgres.Parameters{}) + + assert.DeepEqual(t, actual["ignore_slots"], []any{ + map[string]any{"name": "mine", "type": "physical"}, + map[string]any{"type": "logical", "plugin": "pgoutput"}, + }) + }) + + t.Run("one matcher regardless of replica count", func(t *testing.T) { + actual := DynamicConfiguration(newCluster("a", "b", "c"), nil, postgres.HBAs{}, postgres.Parameters{}) + + ignored, ok := actual["ignore_slots"].([]any) + assert.Assert(t, ok) + assert.Equal(t, len(ignored), 1) + }) +} + func TestInstanceConfigFiles(t *testing.T) { t.Parallel() diff --git a/internal/pgbackrest/reconcile.go b/internal/pgbackrest/reconcile.go index c946aa0e3b..b8fd434741 100644 --- a/internal/pgbackrest/reconcile.go +++ b/internal/pgbackrest/reconcile.go @@ -235,17 +235,22 @@ func AddConfigToRestorePod( addConfigVolumeAndMounts(pod, append(sources, configmap, secret)) } +// ConfigVolumeMount returns the mount [AddConfigToRestorePod] adds to a Pod. +func ConfigVolumeMount() corev1.VolumeMount { + return corev1.VolumeMount{ + Name: "pgbackrest-config", + MountPath: configDirectory, + ReadOnly: true, + } +} + // addConfigVolumeAndMounts adds the config projections to pod as the // configuration volume. It mounts that volume to the database container and // all pgBackRest containers in pod. func addConfigVolumeAndMounts( pod *corev1.PodSpec, config []corev1.VolumeProjection, ) { - configVolumeMount := corev1.VolumeMount{ - Name: "pgbackrest-config", - MountPath: configDirectory, - ReadOnly: true, - } + configVolumeMount := ConfigVolumeMount() configVolume := corev1.Volume{ Name: configVolumeMount.Name, diff --git a/percona/controller/pgcluster/controller.go b/percona/controller/pgcluster/controller.go index e872ceb11d..5c6ca133c6 100644 --- a/percona/controller/pgcluster/controller.go +++ b/percona/controller/pgcluster/controller.go @@ -90,7 +90,7 @@ func (r *PGClusterReconciler) SetupWithManager(ctx context.Context, mgr manager. if err := r.CrunchyController.Watch(source.Kind(mgr.GetCache(), &corev1.Secret{}, r.watchSecrets())); err != nil { return errors.Wrap(err, "unable to watch secrets") } - if err := r.CrunchyController.Watch(source.Kind(mgr.GetCache(), &batchv1.Job{}, r.watchBackupJobs())); err != nil { + if err := r.CrunchyController.Watch(source.Kind(mgr.GetCache(), &batchv1.Job{}, r.watchJobs(naming.LabelPGBackRestRepo))); err != nil { return errors.Wrap(err, "unable to watch jobs") } if err := r.CrunchyController.Watch(source.Kind(mgr.GetCache(), &v2.PerconaPGBackup{}, r.watchPGBackups())); err != nil { @@ -112,8 +112,10 @@ func (r *PGClusterReconciler) SetupWithManager(ctx context.Context, mgr manager. Watches(&corev1.Secret{}, r.watchPGBouncerUserSecrets()). Watches(&corev1.ConfigMap{}, r.watchLogRotateExtraConfig()). WatchesRawSource(source.Kind(mgr.GetCache(), &corev1.Secret{}, r.watchSecrets())). - WatchesRawSource(source.Kind(mgr.GetCache(), &batchv1.Job{}, r.watchBackupJobs())). + WatchesRawSource(source.Kind(mgr.GetCache(), &batchv1.Job{}, r.watchJobs(naming.LabelPGBackRestRepo))). + WatchesRawSource(source.Kind(mgr.GetCache(), &batchv1.Job{}, r.watchJobs(pNaming.LabelLogicalReplica))). WatchesRawSource(source.Kind(mgr.GetCache(), &v2.PerconaPGBackup{}, r.watchPGBackups())). + WatchesRawSource(source.Kind(mgr.GetCache(), &v2.PerconaPGRestore{}, r.watchPGRestores())). WatchesRawSource(source.Channel(standbyClusterEvents, &handler.EnqueueRequestForObject{})). Complete(r) } @@ -134,14 +136,16 @@ func (r *PGClusterReconciler) watchServices() handler.TypedFuncs[*corev1.Service } } -func (r *PGClusterReconciler) watchBackupJobs() handler.TypedFuncs[*batchv1.Job, reconcile.Request] { +// watchJobs enqueues the cluster of a Job whose status changed, as long as the +// Job carries ownerLabel. Every Job the operator creates is labelled with its +// cluster, so LabelCluster alone would wake the reconciler for all of them. +func (r *PGClusterReconciler) watchJobs(ownerLabel string) handler.TypedFuncs[*batchv1.Job, reconcile.Request] { return handler.TypedFuncs[*batchv1.Job, reconcile.Request]{ UpdateFunc: func(ctx context.Context, e event.TypedUpdateEvent[*batchv1.Job], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { labels := e.ObjectNew.GetLabels() crName := labels[naming.LabelCluster] - repoName := labels[naming.LabelPGBackRestRepo] - if len(crName) != 0 && len(repoName) != 0 && + if len(crName) != 0 && len(labels[ownerLabel]) != 0 && !reflect.DeepEqual(e.ObjectNew.Status, e.ObjectOld.Status) { q.Add(reconcile.Request{NamespacedName: client.ObjectKey{ Namespace: e.ObjectNew.GetNamespace(), @@ -171,6 +175,28 @@ func (r *PGClusterReconciler) watchPGBackups() handler.TypedFuncs[*v2.PerconaPGB } } +// watchPGRestores enqueues the cluster a restore targets. +func (r *PGClusterReconciler) watchPGRestores() handler.TypedFuncs[*v2.PerconaPGRestore, reconcile.Request] { + enqueue := func(pgRestore *v2.PerconaPGRestore, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + q.Add(reconcile.Request{NamespacedName: client.ObjectKey{ + Namespace: pgRestore.GetNamespace(), + Name: pgRestore.Spec.PGCluster, + }}) + } + + return handler.TypedFuncs[*v2.PerconaPGRestore, reconcile.Request]{ + CreateFunc: func(ctx context.Context, e event.TypedCreateEvent[*v2.PerconaPGRestore], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + enqueue(e.Object, q) + }, + UpdateFunc: func(ctx context.Context, e event.TypedUpdateEvent[*v2.PerconaPGRestore], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + enqueue(e.ObjectNew, q) + }, + DeleteFunc: func(ctx context.Context, e event.TypedDeleteEvent[*v2.PerconaPGRestore], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + enqueue(e.Object, q) + }, + } +} + func (r *PGClusterReconciler) watchEnvFromSecrets() handler.TypedEventHandler[client.Object, reconcile.Request] { return handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []reconcile.Request { log := logf.FromContext(ctx).WithName("watchEnvFromSecrets") @@ -277,6 +303,12 @@ func (r *PGClusterReconciler) watchSecrets() handler.TypedFuncs[*corev1.Secret, // +kubebuilder:rbac:groups=batch,resources=jobs,verbs=create;list;update // +kubebuilder:rbac:groups="",resources="pods",verbs=create;delete // +kubebuilder:rbac:groups="",resources="persistentvolumeclaims",verbs=create;update +// +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=create;delete;get;list;patch;update;watch +// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=delete;get;watch +// +kubebuilder:rbac:groups="",resources="services",verbs=create;delete;get;list;patch;update;watch +// +kubebuilder:rbac:groups="",resources="configmaps",verbs=create;delete;get;list;patch;update;watch +// +kubebuilder:rbac:groups="",resources="persistentvolumeclaims",verbs=delete;get;list;patch;watch +// +kubebuilder:rbac:groups=pgv2.percona.com,resources=perconapgrestores,verbs=get;list;watch func (r *PGClusterReconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { log := logging.FromContext(ctx).WithValues("cluster", request.Name, "namespace", request.Namespace) @@ -465,6 +497,15 @@ func (r *PGClusterReconciler) Reconcile(ctx context.Context, request reconcile.R return reconcile.Result{}, errors.Wrap(err, "reconcile owner ref migration status") } + requeueLogicalReplicas, err := r.reconcileLogicalReplicas(ctx, cr, postgresCluster) + if err != nil { + return reconcile.Result{}, errors.Wrap(err, "reconcile logical replicas") + } + + if requeueLogicalReplicas { + return reconcile.Result{RequeueAfter: 5 * time.Second}, nil + } + return ctrl.Result{}, nil } diff --git a/percona/controller/pgcluster/logicalreplica.go b/percona/controller/pgcluster/logicalreplica.go new file mode 100644 index 0000000000..6e1d0e462a --- /dev/null +++ b/percona/controller/pgcluster/logicalreplica.go @@ -0,0 +1,1716 @@ +package pgcluster + +import ( + "context" + "fmt" + "io" + "maps" + "path" + "slices" + "strconv" + "strings" + + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + "github.com/percona/percona-postgresql-operator/v2/internal/initialize" + "github.com/percona/percona-postgresql-operator/v2/internal/logging" + "github.com/percona/percona-postgresql-operator/v2/internal/logicalreplica" + "github.com/percona/percona-postgresql-operator/v2/internal/naming" + "github.com/percona/percona-postgresql-operator/v2/internal/patroni" + "github.com/percona/percona-postgresql-operator/v2/internal/pgbackrest" + "github.com/percona/percona-postgresql-operator/v2/internal/postgres" + perconaController "github.com/percona/percona-postgresql-operator/v2/percona/controller" + "github.com/percona/percona-postgresql-operator/v2/percona/k8s" + pNaming "github.com/percona/percona-postgresql-operator/v2/percona/naming" + perconaPG "github.com/percona/percona-postgresql-operator/v2/percona/postgres" + v2 "github.com/percona/percona-postgresql-operator/v2/pkg/apis/pgv2.percona.com/v2" + "github.com/percona/percona-postgresql-operator/v2/pkg/apis/upstream.pgv2.percona.com/v1beta1" +) + +const ( + // logicalReplicaConfigMountPath is deliberately not naming.ConfigMountPath, + // which spec.config.files owns. + logicalReplicaConfigMountPath = "/etc/logical-replica" + + logicalReplicaConfigFile = "postgresql.conf" + + // logicalReplicaBootstrapConfigFile is the postgresql.conf that + // pg_createsubscriber runs the target with during the conversion. + logicalReplicaBootstrapConfigFile = "bootstrap.conf" + + logicalReplicaConfigVolume = "logical-replica-config" + logicalReplicaBootstrapContainer = "logical-replica-bootstrap" + logicalReplicaComponent = "logical-replica" + + // logicalReplicaRecoveryTimeout bounds how long pg_createsubscriber waits + // for the freshly seeded standby to replay up to the conversion LSN. + logicalReplicaRecoveryTimeout = 3600 + + // logicalReplicaPGCtlTimeout bounds how long the bootstrap Job waits for the + // converted target to start and to stop again. + logicalReplicaPGCtlTimeout = 300 + + // logicalReplicaWorkerHeadroom is added on top of one worker per database + // when sizing max_worker_processes, which must be strictly greater than the + // number of databases. + logicalReplicaWorkerHeadroom = 8 +) + +func logicalReplicaName(cr *v2.PerconaPGCluster, replica string) string { + return cr.Name + "-lr-" + replica +} + +func logicalReplicaObjectName(cr *v2.PerconaPGCluster, replica string) string { + return naming.SafeDNSUniqueName(logicalReplicaName(cr, replica)) +} + +func logicalReplicaPVCName(cr *v2.PerconaPGCluster, replica string) string { + return naming.SafeDNSUniqueName(logicalReplicaName(cr, replica) + "-pgdata") +} + +func logicalReplicaJobName(cr *v2.PerconaPGCluster, replica string) string { + return naming.SafeDNSUniqueName(logicalReplicaName(cr, replica) + "-bootstrap") +} + +func logicalReplicaConfigMapName(cr *v2.PerconaPGCluster, replica string) string { + return naming.SafeDNSUniqueName(logicalReplicaName(cr, replica) + "-config") +} + +func logicalReplicaUserSecretName(cr *v2.PerconaPGCluster) string { + return cr.Name + "-" + naming.RolePostgresUser + "-" + v2.UserLogicalReplication +} + +func logicalReplicaSelector(cr *v2.PerconaPGCluster, replica string) map[string]string { + return map[string]string{ + naming.LabelCluster: cr.Name, + pNaming.LabelLogicalReplica: replica, + } +} + +func logicalReplicaLabels(cr *v2.PerconaPGCluster, replica string) map[string]string { + return naming.WithPerconaLabels(naming.Merge( + logicalReplicaSelector(cr, replica), + map[string]string{pNaming.LabelOperatorVersion: cr.Spec.CRVersion}, + ), cr.Name, logicalReplicaComponent, cr.Spec.CRVersion) +} + +// reconcileLogicalReplicas brings every logical replica in the spec to life and +// tears down the ones that were removed from it. It reports whether any replica +// is still working towards being ready. +func (r *PGClusterReconciler) reconcileLogicalReplicas( + ctx context.Context, cr *v2.PerconaPGCluster, crunchyCR *v1beta1.PostgresCluster, +) (bool, error) { + if cr.CompareVersion("3.1.0") < 0 { + return false, nil + } + + // Fast path: no API calls for clusters that never used the feature. + if len(cr.Spec.LogicalReplicas) == 0 && len(cr.Status.LogicalReplicas) == 0 { + return false, nil + } + + // Must come first: a restore deletes the instance StatefulSets, so the state + // gate below would skip the teardown too. No requeue - both ends of a restore + // write this CR or a PerconaPGRestore, and this controller watches both. + restore, err := r.observeSourceRestore(ctx, cr) + if err != nil { + return false, errors.Wrap(err, "observe restore") + } + if restore.InFlight { + return false, r.suspendLogicalReplicas(ctx, cr, restore) + } + + // Runs even when the section is empty: that is the case where the last + // replica was just removed. + deferred, err := r.cleanupRemovedLogicalReplicas(ctx, cr) + if err != nil { + return false, errors.Wrap(err, "clean up removed logical replicas") + } + + if len(cr.Spec.LogicalReplicas) == 0 { + if len(deferred) > 0 { + // Nothing else will bring the primary back into view, so this is + // the one case where the teardown has to be polled for. + return true, r.updateLogicalReplicaStatus(ctx, cr, deferred, nil) + } + + // The condition goes with the last replica, so the next reconcile can + // take the shortcut above with nothing stale left behind. + return false, r.updateLogicalReplicaStatus(ctx, cr, nil, nil) + } + + // Everything below talks to the primary. No requeue: updateStatus writes the + // state just before this runs, so the cluster becoming ready is itself a + // status change that wakes this controller. + if cr.Status.State != v2.AppStateReady { + return len(deferred) > 0, nil + } + + log := logging.FromContext(ctx).WithName("LogicalReplication") + ctx = logging.NewContext(ctx, log) + + // Once per reconcile: cluster-wide, and it costs an exec. + readiness := r.observePrimaryReadiness(ctx, cr) + + // Replicas whose teardown could not be finished keep their status entry: + // forgetting one leaks the logical slots it left on the primary. + statuses := make([]v2.LogicalReplicaStatus, 0, len(cr.Spec.LogicalReplicas)+len(deferred)) + statuses = append(statuses, deferred...) + requeue := len(deferred) > 0 + for i := range cr.Spec.LogicalReplicas { + spec := &cr.Spec.LogicalReplicas[i] + + status, err := r.reconcileLogicalReplica(ctx, cr, crunchyCR, spec, + readiness.Status == metav1.ConditionTrue) + if err != nil { + // One broken replica must not stall the others or the rest of the + // cluster: record it and carry on. Every error path below hands back + // the status it was working on, so this is never nil. + log.Error(err, "reconcile logical replica", "logicalReplica", spec.Name) + + status.State = v2.LogicalReplicaStateBroken + status.Message = err.Error() + } + + if !logicalReplicaSettled(status) { + requeue = true + } + + statuses = append(statuses, *status) + } + + return requeue, r.updateLogicalReplicaStatus(ctx, cr, statuses, &readiness) +} + +func logicalReplicaSettled(status *v2.LogicalReplicaStatus) bool { + if status.State == v2.LogicalReplicaStateReady { + return true + } + if status.State != v2.LogicalReplicaStateBroken { + return false + } + + switch status.Reason { + case v2.LogicalReplicaReasonSourceRestored, + v2.LogicalReplicaReasonSourceSlotMissing, + v2.LogicalReplicaReasonBootstrapFailed: + // All three mean the same thing: the data on this replica cannot be + // reconciled with the primary, and only seeding it again fixes that. + return true + } + + return false +} + +// observePrimaryReadiness reports whether the primary carries everything a +// logical replica bootstrap needs, all of it written by the PostgresCluster +// controller, which reconciles independently of this one. +func (r *PGClusterReconciler) observePrimaryReadiness( + ctx context.Context, cr *v2.PerconaPGCluster, +) metav1.Condition { + // LastTransitionTime left zero on purpose: meta.SetStatusCondition only + // stamps it when Status changes, so a message that moves on its own does not + // look like a transition. + condition := metav1.Condition{ + Type: pNaming.ConditionReadyForLogicalReplication, + Status: metav1.ConditionFalse, + } + + primary, err := perconaPG.GetPrimaryPod(ctx, r.Client, cr) + if err != nil { + condition.Reason = "PrimaryPodNotFound" + condition.Message = err.Error() + return condition + } + + // The Job reads the password from this Secret through a secretKeyRef, and a + // pod stuck on a missing one says nothing about why. + secret := &corev1.Secret{} + key := client.ObjectKey{Name: logicalReplicaUserSecretName(cr), Namespace: cr.Namespace} + if err := r.Client.Get(ctx, key, secret); err != nil || len(secret.Data["password"]) == 0 { + condition.Reason = "ReplicationSecretMissing" + condition.Message = "waiting for the " + key.Name + " secret" + return condition + } + + // The exact signal handlePatroniRestarts acts on. The query below sees a + // pending restart sooner and reports it under the same reason. + if patroni.PodRequiresRestart(primary) { + condition.Reason = logicalreplica.ReasonRestartPending + condition.Message = logicalreplica.PrimaryReadinessMessage(logicalreplica.ReasonRestartPending) + return condition + } + + stdout, err := r.execOnPod(ctx, primary, "", logicalreplica.PrimaryReadinessQuery()) + if err != nil { + logging.FromContext(ctx).V(1).Info("cannot query the primary for logical replication readiness", + "error", err.Error()) + + condition.Status = metav1.ConditionUnknown + condition.Reason = "PrimaryUnreachable" + condition.Message = err.Error() + return condition + } + + if reasons := logicalreplica.ParsePrimaryReadinessReasons(stdout); len(reasons) > 0 { + messages := make([]string, 0, len(reasons)) + for _, reason := range reasons { + messages = append(messages, logicalreplica.PrimaryReadinessMessage(reason)) + } + + condition.Reason = reasons[0] + condition.Message = strings.Join(messages, "; ") + return condition + } + + condition.Status = metav1.ConditionTrue + condition.Reason = "PrimaryReady" + condition.Message = "the primary is ready for logical replica bootstrap" + return condition +} + +const logicalReplicaReseedInstructions = "remove it from spec.logicalReplicas, " + + "wait for its entry to leave status.logicalReplicas, and add it back to seed it again" + +// recordedLogicalReplicaReason reports whether the persisted status of a replica +// already carries reason, so callers can log and emit an event once rather than +// on every pass. +func recordedLogicalReplicaReason(cr *v2.PerconaPGCluster, replica, reason string) bool { + for i := range cr.Status.LogicalReplicas { + if cr.Status.LogicalReplicas[i].Name == replica { + return cr.Status.LogicalReplicas[i].Reason == reason + } + } + + return false +} + +// logicalReplicaStatusFor returns a copy of the recorded status of a replica, or +// a fresh one if it has none yet. +func logicalReplicaStatusFor(cr *v2.PerconaPGCluster, replica string) *v2.LogicalReplicaStatus { + for i := range cr.Status.LogicalReplicas { + if cr.Status.LogicalReplicas[i].Name == replica { + status := cr.Status.LogicalReplicas[i].DeepCopy() + // Reason and message are re-derived on every pass. + status.Reason = "" + status.Message = "" + return status + } + } + + return &v2.LogicalReplicaStatus{ + Name: replica, + State: v2.LogicalReplicaStateBootstrapping, + } +} + +// updateLogicalReplicaStatus writes the logical replica statuses and the +// ReadyForLogicalReplication condition in one update, so the two can never +// disagree. A nil readiness removes the condition, which is what happens when +// the last replica leaves the spec. +func (r *PGClusterReconciler) updateLogicalReplicaStatus( + ctx context.Context, cr *v2.PerconaPGCluster, + statuses []v2.LogicalReplicaStatus, readiness *metav1.Condition, +) error { + return errors.Wrap(retry.RetryOnConflict(retry.DefaultRetry, func() error { + cluster := &v2.PerconaPGCluster{} + if err := r.Client.Get(ctx, types.NamespacedName{ + Name: cr.Name, + Namespace: cr.Namespace, + }, cluster); err != nil { + return errors.Wrap(err, "get PerconaPGCluster") + } + + cluster.Status.LogicalReplicas = statuses + + if readiness == nil { + meta.RemoveStatusCondition(&cluster.Status.Conditions, + pNaming.ConditionReadyForLogicalReplication) + } else { + condition := *readiness + condition.ObservedGeneration = cluster.Generation + meta.SetStatusCondition(&cluster.Status.Conditions, condition) + } + + return r.Client.Status().Update(ctx, cluster) + }), "update logical replica status") +} + +// reconcileLogicalReplica drives a single replica through its lifecycle: +// resolve databases, seed and convert with a one-shot Job, then run it. +func (r *PGClusterReconciler) reconcileLogicalReplica( + ctx context.Context, cr *v2.PerconaPGCluster, crunchyCR *v1beta1.PostgresCluster, + spec *v2.LogicalReplicaSpec, primaryReady bool, +) (*v2.LogicalReplicaStatus, error) { + log := logging.FromContext(ctx).WithValues("logicalReplica", spec.Name) + status := logicalReplicaStatusFor(cr, spec.Name) + + // See v2.LogicalReplicaReasonSourceRestored: not recoverable in place, and + // the system identifier is unchanged, so starting it again would have it + // serve data the cluster no longer has. + if status.InvalidatedAt != nil { + if err := r.scaleLogicalReplica(ctx, cr, spec.Name, 0); err != nil { + return status, errors.Wrap(err, "stop invalidated replica") + } + + if !recordedLogicalReplicaReason(cr, spec.Name, v2.LogicalReplicaReasonSourceRestored) { + log.Info("logical replica needs to be seeded again after a restore of the cluster", + "invalidatedAt", status.InvalidatedAt) + r.Recorder.Eventf(cr, corev1.EventTypeWarning, "LogicalReplicaInvalidated", + "Logical replica %q must be seeded again: the cluster was restored in place after it was seeded", + spec.Name) + } + + status.State = v2.LogicalReplicaStateBroken + status.Reason = v2.LogicalReplicaReasonSourceRestored + status.Message = "the cluster was restored in place after this replica was seeded, so its data " + + "can no longer be reconciled with the primary; " + logicalReplicaReseedInstructions + return status, nil + } + + // A bootstrap that starts too early cannot be retried: pg_createsubscriber + // leaves the data volume unusable. Replicas already bootstrapped skip this + // deliberately, so an unrelated pending restart never stops managing the ones + // that are running. + if status.SeededAt == nil && !primaryReady { + status.State = v2.LogicalReplicaStateBootstrapping + status.Reason = v2.LogicalReplicaReasonPrimaryNotReady + status.Message = "waiting for the primary; see the " + + pNaming.ConditionReadyForLogicalReplication + " condition" + return status, nil + } + + // Resolved exactly once and then frozen: the publications, subscriptions and + // slots are all named after this list, so it must not drift when databases + // are created or dropped later. That is what makes the waits below worth + // having - a half-resolved list would be just as permanent as a complete one. + if len(status.Databases) == 0 { + // DatabaseRevision is written only once the PostgresCluster controller's + // whole create pass has succeeded. No requeue: this reconciler owns that + // CR, so the status write that sets the revision wakes it. + if crunchyCR.Status.DatabaseRevision == "" { + status.State = v2.LogicalReplicaStateBootstrapping + status.Reason = v2.LogicalReplicaReasonWaitingForDatabases + status.Message = "waiting for the operator to create the databases of the cluster" + return status, nil + } + + databases, missing, err := r.resolveLogicalReplicaDatabases(ctx, cr, spec) + if err != nil { + return status, errors.Wrap(err, "resolve databases") + } + if len(missing) > 0 { + // pg_createsubscriber cannot subscribe to a database that is not + // there, and the Job gets exactly one attempt. + status.State = v2.LogicalReplicaStateBootstrapping + status.Reason = v2.LogicalReplicaReasonWaitingForDatabases + status.Message = "waiting for these databases of spec.logicalReplicas[].databases " + + "to be created: " + strings.Join(missing, ", ") + return status, nil + } + if len(databases) == 0 { + status.State = v2.LogicalReplicaStateBootstrapping + status.Reason = v2.LogicalReplicaReasonWaitingForDatabases + status.Message = "the cluster has no databases to replicate; create one, or name the " + + "databases this replica covers in spec.logicalReplicas[].databases" + return status, nil + } + + status.Databases = databases + status.State = v2.LogicalReplicaStateBootstrapping + + // Persist the resolved list before anything acts on it. + return status, nil + } + + ready, err := r.reconcileLogicalReplicaPVC(ctx, cr, spec) + if err != nil { + return status, errors.Wrap(err, "reconcile pvc") + } + if !ready { + // See reconcileLogicalReplicaPVC: the claim of an earlier incarnation is + // still on its way out. + status.State = v2.LogicalReplicaStateBootstrapping + status.Reason = v2.LogicalReplicaReasonWaitingForDataVolume + status.Message = "waiting for the " + logicalReplicaPVCName(cr, spec.Name) + " volume to be deleted" + return status, nil + } + + // The ConfigMap has to exist before the bootstrap Job: pg_createsubscriber + // runs the target server with it, which is what keeps the inherited + // pgBackRest archive_command from firing during the conversion. + if err := r.reconcileLogicalReplicaConfigMap(ctx, cr, crunchyCR, spec, status); err != nil { + return status, errors.Wrap(err, "reconcile configmap") + } + + if status.SeededAt == nil { + bootstrapped, err := r.reconcileLogicalReplicaBootstrap(ctx, cr, crunchyCR, spec, status) + if err != nil { + return status, err + } + if !bootstrapped { + return status, nil + } + + if status.SeededAt == nil { + status.SeededAt = new(metav1.Now()) + } + log.Info("logical replica bootstrapped", "databases", status.Databases, "seededAt", status.SeededAt) + } + + // Asserts one replica, which is also what starts one again after a restore + // that was abandoned before it replaced the data directory. + if err := r.reconcileLogicalReplicaStatefulSet(ctx, cr, crunchyCR, spec); err != nil { + return status, errors.Wrap(err, "reconcile statefulset") + } + if err := r.reconcileLogicalReplicaService(ctx, cr, spec); err != nil { + return status, errors.Wrap(err, "reconcile service") + } + + return r.checkLogicalReplicaHealth(ctx, cr, spec, status) +} + +// execOnPrimary runs sql on the primary and returns its trimmed output. +func (r *PGClusterReconciler) execOnPrimary(ctx context.Context, cr *v2.PerconaPGCluster, database, sql string) (string, error) { + primary, err := perconaPG.GetPrimaryPod(ctx, r.Client, cr) + if err != nil { + return "", errors.Wrap(ErrPrimaryPodNotFound, err.Error()) + } + + return r.execOnPod(ctx, primary, database, sql) +} + +// resolveLogicalReplicaDatabases returns the databases the replica covers, and +// the ones it was told to cover that do not exist on the primary. An empty +// spec.databases means every database a user could connect to. +func (r *PGClusterReconciler) resolveLogicalReplicaDatabases( + ctx context.Context, cr *v2.PerconaPGCluster, spec *v2.LogicalReplicaSpec, +) (databases, missing []string, err error) { + primary, err := perconaPG.GetPrimaryPod(ctx, r.Client, cr) + if err != nil { + return nil, nil, errors.Wrap(ErrPrimaryPodNotFound, err.Error()) + } + + if len(spec.Databases) == 0 { + databases, err = r.queryDatabases(ctx, primary, + `datallowconn AND NOT datistemplate AND datname <> 'postgres'`) + + return databases, nil, err + } + + present, err := r.presentDatabases(ctx, primary) + if err != nil { + return nil, nil, err + } + + databases = make([]string, 0, len(spec.Databases)) + for _, db := range spec.Databases { + databases = append(databases, string(db)) + if !slices.Contains(present, string(db)) { + missing = append(missing, string(db)) + } + } + + return databases, missing, nil +} + +// checkPrimaryCapacity verifies the primary can host the replication slots and +// WAL senders this replica needs. +func (r *PGClusterReconciler) checkPrimaryCapacity( + ctx context.Context, cr *v2.PerconaPGCluster, neededSlots, neededSenders int, +) error { + const sql = `SELECT current_setting('max_replication_slots')::int - (SELECT count(*) FROM pg_catalog.pg_replication_slots), ` + + `current_setting('max_wal_senders')::int - (SELECT count(*) FROM pg_catalog.pg_stat_replication);` + + stdout, err := r.execOnPrimary(ctx, cr, "", sql) + if err != nil { + return err + } + + // One entry per column of the query above, in order. + limits := []struct { + noun, parameter string + needed int + }{ + {"replication slots", "max_replication_slots", neededSlots}, + {"WAL senders", "max_wal_senders", neededSenders}, + } + + fields := strings.Split(stdout, "|") + if len(fields) != len(limits) { + return errors.Errorf("unexpected capacity query output: %q", stdout) + } + + for i, limit := range limits { + free, err := strconv.Atoi(strings.TrimSpace(fields[i])) + if err != nil { + return errors.Wrapf(err, "parse free %s: %q", limit.noun, fields[i]) + } + if free < limit.needed { + return errors.Errorf( + "primary has %d free %s but %d are needed; raise %s via spec.patroni.dynamicConfiguration", + free, limit.noun, limit.needed, limit.parameter) + } + } + + return nil +} + +// logicalReplicaCapacity returns how many replication slots and WAL senders the +// primary has to have free before this replica can be bootstrapped. +func logicalReplicaCapacity(spec *v2.LogicalReplicaSpec, databases int) (slots, senders int) { + slots, senders = databases, databases + + if spec.BootstrapMethodOrDefault() == v2.LogicalReplicaBootstrapMethodPGBaseBackup { + // "--wal-method=stream" holds two WAL senders and a temporary slot at + // once. The primary drops all of it well before the per-database logical + // slots are created, so this is a floor rather than an addition. + slots, senders = max(slots, 1), max(senders, 2) + } + + return slots, senders +} + +// reconcileLogicalReplicaPVC creates or updates the data volume of a replica +// and reports whether it is usable. +func (r *PGClusterReconciler) reconcileLogicalReplicaPVC( + ctx context.Context, cr *v2.PerconaPGCluster, spec *v2.LogicalReplicaSpec, +) (bool, error) { + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: logicalReplicaPVCName(cr, spec.Name), + Namespace: cr.Namespace, + }, + } + + // A claim on its way out must not be adopted: CreateOrUpdate would update it + // and report success, and the bootstrap Job would then mount a volume that + // disappears from under it - or the old data, which the Job refuses to seed + // over. A replica removed from the spec and added straight back, or a + // cancelled bootstrap, opens that window. + existing := &corev1.PersistentVolumeClaim{} + switch err := r.Client.Get(ctx, client.ObjectKeyFromObject(pvc), existing); { + case err == nil && existing.DeletionTimestamp != nil: + return false, nil + case err != nil && !apierrors.IsNotFound(err): + return false, errors.Wrap(err, "get pvc") + } + + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, pvc, func() error { + if pvc.CreationTimestamp.IsZero() { + // The claim spec is immutable apart from resources, so it is only + // set on creation. + pvc.Spec = spec.DataVolumeClaimSpec + } else { + pvc.Spec.Resources = spec.DataVolumeClaimSpec.Resources + } + pvc.Labels = logicalReplicaLabels(cr, spec.Name) + + return controllerutil.SetControllerReference(cr, pvc, r.Client.Scheme()) + }) + + return err == nil, errors.Wrap(err, "create or update pvc") +} + +// reconcileLogicalReplicaBootstrap runs the one-shot Job that seeds the data +// volume from the primary and converts it into a logical subscriber. It reports +// whether the conversion has completed. +func (r *PGClusterReconciler) reconcileLogicalReplicaBootstrap( + ctx context.Context, cr *v2.PerconaPGCluster, crunchyCR *v1beta1.PostgresCluster, spec *v2.LogicalReplicaSpec, status *v2.LogicalReplicaStatus, +) (bool, error) { + job := &batchv1.Job{} + key := client.ObjectKey{Name: logicalReplicaJobName(cr, spec.Name), Namespace: cr.Namespace} + + err := r.Client.Get(ctx, key, job) + switch { + case err != nil && !apierrors.IsNotFound(err): + return false, errors.Wrap(err, "get bootstrap job") + + case err == nil && perconaController.JobCompleted(job): + // The Job holds the data volume open; drop it so the StatefulSet can + // take over. + return true, r.deleteLogicalReplicaJob(ctx, cr, spec.Name) + + case err == nil && perconaController.JobFailed(job): + status.State = v2.LogicalReplicaStateBroken + status.Reason = v2.LogicalReplicaReasonBootstrapFailed + status.Message = "bootstrap job failed: " + jobFailedMessage(job) + + "; inspect the " + job.Name + " job, then delete the logical replica and recreate it" + return false, nil + + case err == nil: + status.State = v2.LogicalReplicaStateBootstrapping + return false, nil + } + + // No Job, and as far as the status is concerned this replica has never been + // bootstrapped. A StatefulSet says otherwise and is the more trustworthy of + // the two: it only ever exists after a completed bootstrap, and it outlives + // the Job. Seeding again would run over a replica that is already + // replicating. + sts, err := r.logicalReplicaStatefulSet(ctx, cr, spec.Name) + if err != nil { + return false, err + } + if sts != nil { + logging.FromContext(ctx).Info( + "logical replica has a StatefulSet but no record of being bootstrapped; adopting it rather than seeding again", + "statefulSet", sts.Name) + + status.SeededAt = sts.CreationTimestamp.DeepCopy() + return true, nil + } + + // Fail fast if the primary cannot host the slots we need, rather than after + // a full base backup. + neededSlots, neededSenders := logicalReplicaCapacity(spec, len(status.Databases)) + if err := r.checkPrimaryCapacity(ctx, cr, neededSlots, neededSenders); err != nil { + status.State = v2.LogicalReplicaStateBroken + status.Message = err.Error() + // Only the user can raise these settings, so retrying with backoff would + // never make it true. + //nolint:nilerr + return false, nil + } + + job, err = r.generateLogicalReplicaBootstrapJob(ctx, cr, crunchyCR, spec, status) + if err != nil { + return false, errors.Wrap(err, "generate bootstrap job") + } + if err := r.Client.Create(ctx, job); err != nil { + return false, errors.Wrap(err, "create bootstrap job") + } + + status.State = v2.LogicalReplicaStateBootstrapping + return false, nil +} + +// deleteLogicalReplicaJob deletes the bootstrap Job of a replica. Foreground, so +// the pod is gone before the claim it holds can be deleted. +func (r *PGClusterReconciler) deleteLogicalReplicaJob( + ctx context.Context, cr *v2.PerconaPGCluster, replica string, +) error { + job := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{ + Name: logicalReplicaJobName(cr, replica), + Namespace: cr.Namespace, + }} + + err := r.Client.Delete(ctx, job, &client.DeleteOptions{ + PropagationPolicy: new(metav1.DeletePropagationForeground), + }) + + return errors.Wrap(client.IgnoreNotFound(err), "delete bootstrap job") +} + +// logicalReplicaStatefulSet returns the StatefulSet of a replica, or nil when +// there is none. One only ever exists after a completed bootstrap. +func (r *PGClusterReconciler) logicalReplicaStatefulSet( + ctx context.Context, cr *v2.PerconaPGCluster, replica string, +) (*appsv1.StatefulSet, error) { + sts := &appsv1.StatefulSet{} + key := client.ObjectKey{Name: logicalReplicaObjectName(cr, replica), Namespace: cr.Namespace} + + switch err := r.Client.Get(ctx, key, sts); { + case apierrors.IsNotFound(err): + return nil, nil + case err != nil: + return nil, errors.Wrap(err, "get statefulset") + } + + return sts, nil +} + +// jobFailedMessage returns the message Kubernetes recorded on the JobFailed +// condition, which says whether the Job hit its backoff limit or its deadline. +func jobFailedMessage(job *batchv1.Job) string { + for i := range job.Status.Conditions { + if job.Status.Conditions[i].Type == batchv1.JobFailed { + return job.Status.Conditions[i].Message + } + } + + return "" +} + +// logicalReplicaBootstrapScript is the body of the bootstrap Job: seed a +// physical standby with logicalReplicaSeedCommand, then convert it. +// +// It never starts Postgres before the conversion. Finishing recovery here, the +// way the in-place restore Job does, ends recovery and promotes, and +// pg_createsubscriber refuses a target that is no longer a standby. +func logicalReplicaBootstrapScript( + dataDir, primaryHost string, port int32, databases []string, replica, seed string, +) string { + args := []string{ + " pg_createsubscriber", + "--verbose", + "--pgdata=" + shellQuote(dataDir), + "--publisher-server=\"${PUBLISHER_CONNINFO}\"", + // Where the bootstrap config below tells the target to put its socket. + "--socketdir=" + shellQuote(postgres.SocketDirectory), + "--recovery-timeout=" + strconv.Itoa(logicalReplicaRecoveryTimeout), + // Otherwise the target runs with the postgresql.conf the seed brought + // back from the primary, which carries archive_mode=on and the pgBackRest + // archive_command: the conversion promotes before it resets the system + // identifier, so the target would archive WAL into the source cluster's + // stanza on a diverged timeline. It is also how the target gets the slot + // and worker settings the conversion checks for. + "--config-file=" + shellQuote(logicalReplicaConfigMountPath+"/"+logicalReplicaBootstrapConfigFile), + } + + for _, db := range databases { + args = append(args, + "--database="+shellQuote(db), + "--publication="+logicalreplica.PublicationName(replica, db), + "--subscription="+logicalreplica.SubscriptionName(replica, db), + "--replication-slot="+logicalreplica.SlotName(replica, db), + ) + } + args = append(args, `"$@"`) + + return strings.Join([]string{ + `set -euo pipefail`, + ``, + // Carries no credential, and must not grow one: pg_createsubscriber + // stores it verbatim in pg_subscription.subconninfo. The password comes + // from PGPASSWORD instead - see logicalReplicaEnvironment. + `PUBLISHER_CONNINFO="host=` + primaryHost + ` port=` + strconv.Itoa(int(port)) + + ` user=` + logicalreplica.ReplicationUser + ` dbname=postgres sslmode=verify-ca sslrootcert=` + + naming.CertMountPath + `/ca.crt"`, + ``, + `createsubscriber() {`, + strings.Join(args, " \\\n "), + `}`, + ``, + `if [ -s ` + shellQuote(path.Join(dataDir, "PG_VERSION")) + ` ]; then`, + ` echo "data directory is not empty, refusing to seed over it" >&2`, + ` exit 1`, + `fi`, + ``, + `install --directory --mode=0700 ` + shellQuote(dataDir), + ``, + // Postgres creates its socket but not the directory holding it, and the + // Job mounts an empty volume over /tmp. + `install --directory --mode=0700 ` + shellQuote(postgres.SocketDirectory), + ``, + seed, + ``, + // On the archive alone the target can only replay as far as the last + // segment the primary happened to push, so it would sit out the whole + // --recovery-timeout waiting for the one holding the conversion LSN. + // Streaming closes that gap; a pg_basebackup seed has no restore_command + // at all, so it is the only way that one ever catches up. + // + // Postgres reads postgresql.auto.conf whatever --config-file says, so this + // survives into the conversion. + `printf >> ` + shellQuote(path.Join(dataDir, "postgresql.auto.conf")) + + ` "primary_conninfo = '%s'\n" "${PUBLISHER_CONNINFO}"`, + ``, + // Ignore any Patroni settings present in the backup: this replica is + // configured from the operator-rendered ConfigMap instead. + `rm -f ` + shellQuote(path.Join(dataDir, "patroni.dynamic.json")), + ``, + `echo "validating prerequisites"`, + `createsubscriber --dry-run`, + ``, + `echo "converting to a logical replica"`, + `createsubscriber`, + ``, + `echo "disabling the new subscriptions on their first apply error"`, + logicalReplicaDisableOnErrorCommand(dataDir, replica, databases), + ``, + `echo "done"`, + }, "\n") +} + +// logicalReplicaDisableOnErrorCommand returns the shell command that sets +// "disable_on_error" on every subscription the conversion has just created. +func logicalReplicaDisableOnErrorCommand(dataDir, replica string, databases []string) string { + options := []string{ + // Same reason pg_createsubscriber is given this file above. + "-c config_file=" + logicalReplicaConfigMountPath + "/" + logicalReplicaBootstrapConfigFile, + // The Job's pod carries the labels the replica's Service selects on, so + // the config file's "listen_addresses = *" would put a half-finished + // replica behind that Service for as long as this takes. + "-c listen_addresses=''", + // An apply worker started here would apply without the setting these + // statements are here to make. + "-c max_logical_replication_workers=0", + } + + timeout := " --timeout=" + strconv.Itoa(logicalReplicaPGCtlTimeout) + + lines := []string{ + `pg_ctl start --wait` + timeout + ` --pgdata=` + shellQuote(dataDir) + + ` --options="` + strings.Join(options, " ") + `"`, + } + + // A subscription only exists in the database it replicates, so each one takes + // its own connection, over the local socket that pg_createsubscriber used too. + for _, db := range databases { + lines = append(lines, `psql --no-psqlrc --set=ON_ERROR_STOP=on`+ + ` --dbname=`+shellQuote(db)+ + ` --command=`+shellQuote(logicalreplica.DisableOnErrorSQL(replica, db))) + } + + // Fast: nothing else is connected, and this leaves the data directory as + // cleanly shut down as the conversion found it, so the replica's postmaster + // starts without recovery. + return strings.Join(append(lines, + `pg_ctl stop --wait`+timeout+` --mode=fast --pgdata=`+shellQuote(dataDir)), "\n") +} + +// logicalReplicaSeedCommand returns the shell command that fills the data +// directory with a physical copy of the primary. Whichever method produces it +// has to leave a directory that is still a standby: pg_createsubscriber converts +// nothing else. +func logicalReplicaSeedCommand( + crunchyCR *v1beta1.PostgresCluster, spec *v2.LogicalReplicaSpec, dataDir string, +) (string, error) { + switch method := spec.BootstrapMethodOrDefault(); method { + case v2.LogicalReplicaBootstrapMethodPGBaseBackup: + return logicalReplicaBaseBackupCommand(crunchyCR, dataDir), nil + + case v2.LogicalReplicaBootstrapMethodPGBackRest: + return logicalReplicaRestoreCommand(crunchyCR, dataDir) + + default: + return "", errors.Errorf("unknown logical replica bootstrap method %q", method) + } +} + +// logicalReplicaRestoreCommand seeds the volume from the cluster's own backups. +func logicalReplicaRestoreCommand(crunchyCR *v1beta1.PostgresCluster, dataDir string) (string, error) { + opts, err := logicalReplicaRestoreOptions(crunchyCR, dataDir) + if err != nil { + return "", err + } + + return `echo "restoring the data directory from pgBackRest"` + "\n" + + "pgbackrest restore " + strings.Join(opts, " "), nil +} + +// logicalReplicaBaseBackupCommand seeds the volume straight from the primary, +// which is the only way to do it when the cluster keeps no backups. +func logicalReplicaBaseBackupCommand(crunchyCR *v1beta1.PostgresCluster, dataDir string) string { + // Patroni is told the same thing when it creates a physical replica: plain + // pg_basebackup does not understand what pg_tde encrypts. + command := "pg_basebackup" + if crunchyCR.Spec.Extensions.PGTDE.Enabled { + command = "pg_tde_basebackup" + } + + args := []string{ + command, + // pg_basebackup drops dbname and forces replication=true, so this is + // matched by the primary's hostssl replication rule. + `--dbname="${PUBLISHER_CONNINFO}"`, + "--pgdata=" + shellQuote(dataDir), + // Ships the WAL written while the backup runs. Unlike a pgBackRest restore + // this leaves no restore_command behind. + "--wal-method=stream", + // Otherwise the backup waits out a spread checkpoint before copying a + // single byte. + "--checkpoint=fast", + "--no-password", + // The only diagnostic for a seed that takes hours. + "--verbose", + "--progress", + } + + // No --write-recovery-conf: it derives primary_conninfo from libpq, which has + // resolved PGPASSWORD by then, so it would write the password into + // postgresql.auto.conf in plain text. The script appends its own conninfo. + // + // No --slot: without one "--wal-method=stream" uses a temporary slot that the + // primary drops with the connection, whereas a named slot would survive a + // failed Job and pin WAL for good. + return `echo "streaming a base backup from the primary"` + "\n" + + strings.Join(args, " \\\n ") + "\n\n" + + // What "pgbackrest restore --type=standby" writes on the other path. + `touch ` + shellQuote(path.Join(dataDir, "standby.signal")) +} + +// logicalReplicaRestoreOptions returns the pgBackRest options that seed a +// logical replica's volume from the cluster's own backups. +func logicalReplicaRestoreOptions(crunchyCR *v1beta1.PostgresCluster, dataDir string) ([]string, error) { + repos := crunchyCR.Spec.Backups.PGBackRest.Repos + if len(repos) == 0 { + return nil, errors.New("cluster has no pgBackRest repository to restore from; " + + "configure spec.backups.pgbackrest.repos or set " + + "spec.logicalReplicas[].bootstrapMethod to pg_basebackup") + } + + opts := []string{ + "--stanza=" + pgbackrest.DefaultStanzaName, + "--pg1-path=" + dataDir, + "--repo=" + strings.TrimPrefix(repos[0].Name, "repo"), + + // pg_createsubscriber converts a target that is still in recovery, and + // this is what writes standby.signal. + "--type=standby", + "--log-level-file=off", + "--log-level-console=info", + "--log-level-stderr=info", + } + + // A logical replica keeps its WAL on the data volume, so pg_wal has to be + // remapped when the instance the backup came from kept its own elsewhere. + for i := range crunchyCR.Spec.InstanceSets { + if crunchyCR.Spec.InstanceSets[i].WALVolumeClaimSpec != nil { + opts = append(opts, "--link-map=pg_wal="+path.Join(dataDir, "pg_wal")) + break + } + } + + return opts, nil +} + +// shellQuote wraps s in single quotes for safe interpolation into the bootstrap +// script: database names may contain almost anything. +func shellQuote(s string) string { + return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" +} + +func (r *PGClusterReconciler) generateLogicalReplicaBootstrapJob( + ctx context.Context, cr *v2.PerconaPGCluster, crunchyCR *v1beta1.PostgresCluster, spec *v2.LogicalReplicaSpec, status *v2.LogicalReplicaStatus, +) (*batchv1.Job, error) { + dataDir := postgres.DataDirectory(crunchyCR) + primaryHost := naming.ClusterPrimaryService(crunchyCR).Name + "." + cr.Namespace + ".svc" + + seed, err := logicalReplicaSeedCommand(crunchyCR, spec, dataDir) + if err != nil { + return nil, err + } + + script := logicalReplicaBootstrapScript( + dataDir, primaryHost, *cr.Spec.Port, status.Databases, spec.Name, seed) + + initImage, err := k8s.InitImage(ctx, r.Client, crunchyCR, nil) + if err != nil { + return nil, errors.Wrap(err, "get init image") + } + + container := corev1.Container{ + Name: logicalReplicaBootstrapContainer, + Image: cr.PostgresImage(), + ImagePullPolicy: cr.Spec.ImagePullPolicy, + Command: []string{"bash", "-c", script}, + Env: logicalReplicaEnvironment(cr, dataDir), + Resources: spec.Resources, + SecurityContext: initialize.RestrictedSecurityContext(true), + VolumeMounts: []corev1.VolumeMount{ + { + Name: "tmp", + MountPath: "/tmp", + }, + logicalReplicaCertVolumeMount(), + postgres.DataVolumeMount(), + { + Name: pNaming.CrunchyBinVolumeName, + MountPath: pNaming.CrunchyBinVolumePath, + }, + logicalReplicaConfigVolumeMount(), + }, + } + + // Follows the repository, not the bootstrap method: the restore_command in the + // inherited postgresql.conf is a useful WAL fallback either way, and a + // non-optional projection with no ConfigMap leaves the Job unschedulable. + hasPGBackRestRepo := len(crunchyCR.Spec.Backups.PGBackRest.Repos) > 0 + if hasPGBackRestRepo { + container.VolumeMounts = append(container.VolumeMounts, pgbackrest.ConfigVolumeMount()) + } + + initContainer := k8s.InitContainer( + crunchyCR, + naming.ContainerDatabase, + initImage, + cr.Spec.ImagePullPolicy, + initialize.RestrictedSecurityContext(true), + container.Resources, + nil) + + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: logicalReplicaJobName(cr, spec.Name), + Namespace: cr.Namespace, + Labels: logicalReplicaLabels(cr, spec.Name), + }, + Spec: batchv1.JobSpec{ + BackoffLimit: new(int32(0)), + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: logicalReplicaLabels(cr, spec.Name), + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + InitContainers: []corev1.Container{initContainer}, + Containers: []corev1.Container{container}, + Volumes: logicalReplicaVolumes(cr, crunchyCR, spec), + SecurityContext: postgres.PodSecurityContext(crunchyCR), + ImagePullSecrets: cr.Spec.ImagePullSecrets, + Affinity: spec.Affinity, + Tolerations: spec.Tolerations, + AutomountServiceAccountToken: new(false), + EnableServiceLinks: new(false), + }, + }, + }, + } + if spec.PriorityClassName != nil { + job.Spec.Template.Spec.PriorityClassName = *spec.PriorityClassName + } + + // The Job restores from the cluster's own repository, so it is both clusters + // here. Gated with the mount above: a mount with no volume makes the Pod + // invalid. + if hasPGBackRestRepo { + pgbackrest.AddConfigToRestorePod(crunchyCR, crunchyCR, &job.Spec.Template.Spec) + } + + return job, controllerutil.SetControllerReference(cr, job, r.Client.Scheme()) +} + +func logicalReplicaCertVolumeMount() corev1.VolumeMount { + return corev1.VolumeMount{ + Name: naming.CertVolume, + MountPath: naming.CertMountPath, + ReadOnly: true, + } +} + +func logicalReplicaConfigVolumeMount() corev1.VolumeMount { + return corev1.VolumeMount{ + Name: logicalReplicaConfigVolume, + MountPath: logicalReplicaConfigMountPath, + ReadOnly: true, + } +} + +// logicalReplicaProbe returns a probe that waits for the replica's postmaster to +// accept connections on its socket. +func logicalReplicaProbe(delay, period, failureThreshold int32) *corev1.Probe { + return &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"pg_isready", "-h", postgres.SocketDirectory}, + }, + }, + InitialDelaySeconds: delay, + PeriodSeconds: period, + FailureThreshold: failureThreshold, + } +} + +// logicalReplicaEnvironment returns the environment shared by the bootstrap Job +// and the StatefulSet. +func logicalReplicaEnvironment(cr *v2.PerconaPGCluster, dataDir string) []corev1.EnvVar { + return []corev1.EnvVar{ + {Name: "PGDATA", Value: dataDir}, + {Name: "PGHOST", Value: postgres.SocketDirectory}, + {Name: "PGPORT", Value: strconv.Itoa(int(*cr.Spec.Port))}, + {Name: "PGPASSWORD", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: logicalReplicaUserSecretName(cr), + }, + Key: "password", + }, + }}, + } +} + +// logicalReplicaVolumes returns the volumes shared by the bootstrap Job and the StatefulSet. +func logicalReplicaVolumes( + cr *v2.PerconaPGCluster, crunchyCR *v1beta1.PostgresCluster, spec *v2.LogicalReplicaSpec, +) []corev1.Volume { + certSecret := naming.PostgresTLSSecret(crunchyCR).Name + if cr.Spec.Secrets.CustomTLSSecret != nil && cr.Spec.Secrets.CustomTLSSecret.Name != "" { + certSecret = cr.Spec.Secrets.CustomTLSSecret.Name + } + + return []corev1.Volume{ + { + Name: naming.CertVolume, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: certSecret, + // PostgreSQL refuses to start when the server key is + // readable by anyone else. + DefaultMode: new(int32(0o600)), + }, + }, + }, + { + Name: postgres.DataVolumeMount().Name, + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: logicalReplicaPVCName(cr, spec.Name), + }, + }, + }, + { + Name: "tmp", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{ + Medium: corev1.StorageMediumMemory, + }, + }, + }, + { + Name: pNaming.CrunchyBinVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: logicalReplicaConfigVolume, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: logicalReplicaConfigMapName(cr, spec.Name), + }, + }, + }, + }, + } +} + +// logicalReplicaPostgresConfig renders the postgresql.conf of a logical replica. +func logicalReplicaPostgresConfig(cr *v2.PerconaPGCluster, crunchyCR *v1beta1.PostgresCluster, databases int, readOnly bool) string { + inherited := path.Join(postgres.DataDirectory(crunchyCR), "postgresql.conf") + + lines := []string{ + "# Generated by percona-postgresql-operator. Do not edit.", + "", + "# The primary's own configuration, as it came back from the backup.", + "# Settings below this line override it.", + fmt.Sprintf("include_if_exists '%s'", inherited), + "", + "listen_addresses = '*'", + fmt.Sprintf("port = %d", *cr.Spec.Port), + fmt.Sprintf("unix_socket_directories = '%s'", postgres.SocketDirectory), + "", + "ssl = on", + fmt.Sprintf("ssl_cert_file = '%s/tls.crt'", naming.CertMountPath), + fmt.Sprintf("ssl_key_file = '%s/tls.key'", naming.CertMountPath), + fmt.Sprintf("ssl_ca_file = '%s/ca.crt'", naming.CertMountPath), + "", + "# Without this the replica would inherit the pgBackRest archive_command and", + "# push WAL from a diverged timeline into the source cluster's stanza.", + "archive_mode = off", + "archive_command = ''", + "", + "# One apply worker and one origin per subscribed database.", + fmt.Sprintf("max_replication_slots = %d", databases), + fmt.Sprintf("max_logical_replication_workers = %d", databases), + fmt.Sprintf("max_worker_processes = %d", databases+logicalReplicaWorkerHeadroom), + "", + "password_encryption = 'scram-sha-256'", + } + + if readOnly { + lines = append(lines, + "", + "# Writing to a logical replica diverges it from the primary, and the", + "# first conflicting row breaks apply for good. Replication itself keeps", + "# working: this only disallows SQL commands, which the apply worker", + "# bypasses.", + "default_transaction_read_only = on", + ) + } + + return strings.Join(lines, "\n") + "\n" +} + +func (r *PGClusterReconciler) reconcileLogicalReplicaConfigMap( + ctx context.Context, cr *v2.PerconaPGCluster, crunchyCR *v1beta1.PostgresCluster, spec *v2.LogicalReplicaSpec, status *v2.LogicalReplicaStatus, +) error { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: logicalReplicaConfigMapName(cr, spec.Name), + Namespace: cr.Namespace, + }, + } + + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, cm, func() error { + cm.Labels = logicalReplicaLabels(cr, spec.Name) + cm.Data = map[string]string{ + logicalReplicaConfigFile: logicalReplicaPostgresConfig(cr, crunchyCR, len(status.Databases), true /* read only */), + logicalReplicaBootstrapConfigFile: logicalReplicaPostgresConfig(cr, crunchyCR, len(status.Databases), false /* read only */), + } + + return controllerutil.SetControllerReference(cr, cm, r.Client.Scheme()) + }) + + return errors.Wrap(err, "create or update configmap") +} + +// reconcileLogicalReplicaStatefulSet renders the StatefulSet that runs the +// replica. Stopping one goes through scaleLogicalReplica instead. +func (r *PGClusterReconciler) reconcileLogicalReplicaStatefulSet( + ctx context.Context, cr *v2.PerconaPGCluster, crunchyCR *v1beta1.PostgresCluster, + spec *v2.LogicalReplicaSpec, +) error { + name := logicalReplicaObjectName(cr, spec.Name) + dataDir := postgres.DataDirectory(crunchyCR) + + sts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: cr.Namespace}, + } + + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, sts, func() error { + sts.Labels = logicalReplicaLabels(cr, spec.Name) + sts.Spec.Replicas = new(int32(1)) + sts.Spec.ServiceName = name + sts.Spec.Selector = &metav1.LabelSelector{MatchLabels: logicalReplicaSelector(cr, spec.Name)} + sts.Spec.Template.Labels = naming.Merge( + logicalReplicaLabels(cr, spec.Name), spec.Metadata.GetLabelsOrNil()) + sts.Spec.Template.Annotations = spec.Metadata.GetAnnotationsOrNil() + + container := corev1.Container{ + Name: naming.ContainerDatabase, + Image: cr.PostgresImage(), + ImagePullPolicy: cr.Spec.ImagePullPolicy, + Command: []string{ + "postgres", + "-D", dataDir, + "-c", "config_file=" + logicalReplicaConfigMountPath + "/" + logicalReplicaConfigFile, + }, + Env: logicalReplicaEnvironment(cr, dataDir), + Resources: spec.Resources, + Ports: []corev1.ContainerPort{{ + Name: naming.PortPostgreSQL, + ContainerPort: *cr.Spec.Port, + Protocol: corev1.ProtocolTCP, + }}, + SecurityContext: initialize.RestrictedSecurityContext(true), + VolumeMounts: []corev1.VolumeMount{ + logicalReplicaCertVolumeMount(), + postgres.DataVolumeMount(), + {Name: "tmp", MountPath: postgres.SocketDirectory}, + logicalReplicaConfigVolumeMount(), + }, + ReadinessProbe: logicalReplicaProbe(5, 10, 0), + LivenessProbe: logicalReplicaProbe(30, 20, 6), + } + + sts.Spec.Template.Spec.Containers = []corev1.Container{container} + sts.Spec.Template.Spec.Volumes = logicalReplicaVolumes(cr, crunchyCR, spec) + sts.Spec.Template.Spec.SecurityContext = postgres.PodSecurityContext(crunchyCR) + sts.Spec.Template.Spec.ImagePullSecrets = cr.Spec.ImagePullSecrets + sts.Spec.Template.Spec.Affinity = spec.Affinity + sts.Spec.Template.Spec.Tolerations = spec.Tolerations + sts.Spec.Template.Spec.EnableServiceLinks = new(false) + if spec.PriorityClassName != nil { + sts.Spec.Template.Spec.PriorityClassName = *spec.PriorityClassName + } + + return controllerutil.SetControllerReference(cr, sts, r.Client.Scheme()) + }) + + return errors.Wrap(err, "create or update statefulset") +} + +func (r *PGClusterReconciler) reconcileLogicalReplicaService( + ctx context.Context, cr *v2.PerconaPGCluster, spec *v2.LogicalReplicaSpec, +) error { + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: logicalReplicaObjectName(cr, spec.Name), + Namespace: cr.Namespace, + }, + } + + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, svc, func() error { + svc.Labels = logicalReplicaLabels(cr, spec.Name) + svc.Spec.Selector = logicalReplicaSelector(cr, spec.Name) + svc.Spec.Ports = []corev1.ServicePort{{ + Name: naming.PortPostgreSQL, + Port: *cr.Spec.Port, + TargetPort: intstr.FromString(naming.PortPostgreSQL), + Protocol: corev1.ProtocolTCP, + }} + + if spec.Expose != nil { + svc.Spec.Type = corev1.ServiceType(spec.Expose.Type) + svc.Spec.LoadBalancerSourceRanges = spec.Expose.LoadBalancerSourceRanges + if len(spec.Expose.Annotations) > 0 { + initialize.Annotations(svc) + maps.Copy(svc.Annotations, spec.Expose.Annotations) + } + maps.Copy(svc.Labels, spec.Expose.Labels) + } + + return controllerutil.SetControllerReference(cr, svc, r.Client.Scheme()) + }) + + return errors.Wrap(err, "create or update service") +} + +// checkLogicalReplicaHealth reports whether replication is still flowing. +func (r *PGClusterReconciler) checkLogicalReplicaHealth( + ctx context.Context, cr *v2.PerconaPGCluster, spec *v2.LogicalReplicaSpec, status *v2.LogicalReplicaStatus, +) (*v2.LogicalReplicaStatus, error) { + // Patroni's permanent-slot copying is gated behind use_slots, off by default, + // so the slots only ever live on the primary that created them: losing them is + // the expected outcome of a failover. + names := make([]string, 0, len(status.Databases)) + for _, db := range status.Databases { + names = append(names, postgres.QuoteLiteral(logicalreplica.SlotName(spec.Name, db))) + } + + sql := "SELECT count(*) FROM pg_catalog.pg_replication_slots WHERE slot_name IN (" + + strings.Join(names, ",") + ");" + + stdout, err := r.execOnPrimary(ctx, cr, "", sql) + if err != nil { + return status, errors.Wrap(err, "count replication slots") + } + + present, err := strconv.Atoi(strings.TrimSpace(stdout)) + if err != nil { + return status, errors.Wrapf(err, "parse replication slot count: %q", stdout) + } + + if present < len(status.Databases) { + status.State = v2.LogicalReplicaStateBroken + status.Reason = v2.LogicalReplicaReasonSourceSlotMissing + status.Message = fmt.Sprintf( + "%d of %d replication slots are missing on the primary, most likely because it failed over; "+ + "this replica can no longer catch up, so "+logicalReplicaReseedInstructions, + len(status.Databases)-present, len(status.Databases)) + return status, nil + } + + // A live slot only proves the subscription was set up, not that it is running: + // an apply worker that cannot connect exits and is restarted forever while the + // slot sits there. + pod, err := r.logicalReplicaPod(ctx, cr, spec.Name) + if err != nil { + status.State = v2.LogicalReplicaStateBootstrapping + status.Reason = v2.LogicalReplicaReasonPodNotFound + // The StatefulSet was only just created, or the pod is restarting. + //nolint:nilerr + return status, nil + } + + for _, db := range status.Databases { + subscription := logicalreplica.SubscriptionName(spec.Name, db) + + // pg_subscription.subenabled says it should be running, + // pg_stat_subscription.pid says it actually is. + sql := "SELECT s.subenabled, (st.pid IS NOT NULL) " + + "FROM pg_catalog.pg_subscription s " + + "LEFT JOIN pg_catalog.pg_stat_subscription st ON st.subid = s.oid AND st.relid IS NULL " + + "WHERE s.subname = " + postgres.QuoteLiteral(subscription) + ";" + + stdout, err := r.execOnPod(ctx, pod, db, sql) + if err != nil { + return status, errors.Wrapf(err, "check subscription %q", subscription) + } + + enabled, running, err := parseSubscriptionHealth(stdout) + if err != nil { + return status, errors.Wrapf(err, "check subscription %q", subscription) + } + + switch { + case !enabled: + status.State = v2.LogicalReplicaStateBroken + status.Reason = v2.LogicalReplicaReasonSubscriptionDisabled + // The bootstrap sets disable_on_error, so this is what an apply error + // looks like from here. + status.Message = fmt.Sprintf( + "subscription %q on database %q is disabled, most likely because applying a "+ + "change from the primary failed; check the logical replica's logs for the error", + subscription, db) + return status, nil + + case !running: + status.State = v2.LogicalReplicaStateBroken + status.Reason = v2.LogicalReplicaReasonApplyWorkerDown + status.Message = fmt.Sprintf( + "subscription %q on database %q is enabled but has no running apply worker; "+ + "check the logical replica's logs for why it cannot reach the primary", + subscription, db) + return status, nil + } + } + + status.State = v2.LogicalReplicaStateReady + return status, nil +} + +// parseSubscriptionHealth reads the "enabled|running" row that +// checkLogicalReplicaHealth queries. An empty result means the subscription is +// gone, which counts as neither. +func parseSubscriptionHealth(stdout string) (enabled, running bool, err error) { + row := strings.TrimSpace(stdout) + if row == "" { + return false, false, nil + } + + fields := strings.Split(row, "|") + if len(fields) != 2 { + return false, false, errors.Errorf("unexpected subscription query output: %q", stdout) + } + + return strings.TrimSpace(fields[0]) == "t", strings.TrimSpace(fields[1]) == "t", nil +} + +// cleanupRemovedLogicalReplicas tears down replicas that are no longer in the +// spec. Dropping the publications and the replication slots is not optional: an +// orphaned logical slot holds WAL on the primary forever. +// +// It returns the statuses of the replicas whose teardown could not be finished, +// which the caller has to keep: the recorded status is the only record of the +// objects that are still to be dropped. +func (r *PGClusterReconciler) cleanupRemovedLogicalReplicas( + ctx context.Context, cr *v2.PerconaPGCluster, +) ([]v2.LogicalReplicaStatus, error) { + wanted := make(map[string]struct{}, len(cr.Spec.LogicalReplicas)) + for _, replica := range cr.Spec.LogicalReplicas { + wanted[replica.Name] = struct{}{} + } + + var deferred []v2.LogicalReplicaStatus + for i := range cr.Status.LogicalReplicas { + recorded := cr.Status.LogicalReplicas[i] + if _, ok := wanted[recorded.Name]; ok { + continue + } + + err := r.deleteLogicalReplica(ctx, cr, recorded.Name, recorded.Databases) + switch { + case errors.Is(err, ErrPrimaryPodNotFound): + // Failing the reconcile over this would take the rest of the cluster + // down with it. Keep the replica on the books and try again. + logging.FromContext(ctx).Info("deferring logical replica teardown, the primary is unavailable", + "logicalReplica", recorded.Name, "error", err.Error()) + + held := *recorded.DeepCopy() + held.State = v2.LogicalReplicaStateBroken + held.Reason = v2.LogicalReplicaReasonAwaitingCleanup + held.Message = "removed from the spec, waiting for a primary to drop its " + + "replication slots and publications on" + deferred = append(deferred, held) + + case err != nil: + return deferred, errors.Wrapf(err, "delete logical replica %q", recorded.Name) + } + } + + return deferred, nil +} + +// deleteLogicalReplica removes the replication objects on the primary and then +// every Kubernetes object belonging to the replica. +func (r *PGClusterReconciler) deleteLogicalReplica( + ctx context.Context, cr *v2.PerconaPGCluster, replica string, databases []string, +) error { + log := logging.FromContext(ctx).WithValues("logicalReplica", replica) + + if err := r.dropLogicalReplicaObjects(ctx, cr, replica, databases); err != nil { + // Never delete the Kubernetes objects while slots may still be held on + // the primary: retrying beats leaking WAL retention. + return errors.Wrap(err, "drop replication objects") + } + + if err := r.deleteLogicalReplicaJob(ctx, cr, replica); err != nil { + return err + } + + name := logicalReplicaObjectName(cr, replica) + objects := []client.Object{ + &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: cr.Namespace}}, + &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: cr.Namespace}}, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: logicalReplicaConfigMapName(cr, replica), Namespace: cr.Namespace}}, + &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: logicalReplicaPVCName(cr, replica), Namespace: cr.Namespace}}, + } + + for _, object := range objects { + if err := r.Client.Delete(ctx, object); client.IgnoreNotFound(err) != nil { + return errors.Wrapf(err, "delete %T", object) + } + } + + log.Info("logical replica removed") + + return nil +} + +// dropLogicalReplicaObjects drops the subscriptions on the replica and then the +// slots and publications on the primary. +func (r *PGClusterReconciler) dropLogicalReplicaObjects( + ctx context.Context, cr *v2.PerconaPGCluster, replica string, databases []string, +) error { + log := logging.FromContext(ctx).WithValues("logicalReplica", replica) + + // A replica removed before it ever resolved its databases has nothing on the + // primary, so it must not need one to be torn down. + if len(databases) == 0 { + return nil + } + + // Drop the subscriptions first so the slots go inactive. The replica may + // already be gone, in which case they are inactive anyway. + if pod, err := r.logicalReplicaPod(ctx, cr, replica); err != nil { + log.Info("skipping subscription cleanup, replica pod is unavailable", "error", err.Error()) + } else { + for _, db := range databases { + subscription := logicalreplica.SubscriptionName(replica, db) + sql := fmt.Sprintf( + // Detaching the slot first keeps DROP SUBSCRIPTION from trying + // to reach the primary, which may already be unreachable. + "ALTER SUBSCRIPTION %q DISABLE; ALTER SUBSCRIPTION %q SET (slot_name = NONE); DROP SUBSCRIPTION %q;", + subscription, subscription, subscription) + + if _, err := r.execOnPod(ctx, pod, db, sql); err != nil { + log.Info("could not drop subscription", "subscription", subscription, "error", err.Error()) + } + } + } + + // Resolved once: the loop below runs three statements per database, and each + // lookup lists every pod in the namespace. + primary, err := perconaPG.GetPrimaryPod(ctx, r.Client, cr) + if err != nil { + return errors.Wrap(ErrPrimaryPodNotFound, err.Error()) + } + + // A database this replica covered may be gone - a point-in-time restore can + // rewind the primary past its creation, and users drop databases. Its + // publication went with it, but psql cannot connect to it at all, so the drop + // below has to be skipped rather than fail the teardown forever. + present, err := r.presentDatabases(ctx, primary) + if err != nil { + return errors.Wrap(err, "list databases") + } + + for _, db := range databases { + slot := logicalreplica.SlotName(replica, db) + publication := logicalreplica.PublicationName(replica, db) + + log.Info("dropping replication objects", "slot", slot, "publication", publication) + + terminateSQL := fmt.Sprintf("SELECT pg_terminate_backend(active_pid) FROM pg_replication_slots WHERE slot_name = %s AND active = true;", + postgres.QuoteLiteral(slot)) + if _, err := r.execOnPod(ctx, primary, "", terminateSQL); err != nil { + return errors.Wrapf(err, "terminate backend on replication slot %q", slot) + } + + sql := fmt.Sprintf( + "SELECT pg_catalog.pg_drop_replication_slot(slot_name) FROM pg_catalog.pg_replication_slots WHERE slot_name = %s;", + postgres.QuoteLiteral(slot)) + if _, err := r.execOnPod(ctx, primary, "", sql); err != nil { + return errors.Wrapf(err, "drop replication slot %q", slot) + } + + // The slot is cluster-wide and is dropped either way, which is the part + // that matters: it is what pins WAL on the primary. + if !slices.Contains(present, db) { + log.Info("skipping publication, its database is gone from the primary", + "database", db, "publication", publication) + continue + } + + if _, err := r.execOnPod(ctx, primary, db, fmt.Sprintf("DROP PUBLICATION IF EXISTS %q;", publication)); err != nil { + return errors.Wrapf(err, "drop publication %q", publication) + } + } + + return nil +} + +// queryDatabases returns the databases on pod that match where, ordered by name: +// the list is frozen for a replica's lifetime, so it has to be stable. +func (r *PGClusterReconciler) queryDatabases( + ctx context.Context, pod *corev1.Pod, where string, +) ([]string, error) { + stdout, err := r.execOnPod(ctx, pod, "", + `SELECT datname FROM pg_catalog.pg_database WHERE `+where+` ORDER BY datname;`) + if err != nil { + return nil, err + } + + var databases []string + for line := range strings.SplitSeq(stdout, "\n") { + if name := strings.TrimSpace(line); name != "" { + databases = append(databases, name) + } + } + + return databases, nil +} + +// presentDatabases returns the databases that exist on pod and can be connected +// to. +func (r *PGClusterReconciler) presentDatabases(ctx context.Context, pod *corev1.Pod) ([]string, error) { + return r.queryDatabases(ctx, pod, "datallowconn") +} + +func (r *PGClusterReconciler) logicalReplicaPod(ctx context.Context, cr *v2.PerconaPGCluster, replica string) (*corev1.Pod, error) { + pods := &corev1.PodList{} + if err := r.Client.List(ctx, pods, &client.ListOptions{ + Namespace: cr.Namespace, + LabelSelector: labels.SelectorFromSet(logicalReplicaSelector(cr, replica)), + }); err != nil { + return nil, errors.Wrap(err, "list pods") + } + + for i := range pods.Items { + pod := &pods.Items[i] + if pod.DeletionTimestamp != nil || pod.Status.Phase != corev1.PodRunning { + continue + } + + // Running is not enough: psql fails against a postmaster that is not + // accepting connections yet, and callers treat an exec failure as broken, + // which would turn a few seconds of startup into a hard error. + if slices.ContainsFunc(pod.Status.Conditions, func(c corev1.PodCondition) bool { + return c.Type == corev1.PodReady && c.Status == corev1.ConditionTrue + }) { + return pod, nil + } + } + + return nil, errors.New("no ready logical replica pod") +} + +// execOnPod runs sql inside a logical replica pod and returns its trimmed +// output. +func (r *PGClusterReconciler) execOnPod(ctx context.Context, pod *corev1.Pod, database, sql string) (string, error) { + exec := postgres.Executor(func(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer, command ...string) error { + return r.PodExec(ctx, pod.GetNamespace(), pod.GetName(), naming.ContainerDatabase, stdin, stdout, stderr, command...) + }) + + options := []string{"-t"} + if database != "" { + options = append(options, "--dbname="+database) + } + + stdout, stderr, err := exec.Exec(ctx, strings.NewReader(sql), map[string]string{ + "ON_ERROR_STOP": "on", + "QUIET": "on", + }, options) + if err != nil { + return "", errors.Wrapf(err, "execute query: stderr=%s", stderr) + } + + return strings.TrimSpace(stdout), nil +} diff --git a/percona/controller/pgcluster/logicalreplica_envtest_test.go b/percona/controller/pgcluster/logicalreplica_envtest_test.go new file mode 100644 index 0000000000..02fdc414dd --- /dev/null +++ b/percona/controller/pgcluster/logicalreplica_envtest_test.go @@ -0,0 +1,260 @@ +//go:build envtest + +package pgcluster + +import ( + "context" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/percona/percona-postgresql-operator/v2/internal/logicalreplica" + pNaming "github.com/percona/percona-postgresql-operator/v2/percona/naming" + v2 "github.com/percona/percona-postgresql-operator/v2/pkg/apis/pgv2.percona.com/v2" +) + +var _ = Describe("Logical replicas", Ordered, func() { + ctx := context.Background() + + const crName = "logical-replicas" + const ns = crName + + namespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: crName, Namespace: ns}, + } + + BeforeAll(func() { + By("Creating the Namespace to perform the tests") + Expect(k8sClient.Create(ctx, namespace)).To(Succeed()) + }) + + AfterAll(func() { + By("Deleting the Namespace to perform the tests") + _ = k8sClient.Delete(ctx, namespace) + }) + + replica := func() v2.LogicalReplicaSpec { + return v2.LogicalReplicaSpec{ + Name: "analytics", + DataVolumeClaimSpec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + } + } + + cluster := func(name string, postgresVersion int, replicas ...v2.LogicalReplicaSpec) *v2.PerconaPGCluster { + cr, err := readDefaultCR(name, ns) + Expect(err).NotTo(HaveOccurred()) + + cr.Spec.PostgresVersion = postgresVersion + cr.Spec.LogicalReplicas = replicas + + return cr + } + + It("should reject logical replicas on PostgreSQL 16", func() { + err := k8sClient.Create(ctx, cluster("lr-pg16", 16, replica())) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("spec.logicalReplicas requires spec.postgresVersion >= 17")) + }) + + It("should accept logical replicas on PostgreSQL 17", func() { + cr := cluster("lr-pg17", 17, replica()) + + Expect(k8sClient.Create(ctx, cr)).To(Succeed()) + DeferCleanup(func() { _ = k8sClient.Delete(ctx, cr) }) + + created := new(v2.PerconaPGCluster) + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(cr), created)).To(Succeed()) + Expect(created.Spec.LogicalReplicas).To(HaveLen(1)) + Expect(created.Spec.LogicalReplicas[0].Name).To(Equal("analytics")) + // Omitting databases means "replicate everything"; it must stay unset + // so the operator resolves it against the primary. + Expect(created.Spec.LogicalReplicas[0].Databases).To(BeEmpty()) + }) + + It("should accept a cluster without logical replicas on PostgreSQL 16", func() { + cr := cluster("lr-none", 16) + + Expect(k8sClient.Create(ctx, cr)).To(Succeed()) + DeferCleanup(func() { _ = k8sClient.Delete(ctx, cr) }) + }) + + It("should default the bootstrap method to pgbackrest", func() { + cr := cluster("lr-default-method", 17, replica()) + + Expect(k8sClient.Create(ctx, cr)).To(Succeed()) + DeferCleanup(func() { _ = k8sClient.Delete(ctx, cr) }) + + created := new(v2.PerconaPGCluster) + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(cr), created)).To(Succeed()) + Expect(created.Spec.LogicalReplicas[0].BootstrapMethod). + To(Equal(v2.LogicalReplicaBootstrapMethodPGBackRest)) + }) + + It("should reject a bootstrap method it does not know", func() { + bad := replica() + bad.BootstrapMethod = "pg_dump" + + err := k8sClient.Create(ctx, cluster("lr-badmethod", 17, bad)) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("bootstrapMethod")) + }) + + // Without backups the cluster has no repository, so "pgbackrest" has nothing + // to restore from and the replica would only ever report itself broken. + It("should reject pgbackrest seeding without backups", func() { + cr := cluster("lr-nobackups-pgbackrest", 17, replica()) + cr.Spec.Backups.Enabled = new(false) + + err := k8sClient.Create(ctx, cr) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring( + "bootstrapMethod must be 'pg_basebackup' when spec.backups.enabled is false")) + }) + + It("should accept pg_basebackup seeding without backups", func() { + seeded := replica() + seeded.BootstrapMethod = v2.LogicalReplicaBootstrapMethodPGBaseBackup + + cr := cluster("lr-nobackups-basebackup", 17, seeded) + cr.Spec.Backups.Enabled = new(false) + + Expect(k8sClient.Create(ctx, cr)).To(Succeed()) + DeferCleanup(func() { _ = k8sClient.Delete(ctx, cr) }) + + created := new(v2.PerconaPGCluster) + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(cr), created)).To(Succeed()) + Expect(created.Spec.LogicalReplicas[0].BootstrapMethod). + To(Equal(v2.LogicalReplicaBootstrapMethodPGBaseBackup)) + }) + + It("should reject an invalid replica name", func() { + bad := replica() + bad.Name = "Analytics_1" + + err := k8sClient.Create(ctx, cluster("lr-badname", 17, bad)) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("spec.logicalReplicas")) + }) + + // The API server validates condition reasons against a pattern, and a reason + // it rejects would fail the status update rather than the bootstrap, so every + // reason the operator can report has to be checked against a real one. + It("should accept every ReadyForLogicalReplication reason", func() { + cr := cluster("lr-conditions", 17, replica()) + Expect(k8sClient.Create(ctx, cr)).To(Succeed()) + DeferCleanup(func() { _ = k8sClient.Delete(ctx, cr) }) + + r := &PGClusterReconciler{Client: k8sClient} + statuses := []v2.LogicalReplicaStatus{{ + Name: "analytics", + State: v2.LogicalReplicaStateBootstrapping, + Reason: v2.LogicalReplicaReasonPrimaryNotReady, + Message: "waiting for the primary", + }} + + for _, reason := range []string{ + "PrimaryPodNotFound", "ReplicationSecretMissing", "PrimaryUnreachable", "PrimaryReady", + logicalreplica.ReasonPrimaryInRecovery, + logicalreplica.ReasonWALLevelNotLogical, + logicalreplica.ReasonRestartPending, + logicalreplica.ReasonReplicationRoleNotReady, + logicalreplica.ReasonReplicationHBAMissing, + } { + Expect(r.updateLogicalReplicaStatus(ctx, cr, statuses, &metav1.Condition{ + Type: pNaming.ConditionReadyForLogicalReplication, + Status: metav1.ConditionFalse, + Reason: reason, + Message: logicalreplica.PrimaryReadinessMessage(reason), + })).To(Succeed(), "reason %q", reason) + + updated := new(v2.PerconaPGCluster) + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(cr), updated)).To(Succeed()) + + cond := meta.FindStatusCondition(updated.Status.Conditions, + pNaming.ConditionReadyForLogicalReplication) + Expect(cond).NotTo(BeNil()) + Expect(cond.Reason).To(Equal(reason)) + Expect(updated.Status.LogicalReplicas).To(HaveLen(1)) + } + + By("removing the condition along with the last replica") + Expect(r.updateLogicalReplicaStatus(ctx, cr, nil, nil)).To(Succeed()) + + updated := new(v2.PerconaPGCluster) + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(cr), updated)).To(Succeed()) + Expect(meta.FindStatusCondition(updated.Status.Conditions, + pNaming.ConditionReadyForLogicalReplication)).To(BeNil()) + Expect(updated.Status.LogicalReplicas).To(BeEmpty()) + }) + + // The state and the reason go into a status the API server validates against + // the generated CRD, and the timestamps have to survive a round trip: they + // are what keeps a replica invalidated by a restore from being started again. + It("should accept every logical replica state, reason and timestamp", func() { + cr := cluster("lr-states", 17, replica()) + Expect(k8sClient.Create(ctx, cr)).To(Succeed()) + DeferCleanup(func() { _ = k8sClient.Delete(ctx, cr) }) + + r := &PGClusterReconciler{Client: k8sClient} + seeded := metav1.NewTime(metav1.Now().Rfc3339Copy().Time) + invalidated := metav1.NewTime(seeded.Add(time.Hour)) + + for _, state := range []v2.LogicalReplicaState{ + v2.LogicalReplicaStateBootstrapping, + v2.LogicalReplicaStateReady, + v2.LogicalReplicaStateBroken, + v2.LogicalReplicaStateSuspended, + } { + for _, reason := range []string{ + v2.LogicalReplicaReasonSourceSlotMissing, + v2.LogicalReplicaReasonSubscriptionDisabled, + v2.LogicalReplicaReasonApplyWorkerDown, + v2.LogicalReplicaReasonBootstrapFailed, + v2.LogicalReplicaReasonPodNotFound, + v2.LogicalReplicaReasonPrimaryNotReady, + v2.LogicalReplicaReasonSourceRestoring, + v2.LogicalReplicaReasonSourceRestored, + v2.LogicalReplicaReasonWaitingForDataVolume, + v2.LogicalReplicaReasonAwaitingCleanup, + } { + Expect(r.updateLogicalReplicaStatus(ctx, cr, []v2.LogicalReplicaStatus{{ + Name: "analytics", + State: state, + Reason: reason, + Message: "message", + Databases: []string{"cluster1"}, + SeededAt: &seeded, + InvalidatedAt: &invalidated, + }}, nil)).To(Succeed(), "state %q reason %q", state, reason) + + updated := new(v2.PerconaPGCluster) + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(cr), updated)).To(Succeed()) + Expect(updated.Status.LogicalReplicas).To(HaveLen(1)) + + recorded := updated.Status.LogicalReplicas[0] + Expect(recorded.State).To(Equal(state)) + Expect(recorded.Reason).To(Equal(reason)) + Expect(recorded.SeededAt).NotTo(BeNil()) + Expect(recorded.InvalidatedAt).NotTo(BeNil()) + Expect(recorded.InvalidatedAt.After(recorded.SeededAt.Time)).To(BeTrue()) + } + } + }) +}) diff --git a/percona/controller/pgcluster/logicalreplica_restore.go b/percona/controller/pgcluster/logicalreplica_restore.go new file mode 100644 index 0000000000..ffab409e3c --- /dev/null +++ b/percona/controller/pgcluster/logicalreplica_restore.go @@ -0,0 +1,221 @@ +package pgcluster + +import ( + "context" + + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/percona/percona-postgresql-operator/v2/internal/controller/postgrescluster" + "github.com/percona/percona-postgresql-operator/v2/internal/logging" + "github.com/percona/percona-postgresql-operator/v2/internal/naming" + pNaming "github.com/percona/percona-postgresql-operator/v2/percona/naming" + v2 "github.com/percona/percona-postgresql-operator/v2/pkg/apis/pgv2.percona.com/v2" +) + +// sourceRestore is what a restore of the cluster means for its logical +// replicas. +type sourceRestore struct { + // InFlight means a restore has been asked for and has not finished, so the + // primary is about to go away or already has. + InFlight bool + + // DataReplaced means the restore has passed the point where the cluster could + // still be put back the way it was. + DataReplaced bool +} + +// observeSourceRestore reports what the restores of this cluster mean for its +// logical replicas. +// +// It never looks at the restore Job: both prepareForRestore and this operator's +// own PGBackRestRestore.Start blank status.pgbackrest.restore, so anything +// derived from it is ambiguous exactly while a restore is starting. +func (r *PGClusterReconciler) observeSourceRestore( + ctx context.Context, cr *v2.PerconaPGCluster, +) (sourceRestore, error) { + restore := sourceRestore{} + + // Set by PGBackRestRestore.Start before anything is torn down and cleared by + // DisableRestore on every terminal outcome. The earliest signal there is, and + // both edges write this CR, so the controller is woken for free. + if enabled := cr.Spec.Backups.PGBackRest.Restore; enabled != nil && + enabled.Enabled != nil && *enabled.Enabled { + restore.InFlight = true + } + if cr.GetAnnotations()[naming.PGBackRestRestore] != "" { + restore.InFlight = true + } + + // Raised by prepareForRestore as it deletes the instance runners. Nothing + // removes it when a restore fails, which is right: a half-restored data + // directory invalidates a replica just as thoroughly as a finished one. + if meta.IsStatusConditionTrue(cr.Status.Conditions, + postgrescluster.ConditionPGBackRestRestoreProgressing) { + restore.DataReplaced = true + } + + // The only signal that covers a snapshot restore with no point-in-time + // recovery: that path never calls PGBackRestRestore.Start, and a volume + // snapshot of PGDATA carries pg_replslot with it, so the health check would + // not find the slots missing either. + restores := &v2.PerconaPGRestoreList{} + if err := r.Client.List(ctx, restores, client.InNamespace(cr.Namespace)); err != nil { + return restore, errors.Wrap(err, "list restores") + } + + for i := range restores.Items { + pgRestore := &restores.Items[i] + if pgRestore.Spec.PGCluster != cr.Name || pgRestore.DeletionTimestamp != nil { + continue + } + + // A restore that has not started yet is deliberately not counted: it has + // touched nothing, and one left behind in that state would keep the + // replicas down for good. No window is missed - a pgBackRest restore sets + // the two signals above in the same pass that moves it out of this state. + switch pgRestore.Status.State { + case v2.RestoreRunning: + restore.InFlight = true + restore.DataReplaced = true + case v2.RestoreStarting: + restore.InFlight = true + default: + } + } + + return restore, nil +} + +// suspendLogicalReplicas stops every logical replica for the duration of a +// restore of the cluster they replicate. See +// [v2.LogicalReplicaReasonSourceRestored] for why they cannot keep running. +// +// Whether they can be resumed afterwards is deliberately not decided here: a +// restore that fails before it touches the data directory leaves them perfectly +// valid. Nothing is destroyed and nothing is forgotten. +func (r *PGClusterReconciler) suspendLogicalReplicas( + ctx context.Context, cr *v2.PerconaPGCluster, restore sourceRestore, +) error { + log := logging.FromContext(ctx).WithName("LogicalReplication") + + // Driven by the status rather than the spec: a replica removed from the + // spec mid-restore still has a running StatefulSet, and its status entry is + // the only record of the objects that have to be dropped on the primary + // once there is one again. + statuses := make([]v2.LogicalReplicaStatus, 0, len(cr.Status.LogicalReplicas)) + for i := range cr.Status.LogicalReplicas { + status := cr.Status.LogicalReplicas[i].DeepCopy() + status.Reason = v2.LogicalReplicaReasonSourceRestoring + + if err := r.scaleLogicalReplica(ctx, cr, status.Name, 0); err != nil { + return errors.Wrapf(err, "stop logical replica %q", status.Name) + } + + switch { + case status.SeededAt != nil: + status.State = v2.LogicalReplicaStateSuspended + status.Message = "the cluster is being restored in place" + + if restore.DataReplaced && status.InvalidatedAt == nil { + log.Info("logical replica invalidated by a restore of the cluster", + "logicalReplica", status.Name) + status.InvalidatedAt = new(metav1.Now()) + } + + default: + // The bootstrap Job has a backoff limit of zero, so one interrupted by + // the restore can never succeed, and the half-written data directory it + // leaves behind is what the next attempt refuses to seed over. Throw + // both away so the replica bootstraps from scratch. + if err := r.discardLogicalReplicaBootstrap(ctx, cr, status.Name); err != nil { + return errors.Wrapf(err, "discard bootstrap of logical replica %q", status.Name) + } + + status.State = v2.LogicalReplicaStateBootstrapping + status.Message = "the bootstrap was cancelled because the cluster is being restored in place" + status.Databases = nil + } + + statuses = append(statuses, *status) + } + + // Synthesised rather than observed: observePrimaryReadiness execs on a + // primary that a restore has taken away, and the answer is known anyway. + readiness := metav1.Condition{ + Type: pNaming.ConditionReadyForLogicalReplication, + Status: metav1.ConditionFalse, + Reason: v2.LogicalReplicaReasonSourceRestoring, + Message: "the cluster is being restored in place; logical replicas are stopped", + } + + return r.updateLogicalReplicaStatus(ctx, cr, statuses, &readiness) +} + +// scaleLogicalReplica sets the replica count of a logical replica's StatefulSet. +// Zero stops a replica while keeping the StatefulSet and the data volume, so +// nothing has to be rebuilt to start it again. +func (r *PGClusterReconciler) scaleLogicalReplica( + ctx context.Context, cr *v2.PerconaPGCluster, replica string, replicas int32, +) error { + sts := &appsv1.StatefulSet{} + key := client.ObjectKey{Name: logicalReplicaObjectName(cr, replica), Namespace: cr.Namespace} + + return errors.Wrap(retry.RetryOnConflict(retry.DefaultBackoff, func() error { + if err := r.Client.Get(ctx, key, sts); err != nil { + return client.IgnoreNotFound(err) + } + if sts.Spec.Replicas != nil && *sts.Spec.Replicas == replicas { + return nil + } + + orig := sts.DeepCopy() + sts.Spec.Replicas = &replicas + + return r.Client.Patch(ctx, sts, client.MergeFrom(orig)) + }), "scale statefulset") +} + +// discardLogicalReplicaBootstrap throws away a bootstrap that was interrupted, +// along with whatever it managed to write to the data volume. +func (r *PGClusterReconciler) discardLogicalReplicaBootstrap( + ctx context.Context, cr *v2.PerconaPGCluster, replica string, +) error { + job := &batchv1.Job{} + key := client.ObjectKey{Name: logicalReplicaJobName(cr, replica), Namespace: cr.Namespace} + switch err := r.Client.Get(ctx, key, job); { + case apierrors.IsNotFound(err): + // Nothing ever ran, so the data volume is pristine and worth keeping. + return nil + case err != nil: + return errors.Wrap(err, "get bootstrap job") + } + + if err := r.deleteLogicalReplicaJob(ctx, cr, replica); err != nil { + return err + } + + // A StatefulSet means the volume holds a replica rather than a partial copy, + // whatever the status says. Deleting the claim would destroy it. + sts, err := r.logicalReplicaStatefulSet(ctx, cr, replica) + if err != nil || sts != nil { + return err + } + + pvc := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{ + Name: logicalReplicaPVCName(cr, replica), + Namespace: cr.Namespace, + }} + if err := r.Client.Delete(ctx, pvc); client.IgnoreNotFound(err) != nil { + return errors.Wrap(err, "delete data volume") + } + + return nil +} diff --git a/percona/controller/pgcluster/logicalreplica_restore_test.go b/percona/controller/pgcluster/logicalreplica_restore_test.go new file mode 100644 index 0000000000..2c6e713d46 --- /dev/null +++ b/percona/controller/pgcluster/logicalreplica_restore_test.go @@ -0,0 +1,634 @@ +package pgcluster + +import ( + "context" + "fmt" + "io" + "slices" + "strings" + "testing" + + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/percona/percona-postgresql-operator/v2/internal/controller/postgrescluster" + "github.com/percona/percona-postgresql-operator/v2/internal/naming" + v2 "github.com/percona/percona-postgresql-operator/v2/pkg/apis/pgv2.percona.com/v2" + crunchyv1beta1 "github.com/percona/percona-postgresql-operator/v2/pkg/apis/upstream.pgv2.percona.com/v1beta1" +) + +// restoreTestCluster is a cluster that uses the logical replica feature, with no +// restore of any kind in progress. +func restoreTestCluster(t *testing.T, replicas ...v2.LogicalReplicaSpec) *v2.PerconaPGCluster { + t.Helper() + + cr, err := readDefaultCR("cluster1", "pg") + require.NoError(t, err) + cr.Default() + cr.Spec.CRVersion = "3.1.0" + cr.Spec.Port = new(int32(5432)) + cr.Spec.LogicalReplicas = replicas + cr.Status.State = v2.AppStateReady + + return cr +} + +func pgRestoreFor(cr *v2.PerconaPGCluster, name string, state v2.PGRestoreState) *v2.PerconaPGRestore { + return &v2.PerconaPGRestore{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: cr.Namespace}, + Spec: v2.PerconaPGRestoreSpec{PGCluster: cr.Name, RepoName: new("repo1")}, + Status: v2.PerconaPGRestoreStatus{State: state}, + } +} + +func TestObserveSourceRestore(t *testing.T) { + for _, tt := range []struct { + name string + mutate func(*v2.PerconaPGCluster) + restores []*v2.PerconaPGRestore + inFlight bool + dataReplaced bool + }{ + { + name: "nothing in progress", + }, + { + name: "the spec flag alone", + // Set by PGBackRestRestore.Start before anything is torn down, so + // this is what stops the replicas earliest. + mutate: func(cr *v2.PerconaPGCluster) { + cr.Spec.Backups.PGBackRest.Restore = &crunchyv1beta1.PGBackRestRestore{ + Enabled: new(true), + } + }, + inFlight: true, + }, + { + name: "a disabled spec flag is not a restore", + mutate: func(cr *v2.PerconaPGCluster) { + cr.Spec.Backups.PGBackRest.Restore = &crunchyv1beta1.PGBackRestRestore{ + Enabled: new(false), + } + }, + }, + { + name: "the crunchy annotation alone", + mutate: func(cr *v2.PerconaPGCluster) { + cr.Annotations = map[string]string{naming.PGBackRestRestore: "restore1"} + }, + inFlight: true, + }, + { + // With no restore left in flight this is a restore that failed: + // nothing removes the condition then, and that is right, because a + // half-restored data directory invalidates a replica just as + // thoroughly as a finished one. + name: "the progressing condition means the data was replaced", + mutate: func(cr *v2.PerconaPGCluster) { + cr.Status.Conditions = []metav1.Condition{{ + Type: postgrescluster.ConditionPGBackRestRestoreProgressing, + Status: metav1.ConditionTrue, + Reason: "ReadyForRestore", + }} + }, + dataReplaced: true, + }, + { + // The only signal that covers a snapshot restore with no + // point-in-time recovery, which never sets the two above. + name: "a starting restore", + restores: []*v2.PerconaPGRestore{{}}, + inFlight: true, + }, + { + name: "a running restore", + restores: []*v2.PerconaPGRestore{{}}, + inFlight: true, + dataReplaced: true, + }, + } { + t.Run(tt.name, func(t *testing.T) { + cr := restoreTestCluster(t) + if tt.mutate != nil { + tt.mutate(cr) + } + + objs := []client.Object{ + // A restore of another cluster, which must be ignored. + &v2.PerconaPGRestore{ + ObjectMeta: metav1.ObjectMeta{Name: "other", Namespace: cr.Namespace}, + Spec: v2.PerconaPGRestoreSpec{PGCluster: "cluster2", RepoName: new("repo1")}, + Status: v2.PerconaPGRestoreStatus{State: v2.RestoreRunning}, + }, + // A finished restore of this one, which must be ignored too. + pgRestoreFor(cr, "done", v2.RestoreSucceeded), + } + if len(tt.restores) > 0 { + state := v2.RestoreStarting + if tt.dataReplaced { + state = v2.RestoreRunning + } + objs = append(objs, pgRestoreFor(cr, "current", state)) + } + + cl, err := buildFakeClient(t.Context(), cr, objs...) + require.NoError(t, err) + r := &PGClusterReconciler{Client: cl} + + restore, err := r.observeSourceRestore(t.Context(), cr) + require.NoError(t, err) + + assert.Equal(t, tt.inFlight, restore.InFlight, "InFlight") + assert.Equal(t, tt.dataReplaced, restore.DataReplaced, "DataReplaced") + }) + } +} + +func TestSuspendLogicalReplicas(t *testing.T) { + spec := v2.LogicalReplicaSpec{Name: "analytics"} + + statefulSet := func(cr *v2.PerconaPGCluster) *appsv1.StatefulSet { + return &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: logicalReplicaObjectName(cr, spec.Name), + Namespace: cr.Namespace, + }, + Spec: appsv1.StatefulSetSpec{Replicas: new(int32(1))}, + } + } + + dataVolume := func(cr *v2.PerconaPGCluster) *corev1.PersistentVolumeClaim { + return &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{ + Name: logicalReplicaPVCName(cr, spec.Name), + Namespace: cr.Namespace, + }} + } + + recorded := func(t *testing.T, cl client.Client, cr *v2.PerconaPGCluster) v2.LogicalReplicaStatus { + t.Helper() + + updated := new(v2.PerconaPGCluster) + require.NoError(t, cl.Get(t.Context(), + client.ObjectKey{Name: cr.Name, Namespace: cr.Namespace}, updated)) + require.Len(t, updated.Status.LogicalReplicas, 1) + + return updated.Status.LogicalReplicas[0] + } + + t.Run("a seeded replica is stopped but kept", func(t *testing.T) { + cr := restoreTestCluster(t, spec) + cr.Status.LogicalReplicas = []v2.LogicalReplicaStatus{{ + Name: spec.Name, + State: v2.LogicalReplicaStateReady, + Databases: []string{"cluster1"}, + SeededAt: new(metav1.Now()), + }} + + cl, err := buildFakeClient(t.Context(), cr, statefulSet(cr), dataVolume(cr)) + require.NoError(t, err) + r := &PGClusterReconciler{Client: cl} + + require.NoError(t, r.suspendLogicalReplicas(t.Context(), cr, + sourceRestore{InFlight: true})) + + sts := new(appsv1.StatefulSet) + require.NoError(t, cl.Get(t.Context(), client.ObjectKeyFromObject(statefulSet(cr)), sts)) + assert.Equal(t, int32(0), *sts.Spec.Replicas) + + // Nothing is destroyed: a restore that fails before it touches the data + // directory leaves this replica perfectly valid. + require.NoError(t, cl.Get(t.Context(), client.ObjectKeyFromObject(dataVolume(cr)), + new(corev1.PersistentVolumeClaim))) + + status := recorded(t, cl, cr) + assert.Equal(t, v2.LogicalReplicaStateSuspended, status.State) + assert.Equal(t, v2.LogicalReplicaReasonSourceRestoring, status.Reason) + assert.Nil(t, status.InvalidatedAt, "the data directory has not been replaced yet") + assert.NotNil(t, status.SeededAt) + assert.Equal(t, []string{"cluster1"}, status.Databases) + }) + + t.Run("a seeded replica is invalidated once the data is replaced", func(t *testing.T) { + cr := restoreTestCluster(t, spec) + cr.Status.LogicalReplicas = []v2.LogicalReplicaStatus{{ + Name: spec.Name, + State: v2.LogicalReplicaStateReady, + Databases: []string{"cluster1"}, + SeededAt: new(metav1.Now()), + }} + + cl, err := buildFakeClient(t.Context(), cr, statefulSet(cr)) + require.NoError(t, err) + r := &PGClusterReconciler{Client: cl} + + require.NoError(t, r.suspendLogicalReplicas(t.Context(), cr, + sourceRestore{InFlight: true, DataReplaced: true})) + + assert.NotNil(t, recorded(t, cl, cr).InvalidatedAt) + }) + + t.Run("an interrupted bootstrap is thrown away", func(t *testing.T) { + // The Job has a backoff limit of zero, so one the restore interrupts can + // never succeed, and the half-written data directory it leaves is what + // the next attempt refuses to seed over. + cr := restoreTestCluster(t, spec) + cr.Status.LogicalReplicas = []v2.LogicalReplicaStatus{{ + Name: spec.Name, + State: v2.LogicalReplicaStateBootstrapping, + Databases: []string{"cluster1"}, + }} + + job := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{ + Name: logicalReplicaJobName(cr, spec.Name), + Namespace: cr.Namespace, + }} + + cl, err := buildFakeClient(t.Context(), cr, job, dataVolume(cr)) + require.NoError(t, err) + r := &PGClusterReconciler{Client: cl} + + require.NoError(t, r.suspendLogicalReplicas(t.Context(), cr, + sourceRestore{InFlight: true, DataReplaced: true})) + + err = cl.Get(t.Context(), client.ObjectKeyFromObject(job), new(batchv1.Job)) + assert.True(t, apierrors.IsNotFound(err), "the bootstrap job must be deleted: %v", err) + + err = cl.Get(t.Context(), client.ObjectKeyFromObject(dataVolume(cr)), + new(corev1.PersistentVolumeClaim)) + assert.True(t, apierrors.IsNotFound(err), "the partial data volume must be deleted: %v", err) + + status := recorded(t, cl, cr) + assert.Equal(t, v2.LogicalReplicaStateBootstrapping, status.State) + assert.Nil(t, status.SeededAt) + assert.Nil(t, status.Databases, "the frozen list is re-resolved against the restored cluster") + assert.Nil(t, status.InvalidatedAt, "an unseeded replica has no data to invalidate") + }) + + t.Run("a replica with a statefulset keeps its data volume", func(t *testing.T) { + // A StatefulSet only ever exists once a bootstrap has completed, so it + // is the more trustworthy record: the volume holds a replica rather than + // a partial copy, whatever the status says. + cr := restoreTestCluster(t, spec) + cr.Status.LogicalReplicas = []v2.LogicalReplicaStatus{{ + Name: spec.Name, + State: v2.LogicalReplicaStateBootstrapping, + }} + + job := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{ + Name: logicalReplicaJobName(cr, spec.Name), + Namespace: cr.Namespace, + }} + + cl, err := buildFakeClient(t.Context(), cr, job, statefulSet(cr), dataVolume(cr)) + require.NoError(t, err) + r := &PGClusterReconciler{Client: cl} + + require.NoError(t, r.suspendLogicalReplicas(t.Context(), cr, + sourceRestore{InFlight: true, DataReplaced: true})) + + require.NoError(t, cl.Get(t.Context(), client.ObjectKeyFromObject(dataVolume(cr)), + new(corev1.PersistentVolumeClaim))) + + sts := new(appsv1.StatefulSet) + require.NoError(t, cl.Get(t.Context(), client.ObjectKeyFromObject(statefulSet(cr)), sts)) + assert.Equal(t, int32(0), *sts.Spec.Replicas) + }) + + t.Run("a replica that never started is left alone", func(t *testing.T) { + // No Job ever ran, so the claim is pristine and worth keeping: deleting + // it would make the cluster provision storage again for no reason. + cr := restoreTestCluster(t, spec) + cr.Status.LogicalReplicas = []v2.LogicalReplicaStatus{{ + Name: spec.Name, + State: v2.LogicalReplicaStateBootstrapping, + }} + + cl, err := buildFakeClient(t.Context(), cr, dataVolume(cr)) + require.NoError(t, err) + r := &PGClusterReconciler{Client: cl} + + require.NoError(t, r.suspendLogicalReplicas(t.Context(), cr, + sourceRestore{InFlight: true, DataReplaced: true})) + + require.NoError(t, cl.Get(t.Context(), client.ObjectKeyFromObject(dataVolume(cr)), + new(corev1.PersistentVolumeClaim))) + }) + + t.Run("a replica removed from the spec keeps its status", func(t *testing.T) { + // Its status entry is the only record of the slots and publications it + // left on the primary. Dropping it here would leak them. + cr := restoreTestCluster(t) + cr.Status.LogicalReplicas = []v2.LogicalReplicaStatus{{ + Name: spec.Name, + State: v2.LogicalReplicaStateReady, + Databases: []string{"cluster1"}, + SeededAt: new(metav1.Now()), + }} + + cl, err := buildFakeClient(t.Context(), cr, statefulSet(cr)) + require.NoError(t, err) + r := &PGClusterReconciler{Client: cl} + + require.NoError(t, r.suspendLogicalReplicas(t.Context(), cr, + sourceRestore{InFlight: true})) + + assert.Equal(t, []string{"cluster1"}, recorded(t, cl, cr).Databases) + }) +} + +// TestRestoreDoesNotErrorWhenReplicaRemoved covers the reconcile that used to +// fail: the teardown needs the primary, and a restore has taken it away. +func TestRestoreDoesNotErrorWhenReplicaRemoved(t *testing.T) { + cr := restoreTestCluster(t) + cr.Spec.Backups.PGBackRest.Restore = &crunchyv1beta1.PGBackRestRestore{Enabled: new(true)} + cr.Status.State = v2.AppStateInit + cr.Status.LogicalReplicas = []v2.LogicalReplicaStatus{{ + Name: "analytics", + State: v2.LogicalReplicaStateReady, + Databases: []string{"cluster1"}, + SeededAt: new(metav1.Now()), + }} + + cl, err := buildFakeClient(t.Context(), cr) + require.NoError(t, err) + // PodExec is nil: that nothing tried to reach the primary is the assertion. + r := &PGClusterReconciler{Client: cl} + + requeue, err := r.reconcileLogicalReplicas(t.Context(), cr, new(crunchyv1beta1.PostgresCluster)) + require.NoError(t, err) + assert.False(t, requeue, "both ends of a restore wake this controller on their own") + + updated := new(v2.PerconaPGCluster) + require.NoError(t, cl.Get(t.Context(), + client.ObjectKey{Name: cr.Name, Namespace: cr.Namespace}, updated)) + require.Len(t, updated.Status.LogicalReplicas, 1, + "the teardown is deferred, not forgotten") +} + +// TestCleanupDefersWithoutPrimary covers the WAL leak: a replica whose teardown +// could not be finished used to be dropped from the status, which is the only +// record of the slots it left behind. +func TestCleanupDefersWithoutPrimary(t *testing.T) { + cr := restoreTestCluster(t) + cr.Status.LogicalReplicas = []v2.LogicalReplicaStatus{{ + Name: "analytics", + State: v2.LogicalReplicaStateReady, + Databases: []string{"cluster1"}, + SeededAt: new(metav1.Now()), + }} + + cl, err := buildFakeClient(t.Context(), cr) + require.NoError(t, err) + r := &PGClusterReconciler{Client: cl} + + requeue, err := r.reconcileLogicalReplicas(t.Context(), cr, new(crunchyv1beta1.PostgresCluster)) + require.NoError(t, err, "one unreachable primary must not fail the whole cluster reconcile") + assert.True(t, requeue, "nothing else brings the primary back into view") + + updated := new(v2.PerconaPGCluster) + require.NoError(t, cl.Get(t.Context(), + client.ObjectKey{Name: cr.Name, Namespace: cr.Namespace}, updated)) + + require.Len(t, updated.Status.LogicalReplicas, 1) + status := updated.Status.LogicalReplicas[0] + assert.Equal(t, v2.LogicalReplicaReasonAwaitingCleanup, status.Reason) + assert.Equal(t, []string{"cluster1"}, status.Databases, + "the database list is what names the slots still to be dropped") +} + +func TestInvalidatedReplicaStaysStopped(t *testing.T) { + spec := v2.LogicalReplicaSpec{Name: "analytics"} + + cr := restoreTestCluster(t, spec) + cr.Status.LogicalReplicas = []v2.LogicalReplicaStatus{{ + Name: spec.Name, + State: v2.LogicalReplicaStateSuspended, + Reason: v2.LogicalReplicaReasonSourceRestoring, + Databases: []string{"cluster1"}, + SeededAt: new(metav1.Now()), + InvalidatedAt: new(metav1.Now()), + }} + + sts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: logicalReplicaObjectName(cr, spec.Name), + Namespace: cr.Namespace, + }, + // As if something else had started it again. + Spec: appsv1.StatefulSetSpec{Replicas: new(int32(1))}, + } + + cl, err := buildFakeClient(t.Context(), cr, sts, + primaryPodForCluster(cr), logicalReplicaUserSecret(cr)) + require.NoError(t, err) + + recorder := record.NewFakeRecorder(10) + r := &PGClusterReconciler{Client: cl, Recorder: recorder} + r.PodExec = func(_ context.Context, _, _, _ string, + _ io.Reader, out, _ io.Writer, _ ...string, + ) error { + // Only the cluster-wide readiness probe may run: the replica itself must + // not be queried at all. + _, err := fmt.Fprintln(out, "") + return err + } + + requeue, err := r.reconcileLogicalReplicas(t.Context(), cr, new(crunchyv1beta1.PostgresCluster)) + require.NoError(t, err) + assert.False(t, requeue, "this replica is waiting for a person, not for the controller") + + require.NoError(t, cl.Get(t.Context(), client.ObjectKeyFromObject(sts), sts)) + assert.Equal(t, int32(0), *sts.Spec.Replicas) + + updated := new(v2.PerconaPGCluster) + require.NoError(t, cl.Get(t.Context(), + client.ObjectKey{Name: cr.Name, Namespace: cr.Namespace}, updated)) + + require.Len(t, updated.Status.LogicalReplicas, 1) + status := updated.Status.LogicalReplicas[0] + assert.Equal(t, v2.LogicalReplicaStateBroken, status.State) + assert.Equal(t, v2.LogicalReplicaReasonSourceRestored, status.Reason) + assert.Contains(t, status.Message, "remove it from spec.logicalReplicas") + assert.NotNil(t, status.InvalidatedAt) + + select { + case event := <-recorder.Events: + assert.Contains(t, event, "LogicalReplicaInvalidated") + default: + t.Error("expected an event on the pass that establishes the replica is invalid") + } +} + +// TestFailedRestoreResumesReplica covers a restore abandoned before it touched +// the data directory: the replica is still valid, so it is simply started again. +func TestFailedRestoreResumesReplica(t *testing.T) { + spec := v2.LogicalReplicaSpec{Name: "analytics"} + + cr := restoreTestCluster(t, spec) + cr.Status.LogicalReplicas = []v2.LogicalReplicaStatus{{ + Name: spec.Name, + State: v2.LogicalReplicaStateSuspended, + Reason: v2.LogicalReplicaReasonSourceRestoring, + Databases: []string{"cluster1"}, + SeededAt: new(metav1.Now()), + }} + + sts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: logicalReplicaObjectName(cr, spec.Name), + Namespace: cr.Namespace, + }, + Spec: appsv1.StatefulSetSpec{Replicas: new(int32(0))}, + } + + crunchyCR := &crunchyv1beta1.PostgresCluster{ + ObjectMeta: metav1.ObjectMeta{Name: cr.Name, Namespace: cr.Namespace}, + } + crunchyCR.Spec.PostgresVersion = 17 + + cl, err := buildFakeClient(t.Context(), cr, sts, + primaryPodForCluster(cr), logicalReplicaUserSecret(cr)) + require.NoError(t, err) + + r := &PGClusterReconciler{Client: cl} + r.PodExec = func(_ context.Context, _, _, _ string, + stdin io.Reader, out, _ io.Writer, _ ...string, + ) error { + sql, err := io.ReadAll(stdin) + require.NoError(t, err) + if strings.Contains(string(sql), "pg_hba_file_rules") { + _, err = fmt.Fprintln(out, "") + return err + } + // The health check runs, which is the point: the replica is managed + // again rather than held down. + return errors.New("FATAL: the database system is starting up") + } + + _, err = r.reconcileLogicalReplicas(t.Context(), cr, crunchyCR) + require.NoError(t, err) + + require.NoError(t, cl.Get(t.Context(), client.ObjectKeyFromObject(sts), sts)) + assert.Equal(t, int32(1), *sts.Spec.Replicas, "an abandoned restore costs the replica nothing") +} + +func TestReconcileLogicalReplicaPVCWaitsForDeletion(t *testing.T) { + spec := &v2.LogicalReplicaSpec{Name: "analytics"} + + cr := restoreTestCluster(t, *spec) + + // A finalizer is what keeps the claim around long enough to be observed + // Terminating, which is exactly what happens in the cluster: the claim + // cannot go before the pod holding it does. + pvc := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{ + Name: logicalReplicaPVCName(cr, spec.Name), + Namespace: cr.Namespace, + Finalizers: []string{"kubernetes.io/pvc-protection"}, + }} + + cl, err := buildFakeClient(t.Context(), cr, pvc) + require.NoError(t, err) + require.NoError(t, cl.Delete(t.Context(), pvc)) + + r := &PGClusterReconciler{Client: cl} + + ready, err := r.reconcileLogicalReplicaPVC(t.Context(), cr, spec) + require.NoError(t, err) + assert.False(t, ready, "a claim on its way out must not be adopted") + + crunchyCR := &crunchyv1beta1.PostgresCluster{ + ObjectMeta: metav1.ObjectMeta{Name: cr.Name, Namespace: cr.Namespace}, + } + crunchyCR.Spec.PostgresVersion = 17 + + status := &v2.LogicalReplicaStatus{ + Name: spec.Name, + Databases: []string{"cluster1"}, + } + cr.Status.LogicalReplicas = []v2.LogicalReplicaStatus{*status} + + got, err := r.reconcileLogicalReplica(t.Context(), cr, crunchyCR, spec, true) + require.NoError(t, err) + assert.Equal(t, v2.LogicalReplicaReasonWaitingForDataVolume, got.Reason) + + err = cl.Get(t.Context(), client.ObjectKey{ + Name: logicalReplicaJobName(cr, spec.Name), Namespace: cr.Namespace}, new(batchv1.Job)) + assert.True(t, apierrors.IsNotFound(err), + "no bootstrap job may be created against a dying volume: %v", err) +} + +// TestDropLogicalReplicaObjectsSkipsMissingDatabase covers a point-in-time +// restore that rewound the primary past the creation of a database the replica +// covered: psql cannot connect to it, so the publication drop has to be skipped +// rather than allowed to fail the teardown forever. +func TestDropLogicalReplicaObjectsSkipsMissingDatabase(t *testing.T) { + cr := restoreTestCluster(t) + + cl, err := buildFakeClient(t.Context(), cr, primaryPodForCluster(cr)) + require.NoError(t, err) + + statements := make([]string, 0) + r := &PGClusterReconciler{Client: cl} + r.PodExec = func(_ context.Context, _, _, _ string, + stdin io.Reader, out, _ io.Writer, command ...string, + ) error { + sql, err := io.ReadAll(stdin) + require.NoError(t, err) + statements = append(statements, string(sql)) + + if strings.Contains(string(sql), "datallowconn") { + _, err = fmt.Fprintln(out, "cluster1") + return err + } + if slices.Contains(command, "--dbname=gone") { + return errors.New(`FATAL: database "gone" does not exist`) + } + + _, err = fmt.Fprintln(out, "") + return err + } + + err = r.dropLogicalReplicaObjects(t.Context(), cr, "analytics", []string{"cluster1", "gone"}) + require.NoError(t, err) + + joined := strings.Join(statements, "\n") + // The slot is cluster-wide and is dropped either way. That is the part that + // matters: it is what pins WAL on the primary. + assert.Contains(t, joined, "pg_drop_replication_slot") + assert.Equal(t, 1, strings.Count(joined, "DROP PUBLICATION"), + "only the database that still exists may have its publication dropped") +} + +func TestLogicalReplicaSettled(t *testing.T) { + for _, tt := range []struct { + state v2.LogicalReplicaState + reason string + settled bool + }{ + {state: v2.LogicalReplicaStateReady, settled: true}, + {state: v2.LogicalReplicaStateBootstrapping}, + {state: v2.LogicalReplicaStateBootstrapping, reason: v2.LogicalReplicaReasonPodNotFound}, + {state: v2.LogicalReplicaStateSuspended, reason: v2.LogicalReplicaReasonSourceRestoring}, + {state: v2.LogicalReplicaStateBroken, reason: v2.LogicalReplicaReasonApplyWorkerDown}, + {state: v2.LogicalReplicaStateBroken, reason: v2.LogicalReplicaReasonAwaitingCleanup}, + // All three mean the replica has to be seeded again, which only a person + // can ask for, and the edit that asks wakes this controller anyway. + {state: v2.LogicalReplicaStateBroken, reason: v2.LogicalReplicaReasonSourceRestored, settled: true}, + {state: v2.LogicalReplicaStateBroken, reason: v2.LogicalReplicaReasonSourceSlotMissing, settled: true}, + {state: v2.LogicalReplicaStateBroken, reason: v2.LogicalReplicaReasonBootstrapFailed, settled: true}, + } { + t.Run(string(tt.state)+"/"+tt.reason, func(t *testing.T) { + assert.Equal(t, tt.settled, logicalReplicaSettled(&v2.LogicalReplicaStatus{ + State: tt.state, Reason: tt.reason, + })) + }) + } +} diff --git a/percona/controller/pgcluster/logicalreplica_test.go b/percona/controller/pgcluster/logicalreplica_test.go new file mode 100644 index 0000000000..06070b7b43 --- /dev/null +++ b/percona/controller/pgcluster/logicalreplica_test.go @@ -0,0 +1,1528 @@ +package pgcluster + +import ( + "context" + "errors" + "fmt" + "io" + "strconv" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/percona/percona-postgresql-operator/v2/internal/logicalreplica" + "github.com/percona/percona-postgresql-operator/v2/internal/naming" + "github.com/percona/percona-postgresql-operator/v2/internal/pgbackrest" + pNaming "github.com/percona/percona-postgresql-operator/v2/percona/naming" + v2 "github.com/percona/percona-postgresql-operator/v2/pkg/apis/pgv2.percona.com/v2" + crunchyv1beta1 "github.com/percona/percona-postgresql-operator/v2/pkg/apis/upstream.pgv2.percona.com/v1beta1" +) + +func testLogicalReplicaCluster() *v2.PerconaPGCluster { + return &v2.PerconaPGCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster1", Namespace: "pg"}, + Spec: v2.PerconaPGClusterSpec{ + CRVersion: "3.1.0", + PostgresVersion: 17, + Port: new(int32(5432)), + }, + } +} + +func TestLogicalReplicaObjectNames(t *testing.T) { + cr := testLogicalReplicaCluster() + + assert.Equal(t, "cluster1-lr-analytics", logicalReplicaObjectName(cr, "analytics")) + assert.Equal(t, "cluster1-lr-analytics-pgdata", logicalReplicaPVCName(cr, "analytics")) + assert.Equal(t, "cluster1-lr-analytics-bootstrap", logicalReplicaJobName(cr, "analytics")) + assert.Equal(t, "cluster1-lr-analytics-config", logicalReplicaConfigMapName(cr, "analytics")) +} + +func TestLogicalReplicaPostgresConfig(t *testing.T) { + cr := testLogicalReplicaCluster() + cc := &crunchyv1beta1.PostgresCluster{} + cc.Spec.PostgresVersion = 17 + + config := logicalReplicaPostgresConfig(cr, cc, 3, false /* read only */) + + // One apply worker and one origin per database, and max_worker_processes has + // to be strictly greater than the number of databases. + assert.Contains(t, config, "max_replication_slots = 3") + assert.Contains(t, config, "max_logical_replication_workers = 3") + assert.Contains(t, config, "max_worker_processes = 11") + + assert.Contains(t, config, "port = 5432") + assert.Contains(t, config, "ssl = on") + assert.Contains(t, config, "ssl_cert_file = '/pgconf/tls/tls.crt'") + assert.Contains(t, config, "unix_socket_directories = '/tmp/postgres'") + + t.Run("inherits whatever the primary's file holds", func(t *testing.T) { + // shared_preload_libraries above all: without it every database carrying + // pgaudit or pg_stat_monitor rejects statements with "must be loaded via + // shared_preload_libraries", including the DDL that pg_createsubscriber + // runs. The pg-tde key command and custom GUCs come along the same way, + // which is why the file is inherited rather than restated. + // + // It is include_if_exists because the file is only there once the + // restore has run. + include := strings.Index(config, "include_if_exists '/pgdata/pg17/postgresql.conf'") + require.NotEqual(t, -1, include, "config does not include the primary's:\n%s", config) + + assert.NotContains(t, config, "shared_preload_libraries", + "restating this would drop whatever the primary preloads") + + // Overrides only win if they come after the include. + assert.Less(t, include, strings.Index(config, "archive_mode = off")) + assert.Less(t, include, strings.Index(config, "max_replication_slots")) + }) + + t.Run("never archives", func(t *testing.T) { + // The restore brings back the primary's data directory, so without this + // the replica inherits the pgBackRest archive_command and pushes WAL + // from a diverged timeline into the source cluster's stanza. + assert.Contains(t, config, "archive_mode = off") + assert.Contains(t, config, "archive_command = ''") + }) + + t.Run("the conversion config is writable", func(t *testing.T) { + // pg_createsubscriber creates subscriptions, advances origins and drops + // publications on the target, so this one must not be read-only. + assert.NotContains(t, config, "default_transaction_read_only") + }) + + t.Run("read-only when asked", func(t *testing.T) { + assert.Contains(t, + logicalReplicaPostgresConfig(cr, cc, 3, true /* read only */), + "default_transaction_read_only = on") + }) +} + +func TestLogicalReplicaBootstrapScript(t *testing.T) { + databases := []string{"cluster1", "reporting"} + script := logicalReplicaBootstrapScript( + "/pgdata/pg17", "cluster1-primary.pg.svc", 5432, databases, "analytics", + "echo \"restoring the data directory from pgBackRest\"\n"+ + "pgbackrest restore --stanza=db --pg1-path=/pgdata/pg17 --repo=1 --type=standby") + + // commandAt locates a command by its own line, so that a mention of it in a + // comment does not count as the invocation. + commandAt := func(t *testing.T, name string) int { + t.Helper() + i := strings.Index(script, "\n"+name) + require.NotEqual(t, -1, i, "%s is never invoked", name) + return i + } + + t.Run("seeds a physical standby before converting it", func(t *testing.T) { + // pg_createsubscriber only converts an existing physical standby, and + // "--type=standby" is what makes the restored directory into one. + assert.Less(t, commandAt(t, "pgbackrest restore"), commandAt(t, "createsubscriber --dry-run")) + assert.Contains(t, script, "--type=standby") + + // pgBackRest leaves a restore_command but cannot know where the primary + // is. Without this the target replays only what has been archived, and + // the conversion waits out its whole --recovery-timeout for the segment + // holding the LSN it recovers to. + assert.Contains(t, script, "primary_conninfo") + assert.Less(t, commandAt(t, "pgbackrest restore"), strings.Index(script, "primary_conninfo")) + }) + + t.Run("never starts Postgres before the conversion", func(t *testing.T) { + // pg_createsubscriber starts the target, replays it to the LSN it reads + // from the publisher and stops it again. Finishing recovery here first + // would end recovery and promote, and the conversion refuses a target + // that is no longer a standby. + assert.NotContains(t, script, "pg_is_in_recovery") + assert.Less(t, commandAt(t, "createsubscriber\n"), commandAt(t, "pg_ctl start")) + }) + + t.Run("disables the new subscriptions on their first apply error", func(t *testing.T) { + // Set here or never: the option cannot be given to pg_createsubscriber, + // and the replica's own postmaster starts applying as soon as it comes + // up. Left unset, an apply error is retried forever behind a + // subscription that still reports itself enabled. + for _, db := range databases { + assert.Contains(t, script, + `--command='`+logicalreplica.DisableOnErrorSQL("analytics", db)+`'`) + } + + // Between a start and a matching clean stop, both of which the + // conversion has to have finished with first. + start := commandAt(t, "pg_ctl start") + stop := commandAt(t, "pg_ctl stop") + assert.Less(t, start, commandAt(t, "psql ")) + assert.Less(t, commandAt(t, "psql "), stop) + + // The target is a promoted standalone primary by now. On the config it + // inherited from the primary it would archive WAL from a diverged + // timeline into the source cluster's stanza. + assert.Contains(t, script[start:stop], + "-c config_file=/etc/logical-replica/bootstrap.conf") + + // No apply worker may run before the ALTERs above, and nothing outside + // this container may connect: the Job's pod carries the labels the + // replica's Service selects on. + assert.Contains(t, script[start:stop], "-c max_logical_replication_workers=0") + assert.Contains(t, script[start:stop], "-c listen_addresses=''") + }) + + t.Run("validates before doing anything irreversible", func(t *testing.T) { + // Once pg_createsubscriber promotes the target the data directory is + // neither a standby nor a subscriber, and the only way forward is to + // seed it again. The dry run runs the same prerequisite checks without + // promoting, so a settings or connectivity problem costs a retry + // instead of a re-seed. + dry := commandAt(t, "createsubscriber --dry-run") + real := commandAt(t, "createsubscriber\n") + + assert.Less(t, dry, real, "the dry run must come first") + + // Both go through one definition, so they cannot drift apart. + assert.Equal(t, 1, strings.Count(script, "pg_createsubscriber \\")) + assert.Contains(t, script, `"$@"`) + }) + + t.Run("refuses to overwrite an existing data directory", func(t *testing.T) { + assert.Contains(t, script, "PG_VERSION") + assert.Contains(t, script, "refusing to seed over it") + }) + + t.Run("names every replication object explicitly", func(t *testing.T) { + // --all cannot be combined with --replication-slot, and the operator has + // to own the slot names so it can drop them again later. + assert.NotContains(t, script, "--all") + + for _, db := range databases { + assert.Contains(t, script, "--database='"+db+"'") + assert.Contains(t, script, "--publication="+logicalreplica.PublicationName("analytics", db)) + assert.Contains(t, script, "--subscription="+logicalreplica.SubscriptionName("analytics", db)) + assert.Contains(t, script, "--replication-slot="+logicalreplica.SlotName("analytics", db)) + } + }) + + t.Run("connects to the primary over verified TLS", func(t *testing.T) { + assert.Contains(t, script, "user="+logicalreplica.ReplicationUser) + assert.Contains(t, script, "sslmode=verify-ca") + assert.Contains(t, script, "host=cluster1-primary.pg.svc") + }) + + t.Run("fails on the first error", func(t *testing.T) { + assert.Contains(t, script, "set -euo pipefail") + }) + + t.Run("the connection string carries no credential", func(t *testing.T) { + // pg_createsubscriber stores the publisher connection string verbatim in + // pg_subscription, where anyone who can read the catalog sees it and + // pg_dump copies it out. The password reaches libpq through PGPASSWORD, + // which logicalReplicaEnvironment puts on both the Job and the + // StatefulSet the apply worker later runs in. + assert.NotContains(t, script, "password=") + assert.NotContains(t, script, "passfile") + assert.NotContains(t, script, ".pgpass") + }) + + t.Run("the connection string is built before anything connects", func(t *testing.T) { + assert.Less(t, commandAt(t, "PUBLISHER_CONNINFO="), + commandAt(t, "createsubscriber --dry-run")) + }) + + t.Run("converts with the operator's config, not the primary's", func(t *testing.T) { + // pg_createsubscriber promotes the target before it resets the system + // identifier. Left on the primary's inherited config, the target would + // archive WAL into the source cluster's pgBackRest stanza during that + // window. + assert.Contains(t, script, "--config-file='/etc/logical-replica/bootstrap.conf'") + }) +} + +func TestLogicalReplicaRestoreOptions(t *testing.T) { + cc := &crunchyv1beta1.PostgresCluster{} + cc.Spec.PostgresVersion = 17 + + t.Run("needs a repository", func(t *testing.T) { + _, err := logicalReplicaRestoreOptions(cc, "/pgdata/pg17") + require.Error(t, err) + }) + + cc.Spec.Backups.PGBackRest.Repos = []crunchyv1beta1.PGBackRestRepo{{Name: "repo2"}} + + t.Run("restores a standby from the cluster's own repository", func(t *testing.T) { + opts, err := logicalReplicaRestoreOptions(cc, "/pgdata/pg17") + require.NoError(t, err) + + assert.Contains(t, opts, "--stanza=db") + assert.Contains(t, opts, "--pg1-path=/pgdata/pg17") + assert.Contains(t, opts, "--repo=2") + + // pg_createsubscriber converts a target that is still in recovery, so + // the restore must not be one that promotes. + assert.Contains(t, opts, "--type=standby") + + // Nothing to remap when the instances keep WAL on the data volume too. + assert.NotContains(t, strings.Join(opts, " "), "--link-map") + }) + + t.Run("brings pg_wal back inside the data directory", func(t *testing.T) { + // A logical replica has no separate WAL volume, so pg_wal is a link in + // the backup that has nowhere to point on this pod. + cc.Spec.InstanceSets = []crunchyv1beta1.PostgresInstanceSetSpec{{ + Name: "instance1", + WALVolumeClaimSpec: &corev1.PersistentVolumeClaimSpec{}, + }} + + opts, err := logicalReplicaRestoreOptions(cc, "/pgdata/pg17") + require.NoError(t, err) + + assert.Contains(t, opts, "--link-map=pg_wal=/pgdata/pg17/pg_wal") + }) +} + +func TestLogicalReplicaSeedCommand(t *testing.T) { + withRepo := func() *crunchyv1beta1.PostgresCluster { + cc := &crunchyv1beta1.PostgresCluster{} + cc.Spec.PostgresVersion = 17 + cc.Spec.Backups.PGBackRest.Repos = []crunchyv1beta1.PGBackRestRepo{{Name: "repo1"}} + return cc + } + + t.Run("defaults to pgbackrest", func(t *testing.T) { + // The field carries a CRD default, but a spec the API server has not + // defaulted must not silently pick the other method. + unset, err := logicalReplicaSeedCommand(withRepo(), &v2.LogicalReplicaSpec{}, "/pgdata/pg17") + require.NoError(t, err) + + explicit, err := logicalReplicaSeedCommand(withRepo(), &v2.LogicalReplicaSpec{ + BootstrapMethod: v2.LogicalReplicaBootstrapMethodPGBackRest, + }, "/pgdata/pg17") + require.NoError(t, err) + + assert.Equal(t, explicit, unset) + assert.Contains(t, unset, "pgbackrest restore") + }) + + t.Run("pgbackrest restores a standby", func(t *testing.T) { + seed, err := logicalReplicaSeedCommand(withRepo(), &v2.LogicalReplicaSpec{ + BootstrapMethod: v2.LogicalReplicaBootstrapMethodPGBackRest, + }, "/pgdata/pg17") + require.NoError(t, err) + + assert.Contains(t, seed, "pgbackrest restore") + assert.Contains(t, seed, "--type=standby") + assert.NotContains(t, seed, "pg_basebackup") + }) + + t.Run("pgbackrest needs a repository", func(t *testing.T) { + cc := &crunchyv1beta1.PostgresCluster{} + cc.Spec.PostgresVersion = 17 + + _, err := logicalReplicaSeedCommand(cc, &v2.LogicalReplicaSpec{ + BootstrapMethod: v2.LogicalReplicaBootstrapMethodPGBackRest, + }, "/pgdata/pg17") + require.Error(t, err) + + // The message has to name the way out, because this is the only error a + // user hits by turning backups off. + assert.Contains(t, err.Error(), "pg_basebackup") + }) + + t.Run("pg_basebackup needs no repository", func(t *testing.T) { + // The whole point of the method: a cluster with backups disabled has no + // repository at all, and used to be unable to host a logical replica. + cc := &crunchyv1beta1.PostgresCluster{} + cc.Spec.PostgresVersion = 17 + + seed, err := logicalReplicaSeedCommand(cc, &v2.LogicalReplicaSpec{ + BootstrapMethod: v2.LogicalReplicaBootstrapMethodPGBaseBackup, + }, "/pgdata/pg17") + require.NoError(t, err) + + assert.Contains(t, seed, "pg_basebackup") + assert.NotContains(t, seed, "pgbackrest") + }) + + t.Run("pg_basebackup leaves a standby behind", func(t *testing.T) { + seed, err := logicalReplicaSeedCommand(withRepo(), &v2.LogicalReplicaSpec{ + BootstrapMethod: v2.LogicalReplicaBootstrapMethodPGBaseBackup, + }, "/pgdata/pg17") + require.NoError(t, err) + + // pg_createsubscriber converts nothing but a standby, and this is what + // "pgbackrest restore --type=standby" writes on the other path. + assert.Contains(t, seed, "touch '/pgdata/pg17/standby.signal'") + + // --write-recovery-conf would write the signal file too, but it derives + // primary_conninfo from libpq, which has resolved PGPASSWORD by then. + // The password would land in postgresql.auto.conf in plain text and + // outlive the Job. + assert.NotContains(t, seed, "--write-recovery-conf") + assert.NotContains(t, seed, "-R") + }) + + t.Run("pg_basebackup reuses the one connection string", func(t *testing.T) { + seed, err := logicalReplicaSeedCommand(withRepo(), &v2.LogicalReplicaSpec{ + BootstrapMethod: v2.LogicalReplicaBootstrapMethodPGBaseBackup, + }, "/pgdata/pg17") + require.NoError(t, err) + + assert.Contains(t, seed, `--dbname="${PUBLISHER_CONNINFO}"`) + assert.Contains(t, seed, "--pgdata='/pgdata/pg17'") + + // Nothing here can prompt, and nothing can stall: there is no terminal, + // and a spread checkpoint would hold the backup for checkpoint_timeout. + assert.Contains(t, seed, "--no-password") + assert.Contains(t, seed, "--checkpoint=fast") + + // No restore_command on this path, so the WAL written during the backup + // has to ship over the second connection. + assert.Contains(t, seed, "--wal-method=stream") + }) + + t.Run("pg_basebackup leaves nothing behind on the primary", func(t *testing.T) { + seed, err := logicalReplicaSeedCommand(withRepo(), &v2.LogicalReplicaSpec{ + BootstrapMethod: v2.LogicalReplicaBootstrapMethodPGBaseBackup, + }, "/pgdata/pg17") + require.NoError(t, err) + + // Without --slot, "--wal-method=stream" uses a temporary slot the primary + // drops with the connection. A named one would survive a failed Job and + // pin WAL on the primary for good. + assert.NotContains(t, seed, "--slot") + assert.NotContains(t, seed, "--create-slot") + }) + + t.Run("pg_basebackup keeps pg_wal on the data volume", func(t *testing.T) { + // A logical replica has one volume. The server sends pg_wal as an empty + // directory even when the primary keeps it elsewhere, so this needs no + // counterpart to the pgBackRest --link-map. + cc := withRepo() + cc.Spec.InstanceSets = []crunchyv1beta1.PostgresInstanceSetSpec{{ + Name: "instance1", + WALVolumeClaimSpec: &corev1.PersistentVolumeClaimSpec{}, + }} + + seed, err := logicalReplicaSeedCommand(cc, &v2.LogicalReplicaSpec{ + BootstrapMethod: v2.LogicalReplicaBootstrapMethodPGBaseBackup, + }, "/pgdata/pg17") + require.NoError(t, err) + + assert.NotContains(t, seed, "--waldir") + assert.NotContains(t, seed, "--link-map") + }) + + t.Run("pg_tde needs its own basebackup", func(t *testing.T) { + // Plain pg_basebackup does not understand what pg_tde encrypts, which is + // why Patroni is told the same thing for creating a physical replica. + cc := withRepo() + cc.Spec.Extensions.PGTDE.Enabled = true + + seed, err := logicalReplicaSeedCommand(cc, &v2.LogicalReplicaSpec{ + BootstrapMethod: v2.LogicalReplicaBootstrapMethodPGBaseBackup, + }, "/pgdata/pg17") + require.NoError(t, err) + + assert.Contains(t, seed, "pg_tde_basebackup") + assert.NotContains(t, seed, "\npg_basebackup") + }) + + t.Run("rejects a method it does not know", func(t *testing.T) { + _, err := logicalReplicaSeedCommand(withRepo(), &v2.LogicalReplicaSpec{ + BootstrapMethod: "pg_dump", + }, "/pgdata/pg17") + + require.Error(t, err) + assert.Contains(t, err.Error(), "pg_dump") + }) +} + +func TestLogicalReplicaBootstrapScriptWithBaseBackup(t *testing.T) { + cc := &crunchyv1beta1.PostgresCluster{} + cc.Spec.PostgresVersion = 17 + + seed, err := logicalReplicaSeedCommand(cc, &v2.LogicalReplicaSpec{ + BootstrapMethod: v2.LogicalReplicaBootstrapMethodPGBaseBackup, + }, "/pgdata/pg17") + require.NoError(t, err) + + script := logicalReplicaBootstrapScript( + "/pgdata/pg17", "cluster1-primary.pg.svc", 5432, []string{"cluster1"}, "analytics", seed) + + at := func(t *testing.T, needle string) int { + t.Helper() + i := strings.Index(script, needle) + require.NotEqual(t, -1, i, "%s is missing from the script", needle) + return i + } + + t.Run("seeds, marks the standby, then converts", func(t *testing.T) { + assert.Less(t, at(t, "\npg_basebackup"), at(t, "touch '/pgdata/pg17/standby.signal'")) + assert.Less(t, at(t, "touch '/pgdata/pg17/standby.signal'"), at(t, "primary_conninfo")) + assert.Less(t, at(t, "primary_conninfo"), at(t, "\ncreatesubscriber --dry-run")) + }) + + t.Run("the connection string is set before the seed connects", func(t *testing.T) { + // pg_basebackup authenticates as the same role over the same conninfo, + // and would expand it to the empty string otherwise. + assert.Less(t, at(t, "\nPUBLISHER_CONNINFO="), at(t, "\npg_basebackup")) + }) + + t.Run("the password never reaches the data directory", func(t *testing.T) { + // primary_conninfo lands in postgresql.auto.conf, so anything the + // conninfo carries outlives the Job. pg_basebackup authenticates from + // PGPASSWORD instead. + assert.NotContains(t, script, "password=") + assert.NotContains(t, script, "passfile") + }) +} + +func TestLogicalReplicaCapacity(t *testing.T) { + basebackup := &v2.LogicalReplicaSpec{ + BootstrapMethod: v2.LogicalReplicaBootstrapMethodPGBaseBackup, + } + + t.Run("pgbackrest needs one of each per database", func(t *testing.T) { + slots, senders := logicalReplicaCapacity(&v2.LogicalReplicaSpec{}, 3) + + assert.Equal(t, 3, slots) + assert.Equal(t, 3, senders) + }) + + t.Run("pg_basebackup needs two WAL senders at once", func(t *testing.T) { + // One for the base backup, one for the WAL stream. A single-database + // replica would otherwise pass this check with one sender free and die + // mid-seed, after a full copy of the cluster. + slots, senders := logicalReplicaCapacity(basebackup, 1) + + assert.Equal(t, 1, slots) + assert.Equal(t, 2, senders) + }) + + t.Run("the seed and the subscriptions never overlap", func(t *testing.T) { + // The primary drops the temporary slot and both senders well before + // pg_createsubscriber creates the per-database logical slots, so this is + // a floor rather than something to add on top. + slots, senders := logicalReplicaCapacity(basebackup, 3) + + assert.Equal(t, 3, slots) + assert.Equal(t, 3, senders) + }) +} + +func TestParseSubscriptionHealth(t *testing.T) { + for _, tt := range []struct { + name string + stdout string + wantEnabled bool + wantRunning bool + wantErr bool + }{ + {name: "enabled and running", stdout: " t | t", wantEnabled: true, wantRunning: true}, + {name: "enabled but no worker", stdout: " t | f", wantEnabled: true}, + {name: "disabled", stdout: " f | f"}, + {name: "subscription is gone", stdout: ""}, + {name: "only whitespace", stdout: " \n "}, + {name: "unexpected shape", stdout: "t", wantErr: true}, + } { + t.Run(tt.name, func(t *testing.T) { + enabled, running, err := parseSubscriptionHealth(tt.stdout) + + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tt.wantEnabled, enabled) + assert.Equal(t, tt.wantRunning, running) + }) + } +} + +func TestShellQuote(t *testing.T) { + assert.Equal(t, `'plain'`, shellQuote("plain")) + assert.Equal(t, `'it'\''s'`, shellQuote("it's")) + assert.Equal(t, `'; rm -rf /'`, shellQuote("; rm -rf /")) +} + +func TestGenerateLogicalReplicaBootstrapJob(t *testing.T) { + ctx := context.Background() + + cr := testLogicalReplicaCluster() + cr.Spec.PostgresVersion = 17 + cr.Spec.InstanceSets = v2.PGInstanceSets{{ + Name: "instance1", + Replicas: new(int32(1)), + DataVolumeClaimSpec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + }, + }} + + crunchyCR := &crunchyv1beta1.PostgresCluster{ + ObjectMeta: metav1.ObjectMeta{Name: cr.Name, Namespace: cr.Namespace}, + } + crunchyCR.Spec.PostgresVersion = 17 + // Pinning the image keeps k8s.InitImage from having to look up the + // operator Pod, which does not exist under the fake client. + crunchyCR.Spec.InitContainer = &crunchyv1beta1.InitContainerSpec{Image: "operator:test"} + // The Job restores from the cluster's own repository. + crunchyCR.Spec.Backups.PGBackRest.Repos = []crunchyv1beta1.PGBackRestRepo{{Name: "repo1"}} + + cl, err := buildFakeClient(ctx, cr) + require.NoError(t, err) + r := &PGClusterReconciler{Client: cl} + + spec := &v2.LogicalReplicaSpec{Name: "analytics"} + status := &v2.LogicalReplicaStatus{Name: "analytics", Databases: []string{"cluster1"}} + + job, err := r.generateLogicalReplicaBootstrapJob(ctx, cr, crunchyCR, spec, status) + require.NoError(t, err) + + podSpec := job.Spec.Template.Spec + + t.Run("injects the operator init container", func(t *testing.T) { + // K8SPG-708: the same container instance pods get, so the bootstrap Job + // has the shared scripts available under /opt/crunchy/bin. + require.Len(t, podSpec.InitContainers, 1) + + init := podSpec.InitContainers[0] + assert.Equal(t, "database-init", init.Name) + assert.Equal(t, "operator:test", init.Image) + assert.Equal(t, []string{"/usr/local/bin/init-entrypoint.sh"}, init.Command) + + require.Len(t, init.VolumeMounts, 1) + assert.Equal(t, pNaming.CrunchyBinVolumeName, init.VolumeMounts[0].Name) + assert.Equal(t, pNaming.CrunchyBinVolumePath, init.VolumeMounts[0].MountPath) + }) + + t.Run("the bootstrap container can reach the scripts", func(t *testing.T) { + require.Len(t, podSpec.Containers, 1) + + var mount *corev1.VolumeMount + for i := range podSpec.Containers[0].VolumeMounts { + if podSpec.Containers[0].VolumeMounts[i].Name == pNaming.CrunchyBinVolumeName { + mount = &podSpec.Containers[0].VolumeMounts[i] + } + } + require.NotNil(t, mount, "bootstrap container does not mount the scripts volume") + assert.Equal(t, pNaming.CrunchyBinVolumePath, mount.MountPath) + }) + + t.Run("the shared volume exists on the pod", func(t *testing.T) { + var volume *corev1.Volume + for i := range podSpec.Volumes { + if podSpec.Volumes[i].Name == pNaming.CrunchyBinVolumeName { + volume = &podSpec.Volumes[i] + } + } + require.NotNil(t, volume, "scripts volume is missing from the pod") + assert.NotNil(t, volume.EmptyDir) + }) + + t.Run("never retries", func(t *testing.T) { + // A half-seeded data directory cannot be re-used: the script refuses to + // run over it, so a retry would only burn time. + require.NotNil(t, job.Spec.BackoffLimit) + assert.Equal(t, int32(0), *job.Spec.BackoffLimit) + assert.Equal(t, corev1.RestartPolicyNever, podSpec.RestartPolicy) + }) + + t.Run("carries the replication password", func(t *testing.T) { + require.Len(t, podSpec.Containers, 1) + + // The conninfo the script builds holds no credential of its own, so + // pg_createsubscriber and pg_basebackup reach the primary on this alone. + assertLogicalReplicaPassword(t, cr, podSpec.Containers[0].Env) + }) +} + +// assertLogicalReplicaPassword checks that env resolves PGPASSWORD from the +// logicalrepl user Secret. Both the bootstrap Job and the replica's StatefulSet +// depend on it: see logicalReplicaEnvironment. +func assertLogicalReplicaPassword(t *testing.T, cr *v2.PerconaPGCluster, env []corev1.EnvVar) { + t.Helper() + + var found *corev1.EnvVar + for i := range env { + if env[i].Name == "PGPASSWORD" { + found = &env[i] + } + } + require.NotNil(t, found, "PGPASSWORD is missing") + + require.NotNil(t, found.ValueFrom) + require.NotNil(t, found.ValueFrom.SecretKeyRef) + assert.Equal(t, logicalReplicaUserSecretName(cr), found.ValueFrom.SecretKeyRef.Name) + assert.Equal(t, "password", found.ValueFrom.SecretKeyRef.Key) + + // Never the literal: it would be readable in the Pod spec by anyone who can + // list Pods, whether or not they can read the Secret. + assert.Empty(t, found.Value) +} + +func TestReconcileLogicalReplicaStatefulSet(t *testing.T) { + ctx := context.Background() + + cr := testLogicalReplicaCluster() + crunchyCR := &crunchyv1beta1.PostgresCluster{ + ObjectMeta: metav1.ObjectMeta{Name: cr.Name, Namespace: cr.Namespace}, + } + crunchyCR.Spec.PostgresVersion = 17 + + cl, err := buildFakeClient(ctx, cr) + require.NoError(t, err) + r := &PGClusterReconciler{Client: cl} + + spec := &v2.LogicalReplicaSpec{Name: "analytics"} + require.NoError(t, r.reconcileLogicalReplicaStatefulSet(ctx, cr, crunchyCR, spec)) + + sts := &appsv1.StatefulSet{} + require.NoError(t, cl.Get(ctx, client.ObjectKey{ + Name: logicalReplicaObjectName(cr, spec.Name), Namespace: cr.Namespace, + }, sts)) + + require.Len(t, sts.Spec.Template.Spec.Containers, 1) + + // The load-bearing one. pg_createsubscriber stored a conninfo with no + // credential in pg_subscription, and the apply worker runs in this + // postmaster and resolves the password out of its environment. Without this + // the replica bootstraps and then never replicates. + assertLogicalReplicaPassword(t, cr, sts.Spec.Template.Spec.Containers[0].Env) +} + +func TestGenerateLogicalReplicaBootstrapJobPGBackRestConfig(t *testing.T) { + ctx := context.Background() + + // hasPGBackRestConfig reports whether the Job both projects the pgBackRest + // configuration and mounts it. The two travel together: a mount with no + // volume makes the Pod invalid, and a volume with no mount hands the + // repository credentials to a container with no use for them. + hasPGBackRestConfig := func(t *testing.T, job *batchv1.Job) (volume, mount bool) { + t.Helper() + name := pgbackrest.ConfigVolumeMount().Name + + podSpec := job.Spec.Template.Spec + for i := range podSpec.Volumes { + if podSpec.Volumes[i].Name == name { + volume = true + } + } + require.Len(t, podSpec.Containers, 1) + for _, m := range podSpec.Containers[0].VolumeMounts { + if m.Name == name { + mount = true + } + } + return volume, mount + } + + generate := func(t *testing.T, repos int, method v2.LogicalReplicaBootstrapMethod) *batchv1.Job { + t.Helper() + + cr := testLogicalReplicaCluster() + cr.Spec.PostgresVersion = 17 + + crunchyCR := &crunchyv1beta1.PostgresCluster{ + ObjectMeta: metav1.ObjectMeta{Name: cr.Name, Namespace: cr.Namespace}, + } + crunchyCR.Spec.PostgresVersion = 17 + crunchyCR.Spec.InitContainer = &crunchyv1beta1.InitContainerSpec{Image: "operator:test"} + for i := range repos { + crunchyCR.Spec.Backups.PGBackRest.Repos = append( + crunchyCR.Spec.Backups.PGBackRest.Repos, + crunchyv1beta1.PGBackRestRepo{Name: "repo" + strconv.Itoa(i+1)}) + } + + cl, err := buildFakeClient(ctx, cr) + require.NoError(t, err) + r := &PGClusterReconciler{Client: cl} + + job, err := r.generateLogicalReplicaBootstrapJob(ctx, cr, crunchyCR, + &v2.LogicalReplicaSpec{Name: "analytics", BootstrapMethod: method}, + &v2.LogicalReplicaStatus{Name: "analytics", Databases: []string{"cluster1"}}) + require.NoError(t, err) + + return job + } + + t.Run("projected when the cluster has a repository", func(t *testing.T) { + volume, mount := hasPGBackRestConfig(t, generate(t, 1, v2.LogicalReplicaBootstrapMethodPGBackRest)) + + assert.True(t, volume) + assert.True(t, mount) + }) + + t.Run("projected for pg_basebackup too", func(t *testing.T) { + // This follows the repository rather than the method: the seed brings the + // primary's postgresql.conf along, and the pgBackRest restore_command in + // it is a useful fallback for WAL the primary has already recycled. + volume, mount := hasPGBackRestConfig(t, generate(t, 1, v2.LogicalReplicaBootstrapMethodPGBaseBackup)) + + assert.True(t, volume) + assert.True(t, mount) + }) + + t.Run("absent when the cluster keeps no backups", func(t *testing.T) { + // The ConfigMap does not exist, and the projection is not optional, so + // leaving it in would hang the Pod unschedulable rather than fail it. + volume, mount := hasPGBackRestConfig(t, generate(t, 0, v2.LogicalReplicaBootstrapMethodPGBaseBackup)) + + assert.False(t, volume) + assert.False(t, mount) + }) +} + +func TestLogicalReplicaStatusFor(t *testing.T) { + cr := testLogicalReplicaCluster() + cr.Status.LogicalReplicas = []v2.LogicalReplicaStatus{{ + Name: "analytics", + State: v2.LogicalReplicaStateBroken, + Reason: v2.LogicalReplicaReasonSourceSlotMissing, + Message: "stale", + Databases: []string{"cluster1"}, + }} + + t.Run("keeps the resolved databases and clears the verdict", func(t *testing.T) { + // The database list is frozen once resolved, because the slot, + // publication and subscription names all derive from it. + status := logicalReplicaStatusFor(cr, "analytics") + + assert.Equal(t, []string{"cluster1"}, status.Databases) + assert.Empty(t, status.Reason) + assert.Empty(t, status.Message) + }) + + t.Run("does not alias the recorded status", func(t *testing.T) { + status := logicalReplicaStatusFor(cr, "analytics") + status.Databases[0] = "mutated" + + assert.Equal(t, "cluster1", cr.Status.LogicalReplicas[0].Databases[0]) + }) + + t.Run("fresh status for an unknown replica", func(t *testing.T) { + status := logicalReplicaStatusFor(cr, "reporting") + + assert.Equal(t, "reporting", status.Name) + assert.Equal(t, v2.LogicalReplicaStateBootstrapping, status.State) + assert.Empty(t, status.Databases) + }) +} + +// logicalReplicaUserSecret is the secret the bootstrap Job reads its password +// from, which observePrimaryReadiness waits for. +func logicalReplicaUserSecret(cr *v2.PerconaPGCluster) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: logicalReplicaUserSecretName(cr), + Namespace: cr.Namespace, + }, + Data: map[string][]byte{"password": []byte("secret")}, + } +} + +func TestObservePrimaryReadiness(t *testing.T) { + newCluster := func() *v2.PerconaPGCluster { + cr, err := readDefaultCR("cluster1", "pg") + require.NoError(t, err) + cr.Default() + cr.Spec.CRVersion = "3.1.0" + cr.Status.State = v2.AppStateReady + return cr + } + + // Records whether the primary was queried at all: the cheap checks must + // short-circuit before paying for an exec. + type execRecorder struct { + called bool + query string + } + + newReconciler := func(t *testing.T, cr *v2.PerconaPGCluster, rec *execRecorder, + stdout string, execErr error, objs ...client.Object, + ) *PGClusterReconciler { + cl, err := buildFakeClient(t.Context(), cr, objs...) + require.NoError(t, err) + + return &PGClusterReconciler{ + Client: cl, + PodExec: func(_ context.Context, _, _, _ string, + stdin io.Reader, out, _ io.Writer, _ ...string, + ) error { + rec.called = true + if stdin != nil { + sql, err := io.ReadAll(stdin) + require.NoError(t, err) + rec.query = string(sql) + } + if execErr != nil { + return execErr + } + _, err := fmt.Fprintln(out, stdout) + return err + }, + } + } + + t.Run("no primary pod", func(t *testing.T) { + cr := newCluster() + rec := new(execRecorder) + r := newReconciler(t, cr, rec, "", nil, logicalReplicaUserSecret(cr)) + + cond := r.observePrimaryReadiness(t.Context(), cr) + + assert.Equal(t, metav1.ConditionFalse, cond.Status) + assert.Equal(t, "PrimaryPodNotFound", cond.Reason) + assert.False(t, rec.called, "must not exec without a primary") + }) + + t.Run("replication secret missing", func(t *testing.T) { + cr := newCluster() + rec := new(execRecorder) + r := newReconciler(t, cr, rec, "", nil, primaryPodForCluster(cr)) + + cond := r.observePrimaryReadiness(t.Context(), cr) + + assert.Equal(t, metav1.ConditionFalse, cond.Status) + assert.Equal(t, "ReplicationSecretMissing", cond.Reason) + assert.Contains(t, cond.Message, logicalReplicaUserSecretName(cr)) + assert.False(t, rec.called, "must not exec before the secret exists") + }) + + t.Run("secret present but empty", func(t *testing.T) { + cr := newCluster() + secret := logicalReplicaUserSecret(cr) + secret.Data = nil + rec := new(execRecorder) + r := newReconciler(t, cr, rec, "", nil, primaryPodForCluster(cr), secret) + + cond := r.observePrimaryReadiness(t.Context(), cr) + + assert.Equal(t, "ReplicationSecretMissing", cond.Reason) + assert.False(t, rec.called) + }) + + t.Run("patroni reports a pending restart", func(t *testing.T) { + cr := newCluster() + pod := primaryPodForCluster(cr) + // The annotation Patroni writes, and the signal handlePatroniRestarts + // acts on to bounce the primary. + pod.Annotations = map[string]string{"status": `{"role":"primary","pending_restart":true}`} + + rec := new(execRecorder) + r := newReconciler(t, cr, rec, "", nil, pod, logicalReplicaUserSecret(cr)) + + cond := r.observePrimaryReadiness(t.Context(), cr) + + assert.Equal(t, metav1.ConditionFalse, cond.Status) + assert.Equal(t, logicalreplica.ReasonRestartPending, cond.Reason) + assert.False(t, rec.called, "a pending restart is decided before the query") + }) + + t.Run("primary is ready", func(t *testing.T) { + cr := newCluster() + rec := new(execRecorder) + r := newReconciler(t, cr, rec, "", nil, + primaryPodForCluster(cr), logicalReplicaUserSecret(cr)) + + cond := r.observePrimaryReadiness(t.Context(), cr) + + assert.Equal(t, metav1.ConditionTrue, cond.Status) + assert.Equal(t, "PrimaryReady", cond.Reason) + assert.Equal(t, pNaming.ConditionReadyForLogicalReplication, cond.Type) + // Left for meta.SetStatusCondition to stamp, so a reason that moves on + // its own does not look like a transition. + assert.True(t, cond.LastTransitionTime.IsZero()) + + assert.True(t, rec.called) + assert.Equal(t, logicalreplica.PrimaryReadinessQuery(), rec.query) + }) + + t.Run("one unmet prerequisite", func(t *testing.T) { + cr := newCluster() + rec := new(execRecorder) + r := newReconciler(t, cr, rec, logicalreplica.ReasonReplicationHBAMissing, nil, + primaryPodForCluster(cr), logicalReplicaUserSecret(cr)) + + cond := r.observePrimaryReadiness(t.Context(), cr) + + assert.Equal(t, metav1.ConditionFalse, cond.Status) + assert.Equal(t, logicalreplica.ReasonReplicationHBAMissing, cond.Reason) + assert.Equal(t, + logicalreplica.PrimaryReadinessMessage(logicalreplica.ReasonReplicationHBAMissing), + cond.Message) + }) + + t.Run("several unmet prerequisites", func(t *testing.T) { + cr := newCluster() + rec := new(execRecorder) + r := newReconciler(t, cr, rec, + logicalreplica.ReasonRestartPending+","+logicalreplica.ReasonReplicationHBAMissing, nil, + primaryPodForCluster(cr), logicalReplicaUserSecret(cr)) + + cond := r.observePrimaryReadiness(t.Context(), cr) + + // The reason is the first, but the message has to name all of them: the + // user fixes them together. + assert.Equal(t, logicalreplica.ReasonRestartPending, cond.Reason) + assert.Contains(t, cond.Message, + logicalreplica.PrimaryReadinessMessage(logicalreplica.ReasonRestartPending)) + assert.Contains(t, cond.Message, + logicalreplica.PrimaryReadinessMessage(logicalreplica.ReasonReplicationHBAMissing)) + }) + + t.Run("primary cannot be queried", func(t *testing.T) { + cr := newCluster() + rec := new(execRecorder) + r := newReconciler(t, cr, rec, "", errors.New("connection refused"), + primaryPodForCluster(cr), logicalReplicaUserSecret(cr)) + + cond := r.observePrimaryReadiness(t.Context(), cr) + + // Unknown rather than False: the operator does not know, and a bootstrap + // still must not start. + assert.Equal(t, metav1.ConditionUnknown, cond.Status) + assert.Equal(t, "PrimaryUnreachable", cond.Reason) + assert.Contains(t, cond.Message, "connection refused") + }) +} + +func TestReconcileLogicalReplicaGatesBootstrap(t *testing.T) { + cr, err := readDefaultCR("cluster1", "pg") + require.NoError(t, err) + cr.Default() + cr.Spec.CRVersion = "3.1.0" + cr.Status.State = v2.AppStateReady + + spec := &v2.LogicalReplicaSpec{Name: "analytics"} + + cl, err := buildFakeClient(t.Context(), cr) + require.NoError(t, err) + r := &PGClusterReconciler{Client: cl} + + crunchyCR := &crunchyv1beta1.PostgresCluster{} + crunchyCR.Spec.PostgresVersion = 17 + + status, err := r.reconcileLogicalReplica(t.Context(), cr, crunchyCR, spec, false) + require.NoError(t, err) + + assert.Equal(t, v2.LogicalReplicaStateBootstrapping, status.State) + assert.Equal(t, v2.LogicalReplicaReasonPrimaryNotReady, status.Reason) + assert.Contains(t, status.Message, pNaming.ConditionReadyForLogicalReplication) + + // The gate sits ahead of everything, so nothing at all was created - not the + // Job, and not the volume or config it would need. Resolving the databases + // would have queried the primary, and PodExec is nil here: that this did not + // panic is itself the assertion. + unwanted := map[string]client.Object{ + logicalReplicaJobName(cr, spec.Name): &batchv1.Job{}, + logicalReplicaPVCName(cr, spec.Name): &corev1.PersistentVolumeClaim{}, + logicalReplicaConfigMapName(cr, spec.Name): &corev1.ConfigMap{}, + } + for name, obj := range unwanted { + err := cl.Get(t.Context(), client.ObjectKey{Name: name, Namespace: cr.Namespace}, obj) + assert.True(t, apierrors.IsNotFound(err), "%s should not exist: %v", name, err) + } +} + +// TestReconcileLogicalReplicaWaitsForDatabases covers the way a cluster that has +// no databases yet used to be reported. The list is frozen for the lifetime of +// the replica, so resolving it too early is permanent: an empty list left the +// replica broken, and a database named in the spec that had still to be created +// was seeded anyway and failed the one bootstrap attempt the Job gets. +func TestReconcileLogicalReplicaWaitsForDatabases(t *testing.T) { + newCluster := func(t *testing.T) *v2.PerconaPGCluster { + cr, err := readDefaultCR("cluster1", "pg") + require.NoError(t, err) + cr.Default() + cr.Spec.CRVersion = "3.1.0" + cr.Status.State = v2.AppStateReady + cr.Spec.Port = new(int32(5432)) + return cr + } + + // The databases the primary reports, whatever it is asked. Both queries this + // covers read one name per line. + primaryWith := func(databases ...string) func(context.Context, string, string, string, + io.Reader, io.Writer, io.Writer, ...string) error { + return func(_ context.Context, _, _, _ string, + _ io.Reader, out, _ io.Writer, _ ...string, + ) error { + for _, database := range databases { + if _, err := fmt.Fprintln(out, database); err != nil { + return err + } + } + return nil + } + } + + // Nothing may be created while a replica is waiting: the bootstrap Job gets + // one attempt, and running it against the wrong set of databases spends it. + assertNothingCreated := func(t *testing.T, cl client.Client, cr *v2.PerconaPGCluster, replica string) { + t.Helper() + for name, obj := range map[string]client.Object{ + logicalReplicaJobName(cr, replica): &batchv1.Job{}, + logicalReplicaPVCName(cr, replica): &corev1.PersistentVolumeClaim{}, + logicalReplicaConfigMapName(cr, replica): &corev1.ConfigMap{}, + } { + err := cl.Get(t.Context(), client.ObjectKey{Name: name, Namespace: cr.Namespace}, obj) + assert.True(t, apierrors.IsNotFound(err), "%s should not exist: %v", name, err) + } + } + + t.Run("waits for the operator to create the databases", func(t *testing.T) { + cr := newCluster(t) + spec := &v2.LogicalReplicaSpec{Name: "analytics"} + + cl, err := buildFakeClient(t.Context(), cr, primaryPodForCluster(cr)) + require.NoError(t, err) + + // No databaseRevision: the PostgresCluster controller has not finished a + // create pass, so the list on the primary may be empty or half of what it + // is about to be. PodExec is nil here - that this did not panic is the + // assertion that the primary was not queried at all. + crunchyCR := &crunchyv1beta1.PostgresCluster{} + crunchyCR.Spec.PostgresVersion = 17 + + r := &PGClusterReconciler{Client: cl} + status, err := r.reconcileLogicalReplica(t.Context(), cr, crunchyCR, spec, true) + require.NoError(t, err) + + assert.Equal(t, v2.LogicalReplicaStateBootstrapping, status.State) + assert.Equal(t, v2.LogicalReplicaReasonWaitingForDatabases, status.Reason) + assert.Empty(t, status.Databases) + assertNothingCreated(t, cl, cr, spec.Name) + }) + + t.Run("waits when the cluster has no databases of its own", func(t *testing.T) { + cr := newCluster(t) + spec := &v2.LogicalReplicaSpec{Name: "analytics"} + + cl, err := buildFakeClient(t.Context(), cr, primaryPodForCluster(cr)) + require.NoError(t, err) + + crunchyCR := &crunchyv1beta1.PostgresCluster{} + crunchyCR.Spec.PostgresVersion = 17 + crunchyCR.Status.DatabaseRevision = "abc123" + + r := &PGClusterReconciler{Client: cl} + // The query already filters out templates and "postgres", so a cluster + // with no databases of its own answers with nothing. + r.PodExec = primaryWith() + + status, err := r.reconcileLogicalReplica(t.Context(), cr, crunchyCR, spec, true) + require.NoError(t, err) + + // Bootstrapping, not broken: a database created at any point makes this + // replica viable, and nothing about it needs to be rebuilt first. + assert.Equal(t, v2.LogicalReplicaStateBootstrapping, status.State) + assert.Equal(t, v2.LogicalReplicaReasonWaitingForDatabases, status.Reason) + assert.Contains(t, status.Message, "spec.logicalReplicas[].databases") + assert.Empty(t, status.Databases) + assertNothingCreated(t, cl, cr, spec.Name) + }) + + t.Run("waits for a database named in the spec that does not exist", func(t *testing.T) { + cr := newCluster(t) + spec := &v2.LogicalReplicaSpec{ + Name: "analytics", + Databases: []crunchyv1beta1.PostgresIdentifier{"shop", "warehouse"}, + } + + cl, err := buildFakeClient(t.Context(), cr, primaryPodForCluster(cr)) + require.NoError(t, err) + + crunchyCR := &crunchyv1beta1.PostgresCluster{} + crunchyCR.Spec.PostgresVersion = 17 + crunchyCR.Status.DatabaseRevision = "abc123" + + r := &PGClusterReconciler{Client: cl} + r.PodExec = primaryWith("postgres", "template1", "shop") + + status, err := r.reconcileLogicalReplica(t.Context(), cr, crunchyCR, spec, true) + require.NoError(t, err) + + assert.Equal(t, v2.LogicalReplicaStateBootstrapping, status.State) + assert.Equal(t, v2.LogicalReplicaReasonWaitingForDatabases, status.Reason) + assert.Contains(t, status.Message, "warehouse") + assert.NotContains(t, status.Message, "shop,") + + // Nothing is frozen while one of them is missing: seeding the half that + // exists would produce a replica that can never cover the other. + assert.Empty(t, status.Databases) + assertNothingCreated(t, cl, cr, spec.Name) + }) + + t.Run("freezes the list once every database exists", func(t *testing.T) { + cr := newCluster(t) + spec := &v2.LogicalReplicaSpec{ + Name: "analytics", + Databases: []crunchyv1beta1.PostgresIdentifier{"shop", "warehouse"}, + } + + cl, err := buildFakeClient(t.Context(), cr, primaryPodForCluster(cr)) + require.NoError(t, err) + + crunchyCR := &crunchyv1beta1.PostgresCluster{} + crunchyCR.Spec.PostgresVersion = 17 + crunchyCR.Status.DatabaseRevision = "abc123" + + r := &PGClusterReconciler{Client: cl} + r.PodExec = primaryWith("postgres", "template1", "shop", "warehouse") + + status, err := r.reconcileLogicalReplica(t.Context(), cr, crunchyCR, spec, true) + require.NoError(t, err) + + assert.Equal(t, v2.LogicalReplicaStateBootstrapping, status.State) + assert.Empty(t, status.Reason) + assert.Equal(t, []string{"shop", "warehouse"}, status.Databases) + + // The resolved list is persisted before anything acts on it, so this pass + // still stops short of the Job. + assertNothingCreated(t, cl, cr, spec.Name) + }) +} + +func TestReconcileLogicalReplicasRequeue(t *testing.T) { + newCluster := func(state v2.AppState, replicas ...v2.LogicalReplicaSpec) *v2.PerconaPGCluster { + cr, err := readDefaultCR("cluster1", "pg") + require.NoError(t, err) + cr.Default() + cr.Spec.CRVersion = "3.1.0" + cr.Spec.LogicalReplicas = replicas + cr.Status.State = state + return cr + } + + newReconciler := func(t *testing.T, cr *v2.PerconaPGCluster) *PGClusterReconciler { + cl, err := buildFakeClient(t.Context(), cr) + require.NoError(t, err) + return &PGClusterReconciler{Client: cl} + } + + crunchyCR := &crunchyv1beta1.PostgresCluster{} + + t.Run("no requeue when the feature is unused", func(t *testing.T) { + cr := newCluster(v2.AppStateReady) + + requeue, err := newReconciler(t, cr).reconcileLogicalReplicas(t.Context(), cr, crunchyCR) + + require.NoError(t, err) + assert.False(t, requeue) + }) + + t.Run("no requeue while the cluster is not ready", func(t *testing.T) { + // The state is written by updateStatus just before this runs, so the + // cluster becoming ready wakes the controller on its own. Polling here + // would spin on a paused cluster forever. + cr := newCluster(v2.AppStatePaused, v2.LogicalReplicaSpec{Name: "analytics"}) + + requeue, err := newReconciler(t, cr).reconcileLogicalReplicas(t.Context(), cr, crunchyCR) + + require.NoError(t, err) + assert.False(t, requeue) + }) + + t.Run("requeues a replica that is not ready", func(t *testing.T) { + // No primary pod, so the gate stays shut. Nothing this controller + // watches will change to reopen it. + cr := newCluster(v2.AppStateReady, v2.LogicalReplicaSpec{Name: "analytics"}) + r := newReconciler(t, cr) + + requeue, err := r.reconcileLogicalReplicas(t.Context(), cr, crunchyCR) + + require.NoError(t, err) + assert.True(t, requeue) + + updated := new(v2.PerconaPGCluster) + require.NoError(t, r.Client.Get(t.Context(), + client.ObjectKey{Name: cr.Name, Namespace: cr.Namespace}, updated)) + + require.Len(t, updated.Status.LogicalReplicas, 1) + assert.Equal(t, v2.LogicalReplicaReasonPrimaryNotReady, updated.Status.LogicalReplicas[0].Reason) + + cond := meta.FindStatusCondition(updated.Status.Conditions, + pNaming.ConditionReadyForLogicalReplication) + require.NotNil(t, cond) + assert.Equal(t, "PrimaryPodNotFound", cond.Reason) + }) +} + +// TestBootstrapIsNotRepeated covers the way a completed bootstrap used to be +// forgotten. The Job is deleted as soon as it completes, so status.seededAt is +// the only remaining record that the replica was seeded; anything that failed +// after it was set - the pod not accepting connections yet, most easily - used to +// discard it and seed the replica a second time. +func TestBootstrapIsNotRepeated(t *testing.T) { + cr, err := readDefaultCR("cluster1", "pg") + require.NoError(t, err) + cr.Default() + cr.Spec.CRVersion = "3.1.0" + cr.Status.State = v2.AppStateReady + // Not set by deploy/cr.yaml, and the rendered replica config needs it. + cr.Spec.Port = new(int32(5432)) + + spec := &v2.LogicalReplicaSpec{Name: "analytics"} + + crunchyCR := &crunchyv1beta1.PostgresCluster{ + ObjectMeta: metav1.ObjectMeta{Name: cr.Name, Namespace: cr.Namespace}, + } + crunchyCR.Spec.PostgresVersion = 17 + crunchyCR.Spec.Backups.PGBackRest.Repos = []crunchyv1beta1.PGBackRestRepo{{Name: "repo1"}} + + completedJob := func() *batchv1.Job { + return &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: logicalReplicaJobName(cr, spec.Name), + Namespace: cr.Namespace, + }, + Status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{ + {Type: batchv1.JobComplete, Status: corev1.ConditionTrue}, + }}, + } + } + + t.Run("the statefulset is created in the same pass as the bootstrap", func(t *testing.T) { + crWithDatabases := cr.DeepCopy() + crWithDatabases.Status.LogicalReplicas = []v2.LogicalReplicaStatus{{ + Name: spec.Name, + State: v2.LogicalReplicaStateBootstrapping, + Databases: []string{"cluster1"}, + }} + + cl, err := buildFakeClient(t.Context(), crWithDatabases, completedJob(), + primaryPodForCluster(crWithDatabases)) + require.NoError(t, err) + + r := &PGClusterReconciler{Client: cl} + r.PodExec = func(_ context.Context, _, _, _ string, + _ io.Reader, out, _ io.Writer, _ ...string, + ) error { + // The slot the bootstrap created is on the primary. The health check + // then goes looking for the replica's own pod, which the fake client + // has none of, and settles on LogicalReplicaPodNotFound. + _, err := fmt.Fprintln(out, "1") + return err + } + + status, err := r.reconcileLogicalReplica(t.Context(), crWithDatabases, crunchyCR, spec, true) + require.NoError(t, err) + + // Nothing waits for another pass: the replica is seeded, the objects + // that run it exist, and the only thing left is the pod coming up. + require.NotNil(t, status.SeededAt) + assert.Equal(t, v2.LogicalReplicaStateBootstrapping, status.State) + assert.Equal(t, v2.LogicalReplicaReasonPodNotFound, status.Reason) + + sts := &appsv1.StatefulSet{} + require.NoError(t, cl.Get(t.Context(), client.ObjectKey{ + Name: logicalReplicaObjectName(cr, spec.Name), Namespace: cr.Namespace}, sts)) + assert.Equal(t, int32(1), *sts.Spec.Replicas) + + require.NoError(t, cl.Get(t.Context(), client.ObjectKey{ + Name: logicalReplicaObjectName(cr, spec.Name), Namespace: cr.Namespace}, new(corev1.Service))) + + // seededAt is now the only record that the replica was seeded: the Job + // is deleted the moment it completes. The subtest below is what keeps a + // later failure in this same pass from discarding it. + err = cl.Get(t.Context(), client.ObjectKey{ + Name: logicalReplicaJobName(cr, spec.Name), Namespace: cr.Namespace}, new(batchv1.Job)) + assert.True(t, apierrors.IsNotFound(err), "the bootstrap job must be deleted: %v", err) + }) + + // The reported symptom: the health check cannot reach the replica yet, the + // replica is recorded broken, and the next pass bootstraps it again. + t.Run("a health check failure does not forget the bootstrap", func(t *testing.T) { + // Bootstrapped, and the pod is not queryable yet, so the health check + // fails. The recorded status must not lose seededAt. + bootstrapped := cr.DeepCopy() + bootstrapped.Status.LogicalReplicas = []v2.LogicalReplicaStatus{{ + Name: spec.Name, + State: v2.LogicalReplicaStateBootstrapping, + Reason: v2.LogicalReplicaReasonPodNotFound, + Databases: []string{"cluster1"}, + SeededAt: new(metav1.Now()), + }} + bootstrapped.Spec.LogicalReplicas = v2.LogicalReplicas{*spec} + + cl, err := buildFakeClient(t.Context(), bootstrapped, + primaryPodForCluster(bootstrapped), logicalReplicaUserSecret(bootstrapped)) + require.NoError(t, err) + + r := &PGClusterReconciler{Client: cl} + r.PodExec = func(_ context.Context, _, _, _ string, + stdin io.Reader, out, _ io.Writer, _ ...string, + ) error { + sql, err := io.ReadAll(stdin) + require.NoError(t, err) + // The readiness probe query succeeds; the slot count on the primary + // does not. + if strings.Contains(string(sql), "pg_hba_file_rules") { + _, err = fmt.Fprintln(out, "") + return err + } + return errors.New("FATAL: the database system is starting up") + } + + _, err = r.reconcileLogicalReplicas(t.Context(), bootstrapped, crunchyCR) + require.NoError(t, err) + + updated := new(v2.PerconaPGCluster) + require.NoError(t, cl.Get(t.Context(), + client.ObjectKey{Name: cr.Name, Namespace: cr.Namespace}, updated)) + + require.Len(t, updated.Status.LogicalReplicas, 1) + recorded := updated.Status.LogicalReplicas[0] + assert.Equal(t, v2.LogicalReplicaStateBroken, recorded.State) + assert.NotNil(t, recorded.SeededAt, "seededAt must survive a failed pass") + assert.Equal(t, []string{"cluster1"}, recorded.Databases, + "the frozen database list must survive a failed pass") + }) + + t.Run("an existing statefulset is adopted rather than seeded again", func(t *testing.T) { + // seededAt lost by any means at all: the StatefulSet is the more + // trustworthy record, because it only exists after a bootstrap. + crWithDatabases := cr.DeepCopy() + crWithDatabases.Status.LogicalReplicas = []v2.LogicalReplicaStatus{{ + Name: spec.Name, + State: v2.LogicalReplicaStateBootstrapping, + Databases: []string{"cluster1"}, + }} + + // Truncated because metav1.Time round-trips at second granularity. + created := metav1.NewTime(time.Now().Add(-time.Hour).Truncate(time.Second)) + sts := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{ + Name: logicalReplicaObjectName(cr, spec.Name), + Namespace: cr.Namespace, + CreationTimestamp: created, + }} + + cl, err := buildFakeClient(t.Context(), crWithDatabases, sts) + require.NoError(t, err) + r := &PGClusterReconciler{Client: cl} + + status := logicalReplicaStatusFor(crWithDatabases, spec.Name) + bootstrapped, err := r.reconcileLogicalReplicaBootstrap( + t.Context(), crWithDatabases, crunchyCR, spec, status) + + require.NoError(t, err) + assert.True(t, bootstrapped) + require.NotNil(t, status.SeededAt) + // Taken from the StatefulSet, not stamped as now. + assert.True(t, status.SeededAt.Equal(&created), + "expected %v, got %v", created, status.SeededAt) + + job := &batchv1.Job{} + err = cl.Get(t.Context(), client.ObjectKey{ + Name: logicalReplicaJobName(cr, spec.Name), Namespace: cr.Namespace}, job) + assert.True(t, apierrors.IsNotFound(err), "no bootstrap job may be created: %v", err) + }) +} + +func TestLogicalReplicaPodRequiresReady(t *testing.T) { + cr, err := readDefaultCR("cluster1", "pg") + require.NoError(t, err) + cr.Default() + + pod := func(ready corev1.ConditionStatus) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1-lr-analytics-0", + Namespace: cr.Namespace, + Labels: map[string]string{ + naming.LabelCluster: cr.Name, + pNaming.LabelLogicalReplica: "analytics", + }, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: ready}}, + }, + } + } + + // Running but not ready is the state a replica pod is in for the first few + // seconds. Returning it made psql fail and the replica look broken. + cl, err := buildFakeClient(t.Context(), cr.DeepCopy(), pod(corev1.ConditionFalse)) + require.NoError(t, err) + _, err = (&PGClusterReconciler{Client: cl}).logicalReplicaPod(t.Context(), cr, "analytics") + require.Error(t, err) + + cl, err = buildFakeClient(t.Context(), cr.DeepCopy(), pod(corev1.ConditionTrue)) + require.NoError(t, err) + found, err := (&PGClusterReconciler{Client: cl}).logicalReplicaPod(t.Context(), cr, "analytics") + require.NoError(t, err) + assert.Equal(t, "cluster1-lr-analytics-0", found.Name) +} + +func TestUpdateLogicalReplicaStatus(t *testing.T) { + cr, err := readDefaultCR("cluster1", "pg") + require.NoError(t, err) + cr.Default() + + cl, err := buildFakeClient(t.Context(), cr) + require.NoError(t, err) + r := &PGClusterReconciler{Client: cl} + + read := func() *v2.PerconaPGCluster { + out := new(v2.PerconaPGCluster) + require.NoError(t, cl.Get(t.Context(), + client.ObjectKey{Name: cr.Name, Namespace: cr.Namespace}, out)) + return out + } + + statuses := []v2.LogicalReplicaStatus{{Name: "analytics", State: v2.LogicalReplicaStateBootstrapping}} + + t.Run("writes the statuses and the condition together", func(t *testing.T) { + require.NoError(t, r.updateLogicalReplicaStatus(t.Context(), cr, statuses, &metav1.Condition{ + Type: pNaming.ConditionReadyForLogicalReplication, + Status: metav1.ConditionFalse, + Reason: logicalreplica.ReasonReplicationHBAMissing, + Message: "first", + })) + + out := read() + require.Len(t, out.Status.LogicalReplicas, 1) + + cond := meta.FindStatusCondition(out.Status.Conditions, + pNaming.ConditionReadyForLogicalReplication) + require.NotNil(t, cond) + assert.Equal(t, metav1.ConditionFalse, cond.Status) + assert.Equal(t, out.Generation, cond.ObservedGeneration) + }) + + t.Run("a message that changes on its own is not a transition", func(t *testing.T) { + before := meta.FindStatusCondition(read().Status.Conditions, + pNaming.ConditionReadyForLogicalReplication).LastTransitionTime + require.False(t, before.IsZero()) + + require.NoError(t, r.updateLogicalReplicaStatus(t.Context(), cr, statuses, &metav1.Condition{ + Type: pNaming.ConditionReadyForLogicalReplication, + Status: metav1.ConditionFalse, + Reason: logicalreplica.ReasonRestartPending, + Message: "second", + })) + + cond := meta.FindStatusCondition(read().Status.Conditions, + pNaming.ConditionReadyForLogicalReplication) + assert.Equal(t, logicalreplica.ReasonRestartPending, cond.Reason) + assert.Equal(t, "second", cond.Message) + assert.True(t, before.Equal(&cond.LastTransitionTime)) + }) + + t.Run("the condition goes with the last replica", func(t *testing.T) { + require.NoError(t, r.updateLogicalReplicaStatus(t.Context(), cr, nil, nil)) + + out := read() + assert.Empty(t, out.Status.LogicalReplicas) + assert.Nil(t, meta.FindStatusCondition(out.Status.Conditions, + pNaming.ConditionReadyForLogicalReplication)) + }) +} diff --git a/percona/controller/pgcluster/status.go b/percona/controller/pgcluster/status.go index 79d46332c5..026b7a6f04 100644 --- a/percona/controller/pgcluster/status.go +++ b/percona/controller/pgcluster/status.go @@ -206,6 +206,7 @@ var perconaOwnedConditions = []string{ pNaming.ConditionClusterIsReadyForBackup, pNaming.ConditionAPIGroupMigration, pNaming.ConditionStandbyLagging, + pNaming.ConditionReadyForLogicalReplication, v2.ConditionPMMReady, } diff --git a/percona/controller/pgcluster/status_test.go b/percona/controller/pgcluster/status_test.go index 9ba24fb8da..57af808cc2 100644 --- a/percona/controller/pgcluster/status_test.go +++ b/percona/controller/pgcluster/status_test.go @@ -672,6 +672,11 @@ func TestUpdateConditions(t *testing.T) { Status: metav1.ConditionFalse, Reason: "LagNotDetected", }, + { + Type: pNaming.ConditionReadyForLogicalReplication, + Status: metav1.ConditionTrue, + Reason: "PrimaryReady", + }, }, statusConditions: []metav1.Condition{ { @@ -695,6 +700,7 @@ func TestUpdateConditions(t *testing.T) { v2.ConditionPMMReady, pNaming.ConditionAPIGroupMigration, pNaming.ConditionStandbyLagging, + pNaming.ConditionReadyForLogicalReplication, }, }, } diff --git a/percona/naming/conditions.go b/percona/naming/conditions.go index 6d9747a992..bd0440f92d 100644 --- a/percona/naming/conditions.go +++ b/percona/naming/conditions.go @@ -7,4 +7,9 @@ const ( // ConditionStandbyLagging is the type used in a condition to indicate whether or not // the standby cluster is lagging behind the main site ConditionStandbyLagging = "StandbyLagging" + + // ConditionReadyForLogicalReplication reports whether the primary carries + // everything a logical replica bootstrap needs. A bootstrap started too + // early cannot be retried without re-seeding. + ConditionReadyForLogicalReplication = "ReadyForLogicalReplication" ) diff --git a/percona/naming/label.go b/percona/naming/label.go index 55e47ee6d3..94bb16f427 100644 --- a/percona/naming/label.go +++ b/percona/naming/label.go @@ -2,4 +2,8 @@ package naming const ( LabelOperatorVersion = PrefixPerconaPGV2 + "version" + + // LabelLogicalReplica marks every object that belongs to a logical replica + // with the replica's name. + LabelLogicalReplica = PrefixPerconaPGV2 + "logical-replica" ) diff --git a/pkg/apis/pgv2.percona.com/v2/logicalreplica_types_test.go b/pkg/apis/pgv2.percona.com/v2/logicalreplica_types_test.go new file mode 100644 index 0000000000..1b6a5f65bf --- /dev/null +++ b/pkg/apis/pgv2.percona.com/v2/logicalreplica_types_test.go @@ -0,0 +1,199 @@ +package v2 + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + + "github.com/percona/percona-postgresql-operator/v2/percona/version" + crunchyv1beta1 "github.com/percona/percona-postgresql-operator/v2/pkg/apis/upstream.pgv2.percona.com/v1beta1" +) + +func logicalReplicaCR(replicas ...LogicalReplicaSpec) *PerconaPGCluster { + return &PerconaPGCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "test-cluster", Namespace: "test-namespace"}, + Spec: PerconaPGClusterSpec{ + CRVersion: version.Version(), + PostgresVersion: 17, + InstanceSets: PGInstanceSets{{ + Name: "instance1", + Replicas: new(int32(1)), + DataVolumeClaimSpec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + }, + }}, + Backups: Backups{ + PGBackRest: PGBackRestArchive{ + Repos: []crunchyv1beta1.PGBackRestRepo{{Name: "repo1"}}, + }, + }, + LogicalReplicas: replicas, + }, + } +} + +func TestLogicalReplicasEnabled(t *testing.T) { + t.Run("disabled when the section is empty", func(t *testing.T) { + assert.False(t, logicalReplicaCR().LogicalReplicasEnabled()) + }) + + t.Run("enabled when a replica is configured", func(t *testing.T) { + cr := logicalReplicaCR(LogicalReplicaSpec{Name: "analytics"}) + assert.True(t, cr.LogicalReplicasEnabled()) + }) + + t.Run("disabled for CRVersion < 3.1.0", func(t *testing.T) { + cr := logicalReplicaCR(LogicalReplicaSpec{Name: "analytics"}) + cr.Spec.CRVersion = "3.0.0" + assert.False(t, cr.LogicalReplicasEnabled()) + }) +} + +func TestBootstrapMethodOrDefault(t *testing.T) { + t.Run("unset seeds from pgBackRest", func(t *testing.T) { + // The CRD defaults this, so an empty value only reaches here from a spec + // the API server has not seen: a unit test, or an object stored before + // the field existed. Neither may quietly switch method. + spec := &LogicalReplicaSpec{Name: "analytics"} + assert.Equal(t, LogicalReplicaBootstrapMethodPGBackRest, spec.BootstrapMethodOrDefault()) + }) + + t.Run("an explicit method is kept", func(t *testing.T) { + for _, method := range []LogicalReplicaBootstrapMethod{ + LogicalReplicaBootstrapMethodPGBackRest, + LogicalReplicaBootstrapMethodPGBaseBackup, + } { + spec := &LogicalReplicaSpec{Name: "analytics", BootstrapMethod: method} + assert.Equal(t, method, spec.BootstrapMethodOrDefault()) + } + }) +} + +func TestValidateLogicalReplicas(t *testing.T) { + t.Run("empty is valid", func(t *testing.T) { + require.NoError(t, logicalReplicaCR().ValidateLogicalReplicas()) + }) + + t.Run("distinct names are valid", func(t *testing.T) { + cr := logicalReplicaCR( + LogicalReplicaSpec{Name: "analytics"}, + LogicalReplicaSpec{Name: "reporting"}, + ) + require.NoError(t, cr.ValidateLogicalReplicas()) + }) + + t.Run("rejects duplicate names", func(t *testing.T) { + cr := logicalReplicaCR( + LogicalReplicaSpec{Name: "analytics"}, + LogicalReplicaSpec{Name: "analytics"}, + ) + err := cr.ValidateLogicalReplicas() + require.Error(t, err) + assert.Contains(t, err.Error(), "duplicate spec.logicalReplicas name") + }) + + t.Run("rejects a name that collides with an instance set", func(t *testing.T) { + // Both would want the same StatefulSet name. + cr := logicalReplicaCR(LogicalReplicaSpec{Name: "instance1"}) + err := cr.ValidateLogicalReplicas() + require.Error(t, err) + assert.Contains(t, err.Error(), "conflicts with an instance set") + }) + + t.Run("rejects duplicate databases within a replica", func(t *testing.T) { + cr := logicalReplicaCR(LogicalReplicaSpec{ + Name: "analytics", + Databases: []crunchyv1beta1.PostgresIdentifier{"db1", "db1"}, + }) + err := cr.ValidateLogicalReplicas() + require.Error(t, err) + assert.Contains(t, err.Error(), "duplicate database") + }) + + t.Run("reached through Validate", func(t *testing.T) { + cr := logicalReplicaCR( + LogicalReplicaSpec{Name: "analytics"}, + LogicalReplicaSpec{Name: "analytics"}, + ) + cr.Default() + require.Error(t, cr.Validate()) + }) +} + +func TestLogicalReplicasToCrunchy(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, AddToScheme(scheme)) + require.NoError(t, crunchyv1beta1.AddToScheme(scheme)) + + t.Run("no logicalrepl user without logical replicas", func(t *testing.T) { + cr := logicalReplicaCR() + cr.Default() + + actual, err := cr.ToCrunchy(context.Background(), nil, scheme) + require.NoError(t, err) + + assert.Empty(t, actual.Spec.LogicalReplicas) + for _, user := range actual.Spec.Users { + assert.NotEqual(t, PostgresIdentifierOf(UserLogicalReplication), user.Name) + } + }) + + t.Run("injects the reserved superuser and mirrors the names", func(t *testing.T) { + cr := logicalReplicaCR( + LogicalReplicaSpec{Name: "analytics"}, + LogicalReplicaSpec{Name: "reporting"}, + ) + cr.Default() + + actual, err := cr.ToCrunchy(context.Background(), nil, scheme) + require.NoError(t, err) + + require.Len(t, actual.Spec.LogicalReplicas, 2) + assert.Equal(t, "analytics", actual.Spec.LogicalReplicas[0].Name) + assert.Equal(t, "reporting", actual.Spec.LogicalReplicas[1].Name) + + // pg_createsubscriber creates a publication FOR ALL TABLES, which needs + // a superuser, and the replica streams from the primary as this role, + // which needs REPLICATION. + var found *crunchyv1beta1.PostgresUserSpec + for i := range actual.Spec.Users { + if actual.Spec.Users[i].Name == PostgresIdentifierOf(UserLogicalReplication) { + found = &actual.Spec.Users[i] + } + } + require.NotNil(t, found, "logicalrepl user was not injected") + assert.Equal(t, "SUPERUSER REPLICATION", found.Options) + require.NotNil(t, found.Password) + }) + + t.Run("a user-declared logicalrepl is ignored", func(t *testing.T) { + cr := logicalReplicaCR(LogicalReplicaSpec{Name: "analytics"}) + cr.Spec.Users = []crunchyv1beta1.PostgresUserSpec{{ + Name: PostgresIdentifierOf(UserLogicalReplication), + Options: "NOSUPERUSER", + }} + cr.Default() + + actual, err := cr.ToCrunchy(context.Background(), nil, scheme) + require.NoError(t, err) + + count := 0 + for _, user := range actual.Spec.Users { + if user.Name == PostgresIdentifierOf(UserLogicalReplication) { + count++ + assert.Equal(t, "SUPERUSER REPLICATION", user.Options) + } + } + assert.Equal(t, 1, count) + }) +} + +// PostgresIdentifierOf is a readability helper for the tests above. +func PostgresIdentifierOf(s string) crunchyv1beta1.PostgresIdentifier { + return crunchyv1beta1.PostgresIdentifier(s) +} diff --git a/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go b/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go index 8cee3ea4c0..84c68d17a8 100644 --- a/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go +++ b/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go @@ -53,6 +53,8 @@ type PerconaPGCluster struct { // +kubebuilder:validation:XValidation:rule="!has(self.extensions) || !has(self.extensions.pg_tde) || !has(self.extensions.pg_tde.enabled) || !self.extensions.pg_tde.enabled || self.postgresVersion >= 17",message="pg_tde is only supported for PG17 and above" // +kubebuilder:validation:XValidation:rule="!has(self.users) || self.postgresVersion >= 15 || self.users.all(u, !has(u.grantPublicSchemaAccess) || !u.grantPublicSchemaAccess)",message="PostgresVersion must be >= 15 if grantPublicSchemaAccess exists and is true" +// +kubebuilder:validation:XValidation:rule="!has(self.logicalReplicas) || size(self.logicalReplicas) == 0 || self.postgresVersion >= 17",message="spec.logicalReplicas requires spec.postgresVersion >= 17" +// +kubebuilder:validation:XValidation:rule="!has(self.logicalReplicas) || size(self.logicalReplicas) == 0 || !has(self.backups) || !has(self.backups.enabled) || self.backups.enabled || self.logicalReplicas.all(r, has(r.bootstrapMethod) && r.bootstrapMethod == 'pg_basebackup')",message="spec.logicalReplicas[].bootstrapMethod must be 'pg_basebackup' when spec.backups.enabled is false" type PerconaPGClusterSpec struct { // +optional Metadata *crunchyv1beta1.Metadata `json:"metadata,omitempty"` @@ -205,6 +207,109 @@ type PerconaPGClusterSpec struct { // files such as an LDAP CA certificate. // +optional Authentication *crunchyv1beta1.PostgresClusterAuthentication `json:"authentication,omitempty"` + + // Logical replicas are read-write PostgreSQL instances in this cluster that + // receive changes from the primary over logical replication. Each one is + // seeded with a physical copy of the primary, taken the way its + // bootstrapMethod says, and converted with pg_createsubscriber, which + // requires spec.postgresVersion to be 17 or higher. + // +optional + LogicalReplicas LogicalReplicas `json:"logicalReplicas,omitempty"` +} + +type LogicalReplicas []LogicalReplicaSpec + +// ToCrunchy projects the logical replicas onto the Crunchy spec, which needs +// only their names. +func (l LogicalReplicas) ToCrunchy() []crunchyv1beta1.LogicalReplicaSpec { + if len(l) == 0 { + return nil + } + + out := make([]crunchyv1beta1.LogicalReplicaSpec, 0, len(l)) + for _, replica := range l { + out = append(out, crunchyv1beta1.LogicalReplicaSpec{Name: replica.Name}) + } + return out +} + +type LogicalReplicaSpec struct { + // Name of the logical replica. It is used to name the StatefulSet, Service + // and PersistentVolumeClaim of the replica, as well as the publications, + // subscriptions and replication slots backing it. + // +kubebuilder:validation:Required + // +kubebuilder:validation:MaxLength=20 + // +kubebuilder:validation:Pattern=`^[a-z][a-z0-9-]*[a-z0-9]$` + Name string `json:"name"` + + // Databases to replicate. When empty, every database in the cluster except + // the templates and "postgres" is replicated. + // +optional + Databases []crunchyv1beta1.PostgresIdentifier `json:"databases,omitempty"` + + // BootstrapMethod selects how the data volume is seeded before + // pg_createsubscriber converts it into a subscriber. + // + // "pgbackrest" restores the cluster's most recent backup and puts no load on + // the primary. "pg_basebackup" streams a fresh copy straight from the + // primary and needs no pgBackRest repository, which is the only option when + // spec.backups.enabled is false. + // + // It is only read while the replica is being bootstrapped; changing it on a + // replica that already exists has no effect. + // +kubebuilder:validation:Enum={pgbackrest,pg_basebackup} + // +kubebuilder:default=pgbackrest + // +optional + BootstrapMethod LogicalReplicaBootstrapMethod `json:"bootstrapMethod,omitempty"` + + // Defines the data volume of the logical replica. + // +kubebuilder:validation:Required + DataVolumeClaimSpec corev1.PersistentVolumeClaimSpec `json:"dataVolumeClaimSpec"` + + // +optional + Metadata *crunchyv1beta1.Metadata `json:"metadata,omitempty"` + + // +optional + Resources corev1.ResourceRequirements `json:"resources,omitempty"` + + // +optional + Affinity *corev1.Affinity `json:"affinity,omitempty"` + + // +optional + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + + // +optional + PriorityClassName *string `json:"priorityClassName,omitempty"` + + // Specification of the service that exposes this logical replica. + // +optional + Expose *ServiceExpose `json:"expose,omitempty"` +} + +// LogicalReplicaBootstrapMethod selects how the data volume of a logical +// replica is seeded with a physical copy of the primary, before +// pg_createsubscriber converts it into a subscriber. +type LogicalReplicaBootstrapMethod string + +const ( + LogicalReplicaBootstrapMethodPGBackRest LogicalReplicaBootstrapMethod = "pgbackrest" + LogicalReplicaBootstrapMethodPGBaseBackup LogicalReplicaBootstrapMethod = "pg_basebackup" +) + +// BootstrapMethodOrDefault returns the method the data volume of the replica is +// seeded with. The field carries a CRD default, so this only matters for a spec +// the API server has not defaulted. +func (s *LogicalReplicaSpec) BootstrapMethodOrDefault() LogicalReplicaBootstrapMethod { + if s.BootstrapMethod == "" { + return LogicalReplicaBootstrapMethodPGBackRest + } + return s.BootstrapMethod +} + +// LogicalReplicasEnabled returns whether the cluster has any logical replica +// configured. +func (cr *PerconaPGCluster) LogicalReplicasEnabled() bool { + return cr.CompareVersion("3.1.0") >= 0 && len(cr.Spec.LogicalReplicas) > 0 } type ContainerOptions struct { @@ -385,6 +490,46 @@ func (cr *PerconaPGCluster) Validate() error { if err := cr.ValidateDynamicConfiguration(); err != nil { return err } + if err := cr.ValidateLogicalReplicas(); err != nil { + return err + } + return nil +} + +// ValidateLogicalReplicas checks the invariants of spec.logicalReplicas that +// cannot be expressed with kubebuilder markers. +func (cr *PerconaPGCluster) ValidateLogicalReplicas() error { + if len(cr.Spec.LogicalReplicas) == 0 { + return nil + } + + // A logical replica names a StatefulSet in the same namespace as the + // instance sets do, so the names must not collide. + instanceSets := make(map[string]struct{}, len(cr.Spec.InstanceSets)) + for _, set := range cr.Spec.InstanceSets { + instanceSets[set.Name] = struct{}{} + } + + seen := make(map[string]struct{}, len(cr.Spec.LogicalReplicas)) + for _, replica := range cr.Spec.LogicalReplicas { + if _, ok := seen[replica.Name]; ok { + return errors.Errorf("duplicate spec.logicalReplicas name %q", replica.Name) + } + seen[replica.Name] = struct{}{} + + if _, ok := instanceSets[replica.Name]; ok { + return errors.Errorf("spec.logicalReplicas name %q conflicts with an instance set of the same name", replica.Name) + } + + dbs := make(map[crunchyv1beta1.PostgresIdentifier]struct{}, len(replica.Databases)) + for _, db := range replica.Databases { + if _, ok := dbs[db]; ok { + return errors.Errorf("duplicate database %q in spec.logicalReplicas %q", db, replica.Name) + } + dbs[db] = struct{}{} + } + } + return nil } @@ -502,6 +647,10 @@ func (cr *PerconaPGCluster) ToCrunchy(ctx context.Context, postgresCluster *crun log.Info(UserMonitoring + " user is reserved, it'll be ignored.") continue } + if user.Name == UserLogicalReplication { + log.Info(UserLogicalReplication + " user is reserved, it'll be ignored.") + continue + } users = append(users, user) } @@ -528,8 +677,24 @@ func (cr *PerconaPGCluster) ToCrunchy(ctx context.Context, postgresCluster *crun } } + // SUPERUSER because pg_createsubscriber creates a publication FOR + // ALL TABLES, which plain REPLICATION roles such as _crunchyrepl cannot do. + if cr.LogicalReplicasEnabled() { + users = append(users, crunchyv1beta1.PostgresUserSpec{ + Name: UserLogicalReplication, + Options: "SUPERUSER REPLICATION", + Password: &crunchyv1beta1.PostgresPasswordSpec{ + Type: crunchyv1beta1.PostgresPasswordTypeAlphaNumeric, + }, + }) + } + postgresCluster.Spec.Users = users + // crunchy layer renders pg_hba rules, server parameters and + // Patroni's ignore_slots from this. + postgresCluster.Spec.LogicalReplicas = cr.Spec.LogicalReplicas.ToCrunchy() + postgresCluster.Spec.InstanceSets = cr.Spec.InstanceSets.ToCrunchy() postgresCluster.Spec.Proxy = cr.Spec.Proxy.ToCrunchy(cr.Spec.CRVersion) @@ -673,6 +838,12 @@ type PerconaPGClusterStatus struct { // +optional // +operator-sdk:csv:customresourcedefinitions:type=status Standby *StandbyStatus `json:"standby,omitempty"` + + // +optional + // +listType=map + // +listMapKey=name + // +operator-sdk:csv:customresourcedefinitions:type=status + LogicalReplicas []LogicalReplicaStatus `json:"logicalReplicas,omitempty"` } type StandbyStatus struct { @@ -680,6 +851,101 @@ type StandbyStatus struct { LagBytes int64 `json:"lagBytes,omitempty"` } +type LogicalReplicaState string + +const ( + // LogicalReplicaStateBootstrapping means the replica is being seeded and + // converted by the bootstrap Job, or has been and is not serving yet. + LogicalReplicaStateBootstrapping LogicalReplicaState = "bootstrapping" + + LogicalReplicaStateReady LogicalReplicaState = "ready" + + // LogicalReplicaStateBroken means replication has stopped and the replica + // needs to be recreated. + LogicalReplicaStateBroken LogicalReplicaState = "broken" + + // LogicalReplicaStateSuspended means the replica was stopped on purpose, + // because the cluster it replicates is being restored. Not an error. + LogicalReplicaStateSuspended LogicalReplicaState = "suspended" +) + +const ( + // LogicalReplicaReasonSourceSlotMissing means the replication slot backing + // this replica is gone from the primary. A slot lives only on the primary + // that created it, so a failover is the most common cause. + LogicalReplicaReasonSourceSlotMissing = "SourceSlotMissing" + + LogicalReplicaReasonSubscriptionDisabled = "SubscriptionDisabled" + + // LogicalReplicaReasonApplyWorkerDown means a subscription is enabled but has + // no running apply worker, usually because it cannot reach the primary. It + // retries forever without disabling the subscription, so nothing else shows + // that replication has stopped. + LogicalReplicaReasonApplyWorkerDown = "ApplyWorkerDown" + + LogicalReplicaReasonBootstrapFailed = "BootstrapFailed" + LogicalReplicaReasonPodNotFound = "LogicalReplicaPodNotFound" + + // LogicalReplicaReasonPrimaryNotReady means the primary does not yet carry + // everything the bootstrap needs; the ReadyForLogicalReplication condition + // says which prerequisite is missing. + LogicalReplicaReasonPrimaryNotReady = "PrimaryNotReady" + + LogicalReplicaReasonSourceRestoring = "SourceRestoring" + + // LogicalReplicaReasonSourceRestored means the data directory this replica was + // seeded from has been replaced by a restore. The replication slots went with + // it - pgBackRest does not back up pg_replslot - and a point-in-time restore + // has rewound the primary past changes the replica already applied, so nothing + // short of seeding it again fixes it. + LogicalReplicaReasonSourceRestored = "SourceRestored" + + // LogicalReplicaReasonWaitingForDataVolume means the data volume of an + // earlier incarnation of this replica is still being deleted. + LogicalReplicaReasonWaitingForDataVolume = "WaitingForDataVolume" + + // LogicalReplicaReasonWaitingForDatabases means the databases this replica + // would replicate do not exist on the primary yet. The set is frozen for the + // replica's lifetime, so it waits rather than seeding from a partial list. + LogicalReplicaReasonWaitingForDatabases = "WaitingForDatabases" + + // LogicalReplicaReasonAwaitingCleanup means the replica has been removed from + // the spec but its objects on the primary could not be dropped yet. Its status + // is kept until they are: forgetting it leaks a slot that pins WAL. + LogicalReplicaReasonAwaitingCleanup = "AwaitingCleanup" +) + +type LogicalReplicaStatus struct { + Name string `json:"name"` + + // +optional + State LogicalReplicaState `json:"state,omitempty"` + + // +optional + Reason string `json:"reason,omitempty"` + + // +optional + Message string `json:"message,omitempty"` + + // Databases replicated by this replica. It is resolved once, when the + // replica is bootstrapped, and does not change afterwards. + // +optional + Databases []string `json:"databases,omitempty"` + + // SeededAt is when the data this replica serves was copied from the + // cluster. + // +optional + SeededAt *metav1.Time `json:"seededAt,omitempty"` + + // InvalidatedAt is when the operator established that the data on this + // replica can no longer be reconciled with the cluster, because the cluster + // was restored in place after the replica was seeded from it. The replica + // stays stopped until it is removed from spec.logicalReplicas and added + // back, which seeds it again from scratch. + // +optional + InvalidatedAt *metav1.Time `json:"invalidatedAt,omitempty"` +} + type Patroni struct { // +optional Status *crunchyv1beta1.PatroniStatus `json:"status,omitempty"` @@ -1527,6 +1793,11 @@ const ConditionPMMReady = "PMMReady" const ( UserMonitoring = "monitor" + + // UserLogicalReplication is the reserved superuser that pg_createsubscriber + // connects to the primary as, and that the replica streams as. It only exists + // while the cluster has at least one logical replica. + UserLogicalReplication = "logicalrepl" ) // UserMonitoring constructs the monitoring user. diff --git a/pkg/apis/pgv2.percona.com/v2/zz_generated.deepcopy.go b/pkg/apis/pgv2.percona.com/v2/zz_generated.deepcopy.go index 7d13aa6114..b4df1cca83 100644 --- a/pkg/apis/pgv2.percona.com/v2/zz_generated.deepcopy.go +++ b/pkg/apis/pgv2.percona.com/v2/zz_generated.deepcopy.go @@ -321,6 +321,104 @@ func (in *LogRotateSpec) DeepCopy() *LogRotateSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LogicalReplicaSpec) DeepCopyInto(out *LogicalReplicaSpec) { + *out = *in + if in.Databases != nil { + in, out := &in.Databases, &out.Databases + *out = make([]v1beta1.PostgresIdentifier, len(*in)) + copy(*out, *in) + } + in.DataVolumeClaimSpec.DeepCopyInto(&out.DataVolumeClaimSpec) + if in.Metadata != nil { + in, out := &in.Metadata, &out.Metadata + *out = new(v1beta1.Metadata) + (*in).DeepCopyInto(*out) + } + in.Resources.DeepCopyInto(&out.Resources) + if in.Affinity != nil { + in, out := &in.Affinity, &out.Affinity + *out = new(corev1.Affinity) + (*in).DeepCopyInto(*out) + } + if in.Tolerations != nil { + in, out := &in.Tolerations, &out.Tolerations + *out = make([]corev1.Toleration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.PriorityClassName != nil { + in, out := &in.PriorityClassName, &out.PriorityClassName + *out = new(string) + **out = **in + } + if in.Expose != nil { + in, out := &in.Expose, &out.Expose + *out = new(ServiceExpose) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LogicalReplicaSpec. +func (in *LogicalReplicaSpec) DeepCopy() *LogicalReplicaSpec { + if in == nil { + return nil + } + out := new(LogicalReplicaSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LogicalReplicaStatus) DeepCopyInto(out *LogicalReplicaStatus) { + *out = *in + if in.Databases != nil { + in, out := &in.Databases, &out.Databases + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.SeededAt != nil { + in, out := &in.SeededAt, &out.SeededAt + *out = (*in).DeepCopy() + } + if in.InvalidatedAt != nil { + in, out := &in.InvalidatedAt, &out.InvalidatedAt + *out = (*in).DeepCopy() + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LogicalReplicaStatus. +func (in *LogicalReplicaStatus) DeepCopy() *LogicalReplicaStatus { + if in == nil { + return nil + } + out := new(LogicalReplicaStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in LogicalReplicas) DeepCopyInto(out *LogicalReplicas) { + { + in := &in + *out = make(LogicalReplicas, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LogicalReplicas. +func (in LogicalReplicas) DeepCopy() LogicalReplicas { + if in == nil { + return nil + } + out := new(LogicalReplicas) + in.DeepCopyInto(out) + return *out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OfflineSnapshotConfig) DeepCopyInto(out *OfflineSnapshotConfig) { *out = *in @@ -1128,6 +1226,13 @@ func (in *PerconaPGClusterSpec) DeepCopyInto(out *PerconaPGClusterSpec) { *out = new(v1beta1.PostgresClusterAuthentication) (*in).DeepCopyInto(*out) } + if in.LogicalReplicas != nil { + in, out := &in.LogicalReplicas, &out.LogicalReplicas + *out = make(LogicalReplicas, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PerconaPGClusterSpec. @@ -1168,6 +1273,13 @@ func (in *PerconaPGClusterStatus) DeepCopyInto(out *PerconaPGClusterStatus) { *out = new(StandbyStatus) (*in).DeepCopyInto(*out) } + if in.LogicalReplicas != nil { + in, out := &in.LogicalReplicas, &out.LogicalReplicas + *out = make([]LogicalReplicaStatus, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PerconaPGClusterStatus. diff --git a/pkg/apis/upstream.pgv2.percona.com/v1beta1/postgrescluster_types.go b/pkg/apis/upstream.pgv2.percona.com/v1beta1/postgrescluster_types.go index 6c78c6d587..fad4d1425f 100644 --- a/pkg/apis/upstream.pgv2.percona.com/v1beta1/postgrescluster_types.go +++ b/pkg/apis/upstream.pgv2.percona.com/v1beta1/postgrescluster_types.go @@ -174,6 +174,14 @@ type PostgresClusterSpec struct { // +optional Standby *PostgresStandbySpec `json:"standby,omitempty"` + // Logical replicas of this cluster, managed by the Percona layer. Only the + // names are mirrored here: this layer uses their presence to render the + // pg_hba rules for the logical replication user and Patroni's ignore_slots. + // +listType=map + // +listMapKey=name + // +optional + LogicalReplicas []LogicalReplicaSpec `json:"logicalReplicas,omitempty"` + // A list of group IDs applied to the process of a container. These can be // useful when accessing shared file systems with constrained permissions. // More info: https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context @@ -833,6 +841,16 @@ type PostgresStandbySpec struct { Port *int32 `json:"port,omitempty"` } +// LogicalReplicaSpec is the projection of a Percona logical replica onto this +// spec. The Percona layer owns the full definition and the whole lifecycle of +// the replica; only what this layer needs to render server configuration is +// carried over. +type LogicalReplicaSpec struct { + // Name of the logical replica. + // +kubebuilder:validation:Required + Name string `json:"name"` +} + // UserInterfaceSpec is a union of the supported PostgreSQL user interfaces. type UserInterfaceSpec struct { // Defines a pgAdmin user interface. diff --git a/pkg/apis/upstream.pgv2.percona.com/v1beta1/zz_generated.deepcopy.go b/pkg/apis/upstream.pgv2.percona.com/v1beta1/zz_generated.deepcopy.go index 7b41f10f42..3e3d364913 100644 --- a/pkg/apis/upstream.pgv2.percona.com/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/upstream.pgv2.percona.com/v1beta1/zz_generated.deepcopy.go @@ -530,6 +530,21 @@ func (in *InstanceSidecars) DeepCopy() *InstanceSidecars { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LogicalReplicaSpec) DeepCopyInto(out *LogicalReplicaSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LogicalReplicaSpec. +func (in *LogicalReplicaSpec) DeepCopy() *LogicalReplicaSpec { + if in == nil { + return nil + } + out := new(LogicalReplicaSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Metadata) DeepCopyInto(out *Metadata) { *out = *in @@ -2085,6 +2100,11 @@ func (in *PostgresClusterSpec) DeepCopyInto(out *PostgresClusterSpec) { *out = new(PostgresStandbySpec) (*in).DeepCopyInto(*out) } + if in.LogicalReplicas != nil { + in, out := &in.LogicalReplicas, &out.LogicalReplicas + *out = make([]LogicalReplicaSpec, len(*in)) + copy(*out, *in) + } if in.SupplementalGroups != nil { in, out := &in.SupplementalGroups, &out.SupplementalGroups *out = make([]int64, len(*in)) From 6ff1a50bfcb3c9cd072cfcf177247d70683f7601 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Mon, 17 Aug 2026 10:24:51 +0300 Subject: [PATCH 2/9] suspend logical replicas when cluster is paused --- .../controller/pgcluster/logicalreplica.go | 9 +- .../pgcluster/logicalreplica_restore.go | 97 +++++----- .../pgcluster/logicalreplica_restore_test.go | 172 +++++++++++++++--- .../v2/perconapgcluster_types.go | 6 + .../v2/perconapgcluster_types_test.go | 23 +++ 5 files changed, 221 insertions(+), 86 deletions(-) diff --git a/percona/controller/pgcluster/logicalreplica.go b/percona/controller/pgcluster/logicalreplica.go index 6e1d0e462a..b92ceb64da 100644 --- a/percona/controller/pgcluster/logicalreplica.go +++ b/percona/controller/pgcluster/logicalreplica.go @@ -121,15 +121,12 @@ func (r *PGClusterReconciler) reconcileLogicalReplicas( return false, nil } - // Must come first: a restore deletes the instance StatefulSets, so the state - // gate below would skip the teardown too. No requeue - both ends of a restore - // write this CR or a PerconaPGRestore, and this controller watches both. - restore, err := r.observeSourceRestore(ctx, cr) + suspension, err := r.shouldSuspendLogicalReplicas(ctx, cr) if err != nil { return false, errors.Wrap(err, "observe restore") } - if restore.InFlight { - return false, r.suspendLogicalReplicas(ctx, cr, restore) + if suspension.Needed { + return false, r.suspendLogicalReplicas(ctx, cr, suspension) } // Runs even when the section is empty: that is the case where the last diff --git a/percona/controller/pgcluster/logicalreplica_restore.go b/percona/controller/pgcluster/logicalreplica_restore.go index ffab409e3c..41c675f3aa 100644 --- a/percona/controller/pgcluster/logicalreplica_restore.go +++ b/percona/controller/pgcluster/logicalreplica_restore.go @@ -20,46 +20,50 @@ import ( v2 "github.com/percona/percona-postgresql-operator/v2/pkg/apis/pgv2.percona.com/v2" ) -// sourceRestore is what a restore of the cluster means for its logical -// replicas. -type sourceRestore struct { - // InFlight means a restore has been asked for and has not finished, so the - // primary is about to go away or already has. - InFlight bool - - // DataReplaced means the restore has passed the point where the cluster could - // still be put back the way it was. - DataReplaced bool +type suspension struct { + Needed bool + + // Invalidate communicates that logical replication can not continue + // User needs to fix it by re-seeding the replica + Invalidate bool + + Reason string + Message string } -// observeSourceRestore reports what the restores of this cluster mean for its -// logical replicas. -// -// It never looks at the restore Job: both prepareForRestore and this operator's -// own PGBackRestRestore.Start blank status.pgbackrest.restore, so anything -// derived from it is ambiguous exactly while a restore is starting. -func (r *PGClusterReconciler) observeSourceRestore( - ctx context.Context, cr *v2.PerconaPGCluster, -) (sourceRestore, error) { - restore := sourceRestore{} +// shouldSuspendLogicalReplicas reports the need of suspending logical replicas +// and the reason for suspension +func (r *PGClusterReconciler) shouldSuspendLogicalReplicas(ctx context.Context, cr *v2.PerconaPGCluster) (suspension, error) { + s := suspension{} + + if cr.IsPaused() { + s.Needed = true + s.Reason = v2.LogicalReplicaReasonClusterPaused + s.Message = "the cluster is paused" + } // Set by PGBackRestRestore.Start before anything is torn down and cleared by // DisableRestore on every terminal outcome. The earliest signal there is, and // both edges write this CR, so the controller is woken for free. if enabled := cr.Spec.Backups.PGBackRest.Restore; enabled != nil && enabled.Enabled != nil && *enabled.Enabled { - restore.InFlight = true + s.Needed = true + s.Reason = v2.LogicalReplicaReasonSourceRestoring + s.Message = "the cluster is being restored in place" } if cr.GetAnnotations()[naming.PGBackRestRestore] != "" { - restore.InFlight = true + s.Needed = true + s.Reason = v2.LogicalReplicaReasonSourceRestoring + s.Message = "the cluster is being restored in place" } // Raised by prepareForRestore as it deletes the instance runners. Nothing // removes it when a restore fails, which is right: a half-restored data // directory invalidates a replica just as thoroughly as a finished one. - if meta.IsStatusConditionTrue(cr.Status.Conditions, - postgrescluster.ConditionPGBackRestRestoreProgressing) { - restore.DataReplaced = true + if meta.IsStatusConditionTrue(cr.Status.Conditions, postgrescluster.ConditionPGBackRestRestoreProgressing) { + s.Invalidate = true + s.Reason = v2.LogicalReplicaReasonSourceRestored + s.Message = "logical replica invalidated by a restore of the cluster" } // The only signal that covers a snapshot restore with no point-in-time @@ -68,7 +72,7 @@ func (r *PGClusterReconciler) observeSourceRestore( // not find the slots missing either. restores := &v2.PerconaPGRestoreList{} if err := r.Client.List(ctx, restores, client.InNamespace(cr.Namespace)); err != nil { - return restore, errors.Wrap(err, "list restores") + return s, errors.Wrap(err, "list restores") } for i := range restores.Items { @@ -83,27 +87,23 @@ func (r *PGClusterReconciler) observeSourceRestore( // the two signals above in the same pass that moves it out of this state. switch pgRestore.Status.State { case v2.RestoreRunning: - restore.InFlight = true - restore.DataReplaced = true + s.Needed = true + s.Invalidate = true + s.Reason = v2.LogicalReplicaReasonSourceRestored + s.Message = "logical replica invalidated by a restore of the cluster" case v2.RestoreStarting: - restore.InFlight = true + s.Needed = true + s.Reason = v2.LogicalReplicaReasonSourceRestoring + s.Message = "the cluster is being restored in place" default: } } - return restore, nil + return s, nil } -// suspendLogicalReplicas stops every logical replica for the duration of a -// restore of the cluster they replicate. See -// [v2.LogicalReplicaReasonSourceRestored] for why they cannot keep running. -// -// Whether they can be resumed afterwards is deliberately not decided here: a -// restore that fails before it touches the data directory leaves them perfectly -// valid. Nothing is destroyed and nothing is forgotten. -func (r *PGClusterReconciler) suspendLogicalReplicas( - ctx context.Context, cr *v2.PerconaPGCluster, restore sourceRestore, -) error { +// suspendLogicalReplicas stops every logical replica +func (r *PGClusterReconciler) suspendLogicalReplicas(ctx context.Context, cr *v2.PerconaPGCluster, s suspension) error { log := logging.FromContext(ctx).WithName("LogicalReplication") // Driven by the status rather than the spec: a replica removed from the @@ -113,7 +113,6 @@ func (r *PGClusterReconciler) suspendLogicalReplicas( statuses := make([]v2.LogicalReplicaStatus, 0, len(cr.Status.LogicalReplicas)) for i := range cr.Status.LogicalReplicas { status := cr.Status.LogicalReplicas[i].DeepCopy() - status.Reason = v2.LogicalReplicaReasonSourceRestoring if err := r.scaleLogicalReplica(ctx, cr, status.Name, 0); err != nil { return errors.Wrapf(err, "stop logical replica %q", status.Name) @@ -122,11 +121,11 @@ func (r *PGClusterReconciler) suspendLogicalReplicas( switch { case status.SeededAt != nil: status.State = v2.LogicalReplicaStateSuspended - status.Message = "the cluster is being restored in place" + status.Reason = s.Reason + status.Message = s.Message - if restore.DataReplaced && status.InvalidatedAt == nil { - log.Info("logical replica invalidated by a restore of the cluster", - "logicalReplica", status.Name) + if s.Invalidate && status.InvalidatedAt == nil { + log.Info("logical replica invalidated by a restore of the cluster", "logicalReplica", status.Name) status.InvalidatedAt = new(metav1.Now()) } @@ -140,20 +139,18 @@ func (r *PGClusterReconciler) suspendLogicalReplicas( } status.State = v2.LogicalReplicaStateBootstrapping - status.Message = "the bootstrap was cancelled because the cluster is being restored in place" + status.Message = "the bootstrap was cancelled because the logical replica is suspended" status.Databases = nil } statuses = append(statuses, *status) } - // Synthesised rather than observed: observePrimaryReadiness execs on a - // primary that a restore has taken away, and the answer is known anyway. readiness := metav1.Condition{ Type: pNaming.ConditionReadyForLogicalReplication, Status: metav1.ConditionFalse, - Reason: v2.LogicalReplicaReasonSourceRestoring, - Message: "the cluster is being restored in place; logical replicas are stopped", + Reason: s.Reason, + Message: s.Message, } return r.updateLogicalReplicaStatus(ctx, cr, statuses, &readiness) diff --git a/percona/controller/pgcluster/logicalreplica_restore_test.go b/percona/controller/pgcluster/logicalreplica_restore_test.go index 2c6e713d46..32eea09f32 100644 --- a/percona/controller/pgcluster/logicalreplica_restore_test.go +++ b/percona/controller/pgcluster/logicalreplica_restore_test.go @@ -15,12 +15,14 @@ import ( batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/percona/percona-postgresql-operator/v2/internal/controller/postgrescluster" "github.com/percona/percona-postgresql-operator/v2/internal/naming" + pNaming "github.com/percona/percona-postgresql-operator/v2/percona/naming" v2 "github.com/percona/percona-postgresql-operator/v2/pkg/apis/pgv2.percona.com/v2" crunchyv1beta1 "github.com/percona/percona-postgresql-operator/v2/pkg/apis/upstream.pgv2.percona.com/v1beta1" ) @@ -49,17 +51,36 @@ func pgRestoreFor(cr *v2.PerconaPGCluster, name string, state v2.PGRestoreState) } } -func TestObserveSourceRestore(t *testing.T) { +func TestShouldSuspendLogicalReplicas(t *testing.T) { for _, tt := range []struct { - name string - mutate func(*v2.PerconaPGCluster) - restores []*v2.PerconaPGRestore - inFlight bool - dataReplaced bool + name string + mutate func(*v2.PerconaPGCluster) + restores []*v2.PerconaPGRestore + needed bool + invalidate bool + reason string + message string }{ { name: "nothing in progress", }, + { + // A paused cluster has no primary to replicate from, but nothing + // has touched the data directory, so the replicas stay valid. + name: "a paused cluster", + mutate: func(cr *v2.PerconaPGCluster) { + cr.Spec.Pause = new(true) + }, + needed: true, + reason: v2.LogicalReplicaReasonClusterPaused, + message: "the cluster is paused", + }, + { + name: "an unpaused cluster", + mutate: func(cr *v2.PerconaPGCluster) { + cr.Spec.Pause = new(false) + }, + }, { name: "the spec flag alone", // Set by PGBackRestRestore.Start before anything is torn down, so @@ -69,7 +90,9 @@ func TestObserveSourceRestore(t *testing.T) { Enabled: new(true), } }, - inFlight: true, + needed: true, + reason: v2.LogicalReplicaReasonSourceRestoring, + message: "the cluster is being restored in place", }, { name: "a disabled spec flag is not a restore", @@ -84,7 +107,9 @@ func TestObserveSourceRestore(t *testing.T) { mutate: func(cr *v2.PerconaPGCluster) { cr.Annotations = map[string]string{naming.PGBackRestRestore: "restore1"} }, - inFlight: true, + needed: true, + reason: v2.LogicalReplicaReasonSourceRestoring, + message: "the cluster is being restored in place", }, { // With no restore left in flight this is a restore that failed: @@ -99,20 +124,39 @@ func TestObserveSourceRestore(t *testing.T) { Reason: "ReadyForRestore", }} }, - dataReplaced: true, + invalidate: true, + reason: v2.LogicalReplicaReasonSourceRestored, + message: "logical replica invalidated by a restore of the cluster", }, { // The only signal that covers a snapshot restore with no // point-in-time recovery, which never sets the two above. name: "a starting restore", restores: []*v2.PerconaPGRestore{{}}, - inFlight: true, + needed: true, + reason: v2.LogicalReplicaReasonSourceRestoring, + message: "the cluster is being restored in place", + }, + { + name: "a running restore", + restores: []*v2.PerconaPGRestore{{}}, + needed: true, + invalidate: true, + reason: v2.LogicalReplicaReasonSourceRestored, + message: "logical replica invalidated by a restore of the cluster", }, { - name: "a running restore", - restores: []*v2.PerconaPGRestore{{}}, - inFlight: true, - dataReplaced: true, + // A restore of a paused cluster: the restore is the more specific + // answer, and the only one that invalidates. + name: "a running restore of a paused cluster", + mutate: func(cr *v2.PerconaPGCluster) { + cr.Spec.Pause = new(true) + }, + restores: []*v2.PerconaPGRestore{{}}, + needed: true, + invalidate: true, + reason: v2.LogicalReplicaReasonSourceRestored, + message: "logical replica invalidated by a restore of the cluster", }, } { t.Run(tt.name, func(t *testing.T) { @@ -133,7 +177,7 @@ func TestObserveSourceRestore(t *testing.T) { } if len(tt.restores) > 0 { state := v2.RestoreStarting - if tt.dataReplaced { + if tt.invalidate { state = v2.RestoreRunning } objs = append(objs, pgRestoreFor(cr, "current", state)) @@ -143,11 +187,13 @@ func TestObserveSourceRestore(t *testing.T) { require.NoError(t, err) r := &PGClusterReconciler{Client: cl} - restore, err := r.observeSourceRestore(t.Context(), cr) + s, err := r.shouldSuspendLogicalReplicas(t.Context(), cr) require.NoError(t, err) - assert.Equal(t, tt.inFlight, restore.InFlight, "InFlight") - assert.Equal(t, tt.dataReplaced, restore.DataReplaced, "DataReplaced") + assert.Equal(t, tt.needed, s.Needed, "Needed") + assert.Equal(t, tt.invalidate, s.Invalidate, "Invalidate") + assert.Equal(t, tt.reason, s.Reason, "Reason") + assert.Equal(t, tt.message, s.Message, "Message") }) } } @@ -155,6 +201,22 @@ func TestObserveSourceRestore(t *testing.T) { func TestSuspendLogicalReplicas(t *testing.T) { spec := v2.LogicalReplicaSpec{Name: "analytics"} + // What shouldSuspendLogicalReplicas reports for a restore that has started + // but has not yet touched the data directory. + restoring := suspension{ + Needed: true, + Reason: v2.LogicalReplicaReasonSourceRestoring, + Message: "the cluster is being restored in place", + } + + // ... and once it has, which is what invalidates a replica. + restored := suspension{ + Needed: true, + Invalidate: true, + Reason: v2.LogicalReplicaReasonSourceRestored, + Message: "logical replica invalidated by a restore of the cluster", + } + statefulSet := func(cr *v2.PerconaPGCluster) *appsv1.StatefulSet { return &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ @@ -196,8 +258,7 @@ func TestSuspendLogicalReplicas(t *testing.T) { require.NoError(t, err) r := &PGClusterReconciler{Client: cl} - require.NoError(t, r.suspendLogicalReplicas(t.Context(), cr, - sourceRestore{InFlight: true})) + require.NoError(t, r.suspendLogicalReplicas(t.Context(), cr, restoring)) sts := new(appsv1.StatefulSet) require.NoError(t, cl.Get(t.Context(), client.ObjectKeyFromObject(statefulSet(cr)), sts)) @@ -216,6 +277,54 @@ func TestSuspendLogicalReplicas(t *testing.T) { assert.Equal(t, []string{"cluster1"}, status.Databases) }) + t.Run("a paused cluster suspends without invalidating", func(t *testing.T) { + // Pausing takes the primary away just as a restore does, but it leaves + // the data directory alone, so unpausing brings the replica back. + paused := suspension{ + Needed: true, + Reason: v2.LogicalReplicaReasonClusterPaused, + Message: "the cluster is paused", + } + + cr := restoreTestCluster(t, spec) + cr.Spec.Pause = new(true) + cr.Status.LogicalReplicas = []v2.LogicalReplicaStatus{{ + Name: spec.Name, + State: v2.LogicalReplicaStateReady, + Databases: []string{"cluster1"}, + SeededAt: new(metav1.Now()), + }} + + cl, err := buildFakeClient(t.Context(), cr, statefulSet(cr), dataVolume(cr)) + require.NoError(t, err) + r := &PGClusterReconciler{Client: cl} + + require.NoError(t, r.suspendLogicalReplicas(t.Context(), cr, paused)) + + sts := new(appsv1.StatefulSet) + require.NoError(t, cl.Get(t.Context(), client.ObjectKeyFromObject(statefulSet(cr)), sts)) + assert.Equal(t, int32(0), *sts.Spec.Replicas) + + status := recorded(t, cl, cr) + assert.Equal(t, v2.LogicalReplicaStateSuspended, status.State) + assert.Equal(t, v2.LogicalReplicaReasonClusterPaused, status.Reason) + assert.Equal(t, paused.Message, status.Message) + assert.Nil(t, status.InvalidatedAt, "a pause destroys nothing") + assert.NotNil(t, status.SeededAt) + + // The condition says why, so unpausing is all it takes to undo this. + updated := new(v2.PerconaPGCluster) + require.NoError(t, cl.Get(t.Context(), + client.ObjectKey{Name: cr.Name, Namespace: cr.Namespace}, updated)) + + condition := meta.FindStatusCondition(updated.Status.Conditions, + pNaming.ConditionReadyForLogicalReplication) + require.NotNil(t, condition) + assert.Equal(t, metav1.ConditionFalse, condition.Status) + assert.Equal(t, v2.LogicalReplicaReasonClusterPaused, condition.Reason) + assert.Equal(t, paused.Message, condition.Message) + }) + t.Run("a seeded replica is invalidated once the data is replaced", func(t *testing.T) { cr := restoreTestCluster(t, spec) cr.Status.LogicalReplicas = []v2.LogicalReplicaStatus{{ @@ -229,8 +338,7 @@ func TestSuspendLogicalReplicas(t *testing.T) { require.NoError(t, err) r := &PGClusterReconciler{Client: cl} - require.NoError(t, r.suspendLogicalReplicas(t.Context(), cr, - sourceRestore{InFlight: true, DataReplaced: true})) + require.NoError(t, r.suspendLogicalReplicas(t.Context(), cr, restored)) assert.NotNil(t, recorded(t, cl, cr).InvalidatedAt) }) @@ -255,8 +363,7 @@ func TestSuspendLogicalReplicas(t *testing.T) { require.NoError(t, err) r := &PGClusterReconciler{Client: cl} - require.NoError(t, r.suspendLogicalReplicas(t.Context(), cr, - sourceRestore{InFlight: true, DataReplaced: true})) + require.NoError(t, r.suspendLogicalReplicas(t.Context(), cr, restored)) err = cl.Get(t.Context(), client.ObjectKeyFromObject(job), new(batchv1.Job)) assert.True(t, apierrors.IsNotFound(err), "the bootstrap job must be deleted: %v", err) @@ -291,8 +398,7 @@ func TestSuspendLogicalReplicas(t *testing.T) { require.NoError(t, err) r := &PGClusterReconciler{Client: cl} - require.NoError(t, r.suspendLogicalReplicas(t.Context(), cr, - sourceRestore{InFlight: true, DataReplaced: true})) + require.NoError(t, r.suspendLogicalReplicas(t.Context(), cr, restored)) require.NoError(t, cl.Get(t.Context(), client.ObjectKeyFromObject(dataVolume(cr)), new(corev1.PersistentVolumeClaim))) @@ -315,8 +421,7 @@ func TestSuspendLogicalReplicas(t *testing.T) { require.NoError(t, err) r := &PGClusterReconciler{Client: cl} - require.NoError(t, r.suspendLogicalReplicas(t.Context(), cr, - sourceRestore{InFlight: true, DataReplaced: true})) + require.NoError(t, r.suspendLogicalReplicas(t.Context(), cr, restored)) require.NoError(t, cl.Get(t.Context(), client.ObjectKeyFromObject(dataVolume(cr)), new(corev1.PersistentVolumeClaim))) @@ -337,8 +442,7 @@ func TestSuspendLogicalReplicas(t *testing.T) { require.NoError(t, err) r := &PGClusterReconciler{Client: cl} - require.NoError(t, r.suspendLogicalReplicas(t.Context(), cr, - sourceRestore{InFlight: true})) + require.NoError(t, r.suspendLogicalReplicas(t.Context(), cr, restoring)) assert.Equal(t, []string{"cluster1"}, recorded(t, cl, cr).Databases) }) @@ -371,6 +475,14 @@ func TestRestoreDoesNotErrorWhenReplicaRemoved(t *testing.T) { client.ObjectKey{Name: cr.Name, Namespace: cr.Namespace}, updated)) require.Len(t, updated.Status.LogicalReplicas, 1, "the teardown is deferred, not forgotten") + + // The spec flag is the earliest restore signal and the only one set here, so + // it has to name a reason of its own: the API server rejects an empty one. + condition := meta.FindStatusCondition(updated.Status.Conditions, + pNaming.ConditionReadyForLogicalReplication) + require.NotNil(t, condition) + assert.Equal(t, v2.LogicalReplicaReasonSourceRestoring, condition.Reason) + assert.NotEmpty(t, condition.Message) } // TestCleanupDefersWithoutPrimary covers the WAL leak: a replica whose teardown diff --git a/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go b/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go index 84c68d17a8..fe51a49162 100644 --- a/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go +++ b/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go @@ -286,6 +286,10 @@ type LogicalReplicaSpec struct { Expose *ServiceExpose `json:"expose,omitempty"` } +func (cr *PerconaPGCluster) IsPaused() bool { + return cr.Spec.Pause != nil && *cr.Spec.Pause == true +} + // LogicalReplicaBootstrapMethod selects how the data volume of a logical // replica is seeded with a physical copy of the primary, before // pg_createsubscriber converts it into a subscriber. @@ -891,6 +895,8 @@ const ( // says which prerequisite is missing. LogicalReplicaReasonPrimaryNotReady = "PrimaryNotReady" + LogicalReplicaReasonClusterPaused = "ClusterPaused" + LogicalReplicaReasonSourceRestoring = "SourceRestoring" // LogicalReplicaReasonSourceRestored means the data directory this replica was diff --git a/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types_test.go b/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types_test.go index ae0c78d719..4e100a93fd 100644 --- a/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types_test.go +++ b/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types_test.go @@ -83,6 +83,29 @@ func TestPerconaPGCluster_BackupsEnabled(t *testing.T) { } } +func TestPerconaPGCluster_IsPaused(t *testing.T) { + trueVal := true + falseVal := false + + tests := map[string]struct { + pause *bool + expected bool + }{ + "unset means running": {pause: nil}, + "explicitly false": {pause: &falseVal}, + "true": {pause: &trueVal, expected: true}, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + cr := new(PerconaPGCluster) + cr.Spec.Pause = tt.pause + + assert.Equal(t, tt.expected, cr.IsPaused()) + }) + } +} + func TestPerconaPGCluster_Validate(t *testing.T) { t.Run("rejects pg_stat_monitor and pg_stat_statements together", func(t *testing.T) { cluster := new(PerconaPGCluster) From 05b6430f62f03cc6586cfce704b96d9e98e1baf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Mon, 17 Aug 2026 10:33:32 +0300 Subject: [PATCH 3/9] remove primary ready condition if all logical replicas are removed --- percona/controller/pgcluster/logicalreplica.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/percona/controller/pgcluster/logicalreplica.go b/percona/controller/pgcluster/logicalreplica.go index b92ceb64da..d411d183f8 100644 --- a/percona/controller/pgcluster/logicalreplica.go +++ b/percona/controller/pgcluster/logicalreplica.go @@ -116,9 +116,8 @@ func (r *PGClusterReconciler) reconcileLogicalReplicas( return false, nil } - // Fast path: no API calls for clusters that never used the feature. if len(cr.Spec.LogicalReplicas) == 0 && len(cr.Status.LogicalReplicas) == 0 { - return false, nil + return false, r.updateLogicalReplicaStatus(ctx, cr, cr.Status.LogicalReplicas, nil) } suspension, err := r.shouldSuspendLogicalReplicas(ctx, cr) @@ -158,7 +157,6 @@ func (r *PGClusterReconciler) reconcileLogicalReplicas( log := logging.FromContext(ctx).WithName("LogicalReplication") ctx = logging.NewContext(ctx, log) - // Once per reconcile: cluster-wide, and it costs an exec. readiness := r.observePrimaryReadiness(ctx, cr) // Replicas whose teardown could not be finished keep their status entry: From f2ef6d10dfeb637070a6a9187adefd4e6b8e6e57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Mon, 17 Aug 2026 10:36:36 +0300 Subject: [PATCH 4/9] update cr example to empty databases --- deploy/cr.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/deploy/cr.yaml b/deploy/cr.yaml index 22866790ff..e305a1fcf0 100644 --- a/deploy/cr.yaml +++ b/deploy/cr.yaml @@ -378,8 +378,7 @@ spec: # logicalReplicas: # - name: analytics -# databases: -# - cluster1 +# databases: [] # empty = all non-template databases except "postgres" # bootstrapMethod: pgbackrest # dataVolumeClaimSpec: # accessModes: From 823e938c7bfd66ef06a4073127aaa3e4c73ab676 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Mon, 17 Aug 2026 11:12:00 +0300 Subject: [PATCH 5/9] bootstrap logical replicas one by one --- percona/controller/pgcluster/logicalreplica.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/percona/controller/pgcluster/logicalreplica.go b/percona/controller/pgcluster/logicalreplica.go index d411d183f8..47b4af2e79 100644 --- a/percona/controller/pgcluster/logicalreplica.go +++ b/percona/controller/pgcluster/logicalreplica.go @@ -164,6 +164,7 @@ func (r *PGClusterReconciler) reconcileLogicalReplicas( statuses := make([]v2.LogicalReplicaStatus, 0, len(cr.Spec.LogicalReplicas)+len(deferred)) statuses = append(statuses, deferred...) requeue := len(deferred) > 0 + for i := range cr.Spec.LogicalReplicas { spec := &cr.Spec.LogicalReplicas[i] @@ -179,11 +180,18 @@ func (r *PGClusterReconciler) reconcileLogicalReplicas( status.Message = err.Error() } + statuses = append(statuses, *status) + + // bootstrap logical replicas one by one + // otherwise we might think primary has enough free slots + // even when it hasn't to accomodate all pending replicas + if status.State == v2.LogicalReplicaStateBootstrapping { + return true, r.updateLogicalReplicaStatus(ctx, cr, statuses, &readiness) + } + if !logicalReplicaSettled(status) { requeue = true } - - statuses = append(statuses, *status) } return requeue, r.updateLogicalReplicaStatus(ctx, cr, statuses, &readiness) From ae64bfbd848d4ef02dcc81550c4a2bea78c85b1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Mon, 17 Aug 2026 11:18:36 +0300 Subject: [PATCH 6/9] add uniqueness validation for logical replicas and databases --- .../percona/generated/pgv2.percona.com_perconapgclusters.yaml | 4 ++++ config/crd/bases/pgv2.percona.com_perconapgclusters.yaml | 4 ++++ deploy/bundle.yaml | 4 ++++ deploy/crd.yaml | 4 ++++ deploy/cw-bundle.yaml | 4 ++++ pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go | 3 +++ 6 files changed, 23 insertions(+) diff --git a/build/crd/percona/generated/pgv2.percona.com_perconapgclusters.yaml b/build/crd/percona/generated/pgv2.percona.com_perconapgclusters.yaml index 37a1f1ff4f..a45647c73f 100644 --- a/build/crd/percona/generated/pgv2.percona.com_perconapgclusters.yaml +++ b/build/crd/percona/generated/pgv2.percona.com_perconapgclusters.yaml @@ -12890,6 +12890,7 @@ spec: minLength: 1 type: string type: array + x-kubernetes-list-type: set expose: properties: annotations: @@ -12988,6 +12989,9 @@ spec: - name type: object type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map metadata: properties: annotations: diff --git a/config/crd/bases/pgv2.percona.com_perconapgclusters.yaml b/config/crd/bases/pgv2.percona.com_perconapgclusters.yaml index ca737d09da..2d531bc2e2 100644 --- a/config/crd/bases/pgv2.percona.com_perconapgclusters.yaml +++ b/config/crd/bases/pgv2.percona.com_perconapgclusters.yaml @@ -13304,6 +13304,7 @@ spec: minLength: 1 type: string type: array + x-kubernetes-list-type: set expose: properties: annotations: @@ -13402,6 +13403,9 @@ spec: - name type: object type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map metadata: properties: annotations: diff --git a/deploy/bundle.yaml b/deploy/bundle.yaml index 7e1c39ffdf..6a066d4f94 100644 --- a/deploy/bundle.yaml +++ b/deploy/bundle.yaml @@ -13505,6 +13505,7 @@ spec: minLength: 1 type: string type: array + x-kubernetes-list-type: set expose: properties: annotations: @@ -13603,6 +13604,9 @@ spec: - name type: object type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map metadata: properties: annotations: diff --git a/deploy/crd.yaml b/deploy/crd.yaml index 3af3c94930..c84524dd52 100644 --- a/deploy/crd.yaml +++ b/deploy/crd.yaml @@ -13505,6 +13505,7 @@ spec: minLength: 1 type: string type: array + x-kubernetes-list-type: set expose: properties: annotations: @@ -13603,6 +13604,9 @@ spec: - name type: object type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map metadata: properties: annotations: diff --git a/deploy/cw-bundle.yaml b/deploy/cw-bundle.yaml index 78c90861a7..4034dcb65e 100644 --- a/deploy/cw-bundle.yaml +++ b/deploy/cw-bundle.yaml @@ -13505,6 +13505,7 @@ spec: minLength: 1 type: string type: array + x-kubernetes-list-type: set expose: properties: annotations: @@ -13603,6 +13604,9 @@ spec: - name type: object type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map metadata: properties: annotations: diff --git a/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go b/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go index fe51a49162..8eddb5dc95 100644 --- a/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go +++ b/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go @@ -217,6 +217,8 @@ type PerconaPGClusterSpec struct { LogicalReplicas LogicalReplicas `json:"logicalReplicas,omitempty"` } +// +listType=map +// +listMapKey=name type LogicalReplicas []LogicalReplicaSpec // ToCrunchy projects the logical replicas onto the Crunchy spec, which needs @@ -244,6 +246,7 @@ type LogicalReplicaSpec struct { // Databases to replicate. When empty, every database in the cluster except // the templates and "postgres" is replicated. + // +listType=set // +optional Databases []crunchyv1beta1.PostgresIdentifier `json:"databases,omitempty"` From 4f28e8db49089947752ebbd5bfcae7b843430446 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Mon, 17 Aug 2026 17:02:30 +0300 Subject: [PATCH 7/9] address george --- internal/logicalreplica/postgres.go | 28 +++++++++---------- internal/logicalreplica/postgres_test.go | 6 ++-- .../pgcluster/logicalreplica_test.go | 23 ++++++++------- .../v2/perconapgcluster_types_test.go | 7 ++--- 4 files changed, 30 insertions(+), 34 deletions(-) diff --git a/internal/logicalreplica/postgres.go b/internal/logicalreplica/postgres.go index f67cf04d78..4d5b053ca1 100644 --- a/internal/logicalreplica/postgres.go +++ b/internal/logicalreplica/postgres.go @@ -18,7 +18,7 @@ const ( // ReplicationUser must match v2.UserLogicalReplication. ReplicationUser = "logicalrepl" - OutputPlugin = "pgoutput" + outputPlugin = "pgoutput" // maxIdentifierLength is PostgreSQL's NAMEDATALEN-1. maxIdentifierLength = 63 @@ -32,11 +32,11 @@ const ( // Reasons reported by [PrimaryReadinessQuery], rendered by // [PrimaryReadinessMessage]. const ( - ReasonPrimaryInRecovery = "PrimaryInRecovery" - ReasonWALLevelNotLogical = "WALLevelNotLogical" + reasonPrimaryInRecovery = "PrimaryInRecovery" + reasonWALLevelNotLogical = "WALLevelNotLogical" + reasonReplicationRoleNotReady = "ReplicationRoleNotReady" + reasonReplicationHBAMissing = "ReplicationHBAMissing" ReasonRestartPending = "RestartPending" - ReasonReplicationRoleNotReady = "ReplicationRoleNotReady" - ReasonReplicationHBAMissing = "ReplicationHBAMissing" ) // PrimaryReadinessQuery returns a query whose single column lists, comma @@ -70,11 +70,11 @@ func PrimaryReadinessQuery() string { ) AS c(n, name, satisfied) WHERE NOT c.satisfied;`, postgres.QuoteLiteral(ReplicationUser), - postgres.QuoteLiteral(ReasonPrimaryInRecovery), - postgres.QuoteLiteral(ReasonWALLevelNotLogical), + postgres.QuoteLiteral(reasonPrimaryInRecovery), + postgres.QuoteLiteral(reasonWALLevelNotLogical), postgres.QuoteLiteral(ReasonRestartPending), - postgres.QuoteLiteral(ReasonReplicationRoleNotReady), - postgres.QuoteLiteral(ReasonReplicationHBAMissing), + postgres.QuoteLiteral(reasonReplicationRoleNotReady), + postgres.QuoteLiteral(reasonReplicationHBAMissing), postgres.QuoteLiteral(hbaOrigin), postgres.QuoteLiteral(hbaAuthMethod)) } @@ -97,15 +97,15 @@ func ParsePrimaryReadinessReasons(stdout string) []string { // condition message. func PrimaryReadinessMessage(reason string) string { switch reason { - case ReasonPrimaryInRecovery: + case reasonPrimaryInRecovery: return "the primary is in recovery; pg_createsubscriber needs a writable publisher" - case ReasonWALLevelNotLogical: + case reasonWALLevelNotLogical: return `"wal_level" is not "logical"; set it back via spec.patroni.dynamicConfiguration` case ReasonRestartPending: return "PostgreSQL has a parameter change pending a restart, which would cut off the bootstrap" - case ReasonReplicationRoleNotReady: + case reasonReplicationRoleNotReady: return "the " + ReplicationUser + " role has not been created yet" - case ReasonReplicationHBAMissing: + case reasonReplicationHBAMissing: return "the pg_hba rules that let " + ReplicationUser + " reach the primary have not been written yet" default: return reason @@ -151,7 +151,7 @@ func IgnoreSlotsMatchers(inCluster *v1beta1.PostgresCluster) []any { return []any{map[string]any{ "type": "logical", - "plugin": OutputPlugin, + "plugin": outputPlugin, }} } diff --git a/internal/logicalreplica/postgres_test.go b/internal/logicalreplica/postgres_test.go index 5e1399aef2..1830cd6a44 100644 --- a/internal/logicalreplica/postgres_test.go +++ b/internal/logicalreplica/postgres_test.go @@ -77,8 +77,8 @@ func TestPrimaryReadinessQuery(t *testing.T) { // Every reason the parser can report has to be one the caller knows how to // render, and each has to be reachable. for _, reason := range []string{ - ReasonPrimaryInRecovery, ReasonWALLevelNotLogical, ReasonRestartPending, - ReasonReplicationRoleNotReady, ReasonReplicationHBAMissing, + reasonPrimaryInRecovery, reasonWALLevelNotLogical, ReasonRestartPending, + reasonReplicationRoleNotReady, reasonReplicationHBAMissing, } { assert.Equal(t, strings.Count(query, postgres.QuoteLiteral(reason)), 1, reason) assert.Assert(t, PrimaryReadinessMessage(reason) != reason, reason) @@ -161,7 +161,7 @@ func TestIgnoreSlotsMatchers(t *testing.T) { matcher, ok := matchers[0].(map[string]any) assert.Assert(t, ok) assert.Equal(t, matcher["type"], "logical") - assert.Equal(t, matcher["plugin"], OutputPlugin) + assert.Equal(t, matcher["plugin"], outputPlugin) _, hasName := matcher["name"] assert.Assert(t, !hasName) } diff --git a/percona/controller/pgcluster/logicalreplica_test.go b/percona/controller/pgcluster/logicalreplica_test.go index 06070b7b43..02bec3b987 100644 --- a/percona/controller/pgcluster/logicalreplica_test.go +++ b/percona/controller/pgcluster/logicalreplica_test.go @@ -921,7 +921,7 @@ func TestObservePrimaryReadiness(t *testing.T) { cond := r.observePrimaryReadiness(t.Context(), cr) assert.Equal(t, metav1.ConditionFalse, cond.Status) - assert.Equal(t, logicalreplica.ReasonRestartPending, cond.Reason) + assert.Equal(t, "RestartPending", cond.Reason) assert.False(t, rec.called, "a pending restart is decided before the query") }) @@ -947,34 +947,33 @@ func TestObservePrimaryReadiness(t *testing.T) { t.Run("one unmet prerequisite", func(t *testing.T) { cr := newCluster() rec := new(execRecorder) - r := newReconciler(t, cr, rec, logicalreplica.ReasonReplicationHBAMissing, nil, + r := newReconciler(t, cr, rec, "ReplicationHBAMissing", nil, primaryPodForCluster(cr), logicalReplicaUserSecret(cr)) cond := r.observePrimaryReadiness(t.Context(), cr) assert.Equal(t, metav1.ConditionFalse, cond.Status) - assert.Equal(t, logicalreplica.ReasonReplicationHBAMissing, cond.Reason) + assert.Equal(t, "ReplicationHBAMissing", cond.Reason) assert.Equal(t, - logicalreplica.PrimaryReadinessMessage(logicalreplica.ReasonReplicationHBAMissing), + logicalreplica.PrimaryReadinessMessage("ReplicationHBAMissing"), cond.Message) }) t.Run("several unmet prerequisites", func(t *testing.T) { cr := newCluster() rec := new(execRecorder) - r := newReconciler(t, cr, rec, - logicalreplica.ReasonRestartPending+","+logicalreplica.ReasonReplicationHBAMissing, nil, + r := newReconciler(t, cr, rec, "RestartPending,ReplicationHBAMissing", nil, primaryPodForCluster(cr), logicalReplicaUserSecret(cr)) cond := r.observePrimaryReadiness(t.Context(), cr) // The reason is the first, but the message has to name all of them: the // user fixes them together. - assert.Equal(t, logicalreplica.ReasonRestartPending, cond.Reason) + assert.Equal(t, "RestartPending", cond.Reason) assert.Contains(t, cond.Message, - logicalreplica.PrimaryReadinessMessage(logicalreplica.ReasonRestartPending)) + logicalreplica.PrimaryReadinessMessage("RestartPending")) assert.Contains(t, cond.Message, - logicalreplica.PrimaryReadinessMessage(logicalreplica.ReasonReplicationHBAMissing)) + logicalreplica.PrimaryReadinessMessage("ReplicationHBAMissing")) }) t.Run("primary cannot be queried", func(t *testing.T) { @@ -1484,7 +1483,7 @@ func TestUpdateLogicalReplicaStatus(t *testing.T) { require.NoError(t, r.updateLogicalReplicaStatus(t.Context(), cr, statuses, &metav1.Condition{ Type: pNaming.ConditionReadyForLogicalReplication, Status: metav1.ConditionFalse, - Reason: logicalreplica.ReasonReplicationHBAMissing, + Reason: "ReplicationHBAMissing", Message: "first", })) @@ -1506,13 +1505,13 @@ func TestUpdateLogicalReplicaStatus(t *testing.T) { require.NoError(t, r.updateLogicalReplicaStatus(t.Context(), cr, statuses, &metav1.Condition{ Type: pNaming.ConditionReadyForLogicalReplication, Status: metav1.ConditionFalse, - Reason: logicalreplica.ReasonRestartPending, + Reason: "RestartPending", Message: "second", })) cond := meta.FindStatusCondition(read().Status.Conditions, pNaming.ConditionReadyForLogicalReplication) - assert.Equal(t, logicalreplica.ReasonRestartPending, cond.Reason) + assert.Equal(t, "RestartPending", cond.Reason) assert.Equal(t, "second", cond.Message) assert.True(t, before.Equal(&cond.LastTransitionTime)) }) diff --git a/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types_test.go b/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types_test.go index 4e100a93fd..f35a64121c 100644 --- a/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types_test.go +++ b/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types_test.go @@ -84,16 +84,13 @@ func TestPerconaPGCluster_BackupsEnabled(t *testing.T) { } func TestPerconaPGCluster_IsPaused(t *testing.T) { - trueVal := true - falseVal := false - tests := map[string]struct { pause *bool expected bool }{ "unset means running": {pause: nil}, - "explicitly false": {pause: &falseVal}, - "true": {pause: &trueVal, expected: true}, + "explicitly false": {pause: new(false)}, + "true": {pause: new(true), expected: true}, } for name, tt := range tests { From c30b6a4f2d717883fb1e9fb0751bfdb6a9736261 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Tue, 18 Aug 2026 09:31:19 +0300 Subject: [PATCH 8/9] fix envtest and linters --- internal/logicalreplica/postgres.go | 2 +- percona/controller/pgcluster/controller.go | 2 +- percona/controller/pgcluster/logicalreplica.go | 4 ++-- .../pgcluster/logicalreplica_envtest_test.go | 15 +++++++++------ .../pgcluster/logicalreplica_restore.go | 2 +- 5 files changed, 14 insertions(+), 11 deletions(-) diff --git a/internal/logicalreplica/postgres.go b/internal/logicalreplica/postgres.go index 4d5b053ca1..10fd1a1c03 100644 --- a/internal/logicalreplica/postgres.go +++ b/internal/logicalreplica/postgres.go @@ -180,7 +180,7 @@ func DisableOnErrorSQL(replica, db string) string { } // identifier folds the given parts to [a-z0-9_], caps the result at -// NAMEDATALEN-1, and always appends a hash so that names which sanitise or +// NAMEDATALEN-1, and always appends a hash so that names which sanitize or // truncate alike stay distinct. func identifier(prefix, replica, db string) string { sum := sha256.Sum256([]byte(replica + "\x00" + db)) diff --git a/percona/controller/pgcluster/controller.go b/percona/controller/pgcluster/controller.go index 5c6ca133c6..e272cc5569 100644 --- a/percona/controller/pgcluster/controller.go +++ b/percona/controller/pgcluster/controller.go @@ -137,7 +137,7 @@ func (r *PGClusterReconciler) watchServices() handler.TypedFuncs[*corev1.Service } // watchJobs enqueues the cluster of a Job whose status changed, as long as the -// Job carries ownerLabel. Every Job the operator creates is labelled with its +// Job carries ownerLabel. Every Job the operator creates is labeled with its // cluster, so LabelCluster alone would wake the reconciler for all of them. func (r *PGClusterReconciler) watchJobs(ownerLabel string) handler.TypedFuncs[*batchv1.Job, reconcile.Request] { return handler.TypedFuncs[*batchv1.Job, reconcile.Request]{ diff --git a/percona/controller/pgcluster/logicalreplica.go b/percona/controller/pgcluster/logicalreplica.go index 47b4af2e79..4fea2db840 100644 --- a/percona/controller/pgcluster/logicalreplica.go +++ b/percona/controller/pgcluster/logicalreplica.go @@ -184,7 +184,7 @@ func (r *PGClusterReconciler) reconcileLogicalReplicas( // bootstrap logical replicas one by one // otherwise we might think primary has enough free slots - // even when it hasn't to accomodate all pending replicas + // even when it hasn't to accommodate all pending replicas if status.State == v2.LogicalReplicaStateBootstrapping { return true, r.updateLogicalReplicaStatus(ctx, cr, statuses, &readiness) } @@ -602,7 +602,7 @@ func (r *PGClusterReconciler) reconcileLogicalReplicaPVC( // and report success, and the bootstrap Job would then mount a volume that // disappears from under it - or the old data, which the Job refuses to seed // over. A replica removed from the spec and added straight back, or a - // cancelled bootstrap, opens that window. + // canceled bootstrap, opens that window. existing := &corev1.PersistentVolumeClaim{} switch err := r.Client.Get(ctx, client.ObjectKeyFromObject(pvc), existing); { case err == nil && existing.DeletionTimestamp != nil: diff --git a/percona/controller/pgcluster/logicalreplica_envtest_test.go b/percona/controller/pgcluster/logicalreplica_envtest_test.go index 02fdc414dd..0581fdea36 100644 --- a/percona/controller/pgcluster/logicalreplica_envtest_test.go +++ b/percona/controller/pgcluster/logicalreplica_envtest_test.go @@ -170,12 +170,15 @@ var _ = Describe("Logical replicas", Ordered, func() { }} for _, reason := range []string{ - "PrimaryPodNotFound", "ReplicationSecretMissing", "PrimaryUnreachable", "PrimaryReady", - logicalreplica.ReasonPrimaryInRecovery, - logicalreplica.ReasonWALLevelNotLogical, - logicalreplica.ReasonRestartPending, - logicalreplica.ReasonReplicationRoleNotReady, - logicalreplica.ReasonReplicationHBAMissing, + "PrimaryPodNotFound", + "ReplicationSecretMissing", + "PrimaryUnreachable", + "PrimaryReady", + "PrimaryInRecovery", + "WALLevelNotLogical", + "RestartPending", + "ReplicationRoleNotReady", + "ReplicationHBAMissing", } { Expect(r.updateLogicalReplicaStatus(ctx, cr, statuses, &metav1.Condition{ Type: pNaming.ConditionReadyForLogicalReplication, diff --git a/percona/controller/pgcluster/logicalreplica_restore.go b/percona/controller/pgcluster/logicalreplica_restore.go index 41c675f3aa..3ee2293f78 100644 --- a/percona/controller/pgcluster/logicalreplica_restore.go +++ b/percona/controller/pgcluster/logicalreplica_restore.go @@ -139,7 +139,7 @@ func (r *PGClusterReconciler) suspendLogicalReplicas(ctx context.Context, cr *v2 } status.State = v2.LogicalReplicaStateBootstrapping - status.Message = "the bootstrap was cancelled because the logical replica is suspended" + status.Message = "the bootstrap was canceled because the logical replica is suspended" status.Databases = nil } From e584bf2144a13521496d6b08d70ad79221346aea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Tue, 18 Aug 2026 09:36:10 +0300 Subject: [PATCH 9/9] fix golangci-lint --- pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go b/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go index 8eddb5dc95..85cda707c5 100644 --- a/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go +++ b/pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go @@ -290,7 +290,7 @@ type LogicalReplicaSpec struct { } func (cr *PerconaPGCluster) IsPaused() bool { - return cr.Spec.Pause != nil && *cr.Spec.Pause == true + return cr.Spec.Pause != nil && *cr.Spec.Pause } // LogicalReplicaBootstrapMethod selects how the data volume of a logical