From 003239c55c96bb3a0fa71b037afa2606214ce1bf Mon Sep 17 00:00:00 2001 From: Andrii Dema Date: Wed, 12 Aug 2026 00:57:43 +0300 Subject: [PATCH 1/5] K8SPG-1045: rewrite `cert-management-policy` test https://perconadev.atlassian.net/browse/K8SPG-1045 --- e2e-tests/functions | 479 +++++++++++------- .../00-deploy-operator.yaml | 2 +- .../cert-management-policy/01-assert.yaml | 90 +--- ...al-cluster.yaml => 01-create-cluster.yaml} | 4 +- .../02-create-root-ca.yaml | 29 ++ .../02-save-secrets.yaml | 33 -- .../03-create-postgres-certificate.yaml | 46 ++ .../03-delete-cluster-and-secrets.yaml | 30 -- .../cert-management-policy/04-assert.yaml | 19 - .../04-create-replication-certificate.yaml | 30 ++ .../04-create-user-provided-cluster.yaml | 17 - .../cert-management-policy/05-assert.yaml | 36 -- .../05-create-pgbackrest-certificates.yaml | 36 ++ .../05-restore-secrets.yaml | 13 - .../cert-management-policy/06-assert.yaml | 60 --- .../06-create-pgbouncer-certificate.yaml | 51 ++ .../06-restore-instance-secret.yaml | 23 - .../cert-management-policy/07-assert.yaml | 21 + ...-update-pgbackrest-client-certificate.yaml | 46 ++ .../09-create-instance-certificate.yaml | 56 ++ .../cert-management-policy/10-assert.yaml | 69 +++ .../11-verify-served-certificates.yaml | 59 +++ .../12-verify-secrets-unchanged.yaml | 45 ++ .../99-remove-cluster-gracefully.yaml | 1 + percona/controller/pgcluster/status.go | 4 +- percona/controller/pgcluster/status_test.go | 1 + 26 files changed, 791 insertions(+), 509 deletions(-) rename e2e-tests/tests/cert-management-policy/{01-create-initial-cluster.yaml => 01-create-cluster.yaml} (71%) create mode 100644 e2e-tests/tests/cert-management-policy/02-create-root-ca.yaml delete mode 100644 e2e-tests/tests/cert-management-policy/02-save-secrets.yaml create mode 100644 e2e-tests/tests/cert-management-policy/03-create-postgres-certificate.yaml delete mode 100644 e2e-tests/tests/cert-management-policy/03-delete-cluster-and-secrets.yaml delete mode 100644 e2e-tests/tests/cert-management-policy/04-assert.yaml create mode 100644 e2e-tests/tests/cert-management-policy/04-create-replication-certificate.yaml delete mode 100644 e2e-tests/tests/cert-management-policy/04-create-user-provided-cluster.yaml delete mode 100644 e2e-tests/tests/cert-management-policy/05-assert.yaml create mode 100644 e2e-tests/tests/cert-management-policy/05-create-pgbackrest-certificates.yaml delete mode 100644 e2e-tests/tests/cert-management-policy/05-restore-secrets.yaml delete mode 100644 e2e-tests/tests/cert-management-policy/06-assert.yaml create mode 100644 e2e-tests/tests/cert-management-policy/06-create-pgbouncer-certificate.yaml delete mode 100644 e2e-tests/tests/cert-management-policy/06-restore-instance-secret.yaml create mode 100644 e2e-tests/tests/cert-management-policy/07-assert.yaml create mode 100644 e2e-tests/tests/cert-management-policy/08-update-pgbackrest-client-certificate.yaml create mode 100644 e2e-tests/tests/cert-management-policy/09-create-instance-certificate.yaml create mode 100644 e2e-tests/tests/cert-management-policy/10-assert.yaml create mode 100644 e2e-tests/tests/cert-management-policy/11-verify-served-certificates.yaml create mode 100644 e2e-tests/tests/cert-management-policy/12-verify-secrets-unchanged.yaml diff --git a/e2e-tests/functions b/e2e-tests/functions index b4604a42ed..9ff65a0ba4 100644 --- a/e2e-tests/functions +++ b/e2e-tests/functions @@ -288,9 +288,9 @@ get_repo_auth() { local secret=$4 auth="${repo}-${type}-key=${key}\n${repo}-${type}-key-secret=${secret}" - if [[ "$type" == "azure" ]]; then + if [[ $type == "azure" ]]; then auth="${repo}-${type}-account=${key}\n${repo}-${type}-key=${secret}" - elif [[ "$type" == "gcs" ]]; then + elif [[ $type == "gcs" ]]; then auth="${repo}-${type}-key=${key}" fi @@ -312,11 +312,11 @@ deploy_s3_secrets() { case $test_name in "demand-backup") repo_auth+=$(get_repo_auth repo3 s3 $key $secret) - ;; + ;; "scheduled-backup") repo_auth+=$(get_repo_auth repo2 s3 $key $secret) repo_auth+=$(get_repo_auth repo3 s3 $key $secret) - ;; + ;; esac else key=$(yq eval 'select(.metadata.name=="*s3*").data.AWS_ACCESS_KEY_ID' "$TESTS_CONFIG_DIR/cloud-secret.yml" | base64 -d) @@ -328,22 +328,22 @@ deploy_s3_secrets() { case $test_name in "demand-backup") repo_auth+=$(get_repo_auth repo3 azure $azure_key $azure_secret) - ;; + ;; "scheduled-backup") repo_auth+=$(get_repo_auth repo2 gcs "/etc/pgbackrest/conf.d/gcs-key.json") repo_auth+=$(get_repo_auth repo3 azure $azure_key $azure_secret) - yq eval '.stringData["credentials.json"]' "$TESTS_CONFIG_DIR/cloud-secret-minio-gw.yml" > "$TEMP_DIR/gcs-key.json" + yq eval '.stringData["credentials.json"]' "$TESTS_CONFIG_DIR/cloud-secret-minio-gw.yml" >"$TEMP_DIR/gcs-key.json" secret_args+=(--from-file=gcs-key.json="$TEMP_DIR/gcs-key.json") - ;; + ;; esac fi - printf "[global]%b" "$repo_auth" > "$TEMP_DIR/pgbackrest-secret.ini" + printf "[global]%b" "$repo_auth" >"$TEMP_DIR/pgbackrest-secret.ini" secret_args+=(--from-file=cloud.conf="$TEMP_DIR/pgbackrest-secret.ini") kubectl -n "$NAMESPACE" create secret generic "${test_name}-pgbackrest-secrets" "${secret_args[@]}" case $test_name in - "custom-extensions" | "builtin-extensions" | major-upgrade* ) + "custom-extensions" | "builtin-extensions" | major-upgrade*) kubectl -n "$NAMESPACE" apply -f "$TESTS_CONFIG_DIR/cloud-secret.yml" kubectl -n "$NAMESPACE" apply -f "$TESTS_CONFIG_DIR/minio-secret.yml" ;; @@ -357,7 +357,7 @@ deploy_client() { } deploy_cmctl() { - envsubst < "${TESTS_CONFIG_DIR}/cmctl.yml" | kubectl -n "${NAMESPACE}" apply -f - + envsubst <"${TESTS_CONFIG_DIR}/cmctl.yml" | kubectl -n "${NAMESPACE}" apply -f - kubectl -n "${NAMESPACE}" wait deployment cmctl --for=condition=available --timeout=60s } @@ -377,14 +377,14 @@ get_cr() { local cr_file="$TEST_CONFIG_DIR/${cr_name}.yaml" local spaces_cr_file="$TEST_CONFIG_DIR/${cr_name}-spaces.yaml" local aks_cr_file="$TEST_CONFIG_DIR/${cr_name}-aks.yaml" - if [[ $platform == "digitalocean" && -f "$spaces_cr_file" ]]; then + if [[ $platform == "digitalocean" && -f $spaces_cr_file ]]; then cr_file=$spaces_cr_file - elif [[ $platform == "aks" && -f "$aks_cr_file" ]]; then + elif [[ $platform == "aks" && -f $aks_cr_file ]]; then cr_file=$aks_cr_file fi local crs=("$DEPLOY_DIR/cr.yaml") - if [[ -f "$cr_file" ]]; then + if [[ -f $cr_file ]]; then crs+=($cr_file) fi @@ -404,7 +404,7 @@ get_cr() { .spec.pmm.secret = "'${cr_name}'-pmm-secret" | .spec.pmm.customClusterName = "'${cr_name}'-pmm-custom-name" | .spec.pmm.postgresParams = "--environment=dev-postgres" - ' "${crs[@]}" > "$TEMP_DIR/cr.yaml" + ' "${crs[@]}" >"$TEMP_DIR/cr.yaml" $sed -i "s||$repo_path|g" "$TEMP_DIR/cr.yaml" $sed -i "s||$source_path|g" "$TEMP_DIR/cr.yaml" @@ -447,7 +447,7 @@ run_psql_command() { local command=${1} local uri=${2} local driver=${3:-postgres} - + kubectl -n ${NAMESPACE} exec $(get_client_pod) -- \ psql -v ON_ERROR_STOP=1 -t -q "${driver}://${uri}" -c "${command}" } @@ -1174,10 +1174,10 @@ get_container_image() { if [[ ${IMAGE} == percona/* || ${IMAGE} == docker.io/percona/* ]] || [[ -n ${FORCE_RELEASE_RUN} ]]; then local key case $component in - pgbouncer) key="IMAGE_PGBOUNCER${pgVersion}" ;; + pgbouncer) key="IMAGE_PGBOUNCER${pgVersion}" ;; pgbackrest) key="IMAGE_BACKREST${pgVersion}" ;; - postgis) key="IMAGE_POSTGIS${pgVersion}" ;; - *) key="IMAGE_POSTGRESQL${pgVersion}" ;; + postgis) key="IMAGE_POSTGIS${pgVersion}" ;; + *) key="IMAGE_POSTGRESQL${pgVersion}" ;; esac get_release_image "$key" return @@ -1236,94 +1236,94 @@ wait_for_generation() { # Reads all environment variables from a pod (single kubectl exec call) get_envs_from_pod() { - local namespace=$1 - local pod=$2 - local container=$3 + local namespace=$1 + local pod=$2 + local container=$3 - if [ -n "$container" ]; then - kubectl exec -n "$namespace" "$pod" -c "$container" -- printenv 2>/dev/null || true - else - kubectl exec -n "$namespace" "$pod" -- printenv 2>/dev/null || true - fi + if [ -n "$container" ]; then + kubectl exec -n "$namespace" "$pod" -c "$container" -- printenv 2>/dev/null || true + else + kubectl exec -n "$namespace" "$pod" -- printenv 2>/dev/null || true + fi } # Verifies a single variable in given env content check_env_in_pod() { - local check_type=$1 - local pod=$2 - local var_name=$3 - local expected_value=$4 - local env_content=$5 - - local actual_value - actual_value=$(echo "$env_content" | grep -E "^${var_name}=" | cut -d'=' -f2- || true) - - if [[ "$check_type" == "add" ]]; then - if [ "$actual_value" != "$expected_value" ]; then - echo "ERROR: $var_name in $pod — expected '$expected_value', got '${actual_value:-}'" - return 1 - else - echo "OK: $var_name=$actual_value in $pod" - fi - elif [[ "$check_type" == "delete" ]]; then - if [ -n "$actual_value" ]; then - echo "ERROR: $var_name exists in $pod (should not exist)" - return 1 - else - echo "OK: $var_name deleted in $pod" - fi - else - echo "ERROR: unknown check type '$check_type'" - return 1 - fi + local check_type=$1 + local pod=$2 + local var_name=$3 + local expected_value=$4 + local env_content=$5 + + local actual_value + actual_value=$(echo "$env_content" | grep -E "^${var_name}=" | cut -d'=' -f2- || true) + + if [[ $check_type == "add" ]]; then + if [ "$actual_value" != "$expected_value" ]; then + echo "ERROR: $var_name in $pod — expected '$expected_value', got '${actual_value:-}'" + return 1 + else + echo "OK: $var_name=$actual_value in $pod" + fi + elif [[ $check_type == "delete" ]]; then + if [ -n "$actual_value" ]; then + echo "ERROR: $var_name exists in $pod (should not exist)" + return 1 + else + echo "OK: $var_name deleted in $pod" + fi + else + echo "ERROR: unknown check type '$check_type'" + return 1 + fi } # Checks multiple env vars in one or more components check_envs_for_component() { - local check_type=$1 # add | delete - local component=$2 # instance | pgbouncer | repohost - local vars=("${@:3}") # everything after the 2nd argument - - case "$component" in - instance) - POD=$(kubectl get -n "${NAMESPACE}" pod -l postgres-operator.crunchydata.com/instance-set=instance1 -o 'jsonpath={.items[0].metadata.name}') - CONTAINER="" - ;; - pgbouncer) - POD=$(kubectl get -n "${NAMESPACE}" pod -l postgres-operator.crunchydata.com/role=pgbouncer -o 'jsonpath={.items[0].metadata.name}') - CONTAINER="pgbouncer" - ;; - repohost) - POD=$(kubectl get -n "${NAMESPACE}" pod -l postgres-operator.crunchydata.com/data=pgbackrest -o 'jsonpath={.items[0].metadata.name}') - CONTAINER="pgbackrest" - ;; - *) - echo "ERROR: unknown component '$component'" - return 1 - ;; - esac - - echo "Fetching environment variables for $component pod $POD..." - local env_content - env_content=$(get_envs_from_pod "${NAMESPACE}" "$POD" "$CONTAINER") - - local errors=0 - for var_entry in "${vars[@]}"; do - if [[ "$check_type" == "add" ]]; then - local var_name="${var_entry%%=*}" - local var_expected="${var_entry#*=}" - check_env_in_pod add "$POD" "$var_name" "$var_expected" "$env_content" || errors=$((errors+1)) - else - check_env_in_pod delete "$POD" "$var_entry" "" "$env_content" || errors=$((errors+1)) - fi - done - - if (( errors > 0 )); then - echo "$errors environment check(s) failed for component '$component'" - return 1 - else - echo "All environment checks passed for component '$component'" - fi + local check_type=$1 # add | delete + local component=$2 # instance | pgbouncer | repohost + local vars=("${@:3}") # everything after the 2nd argument + + case "$component" in + instance) + POD=$(kubectl get -n "${NAMESPACE}" pod -l postgres-operator.crunchydata.com/instance-set=instance1 -o 'jsonpath={.items[0].metadata.name}') + CONTAINER="" + ;; + pgbouncer) + POD=$(kubectl get -n "${NAMESPACE}" pod -l postgres-operator.crunchydata.com/role=pgbouncer -o 'jsonpath={.items[0].metadata.name}') + CONTAINER="pgbouncer" + ;; + repohost) + POD=$(kubectl get -n "${NAMESPACE}" pod -l postgres-operator.crunchydata.com/data=pgbackrest -o 'jsonpath={.items[0].metadata.name}') + CONTAINER="pgbackrest" + ;; + *) + echo "ERROR: unknown component '$component'" + return 1 + ;; + esac + + echo "Fetching environment variables for $component pod $POD..." + local env_content + env_content=$(get_envs_from_pod "${NAMESPACE}" "$POD" "$CONTAINER") + + local errors=0 + for var_entry in "${vars[@]}"; do + if [[ $check_type == "add" ]]; then + local var_name="${var_entry%%=*}" + local var_expected="${var_entry#*=}" + check_env_in_pod add "$POD" "$var_name" "$var_expected" "$env_content" || errors=$((errors + 1)) + else + check_env_in_pod delete "$POD" "$var_entry" "" "$env_content" || errors=$((errors + 1)) + fi + done + + if ((errors > 0)); then + echo "$errors environment check(s) failed for component '$component'" + return 1 + else + echo "All environment checks passed for component '$component'" + fi } detect_k8s_platform() { @@ -1354,7 +1354,7 @@ detect_k8s_platform() { ensure_ebs_gp3_storage_class() { local platform=${1:-$(detect_k8s_platform)} - if [[ "$platform" != "eks" && "$platform" != "openshift" ]]; then + if [[ $platform != "eks" && $platform != "openshift" ]]; then return 0 fi @@ -1401,21 +1401,21 @@ enable_hugepages() { } enable_hugepages_gke() { - local hugepage_count=$1 + local hugepage_count=$1 - echo "Hugepages configuration is now handled during cluster creation" - echo "Skipping runtime configuration for GKE" + echo "Hugepages configuration is now handled during cluster creation" + echo "Skipping runtime configuration for GKE" - echo "Verifying hugepages configuration..." - local node_name=$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}') + echo "Verifying hugepages configuration..." + local node_name=$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}') - if kubectl get node ${node_name} -o jsonpath='{.status.allocatable.hugepages-2Mi}' | grep -q '[0-9]'; then - echo "✓ Hugepages are already configured on nodes" - return 0 - else - echo "✗ Warning: Hugepages not found in node allocatable resources" - return 1 - fi + if kubectl get node ${node_name} -o jsonpath='{.status.allocatable.hugepages-2Mi}' | grep -q '[0-9]'; then + echo "✓ Hugepages are already configured on nodes" + return 0 + else + echo "✗ Warning: Hugepages not found in node allocatable resources" + return 1 + fi } enable_hugepages_eks() { @@ -1430,8 +1430,8 @@ enable_hugepages_eks() { enable_hugepages_doks() { local hugepage_count=$1 - - if verify_hugepages_on_nodes; then + + if verify_hugepages_on_nodes; then echo "Hugepages already configured" return 0 fi @@ -1638,128 +1638,128 @@ spec: path: /etc/sysctl.d/99-hugepages.conf EOF - [ $? -eq 0 ] || return 1 + [ $? -eq 0 ] || return 1 - echo "MachineConfig created" - echo "Waiting for worker pool to update (~10 minutes)..." + echo "MachineConfig created" + echo "Waiting for worker pool to update (~10 minutes)..." - kubectl wait --for=condition=Updated mcp/worker --timeout=900s 2>/dev/null || { - echo "Update taking longer than expected" - return 1 - } + kubectl wait --for=condition=Updated mcp/worker --timeout=900s 2>/dev/null || { + echo "Update taking longer than expected" + return 1 + } - echo "Worker pool updated" + echo "Worker pool updated" - sleep 10 - verify_hugepages_on_nodes + sleep 10 + verify_hugepages_on_nodes } verify_hugepages_on_nodes() { - echo "Verifying hugepages on nodes" + echo "Verifying hugepages on nodes" - # Get first worker node, fallback to first non-master, fallback to any node - local node_name=$( - kubectl get nodes -l node-role.kubernetes.io/worker -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || \ - kubectl get nodes -l '!node-role.kubernetes.io/master,!node-role.kubernetes.io/control-plane' -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || \ - kubectl get nodes -o jsonpath='{.items[0].metadata.name}' - ) + # Get first worker node, fallback to first non-master, fallback to any node + local node_name=$( + kubectl get nodes -l node-role.kubernetes.io/worker -o jsonpath='{.items[0].metadata.name}' 2>/dev/null \ + || kubectl get nodes -l '!node-role.kubernetes.io/master,!node-role.kubernetes.io/control-plane' -o jsonpath='{.items[0].metadata.name}' 2>/dev/null \ + || kubectl get nodes -o jsonpath='{.items[0].metadata.name}' + ) - if [ -z "${node_name}" ]; then - echo "No nodes found" - return 1 - fi + if [ -z "${node_name}" ]; then + echo "No nodes found" + return 1 + fi - echo "Checking node: ${node_name}" + echo "Checking node: ${node_name}" - local hugepages_capacity=$(kubectl get node ${node_name} \ - -o jsonpath='{.status.capacity.hugepages-2Mi}') + local hugepages_capacity=$(kubectl get node ${node_name} \ + -o jsonpath='{.status.capacity.hugepages-2Mi}') - if [ -n "${hugepages_capacity}" ] && [ "${hugepages_capacity}" != "0" ]; then - echo "Node has hugepages capacity: ${hugepages_capacity}" - return 0 - else - echo "No hugepages capacity found on node ${node_name}" - return 1 - fi + if [ -n "${hugepages_capacity}" ] && [ "${hugepages_capacity}" != "0" ]; then + echo "Node has hugepages capacity: ${hugepages_capacity}" + return 0 + else + echo "No hugepages capacity found on node ${node_name}" + return 1 + fi } verify_hugepages_in_pod() { - local pod_name=$1 - local namespace=$2 - local container=${3:-postgres} + local pod_name=$1 + local namespace=$2 + local container=${3:-postgres} - echo "Verifying hugepages in pod ${pod_name}" + echo "Verifying hugepages in pod ${pod_name}" - # Check /proc/meminfo - local hugepages_total=$(kubectl exec ${pod_name} -n ${namespace} -c ${container} -- \ - grep HugePages_Total /proc/meminfo | awk '{print $2}') + # Check /proc/meminfo + local hugepages_total=$(kubectl exec ${pod_name} -n ${namespace} -c ${container} -- \ + grep HugePages_Total /proc/meminfo | awk '{print $2}') - local hugepages_free=$(kubectl exec ${pod_name} -n ${namespace} -c ${container} -- \ - grep HugePages_Free /proc/meminfo | awk '{print $2}') + local hugepages_free=$(kubectl exec ${pod_name} -n ${namespace} -c ${container} -- \ + grep HugePages_Free /proc/meminfo | awk '{print $2}') - echo "HugePages_Total: ${hugepages_total}" - echo "HugePages_Free: ${hugepages_free}" + echo "HugePages_Total: ${hugepages_total}" + echo "HugePages_Free: ${hugepages_free}" - if [ "${hugepages_total}" -gt 0 ]; then - echo "Hugepages are available in pod" - return 0 - else - echo "No hugepages in pod" - return 1 - fi + if [ "${hugepages_total}" -gt 0 ]; then + echo "Hugepages are available in pod" + return 0 + else + echo "No hugepages in pod" + return 1 + fi } verify_postgresql_hugepages_setting() { - local cluster_name=$1 - local expected_value=${2:-try} + local cluster_name=$1 + local expected_value=${2:-try} - echo "Verifying PostgreSQL huge_pages setting..." + echo "Verifying PostgreSQL huge_pages setting..." - local huge_pages=$(run_psql_local \ - "SHOW huge_pages;" \ - "postgres:$(get_psql_user_pass ${cluster_name}-pguser-postgres)@$(get_psql_user_host ${cluster_name}-pguser-postgres)") + local huge_pages=$(run_psql_local \ + "SHOW huge_pages;" \ + "postgres:$(get_psql_user_pass ${cluster_name}-pguser-postgres)@$(get_psql_user_host ${cluster_name}-pguser-postgres)") - echo "huge_pages: ${huge_pages}" + echo "huge_pages: ${huge_pages}" - if [[ "${huge_pages}" == *"${expected_value}"* ]]; then - echo "PostgreSQL huge_pages is set to '${expected_value}'" - return 0 - else - echo "PostgreSQL huge_pages not set to '${expected_value}' (value: ${huge_pages})" - return 1 - fi + if [[ ${huge_pages} == *"${expected_value}"* ]]; then + echo "PostgreSQL huge_pages is set to '${expected_value}'" + return 0 + else + echo "PostgreSQL huge_pages not set to '${expected_value}' (value: ${huge_pages})" + return 1 + fi } verify_hugepages_usage() { - local pod_name=$1 - local namespace=$2 - local container=${3:-database} + local pod_name=$1 + local namespace=$2 + local container=${3:-database} - echo "Checking hugepages usage..." + echo "Checking hugepages usage..." - kubectl -n ${namespace} exec ${pod_name} -c ${container} -- \ - grep HugePages /proc/meminfo + kubectl -n ${namespace} exec ${pod_name} -c ${container} -- \ + grep HugePages /proc/meminfo - local hugepages_total=$(kubectl -n ${namespace} exec ${pod_name} -c ${container} -- \ - grep HugePages_Total /proc/meminfo | awk '{print $2}') + local hugepages_total=$(kubectl -n ${namespace} exec ${pod_name} -c ${container} -- \ + grep HugePages_Total /proc/meminfo | awk '{print $2}') - local hugepages_free=$(kubectl -n ${namespace} exec ${pod_name} -c ${container} -- \ - grep HugePages_Free /proc/meminfo | awk '{print $2}') + local hugepages_free=$(kubectl -n ${namespace} exec ${pod_name} -c ${container} -- \ + grep HugePages_Free /proc/meminfo | awk '{print $2}') - local hugepages_used=$((hugepages_total - hugepages_free)) + local hugepages_used=$((hugepages_total - hugepages_free)) - echo "" - echo "HugePages usage:" - echo " Total: ${hugepages_total}" - echo " Used: ${hugepages_used}" + echo "" + echo "HugePages usage:" + echo " Total: ${hugepages_total}" + echo " Used: ${hugepages_used}" - if [ "${hugepages_used}" -gt 0 ]; then - echo "PostgreSQL is using hugepages" - return 0 - else - echo "Hugepages available but NOT being used by PostgreSQL" - return 1 - fi + if [ "${hugepages_used}" -gt 0 ]; then + echo "PostgreSQL is using hugepages" + return 0 + else + echo "Hugepages available but NOT being used by PostgreSQL" + return 1 + fi } get_available_k8s_versions() { @@ -2177,3 +2177,88 @@ storage \"file\" { --from-literal=token=${token} \ --from-file=ca.crt=${tmp_dir}/vault.ca } + +issue_root_ca() { + cat >"$TEMP_DIR/root-ca.conf" < 64)); then + printf 'certificate Common Name exceeds 64 characters: %s\n' \ + "$common_name" >&2 + return 1 + fi + + cat >"$TEMP_DIR/${name}.conf" <>"$TEMP_DIR/${name}.conf" + index=$((index + 1)) + done + + openssl ecparam -name prime256v1 -genkey \ + -noout -out "$TEMP_DIR/${name}.key" + openssl req -new \ + -key "$TEMP_DIR/${name}.key" \ + -out "$TEMP_DIR/${name}.csr" \ + -config "$TEMP_DIR/${name}.conf" + openssl x509 -req \ + -in "$TEMP_DIR/${name}.csr" \ + -CA "$TEMP_DIR/root-ca.crt" \ + -CAkey "$TEMP_DIR/root-ca.key" \ + -CAcreateserial \ + -days 365 \ + -sha384 \ + -extensions extensions \ + -extfile "$TEMP_DIR/${name}.conf" \ + -out "$TEMP_DIR/${name}.crt" +} diff --git a/e2e-tests/tests/cert-management-policy/00-deploy-operator.yaml b/e2e-tests/tests/cert-management-policy/00-deploy-operator.yaml index 4dc6d5cbe2..e54aae9fef 100644 --- a/e2e-tests/tests/cert-management-policy/00-deploy-operator.yaml +++ b/e2e-tests/tests/cert-management-policy/00-deploy-operator.yaml @@ -9,5 +9,5 @@ commands: source ../../functions init_temp_dir - destroy_cert_manager + deploy_cert_manager deploy_operator diff --git a/e2e-tests/tests/cert-management-policy/01-assert.yaml b/e2e-tests/tests/cert-management-policy/01-assert.yaml index 1f6cfdb0cc..89f5eca5a2 100644 --- a/e2e-tests/tests/cert-management-policy/01-assert.yaml +++ b/e2e-tests/tests/cert-management-policy/01-assert.yaml @@ -1,79 +1,15 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert -timeout: 300 ---- -apiVersion: v1 -kind: Secret -metadata: - name: cert-management-policy-cluster-ca-cert ---- -apiVersion: v1 -kind: Secret -metadata: - name: cert-management-policy-cluster-cert ---- -apiVersion: v1 -kind: Secret -metadata: - name: cert-management-policy-replication-cert ---- -kind: StatefulSet -apiVersion: apps/v1 -metadata: - labels: - postgres-operator.crunchydata.com/cluster: cert-management-policy - postgres-operator.crunchydata.com/data: postgres - postgres-operator.crunchydata.com/instance-set: instance1 -status: - observedGeneration: 1 - replicas: 1 - readyReplicas: 1 - currentReplicas: 1 - updatedReplicas: 1 - collisionCount: 0 ---- -kind: Deployment -apiVersion: apps/v1 -metadata: - name: cert-management-policy-pgbouncer - labels: - postgres-operator.crunchydata.com/cluster: cert-management-policy - postgres-operator.crunchydata.com/role: pgbouncer -status: - observedGeneration: 1 - replicas: 1 - updatedReplicas: 1 - readyReplicas: 1 ---- -apiVersion: upstream.pgv2.percona.com/v1beta1 -kind: PostgresCluster -metadata: - name: cert-management-policy -status: - instances: - - name: instance1 - readyReplicas: 1 - replicas: 1 - updatedReplicas: 1 - observedGeneration: 1 - proxy: - pgBouncer: - readyReplicas: 1 - replicas: 1 ---- -apiVersion: pgv2.percona.com/v2 -kind: PerconaPGCluster -metadata: - name: cert-management-policy -status: - pgbouncer: - ready: 1 - size: 1 - postgres: - instances: - - name: instance1 - ready: 1 - size: 1 - ready: 1 - size: 1 - state: ready +timeout: 180 +resourceRefs: + - apiVersion: pgv2.percona.com/v2 + kind: PerconaPGCluster + name: cert-management-policy + ref: cluster +assertAll: + - celExpr: cluster.status.conditions.exists(c, c.type == "ReadyForBackup" && c.status == "False" && c.reason == "PGBackRestRepoHostReady" && has(c.observedGeneration) && c.observedGeneration == cluster.metadata.generation) + message: ReadyForBackup should be False with reason PGBackRestRepoHostReady + - celExpr: cluster.status.conditions.exists(c, c.type == "TLSSecretsReady" && c.status == "False" && c.reason == "TLSSecretsMissing" && has(c.observedGeneration) && c.observedGeneration == cluster.metadata.generation) + message: TLSSecretsReady should be False with reason TLSSecretsMissing + - celExpr: cluster.status.conditions.exists(c, c.type == "Progressing" && c.status == "False" && c.reason == "Paused" && has(c.observedGeneration) && c.observedGeneration == cluster.metadata.generation) + message: Progressing should be False with reason Paused diff --git a/e2e-tests/tests/cert-management-policy/01-create-initial-cluster.yaml b/e2e-tests/tests/cert-management-policy/01-create-cluster.yaml similarity index 71% rename from e2e-tests/tests/cert-management-policy/01-create-initial-cluster.yaml rename to e2e-tests/tests/cert-management-policy/01-create-cluster.yaml index dc15a63f45..6e83c569a4 100644 --- a/e2e-tests/tests/cert-management-policy/01-create-initial-cluster.yaml +++ b/e2e-tests/tests/cert-management-policy/01-create-cluster.yaml @@ -9,7 +9,7 @@ commands: source ../../functions get_cr "cert-management-policy" \ - | yq '.spec.tls.certManagementPolicy = "auto"' \ + | yq '.spec.tls.certManagementPolicy = "userProvidedOnly"' \ | yq '.spec.proxy.pgBouncer.replicas = 1' \ - | yq '.spec.instances[].replicas = 1' \ + | yq '.spec.instances[].replicas = 3' \ | kubectl -n "$NAMESPACE" apply -f - diff --git a/e2e-tests/tests/cert-management-policy/02-create-root-ca.yaml b/e2e-tests/tests/cert-management-policy/02-create-root-ca.yaml new file mode 100644 index 0000000000..4687f9ea22 --- /dev/null +++ b/e2e-tests/tests/cert-management-policy/02-create-root-ca.yaml @@ -0,0 +1,29 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o pipefail + set -o xtrace + + source ../../functions + + cluster="cert-management-policy" + + issue_root_ca + + kubectl -n "$NAMESPACE" create secret generic "${cluster}-cluster-ca-cert" \ + --from-file=root.crt="$TEMP_DIR/root-ca.crt" \ + --from-file=root.key="$TEMP_DIR/root-ca.key" \ + --dry-run=client -o yaml \ + | kubectl label --local -f - -o yaml \ + postgres-operator.crunchydata.com/cluster="$cluster" \ + | kubectl -n "$NAMESPACE" apply -f - + + secret="${cluster}-cluster-ca-cert" + checksum=$(kubectl -n "$NAMESPACE" get secret "$secret" -o json \ + | jq --compact-output --sort-keys '.data' \ + | sha256sum | awk '{print $1}') + kubectl -n "$NAMESPACE" annotate secret "$secret" \ + e2e.percona.com/secret-checksum="$checksum" \ + --overwrite diff --git a/e2e-tests/tests/cert-management-policy/02-save-secrets.yaml b/e2e-tests/tests/cert-management-policy/02-save-secrets.yaml deleted file mode 100644 index d66854c5c6..0000000000 --- a/e2e-tests/tests/cert-management-policy/02-save-secrets.yaml +++ /dev/null @@ -1,33 +0,0 @@ -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -timeout: 180 -commands: - - script: |- - set -o errexit - set -o pipefail - set -o xtrace - - source ../../functions - - cluster="cert-management-policy" - instance=$(kubectl -n "$NAMESPACE" get statefulset \ - -l postgres-operator.crunchydata.com/cluster="$cluster",postgres-operator.crunchydata.com/data=postgres \ - -o jsonpath='{.items[0].metadata.name}') - - kubectl -n "$NAMESPACE" get secret \ - "${cluster}-cluster-ca-cert" \ - "${cluster}-cluster-cert" \ - "${cluster}-replication-cert" \ - "${cluster}-pgbackrest" \ - "${cluster}-pgbouncer" \ - "${instance}-certs" \ - -o yaml \ - | yq eval 'del( - .metadata, - .items[].metadata.creationTimestamp, - .items[].metadata.managedFields, - .items[].metadata.ownerReferences, - .items[].metadata.resourceVersion, - .items[].metadata.uid - )' - \ - >"$TEMP_DIR/tls-secrets.yaml" diff --git a/e2e-tests/tests/cert-management-policy/03-create-postgres-certificate.yaml b/e2e-tests/tests/cert-management-policy/03-create-postgres-certificate.yaml new file mode 100644 index 0000000000..af999beb5d --- /dev/null +++ b/e2e-tests/tests/cert-management-policy/03-create-postgres-certificate.yaml @@ -0,0 +1,46 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o pipefail + set -o xtrace + + source ../../functions + + cluster="cert-management-policy" + + primary="${cluster}-primary" + replicas="${cluster}-replicas" + primary_names=( + "${primary}.${NAMESPACE}.svc.cluster.local" + "${primary}.${NAMESPACE}.svc" + "${primary}.${NAMESPACE}" + "$primary" + ) + replica_names=( + "${replicas}.${NAMESPACE}.svc.cluster.local" + "${replicas}.${NAMESPACE}.svc" + "${replicas}.${NAMESPACE}" + "$replicas" + ) + issue_certificate postgres \ + "$primary" serverAuth \ + "${primary_names[@]}" "${replica_names[@]}" + + kubectl -n "$NAMESPACE" create secret generic "${cluster}-cluster-cert" \ + --from-file=tls.crt="$TEMP_DIR/postgres.crt" \ + --from-file=tls.key="$TEMP_DIR/postgres.key" \ + --from-file=ca.crt="$TEMP_DIR/root-ca.crt" \ + --dry-run=client -o yaml \ + | kubectl label --local -f - -o yaml \ + postgres-operator.crunchydata.com/cluster="$cluster" \ + | kubectl -n "$NAMESPACE" apply -f - + + secret="${cluster}-cluster-cert" + checksum=$(kubectl -n "$NAMESPACE" get secret "$secret" -o json \ + | jq --compact-output --sort-keys '.data' \ + | sha256sum | awk '{print $1}') + kubectl -n "$NAMESPACE" annotate secret "$secret" \ + e2e.percona.com/secret-checksum="$checksum" \ + --overwrite diff --git a/e2e-tests/tests/cert-management-policy/03-delete-cluster-and-secrets.yaml b/e2e-tests/tests/cert-management-policy/03-delete-cluster-and-secrets.yaml deleted file mode 100644 index c89a5a6ec9..0000000000 --- a/e2e-tests/tests/cert-management-policy/03-delete-cluster-and-secrets.yaml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -timeout: 180 -commands: - - script: |- - set -o errexit - set -o pipefail - set -o xtrace - - source ../../functions - - cluster="cert-management-policy" - - kubectl -n "$NAMESPACE" delete perconapgcluster "$cluster" --wait=false - remove_all_finalizers - kubectl -n "$NAMESPACE" wait --for=delete "perconapgcluster/$cluster" --timeout=120s - kubectl -n "$NAMESPACE" wait --for=delete "postgrescluster/$cluster" --timeout=120s - - kubectl -n "$NAMESPACE" delete pvc \ - -l postgres-operator.crunchydata.com/cluster="$cluster" \ - --ignore-not-found - - kubectl -n "$NAMESPACE" delete secret \ - -l postgres-operator.crunchydata.com/cluster="$cluster" \ - --ignore-not-found - kubectl -n "$NAMESPACE" delete secret \ - "${cluster}-cluster-ca-cert" \ - "${cluster}-cluster-cert" \ - "${cluster}-replication-cert" \ - --ignore-not-found diff --git a/e2e-tests/tests/cert-management-policy/04-assert.yaml b/e2e-tests/tests/cert-management-policy/04-assert.yaml deleted file mode 100644 index d8632c2399..0000000000 --- a/e2e-tests/tests/cert-management-policy/04-assert.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 60 ---- -apiVersion: pgv2.percona.com/v2 -kind: PerconaPGCluster -metadata: - name: cert-management-policy -status: - conditions: - - reason: PGBackRestRepoHostReady - status: "False" - type: ReadyForBackup - - reason: TLSSecretsMissing - status: "False" - type: TLSSecretsReady - - reason: Paused - status: "False" - type: Progressing diff --git a/e2e-tests/tests/cert-management-policy/04-create-replication-certificate.yaml b/e2e-tests/tests/cert-management-policy/04-create-replication-certificate.yaml new file mode 100644 index 0000000000..6be5c0ebf0 --- /dev/null +++ b/e2e-tests/tests/cert-management-policy/04-create-replication-certificate.yaml @@ -0,0 +1,30 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o pipefail + set -o xtrace + + source ../../functions + + cluster="cert-management-policy" + + issue_certificate replication _crunchyrepl clientAuth _crunchyrepl + + kubectl -n "$NAMESPACE" create secret generic "${cluster}-replication-cert" \ + --from-file=tls.crt="$TEMP_DIR/replication.crt" \ + --from-file=tls.key="$TEMP_DIR/replication.key" \ + --from-file=ca.crt="$TEMP_DIR/root-ca.crt" \ + --dry-run=client -o yaml \ + | kubectl label --local -f - -o yaml \ + postgres-operator.crunchydata.com/cluster="$cluster" \ + | kubectl -n "$NAMESPACE" apply -f - + + secret="${cluster}-replication-cert" + checksum=$(kubectl -n "$NAMESPACE" get secret "$secret" -o json \ + | jq --compact-output --sort-keys '.data' \ + | sha256sum | awk '{print $1}') + kubectl -n "$NAMESPACE" annotate secret "$secret" \ + e2e.percona.com/secret-checksum="$checksum" \ + --overwrite diff --git a/e2e-tests/tests/cert-management-policy/04-create-user-provided-cluster.yaml b/e2e-tests/tests/cert-management-policy/04-create-user-provided-cluster.yaml deleted file mode 100644 index c9620a0bab..0000000000 --- a/e2e-tests/tests/cert-management-policy/04-create-user-provided-cluster.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - script: |- - set -o errexit - set -o pipefail - set -o xtrace - - source ../../functions - - cluster="cert-management-policy" - - get_cr "$cluster" \ - | yq '.spec.tls.certManagementPolicy = "userProvidedOnly"' \ - | yq '.spec.proxy.pgBouncer.replicas = 1' \ - | yq '.spec.instances[].replicas = 1' \ - | kubectl -n "$NAMESPACE" apply -f - diff --git a/e2e-tests/tests/cert-management-policy/05-assert.yaml b/e2e-tests/tests/cert-management-policy/05-assert.yaml deleted file mode 100644 index 58d79931e7..0000000000 --- a/e2e-tests/tests/cert-management-policy/05-assert.yaml +++ /dev/null @@ -1,36 +0,0 @@ -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 180 ---- -kind: StatefulSet -apiVersion: apps/v1 -metadata: - labels: - postgres-operator.crunchydata.com/cluster: cert-management-policy - postgres-operator.crunchydata.com/data: postgres - postgres-operator.crunchydata.com/instance-set: instance1 ---- -apiVersion: pgv2.percona.com/v2 -kind: PerconaPGCluster -metadata: - name: cert-management-policy -status: - conditions: - - reason: PGBackRestRepoHostReady - status: "False" - type: ReadyForBackup - - reason: TLSSecretsMissing - status: "False" - type: TLSSecretsReady - - reason: Paused - status: "False" - type: Progressing - - reason: RepoHostNotReady - status: "False" - type: PGBackRestRepoHostReady - - reason: StanzaNotCreated - status: "False" - type: PGBackRestReplicaRepoReady - - reason: RepoBackupNotComplete - status: "False" - type: PGBackRestReplicaCreate diff --git a/e2e-tests/tests/cert-management-policy/05-create-pgbackrest-certificates.yaml b/e2e-tests/tests/cert-management-policy/05-create-pgbackrest-certificates.yaml new file mode 100644 index 0000000000..9094aac80f --- /dev/null +++ b/e2e-tests/tests/cert-management-policy/05-create-pgbackrest-certificates.yaml @@ -0,0 +1,36 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o pipefail + set -o xtrace + + source ../../functions + + cluster="cert-management-policy" + + # TODO: The user should know the PostgresCluster UID after the cluster is + # created so that the pgBackRest client certificate can use it. + pgbackrest_client="pgbackrest@pending" + issue_certificate pgbackrest-client \ + "$pgbackrest_client" clientAuth "$pgbackrest_client" + + repo_name="${cluster}-repo-host-0.${cluster}-pods" + issue_certificate pgbackrest-repo \ + "${cluster}-repo-host" serverAuth \ + "${repo_name}.${NAMESPACE}.svc.cluster.local" \ + "${repo_name}.${NAMESPACE}.svc" \ + "${repo_name}.${NAMESPACE}" \ + "$repo_name" + + kubectl -n "$NAMESPACE" create secret generic "${cluster}-pgbackrest" \ + --from-file=pgbackrest.ca-roots="$TEMP_DIR/root-ca.crt" \ + --from-file=pgbackrest-client.crt="$TEMP_DIR/pgbackrest-client.crt" \ + --from-file=pgbackrest-client.key="$TEMP_DIR/pgbackrest-client.key" \ + --from-file=pgbackrest-repo-host.crt="$TEMP_DIR/pgbackrest-repo.crt" \ + --from-file=pgbackrest-repo-host.key="$TEMP_DIR/pgbackrest-repo.key" \ + --dry-run=client -o yaml \ + | kubectl label --local -f - -o yaml \ + postgres-operator.crunchydata.com/cluster="$cluster" \ + | kubectl -n "$NAMESPACE" apply -f - diff --git a/e2e-tests/tests/cert-management-policy/05-restore-secrets.yaml b/e2e-tests/tests/cert-management-policy/05-restore-secrets.yaml deleted file mode 100644 index 8f0a7a5bb4..0000000000 --- a/e2e-tests/tests/cert-management-policy/05-restore-secrets.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - script: |- - set -o errexit - set -o xtrace - - source ../../functions - - yq eval \ - 'del(.items[] | select(.metadata.name | test("-certs$")))' \ - "$TEMP_DIR/tls-secrets.yaml" \ - | kubectl -n "$NAMESPACE" apply -f - diff --git a/e2e-tests/tests/cert-management-policy/06-assert.yaml b/e2e-tests/tests/cert-management-policy/06-assert.yaml deleted file mode 100644 index ff107dfb3b..0000000000 --- a/e2e-tests/tests/cert-management-policy/06-assert.yaml +++ /dev/null @@ -1,60 +0,0 @@ -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 600 ---- -apiVersion: v1 -kind: Secret -metadata: - name: cert-management-policy-cluster-ca-cert ---- -apiVersion: v1 -kind: Secret -metadata: - name: cert-management-policy-cluster-cert ---- -apiVersion: v1 -kind: Secret -metadata: - name: cert-management-policy-replication-cert ---- -apiVersion: v1 -kind: Secret -metadata: - name: cert-management-policy-pgbackrest ---- -apiVersion: v1 -kind: Secret -metadata: - name: cert-management-policy-pgbouncer ---- -kind: StatefulSet -apiVersion: apps/v1 -metadata: - labels: - postgres-operator.crunchydata.com/cluster: cert-management-policy - postgres-operator.crunchydata.com/data: postgres - postgres-operator.crunchydata.com/instance-set: instance1 -status: - replicas: 1 - readyReplicas: 1 ---- -kind: Deployment -apiVersion: apps/v1 -metadata: - name: cert-management-policy-pgbouncer -status: - replicas: 1 - readyReplicas: 1 ---- -apiVersion: pgv2.percona.com/v2 -kind: PerconaPGCluster -metadata: - name: cert-management-policy -status: - pgbouncer: - ready: 1 - size: 1 - postgres: - ready: 1 - size: 1 - state: ready diff --git a/e2e-tests/tests/cert-management-policy/06-create-pgbouncer-certificate.yaml b/e2e-tests/tests/cert-management-policy/06-create-pgbouncer-certificate.yaml new file mode 100644 index 0000000000..e274465eb3 --- /dev/null +++ b/e2e-tests/tests/cert-management-policy/06-create-pgbouncer-certificate.yaml @@ -0,0 +1,51 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o pipefail + set -o xtrace + + source ../../functions + + cluster="cert-management-policy" + + pgbouncer="${cluster}-pgbouncer" + pgbouncer_names=( + "${pgbouncer}.${NAMESPACE}.svc.cluster.local" + "${pgbouncer}.${NAMESPACE}.svc" + "${pgbouncer}.${NAMESPACE}" + "$pgbouncer" + ) + issue_certificate pgbouncer \ + "$pgbouncer" serverAuth "${pgbouncer_names[@]}" + + # TODO: In userProvidedOnly mode, the operator should generate everything + # in this Secret except the user-provided certificates. + pgbouncer_password="datalake" + pgbouncer_admin_password="pgbouncer-admin-password" + pgbouncer_verifier='SCRAM-SHA-256$4096:aDFwcDBwNHJ0eTIwMjA=$xHkOo65LX9eBB8a6v+axqvs3+aMBTH0sCT7w/Nxzh5M=:PXuFoeJNuAGSeExskYSqkwUyiUJu8LPC9DgwDWQ9ARQ=' + pgbouncer_users=$(printf \ + '"_crunchypgbouncer" "%s"\n"_crunchypgbounceradmin" "%s"\n' \ + "$pgbouncer_password" "$pgbouncer_admin_password") + + kubectl -n "$NAMESPACE" create secret generic "${cluster}-pgbouncer" \ + --from-file=pgbouncer-frontend.ca-roots="$TEMP_DIR/root-ca.crt" \ + --from-file=pgbouncer-frontend.crt="$TEMP_DIR/pgbouncer.crt" \ + --from-file=pgbouncer-frontend.key="$TEMP_DIR/pgbouncer.key" \ + --from-literal=pgbouncer-password="$pgbouncer_password" \ + --from-literal=pgbouncer-verifier="$pgbouncer_verifier" \ + --from-literal=pgbouncer-admin-password="$pgbouncer_admin_password" \ + --from-literal=pgbouncer-users.txt="$pgbouncer_users" \ + --dry-run=client -o yaml \ + | kubectl label --local -f - -o yaml \ + postgres-operator.crunchydata.com/cluster="$cluster" \ + | kubectl -n "$NAMESPACE" apply -f - + + secret="${cluster}-pgbouncer" + checksum=$(kubectl -n "$NAMESPACE" get secret "$secret" -o json \ + | jq --compact-output --sort-keys '.data' \ + | sha256sum | awk '{print $1}') + kubectl -n "$NAMESPACE" annotate secret "$secret" \ + e2e.percona.com/secret-checksum="$checksum" \ + --overwrite diff --git a/e2e-tests/tests/cert-management-policy/06-restore-instance-secret.yaml b/e2e-tests/tests/cert-management-policy/06-restore-instance-secret.yaml deleted file mode 100644 index 5b5aaeb809..0000000000 --- a/e2e-tests/tests/cert-management-policy/06-restore-instance-secret.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - script: |- - set -o errexit - set -o pipefail - set -o xtrace - - source ../../functions - - cluster="cert-management-policy" - instance=$(kubectl -n "$NAMESPACE" get statefulset \ - -l postgres-operator.crunchydata.com/cluster="$cluster",postgres-operator.crunchydata.com/data=postgres \ - -o jsonpath='{.items[0].metadata.name}') - - new_instance_secret="${instance}-certs" - - instance="$instance" new_instance_secret="$new_instance_secret" yq eval \ - '.items[] | select(.metadata.name | test("-certs$")) | - .metadata.name = strenv(new_instance_secret) | - .metadata.labels."postgres-operator.crunchydata.com/instance" = strenv(instance)' \ - "$TEMP_DIR/tls-secrets.yaml" \ - | kubectl -n "$NAMESPACE" apply -f - diff --git a/e2e-tests/tests/cert-management-policy/07-assert.yaml b/e2e-tests/tests/cert-management-policy/07-assert.yaml new file mode 100644 index 0000000000..bde525ff76 --- /dev/null +++ b/e2e-tests/tests/cert-management-policy/07-assert.yaml @@ -0,0 +1,21 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 240 +--- +apiVersion: upstream.pgv2.percona.com/v1beta1 +kind: PostgresCluster +metadata: + name: cert-management-policy +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: cert-management-policy-repo-host +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + postgres-operator.crunchydata.com/cluster: cert-management-policy + postgres-operator.crunchydata.com/data: postgres + postgres-operator.crunchydata.com/instance-set: instance1 diff --git a/e2e-tests/tests/cert-management-policy/08-update-pgbackrest-client-certificate.yaml b/e2e-tests/tests/cert-management-policy/08-update-pgbackrest-client-certificate.yaml new file mode 100644 index 0000000000..eb83e897bd --- /dev/null +++ b/e2e-tests/tests/cert-management-policy/08-update-pgbackrest-client-certificate.yaml @@ -0,0 +1,46 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o pipefail + set -o xtrace + + source ../../functions + + cluster="cert-management-policy" + + # TODO: The user should know the PostgresCluster UID after the cluster is + # created so that the pgBackRest client certificate can use it. + cluster_uid=$(kubectl -n "$NAMESPACE" get postgrescluster "$cluster" \ + -o jsonpath='{.metadata.uid}') + pgbackrest_client="pgbackrest@${cluster_uid}" + + issue_certificate pgbackrest-client \ + "$pgbackrest_client" clientAuth "$pgbackrest_client" + + pgbackrest_cert=$(base64 <"$TEMP_DIR/pgbackrest-client.crt" \ + | tr -d '\n') + pgbackrest_key=$(base64 <"$TEMP_DIR/pgbackrest-client.key" \ + | tr -d '\n') + kubectl -n "$NAMESPACE" get secret "${cluster}-pgbackrest" -o yaml \ + | pgbackrest_cert="$pgbackrest_cert" pgbackrest_key="$pgbackrest_key" \ + yq eval ' + .data."pgbackrest-client.crt" = strenv(pgbackrest_cert) | + .data."pgbackrest-client.key" = strenv(pgbackrest_key) | + del( + .metadata.creationTimestamp, + .metadata.managedFields, + .metadata.resourceVersion, + .metadata.uid + ) + ' - \ + | kubectl -n "$NAMESPACE" apply -f - + + secret="${cluster}-pgbackrest" + checksum=$(kubectl -n "$NAMESPACE" get secret "$secret" -o json \ + | jq --compact-output --sort-keys '.data' \ + | sha256sum | awk '{print $1}') + kubectl -n "$NAMESPACE" annotate secret "$secret" \ + e2e.percona.com/secret-checksum="$checksum" \ + --overwrite diff --git a/e2e-tests/tests/cert-management-policy/09-create-instance-certificate.yaml b/e2e-tests/tests/cert-management-policy/09-create-instance-certificate.yaml new file mode 100644 index 0000000000..639a0a38ec --- /dev/null +++ b/e2e-tests/tests/cert-management-policy/09-create-instance-certificate.yaml @@ -0,0 +1,56 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o pipefail + set -o xtrace + + source ../../functions + + cluster="cert-management-policy" + statefulsets=$(kubectl -n "$NAMESPACE" get statefulset \ + -l postgres-operator.crunchydata.com/cluster="$cluster",postgres-operator.crunchydata.com/data=postgres \ + -o json) + jq -e '.items | length == 3' <<<"$statefulsets" >/dev/null + instances=$(jq -r '.items[].metadata.name' <<<"$statefulsets" | sort) + + for instance in $instances; do + instance_service=$(kubectl -n "$NAMESPACE" get statefulset "$instance" \ + -o jsonpath='{.spec.serviceName}') + instance_name="${instance}-0.${instance_service}" + instance_names=( + "${instance_name}.${NAMESPACE}.svc.cluster.local" + "${instance_name}.${NAMESPACE}.svc" + "${instance_name}.${NAMESPACE}" + "$instance_name" + ) + + issue_certificate "$instance" \ + "${instance}-0" serverAuth \ + "${instance_names[@]}" + cp "$TEMP_DIR/${instance}.key" "$TEMP_DIR/${instance}.patroni.crt-combined" + cat "$TEMP_DIR/${instance}.crt" >>"$TEMP_DIR/${instance}.patroni.crt-combined" + + kubectl -n "$NAMESPACE" create secret generic "${instance}-certs" \ + --from-file=dns.crt="$TEMP_DIR/${instance}.crt" \ + --from-file=dns.key="$TEMP_DIR/${instance}.key" \ + --from-file=patroni.ca-roots="$TEMP_DIR/root-ca.crt" \ + --from-file=patroni.crt-combined="$TEMP_DIR/${instance}.patroni.crt-combined" \ + --from-file=pgbackrest-server.crt="$TEMP_DIR/${instance}.crt" \ + --from-file=pgbackrest-server.key="$TEMP_DIR/${instance}.key" \ + --dry-run=client -o yaml \ + | kubectl label --local -f - -o yaml \ + postgres-operator.crunchydata.com/cluster="$cluster" \ + postgres-operator.crunchydata.com/instance="$instance" \ + postgres-operator.crunchydata.com/instance-set=instance1 \ + | kubectl -n "$NAMESPACE" apply -f - + + secret="${instance}-certs" + checksum=$(kubectl -n "$NAMESPACE" get secret "$secret" -o json \ + | jq --compact-output --sort-keys '.data' \ + | sha256sum | awk '{print $1}') + kubectl -n "$NAMESPACE" annotate secret "$secret" \ + e2e.percona.com/secret-checksum="$checksum" \ + --overwrite + done diff --git a/e2e-tests/tests/cert-management-policy/10-assert.yaml b/e2e-tests/tests/cert-management-policy/10-assert.yaml new file mode 100644 index 0000000000..927dbc801b --- /dev/null +++ b/e2e-tests/tests/cert-management-policy/10-assert.yaml @@ -0,0 +1,69 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +commands: + - script: |- + set -o errexit + set -o xtrace + + certificates=$(kubectl -n "$NAMESPACE" get certificate -o name) + if [[ -n "$certificates" ]]; then + echo "Certificate resources were generated in userProvidedOnly mode: $certificates" >&2 + exit 1 + fi +resourceRefs: + - apiVersion: pgv2.percona.com/v2 + kind: PerconaPGCluster + name: cert-management-policy + ref: cluster +assertAll: + - celExpr: cluster.status.conditions.exists(c, c.type == "ReadyForBackup" && c.status == "True" && has(c.observedGeneration) && c.observedGeneration == cluster.metadata.generation) + message: ReadyForBackup should be True + - celExpr: cluster.status.conditions.exists(c, c.type == "TLSSecretsReady" && c.status == "True" && c.reason == "TLSSecretsFound" && has(c.observedGeneration) && c.observedGeneration == cluster.metadata.generation) + message: TLSSecretsReady should be True with reason TLSSecretsFound +--- +apiVersion: v1 +kind: Secret +metadata: + name: cert-management-policy-cluster-ca-cert +--- +apiVersion: v1 +kind: Secret +metadata: + name: cert-management-policy-cluster-cert +--- +apiVersion: v1 +kind: Secret +metadata: + name: cert-management-policy-replication-cert +--- +apiVersion: v1 +kind: Secret +metadata: + name: cert-management-policy-pgbackrest +--- +apiVersion: v1 +kind: Secret +metadata: + name: cert-management-policy-pgbouncer +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: cert-management-policy-pgbouncer +status: + replicas: 1 + readyReplicas: 1 +--- +apiVersion: pgv2.percona.com/v2 +kind: PerconaPGCluster +metadata: + name: cert-management-policy +status: + pgbouncer: + ready: 1 + size: 1 + postgres: + ready: 3 + size: 3 + state: ready diff --git a/e2e-tests/tests/cert-management-policy/11-verify-served-certificates.yaml b/e2e-tests/tests/cert-management-policy/11-verify-served-certificates.yaml new file mode 100644 index 0000000000..e745afe0d0 --- /dev/null +++ b/e2e-tests/tests/cert-management-policy/11-verify-served-certificates.yaml @@ -0,0 +1,59 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 60 +commands: + - script: |- + set -o errexit + set -o pipefail + set -o xtrace + + source ../../functions + + cluster="cert-management-policy" + primary=$(kubectl -n "$NAMESPACE" get pod \ + -l postgres-operator.crunchydata.com/cluster="$cluster",postgres-operator.crunchydata.com/role=primary \ + -o jsonpath='{.items[0].metadata.name}') + [[ -n "$primary" ]] + + postgres_secret_fingerprint=$(kubectl -n "$NAMESPACE" get secret \ + "${cluster}-cluster-cert" -o jsonpath='{.data.tls\.crt}' \ + | base64 -d \ + | openssl x509 -noout -fingerprint -sha256 \ + | sed 's/.*=//' \ + | tr -d ':[:space:]') + postgres_served_fingerprint=$(kubectl -n "$NAMESPACE" exec "$primary" -c database -- \ + bash -c "timeout 20 openssl s_client -connect ${cluster}-primary:5432 -starttls postgres /dev/null | openssl x509 -noout -fingerprint -sha256 | sed 's/.*=//'" \ + | tr -d ':[:space:]') + + if [[ "$postgres_served_fingerprint" != "$postgres_secret_fingerprint" ]]; then + echo "PostgreSQL is not serving the user-provided certificate" >&2 + echo "Secret: $postgres_secret_fingerprint" >&2 + echo "Served: $postgres_served_fingerprint" >&2 + exit 1 + fi + + pgbouncer_secret_fingerprint=$(kubectl -n "$NAMESPACE" get secret \ + "${cluster}-pgbouncer" -o jsonpath='{.data.pgbouncer-frontend\.crt}' \ + | base64 -d \ + | openssl x509 -noout -fingerprint -sha256 \ + | sed 's/.*=//' \ + | tr -d ':[:space:]') + pgbouncer_served_fingerprint=$(kubectl -n "$NAMESPACE" exec "$primary" -c database -- \ + bash -c "timeout 20 openssl s_client -connect ${cluster}-pgbouncer:5432 -starttls postgres /dev/null | openssl x509 -noout -fingerprint -sha256 | sed 's/.*=//'" \ + | tr -d ':[:space:]') + + if [[ "$pgbouncer_served_fingerprint" != "$pgbouncer_secret_fingerprint" ]]; then + echo "PgBouncer is not serving the user-provided frontend certificate" >&2 + echo "Secret: $pgbouncer_secret_fingerprint" >&2 + echo "Served: $pgbouncer_served_fingerprint" >&2 + exit 1 + fi + + replication_connections=$(kubectl -n "$NAMESPACE" exec "$primary" -c database -- \ + psql -d postgres -Atqc \ + "SELECT count(*) FROM pg_stat_ssl s JOIN pg_stat_replication r ON s.pid = r.pid WHERE s.ssl = true AND s.client_dn LIKE '%CN=_crunchyrepl%';") + replication_connections=$(tr -d '[:space:]' <<<"$replication_connections") + if [[ "$replication_connections" != "2" ]]; then + echo "Expected two SSL replication connections using the user-provided _crunchyrepl identity, got: $replication_connections" >&2 + exit 1 + fi diff --git a/e2e-tests/tests/cert-management-policy/12-verify-secrets-unchanged.yaml b/e2e-tests/tests/cert-management-policy/12-verify-secrets-unchanged.yaml new file mode 100644 index 0000000000..6b4a05dbe1 --- /dev/null +++ b/e2e-tests/tests/cert-management-policy/12-verify-secrets-unchanged.yaml @@ -0,0 +1,45 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o pipefail + set -o xtrace + + source ../../functions + + cluster="cert-management-policy" + + assert_secret_unchanged() { + local secret="$1" + local expected + local actual + + expected=$(kubectl -n "$NAMESPACE" get secret "$secret" \ + -o jsonpath='{.metadata.annotations.e2e\.percona\.com/secret-checksum}') + actual=$(kubectl -n "$NAMESPACE" get secret "$secret" -o json \ + | jq --compact-output --sort-keys '.data' \ + | sha256sum | awk '{print $1}') + + if [[ -z "$expected" || "$actual" != "$expected" ]]; then + echo "User-provided Secret changed: $secret" >&2 + echo "Expected checksum: ${expected:-missing}" >&2 + echo "Actual checksum: $actual" >&2 + return 1 + fi + } + + assert_secret_unchanged "${cluster}-cluster-ca-cert" + assert_secret_unchanged "${cluster}-cluster-cert" + assert_secret_unchanged "${cluster}-replication-cert" + assert_secret_unchanged "${cluster}-pgbackrest" + assert_secret_unchanged "${cluster}-pgbouncer" + + statefulsets=$(kubectl -n "$NAMESPACE" get statefulset \ + -l postgres-operator.crunchydata.com/cluster="$cluster",postgres-operator.crunchydata.com/data=postgres \ + -o json) + jq -e '.items | length == 3' <<<"$statefulsets" >/dev/null + instances=$(jq -r '.items[].metadata.name' <<<"$statefulsets" | sort) + for instance in $instances; do + assert_secret_unchanged "${instance}-certs" + done diff --git a/e2e-tests/tests/cert-management-policy/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/cert-management-policy/99-remove-cluster-gracefully.yaml index f7840e50f5..7b2bca34b8 100644 --- a/e2e-tests/tests/cert-management-policy/99-remove-cluster-gracefully.yaml +++ b/e2e-tests/tests/cert-management-policy/99-remove-cluster-gracefully.yaml @@ -28,4 +28,5 @@ commands: --ignore-not-found || true check_operator_panic destroy_operator + destroy_cert_manager timeout: 60 diff --git a/percona/controller/pgcluster/status.go b/percona/controller/pgcluster/status.go index 79d46332c5..027061a490 100644 --- a/percona/controller/pgcluster/status.go +++ b/percona/controller/pgcluster/status.go @@ -168,12 +168,14 @@ func (r *PGClusterReconciler) updateStatus(ctx context.Context, cr *v2.PerconaPG func updateConditions(cr *v2.PerconaPGCluster, status *v1beta1.PostgresClusterStatus) { setClusterNotReadyCondition := func(status metav1.ConditionStatus, reason string) { existing := meta.FindStatusCondition(cr.Status.Conditions, pNaming.ConditionClusterIsReadyForBackup) - if existing == nil || existing.Status != status || existing.Reason != reason { + if existing == nil || existing.Status != status || existing.Reason != reason || + existing.ObservedGeneration != cr.Generation { _ = meta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{ Type: pNaming.ConditionClusterIsReadyForBackup, Status: status, LastTransitionTime: metav1.Now(), Reason: reason, + ObservedGeneration: cr.Generation, }) } } diff --git a/percona/controller/pgcluster/status_test.go b/percona/controller/pgcluster/status_test.go index 9ba24fb8da..fed91fe4e4 100644 --- a/percona/controller/pgcluster/status_test.go +++ b/percona/controller/pgcluster/status_test.go @@ -730,6 +730,7 @@ func TestUpdateConditions(t *testing.T) { require.NotNil(t, condition, "ClusterIsReadyForBackup condition should be set") assert.Equal(t, tt.expectedReadyForBackupStatus, condition.Status, "ClusterIsReadyForBackup status mismatch") assert.Equal(t, tt.expectedReadyForBackupReason, condition.Reason, "ClusterIsReadyForBackup reason mismatch") + assert.Equal(t, cr.Generation, condition.ObservedGeneration, "ClusterIsReadyForBackup generation mismatch") // Verify synced conditions for _, expected := range tt.expectedSyncedConditions { From 5c461ac2a56308ea6df1243b541342da2ec52a50 Mon Sep 17 00:00:00 2001 From: Andrii Dema Date: Wed, 12 Aug 2026 11:35:19 +0300 Subject: [PATCH 2/5] preserve pgbouncer certs --- .../06-create-pgbouncer-certificate.yaml | 16 +----- .../cert-management-policy/10-assert.yaml | 8 +++ .../12-verify-secrets-unchanged.yaml | 6 ++- .../controller/postgrescluster/pgbouncer.go | 6 ++- internal/controller/postgrescluster/pki.go | 32 +++++++++++- .../controller/postgrescluster/pki_test.go | 52 ++++++++++++++++++- internal/pgbouncer/certificates.go | 14 ++--- internal/pgbouncer/reconcile.go | 41 +++++++++------ internal/pgbouncer/reconcile_test.go | 40 ++++++++++++++ 9 files changed, 171 insertions(+), 44 deletions(-) diff --git a/e2e-tests/tests/cert-management-policy/06-create-pgbouncer-certificate.yaml b/e2e-tests/tests/cert-management-policy/06-create-pgbouncer-certificate.yaml index e274465eb3..13ed899161 100644 --- a/e2e-tests/tests/cert-management-policy/06-create-pgbouncer-certificate.yaml +++ b/e2e-tests/tests/cert-management-policy/06-create-pgbouncer-certificate.yaml @@ -20,23 +20,10 @@ commands: issue_certificate pgbouncer \ "$pgbouncer" serverAuth "${pgbouncer_names[@]}" - # TODO: In userProvidedOnly mode, the operator should generate everything - # in this Secret except the user-provided certificates. - pgbouncer_password="datalake" - pgbouncer_admin_password="pgbouncer-admin-password" - pgbouncer_verifier='SCRAM-SHA-256$4096:aDFwcDBwNHJ0eTIwMjA=$xHkOo65LX9eBB8a6v+axqvs3+aMBTH0sCT7w/Nxzh5M=:PXuFoeJNuAGSeExskYSqkwUyiUJu8LPC9DgwDWQ9ARQ=' - pgbouncer_users=$(printf \ - '"_crunchypgbouncer" "%s"\n"_crunchypgbounceradmin" "%s"\n' \ - "$pgbouncer_password" "$pgbouncer_admin_password") - kubectl -n "$NAMESPACE" create secret generic "${cluster}-pgbouncer" \ --from-file=pgbouncer-frontend.ca-roots="$TEMP_DIR/root-ca.crt" \ --from-file=pgbouncer-frontend.crt="$TEMP_DIR/pgbouncer.crt" \ --from-file=pgbouncer-frontend.key="$TEMP_DIR/pgbouncer.key" \ - --from-literal=pgbouncer-password="$pgbouncer_password" \ - --from-literal=pgbouncer-verifier="$pgbouncer_verifier" \ - --from-literal=pgbouncer-admin-password="$pgbouncer_admin_password" \ - --from-literal=pgbouncer-users.txt="$pgbouncer_users" \ --dry-run=client -o yaml \ | kubectl label --local -f - -o yaml \ postgres-operator.crunchydata.com/cluster="$cluster" \ @@ -44,7 +31,8 @@ commands: secret="${cluster}-pgbouncer" checksum=$(kubectl -n "$NAMESPACE" get secret "$secret" -o json \ - | jq --compact-output --sort-keys '.data' \ + | jq --compact-output --sort-keys \ + '.data | with_entries(select(.key | startswith("pgbouncer-frontend.")))' \ | sha256sum | awk '{print $1}') kubectl -n "$NAMESPACE" annotate secret "$secret" \ e2e.percona.com/secret-checksum="$checksum" \ diff --git a/e2e-tests/tests/cert-management-policy/10-assert.yaml b/e2e-tests/tests/cert-management-policy/10-assert.yaml index 927dbc801b..65e192d603 100644 --- a/e2e-tests/tests/cert-management-policy/10-assert.yaml +++ b/e2e-tests/tests/cert-management-policy/10-assert.yaml @@ -11,6 +11,14 @@ commands: echo "Certificate resources were generated in userProvidedOnly mode: $certificates" >&2 exit 1 fi + + kubectl -n "$NAMESPACE" get secret cert-management-policy-pgbouncer -o json \ + | jq -e ' + .data."pgbouncer-password" != null and + .data."pgbouncer-verifier" != null and + .data."pgbouncer-admin-password" != null and + .data."pgbouncer-users.txt" != null + ' resourceRefs: - apiVersion: pgv2.percona.com/v2 kind: PerconaPGCluster diff --git a/e2e-tests/tests/cert-management-policy/12-verify-secrets-unchanged.yaml b/e2e-tests/tests/cert-management-policy/12-verify-secrets-unchanged.yaml index 6b4a05dbe1..7b22fcd5d5 100644 --- a/e2e-tests/tests/cert-management-policy/12-verify-secrets-unchanged.yaml +++ b/e2e-tests/tests/cert-management-policy/12-verify-secrets-unchanged.yaml @@ -12,13 +12,14 @@ commands: assert_secret_unchanged() { local secret="$1" + local filter="${2:-.data}" local expected local actual expected=$(kubectl -n "$NAMESPACE" get secret "$secret" \ -o jsonpath='{.metadata.annotations.e2e\.percona\.com/secret-checksum}') actual=$(kubectl -n "$NAMESPACE" get secret "$secret" -o json \ - | jq --compact-output --sort-keys '.data' \ + | jq --compact-output --sort-keys "$filter" \ | sha256sum | awk '{print $1}') if [[ -z "$expected" || "$actual" != "$expected" ]]; then @@ -33,7 +34,8 @@ commands: assert_secret_unchanged "${cluster}-cluster-cert" assert_secret_unchanged "${cluster}-replication-cert" assert_secret_unchanged "${cluster}-pgbackrest" - assert_secret_unchanged "${cluster}-pgbouncer" + assert_secret_unchanged "${cluster}-pgbouncer" \ + '.data | with_entries(select(.key | startswith("pgbouncer-frontend.")))' statefulsets=$(kubectl -n "$NAMESPACE" get statefulset \ -l postgres-operator.crunchydata.com/cluster="$cluster",postgres-operator.crunchydata.com/data=postgres \ diff --git a/internal/controller/postgrescluster/pgbouncer.go b/internal/controller/postgrescluster/pgbouncer.go index 5c414db6df..3df5b8ef3a 100644 --- a/internal/controller/postgrescluster/pgbouncer.go +++ b/internal/controller/postgrescluster/pgbouncer.go @@ -251,6 +251,8 @@ func (r *Reconciler) reconcileCertManagerPGBouncerSecret(ctx context.Context, cl // When the root CA is internal but a stale Certificate CR was left by // K8SPG-1017, the CR is reconciled to update its ownerRef (K8SPG-1007 // recovery) before populating the secret from the internal PKI. +// In userProvidedOnly mode, the operator preserves the user-provided +// certificates. func (r *Reconciler) reconcilePGBouncerSecret( ctx context.Context, cluster *v1beta1.PostgresCluster, root *pki.RootCertificateAuthority, service *corev1.Service, @@ -278,7 +280,6 @@ func (r *Reconciler) reconcilePGBouncerSecret( if !secretFound { return nil, errors.Errorf("user-provided PgBouncer secret %q is missing", naming.ClusterPGBouncer(cluster).Name) } - return existing, nil } var userSecret *corev1.Secret if ref := cluster.Spec.Proxy.PGBouncer.UsersSecret; ref != nil && ref.Name != "" { @@ -793,7 +794,8 @@ func listPGBouncerPods(ctx context.Context, cl client.Client, cluster *v1beta1.P return nil, errors.Wrap(err, "pgbouncer selector") } podList := &corev1.PodList{} - if err := cl.List(ctx, podList, + if err := cl.List( + ctx, podList, client.InNamespace(cluster.Namespace), client.MatchingLabelsSelector{Selector: selector}, ); err != nil { diff --git a/internal/controller/postgrescluster/pki.go b/internal/controller/postgrescluster/pki.go index ea7fdf3682..1f9dbaa309 100644 --- a/internal/controller/postgrescluster/pki.go +++ b/internal/controller/postgrescluster/pki.go @@ -19,6 +19,7 @@ import ( "github.com/percona/percona-postgresql-operator/v2/internal/logging" "github.com/percona/percona-postgresql-operator/v2/internal/naming" + "github.com/percona/percona-postgresql-operator/v2/internal/pgbouncer" "github.com/percona/percona-postgresql-operator/v2/internal/pki" "github.com/percona/percona-postgresql-operator/v2/percona/certmanager" "github.com/percona/percona-postgresql-operator/v2/pkg/apis/upstream.pgv2.percona.com/v1beta1" @@ -47,8 +48,9 @@ func (r *Reconciler) reconcileTLSCondition(ctx context.Context, cluster *v1beta1 } var missing []string + var invalid []string - checkSecret := func(projection *corev1.SecretProjection, secretName string) error { + checkSecret := func(projection *corev1.SecretProjection, secretName string, requiredKeys ...string) error { if projection != nil { secretName = projection.Name } @@ -62,6 +64,17 @@ func (r *Reconciler) reconcileTLSCondition(ctx context.Context, cluster *v1beta1 } if k8serrors.IsNotFound(err) { missing = append(missing, secret.Name) + return nil + } + + var emptyKeys []string + for _, key := range requiredKeys { + if len(secret.Data[key]) == 0 { + emptyKeys = append(emptyKeys, key) + } + } + if len(emptyKeys) > 0 { + invalid = append(invalid, secret.Name+" (missing or empty keys: "+strings.Join(emptyKeys, ", ")+")") } return nil @@ -81,7 +94,15 @@ func (r *Reconciler) reconcileTLSCondition(ctx context.Context, cluster *v1beta1 } if cluster.Spec.Proxy != nil && cluster.Spec.Proxy.PGBouncer != nil { - if err := checkSecret(nil, naming.ClusterPGBouncer(cluster).Name); err != nil { + var requiredKeys []string + if cluster.Spec.Proxy.PGBouncer.CustomTLSSecret == nil { + requiredKeys = []string{ + pgbouncer.CertFrontendAuthoritySecretKey, + pgbouncer.CertFrontendSecretKey, + pgbouncer.CertFrontendPrivateKeySecretKey, + } + } + if err := checkSecret(nil, naming.ClusterPGBouncer(cluster).Name, requiredKeys...); err != nil { return errors.Wrap(err, "check PgBouncer TLS secret") } } @@ -113,6 +134,13 @@ func (r *Reconciler) reconcileTLSCondition(ctx context.Context, cluster *v1beta1 meta.SetStatusCondition(&cluster.Status.Conditions, cond) return nil } + if len(invalid) > 0 { + cond.Message = "Invalid user-provided TLS secrets: " + strings.Join(invalid, ", ") + ". certManagementPolicy is userProvidedOnly" + cond.Reason = "TLSSecretsInvalid" + cond.Status = metav1.ConditionFalse + meta.SetStatusCondition(&cluster.Status.Conditions, cond) + return nil + } meta.SetStatusCondition(&cluster.Status.Conditions, cond) return nil diff --git a/internal/controller/postgrescluster/pki_test.go b/internal/controller/postgrescluster/pki_test.go index e4b8109c4c..f1d106f7cd 100644 --- a/internal/controller/postgrescluster/pki_test.go +++ b/internal/controller/postgrescluster/pki_test.go @@ -27,6 +27,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" "github.com/percona/percona-postgresql-operator/v2/internal/naming" + "github.com/percona/percona-postgresql-operator/v2/internal/pgbouncer" "github.com/percona/percona-postgresql-operator/v2/internal/pki" "github.com/percona/percona-postgresql-operator/v2/internal/testing/require" "github.com/percona/percona-postgresql-operator/v2/percona/certmanager" @@ -127,7 +128,14 @@ func TestReconcileTLSCondition(t *testing.T) { &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "custom-postgres-tls", Namespace: cluster.Namespace}}, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "custom-replication", Namespace: cluster.Namespace}}, &corev1.Secret{ObjectMeta: naming.PGBackRestSecret(cluster)}, - &corev1.Secret{ObjectMeta: naming.ClusterPGBouncer(cluster)}, + &corev1.Secret{ + ObjectMeta: naming.ClusterPGBouncer(cluster), + Data: map[string][]byte{ + pgbouncer.CertFrontendAuthoritySecretKey: []byte("ca"), + pgbouncer.CertFrontendSecretKey: []byte("cert"), + pgbouncer.CertFrontendPrivateKeySecretKey: []byte("key"), + }, + }, } r := &Reconciler{Client: fake.NewClientBuilder().WithObjects(objects...).Build()} @@ -138,6 +146,48 @@ func TestReconcileTLSCondition(t *testing.T) { assert.Equal(t, condition.Message, "") assert.Equal(t, condition.ObservedGeneration, int64(13)) }) + + t.Run("PgBouncer secret has missing certificate data", func(t *testing.T) { + cluster := testCluster() + cluster.Namespace = "postgres-operator" + cluster.Spec.TLS = &v1beta1.TLSSpec{ + CertManagementPolicy: v1beta1.CertManagementUserProvidedOnly, + } + cluster.Spec.CustomRootCATLSSecret = &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{Name: "custom-root-ca"}, + } + cluster.Spec.CustomTLSSecret = &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{Name: "custom-postgres-tls"}, + } + cluster.Spec.CustomReplicationClientTLSSecret = &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{Name: "custom-replication"}, + } + + objects := []client.Object{ + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "custom-root-ca", Namespace: cluster.Namespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "custom-postgres-tls", Namespace: cluster.Namespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "custom-replication", Namespace: cluster.Namespace}}, + &corev1.Secret{ObjectMeta: naming.PGBackRestSecret(cluster)}, + &corev1.Secret{ + ObjectMeta: naming.ClusterPGBouncer(cluster), + Data: map[string][]byte{ + pgbouncer.CertFrontendAuthoritySecretKey: []byte("ca"), + pgbouncer.CertFrontendSecretKey: nil, + }, + }, + } + + r := &Reconciler{Client: fake.NewClientBuilder().WithObjects(objects...).Build()} + assert.NilError(t, r.reconcileTLSCondition(t.Context(), cluster)) + + condition := condition(t, cluster, metav1.ConditionFalse) + assert.Equal(t, condition.Reason, "TLSSecretsInvalid") + assert.Equal(t, condition.Message, "Invalid user-provided TLS secrets: "+ + naming.ClusterPGBouncer(cluster).Name+ + " (missing or empty keys: "+pgbouncer.CertFrontendSecretKey+", "+ + pgbouncer.CertFrontendPrivateKeySecretKey+"). "+ + "certManagementPolicy is userProvidedOnly") + }) } // TestReconcileCerts tests the proper reconciliation of the root ca certificate diff --git a/internal/pgbouncer/certificates.go b/internal/pgbouncer/certificates.go index 6668e83579..c88c6b8477 100644 --- a/internal/pgbouncer/certificates.go +++ b/internal/pgbouncer/certificates.go @@ -24,9 +24,9 @@ const ( certFrontendPrivateKeyProjectionPath = "~postgres-operator/frontend-tls.key" certFrontendProjectionPath = "~postgres-operator/frontend-tls.crt" - certFrontendAuthoritySecretKey = "pgbouncer-frontend.ca-roots" - certFrontendPrivateKeySecretKey = "pgbouncer-frontend.key" - certFrontendSecretKey = "pgbouncer-frontend.crt" + CertFrontendAuthoritySecretKey = "pgbouncer-frontend.ca-roots" + CertFrontendPrivateKeySecretKey = "pgbouncer-frontend.key" + CertFrontendSecretKey = "pgbouncer-frontend.crt" ) // backendAuthority creates a volume projection of the PostgreSQL server @@ -78,15 +78,15 @@ func frontendCertificate( }, Items: []corev1.KeyToPath{ { - Key: certFrontendAuthoritySecretKey, + Key: CertFrontendAuthoritySecretKey, Path: certFrontendAuthorityProjectionPath, }, { - Key: certFrontendPrivateKeySecretKey, + Key: CertFrontendPrivateKeySecretKey, Path: certFrontendPrivateKeyProjectionPath, }, { - Key: certFrontendSecretKey, + Key: CertFrontendSecretKey, Path: certFrontendProjectionPath, }, }, @@ -156,7 +156,7 @@ func frontendCertificate( Name: secret.Name, }, Items: []corev1.KeyToPath{{ - Key: certFrontendAuthoritySecretKey, + Key: CertFrontendAuthoritySecretKey, Path: certFrontendAuthorityProjectionPath, }}, }, diff --git a/internal/pgbouncer/reconcile.go b/internal/pgbouncer/reconcile.go index 735c63144f..d7aca3608f 100644 --- a/internal/pgbouncer/reconcile.go +++ b/internal/pgbouncer/reconcile.go @@ -129,13 +129,19 @@ func Secret(ctx context.Context, } if inCluster.Spec.Proxy.PGBouncer.CustomTLSSecret == nil { - if frontendCertManagerSecret != nil { + if inCluster.Spec.TLS.GetCertManagementPolicy() == v1beta1.CertManagementUserProvidedOnly { + // In userProvidedOnly mode, the certificates should be the only + // user-managed values in the secret. We should preserve them. + outSecret.Data[CertFrontendAuthoritySecretKey] = inSecret.Data[CertFrontendAuthoritySecretKey] + outSecret.Data[CertFrontendPrivateKeySecretKey] = inSecret.Data[CertFrontendPrivateKeySecretKey] + outSecret.Data[CertFrontendSecretKey] = inSecret.Data[CertFrontendSecretKey] + } else if frontendCertManagerSecret != nil { if err == nil { - outSecret.Data[certFrontendAuthoritySecretKey], err = frontendAuthorityCert(inRoot, frontendCertManagerSecret) + outSecret.Data[CertFrontendAuthoritySecretKey], err = frontendAuthorityCert(inRoot, frontendCertManagerSecret) } if err == nil { - outSecret.Data[certFrontendSecretKey] = frontendCertManagerSecret.Data[corev1.TLSCertKey] - outSecret.Data[certFrontendPrivateKeySecretKey] = frontendCertManagerSecret.Data[corev1.TLSPrivateKeyKey] + outSecret.Data[CertFrontendSecretKey] = frontendCertManagerSecret.Data[corev1.TLSCertKey] + outSecret.Data[CertFrontendPrivateKeySecretKey] = frontendCertManagerSecret.Data[corev1.TLSPrivateKeyKey] } } else if inRoot == nil { err = errors.New("waiting for cert-manager to issue pgbouncer frontend certificate") @@ -156,21 +162,21 @@ func Secret(ctx context.Context, // Unmarshal and validate the stored leaf. These first errors can // be ignored because they result in an invalid leaf which is then // correctly regenerated. - _ = leaf.Certificate.UnmarshalText(inSecret.Data[certFrontendSecretKey]) - _ = leaf.PrivateKey.UnmarshalText(inSecret.Data[certFrontendPrivateKeySecretKey]) + _ = leaf.Certificate.UnmarshalText(inSecret.Data[CertFrontendSecretKey]) + _ = leaf.PrivateKey.UnmarshalText(inSecret.Data[CertFrontendPrivateKeySecretKey]) leaf, err = inRoot.RegenerateLeafWhenNecessary(leaf, dnsFQDN, dnsNames) err = errors.WithStack(err) } if err == nil { - outSecret.Data[certFrontendAuthoritySecretKey], err = inRoot.Certificate.MarshalText() + outSecret.Data[CertFrontendAuthoritySecretKey], err = inRoot.Certificate.MarshalText() } if err == nil { - outSecret.Data[certFrontendPrivateKeySecretKey], err = leaf.PrivateKey.MarshalText() + outSecret.Data[CertFrontendPrivateKeySecretKey], err = leaf.PrivateKey.MarshalText() } if err == nil { - outSecret.Data[certFrontendSecretKey], err = leaf.Certificate.MarshalText() + outSecret.Data[CertFrontendSecretKey], err = leaf.Certificate.MarshalText() } } } @@ -179,15 +185,17 @@ func Secret(ctx context.Context, // bundle so PgBouncer also trusts them when verifying client // certificates. Entries keep their given order so identical inputs // always produce identical bundle bytes. - if err == nil && len(additionalCAs) > 0 { - bundle := outSecret.Data[certFrontendAuthoritySecretKey] + if err == nil && len(additionalCAs) > 0 && + (inCluster.Spec.Proxy.PGBouncer.CustomTLSSecret != nil || + inCluster.Spec.TLS.GetCertManagementPolicy() != v1beta1.CertManagementUserProvidedOnly) { + bundle := outSecret.Data[CertFrontendAuthoritySecretKey] for _, ca := range additionalCAs { if len(bundle) > 0 && bundle[len(bundle)-1] != '\n' { bundle = append(bundle, '\n') } bundle = append(bundle, ca...) } - outSecret.Data[certFrontendAuthoritySecretKey] = bundle + outSecret.Data[CertFrontendAuthoritySecretKey] = bundle } return err @@ -227,10 +235,11 @@ func Pod( } configVolume := corev1.Volume{Name: configVolumeMount.Name} configVolume.Projected = &corev1.ProjectedVolumeSource{ - Sources: append(append(append([]corev1.VolumeProjection{}, - podConfigFiles(inCluster, inConfigMap, inSecret)...), - frontendCertificate(inCluster.Spec.Proxy.PGBouncer.CustomTLSSecret, inSecret, - len(inCluster.Spec.Proxy.PGBouncer.AdditionalTrustedCAs) > 0)...), + Sources: append( + append(append([]corev1.VolumeProjection{}, + podConfigFiles(inCluster, inConfigMap, inSecret)...), + frontendCertificate(inCluster.Spec.Proxy.PGBouncer.CustomTLSSecret, inSecret, + len(inCluster.Spec.Proxy.PGBouncer.AdditionalTrustedCAs) > 0)...), backendAuthority(inPostgreSQLCertificate), ), } diff --git a/internal/pgbouncer/reconcile_test.go b/internal/pgbouncer/reconcile_test.go index 9c8129d7ec..91a8d18088 100644 --- a/internal/pgbouncer/reconcile_test.go +++ b/internal/pgbouncer/reconcile_test.go @@ -215,6 +215,46 @@ func TestSecretAdminPassword(t *testing.T) { }) } +func TestSecretUserProvidedOnly(t *testing.T) { + t.Parallel() + + ctx := t.Context() + cluster := new(v1beta1.PostgresCluster) + cluster.SetLabels(map[string]string{naming.LabelVersion: "3.1.0"}) + cluster.Spec.Proxy = &v1beta1.PostgresProxySpec{ + PGBouncer: new(v1beta1.PGBouncerPodSpec), + } + cluster.Spec.TLS = &v1beta1.TLSSpec{ + CertManagementPolicy: v1beta1.CertManagementUserProvidedOnly, + } + assert.NilError(t, cluster.Default(ctx, nil)) + + existing := &corev1.Secret{Data: map[string][]byte{ + "pgbouncer-frontend.ca-roots": []byte("user-ca"), + "pgbouncer-frontend.crt": []byte("user-cert"), + "pgbouncer-frontend.key": []byte("user-key"), + }} + intent := new(corev1.Secret) + + require.NoError(t, Secret(ctx, cluster, nil, existing, nil, + new(corev1.Service), intent, nil, [][]byte{[]byte("additional-ca")})) + + assert.DeepEqual(t, intent.Data["pgbouncer-frontend.ca-roots"], []byte("user-ca")) + assert.DeepEqual(t, intent.Data["pgbouncer-frontend.crt"], []byte("user-cert")) + assert.DeepEqual(t, intent.Data["pgbouncer-frontend.key"], []byte("user-key")) + assert.Equal(t, len(intent.Data["pgbouncer-password"]), 32) + assert.Assert(t, len(intent.Data["pgbouncer-verifier"]) > 0) + assert.Equal(t, len(intent.Data["pgbouncer-admin-password"]), 32) + assert.Assert(t, len(intent.Data["pgbouncer-users.txt"]) > 0) + + // Operator-managed credentials remain stable on later reconciles. + existing.Data = intent.Data + before := intent.DeepCopy() + require.NoError(t, Secret(ctx, cluster, nil, existing, nil, + new(corev1.Service), intent, nil, [][]byte{[]byte("additional-ca")})) + assert.DeepEqual(t, intent, before) +} + func TestSecretAdditionalCAs(t *testing.T) { t.Parallel() From e598f8822e8bfc255441f1157461d7d7cb5889bc Mon Sep 17 00:00:00 2001 From: Andrii Dema Date: Wed, 12 Aug 2026 13:31:50 +0300 Subject: [PATCH 3/5] small fix --- .../05-create-pgbackrest-certificates.yaml | 14 ++++-- ...ml => 08-create-instance-certificate.yaml} | 0 ...-update-pgbackrest-client-certificate.yaml | 46 ------------------- .../{10-assert.yaml => 09-assert.yaml} | 0 ...aml => 10-verify-served-certificates.yaml} | 0 ....yaml => 11-verify-secrets-unchanged.yaml} | 0 6 files changed, 11 insertions(+), 49 deletions(-) rename e2e-tests/tests/cert-management-policy/{09-create-instance-certificate.yaml => 08-create-instance-certificate.yaml} (100%) delete mode 100644 e2e-tests/tests/cert-management-policy/08-update-pgbackrest-client-certificate.yaml rename e2e-tests/tests/cert-management-policy/{10-assert.yaml => 09-assert.yaml} (100%) rename e2e-tests/tests/cert-management-policy/{11-verify-served-certificates.yaml => 10-verify-served-certificates.yaml} (100%) rename e2e-tests/tests/cert-management-policy/{12-verify-secrets-unchanged.yaml => 11-verify-secrets-unchanged.yaml} (100%) diff --git a/e2e-tests/tests/cert-management-policy/05-create-pgbackrest-certificates.yaml b/e2e-tests/tests/cert-management-policy/05-create-pgbackrest-certificates.yaml index 9094aac80f..3ee78ba814 100644 --- a/e2e-tests/tests/cert-management-policy/05-create-pgbackrest-certificates.yaml +++ b/e2e-tests/tests/cert-management-policy/05-create-pgbackrest-certificates.yaml @@ -10,9 +10,9 @@ commands: cluster="cert-management-policy" - # TODO: The user should know the PostgresCluster UID after the cluster is - # created so that the pgBackRest client certificate can use it. - pgbackrest_client="pgbackrest@pending" + cluster_uid=$(kubectl -n "$NAMESPACE" get postgrescluster "$cluster" \ + -o jsonpath='{.metadata.uid}') + pgbackrest_client="pgbackrest@${cluster_uid}" issue_certificate pgbackrest-client \ "$pgbackrest_client" clientAuth "$pgbackrest_client" @@ -34,3 +34,11 @@ commands: | kubectl label --local -f - -o yaml \ postgres-operator.crunchydata.com/cluster="$cluster" \ | kubectl -n "$NAMESPACE" apply -f - + + secret="${cluster}-pgbackrest" + checksum=$(kubectl -n "$NAMESPACE" get secret "$secret" -o json \ + | jq --compact-output --sort-keys '.data' \ + | sha256sum | awk '{print $1}') + kubectl -n "$NAMESPACE" annotate secret "$secret" \ + e2e.percona.com/secret-checksum="$checksum" \ + --overwrite diff --git a/e2e-tests/tests/cert-management-policy/09-create-instance-certificate.yaml b/e2e-tests/tests/cert-management-policy/08-create-instance-certificate.yaml similarity index 100% rename from e2e-tests/tests/cert-management-policy/09-create-instance-certificate.yaml rename to e2e-tests/tests/cert-management-policy/08-create-instance-certificate.yaml diff --git a/e2e-tests/tests/cert-management-policy/08-update-pgbackrest-client-certificate.yaml b/e2e-tests/tests/cert-management-policy/08-update-pgbackrest-client-certificate.yaml deleted file mode 100644 index eb83e897bd..0000000000 --- a/e2e-tests/tests/cert-management-policy/08-update-pgbackrest-client-certificate.yaml +++ /dev/null @@ -1,46 +0,0 @@ -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - script: |- - set -o errexit - set -o pipefail - set -o xtrace - - source ../../functions - - cluster="cert-management-policy" - - # TODO: The user should know the PostgresCluster UID after the cluster is - # created so that the pgBackRest client certificate can use it. - cluster_uid=$(kubectl -n "$NAMESPACE" get postgrescluster "$cluster" \ - -o jsonpath='{.metadata.uid}') - pgbackrest_client="pgbackrest@${cluster_uid}" - - issue_certificate pgbackrest-client \ - "$pgbackrest_client" clientAuth "$pgbackrest_client" - - pgbackrest_cert=$(base64 <"$TEMP_DIR/pgbackrest-client.crt" \ - | tr -d '\n') - pgbackrest_key=$(base64 <"$TEMP_DIR/pgbackrest-client.key" \ - | tr -d '\n') - kubectl -n "$NAMESPACE" get secret "${cluster}-pgbackrest" -o yaml \ - | pgbackrest_cert="$pgbackrest_cert" pgbackrest_key="$pgbackrest_key" \ - yq eval ' - .data."pgbackrest-client.crt" = strenv(pgbackrest_cert) | - .data."pgbackrest-client.key" = strenv(pgbackrest_key) | - del( - .metadata.creationTimestamp, - .metadata.managedFields, - .metadata.resourceVersion, - .metadata.uid - ) - ' - \ - | kubectl -n "$NAMESPACE" apply -f - - - secret="${cluster}-pgbackrest" - checksum=$(kubectl -n "$NAMESPACE" get secret "$secret" -o json \ - | jq --compact-output --sort-keys '.data' \ - | sha256sum | awk '{print $1}') - kubectl -n "$NAMESPACE" annotate secret "$secret" \ - e2e.percona.com/secret-checksum="$checksum" \ - --overwrite diff --git a/e2e-tests/tests/cert-management-policy/10-assert.yaml b/e2e-tests/tests/cert-management-policy/09-assert.yaml similarity index 100% rename from e2e-tests/tests/cert-management-policy/10-assert.yaml rename to e2e-tests/tests/cert-management-policy/09-assert.yaml diff --git a/e2e-tests/tests/cert-management-policy/11-verify-served-certificates.yaml b/e2e-tests/tests/cert-management-policy/10-verify-served-certificates.yaml similarity index 100% rename from e2e-tests/tests/cert-management-policy/11-verify-served-certificates.yaml rename to e2e-tests/tests/cert-management-policy/10-verify-served-certificates.yaml diff --git a/e2e-tests/tests/cert-management-policy/12-verify-secrets-unchanged.yaml b/e2e-tests/tests/cert-management-policy/11-verify-secrets-unchanged.yaml similarity index 100% rename from e2e-tests/tests/cert-management-policy/12-verify-secrets-unchanged.yaml rename to e2e-tests/tests/cert-management-policy/11-verify-secrets-unchanged.yaml From fc44975887aaebe09554a3b0258a3788ae20cada Mon Sep 17 00:00:00 2001 From: Andrii Dema Date: Wed, 12 Aug 2026 14:17:53 +0300 Subject: [PATCH 4/5] match the behavior from PR #1736 --- .../controller/postgrescluster/pgbouncer.go | 6 +- internal/controller/postgrescluster/pki.go | 13 ++- .../controller/postgrescluster/pki_test.go | 17 ++- internal/pgbouncer/reconcile.go | 5 + internal/pgbouncer/reconcile_test.go | 108 ++++++++++++------ 5 files changed, 98 insertions(+), 51 deletions(-) diff --git a/internal/controller/postgrescluster/pgbouncer.go b/internal/controller/postgrescluster/pgbouncer.go index 3df5b8ef3a..04c400f8f6 100644 --- a/internal/controller/postgrescluster/pgbouncer.go +++ b/internal/controller/postgrescluster/pgbouncer.go @@ -276,7 +276,8 @@ func (r *Reconciler) reconcilePGBouncerSecret( err = client.IgnoreNotFound(err) - if cluster.Spec.TLS.GetCertManagementPolicy() == v1beta1.CertManagementUserProvidedOnly { + if cluster.Spec.TLS.GetCertManagementPolicy() == v1beta1.CertManagementUserProvidedOnly && + cluster.Spec.Proxy.PGBouncer.CustomTLSSecret == nil { if !secretFound { return nil, errors.Errorf("user-provided PgBouncer secret %q is missing", naming.ClusterPGBouncer(cluster).Name) } @@ -290,7 +291,8 @@ func (r *Reconciler) reconcilePGBouncerSecret( } var frontendCertManagerSecret *corev1.Secret - if cluster.Spec.Proxy.PGBouncer.CustomTLSSecret == nil { + if cluster.Spec.Proxy.PGBouncer.CustomTLSSecret == nil && + cluster.Spec.TLS.GetCertManagementPolicy() != v1beta1.CertManagementUserProvidedOnly { certManagerManaged, certErr := r.isRootCACertManagerManaged(ctx, cluster) if certErr != nil { return nil, errors.Wrap(certErr, "failed to check if cert-manager manages root CA") diff --git a/internal/controller/postgrescluster/pki.go b/internal/controller/postgrescluster/pki.go index 1f9dbaa309..2968a8a408 100644 --- a/internal/controller/postgrescluster/pki.go +++ b/internal/controller/postgrescluster/pki.go @@ -94,16 +94,17 @@ func (r *Reconciler) reconcileTLSCondition(ctx context.Context, cluster *v1beta1 } if cluster.Spec.Proxy != nil && cluster.Spec.Proxy.PGBouncer != nil { - var requiredKeys []string - if cluster.Spec.Proxy.PGBouncer.CustomTLSSecret == nil { - requiredKeys = []string{ + customTLSSecret := cluster.Spec.Proxy.PGBouncer.CustomTLSSecret + if customTLSSecret == nil { + if err := checkSecret(nil, naming.ClusterPGBouncer(cluster).Name, pgbouncer.CertFrontendAuthoritySecretKey, pgbouncer.CertFrontendSecretKey, pgbouncer.CertFrontendPrivateKeySecretKey, + ); err != nil { + return errors.Wrap(err, "check PgBouncer TLS secret") } - } - if err := checkSecret(nil, naming.ClusterPGBouncer(cluster).Name, requiredKeys...); err != nil { - return errors.Wrap(err, "check PgBouncer TLS secret") + } else if err := checkSecret(customTLSSecret, naming.ClusterPGBouncer(cluster).Name); err != nil { + return errors.Wrap(err, "check custom PgBouncer TLS secret") } } diff --git a/internal/controller/postgrescluster/pki_test.go b/internal/controller/postgrescluster/pki_test.go index f1d106f7cd..ec24815d9e 100644 --- a/internal/controller/postgrescluster/pki_test.go +++ b/internal/controller/postgrescluster/pki_test.go @@ -76,6 +76,9 @@ func TestReconcileTLSCondition(t *testing.T) { cluster.Spec.CustomReplicationClientTLSSecret = &corev1.SecretProjection{ LocalObjectReference: corev1.LocalObjectReference{Name: "custom-replication"}, } + cluster.Spec.Proxy.PGBouncer.CustomTLSSecret = &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{Name: "custom-pgbouncer-tls"}, + } instance := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{ Name: "hippo-instance1-abcd", @@ -101,7 +104,7 @@ func TestReconcileTLSCondition(t *testing.T) { naming.PostgresTLSSecret(cluster).Name, "custom-replication", naming.PGBackRestSecret(cluster).Name, - naming.ClusterPGBouncer(cluster).Name, + "custom-pgbouncer-tls", naming.InstanceCertificates(instance).Name, }, ", ")+". certManagementPolicy is userProvidedOnly") }) @@ -122,20 +125,16 @@ func TestReconcileTLSCondition(t *testing.T) { cluster.Spec.CustomReplicationClientTLSSecret = &corev1.SecretProjection{ LocalObjectReference: corev1.LocalObjectReference{Name: "custom-replication"}, } + cluster.Spec.Proxy.PGBouncer.CustomTLSSecret = &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{Name: "custom-pgbouncer-tls"}, + } objects := []client.Object{ &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "custom-root-ca", Namespace: cluster.Namespace}}, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "custom-postgres-tls", Namespace: cluster.Namespace}}, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "custom-replication", Namespace: cluster.Namespace}}, &corev1.Secret{ObjectMeta: naming.PGBackRestSecret(cluster)}, - &corev1.Secret{ - ObjectMeta: naming.ClusterPGBouncer(cluster), - Data: map[string][]byte{ - pgbouncer.CertFrontendAuthoritySecretKey: []byte("ca"), - pgbouncer.CertFrontendSecretKey: []byte("cert"), - pgbouncer.CertFrontendPrivateKeySecretKey: []byte("key"), - }, - }, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "custom-pgbouncer-tls", Namespace: cluster.Namespace}}, } r := &Reconciler{Client: fake.NewClientBuilder().WithObjects(objects...).Build()} diff --git a/internal/pgbouncer/reconcile.go b/internal/pgbouncer/reconcile.go index d7aca3608f..085609e19a 100644 --- a/internal/pgbouncer/reconcile.go +++ b/internal/pgbouncer/reconcile.go @@ -185,6 +185,11 @@ func Secret(ctx context.Context, // bundle so PgBouncer also trusts them when verifying client // certificates. Entries keep their given order so identical inputs // always produce identical bundle bytes. + // + // K8SPG-1045: Add additional CAs when the certManagementPolicy is auto + // or when a custom pgbouncer tls secret is configured. + // In userProvidedOnly mode without a custom Secret, operator should + // keep the user-provided CA bundle unchanged. if err == nil && len(additionalCAs) > 0 && (inCluster.Spec.Proxy.PGBouncer.CustomTLSSecret != nil || inCluster.Spec.TLS.GetCertManagementPolicy() != v1beta1.CertManagementUserProvidedOnly) { diff --git a/internal/pgbouncer/reconcile_test.go b/internal/pgbouncer/reconcile_test.go index 91a8d18088..cfff674c6f 100644 --- a/internal/pgbouncer/reconcile_test.go +++ b/internal/pgbouncer/reconcile_test.go @@ -14,6 +14,7 @@ import ( "gotest.tools/v3/assert" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/percona/percona-postgresql-operator/v2/internal/feature" "github.com/percona/percona-postgresql-operator/v2/internal/naming" @@ -215,44 +216,83 @@ func TestSecretAdminPassword(t *testing.T) { }) } -func TestSecretUserProvidedOnly(t *testing.T) { +func TestSecretCertManagementPolicy(t *testing.T) { t.Parallel() - ctx := t.Context() - cluster := new(v1beta1.PostgresCluster) - cluster.SetLabels(map[string]string{naming.LabelVersion: "3.1.0"}) - cluster.Spec.Proxy = &v1beta1.PostgresProxySpec{ - PGBouncer: new(v1beta1.PGBouncerPodSpec), - } - cluster.Spec.TLS = &v1beta1.TLSSpec{ - CertManagementPolicy: v1beta1.CertManagementUserProvidedOnly, - } - assert.NilError(t, cluster.Default(ctx, nil)) - - existing := &corev1.Secret{Data: map[string][]byte{ - "pgbouncer-frontend.ca-roots": []byte("user-ca"), - "pgbouncer-frontend.crt": []byte("user-cert"), - "pgbouncer-frontend.key": []byte("user-key"), - }} - intent := new(corev1.Secret) - - require.NoError(t, Secret(ctx, cluster, nil, existing, nil, - new(corev1.Service), intent, nil, [][]byte{[]byte("additional-ca")})) + root, err := pki.NewRootCertificateAuthority() + assert.NilError(t, err) - assert.DeepEqual(t, intent.Data["pgbouncer-frontend.ca-roots"], []byte("user-ca")) - assert.DeepEqual(t, intent.Data["pgbouncer-frontend.crt"], []byte("user-cert")) - assert.DeepEqual(t, intent.Data["pgbouncer-frontend.key"], []byte("user-key")) - assert.Equal(t, len(intent.Data["pgbouncer-password"]), 32) - assert.Assert(t, len(intent.Data["pgbouncer-verifier"]) > 0) - assert.Equal(t, len(intent.Data["pgbouncer-admin-password"]), 32) - assert.Assert(t, len(intent.Data["pgbouncer-users.txt"]) > 0) + tests := []struct { + name string + policy v1beta1.CertManagementPolicy + customTLS bool + expectTLSData bool + }{ + { + name: "auto generates TLS data in the operator Secret", + policy: v1beta1.CertManagementAuto, + expectTLSData: true, + }, + { + name: "auto with custom TLS leaves TLS data out of the operator Secret", + policy: v1beta1.CertManagementAuto, + customTLS: true, + }, + { + name: "user provided only preserves TLS data", + policy: v1beta1.CertManagementUserProvidedOnly, + expectTLSData: true, + }, + { + name: "user provided only with custom TLS does not generate TLS data", + policy: v1beta1.CertManagementUserProvidedOnly, + customTLS: true, + }, + } - // Operator-managed credentials remain stable on later reconciles. - existing.Data = intent.Data - before := intent.DeepCopy() - require.NoError(t, Secret(ctx, cluster, nil, existing, nil, - new(corev1.Service), intent, nil, [][]byte{[]byte("additional-ca")})) - assert.DeepEqual(t, intent, before) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := t.Context() + cluster := new(v1beta1.PostgresCluster) + cluster.Spec.Proxy = &v1beta1.PostgresProxySpec{ + PGBouncer: new(v1beta1.PGBouncerPodSpec), + } + cluster.Spec.TLS = &v1beta1.TLSSpec{CertManagementPolicy: tt.policy} + if tt.customTLS { + cluster.Spec.Proxy.PGBouncer.CustomTLSSecret = &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{Name: "custom-pgbouncer-tls"}, + } + } + assert.NilError(t, cluster.Default(ctx, nil)) + + existing := &corev1.Secret{Data: map[string][]byte{ + "pgbouncer-frontend.ca-roots": []byte("user-ca"), + "pgbouncer-frontend.crt": []byte("user-cert"), + "pgbouncer-frontend.key": []byte("user-key"), + }} + intent := new(corev1.Secret) + service := &corev1.Service{ObjectMeta: metav1.ObjectMeta{ + Namespace: "ns1", Name: "some-name", + }} + + err := Secret(ctx, cluster, root, existing, nil, + service, intent, nil, nil) + require.NoError(t, err) + + assert.Assert(t, len(intent.Data["pgbouncer-password"]) != 0) + assert.Assert(t, len(intent.Data["pgbouncer-verifier"]) != 0) + assert.Assert(t, len(intent.Data["pgbouncer-users.txt"]) != 0) + + for _, key := range []string{ + "pgbouncer-frontend.ca-roots", + "pgbouncer-frontend.crt", + "pgbouncer-frontend.key", + } { + _, found := intent.Data[key] + assert.Equal(t, found, tt.expectTLSData, key) + } + }) + } } func TestSecretAdditionalCAs(t *testing.T) { From 1859818456d2ebcf6701000be5cafc295d4c9e36 Mon Sep 17 00:00:00 2001 From: Andrii Dema Date: Wed, 12 Aug 2026 14:30:59 +0300 Subject: [PATCH 5/5] skip unused additional ca lookup --- .../controller/postgrescluster/pgbouncer.go | 5 +++++ .../postgrescluster/pgbouncer_test.go | 19 +++++++++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/internal/controller/postgrescluster/pgbouncer.go b/internal/controller/postgrescluster/pgbouncer.go index 04c400f8f6..777f1e0005 100644 --- a/internal/controller/postgrescluster/pgbouncer.go +++ b/internal/controller/postgrescluster/pgbouncer.go @@ -358,6 +358,11 @@ func (r *Reconciler) reconcilePGBouncerSecret( func (r *Reconciler) getAdditionalTrustedCAs(ctx context.Context, cluster *v1beta1.PostgresCluster) ([][]byte, error) { pgBouncer := cluster.Spec.Proxy.PGBouncer + // K8SPG-1045: operator should keep user-provided CA bundle as-is in the userProvidedOnly mode + if cluster.Spec.TLS.GetCertManagementPolicy() == v1beta1.CertManagementUserProvidedOnly && + pgBouncer.CustomTLSSecret == nil { + return nil, nil + } if len(pgBouncer.AdditionalTrustedCAs) == 0 { return nil, nil } diff --git a/internal/controller/postgrescluster/pgbouncer_test.go b/internal/controller/postgrescluster/pgbouncer_test.go index 1f5800b212..076e354a30 100644 --- a/internal/controller/postgrescluster/pgbouncer_test.go +++ b/internal/controller/postgrescluster/pgbouncer_test.go @@ -50,8 +50,9 @@ func TestGetAdditionalTrustedCAs(t *testing.T) { customCA := []byte("-----BEGIN CERTIFICATE-----\nCCCC\n-----END CERTIFICATE-----\n") for name, data := range map[string]map[string][]byte{ - "ca-one": {"ca.crt": ca1}, - "ca-two": {"ca.crt": ca2}, + "ca-one": {"ca.crt": ca1}, + "ca-two": {"ca.crt": ca2}, + "ca-without-cert": {}, "custom-tls": { "my-ca": customCA, "tls.crt": []byte("cert"), @@ -80,6 +81,20 @@ func TestGetAdditionalTrustedCAs(t *testing.T) { assert.DeepEqual(t, cas, [][]byte{ca1, ca2}) }) + t.Run("UserProvidedOnlyWithoutCustomTLSIgnoresReferences", func(t *testing.T) { + cluster := cluster.DeepCopy() + cluster.Spec.TLS = &v1beta1.TLSSpec{ + CertManagementPolicy: v1beta1.CertManagementUserProvidedOnly, + } + cluster.Spec.Proxy.PGBouncer.AdditionalTrustedCAs = []corev1.LocalObjectReference{ + {Name: "ca-without-cert"}, + } + + cas, err := reconciler.getAdditionalTrustedCAs(ctx, cluster) + assert.NilError(t, err) + assert.Equal(t, len(cas), 0) + }) + // K8SPG-952: in manual TLS mode the frontend CA file is built solely // from this list, so it must begin with the authority of the custom // TLS Secret.