From c3b1f29f8e20e643970b3f0c4c176b181a51e8b7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= <zbynek.drapela@gmail.com>
Date: Fri, 24 Jul 2026 10:12:56 +0200
Subject: [PATCH 1/6] =?UTF-8?q?test(ci):=20verify=20operator-local=20Fedor?=
 =?UTF-8?q?a=20PostgreSQL=2015=E2=86=9218=20dump/restore?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Exercise RHIDP-14594 on e2e-ocp-operator-nightly: dump/wipe/restore
operator-managed Fedora Postgres (RELATED_IMAGE_postgresql), prove catalog
data survives, and wait for Backstage pod replacement before Playwright.
---
 .ci/pipelines/jobs/ocp-operator.sh            | 196 ++++--
 .ci/pipelines/lib/postgres.sh                 | 620 ++++++++++++++++++
 .../resources/pg-upgrade/catalog-info.yaml    |  13 +
 .../pg-upgrade/data-proof-server.yaml         | 104 +++
 .../e2e/pg-upgrade-data-proof.spec.ts         |  47 ++
 5 files changed, 926 insertions(+), 54 deletions(-)
 create mode 100644 .ci/pipelines/lib/postgres.sh
 create mode 100644 .ci/pipelines/resources/pg-upgrade/catalog-info.yaml
 create mode 100644 .ci/pipelines/resources/pg-upgrade/data-proof-server.yaml
 create mode 100644 e2e-tests/playwright/e2e/pg-upgrade-data-proof.spec.ts

diff --git a/.ci/pipelines/jobs/ocp-operator.sh b/.ci/pipelines/jobs/ocp-operator.sh
index 13240ec159..b9d908c690 100644
--- a/.ci/pipelines/jobs/ocp-operator.sh
+++ b/.ci/pipelines/jobs/ocp-operator.sh
@@ -10,6 +10,8 @@ source "$DIR"/utils.sh
 source "$DIR"/install-methods/operator.sh
 # shellcheck source=.ci/pipelines/lib/testing.sh
 source "$DIR"/lib/testing.sh
+# shellcheck source=.ci/pipelines/lib/postgres.sh
+source "$DIR"/lib/postgres.sh
 # shellcheck source=.ci/pipelines/playwright-projects.sh
 source "$DIR"/playwright-projects.sh
 # shellcheck source=.ci/pipelines/lib/schema-mode-env.sh
@@ -17,8 +19,12 @@ source "$DIR"/lib/schema-mode-env.sh
 
 export INSTALL_METHOD=operator
 
-initiate_operator_deployments() {
-  log::info "Initiating Operator-backed deployments on OCP"
+# Default operator-local DB image (RELATED_IMAGE_postgresql). Fedora path: 15 → 18 dump/restore.
+OPERATOR_PG15_IMAGE="${OPERATOR_PG15_IMAGE:-quay.io/fedora/postgresql-15:latest}"
+OPERATOR_PG18_IMAGE="${OPERATOR_PG18_IMAGE:-quay.io/fedora/postgresql-18:latest}"
+
+initiate_operator_showcase_only() {
+  log::info "Initiating Operator-backed showcase deployment (local Fedora PostgreSQL)"
 
   namespace::configure "${NAME_SPACE}"
   deploy_test_backstage_customization_provider "${NAME_SPACE}"
@@ -28,90 +34,172 @@ initiate_operator_deployments() {
   oc apply -f /tmp/configmap-dynamic-plugins.yaml -n "${NAME_SPACE}"
   deploy_redis_cache "${NAME_SPACE}"
   deploy_rhdh_operator "${NAME_SPACE}" "${DIR}/resources/rhdh-operator/rhdh-start.yaml"
-
-  namespace::configure "${NAME_SPACE_RBAC}"
-  config::prepare_operator_app_config "${DIR}/resources/config_map/app-config-rhdh-rbac.yaml"
-  local rbac_rhdh_base_url="https://backstage-${RELEASE_NAME_RBAC}-${NAME_SPACE_RBAC}.${K8S_CLUSTER_ROUTER_BASE}"
-  apply_yaml_files "${DIR}" "${NAME_SPACE_RBAC}" "${rbac_rhdh_base_url}"
-  config::create_dynamic_plugins_config "${DIR}/value_files/${HELM_CHART_RBAC_VALUE_FILE_NAME}" "/tmp/configmap-dynamic-plugins-rbac.yaml"
-  oc apply -f /tmp/configmap-dynamic-plugins-rbac.yaml -n "${NAME_SPACE_RBAC}"
-  wait_for_crunchy_crd || return 1
-  deploy_rhdh_operator "${NAME_SPACE_RBAC}" "${DIR}/resources/rhdh-operator/rhdh-start-rbac.yaml"
 }
 
-initiate_operator_deployments_osd_gcp() {
-  log::info "Initiating Operator-backed deployments on OSD-GCP"
+# Wait for Postgres + Backstage, run Playwright, always persist artifacts under a unique subdir.
+# Args:
+#   $1 - label
+#   $2 - artifacts_subdir
+#   $3 - url
+#   $4 - expected postgres image substring (e.g. postgresql-18)
+#   $5 - optional max wait seconds (default: 600)
+#   $6 - optional previous RHDH pod UID
+#   $7 - optional "seed" to register persistence-proof catalog entity
+_pg_upgrade_verify_and_test() {
+  local label=$1
+  local artifacts_subdir=$2
+  local url=$3
+  local expected_image_substr=$4
+  local max_wait=${5:-600}
+  local previous_rhdh_uid=${6:-}
+  local seed_data=${7:-}
+  local deploy
+  deploy=$(rhdh_deployment_name "${RELEASE_NAME}")
+
+  log::info "=== ${label}: wait for PostgreSQL (${expected_image_substr}) + Backstage, then Playwright ==="
+  log::info "Artifacts subdir: ${ARTIFACT_DIR:-<unset>}/${artifacts_subdir}"
+
+  if ! wait_for_postgres_ready "${NAME_SPACE}" "${max_wait}" "${expected_image_substr}" > /dev/null; then
+    log::error "PostgreSQL not Ready after ${label}"
+    dump_postgres_diagnostics "${NAME_SPACE}"
+    _pg_save_phase_artifacts "${artifacts_subdir}" "${label}"
+    return 1
+  fi
+  log_postgres_version "${NAME_SPACE}"
+
+  if [[ -n "${previous_rhdh_uid}" ]]; then
+    if ! ensure_rhdh_pod_replaced "${RELEASE_NAME}" "${NAME_SPACE}" "${previous_rhdh_uid}" "${max_wait}"; then
+      log::error "Previous RHDH pod did not terminate cleanly after ${label}"
+      dump_postgres_diagnostics "${NAME_SPACE}"
+      _pg_save_phase_artifacts "${artifacts_subdir}" "${label}"
+      return 1
+    fi
+  fi
 
-  namespace::configure "${NAME_SPACE}"
-  deploy_test_backstage_customization_provider "${NAME_SPACE}"
-  local rhdh_base_url="https://backstage-${RELEASE_NAME}-${NAME_SPACE}.${K8S_CLUSTER_ROUTER_BASE}"
-  apply_yaml_files "${DIR}" "${NAME_SPACE}" "${rhdh_base_url}"
+  oc rollout status "deployment/${deploy}" -n "${NAME_SPACE}" --timeout=10m \
+    || log::warn "Backstage rollout status check timed out after ${label}"
 
-  config::create_dynamic_plugins_config "${DIR}/value_files/${HELM_CHART_VALUE_FILE_NAME}" "/tmp/configmap-dynamic-plugins.yaml"
-  common::save_artifact "${PW_PROJECT_SHOWCASE_OPERATOR}" "/tmp/configmap-dynamic-plugins.yaml"
+  if ! testing::check_backstage_running "${RELEASE_NAME}" "${NAME_SPACE}" "${url}" "${artifacts_subdir}" 40 30; then
+    log::error "Backstage not healthy after ${label}"
+    dump_postgres_diagnostics "${NAME_SPACE}"
+    _pg_save_phase_artifacts "${artifacts_subdir}" "${label}"
+    return 1
+  fi
 
-  oc apply -f /tmp/configmap-dynamic-plugins.yaml -n "${NAME_SPACE}"
-  deploy_redis_cache "${NAME_SPACE}"
-  deploy_rhdh_operator "${NAME_SPACE}" "${DIR}/resources/rhdh-operator/rhdh-start.yaml"
+  if [[ "${seed_data}" == "seed" ]]; then
+    if ! seed_pg_upgrade_data_proof "${url}" "${NAME_SPACE}"; then
+      _pg_save_phase_artifacts "${artifacts_subdir}" "${label}"
+      return 1
+    fi
+  elif [[ "${PG_UPGRADE_DATA_PROOF:-}" == "1" ]]; then
+    if ! assert_pg_upgrade_data_proof_api "${url}"; then
+      log::error "Persistence proof entity missing via API after ${label}"
+      _pg_save_phase_artifacts "${artifacts_subdir}" "${label}"
+      return 1
+    fi
+  fi
 
-  namespace::configure "${NAME_SPACE_RBAC}"
-  config::prepare_operator_app_config "${DIR}/resources/config_map/app-config-rhdh-rbac.yaml"
-  local rbac_rhdh_base_url="https://backstage-${RELEASE_NAME_RBAC}-${NAME_SPACE_RBAC}.${K8S_CLUSTER_ROUTER_BASE}"
-  apply_yaml_files "${DIR}" "${NAME_SPACE_RBAC}" "${rbac_rhdh_base_url}"
+  export PG_UPGRADE_DATA_PROOF=1
 
-  config::create_dynamic_plugins_config "${DIR}/value_files/${HELM_CHART_RBAC_VALUE_FILE_NAME}" "/tmp/configmap-dynamic-plugins-rbac.yaml"
-  common::save_artifact "${PW_PROJECT_SHOWCASE_OPERATOR_RBAC}" "/tmp/configmap-dynamic-plugins-rbac.yaml"
+  testing::check_and_test "${RELEASE_NAME}" "${NAME_SPACE}" "${PW_PROJECT_SHOWCASE_OPERATOR}" "${url}" "" "" "${artifacts_subdir}"
 
-  oc apply -f /tmp/configmap-dynamic-plugins-rbac.yaml -n "${NAME_SPACE_RBAC}"
-  wait_for_crunchy_crd || return 1
-  deploy_rhdh_operator "${NAME_SPACE_RBAC}" "${DIR}/resources/rhdh-operator/rhdh-start-rbac.yaml"
+  _pg_save_phase_artifacts "${artifacts_subdir}" "${label}"
+
+  if [[ "${OVERALL_RESULT:-0}" -ne 0 ]]; then
+    log::error "Playwright (or prior step) failed during ${label}; aborting remaining upgrade phases"
+    return 1
+  fi
 }
 
-run_operator_runtime_config_change_tests() {
-  # Runtime tests handle their own deployment via TypeScript (runtime-deploy.ts).
-  # The first test file (config-map.spec.ts) calls ensureRuntimeDeployed() which:
-  #   - Creates the namespace
-  #   - Deploys RHDH via the operator with internal PostgreSQL
-  #   - Configures schema-mode env vars for port-forwarding
-  # Subsequent test files reuse the existing deployment (workers: 1).
-  #
-  # INSTALL_METHOD=operator is already exported in handle_ocp_operator().
-  #
-  # No URL is passed on purpose - see the same note in jobs/ocp-nightly.sh.
-  # A pre-set BASE_URL makes global-setup.ts skip the deploy branch and poll a
-  # route nothing has created yet; ensureRuntimeDeployed() sets BASE_URL itself.
-  export RUNTIME_AUTO_DEPLOY="true"
-  testing::run_tests "${RELEASE_NAME}" "${NAME_SPACE_RUNTIME}" "${PW_PROJECT_SHOWCASE_RUNTIME}" || true
+_pg_save_phase_artifacts() {
+  local artifacts_subdir=$1
+  local label=$2
+  local diag_file="/tmp/pg-diagnostics-${artifacts_subdir}.txt"
+
+  log::info "Saving phase artifacts for '${label}' → ${ARTIFACT_DIR:-<unset>}/${artifacts_subdir}"
+  dump_postgres_diagnostics "${NAME_SPACE}" > "${diag_file}" 2>&1 || true
+  common::save_artifact "${artifacts_subdir}" "${diag_file}" "postgres" || true
+  save_all_pod_logs "${NAME_SPACE}" "${artifacts_subdir}"
 }
 
+# RHIDP-14594: Operator-local Fedora PostgreSQL upgrade via dump/restore.
+# Operator defaults RELATED_IMAGE_postgresql to quay.io/fedora/postgresql-15.
+# Same Fedora constraints as Helm path (#5141): no reliable POSTGRESQL_UPGRADE=copy,
+# no fedora/postgresql-17 — jump 15 → 18 with pg_dumpall / wipe PVC / restore.
+# Showcase-only (RBAC + runtime skipped) to keep the job focused on DB upgrade evidence.
 handle_ocp_operator() {
   export NAME_SPACE="${NAME_SPACE:-showcase}"
   export NAME_SPACE_RBAC="${NAME_SPACE_RBAC:-showcase-rbac}"
   export NAME_SPACE_RUNTIME="${NAME_SPACE_RUNTIME:-showcase-runtime}"
   export NAME_SPACE_POSTGRES_DB="${NAME_SPACE_POSTGRES_DB:-postgress-external-db}"
+  export POSTGRES_CR_NAME="${POSTGRES_CR_NAME:-${RELEASE_NAME:-rhdh}}"
 
   common::oc_login
 
   K8S_CLUSTER_ROUTER_BASE=$(oc get route console -n openshift-console -o=jsonpath='{.spec.host}' | sed 's/^[^.]*\.//')
   export K8S_CLUSTER_ROUTER_BASE
   local url="https://backstage-${RELEASE_NAME}-${NAME_SPACE}.${K8S_CLUSTER_ROUTER_BASE}"
-  local rbac_url="https://backstage-${RELEASE_NAME_RBAC}-${NAME_SPACE_RBAC}.${K8S_CLUSTER_ROUTER_BASE}"
+  local art_pg15="${PW_PROJECT_SHOWCASE_OPERATOR}-pg15"
+  local art_pg18="${PW_PROJECT_SHOWCASE_OPERATOR}-pg18"
+  local deploy
+  deploy=$(rhdh_deployment_name "${RELEASE_NAME}")
 
   cluster_setup_ocp_operator
 
   if [[ "${JOB_NAME}" =~ osd-gcp ]]; then
-    # OSD-GCP internal registry is unreliable under parallel load; reduce
-    # concurrent skopeo pushes and allow retries (RHDHBUGS-1136).
     export MAX_PARALLEL=3
     prepare_operator 3
-    initiate_operator_deployments_osd_gcp
   else
     prepare_operator
-    initiate_operator_deployments
   fi
 
-  testing::check_and_test "${RELEASE_NAME}" "${NAME_SPACE}" "${PW_PROJECT_SHOWCASE_OPERATOR}" "${url}"
-  testing::check_and_test "${RELEASE_NAME_RBAC}" "${NAME_SPACE_RBAC}" "${PW_PROJECT_SHOWCASE_OPERATOR_RBAC}" "${rbac_url}"
+  # Ensure baseline image is Fedora PG15 (operator CSV default; set explicitly for evidence).
+  if ! set_operator_postgresql_related_image "${OPERATOR_PG15_IMAGE}"; then
+    log::error "Failed to set operator RELATED_IMAGE_postgresql to PG15"
+    return 1
+  fi
+
+  initiate_operator_showcase_only
+
+  log::info "Phase PG15: operator-local ${OPERATOR_PG15_IMAGE}"
+  if ! _pg_upgrade_verify_and_test "PG15 baseline" "${art_pg15}" "${url}" "postgresql-15" 600 "" "seed"; then
+    return 1
+  fi
+
+  local rhdh_uid_before_18
+  rhdh_uid_before_18=$(get_rhdh_pod_uid "${RELEASE_NAME}" "${NAME_SPACE}")
+  log::info "RHDH pod uid before PG18 dump/restore: ${rhdh_uid_before_18:-<none>}"
+
+  log::info "Phase PG18: dump/restore 15 → 18 (operator-local Fedora; skip PG16)"
+  local dumpfile="/tmp/rhdh-operator-pg15-dumpall.sql"
+  if ! postgres_dumpall_to_file "${NAME_SPACE}" "${dumpfile}"; then
+    _pg_save_phase_artifacts "${art_pg18}" "PG18 dump failed"
+    return 1
+  fi
+  common::save_artifact "${art_pg18}" "${dumpfile}" "postgres" || true
+
+  # Switch operator default DB image to PG18 before wiping so recreate uses 18.
+  if ! set_operator_postgresql_related_image "${OPERATOR_PG18_IMAGE}"; then
+    dump_postgres_diagnostics "${NAME_SPACE}"
+    _pg_save_phase_artifacts "${art_pg18}" "PG18 RELATED_IMAGE update failed"
+    return 1
+  fi
+
+  postgres_wipe_persistent_volume "${NAME_SPACE}"
+
+  if ! postgres_restore_dumpall_file "${NAME_SPACE}" "${dumpfile}" "postgresql-18"; then
+    dump_postgres_diagnostics "${NAME_SPACE}"
+    _pg_save_phase_artifacts "${art_pg18}" "PG18 restore failed"
+    return 1
+  fi
+  refresh_postgres_collation_versions "${NAME_SPACE}" 600 "postgresql-18"
+
+  oc rollout restart "deployment/${deploy}" -n "${NAME_SPACE}" || true
+
+  if ! _pg_upgrade_verify_and_test "PG18 after dump/restore" "${art_pg18}" "${url}" "postgresql-18" 900 "${rhdh_uid_before_18}"; then
+    return 1
+  fi
 
-  run_operator_runtime_config_change_tests
+  log::info "PostgreSQL 15 → 18 Operator Fedora dump/restore sequence completed (Playwright after each major)"
+  log::info "Skipping operator RBAC + runtime suites on this evidence PR (showcase-only)"
 }
diff --git a/.ci/pipelines/lib/postgres.sh b/.ci/pipelines/lib/postgres.sh
new file mode 100644
index 0000000000..309e646d03
--- /dev/null
+++ b/.ci/pipelines/lib/postgres.sh
@@ -0,0 +1,620 @@
+#!/usr/bin/env bash
+
+# PostgreSQL helpers for chart-managed / operator-local database upgrades.
+# Dependencies: oc, lib/log.sh
+#
+# Install method awareness (INSTALL_METHOD=helm|operator):
+#   helm     — label app.kubernetes.io/name=postgresql, deploy ${release}-developer-hub
+#   operator — label rhdh.redhat.com/app=backstage-psql-${cr}, deploy backstage-${cr}
+#
+# IMPORTANT: functions that print a pod name on stdout (for capture) MUST send
+# all diagnostics to stderr, otherwise callers using $(...) swallow the evidence.
+
+# Prevent re-sourcing
+if [[ -n "${POSTGRES_LIB_SOURCED:-}" ]]; then
+  return 0
+fi
+readonly POSTGRES_LIB_SOURCED=1
+
+# shellcheck source=.ci/pipelines/lib/log.sh
+source "${DIR}/lib/log.sh"
+
+# ---------------------------------------------------------------------------
+# Install-method helpers
+# ---------------------------------------------------------------------------
+
+# Backstage Deployment name for the given release / CR name.
+rhdh_deployment_name() {
+  local release_name=$1
+  if [[ "${INSTALL_METHOD:-helm}" == "operator" ]]; then
+    echo "backstage-${release_name}"
+  else
+    echo "${release_name}-developer-hub"
+  fi
+}
+
+# CR / release name used for operator-local Postgres resources.
+_postgres_cr_name() {
+  echo "${POSTGRES_CR_NAME:-${RELEASE_NAME:-rhdh}}"
+}
+
+# Resolve a Ready-candidate Postgres pod name in the namespace (or empty).
+_postgres_find_pod() {
+  local namespace=$1
+  local pod=""
+  if [[ "${INSTALL_METHOD:-helm}" == "operator" ]]; then
+    local cr
+    cr=$(_postgres_cr_name)
+    pod=$(oc get pods -n "${namespace}" -l "rhdh.redhat.com/app=backstage-psql-${cr}" \
+      -o jsonpath='{.items[0].metadata.name}' 2> /dev/null || true)
+    if [[ -z "${pod}" ]]; then
+      pod=$(oc get pods -n "${namespace}" \
+        -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2> /dev/null \
+        | grep -E "^backstage-psql-${cr}-" | head -1 || true)
+    fi
+  else
+    pod=$(oc get pods -n "${namespace}" -l "app.kubernetes.io/name=postgresql" \
+      -o jsonpath='{.items[0].metadata.name}' 2> /dev/null || true)
+  fi
+  echo "${pod}"
+}
+
+# StatefulSet readyReplicas for the managed Postgres instance (or 0).
+_postgres_sts_ready_replicas() {
+  local namespace=$1
+  local sts_ready=""
+  if [[ "${INSTALL_METHOD:-helm}" == "operator" ]]; then
+    local cr
+    cr=$(_postgres_cr_name)
+    sts_ready=$(oc get statefulset -n "${namespace}" "backstage-psql-${cr}" \
+      -o jsonpath='{.status.readyReplicas}' 2> /dev/null || true)
+    if [[ -z "${sts_ready}" ]]; then
+      sts_ready=$(oc get statefulset -n "${namespace}" \
+        -l "rhdh.redhat.com/app=backstage-psql-${cr}" \
+        -o jsonpath='{.items[0].status.readyReplicas}' 2> /dev/null || true)
+    fi
+  else
+    sts_ready=$(oc get statefulset -n "${namespace}" -l "app.kubernetes.io/name=postgresql" \
+      -o jsonpath='{.items[0].status.readyReplicas}' 2> /dev/null || true)
+  fi
+  echo "${sts_ready:-0}"
+}
+
+# Point the RHDH operator at a different Fedora/RHEL PostgreSQL image via RELATED_IMAGE_postgresql.
+# Args:
+#   $1 - full image ref (e.g. quay.io/fedora/postgresql-18:latest)
+set_operator_postgresql_related_image() {
+  local image=$1
+  local ns="${OPERATOR_MANAGER:-rhdh-operator}"
+  local deploy
+
+  if [[ -z "${image}" ]]; then
+    log::error "set_operator_postgresql_related_image: image required"
+    return 1
+  fi
+
+  log::info "Setting RELATED_IMAGE_postgresql=${image} on operator controller in ${ns}"
+  deploy=$(oc get deploy -n "${ns}" -l control-plane=controller-manager \
+    -o jsonpath='{.items[0].metadata.name}' 2> /dev/null || true)
+  if [[ -z "${deploy}" ]]; then
+    deploy=$(oc get deploy -n "${ns}" -o jsonpath='{.items[0].metadata.name}' 2> /dev/null || true)
+  fi
+  if [[ -z "${deploy}" ]]; then
+    log::error "No operator Deployment found in ${ns}"
+    return 1
+  fi
+
+  oc set env "deployment/${deploy}" -n "${ns}" "RELATED_IMAGE_postgresql=${image}"
+  if ! oc rollout status "deployment/${deploy}" -n "${ns}" --timeout=5m; then
+    log::error "Operator rollout failed after RELATED_IMAGE_postgresql update"
+    oc get pods -n "${ns}" -o wide >&2 || true
+    oc logs -n "${ns}" "deploy/${deploy}" --tail=80 >&2 || true
+    return 1
+  fi
+  log::info "Operator controller ready with RELATED_IMAGE_postgresql=${image}"
+}
+
+# Dump postgres pod diagnostics to stderr (safe under $(...) capture).
+# Args:
+#   $1 - namespace
+dump_postgres_diagnostics() {
+  local namespace=$1
+  {
+    log::info "PostgreSQL diagnostics for namespace: ${namespace}"
+    oc get pods,statefulset,pvc -n "${namespace}" -o wide 2>&1 | grep -iE 'postgres|NAME' || true
+    oc get pods -n "${namespace}" -o wide 2>&1 | head -40 || true
+    local pg_pod
+    pg_pod=$(_postgres_find_pod "${namespace}")
+    if [[ -z "$pg_pod" ]]; then
+      log::warn "No postgresql pod found; listing all pods containing 'postgres'"
+      oc get pods -n "${namespace}" 2>&1 | grep -i postgres || true
+      return 0
+    fi
+    log::info "Pod image/status for ${pg_pod}:"
+    oc get pod -n "${namespace}" "${pg_pod}" -o jsonpath='image={.spec.containers[0].image} phase={.status.phase} ready={.status.conditions[?(@.type=="Ready")].status}{"\n"}' 2>&1 || true
+    log::info "Describe pod ${pg_pod}:"
+    oc describe pod -n "${namespace}" "${pg_pod}" 2>&1 | tail -100 || true
+    log::info "Logs for ${pg_pod} (current):"
+    oc logs -n "${namespace}" "${pg_pod}" --tail=200 2>&1 || true
+    log::info "Logs for ${pg_pod} (previous):"
+    oc logs -n "${namespace}" "${pg_pod}" --previous --tail=200 2>&1 || true
+    if [[ "${INSTALL_METHOD:-helm}" == "operator" ]]; then
+      log::info "Operator RELATED_IMAGE_postgresql:"
+      oc get deploy -n "${OPERATOR_MANAGER:-rhdh-operator}" -o jsonpath='{range .items[*]}{.metadata.name}{" RELATED_IMAGE_postgresql="}{range .spec.template.spec.containers[0].env[?(@.name=="RELATED_IMAGE_postgresql")]}{.value}{end}{"\n"}{end}' 2>&1 || true
+      log::info "Backstage CR (database):"
+      oc get backstage -n "${namespace}" -o yaml 2>&1 | grep -A20 -E 'database:|enableLocalDb' || true
+    else
+      log::info "Helm values snapshot (postgresql image):"
+      helm get values rhdh -n "${namespace}" 2>&1 | grep -A20 -E 'postgresql:|^  image:' || true
+    fi
+  } >&2
+}
+
+# Wait until PostgreSQL is Ready and (optionally) running an expected image substring.
+# Args:
+#   $1 - namespace
+#   $2 - optional max wait seconds (default: 600)
+#   $3 - optional image substring that must appear in the container image
+# Prints the pod name on success; returns 1 on timeout.
+wait_for_postgres_ready() {
+  local namespace=$1
+  local max_wait=${2:-600}
+  local expected_image_substr=${3:-}
+  local pg_pod=""
+  local waited=0
+
+  if [[ -n "${expected_image_substr}" ]]; then
+    log::info "Waiting for PostgreSQL Ready with image containing '${expected_image_substr}' in ${namespace}"
+  else
+    log::info "Waiting for PostgreSQL pod to be Ready in namespace: ${namespace}"
+  fi
+
+  while [[ $waited -lt $max_wait ]]; do
+    pg_pod=$(_postgres_find_pod "${namespace}")
+    if [[ -n "$pg_pod" ]]; then
+      local phase ready image sts_ready
+      phase=$(oc get pod -n "${namespace}" "${pg_pod}" -o jsonpath='{.status.phase}' 2> /dev/null || true)
+      ready=$(oc get pod -n "${namespace}" "${pg_pod}" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2> /dev/null || true)
+      image=$(oc get pod -n "${namespace}" "${pg_pod}" -o jsonpath='{.spec.containers[0].image}' 2> /dev/null || true)
+      sts_ready=$(_postgres_sts_ready_replicas "${namespace}")
+
+      # Periodic progress on INFO so CI logs show what we are waiting for.
+      if ((waited % 60 == 0)); then
+        log::info "Still waiting… pod=${pg_pod:-none} phase=${phase:-?} ready=${ready:-?} sts_ready=${sts_ready:-0} image=${image:-?} (${waited}s/${max_wait}s)"
+      fi
+
+      if [[ -n "${expected_image_substr}" && "${image}" != *"${expected_image_substr}"* ]]; then
+        :
+      elif [[ "$phase" == "Running" && "$ready" == "True" && "${sts_ready:-0}" -ge 1 ]]; then
+        log::info "PostgreSQL pod is Ready: ${pg_pod} (image=${image})"
+        echo "${pg_pod}"
+        return 0
+      fi
+    elif ((waited % 60 == 0)); then
+      log::info "Still waiting… no postgresql pod yet (${waited}s/${max_wait}s)"
+      oc get pods -n "${namespace}" -o wide >&2 || true
+    fi
+    sleep 10
+    waited=$((waited + 10))
+  done
+
+  log::error "PostgreSQL pod not Ready in namespace ${namespace} after ${max_wait}s"
+  dump_postgres_diagnostics "${namespace}"
+  return 1
+}
+
+# Log the running PostgreSQL server version (for upgrade evidence).
+log_postgres_version() {
+  local namespace=$1
+  local pg_pod=${2:-}
+
+  if [[ -z "$pg_pod" ]]; then
+    pg_pod=$(_postgres_find_pod "${namespace}")
+  fi
+  if [[ -z "$pg_pod" ]]; then
+    log::warn "Cannot log PostgreSQL version: no pod found in ${namespace}"
+    return 0
+  fi
+
+  local version
+  version=$(oc exec -n "${namespace}" "${pg_pod}" -- psql -U postgres -t -A -c "SHOW server_version;" 2> /dev/null | tr -d '[:space:]' || true)
+  local image
+  image=$(oc get pod -n "${namespace}" "${pg_pod}" -o jsonpath='{.spec.containers[0].image}' 2> /dev/null || true)
+  log::info "PostgreSQL evidence — version='${version:-unknown}' image='${image:-unknown}' pod='${pg_pod}'"
+}
+
+# Refresh PostgreSQL collation versions after a major version upgrade.
+# Args:
+#   $1 - namespace
+#   $2 - optional max wait seconds (default: 600)
+#   $3 - optional expected image substring (e.g. postgresql-18)
+refresh_postgres_collation_versions() {
+  local namespace=$1
+  local max_wait=${2:-600}
+  local expected_image_substr=${3:-}
+
+  log::info "Refreshing PostgreSQL collation versions in namespace: ${namespace}"
+
+  local pg_pod
+  if ! pg_pod=$(wait_for_postgres_ready "${namespace}" "${max_wait}" "${expected_image_substr}"); then
+    log::warn "Skipping collation refresh; PostgreSQL not Ready."
+    return 0
+  fi
+
+  log::info "Found PostgreSQL pod: ${pg_pod}"
+
+  # Keep stderr separate so WARNING/DETAIL/HINT lines are not treated as DB names.
+  local databases
+  databases=$(oc exec -n "${namespace}" "${pg_pod}" -- psql -U postgres -t -A -c \
+    "SELECT datname FROM pg_database WHERE datistemplate = false AND datname NOT IN ('postgres');" 2> /dev/null \
+    | grep -E '^[A-Za-z0-9_.:-]+$' || true)
+  log::info "User databases for collation refresh: ${databases:-<none>}"
+
+  local db out
+  for db in postgres template1 $databases; do
+    if [[ -z "$db" ]]; then
+      continue
+    fi
+    log::info "Refreshing collation version for database: ${db}"
+    out=$(oc exec -n "${namespace}" "${pg_pod}" -- psql -U postgres -v ON_ERROR_STOP=1 -c \
+      "ALTER DATABASE \"${db}\" REFRESH COLLATION VERSION;" 2>&1) \
+      && log::debug "${out}" \
+      || log::warn "Failed to refresh collation for ${db}: ${out}"
+  done
+
+  log::info "Collation version refresh completed for namespace: ${namespace}"
+}
+
+# Dump all databases from the chart-managed PostgreSQL (for dump/restore upgrades).
+# Args:
+#   $1 - namespace
+#   $2 - output file path
+postgres_dumpall_to_file() {
+  local namespace=$1
+  local outfile=$2
+  local pg_pod
+
+  if ! pg_pod=$(wait_for_postgres_ready "${namespace}" 300); then
+    log::error "Cannot dumpall: PostgreSQL not Ready in ${namespace}"
+    return 1
+  fi
+
+  log::info "Running pg_dumpall from ${pg_pod} → ${outfile}"
+  mkdir -p "$(dirname "${outfile}")"
+  if ! oc exec -n "${namespace}" "${pg_pod}" -- pg_dumpall -U postgres > "${outfile}"; then
+    log::error "pg_dumpall failed"
+    return 1
+  fi
+  log::info "pg_dumpall wrote $(wc -c < "${outfile}" | tr -d ' ') bytes"
+}
+
+# Wipe the chart-managed Postgres PVC so the next helm install can initdb fresh.
+# Args:
+#   $1 - namespace
+postgres_wipe_persistent_volume() {
+  local namespace=$1
+  local cr pvc
+
+  log::info "Wiping PostgreSQL StatefulSet + PVC in ${namespace} for dump/restore upgrade (INSTALL_METHOD=${INSTALL_METHOD:-helm})"
+  if [[ "${INSTALL_METHOD:-helm}" == "operator" ]]; then
+    cr=$(_postgres_cr_name)
+    # Keep backstage-psql-secret-${cr} so RHDH credentials remain valid after restore.
+    oc delete statefulset -n "${namespace}" "backstage-psql-${cr}" --wait=true --timeout=5m 2>&1 || true
+    oc delete pod -n "${namespace}" -l "rhdh.redhat.com/app=backstage-psql-${cr}" --force --grace-period=0 2>&1 || true
+    pvc=$(oc get pvc -n "${namespace}" -o name 2> /dev/null | grep -i "backstage-psql-${cr}" | head -1 || true)
+    if [[ -z "${pvc}" ]]; then
+      pvc=$(oc get pvc -n "${namespace}" -o name 2> /dev/null | grep -i postgres | head -1 || true)
+    fi
+    if [[ -n "${pvc}" ]]; then
+      log::info "Deleting ${pvc}"
+      oc delete -n "${namespace}" "${pvc}" --wait=true --timeout=5m 2>&1 || true
+    else
+      log::warn "No postgres PVC found to delete"
+    fi
+    # Nudge the Backstage CR so the operator recreates the STS with the new RELATED_IMAGE.
+    oc annotate backstage "${cr}" -n "${namespace}" "rhdh.redhat.com/pg-upgrade-reconcile=$(date +%s)" --overwrite 2>&1 || true
+  else
+    oc delete statefulset -n "${namespace}" -l "app.kubernetes.io/name=postgresql" --wait=true --timeout=5m 2>&1 || true
+    oc delete pod -n "${namespace}" -l "app.kubernetes.io/name=postgresql" --force --grace-period=0 2>&1 || true
+    pvc=$(oc get pvc -n "${namespace}" -o name 2> /dev/null | grep -i postgres | head -1 || true)
+    if [[ -n "${pvc}" ]]; then
+      log::info "Deleting ${pvc}"
+      oc delete -n "${namespace}" "${pvc}" --wait=true --timeout=5m 2>&1 || true
+    else
+      log::warn "No postgres PVC found to delete"
+    fi
+  fi
+}
+
+# Restore a pg_dumpall file into a Ready PostgreSQL pod.
+# Args:
+#   $1 - namespace
+#   $2 - dump file path
+#   $3 - optional expected image substring
+postgres_restore_dumpall_file() {
+  local namespace=$1
+  local dumpfile=$2
+  local expected_image_substr=${3:-}
+  local pg_pod
+
+  if ! pg_pod=$(wait_for_postgres_ready "${namespace}" 600 "${expected_image_substr}"); then
+    log::error "Cannot restore: PostgreSQL not Ready in ${namespace}"
+    return 1
+  fi
+
+  log::info "Restoring pg_dumpall into ${pg_pod} from ${dumpfile}"
+  # Ignore benign "already exists" errors from globals/roles created by image init.
+  if ! oc exec -i -n "${namespace}" "${pg_pod}" -- psql -U postgres -v ON_ERROR_STOP=0 < "${dumpfile}"; then
+    log::error "psql restore failed"
+    return 1
+  fi
+  log::info "pg_dumpall restore completed"
+}
+
+# Return the UID of a Running RHDH (developer-hub) pod for the release, or empty.
+# Args:
+#   $1 - release name (e.g. rhdh)
+#   $2 - namespace
+get_rhdh_pod_uid() {
+  local release_name=$1
+  local namespace=$2
+  local name uid
+  local deploy
+  deploy=$(rhdh_deployment_name "${release_name}")
+
+  while IFS=$'\t' read -r name uid; do
+    if [[ -n "${name}" && "${name}" == "${deploy}"* && -n "${uid}" ]]; then
+      echo "${uid}"
+      return 0
+    fi
+  done < <(oc get pods -n "${namespace}" --field-selector=status.phase=Running \
+    -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.uid}{"\n"}{end}' 2> /dev/null || true)
+
+  return 0
+}
+
+# Wait until the previous RHDH pod UID is gone and a different Ready pod exists.
+# Args:
+#   $1 - release name
+#   $2 - namespace
+#   $3 - previous pod UID
+#   $4 - optional max wait seconds (default: 600)
+wait_for_rhdh_pod_replaced() {
+  local release_name=$1
+  local namespace=$2
+  local previous_uid=$3
+  local max_wait=${4:-600}
+  local waited=0
+  local deploy
+  deploy=$(rhdh_deployment_name "${release_name}")
+
+  if [[ -z "${previous_uid}" ]]; then
+    log::warn "No previous RHDH pod UID provided; skipping pod-replacement wait"
+    return 0
+  fi
+
+  log::info "Waiting for previous RHDH pod (uid=${previous_uid}) to terminate and a new Ready pod for ${deploy}"
+
+  while [[ $waited -lt $max_wait ]]; do
+    local uid_still_present="false"
+    local uids
+    uids=$(oc get pods -n "${namespace}" -o jsonpath='{range .items[*]}{.metadata.uid}{"\n"}{end}' 2> /dev/null || true)
+    if grep -qx "${previous_uid}" <<< "${uids}"; then
+      uid_still_present="true"
+    fi
+
+    local new_name="" new_uid="" phase="" ready=""
+    while IFS=$'\t' read -r new_name new_uid phase ready; do
+      if [[ -n "${new_name}" && "${new_name}" == "${deploy}"* ]]; then
+        break
+      fi
+      new_name=""
+      new_uid=""
+    done < <(oc get pods -n "${namespace}" \
+      -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.uid}{"\t"}{.status.phase}{"\t"}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' 2> /dev/null || true)
+
+    if [[ "${uid_still_present}" != "true" && -n "${new_uid}" && "${new_uid}" != "${previous_uid}" && "${phase}" == "Running" && "${ready}" == "True" ]]; then
+      log::info "New RHDH pod is Ready: name=${new_name} uid=${new_uid} (previous uid=${previous_uid} terminated)"
+      echo "${new_uid}"
+      return 0
+    fi
+
+    if ((waited % 30 == 0)); then
+      log::info "Still waiting for RHDH pod replacement… previous_present=${uid_still_present} new=${new_name:-none} phase=${phase:-?} ready=${ready:-?} (${waited}s/${max_wait}s)"
+      oc get pods -n "${namespace}" -o wide >&2 | grep -E "${deploy}|NAME" || true
+    fi
+
+    sleep 5
+    waited=$((waited + 5))
+  done
+
+  log::error "Timed out waiting for RHDH pod replacement (previous uid=${previous_uid})"
+  oc get pods -n "${namespace}" -o wide >&2 || true
+  return 1
+}
+
+# Ensure RHDH has rolled onto a new pod after a DB upgrade hop.
+# Restarts the deployment when the old UID is still current, then waits for replacement.
+# Args:
+#   $1 - release name
+#   $2 - namespace
+#   $3 - previous pod UID
+#   $4 - optional max wait seconds (default: 600)
+ensure_rhdh_pod_replaced() {
+  local release_name=$1
+  local namespace=$2
+  local previous_uid=$3
+  local max_wait=${4:-600}
+  local deploy
+  local current_uid
+  deploy=$(rhdh_deployment_name "${release_name}")
+
+  if [[ -z "${previous_uid}" ]]; then
+    return 0
+  fi
+
+  current_uid=$(get_rhdh_pod_uid "${release_name}" "${namespace}")
+  if [[ "${current_uid}" == "${previous_uid}" ]]; then
+    log::info "RHDH pod uid unchanged after DB upgrade (${current_uid}); restarting ${deploy} so verification hits a new pod"
+    oc rollout restart "deployment/${deploy}" -n "${namespace}" || true
+  else
+    log::info "RHDH pod uid already changed (${previous_uid} → ${current_uid:-none}); waiting for previous pod to terminate"
+  fi
+
+  wait_for_rhdh_pod_replaced "${release_name}" "${namespace}" "${previous_uid}" "${max_wait}" > /dev/null
+}
+
+# Resolve a cluster-local runtime for the data-proof static server.
+# Prints: image<TAB>command<TAB>scriptPath
+_pg_upgrade_data_proof_runtime() {
+  local python_tag nodejs_tag
+
+  python_tag=$(oc get imagestream python -n openshift -o jsonpath='{.spec.tags[*].name}' 2> /dev/null \
+    | tr ' ' '\n' | grep -E '^[0-9]+\.[0-9]+-ubi[0-9]+$' | sort -V | tail -1 || true)
+  if [[ -n "${python_tag}" ]]; then
+    printf '%s\t%s\t%s\n' \
+      "image-registry.openshift-image-registry.svc:5000/openshift/python:${python_tag}" \
+      "python3" \
+      "/www/serve.py"
+    return 0
+  fi
+
+  nodejs_tag=$(oc get imagestream nodejs -n openshift -o jsonpath='{.spec.tags[*].name}' 2> /dev/null \
+    | tr ' ' '\n' | grep -E '^[0-9]+-ubi[0-9]+$' | sort -t'-' -k1 -n | tail -1 || true)
+  nodejs_tag="${nodejs_tag:-18-ubi8}"
+  printf '%s\t%s\t%s\n' \
+    "image-registry.openshift-image-registry.svc:5000/openshift/nodejs:${nodejs_tag}" \
+    "node" \
+    "/www/serve.js"
+}
+
+# Deploy an in-cluster static server that hosts the persistence-proof catalog entity.
+# Args:
+#   $1 - namespace
+deploy_pg_upgrade_data_proof_fixture() {
+  local namespace=$1
+  local manifest="${DIR}/resources/pg-upgrade/data-proof-server.yaml"
+  local runtime image cmd script
+  local rendered="/tmp/pg-upgrade-data-proof-server.yaml"
+
+  if [[ ! -f "${manifest}" ]]; then
+    log::error "Missing data-proof fixture manifest: ${manifest}"
+    return 1
+  fi
+
+  runtime=$(_pg_upgrade_data_proof_runtime)
+  image=$(cut -f1 <<< "${runtime}")
+  cmd=$(cut -f2 <<< "${runtime}")
+  script=$(cut -f3 <<< "${runtime}")
+
+  log::info "Deploying pg-upgrade data-proof fixture in ${namespace} using ${image} (${cmd} ${script})"
+
+  sed \
+    -e "s|PG_UPGRADE_DATA_PROOF_IMAGE|${image}|g" \
+    -e "s|PG_UPGRADE_DATA_PROOF_COMMAND|${cmd}|g" \
+    -e "s|PG_UPGRADE_DATA_PROOF_SCRIPT|${script}|g" \
+    "${manifest}" > "${rendered}"
+
+  oc apply -n "${namespace}" -f "${rendered}"
+  if ! oc rollout status "deployment/pg-upgrade-data-proof" -n "${namespace}" --timeout=3m; then
+    log::error "pg-upgrade data-proof fixture failed to become Ready"
+    oc get pods,svc -n "${namespace}" -l app=pg-upgrade-data-proof -o wide >&2 || true
+    oc describe deployment/pg-upgrade-data-proof -n "${namespace}" >&2 || true
+    oc logs -n "${namespace}" deploy/pg-upgrade-data-proof --tail=50 >&2 || true
+    return 1
+  fi
+}
+
+# Resolve a publicly fetchable catalog-info URL for the persistence-proof entity.
+# Prefer an explicit override, else GitHub blob URL at the PR/head SHA (Backstage
+# blocks most in-cluster http://*.svc targets via URL reader / SSRF guards).
+pg_upgrade_data_proof_target_url() {
+  if [[ -n "${PG_UPGRADE_PROOF_URL:-}" ]]; then
+    echo "${PG_UPGRADE_PROOF_URL}"
+    return 0
+  fi
+
+  local sha="${PULL_PULL_SHA:-${PULL_BASE_SHA:-}}"
+  if [[ -z "${sha}" ]]; then
+    sha=$(git -C "${DIR}/../.." rev-parse HEAD 2> /dev/null || true)
+  fi
+  local repo="${PG_UPGRADE_PROOF_REPO:-zdrapela/rhdh}"
+  if [[ -z "${sha}" ]]; then
+    log::error "Cannot resolve pg-upgrade proof catalog URL (set PG_UPGRADE_PROOF_URL or PULL_PULL_SHA)"
+    return 1
+  fi
+  echo "https://github.com/${repo}/blob/${sha}/.ci/pipelines/resources/pg-upgrade/catalog-info.yaml"
+}
+
+# Register the persistence-proof Component via the Catalog Locations API and wait until it is queryable.
+# Args:
+#   $1 - Backstage base URL
+#   $2 - namespace (unused; kept for call-site compatibility)
+seed_pg_upgrade_data_proof() {
+  local base_url=$1
+  local _namespace=$2
+  local target
+  local entity_url="${base_url}/api/catalog/entities/by-name/component/default/pg-upgrade-data-proof"
+  local locations_url="${base_url}/api/catalog/locations"
+  local max_wait=${3:-180}
+  local waited=0
+  local status body
+
+  if ! target=$(pg_upgrade_data_proof_target_url); then
+    return 1
+  fi
+
+  log::info "Seeding pg-upgrade data proof location: ${target}"
+
+  status=$(curl --insecure -s -o /tmp/pg-upgrade-seed-location.json -w "%{http_code}" \
+    -X POST "${locations_url}" \
+    -H "Content-Type: application/json" \
+    -d "{\"type\":\"url\",\"target\":\"${target}\"}" || echo "000")
+
+  if [[ "${status}" != "201" && "${status}" != "409" ]]; then
+    log::error "Failed to register data-proof location (HTTP ${status})"
+    cat /tmp/pg-upgrade-seed-location.json >&2 || true
+    return 1
+  fi
+  log::info "Location register response HTTP ${status}"
+
+  log::info "Waiting for entity pg-upgrade-data-proof to appear in catalog"
+  while [[ $waited -lt $max_wait ]]; do
+    status=$(curl --insecure -s -o /tmp/pg-upgrade-seed-entity.json -w "%{http_code}" "${entity_url}" || echo "000")
+    if [[ "${status}" == "200" ]]; then
+      body=$(cat /tmp/pg-upgrade-seed-entity.json)
+      if grep -q "RHIDP-14594" <<< "${body}" && grep -q "pg-upgrade-data-proof" <<< "${body}"; then
+        log::info "Seeded catalog entity is present (RHIDP-14594 persistence proof)"
+        return 0
+      fi
+    fi
+    if ((waited % 30 == 0)); then
+      log::info "Still waiting for seeded entity… HTTP ${status} (${waited}s/${max_wait}s)"
+    fi
+    sleep 5
+    waited=$((waited + 5))
+  done
+
+  log::error "Timed out waiting for seeded entity pg-upgrade-data-proof"
+  cat /tmp/pg-upgrade-seed-entity.json >&2 || true
+  curl --insecure -s "${locations_url}" >&2 || true
+  return 1
+}
+
+# Log API evidence that the persistence-proof entity still exists (post-upgrade).
+# Args:
+#   $1 - Backstage base URL
+assert_pg_upgrade_data_proof_api() {
+  local base_url=$1
+  local entity_url="${base_url}/api/catalog/entities/by-name/component/default/pg-upgrade-data-proof"
+  local status body
+
+  status=$(curl --insecure -s -o /tmp/pg-upgrade-verify-entity.json -w "%{http_code}" "${entity_url}" || echo "000")
+  body=$(cat /tmp/pg-upgrade-verify-entity.json 2> /dev/null || true)
+  if [[ "${status}" != "200" ]] || ! grep -q "RHIDP-14594" <<< "${body}"; then
+    log::error "Persistence proof entity missing or incomplete via API (HTTP ${status})"
+    echo "${body}" >&2
+    return 1
+  fi
+  log::info "API persistence proof OK: component/default/pg-upgrade-data-proof still present after upgrade"
+}
diff --git a/.ci/pipelines/resources/pg-upgrade/catalog-info.yaml b/.ci/pipelines/resources/pg-upgrade/catalog-info.yaml
new file mode 100644
index 0000000000..b3883270c0
--- /dev/null
+++ b/.ci/pipelines/resources/pg-upgrade/catalog-info.yaml
@@ -0,0 +1,13 @@
+apiVersion: backstage.io/v1alpha1
+kind: Component
+metadata:
+  name: pg-upgrade-data-proof
+  title: PG Upgrade Data Proof
+  description: RHIDP-14594 persistence proof seeded before PostgreSQL major upgrade
+  tags:
+    - pg-upgrade-proof
+    - rhidp-14594
+spec:
+  type: service
+  lifecycle: experimental
+  owner: guests
diff --git a/.ci/pipelines/resources/pg-upgrade/data-proof-server.yaml b/.ci/pipelines/resources/pg-upgrade/data-proof-server.yaml
new file mode 100644
index 0000000000..3ecccee642
--- /dev/null
+++ b/.ci/pipelines/resources/pg-upgrade/data-proof-server.yaml
@@ -0,0 +1,104 @@
+# Serves catalog-info.yaml in-cluster so Backstage can register a location
+# without depending on GitHub raw URLs from fork PRs.
+# Image/command placeholders are substituted by deploy_pg_upgrade_data_proof_fixture().
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: pg-upgrade-data-proof
+  labels:
+    app: pg-upgrade-data-proof
+data:
+  catalog-info.yaml: |
+    apiVersion: backstage.io/v1alpha1
+    kind: Component
+    metadata:
+      name: pg-upgrade-data-proof
+      title: PG Upgrade Data Proof
+      description: RHIDP-14594 persistence proof seeded before PostgreSQL major upgrade
+      tags:
+        - pg-upgrade-proof
+        - rhidp-14594
+    spec:
+      type: service
+      lifecycle: experimental
+      owner: guests
+  serve.py: |
+    #!/usr/bin/env python3
+    from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
+    import os
+    os.chdir("/www")
+    ThreadingHTTPServer(("0.0.0.0", 8080), SimpleHTTPRequestHandler).serve_forever()
+  serve.js: |
+    const http = require("http");
+    const fs = require("fs");
+    const path = require("path");
+    http.createServer((req, res) => {
+      const file = path.join("/www", req.url === "/" ? "catalog-info.yaml" : req.url);
+      fs.readFile(file, (err, data) => {
+        if (err) { res.writeHead(404); res.end(String(err)); return; }
+        res.writeHead(200, { "Content-Type": "text/yaml" });
+        res.end(data);
+      });
+    }).listen(8080, "0.0.0.0");
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: pg-upgrade-data-proof
+  labels:
+    app: pg-upgrade-data-proof
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: pg-upgrade-data-proof
+  template:
+    metadata:
+      labels:
+        app: pg-upgrade-data-proof
+    spec:
+      containers:
+        - name: http
+          image: PG_UPGRADE_DATA_PROOF_IMAGE
+          imagePullPolicy: IfNotPresent
+          command:
+            - PG_UPGRADE_DATA_PROOF_COMMAND
+          args:
+            - PG_UPGRADE_DATA_PROOF_SCRIPT
+          ports:
+            - containerPort: 8080
+              name: http
+          volumeMounts:
+            - name: www
+              mountPath: /www
+              readOnly: true
+          readinessProbe:
+            tcpSocket:
+              port: 8080
+            initialDelaySeconds: 2
+            periodSeconds: 5
+          resources:
+            requests:
+              cpu: 10m
+              memory: 32Mi
+            limits:
+              memory: 128Mi
+      volumes:
+        - name: www
+          configMap:
+            name: pg-upgrade-data-proof
+            defaultMode: 0555
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: pg-upgrade-data-proof
+  labels:
+    app: pg-upgrade-data-proof
+spec:
+  selector:
+    app: pg-upgrade-data-proof
+  ports:
+    - name: http
+      port: 8080
+      targetPort: 8080
diff --git a/e2e-tests/playwright/e2e/pg-upgrade-data-proof.spec.ts b/e2e-tests/playwright/e2e/pg-upgrade-data-proof.spec.ts
new file mode 100644
index 0000000000..879d236c9f
--- /dev/null
+++ b/e2e-tests/playwright/e2e/pg-upgrade-data-proof.spec.ts
@@ -0,0 +1,47 @@
+import { expect } from "@playwright/test";
+import { test } from "@support/coverage/test";
+
+import { getTranslations, getCurrentLanguage } from "../e2e/localization/locale";
+import { CatalogBrowsePage } from "../support/pages/catalog-browse-page";
+
+const t = getTranslations();
+const lang = getCurrentLanguage();
+
+/**
+ * RHIDP-14594: prove catalog data seeded on PG15 survives major upgrades.
+ * Enabled only when CI sets PG_UPGRADE_DATA_PROOF=1 (see ocp-pull.sh / ocp-operator.sh).
+ */
+test.describe("PostgreSQL upgrade data persistence proof", () => {
+  test.skip(
+    () => process.env.PG_UPGRADE_DATA_PROOF !== "1",
+    "Runs only during chart-managed or operator-local PostgreSQL upgrade CI (PG_UPGRADE_DATA_PROOF=1)",
+  );
+
+  let catalogBrowsePage: CatalogBrowsePage;
+
+  test.beforeAll(({ rhdhGuestPage }) => {
+    test.info().annotations.push({
+      type: "component",
+      description: "core",
+    });
+    catalogBrowsePage = new CatalogBrowsePage(rhdhGuestPage);
+  });
+
+  test("Seeded pg-upgrade-data-proof component is visible in Catalog UI", async ({
+    rhdhGuestPage,
+  }) => {
+    // Catalog table shows metadata.title when set ("PG Upgrade Data Proof"),
+    // not metadata.name — match the same navigation pattern as catalog-timestamp.
+    await catalogBrowsePage.openSidebar(t["rhdh"][lang]["menuItem.catalog"]);
+    await catalogBrowsePage.verifyHeading(
+      t["catalog"][lang]["indexPage.title"].replace("{{orgName}}", "My Org"),
+    );
+    await catalogBrowsePage.openCatalogSidebar("Component");
+    await catalogBrowsePage.searchCatalog("PG Upgrade Data Proof");
+    await catalogBrowsePage.verifyText("PG Upgrade Data Proof");
+
+    await catalogBrowsePage.openEntityLink("PG Upgrade Data Proof");
+    await catalogBrowsePage.verifyHeading("PG Upgrade Data Proof");
+    await expect(rhdhGuestPage.getByText(/RHIDP-14594 persistence proof/i)).toBeVisible();
+  });
+});

From 95e03eb51331acd7848592a63a3f4bbcf200609d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= <zbynek.drapela@gmail.com>
Date: Fri, 24 Jul 2026 11:12:27 +0200
Subject: [PATCH 2/6] fix(ci): retry operator prepare when Backstage CRD never
 appears

Full install+CRD cycles with OLM diagnostics, always use 3 attempts on
OCP operator nightly, and satisfy prefer-unicode-regexp in the proof spec.
---
 .ci/pipelines/install-methods/operator.sh     | 47 +++++++++++++++++--
 .ci/pipelines/jobs/ocp-operator.sh            |  7 +--
 .../e2e/pg-upgrade-data-proof.spec.ts         |  2 +-
 3 files changed, 47 insertions(+), 9 deletions(-)

diff --git a/.ci/pipelines/install-methods/operator.sh b/.ci/pipelines/install-methods/operator.sh
index 864c981217..16d037d100 100755
--- a/.ci/pipelines/install-methods/operator.sh
+++ b/.ci/pipelines/install-methods/operator.sh
@@ -43,13 +43,50 @@ install_rhdh_operator() {
   fi
 }
 
+# Dump OLM / operator-manager state when Backstage CRD registration fails.
+_operator_olm_debug_info() {
+  local ns="${OPERATOR_MANAGER:-rhdh-operator}"
+  log::info "OLM/operator diagnostics in namespace: ${ns}"
+  oc get csv,sub,ip,catalogsource,deploy,pods -n "${ns}" -o wide 2>&1 || true
+  oc get crd backstages.rhdh.redhat.com -o yaml 2>&1 | head -40 || true
+  oc logs -n "${ns}" -l control-plane=controller-manager --tail=100 2>&1 || true
+  oc get events -n "${ns}" --sort-by='.lastTimestamp' 2>&1 | tail -30 || true
+}
+
+# Install the RHDH operator and wait for the Backstage CRD.
+# Retries the *entire* cycle (namespace recreate → install → CRD wait) up to N times.
+# Args:
+#   $1 - max full-cycle attempts (default: 1)
 prepare_operator() {
-  local retry_operator_installation="${1:-1}"
-  namespace::configure "${OPERATOR_MANAGER}"
-  install_rhdh_operator "${OPERATOR_MANAGER}" "$retry_operator_installation"
+  local max_cycles=${1:-1}
+  local attempt=1
+
+  while ((attempt <= max_cycles)); do
+    log::info "Preparing RHDH operator (attempt ${attempt}/${max_cycles})"
+    namespace::configure "${OPERATOR_MANAGER}"
+    # One install-script attempt per cycle; full-cycle retries cover "script OK but CRD missing".
+    if ! install_rhdh_operator "${OPERATOR_MANAGER}" 1; then
+      log::error "Operator install script failed on attempt ${attempt}/${max_cycles}"
+      _operator_olm_debug_info
+      if ((attempt < max_cycles)); then
+        attempt=$((attempt + 1))
+        continue
+      fi
+      return 1
+    fi
+
+    if k8s_wait::crd "backstages.rhdh.redhat.com" 300 10; then
+      log::info "Backstage CRD available after operator prepare attempt ${attempt}"
+      return 0
+    fi
+
+    log::error "Backstage CRD not available after install attempt ${attempt}/${max_cycles}"
+    _operator_olm_debug_info
+    attempt=$((attempt + 1))
+  done
 
-  # Wait for Backstage CRD to be available after operator installation
-  k8s_wait::crd "backstages.rhdh.redhat.com" 300 10 || return 1
+  log::error "Failed to prepare RHDH operator after ${max_cycles} full cycle(s)"
+  return 1
 }
 
 # Waits for the Crunchy Data PostgreSQL Operator's PostgresCluster CRD to become available.
diff --git a/.ci/pipelines/jobs/ocp-operator.sh b/.ci/pipelines/jobs/ocp-operator.sh
index b9d908c690..230eb49cc5 100644
--- a/.ci/pipelines/jobs/ocp-operator.sh
+++ b/.ci/pipelines/jobs/ocp-operator.sh
@@ -147,11 +147,12 @@ handle_ocp_operator() {
   cluster_setup_ocp_operator
 
   if [[ "${JOB_NAME}" =~ osd-gcp ]]; then
+    # OSD-GCP internal registry is unreliable under parallel load; reduce
+    # concurrent skopeo pushes and allow retries (RHDHBUGS-1136).
     export MAX_PARALLEL=3
-    prepare_operator 3
-  else
-    prepare_operator
   fi
+  # Full install+CRD cycles (covers "install script OK but CRD never appears").
+  prepare_operator 3
 
   # Ensure baseline image is Fedora PG15 (operator CSV default; set explicitly for evidence).
   if ! set_operator_postgresql_related_image "${OPERATOR_PG15_IMAGE}"; then
diff --git a/e2e-tests/playwright/e2e/pg-upgrade-data-proof.spec.ts b/e2e-tests/playwright/e2e/pg-upgrade-data-proof.spec.ts
index 879d236c9f..b0b7ce7320 100644
--- a/e2e-tests/playwright/e2e/pg-upgrade-data-proof.spec.ts
+++ b/e2e-tests/playwright/e2e/pg-upgrade-data-proof.spec.ts
@@ -42,6 +42,6 @@ test.describe("PostgreSQL upgrade data persistence proof", () => {
 
     await catalogBrowsePage.openEntityLink("PG Upgrade Data Proof");
     await catalogBrowsePage.verifyHeading("PG Upgrade Data Proof");
-    await expect(rhdhGuestPage.getByText(/RHIDP-14594 persistence proof/i)).toBeVisible();
+    await expect(rhdhGuestPage.getByText(/RHIDP-14594 persistence proof/iu)).toBeVisible();
   });
 });

From 936333fd96f34319e4ee38db283eeb47fec5632c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= <zbynek.drapela@gmail.com>
Date: Fri, 24 Jul 2026 11:54:46 +0200
Subject: [PATCH 3/6] fix(ci): force OLM v0 and surface operator install logs

Auto-detected OLM v1 can exit successfully without registering the
Backstage CRD. Use classic Subscription install, stream script output,
and fail fast if Subscription/rhdh is missing after install.
---
 .ci/pipelines/install-methods/operator.sh | 68 ++++++++++++++++++-----
 1 file changed, 53 insertions(+), 15 deletions(-)

diff --git a/.ci/pipelines/install-methods/operator.sh b/.ci/pipelines/install-methods/operator.sh
index 16d037d100..2462077dab 100755
--- a/.ci/pipelines/install-methods/operator.sh
+++ b/.ci/pipelines/install-methods/operator.sh
@@ -5,9 +5,28 @@ source "$DIR"/lib/log.sh
 # shellcheck source=.ci/pipelines/utils.sh
 source "$DIR"/utils.sh
 
+# Run install-rhdh-catalog-source.sh with visible logs (common::retry swallows stdout/stderr).
+# Forces OLM v0: auto-detect can pick OLM v1 (ClusterExtension) which returns success without
+# waiting for the operator CSV/CRDs, leaving Backstage CRD missing.
+_install_rhdh_operator_once() {
+  local -a install_args=()
+
+  if [[ "${RELEASE_VERSION}" == "next" ]]; then
+    log::info "Installing RHDH operator with '--next --olm-version v0'"
+    install_args=(--next --olm-version v0 --install-operator rhdh)
+  else
+    log::info "Installing RHDH operator with '-v ${RELEASE_VERSION} --olm-version v0'"
+    install_args=(-v "${RELEASE_VERSION}" --olm-version v0 --install-operator rhdh)
+  fi
+
+  # Stream script output to the CI log; do not capture (common::retry hides failures).
+  bash -x /tmp/install-rhdh-catalog-source.sh "${install_args[@]}"
+}
+
 install_rhdh_operator() {
   local namespace=$1
   local max_attempts=$2
+  local attempt=1
 
   namespace::configure "$namespace"
 
@@ -28,29 +47,37 @@ install_rhdh_operator() {
   fi
   chmod +x /tmp/install-rhdh-catalog-source.sh
 
-  if [[ "$RELEASE_VERSION" == "next" ]]; then
-    log::info "Installing RHDH operator with '--next' flag"
-    if ! common::retry "$max_attempts" 10 bash -x /tmp/install-rhdh-catalog-source.sh --next --install-operator rhdh; then
-      log::error "Failed install RHDH Operator after ${max_attempts} attempts."
-      return 1
+  while ((attempt <= max_attempts)); do
+    log::info "Operator install script attempt ${attempt}/${max_attempts}"
+    if _install_rhdh_operator_once; then
+      log::success "Operator install script succeeded on attempt ${attempt}"
+      return 0
     fi
-  else
-    log::info "Installing RHDH operator with '-v $RELEASE_VERSION' flag"
-    if ! common::retry "$max_attempts" 10 bash -x /tmp/install-rhdh-catalog-source.sh -v "$RELEASE_VERSION" --install-operator rhdh; then
-      log::error "Failed install RHDH Operator after ${max_attempts} attempts."
-      return 1
+    log::warn "Operator install script failed on attempt ${attempt}/${max_attempts}"
+    if ((attempt < max_attempts)); then
+      sleep 10
     fi
-  fi
+    attempt=$((attempt + 1))
+  done
+
+  log::error "Failed install RHDH Operator after ${max_attempts} attempts."
+  return 1
 }
 
 # Dump OLM / operator-manager state when Backstage CRD registration fails.
 _operator_olm_debug_info() {
   local ns="${OPERATOR_MANAGER:-rhdh-operator}"
   log::info "OLM/operator diagnostics in namespace: ${ns}"
-  oc get csv,sub,ip,catalogsource,deploy,pods -n "${ns}" -o wide 2>&1 || true
+  oc get operatorgroup,csv,sub,ip,deploy,pods -n "${ns}" -o wide 2>&1 || true
+  log::info "CatalogSource in openshift-marketplace / olm:"
+  oc get catalogsource -n openshift-marketplace -o wide 2>&1 || true
+  oc get catalogsource -n olm -o wide 2>&1 || true
+  log::info "ClusterExtension / ClusterCatalog (OLM v1), if present:"
+  oc get clusterextension,clustercatalog 2>&1 || true
+  log::info "Backstage CRD:"
   oc get crd backstages.rhdh.redhat.com -o yaml 2>&1 | head -40 || true
   oc logs -n "${ns}" -l control-plane=controller-manager --tail=100 2>&1 || true
-  oc get events -n "${ns}" --sort-by='.lastTimestamp' 2>&1 | tail -30 || true
+  oc get events -n "${ns}" --sort-by='.lastTimestamp' 2>&1 | tail -40 || true
 }
 
 # Install the RHDH operator and wait for the Backstage CRD.
@@ -63,8 +90,7 @@ prepare_operator() {
 
   while ((attempt <= max_cycles)); do
     log::info "Preparing RHDH operator (attempt ${attempt}/${max_cycles})"
-    namespace::configure "${OPERATOR_MANAGER}"
-    # One install-script attempt per cycle; full-cycle retries cover "script OK but CRD missing".
+    # install_rhdh_operator calls namespace::configure — avoid a second wipe here.
     if ! install_rhdh_operator "${OPERATOR_MANAGER}" 1; then
       log::error "Operator install script failed on attempt ${attempt}/${max_cycles}"
       _operator_olm_debug_info
@@ -75,6 +101,18 @@ prepare_operator() {
       return 1
     fi
 
+    # Subscription should exist quickly; fail fast with diagnostics if the script
+    # returned 0 without creating OLM v0 resources.
+    if ! oc get subscription rhdh -n "${OPERATOR_MANAGER}" &> /dev/null; then
+      log::error "Subscription/rhdh missing in ${OPERATOR_MANAGER} after install script success"
+      _operator_olm_debug_info
+      if ((attempt < max_cycles)); then
+        attempt=$((attempt + 1))
+        continue
+      fi
+      return 1
+    fi
+
     if k8s_wait::crd "backstages.rhdh.redhat.com" 300 10; then
       log::info "Backstage CRD available after operator prepare attempt ${attempt}"
       return 0

From dedd613f4b02b144a6149350bf52fe1b6ebad8cb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= <zbynek.drapela@gmail.com>
Date: Fri, 24 Jul 2026 12:40:22 +0200
Subject: [PATCH 4/6] fix(ci): disable default flavours on operator PG upgrade
 showcase CR

Lightspeed default flavour pulls a missing OCI digest and CrashLoops
install-dynamic-plugins before PG upgrade verification can run.
---
 .ci/pipelines/resources/rhdh-operator/rhdh-start.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.ci/pipelines/resources/rhdh-operator/rhdh-start.yaml b/.ci/pipelines/resources/rhdh-operator/rhdh-start.yaml
index f97aa1afb7..3bafb6ace8 100644
--- a/.ci/pipelines/resources/rhdh-operator/rhdh-start.yaml
+++ b/.ci/pipelines/resources/rhdh-operator/rhdh-start.yaml
@@ -3,6 +3,9 @@ apiVersion: rhdh.redhat.com/v1alpha5
 metadata:
   name: rhdh
 spec:
+  # Disable default flavours (e.g. lightspeed) — this evidence job only needs
+  # local Postgres + showcase plugins; lightspeed OCI digests have been flaky in CI.
+  flavours: []
   deployment:
     patch:
       spec:

From 97b59b71a7b685ddf09895da168a3c7477d18b97 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= <zbynek.drapela@gmail.com>
Date: Fri, 24 Jul 2026 13:52:03 +0200
Subject: [PATCH 5/6] fix(ci): pin operator PG RELATED_IMAGE via Subscription
 config

OLM was reverting oc set env on the operator Deployment back to the CSV
RHEL postgresql-15 image, so the post-wipe STS never came up on Fedora 18.
---
 .ci/pipelines/lib/postgres.sh | 70 +++++++++++++++++++++++++++++++++--
 1 file changed, 67 insertions(+), 3 deletions(-)

diff --git a/.ci/pipelines/lib/postgres.sh b/.ci/pipelines/lib/postgres.sh
index 309e646d03..ca645424bc 100644
--- a/.ci/pipelines/lib/postgres.sh
+++ b/.ci/pipelines/lib/postgres.sh
@@ -80,20 +80,37 @@ _postgres_sts_ready_replicas() {
   echo "${sts_ready:-0}"
 }
 
+# Current RELATED_IMAGE_postgresql on the operator Deployment (or empty).
+_operator_related_postgresql_image() {
+  local ns="${OPERATOR_MANAGER:-rhdh-operator}"
+  local deploy=$1
+  oc get "deploy/${deploy}" -n "${ns}" \
+    -o jsonpath='{range .spec.template.spec.containers[0].env[?(@.name=="RELATED_IMAGE_postgresql")]}{.value}{end}' \
+    2> /dev/null || true
+}
+
 # Point the RHDH operator at a different Fedora/RHEL PostgreSQL image via RELATED_IMAGE_postgresql.
+#
+# OLM owns the operator Deployment: `oc set env` is raced/reverted to the CSV
+# default (often registry.redhat.io/rhel9/postgresql-15@sha256:…). Pin the value
+# on Subscription.spec.config.env so OLM keeps applying our override.
+#
 # Args:
 #   $1 - full image ref (e.g. quay.io/fedora/postgresql-18:latest)
 set_operator_postgresql_related_image() {
   local image=$1
   local ns="${OPERATOR_MANAGER:-rhdh-operator}"
+  local sub="${OPERATOR_SUBSCRIPTION_NAME:-rhdh}"
   local deploy
+  local current=""
+  local waited=0
+  local max_wait=180
 
   if [[ -z "${image}" ]]; then
     log::error "set_operator_postgresql_related_image: image required"
     return 1
   fi
 
-  log::info "Setting RELATED_IMAGE_postgresql=${image} on operator controller in ${ns}"
   deploy=$(oc get deploy -n "${ns}" -l control-plane=controller-manager \
     -o jsonpath='{.items[0].metadata.name}' 2> /dev/null || true)
   if [[ -z "${deploy}" ]]; then
@@ -104,14 +121,58 @@ set_operator_postgresql_related_image() {
     return 1
   fi
 
-  oc set env "deployment/${deploy}" -n "${ns}" "RELATED_IMAGE_postgresql=${image}"
+  if ! oc get subscription "${sub}" -n "${ns}" &> /dev/null; then
+    log::error "Subscription/${sub} not found in ${ns}; cannot pin RELATED_IMAGE_postgresql via OLM"
+    oc get subscription -n "${ns}" -o wide >&2 || true
+    return 1
+  fi
+
+  log::info "Pinning RELATED_IMAGE_postgresql=${image} via Subscription/${sub} spec.config.env (OLM-safe)"
+  # Merge-patch replaces config.env; this job only needs the postgresql related image.
+  if ! oc patch subscription "${sub}" -n "${ns}" --type=merge \
+    -p "{\"spec\":{\"config\":{\"env\":[{\"name\":\"RELATED_IMAGE_postgresql\",\"value\":\"${image}\"}]}}}"; then
+    log::error "Failed to patch Subscription/${sub} config.env"
+    return 1
+  fi
+
+  # Also set on the Deployment for a faster first apply; OLM will re-apply from Subscription.
+  oc set env "deployment/${deploy}" -n "${ns}" "RELATED_IMAGE_postgresql=${image}" 2>&1 || true
+
+  log::info "Waiting up to ${max_wait}s for Deployment/${deploy} RELATED_IMAGE_postgresql to equal ${image}"
+  while ((waited < max_wait)); do
+    current=$(_operator_related_postgresql_image "${deploy}")
+    if [[ "${current}" == "${image}" ]]; then
+      break
+    fi
+    log::info "Still waiting… RELATED_IMAGE_postgresql=${current:-<unset>} (${waited}s/${max_wait}s)"
+    sleep 5
+    waited=$((waited + 5))
+  done
+
+  if [[ "${current}" != "${image}" ]]; then
+    log::error "RELATED_IMAGE_postgresql did not stick (got '${current:-<unset>}', want '${image}')"
+    oc get subscription "${sub}" -n "${ns}" -o yaml >&2 || true
+    oc get "deploy/${deploy}" -n "${ns}" -o yaml >&2 || true
+    return 1
+  fi
+
   if ! oc rollout status "deployment/${deploy}" -n "${ns}" --timeout=5m; then
     log::error "Operator rollout failed after RELATED_IMAGE_postgresql update"
     oc get pods -n "${ns}" -o wide >&2 || true
     oc logs -n "${ns}" "deploy/${deploy}" --tail=80 >&2 || true
     return 1
   fi
-  log::info "Operator controller ready with RELATED_IMAGE_postgresql=${image}"
+
+  # Settle check: OLM must not have reverted after rollout status returned.
+  sleep 5
+  current=$(_operator_related_postgresql_image "${deploy}")
+  if [[ "${current}" != "${image}" ]]; then
+    log::error "OLM reverted RELATED_IMAGE_postgresql after rollout (got '${current:-<unset>}', want '${image}')"
+    oc get subscription "${sub}" -n "${ns}" -o yaml >&2 || true
+    return 1
+  fi
+
+  log::info "Operator controller ready with RELATED_IMAGE_postgresql=${image} (Subscription-pinned)"
 }
 
 # Dump postgres pod diagnostics to stderr (safe under $(...) capture).
@@ -141,6 +202,9 @@ dump_postgres_diagnostics() {
     if [[ "${INSTALL_METHOD:-helm}" == "operator" ]]; then
       log::info "Operator RELATED_IMAGE_postgresql:"
       oc get deploy -n "${OPERATOR_MANAGER:-rhdh-operator}" -o jsonpath='{range .items[*]}{.metadata.name}{" RELATED_IMAGE_postgresql="}{range .spec.template.spec.containers[0].env[?(@.name=="RELATED_IMAGE_postgresql")]}{.value}{end}{"\n"}{end}' 2>&1 || true
+      log::info "Subscription config.env (RELATED_IMAGE_postgresql pin):"
+      oc get subscription "${OPERATOR_SUBSCRIPTION_NAME:-rhdh}" -n "${OPERATOR_MANAGER:-rhdh-operator}" \
+        -o jsonpath='{range .spec.config.env[*]}{.name}{"="}{.value}{"\n"}{end}' 2>&1 || true
       log::info "Backstage CR (database):"
       oc get backstage -n "${namespace}" -o yaml 2>&1 | grep -A20 -E 'database:|enableLocalDb' || true
     else

From 19178c4716e0579f8d776587b78f8ea2f8e599a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= <zbynek.drapela@gmail.com>
Date: Fri, 24 Jul 2026 15:07:42 +0200
Subject: [PATCH 6/6] fix(ci): wait for Subscription/Deployment before pinning
 PG image

Backstage CRD can appear before the operator Deployment exists; pin via
Subscription first, then wait for the controller env to match.
---
 .ci/pipelines/lib/postgres.sh | 71 +++++++++++++++++++++++------------
 1 file changed, 47 insertions(+), 24 deletions(-)

diff --git a/.ci/pipelines/lib/postgres.sh b/.ci/pipelines/lib/postgres.sh
index ca645424bc..59cef745a7 100644
--- a/.ci/pipelines/lib/postgres.sh
+++ b/.ci/pipelines/lib/postgres.sh
@@ -89,41 +89,57 @@ _operator_related_postgresql_image() {
     2> /dev/null || true
 }
 
+# Resolve operator controller Deployment name in OPERATOR_MANAGER (or empty).
+_operator_controller_deploy_name() {
+  local ns="${OPERATOR_MANAGER:-rhdh-operator}"
+  local deploy
+  deploy=$(oc get deploy -n "${ns}" -l control-plane=controller-manager \
+    -o jsonpath='{.items[0].metadata.name}' 2> /dev/null || true)
+  if [[ -z "${deploy}" ]]; then
+    deploy=$(oc get deploy -n "${ns}" -o jsonpath='{.items[0].metadata.name}' 2> /dev/null || true)
+  fi
+  echo "${deploy}"
+}
+
 # Point the RHDH operator at a different Fedora/RHEL PostgreSQL image via RELATED_IMAGE_postgresql.
 #
 # OLM owns the operator Deployment: `oc set env` is raced/reverted to the CSV
 # default (often registry.redhat.io/rhel9/postgresql-15@sha256:…). Pin the value
 # on Subscription.spec.config.env so OLM keeps applying our override.
 #
+# The Backstage CRD can appear before the operator Deployment exists — patch the
+# Subscription first, then wait for Deployment + env.
+#
 # Args:
 #   $1 - full image ref (e.g. quay.io/fedora/postgresql-18:latest)
 set_operator_postgresql_related_image() {
   local image=$1
   local ns="${OPERATOR_MANAGER:-rhdh-operator}"
   local sub="${OPERATOR_SUBSCRIPTION_NAME:-rhdh}"
-  local deploy
+  local deploy=""
   local current=""
   local waited=0
-  local max_wait=180
+  local max_wait=300
 
   if [[ -z "${image}" ]]; then
     log::error "set_operator_postgresql_related_image: image required"
     return 1
   fi
 
-  deploy=$(oc get deploy -n "${ns}" -l control-plane=controller-manager \
-    -o jsonpath='{.items[0].metadata.name}' 2> /dev/null || true)
-  if [[ -z "${deploy}" ]]; then
-    deploy=$(oc get deploy -n "${ns}" -o jsonpath='{.items[0].metadata.name}' 2> /dev/null || true)
-  fi
-  if [[ -z "${deploy}" ]]; then
-    log::error "No operator Deployment found in ${ns}"
-    return 1
-  fi
-
+  log::info "Waiting up to ${max_wait}s for Subscription/${sub} in ${ns}"
+  waited=0
+  while ((waited < max_wait)); do
+    if oc get subscription "${sub}" -n "${ns}" &> /dev/null; then
+      break
+    fi
+    log::info "Still waiting… Subscription/${sub} not found (${waited}s/${max_wait}s)"
+    oc get subscription,csv,ip,deploy,pods -n "${ns}" -o wide >&2 || true
+    sleep 5
+    waited=$((waited + 5))
+  done
   if ! oc get subscription "${sub}" -n "${ns}" &> /dev/null; then
     log::error "Subscription/${sub} not found in ${ns}; cannot pin RELATED_IMAGE_postgresql via OLM"
-    oc get subscription -n "${ns}" -o wide >&2 || true
+    oc get subscription,csv,ip,deploy,pods -n "${ns}" -o wide >&2 || true
     return 1
   fi
 
@@ -135,24 +151,31 @@ set_operator_postgresql_related_image() {
     return 1
   fi
 
-  # Also set on the Deployment for a faster first apply; OLM will re-apply from Subscription.
-  oc set env "deployment/${deploy}" -n "${ns}" "RELATED_IMAGE_postgresql=${image}" 2>&1 || true
-
-  log::info "Waiting up to ${max_wait}s for Deployment/${deploy} RELATED_IMAGE_postgresql to equal ${image}"
+  log::info "Waiting up to ${max_wait}s for operator Deployment with RELATED_IMAGE_postgresql=${image}"
+  waited=0
   while ((waited < max_wait)); do
-    current=$(_operator_related_postgresql_image "${deploy}")
-    if [[ "${current}" == "${image}" ]]; then
-      break
+    deploy=$(_operator_controller_deploy_name)
+    if [[ -n "${deploy}" ]]; then
+      # Best-effort immediate apply; OLM re-applies from Subscription.
+      oc set env "deployment/${deploy}" -n "${ns}" "RELATED_IMAGE_postgresql=${image}" 2>&1 || true
+      current=$(_operator_related_postgresql_image "${deploy}")
+      if [[ "${current}" == "${image}" ]]; then
+        break
+      fi
+      log::info "Still waiting… deploy=${deploy} RELATED_IMAGE_postgresql=${current:-<unset>} (${waited}s/${max_wait}s)"
+    else
+      log::info "Still waiting… no operator Deployment yet (${waited}s/${max_wait}s)"
+      oc get subscription,csv,ip,deploy,pods -n "${ns}" -o wide >&2 || true
     fi
-    log::info "Still waiting… RELATED_IMAGE_postgresql=${current:-<unset>} (${waited}s/${max_wait}s)"
     sleep 5
     waited=$((waited + 5))
   done
 
-  if [[ "${current}" != "${image}" ]]; then
-    log::error "RELATED_IMAGE_postgresql did not stick (got '${current:-<unset>}', want '${image}')"
+  if [[ -z "${deploy}" || "${current}" != "${image}" ]]; then
+    log::error "RELATED_IMAGE_postgresql did not stick (deploy='${deploy:-<none>}', got '${current:-<unset>}', want '${image}')"
     oc get subscription "${sub}" -n "${ns}" -o yaml >&2 || true
-    oc get "deploy/${deploy}" -n "${ns}" -o yaml >&2 || true
+    oc get deploy,pods -n "${ns}" -o wide >&2 || true
+    [[ -n "${deploy}" ]] && oc get "deploy/${deploy}" -n "${ns}" -o yaml >&2 || true
     return 1
   fi