test(ci): operator Fedora PostgreSQL 15 to 18 dump/restore upgrade path

.ci/pipelines/install-methods/operator.sh

@@ -5,9 +5,28 @@ source "$DIR"/lib/log.sh
 # shellcheck source=.ci/pipelines/utils.sh
 source "$DIR"/utils.sh
 
+# Run install-rhdh-catalog-source.sh with visible logs (common::retry swallows stdout/stderr).
+# Forces OLM v0: auto-detect can pick OLM v1 (ClusterExtension) which returns success without
+# waiting for the operator CSV/CRDs, leaving Backstage CRD missing.
+_install_rhdh_operator_once() {
+  local -a install_args=()
+
+  if [[ "${RELEASE_VERSION}" == "next" ]]; then
+    log::info "Installing RHDH operator with '--next --olm-version v0'"
+    install_args=(--next --olm-version v0 --install-operator rhdh)
+  else
+    log::info "Installing RHDH operator with '-v ${RELEASE_VERSION} --olm-version v0'"
+    install_args=(-v "${RELEASE_VERSION}" --olm-version v0 --install-operator rhdh)
+  fi
+
+  # Stream script output to the CI log; do not capture (common::retry hides failures).
+  bash -x /tmp/install-rhdh-catalog-source.sh "${install_args[@]}"
+}
+
 install_rhdh_operator() {
   local namespace=$1
   local max_attempts=$2
+  local attempt=1
 
   namespace::configure "$namespace"
 
@@ -28,28 +47,84 @@ install_rhdh_operator() {
   fi
   chmod +x /tmp/install-rhdh-catalog-source.sh
 
-  if [[ "$RELEASE_VERSION" == "next" ]]; then
-    log::info "Installing RHDH operator with '--next' flag"
-    if ! common::retry "$max_attempts" 10 bash -x /tmp/install-rhdh-catalog-source.sh --next --install-operator rhdh; then
-      log::error "Failed install RHDH Operator after ${max_attempts} attempts."
-      return 1
+  while ((attempt <= max_attempts)); do
+    log::info "Operator install script attempt ${attempt}/${max_attempts}"
+    if _install_rhdh_operator_once; then
+      log::success "Operator install script succeeded on attempt ${attempt}"
+      return 0
     fi
-  else
-    log::info "Installing RHDH operator with '-v $RELEASE_VERSION' flag"
-    if ! common::retry "$max_attempts" 10 bash -x /tmp/install-rhdh-catalog-source.sh -v "$RELEASE_VERSION" --install-operator rhdh; then
-      log::error "Failed install RHDH Operator after ${max_attempts} attempts."
-      return 1
+    log::warn "Operator install script failed on attempt ${attempt}/${max_attempts}"
+    if ((attempt < max_attempts)); then
+      sleep 10
     fi
-  fi
+    attempt=$((attempt + 1))
+  done
+
+  log::error "Failed install RHDH Operator after ${max_attempts} attempts."
+  return 1
 }
 
+# Dump OLM / operator-manager state when Backstage CRD registration fails.
+_operator_olm_debug_info() {
+  local ns="${OPERATOR_MANAGER:-rhdh-operator}"
+  log::info "OLM/operator diagnostics in namespace: ${ns}"
+  oc get operatorgroup,csv,sub,ip,deploy,pods -n "${ns}" -o wide 2>&1 || true
+  log::info "CatalogSource in openshift-marketplace / olm:"
+  oc get catalogsource -n openshift-marketplace -o wide 2>&1 || true
+  oc get catalogsource -n olm -o wide 2>&1 || true
+  log::info "ClusterExtension / ClusterCatalog (OLM v1), if present:"
+  oc get clusterextension,clustercatalog 2>&1 || true
+  log::info "Backstage CRD:"
+  oc get crd backstages.rhdh.redhat.com -o yaml 2>&1 | head -40 || true
+  oc logs -n "${ns}" -l control-plane=controller-manager --tail=100 2>&1 || true
+  oc get events -n "${ns}" --sort-by='.lastTimestamp' 2>&1 | tail -40 || true
+}
+
+# Install the RHDH operator and wait for the Backstage CRD.
+# Retries the *entire* cycle (namespace recreate → install → CRD wait) up to N times.
+# Args:
+#   $1 - max full-cycle attempts (default: 1)
 prepare_operator() {
-  local retry_operator_installation="${1:-1}"
-  namespace::configure "${OPERATOR_MANAGER}"
-  install_rhdh_operator "${OPERATOR_MANAGER}" "$retry_operator_installation"
+  local max_cycles=${1:-1}
+  local attempt=1
+
+  while ((attempt <= max_cycles)); do
+    log::info "Preparing RHDH operator (attempt ${attempt}/${max_cycles})"
+    # install_rhdh_operator calls namespace::configure — avoid a second wipe here.
+    if ! install_rhdh_operator "${OPERATOR_MANAGER}" 1; then
+      log::error "Operator install script failed on attempt ${attempt}/${max_cycles}"
+      _operator_olm_debug_info
+      if ((attempt < max_cycles)); then
+        attempt=$((attempt + 1))
+        continue
+      fi
+      return 1
+    fi
+
+    # Subscription should exist quickly; fail fast with diagnostics if the script
+    # returned 0 without creating OLM v0 resources.
+    if ! oc get subscription rhdh -n "${OPERATOR_MANAGER}" &> /dev/null; then
+      log::error "Subscription/rhdh missing in ${OPERATOR_MANAGER} after install script success"
+      _operator_olm_debug_info
+      if ((attempt < max_cycles)); then
+        attempt=$((attempt + 1))
+        continue
+      fi
+      return 1
+    fi
+
+    if k8s_wait::crd "backstages.rhdh.redhat.com" 300 10; then
+      log::info "Backstage CRD available after operator prepare attempt ${attempt}"
+      return 0
+    fi
+
+    log::error "Backstage CRD not available after install attempt ${attempt}/${max_cycles}"
+    _operator_olm_debug_info
+    attempt=$((attempt + 1))
+  done
 
-  # Wait for Backstage CRD to be available after operator installation
-  k8s_wait::crd "backstages.rhdh.redhat.com" 300 10 || return 1
+  log::error "Failed to prepare RHDH operator after ${max_cycles} full cycle(s)"
+  return 1
 }
 
 # Waits for the Crunchy Data PostgreSQL Operator's PostgresCluster CRD to become available.

.ci/pipelines/jobs/ocp-operator.sh

@@ -10,15 +10,21 @@ source "$DIR"/utils.sh
 source "$DIR"/install-methods/operator.sh
 # shellcheck source=.ci/pipelines/lib/testing.sh
 source "$DIR"/lib/testing.sh
+# shellcheck source=.ci/pipelines/lib/postgres.sh
+source "$DIR"/lib/postgres.sh
 # shellcheck source=.ci/pipelines/playwright-projects.sh
 source "$DIR"/playwright-projects.sh
 # shellcheck source=.ci/pipelines/lib/schema-mode-env.sh
 source "$DIR"/lib/schema-mode-env.sh
 
 export INSTALL_METHOD=operator
 
-initiate_operator_deployments() {
-  log::info "Initiating Operator-backed deployments on OCP"
+# Default operator-local DB image (RELATED_IMAGE_postgresql). Fedora path: 15 → 18 dump/restore.
+OPERATOR_PG15_IMAGE="${OPERATOR_PG15_IMAGE:-quay.io/fedora/postgresql-15:latest}"
+OPERATOR_PG18_IMAGE="${OPERATOR_PG18_IMAGE:-quay.io/fedora/postgresql-18:latest}"
+
+initiate_operator_showcase_only() {
+  log::info "Initiating Operator-backed showcase deployment (local Fedora PostgreSQL)"
 
   namespace::configure "${NAME_SPACE}"
   deploy_test_backstage_customization_provider "${NAME_SPACE}"
@@ -28,90 +34,173 @@ initiate_operator_deployments() {
   oc apply -f /tmp/configmap-dynamic-plugins.yaml -n "${NAME_SPACE}"
   deploy_redis_cache "${NAME_SPACE}"
   deploy_rhdh_operator "${NAME_SPACE}" "${DIR}/resources/rhdh-operator/rhdh-start.yaml"
-
-  namespace::configure "${NAME_SPACE_RBAC}"
-  config::prepare_operator_app_config "${DIR}/resources/config_map/app-config-rhdh-rbac.yaml"
-  local rbac_rhdh_base_url="https://backstage-${RELEASE_NAME_RBAC}-${NAME_SPACE_RBAC}.${K8S_CLUSTER_ROUTER_BASE}"
-  apply_yaml_files "${DIR}" "${NAME_SPACE_RBAC}" "${rbac_rhdh_base_url}"
-  config::create_dynamic_plugins_config "${DIR}/value_files/${HELM_CHART_RBAC_VALUE_FILE_NAME}" "/tmp/configmap-dynamic-plugins-rbac.yaml"
-  oc apply -f /tmp/configmap-dynamic-plugins-rbac.yaml -n "${NAME_SPACE_RBAC}"
-  wait_for_crunchy_crd || return 1
-  deploy_rhdh_operator "${NAME_SPACE_RBAC}" "${DIR}/resources/rhdh-operator/rhdh-start-rbac.yaml"
 }
 
-initiate_operator_deployments_osd_gcp() {
-  log::info "Initiating Operator-backed deployments on OSD-GCP"
+# Wait for Postgres + Backstage, run Playwright, always persist artifacts under a unique subdir.
+# Args:
+#   $1 - label
+#   $2 - artifacts_subdir
+#   $3 - url
+#   $4 - expected postgres image substring (e.g. postgresql-18)
+#   $5 - optional max wait seconds (default: 600)
+#   $6 - optional previous RHDH pod UID
+#   $7 - optional "seed" to register persistence-proof catalog entity
+_pg_upgrade_verify_and_test() {
+  local label=$1
+  local artifacts_subdir=$2
+  local url=$3
+  local expected_image_substr=$4
+  local max_wait=${5:-600}
+  local previous_rhdh_uid=${6:-}
+  local seed_data=${7:-}
+  local deploy
+  deploy=$(rhdh_deployment_name "${RELEASE_NAME}")
+
+  log::info "=== ${label}: wait for PostgreSQL (${expected_image_substr}) + Backstage, then Playwright ==="
+  log::info "Artifacts subdir: ${ARTIFACT_DIR:-<unset>}/${artifacts_subdir}"
+
+  if ! wait_for_postgres_ready "${NAME_SPACE}" "${max_wait}" "${expected_image_substr}" > /dev/null; then
+    log::error "PostgreSQL not Ready after ${label}"
+    dump_postgres_diagnostics "${NAME_SPACE}"
+    _pg_save_phase_artifacts "${artifacts_subdir}" "${label}"
+    return 1
+  fi
+  log_postgres_version "${NAME_SPACE}"
+
+  if [[ -n "${previous_rhdh_uid}" ]]; then
+    if ! ensure_rhdh_pod_replaced "${RELEASE_NAME}" "${NAME_SPACE}" "${previous_rhdh_uid}" "${max_wait}"; then
+      log::error "Previous RHDH pod did not terminate cleanly after ${label}"
+      dump_postgres_diagnostics "${NAME_SPACE}"
+      _pg_save_phase_artifacts "${artifacts_subdir}" "${label}"
+      return 1
+    fi
+  fi
 
-  namespace::configure "${NAME_SPACE}"
-  deploy_test_backstage_customization_provider "${NAME_SPACE}"
-  local rhdh_base_url="https://backstage-${RELEASE_NAME}-${NAME_SPACE}.${K8S_CLUSTER_ROUTER_BASE}"
-  apply_yaml_files "${DIR}" "${NAME_SPACE}" "${rhdh_base_url}"
+  oc rollout status "deployment/${deploy}" -n "${NAME_SPACE}" --timeout=10m \
+    || log::warn "Backstage rollout status check timed out after ${label}"
 
-  config::create_dynamic_plugins_config "${DIR}/value_files/${HELM_CHART_VALUE_FILE_NAME}" "/tmp/configmap-dynamic-plugins.yaml"
-  common::save_artifact "${PW_PROJECT_SHOWCASE_OPERATOR}" "/tmp/configmap-dynamic-plugins.yaml"
+  if ! testing::check_backstage_running "${RELEASE_NAME}" "${NAME_SPACE}" "${url}" "${artifacts_subdir}" 40 30; then
+    log::error "Backstage not healthy after ${label}"
+    dump_postgres_diagnostics "${NAME_SPACE}"
+    _pg_save_phase_artifacts "${artifacts_subdir}" "${label}"
+    return 1
+  fi
 
-  oc apply -f /tmp/configmap-dynamic-plugins.yaml -n "${NAME_SPACE}"
-  deploy_redis_cache "${NAME_SPACE}"
-  deploy_rhdh_operator "${NAME_SPACE}" "${DIR}/resources/rhdh-operator/rhdh-start.yaml"
+  if [[ "${seed_data}" == "seed" ]]; then
+    if ! seed_pg_upgrade_data_proof "${url}" "${NAME_SPACE}"; then
+      _pg_save_phase_artifacts "${artifacts_subdir}" "${label}"
+      return 1
+    fi
+  elif [[ "${PG_UPGRADE_DATA_PROOF:-}" == "1" ]]; then
+    if ! assert_pg_upgrade_data_proof_api "${url}"; then
+      log::error "Persistence proof entity missing via API after ${label}"
+      _pg_save_phase_artifacts "${artifacts_subdir}" "${label}"
+      return 1
+    fi
+  fi
 
-  namespace::configure "${NAME_SPACE_RBAC}"
-  config::prepare_operator_app_config "${DIR}/resources/config_map/app-config-rhdh-rbac.yaml"
-  local rbac_rhdh_base_url="https://backstage-${RELEASE_NAME_RBAC}-${NAME_SPACE_RBAC}.${K8S_CLUSTER_ROUTER_BASE}"
-  apply_yaml_files "${DIR}" "${NAME_SPACE_RBAC}" "${rbac_rhdh_base_url}"
+  export PG_UPGRADE_DATA_PROOF=1
 
-  config::create_dynamic_plugins_config "${DIR}/value_files/${HELM_CHART_RBAC_VALUE_FILE_NAME}" "/tmp/configmap-dynamic-plugins-rbac.yaml"
-  common::save_artifact "${PW_PROJECT_SHOWCASE_OPERATOR_RBAC}" "/tmp/configmap-dynamic-plugins-rbac.yaml"
+  testing::check_and_test "${RELEASE_NAME}" "${NAME_SPACE}" "${PW_PROJECT_SHOWCASE_OPERATOR}" "${url}" "" "" "${artifacts_subdir}"
 
-  oc apply -f /tmp/configmap-dynamic-plugins-rbac.yaml -n "${NAME_SPACE_RBAC}"
-  wait_for_crunchy_crd || return 1
-  deploy_rhdh_operator "${NAME_SPACE_RBAC}" "${DIR}/resources/rhdh-operator/rhdh-start-rbac.yaml"
+  _pg_save_phase_artifacts "${artifacts_subdir}" "${label}"
+
+  if [[ "${OVERALL_RESULT:-0}" -ne 0 ]]; then
+    log::error "Playwright (or prior step) failed during ${label}; aborting remaining upgrade phases"
+    return 1
+  fi
 }
 
-run_operator_runtime_config_change_tests() {
-  # Runtime tests handle their own deployment via TypeScript (runtime-deploy.ts).
-  # The first test file (config-map.spec.ts) calls ensureRuntimeDeployed() which:
-  #   - Creates the namespace
-  #   - Deploys RHDH via the operator with internal PostgreSQL
-  #   - Configures schema-mode env vars for port-forwarding
-  # Subsequent test files reuse the existing deployment (workers: 1).
-  #
-  # INSTALL_METHOD=operator is already exported in handle_ocp_operator().
-  #
-  # No URL is passed on purpose - see the same note in jobs/ocp-nightly.sh.
-  # A pre-set BASE_URL makes global-setup.ts skip the deploy branch and poll a
-  # route nothing has created yet; ensureRuntimeDeployed() sets BASE_URL itself.
-  export RUNTIME_AUTO_DEPLOY="true"
-  testing::run_tests "${RELEASE_NAME}" "${NAME_SPACE_RUNTIME}" "${PW_PROJECT_SHOWCASE_RUNTIME}" || true
+_pg_save_phase_artifacts() {
+  local artifacts_subdir=$1
+  local label=$2
+  local diag_file="/tmp/pg-diagnostics-${artifacts_subdir}.txt"
+
+  log::info "Saving phase artifacts for '${label}' → ${ARTIFACT_DIR:-<unset>}/${artifacts_subdir}"
+  dump_postgres_diagnostics "${NAME_SPACE}" > "${diag_file}" 2>&1 || true
+  common::save_artifact "${artifacts_subdir}" "${diag_file}" "postgres" || true
+  save_all_pod_logs "${NAME_SPACE}" "${artifacts_subdir}"
 }
 
+# RHIDP-14594: Operator-local Fedora PostgreSQL upgrade via dump/restore.
+# Operator defaults RELATED_IMAGE_postgresql to quay.io/fedora/postgresql-15.
+# Same Fedora constraints as Helm path (#5141): no reliable POSTGRESQL_UPGRADE=copy,
+# no fedora/postgresql-17 — jump 15 → 18 with pg_dumpall / wipe PVC / restore.
+# Showcase-only (RBAC + runtime skipped) to keep the job focused on DB upgrade evidence.
 handle_ocp_operator() {
   export NAME_SPACE="${NAME_SPACE:-showcase}"
   export NAME_SPACE_RBAC="${NAME_SPACE_RBAC:-showcase-rbac}"
   export NAME_SPACE_RUNTIME="${NAME_SPACE_RUNTIME:-showcase-runtime}"
   export NAME_SPACE_POSTGRES_DB="${NAME_SPACE_POSTGRES_DB:-postgress-external-db}"
+  export POSTGRES_CR_NAME="${POSTGRES_CR_NAME:-${RELEASE_NAME:-rhdh}}"
 
   common::oc_login
 
   K8S_CLUSTER_ROUTER_BASE=$(oc get route console -n openshift-console -o=jsonpath='{.spec.host}' | sed 's/^[^.]*\.//')
   export K8S_CLUSTER_ROUTER_BASE
   local url="https://backstage-${RELEASE_NAME}-${NAME_SPACE}.${K8S_CLUSTER_ROUTER_BASE}"
-  local rbac_url="https://backstage-${RELEASE_NAME_RBAC}-${NAME_SPACE_RBAC}.${K8S_CLUSTER_ROUTER_BASE}"
+  local art_pg15="${PW_PROJECT_SHOWCASE_OPERATOR}-pg15"
+  local art_pg18="${PW_PROJECT_SHOWCASE_OPERATOR}-pg18"
+  local deploy
+  deploy=$(rhdh_deployment_name "${RELEASE_NAME}")
 
   cluster_setup_ocp_operator
 
   if [[ "${JOB_NAME}" =~ osd-gcp ]]; then
     # OSD-GCP internal registry is unreliable under parallel load; reduce
     # concurrent skopeo pushes and allow retries (RHDHBUGS-1136).
     export MAX_PARALLEL=3
-    prepare_operator 3
-    initiate_operator_deployments_osd_gcp
-  else
-    prepare_operator
-    initiate_operator_deployments
   fi
+  # Full install+CRD cycles (covers "install script OK but CRD never appears").
+  prepare_operator 3
+
+  # Ensure baseline image is Fedora PG15 (operator CSV default; set explicitly for evidence).
+  if ! set_operator_postgresql_related_image "${OPERATOR_PG15_IMAGE}"; then
+    log::error "Failed to set operator RELATED_IMAGE_postgresql to PG15"
+    return 1
+  fi
+
+  initiate_operator_showcase_only
+
+  log::info "Phase PG15: operator-local ${OPERATOR_PG15_IMAGE}"
+  if ! _pg_upgrade_verify_and_test "PG15 baseline" "${art_pg15}" "${url}" "postgresql-15" 600 "" "seed"; then
+    return 1
+  fi
+
+  local rhdh_uid_before_18
+  rhdh_uid_before_18=$(get_rhdh_pod_uid "${RELEASE_NAME}" "${NAME_SPACE}")
+  log::info "RHDH pod uid before PG18 dump/restore: ${rhdh_uid_before_18:-<none>}"
+
+  log::info "Phase PG18: dump/restore 15 → 18 (operator-local Fedora; skip PG16)"
+  local dumpfile="/tmp/rhdh-operator-pg15-dumpall.sql"
+  if ! postgres_dumpall_to_file "${NAME_SPACE}" "${dumpfile}"; then
+    _pg_save_phase_artifacts "${art_pg18}" "PG18 dump failed"
+    return 1
+  fi
+  common::save_artifact "${art_pg18}" "${dumpfile}" "postgres" || true
+
+  # Switch operator default DB image to PG18 before wiping so recreate uses 18.
+  if ! set_operator_postgresql_related_image "${OPERATOR_PG18_IMAGE}"; then
+    dump_postgres_diagnostics "${NAME_SPACE}"
+    _pg_save_phase_artifacts "${art_pg18}" "PG18 RELATED_IMAGE update failed"
+    return 1
+  fi
+
+  postgres_wipe_persistent_volume "${NAME_SPACE}"
+
+  if ! postgres_restore_dumpall_file "${NAME_SPACE}" "${dumpfile}" "postgresql-18"; then
+    dump_postgres_diagnostics "${NAME_SPACE}"
+    _pg_save_phase_artifacts "${art_pg18}" "PG18 restore failed"
+    return 1
+  fi
+  refresh_postgres_collation_versions "${NAME_SPACE}" 600 "postgresql-18"
 
-  testing::check_and_test "${RELEASE_NAME}" "${NAME_SPACE}" "${PW_PROJECT_SHOWCASE_OPERATOR}" "${url}"
-  testing::check_and_test "${RELEASE_NAME_RBAC}" "${NAME_SPACE_RBAC}" "${PW_PROJECT_SHOWCASE_OPERATOR_RBAC}" "${rbac_url}"
+  oc rollout restart "deployment/${deploy}" -n "${NAME_SPACE}" || true
+
+  if ! _pg_upgrade_verify_and_test "PG18 after dump/restore" "${art_pg18}" "${url}" "postgresql-18" 900 "${rhdh_uid_before_18}"; then
+    return 1
+  fi
 
-  run_operator_runtime_config_change_tests
+  log::info "PostgreSQL 15 → 18 Operator Fedora dump/restore sequence completed (Playwright after each major)"
+  log::info "Skipping operator RBAC + runtime suites on this evidence PR (showcase-only)"
 }