diff --git a/.ci/pipelines/install-methods/operator.sh b/.ci/pipelines/install-methods/operator.sh index 864c981217..2462077dab 100755 --- a/.ci/pipelines/install-methods/operator.sh +++ b/.ci/pipelines/install-methods/operator.sh @@ -5,9 +5,28 @@ source "$DIR"/lib/log.sh # shellcheck source=.ci/pipelines/utils.sh source "$DIR"/utils.sh +# Run install-rhdh-catalog-source.sh with visible logs (common::retry swallows stdout/stderr). +# Forces OLM v0: auto-detect can pick OLM v1 (ClusterExtension) which returns success without +# waiting for the operator CSV/CRDs, leaving Backstage CRD missing. +_install_rhdh_operator_once() { + local -a install_args=() + + if [[ "${RELEASE_VERSION}" == "next" ]]; then + log::info "Installing RHDH operator with '--next --olm-version v0'" + install_args=(--next --olm-version v0 --install-operator rhdh) + else + log::info "Installing RHDH operator with '-v ${RELEASE_VERSION} --olm-version v0'" + install_args=(-v "${RELEASE_VERSION}" --olm-version v0 --install-operator rhdh) + fi + + # Stream script output to the CI log; do not capture (common::retry hides failures). + bash -x /tmp/install-rhdh-catalog-source.sh "${install_args[@]}" +} + install_rhdh_operator() { local namespace=$1 local max_attempts=$2 + local attempt=1 namespace::configure "$namespace" @@ -28,28 +47,84 @@ install_rhdh_operator() { fi chmod +x /tmp/install-rhdh-catalog-source.sh - if [[ "$RELEASE_VERSION" == "next" ]]; then - log::info "Installing RHDH operator with '--next' flag" - if ! common::retry "$max_attempts" 10 bash -x /tmp/install-rhdh-catalog-source.sh --next --install-operator rhdh; then - log::error "Failed install RHDH Operator after ${max_attempts} attempts." - return 1 + while ((attempt <= max_attempts)); do + log::info "Operator install script attempt ${attempt}/${max_attempts}" + if _install_rhdh_operator_once; then + log::success "Operator install script succeeded on attempt ${attempt}" + return 0 fi - else - log::info "Installing RHDH operator with '-v $RELEASE_VERSION' flag" - if ! common::retry "$max_attempts" 10 bash -x /tmp/install-rhdh-catalog-source.sh -v "$RELEASE_VERSION" --install-operator rhdh; then - log::error "Failed install RHDH Operator after ${max_attempts} attempts." - return 1 + log::warn "Operator install script failed on attempt ${attempt}/${max_attempts}" + if ((attempt < max_attempts)); then + sleep 10 fi - fi + attempt=$((attempt + 1)) + done + + log::error "Failed install RHDH Operator after ${max_attempts} attempts." + return 1 } +# Dump OLM / operator-manager state when Backstage CRD registration fails. +_operator_olm_debug_info() { + local ns="${OPERATOR_MANAGER:-rhdh-operator}" + log::info "OLM/operator diagnostics in namespace: ${ns}" + oc get operatorgroup,csv,sub,ip,deploy,pods -n "${ns}" -o wide 2>&1 || true + log::info "CatalogSource in openshift-marketplace / olm:" + oc get catalogsource -n openshift-marketplace -o wide 2>&1 || true + oc get catalogsource -n olm -o wide 2>&1 || true + log::info "ClusterExtension / ClusterCatalog (OLM v1), if present:" + oc get clusterextension,clustercatalog 2>&1 || true + log::info "Backstage CRD:" + oc get crd backstages.rhdh.redhat.com -o yaml 2>&1 | head -40 || true + oc logs -n "${ns}" -l control-plane=controller-manager --tail=100 2>&1 || true + oc get events -n "${ns}" --sort-by='.lastTimestamp' 2>&1 | tail -40 || true +} + +# Install the RHDH operator and wait for the Backstage CRD. +# Retries the *entire* cycle (namespace recreate → install → CRD wait) up to N times. +# Args: +# $1 - max full-cycle attempts (default: 1) prepare_operator() { - local retry_operator_installation="${1:-1}" - namespace::configure "${OPERATOR_MANAGER}" - install_rhdh_operator "${OPERATOR_MANAGER}" "$retry_operator_installation" + local max_cycles=${1:-1} + local attempt=1 + + while ((attempt <= max_cycles)); do + log::info "Preparing RHDH operator (attempt ${attempt}/${max_cycles})" + # install_rhdh_operator calls namespace::configure — avoid a second wipe here. + if ! install_rhdh_operator "${OPERATOR_MANAGER}" 1; then + log::error "Operator install script failed on attempt ${attempt}/${max_cycles}" + _operator_olm_debug_info + if ((attempt < max_cycles)); then + attempt=$((attempt + 1)) + continue + fi + return 1 + fi + + # Subscription should exist quickly; fail fast with diagnostics if the script + # returned 0 without creating OLM v0 resources. + if ! oc get subscription rhdh -n "${OPERATOR_MANAGER}" &> /dev/null; then + log::error "Subscription/rhdh missing in ${OPERATOR_MANAGER} after install script success" + _operator_olm_debug_info + if ((attempt < max_cycles)); then + attempt=$((attempt + 1)) + continue + fi + return 1 + fi + + if k8s_wait::crd "backstages.rhdh.redhat.com" 300 10; then + log::info "Backstage CRD available after operator prepare attempt ${attempt}" + return 0 + fi + + log::error "Backstage CRD not available after install attempt ${attempt}/${max_cycles}" + _operator_olm_debug_info + attempt=$((attempt + 1)) + done - # Wait for Backstage CRD to be available after operator installation - k8s_wait::crd "backstages.rhdh.redhat.com" 300 10 || return 1 + log::error "Failed to prepare RHDH operator after ${max_cycles} full cycle(s)" + return 1 } # Waits for the Crunchy Data PostgreSQL Operator's PostgresCluster CRD to become available. diff --git a/.ci/pipelines/jobs/ocp-operator.sh b/.ci/pipelines/jobs/ocp-operator.sh index 13240ec159..230eb49cc5 100644 --- a/.ci/pipelines/jobs/ocp-operator.sh +++ b/.ci/pipelines/jobs/ocp-operator.sh @@ -10,6 +10,8 @@ source "$DIR"/utils.sh source "$DIR"/install-methods/operator.sh # shellcheck source=.ci/pipelines/lib/testing.sh source "$DIR"/lib/testing.sh +# shellcheck source=.ci/pipelines/lib/postgres.sh +source "$DIR"/lib/postgres.sh # shellcheck source=.ci/pipelines/playwright-projects.sh source "$DIR"/playwright-projects.sh # shellcheck source=.ci/pipelines/lib/schema-mode-env.sh @@ -17,8 +19,12 @@ source "$DIR"/lib/schema-mode-env.sh export INSTALL_METHOD=operator -initiate_operator_deployments() { - log::info "Initiating Operator-backed deployments on OCP" +# Default operator-local DB image (RELATED_IMAGE_postgresql). Fedora path: 15 → 18 dump/restore. +OPERATOR_PG15_IMAGE="${OPERATOR_PG15_IMAGE:-quay.io/fedora/postgresql-15:latest}" +OPERATOR_PG18_IMAGE="${OPERATOR_PG18_IMAGE:-quay.io/fedora/postgresql-18:latest}" + +initiate_operator_showcase_only() { + log::info "Initiating Operator-backed showcase deployment (local Fedora PostgreSQL)" namespace::configure "${NAME_SPACE}" deploy_test_backstage_customization_provider "${NAME_SPACE}" @@ -28,74 +34,115 @@ initiate_operator_deployments() { oc apply -f /tmp/configmap-dynamic-plugins.yaml -n "${NAME_SPACE}" deploy_redis_cache "${NAME_SPACE}" deploy_rhdh_operator "${NAME_SPACE}" "${DIR}/resources/rhdh-operator/rhdh-start.yaml" - - namespace::configure "${NAME_SPACE_RBAC}" - config::prepare_operator_app_config "${DIR}/resources/config_map/app-config-rhdh-rbac.yaml" - local rbac_rhdh_base_url="https://backstage-${RELEASE_NAME_RBAC}-${NAME_SPACE_RBAC}.${K8S_CLUSTER_ROUTER_BASE}" - apply_yaml_files "${DIR}" "${NAME_SPACE_RBAC}" "${rbac_rhdh_base_url}" - config::create_dynamic_plugins_config "${DIR}/value_files/${HELM_CHART_RBAC_VALUE_FILE_NAME}" "/tmp/configmap-dynamic-plugins-rbac.yaml" - oc apply -f /tmp/configmap-dynamic-plugins-rbac.yaml -n "${NAME_SPACE_RBAC}" - wait_for_crunchy_crd || return 1 - deploy_rhdh_operator "${NAME_SPACE_RBAC}" "${DIR}/resources/rhdh-operator/rhdh-start-rbac.yaml" } -initiate_operator_deployments_osd_gcp() { - log::info "Initiating Operator-backed deployments on OSD-GCP" +# Wait for Postgres + Backstage, run Playwright, always persist artifacts under a unique subdir. +# Args: +# $1 - label +# $2 - artifacts_subdir +# $3 - url +# $4 - expected postgres image substring (e.g. postgresql-18) +# $5 - optional max wait seconds (default: 600) +# $6 - optional previous RHDH pod UID +# $7 - optional "seed" to register persistence-proof catalog entity +_pg_upgrade_verify_and_test() { + local label=$1 + local artifacts_subdir=$2 + local url=$3 + local expected_image_substr=$4 + local max_wait=${5:-600} + local previous_rhdh_uid=${6:-} + local seed_data=${7:-} + local deploy + deploy=$(rhdh_deployment_name "${RELEASE_NAME}") + + log::info "=== ${label}: wait for PostgreSQL (${expected_image_substr}) + Backstage, then Playwright ===" + log::info "Artifacts subdir: ${ARTIFACT_DIR:-}/${artifacts_subdir}" + + if ! wait_for_postgres_ready "${NAME_SPACE}" "${max_wait}" "${expected_image_substr}" > /dev/null; then + log::error "PostgreSQL not Ready after ${label}" + dump_postgres_diagnostics "${NAME_SPACE}" + _pg_save_phase_artifacts "${artifacts_subdir}" "${label}" + return 1 + fi + log_postgres_version "${NAME_SPACE}" + + if [[ -n "${previous_rhdh_uid}" ]]; then + if ! ensure_rhdh_pod_replaced "${RELEASE_NAME}" "${NAME_SPACE}" "${previous_rhdh_uid}" "${max_wait}"; then + log::error "Previous RHDH pod did not terminate cleanly after ${label}" + dump_postgres_diagnostics "${NAME_SPACE}" + _pg_save_phase_artifacts "${artifacts_subdir}" "${label}" + return 1 + fi + fi - namespace::configure "${NAME_SPACE}" - deploy_test_backstage_customization_provider "${NAME_SPACE}" - local rhdh_base_url="https://backstage-${RELEASE_NAME}-${NAME_SPACE}.${K8S_CLUSTER_ROUTER_BASE}" - apply_yaml_files "${DIR}" "${NAME_SPACE}" "${rhdh_base_url}" + oc rollout status "deployment/${deploy}" -n "${NAME_SPACE}" --timeout=10m \ + || log::warn "Backstage rollout status check timed out after ${label}" - config::create_dynamic_plugins_config "${DIR}/value_files/${HELM_CHART_VALUE_FILE_NAME}" "/tmp/configmap-dynamic-plugins.yaml" - common::save_artifact "${PW_PROJECT_SHOWCASE_OPERATOR}" "/tmp/configmap-dynamic-plugins.yaml" + if ! testing::check_backstage_running "${RELEASE_NAME}" "${NAME_SPACE}" "${url}" "${artifacts_subdir}" 40 30; then + log::error "Backstage not healthy after ${label}" + dump_postgres_diagnostics "${NAME_SPACE}" + _pg_save_phase_artifacts "${artifacts_subdir}" "${label}" + return 1 + fi - oc apply -f /tmp/configmap-dynamic-plugins.yaml -n "${NAME_SPACE}" - deploy_redis_cache "${NAME_SPACE}" - deploy_rhdh_operator "${NAME_SPACE}" "${DIR}/resources/rhdh-operator/rhdh-start.yaml" + if [[ "${seed_data}" == "seed" ]]; then + if ! seed_pg_upgrade_data_proof "${url}" "${NAME_SPACE}"; then + _pg_save_phase_artifacts "${artifacts_subdir}" "${label}" + return 1 + fi + elif [[ "${PG_UPGRADE_DATA_PROOF:-}" == "1" ]]; then + if ! assert_pg_upgrade_data_proof_api "${url}"; then + log::error "Persistence proof entity missing via API after ${label}" + _pg_save_phase_artifacts "${artifacts_subdir}" "${label}" + return 1 + fi + fi - namespace::configure "${NAME_SPACE_RBAC}" - config::prepare_operator_app_config "${DIR}/resources/config_map/app-config-rhdh-rbac.yaml" - local rbac_rhdh_base_url="https://backstage-${RELEASE_NAME_RBAC}-${NAME_SPACE_RBAC}.${K8S_CLUSTER_ROUTER_BASE}" - apply_yaml_files "${DIR}" "${NAME_SPACE_RBAC}" "${rbac_rhdh_base_url}" + export PG_UPGRADE_DATA_PROOF=1 - config::create_dynamic_plugins_config "${DIR}/value_files/${HELM_CHART_RBAC_VALUE_FILE_NAME}" "/tmp/configmap-dynamic-plugins-rbac.yaml" - common::save_artifact "${PW_PROJECT_SHOWCASE_OPERATOR_RBAC}" "/tmp/configmap-dynamic-plugins-rbac.yaml" + testing::check_and_test "${RELEASE_NAME}" "${NAME_SPACE}" "${PW_PROJECT_SHOWCASE_OPERATOR}" "${url}" "" "" "${artifacts_subdir}" - oc apply -f /tmp/configmap-dynamic-plugins-rbac.yaml -n "${NAME_SPACE_RBAC}" - wait_for_crunchy_crd || return 1 - deploy_rhdh_operator "${NAME_SPACE_RBAC}" "${DIR}/resources/rhdh-operator/rhdh-start-rbac.yaml" + _pg_save_phase_artifacts "${artifacts_subdir}" "${label}" + + if [[ "${OVERALL_RESULT:-0}" -ne 0 ]]; then + log::error "Playwright (or prior step) failed during ${label}; aborting remaining upgrade phases" + return 1 + fi } -run_operator_runtime_config_change_tests() { - # Runtime tests handle their own deployment via TypeScript (runtime-deploy.ts). - # The first test file (config-map.spec.ts) calls ensureRuntimeDeployed() which: - # - Creates the namespace - # - Deploys RHDH via the operator with internal PostgreSQL - # - Configures schema-mode env vars for port-forwarding - # Subsequent test files reuse the existing deployment (workers: 1). - # - # INSTALL_METHOD=operator is already exported in handle_ocp_operator(). - # - # No URL is passed on purpose - see the same note in jobs/ocp-nightly.sh. - # A pre-set BASE_URL makes global-setup.ts skip the deploy branch and poll a - # route nothing has created yet; ensureRuntimeDeployed() sets BASE_URL itself. - export RUNTIME_AUTO_DEPLOY="true" - testing::run_tests "${RELEASE_NAME}" "${NAME_SPACE_RUNTIME}" "${PW_PROJECT_SHOWCASE_RUNTIME}" || true +_pg_save_phase_artifacts() { + local artifacts_subdir=$1 + local label=$2 + local diag_file="/tmp/pg-diagnostics-${artifacts_subdir}.txt" + + log::info "Saving phase artifacts for '${label}' → ${ARTIFACT_DIR:-}/${artifacts_subdir}" + dump_postgres_diagnostics "${NAME_SPACE}" > "${diag_file}" 2>&1 || true + common::save_artifact "${artifacts_subdir}" "${diag_file}" "postgres" || true + save_all_pod_logs "${NAME_SPACE}" "${artifacts_subdir}" } +# RHIDP-14594: Operator-local Fedora PostgreSQL upgrade via dump/restore. +# Operator defaults RELATED_IMAGE_postgresql to quay.io/fedora/postgresql-15. +# Same Fedora constraints as Helm path (#5141): no reliable POSTGRESQL_UPGRADE=copy, +# no fedora/postgresql-17 — jump 15 → 18 with pg_dumpall / wipe PVC / restore. +# Showcase-only (RBAC + runtime skipped) to keep the job focused on DB upgrade evidence. handle_ocp_operator() { export NAME_SPACE="${NAME_SPACE:-showcase}" export NAME_SPACE_RBAC="${NAME_SPACE_RBAC:-showcase-rbac}" export NAME_SPACE_RUNTIME="${NAME_SPACE_RUNTIME:-showcase-runtime}" export NAME_SPACE_POSTGRES_DB="${NAME_SPACE_POSTGRES_DB:-postgress-external-db}" + export POSTGRES_CR_NAME="${POSTGRES_CR_NAME:-${RELEASE_NAME:-rhdh}}" common::oc_login K8S_CLUSTER_ROUTER_BASE=$(oc get route console -n openshift-console -o=jsonpath='{.spec.host}' | sed 's/^[^.]*\.//') export K8S_CLUSTER_ROUTER_BASE local url="https://backstage-${RELEASE_NAME}-${NAME_SPACE}.${K8S_CLUSTER_ROUTER_BASE}" - local rbac_url="https://backstage-${RELEASE_NAME_RBAC}-${NAME_SPACE_RBAC}.${K8S_CLUSTER_ROUTER_BASE}" + local art_pg15="${PW_PROJECT_SHOWCASE_OPERATOR}-pg15" + local art_pg18="${PW_PROJECT_SHOWCASE_OPERATOR}-pg18" + local deploy + deploy=$(rhdh_deployment_name "${RELEASE_NAME}") cluster_setup_ocp_operator @@ -103,15 +150,57 @@ handle_ocp_operator() { # OSD-GCP internal registry is unreliable under parallel load; reduce # concurrent skopeo pushes and allow retries (RHDHBUGS-1136). export MAX_PARALLEL=3 - prepare_operator 3 - initiate_operator_deployments_osd_gcp - else - prepare_operator - initiate_operator_deployments fi + # Full install+CRD cycles (covers "install script OK but CRD never appears"). + prepare_operator 3 + + # Ensure baseline image is Fedora PG15 (operator CSV default; set explicitly for evidence). + if ! set_operator_postgresql_related_image "${OPERATOR_PG15_IMAGE}"; then + log::error "Failed to set operator RELATED_IMAGE_postgresql to PG15" + return 1 + fi + + initiate_operator_showcase_only + + log::info "Phase PG15: operator-local ${OPERATOR_PG15_IMAGE}" + if ! _pg_upgrade_verify_and_test "PG15 baseline" "${art_pg15}" "${url}" "postgresql-15" 600 "" "seed"; then + return 1 + fi + + local rhdh_uid_before_18 + rhdh_uid_before_18=$(get_rhdh_pod_uid "${RELEASE_NAME}" "${NAME_SPACE}") + log::info "RHDH pod uid before PG18 dump/restore: ${rhdh_uid_before_18:-}" + + log::info "Phase PG18: dump/restore 15 → 18 (operator-local Fedora; skip PG16)" + local dumpfile="/tmp/rhdh-operator-pg15-dumpall.sql" + if ! postgres_dumpall_to_file "${NAME_SPACE}" "${dumpfile}"; then + _pg_save_phase_artifacts "${art_pg18}" "PG18 dump failed" + return 1 + fi + common::save_artifact "${art_pg18}" "${dumpfile}" "postgres" || true + + # Switch operator default DB image to PG18 before wiping so recreate uses 18. + if ! set_operator_postgresql_related_image "${OPERATOR_PG18_IMAGE}"; then + dump_postgres_diagnostics "${NAME_SPACE}" + _pg_save_phase_artifacts "${art_pg18}" "PG18 RELATED_IMAGE update failed" + return 1 + fi + + postgres_wipe_persistent_volume "${NAME_SPACE}" + + if ! postgres_restore_dumpall_file "${NAME_SPACE}" "${dumpfile}" "postgresql-18"; then + dump_postgres_diagnostics "${NAME_SPACE}" + _pg_save_phase_artifacts "${art_pg18}" "PG18 restore failed" + return 1 + fi + refresh_postgres_collation_versions "${NAME_SPACE}" 600 "postgresql-18" - testing::check_and_test "${RELEASE_NAME}" "${NAME_SPACE}" "${PW_PROJECT_SHOWCASE_OPERATOR}" "${url}" - testing::check_and_test "${RELEASE_NAME_RBAC}" "${NAME_SPACE_RBAC}" "${PW_PROJECT_SHOWCASE_OPERATOR_RBAC}" "${rbac_url}" + oc rollout restart "deployment/${deploy}" -n "${NAME_SPACE}" || true + + if ! _pg_upgrade_verify_and_test "PG18 after dump/restore" "${art_pg18}" "${url}" "postgresql-18" 900 "${rhdh_uid_before_18}"; then + return 1 + fi - run_operator_runtime_config_change_tests + log::info "PostgreSQL 15 → 18 Operator Fedora dump/restore sequence completed (Playwright after each major)" + log::info "Skipping operator RBAC + runtime suites on this evidence PR (showcase-only)" } diff --git a/.ci/pipelines/lib/postgres.sh b/.ci/pipelines/lib/postgres.sh new file mode 100644 index 0000000000..59cef745a7 --- /dev/null +++ b/.ci/pipelines/lib/postgres.sh @@ -0,0 +1,707 @@ +#!/usr/bin/env bash + +# PostgreSQL helpers for chart-managed / operator-local database upgrades. +# Dependencies: oc, lib/log.sh +# +# Install method awareness (INSTALL_METHOD=helm|operator): +# helm — label app.kubernetes.io/name=postgresql, deploy ${release}-developer-hub +# operator — label rhdh.redhat.com/app=backstage-psql-${cr}, deploy backstage-${cr} +# +# IMPORTANT: functions that print a pod name on stdout (for capture) MUST send +# all diagnostics to stderr, otherwise callers using $(...) swallow the evidence. + +# Prevent re-sourcing +if [[ -n "${POSTGRES_LIB_SOURCED:-}" ]]; then + return 0 +fi +readonly POSTGRES_LIB_SOURCED=1 + +# shellcheck source=.ci/pipelines/lib/log.sh +source "${DIR}/lib/log.sh" + +# --------------------------------------------------------------------------- +# Install-method helpers +# --------------------------------------------------------------------------- + +# Backstage Deployment name for the given release / CR name. +rhdh_deployment_name() { + local release_name=$1 + if [[ "${INSTALL_METHOD:-helm}" == "operator" ]]; then + echo "backstage-${release_name}" + else + echo "${release_name}-developer-hub" + fi +} + +# CR / release name used for operator-local Postgres resources. +_postgres_cr_name() { + echo "${POSTGRES_CR_NAME:-${RELEASE_NAME:-rhdh}}" +} + +# Resolve a Ready-candidate Postgres pod name in the namespace (or empty). +_postgres_find_pod() { + local namespace=$1 + local pod="" + if [[ "${INSTALL_METHOD:-helm}" == "operator" ]]; then + local cr + cr=$(_postgres_cr_name) + pod=$(oc get pods -n "${namespace}" -l "rhdh.redhat.com/app=backstage-psql-${cr}" \ + -o jsonpath='{.items[0].metadata.name}' 2> /dev/null || true) + if [[ -z "${pod}" ]]; then + pod=$(oc get pods -n "${namespace}" \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2> /dev/null \ + | grep -E "^backstage-psql-${cr}-" | head -1 || true) + fi + else + pod=$(oc get pods -n "${namespace}" -l "app.kubernetes.io/name=postgresql" \ + -o jsonpath='{.items[0].metadata.name}' 2> /dev/null || true) + fi + echo "${pod}" +} + +# StatefulSet readyReplicas for the managed Postgres instance (or 0). +_postgres_sts_ready_replicas() { + local namespace=$1 + local sts_ready="" + if [[ "${INSTALL_METHOD:-helm}" == "operator" ]]; then + local cr + cr=$(_postgres_cr_name) + sts_ready=$(oc get statefulset -n "${namespace}" "backstage-psql-${cr}" \ + -o jsonpath='{.status.readyReplicas}' 2> /dev/null || true) + if [[ -z "${sts_ready}" ]]; then + sts_ready=$(oc get statefulset -n "${namespace}" \ + -l "rhdh.redhat.com/app=backstage-psql-${cr}" \ + -o jsonpath='{.items[0].status.readyReplicas}' 2> /dev/null || true) + fi + else + sts_ready=$(oc get statefulset -n "${namespace}" -l "app.kubernetes.io/name=postgresql" \ + -o jsonpath='{.items[0].status.readyReplicas}' 2> /dev/null || true) + fi + echo "${sts_ready:-0}" +} + +# Current RELATED_IMAGE_postgresql on the operator Deployment (or empty). +_operator_related_postgresql_image() { + local ns="${OPERATOR_MANAGER:-rhdh-operator}" + local deploy=$1 + oc get "deploy/${deploy}" -n "${ns}" \ + -o jsonpath='{range .spec.template.spec.containers[0].env[?(@.name=="RELATED_IMAGE_postgresql")]}{.value}{end}' \ + 2> /dev/null || true +} + +# Resolve operator controller Deployment name in OPERATOR_MANAGER (or empty). +_operator_controller_deploy_name() { + local ns="${OPERATOR_MANAGER:-rhdh-operator}" + local deploy + deploy=$(oc get deploy -n "${ns}" -l control-plane=controller-manager \ + -o jsonpath='{.items[0].metadata.name}' 2> /dev/null || true) + if [[ -z "${deploy}" ]]; then + deploy=$(oc get deploy -n "${ns}" -o jsonpath='{.items[0].metadata.name}' 2> /dev/null || true) + fi + echo "${deploy}" +} + +# Point the RHDH operator at a different Fedora/RHEL PostgreSQL image via RELATED_IMAGE_postgresql. +# +# OLM owns the operator Deployment: `oc set env` is raced/reverted to the CSV +# default (often registry.redhat.io/rhel9/postgresql-15@sha256:…). Pin the value +# on Subscription.spec.config.env so OLM keeps applying our override. +# +# The Backstage CRD can appear before the operator Deployment exists — patch the +# Subscription first, then wait for Deployment + env. +# +# Args: +# $1 - full image ref (e.g. quay.io/fedora/postgresql-18:latest) +set_operator_postgresql_related_image() { + local image=$1 + local ns="${OPERATOR_MANAGER:-rhdh-operator}" + local sub="${OPERATOR_SUBSCRIPTION_NAME:-rhdh}" + local deploy="" + local current="" + local waited=0 + local max_wait=300 + + if [[ -z "${image}" ]]; then + log::error "set_operator_postgresql_related_image: image required" + return 1 + fi + + log::info "Waiting up to ${max_wait}s for Subscription/${sub} in ${ns}" + waited=0 + while ((waited < max_wait)); do + if oc get subscription "${sub}" -n "${ns}" &> /dev/null; then + break + fi + log::info "Still waiting… Subscription/${sub} not found (${waited}s/${max_wait}s)" + oc get subscription,csv,ip,deploy,pods -n "${ns}" -o wide >&2 || true + sleep 5 + waited=$((waited + 5)) + done + if ! oc get subscription "${sub}" -n "${ns}" &> /dev/null; then + log::error "Subscription/${sub} not found in ${ns}; cannot pin RELATED_IMAGE_postgresql via OLM" + oc get subscription,csv,ip,deploy,pods -n "${ns}" -o wide >&2 || true + return 1 + fi + + log::info "Pinning RELATED_IMAGE_postgresql=${image} via Subscription/${sub} spec.config.env (OLM-safe)" + # Merge-patch replaces config.env; this job only needs the postgresql related image. + if ! oc patch subscription "${sub}" -n "${ns}" --type=merge \ + -p "{\"spec\":{\"config\":{\"env\":[{\"name\":\"RELATED_IMAGE_postgresql\",\"value\":\"${image}\"}]}}}"; then + log::error "Failed to patch Subscription/${sub} config.env" + return 1 + fi + + log::info "Waiting up to ${max_wait}s for operator Deployment with RELATED_IMAGE_postgresql=${image}" + waited=0 + while ((waited < max_wait)); do + deploy=$(_operator_controller_deploy_name) + if [[ -n "${deploy}" ]]; then + # Best-effort immediate apply; OLM re-applies from Subscription. + oc set env "deployment/${deploy}" -n "${ns}" "RELATED_IMAGE_postgresql=${image}" 2>&1 || true + current=$(_operator_related_postgresql_image "${deploy}") + if [[ "${current}" == "${image}" ]]; then + break + fi + log::info "Still waiting… deploy=${deploy} RELATED_IMAGE_postgresql=${current:-} (${waited}s/${max_wait}s)" + else + log::info "Still waiting… no operator Deployment yet (${waited}s/${max_wait}s)" + oc get subscription,csv,ip,deploy,pods -n "${ns}" -o wide >&2 || true + fi + sleep 5 + waited=$((waited + 5)) + done + + if [[ -z "${deploy}" || "${current}" != "${image}" ]]; then + log::error "RELATED_IMAGE_postgresql did not stick (deploy='${deploy:-}', got '${current:-}', want '${image}')" + oc get subscription "${sub}" -n "${ns}" -o yaml >&2 || true + oc get deploy,pods -n "${ns}" -o wide >&2 || true + [[ -n "${deploy}" ]] && oc get "deploy/${deploy}" -n "${ns}" -o yaml >&2 || true + return 1 + fi + + if ! oc rollout status "deployment/${deploy}" -n "${ns}" --timeout=5m; then + log::error "Operator rollout failed after RELATED_IMAGE_postgresql update" + oc get pods -n "${ns}" -o wide >&2 || true + oc logs -n "${ns}" "deploy/${deploy}" --tail=80 >&2 || true + return 1 + fi + + # Settle check: OLM must not have reverted after rollout status returned. + sleep 5 + current=$(_operator_related_postgresql_image "${deploy}") + if [[ "${current}" != "${image}" ]]; then + log::error "OLM reverted RELATED_IMAGE_postgresql after rollout (got '${current:-}', want '${image}')" + oc get subscription "${sub}" -n "${ns}" -o yaml >&2 || true + return 1 + fi + + log::info "Operator controller ready with RELATED_IMAGE_postgresql=${image} (Subscription-pinned)" +} + +# Dump postgres pod diagnostics to stderr (safe under $(...) capture). +# Args: +# $1 - namespace +dump_postgres_diagnostics() { + local namespace=$1 + { + log::info "PostgreSQL diagnostics for namespace: ${namespace}" + oc get pods,statefulset,pvc -n "${namespace}" -o wide 2>&1 | grep -iE 'postgres|NAME' || true + oc get pods -n "${namespace}" -o wide 2>&1 | head -40 || true + local pg_pod + pg_pod=$(_postgres_find_pod "${namespace}") + if [[ -z "$pg_pod" ]]; then + log::warn "No postgresql pod found; listing all pods containing 'postgres'" + oc get pods -n "${namespace}" 2>&1 | grep -i postgres || true + return 0 + fi + log::info "Pod image/status for ${pg_pod}:" + oc get pod -n "${namespace}" "${pg_pod}" -o jsonpath='image={.spec.containers[0].image} phase={.status.phase} ready={.status.conditions[?(@.type=="Ready")].status}{"\n"}' 2>&1 || true + log::info "Describe pod ${pg_pod}:" + oc describe pod -n "${namespace}" "${pg_pod}" 2>&1 | tail -100 || true + log::info "Logs for ${pg_pod} (current):" + oc logs -n "${namespace}" "${pg_pod}" --tail=200 2>&1 || true + log::info "Logs for ${pg_pod} (previous):" + oc logs -n "${namespace}" "${pg_pod}" --previous --tail=200 2>&1 || true + if [[ "${INSTALL_METHOD:-helm}" == "operator" ]]; then + log::info "Operator RELATED_IMAGE_postgresql:" + oc get deploy -n "${OPERATOR_MANAGER:-rhdh-operator}" -o jsonpath='{range .items[*]}{.metadata.name}{" RELATED_IMAGE_postgresql="}{range .spec.template.spec.containers[0].env[?(@.name=="RELATED_IMAGE_postgresql")]}{.value}{end}{"\n"}{end}' 2>&1 || true + log::info "Subscription config.env (RELATED_IMAGE_postgresql pin):" + oc get subscription "${OPERATOR_SUBSCRIPTION_NAME:-rhdh}" -n "${OPERATOR_MANAGER:-rhdh-operator}" \ + -o jsonpath='{range .spec.config.env[*]}{.name}{"="}{.value}{"\n"}{end}' 2>&1 || true + log::info "Backstage CR (database):" + oc get backstage -n "${namespace}" -o yaml 2>&1 | grep -A20 -E 'database:|enableLocalDb' || true + else + log::info "Helm values snapshot (postgresql image):" + helm get values rhdh -n "${namespace}" 2>&1 | grep -A20 -E 'postgresql:|^ image:' || true + fi + } >&2 +} + +# Wait until PostgreSQL is Ready and (optionally) running an expected image substring. +# Args: +# $1 - namespace +# $2 - optional max wait seconds (default: 600) +# $3 - optional image substring that must appear in the container image +# Prints the pod name on success; returns 1 on timeout. +wait_for_postgres_ready() { + local namespace=$1 + local max_wait=${2:-600} + local expected_image_substr=${3:-} + local pg_pod="" + local waited=0 + + if [[ -n "${expected_image_substr}" ]]; then + log::info "Waiting for PostgreSQL Ready with image containing '${expected_image_substr}' in ${namespace}" + else + log::info "Waiting for PostgreSQL pod to be Ready in namespace: ${namespace}" + fi + + while [[ $waited -lt $max_wait ]]; do + pg_pod=$(_postgres_find_pod "${namespace}") + if [[ -n "$pg_pod" ]]; then + local phase ready image sts_ready + phase=$(oc get pod -n "${namespace}" "${pg_pod}" -o jsonpath='{.status.phase}' 2> /dev/null || true) + ready=$(oc get pod -n "${namespace}" "${pg_pod}" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2> /dev/null || true) + image=$(oc get pod -n "${namespace}" "${pg_pod}" -o jsonpath='{.spec.containers[0].image}' 2> /dev/null || true) + sts_ready=$(_postgres_sts_ready_replicas "${namespace}") + + # Periodic progress on INFO so CI logs show what we are waiting for. + if ((waited % 60 == 0)); then + log::info "Still waiting… pod=${pg_pod:-none} phase=${phase:-?} ready=${ready:-?} sts_ready=${sts_ready:-0} image=${image:-?} (${waited}s/${max_wait}s)" + fi + + if [[ -n "${expected_image_substr}" && "${image}" != *"${expected_image_substr}"* ]]; then + : + elif [[ "$phase" == "Running" && "$ready" == "True" && "${sts_ready:-0}" -ge 1 ]]; then + log::info "PostgreSQL pod is Ready: ${pg_pod} (image=${image})" + echo "${pg_pod}" + return 0 + fi + elif ((waited % 60 == 0)); then + log::info "Still waiting… no postgresql pod yet (${waited}s/${max_wait}s)" + oc get pods -n "${namespace}" -o wide >&2 || true + fi + sleep 10 + waited=$((waited + 10)) + done + + log::error "PostgreSQL pod not Ready in namespace ${namespace} after ${max_wait}s" + dump_postgres_diagnostics "${namespace}" + return 1 +} + +# Log the running PostgreSQL server version (for upgrade evidence). +log_postgres_version() { + local namespace=$1 + local pg_pod=${2:-} + + if [[ -z "$pg_pod" ]]; then + pg_pod=$(_postgres_find_pod "${namespace}") + fi + if [[ -z "$pg_pod" ]]; then + log::warn "Cannot log PostgreSQL version: no pod found in ${namespace}" + return 0 + fi + + local version + version=$(oc exec -n "${namespace}" "${pg_pod}" -- psql -U postgres -t -A -c "SHOW server_version;" 2> /dev/null | tr -d '[:space:]' || true) + local image + image=$(oc get pod -n "${namespace}" "${pg_pod}" -o jsonpath='{.spec.containers[0].image}' 2> /dev/null || true) + log::info "PostgreSQL evidence — version='${version:-unknown}' image='${image:-unknown}' pod='${pg_pod}'" +} + +# Refresh PostgreSQL collation versions after a major version upgrade. +# Args: +# $1 - namespace +# $2 - optional max wait seconds (default: 600) +# $3 - optional expected image substring (e.g. postgresql-18) +refresh_postgres_collation_versions() { + local namespace=$1 + local max_wait=${2:-600} + local expected_image_substr=${3:-} + + log::info "Refreshing PostgreSQL collation versions in namespace: ${namespace}" + + local pg_pod + if ! pg_pod=$(wait_for_postgres_ready "${namespace}" "${max_wait}" "${expected_image_substr}"); then + log::warn "Skipping collation refresh; PostgreSQL not Ready." + return 0 + fi + + log::info "Found PostgreSQL pod: ${pg_pod}" + + # Keep stderr separate so WARNING/DETAIL/HINT lines are not treated as DB names. + local databases + databases=$(oc exec -n "${namespace}" "${pg_pod}" -- psql -U postgres -t -A -c \ + "SELECT datname FROM pg_database WHERE datistemplate = false AND datname NOT IN ('postgres');" 2> /dev/null \ + | grep -E '^[A-Za-z0-9_.:-]+$' || true) + log::info "User databases for collation refresh: ${databases:-}" + + local db out + for db in postgres template1 $databases; do + if [[ -z "$db" ]]; then + continue + fi + log::info "Refreshing collation version for database: ${db}" + out=$(oc exec -n "${namespace}" "${pg_pod}" -- psql -U postgres -v ON_ERROR_STOP=1 -c \ + "ALTER DATABASE \"${db}\" REFRESH COLLATION VERSION;" 2>&1) \ + && log::debug "${out}" \ + || log::warn "Failed to refresh collation for ${db}: ${out}" + done + + log::info "Collation version refresh completed for namespace: ${namespace}" +} + +# Dump all databases from the chart-managed PostgreSQL (for dump/restore upgrades). +# Args: +# $1 - namespace +# $2 - output file path +postgres_dumpall_to_file() { + local namespace=$1 + local outfile=$2 + local pg_pod + + if ! pg_pod=$(wait_for_postgres_ready "${namespace}" 300); then + log::error "Cannot dumpall: PostgreSQL not Ready in ${namespace}" + return 1 + fi + + log::info "Running pg_dumpall from ${pg_pod} → ${outfile}" + mkdir -p "$(dirname "${outfile}")" + if ! oc exec -n "${namespace}" "${pg_pod}" -- pg_dumpall -U postgres > "${outfile}"; then + log::error "pg_dumpall failed" + return 1 + fi + log::info "pg_dumpall wrote $(wc -c < "${outfile}" | tr -d ' ') bytes" +} + +# Wipe the chart-managed Postgres PVC so the next helm install can initdb fresh. +# Args: +# $1 - namespace +postgres_wipe_persistent_volume() { + local namespace=$1 + local cr pvc + + log::info "Wiping PostgreSQL StatefulSet + PVC in ${namespace} for dump/restore upgrade (INSTALL_METHOD=${INSTALL_METHOD:-helm})" + if [[ "${INSTALL_METHOD:-helm}" == "operator" ]]; then + cr=$(_postgres_cr_name) + # Keep backstage-psql-secret-${cr} so RHDH credentials remain valid after restore. + oc delete statefulset -n "${namespace}" "backstage-psql-${cr}" --wait=true --timeout=5m 2>&1 || true + oc delete pod -n "${namespace}" -l "rhdh.redhat.com/app=backstage-psql-${cr}" --force --grace-period=0 2>&1 || true + pvc=$(oc get pvc -n "${namespace}" -o name 2> /dev/null | grep -i "backstage-psql-${cr}" | head -1 || true) + if [[ -z "${pvc}" ]]; then + pvc=$(oc get pvc -n "${namespace}" -o name 2> /dev/null | grep -i postgres | head -1 || true) + fi + if [[ -n "${pvc}" ]]; then + log::info "Deleting ${pvc}" + oc delete -n "${namespace}" "${pvc}" --wait=true --timeout=5m 2>&1 || true + else + log::warn "No postgres PVC found to delete" + fi + # Nudge the Backstage CR so the operator recreates the STS with the new RELATED_IMAGE. + oc annotate backstage "${cr}" -n "${namespace}" "rhdh.redhat.com/pg-upgrade-reconcile=$(date +%s)" --overwrite 2>&1 || true + else + oc delete statefulset -n "${namespace}" -l "app.kubernetes.io/name=postgresql" --wait=true --timeout=5m 2>&1 || true + oc delete pod -n "${namespace}" -l "app.kubernetes.io/name=postgresql" --force --grace-period=0 2>&1 || true + pvc=$(oc get pvc -n "${namespace}" -o name 2> /dev/null | grep -i postgres | head -1 || true) + if [[ -n "${pvc}" ]]; then + log::info "Deleting ${pvc}" + oc delete -n "${namespace}" "${pvc}" --wait=true --timeout=5m 2>&1 || true + else + log::warn "No postgres PVC found to delete" + fi + fi +} + +# Restore a pg_dumpall file into a Ready PostgreSQL pod. +# Args: +# $1 - namespace +# $2 - dump file path +# $3 - optional expected image substring +postgres_restore_dumpall_file() { + local namespace=$1 + local dumpfile=$2 + local expected_image_substr=${3:-} + local pg_pod + + if ! pg_pod=$(wait_for_postgres_ready "${namespace}" 600 "${expected_image_substr}"); then + log::error "Cannot restore: PostgreSQL not Ready in ${namespace}" + return 1 + fi + + log::info "Restoring pg_dumpall into ${pg_pod} from ${dumpfile}" + # Ignore benign "already exists" errors from globals/roles created by image init. + if ! oc exec -i -n "${namespace}" "${pg_pod}" -- psql -U postgres -v ON_ERROR_STOP=0 < "${dumpfile}"; then + log::error "psql restore failed" + return 1 + fi + log::info "pg_dumpall restore completed" +} + +# Return the UID of a Running RHDH (developer-hub) pod for the release, or empty. +# Args: +# $1 - release name (e.g. rhdh) +# $2 - namespace +get_rhdh_pod_uid() { + local release_name=$1 + local namespace=$2 + local name uid + local deploy + deploy=$(rhdh_deployment_name "${release_name}") + + while IFS=$'\t' read -r name uid; do + if [[ -n "${name}" && "${name}" == "${deploy}"* && -n "${uid}" ]]; then + echo "${uid}" + return 0 + fi + done < <(oc get pods -n "${namespace}" --field-selector=status.phase=Running \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.uid}{"\n"}{end}' 2> /dev/null || true) + + return 0 +} + +# Wait until the previous RHDH pod UID is gone and a different Ready pod exists. +# Args: +# $1 - release name +# $2 - namespace +# $3 - previous pod UID +# $4 - optional max wait seconds (default: 600) +wait_for_rhdh_pod_replaced() { + local release_name=$1 + local namespace=$2 + local previous_uid=$3 + local max_wait=${4:-600} + local waited=0 + local deploy + deploy=$(rhdh_deployment_name "${release_name}") + + if [[ -z "${previous_uid}" ]]; then + log::warn "No previous RHDH pod UID provided; skipping pod-replacement wait" + return 0 + fi + + log::info "Waiting for previous RHDH pod (uid=${previous_uid}) to terminate and a new Ready pod for ${deploy}" + + while [[ $waited -lt $max_wait ]]; do + local uid_still_present="false" + local uids + uids=$(oc get pods -n "${namespace}" -o jsonpath='{range .items[*]}{.metadata.uid}{"\n"}{end}' 2> /dev/null || true) + if grep -qx "${previous_uid}" <<< "${uids}"; then + uid_still_present="true" + fi + + local new_name="" new_uid="" phase="" ready="" + while IFS=$'\t' read -r new_name new_uid phase ready; do + if [[ -n "${new_name}" && "${new_name}" == "${deploy}"* ]]; then + break + fi + new_name="" + new_uid="" + done < <(oc get pods -n "${namespace}" \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.uid}{"\t"}{.status.phase}{"\t"}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' 2> /dev/null || true) + + if [[ "${uid_still_present}" != "true" && -n "${new_uid}" && "${new_uid}" != "${previous_uid}" && "${phase}" == "Running" && "${ready}" == "True" ]]; then + log::info "New RHDH pod is Ready: name=${new_name} uid=${new_uid} (previous uid=${previous_uid} terminated)" + echo "${new_uid}" + return 0 + fi + + if ((waited % 30 == 0)); then + log::info "Still waiting for RHDH pod replacement… previous_present=${uid_still_present} new=${new_name:-none} phase=${phase:-?} ready=${ready:-?} (${waited}s/${max_wait}s)" + oc get pods -n "${namespace}" -o wide >&2 | grep -E "${deploy}|NAME" || true + fi + + sleep 5 + waited=$((waited + 5)) + done + + log::error "Timed out waiting for RHDH pod replacement (previous uid=${previous_uid})" + oc get pods -n "${namespace}" -o wide >&2 || true + return 1 +} + +# Ensure RHDH has rolled onto a new pod after a DB upgrade hop. +# Restarts the deployment when the old UID is still current, then waits for replacement. +# Args: +# $1 - release name +# $2 - namespace +# $3 - previous pod UID +# $4 - optional max wait seconds (default: 600) +ensure_rhdh_pod_replaced() { + local release_name=$1 + local namespace=$2 + local previous_uid=$3 + local max_wait=${4:-600} + local deploy + local current_uid + deploy=$(rhdh_deployment_name "${release_name}") + + if [[ -z "${previous_uid}" ]]; then + return 0 + fi + + current_uid=$(get_rhdh_pod_uid "${release_name}" "${namespace}") + if [[ "${current_uid}" == "${previous_uid}" ]]; then + log::info "RHDH pod uid unchanged after DB upgrade (${current_uid}); restarting ${deploy} so verification hits a new pod" + oc rollout restart "deployment/${deploy}" -n "${namespace}" || true + else + log::info "RHDH pod uid already changed (${previous_uid} → ${current_uid:-none}); waiting for previous pod to terminate" + fi + + wait_for_rhdh_pod_replaced "${release_name}" "${namespace}" "${previous_uid}" "${max_wait}" > /dev/null +} + +# Resolve a cluster-local runtime for the data-proof static server. +# Prints: imagecommandscriptPath +_pg_upgrade_data_proof_runtime() { + local python_tag nodejs_tag + + python_tag=$(oc get imagestream python -n openshift -o jsonpath='{.spec.tags[*].name}' 2> /dev/null \ + | tr ' ' '\n' | grep -E '^[0-9]+\.[0-9]+-ubi[0-9]+$' | sort -V | tail -1 || true) + if [[ -n "${python_tag}" ]]; then + printf '%s\t%s\t%s\n' \ + "image-registry.openshift-image-registry.svc:5000/openshift/python:${python_tag}" \ + "python3" \ + "/www/serve.py" + return 0 + fi + + nodejs_tag=$(oc get imagestream nodejs -n openshift -o jsonpath='{.spec.tags[*].name}' 2> /dev/null \ + | tr ' ' '\n' | grep -E '^[0-9]+-ubi[0-9]+$' | sort -t'-' -k1 -n | tail -1 || true) + nodejs_tag="${nodejs_tag:-18-ubi8}" + printf '%s\t%s\t%s\n' \ + "image-registry.openshift-image-registry.svc:5000/openshift/nodejs:${nodejs_tag}" \ + "node" \ + "/www/serve.js" +} + +# Deploy an in-cluster static server that hosts the persistence-proof catalog entity. +# Args: +# $1 - namespace +deploy_pg_upgrade_data_proof_fixture() { + local namespace=$1 + local manifest="${DIR}/resources/pg-upgrade/data-proof-server.yaml" + local runtime image cmd script + local rendered="/tmp/pg-upgrade-data-proof-server.yaml" + + if [[ ! -f "${manifest}" ]]; then + log::error "Missing data-proof fixture manifest: ${manifest}" + return 1 + fi + + runtime=$(_pg_upgrade_data_proof_runtime) + image=$(cut -f1 <<< "${runtime}") + cmd=$(cut -f2 <<< "${runtime}") + script=$(cut -f3 <<< "${runtime}") + + log::info "Deploying pg-upgrade data-proof fixture in ${namespace} using ${image} (${cmd} ${script})" + + sed \ + -e "s|PG_UPGRADE_DATA_PROOF_IMAGE|${image}|g" \ + -e "s|PG_UPGRADE_DATA_PROOF_COMMAND|${cmd}|g" \ + -e "s|PG_UPGRADE_DATA_PROOF_SCRIPT|${script}|g" \ + "${manifest}" > "${rendered}" + + oc apply -n "${namespace}" -f "${rendered}" + if ! oc rollout status "deployment/pg-upgrade-data-proof" -n "${namespace}" --timeout=3m; then + log::error "pg-upgrade data-proof fixture failed to become Ready" + oc get pods,svc -n "${namespace}" -l app=pg-upgrade-data-proof -o wide >&2 || true + oc describe deployment/pg-upgrade-data-proof -n "${namespace}" >&2 || true + oc logs -n "${namespace}" deploy/pg-upgrade-data-proof --tail=50 >&2 || true + return 1 + fi +} + +# Resolve a publicly fetchable catalog-info URL for the persistence-proof entity. +# Prefer an explicit override, else GitHub blob URL at the PR/head SHA (Backstage +# blocks most in-cluster http://*.svc targets via URL reader / SSRF guards). +pg_upgrade_data_proof_target_url() { + if [[ -n "${PG_UPGRADE_PROOF_URL:-}" ]]; then + echo "${PG_UPGRADE_PROOF_URL}" + return 0 + fi + + local sha="${PULL_PULL_SHA:-${PULL_BASE_SHA:-}}" + if [[ -z "${sha}" ]]; then + sha=$(git -C "${DIR}/../.." rev-parse HEAD 2> /dev/null || true) + fi + local repo="${PG_UPGRADE_PROOF_REPO:-zdrapela/rhdh}" + if [[ -z "${sha}" ]]; then + log::error "Cannot resolve pg-upgrade proof catalog URL (set PG_UPGRADE_PROOF_URL or PULL_PULL_SHA)" + return 1 + fi + echo "https://github.com/${repo}/blob/${sha}/.ci/pipelines/resources/pg-upgrade/catalog-info.yaml" +} + +# Register the persistence-proof Component via the Catalog Locations API and wait until it is queryable. +# Args: +# $1 - Backstage base URL +# $2 - namespace (unused; kept for call-site compatibility) +seed_pg_upgrade_data_proof() { + local base_url=$1 + local _namespace=$2 + local target + local entity_url="${base_url}/api/catalog/entities/by-name/component/default/pg-upgrade-data-proof" + local locations_url="${base_url}/api/catalog/locations" + local max_wait=${3:-180} + local waited=0 + local status body + + if ! target=$(pg_upgrade_data_proof_target_url); then + return 1 + fi + + log::info "Seeding pg-upgrade data proof location: ${target}" + + status=$(curl --insecure -s -o /tmp/pg-upgrade-seed-location.json -w "%{http_code}" \ + -X POST "${locations_url}" \ + -H "Content-Type: application/json" \ + -d "{\"type\":\"url\",\"target\":\"${target}\"}" || echo "000") + + if [[ "${status}" != "201" && "${status}" != "409" ]]; then + log::error "Failed to register data-proof location (HTTP ${status})" + cat /tmp/pg-upgrade-seed-location.json >&2 || true + return 1 + fi + log::info "Location register response HTTP ${status}" + + log::info "Waiting for entity pg-upgrade-data-proof to appear in catalog" + while [[ $waited -lt $max_wait ]]; do + status=$(curl --insecure -s -o /tmp/pg-upgrade-seed-entity.json -w "%{http_code}" "${entity_url}" || echo "000") + if [[ "${status}" == "200" ]]; then + body=$(cat /tmp/pg-upgrade-seed-entity.json) + if grep -q "RHIDP-14594" <<< "${body}" && grep -q "pg-upgrade-data-proof" <<< "${body}"; then + log::info "Seeded catalog entity is present (RHIDP-14594 persistence proof)" + return 0 + fi + fi + if ((waited % 30 == 0)); then + log::info "Still waiting for seeded entity… HTTP ${status} (${waited}s/${max_wait}s)" + fi + sleep 5 + waited=$((waited + 5)) + done + + log::error "Timed out waiting for seeded entity pg-upgrade-data-proof" + cat /tmp/pg-upgrade-seed-entity.json >&2 || true + curl --insecure -s "${locations_url}" >&2 || true + return 1 +} + +# Log API evidence that the persistence-proof entity still exists (post-upgrade). +# Args: +# $1 - Backstage base URL +assert_pg_upgrade_data_proof_api() { + local base_url=$1 + local entity_url="${base_url}/api/catalog/entities/by-name/component/default/pg-upgrade-data-proof" + local status body + + status=$(curl --insecure -s -o /tmp/pg-upgrade-verify-entity.json -w "%{http_code}" "${entity_url}" || echo "000") + body=$(cat /tmp/pg-upgrade-verify-entity.json 2> /dev/null || true) + if [[ "${status}" != "200" ]] || ! grep -q "RHIDP-14594" <<< "${body}"; then + log::error "Persistence proof entity missing or incomplete via API (HTTP ${status})" + echo "${body}" >&2 + return 1 + fi + log::info "API persistence proof OK: component/default/pg-upgrade-data-proof still present after upgrade" +} diff --git a/.ci/pipelines/resources/pg-upgrade/catalog-info.yaml b/.ci/pipelines/resources/pg-upgrade/catalog-info.yaml new file mode 100644 index 0000000000..b3883270c0 --- /dev/null +++ b/.ci/pipelines/resources/pg-upgrade/catalog-info.yaml @@ -0,0 +1,13 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: pg-upgrade-data-proof + title: PG Upgrade Data Proof + description: RHIDP-14594 persistence proof seeded before PostgreSQL major upgrade + tags: + - pg-upgrade-proof + - rhidp-14594 +spec: + type: service + lifecycle: experimental + owner: guests diff --git a/.ci/pipelines/resources/pg-upgrade/data-proof-server.yaml b/.ci/pipelines/resources/pg-upgrade/data-proof-server.yaml new file mode 100644 index 0000000000..3ecccee642 --- /dev/null +++ b/.ci/pipelines/resources/pg-upgrade/data-proof-server.yaml @@ -0,0 +1,104 @@ +# Serves catalog-info.yaml in-cluster so Backstage can register a location +# without depending on GitHub raw URLs from fork PRs. +# Image/command placeholders are substituted by deploy_pg_upgrade_data_proof_fixture(). +apiVersion: v1 +kind: ConfigMap +metadata: + name: pg-upgrade-data-proof + labels: + app: pg-upgrade-data-proof +data: + catalog-info.yaml: | + apiVersion: backstage.io/v1alpha1 + kind: Component + metadata: + name: pg-upgrade-data-proof + title: PG Upgrade Data Proof + description: RHIDP-14594 persistence proof seeded before PostgreSQL major upgrade + tags: + - pg-upgrade-proof + - rhidp-14594 + spec: + type: service + lifecycle: experimental + owner: guests + serve.py: | + #!/usr/bin/env python3 + from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer + import os + os.chdir("/www") + ThreadingHTTPServer(("0.0.0.0", 8080), SimpleHTTPRequestHandler).serve_forever() + serve.js: | + const http = require("http"); + const fs = require("fs"); + const path = require("path"); + http.createServer((req, res) => { + const file = path.join("/www", req.url === "/" ? "catalog-info.yaml" : req.url); + fs.readFile(file, (err, data) => { + if (err) { res.writeHead(404); res.end(String(err)); return; } + res.writeHead(200, { "Content-Type": "text/yaml" }); + res.end(data); + }); + }).listen(8080, "0.0.0.0"); +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pg-upgrade-data-proof + labels: + app: pg-upgrade-data-proof +spec: + replicas: 1 + selector: + matchLabels: + app: pg-upgrade-data-proof + template: + metadata: + labels: + app: pg-upgrade-data-proof + spec: + containers: + - name: http + image: PG_UPGRADE_DATA_PROOF_IMAGE + imagePullPolicy: IfNotPresent + command: + - PG_UPGRADE_DATA_PROOF_COMMAND + args: + - PG_UPGRADE_DATA_PROOF_SCRIPT + ports: + - containerPort: 8080 + name: http + volumeMounts: + - name: www + mountPath: /www + readOnly: true + readinessProbe: + tcpSocket: + port: 8080 + initialDelaySeconds: 2 + periodSeconds: 5 + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + memory: 128Mi + volumes: + - name: www + configMap: + name: pg-upgrade-data-proof + defaultMode: 0555 +--- +apiVersion: v1 +kind: Service +metadata: + name: pg-upgrade-data-proof + labels: + app: pg-upgrade-data-proof +spec: + selector: + app: pg-upgrade-data-proof + ports: + - name: http + port: 8080 + targetPort: 8080 diff --git a/.ci/pipelines/resources/rhdh-operator/rhdh-start.yaml b/.ci/pipelines/resources/rhdh-operator/rhdh-start.yaml index f97aa1afb7..3bafb6ace8 100644 --- a/.ci/pipelines/resources/rhdh-operator/rhdh-start.yaml +++ b/.ci/pipelines/resources/rhdh-operator/rhdh-start.yaml @@ -3,6 +3,9 @@ apiVersion: rhdh.redhat.com/v1alpha5 metadata: name: rhdh spec: + # Disable default flavours (e.g. lightspeed) — this evidence job only needs + # local Postgres + showcase plugins; lightspeed OCI digests have been flaky in CI. + flavours: [] deployment: patch: spec: diff --git a/e2e-tests/playwright/e2e/pg-upgrade-data-proof.spec.ts b/e2e-tests/playwright/e2e/pg-upgrade-data-proof.spec.ts new file mode 100644 index 0000000000..b0b7ce7320 --- /dev/null +++ b/e2e-tests/playwright/e2e/pg-upgrade-data-proof.spec.ts @@ -0,0 +1,47 @@ +import { expect } from "@playwright/test"; +import { test } from "@support/coverage/test"; + +import { getTranslations, getCurrentLanguage } from "../e2e/localization/locale"; +import { CatalogBrowsePage } from "../support/pages/catalog-browse-page"; + +const t = getTranslations(); +const lang = getCurrentLanguage(); + +/** + * RHIDP-14594: prove catalog data seeded on PG15 survives major upgrades. + * Enabled only when CI sets PG_UPGRADE_DATA_PROOF=1 (see ocp-pull.sh / ocp-operator.sh). + */ +test.describe("PostgreSQL upgrade data persistence proof", () => { + test.skip( + () => process.env.PG_UPGRADE_DATA_PROOF !== "1", + "Runs only during chart-managed or operator-local PostgreSQL upgrade CI (PG_UPGRADE_DATA_PROOF=1)", + ); + + let catalogBrowsePage: CatalogBrowsePage; + + test.beforeAll(({ rhdhGuestPage }) => { + test.info().annotations.push({ + type: "component", + description: "core", + }); + catalogBrowsePage = new CatalogBrowsePage(rhdhGuestPage); + }); + + test("Seeded pg-upgrade-data-proof component is visible in Catalog UI", async ({ + rhdhGuestPage, + }) => { + // Catalog table shows metadata.title when set ("PG Upgrade Data Proof"), + // not metadata.name — match the same navigation pattern as catalog-timestamp. + await catalogBrowsePage.openSidebar(t["rhdh"][lang]["menuItem.catalog"]); + await catalogBrowsePage.verifyHeading( + t["catalog"][lang]["indexPage.title"].replace("{{orgName}}", "My Org"), + ); + await catalogBrowsePage.openCatalogSidebar("Component"); + await catalogBrowsePage.searchCatalog("PG Upgrade Data Proof"); + await catalogBrowsePage.verifyText("PG Upgrade Data Proof"); + + await catalogBrowsePage.openEntityLink("PG Upgrade Data Proof"); + await catalogBrowsePage.verifyHeading("PG Upgrade Data Proof"); + await expect(rhdhGuestPage.getByText(/RHIDP-14594 persistence proof/iu)).toBeVisible(); + }); +});