diff --git a/.ci/pipelines/jobs/ocp-disconnected-helm.sh b/.ci/pipelines/jobs/ocp-disconnected-helm.sh new file mode 100644 index 0000000000..768a1916e4 --- /dev/null +++ b/.ci/pipelines/jobs/ocp-disconnected-helm.sh @@ -0,0 +1,209 @@ +#!/bin/bash + +# shellcheck source=.ci/pipelines/lib/log.sh +source "$DIR"/lib/log.sh +# shellcheck source=.ci/pipelines/lib/common.sh +source "$DIR"/lib/common.sh +# shellcheck source=.ci/pipelines/utils.sh +source "$DIR"/utils.sh +# shellcheck source=.ci/pipelines/lib/testing.sh +source "$DIR"/lib/testing.sh +# shellcheck source=.ci/pipelines/playwright-projects.sh +source "$DIR"/playwright-projects.sh +# shellcheck source=.ci/pipelines/lib/disconnected.sh +source "$DIR"/lib/disconnected.sh + +export INSTALL_METHOD="helm" + +handle_ocp_disconnected_helm() { + export NAME_SPACE="${NAME_SPACE:-showcase-ci-disconnected}" + + disconnected::require_env + disconnected::setup_auth + + common::oc_login + + K8S_CLUSTER_ROUTER_BASE=$(oc get route console -n openshift-console -o=jsonpath='{.spec.host}' | sed 's/^[^.]*\.//') + export K8S_CLUSTER_ROUTER_BASE + + log::section "Chart Resolution" + + local is_ga="false" + if [[ "${IMAGE_REGISTRY}" == "registry.redhat.io" ]]; then + is_ga="true" + fi + + if [[ "${is_ga}" == "true" ]]; then + helm repo add openshift-helm-charts https://charts.openshift.io 2> /dev/null || true + helm repo update openshift-helm-charts + log::info "Pulling GA chart from charts.openshift.io (version: ${RELEASE_VERSION})" + helm pull openshift-helm-charts/redhat-developer-hub \ + --version "${RELEASE_VERSION}" \ + -d "${DISCONNECTED_TMPDIR}" || { + log::error "Failed to pull chart from charts.openshift.io" + return 1 + } + else + log::info "Pulling CI chart from ${HELM_CHART_URL} (version: ${CHART_VERSION})" + helm pull "${HELM_CHART_URL}" --version "${CHART_VERSION}" \ + -d "${DISCONNECTED_TMPDIR}" || { + log::error "Failed to pull chart from ${HELM_CHART_URL}" + return 1 + } + fi + + CHART_LOCAL_TGZ=$(find "${DISCONNECTED_TMPDIR}" -maxdepth 1 -name '*.tgz' | head -1) + export CHART_LOCAL_TGZ + + if [[ -z "${CHART_LOCAL_TGZ}" ]]; then + log::error "No chart .tgz found in ${DISCONNECTED_TMPDIR}" + return 1 + fi + log::success "Chart pulled: ${CHART_LOCAL_TGZ}" + + # Resolve PostgreSQL image from chart values + local helm_values + helm_values=$(helm show values "${CHART_LOCAL_TGZ}" 2> /dev/null || true) + + export PG_REGISTRY PG_REPO PG_TAG PG_SEPARATOR + PG_REGISTRY=$(echo "${helm_values}" | yq '.upstream.postgresql.image.registry' || true) + PG_REPO=$(echo "${helm_values}" | yq '.upstream.postgresql.image.repository' || true) + PG_TAG=$(echo "${helm_values}" | yq '.upstream.postgresql.image.tag' || true) + PG_REGISTRY="${PG_REGISTRY:-registry.redhat.io}" + PG_REPO="${PG_REPO:-rhel9/postgresql-15}" + PG_TAG="${PG_TAG:-latest}" + + # The chart encodes digest refs as repository: "repo@sha256" + tag: "". + # Normalize: extract the digest qualifier into PG_SEPARATOR so that: + # - PG_REPO is always a clean path (usable in IDMS source/mirror fields) + # - Full ref is ${PG_REGISTRY}/${PG_REPO}${PG_SEPARATOR}${PG_TAG} + PG_SEPARATOR=":" + if [[ "${PG_REPO}" == *"@"* ]]; then + PG_SEPARATOR="@${PG_REPO##*@}:" # e.g., "@sha256:" + PG_REPO="${PG_REPO%@*}" # e.g., "rhel9/postgresql-15" + fi + + log::info "PostgreSQL image from chart: ${PG_REGISTRY}/${PG_REPO}${PG_SEPARATOR}${PG_TAG}" + + # Resolve catalog index image (same digest encoding as PG). + # The init container pulls this by digest; it must be in the mirror. + export CI_REGISTRY CI_REPO CI_TAG CI_SEPARATOR + CI_REGISTRY=$(echo "${helm_values}" | yq '.global.catalogIndex.image.registry' || true) + CI_REPO=$(echo "${helm_values}" | yq '.global.catalogIndex.image.repository' || true) + CI_TAG=$(echo "${helm_values}" | yq '.global.catalogIndex.image.tag' || true) + CI_REGISTRY="${CI_REGISTRY:-quay.io}" + CI_REPO="${CI_REPO:-rhdh/plugin-catalog-index}" + CI_TAG="${CI_TAG:-latest}" + CI_SEPARATOR=":" + if [[ "${CI_REPO}" == *"@"* ]]; then + CI_SEPARATOR="@${CI_REPO##*@}:" + CI_REPO="${CI_REPO%@*}" + fi + log::info "Catalog index from chart: ${CI_REGISTRY}/${CI_REPO}${CI_SEPARATOR}${CI_TAG}" + + echo "${helm_values}" > "${ARTIFACT_DIR}/disconnected-helm-chart-values.yaml" 2> /dev/null || true + + log::section "Image Mirroring" + + local imageset_config="${DISCONNECTED_TMPDIR}/imageset-config.yaml" + disconnected::build_imageset_config "${imageset_config}" || { + log::error "Failed to build ImageSetConfiguration" + return 1 + } + + local workspace="${DISCONNECTED_TMPDIR}/oc-mirror-workspace" + disconnected::run_oc_mirror "${imageset_config}" "${workspace}" || { + log::error "oc-mirror failed — aborting" + return 1 + } + + log::section "Cluster Resources" + + disconnected::patch_idms "${OC_MIRROR_IDMS_FILE}" + + oc apply -f "${OC_MIRROR_IDMS_FILE}" || { + log::error "Failed to apply IDMS — aborting" + return 1 + } + log::success "ImageDigestMirrorSet applied" + + if [[ -n "${OC_MIRROR_ITMS_FILE:-}" ]]; then + oc apply -f "${OC_MIRROR_ITMS_FILE}" || { + log::error "Failed to apply ITMS — aborting" + return 1 + } + log::success "ImageTagMirrorSet applied" + fi + + # IDMS/ITMS changes trigger a MachineConfig update which rolls worker nodes + # (drain → apply config → reboot). Wait for all MachineConfigPools to finish + # before deploying workloads, otherwise pods get evicted mid-startup. + disconnected::wait_mcp_updated + + log::section "Plugin Mirroring" + disconnected::mirror_plugins || return 1 + + log::section "Namespace and Secrets" + + namespace::configure "${NAME_SPACE}" + disconnected::apply_plugin_mirror_configmap "${NAME_SPACE}" || return 1 + + # Mirror CA + registry auth for install-dynamic-plugins (skopeo). + # Post-renderer mounts CA at /etc/containers/certs.d//ca.crt; + # the chart mounts ${RELEASE_NAME}-dynamic-plugins-registry-auth at + # /opt/app-root/src/.config/containers. + disconnected::create_mirror_registry_ca_configmap "${NAME_SPACE}" || return 1 + disconnected::create_plugin_registry_auth_secret "${NAME_SPACE}" || return 1 + + log::section "Helm Deployment" + + # Prefer the chart from oc-mirror workspace, fall back to the pulled tgz + local chart_install_path + chart_install_path="${OC_MIRROR_CHART_PATH:-${CHART_LOCAL_TGZ}}" + log::info "Installing chart from: ${chart_install_path}" + + local helm_set_flags=( + --set global.clusterRouterBase="${K8S_CLUSTER_ROUTER_BASE}" + --set upstream.backstage.image.registry="${MIRROR_REGISTRY_URL}" + --set upstream.backstage.image.repository="${IMAGE_REPO}" + --set upstream.backstage.image.tag="${TAG_NAME}" + --set upstream.postgresql.image.registry="${MIRROR_REGISTRY_URL}" + ) + + if [[ -n "${CATALOG_INDEX_IMAGE:-}" ]]; then + helm_set_flags+=( + --set global.catalogIndex.image.registry="${MIRROR_REGISTRY_URL}" + --set global.catalogIndex.image.repository="${CATALOG_INDEX_REPO}" + --set global.catalogIndex.image.tag="${CATALOG_INDEX_TAG}" + ) + fi + + # Post-renderer appends disconnected volumes (registries.conf, mirror CA) + # to the rendered Deployment, avoiding the Helm "array clobber" pitfall. + local post_renderer="${DIR}/resources/disconnected/helm-post-renderer.sh" + + helm upgrade -i "${RELEASE_NAME}" -n "${NAME_SPACE}" \ + "${chart_install_path}" \ + -f "${DIR}/value_files/values_disconnected-smoke.yaml" \ + --post-renderer "${post_renderer}" \ + --post-renderer-args "${MIRROR_REGISTRY_URL}" \ + "${helm_set_flags[@]}" || { + log::error "Helm deployment failed" + return 1 + } + + log::success "RHDH deployed via Helm with mirrored images" + + printf '%s\n' "${helm_set_flags[@]}" > "${ARTIFACT_DIR}/disconnected-helm-set-flags.txt" 2> /dev/null || true + + log::section "Smoke Test" + + if [[ -n "${HTTPS_PROXY:-}" ]]; then + log::info "HTTPS_PROXY is set (Playwright will use it): ${HTTPS_PROXY%%@*}@***" + fi + + local url="https://${RELEASE_NAME}-developer-hub-${NAME_SPACE}.${K8S_CLUSTER_ROUTER_BASE}" + testing::check_and_test "${RELEASE_NAME}" "${NAME_SPACE}" "${PW_PROJECT_SMOKE_TEST}" "${url}" + + log::success "Disconnected Helm smoke test completed" +} diff --git a/.ci/pipelines/jobs/ocp-disconnected-operator.sh b/.ci/pipelines/jobs/ocp-disconnected-operator.sh new file mode 100644 index 0000000000..bca3aa52b4 --- /dev/null +++ b/.ci/pipelines/jobs/ocp-disconnected-operator.sh @@ -0,0 +1,261 @@ +#!/bin/bash + +# shellcheck source=.ci/pipelines/lib/log.sh +source "$DIR"/lib/log.sh +# shellcheck source=.ci/pipelines/lib/common.sh +source "$DIR"/lib/common.sh +# shellcheck source=.ci/pipelines/utils.sh +source "$DIR"/utils.sh +# shellcheck source=.ci/pipelines/install-methods/operator.sh +source "$DIR"/install-methods/operator.sh +# shellcheck source=.ci/pipelines/lib/testing.sh +source "$DIR"/lib/testing.sh +# shellcheck source=.ci/pipelines/playwright-projects.sh +source "$DIR"/playwright-projects.sh +# shellcheck source=.ci/pipelines/lib/disconnected.sh +source "$DIR"/lib/disconnected.sh + +export INSTALL_METHOD="operator" + +handle_ocp_disconnected_operator() { + # Force a dedicated namespace (env_variables.sh may already set NAME_SPACE=showcase). + export NAME_SPACE="showcase-disconnected" + + disconnected::require_env + disconnected::setup_auth + + common::oc_login + + K8S_CLUSTER_ROUTER_BASE=$(oc get route console -n openshift-console -o=jsonpath='{.spec.host}' | sed 's/^[^.]*\.//') + export K8S_CLUSTER_ROUTER_BASE + + # Uses prepare-restricted-environment.sh from rhdh-operator, which handles + # mirroring operator/operand images and installing the operator CatalogSource. + log::section "Operator Mirroring and Installation" + + # Fetch prepare-restricted-environment.sh from RELEASE_BRANCH_NAME (default + # main). rhdh-operator#3259 (native oc-mirror cc-* catalogs for OLM v1) is + # merged on main. + local prepare_script_path="${DISCONNECTED_TMPDIR}/prepare-restricted-environment.sh" + disconnected::fetch_script "prepare-restricted-environment.sh" \ + "${prepare_script_path}" || { + log::error "Failed to fetch prepare-restricted-environment.sh — aborting" + return 1 + } + + # Use oc-mirror (documented air-gapped OCP path) instead of the script's + # default skopeo/umoci/podman-build path. Nested Podman in this CI pod cannot + # initialize storage (newuidmap / VFS chown both fail under hostUsers: false). + # CATALOG_INDEX_IMAGE is the plugin catalog index — do not pass it as + # --index-image (OLM operator catalog). Keep it for mirror-plugins.sh below. + # + # OLM version: leave default (auto). On OCP 4.21+ this selects OLM v1. + local filter_versions="${RELEASE_VERSION}" + if [[ "${filter_versions}" == "next" || "${filter_versions}" == "*" ]]; then + filter_versions="*" + fi + + # CONTAINER_PLATFORM_VERSION is set by e2e-tests/container-init.sh from the + # live cluster (oc version → major.minor). + local ocp_version="${CONTAINER_PLATFORM_VERSION:-}" + if [[ -z "${ocp_version}" || "${ocp_version}" == "unknown" || ! "${ocp_version}" =~ ^[0-9]+\.[0-9]+$ ]]; then + log::error "CONTAINER_PLATFORM_VERSION is unset or invalid ('${ocp_version}'); expected OpenShift major.minor" + return 1 + fi + local index_image="registry.redhat.io/redhat/redhat-operator-index:v${ocp_version}" + log::info "Using OCP ${ocp_version} (CONTAINER_PLATFORM_VERSION); index image ${index_image}" + + local prepare_args=( + --use-oc-mirror true + --to-registry "${MIRROR_REGISTRY_URL}" + --index-image "${index_image}" + --filter-versions "${filter_versions}" + ) + + # prepare-restricted-environment.sh skips OLM v1 pull-secret/CA setup for + # external registries. Catalogd must trust the mirror CA and authenticate + # before ClusterCatalog can reach Serving=True. + disconnected::ensure_olm_mirror_pull_secret || return 1 + disconnected::ensure_mirror_registry_ca || return 1 + disconnected::wait_mcp_updated + + log::info "Running prepare-restricted-environment.sh with: ${prepare_args[*]}" + if ! disconnected::with_unset_registry_auth_file \ + bash "${prepare_script_path}" "${prepare_args[@]}"; then + log::error "prepare-restricted-environment.sh failed — aborting" + return 1 + fi + log::success "Operator installed via prepare-restricted-environment.sh" + + # prepare patches the operator SA with internal-registry secret names that do + # not exist for an external mirror. Provide a real mirror pull secret and + # attach it to the OLM v1 installer SA used by ClusterExtension. + local operator_ns="rhdh-operator" + oc create secret generic reg-pull-secret \ + --from-file=.dockerconfigjson="${MIRROR_REGISTRY_PULL_SECRET}" \ + --type=kubernetes.io/dockerconfigjson \ + -n "${operator_ns}" \ + --dry-run=client -o yaml | oc apply -f - || { + log::error "Failed to create reg-pull-secret in ${operator_ns}" + return 1 + } + oc patch serviceaccount rhdh-operator-installer -n "${operator_ns}" --type=merge \ + -p '{"imagePullSecrets":[{"name":"reg-pull-secret"}]}' || { + log::warn "Failed to patch rhdh-operator-installer imagePullSecrets — continuing" + } + log::success "Configured mirror pull secret on rhdh-operator-installer SA" + + # prepare-restricted-environment.sh applies IDMS/CatalogSource which triggers + # a MachineConfig update and node rolling. Wait for completion before deploying + # workloads, same as the Helm path. + disconnected::wait_mcp_updated + + # prepare only creates the ClusterExtension; wait until OLM v1 installs the + # operator and the Backstage CRD appears (dump status on timeout). + disconnected::wait_operator_crd_olm_v1 "rhdh-operator" "backstages.rhdh.redhat.com" 600 || { + log::error "Backstage CRD not available after operator installation" + return 1 + } + + log::section "Plugin Mirroring" + disconnected::mirror_plugins || return 1 + + # Homepage ("Welcome back!") comes from dynamic-plugins.default.yaml inside the + # catalog index. Clearing CATALOG_INDEX_IMAGE skips that file and / 404s after + # guest login. Rewrite to the index already pushed by mirror_plugins (same + # path under MIRROR_REGISTRY_URL). Do this AFTER mirroring so mirror_plugins + # still resolves the upstream/index override correctly. + if [[ -n "${CATALOG_INDEX_IMAGE:-}" ]]; then + export CATALOG_INDEX_IMAGE="${MIRROR_REGISTRY_URL}/${CATALOG_INDEX_IMAGE#*/}" + else + export CATALOG_INDEX_IMAGE="${MIRROR_REGISTRY_URL}/rhdh/plugin-catalog-index:${RELEASE_VERSION}" + fi + log::info "Using mirrored CATALOG_INDEX_IMAGE=${CATALOG_INDEX_IMAGE}" + + # Hub must match the catalog index tag (nightly :next). CSV GA hub lacks the + # local homepage path referenced by plugin-catalog-index:next. Mirror the CI + # hub like Helm's oc-mirror additionalImages, digest-pin as HUB_IMAGE, then + # set RELATED_IMAGE_backstage (authoritative for install-dynamic-plugins). + log::section "Hub Image Mirroring" + disconnected::mirror_hub_image || return 1 + disconnected::resolve_hub_image || return 1 + # RELATED_IMAGE_backstage overrides install-dynamic-plugins after CR patches + # (operator getInitContainer clobber). Must run before the Backstage CR. + disconnected::set_operator_related_hub_image "rhdh-operator" || return 1 + + log::section "Namespace and Secrets" + + namespace::configure "${NAME_SPACE}" + disconnected::apply_plugin_mirror_configmap "${NAME_SPACE}" || return 1 + # Same CA/auth secrets as Helm so skopeo in install-dynamic-plugins can pull + # from the mirror (registries.conf alone is not enough — TLS verify fails). + disconnected::create_mirror_registry_ca_configmap "${NAME_SPACE}" || return 1 + disconnected::create_plugin_registry_auth_secret "${NAME_SPACE}" || return 1 + + # Kubelet pull of HUB_IMAGE (CR patch). Cluster pull-secret is already merged; + # namespace secret + imagePullSecrets matches the K8s operator CR pattern. + oc create secret generic mirror-registry-pull \ + --from-file=.dockerconfigjson="${MIRROR_REGISTRY_PULL_SECRET}" \ + --type=kubernetes.io/dockerconfigjson \ + -n "${NAME_SPACE}" \ + --dry-run=client -o yaml | oc apply -f - || { + log::error "Failed to create mirror-registry-pull secret — aborting" + return 1 + } + log::success "Secret mirror-registry-pull created in ${NAME_SPACE}" + + # Operator mounts one volume per extraFiles ConfigMap name. policy.json cannot + # share rhdh-plugin-mirror-conf or reconcile fails with duplicate volume keys. + oc create configmap rhdh-plugin-mirror-policy \ + --from-literal='policy.json={"default":[{"type":"insecureAcceptAnything"}]}' \ + -n "${NAME_SPACE}" \ + --dry-run=client -o yaml | oc apply -f - || { + log::error "Failed to create rhdh-plugin-mirror-policy ConfigMap — aborting" + return 1 + } + log::success "ConfigMap rhdh-plugin-mirror-policy created in ${NAME_SPACE}" + + log::section "Backstage CR Deployment" + + # Minimal guest-auth ConfigMap (full rhdh-start.yaml references ConfigMaps/Secrets + # created by apply_yaml_files(), which this disconnected handler skips). + oc create configmap app-config-rhdh-disconnected-smoke \ + --from-file="app-config-rhdh.yaml=${DIR}/resources/config_map/app-config-rhdh-disconnected-smoke.yaml" \ + --namespace="${NAME_SPACE}" \ + --dry-run=client -o yaml | oc apply -f - || { + log::error "Failed to create app-config ConfigMap — aborting" + return 1 + } + + local rendered_cr + local auth_secret="${RELEASE_NAME}-dynamic-plugins-registry-auth" + # envsubst substitutes HUB_IMAGE into spec.deployment.patch container images. + rendered_cr=$(envsubst < "${DIR}/resources/rhdh-operator/rhdh-start-disconnected-smoke.yaml") + # Parity with helm-post-renderer.sh: registries.conf, policy.json, mirror CA. + # Also mount registry auth.json for authenticated mirror pulls. + # CI yq is mikefarah/yq (no --arg); interpolate paths via the shell. + # shellcheck disable=SC2016 + # Auth: operator secret mounts use subPath and require an existing parent dir. + # /opt/app-root/src/.config/containers is missing in the image, so mount to + # /tmp and point REGISTRY_AUTH_FILE at it (skopeo honors that). + rendered_cr=$(echo "$rendered_cr" | yq eval " + .spec.application.extraEnvs.envs += [ + { + \"name\": \"REGISTRY_AUTH_FILE\", + \"value\": \"/tmp/auth.json\", + \"containers\": [\"install-dynamic-plugins\"] + } + ] | + .spec.application.extraFiles.configMaps = [ + { + \"name\": \"rhdh-plugin-mirror-conf\", + \"key\": \"rhdh-registries.conf\", + \"mountPath\": \"/etc/containers/registries.conf.d\", + \"containers\": [\"install-dynamic-plugins\"] + }, + { + \"name\": \"rhdh-plugin-mirror-policy\", + \"key\": \"policy.json\", + \"mountPath\": \"/etc/containers\", + \"containers\": [\"install-dynamic-plugins\"] + }, + { + \"name\": \"mirror-registry-ca\", + \"key\": \"ca.crt\", + \"mountPath\": \"/etc/containers/certs.d/${MIRROR_REGISTRY_URL}\", + \"containers\": [\"install-dynamic-plugins\"] + } + ] | + .spec.application.extraFiles.secrets = [ + { + \"name\": \"${auth_secret}\", + \"key\": \"auth.json\", + \"mountPath\": \"/tmp\", + \"containers\": [\"install-dynamic-plugins\"] + } + ] + " -) + + local cr_temp="${DISCONNECTED_TMPDIR}/backstage-cr-disconnected.yaml" + echo "$rendered_cr" > "${cr_temp}" + + cp "${cr_temp}" "${ARTIFACT_DIR}/disconnected-backstage-cr.yaml" 2> /dev/null || true + + # Re-assert RELATED_IMAGE in case OLM v1 reconciled the operator Deployment + # back to the CSV GA value while we prepared the namespace. + disconnected::set_operator_related_hub_image "rhdh-operator" || return 1 + + deploy_rhdh_operator "${NAME_SPACE}" "${cr_temp}" + log::success "Backstage CR deployed in ${NAME_SPACE}" + + # Init image comes from RELATED_IMAGE_backstage (not the CR patch alone). + # Read-only verify both containers match HUB_IMAGE before smoke. + disconnected::verify_backstage_hub_images "${NAME_SPACE}" || return 1 + + log::section "Smoke Test" + + local url="https://backstage-${RELEASE_NAME}-${NAME_SPACE}.${K8S_CLUSTER_ROUTER_BASE}" + testing::check_and_test "${RELEASE_NAME}" "${NAME_SPACE}" "${PW_PROJECT_SMOKE_TEST}" "${url}" + + log::success "Disconnected Operator smoke test completed" +} diff --git a/.ci/pipelines/lib/disconnected.sh b/.ci/pipelines/lib/disconnected.sh new file mode 100644 index 0000000000..60e2706b07 --- /dev/null +++ b/.ci/pipelines/lib/disconnected.sh @@ -0,0 +1,661 @@ +#!/usr/bin/env bash + +# Shared utility functions for disconnected CI pipeline handlers. +# Provides environment validation, oc-mirror-based image mirroring, +# auth setup, and external script fetching. +# +# Dependencies: lib/log.sh, lib/common.sh +# Consumers: jobs/ocp-disconnected-helm.sh, jobs/ocp-disconnected-operator.sh + +# Prevent re-sourcing +if [[ -n "${DISCONNECTED_LIB_SOURCED:-}" ]]; then + return 0 +fi +readonly DISCONNECTED_LIB_SOURCED=1 + +# shellcheck source=.ci/pipelines/lib/log.sh +source "${DIR}/lib/log.sh" +# shellcheck source=.ci/pipelines/lib/common.sh +source "${DIR}/lib/common.sh" + +# Create a dedicated temp directory for disconnected CI artifacts. +DISCONNECTED_TMPDIR=$(mktemp -d) +export DISCONNECTED_TMPDIR + +# Validate that all required disconnected environment variables are set. +# These are exported by the step-registry commands.sh before calling +# openshift-ci-tests.sh. +disconnected::require_env() { + if [[ "${DISCONNECTED:-}" != "true" ]]; then + log::error "DISCONNECTED is not set to 'true'. This handler requires a disconnected environment." + log::error "Ensure the step-registry commands.sh has run before this handler." + return 1 + fi + + common::require_vars \ + MIRROR_REGISTRY_URL \ + MIRROR_REGISTRY_PULL_SECRET \ + MIRROR_REGISTRY_CA +} + +# Configure container-tools authentication for skopeo, oc-mirror, and +# mirror-plugins.sh. Places the combined pull secret (which contains +# credentials for both source registries and the mirror registry) in +# the standard locations expected by these tools. +disconnected::setup_auth() { + export HOME="${HOME:-/tmp/home}" + export XDG_RUNTIME_DIR="${HOME}/run" + mkdir -p "${XDG_RUNTIME_DIR}/containers" + + # oc-mirror and skopeo read auth from ${XDG_RUNTIME_DIR}/containers/auth.json + cp "${MIRROR_REGISTRY_PULL_SECRET}" "${XDG_RUNTIME_DIR}/containers/auth.json" + + # REGISTRY_AUTH_FILE is respected by skopeo as an explicit override + export REGISTRY_AUTH_FILE="${MIRROR_REGISTRY_PULL_SECRET}" + + # oc-mirror requires this to be unset + unset REGISTRY_AUTH_PREFERENCE + + log::info "Container auth configured from ${MIRROR_REGISTRY_PULL_SECRET}" +} + +# Build an ImageSetConfiguration for oc-mirror. +# The configuration is dynamically generated based on IMAGE_REGISTRY: +# - registry.redhat.io (GA): uses helm.local with chart pulled from charts.openshift.io +# - anything else (CI/upstream): uses helm.local with chart pulled from OCI +# Args: +# $1 - output_path: Path to write the ImageSetConfiguration YAML +disconnected::build_imageset_config() { + local output_path=$1 + + # Start with the base config + cat > "${output_path}" << EOF +kind: ImageSetConfiguration +apiVersion: mirror.openshift.io/v2alpha1 +mirror: + helm: + local: + - name: redhat-developer-hub + path: ${CHART_LOCAL_TGZ} +EOF + + # Add additional images that need mirroring beyond what the chart references. + # The chart's default images are discovered automatically by oc-mirror. + local additional_images=() + + # When the hub image is overridden (different from chart defaults), add it + # so oc-mirror mirrors the actual image we'll deploy with. + if [[ "${IMAGE_REGISTRY}" != "registry.redhat.io" ]]; then + # CI/upstream: chart defaults to quay.io/rhdh/rhdh-hub-rhel9@sha256:..., + # but we may deploy with a different tag (e.g., rhdh-community/rhdh:next) + additional_images+=("${IMAGE_REGISTRY}/${IMAGE_REPO}:${TAG_NAME}") + fi + + # PG image: CI charts may use quay.io/fedora/postgresql-15 instead of + # registry.redhat.io/rhel9/postgresql-15. Add it if not from registry.redhat.io. + # PG_SEPARATOR accounts for digest (@sha256:) vs tag (:) encoding. + if [[ "${PG_REGISTRY:-registry.redhat.io}" != "registry.redhat.io" ]]; then + additional_images+=("${PG_REGISTRY}/${PG_REPO}${PG_SEPARATOR}${PG_TAG}") + fi + + # Catalog index: the chart references it by digest and the init container + # pulls it at startup. Must be mirrored so IDMS can redirect the pull. + if [[ -n "${CI_REGISTRY:-}" && -n "${CI_REPO:-}" && -n "${CI_TAG:-}" ]]; then + additional_images+=("${CI_REGISTRY}/${CI_REPO}${CI_SEPARATOR:-:}${CI_TAG}") + fi + + if [[ ${#additional_images[@]} -gt 0 ]]; then + { + echo " additionalImages:" + for img in "${additional_images[@]}"; do + echo " - name: ${img}" + done + } >> "${output_path}" + fi + + log::info "ImageSetConfiguration written to ${output_path}" + log::debug "$(cat "${output_path}")" + + cp "${output_path}" "${ARTIFACT_DIR}/disconnected-imageset-config.yaml" 2> /dev/null || true +} + +# Run a command with REGISTRY_AUTH_FILE unset, then restore it. +# oc-mirror (and prepare paths that invoke it) panic when REGISTRY_AUTH_FILE +# is set because distribution/distribution treats it as a storage driver +# config. Auth still comes from ${XDG_RUNTIME_DIR}/containers/auth.json +# (configured by disconnected::setup_auth). +disconnected::with_unset_registry_auth_file() { + local saved_registry_auth_file="${REGISTRY_AUTH_FILE:-}" + unset REGISTRY_AUTH_FILE + + local rc=0 + "$@" || rc=$? + + if [[ -n "${saved_registry_auth_file}" ]]; then + export REGISTRY_AUTH_FILE="${saved_registry_auth_file}" + fi + return "${rc}" +} + +# Run oc-mirror to mirror images to the disconnected mirror registry. +# Sets OC_MIRROR_IDMS_FILE, OC_MIRROR_ITMS_FILE, and OC_MIRROR_CHART_PATH. +# Args: +# $1 - imageset_config: Path to the ImageSetConfiguration YAML +# $2 - workspace_dir: Path to the oc-mirror workspace directory +disconnected::run_oc_mirror() { + local imageset_config=$1 + local workspace_dir=$2 + + mkdir -p "${workspace_dir}" + + log::info "Running oc-mirror --v2 → ${MIRROR_REGISTRY_URL}" + if ! disconnected::with_unset_registry_auth_file oc-mirror \ + -c "${imageset_config}" \ + "docker://${MIRROR_REGISTRY_URL}" \ + --dest-tls-verify=false \ + --v2 \ + --workspace "file://${workspace_dir}"; then + log::error "oc-mirror failed" + return 1 + fi + + local result_dir="${workspace_dir}/working-dir" + + # IDMS (required) + OC_MIRROR_IDMS_FILE="${result_dir}/cluster-resources/idms-oc-mirror.yaml" + if [[ ! -s "${OC_MIRROR_IDMS_FILE}" ]]; then + log::error "oc-mirror did not generate IDMS at ${OC_MIRROR_IDMS_FILE}" + return 1 + fi + export OC_MIRROR_IDMS_FILE + + # ITMS (optional) + OC_MIRROR_ITMS_FILE="${result_dir}/cluster-resources/itms-oc-mirror.yaml" + if [[ ! -s "${OC_MIRROR_ITMS_FILE}" ]]; then + OC_MIRROR_ITMS_FILE="" + fi + export OC_MIRROR_ITMS_FILE + + # Chart path (in the workspace) + OC_MIRROR_CHART_PATH=$(find "${result_dir}/helm/charts" -name '*.tgz' 2> /dev/null | head -1) + export OC_MIRROR_CHART_PATH + + log::success "oc-mirror completed successfully" + log::info "IDMS: ${OC_MIRROR_IDMS_FILE}" + [[ -n "${OC_MIRROR_ITMS_FILE}" ]] && log::info "ITMS: ${OC_MIRROR_ITMS_FILE}" + [[ -n "${OC_MIRROR_CHART_PATH}" ]] && log::info "Chart: ${OC_MIRROR_CHART_PATH}" + + # Save artifacts for debugging + cp "${OC_MIRROR_IDMS_FILE}" "${ARTIFACT_DIR}/disconnected-idms-generated.yaml" 2> /dev/null || true + [[ -n "${OC_MIRROR_ITMS_FILE}" ]] && cp "${OC_MIRROR_ITMS_FILE}" "${ARTIFACT_DIR}/disconnected-itms-generated.yaml" 2> /dev/null || true +} + +# Patch the oc-mirror-generated IDMS to ensure both quay.io and +# registry.redhat.io sources are covered, regardless of what oc-mirror +# discovered from the chart. This is needed because: +# - GA charts reference registry.redhat.io but CI verification may override to quay.io +# - CI charts reference quay.io but post-GA verification uses registry.redhat.io +# Args: +# $1 - idms_file: Path to the IDMS YAML to patch +disconnected::patch_idms() { + local idms_file=$1 + + log::info "Patching IDMS with cross-registry mirror entries" + + # Add mirror entries for the hub image from both registries + for source_registry in "quay.io" "registry.redhat.io"; do + local source="${source_registry}/${IMAGE_REPO}" + local mirror="${MIRROR_REGISTRY_URL}/${IMAGE_REPO}" + + # Skip if this source is already in the IDMS + if yq eval ".spec.imageDigestMirrors[].source" "${idms_file}" 2> /dev/null | grep -qF "${source}"; then + log::debug "IDMS already contains entry for ${source}" + continue + fi + + yq eval -i \ + ".spec.imageDigestMirrors += [{\"mirrors\": [\"${mirror}\"], \"source\": \"${source}\"}]" \ + "${idms_file}" + log::info "Added IDMS entry: ${source} → ${mirror}" + done + + # Add mirror entry for PG image if not already present. + # PG_REPO is already cleaned of @sha256 by the caller (via PG_SEPARATOR). + if [[ -n "${PG_REGISTRY:-}" && -n "${PG_REPO:-}" ]]; then + local pg_source="${PG_REGISTRY}/${PG_REPO}" + local pg_mirror="${MIRROR_REGISTRY_URL}/${PG_REPO}" + + if ! yq eval ".spec.imageDigestMirrors[].source" "${idms_file}" 2> /dev/null | grep -qF "${pg_source}"; then + yq eval -i \ + ".spec.imageDigestMirrors += [{\"mirrors\": [\"${pg_mirror}\"], \"source\": \"${pg_source}\"}]" \ + "${idms_file}" + log::info "Added IDMS entry: ${pg_source} → ${pg_mirror}" + fi + fi + + log::debug "Patched IDMS:" + log::debug "$(cat "${idms_file}")" + + cp "${idms_file}" "${ARTIFACT_DIR}/disconnected-idms-patched.yaml" 2> /dev/null || true +} + +# Fetch an external script from the rhdh-operator repository. +# Args: +# $1 - script_name: Name of the script (e.g., "mirror-plugins.sh") +# $2 - output_path: Local path to save the script +# $3 - ref: (optional) Branch name, 40-char commit SHA, or pull request +# ref ("pull/" / "refs/pull//head"). Defaults to +# $RELEASE_BRANCH_NAME. +disconnected::fetch_script() { + local script_name=$1 + local output_path=$2 + local ref="${3:-${RELEASE_BRANCH_NAME}}" + local url + local ref_label + local pull_number + + if [[ "${ref}" =~ ^[0-9a-f]{40}$ ]]; then + url="https://raw.githubusercontent.com/redhat-developer/rhdh-operator/${ref}/.rhdh/scripts/${script_name}" + ref_label="sha: ${ref}" + elif [[ "${ref}" =~ ^(refs/)?pull/([0-9]+)(/head)?$ ]]; then + # Always the current PR head (updates as the PR is pushed). + pull_number="${BASH_REMATCH[2]}" + url="https://raw.githubusercontent.com/redhat-developer/rhdh-operator/refs/pull/${pull_number}/head/.rhdh/scripts/${script_name}" + ref_label="pull/${pull_number} head" + else + url="https://raw.githubusercontent.com/redhat-developer/rhdh-operator/refs/heads/${ref}/.rhdh/scripts/${script_name}" + ref_label="branch: ${ref}" + fi + + log::info "Fetching ${script_name} from rhdh-operator (${ref_label})..." + if ! curl -fL --max-time 30 -o "${output_path}" "${url}"; then + log::error "Failed to download ${script_name} from ${url}" + return 1 + fi + chmod +x "${output_path}" + log::success "Downloaded ${script_name} to ${output_path}" +} + +# Wait for MachineConfigPool updates after IDMS/CatalogSource changes. +# Warns and continues on timeout (same behavior as both handlers historically). +disconnected::wait_mcp_updated() { + log::info "Waiting for MachineConfigPool updates to complete (up to 20m)..." + if ! oc wait machineconfigpool --all --for=condition=Updated=True --timeout=20m; then + log::warn "MachineConfigPool wait timed out — proceeding anyway" + fi + log::success "All MachineConfigPools are Updated" +} + +# Mirror the CI hub image (IMAGE_REGISTRY/IMAGE_REPO:TAG_NAME) into the +# disconnected mirror registry. Operator prepare/IDMS covers +# registry.redhat.io/rhdh/rhdh-hub-rhel9, not quay.io/rhdh-community/rhdh:next. +# Helm gets this via oc-mirror additionalImages; operator needs an explicit copy +# so install-dynamic-plugins can use local plugins from the matching hub. +disconnected::mirror_hub_image() { + common::require_vars IMAGE_REGISTRY IMAGE_REPO TAG_NAME MIRROR_REGISTRY_URL || return 1 + + local src="docker://${IMAGE_REGISTRY}/${IMAGE_REPO}:${TAG_NAME}" + local dest="docker://${MIRROR_REGISTRY_URL}/${IMAGE_REPO}:${TAG_NAME}" + + log::info "Mirroring hub image ${IMAGE_REGISTRY}/${IMAGE_REPO}:${TAG_NAME} → ${MIRROR_REGISTRY_URL}/${IMAGE_REPO}:${TAG_NAME}" + if ! skopeo copy --all --dest-tls-verify=false "${src}" "${dest}"; then + log::error "Failed to mirror hub image ${src} → ${dest}" + return 1 + fi + log::success "Hub image mirrored to ${MIRROR_REGISTRY_URL}/${IMAGE_REPO}:${TAG_NAME}" +} + +# Resolve the mirrored hub to a digest-pinned ref and export HUB_IMAGE. +# Call after disconnected::mirror_hub_image. HUB_IMAGE is used by the smoke CR +# (envsubst) and by RELATED_IMAGE_backstage so init + backend stay exact. +disconnected::resolve_hub_image() { + common::require_vars IMAGE_REPO TAG_NAME MIRROR_REGISTRY_URL || return 1 + + local ref="${MIRROR_REGISTRY_URL}/${IMAGE_REPO}:${TAG_NAME}" + local digest + + log::info "Resolving digest for mirrored hub ${ref}" + digest=$(skopeo inspect --tls-verify=false "docker://${ref}" --format '{{.Digest}}' 2> /dev/null || true) + if [[ -z "${digest}" || "${digest}" != sha256:* ]]; then + log::error "Failed to resolve digest for docker://${ref}" + return 1 + fi + + export HUB_IMAGE="${MIRROR_REGISTRY_URL}/${IMAGE_REPO}@${digest}" + log::success "HUB_IMAGE=${HUB_IMAGE}" +} + +# Locate the rhdh-operator controller Deployment name in a namespace. +disconnected::_operator_deploy_name() { + local operator_ns="${1:-rhdh-operator}" + local deploy + + deploy=$(oc get deploy -n "${operator_ns}" \ + -l app.kubernetes.io/component=backstage-controller \ + -o jsonpath='{.items[0].metadata.name}' 2> /dev/null || true) + if [[ -z "${deploy}" ]]; then + deploy=$(oc get deploy -n "${operator_ns}" -o name 2> /dev/null \ + | grep -E 'rhdh-operator|backstage-controller' | head -1 | sed 's|^deployment.apps/||;s|^deployments.apps/||;s|^deployment/||') + fi + if [[ -z "${deploy}" ]]; then + log::error "Could not find rhdh-operator Deployment in ${operator_ns}" + oc get deploy -n "${operator_ns}" || true + return 1 + fi + printf '%s\n' "${deploy}" +} + +# Point the operator's RELATED_IMAGE_backstage at HUB_IMAGE (digest-pinned). +# +# Why: rhdh-operator applies spec.deployment.patch via strategic merge (by +# container name), so the CR can set backstage-backend.image. But +# DynamicPlugins.getInitContainer() later force-sets install-dynamic-plugins +# to RELATED_IMAGE_backstage, clobbering the CR init image. RELATED_IMAGE must +# equal the exact hub ref declared on the CR. +# +# Args: +# $1 - optional operator namespace (default: rhdh-operator) +disconnected::set_operator_related_hub_image() { + common::require_vars HUB_IMAGE || return 1 + + local operator_ns="${1:-rhdh-operator}" + local hub_image="${HUB_IMAGE}" + local deploy + local actual + + deploy=$(disconnected::_operator_deploy_name "${operator_ns}") || return 1 + + log::info "Setting RELATED_IMAGE_backstage=${hub_image} on deploy/${deploy} (${operator_ns})" + oc set env "deployment/${deploy}" -n "${operator_ns}" \ + "RELATED_IMAGE_backstage=${hub_image}" || { + log::error "Failed to set RELATED_IMAGE_backstage on deploy/${deploy}" + return 1 + } + if ! oc rollout status "deployment/${deploy}" -n "${operator_ns}" --timeout=300s; then + log::error "Operator rollout after RELATED_IMAGE_backstage update timed out" + return 1 + fi + + actual=$(oc get "deployment/${deploy}" -n "${operator_ns}" \ + -o jsonpath='{range .spec.template.spec.containers[*].env[?(@.name=="RELATED_IMAGE_backstage")]}{.value}{end}' 2> /dev/null || true) + if [[ "${actual}" != "${hub_image}" ]]; then + log::error "RELATED_IMAGE_backstage on deploy/${deploy} is '${actual}', want '${hub_image}' (OLM may have reverted)" + return 1 + fi + log::success "Operator RELATED_IMAGE_backstage → ${hub_image}" +} + +# Read-only check that both Backstage hub containers use HUB_IMAGE. +# Init must match RELATED_IMAGE_backstage (operator getInitContainer); the CR +# patch alone cannot keep install-dynamic-plugins on a different image. +# +# Args: +# $1 - Backstage namespace +disconnected::verify_backstage_hub_images() { + common::require_vars HUB_IMAGE || return 1 + + local namespace="$1" + local hub_image="${HUB_IMAGE}" + local deploy + local backend_img init_img + local attempt + + deploy=$(oc get deployment -n "${namespace}" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' \ + | grep -E '^backstage-' | grep -vE 'psql|postgres' | head -1) + if [[ -z "${deploy}" ]]; then + log::error "No backstage Deployment found in ${namespace} to verify hub images" + oc get deployment -n "${namespace}" || true + return 1 + fi + + # Allow a short reconcile window after CR apply + RELATED_IMAGE set. + for attempt in 1 2 3 4 5 6; do + backend_img=$(oc get "deployment/${deploy}" -n "${namespace}" \ + -o jsonpath='{range .spec.template.spec.containers[?(@.name=="backstage-backend")]}{.image}{end}') + init_img=$(oc get "deployment/${deploy}" -n "${namespace}" \ + -o jsonpath='{range .spec.template.spec.initContainers[?(@.name=="install-dynamic-plugins")]}{.image}{end}') + log::info "deploy/${deploy} images (attempt ${attempt}): backstage-backend=${backend_img} install-dynamic-plugins=${init_img}" + if [[ "${backend_img}" == "${hub_image}" && "${init_img}" == "${hub_image}" ]]; then + log::success "Backstage Deployment hub images match HUB_IMAGE=${hub_image}" + return 0 + fi + sleep 5 + done + + log::error "RELATED_IMAGE_backstage not effective on install-dynamic-plugins (want ${hub_image}; backend=${backend_img}; init=${init_img})" + return 1 +} + +# Fetch and run mirror-plugins.sh against the disconnected mirror registry. +# Uses CATALOG_INDEX_IMAGE when set; otherwise the GA plugin-catalog-index tag. +disconnected::mirror_plugins() { + local mirror_script="${DISCONNECTED_TMPDIR}/mirror-plugins.sh" + + disconnected::fetch_script "mirror-plugins.sh" "${mirror_script}" || { + log::error "Failed to fetch mirror-plugins.sh — aborting" + return 1 + } + + local plugin_index="oci://registry.access.redhat.com/rhdh/plugin-catalog-index:${RELEASE_VERSION}" + if [[ -n "${CATALOG_INDEX_IMAGE:-}" ]]; then + plugin_index="oci://${CATALOG_INDEX_IMAGE}" + fi + + bash "${mirror_script}" \ + --plugin-index "${plugin_index}" \ + --to-registry "${MIRROR_REGISTRY_URL}" || { + log::error "mirror-plugins.sh failed — aborting" + return 1 + } +} + +# Apply the shared plugin-mirror registries.conf ConfigMap in a namespace. +# Args: +# $1 - namespace +disconnected::apply_plugin_mirror_configmap() { + local namespace=$1 + local configmap_template="${DIR}/resources/disconnected/plugin-mirror-configmap.yaml" + + envsubst < "${configmap_template}" \ + | oc apply -n "${namespace}" -f - || { + log::error "Failed to create registries.conf ConfigMap — aborting" + return 1 + } + log::success "ConfigMap rhdh-plugin-mirror-conf created in ${namespace}" + + envsubst < "${configmap_template}" \ + > "${ARTIFACT_DIR}/disconnected-plugin-mirror-configmap.yaml" 2> /dev/null || true +} + +# Create the per-namespace mirror CA ConfigMap used by install-dynamic-plugins +# (mounted at /etc/containers/certs.d//ca.crt so skopeo trusts TLS). +# Args: +# $1 - namespace +disconnected::create_mirror_registry_ca_configmap() { + local namespace=$1 + + if [[ ! -f "${MIRROR_REGISTRY_CA}" ]]; then + log::error "MIRROR_REGISTRY_CA file not found: ${MIRROR_REGISTRY_CA}" + return 1 + fi + + oc create configmap mirror-registry-ca \ + --from-file="ca.crt=${MIRROR_REGISTRY_CA}" \ + -n "${namespace}" \ + --dry-run=client -o yaml | oc apply -f - || { + log::error "Failed to create mirror-registry-ca ConfigMap — aborting" + return 1 + } + log::success "ConfigMap mirror-registry-ca created in ${namespace}" +} + +# Create the registry auth secret for the install-dynamic-plugins init container. +# Args: +# $1 - namespace +# $2 - secret name (default: ${RELEASE_NAME}-dynamic-plugins-registry-auth) +disconnected::create_plugin_registry_auth_secret() { + local namespace=$1 + local secret_name=${2:-${RELEASE_NAME}-dynamic-plugins-registry-auth} + + oc create secret generic "${secret_name}" \ + --from-file="auth.json=${MIRROR_REGISTRY_PULL_SECRET}" \ + -n "${namespace}" \ + --dry-run=client -o yaml | oc apply -f - || { + log::error "Failed to create registry auth secret — aborting" + return 1 + } + log::success "Secret ${secret_name} created in ${namespace}" +} + +# Ensure the mirror registry CA is trusted cluster-wide via +# image.config.openshift.io/cluster additionalTrustedCA so OLM v1 catalogd +# can pull ClusterCatalog images (x509 unknown authority otherwise). +# Triggers an MCP update; callers should run disconnected::wait_mcp_updated after. +disconnected::ensure_mirror_registry_ca() { + local registry_host="${MIRROR_REGISTRY_URL%%/*}" + # OpenShift ConfigMap keys use ".." in place of ":" for host:port. + local cm_key="${registry_host//:/..}" + local cm_name="rhdh-disconnected-mirror-ca" + local existing_cm + + if [[ ! -f "${MIRROR_REGISTRY_CA}" ]]; then + log::error "MIRROR_REGISTRY_CA file not found: ${MIRROR_REGISTRY_CA}" + return 1 + fi + + existing_cm=$(oc get image.config.openshift.io/cluster -o jsonpath='{.spec.additionalTrustedCA.name}' 2> /dev/null || true) + if [[ -n "${existing_cm}" ]]; then + cm_name="${existing_cm}" + log::info "Merging mirror CA into existing additionalTrustedCA ConfigMap ${cm_name}" + oc get configmap "${cm_name}" -n openshift-config -o json \ + | jq --arg key "${cm_key}" --rawfile cert "${MIRROR_REGISTRY_CA}" \ + '.data[$key] = $cert | del(.metadata.resourceVersion, .metadata.uid, .metadata.creationTimestamp)' \ + | oc apply -f - || { + log::error "Failed to merge mirror CA into ConfigMap ${cm_name}" + return 1 + } + else + log::info "Creating additionalTrustedCA ConfigMap ${cm_name} for ${registry_host}" + oc create configmap "${cm_name}" -n openshift-config \ + --from-file="${cm_key}=${MIRROR_REGISTRY_CA}" \ + --dry-run=client -o yaml | oc apply -f - || { + log::error "Failed to create ConfigMap ${cm_name}" + return 1 + } + oc patch image.config.openshift.io/cluster --type=merge \ + -p "{\"spec\":{\"additionalTrustedCA\":{\"name\":\"${cm_name}\"}}}" || { + log::error "Failed to patch image.config.openshift.io/cluster additionalTrustedCA" + return 1 + } + fi + + log::success "Mirror CA trusted for ${registry_host} via ${cm_name}/${cm_key}" +} + +# Merge mirror-registry credentials into openshift-config/pull-secret so OLM v1 +# catalogd/operator-controller can pull mirrored catalog/bundle images. +# prepare-restricted-environment.sh skips this for external registries. +disconnected::ensure_olm_mirror_pull_secret() { + local existing mirror_auth merged + + existing=$(oc get secret pull-secret -n openshift-config -o jsonpath='{.data.\.dockerconfigjson}' | base64 -d) || { + log::error "Failed to read openshift-config/pull-secret" + return 1 + } + mirror_auth=$(cat "${MIRROR_REGISTRY_PULL_SECRET}") || { + log::error "Failed to read ${MIRROR_REGISTRY_PULL_SECRET}" + return 1 + } + + merged=$(jq -n --argjson existing "${existing}" --argjson mirror "${mirror_auth}" \ + '{auths: ($existing.auths + $mirror.auths)}') || { + log::error "Failed to merge mirror credentials into pull-secret JSON" + return 1 + } + + echo "${merged}" | oc set data secret/pull-secret -n openshift-config \ + --from-file=.dockerconfigjson=/dev/stdin || { + log::error "Failed to update openshift-config/pull-secret with mirror credentials" + return 1 + } + log::success "Merged mirror registry credentials into openshift-config/pull-secret" +} + +# Dump OLM v1 install status for debugging when the operator CRD never appears. +disconnected::dump_olm_v1_status() { + local extension_name=${1:-rhdh-operator} + local catalog_name=${2:-rhdh-catalog} + local operator_ns=${3:-rhdh-operator} + + log::info "Dumping OLM v1 status (ClusterCatalog/ClusterExtension/pods)..." + oc get clustercatalog "${catalog_name}" -o yaml > "${ARTIFACT_DIR}/disconnected-clustercatalog.yaml" 2> /dev/null || true + oc get clusterextension "${extension_name}" -o yaml > "${ARTIFACT_DIR}/disconnected-clusterextension.yaml" 2> /dev/null || true + oc get clustercatalog "${catalog_name}" -o wide 2>&1 | tee -a "${ARTIFACT_DIR}/disconnected-olm-v1-status.txt" || true + oc get clusterextension "${extension_name}" -o wide 2>&1 | tee -a "${ARTIFACT_DIR}/disconnected-olm-v1-status.txt" || true + oc describe clusterextension "${extension_name}" 2>&1 | tee -a "${ARTIFACT_DIR}/disconnected-olm-v1-status.txt" || true + oc get pods -n "${operator_ns}" -o wide 2>&1 | tee -a "${ARTIFACT_DIR}/disconnected-olm-v1-status.txt" || true + oc get events -n "${operator_ns}" --sort-by='.lastTimestamp' 2>&1 | tail -50 \ + | tee -a "${ARTIFACT_DIR}/disconnected-olm-v1-status.txt" || true +} + +# Wait for OLM v1 ClusterExtension to report Installed, then for the CRD. +# Args: +# $1 - extension name (default: rhdh-operator) +# $2 - crd name (default: backstages.rhdh.redhat.com) +# $3 - timeout seconds (default: 600) +disconnected::wait_operator_crd_olm_v1() { + local extension_name=${1:-rhdh-operator} + local crd_name=${2:-backstages.rhdh.redhat.com} + local timeout=${3:-600} + local interval=15 + local elapsed=0 + + log::info "Waiting for ClusterExtension/${extension_name} and CRD ${crd_name} (timeout: ${timeout}s)..." + + while ((elapsed < timeout)); do + if oc get crd "${crd_name}" > /dev/null 2>&1; then + log::success "CRD '${crd_name}' is available" + return 0 + fi + + local installed + installed=$(oc get clusterextension "${extension_name}" \ + -o jsonpath='{range .status.conditions[?(@.type=="Installed")]}{.status}{end}' 2> /dev/null || true) + if [[ "${installed}" == "True" ]]; then + log::info "ClusterExtension/${extension_name} reports Installed=True; waiting for CRD..." + else + local progressing reason + progressing=$(oc get clusterextension "${extension_name}" \ + -o jsonpath='{range .status.conditions[?(@.type=="Progressing")]}{.status}{end}' 2> /dev/null || true) + reason=$(oc get clusterextension "${extension_name}" \ + -o jsonpath='{range .status.conditions[?(@.type=="Installed")]}{.reason}{" "}{.message}{end}' 2> /dev/null || true) + log::debug "ClusterExtension Installed=${installed:-unknown} Progressing=${progressing:-unknown} ${reason}" + fi + + sleep "${interval}" + elapsed=$((elapsed + interval)) + done + + log::error "Timeout waiting for CRD '${crd_name}' after ${timeout}s" + disconnected::dump_olm_v1_status "${extension_name}" + return 1 +} + +# Export functions for subshell usage (e.g., timeout bash -c "...") +export -f disconnected::require_env +export -f disconnected::setup_auth +export -f disconnected::fetch_script +export -f disconnected::with_unset_registry_auth_file +export -f disconnected::wait_mcp_updated +export -f disconnected::mirror_hub_image +export -f disconnected::resolve_hub_image +export -f disconnected::set_operator_related_hub_image +export -f disconnected::verify_backstage_hub_images +export -f disconnected::mirror_plugins +export -f disconnected::apply_plugin_mirror_configmap +export -f disconnected::create_mirror_registry_ca_configmap +export -f disconnected::create_plugin_registry_auth_secret +export -f disconnected::ensure_mirror_registry_ca +export -f disconnected::ensure_olm_mirror_pull_secret +export -f disconnected::dump_olm_v1_status +export -f disconnected::wait_operator_crd_olm_v1 diff --git a/.ci/pipelines/openshift-ci-tests.sh b/.ci/pipelines/openshift-ci-tests.sh index 55b3679cbc..f6564e0b5c 100755 --- a/.ci/pipelines/openshift-ci-tests.sh +++ b/.ci/pipelines/openshift-ci-tests.sh @@ -128,6 +128,20 @@ main() { log::info "Calling handle_ocp_localization" handle_ocp_localization ;; + *ocp*disconnected*helm*nightly*) + log::info "Sourcing ocp-disconnected-helm.sh" + # shellcheck source=.ci/pipelines/jobs/ocp-disconnected-helm.sh + source "${DIR}/jobs/ocp-disconnected-helm.sh" + log::info "Calling handle_ocp_disconnected_helm" + handle_ocp_disconnected_helm + ;; + *ocp*disconnected*operator*nightly*) + log::info "Sourcing ocp-disconnected-operator.sh" + # shellcheck source=.ci/pipelines/jobs/ocp-disconnected-operator.sh + source "${DIR}/jobs/ocp-disconnected-operator.sh" + log::info "Calling handle_ocp_disconnected_operator" + handle_ocp_disconnected_operator + ;; *ocp*helm*nightly*) log::info "Sourcing ocp-nightly.sh" # shellcheck source=.ci/pipelines/jobs/ocp-nightly.sh diff --git a/.ci/pipelines/resources/config_map/app-config-rhdh-disconnected-smoke.yaml b/.ci/pipelines/resources/config_map/app-config-rhdh-disconnected-smoke.yaml new file mode 100644 index 0000000000..a2ec462312 --- /dev/null +++ b/.ci/pipelines/resources/config_map/app-config-rhdh-disconnected-smoke.yaml @@ -0,0 +1,8 @@ +# Minimal app-config for disconnected operator smoke (guest login only). +app: + title: Red Hat Developer Hub +auth: + environment: development + providers: + guest: + dangerouslyAllowOutsideDevelopment: true diff --git a/.ci/pipelines/resources/disconnected/helm-post-renderer.sh b/.ci/pipelines/resources/disconnected/helm-post-renderer.sh new file mode 100755 index 0000000000..8bbda459ff --- /dev/null +++ b/.ci/pipelines/resources/disconnected/helm-post-renderer.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Helm post-renderer for disconnected deployments. +# +# Patches the RHDH Deployment's pod spec for disconnected environments: +# 1. Mounts registries.conf so the init container resolves plugin images +# from the mirror registry instead of the original registries. +# 2. Mounts the mirror registry CA at the standard container-tools +# per-registry path (/etc/containers/certs.d//ca.crt) +# so skopeo trusts the mirror's TLS certificate. +# 3. Mounts a permissive policy.json so skopeo accepts unsigned images +# from the mirror (Red Hat signature server is unreachable in +# disconnected environments). +# +# Using a post-renderer avoids the Helm "array clobber" pitfall: +# a values file that defines extraVolumes[] or initContainers[] replaces +# the chart's entire default array, losing any volumes added by newer +# chart versions. A post-renderer patches the already-rendered manifests +# so the chart's defaults are always preserved. +# +# Usage: +# helm upgrade -i ... \ +# --post-renderer ./helm-post-renderer.sh \ +# --post-renderer-args + +set -euo pipefail + +MIRROR_REGISTRY_URL="${1:?Usage: helm-post-renderer.sh }" + +yq eval " + (select(.kind == \"Deployment\" and .metadata.name == \"*-developer-hub\") | + .spec.template.spec.volumes += [ + {\"name\": \"rhdh-plugin-mirror-conf\", \"configMap\": {\"name\": \"rhdh-plugin-mirror-conf\"}}, + {\"name\": \"mirror-registry-ca\", \"configMap\": {\"name\": \"mirror-registry-ca\"}} + ] | + .spec.template.spec.initContainers[0].volumeMounts += [ + {\"mountPath\": \"/etc/containers/registries.conf.d/rhdh-registries.conf\", \"name\": \"rhdh-plugin-mirror-conf\", \"readOnly\": true, \"subPath\": \"rhdh-registries.conf\"}, + {\"mountPath\": \"/etc/containers/policy.json\", \"name\": \"rhdh-plugin-mirror-conf\", \"readOnly\": true, \"subPath\": \"policy.json\"}, + {\"mountPath\": \"/etc/containers/certs.d/${MIRROR_REGISTRY_URL}/ca.crt\", \"name\": \"mirror-registry-ca\", \"readOnly\": true, \"subPath\": \"ca.crt\"} + ] + ) // . +" diff --git a/.ci/pipelines/resources/disconnected/plugin-mirror-configmap.yaml b/.ci/pipelines/resources/disconnected/plugin-mirror-configmap.yaml new file mode 100644 index 0000000000..7b5acdb5c5 --- /dev/null +++ b/.ci/pipelines/resources/disconnected/plugin-mirror-configmap.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: rhdh-plugin-mirror-conf +data: + rhdh-registries.conf: | + [[registry]] + prefix = "registry.access.redhat.com/rhdh" + location = "${MIRROR_REGISTRY_URL}/rhdh" + + [[registry]] + prefix = "quay.io/rhdh" + location = "${MIRROR_REGISTRY_URL}/rhdh" + + [[registry]] + prefix = "ghcr.io/redhat-developer/rhdh-plugin-export-overlays" + location = "${MIRROR_REGISTRY_URL}/rhdh-plugin-export-overlays" + policy.json: | + {"default": [{"type": "insecureAcceptAnything"}]} diff --git a/.ci/pipelines/resources/rhdh-operator/rhdh-start-disconnected-smoke.yaml b/.ci/pipelines/resources/rhdh-operator/rhdh-start-disconnected-smoke.yaml new file mode 100644 index 0000000000..3c5f70b424 --- /dev/null +++ b/.ci/pipelines/resources/rhdh-operator/rhdh-start-disconnected-smoke.yaml @@ -0,0 +1,54 @@ +# Minimal Backstage CR for disconnected operator smoke test. +# Smoke only needs: guest login → homepage. Avoid ConfigMaps/Secrets that the +# normal apply_yaml_files() flow creates (app-config-rhdh, dynamic-plugins-*, +# rhdh-secrets, redis-secret). +# +# flavours: [] disables default flavours (lightspeed) so the deployment does not +# pull/start lightspeed-core / init-rag-data sidecars. +# +# Exact hub image: $HUB_IMAGE is a digest-pinned mirror ref +# (${MIRROR_REGISTRY_URL}/${IMAGE_REPO}@sha256:…). It must match +# CATALOG_INDEX_IMAGE (nightly :next catalog expects local homepage dist from +# the community hub, not GA rhdh-hub-rhel9). +# +# CRD surface: spec.deployment.patch merges containers/initContainers by name +# (K8s strategic merge / operator merge2). That alone is not enough for +# install-dynamic-plugins — DynamicPlugins.getInitContainer() re-applies +# RELATED_IMAGE_backstage after the patch. ocp-disconnected-operator.sh therefore +# sets RELATED_IMAGE_backstage=$HUB_IMAGE on the operator Deployment as well. +# +# CATALOG_INDEX_IMAGE is injected by deploy_rhdh_operator() after +# ocp-disconnected-operator.sh rewrites it to MIRROR_REGISTRY_URL/… (the index +# already mirrored by disconnected::mirror_plugins). +kind: Backstage +apiVersion: rhdh.redhat.com/v1alpha5 +metadata: + name: rhdh +spec: + flavours: [] + deployment: + patch: + spec: + template: + spec: + imagePullSecrets: + - name: mirror-registry-pull + containers: + - name: backstage-backend + image: "$HUB_IMAGE" + initContainers: + - name: install-dynamic-plugins + image: "$HUB_IMAGE" + application: + appConfig: + configMaps: + - name: app-config-rhdh-disconnected-smoke + mountPath: /opt/app-root/src + extraEnvs: + envs: + - name: NODE_OPTIONS + value: "--no-node-snapshot" + - name: NODE_ENV + value: "production" + - name: NODE_TLS_REJECT_UNAUTHORIZED + value: "0" diff --git a/.ci/pipelines/value_files/values_disconnected-smoke.yaml b/.ci/pipelines/value_files/values_disconnected-smoke.yaml new file mode 100644 index 0000000000..eb85cd46f6 --- /dev/null +++ b/.ci/pipelines/value_files/values_disconnected-smoke.yaml @@ -0,0 +1,20 @@ +# Minimal Helm values for disconnected smoke test. +# The smoke test only verifies: login page renders → guest login → homepage. +# No GitHub integrations, external secrets, dynamic plugins, or OIDC needed. +# All optional features (lightspeed, orchestrator) are disabled to avoid +# pulling extra images and referencing missing ConfigMaps/Secrets. +global: + lightspeed: + enabled: false +orchestrator: + enabled: false +upstream: + backstage: + image: + pullPolicy: Always + appConfig: + auth: + environment: development + providers: + guest: + dangerouslyAllowOutsideDevelopment: true diff --git a/e2e-tests/playwright.config.ts b/e2e-tests/playwright.config.ts index 9f055740b8..15a9acdc6a 100644 --- a/e2e-tests/playwright.config.ts +++ b/e2e-tests/playwright.config.ts @@ -4,6 +4,7 @@ import type { ReporterDescription } from "@playwright/test"; /* oxlint-disable import/no-unassigned-import -- intentional side-effect graph wiring */ import "./playwright/entry-graph"; import { PW_PROJECT } from "./playwright/projects"; +import { parseProxy } from "./playwright/utils/proxy"; process.env.JOB_NAME = process.env.JOB_NAME ?? ""; process.env.IS_OPENSHIFT = process.env.IS_OPENSHIFT ?? ""; @@ -61,6 +62,9 @@ export default defineConfig({ locale: process.env.LOCALE ?? "en", baseURL: process.env.BASE_URL, ignoreHTTPSErrors: true, + // Proxy for disconnected environments where the CI runner reaches the + // cluster through a squid proxy (HTTPS_PROXY set by proxy-conf.sh). + proxy: parseProxy(process.env.HTTPS_PROXY), trace: "retain-on-failure", screenshot: "on", ...devices["Desktop Chrome"], diff --git a/e2e-tests/playwright/global-setup.ts b/e2e-tests/playwright/global-setup.ts index 83c4eb9566..a3d634fe43 100644 --- a/e2e-tests/playwright/global-setup.ts +++ b/e2e-tests/playwright/global-setup.ts @@ -1,5 +1,6 @@ import { request as playwrightRequest } from "@playwright/test"; +import { parseProxy } from "./utils/proxy"; import { ensureRuntimeDeployed } from "./utils/runtime-deploy"; import { waitForRhdhReady } from "./utils/wait-for-rhdh-ready"; @@ -27,6 +28,10 @@ export default async function globalSetup(): Promise { const request = await playwrightRequest.newContext({ baseURL, ignoreHTTPSErrors: true, + // In disconnected environments the CI runner reaches the cluster through a + // squid proxy (HTTPS_PROXY). Unlike browser contexts which inherit the + // proxy from playwright.config.ts, APIRequestContext needs it explicitly. + proxy: parseProxy(process.env.HTTPS_PROXY), }); try { await waitForRhdhReady(request); diff --git a/e2e-tests/playwright/utils/proxy.ts b/e2e-tests/playwright/utils/proxy.ts new file mode 100644 index 0000000000..b2777043ba --- /dev/null +++ b/e2e-tests/playwright/utils/proxy.ts @@ -0,0 +1,25 @@ +/** + * Parse an HTTP(S) proxy URL into Playwright's proxy config. + * + * Playwright's `proxy` option expects `username` and `password` as separate + * fields, but CI environments typically set a single `HTTPS_PROXY` URL like + * `http://user:pass@host:3128`. This helper splits the URL into the shape + * Playwright needs. + * + * Returns `undefined` when no proxy URL is provided (connected environments). + */ +export function parseProxy( + proxyUrl: string | undefined, +): { server: string; username?: string; password?: string } | undefined { + if (proxyUrl === undefined || proxyUrl === "") return undefined; + try { + const u = new URL(proxyUrl); + return { + server: `${u.protocol}//${u.host}`, + ...(u.username !== "" && { username: decodeURIComponent(u.username) }), + ...(u.password !== "" && { password: decodeURIComponent(u.password) }), + }; + } catch { + return { server: proxyUrl }; + } +}