From cf24bfafc285b54b5e9d1da75e9ae65a6230f6a4 Mon Sep 17 00:00:00 2001 From: Bryan Cox Date: Fri, 7 Aug 2026 11:07:47 -0400 Subject: [PATCH] feat(aro-hcp): add e2e-aro-hcp-presubmit job with CPO override and nightly support Adds a new `e2e-aro-hcp-presubmit` CI job that tests both the HyperShift Operator (HO) and Control Plane Operator (CPO) from a hypershift PR against an ARO-HCP environment using nightly OCP release images. Changes: - images-push: conditionally push CPO to OCP ACR when HYPERSHIFT_CPO_IMAGE is provided (via job-level dependency override), and write hypershift-cpo-override.env with the CPO_IMAGE_OVERRIDE ref - test-local: source hypershift-cpo-override.env if present, declare ARO_HCP_OPENSHIFT_CHANNEL_GROUP and ARO_HCP_OPENSHIFT_VERSION_ID env vars - ci-operator config: new e2e-aro-hcp-presubmit job that provides the hypershift (CPO) dependency and sets nightly channel group with OCP 5.0 The existing e2e-aro-hcp job (HO-only) is not affected. Co-Authored-By: Claude Opus 4.6 --- .../hypershift/openshift-hypershift-main.yaml | 15 ++++++ ...aro-hcp-hypershift-images-push-commands.sh | 47 ++++++++++++++++++- .../aro-hcp-hypershift-images-push-ref.yaml | 14 ++++-- .../test/local/aro-hcp-test-local-commands.sh | 7 +++ .../test/local/aro-hcp-test-local-ref.yaml | 16 +++++++ 5 files changed, 94 insertions(+), 5 deletions(-) diff --git a/ci-operator/config/openshift/hypershift/openshift-hypershift-main.yaml b/ci-operator/config/openshift/hypershift/openshift-hypershift-main.yaml index 31610853fe5fd..10d1a4a89ed53 100644 --- a/ci-operator/config/openshift/hypershift/openshift-hypershift-main.yaml +++ b/ci-operator/config/openshift/hypershift/openshift-hypershift-main.yaml @@ -210,6 +210,21 @@ tests: ARO_HCP_DEPLOY_ENV: ci01 MULTISTAGE_PARAM_OVERRIDE_LOCATION: westus3 workflow: aro-hcp-hypershift-e2e +- always_run: false + as: e2e-aro-hcp-presubmit + optional: true + steps: + cluster_profile: aro-hcp-dev + env: + ALLOWED_SUBSCRIPTIONS: Hypershift Managed Azure + ARO_HCP_CLOUD: dev + ARO_HCP_DEPLOY_ENV: ci01 + ARO_HCP_E2E_SUITE: hypershift-presubmit/parallel + ARO_HCP_OPENSHIFT_CHANNEL_GROUP: nightly + ARO_HCP_OPENSHIFT_VERSION_ID: "5.0" + MULTISTAGE_PARAM_OVERRIDE_LOCATION: westus3 + PUSH_CPO_IMAGE: "true" + workflow: aro-hcp-hypershift-e2e - as: e2e-aks-override run_if_changed: (/overrides\.yaml$) steps: diff --git a/ci-operator/step-registry/aro-hcp/hypershift-images-push/aro-hcp-hypershift-images-push-commands.sh b/ci-operator/step-registry/aro-hcp/hypershift-images-push/aro-hcp-hypershift-images-push-commands.sh index c9ecff4d9f4e4..795860dcd492d 100755 --- a/ci-operator/step-registry/aro-hcp/hypershift-images-push/aro-hcp-hypershift-images-push-commands.sh +++ b/ci-operator/step-registry/aro-hcp/hypershift-images-push/aro-hcp-hypershift-images-push-commands.sh @@ -27,6 +27,12 @@ SVC_ACR_NAME=$(yq '.acr.svc.name' "${CONFIG_FILE}") SVC_ACR_URL="${SVC_ACR_NAME}.azurecr.io" echo "SVC ACR: ${SVC_ACR_URL} (HO image)" +if [[ "${PUSH_CPO_IMAGE}" == "true" ]]; then + OCP_ACR_NAME=$(yq '.acr.ocp.name' "${CONFIG_FILE}") + OCP_ACR_URL="${OCP_ACR_NAME}.azurecr.io" + echo "OCP ACR: ${OCP_ACR_URL} (CPO image)" +fi + # Authenticate to CI registry export XDG_RUNTIME_DIR="/tmp/run" mkdir -p "${XDG_RUNTIME_DIR}/containers" "${HOME}/.docker" @@ -36,6 +42,11 @@ oc registry login SVC_ACR_TOKEN=$(az acr login --name "${SVC_ACR_NAME}" --expose-token --output tsv --query accessToken) oc registry login --registry "${SVC_ACR_URL}" --auth-basic="00000000-0000-0000-0000-000000000000:${SVC_ACR_TOKEN}" +if [[ "${PUSH_CPO_IMAGE}" == "true" ]]; then + OCP_ACR_TOKEN=$(az acr login --name "${OCP_ACR_NAME}" --expose-token --output tsv --query accessToken) + oc registry login --registry "${OCP_ACR_URL}" --auth-basic="00000000-0000-0000-0000-000000000000:${OCP_ACR_TOKEN}" +fi + IMAGE_TAG="hypershift-pr-${PULL_NUMBER:-unknown}-$(date +%s)" retry() { @@ -56,6 +67,16 @@ HO_ACR_REF="${SVC_ACR_URL}/hypershift-operator:${IMAGE_TAG}" echo "Pushing hypershift-operator: ${HYPERSHIFT_OPERATOR_IMAGE} -> ${HO_ACR_REF}" retry oc image mirror "${HYPERSHIFT_OPERATOR_IMAGE}" "${HO_ACR_REF}" +# Push control-plane-operator (CPO) image to OCP ACR if provided. +# The CPO runs in HCP namespaces on the management cluster, which have pull +# credentials for the OCP ACR but not the SVC ACR. The override must go to +# the same registry that normal CPO images land in via registryOverrides. +if [[ "${PUSH_CPO_IMAGE}" == "true" ]]; then + CPO_ACR_REF="${OCP_ACR_URL}/hypershift:${IMAGE_TAG}" + echo "Pushing control-plane-operator: ${HYPERSHIFT_CPO_IMAGE} -> ${CPO_ACR_REF}" + retry oc image mirror "${HYPERSHIFT_CPO_IMAGE}" "${CPO_ACR_REF}" +fi + # Resolve digests from ACR for the pushed images. # ACR metadata can lag behind a successful push, so retry the query. resolve_digest() { @@ -104,5 +125,27 @@ unset _YQ_REG _YQ_REPO _YQ_DIG echo "Created hypershift image overrides at ${HYPERSHIFT_OVERRIDES}:" cat "${HYPERSHIFT_OVERRIDES}" -echo "Hypershift operator image pushed successfully." -echo "HO: ${SVC_ACR_URL}/hypershift-operator@${HO_DIGEST}" +if [[ "${PUSH_CPO_IMAGE}" == "true" ]]; then + CPO_DIGEST=$(resolve_digest "${OCP_ACR_URL}" "hypershift" "${IMAGE_TAG}") + echo "CPO digest: ${CPO_DIGEST}" + + if [[ -z "${CPO_DIGEST}" ]]; then + echo "ERROR: Failed to resolve digest for hypershift:${IMAGE_TAG} from ${OCP_ACR_URL}" + exit 1 + fi + + # Write CPO image ref for e2e tests to use as an ARM resource tag. + # The CPO override is applied via the aro-hcp.experimental.cluster. + # control-plane-operator-image-override tag (AFEC-gated), not a config key. + CPO_OVERRIDE_FILE="${SHARED_DIR}/hypershift-cpo-override.env" + echo "export CPO_IMAGE_OVERRIDE=\"${OCP_ACR_URL}/hypershift@${CPO_DIGEST}\"" > "${CPO_OVERRIDE_FILE}" + echo "Created CPO override env at ${CPO_OVERRIDE_FILE}:" + cat "${CPO_OVERRIDE_FILE}" + + echo "All hypershift images pushed successfully." + echo "HO: ${SVC_ACR_URL}/hypershift-operator@${HO_DIGEST}" + echo "CPO: ${OCP_ACR_URL}/hypershift@${CPO_DIGEST}" +else + echo "Hypershift operator image pushed successfully." + echo "HO: ${SVC_ACR_URL}/hypershift-operator@${HO_DIGEST}" +fi diff --git a/ci-operator/step-registry/aro-hcp/hypershift-images-push/aro-hcp-hypershift-images-push-ref.yaml b/ci-operator/step-registry/aro-hcp/hypershift-images-push/aro-hcp-hypershift-images-push-ref.yaml index 0c504f24317af..8b46210d32edd 100644 --- a/ci-operator/step-registry/aro-hcp/hypershift-images-push/aro-hcp-hypershift-images-push-ref.yaml +++ b/ci-operator/step-registry/aro-hcp/hypershift-images-push/aro-hcp-hypershift-images-push-ref.yaml @@ -18,9 +18,17 @@ ref: documentation: |- Config environment name. Also used to select the infrastructure subscription from Vault (infra-{ARO_HCP_DEPLOY_ENV}-subscription-id). + - name: PUSH_CPO_IMAGE + default: "false" + documentation: |- + When "true", also pushes the CPO image to the OCP ACR and writes + SHARED_DIR/hypershift-cpo-override.env. Set by jobs that test CPO + from the PR (e.g. e2e-aro-hcp-presubmit). dependencies: - name: hypershift-operator env: HYPERSHIFT_OPERATOR_IMAGE + - name: hypershift + env: HYPERSHIFT_CPO_IMAGE resources: requests: cpu: "1" @@ -30,6 +38,6 @@ ref: Writes SHARED_DIR/hypershift-image-overrides.yaml with HO config overlay (hypershift.image.repository + hypershift.image.digest), merged by hack/ci/provision-environment.sh during provisioning. - Only the HO is overridden; hosted clusters use the CPO from their OCP - release payload so the e2e test validates the HO against the production - control plane stack. + When HYPERSHIFT_CPO_IMAGE is provided (via job-level dependency override), + also pushes the control-plane-operator (CPO) to the OCP ACR and writes + SHARED_DIR/hypershift-cpo-override.env with the CPO_IMAGE_OVERRIDE ref. diff --git a/ci-operator/step-registry/aro-hcp/test/local/aro-hcp-test-local-commands.sh b/ci-operator/step-registry/aro-hcp/test/local/aro-hcp-test-local-commands.sh index db9efaede7bcd..e9ca021380a5c 100644 --- a/ci-operator/step-registry/aro-hcp/test/local/aro-hcp-test-local-commands.sh +++ b/ci-operator/step-registry/aro-hcp/test/local/aro-hcp-test-local-commands.sh @@ -15,6 +15,13 @@ source "${env_file}" export LOCATION="${SELECTED_LOCATION:-${LOCATION:-}}" : "${LOCATION:?LOCATION must be provided by SELECTED_LOCATION or the legacy runtime slot export file}" +# Source CPO image override if present (set by aro-hcp-hypershift-images-push) +cpo_env="${SHARED_DIR}/hypershift-cpo-override.env" +if [[ -f "${cpo_env}" ]]; then + # shellcheck disable=SC1090 + source "${cpo_env}" +fi + export CLUSTER_PROFILE_DIR="/var/run/aro-hcp-${VAULT_SECRET_PROFILE}" export AZURE_CLIENT_ID; AZURE_CLIENT_ID=$(cat "${CLUSTER_PROFILE_DIR}/client-id") diff --git a/ci-operator/step-registry/aro-hcp/test/local/aro-hcp-test-local-ref.yaml b/ci-operator/step-registry/aro-hcp/test/local/aro-hcp-test-local-ref.yaml index 59cb00b7f5af8..d71e9e79a8858 100644 --- a/ci-operator/step-registry/aro-hcp/test/local/aro-hcp-test-local-ref.yaml +++ b/ci-operator/step-registry/aro-hcp/test/local/aro-hcp-test-local-ref.yaml @@ -41,6 +41,22 @@ ref: documentation: |- Optional override for suite parallelism. When unset, parallelism is set by the test binary. + - name: ARO_HCP_E2E_SUITE + default: "" + documentation: |- + Override the e2e test suite to run (e.g. "hypershift-presubmit/parallel"). + When unset, the Makefile defaults to "rp-api-compat-all/parallel". + - name: ARO_HCP_OPENSHIFT_CHANNEL_GROUP + default: "" + documentation: |- + OCP version channel group for cluster creation (e.g. "nightly"). + When unset, the test framework defaults to "candidate". + - name: ARO_HCP_OPENSHIFT_VERSION_ID + default: "" + documentation: |- + OCP minor version for nightly resolution (e.g. "5.0"). + When unset, the test framework defaults to its built-in version. + Only meaningful when ARO_HCP_OPENSHIFT_CHANNEL_GROUP=nightly. documentation: |- Run the ARO HCP local e2e test suite. Fetches svc cluster kubeconfig, discovers service addresses, grants frontend ingress, registers subscription, then runs tests.