✨ feat(olm): validate the bundle locally with the modern validator set

.github/workflows/ci.yaml

@@ -136,6 +136,14 @@ jobs:
           # some sonar_ko_* meta-tests (chart-rendering ones) need it.
           command -v yq >/dev/null 2>&1 || sudo snap install yq
           yq --version
+      # DIST-OH-02: dist_olm_bundle_test.sh hard-requires operator-sdk for the OperatorHub
+      # validator gate (operatorhubv2 / capabilities / categories). Installing it here is what
+      # makes that gate actually run in CI instead of skipping -- a skipped gate would read as
+      # coverage that does not exist. The gate resolves ./bin/operator-sdk first, so install there.
+      - name: Install operator-sdk (OLM bundle validators)
+        env:
+          OPERATOR_SDK_VERSION: v1.42.3
+        run: bash hack/install-operator-sdk.sh ./bin
       - name: Verify nightly advisory race job contract (HY-07)
         run: bash hack/test/hyg_07_nightly_race_test.sh
       - name: Verify manager RBAC grants core Events (not events.k8s.io)

.github/workflows/release.yaml

@@ -496,6 +496,19 @@ jobs:
         with:
           go-version-file: go.mod
 
+      # DIST-OH-02: hack/operatorhub-pr.sh validates the bundle with the modern OperatorHub
+      # validator set before pushing anything to a third-party repo, and hard-fails without
+      # operator-sdk. Install it here so that validation actually runs.
+      - name: Install operator-sdk (OLM bundle validators)
+        # Soft-fail like the submission step it serves: OperatorHub submission is
+        # discoverability only and must never fail the release pipeline. If the download
+        # fails, operatorhub-pr.sh still refuses to submit an unvalidated bundle and the
+        # outcome is reported as a warning — it fails closed, never silently open.
+        continue-on-error: true
+        env:
+          OPERATOR_SDK_VERSION: v1.42.3
+        run: bash hack/install-operator-sdk.sh ./bin
+
       - name: Generate OLM bundle and create OperatorHub PRs
         id: operatorhub
         continue-on-error: true

Makefile

@@ -164,10 +164,12 @@ KUSTOMIZE ?= $(LOCALBIN)/kustomize
 CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen
 ENVTEST ?= $(LOCALBIN)/setup-envtest
 GOLANGCI_LINT = $(LOCALBIN)/golangci-lint
+OPERATOR_SDK ?= $(LOCALBIN)/operator-sdk
 
 ## Tool Versions
 KUSTOMIZE_VERSION ?= v5.8.1
 CONTROLLER_TOOLS_VERSION ?= v0.20.1
+OPERATOR_SDK_VERSION ?= v1.42.3
 
 #ENVTEST_VERSION is the version of controller-runtime release branch to fetch the envtest setup script (i.e. release-0.20)
 ENVTEST_VERSION ?= $(shell v='$(call gomodver,sigs.k8s.io/controller-runtime)'; \
@@ -213,6 +215,15 @@ $(GOLANGCI_LINT): $(LOCALBIN)
 		mv -f $(LOCALBIN)/golangci-lint-custom $(GOLANGCI_LINT); \
 	} || true
 
+# operator-sdk is NOT installed via go-install-tool: the module pulls in
+# github.com/proglottis/gpgme, so `go install .../cmd/operator-sdk@version` needs cgo and
+# pkg-config unless built with CGO_ENABLED=0 -tags containers_image_openpgp. The published
+# release binary is checksum-verified by hack/install-operator-sdk.sh and installs in seconds.
+.PHONY: operator-sdk
+operator-sdk: $(OPERATOR_SDK) ## Download operator-sdk locally if necessary (checksum-verified release binary).
+$(OPERATOR_SDK): $(LOCALBIN)
+	OPERATOR_SDK_VERSION=$(OPERATOR_SDK_VERSION) bash hack/install-operator-sdk.sh "$(LOCALBIN)"
+
 # go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist
 # $1 - target path with name of binary
 # $2 - package url which can be installed
@@ -255,3 +266,45 @@ generate-olm-bundle: ## Generate OLM bundle for OperatorHub submission
 		cp "$$crd" "$$BUNDLE_DIR/manifests/"; \
 	done && \
 	echo "OLM bundle generated at $$BUNDLE_DIR"
+
+# DIST-OH-02: validate a generated bundle locally with the modern OperatorHub validator set.
+#
+# The upstream community-operators pipeline deprecated the "operatorhub" validator in favour of
+# OperatorHubV2Validator, StandardCapabilitiesValidator and StandardCategoriesValidator. The
+# operator-sdk CLI exposes those Go identities under the *label* names used below --
+# `--select-optional name=operatorhub/v2` is a fatal label-parse error ("a valid label must ...
+# consist of alphanumeric characters, '-', '_' or '.'"), so the CLI names deliberately differ from
+# the wording of the deprecation warning. `operator-sdk bundle validate --list-optional` is the
+# source of truth: operatorhubv2 / capabilities / categories.
+#
+# Each validator gets its OWN `bundle validate` invocation ON PURPOSE. Repeating
+# --select-optional within a single invocation does NOT union the selectors: the LAST flag wins
+# and every earlier validator is silently dropped. Collapsing these three lines into one command
+# would still exit 0 on a green bundle while only ever running the last validator -- coverage that
+# does not exist. hack/test/dist_olm_bundle_test.sh pins the one-selector-per-invocation shape.
+#
+# No cluster and no registry are contacted: the input is an on-disk bundle directory, not an image.
+.PHONY: validate-olm-bundle
+validate-olm-bundle: ## Validate a generated OLM bundle (operatorhubv2 + capabilities + categories)
+	@VERSION=$${VERSION:-$(shell git describe --tags --abbrev=0 2>/dev/null | sed "s/^v//")} && \
+	if [ -z "$$VERSION" ]; then \
+		echo "ERROR: VERSION is required (e.g. make validate-olm-bundle VERSION=0.17.0)" >&2; exit 1; \
+	fi && \
+	BUNDLE_DIR=$${BUNDLE_DIR:-dist/olm-bundle/$$VERSION} && \
+	if [ ! -d "$$BUNDLE_DIR" ]; then \
+		echo "ERROR: $$BUNDLE_DIR does not exist. Run: make generate-olm-bundle VERSION=$$VERSION IMAGE_DIGEST=sha256:..." >&2; exit 1; \
+	fi && \
+	SDK="$(OPERATOR_SDK)" && \
+	if [ ! -x "$$SDK" ]; then SDK="$$(command -v operator-sdk || true)"; fi && \
+	if [ -z "$$SDK" ]; then \
+		echo "ERROR: operator-sdk not found." >&2; \
+		echo "  Install the pinned $(OPERATOR_SDK_VERSION) release into ./bin with: make operator-sdk" >&2; \
+		echo "  (equivalently: OPERATOR_SDK_VERSION=$(OPERATOR_SDK_VERSION) bash hack/install-operator-sdk.sh ./bin)" >&2; \
+		echo "  Upstream install docs: https://sdk.operatorframework.io/docs/installation/" >&2; \
+		exit 1; \
+	fi && \
+	echo "Validating OLM bundle $$BUNDLE_DIR with $$SDK ..." && \
+	"$$SDK" bundle validate "$$BUNDLE_DIR" --select-optional name=operatorhubv2 && \
+	"$$SDK" bundle validate "$$BUNDLE_DIR" --select-optional name=capabilities && \
+	"$$SDK" bundle validate "$$BUNDLE_DIR" --select-optional name=categories && \
+	echo "OLM bundle $$BUNDLE_DIR passed operatorhubv2 + capabilities + categories"

hack/install-operator-sdk.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+# Install a pinned operator-sdk release with SHA256-verified binary download.
+# Checksums from https://github.com/operator-framework/operator-sdk/releases/download/${VERSION}/checksums.txt
+# Usage: OPERATOR_SDK_VERSION=v1.42.3 hack/install-operator-sdk.sh [install-dir]
+# Optional: KOLLECT_FORCE_SHA256=<digest> overrides the upstream expected digest (tests only).
+#
+# Why a release binary and not `go install`: the operator-sdk module pulls in
+# github.com/proglottis/gpgme, so `go install .../cmd/operator-sdk@version` needs cgo and
+# pkg-config unless it is built with CGO_ENABLED=0 -tags containers_image_openpgp. The
+# published binary is checksum-verifiable and installs in seconds instead of minutes.
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+# shellcheck source=lib/verify-sha256.sh
+source "${ROOT}/hack/lib/verify-sha256.sh"
+
+VERSION="${OPERATOR_SDK_VERSION:-v1.42.3}"
+INSTALL_DIR="${1:-${ROOT}/bin}"
+
+OS="$(uname -s | tr '[:upper:]' '[:lower:]')"
+ARCH="$(uname -m)"
+case "${ARCH}" in
+  x86_64) ARCH="amd64" ;;
+  aarch64 | arm64) ARCH="arm64" ;;
+  *)
+    echo "unsupported architecture: ${ARCH}" >&2
+    exit 1
+    ;;
+esac
+
+BINARY="operator-sdk_${OS}_${ARCH}"
+BASE_URL="https://github.com/operator-framework/operator-sdk/releases/download/${VERSION}"
+CHECKSUMS_URL="${BASE_URL}/checksums.txt"
+DOWNLOAD_URL="${BASE_URL}/${BINARY}"
+
+if [[ -n "${KOLLECT_FORCE_SHA256:-}" ]]; then
+  EXPECTED_SHA256="${KOLLECT_FORCE_SHA256}"
+else
+  EXPECTED_SHA256="$(curl -fsSL "${CHECKSUMS_URL}" | awk -v file="${BINARY}" '$2 == file {print $1}')"
+fi
+if [[ -z "${EXPECTED_SHA256}" ]]; then
+  echo "failed to resolve checksum for ${BINARY} from ${CHECKSUMS_URL}" >&2
+  exit 1
+fi
+
+TMP_DIR="$(mktemp -d)"
+trap 'rm -rf "${TMP_DIR}"' EXIT
+
+curl -fsSL "${DOWNLOAD_URL}" -o "${TMP_DIR}/${BINARY}"
+verify_sha256 "${TMP_DIR}/${BINARY}" "${EXPECTED_SHA256}"
+
+# The release asset is a bare binary (no tarball), so install it directly.
+mkdir -p "${INSTALL_DIR}"
+install -m 0755 "${TMP_DIR}/${BINARY}" "${INSTALL_DIR}/operator-sdk"
+"${INSTALL_DIR}/operator-sdk" version

hack/operatorhub-pr.sh

@@ -60,7 +60,17 @@ while IFS= read -r crd_file; do
   fi
 done < <(find config/crd/bases -name 'kollect.dev_*.yaml' | sort)
 
-echo "Bundle verified: ${BUNDLE_DIR}"
+echo "Bundle structure verified: ${BUNDLE_DIR}"
+
+# DIST-OH-02: run the modern OperatorHub validator set BEFORE anything is pushed to a
+# third-party repo. The checks above are structural only (files present); they cannot see a
+# non-standard category, a missing alm-examples entry or a malformed minKubeVersion — all of
+# which the upstream hosted pipeline reports only after a PR exists. This target hard-fails
+# (with an install hint) when operator-sdk is missing rather than skipping, because a skipped
+# validation here is indistinguishable from a clean one.
+make validate-olm-bundle VERSION="${VERSION}" BUNDLE_DIR="${BUNDLE_DIR}"
+
+echo "Bundle validated: ${BUNDLE_DIR}"
 
 if [[ "${DRY_RUN:-0}" == "1" ]]; then
   echo "DRY_RUN=1: would submit ${BUNDLE_DIR} to k8s-operatorhub/community-operators and redhat-openshift-ecosystem/community-operators-prod as ${BRANCH}"

hack/test/dist_ci_wiring_test.sh

@@ -22,4 +22,18 @@ if [[ "${glob_line}" -ge "${lint_line}" ]]; then
   fail "dist_* glob step (line ${glob_line}) must come before task lint (line ${lint_line})"
 fi
 pass "ci.yaml globs dist_*_test.sh before lint"
+
+# DIST-OH-02: dist_olm_bundle_test.sh hard-fails without operator-sdk, so CI must install it
+# BEFORE the dist_* glob step. Ordering is the whole point: an install step placed after the
+# glob turns the OperatorHub validator gate into a permanent red instead of a working gate.
+grep -q 'hack/install-operator-sdk\.sh' "${CI_WORKFLOW}" ||
+  fail "ci.yaml must install operator-sdk for the OLM bundle validator gate"
+
+sdk_line="$(grep -n 'hack/install-operator-sdk\.sh' "${CI_WORKFLOW}" | head -1 | cut -d: -f1)"
+[[ -n "${sdk_line}" ]] || fail "could not locate the operator-sdk install step"
+if [[ "${sdk_line}" -ge "${glob_line}" ]]; then
+  fail "operator-sdk install step (line ${sdk_line}) must come before the dist_* glob (line ${glob_line})"
+fi
+pass "ci.yaml installs operator-sdk before the dist_* glob"
+
 echo "All dist CI wiring tests passed."

hack/test/dist_olm_bundle_test.sh

@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 # DIST-OLM-01: generate-olm-bundle must emit a registry+v1 bundle with all owned CRDs.
 # DIST-OH-01: every owned CRD must carry an alm-examples entry.
+# GATE-OWNED-01: spec.customresourcedefinitions.owned must cover config/crd/bases.
+# DIST-OH-02: the generated bundle must pass the modern OperatorHub validator set locally.
 set -euo pipefail
 
 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
@@ -105,6 +107,42 @@ check_alm_examples() {
 check_alm_examples "${TEMPLATE}" "CSV template"
 check_alm_examples "${csv}" "generated CSV"
 
+# owned[] coverage gate (GATE-OWNED-01). The loop above proves each config/crd/bases file is
+# COPIED into the bundle; nothing proved it is DECLARED in spec.customresourcedefinitions.owned.
+# A new CRD base that never reaches owned[] therefore shipped silently: OLM would install the
+# CRD but the console would not list the API, and alm-examples coverage would still pass because
+# it compares against owned[] -- the very list that is missing the kind.
+check_owned_covers_crd_bases() {
+  local manifest="$1" label="$2"
+  local base_kinds owned_kinds
+
+  # Per-file yq: passing several files in one call interleaves "---" document separators
+  # into the output, which would never match the owned[] list.
+  base_kinds="$(while IFS= read -r crd_file; do
+    yq '.spec.names.kind' "${crd_file}"
+  done < <(find config/crd/bases -name 'kollect.dev_*.yaml' | sort) | LC_ALL=C sort -u)"
+  owned_kinds="$(yq '.spec.customresourcedefinitions.owned[].kind' "${manifest}" | LC_ALL=C sort -u)"
+
+  # Both extractions must be non-empty, otherwise a mistyped yq path or an empty
+  # config/crd/bases would make the comparison below pass vacuously.
+  [[ -n "${base_kinds}" ]] ||
+    fail "${label}: extracted no kinds from config/crd/bases -- the owned[] coverage gate would pass vacuously"
+  [[ -n "${owned_kinds}" ]] ||
+    fail "${label}: extracted no kinds from spec.customresourcedefinitions.owned -- the owned[] coverage gate would pass vacuously"
+
+  if [[ "${base_kinds}" != "${owned_kinds}" ]]; then
+    printf 'dist olm bundle: %s spec.customresourcedefinitions.owned does not match config/crd/bases ("<" crd bases, ">" owned):\n' \
+      "${label}" >&2
+    diff <(printf '%s\n' "${base_kinds}") <(printf '%s\n' "${owned_kinds}") >&2 || true
+    fail "${label}: declare every config/crd/bases kind under spec.customresourcedefinitions.owned in config/olm/template/manifests/kollect.clusterserviceversion.yaml (and give it an alm-examples entry)"
+  fi
+
+  pass "${label}: spec.customresourcedefinitions.owned covers all $(printf '%s\n' "${base_kinds}" | wc -l | tr -d ' ') CRD bases"
+}
+
+check_owned_covers_crd_bases "${TEMPLATE}" "CSV template"
+check_owned_covers_crd_bases "${csv}" "generated CSV"
+
 # RBAC drift gate: the CSV clusterPermissions are a hand copy of the
 # controller-gen-generated config/rbac/role.yaml. Without this gate the next
 # +kubebuilder:rbac marker change would silently ship an under-privileged bundle
@@ -144,4 +182,49 @@ grep -Fq 'operators.operatorframework.io.bundle.channels.v1: stable' "${meta}" |
 
 pass "generate-olm-bundle produced complete bundle for ${VERSION}"
 
+# --- DIST-OH-02: modern OperatorHub validator set ------------------------------------------
+#
+# Until now bundle defects were only discovered after pushing to the community-operators repo
+# and reading someone else's pipeline output -- which is how the missing alm-examples were
+# found. These checks pull that feedback local, and into CI.
+
+# Shape gate for the Makefile recipe. `--select-optional` is a plain string flag, not a
+# repeatable slice: passing it several times in ONE `bundle validate` invocation does not union
+# the selectors, the LAST one wins and every earlier validator is silently dropped. A collapsed
+# recipe would still exit 0 on a healthy bundle while only ever running one validator.
+# Executable lines only -- the recipe's own comments name the anti-pattern they ban.
+RECIPE="$(awk '$0 ~ /^validate-olm-bundle:/ {f=1; next} f && $0 !~ /^\t/ {exit} f' "${ROOT}/Makefile" |
+  grep -v '^[[:space:]]*#' || true)"
+[[ -n "${RECIPE}" ]] ||
+  fail "could not extract the validate-olm-bundle recipe from Makefile -- the shape gate would pass vacuously"
+
+# grep -F needs -e here: a pattern starting with -- is otherwise parsed as an option and the
+# check silently never fires.
+INVOCATIONS="$(printf '%s\n' "${RECIPE}" | grep -o -F -e 'bundle validate' | wc -l | tr -d ' ')"
+SELECTORS="$(printf '%s\n' "${RECIPE}" | grep -o -F -e '--select-optional' | wc -l | tr -d ' ')"
+
+[[ "${SELECTORS}" -ge 3 ]] ||
+  fail "validate-olm-bundle must select at least 3 optional validators, found ${SELECTORS}"
+[[ "${INVOCATIONS}" == "${SELECTORS}" ]] ||
+  fail "validate-olm-bundle runs ${INVOCATIONS} 'bundle validate' invocation(s) for ${SELECTORS} --select-optional flag(s): give each validator its own invocation, or all but the last are silently dropped"
+
+for validator in operatorhubv2 capabilities categories; do
+  printf '%s\n' "${RECIPE}" | grep -Fq "name=${validator}" ||
+    fail "validate-olm-bundle must select the ${validator} validator (operator-sdk's CLI name for the modern OperatorHub validator set)"
+done
+
+pass "validate-olm-bundle runs one validator per bundle validate invocation (${INVOCATIONS})"
+
+# The validator run itself. operator-sdk is a hard requirement, exactly like yq and jq above:
+# a gate that skips when the tool is missing reads as coverage that does not exist.
+SDK="${ROOT}/bin/operator-sdk"
+[[ -x "${SDK}" ]] || SDK="$(command -v operator-sdk || true)"
+[[ -n "${SDK}" ]] || fail "operator-sdk is required for the OperatorHub validator gate. Install the pinned release with 'make operator-sdk' (equivalently: bash hack/install-operator-sdk.sh ./bin); upstream docs: https://sdk.operatorframework.io/docs/installation/"
+
+# Validates the on-disk bundle directory, so no cluster and no registry are contacted.
+make validate-olm-bundle VERSION="${VERSION}" ||
+  fail "generated bundle failed operator-sdk bundle validate (operatorhubv2 / capabilities / categories)"
+
+pass "generated bundle passes operatorhubv2 + capabilities + categories"
+
 echo "All dist OLM bundle tests passed."

hack/test/dist_operatorhub_pr_test.sh

@@ -59,6 +59,12 @@ grep -Fq 'DRY_RUN' "${SCRIPT}" ||
 grep -Fq 'operators/kollect' "${SCRIPT}" ||
   fail "operatorhub-pr.sh operator dir must be operators/kollect"
 
+# DIST-OH-02: the structural file-presence checks in operatorhub-pr.sh cannot see a
+# non-standard category, a missing alm-examples entry or a malformed minKubeVersion. The
+# submission path must run the modern validator set before pushing to a third-party repo.
+printf '%s\n' "${CODE}" | grep -Fq 'make validate-olm-bundle' ||
+  fail "operatorhub-pr.sh must run 'make validate-olm-bundle' before submitting the bundle upstream"
+
 grep -Fq 'operatorhub-pr:' "${WORKFLOW}" ||
   fail "release workflow must define operatorhub-pr job"
 grep -Fq 'OPERATORHUB_PAT' "${WORKFLOW}" ||
@@ -67,6 +73,8 @@ grep -Fq 'continue-on-error: true' "${WORKFLOW}" ||
   fail "release workflow operatorhub step must soft-fail"
 grep -Fq 'hack/operatorhub-pr.sh' "${WORKFLOW}" ||
   fail "release workflow must invoke hack/operatorhub-pr.sh"
+grep -Fq 'hack/install-operator-sdk.sh' "${WORKFLOW}" ||
+  fail "release workflow must install operator-sdk — operatorhub-pr.sh hard-fails without it"
 
 command -v yq >/dev/null 2>&1 ||
   fail "yq (mikefarah/yq v4) is required to inspect the operatorhub-pr job"