✨ feat(lab): allowlist lab substrates so Kind-only scenarios run on Talos (LAB-DEKIND)

.github/workflows/ci.yaml

@@ -142,6 +142,13 @@ jobs:
         run: bash hack/test/core_events_rbac_test.sh
       - name: Verify changelog-sync release guard (fbb5196a3 regression lock)
         run: bash hack/test/changelog_sync_release_guard_test.sh
+      # LAB-DEKIND: this suite is the enforcement mechanism for the lab substrate allowlist
+      # (default-deny on kube contexts) and the non-Kind image-delivery policy. It was only
+      # reachable via hack/docs/verify.sh in the path-filtered docs workflow, whose filter
+      # does not include hack/lab/** — so a change to the harness itself never triggered it.
+      # Run it here, unconditionally, where the guard actually protects something.
+      - name: Lab harness offline meta-tests (substrate allowlist, image delivery, pprof)
+        run: bash hack/test/lab_harness_meta_suite.sh
       - name: Run Sonar SECURITY remediation meta-tests
         run: |
           shopt -s nullglob

.github/workflows/e2e-extended.yaml

@@ -38,6 +38,19 @@ jobs:
       - name: Verify multitenant assert diagnostics + collecting wait
         run: bash hack/test/e2e_mt_repro_harden_test.sh
 
+  webhook-existing-cluster-meta:
+    name: webhook-existing-cluster-meta
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      # LAB-DEKIND / U-08: proves the Kind path is byte-for-byte unchanged AND that the
+      # existing-cluster mode refuses a non-allowlisted context without touching it.
+      - name: Verify webhook smoke runs against an existing cluster read-only
+        run: bash hack/test/e2e_webhook_existing_cluster_test.sh
+
   gate:
     name: gate
     runs-on: ubuntu-latest

Taskfile.yml

@@ -241,11 +241,16 @@ tasks:
     cmds:
       - bash hack/perf-report.sh
 
-  perf-kind:quick:
-    desc: LAB-H10 quick Kind pprof path (offline --dry-run; live Kind maintainer opt-in)
+  perf-lab:quick:
+    desc: LAB-H10 quick pprof path (offline --dry-run; live capture on any allowlisted substrate)
     cmds:
       - bash hack/lab/perf-kind.sh --dry-run --run-id perf-quick --objects 100 --seed 42
 
+  perf-kind:quick:
+    desc: Deprecated alias for perf-lab:quick (the path is no longer Kind-only)
+    cmds:
+      - task: perf-lab:quick
+
   test-integration:
     desc: Run integration-tagged tests (testcontainers; requires Docker)
     env:
@@ -467,3 +472,13 @@ tasks:
     desc: Delete kollect-e2e kind cluster
     cmds:
       - bash hack/kind/e2e/teardown.sh
+
+  lab:webhook-smoke:
+    desc: >-
+      U-08 webhook rejection assertions against the EXISTING allowlisted cluster
+      (read-only: server dry-run only, never creates a cluster).
+      Override the target with KOLLECT_RELEASE / KOLLECT_NAMESPACE.
+    env:
+      KOLLECT_E2E_EXISTING_CLUSTER: "1"
+    cmds:
+      - bash hack/e2e/webhook-smoke.sh

hack/e2e/webhook-smoke.sh

@@ -1,39 +1,113 @@
 #!/usr/bin/env bash
 # Tier 1 webhook e2e: assert serving cert + validating webhook rejects invalid family sink CRs.
+#
+# Two modes (LAB-DEKIND / U-08) — "create a cluster" is separable from "assert against one":
+#   default                        CI/Kind. Switches to the kind-${CLUSTER_NAME} context and
+#                                  applies the valid sample for real. Unchanged behaviour.
+#   KOLLECT_E2E_EXISTING_CLUSTER=1 Run the same assertions against the cluster the CURRENT
+#                                  context points at — provided that context is on the lab
+#                                  substrate allowlist (hack/lab/substrates.conf). Never
+#                                  creates a cluster and never mutates: every apply is a
+#                                  server-side dry run, so it is safe against a live release
+#                                  that is holding evidence.
+#
+# Existing-cluster example (kumulus Talos lab, Helm release kollect-op1):
+#   KUBECONFIG=... KOLLECT_E2E_EXISTING_CLUSTER=1 \
+#     KOLLECT_RELEASE=kollect-op1 KOLLECT_NAMESPACE=kollect-op1 bash hack/e2e/webhook-smoke.sh
+#
+# Exit codes:
+#   0  assertions passed
+#   1  assertion failed
+#   2  kube context refused (not on the lab substrate allowlist)
+#   4  webhook stack not installed on the target cluster (precondition, not a product bug)
 set -euo pipefail
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # shellcheck source=../kind/common.sh
 source "${SCRIPT_DIR}/../kind/common.sh"
+# shellcheck source=../lab/lib/substrate.sh
+source "${SCRIPT_DIR}/../lab/lib/substrate.sh"
 
 readonly CLUSTER_NAME="${CLUSTER_NAME:-kollect-e2e}"
 readonly WAIT_TIMEOUT="${WAIT_TIMEOUT:-300s}"
+readonly EXISTING_CLUSTER="${KOLLECT_E2E_EXISTING_CLUSTER:-0}"
+readonly TEST_NAMESPACE="${KOLLECT_E2E_TEST_NAMESPACE:-default}"
 
 _kind_require kubectl
-kind_use_context "$CLUSTER_NAME"
 
 _log() { echo "[webhook-smoke] $*"; }
 
-_log "Waiting for webhook serving Certificate Ready..."
-kubectl wait --for=condition=Ready "certificate/${KOLLECT_RELEASE}-serving-cert" \
-  -n "$KOLLECT_NAMESPACE" \
-  --timeout="$WAIT_TIMEOUT"
-
-_log "Asserting ValidatingWebhookConfiguration registered..."
-if ! kubectl get validatingwebhookconfiguration "${KOLLECT_RELEASE}-validating-webhook-configuration" \
-  >/dev/null 2>&1; then
-  kubectl get validatingwebhookconfiguration
+# KOLLECT_E2E_TEST_NAMESPACE reaches a manifest that is piped to the API server. Validate it
+# as a DNS-1123 label so it can never carry YAML — the manifest itself stays a quoted
+# heredoc and the namespace is supplied via `kubectl -n`, so nothing is interpolated into it.
+if [[ ! "${TEST_NAMESPACE}" =~ ^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$ ]]; then
+  echo "invalid KOLLECT_E2E_TEST_NAMESPACE '${TEST_NAMESPACE}' (want a DNS-1123 label)" >&2
   exit 1
 fi
 
+# Mutating applies are the Kind-only half of this scenario. Against an existing lab cluster
+# the same admission decision is observable with a server-side dry run, so the accept
+# assertion is downgraded to --dry-run=server there (explicit --dry-run=none on Kind keeps
+# the CI behaviour visible rather than implied).
+APPLY_MODE="none"
+
+if ! kollect_e2e_select_context "$CLUSTER_NAME"; then
+  _log "existing-cluster mode requires an allowlisted lab context; refusing to assert"
+  exit 2
+fi
+if [[ "${EXISTING_CLUSTER}" == "1" ]]; then
+  APPLY_MODE="server"
+  _log "existing-cluster mode: read-only assertions (release ${KOLLECT_RELEASE}, namespace ${KOLLECT_NAMESPACE})"
+fi
+
+_webhook_stack_missing() {
+  _log "FAIL: the validating webhook stack is not installed on this cluster."
+  _log "Release '${KOLLECT_RELEASE}' in namespace '${KOLLECT_NAMESPACE}' has no"
+  _log "ValidatingWebhookConfiguration '${KOLLECT_RELEASE}-validating-webhook-configuration'."
+  _log "Install/upgrade the release with webhooks enabled (and cert-manager present), then re-run."
+  kubectl get validatingwebhookconfiguration || true
+}
+
+if [[ "${EXISTING_CLUSTER}" == "1" ]]; then
+  # cert-manager is a Kind-stack assumption: an existing lab release may serve its webhook
+  # cert another way. Only wait for the Certificate when cert-manager actually manages one.
+  if kubectl get crd certificates.cert-manager.io >/dev/null 2>&1 \
+    && kubectl get certificate "${KOLLECT_RELEASE}-serving-cert" -n "$KOLLECT_NAMESPACE" >/dev/null 2>&1; then
+    _log "Waiting for webhook serving Certificate Ready..."
+    kubectl wait --for=condition=Ready "certificate/${KOLLECT_RELEASE}-serving-cert" \
+      -n "$KOLLECT_NAMESPACE" \
+      --timeout="$WAIT_TIMEOUT"
+  else
+    _log "No cert-manager Certificate for this release; asserting the webhook itself instead."
+  fi
+
+  _log "Asserting ValidatingWebhookConfiguration registered..."
+  if ! kubectl get validatingwebhookconfiguration "${KOLLECT_RELEASE}-validating-webhook-configuration" \
+    >/dev/null 2>&1; then
+    _webhook_stack_missing
+    exit 4
+  fi
+else
+  _log "Waiting for webhook serving Certificate Ready..."
+  kubectl wait --for=condition=Ready "certificate/${KOLLECT_RELEASE}-serving-cert" \
+    -n "$KOLLECT_NAMESPACE" \
+    --timeout="$WAIT_TIMEOUT"
+
+  _log "Asserting ValidatingWebhookConfiguration registered..."
+  if ! kubectl get validatingwebhookconfiguration "${KOLLECT_RELEASE}-validating-webhook-configuration" \
+    >/dev/null 2>&1; then
+    kubectl get validatingwebhookconfiguration
+    exit 1
+  fi
+fi
+
 _log "Expect validating webhook to reject git snapshot sink without git block..."
 set +e
-reject_out="$(kubectl apply --dry-run=server -f - 2>&1 <<'EOF'
+reject_out="$(kubectl apply -n "${TEST_NAMESPACE}" --dry-run=server -f - 2>&1 <<'EOF'
 apiVersion: kollect.dev/v1alpha1
 kind: KollectSnapshotSink
 metadata:
   name: webhook-reject-test
-  namespace: default
 spec:
   type: git
   endpoint: https://example.com/repo.git
@@ -45,7 +119,11 @@ if ! echo "$reject_out" | grep -Eiq 'denied|invalid|failed|Forbidden'; then
   exit 1
 fi
 
-_log "Applying valid minimal snapshot sink via webhook..."
-kubectl apply -f "${REPO_ROOT}/config/samples/e2e/snapshot-sink.yaml"
+if [[ "${EXISTING_CLUSTER}" == "1" ]]; then
+  _log "Admitting valid minimal snapshot sink through the webhook (server dry-run only)..."
+else
+  _log "Applying valid minimal snapshot sink via webhook..."
+fi
+kubectl apply "--dry-run=${APPLY_MODE}" -f "${REPO_ROOT}/config/samples/e2e/snapshot-sink.yaml"
 
 _log "Webhook smoke checks passed."

hack/kind/README.md

@@ -82,6 +82,42 @@ task kind-e2e-down
 task test:e2e
 ```
 
+## Running the assertions on a non-Kind cluster (LAB-DEKIND)
+
+Creating a cluster and asserting against one are separate steps. `common.sh` resolves the
+substrate of the **current** kube context through the lab allowlist
+([`hack/lab/substrates.conf`](../lab/substrates.conf)) and refuses anything unlisted, so an
+install can never land on an ambient production context. Substrate also decides image
+delivery: Kind builds and side-loads, everything else **must** use a pinned registry
+reference (`KOLLECT_IMAGE=ghcr.io/platformrelay/kollect:v<semver>`) because there is no
+`kind load` equivalent — a local-only or `:latest`/`:dev` tag is rejected before install.
+
+The webhook scenario (`hack/e2e/webhook-smoke.sh`) can assert against an existing cluster
+without creating anything, using server-side dry runs only:
+
+```sh
+KOLLECT_E2E_EXISTING_CLUSTER=1 KOLLECT_RELEASE=kollect-op1 KOLLECT_NAMESPACE=kollect-op1 \
+  task lab:webhook-smoke
+```
+
+CI is unaffected: with no extra environment set, the Kind path behaves exactly as before.
+
+`kollect_e2e_select_context` in `common.sh` is the seam: it switches to `kind-<cluster>` by
+default and, in existing-cluster mode, validates the current context against the allowlist
+instead. Other scenario scripts can adopt it one line at a time.
+
+### Still Kind-only (deliberate)
+
+| Assumption | Where | Why it was left |
+| --- | --- | --- |
+| Switches to the `kind-kollect-e2e` context, then creates/deletes CRs, namespaces and sinks | `hack/kind/e2e/smoke.sh`, `bootstrap-samples.sh`, `pipeline-cli-smoke.sh`, `hack/e2e/{cert-manager,tenant-mode,multitenant,finalizer-cleanup-assert,git-export-assert}.sh` | These are *mutating* scenarios. Read-only server dry runs cannot express them, so pointing them at a lab cluster that is holding evidence is unsafe by construction. They can adopt `kollect_e2e_select_context` when a disposable lab cluster exists. |
+| Single-node `cluster.yaml`, `kindest/node` version resolution, dev NodePorts 30080/30443 | `hack/kind/e2e/cluster.yaml`, `hack/kind/dev/`, `common.sh` | Only used while *creating* a kind cluster; unreachable on an existing-cluster run. |
+| cert-manager `Certificate` gate for the webhook serving cert | `hack/e2e/webhook-smoke.sh` | Fixed for existing clusters — the wait is skipped when cert-manager does not manage the release's cert, and the webhook itself is asserted instead. |
+| `kind load docker-image` | `common.sh` | Fixed — substrate decides delivery; non-Kind requires a pinned registry reference. |
+
+No storage-class, hostPath or LoadBalancer assumptions exist in the e2e path (the e2e chart
+values request none), so nothing there blocks a bare-metal lab.
+
 ## Prerequisites (dev)
 
 | Tool | Required for |

hack/kind/common.sh

@@ -5,6 +5,11 @@ set -euo pipefail
 KIND_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 REPO_ROOT="$(cd "${KIND_DIR}/../.." && pwd)"
 
+# Substrate allowlist + image-delivery policy (LAB-DEKIND). Sourced here so the install path
+# has ONE auditable place deciding "which cluster" and "how images get there".
+# shellcheck source=../lab/lib/substrate.sh
+source "${KIND_DIR}/../lab/lib/substrate.sh"
+
 # Pin kind CLI version (matches .github/workflows/e2e-nightly.yaml).
 readonly KIND_VERSION="${KIND_VERSION:-0.32.0}"
 
@@ -117,6 +122,24 @@ kind_use_context() {
   kubectl config use-context "kind-${name}" >/dev/null
 }
 
+# Decoupling seam (LAB-DEKIND): "create/select a kind cluster" vs "assert against whatever
+# cluster we are pointed at". Scenario scripts call this instead of kind_use_context so they
+# can run on an existing lab cluster with KOLLECT_E2E_EXISTING_CLUSTER=1. Default (CI) path
+# is unchanged: switch to kind-<cluster>. Returns 2 when the current context is off-allowlist.
+kollect_e2e_select_context() {
+  local cluster="$1"
+  if [[ "${KOLLECT_E2E_EXISTING_CLUSTER:-0}" != "1" ]]; then
+    kind_use_context "$cluster"
+    return 0
+  fi
+  local ctx
+  ctx="$(kubectl config current-context 2>/dev/null || true)"
+  if ! lab_substrate_assert_context "${ctx}"; then
+    return 2
+  fi
+  return 0
+}
+
 kind_create_cluster() {
   local name="$1" config="$2"
   if kind_cluster_exists "$name"; then
@@ -200,13 +223,29 @@ kollect_helm_install() {
   local values_file="$1"
   shift || true
 
-  _kind_log "Installing kollect via Helm (values: ${values_file}, timeout ${KOLLECT_HELM_TIMEOUT})..."
+  # Split repo/tag on the LAST colon only when it follows the last slash, so a registry with
+  # a port (localhost:5000/kollect:v1) is not mangled. The chart renders repository:tag, so a
+  # digest-pinned reference is refused here rather than silently reinterpreted.
+  local image_repo image_tag
+  if [[ "$KOLLECT_IMAGE" == *"@"* ]]; then
+    _kind_log "KOLLECT_IMAGE '${KOLLECT_IMAGE}' is digest-pinned; the chart renders repository:tag — use a pinned v<semver> tag."
+    return 1
+  fi
+  if [[ "${KOLLECT_IMAGE##*/}" == *:* ]]; then
+    image_repo="${KOLLECT_IMAGE%:*}"
+    image_tag="${KOLLECT_IMAGE##*:}"
+  else
+    image_repo="$KOLLECT_IMAGE"
+    image_tag="latest"
+  fi
+
+  _kind_log "Installing kollect via Helm (values: ${values_file}, image ${image_repo}:${image_tag}, timeout ${KOLLECT_HELM_TIMEOUT})..."
   if ! helm upgrade --install "$KOLLECT_RELEASE" "$KOLLECT_HELM_CHART" \
     --namespace "$KOLLECT_NAMESPACE" \
     --create-namespace \
     -f "$values_file" \
-    --set "image.repository=${KOLLECT_IMAGE%%:*}" \
-    --set "image.tag=${KOLLECT_IMAGE##*:}" \
+    --set "image.repository=${image_repo}" \
+    --set "image.tag=${image_tag}" \
     --set image.pullPolicy=IfNotPresent \
     "$@" \
     --wait --timeout "$KOLLECT_HELM_TIMEOUT"; then
@@ -283,12 +322,48 @@ kollect_wait_manager_ready() {
     --timeout="$timeout"
 }
 
+# Resolve the substrate of the cluster the CURRENT context points at (default-deny).
+# Prints the substrate kind; returns 2 when the context is not on the lab allowlist.
+kollect_current_substrate() {
+  local ctx
+  ctx="$(kubectl config current-context 2>/dev/null || true)"
+  local kind_out
+  if ! kind_out="$(lab_substrate_resolve "${ctx}")"; then
+    lab_substrate_err "refusing kube context '${ctx}': not on the lab substrate allowlist [$(lab_substrate_allowlist_summary)]"
+    lab_substrate_err "default-deny — installs only run on a Kind cluster or an allowlisted lab cluster"
+    return 2
+  fi
+  printf '%s' "${kind_out}"
+}
+
+# Deliver the operator image to the target cluster.
+#   Kind   → build locally and side-load (`kind load docker-image`), as CI has always done.
+#   other  → there is NO side-load equivalent (Talos runs containerd on bare metal, no
+#            docker socket to import into), so the image MUST already exist in a registry at
+#            an immutable reference. Anything else is refused loudly here rather than
+#            silently running whatever the nodes cached — a stale image invalidates the run.
+kollect_deliver_image() {
+  local cluster="$1" substrate="$2"
+  if [[ "$(lab_substrate_image_delivery "$substrate")" == "sideload" ]]; then
+    kollect_build_image
+    kollect_load_image "$cluster"
+    return 0
+  fi
+  if ! lab_substrate_require_registry_image "$KOLLECT_IMAGE" "$substrate"; then
+    _kind_log "Substrate '${substrate}' cannot side-load images; set KOLLECT_IMAGE to a pushed, pinned reference."
+    return 1
+  fi
+  _kind_log "Substrate ${substrate}: using pinned registry image ${KOLLECT_IMAGE} (no side-load, no rebuild)."
+  return 0
+}
+
 kollect_install_base() {
   local cluster="$1" values_file="$2"
   shift 2 || true
 
-  kollect_build_image
-  kollect_load_image "$cluster"
+  local substrate
+  substrate="$(kollect_current_substrate)" || return 1
+  kollect_deliver_image "$cluster" "$substrate" || return 1
   kollect_wait_kube_system_ready
   kollect_helm_install "$values_file" "$@"
   kollect_wait_crds_established