CI substrate parity: chainsaw cluster = live recipe (Cilium/GW-API/Rollouts), un-skip 9 suites

.github/workflows/chainsaw.yaml

@@ -1,4 +1,13 @@
-# Chainsaw CI (enabled after E3) — L2 declarative cluster e2e on kind
+# Chainsaw CI (enabled after E3) — L2 declarative cluster e2e on kind.
+#
+# CI SUBSTRATE PARITY (2026-07-16): the CI cluster is built with the SAME recipe
+# as the live kaddy-dev substrate — hack/cluster/kind-up.sh (kind config with
+# disableDefaultCNI:true + kubeProxyMode:none, Cilium + Gateway API CRDs +
+# LB-IPAM, cert-manager + kaddy-local-ca, all pinned via
+# hack/cluster/versions.env). That un-skips the Cilium/Gateway/Rollouts/local-CA
+# dependent suites (gateway, security netpol, rollouts, tls certificate-renewal)
+# so the platform's headline behaviors are enforced regression here, not
+# live-only proofs.
 name: chainsaw
 
 on:
@@ -9,12 +18,14 @@ on:
     paths:
       - 'deploy/**'
       - 'tests/chainsaw/**'
+      - 'hack/cluster/**'
       - 'Taskfile.yml'
       - '.github/workflows/chainsaw.yaml'
   pull_request:
     paths:
       - 'deploy/**'
       - 'tests/chainsaw/**'
+      - 'hack/cluster/**'
       - 'Taskfile.yml'
       - '.github/workflows/chainsaw.yaml'
 
@@ -24,53 +35,78 @@ permissions:
 jobs:
   chainsaw:
     runs-on: ubuntu-latest
+    # Substrate bring-up (Cilium + observability + rollouts + workloads) plus the
+    # full suite run budgets ~20 min; hard-stop well under the 6h default so a
+    # wedged install fails fast.
+    timeout-minutes: 35
     steps:
-      - uses: actions/checkout@v4
+      # SEC-5: all action refs pinned to full commit SHAs (version as comment),
+      # Renovate-compatible (github-actions manager updates sha + comment).
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
       - name: Install Task
-        uses: arduino/setup-task@v2
+        uses: arduino/setup-task@b91d5d2c96a56797b48ac1e0e89220bf64044611 # v2.0.0
         with:
           version: 3.x
           repo-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Set up Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
         with:
           go-version: stable
-      - name: Create kind cluster
-        uses: helm/kind-action@v1
-        with:
-          wait: 120s
       - name: Install Chainsaw
         # SEC-4: pin to an exact tag (was @latest) so builds are reproducible
         # and Renovate can track the bump.
         run: go install github.com/kyverno/chainsaw@v0.2.15
-      # Suites are gated by `skip: true` until their epic lands (labeling→E1b,
-      # security→E1c, tls→E3, identity→E1d, monitoring→E5, rollouts→E7, portal→E10).
-      # Each epic's implementation installs its deps here (Kyverno, kube-prometheus-stack, …)
-      # and flips its suite's skip flag off.
+      - name: Install kind (pinned)
+        # The cluster itself is created by hack/cluster/kind-up.sh (parity with
+        # the live substrate) — helm/kind-action is gone. Only the kind BINARY
+        # is provisioned here; kubectl, helm and jq are runner-preinstalled.
+        # SEC-4: exact version, paired with KIND_NODE_IMAGE v1.33.1
+        # (hack/cluster/versions.env).
+        run: |
+          KIND_VERSION="v0.29.0"
+          sudo curl -sSL -o /usr/local/bin/kind \
+            "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64"
+          sudo chmod +x /usr/local/bin/kind
+      - name: Raise inotify limits (kind on a shared runner)
+        # The parity substrate runs many watch-heavy pods (Cilium, kps, Loki,
+        # Alloy, Kyverno, Rollouts) on one kind node; the runner's default
+        # inotify limits cause spurious "too many open files" crash-loops.
+        run: |
+          sudo sysctl -w fs.inotify.max_user_watches=524288
+          sudo sysctl -w fs.inotify.max_user_instances=512
+      - name: Create kind cluster (live substrate recipe — Cilium + Gateway API + LB-IPAM + cert-manager + kaddy-local-ca)
+        # REUSES hack/cluster/kind-up.sh — the exact scripts + versions.env pins
+        # the live kaddy-dev cluster is built from (that's the parity point).
+        # On the runner: docker is rootful (podman guards no-op), the loopback
+        # 30080/30443 extraPortMappings are free, and the LB-IPAM pool is carved
+        # from the real docker "kind" bridge subnet by install-cilium.sh.
+        run: |
+          bash hack/cluster/kind-up.sh
+          # hack/cluster isolates its kubeconfig at .state/kubeconfig (never the
+          # runner default) — export it for every following step. Taskfile's
+          # KUBECONFIG default points at the same file.
+          echo "KUBECONFIG=${GITHUB_WORKSPACE}/.state/kubeconfig" >> "$GITHUB_ENV"
+      # Suites still skipped here are gated by `skip: true` until their epic
+      # lands (identity→E1d, portal→E10, caddy-mvp→e-caddy-mvp, crossplane→needs
+      # Crossplane in CI, unsigned-image→cosign Enforce flip). Each epic's
+      # implementation installs its deps here and flips its suite's skip off.
       - name: Install Kyverno (labeling/security suites)
         run: |
           if ls tests/chainsaw/labeling/*.yaml tests/chainsaw/security/*.yaml >/dev/null 2>&1; then
-            # E1c: the labeling suite is UN-SKIPPED and runs on vanilla kind
-            # (the test applies its own ClusterPolicies), so this install must
-            # actually succeed — no failure-swallowing || true (TEST-8), and
-            # the same vendored+pinned manifest the live cluster syncs
-            # (deploy/kyverno/, v1.18.2). Server-side apply: the CRDs exceed
-            # client-side annotation limits.
+            # E1c: the labeling suite applies its own ClusterPolicies, so this
+            # install must actually succeed — no failure-swallowing || true
+            # (TEST-8), and the same vendored+pinned manifest the live cluster
+            # syncs (deploy/kyverno/, v1.18.2). Server-side apply: the CRDs
+            # exceed client-side annotation limits.
             kubectl apply --server-side -f deploy/kyverno/install.yaml
             kubectl -n kyverno rollout status deploy/kyverno-admission-controller --timeout=300s
           fi
-      # E3: install cert-manager (the ACME issuers + tls suites need it) — the
-      # ephemeral CI kind cluster has no E1e bootstrap. Pinned to the same version
-      # the live cluster runs (see runbook). The letsencrypt-staging issuer needs
-      # only outbound reachability, which GitHub runners have.
-      - name: Install cert-manager (tls suites)
+      # E3: cert-manager + the kaddy-local-ca ClusterIssuer are already installed
+      # by kind-up.sh (hack/cluster/install-cert-manager.sh, pinned
+      # CERT_MANAGER_VERSION) — only the ACME issuer is applied on top.
+      - name: Apply ACME staging issuer (tls suites)
         if: ${{ hashFiles('tests/chainsaw/tls/*.yaml') != '' }}
         run: |
-          helm repo add jetstack https://charts.jetstack.io
-          helm repo update
-          helm upgrade --install cert-manager jetstack/cert-manager \
-            --namespace cert-manager --create-namespace \
-            --version v1.18.2 --set crds.enabled=true --wait --timeout 5m
           # Only the STAGING issuer is exercised by the tls suite. Applying the
           # prod issuer too would register an account against the Let's Encrypt
           # PRODUCTION ACME API on every PR run — unnecessary prod-API traffic —
@@ -119,5 +155,89 @@ jobs:
           install_app deploy/observability/alloy.yaml
           # The Grafana Loki datasource ConfigMap is a plain manifest (not Helm).
           kubectl apply -f deploy/observability/grafana-loki-datasource.yaml
-      - name: Run Chainsaw suites
+      # E7: Argo Rollouts controller + the Gateway API trafficRouting plugin —
+      # same vendored+pinned manifests the live cluster syncs (deploy/rollouts/).
+      - name: Install Argo Rollouts + Gateway API plugin (rollouts suites)
+        if: ${{ hashFiles('tests/chainsaw/rollouts/*.yaml') != '' }}
+        run: |
+          set -euo pipefail
+          kubectl apply -f deploy/rollouts/namespace.yaml
+          # ARCH NOTE: deploy/rollouts/config.yaml pins the plugin binary for the
+          # LIVE kind node arch (linux-arm64 — Apple-silicon podman). GitHub
+          # runners are linux-amd64, so rewrite ONLY the arch suffix of the same
+          # pinned v0.16.0 release asset at apply time. The live manifest stays
+          # byte-identical; no floating tag is introduced (SEC-4).
+          sed 's#gatewayapi-plugin-linux-arm64#gatewayapi-plugin-linux-amd64#' \
+            deploy/rollouts/config.yaml | kubectl apply -f -
+          # Server-side apply: the Rollout CRD exceeds client-side annotation
+          # limits. -n argo-rollouts: the vendored upstream install.yaml carries
+          # NO namespace fields (live, the Argo CD Application destination
+          # supplies it — deploy/apps/rollouts.yaml). The plugin ConfigMap is
+          # applied BEFORE the controller starts, so it is loaded on first boot —
+          # no `rollout restart` needed (live needs one because Argo CD may sync
+          # the CM after the controller).
+          kubectl apply --server-side -n argo-rollouts -f deploy/rollouts/install.yaml
+          kubectl -n argo-rollouts rollout status deploy/argo-rollouts --timeout=300s
+      # E4/E7: the platform edge + demo workloads the un-skipped suites assert
+      # against — applied from the SAME GitOps manifests the live cluster syncs.
+      - name: Apply platform edge + demo workloads (gateway/tls/security/rollouts suites)
+        if: ${{ hashFiles('tests/chainsaw/gateway/*.yaml', 'tests/chainsaw/security/*.yaml', 'tests/chainsaw/tls/*.yaml', 'tests/chainsaw/rollouts/*.yaml') != '' }}
+        run: |
+          set -euo pipefail
+          # E4 edge: gateway ns + Gateway + HTTPRoutes + clubhouse app + the
+          # kaddy-local-ca-issued Certificate (gateway + tls + security suites).
+          # Namespace FIRST: `kubectl apply -f <dir>` walks files alphabetically,
+          # so namespace.yaml would otherwise land after the resources it holds.
+          kubectl apply -f deploy/gateway/namespace.yaml
+          kubectl apply -f deploy/gateway/
+          kubectl apply -f deploy/workloads/clubhouse/
+          kubectl apply -f deploy/cert-manager/clubhouse-certificate.yaml
+          kubectl -n gateway rollout status deploy/clubhouse --timeout=300s
+          kubectl -n gateway wait --for=condition=Ready certificate/clubhouse-tls --timeout=180s
+          kubectl -n gateway wait --for=condition=Programmed gateway/clubhouse --timeout=180s
+          # E7 mulligan progressive-delivery demo (needs the Rollouts CRDs above).
+          if ls tests/chainsaw/rollouts/*.yaml >/dev/null 2>&1; then
+            kubectl apply -f deploy/workloads/mulligan/namespace.yaml
+            kubectl apply -f deploy/workloads/mulligan/
+            kubectl -n mulligan wait --for=condition=Programmed gateway/mulligan --timeout=180s
+            # Both Rollouts must be Healthy before the suites patch/abort them.
+            kubectl -n mulligan wait --for=jsonpath='{.status.phase}'=Healthy \
+              rollout/mulligan rollout/mulligan-bg --timeout=300s
+          fi
+      # E1c: the default-deny NetworkPolicy baseline (security suites) — applied
+      # AFTER the stacks are up, mirroring the live cutover order. Cilium (the CI
+      # CNI since parity) actually enforces these.
+      - name: Apply default-deny network baseline (security suites)
+        if: ${{ hashFiles('tests/chainsaw/security/*.yaml') != '' }}
+        run: |
+          set -euo pipefail
+          # No Argo CD runs in the ephemeral CI cluster; create the bare argocd
+          # namespace so its namespace-wide deny floor applies and the
+          # default-deny suite asserts the same namespace set as live.
+          kubectl create namespace argocd --dry-run=client -o yaml | kubectl apply -f -
+          kubectl apply -f deploy/policies/network/gateway.yaml
+          kubectl apply -f deploy/policies/network/monitoring.yaml
+          kubectl apply -f deploy/policies/network/argocd.yaml
+          # websites.yaml is NOT applied: the websites namespace belongs to the
+          # Crossplane suite, which stays live-only (no Crossplane in CI).
+      - name: Run Chainsaw suites (default discovery — chainsaw-test.yaml)
         run: task test:chainsaw
+      - name: Run per-scenario Chainsaw suites (non-default file names)
+        # LATENT-GAP FIX (found by this lane): `chainsaw test <dir>` discovers
+        # ONLY files named chainsaw-test.yaml, so the per-scenario suites
+        # (gateway/security/monitoring/rollouts/tls/crossplane — one file per
+        # scenario, the tests/chainsaw/README authoring style) were NEVER
+        # executed by `task test:chainsaw`, in CI or anywhere. Run every
+        # Test-bearing file explicitly — the same invocation each suite's
+        # live-verification annotation documents. Sequential by construction
+        # (the rollouts suites mutate shared mulligan objects). skip:true
+        # suites still skip. Follow-up: fold this into the Taskfile target.
+        run: |
+          set -euo pipefail
+          grep -l '^kind: Test$' \
+            tests/chainsaw/{gateway,security,monitoring,rollouts,tls,crossplane}/*.yaml \
+            | sort | while read -r f; do
+            echo "::group::chainsaw ${f}"
+            chainsaw test --test-file "$(basename "$f")" "$(dirname "$f")"
+            echo "::endgroup::"
+          done

.github/workflows/deck.yaml

@@ -24,19 +24,19 @@ jobs:
   deck:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - uses: pnpm/action-setup@v4
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+      - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4.3.0
         with:
           version: 10
-      - uses: actions/setup-node@v4
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
         with:
           node-version: 22
           cache: pnpm
           cache-dependency-path: slides/pnpm-lock.yaml
       - name: deck exit gate (build + notes + wordcount + iframes + beats)
         run: bash tests/deck/exit-recording-ready.sh
       - name: Upload deck artifact (slides/dist)
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: deck-dist
           path: slides/dist/

.github/workflows/monitoring.yaml

@@ -2,6 +2,15 @@
 name: monitoring
 
 on:
+  # TEST-9: also run on push to main (was PR-only) — the local-merge workflow
+  # pushes straight to main, so a PR-only gate never enforced anything there.
+  push:
+    branches: [main]
+    paths:
+      - 'deploy/monitoring/**'
+      - 'tests/promtool/**'
+      - 'Taskfile.yml'
+      - '.github/workflows/monitoring.yaml'
   pull_request:
     paths:
       - 'deploy/monitoring/**'
@@ -16,9 +25,10 @@ jobs:
   promrules:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      # SEC-5: action refs pinned to full commit SHAs (version as comment).
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
       - name: Install Task
-        uses: arduino/setup-task@v2
+        uses: arduino/setup-task@b91d5d2c96a56797b48ac1e0e89220bf64044611 # v2.0.0
         with:
           version: 3.x
           repo-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/showcase-image.yaml

@@ -6,8 +6,8 @@
 # The in-workflow `cosign verify` step is the proof the Kyverno keyless
 # attestor (issuer + workflow identity) matches what CI actually signs.
 #
-# SEC-4: actions pinned to exact release tags (repo style). SEC-5 rider:
-# upgrade to full commit-SHA pinning alongside the other workflows.
+# SEC-4/SEC-5: actions pinned to full commit SHAs (version as comment),
+# Renovate-compatible — done across ALL workflows (SEC-5 rider, 2026-07-16).
 name: showcase-image
 
 on:
@@ -36,21 +36,25 @@ jobs:
   build-sign:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
 
-      - uses: docker/setup-buildx-action@v4.2.0
+      # Multi-arch: QEMU emulation for the linux/arm64 leg — the live kind node
+      # is Apple-silicon (linux/arm64) and an amd64-only GHCR image fails there
+      # with "no match for platform in manifest".
+      - uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
+      - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
 
       - name: Login to GHCR
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v4.4.0
+        uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Image metadata (tags + OCI labels)
         id: meta
-        uses: docker/metadata-action@v6.2.0
+        uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0
         with:
           images: ${{ env.IMAGE }}
           # git SHA always; semver on v* tag runs; latest tracks main
@@ -63,17 +67,23 @@ jobs:
 
       - name: Build (and push on non-PR)
         id: build
-        uses: docker/build-push-action@v7.3.0
+        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
         with:
           context: .
           file: deploy/showcase/Dockerfile
           push: ${{ github.event_name != 'pull_request' }}
+          # Published images (main + v* tags) are MULTI-ARCH so both the amd64
+          # CI kind node and the arm64 live kind node (Apple-silicon podman)
+          # can pull the same tag. PR builds stay single-arch amd64 for speed
+          # (no push happens there anyway). cosign signs the manifest-LIST
+          # digest below, which covers both arch manifests.
+          platforms: ${{ github.event_name == 'pull_request' && 'linux/amd64' || 'linux/amd64,linux/arm64' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
 
       - name: Install cosign
         if: github.event_name != 'pull_request'
-        uses: sigstore/cosign-installer@v4.1.2
+        uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
 
       - name: Keyless sign (the DIGEST, not a tag)
         if: github.event_name != 'pull_request'

.github/workflows/verify.yaml

@@ -17,7 +17,7 @@ jobs:
   verify:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
       # SEC-1: secret scan in CI (not just bypassable pre-commit). Pinned to the
       # same gitleaks version as .pre-commit-config.yaml, using the repo config.
       - name: Install gitleaks
@@ -28,12 +28,12 @@ jobs:
       - name: gitleaks (secret scan)
         run: gitleaks dir . --config .github/gitleaks.toml --no-banner --redact
       - name: Install Task
-        uses: arduino/setup-task@v2
+        uses: arduino/setup-task@b91d5d2c96a56797b48ac1e0e89220bf64044611 # v2.0.0
         with:
           version: 3.x
           repo-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Install OpenTofu
-        uses: opentofu/setup-opentofu@v1
+        uses: opentofu/setup-opentofu@9d84900f3238fab8cd84ce47d658d25dd008be2f # v1.0.8
       - name: Install conftest
         run: |
           CONFTEST_VERSION="0.56.0"