diff --git a/.github/workflows/chainsaw.yaml b/.github/workflows/chainsaw.yaml index 39a95d0..942c11e 100644 --- a/.github/workflows/chainsaw.yaml +++ b/.github/workflows/chainsaw.yaml @@ -1,4 +1,13 @@ -# Chainsaw CI (enabled after E3) — L2 declarative cluster e2e on kind +# Chainsaw CI (enabled after E3) — L2 declarative cluster e2e on kind. +# +# CI SUBSTRATE PARITY (2026-07-16): the CI cluster is built with the SAME recipe +# as the live kaddy-dev substrate — hack/cluster/kind-up.sh (kind config with +# disableDefaultCNI:true + kubeProxyMode:none, Cilium + Gateway API CRDs + +# LB-IPAM, cert-manager + kaddy-local-ca, all pinned via +# hack/cluster/versions.env). That un-skips the Cilium/Gateway/Rollouts/local-CA +# dependent suites (gateway, security netpol, rollouts, tls certificate-renewal) +# so the platform's headline behaviors are enforced regression here, not +# live-only proofs. name: chainsaw on: @@ -9,12 +18,14 @@ on: paths: - 'deploy/**' - 'tests/chainsaw/**' + - 'hack/cluster/**' - 'Taskfile.yml' - '.github/workflows/chainsaw.yaml' pull_request: paths: - 'deploy/**' - 'tests/chainsaw/**' + - 'hack/cluster/**' - 'Taskfile.yml' - '.github/workflows/chainsaw.yaml' @@ -24,53 +35,78 @@ permissions: jobs: chainsaw: runs-on: ubuntu-latest + # Substrate bring-up (Cilium + observability + rollouts + workloads) plus the + # full suite run budgets ~20 min; hard-stop well under the 6h default so a + # wedged install fails fast. + timeout-minutes: 35 steps: - - uses: actions/checkout@v4 + # SEC-5: all action refs pinned to full commit SHAs (version as comment), + # Renovate-compatible (github-actions manager updates sha + comment). + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Install Task - uses: arduino/setup-task@v2 + uses: arduino/setup-task@b91d5d2c96a56797b48ac1e0e89220bf64044611 # v2.0.0 with: version: 3.x repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Set up Go - uses: actions/setup-go@v5 + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 with: go-version: stable - - name: Create kind cluster - uses: helm/kind-action@v1 - with: - wait: 120s - name: Install Chainsaw # SEC-4: pin to an exact tag (was @latest) so builds are reproducible # and Renovate can track the bump. run: go install github.com/kyverno/chainsaw@v0.2.15 - # Suites are gated by `skip: true` until their epic lands (labeling→E1b, - # security→E1c, tls→E3, identity→E1d, monitoring→E5, rollouts→E7, portal→E10). - # Each epic's implementation installs its deps here (Kyverno, kube-prometheus-stack, …) - # and flips its suite's skip flag off. + - name: Install kind (pinned) + # The cluster itself is created by hack/cluster/kind-up.sh (parity with + # the live substrate) — helm/kind-action is gone. Only the kind BINARY + # is provisioned here; kubectl, helm and jq are runner-preinstalled. + # SEC-4: exact version, paired with KIND_NODE_IMAGE v1.33.1 + # (hack/cluster/versions.env). + run: | + KIND_VERSION="v0.29.0" + sudo curl -sSL -o /usr/local/bin/kind \ + "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64" + sudo chmod +x /usr/local/bin/kind + - name: Raise inotify limits (kind on a shared runner) + # The parity substrate runs many watch-heavy pods (Cilium, kps, Loki, + # Alloy, Kyverno, Rollouts) on one kind node; the runner's default + # inotify limits cause spurious "too many open files" crash-loops. + run: | + sudo sysctl -w fs.inotify.max_user_watches=524288 + sudo sysctl -w fs.inotify.max_user_instances=512 + - name: Create kind cluster (live substrate recipe — Cilium + Gateway API + LB-IPAM + cert-manager + kaddy-local-ca) + # REUSES hack/cluster/kind-up.sh — the exact scripts + versions.env pins + # the live kaddy-dev cluster is built from (that's the parity point). + # On the runner: docker is rootful (podman guards no-op), the loopback + # 30080/30443 extraPortMappings are free, and the LB-IPAM pool is carved + # from the real docker "kind" bridge subnet by install-cilium.sh. + run: | + bash hack/cluster/kind-up.sh + # hack/cluster isolates its kubeconfig at .state/kubeconfig (never the + # runner default) — export it for every following step. Taskfile's + # KUBECONFIG default points at the same file. + echo "KUBECONFIG=${GITHUB_WORKSPACE}/.state/kubeconfig" >> "$GITHUB_ENV" + # Suites still skipped here are gated by `skip: true` until their epic + # lands (identity→E1d, portal→E10, caddy-mvp→e-caddy-mvp, crossplane→needs + # Crossplane in CI, unsigned-image→cosign Enforce flip). Each epic's + # implementation installs its deps here and flips its suite's skip off. - name: Install Kyverno (labeling/security suites) run: | if ls tests/chainsaw/labeling/*.yaml tests/chainsaw/security/*.yaml >/dev/null 2>&1; then - # E1c: the labeling suite is UN-SKIPPED and runs on vanilla kind - # (the test applies its own ClusterPolicies), so this install must - # actually succeed — no failure-swallowing || true (TEST-8), and - # the same vendored+pinned manifest the live cluster syncs - # (deploy/kyverno/, v1.18.2). Server-side apply: the CRDs exceed - # client-side annotation limits. + # E1c: the labeling suite applies its own ClusterPolicies, so this + # install must actually succeed — no failure-swallowing || true + # (TEST-8), and the same vendored+pinned manifest the live cluster + # syncs (deploy/kyverno/, v1.18.2). Server-side apply: the CRDs + # exceed client-side annotation limits. kubectl apply --server-side -f deploy/kyverno/install.yaml kubectl -n kyverno rollout status deploy/kyverno-admission-controller --timeout=300s fi - # E3: install cert-manager (the ACME issuers + tls suites need it) — the - # ephemeral CI kind cluster has no E1e bootstrap. Pinned to the same version - # the live cluster runs (see runbook). The letsencrypt-staging issuer needs - # only outbound reachability, which GitHub runners have. - - name: Install cert-manager (tls suites) + # E3: cert-manager + the kaddy-local-ca ClusterIssuer are already installed + # by kind-up.sh (hack/cluster/install-cert-manager.sh, pinned + # CERT_MANAGER_VERSION) — only the ACME issuer is applied on top. + - name: Apply ACME staging issuer (tls suites) if: ${{ hashFiles('tests/chainsaw/tls/*.yaml') != '' }} run: | - helm repo add jetstack https://charts.jetstack.io - helm repo update - helm upgrade --install cert-manager jetstack/cert-manager \ - --namespace cert-manager --create-namespace \ - --version v1.18.2 --set crds.enabled=true --wait --timeout 5m # Only the STAGING issuer is exercised by the tls suite. Applying the # prod issuer too would register an account against the Let's Encrypt # PRODUCTION ACME API on every PR run — unnecessary prod-API traffic — @@ -119,5 +155,89 @@ jobs: install_app deploy/observability/alloy.yaml # The Grafana Loki datasource ConfigMap is a plain manifest (not Helm). kubectl apply -f deploy/observability/grafana-loki-datasource.yaml - - name: Run Chainsaw suites + # E7: Argo Rollouts controller + the Gateway API trafficRouting plugin — + # same vendored+pinned manifests the live cluster syncs (deploy/rollouts/). + - name: Install Argo Rollouts + Gateway API plugin (rollouts suites) + if: ${{ hashFiles('tests/chainsaw/rollouts/*.yaml') != '' }} + run: | + set -euo pipefail + kubectl apply -f deploy/rollouts/namespace.yaml + # ARCH NOTE: deploy/rollouts/config.yaml pins the plugin binary for the + # LIVE kind node arch (linux-arm64 — Apple-silicon podman). GitHub + # runners are linux-amd64, so rewrite ONLY the arch suffix of the same + # pinned v0.16.0 release asset at apply time. The live manifest stays + # byte-identical; no floating tag is introduced (SEC-4). + sed 's#gatewayapi-plugin-linux-arm64#gatewayapi-plugin-linux-amd64#' \ + deploy/rollouts/config.yaml | kubectl apply -f - + # Server-side apply: the Rollout CRD exceeds client-side annotation + # limits. -n argo-rollouts: the vendored upstream install.yaml carries + # NO namespace fields (live, the Argo CD Application destination + # supplies it — deploy/apps/rollouts.yaml). The plugin ConfigMap is + # applied BEFORE the controller starts, so it is loaded on first boot — + # no `rollout restart` needed (live needs one because Argo CD may sync + # the CM after the controller). + kubectl apply --server-side -n argo-rollouts -f deploy/rollouts/install.yaml + kubectl -n argo-rollouts rollout status deploy/argo-rollouts --timeout=300s + # E4/E7: the platform edge + demo workloads the un-skipped suites assert + # against — applied from the SAME GitOps manifests the live cluster syncs. + - name: Apply platform edge + demo workloads (gateway/tls/security/rollouts suites) + if: ${{ hashFiles('tests/chainsaw/gateway/*.yaml', 'tests/chainsaw/security/*.yaml', 'tests/chainsaw/tls/*.yaml', 'tests/chainsaw/rollouts/*.yaml') != '' }} + run: | + set -euo pipefail + # E4 edge: gateway ns + Gateway + HTTPRoutes + clubhouse app + the + # kaddy-local-ca-issued Certificate (gateway + tls + security suites). + # Namespace FIRST: `kubectl apply -f ` walks files alphabetically, + # so namespace.yaml would otherwise land after the resources it holds. + kubectl apply -f deploy/gateway/namespace.yaml + kubectl apply -f deploy/gateway/ + kubectl apply -f deploy/workloads/clubhouse/ + kubectl apply -f deploy/cert-manager/clubhouse-certificate.yaml + kubectl -n gateway rollout status deploy/clubhouse --timeout=300s + kubectl -n gateway wait --for=condition=Ready certificate/clubhouse-tls --timeout=180s + kubectl -n gateway wait --for=condition=Programmed gateway/clubhouse --timeout=180s + # E7 mulligan progressive-delivery demo (needs the Rollouts CRDs above). + if ls tests/chainsaw/rollouts/*.yaml >/dev/null 2>&1; then + kubectl apply -f deploy/workloads/mulligan/namespace.yaml + kubectl apply -f deploy/workloads/mulligan/ + kubectl -n mulligan wait --for=condition=Programmed gateway/mulligan --timeout=180s + # Both Rollouts must be Healthy before the suites patch/abort them. + kubectl -n mulligan wait --for=jsonpath='{.status.phase}'=Healthy \ + rollout/mulligan rollout/mulligan-bg --timeout=300s + fi + # E1c: the default-deny NetworkPolicy baseline (security suites) — applied + # AFTER the stacks are up, mirroring the live cutover order. Cilium (the CI + # CNI since parity) actually enforces these. + - name: Apply default-deny network baseline (security suites) + if: ${{ hashFiles('tests/chainsaw/security/*.yaml') != '' }} + run: | + set -euo pipefail + # No Argo CD runs in the ephemeral CI cluster; create the bare argocd + # namespace so its namespace-wide deny floor applies and the + # default-deny suite asserts the same namespace set as live. + kubectl create namespace argocd --dry-run=client -o yaml | kubectl apply -f - + kubectl apply -f deploy/policies/network/gateway.yaml + kubectl apply -f deploy/policies/network/monitoring.yaml + kubectl apply -f deploy/policies/network/argocd.yaml + # websites.yaml is NOT applied: the websites namespace belongs to the + # Crossplane suite, which stays live-only (no Crossplane in CI). + - name: Run Chainsaw suites (default discovery — chainsaw-test.yaml) run: task test:chainsaw + - name: Run per-scenario Chainsaw suites (non-default file names) + # LATENT-GAP FIX (found by this lane): `chainsaw test ` discovers + # ONLY files named chainsaw-test.yaml, so the per-scenario suites + # (gateway/security/monitoring/rollouts/tls/crossplane — one file per + # scenario, the tests/chainsaw/README authoring style) were NEVER + # executed by `task test:chainsaw`, in CI or anywhere. Run every + # Test-bearing file explicitly — the same invocation each suite's + # live-verification annotation documents. Sequential by construction + # (the rollouts suites mutate shared mulligan objects). skip:true + # suites still skip. Follow-up: fold this into the Taskfile target. + run: | + set -euo pipefail + grep -l '^kind: Test$' \ + tests/chainsaw/{gateway,security,monitoring,rollouts,tls,crossplane}/*.yaml \ + | sort | while read -r f; do + echo "::group::chainsaw ${f}" + chainsaw test --test-file "$(basename "$f")" "$(dirname "$f")" + echo "::endgroup::" + done diff --git a/.github/workflows/deck.yaml b/.github/workflows/deck.yaml index e9dfde4..77165d4 100644 --- a/.github/workflows/deck.yaml +++ b/.github/workflows/deck.yaml @@ -24,11 +24,11 @@ jobs: deck: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4.3.0 with: version: 10 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 22 cache: pnpm @@ -36,7 +36,7 @@ jobs: - name: deck exit gate (build + notes + wordcount + iframes + beats) run: bash tests/deck/exit-recording-ready.sh - name: Upload deck artifact (slides/dist) - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: deck-dist path: slides/dist/ diff --git a/.github/workflows/monitoring.yaml b/.github/workflows/monitoring.yaml index af8e16a..25f5b1a 100644 --- a/.github/workflows/monitoring.yaml +++ b/.github/workflows/monitoring.yaml @@ -2,6 +2,15 @@ name: monitoring on: + # TEST-9: also run on push to main (was PR-only) — the local-merge workflow + # pushes straight to main, so a PR-only gate never enforced anything there. + push: + branches: [main] + paths: + - 'deploy/monitoring/**' + - 'tests/promtool/**' + - 'Taskfile.yml' + - '.github/workflows/monitoring.yaml' pull_request: paths: - 'deploy/monitoring/**' @@ -16,9 +25,10 @@ jobs: promrules: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + # SEC-5: action refs pinned to full commit SHAs (version as comment). + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Install Task - uses: arduino/setup-task@v2 + uses: arduino/setup-task@b91d5d2c96a56797b48ac1e0e89220bf64044611 # v2.0.0 with: version: 3.x repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/showcase-image.yaml b/.github/workflows/showcase-image.yaml index 53865d3..acfad25 100644 --- a/.github/workflows/showcase-image.yaml +++ b/.github/workflows/showcase-image.yaml @@ -6,8 +6,8 @@ # The in-workflow `cosign verify` step is the proof the Kyverno keyless # attestor (issuer + workflow identity) matches what CI actually signs. # -# SEC-4: actions pinned to exact release tags (repo style). SEC-5 rider: -# upgrade to full commit-SHA pinning alongside the other workflows. +# SEC-4/SEC-5: actions pinned to full commit SHAs (version as comment), +# Renovate-compatible — done across ALL workflows (SEC-5 rider, 2026-07-16). name: showcase-image on: @@ -36,13 +36,17 @@ jobs: build-sign: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - - uses: docker/setup-buildx-action@v4.2.0 + # Multi-arch: QEMU emulation for the linux/arm64 leg — the live kind node + # is Apple-silicon (linux/arm64) and an amd64-only GHCR image fails there + # with "no match for platform in manifest". + - uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0 + - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 - name: Login to GHCR if: github.event_name != 'pull_request' - uses: docker/login-action@v4.4.0 + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -50,7 +54,7 @@ jobs: - name: Image metadata (tags + OCI labels) id: meta - uses: docker/metadata-action@v6.2.0 + uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 with: images: ${{ env.IMAGE }} # git SHA always; semver on v* tag runs; latest tracks main @@ -63,17 +67,23 @@ jobs: - name: Build (and push on non-PR) id: build - uses: docker/build-push-action@v7.3.0 + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 with: context: . file: deploy/showcase/Dockerfile push: ${{ github.event_name != 'pull_request' }} + # Published images (main + v* tags) are MULTI-ARCH so both the amd64 + # CI kind node and the arm64 live kind node (Apple-silicon podman) + # can pull the same tag. PR builds stay single-arch amd64 for speed + # (no push happens there anyway). cosign signs the manifest-LIST + # digest below, which covers both arch manifests. + platforms: ${{ github.event_name == 'pull_request' && 'linux/amd64' || 'linux/amd64,linux/arm64' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - name: Install cosign if: github.event_name != 'pull_request' - uses: sigstore/cosign-installer@v4.1.2 + uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 - name: Keyless sign (the DIGEST, not a tag) if: github.event_name != 'pull_request' diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml index fb2c478..008f394 100644 --- a/.github/workflows/verify.yaml +++ b/.github/workflows/verify.yaml @@ -17,7 +17,7 @@ jobs: verify: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 # SEC-1: secret scan in CI (not just bypassable pre-commit). Pinned to the # same gitleaks version as .pre-commit-config.yaml, using the repo config. - name: Install gitleaks @@ -28,12 +28,12 @@ jobs: - name: gitleaks (secret scan) run: gitleaks dir . --config .github/gitleaks.toml --no-banner --redact - name: Install Task - uses: arduino/setup-task@v2 + uses: arduino/setup-task@b91d5d2c96a56797b48ac1e0e89220bf64044611 # v2.0.0 with: version: 3.x repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install OpenTofu - uses: opentofu/setup-opentofu@v1 + uses: opentofu/setup-opentofu@9d84900f3238fab8cd84ce47d658d25dd008be2f # v1.0.8 - name: Install conftest run: | CONFTEST_VERSION="0.56.0" diff --git a/tests/chainsaw/crossplane/website-claim-composed.yaml b/tests/chainsaw/crossplane/website-claim-composed.yaml index 320fa44..f5bd5f6 100644 --- a/tests/chainsaw/crossplane/website-claim-composed.yaml +++ b/tests/chainsaw/crossplane/website-claim-composed.yaml @@ -14,10 +14,13 @@ metadata: name: crossplane-website-claim-composed annotations: kaddy.io/req: REQ-E6-S02-02 - # skip: true — needs Crossplane + the Website XRD/Composition + cert-manager - # + Gateway API/monitoring CRDs, which the ephemeral CI chainsaw kind cluster - # does NOT provision (same class as the gateway/tls suites). Verified PASSING - # against the live kind-kaddy-dev cluster with: + # skip: true — needs the Crossplane ENGINE + the Website XRD/Composition + # (deploy/crossplane/), which CI does not install. Since the CI substrate + # parity change (2026-07-16) the OTHER dependencies (cert-manager + + # kaddy-local-ca, Gateway API + Cilium, monitoring CRDs) DO exist in CI — + # un-skipping now only requires installing Crossplane + applying + # deploy/crossplane/ in .github/workflows/chainsaw.yaml (follow-up). + # Verified PASSING against the live kind-kaddy-dev cluster with: # chainsaw test --test-file tests/chainsaw/crossplane/website-claim-composed.yaml . kaddy.io/status: live-cluster-only-ci-substrate-lacks-crossplane spec: diff --git a/tests/chainsaw/gateway/clubhouse-ready.yaml b/tests/chainsaw/gateway/clubhouse-ready.yaml index 6103ff2..525bab2 100644 --- a/tests/chainsaw/gateway/clubhouse-ready.yaml +++ b/tests/chainsaw/gateway/clubhouse-ready.yaml @@ -6,16 +6,15 @@ metadata: name: gateway-clubhouse-ready annotations: kaddy.io/req: REQ-E4-S01-01 - # skip: true — this suite needs the clubhouse Deployment/Service in ns `gateway` - # (deploy/workloads/clubhouse), which the ephemeral CI chainsaw kind cluster does - # NOT provision (it installs cert-manager only, no Cilium/Gateway/workloads). - # Verified PASSING against the live kind-kaddy-dev cluster with: + # Un-skipped by CI substrate parity (2026-07-16): chainsaw CI now builds its + # cluster from the LIVE substrate recipe (hack/cluster/kind-up.sh — Cilium + + # Gateway API CRDs + LB-IPAM + cert-manager/kaddy-local-ca, pinned via + # hack/cluster/versions.env) and applies deploy/gateway/ + + # deploy/workloads/clubhouse/ before the suites run. Also verified PASSING + # against the live kind-kaddy-dev cluster with: # chainsaw test --test-file tests/chainsaw/gateway/clubhouse-ready.yaml . - # Un-skip once CI stands up the Cilium substrate + applies the E4 manifests - # (a .github/workflows change — infra follow-up, outside the E4 lane boundary). - kaddy.io/status: live-cluster-only-ci-substrate-lacks-gateway + kaddy.io/status: ci-enforced-2026-07-16-substrate-parity spec: - skip: true description: clubhouse Deployment has ReadyReplicas >= 1 with mandatory labels; Service exposes 8080 namespace: gateway steps: diff --git a/tests/chainsaw/gateway/root-path-200.yaml b/tests/chainsaw/gateway/root-path-200.yaml index fbd4ccf..34eefdb 100644 --- a/tests/chainsaw/gateway/root-path-200.yaml +++ b/tests/chainsaw/gateway/root-path-200.yaml @@ -7,15 +7,16 @@ metadata: name: gateway-root-path-200 annotations: kaddy.io/req: REQ-E4-S02-01 - # skip: true — needs the live Cilium Gateway + HTTPRoute + clubhouse + the - # kaddy-local-ca issuer, none of which the ephemeral CI chainsaw kind cluster - # provisions (cert-manager only; no Cilium/Gateway API/GatewayClass/local-CA). - # Verified PASSING against the live kind-kaddy-dev cluster with: + # Un-skipped by CI substrate parity (2026-07-16): chainsaw CI now builds its + # cluster from the LIVE substrate recipe (hack/cluster/kind-up.sh — Cilium + + # Gateway API CRDs + LB-IPAM + cert-manager/kaddy-local-ca, pinned via + # hack/cluster/versions.env) and applies the E4 edge manifests + # (deploy/gateway/ + clubhouse + the kaddy-local-ca-issued Certificate), so + # the verified-HTTPS curl through the Cilium Gateway runs here too. Also + # verified PASSING against the live kind-kaddy-dev cluster with: # chainsaw test --test-file tests/chainsaw/gateway/root-path-200.yaml . - # Un-skip once CI stands up the Cilium substrate (infra follow-up, out of lane). - kaddy.io/status: live-cluster-only-ci-substrate-lacks-gateway + kaddy.io/status: ci-enforced-2026-07-16-substrate-parity spec: - skip: true description: Gateway Programmed, HTTPRoute Accepted/ResolvedRefs, / returns 200 with clubhouse marker namespace: gateway steps: diff --git a/tests/chainsaw/monitoring/loki-ready.yaml b/tests/chainsaw/monitoring/loki-ready.yaml index 995272e..4b50749 100644 --- a/tests/chainsaw/monitoring/loki-ready.yaml +++ b/tests/chainsaw/monitoring/loki-ready.yaml @@ -20,9 +20,10 @@ spec: labels: app.kubernetes.io/name: loki status: - conditions: - - type: Ready - status: "True" + # jmespath filter (not a bare conditions array): chainsaw + # asserts arrays length-strictly, and this object carries more + # conditions than the one asserted. Same idiom as root-path-200. + (conditions[?type=='Ready'].status | [0]): "True" catch: - describe: apiVersion: v1 diff --git a/tests/chainsaw/monitoring/stack-ready.yaml b/tests/chainsaw/monitoring/stack-ready.yaml index fb5be29..3891357 100644 --- a/tests/chainsaw/monitoring/stack-ready.yaml +++ b/tests/chainsaw/monitoring/stack-ready.yaml @@ -21,9 +21,10 @@ spec: metadata: name: servicemonitors.monitoring.coreos.com status: - conditions: - - type: Established - status: "True" + # jmespath filter (not a bare conditions array): chainsaw + # asserts arrays length-strictly, and this object carries more + # conditions than the one asserted. Same idiom as root-path-200. + (conditions[?type=='Established'].status | [0]): "True" - name: prometheus-and-alertmanager-running try: - assert: diff --git a/tests/chainsaw/rollouts/bluegreen-blocks-bad-promotion.yaml b/tests/chainsaw/rollouts/bluegreen-blocks-bad-promotion.yaml index dfa19b8..425ca55 100644 --- a/tests/chainsaw/rollouts/bluegreen-blocks-bad-promotion.yaml +++ b/tests/chainsaw/rollouts/bluegreen-blocks-bad-promotion.yaml @@ -13,12 +13,16 @@ metadata: name: rollouts-bluegreen-blocks-bad-promotion annotations: kaddy.io/req: REQ-E7-S01-02 - # skip: true — needs the Argo Rollouts controller (absent in the CI chainsaw - # kind cluster). Verified PASSING live with: + # Un-skipped by CI substrate parity (2026-07-16): the chainsaw workflow now + # installs the Argo Rollouts controller (deploy/rollouts/, pinned v1.9.0) + # and applies deploy/workloads/mulligan/ before the suites run. Also + # verified PASSING live with: # chainsaw test --test-file tests/chainsaw/rollouts/bluegreen-blocks-bad-promotion.yaml . - kaddy.io/status: live-cluster-only-ci-substrate-lacks-rollouts-gateway + kaddy.io/status: ci-enforced-2026-07-16-substrate-parity spec: - skip: true + # Serialized alongside the canary suites: the promote/patch churn shares the + # mulligan namespace and the Rollouts controller's reconcile budget. + concurrent: false description: A new blue/green revision stays on preview (not promoted) without an explicit promote. namespace: mulligan steps: @@ -67,6 +71,10 @@ spec: (phase == 'Paused' || phase == 'Progressing'): true - assert: # Preview service has a pod-template-hash selector (green is up on preview). + # Explicit timeout (like the sibling assert): the controller only pins + # preview to green once the green pods are Available — on a loaded CI + # node that exceeds chainsaw's 30s default (observed in the first CI run). + timeout: 120s resource: apiVersion: v1 kind: Service @@ -75,7 +83,11 @@ spec: name: mulligan-bg-preview spec: selector: - (rollouts-pod-template-hash != null): true + # QUOTED identifier: unquoted, JMESPath parses the dashes as + # arithmetic (rollouts - pod - template - hash == null), so the + # expression could NEVER be true — proven by the first real CI + # execution of this suite (discovery gap fixed in this lane). + ("rollouts-pod-template-hash" != null): true catch: - get: apiVersion: argoproj.io/v1alpha1 diff --git a/tests/chainsaw/rollouts/canary-rollback.yaml b/tests/chainsaw/rollouts/canary-rollback.yaml index 7ce3f8e..e409079 100644 --- a/tests/chainsaw/rollouts/canary-rollback.yaml +++ b/tests/chainsaw/rollouts/canary-rollback.yaml @@ -11,13 +11,16 @@ metadata: name: rollouts-canary-rollback annotations: kaddy.io/req: REQ-E7-S02-02 - # skip: true — same substrate dependency as canary-weights (Rollouts + Gateway - # API plugin + Cilium Gateway), absent in the CI chainsaw kind cluster. - # Verified PASSING live with: + # Un-skipped by CI substrate parity (2026-07-16): same substrate as + # canary-weights — Cilium Gateway + Argo Rollouts + the Gateway API plugin + # (linux-amd64 in CI, pinned v0.16.0) + deploy/workloads/mulligan/, all + # stood up by the chainsaw workflow. Also verified PASSING live with: # chainsaw test --test-file tests/chainsaw/rollouts/canary-rollback.yaml . - kaddy.io/status: live-cluster-only-ci-substrate-lacks-rollouts-gateway + kaddy.io/status: ci-enforced-2026-07-16-substrate-parity spec: - skip: true + # This suite and canary-weights mutate the SAME mulligan Rollout/HTTPRoute; + # serialize so a full `chainsaw test tests/chainsaw` run cannot interleave them. + concurrent: false description: After aborting a canary the mulligan HTTPRoute canary weight returns to 0. namespace: mulligan steps: diff --git a/tests/chainsaw/rollouts/canary-weights.yaml b/tests/chainsaw/rollouts/canary-weights.yaml index 4703262..a047116 100644 --- a/tests/chainsaw/rollouts/canary-weights.yaml +++ b/tests/chainsaw/rollouts/canary-weights.yaml @@ -8,18 +8,19 @@ metadata: name: rollouts-canary-weights annotations: kaddy.io/req: REQ-E7-S02-01 - # skip: true — needs the Argo Rollouts controller, the Gateway API - # trafficRouting plugin (downloaded at controller startup), a Cilium Gateway, - # and the mulligan HTTPRoute — NONE of which the ephemeral CI chainsaw kind - # cluster provisions (it installs cert-manager only). Verified PASSING against - # the live kind-kaddy-dev cluster with: + # Un-skipped by CI substrate parity (2026-07-16): chainsaw CI now builds its + # cluster from the LIVE substrate recipe (hack/cluster/kind-up.sh — Cilium + + # Gateway API CRDs + LB-IPAM + cert-manager/kaddy-local-ca, pinned via + # hack/cluster/versions.env), installs Argo Rollouts + + # the Gateway API trafficRouting plugin (arch-rewritten to linux-amd64 for + # the runner, same pinned v0.16.0) and applies deploy/workloads/mulligan/. + # Also verified PASSING against the live kind-kaddy-dev cluster with: # chainsaw test --test-file tests/chainsaw/rollouts/canary-weights.yaml . - # (kick a canary first: kubectl -n mulligan patch rollout mulligan --type merge - # -p '{"spec":{"template":{"metadata":{"annotations":{"x":"1"}}}}}') - # Un-skip once CI stands up the Cilium + Rollouts substrate (infra follow-up). - kaddy.io/status: live-cluster-only-ci-substrate-lacks-rollouts-gateway + kaddy.io/status: ci-enforced-2026-07-16-substrate-parity spec: - skip: true + # This suite and canary-rollback mutate the SAME mulligan Rollout/HTTPRoute; + # serialize so a full `chainsaw test tests/chainsaw` run cannot interleave them. + concurrent: false description: During a canary the mulligan HTTPRoute carries a non-zero canary backend weight. namespace: mulligan steps: diff --git a/tests/chainsaw/security/default-deny.yaml b/tests/chainsaw/security/default-deny.yaml index a3eedb6..877eea4 100644 --- a/tests/chainsaw/security/default-deny.yaml +++ b/tests/chainsaw/security/default-deny.yaml @@ -9,16 +9,16 @@ metadata: name: security-default-deny annotations: kaddy.io/req: REQ-E1c-S01-01 - # skip: true — needs the live policies app (Cilium CNI + the manual-sync - # `policies` Application); the ephemeral CI chainsaw kind cluster runs - # vanilla kindnet with no netpol enforcement and never syncs - # deploy/policies/. Verified PASSING against live kind-kaddy-dev - # (2026-07-16) with: - # cd tests/chainsaw/security && chainsaw test --test-file default-deny.yaml . - # (skip flipped off locally for the run, reverted after) - kaddy.io/status: live-cluster-only-ci-substrate-lacks-cilium-policies + # Un-skipped by CI substrate parity (2026-07-16): chainsaw CI now builds its + # cluster from the LIVE substrate recipe (hack/cluster/kind-up.sh — Cilium + + # Gateway API CRDs + LB-IPAM + cert-manager/kaddy-local-ca, pinned via + # hack/cluster/versions.env), and applies the SAME + # deploy/policies/network/{gateway,monitoring,argocd}.yaml baseline the live + # policies app syncs (argocd ns is created bare in CI — no Argo CD there, + # the namespace-wide deny floor still applies). Also verified PASSING + # against live kind-kaddy-dev (2026-07-16). + kaddy.io/status: ci-enforced-2026-07-16-substrate-parity spec: - skip: true concurrent: false description: default-deny NetworkPolicies present in gateway, monitoring and argocd steps: diff --git a/tests/chainsaw/security/gateway-to-app.yaml b/tests/chainsaw/security/gateway-to-app.yaml index 89b35d4..d723bc9 100644 --- a/tests/chainsaw/security/gateway-to-app.yaml +++ b/tests/chainsaw/security/gateway-to-app.yaml @@ -15,14 +15,15 @@ metadata: name: security-gateway-to-app annotations: kaddy.io/req: REQ-E1c-S01-02 - # skip: true — same live-substrate dependency as default-deny.yaml - # (Cilium Gateway + clubhouse + policies app; CI kind has none). - # Verified PASSING against live kind-kaddy-dev (2026-07-16) with: - # cd tests/chainsaw/security && chainsaw test --test-file gateway-to-app.yaml . - # (skip flipped off locally for the run, reverted after) - kaddy.io/status: live-cluster-only-ci-substrate-lacks-cilium-policies + # Un-skipped by CI substrate parity (2026-07-16): chainsaw CI now builds its + # cluster from the LIVE substrate recipe (hack/cluster/kind-up.sh — Cilium + + # Gateway API CRDs + LB-IPAM + cert-manager/kaddy-local-ca, pinned via + # hack/cluster/versions.env), applies the E4 edge + # (deploy/gateway/ + clubhouse) AND the default-deny network baseline — + # Cilium enforces it, so the sanctioned Gateway->clubhouse path is a real + # allow-proof in CI. Also verified PASSING live (2026-07-16). + kaddy.io/status: ci-enforced-2026-07-16-substrate-parity spec: - skip: true concurrent: false description: gateway (Envoy) -> clubhouse :8080 stays allowed under default-deny steps: diff --git a/tests/chainsaw/security/unauthorized-ingress-fails.yaml b/tests/chainsaw/security/unauthorized-ingress-fails.yaml index 50ae634..cf1f0d4 100644 --- a/tests/chainsaw/security/unauthorized-ingress-fails.yaml +++ b/tests/chainsaw/security/unauthorized-ingress-fails.yaml @@ -9,15 +9,16 @@ metadata: name: security-unauthorized-ingress-fails annotations: kaddy.io/req: REQ-E1c-S01-03 - # skip: true — same live-substrate dependency as default-deny.yaml - # (Cilium netpol enforcement; CI kindnet enforces nothing, so this test - # would FALSE-PASS... actually false-FAIL open: the curl would succeed). - # Verified PASSING against live kind-kaddy-dev (2026-07-16) with: - # cd tests/chainsaw/security && chainsaw test --test-file unauthorized-ingress-fails.yaml . - # (skip flipped off locally for the run, reverted after) - kaddy.io/status: live-cluster-only-ci-substrate-lacks-cilium-policies + # Un-skipped by CI substrate parity (2026-07-16): chainsaw CI now builds its + # cluster from the LIVE substrate recipe (hack/cluster/kind-up.sh — Cilium + + # Gateway API CRDs + LB-IPAM + cert-manager/kaddy-local-ca, pinned via + # hack/cluster/versions.env), so the CNI ENFORCES + # NetworkPolicy — the deny branch is meaningful here (on kindnet the curl + # would have succeeded and failed the test open). The default-deny baseline + # is applied by the workflow before the suites run. Also verified PASSING + # live (2026-07-16). + kaddy.io/status: ci-enforced-2026-07-16-substrate-parity spec: - skip: true concurrent: false description: default-ns pod -> clubhouse :8080 direct is denied under default-deny steps: diff --git a/tests/chainsaw/tls/cert-manager-ready.yaml b/tests/chainsaw/tls/cert-manager-ready.yaml index 20fad02..df04919 100644 --- a/tests/chainsaw/tls/cert-manager-ready.yaml +++ b/tests/chainsaw/tls/cert-manager-ready.yaml @@ -21,9 +21,10 @@ spec: namespace: cert-manager name: cert-manager status: - conditions: - - type: Available - status: "True" + # jmespath filter (not a bare conditions array): chainsaw + # asserts arrays length-strictly, and this object carries more + # conditions than the one asserted. Same idiom as root-path-200. + (conditions[?type=='Available'].status | [0]): "True" - assert: resource: apiVersion: apps/v1 @@ -32,9 +33,10 @@ spec: namespace: cert-manager name: cert-manager-webhook status: - conditions: - - type: Available - status: "True" + # jmespath filter (not a bare conditions array): chainsaw + # asserts arrays length-strictly, and this object carries more + # conditions than the one asserted. Same idiom as root-path-200. + (conditions[?type=='Available'].status | [0]): "True" - assert: resource: apiVersion: apps/v1 @@ -43,9 +45,10 @@ spec: namespace: cert-manager name: cert-manager-cainjector status: - conditions: - - type: Available - status: "True" + # jmespath filter (not a bare conditions array): chainsaw + # asserts arrays length-strictly, and this object carries more + # conditions than the one asserted. Same idiom as root-path-200. + (conditions[?type=='Available'].status | [0]): "True" - name: crds-established try: - assert: @@ -55,6 +58,7 @@ spec: metadata: name: clusterissuers.cert-manager.io status: - conditions: - - type: Established - status: "True" + # jmespath filter (not a bare conditions array): chainsaw + # asserts arrays length-strictly, and this object carries more + # conditions than the one asserted. Same idiom as root-path-200. + (conditions[?type=='Established'].status | [0]): "True" diff --git a/tests/chainsaw/tls/certificate-renewal.yaml b/tests/chainsaw/tls/certificate-renewal.yaml index ef1efb9..e2ccc78 100644 --- a/tests/chainsaw/tls/certificate-renewal.yaml +++ b/tests/chainsaw/tls/certificate-renewal.yaml @@ -6,17 +6,14 @@ metadata: name: tls-certificate-renewal annotations: kaddy.io/req: REQ-E4-S03-05 - # skip: true — needs the clubhouse-tls Certificate issued by the kaddy-local-ca - # ClusterIssuer in ns `gateway`. CI's ephemeral chainsaw cluster installs - # cert-manager + the letsencrypt-staging issuer only — NOT kaddy-local-ca and - # NOT the clubhouse Certificate — so this cannot issue there. - # Verified PASSING against the live kind-kaddy-dev cluster with: + # Un-skipped by CI substrate parity (2026-07-16): kind-up.sh provisions the + # kaddy-local-ca ClusterIssuer (hack/cluster/install-cert-manager.sh) and + # the workflow applies deploy/cert-manager/clubhouse-certificate.yaml and + # waits for it to be Ready — issuance + renewal scheduling happen in CI. + # Also verified PASSING against the live kind-kaddy-dev cluster with: # chainsaw test --test-file tests/chainsaw/tls/certificate-renewal.yaml . - # Un-skip once CI provisions kaddy-local-ca + applies the clubhouse cert - # (infra follow-up, outside the E4 lane boundary). - kaddy.io/status: live-cluster-only-ci-substrate-lacks-local-ca + kaddy.io/status: ci-enforced-2026-07-16-substrate-parity spec: - skip: true description: clubhouse-tls Certificate is Ready, has renewBefore, and cert-manager set a renewalTime namespace: gateway steps: