Taskfile.yml

# yaml-language-server: $schema=https://taskfile.dev/schema.json
---
version: "3"

set:
  - pipefail

run: once

vars:
  MARKDOWNLINT_CONFIG: hack/tooling/.markdownlint-cli2.yaml
  # Pinned markdownlint-cli2 version (SEC-4: Renovate-trackable, like gitleaks/
  # conftest/terramate/kyverno). Mirror this in .github/workflows/verify.yaml.
  MARKDOWNLINT_VERSION: 0.23.1
  # DOC-13: narrowed markdownlint scope — SHIPPABLE docs only. markdownlint-cli2
  # ignores config-file globs when a config is passed via --config, so these are
  # passed positionally (here + the CI step in .github/workflows/verify.yaml).
  # Keep the two lists BYTE-IDENTICAL. Machine/scratch/vendored trees (.claude/,
  # references/, agent-context/, slides/) are excluded by NOT globbing them.
  # Each glob is QUOTED so markdownlint-cli2's globby does the (recursive `**`)
  # expansion — NOT the shell, whose non-recursive `**` would silently under-lint.
  MARKDOWNLINT_GLOBS: >-
    "docs/**/*.md" "openspec/**/*.md" "modules/**/*.md" "evidence/**/*.md"
    "operator/**/*.md" "deploy/**/*.md" "tests/**/*.md" "skills/**/*.md" "*.md"
  DENYLIST: hack/tooling/sanitize-denylist.txt
  # Pinned upstream ArgoCD version (E1 bootstrap). No floating tags — the
  # install manifest and every image it references are versioned to this tag.
  ARGOCD_VERSION: v3.4.5
  KUBECONFIG: '{{.KUBECONFIG | default "./.state/kubeconfig"}}'

tasks:
  default:
    desc: Show available tasks
    cmds:
      - task --list-all
    silent: true

  lint:
    desc: Markdown and shell lint
    cmds:
      - |
        # Pinned version (SEC-4) via npx so local == CI. A PATH-installed
        # markdownlint-cli2 is used only when it already matches the pin (fast
        # path, no download); otherwise fall back to the pinned npx form. Skip
        # gracefully when neither is available (local dev without Node).
        MDL_PIN="markdownlint-cli2@{{.MARKDOWNLINT_VERSION}}"
        if command -v markdownlint-cli2 >/dev/null 2>&1 \
           && markdownlint-cli2 --help 2>&1 | grep -q "v{{.MARKDOWNLINT_VERSION}} "; then
          markdownlint-cli2 --config "{{.MARKDOWNLINT_CONFIG}}" {{.MARKDOWNLINT_GLOBS}}
        elif command -v npx >/dev/null 2>&1; then
          npx -y "$MDL_PIN" --config "{{.MARKDOWNLINT_CONFIG}}" {{.MARKDOWNLINT_GLOBS}}
        else
          echo "markdownlint-cli2 + npx not available — skip (install Node/npm for local lint)"
        fi
      - |
        if command -v shellcheck >/dev/null 2>&1; then
          shellcheck hack/scrub-denylist.sh
        else
          echo "shellcheck not installed — skip"
        fi

  scrub:
    desc: Fail if sanitize denylist patterns appear in tracked paths
    cmds:
      - bash hack/scrub-denylist.sh

  openspec:validate:
    desc: Validate OpenSpec changes when openspec CLI is installed
    cmds:
      - |
        if command -v openspec >/dev/null 2>&1; then
          openspec validate --all
        else
          echo "openspec CLI not installed — validating change folder structure only"
          test -f openspec/config.yaml
          find openspec/changes -name proposal.md | head -1 | grep -q .
        fi

  verify:
    desc: Design-phase gate — lint, scrub, openspec structure, spec test coverage, fmt, docs
    cmds:
      - task: scrub
      - task: lint
      - task: openspec:validate
      - task: test:spec
      - task: test:fmt
      - task: test:unit
      - task: test:image-digests
      - task: test:meta:ci
      - task: docs:tf:check

  test:meta:ci:
    desc: WS3/E12/E8 — offline CI-wiring + Getting Started docs guards
    cmds:
      - bash tests/meta/ws3-test1-strict-advisory.yaml
      - bash tests/meta/verify-spec-test-path-parse.sh
      - bash tests/meta/ws3-sha-pinned-actions.yaml
      - bash tests/meta/e12-deck-ci-wired.yaml
      - bash tests/meta/e12-release-deck-wired.yaml
      - bash tests/meta/e8-s04-getting-started.sh
      - bash tests/smoke/e8-s04-02.sh
      - bash tests/smoke/e8-s04-demo.sh
      - bash tests/smoke/readme-live-status.sh
      - bash tests/deck/readme-badges.sh
      - bash tests/meta/mit-license.yaml
      - bash tests/smoke/caddy-mvp-s02-offline.sh
      - bash tests/smoke/caddy-mvp-s02-03.sh
      - bash tests/smoke/caddy-mvp-s05-03-offline.sh
      - bash tests/smoke/e8-s03-01.sh
      - bash tests/meta/doc10-roadmap-status.yaml
      - bash tests/meta/e1g-standing-policy.yaml
      - bash tests/meta/e1g-standing-ttl.yaml
      - bash tests/meta/release-provenance.yaml
      - bash tests/meta/verify-fetch-depth.yaml
      - bash tests/meta/batch2-smoke-wired.yaml
      - bash tests/smoke/caddy-mvp-monitoring-gitops.sh
      - bash tests/smoke/argocd-autosync-defaults-offline.sh
      - bash tests/smoke/e1g-offline.sh
      - bash tests/smoke/e1g-traefik-gitops-offline.sh
      - bash tests/smoke/gsk-cloud-edge-gitops-offline.sh
      - bash tests/smoke/e1g-caddy-lab-cloud-route.sh
      - bash tests/smoke/e1g-grafana-lab-traefik-netpol.sh
      - bash tests/smoke/e6g-offline.sh
      - bash tests/smoke/e8b-offline.sh
      - bash tests/smoke/e13-offline.sh
      - bash tests/smoke/e14-offline.sh
      - bash tests/smoke/e10-offline.sh
      - bash tests/smoke/gsk-roll-caddy-images-offline.sh
      - bash tests/smoke/rollouts-plugin-arch-overlay.sh
      - bash tests/meta/kps-managed-cluster-components.sh

  deck:build:
    desc: E12 — build the Slidev deck (wraps tests/deck/slidev-build.sh; asserts a FRESH slides/dist)
    cmds:
      - bash tests/deck/slidev-build.sh

  deck:export:
    desc: E12 — export the Slidev deck to PDF via Playwright (wraps tests/deck/slidev-export.sh; asserts a FRESH slides/kaddy-deck.pdf)
    cmds:
      - bash tests/deck/slidev-export.sh

  test:image-digests:
    desc: E1c — fail on :latest in first-party deploy/ image refs (REQ-E1c-S03-01)
    cmds:
      - bash hack/verify-image-digests.sh

  test:fmt:
    desc: L0 — tofu fmt -check in modules/labels
    cmds:
      - |
        set -eu
        if ! command -v tofu >/dev/null 2>&1; then
          echo "tofu not installed — skip test:fmt"
          exit 0
        fi
        tofu fmt -check -recursive modules

  docs:tf:
    desc: Regenerate modules/labels README terraform-docs section
    cmds:
      - |
        set -eu
        if ! command -v terraform-docs >/dev/null 2>&1; then
          echo "terraform-docs not installed — brew install terraform-docs"; exit 1
        fi
        terraform-docs -c .terraform-docs.yml modules/labels

  docs:tf:check:
    desc: Fail if terraform-docs output drifts from committed README
    cmds:
      - |
        set -eu
        if ! command -v terraform-docs >/dev/null 2>&1; then
          echo "terraform-docs not installed — skip docs:tf:check"
          exit 0
        fi
        tmp=$(mktemp)
        cp modules/labels/README.md "$tmp"
        terraform-docs -c .terraform-docs.yml modules/labels
        if ! diff -q "$tmp" modules/labels/README.md >/dev/null; then
          echo "FAIL terraform-docs drift — run 'task docs:tf'" >&2
          exit 1
        fi
        echo "ok   modules/labels/README.md"

  test:spec:
    desc: Meta — every OpenSpec REQ has Test + Verify (STRICT_TEST_FILES=1 for epic exit)
    cmds:
      - bash hack/verify-spec-coverage.sh

  test:unit:
    desc: L0 — tofu test in modules/labels
    cmds:
      - |
        if [ -d modules/labels/tests ]; then
          (cd modules/labels && tofu test)
        else
          echo "modules/labels/tests not yet present (E1b)"
          exit 0
        fi

  test:kyverno:
    desc: L1 — kyverno CLI tests on the e1c ClusterPolicies (offline, pin v1.18.2)
    cmds:
      - |
        if command -v kyverno >/dev/null 2>&1 && [ -d tests/kyverno ]; then
          kyverno test tests/kyverno/
        else
          echo "skip kyverno test (CLI not installed or tests/kyverno absent — CI installs v1.18.2)"
          exit 0
        fi

  test:policy:
    desc: L1 — conftest on OpenTofu plan fixtures (asserts expected outcomes)
    cmds:
      - |
        if command -v conftest >/dev/null 2>&1 && [ -d policy ]; then
          # Rego unit tests prove the deny rule fires (REQ-E1b-S03-01).
          conftest verify --policy policy --policy tests/policy
          # Good fixture must PASS (all mandatory ADR-0301 keys present).
          conftest test --policy policy tests/fixtures/plan-with-tags.json
          # Bad fixture must be DENIED — invert the exit code so the gate is
          # green only when the deny genuinely fires on the missing-tags plan.
          if conftest test --policy policy tests/fixtures/plan-missing-tags.json; then
            echo "FAIL: plan-missing-tags.json was NOT denied (expected deny)" >&2
            exit 1
          else
            echo "OK: plan-missing-tags.json correctly denied"
          fi
        else
          echo "conftest or policy/ not ready — skip (E1b)"
        fi

  test:promrules:
    desc: L1 — promtool unit tests for PrometheusRule alerts (marshal)
    cmds:
      - |
        if ! command -v promtool >/dev/null 2>&1; then
          echo "promtool not installed — see docs/development/testing.md (part of prometheus release)"
          exit 1
        fi
        # TEST-10 hermeticity: marshal.test.yaml loads its rule_file from a /tmp
        # projection of the PrometheusRule CR, produced by extract-rules.sh.
        # Regenerate it every run so the gate can neither false-FAIL on a fresh
        # machine (no projection) nor false-PASS on a STALE /tmp (outdated rules
        # while the committed CR is broken). CI runs the same extract as its own
        # step; doing it here makes `task test:promrules` self-contained.
        # (The caddy-mvp suites are standalone — they load committed projections.)
        if [ -x hack/monitoring/extract-rules.sh ]; then
          hack/monitoring/extract-rules.sh
        else
          echo "hack/monitoring/extract-rules.sh missing or not executable — cannot project rules for promtool" >&2
          exit 1
        fi
        shopt -s nullglob
        found=0
        for t in tests/promtool/*.test.yaml; do
          if ! grep -Eq 'alert_rule_test|promql_expr_test' "$t"; then
            echo "skip $t (placeholder — no test cases yet, E5)"
            continue
          fi
          found=1
          echo "promtool test rules $t"
          promtool test rules "$t"
        done
        if [ "$found" = 0 ]; then echo "no active promtool tests yet (E5)"; fi

  test:chainsaw:
    desc: L2 — Kyverno Chainsaw declarative cluster tests
    cmds:
      - |
        if command -v chainsaw >/dev/null 2>&1; then
          chainsaw test tests/chainsaw "$@"
        else
          echo "chainsaw not installed — see docs/development/testing.md"
          exit 1
        fi

  test:load:
    desc: L3 — k6 load (marshal thresholds; offline structural by default)
    cmds:
      - |
        set -euo pipefail
        # Offline gate (default): structural smoke for the k6 profile.
        # Live k6 run: SCORECARD_FIXTURES=0 task test:load (requires k6 + BASE_URL).
        if [ "${SCORECARD_FIXTURES:-1}" = "1" ]; then
          bash tests/smoke/e8-s01-01.sh
          echo "OK: test:load offline (SCORECARD_FIXTURES=1)"
          exit 0
        fi
        if ! command -v k6 >/dev/null 2>&1; then
          echo "k6 not installed — install k6 or use SCORECARD_FIXTURES=1" >&2
          exit 1
        fi
        [ -f tests/load/marshal-threshold.js ] || {
          echo "missing tests/load/marshal-threshold.js" >&2
          exit 1
        }
        RATE="${RATE:-150}" BASE_URL="${BASE_URL:-http://127.0.0.1:8080}" \
          k6 run tests/load/marshal-threshold.js

  test:scorecard:
    desc: L4 — evidence capture bundle (fixture mode by default)
    cmds:
      - |
        set -euo pipefail
        if [ ! -x hack/scorecard/capture.sh ]; then
          echo "scorecard capture not ready (E8)" >&2
          exit 1
        fi
        # Offline default: synthesize from evidence/fixtures/.
        # Live capture deferred — set SCORECARD_FIXTURES=0 once live mode lands.
        if [ "${SCORECARD_FIXTURES:-1}" = "1" ]; then
          hack/scorecard/capture.sh --fixtures
        else
          hack/scorecard/capture.sh
        fi
        hack/scorecard/validate.sh

  test:operator:
    desc: E9 — operator envtest suite (REQ-E9-EXIT)
    dir: operator
    cmds:
      - make test

  test:
    desc: Run all available test levels (L0→L2; L3/L4 when implemented)
    cmds:
      - task: test:unit
      - task: test:policy
      - |
        if command -v promtool >/dev/null 2>&1; then
          task test:promrules
        else
          echo "skip promrules (no promtool)"
        fi
      - |
        if [ -d operator ] && command -v go >/dev/null 2>&1; then
          task test:operator
        else
          echo "skip operator envtest (no operator/ or go)"
        fi
      - |
        if command -v chainsaw >/dev/null 2>&1 && command -v kubectl >/dev/null 2>&1; then
          task test:chainsaw
        else
          echo "skip chainsaw (no CLI or cluster)"
        fi

  cluster:up:
    desc: E1e — bring up the local kind cluster (Cilium + Gateway API + cert-manager)
    cmds:
      - bash hack/cluster/kind-up.sh

  cluster:down:
    desc: E1e — tear down the local kind cluster
    cmds:
      - bash hack/cluster/kind-down.sh

  test:meta:e1e:
    desc: E1e — offline meta gates (kind config + secure install), no cluster needed
    cmds:
      - bash tests/meta/e1e-kind-config.sh
      - bash tests/meta/e1e-security.sh

  test:smoke:e1e:
    desc: E1e — live smoke bundle (needs a running kaddy-dev cluster; task cluster:up first)
    cmds:
      - bash tests/smoke/e1e-exit.sh

  test:smoke:e1g:
    desc: E1g — OFFLINE gate for gridscale day-0 IaC (terramate codegen + fmt + validate + tofu test + conftest; no cluster/API/creds)
    cmds:
      - bash tests/smoke/e1g-offline.sh
      - bash tests/smoke/e1g-traefik-gitops-offline.sh
      - bash tests/smoke/gsk-cloud-edge-gitops-offline.sh
      - bash tests/smoke/bootstrap-guard.sh

  e1g:generate:
    desc: E1g — regenerate Terramate codegen (provider/backend/labels) into every stack (offline, deterministic)
    cmds:
      - |
        set -eu
        if ! command -v terramate >/dev/null 2>&1; then
          echo "terramate not installed — install to regenerate codegen (see docs/runbooks/gridscale-day0.md)"; exit 1
        fi
        terramate generate

  e1g:status:
    desc: "E1g — OFFLINE standing-substrate status (E1g-S07). Reads evidence/live/.standing-marker only; soft WARN past ~14d / teardown-by; always exit 0. No creds."
    cmds:
      - bash hack/e1g/standing-status.sh status

  e1g:up:
    desc: "E1g — LIVE gridscale day-0 provisioning (COSTS MONEY; needs real creds). Bootstraps object-storage anchor, then network → k8s → lbaas."
    # Auth mapping (KEY-FACT): the provider wants GRIDSCALE_UUID/_TOKEN but .envrc
    # exports GRIDSCALE_USER_UUID/_API_KEY. Map to the TF_VAR_* the codegen'd
    # provider reads. These are the ONLY place credentials touch the stacks.
    env:
      TF_VAR_gridscale_uuid: '{{.GRIDSCALE_USER_UUID}}'
      TF_VAR_gridscale_token: '{{.GRIDSCALE_API_KEY}}'
      TF_PLUGIN_CACHE_DIR: '{{.TF_PLUGIN_CACHE_DIR | default "~/.terraform.d/plugin-cache"}}'
    cmds:
      - |
        set -euo pipefail
        : "${GRIDSCALE_USER_UUID:?set GRIDSCALE_USER_UUID (direnv .envrc) before a live apply}"
        : "${GRIDSCALE_API_KEY:?set GRIDSCALE_API_KEY (direnv .envrc) before a live apply}"
        echo "LIVE gridscale provisioning — this COSTS MONEY. Ctrl-C within 5s to abort."
        sleep 5
        # 1) Bootstrap the state anchor with LOCAL state.
        ( cd stacks/gridscale/object-storage && tofu init -input=false && tofu apply -auto-approve )
        # Downstream stacks (network → k8s → lbaas) init against the S3 backend
        # created above; supply bucket/endpoint/creds via -backend-config here.
        # See docs/runbooks/gridscale-day0.md for the exact backend-config + the
        # output→input wiring (network IPs → lbaas, GSK kubeconfig → ArgoCD).
        echo "object-storage anchor up. Provision network/k8s/lbaas per the runbook (backend-config wiring)."
        # E1g-S07: record the standing substrate so cost meter is visible (soft TTL).
        bash hack/e1g/standing-status.sh write

  e1g:down:
    desc: "E1g — LIVE ruthless teardown (destroy lbaas → k8s → network → object-storage). Run after EVERY live test (cost discipline)."
    env:
      TF_VAR_gridscale_uuid: '{{.GRIDSCALE_USER_UUID}}'
      TF_VAR_gridscale_token: '{{.GRIDSCALE_API_KEY}}'
      TF_PLUGIN_CACHE_DIR: '{{.TF_PLUGIN_CACHE_DIR | default "~/.terraform.d/plugin-cache"}}'
    cmds:
      - |
        set -euo pipefail
        : "${GRIDSCALE_USER_UUID:?set GRIDSCALE_USER_UUID before teardown}"
        : "${GRIDSCALE_API_KEY:?set GRIDSCALE_API_KEY before teardown}"
        # Reverse dependency order. object-storage last (it holds remote state).
        for s in lbaas k8s network; do
          ( cd "stacks/gridscale/${s}" && tofu destroy -auto-approve ) || true
        done
        ( cd stacks/gridscale/object-storage && tofu destroy -auto-approve ) || true
        # Scrub each stack's local `.terraform/` so a leftover backend cache from
        # this live run can't break the next OFFLINE gate (ENV-1).
        for s in lbaas k8s network object-storage; do
          rm -rf "stacks/gridscale/${s}/.terraform"
        done
        # E1g-S07: clear the standing marker so status becomes a silent no-op.
        bash hack/e1g/standing-status.sh clear
        echo "teardown complete — verify in the gridscale panel that nothing lingers."

  test:smoke:e8b:
    desc: E8b — OFFLINE gate for the live-demo env (targets/runbook/manifests + kubeconform + shellcheck; no cluster/API/creds). Set E8B_LIVE=1 for the live serve check.
    cmds:
      - bash tests/smoke/e8b-offline.sh

  e8b:up:
    desc: "E8b — LIVE on-demand demo bring-up (COSTS MONEY; DECIDED-B on-demand, ephemeral-by-default — a standing go-live substrate is a separate recorded and time-boxed carve-out, D-042/E1g-S07). Composes the E1g GSK substrate, re-syncs the GitOps app-of-apps, waits healthy. Tear down with e8b:down after the demo."
    env:
      KUBECONFIG: '{{.KUBECONFIG}}'
    cmds:
      - |
        set -euo pipefail
        echo "LIVE E8b demo bring-up — this COSTS MONEY (gridscale GSK + LBaaS). Ctrl-C within 5s to abort."
        sleep 5
        # 1) Bring up the E1g gridscale substrate (object-storage anchor, then
        #    network → k8s → lbaas per docs/runbooks/gridscale-day0.md wiring).
        task e1g:up
        echo ""
        echo "Now provision network/k8s/lbaas + thread outputs per gridscale-day0.md,"
        echo "then export KUBECONFIG to the GSK kubeconfig before the steps below."
        # 2) Re-sync the phase-1 GitOps app-of-apps onto the GSK substrate. This is
        #    the SAME bootstrap the local cluster uses (E1g-S05 substrate swap):
        #    install ArgoCD, apply the app-of-apps root — the e8b-demo child then
        #    renders the read-only scorecard + Grafana surfaces (E8b-S02).
        : "${KUBECONFIG:?export KUBECONFIG=<GSK kubeconfig> before re-syncing GitOps}"
        task bootstrap:argocd
        task bootstrap:e3
        # 3) Wait for the demo surfaces to come up (serve/health proven by
        #    tests/smoke/e8b-serve.sh; run `E8B_LIVE=1 task test:smoke:e8b`).
        kubectl -n argocd rollout status deploy/argocd-server --timeout=600s
        echo "E8b demo up. Verify + demo per docs/runbooks/gridscale-live-demo.md, then RUN 'task e8b:down'."

  e8b:down:
    desc: "E8b — RUTHLESS teardown of the on-demand demo. Delegates to e1g:down (destroy lbaas → k8s → network → object-storage). Run after EVERY demo (cost discipline)."
    cmds:
      - |
        set -euo pipefail
        # The demo surfaces are GitOps-managed on the ephemeral GSK cluster, so
        # destroying the gridscale substrate reclaims everything. The demo is
        # ephemeral-by-default; a standing go-live substrate is a separate
        # recorded and time-boxed carve-out (D-042 / E1g-S07), not this cycle.
        task e1g:down
        echo "E8b demo torn down (gridscale substrate destroyed via e1g:down)."

  test:smoke:e10:
    desc: E10 — OFFLINE gate for the Backstage portal / IDP (manifests + kubeconform + shellcheck + ingestor-config PR-invariant + read-only RBAC; no cluster/Backstage). The live bring-up is deferred + skip-gated.
    cmds:
      - bash tests/smoke/e10-offline.sh

  test:smoke:e13:
    desc: E13 — OFFLINE gate for the gridscale Marketplace template (terramate codegen + fmt + validate + tofu test + packer fmt/validate + promtool; no API/creds/build)
    cmds:
      - bash tests/smoke/e13-offline.sh

  test:smoke:e14:
    desc: E14 — OFFLINE gate for the Nix golden image (flake.lock committed + nixpkgs-fmt + nix flake check in a nixos/nix container; skip-not-fail if docker/nix absent). The emulated image build is the live proof (task e14:build).
    cmds:
      - bash tests/smoke/e14-offline.sh

  e14:lock:
    desc: E14 — (re)generate nix/flake.lock inside a nixos/nix container (commit the result)
    cmds:
      - |
        set -eu
        command -v docker >/dev/null 2>&1 || { echo "docker not installed — E14 nix runs in a nixos/nix container"; exit 1; }
        docker run --rm \
          -e NIX_CONFIG=$'experimental-features = nix-command flakes\nfilter-syscalls = false' \
          -v "$PWD/nix":/work -w /work {{.KADDY_NIX_IMAGE | default "nixos/nix:latest"}} \
          nix flake lock
        echo "nix/flake.lock updated — commit it."

  e14:fmt:
    desc: E14 — format nix/*.nix with nixpkgs-fmt (in a nixos/nix container)
    cmds:
      - |
        set -eu
        command -v docker >/dev/null 2>&1 || { echo "docker not installed"; exit 1; }
        docker run --rm \
          -e NIX_CONFIG=$'experimental-features = nix-command flakes\nfilter-syscalls = false' \
          -v "$PWD/nix":/work -w /work {{.KADDY_NIX_IMAGE | default "nixos/nix:latest"}} \
          nix run nixpkgs#nixpkgs-fmt -- .

  e14:build:
    desc: "E14-S01 — LIVE build the x86_64 gridscale golden image (nixos-generators raw) in a --platform linux/amd64 nixos/nix container (rosetta/qemu emulation; SLOW). Output: nix/build/nixos.img."
    cmds:
      - |
        set -euo pipefail
        command -v docker >/dev/null 2>&1 || { echo "docker not installed — the Nix build runs in a nixos/nix container"; exit 1; }
        mkdir -p nix/build
        docker run --rm --platform linux/amd64 \
          -e NIX_CONFIG=$'experimental-features = nix-command flakes\nfilter-syscalls = false\nmax-jobs = auto' \
          -v "$PWD/nix":/work -w /work {{.KADDY_NIX_IMAGE | default "nixos/nix:latest"}} \
          bash -c '
            set -euo pipefail
            nix build .#gridscale-image --print-build-logs
            out=$(readlink -f result)
            img=$(find "$out" -maxdepth 1 -type f | head -1)
            echo "built image: $img ($(du -h "$img" | cut -f1))"
            cp "$img" /work/build/nixos.img
          '
        echo "Nix golden image at nix/build/nixos.img — export + register via E14-S02 (see docs/runbooks/nix-golden-image.md)."

  e13:generate:
    desc: E13 — regenerate Terramate codegen (provider/backend/labels) into the marketplace stacks (offline, deterministic)
    cmds:
      - |
        set -eu
        if ! command -v terramate >/dev/null 2>&1; then
          echo "terramate not installed — install to regenerate codegen (see docs/runbooks/gridscale-marketplace-deploy.md)"; exit 1
        fi
        terramate generate

  e13:up:
    desc: "E13/E14 — LIVE Marketplace register + import (COSTS MONEY at export/deploy; needs real creds). Applies stacks/gridscale-marketplace/{caddy,nginx,nix}. Build+export each golden image FIRST per the runbook (nix .gz via the e14-nix-image CI + upload, see docs/runbooks/nix-golden-image.md)."
    env:
      TF_VAR_gridscale_uuid: '{{.GRIDSCALE_USER_UUID}}'
      TF_VAR_gridscale_token: '{{.GRIDSCALE_API_KEY}}'
      TF_PLUGIN_CACHE_DIR: '{{.TF_PLUGIN_CACHE_DIR | default "~/.terraform.d/plugin-cache"}}'
    cmds:
      - |
        set -euo pipefail
        : "${GRIDSCALE_USER_UUID:?set GRIDSCALE_USER_UUID (direnv .envrc) before a live apply}"
        : "${GRIDSCALE_API_KEY:?set GRIDSCALE_API_KEY (direnv .envrc) before a live apply}"
        echo "LIVE gridscale Marketplace register/import — build+export the .gz golden image FIRST (see the runbook). Ctrl-C within 5s to abort."
        sleep 5
        # Register + import per engine. object_storage_path is threaded in via
        # TF_VAR_object_storage_path (the exported .gz path); backend-config wires
        # the S3 remote state (see docs/runbooks/gridscale-marketplace-deploy.md).
        for s in caddy nginx nix; do
          ( cd "stacks/gridscale-marketplace/${s}" && tofu init -input=false && tofu apply -auto-approve )
        done
        echo "Marketplace apps registered + imported (private tenant). Deploy a gridscale_server from the template per the runbook (nix: hack/e14-s03-live-prove.sh)."

  e13:down:
    desc: "E13 — LIVE teardown: destroy the Marketplace apps/imports (+ any deploy-proof server). Run after EVERY live test (cost discipline)."
    env:
      TF_VAR_gridscale_uuid: '{{.GRIDSCALE_USER_UUID}}'
      TF_VAR_gridscale_token: '{{.GRIDSCALE_API_KEY}}'
      TF_PLUGIN_CACHE_DIR: '{{.TF_PLUGIN_CACHE_DIR | default "~/.terraform.d/plugin-cache"}}'
    cmds:
      - |
        set -euo pipefail
        : "${GRIDSCALE_USER_UUID:?set GRIDSCALE_USER_UUID before teardown}"
        : "${GRIDSCALE_API_KEY:?set GRIDSCALE_API_KEY before teardown}"
        for s in nix nginx caddy; do
          ( cd "stacks/gridscale-marketplace/${s}" && tofu destroy -auto-approve ) || true
        done
        echo "Marketplace teardown complete — verify in the gridscale panel that no template/import/VM lingers."

  bootstrap:argocd:
    desc: E1 — install pinned ArgoCD + bootstrap Gateway overlay (idempotent)
    env:
      KUBECONFIG: '{{.KUBECONFIG}}'
    cmds:
      - |
        set -euo pipefail
        VER="{{.ARGOCD_VERSION}}"
        # SAFETY (E1g-S05a): never install into a non-kind context. Real remote
        # (prod) contexts may live in an ambient KUBECONFIG on this workstation —
        # refuse unless we are pointed at the isolated kind cluster, OR at the one
        # named GSK context opted-in via KADDY_GSK_CONTEXT (live gridscale demo).
        . hack/lib/guard-context.sh
        guard_writable_context
        # 1) Namespace + cmd-params-cm (server.insecure) + Gateway/Certificate/Route.
        #    Apply the overlay FIRST so argocd-server reads server.insecure on first boot.
        #    SERVER-SIDE on purpose (E1d): the overlay now carries argocd-cm keys that
        #    must MERGE with the upstream defaults (resource.exclusions etc.). A
        #    client-side apply on an object without last-applied captures the LIVE
        #    object into the annotation and then DELETES every key not in our overlay
        #    on the next run — SSA field ownership merges cleanly instead.
        kubectl apply --server-side --force-conflicts --field-manager=kaddy-bootstrap \
          -f deploy/bootstrap/argocd.yaml
        # 2) Pinned upstream ArgoCD core install (all images versioned to VER).
        #    Server-side apply: the applicationsets CRD exceeds kubectl's client-side
        #    last-applied annotation size limit (262144 bytes).
        kubectl apply --server-side --force-conflicts -n argocd \
          -f "https://raw.githubusercontent.com/argoproj/argo-cd/${VER}/manifests/install.yaml"
        # 3) Re-apply the overlay: the upstream install ships an argocd-cmd-params-cm
        #    without server.insecure and would otherwise overwrite ours. The DISTINCT
        #    field manager is load-bearing: with the default manager the overlay
        #    apply would own (and thus delete) the upstream argocd-cm defaults.
        kubectl apply --server-side --force-conflicts --field-manager=kaddy-bootstrap \
          -f deploy/bootstrap/argocd.yaml
        # 4) Ensure argocd-server is actually running with --insecure. Only restart
        #    when needed (a fresh install already boots with the cm in place; an
        #    over-write by the upstream install would drop the flag). This keeps
        #    re-runs fast and idempotent instead of forcing a rollout every time.
        #    The flag surfaces as env ARGOCD_SERVER_INSECURE (from the cm).
        if kubectl -n argocd get deploy argocd-server >/dev/null 2>&1; then
          insecure="$(kubectl -n argocd exec deploy/argocd-server -c argocd-server -- \
            printenv ARGOCD_SERVER_INSECURE 2>/dev/null || true)"
          if [ "${insecure}" != "true" ]; then
            kubectl -n argocd rollout restart deploy/argocd-server >/dev/null 2>&1 || true
          fi
        fi
        # Generous timeout: a cold install pulls the argocd/redis/dex/repo-server
        # images concurrently on first boot, which can exceed 300s on a fresh node.
        kubectl -n argocd rollout status deploy/argocd-server --timeout=600s

  test:smoke:e1:
    desc: E1 — live smoke bundle (needs a running kaddy-dev cluster; task cluster:up first)
    cmds:
      - bash tests/smoke/e1-exit.sh

  bootstrap:e3:
    desc: E3 — apply the app-of-apps (idempotent; ARGO_TARGET_REV overrides branch)
    env:
      KUBECONFIG: '{{.KUBECONFIG}}'
    vars:
      # Steady-state truth is committed `main`. Set ARGO_TARGET_REV=<branch> for a
      # LIVE pre-merge proof against an un-merged lane (see deploy/apps/README.md).
      ARGO_TARGET_REV: '{{.ARGO_TARGET_REV | default "main"}}'
    cmds:
      - |
        set -euo pipefail
        # SAFETY: never sync into a non-kind context (mirrors bootstrap:argocd).
        # (E1g-S05a) kind-kaddy-dev default; opt-in GSK via KADDY_GSK_CONTEXT.
        . hack/lib/guard-context.sh
        guard_writable_context
        rev="{{.ARGO_TARGET_REV}}"
        # E1c: the restricted AppProjects must exist BEFORE any Application
        # that references them (root itself is project: platform). The root
        # app also manages projects/ via recurse — this direct apply only
        # solves the first-boot ordering and is otherwise a no-op.
        kubectl apply -f deploy/apps/projects/
        if [ "${rev}" = "main" ]; then
          # Steady state: apply root only; Argo CD generates the children verbatim
          # from git (all pinned to main), which resolves once this lane is merged.
          kubectl apply -f deploy/apps/root.yaml
        else
          # LIVE pre-merge proof: the committed children hard-code targetRevision:
          # main, and the paths they reference do not exist on main yet. So we can
          # NOT rely on the root to generate usable children. Instead we override
          # targetRevision on root AND every child, and apply the children directly.
          # The root override also drops selfHeal so it never reverts our
          # branch-pinned children back to the main-pinned git copy. Committed
          # files are untouched (steady-state truth stays main + selfHeal:true).
          tmp="$(mktemp -d)"; trap 'rm -rf "${tmp}"' EXIT
          yq e ".spec.source.targetRevision = \"${rev}\"
               | del(.spec.syncPolicy.automated)" \
            deploy/apps/root.yaml > "${tmp}/root.yaml"
          for f in deploy/apps/*.yaml; do
            b="$(basename "${f}")"; [ "${b}" = "root.yaml" ] && continue
            yq e ".spec.source.targetRevision = \"${rev}\"" "${f}" > "${tmp}/${b}"
          done
          kubectl apply -f "${tmp}/root.yaml"
          # Apply children directly so they are branch-pinned regardless of the
          # root's generated (git-verbatim) copies.
          for f in "${tmp}"/*.yaml; do
            [ "$(basename "${f}")" = "root.yaml" ] && continue
            kubectl apply -f "${f}"
          done
        fi
        echo "app-of-apps applied (targetRevision=${rev}). Watch: kubectl get applications -n argocd"

  test:smoke:e3:
    desc: E3 — live smoke bundle (app-of-apps + observability + TLS; needs a green cluster)
    cmds:
      - bash tests/smoke/e3-exit.sh

  bootstrap:e4:
    desc: E4 — apply the clubhouse workload + platform Gateway + TLS cert (idempotent)
    env:
      KUBECONFIG: '{{.KUBECONFIG}}'
    cmds:
      - |
        set -euo pipefail
        # SAFETY: never apply into a non-kind context (mirrors bootstrap:e3).
        # (E1g-S05a) kind-kaddy-dev default; opt-in GSK via KADDY_GSK_CONTEXT.
        . hack/lib/guard-context.sh
        guard_writable_context
        kubectl apply -f deploy/gateway/namespace.yaml
        kubectl apply -f deploy/cert-manager/clubhouse-certificate.yaml
        kubectl apply -f deploy/workloads/clubhouse/
        kubectl apply -f deploy/gateway/gateway.yaml
        kubectl apply -f deploy/gateway/httproute.yaml
        kubectl apply -f deploy/gateway/httproute-redirect.yaml
        echo "E4 clubhouse + Gateway + TLS applied. Watch: kubectl -n gateway get certificate,gateway,httproute"

  test:smoke:e4:
    desc: E4 — live smoke bundle (clubhouse over verified HTTPS via the Cilium edge; needs a green cluster)
    cmds:
      - bash tests/smoke/e4-exit.sh

  bootstrap:e7:
    desc: E7 — install Argo Rollouts controller + Gateway API plugin + mulligan demo (idempotent)
    env:
      KUBECONFIG: '{{.KUBECONFIG}}'
    cmds:
      - |
        set -euo pipefail
        # SAFETY: never apply into a non-kind context (mirrors bootstrap:e4).
        # (E1g-S05a) kind-kaddy-dev default; opt-in GSK via KADDY_GSK_CONTEXT.
        . hack/lib/guard-context.sh
        guard_writable_context
        # Controller install (vendored+pinned), plugin ConfigMap + HTTPRoute RBAC.
        kubectl apply -f deploy/rollouts/namespace.yaml
        kubectl apply -f deploy/rollouts/install.yaml -n argo-rollouts
        kubectl apply -f deploy/rollouts/config.yaml
        # The controller must restart to (re)load the plugin ConfigMap.
        kubectl -n argo-rollouts rollout restart deploy/argo-rollouts
        kubectl -n argo-rollouts rollout status deploy/argo-rollouts --timeout=180s
        # Demo workloads (blue/green + canary Rollouts, demo Gateway/HTTPRoute).
        kubectl apply -f deploy/workloads/mulligan/namespace.yaml
        kubectl apply -f deploy/workloads/mulligan/
        echo "E7 rollouts + mulligan applied. Watch: kubectl -n mulligan get rollout,httproute"

  test:smoke:e7:
    desc: E7 — live smoke bundle (rollouts controller, live HTTPRoute weight shift, demo; needs a green cluster)
    cmds:
      - bash tests/smoke/e7-exit.sh

  bootstrap:e1c:
    desc: E1c — security-baseline cluster prerequisites (grafana-admin Secret; idempotent)
    env:
      KUBECONFIG: '{{.KUBECONFIG}}'
    cmds:
      - |
        set -euo pipefail
        # SAFETY: never mutate a non-kind context (mirrors bootstrap:e3).
        # (E1g-S05a) kind-kaddy-dev default; opt-in GSK via KADDY_GSK_CONTEXT.
        . hack/lib/guard-context.sh
        guard_writable_context
        # SEC-12: Grafana admin credentials live in a Kubernetes Secret with a
        # random password — never committed, never a chart default. The kps
        # values reference it via grafana.admin.existingSecret. SOPS/KSOPS
        # ownership of this Secret lands with E1d (identity epic).
        if ! kubectl -n monitoring get secret grafana-admin >/dev/null 2>&1; then
          kubectl create namespace monitoring --dry-run=client -o yaml | kubectl apply -f -
          kubectl -n monitoring create secret generic grafana-admin \
            --from-literal=admin-user=admin \
            --from-literal=admin-password="$(openssl rand -base64 24)"
          echo "created monitoring/grafana-admin (random password)"
        else
          echo "monitoring/grafana-admin already present — left untouched"
        fi
        # The Kyverno ENGINE installs via GitOps (deploy/apps/kyverno.yaml,
        # automated). The `policies` Application stays MANUAL by design —
        # sync it deliberately per deploy/policies/README.md, e.g.:
        #   argocd app sync policies --core
        echo "E1c prerequisites done. Policies app is manual-sync by design (see deploy/policies/README.md)."

  test:smoke:e1c:
    desc: E1c — live smoke bundle (Kyverno enforce matrix, netpol baseline, AppProjects, Grafana secret)
    env:
      KUBECONFIG: '{{.KUBECONFIG}}'
    cmds:
      - bash tests/smoke/e1c-exit.sh

  demo:
    desc: Choreographed mulligan demo (E7 — blue/green promote + canary live weight shift)
    env:
      KUBECONFIG: '{{.KUBECONFIG}}'
    cmds:
      - bash hack/demo/mulligan.sh

  demo:chaos:
    desc: E7 — chaos beat — abort a canary → Argo Rollouts auto-rolls HTTPRoute weights back to stable
    env:
      KUBECONFIG: '{{.KUBECONFIG}}'
    cmds:
      - bash hack/demo/mulligan-abort.sh

  demo:fire:
    desc: E5 — marshal fire demo — controlled clubhouse outage -> ClubhouseDown fires in Alertmanager -> restore -> resolved
    env:
      KUBECONFIG: '{{.KUBECONFIG}}'
    cmds:
      - bash hack/demo/marshal-fire.sh

  test:smoke:e5:
    desc: E5 — live smoke bundle (GitOps-synced marshal monitoring, probe, Grafana, THE FIRE DEMO; needs a green cluster)
    env:
      KUBECONFIG: '{{.KUBECONFIG}}'
    cmds:
      - bash tests/smoke/e5-exit.sh

  bootstrap:e1d:
    desc: E1d — KSOPS root secret + repo-server plugin + Dex OIDC wiring (idempotent)
    env:
      KUBECONFIG: '{{.KUBECONFIG}}'
    cmds:
      - |
        set -euo pipefail
        # SAFETY: never mutate a non-kind context (mirrors bootstrap:e3).
        # (E1g-S05a) kind-kaddy-dev default; opt-in GSK via KADDY_GSK_CONTEXT.
        . hack/lib/guard-context.sh
        guard_writable_context
        # 1) The ONE root secret of the SOPS chain (ADR-0110, standard KSOPS
        #    pattern): the operator age PRIVATE key, imperative by design —
        #    it can never live in git and everything downstream of it does
        #    (deploy/secrets/**.enc.yaml). Never printed; piped straight in.
        AGE_KEY_FILE="${SOPS_AGE_KEY_FILE:-${HOME}/.config/sops/age/keys.txt}"
        [ -f "${AGE_KEY_FILE}" ] \
          || { echo "refusing: age key not found at ${AGE_KEY_FILE} (set SOPS_AGE_KEY_FILE)" >&2; exit 1; }
        kubectl -n argocd create secret generic sops-age \
          --from-file=keys.txt="${AGE_KEY_FILE}" \
          --dry-run=client -o yaml | kubectl apply -f - >/dev/null
        echo "sops-age root secret applied (argocd ns)"
        # 2) Bootstrap overlay: dex Gateway listener + Certificate, argocd-cm
        #    (oidc.config + kustomize.buildOptions), argocd-rbac-cm.
        #    SERVER-SIDE with a DISTINCT field manager: merges our argocd-cm
        #    keys with the upstream defaults instead of owning/deleting them
        #    (see bootstrap:argocd steps 1+3 for the wipe gotcha).
        kubectl apply --server-side --force-conflicts --field-manager=kaddy-bootstrap \
          -f deploy/bootstrap/argocd.yaml
        # 3) KSOPS on the repo-server (committed strategic-merge patch;
        #    idempotent — patching with identical content is a no-op).
        kubectl -n argocd patch deploy argocd-repo-server \
          --patch-file deploy/bootstrap/argocd-repo-server-ksops-patch.yaml
        # 4) argocd-server must reach the Dex issuer at its PUBLIC URL
        #    (https://dex.kaddy.local:30443) for OIDC discovery + token
        #    exchange. In-cluster that URL only exists via the node-published
        #    Gateway nodePort, so pin dex.kaddy.local to the node InternalIP
        #    (dynamic per cluster — this is why it is a bootstrap patch, not
        #    a committed manifest).
        NODE_IP="$(kubectl get node -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}')"
        kubectl -n argocd patch deploy argocd-server --type strategic -p \
          "{\"spec\":{\"template\":{\"spec\":{\"hostAliases\":[{\"ip\":\"${NODE_IP}\",\"hostnames\":[\"dex.kaddy.local\"]}]}}}}"
        kubectl -n argocd rollout status deploy/argocd-repo-server --timeout=300s
        kubectl -n argocd rollout status deploy/argocd-server --timeout=300s
        echo "E1d bootstrap done. identity app syncs via GitOps; policies app stays manual:"
        echo "  argocd app sync policies --core   # identity netpols (deploy/policies/README.md)"

  test:smoke:e1d:
    desc: E1d — live smoke bundle (KSOPS render chain, Dex issuer, ArgoCD OIDC redirect, netpol)
    env:
      KUBECONFIG: '{{.KUBECONFIG}}'
    cmds:
      - bash tests/smoke/e1d-exit.sh

  bootstrap:e6:
    desc: E6 — register the Crossplane platform (AppProjects + crossplane child app; Argo CD converges the rest; idempotent)
    env:
      KUBECONFIG: '{{.KUBECONFIG}}'
    cmds:
      - |
        set -euo pipefail
        # SAFETY: never apply into a non-kind context (mirrors bootstrap:e4).
        # (E1g-S05a) kind-kaddy-dev default; opt-in GSK via KADDY_GSK_CONTEXT.
        . hack/lib/guard-context.sh
        guard_writable_context
        # Projects first (first-boot ordering — same rationale as bootstrap:e3),
        # then the crossplane child Application; Argo CD installs the pinned
        # chart, establishes the XRD/Composition and syncs the demo Website XR
        # via the workloads app (retry handles the CRD-before-CR window).
        kubectl apply -f deploy/apps/projects/platform.yaml
        kubectl apply -f deploy/apps/projects/workloads.yaml
        kubectl apply -f deploy/apps/crossplane.yaml
        # The websites namespace netpol baseline rides in the MANUAL-sync
        # policies app (deploy/policies/README.md) — sync it deliberately:
        #   argocd app sync policies --core
        echo "E6 registered. Watch: kubectl -n argocd get app crossplane; kubectl get xrd,composition"

  test:smoke:e6:
    desc: E6 — live smoke bundle (Crossplane core, Website XRD, claim -> TLS edge -> monitored; needs a green cluster)
    env:
      KUBECONFIG: '{{.KUBECONFIG}}'
    cmds:
      - bash tests/smoke/e6-exit.sh

  test:smoke:e6g:
    desc: E6g — OFFLINE gate (provider-gridscale + ProviderConfig + gridscale Website Composition; no cluster, no gridscale API). Live install DEFERRED to the E6g live cycle.
    cmds:
      - bash tests/smoke/e6g-offline.sh