PE-8787 PE-9173: upgrade kairos v4.1.2 & kairos-init v0.16.3

.arg.template

@@ -52,4 +52,59 @@ FORCE_INTERACTIVE_INSTALL=false
 # INCLUDE_MS_SECUREBOOT_KEYS=true       # Adds Microsoft Secure Boot certificates; if you export existing keys from a device, you typically won't need this
 # AUTO_ENROLL_SECUREBOOT_KEYS=false     # Set to true to automatically enroll certificates on devices in Setup Mode, useful for flashing devices without user interaction
 
+# UKI only: when UKI_BRING_YOUR_OWN_KEYS=true, controls whether the Palette
+# systemd-extension certificate (SPECTRO_EXTENSION_CERT_IMAGE) is merged into
+# the UEFI db during +uki-genkey. Leave true unless you don't want any external certificate int the db trust store. In that case, you have to 
+# ENROLL_SPECTRO_EXTENSION_CERT=true
+
+# Bundle Kubernetes binaries and the agent-provider binaries into the provider
+# image (UKI and non-UKI). 
+# false (default): on systemd >= 255. On older systemd, they ARE baked in.
+# true: bundle them regardless of the base image's systemd version. Use when you need a self-contained provider image.
+# BUNDLE_K8S_AND_AGENT_PROVIDER=false
+
 # DRBD_VERSION=9.2.13  # This variable is required for Piraeus pack for drbd module installtion.
+
+# NVIDIA GPU driver pre-install (for running the NVIDIA GPU Operator in air-gapped
+# environments with driver.enabled=false). Bakes the datacenter driver + DKMS
+# kernel modules into the Ubuntu base image so GPU nodes need no host-side network.
+# See scripts/install-nvidia-drivers.sh for details.
+# INSTALL_NVIDIA_GPU_DRIVERS=true
+# NVIDIA_DRIVER_BRANCH=580                  # Driver branch (check: apt-cache search 'nvidia-headless-.*-server')
+# NVIDIA_DRIVER_TYPE=open                   # open | proprietary. Default "open" (REQUIRED on Hopper/Blackwell,
+#                                            safe on Turing/Ampere/Ada). Set "proprietary" only for pre-Turing GPUs.
+# NVIDIA_USE_CUDA_REPO=true                 # Add NVIDIA CUDA network repo at build time (has every -server branch)
+# NVIDIA_INSTALL_FABRICMANAGER=false        # true for NVSwitch / HGX systems
+# NVIDIA_INSTALL_CONTAINER_TOOLKIT=false    # true to also pre-install nvidia-container-toolkit on host (then toolkit.enabled=false)
+# NVIDIA_REBUILD_INITRD=true                # Rebuild initrd so the nouveau blacklist applies at early boot
+
+# AMD Instinct GPU driver pre-install (for running the AMD GPU Operator in air-gapped
+# environments with driver.enable=false). Mutually exclusive with the NVIDIA options
+# above. See scripts/install-amdgpu-drivers.sh and docs/amd-gpu-airgapped.md for details.
+# INSTALL_AMD_GPU_DRIVERS=true
+# AMDGPU_DRIVER_SOURCE=dkms                  # dkms | inbox. "dkms" bakes AMD's amdgpu-dkms
+#                                            #   built against the image kernel (recommended
+#                                            #   for Instinct/MI silicon). "inbox" uses the
+#                                            #   in-tree amdgpu module from linux-modules-*
+#                                            #   and skips the AMD apt repo entirely — use
+#                                            #   when the DKMS build fails against your image
+#                                            #   kernel (see docs).
+# AMDGPU_DRIVER_RELEASE=31.40                # Only used with dkms mode. amdgpu-install
+#                                            #   release marker. Default 31.40 = ROCm 7.14
+#                                            #   GA (amdgpu 6.19.14), production driver for
+#                                            #   GPU Operator v1.5.1 and MI350P / Radeon AI
+#                                            #   PRO (RDNA4). For an older fleet on Operator
+#                                            #   v1.5.0 use 7.2.1 (amdgpu 6.16.13). AMD
+#                                            #   publishes both ROCm-alias (7.2.1, 7.2.4) and
+#                                            #   marker (30.30.x, 31.40) paths; either works.
+#                                            #   See the compatibility table in docs/.
+# AMDGPU_INSTALL_SMI=true                    # Default true. Install the amd-smi host CLI on
+#                                            #   PATH (parity with nvidia-smi), version-matched
+#                                            #   to the driver (amdrocm-amdsmi from repo.amd.com).
+#                                            #   rocm-smi is deprecated and NOT installed.
+#                                            #   Best-effort; a small slice of ROCm user-space.
+# AMDGPU_REBUILD_INITRD=false                # Default false. amdgpu is intentionally
+#                                            #   kept out of the initrd (multi-GPU init
+#                                            #   emits enough udev events to time out
+#                                            #   dracut-initqueue). It loads after
+#                                            #   switch-root via modules-load.d.

.earthlyignore

@@ -1,2 +1,6 @@
 local/
-build/*
+build/*
+# Whitelist AMD driver artifacts (produced by scripts/prebuild-amdgpu-artifact.sh
+# and consumed by the base-image target via COPY). Without this exception,
+# `build/*` above would hide the tarball from Earthly's build context.
+!build/amdgpu-artifact-*.tar.gz

.github/workflows/base-images.yaml

@@ -1,12 +1,299 @@
 name: Build Kairos Init Base Images
 
+# Builds Spectro Kairos base images used as Earthfile BASE_IMAGE inputs.
+#
+# A single matrix dispatches all base-image builds which do not require credentials:
+#
+#   1. kairosify (docker-bake-kairosify.hcl)
+#        Core distro cores tagged $registry_prefix/kairos-<distro>:<version>-core-...
+#          - Ubuntu 20.04 / 22.04 / 24.04
+#          - openSUSE Leap 15.6
+#          - SLEM 5.4
+#        Also supports a --base_os_image=<image> mode to kairosify an arbitrary
+#        upstream OS.
+#
+#   2. custom builders for images that kairosify cannot produce
+#        - ubuntu-uki/24.04/build.sh   → Trusted Boot Ubuntu 24.04 base with
+#                                        AMD/NVIDIA firmware trimmed (LP#1958518)
+#        - hadron/build.sh (modules)   → Hadron modules image (prerequisite)
+#        - hadron/build.sh (standard)  → Hadron base image
+#        - hadron/build.sh --fips      → Hadron FIPS base image
+#        - hadron/build.sh --uki       → Hadron Trusted Boot base image
+#
+# All specialized images are built on every dispatch. Hadron variants share
+# one matrix entry so the modules image is pushed before dependent base images.
+
 on:
   workflow_dispatch:
-
+    inputs:
+      base_os_image:
+        description: "Custom base OS for kairosify (leave empty to use default distro matrix)"
+        required: false
+        type: string
+        default: ""
+      kairos_init_image:
+        description: "Kairos Init image (its version tags all built images)"
+        required: false
+        type: string
+        default: "quay.io/kairos/kairos-init:v0.16.3"
+      arch:
+        description: "Architecture for kairosify and Ubuntu UKI (Hadron builds amd64+arm64)"
+        required: false
+        type: choice
+        options:
+          - amd64
+          - arm64
+        default: "amd64"
+      model:
+        description: "Model passed to kairos-init / kairosify"
+        required: false
+        type: string
+        default: "generic"
+      kairos_version:
+        description: "Kairos Version (kairos-init --version). Not used in image tags."
+        required: false
+        type: string
+        default: "v4.1.2"
+      hadron_version:
+        description: "Upstream Hadron version tag"
+        required: false
+        type: string
+        default: "v0.5.1"
+      registry_prefix:
+        description: "Registry prefix for output images"
+        required: false
+        type: string
+        default: "us-east1-docker.pkg.dev/spectro-images/dev/pe-8787/edge"
 jobs:
+  # Derives the kairos-init version tag and creates the complete build matrix.
   generate-matrix:
-    runs-on: ubuntu-latest
+    runs-on: Luet-BigRunner
+    outputs:
+      build_matrix: ${{ steps.set-matrix.outputs.build_matrix }}
+      init_version: ${{ steps.set-matrix.outputs.init_version }}
+    steps:
+      - name: Generate build matrix
+        id: set-matrix
+        run: |
+          python3 << 'EOF'
+          import json
+          import os
+
+          base_os_image = "${{ github.event.inputs.base_os_image }}".strip()
+          registry_prefix = "${{ github.event.inputs.registry_prefix }}".rstrip("/")
+          arch = "${{ github.event.inputs.arch }}"
+          model = "${{ github.event.inputs.model }}"
+          kairos_init_image = "${{ github.event.inputs.kairos_init_image }}"
+          hadron_version = "${{ github.event.inputs.hadron_version }}"
+
+          def kairos_init_version(image):
+              """Version component of the kairos-init image reference.
+
+              Base images are tagged with the kairos-init version (not the
+              Kairos version) because kairos-init is what determines the
+              layout and contents of the produced image. Earthfile's
+              KAIROS_INIT_VERSION must match this.
+              """
+              ref = image.rsplit("/", 1)[-1]
+              if "@" in ref:  # digest form: kairos-init@sha256:abc...
+                  return ref.split("@", 1)[1].replace(":", "-")
+              if ":" in ref:
+                  return ref.split(":", 1)[1]
+              return "latest"
+
+          init_version = kairos_init_version(kairos_init_image)
+          print(f"Tagging with kairos-init version: {init_version}")
+
+          build_matrix = []
+          if base_os_image:
+              # Custom upstream — single kairosify entry.
+              simple_name = base_os_image.split("/")[-1].replace(":", "-")
+              tag = f"{registry_prefix}/kairos-custom:{simple_name}-core-{arch}-{model}-{init_version}"
+              build_matrix.append({
+                  "name": f"kairos-custom-{simple_name}",
+                  "builder": "kairosify",
+                  "base_os": base_os_image,
+                  "tag": tag,
+              })
+          else:
+              # Images supported by the generic kairosify Dockerfile.
+              combinations = [
+                  ("ubuntu-20.04", "ubuntu:20.04", "kairos-ubuntu:20.04-core"),
+                  ("ubuntu-22.04", "ubuntu:22.04", "kairos-ubuntu:22.04-core"),
+                  ("ubuntu-24.04", "ubuntu:24.04", "kairos-ubuntu:24.04-core"),
+                  ("opensuse-15.6", "opensuse/leap:15.6", "kairos-opensuse:leap-15.6-core"),
+                  (
+                      "slem-5.4",
+                      "registry.suse.com/suse/sle-micro-rancher/5.4:latest",
+                      "kairos-slem:5.4-core",
+                  ),
+              ]
+              for name, base_os, tag_prefix in combinations:
+                  build_matrix.append({
+                      "name": name,
+                      "builder": "kairosify",
+                      "base_os": base_os,
+                      "tag": f"{registry_prefix}/{tag_prefix}-{arch}-{model}-{init_version}",
+                  })
+
+          # These images need their specialized install/init stages and are
+          # always included without separate build toggles.
+          build_matrix.append({
+              "name": "ubuntu-uki-24.04",
+              "builder": "ubuntu-uki",
+              "tag": f"{registry_prefix}/kairos-ubuntu:24.04-core-{arch}-generic-{init_version}-uki",
+          })
+          build_matrix.append({
+              "name": "hadron",
+              "builder": "hadron",
+              "tags": [
+                  f"{registry_prefix}/hadron-modules:{hadron_version}",
+                  f"{registry_prefix}/kairos-hadron:{hadron_version}-core-generic-{init_version}",
+                  f"{registry_prefix}/kairos-hadron:{hadron_version}-core-generic-{init_version}-fips",
+                  f"{registry_prefix}/kairos-hadron:{hadron_version}-core-generic-{init_version}-uki",
+              ],
+          })
+
+          def out(key, value):
+              with open(os.environ["GITHUB_OUTPUT"], "a") as f:
+                  f.write(f"{key}={value}\n")
+
+          out("build_matrix", json.dumps(build_matrix))
+          out("init_version", init_version)
+
+          print(f"build_matrix={build_matrix}")
+          print(f"init_version={init_version}")
+          EOF
+
+  build-base-images:
+    needs: generate-matrix
+    runs-on: Luet-BigRunner
+    strategy:
+      matrix:
+        include: ${{ fromJson(needs.generate-matrix.outputs.build_matrix) }}
+      fail-fast: false
     steps:
-      - name: Generate Matrix
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to registry
+        run: echo "${{ secrets.US_EAST_JSON_KEY_B64 }}" | base64 -d | docker login -u _json_key --password-stdin us-east1-docker.pkg.dev
+
+      - name: Build and push kairosify image
+        if: matrix.builder == 'kairosify'
+        uses: docker/bake-action@v6
+        with:
+          files: docker-bake-kairosify.hcl
+          targets: kairosify
+          push: true
+          set: |
+            kairosify.platform=linux/${{ github.event.inputs.arch }}
+            kairosify.args.BASE_OS_IMAGE=${{ matrix.base_os }}
+            kairosify.args.KAIROS_INIT_IMAGE=${{ github.event.inputs.kairos_init_image }}
+            kairosify.args.KAIROS_VERSION=${{ github.event.inputs.kairos_version }}
+            kairosify.args.MODEL=${{ github.event.inputs.model }}
+            kairosify.tags=${{ matrix.tag }}
+        env:
+          DOCKER_BUILD_SUMMARY: false
+
+      - name: Build and push Ubuntu 24.04 Trusted Boot base
+        if: matrix.builder == 'ubuntu-uki'
+        env:
+          SPECTRO_REPO: ${{ github.event.inputs.registry_prefix }}
+          KAIROS_VERSION: ${{ github.event.inputs.kairos_version }}
+          KAIROS_INIT_VERSION: ${{ needs.generate-matrix.outputs.init_version }}
+          KAIROS_INIT_IMAGE: ${{ github.event.inputs.kairos_init_image }}
+          ARCH: ${{ github.event.inputs.arch }}
+        run: |
+          set -euo pipefail
+          bash ubuntu-uki/24.04/build.sh \
+            --arch "${ARCH}" \
+            --tag "${{ matrix.tag }}" \
+            --push
+
+      - name: Build and push multi-arch Hadron modules and base images
+        if: matrix.builder == 'hadron'
+        env:
+          SPECTRO_REPO: ${{ github.event.inputs.registry_prefix }}
+          KAIROS_VERSION: ${{ github.event.inputs.kairos_version }}
+          KAIROS_INIT_VERSION: ${{ needs.generate-matrix.outputs.init_version }}
+          KAIROS_INIT_IMAGE: ${{ github.event.inputs.kairos_init_image }}
+          HADRON_VERSION: ${{ github.event.inputs.hadron_version }}
+        run: |
+          set -euo pipefail
+          bash hadron/build.sh --target modules --push
+          bash hadron/build.sh --push
+          bash hadron/build.sh --fips --push
+          bash hadron/build.sh --uki --push
+
+      - name: Save build result
+        if: matrix.builder != 'hadron'
+        run: |
+          mkdir -p /tmp/results
+          echo "${{ matrix.tag }}" >> /tmp/results/built_tags.txt
+
+      - name: Save Hadron build results
+        if: matrix.builder == 'hadron'
+        env:
+          SPECTRO_REPO: ${{ github.event.inputs.registry_prefix }}
+          KAIROS_INIT_VERSION: ${{ needs.generate-matrix.outputs.init_version }}
+          HADRON_VERSION: ${{ github.event.inputs.hadron_version }}
+        run: |
+          mkdir -p /tmp/results
+          {
+            echo "${SPECTRO_REPO}/hadron-modules:${HADRON_VERSION}"
+            echo "${SPECTRO_REPO}/kairos-hadron:${HADRON_VERSION}-core-generic-${KAIROS_INIT_VERSION}"
+            echo "${SPECTRO_REPO}/kairos-hadron:${HADRON_VERSION}-core-generic-${KAIROS_INIT_VERSION}-fips"
+            echo "${SPECTRO_REPO}/kairos-hadron:${HADRON_VERSION}-core-generic-${KAIROS_INIT_VERSION}-uki"
+          } >> /tmp/results/built_tags.txt
+
+      - name: Upload build results
+        uses: actions/upload-artifact@v4
+        with:
+          name: build-result-${{ matrix.name }}
+          path: /tmp/results/built_tags.txt
+
+  collect-outputs:
+    needs:
+      - generate-matrix
+      - build-base-images
+    if: always() && !cancelled()
+    runs-on: Luet-BigRunner
+    outputs:
+      built_tags: ${{ steps.combine-tags.outputs.tags }}
+    steps:
+      - name: Download all build results
+        uses: actions/download-artifact@v4
+        with:
+          pattern: build-result-*
+          path: /tmp/results
+          merge-multiple: true
+
+      - name: Combine all tags
+        id: combine-tags
+        run: |
+          ALL_TAGS=$(find /tmp/results -name "*.txt" -type f -exec cat {} \; 2>/dev/null | grep -v '^$' | sort -u | jq -R -s -c 'split("\n") | map(select(length > 0))')
+          echo "tags=$ALL_TAGS" >> "$GITHUB_OUTPUT"
+          echo "All built tags: $ALL_TAGS"
+
+      - name: Summary
         run: |
-          echo "Hello"
+          {
+            echo "## Base Image Build Summary"
+            echo ""
+            echo "**Input Parameters:**"
+            echo "- Base OS Image (kairosify override): ${{ github.event.inputs.base_os_image || 'Default matrix' }}"
+            echo "- Kairos Init Image: ${{ github.event.inputs.kairos_init_image }} (tags all built images)"
+            echo "- Architecture (kairosify/Ubuntu UKI): ${{ github.event.inputs.arch }}"
+            echo "- Hadron architectures: amd64, arm64"
+            echo "- Model: ${{ github.event.inputs.model }}"
+            echo "- Kairos Version: ${{ github.event.inputs.kairos_version }}"
+            echo "- Hadron Version: ${{ github.event.inputs.hadron_version }}"
+            echo "- Registry Prefix: ${{ github.event.inputs.registry_prefix }}"
+            echo ""
+            echo "**Built Images:**"
+            echo '${{ steps.combine-tags.outputs.tags }}' | jq -r '.[] | "- `" + . + "`"'
+          } >> "$GITHUB_STEP_SUMMARY"