diff --git a/.arg.template b/.arg.template index a27d42bd..53f12e29 100644 --- a/.arg.template +++ b/.arg.template @@ -31,3 +31,47 @@ FORCE_INTERACTIVE_INSTALL=false # AUTO_ENROLL_SECUREBOOT_KEYS=false # Set to true to automatically enroll certificates on devices in Setup Mode, useful for flashing devices without user interaction # DRBD_VERSION=9.2.13 # This variable is required for Piraeus pack for drbd module installtion. + +# NVIDIA GPU driver pre-install (for running the NVIDIA GPU Operator in air-gapped +# environments with driver.enabled=false). Bakes the datacenter driver + DKMS +# kernel modules into the Ubuntu base image so GPU nodes need no host-side network. +# See scripts/install-nvidia-drivers.sh for details. +# INSTALL_NVIDIA_GPU_DRIVERS=true +# NVIDIA_DRIVER_BRANCH=580 # Driver branch (check: apt-cache search 'nvidia-headless-.*-server') +# NVIDIA_DRIVER_TYPE=open # open | proprietary. Default "open" (REQUIRED on Hopper/Blackwell, +# safe on Turing/Ampere/Ada). Set "proprietary" only for pre-Turing GPUs. +# NVIDIA_USE_CUDA_REPO=true # Add NVIDIA CUDA network repo at build time (has every -server branch) +# NVIDIA_INSTALL_FABRICMANAGER=false # true for NVSwitch / HGX systems +# NVIDIA_INSTALL_CONTAINER_TOOLKIT=false # true to also pre-install nvidia-container-toolkit on host (then toolkit.enabled=false) +# NVIDIA_REBUILD_INITRD=true # Rebuild initrd so the nouveau blacklist applies at early boot + +# AMD Instinct GPU driver pre-install (for running the AMD GPU Operator in air-gapped +# environments with driver.enable=false). Mutually exclusive with the NVIDIA options +# above. See scripts/install-amdgpu-drivers.sh and docs/amd-gpu-airgapped.md for details. +# INSTALL_AMD_GPU_DRIVERS=true +# AMDGPU_DRIVER_SOURCE=dkms # dkms | inbox. "dkms" bakes AMD's amdgpu-dkms +# # built against the image kernel (recommended +# # for Instinct/MI silicon). "inbox" uses the +# # in-tree amdgpu module from linux-modules-* +# # and skips the AMD apt repo entirely — use +# # when the DKMS build fails against your image +# # kernel (see docs). +# AMDGPU_DRIVER_RELEASE=31.40 # Only used with dkms mode. amdgpu-install +# # release marker. Default 31.40 = ROCm 7.14 +# # GA (amdgpu 6.19.14), production driver for +# # GPU Operator v1.5.1 and MI350P / Radeon AI +# # PRO (RDNA4). For an older fleet on Operator +# # v1.5.0 use 7.2.1 (amdgpu 6.16.13). AMD +# # publishes both ROCm-alias (7.2.1, 7.2.4) and +# # marker (30.30.x, 31.40) paths; either works. +# # See the compatibility table in docs/. +# AMDGPU_INSTALL_SMI=true # Default true. Install the amd-smi host CLI on +# # PATH (parity with nvidia-smi), version-matched +# # to the driver (amdrocm-amdsmi from repo.amd.com). +# # rocm-smi is deprecated and NOT installed. +# # Best-effort; a small slice of ROCm user-space. +# AMDGPU_REBUILD_INITRD=false # Default false. amdgpu is intentionally +# # kept out of the initrd (multi-GPU init +# # emits enough udev events to time out +# # dracut-initqueue). It loads after +# # switch-root via modules-load.d. diff --git a/.earthlyignore b/.earthlyignore index c6fb21c9..486f781e 100644 --- a/.earthlyignore +++ b/.earthlyignore @@ -1,2 +1,6 @@ local/ -build/* \ No newline at end of file +build/* +# Whitelist AMD driver artifacts (produced by scripts/prebuild-amdgpu-artifact.sh +# and consumed by the base-image target via COPY). Without this exception, +# `build/*` above would hide the tarball from Earthly's build context. +!build/amdgpu-artifact-*.tar.gz \ No newline at end of file diff --git a/Earthfile b/Earthfile index 493d6100..9fb5c5d9 100644 --- a/Earthfile +++ b/Earthfile @@ -27,7 +27,7 @@ ARG RKE2_FLAVOR_TAG=rke2r1 ARG BASE_IMAGE_URL=quay.io/kairos ARG OSBUILDER_VERSION=v0.400.3 ARG OSBUILDER_IMAGE=quay.io/kairos/osbuilder-tools:$OSBUILDER_VERSION -ARG AURORABOOT_VERSION=v0.16.0 +ARG AURORABOOT_VERSION=v0.26.2 ARG AURORABOOT_IMAGE=quay.io/kairos/auroraboot:$AURORABOOT_VERSION ARG K3S_PROVIDER_VERSION=v4.9.1 ARG KUBEADM_PROVIDER_VERSION=v4.9.3 @@ -63,6 +63,57 @@ ARG no_proxy=${NO_PROXY} ARG UPDATE_KERNEL=false +# NVIDIA GPU driver pre-install (for air-gapped GPU Operator with driver.enabled=false). +# When true, the NVIDIA data-center driver + DKMS kernel modules are baked into the +# Ubuntu base image so GPU nodes need no host-side network at boot. +ARG INSTALL_NVIDIA_GPU_DRIVERS=false +ARG NVIDIA_DRIVER_BRANCH=580 +ARG NVIDIA_DRIVER_TYPE=open +ARG NVIDIA_USE_CUDA_REPO=true +ARG NVIDIA_INSTALL_FABRICMANAGER=true +ARG NVIDIA_INSTALL_IMEX=true +ARG NVIDIA_INSTALL_CONTAINER_TOOLKIT=false +ARG NVIDIA_REBUILD_INITRD=true + +# AMD Instinct GPU driver pre-install (for air-gapped AMD GPU Operator with +# driver.enable=false). See scripts/install-amdgpu-drivers.sh + docs/amd-gpu-airgapped.md. +ARG INSTALL_AMD_GPU_DRIVERS=false +# dkms | inbox. "dkms" builds AMD's amdgpu-dkms against the image kernel (default, +# recommended for Instinct silicon). "inbox" uses the in-tree amdgpu module shipped +# with linux-modules-* and skips the AMD apt repo — use only when the DKMS build +# fails against your image kernel and you accept the in-tree driver's feature set. +ARG AMDGPU_DRIVER_SOURCE=dkms +# amdgpu-install release marker (URL segment under repo.radeon.com/amdgpu-install//). +# Default 31.40 = ROCm 7.14 GA (amdgpu 6.19.14), the PRODUCTION driver for AMD GPU +# Operator v1.5.1 and the baseline for MI350P + Radeon AI PRO (RDNA4). (7.14 went GA +# on 2026-07-15; the earlier "31.x = tech-preview" note referred to the ROCm 7.13.0 +# preview and is obsolete.) For an older fleet staying on Operator v1.5.0, use 7.2.1 +# (amdgpu 6.16.13, 30.30.1 line). AMD publishes both ROCm-alias (7.2.1, 7.2.4) and +# driver-release-marker (30.30.x, 31.40) URL segments; either form is accepted here. +# See docs/amd-gpu-airgapped.md for the operator<->driver compatibility matrix. +ARG AMDGPU_DRIVER_RELEASE=31.40 +# Path to a driver artifact produced by scripts/prebuild-amdgpu-artifact.sh +# on the build host. Threaded in by earthly.sh when INSTALL_AMD_GPU_DRIVERS=true +# and AMDGPU_DRIVER_SOURCE=dkms. When set, the base-image AMD block skips the +# in-buildkit DKMS install (which fails in buildkit's RUN sandbox -- see docs) +# and simply extracts the pre-built modules + firmware + config drop-ins. +ARG AMDGPU_ARTIFACT_PATH="" +# Default false: amdgpu is intentionally omitted from the initrd (see +# scripts/install-amdgpu-drivers.sh -- multi-GPU amdgpu init emits enough +# udev events to blow past dracut-initqueue's udev-settle timeout, dropping +# the node into emergency mode). amdgpu loads after switch-root via +# /etc/modules-load.d/amdgpu.conf where there is no timeout pressure. +ARG AMDGPU_REBUILD_INITRD=false +# Install the amd-smi / rocm-smi host CLI on PATH (parity with nvidia-smi). Default +# true. Pulls a small slice of ROCm user-space from repo.radeon.com/rocm; best-effort. +ARG AMDGPU_INSTALL_SMI=true + +# NVIDIA and AMD driver pre-install are mutually exclusive within a single image. +IF [ "$INSTALL_NVIDIA_GPU_DRIVERS" = "true" ] && [ "$INSTALL_AMD_GPU_DRIVERS" = "true" ] + RUN echo "ERROR: INSTALL_NVIDIA_GPU_DRIVERS and INSTALL_AMD_GPU_DRIVERS are mutually exclusive. Enable only one." >&2 && \ + exit 1 +END + IF [ "$FIPS_ENABLED" = "true" ] && [ "$UPDATE_KERNEL" = "true" ] RUN echo "ERROR: UPDATE_KERNEL and FIPS_ENABLED are mutually exclusive. Cannot set both to true." >&2 && \ exit 1 @@ -282,7 +333,11 @@ install-k8s: SAVE ARTIFACT --keep-ts /output/ . build-uki-iso: - FROM --platform=linux/${ARCH} $OSBUILDER_IMAGE + # Switched from quay.io/kairos/osbuilder-tools (archived kairos-io/osbuilder + # + kairos-io/enki) to AuroraBoot, which is the maintained successor. The + # build-iso and build-uki subcommands accept a "dir:" source, so the rootfs + # preparation path above is unchanged; only the final CLI invocation differs. + FROM --platform=linux/${ARCH} $AURORABOOT_IMAGE ENV ISO_NAME=${ISO_NAME} COPY overlay/files-iso/ /overlay/ COPY --if-exists +validate-user-data/user-data /overlay/config.yaml @@ -313,22 +368,56 @@ build-uki-iso: WORKDIR /build COPY --platform=linux/${ARCH} --keep-own +iso-image-rootfs/rootfs /build/image + # AuroraBoot v0.26.1 silently ignores --output/-d for "dir:" sources on + # both build-iso and build-uki, dropping the ISO at /tmp/auroraboot/*.iso + # regardless. We hoist it into /iso/ ourselves after the run. + # AuroraBoot uses urfave/cli v2 which follows Go stdlib flag semantics: + # flag parsing stops at the first positional argument. If `dir:/build/image` + # comes before the flags, --overlay-iso / --arch / --sb-key etc. are + # silently discarded as extra positional args (build-iso continues without + # them; build-uki errors "Required flags ... not set"). Always place the + # positional source LAST. Empirically verified against v0.26.2. IF [ "$ARCH" = "arm64" ] - RUN CMD="/entrypoint.sh --name $ISO_NAME build-iso --date=false --overlay-iso /overlay dir:/build/image --output /iso/ --arch $ARCH" && \ - if [ "$DEBUG" = "true" ]; then CMD="$CMD --debug"; else CMD="$CMD"; fi && \ - $CMD + # arm64 UKI ISO is not supported by upstream today; fall through to a + # plain live/installer ISO, matching the previous osbuilder behavior. + RUN CMD="auroraboot" && \ + if [ "$DEBUG" = "true" ]; then CMD="$CMD --debug"; fi && \ + $CMD build-iso --overlay-iso /overlay --arch arm64 dir:/build/image ELSE IF [ "$ARCH" = "amd64" ] COPY secure-boot/enrollment/ secure-boot/private-keys/ secure-boot/public-keys/ /keys RUN ls -liah /keys - RUN mkdir /iso + # AuroraBoot's build-uki takes explicit key paths instead of osbuilder's + # bundled -k /keys. All key files live at /keys/* because the three + # secure-boot/* dirs above are flattened into the same target. IF [ "$AUTO_ENROLL_SECUREBOOT_KEYS" = "true" ] - RUN enki --config-dir /config build-uki dir:/build/image --extend-cmdline "$CMDLINE" --overlay-iso /overlay --secure-boot-enroll force -t iso -d /iso -k /keys --boot-branding "$BRANDING" + RUN CMD="auroraboot" && \ + if [ "$DEBUG" = "true" ]; then CMD="$CMD --debug"; fi && \ + $CMD build-uki -t iso \ + --extend-cmdline "$CMDLINE" \ + --overlay-iso /overlay \ + --boot-branding "$BRANDING" \ + --public-keys /keys \ + --sb-key /keys/db.key \ + --sb-cert /keys/db.pem \ + --tpm-pcr-private-key /keys/tpm2-pcr-private.pem \ + --secure-boot-enroll force \ + dir:/build/image ELSE - RUN enki --config-dir /config build-uki dir:/build/image --extend-cmdline "$CMDLINE" --overlay-iso /overlay -t iso -d /iso -k /keys --boot-branding "$BRANDING" + RUN CMD="auroraboot" && \ + if [ "$DEBUG" = "true" ]; then CMD="$CMD --debug"; fi && \ + $CMD build-uki -t iso \ + --extend-cmdline "$CMDLINE" \ + --overlay-iso /overlay \ + --boot-branding "$BRANDING" \ + --public-keys /keys \ + --sb-key /keys/db.key \ + --sb-cert /keys/db.pem \ + --tpm-pcr-private-key /keys/tpm2-pcr-private.pem \ + dir:/build/image END END - WORKDIR /iso - RUN mv /iso/*.iso $ISO_NAME.iso + RUN mkdir -p /iso && \ + mv /tmp/auroraboot/*.iso "/iso/$ISO_NAME.iso" SAVE ARTIFACT /iso/* iso: @@ -354,7 +443,12 @@ validate-user-data: build-iso: - FROM --platform=linux/${ARCH} $OSBUILDER_IMAGE + # Switched from quay.io/kairos/osbuilder-tools (archived kairos-io/osbuilder + # + kairos-io/enki) to AuroraBoot, which is the maintained successor. The + # build-iso subcommand accepts a "dir:" source with the same semantics as + # osbuilder's /entrypoint.sh build-iso, so the rootfs preparation path + # above is unchanged; only the final CLI invocation differs. + FROM --platform=linux/${ARCH} $AURORABOOT_IMAGE ENV ISO_NAME=${ISO_NAME} COPY overlay/files-iso/ /overlay/ COPY --if-exists +validate-user-data/user-data /overlay/files-iso/config.yaml @@ -398,17 +492,32 @@ build-iso: rm -f /build/image/opt/spectrocloud/local-ui.tar; \ fi + # AuroraBoot uses Go arch names for both amd64 and arm64 (osbuilder used + # "x86_64" for amd64). + # + # Positional source MUST come last. AuroraBoot uses urfave/cli v2 which + # follows Go stdlib flag semantics: flag parsing stops at the first + # positional argument. If dir:/build/image comes first, --overlay-iso + # and --arch are silently discarded as extra positional args. That is + # what caused the Palette-branded /boot/grub2/grub.cfg (and user-data, + # content bundles, cluster config) to silently disappear from produced + # ISOs before this fix. Empirically verified against v0.26.2. + # + # --output/--override-name are still inert on the subcommand path so we + # leave --output default and mv the produced ISO into /iso/ ourselves. IF [ "$ARCH" = "arm64" ] - RUN CMD="/entrypoint.sh --name $ISO_NAME build-iso --date=false --overlay-iso /overlay dir:/build/image --output /iso/ --arch $ARCH" && \ - if [ "$DEBUG" = "true" ]; then CMD="$CMD --debug"; else CMD="$CMD"; fi && \ - $CMD + RUN CMD="auroraboot" && \ + if [ "$DEBUG" = "true" ]; then CMD="$CMD --debug"; fi && \ + $CMD build-iso --overlay-iso /overlay --arch arm64 dir:/build/image ELSE IF [ "$ARCH" = "amd64" ] - RUN CMD="/entrypoint.sh --name $ISO_NAME build-iso --date=false --overlay-iso /overlay dir:/build/image --output /iso/ --arch x86_64" && \ - if [ "$DEBUG" = "true" ]; then CMD="$CMD --debug"; else CMD="$CMD"; fi && \ - $CMD + RUN CMD="auroraboot" && \ + if [ "$DEBUG" = "true" ]; then CMD="$CMD --debug"; fi && \ + $CMD build-iso --overlay-iso /overlay --arch amd64 dir:/build/image END + RUN mkdir -p /iso && \ + mv /tmp/auroraboot/*.iso "/iso/$ISO_NAME.iso" WORKDIR /iso - RUN sha256sum $ISO_NAME.iso > $ISO_NAME.iso.sha256 + RUN sha256sum "$ISO_NAME.iso" > "$ISO_NAME.iso.sha256" SAVE ARTIFACT --keep-ts /iso/* ### UKI targets @@ -725,6 +834,12 @@ base-image: COPY cloudconfigs/80_stylus_maas.yaml /system/oem/80_stylus_maas.yaml END + # Ensure the Renesas xHCI (USB 3.0) host controller driver is bundled into the + # initramfs so installation from USB media works on hardware using that chipset. + # Must run before the distro dracut regeneration below so the driver is included. + RUN mkdir -p /etc/dracut.conf.d && \ + printf '%s\n' 'hostonly="no"' 'add_drivers+=" xhci_pci_renesas "' 'force_drivers+=" xhci_pci_renesas "' > /etc/dracut.conf.d/99-usb-media.conf + # OS == Ubuntu IF [ "$OS_DISTRIBUTION" = "ubuntu" ] && [ "$ARCH" = "amd64" ] IF [ ! -z "$UBUNTU_PRO_KEY" ] @@ -734,7 +849,7 @@ base-image: END RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends kbd zstd vim iputils-ping bridge-utils curl tcpdump ethtool rsyslog logrotate -y + DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends kbd zstd vim iputils-ping bridge-utils curl tcpdump ethtool rsyslog logrotate libpam-pwquality -y LET APT_UPGRADE_FLAGS="-y" IF [ "$UPDATE_KERNEL" = "false" ] @@ -750,7 +865,12 @@ base-image: # tldr: apt-get upgrade -y doesn't install new packages, so we need to use --with-new-pkgs IF [ "$IS_UKI" = "false" ] + # Hold console-setup before the upgrade. Its 1.226ubuntu1.1 postinst prompts + # via debconf (uninitialized DB -> sed "unterminated s command" -> exit 128), + # which breaks this non-interactive image build. Freeze it at the base-image + # version so apt-get upgrade keeps it back. RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ + apt-mark hold console-setup console-setup-linux && \ apt-get upgrade $APT_UPGRADE_FLAGS && \ apt-get install --no-install-recommends -y \ util-linux \ # Provides essential utilities for Linux systems, including disk management tools. @@ -797,6 +917,64 @@ base-image: fi END + # NVIDIA GPU driver + DKMS kernel modules, built against the now-finalized + # image kernel. Runs here (not in the Dockerfile) so the kernel is settled + # first. Reuses install-kernel-headers.sh for ABI-exact headers. + IF [ "$INSTALL_NVIDIA_GPU_DRIVERS" = "true" ] + COPY scripts/install-kernel-headers.sh /tmp/install-kernel-headers.sh + COPY scripts/install-nvidia-drivers.sh /tmp/install-nvidia-drivers.sh + RUN chmod 755 /tmp/install-kernel-headers.sh /tmp/install-nvidia-drivers.sh && \ + NVIDIA_DRIVER_BRANCH="$NVIDIA_DRIVER_BRANCH" \ + NVIDIA_DRIVER_TYPE="$NVIDIA_DRIVER_TYPE" \ + NVIDIA_USE_CUDA_REPO="$NVIDIA_USE_CUDA_REPO" \ + NVIDIA_INSTALL_FABRICMANAGER="$NVIDIA_INSTALL_FABRICMANAGER" \ + NVIDIA_INSTALL_IMEX="$NVIDIA_INSTALL_IMEX" \ + NVIDIA_INSTALL_CONTAINER_TOOLKIT="$NVIDIA_INSTALL_CONTAINER_TOOLKIT" \ + NVIDIA_REBUILD_INITRD="$NVIDIA_REBUILD_INITRD" \ + /tmp/install-nvidia-drivers.sh && \ + rm -f /tmp/install-nvidia-drivers.sh /tmp/install-kernel-headers.sh + END + + # AMD Instinct GPU driver (amdgpu-dkms) + kernel module, built against the + # now-finalized image kernel. Mutually exclusive with the NVIDIA block above. + IF [ "$INSTALL_AMD_GPU_DRIVERS" = "true" ] + # dkms mode with a pre-built artifact (default path when earthly.sh + # produced one via scripts/prebuild-amdgpu-artifact.sh). Buildkit's + # RUN sandbox breaks AMD's amdgpu-dkms ./configure heredoc probe -- + # see docs/amd-gpu-airgapped.md. The prebuild runs on the host in a + # plain `docker run --privileged` against the same base image, and + # this stage just extracts the resulting modules + firmware + drop-ins. + IF [ "$AMDGPU_DRIVER_SOURCE" = "dkms" ] && [ "$AMDGPU_ARTIFACT_PATH" != "" ] + COPY scripts/install-amdgpu-drivers.sh /tmp/install-amdgpu-drivers.sh + COPY "$AMDGPU_ARTIFACT_PATH" /tmp/amdgpu-artifact.tar.gz + RUN --privileged \ + chmod 755 /tmp/install-amdgpu-drivers.sh && \ + AMDGPU_DRIVER_SOURCE=dkms \ + AMDGPU_DRIVER_RELEASE="$AMDGPU_DRIVER_RELEASE" \ + AMDGPU_REBUILD_INITRD="$AMDGPU_REBUILD_INITRD" \ + AMDGPU_INSTALL_SMI="$AMDGPU_INSTALL_SMI" \ + AMDGPU_ARTIFACT_PATH=/tmp/amdgpu-artifact.tar.gz \ + /tmp/install-amdgpu-drivers.sh && \ + rm -f /tmp/install-amdgpu-drivers.sh /tmp/amdgpu-artifact.tar.gz + ELSE + # inbox mode, OR dkms mode without a pre-built artifact (which + # will fail in buildkit's sandbox, but we let install-amdgpu- + # drivers.sh emit its own clear error rather than short-circuit + # here). install-kernel-headers.sh is only needed for the + # in-buildkit DKMS path; inbox mode doesn't use it. + COPY scripts/install-kernel-headers.sh /tmp/install-kernel-headers.sh + COPY scripts/install-amdgpu-drivers.sh /tmp/install-amdgpu-drivers.sh + RUN --privileged \ + chmod 755 /tmp/install-kernel-headers.sh /tmp/install-amdgpu-drivers.sh && \ + AMDGPU_DRIVER_SOURCE="$AMDGPU_DRIVER_SOURCE" \ + AMDGPU_DRIVER_RELEASE="$AMDGPU_DRIVER_RELEASE" \ + AMDGPU_REBUILD_INITRD="$AMDGPU_REBUILD_INITRD" \ + AMDGPU_INSTALL_SMI="$AMDGPU_INSTALL_SMI" \ + /tmp/install-amdgpu-drivers.sh && \ + rm -f /tmp/install-amdgpu-drivers.sh /tmp/install-kernel-headers.sh + END + END + IF [ "$CIS_HARDENING" = "true" ] COPY cis-harden/harden.sh /tmp/harden.sh RUN /tmp/harden.sh && rm /tmp/harden.sh @@ -859,6 +1037,25 @@ base-image: RUN if ! grep -Fq "systemd.unified_cgroup_hierarchy=1" /etc/cos/bootargs.cfg; then \ sed -i 's|\(set baseCmd="[^"]*\)"|\1 systemd.unified_cgroup_hierarchy=1"|' /etc/cos/bootargs.cfg; \ fi + + # Block nouveau and qat_4xxx at the kernel command line on every + # build, and pin PCI BAR layout to firmware assignments. + # + # nouveau: modern NVIDIA data-center GPUs (Ada/Hopper/Blackwell) + # hang in GSP init when initramfs udev auto-loads nouveau before + # switchroot, stalling systemd-udev-settle indefinitely. Applied + # unconditionally — the image may be installed onto NVIDIA hardware + # even when INSTALL_NVIDIA_GPU_DRIVERS=false. rd.driver.blacklist= + # is the load-bearing flag (dracut honors it before udev fires); + # modprobe.blacklist= is belt-and-braces for post-switchroot. + # Harmless when no NVIDIA GPU is present. + # + # qat_4xxx: on Xeon Scalable 4th/5th gen hosts with QAT devices, + # udev auto-loads qat_4xxx in initramfs and its probe/firmware-load + # stalls boot for minutes. CanvOS does not consume QAT acceleration. + RUN if ! grep -Fq "rd.driver.blacklist=nouveau" /etc/cos/bootargs.cfg; then \ + sed -i 's|\(set baseCmd="[^"]*\)"|\1 rd.driver.blacklist=nouveau,qat_4xxx modprobe.blacklist=nouveau,qat_4xxx nouveau.modeset=0"|' /etc/cos/bootargs.cfg; \ + fi END KAIROS_RELEASE: @@ -915,6 +1112,7 @@ iso-image: RUN rm -f /usr/bin/luet END COPY overlay/files/ / + IF [ "$IS_CLOUD_IMAGE" = "true" ] COPY cloud-images/workaround/grubmenu.cfg /etc/kairos/branding/grubmenu.cfg COPY cloud-images/workaround/custom-post-reset.yaml /system/oem/custom-post-reset.yaml diff --git a/docs/amd-gpu-airgapped.md b/docs/amd-gpu-airgapped.md new file mode 100644 index 00000000..692a9ad2 --- /dev/null +++ b/docs/amd-gpu-airgapped.md @@ -0,0 +1,321 @@ +# Pre-installing the AMD Instinct GPU driver for air-gapped GPU Operator + +This guide explains how to pre-provision the AMD **amdgpu** kernel-mode driver +**in a CanvOS Ubuntu base image**, so that AMD Instinct GPU nodes can run the +[AMD GPU Operator](https://instinct.docs.amd.com/projects/gpu-operator/en/latest/specialized_networks/airgapped-install.html) +in a **fully air-gapped** environment — with **no host-side network access** and +**without the operator building/managing the driver**. + +It is the AMD counterpart of [`nvidia-gpu-airgapped.md`](./nvidia-gpu-airgapped.md) +and follows the same "pre-installed driver" model. + +- Script: [`scripts/install-amdgpu-drivers.sh`](../scripts/install-amdgpu-drivers.sh) +- Wired into the `base-image` target in the [`Earthfile`](../Earthfile), + gated by `INSTALL_AMD_GPU_DRIVERS=true`. + +**Supported targets:** Ubuntu **22.04** (jammy) and **24.04** (noble), `amd64`. +The codename is derived from the image at build time. + +## Two driver-source modes (`AMDGPU_DRIVER_SOURCE`) + +| Mode | What ships in the image | When to use | +| --- | --- | --- | +| `dkms` (default) | AMD's `amdgpu-dkms` source is DKMS-built against the image kernel and lands under `/lib/modules//updates/dkms/`. | Recommended for Instinct/MI silicon. The AMD out-of-tree driver typically carries newer SMU firmware interfaces and per-SKU support ahead of what the in-tree amdgpu has. | +| `inbox` | No AMD **driver** repo is added; the script only ensures the in-tree `amdgpu` module (shipped in `linux-modules-`) autoloads. (The ROCm repo is still added *only* for `amd-smi` if `AMDGPU_INSTALL_SMI=true`.) | Fallback when the DKMS build fails against your image kernel — e.g. AMD hasn't yet published a driver release whose source builds against a very new kernel. Requires accepting the in-tree driver's feature set. | + +Both modes still require `driver.enable=false` at the Helm layer — the operator +does not build a driver either way. A marker at +`/etc/canvos/amdgpu-driver-source` on the booted node records which mode ran. + +> **Mutually exclusive with NVIDIA.** A single image supports one GPU vendor. +> Enabling both `INSTALL_AMD_GPU_DRIVERS` and `INSTALL_NVIDIA_GPU_DRIVERS` fails +> the build. + +--- + +## Split of responsibilities + +| Component | Where it lives | Who installs it | +| --- | --- | --- | +| amdgpu kernel module (`amdgpu`) + firmware | **In the OS image** | **This script (build time)** | +| `amd-smi` host diagnostic CLI (`AMDGPU_INSTALL_SMI=true`) | **In the OS image** (`/usr/bin`) | **This script (build time)** | +| Full ROCm user-space, device-plugin, node-labeller, metrics-exporter | Container images | AMD GPU Operator (from your content bundle) | + +The OS carries only the kernel driver (plus the small `amd-smi` CLI for +troubleshooting); everything else is a container image you +mirror into your Palette content bundle. With `driver.enable=false` the operator +"directly uses inbox or pre-installed AMD GPU drivers" and only deploys the +device-plugin / node-labeller / metrics-exporter. + +At Helm-install time you **must** set: + +``` +--set driver.enable=false # note: "enable", not "enabled" +``` + +--- + +## Relationship to the AMD air-gapped guide + +The AMD guide's `driver.enable=true` path has the operator build the out-of-tree +driver at runtime, which needs build packages and (in restricted networks) a +local package mirror. This integration uses the opposite path +(`driver.enable=false`): the driver is either DKMS-built into the image at +build time (`AMDGPU_DRIVER_SOURCE=dkms`) or the in-tree amdgpu is used as-is +(`AMDGPU_DRIVER_SOURCE=inbox`). Either way, **no host-side mirror or network is +needed at boot**. + +--- + +## The key build-time problem this solves (dkms mode) + +Inside the Earthly/Docker build, `uname -r` is the **builder host's** kernel, not +the kernel baked into the image. In `dkms` mode the driver must be compiled +against the image kernel's headers. + +We ran into a second problem too: Earthly's buildkit `RUN` sandbox breaks +AMD's `amdgpu-dkms` `./configure` heredoc probe with +`"cannot detect CFLAGS…"`, even though the same script + base image + host +succeed under plain `docker run --privileged`. Rather than debug buildkit +(some seccomp/apparmor/mount detail we don't control from the Earthfile), +`dkms` mode uses a **two-stage build**: + +1. **Prebuild on the host** (via `scripts/prebuild-amdgpu-artifact.sh`, + auto-invoked by `./earthly.sh`). Runs a plain `docker run --privileged` + against the same kairos base image, executes the DKMS install inside, + tars the resulting `/lib/modules//updates/dkms/` + firmware + + drop-ins into `build/amdgpu-artifact---.tar.gz`. + Cached by (release × base-image digest × kver); ~8–10 min the first + time, instant on cache hit. +2. **Consume in Earthly**: the `base-image` target `COPY`s the tarball and + extracts it, runs `depmod` against the image kernel, and rebuilds the + initrd. No `./configure`, no compile inside buildkit. + +Both stages use the same `scripts/install-amdgpu-drivers.sh` — the in-buildkit +step just takes the `AMDGPU_ARTIFACT_PATH` fast path. The result on-node is +identical to a native DKMS install. + +`inbox` mode skips both stages entirely and only ensures the in-tree amdgpu +autoloads. + +> **No blacklist needed.** Unlike NVIDIA (where `nouveau` must be blacklisted), +> the DKMS `amdgpu` module replaces the in-tree one via `depmod`'s `updates/` +> override. The script just autoloads `amdgpu`. + +> **When is the prebuild helper invoked?** `./earthly.sh` triggers it +> automatically when `INSTALL_AMD_GPU_DRIVERS=true` and +> `AMDGPU_DRIVER_SOURCE=dkms` (either from `.arg` or a CLI override). No +> extra command needed. It's skipped for `inbox`, or for any non-AMD build. + +--- + +## Quick start + +1. Edit `.arg` and enable the feature: + + ```sh + OS_DISTRIBUTION=ubuntu + OS_VERSION=22 # or 24 for Ubuntu 24.04 + ARCH=amd64 + + INSTALL_AMD_GPU_DRIVERS=true + AMDGPU_DRIVER_SOURCE=dkms # or "inbox" — see modes above + AMDGPU_DRIVER_RELEASE=31.40 # ROCm 7.14 GA; pairs with GPU Operator v1.5.1 (dkms mode only) + AMDGPU_INSTALL_SMI=true # install amd-smi host CLI (parity with nvidia-smi; rocm-smi not installed) + ``` + +2. Build as usual, e.g.: + + ```sh + ./earthly.sh +build-all-images --ARCH=amd64 + ``` + + or override on the command line: + + ```sh + ./earthly.sh +base-image --ARCH=amd64 \ + --INSTALL_AMD_GPU_DRIVERS=true \ + --AMDGPU_DRIVER_RELEASE=31.40 + ``` + + If the `dkms` build fails on your image kernel (see the mapping table + below), rebuild with `--AMDGPU_DRIVER_SOURCE=inbox` to fall back to the + in-tree driver instead. + +3. Mirror the AMD GPU Operator container images into your Palette content bundle + and install the operator with `driver.enable=false`. + +--- + +## Configuration reference + +| Variable | Default | Description | +| --- | --- | --- | +| `INSTALL_AMD_GPU_DRIVERS` | `false` | Master switch. Enables the AMD pre-install pipeline. | +| `AMDGPU_DRIVER_SOURCE` | `dkms` | `dkms` (build AMD's out-of-tree driver against the image kernel) or `inbox` (skip the AMD repo and use the in-tree amdgpu). See modes above. | +| `AMDGPU_DRIVER_RELEASE` | `31.40` | **`dkms` mode only.** `amdgpu-install` URL segment under `repo.radeon.com/amdgpu-install//`. AMD publishes both ROCm-alias paths (e.g. `7.2.1`, `7.2.4`) and driver-release-marker paths (e.g. `30.30.x`, `31.40`) — either form works. Default `31.40` = **ROCm 7.14 GA** → **amdgpu-dkms 6.19.14**, the production driver for **GPU Operator v1.5.1** and the baseline for MI350P + Radeon AI PRO (RDNA4). For an older fleet on **v1.5.0**, use `7.2.1` (amdgpu-dkms 6.16.13). | +| `AMDGPU_INSTALL_SMI` | `true` | Install the `amd-smi` host CLI and symlink into `/usr/bin`, for parity with `nvidia-smi`. Prefers `amdrocm-amdsmi` from `repo.amd.com` (user-space matched to recent drivers, e.g. 7.14 for the `31.40` driver), falling back to `amd-smi-lib` from the legacy `repo.radeon.com/rocm/apt` (7.2.x). `rocm-smi` is deprecated and **not** installed. Best-effort — a failure warns but does not fail the build. | +| `AMDGPU_REBUILD_INITRD` | `false` | Rebuild the initrd for the image kernel. Default `false`: amdgpu is intentionally kept out of the initrd (multi-GPU init can time out dracut-initqueue); it loads after switch-root via `modules-load.d`. | + +### Version alignment across the stack + +Several things have to line up to have a supportable node. Start from the +operator version you bundle and follow AMD's release notes / compat matrix: + +``` + GPU Operator ─┐ AMD release notes pair the operator with a specific + │ ROCm user-space release + ROCm user-space (device-plugin / metrics-exporter / etc) + │ AMD user↔kernel compat matrix pairs ROCm with a + │ driver-release marker + amdgpu driver (amdgpu-dkms) → this is what this script installs + │ The DKMS source has a supported kernel window + Image kernel ─┘ + Kubernetes version — validated per operator release +``` + +#### Operator ↔ driver compatibility + +| GPU Operator | `AMDGPU_DRIVER_RELEASE` | amdgpu-dkms | ROCm | Notes | +| --- | --- | --- | --- | --- | +| **v1.5.1** (latest) | **`31.40`** (default) | `6.19.14` | **7.14 GA** | Production. Baseline for **MI350P** + **Radeon AI PRO (RDNA4)**. DRA driver needs `31.40`+. | +| v1.5.0 | `7.2.1` (= `30.30.1`) | `6.16.13` | 7.2.1 | Production. MI300X/MI325X, MI350X/MI355X. | +| v1.5.0 | `7.2.4` (= `30.30.4`) | `6.16.13` | 7.2.4 | Production patch of the 7.2.x line. | + +> **The old "31.x = tech preview" note is obsolete.** `31.x` referred to the +> **ROCm 7.13.0 preview**. **ROCm 7.14.0 went GA on 2026-07-15**, and marker +> `31.40` (amdgpu 6.19.14) is a **production** driver. MI350X/MI355X/MI350P +> (gfx950) and Radeon AI PRO R9700 (RDNA4/gfx1201) are all in the ROCm 7.14 +> production hardware matrix. The **only** remaining preview caveat in Operator +> v1.5.1 is **Auto Node Remediation (ANR)** — "not fully validated on MI350P +> and Radeon AI platforms in this beta release" — which is an optional operator +> *feature*, not the driver. + +**How to pick:** match the driver to the operator you bundle. +- Standardizing on **Operator v1.5.1** (incl. MI350P / RDNA4) → keep the default **`31.40`**. +- Staying on **Operator v1.5.0** with MI300X/MI325X → set **`AMDGPU_DRIVER_RELEASE=7.2.1`**. + +Driver and operator move together — don't bump one without the other. + +Authoritative references: +- [AMD GPU Operator release notes](https://instinct.docs.amd.com/projects/gpu-operator/en/latest/release-notes.html) +- [ROCm compatibility matrix](https://rocm.docs.amd.com/en/latest/compatibility/compatibility-matrix.html) +- [ROCm user↔kernel compat matrix](https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/user-kernel-space-compat-matrix.html) +- Repo index: + +The 30.30.x line shares one driver *source* (`6.16.13`) and builds cleanly +against Ubuntu 24.04's 6.17 kernel; the `31.40` source (`6.19.14`) is the ROCm +7.14 driver. The "EFI variables are not supported on this system" line during +postinst is a cosmetic mokutil warning (sign_tool step); DKMS proceeds and lands +the module under `updates/dkms/`. + +### When the DKMS build fails + +Common causes: + +1. **Kernel outside the driver's supported window** — `make.log` shows + `configure: cannot detect CFLAGS` or unresolved kernel symbols. Move the + image kernel into range, or move the operator/ROCm/driver combo up as a + paired set — don't change `AMDGPU_DRIVER_RELEASE` in isolation just to chase + a newer kernel. +2. **`linux-headers-` not installed for the image kernel** — check the + earlier log lines from `install-kernel-headers.sh`. Fix the headers. +3. **DKMS module signing (mokutil) failure in the container** — surfaces as + "EFI variables are not supported on this system / /sys/firmware/efi/efivars + not found, aborting." The script writes + `/etc/dkms/framework.conf.d/canvos-no-mok-signing.conf` (empty `sign_tool`) + before the apt install to sidestep this. In 30.30.x the AMD postinst + already tolerates the missing EFI vars (it prints the warning and + continues); the sign_tool drop-in is defensive belt-and-suspenders for + 31.x and future releases that may treat it as fatal. + +The script prints the last 60 lines of `make.log` on failure. Read it — +the class of failure matters for the fix. Workaround for any of the above: +rerun with `AMDGPU_DRIVER_SOURCE=inbox` to use the in-tree amdgpu (accepts +the caveats above about missing driver-version label + SMU IF mismatch on +newer silicon). + +--- + +## Verify on a booted node + +```sh +lsmod | grep amdgpu +dmesg | grep -i amdgpu +ls /sys/class/kfd 2>/dev/null && echo "KFD present" +cat /etc/canvos/amdgpu-driver-source # which mode ran + release + AMDGPU_SMI=yes/no +# In dkms mode, expect a module under /lib/modules//updates/dkms/ +find /lib/modules/$(uname -r)/updates -name 'amdgpu.ko*' 2>/dev/null +# amd-smi is baked into the OS image when AMDGPU_INSTALL_SMI=true (default): +amd-smi version # user-space version (should match the driver, e.g. 7.14 for 31.40) +amd-smi list # like `nvidia-smi -L` — enumerates GPUs from the host +amd-smi monitor # live utilization +# (rocm-smi is deprecated and intentionally not installed — amd-smi supersedes it) +``` + +--- + +## Building the air-gapped content (which images to bundle) + +Pre-installing the driver in the OS removes the **driver-build** images (KMM & +friends). The operator still deploys the rest as containers, so those images — +plus cert-manager and a couple of non-image steps — must be handled. +**Bundling images alone is not sufficient.** + +### Images to mirror into your content bundle + +| Image | Needed with `driver.enable=false`? | +| --- | --- | +| `rocm/gpu-operator` (controller-manager) | Yes | +| `rocm/gpu-operator-utils` | Yes | +| `rocm/k8s-device-plugin` | Yes | +| `rocm/k8s-device-plugin:labeller-*` (node labeller) | Yes | +| `rocm/device-metrics-exporter` | Yes, if you want metrics | +| `rocm/device-config-manager` | Yes | +| `busybox:1.36` (init container) | Yes | +| `registry.k8s.io/nfd/node-feature-discovery` | Yes — unless the cluster already runs NFD | +| cert-manager (`controller`, `webhook`, `cainjector`, `acmesolver`) | Yes — hard dependency | +| KMM images (operator / webhook / worker / signimage) | **No — skip** | +| `gcr.io/kaniko-project/executor`, `ubuntu:` (driver build) | **No — skip** | +| `rocm/test-runner` | Optional (testing only) | + +Render the exact set from the chart rather than transcribing tags: + +```sh +helm template amd-gpu ./gpu-operator-.tgz -f operator-values.yaml \ + | grep -Eo 'image: *"?[^"]+' | sort -u +``` + +### Non-image steps + +1. **Install cert-manager first** (with its images pulled from your registry) — + the AMD operator will not start without it. +2. In the `DeviceConfig` CR, set `spec.driver.enable: false`. +3. Override every image (`controllerManager.manager.image`, + `commonConfig.initContainerImage`, `utilsContainer.image`, + `devicePlugin.devicePluginImage`, `devicePlugin.nodeLabellerImage`, + `metricsExporter.image`, `configManager.image`, and the NFD image) to your + bundle/registry; set `imagePullSecrets` as needed. +4. Ensure GPU nodes are labelled (via NFD or manually): + `feature.node.kubernetes.io/amd-gpu=true`. + +### Palette content bundle + +Add the AMD GPU Operator (and cert-manager) as Helm packs in the cluster profile, +then build the content bundle so it includes the rendered images above (minus the +KMM/kaniko/ubuntu build images). Images set only via `values.yaml` may need to be +added to the pack's additional-images list if the bundle builder doesn't +auto-detect them. Verify on a node with `lsmod | grep amdgpu` and by checking the +operator pods reach `Ready`. + +## Limitations / caveats + +- **Secure Boot / UKI is not supported by this path** (unsigned DKMS modules + won't load). Use the standard (non-UKI) Ubuntu image for GPU nodes. +- **amd64 / Ubuntu only.** +- **Version alignment is yours to own** — `AMDGPU_DRIVER_RELEASE` must line up + with the ROCm version of the operator images you bundle. +- **`inbox` mode loses the driver-version node label** — the AMD GPU Operator's + node-labeller reads `/sys/class/drm/card*/device/driver/module/version`, + which only exists when the driver was DKMS-installed. Enumeration and + scheduling still work; driver-version-aware policies won't. diff --git a/docs/nvidia-gpu-airgapped.md b/docs/nvidia-gpu-airgapped.md new file mode 100644 index 00000000..a802230f --- /dev/null +++ b/docs/nvidia-gpu-airgapped.md @@ -0,0 +1,291 @@ +# Pre-installing the NVIDIA GPU driver for air-gapped GPU Operator + +This guide explains how to bake the NVIDIA data-center GPU driver and its +kernel modules **into a CanvOS Ubuntu base image**, so that GPU nodes can run +the [NVIDIA GPU Operator](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/install-gpu-operator-air-gapped.html) +in a **fully air-gapped** environment — with **no host-side network access** and +**without the operator's driver container**. + +- Script: [`scripts/install-nvidia-drivers.sh`](../scripts/install-nvidia-drivers.sh) +- Wired into the `base-image` target in the [`Earthfile`](../Earthfile), + gated by `INSTALL_NVIDIA_GPU_DRIVERS=true`. + +**Supported targets:** Ubuntu **22.04** and **24.04**, `amd64`. The script is +version-agnostic — it derives the CUDA repo tag (`ubuntu2204` / `ubuntu2404`) and +the kernel codename (`jammy` / `noble`) from the image's `/etc/os-release` at +build time, so the same script works for both without changes. + +> For AMD Instinct GPUs, see [`amd-gpu-airgapped.md`](./amd-gpu-airgapped.md). +> The two are **mutually exclusive** — enabling both `INSTALL_NVIDIA_GPU_DRIVERS` +> and `INSTALL_AMD_GPU_DRIVERS` fails the build. + +--- + +## Why do this (the split of responsibilities) + +The GPU Operator normally deploys the NVIDIA driver as a **driver container** that +downloads and compiles the driver at runtime. That requires network access on the +node and a matching kernel-headers source — neither of which exists on an +air-gapped, immutable (Kairos) edge host. + +The supported alternative is the **pre-installed driver** model: + +| Component | Where it lives | Who installs it | +| --- | --- | --- | +| NVIDIA kernel driver + modules (`nvidia`, `nvidia_uvm`, `nvidia_modeset`, `nvidia_drm`) | **In the OS image** | **This script (build time)** | +| `nvidia-smi` / driver user-space | **In the OS image** | **This script (build time)** | +| nvidia-container-toolkit / runtime class | Container image | GPU Operator (from your content bundle) | +| device-plugin, gpu-feature-discovery, DCGM exporter, MIG manager, validator | Container images | GPU Operator (from your content bundle) | + +So: **the OS carries only the driver + kernel modules**; everything else is a +container image you mirror into your Palette content bundle. At boot the node has a +working driver with zero connectivity, and once your bundled operator images are +present the GPU cluster comes up with no external pulls. + +At Helm-install time you **must** tell the operator the driver is pre-installed: + +``` +--set driver.enabled=false +``` + +If you also opt in to pre-installing the container toolkit on the host +(`NVIDIA_INSTALL_CONTAINER_TOOLKIT=true`, off by default), additionally set: + +``` +--set toolkit.enabled=false +``` + +--- + +## Relationship to NVIDIA's "Local Package Repository" section + +The NVIDIA air-gapped guide lists these Ubuntu packages under +**Local Package Repository → Required Packages**: + +``` +ubuntu: + linux-headers-${KERNEL_VERSION} + linux-image-${KERNEL_VERSION} + linux-modules-${KERNEL_VERSION} +``` + +That list belongs to the **driver-container** strategy: the node runs the GPU +Operator's *driver container*, which compiles the driver **at runtime** and pulls +those OS packages from **a local Ubuntu apt mirror you host**. It requires +`driver.enabled=true` plus a maintained mirror. + +This CanvOS integration deliberately uses the **other** supported strategy — +**pre-installed driver in the OS image** (`driver.enabled=false`) — so **no local +apt mirror is needed**. The substance of those three packages is still satisfied, +just at build time inside the image rather than from a runtime mirror: + +| NVIDIA-required package | How this integration satisfies it | +| --- | --- | +| `linux-headers-${KERNEL_VERSION}` | Installed at build time by `install-kernel-headers.sh` (ABI-exact; DKMS builds against these). | +| `linux-image-${KERNEL_VERSION}` | Already shipped in the Kairos base image (the bootable kernel). | +| `linux-modules-${KERNEL_VERSION}` | Already shipped in the Kairos base image (`/lib/modules/${KERNEL_VERSION}/`). | + +If you specifically want the driver-container + local-mirror model instead, this +script is not the right tool — you would host an apt mirror serving the packages +above and leave `driver.enabled=true`. + +## The key build-time problem this solves + +Inside the Earthly/Docker build, `uname -r` is the **builder host's** kernel, **not** +the kernel baked into the image. If DKMS builds "for the running kernel", you get +modules for the wrong ABI (or the build fails). The script therefore: + +1. derives the **target kernel** from `/lib/modules/*` (the kernel that will boot), +2. installs **ABI-exact kernel headers** for it (reusing + [`install-kernel-headers.sh`](../scripts/install-kernel-headers.sh), which falls + back to `snapshot.ubuntu.com` when Ubuntu rotates the ABI out of the live mirror), +3. forces **DKMS build + install + `depmod`** against that target kernel, and +4. **verifies** the resulting `nvidia*.ko` modules actually landed under + `/lib/modules//` — failing the build loudly if they did not. + +It runs in the `base-image` target **after** the kernel is finalized +(hold/upgrade/purge/dracut), so modules are always built against the settled kernel. + +> **Connectivity note:** the build runs where the builder has internet and bakes +> everything into the image. The resulting image needs no network at boot. + +--- + +## Quick start + +1. Edit `.arg` (copied from `.arg.template`) and enable the feature: + + ```sh + OS_DISTRIBUTION=ubuntu + OS_VERSION=22 # or 24 for Ubuntu 24.04 + ARCH=amd64 + + INSTALL_NVIDIA_GPU_DRIVERS=true + NVIDIA_DRIVER_BRANCH=580 # verify the branch exists (see below) + NVIDIA_DRIVER_TYPE=proprietary # or: open (Turing+ only) + ``` + +2. Build as usual, e.g.: + + ```sh + ./earthly.sh +build-all-images --ARCH=amd64 + ``` + + or override on the command line without touching `.arg`: + + ```sh + ./earthly.sh +base-image --ARCH=amd64 \ + --INSTALL_NVIDIA_GPU_DRIVERS=true \ + --NVIDIA_DRIVER_BRANCH=580 \ + --NVIDIA_DRIVER_TYPE=proprietary + ``` + +3. Mirror the GPU Operator container images into your Palette content bundle + (per the NVIDIA air-gapped guide), and install the operator with + `driver.enabled=false`. + +--- + +## Configuration reference + +All variables are optional and have defaults. Set them in `.arg` or pass as +`--VAR=value` on the `earthly.sh` command line. + +| Variable | Default | Description | +| --- | --- | --- | +| `INSTALL_NVIDIA_GPU_DRIVERS` | `false` | Master switch. When `true`, the driver + DKMS modules are baked into the Ubuntu base image. | +| `NVIDIA_DRIVER_BRANCH` | `580` | Driver **branch** to install (e.g. `550`, `570`, `580`). apt installs the latest patch within the branch — it is not pinned to an exact point release (e.g. `580.159.03`). Must be a real `-server` branch — see [Choosing a driver branch](#choosing-a-driver-branch). | +| `NVIDIA_DRIVER_TYPE` | `open` | `open` or `proprietary`. `open` uses the NVIDIA open GPU kernel modules and is **required** on Hopper (H100/H200) and Blackwell (RTX PRO 6000 Blackwell, B100/B200/GB200); the closed modules fail with `RmInitAdapter (0x22:0x56:897)` on those GPUs. Also safe on Turing/Ampere/Ada. Override to `proprietary` only for pre-Turing hardware (Pascal/Volta). See [Choosing the module flavor](#choosing-the-module-flavor-nvidia_driver_type). | +| `NVIDIA_USE_CUDA_REPO` | `true` | Add the NVIDIA CUDA network repo at build time. It carries every `-server` branch; recommended. `false` uses only Ubuntu's own repos. | +| `NVIDIA_INSTALL_FABRICMANAGER` | `true` | Installs `nvidia-fabricmanager-` + `libnvidia-nscq-` + `nvlsm` (NVIDIA Subnet Manager, from the CUDA repo) + `infiniband-diags` (for `ibstat`) and enables the unit — **required** on NVSwitch systems (HGX H100/H200, HGX B200, DGX, GB200 NVL72) for multi-GPU NVLink to come up. NVIDIA's 570+ shipped unit invokes a wrapper (`/usr/share/nvidia/fabricmanager/nvidia-fabricmanager-start.sh`) that probes `ibstat` + `nvlsm` before starting `nv-fabricmanager`; all four packages must be present or the unit fails before FM is ever invoked. On non-NVSwitch hosts FM exits `"No NvSwitch found"` and the unit stays inactive; no kernel side effect, no restart loop, ~70–110 MB image cost. Set `false` to skip if you want to shave the image and know none of your fleet uses NVSwitch. | +| `NVIDIA_INSTALL_IMEX` | `true` | Installs `nvidia-imex-` (Internode Memory Exchange daemon) and enables the unit — **required** on **GB200 NVL72** for multi-node NVLink Sharp (Blackwell, driver 570+). Not needed for single-node HGX B200 or HGX H100. On non-NVL72 hosts the daemon has no `/etc/nvidia-imex/nodes_config.cfg` and exits cleanly, so the unit stays inactive; ~10–20 MB image cost. Best-effort: older driver branches (pre-570) do not publish the package and the install step is skipped with a warning. | +| `NVIDIA_INSTALL_CONTAINER_TOOLKIT` | `false` | Set `true` to also pre-install `nvidia-container-toolkit` **on the host**. Then set `toolkit.enabled=false` in the operator. Off by default because the operator ships the toolkit. | +| `NVIDIA_REBUILD_INITRD` | `true` | Rebuild the initrd so the `nouveau` blacklist applies during early boot. | + +### Choosing a driver branch + +Only certain branches publish the headless `-server` packages. Inside the base +image (or any Ubuntu 22.04 box with the CUDA repo added) you can list them: + +```sh +apt-cache search 'nvidia-headless-.*-server' +``` + +Pick a branch supported by both your GPU generation and the CUDA/toolkit versions +of the operator images you're bundling. + +### Choosing the module flavor (`NVIDIA_DRIVER_TYPE`) + +The default is `open`. It works on every server GPU generation Turing and newer, +and is **required** for Hopper and Blackwell. Override to `proprietary` only for +pre-Turing hardware. + +| GPU generation | Example cards | Required `NVIDIA_DRIVER_TYPE` | +| -------------- | ------------------------------------------------------ | ----------------------------- | +| Blackwell | RTX PRO 6000 Blackwell, B100, B200, GB200 | `open` (only) | +| Hopper | H100, H200 | `open` (only) | +| Ada Lovelace | L4, L40, L40S, RTX 6000 Ada | either (`open` recommended) | +| Ampere | A100, A10, A30, A40 | either | +| Turing | T4, RTX 20xx | either | +| Pre-Turing | V100, P100, P40 | `proprietary` (only) | + +Symptom of the wrong choice on Hopper/Blackwell: `nvidia-smi` reports +`No devices were found`, and `dmesg` shows one line per GPU of the form +`NVRM: GPU : RmInitAdapter failed! (0x22:0x56:897)`. In that state the +GPU Operator's toolkit init container loops on +`Attempting to validate a driver container installation`, containerd never +registers the `nvidia` runtime handler, the device plugin never advertises +`nvidia.com/gpu`, and workload pods stay `Pending` on +`Insufficient nvidia.com/gpu`. + +--- + +## What the script configures on the host + +- `/etc/modprobe.d/blacklist-nouveau.conf` — blacklists the `nouveau` driver. +- `/etc/modules-load.d/nvidia.conf` — autoloads `nvidia`, `nvidia_uvm`, + `nvidia_modeset`, `nvidia_drm` at boot. +- `/etc/modprobe.d/nvidia.conf` — `NVreg_PreserveVideoMemoryAllocations=1`. +- Enables `nvidia-persistenced.service` (recommended for data-center GPUs). +- Runs `depmod -a ` and rebuilds the initrd for the target kernel. + +Verify on a booted node: + +```sh +nvidia-smi +lsmod | grep nvidia +``` + +--- + +## Building the air-gapped content (which images to bundle) + +Pre-installing the driver in the OS only removes the **driver image**. The +operator still deploys everything else as containers, so those images (and a few +non-image steps) must be handled. **Bundling images alone is not sufficient.** + +### Images to mirror into your content bundle + +| Image | Needed with `driver.enabled=false`? | +| --- | --- | +| `gpu-operator` | Yes | +| `gpu-operator-validator` | Yes | +| `container-toolkit` | Yes — **unless** you also pre-installed it on the host (`NVIDIA_INSTALL_CONTAINER_TOOLKIT=true` → then set `toolkit.enabled=false` and skip this image) | +| `k8s-device-plugin` | Yes | +| `gpu-feature-discovery` | Yes | +| `dcgm` + `dcgm-exporter` | Yes, if you want GPU metrics | +| `node-feature-discovery` | Yes — unless the cluster already runs NFD (`nfd.enabled=false`) | +| CUDA validation image (`nvcr.io/nvidia/cuda:…`) | Yes — used by the validator init container (easy to miss) | +| `k8s-mig-manager` | Only if using MIG | +| **`driver`** | **No — skip it (that's the point of pre-installing)** | + +Don't transcribe tags by hand — they change per operator version. Render the +exact set from the chart and mirror precisely that: + +```sh +helm template gpu-operator nvidia/gpu-operator --version \ + --set driver.enabled=false | grep -Eo 'image: *"?[^"]+' | sort -u +``` + +### Non-image steps + +1. `--set driver.enabled=false`. +2. Override **every** image `repository` to your bundle/registry and set + `imagePullSecrets`. +3. **Palette Edge (k3s / rke2) gotcha:** the container-toolkit defaults assume + stock containerd. On k3s/rke2 you must point it at the right socket and + config, e.g.: + + ``` + --set toolkit.env[0].name=CONTAINERD_CONFIG \ + --set toolkit.env[0].value=/var/lib/rancher/k3s/agent/etc/containerd/config.toml \ + --set toolkit.env[1].name=CONTAINERD_SOCKET \ + --set toolkit.env[1].value=/run/k3s/containerd/containerd.sock \ + --set toolkit.env[2].name=CONTAINERD_RUNTIME_CLASS \ + --set toolkit.env[2].value=nvidia + ``` + + (rke2 paths: `/var/lib/rancher/rke2/agent/etc/containerd/config.toml.tmpl`, + `/run/k3s/containerd/containerd.sock`.) Miss this and workloads never get the + GPU runtime even though the driver is present. + +### Palette content bundle + +Add the GPU Operator as a Helm pack in the cluster profile, then build the +content bundle so it includes the rendered images above (minus `driver`). Images +set only via `values.yaml` may need to be added to the pack's additional-images +list if the bundle builder doesn't auto-detect them. Verify on a node with +`nvidia-smi` and by checking the operator's `*-validator` pods reach `Ready`. + +## Limitations / caveats + +- **Secure Boot / UKI is not supported by this path.** When `IS_UKI=true`, DKMS + modules are unsigned and will not load under Secure Boot; that requires MOK + signing, which this script does **not** implement. Use the standard (non-UKI) + Ubuntu image for GPU nodes. +- **amd64 / Ubuntu only.** The script targets apt-based Ubuntu images on + `x86_64` (with a best-effort `sbsa` path for arm64). Non-Ubuntu distributions + are out of scope. +- **Branch/version alignment is yours to own.** Make sure `NVIDIA_DRIVER_BRANCH` + matches the GPU hardware and the CUDA/toolkit versions expected by the operator + images in your bundle. diff --git a/earthly.sh b/earthly.sh index 9c50e22d..1147348e 100755 --- a/earthly.sh +++ b/earthly.sh @@ -309,6 +309,96 @@ if [[ "$1" == "+maas-image" ]]; then exit 0 fi +# --------------------------------------------------------------------------- +# AMD GPU driver prebuild (dkms mode only). +# +# amdgpu-dkms's ./configure fails inside Earthly's buildkit RUN sandbox +# (see docs/amd-gpu-airgapped.md). We pre-compile the module in a plain +# `docker run --privileged` container here on the build host -- the exact +# environment we verified works end-to-end -- and pass the resulting tarball +# to Earthly for a simple COPY + tar-extract + depmod inside the base image. +# --------------------------------------------------------------------------- + +# Read a --FOO=bar override out of $@ without consuming it. Prints the value +# or empty; the arg is still passed through to earthly untouched. +peek_arg() { + local key="$1" + local a + for a in "$@"; do + case "$a" in + --${key}=*) printf '%s' "${a#--${key}=}"; return ;; + esac + done +} + +INSTALL_AMD_GPU_DRIVERS_EFFECTIVE="$(peek_arg INSTALL_AMD_GPU_DRIVERS "$@")" +INSTALL_AMD_GPU_DRIVERS_EFFECTIVE="${INSTALL_AMD_GPU_DRIVERS_EFFECTIVE:-${INSTALL_AMD_GPU_DRIVERS:-false}}" +AMDGPU_DRIVER_SOURCE_EFFECTIVE="$(peek_arg AMDGPU_DRIVER_SOURCE "$@")" +AMDGPU_DRIVER_SOURCE_EFFECTIVE="${AMDGPU_DRIVER_SOURCE_EFFECTIVE:-${AMDGPU_DRIVER_SOURCE:-dkms}}" + +if [ "$INSTALL_AMD_GPU_DRIVERS_EFFECTIVE" = "true" ] && [ "$AMDGPU_DRIVER_SOURCE_EFFECTIVE" = "dkms" ]; then + # Derive the same BASE_IMAGE the Earthfile derives, so we compile against + # the same rootfs Earthly is about to build on. Only Ubuntu is supported + # for AMD driver pre-install; the AMD mutual-exclusion + Ubuntu-only checks + # elsewhere handle other OS_DISTRIBUTIONs. + AMDGPU_BASE_IMAGE="$(peek_arg BASE_IMAGE "$@")" + AMDGPU_BASE_IMAGE="${AMDGPU_BASE_IMAGE:-${BASE_IMAGE:-}}" + if [ -z "$AMDGPU_BASE_IMAGE" ]; then + _os_dist="$(peek_arg OS_DISTRIBUTION "$@")"; _os_dist="${_os_dist:-${OS_DISTRIBUTION:-ubuntu}}" + _os_ver="$(peek_arg OS_VERSION "$@")"; _os_ver="${_os_ver:-${OS_VERSION:-24.04}}" + _arch="$(peek_arg ARCH "$@")"; _arch="${_arch:-${ARCH:-amd64}}" + _kairos_ver="$(peek_arg KAIROS_VERSION "$@")"; _kairos_ver="${_kairos_ver:-${KAIROS_VERSION:-v4.0.4}}" + _kairos_url="$(peek_arg KAIROS_BASE_IMAGE_URL "$@")"; _kairos_url="${_kairos_url:-${KAIROS_BASE_IMAGE_URL:-$SPECTRO_PUB_REPO/edge}}" + _is_uki="$(peek_arg IS_UKI "$@")"; _is_uki="${_is_uki:-${IS_UKI:-false}}" + + if [ "$_os_dist" != "ubuntu" ]; then + echo "AMD GPU driver pre-install requires OS_DISTRIBUTION=ubuntu (got: $_os_dist)." >&2 + exit 1 + fi + # Same tag formula as Earthfile lines ~141-151. + if [ "$_os_ver" = "22" ] || [ "$_os_ver" = "20" ]; then + _tag="kairos-${_os_dist}:${_os_ver}.04-core-${_arch}-generic-${_kairos_ver}" + elif [ "$_is_uki" = "true" ]; then + _tag="kairos-${_os_dist}:${_os_ver}-core-${_arch}-generic-${_kairos_ver}-uki" + else + _tag="kairos-${_os_dist}:${_os_ver}-core-${_arch}-generic-${_kairos_ver}" + fi + AMDGPU_BASE_IMAGE="${_kairos_url}/${_tag}" + fi + + AMDGPU_DRIVER_RELEASE_EFFECTIVE="$(peek_arg AMDGPU_DRIVER_RELEASE "$@")" + AMDGPU_DRIVER_RELEASE_EFFECTIVE="${AMDGPU_DRIVER_RELEASE_EFFECTIVE:-${AMDGPU_DRIVER_RELEASE:-31.40}}" + + echo "=== Pre-building AMD amdgpu driver (dkms mode) ===" + echo " BASE_IMAGE: $AMDGPU_BASE_IMAGE" + echo " AMDGPU_DRIVER_RELEASE: $AMDGPU_DRIVER_RELEASE_EFFECTIVE" + prebuild_out="$( + BASE_IMAGE="$AMDGPU_BASE_IMAGE" \ + AMDGPU_DRIVER_RELEASE="$AMDGPU_DRIVER_RELEASE_EFFECTIVE" \ + AMDGPU_ARTIFACT_DIR="$(pwd)/build" \ + bash scripts/prebuild-amdgpu-artifact.sh + )" || { echo "AMD driver pre-build failed. See lines above." >&2; exit 1; } + + # Last line of prebuild output is: AMDGPU_ARTIFACT_PATH= + AMDGPU_ARTIFACT_PATH="$(printf '%s\n' "$prebuild_out" | tail -1 | sed -n 's/^AMDGPU_ARTIFACT_PATH=//p')" + [ -s "$AMDGPU_ARTIFACT_PATH" ] || { echo "Prebuild did not emit AMDGPU_ARTIFACT_PATH; aborting." >&2; exit 1; } + echo " Artifact: $AMDGPU_ARTIFACT_PATH" + + # Earthly's COPY reads from the repo build-context (the directory containing + # the Earthfile), not from the host filesystem, so we must pass a path + # relative to the repo root -- not the absolute host path. + repo_root="$(pwd)" + case "$AMDGPU_ARTIFACT_PATH" in + "$repo_root"/*) AMDGPU_ARTIFACT_REL="${AMDGPU_ARTIFACT_PATH#$repo_root/}" ;; + *) echo "Prebuild artifact '$AMDGPU_ARTIFACT_PATH' is outside repo root '$repo_root'; \ +COPY into Earthly would fail. Move the artifact under the repo tree." >&2 ; exit 1 ;; + esac + + # Thread the (repo-relative) artifact path through to Earthly. Its Earthfile + # ARG (added in the companion commit) picks this up and consumes the tarball. + set -- "$@" "--AMDGPU_ARTIFACT_PATH=$AMDGPU_ARTIFACT_REL" +fi + # Normal build flow for other targets if [ -z "$HTTP_PROXY" ] && [ -z "$HTTPS_PROXY" ] && [ -z "$(find certs -type f ! -name '.*' -print -quit)" ]; then build_without_proxy "$@" diff --git a/overlay/files-iso/boot/grub2/grub.cfg b/overlay/files-iso/boot/grub2/grub.cfg index 448421e7..62b5af0f 100644 --- a/overlay/files-iso/boot/grub2/grub.cfg +++ b/overlay/files-iso/boot/grub2/grub.cfg @@ -1,33 +1,56 @@ -search --file --set=root /boot/kernel.xz +search --no-floppy --file --set=root /boot/kernel set default={{DEFAULT_ENTRY}} set timeout=5 set timeout_style=menu set linux=linux set initrd=initrd + +# Video/console parameters are firmware-specific. +# +# BIOS/CSM: vga=795 requests VESA mode 0x31B via INT 10h from the legacy video +# BIOS, giving the kernel a framebuffer console. Unchanged. +# +# UEFI: there is no VBE, so vga= is inert. Combined with nomodeset (which +# blocks the native KMS driver) the kernel ends up with no usable +# framebuffer and the screen stays black. The last console= also wins +# for /dev/console, so console=tty0 must come last or all output goes +# to serial. The baud is pinned because an unqualified console=ttyS0 +# defaults to 9600 while BMC SOL typically runs at 115200. if [ "${grub_platform}" = "efi" ]; then - echo "Please press 't' to show the boot menu on this console" + set consoleparams="console=ttyS0,115200n8 console=tty0" + set videoparams="" +else + set consoleparams="console=tty1 console=ttyS0" + set videoparams="vga=795 nomodeset" fi -set font=($root)/boot/${grub_cpu}/loader/grub2/fonts/unicode.pf2 -if [ -f ${font} ];then - loadfont ${font} + +# loadfont switches GRUB to gfxterm. Some UEFI implementations (observed on +# Supermicro with ATEN BMC) render nothing in that mode, leaving a blank menu, +# so keep the graphical terminal on the BIOS path only. +if [ "${grub_platform}" != "efi" ]; then + set font=($root)/boot/${grub_cpu}/loader/grub2/fonts/unicode.pf2 + if [ -f ${font} ];then + loadfont ${font} + fi fi + menuentry "Palette eXtended Kubernetes Edge Installer" --class os --unrestricted { echo Loading kernel... - $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable vga=795 nomodeset nodepair.enable selinux=0 rd.live.overlay.overlayfs rd.immucore.sysrootwait=600 systemd.unified_cgroup_hierarchy=1 + $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 $consoleparams rd.cos.disable $videoparams nodepair.enable selinux=0 rd.live.overlay.overlayfs rd.immucore.sysrootwait=600 systemd.unified_cgroup_hierarchy=1 rd.driver.blacklist=nouveau,qat_4xxx modprobe.blacklist=nouveau,qat_4xxx nouveau.modeset=0 echo Loading initrd... $initrd ($root)/boot/initrd } menuentry "Palette eXtended Kubernetes Edge Installer (manual)" --class os --unrestricted { echo Loading kernel... - $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable selinux=0 rd.live.overlay.overlayfs rd.immucore.sysrootwait=600 systemd.unified_cgroup_hierarchy=1 + $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 $consoleparams rd.cos.disable selinux=0 rd.live.overlay.overlayfs rd.immucore.sysrootwait=600 systemd.unified_cgroup_hierarchy=1 rd.driver.blacklist=nouveau,qat_4xxx modprobe.blacklist=nouveau,qat_4xxx nouveau.modeset=0 echo Loading initrd... $initrd ($root)/boot/initrd } menuentry "Palette Edge Interactive Installer" --class os --unrestricted { echo Loading kernel... - $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable vga=795 nomodeset nodepair.enable selinux=0 rd.live.overlay.overlayfs rd.immucore.sysrootwait=600 systemd.unified_cgroup_hierarchy=1 interactive-install + $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 $consoleparams rd.cos.disable $videoparams nodepair.enable selinux=0 rd.live.overlay.overlayfs rd.immucore.sysrootwait=600 systemd.unified_cgroup_hierarchy=1 rd.driver.blacklist=nouveau,qat_4xxx modprobe.blacklist=nouveau,qat_4xxx nouveau.modeset=0 interactive-install echo Loading initrd... $initrd ($root)/boot/initrd } diff --git a/scripts/install-amdgpu-drivers.sh b/scripts/install-amdgpu-drivers.sh new file mode 100755 index 00000000..994ae974 --- /dev/null +++ b/scripts/install-amdgpu-drivers.sh @@ -0,0 +1,677 @@ +#!/usr/bin/env bash +# +# install-amdgpu-drivers.sh +# +# Pre-provision the AMD Instinct GPU kernel-mode driver INTO a CanvOS / Kairos +# Ubuntu base image, so a node booted from the image can run the AMD GPU Operator +# with `driver.enable=false` in a fully air-gapped environment. +# +# MODES (AMDGPU_DRIVER_SOURCE) +# ---------------------------- +# dkms (default) -- install AMD's amdgpu-dkms. Two execution paths: +# (a) if AMDGPU_ARTIFACT_PATH is set (Earthly build via +# earthly.sh's prebuild helper), extract the pre- +# compiled modules + firmware + drop-ins from the +# tarball, depmod, rebuild initrd. Fast. +# (b) otherwise download from repo.radeon.com and +# DKMS-build against the image kernel in-place. +# Works under `docker run --privileged`; FAILS in +# Earthly's buildkit RUN sandbox at AMD's ./configure +# step -- use path (a) for Earthly. +# inbox -- do NOT install the AMD apt repo or amdgpu-dkms. Rely on +# the in-tree `amdgpu` module that Ubuntu ships with +# linux-modules-$(uname -r) and the firmware blobs in +# linux-firmware. Only ensures amdgpu autoloads. Choose +# this when you accept the in-tree driver's feature set +# (may miss recent SMU / per-SKU support). +# +# WHAT THIS COVERS (dkms mode, OS side only) +# ------------------------------------------ +# * build toolchain (gcc, make, dkms, kmod, libc headers) +# * kernel headers matching the image kernel (via install-kernel-headers.sh) +# * linux-modules-extra for the image kernel +# * amdgpu-dkms + amdgpu-dkms-firmware, built against the IMAGE kernel +# * amdgpu module autoload + initrd refresh +# * amd-smi host CLI on PATH (parity with nvidia-smi) when +# AMDGPU_INSTALL_SMI=true (default). rocm-smi is deprecated and NOT installed. +# * a marker at /etc/canvos/amdgpu-driver-source recording which mode ran +# +# WHAT THIS DOES *NOT* COVER (both modes) -- ship these as container images in +# your content bundle, deployed by the AMD GPU Operator itself: +# * ROCm user-space, device-plugin, node-labeller, metrics exporter, etc. +# +# At Helm-install time you MUST tell the operator the driver is pre-installed: +# --set driver.enable=false # note: "enable", not "enabled" +# +# WHY THE DKMS DANCE (same rationale as install-nvidia-drivers.sh) +# --------------------------------------------------------------- +# In an Earthly/Docker build `uname -r` is the BUILD HOST kernel, not the kernel +# baked into the image. We derive the target kernel from /lib/modules (the +# kernel that will actually boot) and force the DKMS build + module install + +# depmod against THAT kernel. +# +# TUNABLES (environment variables; all optional) +# AMDGPU_DRIVER_SOURCE dkms | inbox. Default: dkms. +# AMDGPU_DRIVER_RELEASE amdgpu-install release marker (URL segment under +# repo.radeon.com/amdgpu-install//). AMD publishes +# both ROCm-alias paths (7.2.1, 7.2.4) and driver- +# release-marker paths (30.30.x, 31.40); either form +# is accepted. Default: 31.40 -- ROCm 7.14 GA (amdgpu +# 6.19.14), a PRODUCTION driver (7.14 went GA on +# 2026-07-15; the earlier "31.x = tech-preview" note +# referred to the ROCm 7.13.0 preview and is obsolete). +# 31.40 is the baseline for GPU Operator v1.5.1 and +# for MI350P + Radeon AI PRO (RDNA4). For an older +# fleet staying on Operator v1.5.0, use 7.2.1 (amdgpu +# 6.16.13, 30.30.1 line). Ignored in inbox mode. See +# docs/amd-gpu-airgapped.md for the operator<->driver +# compatibility matrix. +# AMDGPU_INSTALL_SMI true | false. Default: true. Install the amd-smi +# host CLI and symlink into /usr/bin, for parity with +# nvidia-smi. Prefers amdrocm-amdsmi from repo.amd.com +# (user-space matching recent drivers, e.g. 7.14 for +# the 31.40 driver), falling back to amd-smi-lib from +# repo.radeon.com. Best-effort: a failure warns but +# does not fail the build. +# AMDGPU_ROCM_APT_VERSION Legacy repo.radeon.com/rocm/apt version, used only +# for the amd-smi-lib fallback. Default: latest. +# AMDGPU_REBUILD_INITRD "true" to rebuild the initrd for the image kernel. +# Default: false. amdgpu is intentionally NOT +# included in the initrd (see the dracut omit +# drop-in the script writes). The base image's +# existing initrd already handles rootfs mount; +# amdgpu loads after switch-root via +# /etc/modules-load.d/amdgpu.conf. +# +set -eo pipefail +set -u + +log() { echo "[install-amdgpu-drivers] $*"; } +warn() { echo "[install-amdgpu-drivers] WARNING: $*" >&2; } +die() { echo "[install-amdgpu-drivers] ERROR: $*" >&2; exit 1; } + +# --------------------------------------------------------------------------- +# ensure_rocm_repo: make an amd-smi package installable and record its name in +# AMDSMI_PKG. Preference order: +# 1. already resolvable (e.g. the amdgpu-install deb configured a repo in the +# in-buildkit dkms path) -- prefer amdrocm-amdsmi, else amd-smi-lib; +# 2. the current ROCm repo repo.amd.com/rocm/packages-multi-arch, which ships +# amdrocm-amdsmi matching recent drivers (7.14 user-space for the 31.40 +# driver) -- this is the repo AMD's amd-smi install docs point to, keyed by +# Ubuntu version (noble -> ubuntu2404); +# 3. the legacy repo.radeon.com/rocm/apt, which ships amd-smi-lib (7.2.x). +# Returns 0 with AMDSMI_PKG set on success, non-zero if no repo yields a package. +# --------------------------------------------------------------------------- +ensure_rocm_repo() { + apt-get install -y --no-install-recommends ca-certificates curl gnupg >/dev/null 2>&1 || true + apt-get update >/dev/null 2>&1 || true + + # (1) Already resolvable (repo configured by amdgpu-install). Prefer the + # modern amdrocm-amdsmi; fall back to amd-smi-lib. + if apt-cache show amdrocm-amdsmi >/dev/null 2>&1; then AMDSMI_PKG="amdrocm-amdsmi"; return 0; fi + if apt-cache show amd-smi-lib >/dev/null 2>&1; then AMDSMI_PKG="amd-smi-lib"; return 0; fi + + install -d -m 0755 /etc/apt/keyrings + + # (2) Preferred: repo.amd.com (current ROCm packaging), keyed by Ubuntu + # version. Ships amd-smi user-space matching recent drivers. + if [ -n "${ubuntu_ver:-}" ] \ + && curl -fsSL https://repo.amd.com/rocm/packages-multi-arch/gpg/rocm.gpg 2>/dev/null \ + | gpg --dearmor --yes -o /etc/apt/keyrings/rocm.gpg 2>/dev/null; then + chmod a+r /etc/apt/keyrings/rocm.gpg + echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.amd.com/rocm/packages-multi-arch/ubuntu${ubuntu_ver} stable main" \ + > /etc/apt/sources.list.d/rocm.list + apt-get update >/dev/null 2>&1 || true + if apt-cache show amdrocm-amdsmi >/dev/null 2>&1; then + log "Using amdrocm-amdsmi from repo.amd.com (ubuntu${ubuntu_ver})." + AMDSMI_PKG="amdrocm-amdsmi" + return 0 + fi + warn "amdrocm-amdsmi not found in repo.amd.com/ubuntu${ubuntu_ver}; trying legacy ROCm apt repo." + else + warn "could not add repo.amd.com ROCm repo; trying legacy ROCm apt repo." + fi + + # (3) Fallback: legacy repo.radeon.com/rocm/apt (ships amd-smi-lib). + log "Adding legacy ROCm apt repo (repo.radeon.com/rocm/apt/${AMDGPU_ROCM_APT_VERSION}) for amd-smi ..." + if ! curl -fsSL https://repo.radeon.com/rocm/rocm.gpg.key 2>/dev/null \ + | gpg --dearmor --yes -o /etc/apt/keyrings/rocm.gpg 2>/dev/null; then + warn "could not fetch/dearmor the legacy ROCm gpg key." + return 1 + fi + chmod a+r /etc/apt/keyrings/rocm.gpg + echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/${AMDGPU_ROCM_APT_VERSION} ${codename} main" \ + > /etc/apt/sources.list.d/rocm.list + apt-get update >/dev/null 2>&1 || { warn "apt-get update after adding legacy ROCm repo failed."; return 1; } + apt-cache show amd-smi-lib >/dev/null 2>&1 || { warn "amd-smi-lib not found even after adding the legacy ROCm repo."; return 1; } + AMDSMI_PKG="amd-smi-lib" + return 0 +} + +# --------------------------------------------------------------------------- +# install_amd_smi: install the amd-smi host CLI (amdrocm-amdsmi from repo.amd.com +# where available, else legacy amd-smi-lib) and put it on PATH at /usr/bin, +# mirroring how nvidia-utils gives /usr/bin/nvidia-smi. Best-effort by design -- +# a diagnostic tool must never fail the image build. +# --------------------------------------------------------------------------- +install_amd_smi() { + if [ "${AMDGPU_INSTALL_SMI}" != "true" ]; then + log "AMDGPU_INSTALL_SMI=false; skipping amd-smi host CLI." + return 0 + fi + log "Installing amd-smi host CLI (parity with nvidia-smi) ..." + AMDSMI_PKG="" + if ! ensure_rocm_repo || [ -z "${AMDSMI_PKG}" ]; then + warn "ROCm repo unavailable; skipping amd-smi. Set AMDGPU_INSTALL_SMI=false to silence." + return 0 + fi + # amd-smi dlopens libdrm_amdgpu.so.1 at runtime to enumerate GPUs (it is NOT + # a linked dependency, so apt won't pull it -- and --no-install-recommends + # would drop it anyway). Without it amd-smi fails with + # "Fail to open libdrm_amdgpu.so.1 ... Unable to detect any GPU devices" + # even when amdgpu is loaded. libdrm-amdgpu1 (from the base Ubuntu repo) + # provides the SONAME, so install it explicitly alongside the SMI package. + log "Installing ${AMDSMI_PKG} + libdrm-amdgpu1 ..." + if ! apt-get install -y --no-install-recommends "${AMDSMI_PKG}" libdrm-amdgpu1; then + warn "${AMDSMI_PKG} + libdrm-amdgpu1 install failed; retrying with ${AMDSMI_PKG} only." + apt-get install -y --no-install-recommends "${AMDSMI_PKG}" \ + || { warn "${AMDSMI_PKG} install failed; continuing without amd-smi."; return 0; } + fi + # The amd-smi binary lands under /opt/rocm*/bin (legacy amd-smi-lib) or + # /opt/rocm*/core-*/bin (repo.amd.com core-SDK layout); neither is on the + # default PATH. Symlink it into /usr/bin so `amd-smi` works ootb like + # `nvidia-smi`. rocm-smi is intentionally NOT installed -- it is deprecated + # and fully superseded by amd-smi. + smi_bin="$(ls /opt/rocm*/core-*/bin/amd-smi /opt/rocm*/bin/amd-smi 2>/dev/null | sort -V | tail -1 || true)" + if [ -n "${smi_bin}" ]; then + ln -sf "${smi_bin}" /usr/bin/amd-smi + log "Linked ${smi_bin} -> /usr/bin/amd-smi" + fi + if [ -x /usr/bin/amd-smi ]; then + AMDGPU_SMI_INSTALLED="yes" + else + warn "amd-smi binary not found under /opt/rocm* after installing ${AMDSMI_PKG}." + fi +} + +# --------------------------------------------------------------------------- +# Config +# --------------------------------------------------------------------------- +AMDGPU_DRIVER_SOURCE="${AMDGPU_DRIVER_SOURCE:-dkms}" +AMDGPU_DRIVER_RELEASE="${AMDGPU_DRIVER_RELEASE:-31.40}" +AMDGPU_REBUILD_INITRD="${AMDGPU_REBUILD_INITRD:-false}" +AMDGPU_INSTALL_SMI="${AMDGPU_INSTALL_SMI:-true}" +AMDGPU_ROCM_APT_VERSION="${AMDGPU_ROCM_APT_VERSION:-latest}" +AMDGPU_SMI_INSTALLED="" # set to "yes" by install_amd_smi on success +AMDSMI_PKG="" # set by ensure_rocm_repo to the resolvable amd-smi package +ubuntu_ver="" # e.g. 2404 for repo.amd.com/.../ubuntu2404 (set from os-release below) + +case "${AMDGPU_DRIVER_SOURCE}" in + dkms|inbox) ;; + *) die "AMDGPU_DRIVER_SOURCE must be 'dkms' or 'inbox' (got: '${AMDGPU_DRIVER_SOURCE}')." ;; +esac + +export DEBIAN_FRONTEND=noninteractive + +command -v apt-get >/dev/null 2>&1 || die "this script only supports apt-based (Ubuntu/Debian) images." + +# --------------------------------------------------------------------------- +# 1. Identify the kernel shipped in the image (NOT the build host kernel) +# --------------------------------------------------------------------------- +KVER="$(printf '%s\n' /lib/modules/* 2>/dev/null | xargs -n1 basename 2>/dev/null | sort -V | tail -1)" +[ -n "${KVER}" ] || die "could not determine target kernel from /lib/modules." +log "Target (image) kernel: ${KVER}" +log "Driver source mode: ${AMDGPU_DRIVER_SOURCE}" +[ "${AMDGPU_DRIVER_SOURCE}" = "dkms" ] && log "AMD driver release: ${AMDGPU_DRIVER_RELEASE}" + +# Ubuntu release codename (jammy / noble) read from the image itself. +codename="" +osid="ubuntu" +if [ -r /etc/os-release ]; then + # shellcheck disable=SC1091 + . /etc/os-release + codename="${VERSION_CODENAME:-}" + osid="${ID:-ubuntu}" + # repo.amd.com component tag: VERSION_ID 24.04 -> ubuntu2404, 22.04 -> ubuntu2204. + ubuntu_ver="$(printf '%s' "${VERSION_ID:-}" | tr -d '.')" +fi +[ -n "${codename}" ] || die "could not determine Ubuntu codename from /etc/os-release." + +mkdir -p /etc/canvos + +# --------------------------------------------------------------------------- +# INBOX MODE: skip the AMD apt repo and DKMS entirely. Rely on the in-tree +# amdgpu module shipped with the image's linux-modules-* package. Only ensure +# the module autoloads at boot, then rebuild initrd if requested. +# --------------------------------------------------------------------------- +if [ "${AMDGPU_DRIVER_SOURCE}" = "inbox" ]; then + log "inbox mode: verifying in-tree amdgpu module is present under /lib/modules/${KVER}/kernel/..." + if ! find "/lib/modules/${KVER}" -path '*/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu.ko*' 2>/dev/null | grep -q .; then + die "inbox mode selected but no in-tree amdgpu module found under /lib/modules/${KVER}/kernel/. \ +This image kernel does not ship an in-tree amdgpu driver -- switch to \ +AMDGPU_DRIVER_SOURCE=dkms or pick a different image kernel." + fi + + log "Configuring amdgpu module autoload ..." + cat > /etc/modules-load.d/amdgpu.conf <<'EOF' +# Managed by CanvOS install-amdgpu-drivers.sh (inbox mode) +# Load the in-tree AMD GPU driver at boot so the AMD GPU Operator sees a ready driver. +amdgpu +EOF + + log "Running depmod -a ${KVER} ..." + depmod -a "${KVER}" + + # Even if AMDGPU_REBUILD_INITRD=true, we explicitly OMIT amdgpu from the + # initrd. On multi-GPU MI systems (e.g. 8x MI325X, 8 XCP partitions each) + # amdgpu init emits so many udev events that dracut's initqueue times out + # waiting for udev-settle before rootfs pivot, leaving the node in + # dracut-emergency. amdgpu isn't needed to mount root (NVMe/SATA use their + # own drivers) so it's safe to load it *after* switch-root via + # /etc/modules-load.d/amdgpu.conf where there is no timeout pressure. + log "Configuring dracut to OMIT amdgpu from initrd (avoid init-time udev storm) ..." + mkdir -p /etc/dracut.conf.d + cat > /etc/dracut.conf.d/98-canvos-amdgpu-omit.conf <<'EOF' +# Managed by CanvOS install-amdgpu-drivers.sh +# Keep amdgpu (and its DKMS helpers) OUT of the initrd. amdgpu emits enough +# udev events at load time (per-XCP-partition, per-ring) to blow past dracut's +# initqueue timeout on multi-GPU systems, dropping the node into emergency +# mode. amdgpu is not required to mount the rootfs; systemd loads it via +# modules-load.d after switch-root. +omit_drivers+=" amdgpu amdttm amdkcl amd-sched amddrm_ttm_helper amddrm_buddy amddrm_exec amdxcp " +EOF + + if [ "${AMDGPU_REBUILD_INITRD}" = "true" ]; then + if command -v dracut >/dev/null 2>&1; then + log "Rebuilding initrd for ${KVER} (dracut, amdgpu omitted) ..." + dracut -f "/boot/initrd-${KVER}" "${KVER}" + ln -sf "initrd-${KVER}" /boot/initrd + elif command -v update-initramfs >/dev/null 2>&1; then + log "Rebuilding initramfs for ${KVER} (update-initramfs) ..." + update-initramfs -u -k "${KVER}" + else + warn "no dracut or update-initramfs found; skipping initrd rebuild." + fi + fi + + install_amd_smi || true # best-effort; must never fail the build + + printf 'AMDGPU_DRIVER_SOURCE=inbox\nAMDGPU_SMI=%s\nKVER=%s\n' \ + "${AMDGPU_SMI_INSTALLED:-no}" "${KVER}" > /etc/canvos/amdgpu-driver-source + + apt-get clean || true + rm -rf /var/lib/apt/lists/* + + log "Done. Using in-tree amdgpu driver for kernel ${KVER}." + log "Reminder: install the AMD GPU Operator with 'driver.enable=false'." + exit 0 +fi + +# --------------------------------------------------------------------------- +# DKMS MODE with a pre-built artifact (produced by +# scripts/prebuild-amdgpu-artifact.sh on the build host). +# +# Buildkit's RUN sandbox breaks AMD's amdgpu-dkms ./configure heredoc probe, +# so we compile outside Earthly in `docker run --privileged` and consume the +# resulting tarball here. Structurally this branch just extracts the tarball, +# runs depmod against the target kernel, and rebuilds the initrd. +# --------------------------------------------------------------------------- +AMDGPU_ARTIFACT_PATH="${AMDGPU_ARTIFACT_PATH:-}" +if [ "${AMDGPU_DRIVER_SOURCE}" = "dkms" ] && [ -n "${AMDGPU_ARTIFACT_PATH}" ]; then + log "dkms mode: consuming pre-built artifact ${AMDGPU_ARTIFACT_PATH}" + [ -s "${AMDGPU_ARTIFACT_PATH}" ] || die "AMDGPU_ARTIFACT_PATH='${AMDGPU_ARTIFACT_PATH}' \ +is not a non-empty file inside the build container. Verify the earthly.sh \ +wrapper produced it and Earthly COPYed it in." + + log "Extracting artifact into root filesystem ..." + tar -xzf "${AMDGPU_ARTIFACT_PATH}" -C / \ + || die "tar extraction of ${AMDGPU_ARTIFACT_PATH} failed." + + MODDIR="/lib/modules/${KVER}" + if ! find "${MODDIR}/updates/dkms" -name 'amdgpu.ko*' 2>/dev/null | grep -q .; then + find "${MODDIR}" -name 'amdgpu.ko*' 2>/dev/null | sed 's/^/ /' >&2 || true + die "amdgpu module missing under ${MODDIR}/updates/dkms after extract. \ +Artifact was built for a different kernel? Delete build/amdgpu-artifact-*.tar.gz \ +and rebuild (AMDGPU_FORCE_REBUILD=1) or verify BASE_IMAGE matches." + fi + + log "Running depmod -a ${KVER} ..." + depmod -a "${KVER}" || die "depmod failed for ${KVER}." + + # Even if AMDGPU_REBUILD_INITRD=true, we explicitly OMIT amdgpu from the + # initrd. On multi-GPU MI systems (e.g. 8x MI325X, 8 XCP partitions each) + # amdgpu init emits so many udev events that dracut's initqueue times out + # waiting for udev-settle before rootfs pivot, leaving the node in + # dracut-emergency. amdgpu isn't needed to mount root (NVMe/SATA use their + # own drivers) so it's safe to load it *after* switch-root via + # /etc/modules-load.d/amdgpu.conf where there is no timeout pressure. + log "Configuring dracut to OMIT amdgpu from initrd (avoid init-time udev storm) ..." + mkdir -p /etc/dracut.conf.d + cat > /etc/dracut.conf.d/98-canvos-amdgpu-omit.conf <<'EOF' +# Managed by CanvOS install-amdgpu-drivers.sh +# Keep amdgpu (and its DKMS helpers) OUT of the initrd. amdgpu emits enough +# udev events at load time (per-XCP-partition, per-ring) to blow past dracut's +# initqueue timeout on multi-GPU systems, dropping the node into emergency +# mode. amdgpu is not required to mount the rootfs; systemd loads it via +# modules-load.d after switch-root. +omit_drivers+=" amdgpu amdttm amdkcl amd-sched amddrm_ttm_helper amddrm_buddy amddrm_exec amdxcp " +EOF + + if [ "${AMDGPU_REBUILD_INITRD}" = "true" ]; then + if command -v dracut >/dev/null 2>&1; then + log "Rebuilding initrd for ${KVER} (dracut, amdgpu omitted) ..." + dracut -f "/boot/initrd-${KVER}" "${KVER}" + ln -sf "initrd-${KVER}" /boot/initrd + elif command -v update-initramfs >/dev/null 2>&1; then + log "Rebuilding initramfs for ${KVER} (update-initramfs) ..." + update-initramfs -u -k "${KVER}" + else + warn "no dracut or update-initramfs found; skipping initrd rebuild." + fi + fi + + install_amd_smi || true # best-effort; must never fail the build + + # Marker written by the prebuild is preserved from the tar. Overwrite + # any prebuild-mode marker with the final in-image reality. + printf 'AMDGPU_DRIVER_SOURCE=dkms (artifact)\nAMDGPU_DRIVER_RELEASE=%s\nAMDGPU_SMI=%s\nKVER=%s\n' \ + "${AMDGPU_DRIVER_RELEASE}" "${AMDGPU_SMI_INSTALLED:-no}" "${KVER}" > /etc/canvos/amdgpu-driver-source + + apt-get clean || true + rm -rf /var/lib/apt/lists/* + + log "Done. AMD amdgpu driver (release ${AMDGPU_DRIVER_RELEASE}, artifact) baked in for kernel ${KVER}." + log "Reminder: install the AMD GPU Operator with 'driver.enable=false'." + exit 0 +fi + +# --------------------------------------------------------------------------- +# DKMS MODE in-buildkit (fallback). Runs the full apt + DKMS build inside +# the image. This path fails inside Earthly's buildkit RUN sandbox at AMD's +# ./configure step (see docs) but is retained for: +# - direct `docker run --privileged` invocations (proven working), +# - the scripts/prebuild-amdgpu-artifact.sh helper, which uses this same +# script inside the container it spawns. +# --------------------------------------------------------------------------- + +# --------------------------------------------------------------------------- +# 2. Build toolchain +# --------------------------------------------------------------------------- +log "Installing build toolchain ..." +apt-get update || true +# The full Kbuild bootstrap: gcc/make/libc from build-essential PLUS the +# tools that recent kernel Makefiles pull in unconditionally. Missing any of +# these fails AMD's amdgpu-dkms ./configure at "cannot detect CFLAGS..." -- +# the failure mode is silent because CFLAGS-detection just runs `make -f -` +# and swallows stderr. We enumerate them explicitly instead of relying on +# --install-recommends (which would also pull other unwanted docs/data). +# bc, bison, flex : referenced by kernel Kbuild machinery +# libelf-dev : module utilities (modpost) + BPF +# libssl-dev : signing certificates / hash routines +# dwarves : pahole for BTF debuginfo (amdgpu-dkms explicitly Recommends this) +# cpio, xz-utils : initramfs assembly (may be needed by initramfs-tools trigger) +apt-get install -y --no-install-recommends \ + ca-certificates curl wget gnupg \ + build-essential gcc make \ + dkms kmod libc6-dev initramfs-tools \ + bc bison flex libelf-dev libssl-dev dwarves \ + cpio xz-utils \ + || die "failed to install build toolchain." + +# --------------------------------------------------------------------------- +# 3. Kernel headers + modules-extra matching the image kernel +# --------------------------------------------------------------------------- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +HEADERS_HELPER="" +for cand in "${SCRIPT_DIR}/install-kernel-headers.sh" /tmp/install-kernel-headers.sh; do + [ -r "${cand}" ] && { HEADERS_HELPER="${cand}"; break; } +done + +if [ -n "${HEADERS_HELPER}" ]; then + log "Installing kernel headers via ${HEADERS_HELPER} ..." + bash "${HEADERS_HELPER}" || warn "kernel-headers helper returned non-zero; continuing." +else + log "Header helper not found; attempting a direct header install ..." + apt-get install -y "linux-headers-${KVER}" || \ + apt-get install -y linux-headers-generic || \ + warn "could not install linux-headers-${KVER}." +fi + +# amdgpu depends on modules that live in linux-modules-extra (e.g. for some +# PCIe / crypto / networking helpers). Best-effort -- may be absent if Ubuntu +# rotated the ABI out of the live mirror. +log "Installing linux-modules-extra-${KVER} (best-effort) ..." +apt-get install -y "linux-modules-extra-${KVER}" || \ + warn "linux-modules-extra-${KVER} not available; continuing." + +# DKMS needs /lib/modules//build to point at the headers source tree. +if [ ! -e "/lib/modules/${KVER}/build" ]; then + src="$(ls -d /usr/src/linux-headers-${KVER} 2>/dev/null | head -1)" + if [ -n "${src}" ]; then + ln -sfn "${src}" "/lib/modules/${KVER}/build" + log "Linked /lib/modules/${KVER}/build -> ${src}" + else + warn "no /usr/src/linux-headers-${KVER}; DKMS build will likely fail." + fi +fi + +# --------------------------------------------------------------------------- +# 4. Register the AMD driver repo via the amdgpu-install package +# The amdgpu-install .deb (AMD's blessed entry point) configures the correct +# versioned driver apt repo + GPG key for the requested release. Its +# filename carries a build number, so we auto-discover it from the directory +# listing rather than hardcoding it. +# --------------------------------------------------------------------------- +inst_dir="https://repo.radeon.com/amdgpu-install/${AMDGPU_DRIVER_RELEASE}/${osid}/${codename}" +log "Locating amdgpu-install package under ${inst_dir}/ ..." +deb_name="$(curl -fsSL "${inst_dir}/" 2>/dev/null \ + | grep -oE 'amdgpu-install_[0-9A-Za-z._-]+_all\.deb' | sort -u | tail -1)" +[ -n "${deb_name}" ] || die "could not find an amdgpu-install package for driver \ +release '${AMDGPU_DRIVER_RELEASE}' on ${codename} at ${inst_dir}/. Check available \ +releases at https://repo.radeon.com/amdgpu-install/ and pick one that supports \ +your image kernel (${KVER}); see docs/amd-gpu-airgapped.md for the mapping." + +log "Installing ${deb_name} (configures the AMD driver apt repo) ..." +wget -qO /tmp/amdgpu-install.deb "${inst_dir}/${deb_name}" \ + || die "failed to download ${deb_name}." +apt-get install -y /tmp/amdgpu-install.deb || die "failed to install amdgpu-install." +rm -f /tmp/amdgpu-install.deb +apt-get update || warn "apt-get update after adding the AMD repo failed." + +# --------------------------------------------------------------------------- +# 4b. Disable DKMS module signing before installing amdgpu-dkms. +# +# amdgpu-dkms (>= 6.18 range, and observed on 31.x releases) invokes mokutil +# from the DKMS sign_tool hook to enroll a Machine Owner Key, which reads +# /sys/firmware/efi/efivars. Docker/Earthly build containers don't expose +# efivars, so mokutil aborts with: +# "EFI variables are not supported on this system / +# /sys/firmware/efi/efivars not found, aborting." +# and the amdgpu-dkms postinst returns non-zero. Empty sign_tool tells DKMS +# to skip signing entirely, sidestepping the mokutil invocation. +# +# CAVEAT: modules produced this way are unsigned -- consistent with the +# Secure Boot / UKI limitation already documented in docs/amd-gpu-airgapped.md. +# --------------------------------------------------------------------------- +log "Disabling DKMS module signing (container has no UEFI efivars) ..." +mkdir -p /etc/dkms/framework.conf.d +cat > /etc/dkms/framework.conf.d/canvos-no-mok-signing.conf <<'EOF' +# Managed by CanvOS install-amdgpu-drivers.sh +# Empty sign_tool tells DKMS to skip module signing. Required for building +# amdgpu-dkms inside container image builds where /sys/firmware/efi/efivars +# is not available. Modules are unsigned; this path does not support Secure Boot. +sign_tool="" +EOF + +# --------------------------------------------------------------------------- +# 5. Install the kernel-mode driver (amdgpu-dkms + firmware). +# Any apt/postinst failure surfaces here -- DO NOT swallow errors; a broken +# DKMS build must fail the image build so the user can fix AMDGPU_DRIVER_RELEASE +# or fall back to AMDGPU_DRIVER_SOURCE=inbox. +# --------------------------------------------------------------------------- +log "Installing amdgpu-dkms + amdgpu-dkms-firmware ..." +if ! apt-get install -y --no-install-recommends amdgpu-dkms amdgpu-dkms-firmware; then + # apt/postinst failure -- dump the DKMS build artifacts so root-causing + # doesn't require an interactive session (Earthly's -i tty is often broken). + log "apt install failed. Dumping DKMS build artifacts for diagnosis:" + log "--- dkms status ---" + dkms status 2>&1 | sed 's/^/ /' || true + for f in /var/lib/dkms/amdgpu/*/build/make.log; do + [ -r "$f" ] || continue + log "--- ${f} (tail -150) ---" + tail -n 150 "$f" | sed 's/^/ /' || true + done + log "--- environment probes ---" + log " kernel: $(uname -r); target KVER: ${KVER}" + log " linux-headers pkg: $(dpkg -l "linux-headers-${KVER}" 2>/dev/null | awk '/^ii/{print $2, $3}')" + log " /lib/modules/${KVER}/build: $(readlink -f "/lib/modules/${KVER}/build" 2>/dev/null || echo MISSING)" + log " /usr/src/linux-headers-${KVER}/Module.symvers: $(test -s "/usr/src/linux-headers-${KVER}/Module.symvers" && echo present || echo missing/empty)" + log " sign_tool drop-in: $(test -r /etc/dkms/framework.conf.d/canvos-no-mok-signing.conf && grep -E '^sign_tool' /etc/dkms/framework.conf.d/canvos-no-mok-signing.conf || echo MISSING)" + log " memory: $(awk '/MemAvailable/{print $2/1024" MiB avail"}' /proc/meminfo)" + die "failed to install amdgpu-dkms (release '${AMDGPU_DRIVER_RELEASE}') for \ +kernel ${KVER}. See make.log tail above. Common causes: (1) the AMD driver source \ +in this release does not support this kernel -- bump AMDGPU_DRIVER_RELEASE (see \ +docs/amd-gpu-airgapped.md); (2) linux-headers-${KVER} not installed / Module.symvers \ +empty; (3) DKMS module signing failed reaching /sys/firmware/efi/efivars -- normally \ +handled by the sign_tool='' drop-in above. Workaround: rerun with \ +AMDGPU_DRIVER_SOURCE=inbox to use the in-tree amdgpu." +fi + +# --------------------------------------------------------------------------- +# 6. Build the DKMS module against the IMAGE kernel (not the build host). +# apt-get's postinst may have already tried against $KVER; we re-run +# explicitly and let failures propagate (no `|| true`). +# --------------------------------------------------------------------------- +command -v dkms >/dev/null 2>&1 || die "dkms binary not found after installing amdgpu-dkms." + +log "Building amdgpu DKMS module for kernel ${KVER} ..." +# `dkms status` differs across versions: +# dkms 2.x: "amdgpu, 6.19.4, 6.14.0-36-generic, x86_64: installed" +# dkms 3.x: "amdgpu/6.19.4, 6.14.0-36-generic, x86_64: installed" +# We want the module-name and source-version (not the kernel). +dkms_line="$(dkms status 2>/dev/null | grep -iE '^amdgpu[/,]' | head -1 || true)" +[ -n "${dkms_line}" ] || die "dkms status does not know about the amdgpu module \ +after apt install -- driver package is broken or DKMS registration failed." + +mod="$(printf '%s\n' "${dkms_line}" | sed -E 's/[,/:].*//' | tr -d ' ')" +ver="$(printf '%s\n' "${dkms_line}" | sed -E 's/^[^/,]+[/,] *//' | sed -E 's/[,:].*//' | tr -d ' ')" +# Fallback: grep any version-looking token if the second column wasn't the version. +if ! printf '%s' "${ver}" | grep -qE '^[0-9]+\.[0-9]+'; then + ver="$(printf '%s\n' "${dkms_line}" | grep -oE '[0-9]+\.[0-9]+[0-9.]*' | head -1)" +fi +[ -n "${mod}" ] && [ -n "${ver}" ] || die "could not parse dkms status line: '${dkms_line}'" + +log " dkms build ${mod}/${ver} -k ${KVER}" +if ! dkms build -m "${mod}" -v "${ver}" -k "${KVER}"; then + log "DKMS build failed. Full make.log tail:" + tail -n 60 "/var/lib/dkms/${mod}/${ver}/build/make.log" 2>&1 | sed 's/^/ /' || true + die "DKMS build of ${mod}/${ver} against kernel ${KVER} failed. AMD driver \ +release '${AMDGPU_DRIVER_RELEASE}' likely does not support this kernel. Either \ +bump AMDGPU_DRIVER_RELEASE (see docs/amd-gpu-airgapped.md) or rerun with \ +AMDGPU_DRIVER_SOURCE=inbox." +fi + +log " dkms install ${mod}/${ver} -k ${KVER}" +dkms install -m "${mod}" -v "${ver}" -k "${KVER}" --force \ + || die "dkms install of ${mod}/${ver} against kernel ${KVER} failed." + +log "DKMS status:"; dkms status 2>&1 | sed 's/^/ /' || true + +# --------------------------------------------------------------------------- +# 7. Verify the DKMS-built module actually landed under updates/dkms and that +# dkms considers it installed for the target kernel. The in-tree amdgpu +# that Ubuntu ships under kernel/... does NOT count -- we're only satisfied +# if the OOT driver made it in. +# --------------------------------------------------------------------------- +MODDIR="/lib/modules/${KVER}" +dkms_mod_found="" +if find "${MODDIR}/updates" -name 'amdgpu.ko*' 2>/dev/null | grep -q .; then + dkms_mod_found="yes" +fi + +dkms_installed="" +if dkms status 2>/dev/null \ + | grep -iE "^amdgpu[/,][^,]*,[[:space:]]*${KVER}[,]" \ + | grep -q ': installed'; then + dkms_installed="yes" +fi + +if [ -z "${dkms_mod_found}" ] || [ -z "${dkms_installed}" ]; then + log "Verification failed:" + log " updates/dkms module present under ${MODDIR}/updates: ${dkms_mod_found:-no}" + log " dkms status shows 'installed' for kernel ${KVER}: ${dkms_installed:-no}" + find "${MODDIR}" -name 'amdgpu.ko*' 2>/dev/null | sed 's/^/ /' || true + die "amdgpu DKMS module was NOT built+installed for kernel ${KVER}. The \ +in-tree amdgpu (if any) is NOT sufficient in dkms mode -- rerun with \ +AMDGPU_DRIVER_SOURCE=inbox if that is what you want." +fi + +log "Verified: amdgpu DKMS module installed for ${KVER}." +find "${MODDIR}/updates" -name 'amdgpu.ko*' 2>/dev/null | sed 's/^/ /' + +# --------------------------------------------------------------------------- +# 8. Autoload amdgpu at boot (no blacklist needed -- dkms replaces the in-tree +# module of the same name via depmod's updates/ override). +# --------------------------------------------------------------------------- +log "Configuring amdgpu module autoload ..." +cat > /etc/modules-load.d/amdgpu.conf <<'EOF' +# Managed by CanvOS install-amdgpu-drivers.sh (dkms mode) +# Load the AMD GPU driver at boot so the AMD GPU Operator sees a ready driver. +amdgpu +EOF + +# --------------------------------------------------------------------------- +# 9. depmod for the target kernel so modprobe resolves amdgpu at boot +# --------------------------------------------------------------------------- +log "Running depmod -a ${KVER} ..." +depmod -a "${KVER}" || warn "depmod reported an error." + +# --------------------------------------------------------------------------- +# 10. Drop dracut config that OMITS amdgpu from any rebuilt initrd. See the +# equivalent block in the artifact / inbox branches for full rationale -- +# multi-GPU amdgpu init blows past initqueue's timeout when loaded early. +# --------------------------------------------------------------------------- +mkdir -p /etc/dracut.conf.d +cat > /etc/dracut.conf.d/98-canvos-amdgpu-omit.conf <<'EOF' +# Managed by CanvOS install-amdgpu-drivers.sh +# Keep amdgpu (and its DKMS helpers) OUT of the initrd. See install script. +omit_drivers+=" amdgpu amdttm amdkcl amd-sched amddrm_ttm_helper amddrm_buddy amddrm_exec amdxcp " +EOF + +# --------------------------------------------------------------------------- +# 11. Rebuild the initrd for the target kernel (amdgpu omitted per above). +# --------------------------------------------------------------------------- +if [ "${AMDGPU_REBUILD_INITRD}" = "true" ] && command -v dracut >/dev/null 2>&1; then + log "Rebuilding initrd for ${KVER} (dracut, amdgpu omitted) ..." + if dracut -f "/boot/initrd-${KVER}" "${KVER}"; then + ln -sf "initrd-${KVER}" /boot/initrd + else + warn "dracut initrd rebuild failed." + fi +elif [ "${AMDGPU_REBUILD_INITRD}" = "true" ] && command -v update-initramfs >/dev/null 2>&1; then + log "Rebuilding initramfs for ${KVER} (update-initramfs) ..." + update-initramfs -u -k "${KVER}" || warn "update-initramfs failed." +fi + +# --------------------------------------------------------------------------- +# 11b. Install the amd-smi host CLI (parity with nvidia-smi). The amdgpu-install +# deb above may already have configured the ROCm repo; if not, install_amd_smi +# adds repo.amd.com. Best-effort -- never fails the build. +# --------------------------------------------------------------------------- +install_amd_smi || true # best-effort; must never fail the build + +# --------------------------------------------------------------------------- +# 11c. Record what we did so ops can query it on-node. +# --------------------------------------------------------------------------- +printf 'AMDGPU_DRIVER_SOURCE=dkms\nAMDGPU_DRIVER_RELEASE=%s\nAMDGPU_DKMS_MODULE=%s/%s\nAMDGPU_SMI=%s\nKVER=%s\n' \ + "${AMDGPU_DRIVER_RELEASE}" "${mod}" "${ver}" "${AMDGPU_SMI_INSTALLED:-no}" "${KVER}" \ + > /etc/canvos/amdgpu-driver-source + +# --------------------------------------------------------------------------- +# 12. Cleanup apt caches to keep the image lean +# --------------------------------------------------------------------------- +apt-get clean +rm -rf /var/lib/apt/lists/* + +log "Done. AMD amdgpu driver (release ${AMDGPU_DRIVER_RELEASE}) baked in for kernel ${KVER}." +log "Reminder: install the AMD GPU Operator with 'driver.enable=false'." diff --git a/scripts/install-nvidia-drivers.sh b/scripts/install-nvidia-drivers.sh new file mode 100755 index 00000000..0acde144 --- /dev/null +++ b/scripts/install-nvidia-drivers.sh @@ -0,0 +1,446 @@ +#!/usr/bin/env bash +# +# install-nvidia-drivers.sh +# +# Pre-install the NVIDIA data-center GPU driver and build its kernel modules +# INTO a CanvOS / Kairos Ubuntu base image, so that a node booted from the +# image can run the NVIDIA GPU Operator in a fully air-gapped environment +# WITHOUT any host-side network access and WITHOUT the operator's driver +# container. +# +# WHAT THIS COVERS (OS side only) +# ------------------------------- +# * build toolchain (gcc, make, dkms, kmod, libc headers) +# * kernel headers that match the kernel shipped in the image +# (delegated to scripts/install-kernel-headers.sh) +# * the NVIDIA driver user-space + `nvidia-smi` (nvidia-utils-*-server) +# * the NVIDIA kernel modules (nvidia, nvidia-uvm, nvidia-modeset, +# nvidia-drm, nvidia-peermem) built with DKMS against the IMAGE kernel +# * nouveau blacklist + nvidia module autoload + nvidia-persistenced +# * nvidia-fabricmanager + libnvidia-nscq for HGX / NVSwitch systems +# (HGX H100/H200, HGX B200, DGX, GB200) -- required for multi-GPU NVLink +# * nvidia-imex for GB200 NVL72 multi-node NVLink Sharp (Blackwell) +# +# WHAT THIS DOES *NOT* COVER (ships as container images in your content bundle, +# deployed by the GPU Operator itself): +# * nvidia-container-toolkit / runtime class +# * k8s-device-plugin, gpu-feature-discovery, DCGM exporter, MIG manager, ... +# +# At Helm-install time you MUST tell the operator the driver is pre-installed: +# --set driver.enabled=false +# (and, if you also pre-install the toolkit below, --set toolkit.enabled=false) +# +# WHY THE DKMS DANCE +# ------------------ +# In an Earthly/Docker build `uname -r` is the BUILD HOST kernel, not the kernel +# baked into the image. If we let apt/DKMS build "for the running kernel" the +# modules would target the wrong ABI (or fail). We therefore derive the target +# kernel from /lib/modules (the kernel that will actually boot) and force every +# DKMS build + module install + depmod against THAT kernel. +# +# CONNECTIVITY +# ------------ +# This script runs at BUILD time, where the builder has internet. It bakes +# everything into the image. The resulting image needs no network at boot. +# +# TUNABLES (environment variables; all optional) +# NVIDIA_DRIVER_BRANCH Driver branch to install (e.g. 550, 570, 580). +# Default: 580 (a data-center production branch) +# NVIDIA_DRIVER_TYPE "proprietary" | "open" Default: open +# "open" is REQUIRED on Hopper (H100/H200) and +# Blackwell (RTX PRO 6000 Blackwell, B100, B200, +# GB200) — the closed modules fail with +# "RmInitAdapter (0x22:0x56:897)" on those GPUs +# and `nvidia-smi` reports "No devices were found". +# Also safe on Turing/Ampere/Ada. Override to +# "proprietary" only for pre-Turing hardware +# (Pascal/Volta). +# NVIDIA_USE_CUDA_REPO "true" to add developer.download.nvidia.com CUDA +# repo (recommended, has every -server branch). +# "false" to use only Ubuntu's own repos. +# Default: true +# NVIDIA_INSTALL_FABRICMANAGER "true" for NVSwitch/HGX boxes (H100/B200 HGX, +# DGX, GB200). Default: true. Harmless on non- +# NVSwitch hosts: the unit exits early and stays +# inactive; no restart loop, no kernel effect, +# ~60-90 MB image cost. Also installs the +# matching libnvidia-nscq- explicitly. +# See docs/nvidia-gpu-airgapped.md. +# NVIDIA_INSTALL_IMEX "true" to also install nvidia-imex-, +# the Internode Memory Exchange daemon required +# for GB200 NVL72 multi-node NVLink Sharp +# (Blackwell, driver 570+). Default: true. +# Harmless on non-NVL72 boxes: without +# /etc/nvidia-imex/nodes_config.cfg the daemon +# exits and the unit stays inactive. Best-effort +# -- skipped with a warning on branches that +# predate IMEX (pre-570). +# NVIDIA_INSTALL_CONTAINER_TOOLKIT "true" to ALSO pre-install +# nvidia-container-toolkit on the host (then set +# toolkit.enabled=false in the operator). +# Default: false (operator ships it) +# NVIDIA_REBUILD_INITRD "true" to rebuild the initrd so the nouveau +# blacklist takes effect in early boot. +# Default: true +# +set -u + +log() { echo "[install-nvidia-drivers] $*"; } +warn() { echo "[install-nvidia-drivers] WARNING: $*" >&2; } +die() { echo "[install-nvidia-drivers] ERROR: $*" >&2; exit 1; } + +# --------------------------------------------------------------------------- +# Config +# --------------------------------------------------------------------------- +NVIDIA_DRIVER_BRANCH="${NVIDIA_DRIVER_BRANCH:-580}" +NVIDIA_DRIVER_TYPE="${NVIDIA_DRIVER_TYPE:-open}" +NVIDIA_USE_CUDA_REPO="${NVIDIA_USE_CUDA_REPO:-true}" +NVIDIA_INSTALL_FABRICMANAGER="${NVIDIA_INSTALL_FABRICMANAGER:-true}" +NVIDIA_INSTALL_IMEX="${NVIDIA_INSTALL_IMEX:-true}" +NVIDIA_INSTALL_CONTAINER_TOOLKIT="${NVIDIA_INSTALL_CONTAINER_TOOLKIT:-false}" +NVIDIA_REBUILD_INITRD="${NVIDIA_REBUILD_INITRD:-true}" + +export DEBIAN_FRONTEND=noninteractive + +command -v apt-get >/dev/null 2>&1 || die "this script only supports apt-based (Ubuntu/Debian) images." + +# --------------------------------------------------------------------------- +# 1. Identify the kernel shipped in the image (NOT the build host kernel) +# --------------------------------------------------------------------------- +KVER="$(printf '%s\n' /lib/modules/* 2>/dev/null | xargs -n1 basename 2>/dev/null | sort -V | tail -1)" +[ -n "${KVER}" ] || die "could not determine target kernel from /lib/modules." +log "Target (image) kernel: ${KVER}" +log "Driver branch: ${NVIDIA_DRIVER_BRANCH} (${NVIDIA_DRIVER_TYPE})" + +# --------------------------------------------------------------------------- +# 2. Build toolchain +# --------------------------------------------------------------------------- +log "Installing build toolchain ..." +apt-get update || true +apt-get install -y --no-install-recommends \ + ca-certificates curl wget gnupg \ + build-essential gcc make \ + dkms kmod libc6-dev pkg-config \ + || die "failed to install build toolchain." + +# --------------------------------------------------------------------------- +# 3. Kernel headers matching the image kernel +# Reuse the repo's ABI-exact / snapshot-aware header installer if present. +# --------------------------------------------------------------------------- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +HEADERS_HELPER="" +for cand in "${SCRIPT_DIR}/install-kernel-headers.sh" /tmp/install-kernel-headers.sh; do + [ -r "${cand}" ] && { HEADERS_HELPER="${cand}"; break; } +done + +if [ -n "${HEADERS_HELPER}" ]; then + log "Installing kernel headers via ${HEADERS_HELPER} ..." + bash "${HEADERS_HELPER}" || warn "kernel-headers helper returned non-zero; continuing." +else + log "Header helper not found; attempting a direct header install ..." + apt-get install -y "linux-headers-${KVER}" || \ + apt-get install -y linux-headers-generic || \ + warn "could not install linux-headers-${KVER}." +fi + +# DKMS needs /lib/modules//build to point at the headers source tree. +if [ ! -e "/lib/modules/${KVER}/build" ]; then + # Find the header tree that matches our kernel and symlink it. + src="$(ls -d /usr/src/linux-headers-${KVER} 2>/dev/null | head -1)" + if [ -n "${src}" ]; then + ln -sfn "${src}" "/lib/modules/${KVER}/build" + log "Linked /lib/modules/${KVER}/build -> ${src}" + else + warn "no /usr/src/linux-headers-${KVER}; DKMS build will likely fail." + fi +fi + +# --------------------------------------------------------------------------- +# 4. NVIDIA package repo (CUDA network repo — has every *-server branch) +# --------------------------------------------------------------------------- +if [ "${NVIDIA_USE_CUDA_REPO}" = "true" ]; then + # Derive the CUDA repo "distro" tag from the image (e.g. 22.04 -> ubuntu2204) + osid=""; osver="" + if [ -r /etc/os-release ]; then + # shellcheck disable=SC1091 + . /etc/os-release + osid="${ID:-ubuntu}" + osver="${VERSION_ID:-22.04}" + fi + distro="${osid}$(echo "${osver}" | tr -d '.')" # ubuntu2204, ubuntu2004, ... + case "$(uname -m)" in + x86_64) cudaarch="x86_64" ;; + aarch64) cudaarch="sbsa" ;; + *) cudaarch="x86_64" ;; + esac + repo_base="https://developer.download.nvidia.com/compute/cuda/repos/${distro}/${cudaarch}" + log "Adding NVIDIA CUDA repo: ${repo_base}" + if wget -qO /tmp/cuda-keyring.deb "${repo_base}/cuda-keyring_1.1-1_all.deb"; then + dpkg -i /tmp/cuda-keyring.deb || warn "cuda-keyring install failed." + rm -f /tmp/cuda-keyring.deb + apt-get update || warn "apt-get update after adding CUDA repo failed." + else + warn "could not download cuda-keyring; falling back to Ubuntu repos." + fi +fi + +# --------------------------------------------------------------------------- +# 5. Choose driver packages +# Headless server packages (no Xorg / GUI). nvidia-utils gives nvidia-smi. +# --------------------------------------------------------------------------- +if [ "${NVIDIA_DRIVER_TYPE}" = "open" ]; then + HEADLESS_PKG="nvidia-headless-${NVIDIA_DRIVER_BRANCH}-server-open" +else + HEADLESS_PKG="nvidia-headless-${NVIDIA_DRIVER_BRANCH}-server" +fi +UTILS_PKG="nvidia-utils-${NVIDIA_DRIVER_BRANCH}-server" + +log "Installing NVIDIA driver packages: ${HEADLESS_PKG} ${UTILS_PKG}" +if ! apt-get install -y --no-install-recommends "${HEADLESS_PKG}" "${UTILS_PKG}"; then + warn "'${HEADLESS_PKG}' not available; retrying with generic (non-server) branch." + if [ "${NVIDIA_DRIVER_TYPE}" = "open" ]; then + HEADLESS_PKG="nvidia-headless-${NVIDIA_DRIVER_BRANCH}-open" + else + HEADLESS_PKG="nvidia-headless-${NVIDIA_DRIVER_BRANCH}" + fi + UTILS_PKG="nvidia-utils-${NVIDIA_DRIVER_BRANCH}" + apt-get install -y --no-install-recommends "${HEADLESS_PKG}" "${UTILS_PKG}" \ + || die "failed to install NVIDIA driver packages for branch ${NVIDIA_DRIVER_BRANCH}. \ +Check available branches with: apt-cache search 'nvidia-headless-.*-server'" +fi + +# --------------------------------------------------------------------------- +# 6. Build the DKMS modules against the IMAGE kernel (not the build host) +# --------------------------------------------------------------------------- +# The apt postinst runs `dkms autoinstall`, which only builds for kernels that +# have headers present -- i.e. our target kernel, since the build host kernel's +# headers are absent in the image. We still force it explicitly to be safe. +if command -v dkms >/dev/null 2>&1; then + log "Building NVIDIA DKMS modules for kernel ${KVER} ..." + # Explicitly (re)build every registered nvidia dkms module for the target. + # `dkms status` output differs across versions: + # dkms 2.x: "nvidia, 580.159.03, 6.14.0-36-generic, x86_64: installed" + # dkms 3.x: "nvidia/580.159.03, 6.14.0-36-generic, x86_64: installed" + # Extract the module name (up to the first , / or :) and the first + # version-looking token, which works for both formats. + dkms status 2>/dev/null | grep -i nvidia | while read -r line; do + mod="$(printf '%s\n' "${line}" | sed -E 's/[,/:].*//' | tr -d ' ')" + ver="$(printf '%s\n' "${line}" | grep -oE '[0-9]+\.[0-9]+[0-9.]*' | head -1)" + case "${mod}" in nvidia*) ;; *) continue ;; esac + [ -n "${mod}" ] && [ -n "${ver}" ] || continue + log " dkms install ${mod}/${ver} -k ${KVER}" + dkms build -m "${mod}" -v "${ver}" -k "${KVER}" 2>/dev/null || true + dkms install -m "${mod}" -v "${ver}" -k "${KVER}" --force 2>/dev/null || true + done + # Belt-and-suspenders: try the autoinstaller pinned to the target kernel + # (ignored gracefully by older dkms that lack the -k flag). + dkms autoinstall -k "${KVER}" 2>/dev/null || true + log "DKMS status:"; dkms status 2>/dev/null || true +else + warn "dkms not found; relying on apt postinst build." +fi + +# --------------------------------------------------------------------------- +# 7. Verify the modules actually landed in the image kernel's module tree +# --------------------------------------------------------------------------- +MODDIR="/lib/modules/${KVER}" +if ls "${MODDIR}"/updates/dkms/nvidia*.ko* >/dev/null 2>&1 || \ + ls "${MODDIR}"/kernel/drivers/video/nvidia*.ko* >/dev/null 2>&1 || \ + find "${MODDIR}" -name 'nvidia*.ko*' 2>/dev/null | grep -q .; then + log "Verified: nvidia kernel modules present under ${MODDIR}." + find "${MODDIR}" -name 'nvidia*.ko*' 2>/dev/null | sed 's/^/ /' +else + die "no nvidia*.ko modules found under ${MODDIR} -- DKMS build did not \ +produce modules for the image kernel. Check that linux-headers-${KVER} and a \ +matching gcc are installed." +fi + +# --------------------------------------------------------------------------- +# 8. NVIDIA Fabric Manager + libnvidia-nscq + nvlsm (HGX / NVSwitch systems) +# Required on HGX H100/H200, HGX B200, DGX, GB200 for multi-GPU NVLink. +# libnvidia-nscq- is a fabricmanager dep and gets pulled in +# transitively -- listed explicitly so the install fails loudly if the +# CUDA repo ever drops the auto-dep. +# +# NVIDIA 570+ ships the FM unit with ExecStart wrapped in +# /usr/share/nvidia/fabricmanager/nvidia-fabricmanager-start.sh, which +# probes `ibstat` (infiniband-diags) and `nvlsm` (NVIDIA Subnet Manager) +# before invoking nv-fabricmanager -- needed so the NVLink subnet is up +# on GB200 NVL72. Both binaries must be present or the unit dies: +# "ibstat command not found! Please install ibstat." +# "nvlsm command not found! Please install nvlsm." +# Empirically the wrapper succeeds on standalone HGX topologies once +# both are installed (verified on HGX with nvlsm 2025.10.14-1 from the +# NVIDIA CUDA repo we already added in step 4), so we install them +# alongside FM and let the vendor wrapper stay in charge. +# +# Package sources: nvlsm ships in the CUDA repo under an unversioned +# package name (not nvlsm-). infiniband-diags is Ubuntu-native. +# On non-NVSwitch hosts nv-fabricmanager still exits "No NvSwitch found" +# and the unit stays inactive; no kernel side effect, no restart loop. +# --------------------------------------------------------------------------- +if [ "${NVIDIA_INSTALL_FABRICMANAGER}" = "true" ]; then + FM_PKG="nvidia-fabricmanager-${NVIDIA_DRIVER_BRANCH}" + NSCQ_PKG="libnvidia-nscq-${NVIDIA_DRIVER_BRANCH}" + log "Installing ${FM_PKG} + ${NSCQ_PKG} + nvlsm + infiniband-diags ..." + if apt-get install -y --no-install-recommends \ + "${FM_PKG}" "${NSCQ_PKG}" nvlsm infiniband-diags; then + systemctl enable nvidia-fabricmanager.service 2>/dev/null || true + + # --- ib_umad autoload for fabricmanager precheck -------------------- + # NVIDIA 570+ nvidia-fabricmanager-start.sh --mode precheck (invoked + # by the systemd unit's ExecStartPre) takes the "Detected NVL5+ + # system" branch on HGX B200 / GB200 hardware and requires the + # ib_umad kernel module to be loaded before nv-fabricmanager can + # start -- the wrapper opens /dev/infiniband/umad* to send MADs to + # the NVSwitch fabric (NVSwitch reuses the InfiniBand management- + # datagram shape). Kairos edge images ship the module (it's in the + # kernel-modules-extra set) but don't auto-load it, so precheck + # dies with: + # Detected NVL5+ system + # Kernel module "ib_umad" has not been loaded, + # fabric manager cannot be started + # Please run "modprobe ib_umad" before starting fabric manager + # Load it at boot so the service comes up on NVSwitch hardware + # without any operator intervention. Verified live on 8x HGX B200 + # (driver 580.159.03): with this in place nvidia-fabricmanager.service + # goes active in ~3s at boot and downstream nvidia-cuda-validator + # exits 0 on the next GPU-operator reconcile. + # + # On non-NVSwitch hosts the module load costs ~10 KiB of RSS and has + # no other side effect; fabricmanager still exits "No NvSwitch found" + # and the unit stays inactive as before. + cat > /etc/modules-load.d/nvidia-fabricmanager.conf <<'EOF' +# Managed by CanvOS install-nvidia-drivers.sh +# Required by nvidia-fabricmanager-start.sh --mode precheck on NVL5+ +# systems (HGX B200, GB200) -- see /usr/bin/nvidia-fabricmanager-start.sh. +ib_umad +EOF + else + warn "could not install ${FM_PKG} / ${NSCQ_PKG} / nvlsm / infiniband-diags; skipping fabric manager." + fi +fi + +# --------------------------------------------------------------------------- +# 8b. NVIDIA IMEX -- Internode Memory Exchange daemon +# Required for GB200 NVL72 multi-node NVLink Sharp (Blackwell, driver +# 570+). Not needed for single-node HGX B200 or HGX H100. Package is +# part of the CUDA repo; older driver branches (pre-570) do not publish +# it, so this is best-effort. On single-node boxes the daemon has no +# /etc/nvidia-imex/nodes_config.cfg and exits cleanly -- unit stays +# inactive, ~10-20 MB image cost. +# --------------------------------------------------------------------------- +if [ "${NVIDIA_INSTALL_IMEX}" = "true" ]; then + IMEX_PKG="nvidia-imex-${NVIDIA_DRIVER_BRANCH}" + log "Installing ${IMEX_PKG} (GB200 NVL72 multi-node NVLink Sharp) ..." + if apt-get install -y --no-install-recommends "${IMEX_PKG}"; then + systemctl enable nvidia-imex.service 2>/dev/null || true + else + warn "${IMEX_PKG} not available (branch ${NVIDIA_DRIVER_BRANCH} may predate IMEX -- 570+ only). Skipping." + fi +fi + +# --------------------------------------------------------------------------- +# 9. Optional: nvidia-container-toolkit on the host +# (default OFF -- the GPU Operator ships and configures the toolkit) +# --------------------------------------------------------------------------- +if [ "${NVIDIA_INSTALL_CONTAINER_TOOLKIT}" = "true" ]; then + log "Installing nvidia-container-toolkit on host ..." + install -d -m 0755 /usr/share/keyrings + if curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey \ + | gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg; then + curl -fsSL https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list \ + | sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' \ + > /etc/apt/sources.list.d/nvidia-container-toolkit.list + apt-get update && apt-get install -y --no-install-recommends nvidia-container-toolkit \ + || warn "nvidia-container-toolkit install failed." + else + warn "could not fetch nvidia-container-toolkit gpg key; skipping." + fi +fi + +# --------------------------------------------------------------------------- +# 10. Host module configuration: blacklist nouveau + autoload nvidia +# --------------------------------------------------------------------------- +log "Configuring nouveau blacklist and nvidia module autoload ..." +cat > /etc/modprobe.d/blacklist-nouveau.conf <<'EOF' +# Managed by CanvOS install-nvidia-drivers.sh +blacklist nouveau +blacklist lbm-nouveau +options nouveau modeset=0 +alias nouveau off +alias lbm-nouveau off +EOF + +cat > /etc/modules-load.d/nvidia.conf <<'EOF' +# Managed by CanvOS install-nvidia-drivers.sh +# Load the NVIDIA stack at boot so the GPU Operator sees a ready driver. +# nvidia_drm is intentionally omitted: it grabs KMS/DRM and on headless GPU +# nodes can wedge early boot. It loads on demand if anything wants it. +nvidia +nvidia_uvm +nvidia_modeset +EOF + +# NVIDIA driver run-time module options recommended for datacenter use: +# NVreg_OpenRmEnableUnsupportedGpus is only relevant for the open modules. +cat > /etc/modprobe.d/nvidia.conf <<'EOF' +# Managed by CanvOS install-nvidia-drivers.sh +options nvidia NVreg_PreserveVideoMemoryAllocations=1 +EOF + +# Enable the persistence daemon (recommended for datacenter GPUs). +systemctl enable nvidia-persistenced.service 2>/dev/null || true + +# --------------------------------------------------------------------------- +# 11. depmod for the target kernel so modprobe can resolve nvidia at boot +# --------------------------------------------------------------------------- +log "Running depmod -a ${KVER} ..." +depmod -a "${KVER}" || warn "depmod reported an error." + +# --------------------------------------------------------------------------- +# 12. Make sure the initrd honors the nouveau blacklist and does NOT ship +# nouveau.ko. The /etc/modprobe.d/blacklist-nouveau.conf we wrote above +# lives on the rootfs and is only consulted AFTER switchroot; by then +# nouveau has already been auto-loaded by initramfs udev on modern +# NVIDIA data-center GPUs (Ada/Hopper/Blackwell), where nouveau's GSP-RM +# support hangs on device init and stalls udev-settle forever. +# +# Fix: write a dracut.conf.d snippet so BOTH this script's dracut +# rebuild AND the later Earthfile-driven dracut rebuild produce an +# initrd that (a) omits nouveau entirely and (b) carries the modprobe +# blacklist file, so initramfs modprobe honors it too. +# --------------------------------------------------------------------------- +mkdir -p /etc/dracut.conf.d +cat > /etc/dracut.conf.d/95-blacklist-nouveau.conf <<'EOF' +# Managed by CanvOS install-nvidia-drivers.sh +omit_drivers+=" nouveau lbm-nouveau " +install_items+=" /etc/modprobe.d/blacklist-nouveau.conf " +EOF + +# --------------------------------------------------------------------------- +# 12b. Rebuild the initrd so the nouveau blacklist applies in early boot +# --------------------------------------------------------------------------- +if [ "${NVIDIA_REBUILD_INITRD}" = "true" ] && command -v dracut >/dev/null 2>&1; then + log "Rebuilding initrd for ${KVER} (dracut) ..." + if dracut -f "/boot/initrd-${KVER}" "${KVER}"; then + ln -sf "initrd-${KVER}" /boot/initrd + else + warn "dracut initrd rebuild failed; nouveau blacklist still applies post-switchroot." + fi +elif [ "${NVIDIA_REBUILD_INITRD}" = "true" ] && command -v update-initramfs >/dev/null 2>&1; then + log "Rebuilding initramfs for ${KVER} (update-initramfs) ..." + update-initramfs -u -k "${KVER}" || warn "update-initramfs failed." +fi + +# --------------------------------------------------------------------------- +# 13. Cleanup apt caches to keep the image lean +# --------------------------------------------------------------------------- +apt-get clean +rm -rf /var/lib/apt/lists/* + +log "Done. NVIDIA driver ${NVIDIA_DRIVER_BRANCH} (${NVIDIA_DRIVER_TYPE}) baked in for kernel ${KVER}." +log "Reminder: install the GPU Operator with 'driver.enabled=false'." +if [ "${NVIDIA_INSTALL_CONTAINER_TOOLKIT}" = "true" ]; then + log "Reminder: you pre-installed the container toolkit -> also set 'toolkit.enabled=false'." +fi diff --git a/scripts/prebuild-amdgpu-artifact.sh b/scripts/prebuild-amdgpu-artifact.sh new file mode 100755 index 00000000..3c74deb6 --- /dev/null +++ b/scripts/prebuild-amdgpu-artifact.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash +# +# prebuild-amdgpu-artifact.sh +# +# Compile AMD's amdgpu-dkms module against the kairos base image's kernel in a +# host-side `docker run --privileged` container, then tar the resulting kernel +# module + firmware + support files into an artifact the main Earthly build +# COPYs in. +# +# WHY THIS EXISTS +# --------------- +# Earthly's buildkit RUN sandbox breaks AMD's amdgpu-dkms ./configure heredoc +# probe (fails at "cannot detect CFLAGS..."), despite the same script + same +# base image + same host succeeding under plain `docker run --privileged`. +# The specific buildkit-vs-docker sandbox difference is not something we +# control from the Earthfile. Instead of fighting it, we run the DKMS build +# outside Earthly, in the environment we know works, and let Earthly consume +# the produced artifact via COPY. +# +# WHAT ENDS UP IN THE ARTIFACT +# /lib/modules//updates/dkms/.ko* +# /lib/firmware/amdgpu/* (firmware blobs) +# /etc/dkms/framework.conf.d/canvos-no-mok-signing.conf (defensive) +# /etc/modules-load.d/amdgpu.conf (autoload) +# /etc/canvos/amdgpu-driver-source (on-node marker) +# +# Extracting the tar into the image (via the Earthfile) + running depmod on +# the target kernel is functionally equivalent to running install-amdgpu- +# drivers.sh directly in the image. +# +# CACHING +# Artifacts are stored at build/amdgpu-artifact---.tar.gz +# Cache hits when release + base-image digest + kver match. Docker image +# pulls are hit via the local docker daemon's own cache. +# +# INPUTS (env vars; defaults mirror the Earthfile / .arg.template) +# BASE_IMAGE kairos base image ref (REQUIRED) +# AMDGPU_DRIVER_RELEASE default: 31.40 (ROCm 7.14 GA; pairs with GPU Operator v1.5.1) +# AMDGPU_ARTIFACT_DIR default: ./build +# AMDGPU_FORCE_REBUILD set to 1 to bypass cache +# +# OUTPUT (stdout) +# Absolute path to the produced .tar.gz on the last line, prefixed by +# "AMDGPU_ARTIFACT_PATH=" so callers can `eval "$(prebuild-amdgpu-artifact.sh)"` +# or just take the last line. +# +set -euo pipefail + +log() { echo "[prebuild-amdgpu] $*" >&2; } +die() { echo "[prebuild-amdgpu] ERROR: $*" >&2; exit 1; } + +# --- inputs --------------------------------------------------------------- +: "${BASE_IMAGE:?BASE_IMAGE must be set (kairos base image ref, e.g. us-docker.pkg.dev/palette-images/edge/kairos-ubuntu:24.04-core-amd64-generic-v4.0.4)}" +AMDGPU_DRIVER_RELEASE="${AMDGPU_DRIVER_RELEASE:-31.40}" +AMDGPU_ARTIFACT_DIR="${AMDGPU_ARTIFACT_DIR:-./build}" +AMDGPU_FORCE_REBUILD="${AMDGPU_FORCE_REBUILD:-0}" + +command -v docker >/dev/null 2>&1 || die "docker must be available on the build host." + +# The install script we'll run inside the container. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +INSTALL_SCRIPT="${SCRIPT_DIR}/install-amdgpu-drivers.sh" +HEADERS_SCRIPT="${SCRIPT_DIR}/install-kernel-headers.sh" +[ -r "${INSTALL_SCRIPT}" ] || die "cannot find install-amdgpu-drivers.sh at ${INSTALL_SCRIPT}" +[ -r "${HEADERS_SCRIPT}" ] || die "cannot find install-kernel-headers.sh at ${HEADERS_SCRIPT}" + +mkdir -p "${AMDGPU_ARTIFACT_DIR}" + +# --- discover target kernel + base image digest --------------------------- +log "Pulling base image (may be cached): ${BASE_IMAGE}" +docker pull "${BASE_IMAGE}" >/dev/null || die "failed to pull ${BASE_IMAGE}" + +BASE_DIGEST="$(docker image inspect -f '{{.Id}}' "${BASE_IMAGE}" | sed 's/^sha256://' | cut -c1-12)" +[ -n "${BASE_DIGEST}" ] || die "could not read digest of ${BASE_IMAGE}" + +KVER="$(docker run --rm --entrypoint /bin/sh "${BASE_IMAGE}" -c 'ls /lib/modules | sort -V | tail -1' 2>/dev/null)" +[ -n "${KVER}" ] || die "could not discover kernel from /lib/modules inside ${BASE_IMAGE}" + +ARTIFACT_NAME="amdgpu-artifact-${AMDGPU_DRIVER_RELEASE}-${KVER}-${BASE_DIGEST}.tar.gz" +ARTIFACT_PATH="$(cd "${AMDGPU_ARTIFACT_DIR}" && pwd)/${ARTIFACT_NAME}" + +log "Base image digest : ${BASE_DIGEST}" +log "Target kernel : ${KVER}" +log "Driver release : ${AMDGPU_DRIVER_RELEASE}" +log "Artifact path : ${ARTIFACT_PATH}" + +# --- cache check ---------------------------------------------------------- +if [ "${AMDGPU_FORCE_REBUILD}" != "1" ] && [ -s "${ARTIFACT_PATH}" ]; then + log "Cache hit -- reusing existing artifact. Set AMDGPU_FORCE_REBUILD=1 to override." + echo "AMDGPU_ARTIFACT_PATH=${ARTIFACT_PATH}" + exit 0 +fi + +# --- build ---------------------------------------------------------------- +# Run the install script inside a privileged container against the same base +# image the Earthfile will use, then tar out the produced files. We stream +# the tar over stdout to avoid needing an intermediate volume mount that some +# rootless docker setups can't do cleanly. +STAGE_DIR="$(mktemp -d "${AMDGPU_ARTIFACT_DIR}/.amdgpu-build.XXXXXX")" +trap 'rm -rf "${STAGE_DIR}"' EXIT + +log "Compiling amdgpu-dkms in container. This takes ~8-10 min the first time." +docker run --rm --privileged \ + -e AMDGPU_DRIVER_SOURCE=dkms \ + -e AMDGPU_DRIVER_RELEASE="${AMDGPU_DRIVER_RELEASE}" \ + -e AMDGPU_REBUILD_INITRD=false \ + -e AMDGPU_INSTALL_SMI=false \ + -e KVER_EXPECTED="${KVER}" \ + -v "${INSTALL_SCRIPT}:/tmp/install-amdgpu-drivers.sh:ro" \ + -v "${HEADERS_SCRIPT}:/tmp/install-kernel-headers.sh:ro" \ + --entrypoint /bin/bash \ + "${BASE_IMAGE}" \ + -c ' + set -eo pipefail + # Scripts are bind-mounted read-only from the host; invoke via `bash` + # so we don t need chmod +x (which would fail on the RO mount). + bash /tmp/install-amdgpu-drivers.sh 1>&2 + + # Verify the module actually landed. + MODDIR="/lib/modules/${KVER_EXPECTED}/updates/dkms" + if ! find "${MODDIR}" -name "amdgpu.ko*" 2>/dev/null | grep -q .; then + echo "prebuild: no amdgpu module under ${MODDIR}" >&2 + exit 1 + fi + + # Build the tar to stdout. Paths must exist to be included; the tar + # is anchored at / so extraction inside the image lands under the + # same absolute paths. + TAR_INPUTS=( + "/lib/modules/${KVER_EXPECTED}/updates/dkms" + "/etc/modules-load.d/amdgpu.conf" + "/etc/canvos/amdgpu-driver-source" + ) + # Firmware + framework drop-in are optional but helpful; skip silently if absent. + [ -d /lib/firmware/amdgpu ] && TAR_INPUTS+=("/lib/firmware/amdgpu") + [ -r /etc/dkms/framework.conf.d/canvos-no-mok-signing.conf ] && \ + TAR_INPUTS+=("/etc/dkms/framework.conf.d/canvos-no-mok-signing.conf") + + tar -czf - "${TAR_INPUTS[@]}" + ' > "${STAGE_DIR}/artifact.tar.gz" + +# Move the artifact to its final path IMMEDIATELY so a subsequent verify +# failure leaves a diagnosable file behind (STAGE_DIR is wiped by the trap). +[ -s "${STAGE_DIR}/artifact.tar.gz" ] || die "prebuild produced an empty artifact." +mv "${STAGE_DIR}/artifact.tar.gz" "${ARTIFACT_PATH}" + +# Sanity: tar file must be a valid gzip and contain the amdgpu module. +tar_listing="$(tar -tzf "${ARTIFACT_PATH}" 2>&1)" \ + || die "artifact ${ARTIFACT_PATH} is not a valid gzipped tar. Head of output: $(printf '%s\n' "${tar_listing}" | head -3)" +if ! printf '%s\n' "${tar_listing}" | grep -qE "updates/dkms/.*amdgpu\.ko"; then + log "Artifact contents (first 40 entries):" + printf '%s\n' "${tar_listing}" | head -40 | sed 's/^/ /' >&2 + die "artifact ${ARTIFACT_PATH} does not contain an amdgpu module under updates/dkms/. \ +The tarball is preserved for inspection. Delete it and rerun with \ +AMDGPU_FORCE_REBUILD=1 to try again." +fi + +log "Artifact produced ($(du -h "${ARTIFACT_PATH}" | awk '{print $1}'))." + +# --- output --------------------------------------------------------------- +echo "AMDGPU_ARTIFACT_PATH=${ARTIFACT_PATH}" diff --git a/ubuntu-fips/20.04/99-usb-media.conf b/ubuntu-fips/20.04/99-usb-media.conf new file mode 100644 index 00000000..af815ee8 --- /dev/null +++ b/ubuntu-fips/20.04/99-usb-media.conf @@ -0,0 +1,3 @@ +hostonly="no" +add_drivers+=" xhci_pci_renesas " +force_drivers+=" xhci_pci_renesas " diff --git a/ubuntu-fips/20.04/Dockerfile b/ubuntu-fips/20.04/Dockerfile index 2cf14ee2..1c8365ab 100644 --- a/ubuntu-fips/20.04/Dockerfile +++ b/ubuntu-fips/20.04/Dockerfile @@ -112,6 +112,11 @@ RUN cd /usr/lib/dracut/modules.d/95iscsi && patch < /dracut-broken-iscsi-ubuntu- COPY dracut.conf /etc/dracut.conf.d/kairos-fips.conf +# Bundle the Renesas xHCI (USB 3.0) host controller driver into the initramfs so +# installation from USB media works on hardware using that chipset. Consumed by the +# `kairos-init -s init` dracut run below (the Earthfile skips dracut for FIPS builds). +COPY 99-usb-media.conf /etc/dracut.conf.d/99-usb-media.conf + # Copy the custom dracut modules.fips that includes 2 missing modules COPY modules.fips /tmp/modules.fips RUN kernel=$(ls /lib/modules | grep fips | head -n1) && mv /tmp/modules.fips /lib/modules/${kernel}/modules.fips diff --git a/ubuntu-fips/22.04/99-usb-media.conf b/ubuntu-fips/22.04/99-usb-media.conf new file mode 100644 index 00000000..af815ee8 --- /dev/null +++ b/ubuntu-fips/22.04/99-usb-media.conf @@ -0,0 +1,3 @@ +hostonly="no" +add_drivers+=" xhci_pci_renesas " +force_drivers+=" xhci_pci_renesas " diff --git a/ubuntu-fips/22.04/Dockerfile.ubuntu22.04-fips b/ubuntu-fips/22.04/Dockerfile.ubuntu22.04-fips index 5d4c06be..282bc8a3 100644 --- a/ubuntu-fips/22.04/Dockerfile.ubuntu22.04-fips +++ b/ubuntu-fips/22.04/Dockerfile.ubuntu22.04-fips @@ -32,6 +32,11 @@ RUN --mount=type=secret,id=pro-attach-config \ COPY 22.04/dracut.conf /etc/dracut.conf.d/kairos-fips.conf +# Bundle the Renesas xHCI (USB 3.0) host controller driver into the initramfs so +# installation from USB media works on hardware using that chipset. Consumed by the +# `kairos-init -s init` dracut run below (the Earthfile skips dracut for FIPS builds). +COPY 22.04/99-usb-media.conf /etc/dracut.conf.d/99-usb-media.conf + # Copy the custom dracut modules.fips that includes 2 missing modules COPY 22.04/modules.fips /tmp/modules.fips RUN kernel=$(ls /lib/modules | head -n1) && mv /tmp/modules.fips /lib/modules/${kernel}/modules.fips diff --git a/ubuntu-fips/24.04/99-usb-media.conf b/ubuntu-fips/24.04/99-usb-media.conf new file mode 100644 index 00000000..af815ee8 --- /dev/null +++ b/ubuntu-fips/24.04/99-usb-media.conf @@ -0,0 +1,3 @@ +hostonly="no" +add_drivers+=" xhci_pci_renesas " +force_drivers+=" xhci_pci_renesas " diff --git a/ubuntu-fips/24.04/Dockerfile.ubuntu24.04-fips b/ubuntu-fips/24.04/Dockerfile.ubuntu24.04-fips index a9d1d6de..f8e63381 100644 --- a/ubuntu-fips/24.04/Dockerfile.ubuntu24.04-fips +++ b/ubuntu-fips/24.04/Dockerfile.ubuntu24.04-fips @@ -41,6 +41,11 @@ RUN --mount=type=secret,id=pro-attach-config \ COPY 24.04/modules.fips /tmp/modules.fips RUN kernel=$(ls /lib/modules | head -n1) && mv /tmp/modules.fips /lib/modules/${kernel}/modules.fips +# Bundle the Renesas xHCI (USB 3.0) host controller driver into the initramfs so +# installation from USB media works on hardware using that chipset. Consumed by the +# `kairos-init -s init` dracut run below (the Earthfile skips dracut for FIPS builds). +COPY 24.04/99-usb-media.conf /etc/dracut.conf.d/99-usb-media.conf + COPY 24.04/fix.sh /tmp/fix.sh COPY stig-remediate.sh /tmp/stig-remediate.sh COPY restore-ubuntu-default-banners.sh /tmp/restore-ubuntu-default-banners.sh