diff --git a/README.md b/README.md index efcc2ae7..566443c1 100644 --- a/README.md +++ b/README.md @@ -88,14 +88,15 @@ A successful install looks like this: ```text This operation needs root privileges. Requesting sudo password... - ✓ [1/8] Detecting GPU (0.2s) - ✓ [2/8] Generating values overlay (initial) (0.0s) - ✓ [3/8] Installing helm + k9s (0.0s) - ✓ [4/8] Installing K3s (single-node) (3.8s) - ✓ [5/8] Pulling custom + external images (25.0s) - ✓ [6/8] Deploying ROCm GPU device plugin + node labeller (0.2s) - ✓ [7/8] Refreshing values overlay from node labels (0.2s) - ✓ [8/8] Deploying JupyterHub runtime (helm install + wait) (9.2s) + ✓ [1/9] Detecting GPU (0.2s) + ✓ [2/9] Provisioning GPU device access (0.1s) + ✓ [3/9] Generating values overlay (initial) (0.0s) + ✓ [4/9] Installing helm + k9s (0.0s) + ✓ [5/9] Installing K3s (single-node) (3.8s) + ✓ [6/9] Pulling custom + external images (25.0s) + ✓ [7/9] Deploying ROCm GPU device plugin + node labeller (0.2s) + ✓ [8/9] Refreshing values overlay from node labels (0.2s) + ✓ [9/9] Deploying JupyterHub runtime (helm install + wait) (9.2s) _ _ _ ____ _ _ ____ _ _ / \ | | | | _ \ | | ___ __ _ _ __ _ __ (_)_ __ __ _ / ___| | ___ _ _ __| | @@ -111,6 +112,13 @@ This operation needs root privileges. Requesting sudo password... kubectl is configured at $HOME/.kube/config; try `kubectl get nodes` ``` +The GPU access stage installs AMD's `amdgpu-insecure-instinct-udev-rules` +package, pinned to `30.30.4.0-2341068.24.04`. It sets mode `0666` only on +`/dev/kfd` and DRM `renderD*` nodes; `card*` keeps the normal system policy. The +device plugin remains a separate allocation layer, and the tested ROCm compute +path needs no supplemental GPU group. The offline `pack` bundle carries the +pinned deb for installation without network access. + See the full guide at [Quick Start](https://amdresearch.github.io/aup-learning-cloud/installation/quick-start.html) and [Single-Node Deployment](https://amdresearch.github.io/aup-learning-cloud/installation/single-node.html). ### Uninstall diff --git a/auplc_installer/cli.py b/auplc_installer/cli.py index f1aea4a7..390f2b32 100644 --- a/auplc_installer/cli.py +++ b/auplc_installer/cli.py @@ -15,6 +15,7 @@ import time from collections.abc import Sequence from pathlib import Path +from typing import NoReturn from auplc_installer import __version__ from auplc_installer.catalog import parse_selection_spec @@ -22,6 +23,8 @@ detect_and_configure_gpu, refine_gpu_config_from_node_labels, ) +from auplc_installer.gpu_access import provision_gpu_access +from auplc_installer.gpu_hardware import GpuHardware, classify_gpu_hardware from auplc_installer.helm import ( deploy_runtime, dev_quick_rollout, @@ -319,6 +322,22 @@ def cmd_install(state: InstallerState, *, pull: bool) -> None: keepalive.stop() +def _raise_unreachable_gpu_hardware(hardware: GpuHardware) -> NoReturn: + raise AssertionError(f"Unhandled GPU hardware classification: {hardware!r}") + + +def _provision_gpu_access_for_local_hardware(*, offline_mode: bool, bundle_dir: Path | None) -> None: + match classify_gpu_hardware(): + case GpuHardware.GPU: + provision_gpu_access(offline_mode=offline_mode, bundle_dir=bundle_dir) + case GpuHardware.CPU: + return + case GpuHardware.UNKNOWN: + raise InstallerError("Could not determine local AMD GPU hardware; refusing to modify installer state") + case unreachable: + _raise_unreachable_gpu_hardware(unreachable) + + def _cmd_install_inner(state: InstallerState, *, pull: bool) -> None: """Body of ``cmd_install`` after sudo session has been primed.""" # Pre-compute the image-stage label so the user knows up-front which path @@ -330,13 +349,16 @@ def _cmd_install_inner(state: InstallerState, *, pull: bool) -> None: else: image_stage_label = "Pulling external images & building custom images" - total = 8 + total = 9 with stage("Detecting GPU", idx=1, total=total): detect_and_configure_gpu(state.gpu, gpu_type_override=state.gpu_type) + + with stage("Provisioning GPU device access", idx=2, total=total): + _provision_gpu_access_for_local_hardware(offline_mode=state.offline_mode, bundle_dir=state.bundle_dir) paths = state.runtime_paths() - with stage("Generating values overlay (initial)", idx=2, total=total): + with stage("Generating values overlay (initial)", idx=3, total=total): # First pass: use local detection so image pulls / builds get the # right GPU_TARGET. Overlay is regenerated again below from # labeller-published labels. @@ -349,10 +371,10 @@ def _cmd_install_inner(state: InstallerState, *, pull: bool) -> None: overlay_path=paths.overlay_path, ) - with stage("Installing helm + k9s", idx=3, total=total): + with stage("Installing helm + k9s", idx=4, total=total): install_tools(offline_mode=state.offline_mode, bundle_dir=state.bundle_dir) - with stage("Installing K3s (single-node)", idx=4, total=total): + with stage("Installing K3s (single-node)", idx=5, total=total): install_k3s_single_node( offline_mode=state.offline_mode, bundle_dir=state.bundle_dir, @@ -360,7 +382,7 @@ def _cmd_install_inner(state: InstallerState, *, pull: bool) -> None: mirror_prefix=state.mirror_prefix, ) - with stage(image_stage_label, idx=5, total=total): + with stage(image_stage_label, idx=6, total=total): if state.offline_mode and state.bundle_dir is not None: load_offline_images(state.bundle_dir) elif pull: @@ -397,13 +419,13 @@ def _cmd_install_inner(state: InstallerState, *, pull: bool) -> None: k3s_images_dir=state.k3s_images_dir, ) - with stage("Deploying ROCm GPU device plugin + node labeller", idx=6, total=total): + with stage("Deploying ROCm GPU device plugin + node labeller", idx=7, total=total): deploy_rocm_gpu_device_plugin( offline_mode=state.offline_mode, bundle_dir=state.bundle_dir, ) - with stage("Refreshing values overlay from node labels", idx=7, total=total): + with stage("Refreshing values overlay from node labels", idx=8, total=total): refine_gpu_config_from_node_labels(state.gpu) generate_values_overlay( state.gpu, @@ -414,7 +436,7 @@ def _cmd_install_inner(state: InstallerState, *, pull: bool) -> None: overlay_path=paths.overlay_path, ) - with stage("Deploying JupyterHub runtime (helm install + wait)", idx=8, total=total): + with stage("Deploying JupyterHub runtime (helm install + wait)", idx=9, total=total): deploy_runtime(paths) _print_success_banner() @@ -582,6 +604,7 @@ def cmd_dev_quick(state: InstallerState) -> None: def cmd_dev_deploy(state: InstallerState) -> None: + _provision_gpu_access_for_local_hardware(offline_mode=state.offline_mode, bundle_dir=state.bundle_dir) detect_and_configure_gpu(state.gpu, gpu_type_override=state.gpu_type) paths = state.runtime_paths() refine_gpu_config_from_node_labels(state.gpu) @@ -597,6 +620,7 @@ def cmd_dev_deploy(state: InstallerState) -> None: def cmd_dev_upgrade(state: InstallerState) -> None: + _provision_gpu_access_for_local_hardware(offline_mode=state.offline_mode, bundle_dir=state.bundle_dir) detect_and_configure_gpu(state.gpu, gpu_type_override=state.gpu_type) paths = state.runtime_paths() refine_gpu_config_from_node_labels(state.gpu) @@ -613,6 +637,7 @@ def cmd_dev_upgrade(state: InstallerState) -> None: def cmd_dev_reinstall(state: InstallerState) -> None: + _provision_gpu_access_for_local_hardware(offline_mode=state.offline_mode, bundle_dir=state.bundle_dir) with contextlib.suppress(InstallerError): remove_runtime() time.sleep(0.5) @@ -623,6 +648,7 @@ def cmd_dev_reinstall(state: InstallerState) -> None: def cmd_rt_install(state: InstallerState) -> None: + _provision_gpu_access_for_local_hardware(offline_mode=state.offline_mode, bundle_dir=state.bundle_dir) detect_and_configure_gpu(state.gpu, gpu_type_override=state.gpu_type) paths = state.runtime_paths() refine_gpu_config_from_node_labels(state.gpu) @@ -638,6 +664,7 @@ def cmd_rt_install(state: InstallerState) -> None: def cmd_rt_upgrade(state: InstallerState) -> None: + _provision_gpu_access_for_local_hardware(offline_mode=state.offline_mode, bundle_dir=state.bundle_dir) detect_and_configure_gpu(state.gpu, gpu_type_override=state.gpu_type) paths = state.runtime_paths() refine_gpu_config_from_node_labels(state.gpu) @@ -678,6 +705,7 @@ def cmd_rt_remove(state: InstallerState) -> None: def cmd_rt_reinstall(state: InstallerState) -> None: + _provision_gpu_access_for_local_hardware(offline_mode=state.offline_mode, bundle_dir=state.bundle_dir) with contextlib.suppress(InstallerError): remove_runtime() time.sleep(0.5) diff --git a/auplc_installer/gpu_access.py b/auplc_installer/gpu_access.py new file mode 100644 index 00000000..534e87e2 --- /dev/null +++ b/auplc_installer/gpu_access.py @@ -0,0 +1,235 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +from __future__ import annotations + +import contextlib +import tempfile +from pathlib import Path +from typing import Protocol + +from auplc_installer.util import InstallerError, run, run_capture, verify_sha256 + +AMD_GPU_UDEV_PACKAGE_NAME = "amdgpu-insecure-instinct-udev-rules" +AMD_GPU_UDEV_PACKAGE_VERSION = "30.30.4.0-2341068.24.04" +AMD_GPU_UDEV_PACKAGE_FILENAME = "amdgpu-insecure-instinct-udev-rules_30.30.4.0-2341068.24.04_all.deb" +AMD_GPU_UDEV_PACKAGE_URL = ( + "https://repo.radeon.com/amdgpu/30.30.4/ubuntu/pool/main/a/amdgpu-insecure-instinct-udev-rules/" + f"{AMD_GPU_UDEV_PACKAGE_FILENAME}" +) +AMD_GPU_UDEV_PACKAGE_SHA256 = "4be865985c7a13114c45925e77bc0b411b9fd47d5040ed35df44b9c411766162" +AMD_GPU_UDEV_PACKAGE_RULES_PATH = Path("/etc/udev/rules.d/70-amdgpu.rules") +AMD_GPU_UDEV_PACKAGE_RULES = ( + 'KERNEL=="kfd", GROUP="render", MODE="0666"\nSUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0666"\n' +) + +LEGACY_KFD_RULES_PATH = Path("/etc/udev/rules.d/70-kfd.rules") +LEGACY_AMDGPU_RULES_PATH = Path("/etc/udev/rules.d/70-amdgpu.rules") +LEGACY_ROCM_DEVICES_RULES_PATH = Path("/etc/udev/rules.d/70-rocm-devices.rules") +LEGACY_KFD_RULES = 'KERNEL=="kfd", MODE="0666"\nSUBSYSTEM=="drm", KERNEL=="renderD*", MODE="0666"\n' +LEGACY_AMDGPU_RULES = ( + "# ROCm device permissions\n" + "# Grant render group access to AMD GPU devices\n" + "# Reference: https://rocm.docs.amd.com/projects/install-on-linux/en/latest/install/prerequisites.html#using-udev-rules\n" + 'KERNEL=="kfd", GROUP="render", MODE="0660"\n' + 'SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660"\n' +) +LEGACY_AMDGPU_PXE_RULES = 'KERNEL=="kfd", MODE="0666"\nKERNEL=="renderD[0-9]*", MODE="0666"\n' +LEGACY_ROCM_DEVICES_RULES = ( + "# ROCm device permissions\n" + "# Ensure /dev/kfd and /dev/dri/renderD* are accessible by render group\n" + 'SUBSYSTEM=="kfd", GROUP="render", MODE="0660"\n' + 'SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660"\n' +) +LEGACY_RULE_CONTENTS: dict[Path, frozenset[str]] = { + LEGACY_KFD_RULES_PATH: frozenset((LEGACY_KFD_RULES,)), + LEGACY_AMDGPU_RULES_PATH: frozenset((LEGACY_AMDGPU_RULES, LEGACY_AMDGPU_PXE_RULES)), + LEGACY_ROCM_DEVICES_RULES_PATH: frozenset((LEGACY_ROCM_DEVICES_RULES,)), +} + + +class GpuAccessHost(Protocol): + def read_text(self, path: Path) -> str | None: ... + + def remove_udev_rule(self, path: Path) -> None: ... + + def installed_package_version(self) -> str | None: ... + + def package_owns_rule(self, path: Path) -> bool: ... + + def install_package(self, deb: Path) -> None: ... + + def reload_udev_rules(self) -> None: ... + + def trigger_udev(self) -> None: ... + + def settle_udev(self) -> None: ... + + def is_symlink(self, path: Path) -> bool: ... + + def is_regular_file(self, path: Path) -> bool: ... + + def path_exists(self, path: Path) -> bool: ... + + def is_directory(self, path: Path) -> bool: ... + + +class SystemGpuAccessHost: + def read_text(self, path: Path) -> str | None: + exists = run(["test", "-e", str(path)], sudo=True, check=False) + if exists.returncode != 0: + return None + result = run_capture(["cat", str(path)], sudo=True) + return result.stdout or "" + + def remove_udev_rule(self, path: Path) -> None: + run(["rm", "-f", str(path)], sudo=True) + + def installed_package_version(self) -> str | None: + result = run_capture( + ["dpkg-query", "--show", "--showformat=${Status}\t${Version}", AMD_GPU_UDEV_PACKAGE_NAME], + sudo=True, + check=False, + ) + if result.returncode != 0: + return None + status, separator, version = (result.stdout or "").strip().partition("\t") + if status != "install ok installed" or not separator or not version: + return None + return version + + def package_owns_rule(self, path: Path) -> bool: + result = run_capture( + ["dpkg-query", "--listfiles", AMD_GPU_UDEV_PACKAGE_NAME], + sudo=True, + check=False, + ) + return result.returncode == 0 and str(path) in (result.stdout or "").splitlines() + + def install_package(self, deb: Path) -> None: + run(["dpkg", "--force-confnew", "--install", str(deb)], sudo=True) + + def reload_udev_rules(self) -> None: + run(["udevadm", "control", "--reload-rules"], sudo=True) + + def trigger_udev(self) -> None: + run(["udevadm", "trigger"], sudo=True) + + def settle_udev(self) -> None: + run(["udevadm", "settle"], sudo=True) + + def is_symlink(self, path: Path) -> bool: + return run(["test", "-L", str(path)], sudo=True, check=False).returncode == 0 + + def is_regular_file(self, path: Path) -> bool: + return run(["test", "-f", str(path)], sudo=True, check=False).returncode == 0 + + def path_exists(self, path: Path) -> bool: + return run(["test", "-e", str(path)], sudo=True, check=False).returncode == 0 + + def is_directory(self, path: Path) -> bool: + return run(["test", "-d", str(path)], sudo=True, check=False).returncode == 0 + + +def provision_gpu_access( + host: GpuAccessHost | None = None, + *, + offline_mode: bool = False, + bundle_dir: Path | None = None, +) -> None: + active_host = host if host is not None else SystemGpuAccessHost() + _validate_parent_chain(active_host, AMD_GPU_UDEV_PACKAGE_RULES_PATH.parent) + installed_version = active_host.installed_package_version() + legacy_paths = _legacy_rules_to_remove(active_host) + if installed_version == AMD_GPU_UDEV_PACKAGE_VERSION: + _verify_installed_package(active_host, installed_version) + else: + _install_package(active_host, offline_mode=offline_mode, bundle_dir=bundle_dir) + installed_version = active_host.installed_package_version() + if installed_version is None: + raise InstallerError(f"{AMD_GPU_UDEV_PACKAGE_NAME} was not installed") + _verify_installed_package(active_host, installed_version) + _remove_separate_legacy_rules(active_host, legacy_paths) + + +def _install_package(active_host: GpuAccessHost, *, offline_mode: bool, bundle_dir: Path | None) -> None: + if offline_mode: + if bundle_dir is None: + raise InstallerError("Offline GPU udev package installation requires a bundle directory") + deb = bundle_dir / "packages" / AMD_GPU_UDEV_PACKAGE_FILENAME + if not deb.is_file(): + raise InstallerError(f"Offline GPU udev package is missing: {deb}") + verify_sha256(deb, AMD_GPU_UDEV_PACKAGE_SHA256) + active_host.install_package(deb) + return + + with tempfile.NamedTemporaryFile(prefix="auplc-amdgpu-udev-", suffix=".deb", delete=False) as temporary: + deb = Path(temporary.name) + try: + run(["wget", "-q", AMD_GPU_UDEV_PACKAGE_URL, "-O", str(deb)]) + verify_sha256(deb, AMD_GPU_UDEV_PACKAGE_SHA256) + active_host.install_package(deb) + finally: + with contextlib.suppress(OSError): + deb.unlink() + + +def _verify_installed_package(active_host: GpuAccessHost, installed_version: str) -> None: + if installed_version != AMD_GPU_UDEV_PACKAGE_VERSION: + raise InstallerError( + f"{AMD_GPU_UDEV_PACKAGE_NAME} has version {installed_version}, expected {AMD_GPU_UDEV_PACKAGE_VERSION}" + ) + if not active_host.package_owns_rule(AMD_GPU_UDEV_PACKAGE_RULES_PATH): + raise InstallerError(f"{AMD_GPU_UDEV_PACKAGE_NAME} does not own {AMD_GPU_UDEV_PACKAGE_RULES_PATH}") + rule = _read_regular_text(active_host, AMD_GPU_UDEV_PACKAGE_RULES_PATH) + if rule != AMD_GPU_UDEV_PACKAGE_RULES: + raise InstallerError(f"{AMD_GPU_UDEV_PACKAGE_NAME} rule does not match the pinned package policy") + + +def _read_regular_text(host: GpuAccessHost, path: Path) -> str | None: + if host.is_symlink(path): + raise InstallerError(f"Refusing symlinked GPU udev rule: {path}") + if not host.path_exists(path): + return None + if not host.is_regular_file(path): + raise InstallerError(f"Refusing non-regular GPU udev rule: {path}") + return host.read_text(path) + + +def _validate_parent_chain(host: GpuAccessHost, parent: Path) -> None: + components = [*reversed(parent.parents), parent] + for index, component in enumerate(components): + if host.is_symlink(component): + raise InstallerError(f"Refusing symlinked GPU udev directory: {component}") + if not host.path_exists(component): + if index != len(components) - 1: + raise InstallerError(f"Missing parent GPU udev directory: {component}") + return + if not host.is_directory(component): + raise InstallerError(f"Refusing non-directory GPU udev parent: {component}") + + +def _legacy_rules_to_remove(host: GpuAccessHost) -> list[Path]: + removals: list[Path] = [] + for path, expected_contents in LEGACY_RULE_CONTENTS.items(): + content = _read_regular_text(host, path) + if content is None: + continue + if path == AMD_GPU_UDEV_PACKAGE_RULES_PATH and host.package_owns_rule(path): + continue + if content not in expected_contents: + raise InstallerError(f"Refusing to remove unexpected legacy GPU udev rule: {path}") + removals.append(path) + return removals + + +def _remove_separate_legacy_rules(host: GpuAccessHost, paths: list[Path]) -> None: + removed = False + for path in paths: + if path == AMD_GPU_UDEV_PACKAGE_RULES_PATH: + continue + host.remove_udev_rule(path) + removed = True + if removed: + host.reload_udev_rules() + host.trigger_udev() + host.settle_udev() diff --git a/auplc_installer/gpu_hardware.py b/auplc_installer/gpu_hardware.py new file mode 100644 index 00000000..0b292eb7 --- /dev/null +++ b/auplc_installer/gpu_hardware.py @@ -0,0 +1,63 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +"""Read-only local AMD GPU hardware classification from Linux PCI sysfs.""" + +from __future__ import annotations + +from enum import Enum +from pathlib import Path +from typing import Final + +PCI_DEVICES_ROOT: Final = Path("/sys/bus/pci/devices") +AMD_PCI_VENDOR: Final = "0x1002" +DISPLAY_CLASS_PREFIX: Final = "0x03" +_HEX_DIGITS: Final = frozenset("0123456789abcdef") + + +class GpuHardware(Enum): + """The local host's AMD display-hardware eligibility.""" + + GPU = "gpu" + CPU = "cpu" + UNKNOWN = "unknown" + + +def classify_gpu_hardware(pci_devices_root: Path = PCI_DEVICES_ROOT) -> GpuHardware: + """Classify local hardware using complete PCI vendor and class evidence.""" + try: + devices = tuple(pci_devices_root.iterdir()) + except OSError: + return GpuHardware.UNKNOWN + + if not devices: + return GpuHardware.UNKNOWN + + scan_is_complete = True + for device in devices: + vendor = _read_pci_attribute(device / "vendor") + pci_class = _read_pci_attribute(device / "class") + if vendor is None or pci_class is None or not _has_valid_pci_attributes(vendor, pci_class): + scan_is_complete = False + continue + if vendor == AMD_PCI_VENDOR and pci_class.startswith(DISPLAY_CLASS_PREFIX): + return GpuHardware.GPU + + return GpuHardware.CPU if scan_is_complete else GpuHardware.UNKNOWN + + +def _read_pci_attribute(path: Path) -> str | None: + try: + value = path.read_text(encoding="ascii").strip().lower() + except (OSError, UnicodeDecodeError): + return None + return value or None + + +def _has_valid_pci_attributes(vendor: str, pci_class: str) -> bool: + return _is_pci_hex(vendor, digits=4) and _is_pci_hex(pci_class, digits=6) + + +def _is_pci_hex(value: str, *, digits: int) -> bool: + return ( + len(value) == digits + 2 and value.startswith("0x") and all(character in _HEX_DIGITS for character in value[2:]) + ) diff --git a/auplc_installer/pack.py b/auplc_installer/pack.py index c4a1b4d7..ef7531b4 100644 --- a/auplc_installer/pack.py +++ b/auplc_installer/pack.py @@ -19,6 +19,11 @@ from auplc_installer.catalog import HUB_IMAGE_NAME, CourseSelection from auplc_installer.gpu import GpuConfig, detect_and_configure_gpu +from auplc_installer.gpu_access import ( + AMD_GPU_UDEV_PACKAGE_FILENAME, + AMD_GPU_UDEV_PACKAGE_SHA256, + AMD_GPU_UDEV_PACKAGE_URL, +) from auplc_installer.images import ( EXTERNAL_IMAGES, pull_and_tag, @@ -121,6 +126,14 @@ def pack_download_k3s_images(staging: Path) -> None: ) +def pack_download_gpu_access_package(staging: Path) -> None: + packages_dir = staging / "packages" + packages_dir.mkdir(parents=True, exist_ok=True) + deb = packages_dir / AMD_GPU_UDEV_PACKAGE_FILENAME + run(["wget", "-q", AMD_GPU_UDEV_PACKAGE_URL, "-O", str(deb)]) + verify_sha256(deb, AMD_GPU_UDEV_PACKAGE_SHA256) + + def pack_save_manifests(staging: Path) -> None: log_step("Saving manifests") out_dir = staging / "manifests" @@ -461,6 +474,7 @@ def pack_bundle( pack_download_binaries(staging) pack_download_k3s_images(staging) + pack_download_gpu_access_package(staging) pack_save_manifests(staging) if local_build: diff --git a/deploy/README.md b/deploy/README.md index c1a8e843..29dd70f4 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -53,18 +53,171 @@ sudo ./auplc-installer install ### Multi-Node Cluster +For SSH-preinstalled nodes, edit the Ansible inventory and multi-node values +file directly. PXE remains generator-based because the controller inventory, +rootfs settings, runtime overlay, and GPU policy must be generated as one +consistent artifact set. + +The AMD device plugin and ROCm node labeller are cluster infrastructure +prerequisites owned outside AUPLC. The infrastructure owner must deploy and +maintain them according to AMD's official guidance. If they are not installed, +follow the pinned manual installation commands in the +[Kubernetes components guide](k8s/README.md). Before Helm, verify that the +DaemonSets are ready and that GPU capacity is advertised. + +#### SSH-preinstalled + +Edit `deploy/ansible/inventory.yml` with the server and agent hostnames, IPs, +k3s token, and other site settings. Keep the human template default, +`auplc_gpu_access_enabled: auto`, unquoted on each host. `auto` runs Python 3 on +that host to scan `/sys/bus/pci/devices` for vendor `0x1002` devices whose PCI +class starts with `0x03`. It does not use `lspci` or require `pciutils`. A match +enables ROCm and the AMD GPU access package; a successful scan with no match +skips both. If a scan fails, the play aborts before either is changed, and +`any_errors_fatal` stops the play for all hosts. + +Set an unquoted YAML boolean `true` or `false` only when you need to override +detection. `true` forces ROCm and package installation, while `false` forces +both to be skipped. Either boolean bypasses the scan. Don't quote any of these +values or use alternatives such as `yes` and `no`. + +For example: + +```yaml +k3s_cluster: + children: + server: + hosts: + controller-1: + ansible_host: 192.0.2.10 + auplc_gpu_access_enabled: auto + agent: + hosts: + gpu-worker-1: + ansible_host: 192.0.2.11 + auplc_gpu_access_enabled: auto +``` + +Copy the human-maintained multi-node values example, then edit the copy for the +site's authentication, storage, images, accelerators, and network access: + +```bash +cd .. +REPO_ROOT="$(pwd)" +DEPLOY_SCRIPTS="$REPO_ROOT/skills/deploy-aup-learning-cloud/scripts" +cp runtime/values-multi-nodes.yaml.example runtime/values-multi-nodes.yaml +# Edit deploy/ansible/inventory.yml and runtime/values-multi-nodes.yaml. + +python3 "$DEPLOY_SCRIPTS/validate.py" --repo "$REPO_ROOT" --topology ssh-preinstalled \ + --inventory "$REPO_ROOT/deploy/ansible/inventory.yml" \ + --values "$REPO_ROOT/runtime/values.yaml" \ + --values "$REPO_ROOT/runtime/values-multi-nodes.yaml" \ + --helm-dry-run + +cd "$REPO_ROOT/deploy/ansible" +sudo ansible-playbook -i inventory.yml playbooks/pb-base.yml +sudo ansible-playbook -i inventory.yml playbooks/pb-k3s-site.yml +sudo ansible-playbook -i inventory.yml playbooks/pb-rocm.yml + +kubectl rollout status -n kube-system daemonset/amdgpu-device-plugin-daemonset --timeout=5m +kubectl rollout status -n kube-system daemonset/amdgpu-labeller-daemonset --timeout=5m +kubectl get nodes -o 'custom-columns=NAME:.metadata.name,AMD_GPU:.status.allocatable.amd\.com/gpu' + +cd "$REPO_ROOT" +helm upgrade --install jupyterhub ./runtime/chart \ + --namespace jupyterhub --create-namespace \ + -f runtime/values.yaml \ + -f runtime/values-multi-nodes.yaml +``` + +With `--inventory` alone, the validator accepts exactly one unquoted `auto`, +`true`, or `false` value for `auplc_gpu_access_enabled` on every managed host. +This validates the direct-edit workflow without a generated GPU resolution +report. `--gpu-resolution` may be supplied only with `--inventory`; that pair +is for generated artifacts, whose inventory values and resolution entries must +remain strict booleans. The generator never writes `auto`. + +The installer, Ansible GPU access role, and PXE controller install AMD's +`amdgpu-insecure-instinct-udev-rules` package, pinned to version +`30.30.4.0-2341068.24.04`. Its package-owned rule sets mode `0666` only on +`/dev/kfd` and DRM `/dev/dri/renderD*` nodes. It does not match +`/dev/dri/card*`; card nodes retain the normal system policy, observed as +`root:video 0660`. + +This host permission policy is separate from Kubernetes allocation. The AMD +device plugin remains the visibility boundary: only Pods that request +`amd.com/gpu` receive allocated GPU devices, and the plugin does not change +host inode ownership or mode. AUPLC Hub adds no GPU supplemental group. The +tested ROCm compute path needs none: on both SHC GPU nodes, `rocminfo` succeeded +as UID `12345` with only supplemental GID `100`, while card nodes remained +inaccessible at mode `0660`. The reported agents were `gfx1151` and `gfx1200`. + +`singleuser.fsGid: 100` controls shared notebook storage ownership only. It is +not part of GPU access and must not be treated as a GPU group setting. + +#### PXE-diskless + +Create a fresh spec, set `topology` to `pxe-diskless`, fill the PXE network +fields, and set `pxe.diskless_agents_have_amd_gpus` explicitly. Diskless agent +hardware can't be inferred from the controller. Generation writes the canonical +inventory, controller vars, runtime overlay, and GPU resolution report directly. +These artifacts express the desired deployment inputs; their existence is not +proof that rootfs provisioning succeeded. Review and install them before running +the controller playbook, whose successful completion provisions the rootfs. + ```bash -# 1. Configure Ansible inventory -cd ansible -vim inventory.yml - -# 2. Run playbooks -sudo ansible-playbook playbooks/pb-base.yml -sudo ansible-playbook playbooks/pb-k3s-site.yml - -# 3. Deploy JupyterHub -cd ../../runtime -cp values-multi-nodes.yaml.example values-multi-nodes.yaml -vim values-multi-nodes.yaml -helm upgrade --install jupyterhub ./chart -n jupyterhub --create-namespace -f values-multi-nodes.yaml +cd .. +REPO_ROOT="$(pwd)" +DEPLOY_SCRIPTS="$REPO_ROOT/skills/deploy-aup-learning-cloud/scripts" +python3 "$DEPLOY_SCRIPTS/gen_configs.py" --print-schema > spec.json +# Edit spec.json: choose pxe-diskless and fill the node, network, and PXE fields. +GENERATED_DIR="$REPO_ROOT/generated" + +cd "$REPO_ROOT" +python3 "$DEPLOY_SCRIPTS/gen_configs.py" --spec spec.json --out-dir "$GENERATED_DIR" +install -m 0600 "$GENERATED_DIR/inventory.yml" "$REPO_ROOT/deploy/ansible/inventory.yml" +install -m 0644 "$GENERATED_DIR/values-basic-example.yaml" "$REPO_ROOT/runtime/values-basic-example.yaml" +python3 "$DEPLOY_SCRIPTS/validate.py" --repo "$REPO_ROOT" --topology pxe-diskless \ + --inventory "$REPO_ROOT/deploy/ansible/inventory.yml" \ + --gpu-resolution "$GENERATED_DIR/gpu-access-resolution.json" \ + --values "$REPO_ROOT/runtime/values.yaml" \ + --values "$REPO_ROOT/runtime/values-basic-example.yaml" \ + --pxe-vars "$GENERATED_DIR/pb-pxe-controller.vars.yml" + +cd "$REPO_ROOT/deploy/ansible" +sudo ansible-playbook \ + -i "$GENERATED_DIR/inventory.yml" \ + playbooks/pb-pxe-controller.yml \ + -e @"$GENERATED_DIR/pb-pxe-controller.vars.yml" + +kubectl rollout status -n kube-system daemonset/amdgpu-device-plugin-daemonset --timeout=5m +kubectl rollout status -n kube-system daemonset/amdgpu-labeller-daemonset --timeout=5m +kubectl get nodes -o 'custom-columns=NAME:.metadata.name,AMD_GPU:.status.allocatable.amd\.com/gpu' + +cd "$REPO_ROOT" +helm upgrade --install jupyterhub ./runtime/chart \ + --namespace jupyterhub --create-namespace \ + -f runtime/values.yaml \ + -f runtime/values-basic-example.yaml ``` + +A fresh PXE rootfs receives the pinned AMD udev package during the controller +playbook. A retained rootfs is accepted only when that exact package version and +its unmodified package-owned rule are present, with no conflicting legacy GPU +rule. Rebuild or correct a retained rootfs separately if that safety check fails. + +#### Generator discovery failures + +| Error | Action | +| --- | --- | +| Host is unreachable | Restore passwordless root SSH to that inventory host, then regenerate. | +| `lspci` is missing or fails | Install `pciutils` on the reported host and rerun generation. | +| Host evidence is `UNKNOWN` or AMD GPU BDF probes disagree | Compare AMD display BDFs from `lspci` with vendor `0x1002` display-class devices under `/sys/bus/pci/devices`; fix missing or inconsistent PCI enumeration, then regenerate. | +| Retained PXE rootfs has the wrong AMD udev package version, a modified package rule, or a conflicting legacy GPU rule | Rebuild the rootfs, or correct the package state through a separate reviewed maintenance action before rerunning the playbook. | + +## Deployment branch boundary + +This branch and these instructions do not modify or roll out any live +deployment. SHC, FET, and other deployment branches or environments must +backport the host permission and immediate artifact publication changes before +their own reviewed rollout. diff --git a/deploy/ansible/README.md b/deploy/ansible/README.md index 3608aec7..0c989e0a 100644 --- a/deploy/ansible/README.md +++ b/deploy/ansible/README.md @@ -22,34 +22,39 @@ SOFTWARE. # Ansible Playbooks -K3s cluster setup playbooks based on [k3s-ansible](https://github.com/k3s-io/k3s-ansible/tree/master). - -For full instructions, see [Multi-Node Cluster Deployment](https://amdresearch.github.io/aup-learning-cloud/installation/multi-node.html). - -## Quick Reference - -```bash -# Configure inventory -vim inventory.yml - -# Base setup -sudo ansible-playbook playbooks/pb-base.yml - -# Deploy K3s cluster -sudo ansible-playbook playbooks/pb-k3s-site.yml - -# Install ROCm GPU drivers -sudo ansible-playbook playbooks/pb-rocm.yml - -# Add new nodes (update inventory.yml first) -sudo ansible-playbook playbooks/pb-k3s-site.yml - -# Reset cluster -sudo ansible-playbook playbooks/pb-k3s-reset.yml - -# Reset single node -sudo ansible-playbook playbooks/pb-k3s-reset.yml --limit -``` +K3s cluster setup playbooks based on [k3s-ansible](https://github.com/k3s-io/k3s-ansible). + +For the human SSH-preinstalled workflow, edit `inventory.yml` directly and use +the playbook commands in the [deployment guide](../README.md). Every server and +agent host entry defaults to unquoted `auplc_gpu_access_enabled: auto`. On each +host, `auto` uses Python 3 to scan `/sys/bus/pci/devices` for vendor `0x1002` +and PCI class `0x03*`; it has no `lspci` or `pciutils` dependency. A match +enables ROCm and the GPU access package, while a successful empty scan skips +both. A scan failure aborts before mutation, and `any_errors_fatal` stops the +play. Unquoted `true` and `false` force enablement or disablement and bypass +detection. + +Pass `--inventory` to validate direct values of `auto`, `true`, or `false`. A +generated `--gpu-resolution` report is not required for the human workflow. If +supplied, it requires `--inventory`, and both generated artifacts must use +strict booleans. The deploy skill never generates `auto`. + +The deploy skill has a separate generator-first SSH workflow that discovers GPU +hosts from managed-host evidence. PXE is always generator-based and uses only +`pxe.diskless_agents_have_amd_gpus` as its GPU policy input. See the +[skill scripts guide](../../skills/deploy-aup-learning-cloud/scripts/README.md) +for the complete generator-first skill command sequences. + +The GPU access role installs AMD's `amdgpu-insecure-instinct-udev-rules` +package, pinned to `30.30.4.0-2341068.24.04`, on GPU hosts and GPU-enabled PXE +root filesystems. The package sets mode `0666` only on `/dev/kfd` and DRM +`renderD*` nodes. It does not change `card*` nodes, which retain normal system +policy, observed as `root:video 0660`. + +Device-plugin allocation is a separate layer and remains the visibility +boundary for Pods requesting `amd.com/gpu`; it does not change host inode +permissions. AUPLC Hub adds no GPU supplemental group. No GPU group was needed +for the tested ROCm compute path. ## Prerequisites @@ -57,3 +62,8 @@ sudo ansible-playbook playbooks/pb-k3s-reset.yml --limit - **Python**: 3.12 - **SSH**: Root login with key-based auth to all nodes - **Hosts**: Consistent `/etc/hosts` entries across all nodes +- **GPU integration**: The infrastructure owner must deploy and maintain the AMD + device plugin and ROCm node labeller outside AUPLC. Use the pinned manual + installation in the [Kubernetes components guide](../k8s/README.md) when the + cluster does not already provide them. Before Helm, run the readiness and + capacity checks in the [deployment guide](../README.md). diff --git a/deploy/ansible/inventory.yml b/deploy/ansible/inventory.yml index a210de23..9be9f9eb 100644 --- a/deploy/ansible/inventory.yml +++ b/deploy/ansible/inventory.yml @@ -24,10 +24,13 @@ k3s_cluster: hosts: # suggested: aup-SHC1-395-1 # You need to config the hostname in /etc/hosts + # Set auto to detect AMD display hardware, or true/false to override it. : + auplc_gpu_access_enabled: auto agent: hosts: : + auplc_gpu_access_enabled: auto # strix-5: # phx-1: # phx-64g: diff --git a/deploy/ansible/playbooks/pb-gpu-access-discovery.yml b/deploy/ansible/playbooks/pb-gpu-access-discovery.yml new file mode 100644 index 00000000..e12229b3 --- /dev/null +++ b/deploy/ansible/playbooks/pb-gpu-access-discovery.yml @@ -0,0 +1,140 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +--- +- name: Discover fleet GPU-access evidence + hosts: k3s_cluster + gather_facts: false + become: false + ignore_unreachable: true + vars: + _auplc_gpu_access_unknown_evidence: + reachable: false + lspci: + rc: 255 + stdout: "" + sysfs: + rc: 255 + stdout: "" + pre_tasks: + - name: Require a safe local discovery evidence output path + ansible.builtin.assert: + that: + - gpu_access_discovery_output_path is defined + - gpu_access_discovery_output_path is string + - gpu_access_discovery_output_path is match('^/') + fail_msg: Set gpu_access_discovery_output_path to an absolute controller-local path before running discovery. + delegate_to: localhost + run_once: true + changed_when: false + + - name: Inspect local discovery evidence parent + ansible.builtin.stat: + path: "{{ gpu_access_discovery_output_path | dirname }}" + follow: false + delegate_to: localhost + run_once: true + register: _auplc_discovery_output_parent + changed_when: false + + - name: Require safe local discovery evidence parent + ansible.builtin.assert: + that: + - _auplc_discovery_output_parent.stat.exists + - _auplc_discovery_output_parent.stat.isdir + - not _auplc_discovery_output_parent.stat.islnk + fail_msg: Discovery output parent must be an existing non-symlink directory. + delegate_to: localhost + run_once: true + changed_when: false + + - name: Inspect local discovery evidence destination + ansible.builtin.stat: + path: "{{ gpu_access_discovery_output_path }}" + follow: false + delegate_to: localhost + run_once: true + register: _auplc_discovery_output_destination + changed_when: false + + - name: Require safe local discovery evidence destination + ansible.builtin.assert: + that: + - >- + not _auplc_discovery_output_destination.stat.exists or + (_auplc_discovery_output_destination.stat.isreg and + not _auplc_discovery_output_destination.stat.islnk) + fail_msg: Discovery output destination must be absent or a regular non-symlink file. + delegate_to: localhost + run_once: true + changed_when: false + tasks: + - name: Discover AMD VGA display BDFs with lspci + ansible.builtin.command: + argv: [lspci, -Dnn, -d, "1002::0300"] + register: _auplc_discovery_lspci_vga + changed_when: false + failed_when: false + + - name: Discover AMD 3D display BDFs with lspci + ansible.builtin.command: + argv: [lspci, -Dnn, -d, "1002::0302"] + register: _auplc_discovery_lspci_3d + changed_when: false + failed_when: false + + - name: Discover AMD display-controller BDFs with lspci + ansible.builtin.command: + argv: [lspci, -Dnn, -d, "1002::0380"] + register: _auplc_discovery_lspci_display + changed_when: false + failed_when: false + + - name: Combine AMD display lspci evidence + ansible.builtin.set_fact: + _auplc_discovery_lspci: + rc: >- + {{ 0 if _auplc_discovery_lspci_vga.rc == 0 and + _auplc_discovery_lspci_3d.rc == 0 and + _auplc_discovery_lspci_display.rc == 0 else 1 }} + stdout: >- + {{ [_auplc_discovery_lspci_vga.stdout | default(''), + _auplc_discovery_lspci_3d.stdout | default(''), + _auplc_discovery_lspci_display.stdout | default('')] + | reject('equalto', '') | join('\n') }} + changed_when: false + + - name: Discover AMD display BDFs through shared sysfs detector + ansible.builtin.include_role: + name: gpu_access + tasks_from: detect + + - name: Record machine-readable GPU access discovery evidence + ansible.builtin.set_fact: + _auplc_gpu_access_discovery_evidence: + host: "{{ inventory_hostname }}" + reachable: true + lspci: + rc: "{{ _auplc_discovery_lspci.rc }}" + stdout: "{{ _auplc_discovery_lspci.stdout | default('') }}" + sysfs: + rc: "{{ _auplc_gpu_access_sysfs.rc }}" + stdout: "{{ _auplc_gpu_access_sysfs.stdout | default('') }}" + changed_when: false + + - name: Write machine-readable GPU access discovery evidence locally + ansible.builtin.copy: + content: | + {"version":1,"hosts":[{% for discovery_host in ansible_play_hosts_all %} + {{ ( + hostvars[discovery_host]._auplc_gpu_access_discovery_evidence + | default( + _auplc_gpu_access_unknown_evidence | combine({'host': discovery_host}), + true + ) + | to_json + ) }}{% if not loop.last %},{% endif %} + {% endfor %}]} + dest: "{{ gpu_access_discovery_output_path }}" + mode: "0600" + delegate_to: localhost + run_once: true + changed_when: false diff --git a/deploy/ansible/playbooks/pb-rocm.yml b/deploy/ansible/playbooks/pb-rocm.yml index 504a7b71..7fb43ba8 100644 --- a/deploy/ansible/playbooks/pb-rocm.yml +++ b/deploy/ansible/playbooks/pb-rocm.yml @@ -19,6 +19,25 @@ - name: Install AMD GPU driver for ROCm 7.13.0 hosts: all + any_errors_fatal: true become: yes + pre_tasks: + - name: Resolve GPU access enablement before ROCm mutation + ansible.builtin.include_role: + name: gpu_access + tasks_from: resolve + + - name: Preflight enabled GPU access hosts before ROCm mutation + ansible.builtin.include_role: + name: gpu_access + tasks_from: preflight + when: _auplc_gpu_access_enabled_resolved roles: - - rocm + - role: rocm + when: _auplc_gpu_access_enabled_resolved + tasks: + - name: Apply GPU access after ROCm installation + ansible.builtin.include_role: + name: gpu_access + tasks_from: apply + when: _auplc_gpu_access_enabled_resolved diff --git a/deploy/ansible/playbooks/pb-udev.yml b/deploy/ansible/playbooks/pb-udev.yml index 508b7b42..53826ff1 100644 --- a/deploy/ansible/playbooks/pb-udev.yml +++ b/deploy/ansible/playbooks/pb-udev.yml @@ -19,7 +19,22 @@ - name: Configure ROCm udev rules hosts: all + any_errors_fatal: true become: yes - roles: - - udev-rocm + pre_tasks: + - name: Resolve GPU access enablement before GPU access mutation + ansible.builtin.include_role: + name: gpu_access + tasks_from: resolve + - name: Preflight enabled GPU access hosts + ansible.builtin.include_role: + name: gpu_access + tasks_from: preflight + when: _auplc_gpu_access_enabled_resolved + tasks: + - name: Apply GPU access on enabled hosts + ansible.builtin.include_role: + name: gpu_access + tasks_from: apply + when: _auplc_gpu_access_enabled_resolved diff --git a/deploy/ansible/roles/gpu_access/defaults/main.yml b/deploy/ansible/roles/gpu_access/defaults/main.yml new file mode 100644 index 00000000..65aa625c --- /dev/null +++ b/deploy/ansible/roles/gpu_access/defaults/main.yml @@ -0,0 +1,21 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +--- +auplc_gpu_access_enabled: auto +# Set for a PXE rootfs. Leave empty to configure the live host. +auplc_rootfs_path: "" +# Rootfs adapters must explicitly constrain their writable target below this +# canonical directory. Live hosts leave this empty. +auplc_rootfs_allowed_root: "" +auplc_gpu_udev_package_name: amdgpu-insecure-instinct-udev-rules +auplc_gpu_udev_package_version: 30.30.4.0-2341068.24.04 +auplc_gpu_udev_package_filename: amdgpu-insecure-instinct-udev-rules_30.30.4.0-2341068.24.04_all.deb +auplc_gpu_udev_package_url: >- + https://repo.radeon.com/amdgpu/30.30.4/ubuntu/pool/main/a/amdgpu-insecure-instinct-udev-rules/amdgpu-insecure-instinct-udev-rules_30.30.4.0-2341068.24.04_all.deb +auplc_gpu_udev_package_checksum: sha256:4be865985c7a13114c45925e77bc0b411b9fd47d5040ed35df44b9c411766162 +auplc_gpu_udev_package_cache_path: >- + /var/cache/auplc/amdgpu-udev-rules/amdgpu-insecure-instinct-udev-rules_30.30.4.0-2341068.24.04_all.deb +auplc_gpu_udev_rule_path: /etc/udev/rules.d/70-amdgpu.rules +auplc_gpu_udev_rule_content: | + KERNEL=="kfd", GROUP="render", MODE="0666" + SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0666" diff --git a/deploy/ansible/roles/gpu_access/tasks/apply.yml b/deploy/ansible/roles/gpu_access/tasks/apply.yml new file mode 100644 index 00000000..261d8a35 --- /dev/null +++ b/deploy/ansible/roles/gpu_access/tasks/apply.yml @@ -0,0 +1,128 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +--- +- name: Install AMD udev package when required + block: + - name: Create deterministic AMD udev package cache + ansible.builtin.file: + path: "{{ auplc_gpu_udev_package_cache_path | dirname }}" + state: directory + owner: root + group: root + mode: "0755" + + - name: Download checksummed AMD udev package + ansible.builtin.get_url: + url: "{{ auplc_gpu_udev_package_url }}" + dest: "{{ auplc_gpu_udev_package_cache_path }}" + checksum: "{{ auplc_gpu_udev_package_checksum }}" + owner: root + group: root + mode: "0644" + + - name: Install AMD udev package on live host + ansible.builtin.apt: + deb: "{{ auplc_gpu_udev_package_cache_path }}" + state: present + allow_downgrade: true + dpkg_options: force-confnew + when: _auplc_target_root | length == 0 + + - name: Copy AMD udev package into PXE rootfs + ansible.builtin.copy: + src: "{{ auplc_gpu_udev_package_cache_path }}" + dest: "{{ _auplc_target_root }}/tmp/{{ auplc_gpu_udev_package_filename }}" + remote_src: true + owner: root + group: root + mode: "0644" + when: _auplc_target_root | length > 0 + + - name: Install AMD udev package in PXE rootfs + ansible.builtin.command: + argv: + - chroot + - "{{ _auplc_target_root }}" + - apt-get + - --option=Dpkg::Options::=--force-confnew + - install + - --yes + - --no-install-recommends + - "/tmp/{{ auplc_gpu_udev_package_filename }}" + environment: + DEBIAN_FRONTEND: noninteractive + changed_when: true + when: _auplc_target_root | length > 0 + + - name: Verify installed AMD udev package + ansible.builtin.import_tasks: verify.yml + + always: + - name: Remove temporary AMD udev package from PXE rootfs + ansible.builtin.file: + path: "{{ _auplc_target_root }}/tmp/{{ auplc_gpu_udev_package_filename }}" + state: absent + when: _auplc_target_root | length > 0 + when: _auplc_gpu_udev_install_needed | bool + +- name: Verify installed AMD udev package without installation + ansible.builtin.import_tasks: verify.yml + when: not _auplc_gpu_udev_install_needed | bool + +- name: Recheck recognized project-owned legacy GPU rules before apply + ansible.builtin.stat: + path: "{{ item.path }}" + follow: false + loop: "{{ _auplc_legacy_gpu_rules }}" + register: _auplc_apply_legacy_gpu_rule_stats + +- name: Reject unsafe legacy GPU rules before apply + ansible.builtin.assert: + that: + - not item.stat.exists or (item.stat.isreg and not item.stat.islnk) + fail_msg: "Unexpected legacy GPU rule filesystem type: {{ item.item.path }}" + loop: "{{ _auplc_apply_legacy_gpu_rule_stats.results }}" + +- name: Read recognized project-owned legacy GPU rules before apply + ansible.builtin.slurp: + src: "{{ item.item.path }}" + loop: "{{ _auplc_apply_legacy_gpu_rule_stats.results }}" + when: item.stat.exists + register: _auplc_apply_legacy_gpu_rule_contents + +- name: Reject unexpected legacy GPU rule content before apply + ansible.builtin.assert: + that: + - >- + ((item.content | b64decode) | hash('sha256')) in item.item.item.sha256 or + (item.item.item.path == _auplc_target_root + auplc_gpu_udev_rule_path and + (item.content | b64decode) == auplc_gpu_udev_rule_content) + fail_msg: "Unexpected legacy GPU rule content: {{ item.item.item.path }}" + loop: "{{ _auplc_apply_legacy_gpu_rule_contents.results }}" + when: not item.skipped | default(false) + +- name: Remove recognized project-owned legacy GPU rules + ansible.builtin.file: + path: "{{ item.item.item.path }}" + state: absent + loop: "{{ _auplc_apply_legacy_gpu_rule_contents.results }}" + when: >- + not item.skipped | default(false) and + ((item.content | b64decode) | hash('sha256')) in item.item.item.sha256 + register: _auplc_removed_legacy_gpu_rules + +- name: Reload live udev rules after legacy cleanup + ansible.builtin.command: + argv: [udevadm, control, --reload-rules] + changed_when: false + when: + - _auplc_target_root | length == 0 + - _auplc_removed_legacy_gpu_rules.changed + +- name: Trigger live udev rules after legacy cleanup + ansible.builtin.command: + argv: [udevadm, trigger] + changed_when: false + when: + - _auplc_target_root | length == 0 + - _auplc_removed_legacy_gpu_rules.changed diff --git a/deploy/ansible/roles/gpu_access/tasks/detect.yml b/deploy/ansible/roles/gpu_access/tasks/detect.yml new file mode 100644 index 00000000..939b4a95 --- /dev/null +++ b/deploy/ansible/roles/gpu_access/tasks/detect.yml @@ -0,0 +1,16 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +--- +- name: Detect AMD display BDFs through sysfs + ansible.builtin.command: + argv: + - python3 + - -c + - >- + from pathlib import Path; devices = Path('/sys/bus/pci/devices'); + print('\n'.join(sorted(device.name for device in devices.iterdir() + if (device / 'vendor').read_text().strip() == '0x1002' and + (device / 'class').read_text().strip().startswith('0x03')))) + register: _auplc_gpu_access_sysfs + changed_when: false + failed_when: false diff --git a/deploy/ansible/roles/gpu_access/tasks/main.yml b/deploy/ansible/roles/gpu_access/tasks/main.yml new file mode 100644 index 00000000..c3317bc1 --- /dev/null +++ b/deploy/ansible/roles/gpu_access/tasks/main.yml @@ -0,0 +1,17 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +--- +- name: Resolve GPU access enablement + ansible.builtin.import_tasks: resolve.yml + +- name: Validate GPU access configuration + ansible.builtin.import_tasks: validate.yml + when: _auplc_gpu_access_enabled_resolved + +- name: Preflight GPU access target + ansible.builtin.import_tasks: preflight.yml + when: _auplc_gpu_access_enabled_resolved + +- name: Apply GPU access configuration + ansible.builtin.import_tasks: apply.yml + when: _auplc_gpu_access_enabled_resolved diff --git a/deploy/ansible/roles/gpu_access/tasks/preflight.yml b/deploy/ansible/roles/gpu_access/tasks/preflight.yml new file mode 100644 index 00000000..59810a95 --- /dev/null +++ b/deploy/ansible/roles/gpu_access/tasks/preflight.yml @@ -0,0 +1,212 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +--- +- name: Validate GPU access configuration before target preflight + ansible.builtin.import_tasks: validate.yml + +- name: Inspect GPU access rootfs target + ansible.builtin.stat: + path: "{{ _auplc_target_root }}" + follow: false + register: _auplc_rootfs + when: _auplc_target_root | length > 0 + +- name: Require regular GPU access rootfs directory + ansible.builtin.assert: + that: + - _auplc_rootfs.stat.isdir + - not _auplc_rootfs.stat.islnk + fail_msg: GPU access rootfs must be an existing non-symlink directory. + when: _auplc_target_root | length > 0 + +- name: Inspect AMD udev rule destination parents + ansible.builtin.stat: + path: "{{ _auplc_target_root }}{{ item }}" + follow: false + loop: + - /etc + - /etc/udev + - /etc/udev/rules.d + register: _auplc_destination_parent_stats + +- name: Reject unsafe AMD udev rule destination parents + ansible.builtin.assert: + that: + - not item.stat.exists or (item.stat.isdir and not item.stat.islnk) + fail_msg: "Unsafe AMD udev rule destination parent: {{ item.item }}" + loop: "{{ _auplc_destination_parent_stats.results }}" + +- name: Define recognized project-owned legacy GPU rules + ansible.builtin.set_fact: + _auplc_legacy_gpu_rules: + - path: "{{ _auplc_target_root }}/etc/udev/rules.d/70-kfd.rules" + sha256: + - 79773871430cb63f5a28cf25666e0eccacf2bb27d4d9f48e10d0b05931650cf0 + - path: "{{ _auplc_target_root }}/etc/udev/rules.d/70-amdgpu.rules" + sha256: + - 678b6a1084576de785b47fcfa0c0b3048117a3add62c1fb8dcff83947004005b + - cc5e78a7861477ac5169a4b84edd4e687c1f14b9a88a9557b0c986479ebbaccd + - path: "{{ _auplc_target_root }}/etc/udev/rules.d/70-rocm-devices.rules" + sha256: + - 951fb3d879d2d45b56cfd4cdb0f7ea061a4a0af77d93b9f2a4da9a8c36d20cad + +- name: Inspect AMD udev rule destination + ansible.builtin.stat: + path: "{{ _auplc_target_root }}{{ auplc_gpu_udev_rule_path }}" + follow: false + register: _auplc_destination_rule + +- name: Reject unsafe AMD udev rule destination + ansible.builtin.assert: + that: + - not _auplc_destination_rule.stat.exists or + (_auplc_destination_rule.stat.isreg and not _auplc_destination_rule.stat.islnk) + fail_msg: Unsafe AMD udev rule destination. + +- name: Query installed AMD udev package on live host + ansible.builtin.command: + argv: + - dpkg-query + - --showformat=${Status}\t${Version} + - --show + - "{{ auplc_gpu_udev_package_name }}" + register: _auplc_live_package + changed_when: false + failed_when: false + when: _auplc_target_root | length == 0 + +- name: Query installed AMD udev package in PXE rootfs + ansible.builtin.command: + argv: + - chroot + - "{{ _auplc_target_root }}" + - dpkg-query + - --showformat=${Status}\t${Version} + - --show + - "{{ auplc_gpu_udev_package_name }}" + register: _auplc_rootfs_package + changed_when: false + failed_when: false + when: _auplc_target_root | length > 0 + +- name: Record installed AMD udev package state on live host + ansible.builtin.set_fact: + _auplc_installed_package: "{{ _auplc_live_package }}" + when: _auplc_target_root | length == 0 + +- name: Record installed AMD udev package state in PXE rootfs + ansible.builtin.set_fact: + _auplc_installed_package: "{{ _auplc_rootfs_package }}" + when: _auplc_target_root | length > 0 + +- name: Record whether AMD udev package installation is needed + ansible.builtin.set_fact: + _auplc_gpu_udev_install_needed: >- + {{ _auplc_installed_package.rc != 0 or + _auplc_installed_package.stdout != 'install ok installed' ~ '\t' ~ auplc_gpu_udev_package_version }} + +- name: Query AMD udev rule package ownership on live host before admission + ansible.builtin.command: + argv: + - dpkg-query + - --search + - "{{ auplc_gpu_udev_rule_path }}" + register: _auplc_live_rule_owner + changed_when: false + failed_when: false + when: _auplc_target_root | length == 0 + +- name: Query AMD udev rule package ownership in PXE rootfs before admission + ansible.builtin.command: + argv: + - chroot + - "{{ _auplc_target_root }}" + - dpkg-query + - --search + - "{{ auplc_gpu_udev_rule_path }}" + register: _auplc_rootfs_rule_owner + changed_when: false + failed_when: false + when: _auplc_target_root | length > 0 + +- name: Record AMD udev rule owner on live host + ansible.builtin.set_fact: + _auplc_existing_rule_owner: "{{ _auplc_live_rule_owner }}" + when: _auplc_target_root | length == 0 + +- name: Record AMD udev rule owner in PXE rootfs + ansible.builtin.set_fact: + _auplc_existing_rule_owner: "{{ _auplc_rootfs_rule_owner }}" + when: _auplc_target_root | length > 0 + +- name: Record whether the AMD udev rule is package-owned + ansible.builtin.set_fact: + _auplc_rule_owned_by_amd_package: >- + {{ _auplc_existing_rule_owner.rc == 0 and + _auplc_existing_rule_owner.stdout == auplc_gpu_udev_package_name + ': ' + auplc_gpu_udev_rule_path }} + +- name: Read existing AMD udev rule + ansible.builtin.slurp: + src: "{{ _auplc_target_root }}{{ auplc_gpu_udev_rule_path }}" + register: _auplc_existing_rule + when: _auplc_destination_rule.stat.exists + +- name: Allow package-owned AMD udev rule convergence + ansible.builtin.set_fact: + _auplc_rule_content_admitted: >- + {{ (not _auplc_destination_rule.stat.exists) or + ((_auplc_existing_rule.content | b64decode) == auplc_gpu_udev_rule_content) or + ((_auplc_gpu_udev_install_needed | bool) and + (((_auplc_existing_rule.content | b64decode) | hash('sha256')) in _auplc_legacy_gpu_rules[1].sha256 or + (_auplc_rule_owned_by_amd_package | bool)) }} + +- name: Reject modified AMD udev rule before package installation + ansible.builtin.assert: + that: _auplc_rule_content_admitted | bool + fail_msg: Existing AMD udev rule is neither the package rule nor a recognized legacy rule. + +- name: Inspect recognized project-owned legacy GPU rules + ansible.builtin.stat: + path: "{{ item.path }}" + follow: false + loop: "{{ _auplc_legacy_gpu_rules }}" + register: _auplc_legacy_gpu_rule_stats + +- name: Reject legacy GPU rule symlinks and non-regular files + ansible.builtin.assert: + that: + - not item.stat.exists or (item.stat.isreg and not item.stat.islnk) + fail_msg: "Unexpected legacy GPU rule filesystem type: {{ item.item.path }}" + loop: "{{ _auplc_legacy_gpu_rule_stats.results }}" + +- name: Read recognized project-owned legacy GPU rules + ansible.builtin.slurp: + src: "{{ item.item.path }}" + loop: "{{ _auplc_legacy_gpu_rule_stats.results }}" + when: item.stat.exists + register: _auplc_legacy_gpu_rule_contents + +- name: Reject unexpected legacy GPU rule content + ansible.builtin.assert: + that: + - >- + ( + item.item.item.path != _auplc_target_root + auplc_gpu_udev_rule_path and + ((item.content | b64decode) | hash('sha256')) in item.item.item.sha256 + ) or + ( + item.item.item.path == _auplc_target_root + auplc_gpu_udev_rule_path and + ( + (item.content | b64decode) == auplc_gpu_udev_rule_content or + ( + (_auplc_gpu_udev_install_needed | bool) and + ( + ((item.content | b64decode) | hash('sha256')) in item.item.item.sha256 or + (_auplc_rule_owned_by_amd_package | bool) + ) + ) + ) + ) + fail_msg: "Unexpected legacy GPU rule content: {{ item.item.item.path }}" + loop: "{{ _auplc_legacy_gpu_rule_contents.results }}" + when: not item.skipped | default(false) diff --git a/deploy/ansible/roles/gpu_access/tasks/resolve.yml b/deploy/ansible/roles/gpu_access/tasks/resolve.yml new file mode 100644 index 00000000..b7f36282 --- /dev/null +++ b/deploy/ansible/roles/gpu_access/tasks/resolve.yml @@ -0,0 +1,42 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +--- +- name: Validate GPU access enablement policy + ansible.builtin.assert: + that: + - auplc_gpu_access_enabled is defined + - >- + auplc_gpu_access_enabled is boolean or + (auplc_gpu_access_enabled is string and auplc_gpu_access_enabled == 'auto') + fail_msg: >- + Set auplc_gpu_access_enabled to true, false, or unquoted auto for every + host before running GPU access tasks. + changed_when: false + +- name: Detect GPU access hardware for auto policy + ansible.builtin.import_tasks: detect.yml + when: auplc_gpu_access_enabled == 'auto' + +- name: Require successful GPU access hardware detection for auto policy + ansible.builtin.assert: + that: + - _auplc_gpu_access_sysfs.rc == 0 + fail_msg: >- + GPU access hardware detection failed for auto policy; no GPU access + mutation was attempted. + when: auplc_gpu_access_enabled == 'auto' + changed_when: false + +- name: Resolve GPU access enablement + ansible.builtin.set_fact: + _auplc_gpu_access_enabled_resolved: >- + {{ auplc_gpu_access_enabled if auplc_gpu_access_enabled is boolean + else (_auplc_gpu_access_sysfs.stdout | trim | length > 0) }} + changed_when: false + +- name: Require resolved GPU access enablement boolean + ansible.builtin.assert: + that: + - _auplc_gpu_access_enabled_resolved is boolean + fail_msg: GPU access enablement did not resolve to a boolean. + changed_when: false diff --git a/deploy/ansible/roles/gpu_access/tasks/validate.yml b/deploy/ansible/roles/gpu_access/tasks/validate.yml new file mode 100644 index 00000000..dbfa855e --- /dev/null +++ b/deploy/ansible/roles/gpu_access/tasks/validate.yml @@ -0,0 +1,44 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +--- +- name: Validate GPU access rootfs path syntax + ansible.builtin.assert: + that: + - auplc_rootfs_path is string + - auplc_rootfs_path == '' or auplc_rootfs_path is match('^/') + - auplc_rootfs_path != '/' + - "'..' not in auplc_rootfs_path.split('/')" + - auplc_rootfs_path == '' or auplc_rootfs_allowed_root | length > 0 + fail_msg: auplc_rootfs_path must be a non-root absolute path without traversal and with an allowed root. + +- name: Canonicalize GPU access rootfs path + ansible.builtin.command: + argv: + - realpath + - --canonicalize-missing + - "{{ auplc_rootfs_path }}" + register: _auplc_canonical_rootfs + changed_when: false + when: auplc_rootfs_path | length > 0 + +- name: Canonicalize allowed GPU access rootfs parent + ansible.builtin.command: + argv: + - realpath + - --canonicalize-existing + - "{{ auplc_rootfs_allowed_root }}" + register: _auplc_canonical_allowed_root + changed_when: false + when: auplc_rootfs_path | length > 0 + +- name: Constrain canonical GPU access rootfs path + ansible.builtin.assert: + that: + - _auplc_canonical_rootfs.stdout != '/' + - _auplc_canonical_rootfs.stdout.startswith(_auplc_canonical_allowed_root.stdout + '/') + fail_msg: GPU access rootfs escapes auplc_rootfs_allowed_root. + when: auplc_rootfs_path | length > 0 + +- name: Record canonical GPU access target root + ansible.builtin.set_fact: + _auplc_target_root: "{{ _auplc_canonical_rootfs.stdout if auplc_rootfs_path | length > 0 else '' }}" diff --git a/deploy/ansible/roles/gpu_access/tasks/verify.yml b/deploy/ansible/roles/gpu_access/tasks/verify.yml new file mode 100644 index 00000000..5e6a3be5 --- /dev/null +++ b/deploy/ansible/roles/gpu_access/tasks/verify.yml @@ -0,0 +1,162 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +--- +- name: Validate GPU access configuration before package verification + ansible.builtin.import_tasks: validate.yml + +- name: Inspect GPU access rootfs before package verification + ansible.builtin.stat: + path: "{{ _auplc_target_root }}" + follow: false + register: _auplc_verify_rootfs + when: _auplc_target_root | length > 0 + +- name: Require regular GPU access rootfs before package verification + ansible.builtin.assert: + that: + - _auplc_verify_rootfs.stat.isdir + - not _auplc_verify_rootfs.stat.islnk + fail_msg: GPU access rootfs must be an existing non-symlink directory. + when: _auplc_target_root | length > 0 + +- name: Inspect AMD udev rule parents before package verification + ansible.builtin.stat: + path: "{{ _auplc_target_root }}{{ item }}" + follow: false + loop: + - /etc + - /etc/udev + - /etc/udev/rules.d + register: _auplc_verify_parent_stats + +- name: Require safe AMD udev rule parents before package verification + ansible.builtin.assert: + that: + - item.stat.exists + - item.stat.isdir + - not item.stat.islnk + fail_msg: "Unsafe AMD udev rule parent: {{ item.item }}" + loop: "{{ _auplc_verify_parent_stats.results }}" + +- name: Inspect retained PXE shipped legacy GPU rules + ansible.builtin.stat: + path: "{{ _auplc_target_root }}{{ item }}" + follow: false + loop: + - /etc/udev/rules.d/70-kfd.rules + - /etc/udev/rules.d/70-rocm-devices.rules + register: _auplc_retained_legacy_gpu_rule_stats + when: auplc_reject_legacy_gpu_rules | default(false) | bool + +- name: Reject retained PXE shipped legacy GPU rules + ansible.builtin.assert: + that: not item.stat.exists + fail_msg: "Retained PXE rootfs has a shipped legacy GPU rule: {{ item.item }}" + loop: "{{ _auplc_retained_legacy_gpu_rule_stats.results | default([]) }}" + when: auplc_reject_legacy_gpu_rules | default(false) | bool + +- name: Query installed AMD udev package version on live host + ansible.builtin.command: + argv: + - dpkg-query + - --showformat=${Status}\t${Version} + - --show + - "{{ auplc_gpu_udev_package_name }}" + register: _auplc_verify_live_package + changed_when: false + failed_when: false + when: _auplc_target_root | length == 0 + +- name: Query installed AMD udev package version in PXE rootfs + ansible.builtin.command: + argv: + - chroot + - "{{ _auplc_target_root }}" + - dpkg-query + - --showformat=${Status}\t${Version} + - --show + - "{{ auplc_gpu_udev_package_name }}" + register: _auplc_verify_rootfs_package + changed_when: false + failed_when: false + when: _auplc_target_root | length > 0 + +- name: Require installed AMD udev package status and exact version on live host + ansible.builtin.assert: + that: + - _auplc_verify_live_package.rc == 0 + - _auplc_verify_live_package.stdout == 'install ok installed' ~ '\t' ~ auplc_gpu_udev_package_version + fail_msg: AMD udev package is not installed at the required version. + when: _auplc_target_root | length == 0 + +- name: Require installed AMD udev package status and exact version in PXE rootfs + ansible.builtin.assert: + that: + - _auplc_verify_rootfs_package.rc == 0 + - _auplc_verify_rootfs_package.stdout == 'install ok installed' ~ '\t' ~ auplc_gpu_udev_package_version + fail_msg: AMD udev package is not installed at the required version. + when: _auplc_target_root | length > 0 + +- name: Query AMD udev rule package ownership on live host + ansible.builtin.command: + argv: + - dpkg-query + - --search + - "{{ auplc_gpu_udev_rule_path }}" + register: _auplc_verify_live_rule_owner + changed_when: false + failed_when: false + when: _auplc_target_root | length == 0 + +- name: Query AMD udev rule package ownership in PXE rootfs + ansible.builtin.command: + argv: + - chroot + - "{{ _auplc_target_root }}" + - dpkg-query + - --search + - "{{ auplc_gpu_udev_rule_path }}" + register: _auplc_verify_rootfs_rule_owner + changed_when: false + failed_when: false + when: _auplc_target_root | length > 0 + +- name: Require package-owned AMD udev rule on live host + ansible.builtin.assert: + that: + - _auplc_verify_live_rule_owner.rc == 0 + - "_auplc_verify_live_rule_owner.stdout == auplc_gpu_udev_package_name + ': ' + auplc_gpu_udev_rule_path" + fail_msg: AMD udev rule is not package-owned by the required package. + when: _auplc_target_root | length == 0 + +- name: Require package-owned AMD udev rule in PXE rootfs + ansible.builtin.assert: + that: + - _auplc_verify_rootfs_rule_owner.rc == 0 + - "_auplc_verify_rootfs_rule_owner.stdout == auplc_gpu_udev_package_name + ': ' + auplc_gpu_udev_rule_path" + fail_msg: AMD udev rule is not package-owned by the required package. + when: _auplc_target_root | length > 0 + +- name: Inspect installed AMD udev rule + ansible.builtin.stat: + path: "{{ _auplc_target_root }}{{ auplc_gpu_udev_rule_path }}" + follow: false + register: _auplc_verify_rule + +- name: Require safe installed AMD udev rule + ansible.builtin.assert: + that: + - _auplc_verify_rule.stat.exists + - _auplc_verify_rule.stat.isreg + - not _auplc_verify_rule.stat.islnk + fail_msg: AMD udev rule has an unsafe filesystem type. + +- name: Read installed AMD udev rule + ansible.builtin.slurp: + src: "{{ _auplc_target_root }}{{ auplc_gpu_udev_rule_path }}" + register: _auplc_verify_rule_content + +- name: Require exact AMD udev rule content + ansible.builtin.assert: + that: (_auplc_verify_rule_content.content | b64decode) == auplc_gpu_udev_rule_content + fail_msg: AMD udev rule is a modified package conffile. diff --git a/deploy/ansible/roles/pxe_controller/defaults/main.yml b/deploy/ansible/roles/pxe_controller/defaults/main.yml index e381140f..66ec003c 100644 --- a/deploy/ansible/roles/pxe_controller/defaults/main.yml +++ b/deploy/ansible/roles/pxe_controller/defaults/main.yml @@ -85,11 +85,14 @@ pxe_apt_mirror: "http://tw.archive.ubuntu.com/ubuntu" pxe_rootfs_force_rebuild: true # Run apt-get upgrade inside rootfs during chroot setup pxe_rootfs_upgrade: false +# Explicitly enable GPU access only for a rootfs intended for GPU workers. +pxe_gpu_access_enabled: false # ============================================================ # Paths # ============================================================ pxe_nfs_root: "/srv/nfs/rootfs" +pxe_nfs_allowed_root: "/srv/nfs" pxe_tftp_root: "/srv/tftp" pxe_web_root: "/var/www/html" diff --git a/deploy/ansible/roles/pxe_controller/tasks/gpu_access.yml b/deploy/ansible/roles/pxe_controller/tasks/gpu_access.yml new file mode 100644 index 00000000..adec7028 --- /dev/null +++ b/deploy/ansible/roles/pxe_controller/tasks/gpu_access.yml @@ -0,0 +1,45 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +--- +- name: Record PXE GPU admission disposition + ansible.builtin.set_fact: + _pxe_rootfs_disposition: "{{ 'fresh' if _pxe_rootfs_rebuilt_this_run | bool else 'retained' }}" + +- name: Assert PXE GPU admission phase + ansible.builtin.assert: + that: pxe_gpu_admission_phase in ['retained-read-only', 'final'] + fail_msg: PXE GPU admission phase is invalid. + +- name: Verify retained PXE AMD udev package before lifecycle changes + ansible.builtin.include_role: + name: gpu_access + tasks_from: verify + vars: + auplc_rootfs_path: "{{ pxe_nfs_root }}" + auplc_rootfs_allowed_root: "{{ pxe_nfs_allowed_root }}" + auplc_reject_legacy_gpu_rules: true + when: + - pxe_gpu_access_enabled | bool + - _pxe_rootfs_disposition == 'retained' + +- name: Preflight GPU access after final PXE re-preflight + ansible.builtin.include_role: + name: gpu_access + tasks_from: preflight + vars: + auplc_rootfs_path: "{{ pxe_nfs_root }}" + auplc_rootfs_allowed_root: "{{ pxe_nfs_allowed_root }}" + when: + - pxe_gpu_access_enabled | bool + - pxe_gpu_admission_phase == 'final' + +- name: Apply GPU access after final PXE re-preflight + ansible.builtin.include_role: + name: gpu_access + tasks_from: apply + vars: + auplc_rootfs_path: "{{ pxe_nfs_root }}" + auplc_rootfs_allowed_root: "{{ pxe_nfs_allowed_root }}" + when: + - pxe_gpu_access_enabled | bool + - pxe_gpu_admission_phase == 'final' diff --git a/deploy/ansible/roles/pxe_controller/tasks/main.yml b/deploy/ansible/roles/pxe_controller/tasks/main.yml index b16e2af0..1a34b0e1 100644 --- a/deploy/ansible/roles/pxe_controller/tasks/main.yml +++ b/deploy/ansible/roles/pxe_controller/tasks/main.yml @@ -82,6 +82,86 @@ # 2. Build NFS rootfs with debootstrap # ========================================================== +- name: Validate PXE rootfs path syntax before lifecycle changes + ansible.builtin.assert: + that: + - pxe_nfs_root is string + - pxe_nfs_root is match('^/') + - pxe_nfs_root != '/' + - "'..' not in pxe_nfs_root.split('/')" + - pxe_nfs_allowed_root is string + - pxe_nfs_allowed_root is match('^/') + fail_msg: pxe_nfs_root and pxe_nfs_allowed_root must be absolute non-root paths without traversal. + +- name: Canonicalize PXE rootfs before lifecycle changes + ansible.builtin.command: + argv: [realpath, --canonicalize-missing, "{{ pxe_nfs_root }}"] + register: _pxe_canonical_nfs_root_result + changed_when: false + +- name: Canonicalize trusted PXE rootfs parent before lifecycle changes + ansible.builtin.command: + argv: [realpath, --canonicalize-existing, "{{ pxe_nfs_allowed_root }}"] + register: _pxe_canonical_nfs_allowed_root + changed_when: false + +- name: Inspect PXE rootfs path before canonical lifecycle changes + ansible.builtin.stat: + path: "{{ pxe_nfs_root }}" + follow: false + register: _pxe_rootfs_lstat + +- name: Constrain canonical PXE rootfs before lifecycle changes + ansible.builtin.assert: + that: + - not _pxe_rootfs_lstat.stat.exists or not _pxe_rootfs_lstat.stat.islnk + - _pxe_canonical_nfs_root_result.stdout != '/' + - _pxe_canonical_nfs_root_result.stdout == pxe_nfs_root + - _pxe_canonical_nfs_root_result.stdout.startswith(_pxe_canonical_nfs_allowed_root.stdout + '/') + fail_msg: pxe_nfs_root must be a non-symlink descendant of pxe_nfs_allowed_root. + +- name: Record canonical PXE rootfs for lifecycle operations + ansible.builtin.set_fact: + _pxe_canonical_nfs_root: "{{ _pxe_canonical_nfs_root_result.stdout }}" + pxe_nfs_root: "{{ _pxe_canonical_nfs_root_result.stdout }}" + +- name: Require existing PXE rootfs is a directory + ansible.builtin.assert: + that: + - not _pxe_rootfs_lstat.stat.exists or _pxe_rootfs_lstat.stat.isdir + fail_msg: "pxe_nfs_root must be a directory when it already exists: {{ pxe_nfs_root }}" + +- name: Inspect PXE rootfs readiness before lifecycle changes + ansible.builtin.stat: + path: "{{ pxe_nfs_root }}/bin/bash" + follow: false + register: _pxe_rootfs_start + +- name: Record PXE rootfs state before lifecycle changes + ansible.builtin.set_fact: + _pxe_rootfs_existed_at_start: "{{ _pxe_rootfs_lstat.stat.exists | bool }}" + _pxe_rootfs_rebuilt_this_run: >- + {{ (pxe_rootfs_force_rebuild | bool) or not (_pxe_rootfs_lstat.stat.exists | bool) }} + +- name: Require incomplete PXE rootfs force rebuild + ansible.builtin.assert: + that: + - >- + not (_pxe_rootfs_lstat.stat.exists | bool) or + (_pxe_rootfs_start.stat.exists | bool) or + (pxe_rootfs_force_rebuild | bool) + fail_msg: >- + Existing PXE rootfs is incomplete and must be rebuilt with + pxe_rootfs_force_rebuild=true; debootstrap will not modify it in place. + +- name: Admit retained PXE GPU rootfs read-only before lifecycle changes + ansible.builtin.include_tasks: gpu_access.yml + vars: + pxe_gpu_admission_phase: retained-read-only + when: + - pxe_gpu_access_enabled | bool + - not (_pxe_rootfs_rebuilt_this_run | bool) + - name: Stop NFS before rootfs rebuild when: pxe_rootfs_force_rebuild | bool ansible.builtin.systemd: @@ -92,9 +172,16 @@ - name: Remove existing rootfs (force rebuild) when: pxe_rootfs_force_rebuild | bool ansible.builtin.shell: | - mountpoint -q {{ pxe_nfs_root }}/dev && umount {{ pxe_nfs_root }}/dev || true - mountpoint -q {{ pxe_nfs_root }}/sys && umount {{ pxe_nfs_root }}/sys || true - mountpoint -q {{ pxe_nfs_root }}/proc && umount {{ pxe_nfs_root }}/proc || true + set -e + if mountpoint -q {{ pxe_nfs_root }}/dev; then + umount {{ pxe_nfs_root }}/dev + fi + if mountpoint -q {{ pxe_nfs_root }}/sys; then + umount {{ pxe_nfs_root }}/sys + fi + if mountpoint -q {{ pxe_nfs_root }}/proc; then + umount {{ pxe_nfs_root }}/proc + fi rm -rf {{ pxe_nfs_root }} changed_when: true @@ -219,9 +306,16 @@ always: - name: Unmount virtual filesystems from chroot ansible.builtin.shell: | - mountpoint -q {{ pxe_nfs_root }}/dev && umount {{ pxe_nfs_root }}/dev || true - mountpoint -q {{ pxe_nfs_root }}/sys && umount {{ pxe_nfs_root }}/sys || true - mountpoint -q {{ pxe_nfs_root }}/proc && umount {{ pxe_nfs_root }}/proc || true + set -e + if mountpoint -q {{ pxe_nfs_root }}/dev; then + umount {{ pxe_nfs_root }}/dev + fi + if mountpoint -q {{ pxe_nfs_root }}/sys; then + umount {{ pxe_nfs_root }}/sys + fi + if mountpoint -q {{ pxe_nfs_root }}/proc; then + umount {{ pxe_nfs_root }}/proc + fi changed_when: true - name: Remove chroot setup script @@ -229,6 +323,13 @@ path: "{{ pxe_nfs_root }}/tmp/chroot-setup.sh" state: absent +- name: Re-preflight PXE GPU rootfs before TFTP + ansible.builtin.include_tasks: gpu_access.yml + vars: + pxe_gpu_admission_phase: final + when: + - pxe_gpu_access_enabled | bool + # ========================================================== # 6. Copy kernel and initrd to TFTP # ========================================================== diff --git a/deploy/ansible/roles/pxe_controller/templates/chroot-setup.sh.j2 b/deploy/ansible/roles/pxe_controller/templates/chroot-setup.sh.j2 index 50ea0867..c504a497 100644 --- a/deploy/ansible/roles/pxe_controller/templates/chroot-setup.sh.j2 +++ b/deploy/ansible/roles/pxe_controller/templates/chroot-setup.sh.j2 @@ -50,12 +50,6 @@ echo "PermitRootLogin yes" > /etc/ssh/sshd_config.d/allow-root.conf echo "PermitRootLogin prohibit-password" > /etc/ssh/sshd_config.d/allow-root.conf {% endif %} -# -- GPU udev rules (let containers access AMD GPUs) -- -tee /etc/udev/rules.d/70-amdgpu.rules << RULES -KERNEL=="kfd", MODE="0666" -KERNEL=="renderD[0-9]*", MODE="0666" -RULES - # -- Disable systemd-networkd (kernel ip=dhcp handles NFS root networking) -- rm -f /etc/netplan/*.yaml systemctl disable systemd-networkd 2>/dev/null || true diff --git a/deploy/ansible/roles/rocm/tasks/main.yml b/deploy/ansible/roles/rocm/tasks/main.yml index 525a3a18..d0353f1b 100644 --- a/deploy/ansible/roles/rocm/tasks/main.yml +++ b/deploy/ansible/roles/rocm/tasks/main.yml @@ -53,25 +53,3 @@ apt: name: amdgpu-dkms state: present - -- name: Ensure render group exists with consistent GID - group: - name: render - gid: 993 - state: present - -- name: Set udev rules for ROCm devices with correct permissions - copy: - content: | - # ROCm device permissions - # Grant render group access to AMD GPU devices - # Reference: https://rocm.docs.amd.com/projects/install-on-linux/en/latest/install/prerequisites.html#using-udev-rules - KERNEL=="kfd", GROUP="render", MODE="0660" - SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660" - dest: /etc/udev/rules.d/70-amdgpu.rules - mode: '0644' - register: udev_rules_changed - -- name: Reload udev rules if changed - shell: udevadm control --reload-rules && udevadm trigger - when: udev_rules_changed.changed diff --git a/deploy/ansible/roles/udev/main.yml b/deploy/ansible/roles/udev/main.yml deleted file mode 100644 index 235fe25e..00000000 --- a/deploy/ansible/roles/udev/main.yml +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - ---- -- name: Create /etc/udev/rules.d/70-kfd.rules - copy: - dest: /etc/udev/rules.d/70-kfd.rules - content: | - KERNEL=="kfd", MODE="0666" - SUBSYSTEM=="drm", KERNEL=="renderD*", MODE="0666" - owner: root - group: root - mode: '0644' - -- name: Reload udev rules - command: udevadm control --reload-rules - -- name: Trigger udev rules - command: udevadm trigger - -- name: Reboot the system (optional) - reboot: - msg: "Rebooting to apply udev rule changes" - pre_reboot_delay: 5 - reboot_timeout: 300 - post_reboot_delay: 30 - when: udev_rocm_reboot_enabled - diff --git a/deploy/k8s/README.md b/deploy/k8s/README.md index be3c471c..7334c198 100644 --- a/deploy/k8s/README.md +++ b/deploy/k8s/README.md @@ -40,17 +40,34 @@ which `runtime/values.yaml` uses as `nodeSelector`s. The installer pins the accelerator `nodeSelector` to the real `amd.com/gpu.product-name` detected on the host, so no manual labelling is needed on single-machine deployments. -If you are deploying manually instead: +For multi-node deployments, the AMD device plugin and ROCm node labeller are +cluster infrastructure prerequisites owned outside AUPLC. The infrastructure +owner must select, deploy, and maintain them according to the +[official AMD Kubernetes device plugin project](https://github.com/ROCm/k8s-device-plugin). + +The device plugin allocates devices to Pods; it does not set host device-node +permissions. Host provisioning separately installs the pinned +`amdgpu-insecure-instinct-udev-rules` package at version +`30.30.4.0-2341068.24.04`. That package sets mode `0666` only on `/dev/kfd` and +DRM `renderD*` nodes and leaves `card*` under normal system policy. AUPLC adds +no supplemental GPU group; none is required for the tested ROCm compute path. + +To install the same pinned manifests used by `auplc-installer`: ```bash -# Deploy AMD GPU device plugin -kubectl create -f https://raw.githubusercontent.com/ROCm/k8s-device-plugin/master/k8s-ds-amdgpu-dp.yaml +ROCM_DEVICE_PLUGIN_COMMIT="dea1db13f05159e64d8114bca4c31f48c3cfcac6" +kubectl apply -f \ + "https://raw.githubusercontent.com/ROCm/k8s-device-plugin/$ROCM_DEVICE_PLUGIN_COMMIT/k8s-ds-amdgpu-dp.yaml" +kubectl apply -f \ + "https://raw.githubusercontent.com/ROCm/k8s-device-plugin/$ROCM_DEVICE_PLUGIN_COMMIT/k8s-ds-amdgpu-labeller.yaml" +``` -# Deploy AMD GPU node labeller (publishes amd.com/gpu.* labels) -kubectl create -f https://raw.githubusercontent.com/ROCm/k8s-device-plugin/master/k8s-ds-amdgpu-labeller.yaml +Before deploying the AUPLC Helm release, verify the installation: -# Verify GPU detection and labels -kubectl describe node | grep amd.com/gpu +```bash +kubectl rollout status -n kube-system daemonset/amdgpu-device-plugin-daemonset --timeout=5m +kubectl rollout status -n kube-system daemonset/amdgpu-labeller-daemonset --timeout=5m +kubectl get nodes -o 'custom-columns=NAME:.metadata.name,AMD_GPU:.status.allocatable.amd\.com/gpu' ``` `runtime/values-multi-nodes.yaml.example` now follows `runtime/values.yaml` and diff --git a/dockerfiles/Base/Dockerfile.rocm b/dockerfiles/Base/Dockerfile.rocm index 43cf8c25..3c2267a6 100644 --- a/dockerfiles/Base/Dockerfile.rocm +++ b/dockerfiles/Base/Dockerfile.rocm @@ -226,14 +226,6 @@ RUN if getent passwd 1000 > /dev/null; then \ RUN useradd -m -s /bin/bash -N -u $NB_UID -g $NB_GID $NB_USER && \ echo "$NB_USER ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers -# Add jovyan to video and render groups for ROCm access -RUN if getent group render; then \ - groupmod -g 992 render; \ - else \ - groupadd -g 992 render; \ - fi -RUN usermod -aG video,render ${NB_USER} - # Create necessary Jupyter directories with correct permissions RUN mkdir -p /home/$NB_USER/.jupyter && \ mkdir -p /home/$NB_USER/.local/share/jupyter/runtime && \ @@ -250,15 +242,8 @@ RUN echo '#!/bin/bash' > /home/$NB_USER/start-jupyter.sh && \ # Verify the file exists (will fail build if not) ls -la /home/$NB_USER/start-jupyter.sh -# Set proper permissions for ROCm devices -RUN mkdir -p /etc/udev/rules.d && \ - echo 'SUBSYSTEM=="kfd", GROUP="video", MODE="0666"' > /etc/udev/rules.d/70-kfd.rules && \ - echo 'SUBSYSTEM=="dri", GROUP="video", MODE="0666"' > /etc/udev/rules.d/70-dri.rules - -# Create entrypoint script to set permissions and start services +# Create entrypoint script to start services RUN echo '#!/bin/bash' > /entrypoint.sh && \ - echo 'chmod 666 /dev/kfd 2>/dev/null || true' >> /entrypoint.sh && \ - echo 'chmod 666 /dev/dri/renderD* 2>/dev/null || true' >> /entrypoint.sh && \ echo 'export USER=jovyan' >> /entrypoint.sh && \ echo 'export SHELL=/bin/bash' >> /entrypoint.sh && \ echo 'exec python3 -m jupyterhub.singleuser --ip=0.0.0.0 --port=8888 "$@"' >> /entrypoint.sh && \ diff --git a/runtime/hub/core/spawner/kubernetes.py b/runtime/hub/core/spawner/kubernetes.py index e9d57fbd..dfe2e547 100644 --- a/runtime/hub/core/spawner/kubernetes.py +++ b/runtime/hub/core/spawner/kubernetes.py @@ -169,8 +169,8 @@ def configure_from_config(cls, config: HubConfig) -> None: # Extract code-server link protection settings cls.code_server_extra_trusted_domains = list(config.code_server.extraTrustedDomains) - async def get_user_resources(self) -> list[str]: - """Get available resources for the user based on their JupyterHub group memberships. + def _resolve_user_resources(self) -> list[str]: + """Resolve available resources for the current user from server-side policy. For auto-login/dummy modes, returns all configured resources. For all other users, resolves resources from JupyterHub groups @@ -195,6 +195,43 @@ async def get_user_resources(self) -> list[str]: self.log.debug(f"User '{username}' resolved resources: {available_resources}") return available_resources + async def get_user_resources(self) -> list[str]: + """Get available resources for the user based on their JupyterHub group memberships.""" + return self._resolve_user_resources() + + def _resolve_accelerator_selection(self, resource_type: str, gpu_selection: Any) -> str | None: + """Validate or default the accelerator selection for a resource.""" + requirements = self.resource_requirements[resource_type] + if gpu_selection is None: + selected_accelerator = "" + elif isinstance(gpu_selection, str): + selected_accelerator = gpu_selection.strip() + else: + raise RuntimeError("Accelerator selection must be a string") + + if "amd.com/gpu" not in requirements: + if selected_accelerator: + raise RuntimeError(f"CPU resource '{resource_type}' does not allow GPU selection") + return None + + resource_metadata = self._hub_config.get_resource_metadata(resource_type) if self._hub_config else None + allowed_accelerators = list(getattr(resource_metadata, "acceleratorKeys", []) or []) + if not allowed_accelerators: + raise RuntimeError(f"GPU resource '{resource_type}' has no authorized accelerators configured") + + if not selected_accelerator: + if len(allowed_accelerators) == 1: + selected_accelerator = allowed_accelerators[0] + else: + raise RuntimeError(f"GPU resource '{resource_type}' requires selecting an accelerator") + + if selected_accelerator not in allowed_accelerators: + raise RuntimeError(f"Accelerator '{selected_accelerator}' is not authorized for resource '{resource_type}'") + if selected_accelerator not in self.accelerator_options: + raise RuntimeError(f"Accelerator '{selected_accelerator}' is not configured") + + return selected_accelerator + async def options_form(self, _) -> str: """Generate the HTML form for resource selection. @@ -291,13 +328,17 @@ def options_from_form(self, formdata) -> dict[str, Any]: resource_type = resource_type_list[0] options["resource_type"] = resource_type - # Parse GPU selection if available - gpu_selection = formdata.get(f"gpu_selection_{resource_type}", [None])[0] - options["gpu_selection"] = gpu_selection - # Validate resource type if resource_type not in self.resource_images: raise RuntimeError(f"Unknown Resource: {resource_type}") + if resource_type not in self._resolve_user_resources(): + raise RuntimeError(f"Resource '{resource_type}' is not authorized for this user") + + gpu_selection = self._resolve_accelerator_selection( + resource_type, + formdata.get(f"gpu_selection_{resource_type}", [None])[0], + ) + options["gpu_selection"] = gpu_selection # Configure spawner based on selections self._configure_spawner(resource_type, gpu_selection) @@ -758,6 +799,7 @@ def _reset_per_spawn_state(self) -> None: "init_containers": copy.deepcopy(self.init_containers), "extra_container_config": copy.deepcopy(self.extra_container_config), "environment": copy.deepcopy(self.environment), + "supplemental_gids": copy.deepcopy(self.supplemental_gids), } for key, value in self._resource_baseline_state.items(): @@ -895,13 +937,25 @@ def _configure_spawner(self, resource_type: str, gpu_selection: str | None = Non async def start(self): """Start the spawner and schedule automatic shutdown.""" + runtime_minutes = self.user_options.get("runtime_minutes", 20) + resource_type = self.user_options.get("resource_type", "cpu") + if resource_type not in self.resource_images: + raise RuntimeError(f"Unknown Resource: {resource_type}") + if resource_type not in self._resolve_user_resources(): + raise RuntimeError(f"Resource '{resource_type}' is not authorized for this user") + gpu_selection = self._resolve_accelerator_selection( + resource_type, + self.user_options.get("gpu_selection"), + ) + self.user_options["gpu_selection"] = gpu_selection + self._configure_spawner(resource_type, gpu_selection) + # Ensure pod fails immediately (not retried) when an init container fails. # JupyterHub manages pod lifecycle; Kubernetes should not silently restart pods. - self.extra_pod_config = {"restartPolicy": "Never"} + extra_pod_config = copy.deepcopy(self.extra_pod_config or {}) + extra_pod_config["restartPolicy"] = "Never" + self.extra_pod_config = extra_pod_config - runtime_minutes = self.user_options.get("runtime_minutes", 20) - resource_type = self.user_options.get("resource_type", "cpu") - gpu_selection = self.user_options.get("gpu_selection", None) username = self.user.name.lower() # Determine accelerator type for quota calculation @@ -1099,7 +1153,6 @@ async def start(self): if hasattr(self, "_spawn_start_timestamp"): duration = time.time() - self._spawn_start_timestamp spawn_duration_seconds.observe(duration) - accelerator_type = self.user_options.get("gpu_selection") or "cpu" # active session count is derived from quota manager, not inc/dec except Exception: pass diff --git a/runtime/hub/tests/test_spawner_gpu_access.py b/runtime/hub/tests/test_spawner_gpu_access.py new file mode 100644 index 00000000..4ea6f476 --- /dev/null +++ b/runtime/hub/tests/test_spawner_gpu_access.py @@ -0,0 +1,173 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +import copy +import importlib.util +import sys +import types +from pathlib import Path +from unittest.mock import patch + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +CORE = ROOT / "core" + +if "core" not in sys.modules: + core_module = types.ModuleType("core") + core_module.__path__ = [str(CORE)] + sys.modules["core"] = core_module + + +def load_module(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +class DummyMetric: + def labels(self, **_kwargs): + return self + + def inc(self): + pass + + def observe(self, _value): + pass + + +class TestKubeSpawner: + def get_pod_manifest(self): + manifest = {"spec": copy.deepcopy(self.extra_pod_config or {})} + security_context = manifest["spec"].setdefault("securityContext", {}) + if self.fs_gid is not None: + security_context["fsGroup"] = self.fs_gid + if self.supplemental_gids: + security_context["supplementalGroups"] = list(self.supplemental_gids) + return manifest + + +def load_spawner_module(): + metrics_module = types.ModuleType("core.metrics") + for metric_name in ( + "pod_failure_total", + "repo_clone_failed_total", + "session_runtime_minutes", + "spawn_duration_seconds", + "spawn_failed_total", + "spawn_gpu_total", + ): + setattr(metrics_module, metric_name, DummyMetric()) + + jupyterhub_module = types.ModuleType("jupyterhub") + jupyterhub_module.__path__ = [] + user_module = types.ModuleType("jupyterhub.user") + user_module.User = type("User", (), {}) + kubespawner_module = types.ModuleType("kubespawner") + kubespawner_module.KubeSpawner = TestKubeSpawner + tornado_module = types.ModuleType("tornado") + web_module = types.ModuleType("tornado.web") + web_module.HTTPError = type("HTTPError", (Exception,), {}) + + with patch.dict( + sys.modules, + { + "core.metrics": metrics_module, + "jupyterhub": jupyterhub_module, + "jupyterhub.user": user_module, + "kubespawner": kubespawner_module, + "tornado": tornado_module, + "tornado.web": web_module, + }, + ): + return load_module("gpu_access_test_spawner", CORE / "spawner" / "kubernetes.py") + + +kubernetes = load_spawner_module() +RemoteLabKubeSpawner = kubernetes.RemoteLabKubeSpawner + + +class DummyLog: + def debug(self, _message): + pass + + +class ResourceMetadata: + acceleratorKeys = ["gpu-a"] + acceleratorOverrides = None + allowGitClone = False + defaultPath = None + env = {} + launchMode = None + + +class HubConfig: + def get_resource_metadata(self, _resource_type): + return ResourceMetadata() + + +def make_spawner(supplemental_gids: list[int] | None = None): + spawner = object.__new__(RemoteLabKubeSpawner) + spawner._hub_config = HubConfig() + spawner.resource_images = {"cpu": "cpu-image", "gpu": "gpu-image"} + spawner.resource_requirements = { + "cpu": {"cpu": "1", "memory": "1Gi"}, + "gpu": {"cpu": "1", "memory": "1Gi", "amd.com/gpu": "1"}, + } + spawner.accelerator_options = {"gpu-a": {}} + spawner.node_selector_mapping = {} + spawner.environment_mapping = {} + spawner.cmd = [] + spawner.args = [] + spawner.default_url = "" + spawner.node_affinity_required = [] + spawner.extra_resource_guarantees = {} + spawner.extra_resource_limits = {} + spawner.init_containers = [] + spawner.extra_container_config = {} + spawner.environment = {} + spawner.fs_gid = 100 + spawner.supplemental_gids = list(supplemental_gids or []) + spawner.extra_pod_config = {} + spawner.log = DummyLog() + spawner._resolve_user_resources = lambda: ["cpu", "gpu"] + return spawner + + +def test_gpu_pod_requests_accelerator_without_changing_generic_supplemental_groups(): + spawner = make_spawner(supplemental_gids=[1234]) + + spawner._configure_spawner("gpu", "gpu-a") + gpu_manifest = spawner.get_pod_manifest() + + assert spawner.extra_resource_guarantees == {"amd.com/gpu": "1"} + assert spawner.extra_resource_limits == {"amd.com/gpu": "1"} + assert gpu_manifest["spec"]["securityContext"] == {"fsGroup": 100, "supplementalGroups": [1234]} + + spawner._configure_spawner("cpu") + cpu_manifest = spawner.get_pod_manifest() + + assert spawner.extra_resource_guarantees == {} + assert spawner.extra_resource_limits == {} + assert cpu_manifest["spec"]["securityContext"] == {"fsGroup": 100, "supplementalGroups": [1234]} + + +def test_gpu_pod_without_generic_supplemental_groups_uses_storage_fs_group_only(): + spawner = make_spawner() + + spawner._configure_spawner("gpu", "gpu-a") + + assert spawner.extra_resource_guarantees == {"amd.com/gpu": "1"} + assert spawner.extra_resource_limits == {"amd.com/gpu": "1"} + assert spawner.get_pod_manifest()["spec"]["securityContext"] == {"fsGroup": 100} + + +def test_unauthorized_gpu_selection_is_rejected_before_spawner_configuration(): + spawner = make_spawner() + spawner._resolve_user_resources = lambda: ["cpu"] + spawner._configure_spawner = lambda *_args: pytest.fail("unauthorized resource configured the spawner") + + with pytest.raises(RuntimeError, match="not authorized"): + spawner.options_from_form({"runtime": ["20"], "resource_type": ["gpu"], "gpu_selection_gpu": ["gpu-a"]}) diff --git a/runtime/values-multi-nodes.yaml.example b/runtime/values-multi-nodes.yaml.example index 3db3cffa..4af6d259 100644 --- a/runtime/values-multi-nodes.yaml.example +++ b/runtime/values-multi-nodes.yaml.example @@ -22,7 +22,9 @@ # cp values-multi-nodes.yaml.example values-multi-nodes.yaml # # Prerequisites: -# - Install the AMD GPU device plugin and ROCm node labeller on GPU nodes. +# - The infrastructure owner must deploy and maintain the AMD GPU device plugin +# and ROCm node labeller outside AUPLC. Before Helm, run the readiness and +# capacity checks in deploy/README.md. # - Install an RWX-capable StorageClass for user homes; this example uses the # NFS provisioner from deploy/k8s/nfs-provisioner with class nfs-client. # - Create registry pull secrets only if you use private images. @@ -576,11 +578,9 @@ monitoring: enabled: false singleuser: - extraPodConfig: - securityContext: - fsGroup: 100 - supplementalGroups: - - 993 + # Storage ownership only. AUPLC runtime does not inject GPU groups. + # An amd.com/gpu request is the device-visibility boundary; injected nodes are 0666. + fsGid: 100 storage: dynamic: storageClass: nfs-client diff --git a/runtime/values.yaml b/runtime/values.yaml index da79243a..2a942e02 100644 --- a/runtime/values.yaml +++ b/runtime/values.yaml @@ -668,14 +668,9 @@ monitoring: enabled: false singleuser: - # Security context for user pods to access GPU devices - # supplementalGroups grants container access to host's render group (GID 993) - # This allows non-root users to access /dev/kfd and /dev/dri devices - extraPodConfig: - securityContext: - fsGroup: 100 - supplementalGroups: - - 993 # render group for ROCm GPU access + # Storage ownership only. AUPLC runtime does not inject GPU groups. + # amd.com/gpu requests allocate devices; host udev policy controls node modes. + fsGid: 100 storage: dynamic: diff --git a/skills/deploy-aup-learning-cloud/SKILL.md b/skills/deploy-aup-learning-cloud/SKILL.md index 6946dca6..cc73af5f 100644 --- a/skills/deploy-aup-learning-cloud/SKILL.md +++ b/skills/deploy-aup-learning-cloud/SKILL.md @@ -1,253 +1,135 @@ --- name: deploy-aup-learning-cloud description: >- - Group: Plan & deploy AUP Learning Cloud. Deploys AUP Learning Cloud (a - multi-node JupyterHub-on-k3s platform for AMD - GPUs) onto physical hardware end to end. Use when the user wants to install, - deploy, set up, or stand up AUP Learning Cloud, AUPLC, or "the learning - cloud" on a cluster; mentions a multi-AIPC or 3-node mini-cluster, PXE / - netboot / diskless agents, the Ansible inventory.yml, pb-pxe-controller, - pb-k3s-site, the ROCm GPU device plugin/labeller, an NFS provisioner, or a - JupyterHub values.yaml / Helm chart for this project. Covers both the - PXE-diskless topology and the SSH-preinstalled multi-node topology. Do not - use for the single-node "./auplc-installer install" flow, for building - notebook images, or for non-AUPLC JupyterHub or k3s installs. + Group: Plan and deploy AUP Learning Cloud. Use when the user wants to install + the multi-node JupyterHub-on-k3s platform on physical hardware through either + PXE-diskless or SSH-preinstalled nodes. Do not use for the single-node + ./auplc-installer flow, notebook image builds, or unrelated JupyterHub and + k3s installations. --- # Deploy AUP Learning Cloud -Stand up AUP Learning Cloud on a multi-node k3s cluster: build the cluster with -Ansible, expose AMD GPUs, provide shared storage, and deploy the JupyterHub -chart with Helm so users can log in and spawn GPU notebooks. +Stand up a multi-node AUP Learning Cloud cluster with Ansible, AMD GPU access, +shared storage, and the JupyterHub Helm chart. -This skill is written for any coding agent. Run the commands and edit the files -as described; the full, copy-runnable command sequence and the troubleshooting -table live in **[reference.md](reference.md)**. +Use the [skill scripts guide](scripts/README.md) as the source of truth for the +complete generator-first command sequences and generated files. Use +[deploy/README.md](../../deploy/README.md) for the human direct-edit workflow, +operational background, and troubleshooting. This skill defines the interview +and safety gates around the generated procedure. ## Prerequisites -- A checkout of `aup-learning-cloud` on the operator/service machine. -- The service machine runs Ubuntu 24.04 with a reserved/static IP and internet - access. -- `ansible` on the operator machine; `kubectl` and `helm` for the cluster - (reference.md has the Helm install command). -- For GPU scheduling: AMD GPU nodes with a working in-kernel NIC driver. -- The user supplies the physical hardware. **No site values (IPs, subnet, SSH - keys, tokens) ship in the repo** — this skill generates them. +- A checkout of `aup-learning-cloud` on the operator machine. +- Ubuntu 24.04, a reserved controller IP, internet access, and Ansible. +- Physical node, network, storage, and authentication details from the user. +- Passwordless root SSH to every managed host in the SSH topology. -## Helper script paths +Site values and secrets don't ship in the repository. Generate them locally +and never put tokens, private keys, or credentials in tracked files. -Resolve the deploy helpers before running the commands below. From any directory -in an AUP Learning Cloud checkout: +## Phase 1: Interview -```bash -REPO_ROOT="$(git rev-parse --show-toplevel)" -DEPLOY_SCRIPTS="$REPO_ROOT/skills/deploy-aup-learning-cloud/scripts" -``` +Ask for an explicit topology choice before collecting other details or touching +machines. Never infer the choice from the hardware. -When this skill is installed as a plugin rather than used from a checkout, set -`DEPLOY_SKILL_DIR` to the absolute directory containing the loaded `SKILL.md`, -then derive the helpers from that directory: - -```bash -DEPLOY_SKILL_DIR="/absolute/path/to/deploy-aup-learning-cloud" -DEPLOY_SCRIPTS="$DEPLOY_SKILL_DIR/scripts" -``` - -## Phase 1 — Interview - -Work through this in order. **The deployment-method choice (1a) is a hard gate: -ask it first and get an explicit answer before collecting anything else or -touching the machines.** - -### Phase 1a — Choose the deployment method (ask first, always) - -Ask the user to pick one. **Never assume or auto-select** — even when the -machines "look like" one case, present both options and let the user decide (you -may recommend, but you still need an explicit choice before continuing): - -| Choose | When | +| Choice | Use when | | --- | --- | -| **PXE Diskless Netboot** (`topology: pxe-diskless`) — one service machine netboots diskless agents | Agents have no OS installed; you want zero per-machine install; small teaching lab. This is the [3-node mini-cluster guide](https://amdresearch.github.io/aup-learning-cloud/installation/multi-node/multi-aipc-hardware-deployment.html). | -| **Multi Node SSH Installation** (`topology: ssh-preinstalled`) — every node already runs Ubuntu | Each node has an OS and is reachable over SSH; closer to a long-running lab. This is the [multi-node guide](https://amdresearch.github.io/aup-learning-cloud/installation/multi-node.html). | - -The value in parentheses is the `topology` field for `gen_configs.py` (Phase 3) -and selects the matching section in [reference.md](reference.md). - -### Phase 1b — Collect the rest (some items branch on the choice above) - -Collect, and confirm back to the user, before touching anything: - -1. **Courses** wanted — drives the `values.yaml` course keys + team mappings - (full catalog setup lives in `configure-aup-learning-cloud-courses`). -2. **Node count** and which node is the controller/server, plus its static IP. - - *SSH path only:* also the hostname + IP of every agent node, and confirm - passwordless root SSH already reaches each one. -3. **GPU — do not ask the user to name the model.** Let the tooling find it: the - detectors report the GPUs (`$DEPLOY_SCRIPTS/detect_hardware.sh` in Phase 2) - and the real ROCm `amd.com/gpu.product-name` label - (`$DEPLOY_SCRIPTS/detect_cluster.sh` in Phase 5). Then - **confirm the detected GPU → accelerator-key mapping with the user** before it - goes into the values file. -4. *PXE path only:* service-machine NIC, subnet (CIDR), gateway, and DNS servers - (also auto-detected in Phase 2 and cross-checked), plus at least one SSH - public key for the rootfs and the apache web port. - -Login mode (`custom.authMode`) is unchanged — it stays at its `auto-login` -default; switch it later with `configure-aup-learning-cloud-auth` if needed. The -detailed steps for both paths are in [reference.md](reference.md). - -## Phase 2 — Discover - -On the service machine, run the bundled detector and cross-check its JSON -against the Phase 1 answers: - -```bash -"$DEPLOY_SCRIPTS/detect_hardware.sh" # JSON: nic, ip, subnet_cidr, gateway, dns_servers, gpus[] -``` - -It reports the default-route NIC, the service-machine IP + subnet CIDR, the -gateway, DNS servers, and each AMD GPU (`lspci`, vendor `1002`) with the bound -`kernel_driver`. If a GPU's `kernel_driver` is empty, note its module for -`pxe_initramfs_modules` (PXE path only). Empty fields come back in `warnings` -so you know exactly what to ask the operator for. The detected GPUs are the -source of truth for the accelerator mapping — Phase 1 does not ask the user to -name them, so surface the detected list and confirm it with the user. - -## Phase 3 — Generate config - -Drive `$DEPLOY_SCRIPTS/gen_configs.py` rather than hand-writing YAML — it keeps the -three artifacts consistent, mints the k3s token locally with a CSPRNG (never -printed), `chmod 600`s the inventory, and pins `pxe_k3s_version == k3s_version`. - -```bash -python3 "$DEPLOY_SCRIPTS/gen_configs.py" --print-schema > spec.json # fill from Phase 1 + 2 -GENERATED_DIR="$REPO_ROOT/generated" -python3 "$DEPLOY_SCRIPTS/gen_configs.py" --spec spec.json --out-dir "$GENERATED_DIR" -``` - -It writes, into `--out-dir`: - -1. `inventory.yml` — `server` host + `token` + `k3s_version` (agents empty for - PXE; listed for SSH) plus the `pxe_controller` group for PXE. -2. `pb-pxe-controller.vars.yml` — PXE path only: extra vars passed to - `deploy/ansible/playbooks/pb-pxe-controller.yml` with `-e @` (`pxe_network_interface`, - `pxe_subnet`, `pxe_gateway`, `pxe_dns_servers`, `pxe_controller_ip`, - `pxe_k3s_server_ips`, `pxe_k3s_version`, `pxe_web_port`, - `pxe_rootfs_password`, `pxe_rootfs_authorized_keys`). -3. `values-basic-example.yaml` — `custom.accelerators.*.nodeSelector` (matched - to real GPU labels in Phase 5), `custom.resources.images`, the storage class - (`nfs-client`), `custom.authMode`, and the proxy `NodePort` (e.g. 30890). - -Review the artifacts, install the inventory and runtime overlay into the -checkout, and keep the PXE vars in the generated directory. -**Never commit `inventory.yml` — it holds the token.** Field-by-field guidance -is in [reference.md](reference.md). - -Map the generated artifacts into the checkout before Phase 5 validation: - -```bash -install -m 0600 "$GENERATED_DIR/inventory.yml" "$REPO_ROOT/deploy/ansible/inventory.yml" -install -m 0644 "$GENERATED_DIR/values-basic-example.yaml" "$REPO_ROOT/runtime/values-basic-example.yaml" - -# PXE only: keep this generated secret in place and use its absolute path. -PXE_VARS="$(realpath "$GENERATED_DIR/pb-pxe-controller.vars.yml")" -chmod 0600 "$PXE_VARS" -``` - -The generated `gpu.acceleratorKeys` activates the selected accelerators only -for the generic GPU resource. Wire selected accelerators into course resources -separately with `configure-aup-learning-cloud-courses`. - -## Phase 4 — Execute (with confirmation gates) - -Run the install in order. **Pause for explicit user confirmation before each -risky/irreversible step** (see Safety). The PXE path is, in brief: - -1. Install host packages on the service machine. -2. Run `pb-pxe-controller.yml -e @"$PXE_VARS"` to build the PXE/NFS rootfs, - then verify the - controller (dnsmasq, NFS, apache2, TFTP boot files). -3. `pb-base.yml` + `pb-k3s-site.yml` to install the single-node k3s server. -4. Publish the k3s token + kubeconfig for agents over the apache `/k3s/` endpoint. -5. Netboot the agents; watch them auto-join with `kubectl get nodes -o wide`. - -Run the PXE controller step with the generated vars file: - -```bash -cd "$REPO_ROOT/deploy/ansible" -ansible-playbook -i inventory.yml playbooks/pb-pxe-controller.yml -e @"$PXE_VARS" -``` - -The SSH path runs `pb-base.yml`, `pb-k3s-site.yml`, and `pb-rocm.yml` against -the inventory instead. Full commands for both paths are in [reference.md](reference.md). - -## Phase 5 — GPU, storage, and chart - -1. Install the AMD GPU device plugin + ROCm labeller, then read the **real** - cluster state: - - ```bash -"$DEPLOY_SCRIPTS/detect_cluster.sh" > cluster.json # nodes[], gpu_product_names[], storage_classes[] - ``` - - Confirm the detected GPU → accelerator-key mapping with the user, then patch - `custom.accelerators.*.nodeSelector` so each `amd.com/gpu.product-name` - matches a value in `gpu_product_names`. Gate the install on a clean - pre-flight (exits non-zero on any mismatch): - - ```bash -# Set this to the topology selected in Phase 1a. -DEPLOY_TOPOLOGY=pxe-diskless -python3 "$DEPLOY_SCRIPTS/validate.py" --repo "$REPO_ROOT" --topology "$DEPLOY_TOPOLOGY" \ - --values runtime/values.yaml --values runtime/values-basic-example.yaml \ - --pxe-vars "$PXE_VARS" --cluster cluster.json --helm-dry-run -``` - -For the PXE path, the validator and Ansible receive the same generated vars -file. Omit `--pxe-vars "$PXE_VARS"` for the SSH path. - -2. Create the notebook-PVC NFS export and install the `nfs-subdir-external-provisioner` - (storage class `nfs-client`). -3. Deploy the chart: - -```bash -helm upgrade --install jupyterhub ./runtime/chart \ - --namespace jupyterhub --create-namespace \ - -f runtime/values.yaml \ - -f runtime/values-basic-example.yaml -``` - -## Phase 6 — Validate end to end - -```bash -kubectl get nodes -o wide # server + agents Ready -kubectl get pods -A # nothing CrashLoopBackOff/Pending/ImagePullBackOff -kubectl get storageclass # nfs-client present -``` - -Then open the Hub (NodePort example: `http://:30890`), log in, -spawn a CPU notebook, confirm file persistence across a restart, then spawn a -GPU notebook and confirm its pod lands on a GPU node -(`kubectl get pods -n jupyterhub -o wide`). +| **PXE Diskless Netboot** (`pxe-diskless`) | A controller netboots diskless agents. | +| **Multi Node SSH Installation** (`ssh-preinstalled`) | Every node already runs Ubuntu and accepts root SSH. | + +Then collect and confirm: + +1. Courses and notebook resources. +2. Controller hostname, static IP, subnet, gateway, and DNS. +3. For SSH, every managed hostname and IP. Don't ask for a GPU host list; + generation discovers GPU hosts over SSH. +4. For PXE, the controller NIC, web port, rootfs SSH public key, and whether + diskless agents have AMD GPUs. This explicit yes or no is the sole PXE GPU + policy input because agent hardware can't be inferred from the controller. +5. Shared storage location and the Hub access method. + +Confirm detected GPU product labels before mapping them to accelerator keys in +the runtime values. + +## Phase 2: Generate + +Create a fresh schema and fill only its current fields. Run the generator rather +than writing inventory or GPU policy by hand. + +For SSH, generation performs read-only discovery on every managed host and +publishes canonical artifacts after GPU evidence is consistent. + +For PXE, generation writes the canonical inventory, PXE vars, runtime overlay, +and GPU resolution report directly as desired deployment inputs. Their existence +does not prove rootfs provisioning succeeded. Review, install, and validate those +files, then run the controller playbook with the canonical inventory and PXE +vars; the playbook must complete successfully before proceeding. + +Follow the complete topology command sequence in the +[skill scripts guide](scripts/README.md). Don't substitute the human direct-edit +SSH workflow from `deploy/README.md`; the skill's SSH path remains +generator-first and discovers GPU policy from managed-host evidence. + +## Phase 3: Validate and execute + +Install the canonical generated inventory and runtime overlay into the checkout, +then run the topology's exact validator command from the +[skill scripts guide](scripts/README.md). The validator inputs are: + +- `--repo` +- `--topology` +- `--inventory` to validate generated host booleans +- `--gpu-resolution` with `--inventory` for generated-artifact consistency +- both `--values` files +- `--pxe-vars` for PXE only + +A human direct inventory can be validated by itself with unquoted `auto`, +`true`, or `false`. A resolution report requires an inventory, and that pairing +accepts only generated boolean values. Supply both in this generator-first +workflow so the validator checks their consistency. The skill resolves every +host to `true` or `false` and never generates `auto`. + +Stop on validation failure. After a clean result, continue with the topology's +Ansible, device plugin, and Helm commands in the skill scripts guide. Treat the +AMD device plugin and ROCm node labeller as infrastructure prerequisites owned +outside AUPLC. Verify both existing DaemonSets and advertised GPU capacity +before Helm; do not install these privileged components as part of the AUPLC +procedure. + +Keep the GPU contract distinct from storage configuration. The installer, +Ansible role, and PXE controller install AMD's +`amdgpu-insecure-instinct-udev-rules` package at the pinned version +`30.30.4.0-2341068.24.04`. Its rule sets mode `0666` only on `/dev/kfd` and DRM +`renderD*` nodes. It does not change `card*`, which retains normal system policy, +observed as `root:video 0660`. + +Device-plugin allocation is a separate visibility layer. Only `amd.com/gpu` +requests receive allocated GPU devices, and the plugin does not change Unix +inode permissions. AUPLC Hub adds no GPU supplemental group; none is required +for the tested ROCm compute path. `singleuser.fsGid: 100` is for shared storage +only. + +## Phase 4: Verify + +Check that all expected nodes are Ready, the GPU labels and allocatable resources +match the generated policy, the storage class is available, and JupyterHub pods +are healthy. Open the Hub, start a CPU notebook, verify persistence, then start a +GPU notebook and confirm it schedules on a GPU node. ## Safety -These steps are destructive or hard to reverse — **stop and get explicit user -confirmation before each one**, and never run them silently: - -- Building/rebuilding the PXE rootfs (`pxe_rootfs_force_rebuild: true`). -- Editing `/etc/exports` and restarting `nfs-kernel-server`. -- `kubectl delete node ` (debugging only). -- `helm uninstall` or a cluster reset (`pb-k3s-reset.yml`). -- Changing firmware boot order / disabling Secure Boot on agents. +Pause for explicit user confirmation before rebuilding a PXE rootfs, changing +NFS exports, changing firmware boot settings, resetting a cluster, deleting a +node, or uninstalling a Helm release. -Never commit or push. Never write the k3s token, OAuth secrets, or SSH private -keys into tracked files. Preserve the four AUP Learning Cloud attribution -layers (see the project `AGENTS.md`) if any chart/Hub source is touched. +Never commit or push deployment secrets. Preserve the four AUP Learning Cloud +attribution layers described in the project `AGENTS.md` if Hub or chart sources +are changed. ## Reference -Full step-by-step commands for both topologies, the GPU-label-to-accelerator -mapping, the `values.yaml` field guide, and the troubleshooting table: -[reference.md](reference.md). +- [Complete skill command sequences](scripts/README.md) +- [Human deployment and troubleshooting](../../deploy/README.md) +- [Skill-specific summary](reference.md) diff --git a/skills/deploy-aup-learning-cloud/reference.md b/skills/deploy-aup-learning-cloud/reference.md index e67faf02..b1da7733 100644 --- a/skills/deploy-aup-learning-cloud/reference.md +++ b/skills/deploy-aup-learning-cloud/reference.md @@ -1,442 +1,74 @@ -# Deploy AUP Learning Cloud — Reference +# Deploy AUP Learning Cloud Reference -Full, copy-runnable commands for both deployment topologies, the GPU label -mapping, the `values.yaml` field guide, and the troubleshooting table. The -workflow and confirmation gates are in [SKILL.md](SKILL.md). +The complete generator-first command sequences and generated file list live in +the [skill scripts guide](scripts/README.md). Human direct-edit deployment, +operational background, and failure guidance live in +[deploy/README.md](../../deploy/README.md). Don't copy those commands into this +reference. -## Contents +## Topology contract -- [Source guides](#source-guides) -- [PXE-diskless topology (3-node mini-cluster)](#pxe-diskless-topology-3-node-mini-cluster) -- [SSH-preinstalled topology (standard multi-node)](#ssh-preinstalled-topology-standard-multi-node) -- [GPU label to accelerator key](#gpu-label-to-accelerator-key) -- [values.yaml field guide](#valuesyaml-field-guide) -- [Troubleshooting](#troubleshooting) - -## Source guides - -- 3-node mini-cluster (PXE diskless): -- Standard multi-node (SSH): - -Treat the live docs as the source of truth for version pins; this file -condenses the opinionated path. - -The helper commands are resolved through `DEPLOY_SCRIPTS` as defined in -[SKILL.md](SKILL.md#helper-script-paths), not through a checkout-root -`scripts/` directory. - -The two topology sections below are the two branches of the Phase 1a gate in -[SKILL.md](SKILL.md): **PXE Diskless Netboot** (`topology: pxe-diskless`) → -[PXE-diskless topology](#pxe-diskless-topology-3-node-mini-cluster); **Multi Node -SSH Installation** (`topology: ssh-preinstalled`) → -[SSH-preinstalled topology](#ssh-preinstalled-topology-standard-multi-node). - -## PXE-diskless topology (3-node mini-cluster) - -One service machine (AIPC 1) runs the PXE controller, the single-node k3s -server, NFS, and the apache k3s-credential endpoint. The other machines are -diskless agents that netboot and auto-join. Only AIPC 1 is Ansible-managed. - -### Step 1 — Prepare the service machine - -```bash -sudo apt update -sudo apt install -y git ansible curl ca-certificates jq \ - dnsmasq pxelinux syslinux-common apache2 \ - nfs-kernel-server debootstrap \ - grub-efi-amd64-signed shim-signed - -ip -br addr # record the NIC and IP -ip route # record the gateway -``` - -Give the local `root` a passwordless SSH login (or add `ansible_connection: -local` to the host vars to skip SSH entirely): - -```bash -sudo install -d -m 0700 /root/.ssh -sudo tee -a /root/.ssh/authorized_keys < ~/.ssh/id_ed25519.pub >/dev/null -sudo chmod 0600 /root/.ssh/authorized_keys -ssh root@ true && echo root-ssh-ok -``` - -### Step 2 — Configure the inventory - -Edit `deploy/ansible/inventory.yml`. AIPC 1 is the only host; the `agent` group -stays empty (netboot agents are not Ansible-managed). Generate the token with -`openssl rand -base64 64` and keep it out of chat/VCS. - -```yaml -k3s_cluster: - children: - server: - hosts: - aipc1: - ansible_host: - agent: - hosts: {} # diskless netboot agents auto-join; do NOT list them here - vars: - ansible_user: root - k3s_version: v1.32.3+k3s1 - token: "" # openssl rand -base64 64 - api_endpoint: "{{ hostvars[groups['server'][0]]['ansible_host'] | default(groups['server'][0]) }}" - -pxe_controller: - hosts: - aipc1: - ansible_host: - vars: - ansible_port: 22 - ansible_user: root -``` - -### Step 3 — Prepare the generated PXE controller vars - -The network, controller, server-IP, and SSH-key values are empty by default and -the role asserts on them. `$DEPLOY_SCRIPTS/gen_configs.py` writes these values -to `generated/pb-pxe-controller.vars.yml`. Keep that file at mode `0600`; it can -contain `pxe_rootfs_password`. Resolve its absolute path for the Ansible and -validator commands instead of copying or merging it into the playbook: - -```bash -PXE_VARS="$(realpath ./generated/pb-pxe-controller.vars.yml)" -chmod 0600 "$PXE_VARS" -test "$(stat -c '%a' "$PXE_VARS")" = 600 -``` - -Review the generated values before the first run: - -```yaml -pxe_rootfs_force_rebuild: true # true for the first build (RISKY: rebuilds rootfs) -pxe_network_interface: "enp1s0" # service-machine NIC (Step 1) -pxe_subnet: "192.168.1.0/24" # node subnet, CIDR -pxe_gateway: "192.168.1.1" # default gateway (informational) -pxe_dns_servers: "8.8.8.8,8.8.4.4" -pxe_controller_ip: "192.168.1.10" # this service machine's IP -pxe_k3s_server_ips: - - "192.168.1.10" -pxe_k3s_version: "v1.32.3+k3s1" # MUST match inventory k3s_version -pxe_web_port: 8080 # apache port for the k3s token/kubeconfig (not 80) -pxe_rootfs_password: "" # optional; empty disables password login (use ansible-vault if set) -pxe_rootfs_authorized_keys: - - "ssh-ed25519 AAAA... you@host" # at least one key required -``` - -Set `pxe_rootfs_force_rebuild: false` after the first stable build so you do -not rebuild the rootfs under running agents. The playbook also exposes -`pxe_apt_mirror`, `pxe_rootfs_packages`, and `pxe_initramfs_modules` (add your -NIC module here if it lacks an in-kernel driver) — leave these at their defaults -unless discovery flagged a need. - -### Step 4 — Run the PXE controller playbook - -```bash -cd ~/aup-learning-cloud -REPO_ROOT="$(pwd)" -DEPLOY_SCRIPTS="$REPO_ROOT/skills/deploy-aup-learning-cloud/scripts" -PXE_VARS="$(realpath "$REPO_ROOT/generated/pb-pxe-controller.vars.yml")" -python3 "$DEPLOY_SCRIPTS/validate.py" --repo "$REPO_ROOT" \ - --topology pxe-diskless --pxe-vars "$PXE_VARS" \ - --values runtime/values.yaml --values runtime/values-basic-example.yaml -cd "$REPO_ROOT/deploy/ansible" -ansible-playbook -i inventory.yml playbooks/pb-pxe-controller.yml -e @"$PXE_VARS" -``` - -### Step 5 — Verify the controller - -```bash -systemctl is-active dnsmasq nfs-kernel-server apache2 -showmount -e localhost -ls -l /srv/tftp/pxelinux.0 /srv/tftp/grubnetx64.efi /srv/tftp/vmlinuz /srv/tftp/initrd.img -curl -I http://127.0.0.1:8080/k3s/ # 403 expected (dir exists, empty) -``` - -The `/k3s/` endpoint is served on port 8080 (k3s owns 80/443 for ingress). - -### Step 6 — Install the single-node k3s server - -Run **without** `sudo` (key-based root SSH already connects as root): - -```bash -cd ~/aup-learning-cloud/deploy/ansible -ansible-playbook -i inventory.yml playbooks/pb-base.yml -ansible-playbook -i inventory.yml playbooks/pb-k3s-site.yml -export KUBECONFIG=~/.kube/config # add to ~/.bashrc to persist -kubectl get nodes -o wide -``` - -### Step 7 — Publish k3s credentials for agents - -```bash -sudo install -d -m 0755 /var/www/html/k3s -sudo install -m 0644 /var/lib/rancher/k3s/server/token /var/www/html/k3s/token -sudo sed "s#https://127.0.0.1:6443#https://:6443#g" \ - /etc/rancher/k3s/k3s.yaml | sudo tee /var/www/html/k3s/kubeconfig >/dev/null -sudo chmod 0644 /var/www/html/k3s/token /var/www/html/k3s/kubeconfig -sudo systemctl reload apache2 - -curl -fsS http://127.0.0.1:8080/k3s/token >/dev/null && echo token-ok -curl -fsS http://127.0.0.1:8080/k3s/kubeconfig >/dev/null && echo kubeconfig-ok -``` - -### Step 8 — Netboot the agents - -On each agent: disable Secure Boot, enable network boot, and put PXE before the -local disk in the firmware boot order. Boot, then watch them register: - -```bash -watch kubectl get nodes -o wide -``` - -Agents appear as `agent-` nodes and become `Ready`. - -### Step 9 — Validate agent persistence - -Reboot one agent; confirm it rejoins with the same identity. On the agent: - -```bash -mount | grep /var/lib/rancher/k3s -test -f /var/lib/rancher/k3s/node-password && echo node-password-ok -systemctl status mount-local-disk k3s-agent --no-pager -``` - -`kubectl delete node ` clears a stale node object — **debugging only**, -confirm with the user first. - -Continue with [Step 10 (GPU)](#step-10--amd-gpu-device-plugin-and-labeller). - -## SSH-preinstalled topology (standard multi-node) - -Every node already runs Ubuntu 24.04 and is reachable over passwordless SSH. - -### Prepare SSH and inventory - -Helper scripts in `deploy/scripts/` enable root SSH and distribute kubeconfig: - -```bash -./deploy/scripts/edit_sshd.sh -./deploy/scripts/setup_ssh_root_access.sh -./deploy/scripts/deploy-kubeconfig.sh -``` - -Edit `deploy/ansible/inventory.yml` — list every node under `server`/`agent`: - -```yaml -k3s_cluster: - children: - server: - hosts: - : - agent: - hosts: - : - : - vars: - ansible_port: 22 - ansible_user: root - k3s_version: v1.32.3+k3s1 - token: "" # openssl rand -base64 64 - api_endpoint: "{{ hostvars[groups['server'][0]]['ansible_host'] | default(groups['server'][0]) }}" -``` - -### Build the cluster - -```bash -cd deploy/ansible -sudo ansible-playbook playbooks/pb-base.yml # base OS / packages -sudo ansible-playbook playbooks/pb-k3s-site.yml # deploy k3s -sudo ansible-playbook playbooks/pb-rocm.yml # ROCm on GPU nodes -``` - -Related: `pb-k3s-upgrade.yml` (upgrade), `pb-k3s-reset.yml` (reset — RISKY). -Then install `kubectl`/`helm` on the operator machine (see Helm command below) -and continue with [Step 10 (GPU)](#step-10--amd-gpu-device-plugin-and-labeller). - -### Install Helm - -```bash -wget https://get.helm.sh/helm-v3.17.2-linux-amd64.tar.gz -O /tmp/helm.tar.gz -cd /tmp && tar -zxvf helm.tar.gz -sudo mv /tmp/linux-amd64/helm /usr/local/bin/helm -``` - -## Step 10 — AMD GPU device plugin and labeller - -```bash -kubectl create -f https://raw.githubusercontent.com/ROCm/k8s-device-plugin/master/k8s-ds-amdgpu-dp.yaml -kubectl create -f https://raw.githubusercontent.com/ROCm/k8s-device-plugin/master/k8s-ds-amdgpu-labeller.yaml - -kubectl get pods -A | grep -i amd -kubectl describe node | grep amd.com/gpu -``` - -Use the labels that actually appear. Common keys: -`amd.com/gpu.product-name`, `amd.com/gpu.family`, `amd.com/gpu.device-id`. - -## Step 11 — Shared NFS storage for notebook PVCs - -This is separate from the PXE rootfs export. Append the export directly to -`/etc/exports` (on Ubuntu 24.04 `/etc/exports.d/*.conf` is ignored): - -```bash -sudo mkdir -p -sudo chown -R nobody:nogroup -sudo chmod 0777 -echo " (rw,sync,no_subtree_check,no_root_squash,insecure)" | sudo tee -a /etc/exports -sudo exportfs -ra -sudo systemctl restart nfs-kernel-server -showmount -e localhost -``` - -Install the provisioner (storage class `nfs-client`): - -```bash -cd ~/aup-learning-cloud -cp deploy/k8s/nfs-provisioner/values.yaml deploy/k8s/nfs-provisioner/values.local.yaml -# edit values.local.yaml: nfs.server, nfs.path, storageClass.name = nfs-client -helm repo add nfs-subdir-external-provisioner https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner/ -helm repo update -helm upgrade --install nfs-subdir-external-provisioner \ - nfs-subdir-external-provisioner/nfs-subdir-external-provisioner \ - --namespace nfs-provisioner --create-namespace \ - -f deploy/k8s/nfs-provisioner/values.local.yaml -kubectl get storageclass -``` - -## Step 12 — Configure JupyterHub values - -The generated `runtime/values-basic-example.yaml` is the canonical deployment -overlay. Review and keep it when Phase 3 generated one. Only when no generated -overlay exists, start a manual overlay from the example: - -```bash -cd ~/aup-learning-cloud/runtime -if [ ! -e values-basic-example.yaml ]; then - cp values-multi-nodes.yaml.example values-basic-example.yaml -fi -``` - -Minimum edits (see the [field guide](#valuesyaml-field-guide)): - -```yaml -custom: - authMode: "auto-login" # single-machine default; avoid "dummy" (login 404s) - accelerators: - strix-halo: - nodeSelector: - amd.com/gpu.product-name: "" # from Step 10 - quotaRate: 3 - resources: - images: - cpu: "" - gpu: "" -hub: - db: - pvc: - storageClassName: nfs-client -singleuser: - storage: - dynamic: - storageClass: nfs-client -proxy: - service: - type: NodePort - nodePorts: - http: 30890 -``` - -## Step 13 — Deploy AUP Learning Cloud - -```bash -cd ~/aup-learning-cloud -helm upgrade --install jupyterhub ./runtime/chart \ - --namespace jupyterhub --create-namespace \ - -f runtime/values.yaml \ - -f runtime/values-basic-example.yaml - -kubectl get pods -n jupyterhub -o wide -kubectl get svc -n jupyterhub -``` - -For later config changes, re-run the same `helm upgrade --install`. - -## Step 14 — End-to-end validation - -```bash -kubectl get nodes -o wide -kubectl get pods -A -kubectl get storageclass -kubectl describe node | grep amd.com/gpu -``` - -Then browse to `http://:30890` (or your ingress host), log in, -spawn a CPU notebook, create a file, restart and confirm it persists, then -spawn a GPU notebook and confirm its pod lands on a GPU node. - -## GPU label to accelerator key - -The chart's accelerator catalog (`runtime/values.yaml`) is keyed by accelerator -names; map the ROCm labeller's `amd.com/gpu.product-name` to the right key. The -GPU is auto-detected (Phase 2 and Phase 5), not named by the user in the -interview — use this table to confirm the detected product-name → key mapping -with the user. Verify against the live values file — product names can normalize -differently per fleet. - -| `amd.com/gpu.product-name` (example) | Accelerator key | +| Topology | Generator behavior | | --- | --- | -| `AMD_Radeon_780M_Graphics` | `phx` | -| `AMD_Radeon_890M_Graphics` | `strix` | -| `AMD_Radeon_8060S_Graphics` | `strix-halo` | -| `AMD_Radeon_RX_9070_XT` | `9070xt` | -| `AMD_Radeon_AI_PRO_R9700` | `r9700` | -| `AMD_Radeon_RX_9600_GRE` | `9600gre` | - -If your labeller reports a different product name, update the matching -`custom.accelerators.*.nodeSelector` entry to that exact string. - -## values.yaml field guide - -Sections to review in the generated `values-basic-example.yaml`, or in the -manual `values-multi-nodes.yaml.example` copy when generation was not used: - -| Field | Purpose | -| --- | --- | -| `custom.authMode` | `auto-login` for the single-machine example; OAuth modes for real auth | -| `custom.githubOrgName`, `hub.config.GitHubOAuthenticator` | GitHub OAuth (when not auto-login) | -| `custom.adminUser` | Hub admin | -| `custom.accelerators.*.nodeSelector` | Must match real `amd.com/gpu.*` labels | -| `custom.resources.images` | CPU/GPU/course notebook images | -| `custom.resources.requirements`, `custom.teams.mapping`, `custom.quota` | Per-team resources and quotas | -| `hub.db.pvc.storageClassName`, `singleuser.storage.dynamic.storageClass` | `nfs-client` for multi-node | -| `proxy.service`, `ingress` | NodePort (e.g. 30890) or ingress host | - -## Troubleshooting - -| Symptom | Likely cause | First checks | -| --- | --- | --- | -| Playbook fails immediately on an assert | A required PXE var is empty | Re-check `pxe_controller_ip`, `pxe_subnet`, `pxe_network_interface`, `pxe_dns_servers`, `pxe_k3s_server_ips`, and at least one SSH key | -| Agent never shows the PXE menu | Firmware boot order, network boot disabled, or Proxy-DHCP not reaching the client | Firmware, switch port, `systemctl status dnsmasq`, `journalctl -u dnsmasq` | -| Agent gets an IP but cannot load boot files | TFTP blocked, missing files, or Secure Boot still on | `/srv/tftp`, firewall, Secure Boot disabled, `dnsmasq` logs | -| Agent has no network during netboot | NIC has no in-kernel driver in the initramfs | `lspci -nnk`, add the module to `pxe_initramfs_modules`, rebuild rootfs | -| Agent kernel boots but cannot mount rootfs | NFS export, subnet ACL, or wrong `pxe_controller_ip` | `showmount -e `, `/etc/exports`, rootfs kernel args | -| Agent waits for the k3s token | Token not published or apache ACL blocks the subnet | `curl http://:8080/k3s/token`, apache config | -| Agent joins once but fails after reboot | Missing local k3s persistence or lost node password | `mount-local-disk`, `/var/lib/rancher/k3s/node-password`, `k3s-agent` logs | -| Agent fails to join with a version error | Agent rootfs k3s newer than the server | Align `pxe_k3s_version` with `k3s_version`, rebuild rootfs | -| Agent node does not join (SSH path) | Hostname resolution, token, or `api_endpoint` mismatch | `systemctl status k3s-agent`, `journalctl -u k3s-agent`, `/etc/hosts` | -| GPU notebook stays Pending | Chart `nodeSelector` mismatch or GPUs exhausted | `kubectl describe pod -n jupyterhub`, node labels | -| PVC stays Pending | StorageClass name mismatch or NFS provisioner cannot mount | `kubectl get storageclass`, provisioner logs, NFS export | -| `kubectl` permission denied on `k3s.yaml` | kubeconfig not readable | `export KUBECONFIG=~/.kube/config`, or `--write-kubeconfig-mode=644` in inventory `extra_server_args` | - -For a complete reset (RISKY — confirm with the user): - -```bash -cd deploy/ansible -sudo ansible-playbook playbooks/pb-k3s-reset.yml # whole cluster -sudo ansible-playbook playbooks/pb-k3s-reset.yml --limit # single node -``` - -## Out of scope - -Zot registry mirror, Cloudflare Tunnel ingress, monitoring/Grafana, HA k3s, -external databases, and NPU setup. Add them only after the minimal deployment -boots agents, schedules GPU notebooks, and persists notebook storage. +| `ssh-preinstalled` | Connects to every managed host, discovers GPU hardware, and publishes canonical files when discovery is consistent. | +| `pxe-diskless` | Uses `pxe.diskless_agents_have_amd_gpus` as its sole GPU policy input and publishes canonical desired-input files before the controller playbook runs. Their existence does not prove rootfs provisioning succeeded. | + +The skill is generator-first for both topologies. Don't hand-author generated +GPU policy, including for SSH. Create deployment specs from the current +`--print-schema` output. Generation resolves hosts to strict `true` or `false` +values and never writes `auto`. + +## Canonical validation inputs + +Use the topology's validator command from the +[skill scripts guide](scripts/README.md). It passes: + +- repository root with `--repo` +- selected topology with `--topology` +- installed inventory with `--inventory` +- generated GPU resolution report with `--gpu-resolution` +- base and generated overlays as two `--values` arguments +- canonical PXE vars with `--pxe-vars` for PXE only + +For a human direct inventory, `--inventory` alone accepts exactly one unquoted +`auto`, `true`, or `false` value for `auplc_gpu_access_enabled` on every managed +host. `--gpu-resolution` requires `--inventory`; supplying both checks generated +artifacts and requires strict booleans in the inventory and resolution report. +The skill supplies both because its workflow is generator-first. Generation and +validation must finish before Ansible or Helm changes are made. + +## GPU permission contract + +- Installer, Ansible, and PXE provisioning install AMD's + `amdgpu-insecure-instinct-udev-rules` package at version + `30.30.4.0-2341068.24.04`. +- The package sets mode `0666` only on `/dev/kfd` and DRM + `/dev/dri/renderD*` nodes. +- The package does not change `/dev/dri/card*`. Card nodes retain normal system + policy, observed as `root:video 0660`. +- AUPLC Hub adds no GPU supplemental group. No GPU group is required for the + tested ROCm compute path. +- AMD device-plugin allocation is the visibility boundary. Only Pods requesting + `amd.com/gpu` receive GPU device nodes; the plugin does not change host inode + ownership or mode. +- `singleuser.fsGid: 100` controls shared storage ownership only. + +Operator evidence from SHC showed `rocminfo` reporting `gfx1151` and `gfx1200` +on the two GPU nodes from UID `12345` Pods with only supplemental GID `100`. +Their `card*` nodes remained inaccessible at mode `0660`. + +The infrastructure owner deploys and maintains the AMD device plugin and ROCm +node labeller outside AUPLC. Before Helm, use the readiness and capacity checks +in [deploy/README.md](../../deploy/README.md); do not install these privileged +components as part of the AUPLC procedure. + +## Operator gates + +Keep the topology choice explicit. Confirm network, node, storage, course, and +access details with the user. For PXE, also confirm the GPU-agent boolean and a +rootfs SSH public key. For SSH, verify passwordless root access to every managed +host. + +Require confirmation before rootfs rebuilds, NFS export changes, firmware boot +changes, cluster resets, node deletion, or Helm uninstall. Keep generated +secrets out of version control. diff --git a/skills/deploy-aup-learning-cloud/scripts/README.md b/skills/deploy-aup-learning-cloud/scripts/README.md index 4e212dcc..83946594 100644 --- a/skills/deploy-aup-learning-cloud/scripts/README.md +++ b/skills/deploy-aup-learning-cloud/scripts/README.md @@ -1,77 +1,140 @@ # Helper scripts -Deterministic helpers the deploy skill runs instead of generating commands ad -hoc. They are dependency-light (`bash` + `python3`, plus the obvious system -tools) and agent-agnostic, and follow the script conventions in -[../../../CONTRIBUTING.md](../../../CONTRIBUTING.md). Each emits JSON or a clear -report and uses exit codes the agent can branch on. +These dependency-light helpers support the multi-node deployment skill. This +file is the source of truth for the complete skill command sequences: artifact +generation and installation, validation, Ansible, device plugin readiness, and +Helm. For human direct-edit deployment, operational background, and +troubleshooting, see [deploy/README.md](../../../deploy/README.md). -| Script | Run when | What it does | -| --- | --- | --- | -| `detect_hardware.sh` | Phase 2, on the service machine | Detects the default-route NIC, IPv4 + subnet CIDR, gateway, DNS servers, and AMD GPUs (`lspci`, vendor `1002`) with their kernel driver. Emits JSON for filling PXE / network vars. Read-only. | -| `detect_cluster.sh` | After k3s + the device plugin are up | `kubectl get` of nodes, real `amd.com/gpu.*` labels, storage classes, and whether the ROCm device plugin + labeller DaemonSets are running. Emits JSON. Read-only. | -| `gen_configs.py` | Phase 3 | From a small cluster-spec (`--print-schema`), writes `inventory.yml`, `pb-pxe-controller.vars.yml` (PXE only), and `values-basic-example.yaml`. Generates the k3s token locally with `secrets` (never printed), `chmod 600` on the inventory, and pins `pxe_k3s_version == k3s_version`. | -| `validate.py` | Before each `ansible-playbook` / `helm` run | For `pxe-diskless`, checks required PXE vars and `k3s_version == pxe_k3s_version`; for both topologies, checks GPU labels only for active resource `acceleratorKeys` (when given `detect_cluster.sh` output), and optionally runs a `helm template` dry-run. Exit 1 on any failure. | +| Script | Purpose | +| --- | --- | +| `detect_hardware.sh` | Reports controller network details and local AMD PCI devices as JSON. | +| `detect_cluster.sh` | Reports Kubernetes nodes, AMD GPU labels, storage classes, and GPU DaemonSet state as JSON. | +| `gen_configs.py` | Prints the current spec schema, discovers live GPU state, and directly publishes canonical topology-specific deployment artifacts. | +| `validate.py` | Checks the selected topology against canonical inventory, GPU resolution, values overlays, and PXE vars when applicable. | -## Quick reference +## Generator contract -From any directory in a checkout, resolve helpers with: +The SSH topology discovers GPU hosts from managed-host evidence. Users don't +provide a GPU host list. The PXE topology has one GPU policy input: +`pxe.diskless_agents_have_amd_gpus`. + +Generation resolves every host to `true` or `false`; it never writes `auto`. +Generated inventory and GPU resolution entries are strict booleans so their +consistency can be checked. + +Generate specs from fresh `--print-schema` output. Both topologies write their +canonical artifacts immediately. For PXE, review and validate those files, then +run the controller playbook with the generated `inventory.yml` and +`pb-pxe-controller.vars.yml`. The files express desired inputs; their existence +does not prove the PXE rootfs was provisioned successfully. + +## SSH-preinstalled commands + +Run these commands from a clean checkout. Fill the generated `spec.json` with +the SSH topology, network settings, and every managed host. Don't add a GPU host +list. The generator discovers GPU policy over passwordless root SSH. ```bash -REPO_ROOT="$(git rev-parse --show-toplevel)" +cd /path/to/aup-learning-cloud +REPO_ROOT="$(pwd)" DEPLOY_SCRIPTS="$REPO_ROOT/skills/deploy-aup-learning-cloud/scripts" +python3 "$DEPLOY_SCRIPTS/gen_configs.py" --print-schema > spec.json +# Edit spec.json: choose ssh-preinstalled and fill the node and network fields. +GENERATED_DIR="$REPO_ROOT/generated" +python3 "$DEPLOY_SCRIPTS/gen_configs.py" --spec spec.json --out-dir "$GENERATED_DIR" + +install -m 0600 "$GENERATED_DIR/inventory.yml" "$REPO_ROOT/deploy/ansible/inventory.yml" +install -m 0644 "$GENERATED_DIR/values-basic-example.yaml" "$REPO_ROOT/runtime/values-basic-example.yaml" + +python3 "$DEPLOY_SCRIPTS/validate.py" --repo "$REPO_ROOT" --topology ssh-preinstalled \ + --inventory "$REPO_ROOT/deploy/ansible/inventory.yml" \ + --gpu-resolution "$GENERATED_DIR/gpu-access-resolution.json" \ + --values "$REPO_ROOT/runtime/values.yaml" \ + --values "$REPO_ROOT/runtime/values-basic-example.yaml" ``` -For an installed plugin, set `DEPLOY_SKILL_DIR` to the absolute directory -containing the loaded `SKILL.md`, then use: +After validation passes, run Ansible, check the infrastructure-owned GPU +components, and install the chart with the generated overlay: ```bash -DEPLOY_SKILL_DIR="/absolute/path/to/deploy-aup-learning-cloud" -DEPLOY_SCRIPTS="$DEPLOY_SKILL_DIR/scripts" +cd "$REPO_ROOT/deploy/ansible" +sudo ansible-playbook -i inventory.yml playbooks/pb-base.yml +sudo ansible-playbook -i inventory.yml playbooks/pb-k3s-site.yml +sudo ansible-playbook -i inventory.yml playbooks/pb-rocm.yml + +kubectl rollout status -n kube-system daemonset/amdgpu-device-plugin-daemonset --timeout=5m +kubectl rollout status -n kube-system daemonset/amdgpu-labeller-daemonset --timeout=5m +kubectl get nodes -o 'custom-columns=NAME:.metadata.name,AMD_GPU:.status.allocatable.amd\.com/gpu' + +cd "$REPO_ROOT" +helm upgrade --install jupyterhub ./runtime/chart \ + --namespace jupyterhub --create-namespace \ + -f runtime/values.yaml \ + -f runtime/values-basic-example.yaml ``` -```bash -# Phase 2 — discover the host -"$DEPLOY_SCRIPTS/detect_hardware.sh" # JSON: nic, ip, subnet_cidr, gateway, dns, gpus[] +## PXE-diskless commands -# Phase 3 — generate config from a spec -python3 "$DEPLOY_SCRIPTS/gen_configs.py" --print-schema > spec.json # then edit spec.json +Fill the generated `spec.json` with the PXE topology and all controller, +network, and rootfs fields. Set `pxe.diskless_agents_have_amd_gpus` explicitly. + +```bash +cd /path/to/aup-learning-cloud +REPO_ROOT="$(pwd)" +DEPLOY_SCRIPTS="$REPO_ROOT/skills/deploy-aup-learning-cloud/scripts" +python3 "$DEPLOY_SCRIPTS/gen_configs.py" --print-schema > spec.json +# Edit spec.json: choose pxe-diskless and fill the node, network, and PXE fields. GENERATED_DIR="$REPO_ROOT/generated" python3 "$DEPLOY_SCRIPTS/gen_configs.py" --spec spec.json --out-dir "$GENERATED_DIR" + install -m 0600 "$GENERATED_DIR/inventory.yml" "$REPO_ROOT/deploy/ansible/inventory.yml" install -m 0644 "$GENERATED_DIR/values-basic-example.yaml" "$REPO_ROOT/runtime/values-basic-example.yaml" -# PXE only: keep the generated secret in place and resolve its absolute path. -PXE_VARS="$(realpath "$GENERATED_DIR/pb-pxe-controller.vars.yml")" -chmod 0600 "$PXE_VARS" + +python3 "$DEPLOY_SCRIPTS/validate.py" --repo "$REPO_ROOT" --topology pxe-diskless \ + --inventory "$REPO_ROOT/deploy/ansible/inventory.yml" \ + --gpu-resolution "$GENERATED_DIR/gpu-access-resolution.json" \ + --values "$REPO_ROOT/runtime/values.yaml" \ + --values "$REPO_ROOT/runtime/values-basic-example.yaml" \ + --pxe-vars "$GENERATED_DIR/pb-pxe-controller.vars.yml" + cd "$REPO_ROOT/deploy/ansible" -ansible-playbook -i inventory.yml playbooks/pb-pxe-controller.yml -e @"$PXE_VARS" +sudo ansible-playbook \ + -i "$GENERATED_DIR/inventory.yml" \ + playbooks/pb-pxe-controller.yml \ + -e @"$GENERATED_DIR/pb-pxe-controller.vars.yml" -# Phase 5 — after k3s + device plugin are up -"$DEPLOY_SCRIPTS/detect_cluster.sh" > cluster.json # JSON: nodes[], gpu_product_names[], storage_classes[] +kubectl rollout status -n kube-system daemonset/amdgpu-device-plugin-daemonset --timeout=5m +kubectl rollout status -n kube-system daemonset/amdgpu-labeller-daemonset --timeout=5m +kubectl get nodes -o 'custom-columns=NAME:.metadata.name,AMD_GPU:.status.allocatable.amd\.com/gpu' -# Before running playbooks / helm (set to the selected topology) -DEPLOY_TOPOLOGY=pxe-diskless -python3 "$DEPLOY_SCRIPTS/validate.py" --repo "$REPO_ROOT" --topology "$DEPLOY_TOPOLOGY" \ - --values runtime/values.yaml --values runtime/values-basic-example.yaml \ - --pxe-vars "$PXE_VARS" --cluster cluster.json --helm-dry-run +cd "$REPO_ROOT" +helm upgrade --install jupyterhub ./runtime/chart \ + --namespace jupyterhub --create-namespace \ + -f runtime/values.yaml \ + -f runtime/values-basic-example.yaml ``` -Omit `--pxe-vars "$PXE_VARS"` for `ssh-preinstalled`. For `pxe-diskless`, the -validator and Ansible must receive the same generated file. +The controller playbook must finish successfully before the remaining cluster +and Helm steps begin. A fresh rootfs receives the pinned GPU access package. A +retained rootfs must pass the package version, package-owned rule, and legacy +rule safety checks described in the deployment guide. + +## Validator contract -Generated `gpu.acceleratorKeys` wires the selected accelerators to the generic -GPU resource. Use `configure-aup-learning-cloud-courses` to wire course -resources separately. +The exact topology commands above pass `--repo`, `--topology`, `--inventory`, +`--gpu-resolution`, two `--values` arguments, and `--pxe-vars` for PXE only. +For direct validation, `--inventory` alone accepts exactly one unquoted `auto`, +`true`, or `false` value for `auplc_gpu_access_enabled` on every managed host. +`--gpu-resolution` requires `--inventory`; supplying both switches to generated +consistency validation, where inventory and resolution values must be strict +booleans. The generator-first skill workflow supplies both and never generates +`auto`. ## Conventions -- **JSON to stdout, diagnostics to stderr.** `detect_*.sh` always print a JSON - object; partial detection is reported via empty fields + a `warnings` array - rather than failing, so the agent can decide what to ask the operator. -- **Exit codes mean something.** `0` success (warnings allowed), `1` a real - validation failure, `2` a usage / missing-tooling error. -- **Secrets never touch stdout or VCS.** `gen_configs.py` mints the k3s token - with a CSPRNG, writes it only into `inventory.yml`, and `chmod 600`s it. -- **No third-party Python.** `gen_configs.py` / `validate.py` use the stdlib - only (no PyYAML), so they run on a bare operator machine. YAML is emitted - from templates and parsed with targeted scanning. +- Detection data goes to stdout as JSON. Diagnostics go to stderr. +- Exit code `0` means success, `1` means validation failed, and `2` means usage + or required tooling is wrong. +- Generated secrets stay off stdout and out of version control. +- Python helpers use the standard library only. diff --git a/skills/deploy-aup-learning-cloud/scripts/artifact_store.py b/skills/deploy-aup-learning-cloud/scripts/artifact_store.py new file mode 100644 index 00000000..5b9de061 --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/artifact_store.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +"""Stage and atomically publish generated deployment artifacts.""" + +from __future__ import annotations + +import os +import shutil +import sys +import tempfile +from contextlib import suppress +from pathlib import Path + + +def die(msg: str, code: int = 1) -> None: + print(f"gen_configs: {msg}", file=sys.stderr) + raise SystemExit(code) + + +def preflight_destinations(paths: list[Path], force: bool) -> None: + if force: + return + for path in paths: + if os.path.lexists(path): + die(f"refusing to overwrite existing {path} (use --force)", 1) + + +def stage_file(path: Path, content: str, mode: int) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + fd, staged_path = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + try: + os.fchmod(fd, mode) + with os.fdopen(fd, "w", encoding="utf-8") as staged_file: + staged_file.write(content) + staged_file.flush() + os.fsync(staged_file.fileno()) + except OSError: + with suppress(OSError): + os.close(fd) + Path(staged_path).unlink(missing_ok=True) + raise + return Path(staged_path) + + +def remove_destination(path: Path) -> None: + if path.is_dir() and not path.is_symlink(): + shutil.rmtree(path) + else: + path.unlink(missing_ok=True) + + +def backup_destination(path: Path) -> tuple[Path, Path]: + backup_dir = Path(tempfile.mkdtemp(prefix=f".{path.name}.backup.", dir=path.parent)) + backup_path = backup_dir / path.name + os.replace(path, backup_path) + return backup_dir, backup_path + + +def _fsync_parent(path: Path) -> None: + directory_fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) + + +def publish_artifacts( + artifacts: list[tuple[Path, str, int, bool]], force: bool, remove_paths: tuple[Path, ...] = () +) -> None: + staged: list[tuple[Path, Path, bool]] = [] + published: list[Path] = [] + backups: list[tuple[Path, Path, Path]] = [] + replacement_paths = tuple(path for path, _, _, _ in artifacts) + try: + for path, content, mode, secret in artifacts: + staged.append((path, stage_file(path, content, mode), secret)) + if force: + for path in (*replacement_paths, *(path for path in remove_paths if path not in replacement_paths)): + if os.path.lexists(path): + backup_dir, backup_path = backup_destination(path) + backups.append((path, backup_dir, backup_path)) + _fsync_parent(path) + for path, staged_path, secret in staged: + if force: + os.replace(staged_path, path) + else: + os.link(staged_path, path) + published.append(path) + if not force: + os.unlink(staged_path) + _fsync_parent(path) + print(f"wrote {path}" + (" (chmod 600 -- contains the k3s token)" if secret else "")) + except OSError as exc: + for path in reversed(published): + remove_destination(path) + _fsync_parent(path) + for path, backup_dir, backup_path in reversed(backups): + remove_destination(path) + os.replace(backup_path, path) + _fsync_parent(path) + backup_dir.rmdir() + die(f"could not publish generated artifacts: {exc}") + else: + for _, backup_dir, _ in backups: + shutil.rmtree(backup_dir) + finally: + for _, staged_path, _ in staged: + staged_path.unlink(missing_ok=True) diff --git a/skills/deploy-aup-learning-cloud/scripts/config_common.py b/skills/deploy-aup-learning-cloud/scripts/config_common.py new file mode 100644 index 00000000..edc3f67e --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/config_common.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +"""Shared schema constants and scalar rendering helpers.""" + +from __future__ import annotations + +import json +import sys + +HEADER_HASH = ( + "# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved.\n" + "# Generated by auplc-skills gen_configs.py -- review before use.\n" +) + +DEFAULT_ACCEL_LABELS = { + "phx": "AMD_Radeon_780M_Graphics", + "strix": "AMD_Radeon_890M_Graphics", + "strix-halo": "AMD_Radeon_8060S_Graphics", + "9070xt": "AMD_Radeon_RX_9070_XT", + "r9700": "AMD_Radeon_AI_PRO_R9700", + "9600gre": "AMD_Radeon_RX_9600_GRE", +} + + +class DuplicateJsonKeyError(ValueError): + pass + + +def _unique_json_object(pairs): + document = {} + for key, value in pairs: + if key in document: + raise DuplicateJsonKeyError(f"duplicate JSON key '{key}'") + document[key] = value + return document + + +def strict_json_loads(raw: str): + return json.loads(raw, object_pairs_hook=_unique_json_object) + + +def die(msg: str, code: int = 1) -> None: + print(f"gen_configs: {msg}", file=sys.stderr) + raise SystemExit(code) + + +def require(spec: dict, path: str): + cur = spec + for part in path.split("."): + if not isinstance(cur, dict) or part not in cur or cur[part] in (None, "", []): + die(f"spec is missing required field '{path}'") + cur = cur[part] + return cur + + +def yaml_quote(value: str) -> str: + return '"' + str(value).replace("\\", "\\\\").replace('"', '\\"') + '"' diff --git a/skills/deploy-aup-learning-cloud/scripts/config_generation.py b/skills/deploy-aup-learning-cloud/scripts/config_generation.py new file mode 100644 index 00000000..c0e881ff --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/config_generation.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +"""Validate cluster specifications and render deploy configuration artifacts.""" + +from __future__ import annotations + +import ipaddress +import re + +from config_common import DEFAULT_ACCEL_LABELS, HEADER_HASH, die, require, yaml_quote +from config_rendering import render_inventory, render_pxe_vars, render_values + +__all__ = [ + "DEFAULT_ACCEL_LABELS", + "HEADER_HASH", + "SCHEMA", + "die", + "render_inventory", + "render_pxe_vars", + "render_values", + "require", + "validate_accelerators", + "validate_config_shapes", + "validate_yaml_scalar", + "validate_spec", + "yaml_quote", +] + +SCHEMA = { + "topology": "pxe-diskless | ssh-preinstalled", + "k3s_version": "v1.32.3+k3s1", + "server": {"name": "aipc1", "ip": "192.168.0.140"}, + "agents": [{"name": "aipc2", "ip": "192.168.0.141"}], + "network": { + "interface": "enp1s0", + "subnet": "192.168.0.0/24", + "gateway": "192.168.0.1", + "dns_servers": "8.8.8.8,8.8.4.4", + }, + "pxe": { + "authorized_keys": ["ssh-ed25519 AAAA... you@host"], + "rootfs_password": "", + "web_port": 8080, + "diskless_agents_have_amd_gpus": True, + }, + "accelerators": {"strix-halo": {"product_name": "AMD_Radeon_8060S_Graphics"}}, + "storage": {"class": "nfs-client"}, + "proxy": {"node_port": 30890}, + "auth_mode": "auto-login", + "images": {"cpu": "ghcr.io/amdresearch/auplc-default:latest", "gpu": "ghcr.io/amdresearch/auplc-base:latest"}, +} + +HOSTNAME_PATTERN = re.compile( + r"(?=.{1,253}\Z)(?:[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)(?:\.(?:[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?))*\Z" +) +K3S_VERSION_PATTERN = re.compile(r"v[0-9]+\.[0-9]+\.[0-9]+\+k3s[0-9]+\Z") +IMAGE_KEY_PATTERN = re.compile(r"[A-Za-z][A-Za-z0-9_-]*\Z") + + +def validate_accelerators(spec: dict) -> None: + if "accelerators" not in spec: + return + accelerators = spec["accelerators"] + if not isinstance(accelerators, dict): + die("spec.accelerators must be a mapping") + unsupported = sorted(set(accelerators) - set(DEFAULT_ACCEL_LABELS)) + if len(unsupported) == 1: + die(f"unsupported accelerator key '{unsupported[0]}'") + if unsupported: + die(f"unsupported accelerator keys: {', '.join(unsupported)}") + for key, config in accelerators.items(): + if not isinstance(config, dict): + die(f"accelerators.{key} must be a mapping") + + +def validate_config_shapes(spec: dict) -> None: + if not isinstance(spec, dict): + die("spec must be a mapping") + validate_accelerators(spec) + for key in ("server", "network", "pxe", "storage", "proxy", "images"): + if key in spec and not isinstance(spec[key], dict): + die(f"spec.{key} must be a mapping") + if "agents" in spec and not isinstance(spec["agents"], list): + die("spec.agents must be a list") + + +def _safe_text(value, path: str, *, allow_empty: bool = False) -> str: + if not isinstance(value, str) or (not allow_empty and not value): + die(f"{path} must be a non-empty string" if not allow_empty else f"{path} must be a string") + if any(ord(character) < 32 or ord(character) == 127 for character in value): + die(f"{path} must not contain control characters") + return value + + +def validate_yaml_scalar(value, path: str, *, allow_empty: bool = False) -> str: + return _safe_text(value, path, allow_empty=allow_empty) + + +def _safe_hostname(value, path: str) -> str: + hostname = _safe_text(value, path) + if not HOSTNAME_PATTERN.fullmatch(hostname): + die(f"{path} must be a safe hostname") + return hostname + + +def _safe_ip(value, path: str) -> str: + address = _safe_text(value, path) + try: + ipaddress.ip_address(address) + except ValueError: + die(f"{path} must be a valid IP address") + return address + + +def _safe_port(value, path: str, minimum: int, maximum: int) -> int: + if type(value) is not int or not minimum <= value <= maximum: + die(f"{path} must be an integer between {minimum} and {maximum}") + return value + + +def _validate_server(server: dict, path: str) -> str: + if set(server) != {"name", "ip"}: + die(f"{path} must contain exactly name and ip") + name = _safe_hostname(server["name"], f"{path}.name") + _safe_ip(server["ip"], f"{path}.ip") + return name + + +def _validate_agents(spec: dict, server_name: str) -> None: + agents = spec.get("agents", []) + if not isinstance(agents, list): + die("spec.agents must be a list") + names = {server_name} + for index, agent in enumerate(agents): + path = f"spec.agents[{index}]" + if not isinstance(agent, dict): + die(f"{path} must be a mapping") + name = _validate_server(agent, path) + if name in names: + die("server and agent names must be unique") + names.add(name) + + +def _validate_rendered_options(spec: dict) -> None: + if "auth_mode" in spec: + _safe_text(spec["auth_mode"], "spec.auth_mode") + if "storage" in spec and "class" in spec["storage"]: + _safe_text(spec["storage"]["class"], "spec.storage.class") + if "proxy" in spec and "node_port" in spec["proxy"]: + _safe_port(spec["proxy"]["node_port"], "spec.proxy.node_port", 30000, 32767) + if "images" in spec: + for key, value in spec["images"].items(): + if not isinstance(key, str) or not IMAGE_KEY_PATTERN.fullmatch(key): + die("spec.images key must be a safe identifier") + _safe_text(value, f"spec.images.{key}") + if "accelerators" in spec: + for key, config in spec["accelerators"].items(): + if "product_name" in config: + _safe_text(config["product_name"], f"spec.accelerators.{key}.product_name") + + +def _validate_pxe(spec: dict) -> None: + pxe = require(spec, "pxe") + keys = pxe.get("authorized_keys") + if not isinstance(keys, list) or not keys: + die("pxe.authorized_keys must contain at least one SSH public key") + for index, key in enumerate(keys): + _safe_text(key, f"spec.pxe.authorized_keys[{index}]") + if "rootfs_password" in pxe: + _safe_text(pxe["rootfs_password"], "spec.pxe.rootfs_password", allow_empty=True) + if "web_port" in pxe: + _safe_port(pxe["web_port"], "spec.pxe.web_port", 1, 65535) + if type(pxe.get("diskless_agents_have_amd_gpus")) is not bool: + die("spec.pxe.diskless_agents_have_amd_gpus must be a boolean") + network = require(spec, "network") + _safe_text(require(spec, "network.interface"), "spec.network.interface") + subnet = _safe_text(require(spec, "network.subnet"), "spec.network.subnet") + try: + ipaddress.ip_network(subnet, strict=True) + except ValueError: + die("spec.network.subnet must be a valid network CIDR") + if "gateway" in network: + _safe_ip(network["gateway"], "spec.network.gateway") + if "dns_servers" in network: + for index, address in enumerate(_safe_text(network["dns_servers"], "spec.network.dns_servers").split(",")): + _safe_ip(address.strip(), f"spec.network.dns_servers[{index}]") + + +def validate_spec(spec: dict) -> str: + if not isinstance(spec, dict): + die("spec must be a mapping") + topo = spec.get("topology") + if topo not in ("pxe-diskless", "ssh-preinstalled"): + die("spec.topology must be 'pxe-diskless' or 'ssh-preinstalled'") + validate_config_shapes(spec) + k3s_version = _safe_text(require(spec, "k3s_version"), "spec.k3s_version") + if not K3S_VERSION_PATTERN.fullmatch(k3s_version): + die("spec.k3s_version must be a safe k3s version") + server_name = _validate_server(require(spec, "server"), "spec.server") + _validate_agents(spec, server_name) + _validate_rendered_options(spec) + if topo == "pxe-diskless": + _validate_pxe(spec) + return topo diff --git a/skills/deploy-aup-learning-cloud/scripts/config_rendering.py b/skills/deploy-aup-learning-cloud/scripts/config_rendering.py new file mode 100644 index 00000000..b66a0064 --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/config_rendering.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +"""Render deployment artifacts from resolved configuration values.""" + +from __future__ import annotations + +from config_common import DEFAULT_ACCEL_LABELS, HEADER_HASH, die, require, yaml_quote +from gpu_access_resolution import FleetResolution, HostStatus + + +def render_inventory(spec: dict, token: str, resolution: FleetResolution) -> str: + topo = spec["topology"] + server = spec["server"] + k3s_version = spec["k3s_version"] + host_gpu_enabled = {host.target.name: host.status is HostStatus.GPU for host in resolution.hosts} + lines = [ + HEADER_HASH, + "k3s_cluster:", + " children:", + " server:", + " hosts:", + f" {server['name']}:", + f" ansible_host: {yaml_quote(server['ip'])}", + f" auplc_gpu_access_enabled: {'true' if host_gpu_enabled[server['name']] else 'false'}", + " agent:", + ] + if topo == "ssh-preinstalled" and spec.get("agents"): + lines.append(" hosts:") + for agent in spec["agents"]: + lines.append(f" {agent['name']}:") + lines.append(f" ansible_host: {yaml_quote(agent['ip'])}") + lines.append( + f" auplc_gpu_access_enabled: {'true' if host_gpu_enabled[agent['name']] else 'false'}" + ) + else: + lines.append(" hosts: {}") + lines += [ + " vars:", + " ansible_port: 22", + " ansible_user: root", + f" k3s_version: {yaml_quote(k3s_version)}", + f" token: {yaml_quote(token)}", + " api_endpoint: \"{{ hostvars[groups['server'][0]]['ansible_host'] | default(groups['server'][0]) }}\"", + ] + if topo == "pxe-diskless": + lines += [ + "", + "pxe_controller:", + " hosts:", + f" {server['name']}:", + f" ansible_host: {yaml_quote(server['ip'])}", + " vars:", + " ansible_port: 22", + " ansible_user: root", + ] + return "\n".join(lines) + "\n" + + +def render_pxe_vars(spec: dict, pxe_gpu_access_enabled: bool) -> str: + net = require(spec, "network") + pxe = spec.get("pxe", {}) + keys = pxe.get("authorized_keys", []) + if not keys: + die("pxe.authorized_keys must contain at least one SSH public key") + server_ip = spec["server"]["ip"] + k3s_version = spec["k3s_version"] + lines = [ + HEADER_HASH, + "# Pass this file to pb-pxe-controller.yml with", + "# ansible-playbook ... -e @", + "# pxe_k3s_version is pinned to k3s_version so agents are never newer", + "# than the server.", + "pxe_rootfs_force_rebuild: true # first build only; set false afterwards", + f"pxe_network_interface: {yaml_quote(net['interface'])}", + f"pxe_subnet: {yaml_quote(net['subnet'])}", + f"pxe_gateway: {yaml_quote(net.get('gateway', ''))}", + f"pxe_dns_servers: {yaml_quote(net.get('dns_servers', '8.8.8.8,8.8.4.4'))}", + f"pxe_controller_ip: {yaml_quote(server_ip)}", + "pxe_k3s_server_ips:", + f" - {yaml_quote(server_ip)}", + f"pxe_k3s_version: {yaml_quote(k3s_version)}", + f"pxe_gpu_access_enabled: {'true' if pxe_gpu_access_enabled else 'false'}", + f"pxe_web_port: {int(pxe.get('web_port', 8080))}", + f"pxe_rootfs_password: {yaml_quote(pxe.get('rootfs_password', ''))}", + "pxe_rootfs_authorized_keys:", + ] + for key in keys: + lines.append(f" - {yaml_quote(key)}") + return "\n".join(lines) + "\n" + + +def render_values(spec: dict) -> str: + accel = spec.get("accelerators") or {} + storage_class = (spec.get("storage") or {}).get("class", "nfs-client") + node_port = (spec.get("proxy") or {}).get("node_port", 30890) + auth_mode = spec.get("auth_mode", "auto-login") + images = spec.get("images") or {} + lines = [ + "# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved.", + "# Helm overlay generated by auplc-skills gen_configs.py.", + "# Layer this on top of runtime/values.yaml:", + "# helm upgrade --install jupyterhub ./runtime/chart -n jupyterhub \\", + "# --create-namespace -f runtime/values.yaml -f ", + "custom:", + f" authMode: {yaml_quote(auth_mode)}", + ] + if accel: + lines.append(" accelerators:") + for key, config in accel.items(): + product = (config or {}).get("product_name") or DEFAULT_ACCEL_LABELS.get(key) + if not product: + die( + f"accelerator '{key}' has no product_name and no known default; " + "add accelerators..product_name from `kubectl describe node`" + ) + lines += [ + f" {key}:", + " nodeSelector:", + f" amd.com/gpu.product-name: {yaml_quote(product)}", + ] + if accel or images: + lines.append(" resources:") + if accel: + lines += [" metadata:", " gpu:", " acceleratorKeys:"] + lines.extend(f" - {yaml_quote(key)}" for key in accel) + if images: + lines.append(" images:") + for key, value in images.items(): + lines.append(f" {key}: {yaml_quote(value)}") + lines += [ + "hub:", + " db:", + " pvc:", + f" storageClassName: {yaml_quote(storage_class)}", + "singleuser:", + " storage:", + " dynamic:", + f" storageClass: {yaml_quote(storage_class)}", + "proxy:", + " service:", + " type: NodePort", + " nodePorts:", + f" http: {int(node_port)}", + ] + return "\n".join(lines) + "\n" diff --git a/skills/deploy-aup-learning-cloud/scripts/gen_configs.py b/skills/deploy-aup-learning-cloud/scripts/gen_configs.py index e0d91ee8..e9665c10 100755 --- a/skills/deploy-aup-learning-cloud/scripts/gen_configs.py +++ b/skills/deploy-aup-learning-cloud/scripts/gen_configs.py @@ -2,8 +2,9 @@ # Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. """Generate AUP Learning Cloud deploy artifacts from a small cluster-spec. -Given a JSON cluster-spec (see ``--print-schema``), write the three files the -deploy skill needs, keeping them mutually consistent: +Given a JSON cluster-spec (see ``--print-schema``), discover the managed hosts' +GPU policy. Both topologies immediately write mutually consistent canonical +deployment artifacts: 1. ``inventory.yml`` -- Ansible inventory (server + token + k3s_version; agents listed for the @@ -11,17 +12,19 @@ 2. ``pb-pxe-controller.vars.yml`` -- PXE topology only: extra vars passed to pb-pxe-controller.yml with ``-e @``. - 3. ``values-basic-example.yaml`` -- Helm overlay: accelerator nodeSelectors, - storage class, proxy NodePort, authMode. + 3. ``values-basic-example.yaml`` -- Helm overlay: storage, proxy, and + authentication. + 4. ``gpu-access-resolution.json`` -- Machine-readable resolved host policy. Design choices (deliberate): * stdlib only (json, argparse, secrets, base64, pathlib). No PyYAML, so this runs on a bare operator machine. YAML is emitted from templates, not a serialiser -- the output is small, fixed-shape, and carries the copyright header. - * The k3s token is generated locally with ``secrets`` (CSPRNG) and written - ONLY into inventory.yml. It is never printed to stdout/stderr. Pass - ``--token-file`` to reuse an existing token instead of minting one. + * The k3s token is generated locally with ``secrets`` (CSPRNG). Canonical + output writes it only into ``inventory.yml``. It is never printed to + stdout/stderr. Pass ``--token-file`` to reuse an existing token instead of + minting one. * ``pxe_k3s_version`` is forced equal to ``k3s_version`` so agents can never be newer than the server (k3s refuses that). * Existing files are not overwritten unless ``--force`` is given. @@ -39,58 +42,22 @@ import argparse import base64 import json -import os import secrets -import shutil import sys -import tempfile -from contextlib import suppress from pathlib import Path -HEADER_HASH = ( - "# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved.\n" - "# Generated by auplc-skills gen_configs.py -- review before use.\n" +from artifact_store import preflight_destinations, publish_artifacts +from config_common import DuplicateJsonKeyError, strict_json_loads +from config_generation import ( + SCHEMA, + die, + render_inventory, + render_pxe_vars, + render_values, + validate_spec, + validate_yaml_scalar, ) - -# Default GPU product-name labels, keyed by the accelerator key used in -# runtime/values.yaml (custom.accelerators.). Verified against the chart's -# values.yaml; override per fleet via spec["accelerators"][key]["product_name"]. -DEFAULT_ACCEL_LABELS = { - "phx": "AMD_Radeon_780M_Graphics", - "strix": "AMD_Radeon_890M_Graphics", - "strix-halo": "AMD_Radeon_8060S_Graphics", - "9070xt": "AMD_Radeon_RX_9070_XT", - "r9700": "AMD_Radeon_AI_PRO_R9700", - "9600gre": "AMD_Radeon_RX_9600_GRE", -} - -SCHEMA = { - "topology": "pxe-diskless | ssh-preinstalled", - "k3s_version": "v1.32.3+k3s1", - "server": {"name": "aipc1", "ip": "192.168.0.140"}, - "agents": [{"name": "aipc2", "ip": "192.168.0.141"}], - "network": { - "interface": "enp1s0", - "subnet": "192.168.0.0/24", - "gateway": "192.168.0.1", - "dns_servers": "8.8.8.8,8.8.4.4", - }, - "pxe": { - "authorized_keys": ["ssh-ed25519 AAAA... you@host"], - "rootfs_password": "", - "web_port": 8080, - }, - "accelerators": {"strix-halo": {"product_name": "AMD_Radeon_8060S_Graphics"}}, - "storage": {"class": "nfs-client"}, - "proxy": {"node_port": 30890}, - "auth_mode": "auto-login", - "images": {"cpu": "ghcr.io/amdresearch/auplc-default:latest", "gpu": "ghcr.io/amdresearch/auplc-base:latest"}, -} - - -def die(msg: str, code: int = 1) -> None: - print(f"gen_configs: {msg}", file=sys.stderr) - raise SystemExit(code) +from gpu_artifact_generation import DiscoveryFailure, canonical_paths, discover_gpu_policy, manifest_content def gen_token() -> str: @@ -98,250 +65,6 @@ def gen_token() -> str: return base64.b64encode(secrets.token_bytes(64)).decode("ascii") -def require(spec: dict, path: str): - cur = spec - for part in path.split("."): - if not isinstance(cur, dict) or part not in cur or cur[part] in (None, "", []): - die(f"spec is missing required field '{path}'") - cur = cur[part] - return cur - - -def yaml_quote(s: str) -> str: - return '"' + str(s).replace("\\", "\\\\").replace('"', '\\"') + '"' - - -def validate_accelerators(spec: dict) -> None: - if "accelerators" not in spec: - return - accelerators = spec["accelerators"] - if not isinstance(accelerators, dict): - die("spec.accelerators must be a mapping") - unsupported = sorted(set(accelerators) - set(DEFAULT_ACCEL_LABELS)) - if len(unsupported) == 1: - die(f"unsupported accelerator key '{unsupported[0]}'") - if unsupported: - die(f"unsupported accelerator keys: {', '.join(unsupported)}") - for key, config in accelerators.items(): - if not isinstance(config, dict): - die(f"accelerators.{key} must be a mapping") - - -def validate_config_shapes(spec: dict) -> None: - if not isinstance(spec, dict): - die("spec must be a mapping") - validate_accelerators(spec) - for key in ("network", "pxe", "storage", "proxy", "images"): - if key in spec and not isinstance(spec[key], dict): - die(f"spec.{key} must be a mapping") - - -def render_inventory(spec: dict, token: str) -> str: - topo = spec["topology"] - server = spec["server"] - k3s_version = spec["k3s_version"] - lines = [ - HEADER_HASH, - "k3s_cluster:", - " children:", - " server:", - " hosts:", - f" {server['name']}:", - f" ansible_host: {server['ip']}", - " agent:", - ] - if topo == "ssh-preinstalled" and spec.get("agents"): - lines.append(" hosts:") - for a in spec["agents"]: - lines.append(f" {a['name']}:") - lines.append(f" ansible_host: {a['ip']}") - else: - # PXE diskless agents auto-join by netboot; do NOT list them here. - lines.append(" hosts: {}") - lines += [ - " vars:", - " ansible_port: 22", - " ansible_user: root", - f" k3s_version: {k3s_version}", - f" token: {yaml_quote(token)}", - " api_endpoint: \"{{ hostvars[groups['server'][0]]['ansible_host'] | default(groups['server'][0]) }}\"", - ] - if topo == "pxe-diskless": - lines += [ - "", - "pxe_controller:", - " hosts:", - f" {server['name']}:", - f" ansible_host: {server['ip']}", - " vars:", - " ansible_port: 22", - " ansible_user: root", - ] - return "\n".join(lines) + "\n" - - -def render_pxe_vars(spec: dict) -> str: - net = require(spec, "network") - pxe = spec.get("pxe", {}) - keys = pxe.get("authorized_keys", []) - if not keys: - die("pxe.authorized_keys must contain at least one SSH public key") - server_ip = spec["server"]["ip"] - k3s_version = spec["k3s_version"] - lines = [ - HEADER_HASH, - "# Pass this file to pb-pxe-controller.yml with", - "# ansible-playbook ... -e @", - "# pxe_k3s_version is pinned to k3s_version so agents are never newer", - "# than the server.", - "pxe_rootfs_force_rebuild: true # first build only; set false afterwards", - f"pxe_network_interface: {yaml_quote(net['interface'])}", - f"pxe_subnet: {yaml_quote(net['subnet'])}", - f"pxe_gateway: {yaml_quote(net.get('gateway', ''))}", - f"pxe_dns_servers: {yaml_quote(net.get('dns_servers', '8.8.8.8,8.8.4.4'))}", - f"pxe_controller_ip: {yaml_quote(server_ip)}", - "pxe_k3s_server_ips:", - f" - {yaml_quote(server_ip)}", - f"pxe_k3s_version: {yaml_quote(k3s_version)}", - f"pxe_web_port: {int(pxe.get('web_port', 8080))}", - f"pxe_rootfs_password: {yaml_quote(pxe.get('rootfs_password', ''))}", - "pxe_rootfs_authorized_keys:", - ] - for k in keys: - lines.append(f" - {yaml_quote(k)}") - return "\n".join(lines) + "\n" - - -def render_values(spec: dict) -> str: - accel = spec.get("accelerators") or {} - storage_class = (spec.get("storage") or {}).get("class", "nfs-client") - node_port = (spec.get("proxy") or {}).get("node_port", 30890) - auth_mode = spec.get("auth_mode", "auto-login") - images = spec.get("images") or {} - - lines = [ - "# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved.", - "# Helm overlay generated by auplc-skills gen_configs.py.", - "# Layer this on top of runtime/values.yaml:", - "# helm upgrade --install jupyterhub ./runtime/chart -n jupyterhub \\", - "# --create-namespace -f runtime/values.yaml -f ", - "custom:", - f" authMode: {yaml_quote(auth_mode)}", - ] - if accel: - lines.append(" accelerators:") - for key, cfg in accel.items(): - product = (cfg or {}).get("product_name") or DEFAULT_ACCEL_LABELS.get(key) - if not product: - die( - f"accelerator '{key}' has no product_name and no known default; " - "add accelerators..product_name from `kubectl describe node`" - ) - lines += [ - f" {key}:", - " nodeSelector:", - f" amd.com/gpu.product-name: {yaml_quote(product)}", - ] - if accel or images: - lines.append(" resources:") - if accel: - lines += [" metadata:", " gpu:", " acceleratorKeys:"] - lines.extend(f" - {yaml_quote(key)}" for key in accel) - if images: - lines.append(" images:") - for k, v in images.items(): - lines.append(f" {k}: {yaml_quote(v)}") - lines += [ - "hub:", - " db:", - " pvc:", - f" storageClassName: {yaml_quote(storage_class)}", - "singleuser:", - " storage:", - " dynamic:", - f" storageClass: {yaml_quote(storage_class)}", - "proxy:", - " service:", - " type: NodePort", - " nodePorts:", - f" http: {int(node_port)}", - ] - return "\n".join(lines) + "\n" - - -def preflight_destinations(paths: list[Path], force: bool) -> None: - if force: - return - for path in paths: - if os.path.lexists(path): - die(f"refusing to overwrite existing {path} (use --force)", 1) - - -def stage_file(path: Path, content: str, mode: int) -> Path: - path.parent.mkdir(parents=True, exist_ok=True) - fd, staged_path = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) - try: - os.fchmod(fd, mode) - with os.fdopen(fd, "w", encoding="utf-8") as staged_file: - staged_file.write(content) - staged_file.flush() - os.fsync(staged_file.fileno()) - except OSError: - with suppress(OSError): - os.close(fd) - Path(staged_path).unlink(missing_ok=True) - raise - return Path(staged_path) - - -def remove_destination(path: Path) -> None: - if path.is_dir() and not path.is_symlink(): - shutil.rmtree(path) - else: - path.unlink(missing_ok=True) - - -def backup_destination(path: Path) -> tuple[Path, Path]: - backup_dir = Path(tempfile.mkdtemp(prefix=f".{path.name}.backup.", dir=path.parent)) - backup_path = backup_dir / path.name - os.replace(path, backup_path) - return backup_dir, backup_path - - -def publish_artifacts(artifacts: list[tuple[Path, str, int, bool]], force: bool) -> None: - staged: list[tuple[Path, Path, bool]] = [] - published: list[Path] = [] - backups: list[tuple[Path, Path, Path]] = [] - try: - for path, content, mode, secret in artifacts: - staged.append((path, stage_file(path, content, mode), secret)) - for path, staged_path, secret in staged: - if force and os.path.lexists(path): - backup_dir, backup_path = backup_destination(path) - backups.append((path, backup_dir, backup_path)) - if force: - os.replace(staged_path, path) - else: - os.link(staged_path, path) - os.unlink(staged_path) - published.append(path) - print(f"wrote {path}" + (" (chmod 600 -- contains the k3s token)" if secret else "")) - except OSError as exc: - for path in reversed(published): - remove_destination(path) - for path, backup_dir, backup_path in reversed(backups): - remove_destination(path) - os.replace(backup_path, path) - backup_dir.rmdir() - die(f"could not publish generated artifacts: {exc}") - else: - for _, backup_dir, _ in backups: - shutil.rmtree(backup_dir) - finally: - for _, staged_path, _ in staged: - staged_path.unlink(missing_ok=True) - - def main(argv=None) -> int: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--spec", help="path to the cluster-spec JSON, or - for stdin") @@ -359,32 +82,34 @@ def main(argv=None) -> int: raw = sys.stdin.read() if args.spec == "-" else Path(args.spec).read_text(encoding="utf-8") try: - spec = json.loads(raw) - except json.JSONDecodeError as exc: + spec = strict_json_loads(raw) + except (DuplicateJsonKeyError, json.JSONDecodeError) as exc: die(f"spec is not valid JSON: {exc}") - if not isinstance(spec, dict): - die("spec must be a mapping") - topo = spec.get("topology") - if topo not in ("pxe-diskless", "ssh-preinstalled"): - die("spec.topology must be 'pxe-diskless' or 'ssh-preinstalled'") - require(spec, "k3s_version") - require(spec, "server.name") - require(spec, "server.ip") - validate_config_shapes(spec) - + topo = validate_spec(spec) if args.token_file: token = Path(args.token_file).read_text(encoding="utf-8").strip() - if not token: - die("--token-file is empty") + validate_yaml_scalar(token, "--token-file") else: token = gen_token() out = Path(args.out_dir) - artifacts = [(out / "inventory.yml", render_inventory(spec, token), 0o600, True)] + try: + discovery = discover_gpu_policy(spec, out) + except DiscoveryFailure as error: + die(str(error)) + inventory, values, manifest = canonical_paths(out) + artifacts = [(inventory, render_inventory(spec, token, discovery.resolution), 0o600, True)] + pxe_gpu_access_enabled = None if topo == "pxe-diskless": - artifacts.append((out / "pb-pxe-controller.vars.yml", render_pxe_vars(spec), 0o600, False)) - artifacts.append((out / "values-basic-example.yaml", render_values(spec), 0o644, False)) + pxe_gpu_access_enabled = spec["pxe"]["diskless_agents_have_amd_gpus"] + artifacts.append( + (out / "pb-pxe-controller.vars.yml", render_pxe_vars(spec, pxe_gpu_access_enabled), 0o600, True) + ) + artifacts += [ + (values, render_values(spec), 0o644, False), + (manifest, manifest_content(discovery, pxe_gpu_access_enabled), 0o644, False), + ] preflight_destinations([path for path, _, _, _ in artifacts], args.force) publish_artifacts(artifacts, args.force) diff --git a/skills/deploy-aup-learning-cloud/scripts/gpu_access_resolution.py b/skills/deploy-aup-learning-cloud/scripts/gpu_access_resolution.py new file mode 100644 index 00000000..555c456d --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/gpu_access_resolution.py @@ -0,0 +1,171 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +"""Parse read-only host evidence and resolve a safe fleet GPU-access policy.""" + +import json +import re +from dataclasses import dataclass +from enum import Enum +from typing import Final + +from config_common import DuplicateJsonKeyError, strict_json_loads +from gpu_resolution_manifest import ResolutionManifest, build_resolution_manifest + +EVIDENCE_VERSION: Final = 1 +BDF_PATTERN: Final = re.compile(r"[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[0-9a-fA-F]{2}\.[0-7]") + + +class HostStatus(str, Enum): + """Classify one inventory host from mutually corroborated discovery probes.""" + + GPU = "gpu" + CPU = "cpu" + UNKNOWN = "unknown" + + +class FleetStatus(str, Enum): + """Describe whether fleet evidence yields a publication-safe GPU policy.""" + + GPU_RESOLVED = "gpu_resolved" + CPU_ONLY = "cpu_only" + BLOCKED = "blocked" + + +@dataclass(frozen=True, slots=True) +class EvidenceParseError(ValueError): + """Raised when discovery JSON does not match the fixed evidence schema.""" + + field: str + + def __str__(self) -> str: + return f"Malformed GPU-access discovery evidence at {self.field}" + + +@dataclass(frozen=True, slots=True) +class InventoryTarget: + name: str + + +@dataclass(frozen=True, slots=True) +class CommandEvidence: + rc: int + stdout: str + + +@dataclass(frozen=True, slots=True) +class HostEvidence: + target: InventoryTarget + reachable: bool + lspci: CommandEvidence + sysfs: CommandEvidence + + +@dataclass(frozen=True, slots=True) +class HostResolution: + target: InventoryTarget + status: HostStatus + reason: str | None + + +@dataclass(frozen=True, slots=True) +class FleetResolution: + status: FleetStatus + hosts: tuple[HostResolution, ...] + reason: str | None + + +def parse_fleet_evidence(raw: str) -> tuple[HostEvidence, ...]: + """Parse the exact JSON emitted by the GPU-access discovery playbook.""" + try: + document = strict_json_loads(raw) + except DuplicateJsonKeyError as error: + raise EvidenceParseError(field=str(error)) from error + except (TypeError, json.JSONDecodeError) as error: + raise EvidenceParseError(field="document") from error + _require_mapping(document, "document") + if set(document) != {"version", "hosts"}: + raise EvidenceParseError(field="document") + if type(document["version"]) is not int or document["version"] != EVIDENCE_VERSION: + raise EvidenceParseError(field="version") + if type(document["hosts"]) is not list: + raise EvidenceParseError(field="hosts") + return tuple(_parse_host(item, f"hosts[{index}]") for index, item in enumerate(document["hosts"])) + + +def resolve_fleet(expected_targets: tuple[InventoryTarget, ...], evidence: tuple[HostEvidence, ...]) -> FleetResolution: + """Resolve a fleet only when complete evidence proves one safe policy.""" + resolutions = tuple(_resolve_host(host) for host in evidence) + expected_names = tuple(target.name for target in expected_targets) + actual_names = tuple(host.target.name for host in evidence) + if len(set(expected_names)) != len(expected_names) or len(set(actual_names)) != len(actual_names): + return _blocked(resolutions, "duplicate host") + if set(expected_names) != set(actual_names): + return _blocked(resolutions, "incomplete host coverage") + if any(host.status is HostStatus.UNKNOWN for host in resolutions): + return _blocked(resolutions, "unknown host evidence") + gpu_hosts = tuple(host for host in resolutions if host.status is HostStatus.GPU) + if not gpu_hosts: + return FleetResolution(FleetStatus.CPU_ONLY, resolutions, None) + return FleetResolution(FleetStatus.GPU_RESOLVED, resolutions, None) + + +def resolution_manifest(resolution: FleetResolution) -> ResolutionManifest: + """Build the public serialized manifest for a resolved fleet.""" + return build_resolution_manifest( + status=resolution.status.value, + hosts={host.target.name: host.status is HostStatus.GPU for host in resolution.hosts}, + ) + + +def _parse_host(raw, field: str) -> HostEvidence: + _require_mapping(raw, field) + required = {"host", "reachable", "lspci", "sysfs"} + if set(raw) != required or type(raw["host"]) is not str or not raw["host"]: + raise EvidenceParseError(field=field) + if type(raw["reachable"]) is not bool: + raise EvidenceParseError(field=f"{field}.reachable") + return HostEvidence( + target=InventoryTarget(name=raw["host"]), + reachable=raw["reachable"], + lspci=_parse_command(raw["lspci"], f"{field}.lspci"), + sysfs=_parse_command(raw["sysfs"], f"{field}.sysfs"), + ) + + +def _parse_command(raw, field: str) -> CommandEvidence: + _require_mapping(raw, field) + if set(raw) != {"rc", "stdout"} or type(raw["rc"]) is not int or type(raw["stdout"]) is not str: + raise EvidenceParseError(field=field) + return CommandEvidence(rc=raw["rc"], stdout=raw["stdout"]) + + +def _require_mapping(value, field: str) -> None: + if type(value) is not dict: + raise EvidenceParseError(field=field) + + +def _resolve_host(evidence: HostEvidence) -> HostResolution: + if not evidence.reachable or evidence.lspci.rc != 0 or evidence.sysfs.rc != 0: + return _unknown(evidence, "GPU discovery probe failed") + lspci_bdfs = _bdfs(evidence.lspci.stdout) + sysfs_bdfs = _bdfs(evidence.sysfs.stdout) + if lspci_bdfs is None or sysfs_bdfs is None or lspci_bdfs != sysfs_bdfs: + return _unknown(evidence, "AMD GPU BDF probes disagree") + if not lspci_bdfs: + return HostResolution(evidence.target, HostStatus.CPU, None) + return HostResolution(evidence.target, HostStatus.GPU, None) + + +def _bdfs(stdout: str) -> frozenset[str] | None: + bdfs = frozenset(line.split(maxsplit=1)[0] for line in stdout.splitlines()) + if all(BDF_PATTERN.fullmatch(bdf) for bdf in bdfs): + return bdfs + return None + + +def _unknown(evidence: HostEvidence, reason: str) -> HostResolution: + return HostResolution(evidence.target, HostStatus.UNKNOWN, reason) + + +def _blocked(hosts: tuple[HostResolution, ...], reason: str) -> FleetResolution: + return FleetResolution(FleetStatus.BLOCKED, hosts, reason) diff --git a/skills/deploy-aup-learning-cloud/scripts/gpu_artifact_generation.py b/skills/deploy-aup-learning-cloud/scripts/gpu_artifact_generation.py new file mode 100644 index 00000000..e16fac14 --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/gpu_artifact_generation.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +"""Discover live GPU facts and prepare publication-safe resolved artifacts.""" + +from __future__ import annotations + +import json +import os +import re +import stat +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import NoReturn + +from artifact_store import publish_artifacts +from config_common import yaml_quote +from config_generation import HEADER_HASH +from gpu_access_resolution import ( + EvidenceParseError, + FleetResolution, + FleetStatus, + InventoryTarget, + parse_fleet_evidence, + resolution_manifest, + resolve_fleet, +) +from gpu_resolution_manifest import build_pxe_resolution_manifest + +DISCOVERY_TIMEOUT_BASE_SECONDS = 30 +DISCOVERY_TIMEOUT_PER_TARGET_SECONDS = 15 +DISCOVERY_TIMEOUT_MAX_SECONDS = 300 +DISCOVERY_DIAGNOSTIC_MAX_CHARS = 1200 + + +def assert_never(value: FleetStatus) -> NoReturn: + raise AssertionError(f"unexpected fleet status: {value}") + + +@dataclass(frozen=True, slots=True) +class DiscoveryFailure(Exception): + reason: str + + def __str__(self) -> str: + return self.reason + + +@dataclass(frozen=True, slots=True) +class DiscoveryPaths: + inventory: Path + evidence: Path + + +@dataclass(frozen=True, slots=True) +class DiscoveryResult: + resolution: FleetResolution + + +def canonical_paths(out_dir: Path) -> tuple[Path, Path, Path]: + return ( + out_dir / "inventory.yml", + out_dir / "values-basic-example.yaml", + out_dir / "gpu-access-resolution.json", + ) + + +def discover_gpu_policy(spec: dict, out_dir: Path) -> DiscoveryResult: + targets = live_targets(spec) + paths = stage_private_discovery(spec, out_dir) + run_discovery(paths, len(targets)) + try: + evidence = parse_fleet_evidence(read_regular_file(paths.evidence)) + except EvidenceParseError as error: + raise DiscoveryFailure("GPU discovery evidence is malformed") from error + resolution = resolve_fleet(targets, evidence) + match resolution.status: + case FleetStatus.BLOCKED: + raise DiscoveryFailure(f"GPU discovery is blocked: {resolution.reason}") + case FleetStatus.GPU_RESOLVED | FleetStatus.CPU_ONLY: + pass + case unreachable: + assert_never(unreachable) + return DiscoveryResult(resolution=resolution) + + +def live_targets(spec: dict) -> tuple[InventoryTarget, ...]: + names = [spec["server"]["name"]] + if spec["topology"] == "ssh-preinstalled": + names.extend(agent["name"] for agent in spec.get("agents", [])) + if len(names) != len(set(names)): + raise DiscoveryFailure("live target names must be unique") + return tuple(InventoryTarget(name=name) for name in names) + + +def stage_private_discovery(spec: dict, out_dir: Path) -> DiscoveryPaths: + resolved_out_dir = out_dir.resolve() + paths = DiscoveryPaths( + inventory=resolved_out_dir / ".gpu-access-discovery.inventory.yml", + evidence=resolved_out_dir / ".gpu-access-discovery-evidence.json", + ) + publish_artifacts( + [ + (paths.inventory, render_discovery_inventory(spec), 0o600, False), + (paths.evidence, "", 0o600, False), + ], + force=True, + ) + return paths + + +def render_discovery_inventory(spec: dict) -> str: + server = spec["server"] + lines = [ + HEADER_HASH, + "k3s_cluster:", + " children:", + " server:", + " hosts:", + f" {server['name']}:", + f" ansible_host: {yaml_quote(server['ip'])}", + " agent:", + ] + if spec["topology"] == "ssh-preinstalled" and spec.get("agents"): + lines.append(" hosts:") + for agent in spec["agents"]: + lines += [f" {agent['name']}:", f" ansible_host: {yaml_quote(agent['ip'])}"] + else: + lines.append(" hosts: {}") + lines += [" vars:", " ansible_port: 22", " ansible_user: root"] + return "\n".join(lines) + "\n" + + +def discovery_timeout_seconds(target_count: int) -> int: + configured = os.environ.get("AUPLC_GPU_DISCOVERY_TIMEOUT_SECONDS") + if configured is not None: + try: + timeout = int(configured) + except ValueError as error: + raise DiscoveryFailure("AUPLC_GPU_DISCOVERY_TIMEOUT_SECONDS must be an integer") from error + if not DISCOVERY_TIMEOUT_BASE_SECONDS <= timeout <= DISCOVERY_TIMEOUT_MAX_SECONDS: + raise DiscoveryFailure( + f"AUPLC_GPU_DISCOVERY_TIMEOUT_SECONDS must be between {DISCOVERY_TIMEOUT_BASE_SECONDS} and " + f"{DISCOVERY_TIMEOUT_MAX_SECONDS}" + ) + return timeout + return min( + DISCOVERY_TIMEOUT_MAX_SECONDS, + DISCOVERY_TIMEOUT_BASE_SECONDS + (DISCOVERY_TIMEOUT_PER_TARGET_SECONDS * target_count), + ) + + +def _bounded_diagnostic(*values: str | bytes | None) -> str: + text = "\n".join(value.decode(errors="replace") if isinstance(value, bytes) else value or "" for value in values) + text = re.sub(r"(?i)\b(token|password|secret|private[_-]?key)\s*[:=]\s*\S+", r"\1=", text) + lines = [line.strip() for line in text.splitlines() if line.strip()] + summary = " | ".join(lines[-8:]) + return summary[-DISCOVERY_DIAGNOSTIC_MAX_CHARS:] or "no Ansible diagnostics" + + +def run_discovery(paths: DiscoveryPaths, target_count: int) -> None: + playbook = Path(__file__).resolve().parents[3] / "deploy" / "ansible" / "playbooks" / "pb-gpu-access-discovery.yml" + argv = [ + "ansible-playbook", + "-i", + str(paths.inventory), + str(playbook), + "-e", + f"gpu_access_discovery_output_path={paths.evidence}", + ] + environment = os.environ.copy() + environment["ANSIBLE_CONFIG"] = str(playbook.parents[1] / "ansible.cfg") + environment["ANSIBLE_HOST_KEY_CHECKING"] = "True" + environment["ANSIBLE_SSH_HOST_KEY_CHECKING"] = "True" + environment["ANSIBLE_SSH_ARGS"] = "-o StrictHostKeyChecking=yes" + for key in ( + "ANSIBLE_SSH_COMMON_ARGS", + "ANSIBLE_SSH_EXTRA_ARGS", + "ANSIBLE_SCP_IF_SSH", + "ANSIBLE_SCP_EXTRA_ARGS", + "ANSIBLE_SFTP_EXTRA_ARGS", + ): + environment.pop(key, None) + timeout = discovery_timeout_seconds(target_count) + try: + result = subprocess.run( + argv, + capture_output=True, + check=False, + cwd=playbook.parents[1], + env=environment, + text=True, + timeout=timeout, + ) + except FileNotFoundError as error: + raise DiscoveryFailure("ansible-playbook is required for GPU discovery") from error + except subprocess.TimeoutExpired as error: + diagnostic = _bounded_diagnostic(error.stderr, error.stdout) + raise DiscoveryFailure(f"GPU discovery playbook timed out after {timeout}s: {diagnostic}") from error + if result.returncode != 0: + diagnostic = _bounded_diagnostic(result.stderr, result.stdout) + raise DiscoveryFailure(f"GPU discovery playbook failed with exit code {result.returncode}: {diagnostic}") + + +def read_regular_file(path: Path) -> str: + try: + mode = os.lstat(path).st_mode + except FileNotFoundError as error: + raise DiscoveryFailure("GPU discovery evidence was not written") from error + if not stat.S_ISREG(mode): + raise DiscoveryFailure("GPU discovery evidence must be a regular file") + try: + return path.read_text(encoding="utf-8") + except OSError as error: + raise DiscoveryFailure("GPU discovery evidence could not be read") from error + + +def manifest_content(result: DiscoveryResult, pxe_gpu_access_enabled: bool | None = None) -> str: + base = resolution_manifest(result.resolution) + document = ( + base + if pxe_gpu_access_enabled is None + else build_pxe_resolution_manifest( + base, + gpu_access_enabled=pxe_gpu_access_enabled, + ) + ) + return json.dumps(document, indent=2, sort_keys=True) + "\n" diff --git a/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_manifest.py b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_manifest.py new file mode 100644 index 00000000..e80d0eaf --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_manifest.py @@ -0,0 +1,57 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +"""Typed GPU-resolution manifest schemas and primitive builders.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Final, TypedDict + +MANIFEST_VERSION: Final = 1 + + +class ResolutionManifest(TypedDict): + """Serialized fleet GPU-resolution evidence.""" + + version: int + status: str + hosts: dict[str, bool] + + +class PxeRootfsManifest(TypedDict): + """Serialized GPU policy applied to the PXE root filesystem.""" + + gpu_access_enabled: bool + + +class PxeResolutionManifest(ResolutionManifest): + """Serialized fleet resolution with its PXE rootfs policy.""" + + pxe_rootfs: PxeRootfsManifest + + +def build_resolution_manifest( + *, + status: str, + hosts: Mapping[str, bool], +) -> ResolutionManifest: + """Build a deterministic ordinary dictionary for fleet resolution.""" + return { + "version": MANIFEST_VERSION, + "status": status, + "hosts": {name: hosts[name] for name in sorted(hosts)}, + } + + +def build_pxe_resolution_manifest( + resolution: ResolutionManifest, + *, + gpu_access_enabled: bool, +) -> PxeResolutionManifest: + """Build a PXE manifest without mutating a base fleet manifest.""" + return { + **resolution, + "pxe_rootfs": { + "gpu_access_enabled": gpu_access_enabled, + }, + } diff --git a/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_parsing.py b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_parsing.py new file mode 100644 index 00000000..eb5a38ec --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_parsing.py @@ -0,0 +1,190 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +import re +from dataclasses import dataclass +from pathlib import Path + +from config_common import DuplicateJsonKeyError, strict_json_loads +from gpu_resolution_manifest import MANIFEST_VERSION + + +@dataclass(frozen=True, slots=True) +class GpuInventory: + hosts: dict[str, bool] + + +@dataclass(frozen=True, slots=True) +class GpuInventoryHostScalars: + hosts: dict[str, str] + + +@dataclass(frozen=True, slots=True) +class GpuResolution: + status: str + hosts: dict[str, bool] + pxe_rootfs_enabled: bool | None + + +@dataclass(frozen=True, slots=True) +class PxeGpuPolicy: + enabled: bool + + +def configured_path(repo: Path, value: str) -> Path: + path = Path(value).expanduser() + return path if path.is_absolute() else repo / path + + +def parse_gpu_boolean(value: str) -> bool | None: + normalized = value.strip() + if normalized == "true": + return True + if normalized == "false": + return False + return None + + +def yaml_indent(line: str) -> int: + return len(line) - len(line.lstrip()) + + +def scan_gpu_inventory_host_scalars(text: str) -> tuple[GpuInventoryHostScalars | None, list[str]]: + host_values: dict[str, list[str]] = {} + host_names: list[str] = [] + stack: list[tuple[int, str]] = [] + + for raw_line in text.splitlines(): + line = raw_line.split("#", 1)[0].rstrip() + if not line.strip(): + continue + indent = yaml_indent(line) + stripped = line.strip() + while stack and indent <= stack[-1][0]: + stack.pop() + path = tuple(key for _, key in stack) + mapping_match = re.fullmatch(r"(.+?):(?:\s*(.*))?", stripped) + if not mapping_match: + continue + key = mapping_match.group(1).strip("\"'") + value = (mapping_match.group(2) or "").strip() + if len(path) == 4 and path[:4] in { + ("k3s_cluster", "children", "server", "hosts"), + ("k3s_cluster", "children", "agent", "hosts"), + }: + host_names.append(key) + host_values.setdefault(key, []) + elif ( + len(path) == 5 + and path[:4] + in { + ("k3s_cluster", "children", "server", "hosts"), + ("k3s_cluster", "children", "agent", "hosts"), + } + and key == "auplc_gpu_access_enabled" + ): + host_values.setdefault(path[4], []).append(value) + stack.append((indent, key)) + + parse_errors: list[str] = [] + if not host_names: + parse_errors.append("inventory has no generated k3s server or agent hosts") + if len(set(host_names)) != len(host_names): + parse_errors.append("inventory has duplicate generated host names") + hosts: dict[str, str] = {} + for host in host_names: + values = host_values[host] + if len(values) != 1: + parse_errors.append(f"inventory host '{host}' must define exactly one auplc_gpu_access_enabled") + continue + hosts[host] = values[0] + if parse_errors: + return None, parse_errors + return GpuInventoryHostScalars(hosts=hosts), [] + + +def validate_direct_gpu_inventory(text: str) -> list[str]: + host_scalars, parse_errors = scan_gpu_inventory_host_scalars(text) + if host_scalars is None: + return parse_errors + for host, value in host_scalars.hosts.items(): + if value not in {"auto", "true", "false"}: + parse_errors.append(f"inventory host '{host}' has malformed auplc_gpu_access_enabled") + return parse_errors + + +def parse_gpu_inventory(text: str) -> tuple[GpuInventory | None, list[str]]: + host_scalars, parse_errors = scan_gpu_inventory_host_scalars(text) + if host_scalars is None: + return None, parse_errors + hosts: dict[str, bool] = {} + for host, value in host_scalars.hosts.items(): + enabled = parse_gpu_boolean(value) + if enabled is None: + parse_errors.append(f"inventory host '{host}' has malformed auplc_gpu_access_enabled") + continue + hosts[host] = enabled + if parse_errors: + return None, parse_errors + return GpuInventory(hosts=hosts), [] + + +def parse_gpu_resolution(text: str, topology: str) -> tuple[GpuResolution | None, list[str]]: + try: + document = strict_json_loads(text) + except DuplicateJsonKeyError as exc: + return None, [f"GPU resolution manifest is malformed: {exc}"] + except (TypeError, ValueError) as exc: + return None, [f"GPU resolution manifest is malformed: {exc}"] + if type(document) is not dict: + return None, ["GPU resolution manifest must be a JSON object"] + expected_keys = {"version", "status", "hosts"} + if topology == "pxe-diskless": + expected_keys.add("pxe_rootfs") + if set(document) != expected_keys: + return None, ["GPU resolution manifest has an unexpected schema"] + if type(document["version"]) is not int or document["version"] != MANIFEST_VERSION: + return None, [f"GPU resolution manifest version must be integer {MANIFEST_VERSION}"] + status = document["status"] + if type(status) is not str or status not in {"cpu_only", "gpu_resolved"}: + return None, ["GPU resolution manifest status must be cpu_only or gpu_resolved"] + if type(document["hosts"]) is not dict or not document["hosts"]: + return None, ["GPU resolution manifest hosts must be a non-empty object"] + if any( + type(host) is not str or not host or type(enabled) is not bool for host, enabled in document["hosts"].items() + ): + return None, ["GPU resolution manifest hosts must map non-empty names to booleans"] + if topology == "ssh-preinstalled": + return GpuResolution(status, document["hosts"], None), [] + rootfs = document["pxe_rootfs"] + if type(rootfs) is not dict or set(rootfs) != {"gpu_access_enabled"}: + return None, ["GPU resolution manifest pxe_rootfs has an unexpected schema"] + rootfs_enabled = rootfs["gpu_access_enabled"] + if type(rootfs_enabled) is not bool: + return None, ["GPU resolution manifest pxe_rootfs.gpu_access_enabled must be boolean"] + return GpuResolution(status, document["hosts"], rootfs_enabled), [] + + +def parse_pxe_gpu_policy(text: str) -> tuple[PxeGpuPolicy | None, list[str]]: + values: dict[str, list[str]] = {"pxe_gpu_access_enabled": []} + for raw_line in text.splitlines(): + line = raw_line.split("#", 1)[0].rstrip() + if not line.strip() or yaml_indent(line) != 0: + continue + mapping_match = re.fullmatch(r"(.+?):(?:\s*(.*))?", line.strip()) + if not mapping_match: + continue + key = mapping_match.group(1).strip("\"'") + if key in values: + values[key].append((mapping_match.group(2) or "").strip()) + parse_errors: list[str] = [] + for key, occurrences in values.items(): + if len(occurrences) != 1: + parse_errors.append(f"PXE vars must define exactly one {key}") + if parse_errors: + return None, parse_errors + enabled = parse_gpu_boolean(values["pxe_gpu_access_enabled"][0]) + if enabled is None: + parse_errors.append("PXE vars have malformed pxe_gpu_access_enabled") + if parse_errors: + return None, parse_errors + return PxeGpuPolicy(enabled=enabled), [] diff --git a/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_validation.py b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_validation.py new file mode 100644 index 00000000..b3c3061f --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_validation.py @@ -0,0 +1,119 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +from dataclasses import dataclass +from pathlib import Path + +from gpu_resolution_parsing import ( + configured_path, + parse_gpu_inventory, + parse_gpu_resolution, + parse_pxe_gpu_policy, + validate_direct_gpu_inventory, +) + + +@dataclass(frozen=True, slots=True) +class GpuArtifactValidationRequest: + repo: Path + inventory_path: str + resolution_path: str + topology: str + pxe_vars_path: Path + has_prior_errors: bool + + +@dataclass(frozen=True, slots=True) +class GpuArtifactValidationResult: + errors: list[str] + passed: list[str] + + +@dataclass(frozen=True, slots=True) +class AcceleratorValidationResult: + errors: list[str] + warnings: list[str] + passed: list[str] + + +def check_gpu_inventory(repo: Path, inventory_path: str) -> GpuArtifactValidationResult: + inventory_file = configured_path(repo, inventory_path) + if not inventory_file.exists(): + return GpuArtifactValidationResult([f"inventory not found: {inventory_file}"], []) + errors = validate_direct_gpu_inventory(inventory_file.read_text(encoding="utf-8")) + return GpuArtifactValidationResult(errors, [] if errors else ["GPU access inventory is valid"]) + + +def check_accelerator_labels( + accelerators: dict[str, str], metadata: dict[str, list[str]], cluster: dict | None +) -> AcceleratorValidationResult: + errors: list[str] = [] + warnings: list[str] = [] + passed: list[str] = [] + active_keys = sorted({key for keys in metadata.values() for key in keys}) + if not active_keys: + return AcceleratorValidationResult([], ["no acceleratorKeys found in effective custom.resources.metadata"], []) + declared: list[str] = [] + for key in active_keys: + if key not in accelerators: + errors.append(f"active accelerator '{key}' is not defined under custom.accelerators") + elif not accelerators[key]: + errors.append(f"active accelerator '{key}' has no amd.com/gpu.product-name nodeSelector") + else: + declared.append(accelerators[key]) + if not declared: + return AcceleratorValidationResult(errors, warnings, passed) + if cluster is None: + warnings.append( + "no --cluster snapshot; cannot confirm nodeSelector labels match real " + f"nodes. Declared: {', '.join(declared)}" + ) + return AcceleratorValidationResult(errors, warnings, passed) + real = set(cluster.get("gpu_product_names", [])) + if not real: + errors.append("cluster snapshot has no GPU product labels for active accelerators") + return AcceleratorValidationResult(errors, warnings, passed) + for declared_label in declared: + if declared_label in real: + passed.append(f"nodeSelector '{declared_label}' matches a real node label") + else: + errors.append( + f"nodeSelector '{declared_label}' matches no node label. Real labels: {', '.join(sorted(real))}" + ) + return AcceleratorValidationResult(errors, warnings, passed) + + +def check_gpu_artifacts(request: GpuArtifactValidationRequest) -> GpuArtifactValidationResult: + errors: list[str] = [] + inventory_file = configured_path(request.repo, request.inventory_path) + resolution_file = configured_path(request.repo, request.resolution_path) + if not inventory_file.exists(): + return GpuArtifactValidationResult([f"generated inventory not found: {inventory_file}"], []) + if not resolution_file.exists(): + return GpuArtifactValidationResult([f"GPU resolution manifest not found: {resolution_file}"], []) + inventory, inventory_errors = parse_gpu_inventory(inventory_file.read_text(encoding="utf-8")) + resolution, resolution_errors = parse_gpu_resolution(resolution_file.read_text(encoding="utf-8"), request.topology) + errors.extend([*inventory_errors, *resolution_errors]) + if inventory is None or resolution is None or errors: + return GpuArtifactValidationResult(errors, []) + if set(inventory.hosts) != set(resolution.hosts): + errors.append("inventory hosts do not exactly match GPU resolution manifest hosts") + for host, enabled in inventory.hosts.items(): + if resolution.hosts.get(host) != enabled: + errors.append(f"inventory host '{host}' GPU access boolean disagrees with the resolution manifest") + pxe_policy = None + if request.topology == "pxe-diskless": + if not request.pxe_vars_path.exists(): + return GpuArtifactValidationResult([*errors, f"PXE vars file not found: {request.pxe_vars_path}"], []) + pxe_policy, pxe_errors = parse_pxe_gpu_policy(request.pxe_vars_path.read_text(encoding="utf-8")) + errors.extend(pxe_errors) + if pxe_policy is None or pxe_errors: + return GpuArtifactValidationResult(errors, []) + if pxe_policy.enabled != resolution.pxe_rootfs_enabled: + errors.append("PXE pxe_gpu_access_enabled disagrees with GPU resolution manifest pxe_rootfs") + if resolution.status == "cpu_only": + if any(resolution.hosts.values()): + errors.append("cpu_only GPU resolution requires all host booleans false") + elif not any(resolution.hosts.values()): + errors.append("gpu_resolved GPU resolution requires an enabled host") + passed = [] if request.has_prior_errors or errors else ["GPU access artifacts agree"] + return GpuArtifactValidationResult(errors, passed) diff --git a/skills/deploy-aup-learning-cloud/scripts/helm_validation.py b/skills/deploy-aup-learning-cloud/scripts/helm_validation.py new file mode 100644 index 00000000..cd1c4e39 --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/helm_validation.py @@ -0,0 +1,37 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +import shutil +import subprocess +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + +CHART = "runtime/chart" + + +@dataclass(frozen=True, slots=True) +class HelmValidationReporter: + ok: Callable[[str], None] + warn: Callable[[str], None] + fail: Callable[[str], None] + + +def check_helm(repo: Path, values: list[str], reporter: HelmValidationReporter) -> None: + if not shutil.which("helm"): + reporter.warn("helm not on PATH; skipped chart dry-run") + return + chart = repo / CHART + if not chart.exists(): + reporter.warn(f"chart not found at {CHART}; skipped dry-run") + return + cmd = ["helm", "template", "jupyterhub", str(chart)] + for rel in values or ["runtime/values.yaml"]: + path = (repo / rel) if not Path(rel).is_absolute() else Path(rel) + if path.exists(): + cmd += ["-f", str(path)] + proc = subprocess.run(cmd, capture_output=True, text=True) + if proc.returncode == 0: + reporter.ok("helm template rendered the chart successfully") + else: + tail = (proc.stderr or proc.stdout).strip().splitlines()[-5:] + reporter.fail("helm template failed:\n " + "\n ".join(tail)) diff --git a/skills/deploy-aup-learning-cloud/scripts/validate.py b/skills/deploy-aup-learning-cloud/scripts/validate.py index 8408f020..934e3ea7 100755 --- a/skills/deploy-aup-learning-cloud/scripts/validate.py +++ b/skills/deploy-aup-learning-cloud/scripts/validate.py @@ -12,6 +12,9 @@ * nodeSelectors for the accelerators actually referenced by effective custom.resources.metadata.*.acceleratorKeys, checked against detect_cluster.sh output when supplied; + * direct SSH inventory GPU access values are exact unquoted `auto`, `true`, + or `false`; generated inventory, GPU-resolution manifest, and PXE rootfs + policy agree when both artifacts are supplied; * (optional) the chart does not render: a `helm template` dry-run. This intentionally uses regex/line scanning rather than a YAML parser so it @@ -20,9 +23,10 @@ inspect something. Usage: - validate.py --repo ~/aup-learning-cloud --topology pxe-diskless validate.py --repo ~/aup-learning-cloud \ --topology ssh-preinstalled \ + --inventory generated/inventory.yml \ + --gpu-resolution generated/gpu-access-resolution.json \ --values runtime/values.yaml --values runtime/values-basic-example.yaml \ --cluster cluster.json --helm-dry-run @@ -30,19 +34,24 @@ 2 on a usage error. """ -from __future__ import annotations - import argparse import json import re -import shutil -import subprocess import sys from pathlib import Path +from config_common import DuplicateJsonKeyError, strict_json_loads +from gpu_resolution_validation import ( + GpuArtifactValidationRequest, + check_accelerator_labels, + check_gpu_artifacts, + check_gpu_inventory, +) +from helm_validation import HelmValidationReporter, check_helm +from values_resolution_parsing import collect_effective_values + PXE_PLAYBOOK = "deploy/ansible/playbooks/pb-pxe-controller.yml" INVENTORY = "deploy/ansible/inventory.yml" -CHART = "runtime/chart" errors: list[str] = [] warnings: list[str] = [] @@ -163,176 +172,6 @@ def check_version_sync(repo: Path, configured_path: str | None = None) -> None: ) -def yaml_scalar(value: str) -> str: - return value.strip().strip('"').strip("'") - - -def yaml_optional_scalar(value: str) -> str: - scalar_value = yaml_scalar(value) - return "" if scalar_value in {"", "null", "~"} else scalar_value - - -def yaml_indent(line: str) -> int: - return len(line) - len(line.lstrip()) - - -def parse_inline_list(value: str) -> list[str]: - items = value.strip()[1:-1].strip() - if not items: - return [] - return [yaml_scalar(item) for item in items.split(",") if yaml_scalar(item)] - - -def is_relevant_flow_path(path: tuple[str, ...]) -> bool: - return path == ("custom",) or path[:2] in {("custom", "accelerators"), ("custom", "resources")} - - -def unsupported_yaml_syntax(value: str) -> bool: - return value.startswith(("&", "*", "!", "|", ">")) - - -def parse_values_file(text: str) -> tuple[dict[str, str | None], dict[str, list[str]], list[str]]: - """Extract the deploy-relevant mappings from a fixed-shape values YAML file. - - The helpers deliberately remain stdlib-only. This scanner handles the - mapping/list shapes used by values overlays, rather than pretending to be a - general YAML parser. - """ - accelerators: dict[str, str | None] = {} - metadata: dict[str, list[str]] = {} - parse_errors: list[str] = [] - stack: list[tuple[int, str]] = [] - - for raw_line in text.splitlines(): - line = raw_line.split("#", 1)[0].rstrip() - if not line.strip(): - continue - indent = yaml_indent(line) - stripped = line.strip() - - while stack and indent <= stack[-1][0]: - stack.pop() - path = tuple(key for _, key in stack) - - if stripped.startswith("- "): - if len(path) == 5 and path[:3] == ("custom", "resources", "metadata") and path[-1] == "acceleratorKeys": - metadata.setdefault(path[3], []).append(yaml_scalar(stripped[2:])) - continue - - product_label_match = re.fullmatch( - r"(?:[\"']amd\.com/gpu\.product-name[\"']|amd\.com/gpu\.product-name):\s*(.*)", stripped - ) - if product_label_match: - if len(path) == 4 and path[:2] == ("custom", "accelerators") and path[-1] == "nodeSelector": - value = product_label_match.group(1).strip() - if unsupported_yaml_syntax(value): - parse_errors.append( - f"unsupported YAML syntax at custom.accelerators.{path[2]}.nodeSelector.amd.com/gpu.product-name" - ) - else: - accelerators[path[2]] = yaml_optional_scalar(value) - continue - - mapping_match = re.fullmatch(r"(.+?):(?:\s*(.*))?", stripped) - if not mapping_match: - continue - key = mapping_match.group(1).strip("\"'") - value = (mapping_match.group(2) or "").strip() - candidate_path = path + (key,) - if value.startswith("{") and value != "{}" and is_relevant_flow_path(candidate_path): - parse_errors.append(f"unsupported non-empty flow-style mapping at {'.'.join(candidate_path)}") - if unsupported_yaml_syntax(value) and is_relevant_flow_path(candidate_path): - parse_errors.append(f"unsupported YAML syntax at {'.'.join(candidate_path)}") - if path == ("custom", "accelerators"): - accelerators.setdefault(key, None) - if len(path) == 4 and path[:3] == ("custom", "resources", "metadata") and key == "acceleratorKeys": - resource_key = path[3] - if unsupported_yaml_syntax(value): - parse_errors.append(f"unsupported YAML syntax at {'.'.join(candidate_path)}") - elif value.startswith("[") and value.endswith("]"): - metadata[resource_key] = parse_inline_list(value) - elif not value or value in {"null", "~"}: - metadata[resource_key] = [] - else: - parse_errors.append(f"acceleratorKeys must be a list at {'.'.join(candidate_path)}") - stack.append((indent, key)) - return accelerators, metadata, parse_errors - - -def collect_effective_values(repo: Path, values: list[str]) -> tuple[dict[str, str], dict[str, list[str]], list[str]]: - paths = values or ["runtime/values.yaml"] - accelerators: dict[str, str] = {} - metadata: dict[str, list[str]] = {} - parse_errors: list[str] = [] - for rel in paths: - p = (repo / rel) if not Path(rel).is_absolute() else Path(rel) - if p.exists(): - parsed_accelerators, parsed_metadata, file_errors = parse_values_file(p.read_text(encoding="utf-8")) - for key, selector in parsed_accelerators.items(): - if selector is not None or key not in accelerators: - accelerators[key] = selector - metadata.update(parsed_metadata) - parse_errors.extend(file_errors) - else: - fail(f"values file not found: {rel}") - return accelerators, metadata, parse_errors - - -def check_accelerator_labels( - accelerators: dict[str, str], metadata: dict[str, list[str]], cluster: dict | None -) -> None: - active_keys = sorted({key for keys in metadata.values() for key in keys}) - if not active_keys: - warn("no acceleratorKeys found in effective custom.resources.metadata") - return - declared: list[str] = [] - for key in active_keys: - if key not in accelerators: - fail(f"active accelerator '{key}' is not defined under custom.accelerators") - elif not accelerators[key]: - fail(f"active accelerator '{key}' has no amd.com/gpu.product-name nodeSelector") - else: - declared.append(accelerators[key]) - if not declared: - return - if cluster is None: - warn( - "no --cluster snapshot; cannot confirm nodeSelector labels match real " - f"nodes. Declared: {', '.join(declared)}" - ) - return - real = set(cluster.get("gpu_product_names", [])) - if not real: - fail("cluster snapshot has no GPU product labels for active accelerators") - return - for d in declared: - if d in real: - ok(f"nodeSelector '{d}' matches a real node label") - else: - fail(f"nodeSelector '{d}' matches no node label. Real labels: {', '.join(sorted(real))}") - - -def check_helm(repo: Path, values: list[str]) -> None: - if not shutil.which("helm"): - warn("helm not on PATH; skipped chart dry-run") - return - chart = repo / CHART - if not chart.exists(): - warn(f"chart not found at {CHART}; skipped dry-run") - return - cmd = ["helm", "template", "jupyterhub", str(chart)] - for rel in values or ["runtime/values.yaml"]: - p = (repo / rel) if not Path(rel).is_absolute() else Path(rel) - if p.exists(): - cmd += ["-f", str(p)] - proc = subprocess.run(cmd, capture_output=True, text=True) - if proc.returncode == 0: - ok("helm template rendered the chart successfully") - else: - tail = (proc.stderr or proc.stdout).strip().splitlines()[-5:] - fail("helm template failed:\n " + "\n ".join(tail)) - - def main(argv=None) -> int: global errors, passed, warnings errors = [] @@ -353,6 +192,13 @@ def main(argv=None) -> int: "--pxe-vars", help="PXE vars file to validate instead of deploy/ansible/playbooks/pb-pxe-controller.yml", ) + ap.add_argument( + "--inventory", + help="inventory.yml for direct ssh-preinstalled validation (auto/true/false) or generated checks; pxe requires --gpu-resolution", + ) + ap.add_argument( + "--gpu-resolution", help="generated gpu-access-resolution.json; requires --inventory for consistency checks" + ) ap.add_argument("--cluster", help="detect_cluster.sh JSON output to match labels against") ap.add_argument("--helm-dry-run", action="store_true", help="also run `helm template`") ap.add_argument("--json", action="store_true", help="emit a JSON report instead of text") @@ -366,8 +212,8 @@ def main(argv=None) -> int: cluster = None if args.cluster: try: - cluster = json.loads(Path(args.cluster).read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as exc: + cluster = strict_json_loads(Path(args.cluster).read_text(encoding="utf-8")) + except (DuplicateJsonKeyError, OSError, json.JSONDecodeError) as exc: print(f"validate: cannot read --cluster: {exc}", file=sys.stderr) return 2 @@ -376,12 +222,46 @@ def main(argv=None) -> int: check_version_sync(repo, args.pxe_vars) else: ok("skipped PXE checks for ssh-preinstalled topology") - accelerators, metadata, parse_errors = collect_effective_values(repo, args.values) - for message in parse_errors: + values_result = collect_effective_values(repo, args.values) + for message in values_result.missing_files: + fail(message) + for message in values_result.parse_errors: + fail(message) + accelerator_result = check_accelerator_labels(values_result.accelerators, values_result.metadata, cluster) + for message in accelerator_result.errors: fail(message) - check_accelerator_labels(accelerators, metadata, cluster) + for message in accelerator_result.warnings: + warn(message) + for message in accelerator_result.passed: + ok(message) + if args.gpu_resolution and not args.inventory: + fail("--gpu-resolution requires --inventory") + elif args.inventory and not args.gpu_resolution: + if args.topology == "pxe-diskless": + fail("pxe-diskless inventory validation requires --gpu-resolution") + else: + inventory_result = check_gpu_inventory(repo, args.inventory) + for message in inventory_result.errors: + fail(message) + for message in inventory_result.passed: + ok(message) + elif args.inventory and args.gpu_resolution: + artifact_result = check_gpu_artifacts( + GpuArtifactValidationRequest( + repo=repo, + inventory_path=args.inventory, + resolution_path=args.gpu_resolution, + topology=args.topology, + pxe_vars_path=pxe_vars_path(repo, args.pxe_vars), + has_prior_errors=bool(errors), + ) + ) + for message in artifact_result.errors: + fail(message) + for message in artifact_result.passed: + ok(message) if args.helm_dry_run: - check_helm(repo, args.values) + check_helm(repo, args.values, HelmValidationReporter(ok=ok, warn=warn, fail=fail)) if args.json: print( diff --git a/skills/deploy-aup-learning-cloud/scripts/values_resolution_parsing.py b/skills/deploy-aup-learning-cloud/scripts/values_resolution_parsing.py new file mode 100644 index 00000000..9836162d --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/values_resolution_parsing.py @@ -0,0 +1,130 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +"""Fixed-shape parsing and overlay resolution for deploy values files.""" + +import re +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True, slots=True) +class ValuesFileParseResult: + accelerators: dict[str, str | None] + metadata: dict[str, list[str]] + parse_errors: list[str] + + +@dataclass(frozen=True, slots=True) +class EffectiveValuesResult: + accelerators: dict[str, str] + metadata: dict[str, list[str]] + missing_files: list[str] + parse_errors: list[str] + + +def yaml_scalar(value: str) -> str: + return value.strip().strip('"').strip("'") + + +def yaml_optional_scalar(value: str) -> str: + scalar_value = yaml_scalar(value) + return "" if scalar_value in {"", "null", "~"} else scalar_value + + +def yaml_indent(line: str) -> int: + return len(line) - len(line.lstrip()) + + +def parse_inline_list(value: str) -> list[str]: + items = value.strip()[1:-1].strip() + if not items: + return [] + return [yaml_scalar(item) for item in items.split(",") if yaml_scalar(item)] + + +def is_relevant_flow_path(path: tuple[str, ...]) -> bool: + return path == ("custom",) or path[:2] in {("custom", "accelerators"), ("custom", "resources")} + + +def unsupported_yaml_syntax(value: str) -> bool: + return value.startswith(("&", "*", "!", "|", ">")) + + +def parse_values_file(text: str) -> ValuesFileParseResult: + accelerators: dict[str, str | None] = {} + metadata: dict[str, list[str]] = {} + parse_errors: list[str] = [] + stack: list[tuple[int, str]] = [] + + for raw_line in text.splitlines(): + line = raw_line.split("#", 1)[0].rstrip() + if not line.strip(): + continue + indent = yaml_indent(line) + stripped = line.strip() + + while stack and indent <= stack[-1][0]: + stack.pop() + path = tuple(key for _, key in stack) + + if stripped.startswith("- "): + if len(path) == 5 and path[:3] == ("custom", "resources", "metadata") and path[-1] == "acceleratorKeys": + metadata.setdefault(path[3], []).append(yaml_scalar(stripped[2:])) + continue + + product_label_match = re.fullmatch( + r"(?:[\"']amd\.com/gpu\.product-name[\"']|amd\.com/gpu\.product-name):\s*(.*)", stripped + ) + if product_label_match: + if len(path) == 4 and path[:2] == ("custom", "accelerators") and path[-1] == "nodeSelector": + value = product_label_match.group(1).strip() + if unsupported_yaml_syntax(value): + parse_errors.append( + f"unsupported YAML syntax at custom.accelerators.{path[2]}.nodeSelector.amd.com/gpu.product-name" + ) + else: + accelerators[path[2]] = yaml_optional_scalar(value) + continue + + mapping_match = re.fullmatch(r"(.+?):(?:\s*(.*))?", stripped) + if not mapping_match: + continue + key = mapping_match.group(1).strip("\"'") + value = (mapping_match.group(2) or "").strip() + candidate_path = path + (key,) + if value.startswith("{") and value != "{}" and is_relevant_flow_path(candidate_path): + parse_errors.append(f"unsupported non-empty flow-style mapping at {'.'.join(candidate_path)}") + if unsupported_yaml_syntax(value) and is_relevant_flow_path(candidate_path): + parse_errors.append(f"unsupported YAML syntax at {'.'.join(candidate_path)}") + if path == ("custom", "accelerators"): + accelerators.setdefault(key, None) + if len(path) == 4 and path[:3] == ("custom", "resources", "metadata") and key == "acceleratorKeys": + resource_key = path[3] + if unsupported_yaml_syntax(value): + parse_errors.append(f"unsupported YAML syntax at {'.'.join(candidate_path)}") + elif value.startswith("[") and value.endswith("]"): + metadata[resource_key] = parse_inline_list(value) + elif not value or value in {"null", "~"}: + metadata[resource_key] = [] + else: + parse_errors.append(f"acceleratorKeys must be a list at {'.'.join(candidate_path)}") + stack.append((indent, key)) + return ValuesFileParseResult(accelerators, metadata, parse_errors) + + +def collect_effective_values(repo: Path, values: list[str]) -> EffectiveValuesResult: + accelerators: dict[str, str] = {} + metadata: dict[str, list[str]] = {} + missing_files: list[str] = [] + parse_errors: list[str] = [] + for rel in values or ["runtime/values.yaml"]: + path = (repo / rel) if not Path(rel).is_absolute() else Path(rel) + if not path.exists(): + missing_files.append(f"values file not found: {rel}") + continue + parsed = parse_values_file(path.read_text(encoding="utf-8")) + for key, selector in parsed.accelerators.items(): + if selector is not None or key not in accelerators: + accelerators[key] = selector + metadata.update(parsed.metadata) + parse_errors.extend(parsed.parse_errors) + return EffectiveValuesResult(accelerators, metadata, missing_files, parse_errors) diff --git a/skills/install-aup-learning-cloud-single-node/reference.md b/skills/install-aup-learning-cloud-single-node/reference.md index 514da9f9..656aac83 100644 --- a/skills/install-aup-learning-cloud-single-node/reference.md +++ b/skills/install-aup-learning-cloud-single-node/reference.md @@ -109,6 +109,10 @@ cd auplc-bundle-gfx1151-* sudo ./auplc-installer install ``` +The bundle includes the pinned +`amdgpu-insecure-instinct-udev-rules_30.30.4.0-2341068.24.04_all.deb`; offline +installation verifies and installs it from the bundle. + ## Troubleshooting | Symptom | Likely cause | First checks | diff --git a/tests/installer/test_cli_gpu_access.py b/tests/installer/test_cli_gpu_access.py new file mode 100644 index 00000000..51854223 --- /dev/null +++ b/tests/installer/test_cli_gpu_access.py @@ -0,0 +1,156 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +"""GPU access sequencing tests for installer command orchestration.""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + +import pytest + +from auplc_installer import cli +from auplc_installer.gpu_hardware import GpuHardware +from auplc_installer.helm import RuntimePaths +from auplc_installer.state import InstallerState + + +@pytest.mark.parametrize(("hardware", "expected_provision_count"), [(GpuHardware.GPU, 1), (GpuHardware.CPU, 0)]) +def test_full_install_gates_gpu_access_without_passing_it_to_the_overlay( + monkeypatch, hardware: GpuHardware, expected_provision_count: int +) -> None: + events: list[str] = [] + state = InstallerState() + paths = RuntimePaths(chart_path=Path("chart"), values_path=Path("values.yaml"), overlay_path=Path("overlay.yaml")) + + def fake_overlay(*args: object, **kwargs: object) -> Path: + assert "render_gid" not in kwargs + return paths.overlay_path + + monkeypatch.setattr(state, "runtime_paths", lambda: paths) + monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: hardware) + monkeypatch.setattr(cli, "detect_and_configure_gpu", lambda *args, **kwargs: events.append("detect")) + monkeypatch.setattr(cli, "provision_gpu_access", lambda **kwargs: events.append("provision")) + monkeypatch.setattr(cli, "generate_values_overlay", fake_overlay) + monkeypatch.setattr(cli, "install_tools", lambda **kwargs: events.append("tools")) + monkeypatch.setattr(cli, "install_k3s_single_node", lambda **kwargs: events.append("k3s")) + monkeypatch.setattr(cli, "pull_custom_images", lambda **kwargs: events.append("custom-images")) + monkeypatch.setattr(cli, "pull_external_images", lambda **kwargs: events.append("external-images")) + monkeypatch.setattr(cli, "deploy_rocm_gpu_device_plugin", lambda **kwargs: events.append("device-plugin")) + monkeypatch.setattr(cli, "refine_gpu_config_from_node_labels", lambda *args, **kwargs: events.append("refine")) + monkeypatch.setattr(cli, "deploy_runtime", lambda *args, **kwargs: events.append("runtime")) + monkeypatch.setattr(cli, "_print_success_banner", lambda: events.append("success")) + + cli._cmd_install_inner(state, pull=True) + + assert events.count("provision") == expected_provision_count + if expected_provision_count: + assert events.index("provision") < events.index("device-plugin") + + +@pytest.mark.parametrize(("hardware", "expected_provision_count"), [(GpuHardware.GPU, 1), (GpuHardware.CPU, 0)]) +def test_runtime_upgrade_gates_host_access_without_provisioning_helm_values( + monkeypatch, hardware: GpuHardware, expected_provision_count: int +) -> None: + events: list[str] = [] + state = InstallerState() + paths = RuntimePaths(chart_path=Path("chart"), values_path=Path("values.yaml"), overlay_path=Path("overlay.yaml")) + + def fake_overlay(*args: object, **kwargs: object) -> Path: + assert "render_gid" not in kwargs + return paths.overlay_path + + monkeypatch.setattr(state, "runtime_paths", lambda: paths) + monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: hardware) + monkeypatch.setattr(cli, "provision_gpu_access", lambda **kwargs: events.append("provision")) + monkeypatch.setattr(cli, "detect_and_configure_gpu", lambda *args, **kwargs: events.append("detect")) + monkeypatch.setattr(cli, "refine_gpu_config_from_node_labels", lambda *args, **kwargs: events.append("refine")) + monkeypatch.setattr(cli, "_preserve_courses_for_upgrade", lambda *args, **kwargs: events.append("preserve-courses")) + monkeypatch.setattr(cli, "generate_values_overlay", fake_overlay) + monkeypatch.setattr(cli, "upgrade_runtime", lambda *args, **kwargs: events.append("upgrade-runtime")) + + cli.cmd_rt_upgrade(state) + + assert events.count("provision") == expected_provision_count + + +@pytest.mark.parametrize( + ("reinstall", "delegate_name"), + [(cli.cmd_dev_reinstall, "cmd_dev_deploy"), (cli.cmd_rt_reinstall, "cmd_rt_install")], +) +@pytest.mark.parametrize(("hardware", "expected_provision_count"), [(GpuHardware.GPU, 1), (GpuHardware.CPU, 0)]) +def test_reinstall_gates_host_access_before_removing_runtime( + monkeypatch, + reinstall: Callable[[InstallerState], None], + delegate_name: str, + hardware: GpuHardware, + expected_provision_count: int, +) -> None: + events: list[str] = [] + state = InstallerState() + + monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: hardware) + monkeypatch.setattr(cli, "provision_gpu_access", lambda **kwargs: events.append("provision")) + monkeypatch.setattr(cli, "remove_runtime", lambda: events.append("remove-runtime")) + monkeypatch.setattr(cli.time, "sleep", lambda seconds: events.append("sleep")) + monkeypatch.setattr(cli, delegate_name, lambda current_state: events.append("delegate")) + + reinstall(state) + + assert events.count("provision") == expected_provision_count + assert events.index("remove-runtime") < events.index("delegate") + if expected_provision_count: + assert events.index("provision") < events.index("remove-runtime") + + +def test_unknown_hardware_blocks_full_install_before_gpu_access_mutation(monkeypatch) -> None: + state = InstallerState() + + monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: GpuHardware.UNKNOWN) + monkeypatch.setattr(cli, "detect_and_configure_gpu", lambda *args, **kwargs: None) + monkeypatch.setattr( + cli, "provision_gpu_access", lambda **kwargs: (_ for _ in ()).throw(AssertionError("must not provision")) + ) + + with pytest.raises(RuntimeError, match="hardware"): + cli._cmd_install_inner(state, pull=True) + + +@pytest.mark.parametrize( + ("reinstall", "delegate_name"), + [(cli.cmd_dev_reinstall, "cmd_dev_deploy"), (cli.cmd_rt_reinstall, "cmd_rt_install")], +) +def test_unknown_hardware_blocks_reinstall_before_runtime_removal( + monkeypatch, reinstall: Callable[[InstallerState], None], delegate_name: str +) -> None: + events: list[str] = [] + state = InstallerState() + + monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: GpuHardware.UNKNOWN) + monkeypatch.setattr( + cli, "provision_gpu_access", lambda **kwargs: (_ for _ in ()).throw(AssertionError("must not provision")) + ) + monkeypatch.setattr(cli, "remove_runtime", lambda: events.append("remove-runtime")) + monkeypatch.setattr(cli, delegate_name, lambda current_state: events.append("delegate")) + + with pytest.raises(RuntimeError, match="hardware"): + reinstall(state) + + assert "remove-runtime" not in events + assert "delegate" not in events + + +def test_gpu_hardware_gate_passes_offline_bundle_context_to_package_provisioning( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + # Given: a local GPU installation running from an offline bundle. + bundle = tmp_path / "bundle" + package_calls: list[dict[str, object]] = [] + monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: GpuHardware.GPU) + monkeypatch.setattr(cli, "provision_gpu_access", lambda **kwargs: package_calls.append(kwargs)) + + # When: the CLI's local-hardware gate provisions GPU access. + cli._provision_gpu_access_for_local_hardware(offline_mode=True, bundle_dir=bundle) + + # Then: package provisioning receives the bundle context unchanged. + assert package_calls == [{"offline_mode": True, "bundle_dir": bundle}] diff --git a/tests/installer/test_gpu_access.py b/tests/installer/test_gpu_access.py new file mode 100644 index 00000000..ad3285a1 --- /dev/null +++ b/tests/installer/test_gpu_access.py @@ -0,0 +1,213 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +"""Tests for AMD's packaged single-node GPU udev policy.""" + +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from auplc_installer import gpu_access +from auplc_installer.gpu_access import ( + AMD_GPU_UDEV_PACKAGE_FILENAME, + AMD_GPU_UDEV_PACKAGE_RULES, + AMD_GPU_UDEV_PACKAGE_RULES_PATH, + AMD_GPU_UDEV_PACKAGE_VERSION, + LEGACY_AMDGPU_RULES, + LEGACY_AMDGPU_RULES_PATH, + SystemGpuAccessHost, + provision_gpu_access, +) +from auplc_installer.util import InstallerError + + +class FakeGpuAccessHost: + def __init__( + self, + *, + files: dict[Path, str] | None = None, + installed_version: str | None = None, + package_owns_rule: bool | None = None, + ) -> None: + self.files = dict(files or {}) + self.installed_version = installed_version + self._package_owns_rule = installed_version is not None if package_owns_rule is None else package_owns_rule + self.calls: list[str] = [] + self.symlinks: set[Path] = set() + self.nonregular_files: set[Path] = set() + self.directories = {Path("/"), Path("/etc"), Path("/etc/udev"), Path("/etc/udev/rules.d")} + + def read_text(self, path: Path) -> str | None: + self.calls.append(f"read:{path}") + return self.files.get(path) + + def remove_udev_rule(self, path: Path) -> None: + self.calls.append(f"remove-rule:{path}") + self.files.pop(path, None) + + def installed_package_version(self) -> str | None: + self.calls.append("installed-version") + return self.installed_version + + def package_owns_rule(self, path: Path) -> bool: + self.calls.append(f"owns-rule:{path}") + return self._package_owns_rule + + def install_package(self, deb: Path) -> None: + self.calls.append(f"install-package:{deb}") + self.installed_version = AMD_GPU_UDEV_PACKAGE_VERSION + self._package_owns_rule = True + self.files[AMD_GPU_UDEV_PACKAGE_RULES_PATH] = AMD_GPU_UDEV_PACKAGE_RULES + + def reload_udev_rules(self) -> None: + self.calls.append("reload-udev") + + def trigger_udev(self) -> None: + self.calls.append("trigger-udev") + + def settle_udev(self) -> None: + self.calls.append("settle-udev") + + def is_symlink(self, path: Path) -> bool: + return path in self.symlinks + + def is_regular_file(self, path: Path) -> bool: + return path in self.files + + def path_exists(self, path: Path) -> bool: + return path in self.files or path in self.symlinks or path in self.nonregular_files or path in self.directories + + def is_directory(self, path: Path) -> bool: + return path in self.directories + + +def test_offline_install_replaces_legacy_rule_at_the_package_owned_path( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + # Given: an offline bundle and a legacy rule from an earlier shipped installer. + bundle = tmp_path / "bundle" + deb = bundle / "packages" / AMD_GPU_UDEV_PACKAGE_FILENAME + deb.parent.mkdir(parents=True) + deb.write_bytes(b"package") + host = FakeGpuAccessHost(files={LEGACY_AMDGPU_RULES_PATH: LEGACY_AMDGPU_RULES}) + verified: list[tuple[Path, str]] = [] + monkeypatch.setattr(gpu_access, "verify_sha256", lambda path, checksum: verified.append((Path(path), checksum))) + + # When: GPU access is provisioned from the bundle. + provision_gpu_access(host, offline_mode=True, bundle_dir=bundle) + + # Then: package installation replaces the path without deleting the package-owned rule afterward. + assert not any(call == f"remove-rule:{LEGACY_AMDGPU_RULES_PATH}" for call in host.calls) + assert host.files == {AMD_GPU_UDEV_PACKAGE_RULES_PATH: AMD_GPU_UDEV_PACKAGE_RULES} + assert verified == [(deb, gpu_access.AMD_GPU_UDEV_PACKAGE_SHA256)] + + +def test_online_install_downloads_to_a_temporary_deb_then_removes_it(monkeypatch: pytest.MonkeyPatch) -> None: + # Given: no installed package and a downloader that materializes its destination. + host = FakeGpuAccessHost() + downloads: list[list[str]] = [] + verified: list[Path] = [] + + def fake_run(command: list[str], **_: object) -> SimpleNamespace: + downloads.append(command) + Path(command[-1]).write_bytes(b"package") + return SimpleNamespace(returncode=0) + + monkeypatch.setattr(gpu_access, "run", fake_run) + monkeypatch.setattr(gpu_access, "verify_sha256", lambda path, _: verified.append(Path(path))) + + # When: GPU access is provisioned online. + provision_gpu_access(host, offline_mode=False, bundle_dir=None) + + # Then: the exact Radeon URL is downloaded, verified, installed, and cleaned up. + downloaded_path = Path(downloads[0][-1]) + assert downloads[0][2] == gpu_access.AMD_GPU_UDEV_PACKAGE_URL + assert verified == [downloaded_path] + assert not downloaded_path.exists() + assert host.installed_version == AMD_GPU_UDEV_PACKAGE_VERSION + assert host.files[AMD_GPU_UDEV_PACKAGE_RULES_PATH] == AMD_GPU_UDEV_PACKAGE_RULES + + +def test_installed_package_requires_the_pinned_version_and_its_exact_rule() -> None: + # Given: the package is already present with the expected package-owned rule. + host = FakeGpuAccessHost( + files={AMD_GPU_UDEV_PACKAGE_RULES_PATH: AMD_GPU_UDEV_PACKAGE_RULES}, + installed_version=AMD_GPU_UDEV_PACKAGE_VERSION, + ) + + # When: provisioning is repeated. + provision_gpu_access(host) + + # Then: no download, install, legacy removal, or device probe is performed. + assert not any(call.startswith(("install-package:", "remove-rule:")) for call in host.calls) + assert not any(call in {"reload-udev", "trigger-udev", "settle-udev"} for call in host.calls) + + +@pytest.mark.parametrize( + ("installed_version", "package_owns_rule", "rule"), + [ + (AMD_GPU_UDEV_PACKAGE_VERSION, False, AMD_GPU_UDEV_PACKAGE_RULES), + (AMD_GPU_UDEV_PACKAGE_VERSION, True, 'KERNEL=="kfd", MODE="0660"\n'), + ], +) +def test_installed_package_fails_closed_when_its_version_or_rule_contract_is_wrong( + installed_version: str, package_owns_rule: bool, rule: str +) -> None: + # Given: an installed package that does not satisfy the pinned package contract. + host = FakeGpuAccessHost( + files={AMD_GPU_UDEV_PACKAGE_RULES_PATH: rule}, + installed_version=installed_version, + package_owns_rule=package_owns_rule, + ) + + # When: provisioning checks the installed package. + with pytest.raises(InstallerError): + provision_gpu_access(host) + + # Then: it fails before installing or mutating any udev rule. + assert not any(call.startswith(("install-package:", "remove-rule:")) for call in host.calls) + + +def test_symlinked_legacy_rule_fails_closed_before_installation(monkeypatch: pytest.MonkeyPatch) -> None: + # Given: a legacy-rule path replaced by a symlink. + host = FakeGpuAccessHost() + host.symlinks.add(LEGACY_AMDGPU_RULES_PATH) + monkeypatch.setattr(gpu_access, "run", lambda *args, **kwargs: pytest.fail("must not download")) + + # When: first-time provisioning inspects legacy rules. + with pytest.raises(InstallerError, match="symlinked GPU udev rule"): + provision_gpu_access(host) + + # Then: no package installation is attempted. + assert not any(call.startswith("install-package:") for call in host.calls) + + +def test_official_rule_matches_the_extracted_deb_policy_not_the_old_pxe_shape() -> None: + # Given: the exact package verification constant. + rules = AMD_GPU_UDEV_PACKAGE_RULES + + # When: its policy is inspected. + # Then: it matches the extracted package rule rather than the former two-line PXE shape. + assert rules == ( + 'KERNEL=="kfd", GROUP="render", MODE="0666"\n' + 'SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0666"\n' + ) + assert "card" not in rules + + +def test_system_adapter_uses_dpkg_for_the_package_install(monkeypatch: pytest.MonkeyPatch) -> None: + # Given: the production host adapter and a recorded command runner. + commands: list[list[str]] = [] + monkeypatch.setattr( + gpu_access, + "run", + lambda command, **_: commands.append(command) or SimpleNamespace(returncode=0), + ) + + # When: it installs the verified package artifact. + SystemGpuAccessHost().install_package(Path("/tmp/package.deb")) + + # Then: installation is delegated to dpkg with sudo awareness. + assert commands == [["dpkg", "--force-confnew", "--install", "/tmp/package.deb"]] diff --git a/tests/installer/test_gpu_access_ordering.py b/tests/installer/test_gpu_access_ordering.py new file mode 100644 index 00000000..93635798 --- /dev/null +++ b/tests/installer/test_gpu_access_ordering.py @@ -0,0 +1,149 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from auplc_installer import gpu_access +from auplc_installer.gpu_access import ( + AMD_GPU_UDEV_PACKAGE_FILENAME, + AMD_GPU_UDEV_PACKAGE_RULES, + AMD_GPU_UDEV_PACKAGE_RULES_PATH, + AMD_GPU_UDEV_PACKAGE_VERSION, + LEGACY_KFD_RULES, + LEGACY_KFD_RULES_PATH, + provision_gpu_access, +) +from auplc_installer.util import InstallerError +from tests.installer.test_gpu_access import FakeGpuAccessHost + + +def _offline_bundle(tmp_path: Path) -> Path: + bundle = tmp_path / "bundle" + deb = bundle / "packages" / AMD_GPU_UDEV_PACKAGE_FILENAME + deb.parent.mkdir(parents=True) + deb.write_bytes(b"package") + return bundle + + +def test_wrong_installed_version_downloads_and_converges_to_the_pinned_package(monkeypatch: pytest.MonkeyPatch) -> None: + # Given: a different installed package version and an online downloader. + host = FakeGpuAccessHost( + files={AMD_GPU_UDEV_PACKAGE_RULES_PATH: 'KERNEL=="kfd", MODE="0660"\n'}, + installed_version="30.30.4.0-older", + ) + downloads: list[list[str]] = [] + + def fake_run(command: list[str], **_: object) -> SimpleNamespace: + downloads.append(command) + Path(command[-1]).write_bytes(b"package") + return SimpleNamespace(returncode=0) + + monkeypatch.setattr(gpu_access, "run", fake_run) + monkeypatch.setattr(gpu_access, "verify_sha256", lambda *args: None) + + # When: GPU access is provisioned. + provision_gpu_access(host) + + # Then: the pinned deb is acquired and the installed rule converges to its exact content. + assert downloads[0][2] == gpu_access.AMD_GPU_UDEV_PACKAGE_URL + assert host.installed_version == AMD_GPU_UDEV_PACKAGE_VERSION + assert host.files[AMD_GPU_UDEV_PACKAGE_RULES_PATH] == AMD_GPU_UDEV_PACKAGE_RULES + + +def test_exact_installed_package_skips_network_then_removes_separate_legacy_rule( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Given: an exact package rule plus a separately shipped legacy KFD rule. + host = FakeGpuAccessHost( + files={ + AMD_GPU_UDEV_PACKAGE_RULES_PATH: AMD_GPU_UDEV_PACKAGE_RULES, + LEGACY_KFD_RULES_PATH: LEGACY_KFD_RULES, + }, + installed_version=AMD_GPU_UDEV_PACKAGE_VERSION, + ) + monkeypatch.setattr(gpu_access, "run", lambda *args, **kwargs: pytest.fail("must not download")) + + # When: provisioning checks an otherwise already-correct installation. + provision_gpu_access(host) + + # Then: it removes only the separate legacy file and applies its removal to live udev state. + assert LEGACY_KFD_RULES_PATH not in host.files + assert not any(call.startswith("install-package:") for call in host.calls) + assert host.calls[-3:] == ["reload-udev", "trigger-udev", "settle-udev"] + + +def test_acquires_and_verifies_the_offline_deb_before_deleting_legacy_rules( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + # Given: a first installation from a verified offline bundle and a legacy KFD rule. + bundle = _offline_bundle(tmp_path) + host = FakeGpuAccessHost(files={LEGACY_KFD_RULES_PATH: LEGACY_KFD_RULES}) + monkeypatch.setattr(gpu_access, "verify_sha256", lambda *args: host.calls.append("verify-deb")) + + # When: the package is installed. + provision_gpu_access(host, offline_mode=True, bundle_dir=bundle) + + # Then: package installation completes before the separate legacy rule is deleted. + install_index = next(index for index, call in enumerate(host.calls) if call.startswith("install-package:")) + removal_index = host.calls.index(f"remove-rule:{LEGACY_KFD_RULES_PATH}") + assert host.calls.index("verify-deb") < install_index < removal_index + + +def test_failed_installation_keeps_legacy_rules_intact(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + # Given: a first offline installation whose package install fails. + bundle = _offline_bundle(tmp_path) + host = FakeGpuAccessHost(files={LEGACY_KFD_RULES_PATH: LEGACY_KFD_RULES}) + monkeypatch.setattr(gpu_access, "verify_sha256", lambda *args: None) + + def fail_install(deb: Path) -> None: + host.calls.append(f"install-package:{deb}") + raise InstallerError("dpkg failed") + + monkeypatch.setattr(host, "install_package", fail_install) + + # When: package installation fails. + with pytest.raises(InstallerError, match="dpkg failed"): + provision_gpu_access(host, offline_mode=True, bundle_dir=bundle) + + # Then: the legacy rule remains and no udev refresh occurs. + assert host.files[LEGACY_KFD_RULES_PATH] == LEGACY_KFD_RULES + assert "reload-udev" not in host.calls + + +@pytest.mark.parametrize("installed_version", ["30.30.4.0-older", None]) +def test_package_owned_differing_conffile_converges( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, installed_version: str | None +) -> None: + # Given: a wrong-version or partial package state that owns a differing conffile. + host = FakeGpuAccessHost( + files={AMD_GPU_UDEV_PACKAGE_RULES_PATH: 'KERNEL=="kfd", MODE="0600"\n'}, + installed_version=installed_version, + package_owns_rule=True, + ) + monkeypatch.setattr(gpu_access, "verify_sha256", lambda *args: None) + + # When: the pinned package is installed from an offline bundle. + provision_gpu_access(host, offline_mode=True, bundle_dir=_offline_bundle(tmp_path)) + + # Then: forced installation converges to the exact package rule without legacy deletion. + assert host.files[AMD_GPU_UDEV_PACKAGE_RULES_PATH] == AMD_GPU_UDEV_PACKAGE_RULES + assert not any(call.startswith("remove-rule:") for call in host.calls) + + +def test_unknown_unowned_amdgpu_rule_fails_closed() -> None: + # Given: an unowned, unrecognized rule at the AMD package path. + host = FakeGpuAccessHost( + files={AMD_GPU_UDEV_PACKAGE_RULES_PATH: 'KERNEL=="kfd", MODE="0600"\n'}, + package_owns_rule=False, + ) + + # When: provisioning admits legacy rules. + with pytest.raises(InstallerError, match="unexpected legacy"): + provision_gpu_access(host) + + # Then: no package installation or rule deletion is attempted. + assert not any(call.startswith(("install-package:", "remove-rule:")) for call in host.calls) diff --git a/tests/installer/test_gpu_hardware.py b/tests/installer/test_gpu_hardware.py new file mode 100644 index 00000000..aa8bb15b --- /dev/null +++ b/tests/installer/test_gpu_hardware.py @@ -0,0 +1,78 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +"""Tests for local AMD GPU hardware classification from PCI sysfs evidence.""" + +from __future__ import annotations + +from pathlib import Path + +from auplc_installer.gpu_hardware import GpuHardware, classify_gpu_hardware + + +def test_classify_gpu_hardware_returns_gpu_for_amd_display_controller(tmp_path: Path) -> None: + pci_devices = tmp_path / "devices" + device = pci_devices / "0000:03:00.0" + device.mkdir(parents=True) + (device / "vendor").write_text("0x1002\n", encoding="ascii") + (device / "class").write_text("0x030200\n", encoding="ascii") + + hardware = classify_gpu_hardware(pci_devices) + + assert hardware is GpuHardware.GPU + + +def test_classify_gpu_hardware_returns_cpu_for_complete_scan_without_amd_display(tmp_path: Path) -> None: + pci_devices = tmp_path / "devices" + intel_display = pci_devices / "0000:00:02.0" + intel_display.mkdir(parents=True) + (intel_display / "vendor").write_text("0x8086\n", encoding="ascii") + (intel_display / "class").write_text("0x030000\n", encoding="ascii") + amd_audio = pci_devices / "0000:03:00.1" + amd_audio.mkdir() + (amd_audio / "vendor").write_text("0x1002\n", encoding="ascii") + (amd_audio / "class").write_text("0x040300\n", encoding="ascii") + + hardware = classify_gpu_hardware(pci_devices) + + assert hardware is GpuHardware.CPU + + +def test_classify_gpu_hardware_returns_unknown_when_pci_root_is_missing(tmp_path: Path) -> None: + hardware = classify_gpu_hardware(tmp_path / "missing") + + assert hardware is GpuHardware.UNKNOWN + + +def test_classify_gpu_hardware_returns_unknown_when_pci_root_is_empty(tmp_path: Path) -> None: + pci_devices = tmp_path / "devices" + pci_devices.mkdir() + + hardware = classify_gpu_hardware(pci_devices) + + assert hardware is GpuHardware.UNKNOWN + + +def test_classify_gpu_hardware_returns_unknown_for_incomplete_pci_evidence(tmp_path: Path) -> None: + pci_devices = tmp_path / "devices" + missing_vendor = pci_devices / "0000:00:02.0" + missing_vendor.mkdir(parents=True) + (missing_vendor / "class").write_text("0x030000\n", encoding="ascii") + + hardware = classify_gpu_hardware(pci_devices) + + assert hardware is GpuHardware.UNKNOWN + + +def test_classify_gpu_hardware_prefers_positive_amd_evidence_over_incomplete_sibling(tmp_path: Path) -> None: + pci_devices = tmp_path / "devices" + incomplete_device = pci_devices / "0000:00:02.0" + incomplete_device.mkdir(parents=True) + (incomplete_device / "vendor").write_text("0x8086\n", encoding="ascii") + gpu_device = pci_devices / "0000:03:00.0" + gpu_device.mkdir() + (gpu_device / "vendor").write_text("0x1002\n", encoding="ascii") + (gpu_device / "class").write_text("0x038000\n", encoding="ascii") + + hardware = classify_gpu_hardware(pci_devices) + + assert hardware is GpuHardware.GPU diff --git a/tests/installer/test_overlay.py b/tests/installer/test_overlay.py index b028cdd9..8e4ece63 100644 --- a/tests/installer/test_overlay.py +++ b/tests/installer/test_overlay.py @@ -27,6 +27,7 @@ ) from auplc_installer.gpu import GpuConfig, SkuEntry, append_product from auplc_installer.overlay import ( + GPU_RESOURCE_KEYS, emit_overlay, generate_values_overlay, try_load_courses_from_overlay, @@ -106,6 +107,22 @@ def test_default_selection_round_trips_valid_yaml() -> None: assert "teams" not in parsed["custom"] +def test_overlay_keeps_gpu_resources_without_gpu_access_contract() -> None: + text, parsed = _render( + _strix_halo_cfg(), + courses=CourseSelection.default(), + ) + + custom = parsed["custom"] + assert "gpuAccess" not in custom + assert "renderGid" not in text + assert "supplementalGroups" not in text + assert set(custom["resources"]["images"]) == set(GPU_RESOURCE_KEYS) + assert set(custom["resources"]["metadata"]) == set(GPU_RESOURCE_KEYS) + assert "teams" not in custom + assert "profiles" not in custom + + def test_resource_images_use_primary_tag() -> None: _, parsed = _render(_strix_halo_cfg(), courses=CourseSelection.default()) images = parsed["custom"]["resources"]["images"] diff --git a/tests/installer/test_pack.py b/tests/installer/test_pack.py new file mode 100644 index 00000000..0b984a52 --- /dev/null +++ b/tests/installer/test_pack.py @@ -0,0 +1,37 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +"""Tests for offline bundle package artifacts.""" + +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace + +from auplc_installer import pack +from auplc_installer.gpu_access import ( + AMD_GPU_UDEV_PACKAGE_FILENAME, + AMD_GPU_UDEV_PACKAGE_SHA256, + AMD_GPU_UDEV_PACKAGE_URL, +) + + +def test_pack_downloads_and_checksums_the_offline_gpu_udev_package(tmp_path: Path, monkeypatch) -> None: + # Given: an empty bundle staging directory and a recording downloader. + commands: list[list[str]] = [] + verified: list[tuple[Path, str]] = [] + + def fake_run(command: list[str], **_: object) -> SimpleNamespace: + commands.append(command) + Path(command[-1]).write_bytes(b"package") + return SimpleNamespace(returncode=0) + + monkeypatch.setattr(pack, "run", fake_run) + monkeypatch.setattr(pack, "verify_sha256", lambda path, checksum: verified.append((Path(path), checksum))) + + # When: package artifacts are added to the offline bundle. + pack.pack_download_gpu_access_package(tmp_path) + + # Then: the pinned deb is placed in packages/ and verified before archiving. + deb = tmp_path / "packages" / AMD_GPU_UDEV_PACKAGE_FILENAME + assert commands == [["wget", "-q", AMD_GPU_UDEV_PACKAGE_URL, "-O", str(deb)]] + assert verified == [(deb, AMD_GPU_UDEV_PACKAGE_SHA256)] diff --git a/tests/installer/test_values_gpu_overrides.py b/tests/installer/test_values_gpu_overrides.py index 7f6ef6ad..18de940f 100644 --- a/tests/installer/test_values_gpu_overrides.py +++ b/tests/installer/test_values_gpu_overrides.py @@ -69,3 +69,12 @@ def test_default_values_route_gpu_resources_to_supported_image_tags() -> None: assert overrides[accelerator_key]["image"] == ( f"ghcr.io/amdresearch/{image_name}:latest-{gpu_target}" ), values_file + + +def test_default_values_use_fs_gid_without_overriding_pod_security_context() -> None: + for values_file in VALUES_FILES: + values = _load_values(values_file) + singleuser = values["singleuser"] + + assert singleuser["fsGid"] == 100, values_file + assert "securityContext" not in singleuser.get("extraPodConfig", {}), values_file diff --git a/tests/scripts/test_gpu_image_permissions.py b/tests/scripts/test_gpu_image_permissions.py new file mode 100644 index 00000000..b8885a4b --- /dev/null +++ b/tests/scripts/test_gpu_image_permissions.py @@ -0,0 +1,22 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +from __future__ import annotations + +import re +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +DOCKERFILE = ROOT / "dockerfiles" / "Base" / "Dockerfile.rocm" + + +def test_rocm_base_leaves_gpu_device_permissions_to_the_host() -> None: + dockerfile = DOCKERFILE.read_text(encoding="utf-8") + + forbidden_patterns = ( + r"\b(?:groupadd|groupmod|usermod)\b.*\b(?:video|render)\b", + r"/etc/udev", + r"chmod\s+666\b", + r"chmod\b.*(?:/dev/|kfd|render|card)", + ) + for pattern in forbidden_patterns: + assert re.search(pattern, dockerfile) is None, pattern diff --git a/tests/skills/test_config_generation_security.py b/tests/skills/test_config_generation_security.py new file mode 100644 index 00000000..66d5a49e --- /dev/null +++ b/tests/skills/test_config_generation_security.py @@ -0,0 +1,103 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +GEN_CONFIGS = ROOT / "skills" / "deploy-aup-learning-cloud" / "scripts" / "gen_configs.py" + + +def safe_spec() -> dict[str, object]: + return { + "topology": "ssh-preinstalled", + "k3s_version": "v1.32.3+k3s1", + "server": {"name": "server-1", "ip": "192.168.1.10"}, + "agents": [{"name": "agent-1", "ip": "192.168.1.11"}], + "images": {"cpu": "registry.example/auplc:latest"}, + } + + +def load_config_generation_module(): + sys.path.insert(0, str(GEN_CONFIGS.parent)) + try: + import config_generation + + return config_generation + finally: + sys.path.pop(0) + + +@pytest.mark.parametrize( + ("path", "value", "message"), + [ + (("server", "name"), "server\n vars: {injected: true}", "spec.server.name"), + (("server", "ip"), "192.168.1.10\n injected: true", "spec.server.ip"), + (("k3s_version",), "v1.32.3+k3s1\n injected: true", "spec.k3s_version"), + (("agents",), [{"name": "server-1", "ip": "192.168.1.11"}], "unique"), + (("agents",), [{"name": "agent-1", "ip": "not-an-ip"}], "spec.agents[0].ip"), + (("images",), {"cpu\n injected": "registry.example/auplc:latest"}, "spec.images key"), + ], +) +def test_generator_rejects_unsafe_public_spec_scalars_before_discovery( + path: tuple[str, ...], value: object, message: str, capsys: pytest.CaptureFixture[str] +) -> None: + module = load_config_generation_module() + spec = safe_spec() + if len(path) == 1: + spec[path[0]] = value + else: + target = spec[path[0]] + assert isinstance(target, dict) + target[path[1]] = value + with pytest.raises(SystemExit) as error: + module.validate_spec(spec) + + assert error.value.code == 1 + assert message in capsys.readouterr().err + + +def test_generator_rejects_an_invalid_k3s_version_before_discovery(capsys: pytest.CaptureFixture[str]) -> None: + module = load_config_generation_module() + spec = safe_spec() + spec["k3s_version"] = "v1.32.3+k3s1 # comments are not accepted" + + with pytest.raises(SystemExit) as error: + module.validate_spec(spec) + + assert error.value.code == 1 + assert "spec.k3s_version" in capsys.readouterr().err + + +def test_generator_applies_the_normal_unknown_field_policy_to_draft_gpu_fields() -> None: + module = load_config_generation_module() + spec = safe_spec() + spec["render_gid"] = 993 + spec["gpu_access"] = {"hosts": []} + + assert module.validate_spec(spec) == "ssh-preinstalled" + + +@pytest.mark.parametrize( + "raw", + [ + '{"topology":"ssh-preinstalled","topology":"pxe-diskless"}', + ], +) +def test_generator_rejects_duplicate_public_policy_keys_before_discovery(tmp_path: Path, raw: str) -> None: + spec = tmp_path / "spec.json" + spec.write_text(raw, encoding="utf-8") + + result = subprocess.run( + [sys.executable, str(GEN_CONFIGS), "--spec", str(spec), "--out-dir", str(tmp_path / "generated")], + capture_output=True, + check=False, + text=True, + ) + + assert result.returncode == 1 + assert "duplicate JSON key" in result.stderr diff --git a/tests/skills/test_deploy_scripts.py b/tests/skills/test_deploy_scripts.py index 7e5b6f2c..c4901737 100644 --- a/tests/skills/test_deploy_scripts.py +++ b/tests/skills/test_deploy_scripts.py @@ -19,6 +19,7 @@ DEPLOY_SCRIPTS = ROOT / "skills" / "deploy-aup-learning-cloud" / "scripts" VALIDATE = DEPLOY_SCRIPTS / "validate.py" GEN_CONFIGS = DEPLOY_SCRIPTS / "gen_configs.py" +ARTIFACT_STORE = DEPLOY_SCRIPTS / "artifact_store.py" def run_script(script: Path, *args: str, cwd: Path | None = None) -> subprocess.CompletedProcess[str]: @@ -37,23 +38,100 @@ def write_file(path: Path, content: str) -> Path: return path +@pytest.fixture(autouse=True) +def fake_ansible_playbook(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + fake_bin = tmp_path / "fake-ansible" + fake_bin.mkdir() + fake_ansible = fake_bin / "ansible-playbook" + fake_ansible.write_text( + r"""#!/usr/bin/env python3 +import json +from pathlib import Path +import sys + +arguments = sys.argv[1:] +inventory = Path(arguments[arguments.index('-i') + 1]) +output = next(value.split('=', 1)[1] for value in arguments if value.startswith('gpu_access_discovery_output_path=')) +hosts = [line.strip()[:-1] for line in inventory.read_text(encoding='utf-8').splitlines() if line.startswith(' ') and line.rstrip().endswith(':')] +evidence = { + 'version': 1, + 'hosts': [{ + 'host': host, + 'reachable': True, + 'lspci': {'rc': 0, 'stdout': ''}, + 'sysfs': {'rc': 0, 'stdout': ''}, + } for host in hosts], +} +Path(output).write_text(json.dumps(evidence), encoding='utf-8') +""", + encoding="utf-8", + ) + fake_ansible.chmod(0o755) + monkeypatch.setenv("PATH", f"{fake_bin}:{os.environ['PATH']}") + + def write_cluster(repo: Path, labels: list[str]) -> Path: return write_file(repo / "cluster.json", json.dumps({"gpu_product_names": labels})) -def load_validate_module(): - spec = importlib.util.spec_from_file_location("deploy_validate", VALIDATE) - module = importlib.util.module_from_spec(spec) - assert spec.loader is not None - spec.loader.exec_module(module) - return module +def write_resolved_gpu_artifacts(repo: Path) -> tuple[Path, Path, Path]: + inventory = write_file( + repo / "generated/inventory.yml", + """k3s_cluster: + children: + server: + hosts: + server: + ansible_host: 192.168.1.10 + auplc_gpu_access_enabled: true + agent: + hosts: + agent: + ansible_host: 192.168.1.11 + auplc_gpu_access_enabled: false +""", + ) + values = write_file( + repo / "generated/values-basic-example.yaml", + """custom: + resources: + metadata: {} +""", + ) + resolution = write_file( + repo / "generated/gpu-access-resolution.json", + json.dumps( + { + "version": 1, + "status": "gpu_resolved", + "hosts": {"agent": False, "server": True}, + } + ), + ) + return inventory, values, resolution -def load_generator_module(): - spec = importlib.util.spec_from_file_location("deploy_generator", GEN_CONFIGS) - module = importlib.util.module_from_spec(spec) - assert spec.loader is not None - spec.loader.exec_module(module) +def load_validate_module(): + sys.path.insert(0, str(DEPLOY_SCRIPTS)) + try: + spec = importlib.util.spec_from_file_location("deploy_validate", VALIDATE) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + finally: + sys.path.pop(0) + + +def load_deploy_module(module_name: str, script: Path): + sys.path.insert(0, str(DEPLOY_SCRIPTS)) + try: + spec = importlib.util.spec_from_file_location(module_name, script) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + finally: + sys.path.pop(0) return module @@ -467,69 +545,6 @@ def test_validator_uses_generated_pxe_vars_file_when_requested(tmp_path: Path) - assert "k3s_version == pxe_k3s_version" in result.stdout -def test_validator_preserves_explicit_selector_and_accelerator_key_clears(tmp_path: Path) -> None: - repo = tmp_path / "checkout" - base = write_file( - repo / "runtime/values.yaml", - """custom: - accelerators: - strix-halo: - nodeSelector: - amd.com/gpu.product-name: AMD_Radeon_8060S_Graphics - resources: - metadata: - gpu: - acceleratorKeys: [strix-halo] -""", - ) - selector_clear = write_file( - repo / "selector-clear.yaml", - """custom: - accelerators: - strix-halo: - nodeSelector: - amd.com/gpu.product-name: null -""", - ) - keys_clear = write_file( - repo / "keys-clear.yaml", - """custom: - resources: - metadata: - gpu: - acceleratorKeys: ~ -""", - ) - - selector_result = run_script( - VALIDATE, - "--repo", - str(repo), - "--topology", - "ssh-preinstalled", - "--values", - str(base), - "--values", - str(selector_clear), - ) - keys_result = run_script( - VALIDATE, - "--repo", - str(repo), - "--topology", - "ssh-preinstalled", - "--values", - str(base), - "--values", - str(keys_clear), - ) - - assert selector_result.returncode == 1 - assert "active accelerator 'strix-halo' has no amd.com/gpu.product-name nodeSelector" in selector_result.stdout - assert keys_result.returncode == 0, keys_result.stdout + keys_result.stderr - assert "no acceleratorKeys found" in keys_result.stdout - - def test_validator_honors_every_supported_explicit_clear_syntax(tmp_path: Path) -> None: repo = tmp_path / "checkout" base = write_file( @@ -623,36 +638,6 @@ def test_validator_main_resets_report_state_between_invocations(tmp_path: Path) assert second == 0 -def test_validator_requires_product_labels_under_active_accelerator_node_selectors(tmp_path: Path) -> None: - repo = tmp_path / "checkout" - values = write_file( - repo / "runtime/values.yaml", - """custom: - accelerators: - strix-halo: - env: - amd.com/gpu.product-name: AMD_Radeon_8060S_Graphics - resources: - metadata: - gpu: - acceleratorKeys: [strix-halo] -""", - ) - - result = run_script( - VALIDATE, - "--repo", - str(repo), - "--topology", - "ssh-preinstalled", - "--values", - str(values), - ) - - assert result.returncode == 1 - assert "active accelerator 'strix-halo' has no amd.com/gpu.product-name nodeSelector" in result.stdout - - def test_validator_ignores_accelerators_and_metadata_outside_custom_resources(tmp_path: Path) -> None: repo = tmp_path / "checkout" values = write_file( @@ -751,6 +736,186 @@ def test_validator_fails_when_an_active_accelerator_has_no_product_selector(tmp_ assert "active accelerator 'strix-halo' has no amd.com/gpu.product-name nodeSelector" in result.stdout +def test_validator_accepts_consistent_gpu_resolved_artifacts(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + inventory, values, resolution = write_resolved_gpu_artifacts(repo) + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--inventory", + str(inventory), + "--values", + str(values), + "--gpu-resolution", + str(resolution), + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert "GPU access artifacts agree" in result.stdout + + +@pytest.mark.parametrize( + ("resolution_content", "expected_error"), + [ + ("not JSON", "GPU resolution manifest is malformed"), + ( + '{"version":1,"status":"pending","hosts":{"agent":false,"server":true}}', + "GPU resolution manifest status must be cpu_only or gpu_resolved", + ), + ( + '{"version":1,"status":"gpu_resolved","hosts":{"server":true,"server":false}}', + "duplicate JSON key 'server'", + ), + ( + '{"version":1,"status":"gpu_resolved","hosts":{"ser\\u0076er":true,"server":false}}', + "duplicate JSON key 'server'", + ), + ], +) +def test_validator_rejects_malformed_pending_or_duplicate_gpu_resolution( + tmp_path: Path, resolution_content: str, expected_error: str +) -> None: + repo = tmp_path / "checkout" + inventory, values, resolution = write_resolved_gpu_artifacts(repo) + resolution.write_text(resolution_content, encoding="utf-8") + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--inventory", + str(inventory), + "--values", + str(values), + "--gpu-resolution", + str(resolution), + ) + + assert result.returncode == 1 + assert expected_error in result.stdout + + +def test_validator_rejects_missing_generated_gpu_resolution_artifact(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + inventory, values, _ = write_resolved_gpu_artifacts(repo) + missing_resolution = repo / "generated/missing-gpu-access-resolution.json" + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--inventory", + str(inventory), + "--values", + str(values), + "--gpu-resolution", + str(missing_resolution), + ) + + assert result.returncode == 1 + assert "GPU resolution manifest not found" in result.stdout + + +def test_validator_rejects_mismatched_host_boolean(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + inventory, values, resolution = write_resolved_gpu_artifacts(repo) + resolution.write_text( + json.dumps( + { + "version": 1, + "status": "gpu_resolved", + "hosts": {"agent": True, "server": True}, + } + ), + encoding="utf-8", + ) + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--inventory", + str(inventory), + "--values", + str(values), + "--gpu-resolution", + str(resolution), + ) + + assert result.returncode == 1 + assert "inventory host 'agent' GPU access boolean disagrees" in result.stdout + + +def test_validator_rejects_pxe_rootfs_boolean_mismatch(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + inventory = write_file( + repo / "generated/inventory.yml", + """k3s_cluster: + children: + server: + hosts: + server: + ansible_host: 192.168.1.10 + auplc_gpu_access_enabled: false + agent: + hosts: {} +""", + ) + values = write_file(repo / "generated/values-basic-example.yaml", "custom:\n resources:\n metadata: {}\n") + resolution = write_file( + repo / "generated/gpu-access-resolution.json", + json.dumps( + { + "version": 1, + "status": "cpu_only", + "hosts": {"server": False}, + "pxe_rootfs": {"gpu_access_enabled": True}, + } + ), + ) + pxe_vars = write_file( + repo / "generated/pb-pxe-controller.vars.yml", + """pxe_network_interface: eno1 +pxe_subnet: 192.168.1.0/24 +pxe_controller_ip: 192.168.1.10 +pxe_dns_servers: 8.8.8.8 +pxe_k3s_server_ips: [192.168.1.10] +pxe_rootfs_authorized_keys: [ssh-ed25519-AAA] +pxe_k3s_version: v1.32.3+k3s1 +pxe_gpu_access_enabled: false +""", + ) + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "pxe-diskless", + "--inventory", + str(inventory), + "--values", + str(values), + "--gpu-resolution", + str(resolution), + "--pxe-vars", + str(pxe_vars), + ) + + assert result.returncode == 1 + assert "pxe_gpu_access_enabled disagrees" in result.stdout + + def test_generator_rejects_unknown_accelerator_keys_before_writing_artifacts(tmp_path: Path) -> None: spec = write_file( tmp_path / "spec.json", @@ -824,13 +989,13 @@ def generator_spec(topology: str = "ssh-preinstalled", accelerators: object | No spec["accelerators"] = accelerators if topology == "pxe-diskless": spec["network"] = {"interface": "enp1s0", "subnet": "192.168.1.0/24"} - spec["pxe"] = {"authorized_keys": ["ssh-ed25519 AAAA test@example"]} + spec["pxe"] = {"authorized_keys": ["ssh-ed25519 AAAA test@example"], "diskless_agents_have_amd_gpus": False} return spec def test_generator_validates_all_pxe_requirements_before_writing(tmp_path: Path) -> None: spec = generator_spec("pxe-diskless") - spec["pxe"] = {"authorized_keys": []} + spec["pxe"] = {"authorized_keys": [], "diskless_agents_have_amd_gpus": False} spec_path = write_file(tmp_path / "spec.json", json.dumps(spec)) out_dir = tmp_path / "generated" @@ -924,7 +1089,7 @@ def test_generator_force_replaces_symlink_entry_without_following_target(tmp_pat def test_generator_force_failure_restores_all_original_destination_types( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - module = load_generator_module() + module = load_deploy_module("deploy_artifact_store", ARTIFACT_STORE) inventory = write_file(tmp_path / "inventory.yml", "old inventory\n") pxe_vars = tmp_path / "pb-pxe-controller.vars.yml" pxe_vars.mkdir() @@ -956,6 +1121,29 @@ def fail_late_replace(source, destination): assert values_target.read_text(encoding="utf-8") == "old symlink target\n" +def test_artifact_store_rolls_back_non_force_destination_after_post_link_fsync_failure( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + module = load_deploy_module("deploy_artifact_store_nonforce_fsync", ARTIFACT_STORE) + destination = tmp_path / "inventory.yml" + original_fsync_parent = module._fsync_parent + calls = 0 + + def fail_after_publication(path: Path) -> None: + nonlocal calls + calls += 1 + if calls == 1: + raise OSError("injected parent fsync failure") + original_fsync_parent(path) + + monkeypatch.setattr(module, "_fsync_parent", fail_after_publication) + + with pytest.raises(SystemExit): + module.publish_artifacts([(destination, "new inventory\n", 0o600, True)], force=False) + + assert not destination.exists() + + def test_generated_overlay_activates_selected_accelerators_for_validation(tmp_path: Path) -> None: repo = tmp_path / "checkout" base_values = write_file( diff --git a/tests/skills/test_direct_inventory_validation.py b/tests/skills/test_direct_inventory_validation.py new file mode 100644 index 00000000..805ed26c --- /dev/null +++ b/tests/skills/test_direct_inventory_validation.py @@ -0,0 +1,188 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +"""Public CLI tests for direct SSH inventory validation.""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +VALIDATE = ROOT / "skills" / "deploy-aup-learning-cloud" / "scripts" / "validate.py" + + +def run_validate(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run([sys.executable, str(VALIDATE), *args], capture_output=True, text=True, check=False) + + +def write(path: Path, content: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + return path + + +def valid_inventory() -> str: + return """k3s_cluster: + children: + server: + hosts: + server: + auplc_gpu_access_enabled: true + agent: + hosts: + agent: + auplc_gpu_access_enabled: false +""" + + +def test_validator_requires_gpu_resolution_for_pxe_inventory_only(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + inventory = write( + repo / "inventory.yml", + valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: auto"), + ) + values = write(repo / "values.yaml", "custom:\n resources:\n metadata: {}\n") + write(repo / "deploy/ansible/inventory.yml", "k3s_version: v1.32.3+k3s1\n") + pxe_vars = write( + repo / "pxe-vars.yml", + """pxe_network_interface: eno1 +pxe_subnet: 192.168.1.0/24 +pxe_controller_ip: 192.168.1.10 +pxe_dns_servers: 8.8.8.8 +pxe_k3s_server_ips: [192.168.1.10] +pxe_rootfs_authorized_keys: [ssh-ed25519-AAA] +pxe_k3s_version: v1.32.3+k3s1 +pxe_gpu_access_enabled: false +""", + ) + + result = run_validate( + "--repo", + str(repo), + "--topology", + "pxe-diskless", + "--inventory", + str(inventory), + "--values", + str(values), + "--pxe-vars", + str(pxe_vars), + ) + + assert result.returncode == 1 + assert "pxe-diskless inventory validation requires --gpu-resolution" in result.stdout + + +@pytest.mark.parametrize("value", ("auto", "true", "false")) +def test_validator_accepts_direct_inventory_values_without_resolution_manifest(tmp_path: Path, value: str) -> None: + repo = tmp_path / "checkout" + inventory = write(repo / "inventory.yml", valid_inventory().replace("true", value).replace("false", value)) + values = write(repo / "values.yaml", "custom:\n resources:\n metadata: {}\n") + + result = run_validate( + "--repo", str(repo), "--topology", "ssh-preinstalled", "--inventory", str(inventory), "--values", str(values) + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert "GPU access inventory is valid" in result.stdout + + +def test_validator_rejects_auto_when_inventory_is_cross_checked_with_gpu_resolution(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + inventory = write( + repo / "inventory.yml", + valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: auto"), + ) + values = write(repo / "values.yaml", "custom:\n resources:\n metadata: {}\n") + resolution = write( + repo / "gpu-access-resolution.json", + """{ + "version": 1, + "status": "gpu_resolved", + "hosts": {"agent": false, "server": true} +} +""", + ) + + result = run_validate( + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--inventory", + str(inventory), + "--gpu-resolution", + str(resolution), + "--values", + str(values), + ) + + assert result.returncode == 1 + assert "inventory host 'server' has malformed auplc_gpu_access_enabled" in result.stdout + + +@pytest.mark.parametrize( + ("inventory_content", "expected_error"), + [ + (valid_inventory().replace(" auplc_gpu_access_enabled: true\n", ""), "must define exactly one"), + (valid_inventory().replace("auplc_gpu_access_enabled: true", 'auplc_gpu_access_enabled: "auto"'), "malformed"), + (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: yes"), "malformed"), + (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: AUTO"), "malformed"), + ( + valid_inventory().replace( + " auplc_gpu_access_enabled: true\n", + " auplc_gpu_access_enabled: true\n auplc_gpu_access_enabled: false\n", + ), + "must define exactly one", + ), + ], +) +def test_validator_rejects_invalid_direct_inventory( + tmp_path: Path, inventory_content: str, expected_error: str +) -> None: + repo = tmp_path / "checkout" + inventory = write(repo / "inventory.yml", inventory_content) + values = write(repo / "values.yaml", "custom:\n resources:\n metadata: {}\n") + + result = run_validate( + "--repo", str(repo), "--topology", "ssh-preinstalled", "--inventory", str(inventory), "--values", str(values) + ) + + assert result.returncode == 1 + assert expected_error in result.stdout + + +def test_validator_reports_direct_inventory_not_found(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + values = write(repo / "values.yaml", "custom:\n resources:\n metadata: {}\n") + + result = run_validate( + "--repo", str(repo), "--topology", "ssh-preinstalled", "--inventory", "missing.yml", "--values", str(values) + ) + + assert result.returncode == 1 + assert "inventory not found" in result.stdout + assert "generated inventory not found" not in result.stdout + + +def test_validator_rejects_gpu_resolution_without_inventory(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + values = write(repo / "values.yaml", "custom:\n resources:\n metadata: {}\n") + resolution = write(repo / "resolution.json", "{}\n") + + result = run_validate( + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--gpu-resolution", + str(resolution), + "--values", + str(values), + ) + + assert result.returncode == 1 + assert "--gpu-resolution requires --inventory" in result.stdout diff --git a/tests/skills/test_gpu_access_resolution.py b/tests/skills/test_gpu_access_resolution.py new file mode 100644 index 00000000..6cd29546 --- /dev/null +++ b/tests/skills/test_gpu_access_resolution.py @@ -0,0 +1,182 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +"""Behavior tests for fleet GPU-access discovery resolution.""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +RESOLUTION = ROOT / "skills" / "deploy-aup-learning-cloud" / "scripts" / "gpu_access_resolution.py" +MANIFEST = ROOT / "skills" / "deploy-aup-learning-cloud" / "scripts" / "gpu_resolution_manifest.py" +DISCOVERY_PLAYBOOK = ROOT / "deploy" / "ansible" / "playbooks" / "pb-gpu-access-discovery.yml" +GPU_BDF = "0000:03:00.0" + + +def load_resolution_module(): + sys.path.insert(0, str(RESOLUTION.parent)) + spec = importlib.util.spec_from_file_location("gpu_access_resolution", RESOLUTION) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + try: + spec.loader.exec_module(module) + return module + finally: + sys.path.pop(0) + + +def load_manifest_module(): + spec = importlib.util.spec_from_file_location("gpu_resolution_manifest", MANIFEST) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def host_evidence( + host: str, + *, + lspci_bdfs: list[str] | None = None, + sysfs_bdfs: list[str] | None = None, + lspci_rc: int = 0, + sysfs_rc: int = 0, + reachable: bool = True, +) -> dict: + lspci = "\n".join(lspci_bdfs or []) + sysfs = "\n".join(sysfs_bdfs if sysfs_bdfs is not None else lspci_bdfs or []) + return { + "host": host, + "reachable": reachable, + "lspci": {"rc": lspci_rc, "stdout": lspci}, + "sysfs": {"rc": sysfs_rc, "stdout": sysfs}, + } + + +def evidence_document(*hosts: dict) -> str: + return json.dumps({"version": 1, "hosts": list(hosts)}) + + +def expected_targets(module, *names: str): + return tuple(module.InventoryTarget(name=name) for name in names) + + +def test_discovery_playbook_records_lspci_and_sysfs_evidence() -> None: + playbook = DISCOVERY_PLAYBOOK.read_text(encoding="utf-8") + assert "_auplc_discovery_lspci.rc" in playbook + assert "_auplc_discovery_lspci.stdout" in playbook + assert "_auplc_gpu_access_sysfs.rc" in playbook + assert "_auplc_gpu_access_sysfs.stdout" in playbook + + +def test_parse_fleet_evidence_accepts_the_exact_machine_evidence_schema() -> None: + module = load_resolution_module() + + evidence = module.parse_fleet_evidence(evidence_document(host_evidence("gpu-1", lspci_bdfs=[GPU_BDF]))) + + assert evidence[0].target == module.InventoryTarget(name="gpu-1") + assert evidence[0].lspci.stdout == GPU_BDF + assert evidence[0].sysfs.stdout == GPU_BDF + + +def test_parse_fleet_evidence_rejects_boolean_integer_values() -> None: + module = load_resolution_module() + + with pytest.raises(module.EvidenceParseError): + module.parse_fleet_evidence(json.dumps({"version": True, "hosts": []})) + + +def test_resolve_fleet_classifies_matching_amd_bdfs_as_gpu() -> None: + module = load_resolution_module() + evidence = module.parse_fleet_evidence(evidence_document(host_evidence("gpu-1", lspci_bdfs=[GPU_BDF]))) + + resolution = module.resolve_fleet(expected_targets(module, "gpu-1"), evidence) + + assert resolution.status is module.FleetStatus.GPU_RESOLVED + assert resolution.hosts[0].status is module.HostStatus.GPU + + +def test_resolve_fleet_classifies_two_empty_successful_gpu_probes_as_cpu_only() -> None: + module = load_resolution_module() + evidence = module.parse_fleet_evidence(evidence_document(host_evidence("cpu-1"))) + + resolution = module.resolve_fleet(expected_targets(module, "cpu-1"), evidence) + + assert resolution.status is module.FleetStatus.CPU_ONLY + assert resolution.hosts[0].status is module.HostStatus.CPU + + +def test_resolve_fleet_blocks_disagreeing_lspci_and_sysfs_evidence() -> None: + module = load_resolution_module() + parsed = module.parse_fleet_evidence( + evidence_document(host_evidence("host-1", lspci_bdfs=[GPU_BDF], sysfs_bdfs=["0000:04:00.0"])) + ) + + resolution = module.resolve_fleet(expected_targets(module, "host-1"), parsed) + + assert resolution.status is module.FleetStatus.BLOCKED + assert resolution.hosts[0].status is module.HostStatus.UNKNOWN + + +def test_resolve_fleet_blocks_incomplete_host_evidence() -> None: + module = load_resolution_module() + parsed = module.parse_fleet_evidence(evidence_document(host_evidence("gpu-1", lspci_bdfs=[GPU_BDF]))) + + resolution = module.resolve_fleet(expected_targets(module, "gpu-1", "gpu-2"), parsed) + + assert resolution.status is module.FleetStatus.BLOCKED + assert resolution.reason == "incomplete host coverage" + + +def test_resolve_fleet_accepts_gpu_hosts_without_a_shared_gid() -> None: + module = load_resolution_module() + parsed = module.parse_fleet_evidence( + evidence_document( + host_evidence("gpu-1", lspci_bdfs=[GPU_BDF]), + host_evidence("gpu-2", lspci_bdfs=["0000:04:00.0"]), + ) + ) + + resolution = module.resolve_fleet(expected_targets(module, "gpu-1", "gpu-2"), parsed) + + assert resolution.status is module.FleetStatus.GPU_RESOLVED + assert [host.status for host in resolution.hosts] == [module.HostStatus.GPU, module.HostStatus.GPU] + + +def test_resolution_manifest_preserves_explicit_host_booleans() -> None: + module = load_resolution_module() + parsed = module.parse_fleet_evidence( + evidence_document( + host_evidence("gpu-1", lspci_bdfs=[GPU_BDF]), + host_evidence("cpu-1"), + ) + ) + + manifest = module.resolution_manifest(module.resolve_fleet(expected_targets(module, "gpu-1", "cpu-1"), parsed)) + + assert manifest == { + "version": 1, + "status": "gpu_resolved", + "hosts": {"cpu-1": False, "gpu-1": True}, + } + + +def test_pxe_resolution_manifest_constructs_without_mutating_base_manifest() -> None: + module = load_manifest_module() + base = module.build_resolution_manifest( + status="gpu_resolved", + hosts={"gpu-2": True, "gpu-1": True}, + ) + + manifest = module.build_pxe_resolution_manifest( + base, + gpu_access_enabled=True, + ) + + assert base["hosts"] == {"gpu-1": True, "gpu-2": True} + assert "pxe_rootfs" not in base + assert manifest["pxe_rootfs"] == {"gpu_access_enabled": True} diff --git a/tests/skills/test_gpu_access_role.py b/tests/skills/test_gpu_access_role.py new file mode 100644 index 00000000..f6efb872 --- /dev/null +++ b/tests/skills/test_gpu_access_role.py @@ -0,0 +1,174 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +"""Contract tests for AMD's packaged GPU udev rules in Ansible.""" + +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[2] +ANSIBLE = ROOT / "deploy" / "ansible" +GPU_ACCESS_ROLE = ANSIBLE / "roles" / "gpu_access" +PXE_CONTROLLER_ROLE = ANSIBLE / "roles" / "pxe_controller" +PXE_GPU_ACCESS_TASKS = PXE_CONTROLLER_ROLE / "tasks" / "gpu_access.yml" + +PACKAGE = "amdgpu-insecure-instinct-udev-rules" +VERSION = "30.30.4.0-2341068.24.04" +URL = f"https://repo.radeon.com/amdgpu/30.30.4/ubuntu/pool/main/a/{PACKAGE}/{PACKAGE}_{VERSION}_all.deb" +SHA256 = "4be865985c7a13114c45925e77bc0b411b9fd47d5040ed35df44b9c411766162" +RULE_PATH = "/etc/udev/rules.d/70-amdgpu.rules" +RULE_CONTENT = ( + 'KERNEL=="kfd", GROUP="render", MODE="0666"\nSUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0666"\n' +) + + +def read(path: Path) -> str: + return path.read_text(encoding="utf-8") + + +def test_gpu_access_role_enforces_pinned_package_contract() -> None: + defaults = yaml.safe_load(read(GPU_ACCESS_ROLE / "defaults" / "main.yml")) + apply = read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") + verify = read(GPU_ACCESS_ROLE / "tasks" / "verify.yml") + + assert [ + defaults[key] + for key in ( + "auplc_gpu_udev_package_name", + "auplc_gpu_udev_package_version", + "auplc_gpu_udev_package_url", + "auplc_gpu_udev_package_checksum", + "auplc_gpu_udev_rule_path", + "auplc_gpu_udev_rule_content", + ) + ] == [PACKAGE, VERSION, URL, f"sha256:{SHA256}", RULE_PATH, RULE_CONTENT] + assert all(token in apply for token in ("ansible.builtin.get_url", "ansible.builtin.apt", "checksum:", "deb:")) + assert all( + token in verify + for token in ( + "dpkg-query", + r"--showformat=${Status}\t${Version}", + "--search", + "install ok installed", + "_auplc_verify_live_rule_owner.stdout == auplc_gpu_udev_package_name + ': ' + auplc_gpu_udev_rule_path", + "(_auplc_verify_rule_content.content | b64decode) == auplc_gpu_udev_rule_content", + ) + ) + + +def test_gpu_access_defaults_and_inventory_leave_auto_unquoted() -> None: + defaults = read(GPU_ACCESS_ROLE / "defaults" / "main.yml") + inventory = read(ANSIBLE / "inventory.yml") + + assert "auplc_gpu_access_enabled: auto" in defaults + assert inventory.count("auplc_gpu_access_enabled: auto") == 2 + assert 'auplc_gpu_access_enabled: "auto"' not in inventory + assert "auplc_gpu_access_enabled: 'auto'" not in inventory + + +def test_gpu_access_rootfs_and_legacy_cleanup_remain_contained_and_verified() -> None: + validation = read(GPU_ACCESS_ROLE / "tasks" / "validate.yml") + preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") + apply = read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") + + assert all( + token in validation + for token in ( + "realpath", + "auplc_rootfs_path != '/'", + "_auplc_canonical_rootfs.stdout.startswith(_auplc_canonical_allowed_root.stdout + '/')", + ) + ) + assert all( + token in preflight + for token in ( + "follow: false", + "_auplc_legacy_gpu_rules", + "hash('sha256')", + "70-kfd.rules", + "70-rocm-devices.rules", + ) + ) + assert apply.index("ansible.builtin.import_tasks: verify.yml") < apply.rindex("state: absent") + assert apply.index("item.content | b64decode") < apply.rindex("state: absent") + + +def test_pxe_gpu_access_chroots_without_bind_mounts_and_rejects_unsafe_retained_rules() -> None: + main = read(PXE_CONTROLLER_ROLE / "tasks" / "main.yml") + tasks = read(PXE_GPU_ACCESS_TASKS) + apply = read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") + verify = read(GPU_ACCESS_ROLE / "tasks" / "verify.yml") + + assert main.index("pxe_gpu_admission_phase: retained-read-only") < main.index("rm -rf {{ pxe_nfs_root }}") + assert main.index("pxe_gpu_admission_phase: final") < main.index("ls {{ pxe_nfs_root }}/boot/vmlinuz-") + assert all( + token in tasks + for token in ( + "tasks_from: verify", + "tasks_from: preflight", + "tasks_from: apply", + 'auplc_rootfs_path: "{{ pxe_nfs_root }}"', + 'auplc_rootfs_allowed_root: "{{ pxe_nfs_allowed_root }}"', + "auplc_reject_legacy_gpu_rules: true", + ) + ) + assert "not item.stat.exists" in verify + assert "chroot" in apply + assert "apt-get" in apply + assert "mount --bind" not in apply + + +def test_pxe_unmounts_only_when_present_and_propagates_failures() -> None: + main = read(PXE_CONTROLLER_ROLE / "tasks" / "main.yml") + + assert main.count("set -e") == 2 + for mount in ("dev", "sys", "proc"): + assert main.count("if mountpoint -q {{ pxe_nfs_root }}/" + mount + "; then") == 2 + assert main.count("umount {{ pxe_nfs_root }}/" + mount) == 2 + assert "&& umount" not in main + assert "|| true" not in main + + +def test_gpu_access_resolves_before_preflight_rocm_and_apply_fail_fatally() -> None: + role_main = read(GPU_ACCESS_ROLE / "tasks" / "main.yml") + rocm = read(ANSIBLE / "playbooks" / "pb-rocm.yml") + udev = read(ANSIBLE / "playbooks" / "pb-udev.yml") + + assert role_main.index("import_tasks: resolve.yml") < role_main.index("import_tasks: preflight.yml") + assert role_main.index("import_tasks: preflight.yml") < role_main.index("import_tasks: apply.yml") + for playbook in (rocm, udev): + assert "any_errors_fatal: true" in playbook + assert playbook.index("tasks_from: resolve") < playbook.index("tasks_from: preflight") + assert playbook.index("tasks_from: preflight") < playbook.index("tasks_from: apply") + assert "when: _auplc_gpu_access_enabled_resolved" in playbook + assert rocm.index("tasks_from: preflight") < rocm.index("- role: rocm") < rocm.index("tasks_from: apply") + + +def test_gpu_access_auto_detection_requires_successful_boolean_resolution_before_preflight() -> None: + role_main = read(GPU_ACCESS_ROLE / "tasks" / "main.yml") + resolve = read(GPU_ACCESS_ROLE / "tasks" / "resolve.yml") + + assert "auplc_gpu_access_enabled == 'auto'" in resolve + assert resolve.index("ansible.builtin.import_tasks: detect.yml") < resolve.index("_auplc_gpu_access_sysfs.rc == 0") + assert resolve.index("_auplc_gpu_access_sysfs.rc == 0") < resolve.index("_auplc_gpu_access_enabled_resolved: >-") + assert resolve.index("_auplc_gpu_access_enabled_resolved: >-") < resolve.index( + "_auplc_gpu_access_enabled_resolved is boolean" + ) + assert role_main.index("ansible.builtin.import_tasks: resolve.yml") < role_main.index( + "ansible.builtin.import_tasks: preflight.yml" + ) + + +def test_gpu_access_rejects_unknown_unowned_udev_content_before_deletion() -> None: + role_main = read(GPU_ACCESS_ROLE / "tasks" / "main.yml") + preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") + apply = read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") + admission = preflight.split("_auplc_rule_content_admitted: >-", maxsplit=1)[1] + cleanup = apply.split("register: _auplc_apply_legacy_gpu_rule_contents", maxsplit=1)[1] + + assert "(_auplc_rule_owned_by_amd_package | bool)" in admission + assert "that: _auplc_rule_content_admitted | bool" in admission + assert role_main.index("ansible.builtin.import_tasks: preflight.yml") < role_main.index( + "ansible.builtin.import_tasks: apply.yml" + ) + assert cleanup.index("item.content | b64decode") < cleanup.index("state: absent") + assert "in item.item.item.sha256" in cleanup diff --git a/tests/skills/test_gpu_artifact_generation.py b/tests/skills/test_gpu_artifact_generation.py new file mode 100644 index 00000000..25bec235 --- /dev/null +++ b/tests/skills/test_gpu_artifact_generation.py @@ -0,0 +1,206 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +"""End-to-end contracts for automatic GPU artifact generation.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +GEN_CONFIGS = ROOT / "skills" / "deploy-aup-learning-cloud" / "scripts" / "gen_configs.py" + + +def evidence_host(name: str, *, gpu: bool = False, reachable: bool = True) -> dict: + bdf = "0000:03:00.0" if gpu else "" + return { + "host": name, + "reachable": reachable, + "lspci": {"rc": 0, "stdout": bdf}, + "sysfs": {"rc": 0, "stdout": bdf}, + } + + +def ssh_spec() -> dict: + return { + "topology": "ssh-preinstalled", + "k3s_version": "v1.32.3+k3s1", + "server": {"name": "server", "ip": "192.168.1.10"}, + "agents": [{"name": "agent", "ip": "192.168.1.11"}], + } + + +def write_fake_ansible(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, document: dict) -> None: + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + fake_ansible = fake_bin / "ansible-playbook" + fake_ansible.write_text( + """#!/usr/bin/env python3 +import json +import os +from pathlib import Path +import sys + +arguments = sys.argv[1:] +Path(os.environ["FAKE_ANSIBLE_RECORD"]).write_text(json.dumps(arguments), encoding="utf-8") +environment_record = os.environ.get("FAKE_ANSIBLE_ENV_RECORD") +if environment_record: + Path(environment_record).write_text( + json.dumps({key: os.environ.get(key) for key in ("ANSIBLE_CONFIG", "ANSIBLE_HOST_KEY_CHECKING", "ANSIBLE_SSH_ARGS", "ANSIBLE_SSH_COMMON_ARGS", "ANSIBLE_SSH_EXTRA_ARGS", "ANSIBLE_SSH_HOST_KEY_CHECKING", "ANSIBLE_SCP_IF_SSH", "ANSIBLE_SCP_EXTRA_ARGS", "ANSIBLE_SFTP_EXTRA_ARGS")}), + encoding="utf-8", + ) +output = next(value.split("=", 1)[1] for value in arguments if value.startswith("gpu_access_discovery_output_path=")) +Path(output).write_text(os.environ["FAKE_ANSIBLE_EVIDENCE"], encoding="utf-8") +""", + encoding="utf-8", + ) + fake_ansible.chmod(0o755) + monkeypatch.setenv("FAKE_ANSIBLE_RECORD", str(tmp_path / "ansible-argv.json")) + monkeypatch.setenv("FAKE_ANSIBLE_EVIDENCE", json.dumps(document)) + monkeypatch.setenv("PATH", f"{fake_bin}:{os.environ['PATH']}") + + +def run_generator(spec_path: Path, out_dir: Path, *extra: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(GEN_CONFIGS), "--spec", str(spec_path), "--out-dir", str(out_dir), *extra], + capture_output=True, + check=False, + text=True, + timeout=30, + ) + + +def write_json(path: Path, document: dict) -> Path: + path.write_text(json.dumps(document), encoding="utf-8") + return path + + +def test_generator_forces_repository_host_key_checking_over_disabled_environment( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_fake_ansible( + tmp_path, monkeypatch, {"version": 1, "hosts": [evidence_host("server"), evidence_host("agent")]} + ) + environment_record = tmp_path / "ansible-environment.json" + monkeypatch.setenv("FAKE_ANSIBLE_ENV_RECORD", str(environment_record)) + monkeypatch.setenv("ANSIBLE_CONFIG", str(tmp_path / "disabled-ansible.cfg")) + monkeypatch.setenv("ANSIBLE_HOST_KEY_CHECKING", "False") + monkeypatch.setenv("ANSIBLE_SSH_ARGS", "-o StrictHostKeyChecking=no") + monkeypatch.setenv("ANSIBLE_SSH_COMMON_ARGS", "-o UserKnownHostsFile=/dev/null") + monkeypatch.setenv("ANSIBLE_SSH_HOST_KEY_CHECKING", "False") + monkeypatch.setenv("ANSIBLE_SSH_EXTRA_ARGS", "-o StrictHostKeyChecking=no") + monkeypatch.setenv("ANSIBLE_SCP_IF_SSH", "True") + monkeypatch.setenv("ANSIBLE_SCP_EXTRA_ARGS", "-o UserKnownHostsFile=/dev/null") + monkeypatch.setenv("ANSIBLE_SFTP_EXTRA_ARGS", "-o StrictHostKeyChecking=no") + + result = run_generator(write_json(tmp_path / "spec.json", ssh_spec()), tmp_path / "generated") + + assert result.returncode == 0, result.stderr + assert json.loads(environment_record.read_text(encoding="utf-8")) == { + "ANSIBLE_CONFIG": str(ROOT / "deploy" / "ansible" / "ansible.cfg"), + "ANSIBLE_HOST_KEY_CHECKING": "True", + "ANSIBLE_SSH_ARGS": "-o StrictHostKeyChecking=yes", + "ANSIBLE_SSH_COMMON_ARGS": None, + "ANSIBLE_SSH_EXTRA_ARGS": None, + "ANSIBLE_SSH_HOST_KEY_CHECKING": "True", + "ANSIBLE_SCP_IF_SSH": None, + "ANSIBLE_SCP_EXTRA_ARGS": None, + "ANSIBLE_SFTP_EXTRA_ARGS": None, + } + + +def test_generator_surfaces_redacted_bounded_ansible_failure_diagnostics( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + fake_ansible = fake_bin / "ansible-playbook" + fake_ansible.write_text( + "#!/bin/sh\nprintf '%s\\n' 'fatal: [server]: UNREACHABLE! token=do-not-disclose' >&2\nexit 2\n", + encoding="utf-8", + ) + fake_ansible.chmod(0o755) + monkeypatch.setenv("PATH", f"{fake_bin}:{os.environ['PATH']}") + + result = run_generator(write_json(tmp_path / "spec.json", ssh_spec()), tmp_path / "generated") + + assert result.returncode == 1 + assert "exit code 2" in result.stderr + assert "fatal: [server]: UNREACHABLE!" in result.stderr + assert "do-not-disclose" not in result.stderr + assert "token=" in result.stderr + + +def test_generator_discovers_mixed_ssh_targets_and_publishes_resolved_artifacts( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_fake_ansible( + tmp_path, + monkeypatch, + {"version": 1, "hosts": [evidence_host("server", gpu=True), evidence_host("agent")]}, + ) + out_dir = tmp_path / "generated" + + result = run_generator(write_json(tmp_path / "spec.json", ssh_spec()), out_dir) + + assert result.returncode == 0, result.stderr + inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") + assert inventory.count("auplc_gpu_access_enabled: true") == 1 + assert inventory.count("auplc_gpu_access_enabled: false") == 1 + assert "auplc_render_gid" not in inventory + assert json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) == { + "version": 1, + "status": "gpu_resolved", + "hosts": {"agent": False, "server": True}, + } + + +@pytest.mark.parametrize("failure", ["missing", "nonzero"]) +def test_generator_does_not_publish_when_ansible_is_unavailable_or_fails( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure: str +) -> None: + out_dir = tmp_path / "generated" + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + if failure == "nonzero": + fake_ansible = fake_bin / "ansible-playbook" + fake_ansible.write_text("#!/bin/sh\nexit 1\n", encoding="utf-8") + fake_ansible.chmod(0o755) + monkeypatch.setenv("PATH", str(fake_bin)) + + result = run_generator(write_json(tmp_path / "spec.json", ssh_spec()), out_dir) + + assert result.returncode == 1 + assert not (out_dir / "inventory.yml").exists() + assert not (out_dir / "values-basic-example.yaml").exists() + assert not (out_dir / "gpu-access-resolution.json").exists() + + +def test_generator_keeps_canonical_artifacts_unchanged_when_discovery_blocks( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_fake_ansible( + tmp_path, + monkeypatch, + {"version": 1, "hosts": [evidence_host("server", reachable=False), evidence_host("agent")]}, + ) + out_dir = tmp_path / "generated" + out_dir.mkdir() + inventory = out_dir / "inventory.yml" + values = out_dir / "values-basic-example.yaml" + manifest = out_dir / "gpu-access-resolution.json" + inventory.write_text("previous inventory\n", encoding="utf-8") + values.write_text("previous values\n", encoding="utf-8") + manifest.write_text("previous manifest\n", encoding="utf-8") + + result = run_generator(write_json(tmp_path / "spec.json", ssh_spec()), out_dir, "--force") + + assert result.returncode == 1 + assert inventory.read_text(encoding="utf-8") == "previous inventory\n" + assert values.read_text(encoding="utf-8") == "previous values\n" + assert manifest.read_text(encoding="utf-8") == "previous manifest\n" diff --git a/tests/skills/test_pxe_finalization.py b/tests/skills/test_pxe_finalization.py new file mode 100644 index 00000000..06313b0a --- /dev/null +++ b/tests/skills/test_pxe_finalization.py @@ -0,0 +1,147 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +"""End-to-end contracts for immediate PXE GPU policy generation.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +GEN_CONFIGS = ROOT / "skills" / "deploy-aup-learning-cloud" / "scripts" / "gen_configs.py" + + +def pxe_spec(gpu_agents: bool) -> dict: + return { + "topology": "pxe-diskless", + "k3s_version": "v1.32.3+k3s1", + "server": {"name": "controller", "ip": "192.168.1.10"}, + "agents": [{"name": "diskless-agent", "ip": "192.168.1.11"}], + "network": {"interface": "enp1s0", "subnet": "192.168.1.0/24"}, + "pxe": { + "authorized_keys": ["ssh-ed25519 AAAA test@example"], + "rootfs_password": "do-not-print-this-secret", + "diskless_agents_have_amd_gpus": gpu_agents, + }, + } + + +def write_json(path: Path, document: dict) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(document), encoding="utf-8") + return path + + +def write_fake_ansible(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, *, controller_gpu: bool = False) -> None: + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + fake_ansible = fake_bin / "ansible-playbook" + bdf = "0000:03:00.0" if controller_gpu else "" + fake_ansible.write_text( + f"""#!/usr/bin/env python3 +import json +import sys +from pathlib import Path + +output = next(value.split('=', 1)[1] for value in sys.argv if value.startswith('gpu_access_discovery_output_path=')) +Path(output).write_text(json.dumps({{ + 'version': 1, + 'hosts': [{{ + 'host': 'controller', 'reachable': True, + 'lspci': {{'rc': 0, 'stdout': {bdf!r}}}, + 'sysfs': {{'rc': 0, 'stdout': {bdf!r}}}, + }}], +}}), encoding='utf-8') +""", + encoding="utf-8", + ) + fake_ansible.chmod(0o755) + monkeypatch.setenv("PATH", f"{fake_bin}:{os.environ['PATH']}") + + +def run_generator(*arguments: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(GEN_CONFIGS), *arguments], + capture_output=True, + check=False, + text=True, + timeout=30, + ) + + +@pytest.mark.parametrize("policy", [(True, "true"), (False, "false")]) +def test_pxe_agents_publish_explicit_boolean_rootfs_policy( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, policy: tuple[bool, str] +) -> None: + gpu_agents, expected_policy = policy + write_fake_ansible(tmp_path, monkeypatch) + out_dir = tmp_path / "generated" + + result = run_generator( + "--spec", str(write_json(tmp_path / "spec.json", pxe_spec(gpu_agents))), "--out-dir", str(out_dir) + ) + + assert result.returncode == 0, result.stderr + inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") + manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) + pxe_vars = (out_dir / "pb-pxe-controller.vars.yml").read_text(encoding="utf-8") + assert "auplc_render_gid" not in inventory + pxe_vars + assert f"pxe_gpu_access_enabled: {expected_policy}" in pxe_vars + assert manifest["pxe_rootfs"] == {"gpu_access_enabled": gpu_agents} + assert "do-not-print-this-secret" not in result.stdout + result.stderr + + +def test_pxe_gpu_controller_and_rootfs_publish_independent_booleans( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_fake_ansible(tmp_path, monkeypatch, controller_gpu=True) + out_dir = tmp_path / "generated" + + result = run_generator("--spec", str(write_json(tmp_path / "spec.json", pxe_spec(True))), "--out-dir", str(out_dir)) + + assert result.returncode == 0, result.stderr + inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") + manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) + assert "auplc_gpu_access_enabled: true" in inventory + assert manifest["status"] == "gpu_resolved" + assert manifest["pxe_rootfs"] == {"gpu_access_enabled": True} + + +def test_pxe_generator_does_not_publish_when_controller_discovery_is_unknown( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + fake_ansible = fake_bin / "ansible-playbook" + fake_ansible.write_text( + """#!/usr/bin/env python3 +import json +import sys +from pathlib import Path + +output = next(value.split('=', 1)[1] for value in sys.argv if value.startswith('gpu_access_discovery_output_path=')) +Path(output).write_text(json.dumps({'version': 1, 'hosts': [{'host': 'controller', 'reachable': False, 'lspci': {'rc': 0, 'stdout': ''}, 'sysfs': {'rc': 0, 'stdout': ''}}]}), encoding='utf-8') +""", + encoding="utf-8", + ) + fake_ansible.chmod(0o755) + monkeypatch.setenv("PATH", f"{fake_bin}:{os.environ['PATH']}") + out_dir = tmp_path / "generated" + + result = run_generator("--spec", str(write_json(tmp_path / "spec.json", pxe_spec(True))), "--out-dir", str(out_dir)) + + assert result.returncode == 1 + assert not any( + (out_dir / name).exists() + for name in ( + "inventory.yml", + "pb-pxe-controller.vars.yml", + "values-basic-example.yaml", + "gpu-access-resolution.json", + ) + )