From c8cebedcb8f5669e214fb14457113e335e75fd5d Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:17 +0800 Subject: [PATCH 01/65] feat(installer): add guarded GPU access policy --- auplc_installer/gpu_access.py | 367 +++++++++++++++++ tests/installer/test_gpu_access.py | 627 +++++++++++++++++++++++++++++ 2 files changed, 994 insertions(+) create mode 100644 auplc_installer/gpu_access.py create mode 100644 tests/installer/test_gpu_access.py diff --git a/auplc_installer/gpu_access.py b/auplc_installer/gpu_access.py new file mode 100644 index 00000000..7c46991a --- /dev/null +++ b/auplc_installer/gpu_access.py @@ -0,0 +1,367 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +"""Single-node AMD GPU device-access source of truth. + +The host's existing ``render`` group is authoritative. Its numeric GID is +persisted here so installer reruns and runtime-only commands cannot silently +select a different permission model. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Protocol + +from auplc_installer.util import InstallerError, run, run_capture + +GPU_ACCESS_STATE_VERSION = 1 +MAX_RENDER_GID = (2**32) - 2 +GPU_ACCESS_STATE_PATH = Path("/var/lib/auplc/gpu-access.json") +GPU_ACCESS_RULES_PATH = Path("/etc/udev/rules.d/70-auplc-gpu-access.rules") +LEGACY_KFD_RULES_PATH = Path("/etc/udev/rules.d/70-kfd.rules") +LEGACY_AMDGPU_RULES_PATH = Path("/etc/udev/rules.d/70-amdgpu.rules") +LEGACY_ROCM_DEVICES_RULES_PATH = Path("/etc/udev/rules.d/70-rocm-devices.rules") +LEGACY_KFD_RULES = 'KERNEL=="kfd", MODE="0666"\nSUBSYSTEM=="drm", KERNEL=="renderD*", MODE="0666"\n' +LEGACY_AMDGPU_RULES = ( + "# ROCm device permissions\n" + "# Grant render group access to AMD GPU devices\n" + "# Reference: https://rocm.docs.amd.com/projects/install-on-linux/en/latest/install/prerequisites.html#using-udev-rules\n" + 'KERNEL=="kfd", GROUP="render", MODE="0660"\n' + 'SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660"\n' +) +LEGACY_AMDGPU_PXE_RULES = 'KERNEL=="kfd", MODE="0666"\nKERNEL=="renderD[0-9]*", MODE="0666"\n' +LEGACY_ROCM_DEVICES_RULES = ( + "# ROCm device permissions\n" + "# Ensure /dev/kfd and /dev/dri/renderD* are accessible by render group\n" + 'SUBSYSTEM=="kfd", GROUP="render", MODE="0660"\n' + 'SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660"\n' +) +LEGACY_RULE_CONTENTS: dict[Path, frozenset[str]] = { + LEGACY_KFD_RULES_PATH: frozenset((LEGACY_KFD_RULES,)), + LEGACY_AMDGPU_RULES_PATH: frozenset((LEGACY_AMDGPU_RULES, LEGACY_AMDGPU_PXE_RULES)), + LEGACY_ROCM_DEVICES_RULES_PATH: frozenset((LEGACY_ROCM_DEVICES_RULES,)), +} +UDEV_MANAGED_MARKER = "# Managed by auplc-installer: AMD GPU device access." +CANONICAL_UDEV_RULES = ( + f"{UDEV_MANAGED_MARKER}\n" + 'KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0660"\n' + 'SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0660"\n' +) +_FSYNC_PATH_SCRIPT = ( + "import os\n" + "import sys\n" + "fd = os.open(sys.argv[1], os.O_RDONLY)\n" + "try:\n" + " os.fsync(fd)\n" + "finally:\n" + " os.close(fd)\n" +) +_VERIFY_DEVICE_ACCESS_SCRIPT = ( + "import os, pathlib, stat, sys\n" + "gid = int(sys.argv[1])\n" + "paths = [pathlib.Path('/dev/kfd')]\n" + "for node in pathlib.Path('/sys/class/drm').glob('renderD*'):\n" + " driver = node / 'device' / 'driver'\n" + " if driver.exists() and driver.resolve().name == 'amdgpu': paths.append(pathlib.Path('/dev/dri') / node.name)\n" + "if len(paths) == 1: raise SystemExit('no AMD renderD device found')\n" + "for path in paths:\n" + " data = path.lstat()\n" + " if not stat.S_ISCHR(data.st_mode) or data.st_uid != 0 or data.st_gid != gid or stat.S_IMODE(data.st_mode) != 0o660: raise SystemExit(f'bad GPU device access: {path}')\n" +) + + +@dataclass(frozen=True) +class GpuAccessState: + """Versioned, immutable record of the host render-group GID.""" + + render_gid: int + version: int = GPU_ACCESS_STATE_VERSION + + def __post_init__(self) -> None: + if self.version != GPU_ACCESS_STATE_VERSION: + raise InstallerError(f"Unsupported GPU access state version: {self.version!r}") + _validate_render_gid(self.render_gid) + + +class GpuAccessHost(Protocol): + """Privileged host-operation seam for GPU access provisioning.""" + + def get_group_entry(self, group_name: str) -> str: + """Return the NSS group record for ``group_name``.""" + + def read_text(self, path: Path) -> str | None: + """Return a privileged file's text, or ``None`` when it is absent.""" + + def write_state_atomically(self, path: Path, text: str) -> None: + """Atomically replace a state file with same-directory persistence.""" + + def write_udev_rule(self, path: Path, text: str) -> None: + """Write a managed udev rule after reconciliation has authorized it.""" + + def reload_udev_rules(self) -> None: + """Reload host udev rules.""" + + def trigger_udev(self) -> None: + """Apply reloaded udev rules to current devices.""" + + def settle_udev(self) -> None: + """Wait until triggered udev events finish before inode verification.""" + + def remove_udev_rule(self, path: Path) -> None: + """Remove an explicitly recognized legacy udev rule.""" + + def verify_device_access(self, render_gid: int) -> None: + """Verify the relevant GPU device inodes use the requested access contract.""" + + def is_symlink(self, path: Path) -> bool: + """Return whether ``path`` is a symlink without following it.""" + + def is_regular_file(self, path: Path) -> bool: + """Return whether an existing ``path`` is a regular file.""" + + def path_exists(self, path: Path) -> bool: + """Return whether ``path`` exists after a separate symlink check.""" + + def is_directory(self, path: Path) -> bool: + """Return whether an existing ``path`` is a directory.""" + + +class SystemGpuAccessHost: + """Production host adapter using the installer's sudo-aware command helpers.""" + + def get_group_entry(self, group_name: str) -> str: + result = run_capture(["getent", "group", group_name], check=False) + if result.returncode != 0: + return "" + return result.stdout or "" + + def read_text(self, path: Path) -> str | None: + exists = run(["test", "-e", str(path)], sudo=True, check=False) + if exists.returncode != 0: + return None + result = run_capture(["cat", str(path)], sudo=True) + return result.stdout or "" + + def write_state_atomically(self, path: Path, text: str) -> None: + """Durably replace state with a same-directory temporary file.""" + self._write_text_atomically(path, text) + + def write_udev_rule(self, path: Path, text: str) -> None: + self._write_text_atomically(path, text) + + def _write_text_atomically(self, path: Path, text: str) -> None: + """Durably replace ``path`` after atomically renaming a temporary file.""" + _validate_parent_chain(self, path.parent) + run(["mkdir", "-p", str(path.parent)], sudo=True) + temporary_result = run_capture( + ["mktemp", str(path.parent / f".{path.name}.XXXXXX")], + sudo=True, + ) + temporary_path = (temporary_result.stdout or "").strip() + if not temporary_path: + raise InstallerError(f"Could not create temporary GPU access state beside {path}") + + try: + run(["tee", temporary_path], sudo=True, input_text=text) + run(["chmod", "0644", temporary_path], sudo=True) + self._fsync_path(temporary_path) + run(["mv", "-f", temporary_path, str(path)], sudo=True) + self._fsync_path(str(path.parent)) + except BaseException: + run(["rm", "-f", temporary_path], sudo=True, check=False) + raise + + def _fsync_path(self, path: str) -> None: + run(["python3", "-c", _FSYNC_PATH_SCRIPT, path], sudo=True) + + def reload_udev_rules(self) -> None: + run(["udevadm", "control", "--reload-rules"], sudo=True) + + def trigger_udev(self) -> None: + run(["udevadm", "trigger"], sudo=True) + + def settle_udev(self) -> None: + run(["udevadm", "settle"], sudo=True) + + def remove_udev_rule(self, path: Path) -> None: + run(["rm", "-f", str(path)], sudo=True) + + def verify_device_access(self, render_gid: int) -> None: + run(["python3", "-c", _VERIFY_DEVICE_ACCESS_SCRIPT, str(render_gid)], sudo=True) + + def is_symlink(self, path: Path) -> bool: + return run(["test", "-L", str(path)], sudo=True, check=False).returncode == 0 + + def is_regular_file(self, path: Path) -> bool: + return run(["test", "-f", str(path)], sudo=True, check=False).returncode == 0 + + def path_exists(self, path: Path) -> bool: + return run(["test", "-e", str(path)], sudo=True, check=False).returncode == 0 + + def is_directory(self, path: Path) -> bool: + return run(["test", "-d", str(path)], sudo=True, check=False).returncode == 0 + + +def serialize_gpu_access_state(state: GpuAccessState) -> str: + """Return the canonical on-disk JSON representation for ``state``.""" + return ( + json.dumps( + {"renderGid": state.render_gid, "version": state.version}, + separators=(",", ":"), + sort_keys=True, + ) + + "\n" + ) + + +def parse_gpu_access_state(text: str) -> GpuAccessState: + """Parse strict versioned GPU access state, failing closed on bad input.""" + try: + payload = json.loads(text) + except (TypeError, json.JSONDecodeError) as exc: + raise InstallerError("Malformed GPU access state") from exc + + if not isinstance(payload, dict) or set(payload) != {"renderGid", "version"}: + raise InstallerError("Malformed GPU access state") + + version = payload["version"] + render_gid = payload["renderGid"] + if type(version) is not int or version != GPU_ACCESS_STATE_VERSION: + raise InstallerError("Unsupported GPU access state version") + _validate_render_gid(render_gid) + return GpuAccessState(render_gid=render_gid, version=version) + + +def resolve_render_gid(getent_output: str) -> int: + """Parse the numeric GID from one ``getent group render`` record.""" + if not isinstance(getent_output, str): + raise InstallerError("Could not resolve the host render group") + + lines = getent_output.splitlines() + if len(lines) != 1: + raise InstallerError("Could not resolve the host render group") + + fields = lines[0].split(":") + if len(fields) != 4 or fields[0] != "render": + raise InstallerError("Could not resolve the host render group") + + raw_gid = fields[2] + if not raw_gid.isascii() or not raw_gid.isdecimal(): + raise InstallerError("Could not resolve the host render group") + + render_gid = int(raw_gid) + _validate_render_gid(render_gid) + return render_gid + + +def render_udev_rules() -> str: + """Return the canonical, least-privilege AMD GPU udev rules.""" + return CANONICAL_UDEV_RULES + + +def provision_gpu_access(host: GpuAccessHost | None = None) -> GpuAccessState: + """Create or reuse immutable state and reconcile the managed udev rule. + + When state is absent, adopt the current host ``render`` GID only after the + udev rule has been applied and verified. Existing state must match the host + group before any mutation occurs. + """ + return _reconcile_gpu_access(host if host is not None else SystemGpuAccessHost()) + + +def load_existing_gpu_access(host: GpuAccessHost | None = None) -> GpuAccessState: + """Reconcile runtime GPU access, adopting missing state for pre-change installs. + + Runtime, upgrade, and reinstall paths reuse persisted state when present. + For an installation created before GPU access state existed, this performs a + one-time host ``render`` GID adoption after udev verification. A persisted + GID that differs from the current host group remains a hard failure. + """ + return _reconcile_gpu_access(host if host is not None else SystemGpuAccessHost()) + + +def _reconcile_gpu_access(host: GpuAccessHost) -> GpuAccessState: + _validate_parent_chain(host, GPU_ACCESS_STATE_PATH.parent) + _validate_parent_chain(host, GPU_ACCESS_RULES_PATH.parent) + state_text = _read_regular_text(host, GPU_ACCESS_STATE_PATH) + host_gid = resolve_render_gid(host.get_group_entry("render")) + + if state_text is None: + state = GpuAccessState(render_gid=host_gid) + persist_state = True + else: + state = parse_gpu_access_state(state_text) + if state.render_gid != host_gid: + raise InstallerError( + f"Persisted render GID does not match the current host render group ({state.render_gid} != {host_gid})" + ) + persist_state = False + + legacy_paths = _legacy_rules_to_remove(host) + existing_rule = _read_regular_text(host, GPU_ACCESS_RULES_PATH) + rewrite_rule = _should_rewrite_udev_rule(existing_rule) + + for path in legacy_paths: + host.remove_udev_rule(path) + if rewrite_rule: + host.write_udev_rule(GPU_ACCESS_RULES_PATH, render_udev_rules()) + host.reload_udev_rules() + host.trigger_udev() + host.settle_udev() + host.verify_device_access(state.render_gid) + if persist_state: + host.write_state_atomically(GPU_ACCESS_STATE_PATH, serialize_gpu_access_state(state)) + + return state + + +def _read_regular_text(host: GpuAccessHost, path: Path) -> str | None: + if host.is_symlink(path): + raise InstallerError(f"Refusing symlinked GPU access file: {path}") + if not host.path_exists(path): + return None + if not host.is_regular_file(path): + raise InstallerError(f"Refusing non-regular GPU access file: {path}") + return host.read_text(path) + + +def _validate_parent_chain(host: GpuAccessHost, parent: Path) -> None: + components = [*reversed(parent.parents), parent] + for index, component in enumerate(components): + if host.is_symlink(component): + raise InstallerError(f"Refusing symlinked GPU access directory: {component}") + if not host.path_exists(component): + if index != len(components) - 1: + raise InstallerError(f"Missing parent GPU access directory: {component}") + return + if not host.is_directory(component): + raise InstallerError(f"Refusing non-directory GPU access parent: {component}") + + +def _legacy_rules_to_remove(host: GpuAccessHost) -> list[Path]: + removals: list[Path] = [] + for path, expected_contents in LEGACY_RULE_CONTENTS.items(): + content = _read_regular_text(host, path) + if content is None: + continue + if content not in expected_contents: + raise InstallerError(f"Refusing to remove unexpected legacy GPU udev rule: {path}") + removals.append(path) + return removals + + +def _should_rewrite_udev_rule(existing_rule: str | None) -> bool: + if existing_rule is None: + return True + if existing_rule == render_udev_rules(): + return False + if existing_rule.split("\n", maxsplit=1)[0] != UDEV_MANAGED_MARKER: + raise InstallerError(f"Refusing to overwrite unmanaged GPU udev rule: {GPU_ACCESS_RULES_PATH}") + return True + + +def _validate_render_gid(render_gid: object) -> None: + if type(render_gid) is not int or not 1 <= render_gid <= MAX_RENDER_GID: + raise InstallerError(f"Invalid render group GID: {render_gid!r}") diff --git a/tests/installer/test_gpu_access.py b/tests/installer/test_gpu_access.py new file mode 100644 index 00000000..e742c219 --- /dev/null +++ b/tests/installer/test_gpu_access.py @@ -0,0 +1,627 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +"""Tests for the single-node AMD GPU access source of truth.""" + +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from auplc_installer import gpu_access +from auplc_installer.gpu_access import ( + GPU_ACCESS_RULES_PATH, + GPU_ACCESS_STATE_PATH, + LEGACY_AMDGPU_PXE_RULES, + LEGACY_AMDGPU_RULES, + LEGACY_AMDGPU_RULES_PATH, + LEGACY_KFD_RULES, + LEGACY_KFD_RULES_PATH, + LEGACY_ROCM_DEVICES_RULES, + LEGACY_ROCM_DEVICES_RULES_PATH, + MAX_RENDER_GID, + GpuAccessState, + SystemGpuAccessHost, + load_existing_gpu_access, + parse_gpu_access_state, + provision_gpu_access, + render_udev_rules, + resolve_render_gid, + serialize_gpu_access_state, +) +from auplc_installer.util import InstallerError + + +class FakeGpuAccessHost: + """In-memory adapter for the installer host-operation seam.""" + + def __init__(self, *, getent_output: str, files: dict[Path, str] | None = None) -> None: + self.getent_output = getent_output + self.files = dict(files or {}) + self.calls: list[str] = [] + self.symlinks: set[Path] = set() + self.nonregular_files: set[Path] = set() + self.directories = { + Path("/"), + Path("/etc"), + Path("/etc/udev"), + Path("/etc/udev/rules.d"), + Path("/var"), + Path("/var/lib"), + Path("/var/lib/auplc"), + } + + def get_group_entry(self, group_name: str) -> str: + self.calls.append(f"get-group:{group_name}") + return self.getent_output + + def read_text(self, path: Path) -> str | None: + self.calls.append(f"read:{path}") + return self.files.get(path) + + def write_state_atomically(self, path: Path, text: str) -> None: + self.calls.append(f"write-state:{path}") + self.files[path] = text + + def write_udev_rule(self, path: Path, text: str) -> None: + self.calls.append(f"write-rule:{path}") + self.files[path] = text + + def remove_udev_rule(self, path: Path) -> None: + self.calls.append(f"remove-rule:{path}") + self.files.pop(path, None) + + def reload_udev_rules(self) -> None: + self.calls.append("reload-udev") + + def trigger_udev(self) -> None: + self.calls.append("trigger-udev") + + def settle_udev(self) -> None: + self.calls.append("settle-udev") + + def verify_device_access(self, render_gid: int) -> None: + self.calls.append(f"verify-devices:{render_gid}") + + def is_symlink(self, path: Path) -> bool: + return path in self.symlinks + + def is_regular_file(self, path: Path) -> bool: + return path in self.files + + def path_exists(self, path: Path) -> bool: + return path in self.files or path in self.symlinks or path in self.nonregular_files or path in self.directories + + def is_directory(self, path: Path) -> bool: + return path in self.directories + + +def test_gpu_access_state_round_trips_as_versioned_json() -> None: + state = GpuAccessState(render_gid=993) + + serialized = serialize_gpu_access_state(state) + + assert serialized == '{"renderGid":993,"version":1}\n' + assert parse_gpu_access_state(serialized) == state + + +@pytest.mark.parametrize( + "state_text", + [ + "not json", + '{"renderGid":993,"version":2}', + '{"renderGid":0,"version":1}', + f'{{"renderGid":{MAX_RENDER_GID + 1},"version":1}}', + '{"renderGid":true,"version":1}', + '{"renderGid":993,"unexpected":true,"version":1}', + ], +) +def test_parse_gpu_access_state_rejects_malformed_or_unsupported_state(state_text: str) -> None: + with pytest.raises(RuntimeError): + parse_gpu_access_state(state_text) + + +def test_resolve_render_gid_reads_the_numeric_getent_field() -> None: + assert resolve_render_gid("render:x:993:student\n") == 993 + + +@pytest.mark.parametrize( + "getent_output", + [ + "", + "video:x:44:student\n", + "render:x:0:student\n", + "render:x:not-a-number:student\n", + f"render:x:{MAX_RENDER_GID + 1}:student\n", + "render:x:993:student\nrender:x:994:student\n", + ], +) +def test_resolve_render_gid_rejects_missing_or_invalid_group_records(getent_output: str) -> None: + with pytest.raises(RuntimeError): + resolve_render_gid(getent_output) + + +def test_render_udev_rules_is_the_canonical_least_privilege_policy() -> None: + rules = render_udev_rules() + + assert rules == ( + "# Managed by auplc-installer: AMD GPU device access.\n" + 'KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0660"\n' + 'SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0660"\n' + ) + assert "card" not in rules + assert "0666" not in rules + assert "chmod" not in rules + + +def test_device_verification_uses_lstat_and_requires_character_devices() -> None: + assert "path.lstat()" in gpu_access._VERIFY_DEVICE_ACCESS_SCRIPT + assert "stat.S_ISCHR(data.st_mode)" in gpu_access._VERIFY_DEVICE_ACCESS_SCRIPT + + +@pytest.mark.parametrize("unsafe_parent", [Path("/etc/udev"), Path("/etc/udev/rules.d"), Path("/var/lib/auplc")]) +def test_symlinked_gpu_access_parent_fails_before_any_file_read_or_write(unsafe_parent: Path) -> None: + host = FakeGpuAccessHost(getent_output="render:x:993:student\n") + host.symlinks.add(unsafe_parent) + + with pytest.raises(InstallerError, match="symlinked GPU access directory"): + provision_gpu_access(host) + + assert not any(call.startswith(("read:", "write-", "remove-rule:")) for call in host.calls) + + +def test_nonregular_canonical_rule_fails_before_reading_or_writing_it() -> None: + host = FakeGpuAccessHost(getent_output="render:x:993:student\n") + host.nonregular_files.add(GPU_ACCESS_RULES_PATH) + + with pytest.raises(InstallerError, match="non-regular GPU access file"): + provision_gpu_access(host) + + assert f"read:{GPU_ACCESS_RULES_PATH}" not in host.calls + assert f"write-rule:{GPU_ACCESS_RULES_PATH}" not in host.calls + + +def test_provision_adopts_host_render_gid_and_installs_canonical_rule() -> None: + host = FakeGpuAccessHost(getent_output="render:x:993:student\n") + + state = provision_gpu_access(host) + + assert state == GpuAccessState(render_gid=993) + assert host.files[GPU_ACCESS_STATE_PATH] == '{"renderGid":993,"version":1}\n' + assert host.files[GPU_ACCESS_RULES_PATH] == render_udev_rules() + assert host.calls[-4:] == [ + "trigger-udev", + "settle-udev", + "verify-devices:993", + f"write-state:{GPU_ACCESS_STATE_PATH}", + ] + + +def test_provision_migrates_exact_legacy_rules_then_verifies_before_persisting_state() -> None: + host = FakeGpuAccessHost( + getent_output="render:x:993:student\n", + files={ + LEGACY_KFD_RULES_PATH: ('KERNEL=="kfd", MODE="0666"\nSUBSYSTEM=="drm", KERNEL=="renderD*", MODE="0666"\n'), + LEGACY_AMDGPU_RULES_PATH: ( + "# ROCm device permissions\n" + "# Grant render group access to AMD GPU devices\n" + "# Reference: https://rocm.docs.amd.com/projects/install-on-linux/en/latest/install/prerequisites.html#using-udev-rules\n" + 'KERNEL=="kfd", GROUP="render", MODE="0660"\n' + 'SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660"\n' + ), + }, + ) + + state = provision_gpu_access(host) + + assert state == GpuAccessState(render_gid=993) + assert LEGACY_KFD_RULES == ('KERNEL=="kfd", MODE="0666"\nSUBSYSTEM=="drm", KERNEL=="renderD*", MODE="0666"\n') + assert LEGACY_AMDGPU_RULES == ( + "# ROCm device permissions\n" + "# Grant render group access to AMD GPU devices\n" + "# Reference: https://rocm.docs.amd.com/projects/install-on-linux/en/latest/install/prerequisites.html#using-udev-rules\n" + 'KERNEL=="kfd", GROUP="render", MODE="0660"\n' + 'SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660"\n' + ) + assert LEGACY_KFD_RULES_PATH not in host.files + assert LEGACY_AMDGPU_RULES_PATH not in host.files + assert host.calls.index(f"remove-rule:{LEGACY_KFD_RULES_PATH}") < host.calls.index( + f"write-rule:{GPU_ACCESS_RULES_PATH}" + ) + assert host.calls[-3:] == ["settle-udev", "verify-devices:993", f"write-state:{GPU_ACCESS_STATE_PATH}"] + + +def test_provision_migrates_exact_legacy_rocm_devices_rule() -> None: + host = FakeGpuAccessHost( + getent_output="render:x:993:student\n", + files={ + LEGACY_ROCM_DEVICES_RULES_PATH: ( + "# ROCm device permissions\n" + "# Ensure /dev/kfd and /dev/dri/renderD* are accessible by render group\n" + 'SUBSYSTEM=="kfd", GROUP="render", MODE="0660"\n' + 'SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660"\n' + ), + }, + ) + + state = provision_gpu_access(host) + + assert state == GpuAccessState(render_gid=993) + assert LEGACY_ROCM_DEVICES_RULES == ( + "# ROCm device permissions\n" + "# Ensure /dev/kfd and /dev/dri/renderD* are accessible by render group\n" + 'SUBSYSTEM=="kfd", GROUP="render", MODE="0660"\n' + 'SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660"\n' + ) + assert LEGACY_ROCM_DEVICES_RULES_PATH not in host.files + + +def test_provision_migrates_exact_legacy_pxe_rule_at_amdgpu_path() -> None: + host = FakeGpuAccessHost( + getent_output="render:x:993:student\n", + files={ + LEGACY_AMDGPU_RULES_PATH: ('KERNEL=="kfd", MODE="0666"\nKERNEL=="renderD[0-9]*", MODE="0666"\n'), + }, + ) + + state = provision_gpu_access(host) + + assert state == GpuAccessState(render_gid=993) + assert LEGACY_AMDGPU_PXE_RULES == ('KERNEL=="kfd", MODE="0666"\nKERNEL=="renderD[0-9]*", MODE="0666"\n') + assert LEGACY_AMDGPU_RULES_PATH not in host.files + + +def test_near_legacy_pxe_rule_fails_closed_without_removal() -> None: + near_variant = 'KERNEL=="kfd", MODE="0666"\nKERNEL=="renderD*", MODE="0666"\n' + host = FakeGpuAccessHost(getent_output="render:x:993:student\n", files={LEGACY_AMDGPU_RULES_PATH: near_variant}) + + with pytest.raises(InstallerError, match="unexpected legacy"): + provision_gpu_access(host) + + assert host.files[LEGACY_AMDGPU_RULES_PATH] == near_variant + + +def test_modified_legacy_rocm_devices_rule_fails_closed_without_removal() -> None: + modified = ( + "# ROCm device permissions\n" + "# Ensure /dev/kfd and /dev/dri/renderD* are accessible by render group\n" + 'SUBSYSTEM=="kfd", GROUP="render", MODE="0666"\n' + 'SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660"\n' + ) + host = FakeGpuAccessHost(getent_output="render:x:993:student\n", files={LEGACY_ROCM_DEVICES_RULES_PATH: modified}) + + with pytest.raises(InstallerError, match="unexpected legacy"): + provision_gpu_access(host) + + assert host.files[LEGACY_ROCM_DEVICES_RULES_PATH] == modified + + +def test_provision_reapplies_and_verifies_matching_immutable_state() -> None: + host = FakeGpuAccessHost( + getent_output="render:x:993:student\n", + files={ + GPU_ACCESS_STATE_PATH: '{"renderGid":993,"version":1}\n', + GPU_ACCESS_RULES_PATH: render_udev_rules(), + }, + ) + + state = provision_gpu_access(host) + + assert state == GpuAccessState(render_gid=993) + assert not any(call.startswith("write-") for call in host.calls) + assert host.calls[-4:] == ["reload-udev", "trigger-udev", "settle-udev", "verify-devices:993"] + + +def test_provision_fails_before_mutation_when_persisted_gid_differs_from_host() -> None: + host = FakeGpuAccessHost( + getent_output="render:x:994:student\n", + files={GPU_ACCESS_STATE_PATH: '{"renderGid":993,"version":1}\n'}, + ) + + with pytest.raises(RuntimeError, match="does not match"): + provision_gpu_access(host) + + assert not any(call.startswith("write-") for call in host.calls) + assert "reload-udev" not in host.calls + assert "trigger-udev" not in host.calls + + +def test_provision_fails_before_writing_state_when_rule_is_unmanaged() -> None: + host = FakeGpuAccessHost( + getent_output="render:x:993:student\n", + files={GPU_ACCESS_RULES_PATH: 'KERNEL=="kfd", MODE="0666"\n'}, + ) + + with pytest.raises(RuntimeError, match="unmanaged"): + provision_gpu_access(host) + + assert GPU_ACCESS_STATE_PATH not in host.files + assert not any(call.startswith("write-") for call in host.calls) + + +def test_load_existing_gpu_access_adopts_missing_state_after_verification() -> None: + host = FakeGpuAccessHost(getent_output="render:x:993:student\n") + + state = load_existing_gpu_access(host) + + assert state == GpuAccessState(render_gid=993) + assert host.files[GPU_ACCESS_STATE_PATH] == '{"renderGid":993,"version":1}\n' + assert host.calls[-3:] == ["settle-udev", "verify-devices:993", f"write-state:{GPU_ACCESS_STATE_PATH}"] + + +def test_managed_rule_is_reconciled_and_reloaded_when_content_changes() -> None: + host = FakeGpuAccessHost( + getent_output="render:x:993:student\n", + files={ + GPU_ACCESS_STATE_PATH: '{"renderGid":993,"version":1}\n', + GPU_ACCESS_RULES_PATH: "# Managed by auplc-installer: AMD GPU device access.\nold rule\n", + }, + ) + + state = load_existing_gpu_access(host) + + assert state == GpuAccessState(render_gid=993) + assert host.files[GPU_ACCESS_RULES_PATH] == render_udev_rules() + assert host.calls[-5:] == [ + f"write-rule:{GPU_ACCESS_RULES_PATH}", + "reload-udev", + "trigger-udev", + "settle-udev", + "verify-devices:993", + ] + + +def test_system_adapter_persists_state_with_a_same_directory_temporary_file(monkeypatch) -> None: + commands: list[list[str]] = [] + capture_commands: list[list[str]] = [] + + def fake_run(command: list[str], **kwargs: object) -> SimpleNamespace: + commands.append(command) + if command[:2] == ["test", "-L"]: + return SimpleNamespace(returncode=1) + return SimpleNamespace(returncode=0) + + def fake_run_capture(command: list[str], **kwargs: object) -> SimpleNamespace: + capture_commands.append(command) + return SimpleNamespace(stdout="/var/lib/auplc/.gpu-access.json.temporary\n") + + monkeypatch.setattr(gpu_access, "run", fake_run) + monkeypatch.setattr(gpu_access, "run_capture", fake_run_capture) + + SystemGpuAccessHost().write_state_atomically(GPU_ACCESS_STATE_PATH, "state\n") + + assert capture_commands == [["mktemp", "/var/lib/auplc/.gpu-access.json.XXXXXX"]] + assert [command for command in commands if command[0] != "test"] == [ + ["mkdir", "-p", "/var/lib/auplc"], + ["tee", "/var/lib/auplc/.gpu-access.json.temporary"], + ["chmod", "0644", "/var/lib/auplc/.gpu-access.json.temporary"], + ["python3", "-c", gpu_access._FSYNC_PATH_SCRIPT, "/var/lib/auplc/.gpu-access.json.temporary"], + ["mv", "-f", "/var/lib/auplc/.gpu-access.json.temporary", "/var/lib/auplc/gpu-access.json"], + ["python3", "-c", gpu_access._FSYNC_PATH_SCRIPT, "/var/lib/auplc"], + ] + + +def test_system_adapter_persists_udev_rule_with_durable_atomic_replacement(monkeypatch) -> None: + commands: list[list[str]] = [] + capture_commands: list[list[str]] = [] + + def fake_run(command: list[str], **kwargs: object) -> SimpleNamespace: + commands.append(command) + if command[:2] == ["test", "-L"]: + return SimpleNamespace(returncode=1) + return SimpleNamespace(returncode=0) + + def fake_run_capture(command: list[str], **kwargs: object) -> SimpleNamespace: + capture_commands.append(command) + return SimpleNamespace(stdout="/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary\n") + + monkeypatch.setattr(gpu_access, "run", fake_run) + monkeypatch.setattr(gpu_access, "run_capture", fake_run_capture) + + SystemGpuAccessHost().write_udev_rule(GPU_ACCESS_RULES_PATH, "rule\n") + + assert capture_commands == [["mktemp", "/etc/udev/rules.d/.70-auplc-gpu-access.rules.XXXXXX"]] + assert [command for command in commands if command[0] != "test"] == [ + ["mkdir", "-p", "/etc/udev/rules.d"], + ["tee", "/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary"], + ["chmod", "0644", "/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary"], + ["python3", "-c", gpu_access._FSYNC_PATH_SCRIPT, "/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary"], + [ + "mv", + "-f", + "/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary", + "/etc/udev/rules.d/70-auplc-gpu-access.rules", + ], + ["python3", "-c", gpu_access._FSYNC_PATH_SCRIPT, "/etc/udev/rules.d"], + ] + + +def test_system_adapter_removes_temporary_file_when_durable_write_fails(monkeypatch) -> None: + commands: list[list[str]] = [] + + def fake_run(command: list[str], **kwargs: object) -> SimpleNamespace: + commands.append(command) + if command[:2] == ["test", "-L"]: + return SimpleNamespace(returncode=1) + if command == ["python3", "-c", gpu_access._FSYNC_PATH_SCRIPT, "/var/lib/auplc/.gpu-access.json.temporary"]: + raise InstallerError("fsync failed") + return SimpleNamespace(returncode=0) + + monkeypatch.setattr(gpu_access, "run", fake_run) + monkeypatch.setattr( + gpu_access, + "run_capture", + lambda command, **kwargs: SimpleNamespace(stdout="/var/lib/auplc/.gpu-access.json.temporary\n"), + ) + + with pytest.raises(InstallerError, match="fsync failed"): + SystemGpuAccessHost().write_state_atomically(GPU_ACCESS_STATE_PATH, "state\n") + + assert [command for command in commands if command[0] != "test"] == [ + ["mkdir", "-p", "/var/lib/auplc"], + ["tee", "/var/lib/auplc/.gpu-access.json.temporary"], + ["chmod", "0644", "/var/lib/auplc/.gpu-access.json.temporary"], + ["python3", "-c", gpu_access._FSYNC_PATH_SCRIPT, "/var/lib/auplc/.gpu-access.json.temporary"], + ["rm", "-f", "/var/lib/auplc/.gpu-access.json.temporary"], + ] + + +@pytest.mark.parametrize( + ("failing_method", "expected_calls"), + [ + ( + "write_udev_rule", + [ + "get-group:render", + f"write-rule:{GPU_ACCESS_RULES_PATH}", + ], + ), + ( + "reload_udev_rules", + [ + "get-group:render", + f"write-rule:{GPU_ACCESS_RULES_PATH}", + "reload-udev", + ], + ), + ( + "trigger_udev", + [ + "get-group:render", + f"write-rule:{GPU_ACCESS_RULES_PATH}", + "reload-udev", + "trigger-udev", + ], + ), + ], +) +def test_first_install_does_not_persist_state_until_udev_reconciliation_succeeds( + monkeypatch, + failing_method: str, + expected_calls: list[str], +) -> None: + host = FakeGpuAccessHost(getent_output="render:x:993:student\n") + original_method = getattr(host, failing_method) + + def fail_after_recording(*args: object) -> None: + original_method(*args) + raise InstallerError(f"{failing_method} failed") + + monkeypatch.setattr(host, failing_method, fail_after_recording) + + with pytest.raises(InstallerError, match=f"{failing_method} failed"): + provision_gpu_access(host) + + assert host.calls[-len(expected_calls) :] == expected_calls + assert GPU_ACCESS_STATE_PATH not in host.files + + +def test_failed_udev_reconciliation_never_rewrites_existing_state(monkeypatch) -> None: + original_state = '{"renderGid":993,"version":1}\n' + host = FakeGpuAccessHost( + getent_output="render:x:993:student\n", + files={ + GPU_ACCESS_STATE_PATH: original_state, + GPU_ACCESS_RULES_PATH: "# Managed by auplc-installer: AMD GPU device access.\nold rule\n", + }, + ) + + def fail_reload() -> None: + host.calls.append("reload-udev") + raise InstallerError("reload failed") + + monkeypatch.setattr(host, "reload_udev_rules", fail_reload) + + with pytest.raises(InstallerError, match="reload failed"): + provision_gpu_access(host) + + assert host.files[GPU_ACCESS_STATE_PATH] == original_state + assert not any(call.startswith("write-state:") for call in host.calls) + assert "trigger-udev" not in host.calls + + +def test_failed_reload_is_retried_and_only_persists_state_after_a_later_success(monkeypatch) -> None: + host = FakeGpuAccessHost(getent_output="render:x:993:student\n") + + def fail_reload() -> None: + host.calls.append("reload-udev") + raise InstallerError("reload failed") + + monkeypatch.setattr(host, "reload_udev_rules", fail_reload) + with pytest.raises(InstallerError, match="reload failed"): + provision_gpu_access(host) + assert GPU_ACCESS_STATE_PATH not in host.files + + monkeypatch.setattr(host, "reload_udev_rules", FakeGpuAccessHost.reload_udev_rules.__get__(host)) + state = provision_gpu_access(host) + + assert state == GpuAccessState(render_gid=993) + assert host.calls[-2:] == ["verify-devices:993", f"write-state:{GPU_ACCESS_STATE_PATH}"] + + +def test_failed_settle_is_retried_and_only_persists_state_after_a_later_success(monkeypatch) -> None: + host = FakeGpuAccessHost(getent_output="render:x:993:student\n") + + def fail_settle() -> None: + host.calls.append("settle-udev") + raise InstallerError("settle failed") + + monkeypatch.setattr(host, "settle_udev", fail_settle) + with pytest.raises(InstallerError, match="settle failed"): + provision_gpu_access(host) + assert GPU_ACCESS_STATE_PATH not in host.files + assert "verify-devices:993" not in host.calls + + monkeypatch.setattr(host, "settle_udev", FakeGpuAccessHost.settle_udev.__get__(host)) + state = provision_gpu_access(host) + + assert state == GpuAccessState(render_gid=993) + assert host.calls[-3:] == ["settle-udev", "verify-devices:993", f"write-state:{GPU_ACCESS_STATE_PATH}"] + + +def test_failed_inode_verification_does_not_adopt_state(monkeypatch) -> None: + host = FakeGpuAccessHost(getent_output="render:x:993:student\n") + + def fail_verification(render_gid: int) -> None: + host.calls.append(f"verify-devices:{render_gid}") + raise InstallerError("device ownership mismatch") + + monkeypatch.setattr(host, "verify_device_access", fail_verification) + + with pytest.raises(InstallerError, match="ownership mismatch"): + provision_gpu_access(host) + + assert host.calls[-1] == "verify-devices:993" + assert GPU_ACCESS_STATE_PATH not in host.files + + +@pytest.mark.parametrize("path", [LEGACY_KFD_RULES_PATH, LEGACY_AMDGPU_RULES_PATH, GPU_ACCESS_RULES_PATH]) +def test_symlinked_gpu_access_files_fail_closed_before_mutation(path: Path) -> None: + host = FakeGpuAccessHost(getent_output="render:x:993:student\n") + host.symlinks.add(path) + + with pytest.raises(InstallerError, match="symlinked"): + provision_gpu_access(host) + + assert GPU_ACCESS_STATE_PATH not in host.files + + +@pytest.mark.parametrize( + ("path", "content"), + [ + (LEGACY_KFD_RULES_PATH, 'KERNEL=="kfd", MODE="0666"\n'), + (LEGACY_AMDGPU_RULES_PATH, 'KERNEL=="kfd", GROUP="render", MODE="0660"\n'), + ], +) +def test_one_line_legacy_variants_fail_closed_without_removal(path: Path, content: str) -> None: + host = FakeGpuAccessHost( + getent_output="render:x:993:student\n", + files={path: content}, + ) + + with pytest.raises(InstallerError, match="unexpected legacy"): + provision_gpu_access(host) + + assert host.files[path] == content + assert GPU_ACCESS_STATE_PATH not in host.files From c6701a7d5fbac0cc2c7b5d2d7d57335aece1ca2a Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:17 +0800 Subject: [PATCH 02/65] feat(installer): classify local GPU hardware --- auplc_installer/gpu_hardware.py | 63 +++++++++++++++++ tests/installer/test_gpu_hardware.py | 102 +++++++++++++++++++++++++++ 2 files changed, 165 insertions(+) create mode 100644 auplc_installer/gpu_hardware.py create mode 100644 tests/installer/test_gpu_hardware.py diff --git a/auplc_installer/gpu_hardware.py b/auplc_installer/gpu_hardware.py new file mode 100644 index 00000000..0b292eb7 --- /dev/null +++ b/auplc_installer/gpu_hardware.py @@ -0,0 +1,63 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +"""Read-only local AMD GPU hardware classification from Linux PCI sysfs.""" + +from __future__ import annotations + +from enum import Enum +from pathlib import Path +from typing import Final + +PCI_DEVICES_ROOT: Final = Path("/sys/bus/pci/devices") +AMD_PCI_VENDOR: Final = "0x1002" +DISPLAY_CLASS_PREFIX: Final = "0x03" +_HEX_DIGITS: Final = frozenset("0123456789abcdef") + + +class GpuHardware(Enum): + """The local host's AMD display-hardware eligibility.""" + + GPU = "gpu" + CPU = "cpu" + UNKNOWN = "unknown" + + +def classify_gpu_hardware(pci_devices_root: Path = PCI_DEVICES_ROOT) -> GpuHardware: + """Classify local hardware using complete PCI vendor and class evidence.""" + try: + devices = tuple(pci_devices_root.iterdir()) + except OSError: + return GpuHardware.UNKNOWN + + if not devices: + return GpuHardware.UNKNOWN + + scan_is_complete = True + for device in devices: + vendor = _read_pci_attribute(device / "vendor") + pci_class = _read_pci_attribute(device / "class") + if vendor is None or pci_class is None or not _has_valid_pci_attributes(vendor, pci_class): + scan_is_complete = False + continue + if vendor == AMD_PCI_VENDOR and pci_class.startswith(DISPLAY_CLASS_PREFIX): + return GpuHardware.GPU + + return GpuHardware.CPU if scan_is_complete else GpuHardware.UNKNOWN + + +def _read_pci_attribute(path: Path) -> str | None: + try: + value = path.read_text(encoding="ascii").strip().lower() + except (OSError, UnicodeDecodeError): + return None + return value or None + + +def _has_valid_pci_attributes(vendor: str, pci_class: str) -> bool: + return _is_pci_hex(vendor, digits=4) and _is_pci_hex(pci_class, digits=6) + + +def _is_pci_hex(value: str, *, digits: int) -> bool: + return ( + len(value) == digits + 2 and value.startswith("0x") and all(character in _HEX_DIGITS for character in value[2:]) + ) diff --git a/tests/installer/test_gpu_hardware.py b/tests/installer/test_gpu_hardware.py new file mode 100644 index 00000000..6a455e26 --- /dev/null +++ b/tests/installer/test_gpu_hardware.py @@ -0,0 +1,102 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +"""Tests for local AMD GPU hardware classification from PCI sysfs evidence.""" + +from __future__ import annotations + +from pathlib import Path + +from auplc_installer.gpu_hardware import GpuHardware, classify_gpu_hardware + + +def test_classify_gpu_hardware_returns_gpu_for_amd_display_controller(tmp_path: Path) -> None: + pci_devices = tmp_path / "devices" + device = pci_devices / "0000:03:00.0" + device.mkdir(parents=True) + (device / "vendor").write_text("0x1002\n", encoding="ascii") + (device / "class").write_text("0x030200\n", encoding="ascii") + + hardware = classify_gpu_hardware(pci_devices) + + assert hardware is GpuHardware.GPU + + +def test_classify_gpu_hardware_returns_cpu_for_complete_scan_without_amd_display(tmp_path: Path) -> None: + pci_devices = tmp_path / "devices" + intel_display = pci_devices / "0000:00:02.0" + intel_display.mkdir(parents=True) + (intel_display / "vendor").write_text("0x8086\n", encoding="ascii") + (intel_display / "class").write_text("0x030000\n", encoding="ascii") + amd_audio = pci_devices / "0000:03:00.1" + amd_audio.mkdir() + (amd_audio / "vendor").write_text("0x1002\n", encoding="ascii") + (amd_audio / "class").write_text("0x040300\n", encoding="ascii") + + hardware = classify_gpu_hardware(pci_devices) + + assert hardware is GpuHardware.CPU + + +def test_classify_gpu_hardware_returns_unknown_when_pci_root_is_missing(tmp_path: Path) -> None: + hardware = classify_gpu_hardware(tmp_path / "missing") + + assert hardware is GpuHardware.UNKNOWN + + +def test_classify_gpu_hardware_returns_unknown_when_pci_root_is_empty(tmp_path: Path) -> None: + pci_devices = tmp_path / "devices" + pci_devices.mkdir() + + hardware = classify_gpu_hardware(pci_devices) + + assert hardware is GpuHardware.UNKNOWN + + +def test_classify_gpu_hardware_returns_unknown_for_incomplete_pci_evidence(tmp_path: Path) -> None: + pci_devices = tmp_path / "devices" + missing_vendor = pci_devices / "0000:00:02.0" + missing_vendor.mkdir(parents=True) + (missing_vendor / "class").write_text("0x030000\n", encoding="ascii") + + hardware = classify_gpu_hardware(pci_devices) + + assert hardware is GpuHardware.UNKNOWN + + +def test_classify_gpu_hardware_returns_unknown_for_malformed_pci_evidence(tmp_path: Path) -> None: + pci_devices = tmp_path / "devices" + malformed_vendor = pci_devices / "0000:00:02.0" + malformed_vendor.mkdir(parents=True) + (malformed_vendor / "vendor").write_text("0xZZZZ\n", encoding="ascii") + (malformed_vendor / "class").write_text("0x030000\n", encoding="ascii") + + hardware = classify_gpu_hardware(pci_devices) + + assert hardware is GpuHardware.UNKNOWN + + +def test_classify_gpu_hardware_returns_unknown_for_unreadable_pci_attribute(tmp_path: Path) -> None: + pci_devices = tmp_path / "devices" + unreadable_class = pci_devices / "0000:00:02.0" + unreadable_class.mkdir(parents=True) + (unreadable_class / "vendor").write_text("0x8086\n", encoding="ascii") + (unreadable_class / "class").mkdir() + + hardware = classify_gpu_hardware(pci_devices) + + assert hardware is GpuHardware.UNKNOWN + + +def test_classify_gpu_hardware_prefers_positive_amd_evidence_over_incomplete_sibling(tmp_path: Path) -> None: + pci_devices = tmp_path / "devices" + incomplete_device = pci_devices / "0000:00:02.0" + incomplete_device.mkdir(parents=True) + (incomplete_device / "vendor").write_text("0x8086\n", encoding="ascii") + gpu_device = pci_devices / "0000:03:00.0" + gpu_device.mkdir() + (gpu_device / "vendor").write_text("0x1002\n", encoding="ascii") + (gpu_device / "class").write_text("0x038000\n", encoding="ascii") + + hardware = classify_gpu_hardware(pci_devices) + + assert hardware is GpuHardware.GPU From 3e2bf6b98527d90fb0f3ddb899bd1c4b45725c01 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:17 +0800 Subject: [PATCH 03/65] feat(installer): wire GPU access into workflows --- auplc_installer/cli.py | 52 +++++- tests/installer/test_cli_gpu_access.py | 227 +++++++++++++++++++++++++ tests/installer/test_cli_helpers.py | 1 + 3 files changed, 271 insertions(+), 9 deletions(-) create mode 100644 tests/installer/test_cli_gpu_access.py diff --git a/auplc_installer/cli.py b/auplc_installer/cli.py index f1aea4a7..f2e9b538 100644 --- a/auplc_installer/cli.py +++ b/auplc_installer/cli.py @@ -13,8 +13,9 @@ import contextlib import sys import time -from collections.abc import Sequence +from collections.abc import Callable, Sequence from pathlib import Path +from typing import NoReturn from auplc_installer import __version__ from auplc_installer.catalog import parse_selection_spec @@ -22,6 +23,8 @@ detect_and_configure_gpu, refine_gpu_config_from_node_labels, ) +from auplc_installer.gpu_access import GpuAccessState, load_existing_gpu_access, provision_gpu_access +from auplc_installer.gpu_hardware import GpuHardware, classify_gpu_hardware from auplc_installer.helm import ( deploy_runtime, dev_quick_rollout, @@ -319,6 +322,22 @@ def cmd_install(state: InstallerState, *, pull: bool) -> None: keepalive.stop() +def _raise_unreachable_gpu_hardware(hardware: GpuHardware) -> NoReturn: + raise AssertionError(f"Unhandled GPU hardware classification: {hardware!r}") + + +def _render_gid_for_local_hardware(reconcile_gpu_access: Callable[[], GpuAccessState]) -> int | None: + match classify_gpu_hardware(): + case GpuHardware.GPU: + return reconcile_gpu_access().render_gid + case GpuHardware.CPU: + return None + case GpuHardware.UNKNOWN: + raise InstallerError("Could not determine local AMD GPU hardware; refusing to modify installer state") + case unreachable: + _raise_unreachable_gpu_hardware(unreachable) + + def _cmd_install_inner(state: InstallerState, *, pull: bool) -> None: """Body of ``cmd_install`` after sudo session has been primed.""" # Pre-compute the image-stage label so the user knows up-front which path @@ -330,13 +349,16 @@ def _cmd_install_inner(state: InstallerState, *, pull: bool) -> None: else: image_stage_label = "Pulling external images & building custom images" - total = 8 + total = 9 with stage("Detecting GPU", idx=1, total=total): detect_and_configure_gpu(state.gpu, gpu_type_override=state.gpu_type) + + with stage("Provisioning GPU device access", idx=2, total=total): + render_gid = _render_gid_for_local_hardware(provision_gpu_access) paths = state.runtime_paths() - with stage("Generating values overlay (initial)", idx=2, total=total): + with stage("Generating values overlay (initial)", idx=3, total=total): # First pass: use local detection so image pulls / builds get the # right GPU_TARGET. Overlay is regenerated again below from # labeller-published labels. @@ -346,13 +368,14 @@ def _cmd_install_inner(state: InstallerState, *, pull: bool) -> None: image_tag=state.image_tag, courses=state.courses, offline_mode=state.offline_mode, + render_gid=render_gid, overlay_path=paths.overlay_path, ) - with stage("Installing helm + k9s", idx=3, total=total): + with stage("Installing helm + k9s", idx=4, total=total): install_tools(offline_mode=state.offline_mode, bundle_dir=state.bundle_dir) - with stage("Installing K3s (single-node)", idx=4, total=total): + with stage("Installing K3s (single-node)", idx=5, total=total): install_k3s_single_node( offline_mode=state.offline_mode, bundle_dir=state.bundle_dir, @@ -360,7 +383,7 @@ def _cmd_install_inner(state: InstallerState, *, pull: bool) -> None: mirror_prefix=state.mirror_prefix, ) - with stage(image_stage_label, idx=5, total=total): + with stage(image_stage_label, idx=6, total=total): if state.offline_mode and state.bundle_dir is not None: load_offline_images(state.bundle_dir) elif pull: @@ -397,13 +420,13 @@ def _cmd_install_inner(state: InstallerState, *, pull: bool) -> None: k3s_images_dir=state.k3s_images_dir, ) - with stage("Deploying ROCm GPU device plugin + node labeller", idx=6, total=total): + with stage("Deploying ROCm GPU device plugin + node labeller", idx=7, total=total): deploy_rocm_gpu_device_plugin( offline_mode=state.offline_mode, bundle_dir=state.bundle_dir, ) - with stage("Refreshing values overlay from node labels", idx=7, total=total): + with stage("Refreshing values overlay from node labels", idx=8, total=total): refine_gpu_config_from_node_labels(state.gpu) generate_values_overlay( state.gpu, @@ -411,10 +434,11 @@ def _cmd_install_inner(state: InstallerState, *, pull: bool) -> None: image_tag=state.image_tag, courses=state.courses, offline_mode=state.offline_mode, + render_gid=render_gid, overlay_path=paths.overlay_path, ) - with stage("Deploying JupyterHub runtime (helm install + wait)", idx=8, total=total): + with stage("Deploying JupyterHub runtime (helm install + wait)", idx=9, total=total): deploy_runtime(paths) _print_success_banner() @@ -582,6 +606,7 @@ def cmd_dev_quick(state: InstallerState) -> None: def cmd_dev_deploy(state: InstallerState) -> None: + render_gid = _render_gid_for_local_hardware(load_existing_gpu_access) detect_and_configure_gpu(state.gpu, gpu_type_override=state.gpu_type) paths = state.runtime_paths() refine_gpu_config_from_node_labels(state.gpu) @@ -591,12 +616,14 @@ def cmd_dev_deploy(state: InstallerState) -> None: image_tag=state.image_tag, courses=state.courses, offline_mode=state.offline_mode, + render_gid=render_gid, overlay_path=paths.overlay_path, ) deploy_runtime(paths, dev=True) def cmd_dev_upgrade(state: InstallerState) -> None: + render_gid = _render_gid_for_local_hardware(load_existing_gpu_access) detect_and_configure_gpu(state.gpu, gpu_type_override=state.gpu_type) paths = state.runtime_paths() refine_gpu_config_from_node_labels(state.gpu) @@ -607,12 +634,14 @@ def cmd_dev_upgrade(state: InstallerState) -> None: image_tag=state.image_tag, courses=state.courses, offline_mode=state.offline_mode, + render_gid=render_gid, overlay_path=paths.overlay_path, ) upgrade_runtime(paths, dev=True) def cmd_dev_reinstall(state: InstallerState) -> None: + _render_gid_for_local_hardware(load_existing_gpu_access) with contextlib.suppress(InstallerError): remove_runtime() time.sleep(0.5) @@ -623,6 +652,7 @@ def cmd_dev_reinstall(state: InstallerState) -> None: def cmd_rt_install(state: InstallerState) -> None: + render_gid = _render_gid_for_local_hardware(load_existing_gpu_access) detect_and_configure_gpu(state.gpu, gpu_type_override=state.gpu_type) paths = state.runtime_paths() refine_gpu_config_from_node_labels(state.gpu) @@ -632,12 +662,14 @@ def cmd_rt_install(state: InstallerState) -> None: image_tag=state.image_tag, courses=state.courses, offline_mode=state.offline_mode, + render_gid=render_gid, overlay_path=paths.overlay_path, ) deploy_runtime(paths) def cmd_rt_upgrade(state: InstallerState) -> None: + render_gid = _render_gid_for_local_hardware(load_existing_gpu_access) detect_and_configure_gpu(state.gpu, gpu_type_override=state.gpu_type) paths = state.runtime_paths() refine_gpu_config_from_node_labels(state.gpu) @@ -648,6 +680,7 @@ def cmd_rt_upgrade(state: InstallerState) -> None: image_tag=state.image_tag, courses=state.courses, offline_mode=state.offline_mode, + render_gid=render_gid, overlay_path=paths.overlay_path, ) upgrade_runtime(paths) @@ -678,6 +711,7 @@ def cmd_rt_remove(state: InstallerState) -> None: def cmd_rt_reinstall(state: InstallerState) -> None: + _render_gid_for_local_hardware(load_existing_gpu_access) with contextlib.suppress(InstallerError): remove_runtime() time.sleep(0.5) diff --git a/tests/installer/test_cli_gpu_access.py b/tests/installer/test_cli_gpu_access.py new file mode 100644 index 00000000..ede70311 --- /dev/null +++ b/tests/installer/test_cli_gpu_access.py @@ -0,0 +1,227 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +"""GPU access sequencing tests for installer command orchestration.""" + +from __future__ import annotations + +from collections.abc import Callable +from contextlib import contextmanager +from pathlib import Path + +import pytest + +from auplc_installer import cli +from auplc_installer.gpu_access import GpuAccessState +from auplc_installer.gpu_hardware import GpuHardware +from auplc_installer.helm import RuntimePaths +from auplc_installer.state import InstallerState + + +@pytest.mark.parametrize( + ("hardware", "expected_render_gid", "expected_provision_count"), + [(GpuHardware.GPU, 993, 1), (GpuHardware.CPU, None, 0)], +) +def test_full_install_gates_gpu_access_without_skipping_later_gpu_flow( + monkeypatch, hardware: GpuHardware, expected_render_gid: int | None, expected_provision_count: int +) -> None: + events: list[object] = [] + stages: list[tuple[str, int, int]] = [] + state = InstallerState() + paths = RuntimePaths(chart_path=Path("chart"), values_path=Path("values.yaml"), overlay_path=Path("overlay.yaml")) + + @contextmanager + def fake_stage(label: str, *, idx: int, total: int): + stages.append((label, idx, total)) + yield + + def fake_overlay(*args: object, **kwargs: object) -> Path: + events.append(("overlay", kwargs["render_gid"])) + return paths.overlay_path + + monkeypatch.setattr(state, "runtime_paths", lambda: paths) + monkeypatch.setattr(cli, "stage", fake_stage) + monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: hardware) + monkeypatch.setattr(cli, "detect_and_configure_gpu", lambda *args, **kwargs: events.append("detect")) + monkeypatch.setattr( + cli, "provision_gpu_access", lambda: events.append("provision") or GpuAccessState(render_gid=993) + ) + monkeypatch.setattr(cli, "generate_values_overlay", fake_overlay) + monkeypatch.setattr(cli, "install_tools", lambda **kwargs: events.append("tools")) + monkeypatch.setattr(cli, "install_k3s_single_node", lambda **kwargs: events.append("k3s")) + monkeypatch.setattr(cli, "pull_custom_images", lambda **kwargs: events.append("custom-images")) + monkeypatch.setattr(cli, "pull_external_images", lambda **kwargs: events.append("external-images")) + monkeypatch.setattr(cli, "deploy_rocm_gpu_device_plugin", lambda **kwargs: events.append("device-plugin")) + monkeypatch.setattr(cli, "refine_gpu_config_from_node_labels", lambda *args, **kwargs: events.append("refine")) + monkeypatch.setattr(cli, "deploy_runtime", lambda *args, **kwargs: events.append("runtime")) + monkeypatch.setattr(cli, "_print_success_banner", lambda: events.append("success")) + + cli._cmd_install_inner(state, pull=True) + + assert events.count("provision") == expected_provision_count + if expected_provision_count: + assert events.index("provision") < events.index("device-plugin") + assert [event for event in events if isinstance(event, tuple)] == [ + ("overlay", expected_render_gid), + ("overlay", expected_render_gid), + ] + assert stages == [ + ("Detecting GPU", 1, 9), + ("Provisioning GPU device access", 2, 9), + ("Generating values overlay (initial)", 3, 9), + ("Installing helm + k9s", 4, 9), + ("Installing K3s (single-node)", 5, 9), + ("Pulling custom + external images", 6, 9), + ("Deploying ROCm GPU device plugin + node labeller", 7, 9), + ("Refreshing values overlay from node labels", 8, 9), + ("Deploying JupyterHub runtime (helm install + wait)", 9, 9), + ] + + +@pytest.mark.parametrize( + ("hardware", "expected_render_gid", "expected_load_count"), + [(GpuHardware.GPU, 993, 1), (GpuHardware.CPU, None, 0)], +) +def test_runtime_upgrade_gates_existing_gpu_access_without_provisioning( + monkeypatch, hardware: GpuHardware, expected_render_gid: int | None, expected_load_count: int +) -> None: + events: list[object] = [] + state = InstallerState() + paths = RuntimePaths(chart_path=Path("chart"), values_path=Path("values.yaml"), overlay_path=Path("overlay.yaml")) + + def fake_overlay(*args: object, **kwargs: object) -> Path: + events.append(("overlay", kwargs["render_gid"])) + return paths.overlay_path + + monkeypatch.setattr(state, "runtime_paths", lambda: paths) + monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: hardware) + monkeypatch.setattr( + cli, "load_existing_gpu_access", lambda: events.append("load") or GpuAccessState(render_gid=993) + ) + monkeypatch.setattr( + cli, "provision_gpu_access", lambda: (_ for _ in ()).throw(AssertionError("must not provision")) + ) + monkeypatch.setattr(cli, "detect_and_configure_gpu", lambda *args, **kwargs: events.append("detect")) + monkeypatch.setattr(cli, "refine_gpu_config_from_node_labels", lambda *args, **kwargs: events.append("refine")) + monkeypatch.setattr(cli, "_preserve_courses_for_upgrade", lambda *args, **kwargs: events.append("preserve-courses")) + monkeypatch.setattr(cli, "generate_values_overlay", fake_overlay) + monkeypatch.setattr(cli, "upgrade_runtime", lambda *args, **kwargs: events.append("upgrade-runtime")) + + cli.cmd_rt_upgrade(state) + + assert events.count("load") == expected_load_count + assert events[-5:] == ["detect", "refine", "preserve-courses", ("overlay", expected_render_gid), "upgrade-runtime"] + + +@pytest.mark.parametrize( + ("command", "expected_events"), + [ + (cli.cmd_dev_deploy, ("detect", "refine", "overlay:None", "deploy-runtime")), + (cli.cmd_dev_upgrade, ("detect", "refine", "preserve-courses", "overlay:None", "upgrade-runtime")), + (cli.cmd_rt_install, ("detect", "refine", "overlay:None", "deploy-runtime")), + (cli.cmd_rt_upgrade, ("detect", "refine", "preserve-courses", "overlay:None", "upgrade-runtime")), + ], +) +def test_cpu_hardware_skips_existing_access_and_preserves_runtime_flow( + monkeypatch, command: Callable[[InstallerState], None], expected_events: tuple[str, ...] +) -> None: + events: list[str] = [] + state = InstallerState() + paths = RuntimePaths(chart_path=Path("chart"), values_path=Path("values.yaml"), overlay_path=Path("overlay.yaml")) + + monkeypatch.setattr(state, "runtime_paths", lambda: paths) + monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: GpuHardware.CPU) + monkeypatch.setattr(cli, "load_existing_gpu_access", lambda: (_ for _ in ()).throw(AssertionError("must not load"))) + monkeypatch.setattr(cli, "detect_and_configure_gpu", lambda *args, **kwargs: events.append("detect")) + monkeypatch.setattr(cli, "refine_gpu_config_from_node_labels", lambda *args, **kwargs: events.append("refine")) + monkeypatch.setattr(cli, "_preserve_courses_for_upgrade", lambda *args, **kwargs: events.append("preserve-courses")) + monkeypatch.setattr( + cli, + "generate_values_overlay", + lambda *args, **kwargs: events.append(f"overlay:{kwargs['render_gid']}") or paths.overlay_path, + ) + monkeypatch.setattr(cli, "deploy_runtime", lambda *args, **kwargs: events.append("deploy-runtime")) + monkeypatch.setattr(cli, "upgrade_runtime", lambda *args, **kwargs: events.append("upgrade-runtime")) + + command(state) + + assert events == list(expected_events) + + +@pytest.mark.parametrize( + ("reinstall", "delegate_name"), + [ + (cli.cmd_dev_reinstall, "cmd_dev_deploy"), + (cli.cmd_rt_reinstall, "cmd_rt_install"), + ], +) +@pytest.mark.parametrize( + ("hardware", "expected_access_events"), + [(GpuHardware.GPU, ["load"]), (GpuHardware.CPU, [])], +) +def test_reinstall_gates_existing_gpu_access_before_removing_runtime( + monkeypatch, + reinstall: Callable[[InstallerState], None], + delegate_name: str, + hardware: GpuHardware, + expected_access_events: list[str], +) -> None: + events: list[str] = [] + state = InstallerState() + + monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: hardware) + monkeypatch.setattr( + cli, "load_existing_gpu_access", lambda: events.append("load") or GpuAccessState(render_gid=993) + ) + monkeypatch.setattr(cli, "remove_runtime", lambda: events.append("remove-runtime")) + monkeypatch.setattr(cli.time, "sleep", lambda seconds: events.append("sleep")) + monkeypatch.setattr(cli, delegate_name, lambda current_state: events.append("delegate")) + + reinstall(state) + + assert events == [*expected_access_events, "remove-runtime", "sleep", "delegate"] + + +def test_unknown_hardware_blocks_full_install_before_gpu_access_mutation(monkeypatch) -> None: + events: list[str] = [] + state = InstallerState() + + monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: GpuHardware.UNKNOWN) + monkeypatch.setattr(cli, "detect_and_configure_gpu", lambda *args, **kwargs: events.append("detect")) + monkeypatch.setattr( + cli, + "provision_gpu_access", + lambda: (_ for _ in ()).throw(AssertionError("must not provision")), + ) + + with pytest.raises(RuntimeError, match="hardware"): + cli._cmd_install_inner(state, pull=True) + + assert events == ["detect"] + + +@pytest.mark.parametrize( + ("reinstall", "delegate_name"), + [ + (cli.cmd_dev_reinstall, "cmd_dev_deploy"), + (cli.cmd_rt_reinstall, "cmd_rt_install"), + ], +) +def test_unknown_hardware_blocks_reinstall_before_runtime_removal( + monkeypatch, reinstall: Callable[[InstallerState], None], delegate_name: str +) -> None: + events: list[str] = [] + state = InstallerState() + + monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: GpuHardware.UNKNOWN) + monkeypatch.setattr( + cli, + "load_existing_gpu_access", + lambda: (_ for _ in ()).throw(AssertionError("must not load")), + ) + monkeypatch.setattr(cli, "remove_runtime", lambda: events.append("remove-runtime")) + monkeypatch.setattr(cli, delegate_name, lambda current_state: events.append("delegate")) + + with pytest.raises(RuntimeError, match="hardware"): + reinstall(state) + + assert events == [] diff --git a/tests/installer/test_cli_helpers.py b/tests/installer/test_cli_helpers.py index bc33cffa..2cc863e8 100644 --- a/tests/installer/test_cli_helpers.py +++ b/tests/installer/test_cli_helpers.py @@ -41,6 +41,7 @@ def _write_overlay(path: Path, courses: CourseSelection) -> None: image_tag="v1.0", courses=courses, offline_mode=False, + render_gid=993, overlay_path=path, ) From 0ea997876d7c7659020fa62eae3a6c2e8191164e Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:17 +0800 Subject: [PATCH 04/65] feat(installer): emit resolved render GID overlays --- auplc_installer/overlay.py | 8 +++++ tests/installer/test_overlay.py | 34 ++++++++++++++++++++ tests/installer/test_values_gpu_overrides.py | 9 ++++++ 3 files changed, 51 insertions(+) diff --git a/auplc_installer/overlay.py b/auplc_installer/overlay.py index 5815f391..202b5c6c 100644 --- a/auplc_installer/overlay.py +++ b/auplc_installer/overlay.py @@ -47,6 +47,7 @@ def emit_overlay( image_tag: str, courses: CourseSelection, offline_mode: bool, + render_gid: int | None, ) -> str: """Render the overlay as a string. Pure function — no I/O.""" buf = StringIO() @@ -66,6 +67,11 @@ def emit_overlay( buf.write(f"# Env selection : {courses.description()}\n") buf.write("# Regenerated on install/upgrade.\n") buf.write("custom:\n") + buf.write(" gpuAccess:\n") + if render_gid is None: + buf.write(" renderGid: null\n") + else: + buf.write(f" renderGid: {render_gid}\n") # --- accelerators --- any_accel_emitted = False @@ -164,6 +170,7 @@ def generate_values_overlay( image_tag: str, courses: CourseSelection, offline_mode: bool, + render_gid: int | None, overlay_path: Path, ) -> Path: """Render the overlay and write it to ``overlay_path``. Returns the path.""" @@ -175,6 +182,7 @@ def generate_values_overlay( image_tag=image_tag, courses=courses, offline_mode=offline_mode, + render_gid=render_gid, ) overlay_path.write_text(text, encoding="utf-8") return overlay_path diff --git a/tests/installer/test_overlay.py b/tests/installer/test_overlay.py index b028cdd9..c93e4fc7 100644 --- a/tests/installer/test_overlay.py +++ b/tests/installer/test_overlay.py @@ -27,6 +27,7 @@ ) from auplc_installer.gpu import GpuConfig, SkuEntry, append_product from auplc_installer.overlay import ( + GPU_RESOURCE_KEYS, emit_overlay, generate_values_overlay, try_load_courses_from_overlay, @@ -45,6 +46,7 @@ def _render( courses: CourseSelection, offline_mode: bool = False, image_tag: str = "v1.0", + render_gid: int | None = 993, ) -> tuple[str, dict]: text = emit_overlay( cfg, @@ -52,6 +54,7 @@ def _render( image_tag=image_tag, courses=courses, offline_mode=offline_mode, + render_gid=render_gid, ) return text, yaml.safe_load(text) @@ -93,6 +96,7 @@ def _write_and_read_back(courses: CourseSelection) -> CourseSelection | None: image_tag="v1.0", courses=courses, offline_mode=False, + render_gid=993, overlay_path=path, ) return try_load_courses_from_overlay(path) @@ -106,6 +110,36 @@ def test_default_selection_round_trips_valid_yaml() -> None: assert "teams" not in parsed["custom"] +def test_overlay_emits_explicit_gpu_access_gid_without_global_pod_groups() -> None: + text = emit_overlay( + _strix_halo_cfg(), + image_registry="ghcr.io/amdresearch", + image_tag="v1.0", + courses=CourseSelection.default(), + offline_mode=False, + render_gid=993, + ) + parsed = yaml.safe_load(text) + + assert parsed["custom"]["gpuAccess"]["renderGid"] == 993 + assert "supplementalGroups" not in text + + +def test_overlay_emits_null_render_gid_without_removing_gpu_resources() -> None: + _, parsed = _render( + _strix_halo_cfg(), + courses=CourseSelection.default(), + render_gid=None, + ) + + custom = parsed["custom"] + assert custom["gpuAccess"]["renderGid"] is None + assert set(custom["resources"]["images"]) == set(GPU_RESOURCE_KEYS) + assert set(custom["resources"]["metadata"]) == set(GPU_RESOURCE_KEYS) + assert "teams" not in custom + assert "profiles" not in custom + + def test_resource_images_use_primary_tag() -> None: _, parsed = _render(_strix_halo_cfg(), courses=CourseSelection.default()) images = parsed["custom"]["resources"]["images"] diff --git a/tests/installer/test_values_gpu_overrides.py b/tests/installer/test_values_gpu_overrides.py index 7f6ef6ad..18de940f 100644 --- a/tests/installer/test_values_gpu_overrides.py +++ b/tests/installer/test_values_gpu_overrides.py @@ -69,3 +69,12 @@ def test_default_values_route_gpu_resources_to_supported_image_tags() -> None: assert overrides[accelerator_key]["image"] == ( f"ghcr.io/amdresearch/{image_name}:latest-{gpu_target}" ), values_file + + +def test_default_values_use_fs_gid_without_overriding_pod_security_context() -> None: + for values_file in VALUES_FILES: + values = _load_values(values_file) + singleuser = values["singleuser"] + + assert singleuser["fsGid"] == 100, values_file + assert "securityContext" not in singleuser.get("extraPodConfig", {}), values_file From 50ecc64d59dc847f08e3e058c8d9bc127017b8cf Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:17 +0800 Subject: [PATCH 05/65] feat(chart): model GPU render GID --- runtime/chart/values.schema.json | 2 +- runtime/chart/values.schema.yaml | 14 ++++++++++++++ runtime/chart/values.yaml | 5 +++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/runtime/chart/values.schema.json b/runtime/chart/values.schema.json index ef7efffc..f53227d3 100644 --- a/runtime/chart/values.schema.json +++ b/runtime/chart/values.schema.json @@ -1 +1 @@ -{"$schema":"http://json-schema.org/draft-07/schema#","type":"object","additionalProperties":false,"required":["imagePullSecrets","hub","proxy","singleuser","ingress","prePuller","custom","cull","debug","rbac","global"],"properties":{"enabled":{"type":["boolean","null"]},"fullnameOverride":{"type":["string","null"]},"nameOverride":{"type":["string","null"]},"imagePullSecret":{"type":"object","required":["create"],"if":{"properties":{"create":{"const":true}}},"then":{"additionalProperties":false,"required":["registry","username","password"],"properties":{"create":{"type":"boolean"},"automaticReferenceInjection":{"type":"boolean"},"registry":{"type":"string"},"username":{"type":"string"},"password":{"type":"string"},"email":{"type":["string","null"]}}}},"imagePullSecrets":{"type":"array"},"hub":{"type":"object","additionalProperties":false,"required":["baseUrl"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"config":{"type":"object","additionalProperties":false,"patternProperties":{"^[A-Z].*$":{"type":"object","additionalProperties":true}},"properties":{"JupyterHub":{"type":"object","additionalProperties":true,"properties":{"subdomain_host":{"type":"string"}}}}},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"baseUrl":{"type":"string"},"command":{"type":"array"},"args":{"type":"array"},"cookieSecret":{"type":["string","null"]},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"db":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["sqlite-pvc","sqlite-memory","mysql","postgres","other"]},"pvc":{"type":"object","additionalProperties":false,"required":["storage"],"properties":{"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"selector":{"type":"object","additionalProperties":true},"storage":{"type":"string"},"accessModes":{"type":"array","items":{"type":["string","null"]}},"storageClassName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"upgrade":{"type":["boolean","null"]},"url":{"type":["string","null"]},"password":{"type":["string","null"]}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"initContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"extraConfig":{"type":"object","additionalProperties":true},"fsGid":{"type":["integer","null"],"minimum":0},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"ports":{"type":"object","additionalProperties":false,"properties":{"appProtocol":{"type":["string","null"]},"nodePort":{"type":["integer","null"],"minimum":0}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPorts":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"existingSecret":{"type":["string","null"]},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"activeServerLimit":{"type":["integer","null"]},"allowNamedServers":{"type":["boolean","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"concurrentSpawnLimit":{"type":["integer","null"]},"consecutiveFailureLimit":{"type":["integer","null"]},"podSecurityContext":{"additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"extraContainers":{"type":"array"},"extraVolumeMounts":{"type":"array"},"extraVolumes":{"type":"array"},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"namedServerLimitPerUser":{"type":["integer","null"]},"redirectToServer":{"type":["boolean","null"]},"resources":{"type":"object","additionalProperties":true},"lifecycle":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"services":{"type":"object","additionalProperties":true,"properties":{"name":{"type":"string"},"admin":{"type":"boolean"},"command":{"type":["string","array"]},"url":{"type":"string"},"api_token":{"type":["string","null"]},"apiToken":{"type":["string","null"]}}},"loadRoles":{"type":"object","additionalProperties":true},"shutdownOnLogout":{"type":["boolean","null"]},"templatePaths":{"type":"array"},"templateVars":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"proxy":{"type":"object","additionalProperties":false,"properties":{"chp":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraCommandLineFlags":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"resources":{"type":"object","additionalProperties":true},"defaultTarget":{"type":["string","null"]},"errorTarget":{"type":["string","null"]},"extraPodSpec":{"type":"object","additionalProperties":true}}},"secretToken":{"type":["string","null"]},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"nodePorts":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"loadBalancerPort":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"disableHttpPort":{"type":"boolean"},"extraPorts":{"type":"array"},"externalIPs":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"loadBalancerSourceRanges":{"type":"array"},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"https":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"type":{"enum":[null,"","letsencrypt","manual","offload","secret"]},"letsencrypt":{"type":"object","additionalProperties":false,"properties":{"contactEmail":{"type":["string","null"]},"acmeServer":{"type":["string","null"]}}},"manual":{"type":"object","additionalProperties":false,"properties":{"key":{"type":["string","null"]},"cert":{"type":["string","null"]}}},"secret":{"type":"object","additionalProperties":false,"properties":{"name":{"type":["string","null"]},"key":{"type":["string","null"]},"crt":{"type":["string","null"]}}},"hosts":{"type":"array"}}},"traefik":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraInitContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraDynamicConfig":{"type":"object","additionalProperties":true},"extraPorts":{"type":"array"},"extraStaticConfig":{"type":"object","additionalProperties":true},"extraVolumes":{"type":"array"},"extraVolumeMounts":{"type":"array"},"hsts":{"type":"object","additionalProperties":false,"required":["includeSubdomains","maxAge","preload"],"properties":{"includeSubdomains":{"type":"boolean"},"maxAge":{"type":"integer"},"preload":{"type":"boolean"}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"secretSync":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}}}},"monitoring":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"namespace":{"type":"string","default":"monitoring"},"releaseLabel":{"type":"string","default":"monitoring"},"hubMetrics":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"allowUnauthenticatedScrape":{"type":"boolean","default":false},"serviceAnnotations":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"serviceMonitor":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"interval":{"type":"string","default":"15s"},"authorization":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":true},"type":{"type":"string","default":"Bearer"},"hubServiceName":{"type":"string","minLength":1,"default":"prometheus-metrics"},"secret":{"type":"object","additionalProperties":false,"properties":{"create":{"type":"boolean","default":true},"name":{"type":"string","default":""},"key":{"type":"string","minLength":1,"default":"token"}}}}}}},"grafana":{"type":"object","additionalProperties":false,"properties":{"dashboard":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"prometheusRule":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"singleuser":{"type":"object","additionalProperties":false,"properties":{"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"podNameTemplate":{"type":["string","null"]},"cpu":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","null"]},"guarantee":{"type":["number","null"]}}},"memory":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","string","null"]},"guarantee":{"type":["number","string","null"]}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"initContainers":{"type":"array"},"profileList":{"type":"array"},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"extraEnv":{"type":["object","array"],"additionalProperties":true},"nodeSelector":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"extraNodeAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAntiAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"cloudMetadata":{"type":"object","additionalProperties":false,"required":["blockWithIptables","ip"],"properties":{"blockWithIptables":{"type":"boolean"},"ip":{"type":"string"}}},"cmd":{"type":["array","string","null"]},"defaultUrl":{"type":["string","null"]},"events":{"type":["boolean","null"]},"extraAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraContainers":{"type":"array"},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPodConfig":{"type":"object","additionalProperties":true},"extraResource":{"type":"object","additionalProperties":false,"properties":{"guarantees":{"type":"object","additionalProperties":true},"limits":{"type":"object","additionalProperties":true}}},"fsGid":{"type":["integer","null"]},"lifecycleHooks":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"networkTools":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}},"serviceAccountName":{"type":["string","null"]},"startTimeout":{"type":["integer","null"]},"storage":{"type":"object","additionalProperties":false,"required":["type","homeMountPath"],"properties":{"capacity":{"type":["string","null"]},"dynamic":{"type":"object","additionalProperties":false,"properties":{"pvcNameTemplate":{"type":["string","null"]},"storageAccessModes":{"type":"array","items":{"type":["string","null"]}},"storageClass":{"type":["string","null"]},"subPath":{"type":["string","null"]},"volumeNameTemplate":{"type":["string","null"]}}},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraVolumeMounts":{"type":["object","array","null"]},"extraVolumes":{"type":["object","array","null"]},"homeMountPath":{"type":"string"},"static":{"type":"object","additionalProperties":false,"properties":{"pvcName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"type":{"enum":["dynamic","static","none"]}}},"allowPrivilegeEscalation":{"type":["boolean","null"]},"uid":{"type":["integer","null"]}}},"scheduling":{"type":"object","additionalProperties":false,"properties":{"userScheduler":{"type":"object","additionalProperties":false,"required":["enabled","plugins","pluginConfig","logLevel"],"properties":{"enabled":{"type":"boolean"},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"logLevel":{"type":"integer"},"plugins":{"type":"object","additionalProperties":true},"pluginConfig":{"type":"array"},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"podPriority":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"globalDefault":{"type":"boolean"},"defaultPriority":{"type":"integer"},"imagePullerPriority":{"type":"integer"},"userPlaceholderPriority":{"type":"integer"}}},"userPlaceholder":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraPodSpec":{"type":"object","additionalProperties":true}}},"corePods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}},"userPods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}}}},"ingress":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"ingressClassName":{"type":["string","null"]},"hosts":{"type":"array"},"pathSuffix":{"type":["string","null"]},"pathType":{"enum":["Prefix","Exact","ImplementationSpecific"]},"tls":{"type":"array"},"extraPaths":{"type":"array"}}},"httpRoute":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"hostnames":{"type":"array"},"gateway":{"type":"object","additionalProperties":false,"required":["name"],"properties":{"name":{"type":"string"},"namespace":{"type":"string"},"sectionName":{"type":"string"}}}}},"prePuller":{"type":"object","additionalProperties":false,"required":["hook","continuous"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"hook":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"pullOnlyOnChanges":{"type":"boolean"},"podSchedulingWaitDuration":{"type":"integer"},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"serviceAccountImagePuller":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"continuous":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"pullProfileListImages":{"type":"boolean"},"extraImages":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]}}}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"pause":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}}}}}},"custom":{"type":"object","additionalProperties":true,"properties":{"authMode":{"type":"string","enum":["auto-login","dummy","github","multi"]},"adminUser":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"}}},"notifications":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"topbar":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}},"homepage":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"legacyAnnouncementFallback":{"type":"boolean"},"items":{"type":"array","items":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}}}}}}},"accelerators":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"displayName":{"type":"string"},"description":{"type":"string"},"nodeSelector":{"type":"object","additionalProperties":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"quotaRate":{"type":"integer","minimum":1}}}},"resources":{"type":"object","additionalProperties":false,"properties":{"images":{"type":"object","additionalProperties":{"type":"string"}},"groupOrder":{"type":"array","items":{"type":"string"}},"requirements":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"cpu":{"type":"string"},"memory":{"type":"string"},"memory_limit":{"type":"string"},"amd.com/gpu":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"group":{"type":"string"},"description":{"type":"string"},"subDescription":{"type":"string"},"accelerator":{"type":"string"},"acceleratorKeys":{"type":"array","items":{"type":"string"}},"allowGitClone":{"type":"boolean"},"defaultPath":{"type":["string","null"]},"launchMode":{"type":"string","enum":["jupyterlab","code-server"]},"resourceType":{"type":"string","enum":["notebook","browser-ide"]},"env":{"type":"object","additionalProperties":{"type":"string"}},"acceleratorOverrides":{"type":"object","additionalProperties":{"type":"object","properties":{"image":{"type":"string"},"env":{"type":"object","additionalProperties":{"type":"string"}}}}}}}}}},"teams":{"type":"object","additionalProperties":false,"properties":{"mapping":{"type":"object","additionalProperties":{"type":"array","items":{"type":"string"}}}}},"quota":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"cpuRate":{"type":"integer","minimum":1},"minimumToStart":{"type":"integer","minimum":0},"defaultQuota":{"type":"integer","minimum":0},"refreshRules":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"schedule":{"type":"string"},"action":{"type":"string","enum":["add","set"]},"amount":{"type":"integer"},"maxBalance":{"type":["integer","null"]},"minBalance":{"type":["integer","null"]},"targets":{"type":"object","additionalProperties":false,"properties":{"includeUnlimited":{"type":"boolean"},"balanceBelow":{"type":["integer","null"]},"balanceAbove":{"type":["integer","null"]},"includeUsers":{"type":"array","items":{"type":"string"}},"excludeUsers":{"type":"array","items":{"type":"string"}},"usernamePattern":{"type":"string"}}}}}}}},"gitClone":{"type":"object","additionalProperties":false,"properties":{"initContainerImage":{"type":"string"},"allowedProviders":{"type":"array","items":{"type":"string"}},"maxCloneTimeout":{"type":"integer","minimum":10},"githubAppName":{"type":"string"},"defaultAccessToken":{"type":"string"},"defaultPersistence":{"type":"boolean"},"allowPersistenceChoice":{"type":"boolean"}}},"hub":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"notebook":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"codeServer":{"type":"object","additionalProperties":false,"properties":{"extraTrustedDomains":{"type":"array","items":{"type":"string"}}}},"apiService":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":["","IfNotPresent","Always","Never","null"]}}}}}}},"cull":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"users":{"type":["boolean","null"]},"adminUsers":{"type":["boolean","null"]},"removeNamedServers":{"type":["boolean","null"]},"timeout":{"type":["integer","null"]},"every":{"type":["integer","null"]},"concurrency":{"type":["integer","null"]},"maxAge":{"type":["integer","null"]}}},"debug":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"}}},"rbac":{"type":"object","additionalProperties":false,"required":["create"],"properties":{"enabled":{"type":"boolean"},"create":{"type":"boolean"}}},"global":{"type":"object","additionalProperties":true,"properties":{"safeToShowValues":{"type":"boolean"}}}}} \ No newline at end of file +{"$schema":"http://json-schema.org/draft-07/schema#","type":"object","additionalProperties":false,"required":["imagePullSecrets","hub","proxy","singleuser","ingress","prePuller","custom","cull","debug","rbac","global"],"properties":{"enabled":{"type":["boolean","null"]},"fullnameOverride":{"type":["string","null"]},"nameOverride":{"type":["string","null"]},"imagePullSecret":{"type":"object","required":["create"],"if":{"properties":{"create":{"const":true}}},"then":{"additionalProperties":false,"required":["registry","username","password"],"properties":{"create":{"type":"boolean"},"automaticReferenceInjection":{"type":"boolean"},"registry":{"type":"string"},"username":{"type":"string"},"password":{"type":"string"},"email":{"type":["string","null"]}}}},"imagePullSecrets":{"type":"array"},"hub":{"type":"object","additionalProperties":false,"required":["baseUrl"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"config":{"type":"object","additionalProperties":false,"patternProperties":{"^[A-Z].*$":{"type":"object","additionalProperties":true}},"properties":{"JupyterHub":{"type":"object","additionalProperties":true,"properties":{"subdomain_host":{"type":"string"}}}}},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"baseUrl":{"type":"string"},"command":{"type":"array"},"args":{"type":"array"},"cookieSecret":{"type":["string","null"]},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"db":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["sqlite-pvc","sqlite-memory","mysql","postgres","other"]},"pvc":{"type":"object","additionalProperties":false,"required":["storage"],"properties":{"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"selector":{"type":"object","additionalProperties":true},"storage":{"type":"string"},"accessModes":{"type":"array","items":{"type":["string","null"]}},"storageClassName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"upgrade":{"type":["boolean","null"]},"url":{"type":["string","null"]},"password":{"type":["string","null"]}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"initContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"extraConfig":{"type":"object","additionalProperties":true},"fsGid":{"type":["integer","null"],"minimum":0},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"ports":{"type":"object","additionalProperties":false,"properties":{"appProtocol":{"type":["string","null"]},"nodePort":{"type":["integer","null"],"minimum":0}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPorts":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"existingSecret":{"type":["string","null"]},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"activeServerLimit":{"type":["integer","null"]},"allowNamedServers":{"type":["boolean","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"concurrentSpawnLimit":{"type":["integer","null"]},"consecutiveFailureLimit":{"type":["integer","null"]},"podSecurityContext":{"additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"extraContainers":{"type":"array"},"extraVolumeMounts":{"type":"array"},"extraVolumes":{"type":"array"},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"namedServerLimitPerUser":{"type":["integer","null"]},"redirectToServer":{"type":["boolean","null"]},"resources":{"type":"object","additionalProperties":true},"lifecycle":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"services":{"type":"object","additionalProperties":true,"properties":{"name":{"type":"string"},"admin":{"type":"boolean"},"command":{"type":["string","array"]},"url":{"type":"string"},"api_token":{"type":["string","null"]},"apiToken":{"type":["string","null"]}}},"loadRoles":{"type":"object","additionalProperties":true},"shutdownOnLogout":{"type":["boolean","null"]},"templatePaths":{"type":"array"},"templateVars":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"proxy":{"type":"object","additionalProperties":false,"properties":{"chp":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraCommandLineFlags":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"resources":{"type":"object","additionalProperties":true},"defaultTarget":{"type":["string","null"]},"errorTarget":{"type":["string","null"]},"extraPodSpec":{"type":"object","additionalProperties":true}}},"secretToken":{"type":["string","null"]},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"nodePorts":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"loadBalancerPort":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"disableHttpPort":{"type":"boolean"},"extraPorts":{"type":"array"},"externalIPs":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"loadBalancerSourceRanges":{"type":"array"},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"https":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"type":{"enum":[null,"","letsencrypt","manual","offload","secret"]},"letsencrypt":{"type":"object","additionalProperties":false,"properties":{"contactEmail":{"type":["string","null"]},"acmeServer":{"type":["string","null"]}}},"manual":{"type":"object","additionalProperties":false,"properties":{"key":{"type":["string","null"]},"cert":{"type":["string","null"]}}},"secret":{"type":"object","additionalProperties":false,"properties":{"name":{"type":["string","null"]},"key":{"type":["string","null"]},"crt":{"type":["string","null"]}}},"hosts":{"type":"array"}}},"traefik":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraInitContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraDynamicConfig":{"type":"object","additionalProperties":true},"extraPorts":{"type":"array"},"extraStaticConfig":{"type":"object","additionalProperties":true},"extraVolumes":{"type":"array"},"extraVolumeMounts":{"type":"array"},"hsts":{"type":"object","additionalProperties":false,"required":["includeSubdomains","maxAge","preload"],"properties":{"includeSubdomains":{"type":"boolean"},"maxAge":{"type":"integer"},"preload":{"type":"boolean"}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"secretSync":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}}}},"monitoring":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"namespace":{"type":"string","default":"monitoring"},"releaseLabel":{"type":"string","default":"monitoring"},"hubMetrics":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"allowUnauthenticatedScrape":{"type":"boolean","default":false},"serviceAnnotations":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"serviceMonitor":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"interval":{"type":"string","default":"15s"},"authorization":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":true},"type":{"type":"string","default":"Bearer"},"hubServiceName":{"type":"string","minLength":1,"default":"prometheus-metrics"},"secret":{"type":"object","additionalProperties":false,"properties":{"create":{"type":"boolean","default":true},"name":{"type":"string","default":""},"key":{"type":"string","minLength":1,"default":"token"}}}}}}},"grafana":{"type":"object","additionalProperties":false,"properties":{"dashboard":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"prometheusRule":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"singleuser":{"type":"object","additionalProperties":false,"properties":{"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"podNameTemplate":{"type":["string","null"]},"cpu":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","null"]},"guarantee":{"type":["number","null"]}}},"memory":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","string","null"]},"guarantee":{"type":["number","string","null"]}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"initContainers":{"type":"array"},"profileList":{"type":"array"},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"extraEnv":{"type":["object","array"],"additionalProperties":true},"nodeSelector":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"extraNodeAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAntiAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"cloudMetadata":{"type":"object","additionalProperties":false,"required":["blockWithIptables","ip"],"properties":{"blockWithIptables":{"type":"boolean"},"ip":{"type":"string"}}},"cmd":{"type":["array","string","null"]},"defaultUrl":{"type":["string","null"]},"events":{"type":["boolean","null"]},"extraAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraContainers":{"type":"array"},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPodConfig":{"type":"object","additionalProperties":true},"extraResource":{"type":"object","additionalProperties":false,"properties":{"guarantees":{"type":"object","additionalProperties":true},"limits":{"type":"object","additionalProperties":true}}},"fsGid":{"type":["integer","null"]},"lifecycleHooks":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"networkTools":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}},"serviceAccountName":{"type":["string","null"]},"startTimeout":{"type":["integer","null"]},"storage":{"type":"object","additionalProperties":false,"required":["type","homeMountPath"],"properties":{"capacity":{"type":["string","null"]},"dynamic":{"type":"object","additionalProperties":false,"properties":{"pvcNameTemplate":{"type":["string","null"]},"storageAccessModes":{"type":"array","items":{"type":["string","null"]}},"storageClass":{"type":["string","null"]},"subPath":{"type":["string","null"]},"volumeNameTemplate":{"type":["string","null"]}}},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraVolumeMounts":{"type":["object","array","null"]},"extraVolumes":{"type":["object","array","null"]},"homeMountPath":{"type":"string"},"static":{"type":"object","additionalProperties":false,"properties":{"pvcName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"type":{"enum":["dynamic","static","none"]}}},"allowPrivilegeEscalation":{"type":["boolean","null"]},"uid":{"type":["integer","null"]}}},"scheduling":{"type":"object","additionalProperties":false,"properties":{"userScheduler":{"type":"object","additionalProperties":false,"required":["enabled","plugins","pluginConfig","logLevel"],"properties":{"enabled":{"type":"boolean"},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"logLevel":{"type":"integer"},"plugins":{"type":"object","additionalProperties":true},"pluginConfig":{"type":"array"},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"podPriority":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"globalDefault":{"type":"boolean"},"defaultPriority":{"type":"integer"},"imagePullerPriority":{"type":"integer"},"userPlaceholderPriority":{"type":"integer"}}},"userPlaceholder":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraPodSpec":{"type":"object","additionalProperties":true}}},"corePods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}},"userPods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}}}},"ingress":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"ingressClassName":{"type":["string","null"]},"hosts":{"type":"array"},"pathSuffix":{"type":["string","null"]},"pathType":{"enum":["Prefix","Exact","ImplementationSpecific"]},"tls":{"type":"array"},"extraPaths":{"type":"array"}}},"httpRoute":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"hostnames":{"type":"array"},"gateway":{"type":"object","additionalProperties":false,"required":["name"],"properties":{"name":{"type":"string"},"namespace":{"type":"string"},"sectionName":{"type":"string"}}}}},"prePuller":{"type":"object","additionalProperties":false,"required":["hook","continuous"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"hook":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"pullOnlyOnChanges":{"type":"boolean"},"podSchedulingWaitDuration":{"type":"integer"},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"serviceAccountImagePuller":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"continuous":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"pullProfileListImages":{"type":"boolean"},"extraImages":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]}}}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"pause":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}}}}}},"custom":{"type":"object","additionalProperties":true,"properties":{"authMode":{"type":"string","enum":["auto-login","dummy","github","multi"]},"adminUser":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"}}},"gpuAccess":{"type":"object","additionalProperties":false,"properties":{"renderGid":{"type":["integer","null"],"minimum":1,"maximum":4294967294}}},"notifications":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"topbar":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}},"homepage":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"legacyAnnouncementFallback":{"type":"boolean"},"items":{"type":"array","items":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}}}}}}},"accelerators":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"displayName":{"type":"string"},"description":{"type":"string"},"nodeSelector":{"type":"object","additionalProperties":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"quotaRate":{"type":"integer","minimum":1}}}},"resources":{"type":"object","additionalProperties":false,"properties":{"images":{"type":"object","additionalProperties":{"type":"string"}},"groupOrder":{"type":"array","items":{"type":"string"}},"requirements":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"cpu":{"type":"string"},"memory":{"type":"string"},"memory_limit":{"type":"string"},"amd.com/gpu":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"group":{"type":"string"},"description":{"type":"string"},"subDescription":{"type":"string"},"accelerator":{"type":"string"},"acceleratorKeys":{"type":"array","items":{"type":"string"}},"allowGitClone":{"type":"boolean"},"defaultPath":{"type":["string","null"]},"launchMode":{"type":"string","enum":["jupyterlab","code-server"]},"resourceType":{"type":"string","enum":["notebook","browser-ide"]},"env":{"type":"object","additionalProperties":{"type":"string"}},"acceleratorOverrides":{"type":"object","additionalProperties":{"type":"object","properties":{"image":{"type":"string"},"env":{"type":"object","additionalProperties":{"type":"string"}}}}}}}}}},"teams":{"type":"object","additionalProperties":false,"properties":{"mapping":{"type":"object","additionalProperties":{"type":"array","items":{"type":"string"}}}}},"quota":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"cpuRate":{"type":"integer","minimum":1},"minimumToStart":{"type":"integer","minimum":0},"defaultQuota":{"type":"integer","minimum":0},"refreshRules":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"schedule":{"type":"string"},"action":{"type":"string","enum":["add","set"]},"amount":{"type":"integer"},"maxBalance":{"type":["integer","null"]},"minBalance":{"type":["integer","null"]},"targets":{"type":"object","additionalProperties":false,"properties":{"includeUnlimited":{"type":"boolean"},"balanceBelow":{"type":["integer","null"]},"balanceAbove":{"type":["integer","null"]},"includeUsers":{"type":"array","items":{"type":"string"}},"excludeUsers":{"type":"array","items":{"type":"string"}},"usernamePattern":{"type":"string"}}}}}}}},"gitClone":{"type":"object","additionalProperties":false,"properties":{"initContainerImage":{"type":"string"},"allowedProviders":{"type":"array","items":{"type":"string"}},"maxCloneTimeout":{"type":"integer","minimum":10},"githubAppName":{"type":"string"},"defaultAccessToken":{"type":"string"},"defaultPersistence":{"type":"boolean"},"allowPersistenceChoice":{"type":"boolean"}}},"hub":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"notebook":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"codeServer":{"type":"object","additionalProperties":false,"properties":{"extraTrustedDomains":{"type":"array","items":{"type":"string"}}}},"apiService":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":["","IfNotPresent","Always","Never","null"]}}}}}}},"cull":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"users":{"type":["boolean","null"]},"adminUsers":{"type":["boolean","null"]},"removeNamedServers":{"type":["boolean","null"]},"timeout":{"type":["integer","null"]},"every":{"type":["integer","null"]},"concurrency":{"type":["integer","null"]},"maxAge":{"type":["integer","null"]}}},"debug":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"}}},"rbac":{"type":"object","additionalProperties":false,"required":["create"],"properties":{"enabled":{"type":"boolean"},"create":{"type":"boolean"}}},"global":{"type":"object","additionalProperties":true,"properties":{"safeToShowValues":{"type":"boolean"}}}}} \ No newline at end of file diff --git a/runtime/chart/values.schema.yaml b/runtime/chart/values.schema.yaml index 22ff5fec..3eab8688 100644 --- a/runtime/chart/values.schema.yaml +++ b/runtime/chart/values.schema.yaml @@ -3195,6 +3195,20 @@ properties: Enable auto-admin creation on first install. Credentials will be stored in `jupyterhub-admin-credentials` secret. + gpuAccess: + type: object + additionalProperties: false + description: | + Host group access settings for GPU-enabled user pods. + properties: + renderGid: + type: [integer, "null"] + minimum: 1 + maximum: 4294967294 + description: | + Numeric GID of the host render group. GPU resources receive this + as a supplemental group; CPU resources do not. + notifications: type: object additionalProperties: false diff --git a/runtime/chart/values.yaml b/runtime/chart/values.yaml index a548691f..e888f22f 100644 --- a/runtime/chart/values.yaml +++ b/runtime/chart/values.yaml @@ -50,6 +50,11 @@ custom: # Define these in runtime/values.yaml, not here accelerators: {} + # Host render-group access for GPU user pods. The installer overlay sets this + # to the detected host render GID when GPU access is provisioned. + gpuAccess: + renderGid: null + # Resource images, requirements, and metadata # Define these in runtime/values.yaml, not here resources: From 24325361f7ee84398ea1e3dfe38dbf26be37260f Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:17 +0800 Subject: [PATCH 06/65] fix(runtime): separate storage and GPU groups --- runtime/values-multi-nodes.yaml.example | 12 +++++++----- runtime/values.yaml | 14 ++++++-------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/runtime/values-multi-nodes.yaml.example b/runtime/values-multi-nodes.yaml.example index 3db3cffa..63c48382 100644 --- a/runtime/values-multi-nodes.yaml.example +++ b/runtime/values-multi-nodes.yaml.example @@ -139,6 +139,11 @@ custom: defaultPersistence: true allowPersistenceChoice: false + # Generated deployment overlays resolve this from corroborated host evidence. + # Keep null in the base example; do not choose a fleet GID manually here. + gpuAccess: + renderGid: null + # -------------------------------------------------------------------------- # Accelerator Configuration # -------------------------------------------------------------------------- @@ -576,11 +581,8 @@ monitoring: enabled: false singleuser: - extraPodConfig: - securityContext: - fsGroup: 100 - supplementalGroups: - - 993 + # Must match the storage ownership group used by the shared volume. + fsGid: 100 storage: dynamic: storageClass: nfs-client diff --git a/runtime/values.yaml b/runtime/values.yaml index da79243a..7bce7308 100644 --- a/runtime/values.yaml +++ b/runtime/values.yaml @@ -61,6 +61,10 @@ custom: adminUser: enabled: false + # The installer overlay supplies the host render GID for GPU user pods. + gpuAccess: + renderGid: null + # ============================================================================ # Notifications # ============================================================================ @@ -668,14 +672,8 @@ monitoring: enabled: false singleuser: - # Security context for user pods to access GPU devices - # supplementalGroups grants container access to host's render group (GID 993) - # This allows non-root users to access /dev/kfd and /dev/dri devices - extraPodConfig: - securityContext: - fsGroup: 100 - supplementalGroups: - - 993 # render group for ROCm GPU access + # Preserve storage volume ownership without replacing KubeSpawner's security context. + fsGid: 100 storage: dynamic: From cff8981ff602a08c6d6ab164e7a95da889ef6ee4 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:17 +0800 Subject: [PATCH 07/65] feat(hub): enforce GPU render group policy --- runtime/hub/core/config.py | 31 ++++ runtime/hub/core/spawner/kubernetes.py | 101 ++++++++-- runtime/hub/tests/test_spawner_gpu_access.py | 182 +++++++++++++++++++ 3 files changed, 303 insertions(+), 11 deletions(-) create mode 100644 runtime/hub/tests/test_spawner_gpu_access.py diff --git a/runtime/hub/core/config.py b/runtime/hub/core/config.py index 3925bbf0..28e24ded 100644 --- a/runtime/hub/core/config.py +++ b/runtime/hub/core/config.py @@ -45,6 +45,8 @@ import yaml from pydantic import BaseModel, Field, field_validator +MAX_RENDER_GID = (2**32) - 2 + # ============================================================================= # YAML Configuration Models # ============================================================================= @@ -85,6 +87,25 @@ class QuotaSettings(BaseModel): model_config = {"extra": "allow"} +class GpuAccessSettings(BaseModel): + """Host group access settings for GPU-enabled user pods.""" + + renderGid: int | None = None + + @field_validator("renderGid", mode="before") + @classmethod + def validate_render_gid(cls, value: Any) -> int | None: + """Require a native positive integer GID when GPU access is configured.""" + + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, int) or not 1 <= value <= MAX_RENDER_GID: + raise ValueError(f"custom.gpuAccess.renderGid must be an integer between 1 and {MAX_RENDER_GID}") + return value + + model_config = {"extra": "allow"} + + class AcceleratorOverride(BaseModel): """Per-accelerator overrides for a resource (image and/or env).""" @@ -211,6 +232,7 @@ class ParsedConfig(BaseModel): accelerators: dict[str, AcceleratorConfig] = Field(default_factory=dict) teams: TeamsConfig = Field(default_factory=TeamsConfig) quota: QuotaSettings = Field(default_factory=QuotaSettings) + gpuAccess: GpuAccessSettings = Field(default_factory=GpuAccessSettings) gitClone: GitCloneSettings = Field(default_factory=GitCloneSettings) hub: HubNetworkSettings = Field(default_factory=HubNetworkSettings) notebook: NotebookNetworkSettings = Field(default_factory=NotebookNetworkSettings) @@ -226,6 +248,7 @@ def from_dicts( accelerators: dict | None = None, teams: dict | None = None, quota: dict | None = None, + gpu_access: dict | None = None, git_clone: dict | None = None, hub: dict | None = None, notebook: dict | None = None, @@ -243,6 +266,8 @@ def from_dicts( raw_config["teams"] = teams if quota: raw_config["quota"] = quota + if gpu_access is not None: + raw_config["gpuAccess"] = gpu_access if git_clone: raw_config["gitClone"] = git_clone if hub: @@ -334,6 +359,7 @@ def init(cls, config_path: str | Path) -> HubConfig: accelerators=raw_config.get("accelerators"), teams=raw_config.get("teams"), quota=raw_config.get("quota"), + gpu_access=raw_config.get("gpuAccess"), git_clone=raw_config.get("gitClone"), hub=raw_config.get("hub"), notebook=raw_config.get("notebook"), @@ -411,6 +437,11 @@ def quota(self) -> QuotaSettings: """Get quota configuration.""" return self._config.quota + @property + def gpu_access(self) -> GpuAccessSettings: + """Get GPU pod access configuration.""" + return self._config.gpuAccess + @property def git_clone(self) -> GitCloneSettings: """Get git clone configuration.""" diff --git a/runtime/hub/core/spawner/kubernetes.py b/runtime/hub/core/spawner/kubernetes.py index e9d57fbd..a99b7fff 100644 --- a/runtime/hub/core/spawner/kubernetes.py +++ b/runtime/hub/core/spawner/kubernetes.py @@ -40,6 +40,7 @@ from kubespawner import KubeSpawner from tornado import web +from core.config import MAX_RENDER_GID from core.metrics import ( pod_failure_total, repo_clone_failed_total, @@ -93,6 +94,7 @@ class RemoteLabKubeSpawner(KubeSpawner): auth_mode: str = "auto-login" single_node_mode: bool = False quota_enabled: bool | None = False + render_gid: int | None = None # Resource configuration (set from config) resource_images: dict[str, str] = {} @@ -154,6 +156,7 @@ def configure_from_config(cls, config: HubConfig) -> None: cls.default_quota = config.quota.defaultQuota cls.minimum_quota_to_start = config.quota.minimumToStart cls.quota_enabled = config.quota.enabled + cls.render_gid = config.gpu_access.renderGid # Extract git clone settings (single source of truth: GitCloneSettings) git_config = config.git_clone @@ -169,8 +172,8 @@ def configure_from_config(cls, config: HubConfig) -> None: # Extract code-server link protection settings cls.code_server_extra_trusted_domains = list(config.code_server.extraTrustedDomains) - async def get_user_resources(self) -> list[str]: - """Get available resources for the user based on their JupyterHub group memberships. + def _resolve_user_resources(self) -> list[str]: + """Resolve available resources for the current user from server-side policy. For auto-login/dummy modes, returns all configured resources. For all other users, resolves resources from JupyterHub groups @@ -195,6 +198,43 @@ async def get_user_resources(self) -> list[str]: self.log.debug(f"User '{username}' resolved resources: {available_resources}") return available_resources + async def get_user_resources(self) -> list[str]: + """Get available resources for the user based on their JupyterHub group memberships.""" + return self._resolve_user_resources() + + def _resolve_accelerator_selection(self, resource_type: str, gpu_selection: Any) -> str | None: + """Validate or default the accelerator selection for a resource.""" + requirements = self.resource_requirements[resource_type] + if gpu_selection is None: + selected_accelerator = "" + elif isinstance(gpu_selection, str): + selected_accelerator = gpu_selection.strip() + else: + raise RuntimeError("Accelerator selection must be a string") + + if "amd.com/gpu" not in requirements: + if selected_accelerator: + raise RuntimeError(f"CPU resource '{resource_type}' does not allow GPU selection") + return None + + resource_metadata = self._hub_config.get_resource_metadata(resource_type) if self._hub_config else None + allowed_accelerators = list(getattr(resource_metadata, "acceleratorKeys", []) or []) + if not allowed_accelerators: + raise RuntimeError(f"GPU resource '{resource_type}' has no authorized accelerators configured") + + if not selected_accelerator: + if len(allowed_accelerators) == 1: + selected_accelerator = allowed_accelerators[0] + else: + raise RuntimeError(f"GPU resource '{resource_type}' requires selecting an accelerator") + + if selected_accelerator not in allowed_accelerators: + raise RuntimeError(f"Accelerator '{selected_accelerator}' is not authorized for resource '{resource_type}'") + if selected_accelerator not in self.accelerator_options: + raise RuntimeError(f"Accelerator '{selected_accelerator}' is not configured") + + return selected_accelerator + async def options_form(self, _) -> str: """Generate the HTML form for resource selection. @@ -291,13 +331,17 @@ def options_from_form(self, formdata) -> dict[str, Any]: resource_type = resource_type_list[0] options["resource_type"] = resource_type - # Parse GPU selection if available - gpu_selection = formdata.get(f"gpu_selection_{resource_type}", [None])[0] - options["gpu_selection"] = gpu_selection - # Validate resource type if resource_type not in self.resource_images: raise RuntimeError(f"Unknown Resource: {resource_type}") + if resource_type not in self._resolve_user_resources(): + raise RuntimeError(f"Resource '{resource_type}' is not authorized for this user") + + gpu_selection = self._resolve_accelerator_selection( + resource_type, + formdata.get(f"gpu_selection_{resource_type}", [None])[0], + ) + options["gpu_selection"] = gpu_selection # Configure spawner based on selections self._configure_spawner(resource_type, gpu_selection) @@ -758,6 +802,7 @@ def _reset_per_spawn_state(self) -> None: "init_containers": copy.deepcopy(self.init_containers), "extra_container_config": copy.deepcopy(self.extra_container_config), "environment": copy.deepcopy(self.environment), + "supplemental_gids": copy.deepcopy(self.supplemental_gids), } for key, value in self._resource_baseline_state.items(): @@ -765,6 +810,28 @@ def _reset_per_spawn_state(self) -> None: self._has_git_init_container = False + def _add_gpu_render_gid(self) -> None: + """Add the configured host render group to a GPU resource's pod.""" + if self.render_gid is None: + raise RuntimeError( + "GPU resource requires custom.gpuAccess.renderGid. " + "Set it to the numeric GID of the host render group before spawning GPU resources." + ) + if ( + isinstance(self.render_gid, bool) + or not isinstance(self.render_gid, int) + or not 1 <= self.render_gid <= MAX_RENDER_GID + ): + raise RuntimeError( + "GPU resource requires a valid custom.gpuAccess.renderGid. " + f"Set it to an integer between 1 and {MAX_RENDER_GID} before spawning GPU resources." + ) + + supplemental_gids = list(self.supplemental_gids or []) + if self.render_gid not in supplemental_gids: + supplemental_gids.append(self.render_gid) + self.supplemental_gids = supplemental_gids + def _configure_spawner(self, resource_type: str, gpu_selection: str | None = None) -> None: """Configure the spawner based on the resource type and GPU selection.""" @@ -829,6 +896,7 @@ def _configure_spawner(self, resource_type: str, gpu_selection: str | None = Non if "amd.com/gpu" in requirements: self.extra_resource_guarantees = {"amd.com/gpu": str(requirements["amd.com/gpu"])} self.extra_resource_limits = {"amd.com/gpu": str(requirements["amd.com/gpu"])} + self._add_gpu_render_gid() elif "amd.com/npu" in requirements: self.log.debug("NPU DEVICE PLUGIN are removed, amd.com/npu is no more needed") @@ -895,13 +963,25 @@ def _configure_spawner(self, resource_type: str, gpu_selection: str | None = Non async def start(self): """Start the spawner and schedule automatic shutdown.""" + runtime_minutes = self.user_options.get("runtime_minutes", 20) + resource_type = self.user_options.get("resource_type", "cpu") + if resource_type not in self.resource_images: + raise RuntimeError(f"Unknown Resource: {resource_type}") + if resource_type not in self._resolve_user_resources(): + raise RuntimeError(f"Resource '{resource_type}' is not authorized for this user") + gpu_selection = self._resolve_accelerator_selection( + resource_type, + self.user_options.get("gpu_selection"), + ) + self.user_options["gpu_selection"] = gpu_selection + self._configure_spawner(resource_type, gpu_selection) + # Ensure pod fails immediately (not retried) when an init container fails. # JupyterHub manages pod lifecycle; Kubernetes should not silently restart pods. - self.extra_pod_config = {"restartPolicy": "Never"} + extra_pod_config = copy.deepcopy(self.extra_pod_config or {}) + extra_pod_config["restartPolicy"] = "Never" + self.extra_pod_config = extra_pod_config - runtime_minutes = self.user_options.get("runtime_minutes", 20) - resource_type = self.user_options.get("resource_type", "cpu") - gpu_selection = self.user_options.get("gpu_selection", None) username = self.user.name.lower() # Determine accelerator type for quota calculation @@ -1099,7 +1179,6 @@ async def start(self): if hasattr(self, "_spawn_start_timestamp"): duration = time.time() - self._spawn_start_timestamp spawn_duration_seconds.observe(duration) - accelerator_type = self.user_options.get("gpu_selection") or "cpu" # active session count is derived from quota manager, not inc/dec except Exception: pass diff --git a/runtime/hub/tests/test_spawner_gpu_access.py b/runtime/hub/tests/test_spawner_gpu_access.py new file mode 100644 index 00000000..edacf4c2 --- /dev/null +++ b/runtime/hub/tests/test_spawner_gpu_access.py @@ -0,0 +1,182 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +import copy +import importlib.util +import sys +import types +from pathlib import Path +from unittest.mock import patch + +import pytest +from pydantic import ValidationError + +ROOT = Path(__file__).resolve().parents[1] +CORE = ROOT / "core" + +if "core" not in sys.modules: + core_module = types.ModuleType("core") + core_module.__path__ = [str(CORE)] + sys.modules["core"] = core_module + + +def load_module(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +class DummyMetric: + def labels(self, **_kwargs): + return self + + def inc(self): + pass + + def observe(self, _value): + pass + + +class TestKubeSpawner: + def get_pod_manifest(self): + manifest = {"spec": copy.deepcopy(self.extra_pod_config or {})} + security_context = manifest["spec"].setdefault("securityContext", {}) + if self.fs_gid is not None: + security_context["fsGroup"] = self.fs_gid + if self.supplemental_gids: + security_context["supplementalGroups"] = list(self.supplemental_gids) + return manifest + + +def load_spawner_module(): + metrics_module = types.ModuleType("core.metrics") + for metric_name in ( + "pod_failure_total", + "repo_clone_failed_total", + "session_runtime_minutes", + "spawn_duration_seconds", + "spawn_failed_total", + "spawn_gpu_total", + ): + setattr(metrics_module, metric_name, DummyMetric()) + + jupyterhub_module = types.ModuleType("jupyterhub") + jupyterhub_module.__path__ = [] + user_module = types.ModuleType("jupyterhub.user") + user_module.User = type("User", (), {}) + kubespawner_module = types.ModuleType("kubespawner") + kubespawner_module.KubeSpawner = TestKubeSpawner + tornado_module = types.ModuleType("tornado") + web_module = types.ModuleType("tornado.web") + web_module.HTTPError = type("HTTPError", (Exception,), {}) + + with patch.dict( + sys.modules, + { + "core.metrics": metrics_module, + "jupyterhub": jupyterhub_module, + "jupyterhub.user": user_module, + "kubespawner": kubespawner_module, + "tornado": tornado_module, + "tornado.web": web_module, + }, + ): + return load_module("gpu_access_test_spawner", CORE / "spawner" / "kubernetes.py") + + +config = load_module("core.config", CORE / "config.py") +kubernetes = load_spawner_module() +RemoteLabKubeSpawner = kubernetes.RemoteLabKubeSpawner + + +class DummyLog: + def debug(self, _message): + pass + + +class ResourceMetadata: + acceleratorKeys = ["gpu-a"] + acceleratorOverrides = None + allowGitClone = False + defaultPath = None + env = {} + launchMode = None + + +class HubConfig: + def get_resource_metadata(self, _resource_type): + return ResourceMetadata() + + +def make_spawner(render_gid: int | None, supplemental_gids: list[int] | None = None): + spawner = object.__new__(RemoteLabKubeSpawner) + spawner._hub_config = HubConfig() + spawner.render_gid = render_gid + spawner.resource_images = {"cpu": "cpu-image", "gpu": "gpu-image"} + spawner.resource_requirements = { + "cpu": {"cpu": "1", "memory": "1Gi"}, + "gpu": {"cpu": "1", "memory": "1Gi", "amd.com/gpu": "1"}, + } + spawner.accelerator_options = {"gpu-a": {}} + spawner.node_selector_mapping = {} + spawner.environment_mapping = {} + spawner.cmd = [] + spawner.args = [] + spawner.default_url = "" + spawner.node_affinity_required = [] + spawner.extra_resource_guarantees = {} + spawner.extra_resource_limits = {} + spawner.init_containers = [] + spawner.extra_container_config = {} + spawner.environment = {} + spawner.fs_gid = 100 + spawner.supplemental_gids = list(supplemental_gids or []) + spawner.extra_pod_config = {} + spawner.log = DummyLog() + spawner._resolve_user_resources = lambda: ["cpu", "gpu"] + return spawner + + +def test_gpu_render_gid_is_injected_only_for_gpu_pods(): + spawner = make_spawner(render_gid=993) + + spawner._configure_spawner("gpu", "gpu-a") + gpu_manifest = spawner.get_pod_manifest() + spawner._configure_spawner("cpu") + cpu_manifest = spawner.get_pod_manifest() + + assert gpu_manifest["spec"]["securityContext"] == {"fsGroup": 100, "supplementalGroups": [993]} + assert cpu_manifest["spec"]["securityContext"] == {"fsGroup": 100} + + +def test_gpu_render_gid_preserves_existing_supplemental_groups(): + spawner = make_spawner(render_gid=993, supplemental_gids=[1234]) + + spawner._configure_spawner("gpu", "gpu-a") + + assert spawner.supplemental_gids == [1234, 993] + + +def test_gpu_spawn_requires_a_host_render_gid(): + spawner = make_spawner(render_gid=None) + + with pytest.raises(RuntimeError, match=r"custom\.gpuAccess\.renderGid"): + spawner._configure_spawner("gpu", "gpu-a") + + +def test_gpu_access_config_validates_render_gid(): + assert config.GpuAccessSettings(renderGid=993).renderGid == 993 + assert config.ParsedConfig.from_dicts(gpu_access={"renderGid": 993}).gpuAccess.renderGid == 993 + with pytest.raises(ValidationError, match="renderGid"): + config.GpuAccessSettings(renderGid=True) + + +def test_unauthorized_gpu_selection_is_rejected_before_spawner_configuration(): + spawner = make_spawner(render_gid=993) + spawner._resolve_user_resources = lambda: ["cpu"] + spawner._configure_spawner = lambda *_args: pytest.fail("unauthorized resource configured the spawner") + + with pytest.raises(RuntimeError, match="not authorized"): + spawner.options_from_form({"runtime": ["20"], "resource_type": ["gpu"], "gpu_selection_gpu": ["gpu-a"]}) From 3489075c6b4df2f3b4d6eb41e96290d0cf59df43 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:17 +0800 Subject: [PATCH 08/65] fix(images): remove embedded GPU permission policy --- dockerfiles/Base/Dockerfile.rocm | 17 +----------- tests/scripts/test_gpu_image_permissions.py | 30 +++++++++++++++++++++ 2 files changed, 31 insertions(+), 16 deletions(-) create mode 100644 tests/scripts/test_gpu_image_permissions.py diff --git a/dockerfiles/Base/Dockerfile.rocm b/dockerfiles/Base/Dockerfile.rocm index 43cf8c25..3c2267a6 100644 --- a/dockerfiles/Base/Dockerfile.rocm +++ b/dockerfiles/Base/Dockerfile.rocm @@ -226,14 +226,6 @@ RUN if getent passwd 1000 > /dev/null; then \ RUN useradd -m -s /bin/bash -N -u $NB_UID -g $NB_GID $NB_USER && \ echo "$NB_USER ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers -# Add jovyan to video and render groups for ROCm access -RUN if getent group render; then \ - groupmod -g 992 render; \ - else \ - groupadd -g 992 render; \ - fi -RUN usermod -aG video,render ${NB_USER} - # Create necessary Jupyter directories with correct permissions RUN mkdir -p /home/$NB_USER/.jupyter && \ mkdir -p /home/$NB_USER/.local/share/jupyter/runtime && \ @@ -250,15 +242,8 @@ RUN echo '#!/bin/bash' > /home/$NB_USER/start-jupyter.sh && \ # Verify the file exists (will fail build if not) ls -la /home/$NB_USER/start-jupyter.sh -# Set proper permissions for ROCm devices -RUN mkdir -p /etc/udev/rules.d && \ - echo 'SUBSYSTEM=="kfd", GROUP="video", MODE="0666"' > /etc/udev/rules.d/70-kfd.rules && \ - echo 'SUBSYSTEM=="dri", GROUP="video", MODE="0666"' > /etc/udev/rules.d/70-dri.rules - -# Create entrypoint script to set permissions and start services +# Create entrypoint script to start services RUN echo '#!/bin/bash' > /entrypoint.sh && \ - echo 'chmod 666 /dev/kfd 2>/dev/null || true' >> /entrypoint.sh && \ - echo 'chmod 666 /dev/dri/renderD* 2>/dev/null || true' >> /entrypoint.sh && \ echo 'export USER=jovyan' >> /entrypoint.sh && \ echo 'export SHELL=/bin/bash' >> /entrypoint.sh && \ echo 'exec python3 -m jupyterhub.singleuser --ip=0.0.0.0 --port=8888 "$@"' >> /entrypoint.sh && \ diff --git a/tests/scripts/test_gpu_image_permissions.py b/tests/scripts/test_gpu_image_permissions.py new file mode 100644 index 00000000..ce5f094a --- /dev/null +++ b/tests/scripts/test_gpu_image_permissions.py @@ -0,0 +1,30 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +from __future__ import annotations + +import re +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +DOCKERFILE = ROOT / "dockerfiles" / "Base" / "Dockerfile.rocm" + + +def test_rocm_base_leaves_gpu_device_permissions_to_the_host() -> None: + dockerfile = DOCKERFILE.read_text(encoding="utf-8") + + forbidden_patterns = ( + r"groupmod\s+-g\s+992\s+render", + r"groupadd\s+-g\s+992\s+render", + r"usermod\s+-aG\s+video,render\s+\$\{NB_USER\}", + r"\brender\b", + r"/etc/udev", + r"chmod\s+666\b", + ) + for pattern in forbidden_patterns: + assert re.search(pattern, dockerfile) is None, pattern + + assert "echo 'export USER=jovyan' >> /entrypoint.sh" in dockerfile + assert "echo 'export SHELL=/bin/bash' >> /entrypoint.sh" in dockerfile + assert 'CMD ["/bin/bash", "/entrypoint.sh"]' in dockerfile + assert "USER $NB_UID" in dockerfile + assert "WORKDIR /home/jovyan" in dockerfile From 5176884387ba4cfe4943e460569dedcff1139374 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:17 +0800 Subject: [PATCH 09/65] feat(ansible): define canonical GPU access contract --- deploy/ansible/filter_plugins/auplc_json.py | 41 ++++++++++++++ .../roles/gpu_access/defaults/main.yml | 13 +++++ .../ansible/roles/gpu_access/tasks/main.yml | 14 +++++ .../roles/gpu_access/tasks/validate.yml | 53 +++++++++++++++++++ .../templates/70-auplc-gpu-access.rules.j2 | 3 ++ .../gpu_access/templates/gpu-access.json.j2 | 1 + 6 files changed, 125 insertions(+) create mode 100644 deploy/ansible/filter_plugins/auplc_json.py create mode 100644 deploy/ansible/roles/gpu_access/defaults/main.yml create mode 100644 deploy/ansible/roles/gpu_access/tasks/main.yml create mode 100644 deploy/ansible/roles/gpu_access/tasks/validate.yml create mode 100644 deploy/ansible/roles/gpu_access/templates/70-auplc-gpu-access.rules.j2 create mode 100644 deploy/ansible/roles/gpu_access/templates/gpu-access.json.j2 diff --git a/deploy/ansible/filter_plugins/auplc_json.py b/deploy/ansible/filter_plugins/auplc_json.py new file mode 100644 index 00000000..080de6af --- /dev/null +++ b/deploy/ansible/filter_plugins/auplc_json.py @@ -0,0 +1,41 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +"""Strict JSON filters used by AUP Learning Cloud Ansible roles.""" + +import json +from collections.abc import Callable +from dataclasses import dataclass +from typing import TypeAlias + +from ansible.errors import AnsibleFilterError + +JSONValue: TypeAlias = None | bool | int | float | str | list["JSONValue"] | dict[str, "JSONValue"] + + +@dataclass(frozen=True, slots=True) +class DuplicateJsonKeyError(ValueError): + key: str + + def __str__(self) -> str: + return f"Duplicate JSON object key: {self.key!r}" + + +def _reject_duplicate_keys(pairs: list[tuple[str, JSONValue]]) -> dict[str, JSONValue]: + result: dict[str, JSONValue] = {} + for key, value in pairs: + if key in result: + raise DuplicateJsonKeyError(key) + result[key] = value + return result + + +def auplc_from_json_strict(value: str) -> JSONValue: + try: + return json.loads(value, object_pairs_hook=_reject_duplicate_keys) + except (TypeError, DuplicateJsonKeyError, json.JSONDecodeError): + raise AnsibleFilterError("Invalid JSON value") from None + + +class FilterModule: + def filters(self) -> dict[str, Callable[[str], JSONValue]]: + return {"auplc_from_json_strict": auplc_from_json_strict} diff --git a/deploy/ansible/roles/gpu_access/defaults/main.yml b/deploy/ansible/roles/gpu_access/defaults/main.yml new file mode 100644 index 00000000..74e40ed3 --- /dev/null +++ b/deploy/ansible/roles/gpu_access/defaults/main.yml @@ -0,0 +1,13 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +--- +# Set this explicitly in cluster inventory or PXE extra vars. The role never +# assumes a site-specific GID. +auplc_render_gid: null +auplc_normalize_render_gid: false +auplc_gpu_access_enabled: false +# Set for a PXE rootfs. Leave empty to configure the live host. +auplc_rootfs_path: "" +# Rootfs adapters must explicitly constrain their writable target below this +# canonical directory. Live hosts leave this empty. +auplc_rootfs_allowed_root: "" diff --git a/deploy/ansible/roles/gpu_access/tasks/main.yml b/deploy/ansible/roles/gpu_access/tasks/main.yml new file mode 100644 index 00000000..4826393b --- /dev/null +++ b/deploy/ansible/roles/gpu_access/tasks/main.yml @@ -0,0 +1,14 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +--- +- name: Validate GPU access configuration + ansible.builtin.import_tasks: validate.yml + when: auplc_gpu_access_enabled | bool + +- name: Preflight GPU access target + ansible.builtin.import_tasks: preflight.yml + when: auplc_gpu_access_enabled | bool + +- name: Apply GPU access configuration + ansible.builtin.import_tasks: apply.yml + when: auplc_gpu_access_enabled | bool diff --git a/deploy/ansible/roles/gpu_access/tasks/validate.yml b/deploy/ansible/roles/gpu_access/tasks/validate.yml new file mode 100644 index 00000000..3e32ee7a --- /dev/null +++ b/deploy/ansible/roles/gpu_access/tasks/validate.yml @@ -0,0 +1,53 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +--- +- name: Validate desired render GID + ansible.builtin.assert: + that: + - auplc_render_gid is not none + - auplc_render_gid is integer + - auplc_render_gid >= 1 + - auplc_render_gid <= 4294967294 + fail_msg: auplc_render_gid must be an explicit integer between 1 and 4294967294. + +- name: Validate GPU access rootfs path syntax + ansible.builtin.assert: + that: + - auplc_rootfs_path is string + - auplc_rootfs_path == '' or auplc_rootfs_path is match('^/') + - auplc_rootfs_path != '/' + - "'..' not in auplc_rootfs_path.split('/')" + - auplc_rootfs_path == '' or auplc_rootfs_allowed_root | length > 0 + fail_msg: auplc_rootfs_path must be a non-root absolute path without traversal and with an allowed root. + +- name: Canonicalize GPU access rootfs path + ansible.builtin.command: + argv: + - realpath + - --canonicalize-missing + - "{{ auplc_rootfs_path }}" + register: _auplc_canonical_rootfs + changed_when: false + when: auplc_rootfs_path | length > 0 + +- name: Canonicalize allowed GPU access rootfs parent + ansible.builtin.command: + argv: + - realpath + - --canonicalize-existing + - "{{ auplc_rootfs_allowed_root }}" + register: _auplc_canonical_allowed_root + changed_when: false + when: auplc_rootfs_path | length > 0 + +- name: Constrain canonical GPU access rootfs path + ansible.builtin.assert: + that: + - _auplc_canonical_rootfs.stdout != '/' + - _auplc_canonical_rootfs.stdout.startswith(_auplc_canonical_allowed_root.stdout + '/') + fail_msg: GPU access rootfs escapes auplc_rootfs_allowed_root. + when: auplc_rootfs_path | length > 0 + +- name: Record canonical GPU access target root + ansible.builtin.set_fact: + _auplc_target_root: "{{ _auplc_canonical_rootfs.stdout if auplc_rootfs_path | length > 0 else '' }}" diff --git a/deploy/ansible/roles/gpu_access/templates/70-auplc-gpu-access.rules.j2 b/deploy/ansible/roles/gpu_access/templates/70-auplc-gpu-access.rules.j2 new file mode 100644 index 00000000..c75ec1e2 --- /dev/null +++ b/deploy/ansible/roles/gpu_access/templates/70-auplc-gpu-access.rules.j2 @@ -0,0 +1,3 @@ +# Managed by auplc-installer: AMD GPU device access. +KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0660" +SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0660" diff --git a/deploy/ansible/roles/gpu_access/templates/gpu-access.json.j2 b/deploy/ansible/roles/gpu_access/templates/gpu-access.json.j2 new file mode 100644 index 00000000..89b9110a --- /dev/null +++ b/deploy/ansible/roles/gpu_access/templates/gpu-access.json.j2 @@ -0,0 +1 @@ +{"renderGid":{{ auplc_render_gid | int }},"version":1} From 59110290bd5671e6575bd058819d95c0be8cc1f1 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:17 +0800 Subject: [PATCH 10/65] feat(ansible): apply verified GPU device policy --- .../roles/gpu_access/handlers/main.yml | 17 ++ .../ansible/roles/gpu_access/tasks/apply.yml | 190 ++++++++++++++++ .../roles/gpu_access/tasks/preflight.yml | 206 ++++++++++++++++++ 3 files changed, 413 insertions(+) create mode 100644 deploy/ansible/roles/gpu_access/handlers/main.yml create mode 100644 deploy/ansible/roles/gpu_access/tasks/apply.yml create mode 100644 deploy/ansible/roles/gpu_access/tasks/preflight.yml diff --git a/deploy/ansible/roles/gpu_access/handlers/main.yml b/deploy/ansible/roles/gpu_access/handlers/main.yml new file mode 100644 index 00000000..6afe9532 --- /dev/null +++ b/deploy/ansible/roles/gpu_access/handlers/main.yml @@ -0,0 +1,17 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +--- +- name: Reload udev rules + ansible.builtin.command: + argv: + - udevadm + - control + - --reload-rules + when: auplc_rootfs_path | length == 0 + +- name: Trigger udev rules + ansible.builtin.command: + argv: + - udevadm + - trigger + when: auplc_rootfs_path | length == 0 diff --git a/deploy/ansible/roles/gpu_access/tasks/apply.yml b/deploy/ansible/roles/gpu_access/tasks/apply.yml new file mode 100644 index 00000000..1def00de --- /dev/null +++ b/deploy/ansible/roles/gpu_access/tasks/apply.yml @@ -0,0 +1,190 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +--- +- name: Inspect recognized project-owned legacy GPU rules for apply + ansible.builtin.stat: + path: "{{ item.path }}" + follow: false + loop: "{{ _auplc_legacy_gpu_rules }}" + register: _auplc_apply_legacy_gpu_rule_stats + +- name: Reject legacy GPU rule symlinks and non-regular files before apply + ansible.builtin.assert: + that: + - not item.stat.exists or (item.stat.isreg and not item.stat.islnk) + fail_msg: "Unexpected legacy GPU rule filesystem type: {{ item.item.path }}" + loop: "{{ _auplc_apply_legacy_gpu_rule_stats.results }}" + +- name: Read recognized project-owned legacy GPU rules for apply + ansible.builtin.slurp: + src: "{{ item.item.path }}" + loop: "{{ _auplc_apply_legacy_gpu_rule_stats.results }}" + when: item.stat.exists + register: _auplc_apply_legacy_gpu_rule_contents + +- name: Reject unexpected legacy GPU rule content before apply + ansible.builtin.assert: + that: + - (item.content | b64decode) in item.item.item.contents + fail_msg: "Unexpected legacy GPU rule content: {{ item.item.item.path }}" + loop: "{{ _auplc_apply_legacy_gpu_rule_contents.results }}" + when: not item.skipped | default(false) + +- name: Remove recognized project-owned legacy GPU rules + ansible.builtin.file: + path: "{{ item.item.item.path }}" + state: absent + loop: "{{ _auplc_apply_legacy_gpu_rule_contents.results }}" + when: not item.skipped | default(false) + +- name: Normalize live render GID + ansible.builtin.command: + argv: [groupmod, -g, "{{ auplc_render_gid | string }}", render] + when: + - _auplc_target_root | length == 0 + - (_auplc_current_render_gid | int) != (auplc_render_gid | int) + - auplc_normalize_render_gid | bool + +- name: Normalize rootfs render GID + ansible.builtin.command: + argv: [chroot, "{{ _auplc_target_root }}", groupmod, -g, "{{ auplc_render_gid | string }}", render] + when: + - _auplc_target_root | length > 0 + - (_auplc_current_render_gid | int) != (auplc_render_gid | int) + - auplc_normalize_render_gid | bool + +- name: Verify target render GID + ansible.builtin.command: + argv: >- + {{ ['getent', 'group', 'render'] if _auplc_target_root | length == 0 + else ['chroot', _auplc_target_root, 'getent', 'group', 'render'] }} + register: _auplc_verified_render_group + changed_when: false + +- name: Require verified render GID + ansible.builtin.assert: + that: + - _auplc_verified_render_group.stdout_lines | length == 1 + - _auplc_verified_render_group.stdout.split(':')[0] == 'render' + - (_auplc_verified_render_group.stdout.split(':')[2] | int) == (auplc_render_gid | int) + fail_msg: Target render group did not resolve to auplc_render_gid. + +- name: Create target udev rules directory + ansible.builtin.file: + path: "{{ _auplc_target_root }}/etc/udev/rules.d" + state: directory + owner: root + group: root + mode: "0755" + +- name: Install canonical AMD GPU udev rules + ansible.builtin.template: + src: 70-auplc-gpu-access.rules.j2 + dest: "{{ _auplc_target_root }}/etc/udev/rules.d/70-auplc-gpu-access.rules" + owner: root + group: root + mode: "0644" + +- name: Reload live udev rules on every apply + ansible.builtin.command: + argv: [udevadm, control, --reload-rules] + changed_when: false + when: _auplc_target_root | length == 0 + +- name: Trigger live udev rules on every apply + ansible.builtin.command: + argv: [udevadm, trigger] + changed_when: false + when: _auplc_target_root | length == 0 + +- name: Settle live udev events before inode verification + ansible.builtin.command: + argv: [udevadm, settle] + changed_when: false + when: _auplc_target_root | length == 0 + +- name: Inspect /dev/kfd after live reconciliation + ansible.builtin.stat: + path: /dev/kfd + follow: false + register: _auplc_kfd + when: _auplc_target_root | length == 0 + +- name: Verify /dev/kfd ownership and mode + ansible.builtin.assert: + that: + - _auplc_kfd.stat.exists + - _auplc_kfd.stat.ischr + - _auplc_kfd.stat.uid == 0 + - _auplc_kfd.stat.gid == (auplc_render_gid | int) + - _auplc_kfd.stat.mode == '0660' + fail_msg: /dev/kfd is not root:render with mode 0660 after reconciliation. + when: _auplc_target_root | length == 0 + +- name: Find live DRM render nodes + ansible.builtin.find: + paths: /dev/dri + patterns: renderD* + file_type: any + recurse: false + register: _auplc_render_nodes + when: _auplc_target_root | length == 0 + +- name: Resolve live DRM render node driver symlinks + ansible.builtin.command: + argv: [readlink, -f, "/sys/class/drm/{{ item.path | basename }}/device/driver"] + loop: "{{ _auplc_render_nodes.files }}" + register: _auplc_render_node_drivers + changed_when: false + failed_when: false + when: _auplc_target_root | length == 0 + +- name: Select AMD live DRM render nodes + ansible.builtin.set_fact: + _auplc_amd_render_nodes: >- + {{ (_auplc_amd_render_nodes | default([])) + + ([item.item.path] if item.rc == 0 and (item.stdout | basename) == 'amdgpu' else []) }} + loop: "{{ _auplc_render_node_drivers.results }}" + when: _auplc_target_root | length == 0 + +- name: Require AMD live DRM render nodes + ansible.builtin.assert: + that: _auplc_amd_render_nodes | length > 0 + fail_msg: No AMD renderD node was available for GPU access verification. + when: _auplc_target_root | length == 0 + +- name: Inspect AMD live DRM render nodes + ansible.builtin.stat: + path: "{{ item }}" + follow: false + loop: "{{ _auplc_amd_render_nodes }}" + register: _auplc_amd_render_node_stats + when: _auplc_target_root | length == 0 + +- name: Verify AMD render node ownership and mode + ansible.builtin.assert: + that: + - item.stat.exists + - item.stat.ischr + - item.stat.uid == 0 + - item.stat.gid == (auplc_render_gid | int) + - item.stat.mode == '0660' + fail_msg: "AMD render node {{ item.item }} is not root:render with mode 0660." + loop: "{{ _auplc_amd_render_node_stats.results }}" + when: _auplc_target_root | length == 0 + +- name: Create target GPU access state directory + ansible.builtin.file: + path: "{{ _auplc_target_root }}/var/lib/auplc" + state: directory + owner: root + group: root + mode: "0755" + +- name: Persist target GPU access state + ansible.builtin.template: + src: gpu-access.json.j2 + dest: "{{ _auplc_target_root }}/var/lib/auplc/gpu-access.json" + owner: root + group: root + mode: "0644" diff --git a/deploy/ansible/roles/gpu_access/tasks/preflight.yml b/deploy/ansible/roles/gpu_access/tasks/preflight.yml new file mode 100644 index 00000000..fcb4808b --- /dev/null +++ b/deploy/ansible/roles/gpu_access/tasks/preflight.yml @@ -0,0 +1,206 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +--- +- name: Validate GPU access configuration before target preflight + ansible.builtin.import_tasks: validate.yml + +- name: Inspect GPU access rootfs target + ansible.builtin.stat: + path: "{{ _auplc_target_root }}" + follow: false + register: _auplc_rootfs + when: _auplc_target_root | length > 0 + +- name: Require regular GPU access rootfs directory + ansible.builtin.assert: + that: + - _auplc_rootfs.stat.isdir + - not _auplc_rootfs.stat.islnk + fail_msg: GPU access rootfs must be an existing non-symlink directory. + when: _auplc_target_root | length > 0 + +- name: Inspect canonical GPU access destination parents + ansible.builtin.stat: + path: "{{ _auplc_target_root }}{{ item }}" + follow: false + loop: + - /etc + - /etc/udev + - /etc/udev/rules.d + - /var + - /var/lib + - /var/lib/auplc + register: _auplc_destination_parent_stats + +- name: Reject unsafe canonical GPU access destination parents + ansible.builtin.assert: + that: + - not item.stat.exists or (item.stat.isdir and not item.stat.islnk) + fail_msg: "Unsafe canonical GPU access destination parent: {{ item.item }}" + loop: "{{ _auplc_destination_parent_stats.results }}" + +- name: Inspect canonical GPU access destinations + ansible.builtin.stat: + path: "{{ _auplc_target_root }}{{ item }}" + follow: false + loop: + - /etc/udev/rules.d/70-auplc-gpu-access.rules + - /var/lib/auplc/gpu-access.json + register: _auplc_destination_stats + +- name: Reject unsafe canonical GPU access destinations + ansible.builtin.assert: + that: + - not item.stat.exists or (item.stat.isreg and not item.stat.islnk) + fail_msg: "Unsafe canonical GPU access destination: {{ item.item }}" + loop: "{{ _auplc_destination_stats.results }}" + +- name: Read existing canonical GPU access destinations + ansible.builtin.slurp: + src: "{{ _auplc_target_root }}{{ item.item }}" + loop: "{{ _auplc_destination_stats.results }}" + when: item.stat.exists + register: _auplc_existing_destinations + +- name: Define canonical GPU access rule content + ansible.builtin.set_fact: + _auplc_canonical_rule: | + # Managed by auplc-installer: AMD GPU device access. + KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0660" + SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0660" + +- name: Reject unmanaged canonical GPU access rule + ansible.builtin.assert: + that: (item.content | b64decode) == _auplc_canonical_rule + fail_msg: "Unmanaged canonical GPU access rule: {{ item.item.item }}" + loop: "{{ _auplc_existing_destinations.results }}" + when: + - not item.skipped | default(false) + - item.item.item.endswith('70-auplc-gpu-access.rules') + +- name: Parse existing canonical GPU access state + ansible.builtin.set_fact: + _auplc_existing_state: "{{ item.content | b64decode | auplc_from_json_strict }}" + loop: "{{ _auplc_existing_destinations.results }}" + when: + - not item.skipped | default(false) + - item.item.item.endswith('gpu-access.json') + +- name: Define recognized project-owned legacy GPU rules + ansible.builtin.set_fact: + _auplc_legacy_gpu_rules: + - path: "{{ _auplc_target_root }}/etc/udev/rules.d/70-kfd.rules" + contents: + - "KERNEL==\"kfd\", MODE=\"0666\"\nSUBSYSTEM==\"drm\", KERNEL==\"renderD*\", MODE=\"0666\"\n" + - path: "{{ _auplc_target_root }}/etc/udev/rules.d/70-amdgpu.rules" + contents: + - | + # ROCm device permissions + # Grant render group access to AMD GPU devices + # Reference: https://rocm.docs.amd.com/projects/install-on-linux/en/latest/install/prerequisites.html#using-udev-rules + KERNEL=="kfd", GROUP="render", MODE="0660" + SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660" + - "KERNEL==\"kfd\", MODE=\"0666\"\nKERNEL==\"renderD[0-9]*\", MODE=\"0666\"\n" + - path: "{{ _auplc_target_root }}/etc/udev/rules.d/70-rocm-devices.rules" + contents: + - | + # ROCm device permissions + # Ensure /dev/kfd and /dev/dri/renderD* are accessible by render group + SUBSYSTEM=="kfd", GROUP="render", MODE="0660" + SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660" + +- name: Inspect recognized project-owned legacy GPU rules + ansible.builtin.stat: + path: "{{ item.path }}" + follow: false + loop: "{{ _auplc_legacy_gpu_rules }}" + register: _auplc_legacy_gpu_rule_stats + +- name: Reject legacy GPU rule symlinks and non-regular files + ansible.builtin.assert: + that: + - not item.stat.exists or (item.stat.isreg and not item.stat.islnk) + fail_msg: "Unexpected legacy GPU rule filesystem type: {{ item.item.path }}" + loop: "{{ _auplc_legacy_gpu_rule_stats.results }}" + +- name: Read recognized project-owned legacy GPU rules + ansible.builtin.slurp: + src: "{{ item.item.path }}" + loop: "{{ _auplc_legacy_gpu_rule_stats.results }}" + when: item.stat.exists + register: _auplc_legacy_gpu_rule_contents + +- name: Reject unexpected legacy GPU rule content + ansible.builtin.assert: + that: + - (item.content | b64decode) in item.item.item.contents + fail_msg: "Unexpected legacy GPU rule content: {{ item.item.item.path }}" + loop: "{{ _auplc_legacy_gpu_rule_contents.results }}" + when: not item.skipped | default(false) + +- name: Read target render group + ansible.builtin.command: + argv: >- + {{ ['getent', 'group', 'render'] if _auplc_target_root | length == 0 + else ['chroot', _auplc_target_root, 'getent', 'group', 'render'] }} + register: _auplc_render_group + changed_when: false + failed_when: false + +- name: Require target render group + ansible.builtin.assert: + that: + - _auplc_render_group.rc == 0 + - _auplc_render_group.stdout_lines | length == 1 + - _auplc_render_group.stdout.split(':') | length == 4 + - _auplc_render_group.stdout.split(':')[0] == 'render' + - _auplc_render_group.stdout.split(':')[2] is match('^[1-9][0-9]*$') + - _auplc_render_group.stdout.split(':')[2] | int <= 4294967294 + fail_msg: Target has no valid render group; this role never creates groups. + +- name: Record target render GID + ansible.builtin.set_fact: + _auplc_current_render_gid: "{{ _auplc_render_group.stdout.split(':')[2] | int }}" + +- name: Reject invalid canonical GPU access state except Interrupted normalization retry + ansible.builtin.assert: + that: + - _auplc_existing_state is mapping + - _auplc_existing_state.keys() | list | sort == ['renderGid', 'version'] + - _auplc_existing_state.version is integer + - _auplc_existing_state.version == 1 + - _auplc_existing_state.renderGid is integer + - _auplc_existing_state.renderGid >= 1 + - _auplc_existing_state.renderGid <= 4294967294 + - >- + _auplc_existing_state.renderGid == auplc_render_gid or + ((auplc_normalize_render_gid | bool) and + (_auplc_existing_state.renderGid == _auplc_current_render_gid or + _auplc_current_render_gid == auplc_render_gid)) + fail_msg: Invalid canonical GPU access state. + when: _auplc_existing_state is defined + +- name: List target groups for desired GID collision + ansible.builtin.command: + argv: >- + {{ ['getent', 'group'] if _auplc_target_root | length == 0 + else ['chroot', _auplc_target_root, 'getent', 'group'] }} + register: _auplc_all_groups + changed_when: false + failed_when: false + +- name: Reject desired GID collision + ansible.builtin.assert: + that: + - _auplc_all_groups.rc == 0 + - >- + _auplc_all_groups.stdout_lines + | select('match', '^[^:]*:[^:]*:' ~ (auplc_render_gid | string) ~ ':') + | reject('match', '^render:') | list | length == 0 + fail_msg: auplc_render_gid is already assigned to another target group. + +- name: Reject render GID mismatch without normalization + ansible.builtin.assert: + that: + - _auplc_current_render_gid == auplc_render_gid or (auplc_normalize_render_gid | bool) + fail_msg: Target render GID differs from auplc_render_gid and normalization is disabled. From f22777a3551f9db75f7b64c301a1878dac768771 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:17 +0800 Subject: [PATCH 11/65] feat(ansible): integrate unified GPU access role --- deploy/ansible/playbooks/pb-rocm.yml | 25 +++++++++++++- deploy/ansible/playbooks/pb-udev.yml | 23 +++++++++++-- deploy/ansible/roles/rocm/tasks/main.yml | 22 ------------ deploy/ansible/roles/udev/main.yml | 44 ------------------------ 4 files changed, 45 insertions(+), 69 deletions(-) delete mode 100644 deploy/ansible/roles/udev/main.yml diff --git a/deploy/ansible/playbooks/pb-rocm.yml b/deploy/ansible/playbooks/pb-rocm.yml index 504a7b71..6788bf3d 100644 --- a/deploy/ansible/playbooks/pb-rocm.yml +++ b/deploy/ansible/playbooks/pb-rocm.yml @@ -19,6 +19,29 @@ - name: Install AMD GPU driver for ROCm 7.13.0 hosts: all + any_errors_fatal: true become: yes + pre_tasks: + - name: Assert explicit GPU access enablement + ansible.builtin.assert: + that: + - auplc_gpu_access_enabled is defined + - auplc_gpu_access_enabled is boolean + fail_msg: >- + Set auplc_gpu_access_enabled to true or false for every host in the + inventory before running pb-rocm.yml. + + - name: Preflight enabled GPU access hosts before ROCm mutation + ansible.builtin.include_role: + name: gpu_access + tasks_from: preflight + when: auplc_gpu_access_enabled | bool roles: - - rocm + - role: rocm + when: auplc_gpu_access_enabled | bool + tasks: + - name: Apply GPU access after ROCm installation + ansible.builtin.include_role: + name: gpu_access + tasks_from: apply + when: auplc_gpu_access_enabled | bool diff --git a/deploy/ansible/playbooks/pb-udev.yml b/deploy/ansible/playbooks/pb-udev.yml index 508b7b42..7e77eb80 100644 --- a/deploy/ansible/playbooks/pb-udev.yml +++ b/deploy/ansible/playbooks/pb-udev.yml @@ -19,7 +19,26 @@ - name: Configure ROCm udev rules hosts: all + any_errors_fatal: true become: yes - roles: - - udev-rocm + pre_tasks: + - name: Assert explicit GPU access enablement + ansible.builtin.assert: + that: + - auplc_gpu_access_enabled is defined + - auplc_gpu_access_enabled is boolean + fail_msg: >- + Set auplc_gpu_access_enabled to true or false for every host in the + inventory before running pb-udev.yml. + - name: Preflight enabled GPU access hosts + ansible.builtin.include_role: + name: gpu_access + tasks_from: preflight + when: auplc_gpu_access_enabled | bool + tasks: + - name: Apply GPU access on enabled hosts + ansible.builtin.include_role: + name: gpu_access + tasks_from: apply + when: auplc_gpu_access_enabled | bool diff --git a/deploy/ansible/roles/rocm/tasks/main.yml b/deploy/ansible/roles/rocm/tasks/main.yml index 525a3a18..d0353f1b 100644 --- a/deploy/ansible/roles/rocm/tasks/main.yml +++ b/deploy/ansible/roles/rocm/tasks/main.yml @@ -53,25 +53,3 @@ apt: name: amdgpu-dkms state: present - -- name: Ensure render group exists with consistent GID - group: - name: render - gid: 993 - state: present - -- name: Set udev rules for ROCm devices with correct permissions - copy: - content: | - # ROCm device permissions - # Grant render group access to AMD GPU devices - # Reference: https://rocm.docs.amd.com/projects/install-on-linux/en/latest/install/prerequisites.html#using-udev-rules - KERNEL=="kfd", GROUP="render", MODE="0660" - SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660" - dest: /etc/udev/rules.d/70-amdgpu.rules - mode: '0644' - register: udev_rules_changed - -- name: Reload udev rules if changed - shell: udevadm control --reload-rules && udevadm trigger - when: udev_rules_changed.changed diff --git a/deploy/ansible/roles/udev/main.yml b/deploy/ansible/roles/udev/main.yml deleted file mode 100644 index 235fe25e..00000000 --- a/deploy/ansible/roles/udev/main.yml +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - ---- -- name: Create /etc/udev/rules.d/70-kfd.rules - copy: - dest: /etc/udev/rules.d/70-kfd.rules - content: | - KERNEL=="kfd", MODE="0666" - SUBSYSTEM=="drm", KERNEL=="renderD*", MODE="0666" - owner: root - group: root - mode: '0644' - -- name: Reload udev rules - command: udevadm control --reload-rules - -- name: Trigger udev rules - command: udevadm trigger - -- name: Reboot the system (optional) - reboot: - msg: "Rebooting to apply udev rule changes" - pre_reboot_delay: 5 - reboot_timeout: 300 - post_reboot_delay: 30 - when: udev_rocm_reboot_enabled - From 132917e34c09c92fe85a0d6c0e98d43871ee2ea7 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:17 +0800 Subject: [PATCH 12/65] feat(deploy): add validated config generation primitives --- .../scripts/config_common.py | 57 +++++ .../scripts/config_generation.py | 209 ++++++++++++++++++ .../scripts/config_rendering.py | 163 ++++++++++++++ 3 files changed, 429 insertions(+) create mode 100644 skills/deploy-aup-learning-cloud/scripts/config_common.py create mode 100644 skills/deploy-aup-learning-cloud/scripts/config_generation.py create mode 100644 skills/deploy-aup-learning-cloud/scripts/config_rendering.py diff --git a/skills/deploy-aup-learning-cloud/scripts/config_common.py b/skills/deploy-aup-learning-cloud/scripts/config_common.py new file mode 100644 index 00000000..edc3f67e --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/config_common.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +"""Shared schema constants and scalar rendering helpers.""" + +from __future__ import annotations + +import json +import sys + +HEADER_HASH = ( + "# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved.\n" + "# Generated by auplc-skills gen_configs.py -- review before use.\n" +) + +DEFAULT_ACCEL_LABELS = { + "phx": "AMD_Radeon_780M_Graphics", + "strix": "AMD_Radeon_890M_Graphics", + "strix-halo": "AMD_Radeon_8060S_Graphics", + "9070xt": "AMD_Radeon_RX_9070_XT", + "r9700": "AMD_Radeon_AI_PRO_R9700", + "9600gre": "AMD_Radeon_RX_9600_GRE", +} + + +class DuplicateJsonKeyError(ValueError): + pass + + +def _unique_json_object(pairs): + document = {} + for key, value in pairs: + if key in document: + raise DuplicateJsonKeyError(f"duplicate JSON key '{key}'") + document[key] = value + return document + + +def strict_json_loads(raw: str): + return json.loads(raw, object_pairs_hook=_unique_json_object) + + +def die(msg: str, code: int = 1) -> None: + print(f"gen_configs: {msg}", file=sys.stderr) + raise SystemExit(code) + + +def require(spec: dict, path: str): + cur = spec + for part in path.split("."): + if not isinstance(cur, dict) or part not in cur or cur[part] in (None, "", []): + die(f"spec is missing required field '{path}'") + cur = cur[part] + return cur + + +def yaml_quote(value: str) -> str: + return '"' + str(value).replace("\\", "\\\\").replace('"', '\\"') + '"' diff --git a/skills/deploy-aup-learning-cloud/scripts/config_generation.py b/skills/deploy-aup-learning-cloud/scripts/config_generation.py new file mode 100644 index 00000000..5fdcf84a --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/config_generation.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +"""Validate cluster specifications and render deploy configuration artifacts.""" + +from __future__ import annotations + +import ipaddress +import re + +from config_common import DEFAULT_ACCEL_LABELS, HEADER_HASH, die, require, yaml_quote +from config_rendering import ResolvedGpuPolicy, render_inventory, render_pxe_vars, render_values + +__all__ = [ + "DEFAULT_ACCEL_LABELS", + "HEADER_HASH", + "ResolvedGpuPolicy", + "SCHEMA", + "die", + "render_inventory", + "render_pxe_vars", + "render_values", + "require", + "validate_accelerators", + "validate_config_shapes", + "validate_yaml_scalar", + "validate_spec", + "yaml_quote", +] + +SCHEMA = { + "topology": "pxe-diskless | ssh-preinstalled", + "k3s_version": "v1.32.3+k3s1", + "server": {"name": "aipc1", "ip": "192.168.0.140"}, + "agents": [{"name": "aipc2", "ip": "192.168.0.141"}], + "network": { + "interface": "enp1s0", + "subnet": "192.168.0.0/24", + "gateway": "192.168.0.1", + "dns_servers": "8.8.8.8,8.8.4.4", + }, + "pxe": { + "authorized_keys": ["ssh-ed25519 AAAA... you@host"], + "rootfs_password": "", + "web_port": 8080, + "diskless_agents_have_amd_gpus": True, + }, + "accelerators": {"strix-halo": {"product_name": "AMD_Radeon_8060S_Graphics"}}, + "storage": {"class": "nfs-client"}, + "proxy": {"node_port": 30890}, + "auth_mode": "auto-login", + "images": {"cpu": "ghcr.io/amdresearch/auplc-default:latest", "gpu": "ghcr.io/amdresearch/auplc-base:latest"}, +} + +HOSTNAME_PATTERN = re.compile( + r"(?=.{1,253}\Z)(?:[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)(?:\.(?:[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?))*\Z" +) +K3S_VERSION_PATTERN = re.compile(r"v[0-9]+\.[0-9]+\.[0-9]+\+k3s[0-9]+\Z") +IMAGE_KEY_PATTERN = re.compile(r"[A-Za-z][A-Za-z0-9_-]*\Z") + + +def validate_accelerators(spec: dict) -> None: + if "accelerators" not in spec: + return + accelerators = spec["accelerators"] + if not isinstance(accelerators, dict): + die("spec.accelerators must be a mapping") + unsupported = sorted(set(accelerators) - set(DEFAULT_ACCEL_LABELS)) + if len(unsupported) == 1: + die(f"unsupported accelerator key '{unsupported[0]}'") + if unsupported: + die(f"unsupported accelerator keys: {', '.join(unsupported)}") + for key, config in accelerators.items(): + if not isinstance(config, dict): + die(f"accelerators.{key} must be a mapping") + + +def validate_config_shapes(spec: dict) -> None: + if not isinstance(spec, dict): + die("spec must be a mapping") + validate_accelerators(spec) + for key in ("server", "network", "pxe", "storage", "proxy", "images"): + if key in spec and not isinstance(spec[key], dict): + die(f"spec.{key} must be a mapping") + if "agents" in spec and not isinstance(spec["agents"], list): + die("spec.agents must be a list") + + +def _safe_text(value, path: str, *, allow_empty: bool = False) -> str: + if not isinstance(value, str) or (not allow_empty and not value): + die(f"{path} must be a non-empty string" if not allow_empty else f"{path} must be a string") + if any(ord(character) < 32 or ord(character) == 127 for character in value): + die(f"{path} must not contain control characters") + return value + + +def validate_yaml_scalar(value, path: str, *, allow_empty: bool = False) -> str: + return _safe_text(value, path, allow_empty=allow_empty) + + +def _safe_hostname(value, path: str) -> str: + hostname = _safe_text(value, path) + if not HOSTNAME_PATTERN.fullmatch(hostname): + die(f"{path} must be a safe hostname") + return hostname + + +def _safe_ip(value, path: str) -> str: + address = _safe_text(value, path) + try: + ipaddress.ip_address(address) + except ValueError: + die(f"{path} must be a valid IP address") + return address + + +def _safe_port(value, path: str, minimum: int, maximum: int) -> int: + if type(value) is not int or not minimum <= value <= maximum: + die(f"{path} must be an integer between {minimum} and {maximum}") + return value + + +def _validate_server(server: dict, path: str) -> str: + if set(server) != {"name", "ip"}: + die(f"{path} must contain exactly name and ip") + name = _safe_hostname(server["name"], f"{path}.name") + _safe_ip(server["ip"], f"{path}.ip") + return name + + +def _validate_agents(spec: dict, server_name: str) -> None: + agents = spec.get("agents", []) + if not isinstance(agents, list): + die("spec.agents must be a list") + names = {server_name} + for index, agent in enumerate(agents): + path = f"spec.agents[{index}]" + if not isinstance(agent, dict): + die(f"{path} must be a mapping") + name = _validate_server(agent, path) + if name in names: + die("server and agent names must be unique") + names.add(name) + + +def _validate_rendered_options(spec: dict) -> None: + if "auth_mode" in spec: + _safe_text(spec["auth_mode"], "spec.auth_mode") + if "storage" in spec and "class" in spec["storage"]: + _safe_text(spec["storage"]["class"], "spec.storage.class") + if "proxy" in spec and "node_port" in spec["proxy"]: + _safe_port(spec["proxy"]["node_port"], "spec.proxy.node_port", 30000, 32767) + if "images" in spec: + for key, value in spec["images"].items(): + if not isinstance(key, str) or not IMAGE_KEY_PATTERN.fullmatch(key): + die("spec.images key must be a safe identifier") + _safe_text(value, f"spec.images.{key}") + if "accelerators" in spec: + for key, config in spec["accelerators"].items(): + if "product_name" in config: + _safe_text(config["product_name"], f"spec.accelerators.{key}.product_name") + + +def _validate_pxe(spec: dict) -> None: + pxe = require(spec, "pxe") + keys = pxe.get("authorized_keys") + if not isinstance(keys, list) or not keys: + die("pxe.authorized_keys must contain at least one SSH public key") + for index, key in enumerate(keys): + _safe_text(key, f"spec.pxe.authorized_keys[{index}]") + if "rootfs_password" in pxe: + _safe_text(pxe["rootfs_password"], "spec.pxe.rootfs_password", allow_empty=True) + if "web_port" in pxe: + _safe_port(pxe["web_port"], "spec.pxe.web_port", 1, 65535) + if type(pxe.get("diskless_agents_have_amd_gpus")) is not bool: + die("spec.pxe.diskless_agents_have_amd_gpus must be a boolean") + network = require(spec, "network") + _safe_text(require(spec, "network.interface"), "spec.network.interface") + subnet = _safe_text(require(spec, "network.subnet"), "spec.network.subnet") + try: + ipaddress.ip_network(subnet, strict=True) + except ValueError: + die("spec.network.subnet must be a valid network CIDR") + if "gateway" in network: + _safe_ip(network["gateway"], "spec.network.gateway") + if "dns_servers" in network: + for index, address in enumerate(_safe_text(network["dns_servers"], "spec.network.dns_servers").split(",")): + _safe_ip(address.strip(), f"spec.network.dns_servers[{index}]") + + +def validate_spec(spec: dict) -> str: + if not isinstance(spec, dict): + die("spec must be a mapping") + topo = spec.get("topology") + if topo not in ("pxe-diskless", "ssh-preinstalled"): + die("spec.topology must be 'pxe-diskless' or 'ssh-preinstalled'") + validate_config_shapes(spec) + k3s_version = _safe_text(require(spec, "k3s_version"), "spec.k3s_version") + if not K3S_VERSION_PATTERN.fullmatch(k3s_version): + die("spec.k3s_version must be a safe k3s version") + server_name = _validate_server(require(spec, "server"), "spec.server") + _validate_agents(spec, server_name) + _validate_rendered_options(spec) + if "render_gid" in spec: + die("spec.render_gid is no longer accepted; GPU policy is discovered automatically") + if "gpu_access" in spec: + die("spec.gpu_access is no longer accepted; GPU policy is discovered automatically") + if topo == "pxe-diskless": + _validate_pxe(spec) + return topo diff --git a/skills/deploy-aup-learning-cloud/scripts/config_rendering.py b/skills/deploy-aup-learning-cloud/scripts/config_rendering.py new file mode 100644 index 00000000..2ac59a86 --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/config_rendering.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +"""Render deployment artifacts from resolved configuration values.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from config_common import DEFAULT_ACCEL_LABELS, HEADER_HASH, die, require, yaml_quote +from gpu_access_resolution import FleetResolution, HostStatus + + +@dataclass(frozen=True, slots=True) +class ResolvedGpuPolicy: + host_gpu_enabled: dict[str, bool] + render_gid: int | None + pxe_gpu_enabled: bool + + +def render_inventory(spec: dict, token: str, resolution: FleetResolution) -> str: + topo = spec["topology"] + server = spec["server"] + k3s_version = spec["k3s_version"] + render_gid = resolution.render_gid + host_gpu_enabled = {host.target.name: host.status is HostStatus.GPU for host in resolution.hosts} + lines = [ + HEADER_HASH, + "k3s_cluster:", + " children:", + " server:", + " hosts:", + f" {server['name']}:", + f" ansible_host: {yaml_quote(server['ip'])}", + f" auplc_gpu_access_enabled: {'true' if host_gpu_enabled[server['name']] else 'false'}", + " agent:", + ] + if topo == "ssh-preinstalled" and spec.get("agents"): + lines.append(" hosts:") + for agent in spec["agents"]: + lines.append(f" {agent['name']}:") + lines.append(f" ansible_host: {yaml_quote(agent['ip'])}") + lines.append( + f" auplc_gpu_access_enabled: {'true' if host_gpu_enabled[agent['name']] else 'false'}" + ) + else: + lines.append(" hosts: {}") + lines += [ + " vars:", + " ansible_port: 22", + " ansible_user: root", + f" k3s_version: {yaml_quote(k3s_version)}", + f" auplc_render_gid: {'null' if render_gid is None else render_gid}", + f" token: {yaml_quote(token)}", + " api_endpoint: \"{{ hostvars[groups['server'][0]]['ansible_host'] | default(groups['server'][0]) }}\"", + ] + if topo == "pxe-diskless": + lines += [ + "", + "pxe_controller:", + " hosts:", + f" {server['name']}:", + f" ansible_host: {yaml_quote(server['ip'])}", + " vars:", + " ansible_port: 22", + " ansible_user: root", + ] + return "\n".join(lines) + "\n" + + +def render_pxe_vars(spec: dict, policy: ResolvedGpuPolicy, finalizer_context: str | None = None) -> str: + net = require(spec, "network") + pxe = spec.get("pxe", {}) + keys = pxe.get("authorized_keys", []) + if not keys: + die("pxe.authorized_keys must contain at least one SSH public key") + server_ip = spec["server"]["ip"] + k3s_version = spec["k3s_version"] + render_gid = policy.render_gid + lines = [ + HEADER_HASH, + "# Pass this file to pb-pxe-controller.yml with", + "# ansible-playbook ... -e @", + "# pxe_k3s_version is pinned to k3s_version so agents are never newer", + "# than the server.", + "pxe_rootfs_force_rebuild: true # first build only; set false afterwards", + f"pxe_network_interface: {yaml_quote(net['interface'])}", + f"pxe_subnet: {yaml_quote(net['subnet'])}", + f"pxe_gateway: {yaml_quote(net.get('gateway', ''))}", + f"pxe_dns_servers: {yaml_quote(net.get('dns_servers', '8.8.8.8,8.8.4.4'))}", + f"pxe_controller_ip: {yaml_quote(server_ip)}", + "pxe_k3s_server_ips:", + f" - {yaml_quote(server_ip)}", + f"pxe_k3s_version: {yaml_quote(k3s_version)}", + f"auplc_render_gid: {'null' if render_gid is None else render_gid}", + f"pxe_gpu_access_enabled: {'true' if policy.pxe_gpu_enabled else 'false'}", + f"pxe_web_port: {int(pxe.get('web_port', 8080))}", + f"pxe_rootfs_password: {yaml_quote(pxe.get('rootfs_password', ''))}", + "pxe_rootfs_authorized_keys:", + ] + for key in keys: + lines.append(f" - {yaml_quote(key)}") + if finalizer_context is not None: + lines.append(f"pxe_finalizer_context: {yaml_quote(finalizer_context)}") + return "\n".join(lines) + "\n" + + +def render_values(spec: dict, resolution: FleetResolution) -> str: + accel = spec.get("accelerators") or {} + storage_class = (spec.get("storage") or {}).get("class", "nfs-client") + node_port = (spec.get("proxy") or {}).get("node_port", 30890) + auth_mode = spec.get("auth_mode", "auto-login") + images = spec.get("images") or {} + render_gid = resolution.render_gid + lines = [ + "# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved.", + "# Helm overlay generated by auplc-skills gen_configs.py.", + "# Layer this on top of runtime/values.yaml:", + "# helm upgrade --install jupyterhub ./runtime/chart -n jupyterhub \\", + "# --create-namespace -f runtime/values.yaml -f ", + "custom:", + f" authMode: {yaml_quote(auth_mode)}", + " gpuAccess:", + f" renderGid: {'null' if render_gid is None else render_gid}", + ] + if accel: + lines.append(" accelerators:") + for key, config in accel.items(): + product = (config or {}).get("product_name") or DEFAULT_ACCEL_LABELS.get(key) + if not product: + die( + f"accelerator '{key}' has no product_name and no known default; " + "add accelerators..product_name from `kubectl describe node`" + ) + lines += [ + f" {key}:", + " nodeSelector:", + f" amd.com/gpu.product-name: {yaml_quote(product)}", + ] + if accel or images: + lines.append(" resources:") + if accel: + lines += [" metadata:", " gpu:", " acceleratorKeys:"] + lines.extend(f" - {yaml_quote(key)}" for key in accel) + if images: + lines.append(" images:") + for key, value in images.items(): + lines.append(f" {key}: {yaml_quote(value)}") + lines += [ + "hub:", + " db:", + " pvc:", + f" storageClassName: {yaml_quote(storage_class)}", + "singleuser:", + " storage:", + " dynamic:", + f" storageClass: {yaml_quote(storage_class)}", + "proxy:", + " service:", + " type: NodePort", + " nodePorts:", + f" http: {int(node_port)}", + ] + return "\n".join(lines) + "\n" From e1e838249ffb71278b16feec853c8737f7b3d1a1 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:18 +0800 Subject: [PATCH 13/65] feat(deploy): publish generated artifacts atomically --- .../scripts/artifact_store.py | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 skills/deploy-aup-learning-cloud/scripts/artifact_store.py diff --git a/skills/deploy-aup-learning-cloud/scripts/artifact_store.py b/skills/deploy-aup-learning-cloud/scripts/artifact_store.py new file mode 100644 index 00000000..5b9de061 --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/artifact_store.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +"""Stage and atomically publish generated deployment artifacts.""" + +from __future__ import annotations + +import os +import shutil +import sys +import tempfile +from contextlib import suppress +from pathlib import Path + + +def die(msg: str, code: int = 1) -> None: + print(f"gen_configs: {msg}", file=sys.stderr) + raise SystemExit(code) + + +def preflight_destinations(paths: list[Path], force: bool) -> None: + if force: + return + for path in paths: + if os.path.lexists(path): + die(f"refusing to overwrite existing {path} (use --force)", 1) + + +def stage_file(path: Path, content: str, mode: int) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + fd, staged_path = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + try: + os.fchmod(fd, mode) + with os.fdopen(fd, "w", encoding="utf-8") as staged_file: + staged_file.write(content) + staged_file.flush() + os.fsync(staged_file.fileno()) + except OSError: + with suppress(OSError): + os.close(fd) + Path(staged_path).unlink(missing_ok=True) + raise + return Path(staged_path) + + +def remove_destination(path: Path) -> None: + if path.is_dir() and not path.is_symlink(): + shutil.rmtree(path) + else: + path.unlink(missing_ok=True) + + +def backup_destination(path: Path) -> tuple[Path, Path]: + backup_dir = Path(tempfile.mkdtemp(prefix=f".{path.name}.backup.", dir=path.parent)) + backup_path = backup_dir / path.name + os.replace(path, backup_path) + return backup_dir, backup_path + + +def _fsync_parent(path: Path) -> None: + directory_fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) + + +def publish_artifacts( + artifacts: list[tuple[Path, str, int, bool]], force: bool, remove_paths: tuple[Path, ...] = () +) -> None: + staged: list[tuple[Path, Path, bool]] = [] + published: list[Path] = [] + backups: list[tuple[Path, Path, Path]] = [] + replacement_paths = tuple(path for path, _, _, _ in artifacts) + try: + for path, content, mode, secret in artifacts: + staged.append((path, stage_file(path, content, mode), secret)) + if force: + for path in (*replacement_paths, *(path for path in remove_paths if path not in replacement_paths)): + if os.path.lexists(path): + backup_dir, backup_path = backup_destination(path) + backups.append((path, backup_dir, backup_path)) + _fsync_parent(path) + for path, staged_path, secret in staged: + if force: + os.replace(staged_path, path) + else: + os.link(staged_path, path) + published.append(path) + if not force: + os.unlink(staged_path) + _fsync_parent(path) + print(f"wrote {path}" + (" (chmod 600 -- contains the k3s token)" if secret else "")) + except OSError as exc: + for path in reversed(published): + remove_destination(path) + _fsync_parent(path) + for path, backup_dir, backup_path in reversed(backups): + remove_destination(path) + os.replace(backup_path, path) + _fsync_parent(path) + backup_dir.rmdir() + die(f"could not publish generated artifacts: {exc}") + else: + for _, backup_dir, _ in backups: + shutil.rmtree(backup_dir) + finally: + for _, staged_path, _ in staged: + staged_path.unlink(missing_ok=True) From 1333e152cfa3a95a5195730316c5056c878d6cba Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:18 +0800 Subject: [PATCH 14/65] feat(deploy): resolve unanimous fleet GPU policy --- .../scripts/gpu_access_resolution.py | 314 ++++++++++++++ .../scripts/gpu_resolution_manifest.py | 68 +++ tests/skills/test_gpu_access_resolution.py | 408 ++++++++++++++++++ 3 files changed, 790 insertions(+) create mode 100644 skills/deploy-aup-learning-cloud/scripts/gpu_access_resolution.py create mode 100644 skills/deploy-aup-learning-cloud/scripts/gpu_resolution_manifest.py create mode 100644 tests/skills/test_gpu_access_resolution.py diff --git a/skills/deploy-aup-learning-cloud/scripts/gpu_access_resolution.py b/skills/deploy-aup-learning-cloud/scripts/gpu_access_resolution.py new file mode 100644 index 00000000..b618b2f8 --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/gpu_access_resolution.py @@ -0,0 +1,314 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +"""Parse read-only host evidence and resolve a safe fleet GPU-access policy.""" + +import json +import re +from dataclasses import dataclass +from enum import Enum +from typing import Final + +from config_common import DuplicateJsonKeyError, strict_json_loads +from gpu_resolution_manifest import ResolutionManifest, build_resolution_manifest + +EVIDENCE_VERSION: Final = 2 +MAX_RENDER_GID: Final = 4_294_967_294 +CANONICAL_RULE: Final = ( + "# Managed by auplc-installer: AMD GPU device access.\n" + 'KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0660"\n' + 'SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0660"\n' +) +BDF_PATTERN: Final = re.compile(r"[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[0-9a-fA-F]{2}\.[0-7]") + + +class HostStatus(str, Enum): + """Classify one inventory host from mutually corroborated discovery probes.""" + + GPU = "gpu" + CPU = "cpu" + UNKNOWN = "unknown" + + +class FleetStatus(str, Enum): + """Describe whether fleet evidence yields a publication-safe GPU policy.""" + + GPU_RESOLVED = "gpu_resolved" + CPU_ONLY = "cpu_only" + BLOCKED = "blocked" + + +@dataclass(frozen=True, slots=True) +class EvidenceParseError(ValueError): + """Raised when discovery JSON does not match the fixed evidence schema.""" + + field: str + + def __str__(self) -> str: + return f"Malformed GPU-access discovery evidence at {self.field}" + + +@dataclass(frozen=True, slots=True) +class InventoryTarget: + name: str + + +@dataclass(frozen=True, slots=True) +class CommandEvidence: + rc: int + stdout: str + + +@dataclass(frozen=True, slots=True) +class FileEvidence: + stat_success: bool + content_success: bool + exists: bool + regular: bool + symlink: bool + content: str + + +@dataclass(frozen=True, slots=True) +class LegacyRuleEvidence: + kfd: FileEvidence + amdgpu: FileEvidence + rocm_devices: FileEvidence + + +@dataclass(frozen=True, slots=True) +class HostEvidence: + target: InventoryTarget + reachable: bool + lspci: CommandEvidence + sysfs: CommandEvidence + render_group: CommandEvidence + groups: CommandEvidence + state: FileEvidence + rule: FileEvidence + legacy_rules: LegacyRuleEvidence + + +@dataclass(frozen=True, slots=True) +class HostResolution: + target: InventoryTarget + status: HostStatus + render_gid: int | None + reason: str | None + + +@dataclass(frozen=True, slots=True) +class FleetResolution: + status: FleetStatus + hosts: tuple[HostResolution, ...] + render_gid: int | None + reason: str | None + + +def parse_fleet_evidence(raw: str) -> tuple[HostEvidence, ...]: + """Parse the exact JSON emitted by the GPU-access discovery playbook.""" + try: + document = strict_json_loads(raw) + except DuplicateJsonKeyError as error: + raise EvidenceParseError(field=str(error)) from error + except (TypeError, json.JSONDecodeError) as error: + raise EvidenceParseError(field="document") from error + _require_mapping(document, "document") + if set(document) != {"version", "hosts"}: + raise EvidenceParseError(field="document") + if type(document["version"]) is not int or document["version"] != EVIDENCE_VERSION: + raise EvidenceParseError(field="version") + if type(document["hosts"]) is not list: + raise EvidenceParseError(field="hosts") + return tuple(_parse_host(item, f"hosts[{index}]") for index, item in enumerate(document["hosts"])) + + +def resolve_fleet(expected_targets: tuple[InventoryTarget, ...], evidence: tuple[HostEvidence, ...]) -> FleetResolution: + """Resolve a fleet only when complete evidence proves one safe policy.""" + resolutions = tuple(_resolve_host(host) for host in evidence) + expected_names = tuple(target.name for target in expected_targets) + actual_names = tuple(host.target.name for host in evidence) + if len(set(expected_names)) != len(expected_names) or len(set(actual_names)) != len(actual_names): + return _blocked(resolutions, "duplicate host") + if set(expected_names) != set(actual_names): + return _blocked(resolutions, "incomplete host coverage") + if any(host.status is HostStatus.UNKNOWN for host in resolutions): + return _blocked(resolutions, "unknown host evidence") + gpu_hosts = tuple(host for host in resolutions if host.status is HostStatus.GPU) + if not gpu_hosts: + return FleetResolution(FleetStatus.CPU_ONLY, resolutions, None, None) + gids = {host.render_gid for host in gpu_hosts} + if len(gids) != 1: + return _blocked(resolutions, "GPU render GIDs disagree") + return FleetResolution(FleetStatus.GPU_RESOLVED, resolutions, next(iter(gids)), None) + + +def resolution_manifest(resolution: FleetResolution) -> ResolutionManifest: + """Build the public serialized manifest for a resolved fleet.""" + return build_resolution_manifest( + version=1, + status=resolution.status.value, + render_gid=resolution.render_gid, + hosts={host.target.name: host.status is HostStatus.GPU for host in resolution.hosts}, + ) + + +def _parse_host(raw, field: str) -> HostEvidence: + _require_mapping(raw, field) + required = {"host", "reachable", "lspci", "sysfs", "render_group", "groups", "state", "rule", "legacy_rules"} + if set(raw) != required or type(raw["host"]) is not str or not raw["host"]: + raise EvidenceParseError(field=field) + if type(raw["reachable"]) is not bool: + raise EvidenceParseError(field=f"{field}.reachable") + return HostEvidence( + target=InventoryTarget(name=raw["host"]), + reachable=raw["reachable"], + lspci=_parse_command(raw["lspci"], f"{field}.lspci"), + sysfs=_parse_command(raw["sysfs"], f"{field}.sysfs"), + render_group=_parse_command(raw["render_group"], f"{field}.render_group"), + groups=_parse_command(raw["groups"], f"{field}.groups"), + state=_parse_file(raw["state"], f"{field}.state"), + rule=_parse_file(raw["rule"], f"{field}.rule"), + legacy_rules=_parse_legacy_rules(raw["legacy_rules"], f"{field}.legacy_rules"), + ) + + +def _parse_command(raw, field: str) -> CommandEvidence: + _require_mapping(raw, field) + if set(raw) != {"rc", "stdout"} or type(raw["rc"]) is not int or type(raw["stdout"]) is not str: + raise EvidenceParseError(field=field) + return CommandEvidence(rc=raw["rc"], stdout=raw["stdout"]) + + +def _parse_file(raw, field: str) -> FileEvidence: + _require_mapping(raw, field) + required = {"stat_success", "content_success", "exists", "regular", "symlink", "content"} + if set(raw) != required or any( + type(raw[key]) is not bool for key in ("stat_success", "content_success", "exists", "regular", "symlink") + ): + raise EvidenceParseError(field=field) + if type(raw["content"]) is not str: + raise EvidenceParseError(field=f"{field}.content") + return FileEvidence(**raw) + + +def _parse_legacy_rules(raw, field: str) -> LegacyRuleEvidence: + _require_mapping(raw, field) + if set(raw) != {"kfd", "amdgpu", "rocm_devices"}: + raise EvidenceParseError(field=field) + return LegacyRuleEvidence( + kfd=_parse_file(raw["kfd"], f"{field}.kfd"), + amdgpu=_parse_file(raw["amdgpu"], f"{field}.amdgpu"), + rocm_devices=_parse_file(raw["rocm_devices"], f"{field}.rocm_devices"), + ) + + +def _require_mapping(value, field: str) -> None: + if type(value) is not dict: + raise EvidenceParseError(field=field) + + +def _resolve_host(evidence: HostEvidence) -> HostResolution: + if not evidence.reachable or evidence.lspci.rc != 0 or evidence.sysfs.rc != 0: + return _unknown(evidence, "GPU discovery probe failed") + if not _file_probes_succeeded(evidence): + return _unknown(evidence, "GPU access file probe failed") + lspci_bdfs = _bdfs(evidence.lspci.stdout) + sysfs_bdfs = _bdfs(evidence.sysfs.stdout) + if lspci_bdfs is None or sysfs_bdfs is None or lspci_bdfs != sysfs_bdfs: + return _unknown(evidence, "AMD GPU BDF probes disagree") + if not lspci_bdfs: + if evidence.state.exists or evidence.rule.exists or _legacy_rule_exists(evidence.legacy_rules): + return _unknown(evidence, "CPU host retains GPU access contract") + return HostResolution(evidence.target, HostStatus.CPU, None, None) + render_gid = _render_gid(evidence) + if render_gid is None or not _safe_gpu_files(evidence, render_gid): + return _unknown(evidence, "GPU access contract is unsafe") + return HostResolution(evidence.target, HostStatus.GPU, render_gid, None) + + +def _bdfs(stdout: str) -> frozenset[str] | None: + bdfs = frozenset(line.split(maxsplit=1)[0] for line in stdout.splitlines()) + if all(BDF_PATTERN.fullmatch(bdf) for bdf in bdfs): + return bdfs + return None + + +def _render_gid(evidence: HostEvidence) -> int | None: + if evidence.render_group.rc != 0 or evidence.groups.rc != 0: + return None + record = _group_record(evidence.render_group.stdout) + if record is None or record[0] != "render": + return None + gid = record[1] + groups = tuple(_group_record(line) for line in evidence.groups.stdout.splitlines()) + if not groups or any(group is None for group in groups): + return None + if sum(group[0] == "render" and group[1] == gid for group in groups) != 1: + return None + if any(group[0] != "render" and group[1] == gid for group in groups): + return None + return gid + + +def _group_record(record: str) -> tuple[str, int] | None: + fields = record.split(":") + if len(fields) != 4 or not fields[0] or not fields[2].isascii() or not fields[2].isdecimal(): + return None + gid = int(fields[2]) + if 1 <= gid <= MAX_RENDER_GID: + return fields[0], gid + return None + + +def _safe_gpu_files(evidence: HostEvidence, render_gid: int) -> bool: + if not _safe_file(evidence.state) or not _safe_file(evidence.rule): + return False + if evidence.state.exists and _state_gid(evidence.state.content) != render_gid: + return False + return not evidence.rule.exists or evidence.rule.content == CANONICAL_RULE + + +def _safe_file(evidence: FileEvidence) -> bool: + if not evidence.stat_success or not evidence.content_success: + return False + if evidence.exists: + return evidence.regular and not evidence.symlink + return not evidence.regular and not evidence.symlink and not evidence.content + + +def _file_probes_succeeded(evidence: HostEvidence) -> bool: + return all( + file_evidence.stat_success and file_evidence.content_success + for file_evidence in ( + evidence.state, + evidence.rule, + evidence.legacy_rules.kfd, + evidence.legacy_rules.amdgpu, + evidence.legacy_rules.rocm_devices, + ) + ) + + +def _legacy_rule_exists(evidence: LegacyRuleEvidence) -> bool: + return any(file_evidence.exists for file_evidence in (evidence.kfd, evidence.amdgpu, evidence.rocm_devices)) + + +def _state_gid(raw: str) -> int | None: + try: + state = strict_json_loads(raw) + except (DuplicateJsonKeyError, TypeError, json.JSONDecodeError): + return None + if type(state) is not dict or set(state) != {"renderGid", "version"}: + return None + gid = state["renderGid"] + if type(gid) is not int or type(state["version"]) is not int or state["version"] != 1: + return None + return gid if 1 <= gid <= MAX_RENDER_GID else None + + +def _unknown(evidence: HostEvidence, reason: str) -> HostResolution: + return HostResolution(evidence.target, HostStatus.UNKNOWN, None, reason) + + +def _blocked(hosts: tuple[HostResolution, ...], reason: str) -> FleetResolution: + return FleetResolution(FleetStatus.BLOCKED, hosts, None, reason) diff --git a/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_manifest.py b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_manifest.py new file mode 100644 index 00000000..60a59fd7 --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_manifest.py @@ -0,0 +1,68 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +"""Typed GPU-resolution manifest schemas and primitive builders.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import TypedDict + + +class ResolutionManifest(TypedDict): + """Serialized fleet GPU-resolution evidence.""" + + version: int + status: str + render_gid: int | None + hosts: dict[str, bool] + + +class PxeRootfsManifest(TypedDict): + """Serialized GPU policy applied to the PXE root filesystem.""" + + gpu_access_enabled: bool + render_gid: int | None + + +class PxeResolutionManifest(ResolutionManifest): + """Serialized fleet resolution with its PXE rootfs policy.""" + + pxe_rootfs: PxeRootfsManifest + + +def build_resolution_manifest( + *, + version: int, + status: str, + render_gid: int | None, + hosts: Mapping[str, bool], +) -> ResolutionManifest: + """Build a deterministic ordinary dictionary for fleet resolution.""" + return { + "version": version, + "status": status, + "render_gid": render_gid, + "hosts": {name: hosts[name] for name in sorted(hosts)}, + } + + +def build_pxe_resolution_manifest( + *, + version: int, + status: str, + render_gid: int | None, + hosts: Mapping[str, bool], + gpu_access_enabled: bool, + pxe_render_gid: int | None, +) -> PxeResolutionManifest: + """Build a PXE manifest without mutating a base fleet manifest.""" + return { + "version": version, + "status": status, + "render_gid": render_gid, + "hosts": {name: hosts[name] for name in sorted(hosts)}, + "pxe_rootfs": { + "gpu_access_enabled": gpu_access_enabled, + "render_gid": pxe_render_gid, + }, + } diff --git a/tests/skills/test_gpu_access_resolution.py b/tests/skills/test_gpu_access_resolution.py new file mode 100644 index 00000000..ea853f00 --- /dev/null +++ b/tests/skills/test_gpu_access_resolution.py @@ -0,0 +1,408 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +"""Behavior tests for fleet GPU-access discovery resolution.""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +RESOLUTION = ROOT / "skills" / "deploy-aup-learning-cloud" / "scripts" / "gpu_access_resolution.py" +MANIFEST = ROOT / "skills" / "deploy-aup-learning-cloud" / "scripts" / "gpu_resolution_manifest.py" +GPU_BDF = "0000:03:00.0" + + +def load_resolution_module(): + sys.path.insert(0, str(RESOLUTION.parent)) + spec = importlib.util.spec_from_file_location("gpu_access_resolution", RESOLUTION) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + try: + spec.loader.exec_module(module) + return module + finally: + sys.path.pop(0) + + +def load_manifest_module(): + spec = importlib.util.spec_from_file_location("gpu_resolution_manifest", MANIFEST) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def host_evidence( + host: str, + *, + lspci_bdfs: list[str] | None = None, + sysfs_bdfs: list[str] | None = None, + lspci_rc: int = 0, + sysfs_rc: int = 0, + reachable: bool = True, + render_gid: int = 993, + group_listing: str | None = None, + state: str | None = None, + rule: str | None = None, + state_stat_success: bool = True, + state_content_success: bool = True, + legacy_rules: dict[str, str | None] | None = None, +) -> dict: + lspci = "\n".join(lspci_bdfs or []) + sysfs = "\n".join(sysfs_bdfs if sysfs_bdfs is not None else lspci_bdfs or []) + return { + "host": host, + "reachable": reachable, + "lspci": {"rc": lspci_rc, "stdout": lspci}, + "sysfs": {"rc": sysfs_rc, "stdout": sysfs}, + "render_group": {"rc": 0, "stdout": f"render:x:{render_gid}:\n"}, + "groups": {"rc": 0, "stdout": group_listing or f"render:x:{render_gid}:\n"}, + "state": { + "stat_success": state_stat_success, + "content_success": state_content_success, + "exists": state is not None, + "regular": state is not None, + "symlink": False, + "content": state or "", + }, + "rule": { + "stat_success": True, + "content_success": True, + "exists": rule is not None, + "regular": rule is not None, + "symlink": False, + "content": rule or "", + }, + "legacy_rules": { + key: { + "stat_success": True, + "content_success": True, + "exists": content is not None, + "regular": content is not None, + "symlink": False, + "content": content or "", + } + for key, content in (legacy_rules or {}).items() + } + | { + key: { + "stat_success": True, + "content_success": True, + "exists": False, + "regular": False, + "symlink": False, + "content": "", + } + for key in ("kfd", "amdgpu", "rocm_devices") + if key not in (legacy_rules or {}) + }, + } + + +def evidence_document(*hosts: dict) -> str: + return json.dumps({"version": 2, "hosts": list(hosts)}) + + +def expected_targets(module, *names: str): + return tuple(module.InventoryTarget(name=name) for name in names) + + +def test_parse_fleet_evidence_accepts_the_exact_machine_evidence_schema() -> None: + module = load_resolution_module() + raw = evidence_document(host_evidence("gpu-1", lspci_bdfs=[GPU_BDF])) + + evidence = module.parse_fleet_evidence(raw) + + assert evidence[0].target == module.InventoryTarget(name="gpu-1") + assert evidence[0].lspci.stdout == GPU_BDF + assert evidence[0].state.exists is False + + +@pytest.mark.parametrize( + "replacement", + [ + {"version": True, "hosts": []}, + {"version": 1, "hosts": [], "unexpected": "field"}, + {"version": 1, "hosts": [{"host": "gpu-1"}]}, + {"version": 1, "hosts": [host_evidence("gpu-1", lspci_rc=True)]}, + ], +) +def test_parse_fleet_evidence_rejects_nonexact_or_boolean_integer_values(replacement: dict) -> None: + module = load_resolution_module() + + with pytest.raises(module.EvidenceParseError): + module.parse_fleet_evidence(json.dumps(replacement)) + + +def test_parse_fleet_evidence_rejects_duplicate_json_keys() -> None: + module = load_resolution_module() + + with pytest.raises(module.EvidenceParseError, match="duplicate JSON key 'version'"): + module.parse_fleet_evidence('{"version":2,"version":2,"hosts":[]}') + + +def test_resolve_fleet_blocks_duplicate_persisted_state_render_gid() -> None: + module = load_resolution_module() + parsed = module.parse_fleet_evidence( + evidence_document( + host_evidence("gpu-1", lspci_bdfs=[GPU_BDF], state='{"renderGid":993,"renderGid":993,"version":1}') + ) + ) + + resolution = module.resolve_fleet(expected_targets(module, "gpu-1"), parsed) + + assert resolution.status is module.FleetStatus.BLOCKED + + +@pytest.mark.parametrize( + "state", + [ + '{"renderGid":993,"version":1,"version":1}', + '{"renderGid":993,"version":1,"r\\u0065nderGid":993}', + '{"renderGid":993,"version":1,"v\\u0065rsion":1}', + ], + ids=["duplicate-version", "escaped-render-gid", "escaped-version"], +) +def test_resolve_fleet_blocks_semantic_duplicate_persisted_state_keys(state: str) -> None: + module = load_resolution_module() + parsed = module.parse_fleet_evidence(evidence_document(host_evidence("gpu-1", lspci_bdfs=[GPU_BDF], state=state))) + + resolution = module.resolve_fleet(expected_targets(module, "gpu-1"), parsed) + + assert resolution.status is module.FleetStatus.BLOCKED + + +def test_resolve_fleet_classifies_matching_amd_bdfs_as_gpu() -> None: + module = load_resolution_module() + evidence = module.parse_fleet_evidence(evidence_document(host_evidence("gpu-1", lspci_bdfs=[GPU_BDF]))) + + resolution = module.resolve_fleet(expected_targets(module, "gpu-1"), evidence) + + assert resolution.status is module.FleetStatus.GPU_RESOLVED + assert resolution.render_gid == 993 + assert resolution.hosts[0].status is module.HostStatus.GPU + + +def test_resolve_fleet_classifies_two_empty_successful_gpu_probes_as_cpu_only() -> None: + module = load_resolution_module() + evidence = module.parse_fleet_evidence(evidence_document(host_evidence("cpu-1"))) + + resolution = module.resolve_fleet(expected_targets(module, "cpu-1"), evidence) + + assert resolution.status is module.FleetStatus.CPU_ONLY + assert resolution.render_gid is None + assert resolution.hosts[0].status is module.HostStatus.CPU + + +@pytest.mark.parametrize( + "evidence", + [ + host_evidence("host-1", lspci_bdfs=[GPU_BDF], sysfs_bdfs=["0000:04:00.0"]), + host_evidence("host-1", lspci_bdfs=[GPU_BDF], lspci_rc=1), + host_evidence("host-1", lspci_bdfs=[GPU_BDF], reachable=False), + ], +) +def test_resolve_fleet_blocks_unknown_gpu_evidence(evidence: dict) -> None: + module = load_resolution_module() + parsed = module.parse_fleet_evidence(evidence_document(evidence)) + + resolution = module.resolve_fleet(expected_targets(module, "host-1"), parsed) + + assert resolution.status is module.FleetStatus.BLOCKED + assert resolution.hosts[0].status is module.HostStatus.UNKNOWN + + +@pytest.mark.parametrize( + ("targets", "hosts"), + [ + (("gpu-1", "gpu-2"), ("gpu-1",)), + (("gpu-1",), ("gpu-1", "gpu-2")), + ], +) +def test_resolve_fleet_blocks_incomplete_or_unexpected_host_evidence( + targets: tuple[str, ...], hosts: tuple[str, ...] +) -> None: + module = load_resolution_module() + parsed = module.parse_fleet_evidence( + evidence_document(*(host_evidence(host, lspci_bdfs=[GPU_BDF]) for host in hosts)) + ) + + resolution = module.resolve_fleet(expected_targets(module, *targets), parsed) + + assert resolution.status is module.FleetStatus.BLOCKED + assert resolution.render_gid is None + + +def test_resolve_fleet_blocks_render_gid_collisions() -> None: + module = load_resolution_module() + parsed = module.parse_fleet_evidence( + evidence_document( + host_evidence( + "gpu-1", + lspci_bdfs=[GPU_BDF], + group_listing="render:x:993:\nother:x:993:\n", + ) + ) + ) + + resolution = module.resolve_fleet(expected_targets(module, "gpu-1"), parsed) + + assert resolution.status is module.FleetStatus.BLOCKED + assert resolution.hosts[0].status is module.HostStatus.UNKNOWN + + +def test_resolve_fleet_blocks_cpu_hosts_with_persisted_gpu_access_contracts() -> None: + module = load_resolution_module() + parsed = module.parse_fleet_evidence( + evidence_document(host_evidence("cpu-1", state='{"renderGid":993,"version":1}')) + ) + + resolution = module.resolve_fleet(expected_targets(module, "cpu-1"), parsed) + + assert resolution.status is module.FleetStatus.BLOCKED + assert resolution.hosts[0].status is module.HostStatus.UNKNOWN + + +@pytest.mark.parametrize("legacy_key", ["kfd", "amdgpu", "rocm_devices"]) +def test_resolve_fleet_blocks_cpu_hosts_with_any_legacy_gpu_access_rule(legacy_key: str) -> None: + module = load_resolution_module() + parsed = module.parse_fleet_evidence( + evidence_document(host_evidence("cpu-1", legacy_rules={legacy_key: 'KERNEL=="kfd", MODE="0666"\n'})) + ) + + resolution = module.resolve_fleet(expected_targets(module, "cpu-1"), parsed) + + assert resolution.status is module.FleetStatus.BLOCKED + assert resolution.hosts[0].status is module.HostStatus.UNKNOWN + + +def test_resolve_fleet_keeps_gpu_legacy_rule_admission_for_the_later_exact_migration() -> None: + module = load_resolution_module() + parsed = module.parse_fleet_evidence( + evidence_document(host_evidence("gpu-1", lspci_bdfs=[GPU_BDF], legacy_rules={"amdgpu": "legacy\n"})) + ) + + resolution = module.resolve_fleet(expected_targets(module, "gpu-1"), parsed) + + assert resolution.status is module.FleetStatus.GPU_RESOLVED + + +def test_resolve_fleet_blocks_file_probe_failures_instead_of_treating_them_as_absence() -> None: + module = load_resolution_module() + parsed = module.parse_fleet_evidence( + evidence_document(host_evidence("cpu-1", state_stat_success=False, state_content_success=False)) + ) + + resolution = module.resolve_fleet(expected_targets(module, "cpu-1"), parsed) + + assert resolution.status is module.FleetStatus.BLOCKED + assert resolution.hosts[0].status is module.HostStatus.UNKNOWN + + +@pytest.mark.parametrize( + "contracts", + [ + {"state": '{"renderGid":994,"version":1}'}, + {"rule": 'KERNEL=="kfd", MODE="0666"\n'}, + ], +) +def test_resolve_fleet_blocks_gpu_hosts_with_unsafe_persisted_contracts(contracts: dict) -> None: + module = load_resolution_module() + parsed = module.parse_fleet_evidence(evidence_document(host_evidence("gpu-1", lspci_bdfs=[GPU_BDF], **contracts))) + + resolution = module.resolve_fleet(expected_targets(module, "gpu-1"), parsed) + + assert resolution.status is module.FleetStatus.BLOCKED + assert resolution.hosts[0].status is module.HostStatus.UNKNOWN + + +def test_resolve_fleet_requires_unanimous_gpu_render_gid() -> None: + module = load_resolution_module() + same_gid = module.parse_fleet_evidence( + evidence_document( + host_evidence("gpu-1", lspci_bdfs=[GPU_BDF], render_gid=993), + host_evidence("gpu-2", lspci_bdfs=["0000:04:00.0"], render_gid=993), + ) + ) + mixed_gid = module.parse_fleet_evidence( + evidence_document( + host_evidence("gpu-1", lspci_bdfs=[GPU_BDF], render_gid=993), + host_evidence("gpu-2", lspci_bdfs=["0000:04:00.0"], render_gid=994), + ) + ) + + resolved = module.resolve_fleet(expected_targets(module, "gpu-1", "gpu-2"), same_gid) + blocked = module.resolve_fleet(expected_targets(module, "gpu-1", "gpu-2"), mixed_gid) + + assert resolved.status is module.FleetStatus.GPU_RESOLVED + assert resolved.render_gid == 993 + assert blocked.status is module.FleetStatus.BLOCKED + assert blocked.render_gid is None + + +def test_resolution_manifest_preserves_explicit_host_booleans_and_unanimous_gid() -> None: + module = load_resolution_module() + parsed = module.parse_fleet_evidence( + evidence_document( + host_evidence("gpu-1", lspci_bdfs=[GPU_BDF], render_gid=993), + host_evidence("cpu-1"), + ) + ) + + resolution = module.resolve_fleet(expected_targets(module, "gpu-1", "cpu-1"), parsed) + manifest = module.resolution_manifest(resolution) + + assert manifest == { + "version": 1, + "status": "gpu_resolved", + "render_gid": 993, + "hosts": {"cpu-1": False, "gpu-1": True}, + } + + +def test_resolution_manifest_is_an_ordinary_dict_with_exact_order_and_sorted_hosts() -> None: + manifest = load_manifest_module().build_resolution_manifest( + version=1, + status="gpu_resolved", + render_gid=993, + hosts={"zeta": True, "alpha": False}, + ) + + assert type(manifest) is dict + assert list(manifest) == ["version", "status", "render_gid", "hosts"] + assert list(manifest["hosts"]) == ["alpha", "zeta"] + assert set(manifest) == {"version", "status", "render_gid", "hosts"} + + +def test_pxe_resolution_manifest_constructs_without_mutating_base_manifest() -> None: + module = load_manifest_module() + base = module.build_resolution_manifest( + version=1, + status="gpu_resolved", + render_gid=993, + hosts={"gpu-2": True, "gpu-1": True}, + ) + + manifest = module.build_pxe_resolution_manifest( + version=base["version"], + status=base["status"], + render_gid=base["render_gid"], + hosts=base["hosts"], + gpu_access_enabled=True, + pxe_render_gid=994, + ) + + assert base == { + "version": 1, + "status": "gpu_resolved", + "render_gid": 993, + "hosts": {"gpu-1": True, "gpu-2": True}, + } + assert list(manifest) == ["version", "status", "render_gid", "hosts", "pxe_rootfs"] + assert manifest["pxe_rootfs"] == {"gpu_access_enabled": True, "render_gid": 994} + assert set(manifest["pxe_rootfs"]) == {"gpu_access_enabled", "render_gid"} From d41f301ae342fc9f3607022f7aaabd9c2642124c Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:18 +0800 Subject: [PATCH 15/65] feat(deploy): discover fleet GPU access state --- .../playbooks/pb-gpu-access-discovery.yml | 386 ++++++++++++++++++ .../scripts/gpu_artifact_generation.py | 218 ++++++++++ tests/skills/test_gpu_artifact_generation.py | 308 ++++++++++++++ 3 files changed, 912 insertions(+) create mode 100644 deploy/ansible/playbooks/pb-gpu-access-discovery.yml create mode 100644 skills/deploy-aup-learning-cloud/scripts/gpu_artifact_generation.py create mode 100644 tests/skills/test_gpu_artifact_generation.py diff --git a/deploy/ansible/playbooks/pb-gpu-access-discovery.yml b/deploy/ansible/playbooks/pb-gpu-access-discovery.yml new file mode 100644 index 00000000..cd8c366a --- /dev/null +++ b/deploy/ansible/playbooks/pb-gpu-access-discovery.yml @@ -0,0 +1,386 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +--- +- name: Discover fleet GPU-access evidence + hosts: k3s_cluster + gather_facts: false + become: false + ignore_unreachable: true + vars: + _auplc_gpu_access_unknown_evidence: + reachable: false + lspci: + rc: 255 + stdout: "" + sysfs: + rc: 255 + stdout: "" + render_group: + rc: 255 + stdout: "" + groups: + rc: 255 + stdout: "" + state: + stat_success: false + content_success: false + exists: false + regular: false + symlink: false + content: "" + rule: + stat_success: false + content_success: false + exists: false + regular: false + symlink: false + content: "" + legacy_rules: + kfd: + stat_success: false + content_success: false + exists: false + regular: false + symlink: false + content: "" + amdgpu: + stat_success: false + content_success: false + exists: false + regular: false + symlink: false + content: "" + rocm_devices: + stat_success: false + content_success: false + exists: false + regular: false + symlink: false + content: "" + pre_tasks: + - name: Require a safe local discovery evidence output path + ansible.builtin.assert: + that: + - gpu_access_discovery_output_path is defined + - gpu_access_discovery_output_path is string + - gpu_access_discovery_output_path is match('^/') + fail_msg: >- + Set gpu_access_discovery_output_path to an absolute controller-local + path before running discovery. + delegate_to: localhost + run_once: true + changed_when: false + + - name: Inspect local discovery evidence parent + ansible.builtin.stat: + path: "{{ gpu_access_discovery_output_path | dirname }}" + follow: false + delegate_to: localhost + run_once: true + register: _auplc_discovery_output_parent + changed_when: false + + - name: Require safe local discovery evidence parent + ansible.builtin.assert: + that: + - _auplc_discovery_output_parent.stat.exists + - _auplc_discovery_output_parent.stat.isdir + - not _auplc_discovery_output_parent.stat.islnk + fail_msg: Discovery output parent must be an existing non-symlink directory. + delegate_to: localhost + run_once: true + changed_when: false + + - name: Inspect local discovery evidence destination + ansible.builtin.stat: + path: "{{ gpu_access_discovery_output_path }}" + follow: false + delegate_to: localhost + run_once: true + register: _auplc_discovery_output_destination + changed_when: false + + - name: Require safe local discovery evidence destination + ansible.builtin.assert: + that: + - >- + not _auplc_discovery_output_destination.stat.exists or + (_auplc_discovery_output_destination.stat.isreg and + not _auplc_discovery_output_destination.stat.islnk) + fail_msg: Discovery output destination must be absent or a regular non-symlink file. + delegate_to: localhost + run_once: true + changed_when: false + tasks: + - name: Discover AMD VGA display BDFs with lspci + ansible.builtin.command: + argv: + - lspci + - -Dnn + - -d + - "1002::0300" + register: _auplc_discovery_lspci_vga + changed_when: false + failed_when: false + + - name: Discover AMD 3D display BDFs with lspci + ansible.builtin.command: + argv: + - lspci + - -Dnn + - -d + - "1002::0302" + register: _auplc_discovery_lspci_3d + changed_when: false + failed_when: false + + - name: Discover AMD display-controller BDFs with lspci + ansible.builtin.command: + argv: + - lspci + - -Dnn + - -d + - "1002::0380" + register: _auplc_discovery_lspci_display + changed_when: false + failed_when: false + + - name: Combine AMD display lspci evidence + ansible.builtin.set_fact: + _auplc_discovery_lspci: + rc: >- + {{ 0 if _auplc_discovery_lspci_vga.rc == 0 and + _auplc_discovery_lspci_3d.rc == 0 and + _auplc_discovery_lspci_display.rc == 0 else 1 }} + stdout: >- + {{ [_auplc_discovery_lspci_vga.stdout | default(''), + _auplc_discovery_lspci_3d.stdout | default(''), + _auplc_discovery_lspci_display.stdout | default('')] + | reject('equalto', '') | join('\n') }} + changed_when: false + + - name: Discover AMD display BDFs through sysfs + ansible.builtin.command: + argv: + - python3 + - -c + - >- + from pathlib import Path; devices = Path('/sys/bus/pci/devices'); + print('\n'.join(sorted(device.name for device in devices.iterdir() + if (device / 'vendor').read_text().strip() == '0x1002' and + (device / 'class').read_text().strip().startswith('0x03')))) + register: _auplc_discovery_sysfs + changed_when: false + failed_when: false + + - name: Read render group record + ansible.builtin.command: + argv: + - getent + - group + - render + register: _auplc_discovery_render_group + changed_when: false + failed_when: false + + - name: Read all group records for render GID collision detection + ansible.builtin.command: + argv: + - getent + - group + register: _auplc_discovery_groups + changed_when: false + failed_when: false + + - name: Inspect persisted GPU access state + ansible.builtin.stat: + path: /var/lib/auplc/gpu-access.json + follow: false + register: _auplc_discovery_state + changed_when: false + ignore_errors: true + + - name: Read persisted GPU access state + ansible.builtin.slurp: + src: /var/lib/auplc/gpu-access.json + register: _auplc_discovery_state_content + when: + - _auplc_discovery_state.stat.exists | default(false) + - _auplc_discovery_state.stat.isreg | default(false) + - not (_auplc_discovery_state.stat.islnk | default(false)) + changed_when: false + ignore_errors: true + + - name: Inspect canonical GPU access rule + ansible.builtin.stat: + path: /etc/udev/rules.d/70-auplc-gpu-access.rules + follow: false + register: _auplc_discovery_rule + changed_when: false + ignore_errors: true + + - name: Read canonical GPU access rule + ansible.builtin.slurp: + src: /etc/udev/rules.d/70-auplc-gpu-access.rules + register: _auplc_discovery_rule_content + when: + - _auplc_discovery_rule.stat.exists | default(false) + - _auplc_discovery_rule.stat.isreg | default(false) + - not (_auplc_discovery_rule.stat.islnk | default(false)) + changed_when: false + ignore_errors: true + + - name: Inspect legacy kfd GPU access rule + ansible.builtin.stat: + path: /etc/udev/rules.d/70-kfd.rules + follow: false + register: _auplc_discovery_legacy_kfd + changed_when: false + ignore_errors: true + + - name: Read legacy kfd GPU access rule + ansible.builtin.slurp: + src: /etc/udev/rules.d/70-kfd.rules + register: _auplc_discovery_legacy_kfd_content + when: + - _auplc_discovery_legacy_kfd.stat.exists | default(false) + - _auplc_discovery_legacy_kfd.stat.isreg | default(false) + - not (_auplc_discovery_legacy_kfd.stat.islnk | default(false)) + changed_when: false + ignore_errors: true + + - name: Inspect legacy amdgpu GPU access rule + ansible.builtin.stat: + path: /etc/udev/rules.d/70-amdgpu.rules + follow: false + register: _auplc_discovery_legacy_amdgpu + changed_when: false + ignore_errors: true + + - name: Read legacy amdgpu GPU access rule + ansible.builtin.slurp: + src: /etc/udev/rules.d/70-amdgpu.rules + register: _auplc_discovery_legacy_amdgpu_content + when: + - _auplc_discovery_legacy_amdgpu.stat.exists | default(false) + - _auplc_discovery_legacy_amdgpu.stat.isreg | default(false) + - not (_auplc_discovery_legacy_amdgpu.stat.islnk | default(false)) + changed_when: false + ignore_errors: true + + - name: Inspect legacy ROCm devices GPU access rule + ansible.builtin.stat: + path: /etc/udev/rules.d/70-rocm-devices.rules + follow: false + register: _auplc_discovery_legacy_rocm_devices + changed_when: false + ignore_errors: true + + - name: Read legacy ROCm devices GPU access rule + ansible.builtin.slurp: + src: /etc/udev/rules.d/70-rocm-devices.rules + register: _auplc_discovery_legacy_rocm_devices_content + when: + - _auplc_discovery_legacy_rocm_devices.stat.exists | default(false) + - _auplc_discovery_legacy_rocm_devices.stat.isreg | default(false) + - not (_auplc_discovery_legacy_rocm_devices.stat.islnk | default(false)) + changed_when: false + ignore_errors: true + + - name: Record machine-readable GPU access discovery evidence + ansible.builtin.set_fact: + _auplc_gpu_access_discovery_evidence: + host: "{{ inventory_hostname }}" + reachable: true + lspci: + rc: "{{ _auplc_discovery_lspci.rc }}" + stdout: "{{ _auplc_discovery_lspci.stdout | default('') }}" + sysfs: + rc: "{{ _auplc_discovery_sysfs.rc }}" + stdout: "{{ _auplc_discovery_sysfs.stdout | default('') }}" + render_group: + rc: "{{ _auplc_discovery_render_group.rc }}" + stdout: "{{ _auplc_discovery_render_group.stdout | default('') }}" + groups: + rc: "{{ _auplc_discovery_groups.rc }}" + stdout: "{{ _auplc_discovery_groups.stdout | default('') }}" + state: + stat_success: "{{ not (_auplc_discovery_state.failed | default(false)) }}" + content_success: >- + {{ not (_auplc_discovery_state.failed | default(false)) and + (not (_auplc_discovery_state.stat.exists | default(false)) or + not (_auplc_discovery_state.stat.isreg | default(false)) or + (_auplc_discovery_state.stat.islnk | default(false)) or + not (_auplc_discovery_state_content.failed | default(false))) }} + exists: "{{ _auplc_discovery_state.stat.exists | default(false) }}" + regular: "{{ _auplc_discovery_state.stat.isreg | default(false) }}" + symlink: "{{ _auplc_discovery_state.stat.islnk | default(false) }}" + content: "{{ _auplc_discovery_state_content.content | default('') | b64decode }}" + rule: + stat_success: "{{ not (_auplc_discovery_rule.failed | default(false)) }}" + content_success: >- + {{ not (_auplc_discovery_rule.failed | default(false)) and + (not (_auplc_discovery_rule.stat.exists | default(false)) or + not (_auplc_discovery_rule.stat.isreg | default(false)) or + (_auplc_discovery_rule.stat.islnk | default(false)) or + not (_auplc_discovery_rule_content.failed | default(false))) }} + exists: "{{ _auplc_discovery_rule.stat.exists | default(false) }}" + regular: "{{ _auplc_discovery_rule.stat.isreg | default(false) }}" + symlink: "{{ _auplc_discovery_rule.stat.islnk | default(false) }}" + content: "{{ _auplc_discovery_rule_content.content | default('') | b64decode }}" + legacy_rules: + kfd: + stat_success: "{{ not (_auplc_discovery_legacy_kfd.failed | default(false)) }}" + content_success: >- + {{ not (_auplc_discovery_legacy_kfd.failed | default(false)) and + (not (_auplc_discovery_legacy_kfd.stat.exists | default(false)) or + not (_auplc_discovery_legacy_kfd.stat.isreg | default(false)) or + (_auplc_discovery_legacy_kfd.stat.islnk | default(false)) or + not (_auplc_discovery_legacy_kfd_content.failed | default(false))) }} + exists: "{{ _auplc_discovery_legacy_kfd.stat.exists | default(false) }}" + regular: "{{ _auplc_discovery_legacy_kfd.stat.isreg | default(false) }}" + symlink: "{{ _auplc_discovery_legacy_kfd.stat.islnk | default(false) }}" + content: "{{ _auplc_discovery_legacy_kfd_content.content | default('') | b64decode }}" + amdgpu: + stat_success: "{{ not (_auplc_discovery_legacy_amdgpu.failed | default(false)) }}" + content_success: >- + {{ not (_auplc_discovery_legacy_amdgpu.failed | default(false)) and + (not (_auplc_discovery_legacy_amdgpu.stat.exists | default(false)) or + not (_auplc_discovery_legacy_amdgpu.stat.isreg | default(false)) or + (_auplc_discovery_legacy_amdgpu.stat.islnk | default(false)) or + not (_auplc_discovery_legacy_amdgpu_content.failed | default(false))) }} + exists: "{{ _auplc_discovery_legacy_amdgpu.stat.exists | default(false) }}" + regular: "{{ _auplc_discovery_legacy_amdgpu.stat.isreg | default(false) }}" + symlink: "{{ _auplc_discovery_legacy_amdgpu.stat.islnk | default(false) }}" + content: "{{ _auplc_discovery_legacy_amdgpu_content.content | default('') | b64decode }}" + rocm_devices: + stat_success: "{{ not (_auplc_discovery_legacy_rocm_devices.failed | default(false)) }}" + content_success: >- + {{ not (_auplc_discovery_legacy_rocm_devices.failed | default(false)) and + (not (_auplc_discovery_legacy_rocm_devices.stat.exists | default(false)) or + not (_auplc_discovery_legacy_rocm_devices.stat.isreg | default(false)) or + (_auplc_discovery_legacy_rocm_devices.stat.islnk | default(false)) or + not (_auplc_discovery_legacy_rocm_devices_content.failed | default(false))) }} + exists: "{{ _auplc_discovery_legacy_rocm_devices.stat.exists | default(false) }}" + regular: "{{ _auplc_discovery_legacy_rocm_devices.stat.isreg | default(false) }}" + symlink: "{{ _auplc_discovery_legacy_rocm_devices.stat.islnk | default(false) }}" + content: "{{ _auplc_discovery_legacy_rocm_devices_content.content | default('') | b64decode }}" + changed_when: false + + - name: Write machine-readable GPU access discovery evidence locally + ansible.builtin.copy: + content: | + {"version":2,"hosts":[{% for discovery_host in ansible_play_hosts_all %} + {{ ( + hostvars[discovery_host]._auplc_gpu_access_discovery_evidence + | default( + _auplc_gpu_access_unknown_evidence | combine({'host': discovery_host}), + true + ) + | to_json + ) }}{% if not loop.last %},{% endif %} + {% endfor %}]} + dest: "{{ gpu_access_discovery_output_path }}" + mode: "0600" + delegate_to: localhost + run_once: true + changed_when: false diff --git a/skills/deploy-aup-learning-cloud/scripts/gpu_artifact_generation.py b/skills/deploy-aup-learning-cloud/scripts/gpu_artifact_generation.py new file mode 100644 index 00000000..3c47e1d2 --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/gpu_artifact_generation.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +"""Discover live GPU facts and prepare publication-safe resolved artifacts.""" + +from __future__ import annotations + +import json +import os +import re +import stat +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import NoReturn + +from artifact_store import publish_artifacts +from config_common import yaml_quote +from config_generation import HEADER_HASH +from gpu_access_resolution import ( + EvidenceParseError, + FleetResolution, + FleetStatus, + InventoryTarget, + parse_fleet_evidence, + resolution_manifest, + resolve_fleet, +) + +DISCOVERY_TIMEOUT_BASE_SECONDS = 30 +DISCOVERY_TIMEOUT_PER_TARGET_SECONDS = 15 +DISCOVERY_TIMEOUT_MAX_SECONDS = 300 +DISCOVERY_DIAGNOSTIC_MAX_CHARS = 1200 + + +def assert_never(value: FleetStatus) -> NoReturn: + raise AssertionError(f"unexpected fleet status: {value}") + + +@dataclass(frozen=True, slots=True) +class DiscoveryFailure(Exception): + reason: str + + def __str__(self) -> str: + return self.reason + + +@dataclass(frozen=True, slots=True) +class DiscoveryPaths: + inventory: Path + evidence: Path + + +@dataclass(frozen=True, slots=True) +class DiscoveryResult: + resolution: FleetResolution + + +def canonical_paths(out_dir: Path) -> tuple[Path, Path, Path]: + return ( + out_dir / "inventory.yml", + out_dir / "values-basic-example.yaml", + out_dir / "gpu-access-resolution.json", + ) + + +def discover_gpu_policy(spec: dict, out_dir: Path) -> DiscoveryResult: + targets = live_targets(spec) + paths = stage_private_discovery(spec, out_dir) + run_discovery(paths, len(targets)) + try: + evidence = parse_fleet_evidence(read_regular_file(paths.evidence)) + except EvidenceParseError as error: + raise DiscoveryFailure("GPU discovery evidence is malformed") from error + resolution = resolve_fleet(targets, evidence) + match resolution.status: + case FleetStatus.BLOCKED: + raise DiscoveryFailure(f"GPU discovery is blocked: {resolution.reason}") + case FleetStatus.GPU_RESOLVED | FleetStatus.CPU_ONLY: + pass + case unreachable: + assert_never(unreachable) + return DiscoveryResult(resolution=resolution) + + +def live_targets(spec: dict) -> tuple[InventoryTarget, ...]: + names = [spec["server"]["name"]] + if spec["topology"] == "ssh-preinstalled": + names.extend(agent["name"] for agent in spec.get("agents", [])) + if len(names) != len(set(names)): + raise DiscoveryFailure("live target names must be unique") + return tuple(InventoryTarget(name=name) for name in names) + + +def stage_private_discovery(spec: dict, out_dir: Path) -> DiscoveryPaths: + resolved_out_dir = out_dir.resolve() + paths = DiscoveryPaths( + inventory=resolved_out_dir / ".gpu-access-discovery.inventory.yml", + evidence=resolved_out_dir / ".gpu-access-discovery-evidence.json", + ) + publish_artifacts( + [ + (paths.inventory, render_discovery_inventory(spec), 0o600, False), + (paths.evidence, "", 0o600, False), + ], + force=True, + ) + return paths + + +def render_discovery_inventory(spec: dict) -> str: + server = spec["server"] + lines = [ + HEADER_HASH, + "k3s_cluster:", + " children:", + " server:", + " hosts:", + f" {server['name']}:", + f" ansible_host: {yaml_quote(server['ip'])}", + " agent:", + ] + if spec["topology"] == "ssh-preinstalled" and spec.get("agents"): + lines.append(" hosts:") + for agent in spec["agents"]: + lines += [f" {agent['name']}:", f" ansible_host: {yaml_quote(agent['ip'])}"] + else: + lines.append(" hosts: {}") + lines += [" vars:", " ansible_port: 22", " ansible_user: root"] + return "\n".join(lines) + "\n" + + +def discovery_timeout_seconds(target_count: int) -> int: + configured = os.environ.get("AUPLC_GPU_DISCOVERY_TIMEOUT_SECONDS") + if configured is not None: + try: + timeout = int(configured) + except ValueError as error: + raise DiscoveryFailure("AUPLC_GPU_DISCOVERY_TIMEOUT_SECONDS must be an integer") from error + if not DISCOVERY_TIMEOUT_BASE_SECONDS <= timeout <= DISCOVERY_TIMEOUT_MAX_SECONDS: + raise DiscoveryFailure( + f"AUPLC_GPU_DISCOVERY_TIMEOUT_SECONDS must be between {DISCOVERY_TIMEOUT_BASE_SECONDS} and " + f"{DISCOVERY_TIMEOUT_MAX_SECONDS}" + ) + return timeout + return min( + DISCOVERY_TIMEOUT_MAX_SECONDS, + DISCOVERY_TIMEOUT_BASE_SECONDS + (DISCOVERY_TIMEOUT_PER_TARGET_SECONDS * target_count), + ) + + +def _bounded_diagnostic(*values: str | bytes | None) -> str: + text = "\n".join(value.decode(errors="replace") if isinstance(value, bytes) else value or "" for value in values) + text = re.sub(r"(?i)\b(token|password|secret|private[_-]?key)\s*[:=]\s*\S+", r"\1=", text) + lines = [line.strip() for line in text.splitlines() if line.strip()] + summary = " | ".join(lines[-8:]) + return summary[-DISCOVERY_DIAGNOSTIC_MAX_CHARS:] or "no Ansible diagnostics" + + +def run_discovery(paths: DiscoveryPaths, target_count: int) -> None: + playbook = Path(__file__).resolve().parents[3] / "deploy" / "ansible" / "playbooks" / "pb-gpu-access-discovery.yml" + argv = [ + "ansible-playbook", + "-i", + str(paths.inventory), + str(playbook), + "-e", + f"gpu_access_discovery_output_path={paths.evidence}", + ] + environment = os.environ.copy() + environment["ANSIBLE_CONFIG"] = str(playbook.parents[1] / "ansible.cfg") + environment["ANSIBLE_HOST_KEY_CHECKING"] = "True" + environment["ANSIBLE_SSH_HOST_KEY_CHECKING"] = "True" + environment["ANSIBLE_SSH_ARGS"] = "-o StrictHostKeyChecking=yes" + for key in ( + "ANSIBLE_SSH_COMMON_ARGS", + "ANSIBLE_SSH_EXTRA_ARGS", + "ANSIBLE_SCP_IF_SSH", + "ANSIBLE_SCP_EXTRA_ARGS", + "ANSIBLE_SFTP_EXTRA_ARGS", + ): + environment.pop(key, None) + timeout = discovery_timeout_seconds(target_count) + try: + result = subprocess.run( + argv, + capture_output=True, + check=False, + cwd=playbook.parents[1], + env=environment, + text=True, + timeout=timeout, + ) + except FileNotFoundError as error: + raise DiscoveryFailure("ansible-playbook is required for GPU discovery") from error + except subprocess.TimeoutExpired as error: + diagnostic = _bounded_diagnostic(error.stderr, error.stdout) + raise DiscoveryFailure(f"GPU discovery playbook timed out after {timeout}s: {diagnostic}") from error + if result.returncode != 0: + diagnostic = _bounded_diagnostic(result.stderr, result.stdout) + raise DiscoveryFailure(f"GPU discovery playbook failed with exit code {result.returncode}: {diagnostic}") + + +def read_regular_file(path: Path) -> str: + try: + mode = os.lstat(path).st_mode + except FileNotFoundError as error: + raise DiscoveryFailure("GPU discovery evidence was not written") from error + if not stat.S_ISREG(mode): + raise DiscoveryFailure("GPU discovery evidence must be a regular file") + try: + return path.read_text(encoding="utf-8") + except OSError as error: + raise DiscoveryFailure("GPU discovery evidence could not be read") from error + + +def manifest_content(result: DiscoveryResult) -> str: + document = resolution_manifest(result.resolution) + return json.dumps(document, indent=2, sort_keys=True) + "\n" diff --git a/tests/skills/test_gpu_artifact_generation.py b/tests/skills/test_gpu_artifact_generation.py new file mode 100644 index 00000000..92924077 --- /dev/null +++ b/tests/skills/test_gpu_artifact_generation.py @@ -0,0 +1,308 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +"""End-to-end contracts for automatic GPU artifact generation.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +GEN_CONFIGS = ROOT / "skills" / "deploy-aup-learning-cloud" / "scripts" / "gen_configs.py" + + +def evidence_host(name: str, *, gpu: bool = False, gid: int = 993, reachable: bool = True) -> dict: + bdf = "0000:03:00.0" if gpu else "" + return { + "host": name, + "reachable": reachable, + "lspci": {"rc": 0, "stdout": bdf}, + "sysfs": {"rc": 0, "stdout": bdf}, + "render_group": {"rc": 0, "stdout": f"render:x:{gid}:\n"}, + "groups": {"rc": 0, "stdout": f"render:x:{gid}:\n"}, + "state": { + "stat_success": True, + "content_success": True, + "exists": False, + "regular": False, + "symlink": False, + "content": "", + }, + "rule": { + "stat_success": True, + "content_success": True, + "exists": False, + "regular": False, + "symlink": False, + "content": "", + }, + "legacy_rules": { + key: { + "stat_success": True, + "content_success": True, + "exists": False, + "regular": False, + "symlink": False, + "content": "", + } + for key in ("kfd", "amdgpu", "rocm_devices") + }, + } + + +def ssh_spec() -> dict: + return { + "topology": "ssh-preinstalled", + "k3s_version": "v1.32.3+k3s1", + "server": {"name": "server", "ip": "192.168.1.10"}, + "agents": [{"name": "agent", "ip": "192.168.1.11"}], + } + + +def write_fake_ansible(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, document: dict) -> Path: + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + fake_ansible = fake_bin / "ansible-playbook" + fake_ansible.write_text( + """#!/usr/bin/env python3 +import json +import os +from pathlib import Path +import sys + +arguments = sys.argv[1:] +Path(os.environ["FAKE_ANSIBLE_RECORD"]).write_text(json.dumps(arguments), encoding="utf-8") +environment_record = os.environ.get("FAKE_ANSIBLE_ENV_RECORD") +if environment_record: + Path(environment_record).write_text( + json.dumps({key: os.environ.get(key) for key in ("ANSIBLE_CONFIG", "ANSIBLE_HOST_KEY_CHECKING", "ANSIBLE_SSH_ARGS", "ANSIBLE_SSH_COMMON_ARGS", "ANSIBLE_SSH_EXTRA_ARGS", "ANSIBLE_SSH_HOST_KEY_CHECKING", "ANSIBLE_SCP_IF_SSH", "ANSIBLE_SCP_EXTRA_ARGS", "ANSIBLE_SFTP_EXTRA_ARGS")}), + encoding="utf-8", + ) +output = next(value.split("=", 1)[1] for value in arguments if value.startswith("gpu_access_discovery_output_path=")) +Path(output).write_text(os.environ["FAKE_ANSIBLE_EVIDENCE"], encoding="utf-8") +""", + encoding="utf-8", + ) + fake_ansible.chmod(0o755) + record = tmp_path / "ansible-argv.json" + monkeypatch.setenv("FAKE_ANSIBLE_RECORD", str(record)) + monkeypatch.setenv("FAKE_ANSIBLE_EVIDENCE", json.dumps(document)) + monkeypatch.setenv("PATH", f"{fake_bin}:{os.environ['PATH']}") + return record + + +def test_generator_forces_repository_host_key_checking_over_disabled_environment( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_fake_ansible( + tmp_path, monkeypatch, {"version": 2, "hosts": [evidence_host("server"), evidence_host("agent")]} + ) + environment_record = tmp_path / "ansible-environment.json" + monkeypatch.setenv("FAKE_ANSIBLE_ENV_RECORD", str(environment_record)) + monkeypatch.setenv("ANSIBLE_CONFIG", str(tmp_path / "disabled-ansible.cfg")) + monkeypatch.setenv("ANSIBLE_HOST_KEY_CHECKING", "False") + monkeypatch.setenv("ANSIBLE_SSH_ARGS", "-o StrictHostKeyChecking=no") + monkeypatch.setenv("ANSIBLE_SSH_COMMON_ARGS", "-o UserKnownHostsFile=/dev/null") + monkeypatch.setenv("ANSIBLE_SSH_HOST_KEY_CHECKING", "False") + monkeypatch.setenv("ANSIBLE_SSH_EXTRA_ARGS", "-o StrictHostKeyChecking=no") + monkeypatch.setenv("ANSIBLE_SCP_IF_SSH", "True") + monkeypatch.setenv("ANSIBLE_SCP_EXTRA_ARGS", "-o UserKnownHostsFile=/dev/null") + monkeypatch.setenv("ANSIBLE_SFTP_EXTRA_ARGS", "-o StrictHostKeyChecking=no") + spec_path = write_json(tmp_path / "spec.json", ssh_spec()) + + result = run_generator(spec_path, tmp_path / "generated") + + assert result.returncode == 0, result.stderr + assert json.loads(environment_record.read_text(encoding="utf-8")) == { + "ANSIBLE_CONFIG": str(ROOT / "deploy" / "ansible" / "ansible.cfg"), + "ANSIBLE_HOST_KEY_CHECKING": "True", + "ANSIBLE_SSH_ARGS": "-o StrictHostKeyChecking=yes", + "ANSIBLE_SSH_COMMON_ARGS": None, + "ANSIBLE_SSH_EXTRA_ARGS": None, + "ANSIBLE_SSH_HOST_KEY_CHECKING": "True", + "ANSIBLE_SCP_IF_SSH": None, + "ANSIBLE_SCP_EXTRA_ARGS": None, + "ANSIBLE_SFTP_EXTRA_ARGS": None, + } + + +def test_generator_surfaces_redacted_bounded_ansible_failure_diagnostics( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + fake_ansible = fake_bin / "ansible-playbook" + fake_ansible.write_text( + "#!/bin/sh\nprintf '%s\\n' 'fatal: [server]: UNREACHABLE! token=do-not-disclose' >&2\nexit 2\n", + encoding="utf-8", + ) + fake_ansible.chmod(0o755) + monkeypatch.setenv("PATH", f"{fake_bin}:{os.environ['PATH']}") + spec_path = write_json(tmp_path / "spec.json", ssh_spec()) + + result = run_generator(spec_path, tmp_path / "generated") + + assert result.returncode == 1 + assert "exit code 2" in result.stderr + assert "fatal: [server]: UNREACHABLE!" in result.stderr + assert "do-not-disclose" not in result.stderr + assert "token=" in result.stderr + + +def run_generator(spec_path: Path, out_dir: Path, *extra: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(GEN_CONFIGS), "--spec", str(spec_path), "--out-dir", str(out_dir), *extra], + capture_output=True, + check=False, + text=True, + timeout=30, + ) + + +def write_json(path: Path, document: dict) -> Path: + path.write_text(json.dumps(document), encoding="utf-8") + return path + + +def test_generator_discovers_mixed_ssh_targets_and_publishes_resolved_artifacts( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + record = write_fake_ansible( + tmp_path, + monkeypatch, + {"version": 2, "hosts": [evidence_host("server", gpu=True), evidence_host("agent")]}, + ) + spec_path = write_json(tmp_path / "spec.json", ssh_spec()) + out_dir = tmp_path / "generated" + + result = run_generator(spec_path, out_dir) + + assert result.returncode == 0, result.stderr + inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") + assert "auplc_render_gid: 993" in inventory + assert inventory.count("auplc_gpu_access_enabled: true") == 1 + assert inventory.count("auplc_gpu_access_enabled: false") == 1 + assert "renderGid: 993" in (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") + assert json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) == { + "version": 1, + "status": "gpu_resolved", + "render_gid": 993, + "hosts": {"agent": False, "server": True}, + } + discovery_inventory = out_dir / ".gpu-access-discovery.inventory.yml" + discovery_evidence = out_dir / ".gpu-access-discovery-evidence.json" + assert discovery_inventory.stat().st_mode & 0o777 == 0o600 + assert discovery_evidence.stat().st_mode & 0o777 == 0o600 + assert json.loads(record.read_text(encoding="utf-8")) == [ + "-i", + str(discovery_inventory), + str(ROOT / "deploy" / "ansible" / "playbooks" / "pb-gpu-access-discovery.yml"), + "-e", + f"gpu_access_discovery_output_path={discovery_evidence}", + ] + + +def test_generator_publishes_null_render_gid_for_all_cpu_ssh_targets( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_fake_ansible( + tmp_path, + monkeypatch, + {"version": 2, "hosts": [evidence_host("server"), evidence_host("agent")]}, + ) + spec_path = write_json(tmp_path / "spec.json", ssh_spec()) + out_dir = tmp_path / "generated" + + result = run_generator(spec_path, out_dir) + + assert result.returncode == 0, result.stderr + inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") + assert "auplc_render_gid: null" in inventory + assert inventory.count("auplc_gpu_access_enabled: false") == 2 + assert "renderGid: null" in (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") + manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) + assert manifest == { + "version": 1, + "status": "cpu_only", + "render_gid": None, + "hosts": {"agent": False, "server": False}, + } + + +@pytest.mark.parametrize("failure", ["missing", "nonzero"]) +def test_generator_does_not_publish_when_ansible_is_unavailable_or_fails( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure: str +) -> None: + spec_path = write_json(tmp_path / "spec.json", ssh_spec()) + out_dir = tmp_path / "generated" + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + if failure == "nonzero": + fake_ansible = fake_bin / "ansible-playbook" + fake_ansible.write_text("#!/bin/sh\nexit 1\n", encoding="utf-8") + fake_ansible.chmod(0o755) + monkeypatch.setenv("PATH", str(fake_bin)) + + result = run_generator(spec_path, out_dir) + + assert result.returncode == 1 + assert not (out_dir / "inventory.yml").exists() + assert not (out_dir / "values-basic-example.yaml").exists() + assert not (out_dir / "gpu-access-resolution.json").exists() + + +@pytest.mark.parametrize( + "document", + [ + {"version": 2, "hosts": [evidence_host("server", reachable=False), evidence_host("agent")]}, + { + "version": 2, + "hosts": [evidence_host("server", gpu=True, gid=993), evidence_host("agent", gpu=True, gid=994)], + }, + ], + ids=["unknown", "gid-disagreement"], +) +def test_generator_keeps_canonical_artifacts_unchanged_when_discovery_blocks( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, document: dict +) -> None: + write_fake_ansible(tmp_path, monkeypatch, document) + spec_path = write_json(tmp_path / "spec.json", ssh_spec()) + out_dir = tmp_path / "generated" + out_dir.mkdir() + inventory = out_dir / "inventory.yml" + values = out_dir / "values-basic-example.yaml" + manifest = out_dir / "gpu-access-resolution.json" + inventory.write_text("previous inventory\n", encoding="utf-8") + values.write_text("previous values\n", encoding="utf-8") + manifest.write_text("previous manifest\n", encoding="utf-8") + + result = run_generator(spec_path, out_dir, "--force") + + assert result.returncode == 1 + assert inventory.read_text(encoding="utf-8") == "previous inventory\n" + assert values.read_text(encoding="utf-8") == "previous values\n" + assert manifest.read_text(encoding="utf-8") == "previous manifest\n" + + +@pytest.mark.parametrize( + "field", + ["render_gid", "gpu_access"], +) +def test_generator_rejects_removed_public_gpu_fields_before_running_discovery( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, field: str +) -> None: + record = write_fake_ansible(tmp_path, monkeypatch, {"version": 2, "hosts": []}) + spec = ssh_spec() + spec[field] = 993 if field == "render_gid" else {"hosts": []} + spec_path = write_json(tmp_path / "spec.json", spec) + + result = run_generator(spec_path, tmp_path / "generated") + + assert result.returncode == 1 + assert f"spec.{field} is no longer accepted" in result.stderr + assert not record.exists() From 0f5a1cb77e32585872d0fb1b2390217975b42d49 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:18 +0800 Subject: [PATCH 16/65] feat(deploy): stage atomic PXE GPU finalization --- .../scripts/pxe_finalization.py | 173 ++++++ .../scripts/pxe_finalization_support.py | 257 +++++++++ tests/skills/test_pxe_finalization.py | 516 ++++++++++++++++++ 3 files changed, 946 insertions(+) create mode 100644 skills/deploy-aup-learning-cloud/scripts/pxe_finalization.py create mode 100644 skills/deploy-aup-learning-cloud/scripts/pxe_finalization_support.py create mode 100644 tests/skills/test_pxe_finalization.py diff --git a/skills/deploy-aup-learning-cloud/scripts/pxe_finalization.py b/skills/deploy-aup-learning-cloud/scripts/pxe_finalization.py new file mode 100644 index 00000000..cb9c7c9f --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/pxe_finalization.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +"""Orchestrate transactional PXE configuration finalization.""" + +from __future__ import annotations + +import json +import os +import secrets +from pathlib import Path + +import pxe_finalization_support as _support +from artifact_store import preflight_destinations, publish_artifacts +from config_rendering import ResolvedGpuPolicy, render_inventory, render_pxe_vars, render_values +from gpu_access_resolution import ( + FleetResolution, + FleetStatus, + HostStatus, + resolution_manifest, +) +from gpu_access_resolution import HostResolution as HostResolution +from gpu_resolution_manifest import build_pxe_resolution_manifest +from pxe_finalization_support import MAX_RENDER_GID as MAX_RENDER_GID +from pxe_finalization_support import ( + VERSION, + Artifact, + JsonDocument, +) +from pxe_finalization_support import FinalizationError as FinalizationError +from pxe_finalization_support import PxePaths as PxePaths +from pxe_finalization_support import paths as paths + +_artifact_attestations = _support.artifact_attestations +_completion = _support.completion +_controller_resolution = _support.controller_resolution +_exclusive_lock = _support.exclusive_lock +_final_resolution = _support.final_resolution +_generation_paths = _support.generation_paths +_read_artifact_attestation = _support.read_artifact_attestation +_read_document = _support.read_document +_spec_sha256 = _support.spec_sha256 +_target = _support.target +_valid_gid = _support.valid_gid +_validate = _support.validate +_verify_canonical_artifacts = _support.verify_canonical_artifacts + + +def stage_pending(spec: JsonDocument, token: str, controller: FleetResolution, out_dir: Path, force: bool) -> PxePaths: + pending = paths(out_dir) + if controller.status is FleetStatus.BLOCKED: + raise FinalizationError(f"GPU discovery is blocked: {controller.reason}") + pending.lock.parent.mkdir(parents=True, exist_ok=True) + with _exclusive_lock(pending.lock): + context: JsonDocument = { + "version": VERSION, + "generation": secrets.token_urlsafe(32), + "spec_sha256": _spec_sha256(spec), + "topology": "pxe-diskless", + "spec": spec, + "token": token, + "controller": resolution_manifest(controller), + } + bootstrap = render_pxe_vars(spec, _controller_policy(controller, True), str(pending.context)) + bootstrap += "\n".join( + [ + f"pxe_finalizer_handoff: {_yaml_quote(str(pending.handoff))}", + f"pxe_finalizer_generation: {_yaml_quote(context['generation'])}", + f"pxe_finalizer_spec_sha256: {_yaml_quote(context['spec_sha256'])}", + f"pxe_finalizer_script: {_yaml_quote(str(Path(__file__).with_name('gen_configs.py').resolve()))}", + "", + ] + ) + artifacts: list[Artifact] = [ + (pending.bootstrap_inventory, _render_bootstrap_inventory(spec), 0o600, True), + (pending.bootstrap_vars, bootstrap, 0o600, True), + (pending.context, json.dumps(context, sort_keys=True) + "\n", 0o600, True), + ] + if not force: + preflight_destinations(_generation_paths(pending), False) + publish_artifacts(artifacts, force, _generation_paths(pending)) + return pending + + +def publish_disabled_rootfs( + spec: JsonDocument, token: str, controller: FleetResolution, out_dir: Path, force: bool +) -> None: + pending = paths(out_dir) + if controller.status is FleetStatus.BLOCKED: + raise FinalizationError(f"GPU discovery is blocked: {controller.reason}") + pending.lock.parent.mkdir(parents=True, exist_ok=True) + with _exclusive_lock(pending.lock): + policy = _controller_policy(controller, False) + artifacts: list[Artifact] = [ + (pending.inventory, render_inventory(spec, token, controller), 0o600, True), + (pending.pxe_vars, render_pxe_vars(spec, policy), 0o600, True), + (pending.values, render_values(spec, controller), 0o644, False), + (pending.manifest, _manifest(controller, False, None), 0o644, False), + ] + if not force: + preflight_destinations(_generation_paths(pending), False) + publish_artifacts(artifacts, force, _generation_paths(pending)) + + +def finalize(out_dir: Path, context_path: Path, handoff_path: Path) -> None: + pending = paths(out_dir) + if context_path.resolve() != pending.context or handoff_path.resolve() != pending.handoff: + raise FinalizationError("PXE finalizer context and handoff paths must be the generated private paths") + pending.lock.parent.mkdir(parents=True, exist_ok=True) + with _exclusive_lock(pending.lock): + context = _read_document(pending.context, "PXE finalizer context") + handoff = _read_document(pending.handoff, "PXE finalizer handoff") + spec, controller, rootfs_gid = _validate(context, handoff) + resolution = _final_resolution(controller, rootfs_gid) + policy = _controller_policy(resolution, True) + artifacts: list[Artifact] = [ + (pending.inventory, render_inventory(spec, context["token"], resolution), 0o600, True), + (pending.pxe_vars, render_pxe_vars(spec, policy), 0o600, True), + (pending.values, render_values(spec, resolution), 0o644, False), + (pending.manifest, _manifest(resolution, True, rootfs_gid), 0o644, False), + ] + completion = _completion(context, handoff, _artifact_attestations(artifacts)) + if os.path.lexists(pending.completion): + if _read_document(pending.completion, "PXE finalizer completion") != completion: + raise FinalizationError("PXE finalizer completion does not match the supplied handoff") + _verify_canonical_artifacts(pending, completion["artifacts"]) + return + published: list[Artifact] = [ + *artifacts, + (pending.completion, json.dumps(completion, sort_keys=True) + "\n", 0o600, True), + ] + preflight_destinations([path for path, _, _, _ in published], False) + publish_artifacts(published, False) + + +def _controller_policy(resolution: FleetResolution, rootfs_enabled: bool) -> ResolvedGpuPolicy: + return ResolvedGpuPolicy( + host_gpu_enabled={host.target.name: host.status is HostStatus.GPU for host in resolution.hosts}, + render_gid=resolution.render_gid, + pxe_gpu_enabled=rootfs_enabled, + ) + + +def _manifest(resolution: FleetResolution, rootfs_enabled: bool, rootfs_gid: int | None) -> str: + base = resolution_manifest(resolution) + document = build_pxe_resolution_manifest( + version=base["version"], + status=base["status"], + render_gid=base["render_gid"], + hosts=base["hosts"], + gpu_access_enabled=rootfs_enabled, + pxe_render_gid=rootfs_gid, + ) + return json.dumps(document, indent=2, sort_keys=True) + "\n" + + +def _render_bootstrap_inventory(spec: JsonDocument) -> str: + server = spec["server"] + return "\n".join( + [ + "pxe_controller:", + " hosts:", + f" {server['name']}:", + f" ansible_host: {server['ip']}", + " vars:", + " ansible_port: 22", + " ansible_user: root", + "", + ] + ) + + +def _yaml_quote(value: str) -> str: + return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"' diff --git a/skills/deploy-aup-learning-cloud/scripts/pxe_finalization_support.py b/skills/deploy-aup-learning-cloud/scripts/pxe_finalization_support.py new file mode 100644 index 00000000..e94923f7 --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/pxe_finalization_support.py @@ -0,0 +1,257 @@ +#!/usr/bin/env python3 +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +"""Typed security and verification support for PXE finalization.""" + +from __future__ import annotations + +import fcntl +import hashlib +import json +import os +import stat +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Final, TypeAlias, TypedDict + +from config_common import DuplicateJsonKeyError, strict_json_loads +from gpu_access_resolution import FleetResolution, FleetStatus, HostResolution, HostStatus, InventoryTarget + +VERSION: Final = 1 +MAX_RENDER_GID: Final = 4_294_967_294 + +JsonScalar: TypeAlias = str | int | float | bool | None +JsonValue: TypeAlias = JsonScalar | list["JsonValue"] | dict[str, "JsonValue"] +JsonDocument: TypeAlias = dict[str, JsonValue] +Artifact: TypeAlias = tuple[Path, str, int, bool] + + +class ArtifactAttestation(TypedDict): + sha256: str + mode: int + owner_uid: int + + +ArtifactAttestations: TypeAlias = dict[str, ArtifactAttestation] + + +@dataclass(frozen=True, slots=True) +class FinalizationError(Exception): + reason: str + + def __str__(self) -> str: + return self.reason + + +@dataclass(frozen=True, slots=True) +class PxePaths: + bootstrap_inventory: Path + bootstrap_vars: Path + context: Path + handoff: Path + completion: Path + lock: Path + inventory: Path + pxe_vars: Path + values: Path + manifest: Path + + +def paths(out_dir: Path) -> PxePaths: + root = out_dir.resolve() + return PxePaths( + bootstrap_inventory=root / ".pxe-bootstrap.inventory.yml", + bootstrap_vars=root / ".pxe-bootstrap.vars.yml", + context=root / ".pxe-finalizer-context.json", + handoff=root / ".pxe-finalizer-handoff.json", + completion=root / ".pxe-finalizer-completion.json", + lock=root / ".pxe-finalizer.lock", + inventory=root / "inventory.yml", + pxe_vars=root / "pb-pxe-controller.vars.yml", + values=root / "values-basic-example.yaml", + manifest=root / "gpu-access-resolution.json", + ) + + +def spec_sha256(spec: JsonDocument) -> str: + encoded = json.dumps(spec, sort_keys=True, separators=(",", ":")).encode() + return hashlib.sha256(encoded).hexdigest() + + +def valid_gid(value: JsonValue) -> bool: + return type(value) is int and 1 <= value <= MAX_RENDER_GID + + +def read_document(path: Path, label: str) -> JsonDocument: + try: + descriptor = os.open(path, os.O_RDONLY | os.O_CLOEXEC | os.O_NOFOLLOW) + with os.fdopen(descriptor, encoding="utf-8") as source: + mode = os.fstat(source.fileno()).st_mode + if not stat.S_ISREG(mode): + raise FinalizationError(f"{label} must be a regular file") + document = strict_json_loads(source.read()) + except FinalizationError: + raise + except (DuplicateJsonKeyError, FileNotFoundError, OSError, ValueError, json.JSONDecodeError) as error: + raise FinalizationError(f"{label} cannot be read") from error + if type(document) is not dict: + raise FinalizationError(f"{label} must be a JSON object") + return document + + +def validate(context: JsonDocument, handoff: JsonDocument) -> tuple[JsonDocument, FleetResolution, int]: + required_context = {"version", "generation", "spec_sha256", "topology", "spec", "token", "controller"} + required_handoff = {"version", "generation", "spec_sha256", "topology", "pxe_gpu_access_enabled", "render_gid"} + if set(context) != required_context or set(handoff) != required_handoff: + raise FinalizationError("PXE finalizer context or handoff has an unexpected schema") + if type(context["version"]) is not int or type(handoff["version"]) is not int: + raise FinalizationError("PXE finalizer context or handoff version is invalid") + if context["version"] != VERSION or handoff["version"] != VERSION: + raise FinalizationError("PXE finalizer context or handoff version is unsupported") + if context["topology"] != "pxe-diskless" or handoff["topology"] != "pxe-diskless": + raise FinalizationError("PXE finalizer topology is invalid") + generation = context["generation"] + if type(generation) is not str or not generation or handoff["generation"] != generation: + raise FinalizationError("PXE finalizer generation does not match") + spec = context["spec"] + if type(spec) is not dict or spec_sha256(spec) != context["spec_sha256"]: + raise FinalizationError("PXE finalizer context spec does not match its digest") + if handoff["spec_sha256"] != context["spec_sha256"] or spec.get("topology") != "pxe-diskless": + raise FinalizationError("PXE finalizer handoff does not match its pending spec") + if "render_gid" in spec or "gpu_access" in spec: + raise FinalizationError("PXE finalizer context contains removed public GPU policy fields") + if type(context["token"]) is not str or not context["token"]: + raise FinalizationError("PXE finalizer context token is invalid") + pxe = spec.get("pxe") + if type(pxe) is not dict or pxe.get("diskless_agents_have_amd_gpus") is not True: + raise FinalizationError("PXE finalizer context is not for GPU-enabled diskless agents") + rootfs_gid = handoff["render_gid"] + if handoff["pxe_gpu_access_enabled"] is not True or not valid_gid(rootfs_gid): + raise FinalizationError("PXE finalizer handoff has no valid resolved rootfs GID") + controller = controller_resolution(spec, context["controller"]) + if controller.render_gid is not None and controller.render_gid != rootfs_gid: + raise FinalizationError("PXE rootfs render GID disagrees with the GPU-enabled controller") + return spec, controller, rootfs_gid + + +def controller_resolution(spec: JsonDocument, raw: JsonValue) -> FleetResolution: + if type(raw) is not dict or set(raw) != {"version", "status", "render_gid", "hosts"}: + raise FinalizationError("PXE finalizer context controller evidence is invalid") + server = spec.get("server") + name = server.get("name") if type(server) is dict else None + hosts = raw["hosts"] + if type(name) is not str or type(hosts) is not dict or set(hosts) != {name} or type(hosts[name]) is not bool: + raise FinalizationError("PXE finalizer context controller host is invalid") + enabled = hosts[name] + gid = raw["render_gid"] + if enabled and not valid_gid(gid): + raise FinalizationError("PXE finalizer context controller GID is invalid") + if not enabled and gid is not None: + raise FinalizationError("CPU-only PXE controller must not publish a render GID") + status = HostStatus.GPU if enabled else HostStatus.CPU + fleet_status = FleetStatus.GPU_RESOLVED if enabled else FleetStatus.CPU_ONLY + if type(raw["version"]) is not int or raw["version"] != VERSION or raw["status"] != fleet_status.value: + raise FinalizationError("PXE finalizer context controller status is invalid") + host = HostResolution(target=target(name), status=status, render_gid=gid, reason=None) + return FleetResolution(fleet_status, (host,), gid, None) + + +def target(name: str) -> InventoryTarget: + return InventoryTarget(name=name) + + +def final_resolution(controller: FleetResolution, rootfs_gid: int) -> FleetResolution: + return FleetResolution(FleetStatus.GPU_RESOLVED, controller.hosts, rootfs_gid, None) + + +def generation_paths(pending: PxePaths) -> tuple[Path, ...]: + return ( + pending.bootstrap_inventory, + pending.bootstrap_vars, + pending.context, + pending.handoff, + pending.completion, + pending.inventory, + pending.pxe_vars, + pending.values, + pending.manifest, + ) + + +def artifact_attestations(artifacts: list[Artifact]) -> ArtifactAttestations: + return { + path.name: {"sha256": hashlib.sha256(content.encode()).hexdigest(), "mode": mode, "owner_uid": os.geteuid()} + for path, content, mode, _ in artifacts + } + + +def verify_canonical_artifacts(pending: PxePaths, expected: JsonValue) -> None: + canonical = (pending.inventory, pending.pxe_vars, pending.values, pending.manifest) + if type(expected) is not dict or set(expected) != {path.name for path in canonical}: + raise FinalizationError("PXE finalizer completion artifacts are invalid") + for path in canonical: + attestation = expected[path.name] + if ( + type(attestation) is not dict + or set(attestation) != {"sha256", "mode", "owner_uid"} + or type(attestation["sha256"]) is not str + or type(attestation["mode"]) is not int + or type(attestation["owner_uid"]) is not int + or read_artifact_attestation(path) != attestation + ): + raise FinalizationError(f"PXE finalizer canonical artifact is missing or corrupted: {path.name}") + + +def read_artifact_attestation(path: Path) -> ArtifactAttestation: + try: + descriptor = os.open(path, os.O_RDONLY | os.O_CLOEXEC | os.O_NOFOLLOW) + with os.fdopen(descriptor, "rb") as source: + artifact_stat = os.fstat(source.fileno()) + if not stat.S_ISREG(artifact_stat.st_mode): + raise FinalizationError("PXE finalizer canonical artifact must be a regular file") + digest = hashlib.sha256() + while chunk := source.read(65_536): + digest.update(chunk) + return { + "sha256": digest.hexdigest(), + "mode": stat.S_IMODE(artifact_stat.st_mode), + "owner_uid": artifact_stat.st_uid, + } + except FinalizationError: + raise + except (FileNotFoundError, OSError) as error: + raise FinalizationError("PXE finalizer canonical artifact cannot be read") from error + + +def completion(context: JsonDocument, handoff: JsonDocument, artifacts: ArtifactAttestations) -> JsonDocument: + return { + **{ + key: handoff[key] + for key in ("version", "generation", "spec_sha256", "topology", "pxe_gpu_access_enabled", "render_gid") + }, + "artifacts": artifacts, + } + + +@contextmanager +def exclusive_lock(path: Path) -> Iterator[None]: + descriptor = -1 + locked = False + try: + descriptor = os.open(path, os.O_RDWR | os.O_CREAT | os.O_CLOEXEC | os.O_NOFOLLOW, 0o600) + lock_stat = os.fstat(descriptor) + if not stat.S_ISREG(lock_stat.st_mode): + raise FinalizationError("PXE finalizer lock must be a regular file") + if lock_stat.st_uid != os.geteuid() or stat.S_IMODE(lock_stat.st_mode) != 0o600: + raise FinalizationError("PXE finalizer lock has unsafe owner or mode") + fcntl.flock(descriptor, fcntl.LOCK_EX) + locked = True + yield + except OSError as error: + raise FinalizationError("PXE finalizer lock cannot be opened") from error + finally: + if locked: + fcntl.flock(descriptor, fcntl.LOCK_UN) + if descriptor >= 0: + os.close(descriptor) diff --git a/tests/skills/test_pxe_finalization.py b/tests/skills/test_pxe_finalization.py new file mode 100644 index 00000000..effccb02 --- /dev/null +++ b/tests/skills/test_pxe_finalization.py @@ -0,0 +1,516 @@ +from __future__ import annotations + +import json +import os +import subprocess +import sys +from dataclasses import FrozenInstanceError +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +GEN_CONFIGS = ROOT / "skills" / "deploy-aup-learning-cloud" / "scripts" / "gen_configs.py" +PXE_PLAYBOOK = ROOT / "deploy" / "ansible" / "playbooks" / "pb-pxe-controller.yml" + + +def pxe_spec(gpu_agents: bool) -> dict: + return { + "topology": "pxe-diskless", + "k3s_version": "v1.32.3+k3s1", + "server": {"name": "controller", "ip": "192.168.1.10"}, + "agents": [{"name": "diskless-agent", "ip": "192.168.1.11"}], + "network": {"interface": "enp1s0", "subnet": "192.168.1.0/24"}, + "pxe": { + "authorized_keys": ["ssh-ed25519 AAAA test@example"], + "rootfs_password": "do-not-print-this-secret", + "diskless_agents_have_amd_gpus": gpu_agents, + }, + } + + +def write_json(path: Path, document: dict) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(document), encoding="utf-8") + return path + + +def write_fake_ansible(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, *, controller_gpu: bool = False) -> None: + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + fake_ansible = fake_bin / "ansible-playbook" + bdf = "0000:03:00.0" if controller_gpu else "" + fake_ansible.write_text( + f"""#!/usr/bin/env python3 +import json +import sys +from pathlib import Path + +output = next(value.split('=', 1)[1] for value in sys.argv if value.startswith('gpu_access_discovery_output_path=')) +Path(output).write_text(json.dumps({{ + 'version': 2, + 'hosts': [{{ + 'host': 'controller', 'reachable': True, + 'lspci': {{'rc': 0, 'stdout': {bdf!r}}}, + 'sysfs': {{'rc': 0, 'stdout': {bdf!r}}}, + 'render_group': {{'rc': 0, 'stdout': 'render:x:993\\n'}}, + 'groups': {{'rc': 0, 'stdout': 'render:x:993\\n'}}, + 'state': {{'stat_success': True, 'content_success': True, 'exists': False, 'regular': False, 'symlink': False, 'content': ''}}, + 'rule': {{'stat_success': True, 'content_success': True, 'exists': False, 'regular': False, 'symlink': False, 'content': ''}}, + 'legacy_rules': {{key: {{'stat_success': True, 'content_success': True, 'exists': False, 'regular': False, 'symlink': False, 'content': ''}} for key in ('kfd', 'amdgpu', 'rocm_devices')}}, + }}], +}}), encoding='utf-8') +""", + encoding="utf-8", + ) + fake_ansible.chmod(0o755) + monkeypatch.setenv("PATH", f"{fake_bin}:{os.environ['PATH']}") + + +def run_generator(*arguments: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(GEN_CONFIGS), *arguments], + capture_output=True, + check=False, + text=True, + timeout=30, + ) + + +def load_finalizer_module(): + scripts = GEN_CONFIGS.parent + sys.path.insert(0, str(scripts)) + try: + spec = spec_from_file_location("test_pxe_finalizer", scripts / "pxe_finalization.py") + assert spec is not None and spec.loader is not None + module = module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + finally: + sys.path.pop(0) + + +def pending_handoff(out_dir: Path, *, render_gid: int = 995) -> tuple[Path, Path]: + context_path = out_dir / ".pxe-finalizer-context.json" + context = json.loads(context_path.read_text(encoding="utf-8")) + handoff_path = out_dir / ".pxe-finalizer-handoff.json" + write_json( + handoff_path, + { + "version": 1, + "generation": context["generation"], + "spec_sha256": context["spec_sha256"], + "topology": "pxe-diskless", + "pxe_gpu_access_enabled": True, + "render_gid": render_gid, + }, + ) + return context_path, handoff_path + + +def canonical_artifacts(out_dir: Path) -> tuple[Path, ...]: + return ( + out_dir / "inventory.yml", + out_dir / "pb-pxe-controller.vars.yml", + out_dir / "values-basic-example.yaml", + out_dir / "gpu-access-resolution.json", + out_dir / ".pxe-finalizer-completion.json", + ) + + +def cpu_controller(finalizer): + return finalizer.FleetResolution( + finalizer.FleetStatus.CPU_ONLY, + (finalizer.HostResolution(finalizer._target("controller"), finalizer.HostStatus.CPU, None, None),), + None, + None, + ) + + +def test_pxe_finalizer_preserves_moved_imports_as_immutable_support_types(tmp_path: Path) -> None: + finalizer = load_finalizer_module() + support = sys.modules["pxe_finalization_support"] + + assert finalizer.FinalizationError is support.FinalizationError + assert finalizer.PxePaths is support.PxePaths + assert finalizer.paths is support.paths + assert finalizer.VERSION == support.VERSION == 1 + assert finalizer.MAX_RENDER_GID == support.MAX_RENDER_GID == 4_294_967_294 + assert finalizer._read_document is support.read_document + assert finalizer._generation_paths is support.generation_paths + assert finalizer._artifact_attestations is support.artifact_attestations + assert finalizer._completion is support.completion + assert finalizer._verify_canonical_artifacts is support.verify_canonical_artifacts + assert finalizer._exclusive_lock is support.exclusive_lock + + error = finalizer.FinalizationError("immutable") + pending = finalizer.paths(tmp_path) + with pytest.raises(FrozenInstanceError): + error.reason = "changed" + with pytest.raises(FrozenInstanceError): + pending.context = tmp_path / "changed.json" + + +def test_pxe_gpu_agents_stage_only_private_bootstrap_artifacts(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + write_fake_ansible(tmp_path, monkeypatch) + spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) + out_dir = tmp_path / "generated" + + result = run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)) + + assert result.returncode == 0, result.stderr + assert not (out_dir / "inventory.yml").exists() + assert not (out_dir / "values-basic-example.yaml").exists() + assert not (out_dir / "gpu-access-resolution.json").exists() + assert "pxe_controller:" in (out_dir / ".pxe-bootstrap.inventory.yml").read_text(encoding="utf-8") + bootstrap = (out_dir / ".pxe-bootstrap.vars.yml").read_text(encoding="utf-8") + assert "pxe_gpu_access_enabled: true" in bootstrap + assert "pxe_finalizer_context:" in bootstrap + assert (out_dir / ".pxe-finalizer-context.json").stat().st_mode & 0o777 == 0o600 + + +def test_pxe_cpu_agents_publish_a_disabled_rootfs_policy(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + write_fake_ansible(tmp_path, monkeypatch) + spec_path = write_json(tmp_path / "spec.json", pxe_spec(False)) + out_dir = tmp_path / "generated" + + result = run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)) + + assert result.returncode == 0, result.stderr + assert "pxe_gpu_access_enabled: false" in (out_dir / "pb-pxe-controller.vars.yml").read_text(encoding="utf-8") + assert "auplc_render_gid: null" in (out_dir / "inventory.yml").read_text(encoding="utf-8") + manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) + assert manifest["pxe_rootfs"] == {"gpu_access_enabled": False, "render_gid": None} + + +def test_pxe_disabled_rootfs_force_replaces_private_generation_state_under_the_generation_lock(tmp_path: Path) -> None: + finalizer = load_finalizer_module() + out_dir = tmp_path / "generated" + pending = finalizer.paths(out_dir) + for path in ( + pending.bootstrap_inventory, + pending.bootstrap_vars, + pending.context, + pending.handoff, + pending.completion, + ): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("stale\n", encoding="utf-8") + + finalizer.publish_disabled_rootfs(pxe_spec(False), "token", cpu_controller(finalizer), out_dir, True) + + assert all( + not path.exists() + for path in ( + pending.bootstrap_inventory, + pending.bootstrap_vars, + pending.context, + pending.handoff, + pending.completion, + ) + ) + assert all(path.exists() for path in canonical_artifacts(out_dir)[:-1]) + + +def test_pxe_disabled_rootfs_force_restores_private_and_canonical_generation_when_publication_fails( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + finalizer = load_finalizer_module() + artifact_store = sys.modules["artifact_store"] + out_dir = tmp_path / "generated" + pending = finalizer.paths(out_dir) + finalizer.publish_disabled_rootfs(pxe_spec(False), "old-token", cpu_controller(finalizer), out_dir, False) + for path in ( + pending.bootstrap_inventory, + pending.bootstrap_vars, + pending.context, + pending.handoff, + pending.completion, + ): + path.write_text(f"old {path.name}\n", encoding="utf-8") + tracked = ( + *canonical_artifacts(out_dir)[:-1], + pending.bootstrap_inventory, + pending.bootstrap_vars, + pending.context, + pending.handoff, + pending.completion, + ) + before = {path.name: path.read_bytes() for path in tracked} + original_replace = artifact_store.os.replace + + def fail_values_replace(source, destination): + if Path(destination) == pending.values and ".backup." not in str(source): + raise OSError("injected disabled-rootfs publication failure") + return original_replace(source, destination) + + monkeypatch.setattr(artifact_store.os, "replace", fail_values_replace) + with pytest.raises(SystemExit): + finalizer.publish_disabled_rootfs(pxe_spec(False), "new-token", cpu_controller(finalizer), out_dir, True) + + assert {path.name: path.read_bytes() for path in tracked} == before + + +def test_pxe_finalizer_publishes_resolved_policy_idempotently_without_secret_output( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_fake_ansible(tmp_path, monkeypatch) + spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) + out_dir = tmp_path / "generated" + pending = run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)) + context, handoff = pending_handoff(out_dir) + + first = run_generator( + "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) + ) + second = run_generator( + "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) + ) + + assert pending.returncode == 0, pending.stderr + assert first.returncode == 0, first.stderr + assert second.returncode == 0, second.stderr + assert "do-not-print-this-secret" not in first.stdout + first.stderr + second.stdout + second.stderr + assert "auplc_render_gid: 995" in (out_dir / "inventory.yml").read_text(encoding="utf-8") + assert "auplc_gpu_access_enabled: false" in (out_dir / "inventory.yml").read_text(encoding="utf-8") + assert "renderGid: 995" in (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") + manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) + assert manifest["pxe_rootfs"] == {"gpu_access_enabled": True, "render_gid": 995} + completion = json.loads((out_dir / ".pxe-finalizer-completion.json").read_text(encoding="utf-8")) + assert completion["artifacts"]["inventory.yml"]["mode"] == 0o600 + assert completion["artifacts"]["inventory.yml"]["owner_uid"] == os.geteuid() + + +def test_pxe_finalizer_retry_rejects_canonical_mode_drift(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + write_fake_ansible(tmp_path, monkeypatch) + spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) + out_dir = tmp_path / "generated" + assert run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)).returncode == 0 + context, handoff = pending_handoff(out_dir) + assert ( + run_generator( + "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) + ).returncode + == 0 + ) + (out_dir / "inventory.yml").chmod(0o644) + + result = run_generator( + "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) + ) + + assert result.returncode == 1 + + +def test_pxe_pending_generation_rejects_existing_private_or_canonical_state_without_force( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_fake_ansible(tmp_path, monkeypatch) + spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) + out_dir = tmp_path / "generated" + assert run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)).returncode == 0 + + result = run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)) + + assert result.returncode == 1 + assert "refusing to overwrite" in result.stderr + + +def test_pxe_forced_pending_generation_hides_prior_public_and_private_generation( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_fake_ansible(tmp_path, monkeypatch) + spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) + out_dir = tmp_path / "generated" + assert run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)).returncode == 0 + context, handoff = pending_handoff(out_dir) + assert ( + run_generator( + "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) + ).returncode + == 0 + ) + old_generation = json.loads(context.read_text(encoding="utf-8"))["generation"] + + result = run_generator("--spec", str(spec_path), "--out-dir", str(out_dir), "--force") + + assert result.returncode == 0, result.stderr + assert json.loads(context.read_text(encoding="utf-8"))["generation"] != old_generation + assert not handoff.exists() + assert all(not path.exists() for path in canonical_artifacts(out_dir)) + + +def test_pxe_forced_pending_generation_restores_prior_generation_if_staging_fails( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_fake_ansible(tmp_path, monkeypatch) + spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) + out_dir = tmp_path / "generated" + assert run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)).returncode == 0 + context, handoff = pending_handoff(out_dir) + assert ( + run_generator( + "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) + ).returncode + == 0 + ) + previous = {path.name: path.read_bytes() for path in (*canonical_artifacts(out_dir), context, handoff)} + finalizer = load_finalizer_module() + artifact_store = sys.modules["artifact_store"] + original_replace = artifact_store.os.replace + + def fail_new_bootstrap(source, destination): + if Path(destination) == out_dir / ".pxe-bootstrap.inventory.yml" and ".backup." not in str(source): + raise OSError("injected staging failure") + return original_replace(source, destination) + + monkeypatch.setattr(artifact_store.os, "replace", fail_new_bootstrap) + controller = finalizer._controller_resolution( + pxe_spec(True), json.loads(context.read_text(encoding="utf-8"))["controller"] + ) + with pytest.raises(SystemExit): + finalizer.stage_pending(pxe_spec(True), "replacement-token", controller, out_dir, True) + + assert {path.name: path.read_bytes() for path in (*canonical_artifacts(out_dir), context, handoff)} == previous + + +@pytest.mark.parametrize("document_name", (".pxe-finalizer-context.json", ".pxe-finalizer-handoff.json")) +def test_pxe_finalizer_rejects_duplicate_keys_in_private_documents( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, document_name: str +) -> None: + write_fake_ansible(tmp_path, monkeypatch) + spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) + out_dir = tmp_path / "generated" + assert run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)).returncode == 0 + context, handoff = pending_handoff(out_dir) + document_path = out_dir / document_name + document_path.write_text( + '{"generation":"duplicate",' + document_path.read_text(encoding="utf-8")[1:], encoding="utf-8" + ) + + result = run_generator( + "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) + ) + + assert result.returncode == 1 + assert all(not path.exists() for path in canonical_artifacts(out_dir)) + + +@pytest.mark.parametrize("mutation", ("missing", "tampered")) +def test_pxe_finalizer_retry_rejects_missing_or_tampered_canonical_artifacts( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, mutation: str +) -> None: + write_fake_ansible(tmp_path, monkeypatch) + spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) + out_dir = tmp_path / "generated" + assert run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)).returncode == 0 + context, handoff = pending_handoff(out_dir) + assert ( + run_generator( + "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) + ).returncode + == 0 + ) + inventory = out_dir / "inventory.yml" + if mutation == "missing": + inventory.unlink() + else: + inventory.write_text("tampered\n", encoding="utf-8") + + result = run_generator( + "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) + ) + + assert result.returncode == 1 + + +def test_pxe_finalizer_rejects_symlink_lock_without_touching_its_target( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_fake_ansible(tmp_path, monkeypatch) + spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) + out_dir = tmp_path / "generated" + assert run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)).returncode == 0 + context, handoff = pending_handoff(out_dir) + target = tmp_path / "lock-target" + target.write_text("unchanged\n", encoding="utf-8") + target.chmod(0o644) + lock = out_dir / ".pxe-finalizer.lock" + lock.unlink() + lock.symlink_to(target) + + result = run_generator( + "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) + ) + + assert result.returncode == 1 + assert target.read_text(encoding="utf-8") == "unchanged\n" + assert target.stat().st_mode & 0o777 == 0o644 + + +@pytest.mark.parametrize( + ("field", "value"), + [("generation", "stale"), ("topology", "ssh-preinstalled"), ("render_gid", None), ("version", True)], +) +def test_pxe_finalizer_rejects_invalid_handoffs_without_publishing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, field: str, value: str | int | None +) -> None: + write_fake_ansible(tmp_path, monkeypatch) + spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) + out_dir = tmp_path / "generated" + assert run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)).returncode == 0 + context, handoff = pending_handoff(out_dir) + document = json.loads(handoff.read_text(encoding="utf-8")) + document[field] = value + write_json(handoff, document) + + result = run_generator( + "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) + ) + + assert result.returncode == 1 + assert not (out_dir / "inventory.yml").exists() + assert not (out_dir / "values-basic-example.yaml").exists() + assert not (out_dir / "gpu-access-resolution.json").exists() + + +def test_pxe_finalizer_rolls_back_if_late_canonical_publication_fails( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_fake_ansible(tmp_path, monkeypatch) + spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) + out_dir = tmp_path / "generated" + assert run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)).returncode == 0 + context, handoff = pending_handoff(out_dir) + finalizer = load_finalizer_module() + artifact_store = sys.modules["artifact_store"] + original_link = artifact_store.os.link + + def fail_values_link(source, destination): + if Path(destination).name == "values-basic-example.yaml": + raise OSError("injected publication failure") + return original_link(source, destination) + + monkeypatch.setattr(artifact_store.os, "link", fail_values_link) + with pytest.raises(SystemExit): + finalizer.finalize(out_dir, context, handoff) + + assert not (out_dir / "inventory.yml").exists() + assert not (out_dir / "pb-pxe-controller.vars.yml").exists() + assert not (out_dir / "values-basic-example.yaml").exists() + assert not (out_dir / "gpu-access-resolution.json").exists() + assert not (out_dir / ".pxe-finalizer-completion.json").exists() + + +def test_pxe_playbook_writes_and_finalizes_private_rootfs_handoff_locally() -> None: + playbook = PXE_PLAYBOOK.read_text(encoding="utf-8") + + assert "pxe_finalizer_handoff" in playbook + assert "pxe_finalizer_context" in playbook + assert "--finalize-pxe" in playbook + assert "delegate_to: localhost" in playbook + assert "run_once: true" in playbook + assert "become: false" in playbook + assert "argv:" in playbook From 27f36f1572a08205933565307515165efb1ac117 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:18 +0800 Subject: [PATCH 17/65] feat(deploy): generate and validate GPU artifacts --- .../scripts/gen_configs.py | 381 ++-------- .../scripts/gpu_resolution_parsing.py | 246 +++++++ .../scripts/gpu_resolution_validation.py | 125 ++++ .../scripts/validate.py | 198 ++--- .../scripts/values_resolution_parsing.py | 130 ++++ .../skills/test_config_generation_security.py | 95 +++ tests/skills/test_deploy_scripts.py | 692 +++++++++++++++++- 7 files changed, 1379 insertions(+), 488 deletions(-) create mode 100644 skills/deploy-aup-learning-cloud/scripts/gpu_resolution_parsing.py create mode 100644 skills/deploy-aup-learning-cloud/scripts/gpu_resolution_validation.py create mode 100644 skills/deploy-aup-learning-cloud/scripts/values_resolution_parsing.py create mode 100644 tests/skills/test_config_generation_security.py diff --git a/skills/deploy-aup-learning-cloud/scripts/gen_configs.py b/skills/deploy-aup-learning-cloud/scripts/gen_configs.py index e0d91ee8..3a3bdb9f 100755 --- a/skills/deploy-aup-learning-cloud/scripts/gen_configs.py +++ b/skills/deploy-aup-learning-cloud/scripts/gen_configs.py @@ -2,8 +2,9 @@ # Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. """Generate AUP Learning Cloud deploy artifacts from a small cluster-spec. -Given a JSON cluster-spec (see ``--print-schema``), write the three files the -deploy skill needs, keeping them mutually consistent: +Given a JSON cluster-spec (see ``--print-schema``), discover the managed hosts' +GPU policy. SSH and PXE without GPU-enabled diskless agents immediately write +mutually consistent canonical deployment artifacts: 1. ``inventory.yml`` -- Ansible inventory (server + token + k3s_version; agents listed for the @@ -11,17 +12,26 @@ 2. ``pb-pxe-controller.vars.yml`` -- PXE topology only: extra vars passed to pb-pxe-controller.yml with ``-e @``. - 3. ``values-basic-example.yaml`` -- Helm overlay: accelerator nodeSelectors, - storage class, proxy NodePort, authMode. + 3. ``values-basic-example.yaml`` -- Helm overlay: resolved render GID, + storage, proxy, and authentication. + 4. ``gpu-access-resolution.json`` -- Machine-readable resolved host policy. + +GPU-enabled PXE instead writes private ``.pxe-bootstrap.inventory.yml``, +``.pxe-bootstrap.vars.yml``, and ``.pxe-finalizer-context.json`` files while +canonical artifacts remain absent. The controller playbook publishes the +canonical artifacts only after it resolves the rootfs GID and succeeds. Design choices (deliberate): * stdlib only (json, argparse, secrets, base64, pathlib). No PyYAML, so this runs on a bare operator machine. YAML is emitted from templates, not a serialiser -- the output is small, fixed-shape, and carries the copyright header. - * The k3s token is generated locally with ``secrets`` (CSPRNG) and written - ONLY into inventory.yml. It is never printed to stdout/stderr. Pass - ``--token-file`` to reuse an existing token instead of minting one. + * The k3s token is generated locally with ``secrets`` (CSPRNG). Immediate + canonical output writes it only into ``inventory.yml``. Pending GPU-enabled + PXE stores it only in private ``.pxe-finalizer-context.json`` until the + controller succeeds and finalization writes ``inventory.yml``. It is never + printed to stdout/stderr. Pass ``--token-file`` to reuse an existing token + instead of minting one. * ``pxe_k3s_version`` is forced equal to ``k3s_version`` so agents can never be newer than the server (k3s refuses that). * Existing files are not overwritten unless ``--force`` is given. @@ -39,58 +49,22 @@ import argparse import base64 import json -import os import secrets -import shutil import sys -import tempfile -from contextlib import suppress from pathlib import Path -HEADER_HASH = ( - "# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved.\n" - "# Generated by auplc-skills gen_configs.py -- review before use.\n" +from artifact_store import preflight_destinations, publish_artifacts +from config_common import DuplicateJsonKeyError, strict_json_loads +from config_generation import ( + SCHEMA, + die, + render_inventory, + render_values, + validate_spec, + validate_yaml_scalar, ) - -# Default GPU product-name labels, keyed by the accelerator key used in -# runtime/values.yaml (custom.accelerators.). Verified against the chart's -# values.yaml; override per fleet via spec["accelerators"][key]["product_name"]. -DEFAULT_ACCEL_LABELS = { - "phx": "AMD_Radeon_780M_Graphics", - "strix": "AMD_Radeon_890M_Graphics", - "strix-halo": "AMD_Radeon_8060S_Graphics", - "9070xt": "AMD_Radeon_RX_9070_XT", - "r9700": "AMD_Radeon_AI_PRO_R9700", - "9600gre": "AMD_Radeon_RX_9600_GRE", -} - -SCHEMA = { - "topology": "pxe-diskless | ssh-preinstalled", - "k3s_version": "v1.32.3+k3s1", - "server": {"name": "aipc1", "ip": "192.168.0.140"}, - "agents": [{"name": "aipc2", "ip": "192.168.0.141"}], - "network": { - "interface": "enp1s0", - "subnet": "192.168.0.0/24", - "gateway": "192.168.0.1", - "dns_servers": "8.8.8.8,8.8.4.4", - }, - "pxe": { - "authorized_keys": ["ssh-ed25519 AAAA... you@host"], - "rootfs_password": "", - "web_port": 8080, - }, - "accelerators": {"strix-halo": {"product_name": "AMD_Radeon_8060S_Graphics"}}, - "storage": {"class": "nfs-client"}, - "proxy": {"node_port": 30890}, - "auth_mode": "auto-login", - "images": {"cpu": "ghcr.io/amdresearch/auplc-default:latest", "gpu": "ghcr.io/amdresearch/auplc-base:latest"}, -} - - -def die(msg: str, code: int = 1) -> None: - print(f"gen_configs: {msg}", file=sys.stderr) - raise SystemExit(code) +from gpu_artifact_generation import DiscoveryFailure, canonical_paths, discover_gpu_policy, manifest_content +from pxe_finalization import FinalizationError, finalize, publish_disabled_rootfs, stage_pending def gen_token() -> str: @@ -98,250 +72,6 @@ def gen_token() -> str: return base64.b64encode(secrets.token_bytes(64)).decode("ascii") -def require(spec: dict, path: str): - cur = spec - for part in path.split("."): - if not isinstance(cur, dict) or part not in cur or cur[part] in (None, "", []): - die(f"spec is missing required field '{path}'") - cur = cur[part] - return cur - - -def yaml_quote(s: str) -> str: - return '"' + str(s).replace("\\", "\\\\").replace('"', '\\"') + '"' - - -def validate_accelerators(spec: dict) -> None: - if "accelerators" not in spec: - return - accelerators = spec["accelerators"] - if not isinstance(accelerators, dict): - die("spec.accelerators must be a mapping") - unsupported = sorted(set(accelerators) - set(DEFAULT_ACCEL_LABELS)) - if len(unsupported) == 1: - die(f"unsupported accelerator key '{unsupported[0]}'") - if unsupported: - die(f"unsupported accelerator keys: {', '.join(unsupported)}") - for key, config in accelerators.items(): - if not isinstance(config, dict): - die(f"accelerators.{key} must be a mapping") - - -def validate_config_shapes(spec: dict) -> None: - if not isinstance(spec, dict): - die("spec must be a mapping") - validate_accelerators(spec) - for key in ("network", "pxe", "storage", "proxy", "images"): - if key in spec and not isinstance(spec[key], dict): - die(f"spec.{key} must be a mapping") - - -def render_inventory(spec: dict, token: str) -> str: - topo = spec["topology"] - server = spec["server"] - k3s_version = spec["k3s_version"] - lines = [ - HEADER_HASH, - "k3s_cluster:", - " children:", - " server:", - " hosts:", - f" {server['name']}:", - f" ansible_host: {server['ip']}", - " agent:", - ] - if topo == "ssh-preinstalled" and spec.get("agents"): - lines.append(" hosts:") - for a in spec["agents"]: - lines.append(f" {a['name']}:") - lines.append(f" ansible_host: {a['ip']}") - else: - # PXE diskless agents auto-join by netboot; do NOT list them here. - lines.append(" hosts: {}") - lines += [ - " vars:", - " ansible_port: 22", - " ansible_user: root", - f" k3s_version: {k3s_version}", - f" token: {yaml_quote(token)}", - " api_endpoint: \"{{ hostvars[groups['server'][0]]['ansible_host'] | default(groups['server'][0]) }}\"", - ] - if topo == "pxe-diskless": - lines += [ - "", - "pxe_controller:", - " hosts:", - f" {server['name']}:", - f" ansible_host: {server['ip']}", - " vars:", - " ansible_port: 22", - " ansible_user: root", - ] - return "\n".join(lines) + "\n" - - -def render_pxe_vars(spec: dict) -> str: - net = require(spec, "network") - pxe = spec.get("pxe", {}) - keys = pxe.get("authorized_keys", []) - if not keys: - die("pxe.authorized_keys must contain at least one SSH public key") - server_ip = spec["server"]["ip"] - k3s_version = spec["k3s_version"] - lines = [ - HEADER_HASH, - "# Pass this file to pb-pxe-controller.yml with", - "# ansible-playbook ... -e @", - "# pxe_k3s_version is pinned to k3s_version so agents are never newer", - "# than the server.", - "pxe_rootfs_force_rebuild: true # first build only; set false afterwards", - f"pxe_network_interface: {yaml_quote(net['interface'])}", - f"pxe_subnet: {yaml_quote(net['subnet'])}", - f"pxe_gateway: {yaml_quote(net.get('gateway', ''))}", - f"pxe_dns_servers: {yaml_quote(net.get('dns_servers', '8.8.8.8,8.8.4.4'))}", - f"pxe_controller_ip: {yaml_quote(server_ip)}", - "pxe_k3s_server_ips:", - f" - {yaml_quote(server_ip)}", - f"pxe_k3s_version: {yaml_quote(k3s_version)}", - f"pxe_web_port: {int(pxe.get('web_port', 8080))}", - f"pxe_rootfs_password: {yaml_quote(pxe.get('rootfs_password', ''))}", - "pxe_rootfs_authorized_keys:", - ] - for k in keys: - lines.append(f" - {yaml_quote(k)}") - return "\n".join(lines) + "\n" - - -def render_values(spec: dict) -> str: - accel = spec.get("accelerators") or {} - storage_class = (spec.get("storage") or {}).get("class", "nfs-client") - node_port = (spec.get("proxy") or {}).get("node_port", 30890) - auth_mode = spec.get("auth_mode", "auto-login") - images = spec.get("images") or {} - - lines = [ - "# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved.", - "# Helm overlay generated by auplc-skills gen_configs.py.", - "# Layer this on top of runtime/values.yaml:", - "# helm upgrade --install jupyterhub ./runtime/chart -n jupyterhub \\", - "# --create-namespace -f runtime/values.yaml -f ", - "custom:", - f" authMode: {yaml_quote(auth_mode)}", - ] - if accel: - lines.append(" accelerators:") - for key, cfg in accel.items(): - product = (cfg or {}).get("product_name") or DEFAULT_ACCEL_LABELS.get(key) - if not product: - die( - f"accelerator '{key}' has no product_name and no known default; " - "add accelerators..product_name from `kubectl describe node`" - ) - lines += [ - f" {key}:", - " nodeSelector:", - f" amd.com/gpu.product-name: {yaml_quote(product)}", - ] - if accel or images: - lines.append(" resources:") - if accel: - lines += [" metadata:", " gpu:", " acceleratorKeys:"] - lines.extend(f" - {yaml_quote(key)}" for key in accel) - if images: - lines.append(" images:") - for k, v in images.items(): - lines.append(f" {k}: {yaml_quote(v)}") - lines += [ - "hub:", - " db:", - " pvc:", - f" storageClassName: {yaml_quote(storage_class)}", - "singleuser:", - " storage:", - " dynamic:", - f" storageClass: {yaml_quote(storage_class)}", - "proxy:", - " service:", - " type: NodePort", - " nodePorts:", - f" http: {int(node_port)}", - ] - return "\n".join(lines) + "\n" - - -def preflight_destinations(paths: list[Path], force: bool) -> None: - if force: - return - for path in paths: - if os.path.lexists(path): - die(f"refusing to overwrite existing {path} (use --force)", 1) - - -def stage_file(path: Path, content: str, mode: int) -> Path: - path.parent.mkdir(parents=True, exist_ok=True) - fd, staged_path = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) - try: - os.fchmod(fd, mode) - with os.fdopen(fd, "w", encoding="utf-8") as staged_file: - staged_file.write(content) - staged_file.flush() - os.fsync(staged_file.fileno()) - except OSError: - with suppress(OSError): - os.close(fd) - Path(staged_path).unlink(missing_ok=True) - raise - return Path(staged_path) - - -def remove_destination(path: Path) -> None: - if path.is_dir() and not path.is_symlink(): - shutil.rmtree(path) - else: - path.unlink(missing_ok=True) - - -def backup_destination(path: Path) -> tuple[Path, Path]: - backup_dir = Path(tempfile.mkdtemp(prefix=f".{path.name}.backup.", dir=path.parent)) - backup_path = backup_dir / path.name - os.replace(path, backup_path) - return backup_dir, backup_path - - -def publish_artifacts(artifacts: list[tuple[Path, str, int, bool]], force: bool) -> None: - staged: list[tuple[Path, Path, bool]] = [] - published: list[Path] = [] - backups: list[tuple[Path, Path, Path]] = [] - try: - for path, content, mode, secret in artifacts: - staged.append((path, stage_file(path, content, mode), secret)) - for path, staged_path, secret in staged: - if force and os.path.lexists(path): - backup_dir, backup_path = backup_destination(path) - backups.append((path, backup_dir, backup_path)) - if force: - os.replace(staged_path, path) - else: - os.link(staged_path, path) - os.unlink(staged_path) - published.append(path) - print(f"wrote {path}" + (" (chmod 600 -- contains the k3s token)" if secret else "")) - except OSError as exc: - for path in reversed(published): - remove_destination(path) - for path, backup_dir, backup_path in reversed(backups): - remove_destination(path) - os.replace(backup_path, path) - backup_dir.rmdir() - die(f"could not publish generated artifacts: {exc}") - else: - for _, backup_dir, _ in backups: - shutil.rmtree(backup_dir) - finally: - for _, staged_path, _ in staged: - staged_path.unlink(missing_ok=True) - - def main(argv=None) -> int: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--spec", help="path to the cluster-spec JSON, or - for stdin") @@ -349,44 +79,61 @@ def main(argv=None) -> int: ap.add_argument("--token-file", help="read the k3s token from this file instead of generating one") ap.add_argument("--force", action="store_true", help="overwrite existing files") ap.add_argument("--print-schema", action="store_true", help="print an example cluster-spec and exit") + ap.add_argument("--finalize-pxe", action="store_true", help=argparse.SUPPRESS) + ap.add_argument("--context", help=argparse.SUPPRESS) + ap.add_argument("--handoff", help=argparse.SUPPRESS) args = ap.parse_args(argv) if args.print_schema: print(json.dumps(SCHEMA, indent=2)) return 0 + if args.finalize_pxe: + if args.spec or args.token_file or args.context is None or args.handoff is None: + die("--finalize-pxe requires --out-dir, --context, and --handoff", 2) + try: + finalize(Path(args.out_dir), Path(args.context), Path(args.handoff)) + except FinalizationError as error: + die(str(error)) + return 0 if not args.spec: die("--spec is required (or use --print-schema)", 2) raw = sys.stdin.read() if args.spec == "-" else Path(args.spec).read_text(encoding="utf-8") try: - spec = json.loads(raw) - except json.JSONDecodeError as exc: + spec = strict_json_loads(raw) + except (DuplicateJsonKeyError, json.JSONDecodeError) as exc: die(f"spec is not valid JSON: {exc}") - if not isinstance(spec, dict): - die("spec must be a mapping") - topo = spec.get("topology") - if topo not in ("pxe-diskless", "ssh-preinstalled"): - die("spec.topology must be 'pxe-diskless' or 'ssh-preinstalled'") - require(spec, "k3s_version") - require(spec, "server.name") - require(spec, "server.ip") - validate_config_shapes(spec) - + topo = validate_spec(spec) if args.token_file: token = Path(args.token_file).read_text(encoding="utf-8").strip() - if not token: - die("--token-file is empty") + validate_yaml_scalar(token, "--token-file") else: token = gen_token() out = Path(args.out_dir) - artifacts = [(out / "inventory.yml", render_inventory(spec, token), 0o600, True)] + try: + discovery = discover_gpu_policy(spec, out) + except DiscoveryFailure as error: + die(str(error)) if topo == "pxe-diskless": - artifacts.append((out / "pb-pxe-controller.vars.yml", render_pxe_vars(spec), 0o600, False)) - artifacts.append((out / "values-basic-example.yaml", render_values(spec), 0o644, False)) - preflight_destinations([path for path, _, _, _ in artifacts], args.force) - publish_artifacts(artifacts, args.force) + try: + if spec["pxe"]["diskless_agents_have_amd_gpus"]: + stage_pending(spec, token, discovery.resolution, out, args.force) + print("PXE GPU rootfs is pending finalization after pb-pxe-controller.yml resolves its render GID.") + else: + publish_disabled_rootfs(spec, token, discovery.resolution, out, args.force) + except FinalizationError as error: + die(str(error)) + else: + inventory, values, manifest = canonical_paths(out) + artifacts = [(inventory, render_inventory(spec, token, discovery.resolution), 0o600, True)] + artifacts += [ + (values, render_values(spec, discovery.resolution), 0o644, False), + (manifest, manifest_content(discovery), 0o644, False), + ] + preflight_destinations([path for path, _, _, _ in artifacts], args.force) + publish_artifacts(artifacts, args.force) print( "\nNext: review the files, then copy them into your aup-learning-cloud " diff --git a/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_parsing.py b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_parsing.py new file mode 100644 index 00000000..7f14faad --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_parsing.py @@ -0,0 +1,246 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +import re +from dataclasses import dataclass +from pathlib import Path + +from config_common import DuplicateJsonKeyError, strict_json_loads + +MAX_RENDER_GID = 4_294_967_294 + + +@dataclass(frozen=True, slots=True) +class GpuInventory: + hosts: dict[str, bool] + render_gid: int | None + + +@dataclass(frozen=True, slots=True) +class GpuResolution: + status: str + hosts: dict[str, bool] + render_gid: int | None + pxe_rootfs_enabled: bool | None + pxe_rootfs_gid: int | None + + +@dataclass(frozen=True, slots=True) +class PxeGpuPolicy: + enabled: bool + render_gid: int | None + + +def configured_path(repo: Path, value: str) -> Path: + path = Path(value).expanduser() + return path if path.is_absolute() else repo / path + + +def parse_gpu_gid(value: str) -> int | None | str: + normalized = value.strip() + if normalized in {"null", "~"}: + return None + if normalized.isascii() and normalized.isdecimal(): + gid = int(normalized) + if 1 <= gid <= MAX_RENDER_GID: + return gid + return "invalid" + + +def parse_gpu_boolean(value: str) -> bool | None: + normalized = value.strip() + if normalized == "true": + return True + if normalized == "false": + return False + return None + + +def yaml_indent(line: str) -> int: + return len(line) - len(line.lstrip()) + + +def parse_gpu_inventory(text: str) -> tuple[GpuInventory | None, list[str]]: + host_values: dict[str, list[str]] = {} + host_names: list[str] = [] + render_gids: list[str] = [] + stack: list[tuple[int, str]] = [] + + for raw_line in text.splitlines(): + line = raw_line.split("#", 1)[0].rstrip() + if not line.strip(): + continue + indent = yaml_indent(line) + stripped = line.strip() + while stack and indent <= stack[-1][0]: + stack.pop() + path = tuple(key for _, key in stack) + mapping_match = re.fullmatch(r"(.+?):(?:\s*(.*))?", stripped) + if not mapping_match: + continue + key = mapping_match.group(1).strip("\"'") + value = (mapping_match.group(2) or "").strip() + if len(path) == 4 and path[:4] in { + ("k3s_cluster", "children", "server", "hosts"), + ("k3s_cluster", "children", "agent", "hosts"), + }: + host_names.append(key) + host_values.setdefault(key, []) + elif ( + len(path) == 5 + and path[:4] + in { + ("k3s_cluster", "children", "server", "hosts"), + ("k3s_cluster", "children", "agent", "hosts"), + } + and key == "auplc_gpu_access_enabled" + ): + host_values.setdefault(path[4], []).append(value) + elif path == ("k3s_cluster", "vars") and key == "auplc_render_gid": + render_gids.append(value) + stack.append((indent, key)) + + parse_errors: list[str] = [] + if not host_names: + parse_errors.append("inventory has no generated k3s server or agent hosts") + if len(set(host_names)) != len(host_names): + parse_errors.append("inventory has duplicate generated host names") + hosts: dict[str, bool] = {} + for host in host_names: + values = host_values[host] + if len(values) != 1: + parse_errors.append(f"inventory host '{host}' must define exactly one auplc_gpu_access_enabled") + continue + enabled = parse_gpu_boolean(values[0]) + if enabled is None: + parse_errors.append(f"inventory host '{host}' has malformed auplc_gpu_access_enabled") + continue + hosts[host] = enabled + if len(render_gids) != 1: + parse_errors.append("inventory must define exactly one k3s_cluster.vars.auplc_render_gid") + return None, parse_errors + render_gid = parse_gpu_gid(render_gids[0]) + if render_gid == "invalid": + parse_errors.append("inventory has malformed auplc_render_gid") + return None, parse_errors + if parse_errors: + return None, parse_errors + return GpuInventory(hosts=hosts, render_gid=render_gid), parse_errors + + +def parse_values_gpu_gid(text: str) -> tuple[int | None, bool, list[str]]: + render_gids: list[str] = [] + stack: list[tuple[int, str]] = [] + for raw_line in text.splitlines(): + line = raw_line.split("#", 1)[0].rstrip() + if not line.strip(): + continue + indent = yaml_indent(line) + stripped = line.strip() + while stack and indent <= stack[-1][0]: + stack.pop() + path = tuple(key for _, key in stack) + mapping_match = re.fullmatch(r"(.+?):(?:\s*(.*))?", stripped) + if not mapping_match: + continue + key = mapping_match.group(1).strip("\"'") + value = (mapping_match.group(2) or "").strip() + if path == ("custom", "gpuAccess") and key == "renderGid": + render_gids.append(value) + stack.append((indent, key)) + if not render_gids: + return None, False, [] + if len(render_gids) != 1: + return None, True, ["custom.gpuAccess.renderGid is duplicated"] + render_gid = parse_gpu_gid(render_gids[0]) + if render_gid == "invalid": + return None, True, ["custom.gpuAccess.renderGid is malformed"] + return render_gid, True, [] + + +def collect_effective_gpu_gid(repo: Path, values: list[str]) -> tuple[int | None, list[str]]: + effective_gid: int | None = None + found = False + parse_errors: list[str] = [] + for rel in values or ["runtime/values.yaml"]: + path = configured_path(repo, rel) + if not path.exists(): + continue + render_gid, present, file_errors = parse_values_gpu_gid(path.read_text(encoding="utf-8")) + parse_errors.extend(f"{path}: {error}" for error in file_errors) + if present and not file_errors: + effective_gid = render_gid + found = True + if not found: + parse_errors.append("effective values have no custom.gpuAccess.renderGid") + return effective_gid, parse_errors + + +def parse_gpu_resolution(text: str, topology: str) -> tuple[GpuResolution | None, list[str]]: + try: + document = strict_json_loads(text) + except DuplicateJsonKeyError as exc: + return None, [f"GPU resolution manifest is malformed: {exc}"] + except (TypeError, ValueError) as exc: + return None, [f"GPU resolution manifest is malformed: {exc}"] + if type(document) is not dict: + return None, ["GPU resolution manifest must be a JSON object"] + expected_keys = {"version", "status", "render_gid", "hosts"} + if topology == "pxe-diskless": + expected_keys.add("pxe_rootfs") + if set(document) != expected_keys: + return None, ["GPU resolution manifest has an unexpected schema"] + if type(document["version"]) is not int or document["version"] != 1: + return None, ["GPU resolution manifest version must be integer 1"] + status = document["status"] + if type(status) is not str or status not in {"cpu_only", "gpu_resolved"}: + return None, ["GPU resolution manifest status must be cpu_only or gpu_resolved"] + if type(document["hosts"]) is not dict or not document["hosts"]: + return None, ["GPU resolution manifest hosts must be a non-empty object"] + if any( + type(host) is not str or not host or type(enabled) is not bool for host, enabled in document["hosts"].items() + ): + return None, ["GPU resolution manifest hosts must map non-empty names to booleans"] + render_gid = document["render_gid"] + if render_gid is not None and (type(render_gid) is not int or not 1 <= render_gid <= MAX_RENDER_GID): + return None, ["GPU resolution manifest render_gid must be an integer or null"] + if topology == "ssh-preinstalled": + return GpuResolution(status, document["hosts"], render_gid, None, None), [] + rootfs = document["pxe_rootfs"] + if type(rootfs) is not dict or set(rootfs) != {"gpu_access_enabled", "render_gid"}: + return None, ["GPU resolution manifest pxe_rootfs has an unexpected schema"] + rootfs_enabled = rootfs["gpu_access_enabled"] + rootfs_gid = rootfs["render_gid"] + if type(rootfs_enabled) is not bool: + return None, ["GPU resolution manifest pxe_rootfs.gpu_access_enabled must be boolean"] + if rootfs_gid is not None and (type(rootfs_gid) is not int or not 1 <= rootfs_gid <= MAX_RENDER_GID): + return None, ["GPU resolution manifest pxe_rootfs.render_gid must be an integer or null"] + return GpuResolution(status, document["hosts"], render_gid, rootfs_enabled, rootfs_gid), [] + + +def parse_pxe_gpu_policy(text: str) -> tuple[PxeGpuPolicy | None, list[str]]: + values: dict[str, list[str]] = {"auplc_render_gid": [], "pxe_gpu_access_enabled": []} + for raw_line in text.splitlines(): + line = raw_line.split("#", 1)[0].rstrip() + if not line.strip() or yaml_indent(line) != 0: + continue + mapping_match = re.fullmatch(r"(.+?):(?:\s*(.*))?", line.strip()) + if not mapping_match: + continue + key = mapping_match.group(1).strip("\"'") + if key in values: + values[key].append((mapping_match.group(2) or "").strip()) + parse_errors: list[str] = [] + for key, occurrences in values.items(): + if len(occurrences) != 1: + parse_errors.append(f"PXE vars must define exactly one {key}") + if parse_errors: + return None, parse_errors + render_gid = parse_gpu_gid(values["auplc_render_gid"][0]) + enabled = parse_gpu_boolean(values["pxe_gpu_access_enabled"][0]) + if render_gid == "invalid": + parse_errors.append("PXE vars have malformed auplc_render_gid") + if enabled is None: + parse_errors.append("PXE vars have malformed pxe_gpu_access_enabled") + if parse_errors: + return None, parse_errors + return PxeGpuPolicy(enabled=enabled, render_gid=render_gid), [] diff --git a/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_validation.py b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_validation.py new file mode 100644 index 00000000..1fd8c041 --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_validation.py @@ -0,0 +1,125 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +from dataclasses import dataclass +from pathlib import Path + +from gpu_resolution_parsing import ( + collect_effective_gpu_gid, + configured_path, + parse_gpu_inventory, + parse_gpu_resolution, + parse_pxe_gpu_policy, +) + + +@dataclass(frozen=True, slots=True) +class GpuArtifactValidationRequest: + repo: Path + inventory_path: str + resolution_path: str + values: list[str] + topology: str + pxe_vars_path: Path + has_prior_errors: bool + + +@dataclass(frozen=True, slots=True) +class GpuArtifactValidationResult: + errors: list[str] + passed: list[str] + + +@dataclass(frozen=True, slots=True) +class AcceleratorValidationResult: + errors: list[str] + warnings: list[str] + passed: list[str] + + +def check_accelerator_labels( + accelerators: dict[str, str], metadata: dict[str, list[str]], cluster: dict | None +) -> AcceleratorValidationResult: + errors: list[str] = [] + warnings: list[str] = [] + passed: list[str] = [] + active_keys = sorted({key for keys in metadata.values() for key in keys}) + if not active_keys: + return AcceleratorValidationResult([], ["no acceleratorKeys found in effective custom.resources.metadata"], []) + declared: list[str] = [] + for key in active_keys: + if key not in accelerators: + errors.append(f"active accelerator '{key}' is not defined under custom.accelerators") + elif not accelerators[key]: + errors.append(f"active accelerator '{key}' has no amd.com/gpu.product-name nodeSelector") + else: + declared.append(accelerators[key]) + if not declared: + return AcceleratorValidationResult(errors, warnings, passed) + if cluster is None: + warnings.append( + "no --cluster snapshot; cannot confirm nodeSelector labels match real " + f"nodes. Declared: {', '.join(declared)}" + ) + return AcceleratorValidationResult(errors, warnings, passed) + real = set(cluster.get("gpu_product_names", [])) + if not real: + errors.append("cluster snapshot has no GPU product labels for active accelerators") + return AcceleratorValidationResult(errors, warnings, passed) + for declared_label in declared: + if declared_label in real: + passed.append(f"nodeSelector '{declared_label}' matches a real node label") + else: + errors.append( + f"nodeSelector '{declared_label}' matches no node label. Real labels: {', '.join(sorted(real))}" + ) + return AcceleratorValidationResult(errors, warnings, passed) + + +def check_gpu_artifacts(request: GpuArtifactValidationRequest) -> GpuArtifactValidationResult: + errors: list[str] = [] + inventory_file = configured_path(request.repo, request.inventory_path) + resolution_file = configured_path(request.repo, request.resolution_path) + if not inventory_file.exists(): + return GpuArtifactValidationResult([f"generated inventory not found: {inventory_file}"], []) + if not resolution_file.exists(): + return GpuArtifactValidationResult([f"GPU resolution manifest not found: {resolution_file}"], []) + inventory, inventory_errors = parse_gpu_inventory(inventory_file.read_text(encoding="utf-8")) + resolution, resolution_errors = parse_gpu_resolution(resolution_file.read_text(encoding="utf-8"), request.topology) + helm_gid, helm_errors = collect_effective_gpu_gid(request.repo, request.values) + errors.extend([*inventory_errors, *resolution_errors, *helm_errors]) + if inventory is None or resolution is None or errors: + return GpuArtifactValidationResult(errors, []) + if set(inventory.hosts) != set(resolution.hosts): + errors.append("inventory hosts do not exactly match GPU resolution manifest hosts") + for host, enabled in inventory.hosts.items(): + if resolution.hosts.get(host) != enabled: + errors.append(f"inventory host '{host}' GPU access boolean disagrees with the resolution manifest") + pxe_policy = None + if request.topology == "pxe-diskless": + if not request.pxe_vars_path.exists(): + return GpuArtifactValidationResult([*errors, f"PXE vars file not found: {request.pxe_vars_path}"], []) + pxe_policy, pxe_errors = parse_pxe_gpu_policy(request.pxe_vars_path.read_text(encoding="utf-8")) + errors.extend(pxe_errors) + if pxe_policy is None or pxe_errors: + return GpuArtifactValidationResult(errors, []) + if pxe_policy.enabled != resolution.pxe_rootfs_enabled: + errors.append("PXE pxe_gpu_access_enabled disagrees with GPU resolution manifest pxe_rootfs") + if resolution.pxe_rootfs_enabled and resolution.pxe_rootfs_gid is None: + errors.append("GPU-enabled PXE rootfs requires a numeric render GID") + if not resolution.pxe_rootfs_enabled and resolution.pxe_rootfs_gid is not None: + errors.append("GPU-disabled PXE rootfs requires a null render GID") + if resolution.pxe_rootfs_enabled and pxe_policy.render_gid != resolution.pxe_rootfs_gid: + errors.append("PXE auplc_render_gid disagrees with GPU resolution manifest pxe_rootfs render_gid") + gids = [inventory.render_gid, helm_gid, resolution.render_gid] + if pxe_policy is not None: + gids.append(pxe_policy.render_gid) + if len(set(gids)) != 1: + errors.append("inventory, Helm, PXE, and GPU resolution render GIDs disagree") + enabled_scope = any(resolution.hosts.values()) or resolution.pxe_rootfs_enabled is True + if resolution.status == "cpu_only": + if enabled_scope or resolution.render_gid is not None or any(gid is not None for gid in gids): + errors.append("cpu_only GPU resolution requires all host/rootfs booleans false and all render GIDs null") + elif not enabled_scope or resolution.render_gid is None: + errors.append("gpu_resolved GPU resolution requires an enabled scope and a numeric render GID") + passed = [] if request.has_prior_errors or errors else ["GPU access artifacts agree"] + return GpuArtifactValidationResult(errors, passed) diff --git a/skills/deploy-aup-learning-cloud/scripts/validate.py b/skills/deploy-aup-learning-cloud/scripts/validate.py index 8408f020..ccb5bb6b 100755 --- a/skills/deploy-aup-learning-cloud/scripts/validate.py +++ b/skills/deploy-aup-learning-cloud/scripts/validate.py @@ -12,6 +12,8 @@ * nodeSelectors for the accelerators actually referenced by effective custom.resources.metadata.*.acceleratorKeys, checked against detect_cluster.sh output when supplied; + * generated inventory, GPU-resolution manifest, Helm render GID, and PXE + rootfs policy agree when generated artifacts are supplied; * (optional) the chart does not render: a `helm template` dry-run. This intentionally uses regex/line scanning rather than a YAML parser so it @@ -20,9 +22,10 @@ inspect something. Usage: - validate.py --repo ~/aup-learning-cloud --topology pxe-diskless validate.py --repo ~/aup-learning-cloud \ --topology ssh-preinstalled \ + --inventory generated/inventory.yml \ + --gpu-resolution generated/gpu-access-resolution.json \ --values runtime/values.yaml --values runtime/values-basic-example.yaml \ --cluster cluster.json --helm-dry-run @@ -30,8 +33,6 @@ 2 on a usage error. """ -from __future__ import annotations - import argparse import json import re @@ -40,6 +41,10 @@ import sys from pathlib import Path +from config_common import DuplicateJsonKeyError, strict_json_loads +from gpu_resolution_validation import GpuArtifactValidationRequest, check_accelerator_labels, check_gpu_artifacts +from values_resolution_parsing import collect_effective_values + PXE_PLAYBOOK = "deploy/ansible/playbooks/pb-pxe-controller.yml" INVENTORY = "deploy/ansible/inventory.yml" CHART = "runtime/chart" @@ -163,155 +168,6 @@ def check_version_sync(repo: Path, configured_path: str | None = None) -> None: ) -def yaml_scalar(value: str) -> str: - return value.strip().strip('"').strip("'") - - -def yaml_optional_scalar(value: str) -> str: - scalar_value = yaml_scalar(value) - return "" if scalar_value in {"", "null", "~"} else scalar_value - - -def yaml_indent(line: str) -> int: - return len(line) - len(line.lstrip()) - - -def parse_inline_list(value: str) -> list[str]: - items = value.strip()[1:-1].strip() - if not items: - return [] - return [yaml_scalar(item) for item in items.split(",") if yaml_scalar(item)] - - -def is_relevant_flow_path(path: tuple[str, ...]) -> bool: - return path == ("custom",) or path[:2] in {("custom", "accelerators"), ("custom", "resources")} - - -def unsupported_yaml_syntax(value: str) -> bool: - return value.startswith(("&", "*", "!", "|", ">")) - - -def parse_values_file(text: str) -> tuple[dict[str, str | None], dict[str, list[str]], list[str]]: - """Extract the deploy-relevant mappings from a fixed-shape values YAML file. - - The helpers deliberately remain stdlib-only. This scanner handles the - mapping/list shapes used by values overlays, rather than pretending to be a - general YAML parser. - """ - accelerators: dict[str, str | None] = {} - metadata: dict[str, list[str]] = {} - parse_errors: list[str] = [] - stack: list[tuple[int, str]] = [] - - for raw_line in text.splitlines(): - line = raw_line.split("#", 1)[0].rstrip() - if not line.strip(): - continue - indent = yaml_indent(line) - stripped = line.strip() - - while stack and indent <= stack[-1][0]: - stack.pop() - path = tuple(key for _, key in stack) - - if stripped.startswith("- "): - if len(path) == 5 and path[:3] == ("custom", "resources", "metadata") and path[-1] == "acceleratorKeys": - metadata.setdefault(path[3], []).append(yaml_scalar(stripped[2:])) - continue - - product_label_match = re.fullmatch( - r"(?:[\"']amd\.com/gpu\.product-name[\"']|amd\.com/gpu\.product-name):\s*(.*)", stripped - ) - if product_label_match: - if len(path) == 4 and path[:2] == ("custom", "accelerators") and path[-1] == "nodeSelector": - value = product_label_match.group(1).strip() - if unsupported_yaml_syntax(value): - parse_errors.append( - f"unsupported YAML syntax at custom.accelerators.{path[2]}.nodeSelector.amd.com/gpu.product-name" - ) - else: - accelerators[path[2]] = yaml_optional_scalar(value) - continue - - mapping_match = re.fullmatch(r"(.+?):(?:\s*(.*))?", stripped) - if not mapping_match: - continue - key = mapping_match.group(1).strip("\"'") - value = (mapping_match.group(2) or "").strip() - candidate_path = path + (key,) - if value.startswith("{") and value != "{}" and is_relevant_flow_path(candidate_path): - parse_errors.append(f"unsupported non-empty flow-style mapping at {'.'.join(candidate_path)}") - if unsupported_yaml_syntax(value) and is_relevant_flow_path(candidate_path): - parse_errors.append(f"unsupported YAML syntax at {'.'.join(candidate_path)}") - if path == ("custom", "accelerators"): - accelerators.setdefault(key, None) - if len(path) == 4 and path[:3] == ("custom", "resources", "metadata") and key == "acceleratorKeys": - resource_key = path[3] - if unsupported_yaml_syntax(value): - parse_errors.append(f"unsupported YAML syntax at {'.'.join(candidate_path)}") - elif value.startswith("[") and value.endswith("]"): - metadata[resource_key] = parse_inline_list(value) - elif not value or value in {"null", "~"}: - metadata[resource_key] = [] - else: - parse_errors.append(f"acceleratorKeys must be a list at {'.'.join(candidate_path)}") - stack.append((indent, key)) - return accelerators, metadata, parse_errors - - -def collect_effective_values(repo: Path, values: list[str]) -> tuple[dict[str, str], dict[str, list[str]], list[str]]: - paths = values or ["runtime/values.yaml"] - accelerators: dict[str, str] = {} - metadata: dict[str, list[str]] = {} - parse_errors: list[str] = [] - for rel in paths: - p = (repo / rel) if not Path(rel).is_absolute() else Path(rel) - if p.exists(): - parsed_accelerators, parsed_metadata, file_errors = parse_values_file(p.read_text(encoding="utf-8")) - for key, selector in parsed_accelerators.items(): - if selector is not None or key not in accelerators: - accelerators[key] = selector - metadata.update(parsed_metadata) - parse_errors.extend(file_errors) - else: - fail(f"values file not found: {rel}") - return accelerators, metadata, parse_errors - - -def check_accelerator_labels( - accelerators: dict[str, str], metadata: dict[str, list[str]], cluster: dict | None -) -> None: - active_keys = sorted({key for keys in metadata.values() for key in keys}) - if not active_keys: - warn("no acceleratorKeys found in effective custom.resources.metadata") - return - declared: list[str] = [] - for key in active_keys: - if key not in accelerators: - fail(f"active accelerator '{key}' is not defined under custom.accelerators") - elif not accelerators[key]: - fail(f"active accelerator '{key}' has no amd.com/gpu.product-name nodeSelector") - else: - declared.append(accelerators[key]) - if not declared: - return - if cluster is None: - warn( - "no --cluster snapshot; cannot confirm nodeSelector labels match real " - f"nodes. Declared: {', '.join(declared)}" - ) - return - real = set(cluster.get("gpu_product_names", [])) - if not real: - fail("cluster snapshot has no GPU product labels for active accelerators") - return - for d in declared: - if d in real: - ok(f"nodeSelector '{d}' matches a real node label") - else: - fail(f"nodeSelector '{d}' matches no node label. Real labels: {', '.join(sorted(real))}") - - def check_helm(repo: Path, values: list[str]) -> None: if not shutil.which("helm"): warn("helm not on PATH; skipped chart dry-run") @@ -353,6 +209,8 @@ def main(argv=None) -> int: "--pxe-vars", help="PXE vars file to validate instead of deploy/ansible/playbooks/pb-pxe-controller.yml", ) + ap.add_argument("--inventory", help="generated inventory.yml to cross-check with GPU resolution") + ap.add_argument("--gpu-resolution", help="generated gpu-access-resolution.json to cross-check") ap.add_argument("--cluster", help="detect_cluster.sh JSON output to match labels against") ap.add_argument("--helm-dry-run", action="store_true", help="also run `helm template`") ap.add_argument("--json", action="store_true", help="emit a JSON report instead of text") @@ -366,8 +224,8 @@ def main(argv=None) -> int: cluster = None if args.cluster: try: - cluster = json.loads(Path(args.cluster).read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as exc: + cluster = strict_json_loads(Path(args.cluster).read_text(encoding="utf-8")) + except (DuplicateJsonKeyError, OSError, json.JSONDecodeError) as exc: print(f"validate: cannot read --cluster: {exc}", file=sys.stderr) return 2 @@ -376,10 +234,36 @@ def main(argv=None) -> int: check_version_sync(repo, args.pxe_vars) else: ok("skipped PXE checks for ssh-preinstalled topology") - accelerators, metadata, parse_errors = collect_effective_values(repo, args.values) - for message in parse_errors: + values_result = collect_effective_values(repo, args.values) + for message in values_result.missing_files: + fail(message) + for message in values_result.parse_errors: + fail(message) + accelerator_result = check_accelerator_labels(values_result.accelerators, values_result.metadata, cluster) + for message in accelerator_result.errors: fail(message) - check_accelerator_labels(accelerators, metadata, cluster) + for message in accelerator_result.warnings: + warn(message) + for message in accelerator_result.passed: + ok(message) + if bool(args.inventory) != bool(args.gpu_resolution): + fail("--inventory and --gpu-resolution must be supplied together") + elif args.inventory and args.gpu_resolution: + artifact_result = check_gpu_artifacts( + GpuArtifactValidationRequest( + repo=repo, + inventory_path=args.inventory, + resolution_path=args.gpu_resolution, + values=args.values, + topology=args.topology, + pxe_vars_path=pxe_vars_path(repo, args.pxe_vars), + has_prior_errors=bool(errors), + ) + ) + for message in artifact_result.errors: + fail(message) + for message in artifact_result.passed: + ok(message) if args.helm_dry_run: check_helm(repo, args.values) diff --git a/skills/deploy-aup-learning-cloud/scripts/values_resolution_parsing.py b/skills/deploy-aup-learning-cloud/scripts/values_resolution_parsing.py new file mode 100644 index 00000000..9836162d --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/values_resolution_parsing.py @@ -0,0 +1,130 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +"""Fixed-shape parsing and overlay resolution for deploy values files.""" + +import re +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True, slots=True) +class ValuesFileParseResult: + accelerators: dict[str, str | None] + metadata: dict[str, list[str]] + parse_errors: list[str] + + +@dataclass(frozen=True, slots=True) +class EffectiveValuesResult: + accelerators: dict[str, str] + metadata: dict[str, list[str]] + missing_files: list[str] + parse_errors: list[str] + + +def yaml_scalar(value: str) -> str: + return value.strip().strip('"').strip("'") + + +def yaml_optional_scalar(value: str) -> str: + scalar_value = yaml_scalar(value) + return "" if scalar_value in {"", "null", "~"} else scalar_value + + +def yaml_indent(line: str) -> int: + return len(line) - len(line.lstrip()) + + +def parse_inline_list(value: str) -> list[str]: + items = value.strip()[1:-1].strip() + if not items: + return [] + return [yaml_scalar(item) for item in items.split(",") if yaml_scalar(item)] + + +def is_relevant_flow_path(path: tuple[str, ...]) -> bool: + return path == ("custom",) or path[:2] in {("custom", "accelerators"), ("custom", "resources")} + + +def unsupported_yaml_syntax(value: str) -> bool: + return value.startswith(("&", "*", "!", "|", ">")) + + +def parse_values_file(text: str) -> ValuesFileParseResult: + accelerators: dict[str, str | None] = {} + metadata: dict[str, list[str]] = {} + parse_errors: list[str] = [] + stack: list[tuple[int, str]] = [] + + for raw_line in text.splitlines(): + line = raw_line.split("#", 1)[0].rstrip() + if not line.strip(): + continue + indent = yaml_indent(line) + stripped = line.strip() + + while stack and indent <= stack[-1][0]: + stack.pop() + path = tuple(key for _, key in stack) + + if stripped.startswith("- "): + if len(path) == 5 and path[:3] == ("custom", "resources", "metadata") and path[-1] == "acceleratorKeys": + metadata.setdefault(path[3], []).append(yaml_scalar(stripped[2:])) + continue + + product_label_match = re.fullmatch( + r"(?:[\"']amd\.com/gpu\.product-name[\"']|amd\.com/gpu\.product-name):\s*(.*)", stripped + ) + if product_label_match: + if len(path) == 4 and path[:2] == ("custom", "accelerators") and path[-1] == "nodeSelector": + value = product_label_match.group(1).strip() + if unsupported_yaml_syntax(value): + parse_errors.append( + f"unsupported YAML syntax at custom.accelerators.{path[2]}.nodeSelector.amd.com/gpu.product-name" + ) + else: + accelerators[path[2]] = yaml_optional_scalar(value) + continue + + mapping_match = re.fullmatch(r"(.+?):(?:\s*(.*))?", stripped) + if not mapping_match: + continue + key = mapping_match.group(1).strip("\"'") + value = (mapping_match.group(2) or "").strip() + candidate_path = path + (key,) + if value.startswith("{") and value != "{}" and is_relevant_flow_path(candidate_path): + parse_errors.append(f"unsupported non-empty flow-style mapping at {'.'.join(candidate_path)}") + if unsupported_yaml_syntax(value) and is_relevant_flow_path(candidate_path): + parse_errors.append(f"unsupported YAML syntax at {'.'.join(candidate_path)}") + if path == ("custom", "accelerators"): + accelerators.setdefault(key, None) + if len(path) == 4 and path[:3] == ("custom", "resources", "metadata") and key == "acceleratorKeys": + resource_key = path[3] + if unsupported_yaml_syntax(value): + parse_errors.append(f"unsupported YAML syntax at {'.'.join(candidate_path)}") + elif value.startswith("[") and value.endswith("]"): + metadata[resource_key] = parse_inline_list(value) + elif not value or value in {"null", "~"}: + metadata[resource_key] = [] + else: + parse_errors.append(f"acceleratorKeys must be a list at {'.'.join(candidate_path)}") + stack.append((indent, key)) + return ValuesFileParseResult(accelerators, metadata, parse_errors) + + +def collect_effective_values(repo: Path, values: list[str]) -> EffectiveValuesResult: + accelerators: dict[str, str] = {} + metadata: dict[str, list[str]] = {} + missing_files: list[str] = [] + parse_errors: list[str] = [] + for rel in values or ["runtime/values.yaml"]: + path = (repo / rel) if not Path(rel).is_absolute() else Path(rel) + if not path.exists(): + missing_files.append(f"values file not found: {rel}") + continue + parsed = parse_values_file(path.read_text(encoding="utf-8")) + for key, selector in parsed.accelerators.items(): + if selector is not None or key not in accelerators: + accelerators[key] = selector + metadata.update(parsed.metadata) + parse_errors.extend(parsed.parse_errors) + return EffectiveValuesResult(accelerators, metadata, missing_files, parse_errors) diff --git a/tests/skills/test_config_generation_security.py b/tests/skills/test_config_generation_security.py new file mode 100644 index 00000000..6ec39bf8 --- /dev/null +++ b/tests/skills/test_config_generation_security.py @@ -0,0 +1,95 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +GEN_CONFIGS = ROOT / "skills" / "deploy-aup-learning-cloud" / "scripts" / "gen_configs.py" + + +def safe_spec() -> dict[str, object]: + return { + "topology": "ssh-preinstalled", + "k3s_version": "v1.32.3+k3s1", + "server": {"name": "server-1", "ip": "192.168.1.10"}, + "agents": [{"name": "agent-1", "ip": "192.168.1.11"}], + "images": {"cpu": "registry.example/auplc:latest"}, + } + + +def load_config_generation_module(): + sys.path.insert(0, str(GEN_CONFIGS.parent)) + try: + import config_generation + + return config_generation + finally: + sys.path.pop(0) + + +@pytest.mark.parametrize( + ("path", "value", "message"), + [ + (("server", "name"), "server\n vars: {injected: true}", "spec.server.name"), + (("server", "ip"), "192.168.1.10\n injected: true", "spec.server.ip"), + (("k3s_version",), "v1.32.3+k3s1\n injected: true", "spec.k3s_version"), + (("agents",), [{"name": "server-1", "ip": "192.168.1.11"}], "unique"), + (("agents",), [{"name": "agent-1", "ip": "not-an-ip"}], "spec.agents[0].ip"), + (("images",), {"cpu\n injected": "registry.example/auplc:latest"}, "spec.images key"), + ], +) +def test_generator_rejects_unsafe_public_spec_scalars_before_discovery( + path: tuple[str, ...], value: object, message: str, capsys: pytest.CaptureFixture[str] +) -> None: + module = load_config_generation_module() + spec = safe_spec() + if len(path) == 1: + spec[path[0]] = value + else: + target = spec[path[0]] + assert isinstance(target, dict) + target[path[1]] = value + with pytest.raises(SystemExit) as error: + module.validate_spec(spec) + + assert error.value.code == 1 + assert message in capsys.readouterr().err + + +def test_generator_rejects_an_invalid_k3s_version_before_discovery(capsys: pytest.CaptureFixture[str]) -> None: + module = load_config_generation_module() + spec = safe_spec() + spec["k3s_version"] = "v1.32.3+k3s1 # comments are not accepted" + + with pytest.raises(SystemExit) as error: + module.validate_spec(spec) + + assert error.value.code == 1 + assert "spec.k3s_version" in capsys.readouterr().err + + +@pytest.mark.parametrize( + "raw", + [ + '{"topology":"ssh-preinstalled","topology":"pxe-diskless"}', + '{"topology":"pxe-diskless","k3s_version":"v1.32.3+k3s1","server":{"name":"server","ip":"192.168.1.10"},"network":{"interface":"eno1","subnet":"192.168.1.0/24"},"pxe":{"authorized_keys":["ssh-ed25519 AAA"],"diskless_agents_have_amd_gpus":true,"diskless_agents_have_amd_gpus":false}}', + ], +) +def test_generator_rejects_duplicate_public_policy_keys_before_discovery(tmp_path: Path, raw: str) -> None: + spec = tmp_path / "spec.json" + spec.write_text(raw, encoding="utf-8") + + result = subprocess.run( + [sys.executable, str(GEN_CONFIGS), "--spec", str(spec), "--out-dir", str(tmp_path / "generated")], + capture_output=True, + check=False, + text=True, + ) + + assert result.returncode == 1 + assert "duplicate JSON key" in result.stderr diff --git a/tests/skills/test_deploy_scripts.py b/tests/skills/test_deploy_scripts.py index 7e5b6f2c..0469f4fc 100644 --- a/tests/skills/test_deploy_scripts.py +++ b/tests/skills/test_deploy_scripts.py @@ -19,6 +19,33 @@ DEPLOY_SCRIPTS = ROOT / "skills" / "deploy-aup-learning-cloud" / "scripts" VALIDATE = DEPLOY_SCRIPTS / "validate.py" GEN_CONFIGS = DEPLOY_SCRIPTS / "gen_configs.py" +CONFIG_GENERATION = DEPLOY_SCRIPTS / "config_generation.py" +ARTIFACT_STORE = DEPLOY_SCRIPTS / "artifact_store.py" +VALUES_RESOLUTION_PARSING = DEPLOY_SCRIPTS / "values_resolution_parsing.py" + +EXPECTED_GENERATOR_SCHEMA = { + "topology": "pxe-diskless | ssh-preinstalled", + "k3s_version": "v1.32.3+k3s1", + "server": {"name": "aipc1", "ip": "192.168.0.140"}, + "agents": [{"name": "aipc2", "ip": "192.168.0.141"}], + "network": { + "interface": "enp1s0", + "subnet": "192.168.0.0/24", + "gateway": "192.168.0.1", + "dns_servers": "8.8.8.8,8.8.4.4", + }, + "pxe": { + "authorized_keys": ["ssh-ed25519 AAAA... you@host"], + "rootfs_password": "", + "web_port": 8080, + "diskless_agents_have_amd_gpus": True, + }, + "accelerators": {"strix-halo": {"product_name": "AMD_Radeon_8060S_Graphics"}}, + "storage": {"class": "nfs-client"}, + "proxy": {"node_port": 30890}, + "auth_mode": "auto-login", + "images": {"cpu": "ghcr.io/amdresearch/auplc-default:latest", "gpu": "ghcr.io/amdresearch/auplc-base:latest"}, +} def run_script(script: Path, *args: str, cwd: Path | None = None) -> subprocess.CompletedProcess[str]: @@ -37,23 +64,113 @@ def write_file(path: Path, content: str) -> Path: return path +@pytest.fixture(autouse=True) +def fake_ansible_playbook(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + fake_bin = tmp_path / "fake-ansible" + fake_bin.mkdir() + fake_ansible = fake_bin / "ansible-playbook" + fake_ansible.write_text( + r"""#!/usr/bin/env python3 +import json +from pathlib import Path +import sys + +arguments = sys.argv[1:] +inventory = Path(arguments[arguments.index('-i') + 1]) +output = next(value.split('=', 1)[1] for value in arguments if value.startswith('gpu_access_discovery_output_path=')) +hosts = [line.strip()[:-1] for line in inventory.read_text(encoding='utf-8').splitlines() if line.startswith(' ') and line.rstrip().endswith(':')] +evidence = { + 'version': 2, + 'hosts': [{ + 'host': host, + 'reachable': True, + 'lspci': {'rc': 0, 'stdout': ''}, + 'sysfs': {'rc': 0, 'stdout': ''}, + 'render_group': {'rc': 0, 'stdout': 'render:x:993:\\n'}, + 'groups': {'rc': 0, 'stdout': 'render:x:993:\\n'}, + 'state': {'stat_success': True, 'content_success': True, 'exists': False, 'regular': False, 'symlink': False, 'content': ''}, + 'rule': {'stat_success': True, 'content_success': True, 'exists': False, 'regular': False, 'symlink': False, 'content': ''}, + 'legacy_rules': { + key: {'stat_success': True, 'content_success': True, 'exists': False, 'regular': False, 'symlink': False, 'content': ''} + for key in ('kfd', 'amdgpu', 'rocm_devices') + }, + } for host in hosts], +} +Path(output).write_text(json.dumps(evidence), encoding='utf-8') +""", + encoding="utf-8", + ) + fake_ansible.chmod(0o755) + monkeypatch.setenv("PATH", f"{fake_bin}:{os.environ['PATH']}") + + def write_cluster(repo: Path, labels: list[str]) -> Path: return write_file(repo / "cluster.json", json.dumps({"gpu_product_names": labels})) -def load_validate_module(): - spec = importlib.util.spec_from_file_location("deploy_validate", VALIDATE) - module = importlib.util.module_from_spec(spec) - assert spec.loader is not None - spec.loader.exec_module(module) - return module +def write_resolved_gpu_artifacts(repo: Path) -> tuple[Path, Path, Path]: + inventory = write_file( + repo / "generated/inventory.yml", + """k3s_cluster: + children: + server: + hosts: + server: + ansible_host: 192.168.1.10 + auplc_gpu_access_enabled: true + agent: + hosts: + agent: + ansible_host: 192.168.1.11 + auplc_gpu_access_enabled: false + vars: + auplc_render_gid: 993 +""", + ) + values = write_file( + repo / "generated/values-basic-example.yaml", + """custom: + gpuAccess: + renderGid: 993 + resources: + metadata: {} +""", + ) + resolution = write_file( + repo / "generated/gpu-access-resolution.json", + json.dumps( + { + "version": 1, + "status": "gpu_resolved", + "render_gid": 993, + "hosts": {"agent": False, "server": True}, + } + ), + ) + return inventory, values, resolution -def load_generator_module(): - spec = importlib.util.spec_from_file_location("deploy_generator", GEN_CONFIGS) - module = importlib.util.module_from_spec(spec) - assert spec.loader is not None - spec.loader.exec_module(module) +def load_validate_module(): + sys.path.insert(0, str(DEPLOY_SCRIPTS)) + try: + spec = importlib.util.spec_from_file_location("deploy_validate", VALIDATE) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + finally: + sys.path.pop(0) + + +def load_deploy_module(module_name: str, script: Path): + sys.path.insert(0, str(DEPLOY_SCRIPTS)) + try: + spec = importlib.util.spec_from_file_location(module_name, script) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + finally: + sys.path.pop(0) return module @@ -174,6 +291,43 @@ def test_validator_retains_selectors_from_partial_accelerator_overlays(tmp_path: assert "AMD_Radeon_8060S_Graphics" in result.stdout +def test_values_resolution_parser_preserves_overlay_precedence_and_error_categories(tmp_path: Path) -> None: + parser = load_deploy_module("values_resolution_parsing", VALUES_RESOLUTION_PARSING) + repo = tmp_path / "checkout" + base = write_file( + repo / "base.yaml", + """custom: + accelerators: + strix-halo: + nodeSelector: + amd.com/gpu.product-name: AMD_Radeon_8060S_Graphics + resources: + metadata: + gpu: + acceleratorKeys: [strix-halo] +""", + ) + partial_overlay = write_file( + repo / "partial.yaml", + """custom: + accelerators: + strix-halo: + displayName: Renamed +""", + ) + invalid_overlay = write_file(repo / "invalid.yaml", "custom: *defaults\n") + + result = parser.collect_effective_values( + repo, + [str(base), str(partial_overlay), "missing.yaml", str(invalid_overlay)], + ) + + assert result.accelerators == {"strix-halo": "AMD_Radeon_8060S_Graphics"} + assert result.metadata == {"gpu": ["strix-halo"]} + assert result.missing_files == ["values file not found: missing.yaml"] + assert result.parse_errors == ["unsupported YAML syntax at custom"] + + def test_validator_accepts_quoted_product_label_keys(tmp_path: Path) -> None: repo = tmp_path / "checkout" values = write_file( @@ -751,6 +905,345 @@ def test_validator_fails_when_an_active_accelerator_has_no_product_selector(tmp_ assert "active accelerator 'strix-halo' has no amd.com/gpu.product-name nodeSelector" in result.stdout +def test_validator_accepts_consistent_cpu_only_gpu_artifacts(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + inventory = write_file( + repo / "generated/inventory.yml", + """k3s_cluster: + children: + server: + hosts: + server: + ansible_host: 192.168.1.10 + auplc_gpu_access_enabled: false + agent: + hosts: + agent: + ansible_host: 192.168.1.11 + auplc_gpu_access_enabled: false + vars: + auplc_render_gid: null +""", + ) + values = write_file( + repo / "generated/values-basic-example.yaml", + """custom: + gpuAccess: + renderGid: null + resources: + metadata: {} +""", + ) + resolution = write_file( + repo / "generated/gpu-access-resolution.json", + json.dumps( + { + "version": 1, + "status": "cpu_only", + "render_gid": None, + "hosts": {"agent": False, "server": False}, + } + ), + ) + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--inventory", + str(inventory), + "--values", + str(values), + "--gpu-resolution", + str(resolution), + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert "GPU access artifacts agree" in result.stdout + + +def test_validator_accepts_consistent_gpu_resolved_artifacts(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + inventory, values, resolution = write_resolved_gpu_artifacts(repo) + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--inventory", + str(inventory), + "--values", + str(values), + "--gpu-resolution", + str(resolution), + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert "GPU access artifacts agree" in result.stdout + + +@pytest.mark.parametrize( + ("resolution_content", "expected_error"), + [ + ("not JSON", "GPU resolution manifest is malformed"), + ( + '{"version":1,"status":"pending","render_gid":993,"hosts":{"agent":false,"server":true}}', + "GPU resolution manifest status must be cpu_only or gpu_resolved", + ), + ( + '{"version":1,"status":"gpu_resolved","render_gid":993,"hosts":{"server":true,"server":false}}', + "duplicate JSON key 'server'", + ), + ( + '{"version":1,"status":"gpu_resolved","render_gid":993,"hosts":{"ser\\u0076er":true,"server":false}}', + "duplicate JSON key 'server'", + ), + ], +) +def test_validator_rejects_malformed_pending_or_duplicate_gpu_resolution( + tmp_path: Path, resolution_content: str, expected_error: str +) -> None: + repo = tmp_path / "checkout" + inventory, values, resolution = write_resolved_gpu_artifacts(repo) + resolution.write_text(resolution_content, encoding="utf-8") + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--inventory", + str(inventory), + "--values", + str(values), + "--gpu-resolution", + str(resolution), + ) + + assert result.returncode == 1 + assert expected_error in result.stdout + + +@pytest.mark.parametrize( + ("inventory_content", "expected_error"), + [ + ( + """k3s_cluster: + children: + server: + hosts: + server: + ansible_host: 192.168.1.10 + agent: + hosts: + agent: + ansible_host: 192.168.1.11 + auplc_gpu_access_enabled: false + vars: + auplc_render_gid: 993 +""", + "inventory host 'server' must define exactly one auplc_gpu_access_enabled", + ), + ( + """k3s_cluster: + children: + server: + hosts: + server: + ansible_host: 192.168.1.10 + auplc_gpu_access_enabled: yes + agent: + hosts: + agent: + ansible_host: 192.168.1.11 + auplc_gpu_access_enabled: false + vars: + auplc_render_gid: 993 +""", + "inventory host 'server' has malformed auplc_gpu_access_enabled", + ), + ( + """k3s_cluster: + children: + server: + hosts: + server: + ansible_host: 192.168.1.10 + auplc_gpu_access_enabled: true + auplc_gpu_access_enabled: false + agent: + hosts: + agent: + ansible_host: 192.168.1.11 + auplc_gpu_access_enabled: false + vars: + auplc_render_gid: 993 +""", + "inventory host 'server' must define exactly one auplc_gpu_access_enabled", + ), + ], +) +def test_validator_rejects_missing_malformed_or_duplicate_inventory_host_booleans( + tmp_path: Path, inventory_content: str, expected_error: str +) -> None: + repo = tmp_path / "checkout" + inventory, values, resolution = write_resolved_gpu_artifacts(repo) + inventory.write_text(inventory_content, encoding="utf-8") + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--inventory", + str(inventory), + "--values", + str(values), + "--gpu-resolution", + str(resolution), + ) + + assert result.returncode == 1 + assert expected_error in result.stdout + + +def test_validator_rejects_missing_generated_gpu_resolution_artifact(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + inventory, values, _ = write_resolved_gpu_artifacts(repo) + missing_resolution = repo / "generated/missing-gpu-access-resolution.json" + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--inventory", + str(inventory), + "--values", + str(values), + "--gpu-resolution", + str(missing_resolution), + ) + + assert result.returncode == 1 + assert "GPU resolution manifest not found" in result.stdout + + +def test_validator_rejects_mismatched_host_boolean_and_render_gid(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + inventory, values, resolution = write_resolved_gpu_artifacts(repo) + values.write_text( + """custom: + gpuAccess: + renderGid: 994 + resources: + metadata: {} +""", + encoding="utf-8", + ) + resolution.write_text( + json.dumps( + { + "version": 1, + "status": "gpu_resolved", + "render_gid": 993, + "hosts": {"agent": True, "server": True}, + } + ), + encoding="utf-8", + ) + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--inventory", + str(inventory), + "--values", + str(values), + "--gpu-resolution", + str(resolution), + ) + + assert result.returncode == 1 + assert "inventory host 'agent' GPU access boolean disagrees" in result.stdout + assert "render GIDs disagree" in result.stdout + + +def test_validator_rejects_pxe_rootfs_boolean_and_gid_mismatch(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + inventory = write_file( + repo / "generated/inventory.yml", + """k3s_cluster: + children: + server: + hosts: + server: + ansible_host: 192.168.1.10 + auplc_gpu_access_enabled: false + agent: + hosts: {} + vars: + auplc_render_gid: 993 +""", + ) + values = write_file(repo / "generated/values-basic-example.yaml", "custom:\n gpuAccess:\n renderGid: 993\n") + resolution = write_file( + repo / "generated/gpu-access-resolution.json", + json.dumps( + { + "version": 1, + "status": "gpu_resolved", + "render_gid": 993, + "hosts": {"server": False}, + "pxe_rootfs": {"gpu_access_enabled": True, "render_gid": 993}, + } + ), + ) + pxe_vars = write_file( + repo / "generated/pb-pxe-controller.vars.yml", + """pxe_network_interface: eno1 +pxe_subnet: 192.168.1.0/24 +pxe_controller_ip: 192.168.1.10 +pxe_dns_servers: 8.8.8.8 +pxe_k3s_server_ips: [192.168.1.10] +pxe_rootfs_authorized_keys: [ssh-ed25519-AAA] +pxe_k3s_version: v1.32.3+k3s1 +auplc_render_gid: 994 +pxe_gpu_access_enabled: false +""", + ) + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "pxe-diskless", + "--inventory", + str(inventory), + "--values", + str(values), + "--gpu-resolution", + str(resolution), + "--pxe-vars", + str(pxe_vars), + ) + + assert result.returncode == 1 + assert "pxe_gpu_access_enabled disagrees" in result.stdout + assert "PXE auplc_render_gid disagrees" in result.stdout + + def test_generator_rejects_unknown_accelerator_keys_before_writing_artifacts(tmp_path: Path) -> None: spec = write_file( tmp_path / "spec.json", @@ -824,13 +1317,13 @@ def generator_spec(topology: str = "ssh-preinstalled", accelerators: object | No spec["accelerators"] = accelerators if topology == "pxe-diskless": spec["network"] = {"interface": "enp1s0", "subnet": "192.168.1.0/24"} - spec["pxe"] = {"authorized_keys": ["ssh-ed25519 AAAA test@example"]} + spec["pxe"] = {"authorized_keys": ["ssh-ed25519 AAAA test@example"], "diskless_agents_have_amd_gpus": False} return spec def test_generator_validates_all_pxe_requirements_before_writing(tmp_path: Path) -> None: spec = generator_spec("pxe-diskless") - spec["pxe"] = {"authorized_keys": []} + spec["pxe"] = {"authorized_keys": [], "diskless_agents_have_amd_gpus": False} spec_path = write_file(tmp_path / "spec.json", json.dumps(spec)) out_dir = tmp_path / "generated" @@ -924,7 +1417,7 @@ def test_generator_force_replaces_symlink_entry_without_following_target(tmp_pat def test_generator_force_failure_restores_all_original_destination_types( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - module = load_generator_module() + module = load_deploy_module("deploy_artifact_store", ARTIFACT_STORE) inventory = write_file(tmp_path / "inventory.yml", "old inventory\n") pxe_vars = tmp_path / "pb-pxe-controller.vars.yml" pxe_vars.mkdir() @@ -956,6 +1449,58 @@ def fail_late_replace(source, destination): assert values_target.read_text(encoding="utf-8") == "old symlink target\n" +def test_artifact_store_rolls_back_destination_when_staged_unlink_fails( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + module = load_deploy_module("deploy_artifact_store_unlink", ARTIFACT_STORE) + destination = tmp_path / "inventory.yml" + original_unlink = module.os.unlink + failed = False + + def fail_first_staged_unlink(path, *args, **kwargs): + nonlocal failed + if not failed and Path(path).name.startswith(".inventory.yml."): + failed = True + raise OSError("injected staged unlink failure") + return original_unlink(path, *args, **kwargs) + + monkeypatch.setattr(module.os, "unlink", fail_first_staged_unlink) + + with pytest.raises(SystemExit): + module.publish_artifacts([(destination, "new inventory\n", 0o600, True)], force=False) + + assert not destination.exists() + + +@pytest.mark.parametrize("force", (False, True)) +def test_artifact_store_rolls_back_destination_when_parent_fsync_fails( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, force: bool +) -> None: + module = load_deploy_module(f"deploy_artifact_store_fsync_{force}", ARTIFACT_STORE) + destination = tmp_path / "inventory.yml" + if force: + destination.write_text("old inventory\n", encoding="utf-8") + original_fsync_parent = module._fsync_parent + calls = 0 + + def fail_after_publication(path): + nonlocal calls + calls += 1 + if calls == (2 if force else 1): + raise OSError("injected parent fsync failure") + return original_fsync_parent(path) + + monkeypatch.setattr(module, "_fsync_parent", fail_after_publication) + + with pytest.raises(SystemExit): + module.publish_artifacts([(destination, "new inventory\n", 0o600, True)], force=force) + + if force: + assert destination.read_text(encoding="utf-8") == "old inventory\n" + else: + assert not destination.exists() + + def test_generated_overlay_activates_selected_accelerators_for_validation(tmp_path: Path) -> None: repo = tmp_path / "checkout" base_values = write_file( @@ -1007,3 +1552,122 @@ def test_checkout_root_helper_path_is_a_runnable_public_cli() -> None: assert result.returncode == 0, result.stdout + result.stderr assert '"topology": "pxe-diskless | ssh-preinstalled"' in result.stdout + + +def test_generator_print_schema_is_byte_stable() -> None: + result = run_script(GEN_CONFIGS, "--print-schema") + + assert result.returncode == 0, result.stdout + result.stderr + assert result.stderr == "" + assert result.stdout == json.dumps(EXPECTED_GENERATOR_SCHEMA, indent=2) + "\n" + + +def test_generator_exits_with_usage_error_when_spec_is_omitted() -> None: + result = run_script(GEN_CONFIGS) + + assert result.returncode == 2 + assert result.stdout == "" + assert result.stderr == "gen_configs: --spec is required (or use --print-schema)\n" + + +def test_generator_replaces_colliding_artifacts_when_force_is_given(tmp_path: Path) -> None: + spec_path = write_file(tmp_path / "spec.json", json.dumps(generator_spec("pxe-diskless"))) + token_path = write_file(tmp_path / "token.txt", "characterization-token\n") + out_dir = tmp_path / "generated" + write_file(out_dir / "inventory.yml", "old inventory\n") + write_file(out_dir / "pb-pxe-controller.vars.yml", "old pxe vars\n") + write_file(out_dir / "values-basic-example.yaml", "old values\n") + + result = run_script( + GEN_CONFIGS, + "--spec", + str(spec_path), + "--out-dir", + str(out_dir), + "--token-file", + str(token_path), + "--force", + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert "old inventory" not in (out_dir / "inventory.yml").read_text(encoding="utf-8") + assert "old pxe vars" not in (out_dir / "pb-pxe-controller.vars.yml").read_text(encoding="utf-8") + assert "old values" not in (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") + assert os.stat(out_dir / "inventory.yml").st_mode & 0o777 == 0o600 + assert os.stat(out_dir / "pb-pxe-controller.vars.yml").st_mode & 0o777 == 0o600 + assert os.stat(out_dir / "values-basic-example.yaml").st_mode & 0o777 == 0o644 + + +def test_generator_exposes_extracted_generation_and_artifact_modules() -> None: + generation = load_deploy_module("deploy_config_generation", CONFIG_GENERATION) + artifacts = load_deploy_module("deploy_artifact_store", ARTIFACT_STORE) + + assert generation.SCHEMA == EXPECTED_GENERATOR_SCHEMA + assert generation.validate_spec(generator_spec()) == "ssh-preinstalled" + assert callable(generation.render_inventory) + assert callable(generation.render_pxe_vars) + assert callable(generation.render_values) + assert callable(artifacts.preflight_destinations) + assert callable(artifacts.publish_artifacts) + + +def test_generator_rejects_legacy_public_gpu_policy_fields_before_discovery(tmp_path: Path) -> None: + spec = generator_spec() + spec["render_gid"] = 1055 + spec["gpu_access"] = {"hosts": [], "pxe_rootfs_enabled": False} + spec_path = write_file(tmp_path / "spec.json", json.dumps(spec)) + out_dir = tmp_path / "generated" + + result = run_script(GEN_CONFIGS, "--spec", str(spec_path), "--out-dir", str(out_dir)) + + assert result.returncode == 1 + assert "spec.render_gid is no longer accepted" in result.stderr + assert not out_dir.exists() + + +def test_generator_uses_fake_ansible_discovery_to_publish_resolved_ssh_policy(tmp_path: Path) -> None: + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + fake_ansible = fake_bin / "ansible-playbook" + fake_ansible.write_text( + r"""#!/usr/bin/env python3 +import json +import pathlib +import sys +args = sys.argv[1:] +output = next(arg.split('=', 1)[1] for arg in args if arg.startswith('gpu_access_discovery_output_path=')) +def host(name, bdf): + return { + 'host': name, 'reachable': True, + 'lspci': {'rc': 0, 'stdout': bdf}, 'sysfs': {'rc': 0, 'stdout': bdf}, + 'render_group': {'rc': 0, 'stdout': 'render:x:993:\\n'}, + 'groups': {'rc': 0, 'stdout': 'render:x:993:\\n'}, + 'state': {'stat_success': True, 'content_success': True, 'exists': False, 'regular': False, 'symlink': False, 'content': ''}, + 'rule': {'stat_success': True, 'content_success': True, 'exists': False, 'regular': False, 'symlink': False, 'content': ''}, + 'legacy_rules': {key: {'stat_success': True, 'content_success': True, 'exists': False, 'regular': False, 'symlink': False, 'content': ''} for key in ('kfd', 'amdgpu', 'rocm_devices')}, + } +pathlib.Path(output).write_text(json.dumps({'version': 2, 'hosts': [host('server', '0000:03:00.0'), host('agent', '')]}), encoding='utf-8') +""", + encoding="utf-8", + ) + fake_ansible.chmod(0o755) + spec = generator_spec() + spec["agents"] = [{"name": "agent", "ip": "192.168.1.11"}] + spec_path = write_file(tmp_path / "spec.json", json.dumps(spec)) + out_dir = tmp_path / "generated" + result = subprocess.run( + [sys.executable, str(GEN_CONFIGS), "--spec", str(spec_path), "--out-dir", str(out_dir)], + capture_output=True, + check=False, + env={**os.environ, "PATH": f"{fake_bin}:{os.environ['PATH']}"}, + text=True, + ) + + assert result.returncode == 0, result.stdout + result.stderr + inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") + assert "auplc_render_gid: 993" in inventory + assert inventory.count("auplc_gpu_access_enabled: true") == 1 + assert inventory.count("auplc_gpu_access_enabled: false") == 1 + assert "renderGid: 993" in (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") + manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) + assert manifest["hosts"] == {"agent": False, "server": True} From 09c10f0a6760e13b1819a9b1a93494812ef898fc Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:18 +0800 Subject: [PATCH 18/65] feat(ansible): admit and finalize PXE GPU rootfs --- .../ansible/playbooks/pb-pxe-controller.yml | 38 ++ .../roles/pxe_controller/defaults/main.yml | 3 + .../roles/pxe_controller/tasks/gpu_access.yml | 296 ++++++++++++ .../roles/pxe_controller/tasks/main.yml | 87 ++++ .../templates/chroot-setup.sh.j2 | 6 - tests/skills/test_gpu_access_role.py | 455 ++++++++++++++++++ 6 files changed, 879 insertions(+), 6 deletions(-) create mode 100644 deploy/ansible/roles/pxe_controller/tasks/gpu_access.yml create mode 100644 tests/skills/test_gpu_access_role.py diff --git a/deploy/ansible/playbooks/pb-pxe-controller.yml b/deploy/ansible/playbooks/pb-pxe-controller.yml index 425b939e..250508cc 100644 --- a/deploy/ansible/playbooks/pb-pxe-controller.yml +++ b/deploy/ansible/playbooks/pb-pxe-controller.yml @@ -102,3 +102,41 @@ roles: - role: pxe_controller + + post_tasks: + - name: Write private PXE finalizer handoff from resolved rootfs facts + ansible.builtin.copy: + content: >- + {{ { + 'version': 1, + 'generation': pxe_finalizer_generation, + 'spec_sha256': pxe_finalizer_spec_sha256, + 'topology': 'pxe-diskless', + 'pxe_gpu_access_enabled': pxe_gpu_access_enabled | bool, + 'render_gid': _pxe_resolved_render_gid | default(none) + } | to_json }} + dest: "{{ pxe_finalizer_handoff }}" + mode: "0600" + delegate_to: localhost + run_once: true + become: false + no_log: true + when: pxe_finalizer_context is defined + + - name: Finalize generated PXE GPU policy from resolved rootfs facts + ansible.builtin.command: + argv: + - "{{ pxe_finalizer_script }}" + - --finalize-pxe + - --out-dir + - "{{ pxe_finalizer_context | dirname }}" + - --context + - "{{ pxe_finalizer_context }}" + - --handoff + - "{{ pxe_finalizer_handoff }}" + delegate_to: localhost + run_once: true + become: false + no_log: true + changed_when: false + when: pxe_finalizer_context is defined diff --git a/deploy/ansible/roles/pxe_controller/defaults/main.yml b/deploy/ansible/roles/pxe_controller/defaults/main.yml index e381140f..66ec003c 100644 --- a/deploy/ansible/roles/pxe_controller/defaults/main.yml +++ b/deploy/ansible/roles/pxe_controller/defaults/main.yml @@ -85,11 +85,14 @@ pxe_apt_mirror: "http://tw.archive.ubuntu.com/ubuntu" pxe_rootfs_force_rebuild: true # Run apt-get upgrade inside rootfs during chroot setup pxe_rootfs_upgrade: false +# Explicitly enable GPU access only for a rootfs intended for GPU workers. +pxe_gpu_access_enabled: false # ============================================================ # Paths # ============================================================ pxe_nfs_root: "/srv/nfs/rootfs" +pxe_nfs_allowed_root: "/srv/nfs" pxe_tftp_root: "/srv/tftp" pxe_web_root: "/var/www/html" diff --git a/deploy/ansible/roles/pxe_controller/tasks/gpu_access.yml b/deploy/ansible/roles/pxe_controller/tasks/gpu_access.yml new file mode 100644 index 00000000..06b5d5f8 --- /dev/null +++ b/deploy/ansible/roles/pxe_controller/tasks/gpu_access.yml @@ -0,0 +1,296 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +--- +- name: Record PXE GPU admission disposition + ansible.builtin.set_fact: + _pxe_rootfs_disposition: "{{ 'fresh' if _pxe_rootfs_rebuilt_this_run | bool else 'retained' }}" + _pxe_unanimous_live_render_gid: "{{ auplc_render_gid if auplc_render_gid is defined and auplc_render_gid is not none else none }}" + _pxe_resolved_render_gid: null + +- name: Assert PXE GPU admission phase + ansible.builtin.assert: + that: pxe_gpu_admission_phase in ['retained-read-only', 'final'] + fail_msg: PXE GPU admission phase is invalid. + +- name: Validate optional unanimous live render GID + ansible.builtin.assert: + that: + - _pxe_unanimous_live_render_gid is integer + - _pxe_unanimous_live_render_gid >= 1 + - _pxe_unanimous_live_render_gid <= 4294967294 + fail_msg: auplc_render_gid must be an integer between 1 and 4294967294 when supplied for a PXE GPU rootfs. + when: + - pxe_gpu_access_enabled | bool + - _pxe_unanimous_live_render_gid is not none + +- name: Inspect PXE rootfs render group for GPU admission + ansible.builtin.command: + argv: [chroot, "{{ pxe_nfs_root }}", getent, group, render] + register: _pxe_admission_render_group + changed_when: false + failed_when: false + when: pxe_gpu_access_enabled | bool + +- name: Require fresh PXE render group lookup outcome + ansible.builtin.assert: + that: _pxe_admission_render_group.rc in [0, 2] + fail_msg: Unable to determine whether the fresh PXE rootfs has a render group. + when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'fresh' + +- name: Require strict existing fresh PXE render group + ansible.builtin.assert: + that: + - _pxe_admission_render_group.stdout_lines | length == 1 + - _pxe_admission_render_group.stdout.split(':') | length == 4 + - _pxe_admission_render_group.stdout.split(':')[0] == 'render' + - _pxe_admission_render_group.stdout.split(':')[2] is match('^[1-9][0-9]*$') + - _pxe_admission_render_group.stdout.split(':')[2] | int <= 4294967294 + fail_msg: Fresh PXE rootfs render group is malformed. + when: >- + pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'fresh' and + _pxe_admission_render_group.rc == 0 + +- name: Require strict retained PXE render group + ansible.builtin.assert: + that: + - _pxe_admission_render_group.rc == 0 + - _pxe_admission_render_group.stdout_lines | length == 1 + - _pxe_admission_render_group.stdout.split(':') | length == 4 + - _pxe_admission_render_group.stdout.split(':')[0] == 'render' + - _pxe_admission_render_group.stdout.split(':')[2] is match('^[1-9][0-9]*$') + - _pxe_admission_render_group.stdout.split(':')[2] | int <= 4294967294 + fail_msg: Retained PXE rootfs must already have one valid render group. + when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' + +- name: Record existing PXE render GID + ansible.builtin.set_fact: + _pxe_existing_render_gid: "{{ _pxe_admission_render_group.stdout.split(':')[2] | int }}" + when: + - pxe_gpu_access_enabled | bool + - _pxe_admission_render_group.rc == 0 + +- name: List fresh PXE rootfs groups before render GID creation + ansible.builtin.command: + argv: [chroot, "{{ pxe_nfs_root }}", getent, group] + register: _pxe_fresh_groups + changed_when: false + failed_when: false + when: + - pxe_gpu_access_enabled | bool + - _pxe_rootfs_disposition == 'fresh' + - _pxe_existing_render_gid | default(none) is none + - _pxe_unanimous_live_render_gid is not none + +- name: Reject fresh PXE render GID collision + ansible.builtin.assert: + that: + - _pxe_fresh_groups.rc == 0 + - >- + _pxe_fresh_groups.stdout_lines + | select('match', '^[^:]*:[^:]*:' ~ (_pxe_unanimous_live_render_gid | string) ~ ':') + | reject('match', '^render:') | list | length == 0 + fail_msg: Fresh PXE rootfs render GID is already assigned to another group. + when: + - pxe_gpu_access_enabled | bool + - _pxe_rootfs_disposition == 'fresh' + - _pxe_existing_render_gid | default(none) is none + - _pxe_unanimous_live_render_gid is not none + +- name: Create missing fresh PXE render group + ansible.builtin.command: + argv: >- + {{ ['chroot', pxe_nfs_root, 'groupadd', '--system', '-g', (_pxe_unanimous_live_render_gid | string), 'render'] + if _pxe_unanimous_live_render_gid is not none + else ['chroot', pxe_nfs_root, 'groupadd', '--system', 'render'] }} + changed_when: true + when: + - pxe_gpu_access_enabled | bool + - _pxe_rootfs_disposition == 'fresh' + - _pxe_existing_render_gid | default(none) is none + +- name: Read fresh PXE render group after creation + ansible.builtin.command: + argv: [chroot, "{{ pxe_nfs_root }}", getent, group, render] + register: _pxe_created_render_group + changed_when: false + when: + - pxe_gpu_access_enabled | bool + - _pxe_rootfs_disposition == 'fresh' + - _pxe_existing_render_gid | default(none) is none + +- name: Resolve newly created fresh PXE render GID + ansible.builtin.set_fact: + _pxe_resolved_render_gid: "{{ _pxe_created_render_group.stdout.split(':')[2] | int }}" + when: + - pxe_gpu_access_enabled | bool + - _pxe_rootfs_disposition == 'fresh' + - _pxe_existing_render_gid | default(none) is none + +- name: Resolve existing fresh PXE render GID + ansible.builtin.set_fact: + _pxe_resolved_render_gid: "{{ _pxe_existing_render_gid }}" + when: + - pxe_gpu_access_enabled | bool + - _pxe_rootfs_disposition == 'fresh' + - _pxe_existing_render_gid | default(none) is not none + +- name: Resolve retained PXE render GID + ansible.builtin.set_fact: + _pxe_resolved_render_gid: "{{ _pxe_existing_render_gid }}" + when: + - pxe_gpu_access_enabled | bool + - _pxe_rootfs_disposition == 'retained' + +- name: List retained PXE rootfs groups for render GID collision check + ansible.builtin.command: + argv: [chroot, "{{ pxe_nfs_root }}", getent, group] + register: _pxe_retained_groups + changed_when: false + failed_when: false + when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' + +- name: Reject retained PXE render GID collision + ansible.builtin.assert: + that: + - _pxe_retained_groups.rc == 0 + - >- + _pxe_retained_groups.stdout_lines + | select('match', '^[^:]*:[^:]*:' ~ (_pxe_resolved_render_gid | string) ~ ':') + | reject('match', '^render:') | list | length == 0 + fail_msg: Retained PXE rootfs render GID is already assigned to another group. + when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' + +- name: Require retained PXE render GID matches unanimous live GID + ansible.builtin.assert: + that: _pxe_resolved_render_gid == _pxe_unanimous_live_render_gid + fail_msg: Retained PXE rootfs render GID differs from the supplied unanimous live render GID; rebuild or migrate it separately. + when: + - pxe_gpu_access_enabled | bool + - _pxe_rootfs_disposition == 'retained' + - _pxe_unanimous_live_render_gid is not none + +- name: Inspect retained PXE canonical GPU access parents + ansible.builtin.stat: + path: "{{ pxe_nfs_root }}{{ item }}" + follow: false + loop: + - /etc + - /etc/udev + - /etc/udev/rules.d + - /var + - /var/lib + - /var/lib/auplc + register: _pxe_retained_canonical_gpu_parent_stats + when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' + +- name: Require retained PXE canonical GPU access parents + ansible.builtin.assert: + that: + - item.stat.exists + - item.stat.isdir + - not item.stat.islnk + fail_msg: "Retained PXE rootfs has an unsafe canonical GPU access parent: {{ item.item }}" + loop: "{{ _pxe_retained_canonical_gpu_parent_stats.results }}" + when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' + +- name: Inspect retained PXE GPU policy paths + ansible.builtin.stat: + path: "{{ pxe_nfs_root }}{{ item }}" + follow: false + loop: + - /etc/udev/rules.d/70-kfd.rules + - /etc/udev/rules.d/70-amdgpu.rules + - /etc/udev/rules.d/70-rocm-devices.rules + - /etc/udev/rules.d/70-auplc-gpu-access.rules + - /var/lib/auplc/gpu-access.json + register: _pxe_retained_gpu_policy_stats + when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' + +- name: Require retained PXE legacy GPU rules absent + ansible.builtin.assert: + that: not item.stat.exists + fail_msg: "Retained PXE rootfs has a legacy GPU rule requiring a separate migration: {{ item.item }}" + loop: "{{ _pxe_retained_gpu_policy_stats.results[:3] }}" + when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' + +- name: Require retained PXE canonical GPU destinations + ansible.builtin.assert: + that: + - item.stat.exists + - item.stat.isreg + - not item.stat.islnk + fail_msg: "Retained PXE rootfs requires an exact canonical GPU access destination: {{ item.item }}" + loop: "{{ _pxe_retained_gpu_policy_stats.results[3:] }}" + when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' + +- name: Read retained PXE canonical GPU access destinations + ansible.builtin.slurp: + src: "{{ item.item }}" + loop: "{{ _pxe_retained_gpu_policy_stats.results[3:] }}" + register: _pxe_retained_canonical_gpu_destinations + when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' + +- name: Define retained PXE canonical GPU rule + ansible.builtin.set_fact: + _pxe_retained_canonical_rule: | + # Managed by auplc-installer: AMD GPU device access. + KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0660" + SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0660" + when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' + +- name: Require retained PXE canonical GPU rule + ansible.builtin.assert: + that: (item.content | b64decode) == _pxe_retained_canonical_rule + fail_msg: "Retained PXE rootfs has a non-canonical GPU access rule: {{ item.item.item }}" + loop: "{{ _pxe_retained_canonical_gpu_destinations.results }}" + when: + - pxe_gpu_access_enabled | bool + - _pxe_rootfs_disposition == 'retained' + - item.item.item.endswith('70-auplc-gpu-access.rules') + +- name: Parse retained PXE canonical GPU state + ansible.builtin.set_fact: + _pxe_retained_canonical_state: "{{ item.content | b64decode | auplc_from_json_strict }}" + loop: "{{ _pxe_retained_canonical_gpu_destinations.results }}" + when: + - pxe_gpu_access_enabled | bool + - _pxe_rootfs_disposition == 'retained' + - item.item.item.endswith('gpu-access.json') + +- name: Require retained PXE canonical GPU state + ansible.builtin.assert: + that: + - _pxe_retained_canonical_state is mapping + - _pxe_retained_canonical_state.keys() | list | sort == ['renderGid', 'version'] + - _pxe_retained_canonical_state.version is integer + - _pxe_retained_canonical_state.version == 1 + - _pxe_retained_canonical_state.renderGid is integer + - _pxe_retained_canonical_state.renderGid == _pxe_resolved_render_gid + fail_msg: Retained PXE rootfs has a non-canonical GPU access state. + when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' + +- name: Preflight GPU access after final PXE re-preflight + ansible.builtin.include_role: + name: gpu_access + tasks_from: preflight + vars: + auplc_rootfs_path: "{{ pxe_nfs_root }}" + auplc_rootfs_allowed_root: "{{ pxe_nfs_allowed_root }}" + auplc_render_gid: "{{ _pxe_resolved_render_gid }}" + auplc_normalize_render_gid: "{{ _pxe_rootfs_disposition == 'fresh' }}" + when: + - pxe_gpu_access_enabled | bool + - pxe_gpu_admission_phase == 'final' + +- name: Apply GPU access after final PXE re-preflight + ansible.builtin.include_role: + name: gpu_access + tasks_from: apply + vars: + auplc_rootfs_path: "{{ pxe_nfs_root }}" + auplc_rootfs_allowed_root: "{{ pxe_nfs_allowed_root }}" + auplc_render_gid: "{{ _pxe_resolved_render_gid }}" + auplc_normalize_render_gid: "{{ _pxe_rootfs_disposition == 'fresh' }}" + when: + - pxe_gpu_access_enabled | bool + - pxe_gpu_admission_phase == 'final' diff --git a/deploy/ansible/roles/pxe_controller/tasks/main.yml b/deploy/ansible/roles/pxe_controller/tasks/main.yml index b16e2af0..b5217f60 100644 --- a/deploy/ansible/roles/pxe_controller/tasks/main.yml +++ b/deploy/ansible/roles/pxe_controller/tasks/main.yml @@ -82,6 +82,86 @@ # 2. Build NFS rootfs with debootstrap # ========================================================== +- name: Validate PXE rootfs path syntax before lifecycle changes + ansible.builtin.assert: + that: + - pxe_nfs_root is string + - pxe_nfs_root is match('^/') + - pxe_nfs_root != '/' + - "'..' not in pxe_nfs_root.split('/')" + - pxe_nfs_allowed_root is string + - pxe_nfs_allowed_root is match('^/') + fail_msg: pxe_nfs_root and pxe_nfs_allowed_root must be absolute non-root paths without traversal. + +- name: Canonicalize PXE rootfs before lifecycle changes + ansible.builtin.command: + argv: [realpath, --canonicalize-missing, "{{ pxe_nfs_root }}"] + register: _pxe_canonical_nfs_root_result + changed_when: false + +- name: Canonicalize trusted PXE rootfs parent before lifecycle changes + ansible.builtin.command: + argv: [realpath, --canonicalize-existing, "{{ pxe_nfs_allowed_root }}"] + register: _pxe_canonical_nfs_allowed_root + changed_when: false + +- name: Inspect PXE rootfs path before canonical lifecycle changes + ansible.builtin.stat: + path: "{{ pxe_nfs_root }}" + follow: false + register: _pxe_rootfs_lstat + +- name: Constrain canonical PXE rootfs before lifecycle changes + ansible.builtin.assert: + that: + - not _pxe_rootfs_lstat.stat.exists or not _pxe_rootfs_lstat.stat.islnk + - _pxe_canonical_nfs_root_result.stdout != '/' + - _pxe_canonical_nfs_root_result.stdout == pxe_nfs_root + - _pxe_canonical_nfs_root_result.stdout.startswith(_pxe_canonical_nfs_allowed_root.stdout + '/') + fail_msg: pxe_nfs_root must be a non-symlink descendant of pxe_nfs_allowed_root. + +- name: Record canonical PXE rootfs for lifecycle operations + ansible.builtin.set_fact: + _pxe_canonical_nfs_root: "{{ _pxe_canonical_nfs_root_result.stdout }}" + pxe_nfs_root: "{{ _pxe_canonical_nfs_root_result.stdout }}" + +- name: Require existing PXE rootfs is a directory + ansible.builtin.assert: + that: + - not _pxe_rootfs_lstat.stat.exists or _pxe_rootfs_lstat.stat.isdir + fail_msg: "pxe_nfs_root must be a directory when it already exists: {{ pxe_nfs_root }}" + +- name: Inspect PXE rootfs readiness before lifecycle changes + ansible.builtin.stat: + path: "{{ pxe_nfs_root }}/bin/bash" + follow: false + register: _pxe_rootfs_start + +- name: Record PXE rootfs state before lifecycle changes + ansible.builtin.set_fact: + _pxe_rootfs_existed_at_start: "{{ _pxe_rootfs_lstat.stat.exists | bool }}" + _pxe_rootfs_rebuilt_this_run: >- + {{ (pxe_rootfs_force_rebuild | bool) or not (_pxe_rootfs_lstat.stat.exists | bool) }} + +- name: Require incomplete PXE rootfs force rebuild + ansible.builtin.assert: + that: + - >- + not (_pxe_rootfs_lstat.stat.exists | bool) or + (_pxe_rootfs_start.stat.exists | bool) or + (pxe_rootfs_force_rebuild | bool) + fail_msg: >- + Existing PXE rootfs is incomplete and must be rebuilt with + pxe_rootfs_force_rebuild=true; debootstrap will not modify it in place. + +- name: Admit retained PXE GPU rootfs read-only before lifecycle changes + ansible.builtin.include_tasks: gpu_access.yml + vars: + pxe_gpu_admission_phase: retained-read-only + when: + - pxe_gpu_access_enabled | bool + - not (_pxe_rootfs_rebuilt_this_run | bool) + - name: Stop NFS before rootfs rebuild when: pxe_rootfs_force_rebuild | bool ansible.builtin.systemd: @@ -229,6 +309,13 @@ path: "{{ pxe_nfs_root }}/tmp/chroot-setup.sh" state: absent +- name: Re-preflight PXE GPU rootfs before TFTP + ansible.builtin.include_tasks: gpu_access.yml + vars: + pxe_gpu_admission_phase: final + when: + - pxe_gpu_access_enabled | bool + # ========================================================== # 6. Copy kernel and initrd to TFTP # ========================================================== diff --git a/deploy/ansible/roles/pxe_controller/templates/chroot-setup.sh.j2 b/deploy/ansible/roles/pxe_controller/templates/chroot-setup.sh.j2 index 50ea0867..c504a497 100644 --- a/deploy/ansible/roles/pxe_controller/templates/chroot-setup.sh.j2 +++ b/deploy/ansible/roles/pxe_controller/templates/chroot-setup.sh.j2 @@ -50,12 +50,6 @@ echo "PermitRootLogin yes" > /etc/ssh/sshd_config.d/allow-root.conf echo "PermitRootLogin prohibit-password" > /etc/ssh/sshd_config.d/allow-root.conf {% endif %} -# -- GPU udev rules (let containers access AMD GPUs) -- -tee /etc/udev/rules.d/70-amdgpu.rules << RULES -KERNEL=="kfd", MODE="0666" -KERNEL=="renderD[0-9]*", MODE="0666" -RULES - # -- Disable systemd-networkd (kernel ip=dhcp handles NFS root networking) -- rm -f /etc/netplan/*.yaml systemctl disable systemd-networkd 2>/dev/null || true diff --git a/tests/skills/test_gpu_access_role.py b/tests/skills/test_gpu_access_role.py new file mode 100644 index 00000000..12f76505 --- /dev/null +++ b/tests/skills/test_gpu_access_role.py @@ -0,0 +1,455 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +"""Canonical artifact tests for the multi-node GPU access role.""" + +from pathlib import Path + +import pytest +from ansible.errors import AnsibleFilterError +from jinja2 import Environment + +from deploy.ansible.filter_plugins.auplc_json import ( + DuplicateJsonKeyError, + _reject_duplicate_keys, + auplc_from_json_strict, +) + +ROOT = Path(__file__).resolve().parents[2] +ANSIBLE = ROOT / "deploy" / "ansible" +GPU_ACCESS_ROLE = ANSIBLE / "roles" / "gpu_access" +PXE_CONTROLLER_ROLE = ANSIBLE / "roles" / "pxe_controller" +PXE_GPU_ACCESS_TASKS = PXE_CONTROLLER_ROLE / "tasks" / "gpu_access.yml" + + +def read(path: Path) -> str: + return path.read_text(encoding="utf-8") + + +def test_jinja_integer_test_rejects_boolean_and_float_state_values() -> None: + template = Environment().from_string("{% if value is integer %}integer{% else %}invalid{% endif %}") + + assert template.render(value=993) == "integer" + assert template.render(value=True) == "invalid" + assert template.render(value=993.0) == "invalid" + + +def test_strict_json_filter_parses_canonical_gpu_access_state() -> None: + assert auplc_from_json_strict('{"renderGid":993,"version":1}\n') == { + "renderGid": 993, + "version": 1, + } + + +def test_duplicate_json_key_error_preserves_typed_key() -> None: + with pytest.raises(DuplicateJsonKeyError) as error: + _reject_duplicate_keys([("version", 1), ("version", 2)]) + + assert error.value.key == "version" + assert str(error.value) == "Duplicate JSON object key: 'version'" + + +@pytest.mark.parametrize( + "value", + [ + '{"renderGid":1,"renderGid":993,"version":1}', + '{"renderGid":1,"render\\u0047id":993,"version":1}', + '{"renderGid":1,"version":1,"version":2}', + '{"outer":{"version":1,"version":2}}', + ], +) +def test_strict_json_filter_rejects_semantic_duplicate_keys(value: str) -> None: + with pytest.raises(AnsibleFilterError, match="^Invalid JSON value$"): + auplc_from_json_strict(value) + + +@pytest.mark.parametrize("value", ["{", '{"renderGid":1,}']) +def test_strict_json_filter_rejects_malformed_json(value: str) -> None: + with pytest.raises(AnsibleFilterError, match="^Invalid JSON value$"): + auplc_from_json_strict(value) + + +def test_gpu_access_role_renders_the_unified_render_gid_contract() -> None: + defaults = read(GPU_ACCESS_ROLE / "defaults" / "main.yml") + tasks = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") + read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") + rules = read(GPU_ACCESS_ROLE / "templates" / "70-auplc-gpu-access.rules.j2") + state = read(GPU_ACCESS_ROLE / "templates" / "gpu-access.json.j2") + + assert "auplc_render_gid: null" in defaults + assert "auplc_normalize_render_gid: false" in defaults + assert 'auplc_rootfs_path: ""' in defaults + assert "getent" in tasks + assert "groupmod" in tasks + assert "auplc_normalize_render_gid" in tasks + assert "_auplc_all_groups" in tasks + assert "reject('match', '^render:')" in tasks + assert "_auplc_render_group.stdout.split(':')[2] | int <= 4294967294" in tasks + assert "notify:" not in tasks + assert "Reload live udev rules on every apply" in tasks + assert "Trigger live udev rules on every apply" in tasks + assert "ansible.builtin.group:" not in tasks + assert rules == ( + "# Managed by auplc-installer: AMD GPU device access.\n" + 'KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0660"\n' + 'SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0660"\n' + ) + assert state == '{"renderGid":{{ auplc_render_gid | int }},"version":1}\n' + + +def test_gpu_access_role_is_wired_for_live_hosts_and_pxe_rootfs_without_legacy_udev_paths() -> None: + rocm_playbook = read(ANSIBLE / "playbooks" / "pb-rocm.yml") + udev_playbook = read(ANSIBLE / "playbooks" / "pb-udev.yml") + rocm_tasks = read(ANSIBLE / "roles" / "rocm" / "tasks" / "main.yml") + pxe_tasks = read(ANSIBLE / "roles" / "pxe_controller" / "tasks" / "main.yml") + pxe_gpu_tasks = read(PXE_GPU_ACCESS_TASKS) + pxe_chroot = read(ANSIBLE / "roles" / "pxe_controller" / "templates" / "chroot-setup.sh.j2") + + assert "name: gpu_access" in rocm_playbook + assert "name: gpu_access" in udev_playbook + assert "udev-rocm" not in udev_playbook + assert "render:993" not in rocm_tasks + assert "70-amdgpu.rules" not in rocm_tasks + assert "include_tasks: gpu_access.yml" in pxe_tasks + assert "name: gpu_access" in pxe_gpu_tasks + assert 'auplc_rootfs_path: "{{ pxe_nfs_root }}"' in pxe_gpu_tasks + assert "0666" not in pxe_chroot + assert not (ANSIBLE / "roles" / "udev" / "main.yml").exists() + + +def test_gpu_access_live_host_playbooks_abort_all_hosts_on_preflight_failure() -> None: + rocm_playbook = read(ANSIBLE / "playbooks" / "pb-rocm.yml") + udev_playbook = read(ANSIBLE / "playbooks" / "pb-udev.yml") + + assert "any_errors_fatal: true" in rocm_playbook + assert "any_errors_fatal: true" in udev_playbook + + +def test_gpu_access_role_migrates_only_recognized_legacy_rules_and_reconciles_live_devices() -> None: + tasks = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") + read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") + + assert "70-kfd.rules" in tasks + assert "70-amdgpu.rules" in tasks + assert "contents:" in tasks + assert 'KERNEL==\\"renderD[0-9]*\\", MODE=\\"0666\\"' in tasks + assert "70-rocm-devices.rules" in tasks + assert 'SUBSYSTEM=="kfd", GROUP="render", MODE="0660"' in tasks + assert "islnk" in tasks + assert "ansible.builtin.slurp" in tasks + assert "Define recognized project-owned legacy GPU rules" in tasks + assert "Unexpected legacy GPU rule content" in tasks + assert "udevadm" in tasks + assert "Verify /dev/kfd ownership and mode" in tasks + assert "Verify AMD render node ownership and mode" in tasks + assert "Settle live udev events before inode verification" in tasks + assert ( + tasks.index("Trigger live udev rules on every apply") + < tasks.index("Settle live udev events before inode verification") + < tasks.index("Inspect /dev/kfd after live reconciliation") + ) + assert tasks.index("Verify AMD render node ownership and mode") < tasks.index("Persist target GPU access state") + assert "/sys/class/drm" in tasks + assert "readlink" in tasks + assert "basename" in tasks + assert "DRIVER=amdgpu" not in tasks + assert "notify:" not in tasks + + +def test_gpu_access_role_validates_legacy_rules_before_render_gid_normalization() -> None: + preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") + apply = read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") + tasks = preflight + apply + + assert "Define recognized project-owned legacy GPU rules" in preflight + assert "Inspect recognized project-owned legacy GPU rules" in preflight + assert "Reject legacy GPU rule symlinks and non-regular files" in preflight + assert "Read recognized project-owned legacy GPU rules" in preflight + assert "Reject unexpected legacy GPU rule content" in preflight + assert "follow: false" in preflight + assert "contents:" in preflight + assert 'KERNEL==\\"renderD[0-9]*\\", MODE=\\"0666\\"' in preflight + assert "not item.skipped | default(false)" in preflight + assert "(item.content | b64decode) in item.item.item.contents" in preflight + assert preflight.index("Inspect recognized project-owned legacy GPU rules") < preflight.index( + "Reject legacy GPU rule symlinks and non-regular files" + ) + assert preflight.index("Reject legacy GPU rule symlinks and non-regular files") < preflight.index( + "Read recognized project-owned legacy GPU rules" + ) + assert preflight.index("Read recognized project-owned legacy GPU rules") < preflight.index( + "Reject unexpected legacy GPU rule content" + ) + assert tasks.index("Reject unexpected legacy GPU rule content") < tasks.index("Normalize live render GID") + assert "Remove recognized project-owned legacy GPU rules" in apply + assert "Inspect recognized project-owned legacy GPU rules for apply" in apply + assert "Reject legacy GPU rule symlinks and non-regular files before apply" in apply + assert "Read recognized project-owned legacy GPU rules for apply" in apply + assert "Reject unexpected legacy GPU rule content before apply" in apply + assert "register: _auplc_apply_legacy_gpu_rule_stats" in apply + assert "register: _auplc_apply_legacy_gpu_rule_contents" in apply + assert "_auplc_apply_legacy_gpu_rule_contents.results" in apply + assert "_auplc_legacy_gpu_rule_contents.results" not in apply + assert apply.index("Reject legacy GPU rule symlinks and non-regular files before apply") < apply.index( + "Read recognized project-owned legacy GPU rules for apply" + ) + assert apply.index("Read recognized project-owned legacy GPU rules for apply") < apply.index( + "Reject unexpected legacy GPU rule content before apply" + ) + assert apply.index("Reject unexpected legacy GPU rule content before apply") < apply.index( + "Remove recognized project-owned legacy GPU rules" + ) + assert apply.index("Remove recognized project-owned legacy GPU rules") < apply.index("Normalize live render GID") + + +def test_pxe_rootfs_lifecycle_uses_an_independent_trusted_parent() -> None: + defaults = read(ANSIBLE / "roles" / "pxe_controller" / "defaults" / "main.yml") + tasks = read(ANSIBLE / "roles" / "pxe_controller" / "tasks" / "main.yml") + gpu_tasks = read(PXE_GPU_ACCESS_TASKS) + + assert 'pxe_nfs_allowed_root: "/srv/nfs"' in defaults + assert 'auplc_rootfs_allowed_root: "{{ pxe_nfs_allowed_root }}"' in gpu_tasks + assert "Constrain canonical PXE rootfs before lifecycle changes" in tasks + assert tasks.index("Constrain canonical PXE rootfs before lifecycle changes") < tasks.index( + "Admit retained PXE GPU rootfs read-only before lifecycle changes" + ) + + +def test_pxe_rootfs_is_canonicalized_before_gpu_admission() -> None: + tasks = read(ANSIBLE / "roles" / "pxe_controller" / "tasks" / "main.yml") + + assert "Canonicalize PXE rootfs before lifecycle changes" in tasks + assert tasks.index("Canonicalize PXE rootfs before lifecycle changes") < tasks.index( + "Stop NFS before rootfs rebuild" + ) + assert "_pxe_canonical_nfs_root" in tasks + assert tasks.index("Canonicalize PXE rootfs before lifecycle changes") < tasks.index( + "Admit retained PXE GPU rootfs read-only before lifecycle changes" + ) + + +def test_gpu_access_preflight_refuses_unmanaged_canonical_destinations() -> None: + preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") + + assert "Inspect canonical GPU access destinations" in preflight + assert "70-auplc-gpu-access.rules" in preflight + assert "gpu-access.json" in preflight + assert "follow: false" in preflight + assert "Reject unmanaged canonical GPU access rule" in preflight + assert "Reject invalid canonical GPU access state" in preflight + assert "auplc_from_json_strict" in preflight + assert "| from_json" not in preflight + assert "renderGid" in preflight + assert "version" in preflight + assert 'src: "{{ _auplc_target_root }}{{ item.item }}"' in preflight + assert "Interrupted normalization retry" in preflight + assert "_auplc_current_render_gid == auplc_render_gid" in preflight + assert "_auplc_existing_state.version is integer" in preflight + assert "_auplc_existing_state.renderGid is integer" in preflight + assert "_auplc_existing_state.renderGid | int" not in preflight + + +def test_gpu_access_roles_use_strict_json_for_canonical_state_readers() -> None: + preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") + pxe_tasks = read(PXE_GPU_ACCESS_TASKS) + + assert "Parse existing canonical GPU access state" in preflight + assert "auplc_from_json_strict" in preflight + assert "auplc_from_json_strict" in pxe_tasks + assert "| from_json" not in preflight + assert "| from_json" not in pxe_tasks + + +def test_canonical_gpu_access_state_contract_is_exact_json() -> None: + state = read(GPU_ACCESS_ROLE / "templates" / "gpu-access.json.j2") + + assert state == '{"renderGid":{{ auplc_render_gid | int }},"version":1}\n' + + +def test_gpu_access_role_splits_safe_preflight_and_rootfs_apply() -> None: + defaults = read(GPU_ACCESS_ROLE / "defaults" / "main.yml") + preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") + validation = read(GPU_ACCESS_ROLE / "tasks" / "validate.yml") + pxe_tasks = read(ANSIBLE / "roles" / "pxe_controller" / "tasks" / "main.yml") + pxe_gpu_tasks = read(PXE_GPU_ACCESS_TASKS) + + assert "auplc_gpu_access_enabled: false" in defaults + assert "auplc_rootfs_allowed_root" in defaults + assert "realpath" in validation + assert "auplc_rootfs_path != '/'" in validation + assert "islnk" in preflight + assert "include_tasks: gpu_access.yml" in pxe_tasks + assert "tasks_from: validate" not in pxe_tasks + assert "Constrain canonical PXE rootfs before lifecycle changes" in pxe_tasks + assert pxe_tasks.index("Constrain canonical PXE rootfs before lifecycle changes") < pxe_tasks.index( + "Stop NFS before rootfs rebuild" + ) + assert "tasks_from: preflight" in pxe_gpu_tasks + assert "tasks_from: apply" in pxe_gpu_tasks + assert "auplc_normalize_render_gid:" in pxe_gpu_tasks + assert "pxe_rootfs_force_rebuild | bool" in pxe_tasks + assert "pxe_gpu_access_normalize_render_gid | bool" not in pxe_tasks + assert "pxe_gpu_access_normalize_render_gid" not in read(PXE_CONTROLLER_ROLE / "defaults" / "main.yml") + + +def test_live_playbooks_preflight_gpu_hosts_before_mutating_roles() -> None: + rocm_playbook = read(ANSIBLE / "playbooks" / "pb-rocm.yml") + udev_playbook = read(ANSIBLE / "playbooks" / "pb-udev.yml") + + assert "pre_tasks:" in rocm_playbook + assert "Assert explicit GPU access enablement" in rocm_playbook + assert "auplc_gpu_access_enabled is defined" in rocm_playbook + assert "auplc_gpu_access_enabled is boolean" in rocm_playbook + assert "default(false)" not in rocm_playbook + assert "tasks_from: preflight" in rocm_playbook + assert rocm_playbook.index("tasks_from: preflight") < rocm_playbook.index("- role: rocm") + assert "- role: rocm" in rocm_playbook + assert rocm_playbook.count("auplc_gpu_access_enabled") >= 3 + assert "tasks_from: apply" in rocm_playbook + assert "auplc_gpu_access_enabled" in rocm_playbook + assert "pre_tasks:" in udev_playbook + assert "Assert explicit GPU access enablement" in udev_playbook + assert "auplc_gpu_access_enabled is defined" in udev_playbook + assert "auplc_gpu_access_enabled is boolean" in udev_playbook + assert "default(false)" not in udev_playbook + assert "tasks_from: preflight" in udev_playbook + assert "tasks_from: apply" in udev_playbook + + +def test_gpu_access_discovery_playbook_is_read_only_and_serializes_live_host_evidence() -> None: + playbook = read(ANSIBLE / "playbooks" / "pb-gpu-access-discovery.yml") + + assert "hosts: k3s_cluster" in playbook + assert "gather_facts: false" in playbook + assert "ignore_unreachable: true" in playbook + assert "ansible.builtin.command:" in playbook + assert "ansible.builtin.stat:" in playbook + assert "ansible.builtin.slurp:" in playbook + assert "ansible.builtin.shell:" not in playbook + assert "changed_when: false" in playbook + assert "lspci" in playbook + assert '"1002::0300"' in playbook + assert '"1002::0302"' in playbook + assert '"1002::0380"' in playbook + assert "getent" in playbook + assert "/sys/bus/pci/devices" in playbook + assert "gpu_access_discovery_output_path" in playbook + assert "delegate_to: localhost" in playbook + assert "ansible.builtin.copy:" in playbook + assert "to_json" in playbook + assert "stat_success" in playbook + assert "content_success" in playbook + assert "legacy_rules" in playbook + assert "/etc/udev/rules.d/70-kfd.rules" in playbook + assert "/etc/udev/rules.d/70-amdgpu.rules" in playbook + assert "/etc/udev/rules.d/70-rocm-devices.rules" in playbook + file_probes = playbook[ + playbook.index("Inspect persisted GPU access state") : playbook.index( + "Record machine-readable GPU access discovery evidence" + ) + ] + assert file_probes.count("ignore_errors: true") == 10 + assert "failed_when: false" not in file_probes + assert 'mode: "0600"' in playbook + assert "hosts: pxe_controller" not in playbook + + +def test_pxe_gpu_admission_resolves_fresh_rootfs_and_refuses_retained_migrations() -> None: + assert PXE_GPU_ACCESS_TASKS.exists() + + main = read(PXE_CONTROLLER_ROLE / "tasks" / "main.yml") + tasks = read(PXE_GPU_ACCESS_TASKS) + + assert "Record PXE rootfs state before lifecycle changes" in main + assert "_pxe_rootfs_existed_at_start" in main + assert "_pxe_rootfs_rebuilt_this_run" in main + assert "include_tasks: gpu_access.yml" in main + assert main.index("Record PXE rootfs state before lifecycle changes") < main.index("Stop NFS before rootfs rebuild") + assert main.index("include_tasks: gpu_access.yml") < main.index("Find latest kernel in rootfs") + assert "tasks_from: validate" not in main + assert "tasks_from: preflight" not in main + assert "tasks_from: apply" not in main + + assert "_pxe_rootfs_disposition" in tasks + assert "_pxe_unanimous_live_render_gid" in tasks + assert "_pxe_resolved_render_gid" in tasks + assert "fresh" in tasks + assert "retained" in tasks + assert "groupadd" in tasks + assert "--system" in tasks + assert "groupmod" not in tasks + assert "getent" in tasks + assert 'auplc_render_gid: "{{ _pxe_resolved_render_gid }}"' in tasks + assert "auplc_normalize_render_gid: \"{{ _pxe_rootfs_disposition == 'fresh' }}\"" in tasks + assert "Require retained PXE legacy GPU rules absent" in tasks + assert "Require retained PXE render GID matches unanimous live GID" in tasks + assert "tasks_from: preflight" in tasks + assert "tasks_from: apply" in tasks + assert "render:993" not in tasks + assert "lspci" not in tasks + assert "pxe_gpu_access_normalize_render_gid" not in tasks + + +def test_pxe_gpu_admission_preflights_retained_rootfs_before_lifecycle_mutation() -> None: + main = read(PXE_CONTROLLER_ROLE / "tasks" / "main.yml") + + retained_admission = "Admit retained PXE GPU rootfs read-only before lifecycle changes" + final_admission = "Re-preflight PXE GPU rootfs before TFTP" + + assert main.count("include_tasks: gpu_access.yml") == 2 + assert main.index("Record PXE rootfs state before lifecycle changes") < main.index(retained_admission) + assert main.index(retained_admission) < main.index("Stop NFS before rootfs rebuild") + assert main.index("Remove chroot setup script") < main.index(final_admission) + assert main.index(final_admission) < main.index("Find latest kernel in rootfs") + + retained_branch = main[main.index(retained_admission) : main.index("Stop NFS before rootfs rebuild")] + final_branch = main[main.index(final_admission) : main.index("Find latest kernel in rootfs")] + + assert "pxe_gpu_access_enabled | bool" in retained_branch + assert "not (_pxe_rootfs_rebuilt_this_run | bool)" in retained_branch + assert "pxe_gpu_access_enabled | bool" in final_branch + assert "pxe_gpu_admission_phase: final" in final_branch + + +def test_pxe_rootfs_disposition_uses_initial_root_path_and_rejects_partial_retained_trees() -> None: + main = read(PXE_CONTROLLER_ROLE / "tasks" / "main.yml") + + assert "Require existing PXE rootfs is a directory" in main + assert "Require incomplete PXE rootfs force rebuild" in main + assert '_pxe_rootfs_existed_at_start: "{{ _pxe_rootfs_lstat.stat.exists | bool }}"' in main + assert "not (_pxe_rootfs_lstat.stat.exists | bool)" in main + assert main.index("Require incomplete PXE rootfs force rebuild") < main.index("Stop NFS before rootfs rebuild") + assert main.index("Require incomplete PXE rootfs force rebuild") < main.index("- name: Build NFS rootfs") + + +def test_pxe_retained_admission_is_read_only_until_post_chroot_repreflight_and_apply() -> None: + main = read(PXE_CONTROLLER_ROLE / "tasks" / "main.yml") + tasks = read(PXE_GPU_ACCESS_TASKS) + + assert "Admit retained PXE GPU rootfs read-only before lifecycle changes" in main + assert "Re-preflight PXE GPU rootfs before TFTP" in main + assert main.index("Admit retained PXE GPU rootfs read-only before lifecycle changes") < main.index( + "Stop NFS before rootfs rebuild" + ) + assert main.index("Remove chroot setup script") < main.index("Re-preflight PXE GPU rootfs before TFTP") + assert "pxe_gpu_admission_phase: retained-read-only" in main + assert "pxe_gpu_admission_phase: final" in main + assert "Require retained PXE canonical GPU rule" in tasks + assert "Require retained PXE canonical GPU state" in tasks + assert "Apply GPU access after final PXE re-preflight" in tasks + retained_read_only = tasks[: tasks.index("Preflight GPU access after final PXE re-preflight")] + assert "tasks_from: apply" not in retained_read_only + assert "pxe_gpu_admission_phase == 'final'" in tasks + + +def test_pxe_retained_admission_checks_canonical_parent_chain_before_lifecycle_or_chroot_mutation() -> None: + main = read(PXE_CONTROLLER_ROLE / "tasks" / "main.yml") + tasks = read(PXE_GPU_ACCESS_TASKS) + + assert "Inspect retained PXE canonical GPU access parents" in tasks + assert "Require retained PXE canonical GPU access parents" in tasks + for parent in ("/etc", "/etc/udev", "/etc/udev/rules.d", "/var", "/var/lib", "/var/lib/auplc"): + assert parent in tasks + assert "item.stat.exists" in tasks + assert "item.stat.isdir" in tasks + assert "not item.stat.islnk" in tasks + assert main.index("Admit retained PXE GPU rootfs read-only before lifecycle changes") < main.index( + "Execute chroot setup" + ) From a45d2e8d7f2c65770bf15d8cff96775a9c1477ba Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:46:18 +0800 Subject: [PATCH 19/65] docs(deploy): document unified GPU permission flow --- deploy/README.md | 123 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 110 insertions(+), 13 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index c1a8e843..d2a1a72f 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -53,18 +53,115 @@ sudo ./auplc-installer install ### Multi-Node Cluster +Generate the spec, fill in the normal network and node details, then let the +generator discover GPU hosts and their shared `render` GID. The SSH flow asks +for no GPU host list and no GID. A PXE spec asks one extra GPU question: +`pxe.diskless_agents_have_amd_gpus`. Set it explicitly because the diskless +agents' hardware is not inferred from the controller. + +#### SSH-preinstalled + ```bash -# 1. Configure Ansible inventory -cd ansible -vim inventory.yml - -# 2. Run playbooks -sudo ansible-playbook playbooks/pb-base.yml -sudo ansible-playbook playbooks/pb-k3s-site.yml - -# 3. Deploy JupyterHub -cd ../../runtime -cp values-multi-nodes.yaml.example values-multi-nodes.yaml -vim values-multi-nodes.yaml -helm upgrade --install jupyterhub ./chart -n jupyterhub --create-namespace -f values-multi-nodes.yaml +cd .. +REPO_ROOT="$(pwd)" +DEPLOY_SCRIPTS="$REPO_ROOT/skills/deploy-aup-learning-cloud/scripts" +python3 "$DEPLOY_SCRIPTS/gen_configs.py" --print-schema > spec.json +# Edit spec.json: choose ssh-preinstalled and fill the node/network fields. +GENERATED_DIR="$REPO_ROOT/generated" +python3 "$DEPLOY_SCRIPTS/gen_configs.py" --spec spec.json --out-dir "$GENERATED_DIR" +install -m 0600 "$GENERATED_DIR/inventory.yml" "$REPO_ROOT/deploy/ansible/inventory.yml" +install -m 0644 "$GENERATED_DIR/values-basic-example.yaml" "$REPO_ROOT/runtime/values-basic-example.yaml" +python3 "$DEPLOY_SCRIPTS/validate.py" --repo "$REPO_ROOT" --topology ssh-preinstalled \ + --inventory "$REPO_ROOT/deploy/ansible/inventory.yml" \ + --gpu-resolution "$GENERATED_DIR/gpu-access-resolution.json" \ + --values "$REPO_ROOT/runtime/values.yaml" \ + --values "$REPO_ROOT/runtime/values-basic-example.yaml" + +cd "$REPO_ROOT/deploy/ansible" +sudo ansible-playbook -i inventory.yml playbooks/pb-base.yml +sudo ansible-playbook -i inventory.yml playbooks/pb-k3s-site.yml +sudo ansible-playbook -i inventory.yml playbooks/pb-rocm.yml + +cd "$REPO_ROOT" +helm upgrade --install jupyterhub ./runtime/chart \ + --namespace jupyterhub --create-namespace \ + -f runtime/values.yaml \ + -f runtime/values-basic-example.yaml ``` + +Generation runs read-only Ansible discovery against every managed host. It +cross-checks AMD display BDFs from `lspci` with PCI vendor and display-class +records under `/sys/bus/pci/devices`; it does not require the devices to be +attached to `amdgpu` before ROCm installation. It checks +the `render` group and existing GPU access files, and publishes only when every +GPU host agrees on one GID. CPU-only fleets publish `null` for the generated +inventory and Helm render GID. GPU policy details in generated files are +internal outputs, not fields to maintain by hand. + +Configure notebook storage ownership with `singleuser.fsGid: 100`. Never set +storage `fsGroup` through `extraPodConfig.securityContext`, because that Pod +security-context override can replace the GPU resource's generated +`supplementalGroups`. + +#### PXE-diskless + +After setting `topology` to `pxe-diskless`, fill the PXE network fields and set +only `pxe.diskless_agents_have_amd_gpus` for GPU policy. When it is `true`, the +first generation is pending and creates private bootstrap files instead of +canonical deployment files. + +```bash +cd "$REPO_ROOT" +python3 "$DEPLOY_SCRIPTS/gen_configs.py" --spec spec.json --out-dir "$GENERATED_DIR" +cd "$REPO_ROOT/deploy/ansible" +sudo ansible-playbook \ + -i "$GENERATED_DIR/.pxe-bootstrap.inventory.yml" \ + playbooks/pb-pxe-controller.yml \ + -e @"$GENERATED_DIR/.pxe-bootstrap.vars.yml" + +# pb-pxe-controller finalizes automatically after a successful rootfs build. +install -m 0600 "$GENERATED_DIR/inventory.yml" "$REPO_ROOT/deploy/ansible/inventory.yml" +install -m 0644 "$GENERATED_DIR/values-basic-example.yaml" "$REPO_ROOT/runtime/values-basic-example.yaml" +python3 "$DEPLOY_SCRIPTS/validate.py" --repo "$REPO_ROOT" --topology pxe-diskless \ + --inventory "$REPO_ROOT/deploy/ansible/inventory.yml" \ + --gpu-resolution "$GENERATED_DIR/gpu-access-resolution.json" \ + --values "$REPO_ROOT/runtime/values.yaml" \ + --values "$REPO_ROOT/runtime/values-basic-example.yaml" \ + --pxe-vars "$GENERATED_DIR/pb-pxe-controller.vars.yml" +``` + +Don't invoke the hidden finalizer yourself. The playbook writes a private +handoff and runs finalization locally. `inventory.yml`, +`pb-pxe-controller.vars.yml`, `values-basic-example.yaml`, and +`gpu-access-resolution.json` appear only after success. + +A fresh PXE rootfs can create a missing `render` group and align it with a +unanimous live controller GPU GID after collision checks. A retained rootfs is +never silently changed. It must already contain one valid `render` group and, +when the controller has a resolved GPU GID, the rootfs GID must match. Rebuild +the rootfs or migrate the retained rootfs separately if it doesn't match. +Offline checks don't replace post-boot verification of GPU device ownership, +mode, supplemental groups, and workload access. + +#### Discovery failures and migration + +| Error | Action | +| --- | --- | +| Host is unreachable | Restore passwordless root SSH to that inventory host, then regenerate. | +| `lspci` is missing or fails | Install `pciutils` on the reported host and rerun generation. | +| Host evidence is `UNKNOWN` or AMD GPU BDF probes disagree | Compare AMD display BDFs from `lspci` with vendor `0x1002` display-class devices under `/sys/bus/pci/devices`; fix missing or inconsistent PCI enumeration, then regenerate. | +| GPU host has no valid `render` group | Install the correct GPU userspace or create one valid system `render` group, then regenerate. | +| GPU render GIDs disagree | Plan and perform a reviewed group migration so every GPU host uses one free GID, then regenerate. | +| CPU host retains GPU access contract, or canonical state/rule conflicts | Inspect `/var/lib/auplc/gpu-access.json` and `/etc/udev/rules.d/70-auplc-gpu-access.rules`. Remove stale project-owned files from a truly CPU-only host, or complete the GPU migration. Never overwrite unknown content. | +| Retained PXE rootfs GID differs from the unanimous live GID | Rebuild the rootfs, or migrate that retained rootfs separately before rerunning the playbook. | + +Old unshipped specs aren't compatible. Remove the former manual GPU policy +fields, regenerate the schema, copy the ordinary node and PXE network values +into it, and set only `pxe.diskless_agents_have_amd_gpus` on PXE deployments. + +## Deployment branch boundary + +This branch and these instructions do not modify or roll out any live +deployment. SHC, FET, and other deployment branches or environments must +backport the automatic discovery and generated-artifact changes before their +own reviewed rollout. From 5fa4856ddc3b19d1240ee4ed05ff87a0e60b8b4b Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:54:16 +0800 Subject: [PATCH 20/65] docs(deploy): align automatic GPU workflow references --- deploy/ansible/README.md | 34 +- skills/deploy-aup-learning-cloud/SKILL.md | 304 +++--------- skills/deploy-aup-learning-cloud/reference.md | 458 ++---------------- .../scripts/README.md | 91 +--- 4 files changed, 140 insertions(+), 747 deletions(-) diff --git a/deploy/ansible/README.md b/deploy/ansible/README.md index 3608aec7..108dad42 100644 --- a/deploy/ansible/README.md +++ b/deploy/ansible/README.md @@ -24,32 +24,14 @@ SOFTWARE. K3s cluster setup playbooks based on [k3s-ansible](https://github.com/k3s-io/k3s-ansible/tree/master). -For full instructions, see [Multi-Node Cluster Deployment](https://amdresearch.github.io/aup-learning-cloud/installation/multi-node.html). - -## Quick Reference - -```bash -# Configure inventory -vim inventory.yml - -# Base setup -sudo ansible-playbook playbooks/pb-base.yml - -# Deploy K3s cluster -sudo ansible-playbook playbooks/pb-k3s-site.yml - -# Install ROCm GPU drivers -sudo ansible-playbook playbooks/pb-rocm.yml - -# Add new nodes (update inventory.yml first) -sudo ansible-playbook playbooks/pb-k3s-site.yml - -# Reset cluster -sudo ansible-playbook playbooks/pb-k3s-reset.yml - -# Reset single node -sudo ansible-playbook playbooks/pb-k3s-reset.yml --limit -``` +For the generator, canonical inventory, validator arguments, and topology-specific +playbook commands, see the authoritative [deployment guide](../README.md). + +Don't write GPU policy into the inventory by hand. SSH generation discovers GPU +hosts and their shared `render` group ID. PXE generation uses only +`pxe.diskless_agents_have_amd_gpus`; when enabled, the controller playbook uses +private bootstrap inputs and publishes canonical files automatically after a +successful rootfs build. ## Prerequisites diff --git a/skills/deploy-aup-learning-cloud/SKILL.md b/skills/deploy-aup-learning-cloud/SKILL.md index 6946dca6..2fc9725a 100644 --- a/skills/deploy-aup-learning-cloud/SKILL.md +++ b/skills/deploy-aup-learning-cloud/SKILL.md @@ -1,253 +1,107 @@ --- name: deploy-aup-learning-cloud description: >- - Group: Plan & deploy AUP Learning Cloud. Deploys AUP Learning Cloud (a - multi-node JupyterHub-on-k3s platform for AMD - GPUs) onto physical hardware end to end. Use when the user wants to install, - deploy, set up, or stand up AUP Learning Cloud, AUPLC, or "the learning - cloud" on a cluster; mentions a multi-AIPC or 3-node mini-cluster, PXE / - netboot / diskless agents, the Ansible inventory.yml, pb-pxe-controller, - pb-k3s-site, the ROCm GPU device plugin/labeller, an NFS provisioner, or a - JupyterHub values.yaml / Helm chart for this project. Covers both the - PXE-diskless topology and the SSH-preinstalled multi-node topology. Do not - use for the single-node "./auplc-installer install" flow, for building - notebook images, or for non-AUPLC JupyterHub or k3s installs. + Group: Plan and deploy AUP Learning Cloud. Use when the user wants to install + the multi-node JupyterHub-on-k3s platform on physical hardware through either + PXE-diskless or SSH-preinstalled nodes. Do not use for the single-node + ./auplc-installer flow, notebook image builds, or unrelated JupyterHub and + k3s installations. --- # Deploy AUP Learning Cloud -Stand up AUP Learning Cloud on a multi-node k3s cluster: build the cluster with -Ansible, expose AMD GPUs, provide shared storage, and deploy the JupyterHub -chart with Helm so users can log in and spawn GPU notebooks. +Stand up a multi-node AUP Learning Cloud cluster with Ansible, AMD GPU access, +shared storage, and the JupyterHub Helm chart. -This skill is written for any coding agent. Run the commands and edit the files -as described; the full, copy-runnable command sequence and the troubleshooting -table live in **[reference.md](reference.md)**. +Use [deploy/README.md](../../deploy/README.md) as the source of truth for the +generator schema, commands, generated files, validation, and troubleshooting. +This skill defines the interview and safety gates around that procedure. ## Prerequisites -- A checkout of `aup-learning-cloud` on the operator/service machine. -- The service machine runs Ubuntu 24.04 with a reserved/static IP and internet - access. -- `ansible` on the operator machine; `kubectl` and `helm` for the cluster - (reference.md has the Helm install command). -- For GPU scheduling: AMD GPU nodes with a working in-kernel NIC driver. -- The user supplies the physical hardware. **No site values (IPs, subnet, SSH - keys, tokens) ship in the repo** — this skill generates them. +- A checkout of `aup-learning-cloud` on the operator machine. +- Ubuntu 24.04, a reserved controller IP, internet access, and Ansible. +- Physical node, network, storage, and authentication details from the user. +- Passwordless root SSH to every managed host in the SSH topology. -## Helper script paths +Site values and secrets don't ship in the repository. Generate them locally +and never put tokens, private keys, or credentials in tracked files. -Resolve the deploy helpers before running the commands below. From any directory -in an AUP Learning Cloud checkout: +## Phase 1: Interview -```bash -REPO_ROOT="$(git rev-parse --show-toplevel)" -DEPLOY_SCRIPTS="$REPO_ROOT/skills/deploy-aup-learning-cloud/scripts" -``` +Ask for an explicit topology choice before collecting other details or touching +machines. Never infer the choice from the hardware. -When this skill is installed as a plugin rather than used from a checkout, set -`DEPLOY_SKILL_DIR` to the absolute directory containing the loaded `SKILL.md`, -then derive the helpers from that directory: +| Choice | Use when | +| --- | --- | +| **PXE Diskless Netboot** (`pxe-diskless`) | A controller netboots diskless agents. | +| **Multi Node SSH Installation** (`ssh-preinstalled`) | Every node already runs Ubuntu and accepts root SSH. | -```bash -DEPLOY_SKILL_DIR="/absolute/path/to/deploy-aup-learning-cloud" -DEPLOY_SCRIPTS="$DEPLOY_SKILL_DIR/scripts" -``` +Then collect and confirm: -## Phase 1 — Interview +1. Courses and notebook resources. +2. Controller hostname, static IP, subnet, gateway, and DNS. +3. For SSH, every managed hostname and IP. Don't ask for a GPU host list or a + shared GPU group ID. Generation discovers both over SSH. +4. For PXE, the controller NIC, web port, rootfs SSH public key, and whether + diskless agents have AMD GPUs. This explicit yes or no is the sole PXE GPU + policy input because agent hardware can't be inferred from the controller. +5. Shared storage location and the Hub access method. -Work through this in order. **The deployment-method choice (1a) is a hard gate: -ask it first and get an explicit answer before collecting anything else or -touching the machines.** +Confirm detected GPU product labels before mapping them to accelerator keys in +the runtime values. -### Phase 1a — Choose the deployment method (ask first, always) +## Phase 2: Generate -Ask the user to pick one. **Never assume or auto-select** — even when the -machines "look like" one case, present both options and let the user decide (you -may recommend, but you still need an explicit choice before continuing): +Create a fresh schema and fill only its current fields. Run the generator rather +than writing inventory or GPU policy by hand. -| Choose | When | -| --- | --- | -| **PXE Diskless Netboot** (`topology: pxe-diskless`) — one service machine netboots diskless agents | Agents have no OS installed; you want zero per-machine install; small teaching lab. This is the [3-node mini-cluster guide](https://amdresearch.github.io/aup-learning-cloud/installation/multi-node/multi-aipc-hardware-deployment.html). | -| **Multi Node SSH Installation** (`topology: ssh-preinstalled`) — every node already runs Ubuntu | Each node has an OS and is reachable over SSH; closer to a long-running lab. This is the [multi-node guide](https://amdresearch.github.io/aup-learning-cloud/installation/multi-node.html). | - -The value in parentheses is the `topology` field for `gen_configs.py` (Phase 3) -and selects the matching section in [reference.md](reference.md). - -### Phase 1b — Collect the rest (some items branch on the choice above) - -Collect, and confirm back to the user, before touching anything: - -1. **Courses** wanted — drives the `values.yaml` course keys + team mappings - (full catalog setup lives in `configure-aup-learning-cloud-courses`). -2. **Node count** and which node is the controller/server, plus its static IP. - - *SSH path only:* also the hostname + IP of every agent node, and confirm - passwordless root SSH already reaches each one. -3. **GPU — do not ask the user to name the model.** Let the tooling find it: the - detectors report the GPUs (`$DEPLOY_SCRIPTS/detect_hardware.sh` in Phase 2) - and the real ROCm `amd.com/gpu.product-name` label - (`$DEPLOY_SCRIPTS/detect_cluster.sh` in Phase 5). Then - **confirm the detected GPU → accelerator-key mapping with the user** before it - goes into the values file. -4. *PXE path only:* service-machine NIC, subnet (CIDR), gateway, and DNS servers - (also auto-detected in Phase 2 and cross-checked), plus at least one SSH - public key for the rootfs and the apache web port. - -Login mode (`custom.authMode`) is unchanged — it stays at its `auto-login` -default; switch it later with `configure-aup-learning-cloud-auth` if needed. The -detailed steps for both paths are in [reference.md](reference.md). - -## Phase 2 — Discover - -On the service machine, run the bundled detector and cross-check its JSON -against the Phase 1 answers: - -```bash -"$DEPLOY_SCRIPTS/detect_hardware.sh" # JSON: nic, ip, subnet_cidr, gateway, dns_servers, gpus[] -``` - -It reports the default-route NIC, the service-machine IP + subnet CIDR, the -gateway, DNS servers, and each AMD GPU (`lspci`, vendor `1002`) with the bound -`kernel_driver`. If a GPU's `kernel_driver` is empty, note its module for -`pxe_initramfs_modules` (PXE path only). Empty fields come back in `warnings` -so you know exactly what to ask the operator for. The detected GPUs are the -source of truth for the accelerator mapping — Phase 1 does not ask the user to -name them, so surface the detected list and confirm it with the user. - -## Phase 3 — Generate config - -Drive `$DEPLOY_SCRIPTS/gen_configs.py` rather than hand-writing YAML — it keeps the -three artifacts consistent, mints the k3s token locally with a CSPRNG (never -printed), `chmod 600`s the inventory, and pins `pxe_k3s_version == k3s_version`. - -```bash -python3 "$DEPLOY_SCRIPTS/gen_configs.py" --print-schema > spec.json # fill from Phase 1 + 2 -GENERATED_DIR="$REPO_ROOT/generated" -python3 "$DEPLOY_SCRIPTS/gen_configs.py" --spec spec.json --out-dir "$GENERATED_DIR" -``` - -It writes, into `--out-dir`: - -1. `inventory.yml` — `server` host + `token` + `k3s_version` (agents empty for - PXE; listed for SSH) plus the `pxe_controller` group for PXE. -2. `pb-pxe-controller.vars.yml` — PXE path only: extra vars passed to - `deploy/ansible/playbooks/pb-pxe-controller.yml` with `-e @` (`pxe_network_interface`, - `pxe_subnet`, `pxe_gateway`, `pxe_dns_servers`, `pxe_controller_ip`, - `pxe_k3s_server_ips`, `pxe_k3s_version`, `pxe_web_port`, - `pxe_rootfs_password`, `pxe_rootfs_authorized_keys`). -3. `values-basic-example.yaml` — `custom.accelerators.*.nodeSelector` (matched - to real GPU labels in Phase 5), `custom.resources.images`, the storage class - (`nfs-client`), `custom.authMode`, and the proxy `NodePort` (e.g. 30890). - -Review the artifacts, install the inventory and runtime overlay into the -checkout, and keep the PXE vars in the generated directory. -**Never commit `inventory.yml` — it holds the token.** Field-by-field guidance -is in [reference.md](reference.md). - -Map the generated artifacts into the checkout before Phase 5 validation: - -```bash -install -m 0600 "$GENERATED_DIR/inventory.yml" "$REPO_ROOT/deploy/ansible/inventory.yml" -install -m 0644 "$GENERATED_DIR/values-basic-example.yaml" "$REPO_ROOT/runtime/values-basic-example.yaml" - -# PXE only: keep this generated secret in place and use its absolute path. -PXE_VARS="$(realpath "$GENERATED_DIR/pb-pxe-controller.vars.yml")" -chmod 0600 "$PXE_VARS" -``` - -The generated `gpu.acceleratorKeys` activates the selected accelerators only -for the generic GPU resource. Wire selected accelerators into course resources -separately with `configure-aup-learning-cloud-courses`. - -## Phase 4 — Execute (with confirmation gates) - -Run the install in order. **Pause for explicit user confirmation before each -risky/irreversible step** (see Safety). The PXE path is, in brief: - -1. Install host packages on the service machine. -2. Run `pb-pxe-controller.yml -e @"$PXE_VARS"` to build the PXE/NFS rootfs, - then verify the - controller (dnsmasq, NFS, apache2, TFTP boot files). -3. `pb-base.yml` + `pb-k3s-site.yml` to install the single-node k3s server. -4. Publish the k3s token + kubeconfig for agents over the apache `/k3s/` endpoint. -5. Netboot the agents; watch them auto-join with `kubectl get nodes -o wide`. - -Run the PXE controller step with the generated vars file: - -```bash -cd "$REPO_ROOT/deploy/ansible" -ansible-playbook -i inventory.yml playbooks/pb-pxe-controller.yml -e @"$PXE_VARS" -``` - -The SSH path runs `pb-base.yml`, `pb-k3s-site.yml`, and `pb-rocm.yml` against -the inventory instead. Full commands for both paths are in [reference.md](reference.md). - -## Phase 5 — GPU, storage, and chart - -1. Install the AMD GPU device plugin + ROCm labeller, then read the **real** - cluster state: - - ```bash -"$DEPLOY_SCRIPTS/detect_cluster.sh" > cluster.json # nodes[], gpu_product_names[], storage_classes[] - ``` - - Confirm the detected GPU → accelerator-key mapping with the user, then patch - `custom.accelerators.*.nodeSelector` so each `amd.com/gpu.product-name` - matches a value in `gpu_product_names`. Gate the install on a clean - pre-flight (exits non-zero on any mismatch): - - ```bash -# Set this to the topology selected in Phase 1a. -DEPLOY_TOPOLOGY=pxe-diskless -python3 "$DEPLOY_SCRIPTS/validate.py" --repo "$REPO_ROOT" --topology "$DEPLOY_TOPOLOGY" \ - --values runtime/values.yaml --values runtime/values-basic-example.yaml \ - --pxe-vars "$PXE_VARS" --cluster cluster.json --helm-dry-run -``` - -For the PXE path, the validator and Ansible receive the same generated vars -file. Omit `--pxe-vars "$PXE_VARS"` for the SSH path. - -2. Create the notebook-PVC NFS export and install the `nfs-subdir-external-provisioner` - (storage class `nfs-client`). -3. Deploy the chart: - -```bash -helm upgrade --install jupyterhub ./runtime/chart \ - --namespace jupyterhub --create-namespace \ - -f runtime/values.yaml \ - -f runtime/values-basic-example.yaml -``` - -## Phase 6 — Validate end to end - -```bash -kubectl get nodes -o wide # server + agents Ready -kubectl get pods -A # nothing CrashLoopBackOff/Pending/ImagePullBackOff -kubectl get storageclass # nfs-client present -``` - -Then open the Hub (NodePort example: `http://:30890`), log in, -spawn a CPU notebook, confirm file persistence across a restart, then spawn a -GPU notebook and confirm its pod lands on a GPU node -(`kubectl get pods -n jupyterhub -o wide`). +For SSH, generation performs read-only discovery on every managed host and +publishes canonical artifacts only after GPU evidence and group IDs agree. -## Safety +For PXE with GPU agents, initial generation creates private bootstrap inventory +and vars. Run the PXE controller playbook with those private files. A successful +rootfs build finalizes generation automatically and publishes the canonical +inventory, PXE vars, runtime overlay, and GPU resolution report. + +Follow the exact generation, installation, and playbook commands in +[deploy/README.md](../../deploy/README.md). Don't invent a separate completion +step. + +## Phase 3: Validate and execute + +Install the canonical generated inventory and runtime overlay into the checkout, +then run the validator with the arguments shown in the deployment guide: -These steps are destructive or hard to reverse — **stop and get explicit user -confirmation before each one**, and never run them silently: +- `--repo` +- `--topology` +- `--inventory` +- `--gpu-resolution` +- both `--values` files +- `--pxe-vars` for PXE only + +Stop on validation failure. After a clean result, follow the topology's Ansible, +storage, device plugin, and Helm sequence in +[deploy/README.md](../../deploy/README.md). + +## Phase 4: Verify + +Check that all expected nodes are Ready, the GPU labels and allocatable resources +match the generated policy, the storage class is available, and JupyterHub pods +are healthy. Open the Hub, start a CPU notebook, verify persistence, then start a +GPU notebook and confirm it schedules on a GPU node. + +## Safety -- Building/rebuilding the PXE rootfs (`pxe_rootfs_force_rebuild: true`). -- Editing `/etc/exports` and restarting `nfs-kernel-server`. -- `kubectl delete node ` (debugging only). -- `helm uninstall` or a cluster reset (`pb-k3s-reset.yml`). -- Changing firmware boot order / disabling Secure Boot on agents. +Pause for explicit user confirmation before rebuilding a PXE rootfs, changing +NFS exports, changing firmware boot settings, resetting a cluster, deleting a +node, or uninstalling a Helm release. -Never commit or push. Never write the k3s token, OAuth secrets, or SSH private -keys into tracked files. Preserve the four AUP Learning Cloud attribution -layers (see the project `AGENTS.md`) if any chart/Hub source is touched. +Never commit or push deployment secrets. Preserve the four AUP Learning Cloud +attribution layers described in the project `AGENTS.md` if Hub or chart sources +are changed. ## Reference -Full step-by-step commands for both topologies, the GPU-label-to-accelerator -mapping, the `values.yaml` field guide, and the troubleshooting table: -[reference.md](reference.md). +- [Deployment commands and troubleshooting](../../deploy/README.md) +- [Skill-specific summary](reference.md) diff --git a/skills/deploy-aup-learning-cloud/reference.md b/skills/deploy-aup-learning-cloud/reference.md index e67faf02..4aafa0fa 100644 --- a/skills/deploy-aup-learning-cloud/reference.md +++ b/skills/deploy-aup-learning-cloud/reference.md @@ -1,442 +1,40 @@ -# Deploy AUP Learning Cloud — Reference +# Deploy AUP Learning Cloud Reference -Full, copy-runnable commands for both deployment topologies, the GPU label -mapping, the `values.yaml` field guide, and the troubleshooting table. The -workflow and confirmation gates are in [SKILL.md](SKILL.md). +The authoritative procedure, command lines, generated file list, and failure +guidance live in [deploy/README.md](../../deploy/README.md). Don't copy those +commands into this reference. -## Contents +## Topology contract -- [Source guides](#source-guides) -- [PXE-diskless topology (3-node mini-cluster)](#pxe-diskless-topology-3-node-mini-cluster) -- [SSH-preinstalled topology (standard multi-node)](#ssh-preinstalled-topology-standard-multi-node) -- [GPU label to accelerator key](#gpu-label-to-accelerator-key) -- [values.yaml field guide](#valuesyaml-field-guide) -- [Troubleshooting](#troubleshooting) - -## Source guides - -- 3-node mini-cluster (PXE diskless): -- Standard multi-node (SSH): - -Treat the live docs as the source of truth for version pins; this file -condenses the opinionated path. - -The helper commands are resolved through `DEPLOY_SCRIPTS` as defined in -[SKILL.md](SKILL.md#helper-script-paths), not through a checkout-root -`scripts/` directory. - -The two topology sections below are the two branches of the Phase 1a gate in -[SKILL.md](SKILL.md): **PXE Diskless Netboot** (`topology: pxe-diskless`) → -[PXE-diskless topology](#pxe-diskless-topology-3-node-mini-cluster); **Multi Node -SSH Installation** (`topology: ssh-preinstalled`) → -[SSH-preinstalled topology](#ssh-preinstalled-topology-standard-multi-node). - -## PXE-diskless topology (3-node mini-cluster) - -One service machine (AIPC 1) runs the PXE controller, the single-node k3s -server, NFS, and the apache k3s-credential endpoint. The other machines are -diskless agents that netboot and auto-join. Only AIPC 1 is Ansible-managed. - -### Step 1 — Prepare the service machine - -```bash -sudo apt update -sudo apt install -y git ansible curl ca-certificates jq \ - dnsmasq pxelinux syslinux-common apache2 \ - nfs-kernel-server debootstrap \ - grub-efi-amd64-signed shim-signed - -ip -br addr # record the NIC and IP -ip route # record the gateway -``` - -Give the local `root` a passwordless SSH login (or add `ansible_connection: -local` to the host vars to skip SSH entirely): - -```bash -sudo install -d -m 0700 /root/.ssh -sudo tee -a /root/.ssh/authorized_keys < ~/.ssh/id_ed25519.pub >/dev/null -sudo chmod 0600 /root/.ssh/authorized_keys -ssh root@ true && echo root-ssh-ok -``` - -### Step 2 — Configure the inventory - -Edit `deploy/ansible/inventory.yml`. AIPC 1 is the only host; the `agent` group -stays empty (netboot agents are not Ansible-managed). Generate the token with -`openssl rand -base64 64` and keep it out of chat/VCS. - -```yaml -k3s_cluster: - children: - server: - hosts: - aipc1: - ansible_host: - agent: - hosts: {} # diskless netboot agents auto-join; do NOT list them here - vars: - ansible_user: root - k3s_version: v1.32.3+k3s1 - token: "" # openssl rand -base64 64 - api_endpoint: "{{ hostvars[groups['server'][0]]['ansible_host'] | default(groups['server'][0]) }}" - -pxe_controller: - hosts: - aipc1: - ansible_host: - vars: - ansible_port: 22 - ansible_user: root -``` - -### Step 3 — Prepare the generated PXE controller vars - -The network, controller, server-IP, and SSH-key values are empty by default and -the role asserts on them. `$DEPLOY_SCRIPTS/gen_configs.py` writes these values -to `generated/pb-pxe-controller.vars.yml`. Keep that file at mode `0600`; it can -contain `pxe_rootfs_password`. Resolve its absolute path for the Ansible and -validator commands instead of copying or merging it into the playbook: - -```bash -PXE_VARS="$(realpath ./generated/pb-pxe-controller.vars.yml)" -chmod 0600 "$PXE_VARS" -test "$(stat -c '%a' "$PXE_VARS")" = 600 -``` - -Review the generated values before the first run: - -```yaml -pxe_rootfs_force_rebuild: true # true for the first build (RISKY: rebuilds rootfs) -pxe_network_interface: "enp1s0" # service-machine NIC (Step 1) -pxe_subnet: "192.168.1.0/24" # node subnet, CIDR -pxe_gateway: "192.168.1.1" # default gateway (informational) -pxe_dns_servers: "8.8.8.8,8.8.4.4" -pxe_controller_ip: "192.168.1.10" # this service machine's IP -pxe_k3s_server_ips: - - "192.168.1.10" -pxe_k3s_version: "v1.32.3+k3s1" # MUST match inventory k3s_version -pxe_web_port: 8080 # apache port for the k3s token/kubeconfig (not 80) -pxe_rootfs_password: "" # optional; empty disables password login (use ansible-vault if set) -pxe_rootfs_authorized_keys: - - "ssh-ed25519 AAAA... you@host" # at least one key required -``` - -Set `pxe_rootfs_force_rebuild: false` after the first stable build so you do -not rebuild the rootfs under running agents. The playbook also exposes -`pxe_apt_mirror`, `pxe_rootfs_packages`, and `pxe_initramfs_modules` (add your -NIC module here if it lacks an in-kernel driver) — leave these at their defaults -unless discovery flagged a need. - -### Step 4 — Run the PXE controller playbook - -```bash -cd ~/aup-learning-cloud -REPO_ROOT="$(pwd)" -DEPLOY_SCRIPTS="$REPO_ROOT/skills/deploy-aup-learning-cloud/scripts" -PXE_VARS="$(realpath "$REPO_ROOT/generated/pb-pxe-controller.vars.yml")" -python3 "$DEPLOY_SCRIPTS/validate.py" --repo "$REPO_ROOT" \ - --topology pxe-diskless --pxe-vars "$PXE_VARS" \ - --values runtime/values.yaml --values runtime/values-basic-example.yaml -cd "$REPO_ROOT/deploy/ansible" -ansible-playbook -i inventory.yml playbooks/pb-pxe-controller.yml -e @"$PXE_VARS" -``` - -### Step 5 — Verify the controller - -```bash -systemctl is-active dnsmasq nfs-kernel-server apache2 -showmount -e localhost -ls -l /srv/tftp/pxelinux.0 /srv/tftp/grubnetx64.efi /srv/tftp/vmlinuz /srv/tftp/initrd.img -curl -I http://127.0.0.1:8080/k3s/ # 403 expected (dir exists, empty) -``` - -The `/k3s/` endpoint is served on port 8080 (k3s owns 80/443 for ingress). - -### Step 6 — Install the single-node k3s server - -Run **without** `sudo` (key-based root SSH already connects as root): - -```bash -cd ~/aup-learning-cloud/deploy/ansible -ansible-playbook -i inventory.yml playbooks/pb-base.yml -ansible-playbook -i inventory.yml playbooks/pb-k3s-site.yml -export KUBECONFIG=~/.kube/config # add to ~/.bashrc to persist -kubectl get nodes -o wide -``` - -### Step 7 — Publish k3s credentials for agents - -```bash -sudo install -d -m 0755 /var/www/html/k3s -sudo install -m 0644 /var/lib/rancher/k3s/server/token /var/www/html/k3s/token -sudo sed "s#https://127.0.0.1:6443#https://:6443#g" \ - /etc/rancher/k3s/k3s.yaml | sudo tee /var/www/html/k3s/kubeconfig >/dev/null -sudo chmod 0644 /var/www/html/k3s/token /var/www/html/k3s/kubeconfig -sudo systemctl reload apache2 - -curl -fsS http://127.0.0.1:8080/k3s/token >/dev/null && echo token-ok -curl -fsS http://127.0.0.1:8080/k3s/kubeconfig >/dev/null && echo kubeconfig-ok -``` - -### Step 8 — Netboot the agents - -On each agent: disable Secure Boot, enable network boot, and put PXE before the -local disk in the firmware boot order. Boot, then watch them register: - -```bash -watch kubectl get nodes -o wide -``` - -Agents appear as `agent-` nodes and become `Ready`. - -### Step 9 — Validate agent persistence - -Reboot one agent; confirm it rejoins with the same identity. On the agent: - -```bash -mount | grep /var/lib/rancher/k3s -test -f /var/lib/rancher/k3s/node-password && echo node-password-ok -systemctl status mount-local-disk k3s-agent --no-pager -``` - -`kubectl delete node ` clears a stale node object — **debugging only**, -confirm with the user first. - -Continue with [Step 10 (GPU)](#step-10--amd-gpu-device-plugin-and-labeller). - -## SSH-preinstalled topology (standard multi-node) - -Every node already runs Ubuntu 24.04 and is reachable over passwordless SSH. - -### Prepare SSH and inventory - -Helper scripts in `deploy/scripts/` enable root SSH and distribute kubeconfig: - -```bash -./deploy/scripts/edit_sshd.sh -./deploy/scripts/setup_ssh_root_access.sh -./deploy/scripts/deploy-kubeconfig.sh -``` - -Edit `deploy/ansible/inventory.yml` — list every node under `server`/`agent`: - -```yaml -k3s_cluster: - children: - server: - hosts: - : - agent: - hosts: - : - : - vars: - ansible_port: 22 - ansible_user: root - k3s_version: v1.32.3+k3s1 - token: "" # openssl rand -base64 64 - api_endpoint: "{{ hostvars[groups['server'][0]]['ansible_host'] | default(groups['server'][0]) }}" -``` - -### Build the cluster - -```bash -cd deploy/ansible -sudo ansible-playbook playbooks/pb-base.yml # base OS / packages -sudo ansible-playbook playbooks/pb-k3s-site.yml # deploy k3s -sudo ansible-playbook playbooks/pb-rocm.yml # ROCm on GPU nodes -``` - -Related: `pb-k3s-upgrade.yml` (upgrade), `pb-k3s-reset.yml` (reset — RISKY). -Then install `kubectl`/`helm` on the operator machine (see Helm command below) -and continue with [Step 10 (GPU)](#step-10--amd-gpu-device-plugin-and-labeller). - -### Install Helm - -```bash -wget https://get.helm.sh/helm-v3.17.2-linux-amd64.tar.gz -O /tmp/helm.tar.gz -cd /tmp && tar -zxvf helm.tar.gz -sudo mv /tmp/linux-amd64/helm /usr/local/bin/helm -``` - -## Step 10 — AMD GPU device plugin and labeller - -```bash -kubectl create -f https://raw.githubusercontent.com/ROCm/k8s-device-plugin/master/k8s-ds-amdgpu-dp.yaml -kubectl create -f https://raw.githubusercontent.com/ROCm/k8s-device-plugin/master/k8s-ds-amdgpu-labeller.yaml - -kubectl get pods -A | grep -i amd -kubectl describe node | grep amd.com/gpu -``` - -Use the labels that actually appear. Common keys: -`amd.com/gpu.product-name`, `amd.com/gpu.family`, `amd.com/gpu.device-id`. - -## Step 11 — Shared NFS storage for notebook PVCs - -This is separate from the PXE rootfs export. Append the export directly to -`/etc/exports` (on Ubuntu 24.04 `/etc/exports.d/*.conf` is ignored): - -```bash -sudo mkdir -p -sudo chown -R nobody:nogroup -sudo chmod 0777 -echo " (rw,sync,no_subtree_check,no_root_squash,insecure)" | sudo tee -a /etc/exports -sudo exportfs -ra -sudo systemctl restart nfs-kernel-server -showmount -e localhost -``` - -Install the provisioner (storage class `nfs-client`): - -```bash -cd ~/aup-learning-cloud -cp deploy/k8s/nfs-provisioner/values.yaml deploy/k8s/nfs-provisioner/values.local.yaml -# edit values.local.yaml: nfs.server, nfs.path, storageClass.name = nfs-client -helm repo add nfs-subdir-external-provisioner https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner/ -helm repo update -helm upgrade --install nfs-subdir-external-provisioner \ - nfs-subdir-external-provisioner/nfs-subdir-external-provisioner \ - --namespace nfs-provisioner --create-namespace \ - -f deploy/k8s/nfs-provisioner/values.local.yaml -kubectl get storageclass -``` - -## Step 12 — Configure JupyterHub values - -The generated `runtime/values-basic-example.yaml` is the canonical deployment -overlay. Review and keep it when Phase 3 generated one. Only when no generated -overlay exists, start a manual overlay from the example: - -```bash -cd ~/aup-learning-cloud/runtime -if [ ! -e values-basic-example.yaml ]; then - cp values-multi-nodes.yaml.example values-basic-example.yaml -fi -``` - -Minimum edits (see the [field guide](#valuesyaml-field-guide)): - -```yaml -custom: - authMode: "auto-login" # single-machine default; avoid "dummy" (login 404s) - accelerators: - strix-halo: - nodeSelector: - amd.com/gpu.product-name: "" # from Step 10 - quotaRate: 3 - resources: - images: - cpu: "" - gpu: "" -hub: - db: - pvc: - storageClassName: nfs-client -singleuser: - storage: - dynamic: - storageClass: nfs-client -proxy: - service: - type: NodePort - nodePorts: - http: 30890 -``` - -## Step 13 — Deploy AUP Learning Cloud - -```bash -cd ~/aup-learning-cloud -helm upgrade --install jupyterhub ./runtime/chart \ - --namespace jupyterhub --create-namespace \ - -f runtime/values.yaml \ - -f runtime/values-basic-example.yaml - -kubectl get pods -n jupyterhub -o wide -kubectl get svc -n jupyterhub -``` - -For later config changes, re-run the same `helm upgrade --install`. - -## Step 14 — End-to-end validation - -```bash -kubectl get nodes -o wide -kubectl get pods -A -kubectl get storageclass -kubectl describe node | grep amd.com/gpu -``` - -Then browse to `http://:30890` (or your ingress host), log in, -spawn a CPU notebook, create a file, restart and confirm it persists, then -spawn a GPU notebook and confirm its pod lands on a GPU node. - -## GPU label to accelerator key - -The chart's accelerator catalog (`runtime/values.yaml`) is keyed by accelerator -names; map the ROCm labeller's `amd.com/gpu.product-name` to the right key. The -GPU is auto-detected (Phase 2 and Phase 5), not named by the user in the -interview — use this table to confirm the detected product-name → key mapping -with the user. Verify against the live values file — product names can normalize -differently per fleet. - -| `amd.com/gpu.product-name` (example) | Accelerator key | +| Topology | Generator behavior | | --- | --- | -| `AMD_Radeon_780M_Graphics` | `phx` | -| `AMD_Radeon_890M_Graphics` | `strix` | -| `AMD_Radeon_8060S_Graphics` | `strix-halo` | -| `AMD_Radeon_RX_9070_XT` | `9070xt` | -| `AMD_Radeon_AI_PRO_R9700` | `r9700` | -| `AMD_Radeon_RX_9600_GRE` | `9600gre` | - -If your labeller reports a different product name, update the matching -`custom.accelerators.*.nodeSelector` entry to that exact string. +| `ssh-preinstalled` | Connects to every managed host, discovers GPU hosts and their shared `render` group ID, and publishes canonical files only when discovery is consistent. | +| `pxe-diskless` | Uses `pxe.diskless_agents_have_amd_gpus` as its sole GPU policy input. GPU-enabled first generation emits private bootstrap files; the PXE controller playbook finalizes canonical files after a successful rootfs build. | -## values.yaml field guide +Don't hand-author generated GPU policy. Old unshipped specs should be recreated +from the current `--print-schema` output. -Sections to review in the generated `values-basic-example.yaml`, or in the -manual `values-multi-nodes.yaml.example` copy when generation was not used: - -| Field | Purpose | -| --- | --- | -| `custom.authMode` | `auto-login` for the single-machine example; OAuth modes for real auth | -| `custom.githubOrgName`, `hub.config.GitHubOAuthenticator` | GitHub OAuth (when not auto-login) | -| `custom.adminUser` | Hub admin | -| `custom.accelerators.*.nodeSelector` | Must match real `amd.com/gpu.*` labels | -| `custom.resources.images` | CPU/GPU/course notebook images | -| `custom.resources.requirements`, `custom.teams.mapping`, `custom.quota` | Per-team resources and quotas | -| `hub.db.pvc.storageClassName`, `singleuser.storage.dynamic.storageClass` | `nfs-client` for multi-node | -| `proxy.service`, `ingress` | NodePort (e.g. 30890) or ingress host | +## Canonical validation inputs -## Troubleshooting +Use the validator command from [deploy/README.md](../../deploy/README.md). It +passes: -| Symptom | Likely cause | First checks | -| --- | --- | --- | -| Playbook fails immediately on an assert | A required PXE var is empty | Re-check `pxe_controller_ip`, `pxe_subnet`, `pxe_network_interface`, `pxe_dns_servers`, `pxe_k3s_server_ips`, and at least one SSH key | -| Agent never shows the PXE menu | Firmware boot order, network boot disabled, or Proxy-DHCP not reaching the client | Firmware, switch port, `systemctl status dnsmasq`, `journalctl -u dnsmasq` | -| Agent gets an IP but cannot load boot files | TFTP blocked, missing files, or Secure Boot still on | `/srv/tftp`, firewall, Secure Boot disabled, `dnsmasq` logs | -| Agent has no network during netboot | NIC has no in-kernel driver in the initramfs | `lspci -nnk`, add the module to `pxe_initramfs_modules`, rebuild rootfs | -| Agent kernel boots but cannot mount rootfs | NFS export, subnet ACL, or wrong `pxe_controller_ip` | `showmount -e `, `/etc/exports`, rootfs kernel args | -| Agent waits for the k3s token | Token not published or apache ACL blocks the subnet | `curl http://:8080/k3s/token`, apache config | -| Agent joins once but fails after reboot | Missing local k3s persistence or lost node password | `mount-local-disk`, `/var/lib/rancher/k3s/node-password`, `k3s-agent` logs | -| Agent fails to join with a version error | Agent rootfs k3s newer than the server | Align `pxe_k3s_version` with `k3s_version`, rebuild rootfs | -| Agent node does not join (SSH path) | Hostname resolution, token, or `api_endpoint` mismatch | `systemctl status k3s-agent`, `journalctl -u k3s-agent`, `/etc/hosts` | -| GPU notebook stays Pending | Chart `nodeSelector` mismatch or GPUs exhausted | `kubectl describe pod -n jupyterhub`, node labels | -| PVC stays Pending | StorageClass name mismatch or NFS provisioner cannot mount | `kubectl get storageclass`, provisioner logs, NFS export | -| `kubectl` permission denied on `k3s.yaml` | kubeconfig not readable | `export KUBECONFIG=~/.kube/config`, or `--write-kubeconfig-mode=644` in inventory `extra_server_args` | +- repository root with `--repo` +- selected topology with `--topology` +- installed inventory with `--inventory` +- generated GPU resolution report with `--gpu-resolution` +- base and generated overlays as two `--values` arguments +- canonical PXE vars with `--pxe-vars` for PXE only -For a complete reset (RISKY — confirm with the user): +Generation and validation must finish before Ansible or Helm changes are made. -```bash -cd deploy/ansible -sudo ansible-playbook playbooks/pb-k3s-reset.yml # whole cluster -sudo ansible-playbook playbooks/pb-k3s-reset.yml --limit # single node -``` +## Operator gates -## Out of scope +Keep the topology choice explicit. Confirm network, node, storage, course, and +access details with the user. For PXE, also confirm the GPU-agent boolean and a +rootfs SSH public key. For SSH, verify passwordless root access to every managed +host. -Zot registry mirror, Cloudflare Tunnel ingress, monitoring/Grafana, HA k3s, -external databases, and NPU setup. Add them only after the minimal deployment -boots agents, schedules GPU notebooks, and persists notebook storage. +Require confirmation before rootfs rebuilds, NFS export changes, firmware boot +changes, cluster resets, node deletion, or Helm uninstall. Keep generated +secrets out of version control. diff --git a/skills/deploy-aup-learning-cloud/scripts/README.md b/skills/deploy-aup-learning-cloud/scripts/README.md index 4e212dcc..65bd71c1 100644 --- a/skills/deploy-aup-learning-cloud/scripts/README.md +++ b/skills/deploy-aup-learning-cloud/scripts/README.md @@ -1,77 +1,36 @@ # Helper scripts -Deterministic helpers the deploy skill runs instead of generating commands ad -hoc. They are dependency-light (`bash` + `python3`, plus the obvious system -tools) and agent-agnostic, and follow the script conventions in -[../../../CONTRIBUTING.md](../../../CONTRIBUTING.md). Each emits JSON or a clear -report and uses exit codes the agent can branch on. +These dependency-light helpers support the multi-node deployment skill. See +[deploy/README.md](../../../deploy/README.md) for the authoritative command +sequence and argument paths. -| Script | Run when | What it does | -| --- | --- | --- | -| `detect_hardware.sh` | Phase 2, on the service machine | Detects the default-route NIC, IPv4 + subnet CIDR, gateway, DNS servers, and AMD GPUs (`lspci`, vendor `1002`) with their kernel driver. Emits JSON for filling PXE / network vars. Read-only. | -| `detect_cluster.sh` | After k3s + the device plugin are up | `kubectl get` of nodes, real `amd.com/gpu.*` labels, storage classes, and whether the ROCm device plugin + labeller DaemonSets are running. Emits JSON. Read-only. | -| `gen_configs.py` | Phase 3 | From a small cluster-spec (`--print-schema`), writes `inventory.yml`, `pb-pxe-controller.vars.yml` (PXE only), and `values-basic-example.yaml`. Generates the k3s token locally with `secrets` (never printed), `chmod 600` on the inventory, and pins `pxe_k3s_version == k3s_version`. | -| `validate.py` | Before each `ansible-playbook` / `helm` run | For `pxe-diskless`, checks required PXE vars and `k3s_version == pxe_k3s_version`; for both topologies, checks GPU labels only for active resource `acceleratorKeys` (when given `detect_cluster.sh` output), and optionally runs a `helm template` dry-run. Exit 1 on any failure. | +| Script | Purpose | +| --- | --- | +| `detect_hardware.sh` | Reports controller network details and local AMD PCI devices as JSON. | +| `detect_cluster.sh` | Reports Kubernetes nodes, AMD GPU labels, storage classes, and GPU DaemonSet state as JSON. | +| `gen_configs.py` | Prints the current spec schema, discovers SSH GPU state, and generates topology-specific deployment artifacts. PXE GPU bootstrap files remain private until the controller playbook finalizes them automatically. | +| `validate.py` | Checks the selected topology against canonical inventory, GPU resolution, values overlays, and PXE vars when applicable. | -## Quick reference +## Generator contract -From any directory in a checkout, resolve helpers with: +The SSH topology discovers GPU hosts and their shared `render` group ID. Users +don't provide either value. The PXE topology has one GPU policy input: +`pxe.diskless_agents_have_amd_gpus`. -```bash -REPO_ROOT="$(git rev-parse --show-toplevel)" -DEPLOY_SCRIPTS="$REPO_ROOT/skills/deploy-aup-learning-cloud/scripts" -``` +Generate specs from fresh `--print-schema` output. Don't hand-edit generated GPU +policy or add a separate PXE completion step. -For an installed plugin, set `DEPLOY_SKILL_DIR` to the absolute directory -containing the loaded `SKILL.md`, then use: +## Validator contract -```bash -DEPLOY_SKILL_DIR="/absolute/path/to/deploy-aup-learning-cloud" -DEPLOY_SCRIPTS="$DEPLOY_SKILL_DIR/scripts" -``` - -```bash -# Phase 2 — discover the host -"$DEPLOY_SCRIPTS/detect_hardware.sh" # JSON: nic, ip, subnet_cidr, gateway, dns, gpus[] - -# Phase 3 — generate config from a spec -python3 "$DEPLOY_SCRIPTS/gen_configs.py" --print-schema > spec.json # then edit spec.json -GENERATED_DIR="$REPO_ROOT/generated" -python3 "$DEPLOY_SCRIPTS/gen_configs.py" --spec spec.json --out-dir "$GENERATED_DIR" -install -m 0600 "$GENERATED_DIR/inventory.yml" "$REPO_ROOT/deploy/ansible/inventory.yml" -install -m 0644 "$GENERATED_DIR/values-basic-example.yaml" "$REPO_ROOT/runtime/values-basic-example.yaml" -# PXE only: keep the generated secret in place and resolve its absolute path. -PXE_VARS="$(realpath "$GENERATED_DIR/pb-pxe-controller.vars.yml")" -chmod 0600 "$PXE_VARS" -cd "$REPO_ROOT/deploy/ansible" -ansible-playbook -i inventory.yml playbooks/pb-pxe-controller.yml -e @"$PXE_VARS" - -# Phase 5 — after k3s + device plugin are up -"$DEPLOY_SCRIPTS/detect_cluster.sh" > cluster.json # JSON: nodes[], gpu_product_names[], storage_classes[] - -# Before running playbooks / helm (set to the selected topology) -DEPLOY_TOPOLOGY=pxe-diskless -python3 "$DEPLOY_SCRIPTS/validate.py" --repo "$REPO_ROOT" --topology "$DEPLOY_TOPOLOGY" \ - --values runtime/values.yaml --values runtime/values-basic-example.yaml \ - --pxe-vars "$PXE_VARS" --cluster cluster.json --helm-dry-run -``` - -Omit `--pxe-vars "$PXE_VARS"` for `ssh-preinstalled`. For `pxe-diskless`, the -validator and Ansible must receive the same generated file. - -Generated `gpu.acceleratorKeys` wires the selected accelerators to the generic -GPU resource. Use `configure-aup-learning-cloud-courses` to wire course -resources separately. +Use the exact validator command in +[deploy/README.md](../../../deploy/README.md). Its canonical inputs are +`--repo`, `--topology`, `--inventory`, `--gpu-resolution`, two `--values` +arguments, and `--pxe-vars` for PXE only. ## Conventions -- **JSON to stdout, diagnostics to stderr.** `detect_*.sh` always print a JSON - object; partial detection is reported via empty fields + a `warnings` array - rather than failing, so the agent can decide what to ask the operator. -- **Exit codes mean something.** `0` success (warnings allowed), `1` a real - validation failure, `2` a usage / missing-tooling error. -- **Secrets never touch stdout or VCS.** `gen_configs.py` mints the k3s token - with a CSPRNG, writes it only into `inventory.yml`, and `chmod 600`s it. -- **No third-party Python.** `gen_configs.py` / `validate.py` use the stdlib - only (no PyYAML), so they run on a bare operator machine. YAML is emitted - from templates and parsed with targeted scanning. +- Detection data goes to stdout as JSON. Diagnostics go to stderr. +- Exit code `0` means success, `1` means validation failed, and `2` means usage + or required tooling is wrong. +- Generated secrets stay off stdout and out of version control. +- Python helpers use the standard library only. From c781cb5f5a27560b5552cbe7e8ea0877674bfd02 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 10:20:28 +0800 Subject: [PATCH 21/65] fix(tests): isolate GPU role skill dependencies --- tests/skills/test_gpu_access_role.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/tests/skills/test_gpu_access_role.py b/tests/skills/test_gpu_access_role.py index 12f76505..b195fef5 100644 --- a/tests/skills/test_gpu_access_role.py +++ b/tests/skills/test_gpu_access_role.py @@ -2,12 +2,25 @@ """Canonical artifact tests for the multi-node GPU access role.""" +import sys +import types from pathlib import Path import pytest -from ansible.errors import AnsibleFilterError -from jinja2 import Environment +try: + from ansible.errors import AnsibleFilterError +except ModuleNotFoundError: + ansible_module = types.ModuleType("ansible") + errors_module = types.ModuleType("ansible.errors") + + class AnsibleFilterError(Exception): + pass + + errors_module.AnsibleFilterError = AnsibleFilterError + ansible_module.errors = errors_module + sys.modules["ansible"] = ansible_module + sys.modules["ansible.errors"] = errors_module from deploy.ansible.filter_plugins.auplc_json import ( DuplicateJsonKeyError, _reject_duplicate_keys, @@ -25,14 +38,6 @@ def read(path: Path) -> str: return path.read_text(encoding="utf-8") -def test_jinja_integer_test_rejects_boolean_and_float_state_values() -> None: - template = Environment().from_string("{% if value is integer %}integer{% else %}invalid{% endif %}") - - assert template.render(value=993) == "integer" - assert template.render(value=True) == "invalid" - assert template.render(value=993.0) == "invalid" - - def test_strict_json_filter_parses_canonical_gpu_access_state() -> None: assert auplc_from_json_strict('{"renderGid":993,"version":1}\n') == { "renderGid": 993, From db73537e224dd0949551d41073ab66d19630996b Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:30:28 +0800 Subject: [PATCH 22/65] refactor(hub): remove GPU group injection --- runtime/hub/core/config.py | 31 -------------- runtime/hub/core/spawner/kubernetes.py | 26 ------------ runtime/hub/tests/test_spawner_gpu_access.py | 43 ++++++++------------ 3 files changed, 17 insertions(+), 83 deletions(-) diff --git a/runtime/hub/core/config.py b/runtime/hub/core/config.py index 28e24ded..3925bbf0 100644 --- a/runtime/hub/core/config.py +++ b/runtime/hub/core/config.py @@ -45,8 +45,6 @@ import yaml from pydantic import BaseModel, Field, field_validator -MAX_RENDER_GID = (2**32) - 2 - # ============================================================================= # YAML Configuration Models # ============================================================================= @@ -87,25 +85,6 @@ class QuotaSettings(BaseModel): model_config = {"extra": "allow"} -class GpuAccessSettings(BaseModel): - """Host group access settings for GPU-enabled user pods.""" - - renderGid: int | None = None - - @field_validator("renderGid", mode="before") - @classmethod - def validate_render_gid(cls, value: Any) -> int | None: - """Require a native positive integer GID when GPU access is configured.""" - - if value is None: - return None - if isinstance(value, bool) or not isinstance(value, int) or not 1 <= value <= MAX_RENDER_GID: - raise ValueError(f"custom.gpuAccess.renderGid must be an integer between 1 and {MAX_RENDER_GID}") - return value - - model_config = {"extra": "allow"} - - class AcceleratorOverride(BaseModel): """Per-accelerator overrides for a resource (image and/or env).""" @@ -232,7 +211,6 @@ class ParsedConfig(BaseModel): accelerators: dict[str, AcceleratorConfig] = Field(default_factory=dict) teams: TeamsConfig = Field(default_factory=TeamsConfig) quota: QuotaSettings = Field(default_factory=QuotaSettings) - gpuAccess: GpuAccessSettings = Field(default_factory=GpuAccessSettings) gitClone: GitCloneSettings = Field(default_factory=GitCloneSettings) hub: HubNetworkSettings = Field(default_factory=HubNetworkSettings) notebook: NotebookNetworkSettings = Field(default_factory=NotebookNetworkSettings) @@ -248,7 +226,6 @@ def from_dicts( accelerators: dict | None = None, teams: dict | None = None, quota: dict | None = None, - gpu_access: dict | None = None, git_clone: dict | None = None, hub: dict | None = None, notebook: dict | None = None, @@ -266,8 +243,6 @@ def from_dicts( raw_config["teams"] = teams if quota: raw_config["quota"] = quota - if gpu_access is not None: - raw_config["gpuAccess"] = gpu_access if git_clone: raw_config["gitClone"] = git_clone if hub: @@ -359,7 +334,6 @@ def init(cls, config_path: str | Path) -> HubConfig: accelerators=raw_config.get("accelerators"), teams=raw_config.get("teams"), quota=raw_config.get("quota"), - gpu_access=raw_config.get("gpuAccess"), git_clone=raw_config.get("gitClone"), hub=raw_config.get("hub"), notebook=raw_config.get("notebook"), @@ -437,11 +411,6 @@ def quota(self) -> QuotaSettings: """Get quota configuration.""" return self._config.quota - @property - def gpu_access(self) -> GpuAccessSettings: - """Get GPU pod access configuration.""" - return self._config.gpuAccess - @property def git_clone(self) -> GitCloneSettings: """Get git clone configuration.""" diff --git a/runtime/hub/core/spawner/kubernetes.py b/runtime/hub/core/spawner/kubernetes.py index a99b7fff..dfe2e547 100644 --- a/runtime/hub/core/spawner/kubernetes.py +++ b/runtime/hub/core/spawner/kubernetes.py @@ -40,7 +40,6 @@ from kubespawner import KubeSpawner from tornado import web -from core.config import MAX_RENDER_GID from core.metrics import ( pod_failure_total, repo_clone_failed_total, @@ -94,7 +93,6 @@ class RemoteLabKubeSpawner(KubeSpawner): auth_mode: str = "auto-login" single_node_mode: bool = False quota_enabled: bool | None = False - render_gid: int | None = None # Resource configuration (set from config) resource_images: dict[str, str] = {} @@ -156,7 +154,6 @@ def configure_from_config(cls, config: HubConfig) -> None: cls.default_quota = config.quota.defaultQuota cls.minimum_quota_to_start = config.quota.minimumToStart cls.quota_enabled = config.quota.enabled - cls.render_gid = config.gpu_access.renderGid # Extract git clone settings (single source of truth: GitCloneSettings) git_config = config.git_clone @@ -810,28 +807,6 @@ def _reset_per_spawn_state(self) -> None: self._has_git_init_container = False - def _add_gpu_render_gid(self) -> None: - """Add the configured host render group to a GPU resource's pod.""" - if self.render_gid is None: - raise RuntimeError( - "GPU resource requires custom.gpuAccess.renderGid. " - "Set it to the numeric GID of the host render group before spawning GPU resources." - ) - if ( - isinstance(self.render_gid, bool) - or not isinstance(self.render_gid, int) - or not 1 <= self.render_gid <= MAX_RENDER_GID - ): - raise RuntimeError( - "GPU resource requires a valid custom.gpuAccess.renderGid. " - f"Set it to an integer between 1 and {MAX_RENDER_GID} before spawning GPU resources." - ) - - supplemental_gids = list(self.supplemental_gids or []) - if self.render_gid not in supplemental_gids: - supplemental_gids.append(self.render_gid) - self.supplemental_gids = supplemental_gids - def _configure_spawner(self, resource_type: str, gpu_selection: str | None = None) -> None: """Configure the spawner based on the resource type and GPU selection.""" @@ -896,7 +871,6 @@ def _configure_spawner(self, resource_type: str, gpu_selection: str | None = Non if "amd.com/gpu" in requirements: self.extra_resource_guarantees = {"amd.com/gpu": str(requirements["amd.com/gpu"])} self.extra_resource_limits = {"amd.com/gpu": str(requirements["amd.com/gpu"])} - self._add_gpu_render_gid() elif "amd.com/npu" in requirements: self.log.debug("NPU DEVICE PLUGIN are removed, amd.com/npu is no more needed") diff --git a/runtime/hub/tests/test_spawner_gpu_access.py b/runtime/hub/tests/test_spawner_gpu_access.py index edacf4c2..4ea6f476 100644 --- a/runtime/hub/tests/test_spawner_gpu_access.py +++ b/runtime/hub/tests/test_spawner_gpu_access.py @@ -8,7 +8,6 @@ from unittest.mock import patch import pytest -from pydantic import ValidationError ROOT = Path(__file__).resolve().parents[1] CORE = ROOT / "core" @@ -86,7 +85,6 @@ def load_spawner_module(): return load_module("gpu_access_test_spawner", CORE / "spawner" / "kubernetes.py") -config = load_module("core.config", CORE / "config.py") kubernetes = load_spawner_module() RemoteLabKubeSpawner = kubernetes.RemoteLabKubeSpawner @@ -110,10 +108,9 @@ def get_resource_metadata(self, _resource_type): return ResourceMetadata() -def make_spawner(render_gid: int | None, supplemental_gids: list[int] | None = None): +def make_spawner(supplemental_gids: list[int] | None = None): spawner = object.__new__(RemoteLabKubeSpawner) spawner._hub_config = HubConfig() - spawner.render_gid = render_gid spawner.resource_images = {"cpu": "cpu-image", "gpu": "gpu-image"} spawner.resource_requirements = { "cpu": {"cpu": "1", "memory": "1Gi"}, @@ -139,42 +136,36 @@ def make_spawner(render_gid: int | None, supplemental_gids: list[int] | None = N return spawner -def test_gpu_render_gid_is_injected_only_for_gpu_pods(): - spawner = make_spawner(render_gid=993) +def test_gpu_pod_requests_accelerator_without_changing_generic_supplemental_groups(): + spawner = make_spawner(supplemental_gids=[1234]) spawner._configure_spawner("gpu", "gpu-a") gpu_manifest = spawner.get_pod_manifest() + + assert spawner.extra_resource_guarantees == {"amd.com/gpu": "1"} + assert spawner.extra_resource_limits == {"amd.com/gpu": "1"} + assert gpu_manifest["spec"]["securityContext"] == {"fsGroup": 100, "supplementalGroups": [1234]} + spawner._configure_spawner("cpu") cpu_manifest = spawner.get_pod_manifest() - assert gpu_manifest["spec"]["securityContext"] == {"fsGroup": 100, "supplementalGroups": [993]} - assert cpu_manifest["spec"]["securityContext"] == {"fsGroup": 100} + assert spawner.extra_resource_guarantees == {} + assert spawner.extra_resource_limits == {} + assert cpu_manifest["spec"]["securityContext"] == {"fsGroup": 100, "supplementalGroups": [1234]} -def test_gpu_render_gid_preserves_existing_supplemental_groups(): - spawner = make_spawner(render_gid=993, supplemental_gids=[1234]) +def test_gpu_pod_without_generic_supplemental_groups_uses_storage_fs_group_only(): + spawner = make_spawner() spawner._configure_spawner("gpu", "gpu-a") - assert spawner.supplemental_gids == [1234, 993] - - -def test_gpu_spawn_requires_a_host_render_gid(): - spawner = make_spawner(render_gid=None) - - with pytest.raises(RuntimeError, match=r"custom\.gpuAccess\.renderGid"): - spawner._configure_spawner("gpu", "gpu-a") - - -def test_gpu_access_config_validates_render_gid(): - assert config.GpuAccessSettings(renderGid=993).renderGid == 993 - assert config.ParsedConfig.from_dicts(gpu_access={"renderGid": 993}).gpuAccess.renderGid == 993 - with pytest.raises(ValidationError, match="renderGid"): - config.GpuAccessSettings(renderGid=True) + assert spawner.extra_resource_guarantees == {"amd.com/gpu": "1"} + assert spawner.extra_resource_limits == {"amd.com/gpu": "1"} + assert spawner.get_pod_manifest()["spec"]["securityContext"] == {"fsGroup": 100} def test_unauthorized_gpu_selection_is_rejected_before_spawner_configuration(): - spawner = make_spawner(render_gid=993) + spawner = make_spawner() spawner._resolve_user_resources = lambda: ["cpu"] spawner._configure_spawner = lambda *_args: pytest.fail("unauthorized resource configured the spawner") From 3f2c32238ed1c2a4ea411eabb24825506be5895a Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:30:28 +0800 Subject: [PATCH 23/65] refactor(chart): remove GPU GID settings --- runtime/chart/values.schema.json | 2 +- runtime/chart/values.schema.yaml | 14 -------------- runtime/chart/values.yaml | 5 ----- 3 files changed, 1 insertion(+), 20 deletions(-) diff --git a/runtime/chart/values.schema.json b/runtime/chart/values.schema.json index f53227d3..ef7efffc 100644 --- a/runtime/chart/values.schema.json +++ b/runtime/chart/values.schema.json @@ -1 +1 @@ -{"$schema":"http://json-schema.org/draft-07/schema#","type":"object","additionalProperties":false,"required":["imagePullSecrets","hub","proxy","singleuser","ingress","prePuller","custom","cull","debug","rbac","global"],"properties":{"enabled":{"type":["boolean","null"]},"fullnameOverride":{"type":["string","null"]},"nameOverride":{"type":["string","null"]},"imagePullSecret":{"type":"object","required":["create"],"if":{"properties":{"create":{"const":true}}},"then":{"additionalProperties":false,"required":["registry","username","password"],"properties":{"create":{"type":"boolean"},"automaticReferenceInjection":{"type":"boolean"},"registry":{"type":"string"},"username":{"type":"string"},"password":{"type":"string"},"email":{"type":["string","null"]}}}},"imagePullSecrets":{"type":"array"},"hub":{"type":"object","additionalProperties":false,"required":["baseUrl"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"config":{"type":"object","additionalProperties":false,"patternProperties":{"^[A-Z].*$":{"type":"object","additionalProperties":true}},"properties":{"JupyterHub":{"type":"object","additionalProperties":true,"properties":{"subdomain_host":{"type":"string"}}}}},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"baseUrl":{"type":"string"},"command":{"type":"array"},"args":{"type":"array"},"cookieSecret":{"type":["string","null"]},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"db":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["sqlite-pvc","sqlite-memory","mysql","postgres","other"]},"pvc":{"type":"object","additionalProperties":false,"required":["storage"],"properties":{"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"selector":{"type":"object","additionalProperties":true},"storage":{"type":"string"},"accessModes":{"type":"array","items":{"type":["string","null"]}},"storageClassName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"upgrade":{"type":["boolean","null"]},"url":{"type":["string","null"]},"password":{"type":["string","null"]}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"initContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"extraConfig":{"type":"object","additionalProperties":true},"fsGid":{"type":["integer","null"],"minimum":0},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"ports":{"type":"object","additionalProperties":false,"properties":{"appProtocol":{"type":["string","null"]},"nodePort":{"type":["integer","null"],"minimum":0}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPorts":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"existingSecret":{"type":["string","null"]},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"activeServerLimit":{"type":["integer","null"]},"allowNamedServers":{"type":["boolean","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"concurrentSpawnLimit":{"type":["integer","null"]},"consecutiveFailureLimit":{"type":["integer","null"]},"podSecurityContext":{"additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"extraContainers":{"type":"array"},"extraVolumeMounts":{"type":"array"},"extraVolumes":{"type":"array"},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"namedServerLimitPerUser":{"type":["integer","null"]},"redirectToServer":{"type":["boolean","null"]},"resources":{"type":"object","additionalProperties":true},"lifecycle":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"services":{"type":"object","additionalProperties":true,"properties":{"name":{"type":"string"},"admin":{"type":"boolean"},"command":{"type":["string","array"]},"url":{"type":"string"},"api_token":{"type":["string","null"]},"apiToken":{"type":["string","null"]}}},"loadRoles":{"type":"object","additionalProperties":true},"shutdownOnLogout":{"type":["boolean","null"]},"templatePaths":{"type":"array"},"templateVars":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"proxy":{"type":"object","additionalProperties":false,"properties":{"chp":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraCommandLineFlags":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"resources":{"type":"object","additionalProperties":true},"defaultTarget":{"type":["string","null"]},"errorTarget":{"type":["string","null"]},"extraPodSpec":{"type":"object","additionalProperties":true}}},"secretToken":{"type":["string","null"]},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"nodePorts":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"loadBalancerPort":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"disableHttpPort":{"type":"boolean"},"extraPorts":{"type":"array"},"externalIPs":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"loadBalancerSourceRanges":{"type":"array"},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"https":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"type":{"enum":[null,"","letsencrypt","manual","offload","secret"]},"letsencrypt":{"type":"object","additionalProperties":false,"properties":{"contactEmail":{"type":["string","null"]},"acmeServer":{"type":["string","null"]}}},"manual":{"type":"object","additionalProperties":false,"properties":{"key":{"type":["string","null"]},"cert":{"type":["string","null"]}}},"secret":{"type":"object","additionalProperties":false,"properties":{"name":{"type":["string","null"]},"key":{"type":["string","null"]},"crt":{"type":["string","null"]}}},"hosts":{"type":"array"}}},"traefik":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraInitContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraDynamicConfig":{"type":"object","additionalProperties":true},"extraPorts":{"type":"array"},"extraStaticConfig":{"type":"object","additionalProperties":true},"extraVolumes":{"type":"array"},"extraVolumeMounts":{"type":"array"},"hsts":{"type":"object","additionalProperties":false,"required":["includeSubdomains","maxAge","preload"],"properties":{"includeSubdomains":{"type":"boolean"},"maxAge":{"type":"integer"},"preload":{"type":"boolean"}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"secretSync":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}}}},"monitoring":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"namespace":{"type":"string","default":"monitoring"},"releaseLabel":{"type":"string","default":"monitoring"},"hubMetrics":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"allowUnauthenticatedScrape":{"type":"boolean","default":false},"serviceAnnotations":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"serviceMonitor":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"interval":{"type":"string","default":"15s"},"authorization":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":true},"type":{"type":"string","default":"Bearer"},"hubServiceName":{"type":"string","minLength":1,"default":"prometheus-metrics"},"secret":{"type":"object","additionalProperties":false,"properties":{"create":{"type":"boolean","default":true},"name":{"type":"string","default":""},"key":{"type":"string","minLength":1,"default":"token"}}}}}}},"grafana":{"type":"object","additionalProperties":false,"properties":{"dashboard":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"prometheusRule":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"singleuser":{"type":"object","additionalProperties":false,"properties":{"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"podNameTemplate":{"type":["string","null"]},"cpu":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","null"]},"guarantee":{"type":["number","null"]}}},"memory":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","string","null"]},"guarantee":{"type":["number","string","null"]}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"initContainers":{"type":"array"},"profileList":{"type":"array"},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"extraEnv":{"type":["object","array"],"additionalProperties":true},"nodeSelector":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"extraNodeAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAntiAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"cloudMetadata":{"type":"object","additionalProperties":false,"required":["blockWithIptables","ip"],"properties":{"blockWithIptables":{"type":"boolean"},"ip":{"type":"string"}}},"cmd":{"type":["array","string","null"]},"defaultUrl":{"type":["string","null"]},"events":{"type":["boolean","null"]},"extraAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraContainers":{"type":"array"},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPodConfig":{"type":"object","additionalProperties":true},"extraResource":{"type":"object","additionalProperties":false,"properties":{"guarantees":{"type":"object","additionalProperties":true},"limits":{"type":"object","additionalProperties":true}}},"fsGid":{"type":["integer","null"]},"lifecycleHooks":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"networkTools":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}},"serviceAccountName":{"type":["string","null"]},"startTimeout":{"type":["integer","null"]},"storage":{"type":"object","additionalProperties":false,"required":["type","homeMountPath"],"properties":{"capacity":{"type":["string","null"]},"dynamic":{"type":"object","additionalProperties":false,"properties":{"pvcNameTemplate":{"type":["string","null"]},"storageAccessModes":{"type":"array","items":{"type":["string","null"]}},"storageClass":{"type":["string","null"]},"subPath":{"type":["string","null"]},"volumeNameTemplate":{"type":["string","null"]}}},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraVolumeMounts":{"type":["object","array","null"]},"extraVolumes":{"type":["object","array","null"]},"homeMountPath":{"type":"string"},"static":{"type":"object","additionalProperties":false,"properties":{"pvcName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"type":{"enum":["dynamic","static","none"]}}},"allowPrivilegeEscalation":{"type":["boolean","null"]},"uid":{"type":["integer","null"]}}},"scheduling":{"type":"object","additionalProperties":false,"properties":{"userScheduler":{"type":"object","additionalProperties":false,"required":["enabled","plugins","pluginConfig","logLevel"],"properties":{"enabled":{"type":"boolean"},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"logLevel":{"type":"integer"},"plugins":{"type":"object","additionalProperties":true},"pluginConfig":{"type":"array"},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"podPriority":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"globalDefault":{"type":"boolean"},"defaultPriority":{"type":"integer"},"imagePullerPriority":{"type":"integer"},"userPlaceholderPriority":{"type":"integer"}}},"userPlaceholder":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraPodSpec":{"type":"object","additionalProperties":true}}},"corePods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}},"userPods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}}}},"ingress":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"ingressClassName":{"type":["string","null"]},"hosts":{"type":"array"},"pathSuffix":{"type":["string","null"]},"pathType":{"enum":["Prefix","Exact","ImplementationSpecific"]},"tls":{"type":"array"},"extraPaths":{"type":"array"}}},"httpRoute":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"hostnames":{"type":"array"},"gateway":{"type":"object","additionalProperties":false,"required":["name"],"properties":{"name":{"type":"string"},"namespace":{"type":"string"},"sectionName":{"type":"string"}}}}},"prePuller":{"type":"object","additionalProperties":false,"required":["hook","continuous"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"hook":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"pullOnlyOnChanges":{"type":"boolean"},"podSchedulingWaitDuration":{"type":"integer"},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"serviceAccountImagePuller":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"continuous":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"pullProfileListImages":{"type":"boolean"},"extraImages":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]}}}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"pause":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}}}}}},"custom":{"type":"object","additionalProperties":true,"properties":{"authMode":{"type":"string","enum":["auto-login","dummy","github","multi"]},"adminUser":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"}}},"gpuAccess":{"type":"object","additionalProperties":false,"properties":{"renderGid":{"type":["integer","null"],"minimum":1,"maximum":4294967294}}},"notifications":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"topbar":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}},"homepage":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"legacyAnnouncementFallback":{"type":"boolean"},"items":{"type":"array","items":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}}}}}}},"accelerators":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"displayName":{"type":"string"},"description":{"type":"string"},"nodeSelector":{"type":"object","additionalProperties":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"quotaRate":{"type":"integer","minimum":1}}}},"resources":{"type":"object","additionalProperties":false,"properties":{"images":{"type":"object","additionalProperties":{"type":"string"}},"groupOrder":{"type":"array","items":{"type":"string"}},"requirements":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"cpu":{"type":"string"},"memory":{"type":"string"},"memory_limit":{"type":"string"},"amd.com/gpu":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"group":{"type":"string"},"description":{"type":"string"},"subDescription":{"type":"string"},"accelerator":{"type":"string"},"acceleratorKeys":{"type":"array","items":{"type":"string"}},"allowGitClone":{"type":"boolean"},"defaultPath":{"type":["string","null"]},"launchMode":{"type":"string","enum":["jupyterlab","code-server"]},"resourceType":{"type":"string","enum":["notebook","browser-ide"]},"env":{"type":"object","additionalProperties":{"type":"string"}},"acceleratorOverrides":{"type":"object","additionalProperties":{"type":"object","properties":{"image":{"type":"string"},"env":{"type":"object","additionalProperties":{"type":"string"}}}}}}}}}},"teams":{"type":"object","additionalProperties":false,"properties":{"mapping":{"type":"object","additionalProperties":{"type":"array","items":{"type":"string"}}}}},"quota":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"cpuRate":{"type":"integer","minimum":1},"minimumToStart":{"type":"integer","minimum":0},"defaultQuota":{"type":"integer","minimum":0},"refreshRules":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"schedule":{"type":"string"},"action":{"type":"string","enum":["add","set"]},"amount":{"type":"integer"},"maxBalance":{"type":["integer","null"]},"minBalance":{"type":["integer","null"]},"targets":{"type":"object","additionalProperties":false,"properties":{"includeUnlimited":{"type":"boolean"},"balanceBelow":{"type":["integer","null"]},"balanceAbove":{"type":["integer","null"]},"includeUsers":{"type":"array","items":{"type":"string"}},"excludeUsers":{"type":"array","items":{"type":"string"}},"usernamePattern":{"type":"string"}}}}}}}},"gitClone":{"type":"object","additionalProperties":false,"properties":{"initContainerImage":{"type":"string"},"allowedProviders":{"type":"array","items":{"type":"string"}},"maxCloneTimeout":{"type":"integer","minimum":10},"githubAppName":{"type":"string"},"defaultAccessToken":{"type":"string"},"defaultPersistence":{"type":"boolean"},"allowPersistenceChoice":{"type":"boolean"}}},"hub":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"notebook":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"codeServer":{"type":"object","additionalProperties":false,"properties":{"extraTrustedDomains":{"type":"array","items":{"type":"string"}}}},"apiService":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":["","IfNotPresent","Always","Never","null"]}}}}}}},"cull":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"users":{"type":["boolean","null"]},"adminUsers":{"type":["boolean","null"]},"removeNamedServers":{"type":["boolean","null"]},"timeout":{"type":["integer","null"]},"every":{"type":["integer","null"]},"concurrency":{"type":["integer","null"]},"maxAge":{"type":["integer","null"]}}},"debug":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"}}},"rbac":{"type":"object","additionalProperties":false,"required":["create"],"properties":{"enabled":{"type":"boolean"},"create":{"type":"boolean"}}},"global":{"type":"object","additionalProperties":true,"properties":{"safeToShowValues":{"type":"boolean"}}}}} \ No newline at end of file +{"$schema":"http://json-schema.org/draft-07/schema#","type":"object","additionalProperties":false,"required":["imagePullSecrets","hub","proxy","singleuser","ingress","prePuller","custom","cull","debug","rbac","global"],"properties":{"enabled":{"type":["boolean","null"]},"fullnameOverride":{"type":["string","null"]},"nameOverride":{"type":["string","null"]},"imagePullSecret":{"type":"object","required":["create"],"if":{"properties":{"create":{"const":true}}},"then":{"additionalProperties":false,"required":["registry","username","password"],"properties":{"create":{"type":"boolean"},"automaticReferenceInjection":{"type":"boolean"},"registry":{"type":"string"},"username":{"type":"string"},"password":{"type":"string"},"email":{"type":["string","null"]}}}},"imagePullSecrets":{"type":"array"},"hub":{"type":"object","additionalProperties":false,"required":["baseUrl"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"config":{"type":"object","additionalProperties":false,"patternProperties":{"^[A-Z].*$":{"type":"object","additionalProperties":true}},"properties":{"JupyterHub":{"type":"object","additionalProperties":true,"properties":{"subdomain_host":{"type":"string"}}}}},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"baseUrl":{"type":"string"},"command":{"type":"array"},"args":{"type":"array"},"cookieSecret":{"type":["string","null"]},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"db":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["sqlite-pvc","sqlite-memory","mysql","postgres","other"]},"pvc":{"type":"object","additionalProperties":false,"required":["storage"],"properties":{"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"selector":{"type":"object","additionalProperties":true},"storage":{"type":"string"},"accessModes":{"type":"array","items":{"type":["string","null"]}},"storageClassName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"upgrade":{"type":["boolean","null"]},"url":{"type":["string","null"]},"password":{"type":["string","null"]}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"initContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"extraConfig":{"type":"object","additionalProperties":true},"fsGid":{"type":["integer","null"],"minimum":0},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"ports":{"type":"object","additionalProperties":false,"properties":{"appProtocol":{"type":["string","null"]},"nodePort":{"type":["integer","null"],"minimum":0}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPorts":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"existingSecret":{"type":["string","null"]},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"activeServerLimit":{"type":["integer","null"]},"allowNamedServers":{"type":["boolean","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"concurrentSpawnLimit":{"type":["integer","null"]},"consecutiveFailureLimit":{"type":["integer","null"]},"podSecurityContext":{"additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"extraContainers":{"type":"array"},"extraVolumeMounts":{"type":"array"},"extraVolumes":{"type":"array"},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"namedServerLimitPerUser":{"type":["integer","null"]},"redirectToServer":{"type":["boolean","null"]},"resources":{"type":"object","additionalProperties":true},"lifecycle":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"services":{"type":"object","additionalProperties":true,"properties":{"name":{"type":"string"},"admin":{"type":"boolean"},"command":{"type":["string","array"]},"url":{"type":"string"},"api_token":{"type":["string","null"]},"apiToken":{"type":["string","null"]}}},"loadRoles":{"type":"object","additionalProperties":true},"shutdownOnLogout":{"type":["boolean","null"]},"templatePaths":{"type":"array"},"templateVars":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"proxy":{"type":"object","additionalProperties":false,"properties":{"chp":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraCommandLineFlags":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"livenessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"readinessProbe":{"type":"object","additionalProperties":true,"required":["enabled"],"if":{"properties":{"enabled":{"const":true}}},"then":{}},"resources":{"type":"object","additionalProperties":true},"defaultTarget":{"type":["string","null"]},"errorTarget":{"type":["string","null"]},"extraPodSpec":{"type":"object","additionalProperties":true}}},"secretToken":{"type":["string","null"]},"service":{"type":"object","additionalProperties":false,"properties":{"type":{"enum":["ClusterIP","NodePort","LoadBalancer","ExternalName"]},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"nodePorts":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"loadBalancerPort":{"type":"object","additionalProperties":false,"properties":{"http":{"type":["integer","null"]},"https":{"type":["integer","null"]}}},"disableHttpPort":{"type":"boolean"},"extraPorts":{"type":"array"},"externalIPs":{"type":"array"},"loadBalancerIP":{"type":["string","null"]},"loadBalancerSourceRanges":{"type":"array"},"ipFamilyPolicy":{"type":["string"]},"ipFamilies":{"type":"array"}}},"https":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"type":{"enum":[null,"","letsencrypt","manual","offload","secret"]},"letsencrypt":{"type":"object","additionalProperties":false,"properties":{"contactEmail":{"type":["string","null"]},"acmeServer":{"type":["string","null"]}}},"manual":{"type":"object","additionalProperties":false,"properties":{"key":{"type":["string","null"]},"cert":{"type":["string","null"]}}},"secret":{"type":"object","additionalProperties":false,"properties":{"name":{"type":["string","null"]},"key":{"type":["string","null"]},"crt":{"type":["string","null"]}}},"hosts":{"type":"array"}}},"traefik":{"type":"object","additionalProperties":false,"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"extraInitContainers":{"type":"array"},"extraEnv":{"type":["object","array"],"additionalProperties":true},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraDynamicConfig":{"type":"object","additionalProperties":true},"extraPorts":{"type":"array"},"extraStaticConfig":{"type":"object","additionalProperties":true},"extraVolumes":{"type":"array"},"extraVolumeMounts":{"type":"array"},"hsts":{"type":"object","additionalProperties":false,"required":["includeSubdomains","maxAge","preload"],"properties":{"includeSubdomains":{"type":"boolean"},"maxAge":{"type":"integer"},"preload":{"type":"boolean"}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"deploymentStrategy":{"type":"object","additionalProperties":false,"properties":{"rollingUpdate":{"type":["string","null"]},"type":{"type":["string","null"]}}},"secretSync":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}}}},"monitoring":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"namespace":{"type":"string","default":"monitoring"},"releaseLabel":{"type":"string","default":"monitoring"},"hubMetrics":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"allowUnauthenticatedScrape":{"type":"boolean","default":false},"serviceAnnotations":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"serviceMonitor":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false},"interval":{"type":"string","default":"15s"},"authorization":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":true},"type":{"type":"string","default":"Bearer"},"hubServiceName":{"type":"string","minLength":1,"default":"prometheus-metrics"},"secret":{"type":"object","additionalProperties":false,"properties":{"create":{"type":"boolean","default":true},"name":{"type":"string","default":""},"key":{"type":"string","minLength":1,"default":"token"}}}}}}},"grafana":{"type":"object","additionalProperties":false,"properties":{"dashboard":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"prometheusRule":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean","default":false}}}}},"singleuser":{"type":"object","additionalProperties":false,"properties":{"networkPolicy":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"ingress":{"type":"array"},"egress":{"type":"array"},"egressAllowRules":{"type":"object","additionalProperties":false,"properties":{"cloudMetadataServer":{"type":"boolean"},"dnsPortsCloudMetadataServer":{"type":"boolean"},"dnsPortsKubeSystemNamespace":{"type":"boolean"},"dnsPortsPrivateIPs":{"type":"boolean"},"nonPrivateIPs":{"type":"boolean"},"privateIPs":{"type":"boolean"}}},"interNamespaceAccessLabels":{"enum":["accept","ignore"]},"allowedIngressPorts":{"type":"array"}}},"podNameTemplate":{"type":["string","null"]},"cpu":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","null"]},"guarantee":{"type":["number","null"]}}},"memory":{"type":"object","additionalProperties":false,"properties":{"limit":{"type":["number","string","null"]},"guarantee":{"type":["number","string","null"]}}},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"initContainers":{"type":"array"},"profileList":{"type":"array"},"extraFiles":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["mountPath"],"oneOf":[{"required":["data"]},{"required":["stringData"]},{"required":["binaryData"]}],"properties":{"mountPath":{"type":"string"},"data":{"type":"object","additionalProperties":true},"stringData":{"type":"string"},"binaryData":{"type":"string"},"mode":{"type":"number"}}}}},"extraEnv":{"type":["object","array"],"additionalProperties":true},"nodeSelector":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"extraNodeAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"extraPodAntiAffinity":{"type":"object","additionalProperties":false,"properties":{"required":{"type":"array"},"preferred":{"type":"array"}}},"cloudMetadata":{"type":"object","additionalProperties":false,"required":["blockWithIptables","ip"],"properties":{"blockWithIptables":{"type":"boolean"},"ip":{"type":"string"}}},"cmd":{"type":["array","string","null"]},"defaultUrl":{"type":["string","null"]},"events":{"type":["boolean","null"]},"extraAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraContainers":{"type":"array"},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraPodConfig":{"type":"object","additionalProperties":true},"extraResource":{"type":"object","additionalProperties":false,"properties":{"guarantees":{"type":"object","additionalProperties":true},"limits":{"type":"object","additionalProperties":true}}},"fsGid":{"type":["integer","null"]},"lifecycleHooks":{"type":"object","additionalProperties":false,"properties":{"postStart":{"type":"object","additionalProperties":true},"preStop":{"type":"object","additionalProperties":true}}},"networkTools":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true}}},"serviceAccountName":{"type":["string","null"]},"startTimeout":{"type":["integer","null"]},"storage":{"type":"object","additionalProperties":false,"required":["type","homeMountPath"],"properties":{"capacity":{"type":["string","null"]},"dynamic":{"type":"object","additionalProperties":false,"properties":{"pvcNameTemplate":{"type":["string","null"]},"storageAccessModes":{"type":"array","items":{"type":["string","null"]}},"storageClass":{"type":["string","null"]},"subPath":{"type":["string","null"]},"volumeNameTemplate":{"type":["string","null"]}}},"extraLabels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"extraVolumeMounts":{"type":["object","array","null"]},"extraVolumes":{"type":["object","array","null"]},"homeMountPath":{"type":"string"},"static":{"type":"object","additionalProperties":false,"properties":{"pvcName":{"type":["string","null"]},"subPath":{"type":["string","null"]}}},"type":{"enum":["dynamic","static","none"]}}},"allowPrivilegeEscalation":{"type":["boolean","null"]},"uid":{"type":["integer","null"]}}},"scheduling":{"type":"object","additionalProperties":false,"properties":{"userScheduler":{"type":"object","additionalProperties":false,"required":["enabled","plugins","pluginConfig","logLevel"],"properties":{"enabled":{"type":"boolean"},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"pdb":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"maxUnavailable":{"type":["integer","null"]},"minAvailable":{"type":["integer","null"]}}},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"logLevel":{"type":"integer"},"plugins":{"type":"object","additionalProperties":true},"pluginConfig":{"type":"array"},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"extraPodSpec":{"type":"object","additionalProperties":true}}},"podPriority":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"globalDefault":{"type":"boolean"},"defaultPriority":{"type":"integer"},"imagePullerPriority":{"type":"integer"},"userPlaceholderPriority":{"type":"integer"}}},"userPlaceholder":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"replicas":{"type":"integer"},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"containerSecurityContext":{"type":"object","additionalProperties":true},"extraPodSpec":{"type":"object","additionalProperties":true}}},"corePods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}},"userPods":{"type":"object","additionalProperties":false,"properties":{"tolerations":{"type":"array"},"nodeAffinity":{"type":"object","additionalProperties":false,"properties":{"matchNodePurpose":{"enum":["ignore","prefer","require"]}}}}}}},"ingress":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"ingressClassName":{"type":["string","null"]},"hosts":{"type":"array"},"pathSuffix":{"type":["string","null"]},"pathType":{"enum":["Prefix","Exact","ImplementationSpecific"]},"tls":{"type":"array"},"extraPaths":{"type":"array"}}},"httpRoute":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"hostnames":{"type":"array"},"gateway":{"type":"object","additionalProperties":false,"required":["name"],"properties":{"name":{"type":"string"},"namespace":{"type":"string"},"sectionName":{"type":"string"}}}}},"prePuller":{"type":"object","additionalProperties":false,"required":["hook","continuous"],"properties":{"revisionHistoryLimit":{"type":["integer","null"],"minimum":0},"labels":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}},"resources":{"type":"object","additionalProperties":true},"extraTolerations":{"type":"array"},"hook":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"pullOnlyOnChanges":{"type":"boolean"},"podSchedulingWaitDuration":{"type":"integer"},"nodeSelector":{"type":"object","additionalProperties":true},"tolerations":{"type":"array"},"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}},"resources":{"type":"object","additionalProperties":true},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"serviceAccountImagePuller":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"continuous":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"serviceAccount":{"type":"object","required":["create"],"additionalProperties":false,"properties":{"create":{"type":"boolean"},"name":{"type":["string","null"]},"annotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"daemonsetAnnotations":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"string"}}}}},"pullProfileListImages":{"type":"boolean"},"extraImages":{"type":"object","additionalProperties":false,"patternProperties":{".*":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]}}}}},"containerSecurityContext":{"type":"object","additionalProperties":true},"pause":{"type":"object","additionalProperties":false,"properties":{"containerSecurityContext":{"type":"object","additionalProperties":true},"image":{"type":"object","additionalProperties":false,"required":["name","tag"],"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":[null,"","IfNotPresent","Always","Never"]},"pullSecrets":{"type":"array"}}}}}}},"custom":{"type":"object","additionalProperties":true,"properties":{"authMode":{"type":"string","enum":["auto-login","dummy","github","multi"]},"adminUser":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"}}},"notifications":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"topbar":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}},"homepage":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"legacyAnnouncementFallback":{"type":"boolean"},"items":{"type":"array","items":{"type":"object","additionalProperties":false,"if":{"required":["enabled"],"properties":{"enabled":{"const":true}}},"then":{"required":["id"],"properties":{"id":{"minLength":1}},"anyOf":[{"required":["title"],"properties":{"title":{"minLength":1}}},{"required":["message"],"properties":{"message":{"minLength":1}}}]},"properties":{"enabled":{"type":"boolean"},"id":{"type":"string"},"version":{"type":"string"},"severity":{"type":"string","enum":["info","success","warning","danger"]},"dismissible":{"type":"boolean"},"eyebrow":{"type":"string"},"title":{"type":"string"},"message":{"type":"string"},"format":{"type":"string","enum":["text","markdown","html"]},"link":{"type":"object","additionalProperties":false,"properties":{"label":{"type":"string"},"url":{"type":"string"}}},"startsAt":{"type":"string"},"endsAt":{"type":"string"}}}}}}}},"accelerators":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"displayName":{"type":"string"},"description":{"type":"string"},"nodeSelector":{"type":"object","additionalProperties":{"type":"string"}},"env":{"type":"object","additionalProperties":{"type":"string"}},"quotaRate":{"type":"integer","minimum":1}}}},"resources":{"type":"object","additionalProperties":false,"properties":{"images":{"type":"object","additionalProperties":{"type":"string"}},"groupOrder":{"type":"array","items":{"type":"string"}},"requirements":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"cpu":{"type":"string"},"memory":{"type":"string"},"memory_limit":{"type":"string"},"amd.com/gpu":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"group":{"type":"string"},"description":{"type":"string"},"subDescription":{"type":"string"},"accelerator":{"type":"string"},"acceleratorKeys":{"type":"array","items":{"type":"string"}},"allowGitClone":{"type":"boolean"},"defaultPath":{"type":["string","null"]},"launchMode":{"type":"string","enum":["jupyterlab","code-server"]},"resourceType":{"type":"string","enum":["notebook","browser-ide"]},"env":{"type":"object","additionalProperties":{"type":"string"}},"acceleratorOverrides":{"type":"object","additionalProperties":{"type":"object","properties":{"image":{"type":"string"},"env":{"type":"object","additionalProperties":{"type":"string"}}}}}}}}}},"teams":{"type":"object","additionalProperties":false,"properties":{"mapping":{"type":"object","additionalProperties":{"type":"array","items":{"type":"string"}}}}},"quota":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":["boolean","null"]},"cpuRate":{"type":"integer","minimum":1},"minimumToStart":{"type":"integer","minimum":0},"defaultQuota":{"type":"integer","minimum":0},"refreshRules":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"schedule":{"type":"string"},"action":{"type":"string","enum":["add","set"]},"amount":{"type":"integer"},"maxBalance":{"type":["integer","null"]},"minBalance":{"type":["integer","null"]},"targets":{"type":"object","additionalProperties":false,"properties":{"includeUnlimited":{"type":"boolean"},"balanceBelow":{"type":["integer","null"]},"balanceAbove":{"type":["integer","null"]},"includeUsers":{"type":"array","items":{"type":"string"}},"excludeUsers":{"type":"array","items":{"type":"string"}},"usernamePattern":{"type":"string"}}}}}}}},"gitClone":{"type":"object","additionalProperties":false,"properties":{"initContainerImage":{"type":"string"},"allowedProviders":{"type":"array","items":{"type":"string"}},"maxCloneTimeout":{"type":"integer","minimum":10},"githubAppName":{"type":"string"},"defaultAccessToken":{"type":"string"},"defaultPersistence":{"type":"boolean"},"allowPersistenceChoice":{"type":"boolean"}}},"hub":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"notebook":{"type":"object","additionalProperties":false,"properties":{"allowedOrigins":{"type":"array","items":{"type":"string"}}}},"codeServer":{"type":"object","additionalProperties":false,"properties":{"extraTrustedDomains":{"type":"array","items":{"type":"string"}}}},"apiService":{"type":"object","additionalProperties":false,"properties":{"image":{"type":"object","additionalProperties":false,"properties":{"name":{"type":"string"},"tag":{"type":"string"},"pullPolicy":{"enum":["","IfNotPresent","Always","Never","null"]}}}}}}},"cull":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"},"users":{"type":["boolean","null"]},"adminUsers":{"type":["boolean","null"]},"removeNamedServers":{"type":["boolean","null"]},"timeout":{"type":["integer","null"]},"every":{"type":["integer","null"]},"concurrency":{"type":["integer","null"]},"maxAge":{"type":["integer","null"]}}},"debug":{"type":"object","additionalProperties":false,"required":["enabled"],"properties":{"enabled":{"type":"boolean"}}},"rbac":{"type":"object","additionalProperties":false,"required":["create"],"properties":{"enabled":{"type":"boolean"},"create":{"type":"boolean"}}},"global":{"type":"object","additionalProperties":true,"properties":{"safeToShowValues":{"type":"boolean"}}}}} \ No newline at end of file diff --git a/runtime/chart/values.schema.yaml b/runtime/chart/values.schema.yaml index 3eab8688..22ff5fec 100644 --- a/runtime/chart/values.schema.yaml +++ b/runtime/chart/values.schema.yaml @@ -3195,20 +3195,6 @@ properties: Enable auto-admin creation on first install. Credentials will be stored in `jupyterhub-admin-credentials` secret. - gpuAccess: - type: object - additionalProperties: false - description: | - Host group access settings for GPU-enabled user pods. - properties: - renderGid: - type: [integer, "null"] - minimum: 1 - maximum: 4294967294 - description: | - Numeric GID of the host render group. GPU resources receive this - as a supplemental group; CPU resources do not. - notifications: type: object additionalProperties: false diff --git a/runtime/chart/values.yaml b/runtime/chart/values.yaml index e888f22f..a548691f 100644 --- a/runtime/chart/values.yaml +++ b/runtime/chart/values.yaml @@ -50,11 +50,6 @@ custom: # Define these in runtime/values.yaml, not here accelerators: {} - # Host render-group access for GPU user pods. The installer overlay sets this - # to the detected host render GID when GPU access is provisioned. - gpuAccess: - renderGid: null - # Resource images, requirements, and metadata # Define these in runtime/values.yaml, not here resources: From 075f29d3742ee8d839ac46a72a1884ffa6bb4998 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:30:28 +0800 Subject: [PATCH 24/65] fix(runtime): keep storage group only --- runtime/values-multi-nodes.yaml.example | 12 +++++------- runtime/values.yaml | 7 ++----- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/runtime/values-multi-nodes.yaml.example b/runtime/values-multi-nodes.yaml.example index 63c48382..4af6d259 100644 --- a/runtime/values-multi-nodes.yaml.example +++ b/runtime/values-multi-nodes.yaml.example @@ -22,7 +22,9 @@ # cp values-multi-nodes.yaml.example values-multi-nodes.yaml # # Prerequisites: -# - Install the AMD GPU device plugin and ROCm node labeller on GPU nodes. +# - The infrastructure owner must deploy and maintain the AMD GPU device plugin +# and ROCm node labeller outside AUPLC. Before Helm, run the readiness and +# capacity checks in deploy/README.md. # - Install an RWX-capable StorageClass for user homes; this example uses the # NFS provisioner from deploy/k8s/nfs-provisioner with class nfs-client. # - Create registry pull secrets only if you use private images. @@ -139,11 +141,6 @@ custom: defaultPersistence: true allowPersistenceChoice: false - # Generated deployment overlays resolve this from corroborated host evidence. - # Keep null in the base example; do not choose a fleet GID manually here. - gpuAccess: - renderGid: null - # -------------------------------------------------------------------------- # Accelerator Configuration # -------------------------------------------------------------------------- @@ -581,7 +578,8 @@ monitoring: enabled: false singleuser: - # Must match the storage ownership group used by the shared volume. + # Storage ownership only. AUPLC runtime does not inject GPU groups. + # An amd.com/gpu request is the device-visibility boundary; injected nodes are 0666. fsGid: 100 storage: dynamic: diff --git a/runtime/values.yaml b/runtime/values.yaml index 7bce7308..3dd59180 100644 --- a/runtime/values.yaml +++ b/runtime/values.yaml @@ -61,10 +61,6 @@ custom: adminUser: enabled: false - # The installer overlay supplies the host render GID for GPU user pods. - gpuAccess: - renderGid: null - # ============================================================================ # Notifications # ============================================================================ @@ -672,7 +668,8 @@ monitoring: enabled: false singleuser: - # Preserve storage volume ownership without replacing KubeSpawner's security context. + # Storage ownership only. AUPLC runtime does not inject GPU groups. + # An amd.com/gpu request is the device-visibility boundary; injected nodes are 0666. fsGid: 100 storage: From ee6c31ce3f85826339f1df5a6fe5a8bab117f257 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:30:28 +0800 Subject: [PATCH 25/65] refactor(installer): simplify GPU host policy --- auplc_installer/gpu_access.py | 213 ++------ tests/installer/test_gpu_access.py | 523 +++++--------------- tests/scripts/test_gpu_image_permissions.py | 6 +- 3 files changed, 160 insertions(+), 582 deletions(-) diff --git a/auplc_installer/gpu_access.py b/auplc_installer/gpu_access.py index 7c46991a..c3a946a7 100644 --- a/auplc_installer/gpu_access.py +++ b/auplc_installer/gpu_access.py @@ -1,24 +1,14 @@ # Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. -"""Single-node AMD GPU device-access source of truth. - -The host's existing ``render`` group is authoritative. Its numeric GID is -persisted here so installer reruns and runtime-only commands cannot silently -select a different permission model. -""" +"""Single-node AMD GPU device-access reconciler.""" from __future__ import annotations -import json -from dataclasses import dataclass from pathlib import Path from typing import Protocol from auplc_installer.util import InstallerError, run, run_capture -GPU_ACCESS_STATE_VERSION = 1 -MAX_RENDER_GID = (2**32) - 2 -GPU_ACCESS_STATE_PATH = Path("/var/lib/auplc/gpu-access.json") GPU_ACCESS_RULES_PATH = Path("/etc/udev/rules.d/70-auplc-gpu-access.rules") LEGACY_KFD_RULES_PATH = Path("/etc/udev/rules.d/70-kfd.rules") LEGACY_AMDGPU_RULES_PATH = Path("/etc/udev/rules.d/70-amdgpu.rules") @@ -46,8 +36,9 @@ UDEV_MANAGED_MARKER = "# Managed by auplc-installer: AMD GPU device access." CANONICAL_UDEV_RULES = ( f"{UDEV_MANAGED_MARKER}\n" - 'KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0660"\n' - 'SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0660"\n' + 'KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0666"\n' + 'SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0666"\n' + 'SUBSYSTEM=="drm", KERNEL=="card*", DRIVERS=="amdgpu", OWNER="root", GROUP="video", MODE="0666"\n' ) _FSYNC_PATH_SCRIPT = ( "import os\n" @@ -59,44 +50,40 @@ " os.close(fd)\n" ) _VERIFY_DEVICE_ACCESS_SCRIPT = ( - "import os, pathlib, stat, sys\n" - "gid = int(sys.argv[1])\n" - "paths = [pathlib.Path('/dev/kfd')]\n" - "for node in pathlib.Path('/sys/class/drm').glob('renderD*'):\n" + "import grp, pathlib, stat\n" + "drm = pathlib.Path('/sys/class/drm')\n" + "devices = [(pathlib.Path('/dev/kfd'), 'render', 0o666)]\n" + "render_nodes = []\n" + "for node in drm.glob('renderD*'):\n" + " driver = node / 'device' / 'driver'\n" + " if driver.exists() and driver.resolve().name == 'amdgpu':\n" + " render_nodes.append(pathlib.Path('/dev/dri') / node.name)\n" + "if not render_nodes: raise SystemExit('no AMD renderD device found')\n" + "devices.extend((path, 'render', 0o666) for path in render_nodes)\n" + "card_nodes = []\n" + "for node in drm.glob('card*'):\n" " driver = node / 'device' / 'driver'\n" - " if driver.exists() and driver.resolve().name == 'amdgpu': paths.append(pathlib.Path('/dev/dri') / node.name)\n" - "if len(paths) == 1: raise SystemExit('no AMD renderD device found')\n" - "for path in paths:\n" + " if driver.exists() and driver.resolve().name == 'amdgpu':\n" + " card_nodes.append(pathlib.Path('/dev/dri') / node.name)\n" + "if not card_nodes: raise SystemExit('no AMD card device found')\n" + "devices.extend((path, 'video', 0o666) for path in card_nodes)\n" + "for path, expected_group, expected_mode in devices:\n" " data = path.lstat()\n" - " if not stat.S_ISCHR(data.st_mode) or data.st_uid != 0 or data.st_gid != gid or stat.S_IMODE(data.st_mode) != 0o660: raise SystemExit(f'bad GPU device access: {path}')\n" + " try:\n" + " group_name = grp.getgrgid(data.st_gid).gr_name\n" + " except KeyError:\n" + " raise SystemExit(f'unknown GPU device group: {path}')\n" + " if not stat.S_ISCHR(data.st_mode) or data.st_uid != 0 or group_name != expected_group or stat.S_IMODE(data.st_mode) != expected_mode:\n" + " raise SystemExit(f'bad GPU device access: {path}')\n" ) -@dataclass(frozen=True) -class GpuAccessState: - """Versioned, immutable record of the host render-group GID.""" - - render_gid: int - version: int = GPU_ACCESS_STATE_VERSION - - def __post_init__(self) -> None: - if self.version != GPU_ACCESS_STATE_VERSION: - raise InstallerError(f"Unsupported GPU access state version: {self.version!r}") - _validate_render_gid(self.render_gid) - - class GpuAccessHost(Protocol): """Privileged host-operation seam for GPU access provisioning.""" - def get_group_entry(self, group_name: str) -> str: - """Return the NSS group record for ``group_name``.""" - def read_text(self, path: Path) -> str | None: """Return a privileged file's text, or ``None`` when it is absent.""" - def write_state_atomically(self, path: Path, text: str) -> None: - """Atomically replace a state file with same-directory persistence.""" - def write_udev_rule(self, path: Path, text: str) -> None: """Write a managed udev rule after reconciliation has authorized it.""" @@ -112,8 +99,8 @@ def settle_udev(self) -> None: def remove_udev_rule(self, path: Path) -> None: """Remove an explicitly recognized legacy udev rule.""" - def verify_device_access(self, render_gid: int) -> None: - """Verify the relevant GPU device inodes use the requested access contract.""" + def verify_device_access(self) -> None: + """Verify the relevant GPU device inodes use the host access contract.""" def is_symlink(self, path: Path) -> bool: """Return whether ``path`` is a symlink without following it.""" @@ -131,12 +118,6 @@ def is_directory(self, path: Path) -> bool: class SystemGpuAccessHost: """Production host adapter using the installer's sudo-aware command helpers.""" - def get_group_entry(self, group_name: str) -> str: - result = run_capture(["getent", "group", group_name], check=False) - if result.returncode != 0: - return "" - return result.stdout or "" - def read_text(self, path: Path) -> str | None: exists = run(["test", "-e", str(path)], sudo=True, check=False) if exists.returncode != 0: @@ -144,10 +125,6 @@ def read_text(self, path: Path) -> str | None: result = run_capture(["cat", str(path)], sudo=True) return result.stdout or "" - def write_state_atomically(self, path: Path, text: str) -> None: - """Durably replace state with a same-directory temporary file.""" - self._write_text_atomically(path, text) - def write_udev_rule(self, path: Path, text: str) -> None: self._write_text_atomically(path, text) @@ -161,7 +138,7 @@ def _write_text_atomically(self, path: Path, text: str) -> None: ) temporary_path = (temporary_result.stdout or "").strip() if not temporary_path: - raise InstallerError(f"Could not create temporary GPU access state beside {path}") + raise InstallerError(f"Could not create temporary GPU access rule beside {path}") try: run(["tee", temporary_path], sudo=True, input_text=text) @@ -188,8 +165,8 @@ def settle_udev(self) -> None: def remove_udev_rule(self, path: Path) -> None: run(["rm", "-f", str(path)], sudo=True) - def verify_device_access(self, render_gid: int) -> None: - run(["python3", "-c", _VERIFY_DEVICE_ACCESS_SCRIPT, str(render_gid)], sudo=True) + def verify_device_access(self) -> None: + run(["python3", "-c", _VERIFY_DEVICE_ACCESS_SCRIPT], sudo=True) def is_symlink(self, path: Path) -> bool: return run(["test", "-L", str(path)], sudo=True, check=False).returncode == 0 @@ -204,117 +181,26 @@ def is_directory(self, path: Path) -> bool: return run(["test", "-d", str(path)], sudo=True, check=False).returncode == 0 -def serialize_gpu_access_state(state: GpuAccessState) -> str: - """Return the canonical on-disk JSON representation for ``state``.""" - return ( - json.dumps( - {"renderGid": state.render_gid, "version": state.version}, - separators=(",", ":"), - sort_keys=True, - ) - + "\n" - ) - - -def parse_gpu_access_state(text: str) -> GpuAccessState: - """Parse strict versioned GPU access state, failing closed on bad input.""" - try: - payload = json.loads(text) - except (TypeError, json.JSONDecodeError) as exc: - raise InstallerError("Malformed GPU access state") from exc - - if not isinstance(payload, dict) or set(payload) != {"renderGid", "version"}: - raise InstallerError("Malformed GPU access state") - - version = payload["version"] - render_gid = payload["renderGid"] - if type(version) is not int or version != GPU_ACCESS_STATE_VERSION: - raise InstallerError("Unsupported GPU access state version") - _validate_render_gid(render_gid) - return GpuAccessState(render_gid=render_gid, version=version) - - -def resolve_render_gid(getent_output: str) -> int: - """Parse the numeric GID from one ``getent group render`` record.""" - if not isinstance(getent_output, str): - raise InstallerError("Could not resolve the host render group") - - lines = getent_output.splitlines() - if len(lines) != 1: - raise InstallerError("Could not resolve the host render group") - - fields = lines[0].split(":") - if len(fields) != 4 or fields[0] != "render": - raise InstallerError("Could not resolve the host render group") - - raw_gid = fields[2] - if not raw_gid.isascii() or not raw_gid.isdecimal(): - raise InstallerError("Could not resolve the host render group") - - render_gid = int(raw_gid) - _validate_render_gid(render_gid) - return render_gid - - def render_udev_rules() -> str: - """Return the canonical, least-privilege AMD GPU udev rules.""" + """Return the canonical AMD GPU host-device udev rules.""" return CANONICAL_UDEV_RULES -def provision_gpu_access(host: GpuAccessHost | None = None) -> GpuAccessState: - """Create or reuse immutable state and reconcile the managed udev rule. - - When state is absent, adopt the current host ``render`` GID only after the - udev rule has been applied and verified. Existing state must match the host - group before any mutation occurs. - """ - return _reconcile_gpu_access(host if host is not None else SystemGpuAccessHost()) - - -def load_existing_gpu_access(host: GpuAccessHost | None = None) -> GpuAccessState: - """Reconcile runtime GPU access, adopting missing state for pre-change installs. - - Runtime, upgrade, and reinstall paths reuse persisted state when present. - For an installation created before GPU access state existed, this performs a - one-time host ``render`` GID adoption after udev verification. A persisted - GID that differs from the current host group remains a hard failure. - """ - return _reconcile_gpu_access(host if host is not None else SystemGpuAccessHost()) - - -def _reconcile_gpu_access(host: GpuAccessHost) -> GpuAccessState: - _validate_parent_chain(host, GPU_ACCESS_STATE_PATH.parent) - _validate_parent_chain(host, GPU_ACCESS_RULES_PATH.parent) - state_text = _read_regular_text(host, GPU_ACCESS_STATE_PATH) - host_gid = resolve_render_gid(host.get_group_entry("render")) - - if state_text is None: - state = GpuAccessState(render_gid=host_gid) - persist_state = True - else: - state = parse_gpu_access_state(state_text) - if state.render_gid != host_gid: - raise InstallerError( - f"Persisted render GID does not match the current host render group ({state.render_gid} != {host_gid})" - ) - persist_state = False - - legacy_paths = _legacy_rules_to_remove(host) - existing_rule = _read_regular_text(host, GPU_ACCESS_RULES_PATH) - rewrite_rule = _should_rewrite_udev_rule(existing_rule) +def provision_gpu_access(host: GpuAccessHost | None = None) -> None: + """Reconcile and verify the canonical AMD GPU host-device policy.""" + active_host = host if host is not None else SystemGpuAccessHost() + _validate_parent_chain(active_host, GPU_ACCESS_RULES_PATH.parent) + legacy_paths = _legacy_rules_to_remove(active_host) + existing_rule = _read_regular_text(active_host, GPU_ACCESS_RULES_PATH) for path in legacy_paths: - host.remove_udev_rule(path) - if rewrite_rule: - host.write_udev_rule(GPU_ACCESS_RULES_PATH, render_udev_rules()) - host.reload_udev_rules() - host.trigger_udev() - host.settle_udev() - host.verify_device_access(state.render_gid) - if persist_state: - host.write_state_atomically(GPU_ACCESS_STATE_PATH, serialize_gpu_access_state(state)) - - return state + active_host.remove_udev_rule(path) + if _should_rewrite_udev_rule(existing_rule): + active_host.write_udev_rule(GPU_ACCESS_RULES_PATH, render_udev_rules()) + active_host.reload_udev_rules() + active_host.trigger_udev() + active_host.settle_udev() + active_host.verify_device_access() def _read_regular_text(host: GpuAccessHost, path: Path) -> str | None: @@ -359,9 +245,4 @@ def _should_rewrite_udev_rule(existing_rule: str | None) -> bool: return False if existing_rule.split("\n", maxsplit=1)[0] != UDEV_MANAGED_MARKER: raise InstallerError(f"Refusing to overwrite unmanaged GPU udev rule: {GPU_ACCESS_RULES_PATH}") - return True - - -def _validate_render_gid(render_gid: object) -> None: - if type(render_gid) is not int or not 1 <= render_gid <= MAX_RENDER_GID: - raise InstallerError(f"Invalid render group GID: {render_gid!r}") + raise InstallerError(f"Refusing to overwrite unrecognized managed GPU udev rule: {GPU_ACCESS_RULES_PATH}") diff --git a/tests/installer/test_gpu_access.py b/tests/installer/test_gpu_access.py index e742c219..230da4a1 100644 --- a/tests/installer/test_gpu_access.py +++ b/tests/installer/test_gpu_access.py @@ -1,6 +1,6 @@ # Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. -"""Tests for the single-node AMD GPU access source of truth.""" +"""Tests for the single-node AMD GPU host device-access reconciler.""" from __future__ import annotations @@ -12,7 +12,6 @@ from auplc_installer import gpu_access from auplc_installer.gpu_access import ( GPU_ACCESS_RULES_PATH, - GPU_ACCESS_STATE_PATH, LEGACY_AMDGPU_PXE_RULES, LEGACY_AMDGPU_RULES, LEGACY_AMDGPU_RULES_PATH, @@ -20,15 +19,9 @@ LEGACY_KFD_RULES_PATH, LEGACY_ROCM_DEVICES_RULES, LEGACY_ROCM_DEVICES_RULES_PATH, - MAX_RENDER_GID, - GpuAccessState, SystemGpuAccessHost, - load_existing_gpu_access, - parse_gpu_access_state, provision_gpu_access, render_udev_rules, - resolve_render_gid, - serialize_gpu_access_state, ) from auplc_installer.util import InstallerError @@ -36,34 +29,17 @@ class FakeGpuAccessHost: """In-memory adapter for the installer host-operation seam.""" - def __init__(self, *, getent_output: str, files: dict[Path, str] | None = None) -> None: - self.getent_output = getent_output + def __init__(self, *, files: dict[Path, str] | None = None) -> None: self.files = dict(files or {}) self.calls: list[str] = [] self.symlinks: set[Path] = set() self.nonregular_files: set[Path] = set() - self.directories = { - Path("/"), - Path("/etc"), - Path("/etc/udev"), - Path("/etc/udev/rules.d"), - Path("/var"), - Path("/var/lib"), - Path("/var/lib/auplc"), - } - - def get_group_entry(self, group_name: str) -> str: - self.calls.append(f"get-group:{group_name}") - return self.getent_output + self.directories = {Path("/"), Path("/etc"), Path("/etc/udev"), Path("/etc/udev/rules.d")} def read_text(self, path: Path) -> str | None: self.calls.append(f"read:{path}") return self.files.get(path) - def write_state_atomically(self, path: Path, text: str) -> None: - self.calls.append(f"write-state:{path}") - self.files[path] = text - def write_udev_rule(self, path: Path, text: str) -> None: self.calls.append(f"write-rule:{path}") self.files[path] = text @@ -81,8 +57,8 @@ def trigger_udev(self) -> None: def settle_udev(self) -> None: self.calls.append("settle-udev") - def verify_device_access(self, render_gid: int) -> None: - self.calls.append(f"verify-devices:{render_gid}") + def verify_device_access(self) -> None: + self.calls.append("verify-devices") def is_symlink(self, path: Path) -> bool: return path in self.symlinks @@ -97,72 +73,34 @@ def is_directory(self, path: Path) -> bool: return path in self.directories -def test_gpu_access_state_round_trips_as_versioned_json() -> None: - state = GpuAccessState(render_gid=993) - - serialized = serialize_gpu_access_state(state) - - assert serialized == '{"renderGid":993,"version":1}\n' - assert parse_gpu_access_state(serialized) == state - - -@pytest.mark.parametrize( - "state_text", - [ - "not json", - '{"renderGid":993,"version":2}', - '{"renderGid":0,"version":1}', - f'{{"renderGid":{MAX_RENDER_GID + 1},"version":1}}', - '{"renderGid":true,"version":1}', - '{"renderGid":993,"unexpected":true,"version":1}', - ], -) -def test_parse_gpu_access_state_rejects_malformed_or_unsupported_state(state_text: str) -> None: - with pytest.raises(RuntimeError): - parse_gpu_access_state(state_text) - - -def test_resolve_render_gid_reads_the_numeric_getent_field() -> None: - assert resolve_render_gid("render:x:993:student\n") == 993 - - -@pytest.mark.parametrize( - "getent_output", - [ - "", - "video:x:44:student\n", - "render:x:0:student\n", - "render:x:not-a-number:student\n", - f"render:x:{MAX_RENDER_GID + 1}:student\n", - "render:x:993:student\nrender:x:994:student\n", - ], -) -def test_resolve_render_gid_rejects_missing_or_invalid_group_records(getent_output: str) -> None: - with pytest.raises(RuntimeError): - resolve_render_gid(getent_output) - - -def test_render_udev_rules_is_the_canonical_least_privilege_policy() -> None: +def test_render_udev_rules_is_the_canonical_host_device_policy() -> None: rules = render_udev_rules() assert rules == ( "# Managed by auplc-installer: AMD GPU device access.\n" - 'KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0660"\n' - 'SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0660"\n' + 'KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0666"\n' + 'SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0666"\n' + 'SUBSYSTEM=="drm", KERNEL=="card*", DRIVERS=="amdgpu", OWNER="root", GROUP="video", MODE="0666"\n' ) - assert "card" not in rules - assert "0666" not in rules assert "chmod" not in rules -def test_device_verification_uses_lstat_and_requires_character_devices() -> None: - assert "path.lstat()" in gpu_access._VERIFY_DEVICE_ACCESS_SCRIPT - assert "stat.S_ISCHR(data.st_mode)" in gpu_access._VERIFY_DEVICE_ACCESS_SCRIPT +def test_device_verification_checks_kfd_and_amd_render_and_card_nodes_without_a_render_gid() -> None: + script = gpu_access._VERIFY_DEVICE_ACCESS_SCRIPT + + assert "path.lstat()" in script + assert "stat.S_ISCHR(data.st_mode)" in script + assert "glob('renderD*')" in script + assert "glob('card*')" in script + assert "'render', 0o666" in script + assert "'video', 0o666" in script + assert "render_gid" not in script + assert "sys.argv[1]" not in script -@pytest.mark.parametrize("unsafe_parent", [Path("/etc/udev"), Path("/etc/udev/rules.d"), Path("/var/lib/auplc")]) +@pytest.mark.parametrize("unsafe_parent", [Path("/etc/udev"), Path("/etc/udev/rules.d")]) def test_symlinked_gpu_access_parent_fails_before_any_file_read_or_write(unsafe_parent: Path) -> None: - host = FakeGpuAccessHost(getent_output="render:x:993:student\n") + host = FakeGpuAccessHost() host.symlinks.add(unsafe_parent) with pytest.raises(InstallerError, match="symlinked GPU access directory"): @@ -172,7 +110,7 @@ def test_symlinked_gpu_access_parent_fails_before_any_file_read_or_write(unsafe_ def test_nonregular_canonical_rule_fails_before_reading_or_writing_it() -> None: - host = FakeGpuAccessHost(getent_output="render:x:993:student\n") + host = FakeGpuAccessHost() host.nonregular_files.add(GPU_ACCESS_RULES_PATH) with pytest.raises(InstallerError, match="non-regular GPU access file"): @@ -182,224 +120,96 @@ def test_nonregular_canonical_rule_fails_before_reading_or_writing_it() -> None: assert f"write-rule:{GPU_ACCESS_RULES_PATH}" not in host.calls -def test_provision_adopts_host_render_gid_and_installs_canonical_rule() -> None: - host = FakeGpuAccessHost(getent_output="render:x:993:student\n") - - state = provision_gpu_access(host) - - assert state == GpuAccessState(render_gid=993) - assert host.files[GPU_ACCESS_STATE_PATH] == '{"renderGid":993,"version":1}\n' - assert host.files[GPU_ACCESS_RULES_PATH] == render_udev_rules() - assert host.calls[-4:] == [ - "trigger-udev", - "settle-udev", - "verify-devices:993", - f"write-state:{GPU_ACCESS_STATE_PATH}", - ] - - -def test_provision_migrates_exact_legacy_rules_then_verifies_before_persisting_state() -> None: - host = FakeGpuAccessHost( - getent_output="render:x:993:student\n", - files={ - LEGACY_KFD_RULES_PATH: ('KERNEL=="kfd", MODE="0666"\nSUBSYSTEM=="drm", KERNEL=="renderD*", MODE="0666"\n'), - LEGACY_AMDGPU_RULES_PATH: ( - "# ROCm device permissions\n" - "# Grant render group access to AMD GPU devices\n" - "# Reference: https://rocm.docs.amd.com/projects/install-on-linux/en/latest/install/prerequisites.html#using-udev-rules\n" - 'KERNEL=="kfd", GROUP="render", MODE="0660"\n' - 'SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660"\n' - ), - }, - ) +def test_provision_reconciles_the_canonical_rule_without_group_lookup_or_state() -> None: + host = FakeGpuAccessHost() - state = provision_gpu_access(host) + result = provision_gpu_access(host) - assert state == GpuAccessState(render_gid=993) - assert LEGACY_KFD_RULES == ('KERNEL=="kfd", MODE="0666"\nSUBSYSTEM=="drm", KERNEL=="renderD*", MODE="0666"\n') - assert LEGACY_AMDGPU_RULES == ( - "# ROCm device permissions\n" - "# Grant render group access to AMD GPU devices\n" - "# Reference: https://rocm.docs.amd.com/projects/install-on-linux/en/latest/install/prerequisites.html#using-udev-rules\n" - 'KERNEL=="kfd", GROUP="render", MODE="0660"\n' - 'SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660"\n' - ) - assert LEGACY_KFD_RULES_PATH not in host.files - assert LEGACY_AMDGPU_RULES_PATH not in host.files - assert host.calls.index(f"remove-rule:{LEGACY_KFD_RULES_PATH}") < host.calls.index( - f"write-rule:{GPU_ACCESS_RULES_PATH}" - ) - assert host.calls[-3:] == ["settle-udev", "verify-devices:993", f"write-state:{GPU_ACCESS_STATE_PATH}"] - - -def test_provision_migrates_exact_legacy_rocm_devices_rule() -> None: - host = FakeGpuAccessHost( - getent_output="render:x:993:student\n", - files={ - LEGACY_ROCM_DEVICES_RULES_PATH: ( - "# ROCm device permissions\n" - "# Ensure /dev/kfd and /dev/dri/renderD* are accessible by render group\n" - 'SUBSYSTEM=="kfd", GROUP="render", MODE="0660"\n' - 'SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660"\n' - ), - }, - ) - - state = provision_gpu_access(host) - - assert state == GpuAccessState(render_gid=993) - assert LEGACY_ROCM_DEVICES_RULES == ( - "# ROCm device permissions\n" - "# Ensure /dev/kfd and /dev/dri/renderD* are accessible by render group\n" - 'SUBSYSTEM=="kfd", GROUP="render", MODE="0660"\n' - 'SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660"\n' - ) - assert LEGACY_ROCM_DEVICES_RULES_PATH not in host.files - - -def test_provision_migrates_exact_legacy_pxe_rule_at_amdgpu_path() -> None: - host = FakeGpuAccessHost( - getent_output="render:x:993:student\n", - files={ - LEGACY_AMDGPU_RULES_PATH: ('KERNEL=="kfd", MODE="0666"\nKERNEL=="renderD[0-9]*", MODE="0666"\n'), - }, - ) + assert result is None + assert host.files == {GPU_ACCESS_RULES_PATH: render_udev_rules()} + assert host.calls[-4:] == ["reload-udev", "trigger-udev", "settle-udev", "verify-devices"] + assert not any("group" in call or "state" in call for call in host.calls) - state = provision_gpu_access(host) - assert state == GpuAccessState(render_gid=993) - assert LEGACY_AMDGPU_PXE_RULES == ('KERNEL=="kfd", MODE="0666"\nKERNEL=="renderD[0-9]*", MODE="0666"\n') - assert LEGACY_AMDGPU_RULES_PATH not in host.files - - -def test_near_legacy_pxe_rule_fails_closed_without_removal() -> None: - near_variant = 'KERNEL=="kfd", MODE="0666"\nKERNEL=="renderD*", MODE="0666"\n' - host = FakeGpuAccessHost(getent_output="render:x:993:student\n", files={LEGACY_AMDGPU_RULES_PATH: near_variant}) +@pytest.mark.parametrize( + ("path", "content"), + [ + (LEGACY_KFD_RULES_PATH, LEGACY_KFD_RULES), + (LEGACY_AMDGPU_RULES_PATH, LEGACY_AMDGPU_RULES), + (LEGACY_AMDGPU_RULES_PATH, LEGACY_AMDGPU_PXE_RULES), + (LEGACY_ROCM_DEVICES_RULES_PATH, LEGACY_ROCM_DEVICES_RULES), + ], +) +def test_provision_removes_only_exact_legacy_rules_before_verifying(path: Path, content: str) -> None: + host = FakeGpuAccessHost(files={path: content}) - with pytest.raises(InstallerError, match="unexpected legacy"): - provision_gpu_access(host) + provision_gpu_access(host) - assert host.files[LEGACY_AMDGPU_RULES_PATH] == near_variant + assert path not in host.files + assert host.files[GPU_ACCESS_RULES_PATH] == render_udev_rules() + assert host.calls.index(f"remove-rule:{path}") < host.calls.index(f"write-rule:{GPU_ACCESS_RULES_PATH}") + assert host.calls[-4:] == ["reload-udev", "trigger-udev", "settle-udev", "verify-devices"] -def test_modified_legacy_rocm_devices_rule_fails_closed_without_removal() -> None: - modified = ( - "# ROCm device permissions\n" - "# Ensure /dev/kfd and /dev/dri/renderD* are accessible by render group\n" - 'SUBSYSTEM=="kfd", GROUP="render", MODE="0666"\n' - 'SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660"\n' - ) - host = FakeGpuAccessHost(getent_output="render:x:993:student\n", files={LEGACY_ROCM_DEVICES_RULES_PATH: modified}) +@pytest.mark.parametrize( + ("path", "content"), + [ + (LEGACY_AMDGPU_RULES_PATH, 'KERNEL=="kfd", MODE="0666"\nKERNEL=="renderD*", MODE="0666"\n'), + ( + LEGACY_ROCM_DEVICES_RULES_PATH, + "# ROCm device permissions\n" + "# Ensure /dev/kfd and /dev/dri/renderD* are accessible by render group\n" + 'SUBSYSTEM=="kfd", GROUP="render", MODE="0666"\n' + 'SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660"\n', + ), + ], +) +def test_near_legacy_rule_fails_closed_without_removal(path: Path, content: str) -> None: + host = FakeGpuAccessHost(files={path: content}) with pytest.raises(InstallerError, match="unexpected legacy"): provision_gpu_access(host) - assert host.files[LEGACY_ROCM_DEVICES_RULES_PATH] == modified + assert host.files[path] == content -def test_provision_reapplies_and_verifies_matching_immutable_state() -> None: - host = FakeGpuAccessHost( - getent_output="render:x:993:student\n", - files={ - GPU_ACCESS_STATE_PATH: '{"renderGid":993,"version":1}\n', - GPU_ACCESS_RULES_PATH: render_udev_rules(), - }, - ) +def test_matching_managed_rule_is_reapplied_and_verified_without_rewriting() -> None: + host = FakeGpuAccessHost(files={GPU_ACCESS_RULES_PATH: render_udev_rules()}) - state = provision_gpu_access(host) + provision_gpu_access(host) - assert state == GpuAccessState(render_gid=993) assert not any(call.startswith("write-") for call in host.calls) - assert host.calls[-4:] == ["reload-udev", "trigger-udev", "settle-udev", "verify-devices:993"] + assert host.calls[-4:] == ["reload-udev", "trigger-udev", "settle-udev", "verify-devices"] -def test_provision_fails_before_mutation_when_persisted_gid_differs_from_host() -> None: - host = FakeGpuAccessHost( - getent_output="render:x:994:student\n", - files={GPU_ACCESS_STATE_PATH: '{"renderGid":993,"version":1}\n'}, - ) +@pytest.mark.parametrize( + "unexpected_rule", + [ + f"{gpu_access.UDEV_MANAGED_MARKER}\n" + 'KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0660"\n' + 'SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0660"\n', + f'{gpu_access.UDEV_MANAGED_MARKER}\nKERNEL=="kfd", MODE="0666"\n', + ], +) +def test_noncanonical_managed_rule_fails_closed_before_mutation(unexpected_rule: str) -> None: + host = FakeGpuAccessHost(files={GPU_ACCESS_RULES_PATH: unexpected_rule}) - with pytest.raises(RuntimeError, match="does not match"): + with pytest.raises(InstallerError, match="unrecognized managed"): provision_gpu_access(host) - assert not any(call.startswith("write-") for call in host.calls) + assert host.files[GPU_ACCESS_RULES_PATH] == unexpected_rule + assert not any(call.startswith(("write-", "remove-rule:")) for call in host.calls) assert "reload-udev" not in host.calls - assert "trigger-udev" not in host.calls -def test_provision_fails_before_writing_state_when_rule_is_unmanaged() -> None: - host = FakeGpuAccessHost( - getent_output="render:x:993:student\n", - files={GPU_ACCESS_RULES_PATH: 'KERNEL=="kfd", MODE="0666"\n'}, - ) +def test_unmanaged_rule_fails_before_any_mutation() -> None: + host = FakeGpuAccessHost(files={GPU_ACCESS_RULES_PATH: 'KERNEL=="kfd", MODE="0666"\n'}) - with pytest.raises(RuntimeError, match="unmanaged"): + with pytest.raises(InstallerError, match="unmanaged"): provision_gpu_access(host) - assert GPU_ACCESS_STATE_PATH not in host.files - assert not any(call.startswith("write-") for call in host.calls) - - -def test_load_existing_gpu_access_adopts_missing_state_after_verification() -> None: - host = FakeGpuAccessHost(getent_output="render:x:993:student\n") - - state = load_existing_gpu_access(host) - - assert state == GpuAccessState(render_gid=993) - assert host.files[GPU_ACCESS_STATE_PATH] == '{"renderGid":993,"version":1}\n' - assert host.calls[-3:] == ["settle-udev", "verify-devices:993", f"write-state:{GPU_ACCESS_STATE_PATH}"] - - -def test_managed_rule_is_reconciled_and_reloaded_when_content_changes() -> None: - host = FakeGpuAccessHost( - getent_output="render:x:993:student\n", - files={ - GPU_ACCESS_STATE_PATH: '{"renderGid":993,"version":1}\n', - GPU_ACCESS_RULES_PATH: "# Managed by auplc-installer: AMD GPU device access.\nold rule\n", - }, - ) - - state = load_existing_gpu_access(host) - - assert state == GpuAccessState(render_gid=993) - assert host.files[GPU_ACCESS_RULES_PATH] == render_udev_rules() - assert host.calls[-5:] == [ - f"write-rule:{GPU_ACCESS_RULES_PATH}", - "reload-udev", - "trigger-udev", - "settle-udev", - "verify-devices:993", - ] - - -def test_system_adapter_persists_state_with_a_same_directory_temporary_file(monkeypatch) -> None: - commands: list[list[str]] = [] - capture_commands: list[list[str]] = [] - - def fake_run(command: list[str], **kwargs: object) -> SimpleNamespace: - commands.append(command) - if command[:2] == ["test", "-L"]: - return SimpleNamespace(returncode=1) - return SimpleNamespace(returncode=0) - - def fake_run_capture(command: list[str], **kwargs: object) -> SimpleNamespace: - capture_commands.append(command) - return SimpleNamespace(stdout="/var/lib/auplc/.gpu-access.json.temporary\n") - - monkeypatch.setattr(gpu_access, "run", fake_run) - monkeypatch.setattr(gpu_access, "run_capture", fake_run_capture) - - SystemGpuAccessHost().write_state_atomically(GPU_ACCESS_STATE_PATH, "state\n") - - assert capture_commands == [["mktemp", "/var/lib/auplc/.gpu-access.json.XXXXXX"]] - assert [command for command in commands if command[0] != "test"] == [ - ["mkdir", "-p", "/var/lib/auplc"], - ["tee", "/var/lib/auplc/.gpu-access.json.temporary"], - ["chmod", "0644", "/var/lib/auplc/.gpu-access.json.temporary"], - ["python3", "-c", gpu_access._FSYNC_PATH_SCRIPT, "/var/lib/auplc/.gpu-access.json.temporary"], - ["mv", "-f", "/var/lib/auplc/.gpu-access.json.temporary", "/var/lib/auplc/gpu-access.json"], - ["python3", "-c", gpu_access._FSYNC_PATH_SCRIPT, "/var/lib/auplc"], - ] + assert not any(call.startswith(("write-", "remove-rule:")) for call in host.calls) + assert "reload-udev" not in host.calls def test_system_adapter_persists_udev_rule_with_durable_atomic_replacement(monkeypatch) -> None: @@ -437,14 +247,19 @@ def fake_run_capture(command: list[str], **kwargs: object) -> SimpleNamespace: ] -def test_system_adapter_removes_temporary_file_when_durable_write_fails(monkeypatch) -> None: +def test_system_adapter_removes_temporary_rule_when_durable_write_fails(monkeypatch) -> None: commands: list[list[str]] = [] def fake_run(command: list[str], **kwargs: object) -> SimpleNamespace: commands.append(command) if command[:2] == ["test", "-L"]: return SimpleNamespace(returncode=1) - if command == ["python3", "-c", gpu_access._FSYNC_PATH_SCRIPT, "/var/lib/auplc/.gpu-access.json.temporary"]: + if command == [ + "python3", + "-c", + gpu_access._FSYNC_PATH_SCRIPT, + "/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary", + ]: raise InstallerError("fsync failed") return SimpleNamespace(returncode=0) @@ -452,56 +267,24 @@ def fake_run(command: list[str], **kwargs: object) -> SimpleNamespace: monkeypatch.setattr( gpu_access, "run_capture", - lambda command, **kwargs: SimpleNamespace(stdout="/var/lib/auplc/.gpu-access.json.temporary\n"), + lambda command, **kwargs: SimpleNamespace(stdout="/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary\n"), ) with pytest.raises(InstallerError, match="fsync failed"): - SystemGpuAccessHost().write_state_atomically(GPU_ACCESS_STATE_PATH, "state\n") + SystemGpuAccessHost().write_udev_rule(GPU_ACCESS_RULES_PATH, "rule\n") assert [command for command in commands if command[0] != "test"] == [ - ["mkdir", "-p", "/var/lib/auplc"], - ["tee", "/var/lib/auplc/.gpu-access.json.temporary"], - ["chmod", "0644", "/var/lib/auplc/.gpu-access.json.temporary"], - ["python3", "-c", gpu_access._FSYNC_PATH_SCRIPT, "/var/lib/auplc/.gpu-access.json.temporary"], - ["rm", "-f", "/var/lib/auplc/.gpu-access.json.temporary"], + ["mkdir", "-p", "/etc/udev/rules.d"], + ["tee", "/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary"], + ["chmod", "0644", "/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary"], + ["python3", "-c", gpu_access._FSYNC_PATH_SCRIPT, "/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary"], + ["rm", "-f", "/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary"], ] -@pytest.mark.parametrize( - ("failing_method", "expected_calls"), - [ - ( - "write_udev_rule", - [ - "get-group:render", - f"write-rule:{GPU_ACCESS_RULES_PATH}", - ], - ), - ( - "reload_udev_rules", - [ - "get-group:render", - f"write-rule:{GPU_ACCESS_RULES_PATH}", - "reload-udev", - ], - ), - ( - "trigger_udev", - [ - "get-group:render", - f"write-rule:{GPU_ACCESS_RULES_PATH}", - "reload-udev", - "trigger-udev", - ], - ), - ], -) -def test_first_install_does_not_persist_state_until_udev_reconciliation_succeeds( - monkeypatch, - failing_method: str, - expected_calls: list[str], -) -> None: - host = FakeGpuAccessHost(getent_output="render:x:993:student\n") +@pytest.mark.parametrize("failing_method", ["write_udev_rule", "reload_udev_rules", "trigger_udev", "settle_udev"]) +def test_reconciliation_stops_when_udev_mutation_fails(monkeypatch, failing_method: str) -> None: + host = FakeGpuAccessHost() original_method = getattr(host, failing_method) def fail_after_recording(*args: object) -> None: @@ -513,78 +296,14 @@ def fail_after_recording(*args: object) -> None: with pytest.raises(InstallerError, match=f"{failing_method} failed"): provision_gpu_access(host) - assert host.calls[-len(expected_calls) :] == expected_calls - assert GPU_ACCESS_STATE_PATH not in host.files - - -def test_failed_udev_reconciliation_never_rewrites_existing_state(monkeypatch) -> None: - original_state = '{"renderGid":993,"version":1}\n' - host = FakeGpuAccessHost( - getent_output="render:x:993:student\n", - files={ - GPU_ACCESS_STATE_PATH: original_state, - GPU_ACCESS_RULES_PATH: "# Managed by auplc-installer: AMD GPU device access.\nold rule\n", - }, - ) - - def fail_reload() -> None: - host.calls.append("reload-udev") - raise InstallerError("reload failed") - - monkeypatch.setattr(host, "reload_udev_rules", fail_reload) - - with pytest.raises(InstallerError, match="reload failed"): - provision_gpu_access(host) - - assert host.files[GPU_ACCESS_STATE_PATH] == original_state - assert not any(call.startswith("write-state:") for call in host.calls) - assert "trigger-udev" not in host.calls - + assert "verify-devices" not in host.calls -def test_failed_reload_is_retried_and_only_persists_state_after_a_later_success(monkeypatch) -> None: - host = FakeGpuAccessHost(getent_output="render:x:993:student\n") - def fail_reload() -> None: - host.calls.append("reload-udev") - raise InstallerError("reload failed") - - monkeypatch.setattr(host, "reload_udev_rules", fail_reload) - with pytest.raises(InstallerError, match="reload failed"): - provision_gpu_access(host) - assert GPU_ACCESS_STATE_PATH not in host.files - - monkeypatch.setattr(host, "reload_udev_rules", FakeGpuAccessHost.reload_udev_rules.__get__(host)) - state = provision_gpu_access(host) - - assert state == GpuAccessState(render_gid=993) - assert host.calls[-2:] == ["verify-devices:993", f"write-state:{GPU_ACCESS_STATE_PATH}"] - - -def test_failed_settle_is_retried_and_only_persists_state_after_a_later_success(monkeypatch) -> None: - host = FakeGpuAccessHost(getent_output="render:x:993:student\n") - - def fail_settle() -> None: - host.calls.append("settle-udev") - raise InstallerError("settle failed") - - monkeypatch.setattr(host, "settle_udev", fail_settle) - with pytest.raises(InstallerError, match="settle failed"): - provision_gpu_access(host) - assert GPU_ACCESS_STATE_PATH not in host.files - assert "verify-devices:993" not in host.calls +def test_failed_inode_verification_leaves_the_reconciled_rule_in_place(monkeypatch) -> None: + host = FakeGpuAccessHost() - monkeypatch.setattr(host, "settle_udev", FakeGpuAccessHost.settle_udev.__get__(host)) - state = provision_gpu_access(host) - - assert state == GpuAccessState(render_gid=993) - assert host.calls[-3:] == ["settle-udev", "verify-devices:993", f"write-state:{GPU_ACCESS_STATE_PATH}"] - - -def test_failed_inode_verification_does_not_adopt_state(monkeypatch) -> None: - host = FakeGpuAccessHost(getent_output="render:x:993:student\n") - - def fail_verification(render_gid: int) -> None: - host.calls.append(f"verify-devices:{render_gid}") + def fail_verification() -> None: + host.calls.append("verify-devices") raise InstallerError("device ownership mismatch") monkeypatch.setattr(host, "verify_device_access", fail_verification) @@ -592,36 +311,16 @@ def fail_verification(render_gid: int) -> None: with pytest.raises(InstallerError, match="ownership mismatch"): provision_gpu_access(host) - assert host.calls[-1] == "verify-devices:993" - assert GPU_ACCESS_STATE_PATH not in host.files + assert host.files[GPU_ACCESS_RULES_PATH] == render_udev_rules() + assert host.calls[-1] == "verify-devices" @pytest.mark.parametrize("path", [LEGACY_KFD_RULES_PATH, LEGACY_AMDGPU_RULES_PATH, GPU_ACCESS_RULES_PATH]) def test_symlinked_gpu_access_files_fail_closed_before_mutation(path: Path) -> None: - host = FakeGpuAccessHost(getent_output="render:x:993:student\n") + host = FakeGpuAccessHost() host.symlinks.add(path) with pytest.raises(InstallerError, match="symlinked"): provision_gpu_access(host) - assert GPU_ACCESS_STATE_PATH not in host.files - - -@pytest.mark.parametrize( - ("path", "content"), - [ - (LEGACY_KFD_RULES_PATH, 'KERNEL=="kfd", MODE="0666"\n'), - (LEGACY_AMDGPU_RULES_PATH, 'KERNEL=="kfd", GROUP="render", MODE="0660"\n'), - ], -) -def test_one_line_legacy_variants_fail_closed_without_removal(path: Path, content: str) -> None: - host = FakeGpuAccessHost( - getent_output="render:x:993:student\n", - files={path: content}, - ) - - with pytest.raises(InstallerError, match="unexpected legacy"): - provision_gpu_access(host) - - assert host.files[path] == content - assert GPU_ACCESS_STATE_PATH not in host.files + assert not any(call.startswith(("write-", "remove-rule:")) for call in host.calls) diff --git a/tests/scripts/test_gpu_image_permissions.py b/tests/scripts/test_gpu_image_permissions.py index ce5f094a..b07047cc 100644 --- a/tests/scripts/test_gpu_image_permissions.py +++ b/tests/scripts/test_gpu_image_permissions.py @@ -13,12 +13,10 @@ def test_rocm_base_leaves_gpu_device_permissions_to_the_host() -> None: dockerfile = DOCKERFILE.read_text(encoding="utf-8") forbidden_patterns = ( - r"groupmod\s+-g\s+992\s+render", - r"groupadd\s+-g\s+992\s+render", - r"usermod\s+-aG\s+video,render\s+\$\{NB_USER\}", - r"\brender\b", + r"\b(?:groupadd|groupmod|usermod)\b.*\b(?:video|render)\b", r"/etc/udev", r"chmod\s+666\b", + r"chmod\b.*(?:/dev/|kfd|render|card)", ) for pattern in forbidden_patterns: assert re.search(pattern, dockerfile) is None, pattern From d5e373c59b09c5c018dadf2c94592a3345922b92 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:30:28 +0800 Subject: [PATCH 26/65] refactor(installer): remove GPU GID workflow --- auplc_installer/cli.py | 30 ++++---- tests/installer/test_cli_gpu_access.py | 97 ++++++++++---------------- tests/installer/test_cli_helpers.py | 1 - 3 files changed, 50 insertions(+), 78 deletions(-) diff --git a/auplc_installer/cli.py b/auplc_installer/cli.py index f2e9b538..609930a9 100644 --- a/auplc_installer/cli.py +++ b/auplc_installer/cli.py @@ -13,7 +13,7 @@ import contextlib import sys import time -from collections.abc import Callable, Sequence +from collections.abc import Sequence from pathlib import Path from typing import NoReturn @@ -23,7 +23,7 @@ detect_and_configure_gpu, refine_gpu_config_from_node_labels, ) -from auplc_installer.gpu_access import GpuAccessState, load_existing_gpu_access, provision_gpu_access +from auplc_installer.gpu_access import provision_gpu_access from auplc_installer.gpu_hardware import GpuHardware, classify_gpu_hardware from auplc_installer.helm import ( deploy_runtime, @@ -326,12 +326,12 @@ def _raise_unreachable_gpu_hardware(hardware: GpuHardware) -> NoReturn: raise AssertionError(f"Unhandled GPU hardware classification: {hardware!r}") -def _render_gid_for_local_hardware(reconcile_gpu_access: Callable[[], GpuAccessState]) -> int | None: +def _provision_gpu_access_for_local_hardware() -> None: match classify_gpu_hardware(): case GpuHardware.GPU: - return reconcile_gpu_access().render_gid + provision_gpu_access() case GpuHardware.CPU: - return None + return case GpuHardware.UNKNOWN: raise InstallerError("Could not determine local AMD GPU hardware; refusing to modify installer state") case unreachable: @@ -355,7 +355,7 @@ def _cmd_install_inner(state: InstallerState, *, pull: bool) -> None: detect_and_configure_gpu(state.gpu, gpu_type_override=state.gpu_type) with stage("Provisioning GPU device access", idx=2, total=total): - render_gid = _render_gid_for_local_hardware(provision_gpu_access) + _provision_gpu_access_for_local_hardware() paths = state.runtime_paths() with stage("Generating values overlay (initial)", idx=3, total=total): @@ -368,7 +368,6 @@ def _cmd_install_inner(state: InstallerState, *, pull: bool) -> None: image_tag=state.image_tag, courses=state.courses, offline_mode=state.offline_mode, - render_gid=render_gid, overlay_path=paths.overlay_path, ) @@ -434,7 +433,6 @@ def _cmd_install_inner(state: InstallerState, *, pull: bool) -> None: image_tag=state.image_tag, courses=state.courses, offline_mode=state.offline_mode, - render_gid=render_gid, overlay_path=paths.overlay_path, ) @@ -606,7 +604,7 @@ def cmd_dev_quick(state: InstallerState) -> None: def cmd_dev_deploy(state: InstallerState) -> None: - render_gid = _render_gid_for_local_hardware(load_existing_gpu_access) + _provision_gpu_access_for_local_hardware() detect_and_configure_gpu(state.gpu, gpu_type_override=state.gpu_type) paths = state.runtime_paths() refine_gpu_config_from_node_labels(state.gpu) @@ -616,14 +614,13 @@ def cmd_dev_deploy(state: InstallerState) -> None: image_tag=state.image_tag, courses=state.courses, offline_mode=state.offline_mode, - render_gid=render_gid, overlay_path=paths.overlay_path, ) deploy_runtime(paths, dev=True) def cmd_dev_upgrade(state: InstallerState) -> None: - render_gid = _render_gid_for_local_hardware(load_existing_gpu_access) + _provision_gpu_access_for_local_hardware() detect_and_configure_gpu(state.gpu, gpu_type_override=state.gpu_type) paths = state.runtime_paths() refine_gpu_config_from_node_labels(state.gpu) @@ -634,14 +631,13 @@ def cmd_dev_upgrade(state: InstallerState) -> None: image_tag=state.image_tag, courses=state.courses, offline_mode=state.offline_mode, - render_gid=render_gid, overlay_path=paths.overlay_path, ) upgrade_runtime(paths, dev=True) def cmd_dev_reinstall(state: InstallerState) -> None: - _render_gid_for_local_hardware(load_existing_gpu_access) + _provision_gpu_access_for_local_hardware() with contextlib.suppress(InstallerError): remove_runtime() time.sleep(0.5) @@ -652,7 +648,7 @@ def cmd_dev_reinstall(state: InstallerState) -> None: def cmd_rt_install(state: InstallerState) -> None: - render_gid = _render_gid_for_local_hardware(load_existing_gpu_access) + _provision_gpu_access_for_local_hardware() detect_and_configure_gpu(state.gpu, gpu_type_override=state.gpu_type) paths = state.runtime_paths() refine_gpu_config_from_node_labels(state.gpu) @@ -662,14 +658,13 @@ def cmd_rt_install(state: InstallerState) -> None: image_tag=state.image_tag, courses=state.courses, offline_mode=state.offline_mode, - render_gid=render_gid, overlay_path=paths.overlay_path, ) deploy_runtime(paths) def cmd_rt_upgrade(state: InstallerState) -> None: - render_gid = _render_gid_for_local_hardware(load_existing_gpu_access) + _provision_gpu_access_for_local_hardware() detect_and_configure_gpu(state.gpu, gpu_type_override=state.gpu_type) paths = state.runtime_paths() refine_gpu_config_from_node_labels(state.gpu) @@ -680,7 +675,6 @@ def cmd_rt_upgrade(state: InstallerState) -> None: image_tag=state.image_tag, courses=state.courses, offline_mode=state.offline_mode, - render_gid=render_gid, overlay_path=paths.overlay_path, ) upgrade_runtime(paths) @@ -711,7 +705,7 @@ def cmd_rt_remove(state: InstallerState) -> None: def cmd_rt_reinstall(state: InstallerState) -> None: - _render_gid_for_local_hardware(load_existing_gpu_access) + _provision_gpu_access_for_local_hardware() with contextlib.suppress(InstallerError): remove_runtime() time.sleep(0.5) diff --git a/tests/installer/test_cli_gpu_access.py b/tests/installer/test_cli_gpu_access.py index ede70311..e4cdaaf9 100644 --- a/tests/installer/test_cli_gpu_access.py +++ b/tests/installer/test_cli_gpu_access.py @@ -11,20 +11,16 @@ import pytest from auplc_installer import cli -from auplc_installer.gpu_access import GpuAccessState from auplc_installer.gpu_hardware import GpuHardware from auplc_installer.helm import RuntimePaths from auplc_installer.state import InstallerState -@pytest.mark.parametrize( - ("hardware", "expected_render_gid", "expected_provision_count"), - [(GpuHardware.GPU, 993, 1), (GpuHardware.CPU, None, 0)], -) -def test_full_install_gates_gpu_access_without_skipping_later_gpu_flow( - monkeypatch, hardware: GpuHardware, expected_render_gid: int | None, expected_provision_count: int +@pytest.mark.parametrize(("hardware", "expected_provision_count"), [(GpuHardware.GPU, 1), (GpuHardware.CPU, 0)]) +def test_full_install_gates_gpu_access_without_passing_it_to_the_overlay( + monkeypatch, hardware: GpuHardware, expected_provision_count: int ) -> None: - events: list[object] = [] + events: list[str] = [] stages: list[tuple[str, int, int]] = [] state = InstallerState() paths = RuntimePaths(chart_path=Path("chart"), values_path=Path("values.yaml"), overlay_path=Path("overlay.yaml")) @@ -35,16 +31,15 @@ def fake_stage(label: str, *, idx: int, total: int): yield def fake_overlay(*args: object, **kwargs: object) -> Path: - events.append(("overlay", kwargs["render_gid"])) + assert "render_gid" not in kwargs + events.append("overlay") return paths.overlay_path monkeypatch.setattr(state, "runtime_paths", lambda: paths) monkeypatch.setattr(cli, "stage", fake_stage) monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: hardware) monkeypatch.setattr(cli, "detect_and_configure_gpu", lambda *args, **kwargs: events.append("detect")) - monkeypatch.setattr( - cli, "provision_gpu_access", lambda: events.append("provision") or GpuAccessState(render_gid=993) - ) + monkeypatch.setattr(cli, "provision_gpu_access", lambda: events.append("provision")) monkeypatch.setattr(cli, "generate_values_overlay", fake_overlay) monkeypatch.setattr(cli, "install_tools", lambda **kwargs: events.append("tools")) monkeypatch.setattr(cli, "install_k3s_single_node", lambda **kwargs: events.append("k3s")) @@ -60,10 +55,7 @@ def fake_overlay(*args: object, **kwargs: object) -> Path: assert events.count("provision") == expected_provision_count if expected_provision_count: assert events.index("provision") < events.index("device-plugin") - assert [event for event in events if isinstance(event, tuple)] == [ - ("overlay", expected_render_gid), - ("overlay", expected_render_gid), - ] + assert events.count("overlay") == 2 assert stages == [ ("Detecting GPU", 1, 9), ("Provisioning GPU device access", 2, 9), @@ -77,29 +69,22 @@ def fake_overlay(*args: object, **kwargs: object) -> Path: ] -@pytest.mark.parametrize( - ("hardware", "expected_render_gid", "expected_load_count"), - [(GpuHardware.GPU, 993, 1), (GpuHardware.CPU, None, 0)], -) -def test_runtime_upgrade_gates_existing_gpu_access_without_provisioning( - monkeypatch, hardware: GpuHardware, expected_render_gid: int | None, expected_load_count: int +@pytest.mark.parametrize(("hardware", "expected_provision_count"), [(GpuHardware.GPU, 1), (GpuHardware.CPU, 0)]) +def test_runtime_upgrade_gates_host_access_without_provisioning_helm_values( + monkeypatch, hardware: GpuHardware, expected_provision_count: int ) -> None: - events: list[object] = [] + events: list[str] = [] state = InstallerState() paths = RuntimePaths(chart_path=Path("chart"), values_path=Path("values.yaml"), overlay_path=Path("overlay.yaml")) def fake_overlay(*args: object, **kwargs: object) -> Path: - events.append(("overlay", kwargs["render_gid"])) + assert "render_gid" not in kwargs + events.append("overlay") return paths.overlay_path monkeypatch.setattr(state, "runtime_paths", lambda: paths) monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: hardware) - monkeypatch.setattr( - cli, "load_existing_gpu_access", lambda: events.append("load") or GpuAccessState(render_gid=993) - ) - monkeypatch.setattr( - cli, "provision_gpu_access", lambda: (_ for _ in ()).throw(AssertionError("must not provision")) - ) + monkeypatch.setattr(cli, "provision_gpu_access", lambda: events.append("provision")) monkeypatch.setattr(cli, "detect_and_configure_gpu", lambda *args, **kwargs: events.append("detect")) monkeypatch.setattr(cli, "refine_gpu_config_from_node_labels", lambda *args, **kwargs: events.append("refine")) monkeypatch.setattr(cli, "_preserve_courses_for_upgrade", lambda *args, **kwargs: events.append("preserve-courses")) @@ -108,20 +93,20 @@ def fake_overlay(*args: object, **kwargs: object) -> Path: cli.cmd_rt_upgrade(state) - assert events.count("load") == expected_load_count - assert events[-5:] == ["detect", "refine", "preserve-courses", ("overlay", expected_render_gid), "upgrade-runtime"] + assert events.count("provision") == expected_provision_count + assert events[-5:] == ["detect", "refine", "preserve-courses", "overlay", "upgrade-runtime"] @pytest.mark.parametrize( ("command", "expected_events"), [ - (cli.cmd_dev_deploy, ("detect", "refine", "overlay:None", "deploy-runtime")), - (cli.cmd_dev_upgrade, ("detect", "refine", "preserve-courses", "overlay:None", "upgrade-runtime")), - (cli.cmd_rt_install, ("detect", "refine", "overlay:None", "deploy-runtime")), - (cli.cmd_rt_upgrade, ("detect", "refine", "preserve-courses", "overlay:None", "upgrade-runtime")), + (cli.cmd_dev_deploy, ("detect", "refine", "overlay", "deploy-runtime")), + (cli.cmd_dev_upgrade, ("detect", "refine", "preserve-courses", "overlay", "upgrade-runtime")), + (cli.cmd_rt_install, ("detect", "refine", "overlay", "deploy-runtime")), + (cli.cmd_rt_upgrade, ("detect", "refine", "preserve-courses", "overlay", "upgrade-runtime")), ], ) -def test_cpu_hardware_skips_existing_access_and_preserves_runtime_flow( +def test_cpu_hardware_skips_host_access_and_preserves_runtime_flow( monkeypatch, command: Callable[[InstallerState], None], expected_events: tuple[str, ...] ) -> None: events: list[str] = [] @@ -130,14 +115,16 @@ def test_cpu_hardware_skips_existing_access_and_preserves_runtime_flow( monkeypatch.setattr(state, "runtime_paths", lambda: paths) monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: GpuHardware.CPU) - monkeypatch.setattr(cli, "load_existing_gpu_access", lambda: (_ for _ in ()).throw(AssertionError("must not load"))) + monkeypatch.setattr( + cli, "provision_gpu_access", lambda: (_ for _ in ()).throw(AssertionError("must not provision")) + ) monkeypatch.setattr(cli, "detect_and_configure_gpu", lambda *args, **kwargs: events.append("detect")) monkeypatch.setattr(cli, "refine_gpu_config_from_node_labels", lambda *args, **kwargs: events.append("refine")) monkeypatch.setattr(cli, "_preserve_courses_for_upgrade", lambda *args, **kwargs: events.append("preserve-courses")) monkeypatch.setattr( cli, "generate_values_overlay", - lambda *args, **kwargs: events.append(f"overlay:{kwargs['render_gid']}") or paths.overlay_path, + lambda *args, **kwargs: events.append("overlay") or paths.overlay_path, ) monkeypatch.setattr(cli, "deploy_runtime", lambda *args, **kwargs: events.append("deploy-runtime")) monkeypatch.setattr(cli, "upgrade_runtime", lambda *args, **kwargs: events.append("upgrade-runtime")) @@ -149,16 +136,12 @@ def test_cpu_hardware_skips_existing_access_and_preserves_runtime_flow( @pytest.mark.parametrize( ("reinstall", "delegate_name"), - [ - (cli.cmd_dev_reinstall, "cmd_dev_deploy"), - (cli.cmd_rt_reinstall, "cmd_rt_install"), - ], + [(cli.cmd_dev_reinstall, "cmd_dev_deploy"), (cli.cmd_rt_reinstall, "cmd_rt_install")], ) @pytest.mark.parametrize( - ("hardware", "expected_access_events"), - [(GpuHardware.GPU, ["load"]), (GpuHardware.CPU, [])], + ("hardware", "expected_access_events"), [(GpuHardware.GPU, ["provision"]), (GpuHardware.CPU, [])] ) -def test_reinstall_gates_existing_gpu_access_before_removing_runtime( +def test_reinstall_gates_host_access_before_removing_runtime( monkeypatch, reinstall: Callable[[InstallerState], None], delegate_name: str, @@ -169,9 +152,7 @@ def test_reinstall_gates_existing_gpu_access_before_removing_runtime( state = InstallerState() monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: hardware) - monkeypatch.setattr( - cli, "load_existing_gpu_access", lambda: events.append("load") or GpuAccessState(render_gid=993) - ) + monkeypatch.setattr(cli, "provision_gpu_access", lambda: events.append("provision")) monkeypatch.setattr(cli, "remove_runtime", lambda: events.append("remove-runtime")) monkeypatch.setattr(cli.time, "sleep", lambda seconds: events.append("sleep")) monkeypatch.setattr(cli, delegate_name, lambda current_state: events.append("delegate")) @@ -188,9 +169,7 @@ def test_unknown_hardware_blocks_full_install_before_gpu_access_mutation(monkeyp monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: GpuHardware.UNKNOWN) monkeypatch.setattr(cli, "detect_and_configure_gpu", lambda *args, **kwargs: events.append("detect")) monkeypatch.setattr( - cli, - "provision_gpu_access", - lambda: (_ for _ in ()).throw(AssertionError("must not provision")), + cli, "provision_gpu_access", lambda: (_ for _ in ()).throw(AssertionError("must not provision")) ) with pytest.raises(RuntimeError, match="hardware"): @@ -201,10 +180,7 @@ def test_unknown_hardware_blocks_full_install_before_gpu_access_mutation(monkeyp @pytest.mark.parametrize( ("reinstall", "delegate_name"), - [ - (cli.cmd_dev_reinstall, "cmd_dev_deploy"), - (cli.cmd_rt_reinstall, "cmd_rt_install"), - ], + [(cli.cmd_dev_reinstall, "cmd_dev_deploy"), (cli.cmd_rt_reinstall, "cmd_rt_install")], ) def test_unknown_hardware_blocks_reinstall_before_runtime_removal( monkeypatch, reinstall: Callable[[InstallerState], None], delegate_name: str @@ -214,9 +190,7 @@ def test_unknown_hardware_blocks_reinstall_before_runtime_removal( monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: GpuHardware.UNKNOWN) monkeypatch.setattr( - cli, - "load_existing_gpu_access", - lambda: (_ for _ in ()).throw(AssertionError("must not load")), + cli, "provision_gpu_access", lambda: (_ for _ in ()).throw(AssertionError("must not provision")) ) monkeypatch.setattr(cli, "remove_runtime", lambda: events.append("remove-runtime")) monkeypatch.setattr(cli, delegate_name, lambda current_state: events.append("delegate")) @@ -225,3 +199,8 @@ def test_unknown_hardware_blocks_reinstall_before_runtime_removal( reinstall(state) assert events == [] + + +def test_cli_exposes_no_render_gid_reconciliation_api() -> None: + assert not hasattr(cli, "_render_gid_for_local_hardware") + assert not hasattr(cli, "load_existing_gpu_access") diff --git a/tests/installer/test_cli_helpers.py b/tests/installer/test_cli_helpers.py index 2cc863e8..bc33cffa 100644 --- a/tests/installer/test_cli_helpers.py +++ b/tests/installer/test_cli_helpers.py @@ -41,7 +41,6 @@ def _write_overlay(path: Path, courses: CourseSelection) -> None: image_tag="v1.0", courses=courses, offline_mode=False, - render_gid=993, overlay_path=path, ) From 3bc557fbee23aa8979ceaeba5545da15a0397181 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:30:28 +0800 Subject: [PATCH 27/65] refactor(installer): remove GPU GID overlays --- auplc_installer/overlay.py | 8 -------- tests/installer/test_overlay.py | 14 +++++--------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/auplc_installer/overlay.py b/auplc_installer/overlay.py index 202b5c6c..5815f391 100644 --- a/auplc_installer/overlay.py +++ b/auplc_installer/overlay.py @@ -47,7 +47,6 @@ def emit_overlay( image_tag: str, courses: CourseSelection, offline_mode: bool, - render_gid: int | None, ) -> str: """Render the overlay as a string. Pure function — no I/O.""" buf = StringIO() @@ -67,11 +66,6 @@ def emit_overlay( buf.write(f"# Env selection : {courses.description()}\n") buf.write("# Regenerated on install/upgrade.\n") buf.write("custom:\n") - buf.write(" gpuAccess:\n") - if render_gid is None: - buf.write(" renderGid: null\n") - else: - buf.write(f" renderGid: {render_gid}\n") # --- accelerators --- any_accel_emitted = False @@ -170,7 +164,6 @@ def generate_values_overlay( image_tag: str, courses: CourseSelection, offline_mode: bool, - render_gid: int | None, overlay_path: Path, ) -> Path: """Render the overlay and write it to ``overlay_path``. Returns the path.""" @@ -182,7 +175,6 @@ def generate_values_overlay( image_tag=image_tag, courses=courses, offline_mode=offline_mode, - render_gid=render_gid, ) overlay_path.write_text(text, encoding="utf-8") return overlay_path diff --git a/tests/installer/test_overlay.py b/tests/installer/test_overlay.py index c93e4fc7..677eb115 100644 --- a/tests/installer/test_overlay.py +++ b/tests/installer/test_overlay.py @@ -46,7 +46,6 @@ def _render( courses: CourseSelection, offline_mode: bool = False, image_tag: str = "v1.0", - render_gid: int | None = 993, ) -> tuple[str, dict]: text = emit_overlay( cfg, @@ -54,7 +53,6 @@ def _render( image_tag=image_tag, courses=courses, offline_mode=offline_mode, - render_gid=render_gid, ) return text, yaml.safe_load(text) @@ -96,7 +94,6 @@ def _write_and_read_back(courses: CourseSelection) -> CourseSelection | None: image_tag="v1.0", courses=courses, offline_mode=False, - render_gid=993, overlay_path=path, ) return try_load_courses_from_overlay(path) @@ -110,30 +107,29 @@ def test_default_selection_round_trips_valid_yaml() -> None: assert "teams" not in parsed["custom"] -def test_overlay_emits_explicit_gpu_access_gid_without_global_pod_groups() -> None: +def test_overlay_never_emits_gpu_access_contract() -> None: text = emit_overlay( _strix_halo_cfg(), image_registry="ghcr.io/amdresearch", image_tag="v1.0", courses=CourseSelection.default(), offline_mode=False, - render_gid=993, ) parsed = yaml.safe_load(text) - assert parsed["custom"]["gpuAccess"]["renderGid"] == 993 + assert "gpuAccess" not in parsed["custom"] + assert "renderGid" not in text assert "supplementalGroups" not in text -def test_overlay_emits_null_render_gid_without_removing_gpu_resources() -> None: +def test_overlay_keeps_gpu_resources_without_gpu_access_contract() -> None: _, parsed = _render( _strix_halo_cfg(), courses=CourseSelection.default(), - render_gid=None, ) custom = parsed["custom"] - assert custom["gpuAccess"]["renderGid"] is None + assert "gpuAccess" not in custom assert set(custom["resources"]["images"]) == set(GPU_RESOURCE_KEYS) assert set(custom["resources"]["metadata"]) == set(GPU_RESOURCE_KEYS) assert "teams" not in custom From e5252db7b99565cd94f95652c474bfb609d827f0 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:30:54 +0800 Subject: [PATCH 28/65] refactor(ansible): simplify GPU access role --- deploy/ansible/filter_plugins/auplc_json.py | 41 -- .../roles/gpu_access/defaults/main.yml | 4 - .../ansible/roles/gpu_access/tasks/apply.yml | 152 +++--- .../roles/gpu_access/tasks/preflight.yml | 120 +---- .../roles/gpu_access/tasks/validate.yml | 9 - .../templates/70-auplc-gpu-access.rules.j2 | 5 +- .../gpu_access/templates/gpu-access.json.j2 | 1 - tests/skills/test_gpu_access_role.py | 503 ++++-------------- 8 files changed, 220 insertions(+), 615 deletions(-) delete mode 100644 deploy/ansible/filter_plugins/auplc_json.py delete mode 100644 deploy/ansible/roles/gpu_access/templates/gpu-access.json.j2 diff --git a/deploy/ansible/filter_plugins/auplc_json.py b/deploy/ansible/filter_plugins/auplc_json.py deleted file mode 100644 index 080de6af..00000000 --- a/deploy/ansible/filter_plugins/auplc_json.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. - -"""Strict JSON filters used by AUP Learning Cloud Ansible roles.""" - -import json -from collections.abc import Callable -from dataclasses import dataclass -from typing import TypeAlias - -from ansible.errors import AnsibleFilterError - -JSONValue: TypeAlias = None | bool | int | float | str | list["JSONValue"] | dict[str, "JSONValue"] - - -@dataclass(frozen=True, slots=True) -class DuplicateJsonKeyError(ValueError): - key: str - - def __str__(self) -> str: - return f"Duplicate JSON object key: {self.key!r}" - - -def _reject_duplicate_keys(pairs: list[tuple[str, JSONValue]]) -> dict[str, JSONValue]: - result: dict[str, JSONValue] = {} - for key, value in pairs: - if key in result: - raise DuplicateJsonKeyError(key) - result[key] = value - return result - - -def auplc_from_json_strict(value: str) -> JSONValue: - try: - return json.loads(value, object_pairs_hook=_reject_duplicate_keys) - except (TypeError, DuplicateJsonKeyError, json.JSONDecodeError): - raise AnsibleFilterError("Invalid JSON value") from None - - -class FilterModule: - def filters(self) -> dict[str, Callable[[str], JSONValue]]: - return {"auplc_from_json_strict": auplc_from_json_strict} diff --git a/deploy/ansible/roles/gpu_access/defaults/main.yml b/deploy/ansible/roles/gpu_access/defaults/main.yml index 74e40ed3..8f5ff6c7 100644 --- a/deploy/ansible/roles/gpu_access/defaults/main.yml +++ b/deploy/ansible/roles/gpu_access/defaults/main.yml @@ -1,10 +1,6 @@ # Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. --- -# Set this explicitly in cluster inventory or PXE extra vars. The role never -# assumes a site-specific GID. -auplc_render_gid: null -auplc_normalize_render_gid: false auplc_gpu_access_enabled: false # Set for a PXE rootfs. Leave empty to configure the live host. auplc_rootfs_path: "" diff --git a/deploy/ansible/roles/gpu_access/tasks/apply.yml b/deploy/ansible/roles/gpu_access/tasks/apply.yml index 1def00de..f830b935 100644 --- a/deploy/ansible/roles/gpu_access/tasks/apply.yml +++ b/deploy/ansible/roles/gpu_access/tasks/apply.yml @@ -1,6 +1,39 @@ # Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. --- +- name: Inspect canonical GPU access rule before apply + ansible.builtin.stat: + path: "{{ _auplc_target_root }}/etc/udev/rules.d/70-auplc-gpu-access.rules" + follow: false + register: _auplc_apply_destination_rule + +- name: Reject unsafe canonical GPU access rule before apply + ansible.builtin.assert: + that: + - not _auplc_apply_destination_rule.stat.exists or + (_auplc_apply_destination_rule.stat.isreg and not _auplc_apply_destination_rule.stat.islnk) + fail_msg: Unsafe canonical GPU access destination. + +- name: Read canonical GPU access rule before apply + ansible.builtin.slurp: + src: "{{ _auplc_target_root }}/etc/udev/rules.d/70-auplc-gpu-access.rules" + register: _auplc_apply_existing_rule + when: _auplc_apply_destination_rule.stat.exists + +- name: Define canonical GPU access rule contents for apply + ansible.builtin.set_fact: + _auplc_apply_canonical_rule: | + # Managed by auplc-installer: AMD GPU device access. + KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0666" + SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0666" + SUBSYSTEM=="drm", KERNEL=="card*", DRIVERS=="amdgpu", OWNER="root", GROUP="video", MODE="0666" + +- name: Recheck canonical GPU access rule before apply + ansible.builtin.assert: + that: (_auplc_apply_existing_rule.content | b64decode) == _auplc_apply_canonical_rule + fail_msg: Unmanaged canonical GPU access rule. + when: _auplc_apply_destination_rule.stat.exists + - name: Inspect recognized project-owned legacy GPU rules for apply ansible.builtin.stat: path: "{{ item.path }}" @@ -37,38 +70,6 @@ loop: "{{ _auplc_apply_legacy_gpu_rule_contents.results }}" when: not item.skipped | default(false) -- name: Normalize live render GID - ansible.builtin.command: - argv: [groupmod, -g, "{{ auplc_render_gid | string }}", render] - when: - - _auplc_target_root | length == 0 - - (_auplc_current_render_gid | int) != (auplc_render_gid | int) - - auplc_normalize_render_gid | bool - -- name: Normalize rootfs render GID - ansible.builtin.command: - argv: [chroot, "{{ _auplc_target_root }}", groupmod, -g, "{{ auplc_render_gid | string }}", render] - when: - - _auplc_target_root | length > 0 - - (_auplc_current_render_gid | int) != (auplc_render_gid | int) - - auplc_normalize_render_gid | bool - -- name: Verify target render GID - ansible.builtin.command: - argv: >- - {{ ['getent', 'group', 'render'] if _auplc_target_root | length == 0 - else ['chroot', _auplc_target_root, 'getent', 'group', 'render'] }} - register: _auplc_verified_render_group - changed_when: false - -- name: Require verified render GID - ansible.builtin.assert: - that: - - _auplc_verified_render_group.stdout_lines | length == 1 - - _auplc_verified_render_group.stdout.split(':')[0] == 'render' - - (_auplc_verified_render_group.stdout.split(':')[2] | int) == (auplc_render_gid | int) - fail_msg: Target render group did not resolve to auplc_render_gid. - - name: Create target udev rules directory ansible.builtin.file: path: "{{ _auplc_target_root }}/etc/udev/rules.d" @@ -116,9 +117,9 @@ - _auplc_kfd.stat.exists - _auplc_kfd.stat.ischr - _auplc_kfd.stat.uid == 0 - - _auplc_kfd.stat.gid == (auplc_render_gid | int) - - _auplc_kfd.stat.mode == '0660' - fail_msg: /dev/kfd is not root:render with mode 0660 after reconciliation. + - _auplc_kfd.stat.gr_name == 'render' + - _auplc_kfd.stat.mode == '0666' + fail_msg: /dev/kfd is not root:render with mode 0666 after reconciliation. when: _auplc_target_root | length == 0 - name: Find live DRM render nodes @@ -130,11 +131,20 @@ register: _auplc_render_nodes when: _auplc_target_root | length == 0 -- name: Resolve live DRM render node driver symlinks +- name: Find live DRM card nodes + ansible.builtin.find: + paths: /dev/dri + patterns: card* + file_type: any + recurse: false + register: _auplc_card_nodes + when: _auplc_target_root | length == 0 + +- name: Resolve live DRM node driver symlinks ansible.builtin.command: argv: [readlink, -f, "/sys/class/drm/{{ item.path | basename }}/device/driver"] - loop: "{{ _auplc_render_nodes.files }}" - register: _auplc_render_node_drivers + loop: "{{ (_auplc_render_nodes.files | default([])) + (_auplc_card_nodes.files | default([])) }}" + register: _auplc_drm_node_drivers changed_when: false failed_when: false when: _auplc_target_root | length == 0 @@ -143,48 +153,68 @@ ansible.builtin.set_fact: _auplc_amd_render_nodes: >- {{ (_auplc_amd_render_nodes | default([])) + - ([item.item.path] if item.rc == 0 and (item.stdout | basename) == 'amdgpu' else []) }} - loop: "{{ _auplc_render_node_drivers.results }}" + ([item.item.path] if item.rc == 0 and (item.stdout | basename) == 'amdgpu' and + (item.item.path | basename) is match('^renderD') else []) }} + loop: "{{ _auplc_drm_node_drivers.results | default([]) }}" + when: _auplc_target_root | length == 0 + +- name: Select AMD live DRM card nodes + ansible.builtin.set_fact: + _auplc_amd_card_nodes: >- + {{ (_auplc_amd_card_nodes | default([])) + + ([item.item.path] if item.rc == 0 and (item.stdout | basename) == 'amdgpu' and + (item.item.path | basename) is match('^card') else []) }} + loop: "{{ _auplc_drm_node_drivers.results | default([]) }}" when: _auplc_target_root | length == 0 - name: Require AMD live DRM render nodes ansible.builtin.assert: - that: _auplc_amd_render_nodes | length > 0 + that: (_auplc_amd_render_nodes | default([])) | length > 0 fail_msg: No AMD renderD node was available for GPU access verification. when: _auplc_target_root | length == 0 +- name: Require AMD live DRM card nodes + ansible.builtin.assert: + that: (_auplc_amd_card_nodes | default([])) | length > 0 + fail_msg: No AMD card node was available for GPU access verification. + when: _auplc_target_root | length == 0 + - name: Inspect AMD live DRM render nodes ansible.builtin.stat: path: "{{ item }}" follow: false - loop: "{{ _auplc_amd_render_nodes }}" + loop: "{{ _auplc_amd_render_nodes | default([]) }}" register: _auplc_amd_render_node_stats when: _auplc_target_root | length == 0 +- name: Inspect AMD live DRM card nodes + ansible.builtin.stat: + path: "{{ item }}" + follow: false + loop: "{{ _auplc_amd_card_nodes | default([]) }}" + register: _auplc_amd_card_node_stats + when: _auplc_target_root | length == 0 + - name: Verify AMD render node ownership and mode ansible.builtin.assert: that: - item.stat.exists - item.stat.ischr - item.stat.uid == 0 - - item.stat.gid == (auplc_render_gid | int) - - item.stat.mode == '0660' - fail_msg: "AMD render node {{ item.item }} is not root:render with mode 0660." - loop: "{{ _auplc_amd_render_node_stats.results }}" + - item.stat.gr_name == 'render' + - item.stat.mode == '0666' + fail_msg: "AMD render node {{ item.item }} is not root:render with mode 0666." + loop: "{{ _auplc_amd_render_node_stats.results | default([]) }}" when: _auplc_target_root | length == 0 -- name: Create target GPU access state directory - ansible.builtin.file: - path: "{{ _auplc_target_root }}/var/lib/auplc" - state: directory - owner: root - group: root - mode: "0755" - -- name: Persist target GPU access state - ansible.builtin.template: - src: gpu-access.json.j2 - dest: "{{ _auplc_target_root }}/var/lib/auplc/gpu-access.json" - owner: root - group: root - mode: "0644" +- name: Verify AMD card node ownership and mode + ansible.builtin.assert: + that: + - item.stat.exists + - item.stat.ischr + - item.stat.uid == 0 + - item.stat.gr_name == 'video' + - item.stat.mode == '0666' + fail_msg: "AMD card node {{ item.item }} is not root:video with mode 0666." + loop: "{{ _auplc_amd_card_node_stats.results | default([]) }}" + when: _auplc_target_root | length == 0 diff --git a/deploy/ansible/roles/gpu_access/tasks/preflight.yml b/deploy/ansible/roles/gpu_access/tasks/preflight.yml index fcb4808b..23d448e4 100644 --- a/deploy/ansible/roles/gpu_access/tasks/preflight.yml +++ b/deploy/ansible/roles/gpu_access/tasks/preflight.yml @@ -27,9 +27,6 @@ - /etc - /etc/udev - /etc/udev/rules.d - - /var - - /var/lib - - /var/lib/auplc register: _auplc_destination_parent_stats - name: Reject unsafe canonical GPU access destination parents @@ -39,52 +36,38 @@ fail_msg: "Unsafe canonical GPU access destination parent: {{ item.item }}" loop: "{{ _auplc_destination_parent_stats.results }}" -- name: Inspect canonical GPU access destinations +- name: Inspect canonical GPU access destination ansible.builtin.stat: - path: "{{ _auplc_target_root }}{{ item }}" + path: "{{ _auplc_target_root }}/etc/udev/rules.d/70-auplc-gpu-access.rules" follow: false - loop: - - /etc/udev/rules.d/70-auplc-gpu-access.rules - - /var/lib/auplc/gpu-access.json - register: _auplc_destination_stats + register: _auplc_destination_rule -- name: Reject unsafe canonical GPU access destinations +- name: Reject unsafe canonical GPU access destination ansible.builtin.assert: that: - - not item.stat.exists or (item.stat.isreg and not item.stat.islnk) - fail_msg: "Unsafe canonical GPU access destination: {{ item.item }}" - loop: "{{ _auplc_destination_stats.results }}" + - not _auplc_destination_rule.stat.exists or + (_auplc_destination_rule.stat.isreg and not _auplc_destination_rule.stat.islnk) + fail_msg: Unsafe canonical GPU access destination. -- name: Read existing canonical GPU access destinations +- name: Read existing canonical GPU access rule ansible.builtin.slurp: - src: "{{ _auplc_target_root }}{{ item.item }}" - loop: "{{ _auplc_destination_stats.results }}" - when: item.stat.exists - register: _auplc_existing_destinations + src: "{{ _auplc_target_root }}/etc/udev/rules.d/70-auplc-gpu-access.rules" + register: _auplc_existing_rule + when: _auplc_destination_rule.stat.exists -- name: Define canonical GPU access rule content +- name: Define canonical GPU access rule contents ansible.builtin.set_fact: _auplc_canonical_rule: | # Managed by auplc-installer: AMD GPU device access. - KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0660" - SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0660" + KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0666" + SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0666" + SUBSYSTEM=="drm", KERNEL=="card*", DRIVERS=="amdgpu", OWNER="root", GROUP="video", MODE="0666" - name: Reject unmanaged canonical GPU access rule ansible.builtin.assert: - that: (item.content | b64decode) == _auplc_canonical_rule - fail_msg: "Unmanaged canonical GPU access rule: {{ item.item.item }}" - loop: "{{ _auplc_existing_destinations.results }}" - when: - - not item.skipped | default(false) - - item.item.item.endswith('70-auplc-gpu-access.rules') - -- name: Parse existing canonical GPU access state - ansible.builtin.set_fact: - _auplc_existing_state: "{{ item.content | b64decode | auplc_from_json_strict }}" - loop: "{{ _auplc_existing_destinations.results }}" - when: - - not item.skipped | default(false) - - item.item.item.endswith('gpu-access.json') + that: (_auplc_existing_rule.content | b64decode) == _auplc_canonical_rule + fail_msg: Unmanaged canonical GPU access rule. + when: _auplc_destination_rule.stat.exists - name: Define recognized project-owned legacy GPU rules ansible.builtin.set_fact: @@ -137,70 +120,3 @@ fail_msg: "Unexpected legacy GPU rule content: {{ item.item.item.path }}" loop: "{{ _auplc_legacy_gpu_rule_contents.results }}" when: not item.skipped | default(false) - -- name: Read target render group - ansible.builtin.command: - argv: >- - {{ ['getent', 'group', 'render'] if _auplc_target_root | length == 0 - else ['chroot', _auplc_target_root, 'getent', 'group', 'render'] }} - register: _auplc_render_group - changed_when: false - failed_when: false - -- name: Require target render group - ansible.builtin.assert: - that: - - _auplc_render_group.rc == 0 - - _auplc_render_group.stdout_lines | length == 1 - - _auplc_render_group.stdout.split(':') | length == 4 - - _auplc_render_group.stdout.split(':')[0] == 'render' - - _auplc_render_group.stdout.split(':')[2] is match('^[1-9][0-9]*$') - - _auplc_render_group.stdout.split(':')[2] | int <= 4294967294 - fail_msg: Target has no valid render group; this role never creates groups. - -- name: Record target render GID - ansible.builtin.set_fact: - _auplc_current_render_gid: "{{ _auplc_render_group.stdout.split(':')[2] | int }}" - -- name: Reject invalid canonical GPU access state except Interrupted normalization retry - ansible.builtin.assert: - that: - - _auplc_existing_state is mapping - - _auplc_existing_state.keys() | list | sort == ['renderGid', 'version'] - - _auplc_existing_state.version is integer - - _auplc_existing_state.version == 1 - - _auplc_existing_state.renderGid is integer - - _auplc_existing_state.renderGid >= 1 - - _auplc_existing_state.renderGid <= 4294967294 - - >- - _auplc_existing_state.renderGid == auplc_render_gid or - ((auplc_normalize_render_gid | bool) and - (_auplc_existing_state.renderGid == _auplc_current_render_gid or - _auplc_current_render_gid == auplc_render_gid)) - fail_msg: Invalid canonical GPU access state. - when: _auplc_existing_state is defined - -- name: List target groups for desired GID collision - ansible.builtin.command: - argv: >- - {{ ['getent', 'group'] if _auplc_target_root | length == 0 - else ['chroot', _auplc_target_root, 'getent', 'group'] }} - register: _auplc_all_groups - changed_when: false - failed_when: false - -- name: Reject desired GID collision - ansible.builtin.assert: - that: - - _auplc_all_groups.rc == 0 - - >- - _auplc_all_groups.stdout_lines - | select('match', '^[^:]*:[^:]*:' ~ (auplc_render_gid | string) ~ ':') - | reject('match', '^render:') | list | length == 0 - fail_msg: auplc_render_gid is already assigned to another target group. - -- name: Reject render GID mismatch without normalization - ansible.builtin.assert: - that: - - _auplc_current_render_gid == auplc_render_gid or (auplc_normalize_render_gid | bool) - fail_msg: Target render GID differs from auplc_render_gid and normalization is disabled. diff --git a/deploy/ansible/roles/gpu_access/tasks/validate.yml b/deploy/ansible/roles/gpu_access/tasks/validate.yml index 3e32ee7a..dbfa855e 100644 --- a/deploy/ansible/roles/gpu_access/tasks/validate.yml +++ b/deploy/ansible/roles/gpu_access/tasks/validate.yml @@ -1,15 +1,6 @@ # Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. --- -- name: Validate desired render GID - ansible.builtin.assert: - that: - - auplc_render_gid is not none - - auplc_render_gid is integer - - auplc_render_gid >= 1 - - auplc_render_gid <= 4294967294 - fail_msg: auplc_render_gid must be an explicit integer between 1 and 4294967294. - - name: Validate GPU access rootfs path syntax ansible.builtin.assert: that: diff --git a/deploy/ansible/roles/gpu_access/templates/70-auplc-gpu-access.rules.j2 b/deploy/ansible/roles/gpu_access/templates/70-auplc-gpu-access.rules.j2 index c75ec1e2..f57d023a 100644 --- a/deploy/ansible/roles/gpu_access/templates/70-auplc-gpu-access.rules.j2 +++ b/deploy/ansible/roles/gpu_access/templates/70-auplc-gpu-access.rules.j2 @@ -1,3 +1,4 @@ # Managed by auplc-installer: AMD GPU device access. -KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0660" -SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0660" +KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0666" +SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0666" +SUBSYSTEM=="drm", KERNEL=="card*", DRIVERS=="amdgpu", OWNER="root", GROUP="video", MODE="0666" diff --git a/deploy/ansible/roles/gpu_access/templates/gpu-access.json.j2 b/deploy/ansible/roles/gpu_access/templates/gpu-access.json.j2 deleted file mode 100644 index 89b9110a..00000000 --- a/deploy/ansible/roles/gpu_access/templates/gpu-access.json.j2 +++ /dev/null @@ -1 +0,0 @@ -{"renderGid":{{ auplc_render_gid | int }},"version":1} diff --git a/tests/skills/test_gpu_access_role.py b/tests/skills/test_gpu_access_role.py index b195fef5..ccece9ce 100644 --- a/tests/skills/test_gpu_access_role.py +++ b/tests/skills/test_gpu_access_role.py @@ -1,32 +1,9 @@ # Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. -"""Canonical artifact tests for the multi-node GPU access role.""" +"""Contract tests for the Ansible GPU device-mode role.""" -import sys -import types from pathlib import Path -import pytest - -try: - from ansible.errors import AnsibleFilterError -except ModuleNotFoundError: - ansible_module = types.ModuleType("ansible") - errors_module = types.ModuleType("ansible.errors") - - class AnsibleFilterError(Exception): - pass - - errors_module.AnsibleFilterError = AnsibleFilterError - ansible_module.errors = errors_module - sys.modules["ansible"] = ansible_module - sys.modules["ansible.errors"] = errors_module -from deploy.ansible.filter_plugins.auplc_json import ( - DuplicateJsonKeyError, - _reject_duplicate_keys, - auplc_from_json_strict, -) - ROOT = Path(__file__).resolve().parents[2] ANSIBLE = ROOT / "deploy" / "ansible" GPU_ACCESS_ROLE = ANSIBLE / "roles" / "gpu_access" @@ -38,423 +15,159 @@ def read(path: Path) -> str: return path.read_text(encoding="utf-8") -def test_strict_json_filter_parses_canonical_gpu_access_state() -> None: - assert auplc_from_json_strict('{"renderGid":993,"version":1}\n') == { - "renderGid": 993, - "version": 1, - } - - -def test_duplicate_json_key_error_preserves_typed_key() -> None: - with pytest.raises(DuplicateJsonKeyError) as error: - _reject_duplicate_keys([("version", 1), ("version", 2)]) - - assert error.value.key == "version" - assert str(error.value) == "Duplicate JSON object key: 'version'" - - -@pytest.mark.parametrize( - "value", - [ - '{"renderGid":1,"renderGid":993,"version":1}', - '{"renderGid":1,"render\\u0047id":993,"version":1}', - '{"renderGid":1,"version":1,"version":2}', - '{"outer":{"version":1,"version":2}}', - ], -) -def test_strict_json_filter_rejects_semantic_duplicate_keys(value: str) -> None: - with pytest.raises(AnsibleFilterError, match="^Invalid JSON value$"): - auplc_from_json_strict(value) - - -@pytest.mark.parametrize("value", ["{", '{"renderGid":1,}']) -def test_strict_json_filter_rejects_malformed_json(value: str) -> None: - with pytest.raises(AnsibleFilterError, match="^Invalid JSON value$"): - auplc_from_json_strict(value) - - -def test_gpu_access_role_renders_the_unified_render_gid_contract() -> None: +def test_gpu_access_role_uses_shc_proven_device_mode_contract() -> None: defaults = read(GPU_ACCESS_ROLE / "defaults" / "main.yml") - tasks = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") + read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") + preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") + apply = read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") rules = read(GPU_ACCESS_ROLE / "templates" / "70-auplc-gpu-access.rules.j2") - state = read(GPU_ACCESS_ROLE / "templates" / "gpu-access.json.j2") - assert "auplc_render_gid: null" in defaults - assert "auplc_normalize_render_gid: false" in defaults + assert "auplc_gpu_access_enabled: false" in defaults assert 'auplc_rootfs_path: ""' in defaults - assert "getent" in tasks - assert "groupmod" in tasks - assert "auplc_normalize_render_gid" in tasks - assert "_auplc_all_groups" in tasks - assert "reject('match', '^render:')" in tasks - assert "_auplc_render_group.stdout.split(':')[2] | int <= 4294967294" in tasks - assert "notify:" not in tasks - assert "Reload live udev rules on every apply" in tasks - assert "Trigger live udev rules on every apply" in tasks - assert "ansible.builtin.group:" not in tasks + assert "auplc_render_gid" not in defaults + assert "normalize" not in defaults + assert "gpu-access.json" not in preflight + assert "groupmod" not in apply + assert "GID collision" not in preflight assert rules == ( "# Managed by auplc-installer: AMD GPU device access.\n" - 'KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0660"\n' - 'SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0660"\n' + 'KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0666"\n' + 'SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0666"\n' + 'SUBSYSTEM=="drm", KERNEL=="card*", DRIVERS=="amdgpu", OWNER="root", GROUP="video", MODE="0666"\n' ) - assert state == '{"renderGid":{{ auplc_render_gid | int }},"version":1}\n' - - -def test_gpu_access_role_is_wired_for_live_hosts_and_pxe_rootfs_without_legacy_udev_paths() -> None: - rocm_playbook = read(ANSIBLE / "playbooks" / "pb-rocm.yml") - udev_playbook = read(ANSIBLE / "playbooks" / "pb-udev.yml") - rocm_tasks = read(ANSIBLE / "roles" / "rocm" / "tasks" / "main.yml") - pxe_tasks = read(ANSIBLE / "roles" / "pxe_controller" / "tasks" / "main.yml") - pxe_gpu_tasks = read(PXE_GPU_ACCESS_TASKS) - pxe_chroot = read(ANSIBLE / "roles" / "pxe_controller" / "templates" / "chroot-setup.sh.j2") - - assert "name: gpu_access" in rocm_playbook - assert "name: gpu_access" in udev_playbook - assert "udev-rocm" not in udev_playbook - assert "render:993" not in rocm_tasks - assert "70-amdgpu.rules" not in rocm_tasks - assert "include_tasks: gpu_access.yml" in pxe_tasks - assert "name: gpu_access" in pxe_gpu_tasks - assert 'auplc_rootfs_path: "{{ pxe_nfs_root }}"' in pxe_gpu_tasks - assert "0666" not in pxe_chroot - assert not (ANSIBLE / "roles" / "udev" / "main.yml").exists() -def test_gpu_access_live_host_playbooks_abort_all_hosts_on_preflight_failure() -> None: - rocm_playbook = read(ANSIBLE / "playbooks" / "pb-rocm.yml") - udev_playbook = read(ANSIBLE / "playbooks" / "pb-udev.yml") - - assert "any_errors_fatal: true" in rocm_playbook - assert "any_errors_fatal: true" in udev_playbook - - -def test_gpu_access_role_migrates_only_recognized_legacy_rules_and_reconciles_live_devices() -> None: - tasks = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") + read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") - - assert "70-kfd.rules" in tasks - assert "70-amdgpu.rules" in tasks - assert "contents:" in tasks - assert 'KERNEL==\\"renderD[0-9]*\\", MODE=\\"0666\\"' in tasks - assert "70-rocm-devices.rules" in tasks - assert 'SUBSYSTEM=="kfd", GROUP="render", MODE="0660"' in tasks - assert "islnk" in tasks - assert "ansible.builtin.slurp" in tasks - assert "Define recognized project-owned legacy GPU rules" in tasks - assert "Unexpected legacy GPU rule content" in tasks - assert "udevadm" in tasks - assert "Verify /dev/kfd ownership and mode" in tasks - assert "Verify AMD render node ownership and mode" in tasks - assert "Settle live udev events before inode verification" in tasks - assert ( - tasks.index("Trigger live udev rules on every apply") - < tasks.index("Settle live udev events before inode verification") - < tasks.index("Inspect /dev/kfd after live reconciliation") - ) - assert tasks.index("Verify AMD render node ownership and mode") < tasks.index("Persist target GPU access state") - assert "/sys/class/drm" in tasks - assert "readlink" in tasks - assert "basename" in tasks - assert "DRIVER=amdgpu" not in tasks - assert "notify:" not in tasks - - -def test_gpu_access_role_validates_legacy_rules_before_render_gid_normalization() -> None: +def test_gpu_access_role_preserves_safe_preflight_and_exact_legacy_admission() -> None: + validation = read(GPU_ACCESS_ROLE / "tasks" / "validate.yml") preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") apply = read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") - tasks = preflight + apply + assert "realpath" in validation + assert "auplc_rootfs_path != '/'" in validation + assert "_auplc_canonical_allowed_root" in validation + assert "Inspect GPU access rootfs target" in preflight + assert "follow: false" in preflight + assert "Reject unsafe canonical GPU access destination parents" in preflight + assert "Reject unsafe canonical GPU access destination" in preflight assert "Define recognized project-owned legacy GPU rules" in preflight - assert "Inspect recognized project-owned legacy GPU rules" in preflight - assert "Reject legacy GPU rule symlinks and non-regular files" in preflight - assert "Read recognized project-owned legacy GPU rules" in preflight assert "Reject unexpected legacy GPU rule content" in preflight - assert "follow: false" in preflight - assert "contents:" in preflight - assert 'KERNEL==\\"renderD[0-9]*\\", MODE=\\"0666\\"' in preflight - assert "not item.skipped | default(false)" in preflight - assert "(item.content | b64decode) in item.item.item.contents" in preflight - assert preflight.index("Inspect recognized project-owned legacy GPU rules") < preflight.index( - "Reject legacy GPU rule symlinks and non-regular files" - ) - assert preflight.index("Reject legacy GPU rule symlinks and non-regular files") < preflight.index( - "Read recognized project-owned legacy GPU rules" - ) - assert preflight.index("Read recognized project-owned legacy GPU rules") < preflight.index( - "Reject unexpected legacy GPU rule content" - ) - assert tasks.index("Reject unexpected legacy GPU rule content") < tasks.index("Normalize live render GID") + assert "70-kfd.rules" in preflight + assert "70-amdgpu.rules" in preflight + assert "70-rocm-devices.rules" in preflight assert "Remove recognized project-owned legacy GPU rules" in apply - assert "Inspect recognized project-owned legacy GPU rules for apply" in apply - assert "Reject legacy GPU rule symlinks and non-regular files before apply" in apply - assert "Read recognized project-owned legacy GPU rules for apply" in apply - assert "Reject unexpected legacy GPU rule content before apply" in apply - assert "register: _auplc_apply_legacy_gpu_rule_stats" in apply - assert "register: _auplc_apply_legacy_gpu_rule_contents" in apply - assert "_auplc_apply_legacy_gpu_rule_contents.results" in apply - assert "_auplc_legacy_gpu_rule_contents.results" not in apply - assert apply.index("Reject legacy GPU rule symlinks and non-regular files before apply") < apply.index( - "Read recognized project-owned legacy GPU rules for apply" - ) - assert apply.index("Read recognized project-owned legacy GPU rules for apply") < apply.index( - "Reject unexpected legacy GPU rule content before apply" - ) assert apply.index("Reject unexpected legacy GPU rule content before apply") < apply.index( "Remove recognized project-owned legacy GPU rules" ) - assert apply.index("Remove recognized project-owned legacy GPU rules") < apply.index("Normalize live render GID") - -def test_pxe_rootfs_lifecycle_uses_an_independent_trusted_parent() -> None: - defaults = read(ANSIBLE / "roles" / "pxe_controller" / "defaults" / "main.yml") - tasks = read(ANSIBLE / "roles" / "pxe_controller" / "tasks" / "main.yml") - gpu_tasks = read(PXE_GPU_ACCESS_TASKS) - assert 'pxe_nfs_allowed_root: "/srv/nfs"' in defaults - assert 'auplc_rootfs_allowed_root: "{{ pxe_nfs_allowed_root }}"' in gpu_tasks - assert "Constrain canonical PXE rootfs before lifecycle changes" in tasks - assert tasks.index("Constrain canonical PXE rootfs before lifecycle changes") < tasks.index( - "Admit retained PXE GPU rootfs read-only before lifecycle changes" - ) - - -def test_pxe_rootfs_is_canonicalized_before_gpu_admission() -> None: - tasks = read(ANSIBLE / "roles" / "pxe_controller" / "tasks" / "main.yml") +def test_gpu_access_role_reconciles_and_verifies_live_devices_only() -> None: + apply = read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") - assert "Canonicalize PXE rootfs before lifecycle changes" in tasks - assert tasks.index("Canonicalize PXE rootfs before lifecycle changes") < tasks.index( - "Stop NFS before rootfs rebuild" - ) - assert "_pxe_canonical_nfs_root" in tasks - assert tasks.index("Canonicalize PXE rootfs before lifecycle changes") < tasks.index( - "Admit retained PXE GPU rootfs read-only before lifecycle changes" + assert "Reload live udev rules on every apply" in apply + assert "Trigger live udev rules on every apply" in apply + assert "Settle live udev events before inode verification" in apply + assert "Inspect /dev/kfd after live reconciliation" in apply + assert "Verify /dev/kfd ownership and mode" in apply + assert "Find live DRM render nodes" in apply + assert "Verify AMD render node ownership and mode" in apply + assert "Find live DRM card nodes" in apply + assert "Verify AMD card node ownership and mode" in apply + assert "/sys/class/drm" in apply + assert "readlink" in apply + assert "basename" in apply + assert "_auplc_kfd.stat.mode == '0666'" in apply + assert "item.stat.mode == '0666'" in apply + assert "item.stat.mode == '0666'" in apply + assert "item.stat.gr_name == 'render'" in apply + assert "item.stat.gr_name == 'video'" in apply + assert ( + apply.index("Trigger live udev rules on every apply") + < apply.index("Settle live udev events before inode verification") + < apply.index("Inspect /dev/kfd after live reconciliation") ) + assert "when: _auplc_target_root | length == 0" in apply + assert "gpu-access.json" not in apply -def test_gpu_access_preflight_refuses_unmanaged_canonical_destinations() -> None: - preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") - - assert "Inspect canonical GPU access destinations" in preflight - assert "70-auplc-gpu-access.rules" in preflight - assert "gpu-access.json" in preflight - assert "follow: false" in preflight - assert "Reject unmanaged canonical GPU access rule" in preflight - assert "Reject invalid canonical GPU access state" in preflight - assert "auplc_from_json_strict" in preflight - assert "| from_json" not in preflight - assert "renderGid" in preflight - assert "version" in preflight - assert 'src: "{{ _auplc_target_root }}{{ item.item }}"' in preflight - assert "Interrupted normalization retry" in preflight - assert "_auplc_current_render_gid == auplc_render_gid" in preflight - assert "_auplc_existing_state.version is integer" in preflight - assert "_auplc_existing_state.renderGid is integer" in preflight - assert "_auplc_existing_state.renderGid | int" not in preflight - - -def test_gpu_access_roles_use_strict_json_for_canonical_state_readers() -> None: +def test_gpu_access_role_rejects_noncanonical_managed_rule_content() -> None: preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") + apply = read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") pxe_tasks = read(PXE_GPU_ACCESS_TASKS) - assert "Parse existing canonical GPU access state" in preflight - assert "auplc_from_json_strict" in preflight - assert "auplc_from_json_strict" in pxe_tasks - assert "| from_json" not in preflight - assert "| from_json" not in pxe_tasks - - -def test_canonical_gpu_access_state_contract_is_exact_json() -> None: - state = read(GPU_ACCESS_ROLE / "templates" / "gpu-access.json.j2") - - assert state == '{"renderGid":{{ auplc_render_gid | int }},"version":1}\n' + assert "_auplc_previous_canonical_rule" not in preflight + assert "(_auplc_existing_rule.content | b64decode) == _auplc_canonical_rule" in preflight + assert "Reject unmanaged canonical GPU access rule" in preflight + assert "_auplc_apply_previous_canonical_rule" not in apply + assert "Recheck canonical GPU access rule before apply" in apply + assert "(_auplc_apply_existing_rule.content | b64decode) == _auplc_apply_canonical_rule" in apply + assert "Unmanaged canonical GPU access rule." in apply + assert "_pxe_retained_previous_canonical_rule" not in pxe_tasks + assert "(_pxe_retained_canonical_gpu_rule.content | b64decode) == _pxe_retained_canonical_rule" in pxe_tasks + assert "non-canonical GPU access rule" in pxe_tasks -def test_gpu_access_role_splits_safe_preflight_and_rootfs_apply() -> None: - defaults = read(GPU_ACCESS_ROLE / "defaults" / "main.yml") - preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") - validation = read(GPU_ACCESS_ROLE / "tasks" / "validate.yml") - pxe_tasks = read(ANSIBLE / "roles" / "pxe_controller" / "tasks" / "main.yml") - pxe_gpu_tasks = read(PXE_GPU_ACCESS_TASKS) +def test_pxe_gpu_access_installs_rules_without_gid_or_state_contract() -> None: + main = read(PXE_CONTROLLER_ROLE / "tasks" / "main.yml") + tasks = read(PXE_GPU_ACCESS_TASKS) - assert "auplc_gpu_access_enabled: false" in defaults - assert "auplc_rootfs_allowed_root" in defaults - assert "realpath" in validation - assert "auplc_rootfs_path != '/'" in validation - assert "islnk" in preflight - assert "include_tasks: gpu_access.yml" in pxe_tasks - assert "tasks_from: validate" not in pxe_tasks - assert "Constrain canonical PXE rootfs before lifecycle changes" in pxe_tasks - assert pxe_tasks.index("Constrain canonical PXE rootfs before lifecycle changes") < pxe_tasks.index( + assert "Admit retained PXE GPU rootfs read-only before lifecycle changes" in main + assert "Re-preflight PXE GPU rootfs before TFTP" in main + assert main.index("Admit retained PXE GPU rootfs read-only before lifecycle changes") < main.index( "Stop NFS before rootfs rebuild" ) - assert "tasks_from: preflight" in pxe_gpu_tasks - assert "tasks_from: apply" in pxe_gpu_tasks - assert "auplc_normalize_render_gid:" in pxe_gpu_tasks - assert "pxe_rootfs_force_rebuild | bool" in pxe_tasks - assert "pxe_gpu_access_normalize_render_gid | bool" not in pxe_tasks - assert "pxe_gpu_access_normalize_render_gid" not in read(PXE_CONTROLLER_ROLE / "defaults" / "main.yml") + assert "Inspect retained PXE canonical GPU access parents" in tasks + assert "Require retained PXE canonical GPU access parents" in tasks + assert "Require retained PXE canonical GPU rule" in tasks + assert "tasks_from: preflight" in tasks + assert "tasks_from: apply" in tasks + assert 'auplc_rootfs_path: "{{ pxe_nfs_root }}"' in tasks + assert 'auplc_rootfs_allowed_root: "{{ pxe_nfs_allowed_root }}"' in tasks + assert "auplc_render_gid" not in tasks + assert "render_gid" not in tasks + assert "groupadd" not in tasks + assert "groupmod" not in tasks + assert "collision" not in tasks.lower() + assert "gpu-access.json" not in tasks + assert "/dev/kfd" not in tasks + assert "/dev/dri" not in tasks -def test_live_playbooks_preflight_gpu_hosts_before_mutating_roles() -> None: +def test_gpu_access_playbooks_keep_two_phase_live_and_rootfs_safety() -> None: rocm_playbook = read(ANSIBLE / "playbooks" / "pb-rocm.yml") udev_playbook = read(ANSIBLE / "playbooks" / "pb-udev.yml") + pxe_playbook = read(ANSIBLE / "playbooks" / "pb-pxe-controller.yml") - assert "pre_tasks:" in rocm_playbook - assert "Assert explicit GPU access enablement" in rocm_playbook - assert "auplc_gpu_access_enabled is defined" in rocm_playbook - assert "auplc_gpu_access_enabled is boolean" in rocm_playbook - assert "default(false)" not in rocm_playbook - assert "tasks_from: preflight" in rocm_playbook + assert "any_errors_fatal: true" in rocm_playbook + assert "any_errors_fatal: true" in udev_playbook assert rocm_playbook.index("tasks_from: preflight") < rocm_playbook.index("- role: rocm") - assert "- role: rocm" in rocm_playbook - assert rocm_playbook.count("auplc_gpu_access_enabled") >= 3 assert "tasks_from: apply" in rocm_playbook - assert "auplc_gpu_access_enabled" in rocm_playbook - assert "pre_tasks:" in udev_playbook - assert "Assert explicit GPU access enablement" in udev_playbook - assert "auplc_gpu_access_enabled is defined" in udev_playbook - assert "auplc_gpu_access_enabled is boolean" in udev_playbook - assert "default(false)" not in udev_playbook assert "tasks_from: preflight" in udev_playbook assert "tasks_from: apply" in udev_playbook + assert "render_gid" not in pxe_playbook -def test_gpu_access_discovery_playbook_is_read_only_and_serializes_live_host_evidence() -> None: - playbook = read(ANSIBLE / "playbooks" / "pb-gpu-access-discovery.yml") - - assert "hosts: k3s_cluster" in playbook - assert "gather_facts: false" in playbook - assert "ignore_unreachable: true" in playbook - assert "ansible.builtin.command:" in playbook - assert "ansible.builtin.stat:" in playbook - assert "ansible.builtin.slurp:" in playbook - assert "ansible.builtin.shell:" not in playbook - assert "changed_when: false" in playbook - assert "lspci" in playbook - assert '"1002::0300"' in playbook - assert '"1002::0302"' in playbook - assert '"1002::0380"' in playbook - assert "getent" in playbook - assert "/sys/bus/pci/devices" in playbook - assert "gpu_access_discovery_output_path" in playbook - assert "delegate_to: localhost" in playbook - assert "ansible.builtin.copy:" in playbook - assert "to_json" in playbook - assert "stat_success" in playbook - assert "content_success" in playbook - assert "legacy_rules" in playbook - assert "/etc/udev/rules.d/70-kfd.rules" in playbook - assert "/etc/udev/rules.d/70-amdgpu.rules" in playbook - assert "/etc/udev/rules.d/70-rocm-devices.rules" in playbook - file_probes = playbook[ - playbook.index("Inspect persisted GPU access state") : playbook.index( - "Record machine-readable GPU access discovery evidence" - ) - ] - assert file_probes.count("ignore_errors: true") == 10 - assert "failed_when: false" not in file_probes - assert 'mode: "0600"' in playbook - assert "hosts: pxe_controller" not in playbook +def test_pxe_controller_playbook_has_no_obsolete_finalizer_post_tasks() -> None: + pxe_playbook = read(ANSIBLE / "playbooks" / "pb-pxe-controller.yml") + assert "post_tasks:" not in pxe_playbook + assert "pxe_finalizer_" not in pxe_playbook + assert "--finalize-pxe" not in pxe_playbook -def test_pxe_gpu_admission_resolves_fresh_rootfs_and_refuses_retained_migrations() -> None: - assert PXE_GPU_ACCESS_TASKS.exists() - main = read(PXE_CONTROLLER_ROLE / "tasks" / "main.yml") - tasks = read(PXE_GPU_ACCESS_TASKS) - - assert "Record PXE rootfs state before lifecycle changes" in main - assert "_pxe_rootfs_existed_at_start" in main - assert "_pxe_rootfs_rebuilt_this_run" in main - assert "include_tasks: gpu_access.yml" in main - assert main.index("Record PXE rootfs state before lifecycle changes") < main.index("Stop NFS before rootfs rebuild") - assert main.index("include_tasks: gpu_access.yml") < main.index("Find latest kernel in rootfs") - assert "tasks_from: validate" not in main - assert "tasks_from: preflight" not in main - assert "tasks_from: apply" not in main - - assert "_pxe_rootfs_disposition" in tasks - assert "_pxe_unanimous_live_render_gid" in tasks - assert "_pxe_resolved_render_gid" in tasks - assert "fresh" in tasks - assert "retained" in tasks - assert "groupadd" in tasks - assert "--system" in tasks - assert "groupmod" not in tasks - assert "getent" in tasks - assert 'auplc_render_gid: "{{ _pxe_resolved_render_gid }}"' in tasks - assert "auplc_normalize_render_gid: \"{{ _pxe_rootfs_disposition == 'fresh' }}\"" in tasks - assert "Require retained PXE legacy GPU rules absent" in tasks - assert "Require retained PXE render GID matches unanimous live GID" in tasks - assert "tasks_from: preflight" in tasks - assert "tasks_from: apply" in tasks - assert "render:993" not in tasks - assert "lspci" not in tasks - assert "pxe_gpu_access_normalize_render_gid" not in tasks - - -def test_pxe_gpu_admission_preflights_retained_rootfs_before_lifecycle_mutation() -> None: - main = read(PXE_CONTROLLER_ROLE / "tasks" / "main.yml") - - retained_admission = "Admit retained PXE GPU rootfs read-only before lifecycle changes" - final_admission = "Re-preflight PXE GPU rootfs before TFTP" - - assert main.count("include_tasks: gpu_access.yml") == 2 - assert main.index("Record PXE rootfs state before lifecycle changes") < main.index(retained_admission) - assert main.index(retained_admission) < main.index("Stop NFS before rootfs rebuild") - assert main.index("Remove chroot setup script") < main.index(final_admission) - assert main.index(final_admission) < main.index("Find latest kernel in rootfs") - - retained_branch = main[main.index(retained_admission) : main.index("Stop NFS before rootfs rebuild")] - final_branch = main[main.index(final_admission) : main.index("Find latest kernel in rootfs")] - - assert "pxe_gpu_access_enabled | bool" in retained_branch - assert "not (_pxe_rootfs_rebuilt_this_run | bool)" in retained_branch - assert "pxe_gpu_access_enabled | bool" in final_branch - assert "pxe_gpu_admission_phase: final" in final_branch - - -def test_pxe_rootfs_disposition_uses_initial_root_path_and_rejects_partial_retained_trees() -> None: - main = read(PXE_CONTROLLER_ROLE / "tasks" / "main.yml") - - assert "Require existing PXE rootfs is a directory" in main - assert "Require incomplete PXE rootfs force rebuild" in main - assert '_pxe_rootfs_existed_at_start: "{{ _pxe_rootfs_lstat.stat.exists | bool }}"' in main - assert "not (_pxe_rootfs_lstat.stat.exists | bool)" in main - assert main.index("Require incomplete PXE rootfs force rebuild") < main.index("Stop NFS before rootfs rebuild") - assert main.index("Require incomplete PXE rootfs force rebuild") < main.index("- name: Build NFS rootfs") - - -def test_pxe_retained_admission_is_read_only_until_post_chroot_repreflight_and_apply() -> None: - main = read(PXE_CONTROLLER_ROLE / "tasks" / "main.yml") - tasks = read(PXE_GPU_ACCESS_TASKS) - - assert "Admit retained PXE GPU rootfs read-only before lifecycle changes" in main - assert "Re-preflight PXE GPU rootfs before TFTP" in main - assert main.index("Admit retained PXE GPU rootfs read-only before lifecycle changes") < main.index( - "Stop NFS before rootfs rebuild" +def test_deploy_ansible_has_no_render_gid_normalization_or_gpu_state_contract() -> None: + forbidden = ( + "auplc_render_gid", + "auplc_normalize_render_gid", + "gpu-access.json", + "auplc_from_json_strict", + "groupmod", + "render GID collision", ) - assert main.index("Remove chroot setup script") < main.index("Re-preflight PXE GPU rootfs before TFTP") - assert "pxe_gpu_admission_phase: retained-read-only" in main - assert "pxe_gpu_admission_phase: final" in main - assert "Require retained PXE canonical GPU rule" in tasks - assert "Require retained PXE canonical GPU state" in tasks - assert "Apply GPU access after final PXE re-preflight" in tasks - retained_read_only = tasks[: tasks.index("Preflight GPU access after final PXE re-preflight")] - assert "tasks_from: apply" not in retained_read_only - assert "pxe_gpu_admission_phase == 'final'" in tasks - -def test_pxe_retained_admission_checks_canonical_parent_chain_before_lifecycle_or_chroot_mutation() -> None: - main = read(PXE_CONTROLLER_ROLE / "tasks" / "main.yml") - tasks = read(PXE_GPU_ACCESS_TASKS) - - assert "Inspect retained PXE canonical GPU access parents" in tasks - assert "Require retained PXE canonical GPU access parents" in tasks - for parent in ("/etc", "/etc/udev", "/etc/udev/rules.d", "/var", "/var/lib", "/var/lib/auplc"): - assert parent in tasks - assert "item.stat.exists" in tasks - assert "item.stat.isdir" in tasks - assert "not item.stat.islnk" in tasks - assert main.index("Admit retained PXE GPU rootfs read-only before lifecycle changes") < main.index( - "Execute chroot setup" + ansible_text = "\n".join( + path.read_text(encoding="utf-8") + for path in ANSIBLE.rglob("*") + if path.is_file() and "__pycache__" not in path.parts ) + + for term in forbidden: + assert term not in ansible_text From b1fd9a35bf5597497432b2a727ffe96d00df2332 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:30:54 +0800 Subject: [PATCH 29/65] refactor(ansible): simplify PXE GPU policy --- .../ansible/playbooks/pb-pxe-controller.yml | 38 --- .../roles/pxe_controller/tasks/gpu_access.yml | 221 ++---------------- 2 files changed, 14 insertions(+), 245 deletions(-) diff --git a/deploy/ansible/playbooks/pb-pxe-controller.yml b/deploy/ansible/playbooks/pb-pxe-controller.yml index 250508cc..425b939e 100644 --- a/deploy/ansible/playbooks/pb-pxe-controller.yml +++ b/deploy/ansible/playbooks/pb-pxe-controller.yml @@ -102,41 +102,3 @@ roles: - role: pxe_controller - - post_tasks: - - name: Write private PXE finalizer handoff from resolved rootfs facts - ansible.builtin.copy: - content: >- - {{ { - 'version': 1, - 'generation': pxe_finalizer_generation, - 'spec_sha256': pxe_finalizer_spec_sha256, - 'topology': 'pxe-diskless', - 'pxe_gpu_access_enabled': pxe_gpu_access_enabled | bool, - 'render_gid': _pxe_resolved_render_gid | default(none) - } | to_json }} - dest: "{{ pxe_finalizer_handoff }}" - mode: "0600" - delegate_to: localhost - run_once: true - become: false - no_log: true - when: pxe_finalizer_context is defined - - - name: Finalize generated PXE GPU policy from resolved rootfs facts - ansible.builtin.command: - argv: - - "{{ pxe_finalizer_script }}" - - --finalize-pxe - - --out-dir - - "{{ pxe_finalizer_context | dirname }}" - - --context - - "{{ pxe_finalizer_context }}" - - --handoff - - "{{ pxe_finalizer_handoff }}" - delegate_to: localhost - run_once: true - become: false - no_log: true - changed_when: false - when: pxe_finalizer_context is defined diff --git a/deploy/ansible/roles/pxe_controller/tasks/gpu_access.yml b/deploy/ansible/roles/pxe_controller/tasks/gpu_access.yml index 06b5d5f8..d0a0bd5b 100644 --- a/deploy/ansible/roles/pxe_controller/tasks/gpu_access.yml +++ b/deploy/ansible/roles/pxe_controller/tasks/gpu_access.yml @@ -4,171 +4,12 @@ - name: Record PXE GPU admission disposition ansible.builtin.set_fact: _pxe_rootfs_disposition: "{{ 'fresh' if _pxe_rootfs_rebuilt_this_run | bool else 'retained' }}" - _pxe_unanimous_live_render_gid: "{{ auplc_render_gid if auplc_render_gid is defined and auplc_render_gid is not none else none }}" - _pxe_resolved_render_gid: null - name: Assert PXE GPU admission phase ansible.builtin.assert: that: pxe_gpu_admission_phase in ['retained-read-only', 'final'] fail_msg: PXE GPU admission phase is invalid. -- name: Validate optional unanimous live render GID - ansible.builtin.assert: - that: - - _pxe_unanimous_live_render_gid is integer - - _pxe_unanimous_live_render_gid >= 1 - - _pxe_unanimous_live_render_gid <= 4294967294 - fail_msg: auplc_render_gid must be an integer between 1 and 4294967294 when supplied for a PXE GPU rootfs. - when: - - pxe_gpu_access_enabled | bool - - _pxe_unanimous_live_render_gid is not none - -- name: Inspect PXE rootfs render group for GPU admission - ansible.builtin.command: - argv: [chroot, "{{ pxe_nfs_root }}", getent, group, render] - register: _pxe_admission_render_group - changed_when: false - failed_when: false - when: pxe_gpu_access_enabled | bool - -- name: Require fresh PXE render group lookup outcome - ansible.builtin.assert: - that: _pxe_admission_render_group.rc in [0, 2] - fail_msg: Unable to determine whether the fresh PXE rootfs has a render group. - when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'fresh' - -- name: Require strict existing fresh PXE render group - ansible.builtin.assert: - that: - - _pxe_admission_render_group.stdout_lines | length == 1 - - _pxe_admission_render_group.stdout.split(':') | length == 4 - - _pxe_admission_render_group.stdout.split(':')[0] == 'render' - - _pxe_admission_render_group.stdout.split(':')[2] is match('^[1-9][0-9]*$') - - _pxe_admission_render_group.stdout.split(':')[2] | int <= 4294967294 - fail_msg: Fresh PXE rootfs render group is malformed. - when: >- - pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'fresh' and - _pxe_admission_render_group.rc == 0 - -- name: Require strict retained PXE render group - ansible.builtin.assert: - that: - - _pxe_admission_render_group.rc == 0 - - _pxe_admission_render_group.stdout_lines | length == 1 - - _pxe_admission_render_group.stdout.split(':') | length == 4 - - _pxe_admission_render_group.stdout.split(':')[0] == 'render' - - _pxe_admission_render_group.stdout.split(':')[2] is match('^[1-9][0-9]*$') - - _pxe_admission_render_group.stdout.split(':')[2] | int <= 4294967294 - fail_msg: Retained PXE rootfs must already have one valid render group. - when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' - -- name: Record existing PXE render GID - ansible.builtin.set_fact: - _pxe_existing_render_gid: "{{ _pxe_admission_render_group.stdout.split(':')[2] | int }}" - when: - - pxe_gpu_access_enabled | bool - - _pxe_admission_render_group.rc == 0 - -- name: List fresh PXE rootfs groups before render GID creation - ansible.builtin.command: - argv: [chroot, "{{ pxe_nfs_root }}", getent, group] - register: _pxe_fresh_groups - changed_when: false - failed_when: false - when: - - pxe_gpu_access_enabled | bool - - _pxe_rootfs_disposition == 'fresh' - - _pxe_existing_render_gid | default(none) is none - - _pxe_unanimous_live_render_gid is not none - -- name: Reject fresh PXE render GID collision - ansible.builtin.assert: - that: - - _pxe_fresh_groups.rc == 0 - - >- - _pxe_fresh_groups.stdout_lines - | select('match', '^[^:]*:[^:]*:' ~ (_pxe_unanimous_live_render_gid | string) ~ ':') - | reject('match', '^render:') | list | length == 0 - fail_msg: Fresh PXE rootfs render GID is already assigned to another group. - when: - - pxe_gpu_access_enabled | bool - - _pxe_rootfs_disposition == 'fresh' - - _pxe_existing_render_gid | default(none) is none - - _pxe_unanimous_live_render_gid is not none - -- name: Create missing fresh PXE render group - ansible.builtin.command: - argv: >- - {{ ['chroot', pxe_nfs_root, 'groupadd', '--system', '-g', (_pxe_unanimous_live_render_gid | string), 'render'] - if _pxe_unanimous_live_render_gid is not none - else ['chroot', pxe_nfs_root, 'groupadd', '--system', 'render'] }} - changed_when: true - when: - - pxe_gpu_access_enabled | bool - - _pxe_rootfs_disposition == 'fresh' - - _pxe_existing_render_gid | default(none) is none - -- name: Read fresh PXE render group after creation - ansible.builtin.command: - argv: [chroot, "{{ pxe_nfs_root }}", getent, group, render] - register: _pxe_created_render_group - changed_when: false - when: - - pxe_gpu_access_enabled | bool - - _pxe_rootfs_disposition == 'fresh' - - _pxe_existing_render_gid | default(none) is none - -- name: Resolve newly created fresh PXE render GID - ansible.builtin.set_fact: - _pxe_resolved_render_gid: "{{ _pxe_created_render_group.stdout.split(':')[2] | int }}" - when: - - pxe_gpu_access_enabled | bool - - _pxe_rootfs_disposition == 'fresh' - - _pxe_existing_render_gid | default(none) is none - -- name: Resolve existing fresh PXE render GID - ansible.builtin.set_fact: - _pxe_resolved_render_gid: "{{ _pxe_existing_render_gid }}" - when: - - pxe_gpu_access_enabled | bool - - _pxe_rootfs_disposition == 'fresh' - - _pxe_existing_render_gid | default(none) is not none - -- name: Resolve retained PXE render GID - ansible.builtin.set_fact: - _pxe_resolved_render_gid: "{{ _pxe_existing_render_gid }}" - when: - - pxe_gpu_access_enabled | bool - - _pxe_rootfs_disposition == 'retained' - -- name: List retained PXE rootfs groups for render GID collision check - ansible.builtin.command: - argv: [chroot, "{{ pxe_nfs_root }}", getent, group] - register: _pxe_retained_groups - changed_when: false - failed_when: false - when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' - -- name: Reject retained PXE render GID collision - ansible.builtin.assert: - that: - - _pxe_retained_groups.rc == 0 - - >- - _pxe_retained_groups.stdout_lines - | select('match', '^[^:]*:[^:]*:' ~ (_pxe_resolved_render_gid | string) ~ ':') - | reject('match', '^render:') | list | length == 0 - fail_msg: Retained PXE rootfs render GID is already assigned to another group. - when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' - -- name: Require retained PXE render GID matches unanimous live GID - ansible.builtin.assert: - that: _pxe_resolved_render_gid == _pxe_unanimous_live_render_gid - fail_msg: Retained PXE rootfs render GID differs from the supplied unanimous live render GID; rebuild or migrate it separately. - when: - - pxe_gpu_access_enabled | bool - - _pxe_rootfs_disposition == 'retained' - - _pxe_unanimous_live_render_gid is not none - - name: Inspect retained PXE canonical GPU access parents ansible.builtin.stat: path: "{{ pxe_nfs_root }}{{ item }}" @@ -177,9 +18,6 @@ - /etc - /etc/udev - /etc/udev/rules.d - - /var - - /var/lib - - /var/lib/auplc register: _pxe_retained_canonical_gpu_parent_stats when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' @@ -202,7 +40,6 @@ - /etc/udev/rules.d/70-amdgpu.rules - /etc/udev/rules.d/70-rocm-devices.rules - /etc/udev/rules.d/70-auplc-gpu-access.rules - - /var/lib/auplc/gpu-access.json register: _pxe_retained_gpu_policy_stats when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' @@ -213,60 +50,34 @@ loop: "{{ _pxe_retained_gpu_policy_stats.results[:3] }}" when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' -- name: Require retained PXE canonical GPU destinations +- name: Require retained PXE canonical GPU rule destination ansible.builtin.assert: that: - - item.stat.exists - - item.stat.isreg - - not item.stat.islnk - fail_msg: "Retained PXE rootfs requires an exact canonical GPU access destination: {{ item.item }}" - loop: "{{ _pxe_retained_gpu_policy_stats.results[3:] }}" + - _pxe_retained_gpu_policy_stats.results[3].stat.exists + - _pxe_retained_gpu_policy_stats.results[3].stat.isreg + - not _pxe_retained_gpu_policy_stats.results[3].stat.islnk + fail_msg: Retained PXE rootfs requires an exact canonical GPU access rule. when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' -- name: Read retained PXE canonical GPU access destinations +- name: Read retained PXE canonical GPU rule ansible.builtin.slurp: - src: "{{ item.item }}" - loop: "{{ _pxe_retained_gpu_policy_stats.results[3:] }}" - register: _pxe_retained_canonical_gpu_destinations + src: "{{ _pxe_retained_gpu_policy_stats.results[3].item }}" + register: _pxe_retained_canonical_gpu_rule when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' -- name: Define retained PXE canonical GPU rule +- name: Define retained PXE canonical GPU rules ansible.builtin.set_fact: _pxe_retained_canonical_rule: | # Managed by auplc-installer: AMD GPU device access. - KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0660" - SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0660" + KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0666" + SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0666" + SUBSYSTEM=="drm", KERNEL=="card*", DRIVERS=="amdgpu", OWNER="root", GROUP="video", MODE="0666" when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' - name: Require retained PXE canonical GPU rule ansible.builtin.assert: - that: (item.content | b64decode) == _pxe_retained_canonical_rule - fail_msg: "Retained PXE rootfs has a non-canonical GPU access rule: {{ item.item.item }}" - loop: "{{ _pxe_retained_canonical_gpu_destinations.results }}" - when: - - pxe_gpu_access_enabled | bool - - _pxe_rootfs_disposition == 'retained' - - item.item.item.endswith('70-auplc-gpu-access.rules') - -- name: Parse retained PXE canonical GPU state - ansible.builtin.set_fact: - _pxe_retained_canonical_state: "{{ item.content | b64decode | auplc_from_json_strict }}" - loop: "{{ _pxe_retained_canonical_gpu_destinations.results }}" - when: - - pxe_gpu_access_enabled | bool - - _pxe_rootfs_disposition == 'retained' - - item.item.item.endswith('gpu-access.json') - -- name: Require retained PXE canonical GPU state - ansible.builtin.assert: - that: - - _pxe_retained_canonical_state is mapping - - _pxe_retained_canonical_state.keys() | list | sort == ['renderGid', 'version'] - - _pxe_retained_canonical_state.version is integer - - _pxe_retained_canonical_state.version == 1 - - _pxe_retained_canonical_state.renderGid is integer - - _pxe_retained_canonical_state.renderGid == _pxe_resolved_render_gid - fail_msg: Retained PXE rootfs has a non-canonical GPU access state. + that: (_pxe_retained_canonical_gpu_rule.content | b64decode) == _pxe_retained_canonical_rule + fail_msg: Retained PXE rootfs has a non-canonical GPU access rule. when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' - name: Preflight GPU access after final PXE re-preflight @@ -276,8 +87,6 @@ vars: auplc_rootfs_path: "{{ pxe_nfs_root }}" auplc_rootfs_allowed_root: "{{ pxe_nfs_allowed_root }}" - auplc_render_gid: "{{ _pxe_resolved_render_gid }}" - auplc_normalize_render_gid: "{{ _pxe_rootfs_disposition == 'fresh' }}" when: - pxe_gpu_access_enabled | bool - pxe_gpu_admission_phase == 'final' @@ -289,8 +98,6 @@ vars: auplc_rootfs_path: "{{ pxe_nfs_root }}" auplc_rootfs_allowed_root: "{{ pxe_nfs_allowed_root }}" - auplc_render_gid: "{{ _pxe_resolved_render_gid }}" - auplc_normalize_render_gid: "{{ _pxe_rootfs_disposition == 'fresh' }}" when: - pxe_gpu_access_enabled | bool - pxe_gpu_admission_phase == 'final' From 117b43b7551a92c6db7751335342b619ab82deec Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:30:54 +0800 Subject: [PATCH 30/65] refactor(deploy): simplify GPU discovery --- .../playbooks/pb-gpu-access-discovery.yml | 249 +----------------- .../scripts/gpu_access_resolution.py | 159 +---------- tests/skills/test_gpu_access_resolution.py | 244 +++-------------- 3 files changed, 54 insertions(+), 598 deletions(-) diff --git a/deploy/ansible/playbooks/pb-gpu-access-discovery.yml b/deploy/ansible/playbooks/pb-gpu-access-discovery.yml index cd8c366a..f788e0d9 100644 --- a/deploy/ansible/playbooks/pb-gpu-access-discovery.yml +++ b/deploy/ansible/playbooks/pb-gpu-access-discovery.yml @@ -1,4 +1,4 @@ -# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. --- - name: Discover fleet GPU-access evidence hosts: k3s_cluster @@ -14,48 +14,6 @@ sysfs: rc: 255 stdout: "" - render_group: - rc: 255 - stdout: "" - groups: - rc: 255 - stdout: "" - state: - stat_success: false - content_success: false - exists: false - regular: false - symlink: false - content: "" - rule: - stat_success: false - content_success: false - exists: false - regular: false - symlink: false - content: "" - legacy_rules: - kfd: - stat_success: false - content_success: false - exists: false - regular: false - symlink: false - content: "" - amdgpu: - stat_success: false - content_success: false - exists: false - regular: false - symlink: false - content: "" - rocm_devices: - stat_success: false - content_success: false - exists: false - regular: false - symlink: false - content: "" pre_tasks: - name: Require a safe local discovery evidence output path ansible.builtin.assert: @@ -63,9 +21,7 @@ - gpu_access_discovery_output_path is defined - gpu_access_discovery_output_path is string - gpu_access_discovery_output_path is match('^/') - fail_msg: >- - Set gpu_access_discovery_output_path to an absolute controller-local - path before running discovery. + fail_msg: Set gpu_access_discovery_output_path to an absolute controller-local path before running discovery. delegate_to: localhost run_once: true changed_when: false @@ -113,33 +69,21 @@ tasks: - name: Discover AMD VGA display BDFs with lspci ansible.builtin.command: - argv: - - lspci - - -Dnn - - -d - - "1002::0300" + argv: [lspci, -Dnn, -d, "1002::0300"] register: _auplc_discovery_lspci_vga changed_when: false failed_when: false - name: Discover AMD 3D display BDFs with lspci ansible.builtin.command: - argv: - - lspci - - -Dnn - - -d - - "1002::0302" + argv: [lspci, -Dnn, -d, "1002::0302"] register: _auplc_discovery_lspci_3d changed_when: false failed_when: false - name: Discover AMD display-controller BDFs with lspci ansible.builtin.command: - argv: - - lspci - - -Dnn - - -d - - "1002::0380" + argv: [lspci, -Dnn, -d, "1002::0380"] register: _auplc_discovery_lspci_display changed_when: false failed_when: false @@ -172,120 +116,6 @@ changed_when: false failed_when: false - - name: Read render group record - ansible.builtin.command: - argv: - - getent - - group - - render - register: _auplc_discovery_render_group - changed_when: false - failed_when: false - - - name: Read all group records for render GID collision detection - ansible.builtin.command: - argv: - - getent - - group - register: _auplc_discovery_groups - changed_when: false - failed_when: false - - - name: Inspect persisted GPU access state - ansible.builtin.stat: - path: /var/lib/auplc/gpu-access.json - follow: false - register: _auplc_discovery_state - changed_when: false - ignore_errors: true - - - name: Read persisted GPU access state - ansible.builtin.slurp: - src: /var/lib/auplc/gpu-access.json - register: _auplc_discovery_state_content - when: - - _auplc_discovery_state.stat.exists | default(false) - - _auplc_discovery_state.stat.isreg | default(false) - - not (_auplc_discovery_state.stat.islnk | default(false)) - changed_when: false - ignore_errors: true - - - name: Inspect canonical GPU access rule - ansible.builtin.stat: - path: /etc/udev/rules.d/70-auplc-gpu-access.rules - follow: false - register: _auplc_discovery_rule - changed_when: false - ignore_errors: true - - - name: Read canonical GPU access rule - ansible.builtin.slurp: - src: /etc/udev/rules.d/70-auplc-gpu-access.rules - register: _auplc_discovery_rule_content - when: - - _auplc_discovery_rule.stat.exists | default(false) - - _auplc_discovery_rule.stat.isreg | default(false) - - not (_auplc_discovery_rule.stat.islnk | default(false)) - changed_when: false - ignore_errors: true - - - name: Inspect legacy kfd GPU access rule - ansible.builtin.stat: - path: /etc/udev/rules.d/70-kfd.rules - follow: false - register: _auplc_discovery_legacy_kfd - changed_when: false - ignore_errors: true - - - name: Read legacy kfd GPU access rule - ansible.builtin.slurp: - src: /etc/udev/rules.d/70-kfd.rules - register: _auplc_discovery_legacy_kfd_content - when: - - _auplc_discovery_legacy_kfd.stat.exists | default(false) - - _auplc_discovery_legacy_kfd.stat.isreg | default(false) - - not (_auplc_discovery_legacy_kfd.stat.islnk | default(false)) - changed_when: false - ignore_errors: true - - - name: Inspect legacy amdgpu GPU access rule - ansible.builtin.stat: - path: /etc/udev/rules.d/70-amdgpu.rules - follow: false - register: _auplc_discovery_legacy_amdgpu - changed_when: false - ignore_errors: true - - - name: Read legacy amdgpu GPU access rule - ansible.builtin.slurp: - src: /etc/udev/rules.d/70-amdgpu.rules - register: _auplc_discovery_legacy_amdgpu_content - when: - - _auplc_discovery_legacy_amdgpu.stat.exists | default(false) - - _auplc_discovery_legacy_amdgpu.stat.isreg | default(false) - - not (_auplc_discovery_legacy_amdgpu.stat.islnk | default(false)) - changed_when: false - ignore_errors: true - - - name: Inspect legacy ROCm devices GPU access rule - ansible.builtin.stat: - path: /etc/udev/rules.d/70-rocm-devices.rules - follow: false - register: _auplc_discovery_legacy_rocm_devices - changed_when: false - ignore_errors: true - - - name: Read legacy ROCm devices GPU access rule - ansible.builtin.slurp: - src: /etc/udev/rules.d/70-rocm-devices.rules - register: _auplc_discovery_legacy_rocm_devices_content - when: - - _auplc_discovery_legacy_rocm_devices.stat.exists | default(false) - - _auplc_discovery_legacy_rocm_devices.stat.isreg | default(false) - - not (_auplc_discovery_legacy_rocm_devices.stat.islnk | default(false)) - changed_when: false - ignore_errors: true - - name: Record machine-readable GPU access discovery evidence ansible.builtin.set_fact: _auplc_gpu_access_discovery_evidence: @@ -297,79 +127,12 @@ sysfs: rc: "{{ _auplc_discovery_sysfs.rc }}" stdout: "{{ _auplc_discovery_sysfs.stdout | default('') }}" - render_group: - rc: "{{ _auplc_discovery_render_group.rc }}" - stdout: "{{ _auplc_discovery_render_group.stdout | default('') }}" - groups: - rc: "{{ _auplc_discovery_groups.rc }}" - stdout: "{{ _auplc_discovery_groups.stdout | default('') }}" - state: - stat_success: "{{ not (_auplc_discovery_state.failed | default(false)) }}" - content_success: >- - {{ not (_auplc_discovery_state.failed | default(false)) and - (not (_auplc_discovery_state.stat.exists | default(false)) or - not (_auplc_discovery_state.stat.isreg | default(false)) or - (_auplc_discovery_state.stat.islnk | default(false)) or - not (_auplc_discovery_state_content.failed | default(false))) }} - exists: "{{ _auplc_discovery_state.stat.exists | default(false) }}" - regular: "{{ _auplc_discovery_state.stat.isreg | default(false) }}" - symlink: "{{ _auplc_discovery_state.stat.islnk | default(false) }}" - content: "{{ _auplc_discovery_state_content.content | default('') | b64decode }}" - rule: - stat_success: "{{ not (_auplc_discovery_rule.failed | default(false)) }}" - content_success: >- - {{ not (_auplc_discovery_rule.failed | default(false)) and - (not (_auplc_discovery_rule.stat.exists | default(false)) or - not (_auplc_discovery_rule.stat.isreg | default(false)) or - (_auplc_discovery_rule.stat.islnk | default(false)) or - not (_auplc_discovery_rule_content.failed | default(false))) }} - exists: "{{ _auplc_discovery_rule.stat.exists | default(false) }}" - regular: "{{ _auplc_discovery_rule.stat.isreg | default(false) }}" - symlink: "{{ _auplc_discovery_rule.stat.islnk | default(false) }}" - content: "{{ _auplc_discovery_rule_content.content | default('') | b64decode }}" - legacy_rules: - kfd: - stat_success: "{{ not (_auplc_discovery_legacy_kfd.failed | default(false)) }}" - content_success: >- - {{ not (_auplc_discovery_legacy_kfd.failed | default(false)) and - (not (_auplc_discovery_legacy_kfd.stat.exists | default(false)) or - not (_auplc_discovery_legacy_kfd.stat.isreg | default(false)) or - (_auplc_discovery_legacy_kfd.stat.islnk | default(false)) or - not (_auplc_discovery_legacy_kfd_content.failed | default(false))) }} - exists: "{{ _auplc_discovery_legacy_kfd.stat.exists | default(false) }}" - regular: "{{ _auplc_discovery_legacy_kfd.stat.isreg | default(false) }}" - symlink: "{{ _auplc_discovery_legacy_kfd.stat.islnk | default(false) }}" - content: "{{ _auplc_discovery_legacy_kfd_content.content | default('') | b64decode }}" - amdgpu: - stat_success: "{{ not (_auplc_discovery_legacy_amdgpu.failed | default(false)) }}" - content_success: >- - {{ not (_auplc_discovery_legacy_amdgpu.failed | default(false)) and - (not (_auplc_discovery_legacy_amdgpu.stat.exists | default(false)) or - not (_auplc_discovery_legacy_amdgpu.stat.isreg | default(false)) or - (_auplc_discovery_legacy_amdgpu.stat.islnk | default(false)) or - not (_auplc_discovery_legacy_amdgpu_content.failed | default(false))) }} - exists: "{{ _auplc_discovery_legacy_amdgpu.stat.exists | default(false) }}" - regular: "{{ _auplc_discovery_legacy_amdgpu.stat.isreg | default(false) }}" - symlink: "{{ _auplc_discovery_legacy_amdgpu.stat.islnk | default(false) }}" - content: "{{ _auplc_discovery_legacy_amdgpu_content.content | default('') | b64decode }}" - rocm_devices: - stat_success: "{{ not (_auplc_discovery_legacy_rocm_devices.failed | default(false)) }}" - content_success: >- - {{ not (_auplc_discovery_legacy_rocm_devices.failed | default(false)) and - (not (_auplc_discovery_legacy_rocm_devices.stat.exists | default(false)) or - not (_auplc_discovery_legacy_rocm_devices.stat.isreg | default(false)) or - (_auplc_discovery_legacy_rocm_devices.stat.islnk | default(false)) or - not (_auplc_discovery_legacy_rocm_devices_content.failed | default(false))) }} - exists: "{{ _auplc_discovery_legacy_rocm_devices.stat.exists | default(false) }}" - regular: "{{ _auplc_discovery_legacy_rocm_devices.stat.isreg | default(false) }}" - symlink: "{{ _auplc_discovery_legacy_rocm_devices.stat.islnk | default(false) }}" - content: "{{ _auplc_discovery_legacy_rocm_devices_content.content | default('') | b64decode }}" changed_when: false - name: Write machine-readable GPU access discovery evidence locally ansible.builtin.copy: content: | - {"version":2,"hosts":[{% for discovery_host in ansible_play_hosts_all %} + {"version":1,"hosts":[{% for discovery_host in ansible_play_hosts_all %} {{ ( hostvars[discovery_host]._auplc_gpu_access_discovery_evidence | default( diff --git a/skills/deploy-aup-learning-cloud/scripts/gpu_access_resolution.py b/skills/deploy-aup-learning-cloud/scripts/gpu_access_resolution.py index b618b2f8..555c456d 100644 --- a/skills/deploy-aup-learning-cloud/scripts/gpu_access_resolution.py +++ b/skills/deploy-aup-learning-cloud/scripts/gpu_access_resolution.py @@ -11,13 +11,7 @@ from config_common import DuplicateJsonKeyError, strict_json_loads from gpu_resolution_manifest import ResolutionManifest, build_resolution_manifest -EVIDENCE_VERSION: Final = 2 -MAX_RENDER_GID: Final = 4_294_967_294 -CANONICAL_RULE: Final = ( - "# Managed by auplc-installer: AMD GPU device access.\n" - 'KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0660"\n' - 'SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0660"\n' -) +EVIDENCE_VERSION: Final = 1 BDF_PATTERN: Final = re.compile(r"[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[0-9a-fA-F]{2}\.[0-7]") @@ -58,41 +52,18 @@ class CommandEvidence: stdout: str -@dataclass(frozen=True, slots=True) -class FileEvidence: - stat_success: bool - content_success: bool - exists: bool - regular: bool - symlink: bool - content: str - - -@dataclass(frozen=True, slots=True) -class LegacyRuleEvidence: - kfd: FileEvidence - amdgpu: FileEvidence - rocm_devices: FileEvidence - - @dataclass(frozen=True, slots=True) class HostEvidence: target: InventoryTarget reachable: bool lspci: CommandEvidence sysfs: CommandEvidence - render_group: CommandEvidence - groups: CommandEvidence - state: FileEvidence - rule: FileEvidence - legacy_rules: LegacyRuleEvidence @dataclass(frozen=True, slots=True) class HostResolution: target: InventoryTarget status: HostStatus - render_gid: int | None reason: str | None @@ -100,7 +71,6 @@ class HostResolution: class FleetResolution: status: FleetStatus hosts: tuple[HostResolution, ...] - render_gid: int | None reason: str | None @@ -135,26 +105,21 @@ def resolve_fleet(expected_targets: tuple[InventoryTarget, ...], evidence: tuple return _blocked(resolutions, "unknown host evidence") gpu_hosts = tuple(host for host in resolutions if host.status is HostStatus.GPU) if not gpu_hosts: - return FleetResolution(FleetStatus.CPU_ONLY, resolutions, None, None) - gids = {host.render_gid for host in gpu_hosts} - if len(gids) != 1: - return _blocked(resolutions, "GPU render GIDs disagree") - return FleetResolution(FleetStatus.GPU_RESOLVED, resolutions, next(iter(gids)), None) + return FleetResolution(FleetStatus.CPU_ONLY, resolutions, None) + return FleetResolution(FleetStatus.GPU_RESOLVED, resolutions, None) def resolution_manifest(resolution: FleetResolution) -> ResolutionManifest: """Build the public serialized manifest for a resolved fleet.""" return build_resolution_manifest( - version=1, status=resolution.status.value, - render_gid=resolution.render_gid, hosts={host.target.name: host.status is HostStatus.GPU for host in resolution.hosts}, ) def _parse_host(raw, field: str) -> HostEvidence: _require_mapping(raw, field) - required = {"host", "reachable", "lspci", "sysfs", "render_group", "groups", "state", "rule", "legacy_rules"} + required = {"host", "reachable", "lspci", "sysfs"} if set(raw) != required or type(raw["host"]) is not str or not raw["host"]: raise EvidenceParseError(field=field) if type(raw["reachable"]) is not bool: @@ -164,11 +129,6 @@ def _parse_host(raw, field: str) -> HostEvidence: reachable=raw["reachable"], lspci=_parse_command(raw["lspci"], f"{field}.lspci"), sysfs=_parse_command(raw["sysfs"], f"{field}.sysfs"), - render_group=_parse_command(raw["render_group"], f"{field}.render_group"), - groups=_parse_command(raw["groups"], f"{field}.groups"), - state=_parse_file(raw["state"], f"{field}.state"), - rule=_parse_file(raw["rule"], f"{field}.rule"), - legacy_rules=_parse_legacy_rules(raw["legacy_rules"], f"{field}.legacy_rules"), ) @@ -179,29 +139,6 @@ def _parse_command(raw, field: str) -> CommandEvidence: return CommandEvidence(rc=raw["rc"], stdout=raw["stdout"]) -def _parse_file(raw, field: str) -> FileEvidence: - _require_mapping(raw, field) - required = {"stat_success", "content_success", "exists", "regular", "symlink", "content"} - if set(raw) != required or any( - type(raw[key]) is not bool for key in ("stat_success", "content_success", "exists", "regular", "symlink") - ): - raise EvidenceParseError(field=field) - if type(raw["content"]) is not str: - raise EvidenceParseError(field=f"{field}.content") - return FileEvidence(**raw) - - -def _parse_legacy_rules(raw, field: str) -> LegacyRuleEvidence: - _require_mapping(raw, field) - if set(raw) != {"kfd", "amdgpu", "rocm_devices"}: - raise EvidenceParseError(field=field) - return LegacyRuleEvidence( - kfd=_parse_file(raw["kfd"], f"{field}.kfd"), - amdgpu=_parse_file(raw["amdgpu"], f"{field}.amdgpu"), - rocm_devices=_parse_file(raw["rocm_devices"], f"{field}.rocm_devices"), - ) - - def _require_mapping(value, field: str) -> None: if type(value) is not dict: raise EvidenceParseError(field=field) @@ -210,20 +147,13 @@ def _require_mapping(value, field: str) -> None: def _resolve_host(evidence: HostEvidence) -> HostResolution: if not evidence.reachable or evidence.lspci.rc != 0 or evidence.sysfs.rc != 0: return _unknown(evidence, "GPU discovery probe failed") - if not _file_probes_succeeded(evidence): - return _unknown(evidence, "GPU access file probe failed") lspci_bdfs = _bdfs(evidence.lspci.stdout) sysfs_bdfs = _bdfs(evidence.sysfs.stdout) if lspci_bdfs is None or sysfs_bdfs is None or lspci_bdfs != sysfs_bdfs: return _unknown(evidence, "AMD GPU BDF probes disagree") if not lspci_bdfs: - if evidence.state.exists or evidence.rule.exists or _legacy_rule_exists(evidence.legacy_rules): - return _unknown(evidence, "CPU host retains GPU access contract") - return HostResolution(evidence.target, HostStatus.CPU, None, None) - render_gid = _render_gid(evidence) - if render_gid is None or not _safe_gpu_files(evidence, render_gid): - return _unknown(evidence, "GPU access contract is unsafe") - return HostResolution(evidence.target, HostStatus.GPU, render_gid, None) + return HostResolution(evidence.target, HostStatus.CPU, None) + return HostResolution(evidence.target, HostStatus.GPU, None) def _bdfs(stdout: str) -> frozenset[str] | None: @@ -233,82 +163,9 @@ def _bdfs(stdout: str) -> frozenset[str] | None: return None -def _render_gid(evidence: HostEvidence) -> int | None: - if evidence.render_group.rc != 0 or evidence.groups.rc != 0: - return None - record = _group_record(evidence.render_group.stdout) - if record is None or record[0] != "render": - return None - gid = record[1] - groups = tuple(_group_record(line) for line in evidence.groups.stdout.splitlines()) - if not groups or any(group is None for group in groups): - return None - if sum(group[0] == "render" and group[1] == gid for group in groups) != 1: - return None - if any(group[0] != "render" and group[1] == gid for group in groups): - return None - return gid - - -def _group_record(record: str) -> tuple[str, int] | None: - fields = record.split(":") - if len(fields) != 4 or not fields[0] or not fields[2].isascii() or not fields[2].isdecimal(): - return None - gid = int(fields[2]) - if 1 <= gid <= MAX_RENDER_GID: - return fields[0], gid - return None - - -def _safe_gpu_files(evidence: HostEvidence, render_gid: int) -> bool: - if not _safe_file(evidence.state) or not _safe_file(evidence.rule): - return False - if evidence.state.exists and _state_gid(evidence.state.content) != render_gid: - return False - return not evidence.rule.exists or evidence.rule.content == CANONICAL_RULE - - -def _safe_file(evidence: FileEvidence) -> bool: - if not evidence.stat_success or not evidence.content_success: - return False - if evidence.exists: - return evidence.regular and not evidence.symlink - return not evidence.regular and not evidence.symlink and not evidence.content - - -def _file_probes_succeeded(evidence: HostEvidence) -> bool: - return all( - file_evidence.stat_success and file_evidence.content_success - for file_evidence in ( - evidence.state, - evidence.rule, - evidence.legacy_rules.kfd, - evidence.legacy_rules.amdgpu, - evidence.legacy_rules.rocm_devices, - ) - ) - - -def _legacy_rule_exists(evidence: LegacyRuleEvidence) -> bool: - return any(file_evidence.exists for file_evidence in (evidence.kfd, evidence.amdgpu, evidence.rocm_devices)) - - -def _state_gid(raw: str) -> int | None: - try: - state = strict_json_loads(raw) - except (DuplicateJsonKeyError, TypeError, json.JSONDecodeError): - return None - if type(state) is not dict or set(state) != {"renderGid", "version"}: - return None - gid = state["renderGid"] - if type(gid) is not int or type(state["version"]) is not int or state["version"] != 1: - return None - return gid if 1 <= gid <= MAX_RENDER_GID else None - - def _unknown(evidence: HostEvidence, reason: str) -> HostResolution: - return HostResolution(evidence.target, HostStatus.UNKNOWN, None, reason) + return HostResolution(evidence.target, HostStatus.UNKNOWN, reason) def _blocked(hosts: tuple[HostResolution, ...], reason: str) -> FleetResolution: - return FleetResolution(FleetStatus.BLOCKED, hosts, None, reason) + return FleetResolution(FleetStatus.BLOCKED, hosts, reason) diff --git a/tests/skills/test_gpu_access_resolution.py b/tests/skills/test_gpu_access_resolution.py index ea853f00..37003f28 100644 --- a/tests/skills/test_gpu_access_resolution.py +++ b/tests/skills/test_gpu_access_resolution.py @@ -6,6 +6,7 @@ import importlib.util import json +import re import sys from pathlib import Path @@ -14,6 +15,7 @@ ROOT = Path(__file__).resolve().parents[2] RESOLUTION = ROOT / "skills" / "deploy-aup-learning-cloud" / "scripts" / "gpu_access_resolution.py" MANIFEST = ROOT / "skills" / "deploy-aup-learning-cloud" / "scripts" / "gpu_resolution_manifest.py" +DISCOVERY_PLAYBOOK = ROOT / "deploy" / "ansible" / "playbooks" / "pb-gpu-access-discovery.yml" GPU_BDF = "0000:03:00.0" @@ -45,13 +47,6 @@ def host_evidence( lspci_rc: int = 0, sysfs_rc: int = 0, reachable: bool = True, - render_gid: int = 993, - group_listing: str | None = None, - state: str | None = None, - rule: str | None = None, - state_stat_success: bool = True, - state_content_success: bool = True, - legacy_rules: dict[str, str | None] | None = None, ) -> dict: lspci = "\n".join(lspci_bdfs or []) sysfs = "\n".join(sysfs_bdfs if sysfs_bdfs is not None else lspci_bdfs or []) @@ -60,67 +55,44 @@ def host_evidence( "reachable": reachable, "lspci": {"rc": lspci_rc, "stdout": lspci}, "sysfs": {"rc": sysfs_rc, "stdout": sysfs}, - "render_group": {"rc": 0, "stdout": f"render:x:{render_gid}:\n"}, - "groups": {"rc": 0, "stdout": group_listing or f"render:x:{render_gid}:\n"}, - "state": { - "stat_success": state_stat_success, - "content_success": state_content_success, - "exists": state is not None, - "regular": state is not None, - "symlink": False, - "content": state or "", - }, - "rule": { - "stat_success": True, - "content_success": True, - "exists": rule is not None, - "regular": rule is not None, - "symlink": False, - "content": rule or "", - }, - "legacy_rules": { - key: { - "stat_success": True, - "content_success": True, - "exists": content is not None, - "regular": content is not None, - "symlink": False, - "content": content or "", - } - for key, content in (legacy_rules or {}).items() - } - | { - key: { - "stat_success": True, - "content_success": True, - "exists": False, - "regular": False, - "symlink": False, - "content": "", - } - for key in ("kfd", "amdgpu", "rocm_devices") - if key not in (legacy_rules or {}) - }, } def evidence_document(*hosts: dict) -> str: - return json.dumps({"version": 2, "hosts": list(hosts)}) + return json.dumps({"version": 1, "hosts": list(hosts)}) def expected_targets(module, *names: str): return tuple(module.InventoryTarget(name=name) for name in names) +def test_discovery_playbook_serializes_the_exact_v1_host_evidence_shape() -> None: + playbook = DISCOVERY_PLAYBOOK.read_text(encoding="utf-8") + evidence_block = playbook.split("_auplc_gpu_access_discovery_evidence:", maxsplit=1)[1].split( + " changed_when:", maxsplit=1 + )[0] + fallback_block = playbook.split("_auplc_gpu_access_unknown_evidence:", maxsplit=1)[1].split( + " pre_tasks:", maxsplit=1 + )[0] + evidence_keys = re.findall(r"^ ([a-z_]+):", evidence_block, re.MULTILINE) + fallback_keys = re.findall(r"^ ([a-z_]+):", fallback_block, re.MULTILINE) + + assert evidence_keys == ["host", "reachable", "lspci", "sysfs"] + assert fallback_keys == ["reachable", "lspci", "sysfs"] + assert "combine({'host': discovery_host})" in playbook + assert '{"version":1,"hosts":[' in playbook + assert "hostvars[discovery_host]._auplc_gpu_access_discovery_evidence" in playbook + assert "| to_json" in playbook + + def test_parse_fleet_evidence_accepts_the_exact_machine_evidence_schema() -> None: module = load_resolution_module() - raw = evidence_document(host_evidence("gpu-1", lspci_bdfs=[GPU_BDF])) - evidence = module.parse_fleet_evidence(raw) + evidence = module.parse_fleet_evidence(evidence_document(host_evidence("gpu-1", lspci_bdfs=[GPU_BDF]))) assert evidence[0].target == module.InventoryTarget(name="gpu-1") assert evidence[0].lspci.stdout == GPU_BDF - assert evidence[0].state.exists is False + assert evidence[0].sysfs.stdout == GPU_BDF @pytest.mark.parametrize( @@ -143,38 +115,7 @@ def test_parse_fleet_evidence_rejects_duplicate_json_keys() -> None: module = load_resolution_module() with pytest.raises(module.EvidenceParseError, match="duplicate JSON key 'version'"): - module.parse_fleet_evidence('{"version":2,"version":2,"hosts":[]}') - - -def test_resolve_fleet_blocks_duplicate_persisted_state_render_gid() -> None: - module = load_resolution_module() - parsed = module.parse_fleet_evidence( - evidence_document( - host_evidence("gpu-1", lspci_bdfs=[GPU_BDF], state='{"renderGid":993,"renderGid":993,"version":1}') - ) - ) - - resolution = module.resolve_fleet(expected_targets(module, "gpu-1"), parsed) - - assert resolution.status is module.FleetStatus.BLOCKED - - -@pytest.mark.parametrize( - "state", - [ - '{"renderGid":993,"version":1,"version":1}', - '{"renderGid":993,"version":1,"r\\u0065nderGid":993}', - '{"renderGid":993,"version":1,"v\\u0065rsion":1}', - ], - ids=["duplicate-version", "escaped-render-gid", "escaped-version"], -) -def test_resolve_fleet_blocks_semantic_duplicate_persisted_state_keys(state: str) -> None: - module = load_resolution_module() - parsed = module.parse_fleet_evidence(evidence_document(host_evidence("gpu-1", lspci_bdfs=[GPU_BDF], state=state))) - - resolution = module.resolve_fleet(expected_targets(module, "gpu-1"), parsed) - - assert resolution.status is module.FleetStatus.BLOCKED + module.parse_fleet_evidence('{"version":1,"version":1,"hosts":[]}') def test_resolve_fleet_classifies_matching_amd_bdfs_as_gpu() -> None: @@ -184,7 +125,6 @@ def test_resolve_fleet_classifies_matching_amd_bdfs_as_gpu() -> None: resolution = module.resolve_fleet(expected_targets(module, "gpu-1"), evidence) assert resolution.status is module.FleetStatus.GPU_RESOLVED - assert resolution.render_gid == 993 assert resolution.hosts[0].status is module.HostStatus.GPU @@ -195,7 +135,6 @@ def test_resolve_fleet_classifies_two_empty_successful_gpu_probes_as_cpu_only() resolution = module.resolve_fleet(expected_targets(module, "cpu-1"), evidence) assert resolution.status is module.FleetStatus.CPU_ONLY - assert resolution.render_gid is None assert resolution.hosts[0].status is module.HostStatus.CPU @@ -235,174 +174,71 @@ def test_resolve_fleet_blocks_incomplete_or_unexpected_host_evidence( resolution = module.resolve_fleet(expected_targets(module, *targets), parsed) assert resolution.status is module.FleetStatus.BLOCKED - assert resolution.render_gid is None + assert resolution.reason == "incomplete host coverage" -def test_resolve_fleet_blocks_render_gid_collisions() -> None: +def test_resolve_fleet_accepts_gpu_hosts_without_a_shared_gid() -> None: module = load_resolution_module() parsed = module.parse_fleet_evidence( evidence_document( - host_evidence( - "gpu-1", - lspci_bdfs=[GPU_BDF], - group_listing="render:x:993:\nother:x:993:\n", - ) + host_evidence("gpu-1", lspci_bdfs=[GPU_BDF]), + host_evidence("gpu-2", lspci_bdfs=["0000:04:00.0"]), ) ) - resolution = module.resolve_fleet(expected_targets(module, "gpu-1"), parsed) - - assert resolution.status is module.FleetStatus.BLOCKED - assert resolution.hosts[0].status is module.HostStatus.UNKNOWN - - -def test_resolve_fleet_blocks_cpu_hosts_with_persisted_gpu_access_contracts() -> None: - module = load_resolution_module() - parsed = module.parse_fleet_evidence( - evidence_document(host_evidence("cpu-1", state='{"renderGid":993,"version":1}')) - ) - - resolution = module.resolve_fleet(expected_targets(module, "cpu-1"), parsed) - - assert resolution.status is module.FleetStatus.BLOCKED - assert resolution.hosts[0].status is module.HostStatus.UNKNOWN - - -@pytest.mark.parametrize("legacy_key", ["kfd", "amdgpu", "rocm_devices"]) -def test_resolve_fleet_blocks_cpu_hosts_with_any_legacy_gpu_access_rule(legacy_key: str) -> None: - module = load_resolution_module() - parsed = module.parse_fleet_evidence( - evidence_document(host_evidence("cpu-1", legacy_rules={legacy_key: 'KERNEL=="kfd", MODE="0666"\n'})) - ) - - resolution = module.resolve_fleet(expected_targets(module, "cpu-1"), parsed) - - assert resolution.status is module.FleetStatus.BLOCKED - assert resolution.hosts[0].status is module.HostStatus.UNKNOWN - - -def test_resolve_fleet_keeps_gpu_legacy_rule_admission_for_the_later_exact_migration() -> None: - module = load_resolution_module() - parsed = module.parse_fleet_evidence( - evidence_document(host_evidence("gpu-1", lspci_bdfs=[GPU_BDF], legacy_rules={"amdgpu": "legacy\n"})) - ) - - resolution = module.resolve_fleet(expected_targets(module, "gpu-1"), parsed) + resolution = module.resolve_fleet(expected_targets(module, "gpu-1", "gpu-2"), parsed) assert resolution.status is module.FleetStatus.GPU_RESOLVED + assert [host.status for host in resolution.hosts] == [module.HostStatus.GPU, module.HostStatus.GPU] -def test_resolve_fleet_blocks_file_probe_failures_instead_of_treating_them_as_absence() -> None: - module = load_resolution_module() - parsed = module.parse_fleet_evidence( - evidence_document(host_evidence("cpu-1", state_stat_success=False, state_content_success=False)) - ) - - resolution = module.resolve_fleet(expected_targets(module, "cpu-1"), parsed) - - assert resolution.status is module.FleetStatus.BLOCKED - assert resolution.hosts[0].status is module.HostStatus.UNKNOWN - - -@pytest.mark.parametrize( - "contracts", - [ - {"state": '{"renderGid":994,"version":1}'}, - {"rule": 'KERNEL=="kfd", MODE="0666"\n'}, - ], -) -def test_resolve_fleet_blocks_gpu_hosts_with_unsafe_persisted_contracts(contracts: dict) -> None: - module = load_resolution_module() - parsed = module.parse_fleet_evidence(evidence_document(host_evidence("gpu-1", lspci_bdfs=[GPU_BDF], **contracts))) - - resolution = module.resolve_fleet(expected_targets(module, "gpu-1"), parsed) - - assert resolution.status is module.FleetStatus.BLOCKED - assert resolution.hosts[0].status is module.HostStatus.UNKNOWN - - -def test_resolve_fleet_requires_unanimous_gpu_render_gid() -> None: - module = load_resolution_module() - same_gid = module.parse_fleet_evidence( - evidence_document( - host_evidence("gpu-1", lspci_bdfs=[GPU_BDF], render_gid=993), - host_evidence("gpu-2", lspci_bdfs=["0000:04:00.0"], render_gid=993), - ) - ) - mixed_gid = module.parse_fleet_evidence( - evidence_document( - host_evidence("gpu-1", lspci_bdfs=[GPU_BDF], render_gid=993), - host_evidence("gpu-2", lspci_bdfs=["0000:04:00.0"], render_gid=994), - ) - ) - - resolved = module.resolve_fleet(expected_targets(module, "gpu-1", "gpu-2"), same_gid) - blocked = module.resolve_fleet(expected_targets(module, "gpu-1", "gpu-2"), mixed_gid) - - assert resolved.status is module.FleetStatus.GPU_RESOLVED - assert resolved.render_gid == 993 - assert blocked.status is module.FleetStatus.BLOCKED - assert blocked.render_gid is None - - -def test_resolution_manifest_preserves_explicit_host_booleans_and_unanimous_gid() -> None: +def test_resolution_manifest_preserves_explicit_host_booleans() -> None: module = load_resolution_module() parsed = module.parse_fleet_evidence( evidence_document( - host_evidence("gpu-1", lspci_bdfs=[GPU_BDF], render_gid=993), + host_evidence("gpu-1", lspci_bdfs=[GPU_BDF]), host_evidence("cpu-1"), ) ) - resolution = module.resolve_fleet(expected_targets(module, "gpu-1", "cpu-1"), parsed) - manifest = module.resolution_manifest(resolution) + manifest = module.resolution_manifest(module.resolve_fleet(expected_targets(module, "gpu-1", "cpu-1"), parsed)) assert manifest == { "version": 1, "status": "gpu_resolved", - "render_gid": 993, "hosts": {"cpu-1": False, "gpu-1": True}, } def test_resolution_manifest_is_an_ordinary_dict_with_exact_order_and_sorted_hosts() -> None: manifest = load_manifest_module().build_resolution_manifest( - version=1, status="gpu_resolved", - render_gid=993, hosts={"zeta": True, "alpha": False}, ) assert type(manifest) is dict - assert list(manifest) == ["version", "status", "render_gid", "hosts"] + assert list(manifest) == ["version", "status", "hosts"] assert list(manifest["hosts"]) == ["alpha", "zeta"] - assert set(manifest) == {"version", "status", "render_gid", "hosts"} + assert set(manifest) == {"version", "status", "hosts"} def test_pxe_resolution_manifest_constructs_without_mutating_base_manifest() -> None: module = load_manifest_module() base = module.build_resolution_manifest( - version=1, status="gpu_resolved", - render_gid=993, hosts={"gpu-2": True, "gpu-1": True}, ) manifest = module.build_pxe_resolution_manifest( - version=base["version"], - status=base["status"], - render_gid=base["render_gid"], - hosts=base["hosts"], + base, gpu_access_enabled=True, - pxe_render_gid=994, ) assert base == { "version": 1, "status": "gpu_resolved", - "render_gid": 993, "hosts": {"gpu-1": True, "gpu-2": True}, } - assert list(manifest) == ["version", "status", "render_gid", "hosts", "pxe_rootfs"] - assert manifest["pxe_rootfs"] == {"gpu_access_enabled": True, "render_gid": 994} - assert set(manifest["pxe_rootfs"]) == {"gpu_access_enabled", "render_gid"} + assert list(manifest) == ["version", "status", "hosts", "pxe_rootfs"] + assert manifest["pxe_rootfs"] == {"gpu_access_enabled": True} + assert set(manifest["pxe_rootfs"]) == {"gpu_access_enabled"} From f217d1fc984c81957f842f29fc2e1f94639adc5c Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:30:54 +0800 Subject: [PATCH 31/65] refactor(deploy): remove draft GPU inputs --- .../scripts/config_generation.py | 7 +------ tests/skills/test_config_generation_security.py | 9 +++++++++ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/skills/deploy-aup-learning-cloud/scripts/config_generation.py b/skills/deploy-aup-learning-cloud/scripts/config_generation.py index 5fdcf84a..c0e881ff 100644 --- a/skills/deploy-aup-learning-cloud/scripts/config_generation.py +++ b/skills/deploy-aup-learning-cloud/scripts/config_generation.py @@ -8,12 +8,11 @@ import re from config_common import DEFAULT_ACCEL_LABELS, HEADER_HASH, die, require, yaml_quote -from config_rendering import ResolvedGpuPolicy, render_inventory, render_pxe_vars, render_values +from config_rendering import render_inventory, render_pxe_vars, render_values __all__ = [ "DEFAULT_ACCEL_LABELS", "HEADER_HASH", - "ResolvedGpuPolicy", "SCHEMA", "die", "render_inventory", @@ -200,10 +199,6 @@ def validate_spec(spec: dict) -> str: server_name = _validate_server(require(spec, "server"), "spec.server") _validate_agents(spec, server_name) _validate_rendered_options(spec) - if "render_gid" in spec: - die("spec.render_gid is no longer accepted; GPU policy is discovered automatically") - if "gpu_access" in spec: - die("spec.gpu_access is no longer accepted; GPU policy is discovered automatically") if topo == "pxe-diskless": _validate_pxe(spec) return topo diff --git a/tests/skills/test_config_generation_security.py b/tests/skills/test_config_generation_security.py index 6ec39bf8..adb6c282 100644 --- a/tests/skills/test_config_generation_security.py +++ b/tests/skills/test_config_generation_security.py @@ -73,6 +73,15 @@ def test_generator_rejects_an_invalid_k3s_version_before_discovery(capsys: pytes assert "spec.k3s_version" in capsys.readouterr().err +def test_generator_applies_the_normal_unknown_field_policy_to_draft_gpu_fields() -> None: + module = load_config_generation_module() + spec = safe_spec() + spec["render_gid"] = 993 + spec["gpu_access"] = {"hosts": []} + + assert module.validate_spec(spec) == "ssh-preinstalled" + + @pytest.mark.parametrize( "raw", [ From 2d28d6c87b413bce11a43987ec0fdae324d057b5 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:30:54 +0800 Subject: [PATCH 32/65] refactor(deploy): render boolean GPU policy --- .../scripts/config_rendering.py | 24 +-- .../scripts/gen_configs.py | 66 +++----- .../scripts/gpu_artifact_generation.py | 13 +- tests/skills/test_gpu_artifact_generation.py | 158 +++++++----------- 4 files changed, 93 insertions(+), 168 deletions(-) diff --git a/skills/deploy-aup-learning-cloud/scripts/config_rendering.py b/skills/deploy-aup-learning-cloud/scripts/config_rendering.py index 2ac59a86..b66a0064 100644 --- a/skills/deploy-aup-learning-cloud/scripts/config_rendering.py +++ b/skills/deploy-aup-learning-cloud/scripts/config_rendering.py @@ -4,24 +4,14 @@ from __future__ import annotations -from dataclasses import dataclass - from config_common import DEFAULT_ACCEL_LABELS, HEADER_HASH, die, require, yaml_quote from gpu_access_resolution import FleetResolution, HostStatus -@dataclass(frozen=True, slots=True) -class ResolvedGpuPolicy: - host_gpu_enabled: dict[str, bool] - render_gid: int | None - pxe_gpu_enabled: bool - - def render_inventory(spec: dict, token: str, resolution: FleetResolution) -> str: topo = spec["topology"] server = spec["server"] k3s_version = spec["k3s_version"] - render_gid = resolution.render_gid host_gpu_enabled = {host.target.name: host.status is HostStatus.GPU for host in resolution.hosts} lines = [ HEADER_HASH, @@ -49,7 +39,6 @@ def render_inventory(spec: dict, token: str, resolution: FleetResolution) -> str " ansible_port: 22", " ansible_user: root", f" k3s_version: {yaml_quote(k3s_version)}", - f" auplc_render_gid: {'null' if render_gid is None else render_gid}", f" token: {yaml_quote(token)}", " api_endpoint: \"{{ hostvars[groups['server'][0]]['ansible_host'] | default(groups['server'][0]) }}\"", ] @@ -67,7 +56,7 @@ def render_inventory(spec: dict, token: str, resolution: FleetResolution) -> str return "\n".join(lines) + "\n" -def render_pxe_vars(spec: dict, policy: ResolvedGpuPolicy, finalizer_context: str | None = None) -> str: +def render_pxe_vars(spec: dict, pxe_gpu_access_enabled: bool) -> str: net = require(spec, "network") pxe = spec.get("pxe", {}) keys = pxe.get("authorized_keys", []) @@ -75,7 +64,6 @@ def render_pxe_vars(spec: dict, policy: ResolvedGpuPolicy, finalizer_context: st die("pxe.authorized_keys must contain at least one SSH public key") server_ip = spec["server"]["ip"] k3s_version = spec["k3s_version"] - render_gid = policy.render_gid lines = [ HEADER_HASH, "# Pass this file to pb-pxe-controller.yml with", @@ -91,26 +79,22 @@ def render_pxe_vars(spec: dict, policy: ResolvedGpuPolicy, finalizer_context: st "pxe_k3s_server_ips:", f" - {yaml_quote(server_ip)}", f"pxe_k3s_version: {yaml_quote(k3s_version)}", - f"auplc_render_gid: {'null' if render_gid is None else render_gid}", - f"pxe_gpu_access_enabled: {'true' if policy.pxe_gpu_enabled else 'false'}", + f"pxe_gpu_access_enabled: {'true' if pxe_gpu_access_enabled else 'false'}", f"pxe_web_port: {int(pxe.get('web_port', 8080))}", f"pxe_rootfs_password: {yaml_quote(pxe.get('rootfs_password', ''))}", "pxe_rootfs_authorized_keys:", ] for key in keys: lines.append(f" - {yaml_quote(key)}") - if finalizer_context is not None: - lines.append(f"pxe_finalizer_context: {yaml_quote(finalizer_context)}") return "\n".join(lines) + "\n" -def render_values(spec: dict, resolution: FleetResolution) -> str: +def render_values(spec: dict) -> str: accel = spec.get("accelerators") or {} storage_class = (spec.get("storage") or {}).get("class", "nfs-client") node_port = (spec.get("proxy") or {}).get("node_port", 30890) auth_mode = spec.get("auth_mode", "auto-login") images = spec.get("images") or {} - render_gid = resolution.render_gid lines = [ "# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved.", "# Helm overlay generated by auplc-skills gen_configs.py.", @@ -119,8 +103,6 @@ def render_values(spec: dict, resolution: FleetResolution) -> str: "# --create-namespace -f runtime/values.yaml -f ", "custom:", f" authMode: {yaml_quote(auth_mode)}", - " gpuAccess:", - f" renderGid: {'null' if render_gid is None else render_gid}", ] if accel: lines.append(" accelerators:") diff --git a/skills/deploy-aup-learning-cloud/scripts/gen_configs.py b/skills/deploy-aup-learning-cloud/scripts/gen_configs.py index 3a3bdb9f..e9665c10 100755 --- a/skills/deploy-aup-learning-cloud/scripts/gen_configs.py +++ b/skills/deploy-aup-learning-cloud/scripts/gen_configs.py @@ -3,8 +3,8 @@ """Generate AUP Learning Cloud deploy artifacts from a small cluster-spec. Given a JSON cluster-spec (see ``--print-schema``), discover the managed hosts' -GPU policy. SSH and PXE without GPU-enabled diskless agents immediately write -mutually consistent canonical deployment artifacts: +GPU policy. Both topologies immediately write mutually consistent canonical +deployment artifacts: 1. ``inventory.yml`` -- Ansible inventory (server + token + k3s_version; agents listed for the @@ -12,26 +12,19 @@ 2. ``pb-pxe-controller.vars.yml`` -- PXE topology only: extra vars passed to pb-pxe-controller.yml with ``-e @``. - 3. ``values-basic-example.yaml`` -- Helm overlay: resolved render GID, - storage, proxy, and authentication. + 3. ``values-basic-example.yaml`` -- Helm overlay: storage, proxy, and + authentication. 4. ``gpu-access-resolution.json`` -- Machine-readable resolved host policy. -GPU-enabled PXE instead writes private ``.pxe-bootstrap.inventory.yml``, -``.pxe-bootstrap.vars.yml``, and ``.pxe-finalizer-context.json`` files while -canonical artifacts remain absent. The controller playbook publishes the -canonical artifacts only after it resolves the rootfs GID and succeeds. - Design choices (deliberate): * stdlib only (json, argparse, secrets, base64, pathlib). No PyYAML, so this runs on a bare operator machine. YAML is emitted from templates, not a serialiser -- the output is small, fixed-shape, and carries the copyright header. - * The k3s token is generated locally with ``secrets`` (CSPRNG). Immediate - canonical output writes it only into ``inventory.yml``. Pending GPU-enabled - PXE stores it only in private ``.pxe-finalizer-context.json`` until the - controller succeeds and finalization writes ``inventory.yml``. It is never - printed to stdout/stderr. Pass ``--token-file`` to reuse an existing token - instead of minting one. + * The k3s token is generated locally with ``secrets`` (CSPRNG). Canonical + output writes it only into ``inventory.yml``. It is never printed to + stdout/stderr. Pass ``--token-file`` to reuse an existing token instead of + minting one. * ``pxe_k3s_version`` is forced equal to ``k3s_version`` so agents can never be newer than the server (k3s refuses that). * Existing files are not overwritten unless ``--force`` is given. @@ -59,12 +52,12 @@ SCHEMA, die, render_inventory, + render_pxe_vars, render_values, validate_spec, validate_yaml_scalar, ) from gpu_artifact_generation import DiscoveryFailure, canonical_paths, discover_gpu_policy, manifest_content -from pxe_finalization import FinalizationError, finalize, publish_disabled_rootfs, stage_pending def gen_token() -> str: @@ -79,22 +72,11 @@ def main(argv=None) -> int: ap.add_argument("--token-file", help="read the k3s token from this file instead of generating one") ap.add_argument("--force", action="store_true", help="overwrite existing files") ap.add_argument("--print-schema", action="store_true", help="print an example cluster-spec and exit") - ap.add_argument("--finalize-pxe", action="store_true", help=argparse.SUPPRESS) - ap.add_argument("--context", help=argparse.SUPPRESS) - ap.add_argument("--handoff", help=argparse.SUPPRESS) args = ap.parse_args(argv) if args.print_schema: print(json.dumps(SCHEMA, indent=2)) return 0 - if args.finalize_pxe: - if args.spec or args.token_file or args.context is None or args.handoff is None: - die("--finalize-pxe requires --out-dir, --context, and --handoff", 2) - try: - finalize(Path(args.out_dir), Path(args.context), Path(args.handoff)) - except FinalizationError as error: - die(str(error)) - return 0 if not args.spec: die("--spec is required (or use --print-schema)", 2) @@ -116,24 +98,20 @@ def main(argv=None) -> int: discovery = discover_gpu_policy(spec, out) except DiscoveryFailure as error: die(str(error)) + inventory, values, manifest = canonical_paths(out) + artifacts = [(inventory, render_inventory(spec, token, discovery.resolution), 0o600, True)] + pxe_gpu_access_enabled = None if topo == "pxe-diskless": - try: - if spec["pxe"]["diskless_agents_have_amd_gpus"]: - stage_pending(spec, token, discovery.resolution, out, args.force) - print("PXE GPU rootfs is pending finalization after pb-pxe-controller.yml resolves its render GID.") - else: - publish_disabled_rootfs(spec, token, discovery.resolution, out, args.force) - except FinalizationError as error: - die(str(error)) - else: - inventory, values, manifest = canonical_paths(out) - artifacts = [(inventory, render_inventory(spec, token, discovery.resolution), 0o600, True)] - artifacts += [ - (values, render_values(spec, discovery.resolution), 0o644, False), - (manifest, manifest_content(discovery), 0o644, False), - ] - preflight_destinations([path for path, _, _, _ in artifacts], args.force) - publish_artifacts(artifacts, args.force) + pxe_gpu_access_enabled = spec["pxe"]["diskless_agents_have_amd_gpus"] + artifacts.append( + (out / "pb-pxe-controller.vars.yml", render_pxe_vars(spec, pxe_gpu_access_enabled), 0o600, True) + ) + artifacts += [ + (values, render_values(spec), 0o644, False), + (manifest, manifest_content(discovery, pxe_gpu_access_enabled), 0o644, False), + ] + preflight_destinations([path for path, _, _, _ in artifacts], args.force) + publish_artifacts(artifacts, args.force) print( "\nNext: review the files, then copy them into your aup-learning-cloud " diff --git a/skills/deploy-aup-learning-cloud/scripts/gpu_artifact_generation.py b/skills/deploy-aup-learning-cloud/scripts/gpu_artifact_generation.py index 3c47e1d2..e16fac14 100644 --- a/skills/deploy-aup-learning-cloud/scripts/gpu_artifact_generation.py +++ b/skills/deploy-aup-learning-cloud/scripts/gpu_artifact_generation.py @@ -25,6 +25,7 @@ resolution_manifest, resolve_fleet, ) +from gpu_resolution_manifest import build_pxe_resolution_manifest DISCOVERY_TIMEOUT_BASE_SECONDS = 30 DISCOVERY_TIMEOUT_PER_TARGET_SECONDS = 15 @@ -213,6 +214,14 @@ def read_regular_file(path: Path) -> str: raise DiscoveryFailure("GPU discovery evidence could not be read") from error -def manifest_content(result: DiscoveryResult) -> str: - document = resolution_manifest(result.resolution) +def manifest_content(result: DiscoveryResult, pxe_gpu_access_enabled: bool | None = None) -> str: + base = resolution_manifest(result.resolution) + document = ( + base + if pxe_gpu_access_enabled is None + else build_pxe_resolution_manifest( + base, + gpu_access_enabled=pxe_gpu_access_enabled, + ) + ) return json.dumps(document, indent=2, sort_keys=True) + "\n" diff --git a/tests/skills/test_gpu_artifact_generation.py b/tests/skills/test_gpu_artifact_generation.py index 92924077..4e40a624 100644 --- a/tests/skills/test_gpu_artifact_generation.py +++ b/tests/skills/test_gpu_artifact_generation.py @@ -16,42 +16,13 @@ GEN_CONFIGS = ROOT / "skills" / "deploy-aup-learning-cloud" / "scripts" / "gen_configs.py" -def evidence_host(name: str, *, gpu: bool = False, gid: int = 993, reachable: bool = True) -> dict: +def evidence_host(name: str, *, gpu: bool = False, reachable: bool = True) -> dict: bdf = "0000:03:00.0" if gpu else "" return { "host": name, "reachable": reachable, "lspci": {"rc": 0, "stdout": bdf}, "sysfs": {"rc": 0, "stdout": bdf}, - "render_group": {"rc": 0, "stdout": f"render:x:{gid}:\n"}, - "groups": {"rc": 0, "stdout": f"render:x:{gid}:\n"}, - "state": { - "stat_success": True, - "content_success": True, - "exists": False, - "regular": False, - "symlink": False, - "content": "", - }, - "rule": { - "stat_success": True, - "content_success": True, - "exists": False, - "regular": False, - "symlink": False, - "content": "", - }, - "legacy_rules": { - key: { - "stat_success": True, - "content_success": True, - "exists": False, - "regular": False, - "symlink": False, - "content": "", - } - for key in ("kfd", "amdgpu", "rocm_devices") - }, } @@ -96,11 +67,26 @@ def write_fake_ansible(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, document return record +def run_generator(spec_path: Path, out_dir: Path, *extra: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(GEN_CONFIGS), "--spec", str(spec_path), "--out-dir", str(out_dir), *extra], + capture_output=True, + check=False, + text=True, + timeout=30, + ) + + +def write_json(path: Path, document: dict) -> Path: + path.write_text(json.dumps(document), encoding="utf-8") + return path + + def test_generator_forces_repository_host_key_checking_over_disabled_environment( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: write_fake_ansible( - tmp_path, monkeypatch, {"version": 2, "hosts": [evidence_host("server"), evidence_host("agent")]} + tmp_path, monkeypatch, {"version": 1, "hosts": [evidence_host("server"), evidence_host("agent")]} ) environment_record = tmp_path / "ansible-environment.json" monkeypatch.setenv("FAKE_ANSIBLE_ENV_RECORD", str(environment_record)) @@ -113,9 +99,8 @@ def test_generator_forces_repository_host_key_checking_over_disabled_environment monkeypatch.setenv("ANSIBLE_SCP_IF_SSH", "True") monkeypatch.setenv("ANSIBLE_SCP_EXTRA_ARGS", "-o UserKnownHostsFile=/dev/null") monkeypatch.setenv("ANSIBLE_SFTP_EXTRA_ARGS", "-o StrictHostKeyChecking=no") - spec_path = write_json(tmp_path / "spec.json", ssh_spec()) - result = run_generator(spec_path, tmp_path / "generated") + result = run_generator(write_json(tmp_path / "spec.json", ssh_spec()), tmp_path / "generated") assert result.returncode == 0, result.stderr assert json.loads(environment_record.read_text(encoding="utf-8")) == { @@ -143,9 +128,8 @@ def test_generator_surfaces_redacted_bounded_ansible_failure_diagnostics( ) fake_ansible.chmod(0o755) monkeypatch.setenv("PATH", f"{fake_bin}:{os.environ['PATH']}") - spec_path = write_json(tmp_path / "spec.json", ssh_spec()) - result = run_generator(spec_path, tmp_path / "generated") + result = run_generator(write_json(tmp_path / "spec.json", ssh_spec()), tmp_path / "generated") assert result.returncode == 1 assert "exit code 2" in result.stderr @@ -154,44 +138,27 @@ def test_generator_surfaces_redacted_bounded_ansible_failure_diagnostics( assert "token=" in result.stderr -def run_generator(spec_path: Path, out_dir: Path, *extra: str) -> subprocess.CompletedProcess[str]: - return subprocess.run( - [sys.executable, str(GEN_CONFIGS), "--spec", str(spec_path), "--out-dir", str(out_dir), *extra], - capture_output=True, - check=False, - text=True, - timeout=30, - ) - - -def write_json(path: Path, document: dict) -> Path: - path.write_text(json.dumps(document), encoding="utf-8") - return path - - def test_generator_discovers_mixed_ssh_targets_and_publishes_resolved_artifacts( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: record = write_fake_ansible( tmp_path, monkeypatch, - {"version": 2, "hosts": [evidence_host("server", gpu=True), evidence_host("agent")]}, + {"version": 1, "hosts": [evidence_host("server", gpu=True), evidence_host("agent")]}, ) - spec_path = write_json(tmp_path / "spec.json", ssh_spec()) out_dir = tmp_path / "generated" - result = run_generator(spec_path, out_dir) + result = run_generator(write_json(tmp_path / "spec.json", ssh_spec()), out_dir) assert result.returncode == 0, result.stderr inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") - assert "auplc_render_gid: 993" in inventory assert inventory.count("auplc_gpu_access_enabled: true") == 1 assert inventory.count("auplc_gpu_access_enabled: false") == 1 - assert "renderGid: 993" in (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") + assert "auplc_render_gid" not in inventory + assert "gpuAccess" not in (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") assert json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) == { "version": 1, "status": "gpu_resolved", - "render_gid": 993, "hosts": {"agent": False, "server": True}, } discovery_inventory = out_dir / ".gpu-access-discovery.inventory.yml" @@ -207,29 +174,46 @@ def test_generator_discovers_mixed_ssh_targets_and_publishes_resolved_artifacts( ] -def test_generator_publishes_null_render_gid_for_all_cpu_ssh_targets( +def test_generator_allows_heterogeneous_gpu_hosts_and_publishes_boolean_only_artifacts( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: write_fake_ansible( tmp_path, monkeypatch, - {"version": 2, "hosts": [evidence_host("server"), evidence_host("agent")]}, + {"version": 1, "hosts": [evidence_host("server", gpu=True), evidence_host("agent", gpu=True)]}, ) - spec_path = write_json(tmp_path / "spec.json", ssh_spec()) out_dir = tmp_path / "generated" - result = run_generator(spec_path, out_dir) + result = run_generator(write_json(tmp_path / "spec.json", ssh_spec()), out_dir) assert result.returncode == 0, result.stderr inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") - assert "auplc_render_gid: null" in inventory - assert inventory.count("auplc_gpu_access_enabled: false") == 2 - assert "renderGid: null" in (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") + values = (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) - assert manifest == { + assert inventory.count("auplc_gpu_access_enabled: true") == 2 + assert "auplc_render_gid" not in inventory + assert "gpuAccess" not in values + assert manifest == {"version": 1, "status": "gpu_resolved", "hosts": {"agent": True, "server": True}} + + +def test_generator_publishes_boolean_only_artifacts_for_all_cpu_ssh_targets( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_fake_ansible( + tmp_path, monkeypatch, {"version": 1, "hosts": [evidence_host("server"), evidence_host("agent")]} + ) + out_dir = tmp_path / "generated" + + result = run_generator(write_json(tmp_path / "spec.json", ssh_spec()), out_dir) + + assert result.returncode == 0, result.stderr + inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") + assert inventory.count("auplc_gpu_access_enabled: false") == 2 + assert "auplc_render_gid" not in inventory + assert "gpuAccess" not in (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") + assert json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) == { "version": 1, "status": "cpu_only", - "render_gid": None, "hosts": {"agent": False, "server": False}, } @@ -238,7 +222,6 @@ def test_generator_publishes_null_render_gid_for_all_cpu_ssh_targets( def test_generator_does_not_publish_when_ansible_is_unavailable_or_fails( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure: str ) -> None: - spec_path = write_json(tmp_path / "spec.json", ssh_spec()) out_dir = tmp_path / "generated" fake_bin = tmp_path / "bin" fake_bin.mkdir() @@ -248,7 +231,7 @@ def test_generator_does_not_publish_when_ansible_is_unavailable_or_fails( fake_ansible.chmod(0o755) monkeypatch.setenv("PATH", str(fake_bin)) - result = run_generator(spec_path, out_dir) + result = run_generator(write_json(tmp_path / "spec.json", ssh_spec()), out_dir) assert result.returncode == 1 assert not (out_dir / "inventory.yml").exists() @@ -256,22 +239,14 @@ def test_generator_does_not_publish_when_ansible_is_unavailable_or_fails( assert not (out_dir / "gpu-access-resolution.json").exists() -@pytest.mark.parametrize( - "document", - [ - {"version": 2, "hosts": [evidence_host("server", reachable=False), evidence_host("agent")]}, - { - "version": 2, - "hosts": [evidence_host("server", gpu=True, gid=993), evidence_host("agent", gpu=True, gid=994)], - }, - ], - ids=["unknown", "gid-disagreement"], -) def test_generator_keeps_canonical_artifacts_unchanged_when_discovery_blocks( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, document: dict + tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - write_fake_ansible(tmp_path, monkeypatch, document) - spec_path = write_json(tmp_path / "spec.json", ssh_spec()) + write_fake_ansible( + tmp_path, + monkeypatch, + {"version": 1, "hosts": [evidence_host("server", reachable=False), evidence_host("agent")]}, + ) out_dir = tmp_path / "generated" out_dir.mkdir() inventory = out_dir / "inventory.yml" @@ -281,28 +256,9 @@ def test_generator_keeps_canonical_artifacts_unchanged_when_discovery_blocks( values.write_text("previous values\n", encoding="utf-8") manifest.write_text("previous manifest\n", encoding="utf-8") - result = run_generator(spec_path, out_dir, "--force") + result = run_generator(write_json(tmp_path / "spec.json", ssh_spec()), out_dir, "--force") assert result.returncode == 1 assert inventory.read_text(encoding="utf-8") == "previous inventory\n" assert values.read_text(encoding="utf-8") == "previous values\n" assert manifest.read_text(encoding="utf-8") == "previous manifest\n" - - -@pytest.mark.parametrize( - "field", - ["render_gid", "gpu_access"], -) -def test_generator_rejects_removed_public_gpu_fields_before_running_discovery( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, field: str -) -> None: - record = write_fake_ansible(tmp_path, monkeypatch, {"version": 2, "hosts": []}) - spec = ssh_spec() - spec[field] = 993 if field == "render_gid" else {"hosts": []} - spec_path = write_json(tmp_path / "spec.json", spec) - - result = run_generator(spec_path, tmp_path / "generated") - - assert result.returncode == 1 - assert f"spec.{field} is no longer accepted" in result.stderr - assert not record.exists() From 709ee3f656ad8151feedb91551eb7452412ca3e6 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:30:54 +0800 Subject: [PATCH 33/65] refactor(deploy): simplify GPU manifests --- .../scripts/gpu_resolution_manifest.py | 23 +--- .../scripts/gpu_resolution_parsing.py | 103 ++---------------- .../scripts/gpu_resolution_validation.py | 25 +---- .../scripts/validate.py | 5 +- tests/skills/test_deploy_scripts.py | 83 ++------------ 5 files changed, 35 insertions(+), 204 deletions(-) diff --git a/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_manifest.py b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_manifest.py index 60a59fd7..e80d0eaf 100644 --- a/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_manifest.py +++ b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_manifest.py @@ -5,7 +5,9 @@ from __future__ import annotations from collections.abc import Mapping -from typing import TypedDict +from typing import Final, TypedDict + +MANIFEST_VERSION: Final = 1 class ResolutionManifest(TypedDict): @@ -13,7 +15,6 @@ class ResolutionManifest(TypedDict): version: int status: str - render_gid: int | None hosts: dict[str, bool] @@ -21,7 +22,6 @@ class PxeRootfsManifest(TypedDict): """Serialized GPU policy applied to the PXE root filesystem.""" gpu_access_enabled: bool - render_gid: int | None class PxeResolutionManifest(ResolutionManifest): @@ -32,37 +32,26 @@ class PxeResolutionManifest(ResolutionManifest): def build_resolution_manifest( *, - version: int, status: str, - render_gid: int | None, hosts: Mapping[str, bool], ) -> ResolutionManifest: """Build a deterministic ordinary dictionary for fleet resolution.""" return { - "version": version, + "version": MANIFEST_VERSION, "status": status, - "render_gid": render_gid, "hosts": {name: hosts[name] for name in sorted(hosts)}, } def build_pxe_resolution_manifest( + resolution: ResolutionManifest, *, - version: int, - status: str, - render_gid: int | None, - hosts: Mapping[str, bool], gpu_access_enabled: bool, - pxe_render_gid: int | None, ) -> PxeResolutionManifest: """Build a PXE manifest without mutating a base fleet manifest.""" return { - "version": version, - "status": status, - "render_gid": render_gid, - "hosts": {name: hosts[name] for name in sorted(hosts)}, + **resolution, "pxe_rootfs": { "gpu_access_enabled": gpu_access_enabled, - "render_gid": pxe_render_gid, }, } diff --git a/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_parsing.py b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_parsing.py index 7f14faad..6fe4c6d1 100644 --- a/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_parsing.py +++ b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_parsing.py @@ -5,29 +5,24 @@ from pathlib import Path from config_common import DuplicateJsonKeyError, strict_json_loads - -MAX_RENDER_GID = 4_294_967_294 +from gpu_resolution_manifest import MANIFEST_VERSION @dataclass(frozen=True, slots=True) class GpuInventory: hosts: dict[str, bool] - render_gid: int | None @dataclass(frozen=True, slots=True) class GpuResolution: status: str hosts: dict[str, bool] - render_gid: int | None pxe_rootfs_enabled: bool | None - pxe_rootfs_gid: int | None @dataclass(frozen=True, slots=True) class PxeGpuPolicy: enabled: bool - render_gid: int | None def configured_path(repo: Path, value: str) -> Path: @@ -35,17 +30,6 @@ def configured_path(repo: Path, value: str) -> Path: return path if path.is_absolute() else repo / path -def parse_gpu_gid(value: str) -> int | None | str: - normalized = value.strip() - if normalized in {"null", "~"}: - return None - if normalized.isascii() and normalized.isdecimal(): - gid = int(normalized) - if 1 <= gid <= MAX_RENDER_GID: - return gid - return "invalid" - - def parse_gpu_boolean(value: str) -> bool | None: normalized = value.strip() if normalized == "true": @@ -62,7 +46,6 @@ def yaml_indent(line: str) -> int: def parse_gpu_inventory(text: str) -> tuple[GpuInventory | None, list[str]]: host_values: dict[str, list[str]] = {} host_names: list[str] = [] - render_gids: list[str] = [] stack: list[tuple[int, str]] = [] for raw_line in text.splitlines(): @@ -95,8 +78,6 @@ def parse_gpu_inventory(text: str) -> tuple[GpuInventory | None, list[str]]: and key == "auplc_gpu_access_enabled" ): host_values.setdefault(path[4], []).append(value) - elif path == ("k3s_cluster", "vars") and key == "auplc_render_gid": - render_gids.append(value) stack.append((indent, key)) parse_errors: list[str] = [] @@ -115,64 +96,9 @@ def parse_gpu_inventory(text: str) -> tuple[GpuInventory | None, list[str]]: parse_errors.append(f"inventory host '{host}' has malformed auplc_gpu_access_enabled") continue hosts[host] = enabled - if len(render_gids) != 1: - parse_errors.append("inventory must define exactly one k3s_cluster.vars.auplc_render_gid") - return None, parse_errors - render_gid = parse_gpu_gid(render_gids[0]) - if render_gid == "invalid": - parse_errors.append("inventory has malformed auplc_render_gid") - return None, parse_errors if parse_errors: return None, parse_errors - return GpuInventory(hosts=hosts, render_gid=render_gid), parse_errors - - -def parse_values_gpu_gid(text: str) -> tuple[int | None, bool, list[str]]: - render_gids: list[str] = [] - stack: list[tuple[int, str]] = [] - for raw_line in text.splitlines(): - line = raw_line.split("#", 1)[0].rstrip() - if not line.strip(): - continue - indent = yaml_indent(line) - stripped = line.strip() - while stack and indent <= stack[-1][0]: - stack.pop() - path = tuple(key for _, key in stack) - mapping_match = re.fullmatch(r"(.+?):(?:\s*(.*))?", stripped) - if not mapping_match: - continue - key = mapping_match.group(1).strip("\"'") - value = (mapping_match.group(2) or "").strip() - if path == ("custom", "gpuAccess") and key == "renderGid": - render_gids.append(value) - stack.append((indent, key)) - if not render_gids: - return None, False, [] - if len(render_gids) != 1: - return None, True, ["custom.gpuAccess.renderGid is duplicated"] - render_gid = parse_gpu_gid(render_gids[0]) - if render_gid == "invalid": - return None, True, ["custom.gpuAccess.renderGid is malformed"] - return render_gid, True, [] - - -def collect_effective_gpu_gid(repo: Path, values: list[str]) -> tuple[int | None, list[str]]: - effective_gid: int | None = None - found = False - parse_errors: list[str] = [] - for rel in values or ["runtime/values.yaml"]: - path = configured_path(repo, rel) - if not path.exists(): - continue - render_gid, present, file_errors = parse_values_gpu_gid(path.read_text(encoding="utf-8")) - parse_errors.extend(f"{path}: {error}" for error in file_errors) - if present and not file_errors: - effective_gid = render_gid - found = True - if not found: - parse_errors.append("effective values have no custom.gpuAccess.renderGid") - return effective_gid, parse_errors + return GpuInventory(hosts=hosts), parse_errors def parse_gpu_resolution(text: str, topology: str) -> tuple[GpuResolution | None, list[str]]: @@ -184,13 +110,13 @@ def parse_gpu_resolution(text: str, topology: str) -> tuple[GpuResolution | None return None, [f"GPU resolution manifest is malformed: {exc}"] if type(document) is not dict: return None, ["GPU resolution manifest must be a JSON object"] - expected_keys = {"version", "status", "render_gid", "hosts"} + expected_keys = {"version", "status", "hosts"} if topology == "pxe-diskless": expected_keys.add("pxe_rootfs") if set(document) != expected_keys: return None, ["GPU resolution manifest has an unexpected schema"] - if type(document["version"]) is not int or document["version"] != 1: - return None, ["GPU resolution manifest version must be integer 1"] + if type(document["version"]) is not int or document["version"] != MANIFEST_VERSION: + return None, [f"GPU resolution manifest version must be integer {MANIFEST_VERSION}"] status = document["status"] if type(status) is not str or status not in {"cpu_only", "gpu_resolved"}: return None, ["GPU resolution manifest status must be cpu_only or gpu_resolved"] @@ -200,25 +126,19 @@ def parse_gpu_resolution(text: str, topology: str) -> tuple[GpuResolution | None type(host) is not str or not host or type(enabled) is not bool for host, enabled in document["hosts"].items() ): return None, ["GPU resolution manifest hosts must map non-empty names to booleans"] - render_gid = document["render_gid"] - if render_gid is not None and (type(render_gid) is not int or not 1 <= render_gid <= MAX_RENDER_GID): - return None, ["GPU resolution manifest render_gid must be an integer or null"] if topology == "ssh-preinstalled": - return GpuResolution(status, document["hosts"], render_gid, None, None), [] + return GpuResolution(status, document["hosts"], None), [] rootfs = document["pxe_rootfs"] - if type(rootfs) is not dict or set(rootfs) != {"gpu_access_enabled", "render_gid"}: + if type(rootfs) is not dict or set(rootfs) != {"gpu_access_enabled"}: return None, ["GPU resolution manifest pxe_rootfs has an unexpected schema"] rootfs_enabled = rootfs["gpu_access_enabled"] - rootfs_gid = rootfs["render_gid"] if type(rootfs_enabled) is not bool: return None, ["GPU resolution manifest pxe_rootfs.gpu_access_enabled must be boolean"] - if rootfs_gid is not None and (type(rootfs_gid) is not int or not 1 <= rootfs_gid <= MAX_RENDER_GID): - return None, ["GPU resolution manifest pxe_rootfs.render_gid must be an integer or null"] - return GpuResolution(status, document["hosts"], render_gid, rootfs_enabled, rootfs_gid), [] + return GpuResolution(status, document["hosts"], rootfs_enabled), [] def parse_pxe_gpu_policy(text: str) -> tuple[PxeGpuPolicy | None, list[str]]: - values: dict[str, list[str]] = {"auplc_render_gid": [], "pxe_gpu_access_enabled": []} + values: dict[str, list[str]] = {"pxe_gpu_access_enabled": []} for raw_line in text.splitlines(): line = raw_line.split("#", 1)[0].rstrip() if not line.strip() or yaml_indent(line) != 0: @@ -235,12 +155,9 @@ def parse_pxe_gpu_policy(text: str) -> tuple[PxeGpuPolicy | None, list[str]]: parse_errors.append(f"PXE vars must define exactly one {key}") if parse_errors: return None, parse_errors - render_gid = parse_gpu_gid(values["auplc_render_gid"][0]) enabled = parse_gpu_boolean(values["pxe_gpu_access_enabled"][0]) - if render_gid == "invalid": - parse_errors.append("PXE vars have malformed auplc_render_gid") if enabled is None: parse_errors.append("PXE vars have malformed pxe_gpu_access_enabled") if parse_errors: return None, parse_errors - return PxeGpuPolicy(enabled=enabled, render_gid=render_gid), [] + return PxeGpuPolicy(enabled=enabled), [] diff --git a/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_validation.py b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_validation.py index 1fd8c041..f33c5cc8 100644 --- a/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_validation.py +++ b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_validation.py @@ -4,7 +4,6 @@ from pathlib import Path from gpu_resolution_parsing import ( - collect_effective_gpu_gid, configured_path, parse_gpu_inventory, parse_gpu_resolution, @@ -17,7 +16,6 @@ class GpuArtifactValidationRequest: repo: Path inventory_path: str resolution_path: str - values: list[str] topology: str pxe_vars_path: Path has_prior_errors: bool @@ -85,8 +83,7 @@ def check_gpu_artifacts(request: GpuArtifactValidationRequest) -> GpuArtifactVal return GpuArtifactValidationResult([f"GPU resolution manifest not found: {resolution_file}"], []) inventory, inventory_errors = parse_gpu_inventory(inventory_file.read_text(encoding="utf-8")) resolution, resolution_errors = parse_gpu_resolution(resolution_file.read_text(encoding="utf-8"), request.topology) - helm_gid, helm_errors = collect_effective_gpu_gid(request.repo, request.values) - errors.extend([*inventory_errors, *resolution_errors, *helm_errors]) + errors.extend([*inventory_errors, *resolution_errors]) if inventory is None or resolution is None or errors: return GpuArtifactValidationResult(errors, []) if set(inventory.hosts) != set(resolution.hosts): @@ -104,22 +101,10 @@ def check_gpu_artifacts(request: GpuArtifactValidationRequest) -> GpuArtifactVal return GpuArtifactValidationResult(errors, []) if pxe_policy.enabled != resolution.pxe_rootfs_enabled: errors.append("PXE pxe_gpu_access_enabled disagrees with GPU resolution manifest pxe_rootfs") - if resolution.pxe_rootfs_enabled and resolution.pxe_rootfs_gid is None: - errors.append("GPU-enabled PXE rootfs requires a numeric render GID") - if not resolution.pxe_rootfs_enabled and resolution.pxe_rootfs_gid is not None: - errors.append("GPU-disabled PXE rootfs requires a null render GID") - if resolution.pxe_rootfs_enabled and pxe_policy.render_gid != resolution.pxe_rootfs_gid: - errors.append("PXE auplc_render_gid disagrees with GPU resolution manifest pxe_rootfs render_gid") - gids = [inventory.render_gid, helm_gid, resolution.render_gid] - if pxe_policy is not None: - gids.append(pxe_policy.render_gid) - if len(set(gids)) != 1: - errors.append("inventory, Helm, PXE, and GPU resolution render GIDs disagree") - enabled_scope = any(resolution.hosts.values()) or resolution.pxe_rootfs_enabled is True if resolution.status == "cpu_only": - if enabled_scope or resolution.render_gid is not None or any(gid is not None for gid in gids): - errors.append("cpu_only GPU resolution requires all host/rootfs booleans false and all render GIDs null") - elif not enabled_scope or resolution.render_gid is None: - errors.append("gpu_resolved GPU resolution requires an enabled scope and a numeric render GID") + if any(resolution.hosts.values()): + errors.append("cpu_only GPU resolution requires all host booleans false") + elif not any(resolution.hosts.values()): + errors.append("gpu_resolved GPU resolution requires an enabled host") passed = [] if request.has_prior_errors or errors else ["GPU access artifacts agree"] return GpuArtifactValidationResult(errors, passed) diff --git a/skills/deploy-aup-learning-cloud/scripts/validate.py b/skills/deploy-aup-learning-cloud/scripts/validate.py index ccb5bb6b..8e3982c1 100755 --- a/skills/deploy-aup-learning-cloud/scripts/validate.py +++ b/skills/deploy-aup-learning-cloud/scripts/validate.py @@ -12,8 +12,8 @@ * nodeSelectors for the accelerators actually referenced by effective custom.resources.metadata.*.acceleratorKeys, checked against detect_cluster.sh output when supplied; - * generated inventory, GPU-resolution manifest, Helm render GID, and PXE - rootfs policy agree when generated artifacts are supplied; + * generated inventory, GPU-resolution manifest, and PXE rootfs policy agree + when generated artifacts are supplied; * (optional) the chart does not render: a `helm template` dry-run. This intentionally uses regex/line scanning rather than a YAML parser so it @@ -254,7 +254,6 @@ def main(argv=None) -> int: repo=repo, inventory_path=args.inventory, resolution_path=args.gpu_resolution, - values=args.values, topology=args.topology, pxe_vars_path=pxe_vars_path(repo, args.pxe_vars), has_prior_errors=bool(errors), diff --git a/tests/skills/test_deploy_scripts.py b/tests/skills/test_deploy_scripts.py index 0469f4fc..c57f916d 100644 --- a/tests/skills/test_deploy_scripts.py +++ b/tests/skills/test_deploy_scripts.py @@ -80,20 +80,12 @@ def fake_ansible_playbook(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> No output = next(value.split('=', 1)[1] for value in arguments if value.startswith('gpu_access_discovery_output_path=')) hosts = [line.strip()[:-1] for line in inventory.read_text(encoding='utf-8').splitlines() if line.startswith(' ') and line.rstrip().endswith(':')] evidence = { - 'version': 2, + 'version': 1, 'hosts': [{ 'host': host, 'reachable': True, 'lspci': {'rc': 0, 'stdout': ''}, 'sysfs': {'rc': 0, 'stdout': ''}, - 'render_group': {'rc': 0, 'stdout': 'render:x:993:\\n'}, - 'groups': {'rc': 0, 'stdout': 'render:x:993:\\n'}, - 'state': {'stat_success': True, 'content_success': True, 'exists': False, 'regular': False, 'symlink': False, 'content': ''}, - 'rule': {'stat_success': True, 'content_success': True, 'exists': False, 'regular': False, 'symlink': False, 'content': ''}, - 'legacy_rules': { - key: {'stat_success': True, 'content_success': True, 'exists': False, 'regular': False, 'symlink': False, 'content': ''} - for key in ('kfd', 'amdgpu', 'rocm_devices') - }, } for host in hosts], } Path(output).write_text(json.dumps(evidence), encoding='utf-8') @@ -123,15 +115,11 @@ def write_resolved_gpu_artifacts(repo: Path) -> tuple[Path, Path, Path]: agent: ansible_host: 192.168.1.11 auplc_gpu_access_enabled: false - vars: - auplc_render_gid: 993 """, ) values = write_file( repo / "generated/values-basic-example.yaml", """custom: - gpuAccess: - renderGid: 993 resources: metadata: {} """, @@ -142,7 +130,6 @@ def write_resolved_gpu_artifacts(repo: Path) -> tuple[Path, Path, Path]: { "version": 1, "status": "gpu_resolved", - "render_gid": 993, "hosts": {"agent": False, "server": True}, } ), @@ -921,15 +908,11 @@ def test_validator_accepts_consistent_cpu_only_gpu_artifacts(tmp_path: Path) -> agent: ansible_host: 192.168.1.11 auplc_gpu_access_enabled: false - vars: - auplc_render_gid: null """, ) values = write_file( repo / "generated/values-basic-example.yaml", """custom: - gpuAccess: - renderGid: null resources: metadata: {} """, @@ -940,7 +923,6 @@ def test_validator_accepts_consistent_cpu_only_gpu_artifacts(tmp_path: Path) -> { "version": 1, "status": "cpu_only", - "render_gid": None, "hosts": {"agent": False, "server": False}, } ), @@ -991,15 +973,15 @@ def test_validator_accepts_consistent_gpu_resolved_artifacts(tmp_path: Path) -> [ ("not JSON", "GPU resolution manifest is malformed"), ( - '{"version":1,"status":"pending","render_gid":993,"hosts":{"agent":false,"server":true}}', + '{"version":1,"status":"pending","hosts":{"agent":false,"server":true}}', "GPU resolution manifest status must be cpu_only or gpu_resolved", ), ( - '{"version":1,"status":"gpu_resolved","render_gid":993,"hosts":{"server":true,"server":false}}', + '{"version":1,"status":"gpu_resolved","hosts":{"server":true,"server":false}}', "duplicate JSON key 'server'", ), ( - '{"version":1,"status":"gpu_resolved","render_gid":993,"hosts":{"ser\\u0076er":true,"server":false}}', + '{"version":1,"status":"gpu_resolved","hosts":{"ser\\u0076er":true,"server":false}}', "duplicate JSON key 'server'", ), ], @@ -1044,8 +1026,6 @@ def test_validator_rejects_malformed_pending_or_duplicate_gpu_resolution( agent: ansible_host: 192.168.1.11 auplc_gpu_access_enabled: false - vars: - auplc_render_gid: 993 """, "inventory host 'server' must define exactly one auplc_gpu_access_enabled", ), @@ -1062,8 +1042,6 @@ def test_validator_rejects_malformed_pending_or_duplicate_gpu_resolution( agent: ansible_host: 192.168.1.11 auplc_gpu_access_enabled: false - vars: - auplc_render_gid: 993 """, "inventory host 'server' has malformed auplc_gpu_access_enabled", ), @@ -1081,8 +1059,6 @@ def test_validator_rejects_malformed_pending_or_duplicate_gpu_resolution( agent: ansible_host: 192.168.1.11 auplc_gpu_access_enabled: false - vars: - auplc_render_gid: 993 """, "inventory host 'server' must define exactly one auplc_gpu_access_enabled", ), @@ -1136,24 +1112,14 @@ def test_validator_rejects_missing_generated_gpu_resolution_artifact(tmp_path: P assert "GPU resolution manifest not found" in result.stdout -def test_validator_rejects_mismatched_host_boolean_and_render_gid(tmp_path: Path) -> None: +def test_validator_rejects_mismatched_host_boolean(tmp_path: Path) -> None: repo = tmp_path / "checkout" inventory, values, resolution = write_resolved_gpu_artifacts(repo) - values.write_text( - """custom: - gpuAccess: - renderGid: 994 - resources: - metadata: {} -""", - encoding="utf-8", - ) resolution.write_text( json.dumps( { "version": 1, "status": "gpu_resolved", - "render_gid": 993, "hosts": {"agent": True, "server": True}, } ), @@ -1176,10 +1142,9 @@ def test_validator_rejects_mismatched_host_boolean_and_render_gid(tmp_path: Path assert result.returncode == 1 assert "inventory host 'agent' GPU access boolean disagrees" in result.stdout - assert "render GIDs disagree" in result.stdout -def test_validator_rejects_pxe_rootfs_boolean_and_gid_mismatch(tmp_path: Path) -> None: +def test_validator_rejects_pxe_rootfs_boolean_mismatch(tmp_path: Path) -> None: repo = tmp_path / "checkout" inventory = write_file( repo / "generated/inventory.yml", @@ -1192,20 +1157,17 @@ def test_validator_rejects_pxe_rootfs_boolean_and_gid_mismatch(tmp_path: Path) - auplc_gpu_access_enabled: false agent: hosts: {} - vars: - auplc_render_gid: 993 """, ) - values = write_file(repo / "generated/values-basic-example.yaml", "custom:\n gpuAccess:\n renderGid: 993\n") + values = write_file(repo / "generated/values-basic-example.yaml", "custom:\n resources:\n metadata: {}\n") resolution = write_file( repo / "generated/gpu-access-resolution.json", json.dumps( { "version": 1, - "status": "gpu_resolved", - "render_gid": 993, + "status": "cpu_only", "hosts": {"server": False}, - "pxe_rootfs": {"gpu_access_enabled": True, "render_gid": 993}, + "pxe_rootfs": {"gpu_access_enabled": True}, } ), ) @@ -1218,7 +1180,6 @@ def test_validator_rejects_pxe_rootfs_boolean_and_gid_mismatch(tmp_path: Path) - pxe_k3s_server_ips: [192.168.1.10] pxe_rootfs_authorized_keys: [ssh-ed25519-AAA] pxe_k3s_version: v1.32.3+k3s1 -auplc_render_gid: 994 pxe_gpu_access_enabled: false """, ) @@ -1241,7 +1202,6 @@ def test_validator_rejects_pxe_rootfs_boolean_and_gid_mismatch(tmp_path: Path) - assert result.returncode == 1 assert "pxe_gpu_access_enabled disagrees" in result.stdout - assert "PXE auplc_render_gid disagrees" in result.stdout def test_generator_rejects_unknown_accelerator_keys_before_writing_artifacts(tmp_path: Path) -> None: @@ -1611,20 +1571,6 @@ def test_generator_exposes_extracted_generation_and_artifact_modules() -> None: assert callable(artifacts.publish_artifacts) -def test_generator_rejects_legacy_public_gpu_policy_fields_before_discovery(tmp_path: Path) -> None: - spec = generator_spec() - spec["render_gid"] = 1055 - spec["gpu_access"] = {"hosts": [], "pxe_rootfs_enabled": False} - spec_path = write_file(tmp_path / "spec.json", json.dumps(spec)) - out_dir = tmp_path / "generated" - - result = run_script(GEN_CONFIGS, "--spec", str(spec_path), "--out-dir", str(out_dir)) - - assert result.returncode == 1 - assert "spec.render_gid is no longer accepted" in result.stderr - assert not out_dir.exists() - - def test_generator_uses_fake_ansible_discovery_to_publish_resolved_ssh_policy(tmp_path: Path) -> None: fake_bin = tmp_path / "bin" fake_bin.mkdir() @@ -1640,13 +1586,8 @@ def host(name, bdf): return { 'host': name, 'reachable': True, 'lspci': {'rc': 0, 'stdout': bdf}, 'sysfs': {'rc': 0, 'stdout': bdf}, - 'render_group': {'rc': 0, 'stdout': 'render:x:993:\\n'}, - 'groups': {'rc': 0, 'stdout': 'render:x:993:\\n'}, - 'state': {'stat_success': True, 'content_success': True, 'exists': False, 'regular': False, 'symlink': False, 'content': ''}, - 'rule': {'stat_success': True, 'content_success': True, 'exists': False, 'regular': False, 'symlink': False, 'content': ''}, - 'legacy_rules': {key: {'stat_success': True, 'content_success': True, 'exists': False, 'regular': False, 'symlink': False, 'content': ''} for key in ('kfd', 'amdgpu', 'rocm_devices')}, } -pathlib.Path(output).write_text(json.dumps({'version': 2, 'hosts': [host('server', '0000:03:00.0'), host('agent', '')]}), encoding='utf-8') +pathlib.Path(output).write_text(json.dumps({'version': 1, 'hosts': [host('server', '0000:03:00.0'), host('agent', '')]}), encoding='utf-8') """, encoding="utf-8", ) @@ -1665,9 +1606,9 @@ def host(name, bdf): assert result.returncode == 0, result.stdout + result.stderr inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") - assert "auplc_render_gid: 993" in inventory assert inventory.count("auplc_gpu_access_enabled: true") == 1 assert inventory.count("auplc_gpu_access_enabled: false") == 1 - assert "renderGid: 993" in (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") + assert "auplc_render_gid" not in inventory + assert "gpuAccess" not in (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) assert manifest["hosts"] == {"agent": False, "server": True} From 131f2f1d5f9de28fbc929c74435ff6f72d1b6578 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:30:54 +0800 Subject: [PATCH 34/65] refactor(deploy): remove PXE finalizer --- .../scripts/pxe_finalization.py | 173 ------- .../scripts/pxe_finalization_support.py | 257 ---------- tests/skills/test_pxe_finalization.py | 457 +++--------------- 3 files changed, 63 insertions(+), 824 deletions(-) delete mode 100644 skills/deploy-aup-learning-cloud/scripts/pxe_finalization.py delete mode 100644 skills/deploy-aup-learning-cloud/scripts/pxe_finalization_support.py diff --git a/skills/deploy-aup-learning-cloud/scripts/pxe_finalization.py b/skills/deploy-aup-learning-cloud/scripts/pxe_finalization.py deleted file mode 100644 index cb9c7c9f..00000000 --- a/skills/deploy-aup-learning-cloud/scripts/pxe_finalization.py +++ /dev/null @@ -1,173 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. -"""Orchestrate transactional PXE configuration finalization.""" - -from __future__ import annotations - -import json -import os -import secrets -from pathlib import Path - -import pxe_finalization_support as _support -from artifact_store import preflight_destinations, publish_artifacts -from config_rendering import ResolvedGpuPolicy, render_inventory, render_pxe_vars, render_values -from gpu_access_resolution import ( - FleetResolution, - FleetStatus, - HostStatus, - resolution_manifest, -) -from gpu_access_resolution import HostResolution as HostResolution -from gpu_resolution_manifest import build_pxe_resolution_manifest -from pxe_finalization_support import MAX_RENDER_GID as MAX_RENDER_GID -from pxe_finalization_support import ( - VERSION, - Artifact, - JsonDocument, -) -from pxe_finalization_support import FinalizationError as FinalizationError -from pxe_finalization_support import PxePaths as PxePaths -from pxe_finalization_support import paths as paths - -_artifact_attestations = _support.artifact_attestations -_completion = _support.completion -_controller_resolution = _support.controller_resolution -_exclusive_lock = _support.exclusive_lock -_final_resolution = _support.final_resolution -_generation_paths = _support.generation_paths -_read_artifact_attestation = _support.read_artifact_attestation -_read_document = _support.read_document -_spec_sha256 = _support.spec_sha256 -_target = _support.target -_valid_gid = _support.valid_gid -_validate = _support.validate -_verify_canonical_artifacts = _support.verify_canonical_artifacts - - -def stage_pending(spec: JsonDocument, token: str, controller: FleetResolution, out_dir: Path, force: bool) -> PxePaths: - pending = paths(out_dir) - if controller.status is FleetStatus.BLOCKED: - raise FinalizationError(f"GPU discovery is blocked: {controller.reason}") - pending.lock.parent.mkdir(parents=True, exist_ok=True) - with _exclusive_lock(pending.lock): - context: JsonDocument = { - "version": VERSION, - "generation": secrets.token_urlsafe(32), - "spec_sha256": _spec_sha256(spec), - "topology": "pxe-diskless", - "spec": spec, - "token": token, - "controller": resolution_manifest(controller), - } - bootstrap = render_pxe_vars(spec, _controller_policy(controller, True), str(pending.context)) - bootstrap += "\n".join( - [ - f"pxe_finalizer_handoff: {_yaml_quote(str(pending.handoff))}", - f"pxe_finalizer_generation: {_yaml_quote(context['generation'])}", - f"pxe_finalizer_spec_sha256: {_yaml_quote(context['spec_sha256'])}", - f"pxe_finalizer_script: {_yaml_quote(str(Path(__file__).with_name('gen_configs.py').resolve()))}", - "", - ] - ) - artifacts: list[Artifact] = [ - (pending.bootstrap_inventory, _render_bootstrap_inventory(spec), 0o600, True), - (pending.bootstrap_vars, bootstrap, 0o600, True), - (pending.context, json.dumps(context, sort_keys=True) + "\n", 0o600, True), - ] - if not force: - preflight_destinations(_generation_paths(pending), False) - publish_artifacts(artifacts, force, _generation_paths(pending)) - return pending - - -def publish_disabled_rootfs( - spec: JsonDocument, token: str, controller: FleetResolution, out_dir: Path, force: bool -) -> None: - pending = paths(out_dir) - if controller.status is FleetStatus.BLOCKED: - raise FinalizationError(f"GPU discovery is blocked: {controller.reason}") - pending.lock.parent.mkdir(parents=True, exist_ok=True) - with _exclusive_lock(pending.lock): - policy = _controller_policy(controller, False) - artifacts: list[Artifact] = [ - (pending.inventory, render_inventory(spec, token, controller), 0o600, True), - (pending.pxe_vars, render_pxe_vars(spec, policy), 0o600, True), - (pending.values, render_values(spec, controller), 0o644, False), - (pending.manifest, _manifest(controller, False, None), 0o644, False), - ] - if not force: - preflight_destinations(_generation_paths(pending), False) - publish_artifacts(artifacts, force, _generation_paths(pending)) - - -def finalize(out_dir: Path, context_path: Path, handoff_path: Path) -> None: - pending = paths(out_dir) - if context_path.resolve() != pending.context or handoff_path.resolve() != pending.handoff: - raise FinalizationError("PXE finalizer context and handoff paths must be the generated private paths") - pending.lock.parent.mkdir(parents=True, exist_ok=True) - with _exclusive_lock(pending.lock): - context = _read_document(pending.context, "PXE finalizer context") - handoff = _read_document(pending.handoff, "PXE finalizer handoff") - spec, controller, rootfs_gid = _validate(context, handoff) - resolution = _final_resolution(controller, rootfs_gid) - policy = _controller_policy(resolution, True) - artifacts: list[Artifact] = [ - (pending.inventory, render_inventory(spec, context["token"], resolution), 0o600, True), - (pending.pxe_vars, render_pxe_vars(spec, policy), 0o600, True), - (pending.values, render_values(spec, resolution), 0o644, False), - (pending.manifest, _manifest(resolution, True, rootfs_gid), 0o644, False), - ] - completion = _completion(context, handoff, _artifact_attestations(artifacts)) - if os.path.lexists(pending.completion): - if _read_document(pending.completion, "PXE finalizer completion") != completion: - raise FinalizationError("PXE finalizer completion does not match the supplied handoff") - _verify_canonical_artifacts(pending, completion["artifacts"]) - return - published: list[Artifact] = [ - *artifacts, - (pending.completion, json.dumps(completion, sort_keys=True) + "\n", 0o600, True), - ] - preflight_destinations([path for path, _, _, _ in published], False) - publish_artifacts(published, False) - - -def _controller_policy(resolution: FleetResolution, rootfs_enabled: bool) -> ResolvedGpuPolicy: - return ResolvedGpuPolicy( - host_gpu_enabled={host.target.name: host.status is HostStatus.GPU for host in resolution.hosts}, - render_gid=resolution.render_gid, - pxe_gpu_enabled=rootfs_enabled, - ) - - -def _manifest(resolution: FleetResolution, rootfs_enabled: bool, rootfs_gid: int | None) -> str: - base = resolution_manifest(resolution) - document = build_pxe_resolution_manifest( - version=base["version"], - status=base["status"], - render_gid=base["render_gid"], - hosts=base["hosts"], - gpu_access_enabled=rootfs_enabled, - pxe_render_gid=rootfs_gid, - ) - return json.dumps(document, indent=2, sort_keys=True) + "\n" - - -def _render_bootstrap_inventory(spec: JsonDocument) -> str: - server = spec["server"] - return "\n".join( - [ - "pxe_controller:", - " hosts:", - f" {server['name']}:", - f" ansible_host: {server['ip']}", - " vars:", - " ansible_port: 22", - " ansible_user: root", - "", - ] - ) - - -def _yaml_quote(value: str) -> str: - return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"' diff --git a/skills/deploy-aup-learning-cloud/scripts/pxe_finalization_support.py b/skills/deploy-aup-learning-cloud/scripts/pxe_finalization_support.py deleted file mode 100644 index e94923f7..00000000 --- a/skills/deploy-aup-learning-cloud/scripts/pxe_finalization_support.py +++ /dev/null @@ -1,257 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. -"""Typed security and verification support for PXE finalization.""" - -from __future__ import annotations - -import fcntl -import hashlib -import json -import os -import stat -from collections.abc import Iterator -from contextlib import contextmanager -from dataclasses import dataclass -from pathlib import Path -from typing import Final, TypeAlias, TypedDict - -from config_common import DuplicateJsonKeyError, strict_json_loads -from gpu_access_resolution import FleetResolution, FleetStatus, HostResolution, HostStatus, InventoryTarget - -VERSION: Final = 1 -MAX_RENDER_GID: Final = 4_294_967_294 - -JsonScalar: TypeAlias = str | int | float | bool | None -JsonValue: TypeAlias = JsonScalar | list["JsonValue"] | dict[str, "JsonValue"] -JsonDocument: TypeAlias = dict[str, JsonValue] -Artifact: TypeAlias = tuple[Path, str, int, bool] - - -class ArtifactAttestation(TypedDict): - sha256: str - mode: int - owner_uid: int - - -ArtifactAttestations: TypeAlias = dict[str, ArtifactAttestation] - - -@dataclass(frozen=True, slots=True) -class FinalizationError(Exception): - reason: str - - def __str__(self) -> str: - return self.reason - - -@dataclass(frozen=True, slots=True) -class PxePaths: - bootstrap_inventory: Path - bootstrap_vars: Path - context: Path - handoff: Path - completion: Path - lock: Path - inventory: Path - pxe_vars: Path - values: Path - manifest: Path - - -def paths(out_dir: Path) -> PxePaths: - root = out_dir.resolve() - return PxePaths( - bootstrap_inventory=root / ".pxe-bootstrap.inventory.yml", - bootstrap_vars=root / ".pxe-bootstrap.vars.yml", - context=root / ".pxe-finalizer-context.json", - handoff=root / ".pxe-finalizer-handoff.json", - completion=root / ".pxe-finalizer-completion.json", - lock=root / ".pxe-finalizer.lock", - inventory=root / "inventory.yml", - pxe_vars=root / "pb-pxe-controller.vars.yml", - values=root / "values-basic-example.yaml", - manifest=root / "gpu-access-resolution.json", - ) - - -def spec_sha256(spec: JsonDocument) -> str: - encoded = json.dumps(spec, sort_keys=True, separators=(",", ":")).encode() - return hashlib.sha256(encoded).hexdigest() - - -def valid_gid(value: JsonValue) -> bool: - return type(value) is int and 1 <= value <= MAX_RENDER_GID - - -def read_document(path: Path, label: str) -> JsonDocument: - try: - descriptor = os.open(path, os.O_RDONLY | os.O_CLOEXEC | os.O_NOFOLLOW) - with os.fdopen(descriptor, encoding="utf-8") as source: - mode = os.fstat(source.fileno()).st_mode - if not stat.S_ISREG(mode): - raise FinalizationError(f"{label} must be a regular file") - document = strict_json_loads(source.read()) - except FinalizationError: - raise - except (DuplicateJsonKeyError, FileNotFoundError, OSError, ValueError, json.JSONDecodeError) as error: - raise FinalizationError(f"{label} cannot be read") from error - if type(document) is not dict: - raise FinalizationError(f"{label} must be a JSON object") - return document - - -def validate(context: JsonDocument, handoff: JsonDocument) -> tuple[JsonDocument, FleetResolution, int]: - required_context = {"version", "generation", "spec_sha256", "topology", "spec", "token", "controller"} - required_handoff = {"version", "generation", "spec_sha256", "topology", "pxe_gpu_access_enabled", "render_gid"} - if set(context) != required_context or set(handoff) != required_handoff: - raise FinalizationError("PXE finalizer context or handoff has an unexpected schema") - if type(context["version"]) is not int or type(handoff["version"]) is not int: - raise FinalizationError("PXE finalizer context or handoff version is invalid") - if context["version"] != VERSION or handoff["version"] != VERSION: - raise FinalizationError("PXE finalizer context or handoff version is unsupported") - if context["topology"] != "pxe-diskless" or handoff["topology"] != "pxe-diskless": - raise FinalizationError("PXE finalizer topology is invalid") - generation = context["generation"] - if type(generation) is not str or not generation or handoff["generation"] != generation: - raise FinalizationError("PXE finalizer generation does not match") - spec = context["spec"] - if type(spec) is not dict or spec_sha256(spec) != context["spec_sha256"]: - raise FinalizationError("PXE finalizer context spec does not match its digest") - if handoff["spec_sha256"] != context["spec_sha256"] or spec.get("topology") != "pxe-diskless": - raise FinalizationError("PXE finalizer handoff does not match its pending spec") - if "render_gid" in spec or "gpu_access" in spec: - raise FinalizationError("PXE finalizer context contains removed public GPU policy fields") - if type(context["token"]) is not str or not context["token"]: - raise FinalizationError("PXE finalizer context token is invalid") - pxe = spec.get("pxe") - if type(pxe) is not dict or pxe.get("diskless_agents_have_amd_gpus") is not True: - raise FinalizationError("PXE finalizer context is not for GPU-enabled diskless agents") - rootfs_gid = handoff["render_gid"] - if handoff["pxe_gpu_access_enabled"] is not True or not valid_gid(rootfs_gid): - raise FinalizationError("PXE finalizer handoff has no valid resolved rootfs GID") - controller = controller_resolution(spec, context["controller"]) - if controller.render_gid is not None and controller.render_gid != rootfs_gid: - raise FinalizationError("PXE rootfs render GID disagrees with the GPU-enabled controller") - return spec, controller, rootfs_gid - - -def controller_resolution(spec: JsonDocument, raw: JsonValue) -> FleetResolution: - if type(raw) is not dict or set(raw) != {"version", "status", "render_gid", "hosts"}: - raise FinalizationError("PXE finalizer context controller evidence is invalid") - server = spec.get("server") - name = server.get("name") if type(server) is dict else None - hosts = raw["hosts"] - if type(name) is not str or type(hosts) is not dict or set(hosts) != {name} or type(hosts[name]) is not bool: - raise FinalizationError("PXE finalizer context controller host is invalid") - enabled = hosts[name] - gid = raw["render_gid"] - if enabled and not valid_gid(gid): - raise FinalizationError("PXE finalizer context controller GID is invalid") - if not enabled and gid is not None: - raise FinalizationError("CPU-only PXE controller must not publish a render GID") - status = HostStatus.GPU if enabled else HostStatus.CPU - fleet_status = FleetStatus.GPU_RESOLVED if enabled else FleetStatus.CPU_ONLY - if type(raw["version"]) is not int or raw["version"] != VERSION or raw["status"] != fleet_status.value: - raise FinalizationError("PXE finalizer context controller status is invalid") - host = HostResolution(target=target(name), status=status, render_gid=gid, reason=None) - return FleetResolution(fleet_status, (host,), gid, None) - - -def target(name: str) -> InventoryTarget: - return InventoryTarget(name=name) - - -def final_resolution(controller: FleetResolution, rootfs_gid: int) -> FleetResolution: - return FleetResolution(FleetStatus.GPU_RESOLVED, controller.hosts, rootfs_gid, None) - - -def generation_paths(pending: PxePaths) -> tuple[Path, ...]: - return ( - pending.bootstrap_inventory, - pending.bootstrap_vars, - pending.context, - pending.handoff, - pending.completion, - pending.inventory, - pending.pxe_vars, - pending.values, - pending.manifest, - ) - - -def artifact_attestations(artifacts: list[Artifact]) -> ArtifactAttestations: - return { - path.name: {"sha256": hashlib.sha256(content.encode()).hexdigest(), "mode": mode, "owner_uid": os.geteuid()} - for path, content, mode, _ in artifacts - } - - -def verify_canonical_artifacts(pending: PxePaths, expected: JsonValue) -> None: - canonical = (pending.inventory, pending.pxe_vars, pending.values, pending.manifest) - if type(expected) is not dict or set(expected) != {path.name for path in canonical}: - raise FinalizationError("PXE finalizer completion artifacts are invalid") - for path in canonical: - attestation = expected[path.name] - if ( - type(attestation) is not dict - or set(attestation) != {"sha256", "mode", "owner_uid"} - or type(attestation["sha256"]) is not str - or type(attestation["mode"]) is not int - or type(attestation["owner_uid"]) is not int - or read_artifact_attestation(path) != attestation - ): - raise FinalizationError(f"PXE finalizer canonical artifact is missing or corrupted: {path.name}") - - -def read_artifact_attestation(path: Path) -> ArtifactAttestation: - try: - descriptor = os.open(path, os.O_RDONLY | os.O_CLOEXEC | os.O_NOFOLLOW) - with os.fdopen(descriptor, "rb") as source: - artifact_stat = os.fstat(source.fileno()) - if not stat.S_ISREG(artifact_stat.st_mode): - raise FinalizationError("PXE finalizer canonical artifact must be a regular file") - digest = hashlib.sha256() - while chunk := source.read(65_536): - digest.update(chunk) - return { - "sha256": digest.hexdigest(), - "mode": stat.S_IMODE(artifact_stat.st_mode), - "owner_uid": artifact_stat.st_uid, - } - except FinalizationError: - raise - except (FileNotFoundError, OSError) as error: - raise FinalizationError("PXE finalizer canonical artifact cannot be read") from error - - -def completion(context: JsonDocument, handoff: JsonDocument, artifacts: ArtifactAttestations) -> JsonDocument: - return { - **{ - key: handoff[key] - for key in ("version", "generation", "spec_sha256", "topology", "pxe_gpu_access_enabled", "render_gid") - }, - "artifacts": artifacts, - } - - -@contextmanager -def exclusive_lock(path: Path) -> Iterator[None]: - descriptor = -1 - locked = False - try: - descriptor = os.open(path, os.O_RDWR | os.O_CREAT | os.O_CLOEXEC | os.O_NOFOLLOW, 0o600) - lock_stat = os.fstat(descriptor) - if not stat.S_ISREG(lock_stat.st_mode): - raise FinalizationError("PXE finalizer lock must be a regular file") - if lock_stat.st_uid != os.geteuid() or stat.S_IMODE(lock_stat.st_mode) != 0o600: - raise FinalizationError("PXE finalizer lock has unsafe owner or mode") - fcntl.flock(descriptor, fcntl.LOCK_EX) - locked = True - yield - except OSError as error: - raise FinalizationError("PXE finalizer lock cannot be opened") from error - finally: - if locked: - fcntl.flock(descriptor, fcntl.LOCK_UN) - if descriptor >= 0: - os.close(descriptor) diff --git a/tests/skills/test_pxe_finalization.py b/tests/skills/test_pxe_finalization.py index effccb02..554d6049 100644 --- a/tests/skills/test_pxe_finalization.py +++ b/tests/skills/test_pxe_finalization.py @@ -1,18 +1,19 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +"""End-to-end contracts for immediate PXE GPU policy generation.""" + from __future__ import annotations import json import os import subprocess import sys -from dataclasses import FrozenInstanceError -from importlib.util import module_from_spec, spec_from_file_location from pathlib import Path import pytest ROOT = Path(__file__).resolve().parents[2] GEN_CONFIGS = ROOT / "skills" / "deploy-aup-learning-cloud" / "scripts" / "gen_configs.py" -PXE_PLAYBOOK = ROOT / "deploy" / "ansible" / "playbooks" / "pb-pxe-controller.yml" def pxe_spec(gpu_agents: bool) -> dict: @@ -49,16 +50,11 @@ def write_fake_ansible(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, *, contr output = next(value.split('=', 1)[1] for value in sys.argv if value.startswith('gpu_access_discovery_output_path=')) Path(output).write_text(json.dumps({{ - 'version': 2, + 'version': 1, 'hosts': [{{ 'host': 'controller', 'reachable': True, 'lspci': {{'rc': 0, 'stdout': {bdf!r}}}, 'sysfs': {{'rc': 0, 'stdout': {bdf!r}}}, - 'render_group': {{'rc': 0, 'stdout': 'render:x:993\\n'}}, - 'groups': {{'rc': 0, 'stdout': 'render:x:993\\n'}}, - 'state': {{'stat_success': True, 'content_success': True, 'exists': False, 'regular': False, 'symlink': False, 'content': ''}}, - 'rule': {{'stat_success': True, 'content_success': True, 'exists': False, 'regular': False, 'symlink': False, 'content': ''}}, - 'legacy_rules': {{key: {{'stat_success': True, 'content_success': True, 'exists': False, 'regular': False, 'symlink': False, 'content': ''}} for key in ('kfd', 'amdgpu', 'rocm_devices')}}, }}], }}), encoding='utf-8') """, @@ -78,439 +74,112 @@ def run_generator(*arguments: str) -> subprocess.CompletedProcess[str]: ) -def load_finalizer_module(): - scripts = GEN_CONFIGS.parent - sys.path.insert(0, str(scripts)) - try: - spec = spec_from_file_location("test_pxe_finalizer", scripts / "pxe_finalization.py") - assert spec is not None and spec.loader is not None - module = module_from_spec(spec) - sys.modules[spec.name] = module - spec.loader.exec_module(module) - return module - finally: - sys.path.pop(0) - - -def pending_handoff(out_dir: Path, *, render_gid: int = 995) -> tuple[Path, Path]: - context_path = out_dir / ".pxe-finalizer-context.json" - context = json.loads(context_path.read_text(encoding="utf-8")) - handoff_path = out_dir / ".pxe-finalizer-handoff.json" - write_json( - handoff_path, - { - "version": 1, - "generation": context["generation"], - "spec_sha256": context["spec_sha256"], - "topology": "pxe-diskless", - "pxe_gpu_access_enabled": True, - "render_gid": render_gid, - }, - ) - return context_path, handoff_path - - def canonical_artifacts(out_dir: Path) -> tuple[Path, ...]: return ( out_dir / "inventory.yml", out_dir / "pb-pxe-controller.vars.yml", out_dir / "values-basic-example.yaml", out_dir / "gpu-access-resolution.json", - out_dir / ".pxe-finalizer-completion.json", ) -def cpu_controller(finalizer): - return finalizer.FleetResolution( - finalizer.FleetStatus.CPU_ONLY, - (finalizer.HostResolution(finalizer._target("controller"), finalizer.HostStatus.CPU, None, None),), - None, - None, - ) - - -def test_pxe_finalizer_preserves_moved_imports_as_immutable_support_types(tmp_path: Path) -> None: - finalizer = load_finalizer_module() - support = sys.modules["pxe_finalization_support"] - - assert finalizer.FinalizationError is support.FinalizationError - assert finalizer.PxePaths is support.PxePaths - assert finalizer.paths is support.paths - assert finalizer.VERSION == support.VERSION == 1 - assert finalizer.MAX_RENDER_GID == support.MAX_RENDER_GID == 4_294_967_294 - assert finalizer._read_document is support.read_document - assert finalizer._generation_paths is support.generation_paths - assert finalizer._artifact_attestations is support.artifact_attestations - assert finalizer._completion is support.completion - assert finalizer._verify_canonical_artifacts is support.verify_canonical_artifacts - assert finalizer._exclusive_lock is support.exclusive_lock - - error = finalizer.FinalizationError("immutable") - pending = finalizer.paths(tmp_path) - with pytest.raises(FrozenInstanceError): - error.reason = "changed" - with pytest.raises(FrozenInstanceError): - pending.context = tmp_path / "changed.json" - - -def test_pxe_gpu_agents_stage_only_private_bootstrap_artifacts(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - write_fake_ansible(tmp_path, monkeypatch) - spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) - out_dir = tmp_path / "generated" - - result = run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)) - - assert result.returncode == 0, result.stderr - assert not (out_dir / "inventory.yml").exists() - assert not (out_dir / "values-basic-example.yaml").exists() - assert not (out_dir / "gpu-access-resolution.json").exists() - assert "pxe_controller:" in (out_dir / ".pxe-bootstrap.inventory.yml").read_text(encoding="utf-8") - bootstrap = (out_dir / ".pxe-bootstrap.vars.yml").read_text(encoding="utf-8") - assert "pxe_gpu_access_enabled: true" in bootstrap - assert "pxe_finalizer_context:" in bootstrap - assert (out_dir / ".pxe-finalizer-context.json").stat().st_mode & 0o777 == 0o600 - - -def test_pxe_cpu_agents_publish_a_disabled_rootfs_policy(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - write_fake_ansible(tmp_path, monkeypatch) - spec_path = write_json(tmp_path / "spec.json", pxe_spec(False)) - out_dir = tmp_path / "generated" - - result = run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)) - - assert result.returncode == 0, result.stderr - assert "pxe_gpu_access_enabled: false" in (out_dir / "pb-pxe-controller.vars.yml").read_text(encoding="utf-8") - assert "auplc_render_gid: null" in (out_dir / "inventory.yml").read_text(encoding="utf-8") - manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) - assert manifest["pxe_rootfs"] == {"gpu_access_enabled": False, "render_gid": None} - - -def test_pxe_disabled_rootfs_force_replaces_private_generation_state_under_the_generation_lock(tmp_path: Path) -> None: - finalizer = load_finalizer_module() - out_dir = tmp_path / "generated" - pending = finalizer.paths(out_dir) - for path in ( - pending.bootstrap_inventory, - pending.bootstrap_vars, - pending.context, - pending.handoff, - pending.completion, - ): - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text("stale\n", encoding="utf-8") - - finalizer.publish_disabled_rootfs(pxe_spec(False), "token", cpu_controller(finalizer), out_dir, True) - - assert all( - not path.exists() - for path in ( - pending.bootstrap_inventory, - pending.bootstrap_vars, - pending.context, - pending.handoff, - pending.completion, - ) - ) - assert all(path.exists() for path in canonical_artifacts(out_dir)[:-1]) - - -def test_pxe_disabled_rootfs_force_restores_private_and_canonical_generation_when_publication_fails( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - finalizer = load_finalizer_module() - artifact_store = sys.modules["artifact_store"] - out_dir = tmp_path / "generated" - pending = finalizer.paths(out_dir) - finalizer.publish_disabled_rootfs(pxe_spec(False), "old-token", cpu_controller(finalizer), out_dir, False) - for path in ( - pending.bootstrap_inventory, - pending.bootstrap_vars, - pending.context, - pending.handoff, - pending.completion, - ): - path.write_text(f"old {path.name}\n", encoding="utf-8") - tracked = ( - *canonical_artifacts(out_dir)[:-1], - pending.bootstrap_inventory, - pending.bootstrap_vars, - pending.context, - pending.handoff, - pending.completion, - ) - before = {path.name: path.read_bytes() for path in tracked} - original_replace = artifact_store.os.replace - - def fail_values_replace(source, destination): - if Path(destination) == pending.values and ".backup." not in str(source): - raise OSError("injected disabled-rootfs publication failure") - return original_replace(source, destination) - - monkeypatch.setattr(artifact_store.os, "replace", fail_values_replace) - with pytest.raises(SystemExit): - finalizer.publish_disabled_rootfs(pxe_spec(False), "new-token", cpu_controller(finalizer), out_dir, True) - - assert {path.name: path.read_bytes() for path in tracked} == before - - -def test_pxe_finalizer_publishes_resolved_policy_idempotently_without_secret_output( +def test_pxe_gpu_agents_publish_immediate_boolean_only_rootfs_artifacts( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: write_fake_ansible(tmp_path, monkeypatch) - spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) out_dir = tmp_path / "generated" - pending = run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)) - context, handoff = pending_handoff(out_dir) - first = run_generator( - "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) - ) - second = run_generator( - "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) - ) + result = run_generator("--spec", str(write_json(tmp_path / "spec.json", pxe_spec(True))), "--out-dir", str(out_dir)) - assert pending.returncode == 0, pending.stderr - assert first.returncode == 0, first.stderr - assert second.returncode == 0, second.stderr - assert "do-not-print-this-secret" not in first.stdout + first.stderr + second.stdout + second.stderr - assert "auplc_render_gid: 995" in (out_dir / "inventory.yml").read_text(encoding="utf-8") - assert "auplc_gpu_access_enabled: false" in (out_dir / "inventory.yml").read_text(encoding="utf-8") - assert "renderGid: 995" in (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") + assert result.returncode == 0, result.stderr + inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") + values = (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) - assert manifest["pxe_rootfs"] == {"gpu_access_enabled": True, "render_gid": 995} - completion = json.loads((out_dir / ".pxe-finalizer-completion.json").read_text(encoding="utf-8")) - assert completion["artifacts"]["inventory.yml"]["mode"] == 0o600 - assert completion["artifacts"]["inventory.yml"]["owner_uid"] == os.geteuid() + pxe_vars = (out_dir / "pb-pxe-controller.vars.yml").read_text(encoding="utf-8") + assert "auplc_render_gid" not in inventory + assert "gpuAccess" not in values + assert "pxe_gpu_access_enabled: true" in pxe_vars + assert manifest == { + "version": 1, + "status": "cpu_only", + "hosts": {"controller": False}, + "pxe_rootfs": {"gpu_access_enabled": True}, + } + assert not list(out_dir.glob(".pxe-finalizer-*")) + assert "do-not-print-this-secret" not in result.stdout + result.stderr -def test_pxe_finalizer_retry_rejects_canonical_mode_drift(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +def test_pxe_cpu_agents_publish_a_disabled_rootfs_policy(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: write_fake_ansible(tmp_path, monkeypatch) - spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) out_dir = tmp_path / "generated" - assert run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)).returncode == 0 - context, handoff = pending_handoff(out_dir) - assert ( - run_generator( - "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) - ).returncode - == 0 - ) - (out_dir / "inventory.yml").chmod(0o644) result = run_generator( - "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) + "--spec", str(write_json(tmp_path / "spec.json", pxe_spec(False))), "--out-dir", str(out_dir) ) - assert result.returncode == 1 - - -def test_pxe_pending_generation_rejects_existing_private_or_canonical_state_without_force( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - write_fake_ansible(tmp_path, monkeypatch) - spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) - out_dir = tmp_path / "generated" - assert run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)).returncode == 0 - - result = run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)) - - assert result.returncode == 1 - assert "refusing to overwrite" in result.stderr + assert result.returncode == 0, result.stderr + pxe_vars = (out_dir / "pb-pxe-controller.vars.yml").read_text(encoding="utf-8") + manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) + assert "pxe_gpu_access_enabled: false" in pxe_vars + assert "auplc_render_gid" not in pxe_vars + assert manifest["pxe_rootfs"] == {"gpu_access_enabled": False} -def test_pxe_forced_pending_generation_hides_prior_public_and_private_generation( +def test_pxe_gpu_controller_and_rootfs_publish_independent_booleans( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - write_fake_ansible(tmp_path, monkeypatch) - spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) + write_fake_ansible(tmp_path, monkeypatch, controller_gpu=True) out_dir = tmp_path / "generated" - assert run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)).returncode == 0 - context, handoff = pending_handoff(out_dir) - assert ( - run_generator( - "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) - ).returncode - == 0 - ) - old_generation = json.loads(context.read_text(encoding="utf-8"))["generation"] - result = run_generator("--spec", str(spec_path), "--out-dir", str(out_dir), "--force") + result = run_generator("--spec", str(write_json(tmp_path / "spec.json", pxe_spec(True))), "--out-dir", str(out_dir)) assert result.returncode == 0, result.stderr - assert json.loads(context.read_text(encoding="utf-8"))["generation"] != old_generation - assert not handoff.exists() - assert all(not path.exists() for path in canonical_artifacts(out_dir)) + inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") + manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) + assert "auplc_gpu_access_enabled: true" in inventory + assert manifest["status"] == "gpu_resolved" + assert manifest["pxe_rootfs"] == {"gpu_access_enabled": True} -def test_pxe_forced_pending_generation_restores_prior_generation_if_staging_fails( +def test_pxe_generator_refuses_existing_canonical_artifacts_without_force( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: write_fake_ansible(tmp_path, monkeypatch) - spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) out_dir = tmp_path / "generated" - assert run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)).returncode == 0 - context, handoff = pending_handoff(out_dir) - assert ( - run_generator( - "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) - ).returncode - == 0 - ) - previous = {path.name: path.read_bytes() for path in (*canonical_artifacts(out_dir), context, handoff)} - finalizer = load_finalizer_module() - artifact_store = sys.modules["artifact_store"] - original_replace = artifact_store.os.replace - - def fail_new_bootstrap(source, destination): - if Path(destination) == out_dir / ".pxe-bootstrap.inventory.yml" and ".backup." not in str(source): - raise OSError("injected staging failure") - return original_replace(source, destination) - - monkeypatch.setattr(artifact_store.os, "replace", fail_new_bootstrap) - controller = finalizer._controller_resolution( - pxe_spec(True), json.loads(context.read_text(encoding="utf-8"))["controller"] - ) - with pytest.raises(SystemExit): - finalizer.stage_pending(pxe_spec(True), "replacement-token", controller, out_dir, True) - - assert {path.name: path.read_bytes() for path in (*canonical_artifacts(out_dir), context, handoff)} == previous + out_dir.mkdir() + existing = out_dir / "values-basic-example.yaml" + existing.write_text("existing\n", encoding="utf-8") - -@pytest.mark.parametrize("document_name", (".pxe-finalizer-context.json", ".pxe-finalizer-handoff.json")) -def test_pxe_finalizer_rejects_duplicate_keys_in_private_documents( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, document_name: str -) -> None: - write_fake_ansible(tmp_path, monkeypatch) - spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) - out_dir = tmp_path / "generated" - assert run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)).returncode == 0 - context, handoff = pending_handoff(out_dir) - document_path = out_dir / document_name - document_path.write_text( - '{"generation":"duplicate",' + document_path.read_text(encoding="utf-8")[1:], encoding="utf-8" - ) - - result = run_generator( - "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) - ) - - assert result.returncode == 1 - assert all(not path.exists() for path in canonical_artifacts(out_dir)) - - -@pytest.mark.parametrize("mutation", ("missing", "tampered")) -def test_pxe_finalizer_retry_rejects_missing_or_tampered_canonical_artifacts( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, mutation: str -) -> None: - write_fake_ansible(tmp_path, monkeypatch) - spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) - out_dir = tmp_path / "generated" - assert run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)).returncode == 0 - context, handoff = pending_handoff(out_dir) - assert ( - run_generator( - "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) - ).returncode - == 0 - ) - inventory = out_dir / "inventory.yml" - if mutation == "missing": - inventory.unlink() - else: - inventory.write_text("tampered\n", encoding="utf-8") - - result = run_generator( - "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) - ) + result = run_generator("--spec", str(write_json(tmp_path / "spec.json", pxe_spec(True))), "--out-dir", str(out_dir)) assert result.returncode == 1 + assert "refusing to overwrite existing" in result.stderr + assert existing.read_text(encoding="utf-8") == "existing\n" + assert all(not path.exists() for path in canonical_artifacts(out_dir) if path != existing) -def test_pxe_finalizer_rejects_symlink_lock_without_touching_its_target( +def test_pxe_generator_does_not_publish_when_controller_discovery_is_unknown( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - write_fake_ansible(tmp_path, monkeypatch) - spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) - out_dir = tmp_path / "generated" - assert run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)).returncode == 0 - context, handoff = pending_handoff(out_dir) - target = tmp_path / "lock-target" - target.write_text("unchanged\n", encoding="utf-8") - target.chmod(0o644) - lock = out_dir / ".pxe-finalizer.lock" - lock.unlink() - lock.symlink_to(target) + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + fake_ansible = fake_bin / "ansible-playbook" + fake_ansible.write_text( + """#!/usr/bin/env python3 +import json +import sys +from pathlib import Path - result = run_generator( - "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) +output = next(value.split('=', 1)[1] for value in sys.argv if value.startswith('gpu_access_discovery_output_path=')) +Path(output).write_text(json.dumps({'version': 1, 'hosts': [{'host': 'controller', 'reachable': False, 'lspci': {'rc': 0, 'stdout': ''}, 'sysfs': {'rc': 0, 'stdout': ''}}]}), encoding='utf-8') +""", + encoding="utf-8", ) - - assert result.returncode == 1 - assert target.read_text(encoding="utf-8") == "unchanged\n" - assert target.stat().st_mode & 0o777 == 0o644 - - -@pytest.mark.parametrize( - ("field", "value"), - [("generation", "stale"), ("topology", "ssh-preinstalled"), ("render_gid", None), ("version", True)], -) -def test_pxe_finalizer_rejects_invalid_handoffs_without_publishing( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, field: str, value: str | int | None -) -> None: - write_fake_ansible(tmp_path, monkeypatch) - spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) + fake_ansible.chmod(0o755) + monkeypatch.setenv("PATH", f"{fake_bin}:{os.environ['PATH']}") out_dir = tmp_path / "generated" - assert run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)).returncode == 0 - context, handoff = pending_handoff(out_dir) - document = json.loads(handoff.read_text(encoding="utf-8")) - document[field] = value - write_json(handoff, document) - result = run_generator( - "--finalize-pxe", "--out-dir", str(out_dir), "--context", str(context), "--handoff", str(handoff) - ) + result = run_generator("--spec", str(write_json(tmp_path / "spec.json", pxe_spec(True))), "--out-dir", str(out_dir)) assert result.returncode == 1 - assert not (out_dir / "inventory.yml").exists() - assert not (out_dir / "values-basic-example.yaml").exists() - assert not (out_dir / "gpu-access-resolution.json").exists() - - -def test_pxe_finalizer_rolls_back_if_late_canonical_publication_fails( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - write_fake_ansible(tmp_path, monkeypatch) - spec_path = write_json(tmp_path / "spec.json", pxe_spec(True)) - out_dir = tmp_path / "generated" - assert run_generator("--spec", str(spec_path), "--out-dir", str(out_dir)).returncode == 0 - context, handoff = pending_handoff(out_dir) - finalizer = load_finalizer_module() - artifact_store = sys.modules["artifact_store"] - original_link = artifact_store.os.link - - def fail_values_link(source, destination): - if Path(destination).name == "values-basic-example.yaml": - raise OSError("injected publication failure") - return original_link(source, destination) - - monkeypatch.setattr(artifact_store.os, "link", fail_values_link) - with pytest.raises(SystemExit): - finalizer.finalize(out_dir, context, handoff) - - assert not (out_dir / "inventory.yml").exists() - assert not (out_dir / "pb-pxe-controller.vars.yml").exists() - assert not (out_dir / "values-basic-example.yaml").exists() - assert not (out_dir / "gpu-access-resolution.json").exists() - assert not (out_dir / ".pxe-finalizer-completion.json").exists() - - -def test_pxe_playbook_writes_and_finalizes_private_rootfs_handoff_locally() -> None: - playbook = PXE_PLAYBOOK.read_text(encoding="utf-8") - - assert "pxe_finalizer_handoff" in playbook - assert "pxe_finalizer_context" in playbook - assert "--finalize-pxe" in playbook - assert "delegate_to: localhost" in playbook - assert "run_once: true" in playbook - assert "become: false" in playbook - assert "argv:" in playbook + assert all(not path.exists() for path in canonical_artifacts(out_dir)) From 0d095733722b2020876b47ff4a17c10b74cbc3ae Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:31:17 +0800 Subject: [PATCH 35/65] docs: update installer stages --- README.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index efcc2ae7..f19aa4b8 100644 --- a/README.md +++ b/README.md @@ -88,14 +88,15 @@ A successful install looks like this: ```text This operation needs root privileges. Requesting sudo password... - ✓ [1/8] Detecting GPU (0.2s) - ✓ [2/8] Generating values overlay (initial) (0.0s) - ✓ [3/8] Installing helm + k9s (0.0s) - ✓ [4/8] Installing K3s (single-node) (3.8s) - ✓ [5/8] Pulling custom + external images (25.0s) - ✓ [6/8] Deploying ROCm GPU device plugin + node labeller (0.2s) - ✓ [7/8] Refreshing values overlay from node labels (0.2s) - ✓ [8/8] Deploying JupyterHub runtime (helm install + wait) (9.2s) + ✓ [1/9] Detecting GPU (0.2s) + ✓ [2/9] Provisioning GPU device access (0.1s) + ✓ [3/9] Generating values overlay (initial) (0.0s) + ✓ [4/9] Installing helm + k9s (0.0s) + ✓ [5/9] Installing K3s (single-node) (3.8s) + ✓ [6/9] Pulling custom + external images (25.0s) + ✓ [7/9] Deploying ROCm GPU device plugin + node labeller (0.2s) + ✓ [8/9] Refreshing values overlay from node labels (0.2s) + ✓ [9/9] Deploying JupyterHub runtime (helm install + wait) (9.2s) _ _ _ ____ _ _ ____ _ _ / \ | | | | _ \ | | ___ __ _ _ __ _ __ (_)_ __ __ _ / ___| | ___ _ _ __| | From d9d973daf7f92a2c7654857a9c55c2ce24a4882b Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:31:17 +0800 Subject: [PATCH 36/65] docs(deploy): document GPU infrastructure contract --- deploy/README.md | 100 ++++++++++++++++++++++++++--------------------- 1 file changed, 55 insertions(+), 45 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index d2a1a72f..23703560 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -53,12 +53,16 @@ sudo ./auplc-installer install ### Multi-Node Cluster -Generate the spec, fill in the normal network and node details, then let the -generator discover GPU hosts and their shared `render` GID. The SSH flow asks -for no GPU host list and no GID. A PXE spec asks one extra GPU question: +Generate the spec and fill in the network and node details. The SSH flow needs +only the managed host details. A PXE spec asks one extra GPU question: `pxe.diskless_agents_have_amd_gpus`. Set it explicitly because the diskless agents' hardware is not inferred from the controller. +The AMD device plugin and ROCm node labeller are cluster infrastructure +prerequisites owned outside AUPLC. The infrastructure owner must deploy and +maintain them according to AMD's official guidance. Before Helm, verify that the +existing DaemonSets are ready and that GPU capacity is advertised. + #### SSH-preinstalled ```bash @@ -82,6 +86,10 @@ sudo ansible-playbook -i inventory.yml playbooks/pb-base.yml sudo ansible-playbook -i inventory.yml playbooks/pb-k3s-site.yml sudo ansible-playbook -i inventory.yml playbooks/pb-rocm.yml +kubectl rollout status -n kube-system daemonset/amdgpu-device-plugin-daemonset --timeout=5m +kubectl rollout status -n kube-system daemonset/amdgpu-labeller-daemonset --timeout=5m +kubectl get nodes -o 'custom-columns=NAME:.metadata.name,AMD_GPU:.status.allocatable.amd\.com/gpu' + cd "$REPO_ROOT" helm upgrade --install jupyterhub ./runtime/chart \ --namespace jupyterhub --create-namespace \ @@ -92,34 +100,35 @@ helm upgrade --install jupyterhub ./runtime/chart \ Generation runs read-only Ansible discovery against every managed host. It cross-checks AMD display BDFs from `lspci` with PCI vendor and display-class records under `/sys/bus/pci/devices`; it does not require the devices to be -attached to `amdgpu` before ROCm installation. It checks -the `render` group and existing GPU access files, and publishes only when every -GPU host agrees on one GID. CPU-only fleets publish `null` for the generated -inventory and Helm render GID. GPU policy details in generated files are -internal outputs, not fields to maintain by hand. +attached to `amdgpu` before ROCm installation. The resulting GPU resolution +report records which managed hosts have AMD display hardware. + +The GPU permission contract is fixed across GPU hosts and PXE root filesystems: -Configure notebook storage ownership with `singleuser.fsGid: 100`. Never set -storage `fsGroup` through `extraPodConfig.securityContext`, because that Pod -security-context override can replace the GPU resource's generated -`supplementalGroups`. +- `/dev/kfd` and AMD `/dev/dri/renderD*` nodes are `root:render` with mode `0666`. +- AMD `/dev/dri/card*` nodes are `root:video` with mode `0666`. +- Every GPU device node injected into a Pod therefore has mode `0666`. +- Host provisioning owns device-node discretionary access control. +- AUPLC Hub adds no GPU supplemental group to user Pods. +- AMD device-plugin allocation is the visibility boundary: only Pods that + request `amd.com/gpu` receive GPU device nodes. The plugin does not set Unix + ownership or modes on host device nodes. + +`singleuser.fsGid: 100` controls shared notebook storage ownership only. It is +not part of GPU access and must not be treated as a GPU group setting. #### PXE-diskless After setting `topology` to `pxe-diskless`, fill the PXE network fields and set -only `pxe.diskless_agents_have_amd_gpus` for GPU policy. When it is `true`, the -first generation is pending and creates private bootstrap files instead of -canonical deployment files. +`pxe.diskless_agents_have_amd_gpus` explicitly. Generation writes the canonical +inventory, controller vars, runtime overlay, and GPU resolution report directly. +These artifacts express the desired deployment inputs; their existence is not +proof that rootfs provisioning succeeded. Review and install them before running +the controller playbook, whose successful completion provisions the rootfs. ```bash cd "$REPO_ROOT" python3 "$DEPLOY_SCRIPTS/gen_configs.py" --spec spec.json --out-dir "$GENERATED_DIR" -cd "$REPO_ROOT/deploy/ansible" -sudo ansible-playbook \ - -i "$GENERATED_DIR/.pxe-bootstrap.inventory.yml" \ - playbooks/pb-pxe-controller.yml \ - -e @"$GENERATED_DIR/.pxe-bootstrap.vars.yml" - -# pb-pxe-controller finalizes automatically after a successful rootfs build. install -m 0600 "$GENERATED_DIR/inventory.yml" "$REPO_ROOT/deploy/ansible/inventory.yml" install -m 0644 "$GENERATED_DIR/values-basic-example.yaml" "$REPO_ROOT/runtime/values-basic-example.yaml" python3 "$DEPLOY_SCRIPTS/validate.py" --repo "$REPO_ROOT" --topology pxe-diskless \ @@ -128,40 +137,41 @@ python3 "$DEPLOY_SCRIPTS/validate.py" --repo "$REPO_ROOT" --topology pxe-diskles --values "$REPO_ROOT/runtime/values.yaml" \ --values "$REPO_ROOT/runtime/values-basic-example.yaml" \ --pxe-vars "$GENERATED_DIR/pb-pxe-controller.vars.yml" -``` -Don't invoke the hidden finalizer yourself. The playbook writes a private -handoff and runs finalization locally. `inventory.yml`, -`pb-pxe-controller.vars.yml`, `values-basic-example.yaml`, and -`gpu-access-resolution.json` appear only after success. +cd "$REPO_ROOT/deploy/ansible" +sudo ansible-playbook \ + -i "$GENERATED_DIR/inventory.yml" \ + playbooks/pb-pxe-controller.yml \ + -e @"$GENERATED_DIR/pb-pxe-controller.vars.yml" + +kubectl rollout status -n kube-system daemonset/amdgpu-device-plugin-daemonset --timeout=5m +kubectl rollout status -n kube-system daemonset/amdgpu-labeller-daemonset --timeout=5m +kubectl get nodes -o 'custom-columns=NAME:.metadata.name,AMD_GPU:.status.allocatable.amd\.com/gpu' + +cd "$REPO_ROOT" +helm upgrade --install jupyterhub ./runtime/chart \ + --namespace jupyterhub --create-namespace \ + -f runtime/values.yaml \ + -f runtime/values-basic-example.yaml +``` -A fresh PXE rootfs can create a missing `render` group and align it with a -unanimous live controller GPU GID after collision checks. A retained rootfs is -never silently changed. It must already contain one valid `render` group and, -when the controller has a resolved GPU GID, the rootfs GID must match. Rebuild -the rootfs or migrate the retained rootfs separately if it doesn't match. -Offline checks don't replace post-boot verification of GPU device ownership, -mode, supplemental groups, and workload access. +A fresh PXE rootfs receives the fixed udev rule during the controller playbook. +A retained rootfs is accepted only when it already contains that exact canonical +rule and no conflicting legacy GPU rule. Rebuild or correct a retained rootfs +separately if that safety check fails. -#### Discovery failures and migration +#### Discovery failures | Error | Action | | --- | --- | | Host is unreachable | Restore passwordless root SSH to that inventory host, then regenerate. | | `lspci` is missing or fails | Install `pciutils` on the reported host and rerun generation. | | Host evidence is `UNKNOWN` or AMD GPU BDF probes disagree | Compare AMD display BDFs from `lspci` with vendor `0x1002` display-class devices under `/sys/bus/pci/devices`; fix missing or inconsistent PCI enumeration, then regenerate. | -| GPU host has no valid `render` group | Install the correct GPU userspace or create one valid system `render` group, then regenerate. | -| GPU render GIDs disagree | Plan and perform a reviewed group migration so every GPU host uses one free GID, then regenerate. | -| CPU host retains GPU access contract, or canonical state/rule conflicts | Inspect `/var/lib/auplc/gpu-access.json` and `/etc/udev/rules.d/70-auplc-gpu-access.rules`. Remove stale project-owned files from a truly CPU-only host, or complete the GPU migration. Never overwrite unknown content. | -| Retained PXE rootfs GID differs from the unanimous live GID | Rebuild the rootfs, or migrate that retained rootfs separately before rerunning the playbook. | - -Old unshipped specs aren't compatible. Remove the former manual GPU policy -fields, regenerate the schema, copy the ordinary node and PXE network values -into it, and set only `pxe.diskless_agents_have_amd_gpus` on PXE deployments. +| Retained PXE rootfs has a legacy or non-canonical GPU rule | Rebuild the rootfs, or replace the conflicting rule through a separate reviewed maintenance action before rerunning the playbook. | ## Deployment branch boundary This branch and these instructions do not modify or roll out any live deployment. SHC, FET, and other deployment branches or environments must -backport the automatic discovery and generated-artifact changes before their -own reviewed rollout. +backport the host permission and immediate artifact publication changes before +their own reviewed rollout. From e74c40d1d26efcc4b0c4b6196772a623a5a5844c Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:31:17 +0800 Subject: [PATCH 37/65] docs(ansible): document GID-free policy --- deploy/ansible/README.md | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/deploy/ansible/README.md b/deploy/ansible/README.md index 108dad42..d698183b 100644 --- a/deploy/ansible/README.md +++ b/deploy/ansible/README.md @@ -22,16 +22,24 @@ SOFTWARE. # Ansible Playbooks -K3s cluster setup playbooks based on [k3s-ansible](https://github.com/k3s-io/k3s-ansible/tree/master). +K3s cluster setup playbooks based on [k3s-ansible](https://github.com/k3s-io/k3s-ansible). For the generator, canonical inventory, validator arguments, and topology-specific playbook commands, see the authoritative [deployment guide](../README.md). Don't write GPU policy into the inventory by hand. SSH generation discovers GPU -hosts and their shared `render` group ID. PXE generation uses only -`pxe.diskless_agents_have_amd_gpus`; when enabled, the controller playbook uses -private bootstrap inputs and publishes canonical files automatically after a -successful rootfs build. +hosts from managed-host evidence. PXE generation uses only +`pxe.diskless_agents_have_amd_gpus` and writes canonical files before the +controller playbook runs. + +The GPU access role sets AMD device-node policy on GPU hosts and GPU-enabled PXE +root filesystems. `/dev/kfd` and AMD `renderD*` nodes are `root:render 0666`; +AMD `card*` nodes are `root:video 0666`. All injected GPU device nodes therefore +use mode `0666`. Device-plugin allocation is the visibility boundary: only Pods +requesting `amd.com/gpu` receive the nodes. The plugin does not change host inode +permissions, and AUPLC Hub adds no GPU supplemental group to user Pods. Ordinary +container group membership does not participate in GPU permissions; host +provisioning owns device-node discretionary access control. ## Prerequisites @@ -39,3 +47,6 @@ successful rootfs build. - **Python**: 3.12 - **SSH**: Root login with key-based auth to all nodes - **Hosts**: Consistent `/etc/hosts` entries across all nodes +- **GPU integration**: The infrastructure owner must deploy and maintain the AMD + device plugin and ROCm node labeller outside AUPLC. Before Helm, run the + readiness and capacity checks in the [deployment guide](../README.md). From a87aba0c0c49b9285ecbdd160dce4c2c8e4d9c93 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:31:17 +0800 Subject: [PATCH 38/65] docs(k8s): require external GPU device management --- deploy/k8s/README.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/deploy/k8s/README.md b/deploy/k8s/README.md index be3c471c..ce25b6c2 100644 --- a/deploy/k8s/README.md +++ b/deploy/k8s/README.md @@ -40,17 +40,18 @@ which `runtime/values.yaml` uses as `nodeSelector`s. The installer pins the accelerator `nodeSelector` to the real `amd.com/gpu.product-name` detected on the host, so no manual labelling is needed on single-machine deployments. -If you are deploying manually instead: +For multi-node deployments, the AMD device plugin and ROCm node labeller are +cluster infrastructure prerequisites owned outside AUPLC. The infrastructure +owner must select, deploy, and maintain them according to the +[official AMD Kubernetes device plugin project](https://github.com/ROCm/k8s-device-plugin). +AUPLC documentation does not install these privileged components. -```bash -# Deploy AMD GPU device plugin -kubectl create -f https://raw.githubusercontent.com/ROCm/k8s-device-plugin/master/k8s-ds-amdgpu-dp.yaml - -# Deploy AMD GPU node labeller (publishes amd.com/gpu.* labels) -kubectl create -f https://raw.githubusercontent.com/ROCm/k8s-device-plugin/master/k8s-ds-amdgpu-labeller.yaml +Before deploying the AUPLC Helm release, verify the existing infrastructure: -# Verify GPU detection and labels -kubectl describe node | grep amd.com/gpu +```bash +kubectl rollout status -n kube-system daemonset/amdgpu-device-plugin-daemonset --timeout=5m +kubectl rollout status -n kube-system daemonset/amdgpu-labeller-daemonset --timeout=5m +kubectl get nodes -o 'custom-columns=NAME:.metadata.name,AMD_GPU:.status.allocatable.amd\.com/gpu' ``` `runtime/values-multi-nodes.yaml.example` now follows `runtime/values.yaml` and From 8c3ba1f50007fc48953b676b7e4794a4ce132cdb Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:31:17 +0800 Subject: [PATCH 39/65] docs(skills): align GPU deployment workflow --- skills/deploy-aup-learning-cloud/SKILL.md | 32 +++++++++++++------ skills/deploy-aup-learning-cloud/reference.md | 26 ++++++++++++--- .../scripts/README.md | 13 +++++--- 3 files changed, 52 insertions(+), 19 deletions(-) diff --git a/skills/deploy-aup-learning-cloud/SKILL.md b/skills/deploy-aup-learning-cloud/SKILL.md index 2fc9725a..1799c447 100644 --- a/skills/deploy-aup-learning-cloud/SKILL.md +++ b/skills/deploy-aup-learning-cloud/SKILL.md @@ -41,8 +41,8 @@ Then collect and confirm: 1. Courses and notebook resources. 2. Controller hostname, static IP, subnet, gateway, and DNS. -3. For SSH, every managed hostname and IP. Don't ask for a GPU host list or a - shared GPU group ID. Generation discovers both over SSH. +3. For SSH, every managed hostname and IP. Don't ask for a GPU host list; + generation discovers GPU hosts over SSH. 4. For PXE, the controller NIC, web port, rootfs SSH public key, and whether diskless agents have AMD GPUs. This explicit yes or no is the sole PXE GPU policy input because agent hardware can't be inferred from the controller. @@ -57,16 +57,16 @@ Create a fresh schema and fill only its current fields. Run the generator rather than writing inventory or GPU policy by hand. For SSH, generation performs read-only discovery on every managed host and -publishes canonical artifacts only after GPU evidence and group IDs agree. +publishes canonical artifacts after GPU evidence is consistent. -For PXE with GPU agents, initial generation creates private bootstrap inventory -and vars. Run the PXE controller playbook with those private files. A successful -rootfs build finalizes generation automatically and publishes the canonical -inventory, PXE vars, runtime overlay, and GPU resolution report. +For PXE, generation writes the canonical inventory, PXE vars, runtime overlay, +and GPU resolution report directly as desired deployment inputs. Their existence +does not prove rootfs provisioning succeeded. Review, install, and validate those +files, then run the controller playbook with the canonical inventory and PXE +vars; the playbook must complete successfully before proceeding. Follow the exact generation, installation, and playbook commands in -[deploy/README.md](../../deploy/README.md). Don't invent a separate completion -step. +[deploy/README.md](../../deploy/README.md). ## Phase 3: Validate and execute @@ -82,7 +82,19 @@ then run the validator with the arguments shown in the deployment guide: Stop on validation failure. After a clean result, follow the topology's Ansible, storage, device plugin, and Helm sequence in -[deploy/README.md](../../deploy/README.md). +[deploy/README.md](../../deploy/README.md). Treat the AMD device plugin and ROCm +node labeller as infrastructure prerequisites owned outside AUPLC. Verify both +existing DaemonSets and advertised GPU capacity before Helm; do not install +these privileged components as part of the AUPLC procedure. + +Keep the GPU contract distinct from storage configuration. GPU hosts use +`root:render 0666` for `/dev/kfd` and AMD `renderD*`, and `root:video 0666` for +AMD `card*`, so every injected GPU device node has mode `0666`. Device-plugin +allocation is the visibility boundary and only `amd.com/gpu` requests receive +GPU nodes. Container group membership does not participate in GPU permissions; +host provisioning owns device-node discretionary access control. AUPLC Hub adds +no GPU supplemental group, and the plugin does not change Unix inode permissions. +`singleuser.fsGid: 100` is for shared storage only. ## Phase 4: Verify diff --git a/skills/deploy-aup-learning-cloud/reference.md b/skills/deploy-aup-learning-cloud/reference.md index 4aafa0fa..ff7d5bd5 100644 --- a/skills/deploy-aup-learning-cloud/reference.md +++ b/skills/deploy-aup-learning-cloud/reference.md @@ -8,11 +8,11 @@ commands into this reference. | Topology | Generator behavior | | --- | --- | -| `ssh-preinstalled` | Connects to every managed host, discovers GPU hosts and their shared `render` group ID, and publishes canonical files only when discovery is consistent. | -| `pxe-diskless` | Uses `pxe.diskless_agents_have_amd_gpus` as its sole GPU policy input. GPU-enabled first generation emits private bootstrap files; the PXE controller playbook finalizes canonical files after a successful rootfs build. | +| `ssh-preinstalled` | Connects to every managed host, discovers GPU hardware, and publishes canonical files when discovery is consistent. | +| `pxe-diskless` | Uses `pxe.diskless_agents_have_amd_gpus` as its sole GPU policy input and publishes canonical desired-input files before the controller playbook runs. Their existence does not prove rootfs provisioning succeeded. | -Don't hand-author generated GPU policy. Old unshipped specs should be recreated -from the current `--print-schema` output. +Don't hand-author generated GPU policy. Create deployment specs from the current +`--print-schema` output. ## Canonical validation inputs @@ -28,6 +28,24 @@ passes: Generation and validation must finish before Ansible or Helm changes are made. +## GPU permission contract + +- Host `/dev/kfd` and AMD `/dev/dri/renderD*` nodes are `root:render 0666`. +- Host AMD `/dev/dri/card*` nodes are `root:video 0666`. +- Every GPU device node injected into a Pod has mode `0666`. +- Container group membership does not participate in GPU permissions; host + provisioning owns device-node discretionary access control. +- AUPLC Hub adds no GPU supplemental group to user Pods. +- AMD device-plugin allocation is the visibility boundary. Only Pods requesting + `amd.com/gpu` receive GPU device nodes; the plugin does not change host inode + ownership or mode. +- `singleuser.fsGid: 100` controls shared storage ownership only. + +The infrastructure owner deploys and maintains the AMD device plugin and ROCm +node labeller outside AUPLC. Before Helm, use the readiness and capacity checks +in [deploy/README.md](../../deploy/README.md); do not install these privileged +components as part of the AUPLC procedure. + ## Operator gates Keep the topology choice explicit. Confirm network, node, storage, course, and diff --git a/skills/deploy-aup-learning-cloud/scripts/README.md b/skills/deploy-aup-learning-cloud/scripts/README.md index 65bd71c1..db9fbfef 100644 --- a/skills/deploy-aup-learning-cloud/scripts/README.md +++ b/skills/deploy-aup-learning-cloud/scripts/README.md @@ -8,17 +8,20 @@ sequence and argument paths. | --- | --- | | `detect_hardware.sh` | Reports controller network details and local AMD PCI devices as JSON. | | `detect_cluster.sh` | Reports Kubernetes nodes, AMD GPU labels, storage classes, and GPU DaemonSet state as JSON. | -| `gen_configs.py` | Prints the current spec schema, discovers SSH GPU state, and generates topology-specific deployment artifacts. PXE GPU bootstrap files remain private until the controller playbook finalizes them automatically. | +| `gen_configs.py` | Prints the current spec schema, discovers live GPU state, and directly publishes canonical topology-specific deployment artifacts. | | `validate.py` | Checks the selected topology against canonical inventory, GPU resolution, values overlays, and PXE vars when applicable. | ## Generator contract -The SSH topology discovers GPU hosts and their shared `render` group ID. Users -don't provide either value. The PXE topology has one GPU policy input: +The SSH topology discovers GPU hosts from managed-host evidence. Users don't +provide a GPU host list. The PXE topology has one GPU policy input: `pxe.diskless_agents_have_amd_gpus`. -Generate specs from fresh `--print-schema` output. Don't hand-edit generated GPU -policy or add a separate PXE completion step. +Generate specs from fresh `--print-schema` output. Both topologies write their +canonical artifacts immediately. For PXE, review and validate those files, then +run the controller playbook with the generated `inventory.yml` and +`pb-pxe-controller.vars.yml`. The files express desired inputs; their existence +does not prove the PXE rootfs was provisioned successfully. ## Validator contract From c64b8cd36ad0abd6c33d547bc80ade99487d307c Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 14:11:50 +0800 Subject: [PATCH 40/65] docs(deploy): restore GPU setup commands --- deploy/README.md | 6 ++++-- deploy/ansible/README.md | 6 ++++-- deploy/k8s/README.md | 13 +++++++++++-- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index 23703560..b53d0a5d 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -60,8 +60,10 @@ agents' hardware is not inferred from the controller. The AMD device plugin and ROCm node labeller are cluster infrastructure prerequisites owned outside AUPLC. The infrastructure owner must deploy and -maintain them according to AMD's official guidance. Before Helm, verify that the -existing DaemonSets are ready and that GPU capacity is advertised. +maintain them according to AMD's official guidance. If they are not installed, +follow the pinned manual installation commands in the +[Kubernetes components guide](k8s/README.md). Before Helm, verify that the +DaemonSets are ready and that GPU capacity is advertised. #### SSH-preinstalled diff --git a/deploy/ansible/README.md b/deploy/ansible/README.md index d698183b..eda9b349 100644 --- a/deploy/ansible/README.md +++ b/deploy/ansible/README.md @@ -48,5 +48,7 @@ provisioning owns device-node discretionary access control. - **SSH**: Root login with key-based auth to all nodes - **Hosts**: Consistent `/etc/hosts` entries across all nodes - **GPU integration**: The infrastructure owner must deploy and maintain the AMD - device plugin and ROCm node labeller outside AUPLC. Before Helm, run the - readiness and capacity checks in the [deployment guide](../README.md). + device plugin and ROCm node labeller outside AUPLC. Use the pinned manual + installation in the [Kubernetes components guide](../k8s/README.md) when the + cluster does not already provide them. Before Helm, run the readiness and + capacity checks in the [deployment guide](../README.md). diff --git a/deploy/k8s/README.md b/deploy/k8s/README.md index ce25b6c2..da148f52 100644 --- a/deploy/k8s/README.md +++ b/deploy/k8s/README.md @@ -44,9 +44,18 @@ For multi-node deployments, the AMD device plugin and ROCm node labeller are cluster infrastructure prerequisites owned outside AUPLC. The infrastructure owner must select, deploy, and maintain them according to the [official AMD Kubernetes device plugin project](https://github.com/ROCm/k8s-device-plugin). -AUPLC documentation does not install these privileged components. -Before deploying the AUPLC Helm release, verify the existing infrastructure: +To install the same pinned manifests used by `auplc-installer`: + +```bash +ROCM_DEVICE_PLUGIN_COMMIT="dea1db13f05159e64d8114bca4c31f48c3cfcac6" +kubectl apply -f \ + "https://raw.githubusercontent.com/ROCm/k8s-device-plugin/$ROCM_DEVICE_PLUGIN_COMMIT/k8s-ds-amdgpu-dp.yaml" +kubectl apply -f \ + "https://raw.githubusercontent.com/ROCm/k8s-device-plugin/$ROCM_DEVICE_PLUGIN_COMMIT/k8s-ds-amdgpu-labeller.yaml" +``` + +Before deploying the AUPLC Helm release, verify the installation: ```bash kubectl rollout status -n kube-system daemonset/amdgpu-device-plugin-daemonset --timeout=5m From 1ca3cb7fd85c9a2b2eb0c915e6b96c5bd2df0a7b Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 19:07:40 +0800 Subject: [PATCH 41/65] refactor(installer): install AMD GPU udev package --- auplc_installer/gpu_access.py | 253 +++++++------- tests/installer/test_gpu_access.py | 359 +++++++------------- tests/installer/test_gpu_access_ordering.py | 164 +++++++++ 3 files changed, 411 insertions(+), 365 deletions(-) create mode 100644 tests/installer/test_gpu_access_ordering.py diff --git a/auplc_installer/gpu_access.py b/auplc_installer/gpu_access.py index c3a946a7..534e87e2 100644 --- a/auplc_installer/gpu_access.py +++ b/auplc_installer/gpu_access.py @@ -1,15 +1,27 @@ # Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. -"""Single-node AMD GPU device-access reconciler.""" - from __future__ import annotations +import contextlib +import tempfile from pathlib import Path from typing import Protocol -from auplc_installer.util import InstallerError, run, run_capture +from auplc_installer.util import InstallerError, run, run_capture, verify_sha256 + +AMD_GPU_UDEV_PACKAGE_NAME = "amdgpu-insecure-instinct-udev-rules" +AMD_GPU_UDEV_PACKAGE_VERSION = "30.30.4.0-2341068.24.04" +AMD_GPU_UDEV_PACKAGE_FILENAME = "amdgpu-insecure-instinct-udev-rules_30.30.4.0-2341068.24.04_all.deb" +AMD_GPU_UDEV_PACKAGE_URL = ( + "https://repo.radeon.com/amdgpu/30.30.4/ubuntu/pool/main/a/amdgpu-insecure-instinct-udev-rules/" + f"{AMD_GPU_UDEV_PACKAGE_FILENAME}" +) +AMD_GPU_UDEV_PACKAGE_SHA256 = "4be865985c7a13114c45925e77bc0b411b9fd47d5040ed35df44b9c411766162" +AMD_GPU_UDEV_PACKAGE_RULES_PATH = Path("/etc/udev/rules.d/70-amdgpu.rules") +AMD_GPU_UDEV_PACKAGE_RULES = ( + 'KERNEL=="kfd", GROUP="render", MODE="0666"\nSUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0666"\n' +) -GPU_ACCESS_RULES_PATH = Path("/etc/udev/rules.d/70-auplc-gpu-access.rules") LEGACY_KFD_RULES_PATH = Path("/etc/udev/rules.d/70-kfd.rules") LEGACY_AMDGPU_RULES_PATH = Path("/etc/udev/rules.d/70-amdgpu.rules") LEGACY_ROCM_DEVICES_RULES_PATH = Path("/etc/udev/rules.d/70-rocm-devices.rules") @@ -33,91 +45,35 @@ LEGACY_AMDGPU_RULES_PATH: frozenset((LEGACY_AMDGPU_RULES, LEGACY_AMDGPU_PXE_RULES)), LEGACY_ROCM_DEVICES_RULES_PATH: frozenset((LEGACY_ROCM_DEVICES_RULES,)), } -UDEV_MANAGED_MARKER = "# Managed by auplc-installer: AMD GPU device access." -CANONICAL_UDEV_RULES = ( - f"{UDEV_MANAGED_MARKER}\n" - 'KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0666"\n' - 'SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0666"\n' - 'SUBSYSTEM=="drm", KERNEL=="card*", DRIVERS=="amdgpu", OWNER="root", GROUP="video", MODE="0666"\n' -) -_FSYNC_PATH_SCRIPT = ( - "import os\n" - "import sys\n" - "fd = os.open(sys.argv[1], os.O_RDONLY)\n" - "try:\n" - " os.fsync(fd)\n" - "finally:\n" - " os.close(fd)\n" -) -_VERIFY_DEVICE_ACCESS_SCRIPT = ( - "import grp, pathlib, stat\n" - "drm = pathlib.Path('/sys/class/drm')\n" - "devices = [(pathlib.Path('/dev/kfd'), 'render', 0o666)]\n" - "render_nodes = []\n" - "for node in drm.glob('renderD*'):\n" - " driver = node / 'device' / 'driver'\n" - " if driver.exists() and driver.resolve().name == 'amdgpu':\n" - " render_nodes.append(pathlib.Path('/dev/dri') / node.name)\n" - "if not render_nodes: raise SystemExit('no AMD renderD device found')\n" - "devices.extend((path, 'render', 0o666) for path in render_nodes)\n" - "card_nodes = []\n" - "for node in drm.glob('card*'):\n" - " driver = node / 'device' / 'driver'\n" - " if driver.exists() and driver.resolve().name == 'amdgpu':\n" - " card_nodes.append(pathlib.Path('/dev/dri') / node.name)\n" - "if not card_nodes: raise SystemExit('no AMD card device found')\n" - "devices.extend((path, 'video', 0o666) for path in card_nodes)\n" - "for path, expected_group, expected_mode in devices:\n" - " data = path.lstat()\n" - " try:\n" - " group_name = grp.getgrgid(data.st_gid).gr_name\n" - " except KeyError:\n" - " raise SystemExit(f'unknown GPU device group: {path}')\n" - " if not stat.S_ISCHR(data.st_mode) or data.st_uid != 0 or group_name != expected_group or stat.S_IMODE(data.st_mode) != expected_mode:\n" - " raise SystemExit(f'bad GPU device access: {path}')\n" -) class GpuAccessHost(Protocol): - """Privileged host-operation seam for GPU access provisioning.""" + def read_text(self, path: Path) -> str | None: ... - def read_text(self, path: Path) -> str | None: - """Return a privileged file's text, or ``None`` when it is absent.""" + def remove_udev_rule(self, path: Path) -> None: ... - def write_udev_rule(self, path: Path, text: str) -> None: - """Write a managed udev rule after reconciliation has authorized it.""" + def installed_package_version(self) -> str | None: ... - def reload_udev_rules(self) -> None: - """Reload host udev rules.""" + def package_owns_rule(self, path: Path) -> bool: ... - def trigger_udev(self) -> None: - """Apply reloaded udev rules to current devices.""" + def install_package(self, deb: Path) -> None: ... - def settle_udev(self) -> None: - """Wait until triggered udev events finish before inode verification.""" + def reload_udev_rules(self) -> None: ... - def remove_udev_rule(self, path: Path) -> None: - """Remove an explicitly recognized legacy udev rule.""" + def trigger_udev(self) -> None: ... - def verify_device_access(self) -> None: - """Verify the relevant GPU device inodes use the host access contract.""" + def settle_udev(self) -> None: ... - def is_symlink(self, path: Path) -> bool: - """Return whether ``path`` is a symlink without following it.""" + def is_symlink(self, path: Path) -> bool: ... - def is_regular_file(self, path: Path) -> bool: - """Return whether an existing ``path`` is a regular file.""" + def is_regular_file(self, path: Path) -> bool: ... - def path_exists(self, path: Path) -> bool: - """Return whether ``path`` exists after a separate symlink check.""" + def path_exists(self, path: Path) -> bool: ... - def is_directory(self, path: Path) -> bool: - """Return whether an existing ``path`` is a directory.""" + def is_directory(self, path: Path) -> bool: ... class SystemGpuAccessHost: - """Production host adapter using the installer's sudo-aware command helpers.""" - def read_text(self, path: Path) -> str | None: exists = run(["test", "-e", str(path)], sudo=True, check=False) if exists.returncode != 0: @@ -125,33 +81,32 @@ def read_text(self, path: Path) -> str | None: result = run_capture(["cat", str(path)], sudo=True) return result.stdout or "" - def write_udev_rule(self, path: Path, text: str) -> None: - self._write_text_atomically(path, text) + def remove_udev_rule(self, path: Path) -> None: + run(["rm", "-f", str(path)], sudo=True) - def _write_text_atomically(self, path: Path, text: str) -> None: - """Durably replace ``path`` after atomically renaming a temporary file.""" - _validate_parent_chain(self, path.parent) - run(["mkdir", "-p", str(path.parent)], sudo=True) - temporary_result = run_capture( - ["mktemp", str(path.parent / f".{path.name}.XXXXXX")], + def installed_package_version(self) -> str | None: + result = run_capture( + ["dpkg-query", "--show", "--showformat=${Status}\t${Version}", AMD_GPU_UDEV_PACKAGE_NAME], sudo=True, + check=False, ) - temporary_path = (temporary_result.stdout or "").strip() - if not temporary_path: - raise InstallerError(f"Could not create temporary GPU access rule beside {path}") - - try: - run(["tee", temporary_path], sudo=True, input_text=text) - run(["chmod", "0644", temporary_path], sudo=True) - self._fsync_path(temporary_path) - run(["mv", "-f", temporary_path, str(path)], sudo=True) - self._fsync_path(str(path.parent)) - except BaseException: - run(["rm", "-f", temporary_path], sudo=True, check=False) - raise - - def _fsync_path(self, path: str) -> None: - run(["python3", "-c", _FSYNC_PATH_SCRIPT, path], sudo=True) + if result.returncode != 0: + return None + status, separator, version = (result.stdout or "").strip().partition("\t") + if status != "install ok installed" or not separator or not version: + return None + return version + + def package_owns_rule(self, path: Path) -> bool: + result = run_capture( + ["dpkg-query", "--listfiles", AMD_GPU_UDEV_PACKAGE_NAME], + sudo=True, + check=False, + ) + return result.returncode == 0 and str(path) in (result.stdout or "").splitlines() + + def install_package(self, deb: Path) -> None: + run(["dpkg", "--force-confnew", "--install", str(deb)], sudo=True) def reload_udev_rules(self) -> None: run(["udevadm", "control", "--reload-rules"], sudo=True) @@ -162,12 +117,6 @@ def trigger_udev(self) -> None: def settle_udev(self) -> None: run(["udevadm", "settle"], sudo=True) - def remove_udev_rule(self, path: Path) -> None: - run(["rm", "-f", str(path)], sudo=True) - - def verify_device_access(self) -> None: - run(["python3", "-c", _VERIFY_DEVICE_ACCESS_SCRIPT], sudo=True) - def is_symlink(self, path: Path) -> bool: return run(["test", "-L", str(path)], sudo=True, check=False).returncode == 0 @@ -181,35 +130,68 @@ def is_directory(self, path: Path) -> bool: return run(["test", "-d", str(path)], sudo=True, check=False).returncode == 0 -def render_udev_rules() -> str: - """Return the canonical AMD GPU host-device udev rules.""" - return CANONICAL_UDEV_RULES - - -def provision_gpu_access(host: GpuAccessHost | None = None) -> None: - """Reconcile and verify the canonical AMD GPU host-device policy.""" +def provision_gpu_access( + host: GpuAccessHost | None = None, + *, + offline_mode: bool = False, + bundle_dir: Path | None = None, +) -> None: active_host = host if host is not None else SystemGpuAccessHost() - _validate_parent_chain(active_host, GPU_ACCESS_RULES_PATH.parent) + _validate_parent_chain(active_host, AMD_GPU_UDEV_PACKAGE_RULES_PATH.parent) + installed_version = active_host.installed_package_version() legacy_paths = _legacy_rules_to_remove(active_host) - existing_rule = _read_regular_text(active_host, GPU_ACCESS_RULES_PATH) - - for path in legacy_paths: - active_host.remove_udev_rule(path) - if _should_rewrite_udev_rule(existing_rule): - active_host.write_udev_rule(GPU_ACCESS_RULES_PATH, render_udev_rules()) - active_host.reload_udev_rules() - active_host.trigger_udev() - active_host.settle_udev() - active_host.verify_device_access() + if installed_version == AMD_GPU_UDEV_PACKAGE_VERSION: + _verify_installed_package(active_host, installed_version) + else: + _install_package(active_host, offline_mode=offline_mode, bundle_dir=bundle_dir) + installed_version = active_host.installed_package_version() + if installed_version is None: + raise InstallerError(f"{AMD_GPU_UDEV_PACKAGE_NAME} was not installed") + _verify_installed_package(active_host, installed_version) + _remove_separate_legacy_rules(active_host, legacy_paths) + + +def _install_package(active_host: GpuAccessHost, *, offline_mode: bool, bundle_dir: Path | None) -> None: + if offline_mode: + if bundle_dir is None: + raise InstallerError("Offline GPU udev package installation requires a bundle directory") + deb = bundle_dir / "packages" / AMD_GPU_UDEV_PACKAGE_FILENAME + if not deb.is_file(): + raise InstallerError(f"Offline GPU udev package is missing: {deb}") + verify_sha256(deb, AMD_GPU_UDEV_PACKAGE_SHA256) + active_host.install_package(deb) + return + + with tempfile.NamedTemporaryFile(prefix="auplc-amdgpu-udev-", suffix=".deb", delete=False) as temporary: + deb = Path(temporary.name) + try: + run(["wget", "-q", AMD_GPU_UDEV_PACKAGE_URL, "-O", str(deb)]) + verify_sha256(deb, AMD_GPU_UDEV_PACKAGE_SHA256) + active_host.install_package(deb) + finally: + with contextlib.suppress(OSError): + deb.unlink() + + +def _verify_installed_package(active_host: GpuAccessHost, installed_version: str) -> None: + if installed_version != AMD_GPU_UDEV_PACKAGE_VERSION: + raise InstallerError( + f"{AMD_GPU_UDEV_PACKAGE_NAME} has version {installed_version}, expected {AMD_GPU_UDEV_PACKAGE_VERSION}" + ) + if not active_host.package_owns_rule(AMD_GPU_UDEV_PACKAGE_RULES_PATH): + raise InstallerError(f"{AMD_GPU_UDEV_PACKAGE_NAME} does not own {AMD_GPU_UDEV_PACKAGE_RULES_PATH}") + rule = _read_regular_text(active_host, AMD_GPU_UDEV_PACKAGE_RULES_PATH) + if rule != AMD_GPU_UDEV_PACKAGE_RULES: + raise InstallerError(f"{AMD_GPU_UDEV_PACKAGE_NAME} rule does not match the pinned package policy") def _read_regular_text(host: GpuAccessHost, path: Path) -> str | None: if host.is_symlink(path): - raise InstallerError(f"Refusing symlinked GPU access file: {path}") + raise InstallerError(f"Refusing symlinked GPU udev rule: {path}") if not host.path_exists(path): return None if not host.is_regular_file(path): - raise InstallerError(f"Refusing non-regular GPU access file: {path}") + raise InstallerError(f"Refusing non-regular GPU udev rule: {path}") return host.read_text(path) @@ -217,13 +199,13 @@ def _validate_parent_chain(host: GpuAccessHost, parent: Path) -> None: components = [*reversed(parent.parents), parent] for index, component in enumerate(components): if host.is_symlink(component): - raise InstallerError(f"Refusing symlinked GPU access directory: {component}") + raise InstallerError(f"Refusing symlinked GPU udev directory: {component}") if not host.path_exists(component): if index != len(components) - 1: - raise InstallerError(f"Missing parent GPU access directory: {component}") + raise InstallerError(f"Missing parent GPU udev directory: {component}") return if not host.is_directory(component): - raise InstallerError(f"Refusing non-directory GPU access parent: {component}") + raise InstallerError(f"Refusing non-directory GPU udev parent: {component}") def _legacy_rules_to_remove(host: GpuAccessHost) -> list[Path]: @@ -232,17 +214,22 @@ def _legacy_rules_to_remove(host: GpuAccessHost) -> list[Path]: content = _read_regular_text(host, path) if content is None: continue + if path == AMD_GPU_UDEV_PACKAGE_RULES_PATH and host.package_owns_rule(path): + continue if content not in expected_contents: raise InstallerError(f"Refusing to remove unexpected legacy GPU udev rule: {path}") removals.append(path) return removals -def _should_rewrite_udev_rule(existing_rule: str | None) -> bool: - if existing_rule is None: - return True - if existing_rule == render_udev_rules(): - return False - if existing_rule.split("\n", maxsplit=1)[0] != UDEV_MANAGED_MARKER: - raise InstallerError(f"Refusing to overwrite unmanaged GPU udev rule: {GPU_ACCESS_RULES_PATH}") - raise InstallerError(f"Refusing to overwrite unrecognized managed GPU udev rule: {GPU_ACCESS_RULES_PATH}") +def _remove_separate_legacy_rules(host: GpuAccessHost, paths: list[Path]) -> None: + removed = False + for path in paths: + if path == AMD_GPU_UDEV_PACKAGE_RULES_PATH: + continue + host.remove_udev_rule(path) + removed = True + if removed: + host.reload_udev_rules() + host.trigger_udev() + host.settle_udev() diff --git a/tests/installer/test_gpu_access.py b/tests/installer/test_gpu_access.py index 230da4a1..159e6a83 100644 --- a/tests/installer/test_gpu_access.py +++ b/tests/installer/test_gpu_access.py @@ -1,6 +1,6 @@ # Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. -"""Tests for the single-node AMD GPU host device-access reconciler.""" +"""Tests for AMD's packaged single-node GPU udev policy.""" from __future__ import annotations @@ -11,26 +11,29 @@ from auplc_installer import gpu_access from auplc_installer.gpu_access import ( - GPU_ACCESS_RULES_PATH, - LEGACY_AMDGPU_PXE_RULES, + AMD_GPU_UDEV_PACKAGE_FILENAME, + AMD_GPU_UDEV_PACKAGE_RULES, + AMD_GPU_UDEV_PACKAGE_RULES_PATH, + AMD_GPU_UDEV_PACKAGE_VERSION, LEGACY_AMDGPU_RULES, LEGACY_AMDGPU_RULES_PATH, - LEGACY_KFD_RULES, - LEGACY_KFD_RULES_PATH, - LEGACY_ROCM_DEVICES_RULES, - LEGACY_ROCM_DEVICES_RULES_PATH, SystemGpuAccessHost, provision_gpu_access, - render_udev_rules, ) from auplc_installer.util import InstallerError class FakeGpuAccessHost: - """In-memory adapter for the installer host-operation seam.""" - - def __init__(self, *, files: dict[Path, str] | None = None) -> None: + def __init__( + self, + *, + files: dict[Path, str] | None = None, + installed_version: str | None = None, + package_owns_rule: bool | None = None, + ) -> None: self.files = dict(files or {}) + self.installed_version = installed_version + self._package_owns_rule = installed_version is not None if package_owns_rule is None else package_owns_rule self.calls: list[str] = [] self.symlinks: set[Path] = set() self.nonregular_files: set[Path] = set() @@ -40,14 +43,24 @@ def read_text(self, path: Path) -> str | None: self.calls.append(f"read:{path}") return self.files.get(path) - def write_udev_rule(self, path: Path, text: str) -> None: - self.calls.append(f"write-rule:{path}") - self.files[path] = text - def remove_udev_rule(self, path: Path) -> None: self.calls.append(f"remove-rule:{path}") self.files.pop(path, None) + def installed_package_version(self) -> str | None: + self.calls.append("installed-version") + return self.installed_version + + def package_owns_rule(self, path: Path) -> bool: + self.calls.append(f"owns-rule:{path}") + return self._package_owns_rule + + def install_package(self, deb: Path) -> None: + self.calls.append(f"install-package:{deb}") + self.installed_version = AMD_GPU_UDEV_PACKAGE_VERSION + self._package_owns_rule = True + self.files[AMD_GPU_UDEV_PACKAGE_RULES_PATH] = AMD_GPU_UDEV_PACKAGE_RULES + def reload_udev_rules(self) -> None: self.calls.append("reload-udev") @@ -57,9 +70,6 @@ def trigger_udev(self) -> None: def settle_udev(self) -> None: self.calls.append("settle-udev") - def verify_device_access(self) -> None: - self.calls.append("verify-devices") - def is_symlink(self, path: Path) -> bool: return path in self.symlinks @@ -73,254 +83,139 @@ def is_directory(self, path: Path) -> bool: return path in self.directories -def test_render_udev_rules_is_the_canonical_host_device_policy() -> None: - rules = render_udev_rules() - - assert rules == ( - "# Managed by auplc-installer: AMD GPU device access.\n" - 'KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0666"\n' - 'SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0666"\n' - 'SUBSYSTEM=="drm", KERNEL=="card*", DRIVERS=="amdgpu", OWNER="root", GROUP="video", MODE="0666"\n' - ) - assert "chmod" not in rules +def test_offline_install_replaces_legacy_rule_at_the_package_owned_path( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + # Given: an offline bundle and a legacy rule from an earlier shipped installer. + bundle = tmp_path / "bundle" + deb = bundle / "packages" / AMD_GPU_UDEV_PACKAGE_FILENAME + deb.parent.mkdir(parents=True) + deb.write_bytes(b"package") + host = FakeGpuAccessHost(files={LEGACY_AMDGPU_RULES_PATH: LEGACY_AMDGPU_RULES}) + verified: list[tuple[Path, str]] = [] + monkeypatch.setattr(gpu_access, "verify_sha256", lambda path, checksum: verified.append((Path(path), checksum))) + # When: GPU access is provisioned from the bundle. + provision_gpu_access(host, offline_mode=True, bundle_dir=bundle) -def test_device_verification_checks_kfd_and_amd_render_and_card_nodes_without_a_render_gid() -> None: - script = gpu_access._VERIFY_DEVICE_ACCESS_SCRIPT + # Then: package installation replaces the path without deleting the package-owned rule afterward. + assert not any(call == f"remove-rule:{LEGACY_AMDGPU_RULES_PATH}" for call in host.calls) + assert host.files == {AMD_GPU_UDEV_PACKAGE_RULES_PATH: AMD_GPU_UDEV_PACKAGE_RULES} + assert verified == [(deb, gpu_access.AMD_GPU_UDEV_PACKAGE_SHA256)] - assert "path.lstat()" in script - assert "stat.S_ISCHR(data.st_mode)" in script - assert "glob('renderD*')" in script - assert "glob('card*')" in script - assert "'render', 0o666" in script - assert "'video', 0o666" in script - assert "render_gid" not in script - assert "sys.argv[1]" not in script - -@pytest.mark.parametrize("unsafe_parent", [Path("/etc/udev"), Path("/etc/udev/rules.d")]) -def test_symlinked_gpu_access_parent_fails_before_any_file_read_or_write(unsafe_parent: Path) -> None: +def test_online_install_downloads_to_a_temporary_deb_then_removes_it(monkeypatch: pytest.MonkeyPatch) -> None: + # Given: no installed package and a downloader that materializes its destination. host = FakeGpuAccessHost() - host.symlinks.add(unsafe_parent) + downloads: list[list[str]] = [] + verified: list[Path] = [] - with pytest.raises(InstallerError, match="symlinked GPU access directory"): - provision_gpu_access(host) - - assert not any(call.startswith(("read:", "write-", "remove-rule:")) for call in host.calls) - - -def test_nonregular_canonical_rule_fails_before_reading_or_writing_it() -> None: - host = FakeGpuAccessHost() - host.nonregular_files.add(GPU_ACCESS_RULES_PATH) - - with pytest.raises(InstallerError, match="non-regular GPU access file"): - provision_gpu_access(host) - - assert f"read:{GPU_ACCESS_RULES_PATH}" not in host.calls - assert f"write-rule:{GPU_ACCESS_RULES_PATH}" not in host.calls - - -def test_provision_reconciles_the_canonical_rule_without_group_lookup_or_state() -> None: - host = FakeGpuAccessHost() - - result = provision_gpu_access(host) - - assert result is None - assert host.files == {GPU_ACCESS_RULES_PATH: render_udev_rules()} - assert host.calls[-4:] == ["reload-udev", "trigger-udev", "settle-udev", "verify-devices"] - assert not any("group" in call or "state" in call for call in host.calls) - - -@pytest.mark.parametrize( - ("path", "content"), - [ - (LEGACY_KFD_RULES_PATH, LEGACY_KFD_RULES), - (LEGACY_AMDGPU_RULES_PATH, LEGACY_AMDGPU_RULES), - (LEGACY_AMDGPU_RULES_PATH, LEGACY_AMDGPU_PXE_RULES), - (LEGACY_ROCM_DEVICES_RULES_PATH, LEGACY_ROCM_DEVICES_RULES), - ], -) -def test_provision_removes_only_exact_legacy_rules_before_verifying(path: Path, content: str) -> None: - host = FakeGpuAccessHost(files={path: content}) - - provision_gpu_access(host) - - assert path not in host.files - assert host.files[GPU_ACCESS_RULES_PATH] == render_udev_rules() - assert host.calls.index(f"remove-rule:{path}") < host.calls.index(f"write-rule:{GPU_ACCESS_RULES_PATH}") - assert host.calls[-4:] == ["reload-udev", "trigger-udev", "settle-udev", "verify-devices"] - - -@pytest.mark.parametrize( - ("path", "content"), - [ - (LEGACY_AMDGPU_RULES_PATH, 'KERNEL=="kfd", MODE="0666"\nKERNEL=="renderD*", MODE="0666"\n'), - ( - LEGACY_ROCM_DEVICES_RULES_PATH, - "# ROCm device permissions\n" - "# Ensure /dev/kfd and /dev/dri/renderD* are accessible by render group\n" - 'SUBSYSTEM=="kfd", GROUP="render", MODE="0666"\n' - 'SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660"\n', - ), - ], -) -def test_near_legacy_rule_fails_closed_without_removal(path: Path, content: str) -> None: - host = FakeGpuAccessHost(files={path: content}) - - with pytest.raises(InstallerError, match="unexpected legacy"): - provision_gpu_access(host) + def fake_run(command: list[str], **_: object) -> SimpleNamespace: + downloads.append(command) + Path(command[-1]).write_bytes(b"package") + return SimpleNamespace(returncode=0) - assert host.files[path] == content + monkeypatch.setattr(gpu_access, "run", fake_run) + monkeypatch.setattr(gpu_access, "verify_sha256", lambda path, _: verified.append(Path(path))) + + # When: GPU access is provisioned online. + provision_gpu_access(host, offline_mode=False, bundle_dir=None) + + # Then: the exact Radeon URL is downloaded, verified, installed, and cleaned up. + downloaded_path = Path(downloads[0][-1]) + assert downloads == [["wget", "-q", gpu_access.AMD_GPU_UDEV_PACKAGE_URL, "-O", str(downloaded_path)]] + assert verified == [downloaded_path] + assert not downloaded_path.exists() + assert host.calls == [ + "installed-version", + f"install-package:{downloaded_path}", + "installed-version", + f"owns-rule:{AMD_GPU_UDEV_PACKAGE_RULES_PATH}", + f"read:{AMD_GPU_UDEV_PACKAGE_RULES_PATH}", + ] -def test_matching_managed_rule_is_reapplied_and_verified_without_rewriting() -> None: - host = FakeGpuAccessHost(files={GPU_ACCESS_RULES_PATH: render_udev_rules()}) +def test_installed_package_requires_the_pinned_version_and_its_exact_rule() -> None: + # Given: the package is already present with the expected package-owned rule. + host = FakeGpuAccessHost( + files={AMD_GPU_UDEV_PACKAGE_RULES_PATH: AMD_GPU_UDEV_PACKAGE_RULES}, + installed_version=AMD_GPU_UDEV_PACKAGE_VERSION, + ) + # When: provisioning is repeated. provision_gpu_access(host) - assert not any(call.startswith("write-") for call in host.calls) - assert host.calls[-4:] == ["reload-udev", "trigger-udev", "settle-udev", "verify-devices"] + # Then: no download, install, legacy removal, or device probe is performed. + assert host.files == {AMD_GPU_UDEV_PACKAGE_RULES_PATH: AMD_GPU_UDEV_PACKAGE_RULES} + assert not any(call.startswith(("install-package:", "remove-rule:")) for call in host.calls) + assert not any(call in {"reload-udev", "trigger-udev", "settle-udev"} for call in host.calls) @pytest.mark.parametrize( - "unexpected_rule", + ("installed_version", "package_owns_rule", "rule"), [ - f"{gpu_access.UDEV_MANAGED_MARKER}\n" - 'KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0660"\n' - 'SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0660"\n', - f'{gpu_access.UDEV_MANAGED_MARKER}\nKERNEL=="kfd", MODE="0666"\n', + (AMD_GPU_UDEV_PACKAGE_VERSION, False, AMD_GPU_UDEV_PACKAGE_RULES), + (AMD_GPU_UDEV_PACKAGE_VERSION, True, 'KERNEL=="kfd", MODE="0660"\n'), ], ) -def test_noncanonical_managed_rule_fails_closed_before_mutation(unexpected_rule: str) -> None: - host = FakeGpuAccessHost(files={GPU_ACCESS_RULES_PATH: unexpected_rule}) +def test_installed_package_fails_closed_when_its_version_or_rule_contract_is_wrong( + installed_version: str, package_owns_rule: bool, rule: str +) -> None: + # Given: an installed package that does not satisfy the pinned package contract. + host = FakeGpuAccessHost( + files={AMD_GPU_UDEV_PACKAGE_RULES_PATH: rule}, + installed_version=installed_version, + package_owns_rule=package_owns_rule, + ) - with pytest.raises(InstallerError, match="unrecognized managed"): + # When: provisioning checks the installed package. + with pytest.raises(InstallerError): provision_gpu_access(host) - assert host.files[GPU_ACCESS_RULES_PATH] == unexpected_rule - assert not any(call.startswith(("write-", "remove-rule:")) for call in host.calls) - assert "reload-udev" not in host.calls + # Then: it fails before installing or mutating any udev rule. + assert not any(call.startswith(("install-package:", "remove-rule:")) for call in host.calls) -def test_unmanaged_rule_fails_before_any_mutation() -> None: - host = FakeGpuAccessHost(files={GPU_ACCESS_RULES_PATH: 'KERNEL=="kfd", MODE="0666"\n'}) +def test_symlinked_legacy_rule_fails_closed_before_installation(monkeypatch: pytest.MonkeyPatch) -> None: + # Given: a legacy-rule path replaced by a symlink. + host = FakeGpuAccessHost() + host.symlinks.add(LEGACY_AMDGPU_RULES_PATH) + monkeypatch.setattr(gpu_access, "run", lambda *args, **kwargs: pytest.fail("must not download")) - with pytest.raises(InstallerError, match="unmanaged"): + # When: first-time provisioning inspects legacy rules. + with pytest.raises(InstallerError, match="symlinked GPU udev rule"): provision_gpu_access(host) - assert not any(call.startswith(("write-", "remove-rule:")) for call in host.calls) - assert "reload-udev" not in host.calls + # Then: no package installation is attempted. + assert not any(call.startswith("install-package:") for call in host.calls) -def test_system_adapter_persists_udev_rule_with_durable_atomic_replacement(monkeypatch) -> None: - commands: list[list[str]] = [] - capture_commands: list[list[str]] = [] - - def fake_run(command: list[str], **kwargs: object) -> SimpleNamespace: - commands.append(command) - if command[:2] == ["test", "-L"]: - return SimpleNamespace(returncode=1) - return SimpleNamespace(returncode=0) +def test_official_rule_matches_the_extracted_deb_policy_not_the_old_pxe_shape() -> None: + # Given: the exact package verification constant. + rules = AMD_GPU_UDEV_PACKAGE_RULES + old_pxe_shape = 'KERNEL=="kfd", MODE="0666"\nKERNEL=="renderD*", MODE="0666"\n' - def fake_run_capture(command: list[str], **kwargs: object) -> SimpleNamespace: - capture_commands.append(command) - return SimpleNamespace(stdout="/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary\n") - - monkeypatch.setattr(gpu_access, "run", fake_run) - monkeypatch.setattr(gpu_access, "run_capture", fake_run_capture) - - SystemGpuAccessHost().write_udev_rule(GPU_ACCESS_RULES_PATH, "rule\n") - - assert capture_commands == [["mktemp", "/etc/udev/rules.d/.70-auplc-gpu-access.rules.XXXXXX"]] - assert [command for command in commands if command[0] != "test"] == [ - ["mkdir", "-p", "/etc/udev/rules.d"], - ["tee", "/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary"], - ["chmod", "0644", "/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary"], - ["python3", "-c", gpu_access._FSYNC_PATH_SCRIPT, "/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary"], - [ - "mv", - "-f", - "/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary", - "/etc/udev/rules.d/70-auplc-gpu-access.rules", - ], - ["python3", "-c", gpu_access._FSYNC_PATH_SCRIPT, "/etc/udev/rules.d"], - ] + # When: its policy is inspected. + # Then: it matches the extracted package rule rather than the former two-line PXE shape. + assert rules == ( + 'KERNEL=="kfd", GROUP="render", MODE="0666"\n' + 'SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0666"\n' + ) + assert rules != old_pxe_shape + assert "card" not in rules -def test_system_adapter_removes_temporary_rule_when_durable_write_fails(monkeypatch) -> None: +def test_system_adapter_uses_dpkg_for_the_package_install(monkeypatch: pytest.MonkeyPatch) -> None: + # Given: the production host adapter and a recorded command runner. commands: list[list[str]] = [] - - def fake_run(command: list[str], **kwargs: object) -> SimpleNamespace: - commands.append(command) - if command[:2] == ["test", "-L"]: - return SimpleNamespace(returncode=1) - if command == [ - "python3", - "-c", - gpu_access._FSYNC_PATH_SCRIPT, - "/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary", - ]: - raise InstallerError("fsync failed") - return SimpleNamespace(returncode=0) - - monkeypatch.setattr(gpu_access, "run", fake_run) monkeypatch.setattr( gpu_access, - "run_capture", - lambda command, **kwargs: SimpleNamespace(stdout="/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary\n"), + "run", + lambda command, **_: commands.append(command) or SimpleNamespace(returncode=0), ) - with pytest.raises(InstallerError, match="fsync failed"): - SystemGpuAccessHost().write_udev_rule(GPU_ACCESS_RULES_PATH, "rule\n") - - assert [command for command in commands if command[0] != "test"] == [ - ["mkdir", "-p", "/etc/udev/rules.d"], - ["tee", "/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary"], - ["chmod", "0644", "/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary"], - ["python3", "-c", gpu_access._FSYNC_PATH_SCRIPT, "/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary"], - ["rm", "-f", "/etc/udev/rules.d/.70-auplc-gpu-access.rules.temporary"], - ] - - -@pytest.mark.parametrize("failing_method", ["write_udev_rule", "reload_udev_rules", "trigger_udev", "settle_udev"]) -def test_reconciliation_stops_when_udev_mutation_fails(monkeypatch, failing_method: str) -> None: - host = FakeGpuAccessHost() - original_method = getattr(host, failing_method) - - def fail_after_recording(*args: object) -> None: - original_method(*args) - raise InstallerError(f"{failing_method} failed") - - monkeypatch.setattr(host, failing_method, fail_after_recording) - - with pytest.raises(InstallerError, match=f"{failing_method} failed"): - provision_gpu_access(host) - - assert "verify-devices" not in host.calls - - -def test_failed_inode_verification_leaves_the_reconciled_rule_in_place(monkeypatch) -> None: - host = FakeGpuAccessHost() - - def fail_verification() -> None: - host.calls.append("verify-devices") - raise InstallerError("device ownership mismatch") - - monkeypatch.setattr(host, "verify_device_access", fail_verification) - - with pytest.raises(InstallerError, match="ownership mismatch"): - provision_gpu_access(host) - - assert host.files[GPU_ACCESS_RULES_PATH] == render_udev_rules() - assert host.calls[-1] == "verify-devices" - - -@pytest.mark.parametrize("path", [LEGACY_KFD_RULES_PATH, LEGACY_AMDGPU_RULES_PATH, GPU_ACCESS_RULES_PATH]) -def test_symlinked_gpu_access_files_fail_closed_before_mutation(path: Path) -> None: - host = FakeGpuAccessHost() - host.symlinks.add(path) - - with pytest.raises(InstallerError, match="symlinked"): - provision_gpu_access(host) + # When: it installs the verified package artifact. + SystemGpuAccessHost().install_package(Path("/tmp/package.deb")) - assert not any(call.startswith(("write-", "remove-rule:")) for call in host.calls) + # Then: installation is delegated to dpkg with sudo awareness. + assert commands == [["dpkg", "--force-confnew", "--install", "/tmp/package.deb"]] diff --git a/tests/installer/test_gpu_access_ordering.py b/tests/installer/test_gpu_access_ordering.py new file mode 100644 index 00000000..6363ffc8 --- /dev/null +++ b/tests/installer/test_gpu_access_ordering.py @@ -0,0 +1,164 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from auplc_installer import gpu_access +from auplc_installer.gpu_access import ( + AMD_GPU_UDEV_PACKAGE_FILENAME, + AMD_GPU_UDEV_PACKAGE_RULES, + AMD_GPU_UDEV_PACKAGE_RULES_PATH, + AMD_GPU_UDEV_PACKAGE_VERSION, + LEGACY_KFD_RULES, + LEGACY_KFD_RULES_PATH, + provision_gpu_access, +) +from auplc_installer.util import InstallerError +from tests.installer.test_gpu_access import FakeGpuAccessHost + + +def _offline_bundle(tmp_path: Path) -> Path: + bundle = tmp_path / "bundle" + deb = bundle / "packages" / AMD_GPU_UDEV_PACKAGE_FILENAME + deb.parent.mkdir(parents=True) + deb.write_bytes(b"package") + return bundle + + +def test_wrong_installed_version_downloads_and_converges_to_the_pinned_package(monkeypatch: pytest.MonkeyPatch) -> None: + # Given: a different installed package version and an online downloader. + host = FakeGpuAccessHost( + files={AMD_GPU_UDEV_PACKAGE_RULES_PATH: 'KERNEL=="kfd", MODE="0660"\n'}, + installed_version="30.30.4.0-older", + ) + downloads: list[list[str]] = [] + + def fake_run(command: list[str], **_: object) -> SimpleNamespace: + downloads.append(command) + Path(command[-1]).write_bytes(b"package") + return SimpleNamespace(returncode=0) + + monkeypatch.setattr(gpu_access, "run", fake_run) + monkeypatch.setattr(gpu_access, "verify_sha256", lambda *args: None) + + # When: GPU access is provisioned. + provision_gpu_access(host) + + # Then: the pinned deb is acquired and the installed rule converges to its exact content. + assert downloads[0][2] == gpu_access.AMD_GPU_UDEV_PACKAGE_URL + assert host.installed_version == AMD_GPU_UDEV_PACKAGE_VERSION + assert host.files[AMD_GPU_UDEV_PACKAGE_RULES_PATH] == AMD_GPU_UDEV_PACKAGE_RULES + + +def test_exact_installed_package_skips_network_then_removes_separate_legacy_rule( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Given: an exact package rule plus a separately shipped legacy KFD rule. + host = FakeGpuAccessHost( + files={ + AMD_GPU_UDEV_PACKAGE_RULES_PATH: AMD_GPU_UDEV_PACKAGE_RULES, + LEGACY_KFD_RULES_PATH: LEGACY_KFD_RULES, + }, + installed_version=AMD_GPU_UDEV_PACKAGE_VERSION, + ) + monkeypatch.setattr(gpu_access, "run", lambda *args, **kwargs: pytest.fail("must not download")) + + # When: provisioning checks an otherwise already-correct installation. + provision_gpu_access(host) + + # Then: it removes only the separate legacy file and applies its removal to live udev state. + assert LEGACY_KFD_RULES_PATH not in host.files + assert not any(call.startswith("install-package:") for call in host.calls) + assert host.calls[-3:] == ["reload-udev", "trigger-udev", "settle-udev"] + + +def test_acquires_and_verifies_the_offline_deb_before_deleting_legacy_rules( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + # Given: a first installation from a verified offline bundle and a legacy KFD rule. + bundle = _offline_bundle(tmp_path) + host = FakeGpuAccessHost(files={LEGACY_KFD_RULES_PATH: LEGACY_KFD_RULES}) + monkeypatch.setattr(gpu_access, "verify_sha256", lambda *args: host.calls.append("verify-deb")) + + # When: the package is installed. + provision_gpu_access(host, offline_mode=True, bundle_dir=bundle) + + # Then: package installation completes before the separate legacy rule is deleted. + install_index = next(index for index, call in enumerate(host.calls) if call.startswith("install-package:")) + removal_index = host.calls.index(f"remove-rule:{LEGACY_KFD_RULES_PATH}") + assert host.calls.index("verify-deb") < install_index < removal_index + + +def test_failed_installation_keeps_legacy_rules_intact(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + # Given: a first offline installation whose package install fails. + bundle = _offline_bundle(tmp_path) + host = FakeGpuAccessHost(files={LEGACY_KFD_RULES_PATH: LEGACY_KFD_RULES}) + monkeypatch.setattr(gpu_access, "verify_sha256", lambda *args: None) + + def fail_install(deb: Path) -> None: + host.calls.append(f"install-package:{deb}") + raise InstallerError("dpkg failed") + + monkeypatch.setattr(host, "install_package", fail_install) + + # When: package installation fails. + with pytest.raises(InstallerError, match="dpkg failed"): + provision_gpu_access(host, offline_mode=True, bundle_dir=bundle) + + # Then: the legacy rule remains and no udev refresh occurs. + assert host.files[LEGACY_KFD_RULES_PATH] == LEGACY_KFD_RULES + assert "reload-udev" not in host.calls + + +def test_wrong_version_package_owned_differing_conffile_converges( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + # Given: a package-owned conffile from a different installed package version. + host = FakeGpuAccessHost( + files={AMD_GPU_UDEV_PACKAGE_RULES_PATH: 'KERNEL=="kfd", MODE="0600"\n'}, + installed_version="30.30.4.0-older", + package_owns_rule=True, + ) + monkeypatch.setattr(gpu_access, "verify_sha256", lambda *args: None) + + # When: the pinned package is installed from an offline bundle. + provision_gpu_access(host, offline_mode=True, bundle_dir=_offline_bundle(tmp_path)) + + # Then: forced installation replaces the differing conffile with the exact package rule. + assert host.files[AMD_GPU_UDEV_PACKAGE_RULES_PATH] == AMD_GPU_UDEV_PACKAGE_RULES + assert not any(call.startswith("remove-rule:") for call in host.calls) + + +def test_partial_package_owned_conffile_converges(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + # Given: a config-files package state that still owns a differing conffile. + host = FakeGpuAccessHost( + files={AMD_GPU_UDEV_PACKAGE_RULES_PATH: 'KERNEL=="kfd", MODE="0600"\n'}, + package_owns_rule=True, + ) + monkeypatch.setattr(gpu_access, "verify_sha256", lambda *args: None) + + # When: the pinned package is installed from an offline bundle. + provision_gpu_access(host, offline_mode=True, bundle_dir=_offline_bundle(tmp_path)) + + # Then: ownership prevents legacy admission and the package converges to the exact rule. + assert host.files[AMD_GPU_UDEV_PACKAGE_RULES_PATH] == AMD_GPU_UDEV_PACKAGE_RULES + assert not any(call.startswith("remove-rule:") for call in host.calls) + + +def test_unknown_unowned_amdgpu_rule_fails_closed() -> None: + # Given: an unowned, unrecognized rule at the AMD package path. + host = FakeGpuAccessHost( + files={AMD_GPU_UDEV_PACKAGE_RULES_PATH: 'KERNEL=="kfd", MODE="0600"\n'}, + package_owns_rule=False, + ) + + # When: provisioning admits legacy rules. + with pytest.raises(InstallerError, match="unexpected legacy"): + provision_gpu_access(host) + + # Then: no package installation or rule deletion is attempted. + assert not any(call.startswith(("install-package:", "remove-rule:")) for call in host.calls) From 2f87d335369fd421397a30b40ef58d2590582f01 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 19:07:40 +0800 Subject: [PATCH 42/65] refactor(installer): pass GPU package context --- auplc_installer/cli.py | 18 ++++++++--------- tests/installer/test_cli_gpu_access.py | 28 ++++++++++++++++++++------ 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/auplc_installer/cli.py b/auplc_installer/cli.py index 609930a9..390f2b32 100644 --- a/auplc_installer/cli.py +++ b/auplc_installer/cli.py @@ -326,10 +326,10 @@ def _raise_unreachable_gpu_hardware(hardware: GpuHardware) -> NoReturn: raise AssertionError(f"Unhandled GPU hardware classification: {hardware!r}") -def _provision_gpu_access_for_local_hardware() -> None: +def _provision_gpu_access_for_local_hardware(*, offline_mode: bool, bundle_dir: Path | None) -> None: match classify_gpu_hardware(): case GpuHardware.GPU: - provision_gpu_access() + provision_gpu_access(offline_mode=offline_mode, bundle_dir=bundle_dir) case GpuHardware.CPU: return case GpuHardware.UNKNOWN: @@ -355,7 +355,7 @@ def _cmd_install_inner(state: InstallerState, *, pull: bool) -> None: detect_and_configure_gpu(state.gpu, gpu_type_override=state.gpu_type) with stage("Provisioning GPU device access", idx=2, total=total): - _provision_gpu_access_for_local_hardware() + _provision_gpu_access_for_local_hardware(offline_mode=state.offline_mode, bundle_dir=state.bundle_dir) paths = state.runtime_paths() with stage("Generating values overlay (initial)", idx=3, total=total): @@ -604,7 +604,7 @@ def cmd_dev_quick(state: InstallerState) -> None: def cmd_dev_deploy(state: InstallerState) -> None: - _provision_gpu_access_for_local_hardware() + _provision_gpu_access_for_local_hardware(offline_mode=state.offline_mode, bundle_dir=state.bundle_dir) detect_and_configure_gpu(state.gpu, gpu_type_override=state.gpu_type) paths = state.runtime_paths() refine_gpu_config_from_node_labels(state.gpu) @@ -620,7 +620,7 @@ def cmd_dev_deploy(state: InstallerState) -> None: def cmd_dev_upgrade(state: InstallerState) -> None: - _provision_gpu_access_for_local_hardware() + _provision_gpu_access_for_local_hardware(offline_mode=state.offline_mode, bundle_dir=state.bundle_dir) detect_and_configure_gpu(state.gpu, gpu_type_override=state.gpu_type) paths = state.runtime_paths() refine_gpu_config_from_node_labels(state.gpu) @@ -637,7 +637,7 @@ def cmd_dev_upgrade(state: InstallerState) -> None: def cmd_dev_reinstall(state: InstallerState) -> None: - _provision_gpu_access_for_local_hardware() + _provision_gpu_access_for_local_hardware(offline_mode=state.offline_mode, bundle_dir=state.bundle_dir) with contextlib.suppress(InstallerError): remove_runtime() time.sleep(0.5) @@ -648,7 +648,7 @@ def cmd_dev_reinstall(state: InstallerState) -> None: def cmd_rt_install(state: InstallerState) -> None: - _provision_gpu_access_for_local_hardware() + _provision_gpu_access_for_local_hardware(offline_mode=state.offline_mode, bundle_dir=state.bundle_dir) detect_and_configure_gpu(state.gpu, gpu_type_override=state.gpu_type) paths = state.runtime_paths() refine_gpu_config_from_node_labels(state.gpu) @@ -664,7 +664,7 @@ def cmd_rt_install(state: InstallerState) -> None: def cmd_rt_upgrade(state: InstallerState) -> None: - _provision_gpu_access_for_local_hardware() + _provision_gpu_access_for_local_hardware(offline_mode=state.offline_mode, bundle_dir=state.bundle_dir) detect_and_configure_gpu(state.gpu, gpu_type_override=state.gpu_type) paths = state.runtime_paths() refine_gpu_config_from_node_labels(state.gpu) @@ -705,7 +705,7 @@ def cmd_rt_remove(state: InstallerState) -> None: def cmd_rt_reinstall(state: InstallerState) -> None: - _provision_gpu_access_for_local_hardware() + _provision_gpu_access_for_local_hardware(offline_mode=state.offline_mode, bundle_dir=state.bundle_dir) with contextlib.suppress(InstallerError): remove_runtime() time.sleep(0.5) diff --git a/tests/installer/test_cli_gpu_access.py b/tests/installer/test_cli_gpu_access.py index e4cdaaf9..b48210b4 100644 --- a/tests/installer/test_cli_gpu_access.py +++ b/tests/installer/test_cli_gpu_access.py @@ -39,7 +39,7 @@ def fake_overlay(*args: object, **kwargs: object) -> Path: monkeypatch.setattr(cli, "stage", fake_stage) monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: hardware) monkeypatch.setattr(cli, "detect_and_configure_gpu", lambda *args, **kwargs: events.append("detect")) - monkeypatch.setattr(cli, "provision_gpu_access", lambda: events.append("provision")) + monkeypatch.setattr(cli, "provision_gpu_access", lambda **kwargs: events.append("provision")) monkeypatch.setattr(cli, "generate_values_overlay", fake_overlay) monkeypatch.setattr(cli, "install_tools", lambda **kwargs: events.append("tools")) monkeypatch.setattr(cli, "install_k3s_single_node", lambda **kwargs: events.append("k3s")) @@ -84,7 +84,7 @@ def fake_overlay(*args: object, **kwargs: object) -> Path: monkeypatch.setattr(state, "runtime_paths", lambda: paths) monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: hardware) - monkeypatch.setattr(cli, "provision_gpu_access", lambda: events.append("provision")) + monkeypatch.setattr(cli, "provision_gpu_access", lambda **kwargs: events.append("provision")) monkeypatch.setattr(cli, "detect_and_configure_gpu", lambda *args, **kwargs: events.append("detect")) monkeypatch.setattr(cli, "refine_gpu_config_from_node_labels", lambda *args, **kwargs: events.append("refine")) monkeypatch.setattr(cli, "_preserve_courses_for_upgrade", lambda *args, **kwargs: events.append("preserve-courses")) @@ -116,7 +116,7 @@ def test_cpu_hardware_skips_host_access_and_preserves_runtime_flow( monkeypatch.setattr(state, "runtime_paths", lambda: paths) monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: GpuHardware.CPU) monkeypatch.setattr( - cli, "provision_gpu_access", lambda: (_ for _ in ()).throw(AssertionError("must not provision")) + cli, "provision_gpu_access", lambda **kwargs: (_ for _ in ()).throw(AssertionError("must not provision")) ) monkeypatch.setattr(cli, "detect_and_configure_gpu", lambda *args, **kwargs: events.append("detect")) monkeypatch.setattr(cli, "refine_gpu_config_from_node_labels", lambda *args, **kwargs: events.append("refine")) @@ -152,7 +152,7 @@ def test_reinstall_gates_host_access_before_removing_runtime( state = InstallerState() monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: hardware) - monkeypatch.setattr(cli, "provision_gpu_access", lambda: events.append("provision")) + monkeypatch.setattr(cli, "provision_gpu_access", lambda **kwargs: events.append("provision")) monkeypatch.setattr(cli, "remove_runtime", lambda: events.append("remove-runtime")) monkeypatch.setattr(cli.time, "sleep", lambda seconds: events.append("sleep")) monkeypatch.setattr(cli, delegate_name, lambda current_state: events.append("delegate")) @@ -169,7 +169,7 @@ def test_unknown_hardware_blocks_full_install_before_gpu_access_mutation(monkeyp monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: GpuHardware.UNKNOWN) monkeypatch.setattr(cli, "detect_and_configure_gpu", lambda *args, **kwargs: events.append("detect")) monkeypatch.setattr( - cli, "provision_gpu_access", lambda: (_ for _ in ()).throw(AssertionError("must not provision")) + cli, "provision_gpu_access", lambda **kwargs: (_ for _ in ()).throw(AssertionError("must not provision")) ) with pytest.raises(RuntimeError, match="hardware"): @@ -190,7 +190,7 @@ def test_unknown_hardware_blocks_reinstall_before_runtime_removal( monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: GpuHardware.UNKNOWN) monkeypatch.setattr( - cli, "provision_gpu_access", lambda: (_ for _ in ()).throw(AssertionError("must not provision")) + cli, "provision_gpu_access", lambda **kwargs: (_ for _ in ()).throw(AssertionError("must not provision")) ) monkeypatch.setattr(cli, "remove_runtime", lambda: events.append("remove-runtime")) monkeypatch.setattr(cli, delegate_name, lambda current_state: events.append("delegate")) @@ -204,3 +204,19 @@ def test_unknown_hardware_blocks_reinstall_before_runtime_removal( def test_cli_exposes_no_render_gid_reconciliation_api() -> None: assert not hasattr(cli, "_render_gid_for_local_hardware") assert not hasattr(cli, "load_existing_gpu_access") + + +def test_gpu_hardware_gate_passes_offline_bundle_context_to_package_provisioning( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + # Given: a local GPU installation running from an offline bundle. + bundle = tmp_path / "bundle" + package_calls: list[dict[str, object]] = [] + monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: GpuHardware.GPU) + monkeypatch.setattr(cli, "provision_gpu_access", lambda **kwargs: package_calls.append(kwargs)) + + # When: the CLI's local-hardware gate provisions GPU access. + cli._provision_gpu_access_for_local_hardware(offline_mode=True, bundle_dir=bundle) + + # Then: package provisioning receives the bundle context unchanged. + assert package_calls == [{"offline_mode": True, "bundle_dir": bundle}] From 36dc762b82164b9c9e241853c4f2df45ee82b214 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 19:07:40 +0800 Subject: [PATCH 43/65] feat(installer): bundle AMD GPU udev package --- auplc_installer/pack.py | 14 ++++++++++++++ tests/installer/test_pack.py | 37 ++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 tests/installer/test_pack.py diff --git a/auplc_installer/pack.py b/auplc_installer/pack.py index c4a1b4d7..ef7531b4 100644 --- a/auplc_installer/pack.py +++ b/auplc_installer/pack.py @@ -19,6 +19,11 @@ from auplc_installer.catalog import HUB_IMAGE_NAME, CourseSelection from auplc_installer.gpu import GpuConfig, detect_and_configure_gpu +from auplc_installer.gpu_access import ( + AMD_GPU_UDEV_PACKAGE_FILENAME, + AMD_GPU_UDEV_PACKAGE_SHA256, + AMD_GPU_UDEV_PACKAGE_URL, +) from auplc_installer.images import ( EXTERNAL_IMAGES, pull_and_tag, @@ -121,6 +126,14 @@ def pack_download_k3s_images(staging: Path) -> None: ) +def pack_download_gpu_access_package(staging: Path) -> None: + packages_dir = staging / "packages" + packages_dir.mkdir(parents=True, exist_ok=True) + deb = packages_dir / AMD_GPU_UDEV_PACKAGE_FILENAME + run(["wget", "-q", AMD_GPU_UDEV_PACKAGE_URL, "-O", str(deb)]) + verify_sha256(deb, AMD_GPU_UDEV_PACKAGE_SHA256) + + def pack_save_manifests(staging: Path) -> None: log_step("Saving manifests") out_dir = staging / "manifests" @@ -461,6 +474,7 @@ def pack_bundle( pack_download_binaries(staging) pack_download_k3s_images(staging) + pack_download_gpu_access_package(staging) pack_save_manifests(staging) if local_build: diff --git a/tests/installer/test_pack.py b/tests/installer/test_pack.py new file mode 100644 index 00000000..0b984a52 --- /dev/null +++ b/tests/installer/test_pack.py @@ -0,0 +1,37 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +"""Tests for offline bundle package artifacts.""" + +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace + +from auplc_installer import pack +from auplc_installer.gpu_access import ( + AMD_GPU_UDEV_PACKAGE_FILENAME, + AMD_GPU_UDEV_PACKAGE_SHA256, + AMD_GPU_UDEV_PACKAGE_URL, +) + + +def test_pack_downloads_and_checksums_the_offline_gpu_udev_package(tmp_path: Path, monkeypatch) -> None: + # Given: an empty bundle staging directory and a recording downloader. + commands: list[list[str]] = [] + verified: list[tuple[Path, str]] = [] + + def fake_run(command: list[str], **_: object) -> SimpleNamespace: + commands.append(command) + Path(command[-1]).write_bytes(b"package") + return SimpleNamespace(returncode=0) + + monkeypatch.setattr(pack, "run", fake_run) + monkeypatch.setattr(pack, "verify_sha256", lambda path, checksum: verified.append((Path(path), checksum))) + + # When: package artifacts are added to the offline bundle. + pack.pack_download_gpu_access_package(tmp_path) + + # Then: the pinned deb is placed in packages/ and verified before archiving. + deb = tmp_path / "packages" / AMD_GPU_UDEV_PACKAGE_FILENAME + assert commands == [["wget", "-q", AMD_GPU_UDEV_PACKAGE_URL, "-O", str(deb)]] + assert verified == [(deb, AMD_GPU_UDEV_PACKAGE_SHA256)] From be284a601f82208fc6bf28674c2c6183c288f7fb Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 19:07:40 +0800 Subject: [PATCH 44/65] refactor(ansible): install AMD GPU udev package --- .../roles/gpu_access/defaults/main.yml | 12 + .../roles/gpu_access/handlers/main.yml | 17 -- .../ansible/roles/gpu_access/tasks/apply.yml | 266 ++++++------------ .../roles/gpu_access/tasks/preflight.yml | 176 +++++++++--- .../ansible/roles/gpu_access/tasks/verify.yml | 162 +++++++++++ .../templates/70-auplc-gpu-access.rules.j2 | 4 - tests/skills/test_gpu_access_role.py | 244 ++++++++++------ 7 files changed, 556 insertions(+), 325 deletions(-) delete mode 100644 deploy/ansible/roles/gpu_access/handlers/main.yml create mode 100644 deploy/ansible/roles/gpu_access/tasks/verify.yml delete mode 100644 deploy/ansible/roles/gpu_access/templates/70-auplc-gpu-access.rules.j2 diff --git a/deploy/ansible/roles/gpu_access/defaults/main.yml b/deploy/ansible/roles/gpu_access/defaults/main.yml index 8f5ff6c7..4a9d4e3b 100644 --- a/deploy/ansible/roles/gpu_access/defaults/main.yml +++ b/deploy/ansible/roles/gpu_access/defaults/main.yml @@ -7,3 +7,15 @@ auplc_rootfs_path: "" # Rootfs adapters must explicitly constrain their writable target below this # canonical directory. Live hosts leave this empty. auplc_rootfs_allowed_root: "" +auplc_gpu_udev_package_name: amdgpu-insecure-instinct-udev-rules +auplc_gpu_udev_package_version: 30.30.4.0-2341068.24.04 +auplc_gpu_udev_package_filename: amdgpu-insecure-instinct-udev-rules_30.30.4.0-2341068.24.04_all.deb +auplc_gpu_udev_package_url: >- + https://repo.radeon.com/amdgpu/30.30.4/ubuntu/pool/main/a/amdgpu-insecure-instinct-udev-rules/amdgpu-insecure-instinct-udev-rules_30.30.4.0-2341068.24.04_all.deb +auplc_gpu_udev_package_checksum: sha256:4be865985c7a13114c45925e77bc0b411b9fd47d5040ed35df44b9c411766162 +auplc_gpu_udev_package_cache_path: >- + /var/cache/auplc/amdgpu-udev-rules/amdgpu-insecure-instinct-udev-rules_30.30.4.0-2341068.24.04_all.deb +auplc_gpu_udev_rule_path: /etc/udev/rules.d/70-amdgpu.rules +auplc_gpu_udev_rule_content: | + KERNEL=="kfd", GROUP="render", MODE="0666" + SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0666" diff --git a/deploy/ansible/roles/gpu_access/handlers/main.yml b/deploy/ansible/roles/gpu_access/handlers/main.yml deleted file mode 100644 index 6afe9532..00000000 --- a/deploy/ansible/roles/gpu_access/handlers/main.yml +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. - ---- -- name: Reload udev rules - ansible.builtin.command: - argv: - - udevadm - - control - - --reload-rules - when: auplc_rootfs_path | length == 0 - -- name: Trigger udev rules - ansible.builtin.command: - argv: - - udevadm - - trigger - when: auplc_rootfs_path | length == 0 diff --git a/deploy/ansible/roles/gpu_access/tasks/apply.yml b/deploy/ansible/roles/gpu_access/tasks/apply.yml index f830b935..261d8a35 100644 --- a/deploy/ansible/roles/gpu_access/tasks/apply.yml +++ b/deploy/ansible/roles/gpu_access/tasks/apply.yml @@ -1,54 +1,89 @@ # Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. --- -- name: Inspect canonical GPU access rule before apply - ansible.builtin.stat: - path: "{{ _auplc_target_root }}/etc/udev/rules.d/70-auplc-gpu-access.rules" - follow: false - register: _auplc_apply_destination_rule - -- name: Reject unsafe canonical GPU access rule before apply - ansible.builtin.assert: - that: - - not _auplc_apply_destination_rule.stat.exists or - (_auplc_apply_destination_rule.stat.isreg and not _auplc_apply_destination_rule.stat.islnk) - fail_msg: Unsafe canonical GPU access destination. - -- name: Read canonical GPU access rule before apply - ansible.builtin.slurp: - src: "{{ _auplc_target_root }}/etc/udev/rules.d/70-auplc-gpu-access.rules" - register: _auplc_apply_existing_rule - when: _auplc_apply_destination_rule.stat.exists - -- name: Define canonical GPU access rule contents for apply - ansible.builtin.set_fact: - _auplc_apply_canonical_rule: | - # Managed by auplc-installer: AMD GPU device access. - KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0666" - SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0666" - SUBSYSTEM=="drm", KERNEL=="card*", DRIVERS=="amdgpu", OWNER="root", GROUP="video", MODE="0666" - -- name: Recheck canonical GPU access rule before apply - ansible.builtin.assert: - that: (_auplc_apply_existing_rule.content | b64decode) == _auplc_apply_canonical_rule - fail_msg: Unmanaged canonical GPU access rule. - when: _auplc_apply_destination_rule.stat.exists - -- name: Inspect recognized project-owned legacy GPU rules for apply +- name: Install AMD udev package when required + block: + - name: Create deterministic AMD udev package cache + ansible.builtin.file: + path: "{{ auplc_gpu_udev_package_cache_path | dirname }}" + state: directory + owner: root + group: root + mode: "0755" + + - name: Download checksummed AMD udev package + ansible.builtin.get_url: + url: "{{ auplc_gpu_udev_package_url }}" + dest: "{{ auplc_gpu_udev_package_cache_path }}" + checksum: "{{ auplc_gpu_udev_package_checksum }}" + owner: root + group: root + mode: "0644" + + - name: Install AMD udev package on live host + ansible.builtin.apt: + deb: "{{ auplc_gpu_udev_package_cache_path }}" + state: present + allow_downgrade: true + dpkg_options: force-confnew + when: _auplc_target_root | length == 0 + + - name: Copy AMD udev package into PXE rootfs + ansible.builtin.copy: + src: "{{ auplc_gpu_udev_package_cache_path }}" + dest: "{{ _auplc_target_root }}/tmp/{{ auplc_gpu_udev_package_filename }}" + remote_src: true + owner: root + group: root + mode: "0644" + when: _auplc_target_root | length > 0 + + - name: Install AMD udev package in PXE rootfs + ansible.builtin.command: + argv: + - chroot + - "{{ _auplc_target_root }}" + - apt-get + - --option=Dpkg::Options::=--force-confnew + - install + - --yes + - --no-install-recommends + - "/tmp/{{ auplc_gpu_udev_package_filename }}" + environment: + DEBIAN_FRONTEND: noninteractive + changed_when: true + when: _auplc_target_root | length > 0 + + - name: Verify installed AMD udev package + ansible.builtin.import_tasks: verify.yml + + always: + - name: Remove temporary AMD udev package from PXE rootfs + ansible.builtin.file: + path: "{{ _auplc_target_root }}/tmp/{{ auplc_gpu_udev_package_filename }}" + state: absent + when: _auplc_target_root | length > 0 + when: _auplc_gpu_udev_install_needed | bool + +- name: Verify installed AMD udev package without installation + ansible.builtin.import_tasks: verify.yml + when: not _auplc_gpu_udev_install_needed | bool + +- name: Recheck recognized project-owned legacy GPU rules before apply ansible.builtin.stat: path: "{{ item.path }}" follow: false loop: "{{ _auplc_legacy_gpu_rules }}" register: _auplc_apply_legacy_gpu_rule_stats -- name: Reject legacy GPU rule symlinks and non-regular files before apply +- name: Reject unsafe legacy GPU rules before apply ansible.builtin.assert: that: - not item.stat.exists or (item.stat.isreg and not item.stat.islnk) fail_msg: "Unexpected legacy GPU rule filesystem type: {{ item.item.path }}" loop: "{{ _auplc_apply_legacy_gpu_rule_stats.results }}" -- name: Read recognized project-owned legacy GPU rules for apply +- name: Read recognized project-owned legacy GPU rules before apply ansible.builtin.slurp: src: "{{ item.item.path }}" loop: "{{ _auplc_apply_legacy_gpu_rule_stats.results }}" @@ -58,7 +93,10 @@ - name: Reject unexpected legacy GPU rule content before apply ansible.builtin.assert: that: - - (item.content | b64decode) in item.item.item.contents + - >- + ((item.content | b64decode) | hash('sha256')) in item.item.item.sha256 or + (item.item.item.path == _auplc_target_root + auplc_gpu_udev_rule_path and + (item.content | b64decode) == auplc_gpu_udev_rule_content) fail_msg: "Unexpected legacy GPU rule content: {{ item.item.item.path }}" loop: "{{ _auplc_apply_legacy_gpu_rule_contents.results }}" when: not item.skipped | default(false) @@ -68,153 +106,23 @@ path: "{{ item.item.item.path }}" state: absent loop: "{{ _auplc_apply_legacy_gpu_rule_contents.results }}" - when: not item.skipped | default(false) + when: >- + not item.skipped | default(false) and + ((item.content | b64decode) | hash('sha256')) in item.item.item.sha256 + register: _auplc_removed_legacy_gpu_rules -- name: Create target udev rules directory - ansible.builtin.file: - path: "{{ _auplc_target_root }}/etc/udev/rules.d" - state: directory - owner: root - group: root - mode: "0755" - -- name: Install canonical AMD GPU udev rules - ansible.builtin.template: - src: 70-auplc-gpu-access.rules.j2 - dest: "{{ _auplc_target_root }}/etc/udev/rules.d/70-auplc-gpu-access.rules" - owner: root - group: root - mode: "0644" - -- name: Reload live udev rules on every apply +- name: Reload live udev rules after legacy cleanup ansible.builtin.command: argv: [udevadm, control, --reload-rules] changed_when: false - when: _auplc_target_root | length == 0 + when: + - _auplc_target_root | length == 0 + - _auplc_removed_legacy_gpu_rules.changed -- name: Trigger live udev rules on every apply +- name: Trigger live udev rules after legacy cleanup ansible.builtin.command: argv: [udevadm, trigger] changed_when: false - when: _auplc_target_root | length == 0 - -- name: Settle live udev events before inode verification - ansible.builtin.command: - argv: [udevadm, settle] - changed_when: false - when: _auplc_target_root | length == 0 - -- name: Inspect /dev/kfd after live reconciliation - ansible.builtin.stat: - path: /dev/kfd - follow: false - register: _auplc_kfd - when: _auplc_target_root | length == 0 - -- name: Verify /dev/kfd ownership and mode - ansible.builtin.assert: - that: - - _auplc_kfd.stat.exists - - _auplc_kfd.stat.ischr - - _auplc_kfd.stat.uid == 0 - - _auplc_kfd.stat.gr_name == 'render' - - _auplc_kfd.stat.mode == '0666' - fail_msg: /dev/kfd is not root:render with mode 0666 after reconciliation. - when: _auplc_target_root | length == 0 - -- name: Find live DRM render nodes - ansible.builtin.find: - paths: /dev/dri - patterns: renderD* - file_type: any - recurse: false - register: _auplc_render_nodes - when: _auplc_target_root | length == 0 - -- name: Find live DRM card nodes - ansible.builtin.find: - paths: /dev/dri - patterns: card* - file_type: any - recurse: false - register: _auplc_card_nodes - when: _auplc_target_root | length == 0 - -- name: Resolve live DRM node driver symlinks - ansible.builtin.command: - argv: [readlink, -f, "/sys/class/drm/{{ item.path | basename }}/device/driver"] - loop: "{{ (_auplc_render_nodes.files | default([])) + (_auplc_card_nodes.files | default([])) }}" - register: _auplc_drm_node_drivers - changed_when: false - failed_when: false - when: _auplc_target_root | length == 0 - -- name: Select AMD live DRM render nodes - ansible.builtin.set_fact: - _auplc_amd_render_nodes: >- - {{ (_auplc_amd_render_nodes | default([])) + - ([item.item.path] if item.rc == 0 and (item.stdout | basename) == 'amdgpu' and - (item.item.path | basename) is match('^renderD') else []) }} - loop: "{{ _auplc_drm_node_drivers.results | default([]) }}" - when: _auplc_target_root | length == 0 - -- name: Select AMD live DRM card nodes - ansible.builtin.set_fact: - _auplc_amd_card_nodes: >- - {{ (_auplc_amd_card_nodes | default([])) + - ([item.item.path] if item.rc == 0 and (item.stdout | basename) == 'amdgpu' and - (item.item.path | basename) is match('^card') else []) }} - loop: "{{ _auplc_drm_node_drivers.results | default([]) }}" - when: _auplc_target_root | length == 0 - -- name: Require AMD live DRM render nodes - ansible.builtin.assert: - that: (_auplc_amd_render_nodes | default([])) | length > 0 - fail_msg: No AMD renderD node was available for GPU access verification. - when: _auplc_target_root | length == 0 - -- name: Require AMD live DRM card nodes - ansible.builtin.assert: - that: (_auplc_amd_card_nodes | default([])) | length > 0 - fail_msg: No AMD card node was available for GPU access verification. - when: _auplc_target_root | length == 0 - -- name: Inspect AMD live DRM render nodes - ansible.builtin.stat: - path: "{{ item }}" - follow: false - loop: "{{ _auplc_amd_render_nodes | default([]) }}" - register: _auplc_amd_render_node_stats - when: _auplc_target_root | length == 0 - -- name: Inspect AMD live DRM card nodes - ansible.builtin.stat: - path: "{{ item }}" - follow: false - loop: "{{ _auplc_amd_card_nodes | default([]) }}" - register: _auplc_amd_card_node_stats - when: _auplc_target_root | length == 0 - -- name: Verify AMD render node ownership and mode - ansible.builtin.assert: - that: - - item.stat.exists - - item.stat.ischr - - item.stat.uid == 0 - - item.stat.gr_name == 'render' - - item.stat.mode == '0666' - fail_msg: "AMD render node {{ item.item }} is not root:render with mode 0666." - loop: "{{ _auplc_amd_render_node_stats.results | default([]) }}" - when: _auplc_target_root | length == 0 - -- name: Verify AMD card node ownership and mode - ansible.builtin.assert: - that: - - item.stat.exists - - item.stat.ischr - - item.stat.uid == 0 - - item.stat.gr_name == 'video' - - item.stat.mode == '0666' - fail_msg: "AMD card node {{ item.item }} is not root:video with mode 0666." - loop: "{{ _auplc_amd_card_node_stats.results | default([]) }}" - when: _auplc_target_root | length == 0 + when: + - _auplc_target_root | length == 0 + - _auplc_removed_legacy_gpu_rules.changed diff --git a/deploy/ansible/roles/gpu_access/tasks/preflight.yml b/deploy/ansible/roles/gpu_access/tasks/preflight.yml index 23d448e4..59810a95 100644 --- a/deploy/ansible/roles/gpu_access/tasks/preflight.yml +++ b/deploy/ansible/roles/gpu_access/tasks/preflight.yml @@ -19,7 +19,7 @@ fail_msg: GPU access rootfs must be an existing non-symlink directory. when: _auplc_target_root | length > 0 -- name: Inspect canonical GPU access destination parents +- name: Inspect AMD udev rule destination parents ansible.builtin.stat: path: "{{ _auplc_target_root }}{{ item }}" follow: false @@ -29,68 +29,141 @@ - /etc/udev/rules.d register: _auplc_destination_parent_stats -- name: Reject unsafe canonical GPU access destination parents +- name: Reject unsafe AMD udev rule destination parents ansible.builtin.assert: that: - not item.stat.exists or (item.stat.isdir and not item.stat.islnk) - fail_msg: "Unsafe canonical GPU access destination parent: {{ item.item }}" + fail_msg: "Unsafe AMD udev rule destination parent: {{ item.item }}" loop: "{{ _auplc_destination_parent_stats.results }}" -- name: Inspect canonical GPU access destination +- name: Define recognized project-owned legacy GPU rules + ansible.builtin.set_fact: + _auplc_legacy_gpu_rules: + - path: "{{ _auplc_target_root }}/etc/udev/rules.d/70-kfd.rules" + sha256: + - 79773871430cb63f5a28cf25666e0eccacf2bb27d4d9f48e10d0b05931650cf0 + - path: "{{ _auplc_target_root }}/etc/udev/rules.d/70-amdgpu.rules" + sha256: + - 678b6a1084576de785b47fcfa0c0b3048117a3add62c1fb8dcff83947004005b + - cc5e78a7861477ac5169a4b84edd4e687c1f14b9a88a9557b0c986479ebbaccd + - path: "{{ _auplc_target_root }}/etc/udev/rules.d/70-rocm-devices.rules" + sha256: + - 951fb3d879d2d45b56cfd4cdb0f7ea061a4a0af77d93b9f2a4da9a8c36d20cad + +- name: Inspect AMD udev rule destination ansible.builtin.stat: - path: "{{ _auplc_target_root }}/etc/udev/rules.d/70-auplc-gpu-access.rules" + path: "{{ _auplc_target_root }}{{ auplc_gpu_udev_rule_path }}" follow: false register: _auplc_destination_rule -- name: Reject unsafe canonical GPU access destination +- name: Reject unsafe AMD udev rule destination ansible.builtin.assert: that: - not _auplc_destination_rule.stat.exists or (_auplc_destination_rule.stat.isreg and not _auplc_destination_rule.stat.islnk) - fail_msg: Unsafe canonical GPU access destination. + fail_msg: Unsafe AMD udev rule destination. + +- name: Query installed AMD udev package on live host + ansible.builtin.command: + argv: + - dpkg-query + - --showformat=${Status}\t${Version} + - --show + - "{{ auplc_gpu_udev_package_name }}" + register: _auplc_live_package + changed_when: false + failed_when: false + when: _auplc_target_root | length == 0 + +- name: Query installed AMD udev package in PXE rootfs + ansible.builtin.command: + argv: + - chroot + - "{{ _auplc_target_root }}" + - dpkg-query + - --showformat=${Status}\t${Version} + - --show + - "{{ auplc_gpu_udev_package_name }}" + register: _auplc_rootfs_package + changed_when: false + failed_when: false + when: _auplc_target_root | length > 0 + +- name: Record installed AMD udev package state on live host + ansible.builtin.set_fact: + _auplc_installed_package: "{{ _auplc_live_package }}" + when: _auplc_target_root | length == 0 + +- name: Record installed AMD udev package state in PXE rootfs + ansible.builtin.set_fact: + _auplc_installed_package: "{{ _auplc_rootfs_package }}" + when: _auplc_target_root | length > 0 + +- name: Record whether AMD udev package installation is needed + ansible.builtin.set_fact: + _auplc_gpu_udev_install_needed: >- + {{ _auplc_installed_package.rc != 0 or + _auplc_installed_package.stdout != 'install ok installed' ~ '\t' ~ auplc_gpu_udev_package_version }} + +- name: Query AMD udev rule package ownership on live host before admission + ansible.builtin.command: + argv: + - dpkg-query + - --search + - "{{ auplc_gpu_udev_rule_path }}" + register: _auplc_live_rule_owner + changed_when: false + failed_when: false + when: _auplc_target_root | length == 0 -- name: Read existing canonical GPU access rule +- name: Query AMD udev rule package ownership in PXE rootfs before admission + ansible.builtin.command: + argv: + - chroot + - "{{ _auplc_target_root }}" + - dpkg-query + - --search + - "{{ auplc_gpu_udev_rule_path }}" + register: _auplc_rootfs_rule_owner + changed_when: false + failed_when: false + when: _auplc_target_root | length > 0 + +- name: Record AMD udev rule owner on live host + ansible.builtin.set_fact: + _auplc_existing_rule_owner: "{{ _auplc_live_rule_owner }}" + when: _auplc_target_root | length == 0 + +- name: Record AMD udev rule owner in PXE rootfs + ansible.builtin.set_fact: + _auplc_existing_rule_owner: "{{ _auplc_rootfs_rule_owner }}" + when: _auplc_target_root | length > 0 + +- name: Record whether the AMD udev rule is package-owned + ansible.builtin.set_fact: + _auplc_rule_owned_by_amd_package: >- + {{ _auplc_existing_rule_owner.rc == 0 and + _auplc_existing_rule_owner.stdout == auplc_gpu_udev_package_name + ': ' + auplc_gpu_udev_rule_path }} + +- name: Read existing AMD udev rule ansible.builtin.slurp: - src: "{{ _auplc_target_root }}/etc/udev/rules.d/70-auplc-gpu-access.rules" + src: "{{ _auplc_target_root }}{{ auplc_gpu_udev_rule_path }}" register: _auplc_existing_rule when: _auplc_destination_rule.stat.exists -- name: Define canonical GPU access rule contents +- name: Allow package-owned AMD udev rule convergence ansible.builtin.set_fact: - _auplc_canonical_rule: | - # Managed by auplc-installer: AMD GPU device access. - KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0666" - SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0666" - SUBSYSTEM=="drm", KERNEL=="card*", DRIVERS=="amdgpu", OWNER="root", GROUP="video", MODE="0666" + _auplc_rule_content_admitted: >- + {{ (not _auplc_destination_rule.stat.exists) or + ((_auplc_existing_rule.content | b64decode) == auplc_gpu_udev_rule_content) or + ((_auplc_gpu_udev_install_needed | bool) and + (((_auplc_existing_rule.content | b64decode) | hash('sha256')) in _auplc_legacy_gpu_rules[1].sha256 or + (_auplc_rule_owned_by_amd_package | bool)) }} -- name: Reject unmanaged canonical GPU access rule +- name: Reject modified AMD udev rule before package installation ansible.builtin.assert: - that: (_auplc_existing_rule.content | b64decode) == _auplc_canonical_rule - fail_msg: Unmanaged canonical GPU access rule. - when: _auplc_destination_rule.stat.exists - -- name: Define recognized project-owned legacy GPU rules - ansible.builtin.set_fact: - _auplc_legacy_gpu_rules: - - path: "{{ _auplc_target_root }}/etc/udev/rules.d/70-kfd.rules" - contents: - - "KERNEL==\"kfd\", MODE=\"0666\"\nSUBSYSTEM==\"drm\", KERNEL==\"renderD*\", MODE=\"0666\"\n" - - path: "{{ _auplc_target_root }}/etc/udev/rules.d/70-amdgpu.rules" - contents: - - | - # ROCm device permissions - # Grant render group access to AMD GPU devices - # Reference: https://rocm.docs.amd.com/projects/install-on-linux/en/latest/install/prerequisites.html#using-udev-rules - KERNEL=="kfd", GROUP="render", MODE="0660" - SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660" - - "KERNEL==\"kfd\", MODE=\"0666\"\nKERNEL==\"renderD[0-9]*\", MODE=\"0666\"\n" - - path: "{{ _auplc_target_root }}/etc/udev/rules.d/70-rocm-devices.rules" - contents: - - | - # ROCm device permissions - # Ensure /dev/kfd and /dev/dri/renderD* are accessible by render group - SUBSYSTEM=="kfd", GROUP="render", MODE="0660" - SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0660" + that: _auplc_rule_content_admitted | bool + fail_msg: Existing AMD udev rule is neither the package rule nor a recognized legacy rule. - name: Inspect recognized project-owned legacy GPU rules ansible.builtin.stat: @@ -116,7 +189,24 @@ - name: Reject unexpected legacy GPU rule content ansible.builtin.assert: that: - - (item.content | b64decode) in item.item.item.contents + - >- + ( + item.item.item.path != _auplc_target_root + auplc_gpu_udev_rule_path and + ((item.content | b64decode) | hash('sha256')) in item.item.item.sha256 + ) or + ( + item.item.item.path == _auplc_target_root + auplc_gpu_udev_rule_path and + ( + (item.content | b64decode) == auplc_gpu_udev_rule_content or + ( + (_auplc_gpu_udev_install_needed | bool) and + ( + ((item.content | b64decode) | hash('sha256')) in item.item.item.sha256 or + (_auplc_rule_owned_by_amd_package | bool) + ) + ) + ) + ) fail_msg: "Unexpected legacy GPU rule content: {{ item.item.item.path }}" loop: "{{ _auplc_legacy_gpu_rule_contents.results }}" when: not item.skipped | default(false) diff --git a/deploy/ansible/roles/gpu_access/tasks/verify.yml b/deploy/ansible/roles/gpu_access/tasks/verify.yml new file mode 100644 index 00000000..5e6a3be5 --- /dev/null +++ b/deploy/ansible/roles/gpu_access/tasks/verify.yml @@ -0,0 +1,162 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +--- +- name: Validate GPU access configuration before package verification + ansible.builtin.import_tasks: validate.yml + +- name: Inspect GPU access rootfs before package verification + ansible.builtin.stat: + path: "{{ _auplc_target_root }}" + follow: false + register: _auplc_verify_rootfs + when: _auplc_target_root | length > 0 + +- name: Require regular GPU access rootfs before package verification + ansible.builtin.assert: + that: + - _auplc_verify_rootfs.stat.isdir + - not _auplc_verify_rootfs.stat.islnk + fail_msg: GPU access rootfs must be an existing non-symlink directory. + when: _auplc_target_root | length > 0 + +- name: Inspect AMD udev rule parents before package verification + ansible.builtin.stat: + path: "{{ _auplc_target_root }}{{ item }}" + follow: false + loop: + - /etc + - /etc/udev + - /etc/udev/rules.d + register: _auplc_verify_parent_stats + +- name: Require safe AMD udev rule parents before package verification + ansible.builtin.assert: + that: + - item.stat.exists + - item.stat.isdir + - not item.stat.islnk + fail_msg: "Unsafe AMD udev rule parent: {{ item.item }}" + loop: "{{ _auplc_verify_parent_stats.results }}" + +- name: Inspect retained PXE shipped legacy GPU rules + ansible.builtin.stat: + path: "{{ _auplc_target_root }}{{ item }}" + follow: false + loop: + - /etc/udev/rules.d/70-kfd.rules + - /etc/udev/rules.d/70-rocm-devices.rules + register: _auplc_retained_legacy_gpu_rule_stats + when: auplc_reject_legacy_gpu_rules | default(false) | bool + +- name: Reject retained PXE shipped legacy GPU rules + ansible.builtin.assert: + that: not item.stat.exists + fail_msg: "Retained PXE rootfs has a shipped legacy GPU rule: {{ item.item }}" + loop: "{{ _auplc_retained_legacy_gpu_rule_stats.results | default([]) }}" + when: auplc_reject_legacy_gpu_rules | default(false) | bool + +- name: Query installed AMD udev package version on live host + ansible.builtin.command: + argv: + - dpkg-query + - --showformat=${Status}\t${Version} + - --show + - "{{ auplc_gpu_udev_package_name }}" + register: _auplc_verify_live_package + changed_when: false + failed_when: false + when: _auplc_target_root | length == 0 + +- name: Query installed AMD udev package version in PXE rootfs + ansible.builtin.command: + argv: + - chroot + - "{{ _auplc_target_root }}" + - dpkg-query + - --showformat=${Status}\t${Version} + - --show + - "{{ auplc_gpu_udev_package_name }}" + register: _auplc_verify_rootfs_package + changed_when: false + failed_when: false + when: _auplc_target_root | length > 0 + +- name: Require installed AMD udev package status and exact version on live host + ansible.builtin.assert: + that: + - _auplc_verify_live_package.rc == 0 + - _auplc_verify_live_package.stdout == 'install ok installed' ~ '\t' ~ auplc_gpu_udev_package_version + fail_msg: AMD udev package is not installed at the required version. + when: _auplc_target_root | length == 0 + +- name: Require installed AMD udev package status and exact version in PXE rootfs + ansible.builtin.assert: + that: + - _auplc_verify_rootfs_package.rc == 0 + - _auplc_verify_rootfs_package.stdout == 'install ok installed' ~ '\t' ~ auplc_gpu_udev_package_version + fail_msg: AMD udev package is not installed at the required version. + when: _auplc_target_root | length > 0 + +- name: Query AMD udev rule package ownership on live host + ansible.builtin.command: + argv: + - dpkg-query + - --search + - "{{ auplc_gpu_udev_rule_path }}" + register: _auplc_verify_live_rule_owner + changed_when: false + failed_when: false + when: _auplc_target_root | length == 0 + +- name: Query AMD udev rule package ownership in PXE rootfs + ansible.builtin.command: + argv: + - chroot + - "{{ _auplc_target_root }}" + - dpkg-query + - --search + - "{{ auplc_gpu_udev_rule_path }}" + register: _auplc_verify_rootfs_rule_owner + changed_when: false + failed_when: false + when: _auplc_target_root | length > 0 + +- name: Require package-owned AMD udev rule on live host + ansible.builtin.assert: + that: + - _auplc_verify_live_rule_owner.rc == 0 + - "_auplc_verify_live_rule_owner.stdout == auplc_gpu_udev_package_name + ': ' + auplc_gpu_udev_rule_path" + fail_msg: AMD udev rule is not package-owned by the required package. + when: _auplc_target_root | length == 0 + +- name: Require package-owned AMD udev rule in PXE rootfs + ansible.builtin.assert: + that: + - _auplc_verify_rootfs_rule_owner.rc == 0 + - "_auplc_verify_rootfs_rule_owner.stdout == auplc_gpu_udev_package_name + ': ' + auplc_gpu_udev_rule_path" + fail_msg: AMD udev rule is not package-owned by the required package. + when: _auplc_target_root | length > 0 + +- name: Inspect installed AMD udev rule + ansible.builtin.stat: + path: "{{ _auplc_target_root }}{{ auplc_gpu_udev_rule_path }}" + follow: false + register: _auplc_verify_rule + +- name: Require safe installed AMD udev rule + ansible.builtin.assert: + that: + - _auplc_verify_rule.stat.exists + - _auplc_verify_rule.stat.isreg + - not _auplc_verify_rule.stat.islnk + fail_msg: AMD udev rule has an unsafe filesystem type. + +- name: Read installed AMD udev rule + ansible.builtin.slurp: + src: "{{ _auplc_target_root }}{{ auplc_gpu_udev_rule_path }}" + register: _auplc_verify_rule_content + +- name: Require exact AMD udev rule content + ansible.builtin.assert: + that: (_auplc_verify_rule_content.content | b64decode) == auplc_gpu_udev_rule_content + fail_msg: AMD udev rule is a modified package conffile. diff --git a/deploy/ansible/roles/gpu_access/templates/70-auplc-gpu-access.rules.j2 b/deploy/ansible/roles/gpu_access/templates/70-auplc-gpu-access.rules.j2 deleted file mode 100644 index f57d023a..00000000 --- a/deploy/ansible/roles/gpu_access/templates/70-auplc-gpu-access.rules.j2 +++ /dev/null @@ -1,4 +0,0 @@ -# Managed by auplc-installer: AMD GPU device access. -KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0666" -SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0666" -SUBSYSTEM=="drm", KERNEL=="card*", DRIVERS=="amdgpu", OWNER="root", GROUP="video", MODE="0666" diff --git a/tests/skills/test_gpu_access_role.py b/tests/skills/test_gpu_access_role.py index ccece9ce..3fe821ec 100644 --- a/tests/skills/test_gpu_access_role.py +++ b/tests/skills/test_gpu_access_role.py @@ -1,6 +1,6 @@ # Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. -"""Contract tests for the Ansible GPU device-mode role.""" +"""Contract tests for AMD's packaged GPU udev rules in Ansible.""" from pathlib import Path @@ -10,33 +10,48 @@ PXE_CONTROLLER_ROLE = ANSIBLE / "roles" / "pxe_controller" PXE_GPU_ACCESS_TASKS = PXE_CONTROLLER_ROLE / "tasks" / "gpu_access.yml" +PACKAGE = "amdgpu-insecure-instinct-udev-rules" +VERSION = "30.30.4.0-2341068.24.04" +FILENAME = f"{PACKAGE}_{VERSION}_all.deb" +URL = f"https://repo.radeon.com/amdgpu/30.30.4/ubuntu/pool/main/a/{PACKAGE}/{FILENAME}" +SHA256 = "4be865985c7a13114c45925e77bc0b411b9fd47d5040ed35df44b9c411766162" +RULE_PATH = "/etc/udev/rules.d/70-amdgpu.rules" +RULE_CONTENT = ( + 'KERNEL=="kfd", GROUP="render", MODE="0666"\nSUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0666"\n' +) + def read(path: Path) -> str: return path.read_text(encoding="utf-8") -def test_gpu_access_role_uses_shc_proven_device_mode_contract() -> None: +def test_gpu_access_role_pins_the_official_amd_package_contract() -> None: defaults = read(GPU_ACCESS_ROLE / "defaults" / "main.yml") preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") apply = read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") - rules = read(GPU_ACCESS_ROLE / "templates" / "70-auplc-gpu-access.rules.j2") - - assert "auplc_gpu_access_enabled: false" in defaults - assert 'auplc_rootfs_path: ""' in defaults - assert "auplc_render_gid" not in defaults - assert "normalize" not in defaults - assert "gpu-access.json" not in preflight - assert "groupmod" not in apply - assert "GID collision" not in preflight - assert rules == ( - "# Managed by auplc-installer: AMD GPU device access.\n" - 'KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0666"\n' - 'SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0666"\n' - 'SUBSYSTEM=="drm", KERNEL=="card*", DRIVERS=="amdgpu", OWNER="root", GROUP="video", MODE="0666"\n' - ) - - -def test_gpu_access_role_preserves_safe_preflight_and_exact_legacy_admission() -> None: + verify = read(GPU_ACCESS_ROLE / "tasks" / "verify.yml") + + assert PACKAGE in defaults + assert VERSION in defaults + assert FILENAME in defaults + assert URL in defaults + assert f"sha256:{SHA256}" in defaults + assert RULE_PATH in defaults + assert " " + RULE_CONTENT.replace("\n", "\n ").rstrip() in defaults + assert "dpkg-query" in preflight + assert r"--showformat=${Status}\t${Version}" in preflight + assert "ansible.builtin.get_url" in apply + assert "ansible.builtin.apt" in apply + assert 'checksum: "{{ auplc_gpu_udev_package_checksum }}"' in apply + assert 'deb: "{{ auplc_gpu_udev_package_cache_path }}"' in apply + assert "dpkg-query" in verify + assert r"--showformat=${Status}\t${Version}" in verify + assert "--search" in verify + assert "package-owned" in verify + assert "modified package conffile" in verify + + +def test_gpu_access_role_preserves_rootfs_and_exact_legacy_safety() -> None: validation = read(GPU_ACCESS_ROLE / "tasks" / "validate.yml") preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") apply = read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") @@ -46,91 +61,165 @@ def test_gpu_access_role_preserves_safe_preflight_and_exact_legacy_admission() - assert "_auplc_canonical_allowed_root" in validation assert "Inspect GPU access rootfs target" in preflight assert "follow: false" in preflight - assert "Reject unsafe canonical GPU access destination parents" in preflight - assert "Reject unsafe canonical GPU access destination" in preflight + assert "Reject unsafe AMD udev rule destination parents" in preflight + assert "Reject unsafe AMD udev rule destination" in preflight assert "Define recognized project-owned legacy GPU rules" in preflight - assert "Reject unexpected legacy GPU rule content" in preflight + assert "hash('sha256')" in preflight assert "70-kfd.rules" in preflight - assert "70-amdgpu.rules" in preflight assert "70-rocm-devices.rules" in preflight + assert "70-auplc-gpu-access.rules" not in preflight + assert "Reject unexpected legacy GPU rule content" in preflight + assert "Recheck recognized project-owned legacy GPU rules before apply" in apply assert "Remove recognized project-owned legacy GPU rules" in apply - assert apply.index("Reject unexpected legacy GPU rule content before apply") < apply.index( + assert apply.index("Download checksummed AMD udev package") < apply.index( + "Remove recognized project-owned legacy GPU rules" + ) + assert apply.index("Verify installed AMD udev package") < apply.index( "Remove recognized project-owned legacy GPU rules" ) + assert "Reload live udev rules after legacy cleanup" in apply + assert "Trigger live udev rules after legacy cleanup" in apply -def test_gpu_access_role_reconciles_and_verifies_live_devices_only() -> None: +def test_gpu_access_role_skips_package_cache_and_download_when_exact_version_is_installed() -> None: + preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") apply = read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") - assert "Reload live udev rules on every apply" in apply - assert "Trigger live udev rules on every apply" in apply - assert "Settle live udev events before inode verification" in apply - assert "Inspect /dev/kfd after live reconciliation" in apply - assert "Verify /dev/kfd ownership and mode" in apply - assert "Find live DRM render nodes" in apply - assert "Verify AMD render node ownership and mode" in apply - assert "Find live DRM card nodes" in apply - assert "Verify AMD card node ownership and mode" in apply - assert "/sys/class/drm" in apply - assert "readlink" in apply - assert "basename" in apply - assert "_auplc_kfd.stat.mode == '0666'" in apply - assert "item.stat.mode == '0666'" in apply - assert "item.stat.mode == '0666'" in apply - assert "item.stat.gr_name == 'render'" in apply - assert "item.stat.gr_name == 'video'" in apply - assert ( - apply.index("Trigger live udev rules on every apply") - < apply.index("Settle live udev events before inode verification") - < apply.index("Inspect /dev/kfd after live reconciliation") - ) - assert "when: _auplc_target_root | length == 0" in apply - assert "gpu-access.json" not in apply + assert "_auplc_gpu_udev_install_needed" in preflight + assert "Install AMD udev package when required" in apply + install_block = apply.split("Install AMD udev package when required", maxsplit=1)[1] + assert "Create deterministic AMD udev package cache" in install_block + assert "Download checksummed AMD udev package" in install_block + assert "when: _auplc_gpu_udev_install_needed | bool" in install_block + assert "Verify installed AMD udev package without installation" in apply + assert "install ok installed" in preflight -def test_gpu_access_role_rejects_noncanonical_managed_rule_content() -> None: +def test_gpu_access_role_requires_installed_status_and_exact_version() -> None: + verify = read(GPU_ACCESS_ROLE / "tasks" / "verify.yml") + + assert "install ok installed" in verify + assert "Require installed AMD udev package status and exact version" in verify + + +def test_preflight_allows_package_owned_wrong_version_rule_for_convergence() -> None: + preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") + + assert "Query AMD udev rule package ownership on live host before admission" in preflight + assert "Query AMD udev rule package ownership in PXE rootfs before admission" in preflight + assert preflight.index("Query installed AMD udev package") < preflight.index("Read existing AMD udev rule") + assert preflight.index("Query AMD udev rule package ownership") < preflight.index("Read existing AMD udev rule") + assert "_auplc_gpu_udev_install_needed | bool" in preflight + assert "_auplc_rule_owned_by_amd_package | bool" in preflight + + +def test_preflight_allows_package_owned_partial_state_rule_for_convergence() -> None: preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") + + assert "Record whether AMD udev package installation is needed" in preflight + assert "Allow package-owned AMD udev rule convergence" in preflight + assert "install ok installed" in preflight + + +def test_preflight_rejects_unknown_unowned_rule_content() -> None: + preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") + + assert "Reject modified AMD udev rule before package installation" in preflight + assert "_auplc_rule_owned_by_amd_package | bool" in preflight + assert "Existing AMD udev rule is neither the package rule nor a recognized legacy rule." in preflight + + +def test_preflight_legacy_admission_matches_package_owned_convergence_admission() -> None: + preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") + primary_admission = preflight.split("Allow package-owned AMD udev rule convergence", maxsplit=1)[1].split( + "Reject modified AMD udev rule before package installation", maxsplit=1 + )[0] + legacy_admission = preflight.split("Reject unexpected legacy GPU rule content", maxsplit=1)[1].split( + "fail_msg:", maxsplit=1 + )[0] + + assert "_auplc_gpu_udev_install_needed | bool" in primary_admission + assert "_auplc_rule_owned_by_amd_package | bool" in primary_admission + assert "_auplc_gpu_udev_install_needed | bool" in legacy_admission + assert "_auplc_rule_owned_by_amd_package | bool" in legacy_admission + assert "auplc_gpu_udev_rule_path" in legacy_admission + assert "auplc_gpu_udev_rule_content" in legacy_admission + + +def test_gpu_access_role_installs_the_package_without_custom_rule_or_device_probes() -> None: apply = read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") - pxe_tasks = read(PXE_GPU_ACCESS_TASKS) - assert "_auplc_previous_canonical_rule" not in preflight - assert "(_auplc_existing_rule.content | b64decode) == _auplc_canonical_rule" in preflight - assert "Reject unmanaged canonical GPU access rule" in preflight - assert "_auplc_apply_previous_canonical_rule" not in apply - assert "Recheck canonical GPU access rule before apply" in apply - assert "(_auplc_apply_existing_rule.content | b64decode) == _auplc_apply_canonical_rule" in apply - assert "Unmanaged canonical GPU access rule." in apply - assert "_pxe_retained_previous_canonical_rule" not in pxe_tasks - assert "(_pxe_retained_canonical_gpu_rule.content | b64decode) == _pxe_retained_canonical_rule" in pxe_tasks - assert "non-canonical GPU access rule" in pxe_tasks + assert not (GPU_ACCESS_ROLE / "templates" / "70-auplc-gpu-access.rules.j2").exists() + assert "ansible.builtin.template" not in apply + assert "70-auplc-gpu-access.rules.j2" not in apply + assert "udevadm settle" not in apply + assert "/dev/kfd" not in apply + assert "/dev/dri" not in apply + assert "/sys/class/drm" not in apply + assert "card" not in apply -def test_pxe_gpu_access_installs_rules_without_gid_or_state_contract() -> None: +def test_gpu_access_role_verifies_exact_installed_package_version_and_rule() -> None: + verify = read(GPU_ACCESS_ROLE / "tasks" / "verify.yml") + + assert "auplc_gpu_udev_package_version" in verify + assert "auplc_gpu_udev_rule_path" in verify + assert "auplc_gpu_udev_rule_content" in verify + assert "Require installed AMD udev package status and exact version" in verify + assert "Require package-owned AMD udev rule" in verify + assert "Require exact AMD udev rule content" in verify + assert "follow: false" in verify + + +def test_pxe_gpu_access_uses_safe_chroot_install_and_strict_retained_admission() -> None: main = read(PXE_CONTROLLER_ROLE / "tasks" / "main.yml") tasks = read(PXE_GPU_ACCESS_TASKS) + apply = read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") + verify = read(GPU_ACCESS_ROLE / "tasks" / "verify.yml") assert "Admit retained PXE GPU rootfs read-only before lifecycle changes" in main assert "Re-preflight PXE GPU rootfs before TFTP" in main assert main.index("Admit retained PXE GPU rootfs read-only before lifecycle changes") < main.index( "Stop NFS before rootfs rebuild" ) - assert "Inspect retained PXE canonical GPU access parents" in tasks - assert "Require retained PXE canonical GPU access parents" in tasks - assert "Require retained PXE canonical GPU rule" in tasks + assert "tasks_from: verify" in tasks assert "tasks_from: preflight" in tasks assert "tasks_from: apply" in tasks assert 'auplc_rootfs_path: "{{ pxe_nfs_root }}"' in tasks assert 'auplc_rootfs_allowed_root: "{{ pxe_nfs_allowed_root }}"' in tasks - assert "auplc_render_gid" not in tasks - assert "render_gid" not in tasks - assert "groupadd" not in tasks - assert "groupmod" not in tasks - assert "collision" not in tasks.lower() - assert "gpu-access.json" not in tasks + assert "auplc_reject_legacy_gpu_rules: true" in tasks + assert "Reject retained PXE shipped legacy GPU rules" in verify + assert "chroot" in apply + assert "apt-get" in apply + assert "Copy AMD udev package into PXE rootfs" in apply + assert "Mount virtual filesystems for AMD udev package installation" not in apply + assert "mount --bind" not in apply + assert "Unmount virtual filesystems after AMD udev package installation" not in apply + assert apply.index("Verify installed AMD udev package") < apply.index( + "Remove temporary AMD udev package from PXE rootfs" + ) + assert main.index("Re-preflight PXE GPU rootfs before TFTP") < main.index("Find latest kernel in rootfs") + assert RULE_CONTENT not in tasks assert "/dev/kfd" not in tasks assert "/dev/dri" not in tasks +def test_pxe_rootfs_unmounts_fail_on_real_errors_but_skip_absent_mounts() -> None: + main = read(PXE_CONTROLLER_ROLE / "tasks" / "main.yml") + + rootfs_removal = main.split("Remove existing rootfs (force rebuild)", maxsplit=1)[1].split( + "Check if NFS rootfs already exists", maxsplit=1 + )[0] + chroot_unmount = main.split("Unmount virtual filesystems from chroot", maxsplit=1)[1].split( + "Remove chroot setup script", maxsplit=1 + )[0] + for task in (rootfs_removal, chroot_unmount): + assert "set -e" in task + assert "if mountpoint -q" in task + assert "&& umount" not in task + assert "|| true" not in task + + def test_gpu_access_playbooks_keep_two_phase_live_and_rootfs_safety() -> None: rocm_playbook = read(ANSIBLE / "playbooks" / "pb-rocm.yml") udev_playbook = read(ANSIBLE / "playbooks" / "pb-udev.yml") @@ -145,15 +234,7 @@ def test_gpu_access_playbooks_keep_two_phase_live_and_rootfs_safety() -> None: assert "render_gid" not in pxe_playbook -def test_pxe_controller_playbook_has_no_obsolete_finalizer_post_tasks() -> None: - pxe_playbook = read(ANSIBLE / "playbooks" / "pb-pxe-controller.yml") - - assert "post_tasks:" not in pxe_playbook - assert "pxe_finalizer_" not in pxe_playbook - assert "--finalize-pxe" not in pxe_playbook - - -def test_deploy_ansible_has_no_render_gid_normalization_or_gpu_state_contract() -> None: +def test_deploy_ansible_has_no_obsolete_gpu_access_policy_or_state_contract() -> None: forbidden = ( "auplc_render_gid", "auplc_normalize_render_gid", @@ -162,7 +243,6 @@ def test_deploy_ansible_has_no_render_gid_normalization_or_gpu_state_contract() "groupmod", "render GID collision", ) - ansible_text = "\n".join( path.read_text(encoding="utf-8") for path in ANSIBLE.rglob("*") From 4e15da9e7cc2b9af9c29a289f534a1077b2a74d1 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 19:08:06 +0800 Subject: [PATCH 45/65] fix(pxe): install GPU udev package safely --- .../roles/pxe_controller/tasks/gpu_access.yml | 80 +++---------------- .../roles/pxe_controller/tasks/main.yml | 26 ++++-- 2 files changed, 31 insertions(+), 75 deletions(-) diff --git a/deploy/ansible/roles/pxe_controller/tasks/gpu_access.yml b/deploy/ansible/roles/pxe_controller/tasks/gpu_access.yml index d0a0bd5b..adec7028 100644 --- a/deploy/ansible/roles/pxe_controller/tasks/gpu_access.yml +++ b/deploy/ansible/roles/pxe_controller/tasks/gpu_access.yml @@ -10,75 +10,17 @@ that: pxe_gpu_admission_phase in ['retained-read-only', 'final'] fail_msg: PXE GPU admission phase is invalid. -- name: Inspect retained PXE canonical GPU access parents - ansible.builtin.stat: - path: "{{ pxe_nfs_root }}{{ item }}" - follow: false - loop: - - /etc - - /etc/udev - - /etc/udev/rules.d - register: _pxe_retained_canonical_gpu_parent_stats - when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' - -- name: Require retained PXE canonical GPU access parents - ansible.builtin.assert: - that: - - item.stat.exists - - item.stat.isdir - - not item.stat.islnk - fail_msg: "Retained PXE rootfs has an unsafe canonical GPU access parent: {{ item.item }}" - loop: "{{ _pxe_retained_canonical_gpu_parent_stats.results }}" - when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' - -- name: Inspect retained PXE GPU policy paths - ansible.builtin.stat: - path: "{{ pxe_nfs_root }}{{ item }}" - follow: false - loop: - - /etc/udev/rules.d/70-kfd.rules - - /etc/udev/rules.d/70-amdgpu.rules - - /etc/udev/rules.d/70-rocm-devices.rules - - /etc/udev/rules.d/70-auplc-gpu-access.rules - register: _pxe_retained_gpu_policy_stats - when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' - -- name: Require retained PXE legacy GPU rules absent - ansible.builtin.assert: - that: not item.stat.exists - fail_msg: "Retained PXE rootfs has a legacy GPU rule requiring a separate migration: {{ item.item }}" - loop: "{{ _pxe_retained_gpu_policy_stats.results[:3] }}" - when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' - -- name: Require retained PXE canonical GPU rule destination - ansible.builtin.assert: - that: - - _pxe_retained_gpu_policy_stats.results[3].stat.exists - - _pxe_retained_gpu_policy_stats.results[3].stat.isreg - - not _pxe_retained_gpu_policy_stats.results[3].stat.islnk - fail_msg: Retained PXE rootfs requires an exact canonical GPU access rule. - when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' - -- name: Read retained PXE canonical GPU rule - ansible.builtin.slurp: - src: "{{ _pxe_retained_gpu_policy_stats.results[3].item }}" - register: _pxe_retained_canonical_gpu_rule - when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' - -- name: Define retained PXE canonical GPU rules - ansible.builtin.set_fact: - _pxe_retained_canonical_rule: | - # Managed by auplc-installer: AMD GPU device access. - KERNEL=="kfd", OWNER="root", GROUP="render", MODE="0666" - SUBSYSTEM=="drm", KERNEL=="renderD*", DRIVERS=="amdgpu", OWNER="root", GROUP="render", MODE="0666" - SUBSYSTEM=="drm", KERNEL=="card*", DRIVERS=="amdgpu", OWNER="root", GROUP="video", MODE="0666" - when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' - -- name: Require retained PXE canonical GPU rule - ansible.builtin.assert: - that: (_pxe_retained_canonical_gpu_rule.content | b64decode) == _pxe_retained_canonical_rule - fail_msg: Retained PXE rootfs has a non-canonical GPU access rule. - when: pxe_gpu_access_enabled | bool and _pxe_rootfs_disposition == 'retained' +- name: Verify retained PXE AMD udev package before lifecycle changes + ansible.builtin.include_role: + name: gpu_access + tasks_from: verify + vars: + auplc_rootfs_path: "{{ pxe_nfs_root }}" + auplc_rootfs_allowed_root: "{{ pxe_nfs_allowed_root }}" + auplc_reject_legacy_gpu_rules: true + when: + - pxe_gpu_access_enabled | bool + - _pxe_rootfs_disposition == 'retained' - name: Preflight GPU access after final PXE re-preflight ansible.builtin.include_role: diff --git a/deploy/ansible/roles/pxe_controller/tasks/main.yml b/deploy/ansible/roles/pxe_controller/tasks/main.yml index b5217f60..1a34b0e1 100644 --- a/deploy/ansible/roles/pxe_controller/tasks/main.yml +++ b/deploy/ansible/roles/pxe_controller/tasks/main.yml @@ -172,9 +172,16 @@ - name: Remove existing rootfs (force rebuild) when: pxe_rootfs_force_rebuild | bool ansible.builtin.shell: | - mountpoint -q {{ pxe_nfs_root }}/dev && umount {{ pxe_nfs_root }}/dev || true - mountpoint -q {{ pxe_nfs_root }}/sys && umount {{ pxe_nfs_root }}/sys || true - mountpoint -q {{ pxe_nfs_root }}/proc && umount {{ pxe_nfs_root }}/proc || true + set -e + if mountpoint -q {{ pxe_nfs_root }}/dev; then + umount {{ pxe_nfs_root }}/dev + fi + if mountpoint -q {{ pxe_nfs_root }}/sys; then + umount {{ pxe_nfs_root }}/sys + fi + if mountpoint -q {{ pxe_nfs_root }}/proc; then + umount {{ pxe_nfs_root }}/proc + fi rm -rf {{ pxe_nfs_root }} changed_when: true @@ -299,9 +306,16 @@ always: - name: Unmount virtual filesystems from chroot ansible.builtin.shell: | - mountpoint -q {{ pxe_nfs_root }}/dev && umount {{ pxe_nfs_root }}/dev || true - mountpoint -q {{ pxe_nfs_root }}/sys && umount {{ pxe_nfs_root }}/sys || true - mountpoint -q {{ pxe_nfs_root }}/proc && umount {{ pxe_nfs_root }}/proc || true + set -e + if mountpoint -q {{ pxe_nfs_root }}/dev; then + umount {{ pxe_nfs_root }}/dev + fi + if mountpoint -q {{ pxe_nfs_root }}/sys; then + umount {{ pxe_nfs_root }}/sys + fi + if mountpoint -q {{ pxe_nfs_root }}/proc; then + umount {{ pxe_nfs_root }}/proc + fi changed_when: true - name: Remove chroot setup script From 04d8ee3baf544ade9fb0d1bee6d1a8d5c07be8aa Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 19:08:06 +0800 Subject: [PATCH 46/65] docs: describe AMD GPU udev package --- README.md | 7 +++++++ runtime/values.yaml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f19aa4b8..566443c1 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,13 @@ This operation needs root privileges. Requesting sudo password... kubectl is configured at $HOME/.kube/config; try `kubectl get nodes` ``` +The GPU access stage installs AMD's `amdgpu-insecure-instinct-udev-rules` +package, pinned to `30.30.4.0-2341068.24.04`. It sets mode `0666` only on +`/dev/kfd` and DRM `renderD*` nodes; `card*` keeps the normal system policy. The +device plugin remains a separate allocation layer, and the tested ROCm compute +path needs no supplemental GPU group. The offline `pack` bundle carries the +pinned deb for installation without network access. + See the full guide at [Quick Start](https://amdresearch.github.io/aup-learning-cloud/installation/quick-start.html) and [Single-Node Deployment](https://amdresearch.github.io/aup-learning-cloud/installation/single-node.html). ### Uninstall diff --git a/runtime/values.yaml b/runtime/values.yaml index 3dd59180..2a942e02 100644 --- a/runtime/values.yaml +++ b/runtime/values.yaml @@ -669,7 +669,7 @@ monitoring: singleuser: # Storage ownership only. AUPLC runtime does not inject GPU groups. - # An amd.com/gpu request is the device-visibility boundary; injected nodes are 0666. + # amd.com/gpu requests allocate devices; host udev policy controls node modes. fsGid: 100 storage: From 5e898d9f43be03d667d85380d40f0e0bc5309953 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 19:08:06 +0800 Subject: [PATCH 47/65] docs(deploy): document AMD GPU udev package --- deploy/README.md | 34 +++++++++++++++++++--------------- deploy/ansible/README.md | 18 ++++++++++-------- deploy/k8s/README.md | 7 +++++++ 3 files changed, 36 insertions(+), 23 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index b53d0a5d..7edbffc8 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -105,16 +105,20 @@ records under `/sys/bus/pci/devices`; it does not require the devices to be attached to `amdgpu` before ROCm installation. The resulting GPU resolution report records which managed hosts have AMD display hardware. -The GPU permission contract is fixed across GPU hosts and PXE root filesystems: - -- `/dev/kfd` and AMD `/dev/dri/renderD*` nodes are `root:render` with mode `0666`. -- AMD `/dev/dri/card*` nodes are `root:video` with mode `0666`. -- Every GPU device node injected into a Pod therefore has mode `0666`. -- Host provisioning owns device-node discretionary access control. -- AUPLC Hub adds no GPU supplemental group to user Pods. -- AMD device-plugin allocation is the visibility boundary: only Pods that - request `amd.com/gpu` receive GPU device nodes. The plugin does not set Unix - ownership or modes on host device nodes. +The installer, Ansible GPU access role, and PXE controller install AMD's +`amdgpu-insecure-instinct-udev-rules` package, pinned to version +`30.30.4.0-2341068.24.04`. Its package-owned rule sets mode `0666` only on +`/dev/kfd` and DRM `/dev/dri/renderD*` nodes. It does not match +`/dev/dri/card*`; card nodes retain the normal system policy, observed as +`root:video 0660`. + +This host permission policy is separate from Kubernetes allocation. The AMD +device plugin remains the visibility boundary: only Pods that request +`amd.com/gpu` receive allocated GPU devices, and the plugin does not change +host inode ownership or mode. AUPLC Hub adds no GPU supplemental group. The +tested ROCm compute path needs none: on both SHC GPU nodes, `rocminfo` succeeded +as UID `12345` with only supplemental GID `100`, while card nodes remained +inaccessible at mode `0660`. The reported agents were `gfx1151` and `gfx1200`. `singleuser.fsGid: 100` controls shared notebook storage ownership only. It is not part of GPU access and must not be treated as a GPU group setting. @@ -157,10 +161,10 @@ helm upgrade --install jupyterhub ./runtime/chart \ -f runtime/values-basic-example.yaml ``` -A fresh PXE rootfs receives the fixed udev rule during the controller playbook. -A retained rootfs is accepted only when it already contains that exact canonical -rule and no conflicting legacy GPU rule. Rebuild or correct a retained rootfs -separately if that safety check fails. +A fresh PXE rootfs receives the pinned AMD udev package during the controller +playbook. A retained rootfs is accepted only when that exact package version and +its unmodified package-owned rule are present, with no conflicting legacy GPU +rule. Rebuild or correct a retained rootfs separately if that safety check fails. #### Discovery failures @@ -169,7 +173,7 @@ separately if that safety check fails. | Host is unreachable | Restore passwordless root SSH to that inventory host, then regenerate. | | `lspci` is missing or fails | Install `pciutils` on the reported host and rerun generation. | | Host evidence is `UNKNOWN` or AMD GPU BDF probes disagree | Compare AMD display BDFs from `lspci` with vendor `0x1002` display-class devices under `/sys/bus/pci/devices`; fix missing or inconsistent PCI enumeration, then regenerate. | -| Retained PXE rootfs has a legacy or non-canonical GPU rule | Rebuild the rootfs, or replace the conflicting rule through a separate reviewed maintenance action before rerunning the playbook. | +| Retained PXE rootfs has the wrong AMD udev package version, a modified package rule, or a conflicting legacy GPU rule | Rebuild the rootfs, or correct the package state through a separate reviewed maintenance action before rerunning the playbook. | ## Deployment branch boundary diff --git a/deploy/ansible/README.md b/deploy/ansible/README.md index eda9b349..cbd02ad1 100644 --- a/deploy/ansible/README.md +++ b/deploy/ansible/README.md @@ -32,14 +32,16 @@ hosts from managed-host evidence. PXE generation uses only `pxe.diskless_agents_have_amd_gpus` and writes canonical files before the controller playbook runs. -The GPU access role sets AMD device-node policy on GPU hosts and GPU-enabled PXE -root filesystems. `/dev/kfd` and AMD `renderD*` nodes are `root:render 0666`; -AMD `card*` nodes are `root:video 0666`. All injected GPU device nodes therefore -use mode `0666`. Device-plugin allocation is the visibility boundary: only Pods -requesting `amd.com/gpu` receive the nodes. The plugin does not change host inode -permissions, and AUPLC Hub adds no GPU supplemental group to user Pods. Ordinary -container group membership does not participate in GPU permissions; host -provisioning owns device-node discretionary access control. +The GPU access role installs AMD's `amdgpu-insecure-instinct-udev-rules` +package, pinned to `30.30.4.0-2341068.24.04`, on GPU hosts and GPU-enabled PXE +root filesystems. The package sets mode `0666` only on `/dev/kfd` and DRM +`renderD*` nodes. It does not change `card*` nodes, which retain normal system +policy, observed as `root:video 0660`. + +Device-plugin allocation is a separate layer and remains the visibility +boundary for Pods requesting `amd.com/gpu`; it does not change host inode +permissions. AUPLC Hub adds no GPU supplemental group. No GPU group was needed +for the tested ROCm compute path. ## Prerequisites diff --git a/deploy/k8s/README.md b/deploy/k8s/README.md index da148f52..7334c198 100644 --- a/deploy/k8s/README.md +++ b/deploy/k8s/README.md @@ -45,6 +45,13 @@ cluster infrastructure prerequisites owned outside AUPLC. The infrastructure owner must select, deploy, and maintain them according to the [official AMD Kubernetes device plugin project](https://github.com/ROCm/k8s-device-plugin). +The device plugin allocates devices to Pods; it does not set host device-node +permissions. Host provisioning separately installs the pinned +`amdgpu-insecure-instinct-udev-rules` package at version +`30.30.4.0-2341068.24.04`. That package sets mode `0666` only on `/dev/kfd` and +DRM `renderD*` nodes and leaves `card*` under normal system policy. AUPLC adds +no supplemental GPU group; none is required for the tested ROCm compute path. + To install the same pinned manifests used by `auplc-installer`: ```bash From 3cf219b707e63a8b015fd8e99a7f1b945d749e18 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 19:08:06 +0800 Subject: [PATCH 48/65] docs(skills): align AMD GPU package workflow --- skills/deploy-aup-learning-cloud/SKILL.md | 20 +++++++++++-------- skills/deploy-aup-learning-cloud/reference.md | 19 ++++++++++++------ .../reference.md | 4 ++++ 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/skills/deploy-aup-learning-cloud/SKILL.md b/skills/deploy-aup-learning-cloud/SKILL.md index 1799c447..2c88018a 100644 --- a/skills/deploy-aup-learning-cloud/SKILL.md +++ b/skills/deploy-aup-learning-cloud/SKILL.md @@ -87,14 +87,18 @@ node labeller as infrastructure prerequisites owned outside AUPLC. Verify both existing DaemonSets and advertised GPU capacity before Helm; do not install these privileged components as part of the AUPLC procedure. -Keep the GPU contract distinct from storage configuration. GPU hosts use -`root:render 0666` for `/dev/kfd` and AMD `renderD*`, and `root:video 0666` for -AMD `card*`, so every injected GPU device node has mode `0666`. Device-plugin -allocation is the visibility boundary and only `amd.com/gpu` requests receive -GPU nodes. Container group membership does not participate in GPU permissions; -host provisioning owns device-node discretionary access control. AUPLC Hub adds -no GPU supplemental group, and the plugin does not change Unix inode permissions. -`singleuser.fsGid: 100` is for shared storage only. +Keep the GPU contract distinct from storage configuration. The installer, +Ansible role, and PXE controller install AMD's +`amdgpu-insecure-instinct-udev-rules` package at the pinned version +`30.30.4.0-2341068.24.04`. Its rule sets mode `0666` only on `/dev/kfd` and DRM +`renderD*` nodes. It does not change `card*`, which retains normal system policy, +observed as `root:video 0660`. + +Device-plugin allocation is a separate visibility layer. Only `amd.com/gpu` +requests receive allocated GPU devices, and the plugin does not change Unix +inode permissions. AUPLC Hub adds no GPU supplemental group; none is required +for the tested ROCm compute path. `singleuser.fsGid: 100` is for shared storage +only. ## Phase 4: Verify diff --git a/skills/deploy-aup-learning-cloud/reference.md b/skills/deploy-aup-learning-cloud/reference.md index ff7d5bd5..9c07ed68 100644 --- a/skills/deploy-aup-learning-cloud/reference.md +++ b/skills/deploy-aup-learning-cloud/reference.md @@ -30,17 +30,24 @@ Generation and validation must finish before Ansible or Helm changes are made. ## GPU permission contract -- Host `/dev/kfd` and AMD `/dev/dri/renderD*` nodes are `root:render 0666`. -- Host AMD `/dev/dri/card*` nodes are `root:video 0666`. -- Every GPU device node injected into a Pod has mode `0666`. -- Container group membership does not participate in GPU permissions; host - provisioning owns device-node discretionary access control. -- AUPLC Hub adds no GPU supplemental group to user Pods. +- Installer, Ansible, and PXE provisioning install AMD's + `amdgpu-insecure-instinct-udev-rules` package at version + `30.30.4.0-2341068.24.04`. +- The package sets mode `0666` only on `/dev/kfd` and DRM + `/dev/dri/renderD*` nodes. +- The package does not change `/dev/dri/card*`. Card nodes retain normal system + policy, observed as `root:video 0660`. +- AUPLC Hub adds no GPU supplemental group. No GPU group is required for the + tested ROCm compute path. - AMD device-plugin allocation is the visibility boundary. Only Pods requesting `amd.com/gpu` receive GPU device nodes; the plugin does not change host inode ownership or mode. - `singleuser.fsGid: 100` controls shared storage ownership only. +Operator evidence from SHC showed `rocminfo` reporting `gfx1151` and `gfx1200` +on the two GPU nodes from UID `12345` Pods with only supplemental GID `100`. +Their `card*` nodes remained inaccessible at mode `0660`. + The infrastructure owner deploys and maintains the AMD device plugin and ROCm node labeller outside AUPLC. Before Helm, use the readiness and capacity checks in [deploy/README.md](../../deploy/README.md); do not install these privileged diff --git a/skills/install-aup-learning-cloud-single-node/reference.md b/skills/install-aup-learning-cloud-single-node/reference.md index 514da9f9..656aac83 100644 --- a/skills/install-aup-learning-cloud-single-node/reference.md +++ b/skills/install-aup-learning-cloud-single-node/reference.md @@ -109,6 +109,10 @@ cd auplc-bundle-gfx1151-* sudo ./auplc-installer install ``` +The bundle includes the pinned +`amdgpu-insecure-instinct-udev-rules_30.30.4.0-2341068.24.04_all.deb`; offline +installation verifies and installs it from the bundle. + ## Troubleshooting | Symptom | Likely cause | First checks | From 544e29c722d67d57cf7f84af3af1dc54afcaee43 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 20:23:54 +0800 Subject: [PATCH 49/65] feat(deploy): validate direct SSH inventory --- .../scripts/gpu_resolution_validation.py | 8 ++ .../scripts/helm_validation.py | 37 ++++++ .../scripts/validate.py | 55 ++++----- .../test_direct_inventory_validation.py | 113 ++++++++++++++++++ 4 files changed, 181 insertions(+), 32 deletions(-) create mode 100644 skills/deploy-aup-learning-cloud/scripts/helm_validation.py create mode 100644 tests/skills/test_direct_inventory_validation.py diff --git a/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_validation.py b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_validation.py index f33c5cc8..6d36271b 100644 --- a/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_validation.py +++ b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_validation.py @@ -34,6 +34,14 @@ class AcceleratorValidationResult: passed: list[str] +def check_gpu_inventory(repo: Path, inventory_path: str) -> GpuArtifactValidationResult: + inventory_file = configured_path(repo, inventory_path) + if not inventory_file.exists(): + return GpuArtifactValidationResult([f"inventory not found: {inventory_file}"], []) + _, errors = parse_gpu_inventory(inventory_file.read_text(encoding="utf-8")) + return GpuArtifactValidationResult(errors, [] if errors else ["GPU access inventory is valid"]) + + def check_accelerator_labels( accelerators: dict[str, str], metadata: dict[str, list[str]], cluster: dict | None ) -> AcceleratorValidationResult: diff --git a/skills/deploy-aup-learning-cloud/scripts/helm_validation.py b/skills/deploy-aup-learning-cloud/scripts/helm_validation.py new file mode 100644 index 00000000..cd1c4e39 --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/helm_validation.py @@ -0,0 +1,37 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +import shutil +import subprocess +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + +CHART = "runtime/chart" + + +@dataclass(frozen=True, slots=True) +class HelmValidationReporter: + ok: Callable[[str], None] + warn: Callable[[str], None] + fail: Callable[[str], None] + + +def check_helm(repo: Path, values: list[str], reporter: HelmValidationReporter) -> None: + if not shutil.which("helm"): + reporter.warn("helm not on PATH; skipped chart dry-run") + return + chart = repo / CHART + if not chart.exists(): + reporter.warn(f"chart not found at {CHART}; skipped dry-run") + return + cmd = ["helm", "template", "jupyterhub", str(chart)] + for rel in values or ["runtime/values.yaml"]: + path = (repo / rel) if not Path(rel).is_absolute() else Path(rel) + if path.exists(): + cmd += ["-f", str(path)] + proc = subprocess.run(cmd, capture_output=True, text=True) + if proc.returncode == 0: + reporter.ok("helm template rendered the chart successfully") + else: + tail = (proc.stderr or proc.stdout).strip().splitlines()[-5:] + reporter.fail("helm template failed:\n " + "\n ".join(tail)) diff --git a/skills/deploy-aup-learning-cloud/scripts/validate.py b/skills/deploy-aup-learning-cloud/scripts/validate.py index 8e3982c1..22b9514d 100755 --- a/skills/deploy-aup-learning-cloud/scripts/validate.py +++ b/skills/deploy-aup-learning-cloud/scripts/validate.py @@ -12,8 +12,9 @@ * nodeSelectors for the accelerators actually referenced by effective custom.resources.metadata.*.acceleratorKeys, checked against detect_cluster.sh output when supplied; - * generated inventory, GPU-resolution manifest, and PXE rootfs policy agree - when generated artifacts are supplied; + * direct inventory GPU access booleans are valid, or generated inventory, + GPU-resolution manifest, and PXE rootfs policy agree when both artifacts + are supplied; * (optional) the chart does not render: a `helm template` dry-run. This intentionally uses regex/line scanning rather than a YAML parser so it @@ -36,18 +37,21 @@ import argparse import json import re -import shutil -import subprocess import sys from pathlib import Path from config_common import DuplicateJsonKeyError, strict_json_loads -from gpu_resolution_validation import GpuArtifactValidationRequest, check_accelerator_labels, check_gpu_artifacts +from gpu_resolution_validation import ( + GpuArtifactValidationRequest, + check_accelerator_labels, + check_gpu_artifacts, + check_gpu_inventory, +) +from helm_validation import HelmValidationReporter, check_helm from values_resolution_parsing import collect_effective_values PXE_PLAYBOOK = "deploy/ansible/playbooks/pb-pxe-controller.yml" INVENTORY = "deploy/ansible/inventory.yml" -CHART = "runtime/chart" errors: list[str] = [] warnings: list[str] = [] @@ -168,27 +172,6 @@ def check_version_sync(repo: Path, configured_path: str | None = None) -> None: ) -def check_helm(repo: Path, values: list[str]) -> None: - if not shutil.which("helm"): - warn("helm not on PATH; skipped chart dry-run") - return - chart = repo / CHART - if not chart.exists(): - warn(f"chart not found at {CHART}; skipped dry-run") - return - cmd = ["helm", "template", "jupyterhub", str(chart)] - for rel in values or ["runtime/values.yaml"]: - p = (repo / rel) if not Path(rel).is_absolute() else Path(rel) - if p.exists(): - cmd += ["-f", str(p)] - proc = subprocess.run(cmd, capture_output=True, text=True) - if proc.returncode == 0: - ok("helm template rendered the chart successfully") - else: - tail = (proc.stderr or proc.stdout).strip().splitlines()[-5:] - fail("helm template failed:\n " + "\n ".join(tail)) - - def main(argv=None) -> int: global errors, passed, warnings errors = [] @@ -209,8 +192,10 @@ def main(argv=None) -> int: "--pxe-vars", help="PXE vars file to validate instead of deploy/ansible/playbooks/pb-pxe-controller.yml", ) - ap.add_argument("--inventory", help="generated inventory.yml to cross-check with GPU resolution") - ap.add_argument("--gpu-resolution", help="generated gpu-access-resolution.json to cross-check") + ap.add_argument("--inventory", help="inventory.yml to validate directly or cross-check with GPU resolution") + ap.add_argument( + "--gpu-resolution", help="generated gpu-access-resolution.json; requires --inventory for consistency checks" + ) ap.add_argument("--cluster", help="detect_cluster.sh JSON output to match labels against") ap.add_argument("--helm-dry-run", action="store_true", help="also run `helm template`") ap.add_argument("--json", action="store_true", help="emit a JSON report instead of text") @@ -246,8 +231,14 @@ def main(argv=None) -> int: warn(message) for message in accelerator_result.passed: ok(message) - if bool(args.inventory) != bool(args.gpu_resolution): - fail("--inventory and --gpu-resolution must be supplied together") + if args.gpu_resolution and not args.inventory: + fail("--gpu-resolution requires --inventory") + elif args.inventory and not args.gpu_resolution: + inventory_result = check_gpu_inventory(repo, args.inventory) + for message in inventory_result.errors: + fail(message) + for message in inventory_result.passed: + ok(message) elif args.inventory and args.gpu_resolution: artifact_result = check_gpu_artifacts( GpuArtifactValidationRequest( @@ -264,7 +255,7 @@ def main(argv=None) -> int: for message in artifact_result.passed: ok(message) if args.helm_dry_run: - check_helm(repo, args.values) + check_helm(repo, args.values, HelmValidationReporter(ok=ok, warn=warn, fail=fail)) if args.json: print( diff --git a/tests/skills/test_direct_inventory_validation.py b/tests/skills/test_direct_inventory_validation.py new file mode 100644 index 00000000..2bceb295 --- /dev/null +++ b/tests/skills/test_direct_inventory_validation.py @@ -0,0 +1,113 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +"""Public CLI tests for direct SSH inventory validation.""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +VALIDATE = ROOT / "skills" / "deploy-aup-learning-cloud" / "scripts" / "validate.py" + + +def run_validate(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run([sys.executable, str(VALIDATE), *args], capture_output=True, text=True, check=False) + + +def write(path: Path, content: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + return path + + +def valid_inventory() -> str: + return """k3s_cluster: + children: + server: + hosts: + server: + auplc_gpu_access_enabled: true + agent: + hosts: + agent: + auplc_gpu_access_enabled: false +""" + + +def test_validator_accepts_direct_inventory_without_resolution_manifest(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + inventory = write(repo / "inventory.yml", valid_inventory()) + values = write(repo / "values.yaml", "custom:\n resources:\n metadata: {}\n") + + result = run_validate( + "--repo", str(repo), "--topology", "ssh-preinstalled", "--inventory", str(inventory), "--values", str(values) + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert "GPU access inventory is valid" in result.stdout + + +@pytest.mark.parametrize( + ("inventory_content", "expected_error"), + [ + (valid_inventory().replace(" auplc_gpu_access_enabled: true\n", ""), "must define exactly one"), + (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: yes"), "malformed"), + ( + valid_inventory().replace( + " auplc_gpu_access_enabled: true\n", + " auplc_gpu_access_enabled: true\n auplc_gpu_access_enabled: false\n", + ), + "must define exactly one", + ), + ], +) +def test_validator_rejects_invalid_direct_inventory( + tmp_path: Path, inventory_content: str, expected_error: str +) -> None: + repo = tmp_path / "checkout" + inventory = write(repo / "inventory.yml", inventory_content) + values = write(repo / "values.yaml", "custom:\n resources:\n metadata: {}\n") + + result = run_validate( + "--repo", str(repo), "--topology", "ssh-preinstalled", "--inventory", str(inventory), "--values", str(values) + ) + + assert result.returncode == 1 + assert expected_error in result.stdout + + +def test_validator_reports_direct_inventory_not_found(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + values = write(repo / "values.yaml", "custom:\n resources:\n metadata: {}\n") + + result = run_validate( + "--repo", str(repo), "--topology", "ssh-preinstalled", "--inventory", "missing.yml", "--values", str(values) + ) + + assert result.returncode == 1 + assert "inventory not found" in result.stdout + assert "generated inventory not found" not in result.stdout + + +def test_validator_rejects_gpu_resolution_without_inventory(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + values = write(repo / "values.yaml", "custom:\n resources:\n metadata: {}\n") + resolution = write(repo / "resolution.json", "{}\n") + + result = run_validate( + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--gpu-resolution", + str(resolution), + "--values", + str(values), + ) + + assert result.returncode == 1 + assert "--gpu-resolution requires --inventory" in result.stdout From 0e513f4b27c28d1dc3c0fdde181842a9e1a018e8 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 20:24:46 +0800 Subject: [PATCH 50/65] fix(ansible): require explicit GPU access flags --- deploy/ansible/inventory.yml | 3 +++ tests/skills/test_gpu_access_role.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/deploy/ansible/inventory.yml b/deploy/ansible/inventory.yml index a210de23..cbda4db7 100644 --- a/deploy/ansible/inventory.yml +++ b/deploy/ansible/inventory.yml @@ -24,10 +24,13 @@ k3s_cluster: hosts: # suggested: aup-SHC1-395-1 # You need to config the hostname in /etc/hosts + # Set true on hosts that provide AMD GPU access. : + auplc_gpu_access_enabled: false agent: hosts: : + auplc_gpu_access_enabled: false # strix-5: # phx-1: # phx-64g: diff --git a/tests/skills/test_gpu_access_role.py b/tests/skills/test_gpu_access_role.py index 3fe821ec..fb11fefe 100644 --- a/tests/skills/test_gpu_access_role.py +++ b/tests/skills/test_gpu_access_role.py @@ -4,6 +4,8 @@ from pathlib import Path +import yaml + ROOT = Path(__file__).resolve().parents[2] ANSIBLE = ROOT / "deploy" / "ansible" GPU_ACCESS_ROLE = ANSIBLE / "roles" / "gpu_access" @@ -51,6 +53,20 @@ def test_gpu_access_role_pins_the_official_amd_package_contract() -> None: assert "modified package conffile" in verify +def test_inventory_placeholders_define_boolean_gpu_access() -> None: + inventory_text = read(ANSIBLE / "inventory.yml") + inventory = yaml.safe_load(inventory_text) + raw_inventory = yaml.load(inventory_text, Loader=yaml.BaseLoader) + hosts = inventory["k3s_cluster"]["children"] + raw_hosts = raw_inventory["k3s_cluster"]["children"] + + for group_name in ("server", "agent"): + for host_name, host in hosts[group_name]["hosts"].items(): + value = host["auplc_gpu_access_enabled"] + assert type(value) is bool + assert raw_hosts[group_name]["hosts"][host_name]["auplc_gpu_access_enabled"] in {"true", "false"} + + def test_gpu_access_role_preserves_rootfs_and_exact_legacy_safety() -> None: validation = read(GPU_ACCESS_ROLE / "tasks" / "validate.yml") preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") From 61a49a5f2313571bb9f96493bb8280fa32f17ccd Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 20:25:24 +0800 Subject: [PATCH 51/65] docs(deploy): restore direct SSH workflow --- deploy/README.md | 73 ++++++++++++++++++++++++++++------------ deploy/ansible/README.md | 20 +++++++---- 2 files changed, 66 insertions(+), 27 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index 7edbffc8..23c29e8c 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -53,10 +53,10 @@ sudo ./auplc-installer install ### Multi-Node Cluster -Generate the spec and fill in the network and node details. The SSH flow needs -only the managed host details. A PXE spec asks one extra GPU question: -`pxe.diskless_agents_have_amd_gpus`. Set it explicitly because the diskless -agents' hardware is not inferred from the controller. +For SSH-preinstalled nodes, edit the Ansible inventory and multi-node values +file directly. PXE remains generator-based because the controller inventory, +rootfs settings, runtime overlay, and GPU policy must be generated as one +consistent artifact set. The AMD device plugin and ROCm node labeller are cluster infrastructure prerequisites owned outside AUPLC. The infrastructure owner must deploy and @@ -67,21 +67,44 @@ DaemonSets are ready and that GPU capacity is advertised. #### SSH-preinstalled +Edit `deploy/ansible/inventory.yml` with the server and agent hostnames, IPs, +k3s token, and other site settings. Every host entry must set +`auplc_gpu_access_enabled` to the YAML boolean `true` or `false`. Use `true` +only for hosts where the AMD GPU access package and ROCm should be installed. +Don't quote the boolean or use alternatives such as `yes` and `no`. + +For example: + +```yaml +k3s_cluster: + children: + server: + hosts: + controller-1: + ansible_host: 192.0.2.10 + auplc_gpu_access_enabled: false + agent: + hosts: + gpu-worker-1: + ansible_host: 192.0.2.11 + auplc_gpu_access_enabled: true +``` + +Copy the human-maintained multi-node values example, then edit the copy for the +site's authentication, storage, images, accelerators, and network access: + ```bash cd .. REPO_ROOT="$(pwd)" DEPLOY_SCRIPTS="$REPO_ROOT/skills/deploy-aup-learning-cloud/scripts" -python3 "$DEPLOY_SCRIPTS/gen_configs.py" --print-schema > spec.json -# Edit spec.json: choose ssh-preinstalled and fill the node/network fields. -GENERATED_DIR="$REPO_ROOT/generated" -python3 "$DEPLOY_SCRIPTS/gen_configs.py" --spec spec.json --out-dir "$GENERATED_DIR" -install -m 0600 "$GENERATED_DIR/inventory.yml" "$REPO_ROOT/deploy/ansible/inventory.yml" -install -m 0644 "$GENERATED_DIR/values-basic-example.yaml" "$REPO_ROOT/runtime/values-basic-example.yaml" +cp runtime/values-multi-nodes.yaml.example runtime/values-multi-nodes.yaml +# Edit deploy/ansible/inventory.yml and runtime/values-multi-nodes.yaml. + python3 "$DEPLOY_SCRIPTS/validate.py" --repo "$REPO_ROOT" --topology ssh-preinstalled \ --inventory "$REPO_ROOT/deploy/ansible/inventory.yml" \ - --gpu-resolution "$GENERATED_DIR/gpu-access-resolution.json" \ --values "$REPO_ROOT/runtime/values.yaml" \ - --values "$REPO_ROOT/runtime/values-basic-example.yaml" + --values "$REPO_ROOT/runtime/values-multi-nodes.yaml" \ + --helm-dry-run cd "$REPO_ROOT/deploy/ansible" sudo ansible-playbook -i inventory.yml playbooks/pb-base.yml @@ -96,14 +119,14 @@ cd "$REPO_ROOT" helm upgrade --install jupyterhub ./runtime/chart \ --namespace jupyterhub --create-namespace \ -f runtime/values.yaml \ - -f runtime/values-basic-example.yaml + -f runtime/values-multi-nodes.yaml ``` -Generation runs read-only Ansible discovery against every managed host. It -cross-checks AMD display BDFs from `lspci` with PCI vendor and display-class -records under `/sys/bus/pci/devices`; it does not require the devices to be -attached to `amdgpu` before ROCm installation. The resulting GPU resolution -report records which managed hosts have AMD display hardware. +The validator checks an inventory supplied by itself for exactly one explicit +YAML boolean `auplc_gpu_access_enabled` on every managed host. This validates +the direct-edit workflow without a generated GPU resolution report. +`--gpu-resolution` may be supplied only with `--inventory`; when both are +supplied, the validator also checks generated-artifact consistency. The installer, Ansible GPU access role, and PXE controller install AMD's `amdgpu-insecure-instinct-udev-rules` package, pinned to version @@ -125,14 +148,22 @@ not part of GPU access and must not be treated as a GPU group setting. #### PXE-diskless -After setting `topology` to `pxe-diskless`, fill the PXE network fields and set -`pxe.diskless_agents_have_amd_gpus` explicitly. Generation writes the canonical +Create a fresh spec, set `topology` to `pxe-diskless`, fill the PXE network +fields, and set `pxe.diskless_agents_have_amd_gpus` explicitly. Diskless agent +hardware can't be inferred from the controller. Generation writes the canonical inventory, controller vars, runtime overlay, and GPU resolution report directly. These artifacts express the desired deployment inputs; their existence is not proof that rootfs provisioning succeeded. Review and install them before running the controller playbook, whose successful completion provisions the rootfs. ```bash +cd .. +REPO_ROOT="$(pwd)" +DEPLOY_SCRIPTS="$REPO_ROOT/skills/deploy-aup-learning-cloud/scripts" +python3 "$DEPLOY_SCRIPTS/gen_configs.py" --print-schema > spec.json +# Edit spec.json: choose pxe-diskless and fill the node, network, and PXE fields. +GENERATED_DIR="$REPO_ROOT/generated" + cd "$REPO_ROOT" python3 "$DEPLOY_SCRIPTS/gen_configs.py" --spec spec.json --out-dir "$GENERATED_DIR" install -m 0600 "$GENERATED_DIR/inventory.yml" "$REPO_ROOT/deploy/ansible/inventory.yml" @@ -166,7 +197,7 @@ playbook. A retained rootfs is accepted only when that exact package version and its unmodified package-owned rule are present, with no conflicting legacy GPU rule. Rebuild or correct a retained rootfs separately if that safety check fails. -#### Discovery failures +#### Generator discovery failures | Error | Action | | --- | --- | diff --git a/deploy/ansible/README.md b/deploy/ansible/README.md index cbd02ad1..fefe0fea 100644 --- a/deploy/ansible/README.md +++ b/deploy/ansible/README.md @@ -24,13 +24,21 @@ SOFTWARE. K3s cluster setup playbooks based on [k3s-ansible](https://github.com/k3s-io/k3s-ansible). -For the generator, canonical inventory, validator arguments, and topology-specific -playbook commands, see the authoritative [deployment guide](../README.md). +For the human SSH-preinstalled workflow, edit `inventory.yml` directly and use +the playbook commands in the [deployment guide](../README.md). Every server and +agent host entry must define `auplc_gpu_access_enabled` as the unquoted YAML +boolean `true` or `false`. Set it to `true` only on hosts where the GPU access +package and ROCm should be installed. Pass `--inventory` to the deployment +validator to check this explicit per-host policy. A generated +`--gpu-resolution` report is not required for the human workflow; if supplied, +it requires `--inventory`, and the validator checks the two generated artifacts +for consistency. -Don't write GPU policy into the inventory by hand. SSH generation discovers GPU -hosts from managed-host evidence. PXE generation uses only -`pxe.diskless_agents_have_amd_gpus` and writes canonical files before the -controller playbook runs. +The deploy skill has a separate generator-first SSH workflow that discovers GPU +hosts from managed-host evidence. PXE is always generator-based and uses only +`pxe.diskless_agents_have_amd_gpus` as its GPU policy input. See the +[skill scripts guide](../../skills/deploy-aup-learning-cloud/scripts/README.md) +for the complete generator-first skill command sequences. The GPU access role installs AMD's `amdgpu-insecure-instinct-udev-rules` package, pinned to `30.30.4.0-2341068.24.04`, on GPU hosts and GPU-enabled PXE From dae1f1efdebfedf4c830883b4354677d6fcf3097 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Tue, 28 Jul 2026 20:26:02 +0800 Subject: [PATCH 52/65] docs(skills): retain generated deployment workflow --- skills/deploy-aup-learning-cloud/SKILL.md | 40 ++++--- skills/deploy-aup-learning-cloud/reference.md | 21 ++-- .../scripts/README.md | 109 ++++++++++++++++-- 3 files changed, 141 insertions(+), 29 deletions(-) diff --git a/skills/deploy-aup-learning-cloud/SKILL.md b/skills/deploy-aup-learning-cloud/SKILL.md index 2c88018a..9af9e4c4 100644 --- a/skills/deploy-aup-learning-cloud/SKILL.md +++ b/skills/deploy-aup-learning-cloud/SKILL.md @@ -13,9 +13,11 @@ description: >- Stand up a multi-node AUP Learning Cloud cluster with Ansible, AMD GPU access, shared storage, and the JupyterHub Helm chart. -Use [deploy/README.md](../../deploy/README.md) as the source of truth for the -generator schema, commands, generated files, validation, and troubleshooting. -This skill defines the interview and safety gates around that procedure. +Use the [skill scripts guide](scripts/README.md) as the source of truth for the +complete generator-first command sequences and generated files. Use +[deploy/README.md](../../deploy/README.md) for the human direct-edit workflow, +operational background, and troubleshooting. This skill defines the interview +and safety gates around the generated procedure. ## Prerequisites @@ -65,27 +67,34 @@ does not prove rootfs provisioning succeeded. Review, install, and validate thos files, then run the controller playbook with the canonical inventory and PXE vars; the playbook must complete successfully before proceeding. -Follow the exact generation, installation, and playbook commands in -[deploy/README.md](../../deploy/README.md). +Follow the complete topology command sequence in the +[skill scripts guide](scripts/README.md). Don't substitute the human direct-edit +SSH workflow from `deploy/README.md`; the skill's SSH path remains +generator-first and discovers GPU policy from managed-host evidence. ## Phase 3: Validate and execute Install the canonical generated inventory and runtime overlay into the checkout, -then run the validator with the arguments shown in the deployment guide: +then run the topology's exact validator command from the +[skill scripts guide](scripts/README.md). The validator inputs are: - `--repo` - `--topology` -- `--inventory` -- `--gpu-resolution` +- `--inventory` to validate explicit host booleans +- `--gpu-resolution` with `--inventory` for generated-artifact consistency - both `--values` files - `--pxe-vars` for PXE only -Stop on validation failure. After a clean result, follow the topology's Ansible, -storage, device plugin, and Helm sequence in -[deploy/README.md](../../deploy/README.md). Treat the AMD device plugin and ROCm -node labeller as infrastructure prerequisites owned outside AUPLC. Verify both -existing DaemonSets and advertised GPU capacity before Helm; do not install -these privileged components as part of the AUPLC procedure. +An inventory can be validated without a GPU resolution report. A resolution +report requires an inventory. Supply both in this generator-first workflow so +the validator also checks their consistency. + +Stop on validation failure. After a clean result, continue with the topology's +Ansible, device plugin, and Helm commands in the skill scripts guide. Treat the +AMD device plugin and ROCm node labeller as infrastructure prerequisites owned +outside AUPLC. Verify both existing DaemonSets and advertised GPU capacity +before Helm; do not install these privileged components as part of the AUPLC +procedure. Keep the GPU contract distinct from storage configuration. The installer, Ansible role, and PXE controller install AMD's @@ -119,5 +128,6 @@ are changed. ## Reference -- [Deployment commands and troubleshooting](../../deploy/README.md) +- [Complete skill command sequences](scripts/README.md) +- [Human deployment and troubleshooting](../../deploy/README.md) - [Skill-specific summary](reference.md) diff --git a/skills/deploy-aup-learning-cloud/reference.md b/skills/deploy-aup-learning-cloud/reference.md index 9c07ed68..e10c8e9c 100644 --- a/skills/deploy-aup-learning-cloud/reference.md +++ b/skills/deploy-aup-learning-cloud/reference.md @@ -1,8 +1,10 @@ # Deploy AUP Learning Cloud Reference -The authoritative procedure, command lines, generated file list, and failure -guidance live in [deploy/README.md](../../deploy/README.md). Don't copy those -commands into this reference. +The complete generator-first command sequences and generated file list live in +the [skill scripts guide](scripts/README.md). Human direct-edit deployment, +operational background, and failure guidance live in +[deploy/README.md](../../deploy/README.md). Don't copy those commands into this +reference. ## Topology contract @@ -11,13 +13,14 @@ commands into this reference. | `ssh-preinstalled` | Connects to every managed host, discovers GPU hardware, and publishes canonical files when discovery is consistent. | | `pxe-diskless` | Uses `pxe.diskless_agents_have_amd_gpus` as its sole GPU policy input and publishes canonical desired-input files before the controller playbook runs. Their existence does not prove rootfs provisioning succeeded. | -Don't hand-author generated GPU policy. Create deployment specs from the current +The skill is generator-first for both topologies. Don't hand-author generated +GPU policy, including for SSH. Create deployment specs from the current `--print-schema` output. ## Canonical validation inputs -Use the validator command from [deploy/README.md](../../deploy/README.md). It -passes: +Use the topology's validator command from the +[skill scripts guide](scripts/README.md). It passes: - repository root with `--repo` - selected topology with `--topology` @@ -26,7 +29,11 @@ passes: - base and generated overlays as two `--values` arguments - canonical PXE vars with `--pxe-vars` for PXE only -Generation and validation must finish before Ansible or Helm changes are made. +`--inventory` alone validates that every managed host has exactly one explicit +YAML boolean `auplc_gpu_access_enabled`. `--gpu-resolution` requires +`--inventory`; supplying both enables generated-artifact consistency checks. +The skill supplies both because its workflow is generator-first. Generation +and validation must finish before Ansible or Helm changes are made. ## GPU permission contract diff --git a/skills/deploy-aup-learning-cloud/scripts/README.md b/skills/deploy-aup-learning-cloud/scripts/README.md index db9fbfef..dad1fd59 100644 --- a/skills/deploy-aup-learning-cloud/scripts/README.md +++ b/skills/deploy-aup-learning-cloud/scripts/README.md @@ -1,8 +1,10 @@ # Helper scripts -These dependency-light helpers support the multi-node deployment skill. See -[deploy/README.md](../../../deploy/README.md) for the authoritative command -sequence and argument paths. +These dependency-light helpers support the multi-node deployment skill. This +file is the source of truth for the complete skill command sequences: artifact +generation and installation, validation, Ansible, device plugin readiness, and +Helm. For human direct-edit deployment, operational background, and +troubleshooting, see [deploy/README.md](../../../deploy/README.md). | Script | Purpose | | --- | --- | @@ -23,12 +25,105 @@ run the controller playbook with the generated `inventory.yml` and `pb-pxe-controller.vars.yml`. The files express desired inputs; their existence does not prove the PXE rootfs was provisioned successfully. +## SSH-preinstalled commands + +Run these commands from a clean checkout. Fill the generated `spec.json` with +the SSH topology, network settings, and every managed host. Don't add a GPU host +list. The generator discovers GPU policy over passwordless root SSH. + +```bash +cd /path/to/aup-learning-cloud +REPO_ROOT="$(pwd)" +DEPLOY_SCRIPTS="$REPO_ROOT/skills/deploy-aup-learning-cloud/scripts" +python3 "$DEPLOY_SCRIPTS/gen_configs.py" --print-schema > spec.json +# Edit spec.json: choose ssh-preinstalled and fill the node and network fields. +GENERATED_DIR="$REPO_ROOT/generated" +python3 "$DEPLOY_SCRIPTS/gen_configs.py" --spec spec.json --out-dir "$GENERATED_DIR" + +install -m 0600 "$GENERATED_DIR/inventory.yml" "$REPO_ROOT/deploy/ansible/inventory.yml" +install -m 0644 "$GENERATED_DIR/values-basic-example.yaml" "$REPO_ROOT/runtime/values-basic-example.yaml" + +python3 "$DEPLOY_SCRIPTS/validate.py" --repo "$REPO_ROOT" --topology ssh-preinstalled \ + --inventory "$REPO_ROOT/deploy/ansible/inventory.yml" \ + --gpu-resolution "$GENERATED_DIR/gpu-access-resolution.json" \ + --values "$REPO_ROOT/runtime/values.yaml" \ + --values "$REPO_ROOT/runtime/values-basic-example.yaml" +``` + +After validation passes, run Ansible, check the infrastructure-owned GPU +components, and install the chart with the generated overlay: + +```bash +cd "$REPO_ROOT/deploy/ansible" +sudo ansible-playbook -i inventory.yml playbooks/pb-base.yml +sudo ansible-playbook -i inventory.yml playbooks/pb-k3s-site.yml +sudo ansible-playbook -i inventory.yml playbooks/pb-rocm.yml + +kubectl rollout status -n kube-system daemonset/amdgpu-device-plugin-daemonset --timeout=5m +kubectl rollout status -n kube-system daemonset/amdgpu-labeller-daemonset --timeout=5m +kubectl get nodes -o 'custom-columns=NAME:.metadata.name,AMD_GPU:.status.allocatable.amd\.com/gpu' + +cd "$REPO_ROOT" +helm upgrade --install jupyterhub ./runtime/chart \ + --namespace jupyterhub --create-namespace \ + -f runtime/values.yaml \ + -f runtime/values-basic-example.yaml +``` + +## PXE-diskless commands + +Fill the generated `spec.json` with the PXE topology and all controller, +network, and rootfs fields. Set `pxe.diskless_agents_have_amd_gpus` explicitly. + +```bash +cd /path/to/aup-learning-cloud +REPO_ROOT="$(pwd)" +DEPLOY_SCRIPTS="$REPO_ROOT/skills/deploy-aup-learning-cloud/scripts" +python3 "$DEPLOY_SCRIPTS/gen_configs.py" --print-schema > spec.json +# Edit spec.json: choose pxe-diskless and fill the node, network, and PXE fields. +GENERATED_DIR="$REPO_ROOT/generated" +python3 "$DEPLOY_SCRIPTS/gen_configs.py" --spec spec.json --out-dir "$GENERATED_DIR" + +install -m 0600 "$GENERATED_DIR/inventory.yml" "$REPO_ROOT/deploy/ansible/inventory.yml" +install -m 0644 "$GENERATED_DIR/values-basic-example.yaml" "$REPO_ROOT/runtime/values-basic-example.yaml" + +python3 "$DEPLOY_SCRIPTS/validate.py" --repo "$REPO_ROOT" --topology pxe-diskless \ + --inventory "$REPO_ROOT/deploy/ansible/inventory.yml" \ + --gpu-resolution "$GENERATED_DIR/gpu-access-resolution.json" \ + --values "$REPO_ROOT/runtime/values.yaml" \ + --values "$REPO_ROOT/runtime/values-basic-example.yaml" \ + --pxe-vars "$GENERATED_DIR/pb-pxe-controller.vars.yml" + +cd "$REPO_ROOT/deploy/ansible" +sudo ansible-playbook \ + -i "$GENERATED_DIR/inventory.yml" \ + playbooks/pb-pxe-controller.yml \ + -e @"$GENERATED_DIR/pb-pxe-controller.vars.yml" + +kubectl rollout status -n kube-system daemonset/amdgpu-device-plugin-daemonset --timeout=5m +kubectl rollout status -n kube-system daemonset/amdgpu-labeller-daemonset --timeout=5m +kubectl get nodes -o 'custom-columns=NAME:.metadata.name,AMD_GPU:.status.allocatable.amd\.com/gpu' + +cd "$REPO_ROOT" +helm upgrade --install jupyterhub ./runtime/chart \ + --namespace jupyterhub --create-namespace \ + -f runtime/values.yaml \ + -f runtime/values-basic-example.yaml +``` + +The controller playbook must finish successfully before the remaining cluster +and Helm steps begin. A fresh rootfs receives the pinned GPU access package. A +retained rootfs must pass the package version, package-owned rule, and legacy +rule safety checks described in the deployment guide. + ## Validator contract -Use the exact validator command in -[deploy/README.md](../../../deploy/README.md). Its canonical inputs are -`--repo`, `--topology`, `--inventory`, `--gpu-resolution`, two `--values` -arguments, and `--pxe-vars` for PXE only. +The exact topology commands above pass `--repo`, `--topology`, `--inventory`, +`--gpu-resolution`, two `--values` arguments, and `--pxe-vars` for PXE only. +`--inventory` alone validates that every managed host defines exactly one +explicit YAML boolean `auplc_gpu_access_enabled`. `--gpu-resolution` requires +`--inventory`; supplying both performs generated-artifact consistency checks. +The generator-first skill workflow supplies both. ## Conventions From ee7ac6e3bc5ed8cac7215b85d49a4133149f1195 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 29 Jul 2026 11:15:32 +0800 Subject: [PATCH 53/65] fix(validation): allow auto only for direct SSH inventory --- .../scripts/gpu_resolution_parsing.py | 35 +++++- .../scripts/gpu_resolution_validation.py | 3 +- .../scripts/validate.py | 24 +++-- .../test_direct_inventory_validation.py | 100 ++++++++++++++++++ 4 files changed, 148 insertions(+), 14 deletions(-) diff --git a/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_parsing.py b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_parsing.py index 6fe4c6d1..eb5a38ec 100644 --- a/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_parsing.py +++ b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_parsing.py @@ -13,6 +13,11 @@ class GpuInventory: hosts: dict[str, bool] +@dataclass(frozen=True, slots=True) +class GpuInventoryHostScalars: + hosts: dict[str, str] + + @dataclass(frozen=True, slots=True) class GpuResolution: status: str @@ -43,7 +48,7 @@ def yaml_indent(line: str) -> int: return len(line) - len(line.lstrip()) -def parse_gpu_inventory(text: str) -> tuple[GpuInventory | None, list[str]]: +def scan_gpu_inventory_host_scalars(text: str) -> tuple[GpuInventoryHostScalars | None, list[str]]: host_values: dict[str, list[str]] = {} host_names: list[str] = [] stack: list[tuple[int, str]] = [] @@ -85,20 +90,42 @@ def parse_gpu_inventory(text: str) -> tuple[GpuInventory | None, list[str]]: parse_errors.append("inventory has no generated k3s server or agent hosts") if len(set(host_names)) != len(host_names): parse_errors.append("inventory has duplicate generated host names") - hosts: dict[str, bool] = {} + hosts: dict[str, str] = {} for host in host_names: values = host_values[host] if len(values) != 1: parse_errors.append(f"inventory host '{host}' must define exactly one auplc_gpu_access_enabled") continue - enabled = parse_gpu_boolean(values[0]) + hosts[host] = values[0] + if parse_errors: + return None, parse_errors + return GpuInventoryHostScalars(hosts=hosts), [] + + +def validate_direct_gpu_inventory(text: str) -> list[str]: + host_scalars, parse_errors = scan_gpu_inventory_host_scalars(text) + if host_scalars is None: + return parse_errors + for host, value in host_scalars.hosts.items(): + if value not in {"auto", "true", "false"}: + parse_errors.append(f"inventory host '{host}' has malformed auplc_gpu_access_enabled") + return parse_errors + + +def parse_gpu_inventory(text: str) -> tuple[GpuInventory | None, list[str]]: + host_scalars, parse_errors = scan_gpu_inventory_host_scalars(text) + if host_scalars is None: + return None, parse_errors + hosts: dict[str, bool] = {} + for host, value in host_scalars.hosts.items(): + enabled = parse_gpu_boolean(value) if enabled is None: parse_errors.append(f"inventory host '{host}' has malformed auplc_gpu_access_enabled") continue hosts[host] = enabled if parse_errors: return None, parse_errors - return GpuInventory(hosts=hosts), parse_errors + return GpuInventory(hosts=hosts), [] def parse_gpu_resolution(text: str, topology: str) -> tuple[GpuResolution | None, list[str]]: diff --git a/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_validation.py b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_validation.py index 6d36271b..b3c3061f 100644 --- a/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_validation.py +++ b/skills/deploy-aup-learning-cloud/scripts/gpu_resolution_validation.py @@ -8,6 +8,7 @@ parse_gpu_inventory, parse_gpu_resolution, parse_pxe_gpu_policy, + validate_direct_gpu_inventory, ) @@ -38,7 +39,7 @@ def check_gpu_inventory(repo: Path, inventory_path: str) -> GpuArtifactValidatio inventory_file = configured_path(repo, inventory_path) if not inventory_file.exists(): return GpuArtifactValidationResult([f"inventory not found: {inventory_file}"], []) - _, errors = parse_gpu_inventory(inventory_file.read_text(encoding="utf-8")) + errors = validate_direct_gpu_inventory(inventory_file.read_text(encoding="utf-8")) return GpuArtifactValidationResult(errors, [] if errors else ["GPU access inventory is valid"]) diff --git a/skills/deploy-aup-learning-cloud/scripts/validate.py b/skills/deploy-aup-learning-cloud/scripts/validate.py index 22b9514d..934e3ea7 100755 --- a/skills/deploy-aup-learning-cloud/scripts/validate.py +++ b/skills/deploy-aup-learning-cloud/scripts/validate.py @@ -12,9 +12,9 @@ * nodeSelectors for the accelerators actually referenced by effective custom.resources.metadata.*.acceleratorKeys, checked against detect_cluster.sh output when supplied; - * direct inventory GPU access booleans are valid, or generated inventory, - GPU-resolution manifest, and PXE rootfs policy agree when both artifacts - are supplied; + * direct SSH inventory GPU access values are exact unquoted `auto`, `true`, + or `false`; generated inventory, GPU-resolution manifest, and PXE rootfs + policy agree when both artifacts are supplied; * (optional) the chart does not render: a `helm template` dry-run. This intentionally uses regex/line scanning rather than a YAML parser so it @@ -192,7 +192,10 @@ def main(argv=None) -> int: "--pxe-vars", help="PXE vars file to validate instead of deploy/ansible/playbooks/pb-pxe-controller.yml", ) - ap.add_argument("--inventory", help="inventory.yml to validate directly or cross-check with GPU resolution") + ap.add_argument( + "--inventory", + help="inventory.yml for direct ssh-preinstalled validation (auto/true/false) or generated checks; pxe requires --gpu-resolution", + ) ap.add_argument( "--gpu-resolution", help="generated gpu-access-resolution.json; requires --inventory for consistency checks" ) @@ -234,11 +237,14 @@ def main(argv=None) -> int: if args.gpu_resolution and not args.inventory: fail("--gpu-resolution requires --inventory") elif args.inventory and not args.gpu_resolution: - inventory_result = check_gpu_inventory(repo, args.inventory) - for message in inventory_result.errors: - fail(message) - for message in inventory_result.passed: - ok(message) + if args.topology == "pxe-diskless": + fail("pxe-diskless inventory validation requires --gpu-resolution") + else: + inventory_result = check_gpu_inventory(repo, args.inventory) + for message in inventory_result.errors: + fail(message) + for message in inventory_result.passed: + ok(message) elif args.inventory and args.gpu_resolution: artifact_result = check_gpu_artifacts( GpuArtifactValidationRequest( diff --git a/tests/skills/test_direct_inventory_validation.py b/tests/skills/test_direct_inventory_validation.py index 2bceb295..370f3e37 100644 --- a/tests/skills/test_direct_inventory_validation.py +++ b/tests/skills/test_direct_inventory_validation.py @@ -51,11 +51,111 @@ def test_validator_accepts_direct_inventory_without_resolution_manifest(tmp_path assert "GPU access inventory is valid" in result.stdout +def test_validator_requires_gpu_resolution_for_pxe_inventory_only(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + inventory = write( + repo / "inventory.yml", + valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: auto"), + ) + values = write(repo / "values.yaml", "custom:\n resources:\n metadata: {}\n") + write(repo / "deploy/ansible/inventory.yml", "k3s_version: v1.32.3+k3s1\n") + pxe_vars = write( + repo / "pxe-vars.yml", + """pxe_network_interface: eno1 +pxe_subnet: 192.168.1.0/24 +pxe_controller_ip: 192.168.1.10 +pxe_dns_servers: 8.8.8.8 +pxe_k3s_server_ips: [192.168.1.10] +pxe_rootfs_authorized_keys: [ssh-ed25519-AAA] +pxe_k3s_version: v1.32.3+k3s1 +pxe_gpu_access_enabled: false +""", + ) + + result = run_validate( + "--repo", + str(repo), + "--topology", + "pxe-diskless", + "--inventory", + str(inventory), + "--values", + str(values), + "--pxe-vars", + str(pxe_vars), + ) + + assert result.returncode == 1 + assert "pxe-diskless inventory validation requires --gpu-resolution" in result.stdout + + +@pytest.mark.parametrize("value", ("auto", "true", "false")) +def test_validator_accepts_supported_direct_inventory_values(tmp_path: Path, value: str) -> None: + repo = tmp_path / "checkout" + inventory = write(repo / "inventory.yml", valid_inventory().replace("true", value).replace("false", value)) + values = write(repo / "values.yaml", "custom:\n resources:\n metadata: {}\n") + + result = run_validate( + "--repo", str(repo), "--topology", "ssh-preinstalled", "--inventory", str(inventory), "--values", str(values) + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert "GPU access inventory is valid" in result.stdout + + +def test_validator_rejects_auto_when_inventory_is_cross_checked_with_gpu_resolution(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + inventory = write( + repo / "inventory.yml", + valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: auto"), + ) + values = write(repo / "values.yaml", "custom:\n resources:\n metadata: {}\n") + resolution = write( + repo / "gpu-access-resolution.json", + """{ + "version": 1, + "status": "gpu_resolved", + "hosts": {"agent": false, "server": true} +} +""", + ) + + result = run_validate( + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--inventory", + str(inventory), + "--gpu-resolution", + str(resolution), + "--values", + str(values), + ) + + assert result.returncode == 1 + assert "inventory host 'server' has malformed auplc_gpu_access_enabled" in result.stdout + + @pytest.mark.parametrize( ("inventory_content", "expected_error"), [ (valid_inventory().replace(" auplc_gpu_access_enabled: true\n", ""), "must define exactly one"), (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: yes"), "malformed"), + (valid_inventory().replace("auplc_gpu_access_enabled: true", 'auplc_gpu_access_enabled: "auto"'), "malformed"), + (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: 'auto'"), "malformed"), + (valid_inventory().replace("auplc_gpu_access_enabled: true", 'auplc_gpu_access_enabled: "true"'), "malformed"), + (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: 'true'"), "malformed"), + (valid_inventory().replace("auplc_gpu_access_enabled: true", 'auplc_gpu_access_enabled: "false"'), "malformed"), + (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: 'false'"), "malformed"), + (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: AUTO"), "malformed"), + (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: TRUE"), "malformed"), + (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: FALSE"), "malformed"), + (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: no"), "malformed"), + ( + valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: malformed"), + "malformed", + ), ( valid_inventory().replace( " auplc_gpu_access_enabled: true\n", From 6875615325c332cad07607e38cb691022df3ef7e Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 29 Jul 2026 11:16:23 +0800 Subject: [PATCH 54/65] feat(ansible): resolve automatic GPU access --- deploy/ansible/inventory.yml | 6 +-- deploy/ansible/playbooks/pb-rocm.yml | 18 ++++---- deploy/ansible/playbooks/pb-udev.yml | 16 +++---- .../roles/gpu_access/defaults/main.yml | 2 +- .../ansible/roles/gpu_access/tasks/detect.yml | 16 +++++++ .../ansible/roles/gpu_access/tasks/main.yml | 9 ++-- .../roles/gpu_access/tasks/resolve.yml | 42 +++++++++++++++++++ tests/skills/test_gpu_access_role.py | 40 ++++++++++++++++-- 8 files changed, 118 insertions(+), 31 deletions(-) create mode 100644 deploy/ansible/roles/gpu_access/tasks/detect.yml create mode 100644 deploy/ansible/roles/gpu_access/tasks/resolve.yml diff --git a/deploy/ansible/inventory.yml b/deploy/ansible/inventory.yml index cbda4db7..9be9f9eb 100644 --- a/deploy/ansible/inventory.yml +++ b/deploy/ansible/inventory.yml @@ -24,13 +24,13 @@ k3s_cluster: hosts: # suggested: aup-SHC1-395-1 # You need to config the hostname in /etc/hosts - # Set true on hosts that provide AMD GPU access. + # Set auto to detect AMD display hardware, or true/false to override it. : - auplc_gpu_access_enabled: false + auplc_gpu_access_enabled: auto agent: hosts: : - auplc_gpu_access_enabled: false + auplc_gpu_access_enabled: auto # strix-5: # phx-1: # phx-64g: diff --git a/deploy/ansible/playbooks/pb-rocm.yml b/deploy/ansible/playbooks/pb-rocm.yml index 6788bf3d..7fb43ba8 100644 --- a/deploy/ansible/playbooks/pb-rocm.yml +++ b/deploy/ansible/playbooks/pb-rocm.yml @@ -22,26 +22,22 @@ any_errors_fatal: true become: yes pre_tasks: - - name: Assert explicit GPU access enablement - ansible.builtin.assert: - that: - - auplc_gpu_access_enabled is defined - - auplc_gpu_access_enabled is boolean - fail_msg: >- - Set auplc_gpu_access_enabled to true or false for every host in the - inventory before running pb-rocm.yml. + - name: Resolve GPU access enablement before ROCm mutation + ansible.builtin.include_role: + name: gpu_access + tasks_from: resolve - name: Preflight enabled GPU access hosts before ROCm mutation ansible.builtin.include_role: name: gpu_access tasks_from: preflight - when: auplc_gpu_access_enabled | bool + when: _auplc_gpu_access_enabled_resolved roles: - role: rocm - when: auplc_gpu_access_enabled | bool + when: _auplc_gpu_access_enabled_resolved tasks: - name: Apply GPU access after ROCm installation ansible.builtin.include_role: name: gpu_access tasks_from: apply - when: auplc_gpu_access_enabled | bool + when: _auplc_gpu_access_enabled_resolved diff --git a/deploy/ansible/playbooks/pb-udev.yml b/deploy/ansible/playbooks/pb-udev.yml index 7e77eb80..53826ff1 100644 --- a/deploy/ansible/playbooks/pb-udev.yml +++ b/deploy/ansible/playbooks/pb-udev.yml @@ -22,23 +22,19 @@ any_errors_fatal: true become: yes pre_tasks: - - name: Assert explicit GPU access enablement - ansible.builtin.assert: - that: - - auplc_gpu_access_enabled is defined - - auplc_gpu_access_enabled is boolean - fail_msg: >- - Set auplc_gpu_access_enabled to true or false for every host in the - inventory before running pb-udev.yml. + - name: Resolve GPU access enablement before GPU access mutation + ansible.builtin.include_role: + name: gpu_access + tasks_from: resolve - name: Preflight enabled GPU access hosts ansible.builtin.include_role: name: gpu_access tasks_from: preflight - when: auplc_gpu_access_enabled | bool + when: _auplc_gpu_access_enabled_resolved tasks: - name: Apply GPU access on enabled hosts ansible.builtin.include_role: name: gpu_access tasks_from: apply - when: auplc_gpu_access_enabled | bool + when: _auplc_gpu_access_enabled_resolved diff --git a/deploy/ansible/roles/gpu_access/defaults/main.yml b/deploy/ansible/roles/gpu_access/defaults/main.yml index 4a9d4e3b..65aa625c 100644 --- a/deploy/ansible/roles/gpu_access/defaults/main.yml +++ b/deploy/ansible/roles/gpu_access/defaults/main.yml @@ -1,7 +1,7 @@ # Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. --- -auplc_gpu_access_enabled: false +auplc_gpu_access_enabled: auto # Set for a PXE rootfs. Leave empty to configure the live host. auplc_rootfs_path: "" # Rootfs adapters must explicitly constrain their writable target below this diff --git a/deploy/ansible/roles/gpu_access/tasks/detect.yml b/deploy/ansible/roles/gpu_access/tasks/detect.yml new file mode 100644 index 00000000..939b4a95 --- /dev/null +++ b/deploy/ansible/roles/gpu_access/tasks/detect.yml @@ -0,0 +1,16 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +--- +- name: Detect AMD display BDFs through sysfs + ansible.builtin.command: + argv: + - python3 + - -c + - >- + from pathlib import Path; devices = Path('/sys/bus/pci/devices'); + print('\n'.join(sorted(device.name for device in devices.iterdir() + if (device / 'vendor').read_text().strip() == '0x1002' and + (device / 'class').read_text().strip().startswith('0x03')))) + register: _auplc_gpu_access_sysfs + changed_when: false + failed_when: false diff --git a/deploy/ansible/roles/gpu_access/tasks/main.yml b/deploy/ansible/roles/gpu_access/tasks/main.yml index 4826393b..c3317bc1 100644 --- a/deploy/ansible/roles/gpu_access/tasks/main.yml +++ b/deploy/ansible/roles/gpu_access/tasks/main.yml @@ -1,14 +1,17 @@ # Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. --- +- name: Resolve GPU access enablement + ansible.builtin.import_tasks: resolve.yml + - name: Validate GPU access configuration ansible.builtin.import_tasks: validate.yml - when: auplc_gpu_access_enabled | bool + when: _auplc_gpu_access_enabled_resolved - name: Preflight GPU access target ansible.builtin.import_tasks: preflight.yml - when: auplc_gpu_access_enabled | bool + when: _auplc_gpu_access_enabled_resolved - name: Apply GPU access configuration ansible.builtin.import_tasks: apply.yml - when: auplc_gpu_access_enabled | bool + when: _auplc_gpu_access_enabled_resolved diff --git a/deploy/ansible/roles/gpu_access/tasks/resolve.yml b/deploy/ansible/roles/gpu_access/tasks/resolve.yml new file mode 100644 index 00000000..b7f36282 --- /dev/null +++ b/deploy/ansible/roles/gpu_access/tasks/resolve.yml @@ -0,0 +1,42 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +--- +- name: Validate GPU access enablement policy + ansible.builtin.assert: + that: + - auplc_gpu_access_enabled is defined + - >- + auplc_gpu_access_enabled is boolean or + (auplc_gpu_access_enabled is string and auplc_gpu_access_enabled == 'auto') + fail_msg: >- + Set auplc_gpu_access_enabled to true, false, or unquoted auto for every + host before running GPU access tasks. + changed_when: false + +- name: Detect GPU access hardware for auto policy + ansible.builtin.import_tasks: detect.yml + when: auplc_gpu_access_enabled == 'auto' + +- name: Require successful GPU access hardware detection for auto policy + ansible.builtin.assert: + that: + - _auplc_gpu_access_sysfs.rc == 0 + fail_msg: >- + GPU access hardware detection failed for auto policy; no GPU access + mutation was attempted. + when: auplc_gpu_access_enabled == 'auto' + changed_when: false + +- name: Resolve GPU access enablement + ansible.builtin.set_fact: + _auplc_gpu_access_enabled_resolved: >- + {{ auplc_gpu_access_enabled if auplc_gpu_access_enabled is boolean + else (_auplc_gpu_access_sysfs.stdout | trim | length > 0) }} + changed_when: false + +- name: Require resolved GPU access enablement boolean + ansible.builtin.assert: + that: + - _auplc_gpu_access_enabled_resolved is boolean + fail_msg: GPU access enablement did not resolve to a boolean. + changed_when: false diff --git a/tests/skills/test_gpu_access_role.py b/tests/skills/test_gpu_access_role.py index fb11fefe..f67ec67d 100644 --- a/tests/skills/test_gpu_access_role.py +++ b/tests/skills/test_gpu_access_role.py @@ -53,18 +53,26 @@ def test_gpu_access_role_pins_the_official_amd_package_contract() -> None: assert "modified package conffile" in verify -def test_inventory_placeholders_define_boolean_gpu_access() -> None: +def test_gpu_access_defaults_and_inventory_placeholders_use_unquoted_auto() -> None: + defaults_text = read(GPU_ACCESS_ROLE / "defaults" / "main.yml") + defaults = yaml.safe_load(defaults_text) inventory_text = read(ANSIBLE / "inventory.yml") inventory = yaml.safe_load(inventory_text) raw_inventory = yaml.load(inventory_text, Loader=yaml.BaseLoader) hosts = inventory["k3s_cluster"]["children"] raw_hosts = raw_inventory["k3s_cluster"]["children"] + assert defaults["auplc_gpu_access_enabled"] == "auto" + assert "auplc_gpu_access_enabled: auto" in defaults_text + assert inventory_text.count("auplc_gpu_access_enabled: auto") == 2 + assert all( + quoted not in inventory_text + for quoted in ('auplc_gpu_access_enabled: "auto"', "auplc_gpu_access_enabled: 'auto'") + ) for group_name in ("server", "agent"): for host_name, host in hosts[group_name]["hosts"].items(): value = host["auplc_gpu_access_enabled"] - assert type(value) is bool - assert raw_hosts[group_name]["hosts"][host_name]["auplc_gpu_access_enabled"] in {"true", "false"} + assert value == raw_hosts[group_name]["hosts"][host_name]["auplc_gpu_access_enabled"] == "auto" def test_gpu_access_role_preserves_rootfs_and_exact_legacy_safety() -> None: @@ -237,12 +245,38 @@ def test_pxe_rootfs_unmounts_fail_on_real_errors_but_skip_absent_mounts() -> Non def test_gpu_access_playbooks_keep_two_phase_live_and_rootfs_safety() -> None: + role_main = read(GPU_ACCESS_ROLE / "tasks" / "main.yml") + resolve = read(GPU_ACCESS_ROLE / "tasks" / "resolve.yml") + detect = read(GPU_ACCESS_ROLE / "tasks" / "detect.yml") rocm_playbook = read(ANSIBLE / "playbooks" / "pb-rocm.yml") udev_playbook = read(ANSIBLE / "playbooks" / "pb-udev.yml") pxe_playbook = read(ANSIBLE / "playbooks" / "pb-pxe-controller.yml") + assert "ansible.builtin.import_tasks: resolve.yml" in role_main + assert "auplc_gpu_access_enabled | bool" not in role_main + assert "when: _auplc_gpu_access_enabled_resolved" in role_main + assert "python3" in detect + assert "/sys/bus/pci/devices" in detect + assert "0x1002" in detect + assert "startswith('0x03')" in detect + assert "sorted(" in detect + assert "register: _auplc_gpu_access_sysfs" in detect + assert "changed_when: false" in detect + assert "failed_when: false" in detect + assert "auplc_gpu_access_enabled is boolean" in resolve + assert "auplc_gpu_access_enabled == 'auto'" in resolve + assert "ansible.builtin.import_tasks: detect.yml" in resolve + assert "_auplc_gpu_access_sysfs.rc == 0" in resolve + assert "_auplc_gpu_access_sysfs.stdout | trim | length > 0" in resolve + assert "_auplc_gpu_access_enabled_resolved is boolean" in resolve + assert resolve.index("ansible.builtin.import_tasks: detect.yml") < resolve.index("_auplc_gpu_access_sysfs.rc == 0") assert "any_errors_fatal: true" in rocm_playbook assert "any_errors_fatal: true" in udev_playbook + for playbook in (rocm_playbook, udev_playbook): + assert "tasks_from: resolve" in playbook + assert playbook.index("tasks_from: resolve") < playbook.index("tasks_from: preflight") + assert "auplc_gpu_access_enabled | bool" not in playbook + assert "when: _auplc_gpu_access_enabled_resolved" in playbook assert rocm_playbook.index("tasks_from: preflight") < rocm_playbook.index("- role: rocm") assert "tasks_from: apply" in rocm_playbook assert "tasks_from: preflight" in udev_playbook From 8853384bd146dc70e861d86ab83cc2574bd24adc Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 29 Jul 2026 11:17:12 +0800 Subject: [PATCH 55/65] refactor(deploy): reuse GPU sysfs detection --- .../playbooks/pb-gpu-access-discovery.yml | 21 ++++++------------- tests/skills/test_gpu_access_resolution.py | 10 ++++++++- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/deploy/ansible/playbooks/pb-gpu-access-discovery.yml b/deploy/ansible/playbooks/pb-gpu-access-discovery.yml index f788e0d9..e12229b3 100644 --- a/deploy/ansible/playbooks/pb-gpu-access-discovery.yml +++ b/deploy/ansible/playbooks/pb-gpu-access-discovery.yml @@ -102,19 +102,10 @@ | reject('equalto', '') | join('\n') }} changed_when: false - - name: Discover AMD display BDFs through sysfs - ansible.builtin.command: - argv: - - python3 - - -c - - >- - from pathlib import Path; devices = Path('/sys/bus/pci/devices'); - print('\n'.join(sorted(device.name for device in devices.iterdir() - if (device / 'vendor').read_text().strip() == '0x1002' and - (device / 'class').read_text().strip().startswith('0x03')))) - register: _auplc_discovery_sysfs - changed_when: false - failed_when: false + - name: Discover AMD display BDFs through shared sysfs detector + ansible.builtin.include_role: + name: gpu_access + tasks_from: detect - name: Record machine-readable GPU access discovery evidence ansible.builtin.set_fact: @@ -125,8 +116,8 @@ rc: "{{ _auplc_discovery_lspci.rc }}" stdout: "{{ _auplc_discovery_lspci.stdout | default('') }}" sysfs: - rc: "{{ _auplc_discovery_sysfs.rc }}" - stdout: "{{ _auplc_discovery_sysfs.stdout | default('') }}" + rc: "{{ _auplc_gpu_access_sysfs.rc }}" + stdout: "{{ _auplc_gpu_access_sysfs.stdout | default('') }}" changed_when: false - name: Write machine-readable GPU access discovery evidence locally diff --git a/tests/skills/test_gpu_access_resolution.py b/tests/skills/test_gpu_access_resolution.py index 37003f28..fdd802ea 100644 --- a/tests/skills/test_gpu_access_resolution.py +++ b/tests/skills/test_gpu_access_resolution.py @@ -66,7 +66,7 @@ def expected_targets(module, *names: str): return tuple(module.InventoryTarget(name=name) for name in names) -def test_discovery_playbook_serializes_the_exact_v1_host_evidence_shape() -> None: +def test_discovery_playbook_preserves_lspci_agreement_and_exact_v1_host_evidence_shape() -> None: playbook = DISCOVERY_PLAYBOOK.read_text(encoding="utf-8") evidence_block = playbook.split("_auplc_gpu_access_discovery_evidence:", maxsplit=1)[1].split( " changed_when:", maxsplit=1 @@ -83,6 +83,14 @@ def test_discovery_playbook_serializes_the_exact_v1_host_evidence_shape() -> Non assert '{"version":1,"hosts":[' in playbook assert "hostvars[discovery_host]._auplc_gpu_access_discovery_evidence" in playbook assert "| to_json" in playbook + assert "name: gpu_access" in playbook + assert "tasks_from: detect" in playbook + assert "_auplc_gpu_access_sysfs.rc" in playbook + assert "_auplc_gpu_access_sysfs.stdout" in playbook + assert "/sys/bus/pci/devices" not in playbook + assert 'argv: [lspci, -Dnn, -d, "1002::0300"]' in playbook + assert 'argv: [lspci, -Dnn, -d, "1002::0302"]' in playbook + assert 'argv: [lspci, -Dnn, -d, "1002::0380"]' in playbook def test_parse_fleet_evidence_accepts_the_exact_machine_evidence_schema() -> None: From f39f890924feb437fc45dc88d0cc6c42efa5ac7c Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 29 Jul 2026 11:18:00 +0800 Subject: [PATCH 56/65] test(deploy): preserve generated boolean policies --- tests/skills/test_gpu_artifact_generation.py | 3 +++ tests/skills/test_pxe_finalization.py | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/tests/skills/test_gpu_artifact_generation.py b/tests/skills/test_gpu_artifact_generation.py index 4e40a624..fe629147 100644 --- a/tests/skills/test_gpu_artifact_generation.py +++ b/tests/skills/test_gpu_artifact_generation.py @@ -154,6 +154,7 @@ def test_generator_discovers_mixed_ssh_targets_and_publishes_resolved_artifacts( inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") assert inventory.count("auplc_gpu_access_enabled: true") == 1 assert inventory.count("auplc_gpu_access_enabled: false") == 1 + assert "auplc_gpu_access_enabled: auto" not in inventory assert "auplc_render_gid" not in inventory assert "gpuAccess" not in (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") assert json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) == { @@ -191,6 +192,7 @@ def test_generator_allows_heterogeneous_gpu_hosts_and_publishes_boolean_only_art values = (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) assert inventory.count("auplc_gpu_access_enabled: true") == 2 + assert "auplc_gpu_access_enabled: auto" not in inventory assert "auplc_render_gid" not in inventory assert "gpuAccess" not in values assert manifest == {"version": 1, "status": "gpu_resolved", "hosts": {"agent": True, "server": True}} @@ -209,6 +211,7 @@ def test_generator_publishes_boolean_only_artifacts_for_all_cpu_ssh_targets( assert result.returncode == 0, result.stderr inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") assert inventory.count("auplc_gpu_access_enabled: false") == 2 + assert "auplc_gpu_access_enabled: auto" not in inventory assert "auplc_render_gid" not in inventory assert "gpuAccess" not in (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") assert json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) == { diff --git a/tests/skills/test_pxe_finalization.py b/tests/skills/test_pxe_finalization.py index 554d6049..dac1fb0a 100644 --- a/tests/skills/test_pxe_finalization.py +++ b/tests/skills/test_pxe_finalization.py @@ -97,8 +97,10 @@ def test_pxe_gpu_agents_publish_immediate_boolean_only_rootfs_artifacts( manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) pxe_vars = (out_dir / "pb-pxe-controller.vars.yml").read_text(encoding="utf-8") assert "auplc_render_gid" not in inventory + assert "auplc_gpu_access_enabled: auto" not in inventory assert "gpuAccess" not in values assert "pxe_gpu_access_enabled: true" in pxe_vars + assert "pxe_gpu_access_enabled: auto" not in pxe_vars assert manifest == { "version": 1, "status": "cpu_only", @@ -121,6 +123,7 @@ def test_pxe_cpu_agents_publish_a_disabled_rootfs_policy(tmp_path: Path, monkeyp pxe_vars = (out_dir / "pb-pxe-controller.vars.yml").read_text(encoding="utf-8") manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) assert "pxe_gpu_access_enabled: false" in pxe_vars + assert "pxe_gpu_access_enabled: auto" not in pxe_vars assert "auplc_render_gid" not in pxe_vars assert manifest["pxe_rootfs"] == {"gpu_access_enabled": False} @@ -137,6 +140,9 @@ def test_pxe_gpu_controller_and_rootfs_publish_independent_booleans( inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) assert "auplc_gpu_access_enabled: true" in inventory + assert "auplc_gpu_access_enabled: auto" not in inventory + pxe_vars = (out_dir / "pb-pxe-controller.vars.yml").read_text(encoding="utf-8") + assert "pxe_gpu_access_enabled: auto" not in pxe_vars assert manifest["status"] == "gpu_resolved" assert manifest["pxe_rootfs"] == {"gpu_access_enabled": True} From 5431365958636be9356e26499b9ab30919c2a06a Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 29 Jul 2026 11:18:52 +0800 Subject: [PATCH 57/65] docs(deploy): document automatic GPU detection --- deploy/README.md | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index 23c29e8c..29dd70f4 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -68,10 +68,18 @@ DaemonSets are ready and that GPU capacity is advertised. #### SSH-preinstalled Edit `deploy/ansible/inventory.yml` with the server and agent hostnames, IPs, -k3s token, and other site settings. Every host entry must set -`auplc_gpu_access_enabled` to the YAML boolean `true` or `false`. Use `true` -only for hosts where the AMD GPU access package and ROCm should be installed. -Don't quote the boolean or use alternatives such as `yes` and `no`. +k3s token, and other site settings. Keep the human template default, +`auplc_gpu_access_enabled: auto`, unquoted on each host. `auto` runs Python 3 on +that host to scan `/sys/bus/pci/devices` for vendor `0x1002` devices whose PCI +class starts with `0x03`. It does not use `lspci` or require `pciutils`. A match +enables ROCm and the AMD GPU access package; a successful scan with no match +skips both. If a scan fails, the play aborts before either is changed, and +`any_errors_fatal` stops the play for all hosts. + +Set an unquoted YAML boolean `true` or `false` only when you need to override +detection. `true` forces ROCm and package installation, while `false` forces +both to be skipped. Either boolean bypasses the scan. Don't quote any of these +values or use alternatives such as `yes` and `no`. For example: @@ -82,12 +90,12 @@ k3s_cluster: hosts: controller-1: ansible_host: 192.0.2.10 - auplc_gpu_access_enabled: false + auplc_gpu_access_enabled: auto agent: hosts: gpu-worker-1: ansible_host: 192.0.2.11 - auplc_gpu_access_enabled: true + auplc_gpu_access_enabled: auto ``` Copy the human-maintained multi-node values example, then edit the copy for the @@ -122,11 +130,12 @@ helm upgrade --install jupyterhub ./runtime/chart \ -f runtime/values-multi-nodes.yaml ``` -The validator checks an inventory supplied by itself for exactly one explicit -YAML boolean `auplc_gpu_access_enabled` on every managed host. This validates -the direct-edit workflow without a generated GPU resolution report. -`--gpu-resolution` may be supplied only with `--inventory`; when both are -supplied, the validator also checks generated-artifact consistency. +With `--inventory` alone, the validator accepts exactly one unquoted `auto`, +`true`, or `false` value for `auplc_gpu_access_enabled` on every managed host. +This validates the direct-edit workflow without a generated GPU resolution +report. `--gpu-resolution` may be supplied only with `--inventory`; that pair +is for generated artifacts, whose inventory values and resolution entries must +remain strict booleans. The generator never writes `auto`. The installer, Ansible GPU access role, and PXE controller install AMD's `amdgpu-insecure-instinct-udev-rules` package, pinned to version From f47b5b29cad2089f603718caf759d534a5dd86c5 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 29 Jul 2026 11:19:41 +0800 Subject: [PATCH 58/65] docs(ansible): document automatic GPU policy --- deploy/ansible/README.md | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/deploy/ansible/README.md b/deploy/ansible/README.md index fefe0fea..0c989e0a 100644 --- a/deploy/ansible/README.md +++ b/deploy/ansible/README.md @@ -26,13 +26,18 @@ K3s cluster setup playbooks based on [k3s-ansible](https://github.com/k3s-io/k3s For the human SSH-preinstalled workflow, edit `inventory.yml` directly and use the playbook commands in the [deployment guide](../README.md). Every server and -agent host entry must define `auplc_gpu_access_enabled` as the unquoted YAML -boolean `true` or `false`. Set it to `true` only on hosts where the GPU access -package and ROCm should be installed. Pass `--inventory` to the deployment -validator to check this explicit per-host policy. A generated -`--gpu-resolution` report is not required for the human workflow; if supplied, -it requires `--inventory`, and the validator checks the two generated artifacts -for consistency. +agent host entry defaults to unquoted `auplc_gpu_access_enabled: auto`. On each +host, `auto` uses Python 3 to scan `/sys/bus/pci/devices` for vendor `0x1002` +and PCI class `0x03*`; it has no `lspci` or `pciutils` dependency. A match +enables ROCm and the GPU access package, while a successful empty scan skips +both. A scan failure aborts before mutation, and `any_errors_fatal` stops the +play. Unquoted `true` and `false` force enablement or disablement and bypass +detection. + +Pass `--inventory` to validate direct values of `auto`, `true`, or `false`. A +generated `--gpu-resolution` report is not required for the human workflow. If +supplied, it requires `--inventory`, and both generated artifacts must use +strict booleans. The deploy skill never generates `auto`. The deploy skill has a separate generator-first SSH workflow that discovers GPU hosts from managed-host evidence. PXE is always generator-based and uses only From e38dd7d37e452f2bd9f3e4cfb283ec2a0637711f Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 29 Jul 2026 11:20:42 +0800 Subject: [PATCH 59/65] docs(skills): distinguish direct and generated GPU policy --- skills/deploy-aup-learning-cloud/SKILL.md | 10 ++++++---- skills/deploy-aup-learning-cloud/reference.md | 14 ++++++++------ skills/deploy-aup-learning-cloud/scripts/README.md | 14 ++++++++++---- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/skills/deploy-aup-learning-cloud/SKILL.md b/skills/deploy-aup-learning-cloud/SKILL.md index 9af9e4c4..cc73af5f 100644 --- a/skills/deploy-aup-learning-cloud/SKILL.md +++ b/skills/deploy-aup-learning-cloud/SKILL.md @@ -80,14 +80,16 @@ then run the topology's exact validator command from the - `--repo` - `--topology` -- `--inventory` to validate explicit host booleans +- `--inventory` to validate generated host booleans - `--gpu-resolution` with `--inventory` for generated-artifact consistency - both `--values` files - `--pxe-vars` for PXE only -An inventory can be validated without a GPU resolution report. A resolution -report requires an inventory. Supply both in this generator-first workflow so -the validator also checks their consistency. +A human direct inventory can be validated by itself with unquoted `auto`, +`true`, or `false`. A resolution report requires an inventory, and that pairing +accepts only generated boolean values. Supply both in this generator-first +workflow so the validator checks their consistency. The skill resolves every +host to `true` or `false` and never generates `auto`. Stop on validation failure. After a clean result, continue with the topology's Ansible, device plugin, and Helm commands in the skill scripts guide. Treat the diff --git a/skills/deploy-aup-learning-cloud/reference.md b/skills/deploy-aup-learning-cloud/reference.md index e10c8e9c..b1da7733 100644 --- a/skills/deploy-aup-learning-cloud/reference.md +++ b/skills/deploy-aup-learning-cloud/reference.md @@ -15,7 +15,8 @@ reference. The skill is generator-first for both topologies. Don't hand-author generated GPU policy, including for SSH. Create deployment specs from the current -`--print-schema` output. +`--print-schema` output. Generation resolves hosts to strict `true` or `false` +values and never writes `auto`. ## Canonical validation inputs @@ -29,11 +30,12 @@ Use the topology's validator command from the - base and generated overlays as two `--values` arguments - canonical PXE vars with `--pxe-vars` for PXE only -`--inventory` alone validates that every managed host has exactly one explicit -YAML boolean `auplc_gpu_access_enabled`. `--gpu-resolution` requires -`--inventory`; supplying both enables generated-artifact consistency checks. -The skill supplies both because its workflow is generator-first. Generation -and validation must finish before Ansible or Helm changes are made. +For a human direct inventory, `--inventory` alone accepts exactly one unquoted +`auto`, `true`, or `false` value for `auplc_gpu_access_enabled` on every managed +host. `--gpu-resolution` requires `--inventory`; supplying both checks generated +artifacts and requires strict booleans in the inventory and resolution report. +The skill supplies both because its workflow is generator-first. Generation and +validation must finish before Ansible or Helm changes are made. ## GPU permission contract diff --git a/skills/deploy-aup-learning-cloud/scripts/README.md b/skills/deploy-aup-learning-cloud/scripts/README.md index dad1fd59..83946594 100644 --- a/skills/deploy-aup-learning-cloud/scripts/README.md +++ b/skills/deploy-aup-learning-cloud/scripts/README.md @@ -19,6 +19,10 @@ The SSH topology discovers GPU hosts from managed-host evidence. Users don't provide a GPU host list. The PXE topology has one GPU policy input: `pxe.diskless_agents_have_amd_gpus`. +Generation resolves every host to `true` or `false`; it never writes `auto`. +Generated inventory and GPU resolution entries are strict booleans so their +consistency can be checked. + Generate specs from fresh `--print-schema` output. Both topologies write their canonical artifacts immediately. For PXE, review and validate those files, then run the controller playbook with the generated `inventory.yml` and @@ -120,10 +124,12 @@ rule safety checks described in the deployment guide. The exact topology commands above pass `--repo`, `--topology`, `--inventory`, `--gpu-resolution`, two `--values` arguments, and `--pxe-vars` for PXE only. -`--inventory` alone validates that every managed host defines exactly one -explicit YAML boolean `auplc_gpu_access_enabled`. `--gpu-resolution` requires -`--inventory`; supplying both performs generated-artifact consistency checks. -The generator-first skill workflow supplies both. +For direct validation, `--inventory` alone accepts exactly one unquoted `auto`, +`true`, or `false` value for `auplc_gpu_access_enabled` on every managed host. +`--gpu-resolution` requires `--inventory`; supplying both switches to generated +consistency validation, where inventory and resolution values must be strict +booleans. The generator-first skill workflow supplies both and never generates +`auto`. ## Conventions From d64b48ddedbe44188a66c42acceea166d9d16331 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:56:13 +0800 Subject: [PATCH 60/65] test(installer): reduce GPU orchestration coverage --- tests/installer/test_cli_gpu_access.py | 84 +++----------------------- tests/installer/test_gpu_hardware.py | 24 -------- tests/installer/test_overlay.py | 19 +----- 3 files changed, 12 insertions(+), 115 deletions(-) diff --git a/tests/installer/test_cli_gpu_access.py b/tests/installer/test_cli_gpu_access.py index b48210b4..51854223 100644 --- a/tests/installer/test_cli_gpu_access.py +++ b/tests/installer/test_cli_gpu_access.py @@ -5,7 +5,6 @@ from __future__ import annotations from collections.abc import Callable -from contextlib import contextmanager from pathlib import Path import pytest @@ -21,22 +20,14 @@ def test_full_install_gates_gpu_access_without_passing_it_to_the_overlay( monkeypatch, hardware: GpuHardware, expected_provision_count: int ) -> None: events: list[str] = [] - stages: list[tuple[str, int, int]] = [] state = InstallerState() paths = RuntimePaths(chart_path=Path("chart"), values_path=Path("values.yaml"), overlay_path=Path("overlay.yaml")) - @contextmanager - def fake_stage(label: str, *, idx: int, total: int): - stages.append((label, idx, total)) - yield - def fake_overlay(*args: object, **kwargs: object) -> Path: assert "render_gid" not in kwargs - events.append("overlay") return paths.overlay_path monkeypatch.setattr(state, "runtime_paths", lambda: paths) - monkeypatch.setattr(cli, "stage", fake_stage) monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: hardware) monkeypatch.setattr(cli, "detect_and_configure_gpu", lambda *args, **kwargs: events.append("detect")) monkeypatch.setattr(cli, "provision_gpu_access", lambda **kwargs: events.append("provision")) @@ -55,18 +46,6 @@ def fake_overlay(*args: object, **kwargs: object) -> Path: assert events.count("provision") == expected_provision_count if expected_provision_count: assert events.index("provision") < events.index("device-plugin") - assert events.count("overlay") == 2 - assert stages == [ - ("Detecting GPU", 1, 9), - ("Provisioning GPU device access", 2, 9), - ("Generating values overlay (initial)", 3, 9), - ("Installing helm + k9s", 4, 9), - ("Installing K3s (single-node)", 5, 9), - ("Pulling custom + external images", 6, 9), - ("Deploying ROCm GPU device plugin + node labeller", 7, 9), - ("Refreshing values overlay from node labels", 8, 9), - ("Deploying JupyterHub runtime (helm install + wait)", 9, 9), - ] @pytest.mark.parametrize(("hardware", "expected_provision_count"), [(GpuHardware.GPU, 1), (GpuHardware.CPU, 0)]) @@ -79,7 +58,6 @@ def test_runtime_upgrade_gates_host_access_without_provisioning_helm_values( def fake_overlay(*args: object, **kwargs: object) -> Path: assert "render_gid" not in kwargs - events.append("overlay") return paths.overlay_path monkeypatch.setattr(state, "runtime_paths", lambda: paths) @@ -94,59 +72,19 @@ def fake_overlay(*args: object, **kwargs: object) -> Path: cli.cmd_rt_upgrade(state) assert events.count("provision") == expected_provision_count - assert events[-5:] == ["detect", "refine", "preserve-courses", "overlay", "upgrade-runtime"] - - -@pytest.mark.parametrize( - ("command", "expected_events"), - [ - (cli.cmd_dev_deploy, ("detect", "refine", "overlay", "deploy-runtime")), - (cli.cmd_dev_upgrade, ("detect", "refine", "preserve-courses", "overlay", "upgrade-runtime")), - (cli.cmd_rt_install, ("detect", "refine", "overlay", "deploy-runtime")), - (cli.cmd_rt_upgrade, ("detect", "refine", "preserve-courses", "overlay", "upgrade-runtime")), - ], -) -def test_cpu_hardware_skips_host_access_and_preserves_runtime_flow( - monkeypatch, command: Callable[[InstallerState], None], expected_events: tuple[str, ...] -) -> None: - events: list[str] = [] - state = InstallerState() - paths = RuntimePaths(chart_path=Path("chart"), values_path=Path("values.yaml"), overlay_path=Path("overlay.yaml")) - - monkeypatch.setattr(state, "runtime_paths", lambda: paths) - monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: GpuHardware.CPU) - monkeypatch.setattr( - cli, "provision_gpu_access", lambda **kwargs: (_ for _ in ()).throw(AssertionError("must not provision")) - ) - monkeypatch.setattr(cli, "detect_and_configure_gpu", lambda *args, **kwargs: events.append("detect")) - monkeypatch.setattr(cli, "refine_gpu_config_from_node_labels", lambda *args, **kwargs: events.append("refine")) - monkeypatch.setattr(cli, "_preserve_courses_for_upgrade", lambda *args, **kwargs: events.append("preserve-courses")) - monkeypatch.setattr( - cli, - "generate_values_overlay", - lambda *args, **kwargs: events.append("overlay") or paths.overlay_path, - ) - monkeypatch.setattr(cli, "deploy_runtime", lambda *args, **kwargs: events.append("deploy-runtime")) - monkeypatch.setattr(cli, "upgrade_runtime", lambda *args, **kwargs: events.append("upgrade-runtime")) - - command(state) - - assert events == list(expected_events) @pytest.mark.parametrize( ("reinstall", "delegate_name"), [(cli.cmd_dev_reinstall, "cmd_dev_deploy"), (cli.cmd_rt_reinstall, "cmd_rt_install")], ) -@pytest.mark.parametrize( - ("hardware", "expected_access_events"), [(GpuHardware.GPU, ["provision"]), (GpuHardware.CPU, [])] -) +@pytest.mark.parametrize(("hardware", "expected_provision_count"), [(GpuHardware.GPU, 1), (GpuHardware.CPU, 0)]) def test_reinstall_gates_host_access_before_removing_runtime( monkeypatch, reinstall: Callable[[InstallerState], None], delegate_name: str, hardware: GpuHardware, - expected_access_events: list[str], + expected_provision_count: int, ) -> None: events: list[str] = [] state = InstallerState() @@ -159,15 +97,17 @@ def test_reinstall_gates_host_access_before_removing_runtime( reinstall(state) - assert events == [*expected_access_events, "remove-runtime", "sleep", "delegate"] + assert events.count("provision") == expected_provision_count + assert events.index("remove-runtime") < events.index("delegate") + if expected_provision_count: + assert events.index("provision") < events.index("remove-runtime") def test_unknown_hardware_blocks_full_install_before_gpu_access_mutation(monkeypatch) -> None: - events: list[str] = [] state = InstallerState() monkeypatch.setattr(cli, "classify_gpu_hardware", lambda: GpuHardware.UNKNOWN) - monkeypatch.setattr(cli, "detect_and_configure_gpu", lambda *args, **kwargs: events.append("detect")) + monkeypatch.setattr(cli, "detect_and_configure_gpu", lambda *args, **kwargs: None) monkeypatch.setattr( cli, "provision_gpu_access", lambda **kwargs: (_ for _ in ()).throw(AssertionError("must not provision")) ) @@ -175,8 +115,6 @@ def test_unknown_hardware_blocks_full_install_before_gpu_access_mutation(monkeyp with pytest.raises(RuntimeError, match="hardware"): cli._cmd_install_inner(state, pull=True) - assert events == ["detect"] - @pytest.mark.parametrize( ("reinstall", "delegate_name"), @@ -198,12 +136,8 @@ def test_unknown_hardware_blocks_reinstall_before_runtime_removal( with pytest.raises(RuntimeError, match="hardware"): reinstall(state) - assert events == [] - - -def test_cli_exposes_no_render_gid_reconciliation_api() -> None: - assert not hasattr(cli, "_render_gid_for_local_hardware") - assert not hasattr(cli, "load_existing_gpu_access") + assert "remove-runtime" not in events + assert "delegate" not in events def test_gpu_hardware_gate_passes_offline_bundle_context_to_package_provisioning( diff --git a/tests/installer/test_gpu_hardware.py b/tests/installer/test_gpu_hardware.py index 6a455e26..aa8bb15b 100644 --- a/tests/installer/test_gpu_hardware.py +++ b/tests/installer/test_gpu_hardware.py @@ -63,30 +63,6 @@ def test_classify_gpu_hardware_returns_unknown_for_incomplete_pci_evidence(tmp_p assert hardware is GpuHardware.UNKNOWN -def test_classify_gpu_hardware_returns_unknown_for_malformed_pci_evidence(tmp_path: Path) -> None: - pci_devices = tmp_path / "devices" - malformed_vendor = pci_devices / "0000:00:02.0" - malformed_vendor.mkdir(parents=True) - (malformed_vendor / "vendor").write_text("0xZZZZ\n", encoding="ascii") - (malformed_vendor / "class").write_text("0x030000\n", encoding="ascii") - - hardware = classify_gpu_hardware(pci_devices) - - assert hardware is GpuHardware.UNKNOWN - - -def test_classify_gpu_hardware_returns_unknown_for_unreadable_pci_attribute(tmp_path: Path) -> None: - pci_devices = tmp_path / "devices" - unreadable_class = pci_devices / "0000:00:02.0" - unreadable_class.mkdir(parents=True) - (unreadable_class / "vendor").write_text("0x8086\n", encoding="ascii") - (unreadable_class / "class").mkdir() - - hardware = classify_gpu_hardware(pci_devices) - - assert hardware is GpuHardware.UNKNOWN - - def test_classify_gpu_hardware_prefers_positive_amd_evidence_over_incomplete_sibling(tmp_path: Path) -> None: pci_devices = tmp_path / "devices" incomplete_device = pci_devices / "0000:00:02.0" diff --git a/tests/installer/test_overlay.py b/tests/installer/test_overlay.py index 677eb115..8e4ece63 100644 --- a/tests/installer/test_overlay.py +++ b/tests/installer/test_overlay.py @@ -107,29 +107,16 @@ def test_default_selection_round_trips_valid_yaml() -> None: assert "teams" not in parsed["custom"] -def test_overlay_never_emits_gpu_access_contract() -> None: - text = emit_overlay( - _strix_halo_cfg(), - image_registry="ghcr.io/amdresearch", - image_tag="v1.0", - courses=CourseSelection.default(), - offline_mode=False, - ) - parsed = yaml.safe_load(text) - - assert "gpuAccess" not in parsed["custom"] - assert "renderGid" not in text - assert "supplementalGroups" not in text - - def test_overlay_keeps_gpu_resources_without_gpu_access_contract() -> None: - _, parsed = _render( + text, parsed = _render( _strix_halo_cfg(), courses=CourseSelection.default(), ) custom = parsed["custom"] assert "gpuAccess" not in custom + assert "renderGid" not in text + assert "supplementalGroups" not in text assert set(custom["resources"]["images"]) == set(GPU_RESOURCE_KEYS) assert set(custom["resources"]["metadata"]) == set(GPU_RESOURCE_KEYS) assert "teams" not in custom From 31afcedf0a9e4f9162948984a5e2fbe6b2d09ec9 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:57:06 +0800 Subject: [PATCH 61/65] test(installer): consolidate GPU package safety coverage --- tests/installer/test_gpu_access.py | 14 +++-------- tests/installer/test_gpu_access_ordering.py | 27 +++++---------------- 2 files changed, 9 insertions(+), 32 deletions(-) diff --git a/tests/installer/test_gpu_access.py b/tests/installer/test_gpu_access.py index 159e6a83..ad3285a1 100644 --- a/tests/installer/test_gpu_access.py +++ b/tests/installer/test_gpu_access.py @@ -123,16 +123,11 @@ def fake_run(command: list[str], **_: object) -> SimpleNamespace: # Then: the exact Radeon URL is downloaded, verified, installed, and cleaned up. downloaded_path = Path(downloads[0][-1]) - assert downloads == [["wget", "-q", gpu_access.AMD_GPU_UDEV_PACKAGE_URL, "-O", str(downloaded_path)]] + assert downloads[0][2] == gpu_access.AMD_GPU_UDEV_PACKAGE_URL assert verified == [downloaded_path] assert not downloaded_path.exists() - assert host.calls == [ - "installed-version", - f"install-package:{downloaded_path}", - "installed-version", - f"owns-rule:{AMD_GPU_UDEV_PACKAGE_RULES_PATH}", - f"read:{AMD_GPU_UDEV_PACKAGE_RULES_PATH}", - ] + assert host.installed_version == AMD_GPU_UDEV_PACKAGE_VERSION + assert host.files[AMD_GPU_UDEV_PACKAGE_RULES_PATH] == AMD_GPU_UDEV_PACKAGE_RULES def test_installed_package_requires_the_pinned_version_and_its_exact_rule() -> None: @@ -146,7 +141,6 @@ def test_installed_package_requires_the_pinned_version_and_its_exact_rule() -> N provision_gpu_access(host) # Then: no download, install, legacy removal, or device probe is performed. - assert host.files == {AMD_GPU_UDEV_PACKAGE_RULES_PATH: AMD_GPU_UDEV_PACKAGE_RULES} assert not any(call.startswith(("install-package:", "remove-rule:")) for call in host.calls) assert not any(call in {"reload-udev", "trigger-udev", "settle-udev"} for call in host.calls) @@ -193,7 +187,6 @@ def test_symlinked_legacy_rule_fails_closed_before_installation(monkeypatch: pyt def test_official_rule_matches_the_extracted_deb_policy_not_the_old_pxe_shape() -> None: # Given: the exact package verification constant. rules = AMD_GPU_UDEV_PACKAGE_RULES - old_pxe_shape = 'KERNEL=="kfd", MODE="0666"\nKERNEL=="renderD*", MODE="0666"\n' # When: its policy is inspected. # Then: it matches the extracted package rule rather than the former two-line PXE shape. @@ -201,7 +194,6 @@ def test_official_rule_matches_the_extracted_deb_policy_not_the_old_pxe_shape() 'KERNEL=="kfd", GROUP="render", MODE="0666"\n' 'SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="render", MODE="0666"\n' ) - assert rules != old_pxe_shape assert "card" not in rules diff --git a/tests/installer/test_gpu_access_ordering.py b/tests/installer/test_gpu_access_ordering.py index 6363ffc8..93635798 100644 --- a/tests/installer/test_gpu_access_ordering.py +++ b/tests/installer/test_gpu_access_ordering.py @@ -114,29 +114,14 @@ def fail_install(deb: Path) -> None: assert "reload-udev" not in host.calls -def test_wrong_version_package_owned_differing_conffile_converges( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path +@pytest.mark.parametrize("installed_version", ["30.30.4.0-older", None]) +def test_package_owned_differing_conffile_converges( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, installed_version: str | None ) -> None: - # Given: a package-owned conffile from a different installed package version. - host = FakeGpuAccessHost( - files={AMD_GPU_UDEV_PACKAGE_RULES_PATH: 'KERNEL=="kfd", MODE="0600"\n'}, - installed_version="30.30.4.0-older", - package_owns_rule=True, - ) - monkeypatch.setattr(gpu_access, "verify_sha256", lambda *args: None) - - # When: the pinned package is installed from an offline bundle. - provision_gpu_access(host, offline_mode=True, bundle_dir=_offline_bundle(tmp_path)) - - # Then: forced installation replaces the differing conffile with the exact package rule. - assert host.files[AMD_GPU_UDEV_PACKAGE_RULES_PATH] == AMD_GPU_UDEV_PACKAGE_RULES - assert not any(call.startswith("remove-rule:") for call in host.calls) - - -def test_partial_package_owned_conffile_converges(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - # Given: a config-files package state that still owns a differing conffile. + # Given: a wrong-version or partial package state that owns a differing conffile. host = FakeGpuAccessHost( files={AMD_GPU_UDEV_PACKAGE_RULES_PATH: 'KERNEL=="kfd", MODE="0600"\n'}, + installed_version=installed_version, package_owns_rule=True, ) monkeypatch.setattr(gpu_access, "verify_sha256", lambda *args: None) @@ -144,7 +129,7 @@ def test_partial_package_owned_conffile_converges(monkeypatch: pytest.MonkeyPatc # When: the pinned package is installed from an offline bundle. provision_gpu_access(host, offline_mode=True, bundle_dir=_offline_bundle(tmp_path)) - # Then: ownership prevents legacy admission and the package converges to the exact rule. + # Then: forced installation converges to the exact package rule without legacy deletion. assert host.files[AMD_GPU_UDEV_PACKAGE_RULES_PATH] == AMD_GPU_UDEV_PACKAGE_RULES assert not any(call.startswith("remove-rule:") for call in host.calls) From 5c1dbd971fa658802575b152a25b004e82fc3e67 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:58:07 +0800 Subject: [PATCH 62/65] test(deploy): trim generated config validation coverage --- .../skills/test_config_generation_security.py | 1 - tests/skills/test_deploy_scripts.py | 431 +----------------- .../test_direct_inventory_validation.py | 29 +- 3 files changed, 9 insertions(+), 452 deletions(-) diff --git a/tests/skills/test_config_generation_security.py b/tests/skills/test_config_generation_security.py index adb6c282..66d5a49e 100644 --- a/tests/skills/test_config_generation_security.py +++ b/tests/skills/test_config_generation_security.py @@ -86,7 +86,6 @@ def test_generator_applies_the_normal_unknown_field_policy_to_draft_gpu_fields() "raw", [ '{"topology":"ssh-preinstalled","topology":"pxe-diskless"}', - '{"topology":"pxe-diskless","k3s_version":"v1.32.3+k3s1","server":{"name":"server","ip":"192.168.1.10"},"network":{"interface":"eno1","subnet":"192.168.1.0/24"},"pxe":{"authorized_keys":["ssh-ed25519 AAA"],"diskless_agents_have_amd_gpus":true,"diskless_agents_have_amd_gpus":false}}', ], ) def test_generator_rejects_duplicate_public_policy_keys_before_discovery(tmp_path: Path, raw: str) -> None: diff --git a/tests/skills/test_deploy_scripts.py b/tests/skills/test_deploy_scripts.py index c57f916d..c4901737 100644 --- a/tests/skills/test_deploy_scripts.py +++ b/tests/skills/test_deploy_scripts.py @@ -19,33 +19,7 @@ DEPLOY_SCRIPTS = ROOT / "skills" / "deploy-aup-learning-cloud" / "scripts" VALIDATE = DEPLOY_SCRIPTS / "validate.py" GEN_CONFIGS = DEPLOY_SCRIPTS / "gen_configs.py" -CONFIG_GENERATION = DEPLOY_SCRIPTS / "config_generation.py" ARTIFACT_STORE = DEPLOY_SCRIPTS / "artifact_store.py" -VALUES_RESOLUTION_PARSING = DEPLOY_SCRIPTS / "values_resolution_parsing.py" - -EXPECTED_GENERATOR_SCHEMA = { - "topology": "pxe-diskless | ssh-preinstalled", - "k3s_version": "v1.32.3+k3s1", - "server": {"name": "aipc1", "ip": "192.168.0.140"}, - "agents": [{"name": "aipc2", "ip": "192.168.0.141"}], - "network": { - "interface": "enp1s0", - "subnet": "192.168.0.0/24", - "gateway": "192.168.0.1", - "dns_servers": "8.8.8.8,8.8.4.4", - }, - "pxe": { - "authorized_keys": ["ssh-ed25519 AAAA... you@host"], - "rootfs_password": "", - "web_port": 8080, - "diskless_agents_have_amd_gpus": True, - }, - "accelerators": {"strix-halo": {"product_name": "AMD_Radeon_8060S_Graphics"}}, - "storage": {"class": "nfs-client"}, - "proxy": {"node_port": 30890}, - "auth_mode": "auto-login", - "images": {"cpu": "ghcr.io/amdresearch/auplc-default:latest", "gpu": "ghcr.io/amdresearch/auplc-base:latest"}, -} def run_script(script: Path, *args: str, cwd: Path | None = None) -> subprocess.CompletedProcess[str]: @@ -278,43 +252,6 @@ def test_validator_retains_selectors_from_partial_accelerator_overlays(tmp_path: assert "AMD_Radeon_8060S_Graphics" in result.stdout -def test_values_resolution_parser_preserves_overlay_precedence_and_error_categories(tmp_path: Path) -> None: - parser = load_deploy_module("values_resolution_parsing", VALUES_RESOLUTION_PARSING) - repo = tmp_path / "checkout" - base = write_file( - repo / "base.yaml", - """custom: - accelerators: - strix-halo: - nodeSelector: - amd.com/gpu.product-name: AMD_Radeon_8060S_Graphics - resources: - metadata: - gpu: - acceleratorKeys: [strix-halo] -""", - ) - partial_overlay = write_file( - repo / "partial.yaml", - """custom: - accelerators: - strix-halo: - displayName: Renamed -""", - ) - invalid_overlay = write_file(repo / "invalid.yaml", "custom: *defaults\n") - - result = parser.collect_effective_values( - repo, - [str(base), str(partial_overlay), "missing.yaml", str(invalid_overlay)], - ) - - assert result.accelerators == {"strix-halo": "AMD_Radeon_8060S_Graphics"} - assert result.metadata == {"gpu": ["strix-halo"]} - assert result.missing_files == ["values file not found: missing.yaml"] - assert result.parse_errors == ["unsupported YAML syntax at custom"] - - def test_validator_accepts_quoted_product_label_keys(tmp_path: Path) -> None: repo = tmp_path / "checkout" values = write_file( @@ -608,69 +545,6 @@ def test_validator_uses_generated_pxe_vars_file_when_requested(tmp_path: Path) - assert "k3s_version == pxe_k3s_version" in result.stdout -def test_validator_preserves_explicit_selector_and_accelerator_key_clears(tmp_path: Path) -> None: - repo = tmp_path / "checkout" - base = write_file( - repo / "runtime/values.yaml", - """custom: - accelerators: - strix-halo: - nodeSelector: - amd.com/gpu.product-name: AMD_Radeon_8060S_Graphics - resources: - metadata: - gpu: - acceleratorKeys: [strix-halo] -""", - ) - selector_clear = write_file( - repo / "selector-clear.yaml", - """custom: - accelerators: - strix-halo: - nodeSelector: - amd.com/gpu.product-name: null -""", - ) - keys_clear = write_file( - repo / "keys-clear.yaml", - """custom: - resources: - metadata: - gpu: - acceleratorKeys: ~ -""", - ) - - selector_result = run_script( - VALIDATE, - "--repo", - str(repo), - "--topology", - "ssh-preinstalled", - "--values", - str(base), - "--values", - str(selector_clear), - ) - keys_result = run_script( - VALIDATE, - "--repo", - str(repo), - "--topology", - "ssh-preinstalled", - "--values", - str(base), - "--values", - str(keys_clear), - ) - - assert selector_result.returncode == 1 - assert "active accelerator 'strix-halo' has no amd.com/gpu.product-name nodeSelector" in selector_result.stdout - assert keys_result.returncode == 0, keys_result.stdout + keys_result.stderr - assert "no acceleratorKeys found" in keys_result.stdout - - def test_validator_honors_every_supported_explicit_clear_syntax(tmp_path: Path) -> None: repo = tmp_path / "checkout" base = write_file( @@ -764,36 +638,6 @@ def test_validator_main_resets_report_state_between_invocations(tmp_path: Path) assert second == 0 -def test_validator_requires_product_labels_under_active_accelerator_node_selectors(tmp_path: Path) -> None: - repo = tmp_path / "checkout" - values = write_file( - repo / "runtime/values.yaml", - """custom: - accelerators: - strix-halo: - env: - amd.com/gpu.product-name: AMD_Radeon_8060S_Graphics - resources: - metadata: - gpu: - acceleratorKeys: [strix-halo] -""", - ) - - result = run_script( - VALIDATE, - "--repo", - str(repo), - "--topology", - "ssh-preinstalled", - "--values", - str(values), - ) - - assert result.returncode == 1 - assert "active accelerator 'strix-halo' has no amd.com/gpu.product-name nodeSelector" in result.stdout - - def test_validator_ignores_accelerators_and_metadata_outside_custom_resources(tmp_path: Path) -> None: repo = tmp_path / "checkout" values = write_file( @@ -892,60 +736,6 @@ def test_validator_fails_when_an_active_accelerator_has_no_product_selector(tmp_ assert "active accelerator 'strix-halo' has no amd.com/gpu.product-name nodeSelector" in result.stdout -def test_validator_accepts_consistent_cpu_only_gpu_artifacts(tmp_path: Path) -> None: - repo = tmp_path / "checkout" - inventory = write_file( - repo / "generated/inventory.yml", - """k3s_cluster: - children: - server: - hosts: - server: - ansible_host: 192.168.1.10 - auplc_gpu_access_enabled: false - agent: - hosts: - agent: - ansible_host: 192.168.1.11 - auplc_gpu_access_enabled: false -""", - ) - values = write_file( - repo / "generated/values-basic-example.yaml", - """custom: - resources: - metadata: {} -""", - ) - resolution = write_file( - repo / "generated/gpu-access-resolution.json", - json.dumps( - { - "version": 1, - "status": "cpu_only", - "hosts": {"agent": False, "server": False}, - } - ), - ) - - result = run_script( - VALIDATE, - "--repo", - str(repo), - "--topology", - "ssh-preinstalled", - "--inventory", - str(inventory), - "--values", - str(values), - "--gpu-resolution", - str(resolution), - ) - - assert result.returncode == 0, result.stdout + result.stderr - assert "GPU access artifacts agree" in result.stdout - - def test_validator_accepts_consistent_gpu_resolved_artifacts(tmp_path: Path) -> None: repo = tmp_path / "checkout" inventory, values, resolution = write_resolved_gpu_artifacts(repo) @@ -1011,84 +801,6 @@ def test_validator_rejects_malformed_pending_or_duplicate_gpu_resolution( assert expected_error in result.stdout -@pytest.mark.parametrize( - ("inventory_content", "expected_error"), - [ - ( - """k3s_cluster: - children: - server: - hosts: - server: - ansible_host: 192.168.1.10 - agent: - hosts: - agent: - ansible_host: 192.168.1.11 - auplc_gpu_access_enabled: false -""", - "inventory host 'server' must define exactly one auplc_gpu_access_enabled", - ), - ( - """k3s_cluster: - children: - server: - hosts: - server: - ansible_host: 192.168.1.10 - auplc_gpu_access_enabled: yes - agent: - hosts: - agent: - ansible_host: 192.168.1.11 - auplc_gpu_access_enabled: false -""", - "inventory host 'server' has malformed auplc_gpu_access_enabled", - ), - ( - """k3s_cluster: - children: - server: - hosts: - server: - ansible_host: 192.168.1.10 - auplc_gpu_access_enabled: true - auplc_gpu_access_enabled: false - agent: - hosts: - agent: - ansible_host: 192.168.1.11 - auplc_gpu_access_enabled: false -""", - "inventory host 'server' must define exactly one auplc_gpu_access_enabled", - ), - ], -) -def test_validator_rejects_missing_malformed_or_duplicate_inventory_host_booleans( - tmp_path: Path, inventory_content: str, expected_error: str -) -> None: - repo = tmp_path / "checkout" - inventory, values, resolution = write_resolved_gpu_artifacts(repo) - inventory.write_text(inventory_content, encoding="utf-8") - - result = run_script( - VALIDATE, - "--repo", - str(repo), - "--topology", - "ssh-preinstalled", - "--inventory", - str(inventory), - "--values", - str(values), - "--gpu-resolution", - str(resolution), - ) - - assert result.returncode == 1 - assert expected_error in result.stdout - - def test_validator_rejects_missing_generated_gpu_resolution_artifact(tmp_path: Path) -> None: repo = tmp_path / "checkout" inventory, values, _ = write_resolved_gpu_artifacts(repo) @@ -1409,56 +1121,27 @@ def fail_late_replace(source, destination): assert values_target.read_text(encoding="utf-8") == "old symlink target\n" -def test_artifact_store_rolls_back_destination_when_staged_unlink_fails( +def test_artifact_store_rolls_back_non_force_destination_after_post_link_fsync_failure( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - module = load_deploy_module("deploy_artifact_store_unlink", ARTIFACT_STORE) - destination = tmp_path / "inventory.yml" - original_unlink = module.os.unlink - failed = False - - def fail_first_staged_unlink(path, *args, **kwargs): - nonlocal failed - if not failed and Path(path).name.startswith(".inventory.yml."): - failed = True - raise OSError("injected staged unlink failure") - return original_unlink(path, *args, **kwargs) - - monkeypatch.setattr(module.os, "unlink", fail_first_staged_unlink) - - with pytest.raises(SystemExit): - module.publish_artifacts([(destination, "new inventory\n", 0o600, True)], force=False) - - assert not destination.exists() - - -@pytest.mark.parametrize("force", (False, True)) -def test_artifact_store_rolls_back_destination_when_parent_fsync_fails( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, force: bool -) -> None: - module = load_deploy_module(f"deploy_artifact_store_fsync_{force}", ARTIFACT_STORE) + module = load_deploy_module("deploy_artifact_store_nonforce_fsync", ARTIFACT_STORE) destination = tmp_path / "inventory.yml" - if force: - destination.write_text("old inventory\n", encoding="utf-8") original_fsync_parent = module._fsync_parent calls = 0 - def fail_after_publication(path): + def fail_after_publication(path: Path) -> None: nonlocal calls calls += 1 - if calls == (2 if force else 1): + if calls == 1: raise OSError("injected parent fsync failure") - return original_fsync_parent(path) + original_fsync_parent(path) monkeypatch.setattr(module, "_fsync_parent", fail_after_publication) with pytest.raises(SystemExit): - module.publish_artifacts([(destination, "new inventory\n", 0o600, True)], force=force) + module.publish_artifacts([(destination, "new inventory\n", 0o600, True)], force=False) - if force: - assert destination.read_text(encoding="utf-8") == "old inventory\n" - else: - assert not destination.exists() + assert not destination.exists() def test_generated_overlay_activates_selected_accelerators_for_validation(tmp_path: Path) -> None: @@ -1512,103 +1195,3 @@ def test_checkout_root_helper_path_is_a_runnable_public_cli() -> None: assert result.returncode == 0, result.stdout + result.stderr assert '"topology": "pxe-diskless | ssh-preinstalled"' in result.stdout - - -def test_generator_print_schema_is_byte_stable() -> None: - result = run_script(GEN_CONFIGS, "--print-schema") - - assert result.returncode == 0, result.stdout + result.stderr - assert result.stderr == "" - assert result.stdout == json.dumps(EXPECTED_GENERATOR_SCHEMA, indent=2) + "\n" - - -def test_generator_exits_with_usage_error_when_spec_is_omitted() -> None: - result = run_script(GEN_CONFIGS) - - assert result.returncode == 2 - assert result.stdout == "" - assert result.stderr == "gen_configs: --spec is required (or use --print-schema)\n" - - -def test_generator_replaces_colliding_artifacts_when_force_is_given(tmp_path: Path) -> None: - spec_path = write_file(tmp_path / "spec.json", json.dumps(generator_spec("pxe-diskless"))) - token_path = write_file(tmp_path / "token.txt", "characterization-token\n") - out_dir = tmp_path / "generated" - write_file(out_dir / "inventory.yml", "old inventory\n") - write_file(out_dir / "pb-pxe-controller.vars.yml", "old pxe vars\n") - write_file(out_dir / "values-basic-example.yaml", "old values\n") - - result = run_script( - GEN_CONFIGS, - "--spec", - str(spec_path), - "--out-dir", - str(out_dir), - "--token-file", - str(token_path), - "--force", - ) - - assert result.returncode == 0, result.stdout + result.stderr - assert "old inventory" not in (out_dir / "inventory.yml").read_text(encoding="utf-8") - assert "old pxe vars" not in (out_dir / "pb-pxe-controller.vars.yml").read_text(encoding="utf-8") - assert "old values" not in (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") - assert os.stat(out_dir / "inventory.yml").st_mode & 0o777 == 0o600 - assert os.stat(out_dir / "pb-pxe-controller.vars.yml").st_mode & 0o777 == 0o600 - assert os.stat(out_dir / "values-basic-example.yaml").st_mode & 0o777 == 0o644 - - -def test_generator_exposes_extracted_generation_and_artifact_modules() -> None: - generation = load_deploy_module("deploy_config_generation", CONFIG_GENERATION) - artifacts = load_deploy_module("deploy_artifact_store", ARTIFACT_STORE) - - assert generation.SCHEMA == EXPECTED_GENERATOR_SCHEMA - assert generation.validate_spec(generator_spec()) == "ssh-preinstalled" - assert callable(generation.render_inventory) - assert callable(generation.render_pxe_vars) - assert callable(generation.render_values) - assert callable(artifacts.preflight_destinations) - assert callable(artifacts.publish_artifacts) - - -def test_generator_uses_fake_ansible_discovery_to_publish_resolved_ssh_policy(tmp_path: Path) -> None: - fake_bin = tmp_path / "bin" - fake_bin.mkdir() - fake_ansible = fake_bin / "ansible-playbook" - fake_ansible.write_text( - r"""#!/usr/bin/env python3 -import json -import pathlib -import sys -args = sys.argv[1:] -output = next(arg.split('=', 1)[1] for arg in args if arg.startswith('gpu_access_discovery_output_path=')) -def host(name, bdf): - return { - 'host': name, 'reachable': True, - 'lspci': {'rc': 0, 'stdout': bdf}, 'sysfs': {'rc': 0, 'stdout': bdf}, - } -pathlib.Path(output).write_text(json.dumps({'version': 1, 'hosts': [host('server', '0000:03:00.0'), host('agent', '')]}), encoding='utf-8') -""", - encoding="utf-8", - ) - fake_ansible.chmod(0o755) - spec = generator_spec() - spec["agents"] = [{"name": "agent", "ip": "192.168.1.11"}] - spec_path = write_file(tmp_path / "spec.json", json.dumps(spec)) - out_dir = tmp_path / "generated" - result = subprocess.run( - [sys.executable, str(GEN_CONFIGS), "--spec", str(spec_path), "--out-dir", str(out_dir)], - capture_output=True, - check=False, - env={**os.environ, "PATH": f"{fake_bin}:{os.environ['PATH']}"}, - text=True, - ) - - assert result.returncode == 0, result.stdout + result.stderr - inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") - assert inventory.count("auplc_gpu_access_enabled: true") == 1 - assert inventory.count("auplc_gpu_access_enabled: false") == 1 - assert "auplc_render_gid" not in inventory - assert "gpuAccess" not in (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") - manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) - assert manifest["hosts"] == {"agent": False, "server": True} diff --git a/tests/skills/test_direct_inventory_validation.py b/tests/skills/test_direct_inventory_validation.py index 370f3e37..805ed26c 100644 --- a/tests/skills/test_direct_inventory_validation.py +++ b/tests/skills/test_direct_inventory_validation.py @@ -38,19 +38,6 @@ def valid_inventory() -> str: """ -def test_validator_accepts_direct_inventory_without_resolution_manifest(tmp_path: Path) -> None: - repo = tmp_path / "checkout" - inventory = write(repo / "inventory.yml", valid_inventory()) - values = write(repo / "values.yaml", "custom:\n resources:\n metadata: {}\n") - - result = run_validate( - "--repo", str(repo), "--topology", "ssh-preinstalled", "--inventory", str(inventory), "--values", str(values) - ) - - assert result.returncode == 0, result.stdout + result.stderr - assert "GPU access inventory is valid" in result.stdout - - def test_validator_requires_gpu_resolution_for_pxe_inventory_only(tmp_path: Path) -> None: repo = tmp_path / "checkout" inventory = write( @@ -90,7 +77,7 @@ def test_validator_requires_gpu_resolution_for_pxe_inventory_only(tmp_path: Path @pytest.mark.parametrize("value", ("auto", "true", "false")) -def test_validator_accepts_supported_direct_inventory_values(tmp_path: Path, value: str) -> None: +def test_validator_accepts_direct_inventory_values_without_resolution_manifest(tmp_path: Path, value: str) -> None: repo = tmp_path / "checkout" inventory = write(repo / "inventory.yml", valid_inventory().replace("true", value).replace("false", value)) values = write(repo / "values.yaml", "custom:\n resources:\n metadata: {}\n") @@ -141,21 +128,9 @@ def test_validator_rejects_auto_when_inventory_is_cross_checked_with_gpu_resolut ("inventory_content", "expected_error"), [ (valid_inventory().replace(" auplc_gpu_access_enabled: true\n", ""), "must define exactly one"), - (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: yes"), "malformed"), (valid_inventory().replace("auplc_gpu_access_enabled: true", 'auplc_gpu_access_enabled: "auto"'), "malformed"), - (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: 'auto'"), "malformed"), - (valid_inventory().replace("auplc_gpu_access_enabled: true", 'auplc_gpu_access_enabled: "true"'), "malformed"), - (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: 'true'"), "malformed"), - (valid_inventory().replace("auplc_gpu_access_enabled: true", 'auplc_gpu_access_enabled: "false"'), "malformed"), - (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: 'false'"), "malformed"), + (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: yes"), "malformed"), (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: AUTO"), "malformed"), - (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: TRUE"), "malformed"), - (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: FALSE"), "malformed"), - (valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: no"), "malformed"), - ( - valid_inventory().replace("auplc_gpu_access_enabled: true", "auplc_gpu_access_enabled: malformed"), - "malformed", - ), ( valid_inventory().replace( " auplc_gpu_access_enabled: true\n", From 21e775aabf5b2a9a8be75b0976361f8aefd6f946 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:59:18 +0800 Subject: [PATCH 63/65] test(deploy): consolidate GPU artifact policy coverage --- tests/skills/test_gpu_access_resolution.py | 98 +++----------------- tests/skills/test_gpu_artifact_generation.py | 67 +------------ tests/skills/test_pxe_finalization.py | 82 ++++------------ 3 files changed, 36 insertions(+), 211 deletions(-) diff --git a/tests/skills/test_gpu_access_resolution.py b/tests/skills/test_gpu_access_resolution.py index fdd802ea..6cd29546 100644 --- a/tests/skills/test_gpu_access_resolution.py +++ b/tests/skills/test_gpu_access_resolution.py @@ -6,7 +6,6 @@ import importlib.util import json -import re import sys from pathlib import Path @@ -66,31 +65,12 @@ def expected_targets(module, *names: str): return tuple(module.InventoryTarget(name=name) for name in names) -def test_discovery_playbook_preserves_lspci_agreement_and_exact_v1_host_evidence_shape() -> None: +def test_discovery_playbook_records_lspci_and_sysfs_evidence() -> None: playbook = DISCOVERY_PLAYBOOK.read_text(encoding="utf-8") - evidence_block = playbook.split("_auplc_gpu_access_discovery_evidence:", maxsplit=1)[1].split( - " changed_when:", maxsplit=1 - )[0] - fallback_block = playbook.split("_auplc_gpu_access_unknown_evidence:", maxsplit=1)[1].split( - " pre_tasks:", maxsplit=1 - )[0] - evidence_keys = re.findall(r"^ ([a-z_]+):", evidence_block, re.MULTILINE) - fallback_keys = re.findall(r"^ ([a-z_]+):", fallback_block, re.MULTILINE) - - assert evidence_keys == ["host", "reachable", "lspci", "sysfs"] - assert fallback_keys == ["reachable", "lspci", "sysfs"] - assert "combine({'host': discovery_host})" in playbook - assert '{"version":1,"hosts":[' in playbook - assert "hostvars[discovery_host]._auplc_gpu_access_discovery_evidence" in playbook - assert "| to_json" in playbook - assert "name: gpu_access" in playbook - assert "tasks_from: detect" in playbook + assert "_auplc_discovery_lspci.rc" in playbook + assert "_auplc_discovery_lspci.stdout" in playbook assert "_auplc_gpu_access_sysfs.rc" in playbook assert "_auplc_gpu_access_sysfs.stdout" in playbook - assert "/sys/bus/pci/devices" not in playbook - assert 'argv: [lspci, -Dnn, -d, "1002::0300"]' in playbook - assert 'argv: [lspci, -Dnn, -d, "1002::0302"]' in playbook - assert 'argv: [lspci, -Dnn, -d, "1002::0380"]' in playbook def test_parse_fleet_evidence_accepts_the_exact_machine_evidence_schema() -> None: @@ -103,27 +83,11 @@ def test_parse_fleet_evidence_accepts_the_exact_machine_evidence_schema() -> Non assert evidence[0].sysfs.stdout == GPU_BDF -@pytest.mark.parametrize( - "replacement", - [ - {"version": True, "hosts": []}, - {"version": 1, "hosts": [], "unexpected": "field"}, - {"version": 1, "hosts": [{"host": "gpu-1"}]}, - {"version": 1, "hosts": [host_evidence("gpu-1", lspci_rc=True)]}, - ], -) -def test_parse_fleet_evidence_rejects_nonexact_or_boolean_integer_values(replacement: dict) -> None: +def test_parse_fleet_evidence_rejects_boolean_integer_values() -> None: module = load_resolution_module() with pytest.raises(module.EvidenceParseError): - module.parse_fleet_evidence(json.dumps(replacement)) - - -def test_parse_fleet_evidence_rejects_duplicate_json_keys() -> None: - module = load_resolution_module() - - with pytest.raises(module.EvidenceParseError, match="duplicate JSON key 'version'"): - module.parse_fleet_evidence('{"version":1,"version":1,"hosts":[]}') + module.parse_fleet_evidence(json.dumps({"version": True, "hosts": []})) def test_resolve_fleet_classifies_matching_amd_bdfs_as_gpu() -> None: @@ -146,17 +110,11 @@ def test_resolve_fleet_classifies_two_empty_successful_gpu_probes_as_cpu_only() assert resolution.hosts[0].status is module.HostStatus.CPU -@pytest.mark.parametrize( - "evidence", - [ - host_evidence("host-1", lspci_bdfs=[GPU_BDF], sysfs_bdfs=["0000:04:00.0"]), - host_evidence("host-1", lspci_bdfs=[GPU_BDF], lspci_rc=1), - host_evidence("host-1", lspci_bdfs=[GPU_BDF], reachable=False), - ], -) -def test_resolve_fleet_blocks_unknown_gpu_evidence(evidence: dict) -> None: +def test_resolve_fleet_blocks_disagreeing_lspci_and_sysfs_evidence() -> None: module = load_resolution_module() - parsed = module.parse_fleet_evidence(evidence_document(evidence)) + parsed = module.parse_fleet_evidence( + evidence_document(host_evidence("host-1", lspci_bdfs=[GPU_BDF], sysfs_bdfs=["0000:04:00.0"])) + ) resolution = module.resolve_fleet(expected_targets(module, "host-1"), parsed) @@ -164,22 +122,11 @@ def test_resolve_fleet_blocks_unknown_gpu_evidence(evidence: dict) -> None: assert resolution.hosts[0].status is module.HostStatus.UNKNOWN -@pytest.mark.parametrize( - ("targets", "hosts"), - [ - (("gpu-1", "gpu-2"), ("gpu-1",)), - (("gpu-1",), ("gpu-1", "gpu-2")), - ], -) -def test_resolve_fleet_blocks_incomplete_or_unexpected_host_evidence( - targets: tuple[str, ...], hosts: tuple[str, ...] -) -> None: +def test_resolve_fleet_blocks_incomplete_host_evidence() -> None: module = load_resolution_module() - parsed = module.parse_fleet_evidence( - evidence_document(*(host_evidence(host, lspci_bdfs=[GPU_BDF]) for host in hosts)) - ) + parsed = module.parse_fleet_evidence(evidence_document(host_evidence("gpu-1", lspci_bdfs=[GPU_BDF]))) - resolution = module.resolve_fleet(expected_targets(module, *targets), parsed) + resolution = module.resolve_fleet(expected_targets(module, "gpu-1", "gpu-2"), parsed) assert resolution.status is module.FleetStatus.BLOCKED assert resolution.reason == "incomplete host coverage" @@ -218,18 +165,6 @@ def test_resolution_manifest_preserves_explicit_host_booleans() -> None: } -def test_resolution_manifest_is_an_ordinary_dict_with_exact_order_and_sorted_hosts() -> None: - manifest = load_manifest_module().build_resolution_manifest( - status="gpu_resolved", - hosts={"zeta": True, "alpha": False}, - ) - - assert type(manifest) is dict - assert list(manifest) == ["version", "status", "hosts"] - assert list(manifest["hosts"]) == ["alpha", "zeta"] - assert set(manifest) == {"version", "status", "hosts"} - - def test_pxe_resolution_manifest_constructs_without_mutating_base_manifest() -> None: module = load_manifest_module() base = module.build_resolution_manifest( @@ -242,11 +177,6 @@ def test_pxe_resolution_manifest_constructs_without_mutating_base_manifest() -> gpu_access_enabled=True, ) - assert base == { - "version": 1, - "status": "gpu_resolved", - "hosts": {"gpu-1": True, "gpu-2": True}, - } - assert list(manifest) == ["version", "status", "hosts", "pxe_rootfs"] + assert base["hosts"] == {"gpu-1": True, "gpu-2": True} + assert "pxe_rootfs" not in base assert manifest["pxe_rootfs"] == {"gpu_access_enabled": True} - assert set(manifest["pxe_rootfs"]) == {"gpu_access_enabled"} diff --git a/tests/skills/test_gpu_artifact_generation.py b/tests/skills/test_gpu_artifact_generation.py index fe629147..25bec235 100644 --- a/tests/skills/test_gpu_artifact_generation.py +++ b/tests/skills/test_gpu_artifact_generation.py @@ -35,7 +35,7 @@ def ssh_spec() -> dict: } -def write_fake_ansible(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, document: dict) -> Path: +def write_fake_ansible(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, document: dict) -> None: fake_bin = tmp_path / "bin" fake_bin.mkdir() fake_ansible = fake_bin / "ansible-playbook" @@ -60,11 +60,9 @@ def write_fake_ansible(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, document encoding="utf-8", ) fake_ansible.chmod(0o755) - record = tmp_path / "ansible-argv.json" - monkeypatch.setenv("FAKE_ANSIBLE_RECORD", str(record)) + monkeypatch.setenv("FAKE_ANSIBLE_RECORD", str(tmp_path / "ansible-argv.json")) monkeypatch.setenv("FAKE_ANSIBLE_EVIDENCE", json.dumps(document)) monkeypatch.setenv("PATH", f"{fake_bin}:{os.environ['PATH']}") - return record def run_generator(spec_path: Path, out_dir: Path, *extra: str) -> subprocess.CompletedProcess[str]: @@ -141,7 +139,7 @@ def test_generator_surfaces_redacted_bounded_ansible_failure_diagnostics( def test_generator_discovers_mixed_ssh_targets_and_publishes_resolved_artifacts( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - record = write_fake_ansible( + write_fake_ansible( tmp_path, monkeypatch, {"version": 1, "hosts": [evidence_host("server", gpu=True), evidence_host("agent")]}, @@ -154,71 +152,12 @@ def test_generator_discovers_mixed_ssh_targets_and_publishes_resolved_artifacts( inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") assert inventory.count("auplc_gpu_access_enabled: true") == 1 assert inventory.count("auplc_gpu_access_enabled: false") == 1 - assert "auplc_gpu_access_enabled: auto" not in inventory assert "auplc_render_gid" not in inventory - assert "gpuAccess" not in (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") assert json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) == { "version": 1, "status": "gpu_resolved", "hosts": {"agent": False, "server": True}, } - discovery_inventory = out_dir / ".gpu-access-discovery.inventory.yml" - discovery_evidence = out_dir / ".gpu-access-discovery-evidence.json" - assert discovery_inventory.stat().st_mode & 0o777 == 0o600 - assert discovery_evidence.stat().st_mode & 0o777 == 0o600 - assert json.loads(record.read_text(encoding="utf-8")) == [ - "-i", - str(discovery_inventory), - str(ROOT / "deploy" / "ansible" / "playbooks" / "pb-gpu-access-discovery.yml"), - "-e", - f"gpu_access_discovery_output_path={discovery_evidence}", - ] - - -def test_generator_allows_heterogeneous_gpu_hosts_and_publishes_boolean_only_artifacts( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - write_fake_ansible( - tmp_path, - monkeypatch, - {"version": 1, "hosts": [evidence_host("server", gpu=True), evidence_host("agent", gpu=True)]}, - ) - out_dir = tmp_path / "generated" - - result = run_generator(write_json(tmp_path / "spec.json", ssh_spec()), out_dir) - - assert result.returncode == 0, result.stderr - inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") - values = (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") - manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) - assert inventory.count("auplc_gpu_access_enabled: true") == 2 - assert "auplc_gpu_access_enabled: auto" not in inventory - assert "auplc_render_gid" not in inventory - assert "gpuAccess" not in values - assert manifest == {"version": 1, "status": "gpu_resolved", "hosts": {"agent": True, "server": True}} - - -def test_generator_publishes_boolean_only_artifacts_for_all_cpu_ssh_targets( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - write_fake_ansible( - tmp_path, monkeypatch, {"version": 1, "hosts": [evidence_host("server"), evidence_host("agent")]} - ) - out_dir = tmp_path / "generated" - - result = run_generator(write_json(tmp_path / "spec.json", ssh_spec()), out_dir) - - assert result.returncode == 0, result.stderr - inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") - assert inventory.count("auplc_gpu_access_enabled: false") == 2 - assert "auplc_gpu_access_enabled: auto" not in inventory - assert "auplc_render_gid" not in inventory - assert "gpuAccess" not in (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") - assert json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) == { - "version": 1, - "status": "cpu_only", - "hosts": {"agent": False, "server": False}, - } @pytest.mark.parametrize("failure", ["missing", "nonzero"]) diff --git a/tests/skills/test_pxe_finalization.py b/tests/skills/test_pxe_finalization.py index dac1fb0a..06313b0a 100644 --- a/tests/skills/test_pxe_finalization.py +++ b/tests/skills/test_pxe_finalization.py @@ -74,60 +74,28 @@ def run_generator(*arguments: str) -> subprocess.CompletedProcess[str]: ) -def canonical_artifacts(out_dir: Path) -> tuple[Path, ...]: - return ( - out_dir / "inventory.yml", - out_dir / "pb-pxe-controller.vars.yml", - out_dir / "values-basic-example.yaml", - out_dir / "gpu-access-resolution.json", - ) - - -def test_pxe_gpu_agents_publish_immediate_boolean_only_rootfs_artifacts( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch +@pytest.mark.parametrize("policy", [(True, "true"), (False, "false")]) +def test_pxe_agents_publish_explicit_boolean_rootfs_policy( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, policy: tuple[bool, str] ) -> None: + gpu_agents, expected_policy = policy write_fake_ansible(tmp_path, monkeypatch) out_dir = tmp_path / "generated" - result = run_generator("--spec", str(write_json(tmp_path / "spec.json", pxe_spec(True))), "--out-dir", str(out_dir)) + result = run_generator( + "--spec", str(write_json(tmp_path / "spec.json", pxe_spec(gpu_agents))), "--out-dir", str(out_dir) + ) assert result.returncode == 0, result.stderr inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") - values = (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) pxe_vars = (out_dir / "pb-pxe-controller.vars.yml").read_text(encoding="utf-8") - assert "auplc_render_gid" not in inventory - assert "auplc_gpu_access_enabled: auto" not in inventory - assert "gpuAccess" not in values - assert "pxe_gpu_access_enabled: true" in pxe_vars - assert "pxe_gpu_access_enabled: auto" not in pxe_vars - assert manifest == { - "version": 1, - "status": "cpu_only", - "hosts": {"controller": False}, - "pxe_rootfs": {"gpu_access_enabled": True}, - } - assert not list(out_dir.glob(".pxe-finalizer-*")) + assert "auplc_render_gid" not in inventory + pxe_vars + assert f"pxe_gpu_access_enabled: {expected_policy}" in pxe_vars + assert manifest["pxe_rootfs"] == {"gpu_access_enabled": gpu_agents} assert "do-not-print-this-secret" not in result.stdout + result.stderr -def test_pxe_cpu_agents_publish_a_disabled_rootfs_policy(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - write_fake_ansible(tmp_path, monkeypatch) - out_dir = tmp_path / "generated" - - result = run_generator( - "--spec", str(write_json(tmp_path / "spec.json", pxe_spec(False))), "--out-dir", str(out_dir) - ) - - assert result.returncode == 0, result.stderr - pxe_vars = (out_dir / "pb-pxe-controller.vars.yml").read_text(encoding="utf-8") - manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) - assert "pxe_gpu_access_enabled: false" in pxe_vars - assert "pxe_gpu_access_enabled: auto" not in pxe_vars - assert "auplc_render_gid" not in pxe_vars - assert manifest["pxe_rootfs"] == {"gpu_access_enabled": False} - - def test_pxe_gpu_controller_and_rootfs_publish_independent_booleans( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -140,30 +108,10 @@ def test_pxe_gpu_controller_and_rootfs_publish_independent_booleans( inventory = (out_dir / "inventory.yml").read_text(encoding="utf-8") manifest = json.loads((out_dir / "gpu-access-resolution.json").read_text(encoding="utf-8")) assert "auplc_gpu_access_enabled: true" in inventory - assert "auplc_gpu_access_enabled: auto" not in inventory - pxe_vars = (out_dir / "pb-pxe-controller.vars.yml").read_text(encoding="utf-8") - assert "pxe_gpu_access_enabled: auto" not in pxe_vars assert manifest["status"] == "gpu_resolved" assert manifest["pxe_rootfs"] == {"gpu_access_enabled": True} -def test_pxe_generator_refuses_existing_canonical_artifacts_without_force( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - write_fake_ansible(tmp_path, monkeypatch) - out_dir = tmp_path / "generated" - out_dir.mkdir() - existing = out_dir / "values-basic-example.yaml" - existing.write_text("existing\n", encoding="utf-8") - - result = run_generator("--spec", str(write_json(tmp_path / "spec.json", pxe_spec(True))), "--out-dir", str(out_dir)) - - assert result.returncode == 1 - assert "refusing to overwrite existing" in result.stderr - assert existing.read_text(encoding="utf-8") == "existing\n" - assert all(not path.exists() for path in canonical_artifacts(out_dir) if path != existing) - - def test_pxe_generator_does_not_publish_when_controller_discovery_is_unknown( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -188,4 +136,12 @@ def test_pxe_generator_does_not_publish_when_controller_discovery_is_unknown( result = run_generator("--spec", str(write_json(tmp_path / "spec.json", pxe_spec(True))), "--out-dir", str(out_dir)) assert result.returncode == 1 - assert all(not path.exists() for path in canonical_artifacts(out_dir)) + assert not any( + (out_dir / name).exists() + for name in ( + "inventory.yml", + "pb-pxe-controller.vars.yml", + "values-basic-example.yaml", + "gpu-access-resolution.json", + ) + ) From 7259d4e71b2d68cfc14a739ffa47a03838f3a986 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 29 Jul 2026 18:00:31 +0800 Subject: [PATCH 64/65] test(ansible): narrow GPU role contract coverage --- tests/skills/test_gpu_access_role.py | 345 +++++++++------------------ 1 file changed, 108 insertions(+), 237 deletions(-) diff --git a/tests/skills/test_gpu_access_role.py b/tests/skills/test_gpu_access_role.py index f67ec67d..f6efb872 100644 --- a/tests/skills/test_gpu_access_role.py +++ b/tests/skills/test_gpu_access_role.py @@ -1,5 +1,4 @@ # Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. - """Contract tests for AMD's packaged GPU udev rules in Ansible.""" from pathlib import Path @@ -14,8 +13,7 @@ PACKAGE = "amdgpu-insecure-instinct-udev-rules" VERSION = "30.30.4.0-2341068.24.04" -FILENAME = f"{PACKAGE}_{VERSION}_all.deb" -URL = f"https://repo.radeon.com/amdgpu/30.30.4/ubuntu/pool/main/a/{PACKAGE}/{FILENAME}" +URL = f"https://repo.radeon.com/amdgpu/30.30.4/ubuntu/pool/main/a/{PACKAGE}/{PACKAGE}_{VERSION}_all.deb" SHA256 = "4be865985c7a13114c45925e77bc0b411b9fd47d5040ed35df44b9c411766162" RULE_PATH = "/etc/udev/rules.d/70-amdgpu.rules" RULE_CONTENT = ( @@ -27,277 +25,150 @@ def read(path: Path) -> str: return path.read_text(encoding="utf-8") -def test_gpu_access_role_pins_the_official_amd_package_contract() -> None: - defaults = read(GPU_ACCESS_ROLE / "defaults" / "main.yml") - preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") +def test_gpu_access_role_enforces_pinned_package_contract() -> None: + defaults = yaml.safe_load(read(GPU_ACCESS_ROLE / "defaults" / "main.yml")) apply = read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") verify = read(GPU_ACCESS_ROLE / "tasks" / "verify.yml") - assert PACKAGE in defaults - assert VERSION in defaults - assert FILENAME in defaults - assert URL in defaults - assert f"sha256:{SHA256}" in defaults - assert RULE_PATH in defaults - assert " " + RULE_CONTENT.replace("\n", "\n ").rstrip() in defaults - assert "dpkg-query" in preflight - assert r"--showformat=${Status}\t${Version}" in preflight - assert "ansible.builtin.get_url" in apply - assert "ansible.builtin.apt" in apply - assert 'checksum: "{{ auplc_gpu_udev_package_checksum }}"' in apply - assert 'deb: "{{ auplc_gpu_udev_package_cache_path }}"' in apply - assert "dpkg-query" in verify - assert r"--showformat=${Status}\t${Version}" in verify - assert "--search" in verify - assert "package-owned" in verify - assert "modified package conffile" in verify + assert [ + defaults[key] + for key in ( + "auplc_gpu_udev_package_name", + "auplc_gpu_udev_package_version", + "auplc_gpu_udev_package_url", + "auplc_gpu_udev_package_checksum", + "auplc_gpu_udev_rule_path", + "auplc_gpu_udev_rule_content", + ) + ] == [PACKAGE, VERSION, URL, f"sha256:{SHA256}", RULE_PATH, RULE_CONTENT] + assert all(token in apply for token in ("ansible.builtin.get_url", "ansible.builtin.apt", "checksum:", "deb:")) + assert all( + token in verify + for token in ( + "dpkg-query", + r"--showformat=${Status}\t${Version}", + "--search", + "install ok installed", + "_auplc_verify_live_rule_owner.stdout == auplc_gpu_udev_package_name + ': ' + auplc_gpu_udev_rule_path", + "(_auplc_verify_rule_content.content | b64decode) == auplc_gpu_udev_rule_content", + ) + ) -def test_gpu_access_defaults_and_inventory_placeholders_use_unquoted_auto() -> None: - defaults_text = read(GPU_ACCESS_ROLE / "defaults" / "main.yml") - defaults = yaml.safe_load(defaults_text) - inventory_text = read(ANSIBLE / "inventory.yml") - inventory = yaml.safe_load(inventory_text) - raw_inventory = yaml.load(inventory_text, Loader=yaml.BaseLoader) - hosts = inventory["k3s_cluster"]["children"] - raw_hosts = raw_inventory["k3s_cluster"]["children"] +def test_gpu_access_defaults_and_inventory_leave_auto_unquoted() -> None: + defaults = read(GPU_ACCESS_ROLE / "defaults" / "main.yml") + inventory = read(ANSIBLE / "inventory.yml") - assert defaults["auplc_gpu_access_enabled"] == "auto" - assert "auplc_gpu_access_enabled: auto" in defaults_text - assert inventory_text.count("auplc_gpu_access_enabled: auto") == 2 - assert all( - quoted not in inventory_text - for quoted in ('auplc_gpu_access_enabled: "auto"', "auplc_gpu_access_enabled: 'auto'") - ) - for group_name in ("server", "agent"): - for host_name, host in hosts[group_name]["hosts"].items(): - value = host["auplc_gpu_access_enabled"] - assert value == raw_hosts[group_name]["hosts"][host_name]["auplc_gpu_access_enabled"] == "auto" + assert "auplc_gpu_access_enabled: auto" in defaults + assert inventory.count("auplc_gpu_access_enabled: auto") == 2 + assert 'auplc_gpu_access_enabled: "auto"' not in inventory + assert "auplc_gpu_access_enabled: 'auto'" not in inventory -def test_gpu_access_role_preserves_rootfs_and_exact_legacy_safety() -> None: +def test_gpu_access_rootfs_and_legacy_cleanup_remain_contained_and_verified() -> None: validation = read(GPU_ACCESS_ROLE / "tasks" / "validate.yml") preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") apply = read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") - assert "realpath" in validation - assert "auplc_rootfs_path != '/'" in validation - assert "_auplc_canonical_allowed_root" in validation - assert "Inspect GPU access rootfs target" in preflight - assert "follow: false" in preflight - assert "Reject unsafe AMD udev rule destination parents" in preflight - assert "Reject unsafe AMD udev rule destination" in preflight - assert "Define recognized project-owned legacy GPU rules" in preflight - assert "hash('sha256')" in preflight - assert "70-kfd.rules" in preflight - assert "70-rocm-devices.rules" in preflight - assert "70-auplc-gpu-access.rules" not in preflight - assert "Reject unexpected legacy GPU rule content" in preflight - assert "Recheck recognized project-owned legacy GPU rules before apply" in apply - assert "Remove recognized project-owned legacy GPU rules" in apply - assert apply.index("Download checksummed AMD udev package") < apply.index( - "Remove recognized project-owned legacy GPU rules" + assert all( + token in validation + for token in ( + "realpath", + "auplc_rootfs_path != '/'", + "_auplc_canonical_rootfs.stdout.startswith(_auplc_canonical_allowed_root.stdout + '/')", + ) ) - assert apply.index("Verify installed AMD udev package") < apply.index( - "Remove recognized project-owned legacy GPU rules" + assert all( + token in preflight + for token in ( + "follow: false", + "_auplc_legacy_gpu_rules", + "hash('sha256')", + "70-kfd.rules", + "70-rocm-devices.rules", + ) ) - assert "Reload live udev rules after legacy cleanup" in apply - assert "Trigger live udev rules after legacy cleanup" in apply - - -def test_gpu_access_role_skips_package_cache_and_download_when_exact_version_is_installed() -> None: - preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") - apply = read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") - - assert "_auplc_gpu_udev_install_needed" in preflight - assert "Install AMD udev package when required" in apply - install_block = apply.split("Install AMD udev package when required", maxsplit=1)[1] - assert "Create deterministic AMD udev package cache" in install_block - assert "Download checksummed AMD udev package" in install_block - assert "when: _auplc_gpu_udev_install_needed | bool" in install_block - assert "Verify installed AMD udev package without installation" in apply - assert "install ok installed" in preflight - - -def test_gpu_access_role_requires_installed_status_and_exact_version() -> None: - verify = read(GPU_ACCESS_ROLE / "tasks" / "verify.yml") - - assert "install ok installed" in verify - assert "Require installed AMD udev package status and exact version" in verify - - -def test_preflight_allows_package_owned_wrong_version_rule_for_convergence() -> None: - preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") - - assert "Query AMD udev rule package ownership on live host before admission" in preflight - assert "Query AMD udev rule package ownership in PXE rootfs before admission" in preflight - assert preflight.index("Query installed AMD udev package") < preflight.index("Read existing AMD udev rule") - assert preflight.index("Query AMD udev rule package ownership") < preflight.index("Read existing AMD udev rule") - assert "_auplc_gpu_udev_install_needed | bool" in preflight - assert "_auplc_rule_owned_by_amd_package | bool" in preflight - - -def test_preflight_allows_package_owned_partial_state_rule_for_convergence() -> None: - preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") - - assert "Record whether AMD udev package installation is needed" in preflight - assert "Allow package-owned AMD udev rule convergence" in preflight - assert "install ok installed" in preflight - - -def test_preflight_rejects_unknown_unowned_rule_content() -> None: - preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") + assert apply.index("ansible.builtin.import_tasks: verify.yml") < apply.rindex("state: absent") + assert apply.index("item.content | b64decode") < apply.rindex("state: absent") - assert "Reject modified AMD udev rule before package installation" in preflight - assert "_auplc_rule_owned_by_amd_package | bool" in preflight - assert "Existing AMD udev rule is neither the package rule nor a recognized legacy rule." in preflight - -def test_preflight_legacy_admission_matches_package_owned_convergence_admission() -> None: - preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") - primary_admission = preflight.split("Allow package-owned AMD udev rule convergence", maxsplit=1)[1].split( - "Reject modified AMD udev rule before package installation", maxsplit=1 - )[0] - legacy_admission = preflight.split("Reject unexpected legacy GPU rule content", maxsplit=1)[1].split( - "fail_msg:", maxsplit=1 - )[0] - - assert "_auplc_gpu_udev_install_needed | bool" in primary_admission - assert "_auplc_rule_owned_by_amd_package | bool" in primary_admission - assert "_auplc_gpu_udev_install_needed | bool" in legacy_admission - assert "_auplc_rule_owned_by_amd_package | bool" in legacy_admission - assert "auplc_gpu_udev_rule_path" in legacy_admission - assert "auplc_gpu_udev_rule_content" in legacy_admission - - -def test_gpu_access_role_installs_the_package_without_custom_rule_or_device_probes() -> None: - apply = read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") - - assert not (GPU_ACCESS_ROLE / "templates" / "70-auplc-gpu-access.rules.j2").exists() - assert "ansible.builtin.template" not in apply - assert "70-auplc-gpu-access.rules.j2" not in apply - assert "udevadm settle" not in apply - assert "/dev/kfd" not in apply - assert "/dev/dri" not in apply - assert "/sys/class/drm" not in apply - assert "card" not in apply - - -def test_gpu_access_role_verifies_exact_installed_package_version_and_rule() -> None: - verify = read(GPU_ACCESS_ROLE / "tasks" / "verify.yml") - - assert "auplc_gpu_udev_package_version" in verify - assert "auplc_gpu_udev_rule_path" in verify - assert "auplc_gpu_udev_rule_content" in verify - assert "Require installed AMD udev package status and exact version" in verify - assert "Require package-owned AMD udev rule" in verify - assert "Require exact AMD udev rule content" in verify - assert "follow: false" in verify - - -def test_pxe_gpu_access_uses_safe_chroot_install_and_strict_retained_admission() -> None: +def test_pxe_gpu_access_chroots_without_bind_mounts_and_rejects_unsafe_retained_rules() -> None: main = read(PXE_CONTROLLER_ROLE / "tasks" / "main.yml") tasks = read(PXE_GPU_ACCESS_TASKS) apply = read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") verify = read(GPU_ACCESS_ROLE / "tasks" / "verify.yml") - assert "Admit retained PXE GPU rootfs read-only before lifecycle changes" in main - assert "Re-preflight PXE GPU rootfs before TFTP" in main - assert main.index("Admit retained PXE GPU rootfs read-only before lifecycle changes") < main.index( - "Stop NFS before rootfs rebuild" + assert main.index("pxe_gpu_admission_phase: retained-read-only") < main.index("rm -rf {{ pxe_nfs_root }}") + assert main.index("pxe_gpu_admission_phase: final") < main.index("ls {{ pxe_nfs_root }}/boot/vmlinuz-") + assert all( + token in tasks + for token in ( + "tasks_from: verify", + "tasks_from: preflight", + "tasks_from: apply", + 'auplc_rootfs_path: "{{ pxe_nfs_root }}"', + 'auplc_rootfs_allowed_root: "{{ pxe_nfs_allowed_root }}"', + "auplc_reject_legacy_gpu_rules: true", + ) ) - assert "tasks_from: verify" in tasks - assert "tasks_from: preflight" in tasks - assert "tasks_from: apply" in tasks - assert 'auplc_rootfs_path: "{{ pxe_nfs_root }}"' in tasks - assert 'auplc_rootfs_allowed_root: "{{ pxe_nfs_allowed_root }}"' in tasks - assert "auplc_reject_legacy_gpu_rules: true" in tasks - assert "Reject retained PXE shipped legacy GPU rules" in verify + assert "not item.stat.exists" in verify assert "chroot" in apply assert "apt-get" in apply - assert "Copy AMD udev package into PXE rootfs" in apply - assert "Mount virtual filesystems for AMD udev package installation" not in apply assert "mount --bind" not in apply - assert "Unmount virtual filesystems after AMD udev package installation" not in apply - assert apply.index("Verify installed AMD udev package") < apply.index( - "Remove temporary AMD udev package from PXE rootfs" - ) - assert main.index("Re-preflight PXE GPU rootfs before TFTP") < main.index("Find latest kernel in rootfs") - assert RULE_CONTENT not in tasks - assert "/dev/kfd" not in tasks - assert "/dev/dri" not in tasks -def test_pxe_rootfs_unmounts_fail_on_real_errors_but_skip_absent_mounts() -> None: +def test_pxe_unmounts_only_when_present_and_propagates_failures() -> None: main = read(PXE_CONTROLLER_ROLE / "tasks" / "main.yml") - rootfs_removal = main.split("Remove existing rootfs (force rebuild)", maxsplit=1)[1].split( - "Check if NFS rootfs already exists", maxsplit=1 - )[0] - chroot_unmount = main.split("Unmount virtual filesystems from chroot", maxsplit=1)[1].split( - "Remove chroot setup script", maxsplit=1 - )[0] - for task in (rootfs_removal, chroot_unmount): - assert "set -e" in task - assert "if mountpoint -q" in task - assert "&& umount" not in task - assert "|| true" not in task + assert main.count("set -e") == 2 + for mount in ("dev", "sys", "proc"): + assert main.count("if mountpoint -q {{ pxe_nfs_root }}/" + mount + "; then") == 2 + assert main.count("umount {{ pxe_nfs_root }}/" + mount) == 2 + assert "&& umount" not in main + assert "|| true" not in main -def test_gpu_access_playbooks_keep_two_phase_live_and_rootfs_safety() -> None: +def test_gpu_access_resolves_before_preflight_rocm_and_apply_fail_fatally() -> None: role_main = read(GPU_ACCESS_ROLE / "tasks" / "main.yml") - resolve = read(GPU_ACCESS_ROLE / "tasks" / "resolve.yml") - detect = read(GPU_ACCESS_ROLE / "tasks" / "detect.yml") - rocm_playbook = read(ANSIBLE / "playbooks" / "pb-rocm.yml") - udev_playbook = read(ANSIBLE / "playbooks" / "pb-udev.yml") - pxe_playbook = read(ANSIBLE / "playbooks" / "pb-pxe-controller.yml") + rocm = read(ANSIBLE / "playbooks" / "pb-rocm.yml") + udev = read(ANSIBLE / "playbooks" / "pb-udev.yml") - assert "ansible.builtin.import_tasks: resolve.yml" in role_main - assert "auplc_gpu_access_enabled | bool" not in role_main - assert "when: _auplc_gpu_access_enabled_resolved" in role_main - assert "python3" in detect - assert "/sys/bus/pci/devices" in detect - assert "0x1002" in detect - assert "startswith('0x03')" in detect - assert "sorted(" in detect - assert "register: _auplc_gpu_access_sysfs" in detect - assert "changed_when: false" in detect - assert "failed_when: false" in detect - assert "auplc_gpu_access_enabled is boolean" in resolve - assert "auplc_gpu_access_enabled == 'auto'" in resolve - assert "ansible.builtin.import_tasks: detect.yml" in resolve - assert "_auplc_gpu_access_sysfs.rc == 0" in resolve - assert "_auplc_gpu_access_sysfs.stdout | trim | length > 0" in resolve - assert "_auplc_gpu_access_enabled_resolved is boolean" in resolve - assert resolve.index("ansible.builtin.import_tasks: detect.yml") < resolve.index("_auplc_gpu_access_sysfs.rc == 0") - assert "any_errors_fatal: true" in rocm_playbook - assert "any_errors_fatal: true" in udev_playbook - for playbook in (rocm_playbook, udev_playbook): - assert "tasks_from: resolve" in playbook + assert role_main.index("import_tasks: resolve.yml") < role_main.index("import_tasks: preflight.yml") + assert role_main.index("import_tasks: preflight.yml") < role_main.index("import_tasks: apply.yml") + for playbook in (rocm, udev): + assert "any_errors_fatal: true" in playbook assert playbook.index("tasks_from: resolve") < playbook.index("tasks_from: preflight") - assert "auplc_gpu_access_enabled | bool" not in playbook + assert playbook.index("tasks_from: preflight") < playbook.index("tasks_from: apply") assert "when: _auplc_gpu_access_enabled_resolved" in playbook - assert rocm_playbook.index("tasks_from: preflight") < rocm_playbook.index("- role: rocm") - assert "tasks_from: apply" in rocm_playbook - assert "tasks_from: preflight" in udev_playbook - assert "tasks_from: apply" in udev_playbook - assert "render_gid" not in pxe_playbook + assert rocm.index("tasks_from: preflight") < rocm.index("- role: rocm") < rocm.index("tasks_from: apply") -def test_deploy_ansible_has_no_obsolete_gpu_access_policy_or_state_contract() -> None: - forbidden = ( - "auplc_render_gid", - "auplc_normalize_render_gid", - "gpu-access.json", - "auplc_from_json_strict", - "groupmod", - "render GID collision", +def test_gpu_access_auto_detection_requires_successful_boolean_resolution_before_preflight() -> None: + role_main = read(GPU_ACCESS_ROLE / "tasks" / "main.yml") + resolve = read(GPU_ACCESS_ROLE / "tasks" / "resolve.yml") + + assert "auplc_gpu_access_enabled == 'auto'" in resolve + assert resolve.index("ansible.builtin.import_tasks: detect.yml") < resolve.index("_auplc_gpu_access_sysfs.rc == 0") + assert resolve.index("_auplc_gpu_access_sysfs.rc == 0") < resolve.index("_auplc_gpu_access_enabled_resolved: >-") + assert resolve.index("_auplc_gpu_access_enabled_resolved: >-") < resolve.index( + "_auplc_gpu_access_enabled_resolved is boolean" ) - ansible_text = "\n".join( - path.read_text(encoding="utf-8") - for path in ANSIBLE.rglob("*") - if path.is_file() and "__pycache__" not in path.parts + assert role_main.index("ansible.builtin.import_tasks: resolve.yml") < role_main.index( + "ansible.builtin.import_tasks: preflight.yml" ) - for term in forbidden: - assert term not in ansible_text + +def test_gpu_access_rejects_unknown_unowned_udev_content_before_deletion() -> None: + role_main = read(GPU_ACCESS_ROLE / "tasks" / "main.yml") + preflight = read(GPU_ACCESS_ROLE / "tasks" / "preflight.yml") + apply = read(GPU_ACCESS_ROLE / "tasks" / "apply.yml") + admission = preflight.split("_auplc_rule_content_admitted: >-", maxsplit=1)[1] + cleanup = apply.split("register: _auplc_apply_legacy_gpu_rule_contents", maxsplit=1)[1] + + assert "(_auplc_rule_owned_by_amd_package | bool)" in admission + assert "that: _auplc_rule_content_admitted | bool" in admission + assert role_main.index("ansible.builtin.import_tasks: preflight.yml") < role_main.index( + "ansible.builtin.import_tasks: apply.yml" + ) + assert cleanup.index("item.content | b64decode") < cleanup.index("state: absent") + assert "in item.item.item.sha256" in cleanup From 8f1bf458b34453fc14d89862dca23b54d99583c2 Mon Sep 17 00:00:00 2001 From: ShifZhan <252984256+MioYuuIH@users.noreply.github.com> Date: Wed, 29 Jul 2026 18:01:45 +0800 Subject: [PATCH 65/65] test(image): focus GPU permission ownership contract --- tests/scripts/test_gpu_image_permissions.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/scripts/test_gpu_image_permissions.py b/tests/scripts/test_gpu_image_permissions.py index b07047cc..b8885a4b 100644 --- a/tests/scripts/test_gpu_image_permissions.py +++ b/tests/scripts/test_gpu_image_permissions.py @@ -20,9 +20,3 @@ def test_rocm_base_leaves_gpu_device_permissions_to_the_host() -> None: ) for pattern in forbidden_patterns: assert re.search(pattern, dockerfile) is None, pattern - - assert "echo 'export USER=jovyan' >> /entrypoint.sh" in dockerfile - assert "echo 'export SHELL=/bin/bash' >> /entrypoint.sh" in dockerfile - assert 'CMD ["/bin/bash", "/entrypoint.sh"]' in dockerfile - assert "USER $NB_UID" in dockerfile - assert "WORKDIR /home/jovyan" in dockerfile