From 874320c1ddb310ce344ae44a61f9cc056b37241b Mon Sep 17 00:00:00 2001 From: Anthony Byrne Date: Fri, 21 Aug 2026 11:03:59 -0400 Subject: [PATCH] Add optional KMM integration for OoT kernel module loading Adds a KMMReconciler sub-controller that creates a KMM Module CR (owned by the ClusterPolicy, with a ResourceClaim deletion guard) when ClusterPolicy.spec.kernelModule is set and KMM is installed. The existing DP and DRA controllers keep their lifecycle logic; KMM handles only out-of-tree module loading (modprobe, kernel mappings, in-cluster builds). KMM availability is detected at startup via API group discovery, and downstream DaemonSets (DP, DRA, XPU Manager) gate on the KMM ready node label so they schedule only once the OoT module is loaded. Behavior is unchanged when kernelModule is unset. Signed-off-by: Anthony Byrne --- api/v1alpha1/clusterpolicy_types.go | 110 ++ api/v1alpha1/clusterpolicy_webhook.go | 103 ++ api/v1alpha1/clusterpolicy_webhook_test.go | 244 ++++ api/v1alpha1/zz_generated.deepcopy.go | 123 ++ build/operator/Dockerfile | 6 +- .../templates/clusterpolicy.yaml | 60 + charts/gpu-base-operator-policy/values.yaml | 41 + .../crds/clusterpolicies.yaml | 162 +++ .../gpu-base-operator/templates/manager.yaml | 2 + .../templates/module_loader_scc.yaml | 40 + .../templates/module_loader_scc_role.yaml | 18 + .../module_loader_scc_rolebinding.yaml | 17 + .../module_loader_serviceaccount.yaml | 10 + charts/gpu-base-operator/templates/role.yaml | 18 + charts/gpu-base-operator/values.yaml | 7 + cmd/main.go | 25 +- .../crd/bases/intel.com_clusterpolicies.yaml | 162 +++ config/manager/manager.yaml | 4 + config/rbac/kustomization.yaml | 1 + config/rbac/module_loader_serviceaccount.yaml | 8 + config/rbac/role.yaml | 18 + go.mod | 1 + go.sum | 2 + .../controller/clusterpolicy_controller.go | 39 +- internal/controller/controller_utils.go | 36 +- .../controller/deviceplugin_controller.go | 2 +- internal/controller/dra_controller.go | 38 +- internal/controller/kmm_controller.go | 313 ++++ internal/controller/kmm_controller_test.go | 1273 +++++++++++++++++ internal/controller/suite_test.go | 15 + internal/controller/xpumanager_controller.go | 2 +- 31 files changed, 2847 insertions(+), 53 deletions(-) create mode 100644 charts/gpu-base-operator/templates/module_loader_scc.yaml create mode 100644 charts/gpu-base-operator/templates/module_loader_scc_role.yaml create mode 100644 charts/gpu-base-operator/templates/module_loader_scc_rolebinding.yaml create mode 100644 charts/gpu-base-operator/templates/module_loader_serviceaccount.yaml create mode 100644 config/rbac/module_loader_serviceaccount.yaml create mode 100644 internal/controller/kmm_controller.go create mode 100644 internal/controller/kmm_controller_test.go diff --git a/api/v1alpha1/clusterpolicy_types.go b/api/v1alpha1/clusterpolicy_types.go index a58a3a1..d9b048e 100644 --- a/api/v1alpha1/clusterpolicy_types.go +++ b/api/v1alpha1/clusterpolicy_types.go @@ -69,6 +69,12 @@ type ClusterPolicySpec struct { // +kubebuilder:validation:Range=0:4 // +kubebuilder:validation:Default=2 LogLevel int32 `json:"logLevel,omitempty"` + + // KernelModule configures out-of-tree kernel module loading via KMM. + // When set, KMM loads the specified OOT driver module on each node. + // When nil, the in-tree kernel driver is used. + // +optional + KernelModule *KernelModuleSpec `json:"kernelModule,omitempty"` } // DynamicResourceAllocationSpec defines the desired state of DynamicResourceAllocation. @@ -147,11 +153,114 @@ type XpuManagerSpec struct { MonitoringResource string `json:"monitoringResource,omitempty"` } +// RegistryTLSSpec configures TLS behavior for accessing container image registries. +type RegistryTLSSpec struct { + Insecure bool `json:"insecure,omitempty"` + InsecureSkipTLSVerify bool `json:"insecureSkipTLSVerify,omitempty"` +} + +// KernelModuleSpec configures out-of-tree kernel module loading via KMM. +type KernelModuleSpec struct { + // ModuleName is the kernel module to load (defaults to "xe"). + // +kubebuilder:default=xe + ModuleName string `json:"moduleName,omitempty"` + + // Version opts into KMM's ordered upgrade + // (https://kmm.sigs.k8s.io/documentation/ordered_upgrade) for advanced, + // low-disruption driver rollouts. When set, KMM loads the module onto a node + // only once a cluster admin labels that node + // "kmm.node.kubernetes.io/version-module..=", + // letting the admin sequence the upgrade node-by-node and drain GPU workloads + // first. Nodes without a matching label are left untouched. + // + // Most users should leave Version unset and instead upgrade by changing the + // containerImage of the relevant kernelMappings entry (see ContainerImage), + // which rolls the new driver out to all selected nodes at once without any + // per-node label choreography. + // +optional + Version string `json:"version,omitempty"` + + // KernelMappings maps kernel version patterns to container images or + // build specifications. Translates directly to KMM KernelMapping objects. + // +kubebuilder:validation:MinItems=1 + KernelMappings []KernelMappingSpec `json:"kernelMappings"` + + // ModulesLoadingOrder specifies softdep-style loading order for + // multi-module drivers. First element must be ModuleName (defaults + // to "xe"); KMM loads in order and unloads in reverse. Must have + // >=2 entries if set. + // +optional + ModulesLoadingOrder []string `json:"modulesLoadingOrder,omitempty"` + + // FirmwarePath is the in-container path where firmware files are stored. + // +optional + FirmwarePath string `json:"firmwarePath,omitempty"` + + // RegistryTLS configures TLS for accessing the module image registry. + // +optional + RegistryTLS *RegistryTLSSpec `json:"registryTLS,omitempty"` +} + +// KernelMappingSpec maps a kernel version pattern to a container image +// or build specification. +type KernelMappingSpec struct { + // Regexp is a regular expression matched against node kernel versions. + // Use anchored patterns (e.g. "^5\\.14\\.0-.*$") for exact matches. + Regexp string `json:"regexp"` + + // ContainerImage is the full image reference for this kernel version. + // Required when Build is nil. KMM template vars (e.g. ${KERNEL_FULL_VERSION}, + // $MOD_NAME) are supported and resolved by KMM at reconcile time. + // + // Changing ContainerImage is the recommended way to upgrade the driver: KMM + // rolls the new image out to all selected nodes at once, briefly disrupting + // GPU workloads as the module reloads. + // +optional + ContainerImage string `json:"containerImage,omitempty"` + + // Build configures in-cluster building of the driver image via KMM. + // When set, KMM builds the image if it doesn't exist in the registry. + // +optional + Build *KernelModuleBuildSpec `json:"build,omitempty"` + + // InTreeModulesToRemove lists additional in-tree modules to unload + // for this mapping. ModuleName is always included automatically. + // +optional + InTreeModulesToRemove []string `json:"inTreeModulesToRemove,omitempty"` + + // RegistryTLS overrides parent-level TLS settings for this mapping. + // +optional + RegistryTLS *RegistryTLSSpec `json:"registryTLS,omitempty"` +} + +// KernelModuleBuildSpec configures in-cluster driver image building. +type KernelModuleBuildSpec struct { + // DockerfileConfigMap references a ConfigMap containing the Dockerfile. + DockerfileConfigMap v1.LocalObjectReference `json:"dockerfileConfigMap"` + + // BuildArgs are key-value pairs passed to the image builder. + // +optional + BuildArgs []BuildArg `json:"buildArgs,omitempty"` + + // Secrets are made available during the build (e.g., for private + // source repos). Not for registry auth -- use pullSecret on + // ClusterPolicySpec. + // +optional + Secrets []v1.LocalObjectReference `json:"secrets,omitempty"` +} + +// BuildArg is a key-value pair passed as a build argument. +type BuildArg struct { + Name string `json:"name"` + Value string `json:"value"` +} + // ClusterPolicyStatus defines the observed state of ClusterPolicy. type ClusterPolicyStatus struct { DevicePluginStatus string `json:"devicePluginStatus,omitempty"` DRAStatus string `json:"draStatus,omitempty"` XPUManagerStatus string `json:"xpuManagerStatus,omitempty"` + KMMStatus string `json:"kmmStatus,omitempty"` Errors []string `json:"errors,omitempty"` } @@ -183,6 +292,7 @@ type LocalQueueSpec struct { // +kubebuilder:printcolumn:name="DP",type=string,JSONPath=`.status.devicePluginStatus` // +kubebuilder:printcolumn:name="DRA",type=string,JSONPath=`.status.draStatus` // +kubebuilder:printcolumn:name="XPU",type=string,JSONPath=`.status.xpuManagerStatus` +// +kubebuilder:printcolumn:name="KMM",type=string,JSONPath=`.status.kmmStatus` // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` // +operator-sdk:csv:customresourcedefinitions:displayName="Intel GPU Cluster Policy" // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/api/v1alpha1/clusterpolicy_webhook.go b/api/v1alpha1/clusterpolicy_webhook.go index 2b58d53..731f763 100644 --- a/api/v1alpha1/clusterpolicy_webhook.go +++ b/api/v1alpha1/clusterpolicy_webhook.go @@ -74,6 +74,10 @@ func (d *ClusterPolicyCustomDefaulter) Default(_ context.Context, cp *ClusterPol spec.XpuManagerSpec.MonitoringResource = "monitoring" } + if spec.KernelModule != nil && spec.KernelModule.ModuleName == "" { + spec.KernelModule.ModuleName = "xe" + } + return nil } @@ -95,6 +99,7 @@ func validateClusterPolicySpec(spec *ClusterPolicySpec) (admission.Warnings, err errs = append(errs, validatePullSecret(spec)...) errs = append(errs, validateConfigMapOverride(spec)...) errs = append(errs, validateKueueSpec(spec)...) + errs = append(errs, validateKernelModuleSpec(spec)...) if w := warnForSpecProblems(spec); w != "" { warnings = append(warnings, w) @@ -223,6 +228,104 @@ func validateKueueSpec(spec *ClusterPolicySpec) []error { return errs } +func validateKernelModuleSpec(spec *ClusterPolicySpec) []error { + if spec.KernelModule == nil { + return nil + } + + km := spec.KernelModule + var errs []error + + for i, m := range km.KernelMappings { + prefix := fmt.Sprintf("kernelModule.kernelMappings[%d]", i) + errs = append(errs, validateKernelMapping(prefix, &m)...) + } + + if len(km.ModulesLoadingOrder) > 0 { + errs = append(errs, validateModulesLoadingOrder(km.ModuleName, km.ModulesLoadingOrder)...) + } + + return errs +} + +// kmmTemplateVarRegexp matches KMM's ${VAR} and $VAR template placeholders +// (e.g. ${KERNEL_FULL_VERSION}, $MOD_NAME), which KMM resolves at reconcile time. +var kmmTemplateVarRegexp = regexp.MustCompile(`\$\{[^}]*\}|\$[A-Za-z_][A-Za-z0-9_]*`) + +// substituteKMMTemplateVars swaps KMM template placeholders for a tag-safe token +// so the reference can be validated as a normal image reference. +func substituteKMMTemplateVars(image string) string { + return kmmTemplateVarRegexp.ReplaceAllString(image, "0") +} + +func validateImageTagOrDigest(field, image string) error { + // Validate a copy with template vars resolved; report the original on error. + resolved := substituteKMMTemplateVars(image) + + if _, err := reference.ParseNormalizedNamed(resolved); err != nil { + return fmt.Errorf("invalid image reference in %s: %q: %w", field, image, err) + } + + if !strings.Contains(resolved, ":") && !strings.Contains(resolved, "@") { + return fmt.Errorf("%s: image %q must include an explicit tag or digest", field, image) + } + + return nil +} + +func validateKernelMapping(prefix string, m *KernelMappingSpec) []error { + var errs []error + + if m.Regexp == "" { + errs = append(errs, fmt.Errorf("%s: regexp is required", prefix)) + } else { + if _, err := regexp.Compile(m.Regexp); err != nil { + errs = append(errs, fmt.Errorf("%s.regexp: invalid regular expression: %w", prefix, err)) + } + } + + if m.ContainerImage != "" { + if err := validateImageTagOrDigest(prefix+".containerImage", m.ContainerImage); err != nil { + errs = append(errs, err) + } + } + + if m.ContainerImage == "" && m.Build == nil { + errs = append(errs, fmt.Errorf("%s: one of containerImage or build must be set", prefix)) + } + + if m.Build != nil { + if m.Build.DockerfileConfigMap.Name == "" { + errs = append(errs, fmt.Errorf("%s.build.dockerfileConfigMap.name is required", prefix)) + } + } + + return errs +} + +func validateModulesLoadingOrder(moduleName string, order []string) []error { + var errs []error + + if len(order) < 2 { + errs = append(errs, fmt.Errorf("kernelModule.modulesLoadingOrder must have at least 2 entries")) + } + + if len(order) > 0 && moduleName != "" && order[0] != moduleName { + errs = append(errs, fmt.Errorf("kernelModule.modulesLoadingOrder[0] must be moduleName %q", moduleName)) + } + + seen := make(map[string]bool, len(order)) + for i, entry := range order { + if seen[entry] { + errs = append(errs, fmt.Errorf("kernelModule.modulesLoadingOrder[%d]: duplicate entry %q", i, entry)) + break + } + seen[entry] = true + } + + return errs +} + // warnForSpecProblems returns a warning message if some old or deprecated option is set. func warnForSpecProblems(spec *ClusterPolicySpec) string { if spec.ResourceRegistration == "dp" && spec.DevicePluginSpec.LevelzeroImage != "" { diff --git a/api/v1alpha1/clusterpolicy_webhook_test.go b/api/v1alpha1/clusterpolicy_webhook_test.go index 45967d1..9404be2 100644 --- a/api/v1alpha1/clusterpolicy_webhook_test.go +++ b/api/v1alpha1/clusterpolicy_webhook_test.go @@ -387,6 +387,250 @@ var _ = Describe("ClusterPolicy Webhook", func() { }) }) + Context("kernelModule validation", func() { + It("accepts nil kernelModule (in-tree mode)", func() { + obj.Spec.KernelModule = nil + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("accepts valid kernelModule with single wildcard mapping", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("accepts kernelModule without moduleName (defaults to xe)", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + KernelMappings: []KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("accepts kernelMappings with each mapping having its own containerImage", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "^5\\.14\\..*", ContainerImage: "registry.example.com/xe-rhel9:1.0"}, + {Regexp: "^6\\.12\\..*", ContainerImage: "registry.example.com/xe-rhel10:1.0"}, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("rejects mapping without containerImage or build", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "^5\\.14\\..*"}, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("one of containerImage or build must be set")) + }) + + It("rejects mapping without regexp", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {ContainerImage: "registry.example.com/xe:1.0"}, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("regexp is required")) + }) + + It("rejects mapping with invalid regexp", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "[invalid", ContainerImage: "registry.example.com/xe:1.0"}, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("invalid regular expression")) + }) + + It("rejects containerImage without tag or digest", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "^5\\.14\\..*", ContainerImage: "registry.example.com/xe-driver"}, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("must include an explicit tag or digest")) + }) + + It("accepts containerImage with a ${KERNEL_FULL_VERSION} template var in the tag", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-oot-kmd:v1.0-${KERNEL_FULL_VERSION}"}, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("accepts containerImage with a bare $KERNEL_FULL_VERSION template var in the tag", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-oot-kmd:v1.0-$KERNEL_FULL_VERSION"}, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("accepts containerImage using $MOD_NAME/$MOD_NAMESPACE template vars", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/${MOD_NAMESPACE}/$MOD_NAME:v1.0"}, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("still rejects a templated containerImage with no tag or digest", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-oot-kmd-${KERNEL_FULL_VERSION}"}, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("must include an explicit tag or digest")) + }) + + It("accepts valid build spec with dockerfileConfigMap", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + { + Regexp: "^5\\.14\\..*", + Build: &KernelModuleBuildSpec{ + DockerfileConfigMap: v1.LocalObjectReference{Name: "xe-dockerfile"}, + BuildArgs: []BuildArg{{Name: "XE_TAG", Value: "v1.0"}}, + }, + }, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("rejects build spec with empty dockerfileConfigMap name", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + { + Regexp: "^5\\.14\\..*", + Build: &KernelModuleBuildSpec{}, + }, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("dockerfileConfigMap.name is required")) + }) + + It("accepts valid registryTLS at top level", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe:1.0"}, + }, + RegistryTLS: &RegistryTLSSpec{InsecureSkipTLSVerify: true}, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("accepts valid registryTLS per-mapping", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + { + Regexp: "^.+$", + ContainerImage: "registry.example.com/xe:1.0", + RegistryTLS: &RegistryTLSSpec{Insecure: true}, + }, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("accepts valid modulesLoadingOrder with moduleName first", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + ModulesLoadingOrder: []string{"xe", "drm_buddy"}, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("rejects modulesLoadingOrder with fewer than 2 entries", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + ModulesLoadingOrder: []string{"xe"}, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("at least 2 entries")) + }) + + It("rejects modulesLoadingOrder where first entry is not moduleName", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + ModulesLoadingOrder: []string{"drm_buddy", "xe"}, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("must be moduleName")) + }) + + It("rejects modulesLoadingOrder with duplicate entries", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + ModulesLoadingOrder: []string{"xe", "drm_buddy", "drm_buddy"}, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("duplicate entry")) + }) + }) + Context("spec warning for Levelzero", func() { It("emits a warning when levelzero image is set in DP mode", func() { obj.Spec.ResourceRegistration = dpName diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 2470cf4..f3e8257 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -25,6 +25,21 @@ import ( "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BuildArg) DeepCopyInto(out *BuildArg) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BuildArg. +func (in *BuildArg) DeepCopy() *BuildArg { + if in == nil { + return nil + } + out := new(BuildArg) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterPolicy) DeepCopyInto(out *ClusterPolicy) { *out = *in @@ -149,6 +164,11 @@ func (in *ClusterPolicySpec) DeepCopyInto(out *ClusterPolicySpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.KernelModule != nil { + in, out := &in.KernelModule, &out.KernelModule + *out = new(KernelModuleSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterPolicySpec. @@ -466,6 +486,94 @@ func (in *HealthinessSpec) DeepCopy() *HealthinessSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KernelMappingSpec) DeepCopyInto(out *KernelMappingSpec) { + *out = *in + if in.Build != nil { + in, out := &in.Build, &out.Build + *out = new(KernelModuleBuildSpec) + (*in).DeepCopyInto(*out) + } + if in.InTreeModulesToRemove != nil { + in, out := &in.InTreeModulesToRemove, &out.InTreeModulesToRemove + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.RegistryTLS != nil { + in, out := &in.RegistryTLS, &out.RegistryTLS + *out = new(RegistryTLSSpec) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KernelMappingSpec. +func (in *KernelMappingSpec) DeepCopy() *KernelMappingSpec { + if in == nil { + return nil + } + out := new(KernelMappingSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KernelModuleBuildSpec) DeepCopyInto(out *KernelModuleBuildSpec) { + *out = *in + out.DockerfileConfigMap = in.DockerfileConfigMap + if in.BuildArgs != nil { + in, out := &in.BuildArgs, &out.BuildArgs + *out = make([]BuildArg, len(*in)) + copy(*out, *in) + } + if in.Secrets != nil { + in, out := &in.Secrets, &out.Secrets + *out = make([]v1.LocalObjectReference, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KernelModuleBuildSpec. +func (in *KernelModuleBuildSpec) DeepCopy() *KernelModuleBuildSpec { + if in == nil { + return nil + } + out := new(KernelModuleBuildSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KernelModuleSpec) DeepCopyInto(out *KernelModuleSpec) { + *out = *in + if in.KernelMappings != nil { + in, out := &in.KernelMappings, &out.KernelMappings + *out = make([]KernelMappingSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.ModulesLoadingOrder != nil { + in, out := &in.ModulesLoadingOrder, &out.ModulesLoadingOrder + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.RegistryTLS != nil { + in, out := &in.RegistryTLS, &out.RegistryTLS + *out = new(RegistryTLSSpec) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KernelModuleSpec. +func (in *KernelModuleSpec) DeepCopy() *KernelModuleSpec { + if in == nil { + return nil + } + out := new(KernelModuleSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KueueQueueSpec) DeepCopyInto(out *KueueQueueSpec) { *out = *in @@ -503,6 +611,21 @@ func (in *LocalQueueSpec) DeepCopy() *LocalQueueSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RegistryTLSSpec) DeepCopyInto(out *RegistryTLSSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RegistryTLSSpec. +func (in *RegistryTLSSpec) DeepCopy() *RegistryTLSSpec { + if in == nil { + return nil + } + out := new(RegistryTLSSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *XpuManagerSpec) DeepCopyInto(out *XpuManagerSpec) { *out = *in diff --git a/build/operator/Dockerfile b/build/operator/Dockerfile index 0a09c39..f288eb4 100644 --- a/build/operator/Dockerfile +++ b/build/operator/Dockerfile @@ -53,8 +53,10 @@ RUN apt-get update -y && apt-get --no-install-recommends -y install file && \ chmod +x checksec && \ ./checksec --file=/workspace/manager --output=csv | grep -q "$CHECKSEC_REF" -RUN go get github.com/google/go-licenses && \ - go run github.com/google/go-licenses save ./cmd/ --save_path licenses && \ +# GOROOT is passed explicitly so go-licenses can recognize stdlib packages even +# when `go list` re-execs a different toolchain than the one that built it. +RUN go install github.com/google/go-licenses@v1.6.0 && \ + GOROOT="$(go env GOROOT)" go-licenses save ./cmd/ --save_path licenses && \ cp /workspace/LICENSE /workspace/licenses/LICENSE.intel-gpu-base-operator ARG BASE_IMAGE diff --git a/charts/gpu-base-operator-policy/templates/clusterpolicy.yaml b/charts/gpu-base-operator-policy/templates/clusterpolicy.yaml index 737355d..84c0aea 100644 --- a/charts/gpu-base-operator-policy/templates/clusterpolicy.yaml +++ b/charts/gpu-base-operator-policy/templates/clusterpolicy.yaml @@ -62,6 +62,66 @@ spec: configMapOverride: {{ .Values.xpu.configMapOverride | default "" }} {{- end }} +{{- if .Values.kernelModule }} + kernelModule: + moduleName: {{ .Values.kernelModule.moduleName }} + {{- if .Values.kernelModule.version }} + version: {{ .Values.kernelModule.version | quote }} + {{- end }} + kernelMappings: + {{- range .Values.kernelModule.kernelMappings }} + - regexp: {{ .regexp | quote }} + {{- if .containerImage }} + containerImage: {{ .containerImage }} + {{- end }} + {{- if .build }} + build: + dockerfileConfigMap: + name: {{ .build.dockerfileConfigMap.name }} + {{- if .build.buildArgs }} + buildArgs: + {{- range .build.buildArgs }} + - name: {{ .name }} + value: {{ .value | quote }} + {{- end }} + {{- end }} + {{- if .build.secrets }} + secrets: + {{- range .build.secrets }} + - name: {{ .name }} + {{- end }} + {{- end }} + {{- end }} + {{- if .inTreeModulesToRemove }} + inTreeModulesToRemove: {{ toYaml .inTreeModulesToRemove | nindent 10 }} + {{- end }} + {{- if .registryTLS }} + registryTLS: + {{- if .registryTLS.insecure }} + insecure: {{ .registryTLS.insecure }} + {{- end }} + {{- if .registryTLS.insecureSkipTLSVerify }} + insecureSkipTLSVerify: {{ .registryTLS.insecureSkipTLSVerify }} + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.kernelModule.modulesLoadingOrder }} + modulesLoadingOrder: {{ toYaml .Values.kernelModule.modulesLoadingOrder | nindent 6 }} + {{- end }} + {{- if .Values.kernelModule.firmwarePath }} + firmwarePath: {{ .Values.kernelModule.firmwarePath }} + {{- end }} + {{- if .Values.kernelModule.registryTLS }} + registryTLS: + {{- if .Values.kernelModule.registryTLS.insecure }} + insecure: {{ .Values.kernelModule.registryTLS.insecure }} + {{- end }} + {{- if .Values.kernelModule.registryTLS.insecureSkipTLSVerify }} + insecureSkipTLSVerify: {{ .Values.kernelModule.registryTLS.insecureSkipTLSVerify }} + {{- end }} + {{- end }} +{{- end }} + {{- if .Values.pullSecret }} pullSecret: {{ .Values.pullSecret }} {{- end }} diff --git a/charts/gpu-base-operator-policy/values.yaml b/charts/gpu-base-operator-policy/values.yaml index fc465f2..6dd3969 100644 --- a/charts/gpu-base-operator-policy/values.yaml +++ b/charts/gpu-base-operator-policy/values.yaml @@ -35,6 +35,47 @@ kueue: - name: gpu-queue namespace: default +# kernelModule: +# moduleName: xe +# # To upgrade the driver, just change the containerImage below; KMM rolls it +# # out to all selected nodes at once (brief GPU-workload disruption). Only set +# # `version` if you want KMM's node-by-node ordered upgrade, which requires you +# # to label each node with +# # kmm.node.kubernetes.io/version-module..= before +# # the driver loads there. See https://kmm.sigs.k8s.io/documentation/ordered_upgrade +# # version: "1.0" +# kernelMappings: +# - regexp: "^.+$" +# containerImage: registry.example.com/xe-driver:1.0 +# +# # Multi-kernel support: +# # kernelMappings: +# # - regexp: "^5\\.14\\.0-.*\\.el9.*\\.x86_64$" +# # containerImage: registry.example.com/xe-driver-rhel9:1.0 +# # - regexp: "^6\\.12\\..*" +# # containerImage: registry.example.com/xe-driver-rhel10:1.0 +# # +# # In-cluster build: +# # kernelMappings: +# # - regexp: "^5\\.14\\..*" +# # build: +# # dockerfileConfigMap: +# # name: xe-dockerfile +# # buildArgs: +# # - name: XE_TAG +# # value: v1.0 +# # +# # Per-mapping extra in-tree modules to remove (moduleName is always included automatically): +# # - regexp: "^5\\.14\\..*" +# # containerImage: registry.example.com/xe:1.0 +# # inTreeModulesToRemove: ["i915"] +# # +# # modulesLoadingOrder: ["xe", "drm_buddy", "drm_ttm_helper"] +# # firmwarePath: /opt/lib/firmware/xe +# # registryTLS: +# # insecure: false +# # insecureSkipTLSVerify: false + pullSecret: null nodeSelector: {} tolerations: [] diff --git a/charts/gpu-base-operator/crds/clusterpolicies.yaml b/charts/gpu-base-operator/crds/clusterpolicies.yaml index 870f9c5..f4ce3f7 100644 --- a/charts/gpu-base-operator/crds/clusterpolicies.yaml +++ b/charts/gpu-base-operator/crds/clusterpolicies.yaml @@ -24,6 +24,9 @@ spec: - jsonPath: .status.xpuManagerStatus name: XPU type: string + - jsonPath: .status.kmmStatus + name: KMM + type: string - jsonPath: .metadata.creationTimestamp name: Age type: date @@ -136,6 +139,163 @@ spec: minimum: 1 type: integer type: object + kernelModule: + description: |- + KernelModule configures out-of-tree kernel module loading via KMM. + When set, KMM loads the specified OOT driver module on each node. + When nil, the in-tree kernel driver is used. + properties: + firmwarePath: + description: FirmwarePath is the in-container path where firmware + files are stored. + type: string + kernelMappings: + description: |- + KernelMappings maps kernel version patterns to container images or + build specifications. Translates directly to KMM KernelMapping objects. + items: + description: |- + KernelMappingSpec maps a kernel version pattern to a container image + or build specification. + properties: + build: + description: |- + Build configures in-cluster building of the driver image via KMM. + When set, KMM builds the image if it doesn't exist in the registry. + properties: + buildArgs: + description: BuildArgs are key-value pairs passed to + the image builder. + items: + description: BuildArg is a key-value pair passed as + a build argument. + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + dockerfileConfigMap: + description: DockerfileConfigMap references a ConfigMap + containing the Dockerfile. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + secrets: + description: |- + Secrets are made available during the build (e.g., for private + source repos). Not for registry auth -- use pullSecret on + ClusterPolicySpec. + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + required: + - dockerfileConfigMap + type: object + containerImage: + description: |- + ContainerImage is the full image reference for this kernel version. + Required when Build is nil. KMM template vars (e.g. ${KERNEL_FULL_VERSION}, + $MOD_NAME) are supported and resolved by KMM at reconcile time. + + Changing ContainerImage is the recommended way to upgrade the driver: KMM + rolls the new image out to all selected nodes at once, briefly disrupting + GPU workloads as the module reloads. + type: string + inTreeModulesToRemove: + description: |- + InTreeModulesToRemove lists additional in-tree modules to unload + for this mapping. ModuleName is always included automatically. + items: + type: string + type: array + regexp: + description: |- + Regexp is a regular expression matched against node kernel versions. + Use anchored patterns (e.g. "^5\\.14\\.0-.*$") for exact matches. + type: string + registryTLS: + description: RegistryTLS overrides parent-level TLS settings + for this mapping. + properties: + insecure: + type: boolean + insecureSkipTLSVerify: + type: boolean + type: object + required: + - regexp + type: object + minItems: 1 + type: array + moduleName: + default: xe + description: ModuleName is the kernel module to load (defaults + to "xe"). + type: string + modulesLoadingOrder: + description: |- + ModulesLoadingOrder specifies softdep-style loading order for + multi-module drivers. First element must be ModuleName (defaults + to "xe"); KMM loads in order and unloads in reverse. Must have + >=2 entries if set. + items: + type: string + type: array + registryTLS: + description: RegistryTLS configures TLS for accessing the module + image registry. + properties: + insecure: + type: boolean + insecureSkipTLSVerify: + type: boolean + type: object + version: + description: |- + Version opts into KMM's ordered upgrade + (https://kmm.sigs.k8s.io/documentation/ordered_upgrade) for advanced, + low-disruption driver rollouts. When set, KMM loads the module onto a node + only once a cluster admin labels that node + "kmm.node.kubernetes.io/version-module..=", + letting the admin sequence the upgrade node-by-node and drain GPU workloads + first. Nodes without a matching label are left untouched. + + Most users should leave Version unset and instead upgrade by changing the + containerImage of the relevant kernelMappings entry (see ContainerImage), + which rolls the new driver out to all selected nodes at once without any + per-node label choreography. + type: string + required: + - kernelMappings + type: object kueue: description: Define Kueue queues properties: @@ -289,6 +449,8 @@ spec: items: type: string type: array + kmmStatus: + type: string xpuManagerStatus: type: string type: object diff --git a/charts/gpu-base-operator/templates/manager.yaml b/charts/gpu-base-operator/templates/manager.yaml index bec40f0..56667f7 100644 --- a/charts/gpu-base-operator/templates/manager.yaml +++ b/charts/gpu-base-operator/templates/manager.yaml @@ -64,6 +64,8 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: MODULE_LOADER_SERVICE_ACCOUNT_NAME + value: "{{ .Values.moduleLoader.serviceAccountName }}" {{- if .Values.privateRegistry.token }} - name: OPERATOR_SECRET value: "{{ .Release.Name }}-operator-private-registry" diff --git a/charts/gpu-base-operator/templates/module_loader_scc.yaml b/charts/gpu-base-operator/templates/module_loader_scc.yaml new file mode 100644 index 0000000..b778c9c --- /dev/null +++ b/charts/gpu-base-operator/templates/module_loader_scc.yaml @@ -0,0 +1,40 @@ +{{- if .Values.openshift.enabled }} +apiVersion: security.openshift.io/v1 +kind: SecurityContextConstraints +metadata: + name: {{ .Release.Name }}-module-loader-scc + labels: + app.kubernetes.io/name: intel-gpu-base-operator + app.kubernetes.io/managed-by: {{ .Release.Service }} +allowPrivilegedContainer: true +allowHostDirVolumePlugin: true +allowHostIPC: false +allowHostNetwork: false +allowHostPID: false +allowHostPorts: false +allowPrivilegeEscalation: true +allowedCapabilities: + - SYS_MODULE +defaultAddCapabilities: null +requiredDropCapabilities: null +fsGroup: + type: RunAsAny +readOnlyRootFilesystem: false +runAsUser: + type: RunAsAny +seLinuxContext: + type: RunAsAny +seccompProfiles: + - "*" +supplementalGroups: + type: RunAsAny +volumes: + - hostPath + - configMap + - secret + - projected + - emptyDir + - downwardAPI +users: [] +groups: [] +{{- end }} diff --git a/charts/gpu-base-operator/templates/module_loader_scc_role.yaml b/charts/gpu-base-operator/templates/module_loader_scc_role.yaml new file mode 100644 index 0000000..72f292f --- /dev/null +++ b/charts/gpu-base-operator/templates/module_loader_scc_role.yaml @@ -0,0 +1,18 @@ +{{- if .Values.openshift.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Release.Name }}-module-loader-scc-role + labels: + app.kubernetes.io/name: intel-gpu-base-operator + app.kubernetes.io/managed-by: {{ .Release.Service }} +rules: + - apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + resourceNames: + - {{ .Release.Name }}-module-loader-scc + verbs: + - use +{{- end }} diff --git a/charts/gpu-base-operator/templates/module_loader_scc_rolebinding.yaml b/charts/gpu-base-operator/templates/module_loader_scc_rolebinding.yaml new file mode 100644 index 0000000..c7d0e2d --- /dev/null +++ b/charts/gpu-base-operator/templates/module_loader_scc_rolebinding.yaml @@ -0,0 +1,17 @@ +{{- if .Values.openshift.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Release.Name }}-module-loader-scc-binding + labels: + app.kubernetes.io/name: intel-gpu-base-operator + app.kubernetes.io/managed-by: {{ .Release.Service }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Release.Name }}-module-loader-scc-role +subjects: + - kind: ServiceAccount + name: {{ .Values.moduleLoader.serviceAccountName }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/charts/gpu-base-operator/templates/module_loader_serviceaccount.yaml b/charts/gpu-base-operator/templates/module_loader_serviceaccount.yaml new file mode 100644 index 0000000..05989ef --- /dev/null +++ b/charts/gpu-base-operator/templates/module_loader_serviceaccount.yaml @@ -0,0 +1,10 @@ +{{- if .Values.moduleLoader.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.moduleLoader.serviceAccountName }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: intel-gpu-base-operator + app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} diff --git a/charts/gpu-base-operator/templates/role.yaml b/charts/gpu-base-operator/templates/role.yaml index c556fbd..51b89f7 100644 --- a/charts/gpu-base-operator/templates/role.yaml +++ b/charts/gpu-base-operator/templates/role.yaml @@ -78,6 +78,24 @@ rules: - get - patch - update +- apiGroups: + - kmm.sigs.x-k8s.io + resources: + - modules + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kmm.sigs.x-k8s.io + resources: + - modules/status + verbs: + - get - apiGroups: - kueue.x-k8s.io resources: diff --git a/charts/gpu-base-operator/values.yaml b/charts/gpu-base-operator/values.yaml index 831dbc5..7e383d8 100644 --- a/charts/gpu-base-operator/values.yaml +++ b/charts/gpu-base-operator/values.yaml @@ -76,3 +76,10 @@ kueue: - deviceClassNames: - "gpu.intel.com" name: "dra.gpu.intel.com" + +openshift: + enabled: false + +moduleLoader: + create: true + serviceAccountName: module-loader diff --git a/cmd/main.go b/cmd/main.go index 59dea7c..f94c9ba 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -55,6 +55,7 @@ import ( intelcomv1alpha1 "github.com/intel/gpu-base-operator/api/v1alpha1" "github.com/intel/gpu-base-operator/internal/controller" buildVersion "github.com/intel/gpu-base-operator/internal/version" + kmmv1beta1 "github.com/kubernetes-sigs/kernel-module-management/api/v1beta1" // +kubebuilder:scaffold:imports ) @@ -63,6 +64,7 @@ var ( setupLog = ctrl.Log.WithName("setup") openShiftGroups = []string{"route.openshift.io", "security.openshift.io"} draGroups = []string{"resource.k8s.io"} + kmmGroups = []string{"kmm.sigs.x-k8s.io"} ) const ( @@ -70,6 +72,7 @@ const ( openshiftCluster = "OpenShift" draCluster = "DRA" + kmmCluster = "KMM" ) func init() { @@ -80,6 +83,7 @@ func init() { utilruntime.Must(prometheusv1.AddToScheme(scheme)) utilruntime.Must(resv1.AddToScheme(scheme)) utilruntime.Must(kueuev1.AddToScheme(scheme)) + utilruntime.Must(kmmv1beta1.AddToScheme(scheme)) // +kubebuilder:scaffold:scheme } @@ -103,6 +107,7 @@ func detectClusterFeatures() (map[string]bool, error) { features := map[string]bool{ openshiftCluster: false, draCluster: false, + kmmCluster: false, } for _, group := range apiGroups.Groups { @@ -112,6 +117,9 @@ func detectClusterFeatures() (map[string]bool, error) { if slices.Contains(draGroups, group.Name) { features[draCluster] = true } + if slices.Contains(kmmGroups, group.Name) { + features[kmmCluster] = true + } } return features, nil @@ -323,12 +331,19 @@ func main() { setupLog.Info("Operator secret supplied", "secret", secret) } + moduleLoaderSAName := os.Getenv("MODULE_LOADER_SERVICE_ACCOUNT_NAME") + if moduleLoaderSAName == "" { + moduleLoaderSAName = "module-loader" + } + copts := controller.ControllerOpts{ - Namespace: ns, - SecretName: secret, - RequeueDelay: time.Second * 5, - DRAEnable: features[draCluster], - OpenShift: features[openshiftCluster], + Namespace: ns, + SecretName: secret, + RequeueDelay: time.Second * 5, + DRAEnable: features[draCluster], + OpenShift: features[openshiftCluster], + KMMEnable: features[kmmCluster], + ModuleLoaderServiceAccountName: moduleLoaderSAName, } if err := (&controller.ClusterPolicyReconciler{ diff --git a/config/crd/bases/intel.com_clusterpolicies.yaml b/config/crd/bases/intel.com_clusterpolicies.yaml index 870f9c5..f4ce3f7 100644 --- a/config/crd/bases/intel.com_clusterpolicies.yaml +++ b/config/crd/bases/intel.com_clusterpolicies.yaml @@ -24,6 +24,9 @@ spec: - jsonPath: .status.xpuManagerStatus name: XPU type: string + - jsonPath: .status.kmmStatus + name: KMM + type: string - jsonPath: .metadata.creationTimestamp name: Age type: date @@ -136,6 +139,163 @@ spec: minimum: 1 type: integer type: object + kernelModule: + description: |- + KernelModule configures out-of-tree kernel module loading via KMM. + When set, KMM loads the specified OOT driver module on each node. + When nil, the in-tree kernel driver is used. + properties: + firmwarePath: + description: FirmwarePath is the in-container path where firmware + files are stored. + type: string + kernelMappings: + description: |- + KernelMappings maps kernel version patterns to container images or + build specifications. Translates directly to KMM KernelMapping objects. + items: + description: |- + KernelMappingSpec maps a kernel version pattern to a container image + or build specification. + properties: + build: + description: |- + Build configures in-cluster building of the driver image via KMM. + When set, KMM builds the image if it doesn't exist in the registry. + properties: + buildArgs: + description: BuildArgs are key-value pairs passed to + the image builder. + items: + description: BuildArg is a key-value pair passed as + a build argument. + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + dockerfileConfigMap: + description: DockerfileConfigMap references a ConfigMap + containing the Dockerfile. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + secrets: + description: |- + Secrets are made available during the build (e.g., for private + source repos). Not for registry auth -- use pullSecret on + ClusterPolicySpec. + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + required: + - dockerfileConfigMap + type: object + containerImage: + description: |- + ContainerImage is the full image reference for this kernel version. + Required when Build is nil. KMM template vars (e.g. ${KERNEL_FULL_VERSION}, + $MOD_NAME) are supported and resolved by KMM at reconcile time. + + Changing ContainerImage is the recommended way to upgrade the driver: KMM + rolls the new image out to all selected nodes at once, briefly disrupting + GPU workloads as the module reloads. + type: string + inTreeModulesToRemove: + description: |- + InTreeModulesToRemove lists additional in-tree modules to unload + for this mapping. ModuleName is always included automatically. + items: + type: string + type: array + regexp: + description: |- + Regexp is a regular expression matched against node kernel versions. + Use anchored patterns (e.g. "^5\\.14\\.0-.*$") for exact matches. + type: string + registryTLS: + description: RegistryTLS overrides parent-level TLS settings + for this mapping. + properties: + insecure: + type: boolean + insecureSkipTLSVerify: + type: boolean + type: object + required: + - regexp + type: object + minItems: 1 + type: array + moduleName: + default: xe + description: ModuleName is the kernel module to load (defaults + to "xe"). + type: string + modulesLoadingOrder: + description: |- + ModulesLoadingOrder specifies softdep-style loading order for + multi-module drivers. First element must be ModuleName (defaults + to "xe"); KMM loads in order and unloads in reverse. Must have + >=2 entries if set. + items: + type: string + type: array + registryTLS: + description: RegistryTLS configures TLS for accessing the module + image registry. + properties: + insecure: + type: boolean + insecureSkipTLSVerify: + type: boolean + type: object + version: + description: |- + Version opts into KMM's ordered upgrade + (https://kmm.sigs.k8s.io/documentation/ordered_upgrade) for advanced, + low-disruption driver rollouts. When set, KMM loads the module onto a node + only once a cluster admin labels that node + "kmm.node.kubernetes.io/version-module..=", + letting the admin sequence the upgrade node-by-node and drain GPU workloads + first. Nodes without a matching label are left untouched. + + Most users should leave Version unset and instead upgrade by changing the + containerImage of the relevant kernelMappings entry (see ContainerImage), + which rolls the new driver out to all selected nodes at once without any + per-node label choreography. + type: string + required: + - kernelMappings + type: object kueue: description: Define Kueue queues properties: @@ -289,6 +449,8 @@ spec: items: type: string type: array + kmmStatus: + type: string xpuManagerStatus: type: string type: object diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 201c738..0bced06 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -69,6 +69,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: MODULE_LOADER_SERVICE_ACCOUNT_NAME + # kustomize prepends the namePrefix (intel-gpu-base-operator-) to the + # ServiceAccount, but not to this literal, so set the resolved name here. + value: intel-gpu-base-operator-module-loader image: ghcr.io/intel/intel-gpu-base-operator:devel imagePullPolicy: IfNotPresent name: manager diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 955856d..47aafc1 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -27,6 +27,7 @@ resources: - gpufirmwareupdate_admin_role.yaml - gpufirmwareupdate_editor_role.yaml - gpufirmwareupdate_viewer_role.yaml +- module_loader_serviceaccount.yaml - clusterpolicy_admin_role.yaml - clusterpolicy_editor_role.yaml - clusterpolicy_viewer_role.yaml diff --git a/config/rbac/module_loader_serviceaccount.yaml b/config/rbac/module_loader_serviceaccount.yaml new file mode 100644 index 0000000..c9719cd --- /dev/null +++ b/config/rbac/module_loader_serviceaccount.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: module-loader + namespace: system + labels: + app.kubernetes.io/name: intel-gpu-base-operator + app.kubernetes.io/managed-by: kustomize diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index e691942..b68ee6b 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -78,6 +78,24 @@ rules: - get - patch - update +- apiGroups: + - kmm.sigs.x-k8s.io + resources: + - modules + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kmm.sigs.x-k8s.io + resources: + - modules/status + verbs: + - get - apiGroups: - kueue.x-k8s.io resources: diff --git a/go.mod b/go.mod index 5f33aa7..d0b4566 100644 --- a/go.mod +++ b/go.mod @@ -63,6 +63,7 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.19.1 // indirect + github.com/kubernetes-sigs/kernel-module-management v1.0.1-0.20260729111759-de3f8c9d5eae github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect diff --git a/go.sum b/go.sum index 536ccaa..fd50d9a 100644 --- a/go.sum +++ b/go.sum @@ -117,6 +117,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kubernetes-sigs/kernel-module-management v1.0.1-0.20260729111759-de3f8c9d5eae h1:u3TS/PJYMOpIhAmAs1blrMlafMp+4FbFvzy4fk+CalY= +github.com/kubernetes-sigs/kernel-module-management v1.0.1-0.20260729111759-de3f8c9d5eae/go.mod h1:tqr/yRJMo+6VzLB9B/25H+wWvB+VMr95CYHD4ObyByA= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= diff --git a/internal/controller/clusterpolicy_controller.go b/internal/controller/clusterpolicy_controller.go index dc44292..b8052bf 100644 --- a/internal/controller/clusterpolicy_controller.go +++ b/internal/controller/clusterpolicy_controller.go @@ -19,6 +19,7 @@ package controller import ( "context" "errors" + "fmt" "reflect" "slices" "time" @@ -41,6 +42,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" v1alpha "github.com/intel/gpu-base-operator/api/v1alpha1" + kmmv1beta1 "github.com/kubernetes-sigs/kernel-module-management/api/v1beta1" ) // ClusterPolicyReconciler reconciles a ClusterPolicy object @@ -52,12 +54,15 @@ type ClusterPolicyReconciler struct { } type ControllerOpts struct { - ReqName string - Namespace string - SecretName string - RequeueDelay time.Duration - DRAEnable bool - OpenShift bool + ReqName string + Namespace string + SecretName string + RequeueDelay time.Duration + DRAEnable bool + OpenShift bool + KMMEnable bool + ModuleLoaderServiceAccountName string + KMMModuleReadyLabel string } type requeueReconcileErr struct { @@ -146,6 +151,11 @@ func (r *ClusterPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Reques var origCp *v1alpha.ClusterPolicy if cp != nil { origCp = cp.DeepCopy() + + // Clear stale errors so Status.Errors reflects the current reconcile only. + // Sub-controllers re-add any errors that are still relevant this pass; the + // deferred DeepEqual status update persists the cleared slice. + cp.Status.Errors = nil } // Defer status update at the end of reconciliation, to ensure we capture any changes made by sub-controllers. @@ -169,7 +179,12 @@ func (r *ClusterPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Reques opts := r.Opts opts.ReqName = req.Name - subControllers := make([]SubControllerInterface, 0, 4) + if opts.KMMEnable { + opts.KMMModuleReadyLabel = fmt.Sprintf("kmm.node.kubernetes.io/%s.%s.ready", + opts.Namespace, kmmModuleName(req.Name)) + } + + subControllers := make([]SubControllerInterface, 0, 5) // Initialize sub-controllers subControllers = append(subControllers, &DevicePluginReconciler{Client: r.Client, Scheme: r.Scheme, Opts: opts}) @@ -178,6 +193,12 @@ func (r *ClusterPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Reques subControllers = append(subControllers, &DRAReconciler{Client: r.Client, Scheme: r.Scheme, Opts: opts}) subControllers = append(subControllers, &MiscReconciler{Client: r.Client, APIReader: r.APIReader, Scheme: r.Scheme, Opts: opts, CrdNames: crdNames}) + // KMM sub-controller gets a copy with the ready label cleared so the + // KMM Module itself doesn't gate on its own readiness. + kmmOpts := opts + kmmOpts.KMMModuleReadyLabel = "" + subControllers = append(subControllers, &KMMReconciler{Client: r.Client, Scheme: r.Scheme, Opts: kmmOpts}) + // Ensure finalizer is present on live (non-deleted) ClusterPolicy objects. if cp != nil && cp.DeletionTimestamp.IsZero() { if !controllerutil.ContainsFinalizer(cp, clusterPolicyFinalizer) { @@ -345,6 +366,10 @@ func (r *ClusterPolicyReconciler) SetupWithManager(mgr ctrl.Manager, opts Contro Named("clusterpolicy"). Owns(&apps.DaemonSet{}) + if opts.KMMEnable { + b = b.Owns(&kmmv1beta1.Module{}) + } + // Only watch DRA pods when DRA is enabled in the cluster, to avoid unnecessary // pod list/watch permissions and reconcile noise when DRA is not in use. if opts.DRAEnable { diff --git a/internal/controller/controller_utils.go b/internal/controller/controller_utils.go index 76839c1..b9c89d0 100644 --- a/internal/controller/controller_utils.go +++ b/internal/controller/controller_utils.go @@ -17,11 +17,16 @@ limitations under the License. package controller import ( + "context" + v1alpha "github.com/intel/gpu-base-operator/api/v1alpha1" core "k8s.io/api/core/v1" + resourcev1 "k8s.io/api/resource/v1" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" ) -func generateNodeSelector(cp *v1alpha.ClusterPolicy) map[string]string { +func generateNodeSelector(cp *v1alpha.ClusterPolicy, opts ControllerOpts) map[string]string { ns := map[string]string{ "kubernetes.io/arch": "amd64", } @@ -36,6 +41,10 @@ func generateNodeSelector(cp *v1alpha.ClusterPolicy) map[string]string { ns["intel.feature.node.kubernetes.io/gpu"] = trueValue } + if opts.KMMModuleReadyLabel != "" && cp.Spec.KernelModule != nil { + ns[opts.KMMModuleReadyLabel] = "" + } + return ns } @@ -102,6 +111,31 @@ func shouldRemoveXpumd(cp *v1alpha.ClusterPolicy) bool { return false } +func anyAllocatedResourceClaims(ctx context.Context, c client.Client, driverName string) bool { + var rcList resourcev1.ResourceClaimList + + if err := c.List(ctx, &rcList); err != nil { + klog.Error(err, "unable to list ResourceClaims, assuming allocated claims exist") + return true + } + + for _, claim := range rcList.Items { + alloc := claim.Status.Allocation + if alloc == nil || len(alloc.Devices.Results) == 0 { + continue + } + + for _, dev := range alloc.Devices.Results { + if dev.Driver == driverName { + klog.Infof("Found allocated ResourceClaim with GPU device: %s", claim.Name) + return true + } + } + } + + return false +} + // Convert the integer based log level to a string based log level for the OTel config. func logLevelForXpum(cp *v1alpha.ClusterPolicy) string { v := cp.Spec.XpuManagerSpec.LogLevel diff --git a/internal/controller/deviceplugin_controller.go b/internal/controller/deviceplugin_controller.go index 7f850e9..4d0e2aa 100644 --- a/internal/controller/deviceplugin_controller.go +++ b/internal/controller/deviceplugin_controller.go @@ -167,7 +167,7 @@ func (r *DevicePluginReconciler) updateDaemonSetObject(ds *apps.DaemonSet, spec ds.Spec.Template.Spec.Containers[0].Args = dpArgs(spec) - ds.Spec.Template.Spec.NodeSelector = generateNodeSelector(spec) + ds.Spec.Template.Spec.NodeSelector = generateNodeSelector(spec, r.Opts) ds.Spec.Template.Spec.Tolerations = generateTolerations(spec) cspec := &ds.Spec.Template.Spec diff --git a/internal/controller/dra_controller.go b/internal/controller/dra_controller.go index 310b564..180cdf0 100644 --- a/internal/controller/dra_controller.go +++ b/internal/controller/dra_controller.go @@ -248,40 +248,6 @@ func (r *DRAReconciler) createOpenShiftResourcesIfNotExists(ctx context.Context, return nil } -func (r *DRAReconciler) anyAllocatedResourceClaims(ctx context.Context) bool { - var rcList resv1.ResourceClaimList - - klog.Info("Checking for allocated ResourceClaims that would prevent DRA removal") - - if err := r.List(ctx, &rcList); err != nil { - klog.Error(err, "unable to list ResourceClaims") - - return false - } - - klog.Infof("Found %d ResourceClaims", len(rcList.Items)) - for _, claim := range rcList.Items { - alloc := claim.Status.Allocation - - if alloc == nil { - continue - } - if len(alloc.Devices.Results) == 0 { - continue - } - - for _, dev := range alloc.Devices.Results { - if dev.Driver == gpuDeviceClass { - klog.Infof("Found allocated ResourceClaim with GPU device: %s", claim.Name) - - return true - } - } - } - - return false -} - func addHealthCheckIfMissing(container *core.Container, port int32) { for _, p := range container.Ports { if p.ContainerPort == port { @@ -358,7 +324,7 @@ func (r *DRAReconciler) updateDaemonSetObject(ds *apps.DaemonSet, spec *v1alpha. ds.Spec.Template.Spec.Containers[0].Image = dspec.Image ds.Spec.Template.Spec.Containers[0].Args = r.generateArgs(spec) - ds.Spec.Template.Spec.NodeSelector = generateNodeSelector(spec) + ds.Spec.Template.Spec.NodeSelector = generateNodeSelector(spec, r.Opts) ds.Spec.Template.Spec.Tolerations = generateTolerations(spec) cspec := &ds.Spec.Template.Spec @@ -424,7 +390,7 @@ func (r *DRAReconciler) removeDeploymentIfExists(ctx context.Context, cp *v1alph // If there are any allocated ResourceClaims, removal of DRA will cause // the Pods using them to be stuck at Terminating. // Requeue and try again later. - if r.anyAllocatedResourceClaims(ctx) { + if anyAllocatedResourceClaims(ctx, r.Client, gpuDeviceClass) { return ctrl.Result{RequeueAfter: r.Opts.RequeueDelay}, requeueReconcileErr{} } diff --git a/internal/controller/kmm_controller.go b/internal/controller/kmm_controller.go new file mode 100644 index 0000000..8195242 --- /dev/null +++ b/internal/controller/kmm_controller.go @@ -0,0 +1,313 @@ +/* +Copyright 2026 Intel Corporation. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + v1alpha "github.com/intel/gpu-base-operator/api/v1alpha1" + kmmv1beta1 "github.com/kubernetes-sigs/kernel-module-management/api/v1beta1" +) + +// +kubebuilder:rbac:groups=kmm.sigs.x-k8s.io,resources=modules,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=kmm.sigs.x-k8s.io,resources=modules/status,verbs=get + +// KMMReconciler manages a KMM Module CR for out-of-tree kernel module loading. +// It only configures the moduleLoader section — DP/DRA lifecycle remains with the native controllers. +type KMMReconciler struct { + client.Client + Scheme *runtime.Scheme + Opts ControllerOpts +} + +const ( + kmmModuleSuffix = "-gpu" + + kmmNotEnabledMsg = "KMM is not installed in the cluster." +) + +func kmmModuleName(cpName string) string { + return cpName + kmmModuleSuffix +} + +func (r *KMMReconciler) Reconcile(ctx context.Context, cp *v1alpha.ClusterPolicy) (ctrl.Result, error) { + moduleName := kmmModuleName(r.Opts.ReqName) + + if !r.Opts.KMMEnable { + if cp != nil && cp.Spec.KernelModule != nil { + addIfMissing(&cp.Status.Errors, kmmNotEnabledMsg) + } + + return ctrl.Result{}, nil + } + + if cp == nil || cp.Spec.KernelModule == nil { + return r.deleteModuleIfExists(ctx, cp, moduleName) + } + + // KMM's Module webhook forbids toggling container.version between empty and + // non-empty in place, so recreate the Module when the user does so. + if res, done, err := r.recreateOnVersionToggle(ctx, cp, moduleName); done || err != nil { + return res, err + } + + mod := &kmmv1beta1.Module{ + ObjectMeta: metav1.ObjectMeta{ + Name: moduleName, + Namespace: r.Opts.Namespace, + }, + } + + result, err := controllerutil.CreateOrPatch(ctx, r.Client, mod, func() error { + return r.setModuleDesiredState(mod, cp) + }) + + r.updateStatus(cp, mod) + + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to reconcile KMM Module %s for ClusterPolicy %s: %w", moduleName, cp.Name, err) + } + + klog.Infof("KMM Module %s %s", moduleName, result) + + // Setting Version opts into KMM's ordered upgrade, where KMM loads the module + // only onto nodes carrying a matching version-module label. Flag this so admins + // aren't left wondering why the driver silently isn't loading (see + // KernelModuleSpec.Version). + if cp.Spec.KernelModule.Version != "" { + klog.Infof("KMM Module %s uses ordered upgrade (version=%q); nodes must be labeled "+ + "kmm.node.kubernetes.io/version-module.%s.%s=%s before the module loads", + moduleName, cp.Spec.KernelModule.Version, r.Opts.Namespace, moduleName, cp.Spec.KernelModule.Version) + } + + return ctrl.Result{}, nil +} + +// recreateOnVersionToggle deletes an existing Module when its +// moduleLoader.container.version is being toggled to or from empty, which KMM's +// webhook rejects as an in-place update. It returns done=true when the caller +// should stop and requeue; the next reconcile recreates the Module with the new +// version once the old one is gone. +func (r *KMMReconciler) recreateOnVersionToggle(ctx context.Context, cp *v1alpha.ClusterPolicy, name string) (ctrl.Result, bool, error) { + existing := &kmmv1beta1.Module{} + key := types.NamespacedName{Name: name, Namespace: r.Opts.Namespace} + if err := r.Get(ctx, key, existing); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, false, nil + } + return ctrl.Result{}, false, fmt.Errorf("failed to get KMM Module %s: %w", name, err) + } + + var currentVersion string + if existing.Spec.ModuleLoader != nil { + currentVersion = existing.Spec.ModuleLoader.Container.Version + } + + // Same emptiness (both set or both empty) means an in-place patch is allowed. + if (currentVersion == "") == (cp.Spec.KernelModule.Version == "") { + return ctrl.Result{}, false, nil + } + + if !existing.DeletionTimestamp.IsZero() { + // Deletion already underway; wait for it to finish before recreating. + cp.Status.KMMStatus = "Recreating" + return ctrl.Result{RequeueAfter: r.Opts.RequeueDelay}, true, nil + } + + // Recreating unloads the driver, so don't yank it out from under in-use GPUs. + if r.Opts.DRAEnable && anyAllocatedResourceClaims(ctx, r.Client, gpuDeviceClass) { + addIfMissing(&cp.Status.Errors, + fmt.Sprintf("allocated GPU ResourceClaims blocking modules.kmm.sigs.x-k8s.io/%s recreation for version change", + name)) + return ctrl.Result{RequeueAfter: r.Opts.RequeueDelay}, true, requeueReconcileErr{} + } + + klog.Infof("KMM Module %s version toggled (%q -> %q); recreating", name, currentVersion, cp.Spec.KernelModule.Version) + if err := r.Delete(ctx, existing); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, false, fmt.Errorf("failed to delete KMM Module %s for version change: %w", name, err) + } + + cp.Status.KMMStatus = "Recreating" + return ctrl.Result{RequeueAfter: r.Opts.RequeueDelay}, true, nil +} + +func (r *KMMReconciler) setModuleDesiredState(mod *kmmv1beta1.Module, cp *v1alpha.ClusterPolicy) error { + if err := ctrl.SetControllerReference(cp, mod, r.Scheme); err != nil { + return fmt.Errorf("failed to set controller reference: %w", err) + } + + mod.Spec.Selector = generateNodeSelector(cp, r.Opts) + mod.Spec.Tolerations = generateTolerations(cp) + mod.Spec.ImageRepoSecret = cp.Spec.PullSecret + + r.setModuleLoader(mod, cp) + + return nil +} + +func (r *KMMReconciler) setModuleLoader(mod *kmmv1beta1.Module, cp *v1alpha.ClusterPolicy) { + km := cp.Spec.KernelModule + + container := kmmv1beta1.ModuleLoaderContainerSpec{ + Modprobe: kmmv1beta1.ModprobeSpec{ + ModuleName: km.ModuleName, + FirmwarePath: km.FirmwarePath, + ModulesLoadingOrder: km.ModulesLoadingOrder, + }, + Version: km.Version, + InTreeModulesToRemove: []string{km.ModuleName}, + ImagePullPolicy: v1.PullIfNotPresent, + } + + if km.RegistryTLS != nil { + container.RegistryTLS = kmmv1beta1.TLSOptions{ + Insecure: km.RegistryTLS.Insecure, + InsecureSkipTLSVerify: km.RegistryTLS.InsecureSkipTLSVerify, + } + } + + mappings := make([]kmmv1beta1.KernelMapping, 0, len(km.KernelMappings)) + for _, m := range km.KernelMappings { + mapping := kmmv1beta1.KernelMapping{ + Regexp: m.Regexp, + ContainerImage: m.ContainerImage, + } + if len(m.InTreeModulesToRemove) > 0 { + mapping.InTreeModulesToRemove = dedupeStrings( + append([]string{km.ModuleName}, m.InTreeModulesToRemove...)) + } + if m.RegistryTLS != nil { + mapping.RegistryTLS = &kmmv1beta1.TLSOptions{ + Insecure: m.RegistryTLS.Insecure, + InsecureSkipTLSVerify: m.RegistryTLS.InsecureSkipTLSVerify, + } + } + if m.Build != nil { + mapping.Build = convertBuildSpec(m.Build, mod.Spec.Selector) + } + mappings = append(mappings, mapping) + } + container.KernelMappings = mappings + + mod.Spec.ModuleLoader = &kmmv1beta1.ModuleLoaderSpec{ + Container: container, + ServiceAccountName: r.Opts.ModuleLoaderServiceAccountName, + } +} + +func convertBuildSpec(src *v1alpha.KernelModuleBuildSpec, selector map[string]string) *kmmv1beta1.Build { + build := &kmmv1beta1.Build{ + DockerfileConfigMap: &v1.LocalObjectReference{Name: src.DockerfileConfigMap.Name}, + Secrets: append([]v1.LocalObjectReference{}, src.Secrets...), + // Build on the same nodes the module targets: in-cluster driver builds + // need the GPU nodes' kernel headers/toolchain to compile against. + Selector: selector, + } + + if len(src.BuildArgs) > 0 { + args := make([]kmmv1beta1.BuildArg, len(src.BuildArgs)) + for i, a := range src.BuildArgs { + args[i] = kmmv1beta1.BuildArg{Name: a.Name, Value: a.Value} + } + build.BuildArgs = args + } + + return build +} + +func dedupeStrings(s []string) []string { + seen := make(map[string]bool, len(s)) + result := make([]string, 0, len(s)) + + for _, v := range s { + if !seen[v] { + seen[v] = true + result = append(result, v) + } + } + + return result +} + +func (r *KMMReconciler) deleteModuleIfExists(ctx context.Context, cp *v1alpha.ClusterPolicy, name string) (ctrl.Result, error) { + mod := &kmmv1beta1.Module{} + key := types.NamespacedName{Name: name, Namespace: r.Opts.Namespace} + + if err := r.Get(ctx, key, mod); err != nil { + if apierrors.IsNotFound(err) { + if cp != nil { + cp.Status.KMMStatus = notAvailableStatus + } + return ctrl.Result{}, nil + } + + return ctrl.Result{}, fmt.Errorf("failed to get KMM Module %s: %w", name, err) + } + + if r.Opts.DRAEnable && anyAllocatedResourceClaims(ctx, r.Client, gpuDeviceClass) { + if cp != nil { + addIfMissing(&cp.Status.Errors, + fmt.Sprintf("allocated GPU ResourceClaims blocking modules.kmm.sigs.x-k8s.io/%s deletion", + name)) + } + return ctrl.Result{RequeueAfter: r.Opts.RequeueDelay}, requeueReconcileErr{} + } + + if !mod.DeletionTimestamp.IsZero() { + if cp != nil { + cp.Status.KMMStatus = "Removing" + addIfMissing(&cp.Status.Errors, + fmt.Sprintf("modules.kmm.sigs.x-k8s.io/%s is pending deletion", + name)) + } + return ctrl.Result{RequeueAfter: r.Opts.RequeueDelay}, nil + } + + klog.Infof("Deleting KMM Module %s", name) + + if err := r.Delete(ctx, mod); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, fmt.Errorf("failed to delete KMM Module %s: %w", name, err) + } + + return ctrl.Result{}, nil +} + +func (r *KMMReconciler) updateStatus(cp *v1alpha.ClusterPolicy, mod *kmmv1beta1.Module) { + if cp.Spec.KernelModule != nil { + mlStatus := mod.Status.ModuleLoader + cp.Status.KMMStatus = fmt.Sprintf("%d/%d", mlStatus.AvailableNumber, mlStatus.DesiredNumber) + + if mlStatus.DesiredNumber > 0 && mlStatus.AvailableNumber < mlStatus.DesiredNumber { + addIfMissing(&cp.Status.Errors, + fmt.Sprintf("module loader not fully available (%d/%d) for modules.kmm.sigs.x-k8s.io/%s", + mlStatus.AvailableNumber, mlStatus.DesiredNumber, mod.Name)) + } + } else { + cp.Status.KMMStatus = notAvailableStatus + } +} diff --git a/internal/controller/kmm_controller_test.go b/internal/controller/kmm_controller_test.go new file mode 100644 index 0000000..dee0a5c --- /dev/null +++ b/internal/controller/kmm_controller_test.go @@ -0,0 +1,1273 @@ +/* +Copyright 2026 Intel Corporation. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + stderrors "errors" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + resourcev1 "k8s.io/api/resource/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + v1alpha "github.com/intel/gpu-base-operator/api/v1alpha1" + kmmv1beta1 "github.com/kubernetes-sigs/kernel-module-management/api/v1beta1" +) + +var _ = Describe("KMM Controller", func() { + Context("When creating a Module with KernelModule spec", func() { + const ( + namespace = "kmm-oot-create" + resourceName = "kmm-oot-create" + ) + + ctx := context.Background() + typeNamespacedName := types.NamespacedName{Name: resourceName} + + BeforeEach(func() { + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: namespace}, + })).To(Succeed()) + }) + + AfterEach(func() { + resource := &v1alpha.ClusterPolicy{} + if err := k8sClient.Get(ctx, typeNamespacedName, resource); err == nil { + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + } + }) + + It("should create a KMM Module CR with moduleLoader", func() { + cp := &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName}, + Spec: v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + } + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := &ClusterPolicyReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + Namespace: namespace, + KMMEnable: true, + ModuleLoaderServiceAccountName: "intel-gpu-module-loader", + }, + } + + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + + By("verifying KMM Module was created") + mod := &kmmv1beta1.Module{} + modKey := types.NamespacedName{Name: resourceName + kmmModuleSuffix, Namespace: namespace} + Expect(k8sClient.Get(ctx, modKey, mod)).To(Succeed()) + + By("verifying ModuleLoader is set with correct modprobe config") + Expect(mod.Spec.ModuleLoader).NotTo(BeNil()) + Expect(mod.Spec.ModuleLoader.Container.Modprobe.ModuleName).To(Equal("xe")) + Expect(mod.Spec.ModuleLoader.Container.ContainerImage).To(BeEmpty()) + Expect(mod.Spec.ModuleLoader.Container.InTreeModulesToRemove).To(Equal([]string{"xe"})) + Expect(mod.Spec.ModuleLoader.Container.KernelMappings).To(HaveLen(1)) + Expect(mod.Spec.ModuleLoader.Container.KernelMappings[0].Regexp).To(Equal("^.+$")) + Expect(mod.Spec.ModuleLoader.Container.KernelMappings[0].ContainerImage).To(Equal("registry.example.com/xe-driver:1.0")) + + By("verifying no DRA or DevicePlugin specs are set") + Expect(mod.Spec.DRA).To(BeNil()) + Expect(mod.Spec.DevicePlugin).To(BeNil()) + + By("verifying owner reference is set") + Expect(mod.OwnerReferences).To(HaveLen(1)) + Expect(mod.OwnerReferences[0].Name).To(Equal(resourceName)) + }) + }) + + Context("When KernelModule is nil", func() { + const ( + namespace = "kmm-no-km" + resourceName = "kmm-no-km" + ) + + ctx := context.Background() + typeNamespacedName := types.NamespacedName{Name: resourceName} + + BeforeEach(func() { + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: namespace}, + })).To(Succeed()) + }) + + AfterEach(func() { + resource := &v1alpha.ClusterPolicy{} + if err := k8sClient.Get(ctx, typeNamespacedName, resource); err == nil { + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + } + }) + + It("should not create a Module CR", func() { + cp := &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName}, + Spec: v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + } + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := &ClusterPolicyReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + Namespace: namespace, + KMMEnable: true, + ModuleLoaderServiceAccountName: "intel-gpu-module-loader", + }, + } + + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + + mod := &kmmv1beta1.Module{} + modKey := types.NamespacedName{Name: resourceName + kmmModuleSuffix, Namespace: namespace} + err = k8sClient.Get(ctx, modKey, mod) + Expect(errors.IsNotFound(err)).To(BeTrue()) + }) + }) + + Context("When KMM is not enabled", func() { + const ( + namespace = "kmm-not-enabled" + resourceName = "kmm-not-enabled" + ) + + ctx := context.Background() + typeNamespacedName := types.NamespacedName{Name: resourceName} + + BeforeEach(func() { + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: namespace}, + })).To(Succeed()) + }) + + AfterEach(func() { + resource := &v1alpha.ClusterPolicy{} + if err := k8sClient.Get(ctx, typeNamespacedName, resource); err == nil { + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + } + }) + + It("should set kmmNotEnabledMsg error and not create a Module", func() { + cp := &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName}, + Spec: v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + } + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := &ClusterPolicyReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + Namespace: namespace, + KMMEnable: false, + ModuleLoaderServiceAccountName: "intel-gpu-module-loader", + }, + } + + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + + By("verifying status has kmmNotEnabledMsg") + Expect(k8sClient.Get(ctx, typeNamespacedName, cp)).To(Succeed()) + Expect(cp.Status.Errors).To(ContainElement(kmmNotEnabledMsg)) + + By("verifying no Module was created") + mod := &kmmv1beta1.Module{} + modKey := types.NamespacedName{Name: resourceName + kmmModuleSuffix, Namespace: namespace} + err = k8sClient.Get(ctx, modKey, mod) + Expect(errors.IsNotFound(err)).To(BeTrue()) + }) + }) + + Context("When a status error condition resolves", func() { + const ( + namespace = "kmm-clear-err" + resourceName = "kmm-clear-err" + ) + + ctx := context.Background() + typeNamespacedName := types.NamespacedName{Name: resourceName} + + BeforeEach(func() { + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: namespace}, + })).To(Succeed()) + }) + + AfterEach(func() { + resource := &v1alpha.ClusterPolicy{} + if err := k8sClient.Get(ctx, typeNamespacedName, resource); err == nil { + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + } + }) + + It("clears a stale error on the next reconcile once the condition is gone", func() { + cp := &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName}, + Spec: v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + } + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + // KMM disabled + KernelModule set => kmmNotEnabledMsg is recorded. + reconciler := &ClusterPolicyReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + Namespace: namespace, + KMMEnable: false, + ModuleLoaderServiceAccountName: "intel-gpu-module-loader", + }, + } + + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + + By("verifying the error is present") + Expect(k8sClient.Get(ctx, typeNamespacedName, cp)).To(Succeed()) + Expect(cp.Status.Errors).To(ContainElement(kmmNotEnabledMsg)) + + By("removing KernelModule so the condition no longer applies") + cp.Spec.KernelModule = nil + Expect(k8sClient.Update(ctx, cp)).To(Succeed()) + + _, err = reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + + By("verifying the stale error was cleared") + Expect(k8sClient.Get(ctx, typeNamespacedName, cp)).To(Succeed()) + Expect(cp.Status.Errors).NotTo(ContainElement(kmmNotEnabledMsg)) + }) + }) + + Context("When KernelModule is removed from ClusterPolicy", func() { + const ( + namespace = "kmm-remove-km" + resourceName = "kmm-remove-km" + ) + + ctx := context.Background() + typeNamespacedName := types.NamespacedName{Name: resourceName} + + BeforeEach(func() { + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: namespace}, + })).To(Succeed()) + }) + + AfterEach(func() { + resource := &v1alpha.ClusterPolicy{} + if err := k8sClient.Get(ctx, typeNamespacedName, resource); err == nil { + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + } + }) + + It("should delete the Module when KernelModule is nilled out", func() { + cp := &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName}, + Spec: v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + } + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := &ClusterPolicyReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + Namespace: namespace, + KMMEnable: true, + ModuleLoaderServiceAccountName: "intel-gpu-module-loader", + }, + } + + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + + modKey := types.NamespacedName{Name: resourceName + kmmModuleSuffix, Namespace: namespace} + Expect(k8sClient.Get(ctx, modKey, &kmmv1beta1.Module{})).To(Succeed()) + + By("removing KernelModule from ClusterPolicy") + Expect(k8sClient.Get(ctx, typeNamespacedName, cp)).To(Succeed()) + cp.Spec.KernelModule = nil + Expect(k8sClient.Update(ctx, cp)).To(Succeed()) + + _, err = reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + + By("verifying Module was deleted") + err = k8sClient.Get(ctx, modKey, &kmmv1beta1.Module{}) + Expect(errors.IsNotFound(err)).To(BeTrue()) + }) + }) + + Context("When ClusterPolicy is deleted", func() { + const ( + namespace = "kmm-del-cp" + resourceName = "kmm-del-cp" + ) + + ctx := context.Background() + typeNamespacedName := types.NamespacedName{Name: resourceName} + + BeforeEach(func() { + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: namespace}, + })).To(Succeed()) + }) + + AfterEach(func() { + resource := &v1alpha.ClusterPolicy{} + if err := k8sClient.Get(ctx, typeNamespacedName, resource); err == nil { + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + } + }) + + It("should delete Module when ClusterPolicy is deleted", func() { + cp := &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName}, + Spec: v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + } + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := &ClusterPolicyReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + Namespace: namespace, + KMMEnable: true, + ModuleLoaderServiceAccountName: "intel-gpu-module-loader", + }, + } + + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + + modKey := types.NamespacedName{Name: resourceName + kmmModuleSuffix, Namespace: namespace} + Expect(k8sClient.Get(ctx, modKey, &kmmv1beta1.Module{})).To(Succeed()) + + By("deleting the ClusterPolicy") + Expect(k8sClient.Delete(ctx, cp)).To(Succeed()) + + By("reconciling the deletion via KMM reconciler directly") + kmmReconciler := &KMMReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + ReqName: resourceName, + Namespace: namespace, + KMMEnable: true, + }, + } + _, err = kmmReconciler.Reconcile(ctx, nil) + Expect(err).NotTo(HaveOccurred()) + + By("verifying Module was deleted") + err = k8sClient.Get(ctx, modKey, &kmmv1beta1.Module{}) + Expect(errors.IsNotFound(err)).To(BeTrue()) + }) + }) + + Context("When ClusterPolicy has tolerations and pull secret", func() { + const ( + namespace = "kmm-opts-create" + resourceName = "kmm-opts-create" + ) + + ctx := context.Background() + typeNamespacedName := types.NamespacedName{Name: resourceName} + + BeforeEach(func() { + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: namespace}, + })).To(Succeed()) + }) + + AfterEach(func() { + resource := &v1alpha.ClusterPolicy{} + if err := k8sClient.Get(ctx, typeNamespacedName, resource); err == nil { + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + } + }) + + It("should propagate tolerations and pull secret to Module", func() { + cp := &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName}, + Spec: v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + Tolerations: []v1.Toleration{ + { + Key: "gpu-dedicated", + Operator: v1.TolerationOpExists, + Effect: v1.TaintEffectNoSchedule, + }, + }, + PullSecret: &v1.LocalObjectReference{ + Name: "my-registry-secret", + }, + }, + } + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := &ClusterPolicyReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + Namespace: namespace, + KMMEnable: true, + ModuleLoaderServiceAccountName: "intel-gpu-module-loader", + }, + } + + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + + mod := &kmmv1beta1.Module{} + modKey := types.NamespacedName{Name: resourceName + kmmModuleSuffix, Namespace: namespace} + Expect(k8sClient.Get(ctx, modKey, mod)).To(Succeed()) + + By("verifying tolerations") + Expect(mod.Spec.Tolerations).To(HaveLen(1)) + Expect(mod.Spec.Tolerations[0].Key).To(Equal("gpu-dedicated")) + + By("verifying pull secret") + Expect(mod.Spec.ImageRepoSecret).NotTo(BeNil()) + Expect(mod.Spec.ImageRepoSecret.Name).To(Equal("my-registry-secret")) + }) + }) + + DescribeTable("expanded KernelModule fields", func( + ns string, + cpSpec v1alpha.ClusterPolicySpec, + assertFn func(mod *kmmv1beta1.Module, cp *v1alpha.ClusterPolicy), + ) { + ctx := context.Background() + resName := ns + + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: ns}, + })).To(Succeed()) + + cp := &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: resName}, + Spec: cpSpec, + } + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := &ClusterPolicyReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + Namespace: ns, + KMMEnable: true, + ModuleLoaderServiceAccountName: "intel-gpu-module-loader", + }, + } + _, err := reconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{Name: resName}, + }) + Expect(err).NotTo(HaveOccurred()) + + mod := &kmmv1beta1.Module{} + modKey := types.NamespacedName{Name: resName + kmmModuleSuffix, Namespace: ns} + Expect(k8sClient.Get(ctx, modKey, mod)).To(Succeed()) + + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: resName}, cp)).To(Succeed()) + + assertFn(mod, cp) + }, + Entry("should configure ModuleLoader with multiple kernel mappings", + "kmm-exp-multi", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^5\\.14\\.0-.*\\.el9", ContainerImage: "registry.example.com/xe-rhel9:1.0"}, + {Regexp: "^6\\.12\\..*", ContainerImage: "registry.example.com/xe-rhel10:1.0"}, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + Expect(mod.Spec.ModuleLoader.Container.KernelMappings).To(HaveLen(2)) + Expect(mod.Spec.ModuleLoader.Container.KernelMappings[0].Regexp).To(Equal("^5\\.14\\.0-.*\\.el9")) + Expect(mod.Spec.ModuleLoader.Container.KernelMappings[0].ContainerImage).To(Equal("registry.example.com/xe-rhel9:1.0")) + Expect(mod.Spec.ModuleLoader.Container.KernelMappings[1].Regexp).To(Equal("^6\\.12\\..*")) + Expect(mod.Spec.ModuleLoader.Container.KernelMappings[1].ContainerImage).To(Equal("registry.example.com/xe-rhel10:1.0")) + Expect(mod.Spec.ModuleLoader.Container.ContainerImage).To(BeEmpty()) + }, + ), + Entry("should set Version on container spec", + "kmm-exp-version", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + Version: "2.0", + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:2.0"}, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + Expect(mod.Spec.ModuleLoader.Container.Version).To(Equal("2.0")) + }, + ), + Entry("should configure Build on kernel mapping", + "kmm-exp-build", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + { + Regexp: "^5\\.14\\..*", + Build: &v1alpha.KernelModuleBuildSpec{ + DockerfileConfigMap: v1.LocalObjectReference{Name: "xe-dockerfile"}, + BuildArgs: []v1alpha.BuildArg{{Name: "XE_TAG", Value: "v1.0"}}, + Secrets: []v1.LocalObjectReference{{Name: "private-repo"}}, + }, + }, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + kmmBuild := mod.Spec.ModuleLoader.Container.KernelMappings[0].Build + Expect(kmmBuild).NotTo(BeNil()) + Expect(kmmBuild.DockerfileConfigMap).NotTo(BeNil()) + Expect(kmmBuild.DockerfileConfigMap.Name).To(Equal("xe-dockerfile")) + Expect(kmmBuild.BuildArgs).To(HaveLen(1)) + Expect(kmmBuild.BuildArgs[0].Name).To(Equal("XE_TAG")) + Expect(kmmBuild.BuildArgs[0].Value).To(Equal("v1.0")) + Expect(kmmBuild.Secrets).To(HaveLen(1)) + Expect(kmmBuild.Secrets[0].Name).To(Equal("private-repo")) + // The build must run on the same nodes the module targets. + Expect(kmmBuild.Selector).To(Equal(mod.Spec.Selector)) + }, + ), + Entry("should propagate FirmwarePath to ModprobeSpec", + "kmm-exp-firmware", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + FirmwarePath: "/opt/lib/firmware/xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + Expect(mod.Spec.ModuleLoader.Container.Modprobe.FirmwarePath).To(Equal("/opt/lib/firmware/xe")) + }, + ), + Entry("should propagate ModulesLoadingOrder to ModprobeSpec", + "kmm-exp-loadorder", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + ModulesLoadingOrder: []string{"xe", "drm_buddy", "drm_ttm_helper"}, + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + Expect(mod.Spec.ModuleLoader.Container.Modprobe.ModulesLoadingOrder).To(Equal([]string{"xe", "drm_buddy", "drm_ttm_helper"})) + }, + ), + Entry("should map RegistryTLS to container-level TLSOptions", + "kmm-exp-tls", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + RegistryTLS: &v1alpha.RegistryTLSSpec{Insecure: true, InsecureSkipTLSVerify: true}, + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + Expect(mod.Spec.ModuleLoader.Container.RegistryTLS.Insecure).To(BeTrue()) + Expect(mod.Spec.ModuleLoader.Container.RegistryTLS.InsecureSkipTLSVerify).To(BeTrue()) + }, + ), + Entry("should map per-mapping RegistryTLS to KernelMapping TLSOptions", + "kmm-exp-tls-map", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + { + Regexp: "^.+$", + ContainerImage: "registry.example.com/xe-driver:1.0", + RegistryTLS: &v1alpha.RegistryTLSSpec{InsecureSkipTLSVerify: true}, + }, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + Expect(mod.Spec.ModuleLoader.Container.KernelMappings[0].RegistryTLS).NotTo(BeNil()) + Expect(mod.Spec.ModuleLoader.Container.KernelMappings[0].RegistryTLS.InsecureSkipTLSVerify).To(BeTrue()) + }, + ), + Entry("should set container-level InTreeModulesToRemove to [ModuleName]", + "kmm-exp-intree", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + Expect(mod.Spec.ModuleLoader.Container.InTreeModulesToRemove).To(Equal([]string{"xe"})) + Expect(mod.Spec.ModuleLoader.Container.KernelMappings[0].InTreeModulesToRemove).To(BeEmpty()) + }, + ), + Entry("should auto-prepend ModuleName to per-mapping InTreeModulesToRemove", + "kmm-exp-dedup", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + { + Regexp: "^.+$", + ContainerImage: "registry.example.com/xe-driver:1.0", + InTreeModulesToRemove: []string{"i915"}, + }, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + Expect(mod.Spec.ModuleLoader.Container.KernelMappings[0].InTreeModulesToRemove).To(Equal([]string{"xe", "i915"})) + }, + ), + Entry("should set per-mapping InTreeModulesToRemove override", + "kmm-exp-permapping", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + { + Regexp: "^5\\.14\\..*", + ContainerImage: "registry.example.com/xe:1.0", + InTreeModulesToRemove: []string{"old_xe"}, + }, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + Expect(mod.Spec.ModuleLoader.Container.KernelMappings[0].InTreeModulesToRemove).To(Equal([]string{"xe", "old_xe"})) + Expect(mod.Spec.ModuleLoader.Container.InTreeModulesToRemove).To(Equal([]string{"xe"})) + }, + ), + ) + + Context("KMM ready-label gating", func() { + It("should not add KMM ready label to the Module selector", func() { + ns := "kmm-label-gate" + resName := ns + ctx := context.Background() + + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: ns}, + })).To(Succeed()) + + cp := &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: resName}, + Spec: v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + } + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := &ClusterPolicyReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + Namespace: ns, + KMMEnable: true, + ModuleLoaderServiceAccountName: "intel-gpu-module-loader", + }, + } + + _, err := reconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{Name: resName}, + }) + Expect(err).NotTo(HaveOccurred()) + + mod := &kmmv1beta1.Module{} + modKey := types.NamespacedName{Name: resName + kmmModuleSuffix, Namespace: ns} + Expect(k8sClient.Get(ctx, modKey, mod)).To(Succeed()) + + readyLabel := "kmm.node.kubernetes.io/" + ns + "." + resName + kmmModuleSuffix + ".ready" + Expect(mod.Spec.Selector).NotTo(HaveKey(readyLabel), + "KMM Module selector must not gate on its own ready label") + }) + + It("should add KMM ready label to generateNodeSelector when KernelModule is set", func() { + readyLabel := "kmm.node.kubernetes.io/test-ns.test-cp-gpu.ready" + cp := &v1alpha.ClusterPolicy{ + Spec: v1alpha.ClusterPolicySpec{ + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe:1.0"}, + }, + }, + }, + } + opts := ControllerOpts{KMMModuleReadyLabel: readyLabel} + ns := generateNodeSelector(cp, opts) + Expect(ns).To(HaveKeyWithValue(readyLabel, "")) + }) + + It("should not add KMM ready label when KernelModule is nil", func() { + readyLabel := "kmm.node.kubernetes.io/test-ns.test-cp-gpu.ready" + cp := &v1alpha.ClusterPolicy{} + opts := ControllerOpts{KMMModuleReadyLabel: readyLabel} + ns := generateNodeSelector(cp, opts) + Expect(ns).NotTo(HaveKey(readyLabel)) + }) + + It("should not add KMM ready label when KMMModuleReadyLabel is empty", func() { + cp := &v1alpha.ClusterPolicy{ + Spec: v1alpha.ClusterPolicySpec{ + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe:1.0"}, + }, + }, + }, + } + opts := ControllerOpts{} + ns := generateNodeSelector(cp, opts) + for k := range ns { + Expect(k).NotTo(ContainSubstring("kmm.node.kubernetes.io")) + } + }) + }) + + Context("When toggling container.version between empty and non-empty", func() { + newReconciler := func(ns string) *KMMReconciler { + return &KMMReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + ReqName: ns, + Namespace: ns, + KMMEnable: true, + RequeueDelay: time.Millisecond, + ModuleLoaderServiceAccountName: "module-loader", + }, + } + } + + cpWithVersion := func(name, version string) *v1alpha.ClusterPolicy { + return &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + Version: version, + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + }, + }, + } + } + + It("recreates the Module when version is added to an existing (empty-version) Module", func() { + ns := "kmm-ver-add" + ctx := context.Background() + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: ns}, + })).To(Succeed()) + + cp := cpWithVersion(ns, "") + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := newReconciler(ns) + modKey := types.NamespacedName{Name: ns + kmmModuleSuffix, Namespace: ns} + + _, err := reconciler.Reconcile(ctx, cp) + Expect(err).NotTo(HaveOccurred()) + + mod := &kmmv1beta1.Module{} + Expect(k8sClient.Get(ctx, modKey, mod)).To(Succeed()) + Expect(mod.Spec.ModuleLoader.Container.Version).To(BeEmpty()) + originalUID := mod.UID + + // Add a version: the reconcile should delete the Module and requeue + // rather than attempt an in-place patch KMM would reject. + cp.Spec.KernelModule.Version = "1.0" + res, err := reconciler.Reconcile(ctx, cp) + Expect(err).NotTo(HaveOccurred()) + Expect(res.RequeueAfter).To(BeNumerically(">", 0)) + Expect(cp.Status.KMMStatus).To(Equal("Recreating")) + Expect(errors.IsNotFound(k8sClient.Get(ctx, modKey, &kmmv1beta1.Module{}))).To(BeTrue()) + + // The next reconcile recreates the Module with the new version. + _, err = reconciler.Reconcile(ctx, cp) + Expect(err).NotTo(HaveOccurred()) + recreated := &kmmv1beta1.Module{} + Expect(k8sClient.Get(ctx, modKey, recreated)).To(Succeed()) + Expect(recreated.Spec.ModuleLoader.Container.Version).To(Equal("1.0")) + Expect(recreated.UID).NotTo(Equal(originalUID)) + }) + + It("recreates the Module when version is removed from an existing (versioned) Module", func() { + ns := "kmm-ver-remove" + ctx := context.Background() + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: ns}, + })).To(Succeed()) + + cp := cpWithVersion(ns, "1.0") + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := newReconciler(ns) + modKey := types.NamespacedName{Name: ns + kmmModuleSuffix, Namespace: ns} + + _, err := reconciler.Reconcile(ctx, cp) + Expect(err).NotTo(HaveOccurred()) + + mod := &kmmv1beta1.Module{} + Expect(k8sClient.Get(ctx, modKey, mod)).To(Succeed()) + Expect(mod.Spec.ModuleLoader.Container.Version).To(Equal("1.0")) + originalUID := mod.UID + + cp.Spec.KernelModule.Version = "" + res, err := reconciler.Reconcile(ctx, cp) + Expect(err).NotTo(HaveOccurred()) + Expect(res.RequeueAfter).To(BeNumerically(">", 0)) + Expect(errors.IsNotFound(k8sClient.Get(ctx, modKey, &kmmv1beta1.Module{}))).To(BeTrue()) + + _, err = reconciler.Reconcile(ctx, cp) + Expect(err).NotTo(HaveOccurred()) + recreated := &kmmv1beta1.Module{} + Expect(k8sClient.Get(ctx, modKey, recreated)).To(Succeed()) + Expect(recreated.Spec.ModuleLoader.Container.Version).To(BeEmpty()) + Expect(recreated.UID).NotTo(Equal(originalUID)) + }) + + It("patches in place (no recreate) when version changes between two non-empty values", func() { + ns := "kmm-ver-bump" + ctx := context.Background() + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: ns}, + })).To(Succeed()) + + cp := cpWithVersion(ns, "1.0") + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := newReconciler(ns) + modKey := types.NamespacedName{Name: ns + kmmModuleSuffix, Namespace: ns} + + _, err := reconciler.Reconcile(ctx, cp) + Expect(err).NotTo(HaveOccurred()) + + mod := &kmmv1beta1.Module{} + Expect(k8sClient.Get(ctx, modKey, mod)).To(Succeed()) + originalUID := mod.UID + + cp.Spec.KernelModule.Version = "2.0" + res, err := reconciler.Reconcile(ctx, cp) + Expect(err).NotTo(HaveOccurred()) + Expect(res.RequeueAfter).To(BeZero()) + + updated := &kmmv1beta1.Module{} + Expect(k8sClient.Get(ctx, modKey, updated)).To(Succeed()) + Expect(updated.Spec.ModuleLoader.Container.Version).To(Equal("2.0")) + Expect(updated.UID).To(Equal(originalUID), "Module should be patched in place, not recreated") + }) + + It("blocks the recreate while GPU ResourceClaims are allocated", func() { + ns := "kmm-ver-inuse" + ctx := context.Background() + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: ns}, + })).To(Succeed()) + + cp := cpWithVersion(ns, "") + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := newReconciler(ns) + reconciler.Opts.DRAEnable = true + modKey := types.NamespacedName{Name: ns + kmmModuleSuffix, Namespace: ns} + + _, err := reconciler.Reconcile(ctx, cp) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient.Get(ctx, modKey, &kmmv1beta1.Module{})).To(Succeed()) + + By("allocating a GPU ResourceClaim") + rc := &resourcev1.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{Name: "test-gpu-claim", Namespace: ns}, + Spec: resourcev1.ResourceClaimSpec{ + Devices: resourcev1.DeviceClaim{ + Requests: []resourcev1.DeviceRequest{ + { + Name: "gpu", + FirstAvailable: []resourcev1.DeviceSubRequest{ + { + Name: "gpu-sub", + DeviceClassName: gpuDeviceClass, + AllocationMode: resourcev1.DeviceAllocationModeExactCount, + Count: 1, + }, + }, + }, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, rc)).To(Succeed()) + DeferCleanup(func() { _ = k8sClient.Delete(ctx, rc) }) + + rc.Status.Allocation = &resourcev1.AllocationResult{ + Devices: resourcev1.DeviceAllocationResult{ + Results: []resourcev1.DeviceRequestAllocationResult{ + {Request: "gpu", Driver: gpuDeviceClass, Pool: "node-pool", Device: "gpu-0"}, + }, + }, + } + Expect(k8sClient.Status().Update(ctx, rc)).To(Succeed()) + + By("toggling version while the claim is allocated: recreate must be blocked") + cp.Spec.KernelModule.Version = "1.0" + res, err := reconciler.Reconcile(ctx, cp) + Expect(stderrors.Is(err, requeueReconcileErr{})).To(BeTrue()) + Expect(res.RequeueAfter).To(BeNumerically(">", 0)) + Expect(cp.Status.Errors).To(ContainElement(ContainSubstring("blocking modules.kmm.sigs.x-k8s.io/" + ns + kmmModuleSuffix))) + Expect(k8sClient.Get(ctx, modKey, &kmmv1beta1.Module{})).To(Succeed(), "Module must not be deleted while GPUs are in use") + + By("freeing the claim: the recreate proceeds") + Expect(k8sClient.Delete(ctx, rc)).To(Succeed()) + _, err = reconciler.Reconcile(ctx, cp) + Expect(err).NotTo(HaveOccurred()) + Expect(errors.IsNotFound(k8sClient.Get(ctx, modKey, &kmmv1beta1.Module{}))).To(BeTrue()) + }) + }) + + Context("When deleting Module with allocated ResourceClaims", func() { + const ( + namespace = "kmm-rc-safety" + resourceName = "kmm-rc-safety" + ) + + ctx := context.Background() + typeNamespacedName := types.NamespacedName{Name: resourceName} + + BeforeEach(func() { + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: namespace}, + })).To(Succeed()) + }) + + AfterEach(func() { + resource := &v1alpha.ClusterPolicy{} + if err := k8sClient.Get(ctx, typeNamespacedName, resource); err == nil { + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + } + + var rcList resourcev1.ResourceClaimList + if err := k8sClient.List(ctx, &rcList); err == nil { + for i := range rcList.Items { + _ = k8sClient.Delete(ctx, &rcList.Items[i]) + } + } + }) + + It("should requeue instead of deleting the Module", func() { + cp := &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName}, + Spec: v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + } + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := &ClusterPolicyReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + Namespace: namespace, + KMMEnable: true, + DRAEnable: true, + RequeueDelay: 5 * time.Second, + ModuleLoaderServiceAccountName: "intel-gpu-module-loader", + }, + } + + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + + modKey := types.NamespacedName{Name: resourceName + kmmModuleSuffix, Namespace: namespace} + Expect(k8sClient.Get(ctx, modKey, &kmmv1beta1.Module{})).To(Succeed()) + + By("creating an allocated ResourceClaim with a GPU device") + rc := &resourcev1.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-gpu-claim", + Namespace: namespace, + }, + Spec: resourcev1.ResourceClaimSpec{ + Devices: resourcev1.DeviceClaim{ + Requests: []resourcev1.DeviceRequest{ + { + Name: "gpu", + FirstAvailable: []resourcev1.DeviceSubRequest{ + { + Name: "gpu-sub", + DeviceClassName: gpuDeviceClass, + AllocationMode: resourcev1.DeviceAllocationModeExactCount, + Count: 1, + }, + }, + }, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, rc)).To(Succeed()) + + rc.Status.Allocation = &resourcev1.AllocationResult{ + Devices: resourcev1.DeviceAllocationResult{ + Results: []resourcev1.DeviceRequestAllocationResult{ + { + Request: "gpu", + Driver: gpuDeviceClass, + Pool: "node-pool", + Device: "gpu-0", + }, + }, + }, + } + Expect(k8sClient.Status().Update(ctx, rc)).To(Succeed()) + + By("removing KernelModule to trigger deletion") + Expect(k8sClient.Get(ctx, typeNamespacedName, cp)).To(Succeed()) + cp.Spec.KernelModule = nil + Expect(k8sClient.Update(ctx, cp)).To(Succeed()) + + result, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.RequeueAfter).To(Equal(5 * time.Second)) + + By("verifying Module was NOT deleted") + Expect(k8sClient.Get(ctx, modKey, &kmmv1beta1.Module{})).To(Succeed()) + + By("removing the ResourceClaim and reconciling again") + Expect(k8sClient.Delete(ctx, rc)).To(Succeed()) + + result, err = reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.RequeueAfter).To(BeZero()) + + By("verifying Module was deleted") + err = k8sClient.Get(ctx, modKey, &kmmv1beta1.Module{}) + Expect(errors.IsNotFound(err)).To(BeTrue()) + }) + }) + + Context("When KernelModule is nil", func() { + const ( + namespace = "kmm-exp-nkmm" + resourceName = "kmm-exp-nkmm" + ) + + ctx := context.Background() + typeNamespacedName := types.NamespacedName{Name: resourceName} + + BeforeEach(func() { + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: namespace}, + })).To(Succeed()) + }) + + AfterEach(func() { + resource := &v1alpha.ClusterPolicy{} + if err := k8sClient.Get(ctx, typeNamespacedName, resource); err == nil { + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + } + }) + + It("should set KMMStatus to N/A", func() { + cp := &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName}, + Spec: v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + } + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := &ClusterPolicyReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + Namespace: namespace, + KMMEnable: true, + ModuleLoaderServiceAccountName: "intel-gpu-module-loader", + }, + } + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + + Expect(k8sClient.Get(ctx, typeNamespacedName, cp)).To(Succeed()) + Expect(cp.Status.KMMStatus).To(Equal("N/A")) + }) + }) +}) diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go index 22daa3c..9bf4bc6 100644 --- a/internal/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -18,8 +18,11 @@ package controller import ( "context" + "fmt" "os" + "os/exec" "path/filepath" + "strings" "testing" . "github.com/onsi/ginkgo/v2" @@ -32,6 +35,7 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + kmmv1beta1 "github.com/kubernetes-sigs/kernel-module-management/api/v1beta1" prometheusv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" corev1 "k8s.io/api/core/v1" resv1 "k8s.io/api/resource/v1" @@ -79,6 +83,8 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) err = corev1.AddToScheme(s) Expect(err).NotTo(HaveOccurred()) + err = kmmv1beta1.AddToScheme(s) + Expect(err).NotTo(HaveOccurred()) // +kubebuilder:scaffold:scheme @@ -88,6 +94,7 @@ var _ = BeforeSuite(func() { CRDDirectoryPaths: []string{ filepath.Join("..", "..", "config", "crd", "bases"), filepath.Join("..", "..", "config", "deployments", "nfd", "crds"), + filepath.Join(goModDir("github.com/kubernetes-sigs/kernel-module-management"), "config", "crd", "bases"), "testdata", }, ErrorIfCRDPathMissing: true, @@ -127,6 +134,14 @@ var _ = AfterSuite(func() { // This function streamlines the process by finding the required binaries, similar to // setting the 'KUBEBUILDER_ASSETS' environment variable. To ensure the binaries are // properly set up, run 'make setup-envtest' beforehand. +func goModDir(module string) string { + out, err := exec.Command("go", "list", "-m", "-f", "{{.Dir}}", module).Output() + if err != nil { + panic(fmt.Sprintf("failed to resolve module directory for %s: %v", module, err)) + } + return strings.TrimSpace(string(out)) +} + func getFirstFoundEnvTestBinaryDir() string { basePath := filepath.Join("..", "..", "bin", "k8s") entries, err := os.ReadDir(basePath) diff --git a/internal/controller/xpumanager_controller.go b/internal/controller/xpumanager_controller.go index 5f6506d..5d920f8 100644 --- a/internal/controller/xpumanager_controller.go +++ b/internal/controller/xpumanager_controller.go @@ -407,7 +407,7 @@ func (r *XpuManagerReconciler) updateDaemonSetObject(ds *apps.DaemonSet, spec *v processContainerResources(ds, spec, draClaim) processXpumdConfigMapMount(ds, otelConfigMapName) - ds.Spec.Template.Spec.NodeSelector = generateNodeSelector(spec) + ds.Spec.Template.Spec.NodeSelector = generateNodeSelector(spec, r.Opts) ds.Spec.Template.Spec.Tolerations = generateTolerations(spec) cspec := &ds.Spec.Template.Spec