diff --git a/api/v1alpha1/clusterpolicy_types.go b/api/v1alpha1/clusterpolicy_types.go index a58a3a1..d9b048e 100644 --- a/api/v1alpha1/clusterpolicy_types.go +++ b/api/v1alpha1/clusterpolicy_types.go @@ -69,6 +69,12 @@ type ClusterPolicySpec struct { // +kubebuilder:validation:Range=0:4 // +kubebuilder:validation:Default=2 LogLevel int32 `json:"logLevel,omitempty"` + + // KernelModule configures out-of-tree kernel module loading via KMM. + // When set, KMM loads the specified OOT driver module on each node. + // When nil, the in-tree kernel driver is used. + // +optional + KernelModule *KernelModuleSpec `json:"kernelModule,omitempty"` } // DynamicResourceAllocationSpec defines the desired state of DynamicResourceAllocation. @@ -147,11 +153,114 @@ type XpuManagerSpec struct { MonitoringResource string `json:"monitoringResource,omitempty"` } +// RegistryTLSSpec configures TLS behavior for accessing container image registries. +type RegistryTLSSpec struct { + Insecure bool `json:"insecure,omitempty"` + InsecureSkipTLSVerify bool `json:"insecureSkipTLSVerify,omitempty"` +} + +// KernelModuleSpec configures out-of-tree kernel module loading via KMM. +type KernelModuleSpec struct { + // ModuleName is the kernel module to load (defaults to "xe"). + // +kubebuilder:default=xe + ModuleName string `json:"moduleName,omitempty"` + + // Version opts into KMM's ordered upgrade + // (https://kmm.sigs.k8s.io/documentation/ordered_upgrade) for advanced, + // low-disruption driver rollouts. When set, KMM loads the module onto a node + // only once a cluster admin labels that node + // "kmm.node.kubernetes.io/version-module..=", + // letting the admin sequence the upgrade node-by-node and drain GPU workloads + // first. Nodes without a matching label are left untouched. + // + // Most users should leave Version unset and instead upgrade by changing the + // containerImage of the relevant kernelMappings entry (see ContainerImage), + // which rolls the new driver out to all selected nodes at once without any + // per-node label choreography. + // +optional + Version string `json:"version,omitempty"` + + // KernelMappings maps kernel version patterns to container images or + // build specifications. Translates directly to KMM KernelMapping objects. + // +kubebuilder:validation:MinItems=1 + KernelMappings []KernelMappingSpec `json:"kernelMappings"` + + // ModulesLoadingOrder specifies softdep-style loading order for + // multi-module drivers. First element must be ModuleName (defaults + // to "xe"); KMM loads in order and unloads in reverse. Must have + // >=2 entries if set. + // +optional + ModulesLoadingOrder []string `json:"modulesLoadingOrder,omitempty"` + + // FirmwarePath is the in-container path where firmware files are stored. + // +optional + FirmwarePath string `json:"firmwarePath,omitempty"` + + // RegistryTLS configures TLS for accessing the module image registry. + // +optional + RegistryTLS *RegistryTLSSpec `json:"registryTLS,omitempty"` +} + +// KernelMappingSpec maps a kernel version pattern to a container image +// or build specification. +type KernelMappingSpec struct { + // Regexp is a regular expression matched against node kernel versions. + // Use anchored patterns (e.g. "^5\\.14\\.0-.*$") for exact matches. + Regexp string `json:"regexp"` + + // ContainerImage is the full image reference for this kernel version. + // Required when Build is nil. KMM template vars (e.g. ${KERNEL_FULL_VERSION}, + // $MOD_NAME) are supported and resolved by KMM at reconcile time. + // + // Changing ContainerImage is the recommended way to upgrade the driver: KMM + // rolls the new image out to all selected nodes at once, briefly disrupting + // GPU workloads as the module reloads. + // +optional + ContainerImage string `json:"containerImage,omitempty"` + + // Build configures in-cluster building of the driver image via KMM. + // When set, KMM builds the image if it doesn't exist in the registry. + // +optional + Build *KernelModuleBuildSpec `json:"build,omitempty"` + + // InTreeModulesToRemove lists additional in-tree modules to unload + // for this mapping. ModuleName is always included automatically. + // +optional + InTreeModulesToRemove []string `json:"inTreeModulesToRemove,omitempty"` + + // RegistryTLS overrides parent-level TLS settings for this mapping. + // +optional + RegistryTLS *RegistryTLSSpec `json:"registryTLS,omitempty"` +} + +// KernelModuleBuildSpec configures in-cluster driver image building. +type KernelModuleBuildSpec struct { + // DockerfileConfigMap references a ConfigMap containing the Dockerfile. + DockerfileConfigMap v1.LocalObjectReference `json:"dockerfileConfigMap"` + + // BuildArgs are key-value pairs passed to the image builder. + // +optional + BuildArgs []BuildArg `json:"buildArgs,omitempty"` + + // Secrets are made available during the build (e.g., for private + // source repos). Not for registry auth -- use pullSecret on + // ClusterPolicySpec. + // +optional + Secrets []v1.LocalObjectReference `json:"secrets,omitempty"` +} + +// BuildArg is a key-value pair passed as a build argument. +type BuildArg struct { + Name string `json:"name"` + Value string `json:"value"` +} + // ClusterPolicyStatus defines the observed state of ClusterPolicy. type ClusterPolicyStatus struct { DevicePluginStatus string `json:"devicePluginStatus,omitempty"` DRAStatus string `json:"draStatus,omitempty"` XPUManagerStatus string `json:"xpuManagerStatus,omitempty"` + KMMStatus string `json:"kmmStatus,omitempty"` Errors []string `json:"errors,omitempty"` } @@ -183,6 +292,7 @@ type LocalQueueSpec struct { // +kubebuilder:printcolumn:name="DP",type=string,JSONPath=`.status.devicePluginStatus` // +kubebuilder:printcolumn:name="DRA",type=string,JSONPath=`.status.draStatus` // +kubebuilder:printcolumn:name="XPU",type=string,JSONPath=`.status.xpuManagerStatus` +// +kubebuilder:printcolumn:name="KMM",type=string,JSONPath=`.status.kmmStatus` // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` // +operator-sdk:csv:customresourcedefinitions:displayName="Intel GPU Cluster Policy" // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/api/v1alpha1/clusterpolicy_webhook.go b/api/v1alpha1/clusterpolicy_webhook.go index 2b58d53..731f763 100644 --- a/api/v1alpha1/clusterpolicy_webhook.go +++ b/api/v1alpha1/clusterpolicy_webhook.go @@ -74,6 +74,10 @@ func (d *ClusterPolicyCustomDefaulter) Default(_ context.Context, cp *ClusterPol spec.XpuManagerSpec.MonitoringResource = "monitoring" } + if spec.KernelModule != nil && spec.KernelModule.ModuleName == "" { + spec.KernelModule.ModuleName = "xe" + } + return nil } @@ -95,6 +99,7 @@ func validateClusterPolicySpec(spec *ClusterPolicySpec) (admission.Warnings, err errs = append(errs, validatePullSecret(spec)...) errs = append(errs, validateConfigMapOverride(spec)...) errs = append(errs, validateKueueSpec(spec)...) + errs = append(errs, validateKernelModuleSpec(spec)...) if w := warnForSpecProblems(spec); w != "" { warnings = append(warnings, w) @@ -223,6 +228,104 @@ func validateKueueSpec(spec *ClusterPolicySpec) []error { return errs } +func validateKernelModuleSpec(spec *ClusterPolicySpec) []error { + if spec.KernelModule == nil { + return nil + } + + km := spec.KernelModule + var errs []error + + for i, m := range km.KernelMappings { + prefix := fmt.Sprintf("kernelModule.kernelMappings[%d]", i) + errs = append(errs, validateKernelMapping(prefix, &m)...) + } + + if len(km.ModulesLoadingOrder) > 0 { + errs = append(errs, validateModulesLoadingOrder(km.ModuleName, km.ModulesLoadingOrder)...) + } + + return errs +} + +// kmmTemplateVarRegexp matches KMM's ${VAR} and $VAR template placeholders +// (e.g. ${KERNEL_FULL_VERSION}, $MOD_NAME), which KMM resolves at reconcile time. +var kmmTemplateVarRegexp = regexp.MustCompile(`\$\{[^}]*\}|\$[A-Za-z_][A-Za-z0-9_]*`) + +// substituteKMMTemplateVars swaps KMM template placeholders for a tag-safe token +// so the reference can be validated as a normal image reference. +func substituteKMMTemplateVars(image string) string { + return kmmTemplateVarRegexp.ReplaceAllString(image, "0") +} + +func validateImageTagOrDigest(field, image string) error { + // Validate a copy with template vars resolved; report the original on error. + resolved := substituteKMMTemplateVars(image) + + if _, err := reference.ParseNormalizedNamed(resolved); err != nil { + return fmt.Errorf("invalid image reference in %s: %q: %w", field, image, err) + } + + if !strings.Contains(resolved, ":") && !strings.Contains(resolved, "@") { + return fmt.Errorf("%s: image %q must include an explicit tag or digest", field, image) + } + + return nil +} + +func validateKernelMapping(prefix string, m *KernelMappingSpec) []error { + var errs []error + + if m.Regexp == "" { + errs = append(errs, fmt.Errorf("%s: regexp is required", prefix)) + } else { + if _, err := regexp.Compile(m.Regexp); err != nil { + errs = append(errs, fmt.Errorf("%s.regexp: invalid regular expression: %w", prefix, err)) + } + } + + if m.ContainerImage != "" { + if err := validateImageTagOrDigest(prefix+".containerImage", m.ContainerImage); err != nil { + errs = append(errs, err) + } + } + + if m.ContainerImage == "" && m.Build == nil { + errs = append(errs, fmt.Errorf("%s: one of containerImage or build must be set", prefix)) + } + + if m.Build != nil { + if m.Build.DockerfileConfigMap.Name == "" { + errs = append(errs, fmt.Errorf("%s.build.dockerfileConfigMap.name is required", prefix)) + } + } + + return errs +} + +func validateModulesLoadingOrder(moduleName string, order []string) []error { + var errs []error + + if len(order) < 2 { + errs = append(errs, fmt.Errorf("kernelModule.modulesLoadingOrder must have at least 2 entries")) + } + + if len(order) > 0 && moduleName != "" && order[0] != moduleName { + errs = append(errs, fmt.Errorf("kernelModule.modulesLoadingOrder[0] must be moduleName %q", moduleName)) + } + + seen := make(map[string]bool, len(order)) + for i, entry := range order { + if seen[entry] { + errs = append(errs, fmt.Errorf("kernelModule.modulesLoadingOrder[%d]: duplicate entry %q", i, entry)) + break + } + seen[entry] = true + } + + return errs +} + // warnForSpecProblems returns a warning message if some old or deprecated option is set. func warnForSpecProblems(spec *ClusterPolicySpec) string { if spec.ResourceRegistration == "dp" && spec.DevicePluginSpec.LevelzeroImage != "" { diff --git a/api/v1alpha1/clusterpolicy_webhook_test.go b/api/v1alpha1/clusterpolicy_webhook_test.go index 45967d1..9404be2 100644 --- a/api/v1alpha1/clusterpolicy_webhook_test.go +++ b/api/v1alpha1/clusterpolicy_webhook_test.go @@ -387,6 +387,250 @@ var _ = Describe("ClusterPolicy Webhook", func() { }) }) + Context("kernelModule validation", func() { + It("accepts nil kernelModule (in-tree mode)", func() { + obj.Spec.KernelModule = nil + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("accepts valid kernelModule with single wildcard mapping", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("accepts kernelModule without moduleName (defaults to xe)", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + KernelMappings: []KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("accepts kernelMappings with each mapping having its own containerImage", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "^5\\.14\\..*", ContainerImage: "registry.example.com/xe-rhel9:1.0"}, + {Regexp: "^6\\.12\\..*", ContainerImage: "registry.example.com/xe-rhel10:1.0"}, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("rejects mapping without containerImage or build", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "^5\\.14\\..*"}, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("one of containerImage or build must be set")) + }) + + It("rejects mapping without regexp", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {ContainerImage: "registry.example.com/xe:1.0"}, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("regexp is required")) + }) + + It("rejects mapping with invalid regexp", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "[invalid", ContainerImage: "registry.example.com/xe:1.0"}, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("invalid regular expression")) + }) + + It("rejects containerImage without tag or digest", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "^5\\.14\\..*", ContainerImage: "registry.example.com/xe-driver"}, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("must include an explicit tag or digest")) + }) + + It("accepts containerImage with a ${KERNEL_FULL_VERSION} template var in the tag", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-oot-kmd:v1.0-${KERNEL_FULL_VERSION}"}, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("accepts containerImage with a bare $KERNEL_FULL_VERSION template var in the tag", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-oot-kmd:v1.0-$KERNEL_FULL_VERSION"}, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("accepts containerImage using $MOD_NAME/$MOD_NAMESPACE template vars", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/${MOD_NAMESPACE}/$MOD_NAME:v1.0"}, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("still rejects a templated containerImage with no tag or digest", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-oot-kmd-${KERNEL_FULL_VERSION}"}, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("must include an explicit tag or digest")) + }) + + It("accepts valid build spec with dockerfileConfigMap", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + { + Regexp: "^5\\.14\\..*", + Build: &KernelModuleBuildSpec{ + DockerfileConfigMap: v1.LocalObjectReference{Name: "xe-dockerfile"}, + BuildArgs: []BuildArg{{Name: "XE_TAG", Value: "v1.0"}}, + }, + }, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("rejects build spec with empty dockerfileConfigMap name", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + { + Regexp: "^5\\.14\\..*", + Build: &KernelModuleBuildSpec{}, + }, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("dockerfileConfigMap.name is required")) + }) + + It("accepts valid registryTLS at top level", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe:1.0"}, + }, + RegistryTLS: &RegistryTLSSpec{InsecureSkipTLSVerify: true}, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("accepts valid registryTLS per-mapping", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + { + Regexp: "^.+$", + ContainerImage: "registry.example.com/xe:1.0", + RegistryTLS: &RegistryTLSSpec{Insecure: true}, + }, + }, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("accepts valid modulesLoadingOrder with moduleName first", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + ModulesLoadingOrder: []string{"xe", "drm_buddy"}, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).NotTo(HaveOccurred()) + }) + + It("rejects modulesLoadingOrder with fewer than 2 entries", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + ModulesLoadingOrder: []string{"xe"}, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("at least 2 entries")) + }) + + It("rejects modulesLoadingOrder where first entry is not moduleName", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + ModulesLoadingOrder: []string{"drm_buddy", "xe"}, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("must be moduleName")) + }) + + It("rejects modulesLoadingOrder with duplicate entries", func() { + obj.Spec.KernelModule = &KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + ModulesLoadingOrder: []string{"xe", "drm_buddy", "drm_buddy"}, + } + _, err := validator.ValidateCreate(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("duplicate entry")) + }) + }) + Context("spec warning for Levelzero", func() { It("emits a warning when levelzero image is set in DP mode", func() { obj.Spec.ResourceRegistration = dpName diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 2470cf4..f3e8257 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -25,6 +25,21 @@ import ( "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BuildArg) DeepCopyInto(out *BuildArg) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BuildArg. +func (in *BuildArg) DeepCopy() *BuildArg { + if in == nil { + return nil + } + out := new(BuildArg) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterPolicy) DeepCopyInto(out *ClusterPolicy) { *out = *in @@ -149,6 +164,11 @@ func (in *ClusterPolicySpec) DeepCopyInto(out *ClusterPolicySpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.KernelModule != nil { + in, out := &in.KernelModule, &out.KernelModule + *out = new(KernelModuleSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterPolicySpec. @@ -466,6 +486,94 @@ func (in *HealthinessSpec) DeepCopy() *HealthinessSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KernelMappingSpec) DeepCopyInto(out *KernelMappingSpec) { + *out = *in + if in.Build != nil { + in, out := &in.Build, &out.Build + *out = new(KernelModuleBuildSpec) + (*in).DeepCopyInto(*out) + } + if in.InTreeModulesToRemove != nil { + in, out := &in.InTreeModulesToRemove, &out.InTreeModulesToRemove + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.RegistryTLS != nil { + in, out := &in.RegistryTLS, &out.RegistryTLS + *out = new(RegistryTLSSpec) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KernelMappingSpec. +func (in *KernelMappingSpec) DeepCopy() *KernelMappingSpec { + if in == nil { + return nil + } + out := new(KernelMappingSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KernelModuleBuildSpec) DeepCopyInto(out *KernelModuleBuildSpec) { + *out = *in + out.DockerfileConfigMap = in.DockerfileConfigMap + if in.BuildArgs != nil { + in, out := &in.BuildArgs, &out.BuildArgs + *out = make([]BuildArg, len(*in)) + copy(*out, *in) + } + if in.Secrets != nil { + in, out := &in.Secrets, &out.Secrets + *out = make([]v1.LocalObjectReference, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KernelModuleBuildSpec. +func (in *KernelModuleBuildSpec) DeepCopy() *KernelModuleBuildSpec { + if in == nil { + return nil + } + out := new(KernelModuleBuildSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KernelModuleSpec) DeepCopyInto(out *KernelModuleSpec) { + *out = *in + if in.KernelMappings != nil { + in, out := &in.KernelMappings, &out.KernelMappings + *out = make([]KernelMappingSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.ModulesLoadingOrder != nil { + in, out := &in.ModulesLoadingOrder, &out.ModulesLoadingOrder + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.RegistryTLS != nil { + in, out := &in.RegistryTLS, &out.RegistryTLS + *out = new(RegistryTLSSpec) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KernelModuleSpec. +func (in *KernelModuleSpec) DeepCopy() *KernelModuleSpec { + if in == nil { + return nil + } + out := new(KernelModuleSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KueueQueueSpec) DeepCopyInto(out *KueueQueueSpec) { *out = *in @@ -503,6 +611,21 @@ func (in *LocalQueueSpec) DeepCopy() *LocalQueueSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RegistryTLSSpec) DeepCopyInto(out *RegistryTLSSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RegistryTLSSpec. +func (in *RegistryTLSSpec) DeepCopy() *RegistryTLSSpec { + if in == nil { + return nil + } + out := new(RegistryTLSSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *XpuManagerSpec) DeepCopyInto(out *XpuManagerSpec) { *out = *in diff --git a/build/operator/Dockerfile b/build/operator/Dockerfile index 0a09c39..f288eb4 100644 --- a/build/operator/Dockerfile +++ b/build/operator/Dockerfile @@ -53,8 +53,10 @@ RUN apt-get update -y && apt-get --no-install-recommends -y install file && \ chmod +x checksec && \ ./checksec --file=/workspace/manager --output=csv | grep -q "$CHECKSEC_REF" -RUN go get github.com/google/go-licenses && \ - go run github.com/google/go-licenses save ./cmd/ --save_path licenses && \ +# GOROOT is passed explicitly so go-licenses can recognize stdlib packages even +# when `go list` re-execs a different toolchain than the one that built it. +RUN go install github.com/google/go-licenses@v1.6.0 && \ + GOROOT="$(go env GOROOT)" go-licenses save ./cmd/ --save_path licenses && \ cp /workspace/LICENSE /workspace/licenses/LICENSE.intel-gpu-base-operator ARG BASE_IMAGE diff --git a/charts/gpu-base-operator-policy/templates/clusterpolicy.yaml b/charts/gpu-base-operator-policy/templates/clusterpolicy.yaml index 737355d..84c0aea 100644 --- a/charts/gpu-base-operator-policy/templates/clusterpolicy.yaml +++ b/charts/gpu-base-operator-policy/templates/clusterpolicy.yaml @@ -62,6 +62,66 @@ spec: configMapOverride: {{ .Values.xpu.configMapOverride | default "" }} {{- end }} +{{- if .Values.kernelModule }} + kernelModule: + moduleName: {{ .Values.kernelModule.moduleName }} + {{- if .Values.kernelModule.version }} + version: {{ .Values.kernelModule.version | quote }} + {{- end }} + kernelMappings: + {{- range .Values.kernelModule.kernelMappings }} + - regexp: {{ .regexp | quote }} + {{- if .containerImage }} + containerImage: {{ .containerImage }} + {{- end }} + {{- if .build }} + build: + dockerfileConfigMap: + name: {{ .build.dockerfileConfigMap.name }} + {{- if .build.buildArgs }} + buildArgs: + {{- range .build.buildArgs }} + - name: {{ .name }} + value: {{ .value | quote }} + {{- end }} + {{- end }} + {{- if .build.secrets }} + secrets: + {{- range .build.secrets }} + - name: {{ .name }} + {{- end }} + {{- end }} + {{- end }} + {{- if .inTreeModulesToRemove }} + inTreeModulesToRemove: {{ toYaml .inTreeModulesToRemove | nindent 10 }} + {{- end }} + {{- if .registryTLS }} + registryTLS: + {{- if .registryTLS.insecure }} + insecure: {{ .registryTLS.insecure }} + {{- end }} + {{- if .registryTLS.insecureSkipTLSVerify }} + insecureSkipTLSVerify: {{ .registryTLS.insecureSkipTLSVerify }} + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.kernelModule.modulesLoadingOrder }} + modulesLoadingOrder: {{ toYaml .Values.kernelModule.modulesLoadingOrder | nindent 6 }} + {{- end }} + {{- if .Values.kernelModule.firmwarePath }} + firmwarePath: {{ .Values.kernelModule.firmwarePath }} + {{- end }} + {{- if .Values.kernelModule.registryTLS }} + registryTLS: + {{- if .Values.kernelModule.registryTLS.insecure }} + insecure: {{ .Values.kernelModule.registryTLS.insecure }} + {{- end }} + {{- if .Values.kernelModule.registryTLS.insecureSkipTLSVerify }} + insecureSkipTLSVerify: {{ .Values.kernelModule.registryTLS.insecureSkipTLSVerify }} + {{- end }} + {{- end }} +{{- end }} + {{- if .Values.pullSecret }} pullSecret: {{ .Values.pullSecret }} {{- end }} diff --git a/charts/gpu-base-operator-policy/values.yaml b/charts/gpu-base-operator-policy/values.yaml index fc465f2..6dd3969 100644 --- a/charts/gpu-base-operator-policy/values.yaml +++ b/charts/gpu-base-operator-policy/values.yaml @@ -35,6 +35,47 @@ kueue: - name: gpu-queue namespace: default +# kernelModule: +# moduleName: xe +# # To upgrade the driver, just change the containerImage below; KMM rolls it +# # out to all selected nodes at once (brief GPU-workload disruption). Only set +# # `version` if you want KMM's node-by-node ordered upgrade, which requires you +# # to label each node with +# # kmm.node.kubernetes.io/version-module..= before +# # the driver loads there. See https://kmm.sigs.k8s.io/documentation/ordered_upgrade +# # version: "1.0" +# kernelMappings: +# - regexp: "^.+$" +# containerImage: registry.example.com/xe-driver:1.0 +# +# # Multi-kernel support: +# # kernelMappings: +# # - regexp: "^5\\.14\\.0-.*\\.el9.*\\.x86_64$" +# # containerImage: registry.example.com/xe-driver-rhel9:1.0 +# # - regexp: "^6\\.12\\..*" +# # containerImage: registry.example.com/xe-driver-rhel10:1.0 +# # +# # In-cluster build: +# # kernelMappings: +# # - regexp: "^5\\.14\\..*" +# # build: +# # dockerfileConfigMap: +# # name: xe-dockerfile +# # buildArgs: +# # - name: XE_TAG +# # value: v1.0 +# # +# # Per-mapping extra in-tree modules to remove (moduleName is always included automatically): +# # - regexp: "^5\\.14\\..*" +# # containerImage: registry.example.com/xe:1.0 +# # inTreeModulesToRemove: ["i915"] +# # +# # modulesLoadingOrder: ["xe", "drm_buddy", "drm_ttm_helper"] +# # firmwarePath: /opt/lib/firmware/xe +# # registryTLS: +# # insecure: false +# # insecureSkipTLSVerify: false + pullSecret: null nodeSelector: {} tolerations: [] diff --git a/charts/gpu-base-operator/crds/clusterpolicies.yaml b/charts/gpu-base-operator/crds/clusterpolicies.yaml index 870f9c5..f4ce3f7 100644 --- a/charts/gpu-base-operator/crds/clusterpolicies.yaml +++ b/charts/gpu-base-operator/crds/clusterpolicies.yaml @@ -24,6 +24,9 @@ spec: - jsonPath: .status.xpuManagerStatus name: XPU type: string + - jsonPath: .status.kmmStatus + name: KMM + type: string - jsonPath: .metadata.creationTimestamp name: Age type: date @@ -136,6 +139,163 @@ spec: minimum: 1 type: integer type: object + kernelModule: + description: |- + KernelModule configures out-of-tree kernel module loading via KMM. + When set, KMM loads the specified OOT driver module on each node. + When nil, the in-tree kernel driver is used. + properties: + firmwarePath: + description: FirmwarePath is the in-container path where firmware + files are stored. + type: string + kernelMappings: + description: |- + KernelMappings maps kernel version patterns to container images or + build specifications. Translates directly to KMM KernelMapping objects. + items: + description: |- + KernelMappingSpec maps a kernel version pattern to a container image + or build specification. + properties: + build: + description: |- + Build configures in-cluster building of the driver image via KMM. + When set, KMM builds the image if it doesn't exist in the registry. + properties: + buildArgs: + description: BuildArgs are key-value pairs passed to + the image builder. + items: + description: BuildArg is a key-value pair passed as + a build argument. + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + dockerfileConfigMap: + description: DockerfileConfigMap references a ConfigMap + containing the Dockerfile. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + secrets: + description: |- + Secrets are made available during the build (e.g., for private + source repos). Not for registry auth -- use pullSecret on + ClusterPolicySpec. + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + required: + - dockerfileConfigMap + type: object + containerImage: + description: |- + ContainerImage is the full image reference for this kernel version. + Required when Build is nil. KMM template vars (e.g. ${KERNEL_FULL_VERSION}, + $MOD_NAME) are supported and resolved by KMM at reconcile time. + + Changing ContainerImage is the recommended way to upgrade the driver: KMM + rolls the new image out to all selected nodes at once, briefly disrupting + GPU workloads as the module reloads. + type: string + inTreeModulesToRemove: + description: |- + InTreeModulesToRemove lists additional in-tree modules to unload + for this mapping. ModuleName is always included automatically. + items: + type: string + type: array + regexp: + description: |- + Regexp is a regular expression matched against node kernel versions. + Use anchored patterns (e.g. "^5\\.14\\.0-.*$") for exact matches. + type: string + registryTLS: + description: RegistryTLS overrides parent-level TLS settings + for this mapping. + properties: + insecure: + type: boolean + insecureSkipTLSVerify: + type: boolean + type: object + required: + - regexp + type: object + minItems: 1 + type: array + moduleName: + default: xe + description: ModuleName is the kernel module to load (defaults + to "xe"). + type: string + modulesLoadingOrder: + description: |- + ModulesLoadingOrder specifies softdep-style loading order for + multi-module drivers. First element must be ModuleName (defaults + to "xe"); KMM loads in order and unloads in reverse. Must have + >=2 entries if set. + items: + type: string + type: array + registryTLS: + description: RegistryTLS configures TLS for accessing the module + image registry. + properties: + insecure: + type: boolean + insecureSkipTLSVerify: + type: boolean + type: object + version: + description: |- + Version opts into KMM's ordered upgrade + (https://kmm.sigs.k8s.io/documentation/ordered_upgrade) for advanced, + low-disruption driver rollouts. When set, KMM loads the module onto a node + only once a cluster admin labels that node + "kmm.node.kubernetes.io/version-module..=", + letting the admin sequence the upgrade node-by-node and drain GPU workloads + first. Nodes without a matching label are left untouched. + + Most users should leave Version unset and instead upgrade by changing the + containerImage of the relevant kernelMappings entry (see ContainerImage), + which rolls the new driver out to all selected nodes at once without any + per-node label choreography. + type: string + required: + - kernelMappings + type: object kueue: description: Define Kueue queues properties: @@ -289,6 +449,8 @@ spec: items: type: string type: array + kmmStatus: + type: string xpuManagerStatus: type: string type: object diff --git a/charts/gpu-base-operator/templates/manager.yaml b/charts/gpu-base-operator/templates/manager.yaml index bec40f0..56667f7 100644 --- a/charts/gpu-base-operator/templates/manager.yaml +++ b/charts/gpu-base-operator/templates/manager.yaml @@ -64,6 +64,8 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: MODULE_LOADER_SERVICE_ACCOUNT_NAME + value: "{{ .Values.moduleLoader.serviceAccountName }}" {{- if .Values.privateRegistry.token }} - name: OPERATOR_SECRET value: "{{ .Release.Name }}-operator-private-registry" diff --git a/charts/gpu-base-operator/templates/module_loader_scc.yaml b/charts/gpu-base-operator/templates/module_loader_scc.yaml new file mode 100644 index 0000000..b778c9c --- /dev/null +++ b/charts/gpu-base-operator/templates/module_loader_scc.yaml @@ -0,0 +1,40 @@ +{{- if .Values.openshift.enabled }} +apiVersion: security.openshift.io/v1 +kind: SecurityContextConstraints +metadata: + name: {{ .Release.Name }}-module-loader-scc + labels: + app.kubernetes.io/name: intel-gpu-base-operator + app.kubernetes.io/managed-by: {{ .Release.Service }} +allowPrivilegedContainer: true +allowHostDirVolumePlugin: true +allowHostIPC: false +allowHostNetwork: false +allowHostPID: false +allowHostPorts: false +allowPrivilegeEscalation: true +allowedCapabilities: + - SYS_MODULE +defaultAddCapabilities: null +requiredDropCapabilities: null +fsGroup: + type: RunAsAny +readOnlyRootFilesystem: false +runAsUser: + type: RunAsAny +seLinuxContext: + type: RunAsAny +seccompProfiles: + - "*" +supplementalGroups: + type: RunAsAny +volumes: + - hostPath + - configMap + - secret + - projected + - emptyDir + - downwardAPI +users: [] +groups: [] +{{- end }} diff --git a/charts/gpu-base-operator/templates/module_loader_scc_role.yaml b/charts/gpu-base-operator/templates/module_loader_scc_role.yaml new file mode 100644 index 0000000..72f292f --- /dev/null +++ b/charts/gpu-base-operator/templates/module_loader_scc_role.yaml @@ -0,0 +1,18 @@ +{{- if .Values.openshift.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Release.Name }}-module-loader-scc-role + labels: + app.kubernetes.io/name: intel-gpu-base-operator + app.kubernetes.io/managed-by: {{ .Release.Service }} +rules: + - apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + resourceNames: + - {{ .Release.Name }}-module-loader-scc + verbs: + - use +{{- end }} diff --git a/charts/gpu-base-operator/templates/module_loader_scc_rolebinding.yaml b/charts/gpu-base-operator/templates/module_loader_scc_rolebinding.yaml new file mode 100644 index 0000000..c7d0e2d --- /dev/null +++ b/charts/gpu-base-operator/templates/module_loader_scc_rolebinding.yaml @@ -0,0 +1,17 @@ +{{- if .Values.openshift.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Release.Name }}-module-loader-scc-binding + labels: + app.kubernetes.io/name: intel-gpu-base-operator + app.kubernetes.io/managed-by: {{ .Release.Service }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Release.Name }}-module-loader-scc-role +subjects: + - kind: ServiceAccount + name: {{ .Values.moduleLoader.serviceAccountName }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/charts/gpu-base-operator/templates/module_loader_serviceaccount.yaml b/charts/gpu-base-operator/templates/module_loader_serviceaccount.yaml new file mode 100644 index 0000000..05989ef --- /dev/null +++ b/charts/gpu-base-operator/templates/module_loader_serviceaccount.yaml @@ -0,0 +1,10 @@ +{{- if .Values.moduleLoader.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.moduleLoader.serviceAccountName }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: intel-gpu-base-operator + app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} diff --git a/charts/gpu-base-operator/templates/role.yaml b/charts/gpu-base-operator/templates/role.yaml index c556fbd..51b89f7 100644 --- a/charts/gpu-base-operator/templates/role.yaml +++ b/charts/gpu-base-operator/templates/role.yaml @@ -78,6 +78,24 @@ rules: - get - patch - update +- apiGroups: + - kmm.sigs.x-k8s.io + resources: + - modules + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kmm.sigs.x-k8s.io + resources: + - modules/status + verbs: + - get - apiGroups: - kueue.x-k8s.io resources: diff --git a/charts/gpu-base-operator/values.yaml b/charts/gpu-base-operator/values.yaml index 831dbc5..7e383d8 100644 --- a/charts/gpu-base-operator/values.yaml +++ b/charts/gpu-base-operator/values.yaml @@ -76,3 +76,10 @@ kueue: - deviceClassNames: - "gpu.intel.com" name: "dra.gpu.intel.com" + +openshift: + enabled: false + +moduleLoader: + create: true + serviceAccountName: module-loader diff --git a/cmd/main.go b/cmd/main.go index 59dea7c..f94c9ba 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -55,6 +55,7 @@ import ( intelcomv1alpha1 "github.com/intel/gpu-base-operator/api/v1alpha1" "github.com/intel/gpu-base-operator/internal/controller" buildVersion "github.com/intel/gpu-base-operator/internal/version" + kmmv1beta1 "github.com/kubernetes-sigs/kernel-module-management/api/v1beta1" // +kubebuilder:scaffold:imports ) @@ -63,6 +64,7 @@ var ( setupLog = ctrl.Log.WithName("setup") openShiftGroups = []string{"route.openshift.io", "security.openshift.io"} draGroups = []string{"resource.k8s.io"} + kmmGroups = []string{"kmm.sigs.x-k8s.io"} ) const ( @@ -70,6 +72,7 @@ const ( openshiftCluster = "OpenShift" draCluster = "DRA" + kmmCluster = "KMM" ) func init() { @@ -80,6 +83,7 @@ func init() { utilruntime.Must(prometheusv1.AddToScheme(scheme)) utilruntime.Must(resv1.AddToScheme(scheme)) utilruntime.Must(kueuev1.AddToScheme(scheme)) + utilruntime.Must(kmmv1beta1.AddToScheme(scheme)) // +kubebuilder:scaffold:scheme } @@ -103,6 +107,7 @@ func detectClusterFeatures() (map[string]bool, error) { features := map[string]bool{ openshiftCluster: false, draCluster: false, + kmmCluster: false, } for _, group := range apiGroups.Groups { @@ -112,6 +117,9 @@ func detectClusterFeatures() (map[string]bool, error) { if slices.Contains(draGroups, group.Name) { features[draCluster] = true } + if slices.Contains(kmmGroups, group.Name) { + features[kmmCluster] = true + } } return features, nil @@ -323,12 +331,19 @@ func main() { setupLog.Info("Operator secret supplied", "secret", secret) } + moduleLoaderSAName := os.Getenv("MODULE_LOADER_SERVICE_ACCOUNT_NAME") + if moduleLoaderSAName == "" { + moduleLoaderSAName = "module-loader" + } + copts := controller.ControllerOpts{ - Namespace: ns, - SecretName: secret, - RequeueDelay: time.Second * 5, - DRAEnable: features[draCluster], - OpenShift: features[openshiftCluster], + Namespace: ns, + SecretName: secret, + RequeueDelay: time.Second * 5, + DRAEnable: features[draCluster], + OpenShift: features[openshiftCluster], + KMMEnable: features[kmmCluster], + ModuleLoaderServiceAccountName: moduleLoaderSAName, } if err := (&controller.ClusterPolicyReconciler{ diff --git a/config/crd/bases/intel.com_clusterpolicies.yaml b/config/crd/bases/intel.com_clusterpolicies.yaml index 870f9c5..f4ce3f7 100644 --- a/config/crd/bases/intel.com_clusterpolicies.yaml +++ b/config/crd/bases/intel.com_clusterpolicies.yaml @@ -24,6 +24,9 @@ spec: - jsonPath: .status.xpuManagerStatus name: XPU type: string + - jsonPath: .status.kmmStatus + name: KMM + type: string - jsonPath: .metadata.creationTimestamp name: Age type: date @@ -136,6 +139,163 @@ spec: minimum: 1 type: integer type: object + kernelModule: + description: |- + KernelModule configures out-of-tree kernel module loading via KMM. + When set, KMM loads the specified OOT driver module on each node. + When nil, the in-tree kernel driver is used. + properties: + firmwarePath: + description: FirmwarePath is the in-container path where firmware + files are stored. + type: string + kernelMappings: + description: |- + KernelMappings maps kernel version patterns to container images or + build specifications. Translates directly to KMM KernelMapping objects. + items: + description: |- + KernelMappingSpec maps a kernel version pattern to a container image + or build specification. + properties: + build: + description: |- + Build configures in-cluster building of the driver image via KMM. + When set, KMM builds the image if it doesn't exist in the registry. + properties: + buildArgs: + description: BuildArgs are key-value pairs passed to + the image builder. + items: + description: BuildArg is a key-value pair passed as + a build argument. + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + dockerfileConfigMap: + description: DockerfileConfigMap references a ConfigMap + containing the Dockerfile. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + secrets: + description: |- + Secrets are made available during the build (e.g., for private + source repos). Not for registry auth -- use pullSecret on + ClusterPolicySpec. + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + required: + - dockerfileConfigMap + type: object + containerImage: + description: |- + ContainerImage is the full image reference for this kernel version. + Required when Build is nil. KMM template vars (e.g. ${KERNEL_FULL_VERSION}, + $MOD_NAME) are supported and resolved by KMM at reconcile time. + + Changing ContainerImage is the recommended way to upgrade the driver: KMM + rolls the new image out to all selected nodes at once, briefly disrupting + GPU workloads as the module reloads. + type: string + inTreeModulesToRemove: + description: |- + InTreeModulesToRemove lists additional in-tree modules to unload + for this mapping. ModuleName is always included automatically. + items: + type: string + type: array + regexp: + description: |- + Regexp is a regular expression matched against node kernel versions. + Use anchored patterns (e.g. "^5\\.14\\.0-.*$") for exact matches. + type: string + registryTLS: + description: RegistryTLS overrides parent-level TLS settings + for this mapping. + properties: + insecure: + type: boolean + insecureSkipTLSVerify: + type: boolean + type: object + required: + - regexp + type: object + minItems: 1 + type: array + moduleName: + default: xe + description: ModuleName is the kernel module to load (defaults + to "xe"). + type: string + modulesLoadingOrder: + description: |- + ModulesLoadingOrder specifies softdep-style loading order for + multi-module drivers. First element must be ModuleName (defaults + to "xe"); KMM loads in order and unloads in reverse. Must have + >=2 entries if set. + items: + type: string + type: array + registryTLS: + description: RegistryTLS configures TLS for accessing the module + image registry. + properties: + insecure: + type: boolean + insecureSkipTLSVerify: + type: boolean + type: object + version: + description: |- + Version opts into KMM's ordered upgrade + (https://kmm.sigs.k8s.io/documentation/ordered_upgrade) for advanced, + low-disruption driver rollouts. When set, KMM loads the module onto a node + only once a cluster admin labels that node + "kmm.node.kubernetes.io/version-module..=", + letting the admin sequence the upgrade node-by-node and drain GPU workloads + first. Nodes without a matching label are left untouched. + + Most users should leave Version unset and instead upgrade by changing the + containerImage of the relevant kernelMappings entry (see ContainerImage), + which rolls the new driver out to all selected nodes at once without any + per-node label choreography. + type: string + required: + - kernelMappings + type: object kueue: description: Define Kueue queues properties: @@ -289,6 +449,8 @@ spec: items: type: string type: array + kmmStatus: + type: string xpuManagerStatus: type: string type: object diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 201c738..0bced06 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -69,6 +69,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: MODULE_LOADER_SERVICE_ACCOUNT_NAME + # kustomize prepends the namePrefix (intel-gpu-base-operator-) to the + # ServiceAccount, but not to this literal, so set the resolved name here. + value: intel-gpu-base-operator-module-loader image: ghcr.io/intel/intel-gpu-base-operator:devel imagePullPolicy: IfNotPresent name: manager diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 955856d..47aafc1 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -27,6 +27,7 @@ resources: - gpufirmwareupdate_admin_role.yaml - gpufirmwareupdate_editor_role.yaml - gpufirmwareupdate_viewer_role.yaml +- module_loader_serviceaccount.yaml - clusterpolicy_admin_role.yaml - clusterpolicy_editor_role.yaml - clusterpolicy_viewer_role.yaml diff --git a/config/rbac/module_loader_serviceaccount.yaml b/config/rbac/module_loader_serviceaccount.yaml new file mode 100644 index 0000000..c9719cd --- /dev/null +++ b/config/rbac/module_loader_serviceaccount.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: module-loader + namespace: system + labels: + app.kubernetes.io/name: intel-gpu-base-operator + app.kubernetes.io/managed-by: kustomize diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index e691942..b68ee6b 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -78,6 +78,24 @@ rules: - get - patch - update +- apiGroups: + - kmm.sigs.x-k8s.io + resources: + - modules + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kmm.sigs.x-k8s.io + resources: + - modules/status + verbs: + - get - apiGroups: - kueue.x-k8s.io resources: diff --git a/go.mod b/go.mod index 5f33aa7..d0b4566 100644 --- a/go.mod +++ b/go.mod @@ -63,6 +63,7 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.19.1 // indirect + github.com/kubernetes-sigs/kernel-module-management v1.0.1-0.20260729111759-de3f8c9d5eae github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect diff --git a/go.sum b/go.sum index 536ccaa..fd50d9a 100644 --- a/go.sum +++ b/go.sum @@ -117,6 +117,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kubernetes-sigs/kernel-module-management v1.0.1-0.20260729111759-de3f8c9d5eae h1:u3TS/PJYMOpIhAmAs1blrMlafMp+4FbFvzy4fk+CalY= +github.com/kubernetes-sigs/kernel-module-management v1.0.1-0.20260729111759-de3f8c9d5eae/go.mod h1:tqr/yRJMo+6VzLB9B/25H+wWvB+VMr95CYHD4ObyByA= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= diff --git a/internal/controller/clusterpolicy_controller.go b/internal/controller/clusterpolicy_controller.go index dc44292..b8052bf 100644 --- a/internal/controller/clusterpolicy_controller.go +++ b/internal/controller/clusterpolicy_controller.go @@ -19,6 +19,7 @@ package controller import ( "context" "errors" + "fmt" "reflect" "slices" "time" @@ -41,6 +42,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" v1alpha "github.com/intel/gpu-base-operator/api/v1alpha1" + kmmv1beta1 "github.com/kubernetes-sigs/kernel-module-management/api/v1beta1" ) // ClusterPolicyReconciler reconciles a ClusterPolicy object @@ -52,12 +54,15 @@ type ClusterPolicyReconciler struct { } type ControllerOpts struct { - ReqName string - Namespace string - SecretName string - RequeueDelay time.Duration - DRAEnable bool - OpenShift bool + ReqName string + Namespace string + SecretName string + RequeueDelay time.Duration + DRAEnable bool + OpenShift bool + KMMEnable bool + ModuleLoaderServiceAccountName string + KMMModuleReadyLabel string } type requeueReconcileErr struct { @@ -146,6 +151,11 @@ func (r *ClusterPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Reques var origCp *v1alpha.ClusterPolicy if cp != nil { origCp = cp.DeepCopy() + + // Clear stale errors so Status.Errors reflects the current reconcile only. + // Sub-controllers re-add any errors that are still relevant this pass; the + // deferred DeepEqual status update persists the cleared slice. + cp.Status.Errors = nil } // Defer status update at the end of reconciliation, to ensure we capture any changes made by sub-controllers. @@ -169,7 +179,12 @@ func (r *ClusterPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Reques opts := r.Opts opts.ReqName = req.Name - subControllers := make([]SubControllerInterface, 0, 4) + if opts.KMMEnable { + opts.KMMModuleReadyLabel = fmt.Sprintf("kmm.node.kubernetes.io/%s.%s.ready", + opts.Namespace, kmmModuleName(req.Name)) + } + + subControllers := make([]SubControllerInterface, 0, 5) // Initialize sub-controllers subControllers = append(subControllers, &DevicePluginReconciler{Client: r.Client, Scheme: r.Scheme, Opts: opts}) @@ -178,6 +193,12 @@ func (r *ClusterPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Reques subControllers = append(subControllers, &DRAReconciler{Client: r.Client, Scheme: r.Scheme, Opts: opts}) subControllers = append(subControllers, &MiscReconciler{Client: r.Client, APIReader: r.APIReader, Scheme: r.Scheme, Opts: opts, CrdNames: crdNames}) + // KMM sub-controller gets a copy with the ready label cleared so the + // KMM Module itself doesn't gate on its own readiness. + kmmOpts := opts + kmmOpts.KMMModuleReadyLabel = "" + subControllers = append(subControllers, &KMMReconciler{Client: r.Client, Scheme: r.Scheme, Opts: kmmOpts}) + // Ensure finalizer is present on live (non-deleted) ClusterPolicy objects. if cp != nil && cp.DeletionTimestamp.IsZero() { if !controllerutil.ContainsFinalizer(cp, clusterPolicyFinalizer) { @@ -345,6 +366,10 @@ func (r *ClusterPolicyReconciler) SetupWithManager(mgr ctrl.Manager, opts Contro Named("clusterpolicy"). Owns(&apps.DaemonSet{}) + if opts.KMMEnable { + b = b.Owns(&kmmv1beta1.Module{}) + } + // Only watch DRA pods when DRA is enabled in the cluster, to avoid unnecessary // pod list/watch permissions and reconcile noise when DRA is not in use. if opts.DRAEnable { diff --git a/internal/controller/controller_utils.go b/internal/controller/controller_utils.go index 76839c1..b9c89d0 100644 --- a/internal/controller/controller_utils.go +++ b/internal/controller/controller_utils.go @@ -17,11 +17,16 @@ limitations under the License. package controller import ( + "context" + v1alpha "github.com/intel/gpu-base-operator/api/v1alpha1" core "k8s.io/api/core/v1" + resourcev1 "k8s.io/api/resource/v1" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" ) -func generateNodeSelector(cp *v1alpha.ClusterPolicy) map[string]string { +func generateNodeSelector(cp *v1alpha.ClusterPolicy, opts ControllerOpts) map[string]string { ns := map[string]string{ "kubernetes.io/arch": "amd64", } @@ -36,6 +41,10 @@ func generateNodeSelector(cp *v1alpha.ClusterPolicy) map[string]string { ns["intel.feature.node.kubernetes.io/gpu"] = trueValue } + if opts.KMMModuleReadyLabel != "" && cp.Spec.KernelModule != nil { + ns[opts.KMMModuleReadyLabel] = "" + } + return ns } @@ -102,6 +111,31 @@ func shouldRemoveXpumd(cp *v1alpha.ClusterPolicy) bool { return false } +func anyAllocatedResourceClaims(ctx context.Context, c client.Client, driverName string) bool { + var rcList resourcev1.ResourceClaimList + + if err := c.List(ctx, &rcList); err != nil { + klog.Error(err, "unable to list ResourceClaims, assuming allocated claims exist") + return true + } + + for _, claim := range rcList.Items { + alloc := claim.Status.Allocation + if alloc == nil || len(alloc.Devices.Results) == 0 { + continue + } + + for _, dev := range alloc.Devices.Results { + if dev.Driver == driverName { + klog.Infof("Found allocated ResourceClaim with GPU device: %s", claim.Name) + return true + } + } + } + + return false +} + // Convert the integer based log level to a string based log level for the OTel config. func logLevelForXpum(cp *v1alpha.ClusterPolicy) string { v := cp.Spec.XpuManagerSpec.LogLevel diff --git a/internal/controller/deviceplugin_controller.go b/internal/controller/deviceplugin_controller.go index 7f850e9..4d0e2aa 100644 --- a/internal/controller/deviceplugin_controller.go +++ b/internal/controller/deviceplugin_controller.go @@ -167,7 +167,7 @@ func (r *DevicePluginReconciler) updateDaemonSetObject(ds *apps.DaemonSet, spec ds.Spec.Template.Spec.Containers[0].Args = dpArgs(spec) - ds.Spec.Template.Spec.NodeSelector = generateNodeSelector(spec) + ds.Spec.Template.Spec.NodeSelector = generateNodeSelector(spec, r.Opts) ds.Spec.Template.Spec.Tolerations = generateTolerations(spec) cspec := &ds.Spec.Template.Spec diff --git a/internal/controller/dra_controller.go b/internal/controller/dra_controller.go index 310b564..180cdf0 100644 --- a/internal/controller/dra_controller.go +++ b/internal/controller/dra_controller.go @@ -248,40 +248,6 @@ func (r *DRAReconciler) createOpenShiftResourcesIfNotExists(ctx context.Context, return nil } -func (r *DRAReconciler) anyAllocatedResourceClaims(ctx context.Context) bool { - var rcList resv1.ResourceClaimList - - klog.Info("Checking for allocated ResourceClaims that would prevent DRA removal") - - if err := r.List(ctx, &rcList); err != nil { - klog.Error(err, "unable to list ResourceClaims") - - return false - } - - klog.Infof("Found %d ResourceClaims", len(rcList.Items)) - for _, claim := range rcList.Items { - alloc := claim.Status.Allocation - - if alloc == nil { - continue - } - if len(alloc.Devices.Results) == 0 { - continue - } - - for _, dev := range alloc.Devices.Results { - if dev.Driver == gpuDeviceClass { - klog.Infof("Found allocated ResourceClaim with GPU device: %s", claim.Name) - - return true - } - } - } - - return false -} - func addHealthCheckIfMissing(container *core.Container, port int32) { for _, p := range container.Ports { if p.ContainerPort == port { @@ -358,7 +324,7 @@ func (r *DRAReconciler) updateDaemonSetObject(ds *apps.DaemonSet, spec *v1alpha. ds.Spec.Template.Spec.Containers[0].Image = dspec.Image ds.Spec.Template.Spec.Containers[0].Args = r.generateArgs(spec) - ds.Spec.Template.Spec.NodeSelector = generateNodeSelector(spec) + ds.Spec.Template.Spec.NodeSelector = generateNodeSelector(spec, r.Opts) ds.Spec.Template.Spec.Tolerations = generateTolerations(spec) cspec := &ds.Spec.Template.Spec @@ -424,7 +390,7 @@ func (r *DRAReconciler) removeDeploymentIfExists(ctx context.Context, cp *v1alph // If there are any allocated ResourceClaims, removal of DRA will cause // the Pods using them to be stuck at Terminating. // Requeue and try again later. - if r.anyAllocatedResourceClaims(ctx) { + if anyAllocatedResourceClaims(ctx, r.Client, gpuDeviceClass) { return ctrl.Result{RequeueAfter: r.Opts.RequeueDelay}, requeueReconcileErr{} } diff --git a/internal/controller/kmm_controller.go b/internal/controller/kmm_controller.go new file mode 100644 index 0000000..8195242 --- /dev/null +++ b/internal/controller/kmm_controller.go @@ -0,0 +1,313 @@ +/* +Copyright 2026 Intel Corporation. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + v1alpha "github.com/intel/gpu-base-operator/api/v1alpha1" + kmmv1beta1 "github.com/kubernetes-sigs/kernel-module-management/api/v1beta1" +) + +// +kubebuilder:rbac:groups=kmm.sigs.x-k8s.io,resources=modules,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=kmm.sigs.x-k8s.io,resources=modules/status,verbs=get + +// KMMReconciler manages a KMM Module CR for out-of-tree kernel module loading. +// It only configures the moduleLoader section — DP/DRA lifecycle remains with the native controllers. +type KMMReconciler struct { + client.Client + Scheme *runtime.Scheme + Opts ControllerOpts +} + +const ( + kmmModuleSuffix = "-gpu" + + kmmNotEnabledMsg = "KMM is not installed in the cluster." +) + +func kmmModuleName(cpName string) string { + return cpName + kmmModuleSuffix +} + +func (r *KMMReconciler) Reconcile(ctx context.Context, cp *v1alpha.ClusterPolicy) (ctrl.Result, error) { + moduleName := kmmModuleName(r.Opts.ReqName) + + if !r.Opts.KMMEnable { + if cp != nil && cp.Spec.KernelModule != nil { + addIfMissing(&cp.Status.Errors, kmmNotEnabledMsg) + } + + return ctrl.Result{}, nil + } + + if cp == nil || cp.Spec.KernelModule == nil { + return r.deleteModuleIfExists(ctx, cp, moduleName) + } + + // KMM's Module webhook forbids toggling container.version between empty and + // non-empty in place, so recreate the Module when the user does so. + if res, done, err := r.recreateOnVersionToggle(ctx, cp, moduleName); done || err != nil { + return res, err + } + + mod := &kmmv1beta1.Module{ + ObjectMeta: metav1.ObjectMeta{ + Name: moduleName, + Namespace: r.Opts.Namespace, + }, + } + + result, err := controllerutil.CreateOrPatch(ctx, r.Client, mod, func() error { + return r.setModuleDesiredState(mod, cp) + }) + + r.updateStatus(cp, mod) + + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to reconcile KMM Module %s for ClusterPolicy %s: %w", moduleName, cp.Name, err) + } + + klog.Infof("KMM Module %s %s", moduleName, result) + + // Setting Version opts into KMM's ordered upgrade, where KMM loads the module + // only onto nodes carrying a matching version-module label. Flag this so admins + // aren't left wondering why the driver silently isn't loading (see + // KernelModuleSpec.Version). + if cp.Spec.KernelModule.Version != "" { + klog.Infof("KMM Module %s uses ordered upgrade (version=%q); nodes must be labeled "+ + "kmm.node.kubernetes.io/version-module.%s.%s=%s before the module loads", + moduleName, cp.Spec.KernelModule.Version, r.Opts.Namespace, moduleName, cp.Spec.KernelModule.Version) + } + + return ctrl.Result{}, nil +} + +// recreateOnVersionToggle deletes an existing Module when its +// moduleLoader.container.version is being toggled to or from empty, which KMM's +// webhook rejects as an in-place update. It returns done=true when the caller +// should stop and requeue; the next reconcile recreates the Module with the new +// version once the old one is gone. +func (r *KMMReconciler) recreateOnVersionToggle(ctx context.Context, cp *v1alpha.ClusterPolicy, name string) (ctrl.Result, bool, error) { + existing := &kmmv1beta1.Module{} + key := types.NamespacedName{Name: name, Namespace: r.Opts.Namespace} + if err := r.Get(ctx, key, existing); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, false, nil + } + return ctrl.Result{}, false, fmt.Errorf("failed to get KMM Module %s: %w", name, err) + } + + var currentVersion string + if existing.Spec.ModuleLoader != nil { + currentVersion = existing.Spec.ModuleLoader.Container.Version + } + + // Same emptiness (both set or both empty) means an in-place patch is allowed. + if (currentVersion == "") == (cp.Spec.KernelModule.Version == "") { + return ctrl.Result{}, false, nil + } + + if !existing.DeletionTimestamp.IsZero() { + // Deletion already underway; wait for it to finish before recreating. + cp.Status.KMMStatus = "Recreating" + return ctrl.Result{RequeueAfter: r.Opts.RequeueDelay}, true, nil + } + + // Recreating unloads the driver, so don't yank it out from under in-use GPUs. + if r.Opts.DRAEnable && anyAllocatedResourceClaims(ctx, r.Client, gpuDeviceClass) { + addIfMissing(&cp.Status.Errors, + fmt.Sprintf("allocated GPU ResourceClaims blocking modules.kmm.sigs.x-k8s.io/%s recreation for version change", + name)) + return ctrl.Result{RequeueAfter: r.Opts.RequeueDelay}, true, requeueReconcileErr{} + } + + klog.Infof("KMM Module %s version toggled (%q -> %q); recreating", name, currentVersion, cp.Spec.KernelModule.Version) + if err := r.Delete(ctx, existing); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, false, fmt.Errorf("failed to delete KMM Module %s for version change: %w", name, err) + } + + cp.Status.KMMStatus = "Recreating" + return ctrl.Result{RequeueAfter: r.Opts.RequeueDelay}, true, nil +} + +func (r *KMMReconciler) setModuleDesiredState(mod *kmmv1beta1.Module, cp *v1alpha.ClusterPolicy) error { + if err := ctrl.SetControllerReference(cp, mod, r.Scheme); err != nil { + return fmt.Errorf("failed to set controller reference: %w", err) + } + + mod.Spec.Selector = generateNodeSelector(cp, r.Opts) + mod.Spec.Tolerations = generateTolerations(cp) + mod.Spec.ImageRepoSecret = cp.Spec.PullSecret + + r.setModuleLoader(mod, cp) + + return nil +} + +func (r *KMMReconciler) setModuleLoader(mod *kmmv1beta1.Module, cp *v1alpha.ClusterPolicy) { + km := cp.Spec.KernelModule + + container := kmmv1beta1.ModuleLoaderContainerSpec{ + Modprobe: kmmv1beta1.ModprobeSpec{ + ModuleName: km.ModuleName, + FirmwarePath: km.FirmwarePath, + ModulesLoadingOrder: km.ModulesLoadingOrder, + }, + Version: km.Version, + InTreeModulesToRemove: []string{km.ModuleName}, + ImagePullPolicy: v1.PullIfNotPresent, + } + + if km.RegistryTLS != nil { + container.RegistryTLS = kmmv1beta1.TLSOptions{ + Insecure: km.RegistryTLS.Insecure, + InsecureSkipTLSVerify: km.RegistryTLS.InsecureSkipTLSVerify, + } + } + + mappings := make([]kmmv1beta1.KernelMapping, 0, len(km.KernelMappings)) + for _, m := range km.KernelMappings { + mapping := kmmv1beta1.KernelMapping{ + Regexp: m.Regexp, + ContainerImage: m.ContainerImage, + } + if len(m.InTreeModulesToRemove) > 0 { + mapping.InTreeModulesToRemove = dedupeStrings( + append([]string{km.ModuleName}, m.InTreeModulesToRemove...)) + } + if m.RegistryTLS != nil { + mapping.RegistryTLS = &kmmv1beta1.TLSOptions{ + Insecure: m.RegistryTLS.Insecure, + InsecureSkipTLSVerify: m.RegistryTLS.InsecureSkipTLSVerify, + } + } + if m.Build != nil { + mapping.Build = convertBuildSpec(m.Build, mod.Spec.Selector) + } + mappings = append(mappings, mapping) + } + container.KernelMappings = mappings + + mod.Spec.ModuleLoader = &kmmv1beta1.ModuleLoaderSpec{ + Container: container, + ServiceAccountName: r.Opts.ModuleLoaderServiceAccountName, + } +} + +func convertBuildSpec(src *v1alpha.KernelModuleBuildSpec, selector map[string]string) *kmmv1beta1.Build { + build := &kmmv1beta1.Build{ + DockerfileConfigMap: &v1.LocalObjectReference{Name: src.DockerfileConfigMap.Name}, + Secrets: append([]v1.LocalObjectReference{}, src.Secrets...), + // Build on the same nodes the module targets: in-cluster driver builds + // need the GPU nodes' kernel headers/toolchain to compile against. + Selector: selector, + } + + if len(src.BuildArgs) > 0 { + args := make([]kmmv1beta1.BuildArg, len(src.BuildArgs)) + for i, a := range src.BuildArgs { + args[i] = kmmv1beta1.BuildArg{Name: a.Name, Value: a.Value} + } + build.BuildArgs = args + } + + return build +} + +func dedupeStrings(s []string) []string { + seen := make(map[string]bool, len(s)) + result := make([]string, 0, len(s)) + + for _, v := range s { + if !seen[v] { + seen[v] = true + result = append(result, v) + } + } + + return result +} + +func (r *KMMReconciler) deleteModuleIfExists(ctx context.Context, cp *v1alpha.ClusterPolicy, name string) (ctrl.Result, error) { + mod := &kmmv1beta1.Module{} + key := types.NamespacedName{Name: name, Namespace: r.Opts.Namespace} + + if err := r.Get(ctx, key, mod); err != nil { + if apierrors.IsNotFound(err) { + if cp != nil { + cp.Status.KMMStatus = notAvailableStatus + } + return ctrl.Result{}, nil + } + + return ctrl.Result{}, fmt.Errorf("failed to get KMM Module %s: %w", name, err) + } + + if r.Opts.DRAEnable && anyAllocatedResourceClaims(ctx, r.Client, gpuDeviceClass) { + if cp != nil { + addIfMissing(&cp.Status.Errors, + fmt.Sprintf("allocated GPU ResourceClaims blocking modules.kmm.sigs.x-k8s.io/%s deletion", + name)) + } + return ctrl.Result{RequeueAfter: r.Opts.RequeueDelay}, requeueReconcileErr{} + } + + if !mod.DeletionTimestamp.IsZero() { + if cp != nil { + cp.Status.KMMStatus = "Removing" + addIfMissing(&cp.Status.Errors, + fmt.Sprintf("modules.kmm.sigs.x-k8s.io/%s is pending deletion", + name)) + } + return ctrl.Result{RequeueAfter: r.Opts.RequeueDelay}, nil + } + + klog.Infof("Deleting KMM Module %s", name) + + if err := r.Delete(ctx, mod); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, fmt.Errorf("failed to delete KMM Module %s: %w", name, err) + } + + return ctrl.Result{}, nil +} + +func (r *KMMReconciler) updateStatus(cp *v1alpha.ClusterPolicy, mod *kmmv1beta1.Module) { + if cp.Spec.KernelModule != nil { + mlStatus := mod.Status.ModuleLoader + cp.Status.KMMStatus = fmt.Sprintf("%d/%d", mlStatus.AvailableNumber, mlStatus.DesiredNumber) + + if mlStatus.DesiredNumber > 0 && mlStatus.AvailableNumber < mlStatus.DesiredNumber { + addIfMissing(&cp.Status.Errors, + fmt.Sprintf("module loader not fully available (%d/%d) for modules.kmm.sigs.x-k8s.io/%s", + mlStatus.AvailableNumber, mlStatus.DesiredNumber, mod.Name)) + } + } else { + cp.Status.KMMStatus = notAvailableStatus + } +} diff --git a/internal/controller/kmm_controller_test.go b/internal/controller/kmm_controller_test.go new file mode 100644 index 0000000..dee0a5c --- /dev/null +++ b/internal/controller/kmm_controller_test.go @@ -0,0 +1,1273 @@ +/* +Copyright 2026 Intel Corporation. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + stderrors "errors" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + resourcev1 "k8s.io/api/resource/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + v1alpha "github.com/intel/gpu-base-operator/api/v1alpha1" + kmmv1beta1 "github.com/kubernetes-sigs/kernel-module-management/api/v1beta1" +) + +var _ = Describe("KMM Controller", func() { + Context("When creating a Module with KernelModule spec", func() { + const ( + namespace = "kmm-oot-create" + resourceName = "kmm-oot-create" + ) + + ctx := context.Background() + typeNamespacedName := types.NamespacedName{Name: resourceName} + + BeforeEach(func() { + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: namespace}, + })).To(Succeed()) + }) + + AfterEach(func() { + resource := &v1alpha.ClusterPolicy{} + if err := k8sClient.Get(ctx, typeNamespacedName, resource); err == nil { + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + } + }) + + It("should create a KMM Module CR with moduleLoader", func() { + cp := &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName}, + Spec: v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + } + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := &ClusterPolicyReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + Namespace: namespace, + KMMEnable: true, + ModuleLoaderServiceAccountName: "intel-gpu-module-loader", + }, + } + + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + + By("verifying KMM Module was created") + mod := &kmmv1beta1.Module{} + modKey := types.NamespacedName{Name: resourceName + kmmModuleSuffix, Namespace: namespace} + Expect(k8sClient.Get(ctx, modKey, mod)).To(Succeed()) + + By("verifying ModuleLoader is set with correct modprobe config") + Expect(mod.Spec.ModuleLoader).NotTo(BeNil()) + Expect(mod.Spec.ModuleLoader.Container.Modprobe.ModuleName).To(Equal("xe")) + Expect(mod.Spec.ModuleLoader.Container.ContainerImage).To(BeEmpty()) + Expect(mod.Spec.ModuleLoader.Container.InTreeModulesToRemove).To(Equal([]string{"xe"})) + Expect(mod.Spec.ModuleLoader.Container.KernelMappings).To(HaveLen(1)) + Expect(mod.Spec.ModuleLoader.Container.KernelMappings[0].Regexp).To(Equal("^.+$")) + Expect(mod.Spec.ModuleLoader.Container.KernelMappings[0].ContainerImage).To(Equal("registry.example.com/xe-driver:1.0")) + + By("verifying no DRA or DevicePlugin specs are set") + Expect(mod.Spec.DRA).To(BeNil()) + Expect(mod.Spec.DevicePlugin).To(BeNil()) + + By("verifying owner reference is set") + Expect(mod.OwnerReferences).To(HaveLen(1)) + Expect(mod.OwnerReferences[0].Name).To(Equal(resourceName)) + }) + }) + + Context("When KernelModule is nil", func() { + const ( + namespace = "kmm-no-km" + resourceName = "kmm-no-km" + ) + + ctx := context.Background() + typeNamespacedName := types.NamespacedName{Name: resourceName} + + BeforeEach(func() { + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: namespace}, + })).To(Succeed()) + }) + + AfterEach(func() { + resource := &v1alpha.ClusterPolicy{} + if err := k8sClient.Get(ctx, typeNamespacedName, resource); err == nil { + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + } + }) + + It("should not create a Module CR", func() { + cp := &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName}, + Spec: v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + } + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := &ClusterPolicyReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + Namespace: namespace, + KMMEnable: true, + ModuleLoaderServiceAccountName: "intel-gpu-module-loader", + }, + } + + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + + mod := &kmmv1beta1.Module{} + modKey := types.NamespacedName{Name: resourceName + kmmModuleSuffix, Namespace: namespace} + err = k8sClient.Get(ctx, modKey, mod) + Expect(errors.IsNotFound(err)).To(BeTrue()) + }) + }) + + Context("When KMM is not enabled", func() { + const ( + namespace = "kmm-not-enabled" + resourceName = "kmm-not-enabled" + ) + + ctx := context.Background() + typeNamespacedName := types.NamespacedName{Name: resourceName} + + BeforeEach(func() { + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: namespace}, + })).To(Succeed()) + }) + + AfterEach(func() { + resource := &v1alpha.ClusterPolicy{} + if err := k8sClient.Get(ctx, typeNamespacedName, resource); err == nil { + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + } + }) + + It("should set kmmNotEnabledMsg error and not create a Module", func() { + cp := &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName}, + Spec: v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + } + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := &ClusterPolicyReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + Namespace: namespace, + KMMEnable: false, + ModuleLoaderServiceAccountName: "intel-gpu-module-loader", + }, + } + + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + + By("verifying status has kmmNotEnabledMsg") + Expect(k8sClient.Get(ctx, typeNamespacedName, cp)).To(Succeed()) + Expect(cp.Status.Errors).To(ContainElement(kmmNotEnabledMsg)) + + By("verifying no Module was created") + mod := &kmmv1beta1.Module{} + modKey := types.NamespacedName{Name: resourceName + kmmModuleSuffix, Namespace: namespace} + err = k8sClient.Get(ctx, modKey, mod) + Expect(errors.IsNotFound(err)).To(BeTrue()) + }) + }) + + Context("When a status error condition resolves", func() { + const ( + namespace = "kmm-clear-err" + resourceName = "kmm-clear-err" + ) + + ctx := context.Background() + typeNamespacedName := types.NamespacedName{Name: resourceName} + + BeforeEach(func() { + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: namespace}, + })).To(Succeed()) + }) + + AfterEach(func() { + resource := &v1alpha.ClusterPolicy{} + if err := k8sClient.Get(ctx, typeNamespacedName, resource); err == nil { + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + } + }) + + It("clears a stale error on the next reconcile once the condition is gone", func() { + cp := &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName}, + Spec: v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + } + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + // KMM disabled + KernelModule set => kmmNotEnabledMsg is recorded. + reconciler := &ClusterPolicyReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + Namespace: namespace, + KMMEnable: false, + ModuleLoaderServiceAccountName: "intel-gpu-module-loader", + }, + } + + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + + By("verifying the error is present") + Expect(k8sClient.Get(ctx, typeNamespacedName, cp)).To(Succeed()) + Expect(cp.Status.Errors).To(ContainElement(kmmNotEnabledMsg)) + + By("removing KernelModule so the condition no longer applies") + cp.Spec.KernelModule = nil + Expect(k8sClient.Update(ctx, cp)).To(Succeed()) + + _, err = reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + + By("verifying the stale error was cleared") + Expect(k8sClient.Get(ctx, typeNamespacedName, cp)).To(Succeed()) + Expect(cp.Status.Errors).NotTo(ContainElement(kmmNotEnabledMsg)) + }) + }) + + Context("When KernelModule is removed from ClusterPolicy", func() { + const ( + namespace = "kmm-remove-km" + resourceName = "kmm-remove-km" + ) + + ctx := context.Background() + typeNamespacedName := types.NamespacedName{Name: resourceName} + + BeforeEach(func() { + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: namespace}, + })).To(Succeed()) + }) + + AfterEach(func() { + resource := &v1alpha.ClusterPolicy{} + if err := k8sClient.Get(ctx, typeNamespacedName, resource); err == nil { + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + } + }) + + It("should delete the Module when KernelModule is nilled out", func() { + cp := &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName}, + Spec: v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + } + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := &ClusterPolicyReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + Namespace: namespace, + KMMEnable: true, + ModuleLoaderServiceAccountName: "intel-gpu-module-loader", + }, + } + + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + + modKey := types.NamespacedName{Name: resourceName + kmmModuleSuffix, Namespace: namespace} + Expect(k8sClient.Get(ctx, modKey, &kmmv1beta1.Module{})).To(Succeed()) + + By("removing KernelModule from ClusterPolicy") + Expect(k8sClient.Get(ctx, typeNamespacedName, cp)).To(Succeed()) + cp.Spec.KernelModule = nil + Expect(k8sClient.Update(ctx, cp)).To(Succeed()) + + _, err = reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + + By("verifying Module was deleted") + err = k8sClient.Get(ctx, modKey, &kmmv1beta1.Module{}) + Expect(errors.IsNotFound(err)).To(BeTrue()) + }) + }) + + Context("When ClusterPolicy is deleted", func() { + const ( + namespace = "kmm-del-cp" + resourceName = "kmm-del-cp" + ) + + ctx := context.Background() + typeNamespacedName := types.NamespacedName{Name: resourceName} + + BeforeEach(func() { + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: namespace}, + })).To(Succeed()) + }) + + AfterEach(func() { + resource := &v1alpha.ClusterPolicy{} + if err := k8sClient.Get(ctx, typeNamespacedName, resource); err == nil { + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + } + }) + + It("should delete Module when ClusterPolicy is deleted", func() { + cp := &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName}, + Spec: v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + } + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := &ClusterPolicyReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + Namespace: namespace, + KMMEnable: true, + ModuleLoaderServiceAccountName: "intel-gpu-module-loader", + }, + } + + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + + modKey := types.NamespacedName{Name: resourceName + kmmModuleSuffix, Namespace: namespace} + Expect(k8sClient.Get(ctx, modKey, &kmmv1beta1.Module{})).To(Succeed()) + + By("deleting the ClusterPolicy") + Expect(k8sClient.Delete(ctx, cp)).To(Succeed()) + + By("reconciling the deletion via KMM reconciler directly") + kmmReconciler := &KMMReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + ReqName: resourceName, + Namespace: namespace, + KMMEnable: true, + }, + } + _, err = kmmReconciler.Reconcile(ctx, nil) + Expect(err).NotTo(HaveOccurred()) + + By("verifying Module was deleted") + err = k8sClient.Get(ctx, modKey, &kmmv1beta1.Module{}) + Expect(errors.IsNotFound(err)).To(BeTrue()) + }) + }) + + Context("When ClusterPolicy has tolerations and pull secret", func() { + const ( + namespace = "kmm-opts-create" + resourceName = "kmm-opts-create" + ) + + ctx := context.Background() + typeNamespacedName := types.NamespacedName{Name: resourceName} + + BeforeEach(func() { + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: namespace}, + })).To(Succeed()) + }) + + AfterEach(func() { + resource := &v1alpha.ClusterPolicy{} + if err := k8sClient.Get(ctx, typeNamespacedName, resource); err == nil { + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + } + }) + + It("should propagate tolerations and pull secret to Module", func() { + cp := &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName}, + Spec: v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + Tolerations: []v1.Toleration{ + { + Key: "gpu-dedicated", + Operator: v1.TolerationOpExists, + Effect: v1.TaintEffectNoSchedule, + }, + }, + PullSecret: &v1.LocalObjectReference{ + Name: "my-registry-secret", + }, + }, + } + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := &ClusterPolicyReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + Namespace: namespace, + KMMEnable: true, + ModuleLoaderServiceAccountName: "intel-gpu-module-loader", + }, + } + + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + + mod := &kmmv1beta1.Module{} + modKey := types.NamespacedName{Name: resourceName + kmmModuleSuffix, Namespace: namespace} + Expect(k8sClient.Get(ctx, modKey, mod)).To(Succeed()) + + By("verifying tolerations") + Expect(mod.Spec.Tolerations).To(HaveLen(1)) + Expect(mod.Spec.Tolerations[0].Key).To(Equal("gpu-dedicated")) + + By("verifying pull secret") + Expect(mod.Spec.ImageRepoSecret).NotTo(BeNil()) + Expect(mod.Spec.ImageRepoSecret.Name).To(Equal("my-registry-secret")) + }) + }) + + DescribeTable("expanded KernelModule fields", func( + ns string, + cpSpec v1alpha.ClusterPolicySpec, + assertFn func(mod *kmmv1beta1.Module, cp *v1alpha.ClusterPolicy), + ) { + ctx := context.Background() + resName := ns + + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: ns}, + })).To(Succeed()) + + cp := &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: resName}, + Spec: cpSpec, + } + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := &ClusterPolicyReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + Namespace: ns, + KMMEnable: true, + ModuleLoaderServiceAccountName: "intel-gpu-module-loader", + }, + } + _, err := reconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{Name: resName}, + }) + Expect(err).NotTo(HaveOccurred()) + + mod := &kmmv1beta1.Module{} + modKey := types.NamespacedName{Name: resName + kmmModuleSuffix, Namespace: ns} + Expect(k8sClient.Get(ctx, modKey, mod)).To(Succeed()) + + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: resName}, cp)).To(Succeed()) + + assertFn(mod, cp) + }, + Entry("should configure ModuleLoader with multiple kernel mappings", + "kmm-exp-multi", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^5\\.14\\.0-.*\\.el9", ContainerImage: "registry.example.com/xe-rhel9:1.0"}, + {Regexp: "^6\\.12\\..*", ContainerImage: "registry.example.com/xe-rhel10:1.0"}, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + Expect(mod.Spec.ModuleLoader.Container.KernelMappings).To(HaveLen(2)) + Expect(mod.Spec.ModuleLoader.Container.KernelMappings[0].Regexp).To(Equal("^5\\.14\\.0-.*\\.el9")) + Expect(mod.Spec.ModuleLoader.Container.KernelMappings[0].ContainerImage).To(Equal("registry.example.com/xe-rhel9:1.0")) + Expect(mod.Spec.ModuleLoader.Container.KernelMappings[1].Regexp).To(Equal("^6\\.12\\..*")) + Expect(mod.Spec.ModuleLoader.Container.KernelMappings[1].ContainerImage).To(Equal("registry.example.com/xe-rhel10:1.0")) + Expect(mod.Spec.ModuleLoader.Container.ContainerImage).To(BeEmpty()) + }, + ), + Entry("should set Version on container spec", + "kmm-exp-version", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + Version: "2.0", + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:2.0"}, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + Expect(mod.Spec.ModuleLoader.Container.Version).To(Equal("2.0")) + }, + ), + Entry("should configure Build on kernel mapping", + "kmm-exp-build", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + { + Regexp: "^5\\.14\\..*", + Build: &v1alpha.KernelModuleBuildSpec{ + DockerfileConfigMap: v1.LocalObjectReference{Name: "xe-dockerfile"}, + BuildArgs: []v1alpha.BuildArg{{Name: "XE_TAG", Value: "v1.0"}}, + Secrets: []v1.LocalObjectReference{{Name: "private-repo"}}, + }, + }, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + kmmBuild := mod.Spec.ModuleLoader.Container.KernelMappings[0].Build + Expect(kmmBuild).NotTo(BeNil()) + Expect(kmmBuild.DockerfileConfigMap).NotTo(BeNil()) + Expect(kmmBuild.DockerfileConfigMap.Name).To(Equal("xe-dockerfile")) + Expect(kmmBuild.BuildArgs).To(HaveLen(1)) + Expect(kmmBuild.BuildArgs[0].Name).To(Equal("XE_TAG")) + Expect(kmmBuild.BuildArgs[0].Value).To(Equal("v1.0")) + Expect(kmmBuild.Secrets).To(HaveLen(1)) + Expect(kmmBuild.Secrets[0].Name).To(Equal("private-repo")) + // The build must run on the same nodes the module targets. + Expect(kmmBuild.Selector).To(Equal(mod.Spec.Selector)) + }, + ), + Entry("should propagate FirmwarePath to ModprobeSpec", + "kmm-exp-firmware", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + FirmwarePath: "/opt/lib/firmware/xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + Expect(mod.Spec.ModuleLoader.Container.Modprobe.FirmwarePath).To(Equal("/opt/lib/firmware/xe")) + }, + ), + Entry("should propagate ModulesLoadingOrder to ModprobeSpec", + "kmm-exp-loadorder", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + ModulesLoadingOrder: []string{"xe", "drm_buddy", "drm_ttm_helper"}, + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + Expect(mod.Spec.ModuleLoader.Container.Modprobe.ModulesLoadingOrder).To(Equal([]string{"xe", "drm_buddy", "drm_ttm_helper"})) + }, + ), + Entry("should map RegistryTLS to container-level TLSOptions", + "kmm-exp-tls", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + RegistryTLS: &v1alpha.RegistryTLSSpec{Insecure: true, InsecureSkipTLSVerify: true}, + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + Expect(mod.Spec.ModuleLoader.Container.RegistryTLS.Insecure).To(BeTrue()) + Expect(mod.Spec.ModuleLoader.Container.RegistryTLS.InsecureSkipTLSVerify).To(BeTrue()) + }, + ), + Entry("should map per-mapping RegistryTLS to KernelMapping TLSOptions", + "kmm-exp-tls-map", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + { + Regexp: "^.+$", + ContainerImage: "registry.example.com/xe-driver:1.0", + RegistryTLS: &v1alpha.RegistryTLSSpec{InsecureSkipTLSVerify: true}, + }, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + Expect(mod.Spec.ModuleLoader.Container.KernelMappings[0].RegistryTLS).NotTo(BeNil()) + Expect(mod.Spec.ModuleLoader.Container.KernelMappings[0].RegistryTLS.InsecureSkipTLSVerify).To(BeTrue()) + }, + ), + Entry("should set container-level InTreeModulesToRemove to [ModuleName]", + "kmm-exp-intree", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + Expect(mod.Spec.ModuleLoader.Container.InTreeModulesToRemove).To(Equal([]string{"xe"})) + Expect(mod.Spec.ModuleLoader.Container.KernelMappings[0].InTreeModulesToRemove).To(BeEmpty()) + }, + ), + Entry("should auto-prepend ModuleName to per-mapping InTreeModulesToRemove", + "kmm-exp-dedup", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + { + Regexp: "^.+$", + ContainerImage: "registry.example.com/xe-driver:1.0", + InTreeModulesToRemove: []string{"i915"}, + }, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + Expect(mod.Spec.ModuleLoader.Container.KernelMappings[0].InTreeModulesToRemove).To(Equal([]string{"xe", "i915"})) + }, + ), + Entry("should set per-mapping InTreeModulesToRemove override", + "kmm-exp-permapping", + v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + { + Regexp: "^5\\.14\\..*", + ContainerImage: "registry.example.com/xe:1.0", + InTreeModulesToRemove: []string{"old_xe"}, + }, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + func(mod *kmmv1beta1.Module, _ *v1alpha.ClusterPolicy) { + Expect(mod.Spec.ModuleLoader.Container.KernelMappings[0].InTreeModulesToRemove).To(Equal([]string{"xe", "old_xe"})) + Expect(mod.Spec.ModuleLoader.Container.InTreeModulesToRemove).To(Equal([]string{"xe"})) + }, + ), + ) + + Context("KMM ready-label gating", func() { + It("should not add KMM ready label to the Module selector", func() { + ns := "kmm-label-gate" + resName := ns + ctx := context.Background() + + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: ns}, + })).To(Succeed()) + + cp := &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: resName}, + Spec: v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + } + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := &ClusterPolicyReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + Namespace: ns, + KMMEnable: true, + ModuleLoaderServiceAccountName: "intel-gpu-module-loader", + }, + } + + _, err := reconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{Name: resName}, + }) + Expect(err).NotTo(HaveOccurred()) + + mod := &kmmv1beta1.Module{} + modKey := types.NamespacedName{Name: resName + kmmModuleSuffix, Namespace: ns} + Expect(k8sClient.Get(ctx, modKey, mod)).To(Succeed()) + + readyLabel := "kmm.node.kubernetes.io/" + ns + "." + resName + kmmModuleSuffix + ".ready" + Expect(mod.Spec.Selector).NotTo(HaveKey(readyLabel), + "KMM Module selector must not gate on its own ready label") + }) + + It("should add KMM ready label to generateNodeSelector when KernelModule is set", func() { + readyLabel := "kmm.node.kubernetes.io/test-ns.test-cp-gpu.ready" + cp := &v1alpha.ClusterPolicy{ + Spec: v1alpha.ClusterPolicySpec{ + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe:1.0"}, + }, + }, + }, + } + opts := ControllerOpts{KMMModuleReadyLabel: readyLabel} + ns := generateNodeSelector(cp, opts) + Expect(ns).To(HaveKeyWithValue(readyLabel, "")) + }) + + It("should not add KMM ready label when KernelModule is nil", func() { + readyLabel := "kmm.node.kubernetes.io/test-ns.test-cp-gpu.ready" + cp := &v1alpha.ClusterPolicy{} + opts := ControllerOpts{KMMModuleReadyLabel: readyLabel} + ns := generateNodeSelector(cp, opts) + Expect(ns).NotTo(HaveKey(readyLabel)) + }) + + It("should not add KMM ready label when KMMModuleReadyLabel is empty", func() { + cp := &v1alpha.ClusterPolicy{ + Spec: v1alpha.ClusterPolicySpec{ + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe:1.0"}, + }, + }, + }, + } + opts := ControllerOpts{} + ns := generateNodeSelector(cp, opts) + for k := range ns { + Expect(k).NotTo(ContainSubstring("kmm.node.kubernetes.io")) + } + }) + }) + + Context("When toggling container.version between empty and non-empty", func() { + newReconciler := func(ns string) *KMMReconciler { + return &KMMReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + ReqName: ns, + Namespace: ns, + KMMEnable: true, + RequeueDelay: time.Millisecond, + ModuleLoaderServiceAccountName: "module-loader", + }, + } + } + + cpWithVersion := func(name, version string) *v1alpha.ClusterPolicy { + return &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + Version: version, + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + }, + }, + } + } + + It("recreates the Module when version is added to an existing (empty-version) Module", func() { + ns := "kmm-ver-add" + ctx := context.Background() + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: ns}, + })).To(Succeed()) + + cp := cpWithVersion(ns, "") + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := newReconciler(ns) + modKey := types.NamespacedName{Name: ns + kmmModuleSuffix, Namespace: ns} + + _, err := reconciler.Reconcile(ctx, cp) + Expect(err).NotTo(HaveOccurred()) + + mod := &kmmv1beta1.Module{} + Expect(k8sClient.Get(ctx, modKey, mod)).To(Succeed()) + Expect(mod.Spec.ModuleLoader.Container.Version).To(BeEmpty()) + originalUID := mod.UID + + // Add a version: the reconcile should delete the Module and requeue + // rather than attempt an in-place patch KMM would reject. + cp.Spec.KernelModule.Version = "1.0" + res, err := reconciler.Reconcile(ctx, cp) + Expect(err).NotTo(HaveOccurred()) + Expect(res.RequeueAfter).To(BeNumerically(">", 0)) + Expect(cp.Status.KMMStatus).To(Equal("Recreating")) + Expect(errors.IsNotFound(k8sClient.Get(ctx, modKey, &kmmv1beta1.Module{}))).To(BeTrue()) + + // The next reconcile recreates the Module with the new version. + _, err = reconciler.Reconcile(ctx, cp) + Expect(err).NotTo(HaveOccurred()) + recreated := &kmmv1beta1.Module{} + Expect(k8sClient.Get(ctx, modKey, recreated)).To(Succeed()) + Expect(recreated.Spec.ModuleLoader.Container.Version).To(Equal("1.0")) + Expect(recreated.UID).NotTo(Equal(originalUID)) + }) + + It("recreates the Module when version is removed from an existing (versioned) Module", func() { + ns := "kmm-ver-remove" + ctx := context.Background() + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: ns}, + })).To(Succeed()) + + cp := cpWithVersion(ns, "1.0") + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := newReconciler(ns) + modKey := types.NamespacedName{Name: ns + kmmModuleSuffix, Namespace: ns} + + _, err := reconciler.Reconcile(ctx, cp) + Expect(err).NotTo(HaveOccurred()) + + mod := &kmmv1beta1.Module{} + Expect(k8sClient.Get(ctx, modKey, mod)).To(Succeed()) + Expect(mod.Spec.ModuleLoader.Container.Version).To(Equal("1.0")) + originalUID := mod.UID + + cp.Spec.KernelModule.Version = "" + res, err := reconciler.Reconcile(ctx, cp) + Expect(err).NotTo(HaveOccurred()) + Expect(res.RequeueAfter).To(BeNumerically(">", 0)) + Expect(errors.IsNotFound(k8sClient.Get(ctx, modKey, &kmmv1beta1.Module{}))).To(BeTrue()) + + _, err = reconciler.Reconcile(ctx, cp) + Expect(err).NotTo(HaveOccurred()) + recreated := &kmmv1beta1.Module{} + Expect(k8sClient.Get(ctx, modKey, recreated)).To(Succeed()) + Expect(recreated.Spec.ModuleLoader.Container.Version).To(BeEmpty()) + Expect(recreated.UID).NotTo(Equal(originalUID)) + }) + + It("patches in place (no recreate) when version changes between two non-empty values", func() { + ns := "kmm-ver-bump" + ctx := context.Background() + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: ns}, + })).To(Succeed()) + + cp := cpWithVersion(ns, "1.0") + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := newReconciler(ns) + modKey := types.NamespacedName{Name: ns + kmmModuleSuffix, Namespace: ns} + + _, err := reconciler.Reconcile(ctx, cp) + Expect(err).NotTo(HaveOccurred()) + + mod := &kmmv1beta1.Module{} + Expect(k8sClient.Get(ctx, modKey, mod)).To(Succeed()) + originalUID := mod.UID + + cp.Spec.KernelModule.Version = "2.0" + res, err := reconciler.Reconcile(ctx, cp) + Expect(err).NotTo(HaveOccurred()) + Expect(res.RequeueAfter).To(BeZero()) + + updated := &kmmv1beta1.Module{} + Expect(k8sClient.Get(ctx, modKey, updated)).To(Succeed()) + Expect(updated.Spec.ModuleLoader.Container.Version).To(Equal("2.0")) + Expect(updated.UID).To(Equal(originalUID), "Module should be patched in place, not recreated") + }) + + It("blocks the recreate while GPU ResourceClaims are allocated", func() { + ns := "kmm-ver-inuse" + ctx := context.Background() + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: ns}, + })).To(Succeed()) + + cp := cpWithVersion(ns, "") + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := newReconciler(ns) + reconciler.Opts.DRAEnable = true + modKey := types.NamespacedName{Name: ns + kmmModuleSuffix, Namespace: ns} + + _, err := reconciler.Reconcile(ctx, cp) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient.Get(ctx, modKey, &kmmv1beta1.Module{})).To(Succeed()) + + By("allocating a GPU ResourceClaim") + rc := &resourcev1.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{Name: "test-gpu-claim", Namespace: ns}, + Spec: resourcev1.ResourceClaimSpec{ + Devices: resourcev1.DeviceClaim{ + Requests: []resourcev1.DeviceRequest{ + { + Name: "gpu", + FirstAvailable: []resourcev1.DeviceSubRequest{ + { + Name: "gpu-sub", + DeviceClassName: gpuDeviceClass, + AllocationMode: resourcev1.DeviceAllocationModeExactCount, + Count: 1, + }, + }, + }, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, rc)).To(Succeed()) + DeferCleanup(func() { _ = k8sClient.Delete(ctx, rc) }) + + rc.Status.Allocation = &resourcev1.AllocationResult{ + Devices: resourcev1.DeviceAllocationResult{ + Results: []resourcev1.DeviceRequestAllocationResult{ + {Request: "gpu", Driver: gpuDeviceClass, Pool: "node-pool", Device: "gpu-0"}, + }, + }, + } + Expect(k8sClient.Status().Update(ctx, rc)).To(Succeed()) + + By("toggling version while the claim is allocated: recreate must be blocked") + cp.Spec.KernelModule.Version = "1.0" + res, err := reconciler.Reconcile(ctx, cp) + Expect(stderrors.Is(err, requeueReconcileErr{})).To(BeTrue()) + Expect(res.RequeueAfter).To(BeNumerically(">", 0)) + Expect(cp.Status.Errors).To(ContainElement(ContainSubstring("blocking modules.kmm.sigs.x-k8s.io/" + ns + kmmModuleSuffix))) + Expect(k8sClient.Get(ctx, modKey, &kmmv1beta1.Module{})).To(Succeed(), "Module must not be deleted while GPUs are in use") + + By("freeing the claim: the recreate proceeds") + Expect(k8sClient.Delete(ctx, rc)).To(Succeed()) + _, err = reconciler.Reconcile(ctx, cp) + Expect(err).NotTo(HaveOccurred()) + Expect(errors.IsNotFound(k8sClient.Get(ctx, modKey, &kmmv1beta1.Module{}))).To(BeTrue()) + }) + }) + + Context("When deleting Module with allocated ResourceClaims", func() { + const ( + namespace = "kmm-rc-safety" + resourceName = "kmm-rc-safety" + ) + + ctx := context.Background() + typeNamespacedName := types.NamespacedName{Name: resourceName} + + BeforeEach(func() { + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: namespace}, + })).To(Succeed()) + }) + + AfterEach(func() { + resource := &v1alpha.ClusterPolicy{} + if err := k8sClient.Get(ctx, typeNamespacedName, resource); err == nil { + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + } + + var rcList resourcev1.ResourceClaimList + if err := k8sClient.List(ctx, &rcList); err == nil { + for i := range rcList.Items { + _ = k8sClient.Delete(ctx, &rcList.Items[i]) + } + } + }) + + It("should requeue instead of deleting the Module", func() { + cp := &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName}, + Spec: v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + KernelModule: &v1alpha.KernelModuleSpec{ + ModuleName: "xe", + KernelMappings: []v1alpha.KernelMappingSpec{ + {Regexp: "^.+$", ContainerImage: "registry.example.com/xe-driver:1.0"}, + }, + }, + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + } + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := &ClusterPolicyReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + Namespace: namespace, + KMMEnable: true, + DRAEnable: true, + RequeueDelay: 5 * time.Second, + ModuleLoaderServiceAccountName: "intel-gpu-module-loader", + }, + } + + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + + modKey := types.NamespacedName{Name: resourceName + kmmModuleSuffix, Namespace: namespace} + Expect(k8sClient.Get(ctx, modKey, &kmmv1beta1.Module{})).To(Succeed()) + + By("creating an allocated ResourceClaim with a GPU device") + rc := &resourcev1.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-gpu-claim", + Namespace: namespace, + }, + Spec: resourcev1.ResourceClaimSpec{ + Devices: resourcev1.DeviceClaim{ + Requests: []resourcev1.DeviceRequest{ + { + Name: "gpu", + FirstAvailable: []resourcev1.DeviceSubRequest{ + { + Name: "gpu-sub", + DeviceClassName: gpuDeviceClass, + AllocationMode: resourcev1.DeviceAllocationModeExactCount, + Count: 1, + }, + }, + }, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, rc)).To(Succeed()) + + rc.Status.Allocation = &resourcev1.AllocationResult{ + Devices: resourcev1.DeviceAllocationResult{ + Results: []resourcev1.DeviceRequestAllocationResult{ + { + Request: "gpu", + Driver: gpuDeviceClass, + Pool: "node-pool", + Device: "gpu-0", + }, + }, + }, + } + Expect(k8sClient.Status().Update(ctx, rc)).To(Succeed()) + + By("removing KernelModule to trigger deletion") + Expect(k8sClient.Get(ctx, typeNamespacedName, cp)).To(Succeed()) + cp.Spec.KernelModule = nil + Expect(k8sClient.Update(ctx, cp)).To(Succeed()) + + result, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.RequeueAfter).To(Equal(5 * time.Second)) + + By("verifying Module was NOT deleted") + Expect(k8sClient.Get(ctx, modKey, &kmmv1beta1.Module{})).To(Succeed()) + + By("removing the ResourceClaim and reconciling again") + Expect(k8sClient.Delete(ctx, rc)).To(Succeed()) + + result, err = reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.RequeueAfter).To(BeZero()) + + By("verifying Module was deleted") + err = k8sClient.Get(ctx, modKey, &kmmv1beta1.Module{}) + Expect(errors.IsNotFound(err)).To(BeTrue()) + }) + }) + + Context("When KernelModule is nil", func() { + const ( + namespace = "kmm-exp-nkmm" + resourceName = "kmm-exp-nkmm" + ) + + ctx := context.Background() + typeNamespacedName := types.NamespacedName{Name: resourceName} + + BeforeEach(func() { + Expect(k8sClient.Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: namespace}, + })).To(Succeed()) + }) + + AfterEach(func() { + resource := &v1alpha.ClusterPolicy{} + if err := k8sClient.Get(ctx, typeNamespacedName, resource); err == nil { + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + } + }) + + It("should set KMMStatus to N/A", func() { + cp := &v1alpha.ClusterPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName}, + Spec: v1alpha.ClusterPolicySpec{ + ResourceRegistration: "dra", + DynamicResourceAllocationSpec: v1alpha.DynamicResourceAllocationSpec{ + Image: "ghcr.io/intel/gpu-dra:v0.11.0", + }, + }, + } + Expect(k8sClient.Create(ctx, cp)).To(Succeed()) + + reconciler := &ClusterPolicyReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Opts: ControllerOpts{ + Namespace: namespace, + KMMEnable: true, + ModuleLoaderServiceAccountName: "intel-gpu-module-loader", + }, + } + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName}) + Expect(err).NotTo(HaveOccurred()) + + Expect(k8sClient.Get(ctx, typeNamespacedName, cp)).To(Succeed()) + Expect(cp.Status.KMMStatus).To(Equal("N/A")) + }) + }) +}) diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go index 22daa3c..9bf4bc6 100644 --- a/internal/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -18,8 +18,11 @@ package controller import ( "context" + "fmt" "os" + "os/exec" "path/filepath" + "strings" "testing" . "github.com/onsi/ginkgo/v2" @@ -32,6 +35,7 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + kmmv1beta1 "github.com/kubernetes-sigs/kernel-module-management/api/v1beta1" prometheusv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" corev1 "k8s.io/api/core/v1" resv1 "k8s.io/api/resource/v1" @@ -79,6 +83,8 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) err = corev1.AddToScheme(s) Expect(err).NotTo(HaveOccurred()) + err = kmmv1beta1.AddToScheme(s) + Expect(err).NotTo(HaveOccurred()) // +kubebuilder:scaffold:scheme @@ -88,6 +94,7 @@ var _ = BeforeSuite(func() { CRDDirectoryPaths: []string{ filepath.Join("..", "..", "config", "crd", "bases"), filepath.Join("..", "..", "config", "deployments", "nfd", "crds"), + filepath.Join(goModDir("github.com/kubernetes-sigs/kernel-module-management"), "config", "crd", "bases"), "testdata", }, ErrorIfCRDPathMissing: true, @@ -127,6 +134,14 @@ var _ = AfterSuite(func() { // This function streamlines the process by finding the required binaries, similar to // setting the 'KUBEBUILDER_ASSETS' environment variable. To ensure the binaries are // properly set up, run 'make setup-envtest' beforehand. +func goModDir(module string) string { + out, err := exec.Command("go", "list", "-m", "-f", "{{.Dir}}", module).Output() + if err != nil { + panic(fmt.Sprintf("failed to resolve module directory for %s: %v", module, err)) + } + return strings.TrimSpace(string(out)) +} + func getFirstFoundEnvTestBinaryDir() string { basePath := filepath.Join("..", "..", "bin", "k8s") entries, err := os.ReadDir(basePath) diff --git a/internal/controller/xpumanager_controller.go b/internal/controller/xpumanager_controller.go index 5f6506d..5d920f8 100644 --- a/internal/controller/xpumanager_controller.go +++ b/internal/controller/xpumanager_controller.go @@ -407,7 +407,7 @@ func (r *XpuManagerReconciler) updateDaemonSetObject(ds *apps.DaemonSet, spec *v processContainerResources(ds, spec, draClaim) processXpumdConfigMapMount(ds, otelConfigMapName) - ds.Spec.Template.Spec.NodeSelector = generateNodeSelector(spec) + ds.Spec.Template.Spec.NodeSelector = generateNodeSelector(spec, r.Opts) ds.Spec.Template.Spec.Tolerations = generateTolerations(spec) cspec := &ds.Spec.Template.Spec