From f0aa66925cad157365e63d65174dd316e11f4889 Mon Sep 17 00:00:00 2001 From: amarkdotdev Date: Thu, 16 Jul 2026 20:09:49 +0300 Subject: [PATCH] feat: add affinity and nodeSelector to common daemonset config Expose spec.daemonsets.affinity and spec.daemonsets.nodeSelector on ClusterPolicy and wire them through Helm. User nodeSelector labels are merged with existing operand nodeSelector entries. Use a schemaless CRD field for affinity so we do not embed the full corev1.Affinity OpenAPI schema in the ClusterPolicy CRD. Signed-off-by: amarkdotdev --- api/nvidia/v1/clusterpolicy_types.go | 15 +++++ api/nvidia/v1/zz_generated.deepcopy.go | 12 ++++ .../manifests/nvidia.com_clusterpolicies.yaml | 10 +++ .../crd/bases/nvidia.com_clusterpolicies.yaml | 10 +++ controllers/object_controls.go | 13 ++++ controllers/transforms_test.go | 65 +++++++++++++++++++ .../crds/nvidia.com_clusterpolicies.yaml | 10 +++ .../gpu-operator/templates/clusterpolicy.yaml | 6 ++ deployments/gpu-operator/values.yaml | 2 + 9 files changed, 143 insertions(+) diff --git a/api/nvidia/v1/clusterpolicy_types.go b/api/nvidia/v1/clusterpolicy_types.go index 61f97b92ca..65299aa2af 100644 --- a/api/nvidia/v1/clusterpolicy_types.go +++ b/api/nvidia/v1/clusterpolicy_types.go @@ -308,6 +308,21 @@ type DaemonsetsSpec struct { // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:advanced,urn:alm:descriptor:io.kubernetes:Tolerations" Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + // Optional: Set nodeSelector merged with operand nodeSelector labels + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="NodeSelector for all Daemonsets" + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:advanced,urn:alm:descriptor:com.tectonic.ui:nodeSelector" + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + + // Optional: Set affinity for all Daemonsets + // +kubebuilder:validation:Type=object + // +kubebuilder:validation:Schemaless + // +kubebuilder:pruning:PreserveUnknownFields + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Affinity for all Daemonsets" + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:advanced,urn:alm:descriptor:com.tectonic.ui:affinity" + Affinity *corev1.Affinity `json:"affinity,omitempty"` + // +kubebuilder:validation:Optional // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="PriorityClassName" diff --git a/api/nvidia/v1/zz_generated.deepcopy.go b/api/nvidia/v1/zz_generated.deepcopy.go index 9e936de60d..8a01aa39d7 100644 --- a/api/nvidia/v1/zz_generated.deepcopy.go +++ b/api/nvidia/v1/zz_generated.deepcopy.go @@ -525,6 +525,18 @@ func (in *DaemonsetsSpec) DeepCopyInto(out *DaemonsetsSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Affinity != nil { + in, out := &in.Affinity, &out.Affinity + *out = new(corev1.Affinity) + (*in).DeepCopyInto(*out) + } if in.RollingUpdate != nil { in, out := &in.RollingUpdate, &out.RollingUpdate *out = new(RollingUpdateSpec) diff --git a/bundle/manifests/nvidia.com_clusterpolicies.yaml b/bundle/manifests/nvidia.com_clusterpolicies.yaml index c1221b6add..7b9ac2d1d6 100644 --- a/bundle/manifests/nvidia.com_clusterpolicies.yaml +++ b/bundle/manifests/nvidia.com_clusterpolicies.yaml @@ -160,6 +160,10 @@ spec: daemonsets: description: Daemonset defines common configuration for all Daemonsets properties: + affinity: + description: 'Optional: Set affinity for all Daemonsets' + type: object + x-kubernetes-preserve-unknown-fields: true annotations: additionalProperties: type: string @@ -176,6 +180,12 @@ spec: (scope and select) objects. May match selectors of replication controllers and services. type: object + nodeSelector: + additionalProperties: + type: string + description: 'Optional: Set nodeSelector merged with operand nodeSelector + labels' + type: object podSecurityContext: description: 'Optional: Set pod-level security context for all DaemonSet pods (applies as defaults to all containers)' diff --git a/config/crd/bases/nvidia.com_clusterpolicies.yaml b/config/crd/bases/nvidia.com_clusterpolicies.yaml index c1221b6add..7b9ac2d1d6 100644 --- a/config/crd/bases/nvidia.com_clusterpolicies.yaml +++ b/config/crd/bases/nvidia.com_clusterpolicies.yaml @@ -160,6 +160,10 @@ spec: daemonsets: description: Daemonset defines common configuration for all Daemonsets properties: + affinity: + description: 'Optional: Set affinity for all Daemonsets' + type: object + x-kubernetes-preserve-unknown-fields: true annotations: additionalProperties: type: string @@ -176,6 +180,12 @@ spec: (scope and select) objects. May match selectors of replication controllers and services. type: object + nodeSelector: + additionalProperties: + type: string + description: 'Optional: Set nodeSelector merged with operand nodeSelector + labels' + type: object podSecurityContext: description: 'Optional: Set pod-level security context for all DaemonSet pods (applies as defaults to all containers)' diff --git a/controllers/object_controls.go b/controllers/object_controls.go index c20387b9ae..ede74a0dbe 100644 --- a/controllers/object_controls.go +++ b/controllers/object_controls.go @@ -812,6 +812,19 @@ func applyCommonDaemonsetConfig(obj *appsv1.DaemonSet, config *gpuv1.ClusterPoli obj.Spec.Template.Spec.Tolerations = config.Daemonsets.Tolerations } + if len(config.Daemonsets.NodeSelector) > 0 { + if obj.Spec.Template.Spec.NodeSelector == nil { + obj.Spec.Template.Spec.NodeSelector = make(map[string]string) + } + for key, value := range config.Daemonsets.NodeSelector { + obj.Spec.Template.Spec.NodeSelector[key] = value + } + } + + if config.Daemonsets.Affinity != nil { + obj.Spec.Template.Spec.Affinity = config.Daemonsets.Affinity + } + // set pod-level security context if specified (applies as defaults to all containers in the pod) if config.Daemonsets.PodSecurityContext != nil { obj.Spec.Template.Spec.SecurityContext = config.Daemonsets.PodSecurityContext diff --git a/controllers/transforms_test.go b/controllers/transforms_test.go index 9a0881072f..3fa3451b5a 100644 --- a/controllers/transforms_test.go +++ b/controllers/transforms_test.go @@ -149,6 +149,16 @@ func (d Daemonset) WithTolerations(tolerations []corev1.Toleration) Daemonset { return d } +func (d Daemonset) WithNodeSelector(nodeSelector map[string]string) Daemonset { + d.Spec.Template.Spec.NodeSelector = nodeSelector + return d +} + +func (d Daemonset) WithAffinity(affinity *corev1.Affinity) Daemonset { + d.Spec.Template.Spec.Affinity = affinity + return d +} + func (d Daemonset) WithPodSecurityContext(psc *corev1.PodSecurityContext) Daemonset { d.Spec.Template.Spec.SecurityContext = psc return d @@ -689,6 +699,61 @@ func TestApplyCommonDaemonSetConfig(t *testing.T) { }, }), }, + { + description: "nodeSelector configured", + ds: NewDaemonset().WithNodeSelector(map[string]string{ + "nvidia.com/gpu.deploy.device-plugin": "true", + }), + dsSpec: gpuv1.DaemonsetsSpec{ + NodeSelector: map[string]string{ + "karpenter.sh/nodepool": "gpu", + }, + }, + expectedDs: NewDaemonset().WithNodeSelector(map[string]string{ + "nvidia.com/gpu.deploy.device-plugin": "true", + "karpenter.sh/nodepool": "gpu", + }), + }, + { + description: "affinity configured", + ds: NewDaemonset(), + dsSpec: gpuv1.DaemonsetsSpec{ + Affinity: &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "karpenter.sh/nodepool", + Operator: corev1.NodeSelectorOpIn, + Values: []string{"gpu"}, + }, + }, + }, + }, + }, + }, + }, + }, + expectedDs: NewDaemonset().WithAffinity(&corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "karpenter.sh/nodepool", + Operator: corev1.NodeSelectorOpIn, + Values: []string{"gpu"}, + }, + }, + }, + }, + }, + }, + }), + }, { description: "invalid updatestrategy configured", ds: NewDaemonset(), diff --git a/deployments/gpu-operator/crds/nvidia.com_clusterpolicies.yaml b/deployments/gpu-operator/crds/nvidia.com_clusterpolicies.yaml index c1221b6add..7b9ac2d1d6 100644 --- a/deployments/gpu-operator/crds/nvidia.com_clusterpolicies.yaml +++ b/deployments/gpu-operator/crds/nvidia.com_clusterpolicies.yaml @@ -160,6 +160,10 @@ spec: daemonsets: description: Daemonset defines common configuration for all Daemonsets properties: + affinity: + description: 'Optional: Set affinity for all Daemonsets' + type: object + x-kubernetes-preserve-unknown-fields: true annotations: additionalProperties: type: string @@ -176,6 +180,12 @@ spec: (scope and select) objects. May match selectors of replication controllers and services. type: object + nodeSelector: + additionalProperties: + type: string + description: 'Optional: Set nodeSelector merged with operand nodeSelector + labels' + type: object podSecurityContext: description: 'Optional: Set pod-level security context for all DaemonSet pods (applies as defaults to all containers)' diff --git a/deployments/gpu-operator/templates/clusterpolicy.yaml b/deployments/gpu-operator/templates/clusterpolicy.yaml index 3ae8890238..81f862660a 100644 --- a/deployments/gpu-operator/templates/clusterpolicy.yaml +++ b/deployments/gpu-operator/templates/clusterpolicy.yaml @@ -43,6 +43,12 @@ spec: {{- if .Values.daemonsets.tolerations }} tolerations: {{ toYaml .Values.daemonsets.tolerations | nindent 6 }} {{- end }} + {{- if .Values.daemonsets.nodeSelector }} + nodeSelector: {{ toYaml .Values.daemonsets.nodeSelector | nindent 6 }} + {{- end }} + {{- if .Values.daemonsets.affinity }} + affinity: {{ toYaml .Values.daemonsets.affinity | nindent 6 }} + {{- end }} {{- if .Values.daemonsets.priorityClassName }} priorityClassName: {{ .Values.daemonsets.priorityClassName }} {{- end }} diff --git a/deployments/gpu-operator/values.yaml b/deployments/gpu-operator/values.yaml index 47b97ebbc7..6942c9aa37 100644 --- a/deployments/gpu-operator/values.yaml +++ b/deployments/gpu-operator/values.yaml @@ -41,6 +41,8 @@ hostPaths: daemonsets: labels: {} annotations: {} + nodeSelector: {} + affinity: {} priorityClassName: system-node-critical tolerations: - key: nvidia.com/gpu