From bb2dc08fde7b2fbe747b8db7f8c3d962567634cb Mon Sep 17 00:00:00 2001 From: Kethan Gaddam Date: Thu, 20 Aug 2026 20:13:44 +0000 Subject: [PATCH] feat: tolerate HyperPod node-health-status taint in training-operator The kubeflow training-operator Deployment has no tolerations, so it cannot be scheduled onto nodes carrying the HyperPod deep health check taint sagemaker.amazonaws.com/node-health-status=Unschedulable:NoSchedule. On clusters where every node is tainted during an on-start deep health check, the operator pod stays Pending and PyTorchJob-based checks such as MultiNodeTraining never get reconciled. Other controllers and device plugins in this chart already tolerate the same taint (mpi-operator, neuron-device-plugin, nvidia-device-plugin, aws-efa-k8s-device-plugin). This makes training-operators consistent. --- .../Deployment/training-operator-kubeflow-Deployment.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/helm_chart/HyperPodHelmChart/charts/training-operators/templates/Deployment/training-operator-kubeflow-Deployment.yaml b/helm_chart/HyperPodHelmChart/charts/training-operators/templates/Deployment/training-operator-kubeflow-Deployment.yaml index 3ef20b51..07c2b24a 100644 --- a/helm_chart/HyperPodHelmChart/charts/training-operators/templates/Deployment/training-operator-kubeflow-Deployment.yaml +++ b/helm_chart/HyperPodHelmChart/charts/training-operators/templates/Deployment/training-operator-kubeflow-Deployment.yaml @@ -55,4 +55,9 @@ spec: securityContext: allowPrivilegeEscalation: false serviceAccountName: training-operator - terminationGracePeriodSeconds: 10 \ No newline at end of file + terminationGracePeriodSeconds: 10 + tolerations: + - key: sagemaker.amazonaws.com/node-health-status + operator: "Equal" + value: "Unschedulable" + effect: "NoSchedule" \ No newline at end of file