From 5d55485d2bd3446d208beb4718b1766e1e365475 Mon Sep 17 00:00:00 2001 From: Ziyao Huang Date: Fri, 21 Aug 2026 22:17:31 +0000 Subject: [PATCH] Add system-node-critical priorityClass to health-monitoring-agent DaemonSets Under node DiskPressure, HMA was evicted moments before a GPU failure went undetected and unremediated (P438078245). Setting priorityClassName to the built-in system-node-critical makes HMA the last pod evicted (after training and other monitoring pods), preserving health monitoring during node pressure. Mirrors what EKSNodeMonitoringAgent already does. Applied to both the health-monitoring-agent (NVIDIA) and health-monitoring-agent-non-nvidia (Trainium/Inferentia) DaemonSets. Configurable via .Values.priorityClassName, defaulting to system-node-critical. --- .../templates/health-monitoring-agent.yaml | 2 ++ .../charts/health-monitoring-agent/values.yaml | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/helm_chart/HyperPodHelmChart/charts/health-monitoring-agent/templates/health-monitoring-agent.yaml b/helm_chart/HyperPodHelmChart/charts/health-monitoring-agent/templates/health-monitoring-agent.yaml index 301548a0..2d5a430a 100644 --- a/helm_chart/HyperPodHelmChart/charts/health-monitoring-agent/templates/health-monitoring-agent.yaml +++ b/helm_chart/HyperPodHelmChart/charts/health-monitoring-agent/templates/health-monitoring-agent.yaml @@ -159,6 +159,7 @@ spec: - name: localtime mountPath: /etc/localtime readOnly: true + priorityClassName: {{ .Values.priorityClassName | default "system-node-critical" }} serviceAccountName: health-monitoring-agent volumes: - name: log @@ -250,6 +251,7 @@ spec: - name: localtime mountPath: /etc/localtime readOnly: true + priorityClassName: {{ .Values.priorityClassName | default "system-node-critical" }} serviceAccountName: health-monitoring-agent volumes: - name: log diff --git a/helm_chart/HyperPodHelmChart/charts/health-monitoring-agent/values.yaml b/helm_chart/HyperPodHelmChart/charts/health-monitoring-agent/values.yaml index 5ead265e..c666b40a 100644 --- a/helm_chart/HyperPodHelmChart/charts/health-monitoring-agent/values.yaml +++ b/helm_chart/HyperPodHelmChart/charts/health-monitoring-agent/values.yaml @@ -30,3 +30,8 @@ hmaimage: "" # Enable debug output for region selection process debug: true + +# PriorityClass for the HMA DaemonSet pods. +# system-node-critical ensures HMA is the last pod evicted under node pressure +# (e.g. DiskPressure), preventing loss of health monitoring. See P438078245. +priorityClassName: system-node-critical