diff --git a/docs/docs/extraction/releasenotes.md b/docs/docs/extraction/releasenotes.md index fea0c01aa..830ede9fa 100644 --- a/docs/docs/extraction/releasenotes.md +++ b/docs/docs/extraction/releasenotes.md @@ -31,6 +31,7 @@ Highlights for the 26.08 release include: ### Retriever Service and deployment { #retriever-service-and-deployment } - Retriever Service v2 adds a scalable multi-pod architecture with gateway, process isolation, and VectorDB integration +- Fixed split-topology Helm rendering so common `service.*` Pod, container, environment, volume, scheduling, and shutdown settings apply to the gateway, realtime, and batch workloads. `service.securityContext` also applies to worker `wait-for-gateway` init containers. - OpenTelemetry basic support for pipeline and service observability - Expanded air-gapped deployment guidance in [deployment options](deployment-options.md) and the Helm chart README diff --git a/nemo_retriever/helm/README.md b/nemo_retriever/helm/README.md index 7d54bf6ba..c3d56753e 100644 --- a/nemo_retriever/helm/README.md +++ b/nemo_retriever/helm/README.md @@ -403,6 +403,17 @@ short list of knobs you'll touch first. | `service.resources.requests` | `16 / 16Gi` | Tune in tandem with `serviceConfig.pipeline.*Workers`. | | `service.resources.limits` | `96 / 96Gi` | | | `service.gpu.enabled` | `false` | The service does **not** need a GPU. | +| `service.podLabels` / `service.podAnnotations` | `{}` / `{}` | Add labels or annotations to every retriever service Pod. | +| `service.priorityClassName` / `service.topologySpreadConstraints` | `""` / `[]` | Apply common scheduling policy to every retriever service Pod. | +| `service.terminationGracePeriodSeconds` | `60` | Set the shutdown grace period for every retriever service Pod. | +| `service.securityContext` | `{}` | Apply a container security context to every retriever service container. | +| `service.envFrom` | `[]` | Import environment variables from Secrets or ConfigMaps into every retriever service container. | +| `service.extraVolumes` / `service.extraVolumeMounts` | `[]` / `[]` | Add common volumes and mounts to every retriever service Pod and container. | + +In `topology.mode: split`, these common `service.*` settings apply to the +gateway, realtime, and batch Deployments. `service.securityContext` also +applies to the `wait-for-gateway` init containers in the realtime and batch +Deployments. Do not use `service.podLabels` to replace chart selector labels. For audio and video extraction, set `service.installFfmpeg=true` when your cluster allows runtime package installation. **OpenShift restricted-v2** blocks diff --git a/nemo_retriever/helm/openshift.md b/nemo_retriever/helm/openshift.md index 88977542b..dddc572c2 100644 --- a/nemo_retriever/helm/openshift.md +++ b/nemo_retriever/helm/openshift.md @@ -68,6 +68,10 @@ topology: enabled: false ``` +In `topology.mode: split`, `service.securityContext` applies to the gateway, +realtime, and batch service containers. It also applies to the +`wait-for-gateway` init containers in the realtime and batch Deployments. + When **`persistence.enabled=true`**, you can keep the default log path under `persistence.mountPath` (`/var/lib/nemo-retriever`) because the PVC is mounted and SCC-assigned `fsGroup` applies. When persistence is off, always relocate logs to `/tmp` (or another path backed by `service.extraVolumes`). ### Audio and video (ffmpeg) on restricted OpenShift { #audio-and-video-ffmpeg-on-restricted-openshift } @@ -200,7 +204,9 @@ oc get pods -n nemo-retriever oc describe pod -l app.kubernetes.io/name=nemo-retriever -n nemo-retriever ``` -You should see SCC-assigned numeric `runAsUser` on containers that declare a `securityContext` block, and no PSA warnings after overrides are applied. +You should see SCC-assigned numeric `runAsUser` on the service and +`wait-for-gateway` init containers that declare a `securityContext` block, and +no PSA warnings after overrides are applied. ### Example install with NIM Operator (in-cluster NIMs) diff --git a/nemo_retriever/helm/templates/deployment.yaml b/nemo_retriever/helm/templates/deployment.yaml index 98dba7c84..d2da8eccf 100644 --- a/nemo_retriever/helm/templates/deployment.yaml +++ b/nemo_retriever/helm/templates/deployment.yaml @@ -254,16 +254,25 @@ spec: metadata: labels: {{- include "nemo-retriever.role.selectorLabels" (dict "context" $ "role" $role) | nindent 8 }} + {{- with $svc.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} annotations: checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") $ | sha256sum }} + {{- with $svc.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} spec: serviceAccountName: {{ include "nemo-retriever.serviceAccountName" $ }} {{- include "nemo-retriever.imagePullSecrets" $ | nindent 6 }} + {{- with $svc.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} {{- with $svc.podSecurityContext }} securityContext: {{- toYaml . | nindent 8 }} {{- end }} - terminationGracePeriodSeconds: {{ $roleCfg.terminationGracePeriodSeconds | default 15 }} + terminationGracePeriodSeconds: {{ $svc.terminationGracePeriodSeconds }} {{- with $roleCfg.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} @@ -276,10 +285,18 @@ spec: affinity: {{- toYaml . | nindent 8 }} {{- end }} + {{- with $svc.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} {{- if ne $role "gateway" }} initContainers: - name: wait-for-gateway image: busybox:1.37 + {{- with $svc.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} command: - sh - -c @@ -297,6 +314,10 @@ spec: - name: nemo-retriever image: "{{ include "nemo-retriever.runtime.image.repository" $ }}:{{ include "nemo-retriever.runtime.image.tag" $ }}" imagePullPolicy: {{ include "nemo-retriever.runtime.image.pullPolicy" $ }} + {{- with $svc.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} args: - retriever - service @@ -362,6 +383,10 @@ spec: {{- with $svc.env }} {{- toYaml . | nindent 12 }} {{- end }} + {{- with $svc.envFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} resources: {{- $resources := deepCopy $roleCfg.resources }} {{- $roleGpuEnabled := and (hasKey $roleCfg "gpu") (index $roleCfg "gpu").enabled }} @@ -393,6 +418,9 @@ spec: mountPath: {{ $scopeTokenMountPath | quote }} readOnly: true {{- end }} + {{- with $svc.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} {{- if $svc.startupProbe.enabled }} startupProbe: {{- omit $svc.startupProbe "enabled" | toYaml | nindent 12 }} @@ -433,5 +461,8 @@ spec: persistentVolumeClaim: claimName: {{ include "nemo-retriever.retrieverResultsPvcName" $ }} {{- end }} + {{- with $svc.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} {{- end }} {{- end }} diff --git a/nemo_retriever/tests/test_helm_split_common_service_settings.py b/nemo_retriever/tests/test_helm_split_common_service_settings.py new file mode 100644 index 000000000..9e1799cba --- /dev/null +++ b/nemo_retriever/tests/test_helm_split_common_service_settings.py @@ -0,0 +1,100 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Common service settings must apply to every split-topology workload.""" + +from __future__ import annotations + +from tests.test_helm_shared_results import _render, _service_deployments + + +def _main_container(deployment: dict) -> dict: + return next( + container + for container in deployment["spec"]["template"]["spec"]["containers"] + if container["name"] == "nemo-retriever" + ) + + +def test_split_roles_inherit_common_service_settings() -> None: + security_context = { + "allowPrivilegeEscalation": False, + "capabilities": {"drop": ["ALL"]}, + "runAsNonRoot": True, + "seccompProfile": {"type": "RuntimeDefault"}, + } + extra_volume = {"name": "common-settings", "emptyDir": {}} + extra_mount = {"name": "common-settings", "mountPath": "/common-settings", "readOnly": True} + spread_constraint = { + "labelSelector": {"matchLabels": {"example.com/common": "enabled"}}, + "maxSkew": 1, + "topologyKey": "kubernetes.io/hostname", + "whenUnsatisfiable": "ScheduleAnyway", + } + + documents = _render( + "--set", + "topology.mode=split", + "--set", + "service.priorityClassName=common-priority", + "--set", + "service.terminationGracePeriodSeconds=77", + "--set-json", + 'service.podLabels={"example.com/common":"enabled"}', + "--set-json", + 'service.podAnnotations={"example.com/common":"annotation"}', + "--set-json", + ( + 'service.securityContext={"runAsNonRoot":true,' + '"allowPrivilegeEscalation":false,' + '"capabilities":{"drop":["ALL"]},' + '"seccompProfile":{"type":"RuntimeDefault"}}' + ), + "--set-json", + 'service.topologySpreadConstraints=[{"maxSkew":1,' + '"topologyKey":"kubernetes.io/hostname",' + '"whenUnsatisfiable":"ScheduleAnyway",' + '"labelSelector":{"matchLabels":{"example.com/common":"enabled"}}}]', + "--set-json", + 'service.envFrom=[{"configMapRef":{"name":"common-settings"}}]', + "--set-json", + 'service.extraVolumes=[{"name":"common-settings","emptyDir":{}}]', + "--set-json", + 'service.extraVolumeMounts=[{"name":"common-settings",' '"mountPath":"/common-settings","readOnly":true}]', + ) + + deployments = _service_deployments(documents) + assert len(deployments) == 3 + + observed_roles = set() + for deployment in deployments: + role = deployment["metadata"]["labels"]["app.kubernetes.io/component"] + observed_roles.add(role) + pod_template = deployment["spec"]["template"] + pod_spec = pod_template["spec"] + container = _main_container(deployment) + + assert pod_template["metadata"]["labels"]["example.com/common"] == "enabled" + assert pod_template["metadata"]["annotations"]["example.com/common"] == "annotation" + assert pod_spec["priorityClassName"] == "common-priority" + assert pod_spec["terminationGracePeriodSeconds"] == 77 + assert pod_spec["topologySpreadConstraints"] == [spread_constraint] + assert container["securityContext"] == security_context + assert container["envFrom"] == [{"configMapRef": {"name": "common-settings"}}] + assert extra_volume in pod_spec["volumes"] + assert extra_mount in container["volumeMounts"] + + if role == "gateway": + assert "initContainers" not in pod_spec + else: + assert pod_spec["initContainers"] == [ + { + "name": "wait-for-gateway", + "image": "busybox:1.37", + "securityContext": security_context, + "command": pod_spec["initContainers"][0]["command"], + } + ] + + assert observed_roles == {"gateway", "realtime", "batch"}