From 3b635673aaa7500b06fcad9438941648bdb779e3 Mon Sep 17 00:00:00 2001 From: Trevor McKay Date: Fri, 31 Jul 2026 12:44:55 -0400 Subject: [PATCH] add a smoke test to ensure server images start correctly --- .github/workflows/test_images.yaml | 32 ++++++++++ ci/docker/README.md | 14 +++++ ci/docker/smoke_image.sh | 94 ++++++++++++++++++++++++++++++ 3 files changed, 140 insertions(+) create mode 100755 ci/docker/smoke_image.sh diff --git a/.github/workflows/test_images.yaml b/.github/workflows/test_images.yaml index afbbab4382..3767e64566 100644 --- a/.github/workflows/test_images.yaml +++ b/.github/workflows/test_images.yaml @@ -87,3 +87,35 @@ jobs: - name: Test cuopt run: | bash ./ci/docker/test_image.sh + + # Host-side startup smoke: exercises ENTRYPOINT/CMD (REST) and + # CUOPT_SERVER_TYPE=grpc. The jobs above run *inside* the image as a GHA + # container and never launch the servers, so they cannot catch packaging + # gaps such as UBI10's RHEL lib/lib64 NCCL path miss. + smoke: + name: smoke-images (${{ inputs.ARCH }}, cuda${{ needs.prepare.outputs.CUDA_SHORT }}) + runs-on: "linux-${{ inputs.ARCH }}-gpu-a100-latest-1" + needs: prepare + steps: + - name: Checkout code repo + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + fetch-depth: 0 + ref: ${{ inputs.sha }} + persist-credentials: false + - name: Smoke Ubuntu image (REST + gRPC) + env: + IMAGE_TAG_PREFIX: ${{ inputs.IMAGE_TAG_PREFIX }} + CUDA_SHORT: ${{ needs.prepare.outputs.CUDA_SHORT }} + PYTHON_SHORT: ${{ needs.prepare.outputs.PYTHON_SHORT }} + run: | + bash ./ci/docker/smoke_image.sh \ + "nvidia/cuopt:${IMAGE_TAG_PREFIX}-cuda${CUDA_SHORT}-py${PYTHON_SHORT}" + - name: Smoke UBI10 image (REST + gRPC) + if: ${{ startsWith(inputs.CUDA_VER, '13.') }} + env: + IMAGE_TAG_PREFIX: ${{ inputs.IMAGE_TAG_PREFIX }} + CUDA_SHORT: ${{ needs.prepare.outputs.CUDA_SHORT }} + run: | + bash ./ci/docker/smoke_image.sh \ + "nvidia/cuopt:${IMAGE_TAG_PREFIX}-cuda${CUDA_SHORT}-ubi10" diff --git a/ci/docker/README.md b/ci/docker/README.md index 8886d4147d..9abf862ce6 100644 --- a/ci/docker/README.md +++ b/ci/docker/README.md @@ -26,3 +26,17 @@ docker run -it --rm --gpus all -u root --volume $PWD:/repo -w /repo --entrypoint # UBI10 image docker run -it --rm --gpus all -u root --volume $PWD:/repo -w /repo --entrypoint "/bin/bash" nvidia/cuopt:[TAG]-ubi10 ./ci/docker/test_image.sh ``` + +### Startup smoke (REST + gRPC) + +`test_image.sh` runs pytest inside the image and does not launch the servers. +To verify the published entrypoint starts both the default REST server and the +gRPC server (`CUOPT_SERVER_TYPE=grpc`): + +```bash +./ci/docker/smoke_image.sh nvidia/cuopt:[TAG] +./ci/docker/smoke_image.sh nvidia/cuopt:[TAG]-ubi10 +``` + +CI runs this for both variants after the multiarch manifests are published +(see `.github/workflows/test_images.yaml` job `smoke`). diff --git a/ci/docker/smoke_image.sh b/ci/docker/smoke_image.sh new file mode 100755 index 0000000000..c67ae29353 --- /dev/null +++ b/ci/docker/smoke_image.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Smoke-test that a published cuOpt image starts its default REST server and +# the gRPC server via CUOPT_SERVER_TYPE=grpc. Runs on the host with docker so +# the real ENTRYPOINT/CMD path is exercised (unlike test_image.sh, which runs +# inside a GHA job container and never launches the servers). +# +# Usage (any published or locally built tag): +# ./ci/docker/smoke_image.sh nvidia/cuopt:[TAG] +# ./ci/docker/smoke_image.sh nvidia/cuopt:[TAG]-ubi10 +# +# Env: +# SMOKE_TIMEOUT_SECS Max seconds to wait for listen (default: 90) +# SMOKE_GPU_ARGS Docker GPU flags (default: --gpus all) + +set -euo pipefail + +IMAGE="${1:?usage: $0 }" +TIMEOUT_SECS="${SMOKE_TIMEOUT_SECS:-90}" +# shellcheck disable=SC2206 +GPU_ARGS=(${SMOKE_GPU_ARGS:---gpus all}) + +pass() { printf 'PASS %s\n' "$*"; } +fail() { printf 'FAIL %s\n' "$*" >&2; exit 1; } +info() { printf 'INFO %s\n' "$*"; } + +smoke_one() { + local label="$1" + local expect_re="$2" + shift 2 + # Remaining args are extra docker run flags (e.g. -e CUOPT_SERVER_TYPE=grpc). + + local name log cid i + name="cuopt-smoke-${label}-$$" + log="$(mktemp)" + cid="" + + cleanup() { + if [[ -n "${cid}" ]]; then + docker rm -f "${cid}" >/dev/null 2>&1 || true + fi + rm -f "${log}" + } + trap cleanup RETURN + + info "Starting ${label} server from ${IMAGE}" + # Do not use --rm: a fast crash (e.g. missing libnccl.so.2) would delete the + # container before we can collect logs. + cid="$(docker run -d --name "${name}" "${GPU_ARGS[@]}" "$@" "${IMAGE}")" + + for ((i = 1; i <= TIMEOUT_SECS; i++)); do + docker logs "${cid}" >"${log}" 2>&1 || true + + if grep -qiE 'error while loading shared libraries|libnccl\.so|FATAL FIPS SELFTEST|OpenSSL internal error' "${log}"; then + echo "----- ${label} logs -----" + cat "${log}" + fail "${label}: loader/crypto failure while starting" + fi + + if grep -qE "${expect_re}" "${log}"; then + pass "${label}: matched /${expect_re}/" + return 0 + fi + + # Container exited before listen — dump logs and fail. + if ! docker inspect -f '{{.State.Running}}' "${cid}" 2>/dev/null | grep -qx true; then + echo "----- ${label} logs -----" + cat "${log}" + fail "${label}: container exited before becoming ready" + fi + + sleep 1 + done + + echo "----- ${label} logs -----" + cat "${log}" + fail "${label}: timed out after ${TIMEOUT_SECS}s waiting for /${expect_re}/" +} + +info "Pulling ${IMAGE}" +if ! docker pull "${IMAGE}"; then + if docker image inspect "${IMAGE}" >/dev/null 2>&1; then + info "Pull failed; using local image ${IMAGE}" + else + fail "Pull failed and no local image named ${IMAGE}" + fi +fi + +smoke_one rest 'Uvicorn running on' +smoke_one grpc 'Listening on' -e CUOPT_SERVER_TYPE=grpc + +pass "Smoke OK for ${IMAGE}"