From 3857ad5de21c39d5b8cc16a35a90defd31ed5165 Mon Sep 17 00:00:00 2001 From: Caitlin Ross Date: Thu, 9 Jul 2026 16:44:47 -0500 Subject: [PATCH 1/3] cmake presets: make tests always verbose make ci tests always run in verbose mode, in case we have situations where tests return success even though they shouldn't. this way we can go back to ci logs and see the output. --- CMakePresets.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakePresets.json b/CMakePresets.json index 59254d7a..3f1a85b8 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -171,7 +171,8 @@ "name": "base", "hidden": true, "output": { - "outputOnFailure": true + "outputOnFailure": true, + "verbosity": "verbose" }, "execution": { "noTestsAction": "error", From cfe778fc302420b3b35c733577a7dec795810637 Mon Sep 17 00:00:00 2001 From: Caitlin Ross Date: Thu, 9 Jul 2026 16:46:55 -0500 Subject: [PATCH 2/3] tests: add non-codes MPI test for CI machines this lets us check if an mpi issue is a codes problem or a CI machine problem --- tests/CMakeLists.txt | 13 ++++++++++++ tests/mpi-multirank-check.c | 42 +++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 tests/mpi-multirank-check.c diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c972badd..294d5c44 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -16,6 +16,19 @@ add_executable(codes-unit-smoke codes-unit-smoke.cxx) target_link_libraries(codes-unit-smoke PRIVATE GTest::gtest_main) add_test(NAME codes-unit-smoke COMMAND codes-unit-smoke) # --------------------------------------------------------------------------- +# MPI launch preflight. Fails loudly if mpiexec can't form a real multi-rank +# MPI_COMM_WORLD, guarding against a broken launcher (e.g. Ubuntu 24.04's mpich +# 4.2.0-5build3, whose Hydra starts every rank as a singleton) silently +# degrading the optimistic (--sync=3) model tests below into N independent +# *sequential* runs that still exit 0 and pass. Launched exactly like the other +# MPI tests (same MPIEXEC_* vars) at np=2; the binary returns nonzero unless +# MPI_Comm_size == 2, so a singleton launcher makes ctest fail here. +add_executable(mpi-multirank-check mpi-multirank-check.c) +target_link_libraries(mpi-multirank-check PRIVATE MPI::MPI_C) +add_test(NAME mpi-multirank-sanity + COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 2 ${MPIEXEC_PREFLAGS} + $ 2 ${MPIEXEC_POSTFLAGS}) +# --------------------------------------------------------------------------- # codes_add_equivalence_test — register an equivalence/determinism test. # diff --git a/tests/mpi-multirank-check.c b/tests/mpi-multirank-check.c new file mode 100644 index 00000000..3e633aae --- /dev/null +++ b/tests/mpi-multirank-check.c @@ -0,0 +1,42 @@ +/* + * MPI multi-rank launch guard. + * + * Asserts that mpiexec actually forms a multi-rank MPI_COMM_WORLD. A broken MPI + * launcher (notably Ubuntu 24.04's mpich 4.2.0-5build3, whose Hydra starts every + * rank as its own singleton) reports MPI_Comm_size == 1 even under `mpiexec -np + * N`. That silently degrades CODES's optimistic (--sync=3) model tests to N + * independent *sequential* runs -- ROSS prints "Defaulting to Sequential + * Simulation, not enough PEs defined" per rank -- while the tests still exit 0 + * and pass. This guard turns that silent degradation into a hard ctest failure, + * now and for any future environment with the same problem. + * + * Exits 0 iff the world size equals the expected size (argv[1], default 2). + */ +#include +#include +#include + +int main(int argc, char** argv) { + MPI_Init(&argc, &argv); + + int rank = 0, size = 0; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); + + int expected = (argc > 1) ? atoi(argv[1]) : 2; + + if (rank == 0) { + printf("MPI multi-rank guard: MPI_COMM_WORLD size = %d (expected %d)\n", size, expected); + if (size != expected) { + fprintf(stderr, + "ERROR: mpiexec did not form a %d-rank world (got %d). The MPI " + "launcher is starting singletons, so parallel/optimistic " + "(--sync=3) tests would silently run sequential. See the Ubuntu " + "24.04 mpich 4.2.0 PMI bug.\n", + expected, size); + } + } + + MPI_Finalize(); + return (size == expected) ? 0 : 1; +} From 260a273803f000a34af9d20b2813ec53c75a764e Mon Sep 17 00:00:00 2001 From: Caitlin Ross Date: Thu, 9 Jul 2026 18:32:48 -0500 Subject: [PATCH 3/3] ci: run the ubuntu-24.04 mpich legs inside codes-ci-ubuntu24-mpich Ubuntu 24.04's apt mpich (4.2.0-5build3, ch4:ucx) is broken on the GitHub runners in two ways: UCX aborts MPI_Init on the runner's verbs device ("ibv_create_srq() failed"), and even past that mpiexec launches size-1 singletons so optimistic (--sync=3) tests silently run sequential. There's no fixed mpich in the noble archive, so the affected legs run inside a container image that ships a source-built mpich (ch4:ofi -- libfabric/TCP, no UCX/verbs) that forms a real multi-rank world. - build matrix: a conditional container (empty string => no container, so every non-24.04-mpich leg runs on the runner exactly as before). - sanitizers + coverage: unconditional container (both are single-config 24.04 mpich); drop their now-redundant apt install steps. Keeping the sanitizer lanes on mpich (quiet under ASan/UBSan) now actually exercises the optimistic paths. - old-ross-compat is left on apt mpich: it only builds, never runs mpiexec. The image (codes-ci-ubuntu24-mpich) and its publish workflow live on a separate branch; it must be published to GHCR and made public before these legs can pull it. mpi-multirank-sanity going green confirms the launcher forms a real world. --- .github/workflows/build.yml | 42 ++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ec156804..04ccc527 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -54,6 +54,14 @@ jobs: build: name: build (${{ matrix.os }}, ${{ matrix.mpi }}, ${{ matrix.cc }}) runs-on: ${{ matrix.os }} + # ubuntu-24.04's apt mpich is broken on the GitHub runners (ch4:ucx -> + # MPI_Init aborts on the runner's verbs device, and even past that mpiexec + # launches size-1 singletons). Run ONLY those legs inside codes-ci-ubuntu24-mpich, + # which ships a source-built mpich (ch4:ofi) that forms a real multi-rank + # world; the empty string on every other leg means "no container -- run + # directly on the runner" as before. (codes-ci-ubuntu24-mpich is published by + # .github/workflows/ubuntu24-mpich-image.yml; make the GHCR package public.) + container: ${{ (matrix.os == 'ubuntu-24.04' && matrix.mpi == 'mpich') && 'ghcr.io/codes-org/codes-ci-ubuntu24-mpich:latest' || '' }} strategy: # Don't let one leg's failure cancel the others — we want to see every # OS x MPI x compiler result on every run. @@ -100,7 +108,11 @@ jobs: path: ross - name: Install system dependencies (Linux) - if: runner.os == 'Linux' + # Skipped on the ubuntu-24.04 mpich legs: those run inside + # codes-ci-ubuntu24-mpich, which already ships the toolchain, cmake/ninja, + # flex/bison, clang, and our source-built mpich (and the image has no + # sudo/apt set up for this step anyway). + if: runner.os == 'Linux' && !(matrix.os == 'ubuntu-24.04' && matrix.mpi == 'mpich') run: | sudo apt-get update sudo apt-get install -y cmake ninja-build pkg-config flex bison @@ -274,6 +286,12 @@ jobs: sanitizers: name: sanitizers (${{ matrix.sanitizer }}) runs-on: ubuntu-24.04 + # Run inside codes-ci-ubuntu24-mpich (source-built mpich, ch4:ofi) so ranks form a + # real multi-rank world -- ubuntu-24.04's apt mpich is broken on the runners + # (see the build job's note and ci/ubuntu24-mpich/Dockerfile). This also keeps the + # sanitizer lanes on mpich, which is much quieter under ASan/UBSan than + # OpenMPI, while actually exercising the optimistic (parallel) paths. + container: ghcr.io/codes-org/codes-ci-ubuntu24-mpich:latest strategy: # See every sanitizer's result on every run. fail-fast: false @@ -310,13 +328,8 @@ jobs: ref: ${{ github.event_name == 'schedule' && 'master' || env.ROSS_REF }} path: ross - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y \ - mpich libmpich-dev \ - cmake ninja-build pkg-config \ - flex bison + # No dependency-install step: the codes-ci-ubuntu24-mpich container already ships + # the toolchain, cmake/ninja, flex/bison, and our source-built mpich. - name: Configure ROSS # ROSS is built without sanitizers (default gcc); CODES links it and is @@ -376,6 +389,10 @@ jobs: coverage: name: coverage (ubuntu-24.04, gcc) runs-on: ubuntu-24.04 + # Run inside codes-ci-ubuntu24-mpich (source-built mpich, ch4:ofi) so the optimistic + # (parallel) paths actually run and get counted -- ubuntu-24.04's apt mpich is + # broken on the runners (see the build job's note and ci/ubuntu24-mpich/Dockerfile). + container: ghcr.io/codes-org/codes-ci-ubuntu24-mpich:latest env: # ROSS install prefix; find_package(ROSS) honors ROSS_ROOT (CODES presets). ROSS_ROOT: ${{ github.workspace }}/ross-install @@ -392,13 +409,8 @@ jobs: ref: ${{ github.event_name == 'schedule' && 'master' || env.ROSS_REF }} path: ross - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y \ - mpich libmpich-dev \ - cmake ninja-build pkg-config \ - flex bison lcov + # No dependency-install step: the codes-ci-ubuntu24-mpich container already ships + # the toolchain, cmake/ninja, flex/bison, lcov, and our source-built mpich. - name: Configure ROSS run: >