diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ec156804..04ccc527 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -54,6 +54,14 @@ jobs: build: name: build (${{ matrix.os }}, ${{ matrix.mpi }}, ${{ matrix.cc }}) runs-on: ${{ matrix.os }} + # ubuntu-24.04's apt mpich is broken on the GitHub runners (ch4:ucx -> + # MPI_Init aborts on the runner's verbs device, and even past that mpiexec + # launches size-1 singletons). Run ONLY those legs inside codes-ci-ubuntu24-mpich, + # which ships a source-built mpich (ch4:ofi) that forms a real multi-rank + # world; the empty string on every other leg means "no container -- run + # directly on the runner" as before. (codes-ci-ubuntu24-mpich is published by + # .github/workflows/ubuntu24-mpich-image.yml; make the GHCR package public.) + container: ${{ (matrix.os == 'ubuntu-24.04' && matrix.mpi == 'mpich') && 'ghcr.io/codes-org/codes-ci-ubuntu24-mpich:latest' || '' }} strategy: # Don't let one leg's failure cancel the others — we want to see every # OS x MPI x compiler result on every run. @@ -100,7 +108,11 @@ jobs: path: ross - name: Install system dependencies (Linux) - if: runner.os == 'Linux' + # Skipped on the ubuntu-24.04 mpich legs: those run inside + # codes-ci-ubuntu24-mpich, which already ships the toolchain, cmake/ninja, + # flex/bison, clang, and our source-built mpich (and the image has no + # sudo/apt set up for this step anyway). + if: runner.os == 'Linux' && !(matrix.os == 'ubuntu-24.04' && matrix.mpi == 'mpich') run: | sudo apt-get update sudo apt-get install -y cmake ninja-build pkg-config flex bison @@ -274,6 +286,12 @@ jobs: sanitizers: name: sanitizers (${{ matrix.sanitizer }}) runs-on: ubuntu-24.04 + # Run inside codes-ci-ubuntu24-mpich (source-built mpich, ch4:ofi) so ranks form a + # real multi-rank world -- ubuntu-24.04's apt mpich is broken on the runners + # (see the build job's note and ci/ubuntu24-mpich/Dockerfile). This also keeps the + # sanitizer lanes on mpich, which is much quieter under ASan/UBSan than + # OpenMPI, while actually exercising the optimistic (parallel) paths. + container: ghcr.io/codes-org/codes-ci-ubuntu24-mpich:latest strategy: # See every sanitizer's result on every run. fail-fast: false @@ -310,13 +328,8 @@ jobs: ref: ${{ github.event_name == 'schedule' && 'master' || env.ROSS_REF }} path: ross - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y \ - mpich libmpich-dev \ - cmake ninja-build pkg-config \ - flex bison + # No dependency-install step: the codes-ci-ubuntu24-mpich container already ships + # the toolchain, cmake/ninja, flex/bison, and our source-built mpich. - name: Configure ROSS # ROSS is built without sanitizers (default gcc); CODES links it and is @@ -376,6 +389,10 @@ jobs: coverage: name: coverage (ubuntu-24.04, gcc) runs-on: ubuntu-24.04 + # Run inside codes-ci-ubuntu24-mpich (source-built mpich, ch4:ofi) so the optimistic + # (parallel) paths actually run and get counted -- ubuntu-24.04's apt mpich is + # broken on the runners (see the build job's note and ci/ubuntu24-mpich/Dockerfile). + container: ghcr.io/codes-org/codes-ci-ubuntu24-mpich:latest env: # ROSS install prefix; find_package(ROSS) honors ROSS_ROOT (CODES presets). ROSS_ROOT: ${{ github.workspace }}/ross-install @@ -392,13 +409,8 @@ jobs: ref: ${{ github.event_name == 'schedule' && 'master' || env.ROSS_REF }} path: ross - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y \ - mpich libmpich-dev \ - cmake ninja-build pkg-config \ - flex bison lcov + # No dependency-install step: the codes-ci-ubuntu24-mpich container already ships + # the toolchain, cmake/ninja, flex/bison, lcov, and our source-built mpich. - name: Configure ROSS run: > diff --git a/CMakePresets.json b/CMakePresets.json index 59254d7a..3f1a85b8 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -171,7 +171,8 @@ "name": "base", "hidden": true, "output": { - "outputOnFailure": true + "outputOnFailure": true, + "verbosity": "verbose" }, "execution": { "noTestsAction": "error", diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c972badd..294d5c44 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -16,6 +16,19 @@ add_executable(codes-unit-smoke codes-unit-smoke.cxx) target_link_libraries(codes-unit-smoke PRIVATE GTest::gtest_main) add_test(NAME codes-unit-smoke COMMAND codes-unit-smoke) # --------------------------------------------------------------------------- +# MPI launch preflight. Fails loudly if mpiexec can't form a real multi-rank +# MPI_COMM_WORLD, guarding against a broken launcher (e.g. Ubuntu 24.04's mpich +# 4.2.0-5build3, whose Hydra starts every rank as a singleton) silently +# degrading the optimistic (--sync=3) model tests below into N independent +# *sequential* runs that still exit 0 and pass. Launched exactly like the other +# MPI tests (same MPIEXEC_* vars) at np=2; the binary returns nonzero unless +# MPI_Comm_size == 2, so a singleton launcher makes ctest fail here. +add_executable(mpi-multirank-check mpi-multirank-check.c) +target_link_libraries(mpi-multirank-check PRIVATE MPI::MPI_C) +add_test(NAME mpi-multirank-sanity + COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 2 ${MPIEXEC_PREFLAGS} + $ 2 ${MPIEXEC_POSTFLAGS}) +# --------------------------------------------------------------------------- # codes_add_equivalence_test — register an equivalence/determinism test. # diff --git a/tests/mpi-multirank-check.c b/tests/mpi-multirank-check.c new file mode 100644 index 00000000..3e633aae --- /dev/null +++ b/tests/mpi-multirank-check.c @@ -0,0 +1,42 @@ +/* + * MPI multi-rank launch guard. + * + * Asserts that mpiexec actually forms a multi-rank MPI_COMM_WORLD. A broken MPI + * launcher (notably Ubuntu 24.04's mpich 4.2.0-5build3, whose Hydra starts every + * rank as its own singleton) reports MPI_Comm_size == 1 even under `mpiexec -np + * N`. That silently degrades CODES's optimistic (--sync=3) model tests to N + * independent *sequential* runs -- ROSS prints "Defaulting to Sequential + * Simulation, not enough PEs defined" per rank -- while the tests still exit 0 + * and pass. This guard turns that silent degradation into a hard ctest failure, + * now and for any future environment with the same problem. + * + * Exits 0 iff the world size equals the expected size (argv[1], default 2). + */ +#include +#include +#include + +int main(int argc, char** argv) { + MPI_Init(&argc, &argv); + + int rank = 0, size = 0; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); + + int expected = (argc > 1) ? atoi(argv[1]) : 2; + + if (rank == 0) { + printf("MPI multi-rank guard: MPI_COMM_WORLD size = %d (expected %d)\n", size, expected); + if (size != expected) { + fprintf(stderr, + "ERROR: mpiexec did not form a %d-rank world (got %d). The MPI " + "launcher is starting singletons, so parallel/optimistic " + "(--sync=3) tests would silently run sequential. See the Ubuntu " + "24.04 mpich 4.2.0 PMI bug.\n", + expected, size); + } + } + + MPI_Finalize(); + return (size == expected) ? 0 : 1; +}