Fix CI MPICH issues

.github/workflows/build.yml

@@ -54,6 +54,14 @@ jobs:
   build:
     name: build (${{ matrix.os }}, ${{ matrix.mpi }}, ${{ matrix.cc }})
     runs-on: ${{ matrix.os }}
+    # ubuntu-24.04's apt mpich is broken on the GitHub runners (ch4:ucx ->
+    # MPI_Init aborts on the runner's verbs device, and even past that mpiexec
+    # launches size-1 singletons). Run ONLY those legs inside codes-ci-ubuntu24-mpich,
+    # which ships a source-built mpich (ch4:ofi) that forms a real multi-rank
+    # world; the empty string on every other leg means "no container -- run
+    # directly on the runner" as before. (codes-ci-ubuntu24-mpich is published by
+    # .github/workflows/ubuntu24-mpich-image.yml; make the GHCR package public.)
+    container: ${{ (matrix.os == 'ubuntu-24.04' && matrix.mpi == 'mpich') && 'ghcr.io/codes-org/codes-ci-ubuntu24-mpich:latest' || '' }}
     strategy:
       # Don't let one leg's failure cancel the others — we want to see every
       # OS x MPI x compiler result on every run.
@@ -100,7 +108,11 @@ jobs:
           path: ross
 
       - name: Install system dependencies (Linux)
-        if: runner.os == 'Linux'
+        # Skipped on the ubuntu-24.04 mpich legs: those run inside
+        # codes-ci-ubuntu24-mpich, which already ships the toolchain, cmake/ninja,
+        # flex/bison, clang, and our source-built mpich (and the image has no
+        # sudo/apt set up for this step anyway).
+        if: runner.os == 'Linux' && !(matrix.os == 'ubuntu-24.04' && matrix.mpi == 'mpich')
         run: |
           sudo apt-get update
           sudo apt-get install -y cmake ninja-build pkg-config flex bison
@@ -274,6 +286,12 @@ jobs:
   sanitizers:
     name: sanitizers (${{ matrix.sanitizer }})
     runs-on: ubuntu-24.04
+    # Run inside codes-ci-ubuntu24-mpich (source-built mpich, ch4:ofi) so ranks form a
+    # real multi-rank world -- ubuntu-24.04's apt mpich is broken on the runners
+    # (see the build job's note and ci/ubuntu24-mpich/Dockerfile). This also keeps the
+    # sanitizer lanes on mpich, which is much quieter under ASan/UBSan than
+    # OpenMPI, while actually exercising the optimistic (parallel) paths.
+    container: ghcr.io/codes-org/codes-ci-ubuntu24-mpich:latest
     strategy:
       # See every sanitizer's result on every run.
       fail-fast: false
@@ -310,13 +328,8 @@ jobs:
           ref: ${{ github.event_name == 'schedule' && 'master' || env.ROSS_REF }}
           path: ross
 
-      - name: Install system dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y \
-            mpich libmpich-dev \
-            cmake ninja-build pkg-config \
-            flex bison
+      # No dependency-install step: the codes-ci-ubuntu24-mpich container already ships
+      # the toolchain, cmake/ninja, flex/bison, and our source-built mpich.
 
       - name: Configure ROSS
         # ROSS is built without sanitizers (default gcc); CODES links it and is
@@ -376,6 +389,10 @@ jobs:
   coverage:
     name: coverage (ubuntu-24.04, gcc)
     runs-on: ubuntu-24.04
+    # Run inside codes-ci-ubuntu24-mpich (source-built mpich, ch4:ofi) so the optimistic
+    # (parallel) paths actually run and get counted -- ubuntu-24.04's apt mpich is
+    # broken on the runners (see the build job's note and ci/ubuntu24-mpich/Dockerfile).
+    container: ghcr.io/codes-org/codes-ci-ubuntu24-mpich:latest
     env:
       # ROSS install prefix; find_package(ROSS) honors ROSS_ROOT (CODES presets).
       ROSS_ROOT: ${{ github.workspace }}/ross-install
@@ -392,13 +409,8 @@ jobs:
           ref: ${{ github.event_name == 'schedule' && 'master' || env.ROSS_REF }}
           path: ross
 
-      - name: Install system dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y \
-            mpich libmpich-dev \
-            cmake ninja-build pkg-config \
-            flex bison lcov
+      # No dependency-install step: the codes-ci-ubuntu24-mpich container already ships
+      # the toolchain, cmake/ninja, flex/bison, lcov, and our source-built mpich.
 
       - name: Configure ROSS
         run: >

CMakePresets.json

@@ -171,7 +171,8 @@
             "name": "base",
             "hidden": true,
             "output": {
-                "outputOnFailure": true
+                "outputOnFailure": true,
+                "verbosity": "verbose"
             },
             "execution": {
                 "noTestsAction": "error",

tests/CMakeLists.txt

@@ -16,6 +16,19 @@ add_executable(codes-unit-smoke codes-unit-smoke.cxx)
 target_link_libraries(codes-unit-smoke PRIVATE GTest::gtest_main)
 add_test(NAME codes-unit-smoke COMMAND codes-unit-smoke)
 # ---------------------------------------------------------------------------
+# MPI launch preflight. Fails loudly if mpiexec can't form a real multi-rank
+# MPI_COMM_WORLD, guarding against a broken launcher (e.g. Ubuntu 24.04's mpich
+# 4.2.0-5build3, whose Hydra starts every rank as a singleton) silently
+# degrading the optimistic (--sync=3) model tests below into N independent
+# *sequential* runs that still exit 0 and pass. Launched exactly like the other
+# MPI tests (same MPIEXEC_* vars) at np=2; the binary returns nonzero unless
+# MPI_Comm_size == 2, so a singleton launcher makes ctest fail here.
+add_executable(mpi-multirank-check mpi-multirank-check.c)
+target_link_libraries(mpi-multirank-check PRIVATE MPI::MPI_C)
+add_test(NAME mpi-multirank-sanity
+    COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 2 ${MPIEXEC_PREFLAGS}
+            $<TARGET_FILE:mpi-multirank-check> 2 ${MPIEXEC_POSTFLAGS})
+# ---------------------------------------------------------------------------
 
 # codes_add_equivalence_test — register an equivalence/determinism test.
 #

tests/mpi-multirank-check.c

@@ -0,0 +1,42 @@
+/*
+ * MPI multi-rank launch guard.
+ *
+ * Asserts that mpiexec actually forms a multi-rank MPI_COMM_WORLD. A broken MPI
+ * launcher (notably Ubuntu 24.04's mpich 4.2.0-5build3, whose Hydra starts every
+ * rank as its own singleton) reports MPI_Comm_size == 1 even under `mpiexec -np
+ * N`. That silently degrades CODES's optimistic (--sync=3) model tests to N
+ * independent *sequential* runs -- ROSS prints "Defaulting to Sequential
+ * Simulation, not enough PEs defined" per rank -- while the tests still exit 0
+ * and pass. This guard turns that silent degradation into a hard ctest failure,
+ * now and for any future environment with the same problem.
+ *
+ * Exits 0 iff the world size equals the expected size (argv[1], default 2).
+ */
+#include <mpi.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+int main(int argc, char** argv) {
+    MPI_Init(&argc, &argv);
+
+    int rank = 0, size = 0;
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+    MPI_Comm_size(MPI_COMM_WORLD, &size);
+
+    int expected = (argc > 1) ? atoi(argv[1]) : 2;
+
+    if (rank == 0) {
+        printf("MPI multi-rank guard: MPI_COMM_WORLD size = %d (expected %d)\n", size, expected);
+        if (size != expected) {
+            fprintf(stderr,
+                    "ERROR: mpiexec did not form a %d-rank world (got %d). The MPI "
+                    "launcher is starting singletons, so parallel/optimistic "
+                    "(--sync=3) tests would silently run sequential. See the Ubuntu "
+                    "24.04 mpich 4.2.0 PMI bug.\n",
+                    expected, size);
+        }
+    }
+
+    MPI_Finalize();
+    return (size == expected) ? 0 : 1;
+}