Deterministic dace toolchain draft

.github/workflows/copilot-setup-steps.yml

@@ -8,7 +8,7 @@ jobs:
       contents: read
     steps:
       - name: Checkout code
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           submodules: recursive
 

.github/workflows/general-ci.yml

@@ -22,7 +22,7 @@ jobs:
         simplify: [0,1,autoopt]
 
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v7
       with:
         submodules: 'recursive'
     - name: Set up Python ${{ matrix.python-version }}
@@ -106,7 +106,7 @@ jobs:
         ./tests/xform_test.sh
         coverage combine .; coverage report; coverage xml
 
-    - uses: codecov/codecov-action@v5
+    - uses: codecov/codecov-action@v6
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
         verbose: true

.github/workflows/gpu-ci.yml

@@ -23,7 +23,7 @@ jobs:
     if: "!contains(github.event.pull_request.labels.*.name, 'no-ci')"
     runs-on: [self-hosted, gpu]
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v7
       with:
         submodules: 'recursive'
     - name: Install dependencies
@@ -36,6 +36,7 @@ jobs:
         pip install flake8 pytest-xdist coverage
         pip install mpi4py
         pip install cupy
+        pip install cutensor-cu12
         pip uninstall -y dace
         pip install -e ".[testing,ml]"
         curl -Os https://uploader.codecov.io/latest/linux/codecov

.github/workflows/heterogeneous-ci.yml

@@ -23,7 +23,7 @@ jobs:
     if: "!contains(github.event.pull_request.labels.*.name, 'no-ci')"
     runs-on: [self-hosted, linux]
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v7
       with:
         submodules: 'recursive'
     - name: Install dependencies

.github/workflows/linting.yml

@@ -14,7 +14,7 @@ jobs:
 
     steps:
     - name: Check repository
-      uses: actions/checkout@v6
+      uses: actions/checkout@v7
 
     - name: Setup Python 3.10
       uses: actions/setup-python@v6
@@ -27,10 +27,4 @@ jobs:
 
     - name: Run linting tools
       id: lint
-      continue-on-error: true
-      run: pre-commit run --all-files
-
-    - name: Show git diff
-      if: steps.lint.outcome == 'failure'
-      run: |
-        ./.github/workflows/scripts/show-git-diff.sh
+      run: pre-commit run --show-diff-on-failure --color=always --all-files

.github/workflows/ml-ci.yml

@@ -22,7 +22,7 @@ jobs:
         simplify: [0,1,autoopt]
 
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v7
       with:
         submodules: 'recursive'
     - name: Set up Python ${{ matrix.python-version }}
@@ -56,7 +56,7 @@ jobs:
         pytest --cov-report=xml --cov=dace --tb=short --timeout_method thread --timeout=600 -v -m "(torch or onnx or autodiff) and not gpu"
         ./codecov
 
-    - uses: codecov/codecov-action@v5
+    - uses: codecov/codecov-action@v6
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
         verbose: true

.github/workflows/pyFV3-ci.yml

@@ -1,17 +1,12 @@
 name: NASA/NOAA pyFV3 repository build test
 
-# Temporarily disabled for main, and instead applied to a specific DaCe v1 maintenance branch (v1/maintenance). Once
-# the FV3 bridge has been adapted to DaCe v1, this will need to be reverted back to apply to main.
 on:
   push:
-    #branches: [ main, ci-fix ]
-    branches: [ v1/maintenance, ci-fix ]
+    branches: [ main, ci-fix, romanc/stree-v2 ]
   pull_request:
-    #branches: [ main, ci-fix ]
-    branches: [ v1/maintenance, ci-fix ]
+    branches: [ main, ci-fix, romanc/stree-v2 ]
   merge_group:
-    #branches: [ main, ci-fix ]
-    branches: [ v1/maintenance, ci-fix ]
+    branches: [ main, ci-fix, romanc/stree-v2 ]
 
 defaults:
     run:
@@ -22,84 +17,80 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-    build_and_validate_pyFV3:
-      if: "!contains(github.event.pull_request.labels.*.name, 'no-ci')"
-      runs-on: ubuntu-latest
-      strategy:
-          matrix:
-              python-version: [3.11.7]
+  configure:
+    runs-on: ubuntu-latest
+    if: "!contains(github.event.pull_request.labels.*.name, 'no-ci')"
+    name: "Read test configuration"
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+     - name: Checkout to repository
+       uses: actions/checkout@v7
+     - name: Set matrix data
+       id: set-matrix
+       run: echo "matrix={\"include\":$(jq -c . < .github/workflows/pyFV3-regression-matrix.json)}" >> $GITHUB_OUTPUT
 
-      steps:
-      - uses: actions/checkout@v6
-        with:
-              repository: 'NOAA-GFDL/PyFV3'
-              ref: 'ci/DaCe'
-              submodules: 'recursive'
-              path: 'pyFV3'
-      - uses: actions/checkout@v6
-        with:
-            path: 'dace'
-            submodules: 'recursive'
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v6
-        with:
-              python-version: ${{ matrix.python-version }}
-      - name: Install library dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y libopenmpi-dev libboost-all-dev
-          gcc --version
-      # Because Github doesn't allow us to do a git checkout in code
-      # we use a trick to checkout DaCe first (not using the external submodule)
-      # install the full suite via requirements_dev, then re-install the correct DaCe
-      - name: Install Python packages
-        run: |
-          python -m pip install --upgrade pip wheel setuptools
-          pip install -e ./pyFV3[develop]
-          pip install -e ./dace
-      - name: Download data
-        run: |
-          cd pyFV3
-          mkdir -p test_data
-          cd test_data
-          wget --retry-connrefused https://portal.nccs.nasa.gov/datashare/astg/smt/pace-regression-data/8.1.3_c12_6ranks_standard.D_SW.tar.gz
-          tar -xzvf 8.1.3_c12_6ranks_standard.D_SW.tar.gz
-          wget --retry-connrefused https://portal.nccs.nasa.gov/datashare/astg/smt/pace-regression-data/8.1.3_c12_6ranks_standard.RiemSolver3.tar.gz
-          tar -xzvf 8.1.3_c12_6ranks_standard.RiemSolver3.tar.gz
-          wget --retry-connrefused https://portal.nccs.nasa.gov/datashare/astg/smt/pace-regression-data/8.1.3_c12_6ranks_standard.Remapping.tar.gz
-          tar -xzvf 8.1.3_c12_6ranks_standard.Remapping.tar.gz
-          cd ../..
-      # Clean up caches between run for stale un-expanded SDFG to trip the build system (NDSL side issue)
-      - name: "Regression test: Riemman Solver on D-grid (RiemSolver3)"
-        env:
-          FV3_DACEMODE: BuildAndRun
-          PACE_CONSTANTS: GFS
-          PACE_LOGLEVEL: Debug
-        run: |
-          pytest -v -s --data_path=./pyFV3/test_data/8.1.3/c12_6ranks_standard/dycore \
-              --backend=dace:cpu --which_modules=Riem_Solver3 \
-              --threshold_overrides_file=./pyFV3/tests/savepoint/translate/overrides/standard.yaml \
-              ./pyFV3/tests/savepoint
-          rm -r ./.gt_cache_FV3_A
-      - name: "Regression test: Shallow water lagrangian dynamics on D-grid (D_SW) (on rank 0 only)"
-        env:
-          FV3_DACEMODE: BuildAndRun
-          PACE_CONSTANTS: GFS
-          PACE_LOGLEVEL: Debug
-        run: |
-          pytest -v -s --data_path=./pyFV3/test_data/8.1.3/c12_6ranks_standard/dycore \
-              --backend=dace:cpu --which_modules=D_SW --which_rank=0 \
-              --threshold_overrides_file=./pyFV3/tests/savepoint/translate/overrides/standard.yaml \
-              ./pyFV3/tests/savepoint
-          rm -r ./.gt_cache_FV3_A
-      - name: "Regression test: Remapping (on rank 0 only)"
-        env:
-          FV3_DACEMODE: BuildAndRun
-          PACE_CONSTANTS: GFS
-          PACE_LOGLEVEL: Debug
-        run: |
-          pytest -v -s --data_path=./pyFV3/test_data/8.1.3/c12_6ranks_standard/dycore \
-              --backend=dace:cpu --which_modules=Remapping --which_rank=0 \
-              --threshold_overrides_file=./pyFV3/tests/savepoint/translate/overrides/standard.yaml \
-              ./pyFV3/tests/savepoint
-          rm -r ./.gt_cache_FV3_A
+  build_and_validate_pyFV3:
+    runs-on: ubuntu-latest
+    needs: configure
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.configure.outputs.matrix) }}
+    name: "${{ matrix.module }} regression test"
+
+    steps:
+    - name: Checkout pyFV3
+      uses: actions/checkout@v7
+      with:
+        repository: 'NOAA-GFDL/pyFV3'
+        path: 'pyFV3'
+
+    - name: Checkout dace
+      uses: actions/checkout@v7
+      with:
+        path: 'dace'
+        submodules: 'recursive'
+
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v6
+      with:
+        python-version: 3.12
+
+    - name: Install OpenMPI
+      run: pip install openmpi
+
+    # We install `dace` after `pyFV3` to build pyFV3 with the current version of DaCe
+    - name: Install pyFV3 and dace
+      run: |
+        pip install ./pyFV3[ndsl,test]
+        pip install ./dace
+
+    - name: Print versions
+      run: |
+        gcc --version
+        python --version
+        pip --version
+        pip list
+
+    - name: Download data
+      # Skip "Riemann Solver on D-grid" (currently failing)
+      if: ${{ !startsWith(matrix.module, 'Riem_Solver3') }}
+      run: |
+        mkdir -p pyFV3/test_data
+        cd pyFV3/test_data
+        wget --retry-connrefused https://portal.nccs.nasa.gov/datashare/astg/smt/pace-regression-data/${{ matrix.archive }}
+        tar -xzvf ${{ matrix.archive }}
+        cd ../..
+
+    - name: "Regression test: ${{ matrix.name }}"
+      # Skip "Riemann Solver on D-grid" (currently failing)
+      if: ${{ !startsWith(matrix.module, 'Riem_Solver3') }}
+      env:
+        FV3_DACEMODE: BuildAndRun
+        NDSL_LOGLEVEL: Debug
+      run: |
+        pytest -v -s --data_path=./pyFV3/test_data/8.1.3/c12_6ranks_standard/dycore \
+            --backend=orch:dace:cpu:KIJ --which_modules=${{ matrix.module }} \
+            ${{ matrix.extra_args }} \
+            --threshold_overrides_file=./pyFV3/tests/savepoint/translate/overrides/standard.yaml \
+            ./pyFV3/tests/savepoint

.github/workflows/pyFV3-regression-matrix.json

@@ -0,0 +1,20 @@
+[
+    {
+        "name": "Shallow water lagrangian dynamics on D-grid (D_SW) (on rank 0 only)",
+        "archive": "8.1.3_c12_6ranks_standard.D_SW.tar.gz",
+        "module": "D_SW",
+        "extra_args": "--which_rank=0"
+    },
+    {
+        "name":"Riemann Solver on D-grid (RiemSolver3)",
+        "archive":"8.1.3_c12_6ranks_standard.RiemSolver3.tar.gz",
+        "module":"Riem_Solver3",
+        "extra_args":"--which_rank=0"
+    },
+    {
+        "name": "Remapping (on rank 0 only)",
+        "archive": "8.1.3_c12_6ranks_standard.Remapping.tar.gz",
+        "module": "Remapping",
+        "extra_args": "--which_rank=0"
+    }
+]

.github/workflows/release.sh

@@ -3,7 +3,7 @@
 set -e
 
 # Install dependencies
-pip install --upgrade twine
+pip install --upgrade twine build
 
 # Synchronize submodules
 git submodule update --init --recursive

.pre-commit-config.yaml

@@ -2,6 +2,11 @@ default_language_version:
   python: python3
 
 repos:
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.15.5
+  hooks:
+    - id: ruff-check
+      args: [--fix]
 - repo: https://github.com/pre-commit/pre-commit-hooks
   rev: v5.0.0
   hooks:

.readthedocs.yaml

@@ -6,9 +6,9 @@
 version: 2
 
 build:
-  os: ubuntu-20.04
+  os: ubuntu-24.04
   tools:
-    python: "3.10"
+    python: "3.14"
 
 # Build documentation in the docs/ directory with Sphinx
 sphinx:

ci/cscs_gpu.yml

@@ -54,6 +54,12 @@ test_cscs_gh200:
     - cd "${WORKDIR}/dace"
     - git fetch --depth 1 origin "${CI_COMMIT_SHA}"
     - git checkout "${CI_COMMIT_SHA}"
+    - uv pip install cutensor-cu12
+    - CUTENSOR_DIR="${VIRTUAL_ENV}/lib/python${PY_VERSION}/site-packages/cutensor"
+    - ln -sf libcutensor.so.2 "${CUTENSOR_DIR}/lib/libcutensor.so"
+    - export CPATH="${CUTENSOR_DIR}/include:${CPATH:-}"
+    - export LIBRARY_PATH="${CUTENSOR_DIR}/lib:${LIBRARY_PATH:-}"
+    - export LD_LIBRARY_PATH="${CUTENSOR_DIR}/lib:${LD_LIBRARY_PATH:-}"
     - uv pip install -e ".[testing]"
     - export DACE_cache=unique
     - pytest --cov-report=xml --cov=dace --tb=short --timeout_method thread --timeout=300 -n 32 -m "${TEST_VARIANTS}"