From 74aba9e3b242b52db8da72b61d36ddc740bcf687 Mon Sep 17 00:00:00 2001 From: Sebastien Valat Date: Fri, 19 Jun 2026 11:06:30 +0200 Subject: [PATCH 1/8] refactor: axis communication by using Buffer (step 1 - add communication and validate they are identic) --- src/fluid/boundary/axis.cpp | 170 +++++++++++++++++++++++++++++++----- src/fluid/boundary/axis.hpp | 6 ++ src/mpi/buffer.hpp | 61 +++++++++++++ 3 files changed, 217 insertions(+), 20 deletions(-) diff --git a/src/fluid/boundary/axis.cpp b/src/fluid/boundary/axis.cpp index 47f0c2954..da658e6c1 100644 --- a/src/fluid/boundary/axis.cpp +++ b/src/fluid/boundary/axis.cpp @@ -5,6 +5,11 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** +#undef NDEBUG +#include +#include +#include + #include #include "axis.hpp" #include "boundary.hpp" @@ -395,17 +400,24 @@ void Axis::ExchangeMPI(int side) { int ibeg,iend,jbeg,jend,kbeg,kend,offset; int nx,ny,nz; auto bufferSend = this->bufferSend; + Buffer bufferSendNew = this->bufferSendNew; IdefixArray1D map = this->mapVars; IdefixArray4D Vc = this->Vc; IdefixArray4D Vs = this->Vs; + //reset pointer + bufferSendNew.ResetPointer(); + // If MPI Persistent, start receiving even before the buffers are filled MPI_Status sendStatus; MPI_Status recvStatus; + MPI_Status sendStatusNew; + MPI_Status recvStatusNew; double tStart = MPI_Wtime(); MPI_SAFE_CALL(MPI_Start(&recvRequest)); + MPI_SAFE_CALL(MPI_Start(&recvRequestNew)); idfx::mpiCallsTimer += MPI_Wtime() - tStart; // Coordinates of the ghost region which needs to be transfered @@ -419,73 +431,145 @@ void Axis::ExchangeMPI(int side) { kbeg = data->beg[KDIR]; kend = data->end[KDIR]; nz = kend - kbeg; + + // Create the box + BoundingBox baseBox; + baseBox[IDIR][0] = ibeg; + baseBox[IDIR][1] = iend; + baseBox[JDIR][0] = jbeg; + baseBox[JDIR][1] = jend; + baseBox[KDIR][0] = kbeg; + baseBox[KDIR][1] = kend; + if(side==left) { + //shift by NY + BoundingBox sendBoxVc = baseBox; + sendBoxVc[JDIR][0] += ny; + sendBoxVc[JDIR][1] += ny; + bufferSendNew.Pack(Vc, map, sendBoxVc); + idefix_for("LoadBufferX2Vc",0,mapNVars,kbeg,kend,jbeg,jend,ibeg,iend, KOKKOS_LAMBDA (int n, int k, int j, int i) { - bufferSend(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + n*nx*ny*nz ) = + const int storeId = i + (j-jbeg)*nx + (k-kbeg)*nx*ny + n*nx*ny*nz; + bufferSend(storeId ) = Vc(map(n),k,j+ny,i); + assert(bufferSend(storeId) == bufferSendNew.getArray()(storeId)); } ); + if (haveMHD) { + //extend by one + BoundingBox sendBoxVsIdir = sendBoxVc; + sendBoxVsIdir[IDIR][1] += 1; + bufferSendNew.Pack(Vs, IDIR, sendBoxVsIdir); + int VsIndex = mapNVars*nx*ny*nz; idefix_for("LoadBufferX2IDIR",kbeg,kend,jbeg,jend,ibeg,iend+1, KOKKOS_LAMBDA (int k, int j, int i) { - bufferSend(i + (j-jbeg)*(nx+1) + (k-kbeg)*(nx+1)*ny + VsIndex ) = + const int storeId = i + (j-jbeg)*(nx+1) + (k-kbeg)*(nx+1)*ny + VsIndex; + bufferSend(storeId ) = Vs(IDIR,k,j+ny,i); + assert(bufferSend(storeId) == bufferSendNew.getArray()(storeId)); } ); + + //extend by one + BoundingBox sendBoxVsKdir = sendBoxVc; + sendBoxVsKdir[KDIR][1] += 1; + bufferSendNew.Pack(Vs, KDIR, sendBoxVsKdir); + VsIndex = mapNVars*nx*ny*nz + (nx+1)*ny*nz; idefix_for("LoadBufferX2KDIR",kbeg,kend+1,jbeg,jend,ibeg,iend, KOKKOS_LAMBDA (int k, int j, int i) { - bufferSend(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + VsIndex ) = + const int storeId = i + (j-jbeg)*nx + (k-kbeg)*nx*ny + VsIndex; + bufferSend(storeId) = Vs(KDIR,k,j+ny,i); + assert(bufferSend(storeId) == bufferSendNew.getArray()(storeId)); } ); } // MHD } else if(side==right) { + //shift by offset - NY (to pub on right) + BoundingBox sendBoxVc = baseBox; + sendBoxVc[JDIR][0] += offset - ny; + sendBoxVc[JDIR][1] += offset - ny; + bufferSendNew.Pack(Vc, map, sendBoxVc); + idefix_for("LoadBufferX2Vc",0,mapNVars,kbeg,kend,jbeg,jend,ibeg,iend, KOKKOS_LAMBDA (int n, int k, int j, int i) { - bufferSend(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + n*nx*ny*nz ) = + const int storeId = i + (j-jbeg)*nx + (k-kbeg)*nx*ny + n*nx*ny*nz; + bufferSend(storeId) = Vc(map(n),k,j+offset-ny,i); + assert(bufferSend(storeId) == bufferSendNew.getArray()(storeId)); } ); // Load face-centered field in the buffer if (haveMHD) { + //extend by one + BoundingBox sendBoxVsIdir = baseBox; + sendBoxVsIdir[IDIR][1] += 1; + sendBoxVsIdir[JDIR][0] += offset - ny; + sendBoxVsIdir[JDIR][1] += offset - ny; + bufferSendNew.Pack(Vs, IDIR, sendBoxVsIdir); + int VsIndex = mapNVars*nx*ny*nz; idefix_for("LoadBufferX2IDIR",kbeg,kend,jbeg,jend,ibeg,iend+1, KOKKOS_LAMBDA (int k, int j, int i) { - bufferSend(i + (j-jbeg)*(nx+1) + (k-kbeg)*(nx+1)*ny + VsIndex ) = + const int storeId = i + (j-jbeg)*(nx+1) + (k-kbeg)*(nx+1)*ny + VsIndex; + bufferSend(storeId) = Vs(IDIR,k,j+offset-ny,i); + assert(bufferSend(storeId) == bufferSendNew.getArray()(storeId)); } ); - VsIndex = mapNVars*nx*ny*nz + (nx+1)*ny*nz; + //extend by one + BoundingBox sendBoxVsKdir = baseBox; + sendBoxVsKdir[KDIR][1] += 1; + sendBoxVsKdir[JDIR][0] += offset - ny; + sendBoxVsKdir[JDIR][1] += offset - ny; + bufferSendNew.Pack(Vs, KDIR, sendBoxVsKdir); + + VsIndex = mapNVars*nx*ny*nz + (nx+1)*ny*nz; idefix_for("LoadBufferX2KDIR",kbeg,kend+1,jbeg,jend,ibeg,iend, KOKKOS_LAMBDA (int k, int j, int i) { - bufferSend(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + VsIndex ) = + const int storeId = i + (j-jbeg)*nx + (k-kbeg)*nx*ny + VsIndex; + bufferSend(storeId ) = Vs(KDIR,k,j+offset-ny,i); + assert(bufferSend(storeId) == bufferSendNew.getArray()(storeId)); } ); } // MHD } // side==right + assert(bufferSendNew.getArray().size() == bufferSend.size()); + Kokkos::fence(); tStart = MPI_Wtime(); MPI_SAFE_CALL(MPI_Start(&sendRequest)); - MPI_Wait(&recvRequest,&recvStatus); + MPI_SAFE_CALL(MPI_Start(&sendRequestNew)); + MPI_Wait(&recvRequest,&recvStatusNew); + MPI_Wait(&recvRequestNew,&recvStatusNew); idfx::mpiCallsTimer += MPI_Wtime() - tStart; // Unpack auto bufferRecv=this->bufferRecv; + Buffer bufferRecvNew=this->bufferRecvNew; + bufferRecvNew.ResetPointer(); auto sVc = this->symmetryVc; + assert(bufferRecvNew.getArray().size() == bufferRecv.size()); + if(side==left) { + //unpack Vc + BoundingBox recvBoxVc = baseBox; + bufferRecvNew.UnpackJDirSymetric(Vc, map, sVc, recvBoxVc); + idefix_for("StoreBufferAxis",0,mapNVars,kbeg,kend,jbeg,jend,ibeg,iend, KOKKOS_LAMBDA (int n, int k, int j, int i) { - Vc(map(n),k,jend-(j-jbeg)-1,i) = - sVc(map(n))*bufferRecv(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + n*nx*ny*nz ); + assert(Vc(map(n),k,jend-(j-jbeg)-1,i) == + sVc(map(n))*bufferRecv(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + n*nx*ny*nz )); } ); @@ -493,27 +577,44 @@ void Axis::ExchangeMPI(int side) { if (haveMHD) { int VsIndex = mapNVars*nx*ny*nz; auto sVs = this->symmetryVs; + + //unpack Vc + BoundingBox recvBoxVsIdir = baseBox; + recvBoxVsIdir[IDIR][1] += 1; + bufferRecvNew.UnpackJDirSymetric(Vs, IDIR, sVs(IDIR), recvBoxVsIdir); + idefix_for("StoreBufferX2IDIR",kbeg,kend,jbeg,jend,ibeg,iend+1, KOKKOS_LAMBDA (int k, int j, int i) { - Vs(IDIR,k,jend-(j-jbeg)-1,i) = - sVs(IDIR)*bufferRecv(i + (j-jbeg)*(nx+1) + (k-kbeg)*(nx+1)*ny + VsIndex ); + assert(Vs(IDIR,k,jend-(j-jbeg)-1,i) == + sVs(IDIR)*bufferRecv(i + (j-jbeg)*(nx+1) + (k-kbeg)*(nx+1)*ny + VsIndex )); } ); VsIndex = mapNVars*nx*ny*nz + (nx+1)*ny*nz; + //unpack Vc + BoundingBox recvBoxVsKdir = baseBox; + recvBoxVsKdir[KDIR][1] += 1; + bufferRecvNew.UnpackJDirSymetric(Vs, KDIR, sVs(KDIR), recvBoxVsKdir); + idefix_for("StoreBufferX2KDIR",kbeg,kend+1,jbeg,jend,ibeg,iend, KOKKOS_LAMBDA ( int k, int j, int i) { - Vs(KDIR,k,jend-(j-jbeg)-1,i) = - sVs(KDIR)*bufferRecv(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + VsIndex ); + assert(Vs(KDIR,k,jend-(j-jbeg)-1,i) == + sVs(KDIR)*bufferRecv(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + VsIndex )); } ); } } else if(side==right) { + //unpack Vc + BoundingBox recvBoxVc = baseBox; + recvBoxVc[JDIR][0] += offset; + recvBoxVc[JDIR][1] += offset; + bufferRecvNew.UnpackJDirSymetric(Vc, map, sVc, recvBoxVc); + idefix_for("StoreBufferAxis",0,mapNVars,kbeg,kend,jbeg,jend,ibeg,iend, KOKKOS_LAMBDA (int n, int k, int j, int i) { - Vc(map(n),k,jend-(j-jbeg)-1+offset,i) = - sVc(map(n))*bufferRecv(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + n*nx*ny*nz ); + assert(Vc(map(n),k,jend-(j-jbeg)-1+offset,i) == + sVc(map(n))*bufferRecv(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + n*nx*ny*nz )); } ); @@ -521,24 +622,40 @@ void Axis::ExchangeMPI(int side) { if (haveMHD) { int VsIndex = mapNVars*nx*ny*nz; auto sVs = this->symmetryVs; + + //unpack Vc + BoundingBox recvBoxVsIdir = baseBox; + recvBoxVsIdir[IDIR][1] += 1; + recvBoxVsIdir[JDIR][0] += offset; + recvBoxVsIdir[JDIR][1] += offset; + bufferRecvNew.UnpackJDirSymetric(Vs, IDIR, sVs(IDIR), recvBoxVsIdir); + idefix_for("StoreBufferX2IDIR",kbeg,kend,jbeg,jend,ibeg,iend+1, KOKKOS_LAMBDA (int k, int j, int i) { - Vs(IDIR,k,jend-(j-jbeg)-1+offset,i) = - sVs(IDIR)*bufferRecv(i + (j-jbeg)*(nx+1) + (k-kbeg)*(nx+1)*ny + VsIndex ); + assert(Vs(IDIR,k,jend-(j-jbeg)-1+offset,i) == + sVs(IDIR)*bufferRecv(i + (j-jbeg)*(nx+1) + (k-kbeg)*(nx+1)*ny + VsIndex )); } ); VsIndex = mapNVars*nx*ny*nz + (nx+1)*ny*nz; + //unpack Vc + BoundingBox recvBoxVsKdir = baseBox; + recvBoxVsKdir[KDIR][1] += 1; + recvBoxVsKdir[JDIR][0] += offset; + recvBoxVsKdir[JDIR][1] += offset; + bufferRecvNew.UnpackJDirSymetric(Vs, KDIR, sVs(KDIR), recvBoxVsKdir); + idefix_for("StoreBufferX2KDIR",kbeg,kend+1,jbeg,jend,ibeg,iend, KOKKOS_LAMBDA ( int k, int j, int i) { - Vs(KDIR,k,jend-(j-jbeg)-1+offset,i) = - sVs(KDIR)*bufferRecv(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + VsIndex ); + assert(Vs(KDIR,k,jend-(j-jbeg)-1+offset,i) == + sVs(KDIR)*bufferRecv(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + VsIndex )); } ); } // MHD } MPI_Wait(&sendRequest, &sendStatus); + MPI_Wait(&sendRequestNew, &sendStatusNew); idfx::mpiCallsTimer += MPI_Wtime() - tStart; @@ -597,6 +714,10 @@ void Axis::InitMPI() { this->bufferRecv = IdefixArray1D("bufferRecvAxis", bufferSize); this->bufferSend = IdefixArray1D("bufferSendAxis", bufferSize); + //build buffers + this->bufferRecvNew = Buffer(bufferSize); + this->bufferSendNew = Buffer(bufferSize); + // init persistent communications // We receive from procRecv, and we send to procSend int procSend, procRecv; @@ -611,6 +732,15 @@ void Axis::InitMPI() { MPI_SAFE_CALL(MPI_Recv_init(bufferRecv.data(), bufferSize, realMPI, procRecv, 650, data->mygrid->AxisComm, &recvRequest)); + assert(bufferSize == bufferSendNew.Size()); + assert(bufferSize == bufferRecvNew.Size()); + + MPI_SAFE_CALL(MPI_Send_init(bufferSendNew.data(), bufferSendNew.Size(), realMPI, procSend, + 1666651, data->mygrid->AxisComm, &sendRequestNew)); + + MPI_SAFE_CALL(MPI_Recv_init(bufferRecvNew.data(), bufferRecvNew.Size(), realMPI, procRecv, + 1666651, data->mygrid->AxisComm, &recvRequestNew)); + #endif idfx::popRegion(); } diff --git a/src/fluid/boundary/axis.hpp b/src/fluid/boundary/axis.hpp index 229c772f6..1cb78504d 100644 --- a/src/fluid/boundary/axis.hpp +++ b/src/fluid/boundary/axis.hpp @@ -11,6 +11,7 @@ #include #include "idefix.hpp" #include "grid.hpp" +#include "buffer.hpp" // Forward class hydro declaration #include "physics.hpp" @@ -58,10 +59,15 @@ class Axis { #ifdef WITH_MPI MPI_Request sendRequest; MPI_Request recvRequest; + MPI_Request sendRequestNew; + MPI_Request recvRequestNew; IdefixArray1D bufferSend; IdefixArray1D bufferRecv; + Buffer bufferSendNew; + Buffer bufferRecvNew; + int bufferSize; IdefixArray1D mapVars; diff --git a/src/mpi/buffer.hpp b/src/mpi/buffer.hpp index ff4dff146..601005663 100644 --- a/src/mpi/buffer.hpp +++ b/src/mpi/buffer.hpp @@ -31,6 +31,10 @@ class Buffer { return(array.data()); } + const IdefixArray1D & getArray(void) const { + return this->array; + } + int Size() { return(array.size()); } @@ -159,6 +163,33 @@ class Buffer { this->pointer += ninjnk; } + void UnpackJDirSymetric(IdefixArray4D& out, + const int var, + const int symMultiplier, + BoundingBox box) { + const int ni = box[IDIR][1]-box[IDIR][0]; + const int ninj = (box[JDIR][1]-box[JDIR][0])*ni; + const int ninjnk = (box[KDIR][1]-box[KDIR][0])*ninj; + const int ibeg = box[IDIR][0]; + const int jbeg = box[JDIR][0]; + const int kbeg = box[KDIR][0]; + const int iend = box[IDIR][1]; + const int jend = box[JDIR][1]; + const int kend = box[KDIR][1]; + const int offset = this->pointer; + + auto arr = this->array; + idefix_for("UnLoadBuffer4D_var",kbeg,kend,jbeg,jend,ibeg,iend, + KOKKOS_LAMBDA (int k, int j, int i) { + const int jinverted = jend-(j-jbeg)-1; + const int arrIndex = i-ibeg + (j-jbeg)*ni + (k-kbeg)*ninj + offset; + out(var,k,jinverted,i) = symMultiplier * arr(arrIndex); + }); + + // Update pointer + this->pointer += ninjnk; + } + void Unpack(IdefixArray4D& out, IdefixArray1D& map, BoundingBox box) { @@ -186,6 +217,36 @@ class Buffer { this->pointer += ninjnk*map.size(); } + void UnpackJDirSymetric(IdefixArray4D& out, + IdefixArray1D& map, + IdefixArray1D& SymMap, + BoundingBox box) { + const int ni = box[IDIR][1]-box[IDIR][0]; + const int ninj = (box[JDIR][1]-box[JDIR][0])*ni; + const int ninjnk = (box[KDIR][1]-box[KDIR][0])*ninj; + const int ibeg = box[IDIR][0]; + const int jbeg = box[JDIR][0]; + const int kbeg = box[KDIR][0]; + const int iend = box[IDIR][1]; + const int jend = box[JDIR][1]; + const int kend = box[KDIR][1]; + const int offset = this->pointer; + + auto arr = this->array; + idefix_for("UnLoadBuffer4D_map",0,map.size(), + kbeg,kend, + jbeg,jend, + ibeg,iend, + KOKKOS_LAMBDA (int n, int k, int j, int i) { + const int jinverted = jend-(j-jbeg)-1; + const int arrIndex = i-ibeg + (j-jbeg)*ni + (k-kbeg)*ninj + n*ninjnk + offset; + out(map(n),k,jinverted,i) = SymMap(map(n)) * arr(arrIndex); + }); + + // Update pointer + this->pointer += ninjnk*map.size(); + } + private: size_t pointer; From da1faf98410b13f02f849078e24acc8a5f17f9b6 Mon Sep 17 00:00:00 2001 From: Sebastien Valat Date: Fri, 19 Jun 2026 13:10:49 +0200 Subject: [PATCH 2/8] refactor: axis communication by using Buffer (step 2 - remove the old code after cross validation) --- src/fluid/boundary/axis.cpp | 174 +++++------------------------------- src/fluid/boundary/axis.hpp | 9 +- 2 files changed, 23 insertions(+), 160 deletions(-) diff --git a/src/fluid/boundary/axis.cpp b/src/fluid/boundary/axis.cpp index da658e6c1..c6402184f 100644 --- a/src/fluid/boundary/axis.cpp +++ b/src/fluid/boundary/axis.cpp @@ -5,11 +5,6 @@ // Licensed under CeCILL 2.1 License, see COPYING for more information // *********************************************************************************** -#undef NDEBUG -#include -#include -#include - #include #include "axis.hpp" #include "boundary.hpp" @@ -399,25 +394,21 @@ void Axis::ExchangeMPI(int side) { // Load the buffers with data int ibeg,iend,jbeg,jend,kbeg,kend,offset; int nx,ny,nz; - auto bufferSend = this->bufferSend; - Buffer bufferSendNew = this->bufferSendNew; + Buffer bufferSend = this->bufferSend; IdefixArray1D map = this->mapVars; IdefixArray4D Vc = this->Vc; IdefixArray4D Vs = this->Vs; //reset pointer - bufferSendNew.ResetPointer(); + bufferSend.ResetPointer(); // If MPI Persistent, start receiving even before the buffers are filled MPI_Status sendStatus; MPI_Status recvStatus; - MPI_Status sendStatusNew; - MPI_Status recvStatusNew; double tStart = MPI_Wtime(); MPI_SAFE_CALL(MPI_Start(&recvRequest)); - MPI_SAFE_CALL(MPI_Start(&recvRequestNew)); idfx::mpiCallsTimer += MPI_Wtime() - tStart; // Coordinates of the ghost region which needs to be transfered @@ -446,63 +437,25 @@ void Axis::ExchangeMPI(int side) { BoundingBox sendBoxVc = baseBox; sendBoxVc[JDIR][0] += ny; sendBoxVc[JDIR][1] += ny; - bufferSendNew.Pack(Vc, map, sendBoxVc); - - idefix_for("LoadBufferX2Vc",0,mapNVars,kbeg,kend,jbeg,jend,ibeg,iend, - KOKKOS_LAMBDA (int n, int k, int j, int i) { - const int storeId = i + (j-jbeg)*nx + (k-kbeg)*nx*ny + n*nx*ny*nz; - bufferSend(storeId ) = - Vc(map(n),k,j+ny,i); - assert(bufferSend(storeId) == bufferSendNew.getArray()(storeId)); - } - ); + bufferSend.Pack(Vc, map, sendBoxVc); if (haveMHD) { //extend by one BoundingBox sendBoxVsIdir = sendBoxVc; sendBoxVsIdir[IDIR][1] += 1; - bufferSendNew.Pack(Vs, IDIR, sendBoxVsIdir); - - int VsIndex = mapNVars*nx*ny*nz; - idefix_for("LoadBufferX2IDIR",kbeg,kend,jbeg,jend,ibeg,iend+1, - KOKKOS_LAMBDA (int k, int j, int i) { - const int storeId = i + (j-jbeg)*(nx+1) + (k-kbeg)*(nx+1)*ny + VsIndex; - bufferSend(storeId ) = - Vs(IDIR,k,j+ny,i); - assert(bufferSend(storeId) == bufferSendNew.getArray()(storeId)); - } - ); + bufferSend.Pack(Vs, IDIR, sendBoxVsIdir); //extend by one BoundingBox sendBoxVsKdir = sendBoxVc; sendBoxVsKdir[KDIR][1] += 1; - bufferSendNew.Pack(Vs, KDIR, sendBoxVsKdir); - - VsIndex = mapNVars*nx*ny*nz + (nx+1)*ny*nz; - idefix_for("LoadBufferX2KDIR",kbeg,kend+1,jbeg,jend,ibeg,iend, - KOKKOS_LAMBDA (int k, int j, int i) { - const int storeId = i + (j-jbeg)*nx + (k-kbeg)*nx*ny + VsIndex; - bufferSend(storeId) = - Vs(KDIR,k,j+ny,i); - assert(bufferSend(storeId) == bufferSendNew.getArray()(storeId)); - } - ); + bufferSend.Pack(Vs, KDIR, sendBoxVsKdir); } // MHD } else if(side==right) { //shift by offset - NY (to pub on right) BoundingBox sendBoxVc = baseBox; sendBoxVc[JDIR][0] += offset - ny; sendBoxVc[JDIR][1] += offset - ny; - bufferSendNew.Pack(Vc, map, sendBoxVc); - - idefix_for("LoadBufferX2Vc",0,mapNVars,kbeg,kend,jbeg,jend,ibeg,iend, - KOKKOS_LAMBDA (int n, int k, int j, int i) { - const int storeId = i + (j-jbeg)*nx + (k-kbeg)*nx*ny + n*nx*ny*nz; - bufferSend(storeId) = - Vc(map(n),k,j+offset-ny,i); - assert(bufferSend(storeId) == bufferSendNew.getArray()(storeId)); - } - ); + bufferSend.Pack(Vc, map, sendBoxVc); // Load face-centered field in the buffer if (haveMHD) { @@ -511,67 +464,33 @@ void Axis::ExchangeMPI(int side) { sendBoxVsIdir[IDIR][1] += 1; sendBoxVsIdir[JDIR][0] += offset - ny; sendBoxVsIdir[JDIR][1] += offset - ny; - bufferSendNew.Pack(Vs, IDIR, sendBoxVsIdir); - - int VsIndex = mapNVars*nx*ny*nz; - idefix_for("LoadBufferX2IDIR",kbeg,kend,jbeg,jend,ibeg,iend+1, - KOKKOS_LAMBDA (int k, int j, int i) { - const int storeId = i + (j-jbeg)*(nx+1) + (k-kbeg)*(nx+1)*ny + VsIndex; - bufferSend(storeId) = - Vs(IDIR,k,j+offset-ny,i); - assert(bufferSend(storeId) == bufferSendNew.getArray()(storeId)); - } - ); + bufferSend.Pack(Vs, IDIR, sendBoxVsIdir); //extend by one BoundingBox sendBoxVsKdir = baseBox; sendBoxVsKdir[KDIR][1] += 1; sendBoxVsKdir[JDIR][0] += offset - ny; sendBoxVsKdir[JDIR][1] += offset - ny; - bufferSendNew.Pack(Vs, KDIR, sendBoxVsKdir); - - VsIndex = mapNVars*nx*ny*nz + (nx+1)*ny*nz; - idefix_for("LoadBufferX2KDIR",kbeg,kend+1,jbeg,jend,ibeg,iend, - KOKKOS_LAMBDA (int k, int j, int i) { - const int storeId = i + (j-jbeg)*nx + (k-kbeg)*nx*ny + VsIndex; - bufferSend(storeId ) = - Vs(KDIR,k,j+offset-ny,i); - assert(bufferSend(storeId) == bufferSendNew.getArray()(storeId)); - } - ); + bufferSend.Pack(Vs, KDIR, sendBoxVsKdir); } // MHD } // side==right - assert(bufferSendNew.getArray().size() == bufferSend.size()); - Kokkos::fence(); tStart = MPI_Wtime(); MPI_SAFE_CALL(MPI_Start(&sendRequest)); - MPI_SAFE_CALL(MPI_Start(&sendRequestNew)); - MPI_Wait(&recvRequest,&recvStatusNew); - MPI_Wait(&recvRequestNew,&recvStatusNew); + MPI_Wait(&recvRequest,&recvStatus); idfx::mpiCallsTimer += MPI_Wtime() - tStart; // Unpack - auto bufferRecv=this->bufferRecv; - Buffer bufferRecvNew=this->bufferRecvNew; - bufferRecvNew.ResetPointer(); + Buffer bufferRecv=this->bufferRecv; + bufferRecv.ResetPointer(); auto sVc = this->symmetryVc; - assert(bufferRecvNew.getArray().size() == bufferRecv.size()); - if(side==left) { //unpack Vc BoundingBox recvBoxVc = baseBox; - bufferRecvNew.UnpackJDirSymetric(Vc, map, sVc, recvBoxVc); - - idefix_for("StoreBufferAxis",0,mapNVars,kbeg,kend,jbeg,jend,ibeg,iend, - KOKKOS_LAMBDA (int n, int k, int j, int i) { - assert(Vc(map(n),k,jend-(j-jbeg)-1,i) == - sVc(map(n))*bufferRecv(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + n*nx*ny*nz )); - } - ); + bufferRecv.UnpackJDirSymetric(Vc, map, sVc, recvBoxVc); // Load face-centered field in the buffer if (haveMHD) { @@ -581,42 +500,19 @@ void Axis::ExchangeMPI(int side) { //unpack Vc BoundingBox recvBoxVsIdir = baseBox; recvBoxVsIdir[IDIR][1] += 1; - bufferRecvNew.UnpackJDirSymetric(Vs, IDIR, sVs(IDIR), recvBoxVsIdir); - - idefix_for("StoreBufferX2IDIR",kbeg,kend,jbeg,jend,ibeg,iend+1, - KOKKOS_LAMBDA (int k, int j, int i) { - assert(Vs(IDIR,k,jend-(j-jbeg)-1,i) == - sVs(IDIR)*bufferRecv(i + (j-jbeg)*(nx+1) + (k-kbeg)*(nx+1)*ny + VsIndex )); - } - ); - - VsIndex = mapNVars*nx*ny*nz + (nx+1)*ny*nz; + bufferRecv.UnpackJDirSymetric(Vs, IDIR, sVs(IDIR), recvBoxVsIdir); //unpack Vc BoundingBox recvBoxVsKdir = baseBox; recvBoxVsKdir[KDIR][1] += 1; - bufferRecvNew.UnpackJDirSymetric(Vs, KDIR, sVs(KDIR), recvBoxVsKdir); - - idefix_for("StoreBufferX2KDIR",kbeg,kend+1,jbeg,jend,ibeg,iend, - KOKKOS_LAMBDA ( int k, int j, int i) { - assert(Vs(KDIR,k,jend-(j-jbeg)-1,i) == - sVs(KDIR)*bufferRecv(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + VsIndex )); - } - ); + bufferRecv.UnpackJDirSymetric(Vs, KDIR, sVs(KDIR), recvBoxVsKdir); } } else if(side==right) { //unpack Vc BoundingBox recvBoxVc = baseBox; recvBoxVc[JDIR][0] += offset; recvBoxVc[JDIR][1] += offset; - bufferRecvNew.UnpackJDirSymetric(Vc, map, sVc, recvBoxVc); - - idefix_for("StoreBufferAxis",0,mapNVars,kbeg,kend,jbeg,jend,ibeg,iend, - KOKKOS_LAMBDA (int n, int k, int j, int i) { - assert(Vc(map(n),k,jend-(j-jbeg)-1+offset,i) == - sVc(map(n))*bufferRecv(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + n*nx*ny*nz )); - } - ); + bufferRecv.UnpackJDirSymetric(Vc, map, sVc, recvBoxVc); // Load face-centered field in the buffer if (haveMHD) { @@ -628,34 +524,18 @@ void Axis::ExchangeMPI(int side) { recvBoxVsIdir[IDIR][1] += 1; recvBoxVsIdir[JDIR][0] += offset; recvBoxVsIdir[JDIR][1] += offset; - bufferRecvNew.UnpackJDirSymetric(Vs, IDIR, sVs(IDIR), recvBoxVsIdir); - - idefix_for("StoreBufferX2IDIR",kbeg,kend,jbeg,jend,ibeg,iend+1, - KOKKOS_LAMBDA (int k, int j, int i) { - assert(Vs(IDIR,k,jend-(j-jbeg)-1+offset,i) == - sVs(IDIR)*bufferRecv(i + (j-jbeg)*(nx+1) + (k-kbeg)*(nx+1)*ny + VsIndex )); - } - ); - VsIndex = mapNVars*nx*ny*nz + (nx+1)*ny*nz; + bufferRecv.UnpackJDirSymetric(Vs, IDIR, sVs(IDIR), recvBoxVsIdir); //unpack Vc BoundingBox recvBoxVsKdir = baseBox; recvBoxVsKdir[KDIR][1] += 1; recvBoxVsKdir[JDIR][0] += offset; recvBoxVsKdir[JDIR][1] += offset; - bufferRecvNew.UnpackJDirSymetric(Vs, KDIR, sVs(KDIR), recvBoxVsKdir); - - idefix_for("StoreBufferX2KDIR",kbeg,kend+1,jbeg,jend,ibeg,iend, - KOKKOS_LAMBDA ( int k, int j, int i) { - assert(Vs(KDIR,k,jend-(j-jbeg)-1+offset,i) == - sVs(KDIR)*bufferRecv(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + VsIndex )); - } - ); + bufferRecv.UnpackJDirSymetric(Vs, KDIR, sVs(KDIR), recvBoxVsKdir); } // MHD } MPI_Wait(&sendRequest, &sendStatus); - MPI_Wait(&sendRequestNew, &sendStatusNew); idfx::mpiCallsTimer += MPI_Wtime() - tStart; @@ -711,12 +591,9 @@ void Axis::InitMPI() { #endif // DIMENSIONS } - this->bufferRecv = IdefixArray1D("bufferRecvAxis", bufferSize); - this->bufferSend = IdefixArray1D("bufferSendAxis", bufferSize); - //build buffers - this->bufferRecvNew = Buffer(bufferSize); - this->bufferSendNew = Buffer(bufferSize); + this->bufferRecv = Buffer(bufferSize); + this->bufferSend = Buffer(bufferSize); // init persistent communications // We receive from procRecv, and we send to procSend @@ -726,21 +603,12 @@ void Axis::InitMPI() { MPI_SAFE_CALL(MPI_Cart_shift(data->mygrid->AxisComm,0,data->mygrid->nproc[KDIR]/2, &procRecv,&procSend )); - MPI_SAFE_CALL(MPI_Send_init(bufferSend.data(), bufferSize, realMPI, procSend, + MPI_SAFE_CALL(MPI_Send_init(bufferSend.data(), bufferSend.Size(), realMPI, procSend, 650, data->mygrid->AxisComm, &sendRequest)); - MPI_SAFE_CALL(MPI_Recv_init(bufferRecv.data(), bufferSize, realMPI, procRecv, + MPI_SAFE_CALL(MPI_Recv_init(bufferRecv.data(), bufferRecv.Size(), realMPI, procRecv, 650, data->mygrid->AxisComm, &recvRequest)); - assert(bufferSize == bufferSendNew.Size()); - assert(bufferSize == bufferRecvNew.Size()); - - MPI_SAFE_CALL(MPI_Send_init(bufferSendNew.data(), bufferSendNew.Size(), realMPI, procSend, - 1666651, data->mygrid->AxisComm, &sendRequestNew)); - - MPI_SAFE_CALL(MPI_Recv_init(bufferRecvNew.data(), bufferRecvNew.Size(), realMPI, procRecv, - 1666651, data->mygrid->AxisComm, &recvRequestNew)); - #endif idfx::popRegion(); } diff --git a/src/fluid/boundary/axis.hpp b/src/fluid/boundary/axis.hpp index 1cb78504d..390a628ce 100644 --- a/src/fluid/boundary/axis.hpp +++ b/src/fluid/boundary/axis.hpp @@ -59,14 +59,9 @@ class Axis { #ifdef WITH_MPI MPI_Request sendRequest; MPI_Request recvRequest; - MPI_Request sendRequestNew; - MPI_Request recvRequestNew; - IdefixArray1D bufferSend; - IdefixArray1D bufferRecv; - - Buffer bufferSendNew; - Buffer bufferRecvNew; + Buffer bufferSend; + Buffer bufferRecv; int bufferSize; From b8ea3ff62bb051ce1d117d7f21565c66212b4b4c Mon Sep 17 00:00:00 2001 From: Sebastien Valat Date: Fri, 19 Jun 2026 13:56:27 +0200 Subject: [PATCH 3/8] refactor: use better comments + give uniq name to the idefix_for new kernels --- src/fluid/boundary/axis.cpp | 22 +++++++++++----------- src/mpi/buffer.hpp | 4 ++-- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/fluid/boundary/axis.cpp b/src/fluid/boundary/axis.cpp index c6402184f..a67621934 100644 --- a/src/fluid/boundary/axis.cpp +++ b/src/fluid/boundary/axis.cpp @@ -423,7 +423,7 @@ void Axis::ExchangeMPI(int side) { kend = data->end[KDIR]; nz = kend - kbeg; - // Create the box + // Create the base box to be patched by the ops BoundingBox baseBox; baseBox[IDIR][0] = ibeg; baseBox[IDIR][1] = iend; @@ -440,18 +440,18 @@ void Axis::ExchangeMPI(int side) { bufferSend.Pack(Vc, map, sendBoxVc); if (haveMHD) { - //extend by one + //extend by one the end on idir BoundingBox sendBoxVsIdir = sendBoxVc; sendBoxVsIdir[IDIR][1] += 1; bufferSend.Pack(Vs, IDIR, sendBoxVsIdir); - //extend by one + //extend by one the end on kdir BoundingBox sendBoxVsKdir = sendBoxVc; sendBoxVsKdir[KDIR][1] += 1; bufferSend.Pack(Vs, KDIR, sendBoxVsKdir); } // MHD } else if(side==right) { - //shift by offset - NY (to pub on right) + //shift by offset - NY (to select the right part) BoundingBox sendBoxVc = baseBox; sendBoxVc[JDIR][0] += offset - ny; sendBoxVc[JDIR][1] += offset - ny; @@ -459,14 +459,14 @@ void Axis::ExchangeMPI(int side) { // Load face-centered field in the buffer if (haveMHD) { - //extend by one + //extend by one the end on idir + take the right part or the mesh on jdir BoundingBox sendBoxVsIdir = baseBox; sendBoxVsIdir[IDIR][1] += 1; sendBoxVsIdir[JDIR][0] += offset - ny; sendBoxVsIdir[JDIR][1] += offset - ny; bufferSend.Pack(Vs, IDIR, sendBoxVsIdir); - //extend by one + //extend by one the end on kdir + take the right part or the mesh on jdir BoundingBox sendBoxVsKdir = baseBox; sendBoxVsKdir[KDIR][1] += 1; sendBoxVsKdir[JDIR][0] += offset - ny; @@ -497,18 +497,18 @@ void Axis::ExchangeMPI(int side) { int VsIndex = mapNVars*nx*ny*nz; auto sVs = this->symmetryVs; - //unpack Vc + //unpack Vs face-centered BoundingBox recvBoxVsIdir = baseBox; recvBoxVsIdir[IDIR][1] += 1; bufferRecv.UnpackJDirSymetric(Vs, IDIR, sVs(IDIR), recvBoxVsIdir); - //unpack Vc + //unpack Vs face-centered BoundingBox recvBoxVsKdir = baseBox; recvBoxVsKdir[KDIR][1] += 1; bufferRecv.UnpackJDirSymetric(Vs, KDIR, sVs(KDIR), recvBoxVsKdir); } } else if(side==right) { - //unpack Vc + //unpack Vc on right part BoundingBox recvBoxVc = baseBox; recvBoxVc[JDIR][0] += offset; recvBoxVc[JDIR][1] += offset; @@ -519,14 +519,14 @@ void Axis::ExchangeMPI(int side) { int VsIndex = mapNVars*nx*ny*nz; auto sVs = this->symmetryVs; - //unpack Vc + //unpack Vs face-centered on right part BoundingBox recvBoxVsIdir = baseBox; recvBoxVsIdir[IDIR][1] += 1; recvBoxVsIdir[JDIR][0] += offset; recvBoxVsIdir[JDIR][1] += offset; bufferRecv.UnpackJDirSymetric(Vs, IDIR, sVs(IDIR), recvBoxVsIdir); - //unpack Vc + //unpack Vs face-centered on right part BoundingBox recvBoxVsKdir = baseBox; recvBoxVsKdir[KDIR][1] += 1; recvBoxVsKdir[JDIR][0] += offset; diff --git a/src/mpi/buffer.hpp b/src/mpi/buffer.hpp index 601005663..35164355b 100644 --- a/src/mpi/buffer.hpp +++ b/src/mpi/buffer.hpp @@ -179,7 +179,7 @@ class Buffer { const int offset = this->pointer; auto arr = this->array; - idefix_for("UnLoadBuffer4D_var",kbeg,kend,jbeg,jend,ibeg,iend, + idefix_for("UnLoadBuffer4D_var_sym",kbeg,kend,jbeg,jend,ibeg,iend, KOKKOS_LAMBDA (int k, int j, int i) { const int jinverted = jend-(j-jbeg)-1; const int arrIndex = i-ibeg + (j-jbeg)*ni + (k-kbeg)*ninj + offset; @@ -233,7 +233,7 @@ class Buffer { const int offset = this->pointer; auto arr = this->array; - idefix_for("UnLoadBuffer4D_map",0,map.size(), + idefix_for("UnLoadBuffer4D_map_sym",0,map.size(), kbeg,kend, jbeg,jend, ibeg,iend, From 56a570b9cd76b54081ca368dbce088a9652542a8 Mon Sep 17 00:00:00 2001 From: Sebastien Valat Date: Fri, 19 Jun 2026 16:48:37 +0200 Subject: [PATCH 4/8] refactor: use Buffer and BoundingBox for the MPI communications of ConstrainedTransport (step 1) --- .../constrainedTransport/EMFexchange.hpp | 158 +++++++++++++++++- .../constrainedTransport.hpp | 70 ++++++++ 2 files changed, 227 insertions(+), 1 deletion(-) diff --git a/src/fluid/constrainedTransport/EMFexchange.hpp b/src/fluid/constrainedTransport/EMFexchange.hpp index f6e90f75a..48d83ab9b 100644 --- a/src/fluid/constrainedTransport/EMFexchange.hpp +++ b/src/fluid/constrainedTransport/EMFexchange.hpp @@ -8,6 +8,8 @@ #ifndef FLUID_CONSTRAINEDTRANSPORT_EMFEXCHANGE_HPP_ #define FLUID_CONSTRAINEDTRANSPORT_EMFEXCHANGE_HPP_ +#include + #include "constrainedTransport.hpp" #include "fluid.hpp" #include "dataBlock.hpp" @@ -36,14 +38,19 @@ void ConstrainedTransport::ExchangeX1(IdefixArray3D ey, IdefixArray3 IdefixArray1D BufferLeft=BufferSendX1[faceLeft]; IdefixArray1D BufferRight=BufferSendX1[faceRight]; + Buffer BufferLeftNew=BufferSendX1New[faceLeft]; + Buffer BufferRightNew=BufferSendX1New[faceRight]; // If MPI Persistent, start receiving even before the buffers are filled MPI_Status sendStatus[2]; MPI_Status recvStatus[2]; + MPI_Status sendStatusNew[2]; + MPI_Status recvStatusNew[2]; double tStart = MPI_Wtime(); MPI_SAFE_CALL(MPI_Startall(2, recvRequestX1)); + MPI_SAFE_CALL(MPI_Startall(2, recvRequestX1New)); idfx::mpiCallsTimer += MPI_Wtime() - tStart; BoundaryType lbound = data->lbound[IDIR]; @@ -59,50 +66,97 @@ void ConstrainedTransport::ExchangeX1(IdefixArray3D ey, IdefixArray3 kend = data->end[KDIR]; nz = kend - kbeg; + // Create the base box to be patched by the ops + BoundingBox baseBox; + baseBox[IDIR][0] = data->beg[IDIR]; + baseBox[IDIR][1] = data->end[IDIR]; + baseBox[JDIR][0] = data->beg[JDIR]; + baseBox[JDIR][1] = data->end[JDIR]; + baseBox[KDIR][0] = data->beg[KDIR]; + baseBox[KDIR][1] = data->end[KDIR]; + + //extend by one the end on jdir && take the ghost on i + BoundingBox sendBoxEz = baseBox; + sendBoxEz[JDIR][1] += 1; + sendBoxEz[IDIR][0] = iright; + sendBoxEz[IDIR][1] = iright + 1; + BufferRightNew.Pack(ez, sendBoxEz); idefix_for("LoadBufferX1Emfz",kbeg,kend,jbeg,jend+1, KOKKOS_LAMBDA (int k, int j) { BufferRight( (j-jbeg) + (k-kbeg)*(ny+1) ) = ez(k,j,iright); + assert(BufferRightNew.getArray()((j-jbeg) + (k-kbeg)*(ny+1)) == ez(k,j,iright)); } ); #if DIMENSIONS == 3 int Vsindex = (ny+1)*nz; + //extend by one the end on kdir && take the ghost on i + BoundingBox sendBoxEy = baseBox; + sendBoxEy[KDIR][1] += 1; + sendBoxEy[IDIR][0] = iright; + sendBoxEy[IDIR][1] = iright + 1; + BufferRightNew.Pack(ez, sendBoxEy); idefix_for("LoadBufferX1Emfy",kbeg,kend+1,jbeg,jend, KOKKOS_LAMBDA (int k, int j) { BufferRight( (j-jbeg) + (k-kbeg)*ny + Vsindex ) = ey(k,j,iright); + assert(BufferRightNew.getArray()( (j-jbeg) + (k-kbeg)*ny + Vsindex ) == ey(k,j,iright)); } ); #endif + assert(BufferRight == BufferRightNew.getArray()); + // Wait for completion before sending out everything Kokkos::fence(); tStart = MPI_Wtime(); MPI_SAFE_CALL(MPI_Startall(2, sendRequestX1)); + MPI_SAFE_CALL(MPI_Startall(2, sendRequestX1New)); // Wait for buffers to be received MPI_Waitall(2,recvRequestX1,recvStatus); + MPI_Waitall(2,recvRequestX1New,recvStatusNew); MPI_Waitall(2, sendRequestX1, sendStatus); + MPI_Waitall(2, sendRequestX1New, sendStatusNew); idfx::mpiCallsTimer += MPI_Wtime() - tStart; // Unpack BufferLeft=BufferRecvX1[faceLeft]; BufferRight=BufferRecvX1[faceRight]; + BufferLeftNew=BufferRecvX1New[faceLeft]; + BufferRightNew=BufferRecvX1New[faceRight]; - // Erase the emf with the one coming from the left process + + assert(BufferLeft == BufferLeftNew.getArray()); + + // Erase the emf with the one coming from the left process + //extend by one the end on jdir && take the ghost revc zone on i + BoundingBox recvBoxEz = baseBox; + recvBoxEz[JDIR][1] += 1; + recvBoxEz[IDIR][0] = ileft; + recvBoxEz[IDIR][1] = ileft + 1; + BufferLeftNew.Unpack(ez, recvBoxEz); idefix_for("StoreBufferX1Emfz",kbeg,kend,jbeg,jend+1, KOKKOS_LAMBDA (int k, int j) { if(lbound == internal || lbound == periodic) { + assert(ez(k,j,ileft) == BufferLeft( (j-jbeg) + (k-kbeg)*(ny+1))); ez(k,j,ileft) = BufferLeft( (j-jbeg) + (k-kbeg)*(ny+1)); } }); #if DIMENSIONS == 3 Vsindex = (ny+1)*nz; + //extend by one the end on kdir && take the ghost revc zone on i + BoundingBox recvBoxEy = baseBox; + recvBoxEy[KDIR][1] += 1; + recvBoxEy[IDIR][0] = ileft; + recvBoxEy[IDIR][1] = ileft + 1; + BufferLeftNew.Unpack(ey, recvBoxEy); idefix_for("StoreBufferX1Emfy",kbeg,kend+1,jbeg,jend, KOKKOS_LAMBDA (int k, int j) { if(lbound == internal || lbound == periodic) { + assert(ey(k,j,ileft) == BufferLeft( (j-jbeg) + (k-kbeg)*ny +Vsindex)); ey(k,j,ileft) = BufferLeft( (j-jbeg) + (k-kbeg)*ny +Vsindex); } }); @@ -123,12 +177,17 @@ void ConstrainedTransport::ExchangeX2(IdefixArray3D ex, IdefixArray3 [[maybe_unused]] int nz; IdefixArray1D BufferLeft=BufferSendX2[faceLeft]; IdefixArray1D BufferRight=BufferSendX2[faceRight]; + Buffer BufferLeftNew=BufferSendX2New[faceLeft]; + Buffer BufferRightNew=BufferSendX2New[faceRight]; // If MPI Persistent, start receiving even before the buffers are filled double tStart = MPI_Wtime(); MPI_Status sendStatus[2]; MPI_Status recvStatus[2]; + MPI_Status sendStatusNew[2]; + MPI_Status recvStatusNew[2]; MPI_SAFE_CALL(MPI_Startall(2, recvRequestX2)); + MPI_SAFE_CALL(MPI_Startall(2, recvRequestX2New)); idfx::mpiCallsTimer += MPI_Wtime() - tStart; BoundaryType lbound = data->lbound[JDIR]; @@ -144,47 +203,93 @@ void ConstrainedTransport::ExchangeX2(IdefixArray3D ex, IdefixArray3 kend = data->end[KDIR]; nz = kend - kbeg; + // Create the base box to be patched by the ops + BoundingBox baseBox; + baseBox[IDIR][0] = data->beg[IDIR]; + baseBox[IDIR][1] = data->end[IDIR]; + baseBox[JDIR][0] = data->beg[JDIR]; + baseBox[JDIR][1] = data->end[JDIR]; + baseBox[KDIR][0] = data->beg[KDIR]; + baseBox[KDIR][1] = data->end[KDIR]; + + //extend by one the end on idir && take the ghost on j + BoundingBox sendBoxEz = baseBox; + sendBoxEz[IDIR][1] += 1; + sendBoxEz[JDIR][0] = jright; + sendBoxEz[JDIR][1] = jright + 1; + BufferRightNew.Pack(ez, sendBoxEz); idefix_for("LoadBufferX2Emfz",kbeg,kend,ibeg,iend+1, KOKKOS_LAMBDA (int k, int i) { BufferRight( (i-ibeg) + (k-kbeg)*(nx+1) ) = ez(k,jright,i); + assert(BufferRightNew.getArray()( (i-ibeg) + (k-kbeg)*(nx+1) ) == ez(k,jright,i)); } ); #if DIMENSIONS == 3 int Vsindex = (nx+1)*nz; + //extend by one the end on idir && take the ghost on j + BoundingBox sendBoxEx = baseBox; + sendBoxEx[KDIR][1] += 1; + sendBoxEx[JDIR][0] = jright; + sendBoxEx[JDIR][1] = jright + 1; + BufferRightNew.Pack(ex, sendBoxEx); idefix_for("LoadBufferX1Emfx",kbeg,kend+1,ibeg,iend, KOKKOS_LAMBDA (int k, int i) { BufferRight( (i-ibeg) + (k-kbeg)*nx + Vsindex ) = ex(k,jright,i); + assert(BufferRightNew.getArray()( (i-ibeg) + (k-kbeg)*nx + Vsindex ) == ex(k,jright,i)); } ); #endif + assert(BufferRightNew.getArray() == BufferRight); + // Wait for completion before sending out everything Kokkos::fence(); tStart = MPI_Wtime(); MPI_SAFE_CALL(MPI_Startall(2, sendRequestX2)); + MPI_SAFE_CALL(MPI_Startall(2, sendRequestX2New)); // Wait for buffers to be received MPI_Waitall(2,recvRequestX2,recvStatus); + MPI_Waitall(2,recvRequestX2New,recvStatus); MPI_Waitall(2, sendRequestX2, sendStatus); + MPI_Waitall(2, sendRequestX2New, sendStatus); idfx::mpiCallsTimer += MPI_Wtime() - tStart; // Unpack BufferLeft=BufferRecvX2[faceLeft]; BufferRight=BufferRecvX2[faceRight]; + BufferLeftNew=BufferRecvX2New[faceLeft]; + BufferRightNew=BufferRecvX2New[faceRight]; + + assert(BufferLeftNew.getArray() == BufferLeft); // We average the edge emfs zones + //extend by one the end on idir && take the ghost revc zone on j + BoundingBox recvBoxEz = baseBox; + recvBoxEz[IDIR][1] += 1; + recvBoxEz[JDIR][0] = jleft; + recvBoxEz[JDIR][1] = jleft + 1; + BufferLeftNew.Unpack(ez, recvBoxEz); idefix_for("StoreBufferX2Emfz",kbeg,kend,ibeg,iend+1, KOKKOS_LAMBDA (int k, int i) { if(lbound == internal || lbound == periodic) { + assert(ez(k,jleft,i) == BufferLeft( (i-ibeg) + (k-kbeg)*(nx+1) )); ez(k,jleft,i) = BufferLeft( (i-ibeg) + (k-kbeg)*(nx+1) ); } }); #if DIMENSIONS == 3 Vsindex = (nx+1)*nz; + //extend by one the end on kdir && take the ghost revc zone on j + BoundingBox recvBoxEx = baseBox; + recvBoxEx[KDIR][1] += 1; + recvBoxEx[JDIR][0] = jleft; + recvBoxEx[JDIR][1] = jleft + 1; + BufferLeftNew.Unpack(ex, recvBoxEx); idefix_for("StoreBufferX1Emfy",kbeg,kend+1,ibeg,iend, KOKKOS_LAMBDA (int k, int i) { if(lbound == internal || lbound == periodic) { + assert(ex(k,jleft,i) == BufferLeft( (i-ibeg) + (k-kbeg)*nx +Vsindex)); ex(k,jleft,i) = BufferLeft( (i-ibeg) + (k-kbeg)*nx +Vsindex); } }); @@ -205,6 +310,8 @@ void ConstrainedTransport::ExchangeX3(IdefixArray3D ex, IdefixArray3 int nx,ny; IdefixArray1D BufferLeft=BufferSendX3[faceLeft]; IdefixArray1D BufferRight=BufferSendX3[faceRight]; + Buffer BufferLeftNew=BufferSendX3New[faceLeft]; + Buffer BufferRightNew=BufferSendX3New[faceRight]; int Vsindex = 0; @@ -213,7 +320,10 @@ void ConstrainedTransport::ExchangeX3(IdefixArray3D ex, IdefixArray3 double tStart = MPI_Wtime(); MPI_Status sendStatus[2]; MPI_Status recvStatus[2]; + MPI_Status sendStatusNew[2]; + MPI_Status recvStatusNew[2]; MPI_SAFE_CALL(MPI_Startall(2, recvRequestX3)); + MPI_SAFE_CALL(MPI_Startall(2, recvRequestX3New)); idfx::mpiCallsTimer += MPI_Wtime() - tStart; BoundaryType lbound = data->lbound[KDIR]; @@ -231,45 +341,91 @@ void ConstrainedTransport::ExchangeX3(IdefixArray3D ex, IdefixArray3 kleft = data->beg[KDIR]; kright = data->end[KDIR]; + // Create the base box to be patched by the ops + BoundingBox baseBox; + baseBox[IDIR][0] = data->beg[IDIR]; + baseBox[IDIR][1] = data->end[IDIR]; + baseBox[JDIR][0] = data->beg[JDIR]; + baseBox[JDIR][1] = data->end[JDIR]; + baseBox[KDIR][0] = data->beg[KDIR]; + baseBox[KDIR][1] = data->end[KDIR]; + + //extend by one the end on jdir && take the ghost on k + BoundingBox sendBoxEz = baseBox; + sendBoxEz[JDIR][1] += 1; + sendBoxEz[KDIR][0] = kright; + sendBoxEz[KDIR][1] = kright + 1; + BufferRightNew.Pack(ez, sendBoxEz); idefix_for("LoadBufferX3Emfx",jbeg,jend+1,ibeg,iend, KOKKOS_LAMBDA (int j, int i) { BufferRight( (i-ibeg) + (j-jbeg)*nx ) = ex(kright,j,i); + assert(BufferRightNew.getArray()( (i-ibeg) + (j-jbeg)*nx ) == ex(kright,j,i)); } ); Vsindex = nx*(ny+1); + //extend by one the end on idir && take the ghost on k + BoundingBox sendBoxEy = baseBox; + sendBoxEy[IDIR][1] += 1; + sendBoxEy[KDIR][0] = kright; + sendBoxEy[KDIR][1] = kright + 1; + BufferRightNew.Pack(ey, sendBoxEy); idefix_for("LoadBufferX3Emfy",jbeg,jend,ibeg,iend+1, KOKKOS_LAMBDA (int j, int i) { BufferRight( (i-ibeg) + (j-jbeg)*(nx+1) + Vsindex ) = ey(kright,j,i); + assert(BufferRightNew.getArray()( (i-ibeg) + (j-jbeg)*(nx+1) + Vsindex ) == ey(kright,j,i)); } ); + assert(BufferRightNew.getArray() == BufferRight); + // Wait for completion before sending out everything Kokkos::fence(); tStart = MPI_Wtime(); MPI_SAFE_CALL(MPI_Startall(2, sendRequestX3)); + MPI_SAFE_CALL(MPI_Startall(2, sendRequestX3New)); // Wait for buffers to be received MPI_Waitall(2,recvRequestX3,recvStatus); + MPI_Waitall(2,recvRequestX3New,recvStatusNew); MPI_Waitall(2, sendRequestX3, sendStatus); + MPI_Waitall(2, sendRequestX3New, sendStatusNew); idfx::mpiCallsTimer += MPI_Wtime() - tStart; // Unpack BufferLeft=BufferRecvX3[faceLeft]; BufferRight=BufferRecvX3[faceRight]; + BufferLeftNew=BufferRecvX3New[faceLeft]; + BufferRightNew=BufferRecvX3New[faceRight]; + + assert(BufferLeftNew.getArray() == BufferLeft); // We average the edge emfs zones + //extend by one the end on jdir && take the ghost revc zone on k + BoundingBox recvBoxEx = baseBox; + recvBoxEx[JDIR][1] += 1; + recvBoxEx[KDIR][0] = kleft; + recvBoxEx[KDIR][1] = kleft + 1; + BufferLeftNew.Unpack(ex, recvBoxEx); idefix_for("StoreBufferX3Emfx",jbeg,jend+1,ibeg,iend, KOKKOS_LAMBDA (int j, int i) { if(lbound == internal || lbound == periodic) { + assert(ex(kleft,j,i) == BufferLeft( (i-ibeg) + (j-jbeg)*nx )); ex(kleft,j,i) = BufferLeft( (i-ibeg) + (j-jbeg)*nx ); } }); Vsindex = nx*(ny+1); + //extend by one the end on idir && take the ghost revc zone on k + BoundingBox recvBoxEy = baseBox; + recvBoxEy[IDIR][1] += 1; + recvBoxEy[KDIR][0] = kleft; + recvBoxEy[KDIR][1] = kleft + 1; + BufferLeftNew.Unpack(ey, recvBoxEy); idefix_for("StoreBufferX3Emfy",jbeg,jend,ibeg,iend+1, KOKKOS_LAMBDA (int j, int i) { if(lbound == internal || lbound == periodic) { + assert(ey(kleft,j,i) == BufferLeft( (i-ibeg) + (j-jbeg)*(nx+1) + Vsindex )); ey(kleft,j,i) = BufferLeft( (i-ibeg) + (j-jbeg)*(nx+1) + Vsindex ); } }); diff --git a/src/fluid/constrainedTransport/constrainedTransport.hpp b/src/fluid/constrainedTransport/constrainedTransport.hpp index 24c317c09..6e76a5baa 100644 --- a/src/fluid/constrainedTransport/constrainedTransport.hpp +++ b/src/fluid/constrainedTransport/constrainedTransport.hpp @@ -11,6 +11,7 @@ #include "idefix.hpp" #include "input.hpp" #include "riemannSolver.hpp" +#include "buffer.hpp" // Forward declarations #include "physics.hpp" @@ -128,6 +129,12 @@ class ConstrainedTransport { IdefixArray1D BufferRecvX1[2]; IdefixArray1D BufferRecvX2[2]; IdefixArray1D BufferRecvX3[2]; + Buffer BufferSendX1New[2]; + Buffer BufferSendX2New[2]; + Buffer BufferSendX3New[2]; + Buffer BufferRecvX1New[2]; + Buffer BufferRecvX2New[2]; + Buffer BufferRecvX3New[2]; IdefixArray1D mapVars; @@ -142,6 +149,12 @@ class ConstrainedTransport { MPI_Request recvRequestX1[2]; MPI_Request recvRequestX2[2]; MPI_Request recvRequestX3[2]; + MPI_Request sendRequestX1New[2]; + MPI_Request sendRequestX2New[2]; + MPI_Request sendRequestX3New[2]; + MPI_Request recvRequestX1New[2]; + MPI_Request recvRequestX2New[2]; + MPI_Request recvRequestX3New[2]; Kokkos::Timer timer; // Internal MPI timer #endif @@ -291,6 +304,11 @@ ConstrainedTransport::ConstrainedTransport(Input &input, Fluid *hydr BufferSendX1[faceLeft ] = IdefixArray1D("EmfSendX1Left", bufferSizeX1); BufferSendX1[faceRight] = IdefixArray1D("EmfSendX1Right",bufferSizeX1); + BufferRecvX1New[faceLeft ] = Buffer(bufferSizeX1); + BufferRecvX1New[faceRight] = Buffer(bufferSizeX1); + BufferSendX1New[faceLeft ] = Buffer(bufferSizeX1); + BufferSendX1New[faceRight] = Buffer(bufferSizeX1); + // Number of cells in X2 boundary condition (only required when problem >2D): #if DIMENSIONS >= 2 @@ -306,6 +324,11 @@ ConstrainedTransport::ConstrainedTransport(Input &input, Fluid *hydr BufferSendX2[faceLeft ] = IdefixArray1D("EmfSendX2Left", bufferSizeX2); BufferSendX2[faceRight] = IdefixArray1D("EmfSendX2Right",bufferSizeX2); + BufferRecvX2New[faceLeft ] = Buffer(bufferSizeX2); + BufferRecvX2New[faceRight] = Buffer(bufferSizeX2); + BufferSendX2New[faceLeft ] = Buffer(bufferSizeX2); + BufferSendX2New[faceRight] = Buffer(bufferSizeX2); + #endif // Number of cells in X3 boundary condition (only required when problem is 3D): #if DIMENSIONS ==3 @@ -318,6 +341,11 @@ ConstrainedTransport::ConstrainedTransport(Input &input, Fluid *hydr BufferRecvX3[faceRight] = IdefixArray1D("EmfRecvX3Right",bufferSizeX3); BufferSendX3[faceLeft ] = IdefixArray1D("EmfSendX3Left", bufferSizeX3); BufferSendX3[faceRight] = IdefixArray1D("EmfSendX3Right",bufferSizeX3); + + BufferRecvX3New[faceLeft ] = Buffer(bufferSizeX3); + BufferRecvX3New[faceRight] = Buffer(bufferSizeX3); + BufferSendX3New[faceLeft ] = Buffer(bufferSizeX3); + BufferSendX3New[faceRight] = Buffer(bufferSizeX3); #endif // DIMENSIONS // Init persistent MPI communications @@ -332,9 +360,15 @@ ConstrainedTransport::ConstrainedTransport(Input &input, Fluid *hydr MPI_SAFE_CALL(MPI_Send_init(BufferSendX1[faceRight].data(), bufferSizeX1, realMPI, procSend, 100, mygrid->CartComm, &sendRequestX1[faceRight])); + MPI_SAFE_CALL(MPI_Send_init(BufferSendX1New[faceRight].data(), bufferSizeX1, realMPI, procSend, + 1100, + mygrid->CartComm, &sendRequestX1New[faceRight])); MPI_SAFE_CALL(MPI_Recv_init(BufferRecvX1[faceLeft].data(), bufferSizeX1, realMPI, procRecv, 100, mygrid->CartComm, &recvRequestX1[faceLeft])); + MPI_SAFE_CALL(MPI_Recv_init(BufferRecvX1New[faceLeft].data(), bufferSizeX1, realMPI, procRecv, + 1100, + mygrid->CartComm, &recvRequestX1New[faceLeft])); // Send to the left // We receive from procRecv, and we send to procSend @@ -345,9 +379,15 @@ ConstrainedTransport::ConstrainedTransport(Input &input, Fluid *hydr MPI_SAFE_CALL(MPI_Send_init(BufferSendX1[faceLeft].data(), bufferSizeX1, realMPI, procSend, 101, mygrid->CartComm, &sendRequestX1[faceLeft])); + MPI_SAFE_CALL(MPI_Send_init(BufferSendX1New[faceLeft].data(), bufferSizeX1, realMPI, procSend, + 1101, + mygrid->CartComm, &sendRequestX1New[faceLeft])); MPI_SAFE_CALL(MPI_Recv_init(BufferRecvX1[faceRight].data(), bufferSizeX1, realMPI, procRecv, 101, mygrid->CartComm, &recvRequestX1[faceRight])); + MPI_SAFE_CALL(MPI_Recv_init(BufferRecvX1New[faceRight].data(), bufferSizeX1, realMPI, procRecv, + 1101, + mygrid->CartComm, &recvRequestX1New[faceRight])); #if DIMENSIONS >= 2 // We receive from procRecv, and we send to procSend @@ -355,9 +395,15 @@ ConstrainedTransport::ConstrainedTransport(Input &input, Fluid *hydr MPI_SAFE_CALL(MPI_Send_init(BufferSendX2[faceRight].data(), bufferSizeX2, realMPI, procSend, 200, mygrid->CartComm, &sendRequestX2[faceRight])); + MPI_SAFE_CALL(MPI_Send_init(BufferSendX2New[faceRight].data(), bufferSizeX2, realMPI, procSend, + 1200, + mygrid->CartComm, &sendRequestX2New[faceRight])); MPI_SAFE_CALL(MPI_Recv_init(BufferRecvX2[faceLeft].data(), bufferSizeX2, realMPI, procRecv, 200, mygrid->CartComm, &recvRequestX2[faceLeft])); + MPI_SAFE_CALL(MPI_Recv_init(BufferRecvX2New[faceLeft].data(), bufferSizeX2, realMPI, procRecv, + 1200, + mygrid->CartComm, &recvRequestX2New[faceLeft])); // Send to the left // We receive from procRecv, and we send to procSend @@ -365,9 +411,15 @@ ConstrainedTransport::ConstrainedTransport(Input &input, Fluid *hydr MPI_SAFE_CALL(MPI_Send_init(BufferSendX2[faceLeft].data(), bufferSizeX2, realMPI, procSend, 201, mygrid->CartComm, &sendRequestX2[faceLeft])); + MPI_SAFE_CALL(MPI_Send_init(BufferSendX2New[faceLeft].data(), bufferSizeX2, realMPI, procSend, + 1201, + mygrid->CartComm, &sendRequestX2New[faceLeft])); MPI_SAFE_CALL(MPI_Recv_init(BufferRecvX2[faceRight].data(), bufferSizeX2, realMPI, procRecv, 201, mygrid->CartComm, &recvRequestX2[faceRight])); + MPI_SAFE_CALL(MPI_Recv_init(BufferRecvX2New[faceRight].data(), bufferSizeX2, realMPI, procRecv, + 1201, + mygrid->CartComm, &recvRequestX2New[faceRight])); #endif #if DIMENSIONS == 3 @@ -376,9 +428,15 @@ ConstrainedTransport::ConstrainedTransport(Input &input, Fluid *hydr MPI_SAFE_CALL(MPI_Send_init(BufferSendX3[faceRight].data(), bufferSizeX3, realMPI, procSend, 300, mygrid->CartComm, &sendRequestX3[faceRight])); + MPI_SAFE_CALL(MPI_Send_init(BufferSendX3New[faceRight].data(), bufferSizeX3, realMPI, procSend, + 1300, + mygrid->CartComm, &sendRequestX3New[faceRight])); MPI_SAFE_CALL(MPI_Recv_init(BufferRecvX3[faceLeft].data(), bufferSizeX3, realMPI, procRecv, 300, mygrid->CartComm, &recvRequestX3[faceLeft])); + MPI_SAFE_CALL(MPI_Recv_init(BufferRecvX3New[faceLeft].data(), bufferSizeX3, realMPI, procRecv, + 1300, + mygrid->CartComm, &recvRequestX3New[faceLeft])); // Send to the left // We receive from procRecv, and we send to procSend @@ -386,9 +444,15 @@ ConstrainedTransport::ConstrainedTransport(Input &input, Fluid *hydr MPI_SAFE_CALL(MPI_Send_init(BufferSendX3[faceLeft].data(), bufferSizeX3, realMPI, procSend, 301, mygrid->CartComm, &sendRequestX3[faceLeft])); + MPI_SAFE_CALL(MPI_Send_init(BufferSendX3New[faceLeft].data(), bufferSizeX3, realMPI, procSend, + 1301, + mygrid->CartComm, &sendRequestX3New[faceLeft])); MPI_SAFE_CALL(MPI_Recv_init(BufferRecvX3[faceRight].data(), bufferSizeX3, realMPI, procRecv, 301, mygrid->CartComm, &recvRequestX3[faceRight])); + MPI_SAFE_CALL(MPI_Recv_init(BufferRecvX3New[faceRight].data(), bufferSizeX3, realMPI, procRecv, + 1301, + mygrid->CartComm, &recvRequestX3New[faceRight])); #endif #endif // WITH_MPI @@ -410,15 +474,21 @@ ConstrainedTransport::~ConstrainedTransport() { for(int i=0 ; i< 2; i++) { MPI_Request_free( &sendRequestX1[i]); MPI_Request_free( &recvRequestX1[i]); + MPI_Request_free( &sendRequestX1New[i]); + MPI_Request_free( &recvRequestX1New[i]); #if DIMENSIONS >= 2 MPI_Request_free( &sendRequestX2[i]); MPI_Request_free( &recvRequestX2[i]); + MPI_Request_free( &sendRequestX2New[i]); + MPI_Request_free( &recvRequestX2New[i]); #endif #if DIMENSIONS == 3 MPI_Request_free( &sendRequestX3[i]); MPI_Request_free( &recvRequestX3[i]); + MPI_Request_free( &sendRequestX3New[i]); + MPI_Request_free( &recvRequestX3New[i]); #endif } #endif From 3914ed46f29682e353937d4342ae6df5d6eb3ecd Mon Sep 17 00:00:00 2001 From: Sebastien Valat Date: Fri, 19 Jun 2026 17:35:49 +0200 Subject: [PATCH 5/8] refactor: disable some Unpack based on lbound variable to mimic the old way (step 2) --- .../constrainedTransport/EMFexchange.hpp | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/src/fluid/constrainedTransport/EMFexchange.hpp b/src/fluid/constrainedTransport/EMFexchange.hpp index 48d83ab9b..37ef1f142 100644 --- a/src/fluid/constrainedTransport/EMFexchange.hpp +++ b/src/fluid/constrainedTransport/EMFexchange.hpp @@ -136,10 +136,11 @@ void ConstrainedTransport::ExchangeX1(IdefixArray3D ey, IdefixArray3 recvBoxEz[JDIR][1] += 1; recvBoxEz[IDIR][0] = ileft; recvBoxEz[IDIR][1] = ileft + 1; - BufferLeftNew.Unpack(ez, recvBoxEz); + if(lbound == internal || lbound == periodic) + BufferLeftNew.Unpack(ez, recvBoxEz); idefix_for("StoreBufferX1Emfz",kbeg,kend,jbeg,jend+1, KOKKOS_LAMBDA (int k, int j) { - if(lbound == internal || lbound == periodic) { + if(lbound == internal || lbound == periodic) {//////////////<<<<<<<<<<<<<<<<<::ExchangeX1(IdefixArray3D ey, IdefixArray3 recvBoxEy[KDIR][1] += 1; recvBoxEy[IDIR][0] = ileft; recvBoxEy[IDIR][1] = ileft + 1; - BufferLeftNew.Unpack(ey, recvBoxEy); + if(lbound == internal || lbound == periodic) + BufferLeftNew.Unpack(ey, recvBoxEy); idefix_for("StoreBufferX1Emfy",kbeg,kend+1,jbeg,jend, KOKKOS_LAMBDA (int k, int j) { - if(lbound == internal || lbound == periodic) { + if(lbound == internal || lbound == periodic) {//////////////<<<<<<<<<<<<<<<<<::ExchangeX2(IdefixArray3D ex, IdefixArray3 recvBoxEz[IDIR][1] += 1; recvBoxEz[JDIR][0] = jleft; recvBoxEz[JDIR][1] = jleft + 1; - BufferLeftNew.Unpack(ez, recvBoxEz); + if(lbound == internal || lbound == periodic) + BufferLeftNew.Unpack(ez, recvBoxEz); idefix_for("StoreBufferX2Emfz",kbeg,kend,ibeg,iend+1, KOKKOS_LAMBDA (int k, int i) { - if(lbound == internal || lbound == periodic) { + if(lbound == internal || lbound == periodic) {//////////////<<<<<<<<<<<<<<<<<::ExchangeX2(IdefixArray3D ex, IdefixArray3 recvBoxEx[KDIR][1] += 1; recvBoxEx[JDIR][0] = jleft; recvBoxEx[JDIR][1] = jleft + 1; - BufferLeftNew.Unpack(ex, recvBoxEx); + if(lbound == internal || lbound == periodic) + BufferLeftNew.Unpack(ex, recvBoxEx); idefix_for("StoreBufferX1Emfy",kbeg,kend+1,ibeg,iend, KOKKOS_LAMBDA (int k, int i) { - if(lbound == internal || lbound == periodic) { + if(lbound == internal || lbound == periodic) {//////////////<<<<<<<<<<<<<<<<<::ExchangeX3(IdefixArray3D ex, IdefixArray3 recvBoxEx[JDIR][1] += 1; recvBoxEx[KDIR][0] = kleft; recvBoxEx[KDIR][1] = kleft + 1; - BufferLeftNew.Unpack(ex, recvBoxEx); + if(lbound == internal || lbound == periodic) + BufferLeftNew.Unpack(ex, recvBoxEx); idefix_for("StoreBufferX3Emfx",jbeg,jend+1,ibeg,iend, KOKKOS_LAMBDA (int j, int i) { - if(lbound == internal || lbound == periodic) { + if(lbound == internal || lbound == periodic) {//////////////<<<<<<<<<<<<<<<<<::ExchangeX3(IdefixArray3D ex, IdefixArray3 recvBoxEy[IDIR][1] += 1; recvBoxEy[KDIR][0] = kleft; recvBoxEy[KDIR][1] = kleft + 1; - BufferLeftNew.Unpack(ey, recvBoxEy); + if(lbound == internal || lbound == periodic) + BufferLeftNew.Unpack(ey, recvBoxEy); idefix_for("StoreBufferX3Emfy",jbeg,jend,ibeg,iend+1, KOKKOS_LAMBDA (int j, int i) { - if(lbound == internal || lbound == periodic) { + if(lbound == internal || lbound == periodic) {//////////////<<<<<<<<<<<<<<<<< Date: Mon, 6 Jul 2026 11:46:49 +0200 Subject: [PATCH 6/8] refactor: remove the temporary duplicated comms and validation code (step 3) --- .../constrainedTransport/EMFexchange.hpp | 199 +++--------------- .../constrainedTransport.hpp | 105 ++------- 2 files changed, 42 insertions(+), 262 deletions(-) diff --git a/src/fluid/constrainedTransport/EMFexchange.hpp b/src/fluid/constrainedTransport/EMFexchange.hpp index 37ef1f142..426d56ad2 100644 --- a/src/fluid/constrainedTransport/EMFexchange.hpp +++ b/src/fluid/constrainedTransport/EMFexchange.hpp @@ -32,25 +32,18 @@ void ConstrainedTransport::ExchangeX1(IdefixArray3D ey, IdefixArray3 // Load the buffers with data - int ileft,iright,jbeg,jend,kbeg,kend; - int ny; - [[maybe_unused]] int nz; + int ileft,iright,jbeg,jend; - IdefixArray1D BufferLeft=BufferSendX1[faceLeft]; - IdefixArray1D BufferRight=BufferSendX1[faceRight]; - Buffer BufferLeftNew=BufferSendX1New[faceLeft]; - Buffer BufferRightNew=BufferSendX1New[faceRight]; + Buffer BufferLeft=BufferSendX1[faceLeft]; + Buffer BufferRight=BufferSendX1[faceRight]; // If MPI Persistent, start receiving even before the buffers are filled MPI_Status sendStatus[2]; MPI_Status recvStatus[2]; - MPI_Status sendStatusNew[2]; - MPI_Status recvStatusNew[2]; double tStart = MPI_Wtime(); MPI_SAFE_CALL(MPI_Startall(2, recvRequestX1)); - MPI_SAFE_CALL(MPI_Startall(2, recvRequestX1New)); idfx::mpiCallsTimer += MPI_Wtime() - tStart; BoundaryType lbound = data->lbound[IDIR]; @@ -61,10 +54,6 @@ void ConstrainedTransport::ExchangeX1(IdefixArray3D ey, IdefixArray3 iright = data->end[IDIR]; jbeg = data->beg[JDIR]; jend = data->end[JDIR]; - ny = jend - jbeg; - kbeg = data->beg[KDIR]; - kend = data->end[KDIR]; - nz = kend - kbeg; // Create the base box to be patched by the ops BoundingBox baseBox; @@ -80,55 +69,31 @@ void ConstrainedTransport::ExchangeX1(IdefixArray3D ey, IdefixArray3 sendBoxEz[JDIR][1] += 1; sendBoxEz[IDIR][0] = iright; sendBoxEz[IDIR][1] = iright + 1; - BufferRightNew.Pack(ez, sendBoxEz); - idefix_for("LoadBufferX1Emfz",kbeg,kend,jbeg,jend+1, - KOKKOS_LAMBDA (int k, int j) { - BufferRight( (j-jbeg) + (k-kbeg)*(ny+1) ) = ez(k,j,iright); - assert(BufferRightNew.getArray()((j-jbeg) + (k-kbeg)*(ny+1)) == ez(k,j,iright)); - } - ); + BufferRight.Pack(ez, sendBoxEz); #if DIMENSIONS == 3 - int Vsindex = (ny+1)*nz; //extend by one the end on kdir && take the ghost on i BoundingBox sendBoxEy = baseBox; sendBoxEy[KDIR][1] += 1; sendBoxEy[IDIR][0] = iright; sendBoxEy[IDIR][1] = iright + 1; - BufferRightNew.Pack(ez, sendBoxEy); - idefix_for("LoadBufferX1Emfy",kbeg,kend+1,jbeg,jend, - KOKKOS_LAMBDA (int k, int j) { - BufferRight( (j-jbeg) + (k-kbeg)*ny + Vsindex ) = ey(k,j,iright); - assert(BufferRightNew.getArray()( (j-jbeg) + (k-kbeg)*ny + Vsindex ) == ey(k,j,iright)); - } - ); + BufferRight.Pack(ez, sendBoxEy); #endif - assert(BufferRight == BufferRightNew.getArray()); - // Wait for completion before sending out everything Kokkos::fence(); tStart = MPI_Wtime(); MPI_SAFE_CALL(MPI_Startall(2, sendRequestX1)); - MPI_SAFE_CALL(MPI_Startall(2, sendRequestX1New)); // Wait for buffers to be received MPI_Waitall(2,recvRequestX1,recvStatus); - MPI_Waitall(2,recvRequestX1New,recvStatusNew); MPI_Waitall(2, sendRequestX1, sendStatus); - MPI_Waitall(2, sendRequestX1New, sendStatusNew); idfx::mpiCallsTimer += MPI_Wtime() - tStart; // Unpack BufferLeft=BufferRecvX1[faceLeft]; BufferRight=BufferRecvX1[faceRight]; - BufferLeftNew=BufferRecvX1New[faceLeft]; - BufferRightNew=BufferRecvX1New[faceRight]; - - - - assert(BufferLeft == BufferLeftNew.getArray()); // Erase the emf with the one coming from the left process //extend by one the end on jdir && take the ghost revc zone on i @@ -137,31 +102,16 @@ void ConstrainedTransport::ExchangeX1(IdefixArray3D ey, IdefixArray3 recvBoxEz[IDIR][0] = ileft; recvBoxEz[IDIR][1] = ileft + 1; if(lbound == internal || lbound == periodic) - BufferLeftNew.Unpack(ez, recvBoxEz); - idefix_for("StoreBufferX1Emfz",kbeg,kend,jbeg,jend+1, - KOKKOS_LAMBDA (int k, int j) { - if(lbound == internal || lbound == periodic) {//////////////<<<<<<<<<<<<<<<<<::ExchangeX2(IdefixArray3D ex, IdefixArray3 idfx::pushRegion("Emf::ExchangeX2"); // Load the buffers with data - int jleft,jright,ibeg,iend,kbeg,kend; - int nx; - [[maybe_unused]] int nz; - IdefixArray1D BufferLeft=BufferSendX2[faceLeft]; - IdefixArray1D BufferRight=BufferSendX2[faceRight]; - Buffer BufferLeftNew=BufferSendX2New[faceLeft]; - Buffer BufferRightNew=BufferSendX2New[faceRight]; + int jleft,jright; + Buffer BufferLeft=BufferSendX2[faceLeft]; + Buffer BufferRight=BufferSendX2[faceRight]; // If MPI Persistent, start receiving even before the buffers are filled double tStart = MPI_Wtime(); MPI_Status sendStatus[2]; MPI_Status recvStatus[2]; - MPI_Status sendStatusNew[2]; - MPI_Status recvStatusNew[2]; MPI_SAFE_CALL(MPI_Startall(2, recvRequestX2)); - MPI_SAFE_CALL(MPI_Startall(2, recvRequestX2New)); idfx::mpiCallsTimer += MPI_Wtime() - tStart; BoundaryType lbound = data->lbound[JDIR]; BoundaryType rbound = data->rbound[JDIR]; // Coordinates of the ghost region which needs to be transfered - ibeg = data->beg[IDIR]; - iend = data->end[IDIR]; - nx = iend - ibeg; jleft = data->beg[JDIR]; jright = data->end[JDIR]; - kbeg = data->beg[KDIR]; - kend = data->end[KDIR]; - nz = kend - kbeg; // Create the base box to be patched by the ops BoundingBox baseBox; @@ -219,52 +156,30 @@ void ConstrainedTransport::ExchangeX2(IdefixArray3D ex, IdefixArray3 sendBoxEz[IDIR][1] += 1; sendBoxEz[JDIR][0] = jright; sendBoxEz[JDIR][1] = jright + 1; - BufferRightNew.Pack(ez, sendBoxEz); - idefix_for("LoadBufferX2Emfz",kbeg,kend,ibeg,iend+1, - KOKKOS_LAMBDA (int k, int i) { - BufferRight( (i-ibeg) + (k-kbeg)*(nx+1) ) = ez(k,jright,i); - assert(BufferRightNew.getArray()( (i-ibeg) + (k-kbeg)*(nx+1) ) == ez(k,jright,i)); - } - ); + BufferRight.Pack(ez, sendBoxEz); #if DIMENSIONS == 3 - int Vsindex = (nx+1)*nz; //extend by one the end on idir && take the ghost on j BoundingBox sendBoxEx = baseBox; sendBoxEx[KDIR][1] += 1; sendBoxEx[JDIR][0] = jright; sendBoxEx[JDIR][1] = jright + 1; - BufferRightNew.Pack(ex, sendBoxEx); - idefix_for("LoadBufferX1Emfx",kbeg,kend+1,ibeg,iend, - KOKKOS_LAMBDA (int k, int i) { - BufferRight( (i-ibeg) + (k-kbeg)*nx + Vsindex ) = ex(k,jright,i); - assert(BufferRightNew.getArray()( (i-ibeg) + (k-kbeg)*nx + Vsindex ) == ex(k,jright,i)); - } - ); + BufferRight.Pack(ex, sendBoxEx); #endif - assert(BufferRightNew.getArray() == BufferRight); - // Wait for completion before sending out everything Kokkos::fence(); tStart = MPI_Wtime(); MPI_SAFE_CALL(MPI_Startall(2, sendRequestX2)); - MPI_SAFE_CALL(MPI_Startall(2, sendRequestX2New)); // Wait for buffers to be received - MPI_Waitall(2,recvRequestX2,recvStatus); - MPI_Waitall(2,recvRequestX2New,recvStatus); + MPI_Waitall(2, recvRequestX2, recvStatus); MPI_Waitall(2, sendRequestX2, sendStatus); - MPI_Waitall(2, sendRequestX2New, sendStatus); idfx::mpiCallsTimer += MPI_Wtime() - tStart; // Unpack BufferLeft=BufferRecvX2[faceLeft]; BufferRight=BufferRecvX2[faceRight]; - BufferLeftNew=BufferRecvX2New[faceLeft]; - BufferRightNew=BufferRecvX2New[faceRight]; - - assert(BufferLeftNew.getArray() == BufferLeft); // We average the edge emfs zones //extend by one the end on idir && take the ghost revc zone on j @@ -273,30 +188,15 @@ void ConstrainedTransport::ExchangeX2(IdefixArray3D ex, IdefixArray3 recvBoxEz[JDIR][0] = jleft; recvBoxEz[JDIR][1] = jleft + 1; if(lbound == internal || lbound == periodic) - BufferLeftNew.Unpack(ez, recvBoxEz); - idefix_for("StoreBufferX2Emfz",kbeg,kend,ibeg,iend+1, - KOKKOS_LAMBDA (int k, int i) { - if(lbound == internal || lbound == periodic) {//////////////<<<<<<<<<<<<<<<<<::ExchangeX3(IdefixArray3D ex, IdefixArray3 // Load the buffers with data - int kleft,kright,ibeg,iend,jbeg,jend; - int nx,ny; - IdefixArray1D BufferLeft=BufferSendX3[faceLeft]; - IdefixArray1D BufferRight=BufferSendX3[faceRight]; - Buffer BufferLeftNew=BufferSendX3New[faceLeft]; - Buffer BufferRightNew=BufferSendX3New[faceRight]; - - int Vsindex = 0; - + int kleft,kright,jbeg,jend; + Buffer BufferLeft=BufferSendX3[faceLeft]; + Buffer BufferRight=BufferSendX3[faceRight]; // If MPI Persistent, start receiving even before the buffers are filled double tStart = MPI_Wtime(); MPI_Status sendStatus[2]; MPI_Status recvStatus[2]; - MPI_Status sendStatusNew[2]; - MPI_Status recvStatusNew[2]; MPI_SAFE_CALL(MPI_Startall(2, recvRequestX3)); - MPI_SAFE_CALL(MPI_Startall(2, recvRequestX3New)); idfx::mpiCallsTimer += MPI_Wtime() - tStart; BoundaryType lbound = data->lbound[KDIR]; BoundaryType rbound = data->rbound[KDIR]; // Coordinates of the ghost region which needs to be transfered - ibeg = data->beg[IDIR]; - iend = data->end[IDIR]; - nx = iend - ibeg; - jbeg = data->beg[JDIR]; jend = data->end[JDIR]; - ny = jend - jbeg; kleft = data->beg[KDIR]; kright = data->end[KDIR]; @@ -359,50 +245,28 @@ void ConstrainedTransport::ExchangeX3(IdefixArray3D ex, IdefixArray3 sendBoxEz[JDIR][1] += 1; sendBoxEz[KDIR][0] = kright; sendBoxEz[KDIR][1] = kright + 1; - BufferRightNew.Pack(ez, sendBoxEz); - idefix_for("LoadBufferX3Emfx",jbeg,jend+1,ibeg,iend, - KOKKOS_LAMBDA (int j, int i) { - BufferRight( (i-ibeg) + (j-jbeg)*nx ) = ex(kright,j,i); - assert(BufferRightNew.getArray()( (i-ibeg) + (j-jbeg)*nx ) == ex(kright,j,i)); - } - ); - Vsindex = nx*(ny+1); + BufferRight.Pack(ez, sendBoxEz); //extend by one the end on idir && take the ghost on k BoundingBox sendBoxEy = baseBox; sendBoxEy[IDIR][1] += 1; sendBoxEy[KDIR][0] = kright; sendBoxEy[KDIR][1] = kright + 1; - BufferRightNew.Pack(ey, sendBoxEy); - idefix_for("LoadBufferX3Emfy",jbeg,jend,ibeg,iend+1, - KOKKOS_LAMBDA (int j, int i) { - BufferRight( (i-ibeg) + (j-jbeg)*(nx+1) + Vsindex ) = ey(kright,j,i); - assert(BufferRightNew.getArray()( (i-ibeg) + (j-jbeg)*(nx+1) + Vsindex ) == ey(kright,j,i)); - } - ); - - assert(BufferRightNew.getArray() == BufferRight); + BufferRight.Pack(ey, sendBoxEy); // Wait for completion before sending out everything Kokkos::fence(); tStart = MPI_Wtime(); MPI_SAFE_CALL(MPI_Startall(2, sendRequestX3)); - MPI_SAFE_CALL(MPI_Startall(2, sendRequestX3New)); // Wait for buffers to be received - MPI_Waitall(2,recvRequestX3,recvStatus); - MPI_Waitall(2,recvRequestX3New,recvStatusNew); + MPI_Waitall(2, recvRequestX3, recvStatus); MPI_Waitall(2, sendRequestX3, sendStatus); - MPI_Waitall(2, sendRequestX3New, sendStatusNew); idfx::mpiCallsTimer += MPI_Wtime() - tStart; // Unpack BufferLeft=BufferRecvX3[faceLeft]; BufferRight=BufferRecvX3[faceRight]; - BufferLeftNew=BufferRecvX3New[faceLeft]; - BufferRightNew=BufferRecvX3New[faceRight]; - - assert(BufferLeftNew.getArray() == BufferLeft); // We average the edge emfs zones //extend by one the end on jdir && take the ghost revc zone on k @@ -411,30 +275,15 @@ void ConstrainedTransport::ExchangeX3(IdefixArray3D ex, IdefixArray3 recvBoxEx[KDIR][0] = kleft; recvBoxEx[KDIR][1] = kleft + 1; if(lbound == internal || lbound == periodic) - BufferLeftNew.Unpack(ex, recvBoxEx); - idefix_for("StoreBufferX3Emfx",jbeg,jend+1,ibeg,iend, - KOKKOS_LAMBDA (int j, int i) { - if(lbound == internal || lbound == periodic) {//////////////<<<<<<<<<<<<<<<<< BufferSendX1[2]; - IdefixArray1D BufferSendX2[2]; - IdefixArray1D BufferSendX3[2]; - IdefixArray1D BufferRecvX1[2]; - IdefixArray1D BufferRecvX2[2]; - IdefixArray1D BufferRecvX3[2]; - Buffer BufferSendX1New[2]; - Buffer BufferSendX2New[2]; - Buffer BufferSendX3New[2]; - Buffer BufferRecvX1New[2]; - Buffer BufferRecvX2New[2]; - Buffer BufferRecvX3New[2]; + Buffer BufferSendX1[2]; + Buffer BufferSendX2[2]; + Buffer BufferSendX3[2]; + Buffer BufferRecvX1[2]; + Buffer BufferRecvX2[2]; + Buffer BufferRecvX3[2]; IdefixArray1D mapVars; @@ -149,12 +143,6 @@ class ConstrainedTransport { MPI_Request recvRequestX1[2]; MPI_Request recvRequestX2[2]; MPI_Request recvRequestX3[2]; - MPI_Request sendRequestX1New[2]; - MPI_Request sendRequestX2New[2]; - MPI_Request sendRequestX3New[2]; - MPI_Request recvRequestX1New[2]; - MPI_Request recvRequestX2New[2]; - MPI_Request recvRequestX3New[2]; Kokkos::Timer timer; // Internal MPI timer #endif @@ -299,15 +287,10 @@ ConstrainedTransport::ConstrainedTransport(Input &input, Fluid *hydr bufferSizeX1 += data->np_int[JDIR] * (data->np_int[KDIR]+KOFFSET); #endif - BufferRecvX1[faceLeft ] = IdefixArray1D("EmfRecvX1Left", bufferSizeX1); - BufferRecvX1[faceRight] = IdefixArray1D("EmfRecvX1Right",bufferSizeX1); - BufferSendX1[faceLeft ] = IdefixArray1D("EmfSendX1Left", bufferSizeX1); - BufferSendX1[faceRight] = IdefixArray1D("EmfSendX1Right",bufferSizeX1); - - BufferRecvX1New[faceLeft ] = Buffer(bufferSizeX1); - BufferRecvX1New[faceRight] = Buffer(bufferSizeX1); - BufferSendX1New[faceLeft ] = Buffer(bufferSizeX1); - BufferSendX1New[faceRight] = Buffer(bufferSizeX1); + BufferRecvX1[faceLeft ] = Buffer(bufferSizeX1); + BufferRecvX1[faceRight] = Buffer(bufferSizeX1); + BufferSendX1[faceLeft ] = Buffer(bufferSizeX1); + BufferSendX1[faceRight] = Buffer(bufferSizeX1); // Number of cells in X2 boundary condition (only required when problem >2D): #if DIMENSIONS >= 2 @@ -319,15 +302,10 @@ ConstrainedTransport::ConstrainedTransport(Input &input, Fluid *hydr bufferSizeX2 += data->np_int[IDIR] * (data->np_int[KDIR]+KOFFSET); #endif - BufferRecvX2[faceLeft ] = IdefixArray1D("EmfRecvX2Left", bufferSizeX2); - BufferRecvX2[faceRight] = IdefixArray1D("EmfRecvX2Right",bufferSizeX2); - BufferSendX2[faceLeft ] = IdefixArray1D("EmfSendX2Left", bufferSizeX2); - BufferSendX2[faceRight] = IdefixArray1D("EmfSendX2Right",bufferSizeX2); - - BufferRecvX2New[faceLeft ] = Buffer(bufferSizeX2); - BufferRecvX2New[faceRight] = Buffer(bufferSizeX2); - BufferSendX2New[faceLeft ] = Buffer(bufferSizeX2); - BufferSendX2New[faceRight] = Buffer(bufferSizeX2); + BufferRecvX2[faceLeft ] = Buffer(bufferSizeX2); + BufferRecvX2[faceRight] = Buffer(bufferSizeX2); + BufferSendX2[faceLeft ] = Buffer(bufferSizeX2); + BufferSendX2[faceRight] = Buffer(bufferSizeX2); #endif // Number of cells in X3 boundary condition (only required when problem is 3D): @@ -337,15 +315,10 @@ ConstrainedTransport::ConstrainedTransport(Input &input, Fluid *hydr // ey bufferSizeX3 += (data->np_int[IDIR]+IOFFSET) * data->np_int[JDIR]; - BufferRecvX3[faceLeft ] = IdefixArray1D("EmfRecvX3Left", bufferSizeX3); - BufferRecvX3[faceRight] = IdefixArray1D("EmfRecvX3Right",bufferSizeX3); - BufferSendX3[faceLeft ] = IdefixArray1D("EmfSendX3Left", bufferSizeX3); - BufferSendX3[faceRight] = IdefixArray1D("EmfSendX3Right",bufferSizeX3); - - BufferRecvX3New[faceLeft ] = Buffer(bufferSizeX3); - BufferRecvX3New[faceRight] = Buffer(bufferSizeX3); - BufferSendX3New[faceLeft ] = Buffer(bufferSizeX3); - BufferSendX3New[faceRight] = Buffer(bufferSizeX3); + BufferRecvX3[faceLeft ] = Buffer(bufferSizeX3); + BufferRecvX3[faceRight] = Buffer(bufferSizeX3); + BufferSendX3[faceLeft ] = Buffer(bufferSizeX3); + BufferSendX3[faceRight] = Buffer(bufferSizeX3); #endif // DIMENSIONS // Init persistent MPI communications @@ -360,15 +333,9 @@ ConstrainedTransport::ConstrainedTransport(Input &input, Fluid *hydr MPI_SAFE_CALL(MPI_Send_init(BufferSendX1[faceRight].data(), bufferSizeX1, realMPI, procSend, 100, mygrid->CartComm, &sendRequestX1[faceRight])); - MPI_SAFE_CALL(MPI_Send_init(BufferSendX1New[faceRight].data(), bufferSizeX1, realMPI, procSend, - 1100, - mygrid->CartComm, &sendRequestX1New[faceRight])); MPI_SAFE_CALL(MPI_Recv_init(BufferRecvX1[faceLeft].data(), bufferSizeX1, realMPI, procRecv, 100, mygrid->CartComm, &recvRequestX1[faceLeft])); - MPI_SAFE_CALL(MPI_Recv_init(BufferRecvX1New[faceLeft].data(), bufferSizeX1, realMPI, procRecv, - 1100, - mygrid->CartComm, &recvRequestX1New[faceLeft])); // Send to the left // We receive from procRecv, and we send to procSend @@ -379,15 +346,9 @@ ConstrainedTransport::ConstrainedTransport(Input &input, Fluid *hydr MPI_SAFE_CALL(MPI_Send_init(BufferSendX1[faceLeft].data(), bufferSizeX1, realMPI, procSend, 101, mygrid->CartComm, &sendRequestX1[faceLeft])); - MPI_SAFE_CALL(MPI_Send_init(BufferSendX1New[faceLeft].data(), bufferSizeX1, realMPI, procSend, - 1101, - mygrid->CartComm, &sendRequestX1New[faceLeft])); MPI_SAFE_CALL(MPI_Recv_init(BufferRecvX1[faceRight].data(), bufferSizeX1, realMPI, procRecv, 101, mygrid->CartComm, &recvRequestX1[faceRight])); - MPI_SAFE_CALL(MPI_Recv_init(BufferRecvX1New[faceRight].data(), bufferSizeX1, realMPI, procRecv, - 1101, - mygrid->CartComm, &recvRequestX1New[faceRight])); #if DIMENSIONS >= 2 // We receive from procRecv, and we send to procSend @@ -395,15 +356,9 @@ ConstrainedTransport::ConstrainedTransport(Input &input, Fluid *hydr MPI_SAFE_CALL(MPI_Send_init(BufferSendX2[faceRight].data(), bufferSizeX2, realMPI, procSend, 200, mygrid->CartComm, &sendRequestX2[faceRight])); - MPI_SAFE_CALL(MPI_Send_init(BufferSendX2New[faceRight].data(), bufferSizeX2, realMPI, procSend, - 1200, - mygrid->CartComm, &sendRequestX2New[faceRight])); MPI_SAFE_CALL(MPI_Recv_init(BufferRecvX2[faceLeft].data(), bufferSizeX2, realMPI, procRecv, 200, mygrid->CartComm, &recvRequestX2[faceLeft])); - MPI_SAFE_CALL(MPI_Recv_init(BufferRecvX2New[faceLeft].data(), bufferSizeX2, realMPI, procRecv, - 1200, - mygrid->CartComm, &recvRequestX2New[faceLeft])); // Send to the left // We receive from procRecv, and we send to procSend @@ -411,15 +366,9 @@ ConstrainedTransport::ConstrainedTransport(Input &input, Fluid *hydr MPI_SAFE_CALL(MPI_Send_init(BufferSendX2[faceLeft].data(), bufferSizeX2, realMPI, procSend, 201, mygrid->CartComm, &sendRequestX2[faceLeft])); - MPI_SAFE_CALL(MPI_Send_init(BufferSendX2New[faceLeft].data(), bufferSizeX2, realMPI, procSend, - 1201, - mygrid->CartComm, &sendRequestX2New[faceLeft])); MPI_SAFE_CALL(MPI_Recv_init(BufferRecvX2[faceRight].data(), bufferSizeX2, realMPI, procRecv, 201, mygrid->CartComm, &recvRequestX2[faceRight])); - MPI_SAFE_CALL(MPI_Recv_init(BufferRecvX2New[faceRight].data(), bufferSizeX2, realMPI, procRecv, - 1201, - mygrid->CartComm, &recvRequestX2New[faceRight])); #endif #if DIMENSIONS == 3 @@ -428,15 +377,9 @@ ConstrainedTransport::ConstrainedTransport(Input &input, Fluid *hydr MPI_SAFE_CALL(MPI_Send_init(BufferSendX3[faceRight].data(), bufferSizeX3, realMPI, procSend, 300, mygrid->CartComm, &sendRequestX3[faceRight])); - MPI_SAFE_CALL(MPI_Send_init(BufferSendX3New[faceRight].data(), bufferSizeX3, realMPI, procSend, - 1300, - mygrid->CartComm, &sendRequestX3New[faceRight])); MPI_SAFE_CALL(MPI_Recv_init(BufferRecvX3[faceLeft].data(), bufferSizeX3, realMPI, procRecv, 300, mygrid->CartComm, &recvRequestX3[faceLeft])); - MPI_SAFE_CALL(MPI_Recv_init(BufferRecvX3New[faceLeft].data(), bufferSizeX3, realMPI, procRecv, - 1300, - mygrid->CartComm, &recvRequestX3New[faceLeft])); // Send to the left // We receive from procRecv, and we send to procSend @@ -444,15 +387,9 @@ ConstrainedTransport::ConstrainedTransport(Input &input, Fluid *hydr MPI_SAFE_CALL(MPI_Send_init(BufferSendX3[faceLeft].data(), bufferSizeX3, realMPI, procSend, 301, mygrid->CartComm, &sendRequestX3[faceLeft])); - MPI_SAFE_CALL(MPI_Send_init(BufferSendX3New[faceLeft].data(), bufferSizeX3, realMPI, procSend, - 1301, - mygrid->CartComm, &sendRequestX3New[faceLeft])); MPI_SAFE_CALL(MPI_Recv_init(BufferRecvX3[faceRight].data(), bufferSizeX3, realMPI, procRecv, 301, mygrid->CartComm, &recvRequestX3[faceRight])); - MPI_SAFE_CALL(MPI_Recv_init(BufferRecvX3New[faceRight].data(), bufferSizeX3, realMPI, procRecv, - 1301, - mygrid->CartComm, &recvRequestX3New[faceRight])); #endif #endif // WITH_MPI @@ -474,21 +411,15 @@ ConstrainedTransport::~ConstrainedTransport() { for(int i=0 ; i< 2; i++) { MPI_Request_free( &sendRequestX1[i]); MPI_Request_free( &recvRequestX1[i]); - MPI_Request_free( &sendRequestX1New[i]); - MPI_Request_free( &recvRequestX1New[i]); #if DIMENSIONS >= 2 MPI_Request_free( &sendRequestX2[i]); MPI_Request_free( &recvRequestX2[i]); - MPI_Request_free( &sendRequestX2New[i]); - MPI_Request_free( &recvRequestX2New[i]); #endif #if DIMENSIONS == 3 MPI_Request_free( &sendRequestX3[i]); MPI_Request_free( &recvRequestX3[i]); - MPI_Request_free( &sendRequestX3New[i]); - MPI_Request_free( &recvRequestX3New[i]); #endif } #endif From bf47d190d13ae3eb2a06799d07f550fcd80eb586 Mon Sep 17 00:00:00 2001 From: Sebastien Valat Date: Mon, 6 Jul 2026 12:20:17 +0200 Subject: [PATCH 7/8] refactor: Remove some none used varaibles following the previous patches in axis.cpp (step 4) --- src/fluid/boundary/axis.cpp | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/src/fluid/boundary/axis.cpp b/src/fluid/boundary/axis.cpp index a67621934..45ea49ea5 100644 --- a/src/fluid/boundary/axis.cpp +++ b/src/fluid/boundary/axis.cpp @@ -393,7 +393,7 @@ void Axis::ExchangeMPI(int side) { #ifdef WITH_MPI // Load the buffers with data int ibeg,iend,jbeg,jend,kbeg,kend,offset; - int nx,ny,nz; + int ny; Buffer bufferSend = this->bufferSend; IdefixArray1D map = this->mapVars; IdefixArray4D Vc = this->Vc; @@ -412,25 +412,17 @@ void Axis::ExchangeMPI(int side) { idfx::mpiCallsTimer += MPI_Wtime() - tStart; // Coordinates of the ghost region which needs to be transfered - ibeg = 0; - iend = data->np_tot[IDIR]; - nx = data->np_tot[IDIR]; // Number of points in x - jbeg = 0; - jend = data->nghost[JDIR]; offset = data->end[JDIR]; // Distance between beginning of left and right ghosts ny = data->nghost[JDIR]; - kbeg = data->beg[KDIR]; - kend = data->end[KDIR]; - nz = kend - kbeg; // Create the base box to be patched by the ops BoundingBox baseBox; - baseBox[IDIR][0] = ibeg; - baseBox[IDIR][1] = iend; - baseBox[JDIR][0] = jbeg; - baseBox[JDIR][1] = jend; - baseBox[KDIR][0] = kbeg; - baseBox[KDIR][1] = kend; + baseBox[IDIR][0] = 0; + baseBox[IDIR][1] = data->np_tot[IDIR]; + baseBox[JDIR][0] = 0; + baseBox[JDIR][1] = data->nghost[JDIR]; + baseBox[KDIR][0] = data->beg[KDIR]; + baseBox[KDIR][1] = data->end[KDIR]; if(side==left) { //shift by NY @@ -494,7 +486,6 @@ void Axis::ExchangeMPI(int side) { // Load face-centered field in the buffer if (haveMHD) { - int VsIndex = mapNVars*nx*ny*nz; auto sVs = this->symmetryVs; //unpack Vs face-centered @@ -516,7 +507,6 @@ void Axis::ExchangeMPI(int side) { // Load face-centered field in the buffer if (haveMHD) { - int VsIndex = mapNVars*nx*ny*nz; auto sVs = this->symmetryVs; //unpack Vs face-centered on right part From 986505bb53c340fa914460f983f9fbd8ef310354 Mon Sep 17 00:00:00 2001 From: Sebastien Valat Date: Mon, 6 Jul 2026 12:38:35 +0200 Subject: [PATCH 8/8] refactor: Remove some temporary includes (step 5) --- src/fluid/constrainedTransport/EMFexchange.hpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/fluid/constrainedTransport/EMFexchange.hpp b/src/fluid/constrainedTransport/EMFexchange.hpp index 426d56ad2..aad123346 100644 --- a/src/fluid/constrainedTransport/EMFexchange.hpp +++ b/src/fluid/constrainedTransport/EMFexchange.hpp @@ -8,8 +8,6 @@ #ifndef FLUID_CONSTRAINEDTRANSPORT_EMFEXCHANGE_HPP_ #define FLUID_CONSTRAINEDTRANSPORT_EMFEXCHANGE_HPP_ -#include - #include "constrainedTransport.hpp" #include "fluid.hpp" #include "dataBlock.hpp"