diff --git a/src/cmake/GeosxConfig.cmake b/src/cmake/GeosxConfig.cmake index 6fd1b36b3a5..d4d2ebbc84f 100644 --- a/src/cmake/GeosxConfig.cmake +++ b/src/cmake/GeosxConfig.cmake @@ -14,6 +14,7 @@ set( PREPROCESSOR_DEFINES BOUNDS_CHECK METIS MKL MPI + MPI_DESYNC_DETECTION PARMETIS PETSC PYGEOSX diff --git a/src/cmake/GeosxOptions.cmake b/src/cmake/GeosxOptions.cmake index cd076e6fad5..25274b14444 100644 --- a/src/cmake/GeosxOptions.cmake +++ b/src/cmake/GeosxOptions.cmake @@ -92,6 +92,8 @@ endif() option( ENABLE_MPI "" ON ) +option( ENABLE_MPI_DESYNC_DETECTION "" OFF ) + option( ENABLE_CUDA "" OFF ) option( ENABLE_HIP "" OFF ) diff --git a/src/coreComponents/common/GeosxConfig.hpp.in b/src/coreComponents/common/GeosxConfig.hpp.in index 8bfab94de92..5e38ad9fab3 100644 --- a/src/coreComponents/common/GeosxConfig.hpp.in +++ b/src/coreComponents/common/GeosxConfig.hpp.in @@ -44,6 +44,9 @@ /// Enables use of MPI (CMake option ENABLE_MPI) #cmakedefine GEOS_USE_MPI +/// Enables use of MPI (CMake option ENABLE_MPI_DESYNC_DETECTION) +#cmakedefine GEOS_USE_MPI_DESYNC_DETECTION + /// Enables use of OpenMP (CMake option ENABLE_OPENMP) #cmakedefine GEOS_USE_OPENMP diff --git a/src/coreComponents/common/MemoryInfos.cpp b/src/coreComponents/common/MemoryInfos.cpp index 2a2c74f5fd6..2bd61c816b1 100644 --- a/src/coreComponents/common/MemoryInfos.cpp +++ b/src/coreComponents/common/MemoryInfos.cpp @@ -140,9 +140,7 @@ void MemoryLogging::memoryStatsReport() const return; umpire::ResourceManager & rm = umpire::ResourceManager::getInstance(); - integer size; - MPI_Comm_size( MPI_COMM_WORLD, &size ); - size_t nbRank = (std::size_t)size; + std::size_t const nbRank = static_cast< std::size_t >( MpiWrapper::commSize( MPI_COMM_WORLD ) ); // Get a list of all the allocators and sort it so that it's in the same order on each rank. stdVector< string > allocatorNames = rm.getAllocatorNames(); std::sort( allocatorNames.begin(), allocatorNames.end() ); diff --git a/src/coreComponents/common/MpiWrapper.cpp b/src/coreComponents/common/MpiWrapper.cpp index 78c9911d0a5..6633291a56e 100644 --- a/src/coreComponents/common/MpiWrapper.cpp +++ b/src/coreComponents/common/MpiWrapper.cpp @@ -1,528 +1,631 @@ -/* - * ------------------------------------------------------------------------------------------------------------ - * SPDX-License-Identifier: LGPL-2.1-only - * - * Copyright (c) 2016-2024 Lawrence Livermore National Security LLC - * Copyright (c) 2018-2024 TotalEnergies - * Copyright (c) 2018-2024 The Board of Trustees of the Leland Stanford Junior University - * Copyright (c) 2023-2024 Chevron - * Copyright (c) 2019- GEOS/GEOSX Contributors - * All rights reserved - * - * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. - * ------------------------------------------------------------------------------------------------------------ - */ - -/** - * @file MpiWrapper.cpp - */ - -#include "MpiWrapper.hpp" -#include - -#if defined(__clang__) - #pragma clang diagnostic push - #pragma clang diagnostic ignored "-Wunused-parameter" -#elif defined(__GNUC__) - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wunused-parameter" -#endif - - -namespace geos -{ - -#ifdef GEOS_USE_MPI -MPI_Comm MPI_COMM_GEOS; -#else -int MPI_COMM_GEOS = 0; -#endif - -void MpiWrapper::barrier( MPI_Comm const & MPI_PARAM( comm ) ) -{ -#ifdef GEOS_USE_MPI - MPI_Barrier( comm ); -#endif -} - -int MpiWrapper::cartCoords( MPI_Comm comm, int rank, int maxdims, int coords[] ) -{ -#ifdef GEOS_USE_MPI - return MPI_Cart_coords( comm, rank, maxdims, coords ); -#else - return 0; -#endif -} - -int MpiWrapper::cartCreate( MPI_Comm comm_old, int ndims, const int dims[], const int periods[], - int reorder, MPI_Comm * comm_cart ) -{ -#ifdef GEOS_USE_MPI - return MPI_Cart_create( comm_old, ndims, dims, periods, reorder, comm_cart ); -#else - return 0; -#endif -} - -int MpiWrapper::cartRank( MPI_Comm comm, const int coords[] ) -{ - int rank = 0; -#ifdef GEOS_USE_MPI - MPI_Cart_rank( comm, coords, &rank ); -#endif - return rank; -} - -void MpiWrapper::commFree( MPI_Comm & comm ) -{ -#ifdef GEOS_USE_MPI - MPI_CHECK_ERROR( MPI_Comm_free( &comm ) ); -#else -// comm = MPI_COMM_NULL; -#endif -} - -int MpiWrapper::commRank( MPI_Comm const & MPI_PARAM( comm ) ) -{ - int rank = 0; -#ifdef GEOS_USE_MPI - MPI_Comm_rank( comm, &rank ); -#endif - return rank; -} - -int MpiWrapper::commSize( MPI_Comm const & MPI_PARAM( comm ) ) -{ - int size = 1; -#ifdef GEOS_USE_MPI - MPI_Comm_size( comm, &size ); -#endif - return size; -} - -bool MpiWrapper::commCompare( MPI_Comm const & comm1, MPI_Comm const & comm2 ) -{ -#ifdef GEOS_USE_MPI - int result; - MPI_Comm_compare( comm1, comm2, &result ); - return result == MPI_IDENT || result == MPI_CONGRUENT; -#else - return comm1 == comm2; -#endif -} - -bool MpiWrapper::initialized() -{ -#ifdef GEOS_USE_MPI - int ret = false; - MPI_CHECK_ERROR( MPI_Initialized( &ret ) ); - return ret; -#else - return false; -#endif -} - -int MpiWrapper::init( int * argc, char * * * argv ) -{ -#ifdef GEOS_USE_MPI - return MPI_Init( argc, argv ); -#else - return 0; -#endif -} - -internal::ManagedResources & internal::getManagedResources() -{ - static ManagedResources instance; - return instance; -} - -void internal::ManagedResources::finalize() -{ - for( MPI_Op resource : m_mpiOps ) - { - MPI_CHECK_ERROR( MPI_Op_free( &resource ) ); - } - m_mpiOps.clear(); - - for( MPI_Datatype resource : m_mpiTypes ) - { - MPI_CHECK_ERROR( MPI_Type_free( &resource ) ); - } - m_mpiTypes.clear(); -} - -void MpiWrapper::finalize() -{ -#ifdef GEOS_USE_MPI - MpiWrapper::commFree( MPI_COMM_GEOS ); - internal::getManagedResources().finalize(); - MPI_CHECK_ERROR( MPI_Finalize() ); -#endif -} - - -MPI_Comm MpiWrapper::commDup( MPI_Comm const comm ) -{ -#ifdef GEOS_USE_MPI - MPI_Comm duplicate; - MPI_CHECK_ERROR( MPI_Comm_dup( comm, &duplicate ) ); - return duplicate; -#else - return comm; -#endif -} - -MPI_Comm MpiWrapper::commSplit( MPI_Comm const comm, int color, int key ) -{ -#ifdef GEOS_USE_MPI - MPI_Comm scomm; - MPI_CHECK_ERROR( MPI_Comm_split( comm, color, key, &scomm ) ); - return scomm; -#else - return comm; -#endif -} - -int MpiWrapper::test( MPI_Request * request, int * flag, MPI_Status * status ) -{ -#ifdef GEOS_USE_MPI - return MPI_Test( request, flag, status ); -#else - *flag = 0; - return 0; -#endif -} - -int MpiWrapper::testAny( int count, MPI_Request array_of_requests[], int * idx, int * flag, MPI_Status array_of_statuses[] ) -{ -#ifdef GEOS_USE_MPI - return MPI_Testany( count, array_of_requests, idx, flag, array_of_statuses ); -#else - *flag = 0; - return 0; -#endif -} - -int MpiWrapper::testSome( int count, MPI_Request array_of_requests[], int * outcount, int array_of_indices[], MPI_Status array_of_statuses[] ) -{ -#ifdef GEOS_USE_MPI - return MPI_Testsome( count, array_of_requests, outcount, array_of_indices, array_of_statuses ); -#else - *outcount = 0; - return 0; -#endif -} - -int MpiWrapper::testAll( int count, MPI_Request array_of_requests[], int * flag, MPI_Status array_of_statuses[] ) -{ -#ifdef GEOS_USE_MPI - return MPI_Testall( count, array_of_requests, flag, array_of_statuses ); -#else - *flag = 0; - return 0; -#endif -} - -int MpiWrapper::check( MPI_Request * request, int * flag, MPI_Status * status ) -{ -#ifdef GEOS_USE_MPI - return MPI_Request_get_status( *request, flag, status ); -#else - *flag = 0; - return 0; -#endif -} - -int MpiWrapper::checkAny( int count, MPI_Request array_of_requests[], int * idx, int * flag, MPI_Status array_of_statuses[] ) -{ -#ifdef GEOS_USE_MPI - bool found = false; - int flagCache = -1; - int rval = MPI_SUCCESS; - stdVector< int > rvals( count ); - for( int jdx = 0; jdx < count; ++jdx ) - { - *flag = 0; - rvals[ jdx ] = MPI_Request_get_status( array_of_requests[ jdx ], flag, &array_of_statuses[ jdx ] ); - if( *flag && !found ) - { - *idx = jdx; - flagCache = *flag; - } - if( rvals[ jdx ] != MPI_SUCCESS ) - { - rval = rvals[ jdx ]; - } - } - if( found ) - { - *flag = flagCache; - } - return rval; -#else - *flag = 0; - return 0; -#endif -} - -int MpiWrapper::checkAll( int count, MPI_Request array_of_requests[], int * flag, MPI_Status array_of_statuses[] ) -{ -#ifdef GEOS_USE_MPI - // assume all passing, any that don't pass set the flag to false - *flag = 1; - int rval = MPI_SUCCESS; - stdVector< int > rvals( count ); - int iFlag = 0; - for( int idx = 0; idx < count; ++idx ) - { - rvals[ idx ] = MPI_Request_get_status( array_of_requests[ idx ], &iFlag, &array_of_statuses[ idx ] ); - if( !iFlag ) - { - *flag = iFlag; - } - if( rvals[ idx ] != MPI_SUCCESS ) - { - rval = rvals[ idx ]; - } - } - return rval; -#else - *flag = 0; - return 0; -#endif -} - -int MpiWrapper::wait( MPI_Request * request, MPI_Status * status ) -{ -#ifdef GEOS_USE_MPI - return MPI_Wait( request, status ); -#else - return 0; -#endif -} - -int MpiWrapper::waitAny( int count, MPI_Request array_of_requests[], int * indx, MPI_Status array_of_statuses[] ) -{ -#ifdef GEOS_USE_MPI - return MPI_Waitany( count, array_of_requests, indx, array_of_statuses ); -#else - return 0; -#endif -} - -int MpiWrapper::waitSome( int count, MPI_Request array_of_requests[], int * outcount, int array_of_indices[], MPI_Status array_of_statuses[] ) -{ -#ifdef GEOS_USE_MPI - return MPI_Waitsome( count, array_of_requests, outcount, array_of_indices, array_of_statuses ); -#else - // *outcount = 0; - return 0; -#endif -} - -int MpiWrapper::waitAll( int count, MPI_Request array_of_requests[], MPI_Status array_of_statuses[] ) -{ -#ifdef GEOS_USE_MPI - return MPI_Waitall( count, array_of_requests, array_of_statuses ); -#else - return 0; -#endif -} - -double MpiWrapper::wtime( void ) -{ -#ifdef GEOS_USE_MPI - return MPI_Wtime( ); -#else - return 0; -#endif - -} - -int MpiWrapper::activeWaitAny( const int count, MPI_Request array_of_requests[], MPI_Status array_of_statuses[], std::function< MPI_Request ( int ) > func ) -{ - int cmp = 0; - while( cmp < count ) - { - int idx = 0; - int err = waitAny( count, array_of_requests, &idx, array_of_statuses ); - if( err != MPI_SUCCESS ) - return err; - if( idx != MPI_UNDEFINED ) // only if all(requests == MPI_REQUEST_NULL) - { - func( idx ); - } - cmp++; - } - return MPI_SUCCESS; -} - -int MpiWrapper::activeWaitSome( const int count, - MPI_Request array_of_requests[], - MPI_Status array_of_statuses[], - std::function< MPI_Request ( int ) > func ) -{ - int cmp = 0; - while( cmp < count ) - { - int rcvd = 0; - stdVector< int > indices( count, -1 ); - int err = waitSome( count, array_of_requests, &rcvd, &indices[0], array_of_statuses ); - if( err != MPI_SUCCESS ) - return err; - if( rcvd > 0 ) - { - for( int ii = 0; ii < rcvd; ++ii ) - { - if( indices[ii] != MPI_UNDEFINED ) - { - func( indices[ii] ); - } - } - } - cmp += rcvd; - } - return MPI_SUCCESS; -} - - -int MpiWrapper::activeWaitSomeCompletePhase( const int participants, - stdVector< std::tuple< MPI_Request *, MPI_Status *, std::function< MPI_Request ( int ) > > > const & phases ) -{ - const int num_phases = phases.size(); - int err = 0; - for( int phase = 0; phase < num_phases; ++phase ) - { - MPI_Request * const requests = std::get< 0 >( phases[phase] ); - MPI_Status * const statuses = std::get< 1 >( phases[phase] ); - std::function< MPI_Request ( int ) > func = std::get< 2 >( phases[phase] ); - if( requests!=nullptr ) - { - err = activeWaitSome( participants, - requests, - statuses, - func ); - } - else - { - for( int idx = 0; idx < participants; ++idx ) - { - func( idx ); - } - } - if( err != MPI_SUCCESS ) - break; - } - return err; -} - -int MpiWrapper::activeWaitOrderedCompletePhase( const int participants, - stdVector< std::tuple< MPI_Request *, MPI_Status *, std::function< MPI_Request ( int ) > > > const & phases ) -{ - const int num_phases = phases.size(); - for( int phase = 0; phase < num_phases; ++phase ) - { - MPI_Request * const requests = std::get< 0 >( phases[phase] ); - MPI_Status * const statuses = std::get< 1 >( phases[phase] ); - std::function< MPI_Request ( int ) > func = std::get< 2 >( phases[phase] ); - - for( int idx = 0; idx < participants; ++idx ) - { - if( requests!=nullptr ) - { - wait( &requests[idx], &statuses[idx] ); - } - func( idx ); - } - } - return MPI_SUCCESS; -} - -int MpiWrapper::nodeCommSize() -{ - // if not initialized then we guess there is no MPI. - if( !initialized() ) - return 1; - - int len; - std::array< char, MPI_MAX_PROCESSOR_NAME + 1 > hostname; - MPI_Get_processor_name( hostname.data(), &len ); - hostname[len] = '\0'; - int color = (int)std::hash< string >{} (hostname.data()); - if( color < 0 ) - color *= -1; - - /** - * Create intra-node communicator - */ - MPI_Comm nodeComm; - int nodeCommSize; - MPI_Comm_split( MPI_COMM_WORLD, color, -1, &nodeComm ); - MPI_Comm_size( nodeComm, &nodeCommSize ); - return nodeCommSize; -} - -namespace internal -{ - -template< typename FIRST, typename SECOND > -MPI_Datatype getMpiCustomPairType() -{ - static auto const createTypeHolder = [] () { - using PAIR_T = MpiWrapper::PairType< FIRST, SECOND >; - static_assert( std::is_standard_layout_v< PAIR_T > ); - static_assert( std::is_trivially_copyable_v< PAIR_T > ); - - MPI_Datatype types[2] = { getMpiType< FIRST >(), getMpiType< SECOND >() }; - MPI_Aint offsets[2] = { offsetof( PAIR_T, first ), offsetof( PAIR_T, second ) }; - int blocksCount[2] = { 1, 1 }; - - MPI_Datatype mpiType; - GEOS_ERROR_IF_NE( MPI_Type_create_struct( 2, blocksCount, offsets, types, &mpiType ), MPI_SUCCESS ); - GEOS_ERROR_IF_NE( MPI_Type_commit( &mpiType ), MPI_SUCCESS ); - // Resource registered to be destroyed at MpiWrapper::finalize(). - internal::getManagedResources().m_mpiTypes.emplace( mpiType ); - return mpiType; - }; - // Static storage to ensure the MPI operation is created only once and reused for all calls to this function. - static MPI_Datatype mpiType{ createTypeHolder() }; - return mpiType; -} - -template<> MPI_Datatype getMpiPairType< int, int >() -{ return MPI_2INT; } - -template<> MPI_Datatype getMpiPairType< long int, int >() -{ return MPI_LONG_INT; } - -template<> MPI_Datatype getMpiPairType< long int, long int >() -{ return getMpiCustomPairType< long int, long int >(); } - -template<> MPI_Datatype getMpiPairType< long long int, long long int >() -{ return getMpiCustomPairType< long long int, long long int >(); } - -template<> MPI_Datatype getMpiPairType< float, int >() -{ return MPI_FLOAT_INT; } - -template<> MPI_Datatype getMpiPairType< double, int >() -{ return MPI_DOUBLE_INT; } - -template<> MPI_Datatype getMpiPairType< double, long int >() -{ return getMpiCustomPairType< double, long int >(); } - -template<> MPI_Datatype getMpiPairType< double, long long int >() -{ return getMpiCustomPairType< double, long long int >(); } - -template<> MPI_Datatype getMpiPairType< double, double >() -{ return getMpiCustomPairType< double, double >(); } - -} /* namespace internal */ - -} /* namespace geos */ - -#if defined(__clang__) - #pragma clang diagnostic pop -#elif defined(__GNUC__) - #pragma GCC diagnostic pop -#endif +/* + * ------------------------------------------------------------------------------------------------------------ + * SPDX-License-Identifier: LGPL-2.1-only + * + * Copyright (c) 2016-2024 Lawrence Livermore National Security LLC + * Copyright (c) 2018-2024 TotalEnergies + * Copyright (c) 2018-2024 The Board of Trustees of the Leland Stanford Junior University + * Copyright (c) 2023-2024 Chevron + * Copyright (c) 2019- GEOS/GEOSX Contributors + * All rights reserved + * + * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. + * ------------------------------------------------------------------------------------------------------------ + */ + +/** + * @file MpiWrapper.cpp + */ + +#include "MpiWrapper.hpp" +#include "LvArray/src/system.hpp" +#include "common/logger/Logger.hpp" +#include +#include +#include + +#if defined( GEOS_USE_MPI ) && defined( GEOS_USE_MPI_DESYNC_DETECTION ) +#if defined(__GLIBC__) || defined(__APPLE__) +#include +#include +#endif +#endif + +#if defined(__clang__) + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wunused-parameter" +#elif defined(__GNUC__) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + + +namespace geos +{ + +#ifdef GEOS_USE_MPI +MPI_Comm MPI_COMM_GEOS; +#else +int MPI_COMM_GEOS = 0; +#endif + +namespace internal +{ + +#if defined( GEOS_USE_MPI ) && defined( GEOS_USE_MPI_DESYNC_DETECTION ) +stdArray< void *, MpiDesyncGuard::maxFrames > g_lastSuccessfulFrames; +int g_lastSuccessfulFrameCount = 0; + +void MpiDesyncGuard::saveStackFrames() +{ + // LvArray already implements an equivalent (see internal `collect()` in LvArray/src/system.cpp) + // but it is not exposed to the public API. (TODO) Remove this in favor of LvArray's `collect()` + // if it becomes public. +#if defined(__GLIBC__) || defined(__APPLE__) + m_frameCount = backtrace( m_frames.data(), MpiDesyncGuard::maxFrames ); +#else + m_frameCount = 0; +#endif +} + +string MpiDesyncGuard::symbolizeStackTrace( stdArray< void *, maxFrames > const & frames, + int const frameCount ) const +{ + // adapted from LvArray::system::getFunctionNameFromFrame() + std::ostringstream oss; +#if defined(__GLIBC__) || defined(__APPLE__) + for( int i = 0; i < m_frameCount; ++i ) + { + Dl_info dli; + bool const dlOk = dladdr( m_frames[ i ], &dli ); + + oss << "Frame " << i << ": " + << ( dlOk ? + ( dli.dli_sname ? LvArray::system::demangle( dli.dli_sname ) : dli.dli_fname ) : + "Unknown" ) + << "\n"; + } +#else + GEOS_UNUSED_VAR( frames, frameCount ); +#endif + return oss.str(); +} + +void MpiDesyncGuard::failed() +{ + GEOS_LOG_RANK_0( GEOS_FMT( "MPI desync detected: rank timed out\n" + "{}\n" + "Last successful stacktrace:\n" + "{}", + symbolizeStackTrace( m_frames, m_frameCount ), + symbolizeStackTrace( g_lastSuccessfulFrames, g_lastSuccessfulFrameCount ) ) ); + MPI_Abort( m_comm, 1 ); +} + +void MpiDesyncGuard::succeeded() +{ + m_collectiveOperationSuccess = true; + g_lastSuccessfulFrames = m_frames; + g_lastSuccessfulFrameCount = m_frameCount; +} + +void MpiDesyncGuard::detectMpiDesync() +{ + MPI_Request request; + MPI_Ibarrier( m_comm, &request ); + + int flag = 0; + double start = MpiWrapper::wtime(); + while( true ) + { + MpiWrapper::test( &request, &flag, MPI_STATUS_IGNORE ); + if( flag ) + { + succeeded(); + return; + } + + if( MpiWrapper::wtime() - start > 10 ) + { + failed(); + return; + } + + std::this_thread::sleep_for( std::chrono::milliseconds( 100 ) ); + } +} +#endif + +} // namespace internal + +void MpiWrapper::barrier( MPI_Comm const & MPI_PARAM( comm ) ) +{ +#ifdef GEOS_USE_MPI +#ifdef GEOS_USE_MPI_DESYNC_DETECTION + internal::MpiDesyncGuard const mpiDesyncGuard( comm ); +#endif + MPI_Barrier( comm ); +#endif +} + +int MpiWrapper::cartCoords( MPI_Comm comm, int rank, int maxdims, int coords[] ) +{ +#ifdef GEOS_USE_MPI + return MPI_Cart_coords( comm, rank, maxdims, coords ); +#else + return 0; +#endif +} + +int MpiWrapper::cartCreate( MPI_Comm comm_old, int ndims, const int dims[], const int periods[], + int reorder, MPI_Comm * comm_cart ) +{ +#ifdef GEOS_USE_MPI + return MPI_Cart_create( comm_old, ndims, dims, periods, reorder, comm_cart ); +#else + return 0; +#endif +} + +int MpiWrapper::cartRank( MPI_Comm comm, const int coords[] ) +{ + int rank = 0; +#ifdef GEOS_USE_MPI + MPI_Cart_rank( comm, coords, &rank ); +#endif + return rank; +} + +void MpiWrapper::commFree( MPI_Comm & comm ) +{ +#ifdef GEOS_USE_MPI + MPI_CHECK_ERROR( MPI_Comm_free( &comm ) ); +#else +// comm = MPI_COMM_NULL; +#endif +} + +int MpiWrapper::commRank( MPI_Comm const & MPI_PARAM( comm ) ) +{ + int rank = 0; +#ifdef GEOS_USE_MPI + MPI_Comm_rank( comm, &rank ); +#endif + return rank; +} + +int MpiWrapper::commSize( MPI_Comm const & MPI_PARAM( comm ) ) +{ + int size = 1; +#ifdef GEOS_USE_MPI + MPI_Comm_size( comm, &size ); +#endif + return size; +} + +bool MpiWrapper::commCompare( MPI_Comm const & comm1, MPI_Comm const & comm2 ) +{ +#ifdef GEOS_USE_MPI + int result; + MPI_Comm_compare( comm1, comm2, &result ); + return result == MPI_IDENT || result == MPI_CONGRUENT; +#else + return comm1 == comm2; +#endif +} + +bool MpiWrapper::initialized() +{ +#ifdef GEOS_USE_MPI + int ret = false; + MPI_CHECK_ERROR( MPI_Initialized( &ret ) ); + return ret; +#else + return false; +#endif +} + +int MpiWrapper::init( int * argc, char * * * argv ) +{ +#ifdef GEOS_USE_MPI + return MPI_Init( argc, argv ); +#else + return 0; +#endif +} + +internal::ManagedResources & internal::getManagedResources() +{ + static ManagedResources instance; + return instance; +} + +void internal::ManagedResources::finalize() +{ + for( MPI_Op resource : m_mpiOps ) + { + MPI_CHECK_ERROR( MPI_Op_free( &resource ) ); + } + m_mpiOps.clear(); + + for( MPI_Datatype resource : m_mpiTypes ) + { + MPI_CHECK_ERROR( MPI_Type_free( &resource ) ); + } + m_mpiTypes.clear(); +} + +void MpiWrapper::finalize() +{ +#ifdef GEOS_USE_MPI + MpiWrapper::commFree( MPI_COMM_GEOS ); + internal::getManagedResources().finalize(); + MPI_CHECK_ERROR( MPI_Finalize() ); +#endif +} + + +MPI_Comm MpiWrapper::commDup( MPI_Comm const comm ) +{ +#ifdef GEOS_USE_MPI + MPI_Comm duplicate; + MPI_CHECK_ERROR( MPI_Comm_dup( comm, &duplicate ) ); + return duplicate; +#else + return comm; +#endif +} + +MPI_Comm MpiWrapper::commSplit( MPI_Comm const comm, int color, int key ) +{ +#ifdef GEOS_USE_MPI + MPI_Comm scomm; + MPI_CHECK_ERROR( MPI_Comm_split( comm, color, key, &scomm ) ); + return scomm; +#else + return comm; +#endif +} + +int MpiWrapper::test( MPI_Request * request, int * flag, MPI_Status * status ) +{ +#ifdef GEOS_USE_MPI + return MPI_Test( request, flag, status ); +#else + *flag = 0; + return 0; +#endif +} + +int MpiWrapper::testAny( int count, MPI_Request array_of_requests[], int * idx, int * flag, MPI_Status array_of_statuses[] ) +{ +#ifdef GEOS_USE_MPI + return MPI_Testany( count, array_of_requests, idx, flag, array_of_statuses ); +#else + *flag = 0; + return 0; +#endif +} + +int MpiWrapper::testSome( int count, MPI_Request array_of_requests[], int * outcount, int array_of_indices[], MPI_Status array_of_statuses[] ) +{ +#ifdef GEOS_USE_MPI + return MPI_Testsome( count, array_of_requests, outcount, array_of_indices, array_of_statuses ); +#else + *outcount = 0; + return 0; +#endif +} + +int MpiWrapper::testAll( int count, MPI_Request array_of_requests[], int * flag, MPI_Status array_of_statuses[] ) +{ +#ifdef GEOS_USE_MPI + return MPI_Testall( count, array_of_requests, flag, array_of_statuses ); +#else + *flag = 0; + return 0; +#endif +} + +int MpiWrapper::check( MPI_Request * request, int * flag, MPI_Status * status ) +{ +#ifdef GEOS_USE_MPI + return MPI_Request_get_status( *request, flag, status ); +#else + *flag = 0; + return 0; +#endif +} + +int MpiWrapper::checkAny( int count, MPI_Request array_of_requests[], int * idx, int * flag, MPI_Status array_of_statuses[] ) +{ +#ifdef GEOS_USE_MPI + bool found = false; + int flagCache = -1; + int rval = MPI_SUCCESS; + stdVector< int > rvals( count ); + for( int jdx = 0; jdx < count; ++jdx ) + { + *flag = 0; + rvals[ jdx ] = MPI_Request_get_status( array_of_requests[ jdx ], flag, &array_of_statuses[ jdx ] ); + if( *flag && !found ) + { + *idx = jdx; + flagCache = *flag; + } + if( rvals[ jdx ] != MPI_SUCCESS ) + { + rval = rvals[ jdx ]; + } + } + if( found ) + { + *flag = flagCache; + } + return rval; +#else + *flag = 0; + return 0; +#endif +} + +int MpiWrapper::checkAll( int count, MPI_Request array_of_requests[], int * flag, MPI_Status array_of_statuses[] ) +{ +#ifdef GEOS_USE_MPI + // assume all passing, any that don't pass set the flag to false + *flag = 1; + int rval = MPI_SUCCESS; + stdVector< int > rvals( count ); + int iFlag = 0; + for( int idx = 0; idx < count; ++idx ) + { + rvals[ idx ] = MPI_Request_get_status( array_of_requests[ idx ], &iFlag, &array_of_statuses[ idx ] ); + if( !iFlag ) + { + *flag = iFlag; + } + if( rvals[ idx ] != MPI_SUCCESS ) + { + rval = rvals[ idx ]; + } + } + return rval; +#else + *flag = 0; + return 0; +#endif +} + +int MpiWrapper::wait( MPI_Request * request, MPI_Status * status ) +{ +#ifdef GEOS_USE_MPI + return MPI_Wait( request, status ); +#else + return 0; +#endif +} + +int MpiWrapper::waitAny( int count, MPI_Request array_of_requests[], int * indx, MPI_Status array_of_statuses[] ) +{ +#ifdef GEOS_USE_MPI + return MPI_Waitany( count, array_of_requests, indx, array_of_statuses ); +#else + return 0; +#endif +} + +int MpiWrapper::waitSome( int count, MPI_Request array_of_requests[], int * outcount, int array_of_indices[], MPI_Status array_of_statuses[] ) +{ +#ifdef GEOS_USE_MPI + return MPI_Waitsome( count, array_of_requests, outcount, array_of_indices, array_of_statuses ); +#else + // *outcount = 0; + return 0; +#endif +} + +int MpiWrapper::waitAll( int count, MPI_Request array_of_requests[], MPI_Status array_of_statuses[] ) +{ +#ifdef GEOS_USE_MPI + return MPI_Waitall( count, array_of_requests, array_of_statuses ); +#else + return 0; +#endif +} + +double MpiWrapper::wtime( void ) +{ +#ifdef GEOS_USE_MPI + return MPI_Wtime( ); +#else + return 0; +#endif + +} + +int MpiWrapper::activeWaitAny( const int count, MPI_Request array_of_requests[], MPI_Status array_of_statuses[], std::function< MPI_Request ( int ) > func ) +{ + int cmp = 0; + while( cmp < count ) + { + int idx = 0; + int err = waitAny( count, array_of_requests, &idx, array_of_statuses ); + if( err != MPI_SUCCESS ) + return err; + if( idx != MPI_UNDEFINED ) // only if all(requests == MPI_REQUEST_NULL) + { + func( idx ); + } + cmp++; + } + return MPI_SUCCESS; +} + +int MpiWrapper::activeWaitSome( const int count, + MPI_Request array_of_requests[], + MPI_Status array_of_statuses[], + std::function< MPI_Request ( int ) > func ) +{ + int cmp = 0; + while( cmp < count ) + { + int rcvd = 0; + stdVector< int > indices( count, -1 ); + int err = waitSome( count, array_of_requests, &rcvd, &indices[0], array_of_statuses ); + if( err != MPI_SUCCESS ) + return err; + if( rcvd > 0 ) + { + for( int ii = 0; ii < rcvd; ++ii ) + { + if( indices[ii] != MPI_UNDEFINED ) + { + func( indices[ii] ); + } + } + } + cmp += rcvd; + } + return MPI_SUCCESS; +} + + +int MpiWrapper::activeWaitSomeCompletePhase( const int participants, + stdVector< std::tuple< MPI_Request *, MPI_Status *, std::function< MPI_Request ( int ) > > > const & phases ) +{ + const int num_phases = phases.size(); + int err = 0; + for( int phase = 0; phase < num_phases; ++phase ) + { + MPI_Request * const requests = std::get< 0 >( phases[phase] ); + MPI_Status * const statuses = std::get< 1 >( phases[phase] ); + std::function< MPI_Request ( int ) > func = std::get< 2 >( phases[phase] ); + if( requests!=nullptr ) + { + err = activeWaitSome( participants, + requests, + statuses, + func ); + } + else + { + for( int idx = 0; idx < participants; ++idx ) + { + func( idx ); + } + } + if( err != MPI_SUCCESS ) + break; + } + return err; +} + +int MpiWrapper::activeWaitOrderedCompletePhase( const int participants, + stdVector< std::tuple< MPI_Request *, MPI_Status *, std::function< MPI_Request ( int ) > > > const & phases ) +{ + const int num_phases = phases.size(); + for( int phase = 0; phase < num_phases; ++phase ) + { + MPI_Request * const requests = std::get< 0 >( phases[phase] ); + MPI_Status * const statuses = std::get< 1 >( phases[phase] ); + std::function< MPI_Request ( int ) > func = std::get< 2 >( phases[phase] ); + + for( int idx = 0; idx < participants; ++idx ) + { + if( requests!=nullptr ) + { + wait( &requests[idx], &statuses[idx] ); + } + func( idx ); + } + } + return MPI_SUCCESS; +} + +int MpiWrapper::nodeCommSize() +{ + // if not initialized then we guess there is no MPI. + if( !initialized() ) + return 1; + + int len; + std::array< char, MPI_MAX_PROCESSOR_NAME + 1 > hostname; + MPI_Get_processor_name( hostname.data(), &len ); + hostname[len] = '\0'; + int color = (int)std::hash< string >{} (hostname.data()); + if( color < 0 ) + color *= -1; + + /** + * Create intra-node communicator + */ + MPI_Comm nodeComm; + int nodeCommSize; + MPI_Comm_split( MPI_COMM_WORLD, color, -1, &nodeComm ); + MPI_Comm_size( nodeComm, &nodeCommSize ); + return nodeCommSize; +} + +namespace internal +{ + +template< typename FIRST, typename SECOND > +MPI_Datatype getMpiCustomPairType() +{ + static auto const createTypeHolder = [] () { + using PAIR_T = MpiWrapper::PairType< FIRST, SECOND >; + static_assert( std::is_standard_layout_v< PAIR_T > ); + static_assert( std::is_trivially_copyable_v< PAIR_T > ); + + MPI_Datatype types[2] = { getMpiType< FIRST >(), getMpiType< SECOND >() }; + MPI_Aint offsets[2] = { offsetof( PAIR_T, first ), offsetof( PAIR_T, second ) }; + int blocksCount[2] = { 1, 1 }; + + MPI_Datatype mpiType; + GEOS_ERROR_IF_NE( MPI_Type_create_struct( 2, blocksCount, offsets, types, &mpiType ), MPI_SUCCESS ); + GEOS_ERROR_IF_NE( MPI_Type_commit( &mpiType ), MPI_SUCCESS ); + // Resource registered to be destroyed at MpiWrapper::finalize(). + internal::getManagedResources().m_mpiTypes.emplace( mpiType ); + return mpiType; + }; + // Static storage to ensure the MPI operation is created only once and reused for all calls to this function. + static MPI_Datatype mpiType{ createTypeHolder() }; + return mpiType; +} + +template<> MPI_Datatype getMpiPairType< int, int >() +{ return MPI_2INT; } + +template<> MPI_Datatype getMpiPairType< long int, int >() +{ return MPI_LONG_INT; } + +template<> MPI_Datatype getMpiPairType< long int, long int >() +{ return getMpiCustomPairType< long int, long int >(); } + +template<> MPI_Datatype getMpiPairType< long long int, long long int >() +{ return getMpiCustomPairType< long long int, long long int >(); } + +template<> MPI_Datatype getMpiPairType< float, int >() +{ return MPI_FLOAT_INT; } + +template<> MPI_Datatype getMpiPairType< double, int >() +{ return MPI_DOUBLE_INT; } + +template<> MPI_Datatype getMpiPairType< double, long int >() +{ return getMpiCustomPairType< double, long int >(); } + +template<> MPI_Datatype getMpiPairType< double, long long int >() +{ return getMpiCustomPairType< double, long long int >(); } + +template<> MPI_Datatype getMpiPairType< double, double >() +{ return getMpiCustomPairType< double, double >(); } + +} /* namespace internal */ + +} /* namespace geos */ + +#if defined(__clang__) + #pragma clang diagnostic pop +#elif defined(__GNUC__) + #pragma GCC diagnostic pop +#endif diff --git a/src/coreComponents/common/MpiWrapper.hpp b/src/coreComponents/common/MpiWrapper.hpp index 30e65f896c0..df45d15b521 100644 --- a/src/coreComponents/common/MpiWrapper.hpp +++ b/src/coreComponents/common/MpiWrapper.hpp @@ -942,7 +942,49 @@ MPI_Op getMpiPairReductionOp() return mpiOp; } -} +#ifdef GEOS_USE_MPI_DESYNC_DETECTION +/** + * @struct MpiDesyncGuard + * @brief RAII helper to detect MPI desynchronizations from MPI collective operations. + */ +struct MpiDesyncGuard +{ + MPI_Comm const & m_comm; + bool m_collectiveOperationSuccess{ false }; + + static constexpr int maxFrames = 30; + stdArray< void *, maxFrames > m_frames{}; + int m_frameCount = 0; + + explicit MpiDesyncGuard( MPI_Comm const & comm ) + : m_comm( comm ) + { saveStackFrames(); } + + ~MpiDesyncGuard() + { detectMpiDesync(); } + + /// @brief Detects MPI desynchronizations from MPI collective operations. + void detectMpiDesync(); + + /// @brief Method ran when a desynchronization is detected. + void failed(); + + /// @brief Method ran when no desynchronizations are detected. + void succeeded(); + + MpiDesyncGuard( MpiDesyncGuard const & ) = delete; + MpiDesyncGuard & operator=( MpiDesyncGuard const & ) = delete; + + /// @brief Save the stacktrace frames + void saveStackFrames(); + /// @brief Symbolize and demangle the captured stacktrace frames + string symbolizeStackTrace( stdArray< void *, maxFrames > const & frames, + int const frameCount ) const; + +}; +#endif + +} // namespace internal inline MPI_Op MpiWrapper::getMpiOp( Reduction const op ) { @@ -986,6 +1028,9 @@ int MpiWrapper::allgather( T_SEND const * const sendbuf, MPI_Comm MPI_PARAM( comm ) ) { #ifdef GEOS_USE_MPI +#ifdef GEOS_USE_MPI_DESYNC_DETECTION + internal::MpiDesyncGuard const mpiDesyncGuard( comm ); +#endif return MPI_Allgather( sendbuf, sendcount, internal::getMpiType< T_SEND >(), recvbuf, recvcount, internal::getMpiType< T_RECV >(), comm ); @@ -1007,6 +1052,9 @@ int MpiWrapper::allgatherv( T_SEND const * const sendbuf, MPI_Comm MPI_PARAM( comm ) ) { #ifdef GEOS_USE_MPI +#ifdef GEOS_USE_MPI_DESYNC_DETECTION + internal::MpiDesyncGuard const mpiDesyncGuard( comm ); +#endif return MPI_Allgatherv( sendbuf, sendcount, internal::getMpiType< T_SEND >(), recvbuf, recvcounts, displacements, internal::getMpiType< T_RECV >(), comm ); @@ -1024,6 +1072,9 @@ template< typename T > void MpiWrapper::allGather( T const myValue, array1d< T > & allValues, MPI_Comm MPI_PARAM( comm ) ) { #ifdef GEOS_USE_MPI +#ifdef GEOS_USE_MPI_DESYNC_DETECTION + internal::MpiDesyncGuard const mpiDesyncGuard( comm ); +#endif int const mpiSize = commSize( comm ); allValues.resize( mpiSize ); @@ -1044,6 +1095,9 @@ int MpiWrapper::allGather( arrayView1d< T const > const & sendValues, { int const sendSize = LvArray::integerConversion< int >( sendValues.size() ); #ifdef GEOS_USE_MPI +#ifdef GEOS_USE_MPI_DESYNC_DETECTION + internal::MpiDesyncGuard const mpiDesyncGuard( comm ); +#endif int const mpiSize = commSize( comm ); allValues.resize( mpiSize * sendSize ); return MPI_Allgather( sendValues.data(), @@ -1071,6 +1125,9 @@ int MpiWrapper::allGatherv( arrayView1d< T const > const & sendValues, { int const sendSize = LvArray::integerConversion< int >( sendValues.size() ); #ifdef GEOS_USE_MPI +#ifdef GEOS_USE_MPI_DESYNC_DETECTION + internal::MpiDesyncGuard const mpiDesyncGuard( comm ); +#endif int const mpiSize = commSize( comm ); array1d< int > counts; allGather( sendSize, counts, comm ); @@ -1104,6 +1161,9 @@ int MpiWrapper::allReduce( T const * const sendbuf, MPI_Comm const MPI_PARAM( comm ) ) { #ifdef GEOS_USE_MPI +#ifdef GEOS_USE_MPI_DESYNC_DETECTION + internal::MpiDesyncGuard const mpiDesyncGuard( comm ); +#endif MPI_Datatype const mpiType = internal::getMpiType< T >(); return MPI_Allreduce( sendbuf == recvbuf ? MPI_IN_PLACE : sendbuf, recvbuf, count, mpiType, op, comm ); #else @@ -1124,6 +1184,9 @@ int MpiWrapper::reduce( T const * const sendbuf, MPI_Comm const MPI_PARAM( comm ) ) { #ifdef GEOS_USE_MPI +#ifdef GEOS_USE_MPI_DESYNC_DETECTION + internal::MpiDesyncGuard const mpiDesyncGuard( comm ); +#endif MPI_Datatype const mpiType = internal::getMpiType< T >(); return MPI_Reduce( sendbuf == recvbuf ? MPI_IN_PLACE : sendbuf, recvbuf, count, mpiType, op, root, comm ); #else @@ -1143,6 +1206,9 @@ int MpiWrapper::scan( T const * const sendbuf, MPI_Comm MPI_PARAM( comm ) ) { #ifdef GEOS_USE_MPI +#ifdef GEOS_USE_MPI_DESYNC_DETECTION + internal::MpiDesyncGuard const mpiDesyncGuard( comm ); +#endif return MPI_Scan( sendbuf, recvbuf, count, internal::getMpiType< T >(), op, comm ); #else memcpy( recvbuf, sendbuf, count*sizeof(T) ); @@ -1158,6 +1224,9 @@ int MpiWrapper::exscan( T const * const MPI_PARAM( sendbuf ), MPI_Comm MPI_PARAM( comm ) ) { #ifdef GEOS_USE_MPI +#ifdef GEOS_USE_MPI_DESYNC_DETECTION + internal::MpiDesyncGuard const mpiDesyncGuard( comm ); +#endif return MPI_Exscan( sendbuf, recvbuf, count, internal::getMpiType< T >(), op, comm ); #else memset( recvbuf, 0, count*sizeof(T) ); @@ -1172,6 +1241,9 @@ int MpiWrapper::bcast( T * const MPI_PARAM( buffer ), MPI_Comm MPI_PARAM( comm ) ) { #ifdef GEOS_USE_MPI +#ifdef GEOS_USE_MPI_DESYNC_DETECTION + internal::MpiDesyncGuard const mpiDesyncGuard( comm ); +#endif return MPI_Bcast( buffer, count, internal::getMpiType< T >(), root, comm ); #else return 0; @@ -1183,6 +1255,9 @@ template< typename T > void MpiWrapper::broadcast( T & MPI_PARAM( value ), int MPI_PARAM( srcRank ), MPI_Comm MPI_PARAM( comm ) ) { #ifdef GEOS_USE_MPI +#ifdef GEOS_USE_MPI_DESYNC_DETECTION + internal::MpiDesyncGuard const mpiDesyncGuard( comm ); +#endif MPI_Bcast( &value, 1, internal::getMpiType< T >(), srcRank, comm ); #endif } @@ -1194,6 +1269,9 @@ void MpiWrapper::broadcast< string >( string & MPI_PARAM( value ), MPI_Comm MPI_PARAM( comm ) ) { #ifdef GEOS_USE_MPI +#ifdef GEOS_USE_MPI_DESYNC_DETECTION + internal::MpiDesyncGuard const mpiDesyncGuard( comm ); +#endif int size = LvArray::integerConversion< int >( value.size() ); broadcast( size, srcRank, comm ); value.resize( size ); @@ -1210,6 +1288,9 @@ int MpiWrapper::gather( TS const * const sendbuf, MPI_Comm MPI_PARAM( comm ) ) { #ifdef GEOS_USE_MPI +#ifdef GEOS_USE_MPI_DESYNC_DETECTION + internal::MpiDesyncGuard const mpiDesyncGuard( comm ); +#endif return MPI_Gather( sendbuf, sendcount, internal::getMpiType< TS >(), recvbuf, recvcount, internal::getMpiType< TR >(), root, comm ); @@ -1234,6 +1315,9 @@ int MpiWrapper::gather( T const & value, GEOS_ERROR_IF_LT_MSG( destValuesBuffer.size(), size_t( commSize() ), "Receive buffer is not large enough to contain the values to receive." ); #ifdef GEOS_USE_MPI +#ifdef GEOS_USE_MPI_DESYNC_DETECTION + internal::MpiDesyncGuard const mpiDesyncGuard( comm ); +#endif return MPI_Gather( &value, sizeof( T ), internal::getMpiType< uint8_t >(), destValuesBuffer.data(), sizeof( T ), internal::getMpiType< uint8_t >(), root, comm ); @@ -1253,6 +1337,9 @@ int MpiWrapper::gatherv( TS const * const sendbuf, MPI_Comm MPI_PARAM( comm ) ) { #ifdef GEOS_USE_MPI +#ifdef GEOS_USE_MPI_DESYNC_DETECTION + internal::MpiDesyncGuard const mpiDesyncGuard( comm ); +#endif return MPI_Gatherv( sendbuf, sendcount, internal::getMpiType< TS >(), recvbuf, recvcounts, displs, internal::getMpiType< TR >(), root, comm ); @@ -1277,6 +1364,9 @@ int MpiWrapper::scatter( TS const * const sendbuf, MPI_Comm MPI_PARAM( comm )) { #ifdef GEOS_USE_MPI +#ifdef GEOS_USE_MPI_DESYNC_DETECTION + internal::MpiDesyncGuard const mpiDesyncGuard( comm ); +#endif return MPI_Scatter( sendbuf, sendcount, internal::getMpiType< TS >(), recvbuf, recvcount, internal::getMpiType< TR >(), root, comm ); @@ -1301,6 +1391,9 @@ int MpiWrapper::scatterv( TS const * const sendbuf, MPI_Comm MPI_PARAM( comm )) { #ifdef GEOS_USE_MPI +#ifdef GEOS_USE_MPI_DESYNC_DETECTION + internal::MpiDesyncGuard const mpiDesyncGuard( comm ); +#endif return MPI_Scatterv( sendbuf, sendcounts, displs, internal::getMpiType< TS >(), recvbuf, recvcount, internal::getMpiType< TR >(), root, comm ); @@ -1566,6 +1659,9 @@ MpiWrapper::PairType< FIRST, SECOND > MpiWrapper::allReduce( PairType< FIRST, SECOND > const & localPair, MPI_Comm comm ) { #ifdef GEOS_USE_MPI +#ifdef GEOS_USE_MPI_DESYNC_DETECTION + internal::MpiDesyncGuard const mpiDesyncGuard( comm ); +#endif auto const type = internal::getMpiPairType< FIRST, SECOND >(); auto const mpiOp = internal::getMpiPairReductionOp< FIRST, SECOND, OP >(); PairType< FIRST, SECOND > pair{ localPair.first, localPair.second }; diff --git a/src/coreComponents/common/logger/Logger.cpp b/src/coreComponents/common/logger/Logger.cpp index 86611998ebc..5b36036e602 100644 --- a/src/coreComponents/common/logger/Logger.cpp +++ b/src/coreComponents/common/logger/Logger.cpp @@ -19,6 +19,7 @@ // Source includes #include "Logger.hpp" +#include "common/MpiWrapper.hpp" #include "common/Path.hpp" namespace geos @@ -53,7 +54,7 @@ void InitializeLogger( MPI_Comm mpi_comm, const std::string & rankOutputDir ) makeDirsForPath( rankOutputDir ); } - MPI_Barrier( mpi_comm ); + MpiWrapper::barrier( mpi_comm ); std::string outputFilePath = rankOutputDir + "/rank_" + internal::g_rankString + ".out"; internal::g_rankStream = new std::ofstream( outputFilePath ); } diff --git a/src/coreComponents/events/EventManager.cpp b/src/coreComponents/events/EventManager.cpp index 5e24698b008..34b76bbe72d 100644 --- a/src/coreComponents/events/EventManager.cpp +++ b/src/coreComponents/events/EventManager.cpp @@ -169,12 +169,8 @@ bool EventManager::run( DomainPartition & domain ) } m_currentSubEvent = 0; -#ifdef GEOS_USE_MPI // Find the min dt across processes - real64 dt_global; - MPI_Allreduce( &m_dt, &dt_global, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_GEOS ); - m_dt = dt_global; -#endif + m_dt = MpiWrapper::min( m_dt, MPI_COMM_GEOS ); } LogPart logPart( "TIMESTEP", MpiWrapper::commRank() == 0 ); diff --git a/src/coreComponents/events/HaltEvent.cpp b/src/coreComponents/events/HaltEvent.cpp index 93cc9d2d581..95316bbf8c1 100644 --- a/src/coreComponents/events/HaltEvent.cpp +++ b/src/coreComponents/events/HaltEvent.cpp @@ -14,6 +14,7 @@ */ #include "HaltEvent.hpp" +#include "common/MpiWrapper.hpp" #include /** @@ -66,11 +67,7 @@ void HaltEvent::estimateEventTiming( real64 const GEOS_UNUSED_PARAM( time ), // The timing for the ranks may differ slightly, so synchronize // TODO: Only do the communication when you are close to the end? -#ifdef GEOS_USE_MPI - integer forecast_global; - MPI_Allreduce( &forecast, &forecast_global, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD ); - forecast = forecast_global; -#endif + forecast = MpiWrapper::min( forecast, MPI_COMM_WORLD ); setForecast( forecast ); diff --git a/src/coreComponents/events/PeriodicEvent.cpp b/src/coreComponents/events/PeriodicEvent.cpp index 86d6c85bb81..a698fc2eb3b 100644 --- a/src/coreComponents/events/PeriodicEvent.cpp +++ b/src/coreComponents/events/PeriodicEvent.cpp @@ -20,6 +20,7 @@ #include "PeriodicEvent.hpp" #include "common/format/Format.hpp" +#include "common/MpiWrapper.hpp" #include "functions/FunctionManager.hpp" namespace geos @@ -172,31 +173,23 @@ void PeriodicEvent::checkOptionalFunctionThreshold( real64 const time, // Because the function applied to an object may differ by rank, synchronize // (Note: this shouldn't occur very often, since it is only called if the base forecast <= 0) -#ifdef GEOS_USE_MPI - real64 result_global; switch( m_functionStatOption ) { case 0: { - MPI_Allreduce( &result, &result_global, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD ); - result = result_global; + result = MpiWrapper::min( result, MPI_COMM_WORLD ); break; } case 1: { - int nprocs; - MPI_Comm_size( MPI_COMM_WORLD, &nprocs ); - MPI_Allreduce( &result, &result_global, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD ); - result = result_global / nprocs; + result = MpiWrapper::sum( result, MPI_COMM_WORLD ) / MpiWrapper::commSize( MPI_COMM_WORLD ); break; } case 2: { - MPI_Allreduce( &result, &result_global, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD ); - result = result_global; + result = MpiWrapper::max( result, MPI_COMM_WORLD ); } } -#endif } // Forcast event diff --git a/src/coreComponents/fileIO/silo/SiloFile.cpp b/src/coreComponents/fileIO/silo/SiloFile.cpp index ab2b090a2ff..f27c33cc26e 100644 --- a/src/coreComponents/fileIO/silo/SiloFile.cpp +++ b/src/coreComponents/fileIO/silo/SiloFile.cpp @@ -277,13 +277,7 @@ SiloFile::~SiloFile() void SiloFile::makeSiloDirectories() { - - int rank=0; -#ifdef GEOS_USE_MPI - MPI_Comm_rank( MPI_COMM_GEOS, &rank ); -#endif - - if( rank==0 ) + if( MpiWrapper::commRank( MPI_COMM_GEOS ) == 0 ) { makeDirsForPath( joinPath( m_siloDirectory, m_siloDataSubDirectory ) ); } @@ -342,11 +336,7 @@ void SiloFile::waitForBatonWrite( int const domainNumber, integer const eventCounter, bool const isRestart ) { - - int rank = 0; -#ifdef GEOS_USE_MPI - MPI_Comm_rank( MPI_COMM_GEOS, &rank ); -#endif + int const rank = MpiWrapper::commRank( MPI_COMM_GEOS ); int const groupRank = PMPIO_GroupRank( m_baton, rank ); if( isRestart ) @@ -376,11 +366,7 @@ void SiloFile::waitForBatonWrite( int const domainNumber, void SiloFile::waitForBaton( int const domainNumber, string const & restartFileName ) { - - int rank = 0; -#ifdef GEOS_USE_MPI - MPI_Comm_rank( MPI_COMM_GEOS, &rank ); -#endif + int const rank = MpiWrapper::commRank( MPI_COMM_GEOS ); int const groupRank = PMPIO_GroupRank( m_baton, rank ); m_baseFileName = restartFileName; @@ -411,11 +397,7 @@ void SiloFile::handOffBaton() { PMPIO_HandOffBaton( m_baton, m_dbFilePtr ); - int rank = 0; -#ifdef GEOS_USE_MPI - MPI_Comm_rank( MPI_COMM_GEOS, &rank ); -#endif - if( rank==0 ) + if( MpiWrapper::commRank( MPI_COMM_GEOS ) == 0 ) { DBClose( m_dbBaseFilePtr ); } @@ -573,11 +555,7 @@ void SiloFile::writeMeshObject( string const & meshName, } // write multimesh object - int rank = 0; -#ifdef GEOS_USE_MPI - MPI_Comm_rank( MPI_COMM_GEOS, &rank ); -#endif - if( rank == 0 ) + if( MpiWrapper::commRank( MPI_COMM_GEOS ) == 0 ) { DBAddOption( optlist, DBOPT_CYCLE, const_cast< int * >(&cycleNumber)); DBAddOption( optlist, DBOPT_DTIME, const_cast< real64 * >(&problemTime)); @@ -648,11 +626,7 @@ void SiloFile::writeBeamMesh( string const & meshName, //----write multimesh object { - int rank = 0; - #ifdef GEOS_USE_MPI - MPI_Comm_rank( MPI_COMM_GEOS, &rank ); - #endif - if( rank == 0 ) + if( MpiWrapper::commRank( MPI_COMM_GEOS ) == 0 ) { DBAddOption( optlist, DBOPT_CYCLE, const_cast< int * >(&cycleNumber)); DBAddOption( optlist, DBOPT_DTIME, const_cast< real64 * >(&problemTime)); @@ -681,11 +655,7 @@ void SiloFile::writePointMesh( string const & meshName, //----write multimesh object { - int rank = 0; - #ifdef GEOS_USE_MPI - MPI_Comm_rank( MPI_COMM_GEOS, &rank ); - #endif - if( rank == 0 ) + if( MpiWrapper::commRank( MPI_COMM_GEOS ) == 0 ) { writeMultiXXXX( DB_POINTMESH, DBPutMultimesh, 0, meshName.c_str(), cycleNumber, "/", optlist ); } @@ -809,10 +779,7 @@ void SiloFile::writeMaterialMapsFullStorage( ElementRegionBase const & elemRegio DBFreeOptlist( optlist ); } // write multimesh object - int rank = 0; - #ifdef GEOS_USE_MPI - MPI_Comm_rank( MPI_COMM_GEOS, &rank ); - #endif + int const rank = MpiWrapper::commRank( MPI_COMM_GEOS ); if( rank == 0 ) { @@ -1966,11 +1933,7 @@ void SiloFile::writePolygonMeshObject( const string & meshName, } // write multimesh object - int rank = 0; -#ifdef GEOS_USE_MPI - MPI_Comm_rank( MPI_COMM_WORLD, &rank ); -#endif - if( rank == 0 ) + if( MpiWrapper::commRank( MPI_COMM_WORLD ) == 0 ) { DBAddOption( optlist, DBOPT_CYCLE, const_cast< int * >(&cycleNumber)); DBAddOption( optlist, DBOPT_DTIME, const_cast< real64 * >(&problemTime)); @@ -2107,10 +2070,7 @@ void SiloFile::writeMultiXXXX( const DBObjectType type, { (void)centering; - int size = 1; -#ifdef GEOS_USE_MPI - MPI_Comm_size( MPI_COMM_GEOS, &size ); -#endif + int const size = MpiWrapper::commSize( MPI_COMM_GEOS ); string_array vBlockNames( size ); array1d< char * > BlockNames( size ); @@ -2255,11 +2215,7 @@ void SiloFile::writeDataField( string const & meshName, } // write multimesh object - int rank = 0; -#ifdef GEOS_USE_MPI - MPI_Comm_rank( MPI_COMM_GEOS, &rank ); -#endif - if( rank == 0 ) + if( MpiWrapper::commRank( MPI_COMM_GEOS ) == 0 ) { int tensorRank = siloFileUtilities::GetTensorRank< TYPE >(); DBAddOption( optlist, DBOPT_TENSOR_RANK, const_cast< int * >(&tensorRank)); @@ -2524,11 +2480,7 @@ void SiloFile::writeDataField( string const & meshName, } // write multimesh object - int rank = 0; -#ifdef GEOS_USE_MPI - MPI_Comm_rank( MPI_COMM_GEOS, &rank ); -#endif - if( rank == 0 ) + if( MpiWrapper::commRank( MPI_COMM_GEOS ) == 0 ) { // int tensorRank = siloTensorRank; // DBAddOption(optlist, DBOPT_TENSOR_RANK, const_cast (&tensorRank)); @@ -2812,11 +2764,7 @@ void SiloFile::writeMaterialDataField( string const & meshName, } // write multimesh object - int rank = 0; -#ifdef GEOS_USE_MPI - MPI_Comm_rank( MPI_COMM_GEOS, &rank ); -#endif - if( rank == 0 ) + if( MpiWrapper::commRank( MPI_COMM_GEOS ) == 0 ) { int tensorRank = siloFileUtilities::GetTensorRank< TYPE >(); DBAddOption( optlist, DBOPT_TENSOR_RANK, const_cast< int * >(&tensorRank)); diff --git a/src/coreComponents/physicsSolvers/FieldStatisticsBase.hpp b/src/coreComponents/physicsSolvers/FieldStatisticsBase.hpp index 13390fe314c..a7f6e82860e 100644 --- a/src/coreComponents/physicsSolvers/FieldStatisticsBase.hpp +++ b/src/coreComponents/physicsSolvers/FieldStatisticsBase.hpp @@ -99,7 +99,7 @@ class FieldStatisticsBase : public TaskBase makeDirsForPath( m_outputDir ); } // wait till the dir is created by rank 0 - MPI_Barrier( MPI_COMM_WORLD ); + MpiWrapper::barrier( MPI_COMM_WORLD ); } } diff --git a/src/coreComponents/physicsSolvers/fluidFlow/wells/WellSolverBase.cpp b/src/coreComponents/physicsSolvers/fluidFlow/wells/WellSolverBase.cpp index 060ff895e73..06e31b14f66 100644 --- a/src/coreComponents/physicsSolvers/fluidFlow/wells/WellSolverBase.cpp +++ b/src/coreComponents/physicsSolvers/fluidFlow/wells/WellSolverBase.cpp @@ -111,7 +111,7 @@ void WellSolverBase::postInputInitialization() makeDirsForPath( m_ratesOutputDir ); } // wait till the dir is created by rank 0 - MPI_Barrier( MPI_COMM_WORLD ); + MpiWrapper::barrier( MPI_COMM_WORLD ); } } diff --git a/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsMPM.cpp b/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsMPM.cpp index bf1814610e5..9e5689c8c8c 100644 --- a/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsMPM.cpp +++ b/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsMPM.cpp @@ -577,12 +577,12 @@ void SolidMechanicsMPM::initialize( NodeManager & nodeManager, BCTableSize = BCTable1D.size(); } - MPI_Bcast( &BCTableSize, 1, MPI_INT, 0, MPI_COMM_GEOS ); // Broadcast the size of BCTable1D to other processes + MpiWrapper::bcast( &BCTableSize, 1, 0, MPI_COMM_GEOS ); // Broadcast the size of BCTable1D to other processes if( rank != 0 ) // All processes except for root resize their versions of BCTable1D { BCTable1D.resize( BCTableSize ); } - MPI_Bcast( BCTable1D.data(), BCTableSize, MPI_DOUBLE, 0, MPI_COMM_GEOS ); // Broadcast BCTable1D to other processes + MpiWrapper::bcast( BCTable1D.data(), BCTableSize, 0, MPI_COMM_GEOS ); // Broadcast BCTable1D to other processes // Technically don't need to reshape BCTable1D into a 2D array, but it makes things more readable and should have little runtime penalty m_bcTable.resize( BCTableSize / 7, 7 ); // Initialize size of m_BCTable @@ -618,12 +618,12 @@ void SolidMechanicsMPM::initialize( NodeManager & nodeManager, FTableSize = FTable1D.size(); } - MPI_Bcast( &FTableSize, 1, MPI_INT, 0, MPI_COMM_GEOS ); // Broadcast the size of FTable1D to other processes + MpiWrapper::bcast( &FTableSize, 1, 0, MPI_COMM_GEOS ); // Broadcast the size of FTable1D to other processes if( rank != 0 ) // All processes except for root resize their versions of FTable1D { FTable1D.resize( FTableSize ); } - MPI_Bcast( FTable1D.data(), FTableSize, MPI_DOUBLE, 0, MPI_COMM_GEOS ); // Broadcast FTable1D to other processes + MpiWrapper::bcast( FTable1D.data(), FTableSize, 0, MPI_COMM_GEOS ); // Broadcast FTable1D to other processes // Techinically don't need to reshape FTable1D into a 2D array, but it makes things more readable and should have little runtime penalty m_fTable.resize( FTableSize / 4, 4 ); // Initialize size of m_fTable @@ -823,12 +823,7 @@ void SolidMechanicsMPM::initialize( NodeManager & nodeManager, { localMinMass = DBL_MAX; } - MPI_Allreduce( &localMinMass, - &globalMinMass, - 1, - MPI_DOUBLE, - MPI_MIN, - MPI_COMM_GEOS ); + globalMinMass = MpiWrapper::min( localMinMass, MPI_COMM_GEOS ); m_smallMass = fmin( globalMinMass * 1.0e-12, m_smallMass ); // Initialize deformation gradient and velocity gradient @@ -884,12 +879,7 @@ void SolidMechanicsMPM::initialize( NodeManager & nodeManager, } } } ); - MPI_Allreduce( &maxLocalGroupNumber, - &maxGlobalGroupNumber, - 1, - MPI_INT, - MPI_MAX, - MPI_COMM_GEOS ); + maxGlobalGroupNumber = MpiWrapper::max( maxLocalGroupNumber, MPI_COMM_GEOS ); // Number of contact groups m_numContactGroups = maxGlobalGroupNumber + 1; @@ -1465,12 +1455,7 @@ void SolidMechanicsMPM::applyEssentialBCs( const real64 dt, real64 globalFaceReactions[6]; for( int face = 0; face < 6; face++ ) { - MPI_Allreduce( &localFaceReactions[face], - &globalFaceReactions[face], - 1, - MPI_DOUBLE, - MPI_SUM, - MPI_COMM_GEOS ); + globalFaceReactions[face] = MpiWrapper::sum( localFaceReactions[face], MPI_COMM_GEOS ); } // Get end-of-step domain dimensions - note that m_domainExtent is updated later @@ -1956,7 +1941,7 @@ void SolidMechanicsMPM::solverProfiling( std::string label ) { if( m_solverProfiling >= 1 ) { - MPI_Barrier( MPI_COMM_GEOS ); + MpiWrapper::barrier( MPI_COMM_GEOS ); GEOS_LOG_RANK_IF( m_solverProfiling == 2, label ); m_profilingTimes.push_back( MPI_Wtime() ); m_profilingLabels.push_back( label ); @@ -2195,18 +2180,8 @@ void SolidMechanicsMPM::optimizeBinSort( ParticleManager & particleManager ) real64 globalWeightedMultiplier; int localNumberOfParticles = particleManager.getNumberOfParticles(); real64 localWeightedMultiplier = optimalMultiplier * localNumberOfParticles; - MPI_Allreduce( &localWeightedMultiplier, - &globalWeightedMultiplier, - 1, - MPI_DOUBLE, - MPI_SUM, - MPI_COMM_GEOS ); - MPI_Allreduce( &localNumberOfParticles, - &globalNumberOfParticles, - 1, - MPI_INT, - MPI_SUM, - MPI_COMM_GEOS ); + globalWeightedMultiplier = MpiWrapper::sum( localWeightedMultiplier, MPI_COMM_GEOS ); + globalNumberOfParticles = MpiWrapper::sum( localNumberOfParticles, MPI_COMM_GEOS ); // Set bin size multiplier m_binSizeMultiplier = std::max( (int) std::round( globalWeightedMultiplier / globalNumberOfParticles ), 1 ); @@ -2752,20 +2727,10 @@ void SolidMechanicsMPM::computeAndWriteBoxAverage( const real64 dt, // so file is directly plottable in excel as CSV or something. for( localIndex i = 0; i < 9; i++ ) { - real64 localSum = boxSums[i]; - real64 globalSum; - MPI_Allreduce( &localSum, - &globalSum, - 1, - MPI_DOUBLE, - MPI_SUM, - MPI_COMM_GEOS ); - boxSums[i] = globalSum; + boxSums[i] = MpiWrapper::sum( boxSums[i], MPI_COMM_GEOS ); } - int rank; - MPI_Comm_rank( MPI_COMM_GEOS, &rank ); - if( rank == 0 ) + if( MpiWrapper::commRank( MPI_COMM_GEOS ) == 0 ) { // Calculate the box volume real64 boxVolume = m_domainExtent[0] * m_domainExtent[1] * m_domainExtent[2]; @@ -3332,7 +3297,7 @@ void SolidMechanicsMPM::printProfilingResults() } // Print out solver profiling - MPI_Barrier( MPI_COMM_GEOS ); + MpiWrapper::barrier( MPI_COMM_GEOS ); if( rank == 0 ) { std::cout << "---------------------------------------------" << std::endl; diff --git a/src/coreComponents/schema/schema.xsd.other b/src/coreComponents/schema/schema.xsd.other index c78f1e527f3..cc2505e312b 100644 --- a/src/coreComponents/schema/schema.xsd.other +++ b/src/coreComponents/schema/schema.xsd.other @@ -526,7 +526,7 @@ A field can represent a physical variable. (pressure, temperature, global compos - + @@ -1603,7 +1603,7 @@ A field can represent a physical variable. (pressure, temperature, global compos - + @@ -2437,11 +2437,11 @@ A field can represent a physical variable. (pressure, temperature, global compos - + - + diff --git a/src/docs/doxygen/GeosxConfig.hpp b/src/docs/doxygen/GeosxConfig.hpp index 424b6923190..0ad6cd12f85 100644 --- a/src/docs/doxygen/GeosxConfig.hpp +++ b/src/docs/doxygen/GeosxConfig.hpp @@ -44,6 +44,9 @@ /// Enables use of MPI (CMake option ENABLE_MPI) #define GEOS_USE_MPI +/// Enables detection of MPI desynchronization (CMake option ENABLE_MPI_DESYNC_DETECTION) +#define GEOS_USE_MPI_DESYNC_DETECTION + /// Enables use of OpenMP (CMake option ENABLE_OPENMP) #define GEOS_USE_OPENMP