AMP_MPI.h

Go to the documentation of this file.

// This file includes a wrapper class for MPI functions
#ifndef included_AMP_MPI
#define included_AMP_MPI
 
 
#include <any>
#include <array>
#include <atomic>
#include <map>
#include <memory>
#include <random>
#include <set>
#include <string>
#include <tuple>
#include <vector>
 
 
// Add the definitions for the TPLs that are used
#include "AMP/AMP_TPLs.h"
 
// Include MPI if we are building with MPI
#include "AMP/utils/UtilityMacros.h"
#ifdef AMP_USE_MPI
DISABLE_WARNINGS
    #include "mpi.h"
ENABLE_WARNINGS
#endif
 
// Define extra comm_world, comm_self, and comm_null ids
#define AMP_COMM_NULL AMP::AMP_MPI::commNull
#define AMP_COMM_SELF AMP::AMP_MPI::commSelf
#define AMP_COMM_WORLD AMP::AMP_MPI::commWorld
 
 
// Define SAMRAI MPI object
#ifdef AMP_USE_SAMRAI
namespace SAMRAI::tbox {
class SAMRAI_MPI;
}
#endif
 
 
namespace AMP {
 
 
class alignas( 8 ) AMP_MPI final
{
public:
    enum class ThreadSupport : int { SINGLE, FUNNELED, SERIALIZED, MULTIPLE };
 
#ifdef AMP_USE_MPI
    typedef MPI_Comm Comm;
    typedef MPI_Datatype Datatype;
    typedef MPI_Request Request2;
    constexpr static bool has_MPI = true;
#else
    typedef uint32_t Comm;
    typedef uint32_t Datatype;
    typedef uint32_t Request2;
    constexpr static bool has_MPI = false;
#endif
    static Comm commNull;
    static Comm commSelf;
    static Comm commWorld;
 
    constexpr static uint64_t hashNull  = 0xcc6bc5507c132516;
    constexpr static uint64_t hashSelf  = 0x070b9699a107fe57;
    constexpr static uint64_t hashWorld = 0x3d5fdf58e4df5a94;
    constexpr static uint64_t hashMPI   = 0x641118b35a0d87cd;
 
    class Request final
    {
    public:
        Request( Request2 request = Request2(), std::any data = std::any() );
        ~Request();
        operator Request2() const { return d_data->first; }
        Request2 *get() { return &d_data->first; }
 
    private:
        std::shared_ptr<std::pair<Request2, std::any>> d_data;
    };
 
 
public: // Constructors
    AMP_MPI();
 
 
    ~AMP_MPI();
 
 
    AMP_MPI( Comm comm, bool manage = false );
 
 
    AMP_MPI( const AMP_MPI &comm );
 
 
    AMP_MPI( AMP_MPI &&rhs );
 
 
    AMP_MPI &operator=( const AMP_MPI &comm );
 
 
    AMP_MPI &operator=( AMP_MPI &&rhs );
 
 
    void reset();
 
 
public: // Member functions
    static std::string getNodeName();
 
 
    static int getNumberOfProcessors();
 
 
    static std::vector<int> getProcessAffinity();
 
 
    static void setProcessAffinity( const std::vector<int> &procs );
 
 
    static void balanceProcesses( const AMP_MPI &comm           = AMP_MPI( AMP_COMM_WORLD ),
                                  int method                    = 1,
                                  const std::vector<int> &procs = std::vector<int>(),
                                  int N_min                     = 1,
                                  int N_max                     = -1 );
 
 
    static ThreadSupport queryThreadSupport();
 
 
    size_t rand() const;
 
 
    AMP_MPI split( int color, int key = -1, bool manage = true ) const;
 
 
    AMP_MPI splitByNode( int key = -1, bool manage = true ) const;
 
 
    AMP_MPI dup( bool manage = true ) const;
 
 
    static AMP_MPI intersect( const AMP_MPI &comm1, const AMP_MPI &comm2 );
 
 
    bool isNull() const;
 
 
    std::vector<int> globalRanks() const;
 
 
    uint64_t hash() const;
 
 
    uint64_t hashRanks() const;
 
 
    const Comm &getCommunicator() const;
 
 
    bool operator==( const AMP_MPI & ) const;
 
 
    bool operator!=( const AMP_MPI & ) const;
 
 
    bool operator<( const AMP_MPI & ) const;
 
 
    bool operator<=( const AMP_MPI & ) const;
 
 
    bool operator>( const AMP_MPI & ) const;
 
 
    bool operator>=( const AMP_MPI & ) const;
 
 
    int compare( const AMP_MPI & ) const;
 
 
    int getRank() const;
 
 
    int getSize() const;
 
 
    int maxTag() const;
 
 
    int newTag() const;
 
 
    void abort() const;
 
 
    void setCallAbortInSerialInsteadOfExit( bool flag = true );
 
 
    bool allReduce( const bool value ) const;
 
 
    bool anyReduce( const bool value ) const;
 
 
    void allReduce( std::vector<bool> &value ) const;
 
 
    void anyReduce( std::vector<bool> &value ) const;
 
 
    template<class type>
    type sumReduce( const type &value ) const;
 
 
    template<class type>
    void sumReduce( type *x, int n = 1 ) const;
 
 
    template<class type>
    void sumReduce( const type *x, type *y, int n = 1 ) const;
 
 
    template<class type>
    type minReduce( const type &value ) const;
 
 
    template<class type>
    void minReduce( type *x, int n ) const;
 
 
    template<class type>
    void minReduce( const type *x, type *y, int n ) const;
 
    template<class type>
    void minReduce( type *x, int n, int *rank ) const;
 
 
    template<class type>
    void minReduce( const type *x, type *y, int n, int *rank ) const;
 
 
    template<class type>
    type maxReduce( const type &value ) const;
 
 
    template<class type>
    void maxReduce( type *x, int n ) const;
 
 
    template<class type>
    void maxReduce( const type *x, type *y, int n ) const;
 
    template<class type>
    void maxReduce( type *x, int n, int *rank ) const;
 
 
    template<class type>
    void maxReduce( const type *x, type *y, int n, int *rank ) const;
 
 
    template<class type>
    type sumScan( const type &x ) const;
 
 
    template<class type>
    void sumScan( const type *x, type *y, int n ) const;
 
 
    template<class type>
    type minScan( const type &x ) const;
 
 
    template<class type>
    void minScan( const type *x, type *y, int n ) const;
 
 
    template<class type>
    type maxScan( const type &x ) const;
 
 
    template<class type>
    void maxScan( const type *x, type *y, int n ) const;
 
 
    template<class type>
    type bcast( const type &value, int root ) const;
 
 
    template<class type>
    void bcast( type *value, int n, int root ) const;
 
 
    void barrier() const;
 
 
    void sleepBarrier( int ms = 10 ) const;
 
 
    template<class type>
    void send( const type &data, int recv, int tag = 0 ) const;
 
 
    template<class type>
    void send( const type *buf, int length, int recv, int tag = 0 ) const;
 
 
    void sendBytes( const void *buf, int N_bytes, int recv, int tag = 0 ) const;
 
 
    template<class type>
    Request Isend( const type &data, int recv_proc, int tag ) const;
 
 
    template<class type>
    Request Isend( const type *buf, int length, int recv_proc, int tag ) const;
 
 
    Request IsendBytes( const void *buf, int N_bytes, int recv_proc, int tag ) const;
 
 
    template<class type>
    type recv( int send, int tag = 0 ) const;
 
 
    template<class type>
    void recv( type *buf, int length, int send, int tag = 0 ) const;
 
 
    template<class type>
    void recv( type *buf, int &length, int send, bool get_length, int tag = 0 ) const;
 
 
    void recvBytes( void *buf, int N_bytes, int send, int tag = 0 ) const;
 
 
    template<class type>
    Request Irecv( type &data, int send_proc, int tag ) const;
 
 
    template<class type>
    Request Irecv( type *buf, int length, int send_proc, int tag ) const;
 
 
    Request IrecvBytes( void *buf, int N_bytes, int send_proc, int tag ) const;
 
 
    template<class type>
    void sendrecv( const type *sendbuf,
                   int sendcount,
                   int dest,
                   int sendtag,
                   type *recvbuf,
                   int recvcount,
                   int source,
                   int recvtag ) const;
 
 
    template<class type>
    std::vector<type> allGather( const type &x ) const;
 
 
    template<class type>
    std::vector<type> allGather( const std::vector<type> &x ) const;
 
 
    template<class type>
    void allGather( const type &x_in, type *x_out ) const;
 
 
    template<class type>
    int allGather( const type *send_data,
                   int send_cnt,
                   type *recv_data,
                   int *recv_cnt   = nullptr,
                   int *recv_disp  = nullptr,
                   bool known_recv = false ) const;
 
 
    template<class type>
    void setGather( std::set<type> &set ) const;
 
 
    template<class KEY, class DATA>
    void mapGather( std::map<KEY, DATA> &map ) const;
 
 
    template<class type>
    std::vector<type> gather( const type &x, int root ) const;
 
 
    template<class type>
    std::vector<type> gather( const std::vector<type> &x, int root ) const;
 
 
    template<class type>
    void gather( const type *send_data,
                 int send_cnt,
                 type *recv_data,
                 const int *recv_cnt,
                 const int *recv_disp,
                 int root ) const;
 
    template<class type>
    void allToAll( int n, const type *send_data, type *recv_data ) const;
 
    template<class type>
    std::vector<type> allToAll( const std::vector<type> &send ) const;
 
 
    template<class type>
    int allToAll( const type *send_data,
                  const int send_cnt[],
                  const int send_disp[],
                  type *recv_data,
                  int *recv_cnt   = nullptr,
                  int *recv_disp  = nullptr,
                  bool known_recv = false ) const;
 
 
    template<class type, class int_type>
    std::vector<type> allToAll( const std::vector<type> &send_data,
                                const std::vector<int_type> &send_cnt,
                                const std::vector<int_type> &send_disp,
                                const std::vector<int_type> &recv_cnt,
                                const std::vector<int_type> &recv_disp ) const;
 
 
    int
    calcAllToAllDisp( const int *send_cnt, int *send_disp, int *recv_cnt, int *recv_disp ) const;
 
 
    int calcAllToAllDisp( const std::vector<int> &send_cnt,
                          std::vector<int> &send_disp,
                          std::vector<int> &recv_cnt,
                          std::vector<int> &recv_disp ) const;
 
 
    std::vector<int> commRanks( const std::vector<int> &ranks ) const;
 
 
    static void wait( const Request &request );
 
 
    static void wait( Request2 request );
 
 
    static int waitAny( int count, const Request *request );
 
 
    static int waitAny( int count, Request2 *request );
 
 
    static void waitAll( int count, const Request *request );
 
    static void waitAll( int count, Request2 *request );
 
 
    static std::vector<int> waitSome( int count, const Request *request );
 
 
    static std::vector<int> waitSome( int count, Request2 *request );
 
 
    std::tuple<int, int, int> Iprobe( int source = -1, int tag = -1 ) const;
 
 
    std::tuple<int, int, int> probe( int source = -1, int tag = -1 ) const;
 
 
    void serializeStart() const;
 
 
    void serializeStop() const;
 
 
    static double time();
 
 
    static double tick();
 
 
    static void changeProfileLevel( int level ) { profile_level = level; }
 
 
    static size_t MPI_Comm_created();
 
    static size_t MPI_Comm_destroyed();
 
    static std::string info();
 
    static std::array<int, 2> version();
 
    static bool MPI_Active();
 
    static void start_MPI( int &argc, char *argv[], int profile_level = 0 );
 
    static void stop_MPI();
 
 
#ifdef AMP_USE_SAMRAI
public: // SAMRAI interfaces
    AMP_MPI( const SAMRAI::tbox::SAMRAI_MPI &comm );
 
    operator SAMRAI::tbox::SAMRAI_MPI() const;
#endif
 
 
private: // Helper functions
    std::mt19937_64 *getRand() const;
 
private: // data members
    using atomic_ptr = std::atomic_int *volatile;
    using atomic_int = volatile std::atomic_int64_t;
    using int_ptr    = int *volatile;
    using rand_ptr   = std::mt19937_64 *volatile;
 
    Comm d_comm             = commNull;     
    bool d_isNull           = true;         
    bool d_manage           = false;        
    bool d_call_abort       = true;         
    int d_rank              = 0;            
    int d_size              = 1;            
    uint64_t d_hash         = hashNull;     
    int_ptr d_currentTag    = nullptr;      
    mutable int_ptr d_ranks = nullptr;      
    atomic_ptr d_count      = nullptr;      
    mutable rand_ptr d_rand = nullptr;      
    static short profile_level;             
    static int d_maxTag;                    
    static atomic_int N_MPI_Comm_created;   
    static atomic_int N_MPI_Comm_destroyed; 
};
 
 
template<class TYPE>
AMP_MPI getComm( const TYPE &obj );
 
 
} // namespace AMP
 
 
#endif