LCOV - code coverage report
Current view: top level - src/mpiwrap - message_passing.F (source / functions) Coverage Total Hit
Test: CP2K Regtests (git:dbd702d) Lines: 62.4 % 1113 694
Test Date: 2026-11-22 01:17:58 Functions: 51.3 % 152 78

            Line data    Source code
       1              : !--------------------------------------------------------------------------------------------------!
       2              : !   CP2K: A general program to perform molecular dynamics simulations                              !
       3              : !   Copyright 2000-2026 CP2K developers group <https://cp2k.org>                                   !
       4              : !                                                                                                  !
       5              : !   SPDX-License-Identifier: GPL-2.0-or-later                                                      !
       6              : !--------------------------------------------------------------------------------------------------!
       7              : 
       8              : ! **************************************************************************************************
       9              : !> \brief Interface to the message passing library MPI
      10              : !> \par History
      11              : !>      JGH (02-Jan-2001): New error handling
      12              : !>                         Performance tools
      13              : !>      JGH (14-Jan-2001): New routines mp_comm_compare, mp_cart_coords,
      14              : !>                                      mp_rank_compare, mp_alltoall
      15              : !>      JGH (06-Feb-2001): New routines mp_comm_free
      16              : !>      JGH (22-Mar-2001): New routines mp_comm_dup
      17              : !>      fawzi (04-NOV-2004): storable performance info (for f77 interface)
      18              : !>      Wrapper routine for mpi_gatherv added (22.12.2005,MK)
      19              : !>      JGH (13-Feb-2006): Flexible precision
      20              : !>      JGH (15-Feb-2006): single precision mp_alltoall
      21              : !> \note Concurrent calls must use thread-private buffers and requests. Concurrent collectives,
      22              : !>       or point-to-point calls that do not use distinct tags, must use distinct communicators.
      23              : !>       Communicator creation, duplication, splitting, ownership changes, and destruction must
      24              : !>       happen outside OpenMP parallel regions.
      25              : !> \author JGH
      26              : ! **************************************************************************************************
      27              : MODULE message_passing
      28              :    USE ISO_C_BINDING, ONLY: C_F_POINTER, C_PTR
      29              :    USE kinds, ONLY: &
      30              :       dp, int_4, int_4_size, int_8, int_8_size, real_4, real_4_size, real_8, &
      31              :       real_8_size, default_string_length
      32              :    USE machine, ONLY: m_abort, m_memory, m_memory_details
      33              :    USE mp_perf_env, ONLY: add_perf, add_mp_perf_env, rm_mp_perf_env
      34              :    USE OMP_LIB, ONLY: omp_get_level, omp_get_thread_num, omp_in_parallel
      35              : #if defined(__MIMIC)
      36              :    USE mcl, ONLY: mcl_initialize, mcl_is_initialized, mcl_abort
      37              : #endif
      38              : 
      39              : #include "../base/base_uses.f90"
      40              : 
      41              : ! To simplify the transition between the old MPI module and the F08-style module,
      42              : ! we introduce these constants to switch between the required handle types
      43              : ! Unfortunately, Fortran does not offer something like typedef in C++
      44              : #if defined(__parallel) && defined(__MPI_F08)
      45              : #define MPI_DATA_TYPE TYPE(MPI_Datatype)
      46              : #define MPI_COMM_TYPE TYPE(MPI_Comm)
      47              : #define MPI_REQUEST_TYPE TYPE(MPI_Request)
      48              : #define MPI_WIN_TYPE TYPE(MPI_Win)
      49              : #define MPI_FILE_TYPE TYPE(MPI_File)
      50              : #define MPI_INFO_TYPE TYPE(MPI_Info)
      51              : #define MPI_STATUS_TYPE TYPE(MPI_Status)
      52              : #define MPI_GROUP_TYPE TYPE(MPI_Group)
      53              : #define MPI_STATUS_EXTRACT(X) %X
      54              : #define MPI_GET_COMP %mpi_val
      55              : #else
      56              : #define MPI_DATA_TYPE INTEGER
      57              : #define MPI_COMM_TYPE INTEGER
      58              : #define MPI_REQUEST_TYPE INTEGER
      59              : #define MPI_WIN_TYPE INTEGER
      60              : #define MPI_FILE_TYPE INTEGER
      61              : #define MPI_INFO_TYPE INTEGER
      62              : #define MPI_STATUS_TYPE INTEGER, DIMENSION(MPI_STATUS_SIZE)
      63              : #define MPI_GROUP_TYPE INTEGER
      64              : #define MPI_STATUS_EXTRACT(X) (X)
      65              : #define MPI_GET_COMP
      66              : #endif
      67              : 
      68              : #if defined(__parallel)
      69              : ! subroutines: unfortunately, mpi implementations do not provide interfaces for all subroutines
      70              : !              (problems with types and ranks explosion),
      71              : !              we do not quite know what is in the module, so we can not include any....
      72              : !              to nevertheless get checking for what is included, we use the mpi module
      73              : !              without use clause, getting all there is
      74              : #if defined(__MPI_F08)
      75              :    USE mpi_f08
      76              : #else
      77              :    USE mpi
      78              : #endif
      79              : #endif
      80              : 
      81              :    IMPLICIT NONE
      82              :    PRIVATE
      83              : 
      84              :    ! parameters that might be needed
      85              : #if defined(__parallel)
      86              :    LOGICAL, PARAMETER :: cp2k_is_parallel = .TRUE.
      87              :    INTEGER, PARAMETER, PUBLIC :: mp_any_tag = MPI_ANY_TAG
      88              :    INTEGER, PARAMETER, PUBLIC :: mp_any_source = MPI_ANY_SOURCE
      89              :    MPI_COMM_TYPE, PARAMETER :: mp_comm_null_handle = MPI_COMM_NULL
      90              :    MPI_COMM_TYPE, PARAMETER :: mp_comm_self_handle = MPI_COMM_SELF
      91              :    MPI_COMM_TYPE, PARAMETER :: mp_comm_world_handle = MPI_COMM_WORLD
      92              :    MPI_REQUEST_TYPE, PARAMETER :: mp_request_null_handle = MPI_REQUEST_NULL
      93              :    MPI_WIN_TYPE, PARAMETER :: mp_win_null_handle = MPI_WIN_NULL
      94              :    MPI_FILE_TYPE, PARAMETER :: mp_file_null_handle = MPI_FILE_NULL
      95              :    MPI_INFO_TYPE, PARAMETER :: mp_info_null_handle = MPI_INFO_NULL
      96              :    MPI_DATA_TYPE, PARAMETER :: mp_datatype_null_handle = MPI_DATATYPE_NULL
      97              :    INTEGER, PARAMETER, PRIVATE :: mp_comm_split_type_shared_id = MPI_COMM_TYPE_SHARED
      98              :    INTEGER, PARAMETER, PUBLIC :: mp_status_size = MPI_STATUS_SIZE
      99              :    INTEGER, PARAMETER, PUBLIC :: mp_proc_null = MPI_PROC_NULL
     100              :    ! Set max allocatable memory by MPI to 2 GiByte
     101              :    INTEGER(KIND=MPI_ADDRESS_KIND), PARAMETER, PRIVATE :: mp_max_memory_size = HUGE(INT(1, KIND=int_4))
     102              : 
     103              :    INTEGER, PARAMETER, PUBLIC :: mp_max_library_version_string = MPI_MAX_LIBRARY_VERSION_STRING
     104              : 
     105              :    INTEGER, PARAMETER, PUBLIC :: file_offset = MPI_OFFSET_KIND
     106              :    INTEGER, PARAMETER, PUBLIC :: address_kind = MPI_ADDRESS_KIND
     107              :    INTEGER, PARAMETER, PUBLIC :: file_amode_create = MPI_MODE_CREATE
     108              :    INTEGER, PARAMETER, PUBLIC :: file_amode_rdonly = MPI_MODE_RDONLY
     109              :    INTEGER, PARAMETER, PUBLIC :: file_amode_wronly = MPI_MODE_WRONLY
     110              :    INTEGER, PARAMETER, PUBLIC :: file_amode_rdwr = MPI_MODE_RDWR
     111              :    INTEGER, PARAMETER, PUBLIC :: file_amode_excl = MPI_MODE_EXCL
     112              :    INTEGER, PARAMETER, PUBLIC :: file_amode_append = MPI_MODE_APPEND
     113              : #else
     114              :    LOGICAL, PARAMETER :: cp2k_is_parallel = .FALSE.
     115              :    INTEGER, PARAMETER, PUBLIC :: mp_any_tag = -1
     116              :    INTEGER, PARAMETER, PUBLIC :: mp_any_source = -2
     117              :    MPI_COMM_TYPE, PARAMETER :: mp_comm_null_handle = -3
     118              :    MPI_COMM_TYPE, PARAMETER :: mp_comm_self_handle = -11
     119              :    MPI_COMM_TYPE, PARAMETER :: mp_comm_world_handle = -12
     120              :    MPI_REQUEST_TYPE, PARAMETER :: mp_request_null_handle = -4
     121              :    MPI_WIN_TYPE, PARAMETER :: mp_win_null_handle = -5
     122              :    MPI_FILE_TYPE, PARAMETER :: mp_file_null_handle = -6
     123              :    MPI_INFO_TYPE, PARAMETER :: mp_info_null_handle = -7
     124              :    MPI_DATA_TYPE, PARAMETER :: mp_datatype_null_handle = -8
     125              :    INTEGER, PARAMETER, PRIVATE :: mp_comm_split_type_shared_id = -13
     126              :    INTEGER, PARAMETER, PUBLIC :: mp_status_size = -9
     127              :    INTEGER, PARAMETER, PUBLIC :: mp_proc_null = -10
     128              :    INTEGER, PARAMETER, PUBLIC :: mp_max_library_version_string = 1
     129              : 
     130              :    INTEGER, PARAMETER, PUBLIC :: file_offset = int_8
     131              :    INTEGER, PARAMETER, PUBLIC :: address_kind = int_8
     132              :    INTEGER, PARAMETER, PUBLIC :: file_amode_create = 1
     133              :    INTEGER, PARAMETER, PUBLIC :: file_amode_rdonly = 2
     134              :    INTEGER, PARAMETER, PUBLIC :: file_amode_wronly = 4
     135              :    INTEGER, PARAMETER, PUBLIC :: file_amode_rdwr = 8
     136              :    INTEGER, PARAMETER, PUBLIC :: file_amode_excl = 64
     137              :    INTEGER, PARAMETER, PUBLIC :: file_amode_append = 128
     138              : #endif
     139              : 
     140              :    ! we need to fix this to a given number (crossing fingers)
     141              :    ! so that the serial code using Fortran stream IO and the MPI have the same sizes.
     142              :    INTEGER, PARAMETER, PUBLIC :: mpi_character_size = 1
     143              :    INTEGER, PARAMETER, PUBLIC :: mpi_integer_size = 4
     144              : 
     145              :    CHARACTER(LEN=*), PARAMETER, PRIVATE :: moduleN = 'message_passing'
     146              : 
     147              :    ! internal reference counter used to debug communicator leaks
     148              :    INTEGER, PRIVATE, SAVE :: debug_comm_count
     149              : 
     150              :    ! Thread-support level provided by MPI. A negative value means that MPI has
     151              :    ! not been initialized by CP2K or queried from an embedding application yet.
     152              :    INTEGER, PRIVATE, SAVE :: mp_thread_level_provided = -1
     153              : 
     154              :    ! MPI can be initialized by CP2K or by an embedding application. Only the
     155              :    ! component that initialized MPI is allowed to finalize it.
     156              : #if defined(__parallel)
     157              :    LOGICAL, PRIVATE, SAVE :: mp_mpi_initialized_by_cp2k = .FALSE.
     158              : #endif
     159              :    LOGICAL, PRIVATE, SAVE :: mp_world_is_initialized = .FALSE.
     160              : 
     161              :    PUBLIC :: mp_comm_type
     162              :    PUBLIC :: mp_request_type
     163              :    PUBLIC :: mp_win_type
     164              :    PUBLIC :: mp_file_type
     165              :    PUBLIC :: mp_info_type
     166              :    PUBLIC :: mp_split_type
     167              :    PUBLIC :: mp_cart_type
     168              : 
     169              :    PUBLIC :: mp_para_env_type, mp_para_env_p_type, mp_para_cart_type
     170              :    PUBLIC :: mp_para_env_create, mp_para_env_release, &
     171              :              mp_para_cart_create, mp_para_cart_release
     172              :    PUBLIC :: mp_mem_avail_per_rank_GB, mp_mem_used_per_rank_GB, mp_print_mem_per_rank
     173              : 
     174              : #if defined(__MIMIC)
     175              :    ! Stores the split world communicator to finalize a MiMiC run
     176              :    MPI_COMM_TYPE, PRIVATE, SAVE :: mimic_comm_world
     177              : #endif
     178              : 
     179              :    TYPE mp_comm_type
     180              :       PRIVATE
     181              :       MPI_COMM_TYPE :: handle = mp_comm_null_handle
     182              :       ! Number of dimensions within a Cartesian topology (useful with mp_cart_type)
     183              :       INTEGER :: ndims = 1
     184              :       ! Meta data to the communicator
     185              :       INTEGER, PUBLIC :: mepos = -1, source = -1, num_pe = -1
     186              :    CONTAINS
     187              :       ! Setters/Getters
     188              :       PROCEDURE, PASS, NON_OVERRIDABLE :: set_handle => mp_comm_type_set_handle
     189              :       PROCEDURE, PASS, NON_OVERRIDABLE :: get_handle => mp_comm_type_get_handle
     190              :       ! Comparisons
     191              :       PROCEDURE, PRIVATE, PASS, NON_OVERRIDABLE :: mp_comm_op_eq
     192              :       PROCEDURE, PRIVATE, PASS, NON_OVERRIDABLE :: mp_comm_op_neq
     193              :       GENERIC, PUBLIC :: operator(==) => mp_comm_op_eq
     194              :       GENERIC, PUBLIC :: operator(/=) => mp_comm_op_neq
     195              :       ! Communication routines
     196              :       PROCEDURE, PRIVATE, PASS(comm), NON_OVERRIDABLE :: &
     197              :          mp_sendrecv_i, mp_sendrecv_l, mp_sendrecv_r, mp_sendrecv_d, &
     198              :          mp_sendrecv_c, mp_sendrecv_z, &
     199              :          mp_sendrecv_iv, mp_sendrecv_im2, mp_sendrecv_im3, mp_sendrecv_im4, &
     200              :          mp_sendrecv_lv, mp_sendrecv_lm2, mp_sendrecv_lm3, mp_sendrecv_lm4, &
     201              :          mp_sendrecv_rv, mp_sendrecv_rm2, mp_sendrecv_rm3, mp_sendrecv_rm4, &
     202              :          mp_sendrecv_dv, mp_sendrecv_dm2, mp_sendrecv_dm3, mp_sendrecv_dm4, &
     203              :          mp_sendrecv_cv, mp_sendrecv_cm2, mp_sendrecv_cm3, mp_sendrecv_cm4, &
     204              :          mp_sendrecv_zv, mp_sendrecv_zm2, mp_sendrecv_zm3, mp_sendrecv_zm4
     205              :       GENERIC, PUBLIC :: sendrecv => mp_sendrecv_i, mp_sendrecv_l, &
     206              :          mp_sendrecv_r, mp_sendrecv_d, mp_sendrecv_c, mp_sendrecv_z, &
     207              :          mp_sendrecv_iv, mp_sendrecv_im2, mp_sendrecv_im3, mp_sendrecv_im4, &
     208              :          mp_sendrecv_lv, mp_sendrecv_lm2, mp_sendrecv_lm3, mp_sendrecv_lm4, &
     209              :          mp_sendrecv_rv, mp_sendrecv_rm2, mp_sendrecv_rm3, mp_sendrecv_rm4, &
     210              :          mp_sendrecv_dv, mp_sendrecv_dm2, mp_sendrecv_dm3, mp_sendrecv_dm4, &
     211              :          mp_sendrecv_cv, mp_sendrecv_cm2, mp_sendrecv_cm3, mp_sendrecv_cm4, &
     212              :          mp_sendrecv_zv, mp_sendrecv_zm2, mp_sendrecv_zm3, mp_sendrecv_zm4
     213              : 
     214              :       PROCEDURE, PRIVATE, PASS(comm), NON_OVERRIDABLE :: mp_minloc_iv, &
     215              :          mp_minloc_lv, mp_minloc_rv, mp_minloc_dv
     216              :       GENERIC, PUBLIC :: minloc => mp_minloc_iv, &
     217              :          mp_minloc_lv, mp_minloc_rv, mp_minloc_dv
     218              : 
     219              :       PROCEDURE, PRIVATE, PASS(comm), NON_OVERRIDABLE :: mp_maxloc_iv, &
     220              :          mp_maxloc_lv, mp_maxloc_rv, mp_maxloc_dv
     221              :       GENERIC, PUBLIC :: maxloc => mp_maxloc_iv, &
     222              :          mp_maxloc_lv, mp_maxloc_rv, mp_maxloc_dv
     223              : 
     224              :       PROCEDURE, PRIVATE, PASS(comm), NON_OVERRIDABLE :: mp_shift_im, mp_shift_i, &
     225              :          mp_shift_lm, mp_shift_l, mp_shift_rm, mp_shift_r, &
     226              :          mp_shift_dm, mp_shift_d, mp_shift_cm, mp_shift_c, &
     227              :          mp_shift_zm, mp_shift_z
     228              :       GENERIC, PUBLIC :: shift => mp_shift_im, mp_shift_i, &
     229              :          mp_shift_lm, mp_shift_l, mp_shift_rm, mp_shift_r, &
     230              :          mp_shift_dm, mp_shift_d, mp_shift_cm, mp_shift_c, &
     231              :          mp_shift_zm, mp_shift_z
     232              : 
     233              :       PROCEDURE, PRIVATE, PASS(comm), NON_OVERRIDABLE :: mp_bcast_i, mp_bcast_iv, mp_bcast_im, mp_bcast_i3, &
     234              :          mp_bcast_l, mp_bcast_lv, mp_bcast_lm, mp_bcast_l3, &
     235              :          mp_bcast_r, mp_bcast_rv, mp_bcast_rm, mp_bcast_r3, &
     236              :          mp_bcast_d, mp_bcast_dv, mp_bcast_dm, mp_bcast_d3, &
     237              :          mp_bcast_c, mp_bcast_cv, mp_bcast_cm, mp_bcast_c3, &
     238              :          mp_bcast_z, mp_bcast_zv, mp_bcast_zm, mp_bcast_z3, &
     239              :          mp_bcast_b, mp_bcast_bv, mp_bcast_av, mp_bcast_am, &
     240              :          mp_bcast_i_src, mp_bcast_iv_src, mp_bcast_im_src, mp_bcast_i3_src, &
     241              :          mp_bcast_l_src, mp_bcast_lv_src, mp_bcast_lm_src, mp_bcast_l3_src, &
     242              :          mp_bcast_r_src, mp_bcast_rv_src, mp_bcast_rm_src, mp_bcast_r3_src, &
     243              :          mp_bcast_d_src, mp_bcast_dv_src, mp_bcast_dm_src, mp_bcast_d3_src, &
     244              :          mp_bcast_c_src, mp_bcast_cv_src, mp_bcast_cm_src, mp_bcast_c3_src, &
     245              :          mp_bcast_z_src, mp_bcast_zv_src, mp_bcast_zm_src, mp_bcast_z3_src, &
     246              :          mp_bcast_b_src, mp_bcast_bv_src, mp_bcast_av_src, mp_bcast_am_src
     247              :       GENERIC, PUBLIC :: bcast => mp_bcast_i, mp_bcast_iv, mp_bcast_im, mp_bcast_i3, &
     248              :          mp_bcast_l, mp_bcast_lv, mp_bcast_lm, mp_bcast_l3, &
     249              :          mp_bcast_r, mp_bcast_rv, mp_bcast_rm, mp_bcast_r3, &
     250              :          mp_bcast_d, mp_bcast_dv, mp_bcast_dm, mp_bcast_d3, &
     251              :          mp_bcast_c, mp_bcast_cv, mp_bcast_cm, mp_bcast_c3, &
     252              :          mp_bcast_z, mp_bcast_zv, mp_bcast_zm, mp_bcast_z3, &
     253              :          mp_bcast_b, mp_bcast_bv, mp_bcast_av, mp_bcast_am, &
     254              :          mp_bcast_i_src, mp_bcast_iv_src, mp_bcast_im_src, mp_bcast_i3_src, &
     255              :          mp_bcast_l_src, mp_bcast_lv_src, mp_bcast_lm_src, mp_bcast_l3_src, &
     256              :          mp_bcast_r_src, mp_bcast_rv_src, mp_bcast_rm_src, mp_bcast_r3_src, &
     257              :          mp_bcast_d_src, mp_bcast_dv_src, mp_bcast_dm_src, mp_bcast_d3_src, &
     258              :          mp_bcast_c_src, mp_bcast_cv_src, mp_bcast_cm_src, mp_bcast_c3_src, &
     259              :          mp_bcast_z_src, mp_bcast_zv_src, mp_bcast_zm_src, mp_bcast_z3_src, &
     260              :          mp_bcast_b_src, mp_bcast_bv_src, mp_bcast_av_src, mp_bcast_am_src
     261              : 
     262              :       PROCEDURE, PRIVATE, PASS(comm), NON_OVERRIDABLE :: mp_ibcast_i, mp_ibcast_iv, &
     263              :          mp_ibcast_l, mp_ibcast_lv, mp_ibcast_r, mp_ibcast_rv, &
     264              :          mp_ibcast_d, mp_ibcast_dv, mp_ibcast_c, mp_ibcast_cv, &
     265              :          mp_ibcast_z, mp_ibcast_zv
     266              :       GENERIC, PUBLIC :: ibcast => mp_ibcast_i, mp_ibcast_iv, &
     267              :          mp_ibcast_l, mp_ibcast_lv, mp_ibcast_r, mp_ibcast_rv, &
     268              :          mp_ibcast_d, mp_ibcast_dv, mp_ibcast_c, mp_ibcast_cv, &
     269              :          mp_ibcast_z, mp_ibcast_zv
     270              : 
     271              :       PROCEDURE, PRIVATE, PASS(comm), NON_OVERRIDABLE :: &
     272              :          mp_sum_i, mp_sum_iv, mp_sum_im, mp_sum_im3, mp_sum_im4, &
     273              :          mp_sum_l, mp_sum_lv, mp_sum_lm, mp_sum_lm3, mp_sum_lm4, &
     274              :          mp_sum_r, mp_sum_rv, mp_sum_rm, mp_sum_rm3, mp_sum_rm4, &
     275              :          mp_sum_d, mp_sum_dv, mp_sum_dm, mp_sum_dm3, mp_sum_dm4, &
     276              :          mp_sum_c, mp_sum_cv, mp_sum_cm, mp_sum_cm3, mp_sum_cm4, &
     277              :          mp_sum_z, mp_sum_zv, mp_sum_zm, mp_sum_zm3, mp_sum_zm4, &
     278              :          mp_sum_root_iv, mp_sum_root_im, mp_sum_root_im3, &
     279              :          mp_sum_root_lv, mp_sum_root_lm, mp_sum_root_lm3, &
     280              :          mp_sum_root_rv, mp_sum_root_rm, mp_sum_root_rm3, &
     281              :          mp_sum_root_dv, mp_sum_root_dm, mp_sum_root_dm3, &
     282              :          mp_sum_root_cv, mp_sum_root_cm, mp_sum_root_cm3, &
     283              :          mp_sum_root_zv, mp_sum_root_zm, mp_sum_root_zm3, &
     284              :          mp_sum_b, mp_sum_bv
     285              :       GENERIC, PUBLIC :: sum => mp_sum_i, mp_sum_iv, mp_sum_im, mp_sum_im3, mp_sum_im4, &
     286              :          mp_sum_l, mp_sum_lv, mp_sum_lm, mp_sum_lm3, mp_sum_lm4, &
     287              :          mp_sum_r, mp_sum_rv, mp_sum_rm, mp_sum_rm3, mp_sum_rm4, &
     288              :          mp_sum_d, mp_sum_dv, mp_sum_dm, mp_sum_dm3, mp_sum_dm4, &
     289              :          mp_sum_c, mp_sum_cv, mp_sum_cm, mp_sum_cm3, mp_sum_cm4, &
     290              :          mp_sum_z, mp_sum_zv, mp_sum_zm, mp_sum_zm3, mp_sum_zm4, &
     291              :          mp_sum_root_iv, mp_sum_root_im, mp_sum_root_im3, &
     292              :          mp_sum_root_lv, mp_sum_root_lm, mp_sum_root_lm3, &
     293              :          mp_sum_root_rv, mp_sum_root_rm, mp_sum_root_rm3, &
     294              :          mp_sum_root_dv, mp_sum_root_dm, mp_sum_root_dm3, &
     295              :          mp_sum_root_cv, mp_sum_root_cm, mp_sum_root_cm3, &
     296              :          mp_sum_root_zv, mp_sum_root_zm, mp_sum_root_zm3, &
     297              :          mp_sum_b, mp_sum_bv
     298              : 
     299              :       PROCEDURE, PRIVATE, PASS(comm), NON_OVERRIDABLE :: mp_isum_iv, &
     300              :          mp_isum_lv, mp_isum_rv, mp_isum_dv, mp_isum_cv, &
     301              :          mp_isum_zv, mp_isum_bv
     302              :       GENERIC, PUBLIC :: isum => mp_isum_iv, &
     303              :          mp_isum_lv, mp_isum_rv, mp_isum_dv, mp_isum_cv, &
     304              :          mp_isum_zv, mp_isum_bv
     305              : 
     306              :       PROCEDURE, PRIVATE, PASS(comm), NON_OVERRIDABLE :: mp_sum_partial_im, &
     307              :          mp_sum_partial_lm, mp_sum_partial_rm, mp_sum_partial_dm, &
     308              :          mp_sum_partial_cm, mp_sum_partial_zm
     309              :       GENERIC, PUBLIC :: sum_partial => mp_sum_partial_im, &
     310              :          mp_sum_partial_lm, mp_sum_partial_rm, mp_sum_partial_dm, &
     311              :          mp_sum_partial_cm, mp_sum_partial_zm
     312              : 
     313              :       PROCEDURE, PRIVATE, PASS(comm), NON_OVERRIDABLE :: mp_max_i, mp_max_iv, &
     314              :          mp_max_im, &
     315              :          mp_max_l, mp_max_lv, mp_max_lm, &
     316              :          mp_max_r, mp_max_rv, mp_max_rm, &
     317              :          mp_max_d, mp_max_dv, mp_max_dm, &
     318              :          mp_max_c, mp_max_cv, mp_max_cm, &
     319              :          mp_max_z, mp_max_zv, mp_max_zm, &
     320              :          mp_max_root_i, mp_max_root_l, &
     321              :          mp_max_root_r, mp_max_root_d, mp_max_root_c, mp_max_root_z, &
     322              :          mp_max_root_im, mp_max_root_lm, mp_max_root_rm, mp_max_root_dm, &
     323              :          mp_max_root_cm, mp_max_root_zm
     324              :       GENERIC, PUBLIC :: max => mp_max_i, mp_max_iv, &
     325              :          mp_max_im, &
     326              :          mp_max_l, mp_max_lv, mp_max_lm, &
     327              :          mp_max_r, mp_max_rv, mp_max_rm, &
     328              :          mp_max_d, mp_max_dv, mp_max_dm, &
     329              :          mp_max_c, mp_max_cv, mp_max_cm, &
     330              :          mp_max_z, mp_max_zv, mp_max_zm, &
     331              :          mp_max_root_i, mp_max_root_l, &
     332              :          mp_max_root_r, mp_max_root_d, mp_max_root_c, mp_max_root_z, &
     333              :          mp_max_root_im, mp_max_root_lm, mp_max_root_rm, mp_max_root_dm, &
     334              :          mp_max_root_cm, mp_max_root_zm
     335              : 
     336              :       PROCEDURE, PRIVATE, PASS(comm), NON_OVERRIDABLE :: mp_min_i, mp_min_iv, &
     337              :          mp_min_im, &
     338              :          mp_min_l, mp_min_lv, mp_min_lm, &
     339              :          mp_min_r, mp_min_rv, mp_min_rm, &
     340              :          mp_min_d, mp_min_dv, mp_min_dm, &
     341              :          mp_min_c, mp_min_cv, mp_min_cm, &
     342              :          mp_min_z, mp_min_zv, mp_min_zm
     343              :       GENERIC, PUBLIC :: min => mp_min_i, mp_min_iv, &
     344              :          mp_min_im, &
     345              :          mp_min_l, mp_min_lv, mp_min_lm, &
     346              :          mp_min_r, mp_min_rv, mp_min_rm, &
     347              :          mp_min_d, mp_min_dv, mp_min_dm, &
     348              :          mp_min_c, mp_min_cv, mp_min_cm, &
     349              :          mp_min_z, mp_min_zv, mp_min_zm
     350              : 
     351              :       PROCEDURE, PUBLIC, PASS(comm), NON_OVERRIDABLE :: &
     352              :          mp_sum_scatter_iv, mp_sum_scatter_lv, mp_sum_scatter_rv, &
     353              :          mp_sum_scatter_dv, mp_sum_scatter_cv, mp_sum_scatter_zv
     354              :       GENERIC, PUBLIC :: sum_scatter => &
     355              :          mp_sum_scatter_iv, mp_sum_scatter_lv, mp_sum_scatter_rv, &
     356              :          mp_sum_scatter_dv, mp_sum_scatter_cv, mp_sum_scatter_zv
     357              : 
     358              :       PROCEDURE, PRIVATE, PASS(comm), NON_OVERRIDABLE :: mp_prod_r, mp_prod_d, mp_prod_c, mp_prod_z
     359              :       GENERIC, PUBLIC :: prod => mp_prod_r, mp_prod_d, mp_prod_c, mp_prod_z
     360              : 
     361              :       PROCEDURE, PRIVATE, PASS(comm), NON_OVERRIDABLE :: mp_gather_i, mp_gather_iv, mp_gather_im, &
     362              :          mp_gather_l, mp_gather_lv, mp_gather_lm, &
     363              :          mp_gather_r, mp_gather_rv, mp_gather_rm, &
     364              :          mp_gather_d, mp_gather_dv, mp_gather_dm, &
     365              :          mp_gather_c, mp_gather_cv, mp_gather_cm, &
     366              :          mp_gather_z, mp_gather_zv, mp_gather_zm, &
     367              :          mp_gather_i_src, mp_gather_iv_src, mp_gather_im_src, &
     368              :          mp_gather_l_src, mp_gather_lv_src, mp_gather_lm_src, &
     369              :          mp_gather_r_src, mp_gather_rv_src, mp_gather_rm_src, &
     370              :          mp_gather_d_src, mp_gather_dv_src, mp_gather_dm_src, &
     371              :          mp_gather_c_src, mp_gather_cv_src, mp_gather_cm_src, &
     372              :          mp_gather_z_src, mp_gather_zv_src, mp_gather_zm_src
     373              :       GENERIC, PUBLIC :: gather => mp_gather_i, mp_gather_iv, mp_gather_im, &
     374              :          mp_gather_l, mp_gather_lv, mp_gather_lm, &
     375              :          mp_gather_r, mp_gather_rv, mp_gather_rm, &
     376              :          mp_gather_d, mp_gather_dv, mp_gather_dm, &
     377              :          mp_gather_c, mp_gather_cv, mp_gather_cm, &
     378              :          mp_gather_z, mp_gather_zv, mp_gather_zm, &
     379              :          mp_gather_i_src, mp_gather_iv_src, mp_gather_im_src, &
     380              :          mp_gather_l_src, mp_gather_lv_src, mp_gather_lm_src, &
     381              :          mp_gather_r_src, mp_gather_rv_src, mp_gather_rm_src, &
     382              :          mp_gather_d_src, mp_gather_dv_src, mp_gather_dm_src, &
     383              :          mp_gather_c_src, mp_gather_cv_src, mp_gather_cm_src, &
     384              :          mp_gather_z_src, mp_gather_zv_src, mp_gather_zm_src
     385              : 
     386              :       PROCEDURE, PRIVATE, PASS(comm), NON_OVERRIDABLE :: mp_gatherv_iv, &
     387              :          mp_gatherv_lv, mp_gatherv_rv, mp_gatherv_dv, &
     388              :          mp_gatherv_cv, mp_gatherv_zv, mp_gatherv_lm2, mp_gatherv_rm2, &
     389              :          mp_gatherv_dm2, mp_gatherv_cm2, mp_gatherv_zm2, mp_gatherv_iv_src, &
     390              :          mp_gatherv_lv_src, mp_gatherv_rv_src, mp_gatherv_dv_src, &
     391              :          mp_gatherv_cv_src, mp_gatherv_zv_src, mp_gatherv_lm2_src, mp_gatherv_rm2_src, &
     392              :          mp_gatherv_dm2_src, mp_gatherv_cm2_src, mp_gatherv_zm2_src
     393              :       GENERIC, PUBLIC :: gatherv => mp_gatherv_iv, &
     394              :          mp_gatherv_lv, mp_gatherv_rv, mp_gatherv_dv, &
     395              :          mp_gatherv_cv, mp_gatherv_zv, mp_gatherv_lm2, mp_gatherv_rm2, &
     396              :          mp_gatherv_dm2, mp_gatherv_cm2, mp_gatherv_zm2, mp_gatherv_iv_src, &
     397              :          mp_gatherv_lv_src, mp_gatherv_rv_src, mp_gatherv_dv_src, &
     398              :          mp_gatherv_cv_src, mp_gatherv_zv_src, mp_gatherv_lm2_src, mp_gatherv_rm2_src, &
     399              :          mp_gatherv_dm2_src, mp_gatherv_cm2_src, mp_gatherv_zm2_src
     400              : 
     401              :       PROCEDURE, PRIVATE, PASS(comm), NON_OVERRIDABLE :: mp_igatherv_iv, &
     402              :          mp_igatherv_lv, mp_igatherv_rv, mp_igatherv_dv, &
     403              :          mp_igatherv_cv, mp_igatherv_zv
     404              :       GENERIC, PUBLIC :: igatherv => mp_igatherv_iv, &
     405              :          mp_igatherv_lv, mp_igatherv_rv, mp_igatherv_dv, &
     406              :          mp_igatherv_cv, mp_igatherv_zv
     407              : 
     408              :       PROCEDURE, PRIVATE, PASS(comm), NON_OVERRIDABLE :: mp_allgather_i, mp_allgather_i2, &
     409              :          mp_allgather_i12, mp_allgather_i23, mp_allgather_i34, &
     410              :          mp_allgather_i22, mp_allgather_l, mp_allgather_l2, &
     411              :          mp_allgather_l12, mp_allgather_l23, mp_allgather_l34, &
     412              :          mp_allgather_l22, mp_allgather_r, mp_allgather_r2, &
     413              :          mp_allgather_r12, mp_allgather_r23, mp_allgather_r34, &
     414              :          mp_allgather_r22, mp_allgather_d, mp_allgather_d2, &
     415              :          mp_allgather_d12, mp_allgather_d23, mp_allgather_d34, &
     416              :          mp_allgather_d22, mp_allgather_c, mp_allgather_c2, &
     417              :          mp_allgather_c12, mp_allgather_c23, mp_allgather_c34, &
     418              :          mp_allgather_c22, mp_allgather_z, mp_allgather_z2, &
     419              :          mp_allgather_z12, mp_allgather_z23, mp_allgather_z34, &
     420              :          mp_allgather_z22
     421              :       GENERIC, PUBLIC :: allgather => mp_allgather_i, mp_allgather_i2, &
     422              :          mp_allgather_i12, mp_allgather_i23, mp_allgather_i34, &
     423              :          mp_allgather_i22, mp_allgather_l, mp_allgather_l2, &
     424              :          mp_allgather_l12, mp_allgather_l23, mp_allgather_l34, &
     425              :          mp_allgather_l22, mp_allgather_r, mp_allgather_r2, &
     426              :          mp_allgather_r12, mp_allgather_r23, mp_allgather_r34, &
     427              :          mp_allgather_r22, mp_allgather_d, mp_allgather_d2, &
     428              :          mp_allgather_d12, mp_allgather_d23, mp_allgather_d34, &
     429              :          mp_allgather_d22, mp_allgather_c, mp_allgather_c2, &
     430              :          mp_allgather_c12, mp_allgather_c23, mp_allgather_c34, &
     431              :          mp_allgather_c22, mp_allgather_z, mp_allgather_z2, &
     432              :          mp_allgather_z12, mp_allgather_z23, mp_allgather_z34, &
     433              :          mp_allgather_z22
     434              : 
     435              :       PROCEDURE, PRIVATE, PASS(comm), NON_OVERRIDABLE ::  mp_allgatherv_iv, mp_allgatherv_lv, &
     436              :          mp_allgatherv_rv, mp_allgatherv_dv, mp_allgatherv_cv, mp_allgatherv_zv, &
     437              :          mp_allgatherv_im2, mp_allgatherv_lm2, mp_allgatherv_rm2, &
     438              :          mp_allgatherv_dm2, mp_allgatherv_cm2, mp_allgatherv_zm2
     439              :       GENERIC, PUBLIC :: allgatherv => mp_allgatherv_iv, mp_allgatherv_lv, &
     440              :          mp_allgatherv_rv, mp_allgatherv_dv, mp_allgatherv_cv, mp_allgatherv_zv, &
     441              :          mp_allgatherv_im2, mp_allgatherv_lm2, mp_allgatherv_rm2, &
     442              :          mp_allgatherv_dm2, mp_allgatherv_cm2, mp_allgatherv_zm2
     443              : 
     444              :       PROCEDURE, PRIVATE, PASS(comm), NON_OVERRIDABLE :: mp_iallgather_i, mp_iallgather_l, &
     445              :          mp_iallgather_r, mp_iallgather_d, mp_iallgather_c, mp_iallgather_z, &
     446              :          mp_iallgather_i11, mp_iallgather_l11, mp_iallgather_r11, mp_iallgather_d11, &
     447              :          mp_iallgather_c11, mp_iallgather_z11, mp_iallgather_i13, mp_iallgather_l13, &
     448              :          mp_iallgather_r13, mp_iallgather_d13, mp_iallgather_c13, mp_iallgather_z13, &
     449              :          mp_iallgather_i22, mp_iallgather_l22, mp_iallgather_r22, mp_iallgather_d22, &
     450              :          mp_iallgather_c22, mp_iallgather_z22, mp_iallgather_i24, mp_iallgather_l24, &
     451              :          mp_iallgather_r24, mp_iallgather_d24, mp_iallgather_c24, mp_iallgather_z24, &
     452              :          mp_iallgather_i33, mp_iallgather_l33, mp_iallgather_r33, mp_iallgather_d33, &
     453              :          mp_iallgather_c33, mp_iallgather_z33
     454              :       GENERIC, PUBLIC :: iallgather => mp_iallgather_i, mp_iallgather_l, &
     455              :          mp_iallgather_r, mp_iallgather_d, mp_iallgather_c, mp_iallgather_z, &
     456              :          mp_iallgather_i11, mp_iallgather_l11, mp_iallgather_r11, mp_iallgather_d11, &
     457              :          mp_iallgather_c11, mp_iallgather_z11, mp_iallgather_i13, mp_iallgather_l13, &
     458              :          mp_iallgather_r13, mp_iallgather_d13, mp_iallgather_c13, mp_iallgather_z13, &
     459              :          mp_iallgather_i22, mp_iallgather_l22, mp_iallgather_r22, mp_iallgather_d22, &
     460              :          mp_iallgather_c22, mp_iallgather_z22, mp_iallgather_i24, mp_iallgather_l24, &
     461              :          mp_iallgather_r24, mp_iallgather_d24, mp_iallgather_c24, mp_iallgather_z24, &
     462              :          mp_iallgather_i33, mp_iallgather_l33, mp_iallgather_r33, mp_iallgather_d33, &
     463              :          mp_iallgather_c33, mp_iallgather_z33
     464              : 
     465              :       PROCEDURE, PRIVATE, PASS(comm), NON_OVERRIDABLE :: mp_iallgatherv_iv, mp_iallgatherv_iv2, &
     466              :          mp_iallgatherv_lv, mp_iallgatherv_lv2, mp_iallgatherv_rv, mp_iallgatherv_rv2, &
     467              :          mp_iallgatherv_dv, mp_iallgatherv_dv2, mp_iallgatherv_cv, mp_iallgatherv_cv2, &
     468              :          mp_iallgatherv_zv, mp_iallgatherv_zv2
     469              :       GENERIC, PUBLIC :: iallgatherv => mp_iallgatherv_iv, mp_iallgatherv_iv2, &
     470              :          mp_iallgatherv_lv, mp_iallgatherv_lv2, mp_iallgatherv_rv, mp_iallgatherv_rv2, &
     471              :          mp_iallgatherv_dv, mp_iallgatherv_dv2, mp_iallgatherv_cv, mp_iallgatherv_cv2, &
     472              :          mp_iallgatherv_zv, mp_iallgatherv_zv2
     473              : 
     474              :       PROCEDURE, PRIVATE, PASS(comm), NON_OVERRIDABLE :: mp_scatter_iv, mp_scatter_lv, &
     475              :          mp_scatter_rv, mp_scatter_dv, mp_scatter_cv, mp_scatter_zv
     476              :       GENERIC, PUBLIC :: scatter => mp_scatter_iv, mp_scatter_lv, &
     477              :          mp_scatter_rv, mp_scatter_dv, mp_scatter_cv, mp_scatter_zv
     478              : 
     479              :       PROCEDURE, PRIVATE, PASS(comm), NON_OVERRIDABLE :: mp_iscatter_i, mp_iscatter_l, &
     480              :          mp_iscatter_r, mp_iscatter_d, mp_iscatter_c, mp_iscatter_z, &
     481              :          mp_iscatter_iv2, mp_iscatter_lv2, mp_iscatter_rv2, mp_iscatter_dv2, &
     482              :          mp_iscatter_cv2, mp_iscatter_zv2
     483              :       GENERIC, PUBLIC :: iscatter => mp_iscatter_i, mp_iscatter_l, &
     484              :          mp_iscatter_r, mp_iscatter_d, mp_iscatter_c, mp_iscatter_z, &
     485              :          mp_iscatter_iv2, mp_iscatter_lv2, mp_iscatter_rv2, mp_iscatter_dv2, &
     486              :          mp_iscatter_cv2, mp_iscatter_zv2
     487              : 
     488              :       PROCEDURE, PRIVATE, PASS(comm), NON_OVERRIDABLE :: mp_iscatterv_iv, mp_iscatterv_lv, &
     489              :          mp_iscatterv_rv, mp_iscatterv_dv, mp_iscatterv_cv, mp_iscatterv_zv
     490              :       GENERIC, PUBLIC :: iscatterv => mp_iscatterv_iv, mp_iscatterv_lv, &
     491              :          mp_iscatterv_rv, mp_iscatterv_dv, mp_iscatterv_cv, mp_iscatterv_zv
     492              : 
     493              :       PROCEDURE, PRIVATE, PASS(comm), NON_OVERRIDABLE :: mp_alltoall_i, mp_alltoall_i22, mp_alltoall_i33, &
     494              :          mp_alltoall_i44, mp_alltoall_i55, mp_alltoall_i45, mp_alltoall_i34, &
     495              :          mp_alltoall_i11v, mp_alltoall_i22v, mp_alltoall_i54, mp_alltoall_i43, &
     496              :          mp_alltoall_l, mp_alltoall_l22, mp_alltoall_l33, &
     497              :          mp_alltoall_l44, mp_alltoall_l55, mp_alltoall_l45, mp_alltoall_l34, &
     498              :          mp_alltoall_l11v, mp_alltoall_l22v, mp_alltoall_l54, mp_alltoall_l43, &
     499              :          mp_alltoall_r, mp_alltoall_r22, mp_alltoall_r33, &
     500              :          mp_alltoall_r44, mp_alltoall_r55, mp_alltoall_r45, mp_alltoall_r34, &
     501              :          mp_alltoall_r11v, mp_alltoall_r22v, mp_alltoall_r54, mp_alltoall_r43, &
     502              :          mp_alltoall_d, mp_alltoall_d22, mp_alltoall_d33, &
     503              :          mp_alltoall_d44, mp_alltoall_d55, mp_alltoall_d45, mp_alltoall_d34, &
     504              :          mp_alltoall_d11v, mp_alltoall_d22v, mp_alltoall_d54, mp_alltoall_d43, &
     505              :          mp_alltoall_c, mp_alltoall_c22, mp_alltoall_c33, &
     506              :          mp_alltoall_c44, mp_alltoall_c55, mp_alltoall_c45, mp_alltoall_c34, &
     507              :          mp_alltoall_c11v, mp_alltoall_c22v, mp_alltoall_c54, mp_alltoall_c43, &
     508              :          mp_alltoall_z, mp_alltoall_z22, mp_alltoall_z33, &
     509              :          mp_alltoall_z44, mp_alltoall_z55, mp_alltoall_z45, mp_alltoall_z34, &
     510              :          mp_alltoall_z11v, mp_alltoall_z22v, mp_alltoall_z54, mp_alltoall_z43
     511              :       GENERIC, PUBLIC :: alltoall => mp_alltoall_i, mp_alltoall_i22, mp_alltoall_i33, &
     512              :          mp_alltoall_i44, mp_alltoall_i55, mp_alltoall_i45, mp_alltoall_i34, &
     513              :          mp_alltoall_i11v, mp_alltoall_i22v, mp_alltoall_i54, mp_alltoall_i43, &
     514              :          mp_alltoall_l, mp_alltoall_l22, mp_alltoall_l33, &
     515              :          mp_alltoall_l44, mp_alltoall_l55, mp_alltoall_l45, mp_alltoall_l34, &
     516              :          mp_alltoall_l11v, mp_alltoall_l22v, mp_alltoall_l54, mp_alltoall_l43, &
     517              :          mp_alltoall_r, mp_alltoall_r22, mp_alltoall_r33, &
     518              :          mp_alltoall_r44, mp_alltoall_r55, mp_alltoall_r45, mp_alltoall_r34, &
     519              :          mp_alltoall_r11v, mp_alltoall_r22v, mp_alltoall_r54, mp_alltoall_r43, &
     520              :          mp_alltoall_d, mp_alltoall_d22, mp_alltoall_d33, &
     521              :          mp_alltoall_d44, mp_alltoall_d55, mp_alltoall_d45, mp_alltoall_d34, &
     522              :          mp_alltoall_d11v, mp_alltoall_d22v, mp_alltoall_d54, mp_alltoall_d43, &
     523              :          mp_alltoall_c, mp_alltoall_c22, mp_alltoall_c33, &
     524              :          mp_alltoall_c44, mp_alltoall_c55, mp_alltoall_c45, mp_alltoall_c34, &
     525              :          mp_alltoall_c11v, mp_alltoall_c22v, mp_alltoall_c54, mp_alltoall_c43, &
     526              :          mp_alltoall_z, mp_alltoall_z22, mp_alltoall_z33, &
     527              :          mp_alltoall_z44, mp_alltoall_z55, mp_alltoall_z45, mp_alltoall_z34, &
     528              :          mp_alltoall_z11v, mp_alltoall_z22v, mp_alltoall_z54, mp_alltoall_z43
     529              : 
     530              :       PROCEDURE, PRIVATE, PASS(comm), NON_OVERRIDABLE :: mp_send_i, mp_send_iv, mp_send_im2, mp_send_im3, &
     531              :          mp_send_l, mp_send_lv, mp_send_lm2, mp_send_lm3, &
     532              :          mp_send_r, mp_send_rv, mp_send_rm2, mp_send_rm3, &
     533              :          mp_send_d, mp_send_dv, mp_send_dm2, mp_send_dm3, &
     534              :          mp_send_c, mp_send_cv, mp_send_cm2, mp_send_cm3, &
     535              :          mp_send_z, mp_send_zv, mp_send_zm2, mp_send_zm3
     536              :       GENERIC, PUBLIC :: send => mp_send_i, mp_send_iv, mp_send_im2, mp_send_im3, &
     537              :          mp_send_l, mp_send_lv, mp_send_lm2, mp_send_lm3, &
     538              :          mp_send_r, mp_send_rv, mp_send_rm2, mp_send_rm3, &
     539              :          mp_send_d, mp_send_dv, mp_send_dm2, mp_send_dm3, &
     540              :          mp_send_c, mp_send_cv, mp_send_cm2, mp_send_cm3, &
     541              :          mp_send_z, mp_send_zv, mp_send_zm2, mp_send_zm3
     542              : 
     543              :       PROCEDURE, PRIVATE, PASS(comm), NON_OVERRIDABLE :: mp_recv_i, mp_recv_iv, mp_recv_im2, mp_recv_im3, &
     544              :          mp_recv_l, mp_recv_lv, mp_recv_lm2, mp_recv_lm3, &
     545              :          mp_recv_r, mp_recv_rv, mp_recv_rm2, mp_recv_rm3, &
     546              :          mp_recv_d, mp_recv_dv, mp_recv_dm2, mp_recv_dm3, &
     547              :          mp_recv_c, mp_recv_cv, mp_recv_cm2, mp_recv_cm3, &
     548              :          mp_recv_z, mp_recv_zv, mp_recv_zm2, mp_recv_zm3
     549              :       GENERIC, PUBLIC :: recv => mp_recv_i, mp_recv_iv, mp_recv_im2, mp_recv_im3, &
     550              :          mp_recv_l, mp_recv_lv, mp_recv_lm2, mp_recv_lm3, &
     551              :          mp_recv_r, mp_recv_rv, mp_recv_rm2, mp_recv_rm3, &
     552              :          mp_recv_d, mp_recv_dv, mp_recv_dm2, mp_recv_dm3, &
     553              :          mp_recv_c, mp_recv_cv, mp_recv_cm2, mp_recv_cm3, &
     554              :          mp_recv_z, mp_recv_zv, mp_recv_zm2, mp_recv_zm3
     555              : 
     556              :       PROCEDURE, PRIVATE, PASS(comm), NON_OVERRIDABLE :: mp_isendrecv_i, mp_isendrecv_iv, &
     557              :          mp_isendrecv_l, mp_isendrecv_lv, mp_isendrecv_r, mp_isendrecv_rv, &
     558              :          mp_isendrecv_d, mp_isendrecv_dv, mp_isendrecv_c, mp_isendrecv_cv, &
     559              :          mp_isendrecv_z, mp_isendrecv_zv
     560              :       GENERIC, PUBLIC :: isendrecv => mp_isendrecv_i, mp_isendrecv_iv, &
     561              :          mp_isendrecv_l, mp_isendrecv_lv, mp_isendrecv_r, mp_isendrecv_rv, &
     562              :          mp_isendrecv_d, mp_isendrecv_dv, mp_isendrecv_c, mp_isendrecv_cv, &
     563              :          mp_isendrecv_z, mp_isendrecv_zv
     564              : 
     565              :       PROCEDURE, PRIVATE, PASS(comm), NON_OVERRIDABLE :: mp_isend_iv, mp_isend_im2, mp_isend_im3, mp_isend_im4, &
     566              :          mp_isend_lv, mp_isend_lm2, mp_isend_lm3, mp_isend_lm4, &
     567              :          mp_isend_rv, mp_isend_rm2, mp_isend_rm3, mp_isend_rm4, &
     568              :          mp_isend_dv, mp_isend_dm2, mp_isend_dm3, mp_isend_dm4, &
     569              :          mp_isend_cv, mp_isend_cm2, mp_isend_cm3, mp_isend_cm4, &
     570              :          mp_isend_zv, mp_isend_zm2, mp_isend_zm3, mp_isend_zm4, &
     571              :          mp_isend_bv, mp_isend_bm3, mp_isend_custom
     572              :       GENERIC, PUBLIC :: isend => mp_isend_iv, mp_isend_im2, mp_isend_im3, mp_isend_im4, &
     573              :          mp_isend_lv, mp_isend_lm2, mp_isend_lm3, mp_isend_lm4, &
     574              :          mp_isend_rv, mp_isend_rm2, mp_isend_rm3, mp_isend_rm4, &
     575              :          mp_isend_dv, mp_isend_dm2, mp_isend_dm3, mp_isend_dm4, &
     576              :          mp_isend_cv, mp_isend_cm2, mp_isend_cm3, mp_isend_cm4, &
     577              :          mp_isend_zv, mp_isend_zm2, mp_isend_zm3, mp_isend_zm4, &
     578              :          mp_isend_bv, mp_isend_bm3, mp_isend_custom
     579              : 
     580              :       PROCEDURE, PRIVATE, PASS(comm), NON_OVERRIDABLE :: mp_irecv_iv, mp_irecv_im2, mp_irecv_im3, mp_irecv_im4, &
     581              :          mp_irecv_lv, mp_irecv_lm2, mp_irecv_lm3, mp_irecv_lm4, &
     582              :          mp_irecv_rv, mp_irecv_rm2, mp_irecv_rm3, mp_irecv_rm4, &
     583              :          mp_irecv_dv, mp_irecv_dm2, mp_irecv_dm3, mp_irecv_dm4, &
     584              :          mp_irecv_cv, mp_irecv_cm2, mp_irecv_cm3, mp_irecv_cm4, &
     585              :          mp_irecv_zv, mp_irecv_zm2, mp_irecv_zm3, mp_irecv_zm4, &
     586              :          mp_irecv_bv, mp_irecv_bm3, mp_irecv_custom
     587              :       GENERIC, PUBLIC :: irecv => mp_irecv_iv, mp_irecv_im2, mp_irecv_im3, mp_irecv_im4, &
     588              :          mp_irecv_lv, mp_irecv_lm2, mp_irecv_lm3, mp_irecv_lm4, &
     589              :          mp_irecv_rv, mp_irecv_rm2, mp_irecv_rm3, mp_irecv_rm4, &
     590              :          mp_irecv_dv, mp_irecv_dm2, mp_irecv_dm3, mp_irecv_dm4, &
     591              :          mp_irecv_cv, mp_irecv_cm2, mp_irecv_cm3, mp_irecv_cm4, &
     592              :          mp_irecv_zv, mp_irecv_zm2, mp_irecv_zm3, mp_irecv_zm4, &
     593              :          mp_irecv_bv, mp_irecv_bm3, mp_irecv_custom
     594              : 
     595              :       PROCEDURE, PUBLIC, PASS(comm), NON_OVERRIDABLE :: probe => mp_probe
     596              : 
     597              :       PROCEDURE, PUBLIC, PASS(comm), NON_OVERRIDABLE :: sync => mp_sync
     598              :       PROCEDURE, PUBLIC, PASS(comm), NON_OVERRIDABLE :: isync => mp_isync
     599              : 
     600              :       PROCEDURE, PUBLIC, PASS(comm1), NON_OVERRIDABLE :: compare => mp_comm_compare
     601              :       PROCEDURE, PUBLIC, PASS(comm1), NON_OVERRIDABLE :: rank_compare => mp_rank_compare
     602              : 
     603              :       PROCEDURE, PUBLIC, PASS(comm2), NON_OVERRIDABLE :: from_dup => mp_comm_dup
     604              :       PROCEDURE, PUBLIC, PASS(comm), NON_OVERRIDABLE :: mp_comm_free
     605              :       GENERIC, PUBLIC :: free => mp_comm_free
     606              : 
     607              :       PROCEDURE, PUBLIC, PASS(comm), NON_OVERRIDABLE :: mp_comm_init
     608              :       GENERIC, PUBLIC :: init => mp_comm_init
     609              : 
     610              :       PROCEDURE, PUBLIC, PASS(comm), NON_OVERRIDABLE :: get_size => mp_comm_size
     611              :       PROCEDURE, PUBLIC, PASS(comm), NON_OVERRIDABLE :: get_rank => mp_comm_rank
     612              :       PROCEDURE, PUBLIC, PASS(comm), NON_OVERRIDABLE :: get_ndims => mp_comm_get_ndims
     613              :       PROCEDURE, PUBLIC, PASS(comm), NON_OVERRIDABLE :: is_source => mp_comm_is_source
     614              : 
     615              :       ! Creation routines
     616              :       PROCEDURE, PRIVATE, PASS(sub_comm), NON_OVERRIDABLE :: mp_comm_split, mp_comm_split_direct
     617              :       GENERIC, PUBLIC :: from_split => mp_comm_split, mp_comm_split_direct
     618              :       PROCEDURE, PUBLIC, PASS(sub_comm), NON_OVERRIDABLE :: from_split_type => mp_comm_split_type
     619              :       PROCEDURE, PUBLIC, PASS(mp_new_comm), NON_OVERRIDABLE :: from_reordering => mp_reordering
     620              :       PROCEDURE, PUBLIC, PASS(comm_new), NON_OVERRIDABLE :: mp_comm_assign
     621              :       GENERIC, PUBLIC :: ASSIGNMENT(=) => mp_comm_assign
     622              : 
     623              :       ! Other Getters
     624              :       PROCEDURE, PRIVATE, PASS(comm), NON_OVERRIDABLE :: mp_comm_get_tag_ub
     625              :       GENERIC, PUBLIC :: get_tag_ub => mp_comm_get_tag_ub
     626              :       PROCEDURE, PRIVATE, PASS(comm), NON_OVERRIDABLE :: mp_comm_get_host_rank
     627              :       GENERIC, PUBLIC :: get_host_rank => mp_comm_get_host_rank
     628              :       PROCEDURE, PRIVATE, PASS(comm), NON_OVERRIDABLE :: mp_comm_get_io_rank
     629              :       GENERIC, PUBLIC :: get_io_rank => mp_comm_get_io_rank
     630              :       PROCEDURE, PRIVATE, PASS(comm), NON_OVERRIDABLE :: mp_comm_get_wtime_is_global
     631              :       GENERIC, PUBLIC :: get_wtime_is_global => mp_comm_get_wtime_is_global
     632              :    END TYPE
     633              : 
     634              :    TYPE mp_request_type
     635              :       PRIVATE
     636              :       MPI_REQUEST_TYPE :: handle = mp_request_null_handle
     637              :    CONTAINS
     638              :       PROCEDURE, PUBLIC, NON_OVERRIDABLE :: set_handle => mp_request_type_set_handle
     639              :       PROCEDURE, PUBLIC, NON_OVERRIDABLE :: get_handle => mp_request_type_get_handle
     640              :       PROCEDURE, PRIVATE, NON_OVERRIDABLE :: mp_request_op_eq
     641              :       PROCEDURE, PRIVATE, NON_OVERRIDABLE :: mp_request_op_neq
     642              :       GENERIC, PUBLIC :: OPERATOR(==) => mp_request_op_eq
     643              :       GENERIC, PUBLIC :: OPERATOR(/=) => mp_request_op_neq
     644              : 
     645              :       PROCEDURE, PUBLIC, PASS(request), NON_OVERRIDABLE :: test => mp_test_1
     646              : 
     647              :       PROCEDURE, PUBLIC, PASS(request), NON_OVERRIDABLE :: wait => mp_wait
     648              :    END TYPE
     649              : 
     650              :    TYPE mp_win_type
     651              :       PRIVATE
     652              :       MPI_WIN_TYPE :: handle = mp_win_null_handle
     653              :    CONTAINS
     654              :       PROCEDURE, PUBLIC, NON_OVERRIDABLE :: set_handle => mp_win_type_set_handle
     655              :       PROCEDURE, PUBLIC, NON_OVERRIDABLE :: get_handle => mp_win_type_get_handle
     656              :       PROCEDURE, PRIVATE, NON_OVERRIDABLE :: mp_win_op_eq
     657              :       PROCEDURE, PRIVATE, NON_OVERRIDABLE :: mp_win_op_neq
     658              :       GENERIC, PUBLIC :: OPERATOR(==) => mp_win_op_eq
     659              :       GENERIC, PUBLIC :: OPERATOR(/=) => mp_win_op_neq
     660              : 
     661              :       PROCEDURE, PRIVATE, PASS(win), NON_OVERRIDABLE :: mp_win_create_iv, mp_win_create_lv, &
     662              :          mp_win_create_rv, mp_win_create_dv, mp_win_create_cv, mp_win_create_zv
     663              :       GENERIC, PUBLIC :: create => mp_win_create_iv, mp_win_create_lv, &
     664              :          mp_win_create_rv, mp_win_create_dv, mp_win_create_cv, mp_win_create_zv
     665              : 
     666              :       PROCEDURE, PRIVATE, PASS(win), NON_OVERRIDABLE :: mp_rget_iv, mp_rget_lv, &
     667              :          mp_rget_rv, mp_rget_dv, mp_rget_cv, mp_rget_zv
     668              :       GENERIC, PUBLIC :: rget => mp_rget_iv, mp_rget_lv, &
     669              :          mp_rget_rv, mp_rget_dv, mp_rget_cv, mp_rget_zv
     670              : 
     671              :       PROCEDURE, PUBLIC, PASS(win), NON_OVERRIDABLE :: free => mp_win_free
     672              :       PROCEDURE, PUBLIC, PASS(win_new), NON_OVERRIDABLE :: mp_win_assign
     673              :       GENERIC, PUBLIC :: ASSIGNMENT(=) => mp_win_assign
     674              : 
     675              :       PROCEDURE, PUBLIC, PASS(win), NON_OVERRIDABLE :: lock_all => mp_win_lock_all
     676              :       PROCEDURE, PUBLIC, PASS(win), NON_OVERRIDABLE :: unlock_all => mp_win_unlock_all
     677              :       PROCEDURE, PUBLIC, PASS(win), NON_OVERRIDABLE :: flush_all => mp_win_flush_all
     678              :    END TYPE
     679              : 
     680              :    TYPE mp_file_type
     681              :       PRIVATE
     682              :       MPI_FILE_TYPE :: handle = mp_file_null_handle
     683              :    CONTAINS
     684              :       PROCEDURE, PUBLIC, NON_OVERRIDABLE :: set_handle => mp_file_type_set_handle
     685              :       PROCEDURE, PUBLIC, NON_OVERRIDABLE :: get_handle => mp_file_type_get_handle
     686              :       PROCEDURE, PRIVATE, NON_OVERRIDABLE :: mp_file_op_eq
     687              :       PROCEDURE, PRIVATE, NON_OVERRIDABLE :: mp_file_op_neq
     688              :       GENERIC, PUBLIC :: OPERATOR(==) => mp_file_op_eq
     689              :       GENERIC, PUBLIC :: OPERATOR(/=) => mp_file_op_neq
     690              : 
     691              :       PROCEDURE, PRIVATE, PASS(fh), NON_OVERRIDABLE :: mp_file_write_at_ch, mp_file_write_at_chv, &
     692              :          mp_file_write_at_i, mp_file_write_at_iv, mp_file_write_at_r, mp_file_write_at_rv, &
     693              :          mp_file_write_at_d, mp_file_write_at_dv, mp_file_write_at_c, mp_file_write_at_cv, &
     694              :          mp_file_write_at_z, mp_file_write_at_zv, mp_file_write_at_l, mp_file_write_at_lv
     695              :       GENERIC, PUBLIC :: write_at => mp_file_write_at_ch, mp_file_write_at_chv, &
     696              :          mp_file_write_at_i, mp_file_write_at_iv, mp_file_write_at_r, mp_file_write_at_rv, &
     697              :          mp_file_write_at_d, mp_file_write_at_dv, mp_file_write_at_c, mp_file_write_at_cv, &
     698              :          mp_file_write_at_z, mp_file_write_at_zv, mp_file_write_at_l, mp_file_write_at_lv
     699              : 
     700              :       PROCEDURE, PRIVATE, PASS(fh), NON_OVERRIDABLE :: mp_file_write_at_all_ch, mp_file_write_at_all_chv, &
     701              :          mp_file_write_at_all_i, mp_file_write_at_all_iv, mp_file_write_at_all_l, mp_file_write_at_all_lv, &
     702              :          mp_file_write_at_all_r, mp_file_write_at_all_rv, mp_file_write_at_all_d, mp_file_write_at_all_dv, &
     703              :          mp_file_write_at_all_c, mp_file_write_at_all_cv, mp_file_write_at_all_z, mp_file_write_at_all_zv
     704              :       GENERIC, PUBLIC :: write_at_all => mp_file_write_at_all_ch, mp_file_write_at_all_chv, &
     705              :          mp_file_write_at_all_i, mp_file_write_at_all_iv, mp_file_write_at_all_l, mp_file_write_at_all_lv, &
     706              :          mp_file_write_at_all_r, mp_file_write_at_all_rv, mp_file_write_at_all_d, mp_file_write_at_all_dv, &
     707              :          mp_file_write_at_all_c, mp_file_write_at_all_cv, mp_file_write_at_all_z, mp_file_write_at_all_zv
     708              : 
     709              :       PROCEDURE, PRIVATE, PASS(fh), NON_OVERRIDABLE :: mp_file_read_at_ch, mp_file_read_at_chv, &
     710              :          mp_file_read_at_i, mp_file_read_at_iv, mp_file_read_at_r, mp_file_read_at_rv, &
     711              :          mp_file_read_at_d, mp_file_read_at_dv, mp_file_read_at_c, mp_file_read_at_cv, &
     712              :          mp_file_read_at_z, mp_file_read_at_zv, mp_file_read_at_l, mp_file_read_at_lv
     713              :       GENERIC, PUBLIC :: read_at => mp_file_read_at_ch, mp_file_read_at_chv, &
     714              :          mp_file_read_at_i, mp_file_read_at_iv, mp_file_read_at_r, mp_file_read_at_rv, &
     715              :          mp_file_read_at_d, mp_file_read_at_dv, mp_file_read_at_c, mp_file_read_at_cv, &
     716              :          mp_file_read_at_z, mp_file_read_at_zv, mp_file_read_at_l, mp_file_read_at_lv
     717              : 
     718              :       PROCEDURE, PRIVATE, PASS(fh), NON_OVERRIDABLE :: mp_file_read_at_all_ch, mp_file_read_at_all_chv, &
     719              :          mp_file_read_at_all_i, mp_file_read_at_all_iv, mp_file_read_at_all_l, mp_file_read_at_all_lv, &
     720              :          mp_file_read_at_all_r, mp_file_read_at_all_rv, mp_file_read_at_all_d, mp_file_read_at_all_dv, &
     721              :          mp_file_read_at_all_c, mp_file_read_at_all_cv, mp_file_read_at_all_z, mp_file_read_at_all_zv
     722              :       GENERIC, PUBLIC :: read_at_all => mp_file_read_at_all_ch, mp_file_read_at_all_chv, &
     723              :          mp_file_read_at_all_i, mp_file_read_at_all_iv, mp_file_read_at_all_l, mp_file_read_at_all_lv, &
     724              :          mp_file_read_at_all_r, mp_file_read_at_all_rv, mp_file_read_at_all_d, mp_file_read_at_all_dv, &
     725              :          mp_file_read_at_all_c, mp_file_read_at_all_cv, mp_file_read_at_all_z, mp_file_read_at_all_zv
     726              : 
     727              :       PROCEDURE, PUBLIC, PASS(fh), NON_OVERRIDABLE :: open => mp_file_open
     728              :       PROCEDURE, PUBLIC, PASS(fh), NON_OVERRIDABLE :: close => mp_file_close
     729              :       PROCEDURE, PRIVATE, PASS(fh_new), NON_OVERRIDABLE :: mp_file_assign
     730              :       GENERIC, PUBLIC :: ASSIGNMENT(=) => mp_file_assign
     731              : 
     732              :       PROCEDURE, PUBLIC, PASS(fh), NON_OVERRIDABLE :: get_size => mp_file_get_size
     733              :       PROCEDURE, PUBLIC, PASS(fh), NON_OVERRIDABLE :: get_position => mp_file_get_position
     734              : 
     735              :       PROCEDURE, PUBLIC, PASS(fh), NON_OVERRIDABLE :: read_all => mp_file_read_all_chv
     736              :       PROCEDURE, PUBLIC, PASS(fh), NON_OVERRIDABLE :: write_all => mp_file_write_all_chv
     737              :    END TYPE
     738              : 
     739              :    TYPE mp_info_type
     740              :       PRIVATE
     741              :       MPI_INFO_TYPE :: handle = mp_info_null_handle
     742              :    CONTAINS
     743              :       PROCEDURE, NON_OVERRIDABLE :: set_handle => mp_info_type_set_handle
     744              :       PROCEDURE, NON_OVERRIDABLE :: get_handle => mp_info_type_get_handle
     745              :       PROCEDURE, PRIVATE, NON_OVERRIDABLE :: mp_info_op_eq
     746              :       PROCEDURE, PRIVATE, NON_OVERRIDABLE :: mp_info_op_neq
     747              :       GENERIC, PUBLIC :: OPERATOR(==) => mp_info_op_eq
     748              :       GENERIC, PUBLIC :: OPERATOR(/=) => mp_info_op_neq
     749              :    END TYPE
     750              : 
     751              :    TYPE, EXTENDS(mp_comm_type) :: mp_cart_type
     752              :       INTEGER, DIMENSION(:), ALLOCATABLE, PUBLIC :: mepos_cart, num_pe_cart
     753              :       LOGICAL, DIMENSION(:), ALLOCATABLE, PUBLIC :: periodic
     754              :    CONTAINS
     755              :       PROCEDURE, PUBLIC, PASS(comm_cart), NON_OVERRIDABLE :: create => mp_cart_create
     756              :       PROCEDURE, PUBLIC, PASS(sub_comm), NON_OVERRIDABLE :: from_sub => mp_cart_sub
     757              : 
     758              :       PROCEDURE, PRIVATE, PASS(comm), NON_OVERRIDABLE :: get_info_cart => mp_cart_get
     759              : 
     760              :       PROCEDURE, PUBLIC, PASS(comm), NON_OVERRIDABLE :: coords => mp_cart_coords
     761              :       PROCEDURE, PUBLIC, PASS(comm), NON_OVERRIDABLE :: rank_cart => mp_cart_rank
     762              :    END TYPE
     763              : 
     764              : ! **************************************************************************************************
     765              : !> \brief stores all the informations relevant to an mpi environment
     766              : !> \param owns_group if it owns the group (and thus should free it when
     767              : !>        this object is deallocated)
     768              : !> \param ref_count the reference count, when it is zero this object gets
     769              : !>        deallocated
     770              : !> \par History
     771              : !>      08.2002 created [fawzi]
     772              : !> \author Fawzi Mohamed
     773              : ! **************************************************************************************************
     774              :    TYPE, EXTENDS(mp_comm_type) :: mp_para_env_type
     775              :       PRIVATE
     776              :       ! We set it to true to have less initialization steps in case we create a new communicator
     777              :       LOGICAL :: owns_group = .TRUE.
     778              :       INTEGER :: ref_count = -1
     779              :    CONTAINS
     780              :       PROCEDURE, PUBLIC, PASS(para_env), NON_OVERRIDABLE :: retain => mp_para_env_retain
     781              :       PROCEDURE, PUBLIC, PASS(para_env), NON_OVERRIDABLE :: is_valid => mp_para_env_is_valid
     782              :    END TYPE mp_para_env_type
     783              : 
     784              : ! **************************************************************************************************
     785              : !> \brief represent a pointer to a para env (to build arrays)
     786              : !> \param para_env the pointer to the para_env
     787              : !> \par History
     788              : !>      07.2003 created [fawzi]
     789              : !> \author Fawzi Mohamed
     790              : ! **************************************************************************************************
     791              :    TYPE mp_para_env_p_type
     792              :       TYPE(mp_para_env_type), POINTER :: para_env => NULL()
     793              :    END TYPE mp_para_env_p_type
     794              : 
     795              : ! **************************************************************************************************
     796              : !> \brief represent a multidimensional parallel environment
     797              : !> \param mepos_cart the position of the actual processor
     798              : !> \param num_pe_cart number of processors in the group in each dimension
     799              : !> \param source_cart id of a special processor (for example the one for i-o,
     800              : !>        or the master
     801              : !> \param owns_group if it owns the group (and thus should free it when
     802              : !>        this object is deallocated)
     803              : !> \param ref_count the reference count, when it is zero this object gets
     804              : !>        deallocated
     805              : !> \note
     806              : !>      not yet implemented for mpi
     807              : !> \par History
     808              : !>      08.2002 created [fawzi]
     809              : !> \author Fawzi Mohamed
     810              : ! **************************************************************************************************
     811              :    TYPE, EXTENDS(mp_cart_type) :: mp_para_cart_type
     812              :       PRIVATE
     813              :       ! We set it to true to have less initialization steps in case we create a new communicator
     814              :       LOGICAL :: owns_group = .TRUE.
     815              :       INTEGER :: ref_count = -1
     816              :    CONTAINS
     817              :       PROCEDURE, PUBLIC, PASS(cart), NON_OVERRIDABLE :: retain => mp_para_cart_retain
     818              :       PROCEDURE, PUBLIC, PASS(cart), NON_OVERRIDABLE :: is_valid => mp_para_cart_is_valid
     819              :    END TYPE mp_para_cart_type
     820              : 
     821              :    ! Create the constants from the corresponding handles
     822              :    TYPE(mp_comm_type), PARAMETER, PUBLIC :: mp_comm_null = mp_comm_type(mp_comm_null_handle)
     823              :    TYPE(mp_comm_type), PARAMETER, PUBLIC :: mp_comm_self = mp_comm_type(mp_comm_self_handle)
     824              :    TYPE(mp_comm_type), PARAMETER, PUBLIC :: mp_comm_world = mp_comm_type(mp_comm_world_handle)
     825              :    TYPE(mp_request_type), PARAMETER, PUBLIC :: mp_request_null = mp_request_type(mp_request_null_handle)
     826              :    TYPE(mp_win_type), PARAMETER, PUBLIC :: mp_win_null = mp_win_type(mp_win_null_handle)
     827              :    TYPE(mp_file_type), PARAMETER, PUBLIC :: mp_file_null = mp_file_type(mp_file_null_handle)
     828              :    TYPE(mp_info_type), PARAMETER, PUBLIC :: mp_info_null = mp_info_type(mp_info_null_handle)
     829              : 
     830              :    ! Bundles an MPI communicator split type with the info hint it requires
     831              :    TYPE mp_split_type
     832              :       PRIVATE
     833              :       INTEGER            :: split_type = mp_comm_split_type_shared_id
     834              :       TYPE(mp_info_type) :: info = mp_info_null
     835              :    END TYPE
     836              : 
     837              :    ! Node-local (shared-memory) split; needs no info hint.
     838              :    TYPE(mp_split_type), PARAMETER, PUBLIC :: mp_comm_split_type_shared = &
     839              :                                              mp_split_type(mp_comm_split_type_shared_id, mp_info_null)
     840              : 
     841              : #if !defined(__parallel)
     842              :    ! This communicator is to be used in serial mode to emulate a valid communicator which is not a compiler constant
     843              :    INTEGER, PARAMETER, PRIVATE :: mp_comm_default_handle = 1
     844              :    TYPE(mp_comm_type), PARAMETER, PRIVATE :: mp_comm_default = mp_comm_type(mp_comm_default_handle)
     845              : #endif
     846              : 
     847              :    ! Constants to compare communicators
     848              :    INTEGER, PARAMETER, PUBLIC :: mp_comm_ident = 0
     849              :    INTEGER, PARAMETER, PUBLIC :: mp_comm_congruent = 1
     850              :    INTEGER, PARAMETER, PUBLIC :: mp_comm_similar = 2
     851              :    INTEGER, PARAMETER, PUBLIC :: mp_comm_unequal = 3
     852              :    INTEGER, PARAMETER, PUBLIC :: mp_comm_compare_default = -1
     853              : 
     854              :    ! init and error
     855              :    PUBLIC :: mp_world_init, mp_world_finalize, mp_world_owns_mpi, mp_query_thread_level
     856              :    PUBLIC :: mp_abort
     857              : 
     858              :    ! informational / generation of sub comms
     859              :    PUBLIC :: mp_dims_create
     860              :    PUBLIC :: cp2k_is_parallel
     861              : 
     862              :    ! message passing
     863              :    PUBLIC :: mp_waitall, mp_waitany
     864              :    PUBLIC :: mp_testall, mp_testany
     865              : 
     866              :    ! Memory management
     867              :    PUBLIC :: mp_allocate, mp_deallocate
     868              : 
     869              :    ! I/O
     870              :    PUBLIC :: mp_file_delete
     871              :    PUBLIC :: mp_file_get_amode
     872              : 
     873              :    ! some 'advanced types' currently only used for dbcsr
     874              :    PUBLIC :: mp_type_descriptor_type
     875              :    PUBLIC :: mp_type_make
     876              :    PUBLIC :: mp_type_size
     877              : 
     878              :    ! vector types
     879              :    PUBLIC :: mp_type_indexed_make_r, mp_type_indexed_make_d, &
     880              :              mp_type_indexed_make_c, mp_type_indexed_make_z
     881              : 
     882              :    ! More I/O types and routines: variable spaced data using bytes for spacings
     883              :    PUBLIC :: mp_file_descriptor_type
     884              :    PUBLIC :: mp_file_type_free
     885              :    PUBLIC :: mp_file_type_hindexed_make_chv
     886              :    PUBLIC :: mp_file_type_set_view_chv
     887              : 
     888              :    PUBLIC :: mp_get_library_version
     889              : 
     890              :    ! assumed to be private
     891              : 
     892              :    INTERFACE mp_waitall
     893              :       MODULE PROCEDURE mp_waitall_1, mp_waitall_2
     894              :    END INTERFACE
     895              : 
     896              :    INTERFACE mp_testall
     897              :       MODULE PROCEDURE mp_testall_tv
     898              :    END INTERFACE
     899              : 
     900              :    INTERFACE mp_testany
     901              :       MODULE PROCEDURE mp_testany_1, mp_testany_2
     902              :    END INTERFACE
     903              : 
     904              :    INTERFACE mp_type_free
     905              :       MODULE PROCEDURE mp_type_free_m, mp_type_free_v
     906              :    END INTERFACE
     907              : 
     908              :    !
     909              :    ! interfaces to deal easily with scalars / vectors / matrices / ...
     910              :    ! of the different types (integers, doubles, logicals, characters)
     911              :    !
     912              :    INTERFACE mp_allocate
     913              :       MODULE PROCEDURE mp_allocate_i, &
     914              :          mp_allocate_l, &
     915              :          mp_allocate_r, &
     916              :          mp_allocate_d, &
     917              :          mp_allocate_c, &
     918              :          mp_allocate_z
     919              :    END INTERFACE
     920              : 
     921              :    INTERFACE mp_deallocate
     922              :       MODULE PROCEDURE mp_deallocate_i, &
     923              :          mp_deallocate_l, &
     924              :          mp_deallocate_r, &
     925              :          mp_deallocate_d, &
     926              :          mp_deallocate_c, &
     927              :          mp_deallocate_z
     928              :    END INTERFACE
     929              : 
     930              :    INTERFACE mp_type_make
     931              :       MODULE PROCEDURE mp_type_make_struct
     932              :       MODULE PROCEDURE mp_type_make_i, mp_type_make_l, &
     933              :          mp_type_make_r, mp_type_make_d, &
     934              :          mp_type_make_c, mp_type_make_z
     935              :    END INTERFACE
     936              : 
     937              :    INTERFACE mp_alloc_mem
     938              :       MODULE PROCEDURE mp_alloc_mem_i, mp_alloc_mem_l, &
     939              :          mp_alloc_mem_d, mp_alloc_mem_z, &
     940              :          mp_alloc_mem_r, mp_alloc_mem_c
     941              :    END INTERFACE
     942              : 
     943              :    INTERFACE mp_free_mem
     944              :       MODULE PROCEDURE mp_free_mem_i, mp_free_mem_l, &
     945              :          mp_free_mem_d, mp_free_mem_z, &
     946              :          mp_free_mem_r, mp_free_mem_c
     947              :    END INTERFACE
     948              : 
     949              : ! Type declarations
     950              :    TYPE mp_indexing_meta_type
     951              :       INTEGER, DIMENSION(:), POINTER :: index => NULL(), chunks => NULL()
     952              :    END TYPE mp_indexing_meta_type
     953              : 
     954              :    TYPE mp_type_descriptor_type
     955              :       MPI_DATA_TYPE :: type_handle = mp_datatype_null_handle
     956              :       INTEGER :: length = -1
     957              : #if defined(__parallel)
     958              :       INTEGER(kind=mpi_address_kind) :: base = -1
     959              : #endif
     960              :       INTEGER(kind=int_4), DIMENSION(:), POINTER :: data_i => NULL()
     961              :       INTEGER(kind=int_8), DIMENSION(:), POINTER :: data_l => NULL()
     962              :       REAL(kind=real_4), DIMENSION(:), POINTER :: data_r => NULL()
     963              :       REAL(kind=real_8), DIMENSION(:), POINTER :: data_d => NULL()
     964              :       COMPLEX(kind=real_4), DIMENSION(:), POINTER :: data_c => NULL()
     965              :       COMPLEX(kind=real_8), DIMENSION(:), POINTER :: data_z => NULL()
     966              :       TYPE(mp_type_descriptor_type), DIMENSION(:), POINTER :: subtype => NULL()
     967              :       INTEGER :: vector_descriptor(2) = -1
     968              :       LOGICAL :: has_indexing = .FALSE.
     969              :       TYPE(mp_indexing_meta_type) :: index_descriptor = mp_indexing_meta_type()
     970              :    END TYPE mp_type_descriptor_type
     971              : 
     972              :    TYPE mp_file_indexing_meta_type
     973              :       INTEGER, DIMENSION(:), POINTER   :: index => NULL()
     974              :       INTEGER(kind=file_offset), &
     975              :          DIMENSION(:), POINTER         :: chunks => NULL()
     976              :    END TYPE mp_file_indexing_meta_type
     977              : 
     978              :    TYPE mp_file_descriptor_type
     979              :       MPI_DATA_TYPE :: type_handle = mp_datatype_null_handle
     980              :       INTEGER                          :: length = -1
     981              :       LOGICAL                          :: has_indexing = .FALSE.
     982              :       TYPE(mp_file_indexing_meta_type) :: index_descriptor = mp_file_indexing_meta_type()
     983              :    END TYPE
     984              : 
     985              :    ! we make some assumptions on the length of INTEGERS, REALS and LOGICALS
     986              :    INTEGER, PARAMETER :: intlen = BIT_SIZE(0)/8
     987              :    INTEGER, PARAMETER :: reallen = 8
     988              :    INTEGER, PARAMETER :: loglen = BIT_SIZE(0)/8
     989              :    INTEGER, PARAMETER :: charlen = 1
     990              : 
     991              :    LOGICAL, PUBLIC, SAVE :: mp_collect_timings = .FALSE.
     992              : 
     993              : CONTAINS
     994              : 
     995              :    #:mute
     996              :       #:set types = ["comm", "request", "win", "file", "info"]
     997              :    #:endmute
     998              :    #:for type in types
     999     10151445 :       LOGICAL FUNCTION mp_${type}$_op_eq(${type}$1, ${type}$2)
    1000              :          CLASS(mp_${type}$_type), INTENT(IN) :: ${type}$1, ${type}$2
    1001              : #if defined(__parallel) && defined(__MPI_F08)
    1002     10151445 :          mp_${type}$_op_eq = (${type}$1%handle%mpi_val == ${type}$2%handle%mpi_val)
    1003              : #else
    1004              :          mp_${type}$_op_eq = (${type}$1%handle == ${type}$2%handle)
    1005              : #endif
    1006              :       END FUNCTION mp_${type}$_op_eq
    1007     10151445 : 
    1008      3968868 :       LOGICAL FUNCTION mp_${type}$_op_neq(${type}$1, ${type}$2)
    1009              :          CLASS(mp_${type}$_type), INTENT(IN) :: ${type}$1, ${type}$2
    1010              : #if defined(__parallel) && defined(__MPI_F08)
    1011      3968868 :          mp_${type}$_op_neq = (${type}$1%handle%mpi_val /= ${type}$2%handle%mpi_val)
    1012              : #else
    1013              :          mp_${type}$_op_neq = (${type}$1%handle /= ${type}$2%handle)
    1014              : #endif
    1015              :       END FUNCTION mp_${type}$_op_neq
    1016      3968868 : 
    1017     10713394 :       ELEMENTAL #{if type=="comm"}#IMPURE #{endif}#SUBROUTINE mp_${type}$_type_set_handle(this, handle #{if type=="comm"}#, ndims#{endif}#)
    1018              :       CLASS(mp_${type}$_type), INTENT(INOUT) :: this
    1019              :       INTEGER, INTENT(IN) :: handle
    1020              :       #:if type=="comm"
    1021              :          INTEGER, INTENT(IN), OPTIONAL :: ndims
    1022              :       #:endif
    1023              : 
    1024              : #if defined(__parallel) && defined(__MPI_F08)
    1025     10713394 :       this%handle%mpi_val = handle
    1026              : #else
    1027              :       this%handle = handle
    1028              : #endif
    1029              : 
    1030              :       #:if type=="comm"
    1031              :          SELECT TYPE (this)
    1032              :          CLASS IS (mp_cart_type)
    1033            0 :             IF (.NOT. PRESENT(ndims)) &
    1034              :                CALL cp_abort(__LOCATION__, &
    1035            0 :                              "Setup of a cartesian communicator requires information on the number of dimensions!")
    1036              :          END SELECT
    1037     10709330 :          IF (PRESENT(ndims)) this%ndims = ndims
    1038     10709330 :          CALL this%init()
    1039              :       #:endif
    1040              : 
    1041     10713394 :       END SUBROUTINE mp_${type}$_type_set_handle
    1042              : 
    1043      2941550 :       ELEMENTAL FUNCTION mp_${type}$_type_get_handle(this) RESULT(handle)
    1044              :          CLASS(mp_${type}$_type), INTENT(IN) :: this
    1045              :          INTEGER :: handle
    1046              : 
    1047              : #if defined(__parallel) && defined(__MPI_F08)
    1048      2941550 :          handle = this%handle%mpi_val
    1049              : #else
    1050              :          handle = this%handle
    1051              : #endif
    1052      2941550 :       END FUNCTION mp_${type}$_type_get_handle
    1053              :       #:endfor
    1054              : 
    1055         5738 :       FUNCTION mp_comm_get_tag_ub(comm) RESULT(tag_ub)
    1056              :          CLASS(mp_comm_type), INTENT(IN) :: comm
    1057              :          INTEGER :: tag_ub
    1058              : 
    1059              : #if defined(__parallel)
    1060              :          INTEGER :: ierr
    1061              :          LOGICAL :: flag
    1062              :          INTEGER(KIND=MPI_ADDRESS_KIND) :: attrval
    1063              : 
    1064         5738 :          CALL MPI_COMM_GET_ATTR(comm%handle, MPI_TAG_UB, attrval, flag, ierr)
    1065         5738 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_comm_get_attr @ mp_comm_get_tag_ub")
    1066         5738 :          IF (.NOT. flag) THEN
    1067              :             CALL cp_warn(__LOCATION__, "Upper bound of tags not available! "// &
    1068            0 :                          "Only the guaranteed minimum of 32767 is used.")
    1069            0 :             tag_ub = 32767
    1070              :          ELSE
    1071         5738 :             tag_ub = INT(attrval, KIND=KIND(tag_ub))
    1072              :          END IF
    1073              : #else
    1074              :          MARK_USED(comm)
    1075              :          tag_ub = HUGE(1)
    1076              : #endif
    1077         5738 :       END FUNCTION mp_comm_get_tag_ub
    1078              : 
    1079            0 :       FUNCTION mp_comm_get_host_rank(comm) RESULT(host_rank)
    1080              :          CLASS(mp_comm_type), INTENT(IN) :: comm
    1081              :          INTEGER :: host_rank
    1082              : 
    1083              : #if defined(__parallel)
    1084              :          INTEGER :: ierr
    1085              :          LOGICAL :: flag
    1086              :          INTEGER(KIND=MPI_ADDRESS_KIND) :: attrval
    1087              : 
    1088            0 :          CALL MPI_COMM_GET_ATTR(comm%handle, MPI_HOST, attrval, flag, ierr)
    1089            0 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_comm_get_attr @ mp_comm_get_host_rank")
    1090            0 :          IF (.NOT. flag) CPABORT("Host process rank not available!")
    1091            0 :          host_rank = INT(attrval, KIND=KIND(host_rank))
    1092              : #else
    1093              :          MARK_USED(comm)
    1094              :          host_rank = 0
    1095              : #endif
    1096            0 :       END FUNCTION mp_comm_get_host_rank
    1097              : 
    1098            0 :       FUNCTION mp_comm_get_io_rank(comm) RESULT(io_rank)
    1099              :          CLASS(mp_comm_type), INTENT(IN) :: comm
    1100              :          INTEGER :: io_rank
    1101              : 
    1102              : #if defined(__parallel)
    1103              :          INTEGER :: ierr
    1104              :          LOGICAL :: flag
    1105              :          INTEGER(KIND=MPI_ADDRESS_KIND) :: attrval
    1106              : 
    1107            0 :          CALL MPI_COMM_GET_ATTR(comm%handle, MPI_IO, attrval, flag, ierr)
    1108            0 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_comm_get_attr @ mp_comm_get_io_rank")
    1109            0 :          IF (.NOT. flag) CPABORT("IO rank not available!")
    1110            0 :          io_rank = INT(attrval, KIND=KIND(io_rank))
    1111              : #else
    1112              :          MARK_USED(comm)
    1113              :          io_rank = 0
    1114              : #endif
    1115            0 :       END FUNCTION mp_comm_get_io_rank
    1116              : 
    1117            0 :       FUNCTION mp_comm_get_wtime_is_global(comm) RESULT(wtime_is_global)
    1118              :          CLASS(mp_comm_type), INTENT(IN) :: comm
    1119              :          LOGICAL :: wtime_is_global
    1120              : 
    1121              : #if defined(__parallel)
    1122              :          INTEGER :: ierr
    1123              :          LOGICAL :: flag
    1124              :          INTEGER(KIND=MPI_ADDRESS_KIND) :: attrval
    1125              : 
    1126            0 :          CALL MPI_COMM_GET_ATTR(comm%handle, MPI_TAG_UB, attrval, flag, ierr)
    1127            0 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_comm_get_attr @ mp_comm_get_wtime_is_global")
    1128            0 :          IF (.NOT. flag) CPABORT("Synchronization state of WTIME not available!")
    1129            0 :          wtime_is_global = (attrval == 1_MPI_ADDRESS_KIND)
    1130              : #else
    1131              :          MARK_USED(comm)
    1132              :          wtime_is_global = .TRUE.
    1133              : #endif
    1134            0 :       END FUNCTION mp_comm_get_wtime_is_global
    1135              : 
    1136              : ! **************************************************************************************************
    1137              : !> \brief Abort if an MPI lifecycle operation is attempted from an OpenMP parallel region.
    1138              : !> \param routine_name name of the calling routine
    1139              : ! **************************************************************************************************
    1140      9036079 :       SUBROUTINE mp_assert_outside_parallel(routine_name)
    1141              :          CHARACTER(LEN=*), INTENT(IN)                     :: routine_name
    1142              : 
    1143      9036079 :          IF (omp_in_parallel()) THEN
    1144              :             CALL cp_abort(__LOCATION__, &
    1145            0 :                           TRIM(routine_name)//" must be called outside an OpenMP parallel region.")
    1146              :          END IF
    1147      9036079 :       END SUBROUTINE mp_assert_outside_parallel
    1148              : 
    1149              : ! **************************************************************************************************
    1150              : !> \brief Abort unless MPI provides MPI_THREAD_MULTIPLE support.
    1151              : ! **************************************************************************************************
    1152         1508 :       SUBROUTINE mp_assert_thread_multiple()
    1153              : #if defined(__parallel)
    1154         1508 :          IF (mp_thread_level_provided < MPI_THREAD_MULTIPLE) THEN
    1155              :             CALL cp_abort(__LOCATION__, &
    1156              :                           "CP2K requires MPI_THREAD_MULTIPLE, but MPI provides "// &
    1157            0 :                           TRIM(mp_get_thread_level_name())//".")
    1158              :          END IF
    1159              : #endif
    1160         1508 :       END SUBROUTINE mp_assert_thread_multiple
    1161              : 
    1162              : ! **************************************************************************************************
    1163              : !> \brief Initialize MPI or attach to an MPI environment provided by an embedding application.
    1164              : !> \param initialize_mpi whether CP2K should initialize MPI when it is not active yet
    1165              : ! **************************************************************************************************
    1166         3016 :       SUBROUTINE mp_initialize_thread_support(initialize_mpi)
    1167              :          LOGICAL, INTENT(IN)                              :: initialize_mpi
    1168              : #if defined(__parallel)
    1169              :          INTEGER                                          :: ierr
    1170              :          LOGICAL                                          :: finalized, initialized
    1171              : 
    1172         1508 :          CALL mpi_finalized(finalized, ierr)
    1173         1508 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_finalized @ mp_initialize_thread_support")
    1174         1508 :          IF (finalized) THEN
    1175            0 :             CALL cp_abort(__LOCATION__, "MPI has already been finalized.")
    1176              :          END IF
    1177              : 
    1178         1508 :          CALL mpi_initialized(initialized, ierr)
    1179         1508 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_initialized @ mp_initialize_thread_support")
    1180         1508 :          IF (.NOT. initialized) THEN
    1181         1508 :             IF (.NOT. initialize_mpi) THEN
    1182              :                CALL cp_abort(__LOCATION__, &
    1183            0 :                              "MPI must be initialized by the embedding application before CP2K.")
    1184              :             END IF
    1185         1508 :             CALL mpi_init_thread(MPI_THREAD_MULTIPLE, mp_thread_level_provided, ierr)
    1186         1508 :             IF (ierr /= 0) CALL mp_stop(ierr, "mpi_init_thread @ mp_initialize_thread_support")
    1187         1508 :             mp_mpi_initialized_by_cp2k = .TRUE.
    1188              :          ELSE
    1189            0 :             CALL mpi_query_thread(mp_thread_level_provided, ierr)
    1190            0 :             IF (ierr /= 0) CALL mp_stop(ierr, "mpi_query_thread @ mp_initialize_thread_support")
    1191              :          END IF
    1192         1508 :          CALL mp_assert_thread_multiple()
    1193              : #else
    1194              :          MARK_USED(initialize_mpi)
    1195              :          mp_thread_level_provided = -1
    1196              : #endif
    1197         1508 :       END SUBROUTINE mp_initialize_thread_support
    1198              : 
    1199              : ! **************************************************************************************************
    1200              : !> \brief Query the MPI thread-support level when MPI is managed by an embedding application.
    1201              : ! **************************************************************************************************
    1202            0 :       SUBROUTINE mp_query_thread_level()
    1203            0 :          CALL mp_assert_outside_parallel("mp_query_thread_level")
    1204            0 :          CALL mp_initialize_thread_support(.FALSE.)
    1205            0 :       END SUBROUTINE mp_query_thread_level
    1206              : 
    1207              : ! **************************************************************************************************
    1208              : !> \brief Check if MPI was initialized by CP2K and will thus be finalized by CP2K.
    1209              : !> \return true if CP2K owns MPI
    1210              : ! **************************************************************************************************
    1211         1486 :       LOGICAL FUNCTION mp_world_owns_mpi()
    1212              : #if defined(__parallel)
    1213         1486 :          mp_world_owns_mpi = mp_mpi_initialized_by_cp2k
    1214              : #else
    1215              :          mp_world_owns_mpi = .FALSE.
    1216              : #endif
    1217         1486 :       END FUNCTION mp_world_owns_mpi
    1218              : 
    1219              : ! **************************************************************************************************
    1220              : !> \brief Return a human-readable name for the observed MPI thread-support level.
    1221              : !> \return MPI thread-support level name
    1222              : ! **************************************************************************************************
    1223            0 :       CHARACTER(LEN=32) FUNCTION mp_get_thread_level_name()
    1224              : #if defined(__parallel)
    1225            0 :          SELECT CASE (mp_thread_level_provided)
    1226              :          CASE (MPI_THREAD_SINGLE)
    1227            0 :             mp_get_thread_level_name = "MPI_THREAD_SINGLE"
    1228              :          CASE (MPI_THREAD_FUNNELED)
    1229            0 :             mp_get_thread_level_name = "MPI_THREAD_FUNNELED"
    1230              :          CASE (MPI_THREAD_SERIALIZED)
    1231            0 :             mp_get_thread_level_name = "MPI_THREAD_SERIALIZED"
    1232              :          CASE (MPI_THREAD_MULTIPLE)
    1233            0 :             mp_get_thread_level_name = "MPI_THREAD_MULTIPLE"
    1234              :          CASE DEFAULT
    1235            0 :             mp_get_thread_level_name = "not initialized"
    1236              :          END SELECT
    1237              : #else
    1238              :          mp_get_thread_level_name = "not applicable"
    1239              : #endif
    1240            0 :       END FUNCTION mp_get_thread_level_name
    1241              : 
    1242              : ! **************************************************************************************************
    1243              : !> \brief initializes the system default communicator
    1244              : !> \param mp_comm [output] : handle of the default communicator
    1245              : !> \par History
    1246              : !>      2.2004 created [Joost VandeVondele ]
    1247              : !> \note
    1248              : !>      While active, this routine may only be called once. If MPI is already initialized,
    1249              : !>      CP2K validates MPI_THREAD_MULTIPLE and attaches without taking ownership.
    1250              : ! **************************************************************************************************
    1251         1508 :       SUBROUTINE mp_world_init(mp_comm)
    1252              :          CLASS(mp_comm_type), INTENT(OUT)                     :: mp_comm
    1253              : #if defined(__parallel)
    1254              :          INTEGER                                  :: ierr
    1255              : #if defined(__MIMIC)
    1256              :          INTEGER                                  :: env_status, mimic_handle
    1257              :          INTEGER(KIND=MPI_ADDRESS_KIND)           :: appnum
    1258              :          LOGICAL                                  :: flag, mimic_requested
    1259              : #endif
    1260              : #endif
    1261              : 
    1262         1508 :          CALL mp_assert_outside_parallel("mp_world_init")
    1263         1508 :          IF (mp_world_is_initialized) THEN
    1264            0 :             CALL cp_abort(__LOCATION__, "mp_world_init called while already initialized.")
    1265              :          END IF
    1266         1508 :          CALL mp_initialize_thread_support(.TRUE.)
    1267              : #if defined(__parallel)
    1268         1508 :          CALL mpi_comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN, ierr)
    1269         1508 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_comm_set_errhandler @ mp_world_init")
    1270              : #endif
    1271         1508 :          debug_comm_count = 1
    1272         1508 :          mp_comm = mp_comm_world
    1273              : #if defined(__MIMIC)
    1274              :          ! Initialize MCL only for MiMiC runs, since MCL creates communicators it never frees.
    1275              :          ! A MiMiC run is requested either explicitly by setting MCL_COMM_MODE or implicitly
    1276              :          ! by launching CP2K as a client (MPI_APPNUM > 0) in an MPMD run (MCL default mode).
    1277         1508 :          CALL get_environment_variable("MCL_COMM_MODE", status=env_status)
    1278         1508 :          CALL mpi_comm_get_attr(MPI_COMM_WORLD, MPI_APPNUM, appnum, flag, ierr)
    1279         1508 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_comm_get_attr @ mp_world_init")
    1280         1508 :          mimic_requested = (env_status == 0)
    1281         1508 :          IF (flag) mimic_requested = mimic_requested .OR. (appnum > 0_MPI_ADDRESS_KIND)
    1282            0 :          IF (mimic_requested) THEN
    1283            0 :             mimic_handle = mp_comm%get_handle()
    1284            0 :             CALL mcl_initialize(mimic_handle)
    1285            0 :             CALL mp_comm%set_handle(mimic_handle)
    1286              : #if defined(__MPI_F08)
    1287            0 :             mimic_comm_world%mpi_val = mimic_handle
    1288              : #else
    1289              :             mimic_comm_world = mimic_handle
    1290              : #endif
    1291              :          ELSE
    1292         1508 :             mimic_comm_world = MPI_COMM_WORLD
    1293              :          END IF
    1294              : #endif
    1295         1508 :          CALL mp_comm%init()
    1296         1508 :          CALL add_mp_perf_env()
    1297         1508 :          mp_world_is_initialized = .TRUE.
    1298         1508 :       END SUBROUTINE mp_world_init
    1299              : 
    1300              : ! **************************************************************************************************
    1301              : !> \brief re-create the system default communicator with a different MPI
    1302              : !>        rank order
    1303              : !> \param mp_comm [output] : handle of the default communicator
    1304              : !> \param mp_new_comm ...
    1305              : !> \param ranks_order ...
    1306              : !> \par History
    1307              : !>      1.2012 created [ Christiane Pousa ]
    1308              : !> \note
    1309              : !>      should only be called once, at very beginning of CP2K run
    1310              : ! **************************************************************************************************
    1311          764 :       SUBROUTINE mp_reordering(mp_comm, mp_new_comm, ranks_order)
    1312              :          CLASS(mp_comm_type), INTENT(IN)                      :: mp_comm
    1313              :          CLASS(mp_comm_type), INTENT(out)                     :: mp_new_comm
    1314              :          INTEGER, DIMENSION(:), CONTIGUOUS, INTENT(IN)                    :: ranks_order
    1315              : 
    1316              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_reordering'
    1317              : 
    1318              :          INTEGER                                  :: handle, ierr
    1319              : #if defined(__parallel)
    1320              :          MPI_GROUP_TYPE                                  :: newgroup, oldgroup
    1321              : #endif
    1322              : 
    1323          764 :          CALL mp_assert_outside_parallel(routineN)
    1324          764 :          CALL mp_timeset(routineN, handle)
    1325              :          ierr = 0
    1326              : #if defined(__parallel)
    1327              : 
    1328          764 :          CALL mpi_comm_group(mp_comm%handle, oldgroup, ierr)
    1329          764 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_comm_group @ mp_reordering")
    1330          764 :          CALL mpi_group_incl(oldgroup, SIZE(ranks_order), ranks_order, newgroup, ierr)
    1331          764 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_group_incl @ mp_reordering")
    1332              : 
    1333          764 :          CALL mpi_comm_create(mp_comm%handle, newgroup, mp_new_comm%handle, ierr)
    1334          764 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_comm_create @ mp_reordering")
    1335              : 
    1336          764 :          CALL mpi_group_free(oldgroup, ierr)
    1337          764 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_group_free @ mp_reordering")
    1338          764 :          CALL mpi_group_free(newgroup, ierr)
    1339          764 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_group_free @ mp_reordering")
    1340              : 
    1341          764 :          CALL add_perf(perf_id=1, count=1)
    1342              : #else
    1343              :          MARK_USED(mp_comm)
    1344              :          MARK_USED(ranks_order)
    1345              :          mp_new_comm%handle = mp_comm_default_handle
    1346              : #endif
    1347          764 :          debug_comm_count = debug_comm_count + 1
    1348          764 :          CALL mp_new_comm%init()
    1349          764 :          CALL mp_timestop(handle)
    1350          764 :       END SUBROUTINE mp_reordering
    1351              : 
    1352              : ! **************************************************************************************************
    1353              : !> \brief Finalize the system default communicator and MPI when CP2K owns MPI.
    1354              : !> \par History
    1355              : !>      2.2004 created [Joost VandeVondele]
    1356              : ! **************************************************************************************************
    1357         3012 :       SUBROUTINE mp_world_finalize()
    1358              : 
    1359              :          CHARACTER(LEN=default_string_length) :: debug_comm_count_char
    1360              : #if defined(__parallel)
    1361              :          INTEGER                              :: ierr
    1362              : #endif
    1363              : 
    1364         1506 :          CALL mp_assert_outside_parallel("mp_world_finalize")
    1365         1506 :          IF (.NOT. mp_world_is_initialized) THEN
    1366              :             CALL cp_abort(__LOCATION__, &
    1367            0 :                           "mp_world_finalize called without a matching mp_world_init.")
    1368              :          END IF
    1369              : #if defined(__parallel)
    1370              : #if defined(__MIMIC)
    1371         1506 :          CALL mpi_barrier(mimic_comm_world, ierr)
    1372              : #else
    1373              :          CALL mpi_barrier(MPI_COMM_WORLD, ierr) ! call mpi directly to avoid 0 stack pointer
    1374              : #endif
    1375              : #endif
    1376         1506 :          CALL rm_mp_perf_env()
    1377              : 
    1378         1506 :          debug_comm_count = debug_comm_count - 1
    1379              : #if defined(__parallel)
    1380         1506 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_barrier @ mp_world_finalize")
    1381              : #endif
    1382         1506 :          IF (debug_comm_count /= 0) THEN
    1383              :             ! A bug, we're leaking or double-freeing communicators. Needs to be fixed where the leak happens.
    1384              :             ! Memory leak checking might be helpful to locate the culprit
    1385            0 :             WRITE (unit=debug_comm_count_char, FMT='(I2)') debug_comm_count
    1386              :             CALL cp_abort(__LOCATION__, "mp_world_finalize: assert failed:"// &
    1387            0 :                           " leaking communicators "//ADJUSTL(TRIM(debug_comm_count_char)))
    1388              :          END IF
    1389              : #if defined(__parallel)
    1390         1506 :          IF (mp_mpi_initialized_by_cp2k) THEN
    1391         1506 :             CALL mpi_finalize(ierr)
    1392         1506 :             IF (ierr /= 0) CALL mp_stop(ierr, "mpi_finalize @ mp_world_finalize")
    1393              :          END IF
    1394              : #endif
    1395         1506 :          mp_thread_level_provided = -1
    1396              : #if defined(__parallel)
    1397         1506 :          mp_mpi_initialized_by_cp2k = .FALSE.
    1398              : #endif
    1399         1506 :          mp_world_is_initialized = .FALSE.
    1400              : 
    1401         1506 :       END SUBROUTINE mp_world_finalize
    1402              : 
    1403              : ! all the following routines should work for a given communicator, not MPI_WORLD
    1404              : 
    1405              : ! **************************************************************************************************
    1406              : !> \brief globally stops all tasks
    1407              : !>       this is intended to be low level, most of CP2K should call cp_abort()
    1408              : ! **************************************************************************************************
    1409            2 :       SUBROUTINE mp_abort()
    1410              :          INTEGER                                            :: ierr
    1411              : #if defined(__MIMIC)
    1412              :          LOGICAL                                            :: mcl_initialized
    1413              : #endif
    1414              : 
    1415            2 :          ierr = 0
    1416              : 
    1417              : #if !defined(__NO_ABORT)
    1418              : #if defined(__parallel)
    1419              : #if defined(__MIMIC)
    1420              :          CALL mcl_is_initialized(mcl_initialized)
    1421              :          IF (mcl_initialized) CALL mcl_abort(1, ierr)
    1422              : #endif
    1423              :          CALL mpi_abort(MPI_COMM_WORLD, 1, ierr)
    1424              : #else
    1425              :          CALL m_abort()
    1426              : #endif
    1427              : #endif
    1428              :          ! this routine never returns and levels with non-zero exit code
    1429            2 :          STOP 1
    1430              :       END SUBROUTINE mp_abort
    1431              : 
    1432              : ! **************************************************************************************************
    1433              : !> \brief stops *after an mpi error* translating the error code
    1434              : !> \param ierr an error code * returned by an mpi call *
    1435              : !> \param prg_code ...
    1436              : !> \note
    1437              : !>       this function is private to message_passing.F
    1438              : ! **************************************************************************************************
    1439            0 :       SUBROUTINE mp_stop(ierr, prg_code)
    1440              :          INTEGER, INTENT(IN)                        :: ierr
    1441              :          CHARACTER(LEN=*), INTENT(IN)               :: prg_code
    1442              : 
    1443              : #if defined(__parallel)
    1444              :          INTEGER                                    :: istat, len
    1445              :          CHARACTER(LEN=MPI_MAX_ERROR_STRING)        :: error_string
    1446              :          CHARACTER(LEN=MPI_MAX_ERROR_STRING + 512)  :: full_error
    1447              : #else
    1448              :          CHARACTER(LEN=512)                         :: full_error
    1449              : #endif
    1450              : 
    1451              : #if defined(__parallel)
    1452            0 :          CALL mpi_error_string(ierr, error_string, len, istat)
    1453            0 :          WRITE (full_error, '(A,I0,A)') ' MPI error ', ierr, ' in '//TRIM(prg_code)//' : '//error_string(1:len)
    1454              : #else
    1455              :          WRITE (full_error, '(A,I0,A)') ' MPI error (!?) ', ierr, ' in '//TRIM(prg_code)
    1456              : #endif
    1457              : 
    1458            0 :          CPABORT(full_error)
    1459              : 
    1460            0 :       END SUBROUTINE mp_stop
    1461              : 
    1462              : ! **************************************************************************************************
    1463              : !> \brief synchronizes with a barrier a given group of mpi tasks
    1464              : !> \param group mpi communicator
    1465              : ! **************************************************************************************************
    1466      1895366 :       SUBROUTINE mp_sync(comm)
    1467              :          CLASS(mp_comm_type), INTENT(IN)                                :: comm
    1468              : 
    1469              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_sync'
    1470              : 
    1471              :          INTEGER                                            :: handle, ierr
    1472              : 
    1473              :          ierr = 0
    1474       947683 :          CALL mp_timeset(routineN, handle)
    1475              : 
    1476              : #if defined(__parallel)
    1477       947683 :          CALL mpi_barrier(comm%handle, ierr)
    1478       947683 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_barrier @ mp_sync")
    1479       947683 :          CALL add_perf(perf_id=5, count=1)
    1480              : #else
    1481              :          MARK_USED(comm)
    1482              : #endif
    1483       947683 :          CALL mp_timestop(handle)
    1484              : 
    1485       947683 :       END SUBROUTINE mp_sync
    1486              : 
    1487              : ! **************************************************************************************************
    1488              : !> \brief synchronizes with a barrier a given group of mpi tasks
    1489              : !> \param comm mpi communicator
    1490              : !> \param request ...
    1491              : ! **************************************************************************************************
    1492            0 :       SUBROUTINE mp_isync(comm, request)
    1493              :          CLASS(mp_comm_type), INTENT(IN)                    :: comm
    1494              :          TYPE(mp_request_type), INTENT(OUT)                 :: request
    1495              : 
    1496              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_isync'
    1497              : 
    1498              :          INTEGER                                            :: handle, ierr
    1499              : 
    1500              :          ierr = 0
    1501            0 :          CALL mp_timeset(routineN, handle)
    1502              : 
    1503              : #if defined(__parallel)
    1504            0 :          CALL mpi_ibarrier(comm%handle, request%handle, ierr)
    1505            0 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_ibarrier @ mp_isync")
    1506            0 :          CALL add_perf(perf_id=26, count=1)
    1507              : #else
    1508              :          MARK_USED(comm)
    1509              :          request = mp_request_null
    1510              : #endif
    1511            0 :          CALL mp_timestop(handle)
    1512              : 
    1513            0 :       END SUBROUTINE mp_isync
    1514              : 
    1515              : ! **************************************************************************************************
    1516              : !> \brief returns task id for a given mpi communicator
    1517              : !> \param taskid The ID of the communicator
    1518              : !> \param comm mpi communicator
    1519              : ! **************************************************************************************************
    1520     52461914 :       SUBROUTINE mp_comm_rank(taskid, comm)
    1521              : 
    1522              :          INTEGER, INTENT(OUT)                               :: taskid
    1523              :          CLASS(mp_comm_type), INTENT(IN)                    :: comm
    1524              : 
    1525              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_comm_rank'
    1526              : 
    1527              :          INTEGER                                            :: handle
    1528              : #if defined(__parallel)
    1529              :          INTEGER :: ierr
    1530              : #endif
    1531              : 
    1532     26230957 :          CALL mp_timeset(routineN, handle)
    1533              : 
    1534              : #if defined(__parallel)
    1535     26230957 :          CALL mpi_comm_rank(comm%handle, taskid, ierr)
    1536     26230957 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_comm_rank @ mp_comm_rank")
    1537              : #else
    1538              :          MARK_USED(comm)
    1539              :          taskid = 0
    1540              : #endif
    1541     26230957 :          CALL mp_timestop(handle)
    1542              : 
    1543     26230957 :       END SUBROUTINE mp_comm_rank
    1544              : 
    1545              : ! **************************************************************************************************
    1546              : !> \brief returns number of tasks for a given mpi communicator
    1547              : !> \param numtask ...
    1548              : !> \param comm mpi communicator
    1549              : ! **************************************************************************************************
    1550     52461914 :       SUBROUTINE mp_comm_size(numtask, comm)
    1551              : 
    1552              :          INTEGER, INTENT(OUT)                               :: numtask
    1553              :          CLASS(mp_comm_type), INTENT(IN)                    :: comm
    1554              : 
    1555              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_comm_size'
    1556              : 
    1557              :          INTEGER                                            :: handle
    1558              : #if defined(__parallel)
    1559              :          INTEGER :: ierr
    1560              : #endif
    1561              : 
    1562     26230957 :          CALL mp_timeset(routineN, handle)
    1563              : 
    1564              : #if defined(__parallel)
    1565     26230957 :          CALL mpi_comm_size(comm%handle, numtask, ierr)
    1566     26230957 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_comm_rank @ mp_comm_size")
    1567              : #else
    1568              :          MARK_USED(comm)
    1569              :          numtask = 1
    1570              : #endif
    1571     26230957 :          CALL mp_timestop(handle)
    1572              : 
    1573     26230957 :       END SUBROUTINE mp_comm_size
    1574              : 
    1575              : ! **************************************************************************************************
    1576              : !> \brief returns info for a given Cartesian MPI communicator
    1577              : !> \param comm ...
    1578              : !> \param ndims ...
    1579              : !> \param dims ...
    1580              : !> \param task_coor ...
    1581              : !> \param periods ...
    1582              : ! **************************************************************************************************
    1583     12653602 :       SUBROUTINE mp_cart_get(comm, dims, task_coor, periods)
    1584              : 
    1585              :          CLASS(mp_cart_type), INTENT(IN)                    :: comm
    1586              :          INTEGER, INTENT(OUT), OPTIONAL                     :: dims(comm%ndims), task_coor(comm%ndims)
    1587              :          LOGICAL, INTENT(out), OPTIONAL                     :: periods(comm%ndims)
    1588              : 
    1589              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_cart_get'
    1590              : 
    1591              :          INTEGER                                            :: handle
    1592              : #if defined(__parallel)
    1593              :          INTEGER :: ierr
    1594     25307204 :          INTEGER                               :: my_dims(comm%ndims), my_task_coor(comm%ndims)
    1595     25307204 :          LOGICAL                               :: my_periods(comm%ndims)
    1596              : #endif
    1597              : 
    1598     12653602 :          CALL mp_timeset(routineN, handle)
    1599              : 
    1600              : #if defined(__parallel)
    1601     12653602 :          CALL mpi_cart_get(comm%handle, comm%ndims, my_dims, my_periods, my_task_coor, ierr)
    1602     12653602 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_cart_get @ mp_cart_get")
    1603     50615510 :          IF (PRESENT(dims)) dims = my_dims
    1604     50615510 :          IF (PRESENT(task_coor)) task_coor = my_task_coor
    1605     50615510 :          IF (PRESENT(periods)) periods = my_periods
    1606              : #else
    1607              :          MARK_USED(comm)
    1608              :          IF (PRESENT(task_coor)) task_coor = 0
    1609              :          IF (PRESENT(dims)) dims = 1
    1610              :          IF (PRESENT(periods)) periods = .FALSE.
    1611              : #endif
    1612     12653602 :          CALL mp_timestop(handle)
    1613              : 
    1614     12653602 :       END SUBROUTINE mp_cart_get
    1615              : 
    1616            0 :       INTEGER ELEMENTAL FUNCTION mp_comm_get_ndims(comm)
    1617              :          CLASS(mp_comm_type), INTENT(IN) :: comm
    1618              : 
    1619            0 :          mp_comm_get_ndims = comm%ndims
    1620              : 
    1621            0 :       END FUNCTION
    1622              : 
    1623              : ! **************************************************************************************************
    1624              : !> \brief creates a cartesian communicator from any communicator
    1625              : !> \param comm_old ...
    1626              : !> \param ndims ...
    1627              : !> \param dims ...
    1628              : !> \param pos ...
    1629              : !> \param comm_cart ...
    1630              : ! **************************************************************************************************
    1631      1786923 :       SUBROUTINE mp_cart_create(comm_old, ndims, dims, comm_cart)
    1632              : 
    1633              :          CLASS(mp_comm_type), INTENT(IN) :: comm_old
    1634              :          INTEGER, INTENT(IN)                      :: ndims
    1635              :          INTEGER, INTENT(INOUT)                   :: dims(ndims)
    1636              :          CLASS(mp_cart_type), INTENT(OUT) :: comm_cart
    1637              : 
    1638              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_cart_create'
    1639              : 
    1640              :          INTEGER                                  :: handle, ierr
    1641              : #if defined(__parallel)
    1642      1786923 :          LOGICAL, DIMENSION(1:ndims)              :: period
    1643              :          LOGICAL                                  :: reorder
    1644              : #endif
    1645              : 
    1646      1786923 :          ierr = 0
    1647      1786923 :          CALL mp_assert_outside_parallel(routineN)
    1648      1786923 :          CALL mp_timeset(routineN, handle)
    1649              : 
    1650      1786923 :          comm_cart%handle = comm_old%handle
    1651              : #if defined(__parallel)
    1652              : 
    1653      5361085 :          IF (ANY(dims == 0)) CALL mpi_dims_create(comm_old%num_pe, ndims, dims, ierr)
    1654      1786923 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_dims_create @ mp_cart_create")
    1655              : 
    1656              :          ! FIX ME.  Quick hack to avoid problems with realspace grids for compilers
    1657              :          ! like IBM that actually reorder the processors when creating the new
    1658              :          ! communicator
    1659      1786923 :          reorder = .FALSE.
    1660      5361889 :          period = .TRUE.
    1661              :          CALL mpi_cart_create(comm_old%handle, ndims, dims, period, reorder, comm_cart%handle, &
    1662      1786923 :                               ierr)
    1663      1786923 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_cart_create @ mp_cart_create")
    1664      1786923 :          CALL add_perf(perf_id=1, count=1)
    1665              : #else
    1666              :          dims = 1
    1667              :          comm_cart%handle = mp_comm_default_handle
    1668              : #endif
    1669      1786923 :          comm_cart%ndims = ndims
    1670      1786923 :          debug_comm_count = debug_comm_count + 1
    1671      1786923 :          CALL comm_cart%init()
    1672      1786923 :          CALL mp_timestop(handle)
    1673              : 
    1674      1786923 :       END SUBROUTINE mp_cart_create
    1675              : 
    1676              : ! **************************************************************************************************
    1677              : !> \brief wrapper to MPI_Cart_coords
    1678              : !> \param comm ...
    1679              : !> \param rank ...
    1680              : !> \param coords ...
    1681              : ! **************************************************************************************************
    1682        79794 :       SUBROUTINE mp_cart_coords(comm, rank, coords)
    1683              : 
    1684              :          CLASS(mp_cart_type), INTENT(IN) :: comm
    1685              :          INTEGER, INTENT(IN)                                :: rank
    1686              :          INTEGER, DIMENSION(:), INTENT(OUT)                 :: coords
    1687              : 
    1688              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_cart_coords'
    1689              : 
    1690              :          INTEGER                                            :: handle, ierr, m
    1691              : 
    1692              :          ierr = 0
    1693        79794 :          CALL mp_timeset(routineN, handle)
    1694              : 
    1695        79794 :          m = SIZE(coords)
    1696              : #if defined(__parallel)
    1697        79794 :          CALL mpi_cart_coords(comm%handle, rank, m, coords, ierr)
    1698        79794 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_cart_coords @ mp_cart_coords")
    1699              : #else
    1700              :          coords = 0
    1701              :          MARK_USED(rank)
    1702              :          MARK_USED(comm)
    1703              : #endif
    1704        79794 :          CALL mp_timestop(handle)
    1705              : 
    1706        79794 :       END SUBROUTINE mp_cart_coords
    1707              : 
    1708              : ! **************************************************************************************************
    1709              : !> \brief wrapper to MPI_Comm_compare
    1710              : !> \param comm1 ...
    1711              : !> \param comm2 ...
    1712              : !> \param res ...
    1713              : ! **************************************************************************************************
    1714         4628 :       FUNCTION mp_comm_compare(comm1, comm2) RESULT(res)
    1715              : 
    1716              :          CLASS(mp_comm_type), INTENT(IN)                    :: comm1, comm2
    1717              :          INTEGER                                            :: res
    1718              : 
    1719              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_comm_compare'
    1720              : 
    1721              :          INTEGER                                            :: handle
    1722              : #if defined(__parallel)
    1723              :          INTEGER :: ierr, iout
    1724              : #endif
    1725              : 
    1726         2314 :          CALL mp_timeset(routineN, handle)
    1727              : 
    1728         2314 :          res = 0
    1729              : #if defined(__parallel)
    1730         2314 :          CALL mpi_comm_compare(comm1%handle, comm2%handle, iout, ierr)
    1731         2314 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_comm_compare @ mp_comm_compare")
    1732              :          SELECT CASE (iout)
    1733              :          CASE (MPI_IDENT)
    1734         2314 :             res = mp_comm_ident
    1735              :          CASE (MPI_CONGRUENT)
    1736         2314 :             res = mp_comm_congruent
    1737              :          CASE (MPI_SIMILAR)
    1738            0 :             res = mp_comm_similar
    1739              :          CASE (MPI_UNEQUAL)
    1740            0 :             res = mp_comm_unequal
    1741              :          CASE default
    1742         2314 :             CPABORT("Unknown comparison state of the communicators!")
    1743              :          END SELECT
    1744              : #else
    1745              :          MARK_USED(comm1)
    1746              :          MARK_USED(comm2)
    1747              : #endif
    1748         2314 :          CALL mp_timestop(handle)
    1749              : 
    1750         2314 :       END FUNCTION mp_comm_compare
    1751              : 
    1752              : ! **************************************************************************************************
    1753              : !> \brief wrapper to MPI_Cart_sub
    1754              : !> \param comm ...
    1755              : !> \param rdim ...
    1756              : !> \param sub_comm ...
    1757              : ! **************************************************************************************************
    1758         2276 :       SUBROUTINE mp_cart_sub(comm, rdim, sub_comm)
    1759              : 
    1760              :          CLASS(mp_cart_type), INTENT(IN)                                :: comm
    1761              :          LOGICAL, DIMENSION(:), CONTIGUOUS, INTENT(IN)                  :: rdim
    1762              :          CLASS(mp_cart_type), INTENT(OUT)                               :: sub_comm
    1763              : 
    1764              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_cart_sub'
    1765              : 
    1766              :          INTEGER                                            :: handle
    1767              : #if defined(__parallel)
    1768              :          INTEGER :: ierr
    1769              : #endif
    1770              : 
    1771         2276 :          CALL mp_assert_outside_parallel(routineN)
    1772         2276 :          CALL mp_timeset(routineN, handle)
    1773              : 
    1774              : #if defined(__parallel)
    1775         2276 :          CALL mpi_cart_sub(comm%handle, rdim, sub_comm%handle, ierr)
    1776         2276 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_cart_sub @ mp_cart_sub")
    1777              : #else
    1778              :          MARK_USED(comm)
    1779              :          MARK_USED(rdim)
    1780              :          sub_comm%handle = mp_comm_default_handle
    1781              : #endif
    1782         9104 :          sub_comm%ndims = COUNT(rdim)
    1783         2276 :          debug_comm_count = debug_comm_count + 1
    1784         2276 :          CALL sub_comm%init()
    1785         2276 :          CALL mp_timestop(handle)
    1786              : 
    1787         2276 :       END SUBROUTINE mp_cart_sub
    1788              : 
    1789              : ! **************************************************************************************************
    1790              : !> \brief wrapper to MPI_Comm_free
    1791              : !> \param comm ...
    1792              : ! **************************************************************************************************
    1793      4002902 :       SUBROUTINE mp_comm_free(comm)
    1794              : 
    1795              :          CLASS(mp_comm_type), INTENT(INOUT)                 :: comm
    1796              : 
    1797              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_comm_free'
    1798              : 
    1799              :          INTEGER                                            :: handle
    1800              :          LOGICAL :: free_comm
    1801              : #if defined(__parallel)
    1802              :          INTEGER :: ierr
    1803              : #endif
    1804              : 
    1805      4002902 :          CALL mp_assert_outside_parallel(routineN)
    1806      4002902 :          free_comm = .TRUE.
    1807              :          SELECT TYPE (comm)
    1808              :          CLASS IS (mp_para_env_type)
    1809       976164 :             free_comm = .FALSE.
    1810       976164 :             IF (comm%ref_count <= 0) &
    1811            0 :                CPABORT("para_env%ref_count <= 0")
    1812       976164 :             comm%ref_count = comm%ref_count - 1
    1813       976164 :             IF (comm%ref_count <= 0) THEN
    1814        47861 :                free_comm = comm%owns_group
    1815              :             END IF
    1816              :          CLASS IS (mp_para_cart_type)
    1817          158 :             free_comm = .FALSE.
    1818          158 :             IF (comm%ref_count <= 0) &
    1819            0 :                CPABORT("para_cart%ref_count <= 0")
    1820          158 :             comm%ref_count = comm%ref_count - 1
    1821          158 :             IF (comm%ref_count <= 0) THEN
    1822          158 :                free_comm = comm%owns_group
    1823              :             END IF
    1824              :          END SELECT
    1825              : 
    1826      4002902 :          CALL mp_timeset(routineN, handle)
    1827              : 
    1828      4002902 :          IF (free_comm) THEN
    1829              : #if defined(__parallel)
    1830      3047558 :             CALL mpi_comm_free(comm%handle, ierr)
    1831      3047558 :             IF (ierr /= 0) CALL mp_stop(ierr, "mpi_comm_free @ mp_comm_free")
    1832              : #else
    1833              :             comm%handle = mp_comm_null_handle
    1834              : #endif
    1835      3047558 :             debug_comm_count = debug_comm_count - 1
    1836              :          END IF
    1837              : 
    1838              :          SELECT TYPE (comm)
    1839              :          CLASS IS (mp_cart_type)
    1840      2256363 :             DEALLOCATE (comm%periodic, comm%mepos_cart, comm%num_pe_cart)
    1841              :          END SELECT
    1842              : 
    1843      4002902 :          CALL mp_timestop(handle)
    1844              : 
    1845      4002902 :       END SUBROUTINE mp_comm_free
    1846              : 
    1847              : ! **************************************************************************************************
    1848              : !> \brief check whether the environment exists
    1849              : !> \param para_env ...
    1850              : !> \return ...
    1851              : ! **************************************************************************************************
    1852       979848 :       ELEMENTAL LOGICAL FUNCTION mp_para_env_is_valid(para_env)
    1853              :          CLASS(mp_para_env_type), INTENT(IN) :: para_env
    1854              : 
    1855       979848 :          mp_para_env_is_valid = para_env%ref_count > 0
    1856              : 
    1857       979848 :       END FUNCTION mp_para_env_is_valid
    1858              : 
    1859              : ! **************************************************************************************************
    1860              : !> \brief increase the reference counter but ensure that you free it later
    1861              : !> \param para_env ...
    1862              : ! **************************************************************************************************
    1863       928338 :       ELEMENTAL IMPURE SUBROUTINE mp_para_env_retain(para_env)
    1864              :          CLASS(mp_para_env_type), INTENT(INOUT) :: para_env
    1865              : 
    1866       928338 :          CALL mp_assert_outside_parallel("mp_para_env_retain")
    1867       928338 :          para_env%ref_count = para_env%ref_count + 1
    1868              : 
    1869       928338 :       END SUBROUTINE mp_para_env_retain
    1870              : 
    1871              : ! **************************************************************************************************
    1872              : !> \brief check whether the given environment is valid, i.e. existent
    1873              : !> \param cart ...
    1874              : !> \return ...
    1875              : ! **************************************************************************************************
    1876          158 :       ELEMENTAL LOGICAL FUNCTION mp_para_cart_is_valid(cart)
    1877              :          CLASS(mp_para_cart_type), INTENT(IN) :: cart
    1878              : 
    1879          158 :          mp_para_cart_is_valid = cart%ref_count > 0
    1880              : 
    1881          158 :       END FUNCTION mp_para_cart_is_valid
    1882              : 
    1883              : ! **************************************************************************************************
    1884              : !> \brief increase the reference counter, don't forget to free it later
    1885              : !> \param cart ...
    1886              : ! **************************************************************************************************
    1887            0 :       ELEMENTAL IMPURE SUBROUTINE mp_para_cart_retain(cart)
    1888              :          CLASS(mp_para_cart_type), INTENT(INOUT) :: cart
    1889              : 
    1890            0 :          CALL mp_assert_outside_parallel("mp_para_cart_retain")
    1891            0 :          cart%ref_count = cart%ref_count + 1
    1892              : 
    1893            0 :       END SUBROUTINE mp_para_cart_retain
    1894              : 
    1895              : ! **************************************************************************************************
    1896              : !> \brief wrapper to MPI_Comm_dup
    1897              : !> \param comm1 ...
    1898              : !> \param comm2 ...
    1899              : ! **************************************************************************************************
    1900       508857 :       SUBROUTINE mp_comm_dup(comm1, comm2)
    1901              : 
    1902              :          CLASS(mp_comm_type), INTENT(IN)                    :: comm1
    1903              :          CLASS(mp_comm_type), INTENT(OUT)                   :: comm2
    1904              : 
    1905              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_comm_dup'
    1906              : 
    1907              :          INTEGER                                            :: handle
    1908              : #if defined(__parallel)
    1909              :          INTEGER :: ierr
    1910              : #endif
    1911              : 
    1912       508857 :          CALL mp_assert_outside_parallel(routineN)
    1913       508857 :          CALL mp_timeset(routineN, handle)
    1914              : 
    1915              : #if defined(__parallel)
    1916       508857 :          CALL mpi_comm_dup(comm1%handle, comm2%handle, ierr)
    1917       508857 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_comm_dup @ mp_comm_dup")
    1918              : #else
    1919              :          MARK_USED(comm1)
    1920              :          comm2%handle = mp_comm_default_handle
    1921              : #endif
    1922       508857 :          comm2%ndims = comm1%ndims
    1923       508857 :          debug_comm_count = debug_comm_count + 1
    1924       508857 :          CALL comm2%init()
    1925       508857 :          CALL mp_timestop(handle)
    1926              : 
    1927       508857 :       END SUBROUTINE mp_comm_dup
    1928              : 
    1929              : ! **************************************************************************************************
    1930              : !> \brief Implements a simple assignment function to overload the assignment operator
    1931              : !> \param comm_new communicator on the r.h.s. of the assignment operator
    1932              : !> \param comm_old communicator on the l.h.s. of the assignment operator
    1933              : ! **************************************************************************************************
    1934     12641530 :       ELEMENTAL IMPURE SUBROUTINE mp_comm_assign(comm_new, comm_old)
    1935              :          CLASS(mp_comm_type), INTENT(IN) :: comm_old
    1936              :          CLASS(mp_comm_type), INTENT(OUT) :: comm_new
    1937              : 
    1938     12641530 :          comm_new%handle = comm_old%handle
    1939     12641530 :          comm_new%ndims = comm_old%ndims
    1940     12641530 :          CALL comm_new%init(.FALSE.)
    1941     12641530 :       END SUBROUTINE
    1942              : 
    1943              : ! **************************************************************************************************
    1944              : !> \brief check whether the local process is the source process
    1945              : !> \param para_env ...
    1946              : !> \return ...
    1947              : ! **************************************************************************************************
    1948     18987584 :       ELEMENTAL LOGICAL FUNCTION mp_comm_is_source(comm)
    1949              :          CLASS(mp_comm_type), INTENT(IN) :: comm
    1950              : 
    1951     18987584 :          mp_comm_is_source = comm%source == comm%mepos
    1952              : 
    1953     18987584 :       END FUNCTION mp_comm_is_source
    1954              : 
    1955              : ! **************************************************************************************************
    1956              : !> \brief Initializes the communicator (mostly relevant for its derived classes)
    1957              : !> \param comm ...
    1958              : ! **************************************************************************************************
    1959     26399945 :       ELEMENTAL IMPURE SUBROUTINE mp_comm_init(comm, owns_group)
    1960              :          CLASS(mp_comm_type), INTENT(INOUT) :: comm
    1961              :          LOGICAL, INTENT(IN), OPTIONAL :: owns_group
    1962              : 
    1963              :          ! Only reference-counted communicators update ownership state during initialization.
    1964              :          SELECT TYPE (comm)
    1965              :          CLASS IS (mp_para_env_type)
    1966        49371 :             CALL mp_assert_outside_parallel("mp_comm_init")
    1967              :          CLASS IS (mp_para_cart_type)
    1968          158 :             CALL mp_assert_outside_parallel("mp_comm_init")
    1969              :          END SELECT
    1970              : 
    1971     26399945 :          IF (comm%handle MPI_GET_COMP /= mp_comm_null_handle MPI_GET_COMP) THEN
    1972     26220470 :             comm%source = 0
    1973     26220470 :             CALL comm%get_size(comm%num_pe)
    1974     26220470 :             CALL comm%get_rank(comm%mepos)
    1975              :          END IF
    1976              : 
    1977              :          SELECT TYPE (comm)
    1978              :          CLASS IS (mp_cart_type)
    1979     12653602 :             IF (ALLOCATED(comm%periodic)) DEALLOCATE (comm%periodic)
    1980     12653602 :             IF (ALLOCATED(comm%mepos_cart)) DEALLOCATE (comm%mepos_cart)
    1981     12653602 :             IF (ALLOCATED(comm%num_pe_cart)) DEALLOCATE (comm%num_pe_cart)
    1982              : 
    1983              :             ASSOCIATE (ndims => comm%ndims)
    1984              : 
    1985            0 :                ALLOCATE (comm%periodic(ndims), comm%mepos_cart(ndims), &
    1986     63268010 :                          comm%num_pe_cart(ndims))
    1987              :             END ASSOCIATE
    1988              : 
    1989     37961908 :             comm%mepos_cart = 0
    1990     37961908 :             comm%periodic = .FALSE.
    1991     12653602 :             IF (comm%handle MPI_GET_COMP /= mp_comm_null_handle MPI_GET_COMP) THEN
    1992              :                CALL comm%get_info_cart(comm%num_pe_cart, comm%mepos_cart, &
    1993     12653602 :                                        comm%periodic)
    1994              :             END IF
    1995              :          END SELECT
    1996              : 
    1997              :          SELECT TYPE (comm)
    1998              :          CLASS IS (mp_para_env_type)
    1999        49371 :             IF (PRESENT(owns_group)) comm%owns_group = owns_group
    2000        49371 :             comm%ref_count = 1
    2001              :          CLASS IS (mp_para_cart_type)
    2002          158 :             IF (PRESENT(owns_group)) comm%owns_group = owns_group
    2003          158 :             comm%ref_count = 1
    2004              :          END SELECT
    2005              : 
    2006     26399945 :       END SUBROUTINE
    2007              : 
    2008              : ! **************************************************************************************************
    2009              : !> \brief creates a new para environment
    2010              : !> \param para_env the new parallel environment
    2011              : !> \param group the id of the actual mpi_group
    2012              : !> \par History
    2013              : !>      08.2002 created [fawzi]
    2014              : !> \author Fawzi Mohamed
    2015              : ! **************************************************************************************************
    2016            0 :       SUBROUTINE mp_para_env_create(para_env, group)
    2017              :          TYPE(mp_para_env_type), POINTER        :: para_env
    2018              :          CLASS(mp_comm_type), INTENT(in)        :: group
    2019              : 
    2020            0 :          CALL mp_assert_outside_parallel("mp_para_env_create")
    2021            0 :          IF (ASSOCIATED(para_env)) &
    2022            0 :             CPABORT("The passed para_env must not be associated!")
    2023            0 :          ALLOCATE (para_env)
    2024            0 :          para_env%mp_comm_type = group
    2025            0 :          CALL para_env%init()
    2026            0 :       END SUBROUTINE mp_para_env_create
    2027              : 
    2028              : ! **************************************************************************************************
    2029              : !> \brief releases the para object (to be called when you don't want anymore
    2030              : !>      the shared copy of this object)
    2031              : !> \param para_env the new group
    2032              : !> \par History
    2033              : !>      08.2002 created [fawzi]
    2034              : !> \author Fawzi Mohamed
    2035              : !> \note
    2036              : !>      to avoid circular dependencies cp_log_handling has a private copy
    2037              : !>      of this method (see cp_log_handling:my_mp_para_env_release)!
    2038              : ! **************************************************************************************************
    2039      1003355 :       SUBROUTINE mp_para_env_release(para_env)
    2040              :          TYPE(mp_para_env_type), POINTER                    :: para_env
    2041              : 
    2042      1003355 :          CALL mp_assert_outside_parallel("mp_para_env_release")
    2043      1003355 :          IF (ASSOCIATED(para_env)) THEN
    2044       965208 :             CALL para_env%free()
    2045       965208 :             IF (.NOT. para_env%is_valid()) DEALLOCATE (para_env)
    2046              :          END IF
    2047      1003355 :          NULLIFY (para_env)
    2048      1003355 :       END SUBROUTINE mp_para_env_release
    2049              : 
    2050              : ! **************************************************************************************************
    2051              : !> \brief creates a cart (multidimensional parallel environment)
    2052              : !> \param cart the cart environment to create
    2053              : !> \param group the mpi communicator
    2054              : !> \author fawzi
    2055              : ! **************************************************************************************************
    2056            0 :       SUBROUTINE mp_para_cart_create(cart, group)
    2057              :          TYPE(mp_para_cart_type), POINTER, INTENT(OUT)      :: cart
    2058              :          CLASS(mp_comm_type), INTENT(in)                    :: group
    2059              : 
    2060            0 :          CALL mp_assert_outside_parallel("mp_para_cart_create")
    2061            0 :          IF (ASSOCIATED(cart)) &
    2062            0 :             CPABORT("The passed para_cart must not be associated!")
    2063            0 :          ALLOCATE (cart)
    2064            0 :          cart%mp_cart_type = group
    2065            0 :          CALL cart%init()
    2066              : 
    2067            0 :       END SUBROUTINE mp_para_cart_create
    2068              : 
    2069              : ! **************************************************************************************************
    2070              : !> \brief releases the given cart
    2071              : !> \param cart the cart to release
    2072              : !> \author fawzi
    2073              : ! **************************************************************************************************
    2074          158 :       SUBROUTINE mp_para_cart_release(cart)
    2075              :          TYPE(mp_para_cart_type), POINTER                   :: cart
    2076              : 
    2077          158 :          CALL mp_assert_outside_parallel("mp_para_cart_release")
    2078          158 :          IF (ASSOCIATED(cart)) THEN
    2079          158 :             CALL cart%free()
    2080          158 :             IF (.NOT. cart%is_valid()) DEALLOCATE (cart)
    2081              :          END IF
    2082          158 :          NULLIFY (cart)
    2083          158 :       END SUBROUTINE mp_para_cart_release
    2084              : 
    2085              : ! **************************************************************************************************
    2086              : !> \brief wrapper to MPI_Group_translate_ranks
    2087              : !> \param comm1 ...
    2088              : !> \param comm2 ...
    2089              : !> \param rank ...
    2090              : ! **************************************************************************************************
    2091      3743173 :       SUBROUTINE mp_rank_compare(comm1, comm2, rank)
    2092              : 
    2093              :          CLASS(mp_comm_type), INTENT(IN)                      :: comm1, comm2
    2094              :          INTEGER, DIMENSION(:), CONTIGUOUS, INTENT(OUT)       :: rank
    2095              : 
    2096              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_rank_compare'
    2097              : 
    2098              :          INTEGER                                  :: handle
    2099              : #if defined(__parallel)
    2100              :          INTEGER                                  :: i, ierr, n, n1, n2
    2101      3743173 :          INTEGER, ALLOCATABLE, DIMENSION(:)       :: rin
    2102              :          MPI_GROUP_TYPE :: g1, g2
    2103              : #endif
    2104              : 
    2105      3743173 :          CALL mp_timeset(routineN, handle)
    2106              : 
    2107     11229519 :          rank = 0
    2108              : #if defined(__parallel)
    2109      3743173 :          CALL mpi_comm_size(comm1%handle, n1, ierr)
    2110      3743173 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_comm_size @ mp_rank_compare")
    2111      3743173 :          CALL mpi_comm_size(comm2%handle, n2, ierr)
    2112      3743173 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_comm_size @ mp_rank_compare")
    2113      3743173 :          n = MAX(n1, n2)
    2114      3743173 :          CALL mpi_comm_group(comm1%handle, g1, ierr)
    2115      3743173 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_comm_group @ mp_rank_compare")
    2116      3743173 :          CALL mpi_comm_group(comm2%handle, g2, ierr)
    2117      3743173 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_comm_group @ mp_rank_compare")
    2118     11229519 :          ALLOCATE (rin(0:n - 1), STAT=ierr)
    2119      3743173 :          IF (ierr /= 0) &
    2120            0 :             CPABORT("allocate @ mp_rank_compare")
    2121     11229519 :          DO i = 0, n - 1
    2122     11229519 :             rin(i) = i
    2123              :          END DO
    2124      3743173 :          CALL mpi_group_translate_ranks(g1, n, rin, g2, rank, ierr)
    2125      3743173 :          IF (ierr /= 0) CALL mp_stop(ierr, &
    2126            0 :                                      "mpi_group_translate_rank @ mp_rank_compare")
    2127      3743173 :          CALL mpi_group_free(g1, ierr)
    2128      3743173 :          IF (ierr /= 0) &
    2129            0 :             CPABORT("group_free @ mp_rank_compare")
    2130      3743173 :          CALL mpi_group_free(g2, ierr)
    2131      3743173 :          IF (ierr /= 0) &
    2132            0 :             CPABORT("group_free @ mp_rank_compare")
    2133      3743173 :          DEALLOCATE (rin)
    2134              : #else
    2135              :          MARK_USED(comm1)
    2136              :          MARK_USED(comm2)
    2137              : #endif
    2138      3743173 :          CALL mp_timestop(handle)
    2139              : 
    2140     26202211 :       END SUBROUTINE mp_rank_compare
    2141              : 
    2142              : ! **************************************************************************************************
    2143              : !> \brief wrapper to MPI_Dims_create
    2144              : !> \param nodes ...
    2145              : !> \param dims ...
    2146              : ! **************************************************************************************************
    2147      1324098 :       SUBROUTINE mp_dims_create(nodes, dims)
    2148              : 
    2149              :          INTEGER, INTENT(IN)                                :: nodes
    2150              :          INTEGER, DIMENSION(:), INTENT(INOUT)               :: dims
    2151              : 
    2152              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_dims_create'
    2153              : 
    2154              :          INTEGER                                            :: handle, ndim
    2155              : #if defined(__parallel)
    2156              :          INTEGER :: ierr
    2157              : #endif
    2158              : 
    2159      1324098 :          CALL mp_timeset(routineN, handle)
    2160              : 
    2161      1324098 :          ndim = SIZE(dims)
    2162              : #if defined(__parallel)
    2163      1324098 :          IF (ANY(dims == 0)) CALL mpi_dims_create(nodes, ndim, dims, ierr)
    2164      1324098 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_dims_create @ mp_dims_create")
    2165              : #else
    2166              :          dims = 1
    2167              :          MARK_USED(nodes)
    2168              : #endif
    2169      1324098 :          CALL mp_timestop(handle)
    2170              : 
    2171      1324098 :       END SUBROUTINE mp_dims_create
    2172              : 
    2173              : ! **************************************************************************************************
    2174              : !> \brief wrapper to MPI_Cart_rank
    2175              : !> \param comm ...
    2176              : !> \param pos ...
    2177              : !> \param rank ...
    2178              : ! **************************************************************************************************
    2179     12364966 :       SUBROUTINE mp_cart_rank(comm, pos, rank)
    2180              :          CLASS(mp_cart_type), INTENT(IN)                    :: comm
    2181              :          INTEGER, DIMENSION(:), INTENT(IN)                  :: pos
    2182              :          INTEGER, INTENT(OUT)                               :: rank
    2183              : 
    2184              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_cart_rank'
    2185              : 
    2186              :          INTEGER                                            :: handle
    2187              : #if defined(__parallel)
    2188              :          INTEGER :: ierr
    2189              : #endif
    2190              : 
    2191     12364966 :          CALL mp_timeset(routineN, handle)
    2192              : 
    2193              : #if defined(__parallel)
    2194     12364966 :          CALL mpi_cart_rank(comm%handle, pos, rank, ierr)
    2195     12364966 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_cart_rank @ mp_cart_rank")
    2196              : #else
    2197              :          rank = 0
    2198              :          MARK_USED(comm)
    2199              :          MARK_USED(pos)
    2200              : #endif
    2201     12364966 :          CALL mp_timestop(handle)
    2202              : 
    2203     12364966 :       END SUBROUTINE mp_cart_rank
    2204              : 
    2205              : ! **************************************************************************************************
    2206              : !> \brief waits for completion of the given request
    2207              : !> \param request ...
    2208              : !> \par History
    2209              : !>      08.2003 created [f&j]
    2210              : !> \author joost & fawzi
    2211              : !> \note
    2212              : !>      see isendrecv
    2213              : ! **************************************************************************************************
    2214        13116 :       SUBROUTINE mp_wait(request)
    2215              :          CLASS(mp_request_type), INTENT(inout)              :: request
    2216              : 
    2217              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_wait'
    2218              : 
    2219              :          INTEGER                                            :: handle
    2220              : #if defined(__parallel)
    2221              :          INTEGER :: ierr
    2222              : #endif
    2223              : 
    2224         6558 :          CALL mp_timeset(routineN, handle)
    2225              : 
    2226              : #if defined(__parallel)
    2227              : 
    2228         6558 :          CALL mpi_wait(request%handle, MPI_STATUS_IGNORE, ierr)
    2229         6558 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_wait @ mp_wait")
    2230              : 
    2231         6558 :          CALL add_perf(perf_id=9, count=1)
    2232              : #else
    2233              :          request%handle = mp_request_null_handle
    2234              : #endif
    2235         6558 :          CALL mp_timestop(handle)
    2236         6558 :       END SUBROUTINE mp_wait
    2237              : 
    2238              : ! **************************************************************************************************
    2239              : !> \brief waits for completion of the given requests
    2240              : !> \param requests ...
    2241              : !> \par History
    2242              : !>      08.2003 created [f&j]
    2243              : !> \author joost & fawzi
    2244              : !> \note
    2245              : !>      see isendrecv
    2246              : ! **************************************************************************************************
    2247      1605943 :       SUBROUTINE mp_waitall_1(requests)
    2248              :          TYPE(mp_request_type), DIMENSION(:), INTENT(inout)     :: requests
    2249              : 
    2250              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_waitall_1'
    2251              : 
    2252              :          INTEGER                                  :: handle
    2253              : #if defined(__parallel)
    2254              :          INTEGER                                  :: count, ierr
    2255              : #endif
    2256              : 
    2257      1605943 :          CALL mp_timeset(routineN, handle)
    2258              : #if defined(__parallel)
    2259      1605943 :          count = SIZE(requests)
    2260      1605943 :          CALL mpi_waitall_internal(count, requests, ierr)
    2261      1605943 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_waitall @ mp_waitall_1")
    2262      1605943 :          CALL add_perf(perf_id=9, count=1)
    2263              : #else
    2264              :          requests = mp_request_null
    2265              : #endif
    2266      1605943 :          CALL mp_timestop(handle)
    2267      1605943 :       END SUBROUTINE mp_waitall_1
    2268              : 
    2269              : ! **************************************************************************************************
    2270              : !> \brief waits for completion of the given requests
    2271              : !> \param requests ...
    2272              : !> \par History
    2273              : !>      08.2003 created [f&j]
    2274              : !> \author joost & fawzi
    2275              : ! **************************************************************************************************
    2276       721821 :       SUBROUTINE mp_waitall_2(requests)
    2277              :          TYPE(mp_request_type), DIMENSION(:, :), INTENT(inout)  :: requests
    2278              : 
    2279              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_waitall_2'
    2280              : 
    2281              :          INTEGER                                  :: handle
    2282              : #if defined(__parallel)
    2283              :          INTEGER                                  :: count, ierr
    2284              : #endif
    2285              : 
    2286       721821 :          CALL mp_timeset(routineN, handle)
    2287              : #if defined(__parallel)
    2288      2165463 :          count = SIZE(requests)
    2289      3941571 :          CALL mpi_waitall_internal(count, requests, ierr)
    2290       721821 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_waitall @ mp_waitall_2")
    2291       721821 :          CALL add_perf(perf_id=9, count=1)
    2292              : #else
    2293              :          requests = mp_request_null
    2294              : #endif
    2295       721821 :          CALL mp_timestop(handle)
    2296       721821 :       END SUBROUTINE mp_waitall_2
    2297              : 
    2298              : ! **************************************************************************************************
    2299              : !> \brief wrapper needed to deal with interfaces as present in openmpi 1.8.1
    2300              : !>        the issue is with the rank or requests
    2301              : !> \param count ...
    2302              : !> \param array_of_requests ...
    2303              : !> \param ierr ...
    2304              : !> \author Joost VandeVondele
    2305              : ! **************************************************************************************************
    2306              : #if defined(__parallel)
    2307      2327764 :       SUBROUTINE mpi_waitall_internal(count, array_of_requests, ierr)
    2308              :          INTEGER, INTENT(in)                                      :: count
    2309              :          TYPE(mp_request_type), DIMENSION(count), INTENT(inout)   :: array_of_requests
    2310              :          INTEGER, INTENT(out)                                     :: ierr
    2311              : 
    2312      2327764 :          MPI_REQUEST_TYPE, ALLOCATABLE, DIMENSION(:), TARGET      :: request_handles
    2313              : 
    2314     20932336 :          ALLOCATE (request_handles(count), SOURCE=array_of_requests(1:count)%handle)
    2315      2327764 :          CALL mpi_waitall(count, request_handles, MPI_STATUSES_IGNORE, ierr)
    2316      8149594 :          array_of_requests(1:count)%handle = request_handles(:)
    2317      2327764 :          DEALLOCATE (request_handles)
    2318              : 
    2319      2327764 :       END SUBROUTINE mpi_waitall_internal
    2320              : #endif
    2321              : 
    2322              : ! **************************************************************************************************
    2323              : !> \brief waits for completion of any of the given requests
    2324              : !> \param requests ...
    2325              : !> \param completed ...
    2326              : !> \par History
    2327              : !>      09.2008 created
    2328              : !> \author Iain Bethune (c) The Numerical Algorithms Group (NAG) Ltd, 2008 on behalf of the HECToR project
    2329              : ! **************************************************************************************************
    2330        14440 :       SUBROUTINE mp_waitany(requests, completed)
    2331              :          TYPE(mp_request_type), DIMENSION(:), INTENT(inout)     :: requests
    2332              :          INTEGER, INTENT(out)                                   :: completed
    2333              : 
    2334              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_waitany'
    2335              : 
    2336              :          INTEGER                                      :: handle
    2337              : #if defined(__parallel)
    2338              :          INTEGER                                      :: count, ierr
    2339        14440 :          MPI_REQUEST_TYPE, ALLOCATABLE, DIMENSION(:)  :: request_handles
    2340              : #endif
    2341              : 
    2342        14440 :          CALL mp_timeset(routineN, handle)
    2343              : 
    2344              : #if defined(__parallel)
    2345        14440 :          count = SIZE(requests)
    2346              :          ! Convert CP2K's request_handles to the plain handle for the library
    2347       115520 :          ALLOCATE (request_handles(count), SOURCE=requests(1:count)%handle)
    2348              : 
    2349        14440 :          CALL mpi_waitany(count, request_handles, completed, MPI_STATUS_IGNORE, ierr)
    2350        14440 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_waitany @ mp_waitany")
    2351              : 
    2352              :          ! Convert the plain handles to CP2K handles
    2353        43320 :          requests(1:count)%handle = request_handles(:)
    2354        14440 :          DEALLOCATE (request_handles)
    2355        14440 :          CALL add_perf(perf_id=9, count=1)
    2356              : #else
    2357              :          requests = mp_request_null
    2358              :          completed = 1
    2359              : #endif
    2360        14440 :          CALL mp_timestop(handle)
    2361        28880 :       END SUBROUTINE mp_waitany
    2362              : 
    2363              : ! **************************************************************************************************
    2364              : !> \brief Tests for completion of the given requests.
    2365              : !> \brief We use mpi_test so that we can use a single status.
    2366              : !> \param requests the list of requests to test
    2367              : !> \return logical which determines if requests are complete
    2368              : !> \par History
    2369              : !>      3.2016 adapted to any shape [Nico Holmberg]
    2370              : !> \author Alfio Lazzaro
    2371              : ! **************************************************************************************************
    2372         6400 :       FUNCTION mp_testall_tv(requests) RESULT(flag)
    2373              :          TYPE(mp_request_type), DIMENSION(:), INTENT(INOUT) :: requests
    2374              :          LOGICAL                               :: flag
    2375              : 
    2376              : #if defined(__parallel)
    2377              :          INTEGER                               :: i, ierr
    2378              :          LOGICAL, DIMENSION(:), POINTER        :: flags
    2379              : #endif
    2380              : 
    2381         6400 :          flag = .TRUE.
    2382              : 
    2383              : #if defined(__parallel)
    2384        19200 :          ALLOCATE (flags(SIZE(requests)))
    2385        25600 :          DO i = 1, SIZE(requests)
    2386        19200 :             CALL mpi_test(requests(i)%handle, flags(i), MPI_STATUS_IGNORE, ierr)
    2387        19200 :             IF (ierr /= 0) CALL mp_stop(ierr, "mpi_testall @ mp_testall_tv")
    2388        45096 :             flag = flag .AND. flags(i)
    2389              :          END DO
    2390         6400 :          DEALLOCATE (flags)
    2391              : #else
    2392              :          requests = mp_request_null
    2393              : #endif
    2394         6400 :       END FUNCTION mp_testall_tv
    2395              : 
    2396              : ! **************************************************************************************************
    2397              : !> \brief Tests for completion of the given request.
    2398              : !> \param request the request
    2399              : !> \param flag logical which determines if the request is completed
    2400              : !> \par History
    2401              : !>      3.2016 created
    2402              : !> \author Nico Holmberg
    2403              : ! **************************************************************************************************
    2404          410 :       FUNCTION mp_test_1(request) RESULT(flag)
    2405              :          CLASS(mp_request_type), INTENT(inout)              :: request
    2406              :          LOGICAL                                            :: flag
    2407              : 
    2408              : #if defined(__parallel)
    2409              :          INTEGER                                            :: ierr
    2410              : 
    2411          410 :          CALL mpi_test(request%handle, flag, MPI_STATUS_IGNORE, ierr)
    2412          410 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_test @ mp_test_1")
    2413              : #else
    2414              :          MARK_USED(request)
    2415              :          flag = .TRUE.
    2416              : #endif
    2417          410 :       END FUNCTION mp_test_1
    2418              : 
    2419              : ! **************************************************************************************************
    2420              : !> \brief tests for completion of the given requests
    2421              : !> \param requests ...
    2422              : !> \param completed ...
    2423              : !> \param flag ...
    2424              : !> \par History
    2425              : !>      08.2011 created
    2426              : !> \author Iain Bethune
    2427              : ! **************************************************************************************************
    2428            0 :       SUBROUTINE mp_testany_1(requests, completed, flag)
    2429              :          TYPE(mp_request_type), DIMENSION(:), INTENT(inout)  :: requests
    2430              :          INTEGER, INTENT(out), OPTIONAL           :: completed
    2431              :          LOGICAL, INTENT(out), OPTIONAL           :: flag
    2432              : 
    2433              : #if defined(__parallel)
    2434              :          INTEGER                                  :: completed_l, count, ierr
    2435              :          LOGICAL                                  :: flag_l
    2436              : 
    2437            0 :          count = SIZE(requests)
    2438              : 
    2439            0 :          CALL mpi_testany_internal(count, requests, completed_l, flag_l, MPI_STATUS_IGNORE, ierr)
    2440            0 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_testany_1 @ mp_testany")
    2441              : 
    2442            0 :          IF (PRESENT(completed)) completed = completed_l
    2443            0 :          IF (PRESENT(flag)) flag = flag_l
    2444              : #else
    2445              :          MARK_USED(requests)
    2446              :          IF (PRESENT(completed)) completed = 1
    2447              :          IF (PRESENT(flag)) flag = .TRUE.
    2448              : #endif
    2449            0 :       END SUBROUTINE mp_testany_1
    2450              : 
    2451              : ! **************************************************************************************************
    2452              : !> \brief tests for completion of the given requests
    2453              : !> \param requests ...
    2454              : !> \param completed ...
    2455              : !> \param flag ...
    2456              : !> \par History
    2457              : !>      08.2011 created
    2458              : !> \author Iain Bethune
    2459              : ! **************************************************************************************************
    2460            0 :       SUBROUTINE mp_testany_2(requests, completed, flag)
    2461              :          TYPE(mp_request_type), DIMENSION(:, :), INTENT(inout)   :: requests
    2462              :          INTEGER, INTENT(out), OPTIONAL           :: completed
    2463              :          LOGICAL, INTENT(out), OPTIONAL           :: flag
    2464              : 
    2465              : #if defined(__parallel)
    2466              :          INTEGER                                  :: completed_l, count, ierr
    2467              :          LOGICAL                                  :: flag_l
    2468              : 
    2469            0 :          count = SIZE(requests)
    2470              : 
    2471            0 :          CALL mpi_testany_internal(count, requests, completed_l, flag_l, MPI_STATUS_IGNORE, ierr)
    2472            0 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_testany_2 @ mp_testany")
    2473              : 
    2474            0 :          IF (PRESENT(completed)) completed = completed_l
    2475            0 :          IF (PRESENT(flag)) flag = flag_l
    2476              : #else
    2477              :          MARK_USED(requests)
    2478              :          IF (PRESENT(completed)) completed = 1
    2479              :          IF (PRESENT(flag)) flag = .TRUE.
    2480              : #endif
    2481            0 :       END SUBROUTINE mp_testany_2
    2482              : 
    2483              : ! **************************************************************************************************
    2484              : !> \brief wrapper needed to deal with interfaces as present in openmpi 1.8.1
    2485              : !>        the issue is with the rank or requests
    2486              : !> \param count ...
    2487              : !> \param array_of_requests ...
    2488              : !> \param index ...
    2489              : !> \param flag ...
    2490              : !> \param status ...
    2491              : !> \param ierr ...
    2492              : !> \author Joost VandeVondele
    2493              : ! **************************************************************************************************
    2494              : #if defined(__parallel)
    2495            0 :       SUBROUTINE mpi_testany_internal(count, array_of_requests, index, flag, status, ierr)
    2496              :          INTEGER, INTENT(in)                                    :: count
    2497              :          TYPE(mp_request_type), DIMENSION(count), INTENT(inout) :: array_of_requests
    2498              :          INTEGER, INTENT(out)                                   :: index
    2499              :          LOGICAL, INTENT(out)                                   :: flag
    2500              :          MPI_STATUS_TYPE, INTENT(out)                           :: status
    2501              :          INTEGER, INTENT(out)                                   :: ierr
    2502              : 
    2503            0 :          MPI_REQUEST_TYPE, ALLOCATABLE, DIMENSION(:) :: request_handles
    2504              : 
    2505            0 :          ALLOCATE (request_handles(count), SOURCE=array_of_requests(1:count)%handle)
    2506            0 :          CALL mpi_testany(count, request_handles, index, flag, status, ierr)
    2507            0 :          array_of_requests(1:count)%handle = request_handles(:)
    2508            0 :          DEALLOCATE (request_handles)
    2509              : 
    2510            0 :       END SUBROUTINE mpi_testany_internal
    2511              : #endif
    2512              : 
    2513              : ! **************************************************************************************************
    2514              : !> \brief the direct way to split a communicator each color is a sub_comm,
    2515              : !>        the rank order is according to the order in the orig comm
    2516              : !> \param comm ...
    2517              : !> \param sub_comm ...
    2518              : !> \param color ...
    2519              : !> \param key ...
    2520              : !> \author Joost VandeVondele
    2521              : ! **************************************************************************************************
    2522       737502 :       SUBROUTINE mp_comm_split_direct(comm, sub_comm, color, key)
    2523              :          CLASS(mp_comm_type), INTENT(in)                    :: comm
    2524              :          CLASS(mp_comm_type), INTENT(OUT)                   :: sub_comm
    2525              :          INTEGER, INTENT(in)                                :: color
    2526              :          INTEGER, INTENT(in), OPTIONAL                      :: key
    2527              : 
    2528              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_comm_split_direct'
    2529              : 
    2530              :          INTEGER                                            :: handle
    2531              : #if defined(__parallel)
    2532              :          INTEGER :: ierr, my_key
    2533              : #endif
    2534              : 
    2535       737502 :          CALL mp_assert_outside_parallel(routineN)
    2536       737502 :          CALL mp_timeset(routineN, handle)
    2537              : 
    2538              : #if defined(__parallel)
    2539       737502 :          my_key = 0
    2540       737502 :          IF (PRESENT(key)) my_key = key
    2541       737502 :          CALL mpi_comm_split(comm%handle, color, my_key, sub_comm%handle, ierr)
    2542       737502 :          IF (ierr /= mpi_success) CALL mp_stop(ierr, routineN)
    2543       737502 :          CALL add_perf(perf_id=10, count=1)
    2544              : #else
    2545              :          sub_comm%handle = mp_comm_default_handle
    2546              :          MARK_USED(comm)
    2547              :          MARK_USED(color)
    2548              :          MARK_USED(key)
    2549              : #endif
    2550       737502 :          debug_comm_count = debug_comm_count + 1
    2551       737502 :          CALL sub_comm%init()
    2552       737502 :          CALL mp_timestop(handle)
    2553              : 
    2554       737502 :       END SUBROUTINE mp_comm_split_direct
    2555              : ! **************************************************************************************************
    2556              : !> \brief splits a communicator by a predefined MPI split type, e.g. mp_comm_split_type_shared
    2557              : !>        to obtain the node-local (shared-memory) sub-communicator each rank belongs to
    2558              : !> \param comm ...
    2559              : !> \param sub_comm ...
    2560              : !> \param split the split-type/info bundle (e.g. mp_comm_split_type_shared)
    2561              : !> \param key controls the rank ordering within each sub_comm (default 0)
    2562              : ! **************************************************************************************************
    2563          768 :       SUBROUTINE mp_comm_split_type(comm, sub_comm, split, key)
    2564              :          CLASS(mp_comm_type), INTENT(in)                    :: comm
    2565              :          CLASS(mp_comm_type), INTENT(OUT)                   :: sub_comm
    2566              :          TYPE(mp_split_type), INTENT(in)                    :: split
    2567              :          INTEGER, INTENT(in), OPTIONAL                      :: key
    2568              : 
    2569              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_comm_split_type'
    2570              : 
    2571              :          INTEGER                                            :: handle
    2572              : #if defined(__parallel)
    2573              :          INTEGER :: ierr, my_key
    2574              : #endif
    2575              : 
    2576          768 :          CALL mp_assert_outside_parallel(routineN)
    2577          768 :          CALL mp_timeset(routineN, handle)
    2578              : 
    2579              : #if defined(__parallel)
    2580          768 :          my_key = 0
    2581          768 :          IF (PRESENT(key)) my_key = key
    2582              :          CALL mpi_comm_split_type(comm%handle, split%split_type, my_key, split%info%handle, &
    2583          768 :                                   sub_comm%handle, ierr)
    2584          768 :          IF (ierr /= mpi_success) CALL mp_stop(ierr, routineN)
    2585          768 :          CALL add_perf(perf_id=10, count=1)
    2586              : #else
    2587              :          sub_comm%handle = mp_comm_default_handle
    2588              :          MARK_USED(comm)
    2589              :          MARK_USED(split)
    2590              :          MARK_USED(key)
    2591              : #endif
    2592          768 :          debug_comm_count = debug_comm_count + 1
    2593          768 :          CALL sub_comm%init()
    2594          768 :          CALL mp_timestop(handle)
    2595              : 
    2596          768 :       END SUBROUTINE mp_comm_split_type
    2597              : ! **************************************************************************************************
    2598              : !> \brief Memory that is currently free on the node, per MPI rank of that node, in GB.
    2599              : !>
    2600              : !>        The operating system reports node-wide memory (/proc/meminfo on Linux), so every rank
    2601              : !>        of a node reads the same figure. It is divided by the number of ranks sharing that node,
    2602              : !>        which are identified by splitting comm with mp_comm_split_type_shared. The result is
    2603              : !>        minimized over all ranks of comm, such that every rank ends up with the same, most
    2604              : !>        pessimistic figure.
    2605              : !>
    2606              : !>        Returns 0.0 if the operating system does not expose the free memory.
    2607              : !> \param comm communicator over which the result is minimized
    2608              : !> \param mem_avail_GB free memory per MPI rank in GB, identical on all ranks of comm
    2609              : ! **************************************************************************************************
    2610          696 :       SUBROUTINE mp_mem_avail_per_rank_GB(comm, mem_avail_GB)
    2611              :          CLASS(mp_comm_type), INTENT(IN)                    :: comm
    2612              :          REAL(KIND=dp), INTENT(OUT)                         :: mem_avail_GB
    2613              : 
    2614              :          REAL(KIND=dp)                                      :: mem_node_used_GB
    2615              : 
    2616          696 :          CALL mp_mem_node_per_rank_GB(comm, mem_avail_GB, mem_node_used_GB)
    2617              : 
    2618          696 :       END SUBROUTINE mp_mem_avail_per_rank_GB
    2619              : 
    2620              : ! **************************************************************************************************
    2621              : !> \brief Free and occupied memory of the node, in GB, each divided by the ranks of comm on the
    2622              : !>        node: the free memory minimized and the occupied memory maximized over all ranks of comm.
    2623              : !>        Unlike the resident memory of a process, the occupied memory of the node also covers
    2624              : !>        memory that a driver holds (e.g., pinned host memory).
    2625              : !> \param comm communicator whose ranks share the node(s)
    2626              : !> \param mem_avail_GB free memory per MPI rank in GB (0.0 if not exposed)
    2627              : !> \param mem_node_used_GB occupied memory of the node per MPI rank in GB
    2628              : ! **************************************************************************************************
    2629         1532 :       SUBROUTINE mp_mem_node_per_rank_GB(comm, mem_avail_GB, mem_node_used_GB)
    2630              :          CLASS(mp_comm_type), INTENT(IN)                    :: comm
    2631              :          REAL(KIND=dp), INTENT(OUT)                         :: mem_avail_GB, mem_node_used_GB
    2632              : 
    2633              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_mem_node_per_rank_GB'
    2634              : 
    2635              :          INTEGER                                            :: handle, ranks_per_node
    2636              :          INTEGER(KIND=int_8)                                :: mem_buffers, mem_cached, mem_free, &
    2637              :                                                                mem_likely_free, mem_slab, &
    2638              :                                                                mem_sreclaimable, mem_total
    2639              :          TYPE(mp_comm_type)                                 :: node_comm
    2640              : 
    2641          766 :          CALL mp_assert_outside_parallel(routineN)
    2642          766 :          CALL mp_timeset(routineN, handle)
    2643              : 
    2644          766 :          CALL node_comm%from_split_type(comm, mp_comm_split_type_shared, key=comm%mepos)
    2645          766 :          ranks_per_node = MAX(node_comm%num_pe, 1)
    2646          766 :          CALL node_comm%free()
    2647              : 
    2648              :          ! m_memory_details assigns all of its (optional) arguments, so all of them must be given
    2649              :          CALL m_memory_details(MemTotal=mem_total, MemFree=mem_free, Buffers=mem_buffers, &
    2650              :                                Cached=mem_cached, Slab=mem_slab, SReclaimable=mem_sreclaimable, &
    2651          766 :                                MemLikelyFree=mem_likely_free)
    2652          766 :          mem_avail_GB = REAL(mem_likely_free, dp)*1.0E-9_dp/REAL(ranks_per_node, dp)
    2653              :          mem_node_used_GB = REAL(MAX(mem_total - mem_likely_free, 0_int_8), dp)*1.0E-9_dp/ &
    2654          766 :                             REAL(ranks_per_node, dp)
    2655          766 :          CALL comm%min(mem_avail_GB)
    2656          766 :          CALL comm%max(mem_node_used_GB)
    2657              : 
    2658          766 :          CALL mp_timestop(handle)
    2659              : 
    2660          766 :       END SUBROUTINE mp_mem_node_per_rank_GB
    2661              : 
    2662              : ! **************************************************************************************************
    2663              : !> \brief Memory that is currently occupied by this process, in GB, maximized over all ranks of
    2664              : !>        comm, such that every rank ends up with the same, most pessimistic figure.
    2665              : !> \param comm communicator over which the result is maximized
    2666              : !> \param mem_used_GB occupied memory per MPI rank in GB, identical on all ranks of comm
    2667              : ! **************************************************************************************************
    2668         1320 :       SUBROUTINE mp_mem_used_per_rank_GB(comm, mem_used_GB)
    2669              :          CLASS(mp_comm_type), INTENT(IN)                    :: comm
    2670              :          REAL(KIND=dp), INTENT(OUT)                         :: mem_used_GB
    2671              : 
    2672              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_mem_used_per_rank_GB'
    2673              : 
    2674              :          INTEGER                                            :: handle
    2675              :          INTEGER(KIND=int_8)                                :: mem_used
    2676              : 
    2677          440 :          CALL mp_assert_outside_parallel(routineN)
    2678          440 :          CALL mp_timeset(routineN, handle)
    2679              : 
    2680          440 :          CALL m_memory(mem_used)
    2681          440 :          mem_used_GB = REAL(mem_used, dp)*1.0E-9_dp
    2682          440 :          CALL comm%max(mem_used_GB)
    2683              : 
    2684          440 :          CALL mp_timestop(handle)
    2685              : 
    2686          440 :       END SUBROUTINE mp_mem_used_per_rank_GB
    2687              : 
    2688              : ! **************************************************************************************************
    2689              : !> \brief Prints the memory available to and used by a single MPI rank at the moment of the call.
    2690              : !>        Collective on comm; only ranks with unit_nr > 0 write.
    2691              : !> \param comm communicator whose ranks share the reported memory
    2692              : !> \param unit_nr output unit, nothing is printed for unit_nr <= 0
    2693              : !> \param label text identifying the point of the run at which the memory is reported
    2694              : ! **************************************************************************************************
    2695          140 :       SUBROUTINE mp_print_mem_per_rank(comm, unit_nr, label)
    2696              :          CLASS(mp_comm_type), INTENT(IN)                    :: comm
    2697              :          INTEGER, INTENT(IN)                                :: unit_nr
    2698              :          CHARACTER(len=*), INTENT(IN), OPTIONAL             :: label
    2699              : 
    2700              :          REAL(KIND=dp)                                      :: mem_avail_GB, mem_node_used_GB, &
    2701              :                                                                mem_used_GB
    2702              : 
    2703           70 :          CALL mp_mem_node_per_rank_GB(comm, mem_avail_GB, mem_node_used_GB)
    2704           70 :          CALL mp_mem_used_per_rank_GB(comm, mem_used_GB)
    2705              : 
    2706           70 :          IF (unit_nr > 0) THEN
    2707           35 :             IF (PRESENT(label)) WRITE (unit_nr, '(T2,A)') TRIM(label)
    2708           35 :             WRITE (unit_nr, '(T2,A,T68,F10.1,A)') 'Available memory per MPI process', &
    2709           70 :                mem_avail_GB, ' GB'
    2710           35 :             WRITE (unit_nr, '(T2,A,T68,F10.1,A)') 'Used memory per MPI process', &
    2711           70 :                mem_used_GB, ' GB'
    2712           35 :             WRITE (unit_nr, '(T2,A,T68,F10.1,A)') 'Used memory of the node per MPI process', &
    2713           70 :                mem_node_used_GB, ' GB'
    2714              :          END IF
    2715              : 
    2716           70 :       END SUBROUTINE mp_print_mem_per_rank
    2717              : ! **************************************************************************************************
    2718              : !> \brief splits the given communicator in group in subgroups trying to organize
    2719              : !>      them in a way that the communication within each subgroup is
    2720              : !>      efficient (but not necessarily the communication between subgroups)
    2721              : !> \param comm the mpi communicator that you want to split
    2722              : !> \param sub_comm the communicator for the subgroup (created, needs to be freed later)
    2723              : !> \param ngroups actual number of groups
    2724              : !> \param group_distribution input  : allocated with array with the nprocs entries (0 .. nprocs-1)
    2725              : !> \param subgroup_min_size the minimum size of the subgroup
    2726              : !> \param n_subgroups the number of subgroups wanted
    2727              : !> \param group_partition n_subgroups sized array containing the number of cpus wanted per group.
    2728              : !>                         should match the total number of cpus (only used if present and associated) (0..ngroups-1)
    2729              : !> \param stride create groups using a stride (default=1) through the ranks of the comm to be split.
    2730              : !> \par History
    2731              : !>      10.2003 created [fawzi]
    2732              : !>      02.2004 modified [Joost VandeVondele]
    2733              : !> \author Fawzi Mohamed
    2734              : !> \note
    2735              : !>      at least one of subgroup_min_size and n_subgroups is needed,
    2736              : !>      the other default to the value needed to use most processors.
    2737              : !>      if less cpus are present than needed for subgroup min size, n_subgroups,
    2738              : !>      just one comm is created that contains all cpus
    2739              : ! **************************************************************************************************
    2740        10487 :       SUBROUTINE mp_comm_split(comm, sub_comm, ngroups, group_distribution, &
    2741        10487 :                                subgroup_min_size, n_subgroups, group_partition, stride)
    2742              :          CLASS(mp_comm_type), INTENT(in)                      :: comm
    2743              :          CLASS(mp_comm_type), INTENT(out)                     :: sub_comm
    2744              :          INTEGER, INTENT(out)                                 :: ngroups
    2745              :          INTEGER, DIMENSION(0:), INTENT(INOUT)                :: group_distribution
    2746              :          INTEGER, INTENT(in), OPTIONAL                        :: subgroup_min_size, &
    2747              :                                                                  n_subgroups
    2748              :          INTEGER, DIMENSION(0:), INTENT(IN), OPTIONAL         :: group_partition
    2749              :          INTEGER, OPTIONAL, INTENT(IN)                        :: stride
    2750              : 
    2751              :          CHARACTER(LEN=*), PARAMETER :: routineN = 'mp_comm_split', &
    2752              :                                         routineP = moduleN//':'//routineN
    2753              : 
    2754              :          INTEGER                                  :: handle, mepos, nnodes
    2755              : #if defined(__parallel)
    2756              :          INTEGER                                  :: color, i, ierr, j, k, &
    2757              :                                                      my_subgroup_min_size, &
    2758              :                                                      istride, local_stride, irank
    2759        10487 :          INTEGER, DIMENSION(:), ALLOCATABLE       :: rank_permutation
    2760              : #endif
    2761              : 
    2762        10487 :          CALL mp_assert_outside_parallel(routineN)
    2763        10487 :          CALL mp_timeset(routineN, handle)
    2764              : 
    2765              :          ! actual number of groups
    2766              : 
    2767        10487 :          IF (.NOT. PRESENT(subgroup_min_size) .AND. .NOT. PRESENT(n_subgroups)) THEN
    2768            0 :             CPABORT(routineP//" missing arguments")
    2769              :          END IF
    2770        10487 :          IF (PRESENT(subgroup_min_size) .AND. PRESENT(n_subgroups)) THEN
    2771            0 :             CPABORT(routineP//" too many arguments")
    2772              :          END IF
    2773              : 
    2774        10487 :          CALL comm%get_size(nnodes)
    2775        10487 :          CALL comm%get_rank(mepos)
    2776              : 
    2777        10487 :          IF (UBOUND(group_distribution, 1) /= nnodes - 1) THEN
    2778            0 :             CPABORT(routineP//" group_distribution wrong bounds")
    2779              :          END IF
    2780              : 
    2781              : #if defined(__parallel)
    2782        10487 :          IF (PRESENT(subgroup_min_size)) THEN
    2783          158 :             IF (subgroup_min_size < 0 .OR. subgroup_min_size > nnodes) THEN
    2784            0 :                CPABORT(routineP//" subgroup_min_size too small or too large")
    2785              :             END IF
    2786          158 :             ngroups = nnodes/subgroup_min_size
    2787          158 :             my_subgroup_min_size = subgroup_min_size
    2788              :          ELSE ! n_subgroups
    2789        10329 :             IF (n_subgroups <= 0) THEN
    2790            0 :                CPABORT(routineP//" n_subgroups too small")
    2791              :             END IF
    2792        10329 :             IF (nnodes/n_subgroups > 0) THEN ! we have a least one cpu per group
    2793        10281 :                ngroups = n_subgroups
    2794              :             ELSE ! well, only one group then
    2795           48 :                ngroups = 1
    2796              :             END IF
    2797        10329 :             my_subgroup_min_size = nnodes/ngroups
    2798              :          END IF
    2799              : 
    2800              :          ! rank_permutation: is a permutation of ranks, so that groups are not necessarily continuous in rank of the master group
    2801              :          ! while the order is not critical (we only color ranks), it can e.g. be used to make groups that have just 1 rank per node
    2802              :          ! (by setting stride equal to the number of mpi ranks per node), or by sharing  a node between two groups (stride 2).
    2803        31461 :          ALLOCATE (rank_permutation(0:nnodes - 1))
    2804        10487 :          local_stride = 1
    2805        10487 :          IF (PRESENT(stride)) local_stride = stride
    2806        10487 :          k = 0
    2807        20974 :          DO istride = 1, local_stride
    2808        20974 :             DO irank = istride - 1, nnodes - 1, local_stride
    2809        20925 :                rank_permutation(k) = irank
    2810        20925 :                k = k + 1
    2811              :             END DO
    2812              :          END DO
    2813              : 
    2814        31412 :          DO i = 0, nnodes - 1
    2815        31412 :             group_distribution(rank_permutation(i)) = MIN(i/my_subgroup_min_size, ngroups - 1)
    2816              :          END DO
    2817              :          ! even the user gave a partition, see if we can use it to overwrite this choice
    2818        10487 :          IF (PRESENT(group_partition)) THEN
    2819        51570 :             IF (ALL(group_partition > 0) .AND. (SUM(group_partition) == nnodes) .AND. (ngroups == SIZE(group_partition))) THEN
    2820           98 :                k = 0
    2821           98 :                DO i = 0, SIZE(group_partition) - 1
    2822          166 :                   DO j = 1, group_partition(i)
    2823           68 :                      group_distribution(rank_permutation(k)) = i
    2824          132 :                      k = k + 1
    2825              :                   END DO
    2826              :                END DO
    2827              :             ELSE
    2828              :                ! just ignore silently as we have reasonable defaults. Probably a warning would not be to bad
    2829              :             END IF
    2830              :          END IF
    2831        10487 :          DEALLOCATE (rank_permutation)
    2832        10487 :          color = group_distribution(mepos)
    2833        10487 :          CALL mpi_comm_split(comm%handle, color, 0, sub_comm%handle, ierr)
    2834        10487 :          IF (ierr /= mpi_success) CALL mp_stop(ierr, "in "//routineP//" split")
    2835              : 
    2836        10487 :          CALL add_perf(perf_id=10, count=1)
    2837              : #else
    2838              :          sub_comm%handle = mp_comm_default_handle
    2839              :          group_distribution(0) = 0
    2840              :          ngroups = 1
    2841              :          MARK_USED(comm)
    2842              :          MARK_USED(stride)
    2843              :          MARK_USED(group_partition)
    2844              : #endif
    2845        10487 :          debug_comm_count = debug_comm_count + 1
    2846        10487 :          CALL sub_comm%init()
    2847        10487 :          CALL mp_timestop(handle)
    2848              : 
    2849        20974 :       END SUBROUTINE mp_comm_split
    2850              : 
    2851              : ! **************************************************************************************************
    2852              : !> \brief probes for an incoming message with any tag
    2853              : !> \param[inout] source the source of the possible incoming message,
    2854              : !>        if MP_ANY_SOURCE it is a blocking one and return value is the source
    2855              : !>        of the next incoming message
    2856              : !>        if source is a different value it is a non-blocking probe returning
    2857              : !>        MP_ANY_SOURCE if there is no incoming message
    2858              : !> \param[in] comm the communicator
    2859              : !> \param[out] tag the tag of the incoming message
    2860              : !> \author Mandes
    2861              : ! **************************************************************************************************
    2862      1422353 :       SUBROUTINE mp_probe(source, comm, tag)
    2863              :          INTEGER, INTENT(INOUT)                   :: source
    2864              :          CLASS(mp_comm_type), INTENT(IN)          :: comm
    2865              :          INTEGER, INTENT(OUT)                     :: tag
    2866              : 
    2867              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_probe'
    2868              : 
    2869              :          INTEGER                                  :: handle
    2870              : #if defined(__parallel)
    2871              :          INTEGER :: ierr
    2872              :          MPI_STATUS_TYPE     :: status_single
    2873              :          LOGICAL                                  :: flag
    2874              : #endif
    2875              : 
    2876              : !   ---------------------------------------------------------------------------
    2877              : 
    2878      1422353 :          CALL mp_timeset(routineN, handle)
    2879              : 
    2880              : #if defined(__parallel)
    2881      1422353 :          IF (source == mp_any_source) THEN
    2882           14 :             CALL mpi_probe(mp_any_source, mp_any_tag, comm%handle, status_single, ierr)
    2883           14 :             IF (ierr /= 0) CALL mp_stop(ierr, "mpi_probe @ mp_probe")
    2884           14 :             source = status_single MPI_STATUS_EXTRACT(MPI_SOURCE)
    2885           14 :             tag = status_single MPI_STATUS_EXTRACT(MPI_TAG)
    2886              :          ELSE
    2887              :             flag = .FALSE.
    2888      1422339 :             CALL mpi_iprobe(source, mp_any_tag, comm%handle, flag, status_single, ierr)
    2889      1422339 :             IF (ierr /= 0) CALL mp_stop(ierr, "mpi_iprobe @ mp_probe")
    2890      1422339 :             IF (flag .EQV. .FALSE.) THEN
    2891      1413801 :                source = mp_any_source
    2892      1413801 :                tag = -1 !status_single(MPI_TAG) ! in case of flag==false status is undefined
    2893              :             ELSE
    2894         8538 :                tag = status_single MPI_STATUS_EXTRACT(MPI_TAG)
    2895              :             END IF
    2896              :          END IF
    2897              : #else
    2898              :          tag = -1
    2899              :          MARK_USED(comm)
    2900              :          MARK_USED(source)
    2901              : #endif
    2902      1422353 :          CALL mp_timestop(handle)
    2903      1422353 :       END SUBROUTINE mp_probe
    2904              : 
    2905              : ! **************************************************************************************************
    2906              : ! Here come the data routines with none of the standard data types.
    2907              : ! **************************************************************************************************
    2908              : 
    2909              : ! **************************************************************************************************
    2910              : !> \brief ...
    2911              : !> \param msg ...
    2912              : !> \param source ...
    2913              : !> \param comm ...
    2914              : ! **************************************************************************************************
    2915       584903 :       SUBROUTINE mp_bcast_b(msg, source, comm)
    2916              :          LOGICAL, INTENT(INOUT)                             :: msg
    2917              :          INTEGER, INTENT(IN)                                :: source
    2918              :          CLASS(mp_comm_type), INTENT(IN) :: comm
    2919              : 
    2920              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_bcast_b'
    2921              : 
    2922              :          INTEGER                                            :: handle
    2923              : #if defined(__parallel)
    2924              :          INTEGER :: ierr, msglen
    2925              : #endif
    2926              : 
    2927       584903 :          CALL mp_timeset(routineN, handle)
    2928              : 
    2929              : #if defined(__parallel)
    2930       584903 :          msglen = 1
    2931       584903 :          CALL mpi_bcast(msg, msglen, MPI_LOGICAL, source, comm%handle, ierr)
    2932       584903 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_bcast @ "//routineN)
    2933       584903 :          CALL add_perf(perf_id=2, count=1, msg_size=msglen*loglen)
    2934              : #else
    2935              :          MARK_USED(msg)
    2936              :          MARK_USED(source)
    2937              :          MARK_USED(comm)
    2938              : #endif
    2939       584903 :          CALL mp_timestop(handle)
    2940       584903 :       END SUBROUTINE mp_bcast_b
    2941              : 
    2942              : ! **************************************************************************************************
    2943              : !> \brief ...
    2944              : !> \param msg ...
    2945              : !> \param source ...
    2946              : !> \param comm ...
    2947              : ! **************************************************************************************************
    2948       678913 :       SUBROUTINE mp_bcast_b_src(msg, comm)
    2949              :          LOGICAL, INTENT(INOUT)                             :: msg
    2950              :          CLASS(mp_comm_type), INTENT(IN) :: comm
    2951              : 
    2952              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_bcast_b_src'
    2953              : 
    2954              :          INTEGER                                            :: handle
    2955              : #if defined(__parallel)
    2956              :          INTEGER :: ierr, msglen
    2957              : #endif
    2958              : 
    2959       678913 :          CALL mp_timeset(routineN, handle)
    2960              : 
    2961              : #if defined(__parallel)
    2962       678913 :          msglen = 1
    2963       678913 :          CALL mpi_bcast(msg, msglen, MPI_LOGICAL, comm%source, comm%handle, ierr)
    2964       678913 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_bcast @ "//routineN)
    2965       678913 :          CALL add_perf(perf_id=2, count=1, msg_size=msglen*loglen)
    2966              : #else
    2967              :          MARK_USED(msg)
    2968              :          MARK_USED(comm)
    2969              : #endif
    2970       678913 :          CALL mp_timestop(handle)
    2971       678913 :       END SUBROUTINE mp_bcast_b_src
    2972              : 
    2973              : ! **************************************************************************************************
    2974              : !> \brief ...
    2975              : !> \param msg ...
    2976              : !> \param source ...
    2977              : !> \param comm ...
    2978              : ! **************************************************************************************************
    2979            0 :       SUBROUTINE mp_bcast_bv(msg, source, comm)
    2980              :          LOGICAL, CONTIGUOUS, INTENT(INOUT)                 :: msg(:)
    2981              :          INTEGER, INTENT(IN)                                :: source
    2982              :          CLASS(mp_comm_type), INTENT(IN) :: comm
    2983              : 
    2984              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_bcast_bv'
    2985              : 
    2986              :          INTEGER                                            :: handle
    2987              : #if defined(__parallel)
    2988              :          INTEGER :: ierr, msglen
    2989              : #endif
    2990              : 
    2991            0 :          CALL mp_timeset(routineN, handle)
    2992              : 
    2993              : #if defined(__parallel)
    2994            0 :          msglen = SIZE(msg)
    2995            0 :          CALL mpi_bcast(msg, msglen, MPI_LOGICAL, source, comm%handle, ierr)
    2996            0 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_bcast @ "//routineN)
    2997            0 :          CALL add_perf(perf_id=2, count=1, msg_size=msglen*loglen)
    2998              : #else
    2999              :          MARK_USED(msg)
    3000              :          MARK_USED(source)
    3001              :          MARK_USED(comm)
    3002              : #endif
    3003            0 :          CALL mp_timestop(handle)
    3004            0 :       END SUBROUTINE mp_bcast_bv
    3005              : 
    3006              : ! **************************************************************************************************
    3007              : !> \brief ...
    3008              : !> \param msg ...
    3009              : !> \param comm ...
    3010              : ! **************************************************************************************************
    3011            0 :       SUBROUTINE mp_bcast_bv_src(msg, comm)
    3012              :          LOGICAL, CONTIGUOUS, INTENT(INOUT)                 :: msg(:)
    3013              :          CLASS(mp_comm_type), INTENT(IN) :: comm
    3014              : 
    3015              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_bcast_bv_src'
    3016              : 
    3017              :          INTEGER                                            :: handle
    3018              : #if defined(__parallel)
    3019              :          INTEGER :: ierr, msglen
    3020              : #endif
    3021              : 
    3022            0 :          CALL mp_timeset(routineN, handle)
    3023              : 
    3024              : #if defined(__parallel)
    3025            0 :          msglen = SIZE(msg)
    3026            0 :          CALL mpi_bcast(msg, msglen, MPI_LOGICAL, comm%source, comm%handle, ierr)
    3027            0 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_bcast @ "//routineN)
    3028            0 :          CALL add_perf(perf_id=2, count=1, msg_size=msglen*loglen)
    3029              : #else
    3030              :          MARK_USED(msg)
    3031              :          MARK_USED(comm)
    3032              : #endif
    3033            0 :          CALL mp_timestop(handle)
    3034            0 :       END SUBROUTINE mp_bcast_bv_src
    3035              : 
    3036              : ! **************************************************************************************************
    3037              : !> \brief Non-blocking send of logical vector data
    3038              : !> \param msgin the input message
    3039              : !> \param dest the destination processor
    3040              : !> \param comm  the communicator object
    3041              : !> \param request communication request index
    3042              : !> \param tag message tag
    3043              : !> \par History
    3044              : !>      3.2016 added _bv subroutine [Nico Holmberg]
    3045              : !> \author fawzi
    3046              : !> \note see mp_irecv_iv
    3047              : !> \note
    3048              : !>      arrays can be pointers or assumed shape, but they must be contiguous!
    3049              : ! **************************************************************************************************
    3050           16 :       SUBROUTINE mp_isend_bv(msgin, dest, comm, request, tag)
    3051              :          LOGICAL, DIMENSION(:), INTENT(IN)        :: msgin
    3052              :          INTEGER, INTENT(IN)                      :: dest
    3053              :          CLASS(mp_comm_type), INTENT(IN) :: comm
    3054              :          TYPE(mp_request_type), INTENT(out)       :: request
    3055              :          INTEGER, INTENT(in), OPTIONAL            :: tag
    3056              : 
    3057              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_isend_bv'
    3058              : 
    3059              :          INTEGER                                  :: handle
    3060              : #if defined(__parallel)
    3061              :          INTEGER                                  :: ierr, msglen, my_tag
    3062              :          LOGICAL                                  :: foo(1)
    3063              : #endif
    3064              : 
    3065           16 :          CALL mp_timeset(routineN, handle)
    3066              : 
    3067              : #if defined(__parallel)
    3068           32 :          CPASSERT(IS_CONTIGUOUS(msgin) .OR. PRODUCT(SHAPE(msgin)) == 0)
    3069              : 
    3070           16 :          my_tag = 0
    3071           16 :          IF (PRESENT(tag)) my_tag = tag
    3072              : 
    3073           16 :          msglen = SIZE(msgin, 1)
    3074           16 :          IF (msglen > 0) THEN
    3075              :             CALL mpi_isend(msgin(1), msglen, MPI_LOGICAL, dest, my_tag, &
    3076           16 :                            comm%handle, request%handle, ierr)
    3077              :          ELSE
    3078              :             CALL mpi_isend(foo, msglen, MPI_LOGICAL, dest, my_tag, &
    3079            0 :                            comm%handle, request%handle, ierr)
    3080              :          END IF
    3081           16 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_isend @ "//routineN)
    3082              : 
    3083           16 :          CALL add_perf(perf_id=11, count=1, msg_size=msglen*loglen)
    3084              : #else
    3085              :          CPABORT("mp_isend called in non parallel case")
    3086              :          MARK_USED(msgin)
    3087              :          MARK_USED(dest)
    3088              :          MARK_USED(comm)
    3089              :          MARK_USED(tag)
    3090              :          request = mp_request_null
    3091              : #endif
    3092           16 :          CALL mp_timestop(handle)
    3093           16 :       END SUBROUTINE mp_isend_bv
    3094              : 
    3095              : ! **************************************************************************************************
    3096              : !> \brief Non-blocking receive of logical vector data
    3097              : !> \param msgout the received message
    3098              : !> \param source the source processor
    3099              : !> \param comm  the communicator object
    3100              : !> \param request communication request index
    3101              : !> \param tag message tag
    3102              : !> \par History
    3103              : !>      3.2016 added _bv subroutine [Nico Holmberg]
    3104              : !> \author fawzi
    3105              : !> \note see mp_irecv_iv
    3106              : !> \note
    3107              : !>      arrays can be pointers or assumed shape, but they must be contiguous!
    3108              : ! **************************************************************************************************
    3109           16 :       SUBROUTINE mp_irecv_bv(msgout, source, comm, request, tag)
    3110              :          LOGICAL, DIMENSION(:), INTENT(INOUT)     :: msgout
    3111              :          INTEGER, INTENT(IN)                      :: source
    3112              :          CLASS(mp_comm_type), INTENT(IN) :: comm
    3113              :          TYPE(mp_request_type), INTENT(out)       :: request
    3114              :          INTEGER, INTENT(in), OPTIONAL            :: tag
    3115              : 
    3116              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_irecv_bv'
    3117              : 
    3118              :          INTEGER                                  :: handle
    3119              : #if defined(__parallel)
    3120              :          INTEGER                                  :: ierr, msglen, my_tag
    3121              :          LOGICAL                                  :: foo(1)
    3122              : #endif
    3123              : 
    3124           16 :          CALL mp_timeset(routineN, handle)
    3125              : 
    3126              : #if defined(__parallel)
    3127           32 :          CPASSERT(IS_CONTIGUOUS(msgout) .OR. PRODUCT(SHAPE(msgout)) == 0)
    3128              : 
    3129           16 :          my_tag = 0
    3130           16 :          IF (PRESENT(tag)) my_tag = tag
    3131              : 
    3132           16 :          msglen = SIZE(msgout, 1)
    3133           16 :          IF (msglen > 0) THEN
    3134              :             CALL mpi_irecv(msgout(1), msglen, MPI_LOGICAL, source, my_tag, &
    3135           16 :                            comm%handle, request%handle, ierr)
    3136              :          ELSE
    3137              :             CALL mpi_irecv(foo, msglen, MPI_LOGICAL, source, my_tag, &
    3138            0 :                            comm%handle, request%handle, ierr)
    3139              :          END IF
    3140           16 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_ircv @ "//routineN)
    3141              : 
    3142           16 :          CALL add_perf(perf_id=12, count=1, msg_size=msglen*loglen)
    3143              : #else
    3144              :          CPABORT("mp_irecv called in non parallel case")
    3145              :          MARK_USED(msgout)
    3146              :          MARK_USED(source)
    3147              :          MARK_USED(comm)
    3148              :          MARK_USED(tag)
    3149              :          request = mp_request_null
    3150              : #endif
    3151           16 :          CALL mp_timestop(handle)
    3152           16 :       END SUBROUTINE mp_irecv_bv
    3153              : 
    3154              : ! **************************************************************************************************
    3155              : !> \brief Non-blocking send of rank-3 logical data
    3156              : !> \param msgin the input message
    3157              : !> \param dest the destination processor
    3158              : !> \param comm  the communicator object
    3159              : !> \param request communication request index
    3160              : !> \param tag message tag
    3161              : !> \par History
    3162              : !>      2.2016 added _bm3 subroutine [Nico Holmberg]
    3163              : !> \author fawzi
    3164              : !> \note see mp_irecv_iv
    3165              : !> \note
    3166              : !>      arrays can be pointers or assumed shape, but they must be contiguous!
    3167              : ! **************************************************************************************************
    3168            0 :       SUBROUTINE mp_isend_bm3(msgin, dest, comm, request, tag)
    3169              :          LOGICAL, DIMENSION(:, :, :), INTENT(INOUT) :: msgin
    3170              :          INTEGER, INTENT(IN)                        :: dest
    3171              :          CLASS(mp_comm_type), INTENT(IN)            :: comm
    3172              :          TYPE(mp_request_type), INTENT(out)         :: request
    3173              :          INTEGER, INTENT(in), OPTIONAL              :: tag
    3174              : 
    3175              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_isend_bm3'
    3176              : 
    3177              :          INTEGER                                    :: handle
    3178              : #if defined(__parallel)
    3179              :          INTEGER                                    :: ierr, msglen, my_tag
    3180              :          LOGICAL                                    :: foo(1)
    3181              : #endif
    3182              : 
    3183            0 :          CALL mp_timeset(routineN, handle)
    3184              : 
    3185              : #if defined(__parallel)
    3186            0 :          CPASSERT(IS_CONTIGUOUS(msgin) .OR. PRODUCT(SHAPE(msgin)) == 0)
    3187              : 
    3188            0 :          my_tag = 0
    3189            0 :          IF (PRESENT(tag)) my_tag = tag
    3190              : 
    3191            0 :          msglen = SIZE(msgin, 1)*SIZE(msgin, 2)*SIZE(msgin, 3)
    3192            0 :          IF (msglen > 0) THEN
    3193              :             CALL mpi_isend(msgin(1, 1, 1), msglen, MPI_LOGICAL, dest, my_tag, &
    3194            0 :                            comm%handle, request%handle, ierr)
    3195              :          ELSE
    3196              :             CALL mpi_isend(foo, msglen, MPI_LOGICAL, dest, my_tag, &
    3197            0 :                            comm%handle, request%handle, ierr)
    3198              :          END IF
    3199            0 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_isend @ "//routineN)
    3200              : 
    3201            0 :          CALL add_perf(perf_id=11, count=1, msg_size=msglen*loglen)
    3202              : #else
    3203              :          CPABORT("mp_isend called in non parallel case")
    3204              :          MARK_USED(msgin)
    3205              :          MARK_USED(dest)
    3206              :          MARK_USED(comm)
    3207              :          MARK_USED(tag)
    3208              :          request = mp_request_null
    3209              : #endif
    3210            0 :          CALL mp_timestop(handle)
    3211            0 :       END SUBROUTINE mp_isend_bm3
    3212              : 
    3213              : ! **************************************************************************************************
    3214              : !> \brief Non-blocking receive of rank-3 logical data
    3215              : !> \param msgout the received message
    3216              : !> \param source the source processor
    3217              : !> \param comm  the communicator object
    3218              : !> \param request communication request index
    3219              : !> \param tag message tag
    3220              : !> \par History
    3221              : !>      2.2016 added _bm3 subroutine [Nico Holmberg]
    3222              : !> \author fawzi
    3223              : !> \note see mp_irecv_iv
    3224              : !> \note
    3225              : !>      arrays can be pointers or assumed shape, but they must be contiguous!
    3226              : ! **************************************************************************************************
    3227            0 :       SUBROUTINE mp_irecv_bm3(msgout, source, comm, request, tag)
    3228              :          LOGICAL, DIMENSION(:, :, :), INTENT(INOUT) :: msgout
    3229              :          INTEGER, INTENT(IN)                        :: source
    3230              :          CLASS(mp_comm_type), INTENT(IN) :: comm
    3231              :          TYPE(mp_request_type), INTENT(out)         :: request
    3232              :          INTEGER, INTENT(in), OPTIONAL              :: tag
    3233              : 
    3234              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_irecv_bm3'
    3235              : 
    3236              :          INTEGER                                    :: handle
    3237              : #if defined(__parallel)
    3238              :          INTEGER                                    :: ierr, msglen, my_tag
    3239              :          LOGICAL                                    :: foo(1)
    3240              : #endif
    3241              : 
    3242            0 :          CALL mp_timeset(routineN, handle)
    3243              : 
    3244              : #if defined(__parallel)
    3245            0 :          CPASSERT(IS_CONTIGUOUS(msgout) .OR. PRODUCT(SHAPE(msgout)) == 0)
    3246              : 
    3247            0 :          my_tag = 0
    3248            0 :          IF (PRESENT(tag)) my_tag = tag
    3249              : 
    3250            0 :          msglen = SIZE(msgout, 1)*SIZE(msgout, 2)*SIZE(msgout, 3)
    3251            0 :          IF (msglen > 0) THEN
    3252              :             CALL mpi_irecv(msgout(1, 1, 1), msglen, MPI_LOGICAL, source, my_tag, &
    3253            0 :                            comm%handle, request%handle, ierr)
    3254              :          ELSE
    3255              :             CALL mpi_irecv(foo, msglen, MPI_LOGICAL, source, my_tag, &
    3256            0 :                            comm%handle, request%handle, ierr)
    3257              :          END IF
    3258            0 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_ircv @ "//routineN)
    3259              : 
    3260            0 :          CALL add_perf(perf_id=12, count=1, msg_size=msglen*loglen)
    3261              : #else
    3262              :          CPABORT("mp_irecv called in non parallel case")
    3263              :          MARK_USED(msgout)
    3264              :          MARK_USED(source)
    3265              :          MARK_USED(comm)
    3266              :          MARK_USED(request)
    3267              :          MARK_USED(tag)
    3268              :          request = mp_request_null
    3269              : #endif
    3270            0 :          CALL mp_timestop(handle)
    3271            0 :       END SUBROUTINE mp_irecv_bm3
    3272              : 
    3273              : ! **************************************************************************************************
    3274              : !> \brief Broadcasts a string.
    3275              : !> \param msg ...
    3276              : !> \param source ...
    3277              : !> \param comm ...
    3278              : ! **************************************************************************************************
    3279      5063424 :       SUBROUTINE mp_bcast_av(msg, source, comm)
    3280              :          CHARACTER(LEN=*), INTENT(INOUT)          :: msg
    3281              :          INTEGER, INTENT(IN)                      :: source
    3282              :          CLASS(mp_comm_type), INTENT(IN)          :: comm
    3283              : 
    3284              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_bcast_av'
    3285              : 
    3286              :          INTEGER                                  :: handle
    3287              : #if defined(__parallel)
    3288              :          INTEGER                                  :: ierr, msglen
    3289              : #endif
    3290              : 
    3291      5063424 :          CALL mp_timeset(routineN, handle)
    3292              : 
    3293              : #if defined(__parallel)
    3294      5063424 :          msglen = LEN(msg)*charlen
    3295      5063424 :          IF (comm%mepos /= source) msg = "" ! need to clear msg
    3296      5063424 :          CALL mpi_bcast(msg, msglen, MPI_CHARACTER, source, comm%handle, ierr)
    3297      5063424 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_bcast @ "//routineN)
    3298      5063424 :          CALL add_perf(perf_id=2, count=1, msg_size=msglen)
    3299              : #else
    3300              :          MARK_USED(msg)
    3301              :          MARK_USED(source)
    3302              :          MARK_USED(comm)
    3303              : #endif
    3304      5063424 :          CALL mp_timestop(handle)
    3305      5063424 :       END SUBROUTINE mp_bcast_av
    3306              : 
    3307              : ! **************************************************************************************************
    3308              : !> \brief Broadcasts a string.
    3309              : !> \param msg ...
    3310              : !> \param comm ...
    3311              : ! **************************************************************************************************
    3312        13975 :       SUBROUTINE mp_bcast_av_src(msg, comm)
    3313              :          CHARACTER(LEN=*), INTENT(INOUT)          :: msg
    3314              :          CLASS(mp_comm_type), INTENT(IN)          :: comm
    3315              : 
    3316              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_bcast_av_src'
    3317              : 
    3318              :          INTEGER                                  :: handle
    3319              : #if defined(__parallel)
    3320              :          INTEGER                                  :: ierr, msglen
    3321              : #endif
    3322              : 
    3323        13975 :          CALL mp_timeset(routineN, handle)
    3324              : 
    3325              : #if defined(__parallel)
    3326        13975 :          msglen = LEN(msg)*charlen
    3327        13975 :          IF (.NOT. comm%is_source()) msg = "" ! need to clear msg
    3328        13975 :          CALL mpi_bcast(msg, msglen, MPI_CHARACTER, comm%source, comm%handle, ierr)
    3329        13975 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_bcast @ "//routineN)
    3330        13975 :          CALL add_perf(perf_id=2, count=1, msg_size=msglen)
    3331              : #else
    3332              :          MARK_USED(msg)
    3333              :          MARK_USED(comm)
    3334              : #endif
    3335        13975 :          CALL mp_timestop(handle)
    3336        13975 :       END SUBROUTINE mp_bcast_av_src
    3337              : 
    3338              : ! **************************************************************************************************
    3339              : !> \brief ...
    3340              : !> \param msg ...
    3341              : !> \param source ...
    3342              : !> \param comm ...
    3343              : ! **************************************************************************************************
    3344           28 :       SUBROUTINE mp_bcast_am(msg, source, comm)
    3345              :          CHARACTER(LEN=*), CONTIGUOUS, INTENT(INOUT)  :: msg(:)
    3346              :          INTEGER, INTENT(IN)                          :: source
    3347              :          CLASS(mp_comm_type), INTENT(IN) :: comm
    3348              : 
    3349              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_bcast_am'
    3350              : 
    3351              :          INTEGER                                  :: handle
    3352              : #if defined(__parallel)
    3353              :          INTEGER                                  :: ierr, msglen
    3354              : #endif
    3355              : 
    3356           28 :          CALL mp_timeset(routineN, handle)
    3357              : 
    3358              : #if defined(__parallel)
    3359           28 :          msglen = SIZE(msg)*LEN(msg(1))*charlen
    3360         1922 :          IF (comm%mepos /= source) msg = "" ! need to clear msg
    3361           28 :          CALL mpi_bcast(msg, msglen, MPI_CHARACTER, source, comm%handle, ierr)
    3362           28 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_bcast @ "//routineN)
    3363           28 :          CALL add_perf(perf_id=2, count=1, msg_size=msglen)
    3364              : #else
    3365              :          MARK_USED(msg)
    3366              :          MARK_USED(source)
    3367              :          MARK_USED(comm)
    3368              : #endif
    3369           28 :          CALL mp_timestop(handle)
    3370           28 :       END SUBROUTINE mp_bcast_am
    3371              : 
    3372        97692 :       SUBROUTINE mp_bcast_am_src(msg, comm)
    3373              :          CHARACTER(LEN=*), CONTIGUOUS, INTENT(INOUT)  :: msg(:)
    3374              :          CLASS(mp_comm_type), INTENT(IN)              :: comm
    3375              : 
    3376              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_bcast_am_src'
    3377              : 
    3378              :          INTEGER                                  :: handle
    3379              : #if defined(__parallel)
    3380              :          INTEGER                                  :: ierr, msglen
    3381              : #endif
    3382              : 
    3383        97692 :          CALL mp_timeset(routineN, handle)
    3384              : 
    3385              : #if defined(__parallel)
    3386        97692 :          msglen = SIZE(msg)*LEN(msg(1))*charlen
    3387     48923692 :          IF (.NOT. comm%is_source()) msg = "" ! need to clear msg
    3388        97692 :          CALL mpi_bcast(msg, msglen, MPI_CHARACTER, comm%source, comm%handle, ierr)
    3389        97692 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_bcast @ "//routineN)
    3390        97692 :          CALL add_perf(perf_id=2, count=1, msg_size=msglen)
    3391              : #else
    3392              :          MARK_USED(msg)
    3393              :          MARK_USED(comm)
    3394              : #endif
    3395        97692 :          CALL mp_timestop(handle)
    3396        97692 :       END SUBROUTINE mp_bcast_am_src
    3397              : 
    3398              : ! **************************************************************************************************
    3399              : !> \brief Finds the location of the minimal element in a vector.
    3400              : !> \param[in,out] msg         Find location of minimum element among these
    3401              : !>                            data (input).
    3402              : !> \param[in] comm            Message passing environment identifier
    3403              : !> \par MPI mapping
    3404              : !>      mpi_allreduce with the MPI_MINLOC reduction function identifier
    3405              : !> \par Invalid data types
    3406              : !>      This routine is invalid for (int_8) data!
    3407              : ! **************************************************************************************************
    3408          862 :       SUBROUTINE mp_minloc_dv(msg, comm)
    3409              :          REAL(kind=real_8), CONTIGUOUS, INTENT(INOUT)         :: msg(:)
    3410              :          CLASS(mp_comm_type), INTENT(IN)                      :: comm
    3411              : 
    3412              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_minloc_dv'
    3413              : 
    3414              :          INTEGER                                  :: handle
    3415              : #if defined(__parallel)
    3416              :          INTEGER                                  :: ierr, msglen
    3417          862 :          REAL(kind=real_8), ALLOCATABLE           :: res(:)
    3418              : #endif
    3419              : 
    3420              :          IF ("d" == "l" .AND. real_8 == int_8) THEN
    3421              :             CPABORT("Minimal location not available with long integers @ "//routineN)
    3422              :          END IF
    3423          862 :          CALL mp_timeset(routineN, handle)
    3424              : 
    3425              : #if defined(__parallel)
    3426          862 :          msglen = SIZE(msg)
    3427         2586 :          ALLOCATE (res(1:msglen), STAT=ierr)
    3428          862 :          IF (ierr /= 0) &
    3429            0 :             CPABORT("allocate @ "//routineN)
    3430          862 :          CALL mpi_allreduce(msg, res, msglen/2, MPI_2DOUBLE_PRECISION, MPI_MINLOC, comm%handle, ierr)
    3431          862 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_allreduce @ "//routineN)
    3432         2586 :          msg = res
    3433          862 :          DEALLOCATE (res)
    3434          862 :          CALL add_perf(perf_id=3, count=1, msg_size=msglen*real_8_size)
    3435              : #else
    3436              :          MARK_USED(msg)
    3437              :          MARK_USED(comm)
    3438              : #endif
    3439          862 :          CALL mp_timestop(handle)
    3440          862 :       END SUBROUTINE mp_minloc_dv
    3441              : 
    3442              : ! **************************************************************************************************
    3443              : !> \brief Finds the location of the minimal element in a vector.
    3444              : !> \param[in,out] msg         Find location of minimum element among these
    3445              : !>                            data (input).
    3446              : !> \param[in] comm            Message passing environment identifier
    3447              : !> \par MPI mapping
    3448              : !>      mpi_allreduce with the MPI_MINLOC reduction function identifier
    3449              : !> \par Invalid data types
    3450              : !>      This routine is invalid for (int_8) data!
    3451              : ! **************************************************************************************************
    3452            0 :       SUBROUTINE mp_minloc_iv(msg, comm)
    3453              :          INTEGER(KIND=int_4), CONTIGUOUS, INTENT(INOUT)       :: msg(:)
    3454              :          CLASS(mp_comm_type), INTENT(IN)                      :: comm
    3455              : 
    3456              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_minloc_iv'
    3457              : 
    3458              :          INTEGER                                  :: handle
    3459              : #if defined(__parallel)
    3460              :          INTEGER                                  :: ierr, msglen
    3461            0 :          INTEGER(KIND=int_4), ALLOCATABLE         :: res(:)
    3462              : #endif
    3463              : 
    3464              :          IF ("i" == "l" .AND. int_4 == int_8) THEN
    3465              :             CPABORT("Minimal location not available with long integers @ "//routineN)
    3466              :          END IF
    3467            0 :          CALL mp_timeset(routineN, handle)
    3468              : 
    3469              : #if defined(__parallel)
    3470            0 :          msglen = SIZE(msg)
    3471            0 :          ALLOCATE (res(1:msglen))
    3472            0 :          CALL mpi_allreduce(msg, res, msglen/2, MPI_2INTEGER, MPI_MINLOC, comm%handle, ierr)
    3473            0 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_allreduce @ "//routineN)
    3474            0 :          msg = res
    3475            0 :          DEALLOCATE (res)
    3476            0 :          CALL add_perf(perf_id=3, count=1, msg_size=msglen*int_4_size)
    3477              : #else
    3478              :          MARK_USED(msg)
    3479              :          MARK_USED(comm)
    3480              : #endif
    3481            0 :          CALL mp_timestop(handle)
    3482            0 :       END SUBROUTINE mp_minloc_iv
    3483              : 
    3484              : ! **************************************************************************************************
    3485              : !> \brief Finds the location of the minimal element in a vector.
    3486              : !> \param[in,out] msg         Find location of minimum element among these
    3487              : !>                            data (input).
    3488              : !> \param[in] comm            Message passing environment identifier
    3489              : !> \par MPI mapping
    3490              : !>      mpi_allreduce with the MPI_MINLOC reduction function identifier
    3491              : !> \par Invalid data types
    3492              : !>      This routine is invalid for (int_8) data!
    3493              : ! **************************************************************************************************
    3494            0 :       SUBROUTINE mp_minloc_lv(msg, comm)
    3495              :          INTEGER(KIND=int_8), CONTIGUOUS, INTENT(INOUT)       :: msg(:)
    3496              :          CLASS(mp_comm_type), INTENT(IN)                      :: comm
    3497              : 
    3498              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_minloc_lv'
    3499              : 
    3500              :          INTEGER                                  :: handle
    3501              : #if defined(__parallel)
    3502              :          INTEGER                                  :: ierr, msglen
    3503            0 :          INTEGER(KIND=int_8), ALLOCATABLE         :: res(:)
    3504              : #endif
    3505              : 
    3506              :          IF ("l" == "l" .AND. int_8 == int_8) THEN
    3507            0 :             CPABORT("Minimal location not available with long integers @ "//routineN)
    3508              :          END IF
    3509            0 :          CALL mp_timeset(routineN, handle)
    3510              : 
    3511              : #if defined(__parallel)
    3512            0 :          msglen = SIZE(msg)
    3513            0 :          ALLOCATE (res(1:msglen))
    3514            0 :          CALL mpi_allreduce(msg, res, msglen/2, MPI_INTEGER8, MPI_MINLOC, comm%handle, ierr)
    3515            0 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_allreduce @ "//routineN)
    3516            0 :          msg = res
    3517            0 :          DEALLOCATE (res)
    3518            0 :          CALL add_perf(perf_id=3, count=1, msg_size=msglen*int_8_size)
    3519              : #else
    3520              :          MARK_USED(msg)
    3521              :          MARK_USED(comm)
    3522              : #endif
    3523            0 :          CALL mp_timestop(handle)
    3524            0 :       END SUBROUTINE mp_minloc_lv
    3525              : 
    3526              : ! **************************************************************************************************
    3527              : !> \brief Finds the location of the minimal element in a vector.
    3528              : !> \param[in,out] msg         Find location of minimum element among these
    3529              : !>                            data (input).
    3530              : !> \param[in] comm            Message passing environment identifier
    3531              : !> \par MPI mapping
    3532              : !>      mpi_allreduce with the MPI_MINLOC reduction function identifier
    3533              : !> \par Invalid data types
    3534              : !>      This routine is invalid for (int_8) data!
    3535              : ! **************************************************************************************************
    3536            0 :       SUBROUTINE mp_minloc_rv(msg, comm)
    3537              :          REAL(kind=real_4), CONTIGUOUS, INTENT(INOUT)         :: msg(:)
    3538              :          CLASS(mp_comm_type), INTENT(IN)                      :: comm
    3539              : 
    3540              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_minloc_rv'
    3541              : 
    3542              :          INTEGER                                  :: handle
    3543              : #if defined(__parallel)
    3544              :          INTEGER                                  :: ierr, msglen
    3545            0 :          REAL(kind=real_4), ALLOCATABLE           :: res(:)
    3546              : #endif
    3547              : 
    3548              :          IF ("r" == "l" .AND. real_4 == int_8) THEN
    3549              :             CPABORT("Minimal location not available with long integers @ "//routineN)
    3550              :          END IF
    3551            0 :          CALL mp_timeset(routineN, handle)
    3552              : 
    3553              : #if defined(__parallel)
    3554            0 :          msglen = SIZE(msg)
    3555            0 :          ALLOCATE (res(1:msglen))
    3556            0 :          CALL mpi_allreduce(msg, res, msglen/2, MPI_2REAL, MPI_MINLOC, comm%handle, ierr)
    3557            0 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_allreduce @ "//routineN)
    3558            0 :          msg = res
    3559            0 :          DEALLOCATE (res)
    3560            0 :          CALL add_perf(perf_id=3, count=1, msg_size=msglen*real_4_size)
    3561              : #else
    3562              :          MARK_USED(msg)
    3563              :          MARK_USED(comm)
    3564              : #endif
    3565            0 :          CALL mp_timestop(handle)
    3566            0 :       END SUBROUTINE mp_minloc_rv
    3567              : 
    3568              : ! **************************************************************************************************
    3569              : !> \brief Finds the location of the maximal element in a vector.
    3570              : !> \param[in,out] msg         Find location of maximum element among these
    3571              : !>                            data (input).
    3572              : !> \param[in] comm            Message passing environment identifier
    3573              : !> \par MPI mapping
    3574              : !>      mpi_allreduce with the MPI_MAXLOC reduction function identifier
    3575              : !> \par Invalid data types
    3576              : !>      This routine is invalid for (int_8) data!
    3577              : ! **************************************************************************************************
    3578     10111720 :       SUBROUTINE mp_maxloc_dv(msg, comm)
    3579              :          REAL(kind=real_8), CONTIGUOUS, INTENT(INOUT)         :: msg(:)
    3580              :          CLASS(mp_comm_type), INTENT(IN)                      :: comm
    3581              : 
    3582              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_maxloc_dv'
    3583              : 
    3584              :          INTEGER                                  :: handle
    3585              : #if defined(__parallel)
    3586              :          INTEGER                                  :: ierr, msglen
    3587     10111720 :          REAL(kind=real_8), ALLOCATABLE           :: res(:)
    3588              : #endif
    3589              : 
    3590              :          IF ("d" == "l" .AND. real_8 == int_8) THEN
    3591              :             CPABORT("Maximal location not available with long integers @ "//routineN)
    3592              :          END IF
    3593     10111720 :          CALL mp_timeset(routineN, handle)
    3594              : 
    3595              : #if defined(__parallel)
    3596     10111720 :          msglen = SIZE(msg)
    3597     30335160 :          ALLOCATE (res(1:msglen))
    3598     10111720 :          CALL mpi_allreduce(msg, res, msglen/2, MPI_2DOUBLE_PRECISION, MPI_MAXLOC, comm%handle, ierr)
    3599     10111720 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_allreduce @ "//routineN)
    3600     30335160 :          msg = res
    3601     10111720 :          DEALLOCATE (res)
    3602     10111720 :          CALL add_perf(perf_id=3, count=1, msg_size=msglen*real_8_size)
    3603              : #else
    3604              :          MARK_USED(msg)
    3605              :          MARK_USED(comm)
    3606              : #endif
    3607     10111720 :          CALL mp_timestop(handle)
    3608     10111720 :       END SUBROUTINE mp_maxloc_dv
    3609              : 
    3610              : ! **************************************************************************************************
    3611              : !> \brief Finds the location of the maximal element in a vector.
    3612              : !> \param[in,out] msg         Find location of maximum element among these
    3613              : !>                            data (input).
    3614              : !> \param[in] comm            Message passing environment identifier
    3615              : !> \par MPI mapping
    3616              : !>      mpi_allreduce with the MPI_MAXLOC reduction function identifier
    3617              : !> \par Invalid data types
    3618              : !>      This routine is invalid for (int_8) data!
    3619              : ! **************************************************************************************************
    3620          218 :       SUBROUTINE mp_maxloc_iv(msg, comm)
    3621              :          INTEGER(KIND=int_4), CONTIGUOUS, INTENT(INOUT)       :: msg(:)
    3622              :          CLASS(mp_comm_type), INTENT(IN)                      :: comm
    3623              : 
    3624              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_maxloc_iv'
    3625              : 
    3626              :          INTEGER                                  :: handle
    3627              : #if defined(__parallel)
    3628              :          INTEGER                                  :: ierr, msglen
    3629          218 :          INTEGER(KIND=int_4), ALLOCATABLE         :: res(:)
    3630              : #endif
    3631              : 
    3632              :          IF ("i" == "l" .AND. int_4 == int_8) THEN
    3633              :             CPABORT("Maximal location not available with long integers @ "//routineN)
    3634              :          END IF
    3635          218 :          CALL mp_timeset(routineN, handle)
    3636              : 
    3637              : #if defined(__parallel)
    3638          218 :          msglen = SIZE(msg)
    3639          654 :          ALLOCATE (res(1:msglen))
    3640          218 :          CALL mpi_allreduce(msg, res, msglen/2, MPI_2INTEGER, MPI_MAXLOC, comm%handle, ierr)
    3641          218 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_allreduce @ "//routineN)
    3642          654 :          msg = res
    3643          218 :          DEALLOCATE (res)
    3644          218 :          CALL add_perf(perf_id=3, count=1, msg_size=msglen*int_4_size)
    3645              : #else
    3646              :          MARK_USED(msg)
    3647              :          MARK_USED(comm)
    3648              : #endif
    3649          218 :          CALL mp_timestop(handle)
    3650          218 :       END SUBROUTINE mp_maxloc_iv
    3651              : 
    3652              : ! **************************************************************************************************
    3653              : !> \brief Finds the location of the maximal element in a vector.
    3654              : !> \param[in,out] msg         Find location of maximum element among these
    3655              : !>                            data (input).
    3656              : !> \param[in] comm            Message passing environment identifier
    3657              : !> \par MPI mapping
    3658              : !>      mpi_allreduce with the MPI_MAXLOC reduction function identifier
    3659              : !> \par Invalid data types
    3660              : !>      This routine is invalid for (int_8) data!
    3661              : ! **************************************************************************************************
    3662            0 :       SUBROUTINE mp_maxloc_lv(msg, comm)
    3663              :          INTEGER(KIND=int_8), CONTIGUOUS, INTENT(INOUT)       :: msg(:)
    3664              :          CLASS(mp_comm_type), INTENT(IN)                      :: comm
    3665              : 
    3666              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_maxloc_lv'
    3667              : 
    3668              :          INTEGER                                  :: handle
    3669              : #if defined(__parallel)
    3670              :          INTEGER                                  :: ierr, msglen
    3671            0 :          INTEGER(KIND=int_8), ALLOCATABLE         :: res(:)
    3672              : #endif
    3673              : 
    3674              :          IF ("l" == "l" .AND. int_8 == int_8) THEN
    3675            0 :             CPABORT("Maximal location not available with long integers @ "//routineN)
    3676              :          END IF
    3677            0 :          CALL mp_timeset(routineN, handle)
    3678              : 
    3679              : #if defined(__parallel)
    3680            0 :          msglen = SIZE(msg)
    3681            0 :          ALLOCATE (res(1:msglen))
    3682            0 :          CALL mpi_allreduce(msg, res, msglen/2, MPI_INTEGER8, MPI_MAXLOC, comm%handle, ierr)
    3683            0 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_allreduce @ "//routineN)
    3684            0 :          msg = res
    3685            0 :          DEALLOCATE (res)
    3686            0 :          CALL add_perf(perf_id=3, count=1, msg_size=msglen*int_8_size)
    3687              : #else
    3688              :          MARK_USED(msg)
    3689              :          MARK_USED(comm)
    3690              : #endif
    3691            0 :          CALL mp_timestop(handle)
    3692            0 :       END SUBROUTINE mp_maxloc_lv
    3693              : 
    3694              : ! **************************************************************************************************
    3695              : !> \brief Finds the location of the maximal element in a vector.
    3696              : !> \param[in,out] msg         Find location of maximum element among these
    3697              : !>                            data (input).
    3698              : !> \param[in] comm            Message passing environment identifier
    3699              : !> \par MPI mapping
    3700              : !>      mpi_allreduce with the MPI_MAXLOC reduction function identifier
    3701              : !> \par Invalid data types
    3702              : !>      This routine is invalid for (int_8) data!
    3703              : ! **************************************************************************************************
    3704            0 :       SUBROUTINE mp_maxloc_rv(msg, comm)
    3705              :          REAL(kind=real_4), CONTIGUOUS, INTENT(INOUT)         :: msg(:)
    3706              :          CLASS(mp_comm_type), INTENT(IN)                      :: comm
    3707              : 
    3708              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_maxloc_rv'
    3709              : 
    3710              :          INTEGER                                  :: handle
    3711              : #if defined(__parallel)
    3712              :          INTEGER                                  :: ierr, msglen
    3713            0 :          REAL(kind=real_4), ALLOCATABLE           :: res(:)
    3714              : #endif
    3715              : 
    3716              :          IF ("r" == "l" .AND. real_4 == int_8) THEN
    3717              :             CPABORT("Maximal location not available with long integers @ "//routineN)
    3718              :          END IF
    3719            0 :          CALL mp_timeset(routineN, handle)
    3720              : 
    3721              : #if defined(__parallel)
    3722            0 :          msglen = SIZE(msg)
    3723            0 :          ALLOCATE (res(1:msglen))
    3724            0 :          CALL mpi_allreduce(msg, res, msglen/2, MPI_2REAL, MPI_MAXLOC, comm%handle, ierr)
    3725            0 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_allreduce @ "//routineN)
    3726            0 :          msg = res
    3727            0 :          DEALLOCATE (res)
    3728            0 :          CALL add_perf(perf_id=3, count=1, msg_size=msglen*real_4_size)
    3729              : #else
    3730              :          MARK_USED(msg)
    3731              :          MARK_USED(comm)
    3732              : #endif
    3733            0 :          CALL mp_timestop(handle)
    3734            0 :       END SUBROUTINE mp_maxloc_rv
    3735              : 
    3736              : ! **************************************************************************************************
    3737              : !> \brief Logical OR reduction
    3738              : !> \param[in,out] msg         Datum to perform inclusive disjunction (input)
    3739              : !>                            and resultant inclusive disjunction (output)
    3740              : !> \param[in] comm            Message passing environment identifier
    3741              : !> \par MPI mapping
    3742              : !>      mpi_allreduce
    3743              : ! **************************************************************************************************
    3744        59009 :       SUBROUTINE mp_sum_b(msg, comm)
    3745              :          LOGICAL, INTENT(INOUT)                             :: msg
    3746              :          CLASS(mp_comm_type), INTENT(IN)                                :: comm
    3747              : 
    3748              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_sum_b'
    3749              : 
    3750              :          INTEGER                                            :: handle
    3751              : #if defined(__parallel)
    3752              :          INTEGER :: ierr, msglen
    3753              : #endif
    3754              : 
    3755        59009 :          CALL mp_timeset(routineN, handle)
    3756              : #if defined(__parallel)
    3757        59009 :          msglen = 1
    3758        59009 :          IF (comm%num_pe > 1) THEN
    3759         3834 :             CALL mpi_allreduce(MPI_IN_PLACE, msg, msglen, MPI_LOGICAL, MPI_LOR, comm%handle, ierr)
    3760         3834 :             IF (ierr /= 0) CALL mp_stop(ierr, "mpi_allreduce @ "//routineN)
    3761              :          END IF
    3762              : #else
    3763              :          MARK_USED(msg)
    3764              :          MARK_USED(comm)
    3765              : #endif
    3766        59009 :          CALL mp_timestop(handle)
    3767        59009 :       END SUBROUTINE mp_sum_b
    3768              : 
    3769              : ! **************************************************************************************************
    3770              : !> \brief Logical OR reduction
    3771              : !> \param[in,out] msg         Datum to perform inclusive disjunction (input)
    3772              : !>                            and resultant inclusive disjunction (output)
    3773              : !> \param[in] comm             Message passing environment identifier
    3774              : !> \par MPI mapping
    3775              : !>      mpi_allreduce
    3776              : ! **************************************************************************************************
    3777            0 :       SUBROUTINE mp_sum_bv(msg, comm)
    3778              :          LOGICAL, DIMENSION(:), CONTIGUOUS, INTENT(INOUT)               :: msg
    3779              :          CLASS(mp_comm_type), INTENT(IN)                                :: comm
    3780              : 
    3781              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_sum_bv'
    3782              : 
    3783              :          INTEGER                                            :: handle
    3784              : #if defined(__parallel)
    3785              :          INTEGER :: ierr, msglen
    3786              : #endif
    3787              : 
    3788            0 :          CALL mp_timeset(routineN, handle)
    3789              : #if defined(__parallel)
    3790            0 :          msglen = SIZE(msg)
    3791            0 :          IF (msglen > 0 .AND. comm%num_pe > 1) THEN
    3792            0 :             CALL mpi_allreduce(MPI_IN_PLACE, msg, msglen, MPI_LOGICAL, MPI_LOR, comm%handle, ierr)
    3793            0 :             IF (ierr /= 0) CALL mp_stop(ierr, "mpi_allreduce @ "//routineN)
    3794              :          END IF
    3795              : #else
    3796              :          MARK_USED(msg)
    3797              :          MARK_USED(comm)
    3798              : #endif
    3799            0 :          CALL mp_timestop(handle)
    3800            0 :       END SUBROUTINE mp_sum_bv
    3801              : 
    3802              : ! **************************************************************************************************
    3803              : !> \brief Logical OR reduction
    3804              : !> \param[in,out] msg         Datum to perform inclusive disjunction (input)
    3805              : !>                            and resultant inclusive disjunction (output)
    3806              : !> \param[in] comm             Message passing environment identifier
    3807              : !> \param request ...
    3808              : !> \par MPI mapping
    3809              : !>      mpi_allreduce
    3810              : ! **************************************************************************************************
    3811            0 :       SUBROUTINE mp_isum_bv(msg, comm, request)
    3812              :          LOGICAL, DIMENSION(:), INTENT(INOUT)               :: msg
    3813              :          CLASS(mp_comm_type), INTENT(IN)                                :: comm
    3814              :          TYPE(mp_request_type), INTENT(INOUT)                             :: request
    3815              : 
    3816              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_isum_bv'
    3817              : 
    3818              :          INTEGER                                            :: handle
    3819              : #if defined(__parallel)
    3820              :          INTEGER :: ierr, msglen
    3821              : #endif
    3822              : 
    3823            0 :          CALL mp_timeset(routineN, handle)
    3824              : #if defined(__parallel)
    3825            0 :          msglen = SIZE(msg)
    3826            0 :          CPASSERT(IS_CONTIGUOUS(msg) .OR. PRODUCT(SHAPE(msg)) == 0)
    3827              : 
    3828            0 :          IF (msglen > 0 .AND. comm%num_pe > 1) THEN
    3829            0 :             CALL mpi_iallreduce(MPI_IN_PLACE, msg, msglen, MPI_LOGICAL, MPI_LOR, comm%handle, request%handle, ierr)
    3830            0 :             IF (ierr /= 0) CALL mp_stop(ierr, "mpi_allreduce @ "//routineN)
    3831              :          ELSE
    3832            0 :             request = mp_request_null
    3833              :          END IF
    3834              : #else
    3835              :          MARK_USED(msg)
    3836              :          MARK_USED(comm)
    3837              :          request = mp_request_null
    3838              : #endif
    3839            0 :          CALL mp_timestop(handle)
    3840            0 :       END SUBROUTINE mp_isum_bv
    3841              : 
    3842              : ! **************************************************************************************************
    3843              : !> \brief Get Version of the MPI Library (MPI 3)
    3844              : !> \param[out] version        Version of the library,
    3845              : !>                            declared as CHARACTER(LEN=mp_max_library_version_string)
    3846              : !> \param[out] resultlen      Length (in printable characters) of
    3847              : !>                            the result returned in version (integer)
    3848              : ! **************************************************************************************************
    3849            0 :       SUBROUTINE mp_get_library_version(version, resultlen)
    3850              :          CHARACTER(len=*), INTENT(OUT)                      :: version
    3851              :          INTEGER, INTENT(OUT)                               :: resultlen
    3852              : 
    3853              : #if defined(__parallel)
    3854              :          INTEGER                                            :: ierr
    3855              : #endif
    3856              : 
    3857            0 :          version = ''
    3858              : 
    3859              : #if defined(__parallel)
    3860              :          ierr = 0
    3861            0 :          CALL mpi_get_library_version(version, resultlen, ierr)
    3862            0 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_get_library_version @ mp_get_library_version")
    3863              : #else
    3864              :          resultlen = 0
    3865              : #endif
    3866            0 :       END SUBROUTINE mp_get_library_version
    3867              : 
    3868              : ! **************************************************************************************************
    3869              : !> \brief Opens a file
    3870              : !> \param[in] groupid    message passing environment identifier
    3871              : !> \param[out] fh        file handle (file storage unit)
    3872              : !> \param[in] filepath   path to the file
    3873              : !> \param amode_status   access mode
    3874              : !> \param info ...
    3875              : !> \par MPI-I/O mapping  mpi_file_open
    3876              : !> \par STREAM-I/O mapping  OPEN
    3877              : !>
    3878              : !> \param[in](optional) info   info object
    3879              : !> \par History
    3880              : !>      11.2012 created [Hossein Bani-Hashemian]
    3881              : ! **************************************************************************************************
    3882         2370 :       SUBROUTINE mp_file_open(groupid, fh, filepath, amode_status, info)
    3883              :          CLASS(mp_comm_type), INTENT(IN)                      :: groupid
    3884              :          CLASS(mp_file_type), INTENT(OUT)                     :: fh
    3885              :          CHARACTER(len=*), INTENT(IN)                         :: filepath
    3886              :          INTEGER, INTENT(IN)                                  :: amode_status
    3887              :          TYPE(mp_info_type), INTENT(IN), OPTIONAL             :: info
    3888              : 
    3889              : #if defined(__parallel)
    3890              :          INTEGER                                  :: ierr
    3891              :          MPI_INFO_TYPE                            :: my_info
    3892              : #else
    3893              :          CHARACTER(LEN=10)                        :: fstatus, fposition
    3894              :          INTEGER                                  :: amode, handle, istat
    3895              :          LOGICAL                                  :: exists, is_open
    3896              : #endif
    3897              : 
    3898              : #if defined(__parallel)
    3899              :          ierr = 0
    3900         2370 :          my_info = mpi_info_null
    3901         2370 :          IF (PRESENT(info)) my_info = info%handle
    3902         2370 :          CALL mpi_file_open(groupid%handle, filepath, amode_status, my_info, fh%handle, ierr)
    3903         2370 :          CALL mpi_file_set_errhandler(fh%handle, MPI_ERRORS_RETURN, ierr)
    3904         2370 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_file_set_errhandler @ mp_file_open")
    3905              : #else
    3906              :          MARK_USED(groupid)
    3907              :          MARK_USED(info)
    3908              :          amode = amode_status
    3909              :          IF (amode > file_amode_append) THEN
    3910              :             fposition = "APPEND"
    3911              :             amode = amode - file_amode_append
    3912              :          ELSE
    3913              :             fposition = "REWIND"
    3914              :          END IF
    3915              :          IF ((amode == file_amode_create) .OR. &
    3916              :              (amode == file_amode_create + file_amode_wronly) .OR. &
    3917              :              (amode == file_amode_create + file_amode_wronly + file_amode_excl)) THEN
    3918              :             fstatus = "UNKNOWN"
    3919              :          ELSE
    3920              :             fstatus = "OLD"
    3921              :          END IF
    3922              :          ! Get a new unit number
    3923              :          DO handle = 1, 999
    3924              :             INQUIRE (UNIT=handle, EXIST=exists, OPENED=is_open, IOSTAT=istat)
    3925              :             IF (exists .AND. (.NOT. is_open) .AND. (istat == 0)) EXIT
    3926              :          END DO
    3927              :          OPEN (UNIT=handle, FILE=filepath, STATUS=fstatus, ACCESS="STREAM", POSITION=fposition)
    3928              :          fh%handle = handle
    3929              : #endif
    3930         2370 :       END SUBROUTINE mp_file_open
    3931              : 
    3932              : ! **************************************************************************************************
    3933              : !> \brief Deletes a file. Auxiliary routine to emulate 'replace' action for mp_file_open.
    3934              : !>        Only the master processor should call this routine.
    3935              : !> \param[in] filepath   path to the file
    3936              : !> \param[in](optional) info   info object
    3937              : !> \par History
    3938              : !>      11.2017 created [Nico Holmberg]
    3939              : ! **************************************************************************************************
    3940          623 :       SUBROUTINE mp_file_delete(filepath, info)
    3941              :          CHARACTER(len=*), INTENT(IN)             :: filepath
    3942              :          TYPE(mp_info_type), INTENT(IN), OPTIONAL :: info
    3943              : 
    3944              : #if defined(__parallel)
    3945              :          INTEGER                                  :: ierr
    3946              :          MPI_INFO_TYPE                            :: my_info
    3947              :          LOGICAL                                  :: exists
    3948              : 
    3949          623 :          ierr = 0
    3950          623 :          my_info = mpi_info_null
    3951          623 :          IF (PRESENT(info)) my_info = info%handle
    3952          623 :          INQUIRE (FILE=filepath, EXIST=exists)
    3953          623 :          IF (exists) CALL mpi_file_delete(filepath, my_info, ierr)
    3954          623 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_file_set_errhandler @ mp_file_delete")
    3955              : #else
    3956              :          MARK_USED(filepath)
    3957              :          MARK_USED(info)
    3958              :          ! Explicit file delete not necessary, handled by subsequent call to open_file with action 'replace'
    3959              : #endif
    3960              : 
    3961          623 :       END SUBROUTINE mp_file_delete
    3962              : 
    3963              : ! **************************************************************************************************
    3964              : !> \brief Closes a file
    3965              : !> \param[in] fh   file handle (file storage unit)
    3966              : !> \par MPI-I/O mapping   mpi_file_close
    3967              : !> \par STREAM-I/O mapping   CLOSE
    3968              : !>
    3969              : !> \par History
    3970              : !>      11.2012 created [Hossein Bani-Hashemian]
    3971              : ! **************************************************************************************************
    3972         4740 :       SUBROUTINE mp_file_close(fh)
    3973              :          CLASS(mp_file_type), INTENT(INOUT)                             :: fh
    3974              : 
    3975              : #if defined(__parallel)
    3976              :          INTEGER                                            :: ierr
    3977              : 
    3978              :          ierr = 0
    3979         2370 :          CALL mpi_file_set_errhandler(fh%handle, MPI_ERRORS_RETURN, ierr)
    3980         2370 :          CALL mpi_file_close(fh%handle, ierr)
    3981         2370 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_file_set_errhandler @ mp_file_close")
    3982              : #else
    3983              :          CLOSE (fh%handle)
    3984              :          fh%handle = mp_file_null_handle
    3985              : #endif
    3986         2370 :       END SUBROUTINE mp_file_close
    3987              : 
    3988            0 :       SUBROUTINE mp_file_assign(fh_new, fh_old)
    3989              :          CLASS(mp_file_type), INTENT(OUT) :: fh_new
    3990              :          CLASS(mp_file_type), INTENT(IN) :: fh_old
    3991              : 
    3992            0 :          fh_new%handle = fh_old%handle
    3993              : 
    3994            0 :       END SUBROUTINE
    3995              : 
    3996              : ! **************************************************************************************************
    3997              : !> \brief Returns the file size
    3998              : !> \param[in] fh file handle (file storage unit)
    3999              : !> \param[out] file_size  the file size
    4000              : !> \par MPI-I/O mapping   mpi_file_get_size
    4001              : !> \par STREAM-I/O mapping   INQUIRE
    4002              : !>
    4003              : !> \par History
    4004              : !>      12.2012 created [Hossein Bani-Hashemian]
    4005              : ! **************************************************************************************************
    4006            0 :       SUBROUTINE mp_file_get_size(fh, file_size)
    4007              :          CLASS(mp_file_type), INTENT(IN)                                :: fh
    4008              :          INTEGER(kind=file_offset), INTENT(OUT)             :: file_size
    4009              : 
    4010              : #if defined(__parallel)
    4011              :          INTEGER                                            :: ierr
    4012              : #endif
    4013              : 
    4014              : #if defined(__parallel)
    4015              :          ierr = 0
    4016            0 :          CALL mpi_file_set_errhandler(fh%handle, MPI_ERRORS_RETURN, ierr)
    4017            0 :          CALL mpi_file_get_size(fh%handle, file_size, ierr)
    4018            0 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_file_set_errhandler @ mp_file_get_size")
    4019              : #else
    4020              :          INQUIRE (UNIT=fh%handle, SIZE=file_size)
    4021              : #endif
    4022            0 :       END SUBROUTINE mp_file_get_size
    4023              : 
    4024              : ! **************************************************************************************************
    4025              : !> \brief Returns the file position
    4026              : !> \param[in] fh file handle (file storage unit)
    4027              : !> \param[out] file_size  the file position
    4028              : !> \par MPI-I/O mapping   mpi_file_get_position
    4029              : !> \par STREAM-I/O mapping   INQUIRE
    4030              : !>
    4031              : !> \par History
    4032              : !>      11.2017 created [Nico Holmberg]
    4033              : ! **************************************************************************************************
    4034         4064 :       SUBROUTINE mp_file_get_position(fh, pos)
    4035              :          CLASS(mp_file_type), INTENT(IN)                                :: fh
    4036              :          INTEGER(kind=file_offset), INTENT(OUT)             :: pos
    4037              : 
    4038              : #if defined(__parallel)
    4039              :          INTEGER                                            :: ierr
    4040              : #endif
    4041              : 
    4042              : #if defined(__parallel)
    4043              :          ierr = 0
    4044         2032 :          CALL mpi_file_set_errhandler(fh%handle, MPI_ERRORS_RETURN, ierr)
    4045         2032 :          CALL mpi_file_get_position(fh%handle, pos, ierr)
    4046         2032 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_file_set_errhandler @ mp_file_get_position")
    4047              : #else
    4048              :          INQUIRE (UNIT=fh%handle, POS=pos)
    4049              : #endif
    4050         2032 :       END SUBROUTINE mp_file_get_position
    4051              : 
    4052              : ! **************************************************************************************************
    4053              : !> \brief (parallel) Blocking individual file write using explicit offsets
    4054              : !>        (serial) Unformatted stream write
    4055              : !> \param[in] fh     file handle (file storage unit)
    4056              : !> \param[in] offset file offset (position)
    4057              : !> \param[in] msg    data to be written to the file
    4058              : !> \param msglen ...
    4059              : !> \par MPI-I/O mapping   mpi_file_write_at
    4060              : !> \par STREAM-I/O mapping   WRITE
    4061              : !> \param[in](optional) msglen number of the elements of data
    4062              : ! **************************************************************************************************
    4063            0 :       SUBROUTINE mp_file_write_at_chv(fh, offset, msg, msglen)
    4064              :          CHARACTER, CONTIGUOUS, INTENT(IN)                      :: msg(:)
    4065              :          CLASS(mp_file_type), INTENT(IN)                        :: fh
    4066              :          INTEGER, INTENT(IN), OPTIONAL              :: msglen
    4067              :          INTEGER(kind=file_offset), INTENT(IN)      :: offset
    4068              : 
    4069              : #if defined(__parallel)
    4070              :          INTEGER                                    :: ierr, msg_len
    4071              : #endif
    4072              : 
    4073              : #if defined(__parallel)
    4074            0 :          msg_len = SIZE(msg)
    4075            0 :          IF (PRESENT(msglen)) msg_len = msglen
    4076            0 :          CALL MPI_FILE_WRITE_AT(fh%handle, offset, msg, msg_len, MPI_CHARACTER, MPI_STATUS_IGNORE, ierr)
    4077            0 :          IF (ierr /= 0) &
    4078            0 :             CPABORT("mpi_file_write_at_chv @ mp_file_write_at_chv")
    4079              : #else
    4080              :          MARK_USED(msglen)
    4081              :          WRITE (UNIT=fh%handle, POS=offset + 1) msg
    4082              : #endif
    4083            0 :       END SUBROUTINE mp_file_write_at_chv
    4084              : 
    4085              : ! **************************************************************************************************
    4086              : !> \brief ...
    4087              : !> \param fh ...
    4088              : !> \param offset ...
    4089              : !> \param msg ...
    4090              : ! **************************************************************************************************
    4091        10073 :       SUBROUTINE mp_file_write_at_ch(fh, offset, msg)
    4092              :          CHARACTER(LEN=*), INTENT(IN)               :: msg
    4093              :          CLASS(mp_file_type), INTENT(IN)            :: fh
    4094              :          INTEGER(kind=file_offset), INTENT(IN)      :: offset
    4095              : 
    4096              : #if defined(__parallel)
    4097              :          INTEGER                                    :: ierr
    4098              : #endif
    4099              : 
    4100              : #if defined(__parallel)
    4101        10073 :          CALL MPI_FILE_WRITE_AT(fh%handle, offset, msg, LEN(msg), MPI_CHARACTER, MPI_STATUS_IGNORE, ierr)
    4102        10073 :          IF (ierr /= 0) &
    4103            0 :             CPABORT("mpi_file_write_at_ch @ mp_file_write_at_ch")
    4104              : #else
    4105              :          WRITE (UNIT=fh%handle, POS=offset + 1) msg
    4106              : #endif
    4107        10073 :       END SUBROUTINE mp_file_write_at_ch
    4108              : 
    4109              : ! **************************************************************************************************
    4110              : !> \brief (parallel) Blocking collective file write using explicit offsets
    4111              : !>        (serial) Unformatted stream write
    4112              : !> \param fh ...
    4113              : !> \param offset ...
    4114              : !> \param msg ...
    4115              : !> \param msglen ...
    4116              : !> \par MPI-I/O mapping   mpi_file_write_at_all
    4117              : !> \par STREAM-I/O mapping   WRITE
    4118              : ! **************************************************************************************************
    4119            0 :       SUBROUTINE mp_file_write_at_all_chv(fh, offset, msg, msglen)
    4120              :          CHARACTER, CONTIGUOUS, INTENT(IN)                      :: msg(:)
    4121              :          CLASS(mp_file_type), INTENT(IN)                        :: fh
    4122              :          INTEGER, INTENT(IN), OPTIONAL              :: msglen
    4123              :          INTEGER(kind=file_offset), INTENT(IN)      :: offset
    4124              : 
    4125              : #if defined(__parallel)
    4126              :          INTEGER                                    :: ierr, msg_len
    4127              : #endif
    4128              : 
    4129              : #if defined(__parallel)
    4130            0 :          msg_len = SIZE(msg)
    4131            0 :          IF (PRESENT(msglen)) msg_len = msglen
    4132            0 :          CALL MPI_FILE_WRITE_AT_ALL(fh%handle, offset, msg, msg_len, MPI_CHARACTER, MPI_STATUS_IGNORE, ierr)
    4133            0 :          IF (ierr /= 0) &
    4134            0 :             CPABORT("mpi_file_write_at_all_chv @ mp_file_write_at_all_chv")
    4135              : #else
    4136              :          MARK_USED(msglen)
    4137              :          WRITE (UNIT=fh%handle, POS=offset + 1) msg
    4138              : #endif
    4139            0 :       END SUBROUTINE mp_file_write_at_all_chv
    4140              : 
    4141              : ! **************************************************************************************************
    4142              : !> \brief wrapper to MPI_File_write_at_all
    4143              : !> \param fh ...
    4144              : !> \param offset ...
    4145              : !> \param msg ...
    4146              : ! **************************************************************************************************
    4147            0 :       SUBROUTINE mp_file_write_at_all_ch(fh, offset, msg)
    4148              :          CHARACTER(LEN=*), INTENT(IN)               :: msg
    4149              :          CLASS(mp_file_type), INTENT(IN)            :: fh
    4150              :          INTEGER(kind=file_offset), INTENT(IN)      :: offset
    4151              : 
    4152              : #if defined(__parallel)
    4153              :          INTEGER                                    :: ierr
    4154              : #endif
    4155              : 
    4156              : #if defined(__parallel)
    4157            0 :          CALL MPI_FILE_WRITE_AT_ALL(fh%handle, offset, msg, LEN(msg), MPI_CHARACTER, MPI_STATUS_IGNORE, ierr)
    4158            0 :          IF (ierr /= 0) &
    4159            0 :             CPABORT("mpi_file_write_at_all_ch @ mp_file_write_at_all_ch")
    4160              : #else
    4161              :          WRITE (UNIT=fh%handle, POS=offset + 1) msg
    4162              : #endif
    4163            0 :       END SUBROUTINE mp_file_write_at_all_ch
    4164              : 
    4165              : ! **************************************************************************************************
    4166              : !> \brief (parallel) Blocking individual file read using explicit offsets
    4167              : !>        (serial) Unformatted stream read
    4168              : !> \param[in] fh     file handle (file storage unit)
    4169              : !> \param[in] offset file offset (position)
    4170              : !> \param[out] msg   data to be read from the file
    4171              : !> \param msglen ...
    4172              : !> \par MPI-I/O mapping   mpi_file_read_at
    4173              : !> \par STREAM-I/O mapping   READ
    4174              : !> \param[in](optional) msglen  number of elements of data
    4175              : ! **************************************************************************************************
    4176            0 :       SUBROUTINE mp_file_read_at_chv(fh, offset, msg, msglen)
    4177              :          CHARACTER, CONTIGUOUS, INTENT(OUT)                     :: msg(:)
    4178              :          CLASS(mp_file_type), INTENT(IN)                        :: fh
    4179              :          INTEGER, INTENT(IN), OPTIONAL              :: msglen
    4180              :          INTEGER(kind=file_offset), INTENT(IN)      :: offset
    4181              : 
    4182              : #if defined(__parallel)
    4183              :          INTEGER                                    :: ierr, msg_len
    4184              : #endif
    4185              : 
    4186              : #if defined(__parallel)
    4187            0 :          msg_len = SIZE(msg)
    4188            0 :          IF (PRESENT(msglen)) msg_len = msglen
    4189            0 :          CALL MPI_FILE_READ_AT(fh%handle, offset, msg, msg_len, MPI_CHARACTER, MPI_STATUS_IGNORE, ierr)
    4190            0 :          IF (ierr /= 0) &
    4191            0 :             CPABORT("mpi_file_read_at_chv @ mp_file_read_at_chv")
    4192              : #else
    4193              :          MARK_USED(msglen)
    4194              :          READ (UNIT=fh%handle, POS=offset + 1) msg
    4195              : #endif
    4196            0 :       END SUBROUTINE mp_file_read_at_chv
    4197              : 
    4198              : ! **************************************************************************************************
    4199              : !> \brief wrapper to MPI_File_read_at
    4200              : !> \param fh ...
    4201              : !> \param offset ...
    4202              : !> \param msg ...
    4203              : ! **************************************************************************************************
    4204            0 :       SUBROUTINE mp_file_read_at_ch(fh, offset, msg)
    4205              :          CHARACTER(LEN=*), INTENT(OUT)              :: msg
    4206              :          CLASS(mp_file_type), INTENT(IN)            :: fh
    4207              :          INTEGER(kind=file_offset), INTENT(IN)      :: offset
    4208              : 
    4209              : #if defined(__parallel)
    4210              :          INTEGER                                    :: ierr
    4211              : #endif
    4212              : 
    4213              : #if defined(__parallel)
    4214            0 :          CALL MPI_FILE_READ_AT(fh%handle, offset, msg, LEN(msg), MPI_CHARACTER, MPI_STATUS_IGNORE, ierr)
    4215            0 :          IF (ierr /= 0) &
    4216            0 :             CPABORT("mpi_file_read_at_ch @ mp_file_read_at_ch")
    4217              : #else
    4218              :          READ (UNIT=fh%handle, POS=offset + 1) msg
    4219              : #endif
    4220            0 :       END SUBROUTINE mp_file_read_at_ch
    4221              : 
    4222              : ! **************************************************************************************************
    4223              : !> \brief (parallel) Blocking collective file read using explicit offsets
    4224              : !>        (serial) Unformatted stream read
    4225              : !> \param fh ...
    4226              : !> \param offset ...
    4227              : !> \param msg ...
    4228              : !> \param msglen ...
    4229              : !> \par MPI-I/O mapping    mpi_file_read_at_all
    4230              : !> \par STREAM-I/O mapping   READ
    4231              : ! **************************************************************************************************
    4232            0 :       SUBROUTINE mp_file_read_at_all_chv(fh, offset, msg, msglen)
    4233              :          CHARACTER, INTENT(OUT)                     :: msg(:)
    4234              :          CLASS(mp_file_type), INTENT(IN)                        :: fh
    4235              :          INTEGER, INTENT(IN), OPTIONAL              :: msglen
    4236              :          INTEGER(kind=file_offset), INTENT(IN)      :: offset
    4237              : 
    4238              : #if defined(__parallel)
    4239              :          INTEGER                                    :: ierr, msg_len
    4240              : #endif
    4241              : 
    4242              : #if defined(__parallel)
    4243            0 :          msg_len = SIZE(msg)
    4244            0 :          IF (PRESENT(msglen)) msg_len = msglen
    4245            0 :          CALL MPI_FILE_READ_AT_ALL(fh%handle, offset, msg, msg_len, MPI_CHARACTER, MPI_STATUS_IGNORE, ierr)
    4246            0 :          IF (ierr /= 0) &
    4247            0 :             CPABORT("mpi_file_read_at_all_chv @ mp_file_read_at_all_chv")
    4248              : #else
    4249              :          MARK_USED(msglen)
    4250              :          READ (UNIT=fh%handle, POS=offset + 1) msg
    4251              : #endif
    4252            0 :       END SUBROUTINE mp_file_read_at_all_chv
    4253              : 
    4254              : ! **************************************************************************************************
    4255              : !> \brief wrapper to MPI_File_read_at_all
    4256              : !> \param fh ...
    4257              : !> \param offset ...
    4258              : !> \param msg ...
    4259              : ! **************************************************************************************************
    4260           30 :       SUBROUTINE mp_file_read_at_all_ch(fh, offset, msg)
    4261              :          CHARACTER(LEN=*), INTENT(OUT)              :: msg
    4262              :          CLASS(mp_file_type), INTENT(IN)            :: fh
    4263              :          INTEGER(kind=file_offset), INTENT(IN)      :: offset
    4264              : 
    4265              : #if defined(__parallel)
    4266              :          INTEGER                                    :: ierr
    4267              : #endif
    4268              : 
    4269              : #if defined(__parallel)
    4270           30 :          CALL MPI_FILE_READ_AT_ALL(fh%handle, offset, msg, LEN(msg), MPI_CHARACTER, MPI_STATUS_IGNORE, ierr)
    4271           30 :          IF (ierr /= 0) &
    4272            0 :             CPABORT("mpi_file_read_at_all_ch @ mp_file_read_at_all_ch")
    4273              : #else
    4274              :          READ (UNIT=fh%handle, POS=offset + 1) msg
    4275              : #endif
    4276           30 :       END SUBROUTINE mp_file_read_at_all_ch
    4277              : 
    4278              : ! **************************************************************************************************
    4279              : !> \brief Returns the size of a data type in bytes
    4280              : !> \param[in] type_descriptor  data type
    4281              : !> \param[out] type_size       size of the data type
    4282              : !> \par MPI mapping
    4283              : !>      mpi_type_size
    4284              : !>
    4285              : ! **************************************************************************************************
    4286            0 :       SUBROUTINE mp_type_size(type_descriptor, type_size)
    4287              :          TYPE(mp_type_descriptor_type), INTENT(IN)          :: type_descriptor
    4288              :          INTEGER, INTENT(OUT)                               :: type_size
    4289              : 
    4290              : #if defined(__parallel)
    4291              :          INTEGER                                            :: ierr
    4292              : 
    4293              :          ierr = 0
    4294            0 :          CALL MPI_TYPE_SIZE(type_descriptor%type_handle, type_size, ierr)
    4295            0 :          IF (ierr /= 0) &
    4296            0 :             CPABORT("mpi_type_size failed @ mp_type_size")
    4297              : #else
    4298              :          SELECT CASE (type_descriptor%type_handle)
    4299              :          CASE (1)
    4300              :             type_size = real_4_size
    4301              :          CASE (3)
    4302              :             type_size = real_8_size
    4303              :          CASE (5)
    4304              :             type_size = 2*real_4_size
    4305              :          CASE (7)
    4306              :             type_size = 2*real_8_size
    4307              :          END SELECT
    4308              : #endif
    4309            0 :       END SUBROUTINE mp_type_size
    4310              : 
    4311              : ! **************************************************************************************************
    4312              : !> \brief wrapper to MPI_Type_create_struct
    4313              : !> \param subtypes ...
    4314              : !> \param vector_descriptor ...
    4315              : !> \param index_descriptor ...
    4316              : !> \return ...
    4317              : ! **************************************************************************************************
    4318            0 :       FUNCTION mp_type_make_struct(subtypes, &
    4319              :                                    vector_descriptor, index_descriptor) &
    4320            0 :          RESULT(type_descriptor)
    4321              :          TYPE(mp_type_descriptor_type), &
    4322              :             DIMENSION(:), INTENT(IN)                :: subtypes
    4323              :          INTEGER, DIMENSION(2), INTENT(IN), &
    4324              :             OPTIONAL                                :: vector_descriptor
    4325              :          TYPE(mp_indexing_meta_type), &
    4326              :             INTENT(IN), OPTIONAL                    :: index_descriptor
    4327              :          TYPE(mp_type_descriptor_type)              :: type_descriptor
    4328              : 
    4329              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_type_make_struct'
    4330              : 
    4331              :          INTEGER                                    :: i, n
    4332            0 :          INTEGER, ALLOCATABLE, DIMENSION(:)         :: lengths
    4333              : #if defined(__parallel)
    4334              :          INTEGER :: ierr
    4335              :          INTEGER(kind=mpi_address_kind), &
    4336            0 :             ALLOCATABLE, DIMENSION(:)               :: displacements
    4337              : #if defined(__MPI_F08)
    4338              :          ! Even OpenMPI 5.x misses mpi_get_address in the F08 interface
    4339              :          EXTERNAL                                   :: mpi_get_address
    4340              : #endif
    4341              : #endif
    4342            0 :          MPI_DATA_TYPE, ALLOCATABLE, DIMENSION(:) :: old_types
    4343              : 
    4344            0 :          n = SIZE(subtypes)
    4345            0 :          type_descriptor%length = 1
    4346              : #if defined(__parallel)
    4347            0 :          ierr = 0
    4348            0 :          CALL mpi_get_address(MPI_BOTTOM, type_descriptor%base, ierr)
    4349            0 :          IF (ierr /= 0) &
    4350            0 :             CPABORT("MPI_get_address @ "//routineN)
    4351            0 :          ALLOCATE (displacements(n))
    4352              : #endif
    4353            0 :          type_descriptor%vector_descriptor(1:2) = 1
    4354            0 :          type_descriptor%has_indexing = .FALSE.
    4355            0 :          ALLOCATE (type_descriptor%subtype(n))
    4356            0 :          type_descriptor%subtype(:) = subtypes(:)
    4357            0 :          ALLOCATE (lengths(n), old_types(n))
    4358            0 :          DO i = 1, SIZE(subtypes)
    4359              : #if defined(__parallel)
    4360            0 :             displacements(i) = subtypes(i)%base
    4361              : #endif
    4362            0 :             old_types(i) = subtypes(i)%type_handle
    4363            0 :             lengths(i) = subtypes(i)%length
    4364              :          END DO
    4365              : #if defined(__parallel)
    4366              :          CALL MPI_Type_create_struct(n, &
    4367              :                                      lengths, displacements, old_types, &
    4368            0 :                                      type_descriptor%type_handle, ierr)
    4369            0 :          IF (ierr /= 0) &
    4370            0 :             CPABORT("MPI_Type_create_struct @ "//routineN)
    4371            0 :          CALL MPI_Type_commit(type_descriptor%type_handle, ierr)
    4372            0 :          IF (ierr /= 0) &
    4373            0 :             CPABORT("MPI_Type_commit @ "//routineN)
    4374              : #endif
    4375            0 :          IF (PRESENT(vector_descriptor) .OR. PRESENT(index_descriptor)) THEN
    4376            0 :             CPABORT(routineN//" Vectors and indices NYI")
    4377              :          END IF
    4378            0 :       END FUNCTION mp_type_make_struct
    4379              : 
    4380              : ! **************************************************************************************************
    4381              : !> \brief wrapper to MPI_Type_free
    4382              : !> \param type_descriptor ...
    4383              : ! **************************************************************************************************
    4384            0 :       RECURSIVE SUBROUTINE mp_type_free_m(type_descriptor)
    4385              :          TYPE(mp_type_descriptor_type), INTENT(inout)       :: type_descriptor
    4386              : 
    4387              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_type_free_m'
    4388              : 
    4389              :          INTEGER                                            :: handle, i
    4390              : #if defined(__parallel)
    4391              :          INTEGER :: ierr
    4392              : #endif
    4393              : 
    4394            0 :          CALL mp_timeset(routineN, handle)
    4395              : 
    4396              :          ! If the subtype is associated, then it's a user-defined data type.
    4397              : 
    4398            0 :          IF (ASSOCIATED(type_descriptor%subtype)) THEN
    4399            0 :             DO i = 1, SIZE(type_descriptor%subtype)
    4400            0 :                CALL mp_type_free_m(type_descriptor%subtype(i))
    4401              :             END DO
    4402            0 :             DEALLOCATE (type_descriptor%subtype)
    4403              :          END IF
    4404              : #if defined(__parallel)
    4405              :          ierr = 0
    4406            0 :          CALL MPI_Type_free(type_descriptor%type_handle, ierr)
    4407            0 :          IF (ierr /= 0) &
    4408            0 :             CPABORT("MPI_Type_free @ "//routineN)
    4409              : #endif
    4410              : 
    4411            0 :          CALL mp_timestop(handle)
    4412              : 
    4413            0 :       END SUBROUTINE mp_type_free_m
    4414              : 
    4415              : ! **************************************************************************************************
    4416              : !> \brief ...
    4417              : !> \param type_descriptors ...
    4418              : ! **************************************************************************************************
    4419            0 :       SUBROUTINE mp_type_free_v(type_descriptors)
    4420              :          TYPE(mp_type_descriptor_type), DIMENSION(:), &
    4421              :             INTENT(inout)                                   :: type_descriptors
    4422              : 
    4423              :          INTEGER                                            :: i
    4424              : 
    4425            0 :          DO i = 1, SIZE(type_descriptors)
    4426            0 :             CALL mp_type_free(type_descriptors(i))
    4427              :          END DO
    4428              : 
    4429            0 :       END SUBROUTINE mp_type_free_v
    4430              : 
    4431              : ! **************************************************************************************************
    4432              : !> \brief Creates an indexed MPI type for arrays of strings using bytes for spacing (hindexed type)
    4433              : !> \param count   number of array blocks to read
    4434              : !> \param lengths lengths of each array block
    4435              : !> \param displs  byte offsets for array blocks
    4436              : !> \return container holding the created type
    4437              : !> \author Nico Holmberg [05.2017]
    4438              : ! **************************************************************************************************
    4439         4156 :       FUNCTION mp_file_type_hindexed_make_chv(count, lengths, displs) &
    4440              :          RESULT(type_descriptor)
    4441              :          INTEGER, INTENT(IN)                       :: count
    4442              :          INTEGER, DIMENSION(1:count), &
    4443              :             INTENT(IN), TARGET                     :: lengths
    4444              :          INTEGER(kind=file_offset), &
    4445              :             DIMENSION(1:count), INTENT(in), TARGET :: displs
    4446              :          TYPE(mp_file_descriptor_type)             :: type_descriptor
    4447              : 
    4448              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_file_hindexed_make_chv'
    4449              : 
    4450              :          INTEGER :: ierr, handle
    4451              : 
    4452              :          ierr = 0
    4453         2078 :          CALL mp_timeset(routineN, handle)
    4454              : 
    4455              : #if defined(__parallel)
    4456              :          CALL MPI_Type_create_hindexed(count, lengths, INT(displs, KIND=address_kind), MPI_CHARACTER, &
    4457       422514 :                                        type_descriptor%type_handle, ierr)
    4458         2078 :          IF (ierr /= 0) &
    4459            0 :             CPABORT("MPI_Type_create_hindexed @ "//routineN)
    4460         2078 :          CALL MPI_Type_commit(type_descriptor%type_handle, ierr)
    4461         2078 :          IF (ierr /= 0) &
    4462            0 :             CPABORT("MPI_Type_commit @ "//routineN)
    4463              : #else
    4464              :          type_descriptor%type_handle = 68
    4465              : #endif
    4466         2078 :          type_descriptor%length = count
    4467         2078 :          type_descriptor%has_indexing = .TRUE.
    4468         2078 :          type_descriptor%index_descriptor%index => lengths
    4469         2078 :          type_descriptor%index_descriptor%chunks => displs
    4470              : 
    4471         2078 :          CALL mp_timestop(handle)
    4472              : 
    4473         2078 :       END FUNCTION mp_file_type_hindexed_make_chv
    4474              : 
    4475              : ! **************************************************************************************************
    4476              : !> \brief Uses a previously created indexed MPI character type to tell the MPI processes
    4477              : !>        how to partition (set_view) an opened file
    4478              : !> \param fh      the file handle associated with the input file
    4479              : !> \param offset  global offset determining where the relevant data begins
    4480              : !> \param type_descriptor container for the MPI type
    4481              : !> \author Nico Holmberg [05.2017]
    4482              : ! **************************************************************************************************
    4483         2078 :       SUBROUTINE mp_file_type_set_view_chv(fh, offset, type_descriptor)
    4484              :          TYPE(mp_file_type), INTENT(IN)                      :: fh
    4485              :          INTEGER(kind=file_offset), INTENT(IN)    :: offset
    4486              :          TYPE(mp_file_descriptor_type)            :: type_descriptor
    4487              : 
    4488              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_file_set_view_chv'
    4489              : 
    4490              :          INTEGER                                   :: handle
    4491              : #if defined(__parallel)
    4492              :          INTEGER :: ierr
    4493              : #endif
    4494              : 
    4495         2078 :          CALL mp_timeset(routineN, handle)
    4496              : 
    4497              : #if defined(__parallel)
    4498              :          ierr = 0
    4499         2078 :          CALL mpi_file_set_errhandler(fh%handle, MPI_ERRORS_RETURN, ierr)
    4500              :          CALL MPI_File_set_view(fh%handle, offset, MPI_CHARACTER, &
    4501         2078 :                                 type_descriptor%type_handle, "native", MPI_INFO_NULL, ierr)
    4502         2078 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_file_set_errhandler @ MPI_File_set_view")
    4503              : #else
    4504              :          ! Uses absolute offsets stored in mp_file_descriptor_type
    4505              :          MARK_USED(fh)
    4506              :          MARK_USED(offset)
    4507              :          MARK_USED(type_descriptor)
    4508              : #endif
    4509              : 
    4510         2078 :          CALL mp_timestop(handle)
    4511              : 
    4512         2078 :       END SUBROUTINE mp_file_type_set_view_chv
    4513              : 
    4514              : ! **************************************************************************************************
    4515              : !> \brief (parallel) Collective, blocking read of a character array from a file. File access pattern
    4516              : !                    determined by a previously set file view.
    4517              : !>        (serial)   Unformatted stream read using explicit offsets
    4518              : !> \param fh     the file handle associated with the input file
    4519              : !> \param msglen the message length of an individual vector component
    4520              : !> \param ndims  the number of vector components
    4521              : !> \param buffer the buffer where the data is placed
    4522              : !> \param type_descriptor container for the MPI type
    4523              : !> \author Nico Holmberg [05.2017]
    4524              : ! **************************************************************************************************
    4525           46 :       SUBROUTINE mp_file_read_all_chv(fh, msglen, ndims, buffer, type_descriptor)
    4526              :          CLASS(mp_file_type), INTENT(IN)                       :: fh
    4527              :          INTEGER, INTENT(IN)                       :: msglen
    4528              :          INTEGER, INTENT(IN)                       :: ndims
    4529              :          CHARACTER(LEN=msglen), DIMENSION(ndims), INTENT(INOUT)   :: buffer
    4530              :          TYPE(mp_file_descriptor_type), &
    4531              :             INTENT(IN), OPTIONAL                   :: type_descriptor
    4532              : 
    4533              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_file_read_all_chv'
    4534              : 
    4535              :          INTEGER                                   :: handle
    4536              : #if defined(__parallel)
    4537              :          INTEGER:: ierr
    4538              : #else
    4539              :          INTEGER :: i
    4540              : #endif
    4541              : 
    4542           46 :          CALL mp_timeset(routineN, handle)
    4543              : 
    4544              : #if defined(__parallel)
    4545              :          ierr = 0
    4546              :          MARK_USED(type_descriptor)
    4547           46 :          CALL MPI_File_read_all(fh%handle, buffer, ndims*msglen, MPI_CHARACTER, MPI_STATUS_IGNORE, ierr)
    4548           46 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_file_set_errhandler @ MPI_File_read_all")
    4549           46 :          CALL add_perf(perf_id=28, count=1, msg_size=ndims*msglen)
    4550              : #else
    4551              :          MARK_USED(msglen)
    4552              :          MARK_USED(ndims)
    4553              :          IF (.NOT. PRESENT(type_descriptor)) &
    4554              :             CALL cp_abort(__LOCATION__, &
    4555              :                           "Container for mp_file_descriptor_type must be present in serial call.")
    4556              :          IF (.NOT. type_descriptor%has_indexing) &
    4557              :             CALL cp_abort(__LOCATION__, &
    4558              :                           "File view has not been set in mp_file_descriptor_type.")
    4559              :          ! Use explicit offsets
    4560              :          DO i = 1, ndims
    4561              :             READ (fh%handle, POS=type_descriptor%index_descriptor%chunks(i)) buffer(i)
    4562              :          END DO
    4563              : #endif
    4564              : 
    4565           46 :          CALL mp_timestop(handle)
    4566              : 
    4567           46 :       END SUBROUTINE mp_file_read_all_chv
    4568              : 
    4569              : ! **************************************************************************************************
    4570              : !> \brief (parallel) Collective, blocking write of a character array to a file. File access pattern
    4571              : !                    determined by a previously set file view.
    4572              : !>        (serial)   Unformatted stream write using explicit offsets
    4573              : !> \param fh     the file handle associated with the output file
    4574              : !> \param msglen the message length of an individual vector component
    4575              : !> \param ndims  the number of vector components
    4576              : !> \param buffer the buffer where the data is placed
    4577              : !> \param type_descriptor container for the MPI type
    4578              : !> \author Nico Holmberg [05.2017]
    4579              : ! **************************************************************************************************
    4580         2032 :       SUBROUTINE mp_file_write_all_chv(fh, msglen, ndims, buffer, type_descriptor)
    4581              :          CLASS(mp_file_type), INTENT(IN)                      :: fh
    4582              :          INTEGER, INTENT(IN)                                  :: msglen
    4583              :          INTEGER, INTENT(IN)                                  :: ndims
    4584              :          CHARACTER(LEN=msglen), DIMENSION(ndims), INTENT(IN)  :: buffer
    4585              :          TYPE(mp_file_descriptor_type), &
    4586              :             INTENT(IN), OPTIONAL                              :: type_descriptor
    4587              : 
    4588              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_file_write_all_chv'
    4589              : 
    4590              :          INTEGER :: handle
    4591              : #if defined(__parallel)
    4592              :          INTEGER :: ierr
    4593              : #else
    4594              :          INTEGER :: i
    4595              : #endif
    4596              : 
    4597         2032 :          CALL mp_timeset(routineN, handle)
    4598              : 
    4599              : #if defined(__parallel)
    4600              :          MARK_USED(type_descriptor)
    4601         2032 :          CALL mpi_file_set_errhandler(fh%handle, MPI_ERRORS_RETURN, ierr)
    4602         2032 :          CALL MPI_File_write_all(fh%handle, buffer, ndims*msglen, MPI_CHARACTER, MPI_STATUS_IGNORE, ierr)
    4603         2032 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_file_set_errhandler @ MPI_File_write_all")
    4604         2032 :          CALL add_perf(perf_id=28, count=1, msg_size=ndims*msglen)
    4605              : #else
    4606              :          MARK_USED(msglen)
    4607              :          MARK_USED(ndims)
    4608              :          IF (.NOT. PRESENT(type_descriptor)) &
    4609              :             CALL cp_abort(__LOCATION__, &
    4610              :                           "Container for mp_file_descriptor_type must be present in serial call.")
    4611              :          IF (.NOT. type_descriptor%has_indexing) &
    4612              :             CALL cp_abort(__LOCATION__, &
    4613              :                           "File view has not been set in mp_file_descriptor_type.")
    4614              :          ! Use explicit offsets
    4615              :          DO i = 1, ndims
    4616              :             WRITE (fh%handle, POS=type_descriptor%index_descriptor%chunks(i)) buffer(i)
    4617              :          END DO
    4618              : #endif
    4619              : 
    4620         2032 :          CALL mp_timestop(handle)
    4621              : 
    4622         2032 :       END SUBROUTINE mp_file_write_all_chv
    4623              : 
    4624              : ! **************************************************************************************************
    4625              : !> \brief Releases the type used for MPI I/O
    4626              : !> \param type_descriptor the container for the MPI type
    4627              : !> \author Nico Holmberg [05.2017]
    4628              : ! **************************************************************************************************
    4629         4156 :       SUBROUTINE mp_file_type_free(type_descriptor)
    4630              :          TYPE(mp_file_descriptor_type)             :: type_descriptor
    4631              : 
    4632              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_file_type_free'
    4633              : 
    4634              :          INTEGER                                   :: handle
    4635              : #if defined(__parallel)
    4636              :          INTEGER :: ierr
    4637              : #endif
    4638              : 
    4639         2078 :          CALL mp_timeset(routineN, handle)
    4640              : 
    4641              : #if defined(__parallel)
    4642         2078 :          CALL MPI_Type_free(type_descriptor%type_handle, ierr)
    4643         2078 :          IF (ierr /= 0) &
    4644            0 :             CPABORT("MPI_Type_free @ "//routineN)
    4645              : #endif
    4646              : #if defined(__parallel) && defined(__MPI_F08)
    4647         2078 :          type_descriptor%type_handle%mpi_val = -1
    4648              : #else
    4649              :          type_descriptor%type_handle = -1
    4650              : #endif
    4651         2078 :          type_descriptor%length = -1
    4652         2078 :          IF (type_descriptor%has_indexing) THEN
    4653         2078 :             NULLIFY (type_descriptor%index_descriptor%index)
    4654         2078 :             NULLIFY (type_descriptor%index_descriptor%chunks)
    4655         2078 :             type_descriptor%has_indexing = .FALSE.
    4656              :          END IF
    4657              : 
    4658         2078 :          CALL mp_timestop(handle)
    4659              : 
    4660         2078 :       END SUBROUTINE mp_file_type_free
    4661              : 
    4662              : ! **************************************************************************************************
    4663              : !> \brief (parallel) Utility routine to determine MPI file access mode based on variables
    4664              : !                    that in the serial case would get passed to the intrinsic OPEN
    4665              : !>        (serial)   No action
    4666              : !> \param mpi_io     flag that determines if MPI I/O will actually be used
    4667              : !> \param replace    flag that indicates whether file needs to be deleted prior to opening it
    4668              : !> \param amode      the MPI I/O access mode
    4669              : !> \param form       formatted or unformatted data?
    4670              : !> \param action     the variable that determines what to do with file
    4671              : !> \param status     the status flag:
    4672              : !> \param position   should the file be appended or rewound
    4673              : !> \author Nico Holmberg [11.2017]
    4674              : ! **************************************************************************************************
    4675         2032 :       SUBROUTINE mp_file_get_amode(mpi_io, replace, amode, form, action, status, position)
    4676              :          LOGICAL, INTENT(INOUT)                             :: mpi_io, replace
    4677              :          INTEGER, INTENT(OUT)                               :: amode
    4678              :          CHARACTER(len=*), INTENT(IN)                       :: form, action, status, position
    4679              : 
    4680         2032 :          amode = -1
    4681              : #if defined(__parallel)
    4682              :          ! Disable mpi io for unformatted access
    4683            0 :          SELECT CASE (form)
    4684              :          CASE ("FORMATTED")
    4685              :             ! Do nothing
    4686              :          CASE ("UNFORMATTED")
    4687            0 :             mpi_io = .FALSE.
    4688              :          CASE DEFAULT
    4689         2032 :             CPABORT("Unknown MPI file form requested.")
    4690              :          END SELECT
    4691              :          ! Determine file access mode (limited set of allowed choices)
    4692         2032 :          SELECT CASE (action)
    4693              :          CASE ("WRITE")
    4694         2032 :             amode = file_amode_wronly
    4695            0 :             SELECT CASE (status)
    4696              :             CASE ("NEW")
    4697              :                ! Try to open new file for writing, crash if file already exists
    4698            0 :                amode = amode + file_amode_create + file_amode_excl
    4699              :             CASE ("UNKNOWN")
    4700              :                ! Open file for writing and create it if file does not exist
    4701         1708 :                amode = amode + file_amode_create
    4702           76 :                SELECT CASE (position)
    4703              :                CASE ("APPEND")
    4704              :                   ! Append existing file
    4705           76 :                   amode = amode + file_amode_append
    4706              :                CASE ("REWIND", "ASIS")
    4707              :                   ! Do nothing
    4708              :                CASE DEFAULT
    4709         1708 :                   CPABORT("Unknown MPI file position requested.")
    4710              :                END SELECT
    4711              :             CASE ("OLD")
    4712          324 :                SELECT CASE (position)
    4713              :                CASE ("APPEND")
    4714              :                   ! Append existing file
    4715            0 :                   amode = amode + file_amode_append
    4716              :                CASE ("REWIND", "ASIS")
    4717              :                   ! Do nothing
    4718              :                CASE DEFAULT
    4719            0 :                   CPABORT("Unknown MPI file position requested.")
    4720              :                END SELECT
    4721              :             CASE ("REPLACE")
    4722              :                ! Overwrite existing file. Must delete existing file first
    4723          324 :                amode = amode + file_amode_create
    4724          324 :                replace = .TRUE.
    4725              :             CASE ("SCRATCH")
    4726              :                ! Disable
    4727            0 :                mpi_io = .FALSE.
    4728              :             CASE DEFAULT
    4729         2032 :                CPABORT("Unknown MPI file status requested.")
    4730              :             END SELECT
    4731              :          CASE ("READ")
    4732            0 :             amode = file_amode_rdonly
    4733            0 :             SELECT CASE (status)
    4734              :             CASE ("NEW")
    4735            0 :                CPABORT("Cannot read from 'NEW' file.")
    4736              :             CASE ("REPLACE")
    4737            0 :                CPABORT("Illegal status 'REPLACE' for read.")
    4738              :             CASE ("UNKNOWN", "OLD")
    4739              :                ! Do nothing
    4740              :             CASE ("SCRATCH")
    4741              :                ! Disable
    4742            0 :                mpi_io = .FALSE.
    4743              :             CASE DEFAULT
    4744            0 :                CPABORT("Unknown MPI file status requested.")
    4745              :             END SELECT
    4746              :          CASE ("READWRITE")
    4747            0 :             amode = file_amode_rdwr
    4748            0 :             SELECT CASE (status)
    4749              :             CASE ("NEW")
    4750              :                ! Try to open new file, crash if file already exists
    4751            0 :                amode = amode + file_amode_create + file_amode_excl
    4752              :             CASE ("UNKNOWN")
    4753              :                ! Open file and create it if file does not exist
    4754            0 :                amode = amode + file_amode_create
    4755            0 :                SELECT CASE (position)
    4756              :                CASE ("APPEND")
    4757              :                   ! Append existing file
    4758            0 :                   amode = amode + file_amode_append
    4759              :                CASE ("REWIND", "ASIS")
    4760              :                   ! Do nothing
    4761              :                CASE DEFAULT
    4762            0 :                   CPABORT("Unknown MPI file position requested.")
    4763              :                END SELECT
    4764              :             CASE ("OLD")
    4765            0 :                SELECT CASE (position)
    4766              :                CASE ("APPEND")
    4767              :                   ! Append existing file
    4768            0 :                   amode = amode + file_amode_append
    4769              :                CASE ("REWIND", "ASIS")
    4770              :                   ! Do nothing
    4771              :                CASE DEFAULT
    4772            0 :                   CPABORT("Unknown MPI file position requested.")
    4773              :                END SELECT
    4774              :             CASE ("REPLACE")
    4775              :                ! Overwrite existing file. Must delete existing file first
    4776            0 :                amode = amode + file_amode_create
    4777            0 :                replace = .TRUE.
    4778              :             CASE ("SCRATCH")
    4779              :                ! Disable
    4780            0 :                mpi_io = .FALSE.
    4781              :             CASE DEFAULT
    4782            0 :                CPABORT("Unknown MPI file status requested.")
    4783              :             END SELECT
    4784              :          CASE DEFAULT
    4785         2032 :             CPABORT("Unknown MPI file action requested.")
    4786              :          END SELECT
    4787              : #else
    4788              :          MARK_USED(replace)
    4789              :          MARK_USED(form)
    4790              :          MARK_USED(position)
    4791              :          MARK_USED(status)
    4792              :          MARK_USED(action)
    4793              :          mpi_io = .FALSE.
    4794              : #endif
    4795              : 
    4796         2032 :       END SUBROUTINE mp_file_get_amode
    4797              : 
    4798              : ! **************************************************************************************************
    4799              : !> \brief Non-blocking send of custom type
    4800              : !> \param msgin ...
    4801              : !> \param dest ...
    4802              : !> \param comm ...
    4803              : !> \param request ...
    4804              : !> \param tag ...
    4805              : ! **************************************************************************************************
    4806            0 :       SUBROUTINE mp_isend_custom(msgin, dest, comm, request, tag)
    4807              :          TYPE(mp_type_descriptor_type), INTENT(IN)          :: msgin
    4808              :          INTEGER, INTENT(IN)                                :: dest
    4809              :          CLASS(mp_comm_type), INTENT(IN) :: comm
    4810              :          TYPE(mp_request_type), INTENT(out)                               :: request
    4811              :          INTEGER, INTENT(in), OPTIONAL                      :: tag
    4812              : 
    4813              :          INTEGER                                            :: ierr, my_tag
    4814              : 
    4815              :          ierr = 0
    4816            0 :          my_tag = 0
    4817              : 
    4818              : #if defined(__parallel)
    4819            0 :          IF (PRESENT(tag)) my_tag = tag
    4820              : 
    4821              :          CALL mpi_isend(MPI_BOTTOM, 1, msgin%type_handle, dest, my_tag, &
    4822            0 :                         comm%handle, request%handle, ierr)
    4823            0 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_isend @ mp_isend_custom")
    4824              : #else
    4825              :          MARK_USED(msgin)
    4826              :          MARK_USED(dest)
    4827              :          MARK_USED(comm)
    4828              :          MARK_USED(tag)
    4829              :          ierr = 1
    4830              :          request = mp_request_null
    4831              :          CALL mp_stop(ierr, "mp_isend called in non parallel case")
    4832              : #endif
    4833            0 :       END SUBROUTINE mp_isend_custom
    4834              : 
    4835              : ! **************************************************************************************************
    4836              : !> \brief Non-blocking receive of vector data
    4837              : !> \param msgout ...
    4838              : !> \param source ...
    4839              : !> \param comm ...
    4840              : !> \param request ...
    4841              : !> \param tag ...
    4842              : ! **************************************************************************************************
    4843            0 :       SUBROUTINE mp_irecv_custom(msgout, source, comm, request, tag)
    4844              :          TYPE(mp_type_descriptor_type), INTENT(INOUT)       :: msgout
    4845              :          INTEGER, INTENT(IN)                                :: source
    4846              :          CLASS(mp_comm_type), INTENT(IN) :: comm
    4847              :          TYPE(mp_request_type), INTENT(out)                               :: request
    4848              :          INTEGER, INTENT(in), OPTIONAL                      :: tag
    4849              : 
    4850              :          INTEGER                                            :: ierr, my_tag
    4851              : 
    4852              :          ierr = 0
    4853            0 :          my_tag = 0
    4854              : 
    4855              : #if defined(__parallel)
    4856            0 :          IF (PRESENT(tag)) my_tag = tag
    4857              : 
    4858              :          CALL mpi_irecv(MPI_BOTTOM, 1, msgout%type_handle, source, my_tag, &
    4859            0 :                         comm%handle, request%handle, ierr)
    4860            0 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_irecv @ mp_irecv_custom")
    4861              : #else
    4862              :          MARK_USED(msgout)
    4863              :          MARK_USED(source)
    4864              :          MARK_USED(comm)
    4865              :          MARK_USED(tag)
    4866              :          ierr = 1
    4867              :          request = mp_request_null
    4868              :          CPABORT("mp_irecv called in non parallel case")
    4869              : #endif
    4870            0 :       END SUBROUTINE mp_irecv_custom
    4871              : 
    4872              : ! **************************************************************************************************
    4873              : !> \brief Window free
    4874              : !> \param win ...
    4875              : ! **************************************************************************************************
    4876            0 :       SUBROUTINE mp_win_free(win)
    4877              :          CLASS(mp_win_type), INTENT(INOUT)                  :: win
    4878              : 
    4879              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_win_free'
    4880              : 
    4881              :          INTEGER                                            :: handle
    4882              : #if defined(__parallel)
    4883              :          INTEGER :: ierr
    4884              : #endif
    4885              : 
    4886            0 :          CALL mp_timeset(routineN, handle)
    4887              : 
    4888              : #if defined(__parallel)
    4889              :          ierr = 0
    4890            0 :          CALL mpi_win_free(win%handle, ierr)
    4891            0 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_win_free @ "//routineN)
    4892              : 
    4893            0 :          CALL add_perf(perf_id=21, count=1)
    4894              : #else
    4895              :          win%handle = mp_win_null_handle
    4896              : #endif
    4897            0 :          CALL mp_timestop(handle)
    4898            0 :       END SUBROUTINE mp_win_free
    4899              : 
    4900            0 :       SUBROUTINE mp_win_assign(win_new, win_old)
    4901              :          CLASS(mp_win_type), INTENT(OUT) :: win_new
    4902              :          CLASS(mp_win_type), INTENT(IN) :: win_old
    4903              : 
    4904            0 :          win_new%handle = win_old%handle
    4905              : 
    4906            0 :       END SUBROUTINE mp_win_assign
    4907              : 
    4908              : ! **************************************************************************************************
    4909              : !> \brief Window flush
    4910              : !> \param win ...
    4911              : ! **************************************************************************************************
    4912            0 :       SUBROUTINE mp_win_flush_all(win)
    4913              :          CLASS(mp_win_type), INTENT(IN)                     :: win
    4914              : 
    4915              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_win_flush_all'
    4916              : 
    4917              :          INTEGER                                            :: handle, ierr
    4918              : 
    4919              :          ierr = 0
    4920            0 :          CALL mp_timeset(routineN, handle)
    4921              : 
    4922              : #if defined(__parallel)
    4923            0 :          CALL mpi_win_flush_all(win%handle, ierr)
    4924            0 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_win_flush_all @ "//routineN)
    4925              : #else
    4926              :          MARK_USED(win)
    4927              : #endif
    4928            0 :          CALL mp_timestop(handle)
    4929            0 :       END SUBROUTINE mp_win_flush_all
    4930              : 
    4931              : ! **************************************************************************************************
    4932              : !> \brief Window lock
    4933              : !> \param win ...
    4934              : ! **************************************************************************************************
    4935            0 :       SUBROUTINE mp_win_lock_all(win)
    4936              :          CLASS(mp_win_type), INTENT(IN)                     :: win
    4937              : 
    4938              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_win_lock_all'
    4939              : 
    4940              :          INTEGER                                            :: handle, ierr
    4941              : 
    4942              :          ierr = 0
    4943            0 :          CALL mp_timeset(routineN, handle)
    4944              : 
    4945              : #if defined(__parallel)
    4946              : 
    4947            0 :          CALL mpi_win_lock_all(MPI_MODE_NOCHECK, win%handle, ierr)
    4948            0 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_win_lock_all @ "//routineN)
    4949              : 
    4950            0 :          CALL add_perf(perf_id=19, count=1)
    4951              : #else
    4952              :          MARK_USED(win)
    4953              : #endif
    4954            0 :          CALL mp_timestop(handle)
    4955            0 :       END SUBROUTINE mp_win_lock_all
    4956              : 
    4957              : ! **************************************************************************************************
    4958              : !> \brief Window lock
    4959              : !> \param win ...
    4960              : ! **************************************************************************************************
    4961            0 :       SUBROUTINE mp_win_unlock_all(win)
    4962              :          CLASS(mp_win_type), INTENT(IN)                     :: win
    4963              : 
    4964              :          CHARACTER(len=*), PARAMETER :: routineN = 'mp_win_unlock_all'
    4965              : 
    4966              :          INTEGER                                            :: handle, ierr
    4967              : 
    4968              :          ierr = 0
    4969            0 :          CALL mp_timeset(routineN, handle)
    4970              : 
    4971              : #if defined(__parallel)
    4972              : 
    4973            0 :          CALL mpi_win_unlock_all(win%handle, ierr)
    4974            0 :          IF (ierr /= 0) CALL mp_stop(ierr, "mpi_win_unlock_all @ "//routineN)
    4975              : 
    4976            0 :          CALL add_perf(perf_id=19, count=1)
    4977              : #else
    4978              :          MARK_USED(win)
    4979              : #endif
    4980            0 :          CALL mp_timestop(handle)
    4981            0 :       END SUBROUTINE mp_win_unlock_all
    4982              : 
    4983              : ! **************************************************************************************************
    4984              : !> \brief Starts a timer region
    4985              : !> \param routineN ...
    4986              : !> \param handle ...
    4987              : ! **************************************************************************************************
    4988    373282160 :       SUBROUTINE mp_timeset(routineN, handle)
    4989              :          CHARACTER(len=*), INTENT(IN)                       :: routineN
    4990              :          INTEGER, INTENT(OUT)                               :: handle
    4991              : 
    4992    186684814 :          handle = -1
    4993    186684814 :          IF (.NOT. mp_collect_timings) RETURN
    4994    186597602 :          IF (omp_get_thread_num() /= 0) RETURN
    4995    186597346 :          IF (omp_get_level() > 1) RETURN
    4996    186597346 :          CALL timeset(routineN, handle)
    4997              :       END SUBROUTINE mp_timeset
    4998              : 
    4999              : ! **************************************************************************************************
    5000              : !> \brief Ends a timer region
    5001              : !> \param handle ...
    5002              : ! **************************************************************************************************
    5003    186684814 :       SUBROUTINE mp_timestop(handle)
    5004              :          INTEGER, INTENT(IN)                                :: handle
    5005              : 
    5006    186684814 :          IF (handle >= 0) &
    5007    186597346 :             CALL timestop(handle)
    5008    186684814 :       END SUBROUTINE mp_timestop
    5009              : 
    5010              :       #:include 'message_passing.fypp'
    5011              : 
    5012    122129432 :    END MODULE message_passing
        

Generated by: LCOV version 2.0-1