Skip to content

Commit 6d85438

Browse files
aradiaradi
committed
Fix MPI testing issues
* Execute tests using mpiexec * Fix double MPI-finalization * Add helper routines for convenient BLACS testing
1 parent 3d38c45 commit 6d85438

File tree

4 files changed

+217
-30
lines changed

4 files changed

+217
-30
lines changed

‎CMakeLists.txt‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,27 @@ option(
5656
OFF
5757
)
5858

59+
set(
60+
SCALAPACKFX_TEST_OMP_THREADS
61+
"1"
62+
CACHE STRING
63+
"Nr. of OpenMP-threads used for testing"
64+
)
65+
66+
set(SCALAPACKFX_TEST_MPI_PROCS
67+
"1"
68+
CACHE STRING
69+
"Nr. of MPI processes used for testing"
70+
)
71+
72+
set(
73+
SCALAPACKFX_MPIEXEC_POSTFLAGS
74+
""
75+
CACHE STRING
76+
"Additional flags for the mpiexec command (to be added after the mpiexec command name)"
77+
)
78+
79+
5980
#[=============================================================================[
6081
# Project configuration #
6182
]=============================================================================]
@@ -74,6 +95,13 @@ scalapackfx_setup_build_type("RelWithDebInfo")
7495

7596
find_package(MPI REQUIRED)
7697

98+
set(
99+
MPI_TEST_RUNNER
100+
${MPIEXEC_PREFLAGS} ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${SCALAPACKFX_TEST_MPI_PROCS}
101+
${MPIEXEC_POSTFLAGS} ${SCALAPACKFX_MPIEXEC_POSTFLAGS}
102+
)
103+
104+
77105
#
78106
# Prerequisites
79107
#

‎test/CMakeLists.txt‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
# Folder for generated mod-files
22
set(moduledir "${CMAKE_CURRENT_BINARY_DIR}/modules")
33

4+
add_library(blacstestutils blacstestutils.f90)
5+
target_link_libraries(blacstestutils PRIVATE scalapackfx)
6+
target_link_libraries(blacstestutils PRIVATE Fortuno::fortuno_mpi)
7+
48
list(APPEND tests
59
test_desc
610
# test_cpg2l
@@ -16,18 +20,23 @@ list(APPEND tests
1620
# test_tran
1721
)
1822

23+
1924
foreach(test IN LISTS tests)
2025
add_executable(${test})
2126
target_sources(${test} PRIVATE ${test}.f90)
2227
set_target_properties(
2328
${test} PROPERTIES Fortran_MODULE_DIRECTORY "${moduledir}"
2429
)
30+
target_link_libraries(${test} PRIVATE blacstestutils)
2531
target_link_libraries(${test} PRIVATE scalapackfx)
2632
target_link_libraries(${test} PRIVATE Fortuno::fortuno_mpi MPI::MPI_Fortran)
2733
target_link_libraries(${test} PRIVATE Scalapack::Scalapack)
28-
2934
add_test(
3035
NAME ${test}
31-
COMMAND ${test}
36+
COMMAND ${MPI_TEST_RUNNER} ./${test}
37+
)
38+
set_tests_properties(
39+
${test}
40+
PROPERTIES ENVIRONMENT OMP_NUM_THREADS=${SCALAPACKFX_TEST_OMP_THREADS}
3241
)
3342
endforeach()

‎test/blacstestutils.f90‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
module blacstestutils
2+
use libscalapackfx_module, only : blacsgrid, blacsfx_exit, blacsfx_pinfo
3+
use fortuno_mpi, only : test_item, mpi_case, check => mpi_check, failed => mpi_failed, num_ranks
4+
implicit none
5+
6+
private
7+
public :: this_proc, num_procs
8+
public :: blacs_test
9+
public :: blacs_grid_env, get_grid_or_fail
10+
11+
12+
!> Implements a test class with BLACS initialization and destruction
13+
type, extends(mpi_case) :: blacs_case
14+
contains
15+
procedure :: run => blacs_case_run
16+
end type blacs_case
17+
18+
19+
abstract interface
20+
!> Interface of the test procedure
21+
subroutine blacs_test_procedure()
22+
end subroutine blacs_test_procedure
23+
end interface
24+
25+
26+
!> Implements a BLACS grid wrapper enforcing grid finalization
27+
type, extends(blacsgrid) :: blacs_grid_env
28+
29+
!> Whether the grid contains all BLACS processes
30+
logical :: has_all_procs = .false.
31+
contains
32+
final :: final_blacs_grid_env
33+
end type blacs_grid_env
34+
35+
36+
! Number of processes available in the BLACS framework
37+
integer :: num_procs_ = -1
38+
39+
! The id of the current process in the BLACS framework
40+
integer :: this_proc_ = -1
41+
42+
contains
43+
44+
45+
!> Returns the id of the current process in the BLACS framework
46+
function this_proc()
47+
integer :: this_proc
48+
this_proc = this_proc_
49+
end function this_proc
50+
51+
52+
!> Returns the number for processes available in the BLACS framework
53+
function num_procs()
54+
integer :: num_procs
55+
num_procs = num_procs_
56+
end function num_procs
57+
58+
59+
!> Wraps a blacs_case instance as test_item suitable for array constructors.
60+
function blacs_test(name, proc) result(testitem)
61+
character(*), intent(in) :: name
62+
procedure(blacs_test_procedure) :: proc
63+
64+
type(test_item) :: testitem
65+
66+
testitem = test_item(blacs_case(name=name, proc=proc))
67+
68+
end function blacs_test
69+
70+
71+
!> Run procedure of the tempfile_case type.
72+
subroutine blacs_case_run(this)
73+
class(blacs_case), intent(in) :: this
74+
75+
call blacsfx_pinfo(this_proc_, num_procs_)
76+
call check(num_procs_ == num_ranks(),&
77+
& "Number of BLACS processes differ from number of MPI ranks")
78+
if (failed()) return
79+
call this%proc()
80+
call blacsfx_exit(keepmpi=.true.)
81+
this_proc_ = -1
82+
num_procs_ = -1
83+
84+
end subroutine blacs_case_run
85+
86+
87+
!> Returns a grid environment or sets the calling test to failed if not possible
88+
!!
89+
!! Note: This routine must be called from within fortuno MPI test procedures.
90+
!!
91+
subroutine get_grid_or_fail(this, nrow, ncol, includeall)
92+
93+
!> Instance
94+
type(blacs_grid_env), intent(out) :: this
95+
96+
!> Number of process rows
97+
integer, optional, intent(in) :: nrow
98+
99+
!> Number of process columns
100+
integer, optional, intent(in) :: ncol
101+
102+
!> Whether it should be ensured that all processes are included in the grid (default: .true.)
103+
logical, optional, intent(in) :: includeall
104+
105+
integer :: nprocs
106+
integer :: nrow_, ncol_
107+
logical :: includeall_, hasall
108+
type(blacsgrid) :: grid
109+
110+
includeall_ = .true.
111+
if (present(includeall)) includeall_ = includeall
112+
113+
nprocs = num_procs()
114+
if (present(nrow) .and. present(ncol)) then
115+
nrow_ = nrow
116+
ncol_ = ncol
117+
else if (present(nrow)) then
118+
nrow_ = nrow
119+
ncol_ = nprocs / nrow_
120+
else if (present(ncol)) then
121+
ncol_ = ncol
122+
nrow_ = nprocs / ncol_
123+
else if (includeall_) then
124+
do nrow_ = floor(sqrt(real(nprocs))), 1, -1
125+
ncol_ = nprocs / nrow_
126+
if (ncol_ * nrow_ == nprocs) exit
127+
end do
128+
else
129+
nrow_ = floor(sqrt(real(nprocs)))
130+
ncol_ = nprocs / nrow_
131+
end if
132+
hasall = ncol_ * nrow_ == nprocs
133+
134+
call check(nrow_ * ncol_ <= nprocs, msg="Required grid needs more processes than available")
135+
if (failed()) return
136+
call check(nrow_ > 0, msg="Could not set up grid with at least one process row")
137+
if (failed()) return
138+
call check(ncol_ > 0, msg="Could not set up grid with at least one process column")
139+
if (failed()) return
140+
call check(.not. includeall_ .or. hasall,&
141+
& msg="Could not include all processes in the required grid")
142+
if (failed()) return
143+
144+
call this%blacsgrid%initgrid(nrow_, ncol_)
145+
this%has_all_procs = hasall
146+
147+
end subroutine get_grid_or_fail
148+
149+
150+
subroutine final_blacs_grid_env(this)
151+
type(blacs_grid_env), intent(inout) :: this
152+
153+
call this%blacsgrid%destruct()
154+
155+
end subroutine final_blacs_grid_env
156+
157+
end module blacstestutils

‎test/test_desc.f90‎

Lines changed: 21 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,14 @@
11
!> Test app driving Fortuno unit tests.
22
module test_scalapackfx
33
use libscalapackfx_module
4-
use fortuno_mpi, only : global_comm, is_equal, test => mpi_case_item, check => mpi_check,&
5-
& test_list, this_rank
4+
use fortuno_mpi, only : check => mpi_check, failed => mpi_failed, skip => mpi_skip, test_list
5+
use blacstestutils, only : blacs_grid_env, get_grid_or_fail, test => blacs_test,&
6+
& num_procs
67
implicit none
78

89
contains
910

11+
1012
function tests()
1113
type(test_list) :: tests
1214

@@ -18,42 +20,33 @@ end function tests
1820

1921

2022
subroutine test_desc()
21-
type(blacsgrid) :: myGrid
22-
type(blocklist) :: block_List
23+
type(blacs_grid_env) :: env
24+
type(blocklist) :: blist
2325
integer :: desc(DLEN_)
24-
integer :: ii, iGlob, iLoc, nElem
25-
integer :: iProc, nProc, nGrid
26-
27-
call blacsfx_pinfo(iProc, nProc)
28-
29-
nGrid = floor(sqrt(real(nProc)))
30-
31-
call myGrid%initgrid(2, 2)
32-
33-
call check(.false.)
26+
integer :: ii, iglob, iloc, nelem
3427

35-
if (myGrid%mycol == -1) then
28+
if (num_procs() /= 4) then
29+
call skip()
30+
return
31+
end if
3632

37-
print *, "IDLE Node:", iProc, " Exiting..."
33+
call get_grid_or_fail(env, nrow=2, ncol=2)
34+
if (failed()) return
3835

36+
if (env%mycol == -1) then
37+
print *, "IDLE Node:", env%iproc, " Exiting..."
3938
else
40-
41-
call scalafx_getdescriptor(myGrid, 5, 5, 2, 2, desc)
42-
call block_List%init(myGrid, desc, "c")
43-
do ii = 1, size(block_List)
44-
call block_List%getblock(ii, iGlob, iLoc, nElem)
45-
print "(6(A,I3,2X))", "PROW:", myGrid%myrow, "PCOL:", myGrid%mycol,&
46-
& "II:", ii, "GLOB:", iGlob, "LOC:", iLoc, "NEL:", nElem
39+
call scalafx_getdescriptor(env%blacsgrid, 5, 5, 2, 2, desc)
40+
call blist%init(env%blacsgrid, desc, "c")
41+
do ii = 1, size(blist)
42+
call blist%getblock(ii, iglob, iloc, nelem)
43+
print "(6(A,I3,2X))", "PROW:", env%myrow, "PCOL:", env%mycol,&
44+
& "II:", ii, "GLOB:", iglob, "LOC:", iloc, "NEL:", nelem
4745
end do
48-
4946
end if
5047

51-
! THEN each rank must contain source rank's value
52-
!call check(is_equal(buffer, sourceval))
5348
call check(.false.)
5449

55-
call blacsfx_exit()
56-
5750
end subroutine test_desc
5851

5952
end module test_scalapackfx

0 commit comments

Comments
 (0)