Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Autoset nprocs and improve max_blocks estimate #949

Merged
merged 8 commits into from
May 10, 2024
Merged

Autoset nprocs and improve max_blocks estimate #949

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
7ac53a9
update max_blocks in ice_domain.F90, enabling max_blocks set on the fly
minghangli-uni Apr 22, 2024
eae27be
Make nprocs optional
anton-seaice May 2, 2024
f8fbdec
only log on master task
anton-seaice May 2, 2024
87640f7
change .eq. to == syntacx for consistency
anton-seaice May 3, 2024
27e1c05
Add CESMCOUPLED back in
anton-seaice May 3, 2024
8e2df00
Add note to docs about set number of procs automatically
anton-seaice May 3, 2024
56304b7
Review feedback
anton-seaice May 6, 2024
14ac9b9
Update ug_case_settings.rst
anton-seaice May 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 27 additions & 21 deletions cicecore/cicedyn/infrastructure/ice_domain.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ subroutine init_domain_blocks
! This routine reads in domain information and calls the routine
! to set up the block decomposition.

use ice_distribution, only: processor_shape
use ice_distribution, only: processor_shape, proc_decomposition
use ice_domain_size, only: ncat, nilyr, nslyr, max_blocks, &
nx_global, ny_global, block_size_x, block_size_y
use ice_fileunits, only: goto_nml
Expand All @@ -112,7 +112,8 @@ subroutine init_domain_blocks
!----------------------------------------------------------------------

integer (int_kind) :: &
nml_error ! namelist read error flag
nml_error, & ! namelist read error flag
nprocs_x, nprocs_y ! procs decomposed into blocks

character(len=char_len) :: nml_name ! text namelist name
character(len=char_len_long) :: tmpstr2 ! for namelist check
Expand Down Expand Up @@ -216,21 +217,36 @@ subroutine init_domain_blocks
call broadcast_scalar(maskhalo_bound, master_task)
call broadcast_scalar(add_mpi_barriers, master_task)
call broadcast_scalar(debug_blocks, master_task)
if (my_task == master_task) then
if (max_blocks < 1) then
max_blocks=( ((nx_global-1)/block_size_x + 1) * &
((ny_global-1)/block_size_y + 1) - 1) / nprocs + 1
max_blocks=max(1,max_blocks)
write(nu_diag,'(/,a52,i6,/)') &
'(ice_domain): max_block < 1: max_block estimated to ',max_blocks
endif
endif
call broadcast_scalar(max_blocks, master_task)
call broadcast_scalar(block_size_x, master_task)
call broadcast_scalar(block_size_y, master_task)
call broadcast_scalar(nx_global, master_task)
call broadcast_scalar(ny_global, master_task)

! Set nprocs if not set in namelist
apcraig marked this conversation as resolved.
Show resolved Hide resolved
#ifdef CESMCOUPLED
nprocs = get_num_procs()
#else
if (nprocs < 0) then
nprocs = get_num_procs()
else if (nprocs /= get_num_procs()) then
write(nu_diag,*) subname,' ERROR: nprocs, get_num_procs = ',nprocs,get_num_procs()
call abort_ice(subname//' ERROR: Input nprocs not same as system (e.g MPI) request', file=__FILE__, line=__LINE__)
endif
#endif

! Determine max_blocks if not set
if (max_blocks < 1) then
call proc_decomposition(nprocs, nprocs_x, nprocs_y)
max_blocks=((nx_global-1)/block_size_x/nprocs_x+1) * &
((ny_global-1)/block_size_y/nprocs_y+1)
max_blocks=max(1,max_blocks)
if (my_task == master_task) then
write(nu_diag,'(/,a52,i6,/)') &
'(ice_domain): max_block < 1: max_block estimated to ',max_blocks
endif
endif

!----------------------------------------------------------------------
!
! perform some basic checks on domain
Expand All @@ -242,16 +258,6 @@ subroutine init_domain_blocks
!*** domain size zero or negative
!***
call abort_ice(subname//' ERROR: Invalid domain: size < 1', file=__FILE__, line=__LINE__) ! no domain
else if (nprocs /= get_num_procs()) then
!***
!*** input nprocs does not match system (eg MPI) request
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we could keep this comment, I think

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Isn't this clear from the ERROR message?

!***
#if (defined CESMCOUPLED)
nprocs = get_num_procs()
#else
write(nu_diag,*) subname,' ERROR: nprocs, get_num_procs = ',nprocs,get_num_procs()
call abort_ice(subname//' ERROR: Input nprocs not same as system request', file=__FILE__, line=__LINE__)
#endif
else if (nghost < 1) then
!***
!*** must have at least 1 layer of ghost cells
Expand Down
3 changes: 2 additions & 1 deletion cicecore/shared/ice_distribution.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,8 @@ module ice_distribution
ice_distributionGet, &
ice_distributionGetBlockLoc, &
ice_distributionGetBlockID, &
create_local_block_ids
create_local_block_ids, &
proc_decomposition

character (char_len), public :: &
processor_shape ! 'square-pop' (approx) POP default config
Expand Down
2 changes: 1 addition & 1 deletion configuration/scripts/ice_in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -302,7 +302,7 @@
/

&domain_nml
nprocs = 4
nprocs = -1
nx_global = 100
ny_global = 116
block_size_x = 25
Expand Down
3 changes: 2 additions & 1 deletion doc/source/user_guide/ug_case_settings.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -369,7 +369,8 @@ domain_nml
"``maskhalo_remap``", "logical", "mask unused halo cells for transport", "``.false.``"
"``maskhalo_bound``", "logical", "mask unused halo cells for boundary updates", "``.false.``"
"``max_blocks``", "integer", "maximum number of blocks per MPI task for memory allocation", "-1"
"``nprocs``", "integer", "number of processors to use", "-1"
"``nprocs``", "integer", "number of MPI tasks to use", "-1"
"", "``-1``", "find number of MPI tasks automatically", ""
"``ns_boundary_type``", "``cyclic``", "periodic boundary conditions in y-direction", "``open``"
"", "``open``", "Dirichlet boundary conditions in y", ""
"", "``tripole``", "U-fold tripole boundary conditions in y", ""
Expand Down
Loading