From 7ac53a9dc592ce4db219ab32a7a7b6196f03ada8 Mon Sep 17 00:00:00 2001 From: minghangli-uni Date: Mon, 22 Apr 2024 11:12:07 +1000 Subject: [PATCH 1/8] update max_blocks in ice_domain.F90, enabling max_blocks set on the fly Co-authored-by: anton-seaice --- .../cicedyn/infrastructure/ice_domain.F90 | 27 +++++++++++-------- cicecore/shared/ice_distribution.F90 | 3 ++- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/cicecore/cicedyn/infrastructure/ice_domain.F90 b/cicecore/cicedyn/infrastructure/ice_domain.F90 index 8b680f2d4..87d181a51 100644 --- a/cicecore/cicedyn/infrastructure/ice_domain.F90 +++ b/cicecore/cicedyn/infrastructure/ice_domain.F90 @@ -90,6 +90,7 @@ module ice_domain integer (int_kind) :: & nprocs ! num of processors + !*********************************************************************** contains @@ -101,7 +102,7 @@ subroutine init_domain_blocks ! This routine reads in domain information and calls the routine ! to set up the block decomposition. - use ice_distribution, only: processor_shape + use ice_distribution, only: processor_shape, proc_decomposition use ice_domain_size, only: ncat, nilyr, nslyr, max_blocks, & nx_global, ny_global, block_size_x, block_size_y use ice_fileunits, only: goto_nml @@ -112,7 +113,8 @@ subroutine init_domain_blocks !---------------------------------------------------------------------- integer (int_kind) :: & - nml_error ! namelist read error flag + nml_error, & ! namelist read error flag + nprocs_x, nprocs_y ! procs decomposed into blocks character(len=char_len) :: nml_name ! text namelist name character(len=char_len_long) :: tmpstr2 ! for namelist check @@ -216,21 +218,24 @@ subroutine init_domain_blocks call broadcast_scalar(maskhalo_bound, master_task) call broadcast_scalar(add_mpi_barriers, master_task) call broadcast_scalar(debug_blocks, master_task) - if (my_task == master_task) then - if (max_blocks < 1) then - max_blocks=( ((nx_global-1)/block_size_x + 1) * & - ((ny_global-1)/block_size_y + 1) - 1) / nprocs + 1 - max_blocks=max(1,max_blocks) - write(nu_diag,'(/,a52,i6,/)') & - '(ice_domain): max_block < 1: max_block estimated to ',max_blocks - endif - endif call broadcast_scalar(max_blocks, master_task) call broadcast_scalar(block_size_x, master_task) call broadcast_scalar(block_size_y, master_task) call broadcast_scalar(nx_global, master_task) call broadcast_scalar(ny_global, master_task) + ! Automatically determine max_blocks if not set. + if (my_task == master_task) then + if (max_blocks < 1) then + call proc_decomposition(nprocs, nprocs_x, nprocs_y) + max_blocks=((nx_global-1)/block_size_x/nprocs_x+1) * & + ((ny_global-1)/block_size_y/nprocs_y+1) + max_blocks=max(1,max_blocks) + write(nu_diag,'(/,a52,i6,/)') & + '(ice_domain): max_block < 1: max_block estimated to ',max_blocks + endif + endif + !---------------------------------------------------------------------- ! ! perform some basic checks on domain diff --git a/cicecore/shared/ice_distribution.F90 b/cicecore/shared/ice_distribution.F90 index 0f3f6c198..0ba62f4a2 100644 --- a/cicecore/shared/ice_distribution.F90 +++ b/cicecore/shared/ice_distribution.F90 @@ -41,7 +41,8 @@ module ice_distribution ice_distributionGet, & ice_distributionGetBlockLoc, & ice_distributionGetBlockID, & - create_local_block_ids + create_local_block_ids, & + proc_decomposition ! mli: add character (char_len), public :: & processor_shape ! 'square-pop' (approx) POP default config From eae27beadc88b05cb9c7299ae0aa9bcb56e6344c Mon Sep 17 00:00:00 2001 From: Anton Steketee Date: Thu, 2 May 2024 15:25:08 +1000 Subject: [PATCH 2/8] Make nprocs optional --- .../cicedyn/infrastructure/ice_domain.F90 | 38 +++++++++---------- cicecore/shared/ice_distribution.F90 | 2 +- configuration/scripts/ice_in | 2 +- 3 files changed, 19 insertions(+), 23 deletions(-) diff --git a/cicecore/cicedyn/infrastructure/ice_domain.F90 b/cicecore/cicedyn/infrastructure/ice_domain.F90 index 87d181a51..74ac62b79 100644 --- a/cicecore/cicedyn/infrastructure/ice_domain.F90 +++ b/cicecore/cicedyn/infrastructure/ice_domain.F90 @@ -90,7 +90,6 @@ module ice_domain integer (int_kind) :: & nprocs ! num of processors - !*********************************************************************** contains @@ -224,18 +223,25 @@ subroutine init_domain_blocks call broadcast_scalar(nx_global, master_task) call broadcast_scalar(ny_global, master_task) - ! Automatically determine max_blocks if not set. - if (my_task == master_task) then - if (max_blocks < 1) then - call proc_decomposition(nprocs, nprocs_x, nprocs_y) - max_blocks=((nx_global-1)/block_size_x/nprocs_x+1) * & - ((ny_global-1)/block_size_y/nprocs_y+1) - max_blocks=max(1,max_blocks) - write(nu_diag,'(/,a52,i6,/)') & - '(ice_domain): max_block < 1: max_block estimated to ',max_blocks - endif + ! Set nprocs if not set in namelist + if (nprocs .eq. -1) then + nprocs = get_num_procs() + else if (nprocs .ne. get_num_procs()) then + write(nu_diag,*) subname,' ERROR: nprocs, get_num_procs = ',nprocs,get_num_procs() + call abort_ice(subname//' ERROR: Input nprocs not same as system request', file=__FILE__, line=__LINE__) + endif + + ! Determine max_blocks if not set + if (max_blocks < 1) then + call proc_decomposition(nprocs, nprocs_x, nprocs_y) + max_blocks=((nx_global-1)/block_size_x/nprocs_x+1) * & + ((ny_global-1)/block_size_y/nprocs_y+1) + max_blocks=max(1,max_blocks) + write(nu_diag,'(/,a52,i6,/)') & + '(ice_domain): max_block < 1: max_block estimated to ',max_blocks endif + !---------------------------------------------------------------------- ! ! perform some basic checks on domain @@ -247,16 +253,6 @@ subroutine init_domain_blocks !*** domain size zero or negative !*** call abort_ice(subname//' ERROR: Invalid domain: size < 1', file=__FILE__, line=__LINE__) ! no domain - else if (nprocs /= get_num_procs()) then - !*** - !*** input nprocs does not match system (eg MPI) request - !*** -#if (defined CESMCOUPLED) - nprocs = get_num_procs() -#else - write(nu_diag,*) subname,' ERROR: nprocs, get_num_procs = ',nprocs,get_num_procs() - call abort_ice(subname//' ERROR: Input nprocs not same as system request', file=__FILE__, line=__LINE__) -#endif else if (nghost < 1) then !*** !*** must have at least 1 layer of ghost cells diff --git a/cicecore/shared/ice_distribution.F90 b/cicecore/shared/ice_distribution.F90 index 0ba62f4a2..6e06069ab 100644 --- a/cicecore/shared/ice_distribution.F90 +++ b/cicecore/shared/ice_distribution.F90 @@ -42,7 +42,7 @@ module ice_distribution ice_distributionGetBlockLoc, & ice_distributionGetBlockID, & create_local_block_ids, & - proc_decomposition ! mli: add + proc_decomposition character (char_len), public :: & processor_shape ! 'square-pop' (approx) POP default config diff --git a/configuration/scripts/ice_in b/configuration/scripts/ice_in index 63a97d7d8..ad29e05ce 100644 --- a/configuration/scripts/ice_in +++ b/configuration/scripts/ice_in @@ -302,7 +302,7 @@ / &domain_nml - nprocs = 4 + nprocs = -1 nx_global = 100 ny_global = 116 block_size_x = 25 From f8fbdec6736965b8f1e24d346019363b6a337069 Mon Sep 17 00:00:00 2001 From: Anton Steketee Date: Thu, 2 May 2024 15:45:34 +1000 Subject: [PATCH 3/8] only log on master task --- cicecore/cicedyn/infrastructure/ice_domain.F90 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cicecore/cicedyn/infrastructure/ice_domain.F90 b/cicecore/cicedyn/infrastructure/ice_domain.F90 index 74ac62b79..8ca03f17b 100644 --- a/cicecore/cicedyn/infrastructure/ice_domain.F90 +++ b/cicecore/cicedyn/infrastructure/ice_domain.F90 @@ -237,8 +237,10 @@ subroutine init_domain_blocks max_blocks=((nx_global-1)/block_size_x/nprocs_x+1) * & ((ny_global-1)/block_size_y/nprocs_y+1) max_blocks=max(1,max_blocks) - write(nu_diag,'(/,a52,i6,/)') & - '(ice_domain): max_block < 1: max_block estimated to ',max_blocks + if (my_task == master_task) then + write(nu_diag,'(/,a52,i6,/)') & + '(ice_domain): max_block < 1: max_block estimated to ',max_blocks + endif endif From 87640f71733aee8773949b7a8542a1d5b4ef7b19 Mon Sep 17 00:00:00 2001 From: anton-seaice Date: Fri, 3 May 2024 11:15:04 +1000 Subject: [PATCH 4/8] change .eq. to == syntacx for consistency --- cicecore/cicedyn/infrastructure/ice_domain.F90 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cicecore/cicedyn/infrastructure/ice_domain.F90 b/cicecore/cicedyn/infrastructure/ice_domain.F90 index 8ca03f17b..47a0f00ed 100644 --- a/cicecore/cicedyn/infrastructure/ice_domain.F90 +++ b/cicecore/cicedyn/infrastructure/ice_domain.F90 @@ -224,9 +224,9 @@ subroutine init_domain_blocks call broadcast_scalar(ny_global, master_task) ! Set nprocs if not set in namelist - if (nprocs .eq. -1) then + if (nprocs == -1) then nprocs = get_num_procs() - else if (nprocs .ne. get_num_procs()) then + else if (nprocs /= get_num_procs()) then write(nu_diag,*) subname,' ERROR: nprocs, get_num_procs = ',nprocs,get_num_procs() call abort_ice(subname//' ERROR: Input nprocs not same as system request', file=__FILE__, line=__LINE__) endif From 27e1c0540ce75327fa8c45be4d0f6daa32c2e9d1 Mon Sep 17 00:00:00 2001 From: anton-seaice Date: Fri, 3 May 2024 11:27:54 +1000 Subject: [PATCH 5/8] Add CESMCOUPLED back in --- cicecore/cicedyn/infrastructure/ice_domain.F90 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cicecore/cicedyn/infrastructure/ice_domain.F90 b/cicecore/cicedyn/infrastructure/ice_domain.F90 index 47a0f00ed..db8b9d9f6 100644 --- a/cicecore/cicedyn/infrastructure/ice_domain.F90 +++ b/cicecore/cicedyn/infrastructure/ice_domain.F90 @@ -224,12 +224,16 @@ subroutine init_domain_blocks call broadcast_scalar(ny_global, master_task) ! Set nprocs if not set in namelist +#ifdef CESMCOUPLED + nprocs = get_num_procs() +#else if (nprocs == -1) then nprocs = get_num_procs() else if (nprocs /= get_num_procs()) then write(nu_diag,*) subname,' ERROR: nprocs, get_num_procs = ',nprocs,get_num_procs() call abort_ice(subname//' ERROR: Input nprocs not same as system request', file=__FILE__, line=__LINE__) endif +#endif ! Determine max_blocks if not set if (max_blocks < 1) then From 8e2df00cb94d114ee2d05f049f6795b60710f032 Mon Sep 17 00:00:00 2001 From: anton-seaice Date: Fri, 3 May 2024 13:16:26 +1000 Subject: [PATCH 6/8] Add note to docs about set number of procs automatically --- doc/source/user_guide/ug_case_settings.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/user_guide/ug_case_settings.rst b/doc/source/user_guide/ug_case_settings.rst index 9f1f8a259..14ad9318e 100644 --- a/doc/source/user_guide/ug_case_settings.rst +++ b/doc/source/user_guide/ug_case_settings.rst @@ -370,6 +370,7 @@ domain_nml "``maskhalo_bound``", "logical", "mask unused halo cells for boundary updates", "``.false.``" "``max_blocks``", "integer", "maximum number of blocks per MPI task for memory allocation", "-1" "``nprocs``", "integer", "number of processors to use", "-1" + "", "``-1``", "find number of processors automatically", "" "``ns_boundary_type``", "``cyclic``", "periodic boundary conditions in y-direction", "``open``" "", "``open``", "Dirichlet boundary conditions in y", "" "", "``tripole``", "U-fold tripole boundary conditions in y", "" From 56304b7bde088337dc167ca7534065853ce83be5 Mon Sep 17 00:00:00 2001 From: anton-seaice Date: Mon, 6 May 2024 14:44:24 +1000 Subject: [PATCH 7/8] Review feedback --- cicecore/cicedyn/infrastructure/ice_domain.F90 | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cicecore/cicedyn/infrastructure/ice_domain.F90 b/cicecore/cicedyn/infrastructure/ice_domain.F90 index db8b9d9f6..df112eb50 100644 --- a/cicecore/cicedyn/infrastructure/ice_domain.F90 +++ b/cicecore/cicedyn/infrastructure/ice_domain.F90 @@ -227,11 +227,11 @@ subroutine init_domain_blocks #ifdef CESMCOUPLED nprocs = get_num_procs() #else - if (nprocs == -1) then + if (nprocs < 0) then nprocs = get_num_procs() else if (nprocs /= get_num_procs()) then write(nu_diag,*) subname,' ERROR: nprocs, get_num_procs = ',nprocs,get_num_procs() - call abort_ice(subname//' ERROR: Input nprocs not same as system request', file=__FILE__, line=__LINE__) + call abort_ice(subname//' ERROR: Input nprocs not same as system (e.g MPI) request', file=__FILE__, line=__LINE__) endif #endif @@ -247,7 +247,6 @@ subroutine init_domain_blocks endif endif - !---------------------------------------------------------------------- ! ! perform some basic checks on domain From 14ac9b94a2c339d7de7ed337ecd1e90a76864bc4 Mon Sep 17 00:00:00 2001 From: Anton Steketee <79179784+anton-seaice@users.noreply.github.com> Date: Fri, 10 May 2024 09:36:52 +1000 Subject: [PATCH 8/8] Update ug_case_settings.rst --- doc/source/user_guide/ug_case_settings.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/user_guide/ug_case_settings.rst b/doc/source/user_guide/ug_case_settings.rst index 14ad9318e..6deab8c11 100644 --- a/doc/source/user_guide/ug_case_settings.rst +++ b/doc/source/user_guide/ug_case_settings.rst @@ -369,8 +369,8 @@ domain_nml "``maskhalo_remap``", "logical", "mask unused halo cells for transport", "``.false.``" "``maskhalo_bound``", "logical", "mask unused halo cells for boundary updates", "``.false.``" "``max_blocks``", "integer", "maximum number of blocks per MPI task for memory allocation", "-1" - "``nprocs``", "integer", "number of processors to use", "-1" - "", "``-1``", "find number of processors automatically", "" + "``nprocs``", "integer", "number of MPI tasks to use", "-1" + "", "``-1``", "find number of MPI tasks automatically", "" "``ns_boundary_type``", "``cyclic``", "periodic boundary conditions in y-direction", "``open``" "", "``open``", "Dirichlet boundary conditions in y", "" "", "``tripole``", "U-fold tripole boundary conditions in y", ""