diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04.yml b/.github/workflows/ci_build_scm_ubuntu_22.04.yml index 6c33a30e5..e89dcb50c 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04.yml @@ -21,7 +21,6 @@ jobs: sp_ROOT: /home/runner/NCEPLIBS-sp w3emc_ROOT: /home/runner/myw3emc SCM_ROOT: /home/runner/work/ccpp-scm/ccpp-scm - intel_ROOT: /home/runner/intel suites: SCM_GFS_v15p2,SCM_GFS_v16,SCM_GFS_v17_p8,SCM_HRRR,SCM_RRFS_v1beta,SCM_RAP,SCM_WoFS_v0,SCM_GFS_v15p2_ps,SCM_GFS_v16_ps,SCM_GFS_v17_p8_ps,SCM_HRRR_ps,SCM_RRFS_v1beta_ps,SCM_RAP_ps,SCM_WoFS_v0_ps # Workflow steps @@ -57,33 +56,17 @@ jobs: ####################################################################################### # Install FORTRAN dependencies ####################################################################################### - - name: Environment for ifort compiler - if: contains(matrix.fortran-compiler, 'ifort') + - name: Install openmpi run: | - echo "CC=icx" >> $GITHUB_ENV - echo "FC=ifort" >> $GITHUB_ENV - echo "NFVERSION=v4.4.4" >> $GITHUB_ENV - - - name: Cache Intel compilers - id: cache-intel-compilers - if: contains(matrix.fortran-compiler, 'ifort') - uses: actions/cache@v2 - with: - path: /home/runner/intel - key: intel-${{ runner.os }}-compilers-b - - # https://software.intel.com/content/www/us/en/develop/articles/installing-intel-oneapi-toolkits-via-apt.html - # List of packages from Docker file at - # https://github.com/intel/oneapi-containers/blob/master/images/docker/hpckit-devel-ubuntu18.04/Dockerfile - - name: Install Intel compilers and libraries - if: contains(matrix.fortran-compiler, 'ifort') && steps.cache-intel-compilers.outputs.cache-hit != 'true' - run: | - wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB - sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB - sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main" - sudo apt-get update - sudo apt-get install intel-hpckit-getting-started intel-oneapi-clck intel-oneapi-common-licensing intel-oneapi-common-vars - sudo apt-get install intel-oneapi-dev-utilities intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic intel-oneapi-compiler-fortran intel-oneapi-itac + wget https://github.com/open-mpi/ompi/archive/refs/tags/v4.1.6.tar.gz + tar -xvf v4.1.6.tar.gz + cd ompi-4.1.6 + ./autogen.pl + ./configure --prefix=/home/runner/ompi-4.1.6 + make -j4 + make install + echo "LD_LIBRARY_PATH=/home/runner/ompi-4.1.6/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "PATH=/home/runner/ompi-4.1.6/bin:$PATH" >> $GITHUB_ENV - name: Cache bacio library v2.4.1 id: cache-bacio-fortran @@ -158,6 +141,11 @@ jobs: sudo make install export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NFHOME}/lib + - name: Environment for openmpi compiler + run: | + echo "FC=mpif90" >> $GITHUB_ENV + echo "CC=mpicc" >> $GITHUB_ENV + ####################################################################################### # Build SCM. ####################################################################################### diff --git a/.github/workflows/ci_run_scm_DEPHY.yml b/.github/workflows/ci_run_scm_DEPHY.yml index eeba4c7fe..326eeb75e 100644 --- a/.github/workflows/ci_run_scm_DEPHY.yml +++ b/.github/workflows/ci_run_scm_DEPHY.yml @@ -54,6 +54,18 @@ jobs: ####################################################################################### # Install FORTRAN dependencies ####################################################################################### + - name: Install openmpi + run: | + wget https://github.com/open-mpi/ompi/archive/refs/tags/v4.1.6.tar.gz + tar -xvf v4.1.6.tar.gz + cd ompi-4.1.6 + ./autogen.pl + ./configure --prefix=/home/runner/ompi-4.1.6 + make -j4 + make install + echo "LD_LIBRARY_PATH=/home/runner/ompi-4.1.6/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "PATH=/home/runner/ompi-4.1.6/bin:$PATH" >> $GITHUB_ENV + - name: Cache bacio library v2.4.1 id: cache-bacio-fortran uses: actions/cache@v3 @@ -128,6 +140,11 @@ jobs: ####################################################################################### # Build SCM. Run DEPHYv1 case. ####################################################################################### + - name: Environment for openmpi compiler + run: | + echo "FC=mpif90" >> $GITHUB_ENV + echo "CC=mpicc" >> $GITHUB_ENV + - name: Download data for SCM run: | cd ${SCM_ROOT} diff --git a/.github/workflows/ci_run_scm_rts.yml b/.github/workflows/ci_run_scm_rts.yml index 9e9efd9aa..632a44fbb 100644 --- a/.github/workflows/ci_run_scm_rts.yml +++ b/.github/workflows/ci_run_scm_rts.yml @@ -59,17 +59,18 @@ jobs: ####################################################################################### # Install FORTRAN dependencies ####################################################################################### - - name: Environment for gfortran compiler - if: contains(matrix.fortran-compiler, 'gnu') - run: | - echo "FC=gfortran-11" >> $GITHUB_ENV - echo "CC=gcc-11" >> $GITHUB_ENV - - name: Environment for ifort compiler - if: contains(matrix.fortran-compiler, 'intel') + - name: Install openmpi run: | - echo "CC=icx" >> $GITHUB_ENV - echo "FC=ifort" >> $GITHUB_ENV + wget https://github.com/open-mpi/ompi/archive/refs/tags/v4.1.6.tar.gz + tar -xvf v4.1.6.tar.gz + cd ompi-4.1.6 + ./autogen.pl + ./configure --prefix=/home/runner/ompi-4.1.6 + make -j4 + make install + echo "LD_LIBRARY_PATH=/home/runner/ompi-4.1.6/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "PATH=/home/runner/ompi-4.1.6/bin:$PATH" >> $GITHUB_ENV - name: Cache bacio library v2.4.1 id: cache-bacio-fortran @@ -142,6 +143,11 @@ jobs: sudo make install export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${NFHOME}/lib + - name: Environment for openmpi compiler + run: | + echo "FC=mpif90" >> $GITHUB_ENV + echo "CC=mpicc" >> $GITHUB_ENV + ####################################################################################### # Build SCM. Run regression tests. ####################################################################################### diff --git a/ccpp/config/ccpp_prebuild_config.py b/ccpp/config/ccpp_prebuild_config.py index 41736151a..d68aaf98b 100755 --- a/ccpp/config/ccpp_prebuild_config.py +++ b/ccpp/config/ccpp_prebuild_config.py @@ -33,6 +33,7 @@ TYPEDEFS_NEW_METADATA = { 'ccpp_types' : { 'ccpp_types' : '', + 'MPI_Comm' : '', 'ccpp_t' : 'cdata', }, 'machine' : { diff --git a/ccpp/framework b/ccpp/framework index f0b9a18b0..011db4f80 160000 --- a/ccpp/framework +++ b/ccpp/framework @@ -1 +1 @@ -Subproject commit f0b9a18b005d950cb9b0038fbc827b6b37500f43 +Subproject commit 011db4f80a02cba6d65958ace56e8efb197be62b diff --git a/ccpp/physics b/ccpp/physics index 6d8fccbea..76cf93fc2 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit 6d8fccbea12c11387c0bc457dcb3855574cd29aa +Subproject commit 76cf93fc26d7b843f6d89ed0ba4799fd5f2c21f7 diff --git a/scm/src/CMakeLists.txt b/scm/src/CMakeLists.txt index 2ef20bb84..e8e8f7bbb 100644 --- a/scm/src/CMakeLists.txt +++ b/scm/src/CMakeLists.txt @@ -105,6 +105,9 @@ ADD_DEFINITIONS(-DCCPP) # Add host-model specific preprocessor flag (needed for some physics schemes) ADD_DEFINITIONS(-DSCM) +ADD_DEFINITIONS(-DMPI) + + #------------------------------------------------------------------------------ # Add model-specific flags for C/C++/Fortran preprocessor ADD_DEFINITIONS(-DMOIST_CAPPA -DUSE_COND -DNEMS_GSM) @@ -257,6 +260,7 @@ endif() # How about using proper compile targets etc? #------------------------------------------------------------------------------ # Configure sources + ADD_SUBDIRECTORY(${CCPP_FRAMEWORK_SRC} ${CMAKE_BINARY_DIR}/ccpp/framework) ADD_SUBDIRECTORY(${CCPP_PHYSICS_SRC} ${CMAKE_BINARY_DIR}/ccpp/physics) ADD_DEPENDENCIES(ccpp_physics ccpp_framework) diff --git a/scm/src/GFS_typedefs.F90 b/scm/src/GFS_typedefs.F90 index cf4a6db0c..51e6ae1ed 100644 --- a/scm/src/GFS_typedefs.F90 +++ b/scm/src/GFS_typedefs.F90 @@ -1,5 +1,6 @@ module GFS_typedefs + use mpi_f08 use machine, only: kind_phys, kind_dbl_prec, kind_sngl_prec use module_radsw_parameters, only: topfsw_type, sfcfsw_type @@ -85,7 +86,7 @@ module GFS_typedefs type GFS_init_type integer :: me !< my MPI-rank integer :: master !< master MPI-rank - integer :: fcst_mpi_comm !< forecast tasks mpi communicator + type(MPI_Comm) :: fcst_mpi_comm !< forecast tasks mpi communicator integer :: fcst_ntasks !< total number of forecast tasks integer :: tile_num !< tile number for this MPI rank integer :: isc !< starting i-index for this MPI-domain @@ -693,7 +694,7 @@ module GFS_typedefs integer :: me !< MPI rank designator integer :: master !< MPI rank of master atmosphere processor - integer :: communicator !< MPI communicator + type(MPI_Comm) :: communicator !< MPI communicator integer :: ntasks !< MPI size in communicator integer :: nthreads !< OpenMP threads available for physics integer :: nlunit !< unit for namelist @@ -1153,6 +1154,7 @@ module GFS_typedefs logical :: lheatstrg !< flag for canopy heat storage parameterization logical :: lseaspray !< flag for sea spray parameterization logical :: cnvcld + logical :: xr_cnvcld !< flag for adding suspended convective clouds to Xu-Randall cloud fraction logical :: random_clds !< flag controls whether clouds are random logical :: shal_cnv !< flag for calling shallow convection logical :: do_deep !< whether to do deep convection @@ -2582,7 +2584,6 @@ subroutine sfcprop_create (Sfcprop, IM, Model) allocate(Sfcprop%iceprv (IM)) allocate(Sfcprop%snowprv (IM)) allocate(Sfcprop%graupelprv(IM)) - Sfcprop%iceprv = clear_val Sfcprop%snowprv = clear_val Sfcprop%graupelprv = clear_val @@ -3297,7 +3298,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & real(kind=kind_phys), dimension(:), intent(in) :: bk logical, intent(in) :: restart logical, intent(in) :: hydrostatic - integer, intent(in) :: communicator + type(MPI_Comm), intent(in) :: communicator integer, intent(in) :: ntasks integer, intent(in) :: nthreads @@ -3307,9 +3308,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & integer :: seed0 logical :: exists real(kind=kind_phys) :: tem - real(kind=kind_phys) :: rinc(5) - real(kind=kind_sngl_prec) :: rinc4(5) - real(kind=kind_dbl_prec) :: rinc8(5) + real(kind=kind_dbl_prec) :: rinc(5) real(kind=kind_phys) :: wrk(1) real(kind=kind_phys), parameter :: con_hr = 3600. @@ -3673,6 +3672,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & logical :: lheatstrg = .false. !< flag for canopy heat storage parameterization logical :: lseaspray = .false. !< flag for sea spray parameterization logical :: cnvcld = .false. + logical :: xr_cnvcld = .true. !< flag for including suspended convective clouds in Xu-Randall cloud fraction logical :: random_clds = .false. !< flag controls whether clouds are random logical :: shal_cnv = .false. !< flag for calling shallow convection integer :: imfshalcnv = 1 !< flag for mass-flux shallow convection scheme @@ -3979,7 +3979,6 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & real(kind=kind_phys) :: radar_tten_limits(2) = (/ limit_unspecified, limit_unspecified /) integer :: itime - integer :: w3kindreal,w3kindint !--- END NAMELIST VARIABLES @@ -4069,8 +4068,8 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & hwrf_samfdeep, hwrf_samfshal,progsigma,betascu,betamcu, & betadcu,h2o_phys, pdfcld, shcnvcw, redrag, hybedmf, satmedmf,& shinhong, do_ysu, dspheat, lheatstrg, lseaspray, cnvcld, & - random_clds, shal_cnv, imfshalcnv, imfdeepcnv, isatmedmf, & - do_deep, jcap, & + xr_cnvcld, random_clds, shal_cnv, imfshalcnv, imfdeepcnv, & + isatmedmf, do_deep, jcap, & cs_parm, flgmin, cgwf, ccwf, cdmbgwd, sup, ctei_rm, crtrh, & dlqf, rbcr, shoc_parm, psauras, prauras, wminras, & do_sppt, do_shum, do_skeb, & @@ -4949,6 +4948,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & Model%lheatstrg = lheatstrg Model%lseaspray = lseaspray Model%cnvcld = cnvcld + Model%xr_cnvcld = xr_cnvcld Model%random_clds = random_clds Model%shal_cnv = shal_cnv Model%imfshalcnv = imfshalcnv @@ -5664,19 +5664,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & Model%cdec = -9999. Model%clstp = -9999 rinc(1:5) = 0 - call w3kind(w3kindreal,w3kindint) - if (w3kindreal == 8) then - rinc8(1:5) = 0 - call w3difdat(jdat,idat,4,rinc8) - rinc = rinc8 - else if (w3kindreal == 4) then - rinc4(1:5) = 0 - call w3difdat(jdat,idat,4,rinc4) - rinc = rinc4 - else - write(0,*)' FATAL ERROR: Invalid w3kindreal' - call abort - endif + call w3difdat(jdat,idat,4,rinc) Model%phour = rinc(4)/con_hr Model%fhour = (rinc(4) + Model%dtp)/con_hr Model%zhour = mod(Model%phour,Model%fhzero) @@ -6251,7 +6239,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & ' do_shoc=', Model%do_shoc, ' nshoc3d=', Model%nshoc_3d, & ' nshoc_2d=', Model%nshoc_2d, ' shoc_cld=', Model%shoc_cld, & ' nkbfshoc=', Model%nkbfshoc, ' nahdshoc=', Model%nahdshoc, & - ' nscfshoc=', Model%nscfshoc, & + ' nscfshoc=', Model%nscfshoc, ' xr_cnvcld=',Model%xr_cnvcld, & ' uni_cld=', Model%uni_cld, & ' ntot3d=', Model%ntot3d, ' ntot2d=', Model%ntot2d, & ' shocaftcnv=',Model%shocaftcnv,' indcld=', Model%indcld, & @@ -6474,7 +6462,7 @@ subroutine control_print(Model) print *, 'basic control parameters' print *, ' me : ', Model%me print *, ' master : ', Model%master - print *, ' communicator : ', Model%communicator + print *, ' communicator : ', Model%communicator%mpi_val print *, ' nlunit : ', Model%nlunit print *, ' fn_nml : ', trim(Model%fn_nml) print *, ' fhzero : ', Model%fhzero @@ -7192,7 +7180,6 @@ subroutine tbd_create (Tbd, IM, Model) allocate (Tbd%h2opl (IM,levh2o,h2o_coeff)) Tbd%h2opl = clear_val Tbd%ozpl = clear_val - !--- ccn and in needs ! DH* allocate only for MG? *DH diff --git a/scm/src/GFS_typedefs.meta b/scm/src/GFS_typedefs.meta index 02f58b37a..2b2d485bf 100644 --- a/scm/src/GFS_typedefs.meta +++ b/scm/src/GFS_typedefs.meta @@ -3346,7 +3346,7 @@ long_name = MPI communicator units = index dimensions = () - type = integer + type = MPI_Comm [ntasks] standard_name = number_of_mpi_tasks long_name = number of MPI tasks in communicator @@ -5424,6 +5424,12 @@ units = flag dimensions = () type = logical +[xr_cnvcld] + standard_name = flag_for_suspended_convective_clouds_in_Xu_Randall + long_name = flag for using suspended convective clouds in Xu Randall + units = flag + dimensions = () + type = logical [shal_cnv] standard_name = flag_for_simplified_arakawa_schubert_shallow_convection long_name = flag for calling shallow convection @@ -10047,7 +10053,7 @@ [ltg1_max] standard_name = lightning_threat_index_1 long_name = lightning threat index 1 - units = flashes 5 min-1 + units = flashes min-1 dimensions = (horizontal_loop_extent) type = real kind = kind_phys @@ -10056,7 +10062,7 @@ [ltg2_max] standard_name = lightning_threat_index_2 long_name = lightning threat index 2 - units = flashes 5 min-1 + units = flashes min-1 dimensions = (horizontal_loop_extent) type = real kind = kind_phys @@ -10065,7 +10071,7 @@ [ltg3_max] standard_name = lightning_threat_index_3 long_name = lightning threat index 3 - units = flashes 5 min-1 + units = flashes min-1 dimensions = (horizontal_loop_extent) type = real kind = kind_phys diff --git a/scm/src/run_scm.py b/scm/src/run_scm.py index 926fb68ba..b499fafac 100755 --- a/scm/src/run_scm.py +++ b/scm/src/run_scm.py @@ -31,6 +31,9 @@ # Path to default bin directory (relative to scm_root) DEFAULT_BIN_DIR = 'scm/bin' +# Default command string to run MPI apps (number of processes should be 1 since SCM is not set up to use more than 1 yet) +DEFAULT_MPI_COMMAND = 'mpirun -np 1' + # Copy executable to run directory if true (otherwise it will be linked) COPY_EXECUTABLE = False @@ -119,6 +122,7 @@ parser.add_argument('-dt', '--timestep', help='timestep (s)', required=False, type=float) parser.add_argument('-v', '--verbose', help='set logging level to debug and write log to file', action='count', default=0) parser.add_argument('-f', '--file', help='name of file where SCM runs are defined') +parser.add_argument('--mpi_command', help='command used to invoke the executable via MPI (including options)', required=False) ############################################################################### # Functions and subroutines # @@ -185,13 +189,14 @@ def parse_arguments(): run_dir = args.run_dir bin_dir = args.bin_dir timestep = args.timestep + mpi_command = args.mpi_command if not sdf: sdf = DEFAULT_SUITE return (file, case, sdf, namelist, tracers, gdb, runtime, runtime_mult, docker, \ verbose, levels, npz_type, vert_coord_file, case_data_dir, n_itt_out, \ - n_itt_diag, run_dir, bin_dir, timestep) + n_itt_diag, run_dir, bin_dir, timestep, mpi_command) def find_gdb(): """Detect gdb, abort if not found""" @@ -715,12 +720,16 @@ def setup_rundir(self): return os.path.join(SCM_RUN, output_dir) -def launch_executable(use_gdb, gdb, ignore_error = False): +def launch_executable(use_gdb, gdb, mpi_command, ignore_error = False): """Configure model run command and pass control to shell/gdb""" if use_gdb: - cmd = '(cd {scm_run} && {gdb} {executable})'.format(scm_run=SCM_RUN, gdb=gdb, executable=EXECUTABLE) + if not mpi_command: + mpi_command = DEFAULT_MPI_COMMAND + ' xterm -e ' + cmd = '(cd {scm_run} && {mpi_command} {gdb} {executable})'.format(scm_run=SCM_RUN, mpi_command=mpi_command, gdb=gdb, executable=EXECUTABLE) else: - cmd = '(cd {scm_run} && time {executable})'.format(scm_run=SCM_RUN, executable=EXECUTABLE) + if not mpi_command: + mpi_command = DEFAULT_MPI_COMMAND + cmd = '(cd {scm_run} && time {mpi_command} {executable})'.format(scm_run=SCM_RUN, mpi_command=mpi_command, executable=EXECUTABLE) logging.info('Passing control to "{0}"'.format(cmd)) time.sleep(1) # This will abort in 'execute' in the event of an error if ignore_error = False @@ -756,7 +765,7 @@ def copy_outdir(exp_dir): def main(): (file, case, sdf, namelist, tracers, use_gdb, runtime, runtime_mult, docker, \ verbose, levels, npz_type, vert_coord_file, case_data_dir, n_itt_out, \ - n_itt_diag, run_dir, bin_dir, timestep) = parse_arguments() + n_itt_diag, run_dir, bin_dir, timestep, mpi_command) = parse_arguments() setup_logging(verbose) @@ -873,7 +882,7 @@ def main(): l_ignore_error = MULTIRUN_IGNORE_ERROR else: l_ignore_error = False - (status, time_elapsed) = launch_executable(use_gdb, gdb, ignore_error = l_ignore_error) + (status, time_elapsed) = launch_executable(use_gdb, gdb, mpi_command, ignore_error = l_ignore_error) # if status == 0: logging.info('Process "(case={0}, suite={1}, namelist={2}" completed successfully'. \ diff --git a/scm/src/scm.F90 b/scm/src/scm.F90 index ba152f1c1..9a05e974a 100644 --- a/scm/src/scm.F90 +++ b/scm/src/scm.F90 @@ -15,7 +15,8 @@ subroutine scm_main_sub() use scm_time_integration use scm_output use scm_type_defs - + use mpi_f08 + use :: ccpp_static_api, & only: ccpp_physics_init, & ccpp_physics_timestep_init, & @@ -30,6 +31,8 @@ subroutine scm_main_sub() type(scm_input_type), target :: scm_input_instance type(scm_reference_type), target :: scm_reference + type(MPI_Comm) :: fcst_mpi_comm + integer :: i, j, kdt_rad, idtend, itrac real(kind=8) :: rinc(5) !(DAYS, HOURS, MINUTES, SECONDS, MILLISECONDS) integer :: jdat(1:8) @@ -39,6 +42,13 @@ subroutine scm_main_sub() character(len=16) :: logfile_name logical :: in_spinup + call MPI_INIT(ierr) + if (ierr/=0) then + write(*,*) 'An error occurred in MPI_INIT: ', ierr + stop 1 + end if + fcst_mpi_comm = MPI_COMM_WORLD + call get_config_nml(scm_state) select case(scm_state%input_type) @@ -94,6 +104,7 @@ subroutine scm_main_sub() open(unit=physics%Init_parm%logunit, file=trim(scm_state%output_dir)//'/'//logfile_name, action='write', status='replace') end if + physics%Init_parm%fcst_mpi_comm = fcst_mpi_comm physics%Init_parm%levs = scm_state%n_levels physics%Init_parm%bdat(1) = scm_state%init_year physics%Init_parm%bdat(2) = scm_state%init_month @@ -120,7 +131,7 @@ subroutine scm_main_sub() call GFS_suite_setup(physics%Model, physics%Statein, physics%Stateout, & physics%Sfcprop, physics%Coupling, physics%Grid, & physics%Tbd, physics%Cldprop, physics%Radtend, & - physics%Diag, physics%Interstitial, 0, 1, 1, & + physics%Diag, physics%Interstitial, 1, 1, & physics%Init_parm, scm_state%n_cols, scm_state%lon, & scm_state%lat, scm_state%area) @@ -418,6 +429,12 @@ subroutine scm_main_sub() write(*,'(a,i0,a)') 'An error occurred in ccpp_physics_finalize: ' // trim(cdata%errmsg) // '. Exiting...' stop 1 end if + + call MPI_FINALIZE(ierr) + if (ierr/=0) then + write(*,*) 'An error occurred in MPI_FINALIZE: ', ierr + stop 1 + end if end subroutine scm_main_sub @@ -429,5 +446,7 @@ end module scm_main !! subroutine \ref scm_main_sub above. program scm use scm_main + use mpi_f08 + call scm_main_sub() end program scm diff --git a/scm/src/scm_setup.F90 b/scm/src/scm_setup.F90 index 30ebf3d47..5f4423e29 100644 --- a/scm/src/scm_setup.F90 +++ b/scm/src/scm_setup.F90 @@ -279,7 +279,7 @@ end subroutine patch_in_ref !-------------- subroutine GFS_suite_setup (Model, Statein, Stateout, Sfcprop, & Coupling, Grid, Tbd, Cldprop, Radtend, Diag, & - Interstitial, communicator, ntasks, nthreads, & + Interstitial, ntasks, nthreads, & Init_parm, n_cols, lon, lat, area) use machine, only: kind_phys @@ -311,7 +311,6 @@ subroutine GFS_suite_setup (Model, Statein, Stateout, Sfcprop, type(GFS_interstitial_type), intent(inout) :: Interstitial type(GFS_init_type), intent(in) :: Init_parm - integer, intent(in) :: communicator integer, intent(in) :: ntasks, nthreads, n_cols real(kind=dp), dimension(n_cols), intent(in) :: lon, lat, area @@ -334,7 +333,7 @@ subroutine GFS_suite_setup (Model, Statein, Stateout, Sfcprop, Init_parm%input_nml_file, Init_parm%tile_num, & Init_parm%blksz, Init_parm%ak, Init_parm%bk, & Init_parm%restart, Init_parm%hydrostatic, & - communicator, ntasks, nthreads) + Init_parm%fcst_mpi_comm, ntasks, nthreads) !--- initialize DDTs diff --git a/test/plot_scm_out.py b/test/plot_scm_out.py index d8474071e..2c2d50ea8 100755 --- a/test/plot_scm_out.py +++ b/test/plot_scm_out.py @@ -82,7 +82,7 @@ def plot_results(file_bl, file_rt=None, vars2plt=None): if file_rt is not None: x2 = SCM_RT[timeD][:].squeeze()/3600. #seconds - >hours # If temporal dimensions disagree, con't compute deltas from experiments, turn off difference plots. - if (len(x1) != len(x2)): + if (x1.shape != x2.shape): plot_diff = False # end if # end if