sync with escomp

.github/pull_request_template.md

@@ -11,39 +11,6 @@ Are changes expected to change answers? (specify if bfb, different at roundoff,
 Any User Interface Changes (namelist or namelist defaults changes)?
 
 ### Testing performed
+Please describe the tests along with the target model and machine(s) 
+If possible, please also added hashes that were used in the testing
 
-Testing performed if application target is CESM:
-- [ ] (recommended) CIME_DRIVER=nuopc scripts_regression_tests.py
-   - machines:
-   - details (e.g. failed tests):
-- [ ] (recommended) CESM testlist_drv.xml
-   - machines and compilers:
-   - details (e.g. failed tests):
-- [ ] (optional) CESM prealpha test
-   - machines and compilers
-   - details (e.g. failed tests):
-- [ ] (other) please described in detail
-   - machines and compilers
-   - details (e.g. failed tests):
-
-Testing performed if application target is UFS-coupled:
-- [ ] (recommended) UFS-coupled testing
-   - description:
-   - details (e.g. failed tests):
-
-Testing performed if application target is UFS-HAFS:
-- [ ] (recommended) UFS-HAFS testing
-   - description:
-   - details (e.g. failed tests):
-
-### Hashes used for testing:
-
-- [ ] CESM:
-  - repository to check out: https://github.com/ESCOMP/CESM.git
-  - branch/hash:
-- [ ] UFS-coupled, then umbrella repostiory to check out and associated hash:
-  - repository to check out:
-  - branch/hash:
-- [ ] UFS-HAFS, then umbrella repostiory to check out and associated hash:
-  - repository to check out:
-  - branch/hash:

.github/workflows/extbuild.yml

@@ -18,11 +18,13 @@ jobs:
       FC: mpifort
       CXX: mpicxx
       CPPFLAGS: "-I/usr/include -I/usr/local/include"
+
       # Versions of all dependencies can be updated here
-      ESMF_VERSION: v8.4.0
+      ESMF_VERSION: v8.4.2
       PNETCDF_VERSION: checkpoint.1.12.3
       NETCDF_FORTRAN_VERSION: v4.6.0
-      PIO_VERSION: pio2_5_10
+      PIO_VERSION: pio2_6_0
+      CDEPS_VERSION: cdeps1.0.15
     steps:
       - uses: actions/checkout@v3
         # Build the ESMF library, if the cache contains a previous build
@@ -50,14 +52,14 @@ jobs:
           key: ${{ runner.os }}-${{ env.PIO_VERSION }}.pio
       - name: Build ParallelIO
         if: steps.cache-ParallelIO.outputs.cache-hit != 'true'
-        uses: NCAR/ParallelIO/.github/actions/parallelio_cmake@9390e30e29d4ebbfbef0fc72162cacd9e8f25e4e
+        uses: NCAR/ParallelIO/.github/actions/parallelio_cmake@pio2_6_0
         with:
           parallelio_version: ${{ env.ParallelIO_VERSION }}
           enable_fortran: True
           install_prefix: $HOME/pio
       - name: Build ESMF
         if: steps.cache-esmf.outputs.cache-hit != 'true'
-        uses: ESCOMP/CDEPS/.github/actions/buildesmf@e06246b560d3132170bb1a5443fa3d65dfbd2040
+        uses: ESCOMP/CDEPS/.github/actions/buildesmf@cdeps1.0.15
         with:
           esmf_version: ${{ env.ESMF_VERSION }}
           esmf_bopt: g
@@ -67,12 +69,39 @@ jobs:
           netcdf_fortran_path: /usr
           pnetcdf_path: /usr
           parallelio_path: $HOME/pio
+      - name: Cache CDEPS
+        id: cache-cdeps
+        uses: actions/cache@v3
+        with:
+          path: $HOME/cdeps
+          key: ${{ runner.os }}-${{ env.CDEPS_VERSION }}.cdeps
+
+      - name: checkout CDEPS
+        uses: actions/checkout@v3
+        with:
+          repository: ESCOMP/CDEPS
+          path: cdeps-src
+          ref: ${{ env.CDEPS_VERSION }}
+      - name: Build CDEPS
+        if: steps.cache-cdeps.outputs.cache-hit != 'true'
+        uses: ESCOMP/CDEPS/.github/actions/[email protected]
+        with:
+          esmfmkfile: $HOME/ESMF/lib/libg/Linux.gfortran.64.openmpi.default/esmf.mk
+          pio_path: $HOME/pio
+          src_root: ${GITHUB_WORKSPACE}/cdeps-src
+          cmake_flags: " -Wno-dev -DCMAKE_BUILD_TYPE=DEBUG -DWERROR=ON  -DCMAKE_Fortran_FLAGS=\"-DCPRGNU -g -Wall \
+          -ffree-form -ffree-line-length-none -fallow-argument-mismatch \""
+
       - name: Build CMEPS
         run: |
           export ESMFMKFILE=$HOME/ESMF/lib/libg/Linux.gfortran.64.openmpi.default/esmf.mk
           export PIO=$HOME/pio
           mkdir build-cmeps
           pushd build-cmeps
-          cmake -DCMAKE_BUILD_TYPE=DEBUG -DCMAKE_Fortran_FLAGS="-g -Wall -Werror -ffree-form -ffree-line-length-none  -Wno-unused-dummy-argument" ../
+          cmake -DCMAKE_BUILD_TYPE=DEBUG -DCMAKE_Fortran_FLAGS="-g -Wall -Werror -ffree-form -ffree-line-length-none  -Wno-unused-dummy-argument -I /home/runner/work/CMEPS/CMEPS/build-cdeps/share" ../
           make VERBOSE=1
           popd
+
+      - name: Setup tmate session
+        if: ${{ failure() }}
+        uses: mxschmitt/action-tmate@v3

.github/workflows/srt.yml

@@ -26,8 +26,8 @@ jobs:
       CPPFLAGS: "-I/usr/include -I/usr/local/include "
       LDFLAGS: "-L/usr/lib/x86_64-linux-gnu -lnetcdf -lnetcdff -lpnetcdf"     
       # Versions of all dependencies can be updated here
-      ESMF_VERSION: v8.4.0
-      PARALLELIO_VERSION: pio2_5_10
+      ESMF_VERSION: v8.5.0
+      PARALLELIO_VERSION: pio2_6_0
       CIME_MODEL: cesm
       CIME_DRIVER: nuopc
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -153,6 +153,7 @@ jobs:
           mkdir -p $HOME/cesm/scratch
           mkdir -p $HOME/cesm/inputdata
           pushd $GITHUB_WORKSPACE/cesm/cime/CIME/tests
+          export SRCROOT=$GITHUB_WORKSPACE/cesm/
           export CIME_TEST_PLATFORM=ubuntu-latest
           export PIO_INCDIR=$HOME/pio/include
           export PIO_LIBDIR=$HOME/pio/lib
@@ -175,6 +176,6 @@ jobs:
           popd
 #     the following can be used by developers to login to the github server in case of errors
 #     see https://github.com/marketplace/actions/debugging-with-tmate for further details
-#      - name: Setup tmate session
-#        if: ${{ failure() }}
-#        uses: mxschmitt/action-tmate@v3
+      - name: Setup tmate session
+        if: ${{ failure() }}
+        uses: mxschmitt/action-tmate@v3

cesm/driver/ensemble_driver.F90

@@ -340,6 +340,9 @@ subroutine SetModelServices(ensemble_driver, rc)
           else
              inst_suffix = ''
           endif
+          ! CESM does not use this ESMF feature and at large processor counts it can be expensive to have it on.
+          call NUOPC_CompAttributeSet(driver, name="HierarchyProtocol", value="off", rc=rc)
+          if (chkerr(rc,__LINE__,u_FILE_u)) return
 
           ! Set the driver instance attributes
           call NUOPC_CompAttributeAdd(driver, attrList=(/'read_restart'/), rc=rc)
@@ -375,7 +378,7 @@ subroutine SetModelServices(ensemble_driver, rc)
        call shr_log_setLogUnit (logunit)
        ! Create a clock for each driver instance
 
-       call esm_time_clockInit(ensemble_driver, driver, logunit, maintask, rc)
+       call esm_time_clockInit(ensemble_driver, driver, logunit, localpet==petList(1), rc)
        if (chkerr(rc,__LINE__,u_FILE_u)) return
 
     enddo

cesm/driver/esm.F90

@@ -796,8 +796,7 @@ subroutine esm_init_pelayout(driver, maxthreads, rc)
 #ifndef NO_MPI2
     use mpi          , only : MPI_COMM_NULL, mpi_comm_size
 #endif
-    use mct_mod      , only : mct_world_init
-    use driver_pio_mod , only : driver_pio_init, driver_pio_component_init
+    use m_MCTWorld   , only : mct_world_init => init
 
 #ifdef MED_PRESENT
     use med_internalstate_mod , only : med_id

cesm/driver/esmApp.F90

@@ -139,7 +139,7 @@ program esmApp
   ! Call Run  for the ensemble driver
   !-----------------------------------------------------------------------------
   call ESMF_GridCompRun(ensemble_driver_comp, userRc=urc, rc=rc)
-  if (ESMF_LogFoundError(rcToCheck=urc, msg=ESMF_LOGERR_PASSTHRU, &
+  if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
        line=__LINE__, &
        file=__FILE__)) &
        call ESMF_Finalize(endflag=ESMF_END_ABORT)

cesm/flux_atmocn/shr_flux_mod.F90

@@ -259,7 +259,17 @@ SUBROUTINE flux_atmOcn(logunit, nMax  ,zbot  ,ubot  ,vbot  ,thbot ,   &
 
 
     qsat(Tk)   = 640380.0_R8 / exp(5107.4_R8/Tk)
-    cdn(Umps)  =   0.0027_R8 / Umps + 0.000142_R8 + 0.0000764_R8 * Umps
+
+    ! Large and Yeager 2009
+    cdn(Umps)  =  0.0027_R8 / min(33.0000_R8,Umps) + 0.000142_R8 + &
+         0.0000764_R8 * min(33.0000_R8,Umps) - 3.14807e-13_r8 * min(33.0000_R8,Umps)**6
+    ! Capped Large and Pond by wind
+    !   cdn(Umps)  =   0.0027_R8 / min(30.0_R8,Umps) + 0.000142_R8 + 0.0000764_R8 * min(30.0_R8,Umps) 
+    ! Capped Large and Pond by Cd
+    !   cdn(Umps) = min(0.0025_R8, (0.0027_R8 / Umps + 0.000142_R8 + 0.0000764_R8 * Umps ))
+    ! Large and Pond
+    !   cdn(Umps)  =   0.0027_R8 / Umps + 0.000142_R8 + 0.0000764_R8 * Umps
+
     psimhu(xd) = log((1.0_R8+xd*(2.0_R8+xd))*(1.0_R8+xd*xd)/8.0_R8) - 2.0_R8*atan(xd) + 1.571_R8
     psixhu(xd) = 2.0_R8 * log((1.0_R8 + xd*xd)/2.0_R8)
 

cesm/nuopc_cap_share/shr_lightning_coupling_mod.F90

@@ -0,0 +1,112 @@
+module shr_lightning_coupling_mod
+
+  !========================================================================
+  ! Module for handling namelist variables related to lightning coupling
+  !========================================================================
+
+  use ESMF         , only : ESMF_VMGetCurrent, ESMF_VM, ESMF_VMGet
+  use ESMF         , only : ESMF_LOGERR_PASSTHRU, ESMF_SUCCESS
+  use ESMF         , only : ESMF_VMBroadCast, ESMF_Logical, assignment(=)
+  use shr_sys_mod  , only : shr_sys_abort
+  use shr_log_mod  , only : shr_log_getLogUnit
+  use shr_nl_mod   , only : shr_nl_find_group_name
+  use nuopc_shr_methods, only : chkerr
+
+  implicit none
+  private
+
+  ! !PUBLIC MEMBER FUNCTIONS
+  public shr_lightning_coupling_readnl  ! Read namelist
+
+  character(len=*), parameter :: &
+       u_FILE_u=__FILE__
+
+  !====================================================================================
+CONTAINS
+  !====================================================================================
+
+  subroutine shr_lightning_coupling_readnl(NLFilename, atm_provides_lightning_out)
+
+    !========================================================================
+    ! reads lightning_coupling_nl namelist and returns a variable specifying
+    ! if atmosphere model provides lightning flash frequency field to mediator
+    !========================================================================
+
+    ! input/output variables
+    character(len=*), intent(in)  :: NLFilename ! Namelist filename
+    logical, intent(out) :: atm_provides_lightning_out ! if TRUE atm will provide lightning flash frequency
+
+    !----- local -----
+    logical :: atm_provides_lightning
+    type(ESMF_VM)     :: vm
+    integer           :: unitn                  ! namelist unit number
+    integer           :: ierr                   ! error code
+    logical           :: exists                 ! if file exists or not
+    type(ESMF_Logical):: ltmp(1)
+    integer           :: rc
+    integer           :: localpet
+    integer           :: mpicom
+    integer           :: s_logunit
+    character(len=*), parameter :: atm_ozone_frequency_not_present = 'NOT_PRESENT'
+    character(len=*), parameter :: subname = '(shr_lightning_coupling_readnl) '
+    ! ------------------------------------------------------------------
+
+    namelist /lightning_coupling_nl/ atm_provides_lightning
+
+    rc = ESMF_SUCCESS
+
+    atm_provides_lightning_out = .false.
+    ltmp(1) = .false.
+
+    !--- Open and read namelist ---
+    if ( len_trim(NLFilename) == 0 ) then
+       call shr_sys_abort( subname//'ERROR: nlfilename not set' )
+    end if
+    call shr_log_getLogUnit(s_logunit)
+    call ESMF_VMGetCurrent(vm, rc=rc)
+    if (chkerr(rc,__LINE__,u_FILE_u)) return
+
+    call ESMF_VMGet(vm, localPet=localpet, mpiCommunicator=mpicom, rc=rc)
+    if (chkerr(rc,__LINE__,u_FILE_u)) return
+
+    if (localpet==0) then
+       ! ------------------------------------------------------------------------
+       ! Set default values in case namelist file doesn't exist, lightning_coupling_nl group
+       ! doesn't exist within the file, or a given variable isn't present in the namelist
+       ! group in the file.
+       ! ------------------------------------------------------------------------
+       atm_provides_lightning = .false.
+
+       ! ------------------------------------------------------------------------
+       ! Read namelist file
+       ! ------------------------------------------------------------------------
+       inquire( file=trim(NLFileName), exist=exists)
+       if ( exists ) then
+          open(newunit=unitn, file=trim(NLFilename), status='old' )
+          write(s_logunit,'(a)') subname,'Read in lightning_coupling_nl namelist from: ', trim(NLFilename)
+          call shr_nl_find_group_name(unitn, 'lightning_coupling_nl', ierr)
+          if (ierr == 0) then
+             ! Note that ierr /= 0 means no namelist is present.
+             read(unitn, lightning_coupling_nl, iostat=ierr)
+             if (ierr > 0) then
+                call shr_sys_abort(subname//'problem reading lightning_coupling_nl')
+             end if
+          end if
+          close( unitn )
+       end if
+
+       ltmp(1) = atm_provides_lightning
+
+    end if
+
+    ! ------------------------------------------------------------------------
+    ! Broadcast values to all tasks
+    ! ------------------------------------------------------------------------
+    call ESMF_VMBroadcast(vm,  ltmp, count=1, rootPet=0, rc=rc)
+    if (chkerr(rc,__LINE__,u_FILE_u)) return
+
+    atm_provides_lightning_out = ltmp(1)
+
+  end subroutine shr_lightning_coupling_readnl
+
+end module shr_lightning_coupling_mod

cime_config/buildexe

@@ -38,6 +38,7 @@ def _main_func():
         num_esp   = case.get_value("NUM_COMP_INST_ESP")
         ocn_model = case.get_value("COMP_OCN")
         gmake_args = get_standard_makefile_args(case)
+        link_libs = case.get_value("CAM_LINKED_LIBS", subgroup="build_component_cam")
         esmf_aware_threading = case.get_value("ESMF_AWARE_THREADING")
 
     # Determine valid components
@@ -65,6 +66,9 @@ def _main_func():
     if ocn_model == 'mom':
         gmake_args += "USE_FMS=TRUE"
 
+    if link_libs is not None:
+        gmake_args += 'USER_SLIBS="{}"'.format(link_libs)
+
     comp_classes = case.get_values("COMP_CLASSES")
     for comp in comp_classes:
         model = case.get_value("COMP_{}".format(comp))

cime_config/buildnml

@@ -125,17 +125,13 @@ def _create_drv_namelists(case, infile, confdir, nmlgen, files):
     elif case.get_value("RUN_TYPE") == "branch":
         config["run_type"] = "branch"
 
+    config['wav_ice_coupling'] = config['COMP_WAV'] == 'ww3dev' and config['COMP_ICE'] == 'cice'
+
     # ----------------------------------------------------
     # Initialize namelist defaults
     # ----------------------------------------------------
     nmlgen.init_defaults(infile, config, skip_default_for_groups=["modelio"])
 
-    # --------------------------------
-    # Set default wav-ice coupling (assumes cice6 as the ice component
-    # --------------------------------
-    if case.get_value("COMP_WAV") == "ww3dev" and case.get_value("COMP_ICE") == "cice":
-        nmlgen.add_default("wavice_coupling", value=".true.")
-
     # --------------------------------
     # Overwrite: set brnch_retain_casename
     # --------------------------------
@@ -620,14 +616,7 @@ def buildnml(case, caseroot, component):
                 major = line[-2] if "MAJOR" in line else major
                 minor = line[-2] if "MINOR" in line else minor
         logger.debug("ESMF version major {} minor {}".format(major, minor))
-        expect(int(major) >= 8, "ESMF version should be 8.1 or newer")
-        if esmf_aware_threading:
-            expect(
-                int(minor) >= 2,
-                "ESMF version should be 8.2.0 or newer when using ESMF_AWARE_THREADING",
-            )
-        else:
-            expect(int(minor) >= 1, "ESMF version should be 8.1.0 or newer")
+        expect(int(major) >= 8 and int(minor) >=4, "ESMF version should be 8.4.1 or newer")
 
     confdir = os.path.join(case.get_value("CASEBUILD"), "cplconf")
     if not os.path.isdir(confdir):