Merge remote-tracking branch 'origin/develop' into block_atmos_copy_bug

CDEPS-interface/cdeps_files.cmake

@@ -12,8 +12,10 @@ list(APPEND cdeps_share_files
   CDEPS/share/shr_assert.h
   CDEPS/share/shr_cal_mod.F90
   CDEPS/share/shr_const_mod.F90
+  CDEPS/share/shr_file_mod.F90
   CDEPS/share/shr_kind_mod.F90
   CDEPS/share/shr_log_mod.F90
+  CDEPS/share/shr_nl_mod.F90
   CDEPS/share/shr_orb_mod.F90
   CDEPS/share/shr_precip_mod.F90
   CDEPS/share/shr_strconvert_mod.F90

CMEPS-interface/CMakeLists.txt

@@ -81,6 +81,11 @@ list(APPEND _mediator_files
   CMEPS/mediator/med_phases_post_rof_mod.F90
   CMEPS/mediator/med_phases_post_wav_mod.F90)
 
+if(CDEPS_INLINE)
+  list(APPEND _mediator_files CMEPS/mediator/med_phases_cdeps_mod.F90)
+  add_definitions(-DCDEPS_INLINE)
+endif()
+
 ###############################################################################
 ### Host model for CCPP
 ###############################################################################
@@ -184,6 +189,12 @@ if(OpenMP_Fortran_FOUND)
   target_link_libraries(cmeps PRIVATE OpenMP::OpenMP_Fortran)
 endif()
 
+### Set dependency for CDEPS if inline capability is requested
+if(CDEPS_INLINE)
+  add_dependencies(cmeps cdeps::cdeps)
+  target_link_libraries(cmeps PUBLIC cdeps::cdeps)
+endif()
+
 ###############################################################################
 ### Install
 ###############################################################################

CMakeLists.txt

@@ -16,7 +16,7 @@ list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/CMakeModules/Modules)
 ###############################################################################
 
 # Valid applications and choices
-list(APPEND VALID_APPS ATM ATMAERO ATMAQ ATMW ATMWM ATML LND S2S S2SA S2SW S2SWA S2SWAL HAFS HAFSW HAFS-ALL NG-GODAS)
+list(APPEND VALID_APPS ATM ATMAERO ATMAQ ATMW ATMWM ATML LND S2S S2SA S2SW S2SWA S2SWAL HAFS HAFSW HAFS-MOM6 HAFS-MOM6W HAFS-ALL NG-GODAS)
 set(APP NONE CACHE BOOL "Application Name")
 if(NOT (APP IN_LIST VALID_APPS))
   message(FATAL_ERROR "${APP} is not a valid application.\nValid Applications are: ${VALID_APPS}")
@@ -66,11 +66,13 @@ set(DISABLE_FMA     OFF CACHE BOOL "Disable Fused Multiply-Add instructions (wor
 set(INLINE_POST     ON  CACHE BOOL "Enable inline post")
 set(MULTI_GASES     OFF CACHE BOOL "Enable MULTI_GASES")
 set(MOVING_NEST     OFF CACHE BOOL "Enable moving nest code")
+set(REGIONAL_MOM6   OFF CACHE BOOL "Enable Regional MOM6")
 set(OPENMP          ON  CACHE BOOL "Enable OpenMP threading")
 set(PARALLEL_NETCDF OFF CACHE BOOL "Enable parallel NetCDF")
 set(JEDI_DRIVER     OFF CACHE BOOL "Enable JEDI as top level driver")
 set(CMEPS_AOFLUX    OFF CACHE BOOL "Enable atmosphere-ocean flux calculation in mediator")
 set(PDLIB           OFF CACHE BOOL "Enable Domain Decomposition in WW3 via PDLIB")
+set(CDEPS_INLINE    OFF CACHE BOOL "Enable CDEPS inline capability")
 
 set(CMAKE_Platform $ENV{CMAKE_Platform})
 if(CMAKE_Platform)
@@ -114,10 +116,12 @@ message("DEBUG ............ ${DEBUG}")
 message("INLINE_POST ...... ${INLINE_POST}")
 message("MULTI_GASES ...... ${MULTI_GASES}")
 message("MOVING_NEST ...... ${MOVING_NEST}")
+message("REGIONAL_MOM6..... ${REGIONAL_MOM6}")
 message("OPENMP ........... ${OPENMP}")
 message("PARALLEL_NETCDF .. ${PARALLEL_NETCDF}")
 message("JEDI_DRIVER ...... ${JEDI_DRIVER}")
 message("CMEPS_AOFLUX ..... ${CMEPS_AOFLUX}")
+message("CDEPS_INLINE ..... ${CDEPS_INLINE}")
 
 message("")
 
@@ -146,7 +150,7 @@ if(FMS)
   find_package(FMS 2022.04 REQUIRED COMPONENTS R4 R8)
   if(APP MATCHES "^(HAFSW)$")
     add_library(fms ALIAS FMS::fms_r4)
-  elseif (APP MATCHES "^(S2S|S2SA|S2SW|S2SWA|S2SWAL|NG-GODAS)$")
+  elseif (APP MATCHES "^(S2S|S2SA|S2SW|S2SWA|S2SWAL|NG-GODAS|HAFS-MOM6|HAFS-MOM6W)$")
     add_library(fms ALIAS FMS::fms_r8)
   endif()
   if(APP MATCHES "^(ATM|ATMAERO|ATMAQ|ATMWM|ATMW|ATML|HAFS|HAFS-ALL)$")

MOM6-interface/CMakeLists.txt

@@ -28,8 +28,14 @@ include("mom6_files.cmake")
 add_library(mom6_obj OBJECT ${mom6_src_files})
 set_target_properties(mom6_obj PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod)
 target_include_directories(mom6_obj PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/mod>)
-target_include_directories(mom6_obj PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/MOM6/config_src/memory/dynamic_nonsymmetric>
-                                            $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/MOM6/src/framework>)
+
+if(REGIONAL_MOM6)
+  target_include_directories(mom6_obj PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/MOM6/config_src/memory/dynamic_symmetric>
+                                              $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/MOM6/src/framework>)
+else()
+  target_include_directories(mom6_obj PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/MOM6/config_src/memory/dynamic_nonsymmetric>
+                                              $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/MOM6/src/framework>)
+endif()
 target_link_libraries(mom6_obj PRIVATE fms
                                        esmf
                                        stochastic_physics
@@ -42,8 +48,13 @@ target_link_libraries(mom6_obj PRIVATE fms
 add_library(mom6_nuopc_obj OBJECT ${mom6_nuopc_src_files})
 set_target_properties(mom6_nuopc_obj PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod)
 target_include_directories(mom6_nuopc_obj PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/mod>)
-target_include_directories(mom6_nuopc_obj PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/MOM6/config_src/memory/dynamic_nonsymmetric>
-                                                  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/MOM6/src/framework>)
+if(REGIONAL_MOM6)
+  target_include_directories(mom6_nuopc_obj PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/MOM6/config_src/memory/dynamic_symmetric>
+                                                    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/MOM6/src/framework>)
+else()
+  target_include_directories(mom6_nuopc_obj PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/MOM6/config_src/memory/dynamic_nonsymmetric>
+                                                    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/MOM6/src/framework>)
+endif()
 target_link_libraries(mom6_nuopc_obj PRIVATE mom6_obj
                                              fms
                                              stochastic_physics

cmake/configure_apps.cmake

@@ -68,17 +68,19 @@ if(APP MATCHES "^(S2S|S2SA|S2SW|S2SWA|S2SWAL)$")
   message("${APP_MSG} mode")
 endif()
 
-if(APP MATCHES "^(HAFS|HAFSW|HAFS-ALL)$")
+if(APP MATCHES "^(HAFS|HAFSW|HAFS-MOM6|HAFS-MOM6W|HAFS-ALL)$")
   set(CMEPS      ON  CACHE BOOL "Enable CMEPS"               FORCE)
-  if(APP MATCHES "^(HAFS-ALL)$")
-    set(CDEPS    ON  CACHE BOOL "Enable CDEPS"               FORCE)
-    message("Configuring UFS app in HAFS with CDEPS mode")
-  endif()
+  set(CDEPS      ON  CACHE BOOL "Enable CDEPS"               FORCE)
   set(FMS        ON  CACHE BOOL "Enable FMS"                 FORCE)
   set(FV3        ON  CACHE BOOL "Enable FV3"                 FORCE)
   set(STOCH_PHYS ON  CACHE BOOL "Enable Stochastic Physics"  FORCE)
-  set(HYCOM      ON  CACHE BOOL "Enable HYCOM"               FORCE)
-  if(APP MATCHES "^(HAFSW|HAFS-ALL)$")
+  if(APP MATCHES "^(HAFS-MOM6|HAFS-MOM6W)$")
+    set(MOM6       ON  CACHE BOOL "Enable MOM6"              FORCE)
+  endif()
+  if(APP MATCHES "^(HAFS|HAFSW|HAFS-ALL)$")
+    set(HYCOM      ON  CACHE BOOL "Enable HYCOM"             FORCE)
+  endif()
+  if(APP MATCHES "^(HAFSW|HAFS-MOM6W|HAFS-ALL)$")
     set(WW3      ON  CACHE BOOL "Enable WAVEWATCH III"       FORCE)
     message("Configuring UFS app in HAFS with Waves mode")
   endif()

tests/bl_date.conf

@@ -1 +1 @@
-export BL_DATE=20240126
+export BL_DATE=20240208

tests/default_vars.sh

@@ -324,6 +324,7 @@ export HAFS=false
 export AQM=false
 export DATM_CDEPS=false
 export DOCN_CDEPS=false
+export CDEPS_INLINE=false
 export POSTAPP='global'
 export USE_MERRA2=.false.
 
@@ -775,6 +776,7 @@ export HAFS=false
 export AQM=false
 export DATM_CDEPS=false
 export DOCN_CDEPS=false
+export CDEPS_INLINE=false
 export FV3BMIC='p8c'
 export BMIC=.false.
 export DAYS=1
@@ -972,6 +974,7 @@ export HAFS=false
 export AQM=false
 export DATM_CDEPS=true
 export DOCN_CDEPS=false
+export CDEPS_INLINE=false
 export DAYS=1
 
 # model configure
@@ -1047,6 +1050,7 @@ export HAFS=true
 export AQM=false
 export DATM_CDEPS=true
 export DOCN_CDEPS=false
+export CDEPS_INLINE=false
 export INPES=$INPES_dflt
 export JNPES=$JNPES_dflt
 export NTILES=1
@@ -1063,6 +1067,7 @@ export S2S=false
 export HAFS=true
 export AQM=false
 export DOCN_CDEPS=true
+export CDEPS_INLINE=false
 export INPES=$INPES_dflt
 export JNPES=$JNPES_dflt
 export NTILES=1
@@ -1081,6 +1086,7 @@ export HAFS=true
 export AQM=false
 export DATM_CDEPS=false
 export DOCN_CDEPS=false
+export CDEPS_INLINE=false
 export INPES=$INPES_dflt
 export JNPES=$JNPES_dflt
 export NTILES=1
@@ -1152,6 +1158,7 @@ export HAFS=true
 export AQM=false
 export DATM_CDEPS=false
 export DOCN_CDEPS=false
+export CDEPS_INLINE=false
 export INPES=$INPES_dflt
 export JNPES=$JNPES_dflt
 export NTILES=1

tests/fv3_conf/control_run.IN

@@ -13,10 +13,15 @@ fi
 echo "inputdir=$inputdir,NPX=$NPX"
 
 OPNREQ_TEST=${OPNREQ_TEST:-false}
+V2_SFC_FILE=${V2_SFC_FILE:-false}
 SUFFIX=${RT_SUFFIX}
 if [ $WARM_START = .false. ]; then
   mkdir INPUT RESTART
-  cp -r @[INPUTDATA_ROOT]/${inputdir}/INPUT_L127/* ./INPUT/.
+  if [ "$V2_SFC_FILE" = "true" ]; then
+    cp -r @[INPUTDATA_ROOT]/${inputdir}/INPUT_L127_v2_sfc/* ./INPUT/.
+  else
+    cp -r @[INPUTDATA_ROOT]/${inputdir}/INPUT_L127/* ./INPUT/.
+  fi
 else
   mkdir INPUT RESTART
 

tests/fv3_conf/cpld_control_run.IN

@@ -15,6 +15,8 @@ fi
 
 ICERES=${OCNRES:0:1}.${OCNRES:1}
 
+V2_SFC_FILE=${V2_SFC_FILE:-false}
+
 if [[ $BMIC == .true. ]]; then
   FV3_IC=@[INPUTDATA_ROOT_BMIC]/${SYEAR}${SMONTH}${SDAY}${SHOUR}/@[FV3BMIC]/@[ATMRES]_L@[NPZ]/INPUT
   MOM_IC=@[INPUTDATA_ROOT_BMIC]/${SYEAR}${SMONTH}${SDAY}${SHOUR}/mom6_da
@@ -25,7 +27,11 @@ else
      MOM_IC=@[INPUTDATA_ROOT]/MOM6_IC/${OCNRES}/${SYEAR}${SMONTH}${SDAY}${SHOUR}
      ICE_IC=@[INPUTDATA_ROOT]/CICE_IC/${OCNRES}/${SYEAR}${SMONTH}${SDAY}${SHOUR}
   else
-     FV3_IC=@[INPUTDATA_ROOT]/${FV3_DIR}/INPUT_L127_mx${OCNRES}
+     if [ "$V2_SFC_FILE" = "true" ]; then
+       FV3_IC=@[INPUTDATA_ROOT]/${FV3_DIR}/INPUT_L127_mx${OCNRES}_v2_sfc
+     else
+       FV3_IC=@[INPUTDATA_ROOT]/${FV3_DIR}/INPUT_L127_mx${OCNRES}
+     fi
      MOM_IC=@[INPUTDATA_ROOT]/MOM6_IC
      ICE_IC=@[INPUTDATA_ROOT]/CICE_IC/${OCNRES}
   fi

tests/fv3_conf/hafs_fv3_run.IN

@@ -17,6 +17,8 @@ elif [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm" ] || \
      [ ${CNTL_DIR} = "hafs_regional_specified_moving_1nest_atm" ] || \
      [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn" ] || \
      [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn_debug" ] || \
+     [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn_wav_inline" ] || \
+     [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn_wav_mom6" ] || \
      [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn_wav" ]; then
   cp -r @[INPUTDATA_ROOT]/FV3_hafs_input_data/INPUT_hafs_regional_storm_following_1nest_atm/* ./INPUT/
 elif [ ${CNTL_DIR} = "hafs_regional_1nest_atm" ] || \
@@ -49,3 +51,16 @@ if [[ $POSTAPP = 'hafs' ]]; then
     cp ${PATHRT}/parm/postxconfig-NT-hafs.txt postxconfig-NT_FH00.txt
     cp ${PATHRT}/parm/params_grib2_tbl_new params_grib2_tbl_new
 fi
+
+if [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn_wav_mom6" ]; then
+  rm -rf OUTPUT
+  mkdir OUTPUT
+  cp -r @[INPUTDATA_ROOT]/FV3_hafs_input_data/MOM6_regional_input_data/* ./INPUT/
+  cp ${PATHRT}/parm/MOM_input_hafs ./MOM_input
+  cp ${PATHRT}/parm/MOM6_data_table_hafs ./data_table
+fi
+if [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn_wav_inline" ] || \
+   [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn_wav_mom6" ]; then
+  cp @[INPUTDATA_ROOT]/FV3_hafs_input_data/CDEPS_input_data/gfs_mesh.nc ./INPUT/
+  cp @[INPUTDATA_ROOT]/FV3_hafs_input_data/CDEPS_input_data/gfs_forcings.nc ./INPUT/
+fi

tests/fv3_conf/hafs_ww3_run.IN

@@ -1,10 +1,14 @@
 
 if [ ${CNTL_DIR} = "hafs_regional_atm_wav" ] || \
    [ ${CNTL_DIR} = "hafs_regional_atm_ocn_wav" ] || \
+   [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn_wav_inline" ] || \
+   [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn_wav_mom6" ] || \
    [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn_wav" ]; then
   cp @[INPUTDATA_ROOT_WW3]/@[WW3_MODDEF] ./mod_def.ww3
   cp @[INPUTDATA_ROOT_WW3]/@[MESH_WAV] ./mesh.hafs.nc
-  if [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn_wav" ]; then
+  if [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn_wav_inline" ] || \
+     [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn_wav_mom6" ] || \
+     [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn_wav" ]; then
     cp @[INPUTDATA_ROOT]/FV3_hafs_input_data/WW3_hafs_regional_input_data/wind.natl_6m_2020082512 ./wind.ww3
   else
     cp @[INPUTDATA_ROOT_WW3]/wind.natl_6m ./wind.ww3

tests/fv3_conf/hycom_hat10_run.IN

@@ -1,6 +1,7 @@
 
 if [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn" ] || \
    [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn_debug" ] || \
+   [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn_wav_inline" ] || \
    [ ${CNTL_DIR} = "hafs_regional_storm_following_1nest_atm_ocn_wav" ]; then
   rsync -arv @[INPUTDATA_ROOT]/FV3_hafs_input_data/HYCOM_hafs_regional_input_data/. ./
 else