Merge remote-tracking branch 'emc/develop' into feature/simplify_res

* emc/develop:
  Replace Jinja namespaces with replace_tmpl filter and disable ACCOUNT_SERVICE option (NOAA-EMC#2775)
  Add task to process reforecast variables to save on WCOSS2 (NOAA-EMC#2680)

.github/workflows/ci_unit_tests.yaml

@@ -47,7 +47,7 @@ jobs:
         run: |
           sudo mkdir -p /scratch1/NCEPDEV
           cd $GITHUB_WORKSPACE/sorc
-          git submodule update --init --recursive
+          git submodule update --init
           ./link_workflow.sh
           cd $GITHUB_WORKSPACE/ci/scripts/tests
           ln -s ../wxflow

jobs/JGLOBAL_EXTRACTVARS

@@ -0,0 +1,47 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "extractvars" -c "base extractvars"
+
+# Set COM Paths
+for grid in '0p25' '0p50' '1p00'; do
+  prod_dir="COMIN_ATMOS_GRIB_${grid}"
+  GRID=${grid} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx "${prod_dir}:COM_ATMOS_GRIB_GRID_TMPL"
+  if [[ ! -d "${!prod_dir}" ]]; then mkdir -p "${!prod_dir}"; fi
+done
+
+YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx \
+    "COMIN_OCEAN_HISTORY:COM_OCEAN_HISTORY_TMPL" \
+    "COMIN_OCEAN_GRIB:COM_OCEAN_GRIB_TMPL" \
+    "COMIN_OCEAN_NETCDF:COM_OCEAN_NETCDF_TMPL" \
+    "COMIN_ICE_HISTORY:COM_ICE_HISTORY_TMPL" \
+    "COMIN_ICE_GRIB:COM_ICE_GRIB_TMPL" \
+    "COMIN_ICE_NETCDF:COM_ICE_NETCDF_TMPL" \
+    "COMIN_WAVE_GRID:COM_WAVE_GRID_TMPL"
+
+if [[ "${DO_ATM}" == "YES" ]]; then
+  if [[ ! -d "${ARC_RFCST_PROD_ATMOS_F2D}" ]]; then mkdir -p "${ARC_RFCST_PROD_ATMOS_F2D}"; fi
+  if [[ ! -d "${ARC_RFCST_PROD_ATMOS_F3D}" ]]; then mkdir -p "${ARC_RFCST_PROD_ATMOS_F3D}"; fi
+fi
+if [[ "${DO_OCN}" == "YES" ]]; then
+  if [[ ! -d "${ARC_RFCST_PROD_OCN}" ]]; then mkdir -p "${ARC_RFCST_PROD_OCN}"; fi
+fi
+if [[ "${DO_ICE}" == "YES" ]]; then
+  if [[ ! -d "${ARC_RFCST_PROD_ICE}" ]]; then mkdir -p "${ARC_RFCST_PROD_ICE}"; fi
+fi
+if [[ "${DO_WAVE}" == "YES" ]]; then
+  if [[ ! -d "${ARC_RFCST_PROD_WAV}" ]]; then mkdir -p "${ARC_RFCST_PROD_WAV}"; fi
+fi
+
+# Execute the Script
+"${SCRgfs}/exglobal_extractvars.sh"
+status=$?
+(( status != 0 )) && exit "${status}"
+
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd "${DATAROOT}" || true
+[[ "${KEEPDATA}" = "NO" ]] && rm -rf "${DATA}"
+
+exit 0

jobs/rocoto/extractvars.sh

@@ -0,0 +1,23 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+echo
+echo "=============== START TO SOURCE FV3GFS WORKFLOW MODULES ==============="
+. "${HOMEgfs}/ush/load_fv3gfs_modules.sh"
+status=$?
+[[ "${status}" -ne 0 ]] && exit "${status}"
+
+export job="extractvars"
+export jobid="${job}.$$"
+
+###############################################################
+echo
+echo "=============== START TO RUN EXTRACTVARS ==============="
+# Execute the JJOB
+"${HOMEgfs}/jobs/JGLOBAL_EXTRACTVARS"
+status=$?
+[[ "${status}" -ne 0 ]] && exit "${status}"
+
+exit 0

parm/archive/master_enkf.yaml.j2

@@ -52,35 +52,21 @@ datasets:
     {% for mem in range(first_group_mem, last_group_mem + 1) %}
 
         # Declare a dict of search and replace terms to run on each template
-        {% set tmpl_dict = {'ROTDIR':ROTDIR,
-                            'RUN':RUN,
-                            'YMD':cycle_YMD,
-                            'HH':cycle_HH,
-                            'MEMDIR':"mem" + '%03d' % mem} %}
-
-        # Replace template variables with tmpl_dict, one key at a time
-        # This must be done in a namespace to overcome jinja scoping
-        #  Variables set inside of a for loop are lost at the end of the loop
-        #  unless they are part of a namespace
-        {% set com_ns = namespace(COMIN_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_TMPL,
-                                  COMIN_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_TMPL,
-                                  COMIN_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_TMPL) %}
-
-        {% for key in tmpl_dict.keys() %}
-            {% set search_term = '${' + key + '}' %}
-            {% set replace_term = tmpl_dict[key] %}
-            {% set com_ns.COMIN_ATMOS_ANALYSIS_MEM =
-                com_ns.COMIN_ATMOS_ANALYSIS_MEM.replace(search_term, replace_term) %}
-            {% set com_ns.COMIN_ATMOS_HISTORY_MEM =
-                com_ns.COMIN_ATMOS_HISTORY_MEM.replace(search_term, replace_term) %}
-            {% set com_ns.COMIN_ATMOS_RESTART_MEM =
-                com_ns.COMIN_ATMOS_RESTART_MEM.replace(search_term, replace_term) %}
-        {% endfor %}
+        {% set mem_char = 'mem%03d' | format(mem) %}
+        {% set tmpl_dict = ({ '${ROTDIR}':ROTDIR,
+                              '${RUN}':RUN,
+                              '${YMD}':cycle_YMD,
+                              '${HH}':cycle_HH,
+                              '${MEMDIR}': mem_char }) %}
+
+        {% set COMIN_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_TMPL | replace_tmpl(tmpl_dict) %}
+        {% set COMIN_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_TMPL | replace_tmpl(tmpl_dict) %}
+        {% set COMIN_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) %}
 
         # Append the member COM directories
-        {% do COMIN_ATMOS_ANALYSIS_MEM_list.append(com_ns.COMIN_ATMOS_ANALYSIS_MEM)%}
-        {% do COMIN_ATMOS_HISTORY_MEM_list.append(com_ns.COMIN_ATMOS_HISTORY_MEM)%}
-        {% do COMIN_ATMOS_RESTART_MEM_list.append(com_ns.COMIN_ATMOS_RESTART_MEM)%}
+        {% do COMIN_ATMOS_ANALYSIS_MEM_list.append(COMIN_ATMOS_ANALYSIS_MEM)%}
+        {% do COMIN_ATMOS_HISTORY_MEM_list.append(COMIN_ATMOS_HISTORY_MEM)%}
+        {% do COMIN_ATMOS_RESTART_MEM_list.append(COMIN_ATMOS_RESTART_MEM)%}
 
     {% endfor %}
 

parm/config/gefs/config.base

@@ -134,6 +134,7 @@ export DO_WAVE="NO"
 export DO_OCN="NO"
 export DO_ICE="NO"
 export DO_AERO="NO"
+export DO_EXTRACTVARS="@DO_EXTRACTVARS@" # Option to process and extract a subset of products to save on disk
 export AERO_FCST_RUN="" # When to run aerosol forecast: gdas, gfs, or both
 export AERO_ANL_RUN="" # When to run aerosol analysis: gdas, gfs, or both
 export WAVE_RUN="" # When to include wave suite: gdas, gfs, or both

parm/config/gefs/config.extractvars

@@ -0,0 +1,41 @@
+#! /usr/bin/env bash                                                                                                                                                                                                                     
+
+########## config.extractvars ##########
+# Extractvars specific
+
+echo "BEGIN: config.extractvars"
+
+. "${EXPDIR}/config.resources" extractvars
+
+export COMPRSCMD=${COMPRSCMD:-bzip2}
+
+export compress_ocn=0 #1: Compress extracted ocean product, 0: Do not compress extracted ocean product
+export compress_ice=0 #1: Compress extracted ice product, 0: Do not compress extracted ice product
+
+export ocnres="5p00" # Resolution of ocean products
+export iceres="5p00" # Resolution of ice products
+export wavres="5p00" # Resolution of wave products
+
+export depthvar_name="z_l" # Name of depth variable in NetCDF ocean products
+export zmin="0." # Minimum depth to extract from NetCDF ocean products
+export zmax="300." # Maximum depth to extract from NetCDF ocean products
+
+export FHOUT_WAV_EXTRACT=6 # Frequency of wave output to be saved on disk
+
+#Paramater Tables used
+export varlist_2d="${PARMgfs}/product/gefs_shortparmlist_2d.parm" # Parameter table for surface variables
+export varlist_3d="${PARMgfs}/product/gefs_shortparmlist_3d_h.parm" # Parameter table for upper air instantaneous variables
+export varlist_3d_d="${PARMgfs}/product/gefs_shortparmlist_3d_d.parm" # Parameter table for upper air daily-averaged variables
+export varlist_wav="${PARMgfs}/product/gefs_wav_shortparmlist.parm" # Parameter table for wave variables
+export varlist_ocn_netcdf="${PARMgfs}/product/gefs_ocn_shortparmlist.parm" # Parameter table for ocean netcdf variables
+export varlist_ice_netcdf="${PARMgfs}/product/gefs_ice_shortparmlist.parm" # Parameter table for ice netcdf variables
+
+#Directory to save extracted variables
+export ARC_RFCST_PROD="${ARCDIR}/rfcst/${PDY:0:4}/${PDY:0:6}/${PDY:0:8}/mem${ENSMEM}"
+export ARC_RFCST_PROD_ATMOS_F2D="${ARC_RFCST_PROD}/atmos/f2d"
+export ARC_RFCST_PROD_ATMOS_F3D="${ARC_RFCST_PROD}/atmos/f3d"
+export ARC_RFCST_PROD_OCN="${ARC_RFCST_PROD}/ocn"
+export ARC_RFCST_PROD_ICE="${ARC_RFCST_PROD}/ice"
+export ARC_RFCST_PROD_WAV="${ARC_RFCST_PROD}/wav"
+
+echo "END: config.extractvars"

parm/config/gefs/config.resources

@@ -279,6 +279,18 @@ case ${step} in
     export NTASKS=${ntasks}
     ;;
 
+  "extractvars")
+    export walltime_gefs="00:30:00"
+    export ntasks_gefs=1
+    export threads_per_task_gefs=1
+    export tasks_per_node_gefs="${ntasks}"
+    export walltime_gfs="${walltime_gefs}"
+    export ntasks_gfs="${ntasks_gefs}"
+    export threads_per_tasks_gfs="${threads_per_task_gefs}"
+    export tasks_per_node_gfs="${tasks_per_node_gefs}"
+    export is_exclusive=False
+    ;;
+
   *)
     echo "FATAL ERROR: Invalid job ${step} passed to ${BASH_SOURCE[0]}"
     exit 1

parm/config/gefs/yaml/defaults.yaml

@@ -7,7 +7,8 @@ base:
   DO_BUFRSND: "NO"
   DO_GEMPAK: "NO"
   DO_AWIPS: "NO"
-  KEEPDATA: "NO" 
+  KEEPDATA: "NO"
+  DO_EXTRACTVARS: "NO"
   FHMAX_GFS: 120
   FHMAX_HF_GFS: 0
   REPLAY_ICS: "NO"

parm/product/gefs_ice_shortparmlist.parm

@@ -0,0 +1,10 @@
+aice_h
+hi_h
+Tsfc_h
+uvel_h
+vvel_h
+hs_h
+albsni_h
+melts_h
+meltb_h
+frzmlt_h

parm/product/gefs_ocn_shortparmlist.parm

@@ -0,0 +1,9 @@
+temp
+SST
+SSH
+SSS
+MLD_003
+taux
+tauy
+SSU
+SSV

parm/product/gefs_shortparmlist_2d.parm

@@ -0,0 +1,38 @@
+:PRES:surface:
+:WEASD:surface:
+:TMP:2 m above ground:
+:TMP:surface:
+:RH:2 m above ground:
+:TMAX:2 m above ground:
+:TMIN:2 m above ground:
+:UGRD:10 m above ground:
+:VGRD:10 m above ground:
+:APCP:surface:
+:CSNOW:surface:
+:CICEP:surface:
+:CFRZR:surface:
+:CRAIN:surface:
+:PWAT:entire atmosphere (considered as a single layer):
+:TCDC:entire atmosphere (considered as a single layer):
+:DSWRF:surface:
+:DLWRF:surface:
+:ULWRF:top of atmosphere:
+:HLCY:3000-0 m above ground:
+:CAPE:180-0 mb above ground:
+:CIN:180-0 mb above ground:
+:PRMSL:mean sea level:
+:USWRF:surface:
+:ULWRF:surface:
+:TSOIL:0-0.1 m below ground:
+:TSOIL:0.1-0.4 m below ground:
+:SOILW:0-0.1 m below ground:
+:SOILW:0.1-0.4 m below ground:
+:SOILW:0.4-1 m below ground:
+:SOILW:1-2 m below ground:
+:PEVPR:surface:
+:LHTFL:surface:
+:SHTFL:surface:
+:WATR:surface:
+:TSNOWP:surface:
+:FDNSSTMP:surface:
+:HGT:highest tropospheric freezing level:

parm/product/gefs_shortparmlist_3d_d.parm

@@ -0,0 +1,34 @@
+:UGRD:1 mb:
+:UGRD:2 mb:
+:UGRD:3 mb:
+:UGRD:5 mb:
+:UGRD:7 mb:
+:UGRD:10 mb:
+:UGRD:20 mb:
+:UGRD:30 mb:
+:UGRD:50 mb:
+:UGRD:70 mb:
+:VGRD:1 mb:
+:VGRD:2 mb:
+:VGRD:3 mb:
+:VGRD:5 mb:
+:VGRD:7 mb:
+:VGRD:10 mb:
+:VGRD:20 mb:
+:VGRD:30 mb:
+:VGRD:50 mb:
+:VGRD:70 mb:
+:TMP:1 mb:
+:TMP:2 mb:
+:TMP:3 mb:
+:TMP:5 mb:
+:TMP:7 mb:
+:TMP:10 mb:
+:TMP:20 mb:
+:TMP:30 mb:
+:TMP:50 mb:
+:TMP:70 mb:
+:HGT:10 mb:
+:HGT:50 mb:
+:O3MR:10 mb:
+:O3MR:50 mb:

parm/product/gefs_shortparmlist_3d_h.parm

@@ -0,0 +1,45 @@
+:HGT:100 mb:
+:TMP:100 mb:
+:UGRD:100 mb:
+:VGRD:100 mb:
+:O3MR:100 mb:
+:HGT:200 mb:
+:TMP:200 mb:
+:RH:200 mb:
+:UGRD:200 mb:
+:VGRD:200 mb:
+:HGT:250 mb:
+:TMP:250 mb:
+:RH:250 mb:
+:UGRD:250 mb:
+:VGRD:250 mb:
+:HGT:500 mb:
+:TMP:500 mb:
+:RH:500 mb:
+:UGRD:500 mb:
+:VGRD:500 mb:
+:HGT:700 mb:
+:TMP:700 mb:
+:RH:700 mb:
+:UGRD:700 mb:
+:VGRD:700 mb:
+:HGT:850 mb:
+:TMP:850 mb:
+:RH:850 mb:
+:VVEL:850 mb:
+:UGRD:850 mb:
+:VGRD:850 mb:
+:HGT:925 mb:
+:TMP:925 mb:
+:RH:925 mb:
+:UGRD:925 mb:
+:VGRD:925 mb:
+:TMP:1000 mb:
+:RH:1000 mb:
+:UGRD:1000 mb:
+:VGRD:1000 mb:
+:HGT:1000 mb:
+:TMP:0.995 sigma level:
+:RH:0.995 sigma level:
+:UGRD:0.995 sigma level:
+:VGRD:0.995 sigma level:

parm/product/gefs_wav_shortparmlist.parm

@@ -0,0 +1,3 @@
+:UGRD:surface:
+:VGRD:surface:
+:HTSGW:surface: