Update for JCB policies and stage DA job files with Jinja2-templates

env/HERCULES.env

@@ -132,6 +132,16 @@ case ${step} in
     [[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max}
     export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalrun} --cpus-per-task=${NTHREADS_OCNANAL}"
  ;;
+"ocnanalecen")
+
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
+    nth_max=$((npe_node_max / npe_node_ocnanalecen))
+
+    export NTHREADS_OCNANALECEN=${nth_ocnanalecen:-${nth_max}}
+    [[ ${NTHREADS_OCNANALECEN} -gt ${nth_max} ]] && export NTHREADS_OCNANALECEN=${nth_max}
+    export APRUN_OCNANALECEN="${launcher} -n ${npe_ocnanalecen} --cpus-per-task=${NTHREADS_OCNANALECEN}"
+;;
  "ocnanalchkpt")
 
     export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"

jobs/rocoto/aeroanlfinal.sh

@@ -11,11 +11,6 @@ status=$?
 export job="aeroanlfinal"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE"

jobs/rocoto/aeroanlinit.sh

@@ -11,12 +11,6 @@ status=$?
 export job="aeroanlinit"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
-
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE"

jobs/rocoto/aeroanlrun.sh

@@ -11,12 +11,6 @@ status=$?
 export job="aeroanlrun"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
-
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_AERO_ANALYSIS_RUN"

jobs/rocoto/atmanlfinal.sh

@@ -11,11 +11,6 @@ status=$?
 export job="atmanlfinal"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FINALIZE"

jobs/rocoto/atmanlfv3inc.sh

@@ -11,12 +11,6 @@ status=$?
 export job="atmanlfv3inc"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
-
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FV3_INCREMENT"

jobs/rocoto/atmanlinit.sh

@@ -11,12 +11,6 @@ status=$?
 export job="atmanlinit"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
-
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE"

jobs/rocoto/atmanlvar.sh

@@ -11,12 +11,6 @@ status=$?
 export job="atmanlvar"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
-
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_VARIATIONAL"

jobs/rocoto/atmensanlfinal.sh

@@ -11,11 +11,6 @@ status=$?
 export job="atmensanlfinal"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE"

jobs/rocoto/atmensanlfv3inc.sh

@@ -11,12 +11,6 @@ status=$?
 export job="atmensanlfv3inc"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
-
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FV3_INCREMENT"

jobs/rocoto/atmensanlinit.sh

@@ -11,12 +11,6 @@ status=$?
 export job="atmensanlinit"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
-
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE"

jobs/rocoto/atmensanlletkf.sh

@@ -11,12 +11,6 @@ status=$?
 export job="atmensanlletkf"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
-
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_LETKF"

jobs/rocoto/oceanice_products.sh

@@ -12,12 +12,6 @@ source "${HOMEgfs}/ush/preamble.sh"
 status=$?
 if (( status != 0 )); then exit "${status}"; fi
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
-
 export job="oceanice_products"
 export jobid="${job}.$$"
 

jobs/rocoto/prepatmiodaobs.sh

@@ -12,11 +12,10 @@ export job="prepatmobs"
 export jobid="${job}.$$"
 
 ###############################################################
-# setup python path for workflow and ioda utilities
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
+# setup python path for ioda utilities
 # shellcheck disable=SC2311
 pyiodaPATH="${HOMEgfs}/sorc/gdas.cd/build/lib/python$(detect_py_ver)/"
-PYTHONPATH="${pyiodaPATH}:${wxflowPATH}:${PYTHONPATH}"
+PYTHONPATH="${pyiodaPATH}:${PYTHONPATH}"
 export PYTHONPATH
 
 ###############################################################

jobs/rocoto/prepobsaero.sh

@@ -11,12 +11,6 @@ status=$?
 export job="prepobsaero"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
-
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_PREP_OBS_AERO"

jobs/rocoto/prepsnowobs.sh

@@ -12,12 +12,11 @@ export job="prepsnowobs"
 export jobid="${job}.$$"
 
 ###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
+# setup python path for ioda utilities
 # shellcheck disable=SC2311
 pyiodaPATH="${HOMEgfs}/sorc/gdas.cd/build/lib/python$(detect_py_ver)/"
 gdasappPATH="${HOMEgfs}/sorc/gdas.cd/sorc/iodaconv/src:${pyiodaPATH}"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}:${gdasappPATH}"
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}:${gdasappPATH}"
 export PYTHONPATH
 
 ###############################################################

jobs/rocoto/snowanl.sh

@@ -11,12 +11,6 @@ status=$?
 export job="snowanl"
 export jobid="${job}.$$"
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
-
 ###############################################################
 # Execute the JJOB
 "${HOMEgfs}/jobs/JGLOBAL_SNOW_ANALYSIS"

jobs/rocoto/upp.sh

@@ -29,18 +29,18 @@ if [[ "${MACHINE_ID}" = "wcoss2" ]]; then
   module load python/3.8.6
   module load crtm/2.4.0  # TODO: This is only needed when UPP_RUN=goes.  Is there a better way to handle this?
   set_trace
+
+  # Add wxflow to PYTHONPATH
+  wxflowPATH="${HOMEgfs}/ush/python"
+  PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/ush:${wxflowPATH}"
+  export PYTHONPATH
+
 else
   . "${HOMEgfs}/ush/load_fv3gfs_modules.sh"
   status=$?
   if (( status != 0 )); then exit "${status}"; fi
 fi
 
-###############################################################
-# setup python path for workflow utilities and tasks
-wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
-export PYTHONPATH
-
 export job="upp"
 export jobid="${job}.$$"
 

parm/config/gfs/config.atmanl

@@ -15,14 +15,17 @@ export INTERP_METHOD='barycentric'
 if [[ ${DOHYBVAR} = "YES" ]]; then
     # shellcheck disable=SC2153
     export CASE_ANL=${CASE_ENS}
-    export BERROR_YAML="background_error_hybrid_${STATICB_TYPE}_${LOCALIZATION_TYPE}"
+    export BERROR_YAML="atmosphere_background_error_hybrid_${STATICB_TYPE}_${LOCALIZATION_TYPE}"
 else
     export CASE_ANL=${CASE}
-    export BERROR_YAML="background_error_static_${STATICB_TYPE}"
+    export BERROR_YAML="atmosphere_background_error_static_${STATICB_TYPE}"
 fi
 
 export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2"
 export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2"
+export VAR_BKG_STAGING_YAML="${PARMgfs}/gdas/staging/atm_var_bkg.yaml.j2"
+export BERROR_STAGING_YAML="${PARMgfs}/gdas/staging/atm_berror_${STATICB_TYPE}.yaml.j2"
+export FV3ENS_STAGING_YAML="${PARMgfs}/gdas/staging/atm_var_fv3ens.yaml.j2"
 
 export layout_x_atmanl=@LAYOUT_X_ATMANL@
 export layout_y_atmanl=@LAYOUT_Y_ATMANL@

parm/config/gfs/config.atmensanl

@@ -12,6 +12,7 @@ export INTERP_METHOD='barycentric'
 
 export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2"
 export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2"
+export LGETKF_BKG_STAGING_YAML="${PARMgfs}/gdas/staging/atm_lgetkf_bkg.yaml.j2"
 
 export layout_x_atmensanl=@LAYOUT_X_ATMENSANL@
 export layout_y_atmensanl=@LAYOUT_Y_ATMENSANL@

parm/gdas/staging/atm_berror_gsibec.yaml.j2

@@ -0,0 +1,8 @@
+{% set fname_list = ['gfs_gsi_global.nml', 'gsi-coeffs-gfs-global.nc4'] %}
+
+mkdir:
+- '{{ DATA }}/berror'
+copy:
+{% for fname in fname_list %}
+- ['{{ HOMEgfs }}/fix/gdas/gsibec/{{ CASE_ANL }}/{{ fname }}', '{{ DATA }}/berror']
+{% endfor %}

parm/gdas/staging/atm_lgetkf_bkg.yaml.j2

@@ -0,0 +1,32 @@
+{% set ftype_list = ['fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data'] %}
+{% set time_list = [current_cycle] %}
+
+mkdir:
+{% for imem in range(1,NMEM_ENS+1) %}
+   {% set memchar = 'mem%03d' | format(imem) %}
+   {% set tmpl_dict = ({ '${ROTDIR}': ROTDIR,
+                         '${RUN}': RUN, 
+                         '${YMD}': current_cycle | to_YMD,	
+                         '${HH}': current_cycle | strftime('%H'),
+                         '${MEMDIR}': memchar }) %}
+- '{{ DATA }}/bkg/{{ memchar }}'
+- '{{ DATA }}/anl/{{ memchar }}'
+- '{{ COM_ATMOS_ANALYSIS_TMPL | replace_tmpl(tmpl_dict) }}'
+{% endfor %}
+copy:
+{% for time in time_list %}
+   {% for imem in range(1,NMEM_ENS+1) %}
+      {% set memchar = 'mem%03d' | format(imem) %}
+      {% set tmpl_dict = ({ '${ROTDIR}': ROTDIR,
+      	      	      	    '${RUN}': 'enkfgdas',
+                            '${YMD}': previous_cycle | to_YMD,	
+                            '${HH}': previous_cycle | strftime('%H'),
+                            '${MEMDIR}': memchar }) %}
+- ['{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ time | to_fv3time }}.coupler.res', '{{ DATA }}/bkg/{{ memchar }}/']
+      {% for ftype in ftype_list %}
+         {% for itile in range(1,7) %}
+- ['{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ time | to_fv3time }}.{{ ftype }}.tile{{ itile }}.nc', '{{ DATA }}/bkg/{{ memchar }}/']
+         {% endfor %}
+      {% endfor %}
+   {% endfor %}
+{% endfor %}

parm/gdas/staging/atm_var_bkg.yaml.j2

@@ -0,0 +1,14 @@
+{% set ftype_list = ['fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data'] %}
+{% set time_list = [current_cycle] %}
+
+mkdir:
+- '{{ DATA }}/bkg'
+copy:
+{% for time in time_list %}
+- ['{{ COM_ATMOS_RESTART_PREV }}/{{ time | to_fv3time }}.coupler.res', '{{ DATA }}/bkg/']
+   {% for ftype in ftype_list %}
+      {% for itile in range(1,ntiles+1) %}
+- ['{{ COM_ATMOS_RESTART_PREV }}/{{ time | to_fv3time }}.{{ ftype }}.tile{{ itile }}.nc', '{{ DATA }}/bkg/']
+      {% endfor %}
+   {% endfor %}
+{% endfor %}

parm/gdas/staging/atm_var_fv3ens.yaml.j2

@@ -0,0 +1,24 @@
+{% set ftype_list = ['fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data'] %}
+{% set time_list = [current_cycle] %}
+
+mkdir:
+{% for imem in range(1,NMEM_ENS+1) %}
+- '{{ DATA }}/ens/{{ 'mem%03d' | format(imem) }}'
+{% endfor %}
+copy:
+{% for time in time_list %}
+   {% for imem in range(1,NMEM_ENS+1) %}
+      {% set memchar = 'mem%03d' | format(imem) %}
+      {% set tmpl_dict = ({ '${ROTDIR}': ROTDIR,
+                            '${RUN}': 'enkfgdas',
+                            '${YMD}': previous_cycle | to_YMD,
+                            '${HH}': previous_cycle | strftime('%H'),
+                            '${MEMDIR}': memchar }) %}
+- ['{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ time | to_fv3time }}.coupler.res', '{{ DATA }}/ens/{{ memchar }}/']
+      {% for ftype in ftype_list %}
+         {% for itile in range(1,ntiles+1) %}
+- ['{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ time | to_fv3time }}.{{ ftype }}.tile{{ itile }}.nc', '{{ DATA }}/ens/{{ memchar }}/']
+         {% endfor %}
+      {% endfor %}
+   {% endfor %}
+{% endfor %}

scripts/exglobal_cleanup.sh

@@ -14,13 +14,13 @@ rm -rf "${DATAROOT}/${RUN}efcs"*"${PDY:-}${cyc}"
 
 # Search and delete files/directories from DATAROOT/ older than ${purge_every_days} days
 # purge_every_days should be a positive integer
-purge_every_days=3
+#purge_every_days=3
 
 # Find and delete files older than ${purge_every_days} days
-find "${DATAROOT}/"* -type f -mtime "+${purge_every_days}" -exec rm -f {} \;
+#find "${DATAROOT}/"* -type f -mtime "+${purge_every_days}" -exec rm -f {} \;
 
 # Find and delete directories older than ${purge_every_days} days
-find "${DATAROOT}/"* -type d -mtime "+${purge_every_days}" -exec rm -rf {} \;
+#find "${DATAROOT}/"* -type d -mtime "+${purge_every_days}" -exec rm -rf {} \;
 
 echo "Cleanup ${DATAROOT} completed!"
 ###############################################################

scripts/exglobal_prep_snow_obs.py

@@ -21,5 +21,5 @@
     # Instantiate the snow prepare task
     SnowAnl = SnowAnalysis(config)
     SnowAnl.prepare_GTS()
-    if f"{ SnowAnl.runtime_config.cyc }" == '18':
+    if f"{ SnowAnl.task_config.cyc }" == '18':
         SnowAnl.prepare_IMS()