Merge branch 'develop' into feature/ocn-letkf

.gitignore

@@ -48,6 +48,8 @@ parm/gdas/io
 parm/gdas/ioda
 parm/gdas/snow
 parm/gdas/soca
+parm/gdas/jcb-gdas
+parm/gdas/jcb-algorithms
 parm/monitor
 parm/post/AEROSOL_LUTS.dat
 parm/post/nam_micro_lookup.dat
@@ -106,6 +108,7 @@ parm/post/ice.csv
 parm/post/ocnicepost.nml.jinja2
 parm/ufs/noahmptable.tbl
 parm/ufs/model_configure.IN
+parm/ufs/model_configure_nest.IN
 parm/ufs/MOM_input_*.IN
 parm/ufs/MOM6_data_table.IN
 parm/ufs/ice_in.IN
@@ -194,3 +197,8 @@ versions/run.ver
 ush/python/wxflow
 workflow/wxflow
 ci/scripts/wxflow
+
+# jcb checkout and symlinks
+ush/python/jcb
+workflow/jcb
+ci/scripts/jcb

.gitmodules

@@ -30,3 +30,7 @@
   path = sorc/upp.fd
   url = https://github.com/NOAA-EMC/UPP.git
   ignore = dirty
+[submodule "sorc/jcb"]
+  path = sorc/jcb
+  url = https://github.com/noaa-emc/jcb
+  fetchRecurseSubmodules = false

ci/cases/pr/C96_atm3DVar.yaml

@@ -15,3 +15,6 @@ arguments:
   gfs_cyc: 1
   start: cold
   yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml
+
+skip_ci_on_hosts:
+  - wcoss2

ci/cases/pr/C96_atm3DVar_extended.yaml

@@ -0,0 +1,22 @@
+experiment:
+  system: gfs
+  mode: cycled
+
+arguments:
+  pslot: {{ 'pslot' | getenv }}
+  app: ATM
+  resdetatmos: 96
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
+  expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
+  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48
+  idate: 2021122018
+  edate: 2021122118
+  nens: 0
+  gfs_cyc: 4
+  start: cold
+  yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_extended_ci.yaml
+
+skip_ci_on_hosts:
+  - hera
+  - orion
+  - hercules

ci/cases/yamls/gfs_extended_ci.yaml

@@ -0,0 +1,12 @@
+defaults:
+  !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml
+
+base:
+  ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }}
+  DO_GOES: "YES"
+  DO_BUFRSND: "YES"
+  DO_GEMPAK: "YES"
+  DO_AWIPS: "NO"
+  DO_NPOESS: "YES"
+  DO_GENESIS_FSU: "NO"
+  FHMAX_GFS: 384

gempak/ush/gfs_meta_opc_na_ver

@@ -33,8 +33,8 @@ fcsthr="f00"
 # seq won't give us any splitting problems, ignore warnings
 # shellcheck disable=SC2207,SC2312
 case ${cyc} in
-    00 | 12) IFS=$'\n' lookbacks=($(seq 6 6 84) $(seq 96 12 120)) ;;
-    06 | 18) IFS=$'\n' lookbacks=($(seq 6 6 84) $(seq 90 12 126)) ;;
+    00 | 12) lookbacks=($(IFS=$'\n' seq 6 6 84) $(IFS=$'\n' seq 96 12 120)) ;;
+    06 | 18) lookbacks=($(IFS=$'\n' seq 6 6 84) $(IFS=$'\n' seq 90 12 126)) ;;
     *)
         echo "FATAL ERROR: Invalid cycle ${cyc} passed to ${BASH_SOURCE[0]}"
         exit 100

gempak/ush/gfs_meta_opc_np_ver

@@ -33,8 +33,8 @@ fcsthr="f00"
 # seq won't give us any splitting problems, ignore warnings
 # shellcheck disable=SC2207,SC2312
 case ${cyc} in
-    00 | 12) IFS=$'\n' lookbacks=($(seq 6 6 84) $(seq 96 12 120)) ;;
-    06 | 18) IFS=$'\n' lookbacks=($(seq 6 6 84) $(seq 90 12 126)) ;;
+    00 | 12) lookbacks=($(IFS=$'\n' seq 6 6 84) $(IFS=$'\n' seq 96 12 120)) ;;
+    06 | 18) lookbacks=($(IFS=$'\n' seq 6 6 84) $(IFS=$'\n' seq 90 12 126)) ;;
     *)
         echo "FATAL ERROR: Invalid cycle ${cyc} passed to ${BASH_SOURCE[0]}"
         exit 100

gempak/ush/gfs_meta_ver.sh

@@ -32,7 +32,7 @@ MDL2="GFSHPC"
 #GENERATING THE METAFILES.
 # seq won't give us any splitting problems, ignore warnings
 # shellcheck disable=SC2207,SC2312
-IFS=$'\n' lookbacks=($(seq 6 6 180) $(seq 192 12 216))
+lookbacks=($(IFS=$'\n' seq 6 6 180) $(IFS=$'\n' seq 192 12 216))
 for lookback in "${lookbacks[@]}"; do
     init_time="$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${lookback} hours")"
     init_PDY=${init_time:0:8}

parm/config/gefs/config.base

@@ -47,9 +47,9 @@ export NOSCRUB="@NOSCRUB@"
 export BASE_GIT="@BASE_GIT@"
 
 # Toggle to turn on/off GFS downstream processing.
-export DO_BUFRSND="NO"     # BUFR sounding products
-export DO_GEMPAK="NO"      # GEMPAK products
-export DO_AWIPS="NO"       # AWIPS products
+export DO_BUFRSND="@DO_BUFRSND@"     # BUFR sounding products
+export DO_GEMPAK="@DO_GEMPAK@"       # GEMPAK products
+export DO_AWIPS="@DO_AWIPS@"         # AWIPS products
 
 # NO for retrospective parallel; YES for real-time parallel
 #  arch.sh uses REALTIME for MOS.  Need to set REALTIME=YES

parm/config/gefs/yaml/defaults.yaml

@@ -4,6 +4,9 @@ base:
   DO_JEDIOCNVAR: "NO"
   DO_JEDISNOWDA: "NO"
   DO_MERGENSST: "NO"
+  DO_BUFRSND: "NO"
+  DO_GEMPAK: "NO"
+  DO_AWIPS: "NO"
   KEEPDATA: "NO" 
   FHMAX_GFS: 120
   USE_OCN_PERTURB_FILES: "false"

parm/config/gfs/config.atmanl

@@ -5,18 +5,20 @@
 
 echo "BEGIN: config.atmanl"
 
-export OBS_LIST="${PARMgfs}/gdas/atm/obs/lists/gdas_prototype_3d.yaml.j2"
-export JEDIYAML="${PARMgfs}/gdas/atm/variational/3dvar_drpcg.yaml.j2"
+export JCB_BASE_YAML="${PARMgfs}/gdas/atm/jcb-base.yaml.j2"
+export JCB_ALGO_YAML="${PARMgfs}/gdas/atm/jcb-prototype_3dvar.yaml.j2"
+
 export STATICB_TYPE="gsibec"
+export LOCALIZATION_TYPE="bump"
 export INTERP_METHOD='barycentric'
 
 if [[ ${DOHYBVAR} = "YES" ]]; then
     # shellcheck disable=SC2153
     export CASE_ANL=${CASE_ENS}
-    export BERROR_YAML="${PARMgfs}/gdas/atm/berror/hybvar_${STATICB_TYPE}.yaml.j2"
+    export BERROR_YAML="background_error_hybrid_${STATICB_TYPE}_${LOCALIZATION_TYPE}"
 else
     export CASE_ANL=${CASE}
-    export BERROR_YAML="${PARMgfs}/gdas/atm/berror/staticb_${STATICB_TYPE}.yaml.j2"
+    export BERROR_YAML="background_error_static_${STATICB_TYPE}"
 fi
 
 export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2"

parm/config/gfs/config.atmanlfv3inc

@@ -8,7 +8,7 @@ echo "BEGIN: config.atmanlfv3inc"
 # Get task specific resources
 . "${EXPDIR}/config.resources" atmanlfv3inc
 
-export JEDIYAML=${PARMgfs}/gdas/atm/utils/fv3jedi_fv3inc_variational.yaml.j2
+export JCB_ALGO=fv3jedi_fv3inc_variational
 export JEDIEXE=${EXECgfs}/fv3jedi_fv3inc.x
 
 echo "END: config.atmanlfv3inc"

parm/config/gfs/config.atmensanl

@@ -5,8 +5,9 @@
 
 echo "BEGIN: config.atmensanl"
 
-export OBS_LIST="${PARMgfs}/gdas/atm/obs/lists/lgetkf_prototype.yaml.j2"
-export JEDIYAML="${PARMgfs}/gdas/atm/lgetkf/lgetkf.yaml.j2"
+export JCB_BASE_YAML="${PARMgfs}/gdas/atm/jcb-base.yaml.j2"
+export JCB_ALGO_YAML="${PARMgfs}/gdas/atm/jcb-prototype_lgetkf.yaml.j2"
+
 export INTERP_METHOD='barycentric'
 
 export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2"

parm/config/gfs/config.base

@@ -65,18 +65,18 @@ export NOSCRUB="@NOSCRUB@"
 export BASE_GIT="@BASE_GIT@"
 
 # Toggle to turn on/off GFS downstream processing.
-export DO_GOES="@DO_GOES@" # GOES products
-export DO_BUFRSND="NO"     # BUFR sounding products
-export DO_GEMPAK="NO"      # GEMPAK products
-export DO_AWIPS="NO"       # AWIPS products
-export DO_NPOESS="NO"      # NPOESS products
-export DO_TRACKER="YES"    # Hurricane track verification
-export DO_GENESIS="YES"    # Cyclone genesis verification
-export DO_GENESIS_FSU="NO" # Cyclone genesis verification (FSU)
-export DO_VERFOZN="YES"    # Ozone data assimilation monitoring
-export DO_VERFRAD="YES"    # Radiance data assimilation monitoring
-export DO_VMINMON="YES"    # GSI minimization monitoring
-export DO_MOS="NO"         # GFS Model Output Statistics - Only supported on WCOSS2
+export DO_GOES="@DO_GOES@"               # GOES products
+export DO_BUFRSND="@DO_BUFRSND@"         # BUFR sounding products
+export DO_GEMPAK="@DO_GEMPAK@"           # GEMPAK products
+export DO_AWIPS="@DO_AWIPS@"             # AWIPS products
+export DO_NPOESS="@DO_NPOESS@"           # NPOESS products
+export DO_TRACKER="@DO_TRACKER@"         # Hurricane track verification
+export DO_GENESIS="@DO_GENESIS@"         # Cyclone genesis verification
+export DO_GENESIS_FSU="@DO_GENESIS_FSU@" # Cyclone genesis verification (FSU)
+export DO_VERFOZN="YES"                  # Ozone data assimilation monitoring
+export DO_VERFRAD="YES"                  # Radiance data assimilation monitoring
+export DO_VMINMON="YES"                  # GSI minimization monitoring
+export DO_MOS="NO"                       # GFS Model Output Statistics - Only supported on WCOSS2
 
 # NO for retrospective parallel; YES for real-time parallel
 #  arch.sh uses REALTIME for MOS.  Need to set REALTIME=YES

parm/config/gfs/yaml/defaults.yaml

@@ -6,6 +6,13 @@ base:
   DO_JEDISNOWDA: "NO"
   DO_MERGENSST: "NO"
   DO_GOES: "NO"
+  DO_BUFRSND: "NO"
+  DO_GEMPAK: "NO"
+  DO_AWIPS: "NO"
+  DO_NPOESS: "NO"
+  DO_TRACKER: "YES"
+  DO_GENESIS: "YES"
+  DO_GENESIS_FSU: "NO"
   FHMAX_GFS: 120
   DO_VRFY_OCEANDA: "NO"
   GSI_SOILANAL: "NO"

scripts/exgfs_atmos_grib2_special_npoess.sh

@@ -153,7 +153,8 @@ for (( fhr=SHOUR; fhr <= FHOUR; fhr = fhr + FHINC )); do
    # existence of the restart files
    ###############################
    export pgm="postcheck"
-   grib_file="${COM_ATMOS_MASTER}/${RUN}.t${cyc}z.goesmasterf${fhr3}.grb2"
+   # grib_file="${COM_ATMOS_MASTER}/${RUN}.t${cyc}z.goesmasterf${fhr3}.grb2"
+   grib_file="${COM_ATMOS_MASTER}/${RUN}.t${cyc}z.special.grb2f${fhr3}"
    if ! wait_for_file "${grib_file}" "${SLEEP_INT}" "${SLEEP_LOOP_MAX}"; then
       echo "FATAL ERROR: GOES master grib file ${grib_file} not available after max sleep time"
       export err=9

sorc/link_workflow.sh

@@ -88,19 +88,24 @@ if [[ "${LINK_NEST:-OFF}" == "ON" ]] ; then
   source "${HOMEgfs}/versions/fix.nest.ver"
 fi
 
-# Link wxflow in ush/python, workflow and ci/scripts
+# Link python pacakges in ush/python
+# TODO: This will be unnecessary when these are part of the virtualenv
+packages=("wxflow" "jcb")
+for package in "${packages[@]}"; do
+    cd "${HOMEgfs}/ush/python" || exit 1
+    [[ -s "${package}" ]] && rm -f "${package}"
+    ${LINK} "${HOMEgfs}/sorc/${package}/src/${package}" .
+done
+
+# Link wxflow in workflow and ci/scripts
 # TODO: This will be unnecessary when wxflow is part of the virtualenv
-cd "${HOMEgfs}/ush/python" || exit 1
-[[ -s "wxflow" ]] && rm -f wxflow
-${LINK} "${HOMEgfs}/sorc/wxflow/src/wxflow" .
 cd "${HOMEgfs}/workflow" || exit 1
 [[ -s "wxflow" ]] && rm -f wxflow
 ${LINK} "${HOMEgfs}/sorc/wxflow/src/wxflow" .
 cd "${HOMEgfs}/ci/scripts" || exit 1
 [[ -s "wxflow" ]] && rm -f wxflow
 ${LINK} "${HOMEgfs}/sorc/wxflow/src/wxflow" .
 
-
 # Link fix directories
 if [[ -n "${FIX_DIR}" ]]; then
   if [[ ! -d "${HOMEgfs}/fix" ]]; then mkdir "${HOMEgfs}/fix" || exit 1; fi
@@ -228,7 +233,7 @@ fi
 #------------------------------
 if [[ -d "${HOMEgfs}/sorc/gdas.cd" ]]; then
   cd "${HOMEgfs}/parm/gdas" || exit 1
-  declare -a gdasapp_comps=("aero" "atm" "io" "ioda" "snow" "soca")
+  declare -a gdasapp_comps=("aero" "atm" "io" "ioda" "snow" "soca" "jcb-gdas" "jcb-algorithms")
   for comp in "${gdasapp_comps[@]}"; do
     [[ -d "${comp}" ]] && rm -rf "${comp}"
     ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/parm/${comp}" .

ush/python/pygfs/task/analysis.py

@@ -6,8 +6,9 @@
 from logging import getLogger
 from pprint import pformat
 from netCDF4 import Dataset
-from typing import List, Dict, Any, Union
+from typing import List, Dict, Any, Union, Optional
 
+from jcb import render
 from wxflow import (parse_j2yaml, FileHandler, rm_p, logit,
                     Task, Executable, WorkflowException, to_fv3time, to_YMD,
                     Template, TemplateConstants)
@@ -46,11 +47,14 @@ def initialize(self) -> None:
         self.link_jediexe()
 
     @logit(logger)
-    def get_jedi_config(self) -> Dict[str, Any]:
+    def get_jedi_config(self, algorithm: Optional[str] = None) -> Dict[str, Any]:
         """Compile a dictionary of JEDI configuration from JEDIYAML template file
 
         Parameters
         ----------
+        algorithm (optional) : str
+            Name of the algorithm to use in the JEDI configuration. Will override the algorithm
+            set in the self.config.JCB_<>_YAML file
 
         Returns
         ----------
@@ -60,7 +64,31 @@ def get_jedi_config(self) -> Dict[str, Any]:
 
         # generate JEDI YAML file
         logger.info(f"Generate JEDI YAML config: {self.task_config.jedi_yaml}")
-        jedi_config = parse_j2yaml(self.task_config.JEDIYAML, self.task_config, searchpath=self.gdasapp_j2tmpl_dir)
+
+        if 'JCB_BASE_YAML' in self.task_config.keys():
+            # Step 1: fill templates of the jcb base YAML file
+            jcb_config = parse_j2yaml(self.task_config.JCB_BASE_YAML, self.task_config)
+
+            # Step 2: (optional) fill templates of algorithm override YAML and merge
+            if 'JCB_ALGO_YAML' in self.task_config.keys():
+                jcb_algo_config = parse_j2yaml(self.task_config.JCB_ALGO_YAML, self.task_config)
+                jcb_config = {**jcb_config, **jcb_algo_config}
+
+            # If algorithm is present override the algorithm in the JEDI config
+            if algorithm:
+                jcb_config['algorithm'] = algorithm
+
+            # Step 3: generate the JEDI Yaml using JCB driving YAML
+            jedi_config = render(jcb_config)
+        elif 'JEDIYAML' in self.task_config.keys():
+            # Generate JEDI YAML file (without using JCB)
+            logger.info(f"Generate JEDI YAML config: {self.task_config.jedi_yaml}")
+            jedi_config = parse_j2yaml(self.task_config.JEDIYAML, self.task_config,
+                                       searchpath=self.gdasapp_j2tmpl_dir)
+            logger.debug(f"JEDI config:\n{pformat(jedi_config)}")
+        else:
+            raise KeyError(f"Task config must contain JCB_BASE_YAML or JEDIYAML")
+
         logger.debug(f"JEDI config:\n{pformat(jedi_config)}")
 
         return jedi_config
@@ -82,7 +110,7 @@ def get_obs_dict(self) -> Dict[str, Any]:
             a dictionary containing the list of observation files to copy for FileHandler
         """
 
-        logger.info(f"Extracting a list of observation files from {self.task_config.JEDIYAML}")
+        logger.info(f"Extracting a list of observation files from Jedi config file")
         observations = find_value_in_nested_dict(self.task_config.jedi_config, 'observations')
         logger.debug(f"observations:\n{pformat(observations)}")
 
@@ -116,7 +144,7 @@ def get_bias_dict(self) -> Dict[str, Any]:
             a dictionary containing the list of observation bias files to copy for FileHandler
         """
 
-        logger.info(f"Extracting a list of bias correction files from {self.task_config.JEDIYAML}")
+        logger.info(f"Extracting a list of bias correction files from Jedi config file")
         observations = find_value_in_nested_dict(self.task_config.jedi_config, 'observations')
         logger.debug(f"observations:\n{pformat(observations)}")
 

ush/python/pygfs/task/atm_analysis.py

@@ -49,6 +49,9 @@ def __init__(self, config):
                 'APREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.",  # TODO: CDUMP is being replaced by RUN
                 'GPREFIX': f"gdas.t{self.runtime_config.previous_cycle.hour:02d}z.",
                 'jedi_yaml': _jedi_yaml,
+                'atm_obsdatain_path': f"{self.runtime_config.DATA}/obs/",
+                'atm_obsdataout_path': f"{self.runtime_config.DATA}/diags/",
+                'BKG_TSTEP': "PT1H"  # Placeholder for 4D applications
             }
         )
 
@@ -137,8 +140,9 @@ def variational(self: Analysis) -> None:
     @logit(logger)
     def init_fv3_increment(self: Analysis) -> None:
         # Setup JEDI YAML file
-        self.task_config.jedi_yaml = os.path.join(self.runtime_config.DATA, os.path.basename(self.task_config.JEDIYAML))
-        save_as_yaml(self.get_jedi_config(), self.task_config.jedi_yaml)
+        self.task_config.jedi_yaml = os.path.join(self.runtime_config.DATA,
+                                                  f"{self.task_config.JCB_ALGO}.yaml")
+        save_as_yaml(self.get_jedi_config(self.task_config.JCB_ALGO), self.task_config.jedi_yaml)
 
         # Link JEDI executable to run directory
         self.task_config.jedi_exe = self.link_jediexe()

ush/python/pygfs/task/atmens_analysis.py

@@ -46,6 +46,9 @@ def __init__(self, config):
                 'APREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.",  # TODO: CDUMP is being replaced by RUN
                 'GPREFIX': f"gdas.t{self.runtime_config.previous_cycle.hour:02d}z.",
                 'jedi_yaml': _jedi_yaml,
+                'atm_obsdatain_path': f"./obs/",
+                'atm_obsdataout_path': f"./diags/",
+                'BKG_TSTEP': "PT1H"  # Placeholder for 4D applications
             }
         )
 

versions/run.hera.ver

@@ -4,7 +4,7 @@ export spack_env=gsi-addon-dev-rocky8
 
 export hpss_ver=hpss
 export ncl_ver=6.6.2
-export R_ver=3.5.0
+export R_ver=3.6.1
 export gempak_ver=7.17.0
 export perl_ver=5.38.0