Add CI test for products (NOAA-EMC#2567)

Adds a new version of the atm3DVar test that runs the full forecast
length and produces most of the secondary products. For now, this test
will only run on WCOSS due to gempak failures on other machines as well
as computational needs. On other machines, the original version will run
(the original version will not run on WCOSS). AWIPS remains off for now
in this extended test due to a bug involving tocgrib2 and the convective
precip fields (see NOAA-EMC#2566).

The new test runs for 4½ cycles and the full 384-hr forecast length to
ensure all gempak scripts are exercised. Since the cycle throttle is 3
and the bulk of the time is in the free forecast, the cycles run mostly
concurrently so it doesn't extend the total test time too much beyond
that of a single 384-hr forecast.

Fixes a bug in NPOESS that was introduced when the post filenames were
reverted to the previous format for the GOES products until the final
filenames are determined (NOAA-EMC#2499).

Also removes the AWIPS g2 job from the rocoto mesh to complete the
retirement of grib1 products.

Resolves NOAA-EMC#2132
Resolves NOAA-EMC#2445

.gitignore

@@ -106,6 +106,7 @@ parm/post/ice.csv
 parm/post/ocnicepost.nml.jinja2
 parm/ufs/noahmptable.tbl
 parm/ufs/model_configure.IN
+parm/ufs/model_configure_nest.IN
 parm/ufs/MOM_input_*.IN
 parm/ufs/MOM6_data_table.IN
 parm/ufs/ice_in.IN

ci/cases/pr/C96_atm3DVar.yaml

@@ -15,3 +15,6 @@ arguments:
   gfs_cyc: 1
   start: cold
   yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml
+
+skip_ci_on_hosts:
+  - wcoss2

ci/cases/pr/C96_atm3DVar_extended.yaml

@@ -0,0 +1,22 @@
+experiment:
+  system: gfs
+  mode: cycled
+
+arguments:
+  pslot: {{ 'pslot' | getenv }}
+  app: ATM
+  resdetatmos: 96
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
+  expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
+  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48
+  idate: 2021122018
+  edate: 2021122118
+  nens: 0
+  gfs_cyc: 4
+  start: cold
+  yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_extended_ci.yaml
+
+skip_ci_on_hosts:
+  - hera
+  - orion
+  - hercules

ci/cases/yamls/gfs_extended_ci.yaml

@@ -0,0 +1,12 @@
+defaults:
+  !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml
+
+base:
+  ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }}
+  DO_GOES: "YES"
+  DO_BUFRSND: "YES"
+  DO_GEMPAK: "YES"
+  DO_AWIPS: "NO"
+  DO_NPOESS: "YES"
+  DO_GENESIS_FSU: "NO"
+  FHMAX_GFS: 384

gempak/ush/gfs_meta_opc_na_ver

@@ -33,8 +33,8 @@ fcsthr="f00"
 # seq won't give us any splitting problems, ignore warnings
 # shellcheck disable=SC2207,SC2312
 case ${cyc} in
-    00 | 12) IFS=$'\n' lookbacks=($(seq 6 6 84) $(seq 96 12 120)) ;;
-    06 | 18) IFS=$'\n' lookbacks=($(seq 6 6 84) $(seq 90 12 126)) ;;
+    00 | 12) lookbacks=($(IFS=$'\n' seq 6 6 84) $(IFS=$'\n' seq 96 12 120)) ;;
+    06 | 18) lookbacks=($(IFS=$'\n' seq 6 6 84) $(IFS=$'\n' seq 90 12 126)) ;;
     *)
         echo "FATAL ERROR: Invalid cycle ${cyc} passed to ${BASH_SOURCE[0]}"
         exit 100

gempak/ush/gfs_meta_opc_np_ver

@@ -33,8 +33,8 @@ fcsthr="f00"
 # seq won't give us any splitting problems, ignore warnings
 # shellcheck disable=SC2207,SC2312
 case ${cyc} in
-    00 | 12) IFS=$'\n' lookbacks=($(seq 6 6 84) $(seq 96 12 120)) ;;
-    06 | 18) IFS=$'\n' lookbacks=($(seq 6 6 84) $(seq 90 12 126)) ;;
+    00 | 12) lookbacks=($(IFS=$'\n' seq 6 6 84) $(IFS=$'\n' seq 96 12 120)) ;;
+    06 | 18) lookbacks=($(IFS=$'\n' seq 6 6 84) $(IFS=$'\n' seq 90 12 126)) ;;
     *)
         echo "FATAL ERROR: Invalid cycle ${cyc} passed to ${BASH_SOURCE[0]}"
         exit 100

gempak/ush/gfs_meta_ver.sh

@@ -32,7 +32,7 @@ MDL2="GFSHPC"
 #GENERATING THE METAFILES.
 # seq won't give us any splitting problems, ignore warnings
 # shellcheck disable=SC2207,SC2312
-IFS=$'\n' lookbacks=($(seq 6 6 180) $(seq 192 12 216))
+lookbacks=($(IFS=$'\n' seq 6 6 180) $(IFS=$'\n' seq 192 12 216))
 for lookback in "${lookbacks[@]}"; do
     init_time="$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${lookback} hours")"
     init_PDY=${init_time:0:8}

parm/config/gefs/config.base

@@ -47,9 +47,9 @@ export NOSCRUB="@NOSCRUB@"
 export BASE_GIT="@BASE_GIT@"
 
 # Toggle to turn on/off GFS downstream processing.
-export DO_BUFRSND="NO"     # BUFR sounding products
-export DO_GEMPAK="NO"      # GEMPAK products
-export DO_AWIPS="NO"       # AWIPS products
+export DO_BUFRSND="@DO_BUFRSND@"     # BUFR sounding products
+export DO_GEMPAK="@DO_GEMPAK@"       # GEMPAK products
+export DO_AWIPS="@DO_AWIPS@"         # AWIPS products
 
 # NO for retrospective parallel; YES for real-time parallel
 #  arch.sh uses REALTIME for MOS.  Need to set REALTIME=YES

parm/config/gefs/yaml/defaults.yaml

@@ -4,6 +4,9 @@ base:
   DO_JEDIOCNVAR: "NO"
   DO_JEDISNOWDA: "NO"
   DO_MERGENSST: "NO"
+  DO_BUFRSND: "NO"
+  DO_GEMPAK: "NO"
+  DO_AWIPS: "NO"
   KEEPDATA: "NO" 
   FHMAX_GFS: 120
   USE_OCN_PERTURB_FILES: "false"

parm/config/gfs/config.base

@@ -65,18 +65,18 @@ export NOSCRUB="@NOSCRUB@"
 export BASE_GIT="@BASE_GIT@"
 
 # Toggle to turn on/off GFS downstream processing.
-export DO_GOES="@DO_GOES@" # GOES products
-export DO_BUFRSND="NO"     # BUFR sounding products
-export DO_GEMPAK="NO"      # GEMPAK products
-export DO_AWIPS="NO"       # AWIPS products
-export DO_NPOESS="NO"      # NPOESS products
-export DO_TRACKER="YES"    # Hurricane track verification
-export DO_GENESIS="YES"    # Cyclone genesis verification
-export DO_GENESIS_FSU="NO" # Cyclone genesis verification (FSU)
-export DO_VERFOZN="YES"    # Ozone data assimilation monitoring
-export DO_VERFRAD="YES"    # Radiance data assimilation monitoring
-export DO_VMINMON="YES"    # GSI minimization monitoring
-export DO_MOS="NO"         # GFS Model Output Statistics - Only supported on WCOSS2
+export DO_GOES="@DO_GOES@"               # GOES products
+export DO_BUFRSND="@DO_BUFRSND@"         # BUFR sounding products
+export DO_GEMPAK="@DO_GEMPAK@"           # GEMPAK products
+export DO_AWIPS="@DO_AWIPS@"             # AWIPS products
+export DO_NPOESS="@DO_NPOESS@"           # NPOESS products
+export DO_TRACKER="@DO_TRACKER@"         # Hurricane track verification
+export DO_GENESIS="@DO_GENESIS@"         # Cyclone genesis verification
+export DO_GENESIS_FSU="@DO_GENESIS_FSU@" # Cyclone genesis verification (FSU)
+export DO_VERFOZN="YES"                  # Ozone data assimilation monitoring
+export DO_VERFRAD="YES"                  # Radiance data assimilation monitoring
+export DO_VMINMON="YES"                  # GSI minimization monitoring
+export DO_MOS="NO"                       # GFS Model Output Statistics - Only supported on WCOSS2
 
 # NO for retrospective parallel; YES for real-time parallel
 #  arch.sh uses REALTIME for MOS.  Need to set REALTIME=YES

parm/config/gfs/yaml/defaults.yaml

@@ -6,6 +6,13 @@ base:
   DO_JEDISNOWDA: "NO"
   DO_MERGENSST: "NO"
   DO_GOES: "NO"
+  DO_BUFRSND: "NO"
+  DO_GEMPAK: "NO"
+  DO_AWIPS: "NO"
+  DO_NPOESS: "NO"
+  DO_TRACKER: "YES"
+  DO_GENESIS: "YES"
+  DO_GENESIS_FSU: "NO"
   FHMAX_GFS: 120
   DO_VRFY_OCEANDA: "NO"
   GSI_SOILANAL: "NO"

scripts/exgfs_atmos_grib2_special_npoess.sh

@@ -153,7 +153,8 @@ for (( fhr=SHOUR; fhr <= FHOUR; fhr = fhr + FHINC )); do
    # existence of the restart files
    ###############################
    export pgm="postcheck"
-   grib_file="${COM_ATMOS_MASTER}/${RUN}.t${cyc}z.goesmasterf${fhr3}.grb2"
+   # grib_file="${COM_ATMOS_MASTER}/${RUN}.t${cyc}z.goesmasterf${fhr3}.grb2"
+   grib_file="${COM_ATMOS_MASTER}/${RUN}.t${cyc}z.special.grb2f${fhr3}"
    if ! wait_for_file "${grib_file}" "${SLEEP_INT}" "${SLEEP_LOOP_MAX}"; then
       echo "FATAL ERROR: GOES master grib file ${grib_file} not available after max sleep time"
       export err=9

versions/run.hera.ver

@@ -4,7 +4,7 @@ export spack_env=gsi-addon-dev-rocky8
 
 export hpss_ver=hpss
 export ncl_ver=6.6.2
-export R_ver=3.5.0
+export R_ver=3.6.1
 export gempak_ver=7.17.0
 export perl_ver=5.38.0
 

workflow/applications/gfs_cycled.py

@@ -264,7 +264,7 @@ def get_task_names(self):
                 gfs_tasks += ['gempakpgrb2spec']
 
         if self.do_awips:
-            gfs_tasks += ['awips_20km_1p0deg', 'awips_g2', 'fbwind']
+            gfs_tasks += ['awips_20km_1p0deg', 'fbwind']
 
         if self.do_mos:
             gfs_tasks += ['mos_stn_prep', 'mos_grd_prep', 'mos_ext_stn_prep', 'mos_ext_grd_prep',

workflow/applications/gfs_forecast_only.py

@@ -124,7 +124,7 @@ def get_task_names(self):
                 tasks += ['gempak', 'gempakmeta', 'gempakncdcupapgif', 'gempakpgrb2spec']
 
             if self.do_awips:
-                tasks += ['awips_20km_1p0deg', 'awips_g2', 'fbwind']
+                tasks += ['awips_20km_1p0deg', 'fbwind']
 
         if self.do_ocean:
             tasks += ['ocean_prod']

workflow/prod.yml

@@ -113,17 +113,6 @@ suites:
                 jgfs_atmos_awips_f( 3,27,6 ):
                   edits:
                     TRDRUN: 'NO'
-            awips_g2:
-              tasks:
-                jgfs_atmos_awips_g2_f( 0,64,6 ):
-                  template: jgfs_atmos_awips_g2_master
-                  triggers:
-                  - task: jgfs_atmos_post_f( )
-                  edits:
-                    FHRGRP: '( )'
-                    FHRLST: 'f( )'
-                    FCSTHR: '( )'
-                    TRDRUN: 'YES'
             gempak:
               tasks:
                 jgfs_atmos_gempak:

workflow/rocoto/gfs_tasks.py

@@ -1433,47 +1433,6 @@ def awips_20km_1p0deg(self):
 
         return task
 
-    def awips_g2(self):
-
-        deps = []
-        dep_dict = {'type': 'metatask', 'name': f'{self.cdump}atmos_prod'}
-        deps.append(rocoto.add_dependency(dep_dict))
-        dependencies = rocoto.create_dependency(dep=deps)
-
-        awipsenvars = self.envars.copy()
-        awipsenvar_dict = {'FHRGRP': '#grp#',
-                           'FHRLST': '#lst#',
-                           'ROTDIR': self.rotdir}
-        for key, value in awipsenvar_dict.items():
-            awipsenvars.append(rocoto.create_envar(name=key, value=str(value)))
-
-        varname1, varname2, varname3 = 'grp', 'dep', 'lst'
-        varval1, varval2, varval3 = self._get_awipsgroups(self.cdump, self._configs['awips'])
-        var_dict = {varname1: varval1, varname2: varval2, varname3: varval3}
-
-        resources = self.get_resource('awips')
-
-        task_name = f'{self.cdump}awips_g2#{varname1}#'
-        task_dict = {'task_name': task_name,
-                     'resources': resources,
-                     'dependency': dependencies,
-                     'envars': awipsenvars,
-                     'cycledef': self.cdump.replace('enkf', ''),
-                     'command': f'{self.HOMEgfs}/jobs/rocoto/awips_g2.sh',
-                     'job_name': f'{self.pslot}_{task_name}_@H',
-                     'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log',
-                     'maxtries': '&MAXTRIES;'
-                     }
-
-        metatask_dict = {'task_name': f'{self.cdump}awips_g2',
-                         'task_dict': task_dict,
-                         'var_dict': var_dict
-                         }
-
-        task = rocoto.create_task(metatask_dict)
-
-        return task
-
     def gempak(self):
 
         deps = []

workflow/rocoto/tasks.py

@@ -27,7 +27,7 @@ class Tasks:
                    'verfozn', 'verfrad', 'vminmon',
                    'metp',
                    'tracker', 'genesis', 'genesis_fsu',
-                   'postsnd', 'awips_g2', 'awips_20km_1p0deg', 'fbwind',
+                   'postsnd', 'awips_20km_1p0deg', 'fbwind',
                    'gempak', 'gempakmeta', 'gempakmetancdc', 'gempakncdcupapgif', 'gempakpgrb2spec', 'npoess_pgrb2_0p5deg'
                    'waveawipsbulls', 'waveawipsgridded', 'wavegempak', 'waveinit',
                    'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt', 'wavepostsbs', 'waveprep',