From 10859c20cfbc2b75a5afbacdf91385eec3eca959 Mon Sep 17 00:00:00 2001 From: henrywinterbottom-wxdev Date: Thu, 2 Nov 2023 08:19:32 -0600 Subject: [PATCH 01/23] Corrected missing local variable reference. --- ush/hpssarch_gen.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/hpssarch_gen.sh b/ush/hpssarch_gen.sh index 8d8ebf4fe9..2caf0040ef 100755 --- a/ush/hpssarch_gen.sh +++ b/ush/hpssarch_gen.sh @@ -165,8 +165,8 @@ if [[ ${type} = "gfs" ]]; then echo "${COM_ATMOS_GRIB_0p25/${ROTDIR}\//}/${head}pgrb2b.0p25.f${fhr}" echo "${COM_ATMOS_GRIB_0p25/${ROTDIR}\//}/${head}pgrb2b.0p25.f${fhr}.idx" if [[ -s "${COM_ATMOS_GRIB_1p00}/${head}pgrb2b.1p00.f${fhr}" ]]; then - echo "${COM_ATMOS_GRIB_1p00/${ROTDIR}\//}/{head}pgrb2b.1p00.f${fhr}" - echo "${COM_ATMOS_GRIB_1p00/${ROTDIR}\//}/{head}pgrb2b.1p00.f${fhr}.idx" + echo "${COM_ATMOS_GRIB_1p00/${ROTDIR}\//}/${head}pgrb2b.1p00.f${fhr}" + echo "${COM_ATMOS_GRIB_1p00/${ROTDIR}\//}/${head}pgrb2b.1p00.f${fhr}.idx" fi } >> "${DATA}/gfs_pgrb2b.txt" fi From 9df253944ce183931197db203e97de3cfb91043e Mon Sep 17 00:00:00 2001 From: henrywinterbottom-wxdev Date: Thu, 2 Nov 2023 08:21:55 -0600 Subject: [PATCH 02/23] Reverting push to develop. --- ush/hpssarch_gen.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/hpssarch_gen.sh b/ush/hpssarch_gen.sh index 2caf0040ef..8d8ebf4fe9 100755 --- a/ush/hpssarch_gen.sh +++ b/ush/hpssarch_gen.sh @@ -165,8 +165,8 @@ if [[ ${type} = "gfs" ]]; then echo "${COM_ATMOS_GRIB_0p25/${ROTDIR}\//}/${head}pgrb2b.0p25.f${fhr}" echo "${COM_ATMOS_GRIB_0p25/${ROTDIR}\//}/${head}pgrb2b.0p25.f${fhr}.idx" if [[ -s "${COM_ATMOS_GRIB_1p00}/${head}pgrb2b.1p00.f${fhr}" ]]; then - echo "${COM_ATMOS_GRIB_1p00/${ROTDIR}\//}/${head}pgrb2b.1p00.f${fhr}" - echo "${COM_ATMOS_GRIB_1p00/${ROTDIR}\//}/${head}pgrb2b.1p00.f${fhr}.idx" + echo "${COM_ATMOS_GRIB_1p00/${ROTDIR}\//}/{head}pgrb2b.1p00.f${fhr}" + echo "${COM_ATMOS_GRIB_1p00/${ROTDIR}\//}/{head}pgrb2b.1p00.f${fhr}.idx" fi } >> "${DATA}/gfs_pgrb2b.txt" fi From 9297b1eb7c7dd3fdca2e8ea2b213d74ba954f68a Mon Sep 17 00:00:00 2001 From: henrywinterbottom-wxdev Date: Wed, 29 Nov 2023 06:29:41 -0700 Subject: [PATCH 03/23] Added new AWIPS jobs for Rocoto. --- jobs/rocoto/awips_20sh.sh | 68 +++++++++++++++++++++++++++++++++++++++ jobs/rocoto/awips_g2sh.sh | 67 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+) create mode 100755 jobs/rocoto/awips_20sh.sh create mode 100755 jobs/rocoto/awips_g2sh.sh diff --git a/jobs/rocoto/awips_20sh.sh b/jobs/rocoto/awips_20sh.sh new file mode 100755 index 0000000000..869542cc16 --- /dev/null +++ b/jobs/rocoto/awips_20sh.sh @@ -0,0 +1,68 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +## Abstract: +## Inline awips driver script +## HOMEgfs : /full/path/to/workflow +## EXPDIR : /full/path/to/config/files +## CDATE : current analysis date (YYYYMMDDHH) +## CDUMP : cycle name (gdas / gfs) +## PDY : current date (YYYYMMDD) +## cyc : current cycle (HH) +############################################################### + +############################################################### +# Source FV3GFS workflow modules +source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" +status=$? +(( status != 0 )) && exit "${status}" + +export job="awips_20sh.sh" +export jobid="${job}.$$" + +# TODO (#1228) - This script is doing more than just calling a j-job +# Also, this forces us to call the config files here instead of the j-job +source "${HOMEgfs}/ush/jjob_header.sh" -e "awips" -c "base awips" + +fhrlst=$(echo ${FHRLST} | sed -e 's/_/ /g; s/f/ /g; s/,/ /g') + +############################################################### + +################################################################################ +echo +echo "=============== BEGIN AWIPS ===============" + +for fhr3 in ${fhrlst}; do + fhr=$(( 10#${fhr3} )) + if (( fhr > FHMAX_GFS )); then + echo "Nothing to process for FHR = ${fhr3}, cycle" + continue + fi + + fhmin=0 + fhmax=84 + if (( fhr >= fhmin && fhr <= fhmax )); then + if ((fhr % 3 == 0)); then + export fcsthrs=${fhr3} + ${AWIPS20SH} + fi + fi + + fhmin=90 + fhmax=240 + if (( fhr >= fhmin && fhr <= fhmax )); then + if ((fhr % 6 == 0)); then + export fcsthrs=${fhr3} + ${AWIPS20SH} + fi + fi +done + + +############################################################### +# Force Exit out cleanly +if [[ ${KEEPDATA:-"NO"} == "NO" ]] ; then rm -rf "${DATA}" ; fi + +exit 0 diff --git a/jobs/rocoto/awips_g2sh.sh b/jobs/rocoto/awips_g2sh.sh new file mode 100755 index 0000000000..c531c5a9d6 --- /dev/null +++ b/jobs/rocoto/awips_g2sh.sh @@ -0,0 +1,67 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +## Abstract: +## Inline awips driver script +## HOMEgfs : /full/path/to/workflow +## EXPDIR : /full/path/to/config/files +## CDATE : current analysis date (YYYYMMDDHH) +## CDUMP : cycle name (gdas / gfs) +## PDY : current date (YYYYMMDD) +## cyc : current cycle (HH) +############################################################### + +############################################################### +# Source FV3GFS workflow modules +source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" +status=$? +(( status != 0 )) && exit "${status}" + +export job="awips_g2sh" +export jobid="${job}.$$" + +# TODO (#1228) - This script is doing more than just calling a j-job +# Also, this forces us to call the config files here instead of the j-job +source "${HOMEgfs}/ush/jjob_header.sh" -e "awips" -c "base awips" + +fhrlst=$(echo ${FHRLST} | sed -e 's/_/ /g; s/f/ /g; s/,/ /g') + +############################################################### + +################################################################################ +echo +echo "=============== BEGIN AWIPS ===============" + +for fhr3 in ${fhrlst}; do + fhr=$(( 10#${fhr3} )) + if (( fhr > FHMAX_GFS )); then + echo "Nothing to process for FHR = ${fhr3}, cycle" + continue + fi + + fhmin=0 + fhmax=84 + if (( fhr >= fhmin && fhr <= fhmax )); then + if ((fhr % 6 == 0)); then + ${AWIPSG2SH} + fi + fi + + fhmin=90 + fhmax=240 + if (( fhr >= fhmin && fhr <= fhmax )); then + if ((fhr % 6 == 0)); then + export fcsthrs=${fhr3} + ${AWIPSG2SH} + fi + fi +done + + +############################################################### +# Force Exit out cleanly +if [[ ${KEEPDATA:-"NO"} == "NO" ]] ; then rm -rf "${DATA}" ; fi + +exit 0 From 42dac7f23c8fe976f13f56ff8b66df0188d2cbbe Mon Sep 17 00:00:00 2001 From: henrywinterbottom-wxdev Date: Wed, 29 Nov 2023 06:30:05 -0700 Subject: [PATCH 04/23] Removed no longer needed script. --- jobs/rocoto/awips.sh | 73 -------------------------------------------- 1 file changed, 73 deletions(-) delete mode 100755 jobs/rocoto/awips.sh diff --git a/jobs/rocoto/awips.sh b/jobs/rocoto/awips.sh deleted file mode 100755 index 57fbd92572..0000000000 --- a/jobs/rocoto/awips.sh +++ /dev/null @@ -1,73 +0,0 @@ -#! /usr/bin/env bash - -source "${HOMEgfs}/ush/preamble.sh" - -############################################################### -## Abstract: -## Inline awips driver script -## HOMEgfs : /full/path/to/workflow -## EXPDIR : /full/path/to/config/files -## CDATE : current analysis date (YYYYMMDDHH) -## CDUMP : cycle name (gdas / gfs) -## PDY : current date (YYYYMMDD) -## cyc : current cycle (HH) -############################################################### - -############################################################### -# Source FV3GFS workflow modules -source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" -status=$? -(( status != 0 )) && exit "${status}" - -export job="awips" -export jobid="${job}.$$" - -# TODO (#1228) - This script is doing more than just calling a j-job -# Also, this forces us to call the config files here instead of the j-job -source "${HOMEgfs}/ush/jjob_header.sh" -e "awips" -c "base awips" - -fhrlst=$(echo ${FHRLST} | sed -e 's/_/ /g; s/f/ /g; s/,/ /g') - -############################################################### - -################################################################################ -echo -echo "=============== BEGIN AWIPS ===============" - -for fhr3 in ${fhrlst}; do - fhr=$(( 10#${fhr3} )) - if (( fhr > FHMAX_GFS )); then - echo "Nothing to process for FHR = ${fhr3}, cycle" - continue - fi - - fhmin=0 - fhmax=84 - if (( fhr >= fhmin && fhr <= fhmax )); then - if ((fhr % 3 == 0)); then - export fcsthrs=${fhr3} - ${AWIPS20SH} - fi - - if ((fhr % 6 == 0)); then - ${AWIPSG2SH} - fi - fi - - fhmin=90 - fhmax=240 - if (( fhr >= fhmin && fhr <= fhmax )); then - if ((fhr % 6 == 0)); then - export fcsthrs=${fhr3} - ${AWIPS20SH} - ${AWIPSG2SH} - fi - fi -done - - -############################################################### -# Force Exit out cleanly -if [[ ${KEEPDATA:-"NO"} == "NO" ]] ; then rm -rf "${DATA}" ; fi - -exit 0 From 82c8f1b2b371b5b763b610aa76f6f2f21b9c1c70 Mon Sep 17 00:00:00 2001 From: henrywinterbottom-wxdev Date: Wed, 29 Nov 2023 06:37:01 -0700 Subject: [PATCH 05/23] Updated Rocoto applications. --- workflow/applications/gfs_cycled.py | 3 ++- workflow/applications/gfs_forecast_only.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 2438a8244e..b2b103c217 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -234,7 +234,8 @@ def get_task_names(self): gfs_tasks += ['gempak'] if self.do_awips: - gfs_tasks += ['awips'] + gfs_tasks += ['awips_20sh'] + gfs_tasks += ['awips_g2sh'] gfs_tasks += ['fbwinds'] if self.do_npoess: diff --git a/workflow/applications/gfs_forecast_only.py b/workflow/applications/gfs_forecast_only.py index 01d31b08ca..545c2df613 100644 --- a/workflow/applications/gfs_forecast_only.py +++ b/workflow/applications/gfs_forecast_only.py @@ -119,7 +119,8 @@ def get_task_names(self): tasks += ['gempak'] if self.do_awips: - tasks += ['awips'] + tasks += ['awips_20sh'] + tasks += ['awips_g2sh'] tasks += ['fbwinds'] if self.do_wafs: From 3e9a24cdcd04369909bf6fc4df9764ca3546e2ed Mon Sep 17 00:00:00 2001 From: henrywinterbottom-wxdev Date: Wed, 29 Nov 2023 06:40:53 -0700 Subject: [PATCH 06/23] Updates to test AWIPS job splitting. --- workflow/rocoto/gfs_tasks.py | 64 ++++++++++++++++++++++++++++++++++-- workflow/rocoto/tasks.py | 2 +- 2 files changed, 63 insertions(+), 3 deletions(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index a842b4704b..409c5e9be4 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -822,7 +822,7 @@ def fbwinds(self): return task - def awips(self): + def awips_20sh(self): def _get_awipsgroups(cdump, config): @@ -878,7 +878,67 @@ def _get_awipsgroups(cdump, config): resources = self.get_resource('awips') task = create_wf_task('awips', resources, cdump=self.cdump, envar=awipsenvars, dependency=dependencies, - metatask='awips', varname=varname1, varval=varval1, vardict=vardict) + metatask='awips_20sh', varname=varname1, varval=varval1, vardict=vardict) + + return task + + def awips_g2sh(self): + + def _get_awipsgroups(cdump, config): + + fhmin = config['FHMIN'] + fhmax = config['FHMAX'] + fhout = config['FHOUT'] + + # Get a list of all forecast hours + fhrs = [] + if cdump in ['gdas']: + fhrs = range(fhmin, fhmax + fhout, fhout) + elif cdump in ['gfs']: + fhmax = np.max( + [config['FHMAX_GFS_00'], config['FHMAX_GFS_06'], config['FHMAX_GFS_12'], config['FHMAX_GFS_18']]) + fhout = config['FHOUT_GFS'] + fhmax_hf = config['FHMAX_HF_GFS'] + fhout_hf = config['FHOUT_HF_GFS'] + if fhmax > 240: + fhmax = 240 + if fhmax_hf > 240: + fhmax_hf = 240 + fhrs_hf = list(range(fhmin, fhmax_hf + fhout_hf, fhout_hf)) + fhrs = fhrs_hf + list(range(fhrs_hf[-1] + fhout, fhmax + fhout, fhout)) + + nawipsgrp = config['NAWIPSGRP'] + ngrps = nawipsgrp if len(fhrs) > nawipsgrp else len(fhrs) + + fhrs = [f'f{fhr:03d}' for fhr in fhrs] + fhrs = np.array_split(fhrs, ngrps) + fhrs = [fhr.tolist() for fhr in fhrs] + + grp = ' '.join([f'_{fhr[0]}-{fhr[-1]}' for fhr in fhrs]) + dep = ' '.join([fhr[-1] for fhr in fhrs]) + lst = ' '.join(['_'.join(fhr) for fhr in fhrs]) + + return grp, dep, lst + + deps = [] + dep_dict = {'type': 'metatask', 'name': f'{self.cdump}post'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + + awipsenvars = self.envars.copy() + awipsenvar_dict = {'FHRGRP': '#grp#', + 'FHRLST': '#lst#', + 'ROTDIR': self._base.get('ROTDIR')} + for key, value in awipsenvar_dict.items(): + awipsenvars.append(rocoto.create_envar(name=key, value=str(value))) + + varname1, varname2, varname3 = 'grp', 'dep', 'lst' + varval1, varval2, varval3 = _get_awipsgroups(self.cdump, self._configs['awips']) + vardict = {varname2: varval2, varname3: varval3} + + resources = self.get_resource('awips') + task = create_wf_task('awips', resources, cdump=self.cdump, envar=awipsenvars, dependency=dependencies, + metatask='awips_g2sh', varname=varname1, varval=varval1, vardict=vardict) return task diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index dea351ac12..36b80d168e 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -22,7 +22,7 @@ class Tasks: 'fcst', 'post', 'ocnpost', 'verfozn', 'verfrad', 'vminmon', 'metp', 'tracker', 'genesis', 'genesis_fsu', - 'postsnd', 'awips', 'fbwinds', 'gempak', + 'postsnd', 'awips_g2sh', 'awips_20sh', 'fbwinds', 'gempak', 'waveawipsbulls', 'waveawipsgridded', 'wavegempak', 'waveinit', 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt', 'wavepostsbs', 'waveprep', 'npoess'] From 5c2b60a1021bc15e21847c2b8badd28b850e663b Mon Sep 17 00:00:00 2001 From: HenryWinterbottom-NOAA Date: Wed, 29 Nov 2023 14:01:00 +0000 Subject: [PATCH 07/23] Added tasks to GFS for AWIPS. --- workflow/rocoto/gfs_tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 409c5e9be4..e7cb7dc328 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -877,7 +877,7 @@ def _get_awipsgroups(cdump, config): vardict = {varname2: varval2, varname3: varval3} resources = self.get_resource('awips') - task = create_wf_task('awips', resources, cdump=self.cdump, envar=awipsenvars, dependency=dependencies, + task = create_wf_task('awips_20sh', resources, cdump=self.cdump, envar=awipsenvars, dependency=dependencies, metatask='awips_20sh', varname=varname1, varval=varval1, vardict=vardict) return task @@ -937,7 +937,7 @@ def _get_awipsgroups(cdump, config): vardict = {varname2: varval2, varname3: varval3} resources = self.get_resource('awips') - task = create_wf_task('awips', resources, cdump=self.cdump, envar=awipsenvars, dependency=dependencies, + task = create_wf_task('awips_g2sh', resources, cdump=self.cdump, envar=awipsenvars, dependency=dependencies, metatask='awips_g2sh', varname=varname1, varval=varval1, vardict=vardict) return task From 76d4e8ced4c3574611c4a3ca0e9c36f381599a2c Mon Sep 17 00:00:00 2001 From: henrywinterbottom-wxdev Date: Wed, 29 Nov 2023 07:17:40 -0700 Subject: [PATCH 08/23] Addressing issues raised by linter. --- jobs/rocoto/awips_20sh.sh | 13 +++++++------ jobs/rocoto/awips_g2sh.sh | 11 ++++++----- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/jobs/rocoto/awips_20sh.sh b/jobs/rocoto/awips_20sh.sh index 869542cc16..0ea150a051 100755 --- a/jobs/rocoto/awips_20sh.sh +++ b/jobs/rocoto/awips_20sh.sh @@ -26,7 +26,8 @@ export jobid="${job}.$$" # Also, this forces us to call the config files here instead of the j-job source "${HOMEgfs}/ush/jjob_header.sh" -e "awips" -c "base awips" -fhrlst=$(echo ${FHRLST} | sed -e 's/_/ /g; s/f/ /g; s/,/ /g') +# shellcheck disable=SC2153 +fhrlst=$(echo "${FHRLST}" | sed -e 's/_/ /g; s/f/ /g; s/,/ /g') ############################################################### @@ -34,7 +35,7 @@ fhrlst=$(echo ${FHRLST} | sed -e 's/_/ /g; s/f/ /g; s/,/ /g') echo echo "=============== BEGIN AWIPS ===============" -for fhr3 in ${fhrlst}; do +for fhr3 in "${fhrlst}"; do fhr=$(( 10#${fhr3} )) if (( fhr > FHMAX_GFS )); then echo "Nothing to process for FHR = ${fhr3}, cycle" @@ -45,8 +46,8 @@ for fhr3 in ${fhrlst}; do fhmax=84 if (( fhr >= fhmin && fhr <= fhmax )); then if ((fhr % 3 == 0)); then - export fcsthrs=${fhr3} - ${AWIPS20SH} + export fcsthrs="${fhr3}" + "${AWIPS20SH}" fi fi @@ -54,8 +55,8 @@ for fhr3 in ${fhrlst}; do fhmax=240 if (( fhr >= fhmin && fhr <= fhmax )); then if ((fhr % 6 == 0)); then - export fcsthrs=${fhr3} - ${AWIPS20SH} + export fcsthrs="${fhr3}" + "${AWIPS20SH}" fi fi done diff --git a/jobs/rocoto/awips_g2sh.sh b/jobs/rocoto/awips_g2sh.sh index c531c5a9d6..0b4cd8ff0f 100755 --- a/jobs/rocoto/awips_g2sh.sh +++ b/jobs/rocoto/awips_g2sh.sh @@ -26,7 +26,8 @@ export jobid="${job}.$$" # Also, this forces us to call the config files here instead of the j-job source "${HOMEgfs}/ush/jjob_header.sh" -e "awips" -c "base awips" -fhrlst=$(echo ${FHRLST} | sed -e 's/_/ /g; s/f/ /g; s/,/ /g') +# shellcheck disable=SC2153 +fhrlst=$(echo "${FHRLST}" | sed -e "s/_/ /g; s/f/ /g; s/,/ /g") ############################################################### @@ -34,7 +35,7 @@ fhrlst=$(echo ${FHRLST} | sed -e 's/_/ /g; s/f/ /g; s/,/ /g') echo echo "=============== BEGIN AWIPS ===============" -for fhr3 in ${fhrlst}; do +for fhr3 in "${fhrlst}"; do fhr=$(( 10#${fhr3} )) if (( fhr > FHMAX_GFS )); then echo "Nothing to process for FHR = ${fhr3}, cycle" @@ -45,7 +46,7 @@ for fhr3 in ${fhrlst}; do fhmax=84 if (( fhr >= fhmin && fhr <= fhmax )); then if ((fhr % 6 == 0)); then - ${AWIPSG2SH} + "${AWIPSG2SH}" fi fi @@ -53,8 +54,8 @@ for fhr3 in ${fhrlst}; do fhmax=240 if (( fhr >= fhmin && fhr <= fhmax )); then if ((fhr % 6 == 0)); then - export fcsthrs=${fhr3} - ${AWIPSG2SH} + export fcsthrs="${fhr3}" + "${AWIPSG2SH}" fi fi done From 7b41f31d79a81d315c73ce68171d91a294db7206 Mon Sep 17 00:00:00 2001 From: henrywinterbottom-wxdev Date: Wed, 29 Nov 2023 07:19:18 -0700 Subject: [PATCH 09/23] Addressing issues raised by linter. --- jobs/rocoto/awips_20sh.sh | 2 +- jobs/rocoto/awips_g2sh.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jobs/rocoto/awips_20sh.sh b/jobs/rocoto/awips_20sh.sh index 0ea150a051..7ed180ad3e 100755 --- a/jobs/rocoto/awips_20sh.sh +++ b/jobs/rocoto/awips_20sh.sh @@ -35,7 +35,7 @@ fhrlst=$(echo "${FHRLST}" | sed -e 's/_/ /g; s/f/ /g; s/,/ /g') echo echo "=============== BEGIN AWIPS ===============" -for fhr3 in "${fhrlst}"; do +for fhr3 in ${fhrlst}; do fhr=$(( 10#${fhr3} )) if (( fhr > FHMAX_GFS )); then echo "Nothing to process for FHR = ${fhr3}, cycle" diff --git a/jobs/rocoto/awips_g2sh.sh b/jobs/rocoto/awips_g2sh.sh index 0b4cd8ff0f..0fff994beb 100755 --- a/jobs/rocoto/awips_g2sh.sh +++ b/jobs/rocoto/awips_g2sh.sh @@ -35,7 +35,7 @@ fhrlst=$(echo "${FHRLST}" | sed -e "s/_/ /g; s/f/ /g; s/,/ /g") echo echo "=============== BEGIN AWIPS ===============" -for fhr3 in "${fhrlst}"; do +for fhr3 in ${fhrlst}; do fhr=$(( 10#${fhr3} )) if (( fhr > FHMAX_GFS )); then echo "Nothing to process for FHR = ${fhr3}, cycle" From 763c5163e38210908ae8af72717df7b10e839b79 Mon Sep 17 00:00:00 2001 From: henrywinterbottom-wxdev Date: Wed, 29 Nov 2023 10:46:50 -0700 Subject: [PATCH 10/23] Removed comment as per reviewer request. --- jobs/rocoto/awips_20sh.sh | 2 -- jobs/rocoto/awips_g2sh.sh | 2 -- 2 files changed, 4 deletions(-) diff --git a/jobs/rocoto/awips_20sh.sh b/jobs/rocoto/awips_20sh.sh index 7ed180ad3e..093ba378ea 100755 --- a/jobs/rocoto/awips_20sh.sh +++ b/jobs/rocoto/awips_20sh.sh @@ -22,8 +22,6 @@ status=$? export job="awips_20sh.sh" export jobid="${job}.$$" -# TODO (#1228) - This script is doing more than just calling a j-job -# Also, this forces us to call the config files here instead of the j-job source "${HOMEgfs}/ush/jjob_header.sh" -e "awips" -c "base awips" # shellcheck disable=SC2153 diff --git a/jobs/rocoto/awips_g2sh.sh b/jobs/rocoto/awips_g2sh.sh index 0fff994beb..d0f751a488 100755 --- a/jobs/rocoto/awips_g2sh.sh +++ b/jobs/rocoto/awips_g2sh.sh @@ -22,8 +22,6 @@ status=$? export job="awips_g2sh" export jobid="${job}.$$" -# TODO (#1228) - This script is doing more than just calling a j-job -# Also, this forces us to call the config files here instead of the j-job source "${HOMEgfs}/ush/jjob_header.sh" -e "awips" -c "base awips" # shellcheck disable=SC2153 From 7e4e358d3bd247ce4e05b5f0f1adea35c7a0c015 Mon Sep 17 00:00:00 2001 From: henrywinterbottom-wxdev Date: Wed, 29 Nov 2023 10:49:22 -0700 Subject: [PATCH 11/23] Renamed tasks as per reviewer suggestion. --- workflow/applications/gfs_cycled.py | 4 ++-- workflow/applications/gfs_forecast_only.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index b2b103c217..b04f9a45de 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -234,8 +234,8 @@ def get_task_names(self): gfs_tasks += ['gempak'] if self.do_awips: - gfs_tasks += ['awips_20sh'] - gfs_tasks += ['awips_g2sh'] + gfs_tasks += ['awips_20km_1p0'] + gfs_tasks += ['awips_g2'] gfs_tasks += ['fbwinds'] if self.do_npoess: diff --git a/workflow/applications/gfs_forecast_only.py b/workflow/applications/gfs_forecast_only.py index 545c2df613..51a5976996 100644 --- a/workflow/applications/gfs_forecast_only.py +++ b/workflow/applications/gfs_forecast_only.py @@ -119,8 +119,8 @@ def get_task_names(self): tasks += ['gempak'] if self.do_awips: - tasks += ['awips_20sh'] - tasks += ['awips_g2sh'] + tasks += ['awips_20km_1p0'] + tasks += ['awips_g2'] tasks += ['fbwinds'] if self.do_wafs: From dd20e83edcea5e6d555019378711f534ad19c925 Mon Sep 17 00:00:00 2001 From: henrywinterbottom-wxdev Date: Wed, 29 Nov 2023 10:52:31 -0700 Subject: [PATCH 12/23] Updates relative to reviewer comments/suggestions. --- workflow/rocoto/gfs_tasks.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index e7cb7dc328..0c9d0f0db0 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -822,7 +822,7 @@ def fbwinds(self): return task - def awips_20sh(self): + def awips_20km_1p0(self): def _get_awipsgroups(cdump, config): @@ -877,12 +877,12 @@ def _get_awipsgroups(cdump, config): vardict = {varname2: varval2, varname3: varval3} resources = self.get_resource('awips') - task = create_wf_task('awips_20sh', resources, cdump=self.cdump, envar=awipsenvars, dependency=dependencies, - metatask='awips_20sh', varname=varname1, varval=varval1, vardict=vardict) + task = create_wf_task('awips_20km_1p0', resources, cdump=self.cdump, envar=awipsenvars, dependency=dependencies, + metatask='awips_20km_1p0', varname=varname1, varval=varval1, vardict=vardict) return task - def awips_g2sh(self): + def awips_g2(self): def _get_awipsgroups(cdump, config): @@ -937,8 +937,8 @@ def _get_awipsgroups(cdump, config): vardict = {varname2: varval2, varname3: varval3} resources = self.get_resource('awips') - task = create_wf_task('awips_g2sh', resources, cdump=self.cdump, envar=awipsenvars, dependency=dependencies, - metatask='awips_g2sh', varname=varname1, varval=varval1, vardict=vardict) + task = create_wf_task('awips_g2', resources, cdump=self.cdump, envar=awipsenvars, dependency=dependencies, + metatask='awips_g2', varname=varname1, varval=varval1, vardict=vardict) return task From ea27c258c8458a32e22e3d9bcecec331d1f0a30f Mon Sep 17 00:00:00 2001 From: "Henry R. Winterbottom" <49202169+HenryWinterbottom-NOAA@users.noreply.github.com> Date: Fri, 1 Dec 2023 09:03:35 -0700 Subject: [PATCH 13/23] Update jobs/rocoto/awips_20sh.sh Co-authored-by: Rahul Mahajan --- jobs/rocoto/awips_20sh.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/rocoto/awips_20sh.sh b/jobs/rocoto/awips_20sh.sh index 093ba378ea..f4d1d3e66e 100755 --- a/jobs/rocoto/awips_20sh.sh +++ b/jobs/rocoto/awips_20sh.sh @@ -19,7 +19,7 @@ source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" status=$? (( status != 0 )) && exit "${status}" -export job="awips_20sh.sh" +export job="awips_20km" export jobid="${job}.$$" source "${HOMEgfs}/ush/jjob_header.sh" -e "awips" -c "base awips" From 920460b2d8b0df9a18247c6a56d6c3a40ba8d292 Mon Sep 17 00:00:00 2001 From: henrywinterbottom-wxdev Date: Fri, 1 Dec 2023 09:17:55 -0700 Subject: [PATCH 14/23] Renamed files. --- jobs/rocoto/awips_20km_1p0.sh | 67 +++++++++++++++++++++++++++++++++++ jobs/rocoto/awips_g2.sh | 66 ++++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100755 jobs/rocoto/awips_20km_1p0.sh create mode 100755 jobs/rocoto/awips_g2.sh diff --git a/jobs/rocoto/awips_20km_1p0.sh b/jobs/rocoto/awips_20km_1p0.sh new file mode 100755 index 0000000000..f4d1d3e66e --- /dev/null +++ b/jobs/rocoto/awips_20km_1p0.sh @@ -0,0 +1,67 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +## Abstract: +## Inline awips driver script +## HOMEgfs : /full/path/to/workflow +## EXPDIR : /full/path/to/config/files +## CDATE : current analysis date (YYYYMMDDHH) +## CDUMP : cycle name (gdas / gfs) +## PDY : current date (YYYYMMDD) +## cyc : current cycle (HH) +############################################################### + +############################################################### +# Source FV3GFS workflow modules +source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" +status=$? +(( status != 0 )) && exit "${status}" + +export job="awips_20km" +export jobid="${job}.$$" + +source "${HOMEgfs}/ush/jjob_header.sh" -e "awips" -c "base awips" + +# shellcheck disable=SC2153 +fhrlst=$(echo "${FHRLST}" | sed -e 's/_/ /g; s/f/ /g; s/,/ /g') + +############################################################### + +################################################################################ +echo +echo "=============== BEGIN AWIPS ===============" + +for fhr3 in ${fhrlst}; do + fhr=$(( 10#${fhr3} )) + if (( fhr > FHMAX_GFS )); then + echo "Nothing to process for FHR = ${fhr3}, cycle" + continue + fi + + fhmin=0 + fhmax=84 + if (( fhr >= fhmin && fhr <= fhmax )); then + if ((fhr % 3 == 0)); then + export fcsthrs="${fhr3}" + "${AWIPS20SH}" + fi + fi + + fhmin=90 + fhmax=240 + if (( fhr >= fhmin && fhr <= fhmax )); then + if ((fhr % 6 == 0)); then + export fcsthrs="${fhr3}" + "${AWIPS20SH}" + fi + fi +done + + +############################################################### +# Force Exit out cleanly +if [[ ${KEEPDATA:-"NO"} == "NO" ]] ; then rm -rf "${DATA}" ; fi + +exit 0 diff --git a/jobs/rocoto/awips_g2.sh b/jobs/rocoto/awips_g2.sh new file mode 100755 index 0000000000..d0f751a488 --- /dev/null +++ b/jobs/rocoto/awips_g2.sh @@ -0,0 +1,66 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +## Abstract: +## Inline awips driver script +## HOMEgfs : /full/path/to/workflow +## EXPDIR : /full/path/to/config/files +## CDATE : current analysis date (YYYYMMDDHH) +## CDUMP : cycle name (gdas / gfs) +## PDY : current date (YYYYMMDD) +## cyc : current cycle (HH) +############################################################### + +############################################################### +# Source FV3GFS workflow modules +source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" +status=$? +(( status != 0 )) && exit "${status}" + +export job="awips_g2sh" +export jobid="${job}.$$" + +source "${HOMEgfs}/ush/jjob_header.sh" -e "awips" -c "base awips" + +# shellcheck disable=SC2153 +fhrlst=$(echo "${FHRLST}" | sed -e "s/_/ /g; s/f/ /g; s/,/ /g") + +############################################################### + +################################################################################ +echo +echo "=============== BEGIN AWIPS ===============" + +for fhr3 in ${fhrlst}; do + fhr=$(( 10#${fhr3} )) + if (( fhr > FHMAX_GFS )); then + echo "Nothing to process for FHR = ${fhr3}, cycle" + continue + fi + + fhmin=0 + fhmax=84 + if (( fhr >= fhmin && fhr <= fhmax )); then + if ((fhr % 6 == 0)); then + "${AWIPSG2SH}" + fi + fi + + fhmin=90 + fhmax=240 + if (( fhr >= fhmin && fhr <= fhmax )); then + if ((fhr % 6 == 0)); then + export fcsthrs="${fhr3}" + "${AWIPSG2SH}" + fi + fi +done + + +############################################################### +# Force Exit out cleanly +if [[ ${KEEPDATA:-"NO"} == "NO" ]] ; then rm -rf "${DATA}" ; fi + +exit 0 From df94d43b7b46569d5f6e7799db97136bcada8c39 Mon Sep 17 00:00:00 2001 From: henrywinterbottom-wxdev Date: Fri, 1 Dec 2023 09:18:14 -0700 Subject: [PATCH 15/23] Files have been renamed. --- jobs/rocoto/awips_20sh.sh | 67 --------------------------------------- jobs/rocoto/awips_g2sh.sh | 66 -------------------------------------- 2 files changed, 133 deletions(-) delete mode 100755 jobs/rocoto/awips_20sh.sh delete mode 100755 jobs/rocoto/awips_g2sh.sh diff --git a/jobs/rocoto/awips_20sh.sh b/jobs/rocoto/awips_20sh.sh deleted file mode 100755 index f4d1d3e66e..0000000000 --- a/jobs/rocoto/awips_20sh.sh +++ /dev/null @@ -1,67 +0,0 @@ -#! /usr/bin/env bash - -source "${HOMEgfs}/ush/preamble.sh" - -############################################################### -## Abstract: -## Inline awips driver script -## HOMEgfs : /full/path/to/workflow -## EXPDIR : /full/path/to/config/files -## CDATE : current analysis date (YYYYMMDDHH) -## CDUMP : cycle name (gdas / gfs) -## PDY : current date (YYYYMMDD) -## cyc : current cycle (HH) -############################################################### - -############################################################### -# Source FV3GFS workflow modules -source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" -status=$? -(( status != 0 )) && exit "${status}" - -export job="awips_20km" -export jobid="${job}.$$" - -source "${HOMEgfs}/ush/jjob_header.sh" -e "awips" -c "base awips" - -# shellcheck disable=SC2153 -fhrlst=$(echo "${FHRLST}" | sed -e 's/_/ /g; s/f/ /g; s/,/ /g') - -############################################################### - -################################################################################ -echo -echo "=============== BEGIN AWIPS ===============" - -for fhr3 in ${fhrlst}; do - fhr=$(( 10#${fhr3} )) - if (( fhr > FHMAX_GFS )); then - echo "Nothing to process for FHR = ${fhr3}, cycle" - continue - fi - - fhmin=0 - fhmax=84 - if (( fhr >= fhmin && fhr <= fhmax )); then - if ((fhr % 3 == 0)); then - export fcsthrs="${fhr3}" - "${AWIPS20SH}" - fi - fi - - fhmin=90 - fhmax=240 - if (( fhr >= fhmin && fhr <= fhmax )); then - if ((fhr % 6 == 0)); then - export fcsthrs="${fhr3}" - "${AWIPS20SH}" - fi - fi -done - - -############################################################### -# Force Exit out cleanly -if [[ ${KEEPDATA:-"NO"} == "NO" ]] ; then rm -rf "${DATA}" ; fi - -exit 0 diff --git a/jobs/rocoto/awips_g2sh.sh b/jobs/rocoto/awips_g2sh.sh deleted file mode 100755 index d0f751a488..0000000000 --- a/jobs/rocoto/awips_g2sh.sh +++ /dev/null @@ -1,66 +0,0 @@ -#! /usr/bin/env bash - -source "${HOMEgfs}/ush/preamble.sh" - -############################################################### -## Abstract: -## Inline awips driver script -## HOMEgfs : /full/path/to/workflow -## EXPDIR : /full/path/to/config/files -## CDATE : current analysis date (YYYYMMDDHH) -## CDUMP : cycle name (gdas / gfs) -## PDY : current date (YYYYMMDD) -## cyc : current cycle (HH) -############################################################### - -############################################################### -# Source FV3GFS workflow modules -source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" -status=$? -(( status != 0 )) && exit "${status}" - -export job="awips_g2sh" -export jobid="${job}.$$" - -source "${HOMEgfs}/ush/jjob_header.sh" -e "awips" -c "base awips" - -# shellcheck disable=SC2153 -fhrlst=$(echo "${FHRLST}" | sed -e "s/_/ /g; s/f/ /g; s/,/ /g") - -############################################################### - -################################################################################ -echo -echo "=============== BEGIN AWIPS ===============" - -for fhr3 in ${fhrlst}; do - fhr=$(( 10#${fhr3} )) - if (( fhr > FHMAX_GFS )); then - echo "Nothing to process for FHR = ${fhr3}, cycle" - continue - fi - - fhmin=0 - fhmax=84 - if (( fhr >= fhmin && fhr <= fhmax )); then - if ((fhr % 6 == 0)); then - "${AWIPSG2SH}" - fi - fi - - fhmin=90 - fhmax=240 - if (( fhr >= fhmin && fhr <= fhmax )); then - if ((fhr % 6 == 0)); then - export fcsthrs="${fhr3}" - "${AWIPSG2SH}" - fi - fi -done - - -############################################################### -# Force Exit out cleanly -if [[ ${KEEPDATA:-"NO"} == "NO" ]] ; then rm -rf "${DATA}" ; fi - -exit 0 From fb7fefb7893f0425098a758f33d48972d119e6b7 Mon Sep 17 00:00:00 2001 From: henrywinterbottom-wxdev Date: Fri, 1 Dec 2023 09:18:43 -0700 Subject: [PATCH 16/23] Updated task names. --- workflow/rocoto/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index 36b80d168e..f1aa680595 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -22,7 +22,7 @@ class Tasks: 'fcst', 'post', 'ocnpost', 'verfozn', 'verfrad', 'vminmon', 'metp', 'tracker', 'genesis', 'genesis_fsu', - 'postsnd', 'awips_g2sh', 'awips_20sh', 'fbwinds', 'gempak', + 'postsnd', 'awips_g2', 'awips_20km_1p0', 'fbwinds', 'gempak', 'waveawipsbulls', 'waveawipsgridded', 'wavegempak', 'waveinit', 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt', 'wavepostsbs', 'waveprep', 'npoess'] From 3347e8e66a7b95df9415e7e28f8affb0f8fdaab9 Mon Sep 17 00:00:00 2001 From: henrywinterbottom-wxdev Date: Fri, 1 Dec 2023 09:19:55 -0700 Subject: [PATCH 17/23] Renamed file to reflect child JJOB. --- jobs/rocoto/awips_20km_1p0deg.sh | 67 ++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100755 jobs/rocoto/awips_20km_1p0deg.sh diff --git a/jobs/rocoto/awips_20km_1p0deg.sh b/jobs/rocoto/awips_20km_1p0deg.sh new file mode 100755 index 0000000000..f4d1d3e66e --- /dev/null +++ b/jobs/rocoto/awips_20km_1p0deg.sh @@ -0,0 +1,67 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +## Abstract: +## Inline awips driver script +## HOMEgfs : /full/path/to/workflow +## EXPDIR : /full/path/to/config/files +## CDATE : current analysis date (YYYYMMDDHH) +## CDUMP : cycle name (gdas / gfs) +## PDY : current date (YYYYMMDD) +## cyc : current cycle (HH) +############################################################### + +############################################################### +# Source FV3GFS workflow modules +source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" +status=$? +(( status != 0 )) && exit "${status}" + +export job="awips_20km" +export jobid="${job}.$$" + +source "${HOMEgfs}/ush/jjob_header.sh" -e "awips" -c "base awips" + +# shellcheck disable=SC2153 +fhrlst=$(echo "${FHRLST}" | sed -e 's/_/ /g; s/f/ /g; s/,/ /g') + +############################################################### + +################################################################################ +echo +echo "=============== BEGIN AWIPS ===============" + +for fhr3 in ${fhrlst}; do + fhr=$(( 10#${fhr3} )) + if (( fhr > FHMAX_GFS )); then + echo "Nothing to process for FHR = ${fhr3}, cycle" + continue + fi + + fhmin=0 + fhmax=84 + if (( fhr >= fhmin && fhr <= fhmax )); then + if ((fhr % 3 == 0)); then + export fcsthrs="${fhr3}" + "${AWIPS20SH}" + fi + fi + + fhmin=90 + fhmax=240 + if (( fhr >= fhmin && fhr <= fhmax )); then + if ((fhr % 6 == 0)); then + export fcsthrs="${fhr3}" + "${AWIPS20SH}" + fi + fi +done + + +############################################################### +# Force Exit out cleanly +if [[ ${KEEPDATA:-"NO"} == "NO" ]] ; then rm -rf "${DATA}" ; fi + +exit 0 From 993956cbedb91f154650c68788ec3a3e739d114f Mon Sep 17 00:00:00 2001 From: henrywinterbottom-wxdev Date: Fri, 1 Dec 2023 09:22:24 -0700 Subject: [PATCH 18/23] Updates to reflect upstream file updates. --- workflow/rocoto/gfs_tasks.py | 6 +++--- workflow/rocoto/tasks.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 0c9d0f0db0..9fa574e64c 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -822,7 +822,7 @@ def fbwinds(self): return task - def awips_20km_1p0(self): + def awips_20km_1p0deg(self): def _get_awipsgroups(cdump, config): @@ -877,8 +877,8 @@ def _get_awipsgroups(cdump, config): vardict = {varname2: varval2, varname3: varval3} resources = self.get_resource('awips') - task = create_wf_task('awips_20km_1p0', resources, cdump=self.cdump, envar=awipsenvars, dependency=dependencies, - metatask='awips_20km_1p0', varname=varname1, varval=varval1, vardict=vardict) + task = create_wf_task('awips_20km_1p0deg', resources, cdump=self.cdump, envar=awipsenvars, dependency=dependencies, + metatask='awips_20km_1p0deg', varname=varname1, varval=varval1, vardict=vardict) return task diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index f1aa680595..aa52e4fa9b 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -22,7 +22,7 @@ class Tasks: 'fcst', 'post', 'ocnpost', 'verfozn', 'verfrad', 'vminmon', 'metp', 'tracker', 'genesis', 'genesis_fsu', - 'postsnd', 'awips_g2', 'awips_20km_1p0', 'fbwinds', 'gempak', + 'postsnd', 'awips_g2', 'awips_20km_1p0deg', 'fbwinds', 'gempak', 'waveawipsbulls', 'waveawipsgridded', 'wavegempak', 'waveinit', 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt', 'wavepostsbs', 'waveprep', 'npoess'] From 78aef530a06f81f5f75166c4a2e59c85de2d8222 Mon Sep 17 00:00:00 2001 From: henrywinterbottom-wxdev Date: Fri, 1 Dec 2023 09:22:44 -0700 Subject: [PATCH 19/23] File has been renamed. --- jobs/rocoto/awips_20km_1p0.sh | 67 ----------------------------------- 1 file changed, 67 deletions(-) delete mode 100755 jobs/rocoto/awips_20km_1p0.sh diff --git a/jobs/rocoto/awips_20km_1p0.sh b/jobs/rocoto/awips_20km_1p0.sh deleted file mode 100755 index f4d1d3e66e..0000000000 --- a/jobs/rocoto/awips_20km_1p0.sh +++ /dev/null @@ -1,67 +0,0 @@ -#! /usr/bin/env bash - -source "${HOMEgfs}/ush/preamble.sh" - -############################################################### -## Abstract: -## Inline awips driver script -## HOMEgfs : /full/path/to/workflow -## EXPDIR : /full/path/to/config/files -## CDATE : current analysis date (YYYYMMDDHH) -## CDUMP : cycle name (gdas / gfs) -## PDY : current date (YYYYMMDD) -## cyc : current cycle (HH) -############################################################### - -############################################################### -# Source FV3GFS workflow modules -source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" -status=$? -(( status != 0 )) && exit "${status}" - -export job="awips_20km" -export jobid="${job}.$$" - -source "${HOMEgfs}/ush/jjob_header.sh" -e "awips" -c "base awips" - -# shellcheck disable=SC2153 -fhrlst=$(echo "${FHRLST}" | sed -e 's/_/ /g; s/f/ /g; s/,/ /g') - -############################################################### - -################################################################################ -echo -echo "=============== BEGIN AWIPS ===============" - -for fhr3 in ${fhrlst}; do - fhr=$(( 10#${fhr3} )) - if (( fhr > FHMAX_GFS )); then - echo "Nothing to process for FHR = ${fhr3}, cycle" - continue - fi - - fhmin=0 - fhmax=84 - if (( fhr >= fhmin && fhr <= fhmax )); then - if ((fhr % 3 == 0)); then - export fcsthrs="${fhr3}" - "${AWIPS20SH}" - fi - fi - - fhmin=90 - fhmax=240 - if (( fhr >= fhmin && fhr <= fhmax )); then - if ((fhr % 6 == 0)); then - export fcsthrs="${fhr3}" - "${AWIPS20SH}" - fi - fi -done - - -############################################################### -# Force Exit out cleanly -if [[ ${KEEPDATA:-"NO"} == "NO" ]] ; then rm -rf "${DATA}" ; fi - -exit 0 From 2977c869f388cc31535d4a8bbafccf4daf41e0bb Mon Sep 17 00:00:00 2001 From: henrywinterbottom-wxdev Date: Fri, 1 Dec 2023 09:24:38 -0700 Subject: [PATCH 20/23] Updated task names based on upstream commits. --- workflow/applications/gfs_cycled.py | 2 +- workflow/applications/gfs_forecast_only.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index b04f9a45de..4e0897f739 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -234,7 +234,7 @@ def get_task_names(self): gfs_tasks += ['gempak'] if self.do_awips: - gfs_tasks += ['awips_20km_1p0'] + gfs_tasks += ['awips_20km_1p0deg'] gfs_tasks += ['awips_g2'] gfs_tasks += ['fbwinds'] diff --git a/workflow/applications/gfs_forecast_only.py b/workflow/applications/gfs_forecast_only.py index 51a5976996..1790c86a2c 100644 --- a/workflow/applications/gfs_forecast_only.py +++ b/workflow/applications/gfs_forecast_only.py @@ -119,7 +119,7 @@ def get_task_names(self): tasks += ['gempak'] if self.do_awips: - tasks += ['awips_20km_1p0'] + tasks += ['awips_20km_1p0deg'] tasks += ['awips_g2'] tasks += ['fbwinds'] From 9ba22875364e59e72f8c6b2504afaf38d3021077 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 1 Dec 2023 15:52:34 -0500 Subject: [PATCH 21/23] patch to reviewer comments on PR 2094 --- jobs/rocoto/awips_20km_1p0deg.sh | 6 +- jobs/rocoto/awips_g2.sh | 2 +- parm/config/gfs/config.awips | 6 +- workflow/rocoto/gfs_tasks.py | 109 +++++++++++-------------------- 4 files changed, 44 insertions(+), 79 deletions(-) diff --git a/jobs/rocoto/awips_20km_1p0deg.sh b/jobs/rocoto/awips_20km_1p0deg.sh index f4d1d3e66e..e1bf623883 100755 --- a/jobs/rocoto/awips_20km_1p0deg.sh +++ b/jobs/rocoto/awips_20km_1p0deg.sh @@ -19,7 +19,7 @@ source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" status=$? (( status != 0 )) && exit "${status}" -export job="awips_20km" +export job="awips_20km_1p0deg" export jobid="${job}.$$" source "${HOMEgfs}/ush/jjob_header.sh" -e "awips" -c "base awips" @@ -45,7 +45,7 @@ for fhr3 in ${fhrlst}; do if (( fhr >= fhmin && fhr <= fhmax )); then if ((fhr % 3 == 0)); then export fcsthrs="${fhr3}" - "${AWIPS20SH}" + "${AWIPS20KM1P0DEGSH}" fi fi @@ -54,7 +54,7 @@ for fhr3 in ${fhrlst}; do if (( fhr >= fhmin && fhr <= fhmax )); then if ((fhr % 6 == 0)); then export fcsthrs="${fhr3}" - "${AWIPS20SH}" + "${AWIPS20KM1P0DEGSH}" fi fi done diff --git a/jobs/rocoto/awips_g2.sh b/jobs/rocoto/awips_g2.sh index d0f751a488..b94a67b242 100755 --- a/jobs/rocoto/awips_g2.sh +++ b/jobs/rocoto/awips_g2.sh @@ -19,7 +19,7 @@ source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" status=$? (( status != 0 )) && exit "${status}" -export job="awips_g2sh" +export job="awips_g2" export jobid="${job}.$$" source "${HOMEgfs}/ush/jjob_header.sh" -e "awips" -c "base awips" diff --git a/parm/config/gfs/config.awips b/parm/config/gfs/config.awips index 9003e9f6b0..3b78d4bb4b 100644 --- a/parm/config/gfs/config.awips +++ b/parm/config/gfs/config.awips @@ -6,10 +6,10 @@ echo "BEGIN: config.awips" # Get task specific resources -. $EXPDIR/config.resources awips +. "${EXPDIR}/config.resources" awips -export AWIPS20SH=$HOMEgfs/jobs/JGFS_ATMOS_AWIPS_20KM_1P0DEG -export AWIPSG2SH=$HOMEgfs/jobs/JGFS_ATMOS_AWIPS_G2 +export AWIPS20KM1P0DEGSH="${HOMEgfs}/jobs/JGFS_ATMOS_AWIPS_20KM_1P0DEG" +export AWIPSG2SH="${HOMEgfs}/jobs/JGFS_ATMOS_AWIPS_G2" # No. of concurrent awips jobs export NAWIPSGRP=42 diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 9fa574e64c..f69579b068 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -822,43 +822,44 @@ def fbwinds(self): return task - def awips_20km_1p0deg(self): - - def _get_awipsgroups(cdump, config): - - fhmin = config['FHMIN'] - fhmax = config['FHMAX'] - fhout = config['FHOUT'] - - # Get a list of all forecast hours - fhrs = [] - if cdump in ['gdas']: - fhrs = range(fhmin, fhmax + fhout, fhout) - elif cdump in ['gfs']: - fhmax = np.max( - [config['FHMAX_GFS_00'], config['FHMAX_GFS_06'], config['FHMAX_GFS_12'], config['FHMAX_GFS_18']]) - fhout = config['FHOUT_GFS'] - fhmax_hf = config['FHMAX_HF_GFS'] - fhout_hf = config['FHOUT_HF_GFS'] - if fhmax > 240: - fhmax = 240 - if fhmax_hf > 240: - fhmax_hf = 240 - fhrs_hf = list(range(fhmin, fhmax_hf + fhout_hf, fhout_hf)) - fhrs = fhrs_hf + list(range(fhrs_hf[-1] + fhout, fhmax + fhout, fhout)) + @staticmethod + def _get_awipsgroups(cdump, config): - nawipsgrp = config['NAWIPSGRP'] - ngrps = nawipsgrp if len(fhrs) > nawipsgrp else len(fhrs) + fhmin = config['FHMIN'] + fhmax = config['FHMAX'] + fhout = config['FHOUT'] - fhrs = [f'f{fhr:03d}' for fhr in fhrs] - fhrs = np.array_split(fhrs, ngrps) - fhrs = [fhr.tolist() for fhr in fhrs] - - grp = ' '.join([f'_{fhr[0]}-{fhr[-1]}' for fhr in fhrs]) - dep = ' '.join([fhr[-1] for fhr in fhrs]) - lst = ' '.join(['_'.join(fhr) for fhr in fhrs]) + # Get a list of all forecast hours + fhrs = [] + if cdump in ['gdas']: + fhrs = range(fhmin, fhmax + fhout, fhout) + elif cdump in ['gfs']: + fhmax = np.max( + [config['FHMAX_GFS_00'], config['FHMAX_GFS_06'], config['FHMAX_GFS_12'], config['FHMAX_GFS_18']]) + fhout = config['FHOUT_GFS'] + fhmax_hf = config['FHMAX_HF_GFS'] + fhout_hf = config['FHOUT_HF_GFS'] + if fhmax > 240: + fhmax = 240 + if fhmax_hf > 240: + fhmax_hf = 240 + fhrs_hf = list(range(fhmin, fhmax_hf + fhout_hf, fhout_hf)) + fhrs = fhrs_hf + list(range(fhrs_hf[-1] + fhout, fhmax + fhout, fhout)) + + nawipsgrp = config['NAWIPSGRP'] + ngrps = nawipsgrp if len(fhrs) > nawipsgrp else len(fhrs) + + fhrs = [f'f{fhr:03d}' for fhr in fhrs] + fhrs = np.array_split(fhrs, ngrps) + fhrs = [fhr.tolist() for fhr in fhrs] + + grp = ' '.join([f'_{fhr[0]}-{fhr[-1]}' for fhr in fhrs]) + dep = ' '.join([fhr[-1] for fhr in fhrs]) + lst = ' '.join(['_'.join(fhr) for fhr in fhrs]) + + return grp, dep, lst - return grp, dep, lst + def awips_20km_1p0deg(self): deps = [] dep_dict = {'type': 'metatask', 'name': f'{self.cdump}post'} @@ -873,7 +874,7 @@ def _get_awipsgroups(cdump, config): awipsenvars.append(rocoto.create_envar(name=key, value=str(value))) varname1, varname2, varname3 = 'grp', 'dep', 'lst' - varval1, varval2, varval3 = _get_awipsgroups(self.cdump, self._configs['awips']) + varval1, varval2, varval3 = self._get_awipsgroups(self.cdump, self._configs['awips']) vardict = {varname2: varval2, varname3: varval3} resources = self.get_resource('awips') @@ -884,42 +885,6 @@ def _get_awipsgroups(cdump, config): def awips_g2(self): - def _get_awipsgroups(cdump, config): - - fhmin = config['FHMIN'] - fhmax = config['FHMAX'] - fhout = config['FHOUT'] - - # Get a list of all forecast hours - fhrs = [] - if cdump in ['gdas']: - fhrs = range(fhmin, fhmax + fhout, fhout) - elif cdump in ['gfs']: - fhmax = np.max( - [config['FHMAX_GFS_00'], config['FHMAX_GFS_06'], config['FHMAX_GFS_12'], config['FHMAX_GFS_18']]) - fhout = config['FHOUT_GFS'] - fhmax_hf = config['FHMAX_HF_GFS'] - fhout_hf = config['FHOUT_HF_GFS'] - if fhmax > 240: - fhmax = 240 - if fhmax_hf > 240: - fhmax_hf = 240 - fhrs_hf = list(range(fhmin, fhmax_hf + fhout_hf, fhout_hf)) - fhrs = fhrs_hf + list(range(fhrs_hf[-1] + fhout, fhmax + fhout, fhout)) - - nawipsgrp = config['NAWIPSGRP'] - ngrps = nawipsgrp if len(fhrs) > nawipsgrp else len(fhrs) - - fhrs = [f'f{fhr:03d}' for fhr in fhrs] - fhrs = np.array_split(fhrs, ngrps) - fhrs = [fhr.tolist() for fhr in fhrs] - - grp = ' '.join([f'_{fhr[0]}-{fhr[-1]}' for fhr in fhrs]) - dep = ' '.join([fhr[-1] for fhr in fhrs]) - lst = ' '.join(['_'.join(fhr) for fhr in fhrs]) - - return grp, dep, lst - deps = [] dep_dict = {'type': 'metatask', 'name': f'{self.cdump}post'} deps.append(rocoto.add_dependency(dep_dict)) @@ -933,7 +898,7 @@ def _get_awipsgroups(cdump, config): awipsenvars.append(rocoto.create_envar(name=key, value=str(value))) varname1, varname2, varname3 = 'grp', 'dep', 'lst' - varval1, varval2, varval3 = _get_awipsgroups(self.cdump, self._configs['awips']) + varval1, varval2, varval3 = self._get_awipsgroups(self.cdump, self._configs['awips']) vardict = {varname2: varval2, varname3: varval3} resources = self.get_resource('awips') From 1066a725f62a517bbefdd549dd364785412f79b1 Mon Sep 17 00:00:00 2001 From: "Henry R. Winterbottom" <49202169+HenryWinterbottom-NOAA@users.noreply.github.com> Date: Mon, 4 Dec 2023 17:26:16 -0700 Subject: [PATCH 22/23] Update jobs/rocoto/awips_g2.sh Co-authored-by: Rahul Mahajan --- jobs/rocoto/awips_g2.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/rocoto/awips_g2.sh b/jobs/rocoto/awips_g2.sh index b94a67b242..2243d3e809 100755 --- a/jobs/rocoto/awips_g2.sh +++ b/jobs/rocoto/awips_g2.sh @@ -41,7 +41,7 @@ for fhr3 in ${fhrlst}; do fi fhmin=0 - fhmax=84 + fhmax=240 if (( fhr >= fhmin && fhr <= fhmax )); then if ((fhr % 6 == 0)); then "${AWIPSG2SH}" From e9c9209c4aec8fdd83594c70d8d53b677157fce9 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Mon, 4 Dec 2023 20:08:30 -0500 Subject: [PATCH 23/23] Update awips_g2.sh --- jobs/rocoto/awips_g2.sh | 9 --------- 1 file changed, 9 deletions(-) diff --git a/jobs/rocoto/awips_g2.sh b/jobs/rocoto/awips_g2.sh index 2243d3e809..121c96d63f 100755 --- a/jobs/rocoto/awips_g2.sh +++ b/jobs/rocoto/awips_g2.sh @@ -47,15 +47,6 @@ for fhr3 in ${fhrlst}; do "${AWIPSG2SH}" fi fi - - fhmin=90 - fhmax=240 - if (( fhr >= fhmin && fhr <= fhmax )); then - if ((fhr % 6 == 0)); then - export fcsthrs="${fhr3}" - "${AWIPSG2SH}" - fi - fi done