From ec4b800b53cd904c7a408f5e0c548c1ee4b98dd1 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Mon, 20 May 2024 20:30:32 +0000 Subject: [PATCH 001/141] Add task to extract reforecast variables A rocoto task has been added to process the reforecast variables. A config file has been created for this task. Resources specific to this task have also been added to config.resources. --- parm/config/gefs/config.extractvars | 39 +++++++++++++++++++++++++++++ parm/config/gefs/config.resources | 12 +++++++++ workflow/applications/gefs.py | 4 ++- workflow/rocoto/gefs_tasks.py | 36 ++++++++++++++++++++++++++ 4 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 parm/config/gefs/config.extractvars diff --git a/parm/config/gefs/config.extractvars b/parm/config/gefs/config.extractvars new file mode 100644 index 0000000000..a40e9e7e35 --- /dev/null +++ b/parm/config/gefs/config.extractvars @@ -0,0 +1,39 @@ +#! /usr/bin/env bash + +########## config.extractvars ########## +# Extractvars specific + +echo "BEGIN: config.extractvars" + +. "${EXPDIR}/config.resources" extractvars + +export RFHOME=/lfs/h2/emc/ens/noscrub/eric.sinsky/GEFSv13_RR +export REFCST=${RFHOME}/REFCST + +export mpmdd="NO" +export atmosextract=1 +export ocniceextract=1 +export wavextract=1 + +export compress_ocn=0 +export compress_ice=0 +export interp_ocn=0 +export interp_wav=1 + +export EXTRCTVARA=${USHgfs}/gefs_atmos_extractvars.sh +export EXTRCTVARO=${USHgfs}/gefs_ocn_extractvars.sh +export EXTRCTVARW=${USHgfs}/gefs_wav_extractvars.sh + +export varlist_2d=${PARMgfs}/product/gefs_shortparmlist_2d.parm +export varlist_3d=${PARMgfs}/product/gefs_shortparmlist_3d_h.parm +export varlist_3d_d=${PARMgfs}/product/gefs_shortparmlist_3d_d.parm + +export varlist_wav=${PARMgfs}/product/gefs_wav_shortparmlist.parm + +export fhmax=222 +export FHMAXHF=240 +export FHOUTHF=3 +export FHOUTLF=6 + + +echo "END: config.extractvars" diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index 04d55ae082..b355a9842d 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -293,6 +293,18 @@ case ${step} in export NTASKS=${npe_wavepostpnt} ;; + "extractvars") + export wtime_extractvars="00:30:00" + export npe_extractvars=6 + export nth_extractvars=1 + export npe_node_extractvars="${npe_extractvars}" + export wtime_extractvars_gfs="${wtime_extractvars}" + export npe_extractvars_gfs="${npe_extractvars}" + export nth_extractvars_gfs="${nth_extractvars}" + export npe_node_extractvars_gfs="${npe_node_extractvars}" + export is_exclusive=True + ;; + *) echo "FATAL ERROR: Invalid job ${step} passed to ${BASH_SOURCE[0]}" exit 1 diff --git a/workflow/applications/gefs.py b/workflow/applications/gefs.py index c165f9d1ca..15ff036ec5 100644 --- a/workflow/applications/gefs.py +++ b/workflow/applications/gefs.py @@ -14,7 +14,7 @@ def _get_app_configs(self): """ Returns the config_files that are involved in gefs """ - configs = ['stage_ic', 'fcst', 'atmos_products'] + configs = ['stage_ic', 'fcst', 'atmos_products', 'extractvars'] if self.nens > 0: configs += ['efcs', 'atmos_ensstat'] @@ -73,4 +73,6 @@ def get_task_names(self): tasks += ['wavepostbndpnt', 'wavepostbndpntbll'] tasks += ['wavepostpnt'] + tasks += ['extractvars'] + return {f"{self._base['CDUMP']}": tasks} diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 86be494549..64f1d50111 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -268,6 +268,42 @@ def _atmosoceaniceprod(self, component: str): return task + def extractvars(self): + deps = [] + dep_dict = {'type': 'task', 'name': f'ocean_prod_#member#'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + + extractvars_envars = self.envars.copy() + extractvars_dict = {'ENSMEM': '#member#', + 'MEMDIR': 'mem#member#', + } + for key, value in extractvars_dict.items(): + extractvars_envars.append(rocoto.create_envar(name=key, value=str(value))) + + resources = self.get_resource('extractvars') + task_name = f'extractvars_mem#member#' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': extractvars_envars, + 'cycledef': 'gefs', + 'command': f'{self.HOMEgfs}/jobs/rocoto/extractvars.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + member_var_dict = {'member': ' '.join([str(mem).zfill(3) for mem in range(0, self.nmem + 1)])} + member_metatask_dict = {'task_name': 'extractvars', + 'task_dict': task_dict, + 'var_dict': member_var_dict + } + + task = rocoto.create_task(member_metatask_dict) + + return task + def atmos_ensstat(self): resources = self.get_resource('atmos_ensstat') From 292378020af79e259e0c6e45a3eecf48a4ffefa2 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 21 May 2024 19:29:30 +0000 Subject: [PATCH 002/141] Add top level script for extractvars task The topmost extractvars script has been added to the global workflow rocoto subdirectory. --- jobs/rocoto/extractvars.sh | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100755 jobs/rocoto/extractvars.sh diff --git a/jobs/rocoto/extractvars.sh b/jobs/rocoto/extractvars.sh new file mode 100755 index 0000000000..1950a7be04 --- /dev/null +++ b/jobs/rocoto/extractvars.sh @@ -0,0 +1,23 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +echo +echo "=============== START TO SOURCE FV3GFS WORKFLOW MODULES ===============" +. "${HOMEgfs}/ush/load_fv3gfs_modules.sh" +status=$? +[[ ${status} -ne 0 ]] && exit ${status} + +export job="extractvars" +export jobid="${job}.$$" + +############################################################### +echo +echo "=============== START TO RUN EXTRACTVARS ===============" +# Execute the JJOB +${HOMEgfs}/jobs/JGLOBAL_EXTRACTVARS +status=$? +[[ ${status} -ne 0 ]] && exit ${status} + +exit 0 From 8845dc73678f11b62124d70c09b37562b5605a91 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 28 May 2024 17:09:14 +0000 Subject: [PATCH 003/141] Update extractvars job dependency The extractvars job dependency has been updated so that extractvars is triggered by the completion of wave_post_grid. --- workflow/rocoto/gefs_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 64f1d50111..1fe14f04d9 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -270,7 +270,7 @@ def _atmosoceaniceprod(self, component: str): def extractvars(self): deps = [] - dep_dict = {'type': 'task', 'name': f'ocean_prod_#member#'} + dep_dict = {'type': 'task', 'name': f'wave_post_grid_mem#member#'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) From f46544e1449644a4693eaecab661e15aedcf3546 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 29 May 2024 13:45:17 +0000 Subject: [PATCH 004/141] Add JGLOBAL script for extractvars task A new JGLOBAL script has been added for the extractvars task. --- jobs/JGLOBAL_EXTRACTVARS | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100755 jobs/JGLOBAL_EXTRACTVARS diff --git a/jobs/JGLOBAL_EXTRACTVARS b/jobs/JGLOBAL_EXTRACTVARS new file mode 100755 index 0000000000..d18e7966f3 --- /dev/null +++ b/jobs/JGLOBAL_EXTRACTVARS @@ -0,0 +1,32 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" +source "${HOMEgfs}/ush/jjob_header.sh" -e "extractvars" -c "base extractvars" + +# Set COM Paths +for grid in '0p25' '0p50' '1p00'; do + prod_dir="COM_ATMOS_GRIB_${grid}" + GRID=${grid} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx "${prod_dir}:COM_ATMOS_GRIB_GRID_TMPL" + if [[ ! -d "${!prod_dir}" ]]; then mkdir -m 775 -p "${!prod_dir}"; fi +done + +YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COM_OCEAN_HISTORY" +YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COM_OCEAN_GRIB" +YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COM_OCEAN_NETCDF" +YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COM_ICE_HISTORY" +YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COM_ICE_GRIB" +YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COM_ICE_NETCDF" + +YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COM_WAVE_GRID" + +# Execute the Script +${SCRgfs}/exglobal_extractvars.sh + +########################################## +# Remove the Temporary working directory +########################################## +cd ${DATAROOT} +[[ ${KEEPDATA} = "NO" ]] && rm -rf ${DATA} + + +exit 0 From e9702c8e6e876501be9f5733a4af8a3e1970be65 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 29 May 2024 18:44:18 +0000 Subject: [PATCH 005/141] Add scripts to process atmos reforecast variables Scripts have been added which allow the processing of atmosphere reforecast variables. --- parm/config/gefs/config.extractvars | 13 ++-- scripts/exglobal_extractvars.sh | 32 +++++++++ ush/gefs_atmos_extractvars.sh | 102 ++++++++++++++++++++++++++++ 3 files changed, 138 insertions(+), 9 deletions(-) create mode 100755 scripts/exglobal_extractvars.sh create mode 100755 ush/gefs_atmos_extractvars.sh diff --git a/parm/config/gefs/config.extractvars b/parm/config/gefs/config.extractvars index a40e9e7e35..bdb564b353 100644 --- a/parm/config/gefs/config.extractvars +++ b/parm/config/gefs/config.extractvars @@ -20,20 +20,15 @@ export compress_ice=0 export interp_ocn=0 export interp_wav=1 -export EXTRCTVARA=${USHgfs}/gefs_atmos_extractvars.sh -export EXTRCTVARO=${USHgfs}/gefs_ocn_extractvars.sh -export EXTRCTVARW=${USHgfs}/gefs_wav_extractvars.sh - export varlist_2d=${PARMgfs}/product/gefs_shortparmlist_2d.parm export varlist_3d=${PARMgfs}/product/gefs_shortparmlist_3d_h.parm export varlist_3d_d=${PARMgfs}/product/gefs_shortparmlist_3d_d.parm export varlist_wav=${PARMgfs}/product/gefs_wav_shortparmlist.parm -export fhmax=222 -export FHMAXHF=240 -export FHOUTHF=3 -export FHOUTLF=6 - +export fhmax=120 +export FHMAXHF=60 +export FHOUTHF=6 +export FHOUTLF=12 echo "END: config.extractvars" diff --git a/scripts/exglobal_extractvars.sh b/scripts/exglobal_extractvars.sh new file mode 100755 index 0000000000..9b3a32b122 --- /dev/null +++ b/scripts/exglobal_extractvars.sh @@ -0,0 +1,32 @@ +#! /usr/bin/env bash +################################################################################ +# Script: +# +source "${USHgfs}/preamble.sh" + +# Programs used +export WGRIB2=${WGRIB2:-${wgrib2_ROOT}/bin/wgrib2} + +# Scripts used +EXTRCTVARA=${EXTRCTVARA:-"${USHgfs}/gefs_atmos_extractvars.sh"} +EXTRCTVARO=${EXTRCTVARO:-"${USHgfs}/gefs_ocn_extractvars.sh"} +EXTRCTVARW=${EXTRCTVARW:-"${USHgfs}/gefs_wav_extractvars.sh"} + +#Paramater Tables used +export varlist_2d=${varlist_2d:-"${PARMgfs}/product/gefs_shortparmlist_2d.parm"} #Parameter table for surface atmos variables +export varlist_3d=${varlist_3d:-"${PARMgfs}/product/gefs_shortparmlist_3d_h.parm"} #Parameter table for upper air instantaneous variables +export varlist_3d_d=${varlist_3d_d:-"${PARMgfs}/product/gefs_shortparmlist_3d_d.parm"} #Parameter table for upper air daily-averaged variables +export varlist_wav=${varlist_wav:-"${PARMgfs}/product/gefs_wav_shortparmlist.parm"} #Parameter table for wave variables + +# Variables used in this job +export FHMIN=${FHMIN:-0} #The total number of lead hours +export fhmax=${fhmax:-120} #The total number of lead hours +export FHMAXHF=${FHMAXHF:-60} #The maximum lead hour that will use a high output frequency +export FHOUTHF=${FHOUTHF:-6} #The high output frequency (in hours) +export FHOUTLF=${FHOUTLF:-12} #The low output frequency (in hours) + +subdata="${DATA}/mem${ENSMEM}_atmos" +if [ ! -d ${subdata} ]; then mkdir -p ${subdata}; fi +${EXTRCTVARA} ${ENSMEM} ${subdata} + +exit 0 diff --git a/ush/gefs_atmos_extractvars.sh b/ush/gefs_atmos_extractvars.sh new file mode 100755 index 0000000000..f7f6a69158 --- /dev/null +++ b/ush/gefs_atmos_extractvars.sh @@ -0,0 +1,102 @@ +#! /usr/bin/env bash +################################################################################ +# Script: +# +source "${USHgfs}/preamble.sh" + +fcnt=1 +dcnt=1 +ensname=$1 +subdata=$2 + +cd $subdata + +for outtype in "f2d" "f3d"; do + + if [[ "$outtype" == "f2d" ]];then + varlist=$varlist_2d + outdirpre="${subdata}/f2d" + elif [[ "$outtype" == "f3d" ]];then + varlist=$varlist_3d + varlist_d=$varlist_3d_d + outdirpre="${subdata}/f3d" + fi + + if [ ! -d ${outdirpre} ]; then mkdir -p ${outdirpre}; fi + + nh=${FHMIN} + while [[ $nh -le $fhmax ]];do + fnh=`printf "%3.3d" ${nh}` + echo "extracting f${fnh}" + + if [[ "$outtype" == "f2d" ]];then + if [[ $nh -le $FHMAXHF ]];then + outres="0p25" + else + outres="0p50" + fi + elif [[ "$outtype" == "f3d" ]];then + outres="1p00" + fi + + if [[ $nh -lt $FHMAXHF ]];then + outfreq=$FHOUTHF + else + outfreq=$FHOUTLF + fi + + echo $ensname ============== + if [[ "$outres" == "0p25" ]];then + infile1=$COM_ATMOS_GRIB_0p25/gefs.${cycle}.pgrb2.$outres.f${fnh} + infile2=$COM_ATMOS_GRIB_0p25/gefs.${cycle}.pgrb2b.$outres.f${fnh} + elif [[ "$outres" == "0p50" ]];then + infile1=$COM_ATMOS_GRIB_0p50/gefs.${cycle}.pgrb2.$outres.f${fnh} + infile2=$COM_ATMOS_GRIB_0p50/gefs.${cycle}.pgrb2b.$outres.f${fnh} + elif [[ "$outres" == "1p00" ]];then + infile1=$COM_ATMOS_GRIB_1p00/gefs.${cycle}.pgrb2.$outres.f${fnh} + infile2=$COM_ATMOS_GRIB_1p00/gefs.${cycle}.pgrb2b.$outres.f${fnh} + fi + oufile=$outdirpre/ge${ensname}.${cycle}.pgrb2.$outres.f${fnh} + rm -f $oufile #remove outfile if it already exists before extraction + + if [ -f $infile1 ]; then #check if input file exists before extraction + $WGRIB2 $infile1 | grep -F -f $varlist | $WGRIB2 -i $infile1 -append -grib $oufile>/dev/null + else + echo "WARNING: $infile1 does not exist. Please check it." + fi + + if [ -f $infile2 ]; then #check if input file exists before extraction + $WGRIB2 $infile2 | grep -F -f $varlist | $WGRIB2 -i $infile2 -append -grib $oufile>/dev/null + else + echo "WARNING: $infile2 does not exist. Please check it." + fi + + #Compute daily average for a subset of variables + if [[ "$outtype" == "f3d" ]];then + if ! (( $nh % 6 ));then + outfile=$subdata/vartmp_raw_vari_ldy${dcnt}.${ensname}.grib2 + $WGRIB2 $infile1 | grep -F -f $varlist_d | $WGRIB2 -i $infile1 -append -grib $outfile>/dev/null + $WGRIB2 $infile2 | grep -F -f $varlist_d | $WGRIB2 -i $infile2 -append -grib $outfile>/dev/null + if [[ $fcnt -eq 4 ]];then + fnd=$(printf "%2.2d" ${dcnt}) + davg_file=$outdirpre/ge${ensname}.${cycle}.pgrb2.$outres.ldy${fnd} + vcnt=1 + while read vari; do + davgtmp=$subdata/ge${ensname}.${cycle}.tmp.pgrb2.$outres.ldy${fnd}.${vcnt} + $WGRIB2 $outfile | grep "$vari" | $WGRIB2 -i $outfile -fcst_ave 6hr $davgtmp>/dev/null + $WGRIB2 $davgtmp | $WGRIB2 -i $davgtmp -append -grib $davg_file>/dev/null + rm -f $davgtmp + vcnt=$(($vcnt + 1)) + done <$varlist_d #variable + fcnt=1 + dcnt=$(($dcnt + 1)) + else + fcnt=$(($fcnt + 1)) + fi #If at final lead hour of a given day + fi #if lead hour is divisible by 6 + fi #if outtype == f3d + nh=$(($nh + $outfreq)) + done #fhr +done #f2d,f3d + +exit 0 From e378727565d3c72ee0e31f3a4e399b29da9c0aca Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 29 May 2024 18:46:06 +0000 Subject: [PATCH 006/141] Add parm files for extractvars task Parm files have been added that provide a variable list specific to reforecast variables to be saved on disk. --- parm/product/gefs_shortparmlist_2d.parm | 38 +++++++++++++++++++ parm/product/gefs_shortparmlist_3d_d.parm | 34 +++++++++++++++++ parm/product/gefs_shortparmlist_3d_h.parm | 45 +++++++++++++++++++++++ parm/product/gefs_wav_shortparmlist.parm | 6 +++ 4 files changed, 123 insertions(+) create mode 100644 parm/product/gefs_shortparmlist_2d.parm create mode 100644 parm/product/gefs_shortparmlist_3d_d.parm create mode 100644 parm/product/gefs_shortparmlist_3d_h.parm create mode 100644 parm/product/gefs_wav_shortparmlist.parm diff --git a/parm/product/gefs_shortparmlist_2d.parm b/parm/product/gefs_shortparmlist_2d.parm new file mode 100644 index 0000000000..018b8c35ac --- /dev/null +++ b/parm/product/gefs_shortparmlist_2d.parm @@ -0,0 +1,38 @@ +PRES:surface: +:WEASD:surface: +:TMP:2 m above ground: +:TMP:surface: +:RH:2 m above ground: +:TMAX:2 m above ground: +:TMIN:2 m above ground: +:UGRD:10 m above ground: +:VGRD:10 m above ground: +:APCP:surface: +:CSNOW:surface: +:CICEP:surface: +:CFRZR:surface: +:CRAIN:surface: +:PWAT:entire atmosphere (considered as a single layer): +:TCDC:entire atmosphere (considered as a single layer): +:DSWRF:surface: +:DLWRF:surface: +:ULWRF:top of atmosphere: +:HLCY:3000-0 m above ground: +:CAPE:180-0 mb above ground: +:CIN:180-0 mb above ground: +:PRMSL:mean sea level: +:USWRF:surface: +:ULWRF:surface: +:TSOIL:0-0.1 m below ground: +:TSOIL:0.1-0.4 m below ground: +:SOILW:0-0.1 m below ground: +:SOILW:0.1-0.4 m below ground: +:SOILW:0.4-1 m below ground: +:SOILW:1-2 m below ground: +:PEVPR:surface: +:LHTFL:surface: +:SHTFL:surface: +:WATR:surface: +:TSNOWP:surface: +:FDNSSTMP:surface: +:HGT:highest tropospheric freezing level: diff --git a/parm/product/gefs_shortparmlist_3d_d.parm b/parm/product/gefs_shortparmlist_3d_d.parm new file mode 100644 index 0000000000..37a2678826 --- /dev/null +++ b/parm/product/gefs_shortparmlist_3d_d.parm @@ -0,0 +1,34 @@ +:UGRD:1 mb: +:UGRD:2 mb: +:UGRD:3 mb: +:UGRD:5 mb: +:UGRD:7 mb: +:UGRD:10 mb: +:UGRD:20 mb: +:UGRD:30 mb: +:UGRD:50 mb: +:UGRD:70 mb: +:VGRD:1 mb: +:VGRD:2 mb: +:VGRD:3 mb: +:VGRD:5 mb: +:VGRD:7 mb: +:VGRD:10 mb: +:VGRD:20 mb: +:VGRD:30 mb: +:VGRD:50 mb: +:VGRD:70 mb: +:TMP:1 mb: +:TMP:2 mb: +:TMP:3 mb: +:TMP:5 mb: +:TMP:7 mb: +:TMP:10 mb: +:TMP:20 mb: +:TMP:30 mb: +:TMP:50 mb: +:TMP:70 mb: +:HGT:10 mb: +:HGT:50 mb: +:O3MR:10 mb: +:O3MR:50 mb: diff --git a/parm/product/gefs_shortparmlist_3d_h.parm b/parm/product/gefs_shortparmlist_3d_h.parm new file mode 100644 index 0000000000..d7241f633c --- /dev/null +++ b/parm/product/gefs_shortparmlist_3d_h.parm @@ -0,0 +1,45 @@ +:HGT:100 mb: +:TMP:100 mb: +:UGRD:100 mb: +:VGRD:100 mb: +:O3MR:100 mb: +:HGT:200 mb: +:TMP:200 mb: +:RH:200 mb: +:UGRD:200 mb: +:VGRD:200 mb: +:HGT:250 mb: +:TMP:250 mb: +:RH:250 mb: +:UGRD:250 mb: +:VGRD:250 mb: +:HGT:500 mb: +:TMP:500 mb: +:RH:500 mb: +:UGRD:500 mb: +:VGRD:500 mb: +:HGT:700 mb: +:TMP:700 mb: +:RH:700 mb: +:UGRD:700 mb: +:VGRD:700 mb: +:HGT:850 mb: +:TMP:850 mb: +:RH:850 mb: +:VVEL:850 mb: +:UGRD:850 mb: +:VGRD:850 mb: +:HGT:925 mb: +:TMP:925 mb: +:RH:925 mb: +:UGRD:925 mb: +:VGRD:925 mb: +:TMP:1000 mb: +:RH:1000 mb: +:UGRD:1000 mb: +:VGRD:1000 mb: +:HGT:1000 mb: +:TMP:0.995 sigma level: +:RH:0.995 sigma level: +:UGRD:0.995 sigma level: +:VGRD:0.995 sigma level: diff --git a/parm/product/gefs_wav_shortparmlist.parm b/parm/product/gefs_wav_shortparmlist.parm new file mode 100644 index 0000000000..14b264a3b1 --- /dev/null +++ b/parm/product/gefs_wav_shortparmlist.parm @@ -0,0 +1,6 @@ +:UOGRD:surface: +:VOGRD:surface: +:UGRD:surface: +:VGRD:surface: +:ICEC:surface: +:HTSGW:surface: From ff36ef2a550761f9231511b7e31e97ff26d7ed45 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 29 May 2024 19:13:25 +0000 Subject: [PATCH 007/141] Refine extractvars scripts The extractvars scripts have been simplified. --- parm/config/gefs/config.extractvars | 2 +- scripts/exglobal_extractvars.sh | 3 ++- ush/gefs_atmos_extractvars.sh | 14 +++++++------- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/parm/config/gefs/config.extractvars b/parm/config/gefs/config.extractvars index bdb564b353..11cb77333e 100644 --- a/parm/config/gefs/config.extractvars +++ b/parm/config/gefs/config.extractvars @@ -26,7 +26,7 @@ export varlist_3d_d=${PARMgfs}/product/gefs_shortparmlist_3d_d.parm export varlist_wav=${PARMgfs}/product/gefs_wav_shortparmlist.parm -export fhmax=120 +export FHMAX=120 export FHMAXHF=60 export FHOUTHF=6 export FHOUTLF=12 diff --git a/scripts/exglobal_extractvars.sh b/scripts/exglobal_extractvars.sh index 9b3a32b122..b98e90cad3 100755 --- a/scripts/exglobal_extractvars.sh +++ b/scripts/exglobal_extractvars.sh @@ -20,10 +20,11 @@ export varlist_wav=${varlist_wav:-"${PARMgfs}/product/gefs_wav_shortparmlist.par # Variables used in this job export FHMIN=${FHMIN:-0} #The total number of lead hours -export fhmax=${fhmax:-120} #The total number of lead hours +export FHMAX=${FHMAX:-120} #The total number of lead hours export FHMAXHF=${FHMAXHF:-60} #The maximum lead hour that will use a high output frequency export FHOUTHF=${FHOUTHF:-6} #The high output frequency (in hours) export FHOUTLF=${FHOUTLF:-12} #The low output frequency (in hours) +export cycle=${cycle:-"t00z"} #cycle subdata="${DATA}/mem${ENSMEM}_atmos" if [ ! -d ${subdata} ]; then mkdir -p ${subdata}; fi diff --git a/ush/gefs_atmos_extractvars.sh b/ush/gefs_atmos_extractvars.sh index f7f6a69158..73bac9cc65 100755 --- a/ush/gefs_atmos_extractvars.sh +++ b/ush/gefs_atmos_extractvars.sh @@ -13,19 +13,18 @@ cd $subdata for outtype in "f2d" "f3d"; do - if [[ "$outtype" == "f2d" ]];then + if [[ "${outtype}" == "f2d" ]];then varlist=$varlist_2d - outdirpre="${subdata}/f2d" - elif [[ "$outtype" == "f3d" ]];then + elif [[ "${outtype}" == "f3d" ]];then varlist=$varlist_3d varlist_d=$varlist_3d_d - outdirpre="${subdata}/f3d" fi + outdirpre="${subdata}/${outtype}" if [ ! -d ${outdirpre} ]; then mkdir -p ${outdirpre}; fi nh=${FHMIN} - while [[ $nh -le $fhmax ]];do + while [[ $nh -le $FHMAX ]];do fnh=`printf "%3.3d" ${nh}` echo "extracting f${fnh}" @@ -62,13 +61,13 @@ for outtype in "f2d" "f3d"; do if [ -f $infile1 ]; then #check if input file exists before extraction $WGRIB2 $infile1 | grep -F -f $varlist | $WGRIB2 -i $infile1 -append -grib $oufile>/dev/null else - echo "WARNING: $infile1 does not exist. Please check it." + echo "WARNING: $infile1 does not exist." fi if [ -f $infile2 ]; then #check if input file exists before extraction $WGRIB2 $infile2 | grep -F -f $varlist | $WGRIB2 -i $infile2 -append -grib $oufile>/dev/null else - echo "WARNING: $infile2 does not exist. Please check it." + echo "WARNING: $infile2 does not exist." fi #Compute daily average for a subset of variables @@ -97,6 +96,7 @@ for outtype in "f2d" "f3d"; do fi #if outtype == f3d nh=$(($nh + $outfreq)) done #fhr + done #f2d,f3d exit 0 From c17397911b13bde6f927a1a3470d4bb933b5695d Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 29 May 2024 19:24:18 +0000 Subject: [PATCH 008/141] Add corrections Corrections were made to some new scripts. --- scripts/exglobal_extractvars.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/exglobal_extractvars.sh b/scripts/exglobal_extractvars.sh index b98e90cad3..ca2bbd177d 100755 --- a/scripts/exglobal_extractvars.sh +++ b/scripts/exglobal_extractvars.sh @@ -13,13 +13,13 @@ EXTRCTVARO=${EXTRCTVARO:-"${USHgfs}/gefs_ocn_extractvars.sh"} EXTRCTVARW=${EXTRCTVARW:-"${USHgfs}/gefs_wav_extractvars.sh"} #Paramater Tables used -export varlist_2d=${varlist_2d:-"${PARMgfs}/product/gefs_shortparmlist_2d.parm"} #Parameter table for surface atmos variables +export varlist_2d=${varlist_2d:-"${PARMgfs}/product/gefs_shortparmlist_2d.parm"} #Parameter table for surface variables export varlist_3d=${varlist_3d:-"${PARMgfs}/product/gefs_shortparmlist_3d_h.parm"} #Parameter table for upper air instantaneous variables export varlist_3d_d=${varlist_3d_d:-"${PARMgfs}/product/gefs_shortparmlist_3d_d.parm"} #Parameter table for upper air daily-averaged variables export varlist_wav=${varlist_wav:-"${PARMgfs}/product/gefs_wav_shortparmlist.parm"} #Parameter table for wave variables # Variables used in this job -export FHMIN=${FHMIN:-0} #The total number of lead hours +export FHMIN=${FHMIN:-0} #The first lead hour export FHMAX=${FHMAX:-120} #The total number of lead hours export FHMAXHF=${FHMAXHF:-60} #The maximum lead hour that will use a high output frequency export FHOUTHF=${FHOUTHF:-6} #The high output frequency (in hours) From 0deabb03a52913829ccac3ffcef9f4e7ae2feb7e Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Fri, 31 May 2024 19:34:19 +0000 Subject: [PATCH 009/141] Add script to process ocean and ice A script has been added to process the ocean and ice variables for the reforecast. Parm files for ocean and ice have also been added. --- parm/config/gefs/config.extractvars | 6 -- parm/product/gefs_ice_shortparmlist.parm | 10 ++++ .../product/gefs_ice_shortparmlist_grib2.parm | 2 + parm/product/gefs_ocn_shortparmlist.parm | 9 +++ .../product/gefs_ocn_shortparmlist_grib2.parm | 2 + scripts/exglobal_extractvars.sh | 34 +++++++++-- ush/gefs_ocnice_extractvars.sh | 58 +++++++++++++++++++ 7 files changed, 111 insertions(+), 10 deletions(-) create mode 100644 parm/product/gefs_ice_shortparmlist.parm create mode 100644 parm/product/gefs_ice_shortparmlist_grib2.parm create mode 100644 parm/product/gefs_ocn_shortparmlist.parm create mode 100644 parm/product/gefs_ocn_shortparmlist_grib2.parm create mode 100755 ush/gefs_ocnice_extractvars.sh diff --git a/parm/config/gefs/config.extractvars b/parm/config/gefs/config.extractvars index 11cb77333e..4b29139963 100644 --- a/parm/config/gefs/config.extractvars +++ b/parm/config/gefs/config.extractvars @@ -20,12 +20,6 @@ export compress_ice=0 export interp_ocn=0 export interp_wav=1 -export varlist_2d=${PARMgfs}/product/gefs_shortparmlist_2d.parm -export varlist_3d=${PARMgfs}/product/gefs_shortparmlist_3d_h.parm -export varlist_3d_d=${PARMgfs}/product/gefs_shortparmlist_3d_d.parm - -export varlist_wav=${PARMgfs}/product/gefs_wav_shortparmlist.parm - export FHMAX=120 export FHMAXHF=60 export FHOUTHF=6 diff --git a/parm/product/gefs_ice_shortparmlist.parm b/parm/product/gefs_ice_shortparmlist.parm new file mode 100644 index 0000000000..07db948fe3 --- /dev/null +++ b/parm/product/gefs_ice_shortparmlist.parm @@ -0,0 +1,10 @@ +aice_h +hi_h +Tsfc_h +uvel_h +vvel_h +hs_h +albsni_h +melts_h +meltb_h +frzmlt_h diff --git a/parm/product/gefs_ice_shortparmlist_grib2.parm b/parm/product/gefs_ice_shortparmlist_grib2.parm new file mode 100644 index 0000000000..f0e519d24e --- /dev/null +++ b/parm/product/gefs_ice_shortparmlist_grib2.parm @@ -0,0 +1,2 @@ +:ICEC:surface: +:ICETMP:surface: diff --git a/parm/product/gefs_ocn_shortparmlist.parm b/parm/product/gefs_ocn_shortparmlist.parm new file mode 100644 index 0000000000..6673ddb16e --- /dev/null +++ b/parm/product/gefs_ocn_shortparmlist.parm @@ -0,0 +1,9 @@ +temp +SST +SSH +SSS +MLD_003 +taux +tauy +SSU +SSV diff --git a/parm/product/gefs_ocn_shortparmlist_grib2.parm b/parm/product/gefs_ocn_shortparmlist_grib2.parm new file mode 100644 index 0000000000..958c4daa61 --- /dev/null +++ b/parm/product/gefs_ocn_shortparmlist_grib2.parm @@ -0,0 +1,2 @@ +:WTMP:241 m below sea level: +:WTMP:267 m below sea level: diff --git a/scripts/exglobal_extractvars.sh b/scripts/exglobal_extractvars.sh index ca2bbd177d..7ddc88f290 100755 --- a/scripts/exglobal_extractvars.sh +++ b/scripts/exglobal_extractvars.sh @@ -9,7 +9,7 @@ export WGRIB2=${WGRIB2:-${wgrib2_ROOT}/bin/wgrib2} # Scripts used EXTRCTVARA=${EXTRCTVARA:-"${USHgfs}/gefs_atmos_extractvars.sh"} -EXTRCTVARO=${EXTRCTVARO:-"${USHgfs}/gefs_ocn_extractvars.sh"} +EXTRCTVARO=${EXTRCTVARO:-"${USHgfs}/gefs_ocnice_extractvars.sh"} EXTRCTVARW=${EXTRCTVARW:-"${USHgfs}/gefs_wav_extractvars.sh"} #Paramater Tables used @@ -17,6 +17,10 @@ export varlist_2d=${varlist_2d:-"${PARMgfs}/product/gefs_shortparmlist_2d.parm"} export varlist_3d=${varlist_3d:-"${PARMgfs}/product/gefs_shortparmlist_3d_h.parm"} #Parameter table for upper air instantaneous variables export varlist_3d_d=${varlist_3d_d:-"${PARMgfs}/product/gefs_shortparmlist_3d_d.parm"} #Parameter table for upper air daily-averaged variables export varlist_wav=${varlist_wav:-"${PARMgfs}/product/gefs_wav_shortparmlist.parm"} #Parameter table for wave variables +export varlist_ocn_netcdf=${varlist_ocn_netcdf:-"${PARMgfs}/product/gefs_ocn_shortparmlist.parm"} #Parameter table for wave variables +export varlist_ice_netcdf=${varlist_ice_netcdf:-"${PARMgfs}/product/gefs_ice_shortparmlist.parm"} #Parameter table for wave variables +export varlist_ocn_grib2=${varlist_ocn_grib2:-"${PARMgfs}/product/gefs_ocn_shortparmlist_grib2.parm"} #Parameter table for wave variables +export varlist_ice_grib2=${varlist_ice_grib2:-"${PARMgfs}/product/gefs_ice_shortparmlist_grib2.parm"} #Parameter table for wave variables # Variables used in this job export FHMIN=${FHMIN:-0} #The first lead hour @@ -25,9 +29,31 @@ export FHMAXHF=${FHMAXHF:-60} #The maximum lead hour that will use a high output export FHOUTHF=${FHOUTHF:-6} #The high output frequency (in hours) export FHOUTLF=${FHOUTLF:-12} #The low output frequency (in hours) export cycle=${cycle:-"t00z"} #cycle +export ocn_dataformat=${ocn_dataformat:-"grib2"} #the data format of the ocean products to be read +export ice_dataformat=${ice_dataformat:-"grib2"} #the data format of the ice products to be read +export ocnres=${ocnres:-"5p00"} #Resolution of ocean products +export iceres=${iceres:-"5p00"} #Resolution of ice products +export compress_ocn=${compress_ocn:-1} #1: compress extracted ocean product, 0: do not compress extracted ocean product +export compress_ice=${compress_ice:-1} #1: compress extracted ice product, 0: do not compress extracted ice product -subdata="${DATA}/mem${ENSMEM}_atmos" -if [ ! -d ${subdata} ]; then mkdir -p ${subdata}; fi -${EXTRCTVARA} ${ENSMEM} ${subdata} +#Extract variables for atmosphere +if [ ! -d "${DATA}/mem${ENSMEM}_atmos" ]; then + mkdir -p "${DATA}/mem${ENSMEM}_atmos" +fi +${EXTRCTVARA} ${ENSMEM} "${DATA}/mem${ENSMEM}_atmos" + +#Extract variables for ocean +export component_name="ocn" +if [ ! -d "${DATA}/mem${ENSMEM}_ocn" ]; then + mkdir -p "${DATA}/mem${ENSMEM}_ocn" +fi +${EXTRCTVARO} ${ENSMEM} "${DATA}/mem${ENSMEM}_ocn" ${varlist_ocn_grib2} ${ocn_dataformat} ${ocnres} ${compress_ocn} + +#Extract variables for ice +export component_name="ice" +if [ ! -d "${DATA}/mem${ENSMEM}_ice" ]; then + mkdir -p "${DATA}/mem${ENSMEM}_ice" +fi +${EXTRCTVARO} ${ENSMEM} "${DATA}/mem${ENSMEM}_ice" ${varlist_ice_grib2} ${ice_dataformat} ${iceres} ${compress_ice} exit 0 diff --git a/ush/gefs_ocnice_extractvars.sh b/ush/gefs_ocnice_extractvars.sh new file mode 100755 index 0000000000..b7ca6bdbf4 --- /dev/null +++ b/ush/gefs_ocnice_extractvars.sh @@ -0,0 +1,58 @@ +#! /usr/bin/env bash +################################################################################ +# Script: +# +source "${USHgfs}/preamble.sh" + +ensname=${1} +outdirpre=${2} +varlist=${3} +dataformat=${4} +datares=${5} +datacompress=${6} +cd ${outdirpre} + +nh=6 +while [[ $nh -le ${FHMAX} ]];do + fnh=`printf "%3.3d" ${nh}` + VDATE=$($NDATE +${nh} ${PDY}${cyc}) + yyyyv=$(echo $VDATE | cut -c1-4) + mmv=$(echo $VDATE | cut -c5-6) + ddv=$(echo $VDATE | cut -c7-8) + hhv=$(echo $VDATE | cut -c9-10) + echo "extracting f${fnh} valid at ${VDATE} for ${ensname}" + + if [[ "${dataformat}" == "grib2" ]];then + if [[ ${component_name} == "ocn" ]];then + infile=${COM_OCEAN_GRIB}/${datares}/gefs.ocean.${cycle}.${datares}.f${fnh}.grib2 + fi + if [[ ${component_name} == "ice" ]];then + infile=${COM_ICE_GRIB}/${datares}/gefs.ice.${cycle}.${datares}.f${fnh}.grib2 + fi + oufile1=$outdirpre/gefs.${component_name}.${cycle}.${datares}.f${fnh}.rfcst.grib2 + fi + + if [[ "${dataformat}" == "netcdf" ]];then + if [[ ${component_name} == "ocn" ]];then + infile=${COM_OCEAN_NETCDF}/${datares}/gefs.ocean.${cycle}.${datares}.f${fnh}.nc + fi + if [[ ${component_name} == "ice" ]];then + infile=${COM_ICE_NETCDF}/${datares}/gefs.ice.${cycle}.${datares}.f${fnh}.nc + fi + oufile1=$outdirpre/gefs.${component_name}.${cycle}.${datares}.f${fnh}.rfcst.grib2 + fi + + if [ -f $infile ]; then #check if input file exists before extraction + $WGRIB2 $infile | grep -F -f ${varlist} | $WGRIB2 -i $infile -append -grib $oufile1>/dev/null + if [[ ${datacompress} -eq 1 ]];then + echo "Compressing ${oufile1}" + bzip2 $oufile1 + fi + else + echo "WARNING: ${infile} does not exist." + fi + + nh=$((${nh} + 6)) +done + +exit 0 From 177783bc29616fff94cdb29a72d1181133bc683e Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Mon, 3 Jun 2024 16:25:07 +0000 Subject: [PATCH 010/141] Add script to process waves A script has been added to process the wave variables for the reforecast. --- scripts/exglobal_extractvars.sh | 10 +++++++ ush/gefs_wav_extractvars.sh | 46 +++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100755 ush/gefs_wav_extractvars.sh diff --git a/scripts/exglobal_extractvars.sh b/scripts/exglobal_extractvars.sh index 7ddc88f290..429a917c59 100755 --- a/scripts/exglobal_extractvars.sh +++ b/scripts/exglobal_extractvars.sh @@ -33,6 +33,9 @@ export ocn_dataformat=${ocn_dataformat:-"grib2"} #the data format of the ocean p export ice_dataformat=${ice_dataformat:-"grib2"} #the data format of the ice products to be read export ocnres=${ocnres:-"5p00"} #Resolution of ocean products export iceres=${iceres:-"5p00"} #Resolution of ice products +export ocnres=${ocnres:-"5p00"} #Resolution of ocean products +export wavinres=${wavinres:-"5p00"} #Resolution of wave products +export wavoures=${wavoures:-"1p00"} #Resolution of wave products export compress_ocn=${compress_ocn:-1} #1: compress extracted ocean product, 0: do not compress extracted ocean product export compress_ice=${compress_ice:-1} #1: compress extracted ice product, 0: do not compress extracted ice product @@ -56,4 +59,11 @@ if [ ! -d "${DATA}/mem${ENSMEM}_ice" ]; then fi ${EXTRCTVARO} ${ENSMEM} "${DATA}/mem${ENSMEM}_ice" ${varlist_ice_grib2} ${ice_dataformat} ${iceres} ${compress_ice} +#Extract variables for wave +export component_name="wav" +if [ ! -d "${DATA}/mem${ENSMEM}_wav" ]; then + mkdir -p "${DATA}/mem${ENSMEM}_wav" +fi +${EXTRCTVARW} ${ENSMEM} "${DATA}/mem${ENSMEM}_wav" + exit 0 diff --git a/ush/gefs_wav_extractvars.sh b/ush/gefs_wav_extractvars.sh new file mode 100755 index 0000000000..7b2acdd63c --- /dev/null +++ b/ush/gefs_wav_extractvars.sh @@ -0,0 +1,46 @@ +#! /usr/bin/env bash +################################################################################ +# Script: +# +source "${USHgfs}/preamble.sh" + +ensname=$1 +DATA=$2 +cd $DATA + +outfreq=6 + +varlist=$varlist_wav + +dointerp=1 +option1=' -set_grib_type same -new_grid_winds earth ' +option21=' -new_grid_interpolation bilinear' +grid1p00="latlon 0:360:1.0 90:181:-1.0" + +nh=6 +while [[ $nh -le $fhmax ]];do + fnh=`printf "%3.3d" ${nh}` + echo "extracting f${fnh}" + + infile=$COM_WAVE_GRID/gefswave.${cycle}.global.${wavinres}.f${fnh}.grib2 + oufile1=$DATA/gefswave.${cycle}.global.${wavinres}.f${fnh}.mem${ensname}.rfcst.grib2 + rm -f $oufile1 #remove outfile if it already exists before extraction + + if [ -f $infile ]; then #check if input file exists before extraction + if [[ $dointerp -eq 1 ]];then + oufile2=$DATA/gefswave.${cycle}.global.${wavoures}.f${fnh}.rfcst.grib2 + $WGRIB2 $infile $option1 $option21 -new_grid $grid1p00 $oufile2>/dev/null + infile=$oufile2 + fi + $WGRIB2 $infile | grep -F -f $varlist | $WGRIB2 -i $infile -append -grib $oufile1>/dev/null + else + echo "WARNING: $infile does not exist." + fi + + nh=$(($nh + $outfreq)) + +done #fhr + + + +exit 0 From d3688ede71a17948cb5f11f03a7aa3100fd998fb Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Mon, 3 Jun 2024 19:04:46 +0000 Subject: [PATCH 011/141] Simplify wave script The script that processes waves in the extractvars task has been cleaned up. --- ush/gefs_wav_extractvars.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/ush/gefs_wav_extractvars.sh b/ush/gefs_wav_extractvars.sh index 7b2acdd63c..be2f3dc6f4 100755 --- a/ush/gefs_wav_extractvars.sh +++ b/ush/gefs_wav_extractvars.sh @@ -10,8 +10,6 @@ cd $DATA outfreq=6 -varlist=$varlist_wav - dointerp=1 option1=' -set_grib_type same -new_grid_winds earth ' option21=' -new_grid_interpolation bilinear' @@ -22,8 +20,8 @@ while [[ $nh -le $fhmax ]];do fnh=`printf "%3.3d" ${nh}` echo "extracting f${fnh}" - infile=$COM_WAVE_GRID/gefswave.${cycle}.global.${wavinres}.f${fnh}.grib2 - oufile1=$DATA/gefswave.${cycle}.global.${wavinres}.f${fnh}.mem${ensname}.rfcst.grib2 + infile=${COM_WAVE_GRID}/gefswave.${cycle}.global.${wavinres}.f${fnh}.grib2 + oufile1=${DATA}/gefswave.${cycle}.global.${wavinres}.f${fnh}.mem${ensname}.rfcst.grib2 rm -f $oufile1 #remove outfile if it already exists before extraction if [ -f $infile ]; then #check if input file exists before extraction @@ -32,7 +30,7 @@ while [[ $nh -le $fhmax ]];do $WGRIB2 $infile $option1 $option21 -new_grid $grid1p00 $oufile2>/dev/null infile=$oufile2 fi - $WGRIB2 $infile | grep -F -f $varlist | $WGRIB2 -i $infile -append -grib $oufile1>/dev/null + $WGRIB2 $infile | grep -F -f $varlist_wav | $WGRIB2 -i $infile -append -grib $oufile1>/dev/null else echo "WARNING: $infile does not exist." fi From f8073740be5970737f68a530fa60c820f777e946 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Mon, 3 Jun 2024 19:12:15 +0000 Subject: [PATCH 012/141] Fix brackets The brackets have been fixed in gefs_tasks.py --- workflow/rocoto/gefs_tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 1fe14f04d9..ab1ab9eafb 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -276,8 +276,8 @@ def extractvars(self): extractvars_envars = self.envars.copy() extractvars_dict = {'ENSMEM': '#member#', - 'MEMDIR': 'mem#member#', - } + 'MEMDIR': 'mem#member#', + } for key, value in extractvars_dict.items(): extractvars_envars.append(rocoto.create_envar(name=key, value=str(value))) From 215a72602829e808f9b7a2e39cd33a268a95b2c1 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Mon, 3 Jun 2024 19:37:52 +0000 Subject: [PATCH 013/141] Add wave forecast frequency variable A wave output frequency variable has been added that allows a specific wave forecast output frequency for data saved on disk. --- scripts/exglobal_extractvars.sh | 1 + ush/gefs_wav_extractvars.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/exglobal_extractvars.sh b/scripts/exglobal_extractvars.sh index 429a917c59..46494be773 100755 --- a/scripts/exglobal_extractvars.sh +++ b/scripts/exglobal_extractvars.sh @@ -38,6 +38,7 @@ export wavinres=${wavinres:-"5p00"} #Resolution of wave products export wavoures=${wavoures:-"1p00"} #Resolution of wave products export compress_ocn=${compress_ocn:-1} #1: compress extracted ocean product, 0: do not compress extracted ocean product export compress_ice=${compress_ice:-1} #1: compress extracted ice product, 0: do not compress extracted ice product +export FHOUT_WAV_NOSCRUB=${FHOUT_WAV_NOSCRUB:-6} #Frequency of wave output to be saved on disk #Extract variables for atmosphere if [ ! -d "${DATA}/mem${ENSMEM}_atmos" ]; then diff --git a/ush/gefs_wav_extractvars.sh b/ush/gefs_wav_extractvars.sh index be2f3dc6f4..acff4f3aa4 100755 --- a/ush/gefs_wav_extractvars.sh +++ b/ush/gefs_wav_extractvars.sh @@ -8,7 +8,7 @@ ensname=$1 DATA=$2 cd $DATA -outfreq=6 +outfreq=${FHOUT_WAV_NOSCRUB} dointerp=1 option1=' -set_grib_type same -new_grid_winds earth ' From b79237cbe910b7a12babb480739617f5a75144d8 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 4 Jun 2024 12:28:17 +0000 Subject: [PATCH 014/141] Run shellnorms on push Shellnorms is set to run on every git push. --- .github/workflows/linters.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/linters.yaml b/.github/workflows/linters.yaml index 488b6a1407..0a8858a167 100644 --- a/.github/workflows/linters.yaml +++ b/.github/workflows/linters.yaml @@ -1,7 +1,6 @@ # name: shellnorms -on: - pull_request: +on: [push, pull_request] permissions: contents: read From e3f61ba1469142b4870be0aeabaa96892e71c45b Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 4 Jun 2024 13:56:42 +0000 Subject: [PATCH 015/141] Fix wave script An FHMAX variable has been added to the wave script in the extractvars task so that an unbound error can be avoided. --- ush/gefs_wav_extractvars.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/gefs_wav_extractvars.sh b/ush/gefs_wav_extractvars.sh index acff4f3aa4..5f062354e6 100755 --- a/ush/gefs_wav_extractvars.sh +++ b/ush/gefs_wav_extractvars.sh @@ -16,7 +16,7 @@ option21=' -new_grid_interpolation bilinear' grid1p00="latlon 0:360:1.0 90:181:-1.0" nh=6 -while [[ $nh -le $fhmax ]];do +while [[ $nh -le $FHMAX_WAV ]];do fnh=`printf "%3.3d" ${nh}` echo "extracting f${fnh}" From c9e49d7b72e07150895975f027cc793734a02f0a Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 4 Jun 2024 14:18:20 +0000 Subject: [PATCH 016/141] Clean up JGLOBAL_EXTRACTVARS script The JGLOBAL_EXTRACTVARS script has been refined. --- jobs/JGLOBAL_EXTRACTVARS | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/jobs/JGLOBAL_EXTRACTVARS b/jobs/JGLOBAL_EXTRACTVARS index d18e7966f3..70c6d0de2f 100755 --- a/jobs/JGLOBAL_EXTRACTVARS +++ b/jobs/JGLOBAL_EXTRACTVARS @@ -20,13 +20,14 @@ YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COM_ICE_NETCDF" YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COM_WAVE_GRID" # Execute the Script -${SCRgfs}/exglobal_extractvars.sh +"${SCRgfs}/exglobal_extractvars.sh" +status=$? +(( status != 0 )) && exit "${status}" ########################################## # Remove the Temporary working directory ########################################## -cd ${DATAROOT} -[[ ${KEEPDATA} = "NO" ]] && rm -rf ${DATA} - +cd ${DATAROOT} || true +[[ "${KEEPDATA}" = "NO" ]] && rm -rf "${DATA}" exit 0 From b1076f386bb1a97c8a3816d0b2d1b43986744df8 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 4 Jun 2024 14:22:56 +0000 Subject: [PATCH 017/141] Clean up more scripts More scripts have been cleaned up. --- jobs/JGLOBAL_EXTRACTVARS | 2 +- jobs/rocoto/extractvars.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/jobs/JGLOBAL_EXTRACTVARS b/jobs/JGLOBAL_EXTRACTVARS index 70c6d0de2f..957cc2d1dd 100755 --- a/jobs/JGLOBAL_EXTRACTVARS +++ b/jobs/JGLOBAL_EXTRACTVARS @@ -27,7 +27,7 @@ status=$? ########################################## # Remove the Temporary working directory ########################################## -cd ${DATAROOT} || true +cd "${DATAROOT}" || true [[ "${KEEPDATA}" = "NO" ]] && rm -rf "${DATA}" exit 0 diff --git a/jobs/rocoto/extractvars.sh b/jobs/rocoto/extractvars.sh index 1950a7be04..a872431358 100755 --- a/jobs/rocoto/extractvars.sh +++ b/jobs/rocoto/extractvars.sh @@ -7,7 +7,7 @@ echo echo "=============== START TO SOURCE FV3GFS WORKFLOW MODULES ===============" . "${HOMEgfs}/ush/load_fv3gfs_modules.sh" status=$? -[[ ${status} -ne 0 ]] && exit ${status} +[[ "${status}" -ne 0 ]] && exit "${status}" export job="extractvars" export jobid="${job}.$$" @@ -16,8 +16,8 @@ export jobid="${job}.$$" echo echo "=============== START TO RUN EXTRACTVARS ===============" # Execute the JJOB -${HOMEgfs}/jobs/JGLOBAL_EXTRACTVARS +"${HOMEgfs}/jobs/JGLOBAL_EXTRACTVARS" status=$? -[[ ${status} -ne 0 ]] && exit ${status} +[[ "${status}" -ne 0 ]] && exit "${status}" exit 0 From e7adce98c2e6fcd5b8ff3fe3713634d5bdfb5d97 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 4 Jun 2024 14:26:19 +0000 Subject: [PATCH 018/141] Revert "Run shellnorms on push" This reverts commit b79237cbe910b7a12babb480739617f5a75144d8. --- .github/workflows/linters.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/linters.yaml b/.github/workflows/linters.yaml index 0a8858a167..488b6a1407 100644 --- a/.github/workflows/linters.yaml +++ b/.github/workflows/linters.yaml @@ -1,6 +1,7 @@ # name: shellnorms -on: [push, pull_request] +on: + pull_request: permissions: contents: read From e1f2e7146e95fc2807f4cf5824a70fe266b66dd6 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 4 Jun 2024 14:33:32 +0000 Subject: [PATCH 019/141] Clean up extractvars script The extactvars script has been cleaned up. --- scripts/exglobal_extractvars.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/exglobal_extractvars.sh b/scripts/exglobal_extractvars.sh index 46494be773..fc13bcdc3e 100755 --- a/scripts/exglobal_extractvars.sh +++ b/scripts/exglobal_extractvars.sh @@ -41,30 +41,30 @@ export compress_ice=${compress_ice:-1} #1: compress extracted ice product, 0: do export FHOUT_WAV_NOSCRUB=${FHOUT_WAV_NOSCRUB:-6} #Frequency of wave output to be saved on disk #Extract variables for atmosphere -if [ ! -d "${DATA}/mem${ENSMEM}_atmos" ]; then +if [[ ! -d "${DATA}/mem${ENSMEM}_atmos" ]]; then mkdir -p "${DATA}/mem${ENSMEM}_atmos" fi -${EXTRCTVARA} ${ENSMEM} "${DATA}/mem${ENSMEM}_atmos" +${EXTRCTVARA} "${ENSMEM}" "${DATA}/mem${ENSMEM}_atmos" #Extract variables for ocean export component_name="ocn" -if [ ! -d "${DATA}/mem${ENSMEM}_ocn" ]; then +if [[ ! -d "${DATA}/mem${ENSMEM}_ocn" ]]; then mkdir -p "${DATA}/mem${ENSMEM}_ocn" fi -${EXTRCTVARO} ${ENSMEM} "${DATA}/mem${ENSMEM}_ocn" ${varlist_ocn_grib2} ${ocn_dataformat} ${ocnres} ${compress_ocn} +${EXTRCTVARO} "${ENSMEM}" "${DATA}/mem${ENSMEM}_ocn" "${varlist_ocn_grib2}" "${ocn_dataformat}" "${ocnres}" "${compress_ocn}" #Extract variables for ice export component_name="ice" -if [ ! -d "${DATA}/mem${ENSMEM}_ice" ]; then +if [[ ! -d "${DATA}/mem${ENSMEM}_ice" ]]; then mkdir -p "${DATA}/mem${ENSMEM}_ice" fi -${EXTRCTVARO} ${ENSMEM} "${DATA}/mem${ENSMEM}_ice" ${varlist_ice_grib2} ${ice_dataformat} ${iceres} ${compress_ice} +${EXTRCTVARO} "${ENSMEM}" "${DATA}/mem${ENSMEM}_ice" "${varlist_ice_grib2}" "${ice_dataformat}" "${iceres}" "${compress_ice}" #Extract variables for wave export component_name="wav" -if [ ! -d "${DATA}/mem${ENSMEM}_wav" ]; then +if [[ ! -d "${DATA}/mem${ENSMEM}_wav" ]]; then mkdir -p "${DATA}/mem${ENSMEM}_wav" fi -${EXTRCTVARW} ${ENSMEM} "${DATA}/mem${ENSMEM}_wav" +${EXTRCTVARW} "${ENSMEM}" "${DATA}/mem${ENSMEM}_wav" exit 0 From a310f0ed82558184d4395558978e05f7c9c3a1b8 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 4 Jun 2024 14:52:26 +0000 Subject: [PATCH 020/141] Clean up extractvars atmos script --- ush/gefs_atmos_extractvars.sh | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/ush/gefs_atmos_extractvars.sh b/ush/gefs_atmos_extractvars.sh index 73bac9cc65..ba230d7d45 100755 --- a/ush/gefs_atmos_extractvars.sh +++ b/ush/gefs_atmos_extractvars.sh @@ -9,36 +9,35 @@ dcnt=1 ensname=$1 subdata=$2 -cd $subdata +cd "${subdata}" || true for outtype in "f2d" "f3d"; do if [[ "${outtype}" == "f2d" ]];then - varlist=$varlist_2d + varlist=${varlist_2d} elif [[ "${outtype}" == "f3d" ]];then - varlist=$varlist_3d - varlist_d=$varlist_3d_d + varlist=${varlist_3d} + varlist_d=${varlist_3d_d} fi outdirpre="${subdata}/${outtype}" - if [ ! -d ${outdirpre} ]; then mkdir -p ${outdirpre}; fi + if [[ ! -d "${outdirpre}" ]]; then mkdir -p "${outdirpre}"; fi nh=${FHMIN} - while [[ $nh -le $FHMAX ]];do - fnh=`printf "%3.3d" ${nh}` - echo "extracting f${fnh}" + while [[ ${nh} -le ${FHMAX} ]];do + fnh=$(printf "%3.3d" "${nh}") - if [[ "$outtype" == "f2d" ]];then - if [[ $nh -le $FHMAXHF ]];then + if [[ "${outtype}" == "f2d" ]];then + if [[ ${nh} -le ${FHMAXHF} ]];then outres="0p25" else outres="0p50" fi - elif [[ "$outtype" == "f3d" ]];then + elif [[ "${outtype}" == "f3d" ]];then outres="1p00" fi - if [[ $nh -lt $FHMAXHF ]];then + if [[ ${nh} -lt ${FHMAXHF} ]];then outfreq=$FHOUTHF else outfreq=$FHOUTLF From 1917d267abdf487e94ed51da07efed2ac964d00d Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 4 Jun 2024 15:11:52 +0000 Subject: [PATCH 021/141] Clean up atmos extractvars script --- ush/gefs_atmos_extractvars.sh | 62 +++++++++++++++++------------------ 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/ush/gefs_atmos_extractvars.sh b/ush/gefs_atmos_extractvars.sh index ba230d7d45..c11153efb6 100755 --- a/ush/gefs_atmos_extractvars.sh +++ b/ush/gefs_atmos_extractvars.sh @@ -38,62 +38,62 @@ for outtype in "f2d" "f3d"; do fi if [[ ${nh} -lt ${FHMAXHF} ]];then - outfreq=$FHOUTHF + outfreq=${FHOUTHF} else - outfreq=$FHOUTLF + outfreq=${FHOUTLF} fi echo $ensname ============== if [[ "$outres" == "0p25" ]];then - infile1=$COM_ATMOS_GRIB_0p25/gefs.${cycle}.pgrb2.$outres.f${fnh} - infile2=$COM_ATMOS_GRIB_0p25/gefs.${cycle}.pgrb2b.$outres.f${fnh} + infile1=${COM_ATMOS_GRIB_0p25}/gefs.${cycle}.pgrb2.${outres}.f${fnh} + infile2=${COM_ATMOS_GRIB_0p25}/gefs.${cycle}.pgrb2b.${outres}.f${fnh} elif [[ "$outres" == "0p50" ]];then - infile1=$COM_ATMOS_GRIB_0p50/gefs.${cycle}.pgrb2.$outres.f${fnh} - infile2=$COM_ATMOS_GRIB_0p50/gefs.${cycle}.pgrb2b.$outres.f${fnh} + infile1=${COM_ATMOS_GRIB_0p50}/gefs.${cycle}.pgrb2.${outres}.f${fnh} + infile2=${COM_ATMOS_GRIB_0p50}/gefs.${cycle}.pgrb2b.${outres}.f${fnh} elif [[ "$outres" == "1p00" ]];then - infile1=$COM_ATMOS_GRIB_1p00/gefs.${cycle}.pgrb2.$outres.f${fnh} - infile2=$COM_ATMOS_GRIB_1p00/gefs.${cycle}.pgrb2b.$outres.f${fnh} + infile1=${COM_ATMOS_GRIB_1p00}/gefs.${cycle}.pgrb2.${outres}.f${fnh} + infile2=${COM_ATMOS_GRIB_1p00}/gefs.${cycle}.pgrb2b.${outres}.f${fnh} fi - oufile=$outdirpre/ge${ensname}.${cycle}.pgrb2.$outres.f${fnh} - rm -f $oufile #remove outfile if it already exists before extraction + oufile=${outdirpre}/ge${ensname}.${cycle}.pgrb2.${outres}.f${fnh} + rm -f ${oufile} #remove outfile if it already exists before extraction - if [ -f $infile1 ]; then #check if input file exists before extraction - $WGRIB2 $infile1 | grep -F -f $varlist | $WGRIB2 -i $infile1 -append -grib $oufile>/dev/null + if [[ -f $infile1 ]]; then #check if input file exists before extraction + ${WGRIB2} "${infile1}" | grep -F -f "${varlist}" | ${WGRIB2} -i "${infile1}" -append -grib "${oufile}">/dev/null else echo "WARNING: $infile1 does not exist." fi if [ -f $infile2 ]; then #check if input file exists before extraction - $WGRIB2 $infile2 | grep -F -f $varlist | $WGRIB2 -i $infile2 -append -grib $oufile>/dev/null + ${WGRIB2} "${infile2}" | grep -F -f "${varlist}" | ${WGRIB2} -i "${infile2}" -append -grib "${oufile}">/dev/null else - echo "WARNING: $infile2 does not exist." + echo "WARNING: ${infile2} does not exist." fi #Compute daily average for a subset of variables - if [[ "$outtype" == "f3d" ]];then - if ! (( $nh % 6 ));then - outfile=$subdata/vartmp_raw_vari_ldy${dcnt}.${ensname}.grib2 - $WGRIB2 $infile1 | grep -F -f $varlist_d | $WGRIB2 -i $infile1 -append -grib $outfile>/dev/null - $WGRIB2 $infile2 | grep -F -f $varlist_d | $WGRIB2 -i $infile2 -append -grib $outfile>/dev/null - if [[ $fcnt -eq 4 ]];then - fnd=$(printf "%2.2d" ${dcnt}) - davg_file=$outdirpre/ge${ensname}.${cycle}.pgrb2.$outres.ldy${fnd} + if [[ "${outtype}" == "f3d" ]];then + if ! (( nh % 6 ));then + outfile=${subdata}/vartmp_raw_vari_ldy${dcnt}.${ensname}.grib2 + ${WGRIB2} "${infile1}" | grep -F -f "${varlist_d}" | ${WGRIB2} -i "${infile1}" -append -grib "${outfile}">/dev/null + ${WGRIB2} "${infile2}" | grep -F -f "${varlist_d}" | ${WGRIB2} -i "${infile2}" -append -grib "${outfile}">/dev/null + if [[ ${fcnt} -eq 4 ]];then + fnd=$(printf "%2.2d" "${dcnt}") + davg_file=${outdirpre}/ge${ensname}.${cycle}.pgrb2.${outres}.ldy${fnd} vcnt=1 while read vari; do - davgtmp=$subdata/ge${ensname}.${cycle}.tmp.pgrb2.$outres.ldy${fnd}.${vcnt} - $WGRIB2 $outfile | grep "$vari" | $WGRIB2 -i $outfile -fcst_ave 6hr $davgtmp>/dev/null - $WGRIB2 $davgtmp | $WGRIB2 -i $davgtmp -append -grib $davg_file>/dev/null - rm -f $davgtmp - vcnt=$(($vcnt + 1)) - done <$varlist_d #variable + davgtmp=${subdata}/ge${ensname}.${cycle}.tmp.pgrb2.${outres}.ldy${fnd}.${vcnt} + ${WGRIB2} "${outfile}" | grep "${vari}" | ${WGRIB2} -i "${outfile}" -fcst_ave 6hr "${davgtmp}">/dev/null + ${WGRIB2} "${davgtmp}" | ${WGRIB2} -i "${davgtmp}" -append -grib "${davg_file}">/dev/null + rm -f "${davgtmp}" + vcnt=$(( vcnt + 1 )) + done <"${varlist_d}" #variable fcnt=1 - dcnt=$(($dcnt + 1)) + dcnt=$(( dcnt + 1 )) else - fcnt=$(($fcnt + 1)) + fcnt=$(( fcnt + 1 )) fi #If at final lead hour of a given day fi #if lead hour is divisible by 6 fi #if outtype == f3d - nh=$(($nh + $outfreq)) + nh=$(( nh + outfreq )) done #fhr done #f2d,f3d From b88bae41a7c6a558a5469417bba9c3a43708ee6d Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 4 Jun 2024 15:26:36 +0000 Subject: [PATCH 022/141] Clean up atmos extractvars script further --- ush/gefs_atmos_extractvars.sh | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/ush/gefs_atmos_extractvars.sh b/ush/gefs_atmos_extractvars.sh index c11153efb6..9f672c2e94 100755 --- a/ush/gefs_atmos_extractvars.sh +++ b/ush/gefs_atmos_extractvars.sh @@ -43,27 +43,26 @@ for outtype in "f2d" "f3d"; do outfreq=${FHOUTLF} fi - echo $ensname ============== - if [[ "$outres" == "0p25" ]];then + if [[ "${outres}" == "0p25" ]];then infile1=${COM_ATMOS_GRIB_0p25}/gefs.${cycle}.pgrb2.${outres}.f${fnh} infile2=${COM_ATMOS_GRIB_0p25}/gefs.${cycle}.pgrb2b.${outres}.f${fnh} - elif [[ "$outres" == "0p50" ]];then + elif [[ "${outres}" == "0p50" ]];then infile1=${COM_ATMOS_GRIB_0p50}/gefs.${cycle}.pgrb2.${outres}.f${fnh} infile2=${COM_ATMOS_GRIB_0p50}/gefs.${cycle}.pgrb2b.${outres}.f${fnh} - elif [[ "$outres" == "1p00" ]];then + elif [[ "${outres}" == "1p00" ]];then infile1=${COM_ATMOS_GRIB_1p00}/gefs.${cycle}.pgrb2.${outres}.f${fnh} infile2=${COM_ATMOS_GRIB_1p00}/gefs.${cycle}.pgrb2b.${outres}.f${fnh} fi oufile=${outdirpre}/ge${ensname}.${cycle}.pgrb2.${outres}.f${fnh} - rm -f ${oufile} #remove outfile if it already exists before extraction + rm -f "${oufile}" #remove outfile if it already exists before extraction - if [[ -f $infile1 ]]; then #check if input file exists before extraction + if [[ -f "${infile1}" ]]; then #check if input file exists before extraction ${WGRIB2} "${infile1}" | grep -F -f "${varlist}" | ${WGRIB2} -i "${infile1}" -append -grib "${oufile}">/dev/null else echo "WARNING: $infile1 does not exist." fi - if [ -f $infile2 ]; then #check if input file exists before extraction + if [ -f "${infile2}" ]; then #check if input file exists before extraction ${WGRIB2} "${infile2}" | grep -F -f "${varlist}" | ${WGRIB2} -i "${infile2}" -append -grib "${oufile}">/dev/null else echo "WARNING: ${infile2} does not exist." @@ -79,7 +78,7 @@ for outtype in "f2d" "f3d"; do fnd=$(printf "%2.2d" "${dcnt}") davg_file=${outdirpre}/ge${ensname}.${cycle}.pgrb2.${outres}.ldy${fnd} vcnt=1 - while read vari; do + while read -r vari; do davgtmp=${subdata}/ge${ensname}.${cycle}.tmp.pgrb2.${outres}.ldy${fnd}.${vcnt} ${WGRIB2} "${outfile}" | grep "${vari}" | ${WGRIB2} -i "${outfile}" -fcst_ave 6hr "${davgtmp}">/dev/null ${WGRIB2} "${davgtmp}" | ${WGRIB2} -i "${davgtmp}" -append -grib "${davg_file}">/dev/null From f2a9e338bb1403d457df2559a400b87e2b0a5838 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 4 Jun 2024 15:46:26 +0000 Subject: [PATCH 023/141] Refine atmos extractvars script --- ush/gefs_atmos_extractvars.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ush/gefs_atmos_extractvars.sh b/ush/gefs_atmos_extractvars.sh index 9f672c2e94..f08dd8df30 100755 --- a/ush/gefs_atmos_extractvars.sh +++ b/ush/gefs_atmos_extractvars.sh @@ -57,13 +57,13 @@ for outtype in "f2d" "f3d"; do rm -f "${oufile}" #remove outfile if it already exists before extraction if [[ -f "${infile1}" ]]; then #check if input file exists before extraction - ${WGRIB2} "${infile1}" | grep -F -f "${varlist}" | ${WGRIB2} -i "${infile1}" -append -grib "${oufile}">/dev/null + ${WGRIB2} "${infile1}" | grep -F -f "${varlist}" #| ${WGRIB2} -i "${infile1}" -append -grib "${oufile}">/dev/null else - echo "WARNING: $infile1 does not exist." + echo "WARNING: ${infile1} does not exist." fi - if [ -f "${infile2}" ]; then #check if input file exists before extraction - ${WGRIB2} "${infile2}" | grep -F -f "${varlist}" | ${WGRIB2} -i "${infile2}" -append -grib "${oufile}">/dev/null + if [[ -f "${infile2}" ]]; then #check if input file exists before extraction + ${WGRIB2} "${infile2}" | grep -F -f "${varlist}" #| ${WGRIB2} -i "${infile2}" -append -grib "${oufile}">/dev/null else echo "WARNING: ${infile2} does not exist." fi @@ -72,8 +72,8 @@ for outtype in "f2d" "f3d"; do if [[ "${outtype}" == "f3d" ]];then if ! (( nh % 6 ));then outfile=${subdata}/vartmp_raw_vari_ldy${dcnt}.${ensname}.grib2 - ${WGRIB2} "${infile1}" | grep -F -f "${varlist_d}" | ${WGRIB2} -i "${infile1}" -append -grib "${outfile}">/dev/null - ${WGRIB2} "${infile2}" | grep -F -f "${varlist_d}" | ${WGRIB2} -i "${infile2}" -append -grib "${outfile}">/dev/null + ${WGRIB2} "${infile1}" | grep -F -f "${varlist_d}" #| ${WGRIB2} -i "${infile1}" -append -grib "${outfile}">/dev/null + ${WGRIB2} "${infile2}" | grep -F -f "${varlist_d}" #| ${WGRIB2} -i "${infile2}" -append -grib "${outfile}">/dev/null if [[ ${fcnt} -eq 4 ]];then fnd=$(printf "%2.2d" "${dcnt}") davg_file=${outdirpre}/ge${ensname}.${cycle}.pgrb2.${outres}.ldy${fnd} From 5b964ef4c9f20144ae4bb1f7464f09127e6dcfd1 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 4 Jun 2024 15:55:10 +0000 Subject: [PATCH 024/141] disable specific spellcheck in atmos script --- ush/gefs_atmos_extractvars.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/ush/gefs_atmos_extractvars.sh b/ush/gefs_atmos_extractvars.sh index f08dd8df30..bc67c63400 100755 --- a/ush/gefs_atmos_extractvars.sh +++ b/ush/gefs_atmos_extractvars.sh @@ -57,13 +57,15 @@ for outtype in "f2d" "f3d"; do rm -f "${oufile}" #remove outfile if it already exists before extraction if [[ -f "${infile1}" ]]; then #check if input file exists before extraction - ${WGRIB2} "${infile1}" | grep -F -f "${varlist}" #| ${WGRIB2} -i "${infile1}" -append -grib "${oufile}">/dev/null + # shellcheck disable=SC2312 + ${WGRIB2} "${infile1}" | grep -F -f "${varlist}" | ${WGRIB2} -i "${infile1}" -append -grib "${oufile}">/dev/null else echo "WARNING: ${infile1} does not exist." fi if [[ -f "${infile2}" ]]; then #check if input file exists before extraction - ${WGRIB2} "${infile2}" | grep -F -f "${varlist}" #| ${WGRIB2} -i "${infile2}" -append -grib "${oufile}">/dev/null + # shellcheck disable=SC2312 + ${WGRIB2} "${infile2}" | grep -F -f "${varlist}" | ${WGRIB2} -i "${infile2}" -append -grib "${oufile}">/dev/null else echo "WARNING: ${infile2} does not exist." fi @@ -72,15 +74,19 @@ for outtype in "f2d" "f3d"; do if [[ "${outtype}" == "f3d" ]];then if ! (( nh % 6 ));then outfile=${subdata}/vartmp_raw_vari_ldy${dcnt}.${ensname}.grib2 - ${WGRIB2} "${infile1}" | grep -F -f "${varlist_d}" #| ${WGRIB2} -i "${infile1}" -append -grib "${outfile}">/dev/null - ${WGRIB2} "${infile2}" | grep -F -f "${varlist_d}" #| ${WGRIB2} -i "${infile2}" -append -grib "${outfile}">/dev/null + # shellcheck disable=SC2312 + ${WGRIB2} "${infile1}" | grep -F -f "${varlist_d}" | ${WGRIB2} -i "${infile1}" -append -grib "${outfile}">/dev/null + # shellcheck disable=SC2312 + ${WGRIB2} "${infile2}" | grep -F -f "${varlist_d}" | ${WGRIB2} -i "${infile2}" -append -grib "${outfile}">/dev/null if [[ ${fcnt} -eq 4 ]];then fnd=$(printf "%2.2d" "${dcnt}") davg_file=${outdirpre}/ge${ensname}.${cycle}.pgrb2.${outres}.ldy${fnd} vcnt=1 while read -r vari; do davgtmp=${subdata}/ge${ensname}.${cycle}.tmp.pgrb2.${outres}.ldy${fnd}.${vcnt} + # shellcheck disable=SC2312 ${WGRIB2} "${outfile}" | grep "${vari}" | ${WGRIB2} -i "${outfile}" -fcst_ave 6hr "${davgtmp}">/dev/null + # shellcheck disable=SC2312 ${WGRIB2} "${davgtmp}" | ${WGRIB2} -i "${davgtmp}" -append -grib "${davg_file}">/dev/null rm -f "${davgtmp}" vcnt=$(( vcnt + 1 )) From 1049801f132b71dea1ddc2a53158ad48b059d85f Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 5 Jun 2024 14:57:35 +0000 Subject: [PATCH 025/141] Add more corrections --- ush/gefs_ocnice_extractvars.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ush/gefs_ocnice_extractvars.sh b/ush/gefs_ocnice_extractvars.sh index b7ca6bdbf4..4ab8b50351 100755 --- a/ush/gefs_ocnice_extractvars.sh +++ b/ush/gefs_ocnice_extractvars.sh @@ -10,11 +10,11 @@ varlist=${3} dataformat=${4} datares=${5} datacompress=${6} -cd ${outdirpre} +cd "${outdirpre}" || true nh=6 -while [[ $nh -le ${FHMAX} ]];do - fnh=`printf "%3.3d" ${nh}` +while [[ ${nh} -le ${FHMAX} ]];do + fnh=$(printf "%3.3d" "${nh}") VDATE=$($NDATE +${nh} ${PDY}${cyc}) yyyyv=$(echo $VDATE | cut -c1-4) mmv=$(echo $VDATE | cut -c5-6) From 1168566c8d673938eb5829457d9a85601970f960 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 5 Jun 2024 15:14:12 +0000 Subject: [PATCH 026/141] Simplify ocnice extractvars script Unneeded lines have been removed from the ocnice extractvars script. --- ush/gefs_ocnice_extractvars.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/ush/gefs_ocnice_extractvars.sh b/ush/gefs_ocnice_extractvars.sh index 4ab8b50351..2ef3500974 100755 --- a/ush/gefs_ocnice_extractvars.sh +++ b/ush/gefs_ocnice_extractvars.sh @@ -15,12 +15,6 @@ cd "${outdirpre}" || true nh=6 while [[ ${nh} -le ${FHMAX} ]];do fnh=$(printf "%3.3d" "${nh}") - VDATE=$($NDATE +${nh} ${PDY}${cyc}) - yyyyv=$(echo $VDATE | cut -c1-4) - mmv=$(echo $VDATE | cut -c5-6) - ddv=$(echo $VDATE | cut -c7-8) - hhv=$(echo $VDATE | cut -c9-10) - echo "extracting f${fnh} valid at ${VDATE} for ${ensname}" if [[ "${dataformat}" == "grib2" ]];then if [[ ${component_name} == "ocn" ]];then From c2a23b718b77c5c6e281a350e5e7cd5d08b1f046 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 5 Jun 2024 15:17:48 +0000 Subject: [PATCH 027/141] Remove unused variable An unused variable ensname has been removed from the ocnice extractvars script. --- scripts/exglobal_extractvars.sh | 4 ++-- ush/gefs_ocnice_extractvars.sh | 11 +++++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/scripts/exglobal_extractvars.sh b/scripts/exglobal_extractvars.sh index fc13bcdc3e..f41bc47897 100755 --- a/scripts/exglobal_extractvars.sh +++ b/scripts/exglobal_extractvars.sh @@ -51,14 +51,14 @@ export component_name="ocn" if [[ ! -d "${DATA}/mem${ENSMEM}_ocn" ]]; then mkdir -p "${DATA}/mem${ENSMEM}_ocn" fi -${EXTRCTVARO} "${ENSMEM}" "${DATA}/mem${ENSMEM}_ocn" "${varlist_ocn_grib2}" "${ocn_dataformat}" "${ocnres}" "${compress_ocn}" +${EXTRCTVARO} "${DATA}/mem${ENSMEM}_ocn" "${varlist_ocn_grib2}" "${ocn_dataformat}" "${ocnres}" "${compress_ocn}" #Extract variables for ice export component_name="ice" if [[ ! -d "${DATA}/mem${ENSMEM}_ice" ]]; then mkdir -p "${DATA}/mem${ENSMEM}_ice" fi -${EXTRCTVARO} "${ENSMEM}" "${DATA}/mem${ENSMEM}_ice" "${varlist_ice_grib2}" "${ice_dataformat}" "${iceres}" "${compress_ice}" +${EXTRCTVARO} "${DATA}/mem${ENSMEM}_ice" "${varlist_ice_grib2}" "${ice_dataformat}" "${iceres}" "${compress_ice}" #Extract variables for wave export component_name="wav" diff --git a/ush/gefs_ocnice_extractvars.sh b/ush/gefs_ocnice_extractvars.sh index 2ef3500974..817d963ad6 100755 --- a/ush/gefs_ocnice_extractvars.sh +++ b/ush/gefs_ocnice_extractvars.sh @@ -4,12 +4,11 @@ # source "${USHgfs}/preamble.sh" -ensname=${1} -outdirpre=${2} -varlist=${3} -dataformat=${4} -datares=${5} -datacompress=${6} +outdirpre=${1} +varlist=${2} +dataformat=${3} +datares=${4} +datacompress=${5} cd "${outdirpre}" || true nh=6 From 1a91e4a2213e5ec538d0299c4415703538595203 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 5 Jun 2024 15:24:57 +0000 Subject: [PATCH 028/141] Add more corrections Additional corrections have been added. --- ush/gefs_ocnice_extractvars.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ush/gefs_ocnice_extractvars.sh b/ush/gefs_ocnice_extractvars.sh index 817d963ad6..78ef289b83 100755 --- a/ush/gefs_ocnice_extractvars.sh +++ b/ush/gefs_ocnice_extractvars.sh @@ -22,7 +22,7 @@ while [[ ${nh} -le ${FHMAX} ]];do if [[ ${component_name} == "ice" ]];then infile=${COM_ICE_GRIB}/${datares}/gefs.ice.${cycle}.${datares}.f${fnh}.grib2 fi - oufile1=$outdirpre/gefs.${component_name}.${cycle}.${datares}.f${fnh}.rfcst.grib2 + oufile1=${outdirpre}/gefs.${component_name}.${cycle}.${datares}.f${fnh}.rfcst.grib2 fi if [[ "${dataformat}" == "netcdf" ]];then @@ -32,20 +32,21 @@ while [[ ${nh} -le ${FHMAX} ]];do if [[ ${component_name} == "ice" ]];then infile=${COM_ICE_NETCDF}/${datares}/gefs.ice.${cycle}.${datares}.f${fnh}.nc fi - oufile1=$outdirpre/gefs.${component_name}.${cycle}.${datares}.f${fnh}.rfcst.grib2 + oufile1=${outdirpre}/gefs.${component_name}.${cycle}.${datares}.f${fnh}.rfcst.grib2 fi - if [ -f $infile ]; then #check if input file exists before extraction - $WGRIB2 $infile | grep -F -f ${varlist} | $WGRIB2 -i $infile -append -grib $oufile1>/dev/null + if [[ -f "${infile}" ]]; then #check if input file exists before extraction + # shellcheck disable=SC2312 + ${WGRIB2} "${infile}" | grep -F -f "${varlist}" | ${WGRIB2} -i "${infile}" -append -grib "${oufile1}">/dev/null if [[ ${datacompress} -eq 1 ]];then echo "Compressing ${oufile1}" - bzip2 $oufile1 + bzip2 "${oufile1}" fi else echo "WARNING: ${infile} does not exist." fi - nh=$((${nh} + 6)) + nh=$(( nh + 6 )) done exit 0 From 9417dd62d26c834d986b9cc0cb804f559194561b Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 5 Jun 2024 15:42:59 +0000 Subject: [PATCH 029/141] Clean up wave script in extractvars task The wave script in the extractvars task has been cleaned up. --- ush/gefs_atmos_extractvars.sh | 4 ++-- ush/gefs_wav_extractvars.sh | 34 +++++++++++++++------------------- 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/ush/gefs_atmos_extractvars.sh b/ush/gefs_atmos_extractvars.sh index bc67c63400..e4a89a1b0b 100755 --- a/ush/gefs_atmos_extractvars.sh +++ b/ush/gefs_atmos_extractvars.sh @@ -6,8 +6,8 @@ source "${USHgfs}/preamble.sh" fcnt=1 dcnt=1 -ensname=$1 -subdata=$2 +ensname=${1} +subdata=${2} cd "${subdata}" || true diff --git a/ush/gefs_wav_extractvars.sh b/ush/gefs_wav_extractvars.sh index 5f062354e6..f903631889 100755 --- a/ush/gefs_wav_extractvars.sh +++ b/ush/gefs_wav_extractvars.sh @@ -4,11 +4,9 @@ # source "${USHgfs}/preamble.sh" -ensname=$1 -DATA=$2 -cd $DATA - -outfreq=${FHOUT_WAV_NOSCRUB} +ensname=${1} +DATA=${2} +cd ${DATA} || true dointerp=1 option1=' -set_grib_type same -new_grid_winds earth ' @@ -16,29 +14,27 @@ option21=' -new_grid_interpolation bilinear' grid1p00="latlon 0:360:1.0 90:181:-1.0" nh=6 -while [[ $nh -le $FHMAX_WAV ]];do +while [[ ${nh} -le ${FHMAX_WAV} ]];do fnh=`printf "%3.3d" ${nh}` - echo "extracting f${fnh}" infile=${COM_WAVE_GRID}/gefswave.${cycle}.global.${wavinres}.f${fnh}.grib2 oufile1=${DATA}/gefswave.${cycle}.global.${wavinres}.f${fnh}.mem${ensname}.rfcst.grib2 - rm -f $oufile1 #remove outfile if it already exists before extraction + rm -f "${oufile1}" #remove outfile if it already exists before extraction - if [ -f $infile ]; then #check if input file exists before extraction - if [[ $dointerp -eq 1 ]];then - oufile2=$DATA/gefswave.${cycle}.global.${wavoures}.f${fnh}.rfcst.grib2 - $WGRIB2 $infile $option1 $option21 -new_grid $grid1p00 $oufile2>/dev/null - infile=$oufile2 - fi - $WGRIB2 $infile | grep -F -f $varlist_wav | $WGRIB2 -i $infile -append -grib $oufile1>/dev/null + if [[ -f "${infile}" ]]; then #check if input file exists before extraction + if [[ ${dointerp} -eq 1 ]];then + oufile2=${DATA}/gefswave.${cycle}.global.${wavoures}.f${fnh}.rfcst.grib2 + ${WGRIB2} "${infile}" "${option1}" "${option21}" -new_grid "${grid1p00}" "${oufile2}">/dev/null + infile=${oufile2} + fi + # shellcheck disable=SC2312 + ${WGRIB2} "${infile}" | grep -F -f "${varlist_wav}" | ${WGRIB2} -i "${infile}" -append -grib "${oufile1}">/dev/null else - echo "WARNING: $infile does not exist." + echo "WARNING: ${infile} does not exist." fi - nh=$(($nh + $outfreq)) + nh=$(( nh + FHOUT_WAV_NOSCRUB )) done #fhr - - exit 0 From 99d3adfcc7bd3396fe97e42ef9fd550867268093 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 5 Jun 2024 15:47:54 +0000 Subject: [PATCH 030/141] Add more fixes to wave script in extractvars More fixes have been added to the wave script in the extractvars task. --- ush/gefs_wav_extractvars.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/gefs_wav_extractvars.sh b/ush/gefs_wav_extractvars.sh index f903631889..7ae13ec597 100755 --- a/ush/gefs_wav_extractvars.sh +++ b/ush/gefs_wav_extractvars.sh @@ -6,7 +6,7 @@ source "${USHgfs}/preamble.sh" ensname=${1} DATA=${2} -cd ${DATA} || true +cd "${DATA}" || true dointerp=1 option1=' -set_grib_type same -new_grid_winds earth ' @@ -15,7 +15,7 @@ grid1p00="latlon 0:360:1.0 90:181:-1.0" nh=6 while [[ ${nh} -le ${FHMAX_WAV} ]];do - fnh=`printf "%3.3d" ${nh}` + fnh=$(printf "%3.3d" "${nh}") infile=${COM_WAVE_GRID}/gefswave.${cycle}.global.${wavinres}.f${fnh}.grib2 oufile1=${DATA}/gefswave.${cycle}.global.${wavinres}.f${fnh}.mem${ensname}.rfcst.grib2 From f3df65e6ca0c2fa4e61a858c4b48047d0dbc4c5c Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 5 Jun 2024 17:11:39 +0000 Subject: [PATCH 031/141] Define COM directories for extractvars task COM directories have been defined for the extractvars task. Data generated in the extractvars task has the ability to be copied to COMROOT. --- jobs/JGLOBAL_EXTRACTVARS | 10 ++++++++++ parm/config/gfs/config.com | 5 +++++ scripts/exglobal_extractvars.sh | 6 +++++- ush/gefs_wav_extractvars.sh | 6 +++--- 4 files changed, 23 insertions(+), 4 deletions(-) diff --git a/jobs/JGLOBAL_EXTRACTVARS b/jobs/JGLOBAL_EXTRACTVARS index 957cc2d1dd..a77dbde852 100755 --- a/jobs/JGLOBAL_EXTRACTVARS +++ b/jobs/JGLOBAL_EXTRACTVARS @@ -19,6 +19,16 @@ YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COM_ICE_NETCDF" YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COM_WAVE_GRID" +YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COM_RFCST_PROD_ATMOS" +YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COM_RFCST_PROD_OCN" +YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COM_RFCST_PROD_ICE" +YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COM_RFCST_PROD_WAV" + +if [[ ! -d "${COM_RFCST_PROD_ATMOS}" ]]; then mkdir -m 775 -p "${COM_RFCST_PROD_ATMOS}"; fi +if [[ ! -d "${COM_RFCST_PROD_OCN}" ]]; then mkdir -m 775 -p "${COM_RFCST_PROD_OCN}"; fi +if [[ ! -d "${COM_RFCST_PROD_WAV}" ]]; then mkdir -m 775 -p "${COM_RFCST_PROD_WAV}"; fi +if [[ ! -d "${COM_RFCST_PROD_ICE}" ]]; then mkdir -m 775 -p "${COM_RFCST_PROD_ICE}"; fi + # Execute the Script "${SCRgfs}/exglobal_extractvars.sh" status=$? diff --git a/parm/config/gfs/config.com b/parm/config/gfs/config.com index 004ca1affb..86b4c9cea3 100644 --- a/parm/config/gfs/config.com +++ b/parm/config/gfs/config.com @@ -92,6 +92,11 @@ declare -rx COM_ICE_NETCDF_TMPL=${COM_BASE}'/products/ice/netcdf' declare -rx COM_ICE_GRIB_TMPL=${COM_BASE}'/products/ice/grib2' declare -rx COM_ICE_GRIB_GRID_TMPL=${COM_ICE_GRIB_TMPL}'/${GRID}' +declare -rx COM_RFCST_PROD_ATMOS_TMPL=${COM_BASE}'/products/rfcst/atmos' +declare -rx COM_RFCST_PROD_OCN_TMPL=${COM_BASE}'/products/rfcst/ocn' +declare -rx COM_RFCST_PROD_ICE_TMPL=${COM_BASE}'/products/rfcst/ice' +declare -rx COM_RFCST_PROD_WAV_TMPL=${COM_BASE}'/products/rfcst/wav' + declare -rx COM_CHEM_HISTORY_TMPL=${COM_BASE}'/model_data/chem/history' declare -rx COM_CHEM_ANALYSIS_TMPL=${COM_BASE}'/analysis/chem' diff --git a/scripts/exglobal_extractvars.sh b/scripts/exglobal_extractvars.sh index f41bc47897..f868f9e01e 100755 --- a/scripts/exglobal_extractvars.sh +++ b/scripts/exglobal_extractvars.sh @@ -45,6 +45,7 @@ if [[ ! -d "${DATA}/mem${ENSMEM}_atmos" ]]; then mkdir -p "${DATA}/mem${ENSMEM}_atmos" fi ${EXTRCTVARA} "${ENSMEM}" "${DATA}/mem${ENSMEM}_atmos" +cp -pr ${DATA}/mem${ENSMEM}_atmos/* ${COM_RFCST_PROD_ATMOS} #Extract variables for ocean export component_name="ocn" @@ -52,13 +53,15 @@ if [[ ! -d "${DATA}/mem${ENSMEM}_ocn" ]]; then mkdir -p "${DATA}/mem${ENSMEM}_ocn" fi ${EXTRCTVARO} "${DATA}/mem${ENSMEM}_ocn" "${varlist_ocn_grib2}" "${ocn_dataformat}" "${ocnres}" "${compress_ocn}" +cp -pr ${DATA}/mem${ENSMEM}_ocn/* ${COM_RFCST_PROD_OCN} #Extract variables for ice export component_name="ice" if [[ ! -d "${DATA}/mem${ENSMEM}_ice" ]]; then mkdir -p "${DATA}/mem${ENSMEM}_ice" -fi +fi ${EXTRCTVARO} "${DATA}/mem${ENSMEM}_ice" "${varlist_ice_grib2}" "${ice_dataformat}" "${iceres}" "${compress_ice}" +cp -pr ${DATA}/mem${ENSMEM}_ice/* ${COM_RFCST_PROD_ICE} #Extract variables for wave export component_name="wav" @@ -66,5 +69,6 @@ if [[ ! -d "${DATA}/mem${ENSMEM}_wav" ]]; then mkdir -p "${DATA}/mem${ENSMEM}_wav" fi ${EXTRCTVARW} "${ENSMEM}" "${DATA}/mem${ENSMEM}_wav" +cp -pr ${DATA}/mem${ENSMEM}_wav/* ${COM_RFCST_PROD_WAV} exit 0 diff --git a/ush/gefs_wav_extractvars.sh b/ush/gefs_wav_extractvars.sh index 7ae13ec597..8753ecde45 100755 --- a/ush/gefs_wav_extractvars.sh +++ b/ush/gefs_wav_extractvars.sh @@ -9,8 +9,8 @@ DATA=${2} cd "${DATA}" || true dointerp=1 -option1=' -set_grib_type same -new_grid_winds earth ' -option21=' -new_grid_interpolation bilinear' +option1="-set_grib_type same -new_grid_winds earth" +option21="-new_grid_interpolation bilinear" grid1p00="latlon 0:360:1.0 90:181:-1.0" nh=6 @@ -24,7 +24,7 @@ while [[ ${nh} -le ${FHMAX_WAV} ]];do if [[ -f "${infile}" ]]; then #check if input file exists before extraction if [[ ${dointerp} -eq 1 ]];then oufile2=${DATA}/gefswave.${cycle}.global.${wavoures}.f${fnh}.rfcst.grib2 - ${WGRIB2} "${infile}" "${option1}" "${option21}" -new_grid "${grid1p00}" "${oufile2}">/dev/null + ${WGRIB2} ${infile} ${option1} ${option21} -new_grid ${grid1p00} ${oufile2}>/dev/null infile=${oufile2} fi # shellcheck disable=SC2312 From 64beacce317de5f785adf1082bbab9577aff9ee0 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Fri, 7 Jun 2024 19:12:01 +0000 Subject: [PATCH 032/141] Create an option to remove extractvars task An option in config.base has been created that allows the user to include or not include the extractvars task in the workflow. --- parm/config/gefs/config.base | 1 + workflow/applications/applications.py | 1 + workflow/applications/gefs.py | 8 ++++++-- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index 1b4f948349..f8d0f42aa5 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -136,6 +136,7 @@ export DO_WAVE="NO" export DO_OCN="NO" export DO_ICE="NO" export DO_AERO="NO" +export DO_EXTRACTVARS="NO" export AERO_FCST_CDUMP="" # When to run aerosol forecast: gdas, gfs, or both export AERO_ANL_CDUMP="" # When to run aerosol analysis: gdas, gfs, or both export WAVE_CDUMP="" # When to include wave suite: gdas, gfs, or both diff --git a/workflow/applications/applications.py b/workflow/applications/applications.py index 50a9a7cdd0..1902daa762 100644 --- a/workflow/applications/applications.py +++ b/workflow/applications/applications.py @@ -64,6 +64,7 @@ def __init__(self, conf: Configuration) -> None: self.do_upp = not _base.get('WRITE_DOPOST', True) self.do_goes = _base.get('DO_GOES', False) self.do_mos = _base.get('DO_MOS', False) + self.do_extractvars = _base.get('DO_EXTRACTVARS', False) self.do_hpssarch = _base.get('HPSSARCH', False) diff --git a/workflow/applications/gefs.py b/workflow/applications/gefs.py index 15ff036ec5..bdff2186d0 100644 --- a/workflow/applications/gefs.py +++ b/workflow/applications/gefs.py @@ -14,7 +14,7 @@ def _get_app_configs(self): """ Returns the config_files that are involved in gefs """ - configs = ['stage_ic', 'fcst', 'atmos_products', 'extractvars'] + configs = ['stage_ic', 'fcst', 'atmos_products'] if self.nens > 0: configs += ['efcs', 'atmos_ensstat'] @@ -30,6 +30,9 @@ def _get_app_configs(self): if self.do_aero: configs += ['prep_emissions'] + if self.do_extractvars: + configs += ['extractvars'] + return configs @staticmethod @@ -73,6 +76,7 @@ def get_task_names(self): tasks += ['wavepostbndpnt', 'wavepostbndpntbll'] tasks += ['wavepostpnt'] - tasks += ['extractvars'] + if self.do_extractvars: + tasks += ['extractvars'] return {f"{self._base['CDUMP']}": tasks} From 2a5aab95c68df18e06d00fcb86648cd037470bf9 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Mon, 10 Jun 2024 19:55:52 +0000 Subject: [PATCH 033/141] Add more fixes to extractvars scripts --- scripts/exglobal_extractvars.sh | 14 +++++++------- ush/gefs_atmos_extractvars.sh | 11 +++++------ ush/gefs_ocnice_extractvars.sh | 7 +++---- ush/gefs_wav_extractvars.sh | 18 ++++-------------- 4 files changed, 19 insertions(+), 31 deletions(-) diff --git a/scripts/exglobal_extractvars.sh b/scripts/exglobal_extractvars.sh index f868f9e01e..cee382359a 100755 --- a/scripts/exglobal_extractvars.sh +++ b/scripts/exglobal_extractvars.sh @@ -34,8 +34,7 @@ export ice_dataformat=${ice_dataformat:-"grib2"} #the data format of the ice pro export ocnres=${ocnres:-"5p00"} #Resolution of ocean products export iceres=${iceres:-"5p00"} #Resolution of ice products export ocnres=${ocnres:-"5p00"} #Resolution of ocean products -export wavinres=${wavinres:-"5p00"} #Resolution of wave products -export wavoures=${wavoures:-"1p00"} #Resolution of wave products +export wavres=${wavinres:-"5p00"} #Resolution of wave products export compress_ocn=${compress_ocn:-1} #1: compress extracted ocean product, 0: do not compress extracted ocean product export compress_ice=${compress_ice:-1} #1: compress extracted ice product, 0: do not compress extracted ice product export FHOUT_WAV_NOSCRUB=${FHOUT_WAV_NOSCRUB:-6} #Frequency of wave output to be saved on disk @@ -44,8 +43,9 @@ export FHOUT_WAV_NOSCRUB=${FHOUT_WAV_NOSCRUB:-6} #Frequency of wave output to be if [[ ! -d "${DATA}/mem${ENSMEM}_atmos" ]]; then mkdir -p "${DATA}/mem${ENSMEM}_atmos" fi -${EXTRCTVARA} "${ENSMEM}" "${DATA}/mem${ENSMEM}_atmos" -cp -pr ${DATA}/mem${ENSMEM}_atmos/* ${COM_RFCST_PROD_ATMOS} +${EXTRCTVARA} "${DATA}/mem${ENSMEM}_atmos" +cp -pr "${DATA}/mem${ENSMEM}_atmos/f2d" "${COM_RFCST_PROD_ATMOS}" +cp -pr "${DATA}/mem${ENSMEM}_atmos/f3d" "${COM_RFCST_PROD_ATMOS}" #Extract variables for ocean export component_name="ocn" @@ -53,7 +53,7 @@ if [[ ! -d "${DATA}/mem${ENSMEM}_ocn" ]]; then mkdir -p "${DATA}/mem${ENSMEM}_ocn" fi ${EXTRCTVARO} "${DATA}/mem${ENSMEM}_ocn" "${varlist_ocn_grib2}" "${ocn_dataformat}" "${ocnres}" "${compress_ocn}" -cp -pr ${DATA}/mem${ENSMEM}_ocn/* ${COM_RFCST_PROD_OCN} +cp -pr "${DATA}/mem${ENSMEM}_ocn/." "${COM_RFCST_PROD_OCN}" #Extract variables for ice export component_name="ice" @@ -61,7 +61,7 @@ if [[ ! -d "${DATA}/mem${ENSMEM}_ice" ]]; then mkdir -p "${DATA}/mem${ENSMEM}_ice" fi ${EXTRCTVARO} "${DATA}/mem${ENSMEM}_ice" "${varlist_ice_grib2}" "${ice_dataformat}" "${iceres}" "${compress_ice}" -cp -pr ${DATA}/mem${ENSMEM}_ice/* ${COM_RFCST_PROD_ICE} +cp -pr "${DATA}/mem${ENSMEM}_ice/." "${COM_RFCST_PROD_ICE}" #Extract variables for wave export component_name="wav" @@ -69,6 +69,6 @@ if [[ ! -d "${DATA}/mem${ENSMEM}_wav" ]]; then mkdir -p "${DATA}/mem${ENSMEM}_wav" fi ${EXTRCTVARW} "${ENSMEM}" "${DATA}/mem${ENSMEM}_wav" -cp -pr ${DATA}/mem${ENSMEM}_wav/* ${COM_RFCST_PROD_WAV} +cp -pr "${DATA}/mem${ENSMEM}_wav/." "${COM_RFCST_PROD_WAV}" exit 0 diff --git a/ush/gefs_atmos_extractvars.sh b/ush/gefs_atmos_extractvars.sh index e4a89a1b0b..cffada9737 100755 --- a/ush/gefs_atmos_extractvars.sh +++ b/ush/gefs_atmos_extractvars.sh @@ -6,8 +6,7 @@ source "${USHgfs}/preamble.sh" fcnt=1 dcnt=1 -ensname=${1} -subdata=${2} +subdata=${1} cd "${subdata}" || true @@ -53,7 +52,7 @@ for outtype in "f2d" "f3d"; do infile1=${COM_ATMOS_GRIB_1p00}/gefs.${cycle}.pgrb2.${outres}.f${fnh} infile2=${COM_ATMOS_GRIB_1p00}/gefs.${cycle}.pgrb2b.${outres}.f${fnh} fi - oufile=${outdirpre}/ge${ensname}.${cycle}.pgrb2.${outres}.f${fnh} + oufile=${outdirpre}/gefs.${cycle}.pgrb2.${outres}.f${fnh} rm -f "${oufile}" #remove outfile if it already exists before extraction if [[ -f "${infile1}" ]]; then #check if input file exists before extraction @@ -73,17 +72,17 @@ for outtype in "f2d" "f3d"; do #Compute daily average for a subset of variables if [[ "${outtype}" == "f3d" ]];then if ! (( nh % 6 ));then - outfile=${subdata}/vartmp_raw_vari_ldy${dcnt}.${ensname}.grib2 + outfile=${subdata}/vartmp_raw_vari_ldy${dcnt}.grib2 # shellcheck disable=SC2312 ${WGRIB2} "${infile1}" | grep -F -f "${varlist_d}" | ${WGRIB2} -i "${infile1}" -append -grib "${outfile}">/dev/null # shellcheck disable=SC2312 ${WGRIB2} "${infile2}" | grep -F -f "${varlist_d}" | ${WGRIB2} -i "${infile2}" -append -grib "${outfile}">/dev/null if [[ ${fcnt} -eq 4 ]];then fnd=$(printf "%2.2d" "${dcnt}") - davg_file=${outdirpre}/ge${ensname}.${cycle}.pgrb2.${outres}.ldy${fnd} + davg_file=${outdirpre}/gefs.${cycle}.pgrb2.${outres}.24hr_avg.ldy${fnd} vcnt=1 while read -r vari; do - davgtmp=${subdata}/ge${ensname}.${cycle}.tmp.pgrb2.${outres}.ldy${fnd}.${vcnt} + davgtmp=${subdata}/gefs.${cycle}.tmp.pgrb2.${outres}.ldy${fnd}.${vcnt} # shellcheck disable=SC2312 ${WGRIB2} "${outfile}" | grep "${vari}" | ${WGRIB2} -i "${outfile}" -fcst_ave 6hr "${davgtmp}">/dev/null # shellcheck disable=SC2312 diff --git a/ush/gefs_ocnice_extractvars.sh b/ush/gefs_ocnice_extractvars.sh index 78ef289b83..999cc1c039 100755 --- a/ush/gefs_ocnice_extractvars.sh +++ b/ush/gefs_ocnice_extractvars.sh @@ -11,7 +11,7 @@ datares=${4} datacompress=${5} cd "${outdirpre}" || true -nh=6 +nh=${FHMIN_GFS} while [[ ${nh} -le ${FHMAX} ]];do fnh=$(printf "%3.3d" "${nh}") @@ -39,14 +39,13 @@ while [[ ${nh} -le ${FHMAX} ]];do # shellcheck disable=SC2312 ${WGRIB2} "${infile}" | grep -F -f "${varlist}" | ${WGRIB2} -i "${infile}" -append -grib "${oufile1}">/dev/null if [[ ${datacompress} -eq 1 ]];then - echo "Compressing ${oufile1}" - bzip2 "${oufile1}" + bzip2 "${oufile1}" fi else echo "WARNING: ${infile} does not exist." fi - nh=$(( nh + 6 )) + nh=$(( nh + FHOUT_OCNICE_GFS )) done exit 0 diff --git a/ush/gefs_wav_extractvars.sh b/ush/gefs_wav_extractvars.sh index 8753ecde45..a7e410c1cc 100755 --- a/ush/gefs_wav_extractvars.sh +++ b/ush/gefs_wav_extractvars.sh @@ -8,27 +8,17 @@ ensname=${1} DATA=${2} cd "${DATA}" || true -dointerp=1 -option1="-set_grib_type same -new_grid_winds earth" -option21="-new_grid_interpolation bilinear" -grid1p00="latlon 0:360:1.0 90:181:-1.0" - nh=6 while [[ ${nh} -le ${FHMAX_WAV} ]];do fnh=$(printf "%3.3d" "${nh}") - infile=${COM_WAVE_GRID}/gefswave.${cycle}.global.${wavinres}.f${fnh}.grib2 - oufile1=${DATA}/gefswave.${cycle}.global.${wavinres}.f${fnh}.mem${ensname}.rfcst.grib2 + infile=${COM_WAVE_GRID}/gefswave.${cycle}.global.${wavres}.f${fnh}.grib2 + oufile1=${DATA}/gefswave.${cycle}.global.${wavres}.f${fnh}.grib2 rm -f "${oufile1}" #remove outfile if it already exists before extraction if [[ -f "${infile}" ]]; then #check if input file exists before extraction - if [[ ${dointerp} -eq 1 ]];then - oufile2=${DATA}/gefswave.${cycle}.global.${wavoures}.f${fnh}.rfcst.grib2 - ${WGRIB2} ${infile} ${option1} ${option21} -new_grid ${grid1p00} ${oufile2}>/dev/null - infile=${oufile2} - fi - # shellcheck disable=SC2312 - ${WGRIB2} "${infile}" | grep -F -f "${varlist_wav}" | ${WGRIB2} -i "${infile}" -append -grib "${oufile1}">/dev/null + # shellcheck disable=SC2312 + ${WGRIB2} "${infile}" | grep -F -f "${varlist_wav}" | ${WGRIB2} -i "${infile}" -append -grib "${oufile1}">/dev/null else echo "WARNING: ${infile} does not exist." fi From 01b601ae45d39a6c6617ee6603ab4385f786cfe2 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Mon, 10 Jun 2024 19:56:36 +0000 Subject: [PATCH 034/141] Modify wave parm file for extractvars task The wave parm file for the extractvars task has been modify to meet stakeholder and resource requirements. --- parm/product/gefs_wav_shortparmlist.parm | 3 --- 1 file changed, 3 deletions(-) diff --git a/parm/product/gefs_wav_shortparmlist.parm b/parm/product/gefs_wav_shortparmlist.parm index 14b264a3b1..a45e023c85 100644 --- a/parm/product/gefs_wav_shortparmlist.parm +++ b/parm/product/gefs_wav_shortparmlist.parm @@ -1,6 +1,3 @@ -:UOGRD:surface: -:VOGRD:surface: :UGRD:surface: :VGRD:surface: -:ICEC:surface: :HTSGW:surface: From 3a8e90fc73ac33f2d4194bf470a30f5288776693 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 11 Jun 2024 14:54:34 +0000 Subject: [PATCH 035/141] Move extractvars task to end in gefs_tasks.py --- workflow/rocoto/gefs_tasks.py | 72 +++++++++++++++++------------------ 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index ab1ab9eafb..cde5859375 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -268,42 +268,6 @@ def _atmosoceaniceprod(self, component: str): return task - def extractvars(self): - deps = [] - dep_dict = {'type': 'task', 'name': f'wave_post_grid_mem#member#'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - - extractvars_envars = self.envars.copy() - extractvars_dict = {'ENSMEM': '#member#', - 'MEMDIR': 'mem#member#', - } - for key, value in extractvars_dict.items(): - extractvars_envars.append(rocoto.create_envar(name=key, value=str(value))) - - resources = self.get_resource('extractvars') - task_name = f'extractvars_mem#member#' - task_dict = {'task_name': task_name, - 'resources': resources, - 'dependency': dependencies, - 'envars': extractvars_envars, - 'cycledef': 'gefs', - 'command': f'{self.HOMEgfs}/jobs/rocoto/extractvars.sh', - 'job_name': f'{self.pslot}_{task_name}_@H', - 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', - 'maxtries': '&MAXTRIES;' - } - - member_var_dict = {'member': ' '.join([str(mem).zfill(3) for mem in range(0, self.nmem + 1)])} - member_metatask_dict = {'task_name': 'extractvars', - 'task_dict': task_dict, - 'var_dict': member_var_dict - } - - task = rocoto.create_task(member_metatask_dict) - - return task - def atmos_ensstat(self): resources = self.get_resource('atmos_ensstat') @@ -499,3 +463,39 @@ def wavepostpnt(self): task = rocoto.create_task(member_metatask_dict) return task + + def extractvars(self): + deps = [] + dep_dict = {'type': 'task', 'name': f'wave_post_grid_mem#member#'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + + extractvars_envars = self.envars.copy() + extractvars_dict = {'ENSMEM': '#member#', + 'MEMDIR': 'mem#member#', + } + for key, value in extractvars_dict.items(): + extractvars_envars.append(rocoto.create_envar(name=key, value=str(value))) + + resources = self.get_resource('extractvars') + task_name = f'extractvars_mem#member#' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': extractvars_envars, + 'cycledef': 'gefs', + 'command': f'{self.HOMEgfs}/jobs/rocoto/extractvars.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + member_var_dict = {'member': ' '.join([str(mem).zfill(3) for mem in range(0, self.nmem + 1)])} + member_metatask_dict = {'task_name': 'extractvars', + 'task_dict': task_dict, + 'var_dict': member_var_dict + } + + task = rocoto.create_task(member_metatask_dict) + + return task From 1ca415e941b3ba9bd86eac1c826a4c48dac9c31d Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 12 Jun 2024 12:27:51 +0000 Subject: [PATCH 036/141] Improve task dependency for extractvars The dependencies have been modified for the extractvars task. --- workflow/rocoto/gefs_tasks.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index cde5859375..2dbdc36393 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -468,8 +468,13 @@ def extractvars(self): deps = [] dep_dict = {'type': 'task', 'name': f'wave_post_grid_mem#member#'} deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - + dep_dict = {'type': 'metatask', 'name': f'ocean_prod_mem#member#'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'metatask', 'name': f'ice_prod_mem#member#'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'metatask', 'name': f'atmos_prod_mem#member#'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) extractvars_envars = self.envars.copy() extractvars_dict = {'ENSMEM': '#member#', 'MEMDIR': 'mem#member#', From a84e0d9153d6484912b2eb03c3f8cf6f615182ad Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 12 Jun 2024 12:52:39 +0000 Subject: [PATCH 037/141] Remove unused variable An unused variable has been removed form the wave script in extractvars. --- ush/gefs_wav_extractvars.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ush/gefs_wav_extractvars.sh b/ush/gefs_wav_extractvars.sh index a7e410c1cc..7547a58cd8 100755 --- a/ush/gefs_wav_extractvars.sh +++ b/ush/gefs_wav_extractvars.sh @@ -4,8 +4,7 @@ # source "${USHgfs}/preamble.sh" -ensname=${1} -DATA=${2} +DATA=${1} cd "${DATA}" || true nh=6 From ecb069ce6a62a6de974df351823981dfebfbed24 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 12 Jun 2024 13:34:41 +0000 Subject: [PATCH 038/141] Add DO_EXTRACTVARS to default yaml The variable DO_EXTRACTVARS has been added to the gefs default yaml. --- parm/config/gefs/yaml/defaults.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/parm/config/gefs/yaml/defaults.yaml b/parm/config/gefs/yaml/defaults.yaml index 5c763ad29e..32f5b8bbe3 100644 --- a/parm/config/gefs/yaml/defaults.yaml +++ b/parm/config/gefs/yaml/defaults.yaml @@ -7,7 +7,8 @@ base: DO_BUFRSND: "NO" DO_GEMPAK: "NO" DO_AWIPS: "NO" - KEEPDATA: "NO" + KEEPDATA: "NO" + DO_EXTRACTVARS: "NO" FHMAX_GFS: 120 USE_OCN_PERTURB_FILES: "false" From e280a36f9c775f965d391adef26732229fa46426 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 12 Jun 2024 13:36:47 +0000 Subject: [PATCH 039/141] Modify config.base for GEFS The config.base for GEFS has been modified to consider a default DO_EXTRACTVARS in the default yaml file. --- parm/config/gefs/config.base | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index 22382539b7..26a6137354 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -136,7 +136,7 @@ export DO_WAVE="NO" export DO_OCN="NO" export DO_ICE="NO" export DO_AERO="NO" -export DO_EXTRACTVARS="NO" +export DO_EXTRACTVARS="@DO_EXTRACTVARS@" export AERO_FCST_CDUMP="" # When to run aerosol forecast: gdas, gfs, or both export AERO_ANL_CDUMP="" # When to run aerosol analysis: gdas, gfs, or both export WAVE_CDUMP="" # When to include wave suite: gdas, gfs, or both From c79a8b682a48c3d1f9ca61d55090151e819c3bdb Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 12 Jun 2024 13:51:58 +0000 Subject: [PATCH 040/141] Fix dependencies for extractvars The metatask dependencies have been corrected for the extractvars task in gefs_tasks.py. --- workflow/rocoto/gefs_tasks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index f8dc39b567..fb845d1b02 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -477,11 +477,11 @@ def extractvars(self): deps = [] dep_dict = {'type': 'task', 'name': f'wave_post_grid_mem#member#'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': f'ocean_prod_mem#member#'} + dep_dict = {'type': 'metatask', 'name': f'ocean_prod'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': f'ice_prod_mem#member#'} + dep_dict = {'type': 'metatask', 'name': f'ice_prod'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': f'atmos_prod_mem#member#'} + dep_dict = {'type': 'metatask', 'name': f'atmos_prod'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) extractvars_envars = self.envars.copy() From b5893dbf632b415c9120d26156bbe8540e97f227 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 12 Jun 2024 15:45:11 +0000 Subject: [PATCH 041/141] Update FHOUT variable The FHOUT_OCNICE variable has been updated to FHOUT_OCN and FHOUT_ICE. --- scripts/exglobal_extractvars.sh | 4 ++-- ush/gefs_ocnice_extractvars.sh | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/exglobal_extractvars.sh b/scripts/exglobal_extractvars.sh index cee382359a..e9795c5dc5 100755 --- a/scripts/exglobal_extractvars.sh +++ b/scripts/exglobal_extractvars.sh @@ -52,7 +52,7 @@ export component_name="ocn" if [[ ! -d "${DATA}/mem${ENSMEM}_ocn" ]]; then mkdir -p "${DATA}/mem${ENSMEM}_ocn" fi -${EXTRCTVARO} "${DATA}/mem${ENSMEM}_ocn" "${varlist_ocn_grib2}" "${ocn_dataformat}" "${ocnres}" "${compress_ocn}" +${EXTRCTVARO} "${DATA}/mem${ENSMEM}_ocn" "${varlist_ocn_grib2}" "${ocn_dataformat}" "${ocnres}" "${compress_ocn}" "${FHOUT_OCN_GFS}" cp -pr "${DATA}/mem${ENSMEM}_ocn/." "${COM_RFCST_PROD_OCN}" #Extract variables for ice @@ -60,7 +60,7 @@ export component_name="ice" if [[ ! -d "${DATA}/mem${ENSMEM}_ice" ]]; then mkdir -p "${DATA}/mem${ENSMEM}_ice" fi -${EXTRCTVARO} "${DATA}/mem${ENSMEM}_ice" "${varlist_ice_grib2}" "${ice_dataformat}" "${iceres}" "${compress_ice}" +${EXTRCTVARO} "${DATA}/mem${ENSMEM}_ice" "${varlist_ice_grib2}" "${ice_dataformat}" "${iceres}" "${compress_ice}" "${FHOUT_ICE_GFS}" cp -pr "${DATA}/mem${ENSMEM}_ice/." "${COM_RFCST_PROD_ICE}" #Extract variables for wave diff --git a/ush/gefs_ocnice_extractvars.sh b/ush/gefs_ocnice_extractvars.sh index 999cc1c039..4f6b4891cb 100755 --- a/ush/gefs_ocnice_extractvars.sh +++ b/ush/gefs_ocnice_extractvars.sh @@ -9,6 +9,7 @@ varlist=${2} dataformat=${3} datares=${4} datacompress=${5} +fhout_ocnice=${6} cd "${outdirpre}" || true nh=${FHMIN_GFS} @@ -45,7 +46,7 @@ while [[ ${nh} -le ${FHMAX} ]];do echo "WARNING: ${infile} does not exist." fi - nh=$(( nh + FHOUT_OCNICE_GFS )) + nh=$(( nh + fhout_ocnice )) done exit 0 From 03ba20175859b68032f3f7ee227193c39e7217c9 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 12 Jun 2024 16:06:14 +0000 Subject: [PATCH 042/141] Remove unused variables from config.extractvars Unused variables have been removed from config.extractvars. A couple of new variables have been added. --- parm/config/gefs/config.extractvars | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/parm/config/gefs/config.extractvars b/parm/config/gefs/config.extractvars index 4b29139963..cb354f57af 100644 --- a/parm/config/gefs/config.extractvars +++ b/parm/config/gefs/config.extractvars @@ -7,10 +7,6 @@ echo "BEGIN: config.extractvars" . "${EXPDIR}/config.resources" extractvars -export RFHOME=/lfs/h2/emc/ens/noscrub/eric.sinsky/GEFSv13_RR -export REFCST=${RFHOME}/REFCST - -export mpmdd="NO" export atmosextract=1 export ocniceextract=1 export wavextract=1 @@ -25,4 +21,7 @@ export FHMAXHF=60 export FHOUTHF=6 export FHOUTLF=12 +export ocn_dataformat="netcdf" +export ice_dataformat="netcdf" + echo "END: config.extractvars" From 2a6071a15cf14de7da8300372989b2bdc73cfc03 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 12 Jun 2024 16:57:42 +0000 Subject: [PATCH 043/141] Remove more unused variables More unused variables have been removed from config.extractvars. --- parm/config/gefs/config.extractvars | 2 -- 1 file changed, 2 deletions(-) diff --git a/parm/config/gefs/config.extractvars b/parm/config/gefs/config.extractvars index cb354f57af..428da045e6 100644 --- a/parm/config/gefs/config.extractvars +++ b/parm/config/gefs/config.extractvars @@ -13,8 +13,6 @@ export wavextract=1 export compress_ocn=0 export compress_ice=0 -export interp_ocn=0 -export interp_wav=1 export FHMAX=120 export FHMAXHF=60 From cdc4d89cb846a9a6e99c4554628dae46a0203baa Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 12 Jun 2024 17:28:32 +0000 Subject: [PATCH 044/141] Remove nonexistant variable A nonexistant variable has been removed from exglobal_extractvars. --- scripts/exglobal_extractvars.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exglobal_extractvars.sh b/scripts/exglobal_extractvars.sh index e9795c5dc5..27a5716321 100755 --- a/scripts/exglobal_extractvars.sh +++ b/scripts/exglobal_extractvars.sh @@ -68,7 +68,7 @@ export component_name="wav" if [[ ! -d "${DATA}/mem${ENSMEM}_wav" ]]; then mkdir -p "${DATA}/mem${ENSMEM}_wav" fi -${EXTRCTVARW} "${ENSMEM}" "${DATA}/mem${ENSMEM}_wav" +${EXTRCTVARW} "${DATA}/mem${ENSMEM}_wav" cp -pr "${DATA}/mem${ENSMEM}_wav/." "${COM_RFCST_PROD_WAV}" exit 0 From 456dbdf03038340ee2a851d5caf7c84a560f4a95 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 12 Jun 2024 18:45:24 +0000 Subject: [PATCH 045/141] Modify scripts to process netcdf input The extractvars scripts have been modified so that there is a capability for netcdf ocn and ice input to be processed. --- scripts/exglobal_extractvars.sh | 8 ++++++-- ush/gefs_ocnice_extractvars.sh | 20 ++++++++++++++------ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/scripts/exglobal_extractvars.sh b/scripts/exglobal_extractvars.sh index 27a5716321..a0824080dd 100755 --- a/scripts/exglobal_extractvars.sh +++ b/scripts/exglobal_extractvars.sh @@ -52,7 +52,9 @@ export component_name="ocn" if [[ ! -d "${DATA}/mem${ENSMEM}_ocn" ]]; then mkdir -p "${DATA}/mem${ENSMEM}_ocn" fi -${EXTRCTVARO} "${DATA}/mem${ENSMEM}_ocn" "${varlist_ocn_grib2}" "${ocn_dataformat}" "${ocnres}" "${compress_ocn}" "${FHOUT_OCN_GFS}" +if [[ "${ocn_dataformat}" == "netcdf" ]]; then varlist_ocn=${varlist_ocn_netcdf}; fi +if [[ "${ocn_dataformat}" == "grib2" ]]; then varlist_ocn=${varlist_ocn_grib2}; fi +${EXTRCTVARO} "${DATA}/mem${ENSMEM}_ocn" "${varlist_ocn}" "${ocn_dataformat}" "${ocnres}" "${compress_ocn}" "${FHOUT_OCN_GFS}" cp -pr "${DATA}/mem${ENSMEM}_ocn/." "${COM_RFCST_PROD_OCN}" #Extract variables for ice @@ -60,7 +62,9 @@ export component_name="ice" if [[ ! -d "${DATA}/mem${ENSMEM}_ice" ]]; then mkdir -p "${DATA}/mem${ENSMEM}_ice" fi -${EXTRCTVARO} "${DATA}/mem${ENSMEM}_ice" "${varlist_ice_grib2}" "${ice_dataformat}" "${iceres}" "${compress_ice}" "${FHOUT_ICE_GFS}" +if [[ "${ice_dataformat}" == "netcdf" ]]; then varlist_ice=${varlist_ice_netcdf}; fi +if [[ "${ice_dataformat}" == "grib2" ]]; then varlist_ice=${varlist_ice_grib2}; fi +${EXTRCTVARO} "${DATA}/mem${ENSMEM}_ice" "${varlist_ice}" "${ice_dataformat}" "${iceres}" "${compress_ice}" "${FHOUT_ICE_GFS}" cp -pr "${DATA}/mem${ENSMEM}_ice/." "${COM_RFCST_PROD_ICE}" #Extract variables for wave diff --git a/ush/gefs_ocnice_extractvars.sh b/ush/gefs_ocnice_extractvars.sh index 4f6b4891cb..5f71f425bf 100755 --- a/ush/gefs_ocnice_extractvars.sh +++ b/ush/gefs_ocnice_extractvars.sh @@ -23,22 +23,30 @@ while [[ ${nh} -le ${FHMAX} ]];do if [[ ${component_name} == "ice" ]];then infile=${COM_ICE_GRIB}/${datares}/gefs.ice.${cycle}.${datares}.f${fnh}.grib2 fi - oufile1=${outdirpre}/gefs.${component_name}.${cycle}.${datares}.f${fnh}.rfcst.grib2 + oufile1=${outdirpre}/gefs.${component_name}.${cycle}.${datares}.f${fnh}.grib2 fi if [[ "${dataformat}" == "netcdf" ]];then if [[ ${component_name} == "ocn" ]];then - infile=${COM_OCEAN_NETCDF}/${datares}/gefs.ocean.${cycle}.${datares}.f${fnh}.nc + infile=${COM_OCEAN_NETCDF}/gefs.ocean.${cycle}.${datares}.f${fnh}.nc fi if [[ ${component_name} == "ice" ]];then - infile=${COM_ICE_NETCDF}/${datares}/gefs.ice.${cycle}.${datares}.f${fnh}.nc + infile=${COM_ICE_NETCDF}/gefs.ice.${cycle}.${datares}.f${fnh}.nc fi - oufile1=${outdirpre}/gefs.${component_name}.${cycle}.${datares}.f${fnh}.rfcst.grib2 + oufile1=${outdirpre}/gefs.${component_name}.${cycle}.${datares}.f${fnh}.nc fi if [[ -f "${infile}" ]]; then #check if input file exists before extraction - # shellcheck disable=SC2312 - ${WGRIB2} "${infile}" | grep -F -f "${varlist}" | ${WGRIB2} -i "${infile}" -append -grib "${oufile1}">/dev/null + if [[ "${dataformat}" == "grib2" ]];then + # shellcheck disable=SC2312 + ${WGRIB2} "${infile}" | grep -F -f "${varlist}" | ${WGRIB2} -i "${infile}" -append -grib "${oufile1}">/dev/null + fi + if [[ "${dataformat}" == "netcdf" ]];then +# mapfile -t -d ocnice_vars < "${varlist}" + ocnice_vars=$(paste -sd, "${varlist}") + echo ${ocnice_vars} + ncks -v "${ocnice_vars}" "$infile" "$oufile1" + fi if [[ ${datacompress} -eq 1 ]];then bzip2 "${oufile1}" fi From e49652905a1bfac81e37ef076fbc95cfe772d12f Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 12 Jun 2024 18:49:53 +0000 Subject: [PATCH 046/141] Fix shellcheck errors --- ush/gefs_ocnice_extractvars.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ush/gefs_ocnice_extractvars.sh b/ush/gefs_ocnice_extractvars.sh index 5f71f425bf..8181d58573 100755 --- a/ush/gefs_ocnice_extractvars.sh +++ b/ush/gefs_ocnice_extractvars.sh @@ -44,8 +44,7 @@ while [[ ${nh} -le ${FHMAX} ]];do if [[ "${dataformat}" == "netcdf" ]];then # mapfile -t -d ocnice_vars < "${varlist}" ocnice_vars=$(paste -sd, "${varlist}") - echo ${ocnice_vars} - ncks -v "${ocnice_vars}" "$infile" "$oufile1" + ncks -v "${ocnice_vars}" "${infile}" "${oufile1}" fi if [[ ${datacompress} -eq 1 ]];then bzip2 "${oufile1}" From c7b7c4df03c4555a36987c81281d9ea3d3cb86d7 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 12 Jun 2024 18:53:12 +0000 Subject: [PATCH 047/141] Remove line that has been commented out --- ush/gefs_ocnice_extractvars.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ush/gefs_ocnice_extractvars.sh b/ush/gefs_ocnice_extractvars.sh index 8181d58573..9701778369 100755 --- a/ush/gefs_ocnice_extractvars.sh +++ b/ush/gefs_ocnice_extractvars.sh @@ -42,7 +42,6 @@ while [[ ${nh} -le ${FHMAX} ]];do ${WGRIB2} "${infile}" | grep -F -f "${varlist}" | ${WGRIB2} -i "${infile}" -append -grib "${oufile1}">/dev/null fi if [[ "${dataformat}" == "netcdf" ]];then -# mapfile -t -d ocnice_vars < "${varlist}" ocnice_vars=$(paste -sd, "${varlist}") ncks -v "${ocnice_vars}" "${infile}" "${oufile1}" fi From 8ab7ec37088c39f660976c096c13994f913dd10a Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Fri, 14 Jun 2024 20:07:17 +0000 Subject: [PATCH 048/141] Add checks for ocean/ice variable extraction Checks have been added to ensure all requested ocean and ice variables are available in the input netcdf files. --- ush/gefs_ocnice_extractvars.sh | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/ush/gefs_ocnice_extractvars.sh b/ush/gefs_ocnice_extractvars.sh index 9701778369..38f386ecb7 100755 --- a/ush/gefs_ocnice_extractvars.sh +++ b/ush/gefs_ocnice_extractvars.sh @@ -42,8 +42,23 @@ while [[ ${nh} -le ${FHMAX} ]];do ${WGRIB2} "${infile}" | grep -F -f "${varlist}" | ${WGRIB2} -i "${infile}" -append -grib "${oufile1}">/dev/null fi if [[ "${dataformat}" == "netcdf" ]];then - ocnice_vars=$(paste -sd, "${varlist}") - ncks -v "${ocnice_vars}" "${infile}" "${oufile1}" + varsrequested=$(paste -s "${varlist}") + varsinfile=$(cdo -showname "${infile}") + varsavailable="" + for i in ${varsrequested}; do + #check if variable from parm file is available in netcdf file. If variable is not in netcdf file, do not try to extract that variable. + if [[ $varsinfile == *"${i}"* ]]; then + varsavailable+="${i}," + else + echo "WARNING: ${i} is not available in ${infile}." + fi + done + if [[ -z "${varsavailable}" ]];then + echo "WARNING: No variables from parm file ${varlist} are available in netcdf file ${infile}" + else + ocnice_vars=${varsavailable::-1} + ncks -v "${ocnice_vars}" "${infile}" "${oufile1}" + fi fi if [[ ${datacompress} -eq 1 ]];then bzip2 "${oufile1}" From 7ddb47008b8b6ec41e8773d109c489fe9e49ae8f Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Fri, 14 Jun 2024 20:11:43 +0000 Subject: [PATCH 049/141] Fix shellcheck errors --- ush/gefs_ocnice_extractvars.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/gefs_ocnice_extractvars.sh b/ush/gefs_ocnice_extractvars.sh index 38f386ecb7..a75683278f 100755 --- a/ush/gefs_ocnice_extractvars.sh +++ b/ush/gefs_ocnice_extractvars.sh @@ -47,7 +47,7 @@ while [[ ${nh} -le ${FHMAX} ]];do varsavailable="" for i in ${varsrequested}; do #check if variable from parm file is available in netcdf file. If variable is not in netcdf file, do not try to extract that variable. - if [[ $varsinfile == *"${i}"* ]]; then + if [[ ${varsinfile} == *"${i}"* ]]; then varsavailable+="${i}," else echo "WARNING: ${i} is not available in ${infile}." From c44a5bc528031e4f40ade7f3dfaa20a03fa93222 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Mon, 17 Jun 2024 17:18:51 +0000 Subject: [PATCH 050/141] Fix warning statement Warning statement in the ocnice extractvars script has been corrected. --- ush/gefs_ocnice_extractvars.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/gefs_ocnice_extractvars.sh b/ush/gefs_ocnice_extractvars.sh index a75683278f..d045337d13 100755 --- a/ush/gefs_ocnice_extractvars.sh +++ b/ush/gefs_ocnice_extractvars.sh @@ -54,7 +54,7 @@ while [[ ${nh} -le ${FHMAX} ]];do fi done if [[ -z "${varsavailable}" ]];then - echo "WARNING: No variables from parm file ${varlist} are available in netcdf file ${infile}" + echo "WARNING: No variables from parm file ${varlist} are available in netcdf file ${infile}." else ocnice_vars=${varsavailable::-1} ncks -v "${ocnice_vars}" "${infile}" "${oufile1}" From b61d98f8a94576538636f80e84782db29f0b6d5d Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Mon, 17 Jun 2024 17:20:02 +0000 Subject: [PATCH 051/141] Fix typo in parmfile A typo in the atmos extractvars parmfile has been fixed. --- parm/product/gefs_shortparmlist_2d.parm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/product/gefs_shortparmlist_2d.parm b/parm/product/gefs_shortparmlist_2d.parm index 018b8c35ac..bc13101926 100644 --- a/parm/product/gefs_shortparmlist_2d.parm +++ b/parm/product/gefs_shortparmlist_2d.parm @@ -1,4 +1,4 @@ -PRES:surface: +:PRES:surface: :WEASD:surface: :TMP:2 m above ground: :TMP:surface: From 4e4715f9f483601691533017b416fe79ce04f7e4 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 18 Jun 2024 13:43:02 +0000 Subject: [PATCH 052/141] Add checks to the extractvars atmos scripts Some checks have been added to the extractvars atmos scripts. A new script has also been created that contains some functions used to process the atmos variables for the extractvars task. --- ush/extractvars_tools.sh | 43 ++++++++++++++++++++++++ ush/gefs_atmos_extractvars.sh | 61 +++++++++++++++++------------------ 2 files changed, 72 insertions(+), 32 deletions(-) create mode 100644 ush/extractvars_tools.sh diff --git a/ush/extractvars_tools.sh b/ush/extractvars_tools.sh new file mode 100644 index 0000000000..fb21310412 --- /dev/null +++ b/ush/extractvars_tools.sh @@ -0,0 +1,43 @@ +#! /usr/bin/env bash + +gen_parmlist() { + infileg=$1 + requestedvar_in_file1=$2 + varlist=$3 + rm -vf ${requestedvar_in_file1} + while read -r vari; do + varinfile=$(${WGRIB2} "${infileg}" | grep "${vari}") || true + if [[ ! -z "${varinfile}" ]];then # if varinfile is not empty + echo "${vari}" >> "${requestedvar_in_file1}" + fi + done <"${varlist}" +} + +check_atmos() { + requestedvar_in_file1=$1 + requestedvar_in_file2=$2 + requestedvar_in_allgrb2file=requestedvar_in_allgrb2file.txt + rm -rvf "${requestedvar_in_allgrb2file}" + cat "${requestedvar_in_file1}" "${requestedvar_in_file2}" >> "${requestedvar_in_allgrb2file}" + mapfile -t requestedvar_in_allgrb2file_arr < "${requestedvar_in_allgrb2file}" + while read -r vari; do + if [[ ! ${requestedvar_in_allgrb2file_arr[*]} =~ "$vari" ]] ;then + echo "WARNING: PARM VARIABLE ($vari) is not available in pgrb and pgrb2b." + fi + done <"${varlist}" +} + +daily_avg_atmos() { + fnd=$(printf "%2.2d" "${dcnt}") + davg_file=${outdirpre}/gefs.${cycle}.pgrb2.${outres}.24hr_avg.ldy${fnd} + vcnt=1 + while read -r vari; do + davgtmp=${subdata}/gefs.${cycle}.tmp.pgrb2.${outres}.ldy${fnd}.${vcnt} + # shellcheck disable=SC2312 + ${WGRIB2} "${outfile}" | grep "${vari}" | ${WGRIB2} -i "${outfile}" -fcst_ave 6hr "${davgtmp}" + # shellcheck disable=SC2312 + ${WGRIB2} "${davgtmp}" | ${WGRIB2} -i "${davgtmp}" -append -grib "${davg_file}" + rm -f "${davgtmp}" + vcnt=$(( vcnt + 1 )) + done <"${varlist_d}" #variable +} diff --git a/ush/gefs_atmos_extractvars.sh b/ush/gefs_atmos_extractvars.sh index cffada9737..c075227b2e 100755 --- a/ush/gefs_atmos_extractvars.sh +++ b/ush/gefs_atmos_extractvars.sh @@ -3,7 +3,7 @@ # Script: # source "${USHgfs}/preamble.sh" - +source "${USHgfs}/extractvars_tools.sh" fcnt=1 dcnt=1 subdata=${1} @@ -54,52 +54,49 @@ for outtype in "f2d" "f3d"; do fi oufile=${outdirpre}/gefs.${cycle}.pgrb2.${outres}.f${fnh} rm -f "${oufile}" #remove outfile if it already exists before extraction - + requestedvars1="partial_parm1.txt" + requestedvars2="partial_parm2.txt" + rm -f "${requestedvars1}" + rm -f "${requestedvars2}" if [[ -f "${infile1}" ]]; then #check if input file exists before extraction + gen_parmlist "${infile1}" "${requestedvars1}" "${varlist}" # shellcheck disable=SC2312 - ${WGRIB2} "${infile1}" | grep -F -f "${varlist}" | ${WGRIB2} -i "${infile1}" -append -grib "${oufile}">/dev/null + ${WGRIB2} "${infile1}" | grep -F -f "${requestedvars1}" | ${WGRIB2} -i "${infile1}" -append -grib "${oufile}">/dev/null else echo "WARNING: ${infile1} does not exist." fi if [[ -f "${infile2}" ]]; then #check if input file exists before extraction + gen_parmlist "${infile2}" "${requestedvars2}" "${varlist}" # shellcheck disable=SC2312 - ${WGRIB2} "${infile2}" | grep -F -f "${varlist}" | ${WGRIB2} -i "${infile2}" -append -grib "${oufile}">/dev/null + ${WGRIB2} "${infile2}" | grep -F -f "${requestedvars2}" | ${WGRIB2} -i "${infile2}" -append -grib "${oufile}">/dev/null else echo "WARNING: ${infile2} does not exist." fi + if [[ ! -f "${requestedvars1}" ]]; then touch "${requestedvars1}"; fi + if [[ ! -f "${requestedvars2}" ]]; then touch "${requestedvars2}"; fi + check_atmos "${requestedvars1}" "${requestedvars2}" + #Compute daily average for a subset of variables - if [[ "${outtype}" == "f3d" ]];then - if ! (( nh % 6 ));then - outfile=${subdata}/vartmp_raw_vari_ldy${dcnt}.grib2 - # shellcheck disable=SC2312 - ${WGRIB2} "${infile1}" | grep -F -f "${varlist_d}" | ${WGRIB2} -i "${infile1}" -append -grib "${outfile}">/dev/null - # shellcheck disable=SC2312 - ${WGRIB2} "${infile2}" | grep -F -f "${varlist_d}" | ${WGRIB2} -i "${infile2}" -append -grib "${outfile}">/dev/null - if [[ ${fcnt} -eq 4 ]];then - fnd=$(printf "%2.2d" "${dcnt}") - davg_file=${outdirpre}/gefs.${cycle}.pgrb2.${outres}.24hr_avg.ldy${fnd} - vcnt=1 - while read -r vari; do - davgtmp=${subdata}/gefs.${cycle}.tmp.pgrb2.${outres}.ldy${fnd}.${vcnt} - # shellcheck disable=SC2312 - ${WGRIB2} "${outfile}" | grep "${vari}" | ${WGRIB2} -i "${outfile}" -fcst_ave 6hr "${davgtmp}">/dev/null - # shellcheck disable=SC2312 - ${WGRIB2} "${davgtmp}" | ${WGRIB2} -i "${davgtmp}" -append -grib "${davg_file}">/dev/null - rm -f "${davgtmp}" - vcnt=$(( vcnt + 1 )) - done <"${varlist_d}" #variable - fcnt=1 - dcnt=$(( dcnt + 1 )) - else - fcnt=$(( fcnt + 1 )) - fi #If at final lead hour of a given day - fi #if lead hour is divisible by 6 - fi #if outtype == f3d + if (( nh % 6 == 0 )) && (( nh != 0 )) && [[ "${outtype}" == "f3d" ]];then + outfile=${subdata}/vartmp_raw_vari_ldy${dcnt}.grib2 + # shellcheck disable=SC2312 + ${WGRIB2} "${infile1}" | grep -F -f "${varlist_d}" | ${WGRIB2} -i "${infile1}" -append -grib "${outfile}" + # shellcheck disable=SC2312 + ${WGRIB2} "${infile2}" | grep -F -f "${varlist_d}" | ${WGRIB2} -i "${infile2}" -append -grib "${outfile}" + if [[ ${fcnt} -eq 4 ]];then + daily_avg_atmos + fcnt=1 + dcnt=$(( dcnt + 1 )) + else + fcnt=$(( fcnt + 1 )) + fi #If at final lead hour of a given day + fi #if lead hour is divisible by 6 and outtype is f3d + nh=$(( nh + outfreq )) done #fhr done #f2d,f3d -exit 0 +exit 0 From 94ebfbb69942ecd872f7e5c228140447a755466c Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 18 Jun 2024 13:53:24 +0000 Subject: [PATCH 053/141] Address shellcheck errors --- ush/extractvars_tools.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ush/extractvars_tools.sh b/ush/extractvars_tools.sh index fb21310412..41412b2bd2 100644 --- a/ush/extractvars_tools.sh +++ b/ush/extractvars_tools.sh @@ -4,10 +4,10 @@ gen_parmlist() { infileg=$1 requestedvar_in_file1=$2 varlist=$3 - rm -vf ${requestedvar_in_file1} + rm -vf "${requestedvar_in_file1}" while read -r vari; do varinfile=$(${WGRIB2} "${infileg}" | grep "${vari}") || true - if [[ ! -z "${varinfile}" ]];then # if varinfile is not empty + if [[ -n "${varinfile}" ]];then # if varinfile is not empty echo "${vari}" >> "${requestedvar_in_file1}" fi done <"${varlist}" @@ -21,8 +21,8 @@ check_atmos() { cat "${requestedvar_in_file1}" "${requestedvar_in_file2}" >> "${requestedvar_in_allgrb2file}" mapfile -t requestedvar_in_allgrb2file_arr < "${requestedvar_in_allgrb2file}" while read -r vari; do - if [[ ! ${requestedvar_in_allgrb2file_arr[*]} =~ "$vari" ]] ;then - echo "WARNING: PARM VARIABLE ($vari) is not available in pgrb and pgrb2b." + if [[ ! ${requestedvar_in_allgrb2file_arr[*]} =~ ${vari} ]] ;then + echo "WARNING: PARM VARIABLE (${vari}) is not available in pgrb and pgrb2b." fi done <"${varlist}" } From 5774c2af4a7ee1e83026d43eed93acccc253beba Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Fri, 21 Jun 2024 12:17:08 +0000 Subject: [PATCH 054/141] Apply initial fix for ocean filename mismatch An initial fix has been added so that the workflow can properly be link the MOM6 native files when there is an offset. --- ush/forecast_postdet.sh | 2 +- ush/forecast_predet.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index 2cc34eaacd..1440ac9bd5 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -437,7 +437,7 @@ MOM6_postdet() { # Looping over MOM6 output hours local fhr fhr3 last_fhr interval midpoint vdate vdate_mid source_file dest_file for fhr in ${MOM6_OUTPUT_FH}; do - fhr3=$(printf %03i "${fhr}") + fhr3=$(printf %03i "$(( fhr - OFFSET_START_HOUR ))") if [[ -z ${last_fhr:-} ]]; then last_fhr=${fhr} diff --git a/ush/forecast_predet.sh b/ush/forecast_predet.sh index 9183e86002..da85336ebd 100755 --- a/ush/forecast_predet.sh +++ b/ush/forecast_predet.sh @@ -636,7 +636,7 @@ MOM6_predet(){ # MOM6 does not have a concept of high frequency output like FV3 # Convert output settings into an explicit list for MOM6 - MOM6_OUTPUT_FH=$(seq -s ' ' "${FHMIN}" "${FHOUT_OCN}" "${FHMAX}") + MOM6_OUTPUT_FH=$(seq -s ' ' "$(( FHMIN + OFFSET_START_HOUR ))" "${FHOUT_OCN}" "$(( FHMAX + OFFSET_START_HOUR ))") # If using stochastic parameterizations, create a seed that does not exceed the # largest signed integer From 9c73505e76048f5c237f2be68f4cfa631e770277 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Sun, 23 Jun 2024 14:09:27 +0000 Subject: [PATCH 055/141] Revert "Apply initial fix for ocean filename mismatch" This reverts commit 5774c2af4a7ee1e83026d43eed93acccc253beba. --- ush/forecast_postdet.sh | 2 +- ush/forecast_predet.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index 1440ac9bd5..2cc34eaacd 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -437,7 +437,7 @@ MOM6_postdet() { # Looping over MOM6 output hours local fhr fhr3 last_fhr interval midpoint vdate vdate_mid source_file dest_file for fhr in ${MOM6_OUTPUT_FH}; do - fhr3=$(printf %03i "$(( fhr - OFFSET_START_HOUR ))") + fhr3=$(printf %03i "${fhr}") if [[ -z ${last_fhr:-} ]]; then last_fhr=${fhr} diff --git a/ush/forecast_predet.sh b/ush/forecast_predet.sh index da85336ebd..9183e86002 100755 --- a/ush/forecast_predet.sh +++ b/ush/forecast_predet.sh @@ -636,7 +636,7 @@ MOM6_predet(){ # MOM6 does not have a concept of high frequency output like FV3 # Convert output settings into an explicit list for MOM6 - MOM6_OUTPUT_FH=$(seq -s ' ' "$(( FHMIN + OFFSET_START_HOUR ))" "${FHOUT_OCN}" "$(( FHMAX + OFFSET_START_HOUR ))") + MOM6_OUTPUT_FH=$(seq -s ' ' "${FHMIN}" "${FHOUT_OCN}" "${FHMAX}") # If using stochastic parameterizations, create a seed that does not exceed the # largest signed integer From 3947d18f29bd702d3615bdf4a8cb743e61ee373c Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Mon, 24 Jun 2024 13:51:26 +0000 Subject: [PATCH 056/141] Apply an improved fix for ocean filename mismatch A new diag_table has been created that is specfically used for when OFFSET_START_HOUR is greater than 0. Using this diag_table will allow for the ocean filenames to properly match to what is created in the workflow. --- parm/ufs/fv3/diag_table_replay | 337 +++++++++++++++++++++++++++++++++ ush/forecast_postdet.sh | 16 +- ush/parsing_namelists_FV3.sh | 14 ++ 3 files changed, 364 insertions(+), 3 deletions(-) create mode 100644 parm/ufs/fv3/diag_table_replay diff --git a/parm/ufs/fv3/diag_table_replay b/parm/ufs/fv3/diag_table_replay new file mode 100644 index 0000000000..01f2cf9794 --- /dev/null +++ b/parm/ufs/fv3/diag_table_replay @@ -0,0 +1,337 @@ +"fv3_history", 0, "hours", 1, "hours", "time" +"fv3_history2d", 0, "hours", 1, "hours", "time" +"@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", @[FHOUT_OCN], "hours", 1, "hours", "time", @[FHOUT_OCN], "hours", "@[SYEAR] @[SMONTH] @[SDAY] @[CHOUR_offset] 0 0", @[FHOUT_OCN], "hours" +"@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", @[FHOUT_OCN], "hours", 1, "hours", "time", @[FHOUT_OCN], "hours", "@[SYEAR1] @[SMONTH1] @[SDAY1] @[CHOUR1] 0 0" + +############## +# Ocean fields first lead time +############## +# static fields +"ocean_model", "geolon", "geolon", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +"ocean_model", "geolat", "geolat", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +"ocean_model", "geolon_c", "geolon_c", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +"ocean_model", "geolat_c", "geolat_c", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +"ocean_model", "geolon_u", "geolon_u", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +"ocean_model", "geolat_u", "geolat_u", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +"ocean_model", "geolon_v", "geolon_v", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +"ocean_model", "geolat_v", "geolat_v", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +#"ocean_model", "depth_ocean", "depth_ocean", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +#"ocean_model", "wet", "wet", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +"ocean_model", "wet_c", "wet_c", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +"ocean_model", "wet_u", "wet_u", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +"ocean_model", "wet_v", "wet_v", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +"ocean_model", "sin_rot", "sin_rot", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +"ocean_model", "cos_rot", "cos_rot", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 + +# ocean output TSUV and others +"ocean_model", "SSH", "SSH", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +"ocean_model", "SST", "SST", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +"ocean_model", "SSS", "SSS", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +"ocean_model", "speed", "speed", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +"ocean_model", "SSU", "SSU", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +"ocean_model", "SSV", "SSV", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +"ocean_model", "frazil", "frazil", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +"ocean_model", "ePBL_h_ML", "ePBL", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +"ocean_model", "MLD_003", "MLD_003", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +"ocean_model", "MLD_0125", "MLD_0125", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +"ocean_model", "tob", "tob", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 + +# Z-Space Fields Provided for CMIP6 (CMOR Names): +"ocean_model_z", "uo", "uo", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +"ocean_model_z", "vo", "vo", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +"ocean_model_z", "so", "so", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +"ocean_model_z", "temp", "temp", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 + +# forcing +"ocean_model", "taux", "taux", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 +"ocean_model", "tauy", "tauy", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 +"ocean_model", "latent", "latent", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 +"ocean_model", "sensible", "sensible", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 +"ocean_model", "SW", "SW", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 +"ocean_model", "LW", "LW", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 +"ocean_model", "evap", "evap", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 +"ocean_model", "lprec", "lprec", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 +"ocean_model", "lrunoff", "lrunoff", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 +#"ocean_model", "frunoff", "frunoff", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 +"ocean_model", "fprec", "fprec", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 +"ocean_model", "LwLatSens", "LwLatSens", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 +"ocean_model", "Heat_PmE", "Heat_PmE", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 + +############## +# Ocean fields second lead time and after +############# +# static fields +ocean_model, "geolon", "geolon", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +ocean_model, "geolat", "geolat", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +ocean_model, "geolon_c", "geolon_c", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +ocean_model, "geolat_c", "geolat_c", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +ocean_model, "geolon_u", "geolon_u", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +ocean_model, "geolat_u", "geolat_u", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +ocean_model, "geolon_v", "geolon_v", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +ocean_model, "geolat_v", "geolat_v", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +#"ocean_model", "depth_ocean", "depth_ocean", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +#"ocean_model", "wet", "wet", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +ocean_model, "wet_c", "wet_c", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +ocean_model, "wet_u", "wet_u", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +ocean_model, "wet_v", "wet_v", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +ocean_model, "sin_rot", "sin_rot", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +ocean_model, "cos_rot", "cos_rot", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 + +# ocean output TSUV and others +ocean_model, "SSH", "SSH", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +ocean_model, "SST", "SST", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +ocean_model, "SSS", "SSS", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +ocean_model, "speed", "speed", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +ocean_model, "SSU", "SSU", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +ocean_model, "SSV", "SSV", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +ocean_model, "frazil", "frazil", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +ocean_model, "ePBL_h_ML", "ePBL", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +ocean_model, "MLD_003", "MLD_003", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +ocean_model, "MLD_0125", "MLD_0125", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +ocean_model, "tob", "tob", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 + +# Z-Space Fields Provided for CMIP6 (CMOR Names): +ocean_model_z, "uo", "uo", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +ocean_model_z, "vo", "vo", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +ocean_model_z, "so", "so", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +ocean_model_z, "temp", "temp", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 + +# forcing +ocean_model, "taux", "taux", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 +ocean_model, "tauy", "tauy", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 +ocean_model, "latent", "latent", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 +ocean_model, "sensible", "sensible", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 +ocean_model, "SW", "SW", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 +ocean_model, "LW", "LW", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 +ocean_model, "evap", "evap", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 +ocean_model, "lprec", "lprec", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 +ocean_model, "lrunoff", "lrunoff", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 +#"ocean_model", "frunoff", "frunoff", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 +ocean_model, "fprec", "fprec", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 +ocean_model, "LwLatSens", "LwLatSens", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 +ocean_model, "Heat_PmE", "Heat_PmE", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 + +################### +# Atmosphere fields +################### +"gfs_dyn", "ucomp", "ugrd", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "vcomp", "vgrd", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "sphum", "spfh", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "temp", "tmp", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "liq_wat", "clwmr", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "ice_wat", "icmr", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "snowwat", "snmr", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "rainwat", "rwmr", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "graupel", "grle", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "ice_nc", "nccice", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "rain_nc", "nconrd", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "o3mr", "o3mr", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "cld_amt", "cld_amt", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "delp", "dpres", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "delz", "delz", "fv3_history", "all", .false., "none", 2 +#"gfs_dyn", "pfhy", "preshy", "fv3_history", "all", .false., "none", 2 +#"gfs_dyn", "pfnh", "presnh", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "w", "dzdt", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "ps", "pressfc", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "hs", "hgtsfc", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "refl_10cm", "refl_10cm", "fv3_history", "all", .false., "none", 2 + +"gfs_phys", "cldfra", "cldfra", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "frzr", "frzr", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "frzrb", "frzrb", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "frozr", "frozr", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "frozrb", "frozrb", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "tsnowp", "tsnowp", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "tsnowpb", "tsnowpb", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "rhonewsn", "rhonewsn", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ALBDO_ave", "albdo_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "cnvprcp_ave", "cprat_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "cnvprcpb_ave", "cpratb_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "totprcp_ave", "prate_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "totprcpb_ave", "prateb_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "DLWRF", "dlwrf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "DLWRFI", "dlwrf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ULWRF", "ulwrf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ULWRFI", "ulwrf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "DSWRF", "dswrf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "DSWRFI", "dswrf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "USWRF", "uswrf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "USWRFI", "uswrf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "DSWRFtoa", "dswrf_avetoa", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "USWRFtoa", "uswrf_avetoa", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ULWRFtoa", "ulwrf_avetoa", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "gflux_ave", "gflux_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "hpbl", "hpbl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "lhtfl_ave", "lhtfl_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "shtfl_ave", "shtfl_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "pwat", "pwat", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "soilm", "soilm", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TCDC_aveclm", "tcdc_aveclm", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TCDC_avebndcl", "tcdc_avebndcl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TCDC_avelcl", "tcdc_avelcl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TCDC_avemcl", "tcdc_avemcl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TCDC_avehcl", "tcdc_avehcl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TCDCcnvcl", "tcdccnvcl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PRES_avelct", "pres_avelct", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PRES_avelcb", "pres_avelcb", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PRES_avemct", "pres_avemct", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PRES_avemcb", "pres_avemcb", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PRES_avehct", "pres_avehct", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PRES_avehcb", "pres_avehcb", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PREScnvclt", "prescnvclt", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PREScnvclb", "prescnvclb", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TEMP_avehct", "tmp_avehct", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TEMP_avemct", "tmp_avemct", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TEMP_avelct", "tmp_avelct", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "u-gwd_ave", "u-gwd_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "v-gwd_ave", "v-gwd_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "dusfc", "uflx_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "dvsfc", "vflx_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "acond", "acond", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "cduvb_ave", "cduvb_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "cpofp", "cpofp", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "duvb_ave", "duvb_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "csdlf_ave", "csdlf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "csusf_ave", "csusf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "csusf_avetoa", "csusftoa", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "csdsf_ave", "csdsf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "csulf_ave", "csulf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "csulf_avetoa", "csulftoa", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "cwork_ave", "cwork_aveclm", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "evbs_ave", "evbs_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "evcw_ave", "evcw_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "fldcp", "fldcp", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "hgt_hyblev1", "hgt_hyblev1", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "spfh_hyblev1", "spfh_hyblev1", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ugrd_hyblev1", "ugrd_hyblev1", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "vgrd_hyblev1", "vgrd_hyblev1", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "tmp_hyblev1", "tmp_hyblev1", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "gfluxi", "gflux", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "lhtfl", "lhtfl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "shtfl", "shtfl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "pevpr", "pevpr", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "pevpr_ave", "pevpr_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "sbsno_ave", "sbsno_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "sfexc", "sfexc", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "snohf", "snohf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "snowc_ave", "snowc_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "spfhmax2m", "spfhmax_max2m", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "spfhmin2m", "spfhmin_min2m", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "tmpmax2m", "tmax_max2m", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "tmpmin2m", "tmin_min2m", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ssrun_acc", "ssrun_acc", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "sunsd_acc", "sunsd_acc", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "watr_acc", "watr_acc", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "wilt", "wilt", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "vbdsf_ave", "vbdsf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "vddsf_ave", "vddsf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "nbdsf_ave", "nbdsf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "nddsf_ave", "nddsf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "trans_ave", "trans_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "psurf", "pressfc", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "u10m", "ugrd10m", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "v10m", "vgrd10m", "fv3_history2d", "all", .false., "none", 2 + +"gfs_phys", "pahi", "pahi", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "pah_ave", "pah_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ecan_acc", "ecan_acc", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "etran_acc", "etran_acc", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "edir_acc", "edir_acc", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "wa_acc", "wa_acc", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "lfrac", "lfrac", "fv3_history2d", "all", .false., "none", 2 + +"gfs_sfc", "crain", "crain", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "tprcp", "tprcp", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "rainc", "cnvprcp", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "hgtsfc", "orog", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "weasd", "weasd", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "f10m", "f10m", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "q2m", "spfh2m", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "t2m", "tmp2m", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "tsfc", "tmpsfc", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "vtype", "vtype", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "stype", "sotyp", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slmsksfc", "land", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "vfracsfc", "veg", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "zorlsfc", "sfcr", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "uustar", "fricv", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilt1", "soilt1" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilt2", "soilt2" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilt3", "soilt3" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilt4", "soilt4" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilw1", "soilw1" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilw2", "soilw2" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilw3", "soilw3" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilw4", "soilw4" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slc_1", "soill1", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slc_2", "soill2", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slc_3", "soill3", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slc_4", "soill4", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slope", "sltyp", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "alnsf", "alnsf", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "alnwf", "alnwf", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "alvsf", "alvsf", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "alvwf", "alvwf", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "canopy", "cnwat", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "facsf", "facsf", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "facwf", "facwf", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "ffhh", "ffhh", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "ffmm", "ffmm", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "fice", "icec", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "hice", "icetk", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "snoalb", "snoalb", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "shdmax", "shdmax", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "shdmin", "shdmin", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "snowd", "snod", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "tg3", "tg3", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "tisfc", "tisfc", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "tref", "tref", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "z_c", "zc", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "c_0", "c0", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "c_d", "cd", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "w_0", "w0", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "w_d", "wd", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xt", "xt", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xz", "xz", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "dt_cool", "dtcool", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xs", "xs", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xu", "xu", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xv", "xv", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xtts", "xtts", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xzts", "xzts", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "d_conv", "dconv", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "qrain", "qrain", "fv3_history2d", "all", .false., "none", 2 + +#============================================================================================= +# +#====> This file can be used with diag_manager/v2.0a (or higher) <==== +# +# +# FORMATS FOR FILE ENTRIES (not all input values are used) +# ------------------------ +# +#"file_name", output_freq, "output_units", format, "time_units", "long_name", +# +# +#output_freq: > 0 output frequency in "output_units" +# = 0 output frequency every time step +# =-1 output frequency at end of run +# +#output_units = units used for output frequency +# (years, months, days, minutes, hours, seconds) +# +#time_units = units used to label the time axis +# (days, minutes, hours, seconds) +# +# +# FORMAT FOR FIELD ENTRIES (not all input values are used) +# ------------------------ +# +#"module_name", "field_name", "output_name", "file_name" "time_sampling", time_avg, "other_opts", packing +# +#time_avg = .true. or .false. +# +#packing = 1 double precision +# = 2 float +# = 4 packed 16-bit integers +# = 8 packed 1-byte (not tested?) diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index 2cc34eaacd..468db908c4 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -448,11 +448,21 @@ MOM6_postdet() { (( midpoint = last_fhr + interval/2 )) vdate=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${fhr} hours" +%Y%m%d%H) - vdate_mid=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${midpoint} hours" +%Y%m%d%H) + #If OFFSET_START_HOUR is greater than 0, OFFSET_START_HOUR should be added to the midpoint for first lead time + if (( OFFSET_START_HOUR > 0 )) && (( fhr == FHOUT_OCN ));then + vdate_mid=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + $(( midpoint + OFFSET_START_HOUR )) hours" +%Y%m%d%H) + else + vdate_mid=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${midpoint} hours" +%Y%m%d%H) + fi # Native model output uses window midpoint in the filename, but we are mapping that to the end of the period for COM - source_file="ocn_${vdate_mid:0:4}_${vdate_mid:4:2}_${vdate_mid:6:2}_${vdate_mid:8:2}.nc" - dest_file="${RUN}.ocean.t${cyc}z.${interval}hr_avg.f${fhr3}.nc" + if (( OFFSET_START_HOUR > 0 )) && (( fhr == FHOUT_OCN ));then + source_file="ocn_lead1_${vdate_mid:0:4}_${vdate_mid:4:2}_${vdate_mid:6:2}_${vdate_mid:8:2}.nc" + dest_file="${RUN}.ocean.t${cyc}z.${interval}hr_avg.f${fhr3}.nc" + else + source_file="ocn_${vdate_mid:0:4}_${vdate_mid:4:2}_${vdate_mid:6:2}_${vdate_mid:8:2}.nc" + dest_file="${RUN}.ocean.t${cyc}z.${interval}hr_avg.f${fhr3}.nc" + fi ${NLN} "${COMOUT_OCEAN_HISTORY}/${dest_file}" "${DATA}/MOM6_OUTPUT/${source_file}" last_fhr=${fhr} diff --git a/ush/parsing_namelists_FV3.sh b/ush/parsing_namelists_FV3.sh index b88849d7e7..3698137e94 100755 --- a/ush/parsing_namelists_FV3.sh +++ b/ush/parsing_namelists_FV3.sh @@ -17,6 +17,10 @@ if (( gwd_opt == 2 )) && [[ ${do_gsl_drag_ls_bl} == ".true." ]]; then cdmbgwd=${cdmbgwd_gsl} fi +if (( OFFSET_START_HOUR != 0 )); then + DIAG_TABLE="${PARMgfs}/ufs/fv3/diag_table_replay" +fi + # ensure non-prognostic tracers are set dnats=${dnats:-0} @@ -42,6 +46,16 @@ local SDAY=${current_cycle:6:2} local CHOUR=${current_cycle:8:2} local MOM6_OUTPUT_DIR="./MOM6_OUTPUT" +if (( OFFSET_START_HOUR > 0 )); then + local current_cycle_p1=$(${NDATE} "${FHOUT_OCN}" "${current_cycle}") + local current_cycle_offset=$(${NDATE} "${OFFSET_START_HOUR}" "${current_cycle}") + local SYEAR1=${current_cycle_p1:0:4} + local SMONTH1=${current_cycle_p1:4:2} + local SDAY1=${current_cycle_p1:6:2} + local CHOUR1=${current_cycle_p1:8:2} + local CHOUR_offset=${current_cycle_offset:8:2} +fi + atparse < "${template}" >> "diag_table" From dd6b1b1f97a2e66d2277d8486843bf581d41cc2d Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Mon, 24 Jun 2024 14:05:20 +0000 Subject: [PATCH 057/141] Fix shellcheck issue --- ush/parsing_namelists_FV3.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ush/parsing_namelists_FV3.sh b/ush/parsing_namelists_FV3.sh index 3698137e94..4cec6c05aa 100755 --- a/ush/parsing_namelists_FV3.sh +++ b/ush/parsing_namelists_FV3.sh @@ -47,8 +47,10 @@ local CHOUR=${current_cycle:8:2} local MOM6_OUTPUT_DIR="./MOM6_OUTPUT" if (( OFFSET_START_HOUR > 0 )); then - local current_cycle_p1=$(${NDATE} "${FHOUT_OCN}" "${current_cycle}") - local current_cycle_offset=$(${NDATE} "${OFFSET_START_HOUR}" "${current_cycle}") + local current_cycle_p1 + current_cycle_p1=$(${NDATE} "${FHOUT_OCN}" "${current_cycle}") + local current_cycle_offset + current_cycle_offset=$(${NDATE} "${OFFSET_START_HOUR}" "${current_cycle}") local SYEAR1=${current_cycle_p1:0:4} local SMONTH1=${current_cycle_p1:4:2} local SDAY1=${current_cycle_p1:6:2} From ecf788f0365b3250ed1da8ba69e25a463e22a6da Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 25 Jun 2024 17:10:51 +0000 Subject: [PATCH 058/141] Increase flexibility of extractvars dependencies A conditional statement has been added to allow for certain dependencies to be omitted from extractvars for components that are not set to run. --- workflow/rocoto/gefs_tasks.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 2eee090225..00f9553434 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -486,14 +486,18 @@ def wavepostpnt(self): def extractvars(self): deps = [] - dep_dict = {'type': 'task', 'name': f'wave_post_grid_mem#member#'} - deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': f'ocean_prod'} - deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': f'ice_prod'} - deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': f'atmos_prod'} - deps.append(rocoto.add_dependency(dep_dict)) + if self.app_config.do_wave: + dep_dict = {'type': 'task', 'name': f'wave_post_grid_mem#member#'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.app_config.do_ocean: + dep_dict = {'type': 'metatask', 'name': f'ocean_prod'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.app_config.do_ice: + dep_dict = {'type': 'metatask', 'name': f'ice_prod'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.app_config.do_atm: + dep_dict = {'type': 'metatask', 'name': f'atmos_prod'} + deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) extractvars_envars = self.envars.copy() extractvars_dict = {'ENSMEM': '#member#', From 1beb0448f48aeb89309ca42adb7f834d7e2f6f49 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 25 Jun 2024 17:18:26 +0000 Subject: [PATCH 059/141] Allow only specific components to be processed Only components that are set to run will be processed in the extractvars task. --- scripts/exglobal_extractvars.sh | 58 +++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 25 deletions(-) diff --git a/scripts/exglobal_extractvars.sh b/scripts/exglobal_extractvars.sh index a0824080dd..78742e701f 100755 --- a/scripts/exglobal_extractvars.sh +++ b/scripts/exglobal_extractvars.sh @@ -40,39 +40,47 @@ export compress_ice=${compress_ice:-1} #1: compress extracted ice product, 0: do export FHOUT_WAV_NOSCRUB=${FHOUT_WAV_NOSCRUB:-6} #Frequency of wave output to be saved on disk #Extract variables for atmosphere -if [[ ! -d "${DATA}/mem${ENSMEM}_atmos" ]]; then - mkdir -p "${DATA}/mem${ENSMEM}_atmos" +if [[ "${DO_ATM}" == "YES" ]];then + if [[ ! -d "${DATA}/mem${ENSMEM}_atmos" ]]; then + mkdir -p "${DATA}/mem${ENSMEM}_atmos" + fi + ${EXTRCTVARA} "${DATA}/mem${ENSMEM}_atmos" + cp -pr "${DATA}/mem${ENSMEM}_atmos/f2d" "${COM_RFCST_PROD_ATMOS}" + cp -pr "${DATA}/mem${ENSMEM}_atmos/f3d" "${COM_RFCST_PROD_ATMOS}" fi -${EXTRCTVARA} "${DATA}/mem${ENSMEM}_atmos" -cp -pr "${DATA}/mem${ENSMEM}_atmos/f2d" "${COM_RFCST_PROD_ATMOS}" -cp -pr "${DATA}/mem${ENSMEM}_atmos/f3d" "${COM_RFCST_PROD_ATMOS}" #Extract variables for ocean -export component_name="ocn" -if [[ ! -d "${DATA}/mem${ENSMEM}_ocn" ]]; then - mkdir -p "${DATA}/mem${ENSMEM}_ocn" +if [[ "${DO_OCN}" == "YES" ]];then + export component_name="ocn" + if [[ ! -d "${DATA}/mem${ENSMEM}_ocn" ]]; then + mkdir -p "${DATA}/mem${ENSMEM}_ocn" + fi + if [[ "${ocn_dataformat}" == "netcdf" ]]; then varlist_ocn=${varlist_ocn_netcdf}; fi + if [[ "${ocn_dataformat}" == "grib2" ]]; then varlist_ocn=${varlist_ocn_grib2}; fi + ${EXTRCTVARO} "${DATA}/mem${ENSMEM}_ocn" "${varlist_ocn}" "${ocn_dataformat}" "${ocnres}" "${compress_ocn}" "${FHOUT_OCN_GFS}" + cp -pr "${DATA}/mem${ENSMEM}_ocn/." "${COM_RFCST_PROD_OCN}" fi -if [[ "${ocn_dataformat}" == "netcdf" ]]; then varlist_ocn=${varlist_ocn_netcdf}; fi -if [[ "${ocn_dataformat}" == "grib2" ]]; then varlist_ocn=${varlist_ocn_grib2}; fi -${EXTRCTVARO} "${DATA}/mem${ENSMEM}_ocn" "${varlist_ocn}" "${ocn_dataformat}" "${ocnres}" "${compress_ocn}" "${FHOUT_OCN_GFS}" -cp -pr "${DATA}/mem${ENSMEM}_ocn/." "${COM_RFCST_PROD_OCN}" #Extract variables for ice -export component_name="ice" -if [[ ! -d "${DATA}/mem${ENSMEM}_ice" ]]; then - mkdir -p "${DATA}/mem${ENSMEM}_ice" -fi -if [[ "${ice_dataformat}" == "netcdf" ]]; then varlist_ice=${varlist_ice_netcdf}; fi -if [[ "${ice_dataformat}" == "grib2" ]]; then varlist_ice=${varlist_ice_grib2}; fi -${EXTRCTVARO} "${DATA}/mem${ENSMEM}_ice" "${varlist_ice}" "${ice_dataformat}" "${iceres}" "${compress_ice}" "${FHOUT_ICE_GFS}" -cp -pr "${DATA}/mem${ENSMEM}_ice/." "${COM_RFCST_PROD_ICE}" +if [[ "${DO_ICE}" == "YES" ]];then + export component_name="ice" + if [[ ! -d "${DATA}/mem${ENSMEM}_ice" ]]; then + mkdir -p "${DATA}/mem${ENSMEM}_ice" + fi + if [[ "${ice_dataformat}" == "netcdf" ]]; then varlist_ice=${varlist_ice_netcdf}; fi + if [[ "${ice_dataformat}" == "grib2" ]]; then varlist_ice=${varlist_ice_grib2}; fi + ${EXTRCTVARO} "${DATA}/mem${ENSMEM}_ice" "${varlist_ice}" "${ice_dataformat}" "${iceres}" "${compress_ice}" "${FHOUT_ICE_GFS}" + cp -pr "${DATA}/mem${ENSMEM}_ice/." "${COM_RFCST_PROD_ICE}" +fi #Extract variables for wave -export component_name="wav" -if [[ ! -d "${DATA}/mem${ENSMEM}_wav" ]]; then - mkdir -p "${DATA}/mem${ENSMEM}_wav" +if [[ "${DO_WAVE}" == "YES" ]];then + export component_name="wav" + if [[ ! -d "${DATA}/mem${ENSMEM}_wav" ]]; then + mkdir -p "${DATA}/mem${ENSMEM}_wav" + fi + ${EXTRCTVARW} "${DATA}/mem${ENSMEM}_wav" + cp -pr "${DATA}/mem${ENSMEM}_wav/." "${COM_RFCST_PROD_WAV}" fi -${EXTRCTVARW} "${DATA}/mem${ENSMEM}_wav" -cp -pr "${DATA}/mem${ENSMEM}_wav/." "${COM_RFCST_PROD_WAV}" exit 0 From 987f27fd66bec525200a4a4a8922723f04fa252e Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 2 Jul 2024 12:23:43 +0000 Subject: [PATCH 060/141] Refine procedure to get fv3 fhours with replay ICs --- ush/forecast_postdet.sh | 4 ++-- ush/forecast_predet.sh | 24 ++++++++++++++++++------ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index 468db908c4..675a2c07a3 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -204,8 +204,8 @@ EOF if [[ "${QUILTING}" = ".true." ]] && [[ "${OUTPUT_GRID}" = "gaussian_grid" ]]; then local FH2 FH3 for fhr in ${FV3_OUTPUT_FH}; do - FH3=$(printf %03i "${fhr}") - FH2=$(printf %02i "${fhr}") + FH3=$(printf %03i "$(printf %.0f $(echo "${fhr}" | bc -l))") + FH2=$(printf %02i "$(printf %.0f $(echo "${fhr}" | bc -l))") ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.atmf${FH3}.nc" "atmf${FH3}.nc" ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.sfcf${FH3}.nc" "sfcf${FH3}.nc" ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.atm.logf${FH3}.txt" "log.atm.f${FH3}" diff --git a/ush/forecast_predet.sh b/ush/forecast_predet.sh index 9183e86002..0bf0d1327f 100755 --- a/ush/forecast_predet.sh +++ b/ush/forecast_predet.sh @@ -145,13 +145,25 @@ FV3_predet(){ fi # Convert output settings into an explicit list for FV3 - FV3_OUTPUT_FH="" - local fhr=${FHMIN} - if (( FHOUT_HF > 0 && FHMAX_HF > 0 )); then - FV3_OUTPUT_FH="${FV3_OUTPUT_FH} $(seq -s ' ' "${FHMIN}" "${FHOUT_HF}" "${FHMAX_HF}")" - fhr=${FHMAX_HF} + if (( OFFSET_START_HOUR > 0 ));then + FV3_OUTPUT_FH="$(echo "scale=5; $OFFSET_START_HOUR+($DELTIM/3600)" | bc -l)" + local FHMIN_REPLAY=$(( FHMIN + FHOUT )) + local fhr=${FHMIN_REPLAY} + if (( FHOUT_HF > 0 && FHMAX_HF > 0 )); then + FV3_OUTPUT_FH="${FV3_OUTPUT_FH} $(seq -s ' ' "${FHMIN_REPLAY}" "${FHOUT_HF}" "${FHMAX_HF}")" + fhr=${FHMAX_HF} + fi + FV3_OUTPUT_FH="${FV3_OUTPUT_FH} $(seq -s ' ' "${fhr}" "${FHOUT}" "${FHMAX}")" + else + FV3_OUTPUT_FH="" + local fhr=${FHMIN} + if (( FHOUT_HF > 0 && FHMAX_HF > 0 )); then + FV3_OUTPUT_FH="${FV3_OUTPUT_FH} $(seq -s ' ' "${FHMIN}" "${FHOUT_HF}" "${FHMAX_HF}")" + fhr=${FHMAX_HF} + fi + FV3_OUTPUT_FH="${FV3_OUTPUT_FH} $(seq -s ' ' "${fhr}" "${FHOUT}" "${FHMAX}")" fi - FV3_OUTPUT_FH="${FV3_OUTPUT_FH} $(seq -s ' ' "${fhr}" "${FHOUT}" "${FHMAX}")" + # Other options MEMBER=$(( 10#${ENSMEM:-"-1"} )) # -1: control, 0: ensemble mean, >0: ensemble member $MEMBER From 674c40f0a527d93c39d73f62d5ef1801b93c740a Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 2 Jul 2024 13:04:24 +0000 Subject: [PATCH 061/141] Fix shellcheck errors --- ush/forecast_postdet.sh | 6 ++++-- ush/forecast_predet.sh | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index 6310a5a069..a41f4a3d72 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -204,8 +204,10 @@ EOF if [[ "${QUILTING}" = ".true." ]] && [[ "${OUTPUT_GRID}" = "gaussian_grid" ]]; then local FH2 FH3 for fhr in ${FV3_OUTPUT_FH}; do - FH3=$(printf %03i "$(printf %.0f $(echo "${fhr}" | bc -l))") - FH2=$(printf %02i "$(printf %.0f $(echo "${fhr}" | bc -l))") + fhrf=$(echo "${fhr}" | bc -l) + fhri=$(printf %.0f "${fhrf}") + FH3=$(printf %03i "${fhri}") + FH2=$(printf %02i "${fhri}") ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.atmf${FH3}.nc" "atmf${FH3}.nc" ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.sfcf${FH3}.nc" "sfcf${FH3}.nc" ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.atm.logf${FH3}.txt" "log.atm.f${FH3}" diff --git a/ush/forecast_predet.sh b/ush/forecast_predet.sh index 0bf0d1327f..9dd78011c2 100755 --- a/ush/forecast_predet.sh +++ b/ush/forecast_predet.sh @@ -146,7 +146,7 @@ FV3_predet(){ # Convert output settings into an explicit list for FV3 if (( OFFSET_START_HOUR > 0 ));then - FV3_OUTPUT_FH="$(echo "scale=5; $OFFSET_START_HOUR+($DELTIM/3600)" | bc -l)" + FV3_OUTPUT_FH="$(echo "scale=5; ${OFFSET_START_HOUR}+(${DELTIM}/3600)" | bc -l)" local FHMIN_REPLAY=$(( FHMIN + FHOUT )) local fhr=${FHMIN_REPLAY} if (( FHOUT_HF > 0 && FHMAX_HF > 0 )); then From 2be9a15aeb03c7e2e8c137e5b9f0f404d8dcdf23 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 2 Jul 2024 18:35:50 +0000 Subject: [PATCH 062/141] Move extractvars variable defs to config The extractvars variables have been moved from exscript to config.extractvars. --- parm/config/gefs/config.extractvars | 23 +++++++++++++---------- scripts/exglobal_extractvars.sh | 17 ----------------- 2 files changed, 13 insertions(+), 27 deletions(-) diff --git a/parm/config/gefs/config.extractvars b/parm/config/gefs/config.extractvars index 428da045e6..315244c52f 100644 --- a/parm/config/gefs/config.extractvars +++ b/parm/config/gefs/config.extractvars @@ -7,19 +7,22 @@ echo "BEGIN: config.extractvars" . "${EXPDIR}/config.resources" extractvars -export atmosextract=1 -export ocniceextract=1 -export wavextract=1 - -export compress_ocn=0 -export compress_ice=0 - +export compress_ocn=0 #1: compress extracted ocean product, 0: do not compress extracted ocean product +export compress_ice=0 #1: compress extracted ice product, 0: do not compress extracted ice product export FHMAX=120 -export FHMAXHF=60 -export FHOUTHF=6 -export FHOUTLF=12 + +export FHMAXHF=60 #The forecast length of the high output frequency (in hours) for atmos +export FHOUTHF=6 #The high output frequency (in hours) for atmos +export FHOUTLF=12 #The low output frequency (in hours) for atmos export ocn_dataformat="netcdf" export ice_dataformat="netcdf" +export ocnres="5p00" #Resolution of ocean products +export iceres="5p00" #Resolution of ice products +export ocnres="5p00" #Resolution of ocean products +export wavres="5p00" #Resolution of wave products + +export FHOUT_WAV_NOSCRUB=6 #Frequency of wave output to be saved on disk + echo "END: config.extractvars" diff --git a/scripts/exglobal_extractvars.sh b/scripts/exglobal_extractvars.sh index 78742e701f..9740da0547 100755 --- a/scripts/exglobal_extractvars.sh +++ b/scripts/exglobal_extractvars.sh @@ -22,23 +22,6 @@ export varlist_ice_netcdf=${varlist_ice_netcdf:-"${PARMgfs}/product/gefs_ice_sho export varlist_ocn_grib2=${varlist_ocn_grib2:-"${PARMgfs}/product/gefs_ocn_shortparmlist_grib2.parm"} #Parameter table for wave variables export varlist_ice_grib2=${varlist_ice_grib2:-"${PARMgfs}/product/gefs_ice_shortparmlist_grib2.parm"} #Parameter table for wave variables -# Variables used in this job -export FHMIN=${FHMIN:-0} #The first lead hour -export FHMAX=${FHMAX:-120} #The total number of lead hours -export FHMAXHF=${FHMAXHF:-60} #The maximum lead hour that will use a high output frequency -export FHOUTHF=${FHOUTHF:-6} #The high output frequency (in hours) -export FHOUTLF=${FHOUTLF:-12} #The low output frequency (in hours) -export cycle=${cycle:-"t00z"} #cycle -export ocn_dataformat=${ocn_dataformat:-"grib2"} #the data format of the ocean products to be read -export ice_dataformat=${ice_dataformat:-"grib2"} #the data format of the ice products to be read -export ocnres=${ocnres:-"5p00"} #Resolution of ocean products -export iceres=${iceres:-"5p00"} #Resolution of ice products -export ocnres=${ocnres:-"5p00"} #Resolution of ocean products -export wavres=${wavinres:-"5p00"} #Resolution of wave products -export compress_ocn=${compress_ocn:-1} #1: compress extracted ocean product, 0: do not compress extracted ocean product -export compress_ice=${compress_ice:-1} #1: compress extracted ice product, 0: do not compress extracted ice product -export FHOUT_WAV_NOSCRUB=${FHOUT_WAV_NOSCRUB:-6} #Frequency of wave output to be saved on disk - #Extract variables for atmosphere if [[ "${DO_ATM}" == "YES" ]];then if [[ ! -d "${DATA}/mem${ENSMEM}_atmos" ]]; then From c2db0a548ca30f222b5494a6d42af95931cbec26 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Mon, 8 Jul 2024 19:54:18 +0000 Subject: [PATCH 063/141] Replace COM with COMIN and COMOUT The COM variables have been replaced with COMIN and COMOUT in the extractvars task. --- jobs/JGLOBAL_EXTRACTVARS | 32 ++++++++++++++++---------------- scripts/exglobal_extractvars.sh | 10 +++++----- ush/gefs_atmos_extractvars.sh | 12 ++++++------ ush/gefs_ocnice_extractvars.sh | 8 ++++---- ush/gefs_wav_extractvars.sh | 2 +- 5 files changed, 32 insertions(+), 32 deletions(-) diff --git a/jobs/JGLOBAL_EXTRACTVARS b/jobs/JGLOBAL_EXTRACTVARS index a77dbde852..795f673a5c 100755 --- a/jobs/JGLOBAL_EXTRACTVARS +++ b/jobs/JGLOBAL_EXTRACTVARS @@ -5,29 +5,29 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "extractvars" -c "base extractvars" # Set COM Paths for grid in '0p25' '0p50' '1p00'; do - prod_dir="COM_ATMOS_GRIB_${grid}" + prod_dir="COMIN_ATMOS_GRIB_${grid}" GRID=${grid} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx "${prod_dir}:COM_ATMOS_GRIB_GRID_TMPL" if [[ ! -d "${!prod_dir}" ]]; then mkdir -m 775 -p "${!prod_dir}"; fi done -YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COM_OCEAN_HISTORY" -YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COM_OCEAN_GRIB" -YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COM_OCEAN_NETCDF" -YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COM_ICE_HISTORY" -YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COM_ICE_GRIB" -YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COM_ICE_NETCDF" +YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COMIN_OCEAN_HISTORY:COM_OCEAN_HISTORY_TMPL" +YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COMIN_OCEAN_GRIB:COM_OCEAN_GRIB_TMPL" +YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COMIN_OCEAN_NETCDF:COM_OCEAN_NETCDF_TMPL" +YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COMIN_ICE_HISTORY:COM_ICE_HISTORY_TMPL" +YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COMIN_ICE_GRIB:COM_ICE_GRIB_TMPL" +YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COMIN_ICE_NETCDF:COM_ICE_NETCDF_TMPL" -YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COM_WAVE_GRID" +YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COMIN_WAVE_GRID:COM_WAVE_GRID_TMPL" -YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COM_RFCST_PROD_ATMOS" -YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COM_RFCST_PROD_OCN" -YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COM_RFCST_PROD_ICE" -YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COM_RFCST_PROD_WAV" +YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COMOUT_RFCST_PROD_ATMOS:COM_RFCST_PROD_ATMOS_TMPL" +YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COMOUT_RFCST_PROD_OCN:COM_RFCST_PROD_OCN_TMPL" +YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COMOUT_RFCST_PROD_ICE:COM_RFCST_PROD_ICE_TMPL" +YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COMOUT_RFCST_PROD_WAV:COM_RFCST_PROD_WAV_TMPL" -if [[ ! -d "${COM_RFCST_PROD_ATMOS}" ]]; then mkdir -m 775 -p "${COM_RFCST_PROD_ATMOS}"; fi -if [[ ! -d "${COM_RFCST_PROD_OCN}" ]]; then mkdir -m 775 -p "${COM_RFCST_PROD_OCN}"; fi -if [[ ! -d "${COM_RFCST_PROD_WAV}" ]]; then mkdir -m 775 -p "${COM_RFCST_PROD_WAV}"; fi -if [[ ! -d "${COM_RFCST_PROD_ICE}" ]]; then mkdir -m 775 -p "${COM_RFCST_PROD_ICE}"; fi +if [[ ! -d "${COMOUT_RFCST_PROD_ATMOS}" ]]; then mkdir -m 775 -p "${COMOUT_RFCST_PROD_ATMOS}"; fi +if [[ ! -d "${COMOUT_RFCST_PROD_OCN}" ]]; then mkdir -m 775 -p "${COMOUT_RFCST_PROD_OCN}"; fi +if [[ ! -d "${COMOUT_RFCST_PROD_WAV}" ]]; then mkdir -m 775 -p "${COMOUT_RFCST_PROD_WAV}"; fi +if [[ ! -d "${COMOUT_RFCST_PROD_ICE}" ]]; then mkdir -m 775 -p "${COMOUT_RFCST_PROD_ICE}"; fi # Execute the Script "${SCRgfs}/exglobal_extractvars.sh" diff --git a/scripts/exglobal_extractvars.sh b/scripts/exglobal_extractvars.sh index 9740da0547..373901a7b3 100755 --- a/scripts/exglobal_extractvars.sh +++ b/scripts/exglobal_extractvars.sh @@ -28,8 +28,8 @@ if [[ "${DO_ATM}" == "YES" ]];then mkdir -p "${DATA}/mem${ENSMEM}_atmos" fi ${EXTRCTVARA} "${DATA}/mem${ENSMEM}_atmos" - cp -pr "${DATA}/mem${ENSMEM}_atmos/f2d" "${COM_RFCST_PROD_ATMOS}" - cp -pr "${DATA}/mem${ENSMEM}_atmos/f3d" "${COM_RFCST_PROD_ATMOS}" + cp -pr "${DATA}/mem${ENSMEM}_atmos/f2d" "${COMOUT_RFCST_PROD_ATMOS}" + cp -pr "${DATA}/mem${ENSMEM}_atmos/f3d" "${COMOUT_RFCST_PROD_ATMOS}" fi #Extract variables for ocean @@ -41,7 +41,7 @@ if [[ "${DO_OCN}" == "YES" ]];then if [[ "${ocn_dataformat}" == "netcdf" ]]; then varlist_ocn=${varlist_ocn_netcdf}; fi if [[ "${ocn_dataformat}" == "grib2" ]]; then varlist_ocn=${varlist_ocn_grib2}; fi ${EXTRCTVARO} "${DATA}/mem${ENSMEM}_ocn" "${varlist_ocn}" "${ocn_dataformat}" "${ocnres}" "${compress_ocn}" "${FHOUT_OCN_GFS}" - cp -pr "${DATA}/mem${ENSMEM}_ocn/." "${COM_RFCST_PROD_OCN}" + cp -pr "${DATA}/mem${ENSMEM}_ocn/." "${COMOUT_RFCST_PROD_OCN}" fi #Extract variables for ice @@ -53,7 +53,7 @@ if [[ "${DO_ICE}" == "YES" ]];then if [[ "${ice_dataformat}" == "netcdf" ]]; then varlist_ice=${varlist_ice_netcdf}; fi if [[ "${ice_dataformat}" == "grib2" ]]; then varlist_ice=${varlist_ice_grib2}; fi ${EXTRCTVARO} "${DATA}/mem${ENSMEM}_ice" "${varlist_ice}" "${ice_dataformat}" "${iceres}" "${compress_ice}" "${FHOUT_ICE_GFS}" - cp -pr "${DATA}/mem${ENSMEM}_ice/." "${COM_RFCST_PROD_ICE}" + cp -pr "${DATA}/mem${ENSMEM}_ice/." "${COMOUT_RFCST_PROD_ICE}" fi #Extract variables for wave @@ -63,7 +63,7 @@ if [[ "${DO_WAVE}" == "YES" ]];then mkdir -p "${DATA}/mem${ENSMEM}_wav" fi ${EXTRCTVARW} "${DATA}/mem${ENSMEM}_wav" - cp -pr "${DATA}/mem${ENSMEM}_wav/." "${COM_RFCST_PROD_WAV}" + cp -pr "${DATA}/mem${ENSMEM}_wav/." "${COMOUT_RFCST_PROD_WAV}" fi exit 0 diff --git a/ush/gefs_atmos_extractvars.sh b/ush/gefs_atmos_extractvars.sh index c075227b2e..3cb6ca870e 100755 --- a/ush/gefs_atmos_extractvars.sh +++ b/ush/gefs_atmos_extractvars.sh @@ -43,14 +43,14 @@ for outtype in "f2d" "f3d"; do fi if [[ "${outres}" == "0p25" ]];then - infile1=${COM_ATMOS_GRIB_0p25}/gefs.${cycle}.pgrb2.${outres}.f${fnh} - infile2=${COM_ATMOS_GRIB_0p25}/gefs.${cycle}.pgrb2b.${outres}.f${fnh} + infile1=${COMIN_ATMOS_GRIB_0p25}/gefs.${cycle}.pgrb2.${outres}.f${fnh} + infile2=${COMIN_ATMOS_GRIB_0p25}/gefs.${cycle}.pgrb2b.${outres}.f${fnh} elif [[ "${outres}" == "0p50" ]];then - infile1=${COM_ATMOS_GRIB_0p50}/gefs.${cycle}.pgrb2.${outres}.f${fnh} - infile2=${COM_ATMOS_GRIB_0p50}/gefs.${cycle}.pgrb2b.${outres}.f${fnh} + infile1=${COMIN_ATMOS_GRIB_0p50}/gefs.${cycle}.pgrb2.${outres}.f${fnh} + infile2=${COMIN_ATMOS_GRIB_0p50}/gefs.${cycle}.pgrb2b.${outres}.f${fnh} elif [[ "${outres}" == "1p00" ]];then - infile1=${COM_ATMOS_GRIB_1p00}/gefs.${cycle}.pgrb2.${outres}.f${fnh} - infile2=${COM_ATMOS_GRIB_1p00}/gefs.${cycle}.pgrb2b.${outres}.f${fnh} + infile1=${COMIN_ATMOS_GRIB_1p00}/gefs.${cycle}.pgrb2.${outres}.f${fnh} + infile2=${COMIN_ATMOS_GRIB_1p00}/gefs.${cycle}.pgrb2b.${outres}.f${fnh} fi oufile=${outdirpre}/gefs.${cycle}.pgrb2.${outres}.f${fnh} rm -f "${oufile}" #remove outfile if it already exists before extraction diff --git a/ush/gefs_ocnice_extractvars.sh b/ush/gefs_ocnice_extractvars.sh index d045337d13..691cd4f447 100755 --- a/ush/gefs_ocnice_extractvars.sh +++ b/ush/gefs_ocnice_extractvars.sh @@ -18,20 +18,20 @@ while [[ ${nh} -le ${FHMAX} ]];do if [[ "${dataformat}" == "grib2" ]];then if [[ ${component_name} == "ocn" ]];then - infile=${COM_OCEAN_GRIB}/${datares}/gefs.ocean.${cycle}.${datares}.f${fnh}.grib2 + infile=${COMIN_OCEAN_GRIB}/${datares}/gefs.ocean.${cycle}.${datares}.f${fnh}.grib2 fi if [[ ${component_name} == "ice" ]];then - infile=${COM_ICE_GRIB}/${datares}/gefs.ice.${cycle}.${datares}.f${fnh}.grib2 + infile=${COMIN_ICE_GRIB}/${datares}/gefs.ice.${cycle}.${datares}.f${fnh}.grib2 fi oufile1=${outdirpre}/gefs.${component_name}.${cycle}.${datares}.f${fnh}.grib2 fi if [[ "${dataformat}" == "netcdf" ]];then if [[ ${component_name} == "ocn" ]];then - infile=${COM_OCEAN_NETCDF}/gefs.ocean.${cycle}.${datares}.f${fnh}.nc + infile=${COMIN_OCEAN_NETCDF}/gefs.ocean.${cycle}.${datares}.f${fnh}.nc fi if [[ ${component_name} == "ice" ]];then - infile=${COM_ICE_NETCDF}/gefs.ice.${cycle}.${datares}.f${fnh}.nc + infile=${COMIN_ICE_NETCDF}/gefs.ice.${cycle}.${datares}.f${fnh}.nc fi oufile1=${outdirpre}/gefs.${component_name}.${cycle}.${datares}.f${fnh}.nc fi diff --git a/ush/gefs_wav_extractvars.sh b/ush/gefs_wav_extractvars.sh index 7547a58cd8..e20785fe67 100755 --- a/ush/gefs_wav_extractvars.sh +++ b/ush/gefs_wav_extractvars.sh @@ -11,7 +11,7 @@ nh=6 while [[ ${nh} -le ${FHMAX_WAV} ]];do fnh=$(printf "%3.3d" "${nh}") - infile=${COM_WAVE_GRID}/gefswave.${cycle}.global.${wavres}.f${fnh}.grib2 + infile=${COMIN_WAVE_GRID}/gefswave.${cycle}.global.${wavres}.f${fnh}.grib2 oufile1=${DATA}/gefswave.${cycle}.global.${wavres}.f${fnh}.grib2 rm -f "${oufile1}" #remove outfile if it already exists before extraction From a140ef67b5a85414a5d88599fb5e21b7303cfd5a Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Mon, 8 Jul 2024 20:26:22 +0000 Subject: [PATCH 064/141] Revert changes related to offsets Changes that are related to offsets have been removed. --- parm/ufs/fv3/diag_table_replay | 337 --------------------------------- ush/forecast_postdet.sh | 22 +-- ush/forecast_predet.sh | 24 +-- ush/parsing_namelists_FV3.sh | 16 -- 4 files changed, 11 insertions(+), 388 deletions(-) delete mode 100644 parm/ufs/fv3/diag_table_replay diff --git a/parm/ufs/fv3/diag_table_replay b/parm/ufs/fv3/diag_table_replay deleted file mode 100644 index 01f2cf9794..0000000000 --- a/parm/ufs/fv3/diag_table_replay +++ /dev/null @@ -1,337 +0,0 @@ -"fv3_history", 0, "hours", 1, "hours", "time" -"fv3_history2d", 0, "hours", 1, "hours", "time" -"@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", @[FHOUT_OCN], "hours", 1, "hours", "time", @[FHOUT_OCN], "hours", "@[SYEAR] @[SMONTH] @[SDAY] @[CHOUR_offset] 0 0", @[FHOUT_OCN], "hours" -"@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", @[FHOUT_OCN], "hours", 1, "hours", "time", @[FHOUT_OCN], "hours", "@[SYEAR1] @[SMONTH1] @[SDAY1] @[CHOUR1] 0 0" - -############## -# Ocean fields first lead time -############## -# static fields -"ocean_model", "geolon", "geolon", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 -"ocean_model", "geolat", "geolat", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 -"ocean_model", "geolon_c", "geolon_c", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 -"ocean_model", "geolat_c", "geolat_c", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 -"ocean_model", "geolon_u", "geolon_u", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 -"ocean_model", "geolat_u", "geolat_u", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 -"ocean_model", "geolon_v", "geolon_v", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 -"ocean_model", "geolat_v", "geolat_v", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 -#"ocean_model", "depth_ocean", "depth_ocean", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 -#"ocean_model", "wet", "wet", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 -"ocean_model", "wet_c", "wet_c", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 -"ocean_model", "wet_u", "wet_u", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 -"ocean_model", "wet_v", "wet_v", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 -"ocean_model", "sin_rot", "sin_rot", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 -"ocean_model", "cos_rot", "cos_rot", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 - -# ocean output TSUV and others -"ocean_model", "SSH", "SSH", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 -"ocean_model", "SST", "SST", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 -"ocean_model", "SSS", "SSS", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 -"ocean_model", "speed", "speed", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 -"ocean_model", "SSU", "SSU", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 -"ocean_model", "SSV", "SSV", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 -"ocean_model", "frazil", "frazil", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 -"ocean_model", "ePBL_h_ML", "ePBL", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 -"ocean_model", "MLD_003", "MLD_003", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 -"ocean_model", "MLD_0125", "MLD_0125", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 -"ocean_model", "tob", "tob", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 - -# Z-Space Fields Provided for CMIP6 (CMOR Names): -"ocean_model_z", "uo", "uo", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 -"ocean_model_z", "vo", "vo", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 -"ocean_model_z", "so", "so", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 -"ocean_model_z", "temp", "temp", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 - -# forcing -"ocean_model", "taux", "taux", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 -"ocean_model", "tauy", "tauy", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 -"ocean_model", "latent", "latent", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 -"ocean_model", "sensible", "sensible", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 -"ocean_model", "SW", "SW", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 -"ocean_model", "LW", "LW", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 -"ocean_model", "evap", "evap", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 -"ocean_model", "lprec", "lprec", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 -"ocean_model", "lrunoff", "lrunoff", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 -#"ocean_model", "frunoff", "frunoff", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 -"ocean_model", "fprec", "fprec", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 -"ocean_model", "LwLatSens", "LwLatSens", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 -"ocean_model", "Heat_PmE", "Heat_PmE", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 - -############## -# Ocean fields second lead time and after -############# -# static fields -ocean_model, "geolon", "geolon", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 -ocean_model, "geolat", "geolat", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 -ocean_model, "geolon_c", "geolon_c", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 -ocean_model, "geolat_c", "geolat_c", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 -ocean_model, "geolon_u", "geolon_u", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 -ocean_model, "geolat_u", "geolat_u", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 -ocean_model, "geolon_v", "geolon_v", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 -ocean_model, "geolat_v", "geolat_v", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 -#"ocean_model", "depth_ocean", "depth_ocean", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 -#"ocean_model", "wet", "wet", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 -ocean_model, "wet_c", "wet_c", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 -ocean_model, "wet_u", "wet_u", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 -ocean_model, "wet_v", "wet_v", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 -ocean_model, "sin_rot", "sin_rot", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 -ocean_model, "cos_rot", "cos_rot", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 - -# ocean output TSUV and others -ocean_model, "SSH", "SSH", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 -ocean_model, "SST", "SST", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 -ocean_model, "SSS", "SSS", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 -ocean_model, "speed", "speed", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 -ocean_model, "SSU", "SSU", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 -ocean_model, "SSV", "SSV", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 -ocean_model, "frazil", "frazil", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 -ocean_model, "ePBL_h_ML", "ePBL", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 -ocean_model, "MLD_003", "MLD_003", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 -ocean_model, "MLD_0125", "MLD_0125", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 -ocean_model, "tob", "tob", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 - -# Z-Space Fields Provided for CMIP6 (CMOR Names): -ocean_model_z, "uo", "uo", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 -ocean_model_z, "vo", "vo", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 -ocean_model_z, "so", "so", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 -ocean_model_z, "temp", "temp", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 - -# forcing -ocean_model, "taux", "taux", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 -ocean_model, "tauy", "tauy", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 -ocean_model, "latent", "latent", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 -ocean_model, "sensible", "sensible", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 -ocean_model, "SW", "SW", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 -ocean_model, "LW", "LW", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 -ocean_model, "evap", "evap", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 -ocean_model, "lprec", "lprec", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 -ocean_model, "lrunoff", "lrunoff", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 -#"ocean_model", "frunoff", "frunoff", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 -ocean_model, "fprec", "fprec", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 -ocean_model, "LwLatSens", "LwLatSens", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 -ocean_model, "Heat_PmE", "Heat_PmE", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 - -################### -# Atmosphere fields -################### -"gfs_dyn", "ucomp", "ugrd", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "vcomp", "vgrd", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "sphum", "spfh", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "temp", "tmp", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "liq_wat", "clwmr", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "ice_wat", "icmr", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "snowwat", "snmr", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "rainwat", "rwmr", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "graupel", "grle", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "ice_nc", "nccice", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "rain_nc", "nconrd", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "o3mr", "o3mr", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "cld_amt", "cld_amt", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "delp", "dpres", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "delz", "delz", "fv3_history", "all", .false., "none", 2 -#"gfs_dyn", "pfhy", "preshy", "fv3_history", "all", .false., "none", 2 -#"gfs_dyn", "pfnh", "presnh", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "w", "dzdt", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "ps", "pressfc", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "hs", "hgtsfc", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "refl_10cm", "refl_10cm", "fv3_history", "all", .false., "none", 2 - -"gfs_phys", "cldfra", "cldfra", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "frzr", "frzr", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "frzrb", "frzrb", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "frozr", "frozr", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "frozrb", "frozrb", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "tsnowp", "tsnowp", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "tsnowpb", "tsnowpb", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "rhonewsn", "rhonewsn", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "ALBDO_ave", "albdo_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "cnvprcp_ave", "cprat_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "cnvprcpb_ave", "cpratb_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "totprcp_ave", "prate_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "totprcpb_ave", "prateb_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "DLWRF", "dlwrf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "DLWRFI", "dlwrf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "ULWRF", "ulwrf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "ULWRFI", "ulwrf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "DSWRF", "dswrf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "DSWRFI", "dswrf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "USWRF", "uswrf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "USWRFI", "uswrf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "DSWRFtoa", "dswrf_avetoa", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "USWRFtoa", "uswrf_avetoa", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "ULWRFtoa", "ulwrf_avetoa", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "gflux_ave", "gflux_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "hpbl", "hpbl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "lhtfl_ave", "lhtfl_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "shtfl_ave", "shtfl_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "pwat", "pwat", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "soilm", "soilm", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TCDC_aveclm", "tcdc_aveclm", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TCDC_avebndcl", "tcdc_avebndcl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TCDC_avelcl", "tcdc_avelcl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TCDC_avemcl", "tcdc_avemcl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TCDC_avehcl", "tcdc_avehcl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TCDCcnvcl", "tcdccnvcl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PRES_avelct", "pres_avelct", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PRES_avelcb", "pres_avelcb", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PRES_avemct", "pres_avemct", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PRES_avemcb", "pres_avemcb", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PRES_avehct", "pres_avehct", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PRES_avehcb", "pres_avehcb", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PREScnvclt", "prescnvclt", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PREScnvclb", "prescnvclb", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TEMP_avehct", "tmp_avehct", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TEMP_avemct", "tmp_avemct", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TEMP_avelct", "tmp_avelct", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "u-gwd_ave", "u-gwd_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "v-gwd_ave", "v-gwd_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "dusfc", "uflx_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "dvsfc", "vflx_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "acond", "acond", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "cduvb_ave", "cduvb_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "cpofp", "cpofp", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "duvb_ave", "duvb_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "csdlf_ave", "csdlf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "csusf_ave", "csusf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "csusf_avetoa", "csusftoa", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "csdsf_ave", "csdsf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "csulf_ave", "csulf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "csulf_avetoa", "csulftoa", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "cwork_ave", "cwork_aveclm", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "evbs_ave", "evbs_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "evcw_ave", "evcw_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "fldcp", "fldcp", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "hgt_hyblev1", "hgt_hyblev1", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "spfh_hyblev1", "spfh_hyblev1", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "ugrd_hyblev1", "ugrd_hyblev1", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "vgrd_hyblev1", "vgrd_hyblev1", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "tmp_hyblev1", "tmp_hyblev1", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "gfluxi", "gflux", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "lhtfl", "lhtfl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "shtfl", "shtfl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "pevpr", "pevpr", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "pevpr_ave", "pevpr_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "sbsno_ave", "sbsno_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "sfexc", "sfexc", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "snohf", "snohf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "snowc_ave", "snowc_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "spfhmax2m", "spfhmax_max2m", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "spfhmin2m", "spfhmin_min2m", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "tmpmax2m", "tmax_max2m", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "tmpmin2m", "tmin_min2m", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "ssrun_acc", "ssrun_acc", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "sunsd_acc", "sunsd_acc", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "watr_acc", "watr_acc", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "wilt", "wilt", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "vbdsf_ave", "vbdsf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "vddsf_ave", "vddsf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "nbdsf_ave", "nbdsf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "nddsf_ave", "nddsf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "trans_ave", "trans_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "psurf", "pressfc", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "u10m", "ugrd10m", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "v10m", "vgrd10m", "fv3_history2d", "all", .false., "none", 2 - -"gfs_phys", "pahi", "pahi", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "pah_ave", "pah_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "ecan_acc", "ecan_acc", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "etran_acc", "etran_acc", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "edir_acc", "edir_acc", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "wa_acc", "wa_acc", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "lfrac", "lfrac", "fv3_history2d", "all", .false., "none", 2 - -"gfs_sfc", "crain", "crain", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "tprcp", "tprcp", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "rainc", "cnvprcp", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "hgtsfc", "orog", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "weasd", "weasd", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "f10m", "f10m", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "q2m", "spfh2m", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "t2m", "tmp2m", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "tsfc", "tmpsfc", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "vtype", "vtype", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "stype", "sotyp", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slmsksfc", "land", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "vfracsfc", "veg", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "zorlsfc", "sfcr", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "uustar", "fricv", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilt1", "soilt1" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilt2", "soilt2" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilt3", "soilt3" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilt4", "soilt4" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilw1", "soilw1" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilw2", "soilw2" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilw3", "soilw3" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilw4", "soilw4" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slc_1", "soill1", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slc_2", "soill2", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slc_3", "soill3", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slc_4", "soill4", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slope", "sltyp", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "alnsf", "alnsf", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "alnwf", "alnwf", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "alvsf", "alvsf", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "alvwf", "alvwf", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "canopy", "cnwat", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "facsf", "facsf", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "facwf", "facwf", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "ffhh", "ffhh", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "ffmm", "ffmm", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "fice", "icec", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "hice", "icetk", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "snoalb", "snoalb", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "shdmax", "shdmax", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "shdmin", "shdmin", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "snowd", "snod", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "tg3", "tg3", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "tisfc", "tisfc", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "tref", "tref", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "z_c", "zc", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "c_0", "c0", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "c_d", "cd", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "w_0", "w0", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "w_d", "wd", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xt", "xt", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xz", "xz", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "dt_cool", "dtcool", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xs", "xs", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xu", "xu", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xv", "xv", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xtts", "xtts", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xzts", "xzts", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "d_conv", "dconv", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "qrain", "qrain", "fv3_history2d", "all", .false., "none", 2 - -#============================================================================================= -# -#====> This file can be used with diag_manager/v2.0a (or higher) <==== -# -# -# FORMATS FOR FILE ENTRIES (not all input values are used) -# ------------------------ -# -#"file_name", output_freq, "output_units", format, "time_units", "long_name", -# -# -#output_freq: > 0 output frequency in "output_units" -# = 0 output frequency every time step -# =-1 output frequency at end of run -# -#output_units = units used for output frequency -# (years, months, days, minutes, hours, seconds) -# -#time_units = units used to label the time axis -# (days, minutes, hours, seconds) -# -# -# FORMAT FOR FIELD ENTRIES (not all input values are used) -# ------------------------ -# -#"module_name", "field_name", "output_name", "file_name" "time_sampling", time_avg, "other_opts", packing -# -#time_avg = .true. or .false. -# -#packing = 1 double precision -# = 2 float -# = 4 packed 16-bit integers -# = 8 packed 1-byte (not tested?) diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index a41f4a3d72..993331d70b 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -204,10 +204,8 @@ EOF if [[ "${QUILTING}" = ".true." ]] && [[ "${OUTPUT_GRID}" = "gaussian_grid" ]]; then local FH2 FH3 for fhr in ${FV3_OUTPUT_FH}; do - fhrf=$(echo "${fhr}" | bc -l) - fhri=$(printf %.0f "${fhrf}") - FH3=$(printf %03i "${fhri}") - FH2=$(printf %02i "${fhri}") + FH3=$(printf %03i "${fhr}") + FH2=$(printf %02i "${fhr}") ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.atmf${FH3}.nc" "atmf${FH3}.nc" ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.sfcf${FH3}.nc" "sfcf${FH3}.nc" ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.atm.logf${FH3}.txt" "log.atm.f${FH3}" @@ -453,21 +451,11 @@ MOM6_postdet() { (( midpoint = last_fhr + interval/2 )) vdate=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${fhr} hours" +%Y%m%d%H) - #If OFFSET_START_HOUR is greater than 0, OFFSET_START_HOUR should be added to the midpoint for first lead time - if (( OFFSET_START_HOUR > 0 )) && (( fhr == FHOUT_OCN ));then - vdate_mid=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + $(( midpoint + OFFSET_START_HOUR )) hours" +%Y%m%d%H) - else - vdate_mid=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${midpoint} hours" +%Y%m%d%H) - fi + vdate_mid=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${midpoint} hours" +%Y%m%d%H) # Native model output uses window midpoint in the filename, but we are mapping that to the end of the period for COM - if (( OFFSET_START_HOUR > 0 )) && (( fhr == FHOUT_OCN ));then - source_file="ocn_lead1_${vdate_mid:0:4}_${vdate_mid:4:2}_${vdate_mid:6:2}_${vdate_mid:8:2}.nc" - dest_file="${RUN}.ocean.t${cyc}z.${interval}hr_avg.f${fhr3}.nc" - else - source_file="ocn_${vdate_mid:0:4}_${vdate_mid:4:2}_${vdate_mid:6:2}_${vdate_mid:8:2}.nc" - dest_file="${RUN}.ocean.t${cyc}z.${interval}hr_avg.f${fhr3}.nc" - fi + source_file="ocn_${vdate_mid:0:4}_${vdate_mid:4:2}_${vdate_mid:6:2}_${vdate_mid:8:2}.nc" + dest_file="${RUN}.ocean.t${cyc}z.${interval}hr_avg.f${fhr3}.nc" ${NLN} "${COMOUT_OCEAN_HISTORY}/${dest_file}" "${DATA}/MOM6_OUTPUT/${source_file}" last_fhr=${fhr} diff --git a/ush/forecast_predet.sh b/ush/forecast_predet.sh index 9dd78011c2..9183e86002 100755 --- a/ush/forecast_predet.sh +++ b/ush/forecast_predet.sh @@ -145,25 +145,13 @@ FV3_predet(){ fi # Convert output settings into an explicit list for FV3 - if (( OFFSET_START_HOUR > 0 ));then - FV3_OUTPUT_FH="$(echo "scale=5; ${OFFSET_START_HOUR}+(${DELTIM}/3600)" | bc -l)" - local FHMIN_REPLAY=$(( FHMIN + FHOUT )) - local fhr=${FHMIN_REPLAY} - if (( FHOUT_HF > 0 && FHMAX_HF > 0 )); then - FV3_OUTPUT_FH="${FV3_OUTPUT_FH} $(seq -s ' ' "${FHMIN_REPLAY}" "${FHOUT_HF}" "${FHMAX_HF}")" - fhr=${FHMAX_HF} - fi - FV3_OUTPUT_FH="${FV3_OUTPUT_FH} $(seq -s ' ' "${fhr}" "${FHOUT}" "${FHMAX}")" - else - FV3_OUTPUT_FH="" - local fhr=${FHMIN} - if (( FHOUT_HF > 0 && FHMAX_HF > 0 )); then - FV3_OUTPUT_FH="${FV3_OUTPUT_FH} $(seq -s ' ' "${FHMIN}" "${FHOUT_HF}" "${FHMAX_HF}")" - fhr=${FHMAX_HF} - fi - FV3_OUTPUT_FH="${FV3_OUTPUT_FH} $(seq -s ' ' "${fhr}" "${FHOUT}" "${FHMAX}")" + FV3_OUTPUT_FH="" + local fhr=${FHMIN} + if (( FHOUT_HF > 0 && FHMAX_HF > 0 )); then + FV3_OUTPUT_FH="${FV3_OUTPUT_FH} $(seq -s ' ' "${FHMIN}" "${FHOUT_HF}" "${FHMAX_HF}")" + fhr=${FHMAX_HF} fi - + FV3_OUTPUT_FH="${FV3_OUTPUT_FH} $(seq -s ' ' "${fhr}" "${FHOUT}" "${FHMAX}")" # Other options MEMBER=$(( 10#${ENSMEM:-"-1"} )) # -1: control, 0: ensemble mean, >0: ensemble member $MEMBER diff --git a/ush/parsing_namelists_FV3.sh b/ush/parsing_namelists_FV3.sh index 4cec6c05aa..b88849d7e7 100755 --- a/ush/parsing_namelists_FV3.sh +++ b/ush/parsing_namelists_FV3.sh @@ -17,10 +17,6 @@ if (( gwd_opt == 2 )) && [[ ${do_gsl_drag_ls_bl} == ".true." ]]; then cdmbgwd=${cdmbgwd_gsl} fi -if (( OFFSET_START_HOUR != 0 )); then - DIAG_TABLE="${PARMgfs}/ufs/fv3/diag_table_replay" -fi - # ensure non-prognostic tracers are set dnats=${dnats:-0} @@ -46,18 +42,6 @@ local SDAY=${current_cycle:6:2} local CHOUR=${current_cycle:8:2} local MOM6_OUTPUT_DIR="./MOM6_OUTPUT" -if (( OFFSET_START_HOUR > 0 )); then - local current_cycle_p1 - current_cycle_p1=$(${NDATE} "${FHOUT_OCN}" "${current_cycle}") - local current_cycle_offset - current_cycle_offset=$(${NDATE} "${OFFSET_START_HOUR}" "${current_cycle}") - local SYEAR1=${current_cycle_p1:0:4} - local SMONTH1=${current_cycle_p1:4:2} - local SDAY1=${current_cycle_p1:6:2} - local CHOUR1=${current_cycle_p1:8:2} - local CHOUR_offset=${current_cycle_offset:8:2} -fi - atparse < "${template}" >> "diag_table" From c7f8364dc031da1e4f478deb87f2d4c9a6b17dc8 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 9 Jul 2024 12:25:12 +0000 Subject: [PATCH 065/141] Rename JGLOBAL_EXTRACTVARS JGLOBAL_EXTRACTVARS has been renamed to JGEFS_EXTRACTVARS. --- jobs/{JGLOBAL_EXTRACTVARS => JGEFS_EXTRACTVARS} | 0 jobs/rocoto/extractvars.sh | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename jobs/{JGLOBAL_EXTRACTVARS => JGEFS_EXTRACTVARS} (100%) diff --git a/jobs/JGLOBAL_EXTRACTVARS b/jobs/JGEFS_EXTRACTVARS similarity index 100% rename from jobs/JGLOBAL_EXTRACTVARS rename to jobs/JGEFS_EXTRACTVARS diff --git a/jobs/rocoto/extractvars.sh b/jobs/rocoto/extractvars.sh index a872431358..bd274f0559 100755 --- a/jobs/rocoto/extractvars.sh +++ b/jobs/rocoto/extractvars.sh @@ -16,7 +16,7 @@ export jobid="${job}.$$" echo echo "=============== START TO RUN EXTRACTVARS ===============" # Execute the JJOB -"${HOMEgfs}/jobs/JGLOBAL_EXTRACTVARS" +"${HOMEgfs}/jobs/JGEFS_EXTRACTVARS" status=$? [[ "${status}" -ne 0 ]] && exit "${status}" From 32c25a6159615193558943898f5e16bb13c41e90 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 9 Jul 2024 12:30:29 +0000 Subject: [PATCH 066/141] Add comment to explain DO_EXTRACTVARS A brief description has been added for DO_EXTRACTVARS. --- parm/config/gefs/config.base | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index 28351aa97a..7717a4aa80 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -137,7 +137,7 @@ export DO_WAVE="NO" export DO_OCN="NO" export DO_ICE="NO" export DO_AERO="NO" -export DO_EXTRACTVARS="@DO_EXTRACTVARS@" +export DO_EXTRACTVARS="@DO_EXTRACTVARS@" #Option to process and extract a subset of products to save on disk export AERO_FCST_CDUMP="" # When to run aerosol forecast: gdas, gfs, or both export AERO_ANL_CDUMP="" # When to run aerosol analysis: gdas, gfs, or both export WAVE_CDUMP="" # When to include wave suite: gdas, gfs, or both From d349d6687953c55e098281ee3c9b3d1d2e6bb586 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 9 Jul 2024 13:46:01 +0000 Subject: [PATCH 067/141] Polish some if-statements in extractvars Some if-statements have been refined in extractvars. --- scripts/exglobal_extractvars.sh | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/scripts/exglobal_extractvars.sh b/scripts/exglobal_extractvars.sh index 373901a7b3..462f9da2d8 100755 --- a/scripts/exglobal_extractvars.sh +++ b/scripts/exglobal_extractvars.sh @@ -38,8 +38,14 @@ if [[ "${DO_OCN}" == "YES" ]];then if [[ ! -d "${DATA}/mem${ENSMEM}_ocn" ]]; then mkdir -p "${DATA}/mem${ENSMEM}_ocn" fi - if [[ "${ocn_dataformat}" == "netcdf" ]]; then varlist_ocn=${varlist_ocn_netcdf}; fi - if [[ "${ocn_dataformat}" == "grib2" ]]; then varlist_ocn=${varlist_ocn_grib2}; fi + if [[ "${ocn_dataformat}" == "netcdf" ]]; then + varlist_ocn=${varlist_ocn_netcdf} + elif [[ "${ocn_dataformat}" == "grib2" ]]; then + varlist_ocn=${varlist_ocn_grib2} + else + echo "FATAL ERROR: Invalid ocean data format provided (${ocn_dataformat})" + export err=1; err_chk + fi ${EXTRCTVARO} "${DATA}/mem${ENSMEM}_ocn" "${varlist_ocn}" "${ocn_dataformat}" "${ocnres}" "${compress_ocn}" "${FHOUT_OCN_GFS}" cp -pr "${DATA}/mem${ENSMEM}_ocn/." "${COMOUT_RFCST_PROD_OCN}" fi @@ -49,9 +55,15 @@ if [[ "${DO_ICE}" == "YES" ]];then export component_name="ice" if [[ ! -d "${DATA}/mem${ENSMEM}_ice" ]]; then mkdir -p "${DATA}/mem${ENSMEM}_ice" - fi - if [[ "${ice_dataformat}" == "netcdf" ]]; then varlist_ice=${varlist_ice_netcdf}; fi - if [[ "${ice_dataformat}" == "grib2" ]]; then varlist_ice=${varlist_ice_grib2}; fi + fi + if [[ "${ice_dataformat}" == "netcdf" ]]; then + varlist_ice=${varlist_ice_netcdf} + elif [[ "${ice_dataformat}" == "grib2" ]]; then + varlist_ice=${varlist_ice_grib2} + else + echo "FATAL ERROR: Invalid ice data format provided (${ice_dataformat})" + export err=1; err_chk + fi ${EXTRCTVARO} "${DATA}/mem${ENSMEM}_ice" "${varlist_ice}" "${ice_dataformat}" "${iceres}" "${compress_ice}" "${FHOUT_ICE_GFS}" cp -pr "${DATA}/mem${ENSMEM}_ice/." "${COMOUT_RFCST_PROD_ICE}" fi From 8ef0bd953f5e98f5b5f8f9644a9025d2e7b307e6 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 9 Jul 2024 13:48:08 +0000 Subject: [PATCH 068/141] Revert "Rename JGLOBAL_EXTRACTVARS" This reverts commit c7f8364dc031da1e4f478deb87f2d4c9a6b17dc8. --- jobs/{JGEFS_EXTRACTVARS => JGLOBAL_EXTRACTVARS} | 0 jobs/rocoto/extractvars.sh | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename jobs/{JGEFS_EXTRACTVARS => JGLOBAL_EXTRACTVARS} (100%) diff --git a/jobs/JGEFS_EXTRACTVARS b/jobs/JGLOBAL_EXTRACTVARS similarity index 100% rename from jobs/JGEFS_EXTRACTVARS rename to jobs/JGLOBAL_EXTRACTVARS diff --git a/jobs/rocoto/extractvars.sh b/jobs/rocoto/extractvars.sh index bd274f0559..a872431358 100755 --- a/jobs/rocoto/extractvars.sh +++ b/jobs/rocoto/extractvars.sh @@ -16,7 +16,7 @@ export jobid="${job}.$$" echo echo "=============== START TO RUN EXTRACTVARS ===============" # Execute the JJOB -"${HOMEgfs}/jobs/JGEFS_EXTRACTVARS" +"${HOMEgfs}/jobs/JGLOBAL_EXTRACTVARS" status=$? [[ "${status}" -ne 0 ]] && exit "${status}" From 5310eaa7ac9628aa5dd6bf0790a26858072a0096 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 9 Jul 2024 14:07:35 +0000 Subject: [PATCH 069/141] Simplify com var defs in gefs_atmos_extractvars The defining of the com var varialbes have been refined in gefs_atmos_extractvars --- ush/gefs_atmos_extractvars.sh | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/ush/gefs_atmos_extractvars.sh b/ush/gefs_atmos_extractvars.sh index 3cb6ca870e..2d6cbcbc51 100755 --- a/ush/gefs_atmos_extractvars.sh +++ b/ush/gefs_atmos_extractvars.sh @@ -42,16 +42,9 @@ for outtype in "f2d" "f3d"; do outfreq=${FHOUTLF} fi - if [[ "${outres}" == "0p25" ]];then - infile1=${COMIN_ATMOS_GRIB_0p25}/gefs.${cycle}.pgrb2.${outres}.f${fnh} - infile2=${COMIN_ATMOS_GRIB_0p25}/gefs.${cycle}.pgrb2b.${outres}.f${fnh} - elif [[ "${outres}" == "0p50" ]];then - infile1=${COMIN_ATMOS_GRIB_0p50}/gefs.${cycle}.pgrb2.${outres}.f${fnh} - infile2=${COMIN_ATMOS_GRIB_0p50}/gefs.${cycle}.pgrb2b.${outres}.f${fnh} - elif [[ "${outres}" == "1p00" ]];then - infile1=${COMIN_ATMOS_GRIB_1p00}/gefs.${cycle}.pgrb2.${outres}.f${fnh} - infile2=${COMIN_ATMOS_GRIB_1p00}/gefs.${cycle}.pgrb2b.${outres}.f${fnh} - fi + com_var="COMIN_ATMOS_GRIB_${outres}" + infile1="${!com_var}/gefs.${cycle}.pgrb2.${outres}.f${fnh}" + infile2="${!com_var}/gefs.${cycle}.pgrb2b.${outres}.f${fnh}" oufile=${outdirpre}/gefs.${cycle}.pgrb2.${outres}.f${fnh} rm -f "${oufile}" #remove outfile if it already exists before extraction requestedvars1="partial_parm1.txt" From 1d570ea22b3296bcb45fe3b8b00294c17ac8aa33 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 9 Jul 2024 14:10:37 +0000 Subject: [PATCH 070/141] Rename oufile in gefs_atmos_extractvars The oufile variable has been renamed to outfile in gefs_atmos_extractvars. --- ush/gefs_atmos_extractvars.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ush/gefs_atmos_extractvars.sh b/ush/gefs_atmos_extractvars.sh index 2d6cbcbc51..cdf7c6c84d 100755 --- a/ush/gefs_atmos_extractvars.sh +++ b/ush/gefs_atmos_extractvars.sh @@ -45,8 +45,8 @@ for outtype in "f2d" "f3d"; do com_var="COMIN_ATMOS_GRIB_${outres}" infile1="${!com_var}/gefs.${cycle}.pgrb2.${outres}.f${fnh}" infile2="${!com_var}/gefs.${cycle}.pgrb2b.${outres}.f${fnh}" - oufile=${outdirpre}/gefs.${cycle}.pgrb2.${outres}.f${fnh} - rm -f "${oufile}" #remove outfile if it already exists before extraction + outfile=${outdirpre}/gefs.${cycle}.pgrb2.${outres}.f${fnh} + rm -f "${outfile}" #remove outfile if it already exists before extraction requestedvars1="partial_parm1.txt" requestedvars2="partial_parm2.txt" rm -f "${requestedvars1}" @@ -54,7 +54,7 @@ for outtype in "f2d" "f3d"; do if [[ -f "${infile1}" ]]; then #check if input file exists before extraction gen_parmlist "${infile1}" "${requestedvars1}" "${varlist}" # shellcheck disable=SC2312 - ${WGRIB2} "${infile1}" | grep -F -f "${requestedvars1}" | ${WGRIB2} -i "${infile1}" -append -grib "${oufile}">/dev/null + ${WGRIB2} "${infile1}" | grep -F -f "${requestedvars1}" | ${WGRIB2} -i "${infile1}" -append -grib "${outfile}">/dev/null else echo "WARNING: ${infile1} does not exist." fi @@ -62,7 +62,7 @@ for outtype in "f2d" "f3d"; do if [[ -f "${infile2}" ]]; then #check if input file exists before extraction gen_parmlist "${infile2}" "${requestedvars2}" "${varlist}" # shellcheck disable=SC2312 - ${WGRIB2} "${infile2}" | grep -F -f "${requestedvars2}" | ${WGRIB2} -i "${infile2}" -append -grib "${oufile}">/dev/null + ${WGRIB2} "${infile2}" | grep -F -f "${requestedvars2}" | ${WGRIB2} -i "${infile2}" -append -grib "${outfile}">/dev/null else echo "WARNING: ${infile2} does not exist." fi From 491be9c406b6cff688427c3d2a2406c4cd8cb74e Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 9 Jul 2024 14:24:46 +0000 Subject: [PATCH 071/141] Add varlist as argument for check_atmos The varlist variable has been added as an argument for the check_atmos function. --- ush/extractvars_tools.sh | 1 + ush/gefs_atmos_extractvars.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ush/extractvars_tools.sh b/ush/extractvars_tools.sh index 41412b2bd2..5692a774a7 100644 --- a/ush/extractvars_tools.sh +++ b/ush/extractvars_tools.sh @@ -16,6 +16,7 @@ gen_parmlist() { check_atmos() { requestedvar_in_file1=$1 requestedvar_in_file2=$2 + varlist=$3 requestedvar_in_allgrb2file=requestedvar_in_allgrb2file.txt rm -rvf "${requestedvar_in_allgrb2file}" cat "${requestedvar_in_file1}" "${requestedvar_in_file2}" >> "${requestedvar_in_allgrb2file}" diff --git a/ush/gefs_atmos_extractvars.sh b/ush/gefs_atmos_extractvars.sh index cdf7c6c84d..8137f1cf62 100755 --- a/ush/gefs_atmos_extractvars.sh +++ b/ush/gefs_atmos_extractvars.sh @@ -69,7 +69,7 @@ for outtype in "f2d" "f3d"; do if [[ ! -f "${requestedvars1}" ]]; then touch "${requestedvars1}"; fi if [[ ! -f "${requestedvars2}" ]]; then touch "${requestedvars2}"; fi - check_atmos "${requestedvars1}" "${requestedvars2}" + check_atmos "${requestedvars1}" "${requestedvars2}" "${varlist}" #Compute daily average for a subset of variables if (( nh % 6 == 0 )) && (( nh != 0 )) && [[ "${outtype}" == "f3d" ]];then From d9a168180b6cbf692caef866663d5202a45ecea3 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 9 Jul 2024 14:57:38 +0000 Subject: [PATCH 072/141] Define a compression utility A variable has been added in config.extractvars that defines a compression utility. --- parm/config/gefs/config.extractvars | 2 ++ ush/gefs_ocnice_extractvars.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/parm/config/gefs/config.extractvars b/parm/config/gefs/config.extractvars index 315244c52f..7b71423c27 100644 --- a/parm/config/gefs/config.extractvars +++ b/parm/config/gefs/config.extractvars @@ -7,6 +7,8 @@ echo "BEGIN: config.extractvars" . "${EXPDIR}/config.resources" extractvars +export COMPRSCMD=${COMPRSCMD:-bzip2} + export compress_ocn=0 #1: compress extracted ocean product, 0: do not compress extracted ocean product export compress_ice=0 #1: compress extracted ice product, 0: do not compress extracted ice product export FHMAX=120 diff --git a/ush/gefs_ocnice_extractvars.sh b/ush/gefs_ocnice_extractvars.sh index 691cd4f447..0ffa15033f 100755 --- a/ush/gefs_ocnice_extractvars.sh +++ b/ush/gefs_ocnice_extractvars.sh @@ -61,7 +61,7 @@ while [[ ${nh} -le ${FHMAX} ]];do fi fi if [[ ${datacompress} -eq 1 ]];then - bzip2 "${oufile1}" + ${COMPRSCMD} "${oufile1}" fi else echo "WARNING: ${infile} does not exist." From a0343a3f31fd6e65a74de66177e0712e506e1a0f Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 9 Jul 2024 15:01:18 +0000 Subject: [PATCH 073/141] Rename oufile1 in gefs_ocnice_extractvars The oufile1 variable has been renamed to outfile in gefs_ocnice_extractvars. --- ush/gefs_ocnice_extractvars.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ush/gefs_ocnice_extractvars.sh b/ush/gefs_ocnice_extractvars.sh index 0ffa15033f..479b55d5ff 100755 --- a/ush/gefs_ocnice_extractvars.sh +++ b/ush/gefs_ocnice_extractvars.sh @@ -23,7 +23,7 @@ while [[ ${nh} -le ${FHMAX} ]];do if [[ ${component_name} == "ice" ]];then infile=${COMIN_ICE_GRIB}/${datares}/gefs.ice.${cycle}.${datares}.f${fnh}.grib2 fi - oufile1=${outdirpre}/gefs.${component_name}.${cycle}.${datares}.f${fnh}.grib2 + outfile=${outdirpre}/gefs.${component_name}.${cycle}.${datares}.f${fnh}.grib2 fi if [[ "${dataformat}" == "netcdf" ]];then @@ -33,13 +33,13 @@ while [[ ${nh} -le ${FHMAX} ]];do if [[ ${component_name} == "ice" ]];then infile=${COMIN_ICE_NETCDF}/gefs.ice.${cycle}.${datares}.f${fnh}.nc fi - oufile1=${outdirpre}/gefs.${component_name}.${cycle}.${datares}.f${fnh}.nc + outfile=${outdirpre}/gefs.${component_name}.${cycle}.${datares}.f${fnh}.nc fi if [[ -f "${infile}" ]]; then #check if input file exists before extraction if [[ "${dataformat}" == "grib2" ]];then # shellcheck disable=SC2312 - ${WGRIB2} "${infile}" | grep -F -f "${varlist}" | ${WGRIB2} -i "${infile}" -append -grib "${oufile1}">/dev/null + ${WGRIB2} "${infile}" | grep -F -f "${varlist}" | ${WGRIB2} -i "${infile}" -append -grib "${outfile}">/dev/null fi if [[ "${dataformat}" == "netcdf" ]];then varsrequested=$(paste -s "${varlist}") @@ -57,11 +57,11 @@ while [[ ${nh} -le ${FHMAX} ]];do echo "WARNING: No variables from parm file ${varlist} are available in netcdf file ${infile}." else ocnice_vars=${varsavailable::-1} - ncks -v "${ocnice_vars}" "${infile}" "${oufile1}" + ncks -v "${ocnice_vars}" "${infile}" "${outfile}" fi fi if [[ ${datacompress} -eq 1 ]];then - ${COMPRSCMD} "${oufile1}" + ${COMPRSCMD} "${outfile}" fi else echo "WARNING: ${infile} does not exist." From ac855af0ad34a778c7e0bf4464bf43cb57399d98 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 9 Jul 2024 15:05:02 +0000 Subject: [PATCH 074/141] Rename oufile1 in gefs_wav_extractvars The oufile1 variable in gefs_wav_extractvars has been renamed to outfile. --- ush/gefs_wav_extractvars.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ush/gefs_wav_extractvars.sh b/ush/gefs_wav_extractvars.sh index e20785fe67..e2746c5d89 100755 --- a/ush/gefs_wav_extractvars.sh +++ b/ush/gefs_wav_extractvars.sh @@ -12,12 +12,12 @@ while [[ ${nh} -le ${FHMAX_WAV} ]];do fnh=$(printf "%3.3d" "${nh}") infile=${COMIN_WAVE_GRID}/gefswave.${cycle}.global.${wavres}.f${fnh}.grib2 - oufile1=${DATA}/gefswave.${cycle}.global.${wavres}.f${fnh}.grib2 - rm -f "${oufile1}" #remove outfile if it already exists before extraction - + outfile=${DATA}/gefswave.${cycle}.global.${wavres}.f${fnh}.grib2 + rm -f "${outfile}" #remove outfile if it already exists before extraction + if [[ -f "${infile}" ]]; then #check if input file exists before extraction # shellcheck disable=SC2312 - ${WGRIB2} "${infile}" | grep -F -f "${varlist_wav}" | ${WGRIB2} -i "${infile}" -append -grib "${oufile1}">/dev/null + ${WGRIB2} "${infile}" | grep -F -f "${varlist_wav}" | ${WGRIB2} -i "${infile}" -append -grib "${outfile}">/dev/null else echo "WARNING: ${infile} does not exist." fi From f768627c59d0340148b33b7ce8d9d4c8d332b97f Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 9 Jul 2024 15:27:44 +0000 Subject: [PATCH 075/141] Remove mode arugment from mkdir The mode argument has been removed from mkdir in JGLOBAL_EXTRACTVARS. --- jobs/JGLOBAL_EXTRACTVARS | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/jobs/JGLOBAL_EXTRACTVARS b/jobs/JGLOBAL_EXTRACTVARS index 795f673a5c..f66f733195 100755 --- a/jobs/JGLOBAL_EXTRACTVARS +++ b/jobs/JGLOBAL_EXTRACTVARS @@ -7,7 +7,7 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "extractvars" -c "base extractvars" for grid in '0p25' '0p50' '1p00'; do prod_dir="COMIN_ATMOS_GRIB_${grid}" GRID=${grid} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx "${prod_dir}:COM_ATMOS_GRIB_GRID_TMPL" - if [[ ! -d "${!prod_dir}" ]]; then mkdir -m 775 -p "${!prod_dir}"; fi + if [[ ! -d "${!prod_dir}" ]]; then mkdir -p "${!prod_dir}"; fi done YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COMIN_OCEAN_HISTORY:COM_OCEAN_HISTORY_TMPL" @@ -24,10 +24,10 @@ YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COMOUT_RFCST_PROD_OCN:COM_RFCST YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COMOUT_RFCST_PROD_ICE:COM_RFCST_PROD_ICE_TMPL" YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COMOUT_RFCST_PROD_WAV:COM_RFCST_PROD_WAV_TMPL" -if [[ ! -d "${COMOUT_RFCST_PROD_ATMOS}" ]]; then mkdir -m 775 -p "${COMOUT_RFCST_PROD_ATMOS}"; fi -if [[ ! -d "${COMOUT_RFCST_PROD_OCN}" ]]; then mkdir -m 775 -p "${COMOUT_RFCST_PROD_OCN}"; fi -if [[ ! -d "${COMOUT_RFCST_PROD_WAV}" ]]; then mkdir -m 775 -p "${COMOUT_RFCST_PROD_WAV}"; fi -if [[ ! -d "${COMOUT_RFCST_PROD_ICE}" ]]; then mkdir -m 775 -p "${COMOUT_RFCST_PROD_ICE}"; fi +if [[ ! -d "${COMOUT_RFCST_PROD_ATMOS}" ]]; then mkdir -p "${COMOUT_RFCST_PROD_ATMOS}"; fi +if [[ ! -d "${COMOUT_RFCST_PROD_OCN}" ]]; then mkdir -p "${COMOUT_RFCST_PROD_OCN}"; fi +if [[ ! -d "${COMOUT_RFCST_PROD_WAV}" ]]; then mkdir -p "${COMOUT_RFCST_PROD_WAV}"; fi +if [[ ! -d "${COMOUT_RFCST_PROD_ICE}" ]]; then mkdir -p "${COMOUT_RFCST_PROD_ICE}"; fi # Execute the Script "${SCRgfs}/exglobal_extractvars.sh" From f35452f2bdcf04a074359b4b09f7d052786ad3e1 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 9 Jul 2024 15:47:26 +0000 Subject: [PATCH 076/141] Remove FHMAX and use FHMAX_GFS The FHMAX variable has been removed from config.extractvars. FHMAX_GFS from config.base is used in the extractvars task instead of FHMAX. --- parm/config/gefs/config.extractvars | 1 - ush/gefs_atmos_extractvars.sh | 2 +- ush/gefs_ocnice_extractvars.sh | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/parm/config/gefs/config.extractvars b/parm/config/gefs/config.extractvars index 7b71423c27..cecfe72d11 100644 --- a/parm/config/gefs/config.extractvars +++ b/parm/config/gefs/config.extractvars @@ -11,7 +11,6 @@ export COMPRSCMD=${COMPRSCMD:-bzip2} export compress_ocn=0 #1: compress extracted ocean product, 0: do not compress extracted ocean product export compress_ice=0 #1: compress extracted ice product, 0: do not compress extracted ice product -export FHMAX=120 export FHMAXHF=60 #The forecast length of the high output frequency (in hours) for atmos export FHOUTHF=6 #The high output frequency (in hours) for atmos diff --git a/ush/gefs_atmos_extractvars.sh b/ush/gefs_atmos_extractvars.sh index 8137f1cf62..94103e30ff 100755 --- a/ush/gefs_atmos_extractvars.sh +++ b/ush/gefs_atmos_extractvars.sh @@ -23,7 +23,7 @@ for outtype in "f2d" "f3d"; do if [[ ! -d "${outdirpre}" ]]; then mkdir -p "${outdirpre}"; fi nh=${FHMIN} - while [[ ${nh} -le ${FHMAX} ]];do + while [[ ${nh} -le ${FHMAX_GFS} ]];do fnh=$(printf "%3.3d" "${nh}") if [[ "${outtype}" == "f2d" ]];then diff --git a/ush/gefs_ocnice_extractvars.sh b/ush/gefs_ocnice_extractvars.sh index 479b55d5ff..16e8038de8 100755 --- a/ush/gefs_ocnice_extractvars.sh +++ b/ush/gefs_ocnice_extractvars.sh @@ -13,7 +13,7 @@ fhout_ocnice=${6} cd "${outdirpre}" || true nh=${FHMIN_GFS} -while [[ ${nh} -le ${FHMAX} ]];do +while [[ ${nh} -le ${FHMAX_GFS} ]];do fnh=$(printf "%3.3d" "${nh}") if [[ "${dataformat}" == "grib2" ]];then From de3e5e6faca9f9d10e56725364e4a3b489f38b80 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 9 Jul 2024 15:49:39 +0000 Subject: [PATCH 077/141] Remove duplicate variable in config.extractvars A duplicate variable ocnres has been removed from config.extractvars. --- parm/config/gefs/config.extractvars | 1 - 1 file changed, 1 deletion(-) diff --git a/parm/config/gefs/config.extractvars b/parm/config/gefs/config.extractvars index cecfe72d11..a9f0f2b120 100644 --- a/parm/config/gefs/config.extractvars +++ b/parm/config/gefs/config.extractvars @@ -21,7 +21,6 @@ export ice_dataformat="netcdf" export ocnres="5p00" #Resolution of ocean products export iceres="5p00" #Resolution of ice products -export ocnres="5p00" #Resolution of ocean products export wavres="5p00" #Resolution of wave products export FHOUT_WAV_NOSCRUB=6 #Frequency of wave output to be saved on disk From ef7799bf42dbb4d4587738eb3c67ef686ddb15a5 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 9 Jul 2024 16:20:28 +0000 Subject: [PATCH 078/141] Remove FHMAXHF, FHMAXLF and FHMAXHF FHMAXHF, FHMAXLF and FHMAXHF have been removed from config.extractvars and have been replaced with FHMAX_HF_GFS, FHOUT_GFS and FHMAX_HF_GFS from config.base, respectively. --- parm/config/gefs/config.extractvars | 4 ---- ush/gefs_atmos_extractvars.sh | 8 ++++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/parm/config/gefs/config.extractvars b/parm/config/gefs/config.extractvars index a9f0f2b120..a84a6ac888 100644 --- a/parm/config/gefs/config.extractvars +++ b/parm/config/gefs/config.extractvars @@ -12,10 +12,6 @@ export COMPRSCMD=${COMPRSCMD:-bzip2} export compress_ocn=0 #1: compress extracted ocean product, 0: do not compress extracted ocean product export compress_ice=0 #1: compress extracted ice product, 0: do not compress extracted ice product -export FHMAXHF=60 #The forecast length of the high output frequency (in hours) for atmos -export FHOUTHF=6 #The high output frequency (in hours) for atmos -export FHOUTLF=12 #The low output frequency (in hours) for atmos - export ocn_dataformat="netcdf" export ice_dataformat="netcdf" diff --git a/ush/gefs_atmos_extractvars.sh b/ush/gefs_atmos_extractvars.sh index 94103e30ff..8baccab4b2 100755 --- a/ush/gefs_atmos_extractvars.sh +++ b/ush/gefs_atmos_extractvars.sh @@ -27,7 +27,7 @@ for outtype in "f2d" "f3d"; do fnh=$(printf "%3.3d" "${nh}") if [[ "${outtype}" == "f2d" ]];then - if [[ ${nh} -le ${FHMAXHF} ]];then + if [[ ${nh} -le ${FHMAX_HF_GFS} ]];then outres="0p25" else outres="0p50" @@ -36,10 +36,10 @@ for outtype in "f2d" "f3d"; do outres="1p00" fi - if [[ ${nh} -lt ${FHMAXHF} ]];then - outfreq=${FHOUTHF} + if [[ ${nh} -lt ${FHMAX_HF_GFS} ]];then + outfreq=${FHOUT_HF_GFS} else - outfreq=${FHOUTLF} + outfreq=${FHOUT_GFS} fi com_var="COMIN_ATMOS_GRIB_${outres}" From 2f89276150785e6db3305eb7bd09e814a82d5b8d Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 9 Jul 2024 16:33:25 +0000 Subject: [PATCH 079/141] Check that FHMAX_GFS is greater than FHMAX_HF_GFS An error check has been added to make sure FHMAX_GFS is greater than FHMAX_HF_GFS. A fatal error will occur if this check fails. --- scripts/exglobal_extractvars.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/exglobal_extractvars.sh b/scripts/exglobal_extractvars.sh index 462f9da2d8..dca6214784 100755 --- a/scripts/exglobal_extractvars.sh +++ b/scripts/exglobal_extractvars.sh @@ -22,6 +22,12 @@ export varlist_ice_netcdf=${varlist_ice_netcdf:-"${PARMgfs}/product/gefs_ice_sho export varlist_ocn_grib2=${varlist_ocn_grib2:-"${PARMgfs}/product/gefs_ocn_shortparmlist_grib2.parm"} #Parameter table for wave variables export varlist_ice_grib2=${varlist_ice_grib2:-"${PARMgfs}/product/gefs_ice_shortparmlist_grib2.parm"} #Parameter table for wave variables +#Check to make sure FHMAX_HF_GFS is less than FHMAX_GFS +if [[ "${FHMAX_GFS}" -lt "${FHMAX_HF_GFS}" ]];then + echo "FATAL ERROR: FHMAX_GFS (${FHMAX_GFS}) is less than FHMAX_HF_GFS (${FHMAX_HF_GFS}). FHMAX_GFS must be greater than FHMAX_HF_GFS." + export err=1; err_chk +fi + #Extract variables for atmosphere if [[ "${DO_ATM}" == "YES" ]];then if [[ ! -d "${DATA}/mem${ENSMEM}_atmos" ]]; then From beb9e5e5953c76130ebb9c5c5795e5a085fd1224 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 9 Jul 2024 16:46:07 +0000 Subject: [PATCH 080/141] Moved parmfile variables to config.extractvars The parmfile variables have been moved from exglobal_extractvars to config.extractvars. Some of the parmfile descriptions have also been corrected. --- parm/config/gefs/config.extractvars | 10 ++++++++++ scripts/exglobal_extractvars.sh | 10 ---------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/parm/config/gefs/config.extractvars b/parm/config/gefs/config.extractvars index a84a6ac888..dc3fefee13 100644 --- a/parm/config/gefs/config.extractvars +++ b/parm/config/gefs/config.extractvars @@ -21,4 +21,14 @@ export wavres="5p00" #Resolution of wave products export FHOUT_WAV_NOSCRUB=6 #Frequency of wave output to be saved on disk +#Paramater Tables used +export varlist_2d=${varlist_2d:-"${PARMgfs}/product/gefs_shortparmlist_2d.parm"} #Parameter table for surface variables +export varlist_3d=${varlist_3d:-"${PARMgfs}/product/gefs_shortparmlist_3d_h.parm"} #Parameter table for upper air instantaneous variables +export varlist_3d_d=${varlist_3d_d:-"${PARMgfs}/product/gefs_shortparmlist_3d_d.parm"} #Parameter table for upper air daily-averaged variables +export varlist_wav=${varlist_wav:-"${PARMgfs}/product/gefs_wav_shortparmlist.parm"} #Parameter table for wave variables +export varlist_ocn_netcdf=${varlist_ocn_netcdf:-"${PARMgfs}/product/gefs_ocn_shortparmlist.parm"} #Parameter table for ocean netcdf variables +export varlist_ice_netcdf=${varlist_ice_netcdf:-"${PARMgfs}/product/gefs_ice_shortparmlist.parm"} #Parameter table for ice netcdf variables +export varlist_ocn_grib2=${varlist_ocn_grib2:-"${PARMgfs}/product/gefs_ocn_shortparmlist_grib2.parm"} #Parameter table ocn grib2 variables +export varlist_ice_grib2=${varlist_ice_grib2:-"${PARMgfs}/product/gefs_ice_shortparmlist_grib2.parm"} #Parameter table ice grib2 variables + echo "END: config.extractvars" diff --git a/scripts/exglobal_extractvars.sh b/scripts/exglobal_extractvars.sh index dca6214784..da800985c8 100755 --- a/scripts/exglobal_extractvars.sh +++ b/scripts/exglobal_extractvars.sh @@ -12,16 +12,6 @@ EXTRCTVARA=${EXTRCTVARA:-"${USHgfs}/gefs_atmos_extractvars.sh"} EXTRCTVARO=${EXTRCTVARO:-"${USHgfs}/gefs_ocnice_extractvars.sh"} EXTRCTVARW=${EXTRCTVARW:-"${USHgfs}/gefs_wav_extractvars.sh"} -#Paramater Tables used -export varlist_2d=${varlist_2d:-"${PARMgfs}/product/gefs_shortparmlist_2d.parm"} #Parameter table for surface variables -export varlist_3d=${varlist_3d:-"${PARMgfs}/product/gefs_shortparmlist_3d_h.parm"} #Parameter table for upper air instantaneous variables -export varlist_3d_d=${varlist_3d_d:-"${PARMgfs}/product/gefs_shortparmlist_3d_d.parm"} #Parameter table for upper air daily-averaged variables -export varlist_wav=${varlist_wav:-"${PARMgfs}/product/gefs_wav_shortparmlist.parm"} #Parameter table for wave variables -export varlist_ocn_netcdf=${varlist_ocn_netcdf:-"${PARMgfs}/product/gefs_ocn_shortparmlist.parm"} #Parameter table for wave variables -export varlist_ice_netcdf=${varlist_ice_netcdf:-"${PARMgfs}/product/gefs_ice_shortparmlist.parm"} #Parameter table for wave variables -export varlist_ocn_grib2=${varlist_ocn_grib2:-"${PARMgfs}/product/gefs_ocn_shortparmlist_grib2.parm"} #Parameter table for wave variables -export varlist_ice_grib2=${varlist_ice_grib2:-"${PARMgfs}/product/gefs_ice_shortparmlist_grib2.parm"} #Parameter table for wave variables - #Check to make sure FHMAX_HF_GFS is less than FHMAX_GFS if [[ "${FHMAX_GFS}" -lt "${FHMAX_HF_GFS}" ]];then echo "FATAL ERROR: FHMAX_GFS (${FHMAX_GFS}) is less than FHMAX_HF_GFS (${FHMAX_HF_GFS}). FHMAX_GFS must be greater than FHMAX_HF_GFS." From 55f65d5de547626011690786402b5aa0ed8d19fd Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 9 Jul 2024 17:38:47 +0000 Subject: [PATCH 081/141] Add extractvars function to copy output to COMROOT A function has been added that checks if an output file exits. If the output file exits, then the output file is copied from RUNDIRS to COMROOT. --- scripts/exglobal_extractvars.sh | 2 -- ush/extractvars_tools.sh | 10 ++++++++++ ush/gefs_atmos_extractvars.sh | 2 ++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/scripts/exglobal_extractvars.sh b/scripts/exglobal_extractvars.sh index da800985c8..4e33c48cf1 100755 --- a/scripts/exglobal_extractvars.sh +++ b/scripts/exglobal_extractvars.sh @@ -24,8 +24,6 @@ if [[ "${DO_ATM}" == "YES" ]];then mkdir -p "${DATA}/mem${ENSMEM}_atmos" fi ${EXTRCTVARA} "${DATA}/mem${ENSMEM}_atmos" - cp -pr "${DATA}/mem${ENSMEM}_atmos/f2d" "${COMOUT_RFCST_PROD_ATMOS}" - cp -pr "${DATA}/mem${ENSMEM}_atmos/f3d" "${COMOUT_RFCST_PROD_ATMOS}" fi #Extract variables for ocean diff --git a/ush/extractvars_tools.sh b/ush/extractvars_tools.sh index 5692a774a7..7b12387855 100644 --- a/ush/extractvars_tools.sh +++ b/ush/extractvars_tools.sh @@ -42,3 +42,13 @@ daily_avg_atmos() { vcnt=$(( vcnt + 1 )) done <"${varlist_d}" #variable } + +copy_to_comout() { + rundir_outfile=$1 #output data file generated in RUNDIR + comout_dir=$2 #destination directory to which to copy the data file + if [[ -f ${rundir_outfile} ]];then + cp -pr ${rundir_outfile} ${comout_dir} + else + echo "WARNING: Output variable (${rundir_outfile}) does not exist." + fi +} diff --git a/ush/gefs_atmos_extractvars.sh b/ush/gefs_atmos_extractvars.sh index 8baccab4b2..70acac5224 100755 --- a/ush/gefs_atmos_extractvars.sh +++ b/ush/gefs_atmos_extractvars.sh @@ -70,6 +70,7 @@ for outtype in "f2d" "f3d"; do if [[ ! -f "${requestedvars1}" ]]; then touch "${requestedvars1}"; fi if [[ ! -f "${requestedvars2}" ]]; then touch "${requestedvars2}"; fi check_atmos "${requestedvars1}" "${requestedvars2}" "${varlist}" + copy_to_comout "${outfile}" "${COMOUT_RFCST_PROD_ATMOS}" #Compute daily average for a subset of variables if (( nh % 6 == 0 )) && (( nh != 0 )) && [[ "${outtype}" == "f3d" ]];then @@ -80,6 +81,7 @@ for outtype in "f2d" "f3d"; do ${WGRIB2} "${infile2}" | grep -F -f "${varlist_d}" | ${WGRIB2} -i "${infile2}" -append -grib "${outfile}" if [[ ${fcnt} -eq 4 ]];then daily_avg_atmos + copy_to_comout "${davg_file}" "${COMOUT_RFCST_PROD_ATMOS}" fcnt=1 dcnt=$(( dcnt + 1 )) else From 98c3c487c7abb73da7e0d91bdd6ccb275871efe4 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 9 Jul 2024 18:08:55 +0000 Subject: [PATCH 082/141] Copy ocn, ice and wave files individually The ocn, ice and wave extractvar scripts have been modified so that the output can be copied to COMROOT individually. --- scripts/exglobal_extractvars.sh | 7 ++----- ush/gefs_ocnice_extractvars.sh | 6 +++++- ush/gefs_wav_extractvars.sh | 3 ++- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/scripts/exglobal_extractvars.sh b/scripts/exglobal_extractvars.sh index 4e33c48cf1..4542a24fed 100755 --- a/scripts/exglobal_extractvars.sh +++ b/scripts/exglobal_extractvars.sh @@ -40,8 +40,7 @@ if [[ "${DO_OCN}" == "YES" ]];then echo "FATAL ERROR: Invalid ocean data format provided (${ocn_dataformat})" export err=1; err_chk fi - ${EXTRCTVARO} "${DATA}/mem${ENSMEM}_ocn" "${varlist_ocn}" "${ocn_dataformat}" "${ocnres}" "${compress_ocn}" "${FHOUT_OCN_GFS}" - cp -pr "${DATA}/mem${ENSMEM}_ocn/." "${COMOUT_RFCST_PROD_OCN}" + ${EXTRCTVARO} "${DATA}/mem${ENSMEM}_ocn" "${varlist_ocn}" "${ocn_dataformat}" "${ocnres}" "${compress_ocn}" "${FHOUT_OCN_GFS}" "${COMOUT_RFCST_PROD_OCN}" fi #Extract variables for ice @@ -58,8 +57,7 @@ if [[ "${DO_ICE}" == "YES" ]];then echo "FATAL ERROR: Invalid ice data format provided (${ice_dataformat})" export err=1; err_chk fi - ${EXTRCTVARO} "${DATA}/mem${ENSMEM}_ice" "${varlist_ice}" "${ice_dataformat}" "${iceres}" "${compress_ice}" "${FHOUT_ICE_GFS}" - cp -pr "${DATA}/mem${ENSMEM}_ice/." "${COMOUT_RFCST_PROD_ICE}" + ${EXTRCTVARO} "${DATA}/mem${ENSMEM}_ice" "${varlist_ice}" "${ice_dataformat}" "${iceres}" "${compress_ice}" "${FHOUT_ICE_GFS}" "${COMOUT_RFCST_PROD_ICE}" fi #Extract variables for wave @@ -69,7 +67,6 @@ if [[ "${DO_WAVE}" == "YES" ]];then mkdir -p "${DATA}/mem${ENSMEM}_wav" fi ${EXTRCTVARW} "${DATA}/mem${ENSMEM}_wav" - cp -pr "${DATA}/mem${ENSMEM}_wav/." "${COMOUT_RFCST_PROD_WAV}" fi exit 0 diff --git a/ush/gefs_ocnice_extractvars.sh b/ush/gefs_ocnice_extractvars.sh index 16e8038de8..89183b4262 100755 --- a/ush/gefs_ocnice_extractvars.sh +++ b/ush/gefs_ocnice_extractvars.sh @@ -3,6 +3,7 @@ # Script: # source "${USHgfs}/preamble.sh" +source "${USHgfs}/extractvars_tools.sh" outdirpre=${1} varlist=${2} @@ -10,6 +11,7 @@ dataformat=${3} datares=${4} datacompress=${5} fhout_ocnice=${6} +comout_rfcst_prod_ocnice=${7} cd "${outdirpre}" || true nh=${FHMIN_GFS} @@ -62,11 +64,13 @@ while [[ ${nh} -le ${FHMAX_GFS} ]];do fi if [[ ${datacompress} -eq 1 ]];then ${COMPRSCMD} "${outfile}" + copy_to_comout "${outfile}.bz2" "${comout_rfcst_prod_ocnice}" + else + copy_to_comout "${outfile}" "${comout_rfcst_prod_ocnice}" fi else echo "WARNING: ${infile} does not exist." fi - nh=$(( nh + fhout_ocnice )) done diff --git a/ush/gefs_wav_extractvars.sh b/ush/gefs_wav_extractvars.sh index e2746c5d89..6d960e1d78 100755 --- a/ush/gefs_wav_extractvars.sh +++ b/ush/gefs_wav_extractvars.sh @@ -3,6 +3,7 @@ # Script: # source "${USHgfs}/preamble.sh" +source "${USHgfs}/extractvars_tools.sh" DATA=${1} cd "${DATA}" || true @@ -21,7 +22,7 @@ while [[ ${nh} -le ${FHMAX_WAV} ]];do else echo "WARNING: ${infile} does not exist." fi - + copy_to_comout "${outfile}" "${COMOUT_RFCST_PROD_WAV}" nh=$(( nh + FHOUT_WAV_NOSCRUB )) done #fhr From 1985224e72eca94b5f1ebea974cdf7c39d1efb5c Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 9 Jul 2024 18:20:30 +0000 Subject: [PATCH 083/141] Split COMOUT RFCST ATMOS into two directories The COMOUT RFCST ATMOS variable has been split into two directories. There is one directory for f2d variables and one directory for f3d variables. --- jobs/JGLOBAL_EXTRACTVARS | 6 ++++-- parm/config/gfs/config.com | 3 ++- ush/gefs_atmos_extractvars.sh | 2 ++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/jobs/JGLOBAL_EXTRACTVARS b/jobs/JGLOBAL_EXTRACTVARS index f66f733195..1c55c24bc5 100755 --- a/jobs/JGLOBAL_EXTRACTVARS +++ b/jobs/JGLOBAL_EXTRACTVARS @@ -19,12 +19,14 @@ YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COMIN_ICE_NETCDF:COM_ICE_NETCDF_ YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COMIN_WAVE_GRID:COM_WAVE_GRID_TMPL" -YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COMOUT_RFCST_PROD_ATMOS:COM_RFCST_PROD_ATMOS_TMPL" +YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COMOUT_RFCST_PROD_ATMOS_F2D:COM_RFCST_PROD_ATMOS_F2D_TMPL" +YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COMOUT_RFCST_PROD_ATMOS_F3D:COM_RFCST_PROD_ATMOS_F3D_TMPL" YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COMOUT_RFCST_PROD_OCN:COM_RFCST_PROD_OCN_TMPL" YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COMOUT_RFCST_PROD_ICE:COM_RFCST_PROD_ICE_TMPL" YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COMOUT_RFCST_PROD_WAV:COM_RFCST_PROD_WAV_TMPL" -if [[ ! -d "${COMOUT_RFCST_PROD_ATMOS}" ]]; then mkdir -p "${COMOUT_RFCST_PROD_ATMOS}"; fi +if [[ ! -d "${COMOUT_RFCST_PROD_ATMOS_F2D}" ]]; then mkdir -p "${COMOUT_RFCST_PROD_ATMOS_F2D}"; fi +if [[ ! -d "${COMOUT_RFCST_PROD_ATMOS_F3D}" ]]; then mkdir -p "${COMOUT_RFCST_PROD_ATMOS_F3D}"; fi if [[ ! -d "${COMOUT_RFCST_PROD_OCN}" ]]; then mkdir -p "${COMOUT_RFCST_PROD_OCN}"; fi if [[ ! -d "${COMOUT_RFCST_PROD_WAV}" ]]; then mkdir -p "${COMOUT_RFCST_PROD_WAV}"; fi if [[ ! -d "${COMOUT_RFCST_PROD_ICE}" ]]; then mkdir -p "${COMOUT_RFCST_PROD_ICE}"; fi diff --git a/parm/config/gfs/config.com b/parm/config/gfs/config.com index b2c55e2a79..01d23c6df1 100644 --- a/parm/config/gfs/config.com +++ b/parm/config/gfs/config.com @@ -94,7 +94,8 @@ declare -rx COM_ICE_NETCDF_TMPL=${COM_BASE}'/products/ice/netcdf' declare -rx COM_ICE_GRIB_TMPL=${COM_BASE}'/products/ice/grib2' declare -rx COM_ICE_GRIB_GRID_TMPL=${COM_ICE_GRIB_TMPL}'/${GRID}' -declare -rx COM_RFCST_PROD_ATMOS_TMPL=${COM_BASE}'/products/rfcst/atmos' +declare -rx COM_RFCST_PROD_ATMOS_F2D_TMPL=${COM_BASE}'/products/rfcst/atmos/f2d' +declare -rx COM_RFCST_PROD_ATMOS_F3D_TMPL=${COM_BASE}'/products/rfcst/atmos/f3d' declare -rx COM_RFCST_PROD_OCN_TMPL=${COM_BASE}'/products/rfcst/ocn' declare -rx COM_RFCST_PROD_ICE_TMPL=${COM_BASE}'/products/rfcst/ice' declare -rx COM_RFCST_PROD_WAV_TMPL=${COM_BASE}'/products/rfcst/wav' diff --git a/ush/gefs_atmos_extractvars.sh b/ush/gefs_atmos_extractvars.sh index 70acac5224..8385a3a8ed 100755 --- a/ush/gefs_atmos_extractvars.sh +++ b/ush/gefs_atmos_extractvars.sh @@ -14,9 +14,11 @@ for outtype in "f2d" "f3d"; do if [[ "${outtype}" == "f2d" ]];then varlist=${varlist_2d} + COMOUT_RFCST_PROD_ATMOS="${COMOUT_RFCST_PROD_ATMOS_F2D}" elif [[ "${outtype}" == "f3d" ]];then varlist=${varlist_3d} varlist_d=${varlist_3d_d} + COMOUT_RFCST_PROD_ATMOS="${COMOUT_RFCST_PROD_ATMOS_F3D}" fi outdirpre="${subdata}/${outtype}" From 861022038e885375d29341a92d480f580edb797f Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 9 Jul 2024 18:23:55 +0000 Subject: [PATCH 084/141] Fix shell check errors --- ush/extractvars_tools.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/extractvars_tools.sh b/ush/extractvars_tools.sh index 7b12387855..98dadecf0d 100644 --- a/ush/extractvars_tools.sh +++ b/ush/extractvars_tools.sh @@ -46,8 +46,8 @@ daily_avg_atmos() { copy_to_comout() { rundir_outfile=$1 #output data file generated in RUNDIR comout_dir=$2 #destination directory to which to copy the data file - if [[ -f ${rundir_outfile} ]];then - cp -pr ${rundir_outfile} ${comout_dir} + if [[ -f "${rundir_outfile}" ]];then + cp -pr "${rundir_outfile}" "${comout_dir}" else echo "WARNING: Output variable (${rundir_outfile}) does not exist." fi From 4f5db1e3f49d9fb85dea6ccbf71cf496e7839cbb Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 9 Jul 2024 20:00:26 +0000 Subject: [PATCH 085/141] Replace gefs string with RUN variable The string "gefs" has been replaced with the variable RUN. --- ush/extractvars_tools.sh | 4 ++-- ush/gefs_atmos_extractvars.sh | 6 +++--- ush/gefs_ocnice_extractvars.sh | 12 ++++++------ ush/gefs_wav_extractvars.sh | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/ush/extractvars_tools.sh b/ush/extractvars_tools.sh index 98dadecf0d..330a9e3997 100644 --- a/ush/extractvars_tools.sh +++ b/ush/extractvars_tools.sh @@ -30,10 +30,10 @@ check_atmos() { daily_avg_atmos() { fnd=$(printf "%2.2d" "${dcnt}") - davg_file=${outdirpre}/gefs.${cycle}.pgrb2.${outres}.24hr_avg.ldy${fnd} + davg_file=${outdirpre}/${RUN}.${cycle}.pgrb2.${outres}.24hr_avg.ldy${fnd} vcnt=1 while read -r vari; do - davgtmp=${subdata}/gefs.${cycle}.tmp.pgrb2.${outres}.ldy${fnd}.${vcnt} + davgtmp=${subdata}/${RUN}.${cycle}.tmp.pgrb2.${outres}.ldy${fnd}.${vcnt} # shellcheck disable=SC2312 ${WGRIB2} "${outfile}" | grep "${vari}" | ${WGRIB2} -i "${outfile}" -fcst_ave 6hr "${davgtmp}" # shellcheck disable=SC2312 diff --git a/ush/gefs_atmos_extractvars.sh b/ush/gefs_atmos_extractvars.sh index 8385a3a8ed..9612188f42 100755 --- a/ush/gefs_atmos_extractvars.sh +++ b/ush/gefs_atmos_extractvars.sh @@ -45,9 +45,9 @@ for outtype in "f2d" "f3d"; do fi com_var="COMIN_ATMOS_GRIB_${outres}" - infile1="${!com_var}/gefs.${cycle}.pgrb2.${outres}.f${fnh}" - infile2="${!com_var}/gefs.${cycle}.pgrb2b.${outres}.f${fnh}" - outfile=${outdirpre}/gefs.${cycle}.pgrb2.${outres}.f${fnh} + infile1="${!com_var}/${RUN}.${cycle}.pgrb2.${outres}.f${fnh}" + infile2="${!com_var}/${RUN}.${cycle}.pgrb2b.${outres}.f${fnh}" + outfile=${outdirpre}/${RUN}.${cycle}.pgrb2.${outres}.f${fnh} rm -f "${outfile}" #remove outfile if it already exists before extraction requestedvars1="partial_parm1.txt" requestedvars2="partial_parm2.txt" diff --git a/ush/gefs_ocnice_extractvars.sh b/ush/gefs_ocnice_extractvars.sh index 89183b4262..184ec96d42 100755 --- a/ush/gefs_ocnice_extractvars.sh +++ b/ush/gefs_ocnice_extractvars.sh @@ -20,22 +20,22 @@ while [[ ${nh} -le ${FHMAX_GFS} ]];do if [[ "${dataformat}" == "grib2" ]];then if [[ ${component_name} == "ocn" ]];then - infile=${COMIN_OCEAN_GRIB}/${datares}/gefs.ocean.${cycle}.${datares}.f${fnh}.grib2 + infile=${COMIN_OCEAN_GRIB}/${datares}/${RUN}.ocean.${cycle}.${datares}.f${fnh}.grib2 fi if [[ ${component_name} == "ice" ]];then - infile=${COMIN_ICE_GRIB}/${datares}/gefs.ice.${cycle}.${datares}.f${fnh}.grib2 + infile=${COMIN_ICE_GRIB}/${datares}/${RUN}.ice.${cycle}.${datares}.f${fnh}.grib2 fi - outfile=${outdirpre}/gefs.${component_name}.${cycle}.${datares}.f${fnh}.grib2 + outfile=${outdirpre}/${RUN}.${component_name}.${cycle}.${datares}.f${fnh}.grib2 fi if [[ "${dataformat}" == "netcdf" ]];then if [[ ${component_name} == "ocn" ]];then - infile=${COMIN_OCEAN_NETCDF}/gefs.ocean.${cycle}.${datares}.f${fnh}.nc + infile=${COMIN_OCEAN_NETCDF}/${RUN}.ocean.${cycle}.${datares}.f${fnh}.nc fi if [[ ${component_name} == "ice" ]];then - infile=${COMIN_ICE_NETCDF}/gefs.ice.${cycle}.${datares}.f${fnh}.nc + infile=${COMIN_ICE_NETCDF}/${RUN}.ice.${cycle}.${datares}.f${fnh}.nc fi - outfile=${outdirpre}/gefs.${component_name}.${cycle}.${datares}.f${fnh}.nc + outfile=${outdirpre}/${RUN}.${component_name}.${cycle}.${datares}.f${fnh}.nc fi if [[ -f "${infile}" ]]; then #check if input file exists before extraction diff --git a/ush/gefs_wav_extractvars.sh b/ush/gefs_wav_extractvars.sh index 6d960e1d78..119be5fa37 100755 --- a/ush/gefs_wav_extractvars.sh +++ b/ush/gefs_wav_extractvars.sh @@ -12,8 +12,8 @@ nh=6 while [[ ${nh} -le ${FHMAX_WAV} ]];do fnh=$(printf "%3.3d" "${nh}") - infile=${COMIN_WAVE_GRID}/gefswave.${cycle}.global.${wavres}.f${fnh}.grib2 - outfile=${DATA}/gefswave.${cycle}.global.${wavres}.f${fnh}.grib2 + infile=${COMIN_WAVE_GRID}/${RUN}wave.${cycle}.global.${wavres}.f${fnh}.grib2 + outfile=${DATA}/${RUN}wave.${cycle}.global.${wavres}.f${fnh}.grib2 rm -f "${outfile}" #remove outfile if it already exists before extraction if [[ -f "${infile}" ]]; then #check if input file exists before extraction From 701a85c371aaf08826313d30f54b80dfea79d54e Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 10 Jul 2024 12:56:30 +0000 Subject: [PATCH 086/141] Replace strings with arithmetic where possible --- scripts/exglobal_extractvars.sh | 2 +- ush/gefs_atmos_extractvars.sh | 4 ++-- ush/gefs_ocnice_extractvars.sh | 2 +- ush/gefs_wav_extractvars.sh | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/exglobal_extractvars.sh b/scripts/exglobal_extractvars.sh index 4542a24fed..11ed39fd82 100755 --- a/scripts/exglobal_extractvars.sh +++ b/scripts/exglobal_extractvars.sh @@ -13,7 +13,7 @@ EXTRCTVARO=${EXTRCTVARO:-"${USHgfs}/gefs_ocnice_extractvars.sh"} EXTRCTVARW=${EXTRCTVARW:-"${USHgfs}/gefs_wav_extractvars.sh"} #Check to make sure FHMAX_HF_GFS is less than FHMAX_GFS -if [[ "${FHMAX_GFS}" -lt "${FHMAX_HF_GFS}" ]];then +if (( FHMAX_GFS < FHMAX_HF_GFS )); then echo "FATAL ERROR: FHMAX_GFS (${FHMAX_GFS}) is less than FHMAX_HF_GFS (${FHMAX_HF_GFS}). FHMAX_GFS must be greater than FHMAX_HF_GFS." export err=1; err_chk fi diff --git a/ush/gefs_atmos_extractvars.sh b/ush/gefs_atmos_extractvars.sh index 9612188f42..c84bff917b 100755 --- a/ush/gefs_atmos_extractvars.sh +++ b/ush/gefs_atmos_extractvars.sh @@ -25,7 +25,7 @@ for outtype in "f2d" "f3d"; do if [[ ! -d "${outdirpre}" ]]; then mkdir -p "${outdirpre}"; fi nh=${FHMIN} - while [[ ${nh} -le ${FHMAX_GFS} ]];do + while (( nh <= FHMAX_GFS )); do fnh=$(printf "%3.3d" "${nh}") if [[ "${outtype}" == "f2d" ]];then @@ -38,7 +38,7 @@ for outtype in "f2d" "f3d"; do outres="1p00" fi - if [[ ${nh} -lt ${FHMAX_HF_GFS} ]];then + if (( nh < FHMAX_HF_GFS )); then outfreq=${FHOUT_HF_GFS} else outfreq=${FHOUT_GFS} diff --git a/ush/gefs_ocnice_extractvars.sh b/ush/gefs_ocnice_extractvars.sh index 184ec96d42..1e2b258961 100755 --- a/ush/gefs_ocnice_extractvars.sh +++ b/ush/gefs_ocnice_extractvars.sh @@ -15,7 +15,7 @@ comout_rfcst_prod_ocnice=${7} cd "${outdirpre}" || true nh=${FHMIN_GFS} -while [[ ${nh} -le ${FHMAX_GFS} ]];do +while (( nh <= FHMAX_GFS )); do fnh=$(printf "%3.3d" "${nh}") if [[ "${dataformat}" == "grib2" ]];then diff --git a/ush/gefs_wav_extractvars.sh b/ush/gefs_wav_extractvars.sh index 119be5fa37..cc84237744 100755 --- a/ush/gefs_wav_extractvars.sh +++ b/ush/gefs_wav_extractvars.sh @@ -9,7 +9,7 @@ DATA=${1} cd "${DATA}" || true nh=6 -while [[ ${nh} -le ${FHMAX_WAV} ]];do +while (( nh <= FHMAX_WAV )); do fnh=$(printf "%3.3d" "${nh}") infile=${COMIN_WAVE_GRID}/${RUN}wave.${cycle}.global.${wavres}.f${fnh}.grib2 From 16bff05c4918565ff5558add9a5c470efa26457d Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 10 Jul 2024 13:30:23 +0000 Subject: [PATCH 087/141] Replace while loops with for loops where possible Loops that have a constant interval have been changed from while loops to for loops. --- ush/gefs_ocnice_extractvars.sh | 4 +--- ush/gefs_wav_extractvars.sh | 7 ++----- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/ush/gefs_ocnice_extractvars.sh b/ush/gefs_ocnice_extractvars.sh index 1e2b258961..ee032b6df6 100755 --- a/ush/gefs_ocnice_extractvars.sh +++ b/ush/gefs_ocnice_extractvars.sh @@ -14,8 +14,7 @@ fhout_ocnice=${6} comout_rfcst_prod_ocnice=${7} cd "${outdirpre}" || true -nh=${FHMIN_GFS} -while (( nh <= FHMAX_GFS )); do +for (( nh = FHMIN_GFS; nh <= FHMAX_GFS; nh = nh + fhout_ocnice )); do fnh=$(printf "%3.3d" "${nh}") if [[ "${dataformat}" == "grib2" ]];then @@ -71,7 +70,6 @@ while (( nh <= FHMAX_GFS )); do else echo "WARNING: ${infile} does not exist." fi - nh=$(( nh + fhout_ocnice )) done exit 0 diff --git a/ush/gefs_wav_extractvars.sh b/ush/gefs_wav_extractvars.sh index cc84237744..262ddf62a6 100755 --- a/ush/gefs_wav_extractvars.sh +++ b/ush/gefs_wav_extractvars.sh @@ -8,8 +8,7 @@ source "${USHgfs}/extractvars_tools.sh" DATA=${1} cd "${DATA}" || true -nh=6 -while (( nh <= FHMAX_WAV )); do +for (( nh = FHOUT_WAV_NOSCRUB; nh <= FHMAX_WAV; nh = nh + FHOUT_WAV_NOSCRUB )); do fnh=$(printf "%3.3d" "${nh}") infile=${COMIN_WAVE_GRID}/${RUN}wave.${cycle}.global.${wavres}.f${fnh}.grib2 @@ -22,9 +21,7 @@ while (( nh <= FHMAX_WAV )); do else echo "WARNING: ${infile} does not exist." fi - copy_to_comout "${outfile}" "${COMOUT_RFCST_PROD_WAV}" - nh=$(( nh + FHOUT_WAV_NOSCRUB )) - + copy_to_comout "${outfile}" "${COMOUT_RFCST_PROD_WAV}" done #fhr exit 0 From 3a920be33079e24927e59e02ba6c849687d5543d Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 10 Jul 2024 13:40:42 +0000 Subject: [PATCH 088/141] Simplify some if-statements --- ush/gefs_ocnice_extractvars.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ush/gefs_ocnice_extractvars.sh b/ush/gefs_ocnice_extractvars.sh index ee032b6df6..bb2cf29558 100755 --- a/ush/gefs_ocnice_extractvars.sh +++ b/ush/gefs_ocnice_extractvars.sh @@ -20,8 +20,7 @@ for (( nh = FHMIN_GFS; nh <= FHMAX_GFS; nh = nh + fhout_ocnice )); do if [[ "${dataformat}" == "grib2" ]];then if [[ ${component_name} == "ocn" ]];then infile=${COMIN_OCEAN_GRIB}/${datares}/${RUN}.ocean.${cycle}.${datares}.f${fnh}.grib2 - fi - if [[ ${component_name} == "ice" ]];then + elif [[ ${component_name} == "ice" ]];then infile=${COMIN_ICE_GRIB}/${datares}/${RUN}.ice.${cycle}.${datares}.f${fnh}.grib2 fi outfile=${outdirpre}/${RUN}.${component_name}.${cycle}.${datares}.f${fnh}.grib2 @@ -30,8 +29,7 @@ for (( nh = FHMIN_GFS; nh <= FHMAX_GFS; nh = nh + fhout_ocnice )); do if [[ "${dataformat}" == "netcdf" ]];then if [[ ${component_name} == "ocn" ]];then infile=${COMIN_OCEAN_NETCDF}/${RUN}.ocean.${cycle}.${datares}.f${fnh}.nc - fi - if [[ ${component_name} == "ice" ]];then + elif [[ ${component_name} == "ice" ]];then infile=${COMIN_ICE_NETCDF}/${RUN}.ice.${cycle}.${datares}.f${fnh}.nc fi outfile=${outdirpre}/${RUN}.${component_name}.${cycle}.${datares}.f${fnh}.nc From 1022872d15ba81227ccd950a03583e48fb096d5f Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 10 Jul 2024 15:37:50 +0000 Subject: [PATCH 089/141] Modify extractvars dependencies The extractvars dependencies have been modified. When the prod jobs for a certain member are finished, the extractvars task for that member will be triggered. Double quotes have also been added to the atmos extractvars script. --- ush/gefs_atmos_extractvars.sh | 2 +- workflow/rocoto/gefs_tasks.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ush/gefs_atmos_extractvars.sh b/ush/gefs_atmos_extractvars.sh index c84bff917b..ca67971490 100755 --- a/ush/gefs_atmos_extractvars.sh +++ b/ush/gefs_atmos_extractvars.sh @@ -47,7 +47,7 @@ for outtype in "f2d" "f3d"; do com_var="COMIN_ATMOS_GRIB_${outres}" infile1="${!com_var}/${RUN}.${cycle}.pgrb2.${outres}.f${fnh}" infile2="${!com_var}/${RUN}.${cycle}.pgrb2b.${outres}.f${fnh}" - outfile=${outdirpre}/${RUN}.${cycle}.pgrb2.${outres}.f${fnh} + outfile="${outdirpre}/${RUN}.${cycle}.pgrb2.${outres}.f${fnh}" rm -f "${outfile}" #remove outfile if it already exists before extraction requestedvars1="partial_parm1.txt" requestedvars2="partial_parm2.txt" diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 00f9553434..314747d7a6 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -490,13 +490,13 @@ def extractvars(self): dep_dict = {'type': 'task', 'name': f'wave_post_grid_mem#member#'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_ocean: - dep_dict = {'type': 'metatask', 'name': f'ocean_prod'} + dep_dict = {'type': 'metatask', 'name': f'ocean_prod_#member#'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_ice: - dep_dict = {'type': 'metatask', 'name': f'ice_prod'} + dep_dict = {'type': 'metatask', 'name': f'ice_prod_#member#'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_atm: - dep_dict = {'type': 'metatask', 'name': f'atmos_prod'} + dep_dict = {'type': 'metatask', 'name': f'atmos_prod_#member#'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) extractvars_envars = self.envars.copy() From 22de8a82a96c4510d5d18081c5444f6722c18ef3 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 10 Jul 2024 16:09:56 +0000 Subject: [PATCH 090/141] Remove gefs string from ush script filenames The string "gefs" has been removed from the extractvars ush script filenames. --- scripts/exglobal_extractvars.sh | 6 +++--- ush/{gefs_atmos_extractvars.sh => atmos_extractvars.sh} | 0 ush/{gefs_ocnice_extractvars.sh => ocnice_extractvars.sh} | 0 ush/{gefs_wav_extractvars.sh => wav_extractvars.sh} | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename ush/{gefs_atmos_extractvars.sh => atmos_extractvars.sh} (100%) rename ush/{gefs_ocnice_extractvars.sh => ocnice_extractvars.sh} (100%) rename ush/{gefs_wav_extractvars.sh => wav_extractvars.sh} (100%) diff --git a/scripts/exglobal_extractvars.sh b/scripts/exglobal_extractvars.sh index 11ed39fd82..b8a32c8b3d 100755 --- a/scripts/exglobal_extractvars.sh +++ b/scripts/exglobal_extractvars.sh @@ -8,9 +8,9 @@ source "${USHgfs}/preamble.sh" export WGRIB2=${WGRIB2:-${wgrib2_ROOT}/bin/wgrib2} # Scripts used -EXTRCTVARA=${EXTRCTVARA:-"${USHgfs}/gefs_atmos_extractvars.sh"} -EXTRCTVARO=${EXTRCTVARO:-"${USHgfs}/gefs_ocnice_extractvars.sh"} -EXTRCTVARW=${EXTRCTVARW:-"${USHgfs}/gefs_wav_extractvars.sh"} +EXTRCTVARA=${EXTRCTVARA:-"${USHgfs}/atmos_extractvars.sh"} +EXTRCTVARO=${EXTRCTVARO:-"${USHgfs}/ocnice_extractvars.sh"} +EXTRCTVARW=${EXTRCTVARW:-"${USHgfs}/wav_extractvars.sh"} #Check to make sure FHMAX_HF_GFS is less than FHMAX_GFS if (( FHMAX_GFS < FHMAX_HF_GFS )); then diff --git a/ush/gefs_atmos_extractvars.sh b/ush/atmos_extractvars.sh similarity index 100% rename from ush/gefs_atmos_extractvars.sh rename to ush/atmos_extractvars.sh diff --git a/ush/gefs_ocnice_extractvars.sh b/ush/ocnice_extractvars.sh similarity index 100% rename from ush/gefs_ocnice_extractvars.sh rename to ush/ocnice_extractvars.sh diff --git a/ush/gefs_wav_extractvars.sh b/ush/wav_extractvars.sh similarity index 100% rename from ush/gefs_wav_extractvars.sh rename to ush/wav_extractvars.sh From 88e4ccfeeb77217f264480e5161865015d097107 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 10 Jul 2024 19:14:33 +0000 Subject: [PATCH 091/141] Update resources used in extractvars The resources requested have been reduced in config.resources. --- parm/config/gefs/config.resources | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index adacb8ec55..3e4f05b4c1 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -295,14 +295,14 @@ case ${step} in "extractvars") export wtime_extractvars="00:30:00" - export npe_extractvars=6 + export npe_extractvars=1 export nth_extractvars=1 export npe_node_extractvars="${npe_extractvars}" export wtime_extractvars_gfs="${wtime_extractvars}" export npe_extractvars_gfs="${npe_extractvars}" export nth_extractvars_gfs="${nth_extractvars}" export npe_node_extractvars_gfs="${npe_node_extractvars}" - export is_exclusive=True + export is_exclusive=False ;; *) From 44eb052e1c8e45a85ae950a15d6a659c67aadf42 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Thu, 11 Jul 2024 16:49:23 +0000 Subject: [PATCH 092/141] Use more efficient method to generate parmlist A more efficient method is used to generate a partial parmlist in extractvars. --- ush/extractvars_tools.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/extractvars_tools.sh b/ush/extractvars_tools.sh index 330a9e3997..96b9f1ae60 100644 --- a/ush/extractvars_tools.sh +++ b/ush/extractvars_tools.sh @@ -5,9 +5,9 @@ gen_parmlist() { requestedvar_in_file1=$2 varlist=$3 rm -vf "${requestedvar_in_file1}" + file_vars=$(${WGRIB2} "${infileg}") while read -r vari; do - varinfile=$(${WGRIB2} "${infileg}" | grep "${vari}") || true - if [[ -n "${varinfile}" ]];then # if varinfile is not empty + if [[ "${file_vars}" =~ "${vari}" && -n "${vari}" ]]; then echo "${vari}" >> "${requestedvar_in_file1}" fi done <"${varlist}" From bffb227bd62bd14b811984a9b94ea8b1a34cba84 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Thu, 11 Jul 2024 17:18:09 +0000 Subject: [PATCH 093/141] Add job-specific variable for FHMAX_HF_GFS A job-specific variable for FHMAX_HF_GFS has been added called FHMAX_HF_EV. FHMAX_HF_GFS has been replaced by FHMAX_HF_EV accordingly. --- scripts/exglobal_extractvars.sh | 7 ++++--- ush/atmos_extractvars.sh | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/exglobal_extractvars.sh b/scripts/exglobal_extractvars.sh index b8a32c8b3d..be1dd52f0a 100755 --- a/scripts/exglobal_extractvars.sh +++ b/scripts/exglobal_extractvars.sh @@ -12,10 +12,11 @@ EXTRCTVARA=${EXTRCTVARA:-"${USHgfs}/atmos_extractvars.sh"} EXTRCTVARO=${EXTRCTVARO:-"${USHgfs}/ocnice_extractvars.sh"} EXTRCTVARW=${EXTRCTVARW:-"${USHgfs}/wav_extractvars.sh"} -#Check to make sure FHMAX_HF_GFS is less than FHMAX_GFS +#Define a job-specific variable for FHMAX_HF_GFS if (( FHMAX_GFS < FHMAX_HF_GFS )); then - echo "FATAL ERROR: FHMAX_GFS (${FHMAX_GFS}) is less than FHMAX_HF_GFS (${FHMAX_HF_GFS}). FHMAX_GFS must be greater than FHMAX_HF_GFS." - export err=1; err_chk + export FHMAX_HF_EV=FHMAX_GFS +else + export FHMAX_HF_EV=FHMAX_HF_GFS fi #Extract variables for atmosphere diff --git a/ush/atmos_extractvars.sh b/ush/atmos_extractvars.sh index ca67971490..381d7177d2 100755 --- a/ush/atmos_extractvars.sh +++ b/ush/atmos_extractvars.sh @@ -29,7 +29,7 @@ for outtype in "f2d" "f3d"; do fnh=$(printf "%3.3d" "${nh}") if [[ "${outtype}" == "f2d" ]];then - if [[ ${nh} -le ${FHMAX_HF_GFS} ]];then + if [[ ${nh} -le ${FHMAX_HF_EV} ]];then outres="0p25" else outres="0p50" @@ -38,7 +38,7 @@ for outtype in "f2d" "f3d"; do outres="1p00" fi - if (( nh < FHMAX_HF_GFS )); then + if (( nh < FHMAX_HF_EV )); then outfreq=${FHOUT_HF_GFS} else outfreq=${FHOUT_GFS} From 39e5582f5959ab0aa5d12a6a671754d9d2b9dbec Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Thu, 11 Jul 2024 17:42:24 +0000 Subject: [PATCH 094/141] Remove unnecessary mkdir commands --- ush/atmos_extractvars.sh | 4 ++-- ush/ocnice_extractvars.sh | 3 ++- ush/wav_extractvars.sh | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/ush/atmos_extractvars.sh b/ush/atmos_extractvars.sh index 381d7177d2..1529508021 100755 --- a/ush/atmos_extractvars.sh +++ b/ush/atmos_extractvars.sh @@ -8,7 +8,7 @@ fcnt=1 dcnt=1 subdata=${1} -cd "${subdata}" || true +[[ -d "${subdata}" ]] || mkdir -p "${subdata}" for outtype in "f2d" "f3d"; do @@ -22,7 +22,7 @@ for outtype in "f2d" "f3d"; do fi outdirpre="${subdata}/${outtype}" - if [[ ! -d "${outdirpre}" ]]; then mkdir -p "${outdirpre}"; fi + [[ -d "${outdirpre}" ]] || mkdir -p "${outdirpre}" nh=${FHMIN} while (( nh <= FHMAX_GFS )); do diff --git a/ush/ocnice_extractvars.sh b/ush/ocnice_extractvars.sh index bb2cf29558..10a81166e0 100755 --- a/ush/ocnice_extractvars.sh +++ b/ush/ocnice_extractvars.sh @@ -12,7 +12,8 @@ datares=${4} datacompress=${5} fhout_ocnice=${6} comout_rfcst_prod_ocnice=${7} -cd "${outdirpre}" || true + +[[ -d "${outdirpre}" ]] || mkdir -p "${outdirpre}" for (( nh = FHMIN_GFS; nh <= FHMAX_GFS; nh = nh + fhout_ocnice )); do fnh=$(printf "%3.3d" "${nh}") diff --git a/ush/wav_extractvars.sh b/ush/wav_extractvars.sh index 262ddf62a6..03196e37de 100755 --- a/ush/wav_extractvars.sh +++ b/ush/wav_extractvars.sh @@ -6,7 +6,8 @@ source "${USHgfs}/preamble.sh" source "${USHgfs}/extractvars_tools.sh" DATA=${1} -cd "${DATA}" || true + +[[ -d "${DATA}" ]] || mkdir -p "${DATA}" for (( nh = FHOUT_WAV_NOSCRUB; nh <= FHMAX_WAV; nh = nh + FHOUT_WAV_NOSCRUB )); do fnh=$(printf "%3.3d" "${nh}") From 9a03c58c4d9d5e4c2972ec5e191d9cfa2ecf5a47 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Thu, 11 Jul 2024 17:50:37 +0000 Subject: [PATCH 095/141] Move temporary atmos files into subdata directory The temporary atmos files for the extractvars task have been moved to the subdata directory. --- ush/atmos_extractvars.sh | 4 ++-- ush/extractvars_tools.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ush/atmos_extractvars.sh b/ush/atmos_extractvars.sh index 1529508021..7bdc1db1ab 100755 --- a/ush/atmos_extractvars.sh +++ b/ush/atmos_extractvars.sh @@ -49,8 +49,8 @@ for outtype in "f2d" "f3d"; do infile2="${!com_var}/${RUN}.${cycle}.pgrb2b.${outres}.f${fnh}" outfile="${outdirpre}/${RUN}.${cycle}.pgrb2.${outres}.f${fnh}" rm -f "${outfile}" #remove outfile if it already exists before extraction - requestedvars1="partial_parm1.txt" - requestedvars2="partial_parm2.txt" + requestedvars1="${subdata}/partial_parm1.txt" + requestedvars2="${subdata}/partial_parm2.txt" rm -f "${requestedvars1}" rm -f "${requestedvars2}" if [[ -f "${infile1}" ]]; then #check if input file exists before extraction diff --git a/ush/extractvars_tools.sh b/ush/extractvars_tools.sh index 96b9f1ae60..4ab6b02d08 100644 --- a/ush/extractvars_tools.sh +++ b/ush/extractvars_tools.sh @@ -17,7 +17,7 @@ check_atmos() { requestedvar_in_file1=$1 requestedvar_in_file2=$2 varlist=$3 - requestedvar_in_allgrb2file=requestedvar_in_allgrb2file.txt + requestedvar_in_allgrb2file="${subdata}/requestedvar_in_allgrb2file.txt" rm -rvf "${requestedvar_in_allgrb2file}" cat "${requestedvar_in_file1}" "${requestedvar_in_file2}" >> "${requestedvar_in_allgrb2file}" mapfile -t requestedvar_in_allgrb2file_arr < "${requestedvar_in_allgrb2file}" From ec7a307791eee01b3063f40ef8f0f3eb6a7c6c07 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Thu, 11 Jul 2024 17:59:25 +0000 Subject: [PATCH 096/141] Rename varlist in check_atmos function The varlist variable in the check_atmos function has been renamed to varlistl. --- ush/extractvars_tools.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/extractvars_tools.sh b/ush/extractvars_tools.sh index 4ab6b02d08..109bcf7913 100644 --- a/ush/extractvars_tools.sh +++ b/ush/extractvars_tools.sh @@ -16,7 +16,7 @@ gen_parmlist() { check_atmos() { requestedvar_in_file1=$1 requestedvar_in_file2=$2 - varlist=$3 + varlistl=$3 requestedvar_in_allgrb2file="${subdata}/requestedvar_in_allgrb2file.txt" rm -rvf "${requestedvar_in_allgrb2file}" cat "${requestedvar_in_file1}" "${requestedvar_in_file2}" >> "${requestedvar_in_allgrb2file}" @@ -25,7 +25,7 @@ check_atmos() { if [[ ! ${requestedvar_in_allgrb2file_arr[*]} =~ ${vari} ]] ;then echo "WARNING: PARM VARIABLE (${vari}) is not available in pgrb and pgrb2b." fi - done <"${varlist}" + done <"${varlistl}" } daily_avg_atmos() { From bee6fa42308941432b7e4645a94362403a1c15e8 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Thu, 11 Jul 2024 18:37:22 +0000 Subject: [PATCH 097/141] Address shell check error and fix error A shell check has been addressed. Also, an empty partial parm file is created before any wgrib2 functions are called in the atmos ush extractvars script. This prevents any no such file or directory error when wgrib2 is used. --- ush/atmos_extractvars.sh | 6 +++--- ush/extractvars_tools.sh | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ush/atmos_extractvars.sh b/ush/atmos_extractvars.sh index 7bdc1db1ab..5b66faa8d7 100755 --- a/ush/atmos_extractvars.sh +++ b/ush/atmos_extractvars.sh @@ -53,6 +53,8 @@ for outtype in "f2d" "f3d"; do requestedvars2="${subdata}/partial_parm2.txt" rm -f "${requestedvars1}" rm -f "${requestedvars2}" + if [[ ! -f "${requestedvars1}" ]]; then touch "${requestedvars1}"; fi + if [[ ! -f "${requestedvars2}" ]]; then touch "${requestedvars2}"; fi if [[ -f "${infile1}" ]]; then #check if input file exists before extraction gen_parmlist "${infile1}" "${requestedvars1}" "${varlist}" # shellcheck disable=SC2312 @@ -69,8 +71,6 @@ for outtype in "f2d" "f3d"; do echo "WARNING: ${infile2} does not exist." fi - if [[ ! -f "${requestedvars1}" ]]; then touch "${requestedvars1}"; fi - if [[ ! -f "${requestedvars2}" ]]; then touch "${requestedvars2}"; fi check_atmos "${requestedvars1}" "${requestedvars2}" "${varlist}" copy_to_comout "${outfile}" "${COMOUT_RFCST_PROD_ATMOS}" @@ -82,7 +82,7 @@ for outtype in "f2d" "f3d"; do # shellcheck disable=SC2312 ${WGRIB2} "${infile2}" | grep -F -f "${varlist_d}" | ${WGRIB2} -i "${infile2}" -append -grib "${outfile}" if [[ ${fcnt} -eq 4 ]];then - daily_avg_atmos + daily_avg_atmos ${outfile} copy_to_comout "${davg_file}" "${COMOUT_RFCST_PROD_ATMOS}" fcnt=1 dcnt=$(( dcnt + 1 )) diff --git a/ush/extractvars_tools.sh b/ush/extractvars_tools.sh index 109bcf7913..fb41a4902e 100644 --- a/ush/extractvars_tools.sh +++ b/ush/extractvars_tools.sh @@ -4,10 +4,9 @@ gen_parmlist() { infileg=$1 requestedvar_in_file1=$2 varlist=$3 - rm -vf "${requestedvar_in_file1}" file_vars=$(${WGRIB2} "${infileg}") while read -r vari; do - if [[ "${file_vars}" =~ "${vari}" && -n "${vari}" ]]; then + if [[ "${file_vars}" =~ ${vari} && -n "${vari}" ]]; then echo "${vari}" >> "${requestedvar_in_file1}" fi done <"${varlist}" @@ -29,13 +28,14 @@ check_atmos() { } daily_avg_atmos() { + outfile_p=$1 fnd=$(printf "%2.2d" "${dcnt}") davg_file=${outdirpre}/${RUN}.${cycle}.pgrb2.${outres}.24hr_avg.ldy${fnd} vcnt=1 while read -r vari; do davgtmp=${subdata}/${RUN}.${cycle}.tmp.pgrb2.${outres}.ldy${fnd}.${vcnt} # shellcheck disable=SC2312 - ${WGRIB2} "${outfile}" | grep "${vari}" | ${WGRIB2} -i "${outfile}" -fcst_ave 6hr "${davgtmp}" + ${WGRIB2} "${outfile_p}" | grep "${vari}" | ${WGRIB2} -i "${outfile_p}" -fcst_ave 6hr "${davgtmp}" # shellcheck disable=SC2312 ${WGRIB2} "${davgtmp}" | ${WGRIB2} -i "${davgtmp}" -append -grib "${davg_file}" rm -f "${davgtmp}" From e565ff09bdd4431e489e67f23218e910e3f0d19d Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Thu, 11 Jul 2024 18:48:30 +0000 Subject: [PATCH 098/141] Add more detail to warning message More detail has been added to a warning message when a requested parm variable cannot be found in any of the input atmos grib2 files for a given lead time and given member. --- ush/atmos_extractvars.sh | 2 +- ush/extractvars_tools.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ush/atmos_extractvars.sh b/ush/atmos_extractvars.sh index 5b66faa8d7..480745a62f 100755 --- a/ush/atmos_extractvars.sh +++ b/ush/atmos_extractvars.sh @@ -71,7 +71,7 @@ for outtype in "f2d" "f3d"; do echo "WARNING: ${infile2} does not exist." fi - check_atmos "${requestedvars1}" "${requestedvars2}" "${varlist}" + check_atmos "${requestedvars1}" "${requestedvars2}" "${varlist}" "${fnh}" copy_to_comout "${outfile}" "${COMOUT_RFCST_PROD_ATMOS}" #Compute daily average for a subset of variables diff --git a/ush/extractvars_tools.sh b/ush/extractvars_tools.sh index fb41a4902e..b1bd924956 100644 --- a/ush/extractvars_tools.sh +++ b/ush/extractvars_tools.sh @@ -16,13 +16,14 @@ check_atmos() { requestedvar_in_file1=$1 requestedvar_in_file2=$2 varlistl=$3 + fnhl=$4 requestedvar_in_allgrb2file="${subdata}/requestedvar_in_allgrb2file.txt" rm -rvf "${requestedvar_in_allgrb2file}" cat "${requestedvar_in_file1}" "${requestedvar_in_file2}" >> "${requestedvar_in_allgrb2file}" mapfile -t requestedvar_in_allgrb2file_arr < "${requestedvar_in_allgrb2file}" while read -r vari; do if [[ ! ${requestedvar_in_allgrb2file_arr[*]} =~ ${vari} ]] ;then - echo "WARNING: PARM VARIABLE (${vari}) is not available in pgrb and pgrb2b." + echo "WARNING: PARM VARIABLE (${vari}) is not available in pgrb and pgrb2b for f${fnhl}." fi done <"${varlistl}" } From 90c692f14a410d86ff9b04822f77d9082ce3c08e Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Thu, 11 Jul 2024 18:53:53 +0000 Subject: [PATCH 099/141] Fix shell check error --- ush/atmos_extractvars.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/atmos_extractvars.sh b/ush/atmos_extractvars.sh index 480745a62f..ad1817762f 100755 --- a/ush/atmos_extractvars.sh +++ b/ush/atmos_extractvars.sh @@ -82,7 +82,7 @@ for outtype in "f2d" "f3d"; do # shellcheck disable=SC2312 ${WGRIB2} "${infile2}" | grep -F -f "${varlist_d}" | ${WGRIB2} -i "${infile2}" -append -grib "${outfile}" if [[ ${fcnt} -eq 4 ]];then - daily_avg_atmos ${outfile} + daily_avg_atmos "${outfile}" copy_to_comout "${davg_file}" "${COMOUT_RFCST_PROD_ATMOS}" fcnt=1 dcnt=$(( dcnt + 1 )) From c968e1f9a4591da1a6899901ccc0413bf955c08a Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 16 Jul 2024 13:14:39 +0000 Subject: [PATCH 100/141] Declare functions and source in exscript Extractvars functions have been declared in extractvars_tools. The extractvars functions are sourced in exglobal_extractvars instead of atmos_extractvars. --- scripts/exglobal_extractvars.sh | 1 + ush/atmos_extractvars.sh | 2 +- ush/extractvars_tools.sh | 5 +++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/exglobal_extractvars.sh b/scripts/exglobal_extractvars.sh index be1dd52f0a..7558d38d30 100755 --- a/scripts/exglobal_extractvars.sh +++ b/scripts/exglobal_extractvars.sh @@ -3,6 +3,7 @@ # Script: # source "${USHgfs}/preamble.sh" +source "${USHgfs}/extractvars_tools.sh" # Programs used export WGRIB2=${WGRIB2:-${wgrib2_ROOT}/bin/wgrib2} diff --git a/ush/atmos_extractvars.sh b/ush/atmos_extractvars.sh index ad1817762f..2eca318ebb 100755 --- a/ush/atmos_extractvars.sh +++ b/ush/atmos_extractvars.sh @@ -3,7 +3,7 @@ # Script: # source "${USHgfs}/preamble.sh" -source "${USHgfs}/extractvars_tools.sh" + fcnt=1 dcnt=1 subdata=${1} diff --git a/ush/extractvars_tools.sh b/ush/extractvars_tools.sh index b1bd924956..b8c86d36a4 100644 --- a/ush/extractvars_tools.sh +++ b/ush/extractvars_tools.sh @@ -53,3 +53,8 @@ copy_to_comout() { echo "WARNING: Output variable (${rundir_outfile}) does not exist." fi } + +declare -xf gen_parmlist +declare -xf check_atmos +declare -xf daily_avg_atmos +declare -xf copy_to_comout From d86b1e96d20d61f52e60c9e515b55536055e09e4 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 16 Jul 2024 13:23:01 +0000 Subject: [PATCH 101/141] Fix indent An indent has been fixed in extractvars_tools. --- ush/extractvars_tools.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/extractvars_tools.sh b/ush/extractvars_tools.sh index b8c86d36a4..1ca2876ce6 100644 --- a/ush/extractvars_tools.sh +++ b/ush/extractvars_tools.sh @@ -50,7 +50,7 @@ copy_to_comout() { if [[ -f "${rundir_outfile}" ]];then cp -pr "${rundir_outfile}" "${comout_dir}" else - echo "WARNING: Output variable (${rundir_outfile}) does not exist." + echo "WARNING: Output variable (${rundir_outfile}) does not exist." fi } From 1f777187678510dc2c2d14bba9fbc87b86ea0266 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 16 Jul 2024 13:24:22 +0000 Subject: [PATCH 102/141] Replace cp with NCP The command cp has been replaced with the variable NCP in extractvars_tools. --- ush/extractvars_tools.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/extractvars_tools.sh b/ush/extractvars_tools.sh index 1ca2876ce6..87e0cd0671 100644 --- a/ush/extractvars_tools.sh +++ b/ush/extractvars_tools.sh @@ -48,7 +48,7 @@ copy_to_comout() { rundir_outfile=$1 #output data file generated in RUNDIR comout_dir=$2 #destination directory to which to copy the data file if [[ -f "${rundir_outfile}" ]];then - cp -pr "${rundir_outfile}" "${comout_dir}" + ${NCP} "${rundir_outfile}" "${comout_dir}" else echo "WARNING: Output variable (${rundir_outfile}) does not exist." fi From 89f5621ea0269d322b76046ec796eef9559629e6 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 16 Jul 2024 13:38:13 +0000 Subject: [PATCH 103/141] Replace NCP with cpfs --- ush/extractvars_tools.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/extractvars_tools.sh b/ush/extractvars_tools.sh index 87e0cd0671..4318451610 100644 --- a/ush/extractvars_tools.sh +++ b/ush/extractvars_tools.sh @@ -48,7 +48,7 @@ copy_to_comout() { rundir_outfile=$1 #output data file generated in RUNDIR comout_dir=$2 #destination directory to which to copy the data file if [[ -f "${rundir_outfile}" ]];then - ${NCP} "${rundir_outfile}" "${comout_dir}" + cpfs "${rundir_outfile}" "${comout_dir}" else echo "WARNING: Output variable (${rundir_outfile}) does not exist." fi From 2e4ca68fdd8c9b446f5261c4bb941617a5a5a224 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 16 Jul 2024 13:40:07 +0000 Subject: [PATCH 104/141] Remove source extractvars_tools from ush scripts The source extractvars_tools.sh has been removed from the ush scripts. Source extractvars_tools has been moved to the exscript. --- ush/ocnice_extractvars.sh | 1 - ush/wav_extractvars.sh | 1 - 2 files changed, 2 deletions(-) diff --git a/ush/ocnice_extractvars.sh b/ush/ocnice_extractvars.sh index 10a81166e0..17e80119c6 100755 --- a/ush/ocnice_extractvars.sh +++ b/ush/ocnice_extractvars.sh @@ -3,7 +3,6 @@ # Script: # source "${USHgfs}/preamble.sh" -source "${USHgfs}/extractvars_tools.sh" outdirpre=${1} varlist=${2} diff --git a/ush/wav_extractvars.sh b/ush/wav_extractvars.sh index 03196e37de..624996c83d 100755 --- a/ush/wav_extractvars.sh +++ b/ush/wav_extractvars.sh @@ -3,7 +3,6 @@ # Script: # source "${USHgfs}/preamble.sh" -source "${USHgfs}/extractvars_tools.sh" DATA=${1} From f8d24f2237b0c1564649225020d4cbfa9249ac57 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 16 Jul 2024 13:43:10 +0000 Subject: [PATCH 105/141] Rename wav_extractvars to wave_extractvars The wav_extractvars script has been renamed to wave_extractvars. --- scripts/exglobal_extractvars.sh | 2 +- ush/{wav_extractvars.sh => wave_extractvars.sh} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename ush/{wav_extractvars.sh => wave_extractvars.sh} (100%) diff --git a/scripts/exglobal_extractvars.sh b/scripts/exglobal_extractvars.sh index 7558d38d30..7c0a79e11e 100755 --- a/scripts/exglobal_extractvars.sh +++ b/scripts/exglobal_extractvars.sh @@ -11,7 +11,7 @@ export WGRIB2=${WGRIB2:-${wgrib2_ROOT}/bin/wgrib2} # Scripts used EXTRCTVARA=${EXTRCTVARA:-"${USHgfs}/atmos_extractvars.sh"} EXTRCTVARO=${EXTRCTVARO:-"${USHgfs}/ocnice_extractvars.sh"} -EXTRCTVARW=${EXTRCTVARW:-"${USHgfs}/wav_extractvars.sh"} +EXTRCTVARW=${EXTRCTVARW:-"${USHgfs}/wave_extractvars.sh"} #Define a job-specific variable for FHMAX_HF_GFS if (( FHMAX_GFS < FHMAX_HF_GFS )); then diff --git a/ush/wav_extractvars.sh b/ush/wave_extractvars.sh similarity index 100% rename from ush/wav_extractvars.sh rename to ush/wave_extractvars.sh From 0bfc12f707f8ffeb7f80f5e3cf5281b1cc440690 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 16 Jul 2024 14:08:06 +0000 Subject: [PATCH 106/141] Pass dcnt into daily_avg_atmos and add description The dcnt variable has been passed into the daily_avg_atmos function. A brief description of dcnt, vcnt and fcnt have also been added. --- ush/atmos_extractvars.sh | 6 +++--- ush/extractvars_tools.sh | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ush/atmos_extractvars.sh b/ush/atmos_extractvars.sh index 2eca318ebb..b206d7ea3f 100755 --- a/ush/atmos_extractvars.sh +++ b/ush/atmos_extractvars.sh @@ -4,8 +4,8 @@ # source "${USHgfs}/preamble.sh" -fcnt=1 -dcnt=1 +fcnt=1 # 1 is 1st quarter, 2 is 2nd quarter and 3 is 3rd quarter of the day +dcnt=1 # lead day subdata=${1} [[ -d "${subdata}" ]] || mkdir -p "${subdata}" @@ -82,7 +82,7 @@ for outtype in "f2d" "f3d"; do # shellcheck disable=SC2312 ${WGRIB2} "${infile2}" | grep -F -f "${varlist_d}" | ${WGRIB2} -i "${infile2}" -append -grib "${outfile}" if [[ ${fcnt} -eq 4 ]];then - daily_avg_atmos "${outfile}" + daily_avg_atmos "${outfile}" "${dcnt}" copy_to_comout "${davg_file}" "${COMOUT_RFCST_PROD_ATMOS}" fcnt=1 dcnt=$(( dcnt + 1 )) diff --git a/ush/extractvars_tools.sh b/ush/extractvars_tools.sh index 4318451610..2620fa91f8 100644 --- a/ush/extractvars_tools.sh +++ b/ush/extractvars_tools.sh @@ -30,9 +30,10 @@ check_atmos() { daily_avg_atmos() { outfile_p=$1 - fnd=$(printf "%2.2d" "${dcnt}") + dcnt_p=$2 + fnd=$(printf "%2.2d" "${dcnt_p}") davg_file=${outdirpre}/${RUN}.${cycle}.pgrb2.${outres}.24hr_avg.ldy${fnd} - vcnt=1 + vcnt=1 #count variables in varlist_d while read -r vari; do davgtmp=${subdata}/${RUN}.${cycle}.tmp.pgrb2.${outres}.ldy${fnd}.${vcnt} # shellcheck disable=SC2312 From 519922c1272d4c331f599d33701435be753a1d74 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 16 Jul 2024 14:20:32 +0000 Subject: [PATCH 107/141] Replace ${cycle} with t${cyc}z The variable ${cycle} has been replaced with t${cyc}z. --- ush/atmos_extractvars.sh | 6 +++--- ush/extractvars_tools.sh | 4 ++-- ush/ocnice_extractvars.sh | 12 ++++++------ ush/wave_extractvars.sh | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/ush/atmos_extractvars.sh b/ush/atmos_extractvars.sh index b206d7ea3f..4ff64e824d 100755 --- a/ush/atmos_extractvars.sh +++ b/ush/atmos_extractvars.sh @@ -45,9 +45,9 @@ for outtype in "f2d" "f3d"; do fi com_var="COMIN_ATMOS_GRIB_${outres}" - infile1="${!com_var}/${RUN}.${cycle}.pgrb2.${outres}.f${fnh}" - infile2="${!com_var}/${RUN}.${cycle}.pgrb2b.${outres}.f${fnh}" - outfile="${outdirpre}/${RUN}.${cycle}.pgrb2.${outres}.f${fnh}" + infile1="${!com_var}/${RUN}.t${cyc}z.pgrb2.${outres}.f${fnh}" + infile2="${!com_var}/${RUN}.t${cyc}z.pgrb2b.${outres}.f${fnh}" + outfile="${outdirpre}/${RUN}.t${cyc}z.pgrb2.${outres}.f${fnh}" rm -f "${outfile}" #remove outfile if it already exists before extraction requestedvars1="${subdata}/partial_parm1.txt" requestedvars2="${subdata}/partial_parm2.txt" diff --git a/ush/extractvars_tools.sh b/ush/extractvars_tools.sh index 2620fa91f8..0ad04b54ca 100644 --- a/ush/extractvars_tools.sh +++ b/ush/extractvars_tools.sh @@ -32,10 +32,10 @@ daily_avg_atmos() { outfile_p=$1 dcnt_p=$2 fnd=$(printf "%2.2d" "${dcnt_p}") - davg_file=${outdirpre}/${RUN}.${cycle}.pgrb2.${outres}.24hr_avg.ldy${fnd} + davg_file=${outdirpre}/${RUN}.t${cyc}z.pgrb2.${outres}.24hr_avg.ldy${fnd} vcnt=1 #count variables in varlist_d while read -r vari; do - davgtmp=${subdata}/${RUN}.${cycle}.tmp.pgrb2.${outres}.ldy${fnd}.${vcnt} + davgtmp=${subdata}/${RUN}.t${cyc}z.tmp.pgrb2.${outres}.ldy${fnd}.${vcnt} # shellcheck disable=SC2312 ${WGRIB2} "${outfile_p}" | grep "${vari}" | ${WGRIB2} -i "${outfile_p}" -fcst_ave 6hr "${davgtmp}" # shellcheck disable=SC2312 diff --git a/ush/ocnice_extractvars.sh b/ush/ocnice_extractvars.sh index 17e80119c6..0495af9d21 100755 --- a/ush/ocnice_extractvars.sh +++ b/ush/ocnice_extractvars.sh @@ -19,20 +19,20 @@ for (( nh = FHMIN_GFS; nh <= FHMAX_GFS; nh = nh + fhout_ocnice )); do if [[ "${dataformat}" == "grib2" ]];then if [[ ${component_name} == "ocn" ]];then - infile=${COMIN_OCEAN_GRIB}/${datares}/${RUN}.ocean.${cycle}.${datares}.f${fnh}.grib2 + infile=${COMIN_OCEAN_GRIB}/${datares}/${RUN}.ocean.t${cyc}z.${datares}.f${fnh}.grib2 elif [[ ${component_name} == "ice" ]];then - infile=${COMIN_ICE_GRIB}/${datares}/${RUN}.ice.${cycle}.${datares}.f${fnh}.grib2 + infile=${COMIN_ICE_GRIB}/${datares}/${RUN}.ice.t${cyc}z.${datares}.f${fnh}.grib2 fi - outfile=${outdirpre}/${RUN}.${component_name}.${cycle}.${datares}.f${fnh}.grib2 + outfile=${outdirpre}/${RUN}.${component_name}.t${cyc}z.${datares}.f${fnh}.grib2 fi if [[ "${dataformat}" == "netcdf" ]];then if [[ ${component_name} == "ocn" ]];then - infile=${COMIN_OCEAN_NETCDF}/${RUN}.ocean.${cycle}.${datares}.f${fnh}.nc + infile=${COMIN_OCEAN_NETCDF}/${RUN}.ocean.t${cyc}z.${datares}.f${fnh}.nc elif [[ ${component_name} == "ice" ]];then - infile=${COMIN_ICE_NETCDF}/${RUN}.ice.${cycle}.${datares}.f${fnh}.nc + infile=${COMIN_ICE_NETCDF}/${RUN}.ice.t${cyc}z.${datares}.f${fnh}.nc fi - outfile=${outdirpre}/${RUN}.${component_name}.${cycle}.${datares}.f${fnh}.nc + outfile=${outdirpre}/${RUN}.${component_name}.t${cyc}z.${datares}.f${fnh}.nc fi if [[ -f "${infile}" ]]; then #check if input file exists before extraction diff --git a/ush/wave_extractvars.sh b/ush/wave_extractvars.sh index 624996c83d..79ac508575 100755 --- a/ush/wave_extractvars.sh +++ b/ush/wave_extractvars.sh @@ -11,8 +11,8 @@ DATA=${1} for (( nh = FHOUT_WAV_NOSCRUB; nh <= FHMAX_WAV; nh = nh + FHOUT_WAV_NOSCRUB )); do fnh=$(printf "%3.3d" "${nh}") - infile=${COMIN_WAVE_GRID}/${RUN}wave.${cycle}.global.${wavres}.f${fnh}.grib2 - outfile=${DATA}/${RUN}wave.${cycle}.global.${wavres}.f${fnh}.grib2 + infile=${COMIN_WAVE_GRID}/${RUN}wave.t${cyc}z.global.${wavres}.f${fnh}.grib2 + outfile=${DATA}/${RUN}wave.t${cyc}z.global.${wavres}.f${fnh}.grib2 rm -f "${outfile}" #remove outfile if it already exists before extraction if [[ -f "${infile}" ]]; then #check if input file exists before extraction From 9d3ba90545481d821a459079658a1101ce8447a9 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 16 Jul 2024 14:40:08 +0000 Subject: [PATCH 108/141] Pass outres into daily_avg_atmos function The outres variable is passed into daily_avg_atmos function. --- ush/atmos_extractvars.sh | 2 +- ush/extractvars_tools.sh | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ush/atmos_extractvars.sh b/ush/atmos_extractvars.sh index 4ff64e824d..6bf3eef95f 100755 --- a/ush/atmos_extractvars.sh +++ b/ush/atmos_extractvars.sh @@ -82,7 +82,7 @@ for outtype in "f2d" "f3d"; do # shellcheck disable=SC2312 ${WGRIB2} "${infile2}" | grep -F -f "${varlist_d}" | ${WGRIB2} -i "${infile2}" -append -grib "${outfile}" if [[ ${fcnt} -eq 4 ]];then - daily_avg_atmos "${outfile}" "${dcnt}" + daily_avg_atmos "${outfile}" "${dcnt}" "${outres}" copy_to_comout "${davg_file}" "${COMOUT_RFCST_PROD_ATMOS}" fcnt=1 dcnt=$(( dcnt + 1 )) diff --git a/ush/extractvars_tools.sh b/ush/extractvars_tools.sh index 0ad04b54ca..4164eff4fa 100644 --- a/ush/extractvars_tools.sh +++ b/ush/extractvars_tools.sh @@ -31,11 +31,12 @@ check_atmos() { daily_avg_atmos() { outfile_p=$1 dcnt_p=$2 + outres_p=$3 fnd=$(printf "%2.2d" "${dcnt_p}") - davg_file=${outdirpre}/${RUN}.t${cyc}z.pgrb2.${outres}.24hr_avg.ldy${fnd} + davg_file=${outdirpre}/${RUN}.t${cyc}z.pgrb2.${outres_p}.24hr_avg.ldy${fnd} vcnt=1 #count variables in varlist_d while read -r vari; do - davgtmp=${subdata}/${RUN}.t${cyc}z.tmp.pgrb2.${outres}.ldy${fnd}.${vcnt} + davgtmp=${subdata}/${RUN}.t${cyc}z.tmp.pgrb2.${outres_p}.ldy${fnd}.${vcnt} # shellcheck disable=SC2312 ${WGRIB2} "${outfile_p}" | grep "${vari}" | ${WGRIB2} -i "${outfile_p}" -fcst_ave 6hr "${davgtmp}" # shellcheck disable=SC2312 From e64a64003df4899d64a8608d5b74edbffcacc59e Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 16 Jul 2024 14:47:14 +0000 Subject: [PATCH 109/141] Rename davgtmp The filename davgtmp has been renamed. --- ush/extractvars_tools.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/extractvars_tools.sh b/ush/extractvars_tools.sh index 4164eff4fa..6bf91f47e9 100644 --- a/ush/extractvars_tools.sh +++ b/ush/extractvars_tools.sh @@ -36,7 +36,7 @@ daily_avg_atmos() { davg_file=${outdirpre}/${RUN}.t${cyc}z.pgrb2.${outres_p}.24hr_avg.ldy${fnd} vcnt=1 #count variables in varlist_d while read -r vari; do - davgtmp=${subdata}/${RUN}.t${cyc}z.tmp.pgrb2.${outres_p}.ldy${fnd}.${vcnt} + davgtmp=${subdata}/atmos_tmp.ldy${fnd}.${vcnt} # shellcheck disable=SC2312 ${WGRIB2} "${outfile_p}" | grep "${vari}" | ${WGRIB2} -i "${outfile_p}" -fcst_ave 6hr "${davgtmp}" # shellcheck disable=SC2312 From f8bd19d33fba4cbcc7f196caff992141ba9219bc Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 16 Jul 2024 15:29:04 +0000 Subject: [PATCH 110/141] Simplify the declaring of COM variables The COM variables for ocean, ice and waves are declared all in one command. --- jobs/JGLOBAL_EXTRACTVARS | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/jobs/JGLOBAL_EXTRACTVARS b/jobs/JGLOBAL_EXTRACTVARS index 1c55c24bc5..4e84809766 100755 --- a/jobs/JGLOBAL_EXTRACTVARS +++ b/jobs/JGLOBAL_EXTRACTVARS @@ -10,20 +10,19 @@ for grid in '0p25' '0p50' '1p00'; do if [[ ! -d "${!prod_dir}" ]]; then mkdir -p "${!prod_dir}"; fi done -YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COMIN_OCEAN_HISTORY:COM_OCEAN_HISTORY_TMPL" -YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COMIN_OCEAN_GRIB:COM_OCEAN_GRIB_TMPL" -YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COMIN_OCEAN_NETCDF:COM_OCEAN_NETCDF_TMPL" -YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COMIN_ICE_HISTORY:COM_ICE_HISTORY_TMPL" -YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COMIN_ICE_GRIB:COM_ICE_GRIB_TMPL" -YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COMIN_ICE_NETCDF:COM_ICE_NETCDF_TMPL" - -YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COMIN_WAVE_GRID:COM_WAVE_GRID_TMPL" - -YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COMOUT_RFCST_PROD_ATMOS_F2D:COM_RFCST_PROD_ATMOS_F2D_TMPL" -YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COMOUT_RFCST_PROD_ATMOS_F3D:COM_RFCST_PROD_ATMOS_F3D_TMPL" -YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COMOUT_RFCST_PROD_OCN:COM_RFCST_PROD_OCN_TMPL" -YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COMOUT_RFCST_PROD_ICE:COM_RFCST_PROD_ICE_TMPL" -YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx "COMOUT_RFCST_PROD_WAV:COM_RFCST_PROD_WAV_TMPL" +YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx \ + "COMIN_OCEAN_HISTORY:COM_OCEAN_HISTORY_TMPL" \ + "COMIN_OCEAN_GRIB:COM_OCEAN_GRIB_TMPL" \ + "COMIN_OCEAN_NETCDF:COM_OCEAN_NETCDF_TMPL" \ + "COMIN_ICE_HISTORY:COM_ICE_HISTORY_TMPL" \ + "COMIN_ICE_GRIB:COM_ICE_GRIB_TMPL" \ + "COMIN_ICE_NETCDF:COM_ICE_NETCDF_TMPL" \ + "COMIN_WAVE_GRID:COM_WAVE_GRID_TMPL" \ + "COMOUT_RFCST_PROD_ATMOS_F2D:COM_RFCST_PROD_ATMOS_F2D_TMPL" \ + "COMOUT_RFCST_PROD_ATMOS_F3D:COM_RFCST_PROD_ATMOS_F3D_TMPL" \ + "COMOUT_RFCST_PROD_OCN:COM_RFCST_PROD_OCN_TMPL" \ + "COMOUT_RFCST_PROD_ICE:COM_RFCST_PROD_ICE_TMPL" \ + "COMOUT_RFCST_PROD_WAV:COM_RFCST_PROD_WAV_TMPL" if [[ ! -d "${COMOUT_RFCST_PROD_ATMOS_F2D}" ]]; then mkdir -p "${COMOUT_RFCST_PROD_ATMOS_F2D}"; fi if [[ ! -d "${COMOUT_RFCST_PROD_ATMOS_F3D}" ]]; then mkdir -p "${COMOUT_RFCST_PROD_ATMOS_F3D}"; fi From db4b43221c9bef36f6c9e263b17bb261266c68c0 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 16 Jul 2024 15:41:15 +0000 Subject: [PATCH 111/141] Rename FHOUT_WAV_NOSCRUB to FHOUT_WAV_EXTRACT --- parm/config/gefs/config.extractvars | 2 +- ush/wave_extractvars.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/config/gefs/config.extractvars b/parm/config/gefs/config.extractvars index dc3fefee13..ffc75e8ece 100644 --- a/parm/config/gefs/config.extractvars +++ b/parm/config/gefs/config.extractvars @@ -19,7 +19,7 @@ export ocnres="5p00" #Resolution of ocean products export iceres="5p00" #Resolution of ice products export wavres="5p00" #Resolution of wave products -export FHOUT_WAV_NOSCRUB=6 #Frequency of wave output to be saved on disk +export FHOUT_WAV_EXTRACT=6 #Frequency of wave output to be saved on disk #Paramater Tables used export varlist_2d=${varlist_2d:-"${PARMgfs}/product/gefs_shortparmlist_2d.parm"} #Parameter table for surface variables diff --git a/ush/wave_extractvars.sh b/ush/wave_extractvars.sh index 79ac508575..1a391407f7 100755 --- a/ush/wave_extractvars.sh +++ b/ush/wave_extractvars.sh @@ -8,7 +8,7 @@ DATA=${1} [[ -d "${DATA}" ]] || mkdir -p "${DATA}" -for (( nh = FHOUT_WAV_NOSCRUB; nh <= FHMAX_WAV; nh = nh + FHOUT_WAV_NOSCRUB )); do +for (( nh = FHOUT_WAV_EXTRACT; nh <= FHMAX_WAV; nh = nh + FHOUT_WAV_EXTRACT )); do fnh=$(printf "%3.3d" "${nh}") infile=${COMIN_WAVE_GRID}/${RUN}wave.t${cyc}z.global.${wavres}.f${fnh}.grib2 From cd1c5d63413a39d3a33eaed9711926c857f6249c Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 16 Jul 2024 16:15:14 +0000 Subject: [PATCH 112/141] Add check for FHOUT_WAV_EXTRACT A check has been added to make sure FHOUT_WAV is a factor of FHOUT_WAV_EXTRACT. If it is not, FHOUT_WAV_EXTRACT is set to FHOUT_WAV. Also added a space to some comments. --- scripts/exglobal_extractvars.sh | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/scripts/exglobal_extractvars.sh b/scripts/exglobal_extractvars.sh index 7c0a79e11e..a7552c29e2 100755 --- a/scripts/exglobal_extractvars.sh +++ b/scripts/exglobal_extractvars.sh @@ -13,14 +13,19 @@ EXTRCTVARA=${EXTRCTVARA:-"${USHgfs}/atmos_extractvars.sh"} EXTRCTVARO=${EXTRCTVARO:-"${USHgfs}/ocnice_extractvars.sh"} EXTRCTVARW=${EXTRCTVARW:-"${USHgfs}/wave_extractvars.sh"} -#Define a job-specific variable for FHMAX_HF_GFS +# Define a job-specific variable for FHMAX_HF_GFS if (( FHMAX_GFS < FHMAX_HF_GFS )); then export FHMAX_HF_EV=FHMAX_GFS else export FHMAX_HF_EV=FHMAX_HF_GFS fi -#Extract variables for atmosphere +# Set FHOUT_WAV_EXTRACT equal to FHOUT_WAV if FHOUT_WAV is not a factor of FHOUT_WAV_EXTRACT +if (( FHOUT_WAV_EXTRACT % FHOUT_WAV != 0 )); then + FHOUT_WAV_EXTRACT=${FHOUT_WAV} +fi + +# Extract variables for atmosphere if [[ "${DO_ATM}" == "YES" ]];then if [[ ! -d "${DATA}/mem${ENSMEM}_atmos" ]]; then mkdir -p "${DATA}/mem${ENSMEM}_atmos" @@ -28,7 +33,7 @@ if [[ "${DO_ATM}" == "YES" ]];then ${EXTRCTVARA} "${DATA}/mem${ENSMEM}_atmos" fi -#Extract variables for ocean +# Extract variables for ocean if [[ "${DO_OCN}" == "YES" ]];then export component_name="ocn" if [[ ! -d "${DATA}/mem${ENSMEM}_ocn" ]]; then @@ -45,7 +50,7 @@ if [[ "${DO_OCN}" == "YES" ]];then ${EXTRCTVARO} "${DATA}/mem${ENSMEM}_ocn" "${varlist_ocn}" "${ocn_dataformat}" "${ocnres}" "${compress_ocn}" "${FHOUT_OCN_GFS}" "${COMOUT_RFCST_PROD_OCN}" fi -#Extract variables for ice +# Extract variables for ice if [[ "${DO_ICE}" == "YES" ]];then export component_name="ice" if [[ ! -d "${DATA}/mem${ENSMEM}_ice" ]]; then @@ -62,7 +67,7 @@ if [[ "${DO_ICE}" == "YES" ]];then ${EXTRCTVARO} "${DATA}/mem${ENSMEM}_ice" "${varlist_ice}" "${ice_dataformat}" "${iceres}" "${compress_ice}" "${FHOUT_ICE_GFS}" "${COMOUT_RFCST_PROD_ICE}" fi -#Extract variables for wave +# Extract variables for wave if [[ "${DO_WAVE}" == "YES" ]];then export component_name="wav" if [[ ! -d "${DATA}/mem${ENSMEM}_wav" ]]; then From 16b546d5e5ae24f41553b1267ecd7268ce8a814c Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 16 Jul 2024 16:17:56 +0000 Subject: [PATCH 113/141] Remove WGRIB2 definition Remove an unneccesary WGRIB2 definition from exglobal_extractvars. --- scripts/exglobal_extractvars.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/exglobal_extractvars.sh b/scripts/exglobal_extractvars.sh index a7552c29e2..1fa8b2f156 100755 --- a/scripts/exglobal_extractvars.sh +++ b/scripts/exglobal_extractvars.sh @@ -5,9 +5,6 @@ source "${USHgfs}/preamble.sh" source "${USHgfs}/extractvars_tools.sh" -# Programs used -export WGRIB2=${WGRIB2:-${wgrib2_ROOT}/bin/wgrib2} - # Scripts used EXTRCTVARA=${EXTRCTVARA:-"${USHgfs}/atmos_extractvars.sh"} EXTRCTVARO=${EXTRCTVARO:-"${USHgfs}/ocnice_extractvars.sh"} From 0384906843ec646ea954be866883a9f3d1af9679 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 16 Jul 2024 16:32:37 +0000 Subject: [PATCH 114/141] Remove variable pass-throughs --- parm/config/gefs/config.extractvars | 16 ++++++++-------- scripts/exglobal_extractvars.sh | 6 +++--- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/parm/config/gefs/config.extractvars b/parm/config/gefs/config.extractvars index ffc75e8ece..5eb5394a28 100644 --- a/parm/config/gefs/config.extractvars +++ b/parm/config/gefs/config.extractvars @@ -22,13 +22,13 @@ export wavres="5p00" #Resolution of wave products export FHOUT_WAV_EXTRACT=6 #Frequency of wave output to be saved on disk #Paramater Tables used -export varlist_2d=${varlist_2d:-"${PARMgfs}/product/gefs_shortparmlist_2d.parm"} #Parameter table for surface variables -export varlist_3d=${varlist_3d:-"${PARMgfs}/product/gefs_shortparmlist_3d_h.parm"} #Parameter table for upper air instantaneous variables -export varlist_3d_d=${varlist_3d_d:-"${PARMgfs}/product/gefs_shortparmlist_3d_d.parm"} #Parameter table for upper air daily-averaged variables -export varlist_wav=${varlist_wav:-"${PARMgfs}/product/gefs_wav_shortparmlist.parm"} #Parameter table for wave variables -export varlist_ocn_netcdf=${varlist_ocn_netcdf:-"${PARMgfs}/product/gefs_ocn_shortparmlist.parm"} #Parameter table for ocean netcdf variables -export varlist_ice_netcdf=${varlist_ice_netcdf:-"${PARMgfs}/product/gefs_ice_shortparmlist.parm"} #Parameter table for ice netcdf variables -export varlist_ocn_grib2=${varlist_ocn_grib2:-"${PARMgfs}/product/gefs_ocn_shortparmlist_grib2.parm"} #Parameter table ocn grib2 variables -export varlist_ice_grib2=${varlist_ice_grib2:-"${PARMgfs}/product/gefs_ice_shortparmlist_grib2.parm"} #Parameter table ice grib2 variables +export varlist_2d="${PARMgfs}/product/gefs_shortparmlist_2d.parm" #Parameter table for surface variables +export varlist_3d="${PARMgfs}/product/gefs_shortparmlist_3d_h.parm" #Parameter table for upper air instantaneous variables +export varlist_3d_d="${PARMgfs}/product/gefs_shortparmlist_3d_d.parm" #Parameter table for upper air daily-averaged variables +export varlist_wav="${PARMgfs}/product/gefs_wav_shortparmlist.parm" #Parameter table for wave variables +export varlist_ocn_netcdf="${PARMgfs}/product/gefs_ocn_shortparmlist.parm" #Parameter table for ocean netcdf variables +export varlist_ice_netcdf="${PARMgfs}/product/gefs_ice_shortparmlist.parm" #Parameter table for ice netcdf variables +export varlist_ocn_grib2="${PARMgfs}/product/gefs_ocn_shortparmlist_grib2.parm" #Parameter table ocn grib2 variables +export varlist_ice_grib2="${PARMgfs}/product/gefs_ice_shortparmlist_grib2.parm" #Parameter table ice grib2 variables echo "END: config.extractvars" diff --git a/scripts/exglobal_extractvars.sh b/scripts/exglobal_extractvars.sh index 1fa8b2f156..710c11a41b 100755 --- a/scripts/exglobal_extractvars.sh +++ b/scripts/exglobal_extractvars.sh @@ -6,9 +6,9 @@ source "${USHgfs}/preamble.sh" source "${USHgfs}/extractvars_tools.sh" # Scripts used -EXTRCTVARA=${EXTRCTVARA:-"${USHgfs}/atmos_extractvars.sh"} -EXTRCTVARO=${EXTRCTVARO:-"${USHgfs}/ocnice_extractvars.sh"} -EXTRCTVARW=${EXTRCTVARW:-"${USHgfs}/wave_extractvars.sh"} +EXTRCTVARA="${USHgfs}/atmos_extractvars.sh" +EXTRCTVARO="${USHgfs}/ocnice_extractvars.sh" +EXTRCTVARW="${USHgfs}/wave_extractvars.sh" # Define a job-specific variable for FHMAX_HF_GFS if (( FHMAX_GFS < FHMAX_HF_GFS )); then From 7b523eb14aec0a54d6503f90a5a68979ea6de2b6 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 16 Jul 2024 16:59:00 +0000 Subject: [PATCH 115/141] Use variable subdata in ocnice/wave ush scripts The variable subdata replaces outdirpre and DATA in the ocnice and wave ush scripts. --- ush/ocnice_extractvars.sh | 8 ++++---- ush/wave_extractvars.sh | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ush/ocnice_extractvars.sh b/ush/ocnice_extractvars.sh index 0495af9d21..3c294bd4d4 100755 --- a/ush/ocnice_extractvars.sh +++ b/ush/ocnice_extractvars.sh @@ -4,7 +4,7 @@ # source "${USHgfs}/preamble.sh" -outdirpre=${1} +subdata=${1} varlist=${2} dataformat=${3} datares=${4} @@ -12,7 +12,7 @@ datacompress=${5} fhout_ocnice=${6} comout_rfcst_prod_ocnice=${7} -[[ -d "${outdirpre}" ]] || mkdir -p "${outdirpre}" +[[ -d "${subdata}" ]] || mkdir -p "${subdata}" for (( nh = FHMIN_GFS; nh <= FHMAX_GFS; nh = nh + fhout_ocnice )); do fnh=$(printf "%3.3d" "${nh}") @@ -23,7 +23,7 @@ for (( nh = FHMIN_GFS; nh <= FHMAX_GFS; nh = nh + fhout_ocnice )); do elif [[ ${component_name} == "ice" ]];then infile=${COMIN_ICE_GRIB}/${datares}/${RUN}.ice.t${cyc}z.${datares}.f${fnh}.grib2 fi - outfile=${outdirpre}/${RUN}.${component_name}.t${cyc}z.${datares}.f${fnh}.grib2 + outfile=${subdata}/${RUN}.${component_name}.t${cyc}z.${datares}.f${fnh}.grib2 fi if [[ "${dataformat}" == "netcdf" ]];then @@ -32,7 +32,7 @@ for (( nh = FHMIN_GFS; nh <= FHMAX_GFS; nh = nh + fhout_ocnice )); do elif [[ ${component_name} == "ice" ]];then infile=${COMIN_ICE_NETCDF}/${RUN}.ice.t${cyc}z.${datares}.f${fnh}.nc fi - outfile=${outdirpre}/${RUN}.${component_name}.t${cyc}z.${datares}.f${fnh}.nc + outfile=${subdata}/${RUN}.${component_name}.t${cyc}z.${datares}.f${fnh}.nc fi if [[ -f "${infile}" ]]; then #check if input file exists before extraction diff --git a/ush/wave_extractvars.sh b/ush/wave_extractvars.sh index 1a391407f7..15c90a36e2 100755 --- a/ush/wave_extractvars.sh +++ b/ush/wave_extractvars.sh @@ -4,15 +4,15 @@ # source "${USHgfs}/preamble.sh" -DATA=${1} +subdata=${1} -[[ -d "${DATA}" ]] || mkdir -p "${DATA}" +[[ -d "${subdata}" ]] || mkdir -p "${subdata}" for (( nh = FHOUT_WAV_EXTRACT; nh <= FHMAX_WAV; nh = nh + FHOUT_WAV_EXTRACT )); do fnh=$(printf "%3.3d" "${nh}") infile=${COMIN_WAVE_GRID}/${RUN}wave.t${cyc}z.global.${wavres}.f${fnh}.grib2 - outfile=${DATA}/${RUN}wave.t${cyc}z.global.${wavres}.f${fnh}.grib2 + outfile=${subdata}/${RUN}wave.t${cyc}z.global.${wavres}.f${fnh}.grib2 rm -f "${outfile}" #remove outfile if it already exists before extraction if [[ -f "${infile}" ]]; then #check if input file exists before extraction From d14c291a5fe0690ffd1b71388179622e8fdca4b7 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Tue, 16 Jul 2024 17:23:31 +0000 Subject: [PATCH 116/141] Remove unneccesary lines in exglobal_extractvars The mkdir portions have been removed from exglobal_extractvars. Also, "mem${ENSMEM}_" has been removed from directory names. --- scripts/exglobal_extractvars.sh | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/scripts/exglobal_extractvars.sh b/scripts/exglobal_extractvars.sh index 710c11a41b..e7e8dd1898 100755 --- a/scripts/exglobal_extractvars.sh +++ b/scripts/exglobal_extractvars.sh @@ -24,18 +24,12 @@ fi # Extract variables for atmosphere if [[ "${DO_ATM}" == "YES" ]];then - if [[ ! -d "${DATA}/mem${ENSMEM}_atmos" ]]; then - mkdir -p "${DATA}/mem${ENSMEM}_atmos" - fi - ${EXTRCTVARA} "${DATA}/mem${ENSMEM}_atmos" + ${EXTRCTVARA} "${DATA}/atmos" fi # Extract variables for ocean if [[ "${DO_OCN}" == "YES" ]];then export component_name="ocn" - if [[ ! -d "${DATA}/mem${ENSMEM}_ocn" ]]; then - mkdir -p "${DATA}/mem${ENSMEM}_ocn" - fi if [[ "${ocn_dataformat}" == "netcdf" ]]; then varlist_ocn=${varlist_ocn_netcdf} elif [[ "${ocn_dataformat}" == "grib2" ]]; then @@ -44,15 +38,12 @@ if [[ "${DO_OCN}" == "YES" ]];then echo "FATAL ERROR: Invalid ocean data format provided (${ocn_dataformat})" export err=1; err_chk fi - ${EXTRCTVARO} "${DATA}/mem${ENSMEM}_ocn" "${varlist_ocn}" "${ocn_dataformat}" "${ocnres}" "${compress_ocn}" "${FHOUT_OCN_GFS}" "${COMOUT_RFCST_PROD_OCN}" + ${EXTRCTVARO} "${DATA}/ocn" "${varlist_ocn}" "${ocn_dataformat}" "${ocnres}" "${compress_ocn}" "${FHOUT_OCN_GFS}" "${COMOUT_RFCST_PROD_OCN}" fi # Extract variables for ice if [[ "${DO_ICE}" == "YES" ]];then export component_name="ice" - if [[ ! -d "${DATA}/mem${ENSMEM}_ice" ]]; then - mkdir -p "${DATA}/mem${ENSMEM}_ice" - fi if [[ "${ice_dataformat}" == "netcdf" ]]; then varlist_ice=${varlist_ice_netcdf} elif [[ "${ice_dataformat}" == "grib2" ]]; then @@ -61,16 +52,13 @@ if [[ "${DO_ICE}" == "YES" ]];then echo "FATAL ERROR: Invalid ice data format provided (${ice_dataformat})" export err=1; err_chk fi - ${EXTRCTVARO} "${DATA}/mem${ENSMEM}_ice" "${varlist_ice}" "${ice_dataformat}" "${iceres}" "${compress_ice}" "${FHOUT_ICE_GFS}" "${COMOUT_RFCST_PROD_ICE}" + ${EXTRCTVARO} "${DATA}/ice" "${varlist_ice}" "${ice_dataformat}" "${iceres}" "${compress_ice}" "${FHOUT_ICE_GFS}" "${COMOUT_RFCST_PROD_ICE}" fi # Extract variables for wave if [[ "${DO_WAVE}" == "YES" ]];then export component_name="wav" - if [[ ! -d "${DATA}/mem${ENSMEM}_wav" ]]; then - mkdir -p "${DATA}/mem${ENSMEM}_wav" - fi - ${EXTRCTVARW} "${DATA}/mem${ENSMEM}_wav" + ${EXTRCTVARW} "${DATA}/wav" fi exit 0 From a7433a371a0090648a82401049a2cb088a670366 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 17 Jul 2024 12:49:02 +0000 Subject: [PATCH 117/141] Remove gen_parmlist and loop through infiles The gen_parmlist function has been removed and a for-loop has been created to loop through infile1 and inflile2. The variable check all occurs in the check_atmos function. --- ush/atmos_extractvars.sh | 30 +++++++++--------------------- ush/extractvars_tools.sh | 27 +++++++++------------------ 2 files changed, 18 insertions(+), 39 deletions(-) diff --git a/ush/atmos_extractvars.sh b/ush/atmos_extractvars.sh index 6bf3eef95f..8fe073e03f 100755 --- a/ush/atmos_extractvars.sh +++ b/ush/atmos_extractvars.sh @@ -49,29 +49,17 @@ for outtype in "f2d" "f3d"; do infile2="${!com_var}/${RUN}.t${cyc}z.pgrb2b.${outres}.f${fnh}" outfile="${outdirpre}/${RUN}.t${cyc}z.pgrb2.${outres}.f${fnh}" rm -f "${outfile}" #remove outfile if it already exists before extraction - requestedvars1="${subdata}/partial_parm1.txt" - requestedvars2="${subdata}/partial_parm2.txt" - rm -f "${requestedvars1}" - rm -f "${requestedvars2}" - if [[ ! -f "${requestedvars1}" ]]; then touch "${requestedvars1}"; fi - if [[ ! -f "${requestedvars2}" ]]; then touch "${requestedvars2}"; fi - if [[ -f "${infile1}" ]]; then #check if input file exists before extraction - gen_parmlist "${infile1}" "${requestedvars1}" "${varlist}" - # shellcheck disable=SC2312 - ${WGRIB2} "${infile1}" | grep -F -f "${requestedvars1}" | ${WGRIB2} -i "${infile1}" -append -grib "${outfile}">/dev/null - else - echo "WARNING: ${infile1} does not exist." - fi - if [[ -f "${infile2}" ]]; then #check if input file exists before extraction - gen_parmlist "${infile2}" "${requestedvars2}" "${varlist}" - # shellcheck disable=SC2312 - ${WGRIB2} "${infile2}" | grep -F -f "${requestedvars2}" | ${WGRIB2} -i "${infile2}" -append -grib "${outfile}">/dev/null - else - echo "WARNING: ${infile2} does not exist." - fi + for infile in "${infile1}" "${infile2}"; do + if [[ -f "${infile}" ]]; then # check if input file exists before extraction + # shellcheck disable=SC2312 + ${WGRIB2} "${infile}" | grep -F -f "${varlist}" | ${WGRIB2} -i "${infile}" -append -grib "${outfile}" > /dev/null + else + echo "WARNING: ${infile} does not exist." + fi + done - check_atmos "${requestedvars1}" "${requestedvars2}" "${varlist}" "${fnh}" + check_atmos "${infile1}" "${infile2}" "${varlist}" "${fnh}" copy_to_comout "${outfile}" "${COMOUT_RFCST_PROD_ATMOS}" #Compute daily average for a subset of variables diff --git a/ush/extractvars_tools.sh b/ush/extractvars_tools.sh index 6bf91f47e9..2ed831e67a 100644 --- a/ush/extractvars_tools.sh +++ b/ush/extractvars_tools.sh @@ -1,25 +1,17 @@ #! /usr/bin/env bash -gen_parmlist() { - infileg=$1 - requestedvar_in_file1=$2 - varlist=$3 - file_vars=$(${WGRIB2} "${infileg}") - while read -r vari; do - if [[ "${file_vars}" =~ ${vari} && -n "${vari}" ]]; then - echo "${vari}" >> "${requestedvar_in_file1}" - fi - done <"${varlist}" -} - check_atmos() { - requestedvar_in_file1=$1 - requestedvar_in_file2=$2 + infile1p=$1 + infile2p=$2 varlistl=$3 fnhl=$4 - requestedvar_in_allgrb2file="${subdata}/requestedvar_in_allgrb2file.txt" - rm -rvf "${requestedvar_in_allgrb2file}" - cat "${requestedvar_in_file1}" "${requestedvar_in_file2}" >> "${requestedvar_in_allgrb2file}" + requestedvar_in_allgrb2file="${subdata}/parmvarsingribfil.txt" + rm -f "${requestedvar_in_allgrb2file}" + touch "${requestedvar_in_allgrb2file}" + for infilep in "${infile1p}" "${infile2p}"; do + # shellcheck disable=SC2312 + ${WGRIB2} "${infilep}" | grep -F -f "${varlist}" >> ${requestedvar_in_allgrb2file} || true + done mapfile -t requestedvar_in_allgrb2file_arr < "${requestedvar_in_allgrb2file}" while read -r vari; do if [[ ! ${requestedvar_in_allgrb2file_arr[*]} =~ ${vari} ]] ;then @@ -56,7 +48,6 @@ copy_to_comout() { fi } -declare -xf gen_parmlist declare -xf check_atmos declare -xf daily_avg_atmos declare -xf copy_to_comout From 68f27142e0fdda06f1e902eb22ea4eca7f095f54 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 17 Jul 2024 12:55:57 +0000 Subject: [PATCH 118/141] Loop through infile when creating davg files A for-loop has been created to loop through the infiles when calculating the daily-averaged files. --- ush/atmos_extractvars.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ush/atmos_extractvars.sh b/ush/atmos_extractvars.sh index 8fe073e03f..b7a9fbf440 100755 --- a/ush/atmos_extractvars.sh +++ b/ush/atmos_extractvars.sh @@ -65,10 +65,10 @@ for outtype in "f2d" "f3d"; do #Compute daily average for a subset of variables if (( nh % 6 == 0 )) && (( nh != 0 )) && [[ "${outtype}" == "f3d" ]];then outfile=${subdata}/vartmp_raw_vari_ldy${dcnt}.grib2 - # shellcheck disable=SC2312 - ${WGRIB2} "${infile1}" | grep -F -f "${varlist_d}" | ${WGRIB2} -i "${infile1}" -append -grib "${outfile}" - # shellcheck disable=SC2312 - ${WGRIB2} "${infile2}" | grep -F -f "${varlist_d}" | ${WGRIB2} -i "${infile2}" -append -grib "${outfile}" + for infile in "${infile1}" "${infile2}"; do + # shellcheck disable=SC2312 + ${WGRIB2} "${infile}" | grep -F -f "${varlist_d}" | ${WGRIB2} -i "${infile}" -append -grib "${outfile}" + done if [[ ${fcnt} -eq 4 ]];then daily_avg_atmos "${outfile}" "${dcnt}" "${outres}" copy_to_comout "${davg_file}" "${COMOUT_RFCST_PROD_ATMOS}" From a2abff705a6b38b2e3870bc93cd7380ad389b0be Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 17 Jul 2024 13:02:43 +0000 Subject: [PATCH 119/141] Address shellcheck error and add comment A comment has been added to explain why an error should not be returned in check_atmos function. --- ush/extractvars_tools.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ush/extractvars_tools.sh b/ush/extractvars_tools.sh index 2ed831e67a..95abb7794d 100644 --- a/ush/extractvars_tools.sh +++ b/ush/extractvars_tools.sh @@ -9,8 +9,9 @@ check_atmos() { rm -f "${requestedvar_in_allgrb2file}" touch "${requestedvar_in_allgrb2file}" for infilep in "${infile1p}" "${infile2p}"; do + # It is permitted for an empty string to return if no parmlist vars are in infilep, therefore do not return exit 1 error # shellcheck disable=SC2312 - ${WGRIB2} "${infilep}" | grep -F -f "${varlist}" >> ${requestedvar_in_allgrb2file} || true + ${WGRIB2} "${infilep}" | grep -F -f "${varlist}" >> "${requestedvar_in_allgrb2file}" || true done mapfile -t requestedvar_in_allgrb2file_arr < "${requestedvar_in_allgrb2file}" while read -r vari; do From 148d8db5982595981a4dfd761db53d75690986c0 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 17 Jul 2024 13:31:44 +0000 Subject: [PATCH 120/141] Add file availability check in atmos script A file availability check has been added for the daily-averaged data file extraction. --- ush/atmos_extractvars.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ush/atmos_extractvars.sh b/ush/atmos_extractvars.sh index b7a9fbf440..af19ae5761 100755 --- a/ush/atmos_extractvars.sh +++ b/ush/atmos_extractvars.sh @@ -66,8 +66,12 @@ for outtype in "f2d" "f3d"; do if (( nh % 6 == 0 )) && (( nh != 0 )) && [[ "${outtype}" == "f3d" ]];then outfile=${subdata}/vartmp_raw_vari_ldy${dcnt}.grib2 for infile in "${infile1}" "${infile2}"; do - # shellcheck disable=SC2312 - ${WGRIB2} "${infile}" | grep -F -f "${varlist_d}" | ${WGRIB2} -i "${infile}" -append -grib "${outfile}" + if [[ -f "${infile}" ]]; then # check if input file exists before extraction + # shellcheck disable=SC2312 + ${WGRIB2} "${infile}" | grep -F -f "${varlist_d}" | ${WGRIB2} -i "${infile}" -append -grib "${outfile}" + else + echo "WARNING: ${infile} does not exist." + fi done if [[ ${fcnt} -eq 4 ]];then daily_avg_atmos "${outfile}" "${dcnt}" "${outres}" From ea0bd8c5ae18616b588f693a1e56c49b68b63d91 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 17 Jul 2024 18:11:07 +0000 Subject: [PATCH 121/141] Add option to extract levels for ocean 3d vars An option has been added to extract a range of levels from the 3d variables in the netcdf ocean products. --- parm/config/gefs/config.extractvars | 4 ++++ ush/ocnice_extractvars.sh | 7 +++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/parm/config/gefs/config.extractvars b/parm/config/gefs/config.extractvars index 5eb5394a28..4403f46260 100644 --- a/parm/config/gefs/config.extractvars +++ b/parm/config/gefs/config.extractvars @@ -19,6 +19,10 @@ export ocnres="5p00" #Resolution of ocean products export iceres="5p00" #Resolution of ice products export wavres="5p00" #Resolution of wave products +export depthvar_name="z_l" #name of depth variable in NetCDF ocean products +export zmin="0." #Minimum depth to extract from NetCDF ocean products +export zmax="300." #Maximum depth to extract from NetCDF ocean products + export FHOUT_WAV_EXTRACT=6 #Frequency of wave output to be saved on disk #Paramater Tables used diff --git a/ush/ocnice_extractvars.sh b/ush/ocnice_extractvars.sh index 3c294bd4d4..599ee25e3f 100755 --- a/ush/ocnice_extractvars.sh +++ b/ush/ocnice_extractvars.sh @@ -22,15 +22,18 @@ for (( nh = FHMIN_GFS; nh <= FHMAX_GFS; nh = nh + fhout_ocnice )); do infile=${COMIN_OCEAN_GRIB}/${datares}/${RUN}.ocean.t${cyc}z.${datares}.f${fnh}.grib2 elif [[ ${component_name} == "ice" ]];then infile=${COMIN_ICE_GRIB}/${datares}/${RUN}.ice.t${cyc}z.${datares}.f${fnh}.grib2 - fi + fi outfile=${subdata}/${RUN}.${component_name}.t${cyc}z.${datares}.f${fnh}.grib2 fi if [[ "${dataformat}" == "netcdf" ]];then if [[ ${component_name} == "ocn" ]];then infile=${COMIN_OCEAN_NETCDF}/${RUN}.ocean.t${cyc}z.${datares}.f${fnh}.nc + # For ocean products, add an argument to extract a subset of levels + otherargs="-d ${depthvar_name},${zmin},${zmax}" elif [[ ${component_name} == "ice" ]];then infile=${COMIN_ICE_NETCDF}/${RUN}.ice.t${cyc}z.${datares}.f${fnh}.nc + otherargs="" fi outfile=${subdata}/${RUN}.${component_name}.t${cyc}z.${datares}.f${fnh}.nc fi @@ -56,7 +59,7 @@ for (( nh = FHMIN_GFS; nh <= FHMAX_GFS; nh = nh + fhout_ocnice )); do echo "WARNING: No variables from parm file ${varlist} are available in netcdf file ${infile}." else ocnice_vars=${varsavailable::-1} - ncks -v "${ocnice_vars}" "${infile}" "${outfile}" + ncks -v "${ocnice_vars}" ${otherargs} "${infile}" "${outfile}" fi fi if [[ ${datacompress} -eq 1 ]];then From 142450a8e644151e6254f3c1c4bd85823b2b2cb3 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 17 Jul 2024 18:38:21 +0000 Subject: [PATCH 122/141] Fix shell check error --- ush/ocnice_extractvars.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ush/ocnice_extractvars.sh b/ush/ocnice_extractvars.sh index 599ee25e3f..f8152f2f0f 100755 --- a/ush/ocnice_extractvars.sh +++ b/ush/ocnice_extractvars.sh @@ -30,10 +30,10 @@ for (( nh = FHMIN_GFS; nh <= FHMAX_GFS; nh = nh + fhout_ocnice )); do if [[ ${component_name} == "ocn" ]];then infile=${COMIN_OCEAN_NETCDF}/${RUN}.ocean.t${cyc}z.${datares}.f${fnh}.nc # For ocean products, add an argument to extract a subset of levels - otherargs="-d ${depthvar_name},${zmin},${zmax}" + otherargs=(-d ${depthvar_name},${zmin},${zmax}) elif [[ ${component_name} == "ice" ]];then infile=${COMIN_ICE_NETCDF}/${RUN}.ice.t${cyc}z.${datares}.f${fnh}.nc - otherargs="" + otherargs=() fi outfile=${subdata}/${RUN}.${component_name}.t${cyc}z.${datares}.f${fnh}.nc fi @@ -59,7 +59,7 @@ for (( nh = FHMIN_GFS; nh <= FHMAX_GFS; nh = nh + fhout_ocnice )); do echo "WARNING: No variables from parm file ${varlist} are available in netcdf file ${infile}." else ocnice_vars=${varsavailable::-1} - ncks -v "${ocnice_vars}" ${otherargs} "${infile}" "${outfile}" + ncks -v "${ocnice_vars}" "${otherargs[@]}" "${infile}" "${outfile}" fi fi if [[ ${datacompress} -eq 1 ]];then From 380dfee5142a4a06e30bd9bfd1e4f65ae45f7527 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 17 Jul 2024 18:44:13 +0000 Subject: [PATCH 123/141] Add quotes --- ush/ocnice_extractvars.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/ocnice_extractvars.sh b/ush/ocnice_extractvars.sh index f8152f2f0f..7b7b28b0ca 100755 --- a/ush/ocnice_extractvars.sh +++ b/ush/ocnice_extractvars.sh @@ -30,7 +30,7 @@ for (( nh = FHMIN_GFS; nh <= FHMAX_GFS; nh = nh + fhout_ocnice )); do if [[ ${component_name} == "ocn" ]];then infile=${COMIN_OCEAN_NETCDF}/${RUN}.ocean.t${cyc}z.${datares}.f${fnh}.nc # For ocean products, add an argument to extract a subset of levels - otherargs=(-d ${depthvar_name},${zmin},${zmax}) + otherargs=(-d "${depthvar_name}","${zmin}","${zmax}") elif [[ ${component_name} == "ice" ]];then infile=${COMIN_ICE_NETCDF}/${RUN}.ice.t${cyc}z.${datares}.f${fnh}.nc otherargs=() From b293a65eee7fd1cdcad7147153cf54f45ddd669b Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 17 Jul 2024 18:51:59 +0000 Subject: [PATCH 124/141] Move commas inside double quotes --- ush/ocnice_extractvars.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/ocnice_extractvars.sh b/ush/ocnice_extractvars.sh index 7b7b28b0ca..4fc8cf9c90 100755 --- a/ush/ocnice_extractvars.sh +++ b/ush/ocnice_extractvars.sh @@ -30,7 +30,7 @@ for (( nh = FHMIN_GFS; nh <= FHMAX_GFS; nh = nh + fhout_ocnice )); do if [[ ${component_name} == "ocn" ]];then infile=${COMIN_OCEAN_NETCDF}/${RUN}.ocean.t${cyc}z.${datares}.f${fnh}.nc # For ocean products, add an argument to extract a subset of levels - otherargs=(-d "${depthvar_name}","${zmin}","${zmax}") + otherargs=(-d "${depthvar_name},""${zmin},""${zmax}") elif [[ ${component_name} == "ice" ]];then infile=${COMIN_ICE_NETCDF}/${RUN}.ice.t${cyc}z.${datares}.f${fnh}.nc otherargs=() From 43a0b10939f73e7054616560c4c0925a5ca36e47 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 17 Jul 2024 19:37:26 +0000 Subject: [PATCH 125/141] Remove grib2 support Grib2 support has been removed from the extractvars task. --- parm/config/gefs/config.extractvars | 5 -- .../product/gefs_ice_shortparmlist_grib2.parm | 2 - .../product/gefs_ocn_shortparmlist_grib2.parm | 2 - scripts/exglobal_extractvars.sh | 20 +----- ush/ocnice_extractvars.sh | 72 +++++++------------ 5 files changed, 29 insertions(+), 72 deletions(-) delete mode 100644 parm/product/gefs_ice_shortparmlist_grib2.parm delete mode 100644 parm/product/gefs_ocn_shortparmlist_grib2.parm diff --git a/parm/config/gefs/config.extractvars b/parm/config/gefs/config.extractvars index 4403f46260..592b8e1802 100644 --- a/parm/config/gefs/config.extractvars +++ b/parm/config/gefs/config.extractvars @@ -12,9 +12,6 @@ export COMPRSCMD=${COMPRSCMD:-bzip2} export compress_ocn=0 #1: compress extracted ocean product, 0: do not compress extracted ocean product export compress_ice=0 #1: compress extracted ice product, 0: do not compress extracted ice product -export ocn_dataformat="netcdf" -export ice_dataformat="netcdf" - export ocnres="5p00" #Resolution of ocean products export iceres="5p00" #Resolution of ice products export wavres="5p00" #Resolution of wave products @@ -32,7 +29,5 @@ export varlist_3d_d="${PARMgfs}/product/gefs_shortparmlist_3d_d.parm" #Parameter export varlist_wav="${PARMgfs}/product/gefs_wav_shortparmlist.parm" #Parameter table for wave variables export varlist_ocn_netcdf="${PARMgfs}/product/gefs_ocn_shortparmlist.parm" #Parameter table for ocean netcdf variables export varlist_ice_netcdf="${PARMgfs}/product/gefs_ice_shortparmlist.parm" #Parameter table for ice netcdf variables -export varlist_ocn_grib2="${PARMgfs}/product/gefs_ocn_shortparmlist_grib2.parm" #Parameter table ocn grib2 variables -export varlist_ice_grib2="${PARMgfs}/product/gefs_ice_shortparmlist_grib2.parm" #Parameter table ice grib2 variables echo "END: config.extractvars" diff --git a/parm/product/gefs_ice_shortparmlist_grib2.parm b/parm/product/gefs_ice_shortparmlist_grib2.parm deleted file mode 100644 index f0e519d24e..0000000000 --- a/parm/product/gefs_ice_shortparmlist_grib2.parm +++ /dev/null @@ -1,2 +0,0 @@ -:ICEC:surface: -:ICETMP:surface: diff --git a/parm/product/gefs_ocn_shortparmlist_grib2.parm b/parm/product/gefs_ocn_shortparmlist_grib2.parm deleted file mode 100644 index 958c4daa61..0000000000 --- a/parm/product/gefs_ocn_shortparmlist_grib2.parm +++ /dev/null @@ -1,2 +0,0 @@ -:WTMP:241 m below sea level: -:WTMP:267 m below sea level: diff --git a/scripts/exglobal_extractvars.sh b/scripts/exglobal_extractvars.sh index e7e8dd1898..29e0f5caf3 100755 --- a/scripts/exglobal_extractvars.sh +++ b/scripts/exglobal_extractvars.sh @@ -30,29 +30,13 @@ fi # Extract variables for ocean if [[ "${DO_OCN}" == "YES" ]];then export component_name="ocn" - if [[ "${ocn_dataformat}" == "netcdf" ]]; then - varlist_ocn=${varlist_ocn_netcdf} - elif [[ "${ocn_dataformat}" == "grib2" ]]; then - varlist_ocn=${varlist_ocn_grib2} - else - echo "FATAL ERROR: Invalid ocean data format provided (${ocn_dataformat})" - export err=1; err_chk - fi - ${EXTRCTVARO} "${DATA}/ocn" "${varlist_ocn}" "${ocn_dataformat}" "${ocnres}" "${compress_ocn}" "${FHOUT_OCN_GFS}" "${COMOUT_RFCST_PROD_OCN}" + ${EXTRCTVARO} "${DATA}/ocn" "${varlist_ocn_netcdf}" "${ocnres}" "${compress_ocn}" "${FHOUT_OCN_GFS}" "${COMOUT_RFCST_PROD_OCN}" fi # Extract variables for ice if [[ "${DO_ICE}" == "YES" ]];then export component_name="ice" - if [[ "${ice_dataformat}" == "netcdf" ]]; then - varlist_ice=${varlist_ice_netcdf} - elif [[ "${ice_dataformat}" == "grib2" ]]; then - varlist_ice=${varlist_ice_grib2} - else - echo "FATAL ERROR: Invalid ice data format provided (${ice_dataformat})" - export err=1; err_chk - fi - ${EXTRCTVARO} "${DATA}/ice" "${varlist_ice}" "${ice_dataformat}" "${iceres}" "${compress_ice}" "${FHOUT_ICE_GFS}" "${COMOUT_RFCST_PROD_ICE}" + ${EXTRCTVARO} "${DATA}/ice" "${varlist_ice_netcdf}" "${iceres}" "${compress_ice}" "${FHOUT_ICE_GFS}" "${COMOUT_RFCST_PROD_ICE}" fi # Extract variables for wave diff --git a/ush/ocnice_extractvars.sh b/ush/ocnice_extractvars.sh index 4fc8cf9c90..916208096d 100755 --- a/ush/ocnice_extractvars.sh +++ b/ush/ocnice_extractvars.sh @@ -6,62 +6,44 @@ source "${USHgfs}/preamble.sh" subdata=${1} varlist=${2} -dataformat=${3} -datares=${4} -datacompress=${5} -fhout_ocnice=${6} -comout_rfcst_prod_ocnice=${7} +datares=${3} +datacompress=${4} +fhout_ocnice=${5} +comout_rfcst_prod_ocnice=${6} [[ -d "${subdata}" ]] || mkdir -p "${subdata}" for (( nh = FHMIN_GFS; nh <= FHMAX_GFS; nh = nh + fhout_ocnice )); do fnh=$(printf "%3.3d" "${nh}") - if [[ "${dataformat}" == "grib2" ]];then - if [[ ${component_name} == "ocn" ]];then - infile=${COMIN_OCEAN_GRIB}/${datares}/${RUN}.ocean.t${cyc}z.${datares}.f${fnh}.grib2 - elif [[ ${component_name} == "ice" ]];then - infile=${COMIN_ICE_GRIB}/${datares}/${RUN}.ice.t${cyc}z.${datares}.f${fnh}.grib2 - fi - outfile=${subdata}/${RUN}.${component_name}.t${cyc}z.${datares}.f${fnh}.grib2 - fi - - if [[ "${dataformat}" == "netcdf" ]];then - if [[ ${component_name} == "ocn" ]];then - infile=${COMIN_OCEAN_NETCDF}/${RUN}.ocean.t${cyc}z.${datares}.f${fnh}.nc - # For ocean products, add an argument to extract a subset of levels - otherargs=(-d "${depthvar_name},""${zmin},""${zmax}") - elif [[ ${component_name} == "ice" ]];then - infile=${COMIN_ICE_NETCDF}/${RUN}.ice.t${cyc}z.${datares}.f${fnh}.nc - otherargs=() - fi - outfile=${subdata}/${RUN}.${component_name}.t${cyc}z.${datares}.f${fnh}.nc + if [[ ${component_name} == "ocn" ]];then + infile=${COMIN_OCEAN_NETCDF}/${RUN}.ocean.t${cyc}z.${datares}.f${fnh}.nc + # For ocean products, add an argument to extract a subset of levels + otherargs=(-d "${depthvar_name},""${zmin},""${zmax}") + elif [[ ${component_name} == "ice" ]];then + infile=${COMIN_ICE_NETCDF}/${RUN}.ice.t${cyc}z.${datares}.f${fnh}.nc + otherargs=() fi + outfile=${subdata}/${RUN}.${component_name}.t${cyc}z.${datares}.f${fnh}.nc if [[ -f "${infile}" ]]; then #check if input file exists before extraction - if [[ "${dataformat}" == "grib2" ]];then - # shellcheck disable=SC2312 - ${WGRIB2} "${infile}" | grep -F -f "${varlist}" | ${WGRIB2} -i "${infile}" -append -grib "${outfile}">/dev/null - fi - if [[ "${dataformat}" == "netcdf" ]];then - varsrequested=$(paste -s "${varlist}") - varsinfile=$(cdo -showname "${infile}") - varsavailable="" - for i in ${varsrequested}; do - #check if variable from parm file is available in netcdf file. If variable is not in netcdf file, do not try to extract that variable. - if [[ ${varsinfile} == *"${i}"* ]]; then - varsavailable+="${i}," - else - echo "WARNING: ${i} is not available in ${infile}." - fi - done - if [[ -z "${varsavailable}" ]];then - echo "WARNING: No variables from parm file ${varlist} are available in netcdf file ${infile}." + varsrequested=$(paste -s "${varlist}") + varsinfile=$(cdo -showname "${infile}") + varsavailable="" + for i in ${varsrequested}; do + #check if variable from parm file is available in netcdf file. If variable is not in netcdf file, do not try to extract that variable. + if [[ ${varsinfile} == *"${i}"* ]]; then + varsavailable+="${i}," else - ocnice_vars=${varsavailable::-1} - ncks -v "${ocnice_vars}" "${otherargs[@]}" "${infile}" "${outfile}" + echo "WARNING: ${i} is not available in ${infile}." fi - fi + done + if [[ -z "${varsavailable}" ]];then + echo "WARNING: No variables from parm file ${varlist} are available in netcdf file ${infile}." + else + ocnice_vars=${varsavailable::-1} + ncks -v "${ocnice_vars}" "${otherargs[@]}" "${infile}" "${outfile}" + fi if [[ ${datacompress} -eq 1 ]];then ${COMPRSCMD} "${outfile}" copy_to_comout "${outfile}.bz2" "${comout_rfcst_prod_ocnice}" From 7bd4b676c86dea09333c52ecb6374d1b5a1466f1 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Wed, 17 Jul 2024 20:08:45 +0000 Subject: [PATCH 126/141] Add fatal error if data file does not exist A fatal error will occur if an output data file does not exist. --- ush/extractvars_tools.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ush/extractvars_tools.sh b/ush/extractvars_tools.sh index 95abb7794d..374d63aa1c 100644 --- a/ush/extractvars_tools.sh +++ b/ush/extractvars_tools.sh @@ -45,7 +45,8 @@ copy_to_comout() { if [[ -f "${rundir_outfile}" ]];then cpfs "${rundir_outfile}" "${comout_dir}" else - echo "WARNING: Output variable (${rundir_outfile}) does not exist." + echo "FATAL ERROR: Output file (${rundir_outfile}) does not exist." + export err=1; err_chk fi } From c7e5d08c8e90e2c59986aefc80e2534d7124470d Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Thu, 18 Jul 2024 11:43:31 +0000 Subject: [PATCH 127/141] Rename FHMAX_HF_EV to FHMAX_HF_GFS The job-specific variable FHMAX_HF_EV has been renamed to FHMAX_HF_GFS according to a reviewer's suggestion. --- scripts/exglobal_extractvars.sh | 6 +++--- ush/atmos_extractvars.sh | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/exglobal_extractvars.sh b/scripts/exglobal_extractvars.sh index 29e0f5caf3..b25f632ba4 100755 --- a/scripts/exglobal_extractvars.sh +++ b/scripts/exglobal_extractvars.sh @@ -10,11 +10,11 @@ EXTRCTVARA="${USHgfs}/atmos_extractvars.sh" EXTRCTVARO="${USHgfs}/ocnice_extractvars.sh" EXTRCTVARW="${USHgfs}/wave_extractvars.sh" -# Define a job-specific variable for FHMAX_HF_GFS +# Set FHMAX_HF_GFS equal to FHMAX_GFS if FHMAX_HF_GFS is greater than FHMAX_GFS if (( FHMAX_GFS < FHMAX_HF_GFS )); then - export FHMAX_HF_EV=FHMAX_GFS + export FHMAX_HF_GFS=FHMAX_GFS else - export FHMAX_HF_EV=FHMAX_HF_GFS + export FHMAX_HF_GFS=FHMAX_HF_GFS fi # Set FHOUT_WAV_EXTRACT equal to FHOUT_WAV if FHOUT_WAV is not a factor of FHOUT_WAV_EXTRACT diff --git a/ush/atmos_extractvars.sh b/ush/atmos_extractvars.sh index af19ae5761..0a746b1e36 100755 --- a/ush/atmos_extractvars.sh +++ b/ush/atmos_extractvars.sh @@ -29,7 +29,7 @@ for outtype in "f2d" "f3d"; do fnh=$(printf "%3.3d" "${nh}") if [[ "${outtype}" == "f2d" ]];then - if [[ ${nh} -le ${FHMAX_HF_EV} ]];then + if [[ ${nh} -le ${FHMAX_HF_GFS} ]];then outres="0p25" else outres="0p50" @@ -38,7 +38,7 @@ for outtype in "f2d" "f3d"; do outres="1p00" fi - if (( nh < FHMAX_HF_EV )); then + if (( nh < FHMAX_HF_GFS )); then outfreq=${FHOUT_HF_GFS} else outfreq=${FHOUT_GFS} From 16425cbb02b127b6b7508d0a8c6139bb7877176b Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Thu, 18 Jul 2024 13:02:38 +0000 Subject: [PATCH 128/141] Change output directory to ARCDIR The output directory for extractvars has been moved from COMOUT to ARCDIR. --- jobs/JGLOBAL_EXTRACTVARS | 13 +------------ parm/config/gefs/config.extractvars | 14 ++++++++++++++ parm/config/gfs/config.com | 6 ------ scripts/exglobal_extractvars.sh | 4 ++-- ush/atmos_extractvars.sh | 8 ++++---- ush/wave_extractvars.sh | 2 +- 6 files changed, 22 insertions(+), 25 deletions(-) diff --git a/jobs/JGLOBAL_EXTRACTVARS b/jobs/JGLOBAL_EXTRACTVARS index 4e84809766..c72bf48780 100755 --- a/jobs/JGLOBAL_EXTRACTVARS +++ b/jobs/JGLOBAL_EXTRACTVARS @@ -17,18 +17,7 @@ YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx \ "COMIN_ICE_HISTORY:COM_ICE_HISTORY_TMPL" \ "COMIN_ICE_GRIB:COM_ICE_GRIB_TMPL" \ "COMIN_ICE_NETCDF:COM_ICE_NETCDF_TMPL" \ - "COMIN_WAVE_GRID:COM_WAVE_GRID_TMPL" \ - "COMOUT_RFCST_PROD_ATMOS_F2D:COM_RFCST_PROD_ATMOS_F2D_TMPL" \ - "COMOUT_RFCST_PROD_ATMOS_F3D:COM_RFCST_PROD_ATMOS_F3D_TMPL" \ - "COMOUT_RFCST_PROD_OCN:COM_RFCST_PROD_OCN_TMPL" \ - "COMOUT_RFCST_PROD_ICE:COM_RFCST_PROD_ICE_TMPL" \ - "COMOUT_RFCST_PROD_WAV:COM_RFCST_PROD_WAV_TMPL" - -if [[ ! -d "${COMOUT_RFCST_PROD_ATMOS_F2D}" ]]; then mkdir -p "${COMOUT_RFCST_PROD_ATMOS_F2D}"; fi -if [[ ! -d "${COMOUT_RFCST_PROD_ATMOS_F3D}" ]]; then mkdir -p "${COMOUT_RFCST_PROD_ATMOS_F3D}"; fi -if [[ ! -d "${COMOUT_RFCST_PROD_OCN}" ]]; then mkdir -p "${COMOUT_RFCST_PROD_OCN}"; fi -if [[ ! -d "${COMOUT_RFCST_PROD_WAV}" ]]; then mkdir -p "${COMOUT_RFCST_PROD_WAV}"; fi -if [[ ! -d "${COMOUT_RFCST_PROD_ICE}" ]]; then mkdir -p "${COMOUT_RFCST_PROD_ICE}"; fi + "COMIN_WAVE_GRID:COM_WAVE_GRID_TMPL" # Execute the Script "${SCRgfs}/exglobal_extractvars.sh" diff --git a/parm/config/gefs/config.extractvars b/parm/config/gefs/config.extractvars index 592b8e1802..7f113104ee 100644 --- a/parm/config/gefs/config.extractvars +++ b/parm/config/gefs/config.extractvars @@ -30,4 +30,18 @@ export varlist_wav="${PARMgfs}/product/gefs_wav_shortparmlist.parm" #Parameter t export varlist_ocn_netcdf="${PARMgfs}/product/gefs_ocn_shortparmlist.parm" #Parameter table for ocean netcdf variables export varlist_ice_netcdf="${PARMgfs}/product/gefs_ice_shortparmlist.parm" #Parameter table for ice netcdf variables +#Directory to save extracted variables +export ARC_RFCST_PROD="${ARCDIR}/rfcst/${PDY:0:4}/${PDY:0:6}/${PDY:0:8}/mem${ENSMEM}" +export ARC_RFCST_PROD_ATMOS_F2D="${ARC_RFCST_PROD}/atmos/f2d" +export ARC_RFCST_PROD_ATMOS_F3D="${ARC_RFCST_PROD}/atmos/f3d" +export ARC_RFCST_PROD_OCN="${ARC_RFCST_PROD}/ocn" +export ARC_RFCST_PROD_ICE="${ARC_RFCST_PROD}/ice" +export ARC_RFCST_PROD_WAV="${ARC_RFCST_PROD}/wav" + +if [[ ! -d "${ARC_RFCST_PROD_ATMOS_F2D}" ]]; then mkdir -p "${ARC_RFCST_PROD_ATMOS_F2D}"; fi +if [[ ! -d "${ARC_RFCST_PROD_ATMOS_F3D}" ]]; then mkdir -p "${ARC_RFCST_PROD_ATMOS_F3D}"; fi +if [[ ! -d "${ARC_RFCST_PROD_OCN}" ]]; then mkdir -p "${ARC_RFCST_PROD_OCN}"; fi +if [[ ! -d "${ARC_RFCST_PROD_WAV}" ]]; then mkdir -p "${ARC_RFCST_PROD_WAV}"; fi +if [[ ! -d "${ARC_RFCST_PROD_ICE}" ]]; then mkdir -p "${ARC_RFCST_PROD_ICE}"; fi + echo "END: config.extractvars" diff --git a/parm/config/gfs/config.com b/parm/config/gfs/config.com index 01d23c6df1..ec867e64ba 100644 --- a/parm/config/gfs/config.com +++ b/parm/config/gfs/config.com @@ -94,12 +94,6 @@ declare -rx COM_ICE_NETCDF_TMPL=${COM_BASE}'/products/ice/netcdf' declare -rx COM_ICE_GRIB_TMPL=${COM_BASE}'/products/ice/grib2' declare -rx COM_ICE_GRIB_GRID_TMPL=${COM_ICE_GRIB_TMPL}'/${GRID}' -declare -rx COM_RFCST_PROD_ATMOS_F2D_TMPL=${COM_BASE}'/products/rfcst/atmos/f2d' -declare -rx COM_RFCST_PROD_ATMOS_F3D_TMPL=${COM_BASE}'/products/rfcst/atmos/f3d' -declare -rx COM_RFCST_PROD_OCN_TMPL=${COM_BASE}'/products/rfcst/ocn' -declare -rx COM_RFCST_PROD_ICE_TMPL=${COM_BASE}'/products/rfcst/ice' -declare -rx COM_RFCST_PROD_WAV_TMPL=${COM_BASE}'/products/rfcst/wav' - declare -rx COM_CHEM_HISTORY_TMPL=${COM_BASE}'/model_data/chem/history' declare -rx COM_CHEM_ANALYSIS_TMPL=${COM_BASE}'/analysis/chem' diff --git a/scripts/exglobal_extractvars.sh b/scripts/exglobal_extractvars.sh index b25f632ba4..6e65bc4d4a 100755 --- a/scripts/exglobal_extractvars.sh +++ b/scripts/exglobal_extractvars.sh @@ -30,13 +30,13 @@ fi # Extract variables for ocean if [[ "${DO_OCN}" == "YES" ]];then export component_name="ocn" - ${EXTRCTVARO} "${DATA}/ocn" "${varlist_ocn_netcdf}" "${ocnres}" "${compress_ocn}" "${FHOUT_OCN_GFS}" "${COMOUT_RFCST_PROD_OCN}" + ${EXTRCTVARO} "${DATA}/ocn" "${varlist_ocn_netcdf}" "${ocnres}" "${compress_ocn}" "${FHOUT_OCN_GFS}" "${ARC_RFCST_PROD_OCN}" fi # Extract variables for ice if [[ "${DO_ICE}" == "YES" ]];then export component_name="ice" - ${EXTRCTVARO} "${DATA}/ice" "${varlist_ice_netcdf}" "${iceres}" "${compress_ice}" "${FHOUT_ICE_GFS}" "${COMOUT_RFCST_PROD_ICE}" + ${EXTRCTVARO} "${DATA}/ice" "${varlist_ice_netcdf}" "${iceres}" "${compress_ice}" "${FHOUT_ICE_GFS}" "${ARC_RFCST_PROD_ICE}" fi # Extract variables for wave diff --git a/ush/atmos_extractvars.sh b/ush/atmos_extractvars.sh index 0a746b1e36..032852a091 100755 --- a/ush/atmos_extractvars.sh +++ b/ush/atmos_extractvars.sh @@ -14,11 +14,11 @@ for outtype in "f2d" "f3d"; do if [[ "${outtype}" == "f2d" ]];then varlist=${varlist_2d} - COMOUT_RFCST_PROD_ATMOS="${COMOUT_RFCST_PROD_ATMOS_F2D}" + ARC_RFCST_PROD_ATMOS="${ARC_RFCST_PROD_ATMOS_F2D}" elif [[ "${outtype}" == "f3d" ]];then varlist=${varlist_3d} varlist_d=${varlist_3d_d} - COMOUT_RFCST_PROD_ATMOS="${COMOUT_RFCST_PROD_ATMOS_F3D}" + ARC_RFCST_PROD_ATMOS="${ARC_RFCST_PROD_ATMOS_F3D}" fi outdirpre="${subdata}/${outtype}" @@ -60,7 +60,7 @@ for outtype in "f2d" "f3d"; do done check_atmos "${infile1}" "${infile2}" "${varlist}" "${fnh}" - copy_to_comout "${outfile}" "${COMOUT_RFCST_PROD_ATMOS}" + copy_to_comout "${outfile}" "${ARC_RFCST_PROD_ATMOS}" #Compute daily average for a subset of variables if (( nh % 6 == 0 )) && (( nh != 0 )) && [[ "${outtype}" == "f3d" ]];then @@ -75,7 +75,7 @@ for outtype in "f2d" "f3d"; do done if [[ ${fcnt} -eq 4 ]];then daily_avg_atmos "${outfile}" "${dcnt}" "${outres}" - copy_to_comout "${davg_file}" "${COMOUT_RFCST_PROD_ATMOS}" + copy_to_comout "${davg_file}" "${ARC_RFCST_PROD_ATMOS}" fcnt=1 dcnt=$(( dcnt + 1 )) else diff --git a/ush/wave_extractvars.sh b/ush/wave_extractvars.sh index 15c90a36e2..c8d00cdd49 100755 --- a/ush/wave_extractvars.sh +++ b/ush/wave_extractvars.sh @@ -21,7 +21,7 @@ for (( nh = FHOUT_WAV_EXTRACT; nh <= FHMAX_WAV; nh = nh + FHOUT_WAV_EXTRACT )); else echo "WARNING: ${infile} does not exist." fi - copy_to_comout "${outfile}" "${COMOUT_RFCST_PROD_WAV}" + copy_to_comout "${outfile}" "${ARC_RFCST_PROD_WAV}" done #fhr exit 0 From bdb429c6715605a232b420dcaa55d818700690df Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Thu, 18 Jul 2024 19:17:29 +0000 Subject: [PATCH 129/141] Move mkdirs for arcdir to J-JOB script The arcdir directories are being created in the JGLOBAL_EXTRACTVARS instead of config.extractvars. Also, directories for a specific component will only be made if that component is set to run. --- jobs/JGLOBAL_EXTRACTVARS | 14 ++++++++++++++ parm/config/gefs/config.extractvars | 6 ------ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/jobs/JGLOBAL_EXTRACTVARS b/jobs/JGLOBAL_EXTRACTVARS index c72bf48780..137429a59e 100755 --- a/jobs/JGLOBAL_EXTRACTVARS +++ b/jobs/JGLOBAL_EXTRACTVARS @@ -19,6 +19,20 @@ YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx \ "COMIN_ICE_NETCDF:COM_ICE_NETCDF_TMPL" \ "COMIN_WAVE_GRID:COM_WAVE_GRID_TMPL" +if [[ "${DO_ATM}" == "YES" ]];then + if [[ ! -d "${ARC_RFCST_PROD_ATMOS_F2D}" ]]; then mkdir -p "${ARC_RFCST_PROD_ATMOS_F2D}"; fi + if [[ ! -d "${ARC_RFCST_PROD_ATMOS_F3D}" ]]; then mkdir -p "${ARC_RFCST_PROD_ATMOS_F3D}"; fi +fi +if [[ "${DO_OCN}" == "YES" ]];then + if [[ ! -d "${ARC_RFCST_PROD_OCN}" ]]; then mkdir -p "${ARC_RFCST_PROD_OCN}"; fi +fi +if [[ "${DO_ICE}" == "YES" ]];then + if [[ ! -d "${ARC_RFCST_PROD_ICE}" ]]; then mkdir -p "${ARC_RFCST_PROD_ICE}"; fi +fi +if [[ "${DO_WAVE}" == "YES" ]];then + if [[ ! -d "${ARC_RFCST_PROD_WAV}" ]]; then mkdir -p "${ARC_RFCST_PROD_WAV}"; fi +fi + # Execute the Script "${SCRgfs}/exglobal_extractvars.sh" status=$? diff --git a/parm/config/gefs/config.extractvars b/parm/config/gefs/config.extractvars index 7f113104ee..8fce4a227f 100644 --- a/parm/config/gefs/config.extractvars +++ b/parm/config/gefs/config.extractvars @@ -38,10 +38,4 @@ export ARC_RFCST_PROD_OCN="${ARC_RFCST_PROD}/ocn" export ARC_RFCST_PROD_ICE="${ARC_RFCST_PROD}/ice" export ARC_RFCST_PROD_WAV="${ARC_RFCST_PROD}/wav" -if [[ ! -d "${ARC_RFCST_PROD_ATMOS_F2D}" ]]; then mkdir -p "${ARC_RFCST_PROD_ATMOS_F2D}"; fi -if [[ ! -d "${ARC_RFCST_PROD_ATMOS_F3D}" ]]; then mkdir -p "${ARC_RFCST_PROD_ATMOS_F3D}"; fi -if [[ ! -d "${ARC_RFCST_PROD_OCN}" ]]; then mkdir -p "${ARC_RFCST_PROD_OCN}"; fi -if [[ ! -d "${ARC_RFCST_PROD_WAV}" ]]; then mkdir -p "${ARC_RFCST_PROD_WAV}"; fi -if [[ ! -d "${ARC_RFCST_PROD_ICE}" ]]; then mkdir -p "${ARC_RFCST_PROD_ICE}"; fi - echo "END: config.extractvars" From 1add8e1c98a61d56180685241d107e3be2a5dc12 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Thu, 18 Jul 2024 19:58:09 +0000 Subject: [PATCH 130/141] Add brief documentation to scripts and functions Brief documentation has been added to the extractvars scripts and functions. --- scripts/exglobal_extractvars.sh | 11 +++++++++-- ush/atmos_extractvars.sh | 11 +++++++++-- ush/extractvars_tools.sh | 6 ++++++ ush/ocnice_extractvars.sh | 12 ++++++++++-- ush/wave_extractvars.sh | 11 +++++++++-- 5 files changed, 43 insertions(+), 8 deletions(-) diff --git a/scripts/exglobal_extractvars.sh b/scripts/exglobal_extractvars.sh index 6e65bc4d4a..d225ab75a2 100755 --- a/scripts/exglobal_extractvars.sh +++ b/scripts/exglobal_extractvars.sh @@ -1,7 +1,14 @@ #! /usr/bin/env bash + ################################################################################ -# Script: -# +## UNIX Script Documentation Block +## Script name: exglobal_extractvars.sh +## Script description: Extracts variables from atmosphere, ocean, ice and wave +## products and saves these variables in arcdir +####################### +# Main body starts here +####################### + source "${USHgfs}/preamble.sh" source "${USHgfs}/extractvars_tools.sh" diff --git a/ush/atmos_extractvars.sh b/ush/atmos_extractvars.sh index 032852a091..210af44eb5 100755 --- a/ush/atmos_extractvars.sh +++ b/ush/atmos_extractvars.sh @@ -1,7 +1,14 @@ #! /usr/bin/env bash + ################################################################################ -# Script: -# +## UNIX Script Documentation Block +## Script name: atmos_extractvars.sh +## Script description: Extracts and calculates 24-hr averages of variables +## from atmosphere products and saves these variables in arcdir +####################### +# Main body starts here +####################### + source "${USHgfs}/preamble.sh" fcnt=1 # 1 is 1st quarter, 2 is 2nd quarter and 3 is 3rd quarter of the day diff --git a/ush/extractvars_tools.sh b/ush/extractvars_tools.sh index 374d63aa1c..737b93315a 100644 --- a/ush/extractvars_tools.sh +++ b/ush/extractvars_tools.sh @@ -1,6 +1,8 @@ #! /usr/bin/env bash check_atmos() { + # Function to check if there are any missing parm variables in any of the input product grib2 files + # A warning will be displayed if there is a parm variable that cannot be found in any of the given input product grib2 files infile1p=$1 infile2p=$2 varlistl=$3 @@ -22,6 +24,9 @@ check_atmos() { } daily_avg_atmos() { + # Function to calculate the 24-hr average of a grib2 file with atmospheric fields + # The input grib2 file must contain all the time records to be averaged (e.g. 6hr, 12hr, 18hr and 24hr record in one grib2 file) + # The interval of the time records in the input grib2 file must be 6 hours outfile_p=$1 dcnt_p=$2 outres_p=$3 @@ -40,6 +45,7 @@ daily_avg_atmos() { } copy_to_comout() { + # Function to copy the output file with the extracted product variables to a user-defined destination directory rundir_outfile=$1 #output data file generated in RUNDIR comout_dir=$2 #destination directory to which to copy the data file if [[ -f "${rundir_outfile}" ]];then diff --git a/ush/ocnice_extractvars.sh b/ush/ocnice_extractvars.sh index 916208096d..e9cee0e6c5 100755 --- a/ush/ocnice_extractvars.sh +++ b/ush/ocnice_extractvars.sh @@ -1,7 +1,15 @@ #! /usr/bin/env bash + ################################################################################ -# Script: -# +## UNIX Script Documentation Block +## Script name: ocnice_extractvars.sh +## Script description: Extracts and optionally compresses variables +## from ocean and ice products +## and saves these variables in arcdir +####################### +# Main body starts here +####################### + source "${USHgfs}/preamble.sh" subdata=${1} diff --git a/ush/wave_extractvars.sh b/ush/wave_extractvars.sh index c8d00cdd49..f1a886db93 100755 --- a/ush/wave_extractvars.sh +++ b/ush/wave_extractvars.sh @@ -1,7 +1,14 @@ #! /usr/bin/env bash + ################################################################################ -# Script: -# +## UNIX Script Documentation Block +## Script name: wave_extractvars.sh +## Script description: Extracts variables from wave products +## and saves these variables in arcdir +####################### +# Main body starts here +####################### + source "${USHgfs}/preamble.sh" subdata=${1} From bf19e74b0136da1c2ded95971baa0ac4af229b4b Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Thu, 18 Jul 2024 20:11:11 +0000 Subject: [PATCH 131/141] Modify description for daily_atmos A portion of the description for daily_atmos has been removed since what was mentioned is not a requirement for the input grib2 file. --- ush/extractvars_tools.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ush/extractvars_tools.sh b/ush/extractvars_tools.sh index 737b93315a..b0a5ad3184 100644 --- a/ush/extractvars_tools.sh +++ b/ush/extractvars_tools.sh @@ -25,7 +25,6 @@ check_atmos() { daily_avg_atmos() { # Function to calculate the 24-hr average of a grib2 file with atmospheric fields - # The input grib2 file must contain all the time records to be averaged (e.g. 6hr, 12hr, 18hr and 24hr record in one grib2 file) # The interval of the time records in the input grib2 file must be 6 hours outfile_p=$1 dcnt_p=$2 From c8692716eb46361a505abced1f2b52a789f4e2c2 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Thu, 18 Jul 2024 20:13:40 +0000 Subject: [PATCH 132/141] Revert "Modify description for daily_atmos" This reverts commit bf19e74b0136da1c2ded95971baa0ac4af229b4b. --- ush/extractvars_tools.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ush/extractvars_tools.sh b/ush/extractvars_tools.sh index b0a5ad3184..737b93315a 100644 --- a/ush/extractvars_tools.sh +++ b/ush/extractvars_tools.sh @@ -25,6 +25,7 @@ check_atmos() { daily_avg_atmos() { # Function to calculate the 24-hr average of a grib2 file with atmospheric fields + # The input grib2 file must contain all the time records to be averaged (e.g. 6hr, 12hr, 18hr and 24hr record in one grib2 file) # The interval of the time records in the input grib2 file must be 6 hours outfile_p=$1 dcnt_p=$2 From f7357eaa8ea8eab4f7524cde6e73da528a5f4750 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Thu, 18 Jul 2024 20:14:29 +0000 Subject: [PATCH 133/141] Modify and correct description for daily_atmos The correct portion of the description for daily_atmos has been removed since what was mentioned is not a requirement for the input grib2 file. Another portion that was incorrectly removed has been reinstated. --- ush/extractvars_tools.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ush/extractvars_tools.sh b/ush/extractvars_tools.sh index 737b93315a..6aecb9fa9e 100644 --- a/ush/extractvars_tools.sh +++ b/ush/extractvars_tools.sh @@ -26,7 +26,6 @@ check_atmos() { daily_avg_atmos() { # Function to calculate the 24-hr average of a grib2 file with atmospheric fields # The input grib2 file must contain all the time records to be averaged (e.g. 6hr, 12hr, 18hr and 24hr record in one grib2 file) - # The interval of the time records in the input grib2 file must be 6 hours outfile_p=$1 dcnt_p=$2 outres_p=$3 From f572f6b2ce4527e5c65522eb61e2db6e747dc574 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 19 Jul 2024 12:24:54 -0400 Subject: [PATCH 134/141] Apply suggestions from code review --- workflow/rocoto/gefs_tasks.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 314747d7a6..f6d1ad6cdc 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -487,16 +487,16 @@ def wavepostpnt(self): def extractvars(self): deps = [] if self.app_config.do_wave: - dep_dict = {'type': 'task', 'name': f'wave_post_grid_mem#member#'} + dep_dict = {'type': 'task', 'name': 'wave_post_grid_mem#member#'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_ocean: - dep_dict = {'type': 'metatask', 'name': f'ocean_prod_#member#'} + dep_dict = {'type': 'metatask', 'name': 'ocean_prod_#member#'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_ice: - dep_dict = {'type': 'metatask', 'name': f'ice_prod_#member#'} + dep_dict = {'type': 'metatask', 'name': 'ice_prod_#member#'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_atm: - dep_dict = {'type': 'metatask', 'name': f'atmos_prod_#member#'} + dep_dict = {'type': 'metatask', 'name': 'atmos_prod_#member#'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) extractvars_envars = self.envars.copy() From 85e18ed4937c5be78c89883070908109cce6a8ce Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Fri, 19 Jul 2024 19:52:39 +0000 Subject: [PATCH 135/141] Remove /dev/null to address reviewer comment --- ush/atmos_extractvars.sh | 2 +- ush/wave_extractvars.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/atmos_extractvars.sh b/ush/atmos_extractvars.sh index 210af44eb5..882c6c7e3d 100755 --- a/ush/atmos_extractvars.sh +++ b/ush/atmos_extractvars.sh @@ -60,7 +60,7 @@ for outtype in "f2d" "f3d"; do for infile in "${infile1}" "${infile2}"; do if [[ -f "${infile}" ]]; then # check if input file exists before extraction # shellcheck disable=SC2312 - ${WGRIB2} "${infile}" | grep -F -f "${varlist}" | ${WGRIB2} -i "${infile}" -append -grib "${outfile}" > /dev/null + ${WGRIB2} "${infile}" | grep -F -f "${varlist}" | ${WGRIB2} -i "${infile}" -append -grib "${outfile}" else echo "WARNING: ${infile} does not exist." fi diff --git a/ush/wave_extractvars.sh b/ush/wave_extractvars.sh index f1a886db93..51020575c9 100755 --- a/ush/wave_extractvars.sh +++ b/ush/wave_extractvars.sh @@ -24,7 +24,7 @@ for (( nh = FHOUT_WAV_EXTRACT; nh <= FHMAX_WAV; nh = nh + FHOUT_WAV_EXTRACT )); if [[ -f "${infile}" ]]; then #check if input file exists before extraction # shellcheck disable=SC2312 - ${WGRIB2} "${infile}" | grep -F -f "${varlist_wav}" | ${WGRIB2} -i "${infile}" -append -grib "${outfile}">/dev/null + ${WGRIB2} "${infile}" | grep -F -f "${varlist_wav}" | ${WGRIB2} -i "${infile}" -append -grib "${outfile}" else echo "WARNING: ${infile} does not exist." fi From 1f40284a1e6402526a1b2270860edc63a1a6a8cf Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Fri, 19 Jul 2024 19:54:21 +0000 Subject: [PATCH 136/141] Simplify if-statement to address reviewer comment --- scripts/exglobal_extractvars.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/exglobal_extractvars.sh b/scripts/exglobal_extractvars.sh index d225ab75a2..1e4ae0405e 100755 --- a/scripts/exglobal_extractvars.sh +++ b/scripts/exglobal_extractvars.sh @@ -20,8 +20,6 @@ EXTRCTVARW="${USHgfs}/wave_extractvars.sh" # Set FHMAX_HF_GFS equal to FHMAX_GFS if FHMAX_HF_GFS is greater than FHMAX_GFS if (( FHMAX_GFS < FHMAX_HF_GFS )); then export FHMAX_HF_GFS=FHMAX_GFS -else - export FHMAX_HF_GFS=FHMAX_HF_GFS fi # Set FHOUT_WAV_EXTRACT equal to FHOUT_WAV if FHOUT_WAV is not a factor of FHOUT_WAV_EXTRACT From 6d9ccb2ff4536a4d6a03c86a4fc0e64bf67777c0 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Fri, 19 Jul 2024 20:14:59 +0000 Subject: [PATCH 137/141] Fix syntax --- scripts/exglobal_extractvars.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exglobal_extractvars.sh b/scripts/exglobal_extractvars.sh index 1e4ae0405e..77db8431de 100755 --- a/scripts/exglobal_extractvars.sh +++ b/scripts/exglobal_extractvars.sh @@ -19,7 +19,7 @@ EXTRCTVARW="${USHgfs}/wave_extractvars.sh" # Set FHMAX_HF_GFS equal to FHMAX_GFS if FHMAX_HF_GFS is greater than FHMAX_GFS if (( FHMAX_GFS < FHMAX_HF_GFS )); then - export FHMAX_HF_GFS=FHMAX_GFS + export FHMAX_HF_GFS=${FHMAX_GFS} fi # Set FHOUT_WAV_EXTRACT equal to FHOUT_WAV if FHOUT_WAV is not a factor of FHOUT_WAV_EXTRACT From 97c2f8fc40005755ca6d803a8f580ad46cda514e Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Sat, 20 Jul 2024 10:19:17 +0000 Subject: [PATCH 138/141] Convert if-statement to arithmetic --- ush/atmos_extractvars.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ush/atmos_extractvars.sh b/ush/atmos_extractvars.sh index 882c6c7e3d..afd3271d9f 100755 --- a/ush/atmos_extractvars.sh +++ b/ush/atmos_extractvars.sh @@ -19,10 +19,10 @@ subdata=${1} for outtype in "f2d" "f3d"; do - if [[ "${outtype}" == "f2d" ]];then + if [[ "${outtype}" == "f2d" ]]; then varlist=${varlist_2d} ARC_RFCST_PROD_ATMOS="${ARC_RFCST_PROD_ATMOS_F2D}" - elif [[ "${outtype}" == "f3d" ]];then + elif [[ "${outtype}" == "f3d" ]]; then varlist=${varlist_3d} varlist_d=${varlist_3d_d} ARC_RFCST_PROD_ATMOS="${ARC_RFCST_PROD_ATMOS_F3D}" @@ -35,13 +35,13 @@ for outtype in "f2d" "f3d"; do while (( nh <= FHMAX_GFS )); do fnh=$(printf "%3.3d" "${nh}") - if [[ "${outtype}" == "f2d" ]];then - if [[ ${nh} -le ${FHMAX_HF_GFS} ]];then + if [[ "${outtype}" == "f2d" ]]; then + if (( nh < FHMAX_HF_GFS )); then outres="0p25" else outres="0p50" fi - elif [[ "${outtype}" == "f3d" ]];then + elif [[ "${outtype}" == "f3d" ]]; then outres="1p00" fi @@ -70,7 +70,7 @@ for outtype in "f2d" "f3d"; do copy_to_comout "${outfile}" "${ARC_RFCST_PROD_ATMOS}" #Compute daily average for a subset of variables - if (( nh % 6 == 0 )) && (( nh != 0 )) && [[ "${outtype}" == "f3d" ]];then + if (( nh % 6 == 0 )) && (( nh != 0 )) && [[ "${outtype}" == "f3d" ]]; then outfile=${subdata}/vartmp_raw_vari_ldy${dcnt}.grib2 for infile in "${infile1}" "${infile2}"; do if [[ -f "${infile}" ]]; then # check if input file exists before extraction @@ -80,7 +80,7 @@ for outtype in "f2d" "f3d"; do echo "WARNING: ${infile} does not exist." fi done - if [[ ${fcnt} -eq 4 ]];then + if [[ ${fcnt} -eq 4 ]]; then daily_avg_atmos "${outfile}" "${dcnt}" "${outres}" copy_to_comout "${davg_file}" "${ARC_RFCST_PROD_ATMOS}" fcnt=1 From 5aa68a763fbd32c663e1dc141c4b3076f45aac23 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Sat, 20 Jul 2024 10:36:36 +0000 Subject: [PATCH 139/141] Add proper spacing in comments and loops --- jobs/JGLOBAL_EXTRACTVARS | 8 ++++---- parm/config/gefs/config.extractvars | 30 ++++++++++++++--------------- scripts/exglobal_extractvars.sh | 8 ++++---- ush/atmos_extractvars.sh | 10 +++++----- ush/extractvars_tools.sh | 8 ++++---- ush/ocnice_extractvars.sh | 12 ++++++------ ush/wave_extractvars.sh | 6 +++--- 7 files changed, 41 insertions(+), 41 deletions(-) diff --git a/jobs/JGLOBAL_EXTRACTVARS b/jobs/JGLOBAL_EXTRACTVARS index 137429a59e..3478ca3976 100755 --- a/jobs/JGLOBAL_EXTRACTVARS +++ b/jobs/JGLOBAL_EXTRACTVARS @@ -19,17 +19,17 @@ YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx \ "COMIN_ICE_NETCDF:COM_ICE_NETCDF_TMPL" \ "COMIN_WAVE_GRID:COM_WAVE_GRID_TMPL" -if [[ "${DO_ATM}" == "YES" ]];then +if [[ "${DO_ATM}" == "YES" ]]; then if [[ ! -d "${ARC_RFCST_PROD_ATMOS_F2D}" ]]; then mkdir -p "${ARC_RFCST_PROD_ATMOS_F2D}"; fi if [[ ! -d "${ARC_RFCST_PROD_ATMOS_F3D}" ]]; then mkdir -p "${ARC_RFCST_PROD_ATMOS_F3D}"; fi fi -if [[ "${DO_OCN}" == "YES" ]];then +if [[ "${DO_OCN}" == "YES" ]]; then if [[ ! -d "${ARC_RFCST_PROD_OCN}" ]]; then mkdir -p "${ARC_RFCST_PROD_OCN}"; fi fi -if [[ "${DO_ICE}" == "YES" ]];then +if [[ "${DO_ICE}" == "YES" ]]; then if [[ ! -d "${ARC_RFCST_PROD_ICE}" ]]; then mkdir -p "${ARC_RFCST_PROD_ICE}"; fi fi -if [[ "${DO_WAVE}" == "YES" ]];then +if [[ "${DO_WAVE}" == "YES" ]]; then if [[ ! -d "${ARC_RFCST_PROD_WAV}" ]]; then mkdir -p "${ARC_RFCST_PROD_WAV}"; fi fi diff --git a/parm/config/gefs/config.extractvars b/parm/config/gefs/config.extractvars index 8fce4a227f..706fe18450 100644 --- a/parm/config/gefs/config.extractvars +++ b/parm/config/gefs/config.extractvars @@ -9,26 +9,26 @@ echo "BEGIN: config.extractvars" export COMPRSCMD=${COMPRSCMD:-bzip2} -export compress_ocn=0 #1: compress extracted ocean product, 0: do not compress extracted ocean product -export compress_ice=0 #1: compress extracted ice product, 0: do not compress extracted ice product +export compress_ocn=0 #1: Compress extracted ocean product, 0: Do not compress extracted ocean product +export compress_ice=0 #1: Compress extracted ice product, 0: Do not compress extracted ice product -export ocnres="5p00" #Resolution of ocean products -export iceres="5p00" #Resolution of ice products -export wavres="5p00" #Resolution of wave products +export ocnres="5p00" # Resolution of ocean products +export iceres="5p00" # Resolution of ice products +export wavres="5p00" # Resolution of wave products -export depthvar_name="z_l" #name of depth variable in NetCDF ocean products -export zmin="0." #Minimum depth to extract from NetCDF ocean products -export zmax="300." #Maximum depth to extract from NetCDF ocean products +export depthvar_name="z_l" # Name of depth variable in NetCDF ocean products +export zmin="0." # Minimum depth to extract from NetCDF ocean products +export zmax="300." # Maximum depth to extract from NetCDF ocean products -export FHOUT_WAV_EXTRACT=6 #Frequency of wave output to be saved on disk +export FHOUT_WAV_EXTRACT=6 # Frequency of wave output to be saved on disk #Paramater Tables used -export varlist_2d="${PARMgfs}/product/gefs_shortparmlist_2d.parm" #Parameter table for surface variables -export varlist_3d="${PARMgfs}/product/gefs_shortparmlist_3d_h.parm" #Parameter table for upper air instantaneous variables -export varlist_3d_d="${PARMgfs}/product/gefs_shortparmlist_3d_d.parm" #Parameter table for upper air daily-averaged variables -export varlist_wav="${PARMgfs}/product/gefs_wav_shortparmlist.parm" #Parameter table for wave variables -export varlist_ocn_netcdf="${PARMgfs}/product/gefs_ocn_shortparmlist.parm" #Parameter table for ocean netcdf variables -export varlist_ice_netcdf="${PARMgfs}/product/gefs_ice_shortparmlist.parm" #Parameter table for ice netcdf variables +export varlist_2d="${PARMgfs}/product/gefs_shortparmlist_2d.parm" # Parameter table for surface variables +export varlist_3d="${PARMgfs}/product/gefs_shortparmlist_3d_h.parm" # Parameter table for upper air instantaneous variables +export varlist_3d_d="${PARMgfs}/product/gefs_shortparmlist_3d_d.parm" # Parameter table for upper air daily-averaged variables +export varlist_wav="${PARMgfs}/product/gefs_wav_shortparmlist.parm" # Parameter table for wave variables +export varlist_ocn_netcdf="${PARMgfs}/product/gefs_ocn_shortparmlist.parm" # Parameter table for ocean netcdf variables +export varlist_ice_netcdf="${PARMgfs}/product/gefs_ice_shortparmlist.parm" # Parameter table for ice netcdf variables #Directory to save extracted variables export ARC_RFCST_PROD="${ARCDIR}/rfcst/${PDY:0:4}/${PDY:0:6}/${PDY:0:8}/mem${ENSMEM}" diff --git a/scripts/exglobal_extractvars.sh b/scripts/exglobal_extractvars.sh index 77db8431de..a124667679 100755 --- a/scripts/exglobal_extractvars.sh +++ b/scripts/exglobal_extractvars.sh @@ -28,24 +28,24 @@ if (( FHOUT_WAV_EXTRACT % FHOUT_WAV != 0 )); then fi # Extract variables for atmosphere -if [[ "${DO_ATM}" == "YES" ]];then +if [[ "${DO_ATM}" == "YES" ]]; then ${EXTRCTVARA} "${DATA}/atmos" fi # Extract variables for ocean -if [[ "${DO_OCN}" == "YES" ]];then +if [[ "${DO_OCN}" == "YES" ]]; then export component_name="ocn" ${EXTRCTVARO} "${DATA}/ocn" "${varlist_ocn_netcdf}" "${ocnres}" "${compress_ocn}" "${FHOUT_OCN_GFS}" "${ARC_RFCST_PROD_OCN}" fi # Extract variables for ice -if [[ "${DO_ICE}" == "YES" ]];then +if [[ "${DO_ICE}" == "YES" ]]; then export component_name="ice" ${EXTRCTVARO} "${DATA}/ice" "${varlist_ice_netcdf}" "${iceres}" "${compress_ice}" "${FHOUT_ICE_GFS}" "${ARC_RFCST_PROD_ICE}" fi # Extract variables for wave -if [[ "${DO_WAVE}" == "YES" ]];then +if [[ "${DO_WAVE}" == "YES" ]]; then export component_name="wav" ${EXTRCTVARW} "${DATA}/wav" fi diff --git a/ush/atmos_extractvars.sh b/ush/atmos_extractvars.sh index afd3271d9f..c9d0a4e7af 100755 --- a/ush/atmos_extractvars.sh +++ b/ush/atmos_extractvars.sh @@ -69,7 +69,7 @@ for outtype in "f2d" "f3d"; do check_atmos "${infile1}" "${infile2}" "${varlist}" "${fnh}" copy_to_comout "${outfile}" "${ARC_RFCST_PROD_ATMOS}" - #Compute daily average for a subset of variables + # Compute daily average for a subset of variables if (( nh % 6 == 0 )) && (( nh != 0 )) && [[ "${outtype}" == "f3d" ]]; then outfile=${subdata}/vartmp_raw_vari_ldy${dcnt}.grib2 for infile in "${infile1}" "${infile2}"; do @@ -87,12 +87,12 @@ for outtype in "f2d" "f3d"; do dcnt=$(( dcnt + 1 )) else fcnt=$(( fcnt + 1 )) - fi #If at final lead hour of a given day - fi #if lead hour is divisible by 6 and outtype is f3d + fi # If at final lead hour of a given day + fi # if lead hour is divisible by 6 and outtype is f3d nh=$(( nh + outfreq )) - done #fhr + done # nh -done #f2d,f3d +done # f2d,f3d exit 0 diff --git a/ush/extractvars_tools.sh b/ush/extractvars_tools.sh index 6aecb9fa9e..daf61a3d2e 100644 --- a/ush/extractvars_tools.sh +++ b/ush/extractvars_tools.sh @@ -40,14 +40,14 @@ daily_avg_atmos() { ${WGRIB2} "${davgtmp}" | ${WGRIB2} -i "${davgtmp}" -append -grib "${davg_file}" rm -f "${davgtmp}" vcnt=$(( vcnt + 1 )) - done <"${varlist_d}" #variable + done <"${varlist_d}" # variable } copy_to_comout() { # Function to copy the output file with the extracted product variables to a user-defined destination directory - rundir_outfile=$1 #output data file generated in RUNDIR - comout_dir=$2 #destination directory to which to copy the data file - if [[ -f "${rundir_outfile}" ]];then + rundir_outfile=$1 # output data file generated in RUNDIR + comout_dir=$2 # destination directory to which to copy the data file + if [[ -f "${rundir_outfile}" ]]; then cpfs "${rundir_outfile}" "${comout_dir}" else echo "FATAL ERROR: Output file (${rundir_outfile}) does not exist." diff --git a/ush/ocnice_extractvars.sh b/ush/ocnice_extractvars.sh index e9cee0e6c5..f0660bb6ec 100755 --- a/ush/ocnice_extractvars.sh +++ b/ush/ocnice_extractvars.sh @@ -24,11 +24,11 @@ comout_rfcst_prod_ocnice=${6} for (( nh = FHMIN_GFS; nh <= FHMAX_GFS; nh = nh + fhout_ocnice )); do fnh=$(printf "%3.3d" "${nh}") - if [[ ${component_name} == "ocn" ]];then + if [[ ${component_name} == "ocn" ]]; then infile=${COMIN_OCEAN_NETCDF}/${RUN}.ocean.t${cyc}z.${datares}.f${fnh}.nc # For ocean products, add an argument to extract a subset of levels otherargs=(-d "${depthvar_name},""${zmin},""${zmax}") - elif [[ ${component_name} == "ice" ]];then + elif [[ ${component_name} == "ice" ]]; then infile=${COMIN_ICE_NETCDF}/${RUN}.ice.t${cyc}z.${datares}.f${fnh}.nc otherargs=() fi @@ -39,20 +39,20 @@ for (( nh = FHMIN_GFS; nh <= FHMAX_GFS; nh = nh + fhout_ocnice )); do varsinfile=$(cdo -showname "${infile}") varsavailable="" for i in ${varsrequested}; do - #check if variable from parm file is available in netcdf file. If variable is not in netcdf file, do not try to extract that variable. + # Check if variable from parm file is available in netcdf file. If variable is not in netcdf file, do not try to extract that variable. if [[ ${varsinfile} == *"${i}"* ]]; then varsavailable+="${i}," else echo "WARNING: ${i} is not available in ${infile}." fi done - if [[ -z "${varsavailable}" ]];then + if [[ -z "${varsavailable}" ]]; then echo "WARNING: No variables from parm file ${varlist} are available in netcdf file ${infile}." else ocnice_vars=${varsavailable::-1} ncks -v "${ocnice_vars}" "${otherargs[@]}" "${infile}" "${outfile}" fi - if [[ ${datacompress} -eq 1 ]];then + if [[ ${datacompress} -eq 1 ]]; then ${COMPRSCMD} "${outfile}" copy_to_comout "${outfile}.bz2" "${comout_rfcst_prod_ocnice}" else @@ -61,6 +61,6 @@ for (( nh = FHMIN_GFS; nh <= FHMAX_GFS; nh = nh + fhout_ocnice )); do else echo "WARNING: ${infile} does not exist." fi -done +done # nh exit 0 diff --git a/ush/wave_extractvars.sh b/ush/wave_extractvars.sh index 51020575c9..32ee44986b 100755 --- a/ush/wave_extractvars.sh +++ b/ush/wave_extractvars.sh @@ -20,15 +20,15 @@ for (( nh = FHOUT_WAV_EXTRACT; nh <= FHMAX_WAV; nh = nh + FHOUT_WAV_EXTRACT )); infile=${COMIN_WAVE_GRID}/${RUN}wave.t${cyc}z.global.${wavres}.f${fnh}.grib2 outfile=${subdata}/${RUN}wave.t${cyc}z.global.${wavres}.f${fnh}.grib2 - rm -f "${outfile}" #remove outfile if it already exists before extraction + rm -f "${outfile}" # Remove outfile if it already exists before extraction - if [[ -f "${infile}" ]]; then #check if input file exists before extraction + if [[ -f "${infile}" ]]; then # Check if input file exists before extraction # shellcheck disable=SC2312 ${WGRIB2} "${infile}" | grep -F -f "${varlist_wav}" | ${WGRIB2} -i "${infile}" -append -grib "${outfile}" else echo "WARNING: ${infile} does not exist." fi copy_to_comout "${outfile}" "${ARC_RFCST_PROD_WAV}" -done #fhr +done # nh exit 0 From 9a724f411280d6e24f6ff37371b1230eeafdec09 Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Sat, 20 Jul 2024 10:43:49 +0000 Subject: [PATCH 140/141] Change operator in if-statement from < to <= --- ush/atmos_extractvars.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/atmos_extractvars.sh b/ush/atmos_extractvars.sh index c9d0a4e7af..70e86b2f4e 100755 --- a/ush/atmos_extractvars.sh +++ b/ush/atmos_extractvars.sh @@ -45,7 +45,7 @@ for outtype in "f2d" "f3d"; do outres="1p00" fi - if (( nh < FHMAX_HF_GFS )); then + if (( nh <= FHMAX_HF_GFS )); then outfreq=${FHOUT_HF_GFS} else outfreq=${FHOUT_GFS} From bffeb25a0af67c975347269f883c6d894c8901dd Mon Sep 17 00:00:00 2001 From: "Eric.Sinsky" Date: Sat, 20 Jul 2024 10:50:35 +0000 Subject: [PATCH 141/141] Add proper spacing to comment in config.base --- parm/config/gefs/config.base | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index 7717a4aa80..1f5c9228e2 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -137,7 +137,7 @@ export DO_WAVE="NO" export DO_OCN="NO" export DO_ICE="NO" export DO_AERO="NO" -export DO_EXTRACTVARS="@DO_EXTRACTVARS@" #Option to process and extract a subset of products to save on disk +export DO_EXTRACTVARS="@DO_EXTRACTVARS@" # Option to process and extract a subset of products to save on disk export AERO_FCST_CDUMP="" # When to run aerosol forecast: gdas, gfs, or both export AERO_ANL_CDUMP="" # When to run aerosol analysis: gdas, gfs, or both export WAVE_CDUMP="" # When to include wave suite: gdas, gfs, or both