Add observation preparation job for aerosols DA to workflow

env/HERA.env

@@ -102,6 +102,13 @@ elif [[ "${step}" = "atmanlfv3inc" ]]; then
     [[ ${NTHREADS_ATMANLFV3INC} -gt ${nth_max} ]] && export NTHREADS_ATMANLFV3INC=${nth_max}
     export APRUN_ATMANLFV3INC="${launcher} -n ${npe_atmanlfv3inc} --cpus-per-task=${NTHREADS_ATMANLFV3INC}"
 
+elif [[ "${step}" = "prepaeroobs" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_prepaeroobs))
+
+    export NTHREADS_PREPAEROOBS=${nth_prepaeroobs:-1}
+    export APRUN_PREPAEROOBS="${launcher} -n 1 --cpus-per-task=${NTHREADS_PREPAEROOBS}"
+
 elif [[ "${step}" = "snowanl" ]]; then
 
     nth_max=$((npe_node_max / npe_node_snowanl))

env/HERCULES.env

@@ -99,6 +99,13 @@
     [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
     export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun} --cpus-per-task=${NTHREADS_AEROANL}"
  ;;
+
+ "prepaeroobs")
+    nth_max=$((npe_node_max / npe_node_prepaeroobs))
+
+    export NTHREADS_PREPAEROOBS=${nth_prepaeroobs:-1}
+    export APRUN_PREPAEROOBS="${launcher} -n 1 --cpus-per-task=${NTHREADS_PREPAEROOBS}"
+
  "snowanl")
 
     nth_max=$((npe_node_max / npe_node_snowanl))

env/JET.env

@@ -82,6 +82,13 @@ elif [[ "${step}" = "aeroanlrun" ]]; then
     [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
     export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}"
 
+elif [[ "${step}" = "prepaeroobs" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_prepaeroobs))
+
+    export NTHREADS_PREPAEROOBS=${nth_prepaeroobs:-1}
+    export APRUN_PREPAEROOBS="${launcher} -n 1 --cpus-per-task=${NTHREADS_PREPAEROOBS}"
+
 elif [[ "${step}" = "snowanl" ]]; then
 
     nth_max=$((npe_node_max / npe_node_snowanl))

env/ORION.env

@@ -90,6 +90,13 @@ elif [[ "${step}" = "aeroanlrun" ]]; then
     [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
     export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun} --cpus-per-task=${NTHREADS_AEROANL}"
 
+elif [[ "${step}" = "prepaeroobs" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_prepaeroobs))
+
+    export NTHREADS_PREPAEROOBS=${nth_prepaeroobs:-1}
+    export APRUN_PREPAEROOBS="${launcher} -n 1 --cpus-per-task=${NTHREADS_PREPAEROOBS}"
+
 elif [[ "${step}" = "snowanl" ]]; then
 
     nth_max=$((npe_node_max / npe_node_snowanl))

env/S4.env

@@ -82,6 +82,13 @@ elif [[ "${step}" = "aeroanlrun" ]]; then
     [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
     export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}"
 
+elif [[ "${step}" = "prepaeroobs" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_prepaeroobs))
+
+    export NTHREADS_PREPAEROOBS=${nth_prepaeroobs:-1}
+    export APRUN_PREPAEROOBS="${launcher} -n 1 --cpus-per-task=${NTHREADS_PREPAEROOBS}"
+
 elif [[ "${step}" = "snowanl" ]]; then
 
     nth_max=$((npe_node_max / npe_node_snowanl))

env/WCOSS2.env

@@ -76,6 +76,13 @@ elif [[ "${step}" = "aeroanlrun" ]]; then
     [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
     export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}"
 
+elif [[ "${step}" = "prepaeroobs" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_prepaeroobs))
+
+    export NTHREADS_PREPAEROOBS=${nth_prepaeroobs:-1}
+    export APRUN_PREPAEROOBS="${launcher} -n 1 --cpus-per-task=${NTHREADS_PREPAEROOBS}"
+
 elif [[ "${step}" = "snowanl" ]]; then
 
     nth_max=$((npe_node_max / npe_node_snowanl))

jobs/JGLOBAL_PREP_AERO_OBS

@@ -0,0 +1,49 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+export DATA=${DATA:-${DATAROOT}/${RUN}aeroanl_${cyc}}
+source "${HOMEgfs}/ush/jjob_header.sh" -e "prepaeroobs" -c "base aeroanl prepaeroobs"
+
+##############################################
+# Set variables used in the script
+##############################################
+# shellcheck disable=SC2153
+GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours")
+gPDY=${GDATE:0:8}
+gcyc=${GDATE:8:2}
+GDUMP="gdas"
+
+
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+
+# Generate COM variables from templates
+YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OBS COM_CHEM_ANALYSIS
+
+RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
+    COM_CHEM_ANALYSIS_PREV:COM_CHEM_ANALYSIS_TMPL \
+    COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL
+
+mkdir -m 775 -p "${COM_CHEM_ANALYSIS}"
+
+###############################################################
+# Run relevant script
+
+EXSCRIPT=${GDASPREPAEROOBSPY:-${SCRgfs}/exglobal_prep_aero_obs.py}
+${EXSCRIPT}
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]] ; then
+  cat "${pgmout}"
+fi
+
+exit 0

jobs/rocoto/prepaeroobs.sh

@@ -0,0 +1,24 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+# Source UFSDA workflow modules
+. "${HOMEgfs}/ush/load_ufsda_modules.sh"
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+export job="prepaeroobs"
+export jobid="${job}.$$"
+
+###############################################################
+# setup python path for workflow utilities and tasks
+wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
+export PYTHONPATH
+
+###############################################################
+# Execute the JJOB
+"${HOMEgfs}/jobs/JGLOBAL_PREP_AERO_OBS"
+status=$?
+exit "${status}"

parm/config/gfs/config.aeroanl

@@ -20,6 +20,7 @@ export io_layout_y=@IO_LAYOUT_Y@
 
 export JEDIEXE="${EXECgfs}/gdas.x"
 
+
 if [[ "${DOIAU}" == "YES" ]]; then
   export aero_bkg_times="3,6,9"
   export JEDIYAML="${PARMgfs}/gdas/aero/variational/3dvar_fgat_gfs_aero.yaml.j2"

parm/config/gfs/config.base

@@ -178,6 +178,7 @@ export DO_WAVE="NO"
 export DO_OCN="NO"
 export DO_ICE="NO"
 export DO_AERO="NO"
+export DO_PREP_AERO_OBS="YES"
 export AERO_FCST_CDUMP="" # When to run aerosol forecast: gdas, gfs, or both
 export AERO_ANL_CDUMP="" # When to run aerosol analysis: gdas, gfs, or both
 export WAVE_CDUMP="" # When to include wave suite: gdas, gfs, or both

parm/config/gfs/config.prepaeroobs

@@ -0,0 +1,17 @@
+#!/bin/bash -x
+
+########## config.prepaeroobs ##########
+# Aerosol Variance specific
+
+echo "BEGIN: config.prepaeroobs"
+
+# Get task specific resources
+source "${EXPDIR}/config.resources" prepaeroobs
+
+export OBSPROCYAML="${PARMgfs}/gdas/aero/obs/list/aero_obsproc.yaml.j2"
+export OBSPROCEXE="${EXECgfs}/gdas_obsprovider2ioda.x"
+export VIIRS_DATA_DIR="/scratch2/NCEPDEV/stmp3/Yaping.Wang/VIIRS/AWS/"
+export SENSORS="npp,n20"
+
+
+echo "END: config.prepaeroobs"

parm/config/gfs/config.resources

@@ -13,7 +13,7 @@ if (( $# != 1 )); then
     echo "atmanlinit atmanlvar atmanlfv3inc atmanlfinal"
     echo "atmensanlinit atmensanlletkf atmensanlfv3inc atmensanlfinal"
     echo "snowanl"
-    echo "aeroanlinit aeroanlrun aeroanlfinal"
+    echo "prepaeroobs aeroanlinit aeroanlrun aeroanlfinal"
     echo "anal sfcanl analcalc analdiag fcst echgres"
     echo "upp atmos_products"
     echo "tracker genesis genesis_fsu"
@@ -289,6 +289,14 @@ case ${step} in
     export npe_node_snowanl=$(( npe_node_max / nth_snowanl ))
     ;;
 
+  "prepaeroobs")
+    export wtime_prepaeroobs="00:15:00"
+    export npe_prepaeroobs=1
+    export nth_prepaeroobs=1
+    export npe_node_prepaeroobs=1
+    export memory_prepaeroobs="96GB"
+    ;;
+
   "aeroanlinit")
     # below lines are for creating JEDI YAML
     case ${CASE} in

scripts/exglobal_prep_aero_obs.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+# exglobal_prep_aero_obs.py
+# This script collect available viirs
+# obs files, combine and preprocess
+# them.
+import os
+
+from wxflow import Logger, cast_strdict_as_dtypedict
+from pygfs.task.aero_prepobs import AerosolObsPrep
+
+# Initialize root logger
+logger = Logger(level='DEBUG', colored_log=True)
+
+
+if __name__ == '__main__':
+
+    # Take configuration from environment and cast it as python dictionary
+    config = cast_strdict_as_dtypedict(os.environ)
+
+    # Instantiate the aerosol prep obs tasks
+    AeroObs = AerosolObsPrep(config)
+    AeroObs.initialize()
+    AeroObs.runConverter()
+    AeroObs.finalize()