Merge branch 'develop' into feature/ocn-letkf

env/HERA.env

@@ -49,6 +49,10 @@ elif [[ "${step}" = "prepsnowobs" ]]; then
 
     export APRUN_CALCFIMS="${launcher} -n 1"
 
+elif [[ "${step}" = "prep_emissions" ]]; then
+
+    export APRUN="${launcher} -n 1"
+
 elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then
 
     export CFP_MP="YES"

env/HERCULES.env

@@ -45,6 +45,10 @@ case ${step} in
 
     export APRUN_CALCFIMS="${launcher} -n 1"
  ;;
+ "prep_emissions")
+
+    export APRUN="${launcher} -n 1"
+ ;;
  "waveinit" | "waveprep" | "wavepostsbs" | "wavepostbndpnt" | "wavepostpnt" | "wavepostbndpntbll")
 
     export CFP_MP="YES"

env/JET.env

@@ -37,6 +37,10 @@ elif [[ "${step}" = "prepsnowobs" ]]; then
 
     export APRUN_CALCFIMS="${launcher} -n 1"
 
+elif [[ "${step}" = "prep_emissions" ]]; then
+
+    export APRUN="${launcher} -n 1"
+
 elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then
 
     export CFP_MP="YES"

env/ORION.env

@@ -44,6 +44,10 @@ elif [[ "${step}" = "prepsnowobs" ]]; then
 
     export APRUN_CALCFIMS="${launcher} -n 1"
 
+elif [[ "${step}" = "prep_emissions" ]]; then
+
+    export APRUN="${launcher} -n 1"
+
 elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || \
     [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostpnt" ]] || [[ "${step}" == "wavepostbndpntbll" ]]; then
 

env/S4.env

@@ -37,6 +37,10 @@ elif [[ "${step}" = "prepsnowobs" ]]; then
 
     export APRUN_CALCFIMS="${launcher} -n 1"
 
+elif [[ "${step}" = "prep_emissions" ]]; then
+
+    export APRUN="${launcher} -n 1"
+
 elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then
 
     export CFP_MP="YES"

env/WCOSS2.env

@@ -31,6 +31,10 @@ elif [[ "${step}" = "prepsnowobs" ]]; then
 
     export APRUN_CALCFIMS="${launcher} -n 1"
 
+elif [[ "${step}" = "prep_emissions" ]]; then
+
+    export APRUN="${launcher} -n 1"
+
 elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll"  ]] || [[ "${step}" = "wavepostpnt" ]]; then
 
     export USE_CFP="YES"

jobs/JGLOBAL_PREP_EMISSIONS

@@ -0,0 +1,35 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "prep_emissions" -c "base prep_emissions"
+
+##############################################
+# Set variables used in the script
+##############################################
+# TODO: Set local variables used in this script e.g. GDATE may be needed for previous cycle
+
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+# Generate COM variables from templates
+# TODO: Add necessary COMIN, COMOUT variables for this job
+
+###############################################################
+# Run relevant script
+EXSCRIPT=${PREP_EMISSIONS_PY:-${SCRgfs}/exglobal_prep_emissions.py}
+${EXSCRIPT}
+status=$?
+(( status != 0 )) && ( echo "FATAL ERROR: Error executing ${EXSCRIPT}, ABORT!"; exit "${status}" )
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]] ; then
+  cat "${pgmout}"
+fi
+
+exit 0

jobs/rocoto/prep_emissions.sh

@@ -0,0 +1,23 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+# Source UFSDA workflow modules
+source "${HOMEgfs}/ush/load_fv3gfs_modules.sh"
+status=$?
+(( status !=  0 )) && exit "${status}"
+
+export job="prep_emissions"
+export jobid="${job}.$$"
+
+###############################################################
+# setup python path for workflow utilities and tasks
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/ush/python"
+export PYTHONPATH
+
+###############################################################
+# Execute the JJOB
+"${HOMEgfs}/jobs/JGLOBAL_PREP_EMISSIONS"
+status=$?
+exit "${status}"

parm/config/gefs/config.prep_emissions

@@ -0,0 +1,11 @@
+#! /usr/bin/env bash
+
+########## config.prep_emissions ##########
+# aerosol emissions preprocessing specific
+
+echo "BEGIN: config.prep_emissions"
+
+# Get task specific resources
+source "${EXPDIR}/config.resources" prep_emissions
+
+echo "END: config.prep_emissions"

parm/config/gefs/config.resources

@@ -69,6 +69,14 @@ case ${step} in
     export memory_waveinit="2GB"
     ;;
 
+  "prep_emissions")
+    export wtime_prep_emissions="00:10:00"
+    export npe_prep_emissions=1
+    export nth_prep_emissions=1
+    export npe_node_prep_emissions=$(( npe_node_max / nth_prep_emissions ))
+    export memory_prep_emissions="1GB"
+    ;;
+
   "fcst" | "efcs")
     export is_exclusive=True
 

parm/config/gfs/config.aeroanl

@@ -18,7 +18,7 @@ export JEDI_FIX_YAML="${PARMgfs}/gdas/aero_jedi_fix.yaml.j2"
 export io_layout_x=@IO_LAYOUT_X@
 export io_layout_y=@IO_LAYOUT_Y@
 
-export JEDIEXE="${EXECgfs}/fv3jedi_var.x"
+export JEDIEXE="${EXECgfs}/gdas.x"
 
 if [[ "${DOIAU}" == "YES" ]]; then
   export aero_bkg_times="3,6,9"

parm/config/gfs/config.atmanl

@@ -28,6 +28,6 @@ export layout_y_atmanl=@LAYOUT_Y_ATMANL@
 export io_layout_x=@IO_LAYOUT_X@
 export io_layout_y=@IO_LAYOUT_Y@
 
-export JEDIEXE=${EXECgfs}/fv3jedi_var.x
+export JEDIEXE=${EXECgfs}/gdas.x
 
 echo "END: config.atmanl"

parm/config/gfs/config.atmensanl

@@ -18,6 +18,6 @@ export layout_y_atmensanl=@LAYOUT_Y_ATMENSANL@
 export io_layout_x=@IO_LAYOUT_X@
 export io_layout_y=@IO_LAYOUT_Y@
 
-export JEDIEXE=${EXECgfs}/fv3jedi_letkf.x
+export JEDIEXE=${EXECgfs}/gdas.x
 
 echo "END: config.atmensanl"

parm/config/gfs/config.snowanl

@@ -11,7 +11,7 @@ source "${EXPDIR}/config.resources" snowanl
 export OBS_LIST="${PARMgfs}/gdas/snow/obs/lists/gdas_snow.yaml.j2"
 
 # Name of the JEDI executable and its yaml template
-export JEDIEXE="${EXECgfs}/fv3jedi_letkf.x"
+export JEDIEXE="${EXECgfs}/gdas.x"
 export JEDIYAML="${PARMgfs}/gdas/snow/letkfoi/letkfoi.yaml.j2"
 
 # Ensemble member properties

scripts/exglobal_prep_emissions.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+# exglobal_prep_emissions.py
+# This script creates a emissions object
+# which perform the pre-processing for aerosol emissions
+import os
+
+from wxflow import Logger, cast_strdict_as_dtypedict
+from pygfs import AerosolEmissions
+
+
+# Initialize root logger
+logger = Logger(level=os.environ.get("LOGGING_LEVEL", "DEBUG"), colored_log=True)
+
+
+if __name__ == '__main__':
+
+    # Take configuration from environment and cast it as python dictionary
+    config = cast_strdict_as_dtypedict(os.environ)
+
+    # Instantiate the emissions pre-processing task
+    emissions = AerosolEmissions(config)
+    emissions.initialize()
+    emissions.configure()
+    emissions.execute(emissions.task_config.DATA, emissions.task_config.APRUN)
+    emissions.finalize()

sorc/link_workflow.sh

@@ -334,33 +334,16 @@ fi
 
 # GDASApp
 if [[ -d "${HOMEgfs}/sorc/gdas.cd/build" ]]; then
-  declare -a JEDI_EXE=("fv3jedi_addincrement.x" \
-                       "fv3jedi_diffstates.x" \
-                       "fv3jedi_ensvariance.x" \
-                       "fv3jedi_hofx.x" \
-                       "fv3jedi_var.x" \
-                       "fv3jedi_convertincrement.x" \
-                       "fv3jedi_dirac.x" \
-                       "fv3jedi_error_covariance_training.x" \
-                       "fv3jedi_letkf.x" \
-                       "fv3jedi_convertstate.x" \
-                       "fv3jedi_eda.x" \
-                       "fv3jedi_forecast.x" \
+  declare -a JEDI_EXE=("gdas.x" \
+                       "gdas_soca_gridgen.x" \
+                       "gdas_soca_error_covariance_toolbox.x" \
+                       "gdas_soca_setcorscales.x" \
                        "fv3jedi_plot_field.x" \
-                       "fv3jedi_data_checker.py" \
-                       "fv3jedi_enshofx.x" \
-                       "fv3jedi_hofx_nomodel.x" \
-                       "fv3jedi_testdata_downloader.py" \
                        "fv3jedi_fv3inc.x" \
                        "gdas_ens_handler.x" \
                        "gdas_incr_handler.x" \
                        "gdas_obsprovider2ioda.x" \
                        "gdas_socahybridweights.x" \
-                       "soca_convertincrement.x" \
-                       "soca_error_covariance_training.x" \
-                       "soca_setcorscales.x" \
-                       "soca_gridgen.x" \
-                       "soca_var.x" \
                        "bufr2ioda.x" \
                        "calcfIMS.exe" \
                        "apply_incr.exe" )

ush/python/pygfs/__init__.py

@@ -0,0 +1,16 @@
+
+import os
+
+from .task.analysis import Analysis
+from .task.aero_emissions import AerosolEmissions
+from .task.aero_analysis import AerosolAnalysis
+from .task.atm_analysis import AtmAnalysis
+from .task.atmens_analysis import AtmEnsAnalysis
+from .task.snow_analysis import SnowAnalysis
+from .task.upp import UPP
+from .task.oceanice_products import OceanIceProducts
+from .task.gfs_forecast import GFSForecast
+
+__docformat__ = "restructuredtext"
+__version__ = "0.1.0"
+pygfs_directory = os.path.dirname(__file__)

ush/python/pygfs/task/aero_analysis.py

@@ -109,8 +109,10 @@ def execute(self: Analysis) -> None:
         chdir(self.task_config.DATA)
 
         exec_cmd = Executable(self.task_config.APRUN_AEROANL)
-        exec_name = os.path.join(self.task_config.DATA, 'fv3jedi_var.x')
+        exec_name = os.path.join(self.task_config.DATA, 'gdas.x')
         exec_cmd.add_default_arg(exec_name)
+        exec_cmd.add_default_arg('fv3jedi')
+        exec_cmd.add_default_arg('variational')
         exec_cmd.add_default_arg(self.task_config.jedi_yaml)
 
         try:

ush/python/pygfs/task/aero_emissions.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+
+import os
+from logging import getLogger
+from typing import Dict, Any, Union
+from pprint import pformat
+
+from wxflow import (AttrDict,
+                    parse_j2yaml,
+                    FileHandler,
+                    Jinja,
+                    logit,
+                    Task,
+                    add_to_datetime, to_timedelta,
+                    WorkflowException,
+                    Executable, which)
+
+logger = getLogger(__name__.split('.')[-1])
+
+
+class AerosolEmissions(Task):
+    """Aerosol Emissions pre-processing Task
+    """
+
+    @logit(logger, name="AerosolEmissions")
+    def __init__(self, config: Dict[str, Any]) -> None:
+        """Constructor for the Aerosol Emissions task
+
+        Parameters
+        ----------
+        config : Dict[str, Any]
+            Incoming configuration for the task from the environment
+
+        Returns
+        -------
+        None
+        """
+        super().__init__(config)
+
+        local_variable = "something"
+
+        localdict = AttrDict(
+            {'variable_used_repeatedly': local_variable}
+        )
+        self.task_config = AttrDict(**self.config, **self.runtime_config, **localdict)
+
+    @staticmethod
+    @logit(logger)
+    def initialize() -> None:
+        """Initialize the work directory
+        """
+
+    @staticmethod
+    @logit(logger)
+    def configure() -> None:
+        """Configure the artifacts in the work directory.
+        Copy run specific data to run directory
+        """
+
+    @staticmethod
+    @logit(logger)
+    def execute(workdir: Union[str, os.PathLike], aprun_cmd: str) -> None:
+        """Run the executable (if any)
+
+        Parameters
+        ----------
+        workdir : str | os.PathLike
+            work directory with the staged data, parm files, namelists, etc.
+        aprun_cmd : str
+            launcher command for executable.x
+
+        Returns
+        -------
+        None
+        """
+
+    @staticmethod
+    @logit(logger)
+    def finalize() -> None:
+        """Perform closing actions of the task.
+        Copy data back from the DATA/ directory to COM/
+        """