From 3914c2877758887387befd547eb348ccc44d63fd Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Mon, 8 Jul 2024 16:21:30 -0500
Subject: [PATCH 01/13] update PR template for submods. Define DATAROOT with
 RUN and PDYcyc to aid in clean-up

---
 .github/pull_request_template.md | 11 +++++++++++
 parm/config/gefs/config.base     |  2 +-
 parm/config/gfs/config.base      |  2 +-
 scripts/exglobal_cleanup.sh      | 14 +++++---------
 4 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index dbebfe8f6e..317a58c2e3 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -33,6 +33,17 @@
 # Change characteristics
 - Is this a breaking change (a change in existing functionality)? YES/NO
 - Does this change require a documentation update? YES/NO
+- Does this change require an update to any of the following submodules? YES/NO
+  - [ ] UFS-weather-model
+  - [ ] UFS-UTILS
+  - [ ] GSI
+  - [ ] GSI-utils
+  - [ ] GSI-monitor
+  - [ ] GDAS
+  - [ ] EMC verif-global
+  - [ ] wxflow
+  - [ ] GFS-utils
+
 
 # How has this been tested?
 <!-- Please list any test you conducted, including the machine.
diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base
index 9808b96579..eb461ecce1 100644
--- a/parm/config/gefs/config.base
+++ b/parm/config/gefs/config.base
@@ -98,7 +98,7 @@ export PSLOT="@PSLOT@"
 export EXPDIR="@EXPDIR@/${PSLOT}"
 export ROTDIR="@COMROOT@/${PSLOT}"
 
-export DATAROOT="${STMP}/RUNDIRS/${PSLOT}"  # TODO: set via prod_envir in Ops
+export DATAROOT="${STMP}/RUNDIRS/${PSLOT}/${RUN}.${PDY}${cyc}"  # TODO: set via prod_envir in Ops
 export ARCDIR="${NOSCRUB}/archive/${PSLOT}"
 export ATARDIR="@ATARDIR@"
 
diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base
index 9fd66bf0c7..95a028a789 100644
--- a/parm/config/gfs/config.base
+++ b/parm/config/gfs/config.base
@@ -128,7 +128,7 @@ export DUMP_SUFFIX=""
 if [[ "${PDY}${cyc}" -ge "2019092100" && "${PDY}${cyc}" -le "2019110700" ]]; then
     export DUMP_SUFFIX="p"              # Use dumps from NCO GFS v15.3 parallel
 fi
-export DATAROOT="${STMP}/RUNDIRS/${PSLOT}"  # TODO: set via prod_envir in Ops
+export DATAROOT="${STMP}/RUNDIRS/${PSLOT}/${RUN}.${PDY}${cyc}"  # TODO: set via prod_envir in Ops
 export ARCDIR="${NOSCRUB}/archive/${PSLOT}"
 export ATARDIR="@ATARDIR@"
 
diff --git a/scripts/exglobal_cleanup.sh b/scripts/exglobal_cleanup.sh
index dcf1baef31..bfdce0fe06 100755
--- a/scripts/exglobal_cleanup.sh
+++ b/scripts/exglobal_cleanup.sh
@@ -12,15 +12,11 @@ if [[ -d "${DATAfcst}" ]]; then rm -rf "${DATAfcst}"; fi
 #DATAefcs="${DATAROOT}/${RUN}efcs???${PDY:-}${cyc}"
 rm -rf "${DATAROOT}/${RUN}efcs"*"${PDY:-}${cyc}"
 
-# Search and delete files/directories from DATAROOT/ older than ${purge_every_days} days
-# purge_every_days should be a positive integer
-#purge_every_days=3
-
-# Find and delete files older than ${purge_every_days} days
-#find "${DATAROOT}/"* -type f -mtime "+${purge_every_days}" -exec rm -f {} \;
-
-# Find and delete directories older than ${purge_every_days} days
-#find "${DATAROOT}/"* -type d -mtime "+${purge_every_days}" -exec rm -rf {} \;
+# In config.base, DATAROOT is defined as:
+#DATAROOT="${STMP}/RUNDIRS/${PSLOT}/${RUN}.${PDY}${cyc}"
+# cleanup is only executed after the entire cycle is successfully completed.
+# removing DATAROOT should be possible if that is the case.
+rm -rf "${DATAROOT}"
 
 echo "Cleanup ${DATAROOT} completed!"
 ###############################################################

From 72933ecab8cb59feb27573e0148d93942938fa62 Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Tue, 9 Jul 2024 16:05:16 -0400
Subject: [PATCH 02/13] source configs in jjob_header before creating DATA.
 remove DATAROOT from tasks.py

---
 ush/jjob_header.sh                         | 51 +++++++++++-----------
 workflow/applications/gfs_forecast_only.py |  2 +-
 workflow/rocoto/tasks.py                   |  3 +-
 3 files changed, 27 insertions(+), 29 deletions(-)

diff --git a/ush/jjob_header.sh b/ush/jjob_header.sh
index 45fa6402ae..6545a7aa79 100755
--- a/ush/jjob_header.sh
+++ b/ush/jjob_header.sh
@@ -39,7 +39,7 @@
 #                     [default: "YES"]
 #   - $pid          : Override the default process id
 #                     [default: $$]
-# 
+#
 
 OPTIND=1
 while getopts "c:e:" option; do
@@ -63,6 +63,30 @@ if [[ -z ${env_job} ]]; then
     exit 1
 fi
 
+#############################
+# Source relevant config files
+#############################
+export EXPDIR="${EXPDIR:-${HOMEgfs}/parm/config}"
+for config in "${configs[@]:-''}"; do
+    source "${EXPDIR}/config.${config}"
+    status=$?
+    if (( status != 0 )); then
+        echo "FATAL [${BASH_SOURCE[0]}]: Unable to load config config.${config}"
+        exit "${status}"
+    fi
+done
+
+
+##########################################
+# Source machine runtime environment
+##########################################
+source "${HOMEgfs}/env/${machine}.env" "${env_job}"
+status=$?
+if (( status != 0 )); then
+    echo "FATAL [${BASH_SOURCE[0]}]: Error while sourcing machine environment ${machine}.env for job ${env_job}"
+    exit "${status}"
+fi
+
 ##############################################
 # make temp directory
 ##############################################
@@ -88,28 +112,3 @@ source ./PDY
 export pid="${pid:-$$}"
 export pgmout="OUTPUT.${pid}"
 export pgmerr=errfile
-
-
-#############################
-# Source relevant config files
-#############################
-export EXPDIR="${EXPDIR:-${HOMEgfs}/parm/config}"
-for config in "${configs[@]:-''}"; do
-    source "${EXPDIR}/config.${config}"
-    status=$?
-    if (( status != 0 )); then
-        echo "FATAL [${BASH_SOURCE[0]}]: Unable to load config config.${config}"
-    	exit "${status}"
-    fi
-done
-
-
-##########################################
-# Source machine runtime environment
-##########################################
-source "${HOMEgfs}/env/${machine}.env" "${env_job}"
-status=$?
-if (( status != 0 )); then
-    echo "FATAL [${BASH_SOURCE[0]}]: Error while sourcing machine environment ${machine}.env for job ${env_job}"
-	exit "${status}"
-fi
diff --git a/workflow/applications/gfs_forecast_only.py b/workflow/applications/gfs_forecast_only.py
index e219032551..a82380eb59 100644
--- a/workflow/applications/gfs_forecast_only.py
+++ b/workflow/applications/gfs_forecast_only.py
@@ -87,7 +87,7 @@ def get_task_names(self):
         tasks = ['stage_ic']
 
         if self.do_aero:
-            aero_fcst_cdump = _base.get('AERO_FCST_CDUMP', 'BOTH').lower()
+            aero_fcst_cdump = self._base.get('AERO_FCST_CDUMP', 'BOTH').lower()
             if self._base['CDUMP'] in aero_fcst_cdump or aero_fcst_cdump == "both":
                 tasks += ['aerosol_init']
 
diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py
index 097d1adef5..966fc361af 100644
--- a/workflow/rocoto/tasks.py
+++ b/workflow/rocoto/tasks.py
@@ -64,8 +64,7 @@ def __init__(self, app_config: AppConfig, cdump: str) -> None:
                       'CDATE': '<cyclestr>@Y@m@d@H</cyclestr>',
                       'PDY': '<cyclestr>@Y@m@d</cyclestr>',
                       'cyc': '<cyclestr>@H</cyclestr>',
-                      'COMROOT': self._base.get('COMROOT'),
-                      'DATAROOT': self._base.get('DATAROOT')}
+                      'COMROOT': self._base.get('COMROOT')}
 
         self.envars = self._set_envars(envar_dict)
 

From 3a587dd0c526bb93ebf8bb2198c2c6934e1d5101 Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Tue, 9 Jul 2024 16:28:13 -0400
Subject: [PATCH 03/13] missed aerosol forecast in GFS

---
 workflow/applications/applications.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/workflow/applications/applications.py b/workflow/applications/applications.py
index 6a4d240fe5..64440830c1 100644
--- a/workflow/applications/applications.py
+++ b/workflow/applications/applications.py
@@ -79,12 +79,18 @@ def __init__(self, conf: Configuration) -> None:
                 self.wave_cdumps = [wave_cdump]
 
         self.aero_anl_cdumps = None
+        self.aero_fcst_cdumps = None
         if self.do_aero:
             aero_anl_cdump = _base.get('AERO_ANL_CDUMP', 'BOTH').lower()
             if aero_anl_cdump in ['both']:
                 self.aero_anl_cdumps = ['gfs', 'gdas']
             elif aero_anl_cdump in ['gfs', 'gdas']:
                 self.aero_anl_cdumps = [aero_anl_cdump]
+            aero_fcst_cdump = _base.get('AERO_FCST_CDUMP', None).lower()
+            if aero_fcst_cdump in ['both']:
+                self.aero_fcst_cdumps = ['gfs', 'gdas']
+            elif aero_fcst_cdump in ['gfs', 'gdas']:
+                self.aero_fcst_cdumps = [aero_fcst_cdump]
 
     def _init_finalize(self, conf: Configuration):
         print("Finalizing initialize")

From 579130ad303272a45de911b2e0d5eed22a73a7f9 Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Tue, 9 Jul 2024 16:48:23 -0400
Subject: [PATCH 04/13] aerosol_init should not be called when it is a warm
 start

---
 workflow/applications/gfs_forecast_only.py | 6 ++++--
 workflow/rocoto/gfs_tasks.py               | 3 ++-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/workflow/applications/gfs_forecast_only.py b/workflow/applications/gfs_forecast_only.py
index a82380eb59..1e2d14b948 100644
--- a/workflow/applications/gfs_forecast_only.py
+++ b/workflow/applications/gfs_forecast_only.py
@@ -25,7 +25,8 @@ def _get_app_configs(self):
             configs += ['atmos_products']
 
             if self.do_aero:
-                configs += ['aerosol_init']
+                if self._base['EXP_WARM_START'] in ['.false.']:
+                    configs += ['aerosol_init']
 
             if self.do_tracker:
                 configs += ['tracker']
@@ -89,7 +90,8 @@ def get_task_names(self):
         if self.do_aero:
             aero_fcst_cdump = self._base.get('AERO_FCST_CDUMP', 'BOTH').lower()
             if self._base['CDUMP'] in aero_fcst_cdump or aero_fcst_cdump == "both":
-                tasks += ['aerosol_init']
+                if self._base['EXP_WARM_START'] in ['.false.']:
+                    tasks += ['aerosol_init']
 
         if self.do_wave:
             tasks += ['waveinit']
diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py
index 530ea465c4..2c3896db36 100644
--- a/workflow/rocoto/gfs_tasks.py
+++ b/workflow/rocoto/gfs_tasks.py
@@ -856,7 +856,8 @@ def _fcst_forecast_only(self):
             dep_dict = {'type': 'task', 'name': f'{self.cdump}{wave_job}'}
             dependencies.append(rocoto.add_dependency(dep_dict))
 
-        if self.app_config.do_aero and self.cdump in self.app_config.aero_fcst_cdumps:
+        if self.app_config.do_aero and self.cdump in self.app_config.aero_fcst_cdumps and \
+            self._base['EXP_WARM_START'] in ['.false.']:
             # Calculate offset based on CDUMP = gfs | gdas
             interval = None
             if self.cdump in ['gfs']:

From 94abc57084d436831e4d066184410169444864c6 Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Tue, 9 Jul 2024 16:52:57 -0400
Subject: [PATCH 05/13] fix pynorms

---
 workflow/rocoto/gfs_tasks.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py
index 2c3896db36..0a8f3c1a13 100644
--- a/workflow/rocoto/gfs_tasks.py
+++ b/workflow/rocoto/gfs_tasks.py
@@ -856,8 +856,9 @@ def _fcst_forecast_only(self):
             dep_dict = {'type': 'task', 'name': f'{self.cdump}{wave_job}'}
             dependencies.append(rocoto.add_dependency(dep_dict))
 
-        if self.app_config.do_aero and self.cdump in self.app_config.aero_fcst_cdumps and \
-            self._base['EXP_WARM_START'] in ['.false.']:
+        if self.app_config.do_aero and \
+           self.cdump in self.app_config.aero_fcst_cdumps and \
+           self._base['EXP_WARM_START'] in ['.false.']:
             # Calculate offset based on CDUMP = gfs | gdas
             interval = None
             if self.cdump in ['gfs']:

From 5fc21935821bb7927d5d3718b28da4a90bf77a4c Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Thu, 11 Jul 2024 09:27:37 -0500
Subject: [PATCH 06/13] revert GFS f-o changes related to app=S2SWA

---
 workflow/applications/applications.py      | 6 ------
 workflow/applications/gfs_forecast_only.py | 8 +++-----
 workflow/rocoto/gfs_tasks.py               | 4 +---
 3 files changed, 4 insertions(+), 14 deletions(-)

diff --git a/workflow/applications/applications.py b/workflow/applications/applications.py
index 64440830c1..6a4d240fe5 100644
--- a/workflow/applications/applications.py
+++ b/workflow/applications/applications.py
@@ -79,18 +79,12 @@ def __init__(self, conf: Configuration) -> None:
                 self.wave_cdumps = [wave_cdump]
 
         self.aero_anl_cdumps = None
-        self.aero_fcst_cdumps = None
         if self.do_aero:
             aero_anl_cdump = _base.get('AERO_ANL_CDUMP', 'BOTH').lower()
             if aero_anl_cdump in ['both']:
                 self.aero_anl_cdumps = ['gfs', 'gdas']
             elif aero_anl_cdump in ['gfs', 'gdas']:
                 self.aero_anl_cdumps = [aero_anl_cdump]
-            aero_fcst_cdump = _base.get('AERO_FCST_CDUMP', None).lower()
-            if aero_fcst_cdump in ['both']:
-                self.aero_fcst_cdumps = ['gfs', 'gdas']
-            elif aero_fcst_cdump in ['gfs', 'gdas']:
-                self.aero_fcst_cdumps = [aero_fcst_cdump]
 
     def _init_finalize(self, conf: Configuration):
         print("Finalizing initialize")
diff --git a/workflow/applications/gfs_forecast_only.py b/workflow/applications/gfs_forecast_only.py
index 1e2d14b948..e219032551 100644
--- a/workflow/applications/gfs_forecast_only.py
+++ b/workflow/applications/gfs_forecast_only.py
@@ -25,8 +25,7 @@ def _get_app_configs(self):
             configs += ['atmos_products']
 
             if self.do_aero:
-                if self._base['EXP_WARM_START'] in ['.false.']:
-                    configs += ['aerosol_init']
+                configs += ['aerosol_init']
 
             if self.do_tracker:
                 configs += ['tracker']
@@ -88,10 +87,9 @@ def get_task_names(self):
         tasks = ['stage_ic']
 
         if self.do_aero:
-            aero_fcst_cdump = self._base.get('AERO_FCST_CDUMP', 'BOTH').lower()
+            aero_fcst_cdump = _base.get('AERO_FCST_CDUMP', 'BOTH').lower()
             if self._base['CDUMP'] in aero_fcst_cdump or aero_fcst_cdump == "both":
-                if self._base['EXP_WARM_START'] in ['.false.']:
-                    tasks += ['aerosol_init']
+                tasks += ['aerosol_init']
 
         if self.do_wave:
             tasks += ['waveinit']
diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py
index 0a8f3c1a13..530ea465c4 100644
--- a/workflow/rocoto/gfs_tasks.py
+++ b/workflow/rocoto/gfs_tasks.py
@@ -856,9 +856,7 @@ def _fcst_forecast_only(self):
             dep_dict = {'type': 'task', 'name': f'{self.cdump}{wave_job}'}
             dependencies.append(rocoto.add_dependency(dep_dict))
 
-        if self.app_config.do_aero and \
-           self.cdump in self.app_config.aero_fcst_cdumps and \
-           self._base['EXP_WARM_START'] in ['.false.']:
+        if self.app_config.do_aero and self.cdump in self.app_config.aero_fcst_cdumps:
             # Calculate offset based on CDUMP = gfs | gdas
             interval = None
             if self.cdump in ['gfs']:

From eaf9a44a214e488186bacaca92e2ada194e6ad55 Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Tue, 23 Jul 2024 11:18:12 -0400
Subject: [PATCH 07/13] order matters in jjob_header.  retain it.  Use the
 suggestion from DH on constructing DATAROOT in tasks, and remove it from
 config.base

---
 parm/config/gefs/config.base |  1 -
 parm/config/gfs/config.base  |  1 -
 ush/jjob_header.sh           | 50 ++++++++++++++++++------------------
 workflow/rocoto/tasks.py     |  5 +++-
 4 files changed, 29 insertions(+), 28 deletions(-)

diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base
index eb461ecce1..793f68abdd 100644
--- a/parm/config/gefs/config.base
+++ b/parm/config/gefs/config.base
@@ -98,7 +98,6 @@ export PSLOT="@PSLOT@"
 export EXPDIR="@EXPDIR@/${PSLOT}"
 export ROTDIR="@COMROOT@/${PSLOT}"
 
-export DATAROOT="${STMP}/RUNDIRS/${PSLOT}/${RUN}.${PDY}${cyc}"  # TODO: set via prod_envir in Ops
 export ARCDIR="${NOSCRUB}/archive/${PSLOT}"
 export ATARDIR="@ATARDIR@"
 
diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base
index 95a028a789..59e9e0f625 100644
--- a/parm/config/gfs/config.base
+++ b/parm/config/gfs/config.base
@@ -128,7 +128,6 @@ export DUMP_SUFFIX=""
 if [[ "${PDY}${cyc}" -ge "2019092100" && "${PDY}${cyc}" -le "2019110700" ]]; then
     export DUMP_SUFFIX="p"              # Use dumps from NCO GFS v15.3 parallel
 fi
-export DATAROOT="${STMP}/RUNDIRS/${PSLOT}/${RUN}.${PDY}${cyc}"  # TODO: set via prod_envir in Ops
 export ARCDIR="${NOSCRUB}/archive/${PSLOT}"
 export ATARDIR="@ATARDIR@"
 
diff --git a/ush/jjob_header.sh b/ush/jjob_header.sh
index 6545a7aa79..dc75437f1a 100755
--- a/ush/jjob_header.sh
+++ b/ush/jjob_header.sh
@@ -39,7 +39,6 @@
 #                     [default: "YES"]
 #   - $pid          : Override the default process id
 #                     [default: $$]
-#
 
 OPTIND=1
 while getopts "c:e:" option; do
@@ -63,30 +62,6 @@ if [[ -z ${env_job} ]]; then
     exit 1
 fi
 
-#############################
-# Source relevant config files
-#############################
-export EXPDIR="${EXPDIR:-${HOMEgfs}/parm/config}"
-for config in "${configs[@]:-''}"; do
-    source "${EXPDIR}/config.${config}"
-    status=$?
-    if (( status != 0 )); then
-        echo "FATAL [${BASH_SOURCE[0]}]: Unable to load config config.${config}"
-        exit "${status}"
-    fi
-done
-
-
-##########################################
-# Source machine runtime environment
-##########################################
-source "${HOMEgfs}/env/${machine}.env" "${env_job}"
-status=$?
-if (( status != 0 )); then
-    echo "FATAL [${BASH_SOURCE[0]}]: Error while sourcing machine environment ${machine}.env for job ${env_job}"
-    exit "${status}"
-fi
-
 ##############################################
 # make temp directory
 ##############################################
@@ -112,3 +87,28 @@ source ./PDY
 export pid="${pid:-$$}"
 export pgmout="OUTPUT.${pid}"
 export pgmerr=errfile
+
+
+#############################
+# Source relevant config files
+#############################
+export EXPDIR="${EXPDIR:-${HOMEgfs}/parm/config}"
+for config in "${configs[@]:-''}"; do
+    source "${EXPDIR}/config.${config}"
+    status=$?
+    if (( status != 0 )); then
+        echo "FATAL [${BASH_SOURCE[0]}]: Unable to load config config.${config}"
+        exit "${status}"
+    fi
+done
+
+
+##########################################
+# Source machine runtime environment
+##########################################
+source "${HOMEgfs}/env/${machine}.env" "${env_job}"
+status=$?
+if (( status != 0 )); then
+    echo "FATAL [${BASH_SOURCE[0]}]: Error while sourcing machine environment ${machine}.env for job ${env_job}"
+    exit "${status}"
+fi
diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py
index 966fc361af..5b0338e226 100644
--- a/workflow/rocoto/tasks.py
+++ b/workflow/rocoto/tasks.py
@@ -55,6 +55,8 @@ def __init__(self, app_config: AppConfig, cdump: str) -> None:
 
         self.n_tiles = 6  # TODO - this needs to be elsewhere
 
+        # DATAROOT is set by prod_envir in ops.  Here, we use `STMP` to construct DATAROOT
+        dataroot_str = f"{self._base.get('STMP')}/RUNDIRS/{self._base.get('PSLOT')}/{self.cdump}.<cyclestr>@Y@m@d@H</cyclestr>"
         envar_dict = {'RUN_ENVIR': self._base.get('RUN_ENVIR', 'emc'),
                       'HOMEgfs': self.HOMEgfs,
                       'EXPDIR': self._base.get('EXPDIR'),
@@ -64,7 +66,8 @@ def __init__(self, app_config: AppConfig, cdump: str) -> None:
                       'CDATE': '<cyclestr>@Y@m@d@H</cyclestr>',
                       'PDY': '<cyclestr>@Y@m@d</cyclestr>',
                       'cyc': '<cyclestr>@H</cyclestr>',
-                      'COMROOT': self._base.get('COMROOT')}
+                      'COMROOT': self._base.get('COMROOT'),
+                      'DATAROOT': dataroot_str}
 
         self.envars = self._set_envars(envar_dict)
 

From 6cc64ceca1087cb23e5d0baee1deb3e85e9124a3 Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Tue, 23 Jul 2024 11:21:24 -0400
Subject: [PATCH 08/13] update comment to be correct

---
 scripts/exglobal_cleanup.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/exglobal_cleanup.sh b/scripts/exglobal_cleanup.sh
index bfdce0fe06..bc0816f3d6 100755
--- a/scripts/exglobal_cleanup.sh
+++ b/scripts/exglobal_cleanup.sh
@@ -12,7 +12,7 @@ if [[ -d "${DATAfcst}" ]]; then rm -rf "${DATAfcst}"; fi
 #DATAefcs="${DATAROOT}/${RUN}efcs???${PDY:-}${cyc}"
 rm -rf "${DATAROOT}/${RUN}efcs"*"${PDY:-}${cyc}"
 
-# In config.base, DATAROOT is defined as:
+# In XML, DATAROOT is defined as:
 #DATAROOT="${STMP}/RUNDIRS/${PSLOT}/${RUN}.${PDY}${cyc}"
 # cleanup is only executed after the entire cycle is successfully completed.
 # removing DATAROOT should be possible if that is the case.

From 7b10afcfd0a06a003e8743681e0f6b7acd6138b9 Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Tue, 30 Jul 2024 09:55:30 -0400
Subject: [PATCH 09/13] alphabetize and add note on pending PRs

---
 .github/pull_request_template.md | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index 317a58c2e3..3f8fe65065 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -33,16 +33,16 @@
 # Change characteristics
 - Is this a breaking change (a change in existing functionality)? YES/NO
 - Does this change require a documentation update? YES/NO
-- Does this change require an update to any of the following submodules? YES/NO
-  - [ ] UFS-weather-model
-  - [ ] UFS-UTILS
+- Does this change require an update to any of the following submodules? YES/NO (If YES, please add a link to any PRs that are pending.)
+  - [ ] EMC verif-global
+  - [ ] GDAS
+  - [ ] GFS-utils
   - [ ] GSI
-  - [ ] GSI-utils
   - [ ] GSI-monitor
-  - [ ] GDAS
-  - [ ] EMC verif-global
+  - [ ] GSI-utils
+  - [ ] UFS-utils
+  - [ ] UFS-weather-model
   - [ ] wxflow
-  - [ ] GFS-utils
 
 
 # How has this been tested?

From b1e6dbb0f24063080f9a7c438577c9fc68f63086 Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Tue, 30 Jul 2024 15:25:12 -0400
Subject: [PATCH 10/13] clean-up and correct the issue in JGLOBAL_CLEANUP

---
 jobs/JGLOBAL_CLEANUP        |  6 +++---
 scripts/exglobal_cleanup.sh | 19 ++++---------------
 2 files changed, 7 insertions(+), 18 deletions(-)

diff --git a/jobs/JGLOBAL_CLEANUP b/jobs/JGLOBAL_CLEANUP
index 2f81003989..f3cfcae511 100755
--- a/jobs/JGLOBAL_CLEANUP
+++ b/jobs/JGLOBAL_CLEANUP
@@ -5,13 +5,13 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "cleanup" -c "base cleanup"
 
 "${SCRgfs}/exglobal_cleanup.sh"
 status=$?
-[[ ${status} -ne 0 ]] && exit "${status}"
+(( status != 0 )) && exit "${status}"
 
 ##########################################
 # Remove the Temporary working directory
 ##########################################
-cd "${DATAROOT}" || (echo "${DATAROOT} does not exist. ABORT!"; exit 1)
-[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}"
+# DATAROOT="${STMP}/RUNDIRS/${PSLOT}/${RUN}.${PDY}${cyc}"
+# is removed in exglobal_cleanup.sh, nothing to do here.
 
 exit 0
 
diff --git a/scripts/exglobal_cleanup.sh b/scripts/exglobal_cleanup.sh
index bc0816f3d6..75b1f927bc 100755
--- a/scripts/exglobal_cleanup.sh
+++ b/scripts/exglobal_cleanup.sh
@@ -21,28 +21,17 @@ rm -rf "${DATAROOT}"
 echo "Cleanup ${DATAROOT} completed!"
 ###############################################################
 
-###############################################################
-# Clean up previous cycles; various depths
-# PRIOR CYCLE: Leave the prior cycle alone
-# shellcheck disable=SC2153
-GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${assim_freq} hours")
-# PREVIOUS to the PRIOR CYCLE
-GDATE=$(date --utc +%Y%m%d%H -d "${GDATE:0:8} ${GDATE:8:2} -${assim_freq} hours")
-
-# Remove the TMPDIR directory
-# TODO Only prepbufr is currently using this directory, and all jobs should be
-#   cleaning up after themselves anyway
-COMIN="${DATAROOT}/${GDATE}"
-[[ -d ${COMIN} ]] && rm -rf "${COMIN}"
-
 if [[ "${CLEANUP_COM:-YES}" == NO ]] ; then
     exit 0
 fi
 
+###############################################################
+# Clean up previous cycles; various depths
+
 # Step back every assim_freq hours and remove old rotating directories
 # for successful cycles (defaults from 24h to 120h).
 # Retain files needed by Fit2Obs
-last_date=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${RMOLDEND:-24} hours" )
+last_date=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${RMOLDEND:-24} hours")
 first_date=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${RMOLDSTD:-120} hours")
 last_rtofs=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${RMOLDRTOFS:-48} hours")
 function remove_files() {

From 2ebfcd82dbc4c774507299cb71b785044ab89639 Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Wed, 31 Jul 2024 10:50:32 -0400
Subject: [PATCH 11/13] Update workflow/rocoto/tasks.py

Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com>
---
 workflow/rocoto/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py
index 1bb49bd8e1..db21298a25 100644
--- a/workflow/rocoto/tasks.py
+++ b/workflow/rocoto/tasks.py
@@ -62,7 +62,7 @@ def __init__(self, app_config: AppConfig, run: str) -> None:
         self.n_tiles = 6  # TODO - this needs to be elsewhere
 
         # DATAROOT is set by prod_envir in ops.  Here, we use `STMP` to construct DATAROOT
-        dataroot_str = f"{self._base.get('STMP')}/RUNDIRS/{self._base.get('PSLOT')}/{self.cdump}.<cyclestr>@Y@m@d@H</cyclestr>"
+        dataroot_str = f"{self._base.get('STMP')}/RUNDIRS/{self._base.get('PSLOT')}/{self.run}.<cyclestr>@Y@m@d@H</cyclestr>"
         envar_dict = {'RUN_ENVIR': self._base.get('RUN_ENVIR', 'emc'),
                       'HOMEgfs': self.HOMEgfs,
                       'EXPDIR': self._base.get('EXPDIR'),

From e785f36829b4ced5093aeac3092fec79b8d6352c Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Thu, 1 Aug 2024 13:44:31 +0000
Subject: [PATCH 12/13] Make all jobs source config.resources.

---
 jobs/JGFS_ATMOS_FBWIND            |  2 +-
 jobs/JGFS_ATMOS_GEMPAK_META       |  2 +-
 jobs/JGFS_ATMOS_GEMPAK_PGRB2_SPEC |  2 +-
 parm/config/gfs/config.fbwind     | 11 +++++++++++
 parm/config/gfs/config.resources  |  7 +++++++
 5 files changed, 21 insertions(+), 3 deletions(-)
 create mode 100644 parm/config/gfs/config.fbwind

diff --git a/jobs/JGFS_ATMOS_FBWIND b/jobs/JGFS_ATMOS_FBWIND
index 38e524afea..2071954b0d 100755
--- a/jobs/JGFS_ATMOS_FBWIND
+++ b/jobs/JGFS_ATMOS_FBWIND
@@ -6,7 +6,7 @@
 # GFS FBWIND PRODUCT GENERATION
 ############################################
 source "${HOMEgfs}/ush/preamble.sh"
-source "${HOMEgfs}/ush/jjob_header.sh" -e "fbwind" -c "base"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "fbwind" -c "base fbwind"
 
 ###################################
 # Specify NET and RUN Name and model
diff --git a/jobs/JGFS_ATMOS_GEMPAK_META b/jobs/JGFS_ATMOS_GEMPAK_META
index 3af463e241..480dc0f1d6 100755
--- a/jobs/JGFS_ATMOS_GEMPAK_META
+++ b/jobs/JGFS_ATMOS_GEMPAK_META
@@ -6,7 +6,7 @@
 # GFS GEMPAK META PRODUCT GENERATION
 ############################################
 source "${HOMEgfs}/ush/preamble.sh"
-source "${HOMEgfs}/ush/jjob_header.sh" -e "gempak_meta" -c "base"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "gempak_meta" -c "base gempak"
 
 
 ###############################################
diff --git a/jobs/JGFS_ATMOS_GEMPAK_PGRB2_SPEC b/jobs/JGFS_ATMOS_GEMPAK_PGRB2_SPEC
index 0be52c0d19..a82d2805eb 100755
--- a/jobs/JGFS_ATMOS_GEMPAK_PGRB2_SPEC
+++ b/jobs/JGFS_ATMOS_GEMPAK_PGRB2_SPEC
@@ -1,7 +1,7 @@
 #! /usr/bin/env bash
 
 source "${HOMEgfs}/ush/preamble.sh"
-source "${HOMEgfs}/ush/jjob_header.sh" -e "gempak_spec" -c "base"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "gempak_spec" -c "base gempak"
 
 ############################################
 # Set up model and cycle specific variables
diff --git a/parm/config/gfs/config.fbwind b/parm/config/gfs/config.fbwind
new file mode 100644
index 0000000000..c96f35bf7a
--- /dev/null
+++ b/parm/config/gfs/config.fbwind
@@ -0,0 +1,11 @@
+#! /usr/bin/env bash
+
+########## config.gempak ##########
+# GFS fbwind step specific
+
+echo "BEGIN: config.fbwind"
+
+# Get task specific resources
+. $EXPDIR/config.resources fbwind
+
+echo "END: config.fbwind"
diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
index ebdfd5d713..ad23fec434 100644
--- a/parm/config/gfs/config.resources
+++ b/parm/config/gfs/config.resources
@@ -1124,6 +1124,13 @@ case ${step} in
     memory_gfs="2GB"
     ;;
 
+  "fbwind")
+    walltime="00:05:00"
+    ntasks=1
+    threads_per_task=1
+    memory="4GB"
+    ;;
+
   "mos_stn_prep")
     walltime="00:10:00"
     ntasks=3

From 3c02bc025eb25c5438015bcff354c1903dc5e234 Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Thu, 1 Aug 2024 10:40:22 -0400
Subject: [PATCH 13/13] Update parm/config/gfs/config.fbwind

---
 parm/config/gfs/config.fbwind | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parm/config/gfs/config.fbwind b/parm/config/gfs/config.fbwind
index c96f35bf7a..49fdb9e7b4 100644
--- a/parm/config/gfs/config.fbwind
+++ b/parm/config/gfs/config.fbwind
@@ -6,6 +6,6 @@
 echo "BEGIN: config.fbwind"
 
 # Get task specific resources
-. $EXPDIR/config.resources fbwind
+source "${EXPDIR}/config.resources" fbwind
 
 echo "END: config.fbwind"