Merge remote-tracking branch 'origin/develop' into feature/unavailabl…

…e_rocoto * origin/develop: Jenkins Pipeline Updates (#2815) Add Gaea C5 to CI (#2814) Add support for forecast-only runs on AWS (#2711) Add fixes to products for when REPLAY IC's are used (#2755) Add capability to run forecast in segments (#2795)
NOAA-EMC · Aug 13, 2024 · 7e7fbc6 · 7e7fbc6
2 parents 4c529c3 + d994642
commit 7e7fbc6
Show file tree

Hide file tree

Showing 64 changed files with 922 additions and 322 deletions.
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
diff --git a/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml b/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml
@@ -19,5 +19,6 @@ arguments:
 
 skip_ci_on_hosts:
   - wcoss2
+  - gaea
   - orion
   - hercules
diff --git a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml
@@ -19,6 +19,7 @@ arguments:
 
 skip_ci_on_hosts:
   - hera
+  - gaea
   - orion
   - hercules
 
diff --git a/ci/cases/pr/C96_atm3DVar_extended.yaml b/ci/cases/pr/C96_atm3DVar_extended.yaml
@@ -18,5 +18,6 @@ arguments:
 
 skip_ci_on_hosts:
   - hera
+  - gaea
   - orion
   - hercules
diff --git a/ci/cases/pr/C96_atmaerosnowDA.yaml b/ci/cases/pr/C96_atmaerosnowDA.yaml
@@ -18,4 +18,5 @@ arguments:
 
 skip_ci_on_hosts:
   - orion
+  - gaea
   - hercules
diff --git a/ci/cases/yamls/gfs_extended_ci.yaml b/ci/cases/yamls/gfs_extended_ci.yaml
@@ -9,5 +9,6 @@ base:
   DO_AWIPS: "NO"
   DO_NPOESS: "YES"
   DO_GENESIS_FSU: "NO"
+  FCST_BREAKPOINTS: 192
   FHMAX_GFS: 384
   FHMAX_HF_GFS: 120
diff --git a/ci/platforms/config.gaea b/ci/platforms/config.gaea
@@ -0,0 +1,8 @@
+#!/usr/bin/bash
+
+export GFS_CI_ROOT=/gpfs/f5/epic/proj-shared/global/GFS_CI_ROOT
+export ICSDIR_ROOT=/gpfs/f5/epic/proj-shared/global/glopara/data/ICSDIR
+export STMP="/gpfs/f5/epic/scratch/${USER}"
+export SLURM_ACCOUNT=ufs-ard
+export max_concurrent_cases=5
+export max_concurrent_pr=4
diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
@@ -21,7 +21,7 @@ REPO_URL=${REPO_URL:-"[email protected]:NOAA-EMC/global-workflow.git"}
 
 source "${HOMEgfs}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
-  hera | orion | hercules | wcoss2)
+  hera | orion | hercules | wcoss2 | gaea)
    echo "Running Automated Testing on ${MACHINE_ID}"
    source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
    ;;

diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh
@@ -30,7 +30,7 @@ export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
 
 source "${ROOT_DIR}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
-  hera | orion | hercules | wcoss2)
+  hera | orion | hercules | wcoss2 | gaea)
     echo "Running Automated Testing on ${MACHINE_ID}"
     source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}"
     ;;

diff --git a/ci/scripts/driver_weekly.sh b/ci/scripts/driver_weekly.sh
@@ -38,7 +38,7 @@ export PS4='+ $(basename ${BASH_SOURCE[0]})[${LINENO}]'
 
 source "${ROOT_DIR}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
-  hera | orion | hercules | wcoss2)
+  hera | orion | hercules | wcoss2 | gaea)
     echo "Running Automated Testing on ${MACHINE_ID}"
     source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}"
     ;;

diff --git a/ci/scripts/run_ci.sh b/ci/scripts/run_ci.sh
@@ -20,7 +20,7 @@ export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
 
 source "${HOMEgfs}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
-  hera | orion | hercules | wcoss2)
+  hera | orion | hercules | wcoss2 | gaea)
    echo "Running Automated Testing on ${MACHINE_ID}"
    source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
    ;;

diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh
@@ -74,7 +74,7 @@ host=$(hostname)
 
 source "${HOMEgfs}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
-  hera | orion | hercules | wcoss2)
+  hera | orion | hercules | wcoss2 | gaea)
     echo "Launch Jenkins Java Controler on ${MACHINE_ID}";;
   *)
     echo "Unsupported platform. Exiting with error."

diff --git a/env/AWSPW.env b/env/AWSPW.env
@@ -9,8 +9,8 @@ fi
 
 step=$1
 
-export launcher="mpiexec.hydra"
-export mpmd_opt=""
+export launcher="srun -l --export=ALL"
+export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out"
 
 # Configure MPI environment
 export OMP_STACKSIZE=2048000
@@ -35,6 +35,8 @@ fi
 
 if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
 
+    export launcher="srun --mpi=pmi2 -l"
+
     (( nnodes = (ntasks+tasks_per_node-1)/tasks_per_node ))
     (( ufs_ntasks = nnodes*tasks_per_node ))
     # With ESMF threading, the model wants to use the full node

diff --git a/jobs/JGLOBAL_FORECAST b/jobs/JGLOBAL_FORECAST
@@ -116,6 +116,17 @@ fi
 # Remove the Temporary working directory
 ##########################################
 cd "${DATAROOT}" || true
-[[ "${KEEPDATA}" == "NO" ]] && rm -rf "${DATA}" "${DATArestart}"  # do not remove DATAjob. It contains DATAoutput
+# do not remove DATAjob. It contains DATAoutput
+if [[ "${KEEPDATA}" == "NO" ]]; then
+  rm -rf "${DATA}"
+
+  # Determine if this is the last segment
+  commas="${FCST_SEGMENTS//[^,]}"
+  n_segs=${#commas}
+  if (( n_segs - 1 == ${FCST_SEGMENT:-0} )); then
+    # Only delete temporary restarts if it is the last segment
+    rm -rf "${DATArestart}"
+  fi
+fi
 
 exit 0
diff --git a/modulefiles/module_base.noaacloud.lua b/modulefiles/module_base.noaacloud.lua
@@ -0,0 +1,49 @@
+help([[
+Load environment to run GFS on noaacloud
+]])
+
+local spack_mod_path=(os.getenv("spack_mod_path") or "None")
+prepend_path("MODULEPATH", spack_mod_path)
+
+load(pathJoin("stack-intel", (os.getenv("stack_intel_ver") or "None")))
+load(pathJoin("stack-intel-oneapi-mpi", (os.getenv("stack_impi_ver") or "None")))
+load(pathJoin("python", (os.getenv("python_ver") or "None")))
+
+load(pathJoin("jasper", (os.getenv("jasper_ver") or "None")))
+load(pathJoin("libpng", (os.getenv("libpng_ver") or "None")))
+load(pathJoin("cdo", (os.getenv("cdo_ver") or "None")))
+--load(pathJoin("R", (os.getenv("R_ver") or "None")))
+
+load(pathJoin("hdf5", (os.getenv("hdf5_ver") or "None")))
+load(pathJoin("netcdf-c", (os.getenv("netcdf_c_ver") or "None")))
+load(pathJoin("netcdf-fortran", (os.getenv("netcdf_fortran_ver") or "None")))
+
+load(pathJoin("nco", (os.getenv("nco_ver") or "None")))
+load(pathJoin("prod_util", (os.getenv("prod_util_ver") or "None")))
+load(pathJoin("grib-util", (os.getenv("grib_util_ver") or "None")))
+load(pathJoin("g2tmpl", (os.getenv("g2tmpl_ver") or "None")))
+load(pathJoin("gsi-ncdiag", (os.getenv("gsi_ncdiag_ver") or "None")))
+load(pathJoin("crtm", (os.getenv("crtm_ver") or "None")))
+load(pathJoin("bufr", (os.getenv("bufr_ver") or "None")))
+load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None")))
+load(pathJoin("py-f90nml", (os.getenv("py_f90nml_ver") or "None")))
+load(pathJoin("py-netcdf4", (os.getenv("py_netcdf4_ver") or "None")))
+load(pathJoin("py-pyyaml", (os.getenv("py_pyyaml_ver") or "None")))
+load(pathJoin("py-jinja2", (os.getenv("py_jinja2_ver") or "None")))
+load(pathJoin("py-pandas", (os.getenv("py_pandas_ver") or "None")))
+load(pathJoin("py-python-dateutil", (os.getenv("py_python_dateutil_ver") or "None")))
+--load(pathJoin("met", (os.getenv("met_ver") or "None")))
+--load(pathJoin("metplus", (os.getenv("metplus_ver") or "None")))
+load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None")))
+
+setenv("WGRIB2","wgrib2")
+setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None"))
+
+--prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles"))
+--prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/feature-GFSv17_com_reorg_log_update/modulefiles"))
+--load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None")))
+
+--prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles"))
+--load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None")))
+
+whatis("Description: GFS run environment")
diff --git a/modulefiles/module_gwci.noaacloud.lua b/modulefiles/module_gwci.noaacloud.lua
@@ -0,0 +1,15 @@
+help([[
+Load environment to run GFS workflow setup scripts on noaacloud
+]])
+
+prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core")
+
+load(pathJoin("stack-intel", os.getenv("2021.3.0")))
+load(pathJoin("stack-intel-oneapi-mpi", os.getenv("2021.3.0")))
+
+load(pathJoin("netcdf-c", os.getenv("4.9.2")))
+load(pathJoin("netcdf-fortran", os.getenv("4.6.1")))
+load(pathJoin("nccmp","1.9.0.1"))
+load(pathJoin("wgrib2", "2.0.8"))
+
+whatis("Description: GFS run setup CI environment")
diff --git a/modulefiles/module_gwsetup.noaacloud.lua b/modulefiles/module_gwsetup.noaacloud.lua
@@ -0,0 +1,20 @@
+help([[
+Load environment to run GFS workflow setup scripts on noaacloud
+]])
+
+load(pathJoin("rocoto"))
+
+prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core")
+
+local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.3.0"
+local python_ver=os.getenv("python_ver") or "3.10.3"
+
+load(pathJoin("stack-intel", stack_intel_ver))
+load(pathJoin("python", python_ver))
+load("py-jinja2")
+load("py-pyyaml")
+load("py-numpy")
+local git_ver=os.getenv("git_ver") or "1.8.3.1"
+load(pathJoin("git", git_ver))
+
+whatis("Description: GFS run setup environment")
diff --git a/parm/archive/enkf.yaml.j2 b/parm/archive/enkf.yaml.j2
@@ -11,7 +11,7 @@ enkf:
         {% endfor %}
         - "logs/{{ cycle_YMDH }}/{{ RUN }}echgres.log"
         - "logs/{{ cycle_YMDH }}/{{ RUN }}esfc.log"
-        {% for grp in range(iaufhrs | length) %}
+        {% for grp in range(IAUFHRS | length) %}
         - "logs/{{ cycle_YMDH }}/{{ RUN }}ecen{{ '%03d' % grp }}.log"
         {% endfor %}
 
@@ -68,7 +68,7 @@ enkf:
         {% if DOIAU %}
         # IAU increments/analyses
 
-        {% for fhr in iaufhrs if fhr != 6 %}
+        {% for fhr in IAUFHRS if fhr != 6 %}
         {% if do_calc_increment %}
         # Store analyses instead of increments
         - "{{ COMIN_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atma{{ '%03d' % fhr }}.ensmean.nc"

diff --git a/parm/archive/enkf_restarta_grp.yaml.j2 b/parm/archive/enkf_restarta_grp.yaml.j2
@@ -36,14 +36,14 @@ enkf_restarta_grp:
         {% endif %}
 
         # Member increments
-        {% for iaufhr in iaufhrs if iaufhr != 6 %}
+        {% for iaufhr in IAUFHRS if iaufhr != 6 %}
         {% set iaufhr = iaufhr %}
         {% if do_calc_increment %}
         - "{{ COMIN_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}atma{{ '%03d' % iaufhr }}.nc"
         {% else %}
         - "{{ COMIN_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}ratmi{{ '%03d' % iaufhr }}.nc"
         {% endif %}
-        {% endfor %}  # iaufhr in iaufhrs
+        {% endfor %}  # iaufhr in IAUFHRS
 
         # Conventional data
         {% if not lobsdiag_forenkf and not DO_JEDIATMENS %}

diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2
@@ -49,7 +49,7 @@ gdas:
         - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanl.ensres.nc"
                 {% if DOIAU %}
         # Ensemble IAU analysis residuals
-                    {% for fhr in iaufhrs if fhr != 6 %}
+                    {% for fhr in IAUFHRS if fhr != 6 %}
         - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atma{{ '%03d' % fhr }}.ensres.nc"
                     {% endfor %}
                 {% endif %}
@@ -108,7 +108,7 @@ gdas:
         {% endif %}  # End of cycled data
 
         # Forecast and post logs
-        - "logs/{{ cycle_YMDH }}/{{ RUN }}fcst.log"
+        - "logs/{{ cycle_YMDH }}/{{ RUN }}fcst_seg0.log"
 
         {% for fhr in range(0, FHMAX + 1, 3) %}
             {% set fhr3 = '%03d' % fhr %}

diff --git a/parm/archive/gdas_restarta.yaml.j2 b/parm/archive/gdas_restarta.yaml.j2
@@ -6,7 +6,7 @@ gdas_restarta:
         # Deterministic analysis increments
         - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atminc.nc"
         # IAU increments
-        {% for iaufhr in iaufhrs if iaufhr != 6 %}
+        {% for iaufhr in IAUFHRS if iaufhr != 6 %}
         - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmi{{ "%03d" % iaufhr }}.nc"
         {% endfor %}
 

diff --git a/parm/archive/gfs_netcdfa.yaml.j2 b/parm/archive/gfs_netcdfa.yaml.j2
@@ -6,7 +6,7 @@ gfs_netcdfa:
         - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanl.nc"
         - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}sfcanl.nc"
         - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atminc.nc"
-        {% for iauhr in iaufhrs if iauhr != 6 %}
+        {% for iauhr in IAUFHRS if iauhr != 6 %}
         - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmi{{ "%03d" % iauhr }}.nc"
         {% endfor %}
     optional:

diff --git a/parm/archive/master_enkf.yaml.j2 b/parm/archive/master_enkf.yaml.j2
@@ -4,28 +4,6 @@
 {% set cycle_YMDH = current_cycle | to_YMDH %}
 {% set head = RUN + ".t" + cycle_HH + "z." %}
 
-# Split IAUFHRS into a list; typically either "3,6,9" or 6 (integer)
-{% if IAUFHRS is string %}
-    # "3,6,9"
-    {% set iaufhrs = [] %}
-    {% for iaufhr in IAUFHRS.split(",") %}
-        {% do iaufhrs.append(iaufhr | int) %}
-    {% endfor %}
-{% else %}
-    # 6 (integer)
-    {% set iaufhrs = [IAUFHRS] %}
-{% endif %}
-
-# Repeat for IAUFHRS_ENKF
-{% if IAUFHRS_ENKF is string %}
-    {% set iaufhrs_enkf = [] %}
-    {% for iaufhr in IAUFHRS_ENKF.split(",") %}
-        {% do iaufhrs_enkf.append(iaufhr | int) %}
-    {% endfor %}
-{% else %}
-    {% set iaufhrs_enkf = [IAUFHRS_ENKF] %}
-{% endif %}
-
 # Determine which data to archive
 datasets:
 {% if ENSGRP == 0 %}

diff --git a/parm/archive/master_gdas.yaml.j2 b/parm/archive/master_gdas.yaml.j2
@@ -3,16 +3,6 @@
 {% set cycle_YMDH = current_cycle | to_YMDH %}
 {% set head = "gdas.t" + cycle_HH + "z." %}
 
-# Split IAUFHRS into a list; typically either "3,6,9" or 6 (integer)
-{% if IAUFHRS is string %}
-    {% set iaufhrs = [] %}
-    {% for iaufhr in IAUFHRS.split(",") %}
-        {% do iaufhrs.append(iaufhr | int) %}
-    {% endfor %}
-{% else %}
-    {% set iaufhrs = [IAUFHRS] %}
-{% endif %}
-
 datasets:
 # Always archive atmosphere forecast/analysis data
 {% filter indent(width=4) %}

diff --git a/parm/archive/master_gfs.yaml.j2 b/parm/archive/master_gfs.yaml.j2
@@ -3,18 +3,6 @@
 {% set cycle_YMD = current_cycle | to_YMD %}
 {% set cycle_YMDH = current_cycle | to_YMDH %}
 
-# Split IAUFHRS into a list; typically either "3,6,9" or 6 (integer)
-{% if IAUFHRS is string %}
-    # "3,6,9"
-    {% set iaufhrs = [] %}
-    {% for iaufhr in IAUFHRS.split(",") %}
-        {% do iaufhrs.append(iaufhr | int) %}
-    {% endfor %}
-{% else %}
-    # 6 (integer)
-    {% set iaufhrs = [IAUFHRS] %}
-{% endif %}
-
 # Determine which data to archive
 datasets:
 # Always archive atmosphere forecast/analysis data

diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base
@@ -229,8 +229,11 @@ export gfs_cyc=@gfs_cyc@ # 0: no GFS cycle, 1: 00Z only, 2: 00Z and 12Z only, 4:
 
 # GFS output and frequency
 export FHMIN_GFS=0
-export FHMIN=${FHMIN_GFS}
-export FHMAX_GFS=@FHMAX_GFS@
+export FHMAX_GFS="@FHMAX_GFS@"
+# Intermediate times to stop forecast when running in segments
+breakpnts="@FCST_BREAKPOINTS@"
+export FCST_SEGMENTS="${FHMIN_GFS},${breakpnts:+${breakpnts},}${FHMAX_GFS}"
+
 export FHOUT_GFS=6
 export FHMAX_HF_GFS=@FHMAX_HF_GFS@
 export FHOUT_HF_GFS=1

diff --git a/parm/config/gefs/config.extractvars b/parm/config/gefs/config.extractvars
@@ -9,12 +9,12 @@ echo "BEGIN: config.extractvars"
 
 export COMPRSCMD=${COMPRSCMD:-bzip2}
 
-export compress_ocn=0 #1: Compress extracted ocean product, 0: Do not compress extracted ocean product
-export compress_ice=0 #1: Compress extracted ice product, 0: Do not compress extracted ice product
+export compress_ocn=1 #1: Compress extracted ocean product, 0: Do not compress extracted ocean product
+export compress_ice=1 #1: Compress extracted ice product, 0: Do not compress extracted ice product
 
-export ocnres="5p00" # Resolution of ocean products
-export iceres="5p00" # Resolution of ice products
-export wavres="5p00" # Resolution of wave products
+export ocnres="1p00" # Resolution of ocean products
+export iceres="native" # Resolution of ice products
+export wavres="0p25" # Resolution of wave products
 
 export depthvar_name="z_l" # Name of depth variable in NetCDF ocean products
 export zmin="0." # Minimum depth to extract from NetCDF ocean products