Merge branch 'develop' into feature/aero-b

CoryMartin-NOAA · Aug 13, 2024 · 1d4c5ec · 1d4c5ec
2 parents 2af8980 + eba813f
commit 1d4c5ec
Show file tree

Hide file tree

Showing 53 changed files with 827 additions and 231 deletions.
diff --git a/ci/cases/yamls/gfs_extended_ci.yaml b/ci/cases/yamls/gfs_extended_ci.yaml
@@ -9,5 +9,6 @@ base:
   DO_AWIPS: "NO"
   DO_NPOESS: "YES"
   DO_GENESIS_FSU: "NO"
+  FCST_BREAKPOINTS: 192
   FHMAX_GFS: 384
   FHMAX_HF_GFS: 120
diff --git a/env/AWSPW.env b/env/AWSPW.env
@@ -9,8 +9,8 @@ fi
 
 step=$1
 
-export launcher="mpiexec.hydra"
-export mpmd_opt=""
+export launcher="srun -l --export=ALL"
+export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out"
 
 # Configure MPI environment
 export OMP_STACKSIZE=2048000
@@ -35,6 +35,8 @@ fi
 
 if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
 
+    export launcher="srun --mpi=pmi2 -l"
+
     (( nnodes = (ntasks+tasks_per_node-1)/tasks_per_node ))
     (( ufs_ntasks = nnodes*tasks_per_node ))
     # With ESMF threading, the model wants to use the full node

diff --git a/jobs/JGLOBAL_FORECAST b/jobs/JGLOBAL_FORECAST
@@ -116,6 +116,17 @@ fi
 # Remove the Temporary working directory
 ##########################################
 cd "${DATAROOT}" || true
-[[ "${KEEPDATA}" == "NO" ]] && rm -rf "${DATA}" "${DATArestart}"  # do not remove DATAjob. It contains DATAoutput
+# do not remove DATAjob. It contains DATAoutput
+if [[ "${KEEPDATA}" == "NO" ]]; then
+  rm -rf "${DATA}"
+
+  # Determine if this is the last segment
+  commas="${FCST_SEGMENTS//[^,]}"
+  n_segs=${#commas}
+  if (( n_segs - 1 == ${FCST_SEGMENT:-0} )); then
+    # Only delete temporary restarts if it is the last segment
+    rm -rf "${DATArestart}"
+  fi
+fi
 
 exit 0
diff --git a/modulefiles/module_base.noaacloud.lua b/modulefiles/module_base.noaacloud.lua
@@ -0,0 +1,49 @@
+help([[
+Load environment to run GFS on noaacloud
+]])
+
+local spack_mod_path=(os.getenv("spack_mod_path") or "None")
+prepend_path("MODULEPATH", spack_mod_path)
+
+load(pathJoin("stack-intel", (os.getenv("stack_intel_ver") or "None")))
+load(pathJoin("stack-intel-oneapi-mpi", (os.getenv("stack_impi_ver") or "None")))
+load(pathJoin("python", (os.getenv("python_ver") or "None")))
+
+load(pathJoin("jasper", (os.getenv("jasper_ver") or "None")))
+load(pathJoin("libpng", (os.getenv("libpng_ver") or "None")))
+load(pathJoin("cdo", (os.getenv("cdo_ver") or "None")))
+--load(pathJoin("R", (os.getenv("R_ver") or "None")))
+
+load(pathJoin("hdf5", (os.getenv("hdf5_ver") or "None")))
+load(pathJoin("netcdf-c", (os.getenv("netcdf_c_ver") or "None")))
+load(pathJoin("netcdf-fortran", (os.getenv("netcdf_fortran_ver") or "None")))
+
+load(pathJoin("nco", (os.getenv("nco_ver") or "None")))
+load(pathJoin("prod_util", (os.getenv("prod_util_ver") or "None")))
+load(pathJoin("grib-util", (os.getenv("grib_util_ver") or "None")))
+load(pathJoin("g2tmpl", (os.getenv("g2tmpl_ver") or "None")))
+load(pathJoin("gsi-ncdiag", (os.getenv("gsi_ncdiag_ver") or "None")))
+load(pathJoin("crtm", (os.getenv("crtm_ver") or "None")))
+load(pathJoin("bufr", (os.getenv("bufr_ver") or "None")))
+load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None")))
+load(pathJoin("py-f90nml", (os.getenv("py_f90nml_ver") or "None")))
+load(pathJoin("py-netcdf4", (os.getenv("py_netcdf4_ver") or "None")))
+load(pathJoin("py-pyyaml", (os.getenv("py_pyyaml_ver") or "None")))
+load(pathJoin("py-jinja2", (os.getenv("py_jinja2_ver") or "None")))
+load(pathJoin("py-pandas", (os.getenv("py_pandas_ver") or "None")))
+load(pathJoin("py-python-dateutil", (os.getenv("py_python_dateutil_ver") or "None")))
+--load(pathJoin("met", (os.getenv("met_ver") or "None")))
+--load(pathJoin("metplus", (os.getenv("metplus_ver") or "None")))
+load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None")))
+
+setenv("WGRIB2","wgrib2")
+setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None"))
+
+--prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles"))
+--prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/feature-GFSv17_com_reorg_log_update/modulefiles"))
+--load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None")))
+
+--prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles"))
+--load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None")))
+
+whatis("Description: GFS run environment")
diff --git a/modulefiles/module_gwci.noaacloud.lua b/modulefiles/module_gwci.noaacloud.lua
@@ -0,0 +1,15 @@
+help([[
+Load environment to run GFS workflow setup scripts on noaacloud
+]])
+
+prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core")
+
+load(pathJoin("stack-intel", os.getenv("2021.3.0")))
+load(pathJoin("stack-intel-oneapi-mpi", os.getenv("2021.3.0")))
+
+load(pathJoin("netcdf-c", os.getenv("4.9.2")))
+load(pathJoin("netcdf-fortran", os.getenv("4.6.1")))
+load(pathJoin("nccmp","1.9.0.1"))
+load(pathJoin("wgrib2", "2.0.8"))
+
+whatis("Description: GFS run setup CI environment")
diff --git a/modulefiles/module_gwsetup.noaacloud.lua b/modulefiles/module_gwsetup.noaacloud.lua
@@ -0,0 +1,20 @@
+help([[
+Load environment to run GFS workflow setup scripts on noaacloud
+]])
+
+load(pathJoin("rocoto"))
+
+prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core")
+
+local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.3.0"
+local python_ver=os.getenv("python_ver") or "3.10.3"
+
+load(pathJoin("stack-intel", stack_intel_ver))
+load(pathJoin("python", python_ver))
+load("py-jinja2")
+load("py-pyyaml")
+load("py-numpy")
+local git_ver=os.getenv("git_ver") or "1.8.3.1"
+load(pathJoin("git", git_ver))
+
+whatis("Description: GFS run setup environment")
diff --git a/parm/archive/enkf.yaml.j2 b/parm/archive/enkf.yaml.j2
@@ -11,7 +11,7 @@ enkf:
         {% endfor %}
         - "logs/{{ cycle_YMDH }}/{{ RUN }}echgres.log"
         - "logs/{{ cycle_YMDH }}/{{ RUN }}esfc.log"
-        {% for grp in range(iaufhrs | length) %}
+        {% for grp in range(IAUFHRS | length) %}
         - "logs/{{ cycle_YMDH }}/{{ RUN }}ecen{{ '%03d' % grp }}.log"
         {% endfor %}
 
@@ -68,7 +68,7 @@ enkf:
         {% if DOIAU %}
         # IAU increments/analyses
 
-        {% for fhr in iaufhrs if fhr != 6 %}
+        {% for fhr in IAUFHRS if fhr != 6 %}
         {% if do_calc_increment %}
         # Store analyses instead of increments
         - "{{ COMIN_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atma{{ '%03d' % fhr }}.ensmean.nc"

diff --git a/parm/archive/enkf_restarta_grp.yaml.j2 b/parm/archive/enkf_restarta_grp.yaml.j2
@@ -36,14 +36,14 @@ enkf_restarta_grp:
         {% endif %}
 
         # Member increments
-        {% for iaufhr in iaufhrs if iaufhr != 6 %}
+        {% for iaufhr in IAUFHRS if iaufhr != 6 %}
         {% set iaufhr = iaufhr %}
         {% if do_calc_increment %}
         - "{{ COMIN_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}atma{{ '%03d' % iaufhr }}.nc"
         {% else %}
         - "{{ COMIN_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}ratmi{{ '%03d' % iaufhr }}.nc"
         {% endif %}
-        {% endfor %}  # iaufhr in iaufhrs
+        {% endfor %}  # iaufhr in IAUFHRS
 
         # Conventional data
         {% if not lobsdiag_forenkf and not DO_JEDIATMENS %}

diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2
@@ -49,7 +49,7 @@ gdas:
         - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanl.ensres.nc"
                 {% if DOIAU %}
         # Ensemble IAU analysis residuals
-                    {% for fhr in iaufhrs if fhr != 6 %}
+                    {% for fhr in IAUFHRS if fhr != 6 %}
         - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atma{{ '%03d' % fhr }}.ensres.nc"
                     {% endfor %}
                 {% endif %}
@@ -108,7 +108,7 @@ gdas:
         {% endif %}  # End of cycled data
 
         # Forecast and post logs
-        - "logs/{{ cycle_YMDH }}/{{ RUN }}fcst.log"
+        - "logs/{{ cycle_YMDH }}/{{ RUN }}fcst_seg0.log"
 
         {% for fhr in range(0, FHMAX + 1, 3) %}
             {% set fhr3 = '%03d' % fhr %}

diff --git a/parm/archive/gdas_restarta.yaml.j2 b/parm/archive/gdas_restarta.yaml.j2
@@ -6,7 +6,7 @@ gdas_restarta:
         # Deterministic analysis increments
         - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atminc.nc"
         # IAU increments
-        {% for iaufhr in iaufhrs if iaufhr != 6 %}
+        {% for iaufhr in IAUFHRS if iaufhr != 6 %}
         - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmi{{ "%03d" % iaufhr }}.nc"
         {% endfor %}
 

diff --git a/parm/archive/gfs_netcdfa.yaml.j2 b/parm/archive/gfs_netcdfa.yaml.j2
@@ -6,7 +6,7 @@ gfs_netcdfa:
         - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanl.nc"
         - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}sfcanl.nc"
         - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atminc.nc"
-        {% for iauhr in iaufhrs if iauhr != 6 %}
+        {% for iauhr in IAUFHRS if iauhr != 6 %}
         - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmi{{ "%03d" % iauhr }}.nc"
         {% endfor %}
     optional:

diff --git a/parm/archive/master_enkf.yaml.j2 b/parm/archive/master_enkf.yaml.j2
@@ -4,28 +4,6 @@
 {% set cycle_YMDH = current_cycle | to_YMDH %}
 {% set head = RUN + ".t" + cycle_HH + "z." %}
 
-# Split IAUFHRS into a list; typically either "3,6,9" or 6 (integer)
-{% if IAUFHRS is string %}
-    # "3,6,9"
-    {% set iaufhrs = [] %}
-    {% for iaufhr in IAUFHRS.split(",") %}
-        {% do iaufhrs.append(iaufhr | int) %}
-    {% endfor %}
-{% else %}
-    # 6 (integer)
-    {% set iaufhrs = [IAUFHRS] %}
-{% endif %}
-
-# Repeat for IAUFHRS_ENKF
-{% if IAUFHRS_ENKF is string %}
-    {% set iaufhrs_enkf = [] %}
-    {% for iaufhr in IAUFHRS_ENKF.split(",") %}
-        {% do iaufhrs_enkf.append(iaufhr | int) %}
-    {% endfor %}
-{% else %}
-    {% set iaufhrs_enkf = [IAUFHRS_ENKF] %}
-{% endif %}
-
 # Determine which data to archive
 datasets:
 {% if ENSGRP == 0 %}

diff --git a/parm/archive/master_gdas.yaml.j2 b/parm/archive/master_gdas.yaml.j2
@@ -3,16 +3,6 @@
 {% set cycle_YMDH = current_cycle | to_YMDH %}
 {% set head = "gdas.t" + cycle_HH + "z." %}
 
-# Split IAUFHRS into a list; typically either "3,6,9" or 6 (integer)
-{% if IAUFHRS is string %}
-    {% set iaufhrs = [] %}
-    {% for iaufhr in IAUFHRS.split(",") %}
-        {% do iaufhrs.append(iaufhr | int) %}
-    {% endfor %}
-{% else %}
-    {% set iaufhrs = [IAUFHRS] %}
-{% endif %}
-
 datasets:
 # Always archive atmosphere forecast/analysis data
 {% filter indent(width=4) %}

diff --git a/parm/archive/master_gfs.yaml.j2 b/parm/archive/master_gfs.yaml.j2
@@ -3,18 +3,6 @@
 {% set cycle_YMD = current_cycle | to_YMD %}
 {% set cycle_YMDH = current_cycle | to_YMDH %}
 
-# Split IAUFHRS into a list; typically either "3,6,9" or 6 (integer)
-{% if IAUFHRS is string %}
-    # "3,6,9"
-    {% set iaufhrs = [] %}
-    {% for iaufhr in IAUFHRS.split(",") %}
-        {% do iaufhrs.append(iaufhr | int) %}
-    {% endfor %}
-{% else %}
-    # 6 (integer)
-    {% set iaufhrs = [IAUFHRS] %}
-{% endif %}
-
 # Determine which data to archive
 datasets:
 # Always archive atmosphere forecast/analysis data

diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base
@@ -229,8 +229,11 @@ export gfs_cyc=@gfs_cyc@ # 0: no GFS cycle, 1: 00Z only, 2: 00Z and 12Z only, 4:
 
 # GFS output and frequency
 export FHMIN_GFS=0
-export FHMIN=${FHMIN_GFS}
-export FHMAX_GFS=@FHMAX_GFS@
+export FHMAX_GFS="@FHMAX_GFS@"
+# Intermediate times to stop forecast when running in segments
+breakpnts="@FCST_BREAKPOINTS@"
+export FCST_SEGMENTS="${FHMIN_GFS},${breakpnts:+${breakpnts},}${FHMAX_GFS}"
+
 export FHOUT_GFS=6
 export FHMAX_HF_GFS=@FHMAX_HF_GFS@
 export FHOUT_HF_GFS=1

diff --git a/parm/config/gefs/config.extractvars b/parm/config/gefs/config.extractvars
@@ -9,12 +9,12 @@ echo "BEGIN: config.extractvars"
 
 export COMPRSCMD=${COMPRSCMD:-bzip2}
 
-export compress_ocn=0 #1: Compress extracted ocean product, 0: Do not compress extracted ocean product
-export compress_ice=0 #1: Compress extracted ice product, 0: Do not compress extracted ice product
+export compress_ocn=1 #1: Compress extracted ocean product, 0: Do not compress extracted ocean product
+export compress_ice=1 #1: Compress extracted ice product, 0: Do not compress extracted ice product
 
-export ocnres="5p00" # Resolution of ocean products
-export iceres="5p00" # Resolution of ice products
-export wavres="5p00" # Resolution of wave products
+export ocnres="1p00" # Resolution of ocean products
+export iceres="native" # Resolution of ice products
+export wavres="0p25" # Resolution of wave products
 
 export depthvar_name="z_l" # Name of depth variable in NetCDF ocean products
 export zmin="0." # Minimum depth to extract from NetCDF ocean products

diff --git a/parm/config/gefs/config.fcst b/parm/config/gefs/config.fcst
@@ -30,14 +30,19 @@ string="--fv3 ${CASE}"
 # shellcheck disable=SC2086
 source "${EXPDIR}/config.ufs" ${string}
 
-# shellcheck disable=SC2153
-export FHMAX=${FHMAX_GFS}
+# Convert comma-separated string into bash array
+IFS=', ' read -ra segments <<< "${FCST_SEGMENTS}"
+# Determine MIN and MAX based on the forecast segment
+export FHMIN=${segments[${FCST_SEGMENT}]}
+export FHMAX=${segments[${FCST_SEGMENT}+1]}
+# Cap other FHMAX variables at FHMAX for the segment
+export FHMAX_HF=$(( FHMAX_HF_GFS > FHMAX ? FHMAX : FHMAX_HF_GFS ))
+export FHMAX_WAV=$(( FHMAX_WAV > FHMAX ? FHMAX : FHMAX_WAV ))
 # shellcheck disable=SC2153
 export FHOUT=${FHOUT_GFS}
-export FHMAX_HF=${FHMAX_HF_GFS}
 export FHOUT_HF=${FHOUT_HF_GFS}
 export FHOUT_OCN=${FHOUT_OCN_GFS}
-export FHOUT_ICE=${FHOUT_ICE_GFS} 
+export FHOUT_ICE=${FHOUT_ICE_GFS}
 
 # Get task specific resources
 source "${EXPDIR}/config.resources" fcst
@@ -242,7 +247,11 @@ export FSICS="0"
 
 #---------------------------------------------------------------------
 # Write more variables to output
-export DIAG_TABLE="${PARMgfs}/ufs/fv3/diag_table"
+if [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then
+  export DIAG_TABLE="${PARMgfs}/ufs/fv3/diag_table_replay"
+else
+  export DIAG_TABLE="${PARMgfs}/ufs/fv3/diag_table"
+fi
 
 # Write gfs restart files to rerun fcst from any break point
 export restart_interval=${restart_interval_gfs:-12}

diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources
@@ -272,7 +272,7 @@ case ${step} in
     export walltime_gefs="00:30:00"
     export ntasks_gefs=1
     export threads_per_task_gefs=1
-    export tasks_per_node_gefs="${ntasks}"
+    export tasks_per_node_gefs="${ntasks_gefs}"
     export walltime_gfs="${walltime_gefs}"
     export ntasks_gfs="${ntasks_gefs}"
     export threads_per_tasks_gfs="${threads_per_task_gefs}"

diff --git a/parm/config/gefs/yaml/defaults.yaml b/parm/config/gefs/yaml/defaults.yaml
@@ -11,5 +11,6 @@ base:
   DO_EXTRACTVARS: "NO"
   FHMAX_GFS: 120
   FHMAX_HF_GFS: 0
+  FCST_BREAKPOINTS: "48"
   REPLAY_ICS: "NO"
   USE_OCN_PERTURB_FILES: "false"
diff --git a/parm/config/gfs/config.aeroanl b/parm/config/gfs/config.aeroanl
@@ -40,7 +40,7 @@ if [[ "${DOIAU}" == "YES" ]]; then
   export aero_bkg_times="3,6,9"
   export JEDIYAML="${PARMgfs}/gdas/aero/variational/3dvar_fgat_gfs_aero.yaml.j2"
 else
-  export aero_bkg_times="6"
+  export aero_bkg_times="6," # Trailing comma is necessary so this is treated as a list
   export JEDIYAML="${PARMgfs}/gdas/aero/variational/3dvar_gfs_aero.yaml.j2"
 fi