Stat Analysis Implementation

.gitignore

@@ -53,11 +53,8 @@ parm/gdas/soca
 parm/gdas/jcb-gdas
 parm/gdas/jcb-algorithms
 parm/monitor
-parm/post/AEROSOL_LUTS.dat
 parm/post/nam_micro_lookup.dat
 parm/post/optics_luts_DUST.dat
-parm/post/gtg.config.gfs
-parm/post/gtg_imprintings.txt
 parm/post/optics_luts_DUST_nasa.dat
 parm/post/optics_luts_NITR_nasa.dat
 parm/post/optics_luts_SALT.dat
@@ -70,51 +67,19 @@ parm/post/optics_luts_WASO.dat
 parm/post/optics_luts_WASO_nasa.dat
 parm/post/params_grib2_tbl_new
 parm/post/post_tag_gfs128
-parm/post/post_tag_gfs65
-parm/post/postcntrl_gefs.xml
-parm/post/postcntrl_gefs_aerosol.xml
-parm/post/postcntrl_gefs_anl.xml
-parm/post/postcntrl_gefs_chem.xml
-parm/post/postcntrl_gefs_f00.xml
-parm/post/postcntrl_gfs.xml
-parm/post/postcntrl_gfs_anl.xml
-parm/post/postcntrl_gfs_f00.xml
-parm/post/postcntrl_gfs_f00_two.xml
-parm/post/postcntrl_gfs_flux.xml
-parm/post/postcntrl_gfs_flux_f00.xml
-parm/post/postcntrl_gfs_goes.xml
-parm/post/postcntrl_gfs_goes.xml-new
-parm/post/postcntrl_gfs_two.xml
-parm/post/postcntrl_gfs_wafs.xml
-parm/post/postcntrl_gfs_wafs_anl.xml
-parm/post/postxconfig-NT-GEFS-ANL.txt
-parm/post/postxconfig-NT-GEFS-F00.txt
-parm/post/postxconfig-NT-GEFS-F00-aerosol.txt
-parm/post/postxconfig-NT-GEFS-WAFS.txt
-parm/post/postxconfig-NT-GEFS-aerosol.txt
-parm/post/postxconfig-NT-GEFS.txt
-parm/post/postxconfig-NT-GFS-ANL.txt
-parm/post/postxconfig-NT-GFS-F00-TWO.txt
-parm/post/postxconfig-NT-GFS-F00.txt
-parm/post/postxconfig-NT-GFS-FLUX-F00.txt
-parm/post/postxconfig-NT-GFS-FLUX.txt
-parm/post/postxconfig-NT-GFS-GOES.txt
-parm/post/postxconfig-NT-GFS-TWO.txt
-parm/post/postxconfig-NT-GFS-WAFS-ANL.txt
-parm/post/postxconfig-NT-GFS-WAFS.txt
-parm/post/postxconfig-NT-GFS.txt
-parm/post/postxconfig-NT-gefs-aerosol.txt
-parm/post/postxconfig-NT-gefs-chem.txt
+parm/post/gfs
+parm/post/gefs
 parm/post/ocean.csv
 parm/post/ice.csv
 parm/post/ocnicepost.nml.jinja2
 parm/ufs/noahmptable.tbl
 parm/ufs/model_configure.IN
-parm/ufs/model_configure_nest.IN
+parm/ufs/input_global_nest.nml.IN
 parm/ufs/MOM_input_*.IN
 parm/ufs/MOM6_data_table.IN
 parm/ufs/ice_in.IN
 parm/ufs/ufs.configure.*.IN
+parm/ufs/post_itag_gfs
 parm/wafs
 
 # Ignore sorc and logs folders from externals
@@ -200,6 +165,7 @@ ush/month_name.sh
 ush/imsfv3_scf2ioda.py
 ush/atparse.bash
 ush/run_bufr2ioda.py
+ush/bufr2ioda_insitu*
 
 # version files
 versions/build.ver

ci/Jenkinsfile

@@ -78,7 +78,7 @@ pipeline {
                     Machine = machine[0].toUpperCase() + machine.substring(1)
                     echo "Getting Common Workspace for ${Machine}"
                     ws("${custom_workspace[machine]}/${env.CHANGE_ID}") {
-                        properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hercules-EMC', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
+                        properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hercules-EMC', 'Hera-EMC', 'Orion-EMC', 'Gaea'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                         GH = sh(script: "which gh || echo '~/bin/gh'", returnStdout: true).trim()
                         CUSTOM_WORKSPACE = "${WORKSPACE}"
                         sh(script: "mkdir -p ${CUSTOM_WORKSPACE}/RUNTESTS;rm -Rf ${CUSTOM_WORKSPACE}/RUNTESTS/*")
@@ -267,7 +267,7 @@ pipeline {
                 }
             }
         }
-            
+
 
         stage( '5. FINALIZE' ) {
             agent { label NodeName[machine].toLowerCase() }
@@ -297,6 +297,6 @@ pipeline {
                     }
                 }
             }
-        }   
+        }
     }
 }

ci/cases/pr/C48mx500_3DVarAOWCDA.yaml

@@ -9,7 +9,7 @@ arguments:
   resdetocean: 5.0
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500
+  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500/20240610
   idate: 2021032412
   edate: 2021032418
   nens: 0

ci/cases/pr/C96C48_hybatmDA.yaml

@@ -10,7 +10,7 @@ arguments:
   resensatmos: 48
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48
+  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20240610
   idate: 2021122018
   edate: 2021122106
   nens: 2

ci/cases/pr/C96_atmaerosnowDA.yaml → ci/cases/pr/C96C48_hybatmaerosnowDA.yaml

@@ -6,12 +6,13 @@ arguments:
   pslot: {{ 'pslot' | getenv }}
   app: ATMA
   resdetatmos: 96
+  resensatmos: 48
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48
+  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20240610
   idate: 2021122012
   edate: 2021122100
-  nens: 0
+  nens: 2
   gfs_cyc: 1
   start: cold
   yaml: {{ HOMEgfs }}/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml

ci/cases/pr/C96C48_ufs_hybatmDA.yaml

@@ -9,7 +9,7 @@ arguments:
   resensatmos: 48
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48
+  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20240610
   idate: 2024022318
   edate: 2024022400
   nens: 2
@@ -18,7 +18,6 @@ arguments:
   yaml: {{ HOMEgfs }}/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml
 
 skip_ci_on_hosts:
-  - hera
   - gaea
   - orion
   - hercules

ci/cases/pr/C96_atm3DVar.yaml

@@ -8,7 +8,7 @@ arguments:
   resdetatmos: 96
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48
+  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20240610
   idate: 2021122018
   edate: 2021122106
   nens: 0

ci/cases/pr/C96_atm3DVar_extended.yaml

@@ -8,7 +8,7 @@ arguments:
   resdetatmos: 96
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48
+  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20240610
   idate: 2021122018
   edate: 2021122118
   nens: 0

ci/scripts/utils/rocotostat.py

@@ -136,7 +136,7 @@ def rocoto_statcount(rocotostat):
     rocotostat_output = [line.split()[0:4] for line in rocotostat_output]
     rocotostat_output = [line for line in rocotostat_output if len(line) != 1]
 
-    status_cases = ['SUCCEEDED', 'FAIL', 'DEAD', 'RUNNING', 'SUBMITTING', 'QUEUED', 'UNAVAILABLE']
+    status_cases = ['SUCCEEDED', 'FAIL', 'DEAD', 'RUNNING', 'SUBMITTING', 'QUEUED', 'UNAVAILABLE', 'UNKNOWN']
 
     rocoto_status = {}
     status_counts = Counter(case for sublist in rocotostat_output for case in sublist)
@@ -214,14 +214,14 @@ def is_stalled(rocoto_status):
     elif rocoto_status['DEAD'] > 0:
         error_return = rocoto_status['FAIL'] + rocoto_status['DEAD']
         rocoto_state = 'FAIL'
-    elif 'UNAVAILABLE' in rocoto_status or 'UNKNOWN' in rocoto_status:
+    elif rocoto_status['UNAVAILABLE'] > 0 or rocoto_status['UNKNOWN'] > 0:
         rocoto_status = attempt_multiple_times(lambda: rocoto_statcount(rocotostat), 2, 120, ProcessError)
         error_return = 0
         rocoto_state = 'RUNNING'
-        if 'UNAVAILABLE' in rocoto_status:
+        if rocoto_status['UNAVAILABLE'] > 0:
             error_return = rocoto_status['UNAVAILABLE']
             rocoto_state = 'UNAVAILABLE'
-        if 'UNKNOWN' in rocoto_status:
+        if rocoto_status['UNKNOWN'] > 0:
             error_return += rocoto_status['UNKNOWN']
             rocoto_state = 'UNKNOWN'
     elif is_stalled(rocoto_status):

docs/source/errors_faq.rst

@@ -2,6 +2,14 @@
 Common Errors Known Issues
 ==========================
 
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Error: Reserved Variables Causing Workflow Issues
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Several variables are reserved in the workflow and should not be used as environment variables in your shell. Some of the common ones include (but are not limited to):
+``HOMEgfs``, ``machine``, ``ROTDIR``, ``COMROT``, ``COMROOT``, ``COMOUT``, ``COMIN``, ``STMP``, ``PTMP``, ``DATAROOT``, ``DATA``, ``ACCOUNT``, ``PDY``, ``cyc``, ``RUN``, etc.
+If you are using any of these variables in your shell, you may encounter unexpected behavior in the workflow.
+
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 Error: "ImportError" message when running setup script
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -42,4 +50,3 @@ Issue: Directory name change for EnKF folder in ROTDIR
 **Issue:** The EnKF ROTDIR folders were renamed during the GFS v15 development process to remove the period between "enkf" and "gdas": enkf.gdas.$PDY → enkfgdas.$PDY
 
 **Fix:** Older tarballs on HPSS will have the older directory name with the period between 'enkf' and 'gdas'. Make sure to rename folder to 'enkfgdas.$PDY' after obtaining. Only an issue for the initial cycle.
-

docs/source/index.rst

@@ -1,6 +1,6 @@
 
 ###############
-Global Workflow 
+Global Workflow
 ###############
 
 **Global-workflow** is the end-to-end workflow designed to run global configurations of medium range weather forecasting for the UFS weather model. It supports both development and operational implementations. In its current format it supports the Global Forecast System (GFS) and the Global Ensemble Forecast System (GEFS) configurations
@@ -31,7 +31,7 @@ GitHub updates: Users should adjust their "Watch" settings for this repo so they
 Table of Contents
 =================
 
-.. toctree:: 
+.. toctree::
    :numbered:
    :maxdepth: 3
 
@@ -42,3 +42,4 @@ Table of Contents
    output.rst
    run.rst
    noaa_csp.rst
+   errors_faq.rst

docs/source/init.rst

@@ -125,14 +125,14 @@ Start date = 2021032312
    ├── enkfgdas.20210323
    │   ├── 06
    │   │   ├── mem001
-   │   │   │   └── model_data -> ../../../gdas.20210323/06/model_data
+   │   │   │   └── model -> ../../../gdas.20210323/06/model
    │   │   ├── mem002
-   │   │   │   └── model_data -> ../../../gdas.20210323/06/model_data
+   │   │   │   └── model -> ../../../gdas.20210323/06/model
    │   │   ├── mem003
-   │   │   │   └── model_data -> ../../../gdas.20210323/06/model_data
+   │   │   │   └── model -> ../../../gdas.20210323/06/model
    ...
    │   │   └── mem080
-   │   │       └── model_data -> ../../../gdas.20210323/06/model_data
+   │   │       └── model -> ../../../gdas.20210323/06/model
    │   └── 12
    │       ├── mem001
    │       │   └── analysis
@@ -153,7 +153,7 @@ Start date = 2021032312
    │                   └── gdas.t12z.ocninc.nc -> ../../../../../gdas.20210323/12/analysis/ocean/gdas.t12z.ocninc.nc
    └── gdas.20210323
        ├── 06
-       │   └── model_data
+       │   └── model
        │       ├── atmos
        │       │   └── restart
        │       │       ├── 20210323.120000.ca_data.tile1.nc

docs/source/run.rst

@@ -13,4 +13,3 @@ Here we will show how you can run an experiment using the Global Workflow. The G
    start.rst
    monitor_rocoto.rst
    view.rst
-   errors_faq.rst

env/AWSPW.env

@@ -27,7 +27,7 @@ if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:-
     NTHREADS1=${threads_per_task:-1}
     [[ ${NTHREADSmax} -gt ${max_threads_per_task} ]] && NTHREADSmax=${max_threads_per_task}
     [[ ${NTHREADS1} -gt ${max_threads_per_task} ]] && NTHREADS1=${max_threads_per_task}
-    APRUN="${launcher} -n ${ntasks}"
+    APRUN_default="${launcher} -n ${ntasks}"
 else
     echo "ERROR config.resources must be sourced before sourcing AWSPW.env"
     exit 2
@@ -43,45 +43,61 @@ if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
     export APRUN_UFS="${launcher} -n ${ufs_ntasks}"
     unset nnodes ufs_ntasks
 
+elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then
+
+    export CFP_MP="YES"
+    if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi
+    export wavempexec=${launcher}
+    export wave_mpmd=${mpmd_opt}
+
 elif [[ "${step}" = "post" ]]; then
 
     export NTHREADS_NP=${NTHREADS1}
-    export APRUN_NP="${APRUN}"
+    export APRUN_NP="${APRUN_default}"
 
     export NTHREADS_DWN=${threads_per_task_dwn:-1}
     [[ ${NTHREADS_DWN} -gt ${max_threads_per_task} ]] && export NTHREADS_DWN=${max_threads_per_task}
     export APRUN_DWN="${launcher} -n ${ntasks_dwn}"
 
+elif [[ "${step}" = "atmos_products" ]]; then
+
+    export USE_CFP="YES"  # Use MPMD for downstream product generation on Hera
+
+elif [[ "${step}" = "oceanice_products" ]]; then
+
+    export NTHREADS_OCNICEPOST=${NTHREADS1}
+    export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}"
+
 elif [[ "${step}" = "ecen" ]]; then
 
     export NTHREADS_ECEN=${NTHREADSmax}
-    export APRUN_ECEN="${APRUN}"
+    export APRUN_ECEN="${APRUN_default}"
 
     export NTHREADS_CHGRES=${threads_per_task_chgres:-12}
     [[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]] && export NTHREADS_CHGRES=${max_tasks_per_node}
     export APRUN_CHGRES="time"
 
     export NTHREADS_CALCINC=${threads_per_task_calcinc:-1}
     [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task}
-    export APRUN_CALCINC="${APRUN}"
+    export APRUN_CALCINC="${APRUN_default}"
 
 elif [[ "${step}" = "esfc" ]]; then
 
     export NTHREADS_ESFC=${NTHREADSmax}
-    export APRUN_ESFC="${APRUN}"
+    export APRUN_ESFC="${APRUN_default}"
 
     export NTHREADS_CYCLE=${threads_per_task_cycle:-14}
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
-    export APRUN_CYCLE="${APRUN}"
+    export APRUN_CYCLE="${APRUN_default}"
 
 elif [[ "${step}" = "epos" ]]; then
 
     export NTHREADS_EPOS=${NTHREADSmax}
-    export APRUN_EPOS="${APRUN}"
+    export APRUN_EPOS="${APRUN_default}"
 
 elif [[ "${step}" = "fit2obs" ]]; then
 
     export NTHREADS_FIT2OBS=${NTHREADS1}
-    export MPIRUN="${APRUN}"
+    export MPIRUN="${APRUN_default}"
 
 fi