Update realtime branch, gsl_ufs_rtdev1

.github/pull_request_template.md

@@ -33,6 +33,17 @@
 # Change characteristics
 - Is this a breaking change (a change in existing functionality)? YES/NO
 - Does this change require a documentation update? YES/NO
+- Does this change require an update to any of the following submodules? YES/NO (If YES, please add a link to any PRs that are pending.)
+  - [ ] EMC verif-global
+  - [ ] GDAS
+  - [ ] GFS-utils
+  - [ ] GSI
+  - [ ] GSI-monitor
+  - [ ] GSI-utils
+  - [ ] UFS-utils
+  - [ ] UFS-weather-model
+  - [ ] wxflow
+
 
 # How has this been tested?
 <!-- Please list any test you conducted, including the machine.

.github/workflows/ci_unit_tests.yaml

@@ -0,0 +1,64 @@
+name: CI Unit Tests
+on: [pull_request, push, workflow_dispatch]
+
+jobs:
+
+  ci_pytest:
+      runs-on: ubuntu-latest
+      name: Run unit tests on CI system      
+      permissions:
+        checks: write
+
+      steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11.8
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y perl libxml-libxml-perl libxml-libxslt-perl libdatetime-perl
+          python -m pip install --upgrade pip
+          pip install pytest
+          pip install wxflow
+          pip install wget
+
+      - name: Cache Rocoto Install
+        uses: actions/cache@v4
+        with:
+          path: ~/rocoto
+          key: ${{ runner.os }}-rocoto-${{ hashFiles('**/ci-unit_tests.yaml') }}
+
+      - name: Install Rocoto
+        run: |
+          if [ ! -d "$HOME/rocoto/bin" ]; then
+            git clone https://github.com/christopherwharrop/rocoto.git $HOME/rocoto
+            cd $HOME/rocoto
+            ./INSTALL
+          fi
+          echo "$HOME/rocoto/bin" >> $GITHUB_PATH
+
+      - name: Run tests
+        shell: bash
+        run: |
+          sudo mkdir -p /scratch1/NCEPDEV
+          cd $GITHUB_WORKSPACE/sorc
+          git submodule update --init
+          ./link_workflow.sh
+          cd $GITHUB_WORKSPACE/ci/scripts/tests
+          ln -s ../wxflow
+
+          pytest -v --junitxml $GITHUB_WORKSPACE/ci/scripts/tests/test-results.xml
+
+
+      - name: Publish Test Results
+        if: always()
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        with:
+          files: ci/scripts/tests/test-results.xml
+          job_summary: true
+          comment_mode: off

.github/workflows/docs.yaml

@@ -57,7 +57,7 @@ jobs:
         if-no-files-found: ignore
 
     - name: Comment ReadDocs Link in PR
-      if: github.event_name == 'pull_request'
+      if: ${{ github.event_name == 'pull_request' }}
       uses: actions/github-script@v6
       with:
         script: |

.gitignore

@@ -30,32 +30,44 @@ fix/chem
 fix/cice
 fix/cpl
 fix/datm
-fix/gldas
 fix/gdas
 fix/gsi
 fix/lut
 fix/mom6
 fix/orog
-fix/reg2grb2
+fix/orog_nest
 fix/sfc_climo
 fix/ugwd
+fix/ugwd_nest
 fix/verif
 fix/wave
 
 # Ignore parm file symlinks
 #--------------------------
-parm/config/config.base
-parm/gldas
+parm/gdas/aero
+parm/gdas/atm
+parm/gdas/io
+parm/gdas/ioda
+parm/gdas/snow
+parm/gdas/soca
+parm/gdas/jcb-gdas
+parm/gdas/jcb-algorithms
 parm/monitor
 parm/post/AEROSOL_LUTS.dat
 parm/post/nam_micro_lookup.dat
 parm/post/optics_luts_DUST.dat
 parm/post/gtg.config.gfs
 parm/post/gtg_imprintings.txt
+parm/post/optics_luts_DUST_nasa.dat
+parm/post/optics_luts_NITR_nasa.dat
 parm/post/optics_luts_SALT.dat
+parm/post/optics_luts_SALT_nasa.dat
 parm/post/optics_luts_SOOT.dat
+parm/post/optics_luts_SOOT_nasa.dat
 parm/post/optics_luts_SUSO.dat
+parm/post/optics_luts_SUSO_nasa.dat
 parm/post/optics_luts_WASO.dat
+parm/post/optics_luts_WASO_nasa.dat
 parm/post/params_grib2_tbl_new
 parm/post/post_tag_gfs128
 parm/post/post_tag_gfs65
@@ -77,6 +89,9 @@ parm/post/postcntrl_gfs_wafs.xml
 parm/post/postcntrl_gfs_wafs_anl.xml
 parm/post/postxconfig-NT-GEFS-ANL.txt
 parm/post/postxconfig-NT-GEFS-F00.txt
+parm/post/postxconfig-NT-GEFS-F00-aerosol.txt
+parm/post/postxconfig-NT-GEFS-WAFS.txt
+parm/post/postxconfig-NT-GEFS-aerosol.txt
 parm/post/postxconfig-NT-GEFS.txt
 parm/post/postxconfig-NT-GFS-ANL.txt
 parm/post/postxconfig-NT-GFS-F00-TWO.txt
@@ -90,7 +105,16 @@ parm/post/postxconfig-NT-GFS-WAFS.txt
 parm/post/postxconfig-NT-GFS.txt
 parm/post/postxconfig-NT-gefs-aerosol.txt
 parm/post/postxconfig-NT-gefs-chem.txt
+parm/post/ocean.csv
+parm/post/ice.csv
+parm/post/ocnicepost.nml.jinja2
 parm/ufs/noahmptable.tbl
+parm/ufs/model_configure.IN
+parm/ufs/model_configure_nest.IN
+parm/ufs/MOM_input_*.IN
+parm/ufs/MOM6_data_table.IN
+parm/ufs/ice_in.IN
+parm/ufs/ufs.configure.*.IN
 parm/wafs
 
 # Ignore sorc and logs folders from externals
@@ -123,7 +147,6 @@ sorc/radmon_bcor.fd
 sorc/radmon_time.fd
 sorc/rdbfmsua.fd
 sorc/recentersigp.fd
-sorc/reg2grb2.fd
 sorc/supvit.fd
 sorc/syndat_getjtbul.fd
 sorc/syndat_maksynrc.fd
@@ -133,45 +156,43 @@ sorc/tocsbufr.fd
 sorc/upp.fd
 sorc/vint.fd
 sorc/webtitle.fd
+sorc/ocnicepost.fd
 
 # Ignore scripts from externals
 #------------------------------
 # jobs symlinks
-jobs/JGFS_ATMOS_WAFS
-jobs/JGFS_ATMOS_WAFS_BLENDING
-jobs/JGFS_ATMOS_WAFS_BLENDING_0P25
-jobs/JGFS_ATMOS_WAFS_GCIP
-jobs/JGFS_ATMOS_WAFS_GRIB2
-jobs/JGFS_ATMOS_WAFS_GRIB2_0P25
 # scripts symlinks
 scripts/exemcsfc_global_sfc_prep.sh
-scripts/exgfs_atmos_wafs_blending.sh
-scripts/exgfs_atmos_wafs_blending_0p25.sh
-scripts/exgfs_atmos_wafs_gcip.sh
-scripts/exgfs_atmos_wafs_grib.sh
-scripts/exgfs_atmos_wafs_grib2.sh
-scripts/exgfs_atmos_wafs_grib2_0p25.sh
+scripts/exgdas_global_marine_analysis_ecen.py
+scripts/exglobal_prep_ocean_obs.py
 # ush symlinks
+ush/bufr2ioda_insitu_profile_argo.py
+ush/bufr2ioda_insitu_profile_bathy.py
+ush/bufr2ioda_insitu_profile_glider.py
+ush/bufr2ioda_insitu_profile_marinemammal.py
+ush/bufr2ioda_insitu_profile_tesac.py
+ush/bufr2ioda_insitu_profile_xbtctd.py
+ush/bufr2ioda_insitu_surface_altkob.py
+ush/bufr2ioda_insitu_surface_trkob.py
 ush/chgres_cube.sh
 ush/emcsfc_ice_blend.sh
 ush/emcsfc_snow.sh
-ush/exglobal_prep_ocean_obs.py
 ush/fix_precip.sh
 ush/fv3gfs_driver_grid.sh
 ush/fv3gfs_filter_topo.sh
 ush/fv3gfs_make_grid.sh
 ush/fv3gfs_make_orog.sh
+ush/gen_bufr2ioda_json.py
+ush/gen_bufr2ioda_yaml.py
+ush/bufr2ioda_insitu_profile*.py
+ush/bufr2ioda_insitu_surface*.py
 ush/global_chgres.sh
 ush/global_chgres_driver.sh
 ush/global_cycle.sh
 ush/global_cycle_driver.sh
 ush/jediinc2fv3.py
-ush/mkwfsgbl.sh
 ush/ufsda
-ush/wafs_blending.sh
-ush/wafs_grib2.regrid.sh
-ush/wafs_intdsk.sh
-ush/finddate.sh
+ush/soca
 ush/make_NTC_file.pl
 ush/make_ntc_bull.pl
 ush/make_tif.sh
@@ -188,3 +209,8 @@ versions/run.ver
 ush/python/wxflow
 workflow/wxflow
 ci/scripts/wxflow
+
+# jcb checkout and symlinks
+ush/python/jcb
+workflow/jcb
+ci/scripts/jcb

.gitmodules

@@ -1,28 +1,29 @@
 [submodule "sorc/ufs_model.fd"]
-	path = sorc/ufs_model.fd
-	url = https://github.com/NOAA-GSL/ufs-weather-model
-	ignore = dirty
+  path = sorc/ufs_model.fd
+  url = https://github.com/kayeekayee/ufs-weather-model.git
+  branch = 2024jul19_c127601
 [submodule "sorc/wxflow"]
-	path = sorc/wxflow
-	url = https://github.com/NOAA-EMC/wxflow
+  path = sorc/wxflow
+  url = https://github.com/NOAA-EMC/wxflow
 [submodule "sorc/gfs_utils.fd"]
-	path = sorc/gfs_utils.fd
-	url = https://github.com/kayeekayee/gfs-utils
+  path = sorc/gfs_utils.fd
+  url = https://github.com/NOAA-EMC/gfs-utils
 [submodule "sorc/ufs_utils.fd"]
-	path = sorc/ufs_utils.fd
-	url = https://github.com/NOAA-GSL/UFS_UTILS.git
+  path = sorc/ufs_utils.fd
+  url = https://github.com/kayeekayee/UFS_UTILS.git
+  branch = 2024aug4_3ef2e6b
 [submodule "sorc/verif-global.fd"]
-	path = sorc/verif-global.fd
-	url = https://github.com/NOAA-EMC/EMC_verif-global.git
+  path = sorc/verif-global.fd
+  url = https://github.com/NOAA-EMC/EMC_verif-global.git
 [submodule "sorc/gsi_enkf.fd"]
-	path = sorc/gsi_enkf.fd
-	url = https://github.com/NOAA-EMC/GSI.git
+  path = sorc/gsi_enkf.fd
+  url = https://github.com/NOAA-EMC/GSI.git
 [submodule "sorc/gdas.cd"]
-	path = sorc/gdas.cd
-	url = https://github.com/NOAA-EMC/GDASApp.git
+  path = sorc/gdas.cd
+  url = https://github.com/NOAA-EMC/GDASApp.git
 [submodule "sorc/gsi_utils.fd"]
-	path = sorc/gsi_utils.fd
-	url = https://github.com/NOAA-EMC/GSI-Utils.git
+  path = sorc/gsi_utils.fd
+  url = https://github.com/NOAA-EMC/GSI-Utils.git
 [submodule "sorc/gsi_monitor.fd"]
-	path = sorc/gsi_monitor.fd
-	url = https://github.com/NOAA-EMC/GSI-Monitor.git
+  path = sorc/gsi_monitor.fd
+  url = https://github.com/NOAA-EMC/GSI-Monitor.git

.shellcheckrc

@@ -14,3 +14,6 @@ disable=SC1091
 
 # Disable -p -m only applies to deepest directory
 disable=SC2174
+
+# Disable warning of functions in test statements
+disable=SC2310

FV3GFSwfm/rt_v17p8_ugwpv1_c3_mynn/config.aero

@@ -20,6 +20,9 @@ case ${machine} in
   "WCOSS2")
     AERO_INPUTS_DIR="/lfs/h2/emc/global/noscrub/emc.global/data/gocart_emissions"
     ;;
+  "GAEA")
+    AERO_INPUTS_DIR="/gpfs/f5/epic/proj-shared/global/glopara/data/gocart_emissions"
+    ;;
   "JET")
     AERO_INPUTS_DIR="/lfs4/HFIP/hfv3gfs/glopara/data/gocart_emissions"
     ;;
@@ -30,12 +33,12 @@ case ${machine} in
 esac
 export AERO_INPUTS_DIR
 
-export AERO_DIAG_TABLE="${HOMEgfs}/parm/ufs/fv3/diag_table.aero"
-export AERO_FIELD_TABLE="${HOMEgfs}/parm/ufs/fv3/field_table.aero"
+export AERO_DIAG_TABLE="${PARMgfs}/ufs/fv3/diag_table.aero"
+export AERO_FIELD_TABLE="${PARMgfs}/ufs/fv3/field_table.aero"
 # Biomass burning emission dataset. Choose from: gbbepx, qfed, none
 export AERO_EMIS_FIRE="qfed"
 # Directory containing GOCART configuration files
-export AERO_CONFIG_DIR="${HOMEgfs}/parm/ufs/gocart"
+export AERO_CONFIG_DIR="${PARMgfs}/ufs/gocart"
 
 # Aerosol convective scavenging factors (list of string array elements)
 # Element syntax: '<tracer_name>:<factor>'. Use <tracer_name> = * to set default factor for all aerosol tracers

FV3GFSwfm/rt_v17p8_ugwpv1_c3_mynn/config.aeroanl

@@ -6,25 +6,26 @@
 echo "BEGIN: config.aeroanl"
 
 export CASE_ANL=${CASE}
-export OBS_YAML_DIR=${HOMEgfs}/sorc/gdas.cd/parm/aero/obs/config/
-export OBS_LIST=${HOMEgfs}/sorc/gdas.cd/parm/aero/obs/lists/gdas_aero_prototype.yaml
+export OBS_LIST="${PARMgfs}/gdas/aero/obs/lists/gdas_aero.yaml.j2"
 export STATICB_TYPE='identity'
-export BERROR_YAML=${HOMEgfs}/sorc/gdas.cd/parm/aero/berror/staticb_${STATICB_TYPE}.yaml
-export FIXgdas=${HOMEgfs}/fix/gdas
-export BERROR_DATA_DIR=${FIXgdas}/bump/aero/${CASE_ANL}/
+export BERROR_YAML="${PARMgfs}/gdas/aero/berror/staticb_${STATICB_TYPE}.yaml.j2"
+export BERROR_DATA_DIR="${FIXgfs}/gdas/bump/aero/${CASE_ANL}/"
 export BERROR_DATE="20160630.000000"
 
+export CRTM_FIX_YAML="${PARMgfs}/gdas/aero_crtm_coeff.yaml.j2"
+export JEDI_FIX_YAML="${PARMgfs}/gdas/aero_jedi_fix.yaml.j2"
+
 export io_layout_x=1
 export io_layout_y=1
 
-export JEDIEXE=${HOMEgfs}/exec/fv3jedi_var.x
+export JEDIEXE="${EXECgfs}/gdas.x"
 
 if [[ "${DOIAU}" == "YES" ]]; then
   export aero_bkg_times="3,6,9"
-  export AEROVARYAML=${HOMEgfs}/sorc/gdas.cd/parm/aero/variational/3dvar_fgat_gfs_aero.yaml
+  export JEDIYAML="${PARMgfs}/gdas/aero/variational/3dvar_fgat_gfs_aero.yaml.j2"
 else
   export aero_bkg_times="6"
-  export AEROVARYAML=${HOMEgfs}/sorc/gdas.cd/parm/aero/variational/3dvar_gfs_aero.yaml
+  export JEDIYAML="${PARMgfs}/gdas/aero/variational/3dvar_gfs_aero.yaml.j2"
 fi
 
 echo "END: config.aeroanl"

FV3GFSwfm/rt_v17p8_ugwpv1_c3_mynn/config.aeroanlfinal

@@ -6,5 +6,5 @@
 echo "BEGIN: config.aeroanlfinal"
 
 # Get task specific resources
-. $EXPDIR/config.resources aeroanlfinal
+source "${EXPDIR}/config.resources" aeroanlfinal
 echo "END: config.aeroanlfinal"

FV3GFSwfm/rt_v17p8_ugwpv1_c3_mynn/config.aeroanlinit

@@ -6,5 +6,5 @@
 echo "BEGIN: config.aeroanlinit"
 
 # Get task specific resources
-. $EXPDIR/config.resources aeroanlinit
+source "${EXPDIR}/config.resources" aeroanlinit
 echo "END: config.aeroanlinit"

FV3GFSwfm/rt_v17p8_ugwpv1_c3_mynn/config.aeroanlrun

@@ -6,6 +6,6 @@
 echo "BEGIN: config.aeroanlrun"
 
 # Get task specific resources
-. $EXPDIR/config.resources aeroanlrun
+source "${EXPDIR}/config.resources" aeroanlrun
 
 echo "END: config.aeroanlrun"