Feature/jcb darth vader (NOAA-EMC#1256)

This PR updates `ush/ufoeval/run_ufo_hofx_test.sh` to use the JCB framework. This references issue NOAA-EMC#1113 --------- Co-authored-by: CoryMartin-NOAA <[email protected]> Co-authored-by: Emily Liu <[email protected]>
RussTreadon-NOAA · Aug 21, 2024 · 4e21188 · 4e21188
1 parent 0431b26
commit 4e21188
Show file tree

Hide file tree

Showing 4 changed files with 97 additions and 28 deletions.
diff --git a/modulefiles/GDAS/hera.intel.lua b/modulefiles/GDAS/hera.intel.lua
@@ -74,6 +74,7 @@ load("py-scipy/1.11.3")
 load("py-xarray/2023.7.0")
 load("py-f90nml/1.4.3")
 load("py-pip/23.1.2")
+load("py-click/8.1.7")
 
 setenv("CC","mpiicc")
 setenv("FC","mpiifort")

diff --git a/ush/eva/gen_eva_obs_yaml.py b/ush/eva/gen_eva_obs_yaml.py
@@ -63,8 +63,8 @@ def gen_eva_obs_yaml(inputyaml, templateyaml, outputdir):
         os.makedirs(outputdir)
     # now loop over all observation spaces in input JEDI YAML file
     for obsspace in evaobs:
-        name = obsspace['name']
-        cycle = obsspace['diagfile'].split('.')[-2]
+        name = obsspace['name'].replace(" ", "_").lower()
+        cycle = obsspace['diagfile'].split('_')[-1].split('.')[-2]
         logging.info(f'Now processing: {name}')
         # get the dictionary of replacements set up
         replacements = {

diff --git a/ush/ufoeval/run_ufo_hofx_test.sh b/ush/ufoeval/run_ufo_hofx_test.sh
@@ -28,7 +28,7 @@ usage() {
 
 # ==============================================================================
 
-cycle=2021080100
+cycle=2024021900
 run_filtering=YES
 run_eva=YES
 eva_stats_only=NO
@@ -77,13 +77,17 @@ if [ $machine = orion ]; then
       workdir=/work2/noaa/da/$LOGNAME/ufoeval/$cycle/${obtype}
    fi
    GDASApp=${GDASApp:-/work2/noaa/da/$LOGNAME/git/GDASApp/} # Change this to your own branch
+   JCBinstall=${JCBinstall:-/work2/noaa/da/cmartin/CI/GDASApp/opt}
+   JCBpylib=$JCBinstall/lib/python3.7/site-packages
 elif [ $machine = hera ]; then
    if [ $run_filtering == NO ]; then
       workdir=/scratch1/NCEPDEV/stmp2/$LOGNAME/ufoeval/$cycle/${obtype}_noqc
    else
       workdir=/scratch1/NCEPDEV/stmp2/$LOGNAME/ufoeval/$cycle/${obtype}
    fi
    GDASApp=${GDASApp:-/scratch1/NCEPDEV/da/$LOGNAME/git/GDASApp/} # Change this to your own branch
+   JCBinstall=${JCBinstall:-/scratch1/NCEPDEV/da/Cory.R.Martin/CI/GDASApp/opt}
+   JCBpylib=$JCBinstall/lib/python3.10/site-packages
 else
    echo "Machine " $machine "not found"
    exit 1
@@ -94,17 +98,12 @@ if [ $keep_output = YES ]; then
   workdir=${workdir}_datetime
 fi
 
-if [ $run_filtering == NO ]; then
-   yamlpath=$GDASApp/parm/atm/obs/testing/${obtype}_noqc.yaml
-else
-   yamlpath=$GDASApp/parm/atm/obs/testing/${obtype}.yaml
-fi
 exename=test_ObsFilters.x
 
 #-------------- Do not modify below this line ----------------
 # paths that should only be changed by an expert user
 
-dataprocdate=20230811 # Production date of test data
+dataprocdate=20240815 # Production date of test data
 
 obtype_short=${obtype:0:4}
 if [ $obtype_short = "cris" ] || [ $obtype_short = "iasi" ] || [ $obtype_short = "hirs" ] || [ $obtype_short = "sevi" ] || \
@@ -116,7 +115,7 @@ else
 fi
 
 if [ $machine = orion ]; then
-    export Datapath='/work2/noaa/da/eliu/UFO_eval/data/gsi_geovals_l127/nofgat_aug2021/'$dataprocdate 
+    export Datapath='/work2/noaa/da/acollard/UFO_eval/data/gsi_geovals_l127/nofgat_feb2024/'$dataprocdate 
     FixDir=/work2/noaa/da/cmartin/GDASApp/fix
 elif [ $machine = hera ]; then
     export Datapath='/scratch1/NCEPDEV/da/Emily.Liu/UFO_eval/data/gsi_geovals_l127/nofgat_aug2021/'$dataprocdate
@@ -133,14 +132,29 @@ BCDir=$Datapath/bc/
 # other variables that should not change often
 export CDATE=$cycle
 export assim_freq=6
+export half_assim_freq=$(($assim_freq / 2))
 export GDATE=$(date +%Y%m%d%H -d "${CDATE:0:8} ${CDATE:8:2} - ${assim_freq} hours")
+export BDATE=$(date +%Y%m%d%H -d "${CDATE:0:8} ${CDATE:8:2} - ${half_assim_freq} hours")
 export PDY=${CDATE:0:8}
 export cyc=${CDATE:8:2}
+export YYYY=${CDATE:0:4}
+export MM=${CDATE:4:2}
+export DD=${CDATE:6:2}
 export gPDY=${GDATE:0:8}
 export gcyc=${GDATE:8:2}
+export bPDY=${BDATE:0:8}
+export bcyc=${BDATE:8:2}
+export bYYYY=${BDATE:0:4}
+export bMM=${BDATE:4:2}
+export bDD=${BDATE:6:2}
 export CASE="C768"
 export CASE_ANL="C384"
 export LEVS="128"
+export DATA=./
+export COMPONENT=atmos
+export OPREFIX=gdas.t${cyc}z
+export APREFIX=gdas.t${cyc}z
+export GPREFIX=gdas.t${gcyc}z
 
 # Load Modules for GDASApp
 module use $GDASApp/modulefiles
@@ -157,35 +171,89 @@ ln -sf $FixDir/crtm/2.4.0 $workdir/crtm
 
 # copy BC files
 if [ $radiance = "YES" ]; then
-  cp -rf $BCDir/${obtype}*${GDATE}* $workdir/.
+  cp -rf $BCDir/${GPREFIX}.${obtype}.tlapse.txt $workdir/${GPREFIX}.${obtype}.tlapse.txt
+  cp -rf $BCDir/${GPREFIX}.${obtype}.satbias.nc $workdir/${GPREFIX}.${obtype}.satbias.nc
 fi
 
 # Copy obs and geovals
-cp -rf $GeoDir/${obtype}_geoval_${cycle}*.nc4 $workdir/.
-cp -rf $ObsDir/${obtype}_obs_${cycle}*.nc4 $workdir/.
+cp -rf $GeoDir/${obtype}_geoval_${cycle}.nc4 $workdir/${OPREFIX}.${obtype}_geoval.tm00.nc
+cp -rf $ObsDir/${obtype}_obs_${cycle}.nc4 $workdir/${OPREFIX}.${obtype}.tm00.nc
 
 # Link executable
 ln -sf $GDASApp/build/bin/$exename $workdir/.
 
 echo "Generating YAML"
 
 # Copy/generate YAML for test executable
-# First, create the input YAMLs for the genYAML script
-export DATA=./
-export COMPONENT=atmos
-export OPREFIX=gdas.t${cyc}z
-export APREFIX=gdas.t${cyc}z
-export GPREFIX=gdas.t${gcyc}z
+# need to add JCB to PATH and PYTHONPATH
+export PYTHONPATH=$PYTHONPATH:$JCBpylib
+export PATH=$PATH:$JCBinstall/bin
+# First, create the input file for JCB
 
 cat > $workdir/temp.yaml << EOF
-time window:
-  begin: '{{ WINDOW_BEGIN | to_isotime }}'
-  end: '{{ WINDOW_END | to_isotime }}'
-  bound to include: begin
-observations:
-- !INC $yamlpath
+# Search path for model and obs for JCB
+# -------------------------------------
+algorithm_path: "$GDASApp/parm/jcb-algorithms"
+app_path_algorithm: "$GDASApp/parm/jcb-gdas/algorithm/atmosphere"
+app_path_model: "$GDASApp/parm/jcb-gdas/model/atmosphere"
+app_path_observations: "$GDASApp/parm/jcb-gdas/observations/atmosphere"
+app_path_observation_chronicle: "$GDASApp/parm/jcb-gdas/observation_chronicle/atmosphere"
+
+
+# Places where we deviate from the generic file name of a yaml
+# ------------------------------------------------------------
+#final_increment_file: final_increment_gaussian
+final_increment_file: final_increment_cubed_sphere
+output_ensemble_increments_file: output_ensemble_increments_cubed_sphere
+model_file: model_pseudo
+initial_condition_file: background  # Initial conditions for 4D apps is background
+
+
+# Assimilation window
+# -------------------
+window_begin: "${bYYYY}-${bMM}-${bDD}T${bcyc}:00:00Z"
+window_length: "PT${assim_freq}H"
+bound_to_include: begin
+
+# Default background time is for 3D applications
+atmosphere_background_time_iso: "${YYYY}-${MM}-${DD}T${cyc}:00:00Z"
+
+algorithm: test_obs_filters
+
+# Observation things
+# ------------------
+observations: [${obtype}]
+
+crtm_coefficient_path: "$workdir/crtm/"
+
+# Naming conventions for observational files
+atmosphere_obsdatain_path: "$workdir"
+atmosphere_obsdatain_prefix: "$OPREFIX."
+atmosphere_obsdatain_suffix: ".tm00.nc"
+
+atmosphere_obsdataout_path: "$workdir"
+atmosphere_obsdataout_prefix: diag_
+atmosphere_obsdataout_suffix: "_${cycle}.nc"
+
+# Naming conventions for bias correction files
+atmosphere_obsbiasin_path: "$workdir"
+atmosphere_obsbiasin_prefix: "$GPREFIX."
+atmosphere_obsbiasin_suffix: ".satbias.nc"
+atmosphere_obstlapsein_prefix: "$GPREFIX."
+atmosphere_obstlapsein_suffix: ".tlapse.txt"
+atmosphere_obsbiascovin_prefix: "$GPREFIX."
+atmosphere_obsbiascovin_suffix: ".satbias_cov.nc"
+
+atmosphere_obsbiasout_path: "$workdir"
+atmosphere_obsbiasout_prefix: "$APREFIX."
+atmosphere_obsbiasout_suffix: ".satbias.nc"
+atmosphere_obsbiascovout_prefix: "$APREFIX."
+atmosphere_obsbiascovout_suffix: ".satbias_cov.nc"
 EOF
-$GDASApp/ush/genYAML --input $workdir/temp.yaml --output $workdir/${obtype}_${cycle}.yaml
+# jcb render dictionary_of_templates.yaml jedi_config.yaml
+echo "Calling JCB Render"
+jcb render $workdir/temp.yaml $workdir/${obtype}_${cycle}.yaml
+echo "Called JCB Render"
 
 if [ $? -ne 0 ]; then
    echo "YAML creation failed"

diff --git a/ush/ufoeval/test_yamls.sh b/ush/ufoeval/test_yamls.sh
@@ -14,7 +14,7 @@ if [ $? -ne 0 ]; then
 fi
 
 # Process tests wiht QC
-for file in `find ../../parm/atm/obs/testing/*.yaml -type f -not -name "*noqc*"`; do
+for file in `find ../../parm/jcb-gdas/observations/atmosphere -type f`; do
    basefile=${file##*/}
    obtype="${basefile%.*}"
    ./run_ufo_hofx_test.sh -x $obtype > $WORKDIR/$obtype.log 2> $WORKDIR/$obtype.err
@@ -26,7 +26,7 @@ for file in `find ../../parm/atm/obs/testing/*.yaml -type f -not -name "*noqc*"`
 done
 
 # Process tests without QC (HofX + Observation error assignment)
-for file in `ls ../../parm/atm/obs/testing/*_noqc.yaml`; do
+for file in `find ../../parm/jcb-gdas/observations/atmosphere -type f`; do
    basefile=${file##*/}
    obtype="${basefile%_noqc.*}"
    ./run_ufo_hofx_test.sh -x -q $obtype > $WORKDIR/${obtype}_noqc.log 2> $WORKDIR/${obtype}_noqc.err