diff --git a/modulefiles/GDAS/hera.intel.lua b/modulefiles/GDAS/hera.intel.lua index 0d5c080c3..613205f6f 100644 --- a/modulefiles/GDAS/hera.intel.lua +++ b/modulefiles/GDAS/hera.intel.lua @@ -74,6 +74,7 @@ load("py-scipy/1.11.3") load("py-xarray/2023.7.0") load("py-f90nml/1.4.3") load("py-pip/23.1.2") +load("py-click/8.1.7") setenv("CC","mpiicc") setenv("FC","mpiifort") diff --git a/ush/eva/gen_eva_obs_yaml.py b/ush/eva/gen_eva_obs_yaml.py index 5ec301b09..9a500dd07 100755 --- a/ush/eva/gen_eva_obs_yaml.py +++ b/ush/eva/gen_eva_obs_yaml.py @@ -63,8 +63,8 @@ def gen_eva_obs_yaml(inputyaml, templateyaml, outputdir): os.makedirs(outputdir) # now loop over all observation spaces in input JEDI YAML file for obsspace in evaobs: - name = obsspace['name'] - cycle = obsspace['diagfile'].split('.')[-2] + name = obsspace['name'].replace(" ", "_").lower() + cycle = obsspace['diagfile'].split('_')[-1].split('.')[-2] logging.info(f'Now processing: {name}') # get the dictionary of replacements set up replacements = { diff --git a/ush/ufoeval/run_ufo_hofx_test.sh b/ush/ufoeval/run_ufo_hofx_test.sh index 5cb77fb6e..8d685d231 100755 --- a/ush/ufoeval/run_ufo_hofx_test.sh +++ b/ush/ufoeval/run_ufo_hofx_test.sh @@ -28,7 +28,7 @@ usage() { # ============================================================================== -cycle=2021080100 +cycle=2024021900 run_filtering=YES run_eva=YES eva_stats_only=NO @@ -77,6 +77,8 @@ if [ $machine = orion ]; then workdir=/work2/noaa/da/$LOGNAME/ufoeval/$cycle/${obtype} fi GDASApp=${GDASApp:-/work2/noaa/da/$LOGNAME/git/GDASApp/} # Change this to your own branch + JCBinstall=${JCBinstall:-/work2/noaa/da/cmartin/CI/GDASApp/opt} + JCBpylib=$JCBinstall/lib/python3.7/site-packages elif [ $machine = hera ]; then if [ $run_filtering == NO ]; then workdir=/scratch1/NCEPDEV/stmp2/$LOGNAME/ufoeval/$cycle/${obtype}_noqc @@ -84,6 +86,8 @@ elif [ $machine = hera ]; then workdir=/scratch1/NCEPDEV/stmp2/$LOGNAME/ufoeval/$cycle/${obtype} fi GDASApp=${GDASApp:-/scratch1/NCEPDEV/da/$LOGNAME/git/GDASApp/} # Change this to your own branch + JCBinstall=${JCBinstall:-/scratch1/NCEPDEV/da/Cory.R.Martin/CI/GDASApp/opt} + JCBpylib=$JCBinstall/lib/python3.10/site-packages else echo "Machine " $machine "not found" exit 1 @@ -94,17 +98,12 @@ if [ $keep_output = YES ]; then workdir=${workdir}_datetime fi -if [ $run_filtering == NO ]; then - yamlpath=$GDASApp/parm/atm/obs/testing/${obtype}_noqc.yaml -else - yamlpath=$GDASApp/parm/atm/obs/testing/${obtype}.yaml -fi exename=test_ObsFilters.x #-------------- Do not modify below this line ---------------- # paths that should only be changed by an expert user -dataprocdate=20230811 # Production date of test data +dataprocdate=20240815 # Production date of test data obtype_short=${obtype:0:4} if [ $obtype_short = "cris" ] || [ $obtype_short = "iasi" ] || [ $obtype_short = "hirs" ] || [ $obtype_short = "sevi" ] || \ @@ -116,7 +115,7 @@ else fi if [ $machine = orion ]; then - export Datapath='/work2/noaa/da/eliu/UFO_eval/data/gsi_geovals_l127/nofgat_aug2021/'$dataprocdate + export Datapath='/work2/noaa/da/acollard/UFO_eval/data/gsi_geovals_l127/nofgat_feb2024/'$dataprocdate FixDir=/work2/noaa/da/cmartin/GDASApp/fix elif [ $machine = hera ]; then export Datapath='/scratch1/NCEPDEV/da/Emily.Liu/UFO_eval/data/gsi_geovals_l127/nofgat_aug2021/'$dataprocdate @@ -133,14 +132,29 @@ BCDir=$Datapath/bc/ # other variables that should not change often export CDATE=$cycle export assim_freq=6 +export half_assim_freq=$(($assim_freq / 2)) export GDATE=$(date +%Y%m%d%H -d "${CDATE:0:8} ${CDATE:8:2} - ${assim_freq} hours") +export BDATE=$(date +%Y%m%d%H -d "${CDATE:0:8} ${CDATE:8:2} - ${half_assim_freq} hours") export PDY=${CDATE:0:8} export cyc=${CDATE:8:2} +export YYYY=${CDATE:0:4} +export MM=${CDATE:4:2} +export DD=${CDATE:6:2} export gPDY=${GDATE:0:8} export gcyc=${GDATE:8:2} +export bPDY=${BDATE:0:8} +export bcyc=${BDATE:8:2} +export bYYYY=${BDATE:0:4} +export bMM=${BDATE:4:2} +export bDD=${BDATE:6:2} export CASE="C768" export CASE_ANL="C384" export LEVS="128" +export DATA=./ +export COMPONENT=atmos +export OPREFIX=gdas.t${cyc}z +export APREFIX=gdas.t${cyc}z +export GPREFIX=gdas.t${gcyc}z # Load Modules for GDASApp module use $GDASApp/modulefiles @@ -157,12 +171,13 @@ ln -sf $FixDir/crtm/2.4.0 $workdir/crtm # copy BC files if [ $radiance = "YES" ]; then - cp -rf $BCDir/${obtype}*${GDATE}* $workdir/. + cp -rf $BCDir/${GPREFIX}.${obtype}.tlapse.txt $workdir/${GPREFIX}.${obtype}.tlapse.txt + cp -rf $BCDir/${GPREFIX}.${obtype}.satbias.nc $workdir/${GPREFIX}.${obtype}.satbias.nc fi # Copy obs and geovals -cp -rf $GeoDir/${obtype}_geoval_${cycle}*.nc4 $workdir/. -cp -rf $ObsDir/${obtype}_obs_${cycle}*.nc4 $workdir/. +cp -rf $GeoDir/${obtype}_geoval_${cycle}.nc4 $workdir/${OPREFIX}.${obtype}_geoval.tm00.nc +cp -rf $ObsDir/${obtype}_obs_${cycle}.nc4 $workdir/${OPREFIX}.${obtype}.tm00.nc # Link executable ln -sf $GDASApp/build/bin/$exename $workdir/. @@ -170,22 +185,75 @@ ln -sf $GDASApp/build/bin/$exename $workdir/. echo "Generating YAML" # Copy/generate YAML for test executable -# First, create the input YAMLs for the genYAML script -export DATA=./ -export COMPONENT=atmos -export OPREFIX=gdas.t${cyc}z -export APREFIX=gdas.t${cyc}z -export GPREFIX=gdas.t${gcyc}z +# need to add JCB to PATH and PYTHONPATH +export PYTHONPATH=$PYTHONPATH:$JCBpylib +export PATH=$PATH:$JCBinstall/bin +# First, create the input file for JCB cat > $workdir/temp.yaml << EOF -time window: - begin: '{{ WINDOW_BEGIN | to_isotime }}' - end: '{{ WINDOW_END | to_isotime }}' - bound to include: begin -observations: -- !INC $yamlpath +# Search path for model and obs for JCB +# ------------------------------------- +algorithm_path: "$GDASApp/parm/jcb-algorithms" +app_path_algorithm: "$GDASApp/parm/jcb-gdas/algorithm/atmosphere" +app_path_model: "$GDASApp/parm/jcb-gdas/model/atmosphere" +app_path_observations: "$GDASApp/parm/jcb-gdas/observations/atmosphere" +app_path_observation_chronicle: "$GDASApp/parm/jcb-gdas/observation_chronicle/atmosphere" + + +# Places where we deviate from the generic file name of a yaml +# ------------------------------------------------------------ +#final_increment_file: final_increment_gaussian +final_increment_file: final_increment_cubed_sphere +output_ensemble_increments_file: output_ensemble_increments_cubed_sphere +model_file: model_pseudo +initial_condition_file: background # Initial conditions for 4D apps is background + + +# Assimilation window +# ------------------- +window_begin: "${bYYYY}-${bMM}-${bDD}T${bcyc}:00:00Z" +window_length: "PT${assim_freq}H" +bound_to_include: begin + +# Default background time is for 3D applications +atmosphere_background_time_iso: "${YYYY}-${MM}-${DD}T${cyc}:00:00Z" + +algorithm: test_obs_filters + +# Observation things +# ------------------ +observations: [${obtype}] + +crtm_coefficient_path: "$workdir/crtm/" + +# Naming conventions for observational files +atmosphere_obsdatain_path: "$workdir" +atmosphere_obsdatain_prefix: "$OPREFIX." +atmosphere_obsdatain_suffix: ".tm00.nc" + +atmosphere_obsdataout_path: "$workdir" +atmosphere_obsdataout_prefix: diag_ +atmosphere_obsdataout_suffix: "_${cycle}.nc" + +# Naming conventions for bias correction files +atmosphere_obsbiasin_path: "$workdir" +atmosphere_obsbiasin_prefix: "$GPREFIX." +atmosphere_obsbiasin_suffix: ".satbias.nc" +atmosphere_obstlapsein_prefix: "$GPREFIX." +atmosphere_obstlapsein_suffix: ".tlapse.txt" +atmosphere_obsbiascovin_prefix: "$GPREFIX." +atmosphere_obsbiascovin_suffix: ".satbias_cov.nc" + +atmosphere_obsbiasout_path: "$workdir" +atmosphere_obsbiasout_prefix: "$APREFIX." +atmosphere_obsbiasout_suffix: ".satbias.nc" +atmosphere_obsbiascovout_prefix: "$APREFIX." +atmosphere_obsbiascovout_suffix: ".satbias_cov.nc" EOF -$GDASApp/ush/genYAML --input $workdir/temp.yaml --output $workdir/${obtype}_${cycle}.yaml +# jcb render dictionary_of_templates.yaml jedi_config.yaml +echo "Calling JCB Render" +jcb render $workdir/temp.yaml $workdir/${obtype}_${cycle}.yaml +echo "Called JCB Render" if [ $? -ne 0 ]; then echo "YAML creation failed" diff --git a/ush/ufoeval/test_yamls.sh b/ush/ufoeval/test_yamls.sh index d46056bd8..4c721f9af 100755 --- a/ush/ufoeval/test_yamls.sh +++ b/ush/ufoeval/test_yamls.sh @@ -14,7 +14,7 @@ if [ $? -ne 0 ]; then fi # Process tests wiht QC -for file in `find ../../parm/atm/obs/testing/*.yaml -type f -not -name "*noqc*"`; do +for file in `find ../../parm/jcb-gdas/observations/atmosphere -type f`; do basefile=${file##*/} obtype="${basefile%.*}" ./run_ufo_hofx_test.sh -x $obtype > $WORKDIR/$obtype.log 2> $WORKDIR/$obtype.err @@ -26,7 +26,7 @@ for file in `find ../../parm/atm/obs/testing/*.yaml -type f -not -name "*noqc*"` done # Process tests without QC (HofX + Observation error assignment) -for file in `ls ../../parm/atm/obs/testing/*_noqc.yaml`; do +for file in `find ../../parm/jcb-gdas/observations/atmosphere -type f`; do basefile=${file##*/} obtype="${basefile%_noqc.*}" ./run_ufo_hofx_test.sh -x -q $obtype > $WORKDIR/${obtype}_noqc.log 2> $WORKDIR/${obtype}_noqc.err