Merge branch 'develop' into feature/submodule

RussTreadon-NOAA · Dec 21, 2023 · d82da3d · d82da3d
2 parents b052a8a + f933f90
commit d82da3d
Show file tree

Hide file tree

Showing 18 changed files with 112 additions and 44 deletions.
diff --git a/build.sh b/build.sh
@@ -103,6 +103,7 @@ CMAKE_OPTS+=" -DWORKFLOW_TESTS=${WORKFLOW_BUILD}"
 
 # JCSDA changed test data things, need to make a dummy CRTM directory
 if [[ $BUILD_TARGET == 'hera' ]]; then
+  if [ -d "$dir_root/test-data-release/" ]; then rm -rf $dir_root/test-data-release/; fi
   mkdir -p $dir_root/test-data-release/
   ln -sf $GDASAPP_TESTDATA/crtm $dir_root/test-data-release/crtm
 fi

diff --git a/ci/gw_driver.sh b/ci/gw_driver.sh
@@ -67,15 +67,12 @@ for pr in $open_pr_list; do
   # clone global workflow develop branch
   git clone --recursive $workflow_url
 
-  # run checkout script for all other components
-  cd $GDAS_CI_ROOT/workflow/PR/$pr/global-workflow/sorc
-  ./checkout.sh -u
-
   # checkout pull request
-  cd gdas.cd
+  cd $GDAS_CI_ROOT/workflow/PR/$pr/global-workflow/sorc/gdas.cd
   git checkout develop
   git pull
   gh pr checkout $pr
+  git submodule update --init --recursive
 
   # get commit hash
   commit=$(git log --pretty=format:'%h' -n 1)

diff --git a/ci/run_gw_ci.sh b/ci/run_gw_ci.sh
@@ -42,7 +42,7 @@ export WORKFLOW_BUILD="ON"
 cd $repodir/sorc
 module purge
 rm -rf log.build
-./build_all.sh &>> log.build
+./build_all.sh -u &>> log.build
 build_status=$?
 if [ $build_status -eq 0 ]; then
   echo "Build:                                 *SUCCESS*" >> $outfile

diff --git a/ci/stable_driver.sh b/ci/stable_driver.sh
@@ -46,6 +46,7 @@ case ${TARGET} in
     ;;
 esac
 
+set -x
 # ==============================================================================
 datestr="$(date +%Y%m%d)"
 repo_url="https://github.com/NOAA-EMC/GDASApp.git"
@@ -58,12 +59,8 @@ cd $stableroot/$datestr
 # clone global workflow develop branch
 git clone --recursive $workflow_url
 
-# run checkout script for all other components
-cd $stableroot/$datestr/global-workflow/sorc
-./checkout.sh -u
-
 # checkout develop
-cd gdas.cd
+cd $stableroot/$datestr/global-workflow/sorc/gdas.cd
 git checkout develop
 git pull
 

diff --git a/scripts/exglobal_prep_ocean_obs.py b/scripts/exglobal_prep_ocean_obs.py
@@ -6,7 +6,7 @@
 import os
 import prep_marine_obs
 import subprocess
-from wxflow import YAMLFile, save_as_yaml
+from wxflow import YAMLFile, save_as_yaml, FileHandler
 
 # TODO (AFE) figure out why logger is not logging
 # set up logger
@@ -23,6 +23,7 @@
 windowEnd = windowEndDatetime.strftime('%Y-%m-%dT%H:%M:%SZ')
 
 OCNOBS2IODAEXEC = os.getenv('OCNOBS2IODAEXEC')
+COMOUT_OBS = os.getenv('COMOUT_OBS')
 
 OBS_YAML = os.getenv('OBS_YAML')
 # this will fail with FileNotFoundError if all yaml files in OBS_YAML are not
@@ -32,6 +33,8 @@
 OBSPROC_YAML = os.getenv('OBSPROC_YAML')
 obsprocConfig = YAMLFile(OBSPROC_YAML)
 
+filesToSave = []
+
 # TODO (AFE): needs more error handling (missing sources, missing files)
 try:
     # For each of the observation sources (observers) specificed in the OBS_YAML...
@@ -71,6 +74,13 @@
                 save_as_yaml(obsprocSpace, iodaYamlFilename)
 
                 subprocess.run([OCNOBS2IODAEXEC, iodaYamlFilename], check=True)
+
+                filesToSave.append([obsprocSpace['output file'],
+                                    os.path.join(COMOUT_OBS, obsprocSpace['output file'])])
+                filesToSave.append([iodaYamlFilename,
+                                    os.path.join(COMOUT_OBS, iodaYamlFilename)])
 except TypeError:
     print("CRITICAL: Ill-formed OBS_YAML file, exiting")
     raise
+
+FileHandler({'copy': filesToSave}).sync()
diff --git a/test/soca/gw/CMakeLists.txt b/test/soca/gw/CMakeLists.txt
@@ -45,8 +45,8 @@ add_test(NAME test_gdasapp_soca_setup_obsproc
          WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/obsproc)
 
 # Test JGDAS_GLOBAL_OCEAN_ANALYSIS_*
-set(jjob_list "JGLOBAL_PREP_OCEAN_OBS"
-              "JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP"
+set(jjob_list "JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP"
+              "JGLOBAL_PREP_OCEAN_OBS"
               "JGDAS_GLOBAL_OCEAN_ANALYSIS_BMAT"
               "JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN"
               "JGDAS_GLOBAL_OCEAN_ANALYSIS_CHKPT"

diff --git a/test/soca/gw/prepdata.sh b/test/soca/gw/prepdata.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+# called for test_gdasapp_util_prepdata, and by
+# test/soca/gw/setup_obsproc.sh for test_gdasapp_soca_setup_obsproc
+
+
+set -e
+
+cdl2nc4() {
+
+  local output_nc4="$1"
+  local input_cdl="$2"
+
+  echo "Generating ${output_nc4}"
+  ncgen -o "$output_nc4" "$input_cdl"
+}
+
+project_source_dir=$1
+
+
+cdl2nc4 ADT/rads_adt_3a_2021181.nc4 ${project_source_dir}/testdata/rads_adt_3a_2021181.cdl
+cdl2nc4 ADT/rads_adt_3b_2021181.nc4 ${project_source_dir}/testdata/rads_adt_3b_2021181.cdl
+cdl2nc4 icec/icec_amsr2_north_1.nc4 ${project_source_dir}/testdata/icec_amsr2_north_1.cdl
+cdl2nc4 icec/icec_amsr2_north_2.nc4 ${project_source_dir}/testdata/icec_amsr2_north_2.cdl
+cdl2nc4 icec/icec_amsr2_south_1.nc4 ${project_source_dir}/testdata/icec_amsr2_south_1.cdl
+cdl2nc4 icec/icec_amsr2_south_2.nc4 ${project_source_dir}/testdata/icec_amsr2_south_2.cdl
+cdl2nc4 SSS/sss_smap_1.nc4 ${project_source_dir}/testdata/sss_smap_1.cdl
+cdl2nc4 SSS/sss_smap_2.nc4 ${project_source_dir}/testdata/sss_smap_2.cdl
+cdl2nc4 SSS/sss_smos_1.nc4 ${project_source_dir}/testdata/sss_smos_1.cdl
+cdl2nc4 SSS/sss_smos_2.nc4 ${project_source_dir}/testdata/sss_smos_2.cdl
+cdl2nc4 sst/ghrsst_sst_mb_202107010000.nc4 ${project_source_dir}/testdata/ghrsst_sst_mb_202107010000.cdl
+cdl2nc4 sst/ghrsst_sst_mb_202107010100.nc4 ${project_source_dir}/testdata/ghrsst_sst_mb_202107010100.cdl
+cdl2nc4 sst/viirs_aod_1.nc4 ${project_source_dir}/testdata/viirs_aod_1.cdl
+cdl2nc4 sst/viirs_aod_2.nc4 ${project_source_dir}/testdata/viirs_aod_2.cdl
diff --git a/test/soca/gw/setup_obsproc.sh b/test/soca/gw/setup_obsproc.sh
@@ -13,7 +13,7 @@ cd ${test_dmpdir}
 
 mkdir SSS ADT icec sst
 
-${project_source_dir}/utils/test/prepdata.sh ${project_source_dir}/utils/test/
+${project_source_dir}/test/soca/gw/prepdata.sh ${project_source_dir}/utils/test/
 
 
 
diff --git a/utils/obsproc/Ghrsst2Ioda.h b/utils/obsproc/Ghrsst2Ioda.h
@@ -181,8 +181,14 @@ namespace gdasapp {
       // number of obs after subsampling
       int nobs = sst_s.size() * sst_s[0].size();
 
+      // Set the int metadata names
+      std::vector<std::string> intMetadataNames = {"oceanBasin"};
+
+      // Set the float metadata name
+      std::vector<std::string> floatMetadataNames = {};
+
       // Create instance of iodaVars object
-      gdasapp::obsproc::iodavars::IodaVars iodaVars(nobs, {}, {});
+      gdasapp::obsproc::iodavars::IodaVars iodaVars(nobs, floatMetadataNames, intMetadataNames);
 
       // Reference time is Jan 01 1981 00:00:00 GMT+0000
       iodaVars.referenceDate_ = refDate;
@@ -197,6 +203,8 @@ namespace gdasapp {
           iodaVars.obsError_(loc)  = obserror_s[i][j];
           iodaVars.preQc_(loc)     = 0;
           iodaVars.datetime_(loc)  = seconds_s[i][j];
+          // Store optional metadata, set ocean basins to -999 for now
+          iodaVars.intMetadata_.row(loc) << -999;
           loc += 1;
         }
       }

diff --git a/utils/obsproc/IcecAmsr2Ioda.h b/utils/obsproc/IcecAmsr2Ioda.h
@@ -43,8 +43,14 @@ namespace gdasapp {
       int nobs = dimxSize * dimySize;
       int ntimes = dimxSize * dimySize * dimTimeSize;
 
+      // Set the int metadata names
+      std::vector<std::string> intMetadataNames = {"oceanBasin"};
+
+      // Set the float metadata name
+      std::vector<std::string> floatMetadataNames = {};
+
       // Create instance of iodaVars object
-      gdasapp::obsproc::iodavars::IodaVars iodaVars(nobs, {}, {});
+      gdasapp::obsproc::iodavars::IodaVars iodaVars(nobs, floatMetadataNames, intMetadataNames);
 
       oops::Log::debug() << "--- iodaVars.location_: " << iodaVars.location_ << std::endl;
 
@@ -103,6 +109,8 @@ namespace gdasapp {
         iodaVars.obsVal_(i) = static_cast<float>(oneDimObsVal[i]*0.01f);
         iodaVars.obsError_(i) = 0.1;  // Do something for obs error
         iodaVars.preQc_(i) = oneDimFlagsVal[i];
+        // Store optional metadata, set ocean basins to -999 for now
+        iodaVars.intMetadata_.row(i) << -999;
       }
 
       // basic test for iodaVars.trim

diff --git a/utils/obsproc/NetCDFToIodaConverter.h b/utils/obsproc/NetCDFToIodaConverter.h
@@ -153,6 +153,18 @@ namespace gdasapp {
         for (const std::string& strMeta : iodaVars.intMetadataName_) {
           tmpIntMeta = ogrp.vars.createWithScales<int>("MetaData/"+strMeta,
                                                          {ogrp.vars["Location"]}, int_params);
+          // get ocean basin masks if asked in the config
+          obsproc::oceanmask::OceanMask* oceanMask = nullptr;
+          if (fullConfig_.has("ocean basin")) {
+             std::string fileName;
+             fullConfig_.get("ocean basin", fileName);
+             oceanMask = new obsproc::oceanmask::OceanMask(fileName);
+
+             for (int i = 0; i < iodaVars.location_; i++) {
+               iodaVars.intMetadata_.coeffRef(i, size(iodaVars.intMetadataName_)-1) =
+                 oceanMask->getOceanMask(iodaVars.longitude_[i], iodaVars.latitude_[i]);
+             }
+          }
           tmpIntMeta.writeWithEigenRegular(iodaVars.intMetadata_.col(count));
           count++;
         }

diff --git a/utils/obsproc/Rads2Ioda.h b/utils/obsproc/Rads2Ioda.h
@@ -123,19 +123,6 @@ namespace gdasapp {
         (iodaVars.obsVal_ > -4.0 && iodaVars.obsVal_ < 4.0);
       iodaVars.trim(boundsCheck);
 
-      // get ocean basin masks if asked in the config
-      obsproc::oceanmask::OceanMask* oceanMask = nullptr;
-      if (fullConfig_.has("ocean basin")) {
-          std::string fileName;
-          fullConfig_.get("ocean basin", fileName);
-          oceanMask = new obsproc::oceanmask::OceanMask(fileName);
-
-          for (int i = 0; i < iodaVars.location_; i++) {
-            iodaVars.intMetadata_.coeffRef(i, 3) =
-              oceanMask->getOceanMask(iodaVars.longitude_[i], iodaVars.latitude_[i]);
-          }
-      }
-
       return iodaVars;
     };
   };  // class Rads2Ioda

diff --git a/utils/obsproc/Smap2Ioda.h b/utils/obsproc/Smap2Ioda.h
@@ -41,7 +41,14 @@ namespace gdasapp {
       int dim1  = ncFile.getDim("phony_dim_1").getSize();
       int nobs = dim0 * dim1;
 
-      gdasapp::obsproc::iodavars::IodaVars iodaVars(nobs, {}, {});
+      // Set the int metadata names
+      std::vector<std::string> intMetadataNames = {"oceanBasin"};
+
+      // Set the float metadata name
+      std::vector<std::string> floatMetadataNames = {};
+
+      // Create instance of iodaVars object
+      gdasapp::obsproc::iodavars::IodaVars iodaVars(nobs, floatMetadataNames, intMetadataNames);
 
       // TODO(AFE): these arrays can be done as 1D vectors, but those need proper ushorts in
       // the input files, at odd with the current ctests
@@ -102,6 +109,8 @@ namespace gdasapp {
           iodaVars.obsError_(loc) = sss_error[i][j];
           iodaVars.preQc_(loc) = sss_qc[i][j];
           iodaVars.datetime_(loc) =  static_cast<int64_t>(obsTime[j] + secondsSinceEpoch);
+          // Store optional metadata, set ocean basins to -999 for now
+          iodaVars.intMetadata_.row(loc) << -999;
         }
       }
 

diff --git a/utils/obsproc/Smos2Ioda.h b/utils/obsproc/Smos2Ioda.h
@@ -37,7 +37,7 @@ namespace gdasapp {
       // Set the int metadata names
       // TODO(AFE): add other metadata in form of
       // std::vector<std::string> intMetadataNames = {"pass", "cycle", "mission"};
-      std::vector<std::string> intMetadataNames = {};
+      std::vector<std::string> intMetadataNames = {"oceanBasin"};
 
       // Set the float metadata name
       // TODO(AFE): add other metadata in form of
@@ -81,6 +81,8 @@ namespace gdasapp {
         iodaVars.obsError_(i) = sss_error[i];
         iodaVars.preQc_(i) = sss_qc[i];
         iodaVars.datetime_(i) =  static_cast<int64_t>(datetime_[i]*86400.0f) + mjd2000;
+        // Store optional metadata, set ocean basins to -999 for now
+        iodaVars.intMetadata_.row(i) << -999;
       }
 
       // basic test for iodaVars.trim

diff --git a/utils/test/testinput/gdas_ghrsst2ioda.yaml b/utils/test/testinput/gdas_ghrsst2ioda.yaml
@@ -9,6 +9,7 @@ bounds:
   min: -3.0
   max: 50.0
 output file: ghrsst_sst_mb_20210701.ioda.nc
+#ocean basin: RECCAP2_region_masks_all_v20221025.nc
 input files:
 - ghrsst_sst_mb_202107010000.nc4
 - ghrsst_sst_mb_202107010100.nc4

diff --git a/utils/test/testinput/gdas_icecamsr2ioda.yaml b/utils/test/testinput/gdas_icecamsr2ioda.yaml
@@ -1,12 +1,13 @@
-provider: AMSR2
-window begin: 2018-04-15T06:00:00Z
-window end: 2018-04-15T12:00:00Z
-output file: icec_amsr2_north.ioda.nc
-input files:
-- icec_amsr2_north_1.nc4
-- icec_amsr2_north_2.nc4
-
-test:
-  reference filename: testref/icecamsr2ioda.test
-  test output filename: testoutput/icecamsr2ioda.test
-  float relative tolerance: 1e-6
+provider: AMSR2
+window begin: 2018-04-15T06:00:00Z
+window end: 2018-04-15T12:00:00Z
+output file: icec_amsr2_north.ioda.nc
+#ocean basin: RECCAP2_region_masks_all_v20221025.nc
+input files:
+- icec_amsr2_north_1.nc4
+- icec_amsr2_north_2.nc4
+
+test:
+  reference filename: testref/icecamsr2ioda.test
+  test output filename: testoutput/icecamsr2ioda.test
+  float relative tolerance: 1e-6
diff --git a/utils/test/testinput/gdas_smap2ioda.yaml b/utils/test/testinput/gdas_smap2ioda.yaml
@@ -2,6 +2,7 @@ provider: SMAP
 window begin: 2018-04-15T06:00:00Z
 window end: 2018-04-15T12:00:00Z
 output file: sss_smap.ioda.nc
+#ocean basin: RECCAP2_region_masks_all_v20221025.nc
 input files:
 - sss_smap_1.nc4
 - sss_smap_2.nc4

diff --git a/utils/test/testinput/gdas_smos2ioda.yaml b/utils/test/testinput/gdas_smos2ioda.yaml
@@ -2,6 +2,7 @@ provider: SMOS
 window begin: 2018-04-15T06:00:00Z
 window end: 2018-04-15T12:00:00Z
 output file: sss_smos.ioda.nc
+#ocean basin: RECCAP2_region_masks_all_v20221025.nc
 input files:
 - sss_smos_1.nc4
 - sss_smos_2.nc4