IODA converters feature observation redating and error amplification (N…

…OAA-EMC#910) #### This PR has two features that are applied to RADS ADT IODA converters ONLY - Changing the obs. datetime if those do not fall in DA cycle window (i.e. 6 hrs) - <del>Increasing obs. error, however, the method is not yet decided (i.e. find the slope and increase linear error)</del> - Linearly increasing obs. error by time dependent (<del>Errors are adjusted based on 0.1 meter per 6 hours from initial 0.1 meter</del>) - Configuration will provide a slope of linear error - Resolves NOAA-EMC#871 --------- Co-authored-by: Guillaume Vernieres <[email protected]>
RussTreadon-NOAA · Feb 26, 2024 · 60c98b6 · 60c98b6
1 parent 10614c9
commit 60c98b6
Show file tree

Hide file tree

Showing 11 changed files with 155 additions and 110 deletions.
diff --git a/utils/obsproc/Ghrsst2Ioda.h b/utils/obsproc/Ghrsst2Ioda.h
@@ -30,8 +30,6 @@ namespace gdasapp {
       oops::Log::info() << "Processing files provided by GHRSST" << std::endl;
 
       // Get the sst bounds from the configuration
-      std::string sstUnits;
-      fullConfig_.get("bounds.units", sstUnits);
       float sstMin;
       fullConfig_.get("bounds.min", sstMin);
       float sstMax;

diff --git a/utils/obsproc/Rads2Ioda.h b/utils/obsproc/Rads2Ioda.h
@@ -28,6 +28,10 @@ namespace gdasapp {
     gdasapp::obsproc::iodavars::IodaVars providerToIodaVars(const std::string fileName) final {
       oops::Log::info() << "Processing files provided by the RADS" << std::endl;
 
+      // Get the obs. error ratio from the configuration
+      float errRatio;
+      fullConfig_.get("error ratio", errRatio);
+
       // Open the NetCDF file in read-only mode
       netCDF::NcFile ncFile(fileName, netCDF::NcFile::read);
       oops::Log::info() << "Reading... " << fileName << std::endl;
@@ -111,7 +115,7 @@ namespace gdasapp {
         iodaVars.latitude_(i) = static_cast<float>(lat[i])*geoscaleFactor;
         iodaVars.datetime_(i) = static_cast<int64_t>(datetime[i]*86400.0f);
         iodaVars.obsVal_(i) = static_cast<float>(adt[i])*scaleFactor;
-        iodaVars.obsError_(i) = 0.1;  // Do something for obs error
+        iodaVars.obsError_(i) = 0.1;  // only within DA window
         iodaVars.preQc_(i) = 0;
         // Save MDT in optional floatMetadata
         iodaVars.floatMetadata_.row(i) << iodaVars.obsVal_(i) -
@@ -123,7 +127,15 @@ namespace gdasapp {
         (iodaVars.obsVal_ > -4.0 && iodaVars.obsVal_ < 4.0);
       iodaVars.trim(boundsCheck);
 
-      return iodaVars;
+      // Redating and adjusting Errors
+      if (iodaVars.datetime_.size() == 0) {
+        oops::Log::info() << "datetime_ is empty" << std::endl;
+      } else {
+        // Redating and Adjusting Error
+        iodaVars.reDate(windowBegin_, windowEnd_, errRatio);
+
+        return iodaVars;
+      }
     };
   };  // class Rads2Ioda
 }  // namespace gdasapp
diff --git a/utils/obsproc/util.h b/utils/obsproc/util.h
@@ -6,6 +6,9 @@
 #include <string>
 #include <vector>
 
+#include "ioda/../../../../core/IodaUtils.h"
+#include "oops/util/dateFunctions.h"
+
 namespace gdasapp {
   namespace obsproc {
     namespace oceanmask {
@@ -218,6 +221,37 @@ namespace gdasapp {
           oops::Log::test() << checksum(latitude_, "latitude") << std::endl;
           oops::Log::test() << checksum(datetime_, "datetime") << std::endl;
         }
+
+       // Changing the date and Adjusting Errors
+       void reDate(const util::DateTime & windowBegin, const util::DateTime & windowEnd,
+                   float errRatio) {
+         // windowBegin and End into DAwindowTimes
+         std::vector<util::DateTime> DAwindowTimes = {windowBegin, windowEnd};
+         // Epoch DateTime from Provider
+         util::DateTime epochDtime("1858-11-17T00:00:00Z");
+         // Convert DA Window DateTime objects to epoch time offsets in seconds
+         std::vector<int64_t> timeOffsets
+                               = ioda::convertDtimeToTimeOffsets(epochDtime, DAwindowTimes);
+
+         int64_t minDAwindow = timeOffsets[0];
+         int64_t maxDAwindow = timeOffsets[1];
+         for (int i = 0; i < location_; i++) {
+           if (datetime_(i) < minDAwindow) {
+             int delta_t = minDAwindow - datetime_(i);
+             datetime_(i) = minDAwindow + 1;
+             // one second is used for safety falling in min DA window
+             obsError_(i) += errRatio * delta_t;
+           }
+           if (maxDAwindow < datetime_(i)) {
+             int delta_t = datetime_(i) - maxDAwindow;
+             datetime_(i) = maxDAwindow - 1;
+             // one second is used for safety falling in max DA window
+             obsError_(i) += errRatio * delta_t;
+             // Error Ratio comes from configuration based on 0.1 meter per 6 hours
+           }
+         }
+         oops::Log::info() << "IodaVars::IodaVars done redating & adjsting errors." << std::endl;
+       }
       };
     }  // namespace iodavars
   }  // namespace obsproc

diff --git a/utils/test/testdata/rads_adt_3a_2021181.cdl b/utils/test/testdata/rads_adt_3a_2021181.cdl
@@ -77,34 +77,34 @@ variables:
 		:mission_name = "SNTNL-3A" ;
 		:mission_phase = "a" ;
 		:log01 = "2021-07-01 | /Users/rads/bin/rads2nc --ymd=20210630000000,20210701000000 -C1,1000 -S3a -Vsla,adt_egm2008,adt_xgm2016,time_mjd,time_dtg,lon,lat,cycle,pass -X/Users/rads/cron/xgm2016 -X/Users/rads/cron/adt -X/Users/rads/cron/time_dtg -o/Users/rads/adt/2021/181/rads_adt_3a_2021181.nc: RAW data from" ;
-		:history = "Mon Sep 25 17:01:30 2023: ncks -d time,0,10 rads_adt_3a_2021181.nc rads_adt_3a_2021181.ncn\n",
+		:history = "Fri Feb  9 09:44:09 2024: ncks -d time,40000,40010 rads_adt_3a_2021181.nc rads_adt_3a_2021181.ncnn\n",
 			"2021-07-01 21:14:30 : /Users/rads/bin/rads2nc --ymd=20210630000000,20210701000000 -C1,1000 -S3a -Vsla,adt_egm2008,adt_xgm2016,time_mjd,time_dtg,lon,lat,cycle,pass -X/Users/rads/cron/xgm2016 -X/Users/rads/cron/adt -X/Users/rads/cron/time_dtg -o/Users/rads/adt/2021/181/rads_adt_3a_2021181.nc" ;
 		:NCO = "netCDF Operators version 5.0.6 (Homepage = http://nco.sf.net, Code = http://github.com/nco/nco)" ;
 data:
 
- adt_egm2008 = 1674, 2176, 1671, 2320, 2011, 2349, 2139, 2174, 2462, 2441, 
-    2487 ;
+ adt_egm2008 = 12711, 12747, 12881, 12856, 13037, 13230, 13258, 13401, 13518, 
+    13529, 13761 ;
 
- adt_xgm2016 = 1621, 2300, 1938, 2652, 2093, 2246, 1948, 1976, 2406, 2469, 
-    2649 ;
+ adt_xgm2016 = 12494, 12347, 12458, 12721, 12954, 13267, 13452, 13733, 13649, 
+    13275, 13453 ;
 
  cycle = 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73 ;
 
- lat = -43973134, -43915104, -43857071, -43799035, -43740996, -43682953, 
-    -43624907, -43566858, -43508805, -43450749, -43392690 ;
+ lat = -20930965, -20989835, -21048703, -21107570, -21166436, -21225301, 
+    -21284164, -21343026, -21401887, -21460747, -21519605 ;
 
- lon = -21550109, -21571346, -21592549, -21613720, -21634859, -21655964, 
-    -21677037, -21698078, -21719086, -21740062, -21761007 ;
+ lon = -173391486, -173405855, -173420232, -173434617, -173449009, 
+    -173463410, -173477819, -173492236, -173506661, -173521094, -173535536 ;
 
- pass = 517, 517, 517, 517, 517, 517, 517, 517, 517, 517, 517 ;
+ pass = 542, 542, 542, 542, 542, 542, 542, 542, 542, 542, 542 ;
 
- sla = 75, 523, -338, 671, 104, 513, 128, 108, 424, 322, 497 ;
+ sla = 1635, 1674, 1807, 1677, 1881, 1906, 2071, 2202, 2204, 2160, 2296 ;
 
- time_dtg = 20210630000000, 20210630000001, 20210630000002, 20210630000003, 
-    20210630000004, 20210630000005, 20210630000006, 20210630000007, 
-    20210630000008, 20210630000009, 20210630000010 ;
+ time_dtg = 20210630212039, 20210630212040, 20210630212041, 20210630212042, 
+    20210630212043, 20210630212044, 20210630212045, 20210630212046, 
+    20210630212047, 20210630212048, 20210630212049 ;
 
- time_mjd = 59395, 59395.0000115741, 59395.0000231481, 59395.0000347222, 
-    59395.0000462963, 59395.0000578704, 59395.0000694444, 59395.0000810185, 
-    59395.0000925926, 59395.0001041667, 59395.0001157407 ;
+ time_mjd = 59395.8893402778, 59395.8893518519, 59395.8893634259, 
+    59395.889375, 59395.8893865741, 59395.8893981482, 59395.8894097222, 
+    59395.8894212963, 59395.8894328704, 59395.8894444444, 59395.8894560185 ;
 }
diff --git a/utils/test/testdata/rads_adt_3b_2021181.cdl b/utils/test/testdata/rads_adt_3b_2021181.cdl
@@ -77,34 +77,34 @@ variables:
 		:mission_name = "SNTNL-3B" ;
 		:mission_phase = "b" ;
 		:log01 = "2021-07-01 | /Users/rads/bin/rads2nc --ymd=20210630000000,20210701000000 -S3b -Vsla,adt_egm2008,adt_xgm2016,time_mjd,time_dtg,lon,lat,cycle,pass -X/Users/rads/cron/xgm2016 -X/Users/rads/cron/adt -X/Users/rads/cron/time_dtg -o/Users/rads/adt/2021/181/rads_adt_3b_2021181.nc: RAW data from" ;
-		:history = "Mon Sep 25 17:01:31 2023: ncks -d time,0,10 rads_adt_3b_2021181.nc rads_adt_3b_2021181.ncn\n",
+		:history = "Fri Feb  9 09:45:12 2024: ncks -d time,40000,40010 rads_adt_3b_2021181.nc rads_adt_3b_2021181.ncnn\n",
 			"2021-07-01 21:31:13 : /Users/rads/bin/rads2nc --ymd=20210630000000,20210701000000 -S3b -Vsla,adt_egm2008,adt_xgm2016,time_mjd,time_dtg,lon,lat,cycle,pass -X/Users/rads/cron/xgm2016 -X/Users/rads/cron/adt -X/Users/rads/cron/time_dtg -o/Users/rads/adt/2021/181/rads_adt_3b_2021181.nc" ;
 		:NCO = "netCDF Operators version 5.0.6 (Homepage = http://nco.sf.net, Code = http://github.com/nco/nco)" ;
 data:
 
- adt_egm2008 = 6505, 7307, 6026, 5871, 5561, 5246, 4981, 4661, 4391, 4409, 
-    4283 ;
+ adt_egm2008 = 11446, 11985, 11588, 11667, 11692, 11709, 12090, 11956, 12252, 
+    12570, 12648 ;
 
- adt_xgm2016 = 3136, 4247, 4498, 4515, 4579, 4741, 4731, 4351, 4016, 3860, 
-    3887 ;
+ adt_xgm2016 = 11224, 12043, 11839, 11945, 11913, 11973, 12488, 12629, 12692, 
+    12403, 11913 ;
 
  cycle = 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54 ;
 
- lat = 59732735, 59505896, 58767628, 58710774, 58653911, 58597040, 58540159, 
-    58483270, 58426372, 58369465, 58312550 ;
+ lat = 27313032, 27254188, 27195343, 27136497, 27077649, 27018800, 26959949, 
+    26901097, 26842244, 26783389, 26724533 ;
 
- lon = 163417447, 163262248, 162770394, 162733328, 162696369, 162659516, 
-    162622770, 162586128, 162549591, 162513159, 162476830 ;
+ lon = -177633145, -177648592, -177664026, -177679449, -177694860, 
+    -177710259, -177725647, -177741023, -177756388, -177771741, -177787082 ;
 
- pass = 232, 232, 232, 232, 232, 232, 232, 232, 232, 232, 232 ;
+ pass = 258, 258, 258, 258, 258, 258, 258, 258, 258, 258, 258 ;
 
- sla = 4578, 5072, 1437, 756, 254, 91, -2, -170, -484, -313, -244 ;
+ sla = -327, 110, -186, -318, -138, -194, 199, 247, 504, 818, 880 ;
 
- time_dtg = 20210630000629, 20210630000633, 20210630000646, 20210630000647, 
-    20210630000648, 20210630000649, 20210630000650, 20210630000651, 
-    20210630000652, 20210630000653, 20210630000654 ;
+ time_dtg = 20210630220837, 20210630220838, 20210630220839, 20210630220840, 
+    20210630220841, 20210630220842, 20210630220843, 20210630220844, 
+    20210630220845, 20210630220846, 20210630220847 ;
 
- time_mjd = 59395.0045023148, 59395.0045486111, 59395.0046990741, 
-    59395.0047106482, 59395.0047222222, 59395.0047337963, 59395.0047453704, 
-    59395.0047569444, 59395.0047685185, 59395.0047800926, 59395.0047916667 ;
+ time_mjd = 59395.922650463, 59395.922662037, 59395.9226736111, 
+    59395.9226851852, 59395.9226967593, 59395.9227083333, 59395.9227199074, 
+    59395.9227314815, 59395.9227430556, 59395.9227546296, 59395.9227662037 ;
 }
diff --git a/utils/test/testinput/gdas_ghrsst2ioda.yaml b/utils/test/testinput/gdas_ghrsst2ioda.yaml
@@ -1,19 +1,19 @@
-provider: GHRSST
-window begin: 2018-04-15T06:00:00Z
-window end: 2018-04-15T12:00:00Z
-binning:
-  stride: 2
-  min number of obs: 1
-bounds:
-  min: -3.0
-  max: 50.0
-output file: ghrsst_sst_mb_20210701.ioda.nc
-#ocean basin: RECCAP2_region_masks_all_v20221025.nc
-input files:
-- ghrsst_sst_mb_202107010000.nc4
-- ghrsst_sst_mb_202107010100.nc4
-
-test:
-  reference filename: testref/ghrsst2ioda.test
-  test output filename: testoutput/ghrsst2ioda.test
-  float relative tolerance: 1e-6
+provider: GHRSST
+window begin: 2021-06-30T21:00:00Z
+window end: 2021-07-01T03:00:00Z
+binning:
+  stride: 2
+  min number of obs: 1
+bounds:
+  min: -3.0
+  max: 50.0
+output file: ghrsst_sst_mb_20210701.ioda.nc
+#ocean basin: RECCAP2_region_masks_all_v20221025.nc
+input files:
+- ghrsst_sst_mb_202107010000.nc4
+- ghrsst_sst_mb_202107010100.nc4
+
+test:
+  reference filename: testref/ghrsst2ioda.test
+  test output filename: testoutput/ghrsst2ioda.test
+  float relative tolerance: 1e-6
diff --git a/utils/test/testinput/gdas_icecamsr2ioda.yaml b/utils/test/testinput/gdas_icecamsr2ioda.yaml
@@ -1,13 +1,13 @@
-provider: AMSR2
-window begin: 2018-04-15T06:00:00Z
-window end: 2018-04-15T12:00:00Z
-output file: icec_amsr2_north.ioda.nc
-#ocean basin: RECCAP2_region_masks_all_v20221025.nc
-input files:
-- icec_amsr2_north_1.nc4
-- icec_amsr2_north_2.nc4
-
-test:
-  reference filename: testref/icecamsr2ioda.test
-  test output filename: testoutput/icecamsr2ioda.test
-  float relative tolerance: 1e-6
+provider: AMSR2
+window begin: 2021-06-30T21:00:00Z
+window end: 2021-07-01T03:00:00Z
+output file: icec_amsr2_north.ioda.nc
+#ocean basin: RECCAP2_region_masks_all_v20221025.nc
+input files:
+- icec_amsr2_north_1.nc4
+- icec_amsr2_north_2.nc4
+
+test:
+  reference filename: testref/icecamsr2ioda.test
+  test output filename: testoutput/icecamsr2ioda.test
+  float relative tolerance: 1e-6
diff --git a/utils/test/testinput/gdas_rads2ioda.yaml b/utils/test/testinput/gdas_rads2ioda.yaml
@@ -1,6 +1,7 @@
 provider: RADS
-window begin: 2018-04-15T06:00:00Z
-window end: 2018-04-15T12:00:00Z
+window begin: 2021-06-30T21:00:00Z
+window end: 2021-07-01T03:00:00Z
+error ratio: 4.63e-6
 output file: rads_adt_3ab_2021181.ioda.nc
 #ocean basin: RECCAP2_region_masks_all_v20221025.nc
 input files:

diff --git a/utils/test/testinput/gdas_smap2ioda.yaml b/utils/test/testinput/gdas_smap2ioda.yaml
@@ -1,13 +1,13 @@
-provider: SMAP
-window begin: 2018-04-15T06:00:00Z
-window end: 2018-04-15T12:00:00Z
-output file: sss_smap.ioda.nc
-#ocean basin: RECCAP2_region_masks_all_v20221025.nc
-input files:
-- sss_smap_1.nc4
-- sss_smap_2.nc4
-
-test:
-  reference filename: testref/smap2ioda.test
-  test output filename: testoutput/smap2ioda.test
-  float relative tolerance: 1e-6
+provider: SMAP
+window begin: 2021-06-30T21:00:00Z
+window end: 2021-07-01T03:00:00Z
+output file: sss_smap.ioda.nc
+#ocean basin: RECCAP2_region_masks_all_v20221025.nc
+input files:
+- sss_smap_1.nc4
+- sss_smap_2.nc4
+
+test:
+  reference filename: testref/smap2ioda.test
+  test output filename: testoutput/smap2ioda.test
+  float relative tolerance: 1e-6
diff --git a/utils/test/testinput/gdas_smos2ioda.yaml b/utils/test/testinput/gdas_smos2ioda.yaml
@@ -1,13 +1,13 @@
-provider: SMOS
-window begin: 2018-04-15T06:00:00Z
-window end: 2018-04-15T12:00:00Z
-output file: sss_smos.ioda.nc
-#ocean basin: RECCAP2_region_masks_all_v20221025.nc
-input files:
-- sss_smos_1.nc4
-- sss_smos_2.nc4
-
-test:
-  reference filename: testref/smos2ioda.test
-  test output filename: testoutput/smos2ioda.test
-  float relative tolerance: 1e-6
+provider: SMOS
+window begin: 2021-06-30T21:00:00Z
+window end: 2021-07-01T03:00:00Z
+output file: sss_smos.ioda.nc
+#ocean basin: RECCAP2_region_masks_all_v20221025.nc
+input files:
+- sss_smos_1.nc4
+- sss_smos_2.nc4
+
+test:
+  reference filename: testref/smos2ioda.test
+  test output filename: testoutput/smos2ioda.test
+  float relative tolerance: 1e-6
diff --git a/utils/test/testref/rads2ioda.test b/utils/test/testref/rads2ioda.test
@@ -1,9 +1,9 @@
 Reading: [rads_adt_3a_2021181.nc4,rads_adt_3b_2021181.nc4]
 seconds since 1858-11-17T00:00:00Z
 obsVal:
-    Min: 0.1671
-    Max: 0.7307
-    Sum: 8.3145
+    Min: 1.1446
+    Max: 1.3761
+    Sum: 27.6532
 obsError:
     Min: 0.1
     Max: 0.1
@@ -13,14 +13,14 @@ preQc:
     Max: 0
     Sum: 0
 longitude:
-    Min: -21.761
-    Max: 163.417
-    Sum: 1552.07
+    Min: -177.787
+    Max: -173.391
+    Sum: -3862.91
 latitude:
-    Min: -43.9731
-    Max: 59.7327
-    Sum: 165.588
+    Min: -21.5196
+    Max: 27.313
+    Sum: 63.7285
 datetime:
-    Min: 5131727872
-    Max: 5131728384
-    Sum: 112898018816
+    Min: 5131805184
+    Max: 5131807744
+    Sum: 112899742208