Feature #2882 seeps qa (#2987)

* Update seeps.h Change variable names to reduce ambiguity for interpretation and aid useability. * Update seeps.cc Pull through variable name changes and renaming of functions to aid legibility and clarity. Introduced some additional debug print statements. * Update grid-stat.rst Add documentation about the location of the gridded climatology files for SEEPS and which environment variable to use. * Replace read_seeps_scores() with get_seeps_climo_grid() * Manually merging Rachel's patch-1 changes. * Getting close to getting these seeps changes to compile. But it's failing in pair_data_point.cc * Per #2882, get branch feature_2882_seeps_qa compiling again. Recommend revisiting the volume of SEEPS-related Debug log messages and reducing them once its fully tested. * Per #2882, need to update the handling of the PPT24_seepsweights_grid.nc file name. Rename as _v12.0.nc for the updated version with the new names so that the existing regressions tests and nightly builds for main_v11.1 and develop continue to work. We can remove the _v12.0 once this feature branch is merged into develop but for the time being, we need both versions to exist. * Per #2882, rename the SEEPS columns from S12, S13, S21, S23, S31, S32 to the more descriptive ODFL, ODFH, OLFD, OLFH, OHFD, OHFL names. * Per #2882, update SEEPS details * Per #2882, store and report the weighted mean fcst and mean obs, just like the SEEPS score itself so that they're handled in a consistent manner. Note however that it's hard-coded to NOT write the weighted means/score, only the unweighted ones. * Per #2882, change SEEPS debug log levels and correct the storage of mean_fcst and mean_obs values. * Per #2882, correct SEEPS column name lookups * Per #2882, call is_bad_data() instead of is_eq(..., -9999.0) to get rid of compiler warning message. * Per #2882, add 2 more variations of the is_eq() function with mixed float and double inputs to satisfy compiler pb2nc compiler warnings. * Per #2882, switch from dynamically allocated arrays to std::vector * Per #2882, enhance Stat-Analysis to write the SEEPS line type to an output .stat file. * Per #2882, update the aggregated seeps computation to use better-initialized vectors. * Per #2882, resolve a few more SonarQube code smells. * Per #2882, now that this PR is ready to merge, remove the v12.0 version number from the gridded SEEPS climo file name ci-skip-all --------- Co-authored-by: mpm-meto <[email protected]>
dtcenter · Oct 9, 2024 · 8151360 · 8151360
1 parent 7d44b8d
commit 8151360
Show file tree

Hide file tree

Showing 13 changed files with 622 additions and 438 deletions.
diff --git a/data/table_files/met_header_columns_V12.0.txt b/data/table_files/met_header_columns_V12.0.txt
@@ -6,7 +6,7 @@ V12.0 : STAT : ISC      : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID
 V12.0 : STAT : MCTC     : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL (N_CAT) F[0-9]*_O[0-9]* EC_VALUE
 V12.0 : STAT : MCTS     : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL N_CAT ACC ACC_NCL ACC_NCU ACC_BCL ACC_BCU HK HK_BCL HK_BCU HSS HSS_BCL HSS_BCU GER GER_BCL GER_BCU HSS_EC HSS_EC_BCL HSS_EC_BCU EC_VALUE
 V12.0 : STAT : MPR      : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL INDEX OBS_SID OBS_LAT OBS_LON OBS_LVL OBS_ELV FCST OBS OBS_QC OBS_CLIMO_MEAN OBS_CLIMO_STDEV OBS_CLIMO_CDF FCST_CLIMO_MEAN FCST_CLIMO_STDEV
-V12.0 : STAT : SEEPS    : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL S12 S13 S21 S23 S31 S32 PF1 PF2 PF3 PV1 PV2 PV3 MEAN_FCST MEAN_OBS SEEPS
+V12.0 : STAT : SEEPS    : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL ODFL ODFH OLFD OLFH OHFD OHFL PF1 PF2 PF3 PV1 PV2 PV3 MEAN_FCST MEAN_OBS SEEPS
 V12.0 : STAT : SEEPS_MPR    : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE OBS_SID OBS_LAT OBS_LON FCST OBS OBS_QC FCST_CAT OBS_CAT P1 P2 T1 T2 SEEPS
 V12.0 : STAT : NBRCNT   : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL FBS FBS_BCL FBS_BCU FSS FSS_BCL FSS_BCU AFSS AFSS_BCL AFSS_BCU UFSS UFSS_BCL UFSS_BCU F_RATE F_RATE_BCL F_RATE_BCU O_RATE O_RATE_BCL O_RATE_BCU
 V12.0 : STAT : NBRCTC   : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL FY_OY FY_ON FN_OY FN_ON

diff --git a/docs/Users_Guide/grid-stat.rst b/docs/Users_Guide/grid-stat.rst
@@ -80,6 +80,8 @@ The Stable Equitable Error in Probability Space (SEEPS) was devised for monitori
 
 The capability to calculate the SEEPS has also been added to Grid-Stat. This follows the method described in :ref:`North et al, 2022 <North-2022>`, which uses the TRMM 3B42 v7 gridded satellite product for the climatological values and interpolates the forecast and observed products onto this grid for evaluation. A 24-hour TRMM climatology (valid at 00 UTC) constructed from data over the time period 1998-2015 is supplied with the release. Expansion of the capability to other fields will occur as well vetted examples and funding allow.
 
+The gridded climatology required to compute SEEPS is not distributed as part of the code release and can be downloaded from `Zenodo <https://zenodo.org/records/13121064?token=eyJhbGciOiJIUzUxMiJ9.eyJpZCI6ImM5NThkNDU4LTEzNDgtNDlmMy05ZjMwLTVkOWQ0MGZjMTFjNyIsImRhdGEiOnt9LCJyYW5kb20iOiI0NzMxYTM3YmNkMWE0MDA4ZWUyMDU4YTdkOTUyMjE4NCJ9.NJZPN0KkouUCQSmB0QjZbfJEOO6d6ZZQ_Me5VLbVaUY4aWQHSqGE4VLmUdLk-uTjN749Wdv92xLYz0aXay5cNw>`. The path to the file needs to be specified using MET_SEEPS_GRID_CLIMO_NAME.
+
 Fourier Decomposition
 ---------------------
 

diff --git a/docs/Users_Guide/point-stat.rst b/docs/Users_Guide/point-stat.rst
@@ -1549,10 +1549,10 @@ The first set of header columns are common to all of the output files generated
     - Quality control flag for observation
   * - 31
     - FCST_CAT
-    - Forecast category to 3 by 3 matrix
+    - Forecast category (dry, light, or heavy)
   * - 32
     - OBS_CAT
-    - Observationtegory to 3 by 3 matrix
+    - Observation category (dry, light, or heavy)
   * - 33
     - P1
     - Climo-derived probability value for this station (dry)
@@ -1561,10 +1561,10 @@ The first set of header columns are common to all of the output files generated
     - Climo-derived probability value for this station (dry + light)
   * - 35
     - T1
-    - Threshold 1 for p1
+    - Threshold 1 for P1 (dry)
   * - 36
     - T2
-    - Threshold 2 for p2
+    - Threshold 2 for P2 (dry + light)
   * - 37
     - SEEPS
     - SEEPS (Stable Equitable Error in Probability Space) score
@@ -1589,41 +1589,41 @@ The first set of header columns are common to all of the output files generated
     - TOTAL
     - Total number of SEEPS matched pairs
   * - 26
-    - S12
-    - Counts multiplied by the weights for FCST_CAT 1 and OBS_CAT 2
+    - ODFL
+    - Counts multiplied by the weights for the observation dry, forecast light category
   * - 27
-    - S13
-    - Counts multiplied by the weights for FCST_CAT 1 and OBS_CAT 3
+    - ODFH
+    - Counts multiplied by the weights for the observation dry, forecast heavy category
   * - 28
-    - S21
-    - Counts multiplied by the weights for FCST_CAT 2 and OBS_CAT 1
+    - OLFD
+    - Counts multiplied by the weights for the observation light, forecast dry category
   * - 29
-    - S23
-    - Counts multiplied by the weights for FCST_CAT 2 and OBS_CAT 3
+    - OLFH
+    - Counts multiplied by the weights for the observation light, forecast heavy category
   * - 30
-    - S31
-    - Counts multiplied by the weights for FCST_CAT 3 and OBS_CAT 1
+    - OHFD
+    - Counts multiplied by the weights for the observation heavy, forecast dry category
   * - 31
-    - S32
-    - Counts multiplied by the weights for FCST_CAT 3 and OBS_CAT 2
+    - OHFL
+    - Counts multiplied by the weights for the observation heavy, forecast light category
   * - 32
     - PF1
-    - marginal probabilities of the forecast values (FCST_CAT 1)
+    - Marginal probabilities of the forecast dry (FCST_CAT 0)
   * - 33
     - PF2
-    - marginal probabilities of the forecast values (FCST_CAT 2)
+    - Marginal probabilities of the forecast light (FCST_CAT 1)
   * - 34
     - PF3
-    - marginal probabilities of the forecast values (FCST_CAT 3)
+    - Marginal probabilities of the forecast heavy (FCST_CAT 2)
   * - 35
     - PV1
-    - marginal probabilities of the observed values (OBS_CAT 1)
+    - Marginal probabilities of the observed dry (OBS_CAT 0)
   * - 36
     - PV2
-    - marginal probabilities of the observed values (OBS_CAT 2)
+    - Marginal probabilities of the observed light (OBS_CAT 1)
   * - 37
     - PV3
-    - marginal probabilities of the observed values (OBS_CAT 3)
+    - Marginal probabilities of the observed heavy (OBS_CAT 2)
   * - 38
     - SEEPS
     - Averaged SEEPS (Stable Equitable Error in Probability Space) score

diff --git a/internal/test_unit/hdr/met_12_0.hdr b/internal/test_unit/hdr/met_12_0.hdr
@@ -6,7 +6,7 @@ ISC           : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_L
 MCTC          : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL N_CAT _VAR_
 MCTS          : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL N_CAT ACC ACC_NCL ACC_NCU ACC_BCL ACC_BCU HK HK_BCL HK_BCU HSS HSS_BCL HSS_BCU GER GER_BCL GER_BCU HSS_EC HSS_EC_BCL HSS_EC_BCU EC_VALUE
 MPR           : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL INDEX OBS_SID OBS_LAT OBS_LON OBS_LVL OBS_ELV FCST OBS OBS_QC OBS_CLIMO_MEAN OBS_CLIMO_STDEV OBS_CLIMO_CDF FCST_CLIMO_MEAN FCST_CLIMO_STDEV
-SEEPS         : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL S12 S13 S21 S23 S31 S32 PF1 PF2 PF3 PV1 PV2 PV3 MEAN_FCST MEAN_OBS SEEPS
+SEEPS         : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL ODFL ODFH OLFD OLFH OHFD OHFL PF1 PF2 PF3 PV1 PV2 PV3 MEAN_FCST MEAN_OBS SEEPS
 SEEPS_MPR     : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE OBS_SID OBS_LAT OBS_LON FCST OBS OBS_QC FCST_CAT OBS_CAT P1 P2 T1 T2 SEEPS
 NBRCNT        : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL FBS FBS_BCL FBS_BCU FSS FSS_BCL FSS_BCU AFSS AFSS_BCL AFSS_BCU UFSS UFSS_BCL UFSS_BCU F_RATE F_RATE_BCL F_RATE_BCU O_RATE O_RATE_BCL O_RATE_BCU
 NBRCTC        : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL FY_OY FY_ON FN_OY FN_ON

diff --git a/internal/test_unit/xml/unit_grid_stat.xml b/internal/test_unit/xml/unit_grid_stat.xml
@@ -296,7 +296,7 @@
       <pair><name>SEEPS_FLAG</name>      <value>BOTH</value></pair>
       <pair><name>SEEPS_P1_THRESH</name> <value>NA</value></pair>
       <pair><name>OUTPUT_PREFIX</name>   <value>SEEPS</value></pair>
-      <pair><name>SEEPS_GRID_CLIMO_NAME</name><value>&DATA_DIR_CLIMO;/seeps/PPT24_seepsweights_grid.nc</value></pair>
+      <pair><name>SEEPS_GRID_CLIMO_NAME</name><value>&DATA_DIR_CLIMO;/seeps/PPT24_seepsweights_grid_v12.0.nc</value></pair>
     </env>
     <param> \
       &DATA_DIR_MODEL;/seeps/gpm_2021120100_2021120200_trmmgrid.nc \

diff --git a/src/basic/vx_util/stat_column_defs.h b/src/basic/vx_util/stat_column_defs.h
@@ -383,9 +383,9 @@ static const char * const seeps_mpr_columns [] = {
 };
 
 static const char * const seeps_columns [] = {
-   "TOTAL",       "S12",         "S13",
-   "S21",         "S23",         "S31",
-   "S32",         "PF1",         "PF2",
+   "TOTAL",       "ODFL",        "ODFH",
+   "OLFD",        "OLFH",        "OHFD",
+   "OHFL",        "PF1",         "PF2",
    "PF3",         "PV1",         "PV2",
    "PV3",         "MEAN_FCST",   "MEAN_OBS",
    "SEEPS"

diff --git a/src/libcode/vx_analysis_util/stat_job.cc b/src/libcode/vx_analysis_util/stat_job.cc
@@ -1994,6 +1994,7 @@ void STATAnalysisJob::setup_stat_file(int n_row, int n) {
          case STATLineType::prc:    c = get_n_prc_columns(n);   break;
          case STATLineType::eclv:   c = get_n_eclv_columns(n);  break;
          case STATLineType::mpr:    c = n_mpr_columns;          break;
+         case STATLineType::seeps:  c = n_seeps_columns;        break;
          case STATLineType::nbrctc: c = n_nbrctc_columns;       break;
          case STATLineType::nbrcts: c = n_nbrcts_columns;       break;
          case STATLineType::nbrcnt: c = n_nbrcnt_columns;       break;
@@ -2065,6 +2066,7 @@ void STATAnalysisJob::setup_stat_file(int n_row, int n) {
          case STATLineType::prc:    write_prc_header_row   (1, n,                                    stat_at, 0, 0); break;
          case STATLineType::eclv:   write_eclv_header_row  (1, n,                                    stat_at, 0, 0); break;
          case STATLineType::mpr:    write_header_row       (mpr_columns, n_mpr_columns, 1,           stat_at, 0, 0); break;
+         case STATLineType::seeps:  write_header_row       (seeps_columns, n_seeps_columns, 1,       stat_at, 0, 0); break;
          case STATLineType::nbrctc: write_header_row       (nbrctc_columns, n_nbrctc_columns, 1,     stat_at, 0, 0); break;
          case STATLineType::nbrcts: write_header_row       (nbrcts_columns, n_nbrcts_columns, 1,     stat_at, 0, 0); break;
          case STATLineType::nbrcnt: write_header_row       (nbrcnt_columns, n_nbrcnt_columns, 1,     stat_at, 0, 0); break;