Bugfix 2867 point2grid qc flag (#2890)

* #2867 Added compute_adp_qc_flag and adjusted ADP QC flags

* #2867 Added point2grid_GOES_16_ADP_Enterprise_high. Changed AOD QC flags to 0,1,2 (was 1,2,3)

* #2867 Added get_nc_att_values_

* #2867 Added get_nc_att_values. Added the argument allow_conversion to get_nc_data(netCDF::NcVar *, uchar  *data)

* #2867 Read the ADP QC flag values and meanings attributes from DQF variable and set the QC high, meduium, low values to support Enterprise algorithm. Adjusted the ADP QC values by using AOD qc values

* #2867 Cleanup

* #2867 Corrected indent

* #2867 Changed log message

* #2867 Removed unused argument

* #2867 Removed unused argument

* Cleanup

* #2867 Fix SonarQube findings

* #2867 Deleted protected section with no members

* #2867 Cleanup

* #2867 FIxed SonarQube findings; unused local variables, decalare as const, etc

* #2867 MOved include directives to top

* #2867 Changed some argumenmt with references to avoid copying objects

* #2867 Do not filter by QC flag if -qc is not given

* #2867 Use enumj class for GOES QC: HIGH, MEDIUM, and LOW

* #2867 Added log message back which were deleted accidently

* #2867 Chaned statci const to constexpr

* #2867 Initial release. Separated from nc_utils.h

* @2867 Added nc_utils_core.h

* #2867 Moved some blocks to nc_utils_core.h

* #2867 Include nc_utils_core.h

* #2867 Added const references

* Per #2867, fixing typo in comments.

---------

Co-authored-by: Howard Soh <[email protected]>
Co-authored-by: j-opatz <[email protected]>

internal/test_unit/xml/unit_point2grid.xml

@@ -137,7 +137,7 @@
       <grid_nc>&OUTPUT_DIR;/point2grid/point2grid_GOES_16_AOD_TO_G212_compute.nc</grid_nc>
     </output>
   </test>
-  
+
   <test name="point2grid_GOES_16_AOD_TO_G212_GAUSSIAN">
     <exec>&MET_BIN;/point2grid</exec>
     <env>
@@ -155,7 +155,7 @@
       <grid_nc>&OUTPUT_DIR;/point2grid/point2grid_GOES_16_AOD_TO_G212_gaussian.nc</grid_nc>
     </output>
   </test>
-  
+
   <test name="point2grid_GOES_16_ADP">
     <exec>&MET_BIN;/point2grid</exec>
     <env>
@@ -174,7 +174,7 @@
       <grid_nc>&OUTPUT_DIR;/point2grid/point2grid_GOES_16_ADP.nc</grid_nc>
     </output>
   </test>
-  
+
   <test name="point2grid_GOES_16_AOD_TO_G212_GRID_MAP">
     <exec>&MET_BIN;/point2grid</exec>
     <env>
@@ -185,7 +185,7 @@
       G212 \
       &OUTPUT_DIR;/point2grid/point2grid_GOES_16_AOD_TO_G212_grid_map.nc \
       -field 'name="AOD";  level="(*,*)";' \
-      -qc 1,2,3 -method MAX \
+      -qc 0,1,2 -method MAX \
       -v 1
     </param>
     <output>
@@ -205,14 +205,33 @@
       G212 \
       &OUTPUT_DIR;/point2grid/point2grid_GOES_16_AOD_TO_G212.nc \
       -field 'name="AOD";  level="(*,*)";' \
-      -qc 1,2,3 -method MAX \
+      -qc 0,1,2 -method MAX \
       -v 1
     </param>
     <output>
       <grid_nc>&OUTPUT_DIR;/point2grid/point2grid_GOES_16_AOD_TO_G212.nc</grid_nc>
     </output>
   </test>
 
+  <test name="point2grid_GOES_16_ADP_Enterprise_high">
+    <exec>&MET_BIN;/point2grid</exec>
+    <env>
+      <pair><name>MET_TMP_DIR</name>  <value>&OUTPUT_DIR;/point2grid</value></pair>
+    </env>
+    <param> \
+      &DATA_DIR_MODEL;/goes_16/OR_ABI-L2-AODC-M6_G16_s20241100001171_e20241100003544_c20241100006242.nc \
+      G212 \
+      &OUTPUT_DIR;/point2grid/point2grid_GOES_16_ADP_Enterprise_high.nc \
+      -field 'name="AOD_Smoke";  level="(*,*)";' \
+      -adp &DATA_DIR_MODEL;/goes_16/OR_ABI-L2-ADPC-M6_G16_s20241100001171_e20241100003544_c20241100006361.nc \
+      -qc 0,1 -method MAX \
+      -v 1
+    </param>
+    <output>
+      <grid_nc>&OUTPUT_DIR;/point2grid/point2grid_GOES_16_ADP_Enterprise_high.nc</grid_nc>
+    </output>
+  </test>
+
   <test name="point2grid_NCCF_UK">
     <exec>&MET_BIN;/point2grid</exec>
     <env>

src/libcode/vx_nc_util/Makefile.am

@@ -13,7 +13,7 @@ include ${top_srcdir}/Make-include
 noinst_LIBRARIES = libvx_nc_util.a
 libvx_nc_util_a_SOURCES = \
               nc_var_info.cc nc_var_info.h \
-              nc_utils.cc nc_utils.h nc_utils.hpp \
+              nc_utils.cc nc_utils.h nc_utils_core.h nc_utils.hpp \
               write_netcdf.cc write_netcdf.h \
               grid_output.cc grid_output.h \
               load_tc_data.cc load_tc_data.h \

src/libcode/vx_nc_util/Makefile.in

@@ -353,7 +353,7 @@ MAINTAINERCLEANFILES = Makefile.in
 noinst_LIBRARIES = libvx_nc_util.a
 libvx_nc_util_a_SOURCES = \
               nc_var_info.cc nc_var_info.h \
-              nc_utils.cc nc_utils.h nc_utils.hpp \
+              nc_utils.cc nc_utils.h nc_utils_core.h nc_utils.hpp \
               write_netcdf.cc write_netcdf.h \
               grid_output.cc grid_output.h \
               load_tc_data.cc load_tc_data.h \

src/libcode/vx_nc_util/nc_constants.h

@@ -29,10 +29,10 @@
 
 
 // NetCDF keywords
-static const char * const CONFIG_NetCDF_Dimension = "NetCDF_Dimension";
+constexpr char CONFIG_NetCDF_Dimension[] = "NetCDF_Dimension";
 
 // Flag value used to indicate a range of values within a dimension
-static const int range_flag = bad_data_int;
+constexpr int range_flag = bad_data_int;
 
 
 ////////////////////////////////////////////////////////////////////////

src/libcode/vx_nc_util/nc_utils.cc

@@ -537,6 +537,16 @@ bool get_nc_att_value(const NcVar *var, const ConcatString &att_name,
 
 ////////////////////////////////////////////////////////////////////////
 
+bool get_nc_att_values(const NcVar *var, const ConcatString &att_name,
+                       unsigned short *att_val, bool exit_on_error) {
+   static const char *method_name = "get_nc_att_value(NcVar,float) -> ";
+   bool status = get_nc_att_values_(var, att_name, att_val, exit_on_error,
+                                    method_name);
+   return status;
+}
+
+////////////////////////////////////////////////////////////////////////
+
 bool get_nc_att_value(const NcVarAtt *att, ConcatString &att_val) {
    bool status = false;
 
@@ -1750,15 +1760,21 @@ bool get_nc_data(NcVar *var, char **data) {
 
 ////////////////////////////////////////////////////////////////////////
 
-bool get_nc_data(NcVar *var, uchar *data) {
+bool get_nc_data(NcVar *var, uchar *data, bool allow_conversion) {
    bool return_status = false;
    int cell_count = get_data_size(var);
    int data_type = GET_NC_TYPE_ID_P(var);
    static const char *method_name = "get_nc_data(NcVar *, uchar *) -> ";
    if (NC_UBYTE == data_type) return_status = get_nc_data_t(var, data);
-   else if (NC_BYTE == data_type && has_unsigned_attribute(var)) {
+   else if (NC_BYTE == data_type) {
+      if (!has_unsigned_attribute(var) && !allow_conversion) {
+         mlog << Debug(1) << "\n" << method_name
+              << "INFO: Unexpected conversion from 'ncbyte' for variable \""
+              << GET_NC_NAME_P(var) << "\".\n\n";
+      }
       ncbyte *signed_data = new ncbyte[cell_count];
-      if (return_status = get_nc_data_t(var, signed_data)) {
+      return_status = get_nc_data_t(var, signed_data);
+      if (return_status) {
          for (int idx=0; idx<cell_count; idx++) {
             data[idx] = (uchar)signed_data[idx];
          }

src/libcode/vx_nc_util/nc_utils.h

@@ -18,51 +18,19 @@
 
 #include <netcdf>
 
-#ifndef ncbyte
-typedef signed char ncbyte; // from ncvalues.h
-#endif   /*  ncbyte  */
-#ifndef uchar
-typedef unsigned char uchar;
-#endif   /*  uchar  */
-
 #include "concat_string.h"
 #include "int_array.h"
 #include "long_array.h"
 #include "num_array.h"
 #include "nc_var_info.h"
 
-////////////////////////////////////////////////////////////////////////
-
-static const std::string C_unknown_str = std::string("unknown");
-
-#define IS_VALID_NC(ncObj)          (!ncObj.isNull())
-#define IS_VALID_NC_P(ncObjPtr)     ((ncObjPtr != nullptr && !ncObjPtr->isNull()))
-
-#define IS_INVALID_NC(ncObj)        ncObj.isNull()
-#define IS_INVALID_NC_P(ncObjPtr)   (ncObjPtr == nullptr || ncObjPtr->isNull())
-
-#define GET_NC_NAME(ncObj)          ncObj.getName()
-#define GET_NC_NAME_P(ncObjPtr)     ncObjPtr->getName()
-
-#define GET_NC_SIZE(ncObj)          ncObj.getSize()
-#define GET_NC_SIZE_P(ncObjPtr)     ncObjPtr->getSize()
-
-#define GET_SAFE_NC_NAME(ncObj)         (ncObj.isNull() ? C_unknown_str : ncObj.getName())
-#define GET_SAFE_NC_NAME_P(ncObjPtr)    (IS_INVALID_NC_P(ncObjPtr) ? C_unknown_str : ncObjPtr->getName())
-
-#define GET_NC_TYPE_ID(ncObj)           ncObj.getType().getId()
-#define GET_NC_TYPE_ID_P(ncObjPtr)      ncObjPtr->getType().getId()
-#define GET_NC_TYPE_NAME(ncObj)         ncObj.getType().getName()
-#define GET_NC_TYPE_NAME_P(ncObjPtr)    ncObjPtr->getType().getName()
-
-#define GET_NC_DIM_COUNT(ncObj)         ncObj.getDimCount()
-#define GET_NC_DIM_COUNT_P(ncObjPtr)    ncObjPtr->getDimCount()
+#include "nc_utils.hpp"
 
-#define GET_NC_VAR_COUNT(ncObj)         ncObj.getVarCount()
-#define GET_NC_VAR_COUNT_P(ncObjPtr)    ncObjPtr->getVarCount()
+////////////////////////////////////////////////////////////////////////
 
-#define GET_NC_VARS(ncObj)              ncObj.getVars()
-#define GET_NC_VARS_P(ncObjPtr)         ncObjPtr->getVars()
+#ifndef uchar
+typedef unsigned char uchar;
+#endif   /*  uchar  */
 
 ////////////////////////////////////////////////////////////////////////
 
@@ -135,11 +103,9 @@ static const std::string coordinates_att_name          = "coordinates";
 static const std::string coordinate_axis_type_att_name = "_CoordinateAxisType";
 static const std::string cf_att_name                   = "Conventions";
 static const std::string description_att_name          = "description";
-static const std::string fill_value_att_name           = "_FillValue";
 static const std::string grid_mapping_att_name         = "grid_mapping";
 static const std::string grid_mapping_name_att_name    = "grid_mapping_name";
 static const std::string long_name_att_name            = "long_name";
-static const std::string missing_value_att_name        = "missing_value";
 static const std::string projection_att_name           = "Projection";
 static const std::string scale_factor_att_name         = "scale_factor";
 static const std::string standard_name_att_name        = "standard_name";
@@ -183,9 +149,6 @@ extern bool      get_att_no_leap_year(const netCDF::NcVar *);
 
 extern bool      get_cf_conventions(const netCDF::NcFile *, ConcatString&);
 
-extern netCDF::NcVarAtt   *get_nc_att(const netCDF::NcVar  *, const ConcatString &, bool exit_on_error = false);
-extern netCDF::NcGroupAtt *get_nc_att(const netCDF::NcFile *, const ConcatString &, bool exit_on_error = false);
-
 extern bool get_nc_att_value(const netCDF::NcVarAtt *, std::string &);
 extern bool get_nc_att_value(const netCDF::NcVarAtt *, int          &, bool exit_on_error = true);
 extern bool get_nc_att_value(const netCDF::NcVarAtt *, float        &, bool exit_on_error = true);
@@ -195,6 +158,7 @@ extern bool get_nc_att_value(const netCDF::NcVar *, const ConcatString &, Concat
 extern bool get_nc_att_value(const netCDF::NcVar *, const ConcatString &, int          &, bool exit_on_error = false);
 extern bool get_nc_att_value(const netCDF::NcVar *, const ConcatString &, float        &, bool exit_on_error = false);
 extern bool get_nc_att_value(const netCDF::NcVar *, const ConcatString &, double       &, bool exit_on_error = false);
+extern bool get_nc_att_values(const netCDF::NcVar *, const ConcatString &, unsigned short *, bool exit_on_error = false);
 
 extern bool has_att(netCDF::NcFile *, const ConcatString name, bool exit_on_error=false);
 extern bool has_att(netCDF::NcVar *, const ConcatString name, bool do_log=false);
@@ -257,11 +221,10 @@ extern ConcatString* get_string_val(netCDF::NcVar *var, const int index, const i
 extern bool get_nc_data(netCDF::NcVar *, int    *data);
 extern bool get_nc_data(netCDF::NcVar *, char   *data);
 extern bool get_nc_data(netCDF::NcVar *, char  **data);
-extern bool get_nc_data(netCDF::NcVar *, uchar  *data);
+extern bool get_nc_data(netCDF::NcVar *, uchar  *data, bool allow_conversion=false);
 extern bool get_nc_data(netCDF::NcVar *, float  *data);
 extern bool get_nc_data(netCDF::NcVar *, double *data);
 extern bool get_nc_data(netCDF::NcVar *, time_t *data);
-extern bool get_nc_data(netCDF::NcVar *, ncbyte *data);
 extern bool get_nc_data(netCDF::NcVar *, unsigned short *data);
 
 extern bool get_nc_data(netCDF::NcVar *, int    *data, const LongArray &curs);
@@ -359,14 +322,8 @@ extern netCDF::NcDim  add_dim(netCDF::NcFile *, const std::string &);
 extern netCDF::NcDim  add_dim(netCDF::NcFile *, const std::string &, const size_t);
 extern bool   has_dim(netCDF::NcFile *, const char *dim_name);
 extern bool   get_dim(const netCDF::NcFile *, const ConcatString &, int &, bool error_out = false);
-extern int    get_dim_count(const netCDF::NcVar *);
 extern int    get_dim_count(const netCDF::NcFile *);
-extern int    get_dim_size(const netCDF::NcDim *);
-extern int    get_dim_size(const netCDF::NcVar *, const int dim_offset);
 extern int    get_dim_value(const netCDF::NcFile *, const std::string &, const bool error_out = false);
-extern netCDF::NcDim  get_nc_dim(const netCDF::NcFile *, const std::string &dim_name);
-extern netCDF::NcDim  get_nc_dim(const netCDF::NcVar *, const std::string &dim_name);
-extern netCDF::NcDim  get_nc_dim(const netCDF::NcVar *, const int dim_offset);
 extern bool   get_dim_names(const netCDF::NcVar *var, StringArray *dimNames);
 extern bool   get_dim_names(const netCDF::NcFile *nc, StringArray *dimNames);
 
@@ -376,7 +333,6 @@ extern netCDF::NcVar  get_nc_var_time(const netCDF::NcFile *nc);
 extern int    get_index_at_nc_data(netCDF::NcVar *var, double value, const std::string dim_name, bool is_time=false);
 extern netCDF::NcFile* open_ncfile(const char * nc_name, bool write = false);
 
-extern int get_data_size(netCDF::NcVar *);
 extern unixtime get_reference_unixtime(netCDF::NcVar *time_var, int &sec_per_unit,
                                        bool &no_leap_year);
 
@@ -390,10 +346,6 @@ extern void parse_time_string(const char *str, unixtime &ut);
 
 ////////////////////////////////////////////////////////////////////////
 
-#include "nc_utils.hpp"
-
-////////////////////////////////////////////////////////////////////////
-
 #endif   /*  __NC_UTILS_H__  */
 
 ////////////////////////////////////////////////////////////////////////

src/libcode/vx_nc_util/nc_utils.hpp

@@ -13,6 +13,10 @@
 
 ////////////////////////////////////////////////////////////////////////
 
+#include "nc_utils_core.h"
+
+////////////////////////////////////////////////////////////////////////
+
 extern bool get_att_value(const netCDF::NcAtt *att, int &att_val);
 extern bool get_att_value(const netCDF::NcAtt *att, ConcatString &value);
 extern bool get_att_value(const netCDF::NcAtt *att, ncbyte &att_val);
@@ -93,12 +97,33 @@ bool get_nc_att_value_(const netCDF::NcVar *var, const ConcatString &att_name,
    if (!status) {
       mlog << Error << "\n" << caller_name
            << get_log_msg_for_att(att, GET_SAFE_NC_NAME_P(var), att_name);
-      if (exit_on_error) {
-         if (att) delete att;
-         exit(1);
-      }
    }
    if (att) delete att;
+   if (!status && exit_on_error) exit(1);
+
+   return status;
+}
+
+////////////////////////////////////////////////////////////////////////
+
+template <typename T>
+bool get_nc_att_values_(const netCDF::NcVar *var, const ConcatString &att_name,
+                        T *att_vals, bool exit_on_error,
+                        const char *caller_name) {
+   // caller should initialize att_vals
+
+   //
+   // Retrieve the NetCDF variable attribute.
+   //
+   netCDF::NcVarAtt *att = get_nc_att(var, att_name);
+   bool status = IS_VALID_NC_P(att);
+   if (status) att->getValues(att_vals);
+   else {
+      mlog << Error << "\n" << caller_name
+           << get_log_msg_for_att(att, GET_SAFE_NC_NAME_P(var), att_name);
+   }
+   if (att) delete att;
+   if (!status && exit_on_error) exit(1);
 
    return status;
 }