Merge pull request #381 from kenkehoe/global_attribute_qc

Global attribute qc

act/qc/clean.py

@@ -9,6 +9,8 @@
 import numpy as np
 import copy
 
+from act.qc.qcfilter import parse_bit
+
 
 @xr.register_dataset_accessor('clean')
 class CleanDataset(object):
@@ -548,7 +550,8 @@ def link_variables(self):
     def clean_arm_qc(self,
                      override_cf_flag=True,
                      clean_units_string=True,
-                     correct_valid_min_max=True):
+                     correct_valid_min_max=True,
+                     remove_unset_global_tests=True):
         """
         Function to clean up xarray object QC variables.
 
@@ -566,6 +569,9 @@ def clean_arm_qc(self,
             fail_max and fail_detla if the valid_min, valid_max or valid_delta
             is listed in bit discription attribute. If not listed as
             used with QC will assume is being used correctly.
+        remove_unset_global_tests : bool
+            Option to look for globaly defined tests that are not set at the
+            variable level and remove from quality control variable.
 
         """
         global_qc = self.get_attr_info()
@@ -623,6 +629,49 @@ def clean_arm_qc(self,
                 except KeyError:
                     pass
 
+        # If requested remove tests at variable level that were set from global level descriptions.
+        # This is assuming the test was only performed if the limit value is listed with the variable
+        # even if the global level describes the test.
+        if remove_unset_global_tests and global_qc is not None:
+            limit_name_list = ['fail_min', 'fail_max', 'fail_delta']
+
+            for qc_var_name in self.matched_qc_variables:
+                flag_meanings = self._obj[qc_var_name].attrs['flag_meanings']
+                flag_masks = self._obj[qc_var_name].attrs['flag_masks']
+                tests_to_remove = []
+                for ii, flag_meaning in enumerate(flag_meanings):
+
+                    # Loop over usual test attribute names looking to see if they
+                    # are listed in test description. If so use that name for look up.
+                    test_attribute_limit_name = None
+                    for name in limit_name_list:
+                        if name in flag_meaning:
+                            test_attribute_limit_name = name
+                            break
+
+                    if test_attribute_limit_name is None:
+                        continue
+
+                    remove_test = True
+                    test_number = int(parse_bit(flag_masks[ii]))
+                    for attr_name in self._obj[qc_var_name].attrs:
+                        if test_attribute_limit_name == attr_name:
+                            remove_test = False
+                            break
+
+                        index = self._obj.qcfilter.get_qc_test_mask(
+                            qc_var_name=qc_var_name, test_number=test_number)
+                        if np.any(index):
+                            remove_test = False
+                            break
+
+                    if remove_test:
+                        tests_to_remove.append(test_number)
+
+                if len(tests_to_remove) > 0:
+                    for test_to_remove in tests_to_remove:
+                        self._obj.qcfilter.remove_test(qc_var_name=qc_var_name, test_number=test_to_remove)
+
     def normalize_assessment(self, variables=None, exclude_variables=None,
                              qc_lookup={"Incorrect": "Bad", "Suspect": "Indeterminate"}):
 

act/qc/qcfilter.py

@@ -336,15 +336,18 @@ def add_test(self, var_name, index=None, test_number=None,
 
         return test_dict
 
-    def remove_test(self, var_name, test_number=None, flag_value=False,
+    def remove_test(self, var_name=None, qc_var_name=None, test_number=None, flag_value=False,
                     flag_values_reset_value=0):
         """
-        Method to remove a test/filter from a quality control variable.
+        Method to remove a test/filter from a quality control variable. Must set
+        var_name or qc_var_name.
 
         Parameters
         ----------
-        var_name : str
+        var_name : str or None
             Data variable name.
+        qc_var_name : str or None
+            Quality control variable name. Ignored if var_name is set.
         test_number : int
             Test number to remove.
         flag_value : boolean
@@ -360,9 +363,14 @@ def remove_test(self, var_name, test_number=None, flag_value=False,
         """
         if test_number is None:
             raise ValueError('You need to provide a value for test_number '
-                             'keyword when calling the add_test method')
+                             'keyword when calling the add_test() method')
 
-        qc_var_name = self._obj.qcfilter.check_for_ancillary_qc(var_name)
+        if var_name is None and qc_var_name is None:
+            raise ValueError('You need to provide a value for var_name or qc_var_name '
+                             'keyword when calling the add_test() method')
+
+        if var_name is not None:
+            qc_var_name = self._obj.qcfilter.check_for_ancillary_qc(var_name)
 
         # Determine which index is using the test number
         index = None
@@ -385,16 +393,22 @@ def remove_test(self, var_name, test_number=None, flag_value=False,
 
         if flag_value:
             remove_index = self._obj.qcfilter.get_qc_test_mask(
-                var_name, test_number, return_index=True, flag_value=True)
-            self._obj.qcfilter.unset_test(var_name, remove_index, test_number,
-                                          flag_value, flag_values_reset_value)
+                var_name=var_name, qc_var_name=qc_var_name, test_number=test_number,
+                return_index=True, flag_value=True)
+            self._obj.qcfilter.unset_test(var_name=var_name, qc_var_name=qc_var_name,
+                                          index=remove_index, test_number=test_number,
+                                          flag_value=flag_value,
+                                          flag_values_reset_value=flag_values_reset_value)
             del flag_values[index]
             self._obj[qc_var_name].attrs['flag_values'] = flag_values
+
         else:
             remove_index = self._obj.qcfilter.get_qc_test_mask(
-                var_name, test_number, return_index=True)
-            self._obj.qcfilter.unset_test(var_name, remove_index, test_number,
-                                          flag_value, flag_values_reset_value)
+                var_name=var_name, qc_var_name=qc_var_name, test_number=test_number,
+                return_index=True)
+            self._obj.qcfilter.unset_test(var_name=var_name, qc_var_name=qc_var_name,
+                                          index=remove_index, test_number=test_number,
+                                          flag_value=flag_value)
             del flag_masks[index]
             self._obj[qc_var_name].attrs['flag_masks'] = flag_masks
 
@@ -458,15 +472,17 @@ def set_test(self, var_name, index=None, test_number=None,
 
         self._obj[qc_var_name].values = qc_variable
 
-    def unset_test(self, var_name, index=None, test_number=None,
+    def unset_test(self, var_name=None, qc_var_name=None, index=None, test_number=None,
                    flag_value=False, flag_values_reset_value=0):
         """
         Method to unset a test/filter from a quality control variable.
 
         Parameters
         ----------
-        var_name : str
+        var_name : str or None
             Data variable name.
+        qc_var_name : str or None
+            Quality control variable name. Ignored if var_name is set.
         index : int or list or numpy array
             Index to unset test in quality control array. If want to
             unset all values will need to pass in index of all values.
@@ -488,7 +504,12 @@ def unset_test(self, var_name, index=None, test_number=None,
         if index is None:
             return
 
-        qc_var_name = self._obj.qcfilter.check_for_ancillary_qc(var_name)
+        if var_name is None and qc_var_name is None:
+            raise ValueError('You need to provide a value for var_name or qc_var_name '
+                             'keyword when calling the unset_test() method')
+
+        if var_name is not None:
+            qc_var_name = self._obj.qcfilter.check_for_ancillary_qc(var_name)
 
         qc_variable = self._obj[qc_var_name].values
         if flag_value:
@@ -553,18 +574,21 @@ def available_bit(self, qc_var_name, recycle=False):
 
         return int(next_bit)
 
-    def get_qc_test_mask(self, var_name, test_number, flag_value=False,
-                         return_index=False):
+    def get_qc_test_mask(self, var_name=None, test_number=None, qc_var_name=None,
+                         flag_value=False, return_index=False):
         """
         Returns a numpy array of False or True where a particular
-        flag or bit is set in a numpy array.
+        flag or bit is set in a numpy array. Must set var_name or qc_var_name
+        when calling.
 
         Parameters
         ----------
-        var_name : str
+        var_name : str or None
             Data variable name.
         test_number : int
             Test number to return array where test is set.
+        qc_var_name : str or None
+            Quality control variable name. Ignored if var_name is set.
         flag_value : boolean
             Switch to use flag_values integer quality control.
         return_index : boolean
@@ -609,7 +633,16 @@ def get_qc_test_mask(self, var_name, test_number, flag_value=False,
                         dtype=float32)
 
         """
-        qc_var_name = self._obj.qcfilter.check_for_ancillary_qc(var_name)
+        if var_name is None and qc_var_name is None:
+            raise ValueError('You need to provide a value for var_name or qc_var_name '
+                             'keyword when calling the get_qc_test_mask() method')
+
+        if test_number is None:
+            raise ValueError('You need to provide a value for test_number '
+                             'keyword when calling the get_qc_test_mask() method')
+
+        if var_name is not None:
+            qc_var_name = self._obj.qcfilter.check_for_ancillary_qc(var_name)
 
         qc_variable = self._obj[qc_var_name].values
 

act/tests/test_correct.py

@@ -35,10 +35,10 @@ def test_correct_mpl():
                    181.9355])
     np.testing.assert_allclose(
         sig_cross_pol, [-0.5823283, -1.6066532, -1.7153032,
-                        -2.520143, -2.275405], rtol=4e-07)
+                        -2.520143, -2.275405], rtol=4e-06)
     np.testing.assert_allclose(
         sig_co_pol, [12.5631485, 11.035495, 11.999875,
-                     11.09393, 11.388968])
+                     11.09393, 11.388968], rtol=1e-6)
     np.testing.assert_allclose(
         height, [0.00749012, 0.02247084, 0.03745109,
                  0.05243181, 0.06741206, 0.08239277, 0.09737302,
@@ -79,17 +79,12 @@ def test_correct_dl():
     obj = act.io.armfiles.read_netcdf(files)
 
     new_obj = act.corrections.doppler_lidar.correct_dl(obj, fill_value=np.nan)
-    data = new_obj['attenuated_backscatter'].data
-    data[np.isnan(data)] = 0.
-    data = data * 100.
-    data = data.astype(np.int64)
-    assert np.sum(data) == -18633551
+    data = new_obj['attenuated_backscatter'].values
+    np.testing.assert_almost_equal(np.nansum(data), -186479.83, decimal=0.1)
 
     new_obj = act.corrections.doppler_lidar.correct_dl(obj, range_normalize=False)
-    data = new_obj['attenuated_backscatter'].data
-    data[np.isnan(data)] = 0.
-    data = data.astype(np.int64)
-    assert np.sum(data) == -224000
+    data = new_obj['attenuated_backscatter'].values
+    np.testing.assert_almost_equal(np.nansum(data), -200886.0, decimal=0.1)
 
 
 def test_correct_rl():

act/tests/test_qc.py

@@ -18,6 +18,29 @@ def test_fft_shading_test():
     assert np.nansum(qc_data.values) == 456
 
 
+def test_global_qc_cleanup():
+    ds_object = read_netcdf(EXAMPLE_MET1)
+    ds_object.load()
+    ds_object.clean.cleanup()
+
+    assert ds_object['qc_wdir_vec_mean'].attrs['flag_meanings'] == [
+        'Value is equal to missing_value.', 'Value is less than the fail_min.',
+        'Value is greater than the fail_max.']
+    assert ds_object['qc_wdir_vec_mean'].attrs['flag_masks'] == [1, 2, 4]
+    assert ds_object['qc_wdir_vec_mean'].attrs['flag_assessments'] == ['Bad', 'Bad', 'Bad']
+
+    assert ds_object['qc_temp_mean'].attrs['flag_meanings'] == [
+        'Value is equal to missing_value.', 'Value is less than the fail_min.',
+        'Value is greater than the fail_max.',
+        'Difference between current and previous values exceeds fail_delta.']
+    assert ds_object['qc_temp_mean'].attrs['flag_masks'] == [1, 2, 4, 8]
+    assert ds_object['qc_temp_mean'].attrs['flag_assessments'] == ['Bad', 'Bad',
+                                                                   'Bad', 'Indeterminate']
+
+    ds_object.close()
+    del ds_object
+
+
 def test_qc_test_errors():
     ds_object = read_netcdf(EXAMPLE_MET1)
     var_name = 'temp_mean'
@@ -132,7 +155,7 @@ def test_qcfilter():
     # tests are set.
     assert np.sum(ds_object.qcfilter.get_qc_test_mask(
         var_name, result['test_number'], return_index=True) -
-        np.array(index, dtype=np.int)) == 0
+        np.array(index, dtype=int)) == 0
 
     # Unset a test
     ds_object.qcfilter.unset_test(var_name, index=0,
@@ -555,7 +578,9 @@ def test_qctests_dos():
     test_meaning = ('Data failing persistence test. Standard Deviation over a '
                     'window of 10 values less than 0.0001.')
     assert ds_object[qc_var_name].attrs['flag_meanings'][-1] == test_meaning
-    assert np.sum(ds_object[qc_var_name].values) == 1500
+    # There is a precision issue with GitHub testing that makes the number of tests
+    # tripped off by 1. This isclose() option is to account for that.
+    assert np.isclose(np.sum(ds_object[qc_var_name].values), 1500, atol=1)
 
     ds_object.qcfilter.add_persistence_test(var_name, window=10000, prepend_text='DQO')
     test_meaning = ('DQO: Data failing persistence test. Standard Deviation over a window of '