From 3ba19fc51a0cb5cd572dbe2d154ffe9031d5ee3b Mon Sep 17 00:00:00 2001 From: kehoe Date: Tue, 11 Jan 2022 20:31:48 +0000 Subject: [PATCH 1/5] Moving this file because pytest is not recognizing it in /test directory to register the mark. Moving to base directory so pytest finds and registers the mark and stops the warnings. --- act/tests/pytest.ini => pytest.ini | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename act/tests/pytest.ini => pytest.ini (100%) diff --git a/act/tests/pytest.ini b/pytest.ini similarity index 100% rename from act/tests/pytest.ini rename to pytest.ini From 705fe3b11992495712aa85811fd6138a165e3d5a Mon Sep 17 00:00:00 2001 From: kehoe Date: Tue, 11 Jan 2022 21:30:49 +0000 Subject: [PATCH 2/5] Adding logic to remove tests defined at global attribute level when moved to variable attributes but the test was not performed. --- act/qc/clean.py | 51 ++++++++++++++++++++++++++++++- act/qc/qcfilter.py | 71 ++++++++++++++++++++++++++++++++------------ act/tests/test_qc.py | 23 ++++++++++++++ 3 files changed, 125 insertions(+), 20 deletions(-) diff --git a/act/qc/clean.py b/act/qc/clean.py index 45322b9317..ef81b1879d 100644 --- a/act/qc/clean.py +++ b/act/qc/clean.py @@ -9,6 +9,8 @@ import numpy as np import copy +from act.qc.qcfilter import parse_bit + @xr.register_dataset_accessor('clean') class CleanDataset(object): @@ -548,7 +550,8 @@ def link_variables(self): def clean_arm_qc(self, override_cf_flag=True, clean_units_string=True, - correct_valid_min_max=True): + correct_valid_min_max=True, + remove_unset_global_tests=True): """ Function to clean up xarray object QC variables. @@ -566,6 +569,9 @@ def clean_arm_qc(self, fail_max and fail_detla if the valid_min, valid_max or valid_delta is listed in bit discription attribute. If not listed as used with QC will assume is being used correctly. + remove_unset_global_tests : bool + Option to look for globaly defined tests that are not set at the + variable level and remove from quality control variable. """ global_qc = self.get_attr_info() @@ -623,6 +629,49 @@ def clean_arm_qc(self, except KeyError: pass + # If requested remove tests at variable level that were set from global level descriptions. + # This is assuming the test was only performed if the limit value is listed with the variable + # even if the global level describes the test. + if remove_unset_global_tests and global_qc is not None: + limit_name_list = ['fail_min', 'fail_max', 'fail_delta'] + + for qc_var_name in self.matched_qc_variables: + flag_meanings = self._obj[qc_var_name].attrs['flag_meanings'] + flag_masks = self._obj[qc_var_name].attrs['flag_masks'] + tests_to_remove = [] + for ii, flag_meaning in enumerate(flag_meanings): + + # Loop over usual test attribute names looking to see if they + # are listed in test description. If so use that name for look up. + test_attribute_limit_name = None + for name in limit_name_list: + if name in flag_meaning: + test_attribute_limit_name = name + break + + if test_attribute_limit_name is None: + continue + + remove_test = True + test_number = int(parse_bit(flag_masks[ii])) + for attr_name in self._obj[qc_var_name].attrs: + if test_attribute_limit_name == attr_name: + remove_test = False + break + + index = self._obj.qcfilter.get_qc_test_mask( + qc_var_name=qc_var_name, test_number=test_number) + if np.any(index): + remove_test = False + break + + if remove_test: + tests_to_remove.append(test_number) + + if len(tests_to_remove) > 0: + for test_to_remove in tests_to_remove: + self._obj.qcfilter.remove_test(qc_var_name=qc_var_name, test_number=test_to_remove) + def normalize_assessment(self, variables=None, exclude_variables=None, qc_lookup={"Incorrect": "Bad", "Suspect": "Indeterminate"}): diff --git a/act/qc/qcfilter.py b/act/qc/qcfilter.py index 84072076ed..6b3a80fc6f 100644 --- a/act/qc/qcfilter.py +++ b/act/qc/qcfilter.py @@ -336,15 +336,18 @@ def add_test(self, var_name, index=None, test_number=None, return test_dict - def remove_test(self, var_name, test_number=None, flag_value=False, + def remove_test(self, var_name=None, qc_var_name=None, test_number=None, flag_value=False, flag_values_reset_value=0): """ - Method to remove a test/filter from a quality control variable. + Method to remove a test/filter from a quality control variable. Must set + var_name or qc_var_name. Parameters ---------- - var_name : str + var_name : str or None Data variable name. + qc_var_name : str or None + Quality control variable name. Ignored if var_name is set. test_number : int Test number to remove. flag_value : boolean @@ -360,9 +363,14 @@ def remove_test(self, var_name, test_number=None, flag_value=False, """ if test_number is None: raise ValueError('You need to provide a value for test_number ' - 'keyword when calling the add_test method') + 'keyword when calling the add_test() method') - qc_var_name = self._obj.qcfilter.check_for_ancillary_qc(var_name) + if var_name is None and qc_var_name is None: + raise ValueError('You need to provide a value for var_name or qc_var_name ' + 'keyword when calling the add_test() method') + + if var_name is not None: + qc_var_name = self._obj.qcfilter.check_for_ancillary_qc(var_name) # Determine which index is using the test number index = None @@ -385,16 +393,22 @@ def remove_test(self, var_name, test_number=None, flag_value=False, if flag_value: remove_index = self._obj.qcfilter.get_qc_test_mask( - var_name, test_number, return_index=True, flag_value=True) - self._obj.qcfilter.unset_test(var_name, remove_index, test_number, - flag_value, flag_values_reset_value) + var_name=var_name, qc_var_name=qc_var_name, test_number=test_number, + return_index=True, flag_value=True) + self._obj.qcfilter.unset_test(var_name=var_name, qc_var_name=qc_var_name, + index=remove_index, test_number=test_number, + flag_value=flag_value, + flag_values_reset_value=flag_values_reset_value) del flag_values[index] self._obj[qc_var_name].attrs['flag_values'] = flag_values + else: remove_index = self._obj.qcfilter.get_qc_test_mask( - var_name, test_number, return_index=True) - self._obj.qcfilter.unset_test(var_name, remove_index, test_number, - flag_value, flag_values_reset_value) + var_name=var_name, qc_var_name=qc_var_name, test_number=test_number, + return_index=True) + self._obj.qcfilter.unset_test(var_name=var_name, qc_var_name=qc_var_name, + index=remove_index, test_number=test_number, + flag_value=flag_value) del flag_masks[index] self._obj[qc_var_name].attrs['flag_masks'] = flag_masks @@ -458,15 +472,17 @@ def set_test(self, var_name, index=None, test_number=None, self._obj[qc_var_name].values = qc_variable - def unset_test(self, var_name, index=None, test_number=None, + def unset_test(self, var_name=None, qc_var_name=None, index=None, test_number=None, flag_value=False, flag_values_reset_value=0): """ Method to unset a test/filter from a quality control variable. Parameters ---------- - var_name : str + var_name : str or None Data variable name. + qc_var_name : str or None + Quality control variable name. Ignored if var_name is set. index : int or list or numpy array Index to unset test in quality control array. If want to unset all values will need to pass in index of all values. @@ -488,7 +504,12 @@ def unset_test(self, var_name, index=None, test_number=None, if index is None: return - qc_var_name = self._obj.qcfilter.check_for_ancillary_qc(var_name) + if var_name is None and qc_var_name is None: + raise ValueError('You need to provide a value for var_name or qc_var_name ' + 'keyword when calling the unset_test() method') + + if var_name is not None: + qc_var_name = self._obj.qcfilter.check_for_ancillary_qc(var_name) qc_variable = self._obj[qc_var_name].values if flag_value: @@ -553,18 +574,21 @@ def available_bit(self, qc_var_name, recycle=False): return int(next_bit) - def get_qc_test_mask(self, var_name, test_number, flag_value=False, - return_index=False): + def get_qc_test_mask(self, var_name=None, test_number=None, qc_var_name=None, + flag_value=False, return_index=False): """ Returns a numpy array of False or True where a particular - flag or bit is set in a numpy array. + flag or bit is set in a numpy array. Must set var_name or qc_var_name + when calling. Parameters ---------- - var_name : str + var_name : str or None Data variable name. test_number : int Test number to return array where test is set. + qc_var_name : str or None + Quality control variable name. Ignored if var_name is set. flag_value : boolean Switch to use flag_values integer quality control. return_index : boolean @@ -609,7 +633,16 @@ def get_qc_test_mask(self, var_name, test_number, flag_value=False, dtype=float32) """ - qc_var_name = self._obj.qcfilter.check_for_ancillary_qc(var_name) + if var_name is None and qc_var_name is None: + raise ValueError('You need to provide a value for var_name or qc_var_name ' + 'keyword when calling the get_qc_test_mask() method') + + if test_number is None: + raise ValueError('You need to provide a value for test_number ' + 'keyword when calling the get_qc_test_mask() method') + + if var_name is not None: + qc_var_name = self._obj.qcfilter.check_for_ancillary_qc(var_name) qc_variable = self._obj[qc_var_name].values diff --git a/act/tests/test_qc.py b/act/tests/test_qc.py index 455f9c4ba4..221ebfc6c0 100644 --- a/act/tests/test_qc.py +++ b/act/tests/test_qc.py @@ -18,6 +18,29 @@ def test_fft_shading_test(): assert np.nansum(qc_data.values) == 456 +def test_global_qc_cleanup(): + ds_object = read_netcdf(EXAMPLE_MET1) + ds_object.load() + ds_object.clean.cleanup() + + assert ds_object['qc_wdir_vec_mean'].attrs['flag_meanings'] == [ + 'Value is equal to missing_value.', 'Value is less than the fail_min.', + 'Value is greater than the fail_max.'] + assert ds_object['qc_wdir_vec_mean'].attrs['flag_masks'] == [1, 2, 4] + assert ds_object['qc_wdir_vec_mean'].attrs['flag_assessments'] == ['Bad', 'Bad', 'Bad'] + + assert ds_object['qc_temp_mean'].attrs['flag_meanings'] == [ + 'Value is equal to missing_value.', 'Value is less than the fail_min.', + 'Value is greater than the fail_max.', + 'Difference between current and previous values exceeds fail_delta.'] + assert ds_object['qc_temp_mean'].attrs['flag_masks'] == [1, 2, 4, 8] + assert ds_object['qc_temp_mean'].attrs['flag_assessments'] == ['Bad', 'Bad', + 'Bad', 'Indeterminate'] + + ds_object.close() + del ds_object + + def test_qc_test_errors(): ds_object = read_netcdf(EXAMPLE_MET1) var_name = 'temp_mean' From f1072089fc739ce17c0ee5e0f94736b7542cf55c Mon Sep 17 00:00:00 2001 From: kehoe Date: Tue, 11 Jan 2022 21:39:59 +0000 Subject: [PATCH 3/5] Fixing numpy deprecation --- act/tests/test_qc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/act/tests/test_qc.py b/act/tests/test_qc.py index 221ebfc6c0..c8d962ac85 100644 --- a/act/tests/test_qc.py +++ b/act/tests/test_qc.py @@ -155,7 +155,7 @@ def test_qcfilter(): # tests are set. assert np.sum(ds_object.qcfilter.get_qc_test_mask( var_name, result['test_number'], return_index=True) - - np.array(index, dtype=np.int)) == 0 + np.array(index, dtype=int)) == 0 # Unset a test ds_object.qcfilter.unset_test(var_name, index=0, From 1cd1816d239b3f6a583782f3c57a19e6f580560c Mon Sep 17 00:00:00 2001 From: Kenneth Kehoe Date: Tue, 11 Jan 2022 16:39:37 -0700 Subject: [PATCH 4/5] Fixing way test is done for different machine rounding errors? --- act/tests/test_correct.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/act/tests/test_correct.py b/act/tests/test_correct.py index 979203d99d..3e776fa327 100644 --- a/act/tests/test_correct.py +++ b/act/tests/test_correct.py @@ -35,10 +35,10 @@ def test_correct_mpl(): 181.9355]) np.testing.assert_allclose( sig_cross_pol, [-0.5823283, -1.6066532, -1.7153032, - -2.520143, -2.275405], rtol=4e-07) + -2.520143, -2.275405], rtol=4e-06) np.testing.assert_allclose( sig_co_pol, [12.5631485, 11.035495, 11.999875, - 11.09393, 11.388968]) + 11.09393, 11.388968], rtol=1e-6) np.testing.assert_allclose( height, [0.00749012, 0.02247084, 0.03745109, 0.05243181, 0.06741206, 0.08239277, 0.09737302, @@ -79,17 +79,12 @@ def test_correct_dl(): obj = act.io.armfiles.read_netcdf(files) new_obj = act.corrections.doppler_lidar.correct_dl(obj, fill_value=np.nan) - data = new_obj['attenuated_backscatter'].data - data[np.isnan(data)] = 0. - data = data * 100. - data = data.astype(np.int64) - assert np.sum(data) == -18633551 + data = new_obj['attenuated_backscatter'].values + np.testing.assert_almost_equal(np.nansum(data), -186479.83, decimal=0.1) new_obj = act.corrections.doppler_lidar.correct_dl(obj, range_normalize=False) - data = new_obj['attenuated_backscatter'].data - data[np.isnan(data)] = 0. - data = data.astype(np.int64) - assert np.sum(data) == -224000 + data = new_obj['attenuated_backscatter'].values + np.testing.assert_almost_equal(np.nansum(data), -200886.0, decimal=0.1) def test_correct_rl(): From df092b3a633bdd37d25a4d26ea940bff3254e7f4 Mon Sep 17 00:00:00 2001 From: kehoe Date: Wed, 12 Jan 2022 00:10:34 +0000 Subject: [PATCH 5/5] Resolving issue with test having slightly different results between ARM server and GitHub server. --- act/tests/test_qc.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/act/tests/test_qc.py b/act/tests/test_qc.py index c8d962ac85..0a69d89921 100644 --- a/act/tests/test_qc.py +++ b/act/tests/test_qc.py @@ -578,7 +578,9 @@ def test_qctests_dos(): test_meaning = ('Data failing persistence test. Standard Deviation over a ' 'window of 10 values less than 0.0001.') assert ds_object[qc_var_name].attrs['flag_meanings'][-1] == test_meaning - assert np.sum(ds_object[qc_var_name].values) == 1500 + # There is a precision issue with GitHub testing that makes the number of tests + # tripped off by 1. This isclose() option is to account for that. + assert np.isclose(np.sum(ds_object[qc_var_name].values), 1500, atol=1) ds_object.qcfilter.add_persistence_test(var_name, window=10000, prepend_text='DQO') test_meaning = ('DQO: Data failing persistence test. Standard Deviation over a window of '