From 674cff949c32443e6e777c2a2bcb45792c8789dd Mon Sep 17 00:00:00 2001 From: Bart Schilperoort Date: Fri, 11 Aug 2023 20:42:41 +0200 Subject: [PATCH 1/5] Remove sections setter & deleter. --- .../calibration/section_utils.py | 223 ++++++++++++++++++ src/dtscalibration/calibration/utils.py | 0 src/dtscalibration/datastore.py | 100 ++------ tests/test_datastore.py | 20 +- tests/test_dtscalibration.py | 41 ++-- 5 files changed, 269 insertions(+), 115 deletions(-) create mode 100644 src/dtscalibration/calibration/section_utils.py create mode 100644 src/dtscalibration/calibration/utils.py diff --git a/src/dtscalibration/calibration/section_utils.py b/src/dtscalibration/calibration/section_utils.py new file mode 100644 index 00000000..0b7b3213 --- /dev/null +++ b/src/dtscalibration/calibration/section_utils.py @@ -0,0 +1,223 @@ +import xarray as xr +from typing import Dict, List +import yaml +import numpy as np +from dtscalibration.datastore_utils import ufunc_per_section_helper + + +def set_sections(ds: xr.Dataset, sections: Dict[str, List[slice]]) -> xr.Dataset: + sections_validated = None + + if sections is not None: + sections_validated = validate_sections(ds, sections=sections) + + ds.attrs["_sections"] = yaml.dump(sections_validated) + return ds + + +def validate_sections(ds: xr.Dataset, sections: Dict[str, List[slice]]): + assert isinstance(sections, dict) + + # be less restrictive for capitalized labels + # find lower cases label + labels = np.reshape( + [[s.lower(), s] for s in ds.data_vars.keys()], (-1,) + ).tolist() + + sections_fix = dict() + for k, v in sections.items(): + if k.lower() in labels: + i_lower_case = labels.index(k.lower()) + i_normal_case = i_lower_case + 1 + k_normal_case = labels[i_normal_case] + sections_fix[k_normal_case] = v + else: + assert k in ds.data_vars, ( + "The keys of the " + "sections-dictionary should " + "refer to a valid timeserie " + "already stored in " + "ds.data_vars " + ) + + sections_fix_slice_fixed = dict() + + for k, v in sections_fix.items(): + assert isinstance(v, (list, tuple)), ( + "The values of the sections-dictionary " + "should be lists of slice objects." + ) + + for vi in v: + assert isinstance(vi, slice), ( + "The values of the sections-dictionary should " + "be lists of slice objects." + ) + + assert ds.x.sel(x=vi).size > 0, ( + f"Better define the {k} section. You tried {vi}, " + "which is not within the x-dimension" + ) + + # sorted stretches + stretch_unsort = [slice(float(vi.start), float(vi.stop)) for vi in v] + stretch_start = [i.start for i in stretch_unsort] + stretch_i_sorted = np.argsort(stretch_start) + sections_fix_slice_fixed[k] = [stretch_unsort[i] for i in stretch_i_sorted] + + # Prevent overlapping slices + ix_sec = ufunc_per_section( + ds, sections=sections_fix_slice_fixed, x_indices=True, calc_per="all" + ) + assert np.unique(ix_sec).size == ix_sec.size, "The sections are overlapping" + + return sections_fix_slice_fixed + + +def ufunc_per_section( + ds: xr.Dataset, + sections, + func=None, + label=None, + subtract_from_label=None, + temp_err=False, + x_indices=False, + ref_temp_broadcasted=False, + calc_per="stretch", + **func_kwargs, +): + """ + User function applied to parts of the cable. Super useful, + many options and slightly + complicated. + + The function `func` is taken over all the timesteps and calculated + per `calc_per`. This + is returned as a dictionary + + Parameters + ---------- + sections : Dict[str, List[slice]], optional + If `None` is supplied, `ds.sections` is used. Define calibration + sections. Each section requires a reference temperature time series, + such as the temperature measured by an external temperature sensor. + They should already be part of the DataStore object. `sections` + is defined with a dictionary with its keywords of the + names of the reference temperature time series. Its values are + lists of slice objects, where each slice object is a fiber stretch + that has the reference temperature. Afterwards, `sections` is stored + under `ds.sections`. + func : callable, str + A numpy function, or lambda function to apple to each 'calc_per'. + label + subtract_from_label + temp_err : bool + The argument of the function is label minus the reference + temperature. + x_indices : bool + To retreive an integer array with the indices of the + x-coordinates in the section/stretch. The indices are sorted. + ref_temp_broadcasted : bool + calc_per : {'all', 'section', 'stretch'} + func_kwargs : dict + Dictionary with options that are passed to func + + TODO: Spend time on creating a slice instead of appendng everything\ + to a list and concatenating after. + + + Returns + ------- + + Examples + -------- + + 1. Calculate the variance of the residuals in the along ALL the\ + reference sections wrt the temperature of the water baths + + >>> tmpf_var = d.ufunc_per_section( + >>> func='var', + >>> calc_per='all', + >>> label='tmpf', + >>> temp_err=True) + + 2. Calculate the variance of the residuals in the along PER\ + reference section wrt the temperature of the water baths + + >>> tmpf_var = d.ufunc_per_section( + >>> func='var', + >>> calc_per='stretch', + >>> label='tmpf', + >>> temp_err=True) + + 3. Calculate the variance of the residuals in the along PER\ + water bath wrt the temperature of the water baths + + >>> tmpf_var = d.ufunc_per_section( + >>> func='var', + >>> calc_per='section', + >>> label='tmpf', + >>> temp_err=True) + + 4. Obtain the coordinates of the measurements per section + + >>> locs = d.ufunc_per_section( + >>> func=None, + >>> label='x', + >>> temp_err=False, + >>> ref_temp_broadcasted=False, + >>> calc_per='stretch') + + 5. Number of observations per stretch + + >>> nlocs = d.ufunc_per_section( + >>> func=len, + >>> label='x', + >>> temp_err=False, + >>> ref_temp_broadcasted=False, + >>> calc_per='stretch') + + 6. broadcast the temperature of the reference sections to\ + stretch/section/all dimensions. The value of the reference\ + temperature (a timeseries) is broadcasted to the shape of self[\ + label]. The self[label] is not used for anything else. + + >>> temp_ref = d.ufunc_per_section( + >>> label='st', + >>> ref_temp_broadcasted=True, + >>> calc_per='all') + + 7. x-coordinate index + + >>> ix_loc = d.ufunc_per_section(x_indices=True) + + + Note + ---- + If `self[label]` or `self[subtract_from_label]` is a Dask array, a Dask + array is returned else a numpy array is returned + """ + if label is None: + dataarray = None + else: + dataarray = ds[label] + + if x_indices: + x_coords = ds.x + reference_dataset = None + else: + x_coords = None + reference_dataset = {k: ds[k] for k in sections} + + return ufunc_per_section_helper( + x_coords=x_coords, + sections=sections, + func=func, + dataarray=dataarray, + subtract_from_dataarray=subtract_from_label, + reference_dataset=reference_dataset, + subtract_reference_from_dataarray=temp_err, + ref_temp_broadcasted=ref_temp_broadcasted, + calc_per=calc_per, + **func_kwargs, + ) diff --git a/src/dtscalibration/calibration/utils.py b/src/dtscalibration/calibration/utils.py new file mode 100644 index 00000000..e69de29b diff --git a/src/dtscalibration/datastore.py b/src/dtscalibration/datastore.py index 814c21b0..e51dcb56 100644 --- a/src/dtscalibration/datastore.py +++ b/src/dtscalibration/datastore.py @@ -23,6 +23,7 @@ from dtscalibration.datastore_utils import check_timestep_allclose from dtscalibration.datastore_utils import ufunc_per_section_helper from dtscalibration.io_utils import _dim_attrs +from dtscalibration.calibration.section_utils import set_sections, validate_sections dtsattr_namelist = ["double_ended_flag"] dim_attrs = {k: v for kl, v in _dim_attrs.items() for k in kl} @@ -121,7 +122,7 @@ def __init__(self, *args, autofill_dim_attrs=True, **kwargs): self.attrs["_sections"] = yaml.dump(None) if "sections" in kwargs: - self.sections = kwargs["sections"] + self = set_sections(self, kwargs["sections"]) pass @@ -209,78 +210,9 @@ def sections(self): return yaml.load(self.attrs["_sections"], Loader=yaml.UnsafeLoader) - @sections.setter - def sections(self, sections: Dict[str, List[slice]]): - sections_validated = None - - if sections is not None: - sections_validated = self.validate_sections(sections=sections) - - self.attrs["_sections"] = yaml.dump(sections_validated) - pass - @sections.deleter def sections(self): - self.sections = None - pass - - def validate_sections(self, sections: Dict[str, List[slice]]): - assert isinstance(sections, dict) - - # be less restrictive for capitalized labels - # find lower cases label - labels = np.reshape( - [[s.lower(), s] for s in self.data_vars.keys()], (-1,) - ).tolist() - - sections_fix = dict() - for k, v in sections.items(): - if k.lower() in labels: - i_lower_case = labels.index(k.lower()) - i_normal_case = i_lower_case + 1 - k_normal_case = labels[i_normal_case] - sections_fix[k_normal_case] = v - else: - assert k in self.data_vars, ( - "The keys of the " - "sections-dictionary should " - "refer to a valid timeserie " - "already stored in " - "ds.data_vars " - ) - - sections_fix_slice_fixed = dict() - - for k, v in sections_fix.items(): - assert isinstance(v, (list, tuple)), ( - "The values of the sections-dictionary " - "should be lists of slice objects." - ) - - for vi in v: - assert isinstance(vi, slice), ( - "The values of the sections-dictionary should " - "be lists of slice objects." - ) - - assert self.x.sel(x=vi).size > 0, ( - f"Better define the {k} section. You tried {vi}, " - "which is not within the x-dimension" - ) - - # sorted stretches - stretch_unsort = [slice(float(vi.start), float(vi.stop)) for vi in v] - stretch_start = [i.start for i in stretch_unsort] - stretch_i_sorted = np.argsort(stretch_start) - sections_fix_slice_fixed[k] = [stretch_unsort[i] for i in stretch_i_sorted] - - # Prevent overlapping slices - ix_sec = self.ufunc_per_section( - sections=sections_fix_slice_fixed, x_indices=True, calc_per="all" - ) - assert np.unique(ix_sec).size == ix_sec.size, "The sections are overlapping" - - return sections_fix_slice_fixed + self.attrs["_sections"] = yaml.dump(None) def check_reference_section_values(self): """ @@ -952,7 +884,7 @@ def func_cost(p, data, xs): if sections is None: sections = self.sections else: - sections = self.validate_sections(sections) + sections = validate_sections(self, sections) assert self[st_label].dims[0] == "x", "Stokes are transposed" @@ -1132,7 +1064,7 @@ def variance_stokes_exponential( if sections is None: sections = self.sections else: - sections = self.validate_sections(sections) + sections = validate_sections(self, sections) assert self[st_label].dims[0] == "x", "Stokes are transposed" @@ -1390,7 +1322,7 @@ def variance_stokes_linear( if sections is None: sections = self.sections else: - sections = self.validate_sections(sections) + sections = validate_sections(self, sections) assert self[st_label].dims[0] == "x", "Stokes are transposed" _, resid = self.variance_stokes(sections=sections, st_label=st_label) @@ -1616,7 +1548,7 @@ def set_trans_att(self, trans_att=None): If multiple locations are defined, the losses are added. """ - if "trans_att" in self.coords and self.trans_att.size > 0: + if "trans_att" in self.coords and self["trans_att"].size > 0: raise_warning = 0 del_keys = [] @@ -1638,12 +1570,12 @@ def set_trans_att(self, trans_att=None): trans_att = [] self["trans_att"] = trans_att - self.trans_att.attrs = dim_attrs["trans_att"] + self["trans_att"].attrs = dim_attrs["trans_att"] pass def calibration_single_ended( self, - sections=None, + sections, st_var=None, ast_var=None, method="wls", @@ -1790,8 +1722,7 @@ def calibration_single_ended( check_deprecated_kwargs(kwargs) self.set_trans_att(trans_att=trans_att, **kwargs) - if sections is not None: - self.sections = sections + self = set_sections(self, sections) # TODO: don't change object in-place. if method == "wls": assert st_var is not None and ast_var is not None, "Set `st_var`" @@ -1802,7 +1733,7 @@ def calibration_single_ended( nt = self["time"].size nta = self.trans_att.size - assert self.st.dims[0] == "x", "Stokes are transposed" + assert self["st"].dims[0] == "x", "Stokes are transposed" assert self.ast.dims[0] == "x", "Stokes are transposed" if matching_sections: @@ -1996,7 +1927,7 @@ def calibration_single_ended( def calibration_double_ended( self, - sections=None, + sections, st_var=None, ast_var=None, rst_var=None, @@ -2221,8 +2152,7 @@ def calibration_double_ended( self.set_trans_att(trans_att=trans_att, **kwargs) - if sections is not None: - self.sections = sections + self = set_sections(self, sections) # TODO: don't change object in-place. self.check_reference_section_values() @@ -4282,7 +4212,7 @@ def in_confidence_interval(self, ci_label, conf_ints=None, sections=None): if sections is None: sections = self.sections else: - sections = self.validate_sections(sections) + sections = validate_sections(self, sections) if conf_ints is None: conf_ints = self[ci_label].values @@ -4335,7 +4265,7 @@ def temperature_residuals(self, label=None, sections=None): if sections is None: sections = self.sections else: - sections = self.validate_sections(sections) + sections = validate_sections(self, sections) resid_temp = self.ufunc_per_section( sections=sections, label=label, temp_err=True, calc_per="all" diff --git a/tests/test_datastore.py b/tests/test_datastore.py index d23d280f..5fd81967 100644 --- a/tests/test_datastore.py +++ b/tests/test_datastore.py @@ -20,6 +20,7 @@ from dtscalibration.datastore_utils import merge_double_ended from dtscalibration.datastore_utils import shift_double_ended from dtscalibration.datastore_utils import suggest_cable_shift_double_ended +from dtscalibration.calibration.section_utils import set_sections np.random.seed(0) @@ -156,21 +157,20 @@ def test_sections_property(): "probe1Temperature": [slice(0.0, 17.0), slice(70.0, 80.0)], # cold bath "probe2Temperature": [slice(24.0, 34.0), slice(85.0, 95.0)], # warm bath } - ds.sections = sections1 + ds = set_sections(ds, sections1) - assert isinstance(ds._sections, str) + assert isinstance(ds.attrs["_sections"], str) assert ds.sections == sections1 assert ds.sections != sections2 # test if accepts singleton numpy arrays - ds.sections = { - "probe1Temperature": [slice(np.array(0.0), np.array(17.0)), slice(70.0, 80.0)] - } - - # delete property - del ds.sections - assert ds.sections is None + ds = set_sections( + ds, + { + "probe1Temperature": [slice(np.array(0.0), np.array(17.0)), slice(70.0, 80.0)] + } + ) def test_io_sections_property(): @@ -190,7 +190,7 @@ def test_io_sections_property(): } ds["x"].attrs["units"] = "m" - ds.sections = sections + ds = set_sections(ds, sections) # Create a temporary file to write data to. # 'with' method is used so the file is closed by tempfile diff --git a/tests/test_dtscalibration.py b/tests/test_dtscalibration.py index 2b86aa2d..ff9d9114 100644 --- a/tests/test_dtscalibration.py +++ b/tests/test_dtscalibration.py @@ -10,6 +10,7 @@ from dtscalibration import read_silixa_files from dtscalibration.calibrate_utils import wls_sparse from dtscalibration.calibrate_utils import wls_stats +from dtscalibration.calibration.section_utils import set_sections np.random.seed(0) @@ -1089,10 +1090,10 @@ def test_double_ended_wls_estimate_synthetic_df_and_db_are_different(): attrs={"isDoubleEnded": "1"}, ) - ds.sections = { + ds = set_sections(ds, { "cold": [slice(0.0, 0.09 * cable_len)], "warm": [slice(0.9 * cable_len, cable_len)], - } + }) real_ans2 = np.concatenate(([gamma], df, db, E_real[:, 0])) @@ -1207,10 +1208,10 @@ def test_reneaming_old_default_labels_to_new_fixed_labels(): ) ds = ds.rename_labels() - ds.sections = { + ds = set_sections(ds, { "cold": [slice(0.0, 0.09 * cable_len)], "warm": [slice(0.9 * cable_len, cable_len)], - } + }) real_ans2 = np.concatenate(([gamma], df, db, E_real[:, 0])) @@ -1312,10 +1313,10 @@ def test_fail_if_st_labels_are_passed_to_calibration_function(): ) ds = ds.rename_labels() - ds.sections = { + ds = set_sections(ds, { "cold": [slice(0.0, 0.09 * cable_len)], "warm": [slice(0.9 * cable_len, cable_len)], - } + }) ds.calibration_double_ended( st_label="ST", @@ -1418,13 +1419,13 @@ def test_double_ended_asymmetrical_attenuation(): attrs={"isDoubleEnded": "1"}, ) - ds.sections = { + ds = set_sections(ds, { "cold": [slice(0.0, x[nx_per_sec - 1]), slice(x[-nx_per_sec], x[-1])], "warm": [ slice(x[nx_per_sec], x[2 * nx_per_sec - 1]), slice(x[-2 * nx_per_sec], x[-1 * nx_per_sec - 1]), ], - } + }) ds.calibration_double_ended( st_var=1.5, @@ -1554,10 +1555,10 @@ def test_double_ended_one_matching_section_and_one_asym_att(): attrs={"isDoubleEnded": "1"}, ) - ds.sections = { + ds = set_sections(ds, { "cold": [slice(0.0, x[nx_per_sec - 1])], "warm": [slice(x[nx_per_sec], x[2 * nx_per_sec - 1])], - } + }) ds.calibration_double_ended( st_var=1.5, @@ -1672,10 +1673,10 @@ def test_double_ended_two_matching_sections_and_two_asym_atts(): attrs={"isDoubleEnded": "1"}, ) - ds.sections = { + ds = set_sections(ds, { "cold": [slice(0.0, x[nx_per_sec - 1])], "warm": [slice(x[nx_per_sec], x[2 * nx_per_sec - 1])], - } + }) ms = [ ( slice(x[2 * nx_per_sec], x[3 * nx_per_sec - 1]), @@ -2077,10 +2078,10 @@ def test_double_ended_fix_alpha_matching_sections_and_one_asym_att(): attrs={"isDoubleEnded": "1"}, ) - ds.sections = { + ds = set_sections(ds, { "cold": [slice(0.0, x[nx_per_sec - 1])], "warm": [slice(x[nx_per_sec], x[2 * nx_per_sec - 1])], - } + }) ds.calibration_double_ended( st_var=1.5, @@ -2215,10 +2216,10 @@ def test_double_ended_fix_alpha_gamma_matching_sections_and_one_asym_att(): attrs={"isDoubleEnded": "1"}, ) - ds.sections = { + ds = set_sections(ds, { "cold": [slice(0.0, x[nx_per_sec - 1])], "warm": [slice(x[nx_per_sec], x[2 * nx_per_sec - 1])], - } + }) ds.calibration_double_ended( st_var=1.5, @@ -2354,10 +2355,10 @@ def test_double_ended_fix_gamma_matching_sections_and_one_asym_att(): attrs={"isDoubleEnded": "1"}, ) - ds.sections = { + ds = set_sections(ds, { "cold": [slice(0.0, x[nx_per_sec - 1])], "warm": [slice(x[nx_per_sec], x[2 * nx_per_sec - 1])], - } + }) ds.calibration_double_ended( st_var=1.5, @@ -3578,7 +3579,7 @@ def test_average_measurements_single_ended(): ds = ds_.sel(x=slice(0, 100)) # only calibrate parts of the fiber sections = {"probe2Temperature": [slice(6.0, 14.0)]} # warm bath - ds.sections = sections + ds = set_sections(ds, sections) st_var, ast_var = 5.0, 5.0 @@ -3645,7 +3646,7 @@ def test_average_measurements_double_ended(): "probe1Temperature": [slice(7.5, 17.0), slice(70.0, 80.0)], # cold bath "probe2Temperature": [slice(24.0, 34.0), slice(85.0, 95.0)], # warm bath } - ds.sections = sections + ds = set_sections(ds, sections) st_var, ast_var, rst_var, rast_var = 5.0, 5.0, 5.0, 5.0 From e0d5807852c83500c31e67a8d9462d797d652800 Mon Sep 17 00:00:00 2001 From: Bart Schilperoort Date: Sun, 13 Aug 2023 09:37:56 +0200 Subject: [PATCH 2/5] Fix tests: sections is required arg for calibration_ --- tests/test_dtscalibration.py | 50 ++++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/tests/test_dtscalibration.py b/tests/test_dtscalibration.py index ff9d9114..e2e1355b 100644 --- a/tests/test_dtscalibration.py +++ b/tests/test_dtscalibration.py @@ -1090,14 +1090,15 @@ def test_double_ended_wls_estimate_synthetic_df_and_db_are_different(): attrs={"isDoubleEnded": "1"}, ) - ds = set_sections(ds, { + sections = { "cold": [slice(0.0, 0.09 * cable_len)], "warm": [slice(0.9 * cable_len, cable_len)], - }) + } real_ans2 = np.concatenate(([gamma], df, db, E_real[:, 0])) ds.calibration_double_ended( + sections=sections, st_var=1.5, ast_var=1.5, rst_var=1.0, @@ -1208,14 +1209,15 @@ def test_reneaming_old_default_labels_to_new_fixed_labels(): ) ds = ds.rename_labels() - ds = set_sections(ds, { + sections = { "cold": [slice(0.0, 0.09 * cable_len)], "warm": [slice(0.9 * cable_len, cable_len)], - }) + } real_ans2 = np.concatenate(([gamma], df, db, E_real[:, 0])) ds.calibration_double_ended( + sections=sections, st_var=1.5, ast_var=1.5, rst_var=1.0, @@ -1313,12 +1315,13 @@ def test_fail_if_st_labels_are_passed_to_calibration_function(): ) ds = ds.rename_labels() - ds = set_sections(ds, { + sections = { "cold": [slice(0.0, 0.09 * cable_len)], "warm": [slice(0.9 * cable_len, cable_len)], - }) + } ds.calibration_double_ended( + sections=sections, st_label="ST", ast_label="AST", rst_label="REV-ST", @@ -1419,15 +1422,16 @@ def test_double_ended_asymmetrical_attenuation(): attrs={"isDoubleEnded": "1"}, ) - ds = set_sections(ds, { + sections = { "cold": [slice(0.0, x[nx_per_sec - 1]), slice(x[-nx_per_sec], x[-1])], "warm": [ slice(x[nx_per_sec], x[2 * nx_per_sec - 1]), slice(x[-2 * nx_per_sec], x[-1 * nx_per_sec - 1]), ], - }) + } ds.calibration_double_ended( + sections=sections, st_var=1.5, ast_var=1.5, rst_var=1.0, @@ -1457,6 +1461,7 @@ def test_double_ended_asymmetrical_attenuation(): # About to be depreciated ds.calibration_double_ended( + sections=sections, st_var=1.5, ast_var=1.5, rst_var=1.0, @@ -1555,12 +1560,13 @@ def test_double_ended_one_matching_section_and_one_asym_att(): attrs={"isDoubleEnded": "1"}, ) - ds = set_sections(ds, { + sections = { "cold": [slice(0.0, x[nx_per_sec - 1])], "warm": [slice(x[nx_per_sec], x[2 * nx_per_sec - 1])], - }) + } ds.calibration_double_ended( + sections=sections, st_var=1.5, ast_var=1.5, rst_var=1.0, @@ -1673,10 +1679,10 @@ def test_double_ended_two_matching_sections_and_two_asym_atts(): attrs={"isDoubleEnded": "1"}, ) - ds = set_sections(ds, { + sections = { "cold": [slice(0.0, x[nx_per_sec - 1])], "warm": [slice(x[nx_per_sec], x[2 * nx_per_sec - 1])], - }) + } ms = [ ( slice(x[2 * nx_per_sec], x[3 * nx_per_sec - 1]), @@ -1691,6 +1697,7 @@ def test_double_ended_two_matching_sections_and_two_asym_atts(): ] ds.calibration_double_ended( + sections=sections, st_var=0.5, ast_var=0.5, rst_var=0.1, @@ -2078,12 +2085,13 @@ def test_double_ended_fix_alpha_matching_sections_and_one_asym_att(): attrs={"isDoubleEnded": "1"}, ) - ds = set_sections(ds, { + sections = { "cold": [slice(0.0, x[nx_per_sec - 1])], "warm": [slice(x[nx_per_sec], x[2 * nx_per_sec - 1])], - }) + } ds.calibration_double_ended( + sections=sections, st_var=1.5, ast_var=1.5, rst_var=1.0, @@ -2110,6 +2118,7 @@ def test_double_ended_fix_alpha_matching_sections_and_one_asym_att(): alpha_var_adj = ds.alpha_var.values.copy() ds.calibration_double_ended( + sections=sections, st_var=1.5, ast_var=1.5, rst_var=1.0, @@ -2216,12 +2225,13 @@ def test_double_ended_fix_alpha_gamma_matching_sections_and_one_asym_att(): attrs={"isDoubleEnded": "1"}, ) - ds = set_sections(ds, { + sections = { "cold": [slice(0.0, x[nx_per_sec - 1])], "warm": [slice(x[nx_per_sec], x[2 * nx_per_sec - 1])], - }) + } ds.calibration_double_ended( + sections=sections, st_var=1.5, ast_var=1.5, rst_var=1.0, @@ -2248,6 +2258,7 @@ def test_double_ended_fix_alpha_gamma_matching_sections_and_one_asym_att(): alpha_var_adj = ds.alpha_var.values.copy() ds.calibration_double_ended( + sections=sections, st_var=1.5, ast_var=1.5, rst_var=1.0, @@ -2355,12 +2366,13 @@ def test_double_ended_fix_gamma_matching_sections_and_one_asym_att(): attrs={"isDoubleEnded": "1"}, ) - ds = set_sections(ds, { + sections = { "cold": [slice(0.0, x[nx_per_sec - 1])], "warm": [slice(x[nx_per_sec], x[2 * nx_per_sec - 1])], - }) + } ds.calibration_double_ended( + sections=sections, st_var=1.5, ast_var=1.5, rst_var=1.0, @@ -3646,11 +3658,11 @@ def test_average_measurements_double_ended(): "probe1Temperature": [slice(7.5, 17.0), slice(70.0, 80.0)], # cold bath "probe2Temperature": [slice(24.0, 34.0), slice(85.0, 95.0)], # warm bath } - ds = set_sections(ds, sections) st_var, ast_var, rst_var, rast_var = 5.0, 5.0, 5.0, 5.0 ds.calibration_double_ended( + sections=sections, st_var=st_var, ast_var=ast_var, rst_var=rst_var, From 4a426e24ea438eedc8c7fa73fe84ffdd65a7ea26 Mon Sep 17 00:00:00 2001 From: Bart Schilperoort Date: Sun, 13 Aug 2023 09:53:39 +0200 Subject: [PATCH 3/5] Mark slow tests, make them skippable. --- pyproject.toml | 3 +++ tests/test_dtscalibration.py | 9 +++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7b05a985..1a9f77e2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -134,6 +134,9 @@ test = ["pytest ./src/ ./tests/",] # --doctest-modules [tool.pytest.ini_options] testpaths = ["tests"] +markers = [ + "slow: marks tests as slow (deselect with '-m \"not slow\"')", +] [tool.ruff] select = [ # It would be nice to have the commented out checks working. diff --git a/tests/test_dtscalibration.py b/tests/test_dtscalibration.py index e2e1355b..7c88af85 100644 --- a/tests/test_dtscalibration.py +++ b/tests/test_dtscalibration.py @@ -57,6 +57,7 @@ def assert_almost_equal_verbose(actual, desired, verbose=False, **kwargs): pass +@pytest.mark.slow # Execution time ~20 seconds def test_variance_input_types_single(): import dask.array as da @@ -222,6 +223,7 @@ def callable_st_var(stokes): pass +@pytest.mark.slow # Execution time ~0.5 minute def test_variance_input_types_double(): import dask.array as da @@ -454,6 +456,7 @@ def st_var_callable(stokes): pass +@pytest.mark.slow # Execution time ~0.5 minute def test_double_ended_variance_estimate_synthetic(): import dask.array as da @@ -780,6 +783,7 @@ def test_variance_of_stokes_synthetic(): pass +@pytest.mark.slow # Execution time ~20 seconds def test_variance_of_stokes_linear_synthetic(): """ Produces a synthetic Stokes measurement with a known noise distribution. @@ -851,6 +855,7 @@ def test_variance_of_stokes_linear_synthetic(): pass +@pytest.mark.slow # Execution time ~20 seconds def test_exponential_variance_of_stokes(): correct_var = 11.86535 filepath = data_dir_double_ended2 @@ -2556,6 +2561,7 @@ def test_double_ended_exponential_variance_estimate_synthetic(): pass +@pytest.mark.slow # Execution time ~2 minutes. def test_estimate_variance_of_temperature_estimate(): import dask.array as da @@ -3591,12 +3597,11 @@ def test_average_measurements_single_ended(): ds = ds_.sel(x=slice(0, 100)) # only calibrate parts of the fiber sections = {"probe2Temperature": [slice(6.0, 14.0)]} # warm bath - ds = set_sections(ds, sections) st_var, ast_var = 5.0, 5.0 ds.calibration_single_ended( - st_var=st_var, ast_var=ast_var, method="wls", solver="sparse" + sections=sections, st_var=st_var, ast_var=ast_var, method="wls", solver="sparse" ) ds.average_single_ended( p_val="p_val", From 95ac57a869930da23b37accab581baa87ddd1709 Mon Sep 17 00:00:00 2001 From: Bart Schilperoort Date: Sun, 13 Aug 2023 10:27:52 +0200 Subject: [PATCH 4/5] Fix linting issues --- .gitignore | 2 ++ pyproject.toml | 3 ++- src/dtscalibration/calibration/section_utils.py | 14 +++++++------- src/dtscalibration/datastore.py | 5 ++--- tests/test_datastore.py | 9 ++++++--- tests/test_dtscalibration.py | 1 - 6 files changed, 19 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index 29d21eac..b8f681da 100644 --- a/.gitignore +++ b/.gitignore @@ -64,6 +64,8 @@ docs/_build .build .ve .env +.venv +.vscode .cache .pytest .bootstrap diff --git a/pyproject.toml b/pyproject.toml index 1a9f77e2..58b7babf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -184,13 +184,14 @@ max-complexity = 10 py_version=39 force_single_line = true known_first_party = ["dtscalibration"] -skip = [".gitignore", ".tox", "docs"] +skip = [".gitignore", ".tox", "docs", ".venv"] src_paths = ["src", "tests"] line_length = 120 [tool.black] line-length = 88 target-version = ['py39', 'py310', 'py311'] +extend-exclude = ".venv" [tool.mypy] ignore_missing_imports = true # Preferably false, but matplotlib, scipy and statsmodels are missing typing stubs diff --git a/src/dtscalibration/calibration/section_utils.py b/src/dtscalibration/calibration/section_utils.py index 0b7b3213..126d575b 100644 --- a/src/dtscalibration/calibration/section_utils.py +++ b/src/dtscalibration/calibration/section_utils.py @@ -1,7 +1,10 @@ +from typing import Dict +from typing import List + +import numpy as np import xarray as xr -from typing import Dict, List import yaml -import numpy as np + from dtscalibration.datastore_utils import ufunc_per_section_helper @@ -20,9 +23,7 @@ def validate_sections(ds: xr.Dataset, sections: Dict[str, List[slice]]): # be less restrictive for capitalized labels # find lower cases label - labels = np.reshape( - [[s.lower(), s] for s in ds.data_vars.keys()], (-1,) - ).tolist() + labels = np.reshape([[s.lower(), s] for s in ds.data_vars.keys()], (-1,)).tolist() sections_fix = dict() for k, v in sections.items(): @@ -44,8 +45,7 @@ def validate_sections(ds: xr.Dataset, sections: Dict[str, List[slice]]): for k, v in sections_fix.items(): assert isinstance(v, (list, tuple)), ( - "The values of the sections-dictionary " - "should be lists of slice objects." + "The values of the sections-dictionary " "should be lists of slice objects." ) for vi in v: diff --git a/src/dtscalibration/datastore.py b/src/dtscalibration/datastore.py index e51dcb56..01faf0c6 100644 --- a/src/dtscalibration/datastore.py +++ b/src/dtscalibration/datastore.py @@ -1,7 +1,5 @@ import os import warnings -from typing import Dict -from typing import List import dask import dask.array as da @@ -17,13 +15,14 @@ from dtscalibration.calibrate_utils import calibration_single_ended_helper from dtscalibration.calibrate_utils import match_sections from dtscalibration.calibrate_utils import parse_st_var +from dtscalibration.calibration.section_utils import set_sections +from dtscalibration.calibration.section_utils import validate_sections from dtscalibration.datastore_utils import ParameterIndexDoubleEnded from dtscalibration.datastore_utils import ParameterIndexSingleEnded from dtscalibration.datastore_utils import check_deprecated_kwargs from dtscalibration.datastore_utils import check_timestep_allclose from dtscalibration.datastore_utils import ufunc_per_section_helper from dtscalibration.io_utils import _dim_attrs -from dtscalibration.calibration.section_utils import set_sections, validate_sections dtsattr_namelist = ["double_ended_flag"] dim_attrs = {k: v for kl, v in _dim_attrs.items() for k in kl} diff --git a/tests/test_datastore.py b/tests/test_datastore.py index 5fd81967..b6b40c77 100644 --- a/tests/test_datastore.py +++ b/tests/test_datastore.py @@ -17,10 +17,10 @@ from dtscalibration import read_sensornet_files from dtscalibration import read_sensortran_files from dtscalibration import read_silixa_files +from dtscalibration.calibration.section_utils import set_sections from dtscalibration.datastore_utils import merge_double_ended from dtscalibration.datastore_utils import shift_double_ended from dtscalibration.datastore_utils import suggest_cable_shift_double_ended -from dtscalibration.calibration.section_utils import set_sections np.random.seed(0) @@ -168,8 +168,11 @@ def test_sections_property(): ds = set_sections( ds, { - "probe1Temperature": [slice(np.array(0.0), np.array(17.0)), slice(70.0, 80.0)] - } + "probe1Temperature": [ + slice(np.array(0.0), np.array(17.0)), + slice(70.0, 80.0), + ] + }, ) diff --git a/tests/test_dtscalibration.py b/tests/test_dtscalibration.py index 7c88af85..805a900e 100644 --- a/tests/test_dtscalibration.py +++ b/tests/test_dtscalibration.py @@ -10,7 +10,6 @@ from dtscalibration import read_silixa_files from dtscalibration.calibrate_utils import wls_sparse from dtscalibration.calibrate_utils import wls_stats -from dtscalibration.calibration.section_utils import set_sections np.random.seed(0) From 31c45a1a69ef59a4305ed06455c6dd058a1a08d5 Mon Sep 17 00:00:00 2001 From: Bart Schilperoort Date: Sun, 13 Aug 2023 11:02:47 +0200 Subject: [PATCH 5/5] Add fast-test command to hatch --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 58b7babf..315e0bb7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -111,6 +111,7 @@ lint = [ ] format = ["black .", "isort .", "lint",] test = ["pytest ./src/ ./tests/",] # --doctest-modules +fast-test = ["pytest ./tests/ -m \"not slow\"",] coverage = [ "pytest --cov --cov-report term --cov-report xml --junitxml=xunit-result.xml tests/", ]