From 2e8a680cfde2df8800e185cb1feb41a7275ae456 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Mon, 14 Aug 2023 18:00:46 +0200 Subject: [PATCH 01/66] User-friendly error when setting manually ds.sections Solves #200 --- src/dtscalibration/datastore.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/dtscalibration/datastore.py b/src/dtscalibration/datastore.py index 01faf0c6..f2ac8215 100644 --- a/src/dtscalibration/datastore.py +++ b/src/dtscalibration/datastore.py @@ -213,6 +213,12 @@ def sections(self): def sections(self): self.attrs["_sections"] = yaml.dump(None) + @sections.setter + def sections(self, value): + msg = "Not possible anymore. Instead, pass the sections as an argument to \n" \ + "ds.dts.calibrate_single_ended() or ds.dts.calibrate_double_ended()." + raise NotImplementedError(msg) + def check_reference_section_values(self): """ Checks if the values of the used sections are of the right datatype From 07270e371de5ac3bb8555b58dc4b28e12b54b4dd Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Mon, 14 Aug 2023 18:04:45 +0200 Subject: [PATCH 02/66] User-friendly error when setting manually ds.sections Solves #200 --- src/dtscalibration/datastore.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/dtscalibration/datastore.py b/src/dtscalibration/datastore.py index f2ac8215..0d81b2ce 100644 --- a/src/dtscalibration/datastore.py +++ b/src/dtscalibration/datastore.py @@ -219,6 +219,12 @@ def sections(self, value): "ds.dts.calibrate_single_ended() or ds.dts.calibrate_double_ended()." raise NotImplementedError(msg) + @sections.setter + def sections(self, value): + msg = "Not possible anymore. Instead, pass the sections as an argument to \n" \ + "ds.dts.calibrate_single_ended() or ds.dts.calibrate_double_ended()." + raise NotImplementedError(msg) + def check_reference_section_values(self): """ Checks if the values of the used sections are of the right datatype From 66aa9f00e335ebe90774211052be3f749498f972 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Mon, 14 Aug 2023 18:05:55 +0200 Subject: [PATCH 03/66] Revert accidental double implementation of sections.setter --- src/dtscalibration/datastore.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/dtscalibration/datastore.py b/src/dtscalibration/datastore.py index 0d81b2ce..f2ac8215 100644 --- a/src/dtscalibration/datastore.py +++ b/src/dtscalibration/datastore.py @@ -219,12 +219,6 @@ def sections(self, value): "ds.dts.calibrate_single_ended() or ds.dts.calibrate_double_ended()." raise NotImplementedError(msg) - @sections.setter - def sections(self, value): - msg = "Not possible anymore. Instead, pass the sections as an argument to \n" \ - "ds.dts.calibrate_single_ended() or ds.dts.calibrate_double_ended()." - raise NotImplementedError(msg) - def check_reference_section_values(self): """ Checks if the values of the used sections are of the right datatype From 10dfbb42df0d785b39d98d44cd5fa1701bdcf18f Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Thu, 17 Aug 2023 10:12:22 +0200 Subject: [PATCH 04/66] Update documentation of datastore class --- src/dtscalibration/datastore.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/dtscalibration/datastore.py b/src/dtscalibration/datastore.py index f2ac8215..7033bc68 100644 --- a/src/dtscalibration/datastore.py +++ b/src/dtscalibration/datastore.py @@ -876,9 +876,6 @@ def variance_stokes_constant(self, st_label, sections=None, reshape_residuals=Tr dtscalibration/python-dts-calibration/blob/main/examples/notebooks/\ 04Calculate_variance_Stokes.ipynb>`_ """ - - # var_I, resid = variance_stokes_constant_util(st_label, sections=None, reshape_residuals=True) - # def variance_stokes_constant_util(st_label, sections=None, reshape_residuals=True): def func_fit(p, xs): return p[:xs, None] * p[None, xs:] @@ -1020,7 +1017,12 @@ def variance_stokes_exponential( Parameters ---------- - reshape_residuals + suppress_info : bool, optional + Suppress print statements. + use_statsmodels : bool, optional + Use statsmodels to fit the exponential. If `False`, use scipy. + reshape_residuals : bool, optional + Reshape the residuals to the shape of the Stokes intensity st_label : str label of the Stokes, anti-Stokes measurement. E.g., st, ast, rst, rast @@ -1706,6 +1708,9 @@ def calibration_single_ended( variance of the estimate of dalpha. Covariances between alpha and other parameters are not accounted for. + fix_alpha : Tuple[array-like, array-like], optional + A tuple containing two array-likes. The first array-like is the integrated + differential attenuation of length x, and the second item is its variance. Returns ------- @@ -3035,7 +3040,7 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): arr = da.concatenate( ( da.zeros( - (1, i_splice, 1), chunks=((1, i_splice, 1)), dtype=bool + (1, i_splice, 1), chunks=(1, i_splice, 1), dtype=bool ), da.ones( (1, no - i_splice, 1), @@ -3051,7 +3056,7 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): da.ones((1, i_splice, 1), chunks=(1, i_splice, 1), dtype=bool), da.zeros( (1, no - i_splice, 1), - chunks=((1, no - i_splice, 1)), + chunks=(1, no - i_splice, 1), dtype=bool, ), ), @@ -3832,7 +3837,7 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): arr = da.concatenate( ( da.zeros( - (1, i_splice, 1), chunks=((1, i_splice, 1)), dtype=bool + (1, i_splice, 1), chunks=(1, i_splice, 1), dtype=bool ), da.ones( (1, no - i_splice, 1), @@ -3848,7 +3853,7 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): da.ones((1, i_splice, 1), chunks=(1, i_splice, 1), dtype=bool), da.zeros( (1, no - i_splice, 1), - chunks=((1, no - i_splice, 1)), + chunks=(1, no - i_splice, 1), dtype=bool, ), ), From 77d9ede96020e6347feb2a8626a45d4e529624e3 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Fri, 18 Aug 2023 09:16:56 +0200 Subject: [PATCH 05/66] Removed numbagg from dependencies llvmlite is a pain to install with pip. Alternatively, it could be installed with conda --- .github/workflows/python-publish-dry-run.yml | 2 +- pyproject.toml | 18 ++++++++++-------- src/dtscalibration/datastore.py | 15 +++++++-------- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/.github/workflows/python-publish-dry-run.yml b/.github/workflows/python-publish-dry-run.yml index 74f2b3a0..66c2d5de 100644 --- a/.github/workflows/python-publish-dry-run.yml +++ b/.github/workflows/python-publish-dry-run.yml @@ -34,5 +34,5 @@ jobs: pip install build - name: Build package run: python -m build - - name: Test build + - name: Check long description for PyPi run: twine check --strict dist/* diff --git a/pyproject.toml b/pyproject.toml index 315e0bb7..162b2448 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,13 +19,13 @@ disable = true # Requires confirmation when publishing to pypi. [project] name = "dtscalibration" -description = "A Python package to load raw DTS files, perform a calibration, and plot the result." +description = "Load Distributed Temperature Sensing (DTS) files, calibrate the temperature and estimate its uncertainty." readme = "README.rst" license = "BSD-3-Clause" requires-python = ">=3.9, <3.12" authors = [ - {email = "bdestombe@gmail.com"}, - {name = "Bas des Tombe, Bart Schilperoort"} + {name = "Bas des Tombe", email = "bdestombe@gmail.com"}, + {name = "Bart Schilperoort", email = "b.schilperoort@gmail.com"}, ] maintainers = [ {name = "Bas des Tombe", email = "bdestombe@gmail.com"}, @@ -52,10 +52,12 @@ classifiers = [ "Topic :: Utilities", ] dependencies = [ - "numpy>=1.22.4", # xarray: recommended to use >= 1.22 for full quantile method support + "numpy>=1.22.4", # >= 1.22 for full quantile method support in xarray "dask", "pandas", - "xarray[parallel,accel]", + "xarray[parallel]", # numbagg (llvmlite) is a pain to install + "bottleneck", # speed up Xarray + "flox", # speed up Xarray "pyyaml>=6.0.1", "xmltodict", "scipy", @@ -74,13 +76,13 @@ dev = [ "isort", "black[jupyter]", "mypy", - "types-PyYAML", # for pyyaml types + "types-PyYAML", # for pyyaml types "types-xmltodict", # for xmltodict types - "pandas-stubs", # for pandas types + "pandas-stubs", # for pandas types "pytest", "pytest-cov", "jupyter", - "nbformat", # Needed to run the tests + "nbformat", # Needed to run the tests ] docs = [ # Required for ReadTheDocs "IPython", diff --git a/src/dtscalibration/datastore.py b/src/dtscalibration/datastore.py index 7033bc68..434868af 100644 --- a/src/dtscalibration/datastore.py +++ b/src/dtscalibration/datastore.py @@ -215,8 +215,10 @@ def sections(self): @sections.setter def sections(self, value): - msg = "Not possible anymore. Instead, pass the sections as an argument to \n" \ - "ds.dts.calibrate_single_ended() or ds.dts.calibrate_double_ended()." + msg = ( + "Not possible anymore. Instead, pass the sections as an argument to \n" + "ds.dts.calibrate_single_ended() or ds.dts.calibrate_double_ended()." + ) raise NotImplementedError(msg) def check_reference_section_values(self): @@ -876,6 +878,7 @@ def variance_stokes_constant(self, st_label, sections=None, reshape_residuals=Tr dtscalibration/python-dts-calibration/blob/main/examples/notebooks/\ 04Calculate_variance_Stokes.ipynb>`_ """ + def func_fit(p, xs): return p[:xs, None] * p[None, xs:] @@ -3039,9 +3042,7 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): if direction == "fw": arr = da.concatenate( ( - da.zeros( - (1, i_splice, 1), chunks=(1, i_splice, 1), dtype=bool - ), + da.zeros((1, i_splice, 1), chunks=(1, i_splice, 1), dtype=bool), da.ones( (1, no - i_splice, 1), chunks=(1, no - i_splice, 1), @@ -3836,9 +3837,7 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): if direction == "fw": arr = da.concatenate( ( - da.zeros( - (1, i_splice, 1), chunks=(1, i_splice, 1), dtype=bool - ), + da.zeros((1, i_splice, 1), chunks=(1, i_splice, 1), dtype=bool), da.ones( (1, no - i_splice, 1), chunks=(1, no - i_splice, 1), From c18f8eab3e9cc46fe6e7d4a54decef250c424a40 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Fri, 18 Aug 2023 12:42:07 +0200 Subject: [PATCH 06/66] Explicitly pass sections to more functions --- pyproject.toml | 2 +- src/dtscalibration/calibrate_utils.py | 33 ++++++++++------- .../calibration/section_utils.py | 14 ++++---- src/dtscalibration/datastore.py | 36 ++++++++++++++----- src/dtscalibration/plot.py | 24 ++++++++++--- tests/test_dtscalibration.py | 36 ++++++++++++------- 6 files changed, 100 insertions(+), 45 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 162b2448..ccb756f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,7 @@ dependencies = [ "numpy>=1.22.4", # >= 1.22 for full quantile method support in xarray "dask", "pandas", - "xarray[parallel]", # numbagg (llvmlite) is a pain to install + "xarray[parallel]", # numbagg (llvmlite) is a pain to install with pip "bottleneck", # speed up Xarray "flox", # speed up Xarray "pyyaml>=6.0.1", diff --git a/src/dtscalibration/calibrate_utils.py b/src/dtscalibration/calibrate_utils.py index 6e90e208..5c343ac8 100644 --- a/src/dtscalibration/calibrate_utils.py +++ b/src/dtscalibration/calibrate_utils.py @@ -48,6 +48,7 @@ def parse_st_var(ds, st_var, st_label="st"): def calibration_single_ended_helper( self, + sections, st_var, ast_var, fix_alpha, @@ -67,6 +68,7 @@ def calibration_single_ended_helper( calc_cov = True split = calibration_single_ended_solver( self, + sections, st_var, ast_var, calc_cov=calc_cov, @@ -189,6 +191,7 @@ def calibration_single_ended_helper( def calibration_single_ended_solver( # noqa: MC0001 ds, + sections=None, st_var=None, ast_var=None, calc_cov=True, @@ -235,7 +238,7 @@ def calibration_single_ended_solver( # noqa: MC0001 """ # get ix_sec argsort so the sections are in order of increasing x - ix_sec = ds.ufunc_per_section(x_indices=True, calc_per="all") + ix_sec = ds.ufunc_per_section(sections=sections, x_indices=True, calc_per="all") ds_sec = ds.isel(x=ix_sec) x_sec = ds_sec["x"].values @@ -255,7 +258,7 @@ def calibration_single_ended_solver( # noqa: MC0001 # X \gamma # Eq.34 cal_ref = ds.ufunc_per_section( - label="st", ref_temp_broadcasted=True, calc_per="all" + sections=sections, label="st", ref_temp_broadcasted=True, calc_per="all" ) # cal_ref = cal_ref # sort by increasing x data_gamma = 1 / (cal_ref.T.ravel() + 273.15) # gamma @@ -485,6 +488,7 @@ def calibration_single_ended_solver( # noqa: MC0001 def calibration_double_ended_helper( self, + sections, st_var, ast_var, rst_var, @@ -503,6 +507,7 @@ def calibration_double_ended_helper( if fix_alpha or fix_gamma: split = calibration_double_ended_solver( self, + sections, st_var, ast_var, rst_var, @@ -515,6 +520,7 @@ def calibration_double_ended_helper( else: out = calibration_double_ended_solver( self, + sections, st_var, ast_var, rst_var, @@ -887,7 +893,7 @@ def calibration_double_ended_helper( [fix_gamma[0]], out[0][: 2 * nt], E_all_exact, - out[0][2 * nt + n_E_in_cal :], + out[0][2 * nt + n_E_in_cal:], ) ) p_val[1 + 2 * nt + split["ix_from_cal_match_to_glob"]] = out[0][ @@ -987,7 +993,7 @@ def calibration_double_ended_helper( p0_est = np.concatenate( ( split["p0_est"][: 1 + 2 * nt], - split["p0_est"][1 + 2 * nt + n_E_in_cal :], + split["p0_est"][1 + 2 * nt + n_E_in_cal:], ) ) X_E1 = sp.csr_matrix(([], ([], [])), shape=(nt * nx_sec, self.x.size)) @@ -1102,6 +1108,7 @@ def calibration_double_ended_helper( def calibration_double_ended_solver( # noqa: MC0001 ds, + sections=None, st_var=None, ast_var=None, rst_var=None, @@ -1167,7 +1174,7 @@ def calibration_double_ended_solver( # noqa: MC0001 ------- """ - ix_sec = ds.ufunc_per_section(x_indices=True, calc_per="all") + ix_sec = ds.ufunc_per_section(sections=sections, x_indices=True, calc_per="all") ds_sec = ds.isel(x=ix_sec) ix_alpha_is_zero = ix_sec[0] # per definition of E @@ -1187,7 +1194,7 @@ def calibration_double_ended_solver( # noqa: MC0001 rast_var=rast_var, ix_alpha_is_zero=ix_alpha_is_zero, ) - df_est, db_est = calc_df_db_double_est(ds, ix_alpha_is_zero, 485.0) + df_est, db_est = calc_df_db_double_est(ds, sections, ix_alpha_is_zero, 485.0) ( E, @@ -1196,7 +1203,7 @@ def calibration_double_ended_solver( # noqa: MC0001 Zero_d, Z_TA_fw, Z_TA_bw, - ) = construct_submatrices(nt, nx_sec, ds, ds.trans_att.values, x_sec) + ) = construct_submatrices(sections, nt, nx_sec, ds, ds.trans_att.values, x_sec) # y # Eq.41--45 y_F = np.log(ds_sec.st / ds_sec.ast).values.ravel() @@ -1822,7 +1829,7 @@ def construct_submatrices_matching_sections(x, ix_sec, hix, tix, nt, trans_att): ) -def construct_submatrices(nt, nx, ds, trans_att, x_sec): +def construct_submatrices(sections, nt, nx, ds, trans_att, x_sec): """Wrapped in a function to reduce memory usage. E is zero at the first index of the reference section (ds_sec) Constructing: @@ -1840,7 +1847,9 @@ def construct_submatrices(nt, nx, ds, trans_att, x_sec): # Z \gamma # Eq.47 cal_ref = np.array( - ds.ufunc_per_section(label="st", ref_temp_broadcasted=True, calc_per="all") + ds.ufunc_per_section( + sections=sections, label="st", ref_temp_broadcasted=True, calc_per="all" + ) ) data_gamma = 1 / (cal_ref.ravel() + 273.15) # gamma coord_gamma_row = np.arange(nt * nx, dtype=int) @@ -2213,7 +2222,7 @@ def calc_alpha_double( return E, E_var -def calc_df_db_double_est(ds, ix_alpha_is_zero, gamma_est): +def calc_df_db_double_est(ds, sections, ix_alpha_is_zero, gamma_est): Ifwx0 = np.log( ds.st.isel(x=ix_alpha_is_zero) / ds.ast.isel(x=ix_alpha_is_zero) ).values @@ -2221,9 +2230,9 @@ def calc_df_db_double_est(ds, ix_alpha_is_zero, gamma_est): ds.rst.isel(x=ix_alpha_is_zero) / ds.rast.isel(x=ix_alpha_is_zero) ).values ref_temps_refs = ds.ufunc_per_section( - label="st", ref_temp_broadcasted=True, calc_per="all" + sections=sections, label="st", ref_temp_broadcasted=True, calc_per="all" ) - ix_sec = ds.ufunc_per_section(x_indices=True, calc_per="all") + ix_sec = ds.ufunc_per_section(sections=sections, x_indices=True, calc_per="all") ref_temps_x0 = ( ref_temps_refs[ix_sec == ix_alpha_is_zero].flatten().compute() + 273.15 ) diff --git a/src/dtscalibration/calibration/section_utils.py b/src/dtscalibration/calibration/section_utils.py index 126d575b..63dd1e5a 100644 --- a/src/dtscalibration/calibration/section_utils.py +++ b/src/dtscalibration/calibration/section_utils.py @@ -135,7 +135,7 @@ def ufunc_per_section( 1. Calculate the variance of the residuals in the along ALL the\ reference sections wrt the temperature of the water baths - >>> tmpf_var = d.ufunc_per_section( + >>> tmpf_var = d.ufunc_per_section(sections, >>> func='var', >>> calc_per='all', >>> label='tmpf', @@ -144,7 +144,7 @@ def ufunc_per_section( 2. Calculate the variance of the residuals in the along PER\ reference section wrt the temperature of the water baths - >>> tmpf_var = d.ufunc_per_section( + >>> tmpf_var = d.ufunc_per_section(sections, >>> func='var', >>> calc_per='stretch', >>> label='tmpf', @@ -153,7 +153,7 @@ def ufunc_per_section( 3. Calculate the variance of the residuals in the along PER\ water bath wrt the temperature of the water baths - >>> tmpf_var = d.ufunc_per_section( + >>> tmpf_var = d.ufunc_per_section(sections, >>> func='var', >>> calc_per='section', >>> label='tmpf', @@ -161,7 +161,7 @@ def ufunc_per_section( 4. Obtain the coordinates of the measurements per section - >>> locs = d.ufunc_per_section( + >>> locs = d.ufunc_per_section(sections, >>> func=None, >>> label='x', >>> temp_err=False, @@ -170,7 +170,7 @@ def ufunc_per_section( 5. Number of observations per stretch - >>> nlocs = d.ufunc_per_section( + >>> nlocs = d.ufunc_per_section(sections, >>> func=len, >>> label='x', >>> temp_err=False, @@ -182,14 +182,14 @@ def ufunc_per_section( temperature (a timeseries) is broadcasted to the shape of self[\ label]. The self[label] is not used for anything else. - >>> temp_ref = d.ufunc_per_section( + >>> temp_ref = d.ufunc_per_section(sections, >>> label='st', >>> ref_temp_broadcasted=True, >>> calc_per='all') 7. x-coordinate index - >>> ix_loc = d.ufunc_per_section(x_indices=True) + >>> ix_loc = d.ufunc_per_section(sections, x_indices=True) Note diff --git a/src/dtscalibration/datastore.py b/src/dtscalibration/datastore.py index 434868af..69eac39c 100644 --- a/src/dtscalibration/datastore.py +++ b/src/dtscalibration/datastore.py @@ -1754,7 +1754,9 @@ def calibration_single_ended( else: matching_indices = None - ix_sec = self.ufunc_per_section(x_indices=True, calc_per="all") + ix_sec = self.ufunc_per_section( + sections=sections, x_indices=True, calc_per="all" + ) assert not np.any(self.st.isel(x=ix_sec) <= 0.0), ( "There is uncontrolled noise in the ST signal. Are your sections" "correctly defined?" @@ -1767,6 +1769,7 @@ def calibration_single_ended( if method == "wls": p_cov, p_val, p_var = calibration_single_ended_helper( self, + sections, st_var, ast_var, fix_alpha, @@ -2172,7 +2175,9 @@ def calibration_double_ended( nx = self.x.size nt = self["time"].size nta = self.trans_att.size - ix_sec = self.ufunc_per_section(x_indices=True, calc_per="all") + ix_sec = self.ufunc_per_section( + sections=sections, x_indices=True, calc_per="all" + ) nx_sec = ix_sec.size assert self.st.dims[0] == "x", "Stokes are transposed" @@ -2218,6 +2223,7 @@ def calibration_double_ended( if method == "wls": p_cov, p_val, p_var = calibration_double_ended_helper( self, + sections, st_var, ast_var, rst_var, @@ -2899,6 +2905,7 @@ def average_single_ended( def average_double_ended( self, + sections=None, p_val="p_val", p_cov="p_cov", st_var=None, @@ -3083,6 +3090,7 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): pass self.conf_int_double_ended( + sections=sections, p_val=p_val, p_cov=p_cov, st_var=st_var, @@ -3695,6 +3703,7 @@ def conf_int_single_ended( def conf_int_double_ended( self, + sections=None, p_val="p_val", p_cov="p_cov", st_var=None, @@ -3947,7 +3956,9 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): p_cov = self[p_cov].values assert p_cov.shape == (npar, npar) - ix_sec = self.ufunc_per_section(x_indices=True, calc_per="all") + ix_sec = self.ufunc_per_section( + sections=sections, x_indices=True, calc_per="all" + ) nx_sec = ix_sec.size from_i = np.concatenate( ( @@ -4122,7 +4133,9 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): if var_only_sections: # sets the values outside the reference sections to NaN - xi = self.ufunc_per_section(x_indices=True, calc_per="all") + xi = self.ufunc_per_section( + sections=sections, x_indices=True, calc_per="all" + ) x_mask_ = [True if ix in xi else False for ix in range(self.x.size)] x_mask = np.reshape(x_mask_, (1, -1, 1)) self[label + "_mc_set"] = self[label + "_mc_set"].where(x_mask) @@ -4354,6 +4367,7 @@ def ufunc_per_section( reference sections wrt the temperature of the water baths >>> tmpf_var = d.ufunc_per_section( + >>> sections=sections, >>> func='var', >>> calc_per='all', >>> label='tmpf', @@ -4363,6 +4377,7 @@ def ufunc_per_section( reference section wrt the temperature of the water baths >>> tmpf_var = d.ufunc_per_section( + >>> sections=sections, >>> func='var', >>> calc_per='stretch', >>> label='tmpf', @@ -4372,6 +4387,7 @@ def ufunc_per_section( water bath wrt the temperature of the water baths >>> tmpf_var = d.ufunc_per_section( + >>> sections=sections, >>> func='var', >>> calc_per='section', >>> label='tmpf', @@ -4380,6 +4396,7 @@ def ufunc_per_section( 4. Obtain the coordinates of the measurements per section >>> locs = d.ufunc_per_section( + >>> sections=sections, >>> func=None, >>> label='x', >>> temp_err=False, @@ -4389,6 +4406,7 @@ def ufunc_per_section( 5. Number of observations per stretch >>> nlocs = d.ufunc_per_section( + >>> sections=sections, >>> func=len, >>> label='x', >>> temp_err=False, @@ -4407,7 +4425,7 @@ def ufunc_per_section( 7. x-coordinate index - >>> ix_loc = d.ufunc_per_section(x_indices=True) + >>> ix_loc = d.ufunc_per_section(sections=sections, x_indices=True) Note @@ -4415,9 +4433,8 @@ def ufunc_per_section( If `self[label]` or `self[subtract_from_label]` is a Dask array, a Dask array is returned else a numpy array is returned """ - if sections is None: - sections = self.sections - + # if sections is None: + # sections = self.sections if label is None: dataarray = None else: @@ -4426,7 +4443,10 @@ def ufunc_per_section( if x_indices: x_coords = self.x reference_dataset = None + else: + sections = validate_sections(self, sections) + x_coords = None reference_dataset = {k: self[k] for k in sections} diff --git a/src/dtscalibration/plot.py b/src/dtscalibration/plot.py index 27a747e4..43815b86 100755 --- a/src/dtscalibration/plot.py +++ b/src/dtscalibration/plot.py @@ -509,13 +509,14 @@ def plot_accuracy( def plot_sigma_report( - ds, temp_label, temp_var_acc_label, temp_var_prec_label=None, itimes=None + ds, sections, temp_label, temp_var_acc_label, temp_var_prec_label=None, itimes=None ): """Returns two sub-plots. first a temperature with confidence boundaries. Parameters ---------- ds + sections temp_label temp_var_label itimes @@ -573,11 +574,20 @@ def plot_sigma_report( # temp_err=True, # axis=0) sigma_est = ds.ufunc_per_section( - label=temp_label, func=np.std, temp_err=True, calc_per="stretch", axis=0 + sections=sections, + label=temp_label, + func=np.std, + temp_err=True, + calc_per="stretch", + axis=0, ) else: sigma_est = ds.ufunc_per_section( - label=temp_label, func=np.std, temp_err=True, calc_per="stretch" + sections=sections, + label=temp_label, + func=np.std, + temp_err=True, + calc_per="stretch", ) for (k, v), (k_se, v_se) in zip(ds.sections.items(), sigma_est.items()): @@ -628,9 +638,13 @@ def plot_sigma_report( ax1.set_ylabel(r"Temperature [$^\circ$C]") err_ref = ds.ufunc_per_section( - label=temp_label, func=None, temp_err=True, calc_per="stretch" + sections=sections, + label=temp_label, + func=None, + temp_err=True, + calc_per="stretch", ) - x_ref = ds.ufunc_per_section(label="x", calc_per="stretch") + x_ref = ds.ufunc_per_section(sections=sections, label="x", calc_per="stretch") for (k, v), (k_se, v_se), (kx, vx) in zip( ds.sections.items(), err_ref.items(), x_ref.items() diff --git a/tests/test_dtscalibration.py b/tests/test_dtscalibration.py index 805a900e..c4b674c1 100644 --- a/tests/test_dtscalibration.py +++ b/tests/test_dtscalibration.py @@ -321,6 +321,7 @@ def test_variance_input_types_double(): ) ds.conf_int_double_ended( + sections=sections, st_var=st_var, ast_var=st_var, rst_var=st_var, @@ -353,6 +354,7 @@ def st_var_callable(stokes): ) ds.conf_int_double_ended( + sections=sections, st_var=st_var_callable, ast_var=st_var_callable, rst_var=st_var_callable, @@ -382,6 +384,7 @@ def st_var_callable(stokes): ) ds.conf_int_double_ended( + sections=sections, st_var=st_var, ast_var=st_var, rst_var=st_var, @@ -408,6 +411,7 @@ def st_var_callable(stokes): ) ds.conf_int_double_ended( + sections=sections, st_var=st_var, ast_var=st_var, rst_var=st_var, @@ -437,6 +441,7 @@ def st_var_callable(stokes): ) ds.conf_int_double_ended( + sections=sections, st_var=st_var, ast_var=st_var, rst_var=st_var, @@ -565,6 +570,7 @@ def test_double_ended_variance_estimate_synthetic(): assert_almost_equal_verbose(ds.tmpb.mean(), 12.0, decimal=3) ds.conf_int_double_ended( + sections=sections, p_val="p_val", p_cov="p_cov", st_var=mst_var, @@ -577,20 +583,20 @@ def test_double_ended_variance_estimate_synthetic(): ) # Calibrated variance - stdsf1 = ds.ufunc_per_section( + stdsf1 = ds.ufunc_per_section(sections=sections, label="tmpf", func=np.std, temp_err=True, calc_per="stretch" ) - stdsb1 = ds.ufunc_per_section( + stdsb1 = ds.ufunc_per_section(sections=sections, label="tmpb", func=np.std, temp_err=True, calc_per="stretch" ) # Use a single timestep to better check if the parameter uncertainties propagate ds1 = ds.isel(time=1) # Estimated VAR - stdsf2 = ds1.ufunc_per_section( + stdsf2 = ds1.ufunc_per_section(sections=sections, label="tmpf_mc_var", func=np.mean, temp_err=False, calc_per="stretch" ) - stdsb2 = ds1.ufunc_per_section( + stdsb2 = ds1.ufunc_per_section(sections=sections, label="tmpb_mc_var", func=np.mean, temp_err=False, calc_per="stretch" ) @@ -701,14 +707,14 @@ def test_single_ended_variance_estimate_synthetic(): ) # Calibrated variance - stdsf1 = ds.ufunc_per_section( + stdsf1 = ds.ufunc_per_section(sections=sections, label="tmpf", func=np.std, temp_err=True, calc_per="stretch", ddof=1 ) # Use a single timestep to better check if the parameter uncertainties propagate ds1 = ds.isel(time=1) # Estimated VAR - stdsf2 = ds1.ufunc_per_section( + stdsf2 = ds1.ufunc_per_section(sections=sections, label="tmpf_mc_var", func=np.mean, temp_err=False, calc_per="stretch" ) @@ -2513,6 +2519,7 @@ def test_double_ended_exponential_variance_estimate_synthetic(): ) ds.conf_int_double_ended( + sections=sections, p_val="p_val", p_cov="p_cov", st_label=st_label, @@ -2529,10 +2536,10 @@ def test_double_ended_exponential_variance_estimate_synthetic(): ) # Calibrated variance - stdsf1 = ds.ufunc_per_section( + stdsf1 = ds.ufunc_per_section(sections=sections, label="tmpf", func=np.std, temp_err=True, calc_per="stretch" ) - stdsb1 = ds.ufunc_per_section( + stdsb1 = ds.ufunc_per_section(sections=sections, label="tmpb", func=np.std, temp_err=True, calc_per="stretch" ) @@ -2540,10 +2547,10 @@ def test_double_ended_exponential_variance_estimate_synthetic(): ds1 = ds.isel(time=1) # Estimated VAR - stdsf2 = ds1.ufunc_per_section( + stdsf2 = ds1.ufunc_per_section(sections=sections, label="tmpf_mc_var", func=np.mean, temp_err=False, calc_per="stretch" ) - stdsb2 = ds1.ufunc_per_section( + stdsb2 = ds1.ufunc_per_section(sections=sections, label="tmpb_mc_var", func=np.mean, temp_err=False, calc_per="stretch" ) @@ -2662,6 +2669,7 @@ def test_estimate_variance_of_temperature_estimate(): ) ds.conf_int_double_ended( + sections=sections, p_val="p_val", p_cov="p_cov", st_var=mst_var, @@ -3526,7 +3534,7 @@ def test_single_ended_exponential_variance_estimate_synthetic(): ) # Calibrated variance - stdsf1 = ds.ufunc_per_section( + stdsf1 = ds.ufunc_per_section(sections=sections, label="tmpf", func=np.var, temp_err=True, calc_per="stretch", ddof=1 ) @@ -3534,7 +3542,7 @@ def test_single_ended_exponential_variance_estimate_synthetic(): # propagate ds1 = ds.isel(time=1) # Estimated VAR - stdsf2 = ds1.ufunc_per_section( + stdsf2 = ds1.ufunc_per_section(sections=sections, label="tmpf_mc_var", func=np.mean, temp_err=False, calc_per="stretch" ) @@ -3675,6 +3683,7 @@ def test_average_measurements_double_ended(): solver="sparse", ) ds.average_double_ended( + sections=sections, p_val="p_val", p_cov="p_cov", st_var=st_var, @@ -3688,6 +3697,7 @@ def test_average_measurements_double_ended(): ) ix = ds.get_section_indices(slice(6, 10)) ds.average_double_ended( + sections=sections, p_val="p_val", p_cov="p_cov", st_var=st_var, @@ -3704,6 +3714,7 @@ def test_average_measurements_double_ended(): np.datetime64("2018-03-28T00:41:12.084000000"), ) ds.average_double_ended( + sections=sections, p_val="p_val", p_cov="p_cov", st_var=st_var, @@ -3717,6 +3728,7 @@ def test_average_measurements_double_ended(): ci_avg_time_sel=sl, ) ds.average_double_ended( + sections=sections, p_val="p_val", p_cov="p_cov", st_var=st_var, From bb1b477851a02a358067513affdce9af63917c47 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Fri, 18 Aug 2023 12:43:23 +0200 Subject: [PATCH 07/66] Format --- src/dtscalibration/calibrate_utils.py | 4 +- src/dtscalibration/datastore.py | 2 +- tests/test_dtscalibration.py | 82 +++++++++++++++++++-------- 3 files changed, 61 insertions(+), 27 deletions(-) diff --git a/src/dtscalibration/calibrate_utils.py b/src/dtscalibration/calibrate_utils.py index 5c343ac8..fd0117b0 100644 --- a/src/dtscalibration/calibrate_utils.py +++ b/src/dtscalibration/calibrate_utils.py @@ -893,7 +893,7 @@ def calibration_double_ended_helper( [fix_gamma[0]], out[0][: 2 * nt], E_all_exact, - out[0][2 * nt + n_E_in_cal:], + out[0][2 * nt + n_E_in_cal :], ) ) p_val[1 + 2 * nt + split["ix_from_cal_match_to_glob"]] = out[0][ @@ -993,7 +993,7 @@ def calibration_double_ended_helper( p0_est = np.concatenate( ( split["p0_est"][: 1 + 2 * nt], - split["p0_est"][1 + 2 * nt + n_E_in_cal:], + split["p0_est"][1 + 2 * nt + n_E_in_cal :], ) ) X_E1 = sp.csr_matrix(([], ([], [])), shape=(nt * nx_sec, self.x.size)) diff --git a/src/dtscalibration/datastore.py b/src/dtscalibration/datastore.py index 69eac39c..fa367dbf 100644 --- a/src/dtscalibration/datastore.py +++ b/src/dtscalibration/datastore.py @@ -4443,7 +4443,7 @@ def ufunc_per_section( if x_indices: x_coords = self.x reference_dataset = None - + else: sections = validate_sections(self, sections) diff --git a/tests/test_dtscalibration.py b/tests/test_dtscalibration.py index c4b674c1..66fd43c2 100644 --- a/tests/test_dtscalibration.py +++ b/tests/test_dtscalibration.py @@ -583,21 +583,29 @@ def test_double_ended_variance_estimate_synthetic(): ) # Calibrated variance - stdsf1 = ds.ufunc_per_section(sections=sections, - label="tmpf", func=np.std, temp_err=True, calc_per="stretch" + stdsf1 = ds.ufunc_per_section( + sections=sections, label="tmpf", func=np.std, temp_err=True, calc_per="stretch" ) - stdsb1 = ds.ufunc_per_section(sections=sections, - label="tmpb", func=np.std, temp_err=True, calc_per="stretch" + stdsb1 = ds.ufunc_per_section( + sections=sections, label="tmpb", func=np.std, temp_err=True, calc_per="stretch" ) # Use a single timestep to better check if the parameter uncertainties propagate ds1 = ds.isel(time=1) # Estimated VAR - stdsf2 = ds1.ufunc_per_section(sections=sections, - label="tmpf_mc_var", func=np.mean, temp_err=False, calc_per="stretch" + stdsf2 = ds1.ufunc_per_section( + sections=sections, + label="tmpf_mc_var", + func=np.mean, + temp_err=False, + calc_per="stretch", ) - stdsb2 = ds1.ufunc_per_section(sections=sections, - label="tmpb_mc_var", func=np.mean, temp_err=False, calc_per="stretch" + stdsb2 = ds1.ufunc_per_section( + sections=sections, + label="tmpb_mc_var", + func=np.mean, + temp_err=False, + calc_per="stretch", ) for (_, v1), (_, v2) in zip(stdsf1.items(), stdsf2.items()): @@ -707,15 +715,24 @@ def test_single_ended_variance_estimate_synthetic(): ) # Calibrated variance - stdsf1 = ds.ufunc_per_section(sections=sections, - label="tmpf", func=np.std, temp_err=True, calc_per="stretch", ddof=1 + stdsf1 = ds.ufunc_per_section( + sections=sections, + label="tmpf", + func=np.std, + temp_err=True, + calc_per="stretch", + ddof=1, ) # Use a single timestep to better check if the parameter uncertainties propagate ds1 = ds.isel(time=1) # Estimated VAR - stdsf2 = ds1.ufunc_per_section(sections=sections, - label="tmpf_mc_var", func=np.mean, temp_err=False, calc_per="stretch" + stdsf2 = ds1.ufunc_per_section( + sections=sections, + label="tmpf_mc_var", + func=np.mean, + temp_err=False, + calc_per="stretch", ) for (_, v1), (_, v2) in zip(stdsf1.items(), stdsf2.items()): @@ -2536,22 +2553,30 @@ def test_double_ended_exponential_variance_estimate_synthetic(): ) # Calibrated variance - stdsf1 = ds.ufunc_per_section(sections=sections, - label="tmpf", func=np.std, temp_err=True, calc_per="stretch" + stdsf1 = ds.ufunc_per_section( + sections=sections, label="tmpf", func=np.std, temp_err=True, calc_per="stretch" ) - stdsb1 = ds.ufunc_per_section(sections=sections, - label="tmpb", func=np.std, temp_err=True, calc_per="stretch" + stdsb1 = ds.ufunc_per_section( + sections=sections, label="tmpb", func=np.std, temp_err=True, calc_per="stretch" ) # Use a single timestep to better check if the parameter uncertainties propagate ds1 = ds.isel(time=1) # Estimated VAR - stdsf2 = ds1.ufunc_per_section(sections=sections, - label="tmpf_mc_var", func=np.mean, temp_err=False, calc_per="stretch" + stdsf2 = ds1.ufunc_per_section( + sections=sections, + label="tmpf_mc_var", + func=np.mean, + temp_err=False, + calc_per="stretch", ) - stdsb2 = ds1.ufunc_per_section(sections=sections, - label="tmpb_mc_var", func=np.mean, temp_err=False, calc_per="stretch" + stdsb2 = ds1.ufunc_per_section( + sections=sections, + label="tmpb_mc_var", + func=np.mean, + temp_err=False, + calc_per="stretch", ) for (_, v1), (_, v2) in zip(stdsf1.items(), stdsf2.items()): @@ -3534,16 +3559,25 @@ def test_single_ended_exponential_variance_estimate_synthetic(): ) # Calibrated variance - stdsf1 = ds.ufunc_per_section(sections=sections, - label="tmpf", func=np.var, temp_err=True, calc_per="stretch", ddof=1 + stdsf1 = ds.ufunc_per_section( + sections=sections, + label="tmpf", + func=np.var, + temp_err=True, + calc_per="stretch", + ddof=1, ) # Use a single timestep to better check if the parameter uncertainties # propagate ds1 = ds.isel(time=1) # Estimated VAR - stdsf2 = ds1.ufunc_per_section(sections=sections, - label="tmpf_mc_var", func=np.mean, temp_err=False, calc_per="stretch" + stdsf2 = ds1.ufunc_per_section( + sections=sections, + label="tmpf_mc_var", + func=np.mean, + temp_err=False, + calc_per="stretch", ) for (_, v1), (_, v2) in zip(stdsf1.items(), stdsf2.items()): From b453f5965ed54257c2254da10bd8ec2b17677359 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Fri, 18 Aug 2023 15:48:15 +0200 Subject: [PATCH 08/66] Moving away from calibrate_double_ended() inplace self.update(out) is still in place at the end --- src/dtscalibration/datastore.py | 454 ++++++++++++++------------ src/dtscalibration/datastore_utils.py | 39 +++ 2 files changed, 293 insertions(+), 200 deletions(-) diff --git a/src/dtscalibration/datastore.py b/src/dtscalibration/datastore.py index fa367dbf..1c69553d 100644 --- a/src/dtscalibration/datastore.py +++ b/src/dtscalibration/datastore.py @@ -21,6 +21,7 @@ from dtscalibration.datastore_utils import ParameterIndexSingleEnded from dtscalibration.datastore_utils import check_deprecated_kwargs from dtscalibration.datastore_utils import check_timestep_allclose +from dtscalibration.datastore_utils import get_params_from_pval from dtscalibration.datastore_utils import ufunc_per_section_helper from dtscalibration.io_utils import _dim_attrs @@ -2258,215 +2259,243 @@ def calibration_double_ended( assert p_var.size == ip.npar assert p_cov.shape == (ip.npar, ip.npar) - # save estimates and variances to datastore, skip covariances - self["gamma"] = (tuple(), p_val[ip.gamma].item()) - self["alpha"] = (("x",), p_val[ip.alpha]) - self["df"] = (("time",), p_val[ip.df]) - self["db"] = (("time",), p_val[ip.db]) - - if nta: - self["talpha_fw"] = ( - ("time", "trans_att"), - p_val[ip.taf].reshape((nt, nta), order="C"), - ) - self["talpha_bw"] = ( - ("time", "trans_att"), - p_val[ip.tab].reshape((nt, nta), order="C"), - ) - self["talpha_fw_var"] = ( - ("time", "trans_att"), - p_var[ip.taf].reshape((nt, nta), order="C"), - ) - self["talpha_bw_var"] = ( - ("time", "trans_att"), - p_var[ip.tab].reshape((nt, nta), order="C"), - ) - else: - self["talpha_fw"] = (("time", "trans_att"), np.zeros((nt, 0))) - self["talpha_bw"] = (("time", "trans_att"), np.zeros((nt, 0))) - self["talpha_fw_var"] = (("time", "trans_att"), np.zeros((nt, 0))) - self["talpha_bw_var"] = (("time", "trans_att"), np.zeros((nt, 0))) + coords = {"x": self["x"], "time": self["time"], "trans_att": self["trans_att"]} + params = get_params_from_pval(ip, p_val, coords) + param_covs = get_params_from_pval(ip, p_var, coords) + out = xr.Dataset( + { + "tmpf": params["gamma"] + / ( + np.log(self.st / self.ast) + + params["df"] + + params["alpha"] + + params["talpha_fw_full"] + ) + - 273.15, + "tmpb": params["gamma"] + / ( + np.log(self.rst / self.rast) + + params["db"] + - params["alpha"] + + params["talpha_bw_full"] + ) + - 273.15, + } + ) - self["talpha_fw_full"] = ( - ("x", "time"), - ip.get_taf_values( - pval=p_val, x=self.x.values, trans_att=self.trans_att.values, axis="" - ), + # extract covariances and ensure broadcastable to (nx, nt) + param_covs["gamma_df"] = (("time",), p_cov[np.ix_(ip.gamma, ip.df)][0]) + param_covs["gamma_db"] = (("time",), p_cov[np.ix_(ip.gamma, ip.db)][0]) + param_covs["gamma_alpha"] = (("x",), p_cov[np.ix_(ip.alpha, ip.gamma)][:, 0]) + param_covs["df_db"] = ( + ("time",), + p_cov[ip.df, ip.db], ) - self["talpha_bw_full"] = ( - ("x", "time"), - ip.get_tab_values( - pval=p_val, x=self.x.values, trans_att=self.trans_att.values, axis="" + param_covs["alpha_df"] = ( + ( + "x", + "time", ), + p_cov[np.ix_(ip.alpha, ip.df)], ) - - self["tmpf"] = ( - self.gamma - / ( - np.log(self.st / self.ast) - + self["df"] - + self.alpha - + self["talpha_fw_full"] - ) - - 273.15 - ) - - self["tmpb"] = ( - self.gamma - / ( - np.log(self.rst / self.rast) - + self["db"] - - self.alpha - + self["talpha_bw_full"] - ) - - 273.15 + param_covs["alpha_db"] = ( + ( + "x", + "time", + ), + p_cov[np.ix_(ip.alpha, ip.db)], ) - - self["gamma_var"] = (tuple(), p_var[ip.gamma].item()) - self["alpha_var"] = (("x",), p_var[ip.alpha]) - self["df_var"] = (("time",), p_var[ip.df]) - self["db_var"] = (("time",), p_var[ip.db]) - self["talpha_fw_full_var"] = ( - ("x", "time"), + param_covs["tafw_gamma"] = ( + ( + "x", + "time", + ), ip.get_taf_values( - pval=p_var, x=self.x.values, trans_att=self.trans_att.values, axis="" + pval=p_cov[ip.gamma], + x=self.x.values, + trans_att=self.trans_att.values, + axis="", ), ) - self["talpha_bw_full_var"] = ( - ("x", "time"), + param_covs["tabw_gamma"] = ( + ( + "x", + "time", + ), ip.get_tab_values( - pval=p_var, x=self.x.values, trans_att=self.trans_att.values, axis="" + pval=p_cov[ip.gamma], + x=self.x.values, + trans_att=self.trans_att.values, + axis="", ), ) - - # extract covariances and ensure broadcastable to (nx, nt) - sigma2_gamma_df = p_cov[np.ix_(ip.gamma, ip.df)] - sigma2_gamma_db = p_cov[np.ix_(ip.gamma, ip.db)] - sigma2_gamma_alpha = p_cov[np.ix_(ip.alpha, ip.gamma)] - sigma2_df_db = p_cov[ip.df, ip.db][None] # Shape is [0, nt] ? - sigma2_alpha_df = p_cov[np.ix_(ip.alpha, ip.df)] - sigma2_alpha_db = p_cov[np.ix_(ip.alpha, ip.db)] - sigma2_tafw_gamma = ip.get_taf_values( - pval=p_cov[ip.gamma], - x=self.x.values, - trans_att=self.trans_att.values, - axis="", - ) - sigma2_tabw_gamma = ip.get_tab_values( - pval=p_cov[ip.gamma], - x=self.x.values, - trans_att=self.trans_att.values, - axis="", - ) - sigma2_tafw_alpha = ip.get_taf_values( - pval=p_cov[ip.alpha], - x=self.x.values, - trans_att=self.trans_att.values, - axis="x", + param_covs["tafw_alpha"] = ( + ( + "x", + "time", + ), + ip.get_taf_values( + pval=p_cov[ip.alpha], + x=self.x.values, + trans_att=self.trans_att.values, + axis="x", + ), ) - sigma2_tabw_alpha = ip.get_tab_values( - pval=p_cov[ip.alpha], - x=self.x.values, - trans_att=self.trans_att.values, - axis="x", + param_covs["tabw_alpha"] = ( + ( + "x", + "time", + ), + ip.get_tab_values( + pval=p_cov[ip.alpha], + x=self.x.values, + trans_att=self.trans_att.values, + axis="x", + ), ) - sigma2_tafw_df = ip.get_taf_values( - pval=p_cov[ip.df], - x=self.x.values, - trans_att=self.trans_att.values, - axis="time", + param_covs["tafw_df"] = ( + ( + "x", + "time", + ), + ip.get_taf_values( + pval=p_cov[ip.df], + x=self.x.values, + trans_att=self.trans_att.values, + axis="time", + ), ) - sigma2_tafw_db = ip.get_taf_values( - pval=p_cov[ip.db], - x=self.x.values, - trans_att=self.trans_att.values, - axis="time", + param_covs["tafw_db"] = ( + ( + "x", + "time", + ), + ip.get_taf_values( + pval=p_cov[ip.db], + x=self.x.values, + trans_att=self.trans_att.values, + axis="time", + ), ) - sigma2_tabw_db = ip.get_tab_values( - pval=p_cov[ip.db], - x=self.x.values, - trans_att=self.trans_att.values, - axis="time", + param_covs["tabw_db"] = ( + ( + "x", + "time", + ), + ip.get_tab_values( + pval=p_cov[ip.db], + x=self.x.values, + trans_att=self.trans_att.values, + axis="time", + ), ) - sigma2_tabw_df = ip.get_tab_values( - pval=p_cov[ip.df], - x=self.x.values, - trans_att=self.trans_att.values, - axis="time", + param_covs["tabw_df"] = ( + ( + "x", + "time", + ), + ip.get_tab_values( + pval=p_cov[ip.df], + x=self.x.values, + trans_att=self.trans_att.values, + axis="time", + ), ) # sigma2_tafw_tabw - tmpf = self["tmpf"] + 273.15 - tmpb = self["tmpb"] + 273.15 + tmpf = out["tmpf"] + 273.15 + tmpb = out["tmpb"] + 273.15 deriv_dict = dict( - T_gamma_fw=tmpf / self.gamma, - T_st_fw=-(tmpf**2) / (self.gamma * self.st), - T_ast_fw=tmpf**2 / (self.gamma * self.ast), - T_df_fw=-(tmpf**2) / self.gamma, - T_alpha_fw=-(tmpf**2) / self.gamma, - T_ta_fw=-(tmpf**2) / self.gamma, - T_gamma_bw=tmpb / self.gamma, - T_rst_bw=-(tmpb**2) / (self.gamma * self.rst), - T_rast_bw=tmpb**2 / (self.gamma * self.rast), - T_db_bw=-(tmpb**2) / self.gamma, - T_alpha_bw=tmpb**2 / self.gamma, - T_ta_bw=-(tmpb**2) / self.gamma, + T_gamma_fw=tmpf / params["gamma"], + T_st_fw=-(tmpf**2) / (params["gamma"] * self.st), + T_ast_fw=tmpf**2 / (params["gamma"] * self.ast), + T_df_fw=-(tmpf**2) / params["gamma"], + T_alpha_fw=-(tmpf**2) / params["gamma"], + T_ta_fw=-(tmpf**2) / params["gamma"], + T_gamma_bw=tmpb / params["gamma"], + T_rst_bw=-(tmpb**2) / (params["gamma"] * self.rst), + T_rast_bw=tmpb**2 / (params["gamma"] * self.rast), + T_db_bw=-(tmpb**2) / params["gamma"], + T_alpha_bw=tmpb**2 / params["gamma"], + T_ta_bw=-(tmpb**2) / params["gamma"], ) deriv_ds = xr.Dataset(deriv_dict) - self["deriv"] = deriv_ds.to_array(dim="com2") + out["deriv"] = deriv_ds.to_array(dim="com2") var_fw_dict = dict( dT_dst=deriv_ds.T_st_fw**2 * parse_st_var(self, st_var, st_label="st"), dT_dast=deriv_ds.T_ast_fw**2 * parse_st_var(self, ast_var, st_label="ast"), - dT_gamma=deriv_ds.T_gamma_fw**2 * self["gamma_var"], - dT_ddf=deriv_ds.T_df_fw**2 * self["df_var"], - dT_dalpha=deriv_ds.T_alpha_fw**2 * self["alpha_var"], - dT_dta=deriv_ds.T_ta_fw**2 * self["talpha_fw_full_var"], - dgamma_ddf=(2 * deriv_ds.T_gamma_fw * deriv_ds.T_df_fw * sigma2_gamma_df), + dT_gamma=deriv_ds.T_gamma_fw**2 * param_covs["gamma"], + dT_ddf=deriv_ds.T_df_fw**2 * param_covs["df"], + dT_dalpha=deriv_ds.T_alpha_fw**2 * param_covs["alpha"], + dT_dta=deriv_ds.T_ta_fw**2 * param_covs["talpha_fw_full"], + dgamma_ddf=( + 2 * deriv_ds.T_gamma_fw * deriv_ds.T_df_fw * param_covs["gamma_df"] + ), dgamma_dalpha=( - 2 * deriv_ds.T_gamma_fw * deriv_ds.T_alpha_fw * sigma2_gamma_alpha + 2 + * deriv_ds.T_gamma_fw + * deriv_ds.T_alpha_fw + * param_covs["gamma_alpha"] + ), + dalpha_ddf=( + 2 * deriv_ds.T_alpha_fw * deriv_ds.T_df_fw * param_covs["alpha_df"] + ), + dta_dgamma=( + 2 * deriv_ds.T_ta_fw * deriv_ds.T_gamma_fw * param_covs["tafw_gamma"] + ), + dta_ddf=(2 * deriv_ds.T_ta_fw * deriv_ds.T_df_fw * param_covs["tafw_df"]), + dta_dalpha=( + 2 * deriv_ds.T_ta_fw * deriv_ds.T_alpha_fw * param_covs["tafw_alpha"] ), - dalpha_ddf=(2 * deriv_ds.T_alpha_fw * deriv_ds.T_df_fw * sigma2_alpha_df), - dta_dgamma=(2 * deriv_ds.T_ta_fw * deriv_ds.T_gamma_fw * sigma2_tafw_gamma), - dta_ddf=(2 * deriv_ds.T_ta_fw * deriv_ds.T_df_fw * sigma2_tafw_df), - dta_dalpha=(2 * deriv_ds.T_ta_fw * deriv_ds.T_alpha_fw * sigma2_tafw_alpha), ) var_bw_dict = dict( dT_drst=deriv_ds.T_rst_bw**2 * parse_st_var(self, rst_var, st_label="rst"), dT_drast=deriv_ds.T_rast_bw**2 * parse_st_var(self, rast_var, st_label="rast"), - dT_gamma=deriv_ds.T_gamma_bw**2 * self["gamma_var"], - dT_ddb=deriv_ds.T_db_bw**2 * self["db_var"], - dT_dalpha=deriv_ds.T_alpha_bw**2 * self["alpha_var"], - dT_dta=deriv_ds.T_ta_bw**2 * self["talpha_bw_full_var"], - dgamma_ddb=(2 * deriv_ds.T_gamma_bw * deriv_ds.T_db_bw * sigma2_gamma_db), + dT_gamma=deriv_ds.T_gamma_bw**2 * param_covs["gamma"], + dT_ddb=deriv_ds.T_db_bw**2 * param_covs["db"], + dT_dalpha=deriv_ds.T_alpha_bw**2 * param_covs["alpha"], + dT_dta=deriv_ds.T_ta_bw**2 * param_covs["talpha_bw_full"], + dgamma_ddb=( + 2 * deriv_ds.T_gamma_bw * deriv_ds.T_db_bw * param_covs["gamma_db"] + ), dgamma_dalpha=( - 2 * deriv_ds.T_gamma_bw * deriv_ds.T_alpha_bw * sigma2_gamma_alpha + 2 + * deriv_ds.T_gamma_bw + * deriv_ds.T_alpha_bw + * param_covs["gamma_alpha"] + ), + dalpha_ddb=( + 2 * deriv_ds.T_alpha_bw * deriv_ds.T_db_bw * param_covs["alpha_db"] + ), + dta_dgamma=( + 2 * deriv_ds.T_ta_bw * deriv_ds.T_gamma_bw * param_covs["tabw_gamma"] + ), + dta_ddb=(2 * deriv_ds.T_ta_bw * deriv_ds.T_db_bw * param_covs["tabw_db"]), + dta_dalpha=( + 2 * deriv_ds.T_ta_bw * deriv_ds.T_alpha_bw * param_covs["tabw_alpha"] ), - dalpha_ddb=(2 * deriv_ds.T_alpha_bw * deriv_ds.T_db_bw * sigma2_alpha_db), - dta_dgamma=(2 * deriv_ds.T_ta_bw * deriv_ds.T_gamma_bw * sigma2_tabw_gamma), - dta_ddb=(2 * deriv_ds.T_ta_bw * deriv_ds.T_db_bw * sigma2_tabw_db), - dta_dalpha=(2 * deriv_ds.T_ta_bw * deriv_ds.T_alpha_bw * sigma2_tabw_alpha), ) - self["var_fw_da"] = xr.Dataset(var_fw_dict).to_array(dim="comp_fw") - self["var_bw_da"] = xr.Dataset(var_bw_dict).to_array(dim="comp_bw") + out["var_fw_da"] = xr.Dataset(var_fw_dict).to_array(dim="comp_fw") + out["var_bw_da"] = xr.Dataset(var_bw_dict).to_array(dim="comp_bw") - self["tmpf_var"] = self["var_fw_da"].sum(dim="comp_fw") - self["tmpb_var"] = self["var_bw_da"].sum(dim="comp_bw") + out["tmpf_var"] = out["var_fw_da"].sum(dim="comp_fw") + out["tmpb_var"] = out["var_bw_da"].sum(dim="comp_bw") # First estimate of tmpw_var - self["tmpw_var" + "_approx"] = 1 / (1 / self["tmpf_var"] + 1 / self["tmpb_var"]) - self["tmpw"] = ( - (tmpf / self["tmpf_var"] + tmpb / self["tmpb_var"]) - * self["tmpw_var" + "_approx"] + out["tmpw_var" + "_approx"] = 1 / (1 / out["tmpf_var"] + 1 / out["tmpb_var"]) + out["tmpw"] = ( + (tmpf / out["tmpf_var"] + tmpb / out["tmpb_var"]) + * out["tmpw_var" + "_approx"] ) - 273.15 - weightsf = self["tmpw_var" + "_approx"] / self["tmpf_var"] - weightsb = self["tmpw_var" + "_approx"] / self["tmpb_var"] + weightsf = out["tmpw_var" + "_approx"] / out["tmpf_var"] + weightsb = out["tmpw_var" + "_approx"] / out["tmpb_var"] deriv_dict2 = dict( T_gamma_w=weightsf * deriv_dict["T_gamma_fw"] @@ -2493,61 +2522,85 @@ def calibration_double_ended( * parse_st_var(self, rst_var, st_label="rst"), dT_drast=deriv_ds2.T_rast_w**2 * parse_st_var(self, rast_var, st_label="rast"), - dT_gamma=deriv_ds2.T_gamma_w**2 * self["gamma_var"], - dT_ddf=deriv_ds2.T_df_w**2 * self["df_var"], - dT_ddb=deriv_ds2.T_db_w**2 * self["db_var"], - dT_dalpha=deriv_ds2.T_alpha_w**2 * self["alpha_var"], - dT_dtaf=deriv_ds2.T_taf_w**2 * self["talpha_fw_full_var"], - dT_dtab=deriv_ds2.T_tab_w**2 * self["talpha_bw_full_var"], - dgamma_ddf=2 * deriv_ds2.T_gamma_w * deriv_ds2.T_df_w * sigma2_gamma_df, - dgamma_ddb=2 * deriv_ds2.T_gamma_w * deriv_ds2.T_db_w * sigma2_gamma_db, + dT_gamma=deriv_ds2.T_gamma_w**2 * param_covs["gamma"], + dT_ddf=deriv_ds2.T_df_w**2 * param_covs["df"], + dT_ddb=deriv_ds2.T_db_w**2 * param_covs["db"], + dT_dalpha=deriv_ds2.T_alpha_w**2 * param_covs["alpha"], + dT_dtaf=deriv_ds2.T_taf_w**2 * param_covs["talpha_fw_full"], + dT_dtab=deriv_ds2.T_tab_w**2 * param_covs["talpha_bw_full"], + dgamma_ddf=2 + * deriv_ds2.T_gamma_w + * deriv_ds2.T_df_w + * param_covs["gamma_df"], + dgamma_ddb=2 + * deriv_ds2.T_gamma_w + * deriv_ds2.T_db_w + * param_covs["gamma_db"], dgamma_dalpha=2 * deriv_ds2.T_gamma_w * deriv_ds2.T_alpha_w - * sigma2_gamma_alpha, - dgamma_dtaf=2 * deriv_ds2.T_gamma_w * deriv_ds2.T_taf_w * sigma2_tafw_gamma, - dgamma_dtab=2 * deriv_ds2.T_gamma_w * deriv_ds2.T_tab_w * sigma2_tabw_gamma, - ddf_ddb=2 * deriv_ds2.T_df_w * deriv_ds2.T_db_w * sigma2_df_db, - ddf_dalpha=2 * deriv_ds2.T_df_w * deriv_ds2.T_alpha_w * sigma2_alpha_df, - ddf_dtaf=2 * deriv_ds2.T_df_w * deriv_ds2.T_taf_w * sigma2_tafw_df, - ddf_dtab=2 * deriv_ds2.T_df_w * deriv_ds2.T_tab_w * sigma2_tabw_df, - ddb_dalpha=2 * deriv_ds2.T_db_w * deriv_ds2.T_alpha_w * sigma2_alpha_db, - ddb_dtaf=2 * deriv_ds2.T_db_w * deriv_ds2.T_taf_w * sigma2_tafw_db, - ddb_dtab=2 * deriv_ds2.T_db_w * deriv_ds2.T_tab_w * sigma2_tabw_db, - # dtaf_dtab=2 * deriv_ds2.T_tab_w * deriv_ds2.T_tab_w * sigma2_tafw_tabw, + * param_covs["gamma_alpha"], + dgamma_dtaf=2 + * deriv_ds2.T_gamma_w + * deriv_ds2.T_taf_w + * param_covs["tafw_gamma"], + dgamma_dtab=2 + * deriv_ds2.T_gamma_w + * deriv_ds2.T_tab_w + * param_covs["tabw_gamma"], + ddf_ddb=2 * deriv_ds2.T_df_w * deriv_ds2.T_db_w * param_covs["df_db"], + ddf_dalpha=2 + * deriv_ds2.T_df_w + * deriv_ds2.T_alpha_w + * param_covs["alpha_df"], + ddf_dtaf=2 * deriv_ds2.T_df_w * deriv_ds2.T_taf_w * param_covs["tafw_df"], + ddf_dtab=2 * deriv_ds2.T_df_w * deriv_ds2.T_tab_w * param_covs["tabw_df"], + ddb_dalpha=2 + * deriv_ds2.T_db_w + * deriv_ds2.T_alpha_w + * param_covs["alpha_db"], + ddb_dtaf=2 * deriv_ds2.T_db_w * deriv_ds2.T_taf_w * param_covs["tafw_db"], + ddb_dtab=2 * deriv_ds2.T_db_w * deriv_ds2.T_tab_w * param_covs["tabw_db"], + # dtaf_dtab=2 * deriv_ds2.T_tab_w * deriv_ds2.T_tab_w * param_covs["tafw_tabw"], ) - self["var_w_da"] = xr.Dataset(var_w_dict).to_array(dim="comp_w") - self["tmpw_var"] = self["var_w_da"].sum(dim="comp_w") + out["var_w_da"] = xr.Dataset(var_w_dict).to_array(dim="comp_w") + out["tmpw_var"] = out["var_w_da"].sum(dim="comp_w") + # Compute uncertainty solely due to noise in Stokes signal, neglecting parameter uncenrtainty tmpf_var_excl_par = ( - self["var_fw_da"].sel(comp_fw=["dT_dst", "dT_dast"]).sum(dim="comp_fw") + out["var_fw_da"].sel(comp_fw=["dT_dst", "dT_dast"]).sum(dim="comp_fw") ) tmpb_var_excl_par = ( - self["var_bw_da"].sel(comp_bw=["dT_drst", "dT_drast"]).sum(dim="comp_bw") - ) - self["tmpw_var" + "_lower"] = 1 / ( - 1 / tmpf_var_excl_par + 1 / tmpb_var_excl_par + out["var_bw_da"].sel(comp_bw=["dT_drst", "dT_drast"]).sum(dim="comp_bw") ) + out["tmpw_var" + "_lower"] = 1 / (1 / tmpf_var_excl_par + 1 / tmpb_var_excl_par) - self["tmpf"].attrs.update(_dim_attrs[("tmpf",)]) - self["tmpb"].attrs.update(_dim_attrs[("tmpb",)]) - self["tmpw"].attrs.update(_dim_attrs[("tmpw",)]) - self["tmpf_var"].attrs.update(_dim_attrs[("tmpf_var",)]) - self["tmpb_var"].attrs.update(_dim_attrs[("tmpb_var",)]) - self["tmpw_var"].attrs.update(_dim_attrs[("tmpw_var",)]) - self["tmpw_var" + "_approx"].attrs.update(_dim_attrs[("tmpw_var_approx",)]) - self["tmpw_var" + "_lower"].attrs.update(_dim_attrs[("tmpw_var_lower",)]) + out["tmpf"].attrs.update(_dim_attrs[("tmpf",)]) + out["tmpb"].attrs.update(_dim_attrs[("tmpb",)]) + out["tmpw"].attrs.update(_dim_attrs[("tmpw",)]) + out["tmpf_var"].attrs.update(_dim_attrs[("tmpf_var",)]) + out["tmpb_var"].attrs.update(_dim_attrs[("tmpb_var",)]) + out["tmpw_var"].attrs.update(_dim_attrs[("tmpw_var",)]) + out["tmpw_var" + "_approx"].attrs.update(_dim_attrs[("tmpw_var_approx",)]) + out["tmpw_var" + "_lower"].attrs.update(_dim_attrs[("tmpw_var_lower",)]) drop_vars = [ k for k, v in self.items() if {"params1", "params2"}.intersection(v.dims) ] for k in drop_vars: + print(f"removing {k}") del self[k] - self["p_val"] = (("params1",), p_val) - self["p_cov"] = (("params1", "params2"), p_cov) + out["p_val"] = (("params1",), p_val) + out["p_cov"] = (("params1", "params2"), p_cov) + + out.update(params) + for key, da in param_covs.items(): + out[key + "_var"] = da + + self.update(out) pass def average_single_ended( @@ -2879,6 +2932,7 @@ def average_single_ended( ) # The new CI dimension is added as # firsaxis self["tmpf_mc_avg2"] = (("CI", x_dim2), qq) + # Clean up the garbage. All arrays with a Monte Carlo dimension. if mc_remove_set_flag: remove_mc_set = [ diff --git a/src/dtscalibration/datastore_utils.py b/src/dtscalibration/datastore_utils.py index dc2daad3..a4b9f875 100644 --- a/src/dtscalibration/datastore_utils.py +++ b/src/dtscalibration/datastore_utils.py @@ -528,6 +528,45 @@ def get_netcdf_encoding( return encoding +def get_params_from_pval(ip, p_val, coords): + assert len(p_val) == ip.npar, "Length of p_val is incorrect" + + params = xr.Dataset(coords=coords) + + # save estimates and variances to datastore, skip covariances + params["gamma"] = (tuple(), p_val[ip.gamma].item()) + params["alpha"] = (("x",), p_val[ip.alpha]) + params["df"] = (("time",), p_val[ip.df]) + params["db"] = (("time",), p_val[ip.db]) + + if ip.nta: + params["talpha_fw"] = ( + ("time", "trans_att"), + p_val[ip.taf].reshape((ip.nt, ip.nta), order="C"), + ) + params["talpha_bw"] = ( + ("time", "trans_att"), + p_val[ip.tab].reshape((ip.nt, ip.nta), order="C"), + ) + else: + params["talpha_fw"] = (("time", "trans_att"), np.zeros((ip.nt, 0))) + params["talpha_bw"] = (("time", "trans_att"), np.zeros((ip.nt, 0))) + + params["talpha_fw_full"] = ( + ("x", "time"), + ip.get_taf_values( + pval=p_val, x=params.x.values, trans_att=params.trans_att.values, axis="" + ), + ) + params["talpha_bw_full"] = ( + ("x", "time"), + ip.get_tab_values( + pval=p_val, x=params.x.values, trans_att=params.trans_att.values, axis="" + ), + ) + return params + + def merge_double_ended( ds_fw: "DataStore", ds_bw: "DataStore", From e884f945402ea54e5c360bf184477ccacc72d0f0 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Fri, 18 Aug 2023 16:03:06 +0200 Subject: [PATCH 09/66] Refactored calibrate_double_ended covariance calculation --- src/dtscalibration/datastore.py | 127 +---------------- src/dtscalibration/datastore_utils.py | 197 +++++++++++++++++++++----- 2 files changed, 167 insertions(+), 157 deletions(-) diff --git a/src/dtscalibration/datastore.py b/src/dtscalibration/datastore.py index 1c69553d..b5af307e 100644 --- a/src/dtscalibration/datastore.py +++ b/src/dtscalibration/datastore.py @@ -2260,8 +2260,9 @@ def calibration_double_ended( assert p_cov.shape == (ip.npar, ip.npar) coords = {"x": self["x"], "time": self["time"], "trans_att": self["trans_att"]} - params = get_params_from_pval(ip, p_val, coords) - param_covs = get_params_from_pval(ip, p_var, coords) + params = get_params_from_pval(ip, coords, p_val=p_val) + param_covs = get_params_from_pval(ip, coords, p_val=p_var, p_cov=p_cov) + out = xr.Dataset( { "tmpf": params["gamma"] @@ -2283,126 +2284,6 @@ def calibration_double_ended( } ) - # extract covariances and ensure broadcastable to (nx, nt) - param_covs["gamma_df"] = (("time",), p_cov[np.ix_(ip.gamma, ip.df)][0]) - param_covs["gamma_db"] = (("time",), p_cov[np.ix_(ip.gamma, ip.db)][0]) - param_covs["gamma_alpha"] = (("x",), p_cov[np.ix_(ip.alpha, ip.gamma)][:, 0]) - param_covs["df_db"] = ( - ("time",), - p_cov[ip.df, ip.db], - ) - param_covs["alpha_df"] = ( - ( - "x", - "time", - ), - p_cov[np.ix_(ip.alpha, ip.df)], - ) - param_covs["alpha_db"] = ( - ( - "x", - "time", - ), - p_cov[np.ix_(ip.alpha, ip.db)], - ) - param_covs["tafw_gamma"] = ( - ( - "x", - "time", - ), - ip.get_taf_values( - pval=p_cov[ip.gamma], - x=self.x.values, - trans_att=self.trans_att.values, - axis="", - ), - ) - param_covs["tabw_gamma"] = ( - ( - "x", - "time", - ), - ip.get_tab_values( - pval=p_cov[ip.gamma], - x=self.x.values, - trans_att=self.trans_att.values, - axis="", - ), - ) - param_covs["tafw_alpha"] = ( - ( - "x", - "time", - ), - ip.get_taf_values( - pval=p_cov[ip.alpha], - x=self.x.values, - trans_att=self.trans_att.values, - axis="x", - ), - ) - param_covs["tabw_alpha"] = ( - ( - "x", - "time", - ), - ip.get_tab_values( - pval=p_cov[ip.alpha], - x=self.x.values, - trans_att=self.trans_att.values, - axis="x", - ), - ) - param_covs["tafw_df"] = ( - ( - "x", - "time", - ), - ip.get_taf_values( - pval=p_cov[ip.df], - x=self.x.values, - trans_att=self.trans_att.values, - axis="time", - ), - ) - param_covs["tafw_db"] = ( - ( - "x", - "time", - ), - ip.get_taf_values( - pval=p_cov[ip.db], - x=self.x.values, - trans_att=self.trans_att.values, - axis="time", - ), - ) - param_covs["tabw_db"] = ( - ( - "x", - "time", - ), - ip.get_tab_values( - pval=p_cov[ip.db], - x=self.x.values, - trans_att=self.trans_att.values, - axis="time", - ), - ) - param_covs["tabw_df"] = ( - ( - "x", - "time", - ), - ip.get_tab_values( - pval=p_cov[ip.df], - x=self.x.values, - trans_att=self.trans_att.values, - axis="time", - ), - ) - # sigma2_tafw_tabw - tmpf = out["tmpf"] + 273.15 tmpb = out["tmpb"] + 273.15 @@ -2597,7 +2478,7 @@ def calibration_double_ended( out.update(params) - for key, da in param_covs.items(): + for key, da in param_covs.data_vars.items(): out[key + "_var"] = da self.update(out) diff --git a/src/dtscalibration/datastore_utils.py b/src/dtscalibration/datastore_utils.py index a4b9f875..e12cba3a 100644 --- a/src/dtscalibration/datastore_utils.py +++ b/src/dtscalibration/datastore_utils.py @@ -528,42 +528,171 @@ def get_netcdf_encoding( return encoding -def get_params_from_pval(ip, p_val, coords): - assert len(p_val) == ip.npar, "Length of p_val is incorrect" - - params = xr.Dataset(coords=coords) - - # save estimates and variances to datastore, skip covariances - params["gamma"] = (tuple(), p_val[ip.gamma].item()) - params["alpha"] = (("x",), p_val[ip.alpha]) - params["df"] = (("time",), p_val[ip.df]) - params["db"] = (("time",), p_val[ip.db]) - - if ip.nta: - params["talpha_fw"] = ( - ("time", "trans_att"), - p_val[ip.taf].reshape((ip.nt, ip.nta), order="C"), +def get_params_from_pval(ip, coords, p_val=None, p_cov=None): + if p_val is not None: + assert len(p_val) == ip.npar, "Length of p_val is incorrect" + + params = xr.Dataset(coords=coords) + + # save estimates and variances to datastore, skip covariances + params["gamma"] = (tuple(), p_val[ip.gamma].item()) + params["alpha"] = (("x",), p_val[ip.alpha]) + params["df"] = (("time",), p_val[ip.df]) + params["db"] = (("time",), p_val[ip.db]) + + if ip.nta: + params["talpha_fw"] = ( + ("time", "trans_att"), + p_val[ip.taf].reshape((ip.nt, ip.nta), order="C"), + ) + params["talpha_bw"] = ( + ("time", "trans_att"), + p_val[ip.tab].reshape((ip.nt, ip.nta), order="C"), + ) + else: + params["talpha_fw"] = (("time", "trans_att"), np.zeros((ip.nt, 0))) + params["talpha_bw"] = (("time", "trans_att"), np.zeros((ip.nt, 0))) + + params["talpha_fw_full"] = ( + ("x", "time"), + ip.get_taf_values( + pval=p_val, + x=params.x.values, + trans_att=params.trans_att.values, + axis="", + ), ) - params["talpha_bw"] = ( - ("time", "trans_att"), - p_val[ip.tab].reshape((ip.nt, ip.nta), order="C"), + params["talpha_bw_full"] = ( + ("x", "time"), + ip.get_tab_values( + pval=p_val, + x=params.x.values, + trans_att=params.trans_att.values, + axis="", + ), ) - else: - params["talpha_fw"] = (("time", "trans_att"), np.zeros((ip.nt, 0))) - params["talpha_bw"] = (("time", "trans_att"), np.zeros((ip.nt, 0))) - - params["talpha_fw_full"] = ( - ("x", "time"), - ip.get_taf_values( - pval=p_val, x=params.x.values, trans_att=params.trans_att.values, axis="" - ), - ) - params["talpha_bw_full"] = ( - ("x", "time"), - ip.get_tab_values( - pval=p_val, x=params.x.values, trans_att=params.trans_att.values, axis="" - ), - ) + if p_cov is not None: + assert p_cov.shape == (ip.npar, ip.npar), "Shape of p_cov is incorrect" + + # extract covariances and ensure broadcastable to (nx, nt) + params["gamma_df"] = (("time",), p_cov[np.ix_(ip.gamma, ip.df)][0]) + params["gamma_db"] = (("time",), p_cov[np.ix_(ip.gamma, ip.db)][0]) + params["gamma_alpha"] = (("x",), p_cov[np.ix_(ip.alpha, ip.gamma)][:, 0]) + params["df_db"] = ( + ("time",), + p_cov[ip.df, ip.db], + ) + params["alpha_df"] = ( + ( + "x", + "time", + ), + p_cov[np.ix_(ip.alpha, ip.df)], + ) + params["alpha_db"] = ( + ( + "x", + "time", + ), + p_cov[np.ix_(ip.alpha, ip.db)], + ) + params["tafw_gamma"] = ( + ( + "x", + "time", + ), + ip.get_taf_values( + pval=p_cov[ip.gamma], + x=params["x"].values, + trans_att=params["trans_att"].values, + axis="", + ), + ) + params["tabw_gamma"] = ( + ( + "x", + "time", + ), + ip.get_tab_values( + pval=p_cov[ip.gamma], + x=params["x"].values, + trans_att=params["trans_att"].values, + axis="", + ), + ) + params["tafw_alpha"] = ( + ( + "x", + "time", + ), + ip.get_taf_values( + pval=p_cov[ip.alpha], + x=params["x"].values, + trans_att=params["trans_att"].values, + axis="x", + ), + ) + params["tabw_alpha"] = ( + ( + "x", + "time", + ), + ip.get_tab_values( + pval=p_cov[ip.alpha], + x=params["x"].values, + trans_att=params["trans_att"].values, + axis="x", + ), + ) + params["tafw_df"] = ( + ( + "x", + "time", + ), + ip.get_taf_values( + pval=p_cov[ip.df], + x=params["x"].values, + trans_att=params["trans_att"].values, + axis="time", + ), + ) + params["tafw_db"] = ( + ( + "x", + "time", + ), + ip.get_taf_values( + pval=p_cov[ip.db], + x=params["x"].values, + trans_att=params["trans_att"].values, + axis="time", + ), + ) + params["tabw_db"] = ( + ( + "x", + "time", + ), + ip.get_tab_values( + pval=p_cov[ip.db], + x=params["x"].values, + trans_att=params["trans_att"].values, + axis="time", + ), + ) + params["tabw_df"] = ( + ( + "x", + "time", + ), + ip.get_tab_values( + pval=p_cov[ip.df], + x=params["x"].values, + trans_att=params["trans_att"].values, + axis="time", + ), + ) + # sigma2_tafw_tabw return params From a4a1675edb4abb9580a4fa66f2ed4f8dd4ad1cda Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Fri, 18 Aug 2023 19:38:55 +0200 Subject: [PATCH 10/66] Parallel testing --- pyproject.toml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ccb756f8..a3387d12 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,8 +56,8 @@ dependencies = [ "dask", "pandas", "xarray[parallel]", # numbagg (llvmlite) is a pain to install with pip - "bottleneck", # speed up Xarray - "flox", # speed up Xarray + "bottleneck", # optional, speed up Xarray + "flox", # optional, speed up Xarray "pyyaml>=6.0.1", "xmltodict", "scipy", @@ -74,13 +74,14 @@ dev = [ "bump2version", "ruff", "isort", - "black[jupyter]", - "mypy", + "black[jupyter]", # for formatting + "mypy", # for type checking "types-PyYAML", # for pyyaml types "types-xmltodict", # for xmltodict types "pandas-stubs", # for pandas types "pytest", - "pytest-cov", + "pytest-cov", # for coverage + "pytest-xdist", # for parallel testing "jupyter", "nbformat", # Needed to run the tests ] @@ -112,8 +113,8 @@ lint = [ "mypy src/", ] format = ["black .", "isort .", "lint",] -test = ["pytest ./src/ ./tests/",] # --doctest-modules -fast-test = ["pytest ./tests/ -m \"not slow\"",] +test = ["pytest -n auto ./src/ ./tests/",] +fast-test = ["pytest -n auto ./tests/ -m \"not slow\"",] coverage = [ "pytest --cov --cov-report term --cov-report xml --junitxml=xunit-result.xml tests/", ] @@ -149,7 +150,7 @@ select = [ # It would be nice to have the commented out checks working. # "D", # pydocstyle # "C90", # mccabe complexity # "N", # PEP8-naming - # "UP", # pyupgrade (upgrade syntax to current syntax) + "UP", # pyupgrade (upgrade syntax to current syntax) "PLE", # Pylint error https://github.com/charliermarsh/ruff#error-ple # "PLR", # Pylint refactor (e.g. too-many-arguments) # "PLW", # Pylint warning (useless-else-on-loop) From f363413ba6479c2a59db8f9bb58b89f88d1d08e0 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Fri, 18 Aug 2023 21:18:04 +0200 Subject: [PATCH 11/66] refactored variance functions --- pyproject.toml | 6 +- src/dtscalibration/__init__.py | 1 - src/dtscalibration/calibrate_utils.py | 1 - src/dtscalibration/datastore.py | 181 +++---------------------- src/dtscalibration/variance_helpers.py | 169 +++++++++++++++++++++++ 5 files changed, 193 insertions(+), 165 deletions(-) create mode 100644 src/dtscalibration/variance_helpers.py diff --git a/pyproject.toml b/pyproject.toml index a3387d12..959c779e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -113,8 +113,8 @@ lint = [ "mypy src/", ] format = ["black .", "isort .", "lint",] -test = ["pytest -n auto ./src/ ./tests/",] -fast-test = ["pytest -n auto ./tests/ -m \"not slow\"",] +test = ["pytest -n auto --dist worksteal ./src/ ./tests/",] +fast-test = ["pytest -n auto --dist worksteal ./tests/ -m \"not slow\"",] coverage = [ "pytest --cov --cov-report term --cov-report xml --junitxml=xunit-result.xml tests/", ] @@ -150,7 +150,7 @@ select = [ # It would be nice to have the commented out checks working. # "D", # pydocstyle # "C90", # mccabe complexity # "N", # PEP8-naming - "UP", # pyupgrade (upgrade syntax to current syntax) + # "UP", # pyupgrade (upgrade syntax to current syntax) "PLE", # Pylint error https://github.com/charliermarsh/ruff#error-ple # "PLR", # Pylint refactor (e.g. too-many-arguments) # "PLW", # Pylint warning (useless-else-on-loop) diff --git a/src/dtscalibration/__init__.py b/src/dtscalibration/__init__.py index 3836a779..4ecabb02 100644 --- a/src/dtscalibration/__init__.py +++ b/src/dtscalibration/__init__.py @@ -1,4 +1,3 @@ -# coding=utf-8 from dtscalibration.datastore import DataStore from dtscalibration.datastore_utils import check_dims from dtscalibration.datastore_utils import check_timestep_allclose diff --git a/src/dtscalibration/calibrate_utils.py b/src/dtscalibration/calibrate_utils.py index fd0117b0..960fa220 100644 --- a/src/dtscalibration/calibrate_utils.py +++ b/src/dtscalibration/calibrate_utils.py @@ -1,4 +1,3 @@ -# coding=utf-8 import numpy as np import scipy.sparse as sp import statsmodels.api as sm diff --git a/src/dtscalibration/datastore.py b/src/dtscalibration/datastore.py index b5af307e..6bb774f8 100644 --- a/src/dtscalibration/datastore.py +++ b/src/dtscalibration/datastore.py @@ -4,12 +4,9 @@ import dask import dask.array as da import numpy as np -import scipy.sparse as sp import scipy.stats as sst import xarray as xr import yaml -from scipy.optimize import minimize -from scipy.sparse import linalg as ln from dtscalibration.calibrate_utils import calibration_double_ended_helper from dtscalibration.calibrate_utils import calibration_single_ended_helper @@ -24,6 +21,9 @@ from dtscalibration.datastore_utils import get_params_from_pval from dtscalibration.datastore_utils import ufunc_per_section_helper from dtscalibration.io_utils import _dim_attrs +from dtscalibration.variance_helpers import variance_stokes_constant_helper +from dtscalibration.variance_helpers import variance_stokes_exponential_helper +from dtscalibration.variance_helpers import variance_stokes_linear_helper dtsattr_namelist = ["double_ended_flag"] dim_attrs = {k: v for kl, v in _dim_attrs.items() for k in kl} @@ -879,21 +879,12 @@ def variance_stokes_constant(self, st_label, sections=None, reshape_residuals=Tr dtscalibration/python-dts-calibration/blob/main/examples/notebooks/\ 04Calculate_variance_Stokes.ipynb>`_ """ - - def func_fit(p, xs): - return p[:xs, None] * p[None, xs:] - - def func_cost(p, data, xs): - fit = func_fit(p, xs) - return np.sum((fit - data) ** 2) - if sections is None: sections = self.sections else: sections = validate_sections(self, sections) - assert self[st_label].dims[0] == "x", "Stokes are transposed" - + assert self[st_label].dims[0] == "x", f"{st_label} are transposed" check_timestep_allclose(self, eps=0.01) # should maybe be per section. But then residuals @@ -904,25 +895,7 @@ def func_cost(p, data, xs): ) )[0] - resid_list = [] - - for k, v in data_dict.items(): - for vi in v: - nxs, nt = vi.shape - npar = nt + nxs - - p1 = np.ones(npar) * vi.mean() ** 0.5 - - res = minimize(func_cost, p1, args=(vi, nxs), method="Powell") - assert res.success, "Unable to fit. Try variance_stokes_exponential" - - fit = func_fit(res.x, nxs) - resid_list.append(fit - vi) - - resid = np.concatenate(resid_list) - - # unbiased estimater ddof=1, originally thought it was npar - var_I = resid.var(ddof=1) + var_I, resid = variance_stokes_constant_helper(data_dict) if not reshape_residuals: return var_I, resid @@ -1096,90 +1069,13 @@ def variance_stokes_exponential( x_list.append(da.tile(_x, nt)) len_stretch_list.append(_x.size) - n_sections = len(len_stretch_list) # number of sections - n_locs = sum(len_stretch_list) # total number of locations along cable used - # for reference. - x = np.concatenate(x_list) # coordinates are already in memory y = np.concatenate(y_list) - data1 = x - data2 = np.ones(sum(len_stretch_list) * nt) - data = np.concatenate([data1, data2]) - - # alpha is NOT the same for all -> one column per section - coords1row = np.arange(nt * n_locs) - coords1col = np.hstack( - [np.ones(in_locs * nt) * i for i, in_locs in enumerate(len_stretch_list)] - ) # C for - - # second calibration parameter is different per section and per timestep - coords2row = np.arange(nt * n_locs) - coords2col = np.hstack( - [ - np.repeat( - np.arange(i * nt + n_sections, (i + 1) * nt + n_sections), in_locs - ) - for i, in_locs in enumerate(len_stretch_list) - ] - ) # C for - coords = ( - np.concatenate([coords1row, coords2row]), - np.concatenate([coords1col, coords2col]), + var_I, resid = variance_stokes_exponential_helper( + nt, x, y, len_stretch_list, use_statsmodels, suppress_info ) - lny = np.log(y) - w = y.copy() # 1/std. - - ddof = n_sections + nt * n_sections # see numpy documentation on ddof - - if use_statsmodels: - # returns the same answer with statsmodel - import statsmodels.api as sm - - X = sp.coo_matrix( - (data, coords), shape=(nt * n_locs, ddof), dtype=float, copy=False - ) - - mod_wls = sm.WLS(lny, X.toarray(), weights=w**2) - res_wls = mod_wls.fit() - # print(res_wls.summary()) - a = res_wls.params - - else: - wdata = data * np.hstack((w, w)) - wX = sp.coo_matrix( - (wdata, coords), - shape=(nt * n_locs, n_sections + nt * n_sections), - dtype=float, - copy=False, - ) - - wlny = lny * w - - p0_est = np.asarray(n_sections * [0.0] + nt * n_sections * [8]) - # noinspection PyTypeChecker - a = ln.lsqr(wX, wlny, x0=p0_est, show=not suppress_info, calc_var=False)[0] - - beta = a[:n_sections] - beta_expand_to_sec = np.hstack( - [ - np.repeat(float(beta[i]), leni * nt) - for i, leni in enumerate(len_stretch_list) - ] - ) - G = np.asarray(a[n_sections:]) - G_expand_to_sec = np.hstack( - [ - np.repeat(G[i * nt : (i + 1) * nt], leni) - for i, leni in enumerate(len_stretch_list) - ] - ) - - I_est = np.exp(G_expand_to_sec) * np.exp(x * beta_expand_to_sec) - resid = I_est - y - var_I = resid.var(ddof=1) - if not reshape_residuals: return var_I, resid @@ -1336,60 +1232,25 @@ def variance_stokes_linear( sections = validate_sections(self, sections) assert self[st_label].dims[0] == "x", "Stokes are transposed" - _, resid = self.variance_stokes(sections=sections, st_label=st_label) + _, resid = self.variance_stokes( + sections=sections, st_label=st_label, reshape_residuals=False + ) ix_sec = self.ufunc_per_section( sections=sections, x_indices=True, calc_per="all" ) - st = self.isel(x=ix_sec)[st_label].values.ravel() - diff_st = resid.isel(x=ix_sec).values.ravel() - - # Adjust nbin silently to fit residuals in - # rectangular matrix and use numpy for computation - nbin_ = nbin - while st.size % nbin_: - nbin_ -= 1 - - if nbin_ != nbin: - print( - "Estimation of linear variance of", - st_label, - "Adjusting nbin to:", - nbin_, - ) - nbin = nbin_ - - isort = np.argsort(st) - st_sort_mean = st[isort].reshape((nbin, -1)).mean(axis=1) - st_sort_var = diff_st[isort].reshape((nbin, -1)).var(axis=1) - - if through_zero: - # VAR(Stokes) = slope * Stokes - offset = 0.0 - slope = np.linalg.lstsq(st_sort_mean[:, None], st_sort_var, rcond=None)[0] - - else: - # VAR(Stokes) = slope * Stokes + offset - slope, offset = np.linalg.lstsq( - np.hstack((st_sort_mean[:, None], np.ones((nbin, 1)))), - st_sort_var, - rcond=None, - )[0] - - if offset < 0: - warnings.warn( - f"Warning! Offset of variance_stokes_linear() " - f"of {st_label} is negative. This is phisically " - f"not possible. Most likely, your {st_label} do " - f"not vary enough to fit a linear curve. Either " - f"use `through_zero` option or use " - f"`ds.variance_stokes_constant()`. Another reason " - f"could be that your sections are defined to be " - f"wider than they actually are." - ) - def var_fun(stokes): - return slope * stokes + offset + st = self[st_label].isel(x=ix_sec).values.ravel() + diff_st = resid.ravel() + + ( + slope, + offset, + st_sort_mean, + st_sort_var, + resid, + var_fun, + ) = variance_stokes_linear_helper(st, diff_st, nbin, through_zero) if plot_fit: plt.figure() diff --git a/src/dtscalibration/variance_helpers.py b/src/dtscalibration/variance_helpers.py new file mode 100644 index 00000000..e1b930c3 --- /dev/null +++ b/src/dtscalibration/variance_helpers.py @@ -0,0 +1,169 @@ +import warnings + +import numpy as np +import scipy.sparse as sp +from scipy.optimize import minimize +from scipy.sparse import linalg as ln + + +def variance_stokes_constant_helper(data_dict): + def func_fit(p, xs): + return p[:xs, None] * p[None, xs:] + + def func_cost(p, data, xs): + fit = func_fit(p, xs) + return np.sum((fit - data) ** 2) + + resid_list = [] + + for k, v in data_dict.items(): + for vi in v: + nxs, nt = vi.shape + npar = nt + nxs + + p1 = np.ones(npar) * vi.mean() ** 0.5 + + res = minimize(func_cost, p1, args=(vi, nxs), method="Powell") + assert res.success, "Unable to fit. Try variance_stokes_exponential" + + fit = func_fit(res.x, nxs) + resid_list.append(fit - vi) + + resid = np.concatenate(resid_list) + + # unbiased estimater ddof=1, originally thought it was npar + var_I = resid.var(ddof=1) + + return var_I, resid + + +def variance_stokes_exponential_helper( + nt, x, y, len_stretch_list, use_statsmodels, suppress_info +): + n_sections = len(len_stretch_list) # number of sections + n_locs = sum(len_stretch_list) # total number of locations along cable used + # for reference. + + data1 = x + data2 = np.ones(sum(len_stretch_list) * nt) + data = np.concatenate([data1, data2]) + + # alpha is NOT the same for all -> one column per section + coords1row = np.arange(nt * n_locs) + coords1col = np.hstack( + [np.ones(in_locs * nt) * i for i, in_locs in enumerate(len_stretch_list)] + ) # C for + + # second calibration parameter is different per section and per timestep + coords2row = np.arange(nt * n_locs) + coords2col = np.hstack( + [ + np.repeat( + np.arange(i * nt + n_sections, (i + 1) * nt + n_sections), in_locs + ) + for i, in_locs in enumerate(len_stretch_list) + ] + ) # C for + coords = ( + np.concatenate([coords1row, coords2row]), + np.concatenate([coords1col, coords2col]), + ) + + lny = np.log(y) + w = y.copy() # 1/std. + + ddof = n_sections + nt * n_sections # see numpy documentation on ddof + + if use_statsmodels: + # returns the same answer with statsmodel + import statsmodels.api as sm + + X = sp.coo_matrix( + (data, coords), shape=(nt * n_locs, ddof), dtype=float, copy=False + ) + + mod_wls = sm.WLS(lny, X.toarray(), weights=w**2) + res_wls = mod_wls.fit() + # print(res_wls.summary()) + a = res_wls.params + + else: + wdata = data * np.hstack((w, w)) + wX = sp.coo_matrix( + (wdata, coords), + shape=(nt * n_locs, n_sections + nt * n_sections), + dtype=float, + copy=False, + ) + + wlny = lny * w + + p0_est = np.asarray(n_sections * [0.0] + nt * n_sections * [8]) + # noinspection PyTypeChecker + a = ln.lsqr(wX, wlny, x0=p0_est, show=not suppress_info, calc_var=False)[0] + + beta = a[:n_sections] + beta_expand_to_sec = np.hstack( + [ + np.repeat(float(beta[i]), leni * nt) + for i, leni in enumerate(len_stretch_list) + ] + ) + G = np.asarray(a[n_sections:]) + G_expand_to_sec = np.hstack( + [ + np.repeat(G[i * nt : (i + 1) * nt], leni) + for i, leni in enumerate(len_stretch_list) + ] + ) + + I_est = np.exp(G_expand_to_sec) * np.exp(x * beta_expand_to_sec) + resid = I_est - y + var_I = resid.var(ddof=1) + return var_I, resid + + +def variance_stokes_linear_helper(st_sec, resid_sec, nbin, through_zero): + # Adjust nbin silently to fit residuals in + # rectangular matrix and use numpy for computation + nbin_ = nbin + while st_sec.size % nbin_: + nbin_ -= 1 + + if nbin_ != nbin: + print(f"Adjusting nbin to: {nbin_} to fit residuals in ") + nbin = nbin_ + + isort = np.argsort(st_sec) + st_sort_mean = st_sec[isort].reshape((nbin, -1)).mean(axis=1) + st_sort_var = resid_sec[isort].reshape((nbin, -1)).var(axis=1) + + if through_zero: + # VAR(Stokes) = slope * Stokes + offset = 0.0 + slope = np.linalg.lstsq(st_sort_mean[:, None], st_sort_var, rcond=None)[0] + + else: + # VAR(Stokes) = slope * Stokes + offset + slope, offset = np.linalg.lstsq( + np.hstack((st_sort_mean[:, None], np.ones((nbin, 1)))), + st_sort_var, + rcond=None, + )[0] + + if offset < 0: + warnings.warn( + "Warning! Offset of variance_stokes_linear() " + "is negative. This is phisically " + "not possible. Most likely, your Stokes intensities do " + "not vary enough to fit a linear curve. Either " + "use `through_zero` option or use " + "`ds.variance_stokes_constant()`. Another reason " + "could be that your sections are defined to be " + "wider than they actually are." + ) + + def var_fun(stokes): + return slope * stokes + offset + + return slope, offset, st_sort_mean, st_sort_var, resid_sec, var_fun From 7f795b92698fa75ae8dedb0c7eeb2657207b0a87 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Fri, 18 Aug 2023 21:26:46 +0200 Subject: [PATCH 12/66] Include pyupgrade flag of ruff --- pyproject.toml | 4 ++-- src/dtscalibration/calibration/section_utils.py | 6 ++---- src/dtscalibration/datastore.py | 10 +++++----- src/dtscalibration/datastore_utils.py | 1 - src/dtscalibration/io.py | 6 +++--- src/dtscalibration/io_utils.py | 11 +++++------ src/dtscalibration/plot.py | 7 +++---- tests/test_datastore.py | 1 - tests/test_dtscalibration.py | 1 - 9 files changed, 20 insertions(+), 27 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 959c779e..3597d280 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -150,7 +150,7 @@ select = [ # It would be nice to have the commented out checks working. # "D", # pydocstyle # "C90", # mccabe complexity # "N", # PEP8-naming - # "UP", # pyupgrade (upgrade syntax to current syntax) + "UP", # pyupgrade (upgrade syntax to current syntax) "PLE", # Pylint error https://github.com/charliermarsh/ruff#error-ple # "PLR", # Pylint refactor (e.g. too-many-arguments) # "PLW", # Pylint warning (useless-else-on-loop) @@ -166,7 +166,7 @@ ignore = [ "E501", # Line too long (want to have fixed ] # Allow autofix for all enabled rules (when `--fix`) is provided. -fixable = ["E", "F"] +fixable = ["E", "F", "UP", "PLE"] unfixable = [] line-length = 88 exclude = ["docs", "build"] diff --git a/src/dtscalibration/calibration/section_utils.py b/src/dtscalibration/calibration/section_utils.py index 63dd1e5a..648642be 100644 --- a/src/dtscalibration/calibration/section_utils.py +++ b/src/dtscalibration/calibration/section_utils.py @@ -1,5 +1,3 @@ -from typing import Dict -from typing import List import numpy as np import xarray as xr @@ -8,7 +6,7 @@ from dtscalibration.datastore_utils import ufunc_per_section_helper -def set_sections(ds: xr.Dataset, sections: Dict[str, List[slice]]) -> xr.Dataset: +def set_sections(ds: xr.Dataset, sections: dict[str, list[slice]]) -> xr.Dataset: sections_validated = None if sections is not None: @@ -18,7 +16,7 @@ def set_sections(ds: xr.Dataset, sections: Dict[str, List[slice]]) -> xr.Dataset return ds -def validate_sections(ds: xr.Dataset, sections: Dict[str, List[slice]]): +def validate_sections(ds: xr.Dataset, sections: dict[str, list[slice]]): assert isinstance(sections, dict) # be less restrictive for capitalized labels diff --git a/src/dtscalibration/datastore.py b/src/dtscalibration/datastore.py index 6bb774f8..64f7bd35 100644 --- a/src/dtscalibration/datastore.py +++ b/src/dtscalibration/datastore.py @@ -144,11 +144,11 @@ def __repr__(self): unit = "" for k, v in self.sections.items(): - preamble_new += " {0: <23}".format(k) + preamble_new += f" {k: <23}" # Compute statistics reference section timeseries - sec_stat = "({0:6.2f}".format(float(self[k].mean())) - sec_stat += " +/-{0:5.2f}".format(float(self[k].std())) + sec_stat = f"({float(self[k].mean()):6.2f}" + sec_stat += f" +/-{float(self[k].std()):5.2f}" sec_stat += "\N{DEGREE SIGN}C)\t" preamble_new += sec_stat @@ -425,7 +425,7 @@ def to_netcdf( if value is None: self.attrs[attribute] = "" - return super(DataStore, self).to_netcdf( + return super().to_netcdf( path, mode, format=format, @@ -552,7 +552,7 @@ def to_mf_netcdf( paths = [ os.path.join( folder_path, - filename_preamble + "{:04d}".format(ix) + filename_extension, + filename_preamble + f"{ix:04d}" + filename_extension, ) for ix in range(len(x)) ] diff --git a/src/dtscalibration/datastore_utils.py b/src/dtscalibration/datastore_utils.py index e12cba3a..339a45a7 100644 --- a/src/dtscalibration/datastore_utils.py +++ b/src/dtscalibration/datastore_utils.py @@ -1,4 +1,3 @@ -# coding=utf-8 from typing import TYPE_CHECKING from typing import Optional from typing import Union diff --git a/src/dtscalibration/io.py b/src/dtscalibration/io.py index be7d0dd6..edbeac63 100644 --- a/src/dtscalibration/io.py +++ b/src/dtscalibration/io.py @@ -282,7 +282,7 @@ def read_silixa_files( else: raise NotImplementedError( - "Silixa xml version " + "{0} not implemented".format(xml_version) + "Silixa xml version " + f"{xml_version} not implemented" ) ds = DataStore(data_vars=data_vars, coords=coords, attrs=attrs, **kwargs) @@ -332,7 +332,7 @@ def read_sensortran_files( # Check if corresponding temperature file exists if not os.path.isfile(filepathlist_temp[ii]): raise FileNotFoundError( - "Could not find BinaryTemp " + "file corresponding to {}".format(fname) + "Could not find BinaryTemp " + f"file corresponding to {fname}" ) version = sensortran_binary_version_check(filepathlist_dts) @@ -347,7 +347,7 @@ def read_sensortran_files( ) else: raise NotImplementedError( - "Sensortran binary version " + "{0} not implemented".format(version) + "Sensortran binary version " + f"{version} not implemented" ) ds = DataStore(data_vars=data_vars, coords=coords, attrs=attrs, **kwargs) diff --git a/src/dtscalibration/io_utils.py b/src/dtscalibration/io_utils.py index e880a6ca..4b480742 100644 --- a/src/dtscalibration/io_utils.py +++ b/src/dtscalibration/io_utils.py @@ -1,4 +1,3 @@ -# coding=utf-8 import os import struct from contextlib import contextmanager @@ -1263,8 +1262,8 @@ def read_sensortran_files_routine( AST = np.zeros((nx, ntime), dtype=np.int32) TMP = np.zeros((nx, ntime)) - ST_zero = np.zeros((ntime)) - AST_zero = np.zeros((ntime)) + ST_zero = np.zeros(ntime) + AST_zero = np.zeros(ntime) for ii in range(ntime): data_dts, meta_dts = read_sensortran_single(filepathlist_dts[ii]) @@ -1468,7 +1467,7 @@ def read_apsensing_files_routine( + "/{0}wellLogSet/{0}wellLog" ).format(namespace) ) - logdata_tree = logtree.find("./{0}logData".format(namespace)) + logdata_tree = logtree.find(f"./{namespace}logData") # Amount of datapoints is the size of the logdata tree nx = len(logdata_tree) @@ -1490,7 +1489,7 @@ def read_apsensing_files_routine( attrs["backwardMeasurementChannel"] = "N/A" data_item_names = [ - attrs["wellbore:wellLogSet:wellLog:logCurveInfo_{0}:mnemonic".format(x)] + attrs[f"wellbore:wellLogSet:wellLog:logCurveInfo_{x}:mnemonic"] for x in range(0, 4) ] @@ -1591,7 +1590,7 @@ def grab_data_per_file(file_handle): data_vars[tld[name]] = (["x", "time"], data_arri, dim_attrs_apsensing[name]) else: raise ValueError( - "Dont know what to do with the" + " {} data column".format(name) + "Dont know what to do with the" + f" {name} data column" ) _time_dtype = [("filename_tstamp", np.int64), ("acquisitionTime", " Date: Sun, 20 Aug 2023 18:07:33 +0200 Subject: [PATCH 13/66] Refactored single ended routine p_val to params and moved averaging routines --- src/dtscalibration/averaging_utils.py | 75 +++++++++++ src/dtscalibration/datastore.py | 173 +++----------------------- src/dtscalibration/datastore_utils.py | 76 ++++++++++- 3 files changed, 169 insertions(+), 155 deletions(-) create mode 100644 src/dtscalibration/averaging_utils.py diff --git a/src/dtscalibration/averaging_utils.py b/src/dtscalibration/averaging_utils.py new file mode 100644 index 00000000..9169c36e --- /dev/null +++ b/src/dtscalibration/averaging_utils.py @@ -0,0 +1,75 @@ + + +def inverse_variance_weighted_mean( + self, + tmp1="tmpf", + tmp2="tmpb", + tmp1_var="tmpf_mc_var", + tmp2_var="tmpb_mc_var", + tmpw_store="tmpw", + tmpw_var_store="tmpw_var", +): + """ + Average two temperature datasets with the inverse of the variance as + weights. The two + temperature datasets `tmp1` and `tmp2` with their variances + `tmp1_var` and `tmp2_var`, + respectively. Are averaged and stored in the DataStore. + + Parameters + ---------- + tmp1 : str + The label of the first temperature dataset that is averaged + tmp2 : str + The label of the second temperature dataset that is averaged + tmp1_var : str + The variance of tmp1 + tmp2_var : str + The variance of tmp2 + tmpw_store : str + The label of the averaged temperature dataset + tmpw_var_store : str + The label of the variance of the averaged temperature dataset + + Returns + ------- + + """ + + self[tmpw_var_store] = 1 / (1 / self[tmp1_var] + 1 / self[tmp2_var]) + + self[tmpw_store] = ( + self[tmp1] / self[tmp1_var] + self[tmp2] / self[tmp2_var] + ) * self[tmpw_var_store] + + pass + +def inverse_variance_weighted_mean_array( + self, + tmp_label="tmpf", + tmp_var_label="tmpf_mc_var", + tmpw_store="tmpw", + tmpw_var_store="tmpw_var", + dim="time", +): + """ + Calculates the weighted average across a dimension. + + Parameters + ---------- + + Returns + ------- + + See Also + -------- + - https://en.wikipedia.org/wiki/Inverse-variance_weighting + + """ + self[tmpw_var_store] = 1 / (1 / self[tmp_var_label]).sum(dim=dim) + + self[tmpw_store] = (self[tmp_label] / self[tmp_var_label]).sum(dim=dim) / ( + 1 / self[tmp_var_label] + ).sum(dim=dim) + + pass \ No newline at end of file diff --git a/src/dtscalibration/datastore.py b/src/dtscalibration/datastore.py index 64f7bd35..3b6e9b49 100644 --- a/src/dtscalibration/datastore.py +++ b/src/dtscalibration/datastore.py @@ -18,7 +18,8 @@ from dtscalibration.datastore_utils import ParameterIndexSingleEnded from dtscalibration.datastore_utils import check_deprecated_kwargs from dtscalibration.datastore_utils import check_timestep_allclose -from dtscalibration.datastore_utils import get_params_from_pval +from dtscalibration.datastore_utils import get_params_from_pval_double_ended +from dtscalibration.datastore_utils import get_params_from_pval_single_ended from dtscalibration.datastore_utils import ufunc_per_section_helper from dtscalibration.io_utils import _dim_attrs from dtscalibration.variance_helpers import variance_stokes_constant_helper @@ -1331,80 +1332,6 @@ def i_var(self, st_var, ast_var, st_label="st", ast_label="ast"): return st**-2 * st_var + ast**-2 * ast_var - def inverse_variance_weighted_mean( - self, - tmp1="tmpf", - tmp2="tmpb", - tmp1_var="tmpf_mc_var", - tmp2_var="tmpb_mc_var", - tmpw_store="tmpw", - tmpw_var_store="tmpw_var", - ): - """ - Average two temperature datasets with the inverse of the variance as - weights. The two - temperature datasets `tmp1` and `tmp2` with their variances - `tmp1_var` and `tmp2_var`, - respectively. Are averaged and stored in the DataStore. - - Parameters - ---------- - tmp1 : str - The label of the first temperature dataset that is averaged - tmp2 : str - The label of the second temperature dataset that is averaged - tmp1_var : str - The variance of tmp1 - tmp2_var : str - The variance of tmp2 - tmpw_store : str - The label of the averaged temperature dataset - tmpw_var_store : str - The label of the variance of the averaged temperature dataset - - Returns - ------- - - """ - - self[tmpw_var_store] = 1 / (1 / self[tmp1_var] + 1 / self[tmp2_var]) - - self[tmpw_store] = ( - self[tmp1] / self[tmp1_var] + self[tmp2] / self[tmp2_var] - ) * self[tmpw_var_store] - - pass - - def inverse_variance_weighted_mean_array( - self, - tmp_label="tmpf", - tmp_var_label="tmpf_mc_var", - tmpw_store="tmpw", - tmpw_var_store="tmpw_var", - dim="time", - ): - """ - Calculates the weighted average across a dimension. - - Parameters - ---------- - - Returns - ------- - - See Also - -------- - - https://en.wikipedia.org/wiki/Inverse-variance_weighting - - """ - self[tmpw_var_store] = 1 / (1 / self[tmp_var_label]).sum(dim=dim) - - self[tmpw_store] = (self[tmp_label] / self[tmp_var_label]).sum(dim=dim) / ( - 1 / self[tmp_var_label] - ).sum(dim=dim) - - pass - def set_trans_att(self, trans_att=None): """Gracefully set the locations that introduce directional differential attenuation @@ -1669,52 +1596,12 @@ def calibration_single_ended( assert p_cov.shape == (ip.npar, ip.npar) # store calibration parameters in DataStore - self["gamma"] = (tuple(), p_val[ip.gamma].item()) - self["gamma_var"] = (tuple(), p_var[ip.gamma].item()) - - if nta > 0: - self["talpha_fw"] = ( - ("trans_att", "time"), - ip.get_taf_pars(p_val), - ) - self["talpha_fw_var"] = ( - ("trans_att", "time"), - ip.get_taf_pars(p_var), - ) - else: - self["talpha_fw"] = (("trans_att", "time"), np.zeros((0, nt))) - self["talpha_fw_var"] = (("trans_att", "time"), np.zeros((0, nt))) - - self["c"] = (("time",), p_val[ip.c]) - self["c_var"] = (("time",), p_var[ip.c]) - - if fix_alpha: - self["alpha"] = (("x",), fix_alpha[0]) - self["alpha_var"] = (("x",), fix_alpha[1]) - - else: - self["dalpha"] = (tuple(), p_val[ip.dalpha].item()) - self["dalpha_var"] = (tuple(), p_var[ip.dalpha].item()) - - self["alpha"] = self.dalpha * self.x - self["alpha_var"] = self["dalpha_var"] * self.x**2 - - self["talpha_fw_full"] = ( - ("x", "time"), - ip.get_taf_values( - pval=p_val, x=self.x.values, trans_att=self.trans_att.values, axis="" - ), - ) - self["talpha_fw_full_var"] = ( - ("x", "time"), - ip.get_taf_values( - pval=p_var, x=self.x.values, trans_att=self.trans_att.values, axis="" - ), - ) + coords = {"x": self["x"], "time": self["time"], "trans_att": self["trans_att"]} + params, param_covs = get_params_from_pval_single_ended(ip, coords, p_val=p_val, p_var=p_var, p_cov=p_cov, fix_alpha=fix_alpha) tmpf = self.gamma / ( - (np.log(self.st / self.ast) + (self.c + self["talpha_fw_full"])) - + self.alpha + (np.log(self.st / self.ast) + (params["c"] + params["talpha_fw_full"])) + + params["alpha"] ) self["tmpf"] = tmpf - 273.15 @@ -1729,58 +1616,36 @@ def calibration_single_ended( T_ta_fw=-(tmpf**2) / self.gamma, ) deriv_ds = xr.Dataset(deriv_dict) - # self["deriv"] = deriv_ds.to_array(dim="com2") - - sigma2_gamma_c = p_cov[np.ix_(ip.gamma, ip.c)] - sigma2_tafw_gamma = ip.get_taf_values( - pval=p_cov[ip.gamma], - x=self.x.values, - trans_att=self.trans_att.values, - axis="", - ) - sigma2_tafw_c = ip.get_taf_values( - pval=p_cov[ip.c], - x=self.x.values, - trans_att=self.trans_att.values, - axis="time", - ) + var_fw_dict = dict( dT_dst=deriv_ds.T_st_fw**2 * parse_st_var(self, st_var, st_label="st"), dT_dast=deriv_ds.T_ast_fw**2 * parse_st_var(self, ast_var, st_label="ast"), - dT_gamma=deriv_ds.T_gamma_fw**2 * self["gamma_var"], - dT_dc=deriv_ds.T_c_fw**2 * self["c_var"], + dT_gamma=deriv_ds.T_gamma_fw**2 * param_covs["gamma"], + dT_dc=deriv_ds.T_c_fw**2 * param_covs["c"], dT_ddalpha=deriv_ds.T_alpha_fw**2 - * self["alpha_var"], # same as dT_dalpha - dT_dta=deriv_ds.T_ta_fw**2 * self["talpha_fw_full_var"], - dgamma_dc=(2 * deriv_ds.T_gamma_fw * deriv_ds.T_c_fw * sigma2_gamma_c), - dta_dgamma=(2 * deriv_ds.T_ta_fw * deriv_ds.T_gamma_fw * sigma2_tafw_gamma), - dta_dc=(2 * deriv_ds.T_ta_fw * deriv_ds.T_c_fw * sigma2_tafw_c), + * param_covs["alpha"], # same as dT_dalpha + dT_dta=deriv_ds.T_ta_fw**2 * param_covs["talpha_fw_full"], + dgamma_dc=(2 * deriv_ds.T_gamma_fw * deriv_ds.T_c_fw * param_covs["gamma_c"]), + dta_dgamma=(2 * deriv_ds.T_ta_fw * deriv_ds.T_gamma_fw * param_covs["tafw_gamma"]), + dta_dc=(2 * deriv_ds.T_ta_fw * deriv_ds.T_c_fw * param_covs["tafw_c"]), ) if not fix_alpha: # These correlations don't exist in case of fix_alpha. Including them reduces tmpf_var. - sigma2_gamma_dalpha = p_cov[np.ix_(ip.dalpha, ip.gamma)] - sigma2_dalpha_c = p_cov[np.ix_(ip.dalpha, ip.c)] - sigma2_tafw_dalpha = ip.get_taf_values( - pval=p_cov[ip.dalpha], - x=self.x.values, - trans_att=self.trans_att.values, - axis="", - ) var_fw_dict.update( dict( dgamma_ddalpha=( 2 * deriv_ds.T_gamma_fw * deriv_ds.T_dalpha_fw - * sigma2_gamma_dalpha + * param_covs["gamma_dalpha"] ), ddalpha_dc=( - 2 * deriv_ds.T_dalpha_fw * deriv_ds.T_c_fw * sigma2_dalpha_c + 2 * deriv_ds.T_dalpha_fw * deriv_ds.T_c_fw * param_covs["dalpha_c"] ), dta_ddalpha=( - 2 * deriv_ds.T_ta_fw * deriv_ds.T_dalpha_fw * sigma2_tafw_dalpha + 2 * deriv_ds.T_ta_fw * deriv_ds.T_dalpha_fw * param_covs["tafw_dalpha"] ), ) ) @@ -2121,8 +1986,8 @@ def calibration_double_ended( assert p_cov.shape == (ip.npar, ip.npar) coords = {"x": self["x"], "time": self["time"], "trans_att": self["trans_att"]} - params = get_params_from_pval(ip, coords, p_val=p_val) - param_covs = get_params_from_pval(ip, coords, p_val=p_var, p_cov=p_cov) + params = get_params_from_pval_double_ended(ip, coords, p_val=p_val) + param_covs = get_params_from_pval_double_ended(ip, coords, p_val=p_var, p_cov=p_cov) out = xr.Dataset( { diff --git a/src/dtscalibration/datastore_utils.py b/src/dtscalibration/datastore_utils.py index 339a45a7..c36c1c9b 100644 --- a/src/dtscalibration/datastore_utils.py +++ b/src/dtscalibration/datastore_utils.py @@ -527,7 +527,7 @@ def get_netcdf_encoding( return encoding -def get_params_from_pval(ip, coords, p_val=None, p_cov=None): +def get_params_from_pval_double_ended(ip, coords, p_val=None, p_cov=None): if p_val is not None: assert len(p_val) == ip.npar, "Length of p_val is incorrect" @@ -693,6 +693,80 @@ def get_params_from_pval(ip, coords, p_val=None, p_cov=None): ) # sigma2_tafw_tabw return params + + +def get_params_from_pval_single_ended(ip, coords, p_val=None, p_var=None, p_cov=None, fix_alpha=None): + assert len(p_val) == ip.npar, "Length of p_val is incorrect" + + params = xr.Dataset(coords=coords) + param_covs = xr.Dataset(coords=coords) + + params["gamma"] = (tuple(), p_val[ip.gamma].item()) + param_covs["gamma"] = (tuple(), p_var[ip.gamma].item()) + + if ip.nta > 0: + params["talpha_fw"] = ( + ("trans_att", "time"), + ip.get_taf_pars(p_val), + ) + param_covs["talpha_fw"] = ( + ("trans_att", "time"), + ip.get_taf_pars(p_var), + ) + else: + params["talpha_fw"] = (("trans_att", "time"), np.zeros((0, ip.nt))) + param_covs["talpha_fw"] = (("trans_att", "time"), np.zeros((0, ip.nt))) + + params["c"] = (("time",), p_val[ip.c]) + param_covs["c"] = (("time",), p_var[ip.c]) + + if fix_alpha is not None: + params["alpha"] = (("x",), fix_alpha[0]) + param_covs["alpha"] = (("x",), fix_alpha[1]) + + else: + params["dalpha"] = (tuple(), p_val[ip.dalpha].item()) + param_covs["dalpha"] = (tuple(), p_var[ip.dalpha].item()) + + params["alpha"] = params["dalpha"] * params["x"] + param_covs["alpha"] = param_covs["dalpha"] * params["x"]**2 + + param_covs["gamma_dalpha"] = p_cov[np.ix_(ip.dalpha, ip.gamma)] + param_covs["dalpha_c"] = p_cov[np.ix_(ip.dalpha, ip.c)] + param_covs["tafw_dalpha"] = ip.get_taf_values( + pval=p_cov[ip.dalpha], + x=params["x"].values, + trans_att=params["trans_att"].values, + axis="", + ) + + params["talpha_fw_full"] = ( + ("x", "time"), + ip.get_taf_values( + pval=p_val, x=params["x"].values, trans_att=params["trans_att"].values, axis="" + ), + ) + param_covs["talpha_fw_full"] = ( + ("x", "time"), + ip.get_taf_values( + pval=p_var, x=params["x"].values, trans_att=params["trans_att"].values, axis="" + ), + ) + + param_covs["gamma_c"] = p_cov[np.ix_(ip.gamma, ip.c)] + param_covs["tafw_gamma"] = ip.get_taf_values( + pval=p_cov[ip.gamma], + x=params["x"].values, + trans_att=params["trans_att"].values, + axis="", + ) + param_covs["tafw_c"] = ip.get_taf_values( + pval=p_cov[ip.c], + x=params["x"].values, + trans_att=params["trans_att"].values, + axis="time", + ) + return params, param_covs def merge_double_ended( From 0c427d805eeb2fc79c94a0a7bf5c609a4fd8d420 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Mon, 21 Aug 2023 09:47:03 +0200 Subject: [PATCH 14/66] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index b8f681da..f06bd184 100644 --- a/.gitignore +++ b/.gitignore @@ -72,3 +72,4 @@ docs/_build .appveyor.token *.bak .vscode/settings.json +*.code-workspace From 43f42c8849807e77a1c68fb081b6cfe406ab3af7 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Tue, 22 Aug 2023 21:18:41 +0200 Subject: [PATCH 15/66] Refactored calibration routines In preparation of the accessor --- pyproject.toml | 2 +- src/dtscalibration/averaging_utils.py | 5 +- .../calibration/section_utils.py | 1 - src/dtscalibration/datastore.py | 67 ++++++++++++------- src/dtscalibration/datastore_utils.py | 64 +++++++++++------- src/dtscalibration/io.py | 14 ++-- src/dtscalibration/io_utils.py | 4 +- 7 files changed, 92 insertions(+), 65 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3597d280..af0bd1fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["hatchling>=1.8.0", "hatch-vcs", "hatch-fancy-pypi-readme"] +requires = ["hatchling>=1.8.0", "hatch-vcs"] build-backend = "hatchling.build" [tool.hatch.version] diff --git a/src/dtscalibration/averaging_utils.py b/src/dtscalibration/averaging_utils.py index 9169c36e..c94194ba 100644 --- a/src/dtscalibration/averaging_utils.py +++ b/src/dtscalibration/averaging_utils.py @@ -1,5 +1,3 @@ - - def inverse_variance_weighted_mean( self, tmp1="tmpf", @@ -44,6 +42,7 @@ def inverse_variance_weighted_mean( pass + def inverse_variance_weighted_mean_array( self, tmp_label="tmpf", @@ -72,4 +71,4 @@ def inverse_variance_weighted_mean_array( 1 / self[tmp_var_label] ).sum(dim=dim) - pass \ No newline at end of file + pass diff --git a/src/dtscalibration/calibration/section_utils.py b/src/dtscalibration/calibration/section_utils.py index 648642be..2d1ef100 100644 --- a/src/dtscalibration/calibration/section_utils.py +++ b/src/dtscalibration/calibration/section_utils.py @@ -1,4 +1,3 @@ - import numpy as np import xarray as xr import yaml diff --git a/src/dtscalibration/datastore.py b/src/dtscalibration/datastore.py index 3b6e9b49..9e1449c2 100644 --- a/src/dtscalibration/datastore.py +++ b/src/dtscalibration/datastore.py @@ -1597,23 +1597,26 @@ def calibration_single_ended( # store calibration parameters in DataStore coords = {"x": self["x"], "time": self["time"], "trans_att": self["trans_att"]} - params, param_covs = get_params_from_pval_single_ended(ip, coords, p_val=p_val, p_var=p_var, p_cov=p_cov, fix_alpha=fix_alpha) + params, param_covs = get_params_from_pval_single_ended( + ip, coords, p_val=p_val, p_var=p_var, p_cov=p_cov, fix_alpha=fix_alpha + ) - tmpf = self.gamma / ( + tmpf = params["gamma"] / ( (np.log(self.st / self.ast) + (params["c"] + params["talpha_fw_full"])) + params["alpha"] ) - self["tmpf"] = tmpf - 273.15 + out = xr.Dataset({"tmpf": tmpf - 273.15}) + out["tmpf"].attrs.update(_dim_attrs[("tmpf",)]) # tmpf_var deriv_dict = dict( - T_gamma_fw=tmpf / self.gamma, - T_st_fw=-(tmpf**2) / (self.gamma * self.st), - T_ast_fw=tmpf**2 / (self.gamma * self.ast), - T_c_fw=-(tmpf**2) / self.gamma, - T_dalpha_fw=-self.x * (tmpf**2) / self.gamma, - T_alpha_fw=-(tmpf**2) / self.gamma, - T_ta_fw=-(tmpf**2) / self.gamma, + T_gamma_fw=tmpf / params["gamma"], + T_st_fw=-(tmpf**2) / (params["gamma"] * self.st), + T_ast_fw=tmpf**2 / (params["gamma"] * self.ast), + T_c_fw=-(tmpf**2) / params["gamma"], + T_dalpha_fw=-self.x * (tmpf**2) / params["gamma"], + T_alpha_fw=-(tmpf**2) / params["gamma"], + T_ta_fw=-(tmpf**2) / params["gamma"], ) deriv_ds = xr.Dataset(deriv_dict) @@ -1626,8 +1629,12 @@ def calibration_single_ended( dT_ddalpha=deriv_ds.T_alpha_fw**2 * param_covs["alpha"], # same as dT_dalpha dT_dta=deriv_ds.T_ta_fw**2 * param_covs["talpha_fw_full"], - dgamma_dc=(2 * deriv_ds.T_gamma_fw * deriv_ds.T_c_fw * param_covs["gamma_c"]), - dta_dgamma=(2 * deriv_ds.T_ta_fw * deriv_ds.T_gamma_fw * param_covs["tafw_gamma"]), + dgamma_dc=( + 2 * deriv_ds.T_gamma_fw * deriv_ds.T_c_fw * param_covs["gamma_c"] + ), + dta_dgamma=( + 2 * deriv_ds.T_ta_fw * deriv_ds.T_gamma_fw * param_covs["tafw_gamma"] + ), dta_dc=(2 * deriv_ds.T_ta_fw * deriv_ds.T_c_fw * param_covs["tafw_c"]), ) @@ -1642,19 +1649,23 @@ def calibration_single_ended( * param_covs["gamma_dalpha"] ), ddalpha_dc=( - 2 * deriv_ds.T_dalpha_fw * deriv_ds.T_c_fw * param_covs["dalpha_c"] + 2 + * deriv_ds.T_dalpha_fw + * deriv_ds.T_c_fw + * param_covs["dalpha_c"] ), dta_ddalpha=( - 2 * deriv_ds.T_ta_fw * deriv_ds.T_dalpha_fw * param_covs["tafw_dalpha"] + 2 + * deriv_ds.T_ta_fw + * deriv_ds.T_dalpha_fw + * param_covs["tafw_dalpha"] ), ) ) - self["var_fw_da"] = xr.Dataset(var_fw_dict).to_array(dim="comp_fw") - self["tmpf_var"] = self["var_fw_da"].sum(dim="comp_fw") - - self["tmpf"].attrs.update(_dim_attrs[("tmpf",)]) - self["tmpf_var"].attrs.update(_dim_attrs[("tmpf_var",)]) + out["var_fw_da"] = xr.Dataset(var_fw_dict).to_array(dim="comp_fw") + out["tmpf_var"] = out["var_fw_da"].sum(dim="comp_fw") + out["tmpf_var"].attrs.update(_dim_attrs[("tmpf_var",)]) drop_vars = [ k for k, v in self.items() if {"params1", "params2"}.intersection(v.dims) @@ -1663,9 +1674,14 @@ def calibration_single_ended( for k in drop_vars: del self[k] - self["p_val"] = (("params1",), p_val) - self["p_cov"] = (("params1", "params2"), p_cov) + out["p_val"] = (("params1",), p_val) + out["p_cov"] = (("params1", "params2"), p_cov) + + out.update(params) + for key, dataarray in param_covs.data_vars.items(): + out[key + "_var"] = dataarray + self.update(out) pass def calibration_double_ended( @@ -1987,7 +2003,9 @@ def calibration_double_ended( coords = {"x": self["x"], "time": self["time"], "trans_att": self["trans_att"]} params = get_params_from_pval_double_ended(ip, coords, p_val=p_val) - param_covs = get_params_from_pval_double_ended(ip, coords, p_val=p_var, p_cov=p_cov) + param_covs = get_params_from_pval_double_ended( + ip, coords, p_val=p_var, p_cov=p_cov + ) out = xr.Dataset( { @@ -2203,9 +2221,8 @@ def calibration_double_ended( out["p_cov"] = (("params1", "params2"), p_cov) out.update(params) - - for key, da in param_covs.data_vars.items(): - out[key + "_var"] = da + for key, dataarray in param_covs.data_vars.items(): + out[key + "_var"] = dataarray self.update(out) pass diff --git a/src/dtscalibration/datastore_utils.py b/src/dtscalibration/datastore_utils.py index c36c1c9b..0d54dd31 100644 --- a/src/dtscalibration/datastore_utils.py +++ b/src/dtscalibration/datastore_utils.py @@ -693,9 +693,11 @@ def get_params_from_pval_double_ended(ip, coords, p_val=None, p_cov=None): ) # sigma2_tafw_tabw return params - -def get_params_from_pval_single_ended(ip, coords, p_val=None, p_var=None, p_cov=None, fix_alpha=None): + +def get_params_from_pval_single_ended( + ip, coords, p_val=None, p_var=None, p_cov=None, fix_alpha=None +): assert len(p_val) == ip.npar, "Length of p_val is incorrect" params = xr.Dataset(coords=coords) @@ -729,42 +731,56 @@ def get_params_from_pval_single_ended(ip, coords, p_val=None, p_var=None, p_cov= param_covs["dalpha"] = (tuple(), p_var[ip.dalpha].item()) params["alpha"] = params["dalpha"] * params["x"] - param_covs["alpha"] = param_covs["dalpha"] * params["x"]**2 + param_covs["alpha"] = param_covs["dalpha"] * params["x"] ** 2 - param_covs["gamma_dalpha"] = p_cov[np.ix_(ip.dalpha, ip.gamma)] - param_covs["dalpha_c"] = p_cov[np.ix_(ip.dalpha, ip.c)] - param_covs["tafw_dalpha"] = ip.get_taf_values( - pval=p_cov[ip.dalpha], - x=params["x"].values, - trans_att=params["trans_att"].values, - axis="", + param_covs["gamma_dalpha"] = (tuple(), p_cov[np.ix_(ip.dalpha, ip.gamma)][0, 0]) + param_covs["dalpha_c"] = (("time",), p_cov[np.ix_(ip.dalpha, ip.c)][0, :]) + param_covs["tafw_dalpha"] = ( + ("x", "time"), + ip.get_taf_values( + pval=p_cov[ip.dalpha], + x=params["x"].values, + trans_att=params["trans_att"].values, + axis="", + ), ) params["talpha_fw_full"] = ( ("x", "time"), ip.get_taf_values( - pval=p_val, x=params["x"].values, trans_att=params["trans_att"].values, axis="" + pval=p_val, + x=params["x"].values, + trans_att=params["trans_att"].values, + axis="", ), ) param_covs["talpha_fw_full"] = ( ("x", "time"), ip.get_taf_values( - pval=p_var, x=params["x"].values, trans_att=params["trans_att"].values, axis="" + pval=p_var, + x=params["x"].values, + trans_att=params["trans_att"].values, + axis="", ), ) - - param_covs["gamma_c"] = p_cov[np.ix_(ip.gamma, ip.c)] - param_covs["tafw_gamma"] = ip.get_taf_values( - pval=p_cov[ip.gamma], - x=params["x"].values, - trans_att=params["trans_att"].values, - axis="", + param_covs["gamma_c"] = (("time",), p_cov[np.ix_(ip.gamma, ip.c)][0, :]) + param_covs["tafw_gamma"] = ( + ("x", "time"), + ip.get_taf_values( + pval=p_cov[ip.gamma], + x=params["x"].values, + trans_att=params["trans_att"].values, + axis="", + ), ) - param_covs["tafw_c"] = ip.get_taf_values( - pval=p_cov[ip.c], - x=params["x"].values, - trans_att=params["trans_att"].values, - axis="time", + param_covs["tafw_c"] = ( + ("x", "time"), + ip.get_taf_values( + pval=p_cov[ip.c], + x=params["x"].values, + trans_att=params["trans_att"].values, + axis="time", + ), ) return params, param_covs diff --git a/src/dtscalibration/io.py b/src/dtscalibration/io.py index edbeac63..413851f5 100644 --- a/src/dtscalibration/io.py +++ b/src/dtscalibration/io.py @@ -423,10 +423,9 @@ def read_apsensing_files( else: warnings.warn( - "AP sensing device " - '"{0}"'.format(device) - + " has not been tested.\nPlease open an issue on github" - + " and provide an example file" + "AP sensing device {device}" + " has not been tested.\nPlease open an issue on github" + " and provide an example file" ) data_vars, coords, attrs = read_apsensing_files_routine( @@ -544,10 +543,9 @@ def read_sensornet_files( else: flip_reverse_measurements = False warnings.warn( - "Sensornet .dff version " - '"{0}"'.format(ddf_version) - + " has not been tested.\nPlease open an issue on github" - + " and provide an example file" + f"Sensornet .dff version {ddf_version}" + " has not been tested.\nPlease open an issue on github" + " and provide an example file" ) data_vars, coords, attrs = read_sensornet_files_routine_v3( diff --git a/src/dtscalibration/io_utils.py b/src/dtscalibration/io_utils.py index 4b480742..ca5dbb3a 100644 --- a/src/dtscalibration/io_utils.py +++ b/src/dtscalibration/io_utils.py @@ -1589,9 +1589,7 @@ def grab_data_per_file(file_handle): elif name in dim_attrs_apsensing: data_vars[tld[name]] = (["x", "time"], data_arri, dim_attrs_apsensing[name]) else: - raise ValueError( - "Dont know what to do with the" + f" {name} data column" - ) + raise ValueError("Dont know what to do with the" + f" {name} data column") _time_dtype = [("filename_tstamp", np.int64), ("acquisitionTime", " Date: Tue, 22 Aug 2023 21:34:43 +0200 Subject: [PATCH 16/66] Please ruff --- src/dtscalibration/datastore.py | 2 +- src/dtscalibration/plot.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/dtscalibration/datastore.py b/src/dtscalibration/datastore.py index 9e1449c2..1882aa64 100644 --- a/src/dtscalibration/datastore.py +++ b/src/dtscalibration/datastore.py @@ -155,7 +155,7 @@ def __repr__(self): # print sections vl = [ - "{0:.2f}{2} - {1:.2f}{2}".format(vi.start, vi.stop, unit) + f"{vi.start:.2f}{unit} - {vi.stop:.2f}{unit}" for vi in v ] preamble_new += " and ".join(vl) + "\n" diff --git a/src/dtscalibration/plot.py b/src/dtscalibration/plot.py index 7e487879..13c0c111 100755 --- a/src/dtscalibration/plot.py +++ b/src/dtscalibration/plot.py @@ -630,8 +630,7 @@ def plot_sigma_report( ) else: ax1.set_title( - "Projected uncertainty at t={} compared to standard error " - "in baths".format(itimes) + f"Projected uncertainty at t={itimes} compared to standard error in baths" ) ax1.legend() ax1.set_ylabel(r"Temperature [$^\circ$C]") From 49bbae3f820a567e546a24da2dbd92188d17eb75 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Sun, 3 Sep 2023 15:49:12 +0200 Subject: [PATCH 17/66] Refactored Datastore class --- src/dtscalibration/datastore.py | 580 +++++++++++++++++--------------- tests/test_dtscalibration.py | 6 +- 2 files changed, 306 insertions(+), 280 deletions(-) diff --git a/src/dtscalibration/datastore.py b/src/dtscalibration/datastore.py index 1882aa64..db555798 100644 --- a/src/dtscalibration/datastore.py +++ b/src/dtscalibration/datastore.py @@ -154,10 +154,7 @@ def __repr__(self): preamble_new += sec_stat # print sections - vl = [ - f"{vi.start:.2f}{unit} - {vi.stop:.2f}{unit}" - for vi in v - ] + vl = [f"{vi.start:.2f}{unit} - {vi.stop:.2f}{unit}" for vi in v] preamble_new += " and ".join(vl) + "\n" else: @@ -2376,7 +2373,9 @@ def average_single_ended( if var_only_sections is not None: raise NotImplementedError() - self.conf_int_single_ended( + out = xr.Dataset() + + mcparams = self.conf_int_single_ended( p_val=p_val, p_cov=p_cov, st_var=st_var, @@ -2388,85 +2387,86 @@ def average_single_ended( reduce_memory_usage=reduce_memory_usage, **kwargs, ) + mcparams["tmpf"] = self["tmpf"] if ci_avg_time_sel is not None: time_dim2 = "time" + "_avg" x_dim2 = "x" - self.coords[time_dim2] = ( + mcparams.coords[time_dim2] = ( (time_dim2,), - self["time"].sel(**{"time": ci_avg_time_sel}).data, + mcparams["time"].sel(**{"time": ci_avg_time_sel}).data, ) - self["tmpf_avgsec"] = ( + mcparams["tmpf_avgsec"] = ( ("x", time_dim2), - self["tmpf"].sel(**{"time": ci_avg_time_sel}).data, + mcparams["tmpf"].sel(**{"time": ci_avg_time_sel}).data, ) - self["tmpf_mc_set"] = ( + mcparams["tmpf_mc_set"] = ( ("mc", "x", time_dim2), - self["tmpf" + "_mc_set"].sel(**{"time": ci_avg_time_sel}).data, + mcparams["tmpf" + "_mc_set"].sel(**{"time": ci_avg_time_sel}).data, ) elif ci_avg_time_isel is not None: time_dim2 = "time" + "_avg" x_dim2 = "x" - self.coords[time_dim2] = ( + mcparams.coords[time_dim2] = ( (time_dim2,), - self["time"].isel(**{"time": ci_avg_time_isel}).data, + mcparams["time"].isel(**{"time": ci_avg_time_isel}).data, ) - self["tmpf_avgsec"] = ( + mcparams["tmpf_avgsec"] = ( ("x", time_dim2), - self["tmpf"].isel(**{"time": ci_avg_time_isel}).data, + mcparams["tmpf"].isel(**{"time": ci_avg_time_isel}).data, ) - self["tmpf_mc_set"] = ( + mcparams["tmpf_mc_set"] = ( ("mc", "x", time_dim2), - self["tmpf" + "_mc_set"].isel(**{"time": ci_avg_time_isel}).data, + mcparams["tmpf" + "_mc_set"].isel(**{"time": ci_avg_time_isel}).data, ) elif ci_avg_x_sel is not None: time_dim2 = "time" x_dim2 = "x_avg" - self.coords[x_dim2] = ((x_dim2,), self.x.sel(x=ci_avg_x_sel).data) - self["tmpf_avgsec"] = ( + mcparams.coords[x_dim2] = ((x_dim2,), mcparams.x.sel(x=ci_avg_x_sel).data) + mcparams["tmpf_avgsec"] = ( (x_dim2, "time"), - self["tmpf"].sel(x=ci_avg_x_sel).data, + mcparams["tmpf"].sel(x=ci_avg_x_sel).data, ) - self["tmpf_mc_set"] = ( + mcparams["tmpf_mc_set"] = ( ("mc", x_dim2, "time"), - self["tmpf_mc_set"].sel(x=ci_avg_x_sel).data, + mcparams["tmpf_mc_set"].sel(x=ci_avg_x_sel).data, ) elif ci_avg_x_isel is not None: time_dim2 = "time" x_dim2 = "x_avg" - self.coords[x_dim2] = ((x_dim2,), self.x.isel(x=ci_avg_x_isel).data) - self["tmpf_avgsec"] = ( + mcparams.coords[x_dim2] = ((x_dim2,), mcparams.x.isel(x=ci_avg_x_isel).data) + mcparams["tmpf_avgsec"] = ( (x_dim2, time_dim2), - self["tmpf"].isel(x=ci_avg_x_isel).data, + mcparams["tmpf"].isel(x=ci_avg_x_isel).data, ) - self["tmpf_mc_set"] = ( + mcparams["tmpf_mc_set"] = ( ("mc", x_dim2, time_dim2), - self["tmpf_mc_set"].isel(x=ci_avg_x_isel).data, + mcparams["tmpf_mc_set"].isel(x=ci_avg_x_isel).data, ) else: - self["tmpf_avgsec"] = self["tmpf"] + mcparams["tmpf_avgsec"] = mcparams["tmpf"] x_dim2 = "x" time_dim2 = "time" # subtract the mean temperature - q = self["tmpf_mc_set"] - self["tmpf_avgsec"] - self["tmpf_mc" + "_avgsec_var"] = q.var(dim="mc", ddof=1) + q = mcparams["tmpf_mc_set"] - mcparams["tmpf_avgsec"] + out["tmpf_mc" + "_avgsec_var"] = q.var(dim="mc", ddof=1) if ci_avg_x_flag1: # unweighted mean - self["tmpf_avgx1"] = self["tmpf" + "_avgsec"].mean(dim=x_dim2) + out["tmpf_avgx1"] = mcparams["tmpf" + "_avgsec"].mean(dim=x_dim2) - q = self["tmpf_mc_set"] - self["tmpf_avgsec"] + q = mcparams["tmpf_mc_set"] - mcparams["tmpf_avgsec"] qvar = q.var(dim=["mc", x_dim2], ddof=1) - self["tmpf_mc_avgx1_var"] = qvar + out["tmpf_mc_avgx1_var"] = qvar if conf_ints: - new_chunks = (len(conf_ints), self["tmpf_mc_set"].chunks[2]) - avg_axis = self["tmpf_mc_set"].get_axis_num(["mc", x_dim2]) - q = self["tmpf_mc_set"].data.map_blocks( + new_chunks = (len(conf_ints), mcparams["tmpf_mc_set"].chunks[2]) + avg_axis = mcparams["tmpf_mc_set"].get_axis_num(["mc", x_dim2]) + q = mcparams["tmpf_mc_set"].data.map_blocks( lambda x: np.percentile(x, q=conf_ints, axis=avg_axis), chunks=new_chunks, # drop_axis=avg_axis, @@ -2474,28 +2474,28 @@ def average_single_ended( new_axis=0, ) # The new CI dim is added as firsaxis - self["tmpf_mc_avgx1"] = (("CI", time_dim2), q) + out["tmpf_mc_avgx1"] = (("CI", time_dim2), q) if ci_avg_x_flag2: - q = self["tmpf_mc_set"] - self["tmpf_avgsec"] + q = mcparams["tmpf_mc_set"] - mcparams["tmpf_avgsec"] qvar = q.var(dim=["mc"], ddof=1) # Inverse-variance weighting avg_x_var = 1 / (1 / qvar).sum(dim=x_dim2) - self["tmpf_mc_avgx2_var"] = avg_x_var + out["tmpf_mc_avgx2_var"] = avg_x_var - self["tmpf" + "_mc_avgx2_set"] = (self["tmpf_mc_set"] / qvar).sum( + mcparams["tmpf" + "_mc_avgx2_set"] = (mcparams["tmpf_mc_set"] / qvar).sum( dim=x_dim2 ) * avg_x_var - self["tmpf" + "_avgx2"] = self["tmpf" + "_mc_avgx2_set"].mean(dim="mc") + out["tmpf" + "_avgx2"] = mcparams["tmpf" + "_mc_avgx2_set"].mean(dim="mc") if conf_ints: - new_chunks = (len(conf_ints), self["tmpf_mc_set"].chunks[2]) - avg_axis_avgx = self["tmpf_mc_set"].get_axis_num("mc") + new_chunks = (len(conf_ints), mcparams["tmpf_mc_set"].chunks[2]) + avg_axis_avgx = mcparams["tmpf_mc_set"].get_axis_num("mc") - qq = self["tmpf_mc_avgx2_set"].data.map_blocks( + qq = mcparams["tmpf_mc_avgx2_set"].data.map_blocks( lambda x: np.percentile(x, q=conf_ints, axis=avg_axis_avgx), chunks=new_chunks, # drop_axis=avg_axis_avgx, @@ -2504,20 +2504,20 @@ def average_single_ended( dtype=float, ) # The new CI dimension is added as # firsaxis - self["tmpf_mc_avgx2"] = (("CI", time_dim2), qq) + out["tmpf_mc_avgx2"] = (("CI", time_dim2), qq) if ci_avg_time_flag1 is not None: # unweighted mean - self["tmpf_avg1"] = self["tmpf" + "_avgsec"].mean(dim=time_dim2) + out["tmpf_avg1"] = mcparams["tmpf_avgsec"].mean(dim=time_dim2) - q = self["tmpf_mc_set"] - self["tmpf_avgsec"] + q = mcparams["tmpf_mc_set"] - mcparams["tmpf_avgsec"] qvar = q.var(dim=["mc", time_dim2], ddof=1) - self["tmpf_mc_avg1_var"] = qvar + out["tmpf_mc_avg1_var"] = qvar if conf_ints: - new_chunks = (len(conf_ints), self["tmpf_mc_set"].chunks[1]) - avg_axis = self["tmpf_mc_set"].get_axis_num(["mc", time_dim2]) - q = self["tmpf_mc_set"].data.map_blocks( + new_chunks = (len(conf_ints), mcparams["tmpf_mc_set"].chunks[1]) + avg_axis = mcparams["tmpf_mc_set"].get_axis_num(["mc", time_dim2]) + q = mcparams["tmpf_mc_set"].data.map_blocks( lambda x: np.percentile(x, q=conf_ints, axis=avg_axis), chunks=new_chunks, # drop_axis=avg_axis, @@ -2525,28 +2525,28 @@ def average_single_ended( new_axis=0, ) # The new CI dim is added as firsaxis - self["tmpf_mc_avg1"] = (("CI", x_dim2), q) + out["tmpf_mc_avg1"] = (("CI", x_dim2), q) if ci_avg_time_flag2: - q = self["tmpf_mc_set"] - self["tmpf_avgsec"] + q = mcparams["tmpf_mc_set"] - mcparams["tmpf_avgsec"] qvar = q.var(dim=["mc"], ddof=1) # Inverse-variance weighting avg_time_var = 1 / (1 / qvar).sum(dim=time_dim2) - self["tmpf_mc_avg2_var"] = avg_time_var + out["tmpf_mc_avg2_var"] = avg_time_var - self["tmpf" + "_mc_avg2_set"] = (self["tmpf_mc_set"] / qvar).sum( + mcparams["tmpf" + "_mc_avg2_set"] = (mcparams["tmpf_mc_set"] / qvar).sum( dim=time_dim2 ) * avg_time_var - self["tmpf_avg2"] = self["tmpf" + "_mc_avg2_set"].mean(dim="mc") + out["tmpf_avg2"] = mcparams["tmpf" + "_mc_avg2_set"].mean(dim="mc") if conf_ints: - new_chunks = (len(conf_ints), self["tmpf_mc_set"].chunks[1]) - avg_axis_avg2 = self["tmpf_mc_set"].get_axis_num("mc") + new_chunks = (len(conf_ints), mcparams["tmpf_mc_set"].chunks[1]) + avg_axis_avg2 = mcparams["tmpf_mc_set"].get_axis_num("mc") - qq = self["tmpf_mc_avg2_set"].data.map_blocks( + qq = mcparams["tmpf_mc_avg2_set"].data.map_blocks( lambda x: np.percentile(x, q=conf_ints, axis=avg_axis_avg2), chunks=new_chunks, # drop_axis=avg_axis_avg2, @@ -2555,7 +2555,7 @@ def average_single_ended( dtype=float, ) # The new CI dimension is added as # firsaxis - self["tmpf_mc_avg2"] = (("CI", x_dim2), qq) + out["tmpf_mc_avg2"] = (("CI", x_dim2), qq) # Clean up the garbage. All arrays with a Monte Carlo dimension. if mc_remove_set_flag: @@ -2577,8 +2577,10 @@ def average_single_ended( remove_mc_set.append("tmpf_mc_avgsec_var") for k in remove_mc_set: - if k in self: - del self[k] + if k in out: + del out[k] + + self.update(out) pass def average_double_ended( @@ -2767,7 +2769,9 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): else: pass - self.conf_int_double_ended( + out = xr.Dataset() + + mcparams = self.conf_int_double_ended( sections=sections, p_val=p_val, p_cov=p_cov, @@ -2787,83 +2791,89 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): if ci_avg_time_sel is not None: time_dim2 = "time" + "_avg" x_dim2 = "x" - self.coords[time_dim2] = ( + mcparams.coords[time_dim2] = ( (time_dim2,), - self["time"].sel(**{"time": ci_avg_time_sel}).data, + mcparams["time"].sel(**{"time": ci_avg_time_sel}).data, ) - self[label + "_avgsec"] = ( + mcparams[label + "_avgsec"] = ( ("x", time_dim2), self[label].sel(**{"time": ci_avg_time_sel}).data, ) - self[label + "_mc_set"] = ( + mcparams[label + "_mc_set"] = ( ("mc", "x", time_dim2), - self[label + "_mc_set"].sel(**{"time": ci_avg_time_sel}).data, + mcparams[label + "_mc_set"].sel(**{"time": ci_avg_time_sel}).data, ) elif ci_avg_time_isel is not None: time_dim2 = "time" + "_avg" x_dim2 = "x" - self.coords[time_dim2] = ( + mcparams.coords[time_dim2] = ( (time_dim2,), - self["time"].isel(**{"time": ci_avg_time_isel}).data, + mcparams["time"].isel(**{"time": ci_avg_time_isel}).data, ) - self[label + "_avgsec"] = ( + mcparams[label + "_avgsec"] = ( ("x", time_dim2), self[label].isel(**{"time": ci_avg_time_isel}).data, ) - self[label + "_mc_set"] = ( + mcparams[label + "_mc_set"] = ( ("mc", "x", time_dim2), - self[label + "_mc_set"].isel(**{"time": ci_avg_time_isel}).data, + mcparams[label + "_mc_set"].isel(**{"time": ci_avg_time_isel}).data, ) elif ci_avg_x_sel is not None: time_dim2 = "time" x_dim2 = "x_avg" - self.coords[x_dim2] = ((x_dim2,), self.x.sel(x=ci_avg_x_sel).data) - self[label + "_avgsec"] = ( + mcparams.coords[x_dim2] = ( + (x_dim2,), + mcparams.x.sel(x=ci_avg_x_sel).data, + ) + mcparams[label + "_avgsec"] = ( (x_dim2, "time"), self[label].sel(x=ci_avg_x_sel).data, ) - self[label + "_mc_set"] = ( + mcparams[label + "_mc_set"] = ( ("mc", x_dim2, "time"), - self[label + "_mc_set"].sel(x=ci_avg_x_sel).data, + mcparams[label + "_mc_set"].sel(x=ci_avg_x_sel).data, ) elif ci_avg_x_isel is not None: time_dim2 = "time" x_dim2 = "x_avg" - self.coords[x_dim2] = ((x_dim2,), self.x.isel(x=ci_avg_x_isel).data) - self[label + "_avgsec"] = ( + mcparams.coords[x_dim2] = ( + (x_dim2,), + mcparams.x.isel(x=ci_avg_x_isel).data, + ) + mcparams[label + "_avgsec"] = ( (x_dim2, time_dim2), self[label].isel(x=ci_avg_x_isel).data, ) - self[label + "_mc_set"] = ( + mcparams[label + "_mc_set"] = ( ("mc", x_dim2, time_dim2), - self[label + "_mc_set"].isel(x=ci_avg_x_isel).data, + mcparams[label + "_mc_set"].isel(x=ci_avg_x_isel).data, ) else: - self[label + "_avgsec"] = self[label] + mcparams[label + "_avgsec"] = self[label] x_dim2 = "x" time_dim2 = "time" - memchunk = self[label + "_mc_set"].chunks + memchunk = mcparams[label + "_mc_set"].chunks # subtract the mean temperature - q = self[label + "_mc_set"] - self[label + "_avgsec"] - self[label + "_mc" + "_avgsec_var"] = q.var(dim="mc", ddof=1) + q = mcparams[label + "_mc_set"] - mcparams[label + "_avgsec"] + out[label + "_mc" + "_avgsec_var"] = q.var(dim="mc", ddof=1) if ci_avg_x_flag1: # unweighted mean - self[label + "_avgx1"] = self[label + "_avgsec"].mean(dim=x_dim2) + out[label + "_avgx1"] = mcparams[label + "_avgsec"].mean(dim=x_dim2) - q = self[label + "_mc_set"] - self[label + "_avgsec"] + q = mcparams[label + "_mc_set"] - mcparams[label + "_avgsec"] qvar = q.var(dim=["mc", x_dim2], ddof=1) - self[label + "_mc_avgx1_var"] = qvar + out[label + "_mc_avgx1_var"] = qvar if conf_ints: - new_chunks = (len(conf_ints), self[label + "_mc_set"].chunks[2]) - avg_axis = self[label + "_mc_set"].get_axis_num(["mc", x_dim2]) - q = self[label + "_mc_set"].data.map_blocks( + new_chunks = (len(conf_ints), mcparams[label + "_mc_set"].chunks[2]) + avg_axis = mcparams[label + "_mc_set"].get_axis_num(["mc", x_dim2]) + q = mcparams[label + "_mc_set"].data.map_blocks( lambda x: np.percentile(x, q=conf_ints, axis=avg_axis), chunks=new_chunks, # drop_axis=avg_axis, @@ -2871,28 +2881,28 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): new_axis=0, ) # The new CI dim is added as firsaxis - self[label + "_mc_avgx1"] = (("CI", time_dim2), q) + out[label + "_mc_avgx1"] = (("CI", time_dim2), q) if ci_avg_x_flag2: - q = self[label + "_mc_set"] - self[label + "_avgsec"] + q = mcparams[label + "_mc_set"] - mcparams[label + "_avgsec"] qvar = q.var(dim=["mc"], ddof=1) # Inverse-variance weighting avg_x_var = 1 / (1 / qvar).sum(dim=x_dim2) - self[label + "_mc_avgx2_var"] = avg_x_var + out[label + "_mc_avgx2_var"] = avg_x_var - self[label + "_mc_avgx2_set"] = (self[label + "_mc_set"] / qvar).sum( - dim=x_dim2 - ) * avg_x_var - self[label + "_avgx2"] = self[label + "_mc_avgx2_set"].mean(dim="mc") + mcparams[label + "_mc_avgx2_set"] = ( + mcparams[label + "_mc_set"] / qvar + ).sum(dim=x_dim2) * avg_x_var + out[label + "_avgx2"] = mcparams[label + "_mc_avgx2_set"].mean(dim="mc") if conf_ints: - new_chunks = (len(conf_ints), self[label + "_mc_set"].chunks[2]) - avg_axis_avgx = self[label + "_mc_set"].get_axis_num("mc") + new_chunks = (len(conf_ints), mcparams[label + "_mc_set"].chunks[2]) + avg_axis_avgx = mcparams[label + "_mc_set"].get_axis_num("mc") - qq = self[label + "_mc_avgx2_set"].data.map_blocks( + qq = mcparams[label + "_mc_avgx2_set"].data.map_blocks( lambda x: np.percentile(x, q=conf_ints, axis=avg_axis_avgx), chunks=new_chunks, # drop_axis=avg_axis_avgx, @@ -2901,20 +2911,22 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): dtype=float, ) # The new CI dimension is added as # firsaxis - self[label + "_mc_avgx2"] = (("CI", time_dim2), qq) + out[label + "_mc_avgx2"] = (("CI", time_dim2), qq) if ci_avg_time_flag1 is not None: # unweighted mean - self[label + "_avg1"] = self[label + "_avgsec"].mean(dim=time_dim2) + out[label + "_avg1"] = mcparams[label + "_avgsec"].mean(dim=time_dim2) - q = self[label + "_mc_set"] - self[label + "_avgsec"] + q = mcparams[label + "_mc_set"] - mcparams[label + "_avgsec"] qvar = q.var(dim=["mc", time_dim2], ddof=1) - self[label + "_mc_avg1_var"] = qvar + out[label + "_mc_avg1_var"] = qvar if conf_ints: - new_chunks = (len(conf_ints), self[label + "_mc_set"].chunks[1]) - avg_axis = self[label + "_mc_set"].get_axis_num(["mc", time_dim2]) - q = self[label + "_mc_set"].data.map_blocks( + new_chunks = (len(conf_ints), mcparams[label + "_mc_set"].chunks[1]) + avg_axis = mcparams[label + "_mc_set"].get_axis_num( + ["mc", time_dim2] + ) + q = mcparams[label + "_mc_set"].data.map_blocks( lambda x: np.percentile(x, q=conf_ints, axis=avg_axis), chunks=new_chunks, # drop_axis=avg_axis, @@ -2922,28 +2934,28 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): new_axis=0, ) # The new CI dim is added as firsaxis - self[label + "_mc_avg1"] = (("CI", x_dim2), q) + out[label + "_mc_avg1"] = (("CI", x_dim2), q) if ci_avg_time_flag2: - q = self[label + "_mc_set"] - self[label + "_avgsec"] + q = mcparams[label + "_mc_set"] - mcparams[label + "_avgsec"] qvar = q.var(dim=["mc"], ddof=1) # Inverse-variance weighting avg_time_var = 1 / (1 / qvar).sum(dim=time_dim2) - self[label + "_mc_avg2_var"] = avg_time_var + out[label + "_mc_avg2_var"] = avg_time_var - self[label + "_mc_avg2_set"] = (self[label + "_mc_set"] / qvar).sum( - dim=time_dim2 - ) * avg_time_var - self[label + "_avg2"] = self[label + "_mc_avg2_set"].mean(dim="mc") + mcparams[label + "_mc_avg2_set"] = ( + mcparams[label + "_mc_set"] / qvar + ).sum(dim=time_dim2) * avg_time_var + out[label + "_avg2"] = mcparams[label + "_mc_avg2_set"].mean(dim="mc") if conf_ints: - new_chunks = (len(conf_ints), self[label + "_mc_set"].chunks[1]) - avg_axis_avg2 = self[label + "_mc_set"].get_axis_num("mc") + new_chunks = (len(conf_ints), mcparams[label + "_mc_set"].chunks[1]) + avg_axis_avg2 = mcparams[label + "_mc_set"].get_axis_num("mc") - qq = self[label + "_mc_avg2_set"].data.map_blocks( + qq = mcparams[label + "_mc_avg2_set"].data.map_blocks( lambda x: np.percentile(x, q=conf_ints, axis=avg_axis_avg2), chunks=new_chunks, # drop_axis=avg_axis_avg2, @@ -2952,40 +2964,40 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): dtype=float, ) # The new CI dimension is added as # firsaxis - self[label + "_mc_avg2"] = (("CI", x_dim2), qq) + out[label + "_mc_avg2"] = (("CI", x_dim2), qq) # Weighted mean of the forward and backward tmpw_var = 1 / ( - 1 / self["tmpf_mc" + "_avgsec_var"] + 1 / self["tmpb_mc" + "_avgsec_var"] + 1 / out["tmpf_mc" + "_avgsec_var"] + 1 / out["tmpb_mc" + "_avgsec_var"] ) q = ( - self["tmpf_mc_set"] / self["tmpf_mc" + "_avgsec_var"] - + self["tmpb_mc_set"] / self["tmpb_mc" + "_avgsec_var"] + mcparams["tmpf_mc_set"] / out["tmpf_mc" + "_avgsec_var"] + + mcparams["tmpb_mc_set"] / out["tmpb_mc" + "_avgsec_var"] ) * tmpw_var - self["tmpw" + "_mc_set"] = q # + mcparams["tmpw" + "_mc_set"] = q # - # self["tmpw"] = self["tmpw" + '_mc_set'].mean(dim='mc') - self["tmpw" + "_avgsec"] = ( - self["tmpf_avgsec"] / self["tmpf_mc" + "_avgsec_var"] - + self["tmpb_avgsec"] / self["tmpb_mc" + "_avgsec_var"] + # out["tmpw"] = out["tmpw" + '_mc_set'].mean(dim='mc') + out["tmpw" + "_avgsec"] = ( + mcparams["tmpf_avgsec"] / out["tmpf_mc" + "_avgsec_var"] + + mcparams["tmpb_avgsec"] / out["tmpb_mc" + "_avgsec_var"] ) * tmpw_var - q = self["tmpw" + "_mc_set"] - self["tmpw" + "_avgsec"] - self["tmpw" + "_mc" + "_avgsec_var"] = q.var(dim="mc", ddof=1) + q = mcparams["tmpw" + "_mc_set"] - out["tmpw_avgsec"] + out["tmpw" + "_mc" + "_avgsec_var"] = q.var(dim="mc", ddof=1) if ci_avg_time_flag1: - self["tmpw" + "_avg1"] = self["tmpw" + "_avgsec"].mean(dim=time_dim2) + out["tmpw" + "_avg1"] = out["tmpw" + "_avgsec"].mean(dim=time_dim2) - self["tmpw" + "_mc_avg1_var"] = self["tmpw" + "_mc_set"].var( + out["tmpw" + "_mc_avg1_var"] = mcparams["tmpw" + "_mc_set"].var( dim=["mc", time_dim2] ) if conf_ints: new_chunks_weighted = ((len(conf_ints),),) + (memchunk[1],) - avg_axis = self["tmpw" + "_mc_set"].get_axis_num(["mc", time_dim2]) - q2 = self["tmpw" + "_mc_set"].data.map_blocks( + avg_axis = mcparams["tmpw" + "_mc_set"].get_axis_num(["mc", time_dim2]) + q2 = mcparams["tmpw" + "_mc_set"].data.map_blocks( lambda x: np.percentile(x, q=conf_ints, axis=avg_axis), chunks=new_chunks_weighted, # Explicitly define output chunks @@ -2994,32 +3006,32 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): dtype=float, ) # The new CI dimension is added as # first axis - self["tmpw" + "_mc_avg1"] = (("CI", x_dim2), q2) + out["tmpw" + "_mc_avg1"] = (("CI", x_dim2), q2) if ci_avg_time_flag2: tmpw_var_avg2 = 1 / ( - 1 / self["tmpf_mc_avg2_var"] + 1 / self["tmpb_mc_avg2_var"] + 1 / out["tmpf_mc_avg2_var"] + 1 / out["tmpb_mc_avg2_var"] ) q = ( - self["tmpf_mc_avg2_set"] / self["tmpf_mc_avg2_var"] - + self["tmpb_mc_avg2_set"] / self["tmpb_mc_avg2_var"] + mcparams["tmpf_mc_avg2_set"] / out["tmpf_mc_avg2_var"] + + mcparams["tmpb_mc_avg2_set"] / out["tmpb_mc_avg2_var"] ) * tmpw_var_avg2 - self["tmpw" + "_mc_avg2_set"] = q # + mcparams["tmpw" + "_mc_avg2_set"] = q # - self["tmpw" + "_avg2"] = ( - self["tmpf_avg2"] / self["tmpf_mc_avg2_var"] - + self["tmpb_avg2"] / self["tmpb_mc_avg2_var"] + out["tmpw" + "_avg2"] = ( + out["tmpf_avg2"] / out["tmpf_mc_avg2_var"] + + out["tmpb_avg2"] / out["tmpb_mc_avg2_var"] ) * tmpw_var_avg2 - self["tmpw" + "_mc_avg2_var"] = tmpw_var_avg2 + out["tmpw" + "_mc_avg2_var"] = tmpw_var_avg2 if conf_ints: # We first need to know the x-dim-chunk-size new_chunks_weighted = ((len(conf_ints),),) + (memchunk[1],) - avg_axis_avg2 = self["tmpw" + "_mc_avg2_set"].get_axis_num("mc") - q2 = self["tmpw" + "_mc_avg2_set"].data.map_blocks( + avg_axis_avg2 = mcparams["tmpw" + "_mc_avg2_set"].get_axis_num("mc") + q2 = mcparams["tmpw" + "_mc_avg2_set"].data.map_blocks( lambda x: np.percentile(x, q=conf_ints, axis=avg_axis_avg2), chunks=new_chunks_weighted, # Explicitly define output chunks @@ -3027,17 +3039,17 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): new_axis=0, dtype=float, ) # The new CI dimension is added as firstax - self["tmpw" + "_mc_avg2"] = (("CI", x_dim2), q2) + out["tmpw" + "_mc_avg2"] = (("CI", x_dim2), q2) if ci_avg_x_flag1: - self["tmpw" + "_avgx1"] = self["tmpw" + "_avgsec"].mean(dim=x_dim2) + out["tmpw" + "_avgx1"] = out["tmpw" + "_avgsec"].mean(dim=x_dim2) - self["tmpw" + "_mc_avgx1_var"] = self["tmpw" + "_mc_set"].var(dim=x_dim2) + out["tmpw" + "_mc_avgx1_var"] = mcparams["tmpw" + "_mc_set"].var(dim=x_dim2) if conf_ints: new_chunks_weighted = ((len(conf_ints),),) + (memchunk[2],) - avg_axis = self["tmpw" + "_mc_set"].get_axis_num(["mc", x_dim2]) - q2 = self["tmpw" + "_mc_set"].data.map_blocks( + avg_axis = mcparams["tmpw" + "_mc_set"].get_axis_num(["mc", x_dim2]) + q2 = mcparams["tmpw" + "_mc_set"].data.map_blocks( lambda x: np.percentile(x, q=conf_ints, axis=avg_axis), chunks=new_chunks_weighted, # Explicitly define output chunks @@ -3046,32 +3058,32 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): dtype=float, ) # The new CI dimension is added as # first axis - self["tmpw" + "_mc_avgx1"] = (("CI", time_dim2), q2) + out["tmpw" + "_mc_avgx1"] = (("CI", time_dim2), q2) if ci_avg_x_flag2: tmpw_var_avgx2 = 1 / ( - 1 / self["tmpf_mc_avgx2_var"] + 1 / self["tmpb_mc_avgx2_var"] + 1 / out["tmpf_mc_avgx2_var"] + 1 / out["tmpb_mc_avgx2_var"] ) q = ( - self["tmpf_mc_avgx2_set"] / self["tmpf_mc_avgx2_var"] - + self["tmpb_mc_avgx2_set"] / self["tmpb_mc_avgx2_var"] + mcparams["tmpf_mc_avgx2_set"] / out["tmpf_mc_avgx2_var"] + + mcparams["tmpb_mc_avgx2_set"] / out["tmpb_mc_avgx2_var"] ) * tmpw_var_avgx2 - self["tmpw" + "_mc_avgx2_set"] = q # + mcparams["tmpw" + "_mc_avgx2_set"] = q # - self["tmpw" + "_avgx2"] = ( - self["tmpf_avgx2"] / self["tmpf_mc_avgx2_var"] - + self["tmpb_avgx2"] / self["tmpb_mc_avgx2_var"] + out["tmpw" + "_avgx2"] = ( + out["tmpf_avgx2"] / out["tmpf_mc_avgx2_var"] + + out["tmpb_avgx2"] / out["tmpb_mc_avgx2_var"] ) * tmpw_var_avgx2 - self["tmpw" + "_mc_avgx2_var"] = tmpw_var_avgx2 + out["tmpw" + "_mc_avgx2_var"] = tmpw_var_avgx2 if conf_ints: # We first need to know the x-dim-chunk-size new_chunks_weighted = ((len(conf_ints),),) + (memchunk[2],) - avg_axis_avgx2 = self["tmpw" + "_mc_avgx2_set"].get_axis_num("mc") - q2 = self["tmpw" + "_mc_avgx2_set"].data.map_blocks( + avg_axis_avgx2 = mcparams["tmpw" + "_mc_avgx2_set"].get_axis_num("mc") + q2 = mcparams["tmpw" + "_mc_avgx2_set"].data.map_blocks( lambda x: np.percentile(x, q=conf_ints, axis=avg_axis_avgx2), chunks=new_chunks_weighted, # Explicitly define output chunks @@ -3079,7 +3091,7 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): new_axis=0, dtype=float, ) # The new CI dimension is added as firstax - self["tmpw" + "_mc_avgx2"] = (("CI", time_dim2), q2) + out["tmpw" + "_mc_avgx2"] = (("CI", time_dim2), q2) # Clean up the garbage. All arrays with a Monte Carlo dimension. if mc_remove_set_flag: @@ -3104,13 +3116,16 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): remove_mc_set.append(i + "_mc_avgx2_set") remove_mc_set.append(i + "_mc_avgsec_var") - if self.trans_att.size: + if "trans_att" in mcparams and mcparams.trans_att.size: remove_mc_set.append('talpha"_fw_mc') remove_mc_set.append('talpha"_bw_mc') for k in remove_mc_set: - if k in self: - del self[k] + if k in out: + print(f"Removed from results: {k}") + del out[k] + + self.update(out) pass def conf_int_single_ended( @@ -3194,6 +3209,9 @@ def conf_int_single_ended( """ check_deprecated_kwargs(kwargs) + out = xr.Dataset() + params = xr.Dataset() + if da_random_state: state = da_random_state else: @@ -3202,6 +3220,7 @@ def conf_int_single_ended( no, nt = self.st.data.shape if "trans_att" in self.keys(): nta = self.trans_att.size + else: nta = 0 @@ -3219,10 +3238,13 @@ def conf_int_single_ended( else: raise Exception("The size of `p_val` is not what I expected") - self.coords["mc"] = range(mc_sample_size) + params.coords["mc"] = range(mc_sample_size) + params.coords["x"] = self.x + params.coords["time"] = self.time if conf_ints: - self.coords["CI"] = conf_ints + out.coords["CI"] = conf_ints + params.coords["CI"] = conf_ints # WLS if isinstance(p_cov, str): @@ -3232,20 +3254,20 @@ def conf_int_single_ended( p_mc = sst.multivariate_normal.rvs(mean=p_val, cov=p_cov, size=mc_sample_size) if fixed_alpha: - self["alpha_mc"] = (("mc", "x"), p_mc[:, 1 : no + 1]) - self["c_mc"] = (("mc", "time"), p_mc[:, 1 + no : 1 + no + nt]) + params["alpha_mc"] = (("mc", "x"), p_mc[:, 1 : no + 1]) + params["c_mc"] = (("mc", "time"), p_mc[:, 1 + no : 1 + no + nt]) else: - self["dalpha_mc"] = (("mc",), p_mc[:, 1]) - self["c_mc"] = (("mc", "time"), p_mc[:, 2 : nt + 2]) + params["dalpha_mc"] = (("mc",), p_mc[:, 1]) + params["c_mc"] = (("mc", "time"), p_mc[:, 2 : nt + 2]) - self["gamma_mc"] = (("mc",), p_mc[:, 0]) + params["gamma_mc"] = (("mc",), p_mc[:, 0]) if nta: - self["ta_mc"] = ( + params["ta_mc"] = ( ("mc", "trans_att", "time"), np.reshape(p_mc[:, -nt * nta :], (mc_sample_size, nta, nt)), ) - rsize = (self.mc.size, self.x.size, self.time.size) + rsize = (params.mc.size, params.x.size, params.time.size) if reduce_memory_usage: memchunk = da.ones( @@ -3292,7 +3314,7 @@ def conf_int_single_ended( else: st_vari_da = da.from_array(st_vari, chunks=memchunk[1:]) - self[k] = ( + params[k] = ( ("mc", "x", "time"), state.normal( loc=loc, # has chunks=memchunk[1:] @@ -3305,52 +3327,50 @@ def conf_int_single_ended( ta_arr = np.zeros((mc_sample_size, no, nt)) if nta: - for ii, ta in enumerate(self["ta_mc"]): + for ii, ta in enumerate(params["ta_mc"]): for tai, taxi in zip(ta.values, self.trans_att.values): ta_arr[ii, self.x.values >= taxi] = ( ta_arr[ii, self.x.values >= taxi] + tai ) - self["ta_mc_arr"] = (("mc", "x", "time"), ta_arr) + params["ta_mc_arr"] = (("mc", "x", "time"), ta_arr) if fixed_alpha: - self["tmpf_mc_set"] = ( - self["gamma_mc"] + params["tmpf_mc_set"] = ( + params["gamma_mc"] / ( ( - np.log(self["r_st"]) - - np.log(self["r_ast"]) - + (self["c_mc"] + self["ta_mc_arr"]) + np.log(params["r_st"]) + - np.log(params["r_ast"]) + + (params["c_mc"] + params["ta_mc_arr"]) ) - + self["alpha_mc"] + + params["alpha_mc"] ) - 273.15 ) else: - self["tmpf_mc_set"] = ( - self["gamma_mc"] + params["tmpf_mc_set"] = ( + params["gamma_mc"] / ( ( - np.log(self["r_st"]) - - np.log(self["r_ast"]) - + (self["c_mc"] + self["ta_mc_arr"]) + np.log(params["r_st"]) + - np.log(params["r_ast"]) + + (params["c_mc"] + params["ta_mc_arr"]) ) - + (self["dalpha_mc"] * self.x) + + (params["dalpha_mc"] * params.x) ) - 273.15 ) avg_dims = ["mc"] - - avg_axis = self["tmpf_mc_set"].get_axis_num(avg_dims) - - self["tmpf_mc_var"] = (self["tmpf_mc_set"] - self["tmpf"]).var( + avg_axis = params["tmpf_mc_set"].get_axis_num(avg_dims) + out["tmpf_mc_var"] = (params["tmpf_mc_set"] - self["tmpf"]).var( dim=avg_dims, ddof=1 ) if conf_ints: - new_chunks = ((len(conf_ints),),) + self["tmpf_mc_set"].chunks[1:] + new_chunks = ((len(conf_ints),),) + params["tmpf_mc_set"].chunks[1:] - qq = self["tmpf_mc_set"] + qq = params["tmpf_mc_set"] q = qq.data.map_blocks( lambda x: np.percentile(x, q=conf_ints, axis=avg_axis), @@ -3359,25 +3379,13 @@ def conf_int_single_ended( new_axis=0, ) # The new CI dimension is added as first axis - self["tmpf_mc"] = (("CI", "x", "time"), q) + out["tmpf_mc"] = (("CI", "x", "time"), q) - if mc_remove_set_flag: - drop_var = [ - "gamma_mc", - "dalpha_mc", - "alpha_mc", - "c_mc", - "mc", - "r_st", - "r_ast", - "tmpf_mc_set", - "ta_mc_arr", - ] - for k in drop_var: - if k in self: - del self[k] + if not mc_remove_set_flag: + out.update(params) - pass + self.update(out) + return out def conf_int_double_ended( self, @@ -3549,6 +3557,9 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): check_deprecated_kwargs(kwargs) + out = xr.Dataset() + params = xr.Dataset() + if da_random_state: # In testing environments assert isinstance(da_random_state, da.random.RandomState) @@ -3578,9 +3589,13 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): (mc_sample_size, no, nt), chunks={0: -1, 1: "auto", 2: "auto"} ).chunks - self.coords["mc"] = range(mc_sample_size) + params.coords["mc"] = range(mc_sample_size) + params.coords["x"] = self.x + params.coords["time"] = self.time + if conf_ints: self.coords["CI"] = conf_ints + params.coords["CI"] = conf_ints assert isinstance(p_val, (str, np.ndarray, np.generic)) if isinstance(p_val, str): @@ -3600,10 +3615,10 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): d_bw = p_val[1 + nt : 2 * nt + 1] alpha = p_val[2 * nt + 1 : 2 * nt + 1 + no] - self["gamma_mc"] = (tuple(), gamma) - self["alpha_mc"] = (("x",), alpha) - self["df_mc"] = (("time",), d_fw) - self["db_mc"] = (("time",), d_bw) + params["gamma_mc"] = (tuple(), gamma) + params["alpha_mc"] = (("x",), alpha) + params["df_mc"] = (("time",), d_fw) + params["db_mc"] = (("time",), d_bw) if nta: ta = p_val[2 * nt + 1 + no :].reshape((nt, 2, nta), order="F") @@ -3611,19 +3626,19 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): ta_bw = ta[:, 1, :] ta_fw_arr = np.zeros((no, nt)) - for tai, taxi in zip(ta_fw.T, self.coords["trans_att"].values): - ta_fw_arr[self.x.values >= taxi] = ( - ta_fw_arr[self.x.values >= taxi] + tai + for tai, taxi in zip(ta_fw.T, params.coords["trans_att"].values): + ta_fw_arr[params.x.values >= taxi] = ( + ta_fw_arr[params.x.values >= taxi] + tai ) ta_bw_arr = np.zeros((no, nt)) - for tai, taxi in zip(ta_bw.T, self.coords["trans_att"].values): - ta_bw_arr[self.x.values < taxi] = ( - ta_bw_arr[self.x.values < taxi] + tai + for tai, taxi in zip(ta_bw.T, params.coords["trans_att"].values): + ta_bw_arr[params.x.values < taxi] = ( + ta_bw_arr[params.x.values < taxi] + tai ) - self["talpha_fw_mc"] = (("x", "time"), ta_fw_arr) - self["talpha_bw_mc"] = (("x", "time"), ta_bw_arr) + params["talpha_fw_mc"] = (("x", "time"), ta_fw_arr) + params["talpha_bw_mc"] = (("x", "time"), ta_bw_arr) elif isinstance(p_cov, bool) and p_cov: raise NotImplementedError("Not an implemented option. Check p_cov argument") @@ -3657,9 +3672,9 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): d_fw = po_mc[:, 1 : nt + 1] d_bw = po_mc[:, 1 + nt : 2 * nt + 1] - self["gamma_mc"] = (("mc",), gamma) - self["df_mc"] = (("mc", "time"), d_fw) - self["db_mc"] = (("mc", "time"), d_bw) + params["gamma_mc"] = (("mc",), gamma) + params["df_mc"] = (("mc", "time"), d_fw) + params["db_mc"] = (("mc", "time"), d_bw) # calculate alpha seperately alpha = np.zeros((mc_sample_size, no), dtype=float) @@ -3679,7 +3694,7 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): alpha[:, not_ix_sec] = not_alpha_mc - self["alpha_mc"] = (("mc", "x"), alpha) + params["alpha_mc"] = (("mc", "x"), alpha) if nta: ta = po_mc[:, 2 * nt + 1 + nx_sec :].reshape( @@ -3692,10 +3707,10 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): (mc_sample_size, no, nt), chunks=memchunk, dtype=float ) for tai, taxi in zip( - ta_fw.swapaxes(0, 2), self.coords["trans_att"].values + ta_fw.swapaxes(0, 2), params.coords["trans_att"].values ): # iterate over the splices - i_splice = sum(self.x.values < taxi) + i_splice = sum(params.x.values < taxi) mask = create_da_ta2(no, i_splice, direction="fw", chunks=memchunk) ta_fw_arr += mask * tai.T[:, None, :] @@ -3704,15 +3719,15 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): (mc_sample_size, no, nt), chunks=memchunk, dtype=float ) for tai, taxi in zip( - ta_bw.swapaxes(0, 2), self.coords["trans_att"].values + ta_bw.swapaxes(0, 2), params.coords["trans_att"].values ): - i_splice = sum(self.x.values < taxi) + i_splice = sum(params.x.values < taxi) mask = create_da_ta2(no, i_splice, direction="bw", chunks=memchunk) ta_bw_arr += mask * tai.T[:, None, :] - self["talpha_fw_mc"] = (("mc", "x", "time"), ta_fw_arr) - self["talpha_bw_mc"] = (("mc", "x", "time"), ta_bw_arr) + params["talpha_fw_mc"] = (("mc", "x", "time"), ta_fw_arr) + params["talpha_bw_mc"] = (("mc", "x", "time"), ta_bw_arr) # Draw from the normal distributions for the Stokes intensities for k, st_labeli, st_vari in zip( @@ -3752,7 +3767,7 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): else: st_vari_da = da.from_array(st_vari, chunks=memchunk[1:]) - self[k] = ( + params[k] = ( ("mc", "x", "time"), state.normal( loc=loc, # has chunks=memchunk[1:] @@ -3766,45 +3781,45 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): if "tmpw" or label: if label == "tmpf": if nta: - self["tmpf_mc_set"] = ( - self["gamma_mc"] + params["tmpf_mc_set"] = ( + params["gamma_mc"] / ( - np.log(self["r_st"] / self["r_ast"]) - + self["df_mc"] - + self["alpha_mc"] - + self["talpha_fw_mc"] + np.log(params["r_st"] / params["r_ast"]) + + params["df_mc"] + + params["alpha_mc"] + + params["talpha_fw_mc"] ) - 273.15 ) else: - self["tmpf_mc_set"] = ( - self["gamma_mc"] + params["tmpf_mc_set"] = ( + params["gamma_mc"] / ( - np.log(self["r_st"] / self["r_ast"]) - + self["df_mc"] - + self["alpha_mc"] + np.log(params["r_st"] / params["r_ast"]) + + params["df_mc"] + + params["alpha_mc"] ) - 273.15 ) else: if nta: - self["tmpb_mc_set"] = ( - self["gamma_mc"] + params["tmpb_mc_set"] = ( + params["gamma_mc"] / ( - np.log(self["r_rst"] / self["r_rast"]) - + self["db_mc"] - - self["alpha_mc"] - + self["talpha_bw_mc"] + np.log(params["r_rst"] / params["r_rast"]) + + params["db_mc"] + - params["alpha_mc"] + + params["talpha_bw_mc"] ) - 273.15 ) else: - self["tmpb_mc_set"] = ( - self["gamma_mc"] + params["tmpb_mc_set"] = ( + params["gamma_mc"] / ( - np.log(self["r_rst"] / self["r_rast"]) - + self["db_mc"] - - self["alpha_mc"] + np.log(params["r_rst"] / params["r_rast"]) + + params["db_mc"] + - params["alpha_mc"] ) - 273.15 ) @@ -3814,19 +3829,21 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): xi = self.ufunc_per_section( sections=sections, x_indices=True, calc_per="all" ) - x_mask_ = [True if ix in xi else False for ix in range(self.x.size)] + x_mask_ = [ + True if ix in xi else False for ix in range(params.x.size) + ] x_mask = np.reshape(x_mask_, (1, -1, 1)) - self[label + "_mc_set"] = self[label + "_mc_set"].where(x_mask) + params[label + "_mc_set"] = params[label + "_mc_set"].where(x_mask) # subtract the mean temperature - q = self[label + "_mc_set"] - self[label] - self[label + "_mc_var"] = q.var(dim="mc", ddof=1) + q = params[label + "_mc_set"] - self[label] + out[label + "_mc_var"] = q.var(dim="mc", ddof=1) if conf_ints: - new_chunks = list(self[label + "_mc_set"].chunks) + new_chunks = list(params[label + "_mc_set"].chunks) new_chunks[0] = (len(conf_ints),) - avg_axis = self[label + "_mc_set"].get_axis_num("mc") - q = self[label + "_mc_set"].data.map_blocks( + avg_axis = params[label + "_mc_set"].get_axis_num("mc") + q = params[label + "_mc_set"].data.map_blocks( lambda x: np.percentile(x, q=conf_ints, axis=avg_axis), chunks=new_chunks, # drop_axis=avg_axis, @@ -3834,37 +3851,37 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): new_axis=0, ) # The new CI dimension is added as firsaxis - self[label + "_mc"] = (("CI", "x", "time"), q) + out[label + "_mc"] = (("CI", "x", "time"), q) # Weighted mean of the forward and backward - tmpw_var = 1 / (1 / self["tmpf_mc_var"] + 1 / self["tmpb_mc_var"]) + tmpw_var = 1 / (1 / out["tmpf_mc_var"] + 1 / out["tmpb_mc_var"]) q = ( - self["tmpf_mc_set"] / self["tmpf_mc_var"] - + self["tmpb_mc_set"] / self["tmpb_mc_var"] + params["tmpf_mc_set"] / out["tmpf_mc_var"] + + params["tmpb_mc_set"] / out["tmpb_mc_var"] ) * tmpw_var - self["tmpw" + "_mc_set"] = q # + params["tmpw" + "_mc_set"] = q # - self["tmpw"] = ( - self["tmpf"] / self["tmpf_mc_var"] + self["tmpb"] / self["tmpb_mc_var"] + out["tmpw"] = ( + self["tmpf"] / out["tmpf_mc_var"] + self["tmpb"] / out["tmpb_mc_var"] ) * tmpw_var - q = self["tmpw" + "_mc_set"] - self["tmpw"] - self["tmpw" + "_mc_var"] = q.var(dim="mc", ddof=1) + q = params["tmpw" + "_mc_set"] - self["tmpw"] + out["tmpw" + "_mc_var"] = q.var(dim="mc", ddof=1) # Calculate the CI of the weighted MC_set if conf_ints: new_chunks_weighted = ((len(conf_ints),),) + memchunk[1:] - avg_axis = self["tmpw" + "_mc_set"].get_axis_num("mc") - q2 = self["tmpw" + "_mc_set"].data.map_blocks( + avg_axis = params["tmpw" + "_mc_set"].get_axis_num("mc") + q2 = params["tmpw" + "_mc_set"].data.map_blocks( lambda x: np.percentile(x, q=conf_ints, axis=avg_axis), chunks=new_chunks_weighted, # Explicitly define output chunks drop_axis=avg_axis, # avg dimensions are dropped new_axis=0, dtype=float, ) # The new CI dimension is added as first axis - self["tmpw" + "_mc"] = (("CI", "x", "time"), q2) + out["tmpw" + "_mc"] = (("CI", "x", "time"), q2) # Clean up the garbage. All arrays with a Monte Carlo dimension. if mc_remove_set_flag: @@ -3887,9 +3904,14 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): remove_mc_set.append('talpha"_bw_mc') for k in remove_mc_set: - if k in self: - del self[k] - pass + if k in out: + del out[k] + + if not mc_remove_set_flag: + out.update(params) + + self.update(out) + return out def in_confidence_interval(self, ci_label, conf_ints=None, sections=None): """ diff --git a/tests/test_dtscalibration.py b/tests/test_dtscalibration.py index 23516b09..b354c584 100644 --- a/tests/test_dtscalibration.py +++ b/tests/test_dtscalibration.py @@ -130,7 +130,11 @@ def test_variance_input_types_single(): ) ds.conf_int_single_ended( - st_var=st_var, ast_var=st_var, mc_sample_size=100, da_random_state=state + st_var=st_var, + ast_var=st_var, + mc_sample_size=100, + da_random_state=state, + mc_remove_set_flag=False, ) assert_almost_equal_verbose( From a515f8b3e1e73752a1181278f4f015b49b67feab Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Sun, 3 Sep 2023 15:57:19 +0200 Subject: [PATCH 18/66] Updated sections example notebook --- docs/notebooks/03Define_sections.ipynb | 35 +++++--------------------- 1 file changed, 6 insertions(+), 29 deletions(-) diff --git a/docs/notebooks/03Define_sections.ipynb b/docs/notebooks/03Define_sections.ipynb index ef9d97b2..8b1000ea 100644 --- a/docs/notebooks/03Define_sections.ipynb +++ b/docs/notebooks/03Define_sections.ipynb @@ -23,7 +23,7 @@ "source": [ "import os\n", "\n", - "from dtscalibration import read_silixa_files" + "from dtscalibration import read_silixa_files\n" ] }, { @@ -40,7 +40,7 @@ "outputs": [], "source": [ "filepath = os.path.join(\"..\", \"..\", \"tests\", \"data\", \"double_ended2\")\n", - "ds = read_silixa_files(directory=filepath, timezone_netcdf=\"UTC\", file_ext=\"*.xml\")" + "ds = read_silixa_files(directory=filepath, timezone_netcdf=\"UTC\", file_ext=\"*.xml\")\n" ] }, { @@ -66,7 +66,8 @@ "outputs": [], "source": [ "print(ds.timeseries_keys) # list the available timeseeries\n", - "ds.probe1Temperature.plot(figsize=(12, 8)); # plot one of the timeseries" + "ds.probe1Temperature.plot(figsize=(12, 8))\n", + "# plot one of the timeseries\n" ] }, { @@ -115,24 +116,7 @@ "sections = {\n", " \"probe1Temperature\": [slice(7.5, 17.0), slice(70.0, 80.0)], # cold bath\n", " \"probe2Temperature\": [slice(24.0, 34.0), slice(85.0, 95.0)], # warm bath\n", - "}\n", - "ds.sections = sections" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "execution": { - "iopub.execute_input": "2022-04-06T08:09:10.309314Z", - "iopub.status.busy": "2022-04-06T08:09:10.308985Z", - "iopub.status.idle": "2022-04-06T08:09:10.323444Z", - "shell.execute_reply": "2022-04-06T08:09:10.322874Z" - } - }, - "outputs": [], - "source": [ - "ds.sections" + "}\n" ] }, { @@ -141,13 +125,6 @@ "source": [ "NetCDF files do not support reading/writing python dictionaries. Internally the sections dictionary is stored in `ds._sections` as a string encoded with yaml, which can be saved to a netCDF file. Each time the sections dictionary is requested, yaml decodes the string and evaluates it to the Python dictionary. " ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { @@ -166,7 +143,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.11" + "version": "3.10.10" } }, "nbformat": 4, From 39f71c9c5384b182767f1f12630683a49c1cdc71 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Sun, 3 Sep 2023 16:26:53 +0200 Subject: [PATCH 19/66] Updated example notebooks --- .../04Calculate_variance_Stokes.ipynb | 13 ++++--- docs/notebooks/07Calibrate_single_ended.ipynb | 20 +++++------ docs/notebooks/08Calibrate_double_ended.ipynb | 36 ++++++++++--------- .../12Datastore_from_numpy_arrays.ipynb | 23 ++++++------ .../13Fixed_parameter_calibration.ipynb | 19 +++++----- docs/notebooks/14Lossy_splices.ipynb | 29 +++++++-------- docs/notebooks/15Matching_sections.ipynb | 32 +++++++++-------- docs/notebooks/16Averaging_temperatures.ipynb | 26 ++++++++------ ...Temperature_uncertainty_single_ended.ipynb | 18 +++++----- src/dtscalibration/datastore.py | 1 + 10 files changed, 115 insertions(+), 102 deletions(-) diff --git a/docs/notebooks/04Calculate_variance_Stokes.ipynb b/docs/notebooks/04Calculate_variance_Stokes.ipynb index 2e0eba4b..0de14167 100644 --- a/docs/notebooks/04Calculate_variance_Stokes.ipynb +++ b/docs/notebooks/04Calculate_variance_Stokes.ipynb @@ -55,7 +55,7 @@ "source": [ "filepath = os.path.join(\"..\", \"..\", \"tests\", \"data\", \"double_ended2\")\n", "\n", - "ds = read_silixa_files(directory=filepath, timezone_netcdf=\"UTC\", file_ext=\"*.xml\")" + "ds = read_silixa_files(directory=filepath, timezone_netcdf=\"UTC\", file_ext=\"*.xml\")\n" ] }, { @@ -81,8 +81,7 @@ "sections = {\n", " \"probe1Temperature\": [slice(7.5, 17.0), slice(70.0, 80.0)], # cold bath\n", " \"probe2Temperature\": [slice(24.0, 34.0), slice(85.0, 95.0)], # warm bath\n", - "}\n", - "ds.sections = sections" + "}\n" ] }, { @@ -120,13 +119,13 @@ }, "outputs": [], "source": [ - "I_var, residuals = ds.variance_stokes_constant(st_label=\"st\")\n", + "I_var, residuals = ds.variance_stokes_constant(sections=sections, st_label=\"st\")\n", "print(\n", " \"The variance of the Stokes signal along the reference sections \"\n", " \"is approximately {:.2f} on a {:.1f} sec acquisition time\".format(\n", " I_var, ds.userAcquisitionTimeFW.data[0]\n", " )\n", - ")" + ")\n" ] }, { @@ -153,7 +152,7 @@ " robust=True,\n", " units=\"\",\n", " method=\"single\",\n", - ")" + ")\n" ] }, { @@ -187,7 +186,7 @@ "x = np.linspace(mean - 3 * sigma, mean + 3 * sigma, 100)\n", "approximated_normal_fit = scipy.stats.norm.pdf(x, mean, sigma)\n", "residuals.plot.hist(bins=50, figsize=(12, 8), density=True)\n", - "plt.plot(x, approximated_normal_fit);" + "plt.plot(x, approximated_normal_fit)\n" ] }, { diff --git a/docs/notebooks/07Calibrate_single_ended.ipynb b/docs/notebooks/07Calibrate_single_ended.ipynb index 28616d9e..4c591ede 100644 --- a/docs/notebooks/07Calibrate_single_ended.ipynb +++ b/docs/notebooks/07Calibrate_single_ended.ipynb @@ -72,7 +72,7 @@ "outputs": [], "source": [ "filepath = os.path.join(\"..\", \"..\", \"tests\", \"data\", \"single_ended\")\n", - "ds = read_silixa_files(directory=filepath, timezone_netcdf=\"UTC\", file_ext=\"*.xml\")" + "ds = read_silixa_files(directory=filepath, timezone_netcdf=\"UTC\", file_ext=\"*.xml\")\n" ] }, { @@ -90,10 +90,10 @@ "outputs": [], "source": [ "ds = ds.sel(x=slice(-30, 101)) # dismiss parts of the fiber that are not interesting\n", - "ds.sections = {\n", + "sections = {\n", " \"probe1Temperature\": [slice(20, 25.5)], # warm bath\n", " \"probe2Temperature\": [slice(5.5, 15.5)], # cold bath\n", - "}" + "}\n" ] }, { @@ -119,8 +119,8 @@ }, "outputs": [], "source": [ - "st_var, resid = ds.variance_stokes_constant(st_label=\"st\")\n", - "ast_var, _ = ds.variance_stokes_constant(st_label=\"ast\")" + "st_var, resid = ds.variance_stokes_constant(sections=sections, st_label=\"st\")\n", + "ast_var, _ = ds.variance_stokes_constant(sections=sections, st_label=\"ast\")\n" ] }, { @@ -136,7 +136,7 @@ "metadata": {}, "outputs": [], "source": [ - "resid.plot(figsize=(12, 4));" + "resid.plot(figsize=(12, 4))\n" ] }, { @@ -160,7 +160,7 @@ }, "outputs": [], "source": [ - "ds.calibration_single_ended(st_var=st_var, ast_var=ast_var)" + "ds.calibration_single_ended(sections=sections, st_var=st_var, ast_var=ast_var)\n" ] }, { @@ -177,7 +177,7 @@ "metadata": {}, "outputs": [], "source": [ - "ds.tmpf.plot(figsize=(12, 4));" + "ds.tmpf.plot(figsize=(12, 4))\n" ] }, { @@ -189,7 +189,7 @@ "ds1 = ds.isel(time=0)\n", "ds1.tmpf.plot(figsize=(12, 4))\n", "(ds1.tmpf_var**0.5).plot(figsize=(12, 4))\n", - "plt.ylabel(\"$\\sigma$ ($^\\circ$C)\");" + "plt.ylabel(\"$\\sigma$ ($^\\circ$C)\")\n" ] }, { @@ -217,7 +217,7 @@ "outputs": [], "source": [ "ds1.st.plot(figsize=(12, 8))\n", - "ds1.ast.plot();" + "ds1.ast.plot()\n" ] }, { diff --git a/docs/notebooks/08Calibrate_double_ended.ipynb b/docs/notebooks/08Calibrate_double_ended.ipynb index 2ed0f35b..21ba4b45 100644 --- a/docs/notebooks/08Calibrate_double_ended.ipynb +++ b/docs/notebooks/08Calibrate_double_ended.ipynb @@ -70,7 +70,7 @@ "\n", "ds_notaligned = read_silixa_files(\n", " directory=filepath, timezone_netcdf=\"UTC\", file_ext=\"*.xml\"\n", - ")" + ")\n" ] }, { @@ -102,7 +102,7 @@ }, "outputs": [], "source": [ - "ds_notaligned = ds_notaligned.sel(x=slice(0, 100)) # only calibrate parts of the fiber" + "ds_notaligned = ds_notaligned.sel(x=slice(0, 100)) # only calibrate parts of the fiber\n" ] }, { @@ -124,7 +124,7 @@ " plot_result=True,\n", " figsize=(12, 6),\n", ")\n", - "print(suggested_shift)" + "print(suggested_shift)\n" ] }, { @@ -140,7 +140,7 @@ "metadata": {}, "outputs": [], "source": [ - "ds = shift_double_ended(ds_notaligned, suggested_shift[0])" + "ds = shift_double_ended(ds_notaligned, suggested_shift[0])\n" ] }, { @@ -157,10 +157,10 @@ "metadata": {}, "outputs": [], "source": [ - "ds.sections = {\n", + "sections = {\n", " \"probe1Temperature\": [slice(7.5, 17.0), slice(70.0, 80.0)], # cold bath\n", " \"probe2Temperature\": [slice(24.0, 34.0), slice(85.0, 95.0)], # warm bath\n", - "}" + "}\n" ] }, { @@ -186,10 +186,10 @@ }, "outputs": [], "source": [ - "st_var, resid = ds.variance_stokes_constant(st_label=\"st\")\n", - "ast_var, _ = ds.variance_stokes_constant(st_label=\"ast\")\n", - "rst_var, _ = ds.variance_stokes_constant(st_label=\"rst\")\n", - "rast_var, _ = ds.variance_stokes_constant(st_label=\"rast\")" + "st_var, resid = ds.variance_stokes_constant(sections=sections, st_label=\"st\")\n", + "ast_var, _ = ds.variance_stokes_constant(sections=sections, st_label=\"ast\")\n", + "rst_var, _ = ds.variance_stokes_constant(sections=sections, st_label=\"rst\")\n", + "rast_var, _ = ds.variance_stokes_constant(sections=sections, st_label=\"rast\")\n" ] }, { @@ -212,7 +212,7 @@ }, "outputs": [], "source": [ - "resid.plot(figsize=(12, 4));" + "resid.plot(figsize=(12, 4))\n" ] }, { @@ -237,11 +237,12 @@ "outputs": [], "source": [ "ds.calibration_double_ended(\n", + " sections=sections,\n", " st_var=st_var,\n", " ast_var=ast_var,\n", " rst_var=rst_var,\n", " rast_var=rast_var,\n", - ")" + ")\n" ] }, { @@ -257,7 +258,7 @@ }, "outputs": [], "source": [ - "ds.tmpw.plot(figsize=(12, 4));" + "ds.tmpw.plot(figsize=(12, 4))\n" ] }, { @@ -295,7 +296,7 @@ "ds.tmpw_var.plot(figsize=(12, 4))\n", "ds1 = ds.isel(time=-1) # take only the first timestep\n", "(ds1.tmpw_var**0.5).plot(figsize=(12, 4))\n", - "plt.gca().set_ylabel(\"Standard error ($^\\circ$C)\");" + "plt.gca().set_ylabel(\"Standard error ($^\\circ$C)\")\n" ] }, { @@ -323,6 +324,7 @@ "outputs": [], "source": [ "ds.conf_int_double_ended(\n", + " sections=sections,\n", " st_var=st_var,\n", " ast_var=ast_var,\n", " rst_var=rst_var,\n", @@ -331,7 +333,7 @@ " mc_sample_size=500,\n", ") # < increase sample size for better approximation\n", "\n", - "ds.tmpw_mc_var.plot(figsize=(12, 4));" + "ds.tmpw_mc_var.plot(figsize=(12, 4))\n" ] }, { @@ -351,7 +353,7 @@ "ds1.tmpw.plot(linewidth=0.7, figsize=(12, 4))\n", "ds1.tmpw_mc.isel(CI=0).plot(linewidth=0.7, label=\"CI: 2.5%\")\n", "ds1.tmpw_mc.isel(CI=1).plot(linewidth=0.7, label=\"CI: 97.5%\")\n", - "plt.legend(fontsize=\"small\")" + "plt.legend(fontsize=\"small\")\n" ] }, { @@ -380,7 +382,7 @@ "(ds1.tmpb_var**0.5).plot()\n", "(ds1.tmpw_var**0.5).plot()\n", "(ds1.tmpw_mc_var**0.5).plot()\n", - "plt.ylabel(\"$\\sigma$ ($^\\circ$C)\");" + "plt.ylabel(\"$\\sigma$ ($^\\circ$C)\")\n" ] }, { diff --git a/docs/notebooks/12Datastore_from_numpy_arrays.ipynb b/docs/notebooks/12Datastore_from_numpy_arrays.ipynb index 2ddb9a20..1f3d3ce7 100644 --- a/docs/notebooks/12Datastore_from_numpy_arrays.ipynb +++ b/docs/notebooks/12Datastore_from_numpy_arrays.ipynb @@ -26,7 +26,7 @@ "import matplotlib.pyplot as plt\n", "import xarray as xr\n", "\n", - "from dtscalibration import DataStore, read_silixa_files" + "from dtscalibration import DataStore, read_silixa_files\n" ] }, { @@ -61,7 +61,7 @@ "source": [ "filepath = os.path.join(\"..\", \"..\", \"tests\", \"data\", \"single_ended\")\n", "\n", - "ds_silixa = read_silixa_files(directory=filepath, silent=True)" + "ds_silixa = read_silixa_files(directory=filepath, silent=True)\n" ] }, { @@ -89,7 +89,7 @@ "x = ds_silixa.x.values\n", "time = ds_silixa.time.values\n", "ST = ds_silixa.st.values\n", - "AST = ds_silixa.ast.values" + "AST = ds_silixa.ast.values\n" ] }, { @@ -116,7 +116,7 @@ "ds[\"x\"] = (\"x\", x)\n", "ds[\"time\"] = (\"time\", time)\n", "ds[\"st\"] = ([\"x\", \"time\"], ST)\n", - "ds[\"ast\"] = ([\"x\", \"time\"], AST)" + "ds[\"ast\"] = ([\"x\", \"time\"], AST)\n" ] }, { @@ -133,7 +133,7 @@ "outputs": [], "source": [ "ds = DataStore(ds)\n", - "print(ds)" + "print(ds)\n" ] }, { @@ -169,7 +169,7 @@ "ds[\"temp1\"] = ds_silixa[\"probe1Temperature\"]\n", "ds[\"temp2\"] = ds_silixa[\"probe2Temperature\"]\n", "\n", - "ds.attrs[\"isDoubleEnded\"] = \"0\"" + "ds.attrs[\"isDoubleEnded\"] = \"0\"\n" ] }, { @@ -197,13 +197,12 @@ " \"temp1\": [slice(20, 25.5)], # warm bath\n", " \"temp2\": [slice(5.5, 15.5)], # cold bath\n", "}\n", - "ds.sections = sections\n", "\n", - "st_var, resid = ds.variance_stokes_constant(st_label=\"st\")\n", - "ast_var, _ = ds.variance_stokes_constant(st_label=\"ast\")\n", - "ds.calibration_single_ended(st_var=st_var, ast_var=ast_var)\n", + "st_var, resid = ds.variance_stokes_constant(sections=sections, st_label=\"st\")\n", + "ast_var, _ = ds.variance_stokes_constant(sections=sections, st_label=\"ast\")\n", + "ds.calibration_single_ended(sections=sections, st_var=st_var, ast_var=ast_var)\n", "\n", - "ds.isel(time=0).tmpf.plot();" + "ds.isel(time=0).tmpf.plot()\n" ] }, { @@ -212,7 +211,7 @@ "metadata": {}, "outputs": [], "source": [ - "ds" + "ds\n" ] }, { diff --git a/docs/notebooks/13Fixed_parameter_calibration.ipynb b/docs/notebooks/13Fixed_parameter_calibration.ipynb index c5b9fb47..cf5a78c1 100644 --- a/docs/notebooks/13Fixed_parameter_calibration.ipynb +++ b/docs/notebooks/13Fixed_parameter_calibration.ipynb @@ -39,8 +39,7 @@ " \"probe1Temperature\": [\n", " slice(20, 25.5)\n", " ], # we only use the warm bath in this notebook\n", - "}\n", - "ds100.sections = sections" + "}" ] }, { @@ -69,11 +68,15 @@ "fix_gamma = (481.9, 0) # (gamma value, gamma variance)\n", "fix_dalpha = (-2.014e-5, 0) # (alpha value, alpha variance)\n", "\n", - "st_var, resid = ds100.variance_stokes_constant(st_label=\"st\")\n", - "ast_var, _ = ds100.variance_stokes_constant(st_label=\"ast\")\n", + "st_var, resid = ds100.variance_stokes_constant(sections=sections, st_label=\"st\")\n", + "ast_var, _ = ds100.variance_stokes_constant(sections=sections, st_label=\"ast\")\n", "ds100.calibration_single_ended(\n", - " st_var=st_var, ast_var=ast_var, fix_gamma=fix_gamma, fix_dalpha=fix_dalpha\n", - ")" + " sections=sections,\n", + " st_var=st_var,\n", + " ast_var=ast_var,\n", + " fix_gamma=fix_gamma,\n", + " fix_dalpha=fix_dalpha,\n", + ")\n" ] }, { @@ -97,7 +100,7 @@ "outputs": [], "source": [ "print(\"gamma used in calibration:\", ds100.gamma.values)\n", - "print(\"dalpha used in calibration:\", ds100.dalpha.values)" + "print(\"dalpha used in calibration:\", ds100.dalpha.values)\n" ] }, { @@ -129,7 +132,7 @@ " linewidth=1, label=\"Device calibrated\"\n", ") # plot the temperature calibrated by the device\n", "plt.title(\"Temperature at the first time step\")\n", - "plt.legend();" + "plt.legend()\n" ] }, { diff --git a/docs/notebooks/14Lossy_splices.ipynb b/docs/notebooks/14Lossy_splices.ipynb index c2c98e6e..5e41b774 100644 --- a/docs/notebooks/14Lossy_splices.ipynb +++ b/docs/notebooks/14Lossy_splices.ipynb @@ -72,8 +72,7 @@ "sections = {\n", " \"probe1Temperature\": [slice(7.5, 17.0), slice(70.0, 80.0)], # cold bath\n", " \"probe2Temperature\": [slice(24.0, 34.0), slice(85.0, 95.0)], # warm bath\n", - "}\n", - "ds.sections = sections" + "}\n" ] }, { @@ -102,7 +101,7 @@ "ds[\"ast\"] = ds.ast.where(ds.x < 50, ds.ast * 0.82)\n", "\n", "ds[\"rst\"] = ds.rst.where(ds.x > 50, ds.rst * 0.85)\n", - "ds[\"rast\"] = ds.rast.where(ds.x > 50, ds.rast * 0.81)" + "ds[\"rast\"] = ds.rast.where(ds.x > 50, ds.rast * 0.81)\n" ] }, { @@ -122,7 +121,7 @@ "ds.isel(time=0).ast.plot(label=\"ast\")\n", "ds.isel(time=0).rst.plot(label=\"rst\")\n", "ds.isel(time=0).rast.plot(label=\"rast\")\n", - "plt.legend();" + "plt.legend()\n" ] }, { @@ -147,19 +146,20 @@ "source": [ "ds_a = ds.copy(deep=True)\n", "\n", - "st_var, resid = ds_a.variance_stokes(st_label=\"st\")\n", - "ast_var, _ = ds_a.variance_stokes(st_label=\"ast\")\n", - "rst_var, _ = ds_a.variance_stokes(st_label=\"rst\")\n", - "rast_var, _ = ds_a.variance_stokes(st_label=\"rast\")\n", + "st_var, resid = ds_a.variance_stokes(sections=sections, st_label=\"st\")\n", + "ast_var, _ = ds_a.variance_stokes(sections=sections, st_label=\"ast\")\n", + "rst_var, _ = ds_a.variance_stokes(sections=sections, st_label=\"rst\")\n", + "rast_var, _ = ds_a.variance_stokes(sections=sections, st_label=\"rast\")\n", "\n", "ds_a.calibration_double_ended(\n", + " sections=sections,\n", " st_var=st_var,\n", " ast_var=ast_var,\n", " rst_var=rst_var,\n", " rast_var=rast_var,\n", ")\n", "\n", - "ds_a.isel(time=0).tmpw.plot(label=\"calibrated\");" + "ds_a.isel(time=0).tmpw.plot(label=\"calibrated\")\n" ] }, { @@ -184,12 +184,13 @@ }, "outputs": [], "source": [ - "st_var, resid = ds.variance_stokes(st_label=\"st\")\n", - "ast_var, _ = ds.variance_stokes(st_label=\"ast\")\n", - "rst_var, _ = ds.variance_stokes(st_label=\"rst\")\n", - "rast_var, _ = ds.variance_stokes(st_label=\"rast\")\n", + "st_var, resid = ds.variance_stokes(sections=sections, st_label=\"st\")\n", + "ast_var, _ = ds.variance_stokes(sections=sections, st_label=\"ast\")\n", + "rst_var, _ = ds.variance_stokes(sections=sections, st_label=\"rst\")\n", + "rast_var, _ = ds.variance_stokes(sections=sections, st_label=\"rast\")\n", "\n", "ds.calibration_double_ended(\n", + " sections=sections,\n", " st_var=st_var,\n", " ast_var=ast_var,\n", " rst_var=rst_var,\n", @@ -199,7 +200,7 @@ "\n", "ds_a.isel(time=0).tmpw.plot(label=\"no trans. att.\")\n", "ds.isel(time=0).tmpw.plot(label=\"with trans. att.\")\n", - "plt.legend();" + "plt.legend()\n" ] }, { diff --git a/docs/notebooks/15Matching_sections.ipynb b/docs/notebooks/15Matching_sections.ipynb index a83c3730..cb709b50 100644 --- a/docs/notebooks/15Matching_sections.ipynb +++ b/docs/notebooks/15Matching_sections.ipynb @@ -68,8 +68,7 @@ "sections = {\n", " \"probe1Temperature\": [slice(7.5, 17.0)], # cold bath\n", " \"probe2Temperature\": [slice(24.0, 34.0)], # warm bath\n", - "}\n", - "ds.sections = sections" + "}\n" ] }, { @@ -96,7 +95,7 @@ "ds[\"ast\"] = ds.ast.where(ds.x < 50, ds.ast * 0.82)\n", "\n", "ds[\"rst\"] = ds.rst.where(ds.x > 50, ds.rst * 0.85)\n", - "ds[\"rast\"] = ds.rast.where(ds.x > 50, ds.rast * 0.81)" + "ds[\"rast\"] = ds.rast.where(ds.x > 50, ds.rast * 0.81)\n" ] }, { @@ -123,16 +122,20 @@ "source": [ "ds_a = ds.copy(deep=True)\n", "\n", - "st_var, resid = ds_a.variance_stokes(st_label=\"st\")\n", - "ast_var, _ = ds_a.variance_stokes(st_label=\"ast\")\n", - "rst_var, _ = ds_a.variance_stokes(st_label=\"rst\")\n", - "rast_var, _ = ds_a.variance_stokes(st_label=\"rast\")\n", + "st_var, resid = ds_a.variance_stokes(sections=sections, st_label=\"st\")\n", + "ast_var, _ = ds_a.variance_stokes(sections=sections, st_label=\"ast\")\n", + "rst_var, _ = ds_a.variance_stokes(sections=sections, st_label=\"rst\")\n", + "rast_var, _ = ds_a.variance_stokes(sections=sections, st_label=\"rast\")\n", "\n", "ds_a.calibration_double_ended(\n", - " st_var=st_var, ast_var=ast_var, rst_var=rst_var, rast_var=rast_var\n", + " sections=sections,\n", + " st_var=st_var,\n", + " ast_var=ast_var,\n", + " rst_var=rst_var,\n", + " rast_var=rast_var,\n", ")\n", "\n", - "ds_a.isel(time=0).tmpw.plot(label=\"calibrated\")" + "ds_a.isel(time=0).tmpw.plot(label=\"calibrated\")\n" ] }, { @@ -165,12 +168,13 @@ "source": [ "matching_sections = [(slice(7.5, 17.6), slice(69, 79.1), False)]\n", "\n", - "st_var, resid = ds.variance_stokes(st_label=\"st\")\n", - "ast_var, _ = ds.variance_stokes(st_label=\"ast\")\n", - "rst_var, _ = ds.variance_stokes(st_label=\"rst\")\n", - "rast_var, _ = ds.variance_stokes(st_label=\"rast\")\n", + "st_var, resid = ds.variance_stokes(sections=sections, st_label=\"st\")\n", + "ast_var, _ = ds.variance_stokes(sections=sections, st_label=\"ast\")\n", + "rst_var, _ = ds.variance_stokes(sections=sections, st_label=\"rst\")\n", + "rast_var, _ = ds.variance_stokes(sections=sections, st_label=\"rast\")\n", "\n", "ds.calibration_double_ended(\n", + " sections=sections,\n", " st_var=st_var,\n", " ast_var=ast_var,\n", " rst_var=rst_var,\n", @@ -181,7 +185,7 @@ "\n", "ds_a.isel(time=0).tmpw.plot(label=\"normal calibration\")\n", "ds.isel(time=0).tmpw.plot(label=\"matching sections\")\n", - "plt.legend()" + "plt.legend()\n" ] }, { diff --git a/docs/notebooks/16Averaging_temperatures.ipynb b/docs/notebooks/16Averaging_temperatures.ipynb index 6079bd9f..76925061 100644 --- a/docs/notebooks/16Averaging_temperatures.ipynb +++ b/docs/notebooks/16Averaging_temperatures.ipynb @@ -63,8 +63,7 @@ "sections = {\n", " \"probe1Temperature\": [slice(7.5, 17.0), slice(70.0, 80.0)], # cold bath\n", " \"probe2Temperature\": [slice(24.0, 34.0), slice(85.0, 95.0)], # warm bath\n", - "}\n", - "ds.sections = sections" + "}\n" ] }, { @@ -80,10 +79,10 @@ }, "outputs": [], "source": [ - "st_var, resid = ds.variance_stokes(st_label=\"st\")\n", - "ast_var, _ = ds.variance_stokes(st_label=\"ast\")\n", - "rst_var, _ = ds.variance_stokes(st_label=\"rst\")\n", - "rast_var, _ = ds.variance_stokes(st_label=\"rast\")" + "st_var, resid = ds.variance_stokes(sections=sections, st_label=\"st\")\n", + "ast_var, _ = ds.variance_stokes(sections=sections, st_label=\"ast\")\n", + "rst_var, _ = ds.variance_stokes(sections=sections, st_label=\"rst\")\n", + "rast_var, _ = ds.variance_stokes(sections=sections, st_label=\"rast\")\n" ] }, { @@ -100,11 +99,12 @@ "outputs": [], "source": [ "ds.calibration_double_ended(\n", + " sections=sections,\n", " st_var=st_var,\n", " ast_var=ast_var,\n", " rst_var=rst_var,\n", " rast_var=rast_var,\n", - ")" + ")\n" ] }, { @@ -179,6 +179,7 @@ "outputs": [], "source": [ "ds.average_double_ended(\n", + " sections=sections,\n", " st_var=st_var,\n", " ast_var=ast_var,\n", " rst_var=rst_var,\n", @@ -190,7 +191,7 @@ " ci_avg_time_isel=[0, 1, 2, 3, 4, 5],\n", " ci_avg_time_sel=None,\n", ")\n", - "ds.tmpw_mc_avg1.plot(hue=\"CI\", linewidth=0.8);" + "ds.tmpw_mc_avg1.plot(hue=\"CI\", linewidth=0.8)\n" ] }, { @@ -239,6 +240,7 @@ "outputs": [], "source": [ "ds.average_double_ended(\n", + " sections=sections,\n", " st_var=st_var,\n", " ast_var=ast_var,\n", " rst_var=rst_var,\n", @@ -250,7 +252,7 @@ " ci_avg_time_isel=[0, 1, 2, 3, 4, 5],\n", " ci_avg_time_sel=None,\n", ")\n", - "ds.tmpw_mc_avg2.plot(hue=\"CI\", linewidth=0.8);" + "ds.tmpw_mc_avg2.plot(hue=\"CI\", linewidth=0.8)\n" ] }, { @@ -299,6 +301,7 @@ "outputs": [], "source": [ "ds.average_double_ended(\n", + " sections=sections,\n", " st_var=st_var,\n", " ast_var=ast_var,\n", " rst_var=rst_var,\n", @@ -310,7 +313,7 @@ " ci_avg_x_sel=slice(7.5, 17.0),\n", " ci_avg_x_isel=None,\n", ")\n", - "ds.tmpw_mc_avgx1.plot(hue=\"CI\", linewidth=0.8);" + "ds.tmpw_mc_avgx1.plot(hue=\"CI\", linewidth=0.8)\n" ] }, { @@ -359,6 +362,7 @@ "outputs": [], "source": [ "ds.average_double_ended(\n", + " sections=sections,\n", " st_var=st_var,\n", " ast_var=ast_var,\n", " rst_var=rst_var,\n", @@ -370,7 +374,7 @@ " ci_avg_x_sel=slice(7.5, 17.0),\n", " ci_avg_x_isel=None,\n", ")\n", - "ds.tmpw_mc_avgx2.plot(hue=\"CI\", linewidth=0.8);" + "ds.tmpw_mc_avgx2.plot(hue=\"CI\", linewidth=0.8)\n" ] }, { diff --git a/docs/notebooks/17Temperature_uncertainty_single_ended.ipynb b/docs/notebooks/17Temperature_uncertainty_single_ended.ipynb index f270404f..478a490e 100644 --- a/docs/notebooks/17Temperature_uncertainty_single_ended.ipynb +++ b/docs/notebooks/17Temperature_uncertainty_single_ended.ipynb @@ -32,15 +32,15 @@ "ds = read_silixa_files(directory=filepath, timezone_netcdf=\"UTC\", file_ext=\"*.xml\")\n", "\n", "ds = ds.sel(x=slice(-30, 101)) # dismiss parts of the fiber that are not interesting\n", - "ds.sections = {\n", + "sections = {\n", " \"probe1Temperature\": [slice(20, 25.5)], # warm bath\n", " \"probe2Temperature\": [slice(5.5, 15.5)], # cold bath\n", "}\n", "\n", - "st_var, resid = ds.variance_stokes_constant(st_label=\"st\")\n", - "ast_var, _ = ds.variance_stokes_constant(st_label=\"ast\")\n", + "st_var, resid = ds.variance_stokes_constant(sections=sections, st_label=\"st\")\n", + "ast_var, _ = ds.variance_stokes_constant(sections=sections, st_label=\"ast\")\n", "\n", - "ds.calibration_single_ended(st_var=st_var, ast_var=ast_var)" + "ds.calibration_single_ended(sections=sections, st_var=st_var, ast_var=ast_var)" ] }, { @@ -73,7 +73,7 @@ "plt.fill_between(ds1.x, y1=ds1.tmpf_var, y2=stast_var, label=\"Parameter estimation\")\n", "plt.suptitle(\"Variance of the estimated temperature\")\n", "plt.ylabel(\"$\\sigma^2$ ($^\\circ$C$^2$)\")\n", - "plt.legend(fontsize=\"small\");" + "plt.legend(fontsize=\"small\")\n" ] }, { @@ -84,7 +84,7 @@ "outputs": [], "source": [ "# The effects of the parameter uncertainty can be further inspected\n", - "ds1.var_fw_da.plot(hue=\"comp_fw\", figsize=(12, 4));" + "ds1.var_fw_da.plot(hue=\"comp_fw\", figsize=(12, 4))\n" ] }, { @@ -121,7 +121,7 @@ "source": [ "ds.conf_int_single_ended(\n", " st_var=st_var, ast_var=ast_var, conf_ints=[2.5, 97.5], mc_sample_size=500\n", - ")" + ")\n" ] }, { @@ -145,7 +145,7 @@ "(ds1.tmpf_mc_var**0.5).plot(figsize=(12, 4), label=\"Monte Carlo approx.\")\n", "(ds1.tmpf_var**0.5).plot(label=\"Linear error approx.\")\n", "plt.ylabel(\"$\\sigma$ ($^\\circ$C)\")\n", - "plt.legend(fontsize=\"small\");" + "plt.legend(fontsize=\"small\")\n" ] }, { @@ -166,7 +166,7 @@ "ds1.tmpf.plot(linewidth=0.7, figsize=(12, 4))\n", "ds1.tmpf_mc.sel(CI=2.5).plot(linewidth=0.7, label=\"CI: 2.5%\")\n", "ds1.tmpf_mc.sel(CI=97.5).plot(linewidth=0.7, label=\"CI: 97.5%\")\n", - "plt.legend(fontsize=\"small\");" + "plt.legend(fontsize=\"small\")\n" ] }, { diff --git a/src/dtscalibration/datastore.py b/src/dtscalibration/datastore.py index db555798..459a45dd 100644 --- a/src/dtscalibration/datastore.py +++ b/src/dtscalibration/datastore.py @@ -3649,6 +3649,7 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): p_cov = self[p_cov].values assert p_cov.shape == (npar, npar) + assert sections is not None, "Define sections" ix_sec = self.ufunc_per_section( sections=sections, x_indices=True, calc_per="all" ) From a40c810b68bb4971cbc1c7e8f738cff94bfdf0f7 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Sun, 3 Sep 2023 16:31:26 +0200 Subject: [PATCH 20/66] Add api to docs --- docs/api/dtscalibration.DataStore.rst | 7 +++++++ docs/api/dtscalibration.check_dims.rst | 6 ++++++ docs/api/dtscalibration.check_timestep_allclose.rst | 6 ++++++ docs/api/dtscalibration.get_netcdf_encoding.rst | 6 ++++++ docs/api/dtscalibration.merge_double_ended.rst | 6 ++++++ docs/api/dtscalibration.open_datastore.rst | 6 ++++++ docs/api/dtscalibration.open_mf_datastore.rst | 6 ++++++ docs/api/dtscalibration.plot_accuracy.rst | 6 ++++++ ...dtscalibration.plot_location_residuals_double_ended.rst | 6 ++++++ .../dtscalibration.plot_residuals_reference_sections.rst | 6 ++++++ ...alibration.plot_residuals_reference_sections_single.rst | 6 ++++++ docs/api/dtscalibration.plot_sigma_report.rst | 6 ++++++ docs/api/dtscalibration.read_apsensing_files.rst | 6 ++++++ docs/api/dtscalibration.read_sensornet_files.rst | 6 ++++++ docs/api/dtscalibration.read_sensortran_files.rst | 6 ++++++ docs/api/dtscalibration.read_silixa_files.rst | 6 ++++++ docs/api/dtscalibration.shift_double_ended.rst | 6 ++++++ .../dtscalibration.suggest_cable_shift_double_ended.rst | 6 ++++++ 18 files changed, 109 insertions(+) create mode 100644 docs/api/dtscalibration.DataStore.rst create mode 100644 docs/api/dtscalibration.check_dims.rst create mode 100644 docs/api/dtscalibration.check_timestep_allclose.rst create mode 100644 docs/api/dtscalibration.get_netcdf_encoding.rst create mode 100644 docs/api/dtscalibration.merge_double_ended.rst create mode 100644 docs/api/dtscalibration.open_datastore.rst create mode 100644 docs/api/dtscalibration.open_mf_datastore.rst create mode 100644 docs/api/dtscalibration.plot_accuracy.rst create mode 100644 docs/api/dtscalibration.plot_location_residuals_double_ended.rst create mode 100644 docs/api/dtscalibration.plot_residuals_reference_sections.rst create mode 100644 docs/api/dtscalibration.plot_residuals_reference_sections_single.rst create mode 100644 docs/api/dtscalibration.plot_sigma_report.rst create mode 100644 docs/api/dtscalibration.read_apsensing_files.rst create mode 100644 docs/api/dtscalibration.read_sensornet_files.rst create mode 100644 docs/api/dtscalibration.read_sensortran_files.rst create mode 100644 docs/api/dtscalibration.read_silixa_files.rst create mode 100644 docs/api/dtscalibration.shift_double_ended.rst create mode 100644 docs/api/dtscalibration.suggest_cable_shift_double_ended.rst diff --git a/docs/api/dtscalibration.DataStore.rst b/docs/api/dtscalibration.DataStore.rst new file mode 100644 index 00000000..8ed756d9 --- /dev/null +++ b/docs/api/dtscalibration.DataStore.rst @@ -0,0 +1,7 @@ +DataStore +========= + +.. currentmodule:: dtscalibration + +.. autoclass:: DataStore + :show-inheritance: diff --git a/docs/api/dtscalibration.check_dims.rst b/docs/api/dtscalibration.check_dims.rst new file mode 100644 index 00000000..1dc68e7e --- /dev/null +++ b/docs/api/dtscalibration.check_dims.rst @@ -0,0 +1,6 @@ +check_dims +========== + +.. currentmodule:: dtscalibration + +.. autofunction:: check_dims diff --git a/docs/api/dtscalibration.check_timestep_allclose.rst b/docs/api/dtscalibration.check_timestep_allclose.rst new file mode 100644 index 00000000..d655bfe4 --- /dev/null +++ b/docs/api/dtscalibration.check_timestep_allclose.rst @@ -0,0 +1,6 @@ +check_timestep_allclose +======================= + +.. currentmodule:: dtscalibration + +.. autofunction:: check_timestep_allclose diff --git a/docs/api/dtscalibration.get_netcdf_encoding.rst b/docs/api/dtscalibration.get_netcdf_encoding.rst new file mode 100644 index 00000000..820b0a0f --- /dev/null +++ b/docs/api/dtscalibration.get_netcdf_encoding.rst @@ -0,0 +1,6 @@ +get_netcdf_encoding +=================== + +.. currentmodule:: dtscalibration + +.. autofunction:: get_netcdf_encoding diff --git a/docs/api/dtscalibration.merge_double_ended.rst b/docs/api/dtscalibration.merge_double_ended.rst new file mode 100644 index 00000000..c8e512b5 --- /dev/null +++ b/docs/api/dtscalibration.merge_double_ended.rst @@ -0,0 +1,6 @@ +merge_double_ended +================== + +.. currentmodule:: dtscalibration + +.. autofunction:: merge_double_ended diff --git a/docs/api/dtscalibration.open_datastore.rst b/docs/api/dtscalibration.open_datastore.rst new file mode 100644 index 00000000..95a987bd --- /dev/null +++ b/docs/api/dtscalibration.open_datastore.rst @@ -0,0 +1,6 @@ +open_datastore +============== + +.. currentmodule:: dtscalibration + +.. autofunction:: open_datastore diff --git a/docs/api/dtscalibration.open_mf_datastore.rst b/docs/api/dtscalibration.open_mf_datastore.rst new file mode 100644 index 00000000..edab4ebe --- /dev/null +++ b/docs/api/dtscalibration.open_mf_datastore.rst @@ -0,0 +1,6 @@ +open_mf_datastore +================= + +.. currentmodule:: dtscalibration + +.. autofunction:: open_mf_datastore diff --git a/docs/api/dtscalibration.plot_accuracy.rst b/docs/api/dtscalibration.plot_accuracy.rst new file mode 100644 index 00000000..a2e41fc2 --- /dev/null +++ b/docs/api/dtscalibration.plot_accuracy.rst @@ -0,0 +1,6 @@ +plot_accuracy +============= + +.. currentmodule:: dtscalibration + +.. autofunction:: plot_accuracy diff --git a/docs/api/dtscalibration.plot_location_residuals_double_ended.rst b/docs/api/dtscalibration.plot_location_residuals_double_ended.rst new file mode 100644 index 00000000..ad0d16db --- /dev/null +++ b/docs/api/dtscalibration.plot_location_residuals_double_ended.rst @@ -0,0 +1,6 @@ +plot_location_residuals_double_ended +==================================== + +.. currentmodule:: dtscalibration + +.. autofunction:: plot_location_residuals_double_ended diff --git a/docs/api/dtscalibration.plot_residuals_reference_sections.rst b/docs/api/dtscalibration.plot_residuals_reference_sections.rst new file mode 100644 index 00000000..45f2529b --- /dev/null +++ b/docs/api/dtscalibration.plot_residuals_reference_sections.rst @@ -0,0 +1,6 @@ +plot_residuals_reference_sections +================================= + +.. currentmodule:: dtscalibration + +.. autofunction:: plot_residuals_reference_sections diff --git a/docs/api/dtscalibration.plot_residuals_reference_sections_single.rst b/docs/api/dtscalibration.plot_residuals_reference_sections_single.rst new file mode 100644 index 00000000..725d77a0 --- /dev/null +++ b/docs/api/dtscalibration.plot_residuals_reference_sections_single.rst @@ -0,0 +1,6 @@ +plot_residuals_reference_sections_single +======================================== + +.. currentmodule:: dtscalibration + +.. autofunction:: plot_residuals_reference_sections_single diff --git a/docs/api/dtscalibration.plot_sigma_report.rst b/docs/api/dtscalibration.plot_sigma_report.rst new file mode 100644 index 00000000..b047cdeb --- /dev/null +++ b/docs/api/dtscalibration.plot_sigma_report.rst @@ -0,0 +1,6 @@ +plot_sigma_report +================= + +.. currentmodule:: dtscalibration + +.. autofunction:: plot_sigma_report diff --git a/docs/api/dtscalibration.read_apsensing_files.rst b/docs/api/dtscalibration.read_apsensing_files.rst new file mode 100644 index 00000000..04bd67d6 --- /dev/null +++ b/docs/api/dtscalibration.read_apsensing_files.rst @@ -0,0 +1,6 @@ +read_apsensing_files +==================== + +.. currentmodule:: dtscalibration + +.. autofunction:: read_apsensing_files diff --git a/docs/api/dtscalibration.read_sensornet_files.rst b/docs/api/dtscalibration.read_sensornet_files.rst new file mode 100644 index 00000000..541f00cb --- /dev/null +++ b/docs/api/dtscalibration.read_sensornet_files.rst @@ -0,0 +1,6 @@ +read_sensornet_files +==================== + +.. currentmodule:: dtscalibration + +.. autofunction:: read_sensornet_files diff --git a/docs/api/dtscalibration.read_sensortran_files.rst b/docs/api/dtscalibration.read_sensortran_files.rst new file mode 100644 index 00000000..35d92c94 --- /dev/null +++ b/docs/api/dtscalibration.read_sensortran_files.rst @@ -0,0 +1,6 @@ +read_sensortran_files +===================== + +.. currentmodule:: dtscalibration + +.. autofunction:: read_sensortran_files diff --git a/docs/api/dtscalibration.read_silixa_files.rst b/docs/api/dtscalibration.read_silixa_files.rst new file mode 100644 index 00000000..d5adee72 --- /dev/null +++ b/docs/api/dtscalibration.read_silixa_files.rst @@ -0,0 +1,6 @@ +read_silixa_files +================= + +.. currentmodule:: dtscalibration + +.. autofunction:: read_silixa_files diff --git a/docs/api/dtscalibration.shift_double_ended.rst b/docs/api/dtscalibration.shift_double_ended.rst new file mode 100644 index 00000000..0a879ea1 --- /dev/null +++ b/docs/api/dtscalibration.shift_double_ended.rst @@ -0,0 +1,6 @@ +shift_double_ended +================== + +.. currentmodule:: dtscalibration + +.. autofunction:: shift_double_ended diff --git a/docs/api/dtscalibration.suggest_cable_shift_double_ended.rst b/docs/api/dtscalibration.suggest_cable_shift_double_ended.rst new file mode 100644 index 00000000..14a135c0 --- /dev/null +++ b/docs/api/dtscalibration.suggest_cable_shift_double_ended.rst @@ -0,0 +1,6 @@ +suggest_cable_shift_double_ended +================================ + +.. currentmodule:: dtscalibration + +.. autofunction:: suggest_cable_shift_double_ended From cfc4c1b0df46fb596d15c484c3ab00fdc2f361da Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Sun, 3 Sep 2023 16:35:19 +0200 Subject: [PATCH 21/66] Unfortenate formatting of notebooks --- docs/notebooks/03Define_sections.ipynb | 8 +++--- .../04Calculate_variance_Stokes.ipynb | 10 +++---- docs/notebooks/07Calibrate_single_ended.ipynb | 16 ++++++------ docs/notebooks/08Calibrate_double_ended.ipynb | 26 +++++++++---------- .../12Datastore_from_numpy_arrays.ipynb | 16 ++++++------ .../13Fixed_parameter_calibration.ipynb | 6 ++--- docs/notebooks/14Lossy_splices.ipynb | 10 +++---- docs/notebooks/15Matching_sections.ipynb | 8 +++--- docs/notebooks/16Averaging_temperatures.ipynb | 14 +++++----- ...Temperature_uncertainty_single_ended.ipynb | 10 +++---- 10 files changed, 62 insertions(+), 62 deletions(-) diff --git a/docs/notebooks/03Define_sections.ipynb b/docs/notebooks/03Define_sections.ipynb index 8b1000ea..c342cd53 100644 --- a/docs/notebooks/03Define_sections.ipynb +++ b/docs/notebooks/03Define_sections.ipynb @@ -23,7 +23,7 @@ "source": [ "import os\n", "\n", - "from dtscalibration import read_silixa_files\n" + "from dtscalibration import read_silixa_files" ] }, { @@ -40,7 +40,7 @@ "outputs": [], "source": [ "filepath = os.path.join(\"..\", \"..\", \"tests\", \"data\", \"double_ended2\")\n", - "ds = read_silixa_files(directory=filepath, timezone_netcdf=\"UTC\", file_ext=\"*.xml\")\n" + "ds = read_silixa_files(directory=filepath, timezone_netcdf=\"UTC\", file_ext=\"*.xml\")" ] }, { @@ -67,7 +67,7 @@ "source": [ "print(ds.timeseries_keys) # list the available timeseeries\n", "ds.probe1Temperature.plot(figsize=(12, 8))\n", - "# plot one of the timeseries\n" + "# plot one of the timeseries" ] }, { @@ -116,7 +116,7 @@ "sections = {\n", " \"probe1Temperature\": [slice(7.5, 17.0), slice(70.0, 80.0)], # cold bath\n", " \"probe2Temperature\": [slice(24.0, 34.0), slice(85.0, 95.0)], # warm bath\n", - "}\n" + "}" ] }, { diff --git a/docs/notebooks/04Calculate_variance_Stokes.ipynb b/docs/notebooks/04Calculate_variance_Stokes.ipynb index 0de14167..a167e999 100644 --- a/docs/notebooks/04Calculate_variance_Stokes.ipynb +++ b/docs/notebooks/04Calculate_variance_Stokes.ipynb @@ -55,7 +55,7 @@ "source": [ "filepath = os.path.join(\"..\", \"..\", \"tests\", \"data\", \"double_ended2\")\n", "\n", - "ds = read_silixa_files(directory=filepath, timezone_netcdf=\"UTC\", file_ext=\"*.xml\")\n" + "ds = read_silixa_files(directory=filepath, timezone_netcdf=\"UTC\", file_ext=\"*.xml\")" ] }, { @@ -81,7 +81,7 @@ "sections = {\n", " \"probe1Temperature\": [slice(7.5, 17.0), slice(70.0, 80.0)], # cold bath\n", " \"probe2Temperature\": [slice(24.0, 34.0), slice(85.0, 95.0)], # warm bath\n", - "}\n" + "}" ] }, { @@ -125,7 +125,7 @@ " \"is approximately {:.2f} on a {:.1f} sec acquisition time\".format(\n", " I_var, ds.userAcquisitionTimeFW.data[0]\n", " )\n", - ")\n" + ")" ] }, { @@ -152,7 +152,7 @@ " robust=True,\n", " units=\"\",\n", " method=\"single\",\n", - ")\n" + ")" ] }, { @@ -186,7 +186,7 @@ "x = np.linspace(mean - 3 * sigma, mean + 3 * sigma, 100)\n", "approximated_normal_fit = scipy.stats.norm.pdf(x, mean, sigma)\n", "residuals.plot.hist(bins=50, figsize=(12, 8), density=True)\n", - "plt.plot(x, approximated_normal_fit)\n" + "plt.plot(x, approximated_normal_fit)" ] }, { diff --git a/docs/notebooks/07Calibrate_single_ended.ipynb b/docs/notebooks/07Calibrate_single_ended.ipynb index 4c591ede..7edf0a8a 100644 --- a/docs/notebooks/07Calibrate_single_ended.ipynb +++ b/docs/notebooks/07Calibrate_single_ended.ipynb @@ -72,7 +72,7 @@ "outputs": [], "source": [ "filepath = os.path.join(\"..\", \"..\", \"tests\", \"data\", \"single_ended\")\n", - "ds = read_silixa_files(directory=filepath, timezone_netcdf=\"UTC\", file_ext=\"*.xml\")\n" + "ds = read_silixa_files(directory=filepath, timezone_netcdf=\"UTC\", file_ext=\"*.xml\")" ] }, { @@ -93,7 +93,7 @@ "sections = {\n", " \"probe1Temperature\": [slice(20, 25.5)], # warm bath\n", " \"probe2Temperature\": [slice(5.5, 15.5)], # cold bath\n", - "}\n" + "}" ] }, { @@ -120,7 +120,7 @@ "outputs": [], "source": [ "st_var, resid = ds.variance_stokes_constant(sections=sections, st_label=\"st\")\n", - "ast_var, _ = ds.variance_stokes_constant(sections=sections, st_label=\"ast\")\n" + "ast_var, _ = ds.variance_stokes_constant(sections=sections, st_label=\"ast\")" ] }, { @@ -136,7 +136,7 @@ "metadata": {}, "outputs": [], "source": [ - "resid.plot(figsize=(12, 4))\n" + "resid.plot(figsize=(12, 4))" ] }, { @@ -160,7 +160,7 @@ }, "outputs": [], "source": [ - "ds.calibration_single_ended(sections=sections, st_var=st_var, ast_var=ast_var)\n" + "ds.calibration_single_ended(sections=sections, st_var=st_var, ast_var=ast_var)" ] }, { @@ -177,7 +177,7 @@ "metadata": {}, "outputs": [], "source": [ - "ds.tmpf.plot(figsize=(12, 4))\n" + "ds.tmpf.plot(figsize=(12, 4))" ] }, { @@ -189,7 +189,7 @@ "ds1 = ds.isel(time=0)\n", "ds1.tmpf.plot(figsize=(12, 4))\n", "(ds1.tmpf_var**0.5).plot(figsize=(12, 4))\n", - "plt.ylabel(\"$\\sigma$ ($^\\circ$C)\")\n" + "plt.ylabel(\"$\\sigma$ ($^\\circ$C)\")" ] }, { @@ -217,7 +217,7 @@ "outputs": [], "source": [ "ds1.st.plot(figsize=(12, 8))\n", - "ds1.ast.plot()\n" + "ds1.ast.plot()" ] }, { diff --git a/docs/notebooks/08Calibrate_double_ended.ipynb b/docs/notebooks/08Calibrate_double_ended.ipynb index 21ba4b45..5f36c954 100644 --- a/docs/notebooks/08Calibrate_double_ended.ipynb +++ b/docs/notebooks/08Calibrate_double_ended.ipynb @@ -70,7 +70,7 @@ "\n", "ds_notaligned = read_silixa_files(\n", " directory=filepath, timezone_netcdf=\"UTC\", file_ext=\"*.xml\"\n", - ")\n" + ")" ] }, { @@ -102,7 +102,7 @@ }, "outputs": [], "source": [ - "ds_notaligned = ds_notaligned.sel(x=slice(0, 100)) # only calibrate parts of the fiber\n" + "ds_notaligned = ds_notaligned.sel(x=slice(0, 100)) # only calibrate parts of the fiber" ] }, { @@ -124,7 +124,7 @@ " plot_result=True,\n", " figsize=(12, 6),\n", ")\n", - "print(suggested_shift)\n" + "print(suggested_shift)" ] }, { @@ -140,7 +140,7 @@ "metadata": {}, "outputs": [], "source": [ - "ds = shift_double_ended(ds_notaligned, suggested_shift[0])\n" + "ds = shift_double_ended(ds_notaligned, suggested_shift[0])" ] }, { @@ -160,7 +160,7 @@ "sections = {\n", " \"probe1Temperature\": [slice(7.5, 17.0), slice(70.0, 80.0)], # cold bath\n", " \"probe2Temperature\": [slice(24.0, 34.0), slice(85.0, 95.0)], # warm bath\n", - "}\n" + "}" ] }, { @@ -189,7 +189,7 @@ "st_var, resid = ds.variance_stokes_constant(sections=sections, st_label=\"st\")\n", "ast_var, _ = ds.variance_stokes_constant(sections=sections, st_label=\"ast\")\n", "rst_var, _ = ds.variance_stokes_constant(sections=sections, st_label=\"rst\")\n", - "rast_var, _ = ds.variance_stokes_constant(sections=sections, st_label=\"rast\")\n" + "rast_var, _ = ds.variance_stokes_constant(sections=sections, st_label=\"rast\")" ] }, { @@ -212,7 +212,7 @@ }, "outputs": [], "source": [ - "resid.plot(figsize=(12, 4))\n" + "resid.plot(figsize=(12, 4))" ] }, { @@ -242,7 +242,7 @@ " ast_var=ast_var,\n", " rst_var=rst_var,\n", " rast_var=rast_var,\n", - ")\n" + ")" ] }, { @@ -258,7 +258,7 @@ }, "outputs": [], "source": [ - "ds.tmpw.plot(figsize=(12, 4))\n" + "ds.tmpw.plot(figsize=(12, 4))" ] }, { @@ -296,7 +296,7 @@ "ds.tmpw_var.plot(figsize=(12, 4))\n", "ds1 = ds.isel(time=-1) # take only the first timestep\n", "(ds1.tmpw_var**0.5).plot(figsize=(12, 4))\n", - "plt.gca().set_ylabel(\"Standard error ($^\\circ$C)\")\n" + "plt.gca().set_ylabel(\"Standard error ($^\\circ$C)\")" ] }, { @@ -333,7 +333,7 @@ " mc_sample_size=500,\n", ") # < increase sample size for better approximation\n", "\n", - "ds.tmpw_mc_var.plot(figsize=(12, 4))\n" + "ds.tmpw_mc_var.plot(figsize=(12, 4))" ] }, { @@ -353,7 +353,7 @@ "ds1.tmpw.plot(linewidth=0.7, figsize=(12, 4))\n", "ds1.tmpw_mc.isel(CI=0).plot(linewidth=0.7, label=\"CI: 2.5%\")\n", "ds1.tmpw_mc.isel(CI=1).plot(linewidth=0.7, label=\"CI: 97.5%\")\n", - "plt.legend(fontsize=\"small\")\n" + "plt.legend(fontsize=\"small\")" ] }, { @@ -382,7 +382,7 @@ "(ds1.tmpb_var**0.5).plot()\n", "(ds1.tmpw_var**0.5).plot()\n", "(ds1.tmpw_mc_var**0.5).plot()\n", - "plt.ylabel(\"$\\sigma$ ($^\\circ$C)\")\n" + "plt.ylabel(\"$\\sigma$ ($^\\circ$C)\")" ] }, { diff --git a/docs/notebooks/12Datastore_from_numpy_arrays.ipynb b/docs/notebooks/12Datastore_from_numpy_arrays.ipynb index 1f3d3ce7..9cd233a5 100644 --- a/docs/notebooks/12Datastore_from_numpy_arrays.ipynb +++ b/docs/notebooks/12Datastore_from_numpy_arrays.ipynb @@ -26,7 +26,7 @@ "import matplotlib.pyplot as plt\n", "import xarray as xr\n", "\n", - "from dtscalibration import DataStore, read_silixa_files\n" + "from dtscalibration import DataStore, read_silixa_files" ] }, { @@ -61,7 +61,7 @@ "source": [ "filepath = os.path.join(\"..\", \"..\", \"tests\", \"data\", \"single_ended\")\n", "\n", - "ds_silixa = read_silixa_files(directory=filepath, silent=True)\n" + "ds_silixa = read_silixa_files(directory=filepath, silent=True)" ] }, { @@ -89,7 +89,7 @@ "x = ds_silixa.x.values\n", "time = ds_silixa.time.values\n", "ST = ds_silixa.st.values\n", - "AST = ds_silixa.ast.values\n" + "AST = ds_silixa.ast.values" ] }, { @@ -116,7 +116,7 @@ "ds[\"x\"] = (\"x\", x)\n", "ds[\"time\"] = (\"time\", time)\n", "ds[\"st\"] = ([\"x\", \"time\"], ST)\n", - "ds[\"ast\"] = ([\"x\", \"time\"], AST)\n" + "ds[\"ast\"] = ([\"x\", \"time\"], AST)" ] }, { @@ -133,7 +133,7 @@ "outputs": [], "source": [ "ds = DataStore(ds)\n", - "print(ds)\n" + "print(ds)" ] }, { @@ -169,7 +169,7 @@ "ds[\"temp1\"] = ds_silixa[\"probe1Temperature\"]\n", "ds[\"temp2\"] = ds_silixa[\"probe2Temperature\"]\n", "\n", - "ds.attrs[\"isDoubleEnded\"] = \"0\"\n" + "ds.attrs[\"isDoubleEnded\"] = \"0\"" ] }, { @@ -202,7 +202,7 @@ "ast_var, _ = ds.variance_stokes_constant(sections=sections, st_label=\"ast\")\n", "ds.calibration_single_ended(sections=sections, st_var=st_var, ast_var=ast_var)\n", "\n", - "ds.isel(time=0).tmpf.plot()\n" + "ds.isel(time=0).tmpf.plot()" ] }, { @@ -211,7 +211,7 @@ "metadata": {}, "outputs": [], "source": [ - "ds\n" + "ds" ] }, { diff --git a/docs/notebooks/13Fixed_parameter_calibration.ipynb b/docs/notebooks/13Fixed_parameter_calibration.ipynb index cf5a78c1..fc2642ae 100644 --- a/docs/notebooks/13Fixed_parameter_calibration.ipynb +++ b/docs/notebooks/13Fixed_parameter_calibration.ipynb @@ -76,7 +76,7 @@ " ast_var=ast_var,\n", " fix_gamma=fix_gamma,\n", " fix_dalpha=fix_dalpha,\n", - ")\n" + ")" ] }, { @@ -100,7 +100,7 @@ "outputs": [], "source": [ "print(\"gamma used in calibration:\", ds100.gamma.values)\n", - "print(\"dalpha used in calibration:\", ds100.dalpha.values)\n" + "print(\"dalpha used in calibration:\", ds100.dalpha.values)" ] }, { @@ -132,7 +132,7 @@ " linewidth=1, label=\"Device calibrated\"\n", ") # plot the temperature calibrated by the device\n", "plt.title(\"Temperature at the first time step\")\n", - "plt.legend()\n" + "plt.legend()" ] }, { diff --git a/docs/notebooks/14Lossy_splices.ipynb b/docs/notebooks/14Lossy_splices.ipynb index 5e41b774..977969f3 100644 --- a/docs/notebooks/14Lossy_splices.ipynb +++ b/docs/notebooks/14Lossy_splices.ipynb @@ -72,7 +72,7 @@ "sections = {\n", " \"probe1Temperature\": [slice(7.5, 17.0), slice(70.0, 80.0)], # cold bath\n", " \"probe2Temperature\": [slice(24.0, 34.0), slice(85.0, 95.0)], # warm bath\n", - "}\n" + "}" ] }, { @@ -101,7 +101,7 @@ "ds[\"ast\"] = ds.ast.where(ds.x < 50, ds.ast * 0.82)\n", "\n", "ds[\"rst\"] = ds.rst.where(ds.x > 50, ds.rst * 0.85)\n", - "ds[\"rast\"] = ds.rast.where(ds.x > 50, ds.rast * 0.81)\n" + "ds[\"rast\"] = ds.rast.where(ds.x > 50, ds.rast * 0.81)" ] }, { @@ -121,7 +121,7 @@ "ds.isel(time=0).ast.plot(label=\"ast\")\n", "ds.isel(time=0).rst.plot(label=\"rst\")\n", "ds.isel(time=0).rast.plot(label=\"rast\")\n", - "plt.legend()\n" + "plt.legend()" ] }, { @@ -159,7 +159,7 @@ " rast_var=rast_var,\n", ")\n", "\n", - "ds_a.isel(time=0).tmpw.plot(label=\"calibrated\")\n" + "ds_a.isel(time=0).tmpw.plot(label=\"calibrated\")" ] }, { @@ -200,7 +200,7 @@ "\n", "ds_a.isel(time=0).tmpw.plot(label=\"no trans. att.\")\n", "ds.isel(time=0).tmpw.plot(label=\"with trans. att.\")\n", - "plt.legend()\n" + "plt.legend()" ] }, { diff --git a/docs/notebooks/15Matching_sections.ipynb b/docs/notebooks/15Matching_sections.ipynb index cb709b50..3017c04b 100644 --- a/docs/notebooks/15Matching_sections.ipynb +++ b/docs/notebooks/15Matching_sections.ipynb @@ -68,7 +68,7 @@ "sections = {\n", " \"probe1Temperature\": [slice(7.5, 17.0)], # cold bath\n", " \"probe2Temperature\": [slice(24.0, 34.0)], # warm bath\n", - "}\n" + "}" ] }, { @@ -95,7 +95,7 @@ "ds[\"ast\"] = ds.ast.where(ds.x < 50, ds.ast * 0.82)\n", "\n", "ds[\"rst\"] = ds.rst.where(ds.x > 50, ds.rst * 0.85)\n", - "ds[\"rast\"] = ds.rast.where(ds.x > 50, ds.rast * 0.81)\n" + "ds[\"rast\"] = ds.rast.where(ds.x > 50, ds.rast * 0.81)" ] }, { @@ -135,7 +135,7 @@ " rast_var=rast_var,\n", ")\n", "\n", - "ds_a.isel(time=0).tmpw.plot(label=\"calibrated\")\n" + "ds_a.isel(time=0).tmpw.plot(label=\"calibrated\")" ] }, { @@ -185,7 +185,7 @@ "\n", "ds_a.isel(time=0).tmpw.plot(label=\"normal calibration\")\n", "ds.isel(time=0).tmpw.plot(label=\"matching sections\")\n", - "plt.legend()\n" + "plt.legend()" ] }, { diff --git a/docs/notebooks/16Averaging_temperatures.ipynb b/docs/notebooks/16Averaging_temperatures.ipynb index 76925061..2d97bb85 100644 --- a/docs/notebooks/16Averaging_temperatures.ipynb +++ b/docs/notebooks/16Averaging_temperatures.ipynb @@ -63,7 +63,7 @@ "sections = {\n", " \"probe1Temperature\": [slice(7.5, 17.0), slice(70.0, 80.0)], # cold bath\n", " \"probe2Temperature\": [slice(24.0, 34.0), slice(85.0, 95.0)], # warm bath\n", - "}\n" + "}" ] }, { @@ -82,7 +82,7 @@ "st_var, resid = ds.variance_stokes(sections=sections, st_label=\"st\")\n", "ast_var, _ = ds.variance_stokes(sections=sections, st_label=\"ast\")\n", "rst_var, _ = ds.variance_stokes(sections=sections, st_label=\"rst\")\n", - "rast_var, _ = ds.variance_stokes(sections=sections, st_label=\"rast\")\n" + "rast_var, _ = ds.variance_stokes(sections=sections, st_label=\"rast\")" ] }, { @@ -104,7 +104,7 @@ " ast_var=ast_var,\n", " rst_var=rst_var,\n", " rast_var=rast_var,\n", - ")\n" + ")" ] }, { @@ -191,7 +191,7 @@ " ci_avg_time_isel=[0, 1, 2, 3, 4, 5],\n", " ci_avg_time_sel=None,\n", ")\n", - "ds.tmpw_mc_avg1.plot(hue=\"CI\", linewidth=0.8)\n" + "ds.tmpw_mc_avg1.plot(hue=\"CI\", linewidth=0.8)" ] }, { @@ -252,7 +252,7 @@ " ci_avg_time_isel=[0, 1, 2, 3, 4, 5],\n", " ci_avg_time_sel=None,\n", ")\n", - "ds.tmpw_mc_avg2.plot(hue=\"CI\", linewidth=0.8)\n" + "ds.tmpw_mc_avg2.plot(hue=\"CI\", linewidth=0.8)" ] }, { @@ -313,7 +313,7 @@ " ci_avg_x_sel=slice(7.5, 17.0),\n", " ci_avg_x_isel=None,\n", ")\n", - "ds.tmpw_mc_avgx1.plot(hue=\"CI\", linewidth=0.8)\n" + "ds.tmpw_mc_avgx1.plot(hue=\"CI\", linewidth=0.8)" ] }, { @@ -374,7 +374,7 @@ " ci_avg_x_sel=slice(7.5, 17.0),\n", " ci_avg_x_isel=None,\n", ")\n", - "ds.tmpw_mc_avgx2.plot(hue=\"CI\", linewidth=0.8)\n" + "ds.tmpw_mc_avgx2.plot(hue=\"CI\", linewidth=0.8)" ] }, { diff --git a/docs/notebooks/17Temperature_uncertainty_single_ended.ipynb b/docs/notebooks/17Temperature_uncertainty_single_ended.ipynb index 478a490e..301e4963 100644 --- a/docs/notebooks/17Temperature_uncertainty_single_ended.ipynb +++ b/docs/notebooks/17Temperature_uncertainty_single_ended.ipynb @@ -73,7 +73,7 @@ "plt.fill_between(ds1.x, y1=ds1.tmpf_var, y2=stast_var, label=\"Parameter estimation\")\n", "plt.suptitle(\"Variance of the estimated temperature\")\n", "plt.ylabel(\"$\\sigma^2$ ($^\\circ$C$^2$)\")\n", - "plt.legend(fontsize=\"small\")\n" + "plt.legend(fontsize=\"small\")" ] }, { @@ -84,7 +84,7 @@ "outputs": [], "source": [ "# The effects of the parameter uncertainty can be further inspected\n", - "ds1.var_fw_da.plot(hue=\"comp_fw\", figsize=(12, 4))\n" + "ds1.var_fw_da.plot(hue=\"comp_fw\", figsize=(12, 4))" ] }, { @@ -121,7 +121,7 @@ "source": [ "ds.conf_int_single_ended(\n", " st_var=st_var, ast_var=ast_var, conf_ints=[2.5, 97.5], mc_sample_size=500\n", - ")\n" + ")" ] }, { @@ -145,7 +145,7 @@ "(ds1.tmpf_mc_var**0.5).plot(figsize=(12, 4), label=\"Monte Carlo approx.\")\n", "(ds1.tmpf_var**0.5).plot(label=\"Linear error approx.\")\n", "plt.ylabel(\"$\\sigma$ ($^\\circ$C)\")\n", - "plt.legend(fontsize=\"small\")\n" + "plt.legend(fontsize=\"small\")" ] }, { @@ -166,7 +166,7 @@ "ds1.tmpf.plot(linewidth=0.7, figsize=(12, 4))\n", "ds1.tmpf_mc.sel(CI=2.5).plot(linewidth=0.7, label=\"CI: 2.5%\")\n", "ds1.tmpf_mc.sel(CI=97.5).plot(linewidth=0.7, label=\"CI: 97.5%\")\n", - "plt.legend(fontsize=\"small\")\n" + "plt.legend(fontsize=\"small\")" ] }, { From ac0ef5f43e4106b36d91e9d6302788b28c639368 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Thu, 21 Sep 2023 15:11:14 -0600 Subject: [PATCH 22/66] Estimate variance of stokes refactored --- src/dtscalibration/__init__.py | 1 + .../calibration/section_utils.py | 175 ++++-- src/dtscalibration/calibration/utils.py | 0 src/dtscalibration/datastore_accessor.py | 233 ++++++++ src/dtscalibration/variance_stokes.py | 498 ++++++++++++++++++ tests/test_dtscalibration.py | 194 ------- tests/test_variance_stokes.py | 243 +++++++++ 7 files changed, 1111 insertions(+), 233 deletions(-) delete mode 100644 src/dtscalibration/calibration/utils.py create mode 100644 src/dtscalibration/datastore_accessor.py create mode 100644 src/dtscalibration/variance_stokes.py create mode 100644 tests/test_variance_stokes.py diff --git a/src/dtscalibration/__init__.py b/src/dtscalibration/__init__.py index 4ecabb02..d173d689 100644 --- a/src/dtscalibration/__init__.py +++ b/src/dtscalibration/__init__.py @@ -1,4 +1,5 @@ from dtscalibration.datastore import DataStore +from dtscalibration.datastore_accessor import DtsAccessor from dtscalibration.datastore_utils import check_dims from dtscalibration.datastore_utils import check_timestep_allclose from dtscalibration.datastore_utils import get_netcdf_encoding diff --git a/src/dtscalibration/calibration/section_utils.py b/src/dtscalibration/calibration/section_utils.py index 2d1ef100..3e13709a 100644 --- a/src/dtscalibration/calibration/section_utils.py +++ b/src/dtscalibration/calibration/section_utils.py @@ -15,60 +15,157 @@ def set_sections(ds: xr.Dataset, sections: dict[str, list[slice]]) -> xr.Dataset return ds -def validate_sections(ds: xr.Dataset, sections: dict[str, list[slice]]): - assert isinstance(sections, dict) +def validate_no_overlapping_sections(sections: dict[str, list[slice]]): + """ + Check if the sections do not overlap. + + Parameters + ---------- + sections : dict[str, list[slice]] + The keys of the dictionary are the names of the sections. + The values are lists of slice objects. - # be less restrictive for capitalized labels - # find lower cases label - labels = np.reshape([[s.lower(), s] for s in ds.data_vars.keys()], (-1,)).tolist() + Returns + ------- + None + + Raises + ------ + AssertionError + If the sections overlap. + """ + all_stretches = list() - sections_fix = dict() for k, v in sections.items(): - if k.lower() in labels: - i_lower_case = labels.index(k.lower()) - i_normal_case = i_lower_case + 1 - k_normal_case = labels[i_normal_case] - sections_fix[k_normal_case] = v - else: - assert k in ds.data_vars, ( - "The keys of the " - "sections-dictionary should " - "refer to a valid timeserie " - "already stored in " - "ds.data_vars " - ) + for vi in v: + all_stretches.append(vi) + + # Check for overlapping slices + all_start_stop = [[stretch.start, stretch.stop] for stretch in all_stretches] + isorted_start = np.argsort([i[0] for i in all_start_stop]) + all_start_stop_startsort = [all_start_stop[i] for i in isorted_start] + all_start_stop_startsort_flat = sum(all_start_stop_startsort, []) + assert all_start_stop_startsort_flat == sorted( + all_start_stop_startsort_flat + ), "Sections contains overlapping stretches" + pass + + +def validate_sections_definition(sections: dict[str, list[slice]]): + """ + Check if the sections are defined correctly. The sections are defined + correctly if: + - The keys of the sections-dictionary are strings (assertion) + - The values of the sections-dictionary are lists (assertion) + + Parameters + ---------- + sections : dict[str, list[slice]] + The keys of the dictionary are the names of the sections. + The values are lists of slice objects. + + Returns + ------- + None + + Raises + ------ + AssertionError + If the sections are not defined correctly. + """ + assert isinstance(sections, dict) - sections_fix_slice_fixed = dict() + for k, v in sections.items(): + assert isinstance(k, str), ( + "The keys of the " + "sections-dictionary should " + "be strings" + ) - for k, v in sections_fix.items(): assert isinstance(v, (list, tuple)), ( "The values of the sections-dictionary " "should be lists of slice objects." ) - for vi in v: - assert isinstance(vi, slice), ( - "The values of the sections-dictionary should " - "be lists of slice objects." - ) +def validate_sections(ds: xr.Dataset, sections: dict[str, list[slice]]): + """ + Check if the sections are valid. The sections are valid if: + - The keys of the sections-dictionary refer to a valid timeserie + already stored in ds.data_vars (assertion) + - The values of the sections-dictionary are lists of slice objects. + (assertion) + - The slices are within the x-dimension (assertion) + - The slices do not overlap (assertion) + + Parameters + ---------- + ds : xr.Dataset + The dataset that contains the timeseries that are referred to in + the sections-dictionary. + sections : dict[str, list[slice]] + The keys of the dictionary are the names of the sections. + The values are lists of slice objects. + + Returns + ------- + None + + Raises + ------ + AssertionError + If the sections are not valid. + """ + validate_sections_definition(sections=sections) + validate_no_overlapping_sections(sections=sections) + + for k, v in sections.items(): + assert k in ds.data_vars, ( + "The keys of the " + "sections-dictionary should " + "refer to a valid timeserie " + "already stored in " + "ds.data_vars " + ) + + for vi in v: assert ds.x.sel(x=vi).size > 0, ( f"Better define the {k} section. You tried {vi}, " "which is not within the x-dimension" ) - # sorted stretches - stretch_unsort = [slice(float(vi.start), float(vi.stop)) for vi in v] - stretch_start = [i.start for i in stretch_unsort] - stretch_i_sorted = np.argsort(stretch_start) - sections_fix_slice_fixed[k] = [stretch_unsort[i] for i in stretch_i_sorted] - - # Prevent overlapping slices - ix_sec = ufunc_per_section( - ds, sections=sections_fix_slice_fixed, x_indices=True, calc_per="all" - ) - assert np.unique(ix_sec).size == ix_sec.size, "The sections are overlapping" - - return sections_fix_slice_fixed + # for k, v in sections.items(): + # assert isinstance(v, (list, tuple)), ( + # "The values of the sections-dictionary " "should be lists of slice objects." + # ) + + # for vi in v: + # assert isinstance(vi, slice), ( + # "The values of the sections-dictionary should " + # "be lists of slice objects." + # ) + + # assert ds.x.sel(x=vi).size > 0, ( + # f"Better define the {k} section. You tried {vi}, " + # "which is not within the x-dimension" + # ) + + # # sorted stretches + # stretch_unsort = [slice(float(vi.start), float(vi.stop)) for vi in v] + # stretch_start = [i.start for i in stretch_unsort] + # stretch_i_sorted = np.argsort(stretch_start) + # sections_fix_slice_fixed[k] = [stretch_unsort[i] for i in stretch_i_sorted] + # all_stretches.extend(sections_fix_slice_fixed[k]) + + # # Check for overlapping slices + # all_start_stop = [[stretch.start, stretch.stop] for stretch in all_stretches] + # isorted_start = np.argsort([i[0] for i in all_start_stop]) + # all_start_stop_startsort = [all_start_stop[i] for i in isorted_start] + # all_start_stop_startsort_flat = sum(all_start_stop_startsort, []) + # assert all_start_stop_startsort_flat == sorted( + # all_start_stop_startsort_flat), \ + # "Sections contains overlapping stretches" + + pass def ufunc_per_section( diff --git a/src/dtscalibration/calibration/utils.py b/src/dtscalibration/calibration/utils.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/dtscalibration/datastore_accessor.py b/src/dtscalibration/datastore_accessor.py new file mode 100644 index 00000000..66c0b16a --- /dev/null +++ b/src/dtscalibration/datastore_accessor.py @@ -0,0 +1,233 @@ +import dask.array as da +import numpy as np +import xarray as xr +import yaml + +from dtscalibration.datastore_utils import check_timestep_allclose + + +@xr.register_dataset_accessor("dts") +class DtsAccessor: + def __init__(self, xarray_obj): + # check xarray_obj + # check naming convention + assert ( + ("st" in xarray_obj.data_vars) and + ("ast" in xarray_obj.data_vars) and + ("userAcquisitionTimeFW" in xarray_obj.data_vars)), \ + "xarray_obj should have st, ast, userAcquisitionTimeFW" + + # Varying acquisition times not supported. Could be in the future. + # Should actually be moved to the estimating variance functions + check_timestep_allclose(xarray_obj, eps=0.01) + + # cache xarray_obj + self._obj = xarray_obj + self.attrs = xarray_obj.attrs + + # alias commonly used variables + self.x = xarray_obj.x + self.time = xarray_obj.time + self.transatt = xarray_obj.get("transatt") + + self.st = xarray_obj["st"] # required + self.ast = xarray_obj["ast"] # required + self.rst = xarray_obj.get("rst") # None if doesn't exist + self.rast = xarray_obj.get("rast") # None is doesn't exist + + # alias commonly computed variables + self.nx = self.x.size + self.nt = self.time.size + if self.transatt: + self.nta = self.transatt.size + else: + self.nta = 0 + + pass + + def __repr__(self): + # __repr__ from xarray is used and edited. + # 'xarray' is prepended. so we remove it and add 'dtscalibration' + s = xr.core.formatting.dataset_repr(self._obj) + name_module = type(self._obj).__name__ + preamble_new = "" % name_module + + # Add sections to new preamble + preamble_new += "\nSections:" + if hasattr(self._obj, "_sections") and self.sections: + preamble_new += "\n" + unit = self.x.attrs.get("unit", "") + + for k, v in self.sections.items(): + preamble_new += f" {k: <23}" + + # Compute statistics reference section timeseries + sec_stat = f"({float(self._obj[k].mean()):6.2f}" + sec_stat += f" +/-{float(self._obj[k].std()):5.2f}" + sec_stat += "\N{DEGREE SIGN}C)\t" + preamble_new += sec_stat + + # print sections + vl = [f"{vi.start:.2f}{unit} - {vi.stop:.2f}{unit}" for vi in v] + preamble_new += " and ".join(vl) + "\n" + + else: + preamble_new += 18 * " " + "()\n" + + # add new preamble to the remainder of the former __repr__ + len_preamble_old = 8 + len(name_module) + 2 + + # untill the attribute listing + attr_index = s.find("Attributes:") + + # abbreviate attribute listing + attr_list_all = s[attr_index:].split(sep="\n") + if len(attr_list_all) > 10: + s_too_many = ["\n.. and many more attributes. See: ds.attrs"] + attr_list = attr_list_all[:10] + s_too_many + else: + attr_list = attr_list_all + + s_out = preamble_new + s[len_preamble_old:attr_index] + "\n".join(attr_list) + + # return new __repr__ + return s_out + + # noinspection PyIncorrectDocstring + @property + def sections(self): + """ + Define calibration sections. Each section requires a reference + temperature time series, such as the temperature measured by an + external temperature sensor. They should already be part of the + DataStore object. + + Please look at the example notebook on `sections` if you encounter + difficulties. + + Parameters + ---------- + sections : Dict[str, List[slice]] + Sections are defined in a dictionary with its keywords of the + names of the reference + temperature time series. Its values are lists of slice objects, + where each slice object + is a stretch. + Returns + ------- + + """ + if "_sections" not in self._obj.attrs: + self._obj.attrs["_sections"] = yaml.dump(None) + + return yaml.load(self._obj.attrs["_sections"], Loader=yaml.UnsafeLoader) + + @sections.deleter + def sections(self): + self._obj.attrs["_sections"] = yaml.dump(None) + + @sections.setter + def sections(self, value): + msg = ( + "Not possible anymore. Instead, pass the sections as an argument to \n" + "ds.dts.calibrate_single_ended() or ds.dts.calibrate_double_ended()." + ) + raise NotImplementedError(msg) + + def get_default_encoding(self, time_chunks_from_key=None): + """ + Returns a dictionary with sensible compression setting for writing + netCDF files. + + Returns + ------- + + """ + # The following variables are stored with a sufficiently large + # precision in 32 bit + float32l = [ + "st", + "ast", + "rst", + "rast", + "time", + "timestart", + "tmp", + "timeend", + "acquisitionTime", + "x", + ] + int32l = [ + "filename_tstamp", + "acquisitiontimeFW", + "acquisitiontimeBW", + "userAcquisitionTimeFW", + "userAcquisitionTimeBW", + ] + + # default variable compression + compdata = dict( + zlib=True, complevel=6, shuffle=False + ) # , least_significant_digit=None + + # default coordinate compression + compcoords = dict(zlib=True, complevel=4) + + # construct encoding dict + encoding = {var: compdata.copy() for var in self._obj.data_vars} + encoding.update({var: compcoords.copy() for var in self._obj.coords}) + + for k, v in encoding.items(): + if k in float32l: + v["dtype"] = "float32" + + if k in int32l: + v["dtype"] = "int32" + # v['_FillValue'] = -9999 # Int does not support NaN + + if np.issubdtype(self._obj[k].dtype, str) or np.issubdtype( + self._obj[k].dtype, object + ): + # Compression not supported for variable length strings + # https://github.com/Unidata/netcdf4-python/issues/1205 + v["zlib"] = False + + if time_chunks_from_key is not None: + # obtain optimal chunk sizes in time and x dim + if self[time_chunks_from_key].dims == ("x", "time"): + x_chunk, t_chunk = da.ones( + self[time_chunks_from_key].shape, + chunks=(-1, "auto"), + dtype="float32", + ).chunks + + elif self[time_chunks_from_key].dims == ("time", "x"): + x_chunk, t_chunk = da.ones( + self[time_chunks_from_key].shape, + chunks=("auto", -1), + dtype="float32", + ).chunks + else: + assert 0, "something went wrong with your Stokes dimensions" + + for k, v in encoding.items(): + # By writing and compressing the data in chunks, some sort of + # parallism is possible. + if self[k].dims == ("x", "time"): + chunks = (x_chunk[0], t_chunk[0]) + + elif self[k].dims == ("time", "x"): + chunks = (t_chunk[0], x_chunk[0]) + + elif self[k].dims == ("x",): + chunks = (x_chunk[0],) + + elif self[k].dims == ("time",): + chunks = (t_chunk[0],) + + else: + continue + + v["chunksizes"] = chunks + + return encoding diff --git a/src/dtscalibration/variance_stokes.py b/src/dtscalibration/variance_stokes.py new file mode 100644 index 00000000..0393f259 --- /dev/null +++ b/src/dtscalibration/variance_stokes.py @@ -0,0 +1,498 @@ +import dask.array as da +import numpy as np +import xarray as xr + +from dtscalibration.datastore_utils import ufunc_per_section_helper +from dtscalibration.variance_helpers import variance_stokes_constant_helper +from dtscalibration.variance_helpers import variance_stokes_exponential_helper +from dtscalibration.variance_helpers import variance_stokes_linear_helper +from dtscalibration.calibration.section_utils import validate_sections_definition +from dtscalibration.calibration.section_utils import validate_no_overlapping_sections + + +def variance_stokes_constant(st, sections, reshape_residuals=True): + """ + Approximate the variance of the noise in Stokes intensity measurements + with one value, suitable for small setups. + + * `ds.variance_stokes_constant()` for small setups with small variations in\ + intensity. Variance of the Stokes measurements is assumed to be the same\ + along the entire fiber. + + * `ds.variance_stokes_exponential()` for small setups with very few time\ + steps. Too many degrees of freedom results in an under estimation of the\ + noise variance. Almost never the case, but use when calibrating pre time\ + step. + + * `ds.variance_stokes_linear()` for larger setups with more time steps.\ + Assumes Poisson distributed noise with the following model:: + + st_var = a * ds.st + b + + + where `a` and `b` are constants. Requires reference sections at + beginning and end of the fiber, to have residuals at high and low + intensity measurements. + + The Stokes and anti-Stokes intensities are measured with detectors, + which inherently introduce noise to the measurements. Knowledge of the + distribution of the measurement noise is needed for a calibration with + weighted observations (Sections 5 and 6 of [1]_) + and to project the associated uncertainty to the temperature confidence + intervals (Section 7 of [1]_). Two sources dominate the noise + in the Stokes and anti-Stokes intensity measurements + (Hartog, 2017, p.125). Close to the laser, noise from the conversion of + backscatter to electricity dominates the measurement noise. The + detecting component, an avalanche photodiode, produces Poisson- + distributed noise with a variance that increases linearly with the + intensity. The Stokes and anti-Stokes intensities are commonly much + larger than the standard deviation of the noise, so that the Poisson + distribution can be approximated with a Normal distribution with a mean + of zero and a variance that increases linearly with the intensity. At + the far-end of the fiber, noise from the electrical circuit dominates + the measurement noise. It produces Normal-distributed noise with a mean + of zero and a variance that is independent of the intensity. + + Calculates the variance between the measurements and a best fit + at each reference section. This fits a function to the nt * nx + measurements with ns * nt + nx parameters, where nx are the total + number of reference locations along all sections. The temperature is + constant along the reference sections, so the expression of the + Stokes power can be split in a time series per reference section and + a constant per observation location. + + Idea from Discussion at page 127 in Richter, P. H. (1995). Estimating + errors in least-squares fitting. + + The timeseries and the constant are, of course, highly correlated + (Equations 20 and 21 in [1]_), but that is not relevant here as only the + product is of interest. The residuals between the fitted product and the + Stokes intensity measurements are attributed to the + noise from the detector. The variance of the residuals is used as a + proxy for the variance of the noise in the Stokes and anti-Stokes + intensity measurements. A non-uniform temperature of + the reference sections results in an over estimation of the noise + variance estimate because all temperature variation is attributed to + the noise. + + Parameters + ---------- + reshape_residuals + st : DataArray + sections : Dict[str, List[slice]] + + Returns + ------- + I_var : float + Variance of the residuals between measured and best fit + resid : array_like + Residuals between measured and best fit + + Notes + ----- + + * Because there are a large number of unknowns, spend time on\ + calculating an initial estimate. Can be turned off by setting to False. + + * It is often not needed to use measurements from all time steps. If\ + your variance estimate does not change when including measurements\ + additional time steps, you have included enough measurements. + + References + ---------- + .. [1] des Tombe, B., Schilperoort, B., & Bakker, M. (2020). Estimation + of Temperature and Associated Uncertainty from Fiber-Optic Raman- + Spectrum Distributed Temperature Sensing. Sensors, 20(8), 2235. + https://doi.org/10.3390/s20082235 + + Examples + -------- + - `Example notebook 4: Calculate variance Stokes intensity measurements\ + `_ + + TODO: Account for varying acquisition times + """ + validate_sections_definition(sections=sections) + validate_no_overlapping_sections(sections=sections) + + assert st.dims[0] == "x", "DataArray is transposed" + + # should maybe be per section. But then residuals + # seem to be correlated between stretches. I don't know why.. BdT. + data_dict = da.compute( + ufunc_per_section_helper( + sections=sections, + dataarray=st, + calc_per="stretch" + ) + )[0] + + var_I, resid = variance_stokes_constant_helper(data_dict) + + if not reshape_residuals: + return var_I, resid + + else: + ix_resid = ufunc_per_section_helper( + sections=sections, x_coords=st.x, calc_per="all" + ) + + resid_sorted = np.full(shape=st.shape, fill_value=np.nan) + resid_sorted[ix_resid, :] = resid + resid_da = xr.DataArray(data=resid_sorted, coords=st.coords) + + return var_I, resid_da + + +def variance_stokes_exponential( + st, + sections, + use_statsmodels=False, + suppress_info=True, + reshape_residuals=True, +): + """ + Approximate the variance of the noise in Stokes intensity measurements + with one value, suitable for small setups with measurements from only + a few times. + + * `ds.variance_stokes_constant()` for small setups with small variations in\ + intensity. Variance of the Stokes measurements is assumed to be the same\ + along the entire fiber. + + * `ds.variance_stokes_exponential()` for small setups with very few time\ + steps. Too many degrees of freedom results in an under estimation of the\ + noise variance. Almost never the case, but use when calibrating pre time\ + step. + + * `ds.variance_stokes_linear()` for larger setups with more time steps.\ + Assumes Poisson distributed noise with the following model:: + + st_var = a * ds.st + b + + + where `a` and `b` are constants. Requires reference sections at + beginning and end of the fiber, to have residuals at high and low + intensity measurements. + + The Stokes and anti-Stokes intensities are measured with detectors, + which inherently introduce noise to the measurements. Knowledge of the + distribution of the measurement noise is needed for a calibration with + weighted observations (Sections 5 and 6 of [1]_) + and to project the associated uncertainty to the temperature confidence + intervals (Section 7 of [1]_). Two sources dominate the noise + in the Stokes and anti-Stokes intensity measurements + (Hartog, 2017, p.125). Close to the laser, noise from the conversion of + backscatter to electricity dominates the measurement noise. The + detecting component, an avalanche photodiode, produces Poisson- + distributed noise with a variance that increases linearly with the + intensity. The Stokes and anti-Stokes intensities are commonly much + larger than the standard deviation of the noise, so that the Poisson + distribution can be approximated with a Normal distribution with a mean + of zero and a variance that increases linearly with the intensity. At + the far-end of the fiber, noise from the electrical circuit dominates + the measurement noise. It produces Normal-distributed noise with a mean + of zero and a variance that is independent of the intensity. + + Calculates the variance between the measurements and a best fit + at each reference section. This fits a function to the nt * nx + measurements with ns * nt + nx parameters, where nx are the total + number of reference locations along all sections. The temperature is + constant along the reference sections. This fits a two-parameter + exponential to the stokes measurements. The temperature is constant + and there are no splices/sharp bends in each reference section. + Therefore all signal decrease is due to differential attenuation, + which is the same for each reference section. The scale of the + exponential does differ per reference section. + + Assumptions: 1) the temperature is the same along a reference + section. 2) no sharp bends and splices in the reference sections. 3) + Same type of optical cable in each reference section. + + Idea from discussion at page 127 in Richter, P. H. (1995). Estimating + errors in least-squares fitting. For weights used error propagation: + w^2 = 1/sigma(lny)^2 = y^2/sigma(y)^2 = y^2 + + The timeseries and the constant are, of course, highly correlated + (Equations 20 and 21 in [1]_), but that is not relevant here as only the + product is of interest. The residuals between the fitted product and the + Stokes intensity measurements are attributed to the + noise from the detector. The variance of the residuals is used as a + proxy for the variance of the noise in the Stokes and anti-Stokes + intensity measurements. A non-uniform temperature of + the reference sections results in an over estimation of the noise + variance estimate because all temperature variation is attributed to + the noise. + + Parameters + ---------- + suppress_info : bool, optional + Suppress print statements. + use_statsmodels : bool, optional + Use statsmodels to fit the exponential. If `False`, use scipy. + reshape_residuals : bool, optional + Reshape the residuals to the shape of the Stokes intensity + st_label : str + label of the Stokes, anti-Stokes measurement. + E.g., st, ast, rst, rast + sections : Dict[str, List[slice]], optional + If `None` is supplied, `ds.sections` is used. Define calibration + sections. Each section requires a reference temperature time series, + such as the temperature measured by an external temperature sensor. + They should already be part of the DataStore object. `sections` + is defined with a dictionary with its keywords of the + names of the reference temperature time series. Its values are + lists of slice objects, where each slice object is a fiber stretch + that has the reference temperature. Afterwards, `sections` is stored + under `ds.sections`. + + Returns + ------- + I_var : float + Variance of the residuals between measured and best fit + resid : array_like + Residuals between measured and best fit + + Notes + ----- + + * Because there are a large number of unknowns, spend time on\ + calculating an initial estimate. Can be turned off by setting to False. + + * It is often not needed to use measurements from all time steps. If\ + your variance estimate does not change when including measurements from\ + more time steps, you have included enough measurements. + + References + ---------- + .. [1] des Tombe, B., Schilperoort, B., & Bakker, M. (2020). Estimation + of Temperature and Associated Uncertainty from Fiber-Optic Raman- + Spectrum Distributed Temperature Sensing. Sensors, 20(8), 2235. + https://doi.org/10.3390/s20082235 + + Examples + -------- + - `Example notebook 4: Calculate variance Stokes intensity measurements\ + `_ + """ + validate_sections_definition(sections=sections) + validate_no_overlapping_sections(sections=sections) + + assert st.dims[0] == "x", "Stokes are transposed" + nt = st.coords["time"].size + + # number of reference points per section (spatial) + len_stretch_list = [] + y_list = [] # intensities of stokes + x_list = [] # length rel to start of section. for alpha + + for k, stretches in sections.items(): + for stretch in stretches: + y_list.append(st.sel(x=stretch).data.T.reshape(-1)) + _x = st.coords["x"].sel(x=stretch).data.copy() + _x -= _x[0] + x_list.append(da.tile(_x, nt)) + len_stretch_list.append(_x.size) + + x = np.concatenate(x_list) + y = np.concatenate(y_list) + + var_I, resid = variance_stokes_exponential_helper( + nt, x, y, len_stretch_list, use_statsmodels, suppress_info + ) + + if not reshape_residuals: + return var_I, resid + + else: + # restructure the residuals, such that they can be plotted and + # added to ds + resid_res = [] + for leni, lenis, lenie in zip( + len_stretch_list, + nt * np.cumsum([0] + len_stretch_list[:-1]), + nt * np.cumsum(len_stretch_list), + ): + try: + resid_res.append(resid[lenis:lenie].reshape((leni, nt), order="F")) + except: # noqa: E722 + # Dask array does not support order + resid_res.append(resid[lenis:lenie].T.reshape((nt, leni)).T) + + _resid = np.concatenate(resid_res) + # _resid_x = self.ufunc_per_section( + # sections=sections, label="x", calc_per="all" + # ) + _resid_x = ufunc_per_section_helper(sections=sections, dataarray=st.coords["x"], calc_per="all") + isort = np.argsort(_resid_x) + resid_x = _resid_x[isort] # get indices from ufunc directly + resid = _resid[isort, :] + + ix_resid = np.array([np.argmin(np.abs(ai - st.coords["x"].data)) for ai in resid_x]) + + resid_sorted = np.full(shape=st.shape, fill_value=np.nan) + resid_sorted[ix_resid, :] = resid + resid_da = xr.DataArray(data=resid_sorted, coords=st.coords) + + return var_I, resid_da + + +def variance_stokes_linear( + st, sections, nbin=50, through_zero=False, plot_fit=False +): + """ + Approximate the variance of the noise in Stokes intensity measurements + with a linear function of the intensity, suitable for large setups. + + * `ds.variance_stokes_constant()` for small setups with small variations in\ + intensity. Variance of the Stokes measurements is assumed to be the same\ + along the entire fiber. + + * `ds.variance_stokes_exponential()` for small setups with very few time\ + steps. Too many degrees of freedom results in an under estimation of the\ + noise variance. Almost never the case, but use when calibrating pre time\ + step. + + * `ds.variance_stokes_linear()` for larger setups with more time steps.\ + Assumes Poisson distributed noise with the following model:: + + st_var = a * ds.st + b + + + where `a` and `b` are constants. Requires reference sections at + beginning and end of the fiber, to have residuals at high and low + intensity measurements. + + The Stokes and anti-Stokes intensities are measured with detectors, + which inherently introduce noise to the measurements. Knowledge of the + distribution of the measurement noise is needed for a calibration with + weighted observations (Sections 5 and 6 of [1]_) + and to project the associated uncertainty to the temperature confidence + intervals (Section 7 of [1]_). Two sources dominate the noise + in the Stokes and anti-Stokes intensity measurements + (Hartog, 2017, p.125). Close to the laser, noise from the conversion of + backscatter to electricity dominates the measurement noise. The + detecting component, an avalanche photodiode, produces Poisson- + distributed noise with a variance that increases linearly with the + intensity. The Stokes and anti-Stokes intensities are commonly much + larger than the standard deviation of the noise, so that the Poisson + distribution can be approximated with a Normal distribution with a mean + of zero and a variance that increases linearly with the intensity. At + the far-end of the fiber, noise from the electrical circuit dominates + the measurement noise. It produces Normal-distributed noise with a mean + of zero and a variance that is independent of the intensity. + + Calculates the variance between the measurements and a best fit + at each reference section. This fits a function to the nt * nx + measurements with ns * nt + nx parameters, where nx are the total + number of reference locations along all sections. The temperature is + constant along the reference sections, so the expression of the + Stokes power can be split in a time series per reference section and + a constant per observation location. + + Idea from Discussion at page 127 in Richter, P. H. (1995). Estimating + errors in least-squares fitting. + + The timeseries and the constant are, of course, highly correlated + (Equations 20 and 21 in [1]_), but that is not relevant here as only the + product is of interest. The residuals between the fitted product and the + Stokes intensity measurements are attributed to the + noise from the detector. The variance of the residuals is used as a + proxy for the variance of the noise in the Stokes and anti-Stokes + intensity measurements. A non-uniform temperature of + the reference sections results in an over estimation of the noise + variance estimate because all temperature variation is attributed to + the noise. + + Notes + ----- + + * Because there are a large number of unknowns, spend time on\ + calculating an initial estimate. Can be turned off by setting to False. + + * It is often not needed to use measurements from all time steps. If\ + your variance estimate does not change when including measurements \ + from more time steps, you have included enough measurements. + + References + ---------- + .. [1] des Tombe, B., Schilperoort, B., & Bakker, M. (2020). Estimation + of Temperature and Associated Uncertainty from Fiber-Optic Raman- + Spectrum Distributed Temperature Sensing. Sensors, 20(8), 2235. + https://doi.org/10.3390/s20082235 + + Examples + -------- + - `Example notebook 4: Calculate variance Stokes intensity \ + measurements `_ + + Parameters + ---------- + st_label : str + Key under which the Stokes DataArray is stored. E.g., 'st', 'rst' + sections : dict, optional + Define sections. See documentation + nbin : int + Number of bins to compute the variance for, through which the + linear function is fitted. Make sure that that are at least 50 + residuals per bin to compute the variance from. + through_zero : bool + If True, the variance is computed as: VAR(Stokes) = slope * Stokes + If False, VAR(Stokes) = slope * Stokes + offset. + From what we can tell from our inital trails, is that the offset + seems relatively small, so that True seems a better option for + setups where a reference section with very low Stokes intensities + is missing. If data with low Stokes intensities available, it is + better to not fit through zero, but determine the offset from + the data. + plot_fit : bool + If True plot the variances for each bin and plot the fitted + linear function + """ + validate_sections_definition(sections=sections) + validate_no_overlapping_sections(sections=sections) + + assert st.dims[0] == "x", "Stokes are transposed" + _, resid = variance_stokes_constant( + sections=sections, st=st, reshape_residuals=False + ) + ix_sec = ufunc_per_section_helper(sections=sections, x_coords=st.coords["x"], calc_per="all") + + st = st.isel(x=ix_sec).values.ravel() + diff_st = resid.ravel() + + ( + slope, + offset, + st_sort_mean, + st_sort_var, + resid, + var_fun, + ) = variance_stokes_linear_helper(st, diff_st, nbin, through_zero) + + if plot_fit: + import matplotlib.pyplot as plt + plt.figure() + plt.scatter(st_sort_mean, st_sort_var, marker=".", c="black") + plt.plot( + [0.0, st_sort_mean[-1]], + [var_fun(0.0), var_fun(st_sort_mean[-1])], + c="white", + lw=1.3, + ) + plt.plot( + [0.0, st_sort_mean[-1]], + [var_fun(0.0), var_fun(st_sort_mean[-1])], + c="black", + lw=0.8, + ) + plt.xlabel("intensity") + plt.ylabel("intensity variance") + + return slope, offset, st_sort_mean, st_sort_var, resid, var_fun diff --git a/tests/test_dtscalibration.py b/tests/test_dtscalibration.py index b354c584..376b5a3b 100644 --- a/tests/test_dtscalibration.py +++ b/tests/test_dtscalibration.py @@ -746,206 +746,12 @@ def test_single_ended_variance_estimate_synthetic(): pass -@pytest.mark.skip(reason="Not enough measurements in time. Use exponential " "instead.") -def test_variance_of_stokes(): - correct_var = 9.045 - filepath = data_dir_double_ended2 - ds = read_silixa_files(directory=filepath, timezone_netcdf="UTC", file_ext="*.xml") - sections = { - "probe1Temperature": [slice(7.5, 17.0), slice(70.0, 80.0)], # cold bath - "probe2Temperature": [slice(24.0, 34.0), slice(85.0, 95.0)], # warm bath - } - - I_var, _ = ds.variance_stokes(st_label="st", sections=sections) - assert_almost_equal_verbose(I_var, correct_var, decimal=1) - - ds_dask = ds.chunk(chunks={}) - I_var, _ = ds_dask.variance_stokes(st_label="st", sections=sections) - assert_almost_equal_verbose(I_var, correct_var, decimal=1) - - pass - - -def test_variance_of_stokes_synthetic(): - """ - Produces a synthetic Stokes measurement with a known noise distribution. Check if same - variance is obtained. - - Returns - ------- - - """ - yvar = 5.0 - - nx = 500 - x = np.linspace(0.0, 20.0, nx) - - nt = 200 - G = np.linspace(3000, 4000, nt)[None] - - y = G * np.exp(-0.001 * x[:, None]) - - y += stats.norm.rvs(size=y.size, scale=yvar**0.5).reshape(y.shape) - - ds = DataStore( - { - "st": (["x", "time"], y), - "probe1Temperature": (["time"], range(nt)), - "userAcquisitionTimeFW": (["time"], np.ones(nt)), - }, - coords={"x": x, "time": range(nt)}, - attrs={"isDoubleEnded": "0"}, - ) - - sections = {"probe1Temperature": [slice(0.0, 20.0)]} - test_st_var, _ = ds.variance_stokes(st_label="st", sections=sections) - - assert_almost_equal_verbose(test_st_var, yvar, decimal=1) - - test_st_var, _ = ds.variance_stokes(st_label="st", sections=sections) - - assert_almost_equal_verbose(test_st_var, yvar, decimal=1) - pass - - -@pytest.mark.slow # Execution time ~20 seconds -def test_variance_of_stokes_linear_synthetic(): - """ - Produces a synthetic Stokes measurement with a known noise distribution. - Check if same variance is obtained. - - Returns - ------- - - """ - var_slope = 0.01 - - nx = 500 - x = np.linspace(0.0, 20.0, nx) - - nt = 200 - G = np.linspace(500, 4000, nt)[None] - c_no_noise = G * np.exp(-0.001 * x[:, None]) - - c_lin_var_through_zero = stats.norm.rvs( - loc=c_no_noise, - # size=y.size, - scale=(var_slope * c_no_noise) ** 0.5, - ) - ds = DataStore( - { - "st": (["x", "time"], c_no_noise), - "c_lin_var_through_zero": (["x", "time"], c_lin_var_through_zero), - "probe1Temperature": (["time"], range(nt)), - "userAcquisitionTimeFW": (["time"], np.ones(nt)), - }, - coords={"x": x, "time": range(nt)}, - attrs={"isDoubleEnded": "0"}, - ) - - sections = {"probe1Temperature": [slice(0.0, 20.0)]} - test_st_var, _ = ds.variance_stokes(st_label="st", sections=sections) - - # If fit is forced through zero. Only Poisson distributed noise - ( - slope, - offset, - st_sort_mean, - st_sort_var, - resid, - var_fun, - ) = ds.variance_stokes_linear( - "c_lin_var_through_zero", - sections=sections, - nbin=10, - through_zero=True, - plot_fit=False, - ) - assert_almost_equal_verbose(slope, var_slope, decimal=3) - - # Fit accounts for Poisson noise plus white noise - ( - slope, - offset, - st_sort_mean, - st_sort_var, - resid, - var_fun, - ) = ds.variance_stokes_linear( - "c_lin_var_through_zero", sections=sections, nbin=100, through_zero=False - ) - assert_almost_equal_verbose(slope, var_slope, decimal=3) - assert_almost_equal_verbose(offset, 0.0, decimal=0) - - pass - - -@pytest.mark.slow # Execution time ~20 seconds -def test_exponential_variance_of_stokes(): - correct_var = 11.86535 - filepath = data_dir_double_ended2 - ds = read_silixa_files(directory=filepath, timezone_netcdf="UTC", file_ext="*.xml") - sections = { - "probe1Temperature": [slice(7.5, 17.0), slice(70.0, 80.0)], # cold bath - "probe2Temperature": [slice(24.0, 34.0), slice(85.0, 95.0)], # warm bath - } - - I_var, _ = ds.variance_stokes_exponential(st_label="st", sections=sections) - assert_almost_equal_verbose(I_var, correct_var, decimal=5) - - ds_dask = ds.chunk(chunks={}) - I_var, _ = ds_dask.variance_stokes_exponential(st_label="st", sections=sections) - assert_almost_equal_verbose(I_var, correct_var, decimal=5) - - pass - - -def test_exponential_variance_of_stokes_synthetic(): - """ - Produces a synthetic Stokes measurement with a known noise distribution. Check if same - variance is obtained. - - Returns - ------- - - """ - yvar = 5.0 - - nx = 500 - x = np.linspace(0.0, 20.0, nx) - - nt = 200 - beta = np.linspace(3000, 4000, nt)[None] - - y = beta * np.exp(-0.001 * x[:, None]) - - y += stats.norm.rvs(size=y.size, scale=yvar**0.5).reshape(y.shape) - - ds = DataStore( - { - "st": (["x", "time"], y), - "probe1Temperature": (["time"], range(nt)), - "userAcquisitionTimeFW": (["time"], np.ones(nt)), - }, - coords={"x": x, "time": range(nt)}, - attrs={"isDoubleEnded": "0"}, - ) - - sections = {"probe1Temperature": [slice(0.0, 20.0)]} - test_st_var, _ = ds.variance_stokes_exponential(st_label="st", sections=sections) - - assert_almost_equal_verbose(test_st_var, yvar, decimal=1) - pass - - def test_double_ended_wls_estimate_synthetic(): """Checks whether the coefficients are correctly defined by creating a synthetic measurement set, and derive the parameters from this set. Without variance. They should be the same as the parameters used to create the synthetic measurment set""" - from dtscalibration import DataStore - cable_len = 100.0 nt = 50 time = np.arange(nt) diff --git a/tests/test_variance_stokes.py b/tests/test_variance_stokes.py new file mode 100644 index 00000000..c81b2096 --- /dev/null +++ b/tests/test_variance_stokes.py @@ -0,0 +1,243 @@ +import os + +import numpy as np +import pytest +from scipy import stats + +from dtscalibration import DataStore +from dtscalibration import read_silixa_files +from dtscalibration.variance_stokes import variance_stokes_exponential +from dtscalibration.variance_stokes import variance_stokes_constant +from dtscalibration.variance_stokes import variance_stokes_linear + +np.random.seed(0) + +fn = [ + "channel 1_20170921112245510.xml", + "channel 1_20170921112746818.xml", + "channel 1_20170921112746818.xml", +] +fn_single = [ + "channel 2_20180504132202074.xml", + "channel 2_20180504132232903.xml", + "channel 2_20180504132303723.xml", +] + +if 1: + # working dir is tests + wd = os.path.dirname(os.path.abspath(__file__)) + data_dir_single_ended = os.path.join(wd, "data", "single_ended") + data_dir_double_ended = os.path.join(wd, "data", "double_ended") + data_dir_double_ended2 = os.path.join(wd, "data", "double_ended2") + +else: + # working dir is src + data_dir_single_ended = os.path.join("..", "..", "tests", "data", "single_ended") + data_dir_double_ended = os.path.join("..", "..", "tests", "data", "double_ended") + data_dir_double_ended2 = os.path.join("..", "..", "tests", "data", "double_ended2") + + +def assert_almost_equal_verbose(actual, desired, verbose=False, **kwargs): + """Print the actual precision decimals""" + err = np.abs(actual - desired).max() + dec = -np.ceil(np.log10(err)) + + if not (np.isfinite(dec)): + dec = 18.0 + + m = "\n>>>>>The actual precision is: " + str(float(dec)) + + if verbose: + print(m) + + desired2 = np.broadcast_to(desired, actual.shape) + np.testing.assert_almost_equal(actual, desired2, err_msg=m, **kwargs) + pass + + +@pytest.mark.skip(reason="Not enough measurements in time. Use exponential instead.") +def test_variance_of_stokes(): + correct_var = 9.045 + filepath = data_dir_double_ended2 + ds = read_silixa_files(directory=filepath, timezone_netcdf="UTC", file_ext="*.xml") + sections = { + "probe1Temperature": [slice(7.5, 17.0), slice(70.0, 80.0)], # cold bath + "probe2Temperature": [slice(24.0, 34.0), slice(85.0, 95.0)], # warm bath + } + + I_var, _ = variance_stokes_constant(st=ds["st"], sections=sections) + assert_almost_equal_verbose(I_var, correct_var, decimal=1) + + ds_dask = ds.chunk(chunks={}) + I_var, _ = variance_stokes_constant(st=ds_dask["st"], sections=sections) + assert_almost_equal_verbose(I_var, correct_var, decimal=1) + + pass + + +def test_variance_of_stokes_synthetic(): + """ + Produces a synthetic Stokes measurement with a known noise distribution. Check if same + variance is obtained. + + Returns + ------- + + """ + yvar = 5.0 + + nx = 500 + x = np.linspace(0.0, 20.0, nx) + + nt = 200 + G = np.linspace(3000, 4000, nt)[None] + + y = G * np.exp(-0.001 * x[:, None]) + + y += stats.norm.rvs(size=y.size, scale=yvar**0.5).reshape(y.shape) + + ds = DataStore( + { + "st": (["x", "time"], y), + "probe1Temperature": (["time"], range(nt)), + "userAcquisitionTimeFW": (["time"], np.ones(nt)), + }, + coords={"x": x, "time": range(nt)}, + attrs={"isDoubleEnded": "0"}, + ) + + sections = {"probe1Temperature": [slice(0.0, 20.0)]} + test_st_var, _ = variance_stokes_constant(st=ds["st"], sections=sections) + + assert_almost_equal_verbose(test_st_var, yvar, decimal=1) + pass + + +@pytest.mark.slow # Execution time ~20 seconds +def test_variance_of_stokes_linear_synthetic(): + """ + Produces a synthetic Stokes measurement with a known noise distribution. + Check if same variance is obtained. + + Returns + ------- + + """ + var_slope = 0.01 + + nx = 500 + x = np.linspace(0.0, 20.0, nx) + + nt = 200 + G = np.linspace(500, 4000, nt)[None] + c_no_noise = G * np.exp(-0.001 * x[:, None]) + + c_lin_var_through_zero = stats.norm.rvs( + loc=c_no_noise, + # size=y.size, + scale=(var_slope * c_no_noise) ** 0.5, + ) + ds = DataStore( + { + "st": (["x", "time"], c_no_noise), + "c_lin_var_through_zero": (["x", "time"], c_lin_var_through_zero), + "probe1Temperature": (["time"], range(nt)), + "userAcquisitionTimeFW": (["time"], np.ones(nt)), + }, + coords={"x": x, "time": range(nt)}, + attrs={"isDoubleEnded": "0"}, + ) + + sections = {"probe1Temperature": [slice(0.0, 20.0)]} + test_st_var, _ = variance_stokes_constant(st=ds["st"], sections=sections) + + # If fit is forced through zero. Only Poisson distributed noise + ( + slope, + offset, + st_sort_mean, + st_sort_var, + resid, + var_fun, + ) = variance_stokes_linear( + st=ds["c_lin_var_through_zero"], + sections=sections, + nbin=10, + through_zero=True, + plot_fit=False, + ) + assert_almost_equal_verbose(slope, var_slope, decimal=3) + + # Fit accounts for Poisson noise plus white noise + ( + slope, + offset, + st_sort_mean, + st_sort_var, + resid, + var_fun, + ) = variance_stokes_linear( + st=ds["c_lin_var_through_zero"], sections=sections, nbin=100, through_zero=False + ) + assert_almost_equal_verbose(slope, var_slope, decimal=3) + assert_almost_equal_verbose(offset, 0.0, decimal=0) + + pass + + +@pytest.mark.slow # Execution time ~20 seconds +def test_exponential_variance_of_stokes(): + correct_var = 11.86535 + filepath = data_dir_double_ended2 + ds = read_silixa_files(directory=filepath, timezone_netcdf="UTC", file_ext="*.xml") + sections = { + "probe1Temperature": [slice(7.5, 17.0), slice(70.0, 80.0)], # cold bath + "probe2Temperature": [slice(24.0, 34.0), slice(85.0, 95.0)], # warm bath + } + + I_var, _ = variance_stokes_exponential(st=ds["st"], sections=sections) + assert_almost_equal_verbose(I_var, correct_var, decimal=5) + + ds_dask = ds.chunk(chunks={}) + I_var, _ = variance_stokes_exponential(st=ds_dask["st"], sections=sections) + assert_almost_equal_verbose(I_var, correct_var, decimal=5) + + pass + + +def test_exponential_variance_of_stokes_synthetic(): + """ + Produces a synthetic Stokes measurement with a known noise distribution. Check if same + variance is obtained. + + Returns + ------- + + """ + yvar = 5.0 + + nx = 500 + x = np.linspace(0.0, 20.0, nx) + + nt = 200 + beta = np.linspace(3000, 4000, nt)[None] + + y = beta * np.exp(-0.001 * x[:, None]) + + y += stats.norm.rvs(size=y.size, scale=yvar**0.5).reshape(y.shape) + + ds = DataStore( + { + "st": (["x", "time"], y), + "probe1Temperature": (["time"], range(nt)), + "userAcquisitionTimeFW": (["time"], np.ones(nt)), + }, + coords={"x": x, "time": range(nt)}, + attrs={"isDoubleEnded": "0"}, + ) + + sections = {"probe1Temperature": [slice(0.0, 20.0)]} + test_st_var, _ = variance_stokes_exponential(st=ds["st"], sections=sections) + + assert_almost_equal_verbose(test_st_var, yvar, decimal=1) + pass From d0a745d62dbf2687ef621979deb3749980e07c73 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Mon, 9 Oct 2023 23:18:31 +0300 Subject: [PATCH 23/66] single_ended_calibration and single_conf_ints working --- src/dtscalibration/__init__.py | 3 - src/dtscalibration/calibrate_utils.py | 114 +- src/dtscalibration/datastore.py | 45 +- src/dtscalibration/datastore_accessor.py | 1512 +++++++++++++++++++++- src/dtscalibration/datastore_utils.py | 86 +- src/dtscalibration/plot.py | 8 +- src/dtscalibration/variance_helpers.py | 27 + src/dtscalibration/variance_stokes.py | 11 +- tests/test_dtscalibration.py | 382 +++--- 9 files changed, 1796 insertions(+), 392 deletions(-) diff --git a/src/dtscalibration/__init__.py b/src/dtscalibration/__init__.py index d173d689..66fdb7ac 100644 --- a/src/dtscalibration/__init__.py +++ b/src/dtscalibration/__init__.py @@ -1,7 +1,5 @@ from dtscalibration.datastore import DataStore -from dtscalibration.datastore_accessor import DtsAccessor from dtscalibration.datastore_utils import check_dims -from dtscalibration.datastore_utils import check_timestep_allclose from dtscalibration.datastore_utils import get_netcdf_encoding from dtscalibration.datastore_utils import merge_double_ended from dtscalibration.datastore_utils import shift_double_ended @@ -28,7 +26,6 @@ "read_sensortran_files", "read_silixa_files", "check_dims", - "check_timestep_allclose", "get_netcdf_encoding", "merge_double_ended", "shift_double_ended", diff --git a/src/dtscalibration/calibrate_utils.py b/src/dtscalibration/calibrate_utils.py index 960fa220..bd84f7b2 100644 --- a/src/dtscalibration/calibrate_utils.py +++ b/src/dtscalibration/calibrate_utils.py @@ -5,14 +5,16 @@ from scipy.sparse import linalg as ln -def parse_st_var(ds, st_var, st_label="st"): +def parse_st_var(st, st_var): """ Utility function to check the st_var input and to return in DataArray format. Parameters ---------- - ds : DataStore + st : DataArray + Stokes/anti-stokes data variable for which the variance is being parsed. + st_var : float, callable, array-like If `float` the variance of the noise from the Stokes detector is described with a single value. @@ -21,25 +23,23 @@ def parse_st_var(ds, st_var, st_label="st"): Or when the variance is a function of the intensity (Poisson distributed) define a DataArray of the shape as ds.st, where the variance can be a function of time and/or x. - st_label : string - Name of the (reverse) stokes/anti-stokes data variable which is being - parsed. Returns ------- - Parsed st_var + st_var_sec : DataArray + The variance of the noise from the Stokes detector. """ if callable(st_var): - st_var_sec = st_var(ds[st_label]) + st_var_sec = st_var(st) else: - st_var_sec = xr.ones_like(ds[st_label]) * st_var + st_var_sec = xr.ones_like(st) * st_var assert np.all(np.isfinite(st_var_sec)), ( - "NaN/inf values detected in " + st_label + "_var. Please check input." + "NaN/inf values detected in computed st_var. Please check input." ) assert np.all(st_var_sec > 0.0), ( - "Negative values detected in " + st_label + "_var. Please check input." + "Negative values detected in computed st_var. Please check input." ) return st_var_sec @@ -54,16 +54,14 @@ def calibration_single_ended_helper( fix_dalpha, fix_gamma, matching_indices, - nt, - nta, - nx, + trans_att, solver, ): """Only used in `calibration_single_ended()`""" - for input_item in [st_var, ast_var]: - assert input_item is not None, ( - "For wls define all " "variances (`st_var`, " "`ast_var`) " - ) + nt = self.dts.nt + nx = self.dts.nx + nta = len(trans_att) + calc_cov = True split = calibration_single_ended_solver( self, @@ -73,6 +71,7 @@ def calibration_single_ended_helper( calc_cov=calc_cov, solver="external_split", matching_indices=matching_indices, + trans_att=trans_att ) y = split["y"] w = split["w"] @@ -80,11 +79,11 @@ def calibration_single_ended_helper( # Stack all X's if fix_alpha: assert not fix_dalpha, "Use either `fix_dalpha` or `fix_alpha`" - assert fix_alpha[0].size == self.x.size, ( - "fix_alpha also needs to be defined outside the reference " "sections" + assert fix_alpha[0].size == nx, ( + "fix_alpha also needs to be defined outside the reference sections" ) - assert fix_alpha[1].size == self.x.size, ( - "fix_alpha also needs to be defined outside the reference " "sections" + assert fix_alpha[1].size == nx, ( + "fix_alpha also needs to be defined outside the reference sections" ) p_val = split["p0_est_alpha"].copy() @@ -196,6 +195,7 @@ def calibration_single_ended_solver( # noqa: MC0001 calc_cov=True, solver="sparse", matching_indices=None, + trans_att=[], verbose=False, ): """ @@ -237,7 +237,7 @@ def calibration_single_ended_solver( # noqa: MC0001 """ # get ix_sec argsort so the sections are in order of increasing x - ix_sec = ds.ufunc_per_section(sections=sections, x_indices=True, calc_per="all") + ix_sec = ds.dts.ufunc_per_section(sections=sections, x_indices=True, calc_per="all") ds_sec = ds.isel(x=ix_sec) x_sec = ds_sec["x"].values @@ -245,7 +245,7 @@ def calibration_single_ended_solver( # noqa: MC0001 nx = x_sec.size nt = ds.time.size no = ds.x.size - nta = ds.trans_att.size + nta = len(trans_att) nm = matching_indices.shape[0] if np.any(matching_indices) else 0 if np.any(matching_indices): @@ -256,7 +256,7 @@ def calibration_single_ended_solver( # noqa: MC0001 p0_est_alpha = np.asarray([485.0] + no * [0.0] + nt * [1.4] + nta * nt * [0.0]) # X \gamma # Eq.34 - cal_ref = ds.ufunc_per_section( + cal_ref = ds.dts.ufunc_per_section( sections=sections, label="st", ref_temp_broadcasted=True, calc_per="all" ) # cal_ref = cal_ref # sort by increasing x @@ -287,10 +287,10 @@ def calibration_single_ended_solver( # noqa: MC0001 ) # X ta #not documented - if ds.trans_att.size > 0: + if nta > 0: TA_list = [] - for transient_att_xi in ds.trans_att.values: + for transient_att_xi in trans_att: # first index on the right hand side a the difficult splice # Deal with connector outside of fiber if transient_att_xi >= x_sec[-1]: @@ -339,10 +339,10 @@ def calibration_single_ended_solver( # noqa: MC0001 ) # make TA matrix - if ds.trans_att.size > 0: + if nta > 0: transient_m_data = np.zeros((nm, nta)) for ii, row in enumerate(matching_indices): - for jj, transient_att_xi in enumerate(ds.trans_att.values): + for jj, transient_att_xi in enumerate(trans_att): transient_m_data[ii, jj] = np.logical_and( transient_att_xi > x_all[row[0]], transient_att_xi < x_all[row[1]], @@ -388,8 +388,8 @@ def calibration_single_ended_solver( # noqa: MC0001 # w if st_var is not None: - st_var_sec = parse_st_var(ds, st_var, st_label="st").isel(x=ix_sec).values - ast_var_sec = parse_st_var(ds, ast_var, st_label="ast").isel(x=ix_sec).values + st_var_sec = parse_st_var(ds.st, st_var).isel(x=ix_sec).values + ast_var_sec = parse_st_var(ds.ast, ast_var).isel(x=ix_sec).values w = ( 1 @@ -400,22 +400,22 @@ def calibration_single_ended_solver( # noqa: MC0001 if np.any(matching_indices): st_var_ms0 = ( - parse_st_var(ds, st_var, st_label="st") + parse_st_var(ds.st, st_var) .isel(x=matching_indices[:, 0]) .values ) st_var_ms1 = ( - parse_st_var(ds, st_var, st_label="st") + parse_st_var(ds.st, st_var) .isel(x=matching_indices[:, 1]) .values ) ast_var_ms0 = ( - parse_st_var(ds, ast_var, st_label="ast") + parse_st_var(ds.ast, ast_var) .isel(x=matching_indices[:, 0]) .values ) ast_var_ms1 = ( - parse_st_var(ds, ast_var, st_label="ast") + parse_st_var(ds.ast, ast_var) .isel(x=matching_indices[:, 1]) .values ) @@ -1173,14 +1173,14 @@ def calibration_double_ended_solver( # noqa: MC0001 ------- """ - ix_sec = ds.ufunc_per_section(sections=sections, x_indices=True, calc_per="all") + ix_sec = ds.dts.ufunc_per_section(sections=sections, x_indices=True, calc_per="all") ds_sec = ds.isel(x=ix_sec) ix_alpha_is_zero = ix_sec[0] # per definition of E x_sec = ds_sec["x"].values nx_sec = x_sec.size nt = ds.time.size - nta = ds.trans_att.size + nta = ds.dts.nta # Calculate E as initial estimate for the E calibration. # Does not require ta to be passed on @@ -1209,10 +1209,10 @@ def calibration_double_ended_solver( # noqa: MC0001 y_B = np.log(ds_sec.rst / ds_sec.rast).values.ravel() # w - st_var_sec = parse_st_var(ds, st_var, st_label="st").isel(x=ix_sec).values - ast_var_sec = parse_st_var(ds, ast_var, st_label="ast").isel(x=ix_sec).values - rst_var_sec = parse_st_var(ds, rst_var, st_label="rst").isel(x=ix_sec).values - rast_var_sec = parse_st_var(ds, rast_var, st_label="rast").isel(x=ix_sec).values + st_var_sec = parse_st_var(ds.st, st_var).isel(x=ix_sec).values + ast_var_sec = parse_st_var(ds.ast, ast_var).isel(x=ix_sec).values + rst_var_sec = parse_st_var(ds.rst, rst_var).isel(x=ix_sec).values + rast_var_sec = parse_st_var(ds.rast, rast_var).isel(x=ix_sec).values w_F = ( 1 @@ -1335,27 +1335,27 @@ def calibration_double_ended_solver( # noqa: MC0001 y = np.concatenate((y_F, y_B, y_eq1, y_eq2, y_eq3)) - st_var_hix = parse_st_var(ds, st_var, st_label="st").isel(x=hix).values - ast_var_hix = parse_st_var(ds, ast_var, st_label="ast").isel(x=hix).values - rst_var_hix = parse_st_var(ds, rst_var, st_label="rst").isel(x=hix).values - rast_var_hix = parse_st_var(ds, rast_var, st_label="rast").isel(x=hix).values + st_var_hix = parse_st_var(ds.st, st_var).isel(x=hix).values + ast_var_hix = parse_st_var(ds.ast, ast_var).isel(x=hix).values + rst_var_hix = parse_st_var(ds.rst, rst_var).isel(x=hix).values + rast_var_hix = parse_st_var(ds.rast, rast_var).isel(x=hix).values - st_var_tix = parse_st_var(ds, st_var, st_label="st").isel(x=tix).values - ast_var_tix = parse_st_var(ds, ast_var, st_label="ast").isel(x=tix).values - rst_var_tix = parse_st_var(ds, rst_var, st_label="rst").isel(x=tix).values - rast_var_tix = parse_st_var(ds, rast_var, st_label="rast").isel(x=tix).values + st_var_tix = parse_st_var(ds.st, st_var).isel(x=tix).values + ast_var_tix = parse_st_var(ds.ast, ast_var).isel(x=tix).values + rst_var_tix = parse_st_var(ds.rst, rst_var).isel(x=tix).values + rast_var_tix = parse_st_var(ds.rast, rast_var).isel(x=tix).values st_var_mnc = ( - parse_st_var(ds, st_var, st_label="st").isel(x=ix_match_not_cal).values + parse_st_var(ds.st, st_var).isel(x=ix_match_not_cal).values ) ast_var_mnc = ( - parse_st_var(ds, ast_var, st_label="ast").isel(x=ix_match_not_cal).values + parse_st_var(ds.ast, ast_var).isel(x=ix_match_not_cal).values ) rst_var_mnc = ( - parse_st_var(ds, rst_var, st_label="rst").isel(x=ix_match_not_cal).values + parse_st_var(ds.rst, rst_var).isel(x=ix_match_not_cal).values ) rast_var_mnc = ( - parse_st_var(ds, rast_var, st_label="rast").isel(x=ix_match_not_cal).values + parse_st_var(ds.rast, rast_var).isel(x=ix_match_not_cal).values ) w_eq1 = 1 / ( @@ -1456,7 +1456,7 @@ def calibration_double_ended_solver( # noqa: MC0001 # the int diff att for outside the reference sections. # calculate talpha_fw and bw for attenuation - if ds.trans_att.size > 0: + if ds.dts.nta > 0: if np.any(matching_indices): ta = p_sol[1 + 2 * nt + ix_from_cal_match_to_glob.size :].reshape( (nt, 2, nta), order="F" @@ -1846,7 +1846,7 @@ def construct_submatrices(sections, nt, nx, ds, trans_att, x_sec): # Z \gamma # Eq.47 cal_ref = np.array( - ds.ufunc_per_section( + ds.dts.ufunc_per_section( sections=sections, label="st", ref_temp_broadcasted=True, calc_per="all" ) ) @@ -2154,7 +2154,7 @@ def calc_alpha_double( D_F_var = ds["df_var"] D_B_var = ds["db_var"] - if ds.trans_att.size > 0: + if ds.dts.nta > 0: # Can be improved by including covariances. That reduces the # uncert. @@ -2228,10 +2228,10 @@ def calc_df_db_double_est(ds, sections, ix_alpha_is_zero, gamma_est): Ibwx0 = np.log( ds.rst.isel(x=ix_alpha_is_zero) / ds.rast.isel(x=ix_alpha_is_zero) ).values - ref_temps_refs = ds.ufunc_per_section( + ref_temps_refs = ds.dts.ufunc_per_section( sections=sections, label="st", ref_temp_broadcasted=True, calc_per="all" ) - ix_sec = ds.ufunc_per_section(sections=sections, x_indices=True, calc_per="all") + ix_sec = ds.dts.ufunc_per_section(sections=sections, x_indices=True, calc_per="all") ref_temps_x0 = ( ref_temps_refs[ix_sec == ix_alpha_is_zero].flatten().compute() + 273.15 ) diff --git a/src/dtscalibration/datastore.py b/src/dtscalibration/datastore.py index 459a45dd..54973c0e 100644 --- a/src/dtscalibration/datastore.py +++ b/src/dtscalibration/datastore.py @@ -17,7 +17,6 @@ from dtscalibration.datastore_utils import ParameterIndexDoubleEnded from dtscalibration.datastore_utils import ParameterIndexSingleEnded from dtscalibration.datastore_utils import check_deprecated_kwargs -from dtscalibration.datastore_utils import check_timestep_allclose from dtscalibration.datastore_utils import get_params_from_pval_double_ended from dtscalibration.datastore_utils import get_params_from_pval_single_ended from dtscalibration.datastore_utils import ufunc_per_section_helper @@ -883,7 +882,7 @@ def variance_stokes_constant(self, st_label, sections=None, reshape_residuals=Tr sections = validate_sections(self, sections) assert self[st_label].dims[0] == "x", f"{st_label} are transposed" - check_timestep_allclose(self, eps=0.01) + # check_timestep_allclose(self, eps=0.01) # should maybe be per section. But then residuals # seem to be correlated between stretches. I don't know why.. BdT. @@ -1050,7 +1049,7 @@ def variance_stokes_exponential( assert self[st_label].dims[0] == "x", "Stokes are transposed" - check_timestep_allclose(self, eps=0.01) + # check_timestep_allclose(self, eps=0.01) nt = self.time.size @@ -1329,46 +1328,6 @@ def i_var(self, st_var, ast_var, st_label="st", ast_label="ast"): return st**-2 * st_var + ast**-2 * ast_var - def set_trans_att(self, trans_att=None): - """Gracefully set the locations that introduce directional differential - attenuation - - Parameters - ---------- - trans_att : iterable, optional - Splices can cause jumps in differential attenuation. Normal single - ended calibration assumes these are not present. An additional loss - term is added in the 'shadow' of the splice. Each location - introduces an additional nt parameters to solve for. Requiring - either an additional calibration section or matching sections. - If multiple locations are defined, the losses are added. - - """ - if "trans_att" in self.coords and self["trans_att"].size > 0: - raise_warning = 0 - - del_keys = [] - for k, v in self.data_vars.items(): - if "trans_att" in v.dims: - del_keys.append(k) - - for del_key in del_keys: - del self[del_key] - - if raise_warning: - m = ( - "trans_att was set before. All `data_vars` that make use " - "of the `trans_att` coordinates were deleted: " + str(del_keys) - ) - warnings.warn(m) - - if trans_att is None: - trans_att = [] - - self["trans_att"] = trans_att - self["trans_att"].attrs = dim_attrs["trans_att"] - pass - def calibration_single_ended( self, sections, diff --git a/src/dtscalibration/datastore_accessor.py b/src/dtscalibration/datastore_accessor.py index 66c0b16a..29c39619 100644 --- a/src/dtscalibration/datastore_accessor.py +++ b/src/dtscalibration/datastore_accessor.py @@ -2,47 +2,40 @@ import numpy as np import xarray as xr import yaml +import scipy.stats as sst -from dtscalibration.datastore_utils import check_timestep_allclose +from dtscalibration.calibration.section_utils import validate_sections +from dtscalibration.calibrate_utils import calibration_single_ended_helper +from dtscalibration.calibrate_utils import match_sections +from dtscalibration.calibrate_utils import parse_st_var +from dtscalibration.calibration.section_utils import set_sections +from dtscalibration.calibration.section_utils import set_matching_sections +from dtscalibration.datastore_utils import ParameterIndexSingleEnded +from dtscalibration.datastore_utils import get_params_from_pval_single_ended +from dtscalibration.datastore_utils import ufunc_per_section_helper +from dtscalibration.io_utils import dim_attrs @xr.register_dataset_accessor("dts") class DtsAccessor: def __init__(self, xarray_obj): - # check xarray_obj - # check naming convention - assert ( - ("st" in xarray_obj.data_vars) and - ("ast" in xarray_obj.data_vars) and - ("userAcquisitionTimeFW" in xarray_obj.data_vars)), \ - "xarray_obj should have st, ast, userAcquisitionTimeFW" - - # Varying acquisition times not supported. Could be in the future. - # Should actually be moved to the estimating variance functions - check_timestep_allclose(xarray_obj, eps=0.01) - # cache xarray_obj self._obj = xarray_obj self.attrs = xarray_obj.attrs # alias commonly used variables self.x = xarray_obj.x + self.nx = self.x.size self.time = xarray_obj.time - self.transatt = xarray_obj.get("transatt") + self.nt = self.time.size - self.st = xarray_obj["st"] # required - self.ast = xarray_obj["ast"] # required + self.st = xarray_obj.get("st") + self.ast = xarray_obj.get("ast") self.rst = xarray_obj.get("rst") # None if doesn't exist self.rast = xarray_obj.get("rast") # None is doesn't exist - # alias commonly computed variables - self.nx = self.x.size - self.nt = self.time.size - if self.transatt: - self.nta = self.transatt.size - else: - self.nta = 0 - + self.acquisitiontime_fw = xarray_obj.get("userAcquisitionTimeFW") + self.acquisitiontime_bw = xarray_obj.get("userAcquisitionTimeBW") pass def __repr__(self): @@ -133,6 +126,46 @@ def sections(self, value): "ds.dts.calibrate_single_ended() or ds.dts.calibrate_double_ended()." ) raise NotImplementedError(msg) + + # noinspection PyIncorrectDocstring + @property + def matching_sections(self): + """ + Define calibration sections. Each matching_section requires a reference + temperature time series, such as the temperature measured by an + external temperature sensor. They should already be part of the + DataStore object. + + Please look at the example notebook on `matching_sections` if you encounter + difficulties. + + Parameters + ---------- + matching_sections : List[Tuple[slice, slice, bool]], optional + Provide a list of tuples. A tuple per matching section. Each tuple + has three items. The first two items are the slices of the sections + that are matched. The third item is a boolean and is True if the two + sections have a reverse direction ("J-configuration"). + Returns + ------- + + """ + if "_matching_sections" not in self._obj.attrs: + self._obj.attrs["_matching_sections"] = yaml.dump(None) + + return yaml.load(self._obj.attrs["_matching_sections"], Loader=yaml.UnsafeLoader) + + @matching_sections.deleter + def matching_sections(self): + self._obj.attrs["_matching_sections"] = yaml.dump(None) + + @matching_sections.setter + def matching_sections(self, value): + msg = ( + "Not possible anymore. Instead, pass the matching_sections as an argument to \n" + "ds.dts.calibrate_single_ended() or ds.dts.calibrate_double_ended()." + ) + raise NotImplementedError(msg) def get_default_encoding(self, time_chunks_from_key=None): """ @@ -231,3 +264,1434 @@ def get_default_encoding(self, time_chunks_from_key=None): v["chunksizes"] = chunks return encoding + + def ufunc_per_section( + self, + sections=None, + func=None, + label=None, + subtract_from_label=None, + temp_err=False, + x_indices=False, + ref_temp_broadcasted=False, + calc_per="stretch", + **func_kwargs, + ): + """ + User function applied to parts of the cable. Super useful, + many options and slightly + complicated. + + The function `func` is taken over all the timesteps and calculated + per `calc_per`. This + is returned as a dictionary + + Parameters + ---------- + sections : Dict[str, List[slice]], optional + If `None` is supplied, `ds.sections` is used. Define calibration + sections. Each section requires a reference temperature time series, + such as the temperature measured by an external temperature sensor. + They should already be part of the DataStore object. `sections` + is defined with a dictionary with its keywords of the + names of the reference temperature time series. Its values are + lists of slice objects, where each slice object is a fiber stretch + that has the reference temperature. Afterwards, `sections` is stored + under `ds.sections`. + func : callable, str + A numpy function, or lambda function to apple to each 'calc_per'. + label + subtract_from_label + temp_err : bool + The argument of the function is label minus the reference + temperature. + x_indices : bool + To retreive an integer array with the indices of the + x-coordinates in the section/stretch. The indices are sorted. + ref_temp_broadcasted : bool + calc_per : {'all', 'section', 'stretch'} + func_kwargs : dict + Dictionary with options that are passed to func + + TODO: Spend time on creating a slice instead of appendng everything\ + to a list and concatenating after. + + + Returns + ------- + + Examples + -------- + + 1. Calculate the variance of the residuals in the along ALL the\ + reference sections wrt the temperature of the water baths + + >>> tmpf_var = d.ufunc_per_section( + >>> sections=sections, + >>> func='var', + >>> calc_per='all', + >>> label='tmpf', + >>> temp_err=True) + + 2. Calculate the variance of the residuals in the along PER\ + reference section wrt the temperature of the water baths + + >>> tmpf_var = d.ufunc_per_section( + >>> sections=sections, + >>> func='var', + >>> calc_per='stretch', + >>> label='tmpf', + >>> temp_err=True) + + 3. Calculate the variance of the residuals in the along PER\ + water bath wrt the temperature of the water baths + + >>> tmpf_var = d.ufunc_per_section( + >>> sections=sections, + >>> func='var', + >>> calc_per='section', + >>> label='tmpf', + >>> temp_err=True) + + 4. Obtain the coordinates of the measurements per section + + >>> locs = d.ufunc_per_section( + >>> sections=sections, + >>> func=None, + >>> label='x', + >>> temp_err=False, + >>> ref_temp_broadcasted=False, + >>> calc_per='stretch') + + 5. Number of observations per stretch + + >>> nlocs = d.ufunc_per_section( + >>> sections=sections, + >>> func=len, + >>> label='x', + >>> temp_err=False, + >>> ref_temp_broadcasted=False, + >>> calc_per='stretch') + + 6. broadcast the temperature of the reference sections to\ + stretch/section/all dimensions. The value of the reference\ + temperature (a timeseries) is broadcasted to the shape of self[\ + label]. The self[label] is not used for anything else. + + >>> temp_ref = d.ufunc_per_section( + >>> label='st', + >>> ref_temp_broadcasted=True, + >>> calc_per='all') + + 7. x-coordinate index + + >>> ix_loc = d.ufunc_per_section(sections=sections, x_indices=True) + + + Note + ---- + If `self[label]` or `self[subtract_from_label]` is a Dask array, a Dask + array is returned else a numpy array is returned + """ + if label is None: + dataarray = None + else: + dataarray = self._obj[label] + + if x_indices: + x_coords = self.x + reference_dataset = None + + else: + validate_sections(self._obj, sections) + + x_coords = None + reference_dataset = {k: self._obj[k] for k in sections} + + out = ufunc_per_section_helper( + x_coords=x_coords, + sections=sections, + func=func, + dataarray=dataarray, + subtract_from_dataarray=subtract_from_label, + reference_dataset=reference_dataset, + subtract_reference_from_dataarray=temp_err, + ref_temp_broadcasted=ref_temp_broadcasted, + calc_per=calc_per, + **func_kwargs, + ) + return out + + def calibration_single_ended( + self, + sections, + st_var, + ast_var, + method="wls", + solver="sparse", + p_val=None, + p_var=None, + p_cov=None, + matching_sections=None, + trans_att=[], + fix_gamma=None, + fix_dalpha=None, + fix_alpha=None, + ): + r""" + Calibrate the Stokes (`ds.st`) and anti-Stokes (`ds.ast`) data to + temperature using fiber sections with a known temperature + (`ds.sections`) for single-ended setups. The calibrated temperature is + stored under `ds.tmpf` and its variance under `ds.tmpf_var`. + + In single-ended setups, Stokes and anti-Stokes intensity is measured + from a single end of the fiber. The differential attenuation is assumed + constant along the fiber so that the integrated differential attenuation + may be written as (Hausner et al, 2011): + + .. math:: + + \int_0^x{\Delta\\alpha(x')\,\mathrm{d}x'} \\approx \Delta\\alpha x + + The temperature can now be written from Equation 10 [1]_ as: + + .. math:: + + T(x,t) \\approx \\frac{\gamma}{I(x,t) + C(t) + \Delta\\alpha x} + + where + + .. math:: + + I(x,t) = \ln{\left(\\frac{P_+(x,t)}{P_-(x,t)}\\right)} + + + .. math:: + + C(t) = \ln{\left(\\frac{\eta_-(t)K_-/\lambda_-^4}{\eta_+(t)K_+/\lambda_+^4}\\right)} + + where :math:`C` is the lumped effect of the difference in gain at + :math:`x=0` between Stokes and anti-Stokes intensity measurements and + the dependence of the scattering intensity on the wavelength. The + parameters :math:`P_+` and :math:`P_-` are the Stokes and anti-Stokes + intensity measurements, respectively. + The parameters :math:`\gamma`, :math:`C(t)`, and :math:`\Delta\\alpha` + must be estimated from calibration to reference sections, as discussed + in Section 5 [1]_. The parameter :math:`C` must be estimated + for each time and is constant along the fiber. :math:`T` in the listed + equations is in Kelvin, but is converted to Celsius after calibration. + + Parameters + ---------- + p_val : array-like, optional + Define `p_val`, `p_var`, `p_cov` if you used an external function + for calibration. Has size 2 + `nt`. First value is :math:`\gamma`, + second is :math:`\Delta \\alpha`, others are :math:`C` for each + timestep. + p_var : array-like, optional + Define `p_val`, `p_var`, `p_cov` if you used an external function + for calibration. Has size 2 + `nt`. First value is :math:`\gamma`, + second is :math:`\Delta \\alpha`, others are :math:`C` for each + timestep. + p_cov : array-like, optional + The covariances of `p_val`. + If set to False, no uncertainty in the parameters is propagated + into the confidence intervals. Similar to the spec sheets of the DTS + manufacturers. And similar to passing an array filled with zeros. + sections : Dict[str, List[slice]] + Each section requires a reference temperature time series, + such as the temperature measured by an external temperature sensor. + They should already be part of the DataStore object. `sections` + is defined with a dictionary with its keywords of the + names of the reference temperature time series. Its values are + lists of slice objects, where each slice object is a fiber stretch + that has the reference temperature. Afterwards, `sections` is stored + under `ds.sections`. + st_var, ast_var : float, callable, array-like, optional + The variance of the measurement noise of the Stokes signals in the + forward direction. If `float` the variance of the noise from the + Stokes detector is described with a single value. + If `callable` the variance of the noise from the Stokes detector is + a function of the intensity, as defined in the callable function. + Or manually define a variance with a DataArray of the shape + `ds.st.shape`, where the variance can be a function of time and/or + x. Required if method is wls. + method : {'wls',} + Use `'wls'` for weighted least + squares. + solver : {'sparse', 'stats'} + Either use the homemade weighted sparse solver or the weighted + dense matrix solver of statsmodels. The sparse solver uses much less + memory, is faster, and gives the same result as the statsmodels + solver. The statsmodels solver is mostly used to check the sparse + solver. `'stats'` is the default. + matching_sections : List[Tuple[slice, slice, bool]], optional + Provide a list of tuples. A tuple per matching section. Each tuple + has three items. The first two items are the slices of the sections + that are matched. The third item is a boolean and is True if the two + sections have a reverse direction ("J-configuration"). + trans_att : iterable, optional + Splices can cause jumps in differential attenuation. Normal single + ended calibration assumes these are not present. An additional loss + term is added in the 'shadow' of the splice. Each location + introduces an additional nt parameters to solve for. Requiring + either an additional calibration section or matching sections. + If multiple locations are defined, the losses are added. + fix_gamma : Tuple[float, float], optional + A tuple containing two floats. The first float is the value of + gamma, and the second item is the variance of the estimate of gamma. + Covariances between gamma and other parameters are not accounted + for. + fix_dalpha : Tuple[float, float], optional + A tuple containing two floats. The first float is the value of + dalpha (:math:`\Delta \\alpha` in [1]_), and the second item is the + variance of the estimate of dalpha. + Covariances between alpha and other parameters are not accounted + for. + fix_alpha : Tuple[array-like, array-like], optional + A tuple containing two array-likes. The first array-like is the integrated + differential attenuation of length x, and the second item is its variance. + + Returns + ------- + + References + ---------- + .. [1] des Tombe, B., Schilperoort, B., & Bakker, M. (2020). Estimation + of Temperature and Associated Uncertainty from Fiber-Optic Raman- + Spectrum Distributed Temperature Sensing. Sensors, 20(8), 2235. + https://doi.org/10.3390/s20082235 + + Examples + -------- + - `Example notebook 7: Calibrate single ended `_ + + """ + assert self.st.dims[0] == "x", "Stokes are transposed" + assert self.ast.dims[0] == "x", "Stokes are transposed" + + # out contains the state + out = xr.Dataset(coords={"x": self.x, "time": self.time, "trans_att": trans_att}).copy() + out.coords["x"].attrs = dim_attrs["x"] + out.coords["trans_att"].attrs = dim_attrs["trans_att"] + + nta = len(trans_att) + + # check and store sections and matching_sections + validate_sections(self._obj, sections=sections) + set_sections(out, sections) + set_matching_sections(out, matching_sections) + + # Convert sections and matching_sections to indices + ix_sec = self.ufunc_per_section( + sections=sections, x_indices=True, calc_per="all" + ) + if matching_sections: + matching_indices = match_sections(self, matching_sections) + else: + matching_indices = None + + assert not np.any(self.st.isel(x=ix_sec) <= 0.0), ( + "There is uncontrolled noise in the ST signal. Are your sections" + "correctly defined?" + ) + assert not np.any(self.ast.isel(x=ix_sec) <= 0.0), ( + "There is uncontrolled noise in the AST signal. Are your sections" + "correctly defined?" + ) + + if method == "wls": + p_cov, p_val, p_var = calibration_single_ended_helper( + self._obj, + sections, + st_var, + ast_var, + fix_alpha, + fix_dalpha, + fix_gamma, + matching_indices, + trans_att, + solver, + ) + + elif method == "external": + for input_item in [p_val, p_var, p_cov]: + assert ( + input_item is not None + ), "Define p_val, p_var, p_cov when using an external solver" + + else: + raise ValueError("Choose a valid method") + + # all below require the following solution sizes + if fix_alpha: + ip = ParameterIndexSingleEnded( + self.nt, self.nx, nta, includes_alpha=True, includes_dalpha=False + ) + else: + ip = ParameterIndexSingleEnded( + self.nt, self.nx, nta, includes_alpha=False, includes_dalpha=True + ) + + # npar = 1 + 1 + nt + nta * nt + assert p_val.size == ip.npar + assert p_var.size == ip.npar + assert p_cov.shape == (ip.npar, ip.npar) + + # store calibration parameters in DataStore + params, param_covs = get_params_from_pval_single_ended( + ip, out.coords, p_val=p_val, p_var=p_var, p_cov=p_cov, fix_alpha=fix_alpha + ) + + tmpf = params["gamma"] / ( + (np.log(self.st / self.ast) + (params["c"] + params["talpha_fw_full"])) + + params["alpha"] + ) + + out["tmpf"] = tmpf - 273.15 + out["tmpf"].attrs.update(dim_attrs["tmpf"]) + + # tmpf_var + deriv_dict = dict( + T_gamma_fw=tmpf / params["gamma"], + T_st_fw=-(tmpf**2) / (params["gamma"] * self.st), + T_ast_fw=tmpf**2 / (params["gamma"] * self.ast), + T_c_fw=-(tmpf**2) / params["gamma"], + T_dalpha_fw=-self.x * (tmpf**2) / params["gamma"], + T_alpha_fw=-(tmpf**2) / params["gamma"], + T_ta_fw=-(tmpf**2) / params["gamma"], + ) + deriv_ds = xr.Dataset(deriv_dict) + + var_fw_dict = dict( + dT_dst=deriv_ds.T_st_fw**2 * parse_st_var(self.st, st_var), + dT_dast=deriv_ds.T_ast_fw**2 + * parse_st_var(self.ast, ast_var), + dT_gamma=deriv_ds.T_gamma_fw**2 * param_covs["gamma"], + dT_dc=deriv_ds.T_c_fw**2 * param_covs["c"], + dT_ddalpha=deriv_ds.T_alpha_fw**2 + * param_covs["alpha"], # same as dT_dalpha + dT_dta=deriv_ds.T_ta_fw**2 * param_covs["talpha_fw_full"], + dgamma_dc=( + 2 * deriv_ds.T_gamma_fw * deriv_ds.T_c_fw * param_covs["gamma_c"] + ), + dta_dgamma=( + 2 * deriv_ds.T_ta_fw * deriv_ds.T_gamma_fw * param_covs["tafw_gamma"] + ), + dta_dc=(2 * deriv_ds.T_ta_fw * deriv_ds.T_c_fw * param_covs["tafw_c"]), + ) + + if not fix_alpha: + # These correlations don't exist in case of fix_alpha. Including them reduces tmpf_var. + var_fw_dict.update( + dict( + dgamma_ddalpha=( + 2 + * deriv_ds.T_gamma_fw + * deriv_ds.T_dalpha_fw + * param_covs["gamma_dalpha"] + ), + ddalpha_dc=( + 2 + * deriv_ds.T_dalpha_fw + * deriv_ds.T_c_fw + * param_covs["dalpha_c"] + ), + dta_ddalpha=( + 2 + * deriv_ds.T_ta_fw + * deriv_ds.T_dalpha_fw + * param_covs["tafw_dalpha"] + ), + ) + ) + + out["var_fw_da"] = xr.Dataset(var_fw_dict).to_array(dim="comp_fw") + out["tmpf_var"] = out["var_fw_da"].sum(dim="comp_fw") + out["tmpf_var"].attrs.update(dim_attrs["tmpf_var"]) + + out["p_val"] = (("params1",), p_val) + out["p_cov"] = (("params1", "params2"), p_cov) + + out.update(params) + for key, dataarray in param_covs.data_vars.items(): + out[key + "_var"] = dataarray + + return out + + + def conf_int_single_ended( + self, + result, + st_var, + ast_var, + conf_ints=[], + mc_sample_size=100, + da_random_state=None, + reduce_memory_usage=False, + mc_remove_set_flag=True): + """The result object is what comes out of the single_ended_calibration routine) + + TODO: Use get_params_from_pval_single_ended() to extract parameter sets from mc + """ + assert self.st.dims[0] == "x", "Stokes are transposed" + assert self.ast.dims[0] == "x", "Stokes are transposed" + + if da_random_state: + state = da_random_state + else: + state = da.random.RandomState() + + # out contains the state + out = xr.Dataset(coords={"x": self.x, "time": self.time, "trans_att": result["trans_att"]}).copy() + out.coords["x"].attrs = dim_attrs["x"] + out.coords["trans_att"].attrs = dim_attrs["trans_att"] + out.coords["CI"] = conf_ints + + set_sections(out, result.dts.sections) + set_matching_sections(out, result.dts.matching_sections) + + params = out.copy() + params.coords["mc"] = range(mc_sample_size) + + no, nt = self.st.data.shape + nta = result["trans_att"].size + + p_val = result["p_val"].data + p_cov = result["p_cov"].data + + npar = p_val.size + + # check number of parameters + if npar == nt + 2 + nt * nta: + fixed_alpha = False + elif npar == 1 + no + nt + nt * nta: + fixed_alpha = True + else: + raise Exception("The size of `p_val` is not what I expected") + + assert isinstance(p_val, (str, np.ndarray, np.generic)) + if isinstance(p_val, str): + p_val = self[p_val].data + + npar = p_val.size + p_mc = sst.multivariate_normal.rvs(mean=p_val, cov=p_cov, size=mc_sample_size) + + if fixed_alpha: + params["alpha_mc"] = (("mc", "x"), p_mc[:, 1 : no + 1]) + params["c_mc"] = (("mc", "time"), p_mc[:, 1 + no : 1 + no + nt]) + else: + params["dalpha_mc"] = (("mc",), p_mc[:, 1]) + params["c_mc"] = (("mc", "time"), p_mc[:, 2 : nt + 2]) + + params["gamma_mc"] = (("mc",), p_mc[:, 0]) + if nta: + params["ta_mc"] = ( + ("mc", "trans_att", "time"), + np.reshape(p_mc[:, -nt * nta :], (mc_sample_size, nta, nt)), + ) + + rsize = (params.mc.size, params.x.size, params.time.size) + + if reduce_memory_usage: + memchunk = da.ones( + (mc_sample_size, no, nt), chunks={0: -1, 1: 1, 2: "auto"} + ).chunks + else: + memchunk = da.ones( + (mc_sample_size, no, nt), chunks={0: -1, 1: "auto", 2: "auto"} + ).chunks + + + # Draw from the normal distributions for the Stokes intensities + for key_mc, sti, st_vari in zip(["r_st", "r_ast"], [self.st, self.ast], + [st_var, ast_var]): + # for k, st_labeli, st_vari in zip( + # ["r_st", "r_ast"], ["st", "ast"], [st_var, ast_var] + # ): + # Load the mean as chunked Dask array, otherwise eats memory + if type(sti.data) == da.core.Array: + loc = da.asarray(sti.data, chunks=memchunk[1:]) + else: + loc = da.from_array(sti.data, chunks=memchunk[1:]) + + # Make sure variance is of size (no, nt) + if np.size(st_vari) > 1: + if st_vari.shape == sti.shape: + pass + else: + st_vari = np.broadcast_to(st_vari, (no, nt)) + else: + pass + + # Load variance as chunked Dask array, otherwise eats memory + if type(st_vari) == da.core.Array: + st_vari_da = da.asarray(st_vari, chunks=memchunk[1:]) + + elif callable(st_vari) and type(sti.data) == da.core.Array: + st_vari_da = da.asarray( + st_vari(sti).data, chunks=memchunk[1:] + ) + + elif callable(st_vari) and type(sti.data) != da.core.Array: + st_vari_da = da.from_array( + st_vari(sti).data, chunks=memchunk[1:] + ) + + else: + st_vari_da = da.from_array(st_vari, chunks=memchunk[1:]) + + params[key_mc] = ( + ("mc", "x", "time"), + state.normal( + loc=loc, # has chunks=memchunk[1:] + scale=st_vari_da**0.5, + size=rsize, + chunks=memchunk, + ), + ) + + ta_arr = np.zeros((mc_sample_size, no, nt)) + + if nta: + for ii, ta in enumerate(params["ta_mc"]): + for tai, taxi in zip(ta.values, result["trans_att"].values): + ta_arr[ii, self.x.values >= taxi] = ( + ta_arr[ii, self.x.values >= taxi] + tai + ) + params["ta_mc_arr"] = (("mc", "x", "time"), ta_arr) + + if fixed_alpha: + params["tmpf_mc_set"] = ( + params["gamma_mc"] + / ( + ( + np.log(params["r_st"]) + - np.log(params["r_ast"]) + + (params["c_mc"] + params["ta_mc_arr"]) + ) + + params["alpha_mc"] + ) + - 273.15 + ) + else: + params["tmpf_mc_set"] = ( + params["gamma_mc"] + / ( + ( + np.log(params["r_st"]) + - np.log(params["r_ast"]) + + (params["c_mc"] + params["ta_mc_arr"]) + ) + + (params["dalpha_mc"] * params.x) + ) + - 273.15 + ) + + avg_dims = ["mc"] + avg_axis = params["tmpf_mc_set"].get_axis_num(avg_dims) + out["tmpf_mc_var"] = (params["tmpf_mc_set"] - result["tmpf"]).var( + dim=avg_dims, ddof=1 + ) + + if conf_ints: + new_chunks = ((len(conf_ints),),) + params["tmpf_mc_set"].chunks[1:] + + qq = params["tmpf_mc_set"] + + q = qq.data.map_blocks( + lambda x: np.percentile(x, q=conf_ints, axis=avg_axis), + chunks=new_chunks, # + drop_axis=avg_axis, # avg dimesnions are dropped from input arr + new_axis=0, + ) # The new CI dimension is added as first axis + + out["tmpf_mc"] = (("CI", "x", "time"), q) + + if not mc_remove_set_flag: + out.update(params) + + return out + + def conf_int_single_ended_old( + self, + p_val="p_val", + p_cov="p_cov", + st_var=None, + ast_var=None, + conf_ints=None, + mc_sample_size=100, + da_random_state=None, + mc_remove_set_flag=True, + reduce_memory_usage=False + ): + r""" + Estimation of the confidence intervals for the temperatures measured + with a single-ended setup. It consists of five steps. First, the variances + of the Stokes and anti-Stokes intensity measurements are estimated + following the steps in Section 4 [1]_. A Normal + distribution is assigned to each intensity measurement that is centered + at the measurement and using the estimated variance. Second, a multi- + variate Normal distribution is assigned to the estimated parameters + using the covariance matrix from the calibration procedure presented in + Section 5 [1]_. Third, the distributions are sampled, and the + temperature is computed with Equation 12 [1]_. Fourth, step + three is repeated, e.g., 10,000 times for each location and for each + time. The resulting 10,000 realizations of the temperatures + approximate the probability density functions of the estimated + temperature at that location and time. Fifth, the standard uncertainties + are computed with the standard deviations of the realizations of the + temperatures, and the 95\% confidence intervals are computed from the + 2.5\% and 97.5\% percentiles of the realizations of the temperatures. + + + Parameters + ---------- + p_val : array-like, optional + Define `p_val`, `p_var`, `p_cov` if you used an external function + for calibration. Has size 2 + `nt`. First value is :math:`\gamma`, + second is :math:`\Delta \\alpha`, others are :math:`C` for each + timestep. + If set to False, no uncertainty in the parameters is propagated + into the confidence intervals. Similar to the spec sheets of the DTS + manufacturers. And similar to passing an array filled with zeros + p_cov : array-like, optional + The covariances of `p_val`. + st_var, ast_var : float, callable, array-like, optional + The variance of the measurement noise of the Stokes signals in the + forward direction. If `float` the variance of the noise from the + Stokes detector is described with a single value. + If `callable` the variance of the noise from the Stokes detector is + a function of the intensity, as defined in the callable function. + Or manually define a variance with a DataArray of the shape + `ds.st.shape`, where the variance can be a function of time and/or + x. Required if method is wls. + conf_ints : iterable object of float + A list with the confidence boundaries that are calculated. Valid + values are between + [0, 1]. + mc_sample_size : int + Size of the monte carlo parameter set used to calculate the + confidence interval + da_random_state + For testing purposes. Similar to random seed. The seed for dask. + Makes random not so random. To produce reproducable results for + testing environments. + mc_remove_set_flag : bool + Remove the monte carlo data set, from which the CI and the + variance are calculated. + reduce_memory_usage : bool + Use less memory but at the expense of longer computation time + + + References + ---------- + .. [1] des Tombe, B., Schilperoort, B., & Bakker, M. (2020). Estimation + of Temperature and Associated Uncertainty from Fiber-Optic Raman- + Spectrum Distributed Temperature Sensing. Sensors, 20(8), 2235. + https://doi.org/10.3390/s20082235 + """ + out = xr.Dataset() + params = xr.Dataset() + + if da_random_state: + state = da_random_state + else: + state = da.random.RandomState() + + no, nt = self.st.data.shape + if "trans_att" in self.keys(): + nta = self.trans_att.size + + else: + nta = 0 + + assert isinstance(p_val, (str, np.ndarray, np.generic)) + if isinstance(p_val, str): + p_val = self[p_val].data + + npar = p_val.size + + # number of parameters + if npar == nt + 2 + nt * nta: + fixed_alpha = False + elif npar == 1 + no + nt + nt * nta: + fixed_alpha = True + else: + raise Exception("The size of `p_val` is not what I expected") + + params.coords["mc"] = range(mc_sample_size) + params.coords["x"] = self.x + params.coords["time"] = self.time + + if conf_ints: + out.coords["CI"] = conf_ints + params.coords["CI"] = conf_ints + + # WLS + if isinstance(p_cov, str): + p_cov = self[p_cov].data + assert p_cov.shape == (npar, npar) + + p_mc = sst.multivariate_normal.rvs(mean=p_val, cov=p_cov, size=mc_sample_size) + + if fixed_alpha: + params["alpha_mc"] = (("mc", "x"), p_mc[:, 1 : no + 1]) + params["c_mc"] = (("mc", "time"), p_mc[:, 1 + no : 1 + no + nt]) + else: + params["dalpha_mc"] = (("mc",), p_mc[:, 1]) + params["c_mc"] = (("mc", "time"), p_mc[:, 2 : nt + 2]) + + params["gamma_mc"] = (("mc",), p_mc[:, 0]) + if nta: + params["ta_mc"] = ( + ("mc", "trans_att", "time"), + np.reshape(p_mc[:, -nt * nta :], (mc_sample_size, nta, nt)), + ) + + rsize = (params.mc.size, params.x.size, params.time.size) + + if reduce_memory_usage: + memchunk = da.ones( + (mc_sample_size, no, nt), chunks={0: -1, 1: 1, 2: "auto"} + ).chunks + else: + memchunk = da.ones( + (mc_sample_size, no, nt), chunks={0: -1, 1: "auto", 2: "auto"} + ).chunks + + # Draw from the normal distributions for the Stokes intensities + for k, st_labeli, st_vari in zip( + ["r_st", "r_ast"], ["st", "ast"], [st_var, ast_var] + ): + # Load the mean as chunked Dask array, otherwise eats memory + if type(self[st_labeli].data) == da.core.Array: + loc = da.asarray(self[st_labeli].data, chunks=memchunk[1:]) + else: + loc = da.from_array(self[st_labeli].data, chunks=memchunk[1:]) + + # Make sure variance is of size (no, nt) + if np.size(st_vari) > 1: + if st_vari.shape == self[st_labeli].shape: + pass + else: + st_vari = np.broadcast_to(st_vari, (no, nt)) + else: + pass + + # Load variance as chunked Dask array, otherwise eats memory + if type(st_vari) == da.core.Array: + st_vari_da = da.asarray(st_vari, chunks=memchunk[1:]) + + elif callable(st_vari) and type(self[st_labeli].data) == da.core.Array: + st_vari_da = da.asarray( + st_vari(self[st_labeli]).data, chunks=memchunk[1:] + ) + + elif callable(st_vari) and type(self[st_labeli].data) != da.core.Array: + st_vari_da = da.from_array( + st_vari(self[st_labeli]).data, chunks=memchunk[1:] + ) + + else: + st_vari_da = da.from_array(st_vari, chunks=memchunk[1:]) + + params[k] = ( + ("mc", "x", "time"), + state.normal( + loc=loc, # has chunks=memchunk[1:] + scale=st_vari_da**0.5, + size=rsize, + chunks=memchunk, + ), + ) + + ta_arr = np.zeros((mc_sample_size, no, nt)) + + if nta: + for ii, ta in enumerate(params["ta_mc"]): + for tai, taxi in zip(ta.values, self.trans_att.values): + ta_arr[ii, self.x.values >= taxi] = ( + ta_arr[ii, self.x.values >= taxi] + tai + ) + params["ta_mc_arr"] = (("mc", "x", "time"), ta_arr) + + if fixed_alpha: + params["tmpf_mc_set"] = ( + params["gamma_mc"] + / ( + ( + np.log(params["r_st"]) + - np.log(params["r_ast"]) + + (params["c_mc"] + params["ta_mc_arr"]) + ) + + params["alpha_mc"] + ) + - 273.15 + ) + else: + params["tmpf_mc_set"] = ( + params["gamma_mc"] + / ( + ( + np.log(params["r_st"]) + - np.log(params["r_ast"]) + + (params["c_mc"] + params["ta_mc_arr"]) + ) + + (params["dalpha_mc"] * params.x) + ) + - 273.15 + ) + + avg_dims = ["mc"] + avg_axis = params["tmpf_mc_set"].get_axis_num(avg_dims) + out["tmpf_mc_var"] = (params["tmpf_mc_set"] - self["tmpf"]).var( + dim=avg_dims, ddof=1 + ) + + if conf_ints: + new_chunks = ((len(conf_ints),),) + params["tmpf_mc_set"].chunks[1:] + + qq = params["tmpf_mc_set"] + + q = qq.data.map_blocks( + lambda x: np.percentile(x, q=conf_ints, axis=avg_axis), + chunks=new_chunks, # + drop_axis=avg_axis, # avg dimesnions are dropped from input arr + new_axis=0, + ) # The new CI dimension is added as first axis + + out["tmpf_mc"] = (("CI", "x", "time"), q) + + if not mc_remove_set_flag: + out.update(params) + + self.update(out) + return out + + def conf_int_double_ended( + self, + sections=None, + p_val="p_val", + p_cov="p_cov", + st_var=None, + ast_var=None, + rst_var=None, + rast_var=None, + conf_ints=None, + mc_sample_size=100, + var_only_sections=False, + da_random_state=None, + mc_remove_set_flag=True, + reduce_memory_usage=False, + **kwargs, + ): + r""" + Estimation of the confidence intervals for the temperatures measured + with a double-ended setup. + Double-ended setups require four additional steps to estimate the + confidence intervals for the temperature. First, the variances of the + Stokes and anti-Stokes intensity measurements of the forward and + backward channels are estimated following the steps in + Section 4 [1]_. See `ds.variance_stokes_constant()`. + A Normal distribution is assigned to each + intensity measurement that is centered at the measurement and using the + estimated variance. Second, a multi-variate Normal distribution is + assigned to the estimated parameters using the covariance matrix from + the calibration procedure presented in Section 6 [1]_ (`p_cov`). Third, + Normal distributions are assigned for :math:`A` (`ds.alpha`) + for each location + outside of the reference sections. These distributions are centered + around :math:`A_p` and have variance :math:`\sigma^2\left[A_p\\right]` + given by Equations 44 and 45. Fourth, the distributions are sampled + and :math:`T_{\mathrm{F},m,n}` and :math:`T_{\mathrm{B},m,n}` are + computed with Equations 16 and 17, respectively. Fifth, step four is repeated to + compute, e.g., 10,000 realizations (`mc_sample_size`) of :math:`T_{\mathrm{F},m,n}` and + :math:`T_{\mathrm{B},m,n}` to approximate their probability density + functions. Sixth, the standard uncertainties of + :math:`T_{\mathrm{F},m,n}` and :math:`T_{\mathrm{B},m,n}` + (:math:`\sigma\left[T_{\mathrm{F},m,n}\\right]` and + :math:`\sigma\left[T_{\mathrm{B},m,n}\\right]`) are estimated with the + standard deviation of their realizations. Seventh, for each realization + :math:`i` the temperature :math:`T_{m,n,i}` is computed as the weighted + average of :math:`T_{\mathrm{F},m,n,i}` and + :math:`T_{\mathrm{B},m,n,i}`: + + .. math:: + + T_{m,n,i} =\ + \sigma^2\left[T_{m,n}\\right]\left({\\frac{T_{\mathrm{F},m,n,i}}{\ + \sigma^2\left[T_{\mathrm{F},m,n}\\right]} +\ + \\frac{T_{\mathrm{B},m,n,i}}{\ + \sigma^2\left[T_{\mathrm{B},m,n}\\right]}}\\right) + + where + + .. math:: + + \sigma^2\left[T_{m,n}\\right] = \\frac{1}{1 /\ + \sigma^2\left[T_{\mathrm{F},m,n}\\right] + 1 /\ + \sigma^2\left[T_{\mathrm{B},m,n}\\right]} + + The best estimate of the temperature :math:`T_{m,n}` is computed + directly from the best estimates of :math:`T_{\mathrm{F},m,n}` and + :math:`T_{\mathrm{B},m,n}` as: + + .. math:: + T_{m,n} =\ + \sigma^2\left[T_{m,n}\\right]\left({\\frac{T_{\mathrm{F},m,n}}{\ + \sigma^2\left[T_{\mathrm{F},m,n}\\right]} + \\frac{T_{\mathrm{B},m,n}}{\ + \sigma^2\left[T_{\mathrm{B},m,n}\\right]}}\\right) + + Alternatively, the best estimate of :math:`T_{m,n}` can be approximated + with the mean of the :math:`T_{m,n,i}` values. Finally, the 95\% + confidence interval for :math:`T_{m,n}` are estimated with the 2.5\% and + 97.5\% percentiles of :math:`T_{m,n,i}`. + + Assumes sections are set. + + Parameters + ---------- + p_val : array-like, optional + Define `p_val`, `p_var`, `p_cov` if you used an external function + for calibration. Has size `1 + 2 * nt + nx + 2 * nt * nta`. + First value is :math:`\gamma`, then `nt` times + :math:`D_\mathrm{F}`, then `nt` times + :math:`D_\mathrm{B}`, then for each location :math:`D_\mathrm{B}`, + then for each connector that introduces directional attenuation two + parameters per time step. + p_cov : array-like, optional + The covariances of `p_val`. Square matrix. + If set to False, no uncertainty in the parameters is propagated + into the confidence intervals. Similar to the spec sheets of the DTS + manufacturers. And similar to passing an array filled with zeros. + st_var, ast_var, rst_var, rast_var : float, callable, array-like, optional + The variance of the measurement noise of the Stokes signals in the + forward direction. If `float` the variance of the noise from the + Stokes detector is described with a single value. + If `callable` the variance of the noise from the Stokes detector is + a function of the intensity, as defined in the callable function. + Or manually define a variance with a DataArray of the shape + `ds.st.shape`, where the variance can be a function of time and/or + x. Required if method is wls. + conf_ints : iterable object of float + A list with the confidence boundaries that are calculated. Valid + values are between [0, 1]. + mc_sample_size : int + Size of the monte carlo parameter set used to calculate the + confidence interval + var_only_sections : bool + useful if using the ci_avg_x_flag. Only calculates the var over the + sections, so that the values can be compared with accuracy along the + reference sections. Where the accuracy is the variance of the + residuals between the estimated temperature and temperature of the + water baths. + da_random_state + For testing purposes. Similar to random seed. The seed for dask. + Makes random not so random. To produce reproducable results for + testing environments. + mc_remove_set_flag : bool + Remove the monte carlo data set, from which the CI and the + variance are calculated. + reduce_memory_usage : bool + Use less memory but at the expense of longer computation time + + Returns + ------- + + References + ---------- + .. [1] des Tombe, B., Schilperoort, B., & Bakker, M. (2020). Estimation + of Temperature and Associated Uncertainty from Fiber-Optic Raman- + Spectrum Distributed Temperature Sensing. Sensors, 20(8), 2235. + https://doi.org/10.3390/s20082235 + + """ + + def create_da_ta2(no, i_splice, direction="fw", chunks=None): + """create mask array mc, o, nt""" + + if direction == "fw": + arr = da.concatenate( + ( + da.zeros((1, i_splice, 1), chunks=(1, i_splice, 1), dtype=bool), + da.ones( + (1, no - i_splice, 1), + chunks=(1, no - i_splice, 1), + dtype=bool, + ), + ), + axis=1, + ).rechunk((1, chunks[1], 1)) + else: + arr = da.concatenate( + ( + da.ones((1, i_splice, 1), chunks=(1, i_splice, 1), dtype=bool), + da.zeros( + (1, no - i_splice, 1), + chunks=(1, no - i_splice, 1), + dtype=bool, + ), + ), + axis=1, + ).rechunk((1, chunks[1], 1)) + return arr + + out = xr.Dataset() + params = xr.Dataset() + + if da_random_state: + # In testing environments + assert isinstance(da_random_state, da.random.RandomState) + state = da_random_state + else: + state = da.random.RandomState() + + if conf_ints: + assert "tmpw", ( + "Current implementation requires you to " + 'define "tmpw" when estimating confidence ' + "intervals" + ) + + no, nt = self.st.shape + nta = self.trans_att.size + npar = 1 + 2 * nt + no + nt * 2 * nta # number of parameters + + rsize = (mc_sample_size, no, nt) + + if reduce_memory_usage: + memchunk = da.ones( + (mc_sample_size, no, nt), chunks={0: -1, 1: 1, 2: "auto"} + ).chunks + else: + memchunk = da.ones( + (mc_sample_size, no, nt), chunks={0: -1, 1: "auto", 2: "auto"} + ).chunks + + params.coords["mc"] = range(mc_sample_size) + params.coords["x"] = self.x + params.coords["time"] = self.time + + if conf_ints: + self.coords["CI"] = conf_ints + params.coords["CI"] = conf_ints + + assert isinstance(p_val, (str, np.ndarray, np.generic)) + if isinstance(p_val, str): + p_val = self[p_val].values + assert p_val.shape == (npar,), ( + "Did you set 'talpha' as " + "keyword argument of the " + "conf_int_double_ended() function?" + ) + + assert isinstance(p_cov, (str, np.ndarray, np.generic, bool)) + + if isinstance(p_cov, bool) and not p_cov: + # Exclude parameter uncertainty if p_cov == False + gamma = p_val[0] + d_fw = p_val[1 : nt + 1] + d_bw = p_val[1 + nt : 2 * nt + 1] + alpha = p_val[2 * nt + 1 : 2 * nt + 1 + no] + + params["gamma_mc"] = (tuple(), gamma) + params["alpha_mc"] = (("x",), alpha) + params["df_mc"] = (("time",), d_fw) + params["db_mc"] = (("time",), d_bw) + + if nta: + ta = p_val[2 * nt + 1 + no :].reshape((nt, 2, nta), order="F") + ta_fw = ta[:, 0, :] + ta_bw = ta[:, 1, :] + + ta_fw_arr = np.zeros((no, nt)) + for tai, taxi in zip(ta_fw.T, params.coords["trans_att"].values): + ta_fw_arr[params.x.values >= taxi] = ( + ta_fw_arr[params.x.values >= taxi] + tai + ) + + ta_bw_arr = np.zeros((no, nt)) + for tai, taxi in zip(ta_bw.T, params.coords["trans_att"].values): + ta_bw_arr[params.x.values < taxi] = ( + ta_bw_arr[params.x.values < taxi] + tai + ) + + params["talpha_fw_mc"] = (("x", "time"), ta_fw_arr) + params["talpha_bw_mc"] = (("x", "time"), ta_bw_arr) + + elif isinstance(p_cov, bool) and p_cov: + raise NotImplementedError("Not an implemented option. Check p_cov argument") + + else: + # WLS + if isinstance(p_cov, str): + p_cov = self[p_cov].values + assert p_cov.shape == (npar, npar) + + assert sections is not None, "Define sections" + ix_sec = self.ufunc_per_section( + sections=sections, x_indices=True, calc_per="all" + ) + nx_sec = ix_sec.size + from_i = np.concatenate( + ( + np.arange(1 + 2 * nt), + 1 + 2 * nt + ix_sec, + np.arange(1 + 2 * nt + no, 1 + 2 * nt + no + nt * 2 * nta), + ) + ) + iox_sec1, iox_sec2 = np.meshgrid(from_i, from_i, indexing="ij") + po_val = p_val[from_i] + po_cov = p_cov[iox_sec1, iox_sec2] + + po_mc = sst.multivariate_normal.rvs( + mean=po_val, cov=po_cov, size=mc_sample_size + ) + + gamma = po_mc[:, 0] + d_fw = po_mc[:, 1 : nt + 1] + d_bw = po_mc[:, 1 + nt : 2 * nt + 1] + + params["gamma_mc"] = (("mc",), gamma) + params["df_mc"] = (("mc", "time"), d_fw) + params["db_mc"] = (("mc", "time"), d_bw) + + # calculate alpha seperately + alpha = np.zeros((mc_sample_size, no), dtype=float) + alpha[:, ix_sec] = po_mc[:, 1 + 2 * nt : 1 + 2 * nt + nx_sec] + + not_ix_sec = np.array([i for i in range(no) if i not in ix_sec]) + + if np.any(not_ix_sec): + not_alpha_val = p_val[2 * nt + 1 + not_ix_sec] + not_alpha_var = p_cov[2 * nt + 1 + not_ix_sec, 2 * nt + 1 + not_ix_sec] + + not_alpha_mc = np.random.normal( + loc=not_alpha_val, + scale=not_alpha_var**0.5, + size=(mc_sample_size, not_alpha_val.size), + ) + + alpha[:, not_ix_sec] = not_alpha_mc + + params["alpha_mc"] = (("mc", "x"), alpha) + + if nta: + ta = po_mc[:, 2 * nt + 1 + nx_sec :].reshape( + (mc_sample_size, nt, 2, nta), order="F" + ) + ta_fw = ta[:, :, 0, :] + ta_bw = ta[:, :, 1, :] + + ta_fw_arr = da.zeros( + (mc_sample_size, no, nt), chunks=memchunk, dtype=float + ) + for tai, taxi in zip( + ta_fw.swapaxes(0, 2), params.coords["trans_att"].values + ): + # iterate over the splices + i_splice = sum(params.x.values < taxi) + mask = create_da_ta2(no, i_splice, direction="fw", chunks=memchunk) + + ta_fw_arr += mask * tai.T[:, None, :] + + ta_bw_arr = da.zeros( + (mc_sample_size, no, nt), chunks=memchunk, dtype=float + ) + for tai, taxi in zip( + ta_bw.swapaxes(0, 2), params.coords["trans_att"].values + ): + i_splice = sum(params.x.values < taxi) + mask = create_da_ta2(no, i_splice, direction="bw", chunks=memchunk) + + ta_bw_arr += mask * tai.T[:, None, :] + + params["talpha_fw_mc"] = (("mc", "x", "time"), ta_fw_arr) + params["talpha_bw_mc"] = (("mc", "x", "time"), ta_bw_arr) + + # Draw from the normal distributions for the Stokes intensities + for k, st_labeli, st_vari in zip( + ["r_st", "r_ast", "r_rst", "r_rast"], + ["st", "ast", "rst", "rast"], + [st_var, ast_var, rst_var, rast_var], + ): + # Load the mean as chunked Dask array, otherwise eats memory + if type(self[st_labeli].data) == da.core.Array: + loc = da.asarray(self[st_labeli].data, chunks=memchunk[1:]) + else: + loc = da.from_array(self[st_labeli].data, chunks=memchunk[1:]) + + # Make sure variance is of size (no, nt) + if np.size(st_vari) > 1: + if st_vari.shape == self[st_labeli].shape: + pass + else: + st_vari = np.broadcast_to(st_vari, (no, nt)) + else: + pass + + # Load variance as chunked Dask array, otherwise eats memory + if type(st_vari) == da.core.Array: + st_vari_da = da.asarray(st_vari, chunks=memchunk[1:]) + + elif callable(st_vari) and type(self[st_labeli].data) == da.core.Array: + st_vari_da = da.asarray( + st_vari(self[st_labeli]).data, chunks=memchunk[1:] + ) + + elif callable(st_vari) and type(self[st_labeli].data) != da.core.Array: + st_vari_da = da.from_array( + st_vari(self[st_labeli]).data, chunks=memchunk[1:] + ) + + else: + st_vari_da = da.from_array(st_vari, chunks=memchunk[1:]) + + params[k] = ( + ("mc", "x", "time"), + state.normal( + loc=loc, # has chunks=memchunk[1:] + scale=st_vari_da**0.5, + size=rsize, + chunks=memchunk, + ), + ) + + for label in ["tmpf", "tmpb"]: + if "tmpw" or label: + if label == "tmpf": + if nta: + params["tmpf_mc_set"] = ( + params["gamma_mc"] + / ( + np.log(params["r_st"] / params["r_ast"]) + + params["df_mc"] + + params["alpha_mc"] + + params["talpha_fw_mc"] + ) + - 273.15 + ) + else: + params["tmpf_mc_set"] = ( + params["gamma_mc"] + / ( + np.log(params["r_st"] / params["r_ast"]) + + params["df_mc"] + + params["alpha_mc"] + ) + - 273.15 + ) + else: + if nta: + params["tmpb_mc_set"] = ( + params["gamma_mc"] + / ( + np.log(params["r_rst"] / params["r_rast"]) + + params["db_mc"] + - params["alpha_mc"] + + params["talpha_bw_mc"] + ) + - 273.15 + ) + else: + params["tmpb_mc_set"] = ( + params["gamma_mc"] + / ( + np.log(params["r_rst"] / params["r_rast"]) + + params["db_mc"] + - params["alpha_mc"] + ) + - 273.15 + ) + + if var_only_sections: + # sets the values outside the reference sections to NaN + xi = self.ufunc_per_section( + sections=sections, x_indices=True, calc_per="all" + ) + x_mask_ = [ + True if ix in xi else False for ix in range(params.x.size) + ] + x_mask = np.reshape(x_mask_, (1, -1, 1)) + params[label + "_mc_set"] = params[label + "_mc_set"].where(x_mask) + + # subtract the mean temperature + q = params[label + "_mc_set"] - self[label] + out[label + "_mc_var"] = q.var(dim="mc", ddof=1) + + if conf_ints: + new_chunks = list(params[label + "_mc_set"].chunks) + new_chunks[0] = (len(conf_ints),) + avg_axis = params[label + "_mc_set"].get_axis_num("mc") + q = params[label + "_mc_set"].data.map_blocks( + lambda x: np.percentile(x, q=conf_ints, axis=avg_axis), + chunks=new_chunks, # + drop_axis=avg_axis, + # avg dimensions are dropped from input arr + new_axis=0, + ) # The new CI dimension is added as firsaxis + + out[label + "_mc"] = (("CI", "x", "time"), q) + + # Weighted mean of the forward and backward + tmpw_var = 1 / (1 / out["tmpf_mc_var"] + 1 / out["tmpb_mc_var"]) + + q = ( + params["tmpf_mc_set"] / out["tmpf_mc_var"] + + params["tmpb_mc_set"] / out["tmpb_mc_var"] + ) * tmpw_var + + params["tmpw" + "_mc_set"] = q # + + out["tmpw"] = ( + self["tmpf"] / out["tmpf_mc_var"] + self["tmpb"] / out["tmpb_mc_var"] + ) * tmpw_var + + q = params["tmpw" + "_mc_set"] - self["tmpw"] + out["tmpw" + "_mc_var"] = q.var(dim="mc", ddof=1) + + # Calculate the CI of the weighted MC_set + if conf_ints: + new_chunks_weighted = ((len(conf_ints),),) + memchunk[1:] + avg_axis = params["tmpw" + "_mc_set"].get_axis_num("mc") + q2 = params["tmpw" + "_mc_set"].data.map_blocks( + lambda x: np.percentile(x, q=conf_ints, axis=avg_axis), + chunks=new_chunks_weighted, # Explicitly define output chunks + drop_axis=avg_axis, # avg dimensions are dropped + new_axis=0, + dtype=float, + ) # The new CI dimension is added as first axis + out["tmpw" + "_mc"] = (("CI", "x", "time"), q2) + + # Clean up the garbage. All arrays with a Monte Carlo dimension. + if mc_remove_set_flag: + remove_mc_set = [ + "r_st", + "r_ast", + "r_rst", + "r_rast", + "gamma_mc", + "alpha_mc", + "df_mc", + "db_mc", + ] + + for i in ["tmpf", "tmpb", "tmpw"]: + remove_mc_set.append(i + "_mc_set") + + if nta: + remove_mc_set.append('talpha"_fw_mc') + remove_mc_set.append('talpha"_bw_mc') + + for k in remove_mc_set: + if k in out: + del out[k] + + if not mc_remove_set_flag: + out.update(params) + + self.update(out) + return out diff --git a/src/dtscalibration/datastore_utils.py b/src/dtscalibration/datastore_utils.py index 0d54dd31..a7de1d22 100644 --- a/src/dtscalibration/datastore_utils.py +++ b/src/dtscalibration/datastore_utils.py @@ -462,43 +462,41 @@ def check_deprecated_kwargs(kwargs): pass -def check_timestep_allclose(ds: "DataStore", eps: float = 0.01) -> None: - """ - Check if all timesteps are of equal size. For now it is not possible to calibrate - over timesteps if the acquisition time of timesteps varies, as the Stokes variance - would change over time. - - The acquisition time is stored for single ended measurements in userAcquisitionTime, - for double ended measurements in userAcquisitionTimeFW and userAcquisitionTimeBW. - - Parameters - ---------- - ds : DataStore - eps : float - Default accepts 1% of relative variation between min and max acquisition time. - - Returns - ------- - - """ - dim = ds.channel_configuration["chfw"]["acquisitiontime_label"] - dt = ds[dim].data - dtmin = dt.min() - dtmax = dt.max() - dtavg = (dtmin + dtmax) / 2 - assert (dtmax - dtmin) / dtavg < eps, ( - "Acquisition time is Forward channel not equal for all " "time steps" - ) - - if ds.is_double_ended: - dim = ds.channel_configuration["chbw"]["acquisitiontime_label"] - dt = ds[dim].data - dtmin = dt.min() - dtmax = dt.max() - dtavg = (dtmin + dtmax) / 2 - assert (dtmax - dtmin) / dtavg < eps, ( - "Acquisition time Backward channel is not equal " "for all time steps" - ) +# def check_timestep_allclose(ds: "DataStore", eps: float = 0.05) -> None: +# """ +# Check if all timesteps are of equal size. For now it is not possible to calibrate +# over timesteps if the acquisition time of timesteps varies, as the Stokes variance +# would change over time. + +# The acquisition time is stored for single ended measurements in userAcquisitionTime, +# for double ended measurements in userAcquisitionTimeFW and userAcquisitionTimeBW. + +# Parameters +# ---------- +# ds : DataStore +# eps : float +# Default accepts 1% of relative variation between min and max acquisition time. + +# Returns +# ------- +# """ +# dt = ds["userAcquisitionTimeFW"].data +# dtmin = dt.min() +# dtmax = dt.max() +# dtavg = (dtmin + dtmax) / 2 +# assert (dtmax - dtmin) / dtavg < eps, ( +# "Acquisition time is Forward channel not equal for all time steps" +# ) + +# if "userAcquisitionTimeBW" in ds: +# dt = ds["userAcquisitionTimeBW"].data +# dtmin = dt.min() +# dtmax = dt.max() +# dtavg = (dtmin + dtmax) / 2 +# assert (dtmax - dtmin) / dtavg < eps, ( +# "Acquisition time Backward channel is not equal for all time steps" +# ) +# pass def get_netcdf_encoding( @@ -698,7 +696,14 @@ def get_params_from_pval_double_ended(ip, coords, p_val=None, p_cov=None): def get_params_from_pval_single_ended( ip, coords, p_val=None, p_var=None, p_cov=None, fix_alpha=None ): - assert len(p_val) == ip.npar, "Length of p_val is incorrect" + if p_val is not None: + assert len(p_val) == ip.npar, "Length of p_val is incorrect" + + if p_var is not None: + assert len(p_var) == ip.npar, "Length of p_var is incorrect" + + if p_cov is not None: + assert p_cov.shape == (ip.npar, ip.npar), "Shape of p_cov is incorrect" params = xr.Dataset(coords=coords) param_covs = xr.Dataset(coords=coords) @@ -1328,7 +1333,7 @@ def ufunc_per_section_helper( 7. x-coordinate index - >>> ix_loc = ufunc_per_section_helperx_coords=d.x) + >>> ix_loc = ufunc_per_section_helper(x_coords=d.x) Note @@ -1370,6 +1375,7 @@ def func(a): assert callable(func) assert calc_per in ["all", "section", "stretch"] + assert "x_indices" not in func_kwargs, "pass x_coords arg instead" if x_coords is None and ( (dataarray is not None and hasattr(dataarray.data, "chunks")) @@ -1389,6 +1395,8 @@ def func(a): assert subtract_from_dataarray is None assert not subtract_reference_from_dataarray assert not ref_temp_broadcasted + assert not func_kwargs, "Unsupported kwargs" + # so it is slicable with x-indices _x_indices = x_coords.astype(int) * 0 + np.arange(x_coords.size) arg1 = _x_indices.sel(x=stretch).data diff --git a/src/dtscalibration/plot.py b/src/dtscalibration/plot.py index 13c0c111..867abe4f 100755 --- a/src/dtscalibration/plot.py +++ b/src/dtscalibration/plot.py @@ -572,7 +572,7 @@ def plot_sigma_report( # calc_per='stretch', # temp_err=True, # axis=0) - sigma_est = ds.ufunc_per_section( + sigma_est = ds.dts.ufunc_per_section( sections=sections, label=temp_label, func=np.std, @@ -581,7 +581,7 @@ def plot_sigma_report( axis=0, ) else: - sigma_est = ds.ufunc_per_section( + sigma_est = ds.dts.ufunc_per_section( sections=sections, label=temp_label, func=np.std, @@ -635,14 +635,14 @@ def plot_sigma_report( ax1.legend() ax1.set_ylabel(r"Temperature [$^\circ$C]") - err_ref = ds.ufunc_per_section( + err_ref = ds.dts.ufunc_per_section( sections=sections, label=temp_label, func=None, temp_err=True, calc_per="stretch", ) - x_ref = ds.ufunc_per_section(sections=sections, label="x", calc_per="stretch") + x_ref = ds.dts.ufunc_per_section(sections=sections, label="x", calc_per="stretch") for (k, v), (k_se, v_se), (kx, vx) in zip( ds.sections.items(), err_ref.items(), x_ref.items() diff --git a/src/dtscalibration/variance_helpers.py b/src/dtscalibration/variance_helpers.py index e1b930c3..6298810a 100644 --- a/src/dtscalibration/variance_helpers.py +++ b/src/dtscalibration/variance_helpers.py @@ -167,3 +167,30 @@ def var_fun(stokes): return slope * stokes + offset return slope, offset, st_sort_mean, st_sort_var, resid_sec, var_fun + + +def check_allclose_acquisitiontime(acquisitiontime, eps: float = 0.05) -> None: + """ + Check if all acquisition times are of equal duration. For now it is not possible to calibrate + over timesteps if the acquisition time of timesteps varies, as the Stokes variance + would change over time. + + The acquisition time is stored for single ended measurements in userAcquisitionTime, + for double ended measurements in userAcquisitionTimeFW and userAcquisitionTimeBW. + + Parameters + ---------- + ds : DataStore + eps : float + Default accepts 1% of relative variation between min and max acquisition time. + + Returns + ------- + """ + dtmin = acquisitiontime.min() + dtmax = acquisitiontime.max() + dtavg = (dtmin + dtmax) / 2 + assert (dtmax - dtmin) / dtavg < eps, ( + "Acquisition time is Forward channel not equal for all time steps" + ) + pass diff --git a/src/dtscalibration/variance_stokes.py b/src/dtscalibration/variance_stokes.py index 0393f259..eb09041a 100644 --- a/src/dtscalibration/variance_stokes.py +++ b/src/dtscalibration/variance_stokes.py @@ -8,9 +8,10 @@ from dtscalibration.variance_helpers import variance_stokes_linear_helper from dtscalibration.calibration.section_utils import validate_sections_definition from dtscalibration.calibration.section_utils import validate_no_overlapping_sections +from dtscalibration.variance_helpers import check_allclose_acquisitiontime -def variance_stokes_constant(st, sections, reshape_residuals=True): +def variance_stokes_constant(st, sections, acquisitiontime, reshape_residuals=True): """ Approximate the variance of the noise in Stokes intensity measurements with one value, suitable for small setups. @@ -116,6 +117,7 @@ def variance_stokes_constant(st, sections, reshape_residuals=True): """ validate_sections_definition(sections=sections) validate_no_overlapping_sections(sections=sections) + check_allclose_acquisitiontime(acquisitiontime=acquisitiontime) assert st.dims[0] == "x", "DataArray is transposed" @@ -149,6 +151,7 @@ def variance_stokes_constant(st, sections, reshape_residuals=True): def variance_stokes_exponential( st, sections, + acquisitiontime, use_statsmodels=False, suppress_info=True, reshape_residuals=True, @@ -281,6 +284,7 @@ def variance_stokes_exponential( """ validate_sections_definition(sections=sections) validate_no_overlapping_sections(sections=sections) + check_allclose_acquisitiontime(acquisitiontime=acquisitiontime) assert st.dims[0] == "x", "Stokes are transposed" nt = st.coords["time"].size @@ -339,10 +343,10 @@ def variance_stokes_exponential( resid_da = xr.DataArray(data=resid_sorted, coords=st.coords) return var_I, resid_da - + def variance_stokes_linear( - st, sections, nbin=50, through_zero=False, plot_fit=False + st, sections, acquisitiontime, nbin=50, through_zero=False, plot_fit=False ): """ Approximate the variance of the noise in Stokes intensity measurements @@ -457,6 +461,7 @@ def variance_stokes_linear( """ validate_sections_definition(sections=sections) validate_no_overlapping_sections(sections=sections) + check_allclose_acquisitiontime(acquisitiontime=acquisitiontime) assert st.dims[0] == "x", "Stokes are transposed" _, resid = variance_stokes_constant( diff --git a/tests/test_dtscalibration.py b/tests/test_dtscalibration.py index 376b5a3b..b15cee52 100644 --- a/tests/test_dtscalibration.py +++ b/tests/test_dtscalibration.py @@ -4,11 +4,13 @@ import pytest import scipy.sparse as sp from scipy import stats - -from dtscalibration import DataStore +from xarray import Dataset from dtscalibration import read_silixa_files from dtscalibration.calibrate_utils import wls_sparse from dtscalibration.calibrate_utils import wls_stats +from dtscalibration.datastore_accessor import DtsAccessor # noqa: F401 +from dtscalibration.variance_stokes import variance_stokes_exponential + np.random.seed(0) @@ -59,8 +61,6 @@ def assert_almost_equal_verbose(actual, desired, verbose=False, **kwargs): def test_variance_input_types_single(): import dask.array as da - from src.dtscalibration import DataStore - state = da.random.RandomState(0) stokes_m_var = 40.0 @@ -105,7 +105,7 @@ def test_variance_input_types_single(): print("C", np.log(C_p / C_m)) print("x0", x.max()) - ds = DataStore( + ds = Dataset( { "st": (["x", "time"], st_m), "ast": (["x", "time"], ast_m), @@ -125,11 +125,11 @@ def test_variance_input_types_single(): # Test float input st_var = 5.0 - ds.calibration_single_ended( + ds.dts.calibration_single_ended( sections=sections, st_var=st_var, ast_var=st_var, method="wls", solver="sparse" ) - ds.conf_int_single_ended( + ds.dts.conf_int_single_ended( st_var=st_var, ast_var=st_var, mc_sample_size=100, @@ -150,7 +150,7 @@ def callable_st_var(stokes): offset = 0 return slope * stokes + offset - ds.calibration_single_ended( + ds.dts.calibration_single_ended( sections=sections, st_var=callable_st_var, ast_var=callable_st_var, @@ -174,11 +174,11 @@ def callable_st_var(stokes): # Test input with shape of (ntime, nx) st_var = ds.st.values * 0 + 20.0 - ds.calibration_single_ended( + ds.dts.calibration_single_ended( sections=sections, st_var=st_var, ast_var=st_var, method="wls", solver="sparse" ) - ds.conf_int_single_ended( + ds.dts.conf_int_single_ended( st_var=st_var, ast_var=st_var, mc_sample_size=100, da_random_state=state ) @@ -189,11 +189,11 @@ def callable_st_var(stokes): ds.st.mean(dim="time").values * 0 + np.linspace(10, 50, num=ds.st.x.size) ) - ds.calibration_single_ended( + ds.dts.calibration_single_ended( sections=sections, st_var=st_var, ast_var=st_var, method="wls", solver="sparse" ) - ds.conf_int_single_ended( + ds.dts.conf_int_single_ended( st_var=st_var, ast_var=st_var, mc_sample_size=100, da_random_state=state ) @@ -207,11 +207,11 @@ def callable_st_var(stokes): # Test input with shape (ntime) st_var = ds.st.mean(dim="x").values * 0 + np.linspace(5, 200, num=nt) - ds.calibration_single_ended( + ds.dts.calibration_single_ended( sections=sections, st_var=st_var, ast_var=st_var, method="wls", solver="sparse" ) - ds.conf_int_single_ended( + ds.dts.conf_int_single_ended( st_var=st_var, ast_var=st_var, mc_sample_size=100, da_random_state=state ) @@ -229,8 +229,6 @@ def callable_st_var(stokes): def test_variance_input_types_double(): import dask.array as da - from src.dtscalibration import DataStore - state = da.random.RandomState(0) stokes_m_var = 40.0 @@ -290,7 +288,7 @@ def test_variance_input_types_double(): print("C", np.log(C_p / C_m)) print("x0", x.max()) - ds = DataStore( + ds = Dataset( { "st": (["x", "time"], st_m), "ast": (["x", "time"], ast_m), @@ -313,7 +311,7 @@ def test_variance_input_types_double(): # Test float input st_var = 5.0 - ds.calibration_double_ended( + ds.dts.calibration_double_ended( sections=sections, st_var=st_var, ast_var=st_var, @@ -346,7 +344,7 @@ def st_var_callable(stokes): offset = 0 return slope * stokes + offset - ds.calibration_double_ended( + ds.dts.calibration_double_ended( sections=sections, st_var=st_var_callable, ast_var=st_var_callable, @@ -376,7 +374,7 @@ def st_var_callable(stokes): # Test input with shape of (ntime, nx) st_var = ds.st.values * 0 + 20.0 - ds.calibration_double_ended( + ds.dts.calibration_double_ended( sections=sections, st_var=st_var, ast_var=st_var, @@ -403,7 +401,7 @@ def st_var_callable(stokes): ds.st.mean(dim="time").values * 0 + np.linspace(10, 50, num=ds.st.x.size) ) - ds.calibration_double_ended( + ds.dts.calibration_double_ended( sections=sections, st_var=st_var, ast_var=st_var, @@ -433,7 +431,7 @@ def st_var_callable(stokes): # Test input with shape (ntime) st_var = ds.st.mean(dim="x").values * 0 + np.linspace(5, 200, num=nt) - ds.calibration_double_ended( + ds.dts.calibration_double_ended( sections=sections, st_var=st_var, ast_var=st_var, @@ -467,8 +465,6 @@ def st_var_callable(stokes): def test_double_ended_variance_estimate_synthetic(): import dask.array as da - from src.dtscalibration import DataStore - state = da.random.RandomState(0) stokes_m_var = 40.0 @@ -528,7 +524,7 @@ def test_double_ended_variance_estimate_synthetic(): print("C", np.log(C_p / C_m)) print("x0", x.max()) - ds = DataStore( + ds = Dataset( { "st": (["x", "time"], st_m), "ast": (["x", "time"], ast_m), @@ -559,7 +555,7 @@ def test_double_ended_variance_estimate_synthetic(): mrast_var = float(mrast_var) # MC variance - ds.calibration_double_ended( + ds.dts.calibration_double_ended( sections=sections, st_var=mst_var, ast_var=mast_var, @@ -627,8 +623,6 @@ def test_double_ended_variance_estimate_synthetic(): def test_single_ended_variance_estimate_synthetic(): import dask.array as da - from src.dtscalibration import DataStore - state = da.random.RandomState(0) stokes_m_var = 40.0 @@ -673,7 +667,7 @@ def test_single_ended_variance_estimate_synthetic(): print("C", np.log(C_p / C_m)) print("x0", x.max()) - ds = DataStore( + ds = Dataset( { "st": (["x", "time"], st_m), "ast": (["x", "time"], ast_m), @@ -699,7 +693,7 @@ def test_single_ended_variance_estimate_synthetic(): mast_var = float(mast_var) # MC variqnce - ds.calibration_single_ended( + ds.dts.calibration_single_ended( sections=sections, st_var=mst_var, ast_var=mast_var, @@ -707,7 +701,7 @@ def test_single_ended_variance_estimate_synthetic(): solver="sparse", ) - ds.conf_int_single_ended( + ds.dts.conf_int_single_ended( p_val="p_val", p_cov="p_cov", st_var=mst_var, @@ -797,7 +791,7 @@ def test_double_ended_wls_estimate_synthetic(): alpha = np.mean(np.log(rst / rast) - np.log(st / ast), axis=1) / 2 alpha -= alpha[0] # the first x-index is where to start counting - ds = DataStore( + ds = Dataset( { "st": (["x", "time"], st), "ast": (["x", "time"], ast), @@ -818,7 +812,7 @@ def test_double_ended_wls_estimate_synthetic(): } # WLS - ds.calibration_double_ended( + ds.dts.calibration_double_ended( sections=sections, st_var=1e-7, ast_var=1e-7, @@ -842,7 +836,6 @@ def test_double_ended_wls_estimate_synthetic_df_and_db_are_different(): They should be the same as the parameters used to create the synthetic measurment set. This one has a different D for the forward channel than for the backward channel.""" - from dtscalibration import DataStore cable_len = 100.0 nt = 3 @@ -911,7 +904,7 @@ def test_double_ended_wls_estimate_synthetic_df_and_db_are_different(): E_real = (i_bw - i_fw) / 2 + (db - df) / 2 - ds = DataStore( + ds = Dataset( { "st": (["x", "time"], st), "ast": (["x", "time"], ast), @@ -933,7 +926,7 @@ def test_double_ended_wls_estimate_synthetic_df_and_db_are_different(): real_ans2 = np.concatenate(([gamma], df, db, E_real[:, 0])) - ds.calibration_double_ended( + ds.dts.calibration_double_ended( sections=sections, st_var=1.5, ast_var=1.5, @@ -960,7 +953,6 @@ def test_reneaming_old_default_labels_to_new_fixed_labels(): """Same as `test_double_ended_wls_estimate_synthetic_df_and_db_are_different` Which runs fast, but using the renaming function.""" - from dtscalibration import DataStore cable_len = 100.0 nt = 3 @@ -1029,7 +1021,7 @@ def test_reneaming_old_default_labels_to_new_fixed_labels(): E_real = (i_bw - i_fw) / 2 + (db - df) / 2 - ds = DataStore( + ds = Dataset( { "ST": (["x", "time"], st), "AST": (["x", "time"], ast), @@ -1052,7 +1044,7 @@ def test_reneaming_old_default_labels_to_new_fixed_labels(): real_ans2 = np.concatenate(([gamma], df, db, E_real[:, 0])) - ds.calibration_double_ended( + ds.dts.calibration_double_ended( sections=sections, st_var=1.5, ast_var=1.5, @@ -1080,7 +1072,6 @@ def test_fail_if_st_labels_are_passed_to_calibration_function(): """Same as `test_double_ended_wls_estimate_synthetic_df_and_db_are_different` Which runs fast.""" - from dtscalibration import DataStore cable_len = 100.0 nt = 3 @@ -1135,7 +1126,7 @@ def test_fail_if_st_labels_are_passed_to_calibration_function(): / (np.exp(gamma / temp_real_kelvin) - 1) ) - ds = DataStore( + ds = Dataset( { "ST": (["x", "time"], st), "AST": (["x", "time"], ast), @@ -1156,7 +1147,7 @@ def test_fail_if_st_labels_are_passed_to_calibration_function(): "warm": [slice(0.9 * cable_len, cable_len)], } - ds.calibration_double_ended( + ds.dts.calibration_double_ended( sections=sections, st_label="ST", ast_label="AST", @@ -1176,8 +1167,6 @@ def test_fail_if_st_labels_are_passed_to_calibration_function(): def test_double_ended_asymmetrical_attenuation(): - from dtscalibration import DataStore - cable_len = 100.0 nt = 3 time = np.arange(nt) @@ -1242,7 +1231,7 @@ def test_double_ended_asymmetrical_attenuation(): / (np.exp(gamma / temp_real_kelvin) - 1) ) - ds = DataStore( + ds = Dataset( { "TMPR": (["x", "time"], temp_real_celsius), "st": (["x", "time"], st), @@ -1266,7 +1255,7 @@ def test_double_ended_asymmetrical_attenuation(): ], } - ds.calibration_double_ended( + ds.dts.calibration_double_ended( sections=sections, st_var=1.5, ast_var=1.5, @@ -1296,7 +1285,7 @@ def test_double_ended_asymmetrical_attenuation(): assert len(del_keys) == 0, "clear out trans_att config" # About to be depreciated - ds.calibration_double_ended( + ds.dts.calibration_double_ended( sections=sections, st_var=1.5, ast_var=1.5, @@ -1314,8 +1303,6 @@ def test_double_ended_asymmetrical_attenuation(): def test_double_ended_one_matching_section_and_one_asym_att(): - from dtscalibration import DataStore - cable_len = 100.0 nt = 3 time = np.arange(nt) @@ -1380,7 +1367,7 @@ def test_double_ended_one_matching_section_and_one_asym_att(): / (np.exp(gamma / temp_real_kelvin) - 1) ) - ds = DataStore( + ds = Dataset( { "TMPR": (["x", "time"], temp_real_celsius), "st": (["x", "time"], st), @@ -1401,7 +1388,7 @@ def test_double_ended_one_matching_section_and_one_asym_att(): "warm": [slice(x[nx_per_sec], x[2 * nx_per_sec - 1])], } - ds.calibration_double_ended( + ds.dts.calibration_double_ended( sections=sections, st_var=1.5, ast_var=1.5, @@ -1427,7 +1414,6 @@ def test_double_ended_one_matching_section_and_one_asym_att(): def test_double_ended_two_matching_sections_and_two_asym_atts(): """Setup contains two matching sections and two connectors that introduce asymmetrical attenuation. Solves beautifully.""" - from dtscalibration import DataStore cable_len = 100.0 nt = 5 @@ -1499,7 +1485,7 @@ def test_double_ended_two_matching_sections_and_two_asym_atts(): / (np.exp(gamma / temp_real_kelvin) - 1) ) - ds = DataStore( + ds = Dataset( { "TMPR": (["x", "time"], temp_real_celsius), "st": (["x", "time"], st), @@ -1532,7 +1518,7 @@ def test_double_ended_two_matching_sections_and_two_asym_atts(): ), ] - ds.calibration_double_ended( + ds.dts.calibration_double_ended( sections=sections, st_var=0.5, ast_var=0.5, @@ -1556,7 +1542,6 @@ def test_double_ended_wls_fix_gamma_estimate_synthetic(): Without variance. They should be the same as the parameters used to create the synthetic measurment set""" - from dtscalibration import DataStore cable_len = 100.0 nt = 500 @@ -1608,7 +1593,7 @@ def test_double_ended_wls_fix_gamma_estimate_synthetic(): # to ensure the st, rst, ast, rast were correctly defined. np.testing.assert_allclose(alpha2, alpha, atol=1e-15, rtol=0) - ds = DataStore( + ds = Dataset( { "st": (["x", "time"], st), "ast": (["x", "time"], ast), @@ -1629,7 +1614,7 @@ def test_double_ended_wls_fix_gamma_estimate_synthetic(): } # WLS - ds.calibration_double_ended( + ds.dts.calibration_double_ended( sections=sections, st_var=1e-12, ast_var=1e-12, @@ -1655,7 +1640,6 @@ def test_double_ended_wls_fix_alpha_estimate_synthetic(): Without variance. They should be the same as the parameters used to create the synthetic measurment set""" - from dtscalibration import DataStore cable_len = 100.0 nt = 500 @@ -1702,7 +1686,7 @@ def test_double_ended_wls_fix_alpha_estimate_synthetic(): alpha = np.mean(np.log(rst / rast) - np.log(st / ast), axis=1) / 2 alpha -= alpha[0] # the first x-index is where to start counting - ds = DataStore( + ds = Dataset( { "st": (["x", "time"], st), "ast": (["x", "time"], ast), @@ -1723,7 +1707,7 @@ def test_double_ended_wls_fix_alpha_estimate_synthetic(): } # WLS - ds.calibration_double_ended( + ds.dts.calibration_double_ended( sections=sections, st_var=1e-7, ast_var=1e-7, @@ -1749,7 +1733,6 @@ def test_double_ended_wls_fix_alpha_fix_gamma_estimate_synthetic(): Without variance. They should be the same as the parameters used to create the synthetic measurment set""" - from dtscalibration import DataStore cable_len = 100.0 nt = 500 @@ -1796,7 +1779,7 @@ def test_double_ended_wls_fix_alpha_fix_gamma_estimate_synthetic(): alpha = np.mean(np.log(rst / rast) - np.log(st / ast), axis=1) / 2 alpha -= alpha[0] # the first x-index is where to start counting - ds = DataStore( + ds = Dataset( { "st": (["x", "time"], st), "ast": (["x", "time"], ast), @@ -1817,7 +1800,7 @@ def test_double_ended_wls_fix_alpha_fix_gamma_estimate_synthetic(): } # WLS - ds.calibration_double_ended( + ds.dts.calibration_double_ended( sections=sections, st_var=1e-7, ast_var=1e-7, @@ -1839,8 +1822,6 @@ def test_double_ended_wls_fix_alpha_fix_gamma_estimate_synthetic(): def test_double_ended_fix_alpha_matching_sections_and_one_asym_att(): - from dtscalibration import DataStore - cable_len = 100.0 nt = 3 time = np.arange(nt) @@ -1905,7 +1886,7 @@ def test_double_ended_fix_alpha_matching_sections_and_one_asym_att(): / (np.exp(gamma / temp_real_kelvin) - 1) ) - ds = DataStore( + ds = Dataset( { "TMPR": (["x", "time"], temp_real_celsius), "st": (["x", "time"], st), @@ -1926,7 +1907,7 @@ def test_double_ended_fix_alpha_matching_sections_and_one_asym_att(): "warm": [slice(x[nx_per_sec], x[2 * nx_per_sec - 1])], } - ds.calibration_double_ended( + ds.dts.calibration_double_ended( sections=sections, st_var=1.5, ast_var=1.5, @@ -1953,7 +1934,7 @@ def test_double_ended_fix_alpha_matching_sections_and_one_asym_att(): alpha_adj = ds.alpha.values.copy() alpha_var_adj = ds.alpha_var.values.copy() - ds.calibration_double_ended( + ds.dts.calibration_double_ended( sections=sections, st_var=1.5, ast_var=1.5, @@ -1979,8 +1960,6 @@ def test_double_ended_fix_alpha_matching_sections_and_one_asym_att(): def test_double_ended_fix_alpha_gamma_matching_sections_and_one_asym_att(): - from dtscalibration import DataStore - cable_len = 100.0 nt = 3 time = np.arange(nt) @@ -2045,7 +2024,7 @@ def test_double_ended_fix_alpha_gamma_matching_sections_and_one_asym_att(): / (np.exp(gamma / temp_real_kelvin) - 1) ) - ds = DataStore( + ds = Dataset( { "TMPR": (["x", "time"], temp_real_celsius), "st": (["x", "time"], st), @@ -2066,7 +2045,7 @@ def test_double_ended_fix_alpha_gamma_matching_sections_and_one_asym_att(): "warm": [slice(x[nx_per_sec], x[2 * nx_per_sec - 1])], } - ds.calibration_double_ended( + ds.dts.calibration_double_ended( sections=sections, st_var=1.5, ast_var=1.5, @@ -2093,7 +2072,7 @@ def test_double_ended_fix_alpha_gamma_matching_sections_and_one_asym_att(): alpha_adj = ds.alpha.values.copy() alpha_var_adj = ds.alpha_var.values.copy() - ds.calibration_double_ended( + ds.dts.calibration_double_ended( sections=sections, st_var=1.5, ast_var=1.5, @@ -2120,8 +2099,6 @@ def test_double_ended_fix_alpha_gamma_matching_sections_and_one_asym_att(): def test_double_ended_fix_gamma_matching_sections_and_one_asym_att(): - from dtscalibration import DataStore - cable_len = 100.0 nt = 3 time = np.arange(nt) @@ -2186,7 +2163,7 @@ def test_double_ended_fix_gamma_matching_sections_and_one_asym_att(): / (np.exp(gamma / temp_real_kelvin) - 1) ) - ds = DataStore( + ds = Dataset( { "TMPR": (["x", "time"], temp_real_celsius), "st": (["x", "time"], st), @@ -2207,7 +2184,7 @@ def test_double_ended_fix_gamma_matching_sections_and_one_asym_att(): "warm": [slice(x[nx_per_sec], x[2 * nx_per_sec - 1])], } - ds.calibration_double_ended( + ds.dts.calibration_double_ended( sections=sections, st_var=1.5, ast_var=1.5, @@ -2238,8 +2215,6 @@ def test_double_ended_fix_gamma_matching_sections_and_one_asym_att(): def test_double_ended_exponential_variance_estimate_synthetic(): import dask.array as da - from dtscalibration import DataStore - state = da.random.RandomState(0) stokes_m_var = 4.0 @@ -2300,7 +2275,7 @@ def test_double_ended_exponential_variance_estimate_synthetic(): print("C", np.log(C_p / C_m)) print("x0", x.max()) - ds = DataStore( + ds = Dataset( { # 'st': (['x', 'time'], st), # 'ast': (['x', 'time'], ast), @@ -2330,7 +2305,7 @@ def test_double_ended_exponential_variance_estimate_synthetic(): rast_label = "rast" # MC variance - ds.calibration_double_ended( + ds.dts.calibration_double_ended( sections=sections, st_label=st_label, ast_label=ast_label, @@ -2405,8 +2380,6 @@ def test_double_ended_exponential_variance_estimate_synthetic(): def test_estimate_variance_of_temperature_estimate(): import dask.array as da - from dtscalibration import DataStore - state = da.random.RandomState(0) stokes_m_var = 0.1 @@ -2470,7 +2443,7 @@ def test_estimate_variance_of_temperature_estimate(): print("C", np.log(C_p / C_m)) print("x0", x.max()) - ds = DataStore( + ds = Dataset( { "st": (["x", "time"], st_m), "ast": (["x", "time"], ast_m), @@ -2490,7 +2463,7 @@ def test_estimate_variance_of_temperature_estimate(): "warm": [slice(0.5 * cable_len, 0.75 * cable_len)], } # MC variance - ds.calibration_double_ended( + ds.dts.calibration_double_ended( sections=sections, st_var=mst_var, ast_var=mast_var, @@ -2565,8 +2538,7 @@ def test_single_ended_wls_estimate_synthetic(): They should be the same as the parameters used to create the synthetic measurment set""" - from dtscalibration import DataStore - + # from dtscalibration import DataStore cable_len = 100.0 nt = 50 time = np.arange(nt) @@ -2605,7 +2577,7 @@ def test_single_ended_wls_estimate_synthetic(): print("C", np.log(C_p / C_m)) print("x0", x.max()) - ds = DataStore( + ds = Dataset( { "st": (["x", "time"], st), "ast": (["x", "time"], ast), @@ -2623,9 +2595,10 @@ def test_single_ended_wls_estimate_synthetic(): } # WLS - ds.calibration_single_ended( + out = ds.dts.calibration_single_ended( sections=sections, st_var=1.0, ast_var=1.0, method="wls", solver="sparse" ) + ds.update(out) assert_almost_equal_verbose(ds.gamma.values, gamma, decimal=6) assert_almost_equal_verbose(ds.dalpha.values, dalpha_p - dalpha_m, decimal=8) @@ -2641,8 +2614,6 @@ def test_single_ended_wls_fix_dalpha_synthetic(): They should be the same as the parameters used to create the synthetic measurment set""" - from dtscalibration import DataStore - cable_len = 100.0 nt = 50 time = np.arange(nt) @@ -2681,7 +2652,7 @@ def test_single_ended_wls_fix_dalpha_synthetic(): print("C", np.log(C_p / C_m)) print("x0", x.max()) - ds_ori = DataStore( + ds_ori = Dataset( { "st": (["x", "time"], st), "ast": (["x", "time"], ast), @@ -2700,7 +2671,7 @@ def test_single_ended_wls_fix_dalpha_synthetic(): # Test fix_dalpha ds_dalpha = ds_ori.copy() - ds_dalpha.calibration_single_ended( + out = ds_dalpha.dts.calibration_single_ended( sections=sections, st_var=1.0, ast_var=1.0, @@ -2708,19 +2679,18 @@ def test_single_ended_wls_fix_dalpha_synthetic(): solver="sparse", fix_dalpha=(dalpha_p - dalpha_m, 0.0), ) - - assert_almost_equal_verbose(ds_dalpha.gamma.values, gamma, decimal=12) + assert_almost_equal_verbose(out.gamma.values, gamma, decimal=12) assert_almost_equal_verbose( - ds_dalpha.dalpha.values, dalpha_p - dalpha_m, decimal=14 + out.dalpha.values, dalpha_p - dalpha_m, decimal=14 ) assert_almost_equal_verbose( - ds_dalpha.alpha.values, x * (dalpha_p - dalpha_m), decimal=14 + out.alpha.values, x * (dalpha_p - dalpha_m), decimal=14 ) - assert_almost_equal_verbose(ds_dalpha.tmpf.values, temp_real - 273.15, decimal=10) + assert_almost_equal_verbose(out.tmpf.values, temp_real - 273.15, decimal=10) # Test fix_alpha ds_alpha = ds_ori.copy() - ds_alpha.calibration_single_ended( + out = ds_alpha.dts.calibration_single_ended( sections=sections, st_var=1.0, ast_var=1.0, @@ -2729,11 +2699,11 @@ def test_single_ended_wls_fix_dalpha_synthetic(): fix_alpha=(x * (dalpha_p - dalpha_m), 0.0 * x), ) - assert_almost_equal_verbose(ds_alpha.gamma.values, gamma, decimal=12) + assert_almost_equal_verbose(out.gamma.values, gamma, decimal=12) assert_almost_equal_verbose( - ds_dalpha.alpha.values, x * (dalpha_p - dalpha_m), decimal=14 + out.alpha.values, x * (dalpha_p - dalpha_m), decimal=14 ) - assert_almost_equal_verbose(ds_alpha.tmpf.values, temp_real - 273.15, decimal=10) + assert_almost_equal_verbose(out.tmpf.values, temp_real - 273.15, decimal=10) pass @@ -2745,8 +2715,6 @@ def test_single_ended_wls_fix_gamma_synthetic(): They should be the same as the parameters used to create the synthetic measurment set""" - from dtscalibration import DataStore - cable_len = 100.0 nt = 50 time = np.arange(nt) @@ -2785,7 +2753,7 @@ def test_single_ended_wls_fix_gamma_synthetic(): print("C", np.log(C_p / C_m)) print("x0", x.max()) - ds = DataStore( + ds = Dataset( { "st": (["x", "time"], st), "ast": (["x", "time"], ast), @@ -2803,7 +2771,7 @@ def test_single_ended_wls_fix_gamma_synthetic(): } # WLS - ds.calibration_single_ended( + out = ds.dts.calibration_single_ended( sections=sections, st_var=1.0, ast_var=1.0, @@ -2812,9 +2780,9 @@ def test_single_ended_wls_fix_gamma_synthetic(): fix_gamma=(gamma, 0.0), ) - assert_almost_equal_verbose(ds.gamma.values, gamma, decimal=18) - assert_almost_equal_verbose(ds.dalpha.values, dalpha_p - dalpha_m, decimal=10) - assert_almost_equal_verbose(ds.tmpf.values, temp_real - 273.15, decimal=8) + assert_almost_equal_verbose(out.gamma.values, gamma, decimal=18) + assert_almost_equal_verbose(out.dalpha.values, dalpha_p - dalpha_m, decimal=10) + assert_almost_equal_verbose(out.tmpf.values, temp_real - 273.15, decimal=8) pass @@ -2826,8 +2794,6 @@ def test_single_ended_wls_fix_gamma_fix_dalpha_synthetic(): They should be the same as the parameters used to create the synthetic measurment set""" - from dtscalibration import DataStore - cable_len = 100.0 nt = 50 time = np.arange(nt) @@ -2866,7 +2832,7 @@ def test_single_ended_wls_fix_gamma_fix_dalpha_synthetic(): print("C", np.log(C_p / C_m)) print("x0", x.max()) - ds = DataStore( + ds = Dataset( { "st": (["x", "time"], st), "ast": (["x", "time"], ast), @@ -2884,7 +2850,7 @@ def test_single_ended_wls_fix_gamma_fix_dalpha_synthetic(): } # WLS - ds.calibration_single_ended( + out = ds.dts.calibration_single_ended( sections=sections, st_var=1.0, ast_var=1.0, @@ -2894,9 +2860,9 @@ def test_single_ended_wls_fix_gamma_fix_dalpha_synthetic(): fix_dalpha=(dalpha_p - dalpha_m, 0.0), ) - assert_almost_equal_verbose(ds.gamma.values, gamma, decimal=18) - assert_almost_equal_verbose(ds.dalpha.values, dalpha_p - dalpha_m, decimal=18) - assert_almost_equal_verbose(ds.tmpf.values, temp_real - 273.15, decimal=8) + assert_almost_equal_verbose(out.gamma.values, gamma, decimal=18) + assert_almost_equal_verbose(out.dalpha.values, dalpha_p - dalpha_m, decimal=18) + assert_almost_equal_verbose(out.tmpf.values, temp_real - 273.15, decimal=8) pass @@ -2904,7 +2870,6 @@ def test_single_ended_wls_fix_gamma_fix_dalpha_synthetic(): def test_single_ended_trans_att_synthetic(): """Checks whether the transient attenuation routines perform as intended, and calibrate to the correct temperature""" - from dtscalibration import DataStore cable_len = 100.0 nt = 50 @@ -2952,7 +2917,7 @@ def test_single_ended_trans_att_synthetic(): tr_att2 = np.random.rand(nt) * 0.2 + 0.8 st[int(x.size * 0.6) :] *= tr_att2 - ds = DataStore( + ds = Dataset( { "st": (["x", "time"], st), "ast": (["x", "time"], ast), @@ -2977,81 +2942,50 @@ def test_single_ended_trans_att_synthetic(): ds_test = ds.copy(deep=True) # WLS - ds_test.calibration_single_ended( + out = ds_test.dts.calibration_single_ended( sections=sections, st_var=1.0, ast_var=1.0, method="wls", - trans_att=[40, 60], + trans_att=[40., 60.], solver="sparse", ) - assert_almost_equal_verbose(ds_test.gamma.values, gamma, decimal=8) - assert_almost_equal_verbose(ds_test.tmpf.values, temp_real - 273.15, decimal=8) + assert_almost_equal_verbose(out.gamma.values, gamma, decimal=8) + assert_almost_equal_verbose(out.tmpf.values, temp_real - 273.15, decimal=8) assert_almost_equal_verbose( - ds_test.isel(trans_att=0).talpha_fw, -np.log(tr_att), decimal=8 + out.isel(trans_att=0).talpha_fw, -np.log(tr_att), decimal=8 ) assert_almost_equal_verbose( - ds_test.isel(trans_att=1).talpha_fw, -np.log(tr_att2), decimal=8 - ) - - # test `trans_att` related functions - # Clear out old results - ds_test.set_trans_att([]) - - assert ds_test.trans_att.size == 0, "clear out trans_att config" - - del_keys = [] - for k, v in ds_test.data_vars.items(): - if "trans_att" in v.dims: - del_keys.append(k) - - assert len(del_keys) == 0, "clear out trans_att config" - - ds_test.calibration_single_ended( - sections=sections, - st_var=1.0, - ast_var=1.0, - method="wls", - trans_att=[40, 60], - solver="sparse", - ) - - assert_almost_equal_verbose(ds_test.gamma.values, gamma, decimal=8) - assert_almost_equal_verbose(ds_test.tmpf.values, temp_real - 273.15, decimal=8) - assert_almost_equal_verbose( - ds_test.isel(trans_att=0).talpha_fw, -np.log(tr_att), decimal=8 - ) - assert_almost_equal_verbose( - ds_test.isel(trans_att=1).talpha_fw, -np.log(tr_att2), decimal=8 + out.isel(trans_att=1).talpha_fw, -np.log(tr_att2), decimal=8 ) ds_test = ds.copy(deep=True) # Test fixing gamma + transient att. - ds_test.calibration_single_ended( + out = ds_test.dts.calibration_single_ended( sections=sections, st_var=1.0, ast_var=1.0, method="wls", fix_gamma=(482.6, 0), - trans_att=[40, 60], + trans_att=[40., 60.], solver="sparse", ) - assert_almost_equal_verbose(ds_test.gamma.values, gamma, decimal=10) - assert_almost_equal_verbose(ds_test.tmpf.values, temp_real - 273.15, decimal=8) + assert_almost_equal_verbose(out.gamma.values, gamma, decimal=10) + assert_almost_equal_verbose(out.tmpf.values, temp_real - 273.15, decimal=8) assert_almost_equal_verbose( - ds_test.isel(trans_att=0).talpha_fw, -np.log(tr_att), decimal=8 + out.isel(trans_att=0).talpha_fw, -np.log(tr_att), decimal=8 ) assert_almost_equal_verbose( - ds_test.isel(trans_att=1).talpha_fw, -np.log(tr_att2), decimal=8 + out.isel(trans_att=1).talpha_fw, -np.log(tr_att2), decimal=8 ) ds_test = ds.copy(deep=True) # Test fixing alpha + transient att. - ds_test.calibration_single_ended( + out = ds_test.dts.calibration_single_ended( sections=sections, st_var=1.0, ast_var=1.0, @@ -3061,20 +2995,19 @@ def test_single_ended_trans_att_synthetic(): solver="sparse", ) - assert_almost_equal_verbose(ds_test.gamma.values, gamma, decimal=8) - assert_almost_equal_verbose(ds_test.tmpf.values, temp_real - 273.15, decimal=8) + assert_almost_equal_verbose(out.gamma.values, gamma, decimal=8) + assert_almost_equal_verbose(out.tmpf.values, temp_real - 273.15, decimal=8) assert_almost_equal_verbose( - ds_test.isel(trans_att=0).talpha_fw, -np.log(tr_att), decimal=8 + out.isel(trans_att=0).talpha_fw, -np.log(tr_att), decimal=8 ) assert_almost_equal_verbose( - ds_test.isel(trans_att=1).talpha_fw, -np.log(tr_att2), decimal=8 + out.isel(trans_att=1).talpha_fw, -np.log(tr_att2), decimal=8 ) def test_single_ended_matching_sections_synthetic(): """Checks whether the matching sections routines perform as intended, and calibrate to the correct temperature""" - from dtscalibration import DataStore cable_len = 100.0 nt = 50 @@ -3122,7 +3055,7 @@ def test_single_ended_matching_sections_synthetic(): tr_att2 = np.random.rand(nt) * 0.2 + 0.8 st[int(x.size * 0.6) :] *= tr_att2 - ds = DataStore( + ds = Dataset( { "st": (["x", "time"], st), "ast": (["x", "time"], ast), @@ -3156,7 +3089,7 @@ def test_single_ended_matching_sections_synthetic(): ds_test = ds.copy(deep=True) # WLS - ds_test.calibration_single_ended( + out = ds_test.dts.calibration_single_ended( sections=sections, st_var=1.0, ast_var=1.0, @@ -3166,19 +3099,19 @@ def test_single_ended_matching_sections_synthetic(): solver="sparse", ) - assert_almost_equal_verbose(ds_test.gamma.values, gamma, decimal=8) - assert_almost_equal_verbose(ds_test.tmpf.values, temp_real - 273.15, decimal=8) + assert_almost_equal_verbose(out.gamma.values, gamma, decimal=8) + assert_almost_equal_verbose(out.tmpf.values, temp_real - 273.15, decimal=8) assert_almost_equal_verbose( - ds_test.isel(trans_att=0).talpha_fw, -np.log(tr_att), decimal=8 + out.isel(trans_att=0).talpha_fw, -np.log(tr_att), decimal=8 ) assert_almost_equal_verbose( - ds_test.isel(trans_att=1).talpha_fw, -np.log(tr_att2), decimal=8 + out.isel(trans_att=1).talpha_fw, -np.log(tr_att2), decimal=8 ) ds_test = ds.copy(deep=True) # Test fixing gamma + transient att. - ds_test.calibration_single_ended( + out = ds_test.dts.calibration_single_ended( sections=sections, st_var=1.0, ast_var=1.0, @@ -3189,19 +3122,19 @@ def test_single_ended_matching_sections_synthetic(): solver="sparse", ) - assert_almost_equal_verbose(ds_test.gamma.values, gamma, decimal=10) - assert_almost_equal_verbose(ds_test.tmpf.values, temp_real - 273.15, decimal=8) + assert_almost_equal_verbose(out.gamma.values, gamma, decimal=10) + assert_almost_equal_verbose(out.tmpf.values, temp_real - 273.15, decimal=8) assert_almost_equal_verbose( - ds_test.isel(trans_att=0).talpha_fw, -np.log(tr_att), decimal=8 + out.isel(trans_att=0).talpha_fw, -np.log(tr_att), decimal=8 ) assert_almost_equal_verbose( - ds_test.isel(trans_att=1).talpha_fw, -np.log(tr_att2), decimal=8 + out.isel(trans_att=1).talpha_fw, -np.log(tr_att2), decimal=8 ) ds_test = ds.copy(deep=True) # Test fixing dalpha + transient att. - ds_test.calibration_single_ended( + out = ds_test.dts.calibration_single_ended( sections=sections, st_var=1.0, ast_var=1.0, @@ -3212,19 +3145,19 @@ def test_single_ended_matching_sections_synthetic(): solver="sparse", ) - assert_almost_equal_verbose(ds_test.gamma.values, gamma, decimal=10) - assert_almost_equal_verbose(ds_test.tmpf.values, temp_real - 273.15, decimal=8) + assert_almost_equal_verbose(out.gamma.values, gamma, decimal=10) + assert_almost_equal_verbose(out.tmpf.values, temp_real - 273.15, decimal=8) assert_almost_equal_verbose( - ds_test.isel(trans_att=0).talpha_fw, -np.log(tr_att), decimal=8 + out.isel(trans_att=0).talpha_fw, -np.log(tr_att), decimal=8 ) assert_almost_equal_verbose( - ds_test.isel(trans_att=1).talpha_fw, -np.log(tr_att2), decimal=8 + out.isel(trans_att=1).talpha_fw, -np.log(tr_att2), decimal=8 ) ds_test = ds.copy(deep=True) # Test fixing gamma & dalpha + transient att. - ds_test.calibration_single_ended( + out = ds_test.dts.calibration_single_ended( sections=sections, st_var=1.0, ast_var=1.0, @@ -3236,36 +3169,35 @@ def test_single_ended_matching_sections_synthetic(): solver="sparse", ) - assert_almost_equal_verbose(ds_test.gamma.values, gamma, decimal=10) - assert_almost_equal_verbose(ds_test.tmpf.values, temp_real - 273.15, decimal=8) + assert_almost_equal_verbose(out.gamma.values, gamma, decimal=10) + assert_almost_equal_verbose(out.tmpf.values, temp_real - 273.15, decimal=8) assert_almost_equal_verbose( - ds_test.isel(trans_att=0).talpha_fw, -np.log(tr_att), decimal=8 + out.isel(trans_att=0).talpha_fw, -np.log(tr_att), decimal=8 ) assert_almost_equal_verbose( - ds_test.isel(trans_att=1).talpha_fw, -np.log(tr_att2), decimal=8 + out.isel(trans_att=1).talpha_fw, -np.log(tr_att2), decimal=8 ) # Test conf. ints. for the combination of everything - ds_test.conf_int_single_ended( - p_val="p_val", - p_cov="p_cov", + out_conf = ds_test.dts.conf_int_single_ended( + result=out, st_var=1.0, ast_var=1.0, conf_ints=[2.5, 50.0, 97.5], mc_sample_size=50, ) - ds_test_1 = ds_test.isel(time=-1) + out_conf_1 = out_conf.isel(time=-1) # ds_test_1.tmpf # ds_test_1.tmpf_mc.isel(CI=0).values # ds_test_1.tmpf_mc.isel(CI=2).values assert np.all( - np.less(ds_test_1.tmpf_mc.isel(CI=0).values, ds_test_1.tmpf) + np.less(out_conf_1.tmpf_mc.isel(CI=0).values, out.isel(time=-1).tmpf) ), "Single-ended, trans. att.; 2.5% confidence interval is incorrect" assert np.all( - np.greater(ds_test_1.tmpf_mc.isel(CI=2).values, ds_test_1.tmpf) + np.greater(out_conf_1.tmpf_mc.isel(CI=2).values, out.isel(time=-1).tmpf) ), "Single-ended, trans. att.; 97.5% confidence interval is incorrect" @@ -3277,8 +3209,6 @@ def test_single_ended_exponential_variance_estimate_synthetic(): measurment set""" import dask.array as da - from dtscalibration import DataStore - state = da.random.RandomState(0) stokes_m_var = 40.0 @@ -3323,7 +3253,7 @@ def test_single_ended_exponential_variance_estimate_synthetic(): # print('C', np.log(C_p / C_m)) # print('x0', x.max()) - ds = DataStore( + ds = Dataset( { # 'st': (['x', 'time'], st), # 'ast': (['x', 'time'], ast), @@ -3342,33 +3272,36 @@ def test_single_ended_exponential_variance_estimate_synthetic(): "warm": [slice(0.5 * cable_len, cable_len)], } - st_label = "st" - ast_label = "ast" - - mst_var, _ = ds.variance_stokes_exponential(st_label=st_label, sections=sections) - mast_var, _ = ds.variance_stokes_exponential(st_label=ast_label, sections=sections) + mst_var, _ = variance_stokes_exponential( + ds["st"], + sections, + ds["userAcquisitionTimeFW"], + use_statsmodels=False, + suppress_info=False, + reshape_residuals=True, + ) + mast_var, _ = variance_stokes_exponential( + ds["ast"], + sections, + ds["userAcquisitionTimeFW"], + use_statsmodels=False, + suppress_info=False, + reshape_residuals=True, + ) # MC variqnce - ds.calibration_single_ended( + out = ds.dts.calibration_single_ended( sections=sections, st_var=mst_var, ast_var=mast_var, method="wls", solver="sparse", ) - - ds.conf_int_single_ended( - p_val="p_val", - p_cov="p_cov", - st_var=mst_var, - ast_var=mast_var, - conf_ints=[2.5, 50.0, 97.5], - mc_sample_size=50, - da_random_state=state, - ) + ds2 = ds.copy() + ds2.update(out) # Calibrated variance - stdsf1 = ds.ufunc_per_section( + stdsf1 = ds2.dts.ufunc_per_section( sections=sections, label="tmpf", func=np.var, @@ -3376,12 +3309,23 @@ def test_single_ended_exponential_variance_estimate_synthetic(): calc_per="stretch", ddof=1, ) + out_ci = ds2.dts.conf_int_single_ended( + result=out, + st_var=mst_var, + ast_var=mast_var, + conf_ints=[2.5, 50.0, 97.5], + mc_sample_size=50, + da_random_state=state, + mc_remove_set_flag=False + ) + ds2.update(out_ci) + + # Use a single timestep to better check if the parameter uncertainties # propagate - ds1 = ds.isel(time=1) # Estimated VAR - stdsf2 = ds1.ufunc_per_section( + stdsf2 = ds2.isel(time=1).dts.ufunc_per_section( sections=sections, label="tmpf_mc_var", func=np.mean, @@ -3399,7 +3343,7 @@ def test_single_ended_exponential_variance_estimate_synthetic(): print("hoi") -def test_calibrate_wls_procedures(): +def test_calibrate_wls_solver_procedures(): x = np.linspace(0, 10, 25 * 4) np.random.shuffle(x) @@ -3450,7 +3394,7 @@ def test_average_measurements_single_ended(): st_var, ast_var = 5.0, 5.0 - ds.calibration_single_ended( + ds.dts.calibration_single_ended( sections=sections, st_var=st_var, ast_var=ast_var, method="wls", solver="sparse" ) ds.average_single_ended( @@ -3516,7 +3460,7 @@ def test_average_measurements_double_ended(): st_var, ast_var, rst_var, rast_var = 5.0, 5.0, 5.0, 5.0 - ds.calibration_double_ended( + ds.dts.calibration_double_ended( sections=sections, st_var=st_var, ast_var=ast_var, From 89a5c8dc2709a182e495048bdda983471119959e Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Mon, 9 Oct 2023 23:27:04 +0300 Subject: [PATCH 24/66] renamed conf_ints_single_ended to monte_carlo_single_ended and calibration_single_ended to calibrate_ssingle_ended --- .../calibration/section_utils.py | 43 +-- src/dtscalibration/datastore_accessor.py | 262 +----------------- tests/test_dtscalibration.py | 44 +-- 3 files changed, 29 insertions(+), 320 deletions(-) diff --git a/src/dtscalibration/calibration/section_utils.py b/src/dtscalibration/calibration/section_utils.py index 3e13709a..4333c8a1 100644 --- a/src/dtscalibration/calibration/section_utils.py +++ b/src/dtscalibration/calibration/section_utils.py @@ -5,14 +5,12 @@ from dtscalibration.datastore_utils import ufunc_per_section_helper -def set_sections(ds: xr.Dataset, sections: dict[str, list[slice]]) -> xr.Dataset: - sections_validated = None +def set_sections(ds: xr.Dataset, sections: dict[str, list[slice]] = None): + ds.attrs["_sections"] = yaml.dump(sections) - if sections is not None: - sections_validated = validate_sections(ds, sections=sections) - ds.attrs["_sections"] = yaml.dump(sections_validated) - return ds +def set_matching_sections(ds: xr.Dataset, matching_sections: dict[str, list[slice]] = None): + ds.attrs["_matching_sections"] = yaml.dump(matching_sections) def validate_no_overlapping_sections(sections: dict[str, list[slice]]): @@ -132,39 +130,6 @@ def validate_sections(ds: xr.Dataset, sections: dict[str, list[slice]]): f"Better define the {k} section. You tried {vi}, " "which is not within the x-dimension" ) - - # for k, v in sections.items(): - # assert isinstance(v, (list, tuple)), ( - # "The values of the sections-dictionary " "should be lists of slice objects." - # ) - - # for vi in v: - # assert isinstance(vi, slice), ( - # "The values of the sections-dictionary should " - # "be lists of slice objects." - # ) - - # assert ds.x.sel(x=vi).size > 0, ( - # f"Better define the {k} section. You tried {vi}, " - # "which is not within the x-dimension" - # ) - - # # sorted stretches - # stretch_unsort = [slice(float(vi.start), float(vi.stop)) for vi in v] - # stretch_start = [i.start for i in stretch_unsort] - # stretch_i_sorted = np.argsort(stretch_start) - # sections_fix_slice_fixed[k] = [stretch_unsort[i] for i in stretch_i_sorted] - # all_stretches.extend(sections_fix_slice_fixed[k]) - - # # Check for overlapping slices - # all_start_stop = [[stretch.start, stretch.stop] for stretch in all_stretches] - # isorted_start = np.argsort([i[0] for i in all_start_stop]) - # all_start_stop_startsort = [all_start_stop[i] for i in isorted_start] - # all_start_stop_startsort_flat = sum(all_start_stop_startsort, []) - # assert all_start_stop_startsort_flat == sorted( - # all_start_stop_startsort_flat), \ - # "Sections contains overlapping stretches" - pass diff --git a/src/dtscalibration/datastore_accessor.py b/src/dtscalibration/datastore_accessor.py index 29c39619..b53cc203 100644 --- a/src/dtscalibration/datastore_accessor.py +++ b/src/dtscalibration/datastore_accessor.py @@ -422,7 +422,7 @@ def ufunc_per_section( ) return out - def calibration_single_ended( + def calibrate_single_ended( self, sections, st_var, @@ -722,7 +722,7 @@ def calibration_single_ended( return out - def conf_int_single_ended( + def monte_carlo_single_ended( self, result, st_var, @@ -915,263 +915,7 @@ def conf_int_single_ended( return out - def conf_int_single_ended_old( - self, - p_val="p_val", - p_cov="p_cov", - st_var=None, - ast_var=None, - conf_ints=None, - mc_sample_size=100, - da_random_state=None, - mc_remove_set_flag=True, - reduce_memory_usage=False - ): - r""" - Estimation of the confidence intervals for the temperatures measured - with a single-ended setup. It consists of five steps. First, the variances - of the Stokes and anti-Stokes intensity measurements are estimated - following the steps in Section 4 [1]_. A Normal - distribution is assigned to each intensity measurement that is centered - at the measurement and using the estimated variance. Second, a multi- - variate Normal distribution is assigned to the estimated parameters - using the covariance matrix from the calibration procedure presented in - Section 5 [1]_. Third, the distributions are sampled, and the - temperature is computed with Equation 12 [1]_. Fourth, step - three is repeated, e.g., 10,000 times for each location and for each - time. The resulting 10,000 realizations of the temperatures - approximate the probability density functions of the estimated - temperature at that location and time. Fifth, the standard uncertainties - are computed with the standard deviations of the realizations of the - temperatures, and the 95\% confidence intervals are computed from the - 2.5\% and 97.5\% percentiles of the realizations of the temperatures. - - - Parameters - ---------- - p_val : array-like, optional - Define `p_val`, `p_var`, `p_cov` if you used an external function - for calibration. Has size 2 + `nt`. First value is :math:`\gamma`, - second is :math:`\Delta \\alpha`, others are :math:`C` for each - timestep. - If set to False, no uncertainty in the parameters is propagated - into the confidence intervals. Similar to the spec sheets of the DTS - manufacturers. And similar to passing an array filled with zeros - p_cov : array-like, optional - The covariances of `p_val`. - st_var, ast_var : float, callable, array-like, optional - The variance of the measurement noise of the Stokes signals in the - forward direction. If `float` the variance of the noise from the - Stokes detector is described with a single value. - If `callable` the variance of the noise from the Stokes detector is - a function of the intensity, as defined in the callable function. - Or manually define a variance with a DataArray of the shape - `ds.st.shape`, where the variance can be a function of time and/or - x. Required if method is wls. - conf_ints : iterable object of float - A list with the confidence boundaries that are calculated. Valid - values are between - [0, 1]. - mc_sample_size : int - Size of the monte carlo parameter set used to calculate the - confidence interval - da_random_state - For testing purposes. Similar to random seed. The seed for dask. - Makes random not so random. To produce reproducable results for - testing environments. - mc_remove_set_flag : bool - Remove the monte carlo data set, from which the CI and the - variance are calculated. - reduce_memory_usage : bool - Use less memory but at the expense of longer computation time - - - References - ---------- - .. [1] des Tombe, B., Schilperoort, B., & Bakker, M. (2020). Estimation - of Temperature and Associated Uncertainty from Fiber-Optic Raman- - Spectrum Distributed Temperature Sensing. Sensors, 20(8), 2235. - https://doi.org/10.3390/s20082235 - """ - out = xr.Dataset() - params = xr.Dataset() - - if da_random_state: - state = da_random_state - else: - state = da.random.RandomState() - - no, nt = self.st.data.shape - if "trans_att" in self.keys(): - nta = self.trans_att.size - - else: - nta = 0 - - assert isinstance(p_val, (str, np.ndarray, np.generic)) - if isinstance(p_val, str): - p_val = self[p_val].data - - npar = p_val.size - - # number of parameters - if npar == nt + 2 + nt * nta: - fixed_alpha = False - elif npar == 1 + no + nt + nt * nta: - fixed_alpha = True - else: - raise Exception("The size of `p_val` is not what I expected") - - params.coords["mc"] = range(mc_sample_size) - params.coords["x"] = self.x - params.coords["time"] = self.time - - if conf_ints: - out.coords["CI"] = conf_ints - params.coords["CI"] = conf_ints - - # WLS - if isinstance(p_cov, str): - p_cov = self[p_cov].data - assert p_cov.shape == (npar, npar) - - p_mc = sst.multivariate_normal.rvs(mean=p_val, cov=p_cov, size=mc_sample_size) - - if fixed_alpha: - params["alpha_mc"] = (("mc", "x"), p_mc[:, 1 : no + 1]) - params["c_mc"] = (("mc", "time"), p_mc[:, 1 + no : 1 + no + nt]) - else: - params["dalpha_mc"] = (("mc",), p_mc[:, 1]) - params["c_mc"] = (("mc", "time"), p_mc[:, 2 : nt + 2]) - - params["gamma_mc"] = (("mc",), p_mc[:, 0]) - if nta: - params["ta_mc"] = ( - ("mc", "trans_att", "time"), - np.reshape(p_mc[:, -nt * nta :], (mc_sample_size, nta, nt)), - ) - - rsize = (params.mc.size, params.x.size, params.time.size) - - if reduce_memory_usage: - memchunk = da.ones( - (mc_sample_size, no, nt), chunks={0: -1, 1: 1, 2: "auto"} - ).chunks - else: - memchunk = da.ones( - (mc_sample_size, no, nt), chunks={0: -1, 1: "auto", 2: "auto"} - ).chunks - - # Draw from the normal distributions for the Stokes intensities - for k, st_labeli, st_vari in zip( - ["r_st", "r_ast"], ["st", "ast"], [st_var, ast_var] - ): - # Load the mean as chunked Dask array, otherwise eats memory - if type(self[st_labeli].data) == da.core.Array: - loc = da.asarray(self[st_labeli].data, chunks=memchunk[1:]) - else: - loc = da.from_array(self[st_labeli].data, chunks=memchunk[1:]) - - # Make sure variance is of size (no, nt) - if np.size(st_vari) > 1: - if st_vari.shape == self[st_labeli].shape: - pass - else: - st_vari = np.broadcast_to(st_vari, (no, nt)) - else: - pass - - # Load variance as chunked Dask array, otherwise eats memory - if type(st_vari) == da.core.Array: - st_vari_da = da.asarray(st_vari, chunks=memchunk[1:]) - - elif callable(st_vari) and type(self[st_labeli].data) == da.core.Array: - st_vari_da = da.asarray( - st_vari(self[st_labeli]).data, chunks=memchunk[1:] - ) - - elif callable(st_vari) and type(self[st_labeli].data) != da.core.Array: - st_vari_da = da.from_array( - st_vari(self[st_labeli]).data, chunks=memchunk[1:] - ) - - else: - st_vari_da = da.from_array(st_vari, chunks=memchunk[1:]) - - params[k] = ( - ("mc", "x", "time"), - state.normal( - loc=loc, # has chunks=memchunk[1:] - scale=st_vari_da**0.5, - size=rsize, - chunks=memchunk, - ), - ) - - ta_arr = np.zeros((mc_sample_size, no, nt)) - - if nta: - for ii, ta in enumerate(params["ta_mc"]): - for tai, taxi in zip(ta.values, self.trans_att.values): - ta_arr[ii, self.x.values >= taxi] = ( - ta_arr[ii, self.x.values >= taxi] + tai - ) - params["ta_mc_arr"] = (("mc", "x", "time"), ta_arr) - - if fixed_alpha: - params["tmpf_mc_set"] = ( - params["gamma_mc"] - / ( - ( - np.log(params["r_st"]) - - np.log(params["r_ast"]) - + (params["c_mc"] + params["ta_mc_arr"]) - ) - + params["alpha_mc"] - ) - - 273.15 - ) - else: - params["tmpf_mc_set"] = ( - params["gamma_mc"] - / ( - ( - np.log(params["r_st"]) - - np.log(params["r_ast"]) - + (params["c_mc"] + params["ta_mc_arr"]) - ) - + (params["dalpha_mc"] * params.x) - ) - - 273.15 - ) - - avg_dims = ["mc"] - avg_axis = params["tmpf_mc_set"].get_axis_num(avg_dims) - out["tmpf_mc_var"] = (params["tmpf_mc_set"] - self["tmpf"]).var( - dim=avg_dims, ddof=1 - ) - - if conf_ints: - new_chunks = ((len(conf_ints),),) + params["tmpf_mc_set"].chunks[1:] - - qq = params["tmpf_mc_set"] - - q = qq.data.map_blocks( - lambda x: np.percentile(x, q=conf_ints, axis=avg_axis), - chunks=new_chunks, # - drop_axis=avg_axis, # avg dimesnions are dropped from input arr - new_axis=0, - ) # The new CI dimension is added as first axis - - out["tmpf_mc"] = (("CI", "x", "time"), q) - - if not mc_remove_set_flag: - out.update(params) - - self.update(out) - return out - - def conf_int_double_ended( + def monte_carlo_double_ended( self, sections=None, p_val="p_val", diff --git a/tests/test_dtscalibration.py b/tests/test_dtscalibration.py index b15cee52..f9642517 100644 --- a/tests/test_dtscalibration.py +++ b/tests/test_dtscalibration.py @@ -125,7 +125,7 @@ def test_variance_input_types_single(): # Test float input st_var = 5.0 - ds.dts.calibration_single_ended( + ds.dts.calibrate_single_ended( sections=sections, st_var=st_var, ast_var=st_var, method="wls", solver="sparse" ) @@ -150,7 +150,7 @@ def callable_st_var(stokes): offset = 0 return slope * stokes + offset - ds.dts.calibration_single_ended( + ds.dts.calibrate_single_ended( sections=sections, st_var=callable_st_var, ast_var=callable_st_var, @@ -174,7 +174,7 @@ def callable_st_var(stokes): # Test input with shape of (ntime, nx) st_var = ds.st.values * 0 + 20.0 - ds.dts.calibration_single_ended( + ds.dts.calibrate_single_ended( sections=sections, st_var=st_var, ast_var=st_var, method="wls", solver="sparse" ) @@ -189,7 +189,7 @@ def callable_st_var(stokes): ds.st.mean(dim="time").values * 0 + np.linspace(10, 50, num=ds.st.x.size) ) - ds.dts.calibration_single_ended( + ds.dts.calibrate_single_ended( sections=sections, st_var=st_var, ast_var=st_var, method="wls", solver="sparse" ) @@ -207,7 +207,7 @@ def callable_st_var(stokes): # Test input with shape (ntime) st_var = ds.st.mean(dim="x").values * 0 + np.linspace(5, 200, num=nt) - ds.dts.calibration_single_ended( + ds.dts.calibrate_single_ended( sections=sections, st_var=st_var, ast_var=st_var, method="wls", solver="sparse" ) @@ -693,7 +693,7 @@ def test_single_ended_variance_estimate_synthetic(): mast_var = float(mast_var) # MC variqnce - ds.dts.calibration_single_ended( + ds.dts.calibrate_single_ended( sections=sections, st_var=mst_var, ast_var=mast_var, @@ -2595,7 +2595,7 @@ def test_single_ended_wls_estimate_synthetic(): } # WLS - out = ds.dts.calibration_single_ended( + out = ds.dts.calibrate_single_ended( sections=sections, st_var=1.0, ast_var=1.0, method="wls", solver="sparse" ) ds.update(out) @@ -2671,7 +2671,7 @@ def test_single_ended_wls_fix_dalpha_synthetic(): # Test fix_dalpha ds_dalpha = ds_ori.copy() - out = ds_dalpha.dts.calibration_single_ended( + out = ds_dalpha.dts.calibrate_single_ended( sections=sections, st_var=1.0, ast_var=1.0, @@ -2690,7 +2690,7 @@ def test_single_ended_wls_fix_dalpha_synthetic(): # Test fix_alpha ds_alpha = ds_ori.copy() - out = ds_alpha.dts.calibration_single_ended( + out = ds_alpha.dts.calibrate_single_ended( sections=sections, st_var=1.0, ast_var=1.0, @@ -2771,7 +2771,7 @@ def test_single_ended_wls_fix_gamma_synthetic(): } # WLS - out = ds.dts.calibration_single_ended( + out = ds.dts.calibrate_single_ended( sections=sections, st_var=1.0, ast_var=1.0, @@ -2850,7 +2850,7 @@ def test_single_ended_wls_fix_gamma_fix_dalpha_synthetic(): } # WLS - out = ds.dts.calibration_single_ended( + out = ds.dts.calibrate_single_ended( sections=sections, st_var=1.0, ast_var=1.0, @@ -2942,7 +2942,7 @@ def test_single_ended_trans_att_synthetic(): ds_test = ds.copy(deep=True) # WLS - out = ds_test.dts.calibration_single_ended( + out = ds_test.dts.calibrate_single_ended( sections=sections, st_var=1.0, ast_var=1.0, @@ -2963,7 +2963,7 @@ def test_single_ended_trans_att_synthetic(): ds_test = ds.copy(deep=True) # Test fixing gamma + transient att. - out = ds_test.dts.calibration_single_ended( + out = ds_test.dts.calibrate_single_ended( sections=sections, st_var=1.0, ast_var=1.0, @@ -2985,7 +2985,7 @@ def test_single_ended_trans_att_synthetic(): ds_test = ds.copy(deep=True) # Test fixing alpha + transient att. - out = ds_test.dts.calibration_single_ended( + out = ds_test.dts.calibrate_single_ended( sections=sections, st_var=1.0, ast_var=1.0, @@ -3089,7 +3089,7 @@ def test_single_ended_matching_sections_synthetic(): ds_test = ds.copy(deep=True) # WLS - out = ds_test.dts.calibration_single_ended( + out = ds_test.dts.calibrate_single_ended( sections=sections, st_var=1.0, ast_var=1.0, @@ -3111,7 +3111,7 @@ def test_single_ended_matching_sections_synthetic(): ds_test = ds.copy(deep=True) # Test fixing gamma + transient att. - out = ds_test.dts.calibration_single_ended( + out = ds_test.dts.calibrate_single_ended( sections=sections, st_var=1.0, ast_var=1.0, @@ -3134,7 +3134,7 @@ def test_single_ended_matching_sections_synthetic(): ds_test = ds.copy(deep=True) # Test fixing dalpha + transient att. - out = ds_test.dts.calibration_single_ended( + out = ds_test.dts.calibrate_single_ended( sections=sections, st_var=1.0, ast_var=1.0, @@ -3157,7 +3157,7 @@ def test_single_ended_matching_sections_synthetic(): ds_test = ds.copy(deep=True) # Test fixing gamma & dalpha + transient att. - out = ds_test.dts.calibration_single_ended( + out = ds_test.dts.calibrate_single_ended( sections=sections, st_var=1.0, ast_var=1.0, @@ -3179,7 +3179,7 @@ def test_single_ended_matching_sections_synthetic(): ) # Test conf. ints. for the combination of everything - out_conf = ds_test.dts.conf_int_single_ended( + out_conf = ds_test.dts.monte_carlo_single_ended( result=out, st_var=1.0, ast_var=1.0, @@ -3290,7 +3290,7 @@ def test_single_ended_exponential_variance_estimate_synthetic(): ) # MC variqnce - out = ds.dts.calibration_single_ended( + out = ds.dts.calibrate_single_ended( sections=sections, st_var=mst_var, ast_var=mast_var, @@ -3309,7 +3309,7 @@ def test_single_ended_exponential_variance_estimate_synthetic(): calc_per="stretch", ddof=1, ) - out_ci = ds2.dts.conf_int_single_ended( + out_ci = ds2.dts.monte_carlo_single_ended( result=out, st_var=mst_var, ast_var=mast_var, @@ -3394,7 +3394,7 @@ def test_average_measurements_single_ended(): st_var, ast_var = 5.0, 5.0 - ds.dts.calibration_single_ended( + ds.dts.calibrate_single_ended( sections=sections, st_var=st_var, ast_var=ast_var, method="wls", solver="sparse" ) ds.average_single_ended( From ec2a49a782a57a9ba052da126e9036f5cdc41fa4 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Mon, 9 Oct 2023 23:37:49 +0300 Subject: [PATCH 25/66] Removed DataStore from being used --- src/dtscalibration/datastore.py | 6 ------ src/dtscalibration/datastore_utils.py | 5 +---- src/dtscalibration/io.py | 1 - tests/test_datastore.py | 1 - tests/test_variance_stokes.py | 4 ++-- 5 files changed, 3 insertions(+), 14 deletions(-) diff --git a/src/dtscalibration/datastore.py b/src/dtscalibration/datastore.py index 54973c0e..582a773d 100644 --- a/src/dtscalibration/datastore.py +++ b/src/dtscalibration/datastore.py @@ -4123,9 +4123,3 @@ def ufunc_per_section( **func_kwargs, ) return out - - def resample_datastore(*args, **kwargs): - raise ( - "ds.resample_datastore() is deprecated. Use from dtscalibration import DataStore; " - "DataStore(ds.resample()) instead. See example notebook 2." - ) diff --git a/src/dtscalibration/datastore_utils.py b/src/dtscalibration/datastore_utils.py index a7de1d22..41709c32 100644 --- a/src/dtscalibration/datastore_utils.py +++ b/src/dtscalibration/datastore_utils.py @@ -8,12 +8,9 @@ import numpy.typing as npt import xarray as xr -if TYPE_CHECKING: - from dtscalibration import DataStore - def check_dims( - ds: "DataStore", + ds: xr.Dataset, labels: Union[list[str], tuple[str]], correct_dims: Optional[tuple[str]] = None, ) -> None: diff --git a/src/dtscalibration/io.py b/src/dtscalibration/io.py index 413851f5..0ee3483a 100644 --- a/src/dtscalibration/io.py +++ b/src/dtscalibration/io.py @@ -7,7 +7,6 @@ import numpy as np import xarray as xr -from dtscalibration import DataStore from dtscalibration.io_utils import apsensing_xml_version_check from dtscalibration.io_utils import read_apsensing_files_routine from dtscalibration.io_utils import read_sensornet_files_routine_v3 diff --git a/tests/test_datastore.py b/tests/test_datastore.py index 15a6cc38..5620f54b 100644 --- a/tests/test_datastore.py +++ b/tests/test_datastore.py @@ -9,7 +9,6 @@ import numpy as np import pytest -from dtscalibration import DataStore from dtscalibration import open_datastore from dtscalibration import open_mf_datastore from dtscalibration import read_apsensing_files diff --git a/tests/test_variance_stokes.py b/tests/test_variance_stokes.py index c81b2096..6accabe3 100644 --- a/tests/test_variance_stokes.py +++ b/tests/test_variance_stokes.py @@ -3,8 +3,8 @@ import numpy as np import pytest from scipy import stats +import xarray as xr -from dtscalibration import DataStore from dtscalibration import read_silixa_files from dtscalibration.variance_stokes import variance_stokes_exponential from dtscalibration.variance_stokes import variance_stokes_constant @@ -96,7 +96,7 @@ def test_variance_of_stokes_synthetic(): y += stats.norm.rvs(size=y.size, scale=yvar**0.5).reshape(y.shape) - ds = DataStore( + ds = xr.Dataset( { "st": (["x", "time"], y), "probe1Temperature": (["time"], range(nt)), From 5e123b7a8e12c042b754db2bad204914bb4fbefb Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Mon, 9 Oct 2023 23:38:09 +0300 Subject: [PATCH 26/66] removed renaming_labels test --- tests/test_dtscalibration.py | 119 ----------------------------------- 1 file changed, 119 deletions(-) diff --git a/tests/test_dtscalibration.py b/tests/test_dtscalibration.py index f9642517..b81833f8 100644 --- a/tests/test_dtscalibration.py +++ b/tests/test_dtscalibration.py @@ -949,124 +949,6 @@ def test_double_ended_wls_estimate_synthetic_df_and_db_are_different(): pass -def test_reneaming_old_default_labels_to_new_fixed_labels(): - """Same as - `test_double_ended_wls_estimate_synthetic_df_and_db_are_different` - Which runs fast, but using the renaming function.""" - - cable_len = 100.0 - nt = 3 - time = np.arange(nt) - x = np.linspace(0.0, cable_len, 8) - ts_cold = np.ones(nt) * 4.0 + np.cos(time) * 4 - ts_warm = np.ones(nt) * 20.0 + -np.sin(time) * 4 - - C_p = 1324 # 1/2 * E0 * v * K_+/lam_+^4 - eta_pf = np.cos(time) / 10 + 1 # eta_+ (gain factor forward channel) - eta_pb = np.sin(time) / 10 + 1 # eta_- (gain factor backward channel) - C_m = 5000.0 - eta_mf = np.cos(time + np.pi / 8) / 10 + 1 - eta_mb = np.sin(time + np.pi / 8) / 10 + 1 - dalpha_r = 0.005284 - dalpha_m = 0.004961 - dalpha_p = 0.005607 - gamma = 482.6 - - temp_real_kelvin = np.zeros((len(x), nt)) + 273.15 - temp_real_kelvin[x < 0.2 * cable_len] += ts_cold[None] - temp_real_kelvin[x > 0.85 * cable_len] += ts_warm[None] - temp_real_celsius = temp_real_kelvin - 273.15 - - st = ( - eta_pf[None] - * C_p - * np.exp(-dalpha_r * x[:, None]) - * np.exp(-dalpha_p * x[:, None]) - * np.exp(gamma / temp_real_kelvin) - / (np.exp(gamma / temp_real_kelvin) - 1) - ) - ast = ( - eta_mf[None] - * C_m - * np.exp(-dalpha_r * x[:, None]) - * np.exp(-dalpha_m * x[:, None]) - / (np.exp(gamma / temp_real_kelvin) - 1) - ) - rst = ( - eta_pb[None] - * C_p - * np.exp(-dalpha_r * (-x[:, None] + cable_len)) - * np.exp(-dalpha_p * (-x[:, None] + cable_len)) - * np.exp(gamma / temp_real_kelvin) - / (np.exp(gamma / temp_real_kelvin) - 1) - ) - rast = ( - eta_mb[None] - * C_m - * np.exp(-dalpha_r * (-x[:, None] + cable_len)) - * np.exp(-dalpha_m * (-x[:, None] + cable_len)) - / (np.exp(gamma / temp_real_kelvin) - 1) - ) - - c_f = np.log(eta_mf * C_m / (eta_pf * C_p)) - c_b = np.log(eta_mb * C_m / (eta_pb * C_p)) - - dalpha = dalpha_p - dalpha_m # \Delta\alpha - alpha_int = cable_len * dalpha - - df = c_f # reference section starts at first x-index - db = c_b + alpha_int - i_fw = np.log(st / ast) - i_bw = np.log(rst / rast) - - E_real = (i_bw - i_fw) / 2 + (db - df) / 2 - - ds = Dataset( - { - "ST": (["x", "time"], st), - "AST": (["x", "time"], ast), - "REV-ST": (["x", "time"], rst), - "REV-AST": (["x", "time"], rast), - "userAcquisitionTimeFW": (["time"], np.ones(nt)), - "userAcquisitionTimeBW": (["time"], np.ones(nt)), - "cold": (["time"], ts_cold), - "warm": (["time"], ts_warm), - }, - coords={"x": x, "time": time}, - attrs={"isDoubleEnded": "1"}, - ) - ds = ds.rename_labels() - - sections = { - "cold": [slice(0.0, 0.09 * cable_len)], - "warm": [slice(0.9 * cable_len, cable_len)], - } - - real_ans2 = np.concatenate(([gamma], df, db, E_real[:, 0])) - - ds.dts.calibration_double_ended( - sections=sections, - st_var=1.5, - ast_var=1.5, - rst_var=1.0, - rast_var=1.0, - method="wls", - solver="sparse", - fix_gamma=(gamma, 0.0), - ) - - assert_almost_equal_verbose(df, ds.df.values, decimal=14) - assert_almost_equal_verbose(db, ds.db.values, decimal=13) - assert_almost_equal_verbose( - x * (dalpha_p - dalpha_m), ds.alpha.values - ds.alpha.values[0], decimal=13 - ) - assert np.all(np.abs(real_ans2 - ds.p_val.values) < 1e-10) - assert_almost_equal_verbose(temp_real_celsius, ds.tmpf.values, decimal=10) - assert_almost_equal_verbose(temp_real_celsius, ds.tmpb.values, decimal=10) - assert_almost_equal_verbose(temp_real_celsius, ds.tmpw.values, decimal=10) - pass - - @pytest.mark.xfail def test_fail_if_st_labels_are_passed_to_calibration_function(): """Same as @@ -2538,7 +2420,6 @@ def test_single_ended_wls_estimate_synthetic(): They should be the same as the parameters used to create the synthetic measurment set""" - # from dtscalibration import DataStore cable_len = 100.0 nt = 50 time = np.arange(nt) From c18dd68cde2323dcd7d9156676ff49545627b417 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Fri, 13 Oct 2023 09:43:13 +0400 Subject: [PATCH 27/66] conf_int_single renamed to monte_carlo_single --- src/dtscalibration/datastore_accessor.py | 5 +++-- tests/test_dtscalibration.py | 15 ++++++++------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/dtscalibration/datastore_accessor.py b/src/dtscalibration/datastore_accessor.py index b53cc203..b541aad8 100644 --- a/src/dtscalibration/datastore_accessor.py +++ b/src/dtscalibration/datastore_accessor.py @@ -29,10 +29,11 @@ def __init__(self, xarray_obj): self.time = xarray_obj.time self.nt = self.time.size + # None if doesn't exist self.st = xarray_obj.get("st") self.ast = xarray_obj.get("ast") - self.rst = xarray_obj.get("rst") # None if doesn't exist - self.rast = xarray_obj.get("rast") # None is doesn't exist + self.rst = xarray_obj.get("rst") + self.rast = xarray_obj.get("rast") self.acquisitiontime_fw = xarray_obj.get("userAcquisitionTimeFW") self.acquisitiontime_bw = xarray_obj.get("userAcquisitionTimeBW") diff --git a/tests/test_dtscalibration.py b/tests/test_dtscalibration.py index b81833f8..1ff7a32a 100644 --- a/tests/test_dtscalibration.py +++ b/tests/test_dtscalibration.py @@ -10,7 +10,8 @@ from dtscalibration.calibrate_utils import wls_stats from dtscalibration.datastore_accessor import DtsAccessor # noqa: F401 from dtscalibration.variance_stokes import variance_stokes_exponential - +from dtscalibration.variance_stokes import variance_stokes_constant +from dtscalibration.variance_stokes import variance_stokes_linear np.random.seed(0) @@ -129,7 +130,7 @@ def test_variance_input_types_single(): sections=sections, st_var=st_var, ast_var=st_var, method="wls", solver="sparse" ) - ds.dts.conf_int_single_ended( + ds.dts.monte_carlo_single_ended( st_var=st_var, ast_var=st_var, mc_sample_size=100, @@ -158,7 +159,7 @@ def callable_st_var(stokes): solver="sparse", ) - ds.conf_int_single_ended( + ds.monte_carlo_single_ended( st_var=callable_st_var, ast_var=callable_st_var, mc_sample_size=100, @@ -178,7 +179,7 @@ def callable_st_var(stokes): sections=sections, st_var=st_var, ast_var=st_var, method="wls", solver="sparse" ) - ds.dts.conf_int_single_ended( + ds.dts.monte_carlo_single_ended( st_var=st_var, ast_var=st_var, mc_sample_size=100, da_random_state=state ) @@ -193,7 +194,7 @@ def callable_st_var(stokes): sections=sections, st_var=st_var, ast_var=st_var, method="wls", solver="sparse" ) - ds.dts.conf_int_single_ended( + ds.dts.monte_carlo_single_ended( st_var=st_var, ast_var=st_var, mc_sample_size=100, da_random_state=state ) @@ -211,7 +212,7 @@ def callable_st_var(stokes): sections=sections, st_var=st_var, ast_var=st_var, method="wls", solver="sparse" ) - ds.dts.conf_int_single_ended( + ds.dts.monte_carlo_single_ended( st_var=st_var, ast_var=st_var, mc_sample_size=100, da_random_state=state ) @@ -701,7 +702,7 @@ def test_single_ended_variance_estimate_synthetic(): solver="sparse", ) - ds.dts.conf_int_single_ended( + ds.dts.monte_carlo_single_ended( p_val="p_val", p_cov="p_cov", st_var=mst_var, From 9286a01b8122f6a8f6ee95f72e3b60fac28b965b Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Fri, 13 Oct 2023 12:46:16 +0400 Subject: [PATCH 28/66] Initial commit monte_carlo_double_ended --- src/dtscalibration/datastore_accessor.py | 226 ++++++++++------------- 1 file changed, 98 insertions(+), 128 deletions(-) diff --git a/src/dtscalibration/datastore_accessor.py b/src/dtscalibration/datastore_accessor.py index b541aad8..b0dcb116 100644 --- a/src/dtscalibration/datastore_accessor.py +++ b/src/dtscalibration/datastore_accessor.py @@ -757,7 +757,7 @@ def monte_carlo_single_ended( params = out.copy() params.coords["mc"] = range(mc_sample_size) - no, nt = self.st.data.shape + no, nt = self.st.shape nta = result["trans_att"].size p_val = result["p_val"].data @@ -772,12 +772,7 @@ def monte_carlo_single_ended( fixed_alpha = True else: raise Exception("The size of `p_val` is not what I expected") - - assert isinstance(p_val, (str, np.ndarray, np.generic)) - if isinstance(p_val, str): - p_val = self[p_val].data - npar = p_val.size p_mc = sst.multivariate_normal.rvs(mean=p_val, cov=p_cov, size=mc_sample_size) if fixed_alpha: @@ -809,9 +804,6 @@ def monte_carlo_single_ended( # Draw from the normal distributions for the Stokes intensities for key_mc, sti, st_vari in zip(["r_st", "r_ast"], [self.st, self.ast], [st_var, ast_var]): - # for k, st_labeli, st_vari in zip( - # ["r_st", "r_ast"], ["st", "ast"], [st_var, ast_var] - # ): # Load the mean as chunked Dask array, otherwise eats memory if type(sti.data) == da.core.Array: loc = da.asarray(sti.data, chunks=memchunk[1:]) @@ -918,20 +910,18 @@ def monte_carlo_single_ended( def monte_carlo_double_ended( self, - sections=None, - p_val="p_val", - p_cov="p_cov", - st_var=None, - ast_var=None, - rst_var=None, - rast_var=None, - conf_ints=None, + result, + st_var, + ast_var, + rst_var, + rast_var, + conf_ints, mc_sample_size=100, var_only_sections=False, + exclude_parameter_uncertainty=False, da_random_state=None, mc_remove_set_flag=True, - reduce_memory_usage=False, - **kwargs, + reduce_memory_usage=False ): r""" Estimation of the confidence intervals for the temperatures measured @@ -1053,6 +1043,7 @@ def monte_carlo_double_ended( Spectrum Distributed Temperature Sensing. Sensors, 20(8), 2235. https://doi.org/10.3390/s20082235 + TODO: Use get_params_from_pval_double_ended() to extract parameter sets from mc """ def create_da_ta2(no, i_splice, direction="fw", chunks=None): @@ -1084,9 +1075,6 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): ).rechunk((1, chunks[1], 1)) return arr - out = xr.Dataset() - params = xr.Dataset() - if da_random_state: # In testing environments assert isinstance(da_random_state, da.random.RandomState) @@ -1094,16 +1082,26 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): else: state = da.random.RandomState() - if conf_ints: - assert "tmpw", ( - "Current implementation requires you to " - 'define "tmpw" when estimating confidence ' - "intervals" - ) + out = xr.Dataset(coords={"x": self.x, "time": self.time, "trans_att": result["trans_att"]}).copy() + out.coords["x"].attrs = dim_attrs["x"] + out.coords["trans_att"].attrs = dim_attrs["trans_att"] + out.coords["CI"] = conf_ints + + set_sections(out, result.dts.sections) + set_matching_sections(out, result.dts.matching_sections) + + params = out.copy() # Contains all mc sampled parameters + params.coords["mc"] = range(mc_sample_size) no, nt = self.st.shape - nta = self.trans_att.size - npar = 1 + 2 * nt + no + nt * 2 * nta # number of parameters + nta = result["trans_att"].size + + p_val = result["p_val"].data + p_cov = result["p_cov"].data + + npar = p_val.size + npar_valid = 1 + 2 * nt + no + nt * 2 * nta + assert npar == npar_valid, "Inconsistent result object" rsize = (mc_sample_size, no, nt) @@ -1116,26 +1114,7 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): (mc_sample_size, no, nt), chunks={0: -1, 1: "auto", 2: "auto"} ).chunks - params.coords["mc"] = range(mc_sample_size) - params.coords["x"] = self.x - params.coords["time"] = self.time - - if conf_ints: - self.coords["CI"] = conf_ints - params.coords["CI"] = conf_ints - - assert isinstance(p_val, (str, np.ndarray, np.generic)) - if isinstance(p_val, str): - p_val = self[p_val].values - assert p_val.shape == (npar,), ( - "Did you set 'talpha' as " - "keyword argument of the " - "conf_int_double_ended() function?" - ) - - assert isinstance(p_cov, (str, np.ndarray, np.generic, bool)) - - if isinstance(p_cov, bool) and not p_cov: + if exclude_parameter_uncertainty: # Exclude parameter uncertainty if p_cov == False gamma = p_val[0] d_fw = p_val[1 : nt + 1] @@ -1167,16 +1146,8 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): params["talpha_fw_mc"] = (("x", "time"), ta_fw_arr) params["talpha_bw_mc"] = (("x", "time"), ta_bw_arr) - elif isinstance(p_cov, bool) and p_cov: - raise NotImplementedError("Not an implemented option. Check p_cov argument") - else: - # WLS - if isinstance(p_cov, str): - p_cov = self[p_cov].values - assert p_cov.shape == (npar, npar) - - assert sections is not None, "Define sections" + sections = result.dts.sections ix_sec = self.ufunc_per_section( sections=sections, x_indices=True, calc_per="all" ) @@ -1306,80 +1277,79 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): ) for label in ["tmpf", "tmpb"]: - if "tmpw" or label: - if label == "tmpf": - if nta: - params["tmpf_mc_set"] = ( - params["gamma_mc"] - / ( - np.log(params["r_st"] / params["r_ast"]) - + params["df_mc"] - + params["alpha_mc"] - + params["talpha_fw_mc"] - ) - - 273.15 - ) - else: - params["tmpf_mc_set"] = ( - params["gamma_mc"] - / ( - np.log(params["r_st"] / params["r_ast"]) - + params["df_mc"] - + params["alpha_mc"] - ) - - 273.15 + if label == "tmpf": + if nta: + params["tmpf_mc_set"] = ( + params["gamma_mc"] + / ( + np.log(params["r_st"] / params["r_ast"]) + + params["df_mc"] + + params["alpha_mc"] + + params["talpha_fw_mc"] ) + - 273.15 + ) else: - if nta: - params["tmpb_mc_set"] = ( - params["gamma_mc"] - / ( - np.log(params["r_rst"] / params["r_rast"]) - + params["db_mc"] - - params["alpha_mc"] - + params["talpha_bw_mc"] - ) - - 273.15 + params["tmpf_mc_set"] = ( + params["gamma_mc"] + / ( + np.log(params["r_st"] / params["r_ast"]) + + params["df_mc"] + + params["alpha_mc"] ) - else: - params["tmpb_mc_set"] = ( - params["gamma_mc"] - / ( - np.log(params["r_rst"] / params["r_rast"]) - + params["db_mc"] - - params["alpha_mc"] - ) - - 273.15 + - 273.15 + ) + else: + if nta: + params["tmpb_mc_set"] = ( + params["gamma_mc"] + / ( + np.log(params["r_rst"] / params["r_rast"]) + + params["db_mc"] + - params["alpha_mc"] + + params["talpha_bw_mc"] ) - - if var_only_sections: - # sets the values outside the reference sections to NaN - xi = self.ufunc_per_section( - sections=sections, x_indices=True, calc_per="all" + - 273.15 ) - x_mask_ = [ - True if ix in xi else False for ix in range(params.x.size) - ] - x_mask = np.reshape(x_mask_, (1, -1, 1)) - params[label + "_mc_set"] = params[label + "_mc_set"].where(x_mask) - - # subtract the mean temperature - q = params[label + "_mc_set"] - self[label] - out[label + "_mc_var"] = q.var(dim="mc", ddof=1) - - if conf_ints: - new_chunks = list(params[label + "_mc_set"].chunks) - new_chunks[0] = (len(conf_ints),) - avg_axis = params[label + "_mc_set"].get_axis_num("mc") - q = params[label + "_mc_set"].data.map_blocks( - lambda x: np.percentile(x, q=conf_ints, axis=avg_axis), - chunks=new_chunks, # - drop_axis=avg_axis, - # avg dimensions are dropped from input arr - new_axis=0, - ) # The new CI dimension is added as firsaxis - - out[label + "_mc"] = (("CI", "x", "time"), q) + else: + params["tmpb_mc_set"] = ( + params["gamma_mc"] + / ( + np.log(params["r_rst"] / params["r_rast"]) + + params["db_mc"] + - params["alpha_mc"] + ) + - 273.15 + ) + + if var_only_sections: + # sets the values outside the reference sections to NaN + xi = self.ufunc_per_section( + sections=sections, x_indices=True, calc_per="all" + ) + x_mask_ = [ + True if ix in xi else False for ix in range(params.x.size) + ] + x_mask = np.reshape(x_mask_, (1, -1, 1)) + params[label + "_mc_set"] = params[label + "_mc_set"].where(x_mask) + + # subtract the mean temperature + q = params[label + "_mc_set"] - self[label] + out[label + "_mc_var"] = q.var(dim="mc", ddof=1) + + if conf_ints: + new_chunks = list(params[label + "_mc_set"].chunks) + new_chunks[0] = (len(conf_ints),) + avg_axis = params[label + "_mc_set"].get_axis_num("mc") + q = params[label + "_mc_set"].data.map_blocks( + lambda x: np.percentile(x, q=conf_ints, axis=avg_axis), + chunks=new_chunks, # + drop_axis=avg_axis, + # avg dimensions are dropped from input arr + new_axis=0, + ) # The new CI dimension is added as firsaxis + + out[label + "_mc"] = (("CI", "x", "time"), q) # Weighted mean of the forward and backward tmpw_var = 1 / (1 / out["tmpf_mc_var"] + 1 / out["tmpb_mc_var"]) From 98ce96f5cc7799cfa22ee671576a9dc80b892fa3 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Sun, 15 Oct 2023 17:48:48 +0200 Subject: [PATCH 29/66] Single and double ended calibration working and tested, including the monte carlo runs --- src/dtscalibration/calibrate_utils.py | 74 +- src/dtscalibration/datastore_accessor.py | 1461 +++++++++++++++++++++- tests/test_dtscalibration.py | 893 ++----------- tests/test_variance_stokes.py | 693 +++++++++- 4 files changed, 2249 insertions(+), 872 deletions(-) diff --git a/src/dtscalibration/calibrate_utils.py b/src/dtscalibration/calibrate_utils.py index bd84f7b2..e0590d7c 100644 --- a/src/dtscalibration/calibrate_utils.py +++ b/src/dtscalibration/calibrate_utils.py @@ -53,7 +53,7 @@ def calibration_single_ended_helper( fix_alpha, fix_dalpha, fix_gamma, - matching_indices, + matching_sections, trans_att, solver, ): @@ -62,6 +62,11 @@ def calibration_single_ended_helper( nx = self.dts.nx nta = len(trans_att) + if matching_sections: + matching_indices = match_sections(self, matching_sections) + else: + matching_indices = None + calc_cov = True split = calibration_single_ended_solver( self, @@ -497,12 +502,22 @@ def calibration_double_ended_helper( nt, nta, nx, - nx_sec, ix_sec, - matching_indices, + matching_sections, + trans_att, solver, verbose, ): + nt = self.dts.nt + nx = self.dts.nx + + nx_sec = ix_sec.size + + if matching_sections: + matching_indices = match_sections(self, matching_sections) + else: + matching_indices = None + if fix_alpha or fix_gamma: split = calibration_double_ended_solver( self, @@ -514,6 +529,8 @@ def calibration_double_ended_helper( calc_cov=True, solver="external_split", matching_indices=matching_indices, + trans_att=trans_att, + nta=nta, verbose=verbose, ) else: @@ -527,6 +544,8 @@ def calibration_double_ended_helper( calc_cov=True, solver=solver, matching_indices=matching_indices, + trans_att=trans_att, + nta=nta, verbose=verbose, ) @@ -820,7 +839,7 @@ def calibration_double_ended_helper( # put E outside of reference section in solution # concatenating makes a copy of the data instead of using a # pointer - ds_sub = self[["st", "ast", "rst", "rast", "trans_att"]] + ds_sub = self[["st", "ast", "rst", "rast"]] ds_sub["df"] = (("time",), out[0][:nt]) ds_sub["df_var"] = (("time",), out[1][:nt]) ds_sub["db"] = (("time",), out[0][nt : 2 * nt]) @@ -854,6 +873,7 @@ def calibration_double_ended_helper( rst_var=rst_var, rast_var=rast_var, ix_alpha_is_zero=ix_sec[0], + trans_att=trans_att, talpha_fw=talpha_fw, talpha_bw=talpha_bw, talpha_fw_var=talpha_fw_var, @@ -1115,6 +1135,8 @@ def calibration_double_ended_solver( # noqa: MC0001 calc_cov=True, solver="sparse", matching_indices=None, + trans_att=None, + nta=None, verbose=False, ): """ @@ -1180,7 +1202,6 @@ def calibration_double_ended_solver( # noqa: MC0001 x_sec = ds_sec["x"].values nx_sec = x_sec.size nt = ds.time.size - nta = ds.dts.nta # Calculate E as initial estimate for the E calibration. # Does not require ta to be passed on @@ -1192,6 +1213,7 @@ def calibration_double_ended_solver( # noqa: MC0001 rst_var=rst_var, rast_var=rast_var, ix_alpha_is_zero=ix_alpha_is_zero, + trans_att=trans_att ) df_est, db_est = calc_df_db_double_est(ds, sections, ix_alpha_is_zero, 485.0) @@ -1202,7 +1224,7 @@ def calibration_double_ended_solver( # noqa: MC0001 Zero_d, Z_TA_fw, Z_TA_bw, - ) = construct_submatrices(sections, nt, nx_sec, ds, ds.trans_att.values, x_sec) + ) = construct_submatrices(sections, nt, nx_sec, ds, trans_att, x_sec) # y # Eq.41--45 y_F = np.log(ds_sec.st / ds_sec.ast).values.ravel() @@ -1261,7 +1283,7 @@ def calibration_double_ended_solver( # noqa: MC0001 matching_indices[:, 0], matching_indices[:, 1], nt, - ds.trans_att.values, + trans_att, ) p0_est = np.concatenate( @@ -1445,18 +1467,8 @@ def calibration_double_ended_solver( # noqa: MC0001 elif not calc_cov and not verbose: p_sol, p_var = out - # if verbose: - # from dtscalibration.plot import plot_location_residuals_double_ended - # - # dv = plot_location_residuals_double_ended(ds, werr, hix, tix, ix_sec, - # ix_match_not_cal, nt) - - # p_sol contains the int diff att of all the locations within the - # reference sections. po_sol is its expanded version that contains also - # the int diff att for outside the reference sections. - # calculate talpha_fw and bw for attenuation - if ds.dts.nta > 0: + if nta > 0: if np.any(matching_indices): ta = p_sol[1 + 2 * nt + ix_from_cal_match_to_glob.size :].reshape( (nt, 2, nta), order="F" @@ -1481,7 +1493,7 @@ def calibration_double_ended_solver( # noqa: MC0001 # put E outside of reference section in solution # concatenating makes a copy of the data instead of using a pointer - ds_sub = ds[["st", "ast", "rst", "rast", "trans_att"]] + ds_sub = ds[["st", "ast", "rst", "rast"]] ds_sub["df"] = (("time",), p_sol[1 : 1 + nt]) ds_sub["df_var"] = (("time",), p_var[1 : 1 + nt]) ds_sub["db"] = (("time",), p_sol[1 + nt : 1 + 2 * nt]) @@ -1494,6 +1506,7 @@ def calibration_double_ended_solver( # noqa: MC0001 rst_var=rst_var, rast_var=rast_var, ix_alpha_is_zero=ix_alpha_is_zero, + trans_att=trans_att, talpha_fw=talpha_fw, talpha_bw=talpha_bw, talpha_fw_var=talpha_fw_var, @@ -1725,7 +1738,7 @@ def construct_submatrices_matching_sections(x, ix_sec, hix, tix, nt, trans_att): Zero_eq3_gamma = sp.coo_matrix(([], ([], [])), shape=(nt * nx_nm, 1)) # TA - if trans_att.size > 0: + if len(trans_att) > 0: # unpublished BdT TA_eq1_list = [] @@ -1875,7 +1888,7 @@ def construct_submatrices(sections, nt, nx, ds, trans_att, x_sec): # Zero # Eq.45 Zero_d = sp.coo_matrix(([], ([], [])), shape=(nt * nx, nt)) # Zero_E = sp.coo_matrix(([], ([], [])), shape=(nt * nx, (nx - 1))) - if trans_att.size > 0: + if len(trans_att) > 0: # unpublished BdT TA_fw_list = [] @@ -2110,6 +2123,7 @@ def calc_alpha_double( rst_var=None, rast_var=None, ix_alpha_is_zero=-1, + trans_att=None, talpha_fw=None, talpha_bw=None, talpha_fw_var=None, @@ -2138,8 +2152,8 @@ def calc_alpha_double( else: rast_var_val = np.asarray(rast_var) - i_var_fw = ds.i_var(st_var_val, ast_var_val, st_label="st", ast_label="ast") - i_var_bw = ds.i_var(rst_var_val, rast_var_val, st_label="rst", ast_label="rast") + i_var_fw = ds.st**-2 * st_var_val + ds.ast**-2 * ast_var_val + i_var_bw = ds.rst**-2 * rst_var_val + ds.rast**-2 * rast_var_val i_fw = np.log(ds.st / ds.ast) i_bw = np.log(ds.rst / ds.rast) @@ -2154,14 +2168,14 @@ def calc_alpha_double( D_F_var = ds["df_var"] D_B_var = ds["db_var"] - if ds.dts.nta > 0: + if len(trans_att) > 0: # Can be improved by including covariances. That reduces the # uncert. ta_arr_fw = np.zeros((ds.x.size, ds["time"].size)) ta_arr_fw_var = np.zeros((ds.x.size, ds["time"].size)) for tai, taxi, tai_var in zip( - talpha_fw.T, ds.trans_att.values, talpha_fw_var.T + talpha_fw.T, trans_att, talpha_fw_var.T ): ta_arr_fw[ds.x.values >= taxi] = ( ta_arr_fw[ds.x.values >= taxi] + tai @@ -2173,7 +2187,7 @@ def calc_alpha_double( ta_arr_bw = np.zeros((ds.x.size, ds["time"].size)) ta_arr_bw_var = np.zeros((ds.x.size, ds["time"].size)) for tai, taxi, tai_var in zip( - talpha_bw.T, ds.trans_att.values, talpha_bw_var.T + talpha_bw.T, trans_att, talpha_bw_var.T ): ta_arr_bw[ds.x.values < taxi] = ta_arr_bw[ds.x.values < taxi] + tai ta_arr_bw_var[ds.x.values < taxi] = ( @@ -2274,14 +2288,14 @@ def match_sections(ds, matching_sections): txs ) - hix = ds.ufunc_per_section( - sections={0: [i[0] for i in matching_sections]}, x_indices=True, calc_per="all" + hix = ds.dts.ufunc_per_section( + sections={0: [i[0] for i in matching_sections]}, x_indices=True, calc_per="all", suppress_section_validation=True ) tixl = [] for _, tslice, reverse_flag in matching_sections: - ixi = ds.ufunc_per_section( - sections={0: [tslice]}, x_indices=True, calc_per="all" + ixi = ds.dts.ufunc_per_section( + sections={0: [tslice]}, x_indices=True, calc_per="all", suppress_section_validation=True ) if reverse_flag: diff --git a/src/dtscalibration/datastore_accessor.py b/src/dtscalibration/datastore_accessor.py index b0dcb116..089a4de3 100644 --- a/src/dtscalibration/datastore_accessor.py +++ b/src/dtscalibration/datastore_accessor.py @@ -4,13 +4,15 @@ import yaml import scipy.stats as sst -from dtscalibration.calibration.section_utils import validate_sections +from dtscalibration.calibration.section_utils import validate_sections, validate_sections_definition, validate_no_overlapping_sections +from dtscalibration.calibrate_utils import calibration_double_ended_helper from dtscalibration.calibrate_utils import calibration_single_ended_helper -from dtscalibration.calibrate_utils import match_sections from dtscalibration.calibrate_utils import parse_st_var from dtscalibration.calibration.section_utils import set_sections from dtscalibration.calibration.section_utils import set_matching_sections +from dtscalibration.datastore_utils import ParameterIndexDoubleEnded from dtscalibration.datastore_utils import ParameterIndexSingleEnded +from dtscalibration.datastore_utils import get_params_from_pval_double_ended from dtscalibration.datastore_utils import get_params_from_pval_single_ended from dtscalibration.datastore_utils import ufunc_per_section_helper from dtscalibration.io_utils import dim_attrs @@ -228,16 +230,16 @@ def get_default_encoding(self, time_chunks_from_key=None): if time_chunks_from_key is not None: # obtain optimal chunk sizes in time and x dim - if self[time_chunks_from_key].dims == ("x", "time"): + if self._obj[time_chunks_from_key].dims == ("x", "time"): x_chunk, t_chunk = da.ones( - self[time_chunks_from_key].shape, + self._obj[time_chunks_from_key].shape, chunks=(-1, "auto"), dtype="float32", ).chunks - elif self[time_chunks_from_key].dims == ("time", "x"): + elif self._obj[time_chunks_from_key].dims == ("time", "x"): x_chunk, t_chunk = da.ones( - self[time_chunks_from_key].shape, + self._obj[time_chunks_from_key].shape, chunks=("auto", -1), dtype="float32", ).chunks @@ -247,16 +249,16 @@ def get_default_encoding(self, time_chunks_from_key=None): for k, v in encoding.items(): # By writing and compressing the data in chunks, some sort of # parallism is possible. - if self[k].dims == ("x", "time"): + if self._obj[k].dims == ("x", "time"): chunks = (x_chunk[0], t_chunk[0]) - elif self[k].dims == ("time", "x"): + elif self._obj[k].dims == ("time", "x"): chunks = (t_chunk[0], x_chunk[0]) - elif self[k].dims == ("x",): + elif self._obj[k].dims == ("x",): chunks = (x_chunk[0],) - elif self[k].dims == ("time",): + elif self._obj[k].dims == ("time",): chunks = (t_chunk[0],) else: @@ -276,6 +278,7 @@ def ufunc_per_section( x_indices=False, ref_temp_broadcasted=False, calc_per="stretch", + suppress_section_validation=False, **func_kwargs, ): """ @@ -394,6 +397,10 @@ def ufunc_per_section( If `self[label]` or `self[subtract_from_label]` is a Dask array, a Dask array is returned else a numpy array is returned """ + if not suppress_section_validation: + validate_sections_definition(sections=sections) + validate_no_overlapping_sections(sections=sections) + if label is None: dataarray = None else: @@ -404,8 +411,6 @@ def ufunc_per_section( reference_dataset = None else: - validate_sections(self._obj, sections) - x_coords = None reference_dataset = {k: self._obj[k] for k in sections} @@ -570,9 +575,6 @@ def calibrate_single_ended( 07Calibrate_single_wls.ipynb>`_ """ - assert self.st.dims[0] == "x", "Stokes are transposed" - assert self.ast.dims[0] == "x", "Stokes are transposed" - # out contains the state out = xr.Dataset(coords={"x": self.x, "time": self.time, "trans_att": trans_att}).copy() out.coords["x"].attrs = dim_attrs["x"] @@ -585,15 +587,12 @@ def calibrate_single_ended( set_sections(out, sections) set_matching_sections(out, matching_sections) - # Convert sections and matching_sections to indices + assert self.st.dims[0] == "x", "Stokes are transposed" + assert self.ast.dims[0] == "x", "Stokes are transposed" + ix_sec = self.ufunc_per_section( sections=sections, x_indices=True, calc_per="all" ) - if matching_sections: - matching_indices = match_sections(self, matching_sections) - else: - matching_indices = None - assert not np.any(self.st.isel(x=ix_sec) <= 0.0), ( "There is uncontrolled noise in the ST signal. Are your sections" "correctly defined?" @@ -612,7 +611,7 @@ def calibrate_single_ended( fix_alpha, fix_dalpha, fix_gamma, - matching_indices, + matching_sections, trans_att, solver, ) @@ -723,6 +722,509 @@ def calibrate_single_ended( return out + def calibration_double_ended( + self, + sections, + st_var, + ast_var, + rst_var, + rast_var, + method="wls", + solver="sparse", + p_val=None, + p_var=None, + p_cov=None, + trans_att=[], + fix_gamma=None, + fix_alpha=None, + matching_sections=None, + verbose=False, + ): + r""" + See example notebook 8 for an explanation on how to use this function. + Calibrate the Stokes (`ds.st`) and anti-Stokes (`ds.ast`) of the forward + channel and from the backward channel (`ds.rst`, `ds.rast`) data to + temperature using fiber sections with a known temperature + (`ds.sections`) for double-ended setups. The calibrated temperature of + the forward channel is stored under `ds.tmpf` and its variance under + `ds.tmpf_var`, and that of the backward channel under `ds.tmpb` and + `ds.tmpb_var`. The inverse-variance weighted average of the forward and + backward channel is stored under `ds.tmpw` and `ds.tmpw_var`. + In double-ended setups, Stokes and anti-Stokes intensity is measured in + two directions from both ends of the fiber. The forward-channel + measurements are denoted with subscript F, and the backward-channel + measurements are denoted with subscript B. Both measurement channels + start at a different end of the fiber and have opposite directions, and + therefore have different spatial coordinates. + The first processing step + with double-ended measurements is to align the measurements of the two + measurement channels so that they have the same spatial coordinates. The + spatial coordinate :math:`x` (m) is defined here positive in the forward + direction, starting at 0 where the fiber is connected to the forward + channel of the DTS system; the length of the fiber is :math:`L`. + Consequently, the backward-channel measurements are flipped and shifted + to align with the forward-channel measurements. Alignment of the + measurements of the two channels is prone to error because it requires + the exact fiber length (McDaniel et al., 2018). Depending on the DTS system + used, the forward channel and backward channel are measured one after + another by making use of an optical switch, so that only a single + detector is needed. However, it is assumed in this package that the + forward channel and backward channel are measured simultaneously, so + that the temperature of both measurements is the same. This assumption + holds better for short acquisition times with respect to the timescale + of the temperature variation, and when there is no systematic difference + in temperature between the two channels. The temperature may be computed + from the forward-channel measurements (Equation 10 [1]_) with: + .. math:: + T_\mathrm{F} (x,t) = \\frac{\gamma}{I_\mathrm{F}(x,t) + \ + C_\mathrm{F}(t) + \int_0^x{\Delta\\alpha(x')\,\mathrm{d}x'}} + and from the backward-channel measurements with: + .. math:: + T_\mathrm{B} (x,t) = \\frac{\gamma}{I_\mathrm{B}(x,t) + \ + C_\mathrm{B}(t) + \int_x^L{\Delta\\alpha(x')\,\mathrm{d}x'}} + with + .. math:: + I(x,t) = \ln{\left(\\frac{P_+(x,t)}{P_-(x,t)}\\right)} + .. math:: + C(t) = \ln{\left(\\frac{\eta_-(t)K_-/\lambda_-^4}{\eta_+(t)K_+/\lambda_+^4}\\right)} + where :math:`C` is the lumped effect of the difference in gain at + :param mc_conf_ints: + :math:`x=0` between Stokes and anti-Stokes intensity measurements and + the dependence of the scattering intensity on the wavelength. The + parameters :math:`P_+` and :math:`P_-` are the Stokes and anti-Stokes + intensity measurements, respectively. + :math:`C_\mathrm{F}(t)` and :math:`C_\mathrm{B}(t)` are the + parameter :math:`C(t)` for the forward-channel and backward-channel + measurements, respectively. :math:`C_\mathrm{B}(t)` may be different + from :math:`C_\mathrm{F}(t)` due to differences in gain, and difference + in the attenuation between the detectors and the point the fiber end is + connected to the DTS system (:math:`\eta_+` and :math:`\eta_-` in + Equation~\\ref{eqn:c}). :math:`T` in the listed + equations is in Kelvin, but is converted to Celsius after calibration. + The calibration procedure presented in van de + Giesen et al. 2012 approximates :math:`C(t)` to be + the same for the forward and backward-channel measurements, but this + approximation is not made here. + Parameter :math:`A(x)` (`ds.alpha`) is introduced to simplify the notation of the + double-ended calibration procedure and represents the integrated + differential attenuation between locations :math:`x_1` and :math:`x` + along the fiber. Location :math:`x_1` is the first reference section + location (the smallest x-value of all used reference sections). + .. math:: + A(x) = \int_{x_1}^x{\Delta\\alpha(x')\,\mathrm{d}x'} + so that the expressions for temperature may be written as: + .. math:: + T_\mathrm{F} (x,t) = \\frac{\gamma}{I_\mathrm{F}(x,t) + D_\mathrm{F}(t) + A(x)}, + T_\mathrm{B} (x,t) = \\frac{\gamma}{I_\mathrm{B}(x,t) + D_\mathrm{B}(t) - A(x)} + where + .. math:: + D_{\mathrm{F}}(t) = C_{\mathrm{F}}(t) + \int_0^{x_1}{\Delta\\alpha(x')\,\mathrm{d}x'}, + D_{\mathrm{B}}(t) = C_{\mathrm{B}}(t) + \int_{x_1}^L{\Delta\\alpha(x')\,\mathrm{d}x'} + Parameters :math:`D_\mathrm{F}` (`ds.df`) and :math:`D_\mathrm{B}` + (`ds.db`) must be estimated for each time and are constant along the fiber, and parameter + :math:`A` must be estimated for each location and is constant over time. + The calibration procedure is discussed in Section 6. + :math:`T_\mathrm{F}` (`ds.tmpf`) and :math:`T_\mathrm{B}` (`ds.tmpb`) + are separate + approximations of the same temperature at the same time. The estimated + :math:`T_\mathrm{F}` is more accurate near :math:`x=0` because that is + where the signal is strongest. Similarly, the estimated + :math:`T_\mathrm{B}` is more accurate near :math:`x=L`. A single best + estimate of the temperature is obtained from the weighted average of + :math:`T_\mathrm{F}` and :math:`T_\mathrm{B}` as discussed in + Section 7.2 [1]_ . + + + Parameters + ---------- + p_val : array-like, optional + Define `p_val`, `p_var`, `p_cov` if you used an external function + for calibration. Has size `1 + 2 * nt + nx + 2 * nt * nta`. + First value is :math:`\gamma`, then `nt` times + :math:`D_\mathrm{F}`, then `nt` times + :math:`D_\mathrm{B}`, then for each location :math:`D_\mathrm{B}`, + then for each connector that introduces directional attenuation two + parameters per time step. + p_var : array-like, optional + Define `p_val`, `p_var`, `p_cov` if you used an external function + for calibration. Has size `1 + 2 * nt + nx + 2 * nt * nta`. + Is the variance of `p_val`. + p_cov : array-like, optional + The covariances of `p_val`. Square matrix. + If set to False, no uncertainty in the parameters is propagated + into the confidence intervals. Similar to the spec sheets of the DTS + manufacturers. And similar to passing an array filled with zeros. + sections : Dict[str, List[slice]], optional + If `None` is supplied, `ds.sections` is used. Define calibration + sections. Each section requires a reference temperature time series, + such as the temperature measured by an external temperature sensor. + They should already be part of the DataStore object. `sections` + is defined with a dictionary with its keywords of the + names of the reference temperature time series. Its values are + lists of slice objects, where each slice object is a fiber stretch + that has the reference temperature. Afterwards, `sections` is stored + under `ds.sections`. + st_var, ast_var, rst_var, rast_var : float, callable, array-like, optional + The variance of the measurement noise of the Stokes signals in the + forward direction. If `float` the variance of the noise from the + Stokes detector is described with a single value. + If `callable` the variance of the noise from the Stokes detector is + a function of the intensity, as defined in the callable function. + Or manually define a variance with a DataArray of the shape + `ds.st.shape`, where the variance can be a function of time and/or + x. Required if method is wls. + mc_sample_size : int, optional + If set, the variance is also computed using Monte Carlo sampling. + The number of Monte Carlo samples drawn used to estimate the + variance of the forward and backward channel temperature estimates + and estimate the inverse-variance weighted average temperature. + conf_ints : iterable object of float + A list with the confidence boundaries that are calculated. Valid + values are between [0, 1]. + mc_da_random_state + For testing purposes. Similar to random seed. The seed for dask. + Makes random not so random. To produce reproducable results for + testing environments. + mc_remove_set_flag : bool + Remove the monte carlo data set, from which the CI and the + variance are calculated. + variance_suffix : str, optional + String appended for storing the variance. Only used when method + is wls. + method : {'wls', 'external'} + Use `'wls'` for weighted least squares. + solver : {'sparse', 'stats'} + Either use the homemade weighted sparse solver or the weighted + dense matrix solver of statsmodels. The sparse solver uses much less + memory, is faster, and gives the same result as the statsmodels + solver. The statsmodels solver is mostly used to check the sparse + solver. `'stats'` is the default. + trans_att : iterable, optional + Splices can cause jumps in differential attenuation. Normal single + ended calibration assumes these are not present. An additional loss + term is added in the 'shadow' of the splice. Each location + introduces an additional nt parameters to solve for. Requiring + either an additional calibration section or matching sections. + If multiple locations are defined, the losses are added. + fix_gamma : Tuple[float, float], optional + A tuple containing two floats. The first float is the value of + gamma, and the second item is the variance of the estimate of gamma. + Covariances between gamma and other parameters are not accounted + for. + fix_alpha : Tuple[array-like, array-like], optional + A tuple containing two arrays. The first array contains the + values of integrated differential att (:math:`A` in paper), and the + second array contains the variance of the estimate of alpha. + Covariances (in-) between alpha and other parameters are not + accounted for. + matching_sections : List[Tuple[slice, slice, bool]] + Provide a list of tuples. A tuple per matching section. Each tuple + has three items. The first two items are the slices of the sections + that are matched. The third item is a boolean and is True if the two + sections have a reverse direction ("J-configuration"). + matching_indices : array + Provide an array of x-indices of size (npair, 2), where each pair + has the same temperature. Used to improve the estimate of the + integrated differential attenuation. + verbose : bool + Show additional calibration information + Returns + ------- + References + ---------- + .. [1] des Tombe, B., Schilperoort, B., & Bakker, M. (2020). Estimation + of Temperature and Associated Uncertainty from Fiber-Optic Raman- + Spectrum Distributed Temperature Sensing. Sensors, 20(8), 2235. + https://doi.org/10.3390/s20082235 + Examples + -------- + - `Example notebook 8: Calibrate double ended `_ + """ + # out contains the state + out = xr.Dataset(coords={"x": self.x, "time": self.time, "trans_att": trans_att}).copy() + out.coords["x"].attrs = dim_attrs["x"] + out.coords["trans_att"].attrs = dim_attrs["trans_att"] + + nta = len(trans_att) + + # check and store sections and matching_sections + validate_sections(self._obj, sections=sections) + set_sections(out, sections) + set_matching_sections(out, matching_sections) + + # TODO: confidence intervals using the variance approximated by linear error propagation + assert self.st.dims[0] == "x", "Stokes are transposed" + assert self.ast.dims[0] == "x", "Stokes are transposed" + assert self.rst.dims[0] == "x", "Stokes are transposed" + assert self.rast.dims[0] == "x", "Stokes are transposed" + + ix_sec = self.ufunc_per_section( + sections=sections, x_indices=True, calc_per="all" + ) + assert not np.any(self.st.isel(x=ix_sec) <= 0.0), ( + "There is uncontrolled noise in the ST signal. Are your sections" + "correctly defined?" + ) + assert not np.any(self.ast.isel(x=ix_sec) <= 0.0), ( + "There is uncontrolled noise in the AST signal. Are your sections" + "correctly defined?" + ) + assert not np.any(self.rst.isel(x=ix_sec) <= 0.0), ( + "There is uncontrolled noise in the REV-ST signal. Are your " + "sections correctly defined?" + ) + assert not np.any(self.rast.isel(x=ix_sec) <= 0.0), ( + "There is uncontrolled noise in the REV-AST signal. Are your " + "sections correctly defined?" + ) + + if method == "wls": + p_cov, p_val, p_var = calibration_double_ended_helper( + self._obj, + sections, + st_var, + ast_var, + rst_var, + rast_var, + fix_alpha, + fix_gamma, + self.nt, + nta, + self.nx, + ix_sec, + matching_sections, + trans_att, + solver, + verbose, + ) + + elif method == "external": + for input_item in [p_val, p_var, p_cov]: + assert input_item is not None + + elif method == "external_split": + raise ValueError("Not implemented yet") + + else: + raise ValueError("Choose a valid method") + + # all below require the following solution sizes + ip = ParameterIndexDoubleEnded(self.nt, self.nx, nta) + + # npar = 1 + 2 * nt + nx + 2 * nt * nta + assert p_val.size == ip.npar + assert p_var.size == ip.npar + assert p_cov.shape == (ip.npar, ip.npar) + + params = get_params_from_pval_double_ended(ip, out.coords, p_val=p_val) + param_covs = get_params_from_pval_double_ended( + ip, out.coords, p_val=p_var, p_cov=p_cov + ) + + tmpf = params["gamma"] / ( + np.log(self.st / self.ast) + + params["df"] + + params["alpha"] + + params["talpha_fw_full"] + ) + tmpb = params["gamma"] / ( + np.log(self.rst / self.rast) + + params["db"] + - params["alpha"] + + params["talpha_bw_full"] + ) + out["tmpf"] = tmpf - 273.15 + out["tmpb"] = tmpb - 273.15 + + deriv_dict = dict( + T_gamma_fw=tmpf / params["gamma"], + T_st_fw=-(tmpf**2) / (params["gamma"] * self.st), + T_ast_fw=tmpf**2 / (params["gamma"] * self.ast), + T_df_fw=-(tmpf**2) / params["gamma"], + T_alpha_fw=-(tmpf**2) / params["gamma"], + T_ta_fw=-(tmpf**2) / params["gamma"], + T_gamma_bw=tmpb / params["gamma"], + T_rst_bw=-(tmpb**2) / (params["gamma"] * self.rst), + T_rast_bw=tmpb**2 / (params["gamma"] * self.rast), + T_db_bw=-(tmpb**2) / params["gamma"], + T_alpha_bw=tmpb**2 / params["gamma"], + T_ta_bw=-(tmpb**2) / params["gamma"], + ) + deriv_ds = xr.Dataset(deriv_dict) + out["deriv"] = deriv_ds.to_array(dim="com2") + + var_fw_dict = dict( + dT_dst=deriv_ds.T_st_fw**2 * parse_st_var(self.st, st_var), + dT_dast=deriv_ds.T_ast_fw**2 + * parse_st_var(self.ast, ast_var), + dT_gamma=deriv_ds.T_gamma_fw**2 * param_covs["gamma"], + dT_ddf=deriv_ds.T_df_fw**2 * param_covs["df"], + dT_dalpha=deriv_ds.T_alpha_fw**2 * param_covs["alpha"], + dT_dta=deriv_ds.T_ta_fw**2 * param_covs["talpha_fw_full"], + dgamma_ddf=( + 2 * deriv_ds.T_gamma_fw * deriv_ds.T_df_fw * param_covs["gamma_df"] + ), + dgamma_dalpha=( + 2 + * deriv_ds.T_gamma_fw + * deriv_ds.T_alpha_fw + * param_covs["gamma_alpha"] + ), + dalpha_ddf=( + 2 * deriv_ds.T_alpha_fw * deriv_ds.T_df_fw * param_covs["alpha_df"] + ), + dta_dgamma=( + 2 * deriv_ds.T_ta_fw * deriv_ds.T_gamma_fw * param_covs["tafw_gamma"] + ), + dta_ddf=(2 * deriv_ds.T_ta_fw * deriv_ds.T_df_fw * param_covs["tafw_df"]), + dta_dalpha=( + 2 * deriv_ds.T_ta_fw * deriv_ds.T_alpha_fw * param_covs["tafw_alpha"] + ), + ) + var_bw_dict = dict( + dT_drst=deriv_ds.T_rst_bw**2 + * parse_st_var(self.rst, rst_var), + dT_drast=deriv_ds.T_rast_bw**2 + * parse_st_var(self.rast, rast_var), + dT_gamma=deriv_ds.T_gamma_bw**2 * param_covs["gamma"], + dT_ddb=deriv_ds.T_db_bw**2 * param_covs["db"], + dT_dalpha=deriv_ds.T_alpha_bw**2 * param_covs["alpha"], + dT_dta=deriv_ds.T_ta_bw**2 * param_covs["talpha_bw_full"], + dgamma_ddb=( + 2 * deriv_ds.T_gamma_bw * deriv_ds.T_db_bw * param_covs["gamma_db"] + ), + dgamma_dalpha=( + 2 + * deriv_ds.T_gamma_bw + * deriv_ds.T_alpha_bw + * param_covs["gamma_alpha"] + ), + dalpha_ddb=( + 2 * deriv_ds.T_alpha_bw * deriv_ds.T_db_bw * param_covs["alpha_db"] + ), + dta_dgamma=( + 2 * deriv_ds.T_ta_bw * deriv_ds.T_gamma_bw * param_covs["tabw_gamma"] + ), + dta_ddb=(2 * deriv_ds.T_ta_bw * deriv_ds.T_db_bw * param_covs["tabw_db"]), + dta_dalpha=( + 2 * deriv_ds.T_ta_bw * deriv_ds.T_alpha_bw * param_covs["tabw_alpha"] + ), + ) + + out["var_fw_da"] = xr.Dataset(var_fw_dict).to_array(dim="comp_fw") + out["var_bw_da"] = xr.Dataset(var_bw_dict).to_array(dim="comp_bw") + + out["tmpf_var"] = out["var_fw_da"].sum(dim="comp_fw") + out["tmpb_var"] = out["var_bw_da"].sum(dim="comp_bw") + + # First estimate of tmpw_var + out["tmpw_var" + "_approx"] = 1 / (1 / out["tmpf_var"] + 1 / out["tmpb_var"]) + out["tmpw"] = ( + (tmpf / out["tmpf_var"] + tmpb / out["tmpb_var"]) + * out["tmpw_var" + "_approx"] + ) - 273.15 + + weightsf = out["tmpw_var" + "_approx"] / out["tmpf_var"] + weightsb = out["tmpw_var" + "_approx"] / out["tmpb_var"] + + deriv_dict2 = dict( + T_gamma_w=weightsf * deriv_dict["T_gamma_fw"] + + weightsb * deriv_dict["T_gamma_bw"], + T_st_w=weightsf * deriv_dict["T_st_fw"], + T_ast_w=weightsf * deriv_dict["T_ast_fw"], + T_rst_w=weightsb * deriv_dict["T_rst_bw"], + T_rast_w=weightsb * deriv_dict["T_rast_bw"], + T_df_w=weightsf * deriv_dict["T_df_fw"], + T_db_w=weightsb * deriv_dict["T_db_bw"], + T_alpha_w=weightsf * deriv_dict["T_alpha_fw"] + + weightsb * deriv_dict["T_alpha_bw"], + T_taf_w=weightsf * deriv_dict["T_ta_fw"], + T_tab_w=weightsb * deriv_dict["T_ta_bw"], + ) + deriv_ds2 = xr.Dataset(deriv_dict2) + + # TODO: sigma2_tafw_tabw + var_w_dict = dict( + dT_dst=deriv_ds2.T_st_w**2 * parse_st_var(self.st, st_var), + dT_dast=deriv_ds2.T_ast_w**2 + * parse_st_var(self.ast, ast_var), + dT_drst=deriv_ds2.T_rst_w**2 + * parse_st_var(self.rst, rst_var), + dT_drast=deriv_ds2.T_rast_w**2 + * parse_st_var(self.rast, rast_var), + dT_gamma=deriv_ds2.T_gamma_w**2 * param_covs["gamma"], + dT_ddf=deriv_ds2.T_df_w**2 * param_covs["df"], + dT_ddb=deriv_ds2.T_db_w**2 * param_covs["db"], + dT_dalpha=deriv_ds2.T_alpha_w**2 * param_covs["alpha"], + dT_dtaf=deriv_ds2.T_taf_w**2 * param_covs["talpha_fw_full"], + dT_dtab=deriv_ds2.T_tab_w**2 * param_covs["talpha_bw_full"], + dgamma_ddf=2 + * deriv_ds2.T_gamma_w + * deriv_ds2.T_df_w + * param_covs["gamma_df"], + dgamma_ddb=2 + * deriv_ds2.T_gamma_w + * deriv_ds2.T_db_w + * param_covs["gamma_db"], + dgamma_dalpha=2 + * deriv_ds2.T_gamma_w + * deriv_ds2.T_alpha_w + * param_covs["gamma_alpha"], + dgamma_dtaf=2 + * deriv_ds2.T_gamma_w + * deriv_ds2.T_taf_w + * param_covs["tafw_gamma"], + dgamma_dtab=2 + * deriv_ds2.T_gamma_w + * deriv_ds2.T_tab_w + * param_covs["tabw_gamma"], + ddf_ddb=2 * deriv_ds2.T_df_w * deriv_ds2.T_db_w * param_covs["df_db"], + ddf_dalpha=2 + * deriv_ds2.T_df_w + * deriv_ds2.T_alpha_w + * param_covs["alpha_df"], + ddf_dtaf=2 * deriv_ds2.T_df_w * deriv_ds2.T_taf_w * param_covs["tafw_df"], + ddf_dtab=2 * deriv_ds2.T_df_w * deriv_ds2.T_tab_w * param_covs["tabw_df"], + ddb_dalpha=2 + * deriv_ds2.T_db_w + * deriv_ds2.T_alpha_w + * param_covs["alpha_db"], + ddb_dtaf=2 * deriv_ds2.T_db_w * deriv_ds2.T_taf_w * param_covs["tafw_db"], + ddb_dtab=2 * deriv_ds2.T_db_w * deriv_ds2.T_tab_w * param_covs["tabw_db"], + # dtaf_dtab=2 * deriv_ds2.T_tab_w * deriv_ds2.T_tab_w * param_covs["tafw_tabw"], + ) + out["var_w_da"] = xr.Dataset(var_w_dict).to_array(dim="comp_w") + out["tmpw_var"] = out["var_w_da"].sum(dim="comp_w") + + # Compute uncertainty solely due to noise in Stokes signal, neglecting parameter uncenrtainty + tmpf_var_excl_par = ( + out["var_fw_da"].sel(comp_fw=["dT_dst", "dT_dast"]).sum(dim="comp_fw") + ) + tmpb_var_excl_par = ( + out["var_bw_da"].sel(comp_bw=["dT_drst", "dT_drast"]).sum(dim="comp_bw") + ) + out["tmpw_var" + "_lower"] = 1 / (1 / tmpf_var_excl_par + 1 / tmpb_var_excl_par) + + out["tmpf"].attrs.update(dim_attrs["tmpf"]) + out["tmpb"].attrs.update(dim_attrs["tmpb"]) + out["tmpw"].attrs.update(dim_attrs["tmpw"]) + out["tmpf_var"].attrs.update(dim_attrs["tmpf_var"]) + out["tmpb_var"].attrs.update(dim_attrs["tmpb_var"]) + out["tmpw_var"].attrs.update(dim_attrs["tmpw_var"]) + out["tmpw_var" + "_approx"].attrs.update(dim_attrs["tmpw_var_approx"]) + out["tmpw_var" + "_lower"].attrs.update(dim_attrs["tmpw_var_lower"]) + + out["p_val"] = (("params1",), p_val) + out["p_cov"] = (("params1", "params2"), p_cov) + + out.update(params) + for key, dataarray in param_covs.data_vars.items(): + out[key + "_var"] = dataarray + + return out + def monte_carlo_single_ended( self, result, @@ -1229,20 +1731,20 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): params["talpha_bw_mc"] = (("mc", "x", "time"), ta_bw_arr) # Draw from the normal distributions for the Stokes intensities - for k, st_labeli, st_vari in zip( + for k, sti, st_vari in zip( ["r_st", "r_ast", "r_rst", "r_rast"], - ["st", "ast", "rst", "rast"], + [self.st, self.ast, self.rst, self.rast], [st_var, ast_var, rst_var, rast_var], ): # Load the mean as chunked Dask array, otherwise eats memory - if type(self[st_labeli].data) == da.core.Array: - loc = da.asarray(self[st_labeli].data, chunks=memchunk[1:]) + if type(sti.data) == da.core.Array: + loc = da.asarray(sti.data, chunks=memchunk[1:]) else: - loc = da.from_array(self[st_labeli].data, chunks=memchunk[1:]) + loc = da.from_array(sti.data, chunks=memchunk[1:]) # Make sure variance is of size (no, nt) if np.size(st_vari) > 1: - if st_vari.shape == self[st_labeli].shape: + if st_vari.shape == sti.shape: pass else: st_vari = np.broadcast_to(st_vari, (no, nt)) @@ -1253,14 +1755,14 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): if type(st_vari) == da.core.Array: st_vari_da = da.asarray(st_vari, chunks=memchunk[1:]) - elif callable(st_vari) and type(self[st_labeli].data) == da.core.Array: + elif callable(st_vari) and type(sti.data) == da.core.Array: st_vari_da = da.asarray( - st_vari(self[st_labeli]).data, chunks=memchunk[1:] + st_vari(sti).data, chunks=memchunk[1:] ) - elif callable(st_vari) and type(self[st_labeli].data) != da.core.Array: + elif callable(st_vari) and type(sti.data) != da.core.Array: st_vari_da = da.from_array( - st_vari(self[st_labeli]).data, chunks=memchunk[1:] + st_vari(sti).data, chunks=memchunk[1:] ) else: @@ -1334,7 +1836,7 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): params[label + "_mc_set"] = params[label + "_mc_set"].where(x_mask) # subtract the mean temperature - q = params[label + "_mc_set"] - self[label] + q = params[label + "_mc_set"] - result[label] out[label + "_mc_var"] = q.var(dim="mc", ddof=1) if conf_ints: @@ -1362,10 +1864,10 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): params["tmpw" + "_mc_set"] = q # out["tmpw"] = ( - self["tmpf"] / out["tmpf_mc_var"] + self["tmpb"] / out["tmpb_mc_var"] + result["tmpf"] / out["tmpf_mc_var"] + result["tmpb"] / out["tmpb_mc_var"] ) * tmpw_var - q = params["tmpw" + "_mc_set"] - self["tmpw"] + q = params["tmpw" + "_mc_set"] - result["tmpw"] out["tmpw" + "_mc_var"] = q.var(dim="mc", ddof=1) # Calculate the CI of the weighted MC_set @@ -1408,5 +1910,892 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): if not mc_remove_set_flag: out.update(params) - self.update(out) return out + + def average_single_ended( + self, + st_var, + ast_var, + conf_ints=None, + mc_sample_size=100, + ci_avg_time_flag1=False, + ci_avg_time_flag2=False, + ci_avg_time_sel=None, + ci_avg_time_isel=None, + ci_avg_x_flag1=False, + ci_avg_x_flag2=False, + ci_avg_x_sel=None, + ci_avg_x_isel=None, + da_random_state=None, + mc_remove_set_flag=True, + reduce_memory_usage=False, + ): + """ + Average temperatures from single-ended setups. + + Four types of averaging are implemented. Please see Example Notebook 16. + + + Parameters + ---------- + p_val : array-like, optional + Define `p_val`, `p_var`, `p_cov` if you used an external function + for calibration. Has size 2 + `nt`. First value is :math:`\gamma`, + second is :math:`\Delta \\alpha`, others are :math:`C` for each + timestep. + If set to False, no uncertainty in the parameters is propagated + into the confidence intervals. Similar to the spec sheets of the DTS + manufacturers. And similar to passing an array filled with zeros + p_cov : array-like, optional + The covariances of `p_val`. + st_var, ast_var : float, callable, array-like, optional + The variance of the measurement noise of the Stokes signals in the + forward direction. If `float` the variance of the noise from the + Stokes detector is described with a single value. + If `callable` the variance of the noise from the Stokes detector is + a function of the intensity, as defined in the callable function. + Or manually define a variance with a DataArray of the shape + `ds.st.shape`, where the variance can be a function of time and/or + x. Required if method is wls. + conf_ints : iterable object of float + A list with the confidence boundaries that are calculated. Valid + values are between + [0, 1]. + mc_sample_size : int + Size of the monte carlo parameter set used to calculate the + confidence interval + ci_avg_time_flag1 : bool + The confidence intervals differ each time step. Assumes the + temperature varies during the measurement period. Computes the + arithmic temporal mean. If you would like to know the confidence + interfal of: + (1) a single additional measurement. So you can state "if another + measurement were to be taken, it would have this ci" + (2) all measurements. So you can state "The temperature remained + during the entire measurement period between these ci bounds". + Adds "tmpw" + '_avg1' and "tmpw" + '_mc_avg1_var' to the + DataStore. If `conf_ints` are set, also the confidence intervals + `_mc_avg1` are added to the DataStore. Works independently of the + ci_avg_time_flag2 and ci_avg_x_flag. + ci_avg_time_flag2 : bool + The confidence intervals differ each time step. Assumes the + temperature remains constant during the measurement period. + Computes the inverse-variance-weighted-temporal-mean temperature + and its uncertainty. + If you would like to know the confidence interfal of: + (1) I want to estimate a background temperature with confidence + intervals. I hereby assume the temperature does not change over + time and average all measurements to get a better estimate of the + background temperature. + Adds "tmpw" + '_avg2' and "tmpw" + '_mc_avg2_var' to the + DataStore. If `conf_ints` are set, also the confidence intervals + `_mc_avg2` are added to the DataStore. Works independently of the + ci_avg_time_flag1 and ci_avg_x_flag. + ci_avg_time_sel : slice + Compute ci_avg_time_flag1 and ci_avg_time_flag2 using only a + selection of the data + ci_avg_time_isel : iterable of int + Compute ci_avg_time_flag1 and ci_avg_time_flag2 using only a + selection of the data + ci_avg_x_flag1 : bool + The confidence intervals differ at each location. Assumes the + temperature varies over `x` and over time. Computes the + arithmic spatial mean. If you would like to know the confidence + interfal of: + (1) a single additional measurement location. So you can state "if + another measurement location were to be taken, + it would have this ci" + (2) all measurement locations. So you can state "The temperature + along the fiber remained between these ci bounds". + Adds "tmpw" + '_avgx1' and "tmpw" + '_mc_avgx1_var' to the + DataStore. If `conf_ints` are set, also the confidence intervals + `_mc_avgx1` are added to the DataStore. Works independently of the + ci_avg_time_flag1, ci_avg_time_flag2 and ci_avg_x2_flag. + ci_avg_x_flag2 : bool + The confidence intervals differ at each location. Assumes the + temperature is the same at each location but varies over time. + Computes the inverse-variance-weighted-spatial-mean temperature + and its uncertainty. + If you would like to know the confidence interfal of: + (1) I have put a lot of fiber in water, and I know that the + temperature variation in the water is much smaller than along + other parts of the fiber. And I would like to average the + measurements from multiple locations to improve the estimated + temperature. + Adds "tmpw" + '_avg2' and "tmpw" + '_mc_avg2_var' to the + DataStore. If `conf_ints` are set, also the confidence intervals + `_mc_avg2` are added to the DataStore. Works independently of the + ci_avg_time_flag1 and ci_avg_x_flag. + ci_avg_x_sel : slice + Compute ci_avg_time_flag1 and ci_avg_time_flag2 using only a + selection of the data + ci_avg_x_isel : iterable of int + Compute ci_avg_time_flag1 and ci_avg_time_flag2 using only a + selection of the data + da_random_state + For testing purposes. Similar to random seed. The seed for dask. + Makes random not so random. To produce reproducable results for + testing environments. + mc_remove_set_flag : bool + Remove the monte carlo data set, from which the CI and the + variance are calculated. + reduce_memory_usage : bool + Use less memory but at the expense of longer computation time + + Returns + ------- + + """ + out = xr.Dataset() + + mcparams = self.conf_int_single_ended( + p_val=p_val, + p_cov=p_cov, + st_var=st_var, + ast_var=ast_var, + conf_ints=None, + mc_sample_size=mc_sample_size, + da_random_state=da_random_state, + mc_remove_set_flag=False, + reduce_memory_usage=reduce_memory_usage, + ) + mcparams["tmpf"] = self["tmpf"] + + if ci_avg_time_sel is not None: + time_dim2 = "time" + "_avg" + x_dim2 = "x" + mcparams.coords[time_dim2] = ( + (time_dim2,), + mcparams["time"].sel(**{"time": ci_avg_time_sel}).data, + ) + mcparams["tmpf_avgsec"] = ( + ("x", time_dim2), + mcparams["tmpf"].sel(**{"time": ci_avg_time_sel}).data, + ) + mcparams["tmpf_mc_set"] = ( + ("mc", "x", time_dim2), + mcparams["tmpf" + "_mc_set"].sel(**{"time": ci_avg_time_sel}).data, + ) + + elif ci_avg_time_isel is not None: + time_dim2 = "time" + "_avg" + x_dim2 = "x" + mcparams.coords[time_dim2] = ( + (time_dim2,), + mcparams["time"].isel(**{"time": ci_avg_time_isel}).data, + ) + mcparams["tmpf_avgsec"] = ( + ("x", time_dim2), + mcparams["tmpf"].isel(**{"time": ci_avg_time_isel}).data, + ) + mcparams["tmpf_mc_set"] = ( + ("mc", "x", time_dim2), + mcparams["tmpf" + "_mc_set"].isel(**{"time": ci_avg_time_isel}).data, + ) + + elif ci_avg_x_sel is not None: + time_dim2 = "time" + x_dim2 = "x_avg" + mcparams.coords[x_dim2] = ((x_dim2,), mcparams.x.sel(x=ci_avg_x_sel).data) + mcparams["tmpf_avgsec"] = ( + (x_dim2, "time"), + mcparams["tmpf"].sel(x=ci_avg_x_sel).data, + ) + mcparams["tmpf_mc_set"] = ( + ("mc", x_dim2, "time"), + mcparams["tmpf_mc_set"].sel(x=ci_avg_x_sel).data, + ) + + elif ci_avg_x_isel is not None: + time_dim2 = "time" + x_dim2 = "x_avg" + mcparams.coords[x_dim2] = ((x_dim2,), mcparams.x.isel(x=ci_avg_x_isel).data) + mcparams["tmpf_avgsec"] = ( + (x_dim2, time_dim2), + mcparams["tmpf"].isel(x=ci_avg_x_isel).data, + ) + mcparams["tmpf_mc_set"] = ( + ("mc", x_dim2, time_dim2), + mcparams["tmpf_mc_set"].isel(x=ci_avg_x_isel).data, + ) + else: + mcparams["tmpf_avgsec"] = mcparams["tmpf"] + x_dim2 = "x" + time_dim2 = "time" + + # subtract the mean temperature + q = mcparams["tmpf_mc_set"] - mcparams["tmpf_avgsec"] + out["tmpf_mc" + "_avgsec_var"] = q.var(dim="mc", ddof=1) + + if ci_avg_x_flag1: + # unweighted mean + out["tmpf_avgx1"] = mcparams["tmpf" + "_avgsec"].mean(dim=x_dim2) + + q = mcparams["tmpf_mc_set"] - mcparams["tmpf_avgsec"] + qvar = q.var(dim=["mc", x_dim2], ddof=1) + out["tmpf_mc_avgx1_var"] = qvar + + if conf_ints: + new_chunks = (len(conf_ints), mcparams["tmpf_mc_set"].chunks[2]) + avg_axis = mcparams["tmpf_mc_set"].get_axis_num(["mc", x_dim2]) + q = mcparams["tmpf_mc_set"].data.map_blocks( + lambda x: np.percentile(x, q=conf_ints, axis=avg_axis), + chunks=new_chunks, # + drop_axis=avg_axis, + # avg dimensions are dropped from input arr + new_axis=0, + ) # The new CI dim is added as firsaxis + + out["tmpf_mc_avgx1"] = (("CI", time_dim2), q) + + if ci_avg_x_flag2: + q = mcparams["tmpf_mc_set"] - mcparams["tmpf_avgsec"] + + qvar = q.var(dim=["mc"], ddof=1) + + # Inverse-variance weighting + avg_x_var = 1 / (1 / qvar).sum(dim=x_dim2) + + out["tmpf_mc_avgx2_var"] = avg_x_var + + mcparams["tmpf" + "_mc_avgx2_set"] = (mcparams["tmpf_mc_set"] / qvar).sum( + dim=x_dim2 + ) * avg_x_var + out["tmpf" + "_avgx2"] = mcparams["tmpf" + "_mc_avgx2_set"].mean(dim="mc") + + if conf_ints: + new_chunks = (len(conf_ints), mcparams["tmpf_mc_set"].chunks[2]) + avg_axis_avgx = mcparams["tmpf_mc_set"].get_axis_num("mc") + + qq = mcparams["tmpf_mc_avgx2_set"].data.map_blocks( + lambda x: np.percentile(x, q=conf_ints, axis=avg_axis_avgx), + chunks=new_chunks, # + drop_axis=avg_axis_avgx, + # avg dimensions are dropped from input arr + new_axis=0, + dtype=float, + ) # The new CI dimension is added as + # firsaxis + out["tmpf_mc_avgx2"] = (("CI", time_dim2), qq) + + if ci_avg_time_flag1 is not None: + # unweighted mean + out["tmpf_avg1"] = mcparams["tmpf_avgsec"].mean(dim=time_dim2) + + q = mcparams["tmpf_mc_set"] - mcparams["tmpf_avgsec"] + qvar = q.var(dim=["mc", time_dim2], ddof=1) + out["tmpf_mc_avg1_var"] = qvar + + if conf_ints: + new_chunks = (len(conf_ints), mcparams["tmpf_mc_set"].chunks[1]) + avg_axis = mcparams["tmpf_mc_set"].get_axis_num(["mc", time_dim2]) + q = mcparams["tmpf_mc_set"].data.map_blocks( + lambda x: np.percentile(x, q=conf_ints, axis=avg_axis), + chunks=new_chunks, # + drop_axis=avg_axis, + # avg dimensions are dropped from input arr + new_axis=0, + ) # The new CI dim is added as firsaxis + + out["tmpf_mc_avg1"] = (("CI", x_dim2), q) + + if ci_avg_time_flag2: + q = mcparams["tmpf_mc_set"] - mcparams["tmpf_avgsec"] + + qvar = q.var(dim=["mc"], ddof=1) + + # Inverse-variance weighting + avg_time_var = 1 / (1 / qvar).sum(dim=time_dim2) + + out["tmpf_mc_avg2_var"] = avg_time_var + + mcparams["tmpf" + "_mc_avg2_set"] = (mcparams["tmpf_mc_set"] / qvar).sum( + dim=time_dim2 + ) * avg_time_var + out["tmpf_avg2"] = mcparams["tmpf" + "_mc_avg2_set"].mean(dim="mc") + + if conf_ints: + new_chunks = (len(conf_ints), mcparams["tmpf_mc_set"].chunks[1]) + avg_axis_avg2 = mcparams["tmpf_mc_set"].get_axis_num("mc") + + qq = mcparams["tmpf_mc_avg2_set"].data.map_blocks( + lambda x: np.percentile(x, q=conf_ints, axis=avg_axis_avg2), + chunks=new_chunks, # + drop_axis=avg_axis_avg2, + # avg dimensions are dropped from input arr + new_axis=0, + dtype=float, + ) # The new CI dimension is added as + # firsaxis + out["tmpf_mc_avg2"] = (("CI", x_dim2), qq) + + # Clean up the garbage. All arrays with a Monte Carlo dimension. + if mc_remove_set_flag: + remove_mc_set = [ + "r_st", + "r_ast", + "gamma_mc", + "dalpha_mc", + "c_mc", + "x_avg", + "time_avg", + "mc", + "ta_mc_arr", + ] + remove_mc_set.append("tmpf_avgsec") + remove_mc_set.append("tmpf_mc_set") + remove_mc_set.append("tmpf_mc_avg2_set") + remove_mc_set.append("tmpf_mc_avgx2_set") + remove_mc_set.append("tmpf_mc_avgsec_var") + + for k in remove_mc_set: + if k in out: + del out[k] + + self.update(out) + pass + + def average_double_ended( + self, + sections=None, + p_val="p_val", + p_cov="p_cov", + st_var=None, + ast_var=None, + rst_var=None, + rast_var=None, + conf_ints=None, + mc_sample_size=100, + ci_avg_time_flag1=False, + ci_avg_time_flag2=False, + ci_avg_time_sel=None, + ci_avg_time_isel=None, + ci_avg_x_flag1=False, + ci_avg_x_flag2=False, + ci_avg_x_sel=None, + ci_avg_x_isel=None, + da_random_state=None, + mc_remove_set_flag=True, + reduce_memory_usage=False, + **kwargs, + ): + """ + Average temperatures from double-ended setups. + + Four types of averaging are implemented. Please see Example Notebook 16. + + Parameters + ---------- + p_val : array-like, optional + Define `p_val`, `p_var`, `p_cov` if you used an external function + for calibration. Has size 2 + `nt`. First value is :math:`\\gamma`, + second is :math:`\\Delta \\alpha`, others are :math:`C` for each + timestep. + If set to False, no uncertainty in the parameters is propagated + into the confidence intervals. Similar to the spec sheets of the DTS + manufacturers. And similar to passing an array filled with zeros + p_cov : array-like, optional + The covariances of `p_val`. + st_var, ast_var, rst_var, rast_var : float, callable, array-like, optional + The variance of the measurement noise of the Stokes signals in the + forward direction. If `float` the variance of the noise from the + Stokes detector is described with a single value. + If `callable` the variance of the noise from the Stokes detector is + a function of the intensity, as defined in the callable function. + Or manually define a variance with a DataArray of the shape + `ds.st.shape`, where the variance can be a function of time and/or + x. Required if method is wls. + conf_ints : iterable object of float + A list with the confidence boundaries that are calculated. Valid + values are between + [0, 1]. + mc_sample_size : int + Size of the monte carlo parameter set used to calculate the + confidence interval + ci_avg_time_flag1 : bool + The confidence intervals differ each time step. Assumes the + temperature varies during the measurement period. Computes the + arithmic temporal mean. If you would like to know the confidence + interfal of: + (1) a single additional measurement. So you can state "if another + measurement were to be taken, it would have this ci" + (2) all measurements. So you can state "The temperature remained + during the entire measurement period between these ci bounds". + Adds "tmpw" + '_avg1' and "tmpw" + '_mc_avg1_var' to the + DataStore. If `conf_ints` are set, also the confidence intervals + `_mc_avg1` are added to the DataStore. Works independently of the + ci_avg_time_flag2 and ci_avg_x_flag. + ci_avg_time_flag2 : bool + The confidence intervals differ each time step. Assumes the + temperature remains constant during the measurement period. + Computes the inverse-variance-weighted-temporal-mean temperature + and its uncertainty. + If you would like to know the confidence interfal of: + (1) I want to estimate a background temperature with confidence + intervals. I hereby assume the temperature does not change over + time and average all measurements to get a better estimate of the + background temperature. + Adds "tmpw" + '_avg2' and "tmpw" + '_mc_avg2_var' to the + DataStore. If `conf_ints` are set, also the confidence intervals + `_mc_avg2` are added to the DataStore. Works independently of the + ci_avg_time_flag1 and ci_avg_x_flag. + ci_avg_time_sel : slice + Compute ci_avg_time_flag1 and ci_avg_time_flag2 using only a + selection of the data + ci_avg_time_isel : iterable of int + Compute ci_avg_time_flag1 and ci_avg_time_flag2 using only a + selection of the data + ci_avg_x_flag1 : bool + The confidence intervals differ at each location. Assumes the + temperature varies over `x` and over time. Computes the + arithmic spatial mean. If you would like to know the confidence + interfal of: + (1) a single additional measurement location. So you can state "if + another measurement location were to be taken, + it would have this ci" + (2) all measurement locations. So you can state "The temperature + along the fiber remained between these ci bounds". + Adds "tmpw" + '_avgx1' and "tmpw" + '_mc_avgx1_var' to the + DataStore. If `conf_ints` are set, also the confidence intervals + `_mc_avgx1` are added to the DataStore. Works independently of the + ci_avg_time_flag1, ci_avg_time_flag2 and ci_avg_x2_flag. + ci_avg_x_flag2 : bool + The confidence intervals differ at each location. Assumes the + temperature is the same at each location but varies over time. + Computes the inverse-variance-weighted-spatial-mean temperature + and its uncertainty. + If you would like to know the confidence interfal of: + (1) I have put a lot of fiber in water, and I know that the + temperature variation in the water is much smaller than along + other parts of the fiber. And I would like to average the + measurements from multiple locations to improve the estimated + temperature. + Adds "tmpw" + '_avg2' and "tmpw" + '_mc_avg2_var' to the + DataStore. If `conf_ints` are set, also the confidence intervals + `_mc_avg2` are added to the DataStore. Works independently of the + ci_avg_time_flag1 and ci_avg_x_flag. + ci_avg_x_sel : slice + Compute ci_avg_time_flag1 and ci_avg_time_flag2 using only a + selection of the data + ci_avg_x_isel : iterable of int + Compute ci_avg_time_flag1 and ci_avg_time_flag2 using only a + selection of the data + da_random_state + For testing purposes. Similar to random seed. The seed for dask. + Makes random not so random. To produce reproducable results for + testing environments. + mc_remove_set_flag : bool + Remove the monte carlo data set, from which the CI and the + variance are calculated. + reduce_memory_usage : bool + Use less memory but at the expense of longer computation time + + Returns + ------- + + """ + + def create_da_ta2(no, i_splice, direction="fw", chunks=None): + """create mask array mc, o, nt""" + + if direction == "fw": + arr = da.concatenate( + ( + da.zeros((1, i_splice, 1), chunks=(1, i_splice, 1), dtype=bool), + da.ones( + (1, no - i_splice, 1), + chunks=(1, no - i_splice, 1), + dtype=bool, + ), + ), + axis=1, + ).rechunk((1, chunks[1], 1)) + else: + arr = da.concatenate( + ( + da.ones((1, i_splice, 1), chunks=(1, i_splice, 1), dtype=bool), + da.zeros( + (1, no - i_splice, 1), + chunks=(1, no - i_splice, 1), + dtype=bool, + ), + ), + axis=1, + ).rechunk((1, chunks[1], 1)) + return arr + + check_deprecated_kwargs(kwargs) + + if (ci_avg_x_flag1 or ci_avg_x_flag2) and ( + ci_avg_time_flag1 or ci_avg_time_flag2 + ): + raise NotImplementedError( + "Incompatible flags. Can not pick " "the right chunks" + ) + + elif not ( + ci_avg_x_flag1 or ci_avg_x_flag2 or ci_avg_time_flag1 or ci_avg_time_flag2 + ): + raise NotImplementedError("Pick one of the averaging options") + + else: + pass + + out = xr.Dataset() + + mcparams = self.conf_int_double_ended( + sections=sections, + p_val=p_val, + p_cov=p_cov, + st_var=st_var, + ast_var=ast_var, + rst_var=rst_var, + rast_var=rast_var, + conf_ints=None, + mc_sample_size=mc_sample_size, + da_random_state=da_random_state, + mc_remove_set_flag=False, + reduce_memory_usage=reduce_memory_usage, + **kwargs, + ) + + for label in ["tmpf", "tmpb"]: + if ci_avg_time_sel is not None: + time_dim2 = "time" + "_avg" + x_dim2 = "x" + mcparams.coords[time_dim2] = ( + (time_dim2,), + mcparams["time"].sel(**{"time": ci_avg_time_sel}).data, + ) + mcparams[label + "_avgsec"] = ( + ("x", time_dim2), + self[label].sel(**{"time": ci_avg_time_sel}).data, + ) + mcparams[label + "_mc_set"] = ( + ("mc", "x", time_dim2), + mcparams[label + "_mc_set"].sel(**{"time": ci_avg_time_sel}).data, + ) + + elif ci_avg_time_isel is not None: + time_dim2 = "time" + "_avg" + x_dim2 = "x" + mcparams.coords[time_dim2] = ( + (time_dim2,), + mcparams["time"].isel(**{"time": ci_avg_time_isel}).data, + ) + mcparams[label + "_avgsec"] = ( + ("x", time_dim2), + self[label].isel(**{"time": ci_avg_time_isel}).data, + ) + mcparams[label + "_mc_set"] = ( + ("mc", "x", time_dim2), + mcparams[label + "_mc_set"].isel(**{"time": ci_avg_time_isel}).data, + ) + + elif ci_avg_x_sel is not None: + time_dim2 = "time" + x_dim2 = "x_avg" + mcparams.coords[x_dim2] = ( + (x_dim2,), + mcparams.x.sel(x=ci_avg_x_sel).data, + ) + mcparams[label + "_avgsec"] = ( + (x_dim2, "time"), + self[label].sel(x=ci_avg_x_sel).data, + ) + mcparams[label + "_mc_set"] = ( + ("mc", x_dim2, "time"), + mcparams[label + "_mc_set"].sel(x=ci_avg_x_sel).data, + ) + + elif ci_avg_x_isel is not None: + time_dim2 = "time" + x_dim2 = "x_avg" + mcparams.coords[x_dim2] = ( + (x_dim2,), + mcparams.x.isel(x=ci_avg_x_isel).data, + ) + mcparams[label + "_avgsec"] = ( + (x_dim2, time_dim2), + self[label].isel(x=ci_avg_x_isel).data, + ) + mcparams[label + "_mc_set"] = ( + ("mc", x_dim2, time_dim2), + mcparams[label + "_mc_set"].isel(x=ci_avg_x_isel).data, + ) + else: + mcparams[label + "_avgsec"] = self[label] + x_dim2 = "x" + time_dim2 = "time" + + memchunk = mcparams[label + "_mc_set"].chunks + + # subtract the mean temperature + q = mcparams[label + "_mc_set"] - mcparams[label + "_avgsec"] + out[label + "_mc" + "_avgsec_var"] = q.var(dim="mc", ddof=1) + + if ci_avg_x_flag1: + # unweighted mean + out[label + "_avgx1"] = mcparams[label + "_avgsec"].mean(dim=x_dim2) + + q = mcparams[label + "_mc_set"] - mcparams[label + "_avgsec"] + qvar = q.var(dim=["mc", x_dim2], ddof=1) + out[label + "_mc_avgx1_var"] = qvar + + if conf_ints: + new_chunks = (len(conf_ints), mcparams[label + "_mc_set"].chunks[2]) + avg_axis = mcparams[label + "_mc_set"].get_axis_num(["mc", x_dim2]) + q = mcparams[label + "_mc_set"].data.map_blocks( + lambda x: np.percentile(x, q=conf_ints, axis=avg_axis), + chunks=new_chunks, # + drop_axis=avg_axis, + # avg dimensions are dropped from input arr + new_axis=0, + ) # The new CI dim is added as firsaxis + + out[label + "_mc_avgx1"] = (("CI", time_dim2), q) + + if ci_avg_x_flag2: + q = mcparams[label + "_mc_set"] - mcparams[label + "_avgsec"] + + qvar = q.var(dim=["mc"], ddof=1) + + # Inverse-variance weighting + avg_x_var = 1 / (1 / qvar).sum(dim=x_dim2) + + out[label + "_mc_avgx2_var"] = avg_x_var + + mcparams[label + "_mc_avgx2_set"] = ( + mcparams[label + "_mc_set"] / qvar + ).sum(dim=x_dim2) * avg_x_var + out[label + "_avgx2"] = mcparams[label + "_mc_avgx2_set"].mean(dim="mc") + + if conf_ints: + new_chunks = (len(conf_ints), mcparams[label + "_mc_set"].chunks[2]) + avg_axis_avgx = mcparams[label + "_mc_set"].get_axis_num("mc") + + qq = mcparams[label + "_mc_avgx2_set"].data.map_blocks( + lambda x: np.percentile(x, q=conf_ints, axis=avg_axis_avgx), + chunks=new_chunks, # + drop_axis=avg_axis_avgx, + # avg dimensions are dropped from input arr + new_axis=0, + dtype=float, + ) # The new CI dimension is added as + # firsaxis + out[label + "_mc_avgx2"] = (("CI", time_dim2), qq) + + if ci_avg_time_flag1 is not None: + # unweighted mean + out[label + "_avg1"] = mcparams[label + "_avgsec"].mean(dim=time_dim2) + + q = mcparams[label + "_mc_set"] - mcparams[label + "_avgsec"] + qvar = q.var(dim=["mc", time_dim2], ddof=1) + out[label + "_mc_avg1_var"] = qvar + + if conf_ints: + new_chunks = (len(conf_ints), mcparams[label + "_mc_set"].chunks[1]) + avg_axis = mcparams[label + "_mc_set"].get_axis_num( + ["mc", time_dim2] + ) + q = mcparams[label + "_mc_set"].data.map_blocks( + lambda x: np.percentile(x, q=conf_ints, axis=avg_axis), + chunks=new_chunks, # + drop_axis=avg_axis, + # avg dimensions are dropped from input arr + new_axis=0, + ) # The new CI dim is added as firsaxis + + out[label + "_mc_avg1"] = (("CI", x_dim2), q) + + if ci_avg_time_flag2: + q = mcparams[label + "_mc_set"] - mcparams[label + "_avgsec"] + + qvar = q.var(dim=["mc"], ddof=1) + + # Inverse-variance weighting + avg_time_var = 1 / (1 / qvar).sum(dim=time_dim2) + + out[label + "_mc_avg2_var"] = avg_time_var + + mcparams[label + "_mc_avg2_set"] = ( + mcparams[label + "_mc_set"] / qvar + ).sum(dim=time_dim2) * avg_time_var + out[label + "_avg2"] = mcparams[label + "_mc_avg2_set"].mean(dim="mc") + + if conf_ints: + new_chunks = (len(conf_ints), mcparams[label + "_mc_set"].chunks[1]) + avg_axis_avg2 = mcparams[label + "_mc_set"].get_axis_num("mc") + + qq = mcparams[label + "_mc_avg2_set"].data.map_blocks( + lambda x: np.percentile(x, q=conf_ints, axis=avg_axis_avg2), + chunks=new_chunks, # + drop_axis=avg_axis_avg2, + # avg dimensions are dropped from input arr + new_axis=0, + dtype=float, + ) # The new CI dimension is added as + # firsaxis + out[label + "_mc_avg2"] = (("CI", x_dim2), qq) + + # Weighted mean of the forward and backward + tmpw_var = 1 / ( + 1 / out["tmpf_mc" + "_avgsec_var"] + 1 / out["tmpb_mc" + "_avgsec_var"] + ) + + q = ( + mcparams["tmpf_mc_set"] / out["tmpf_mc" + "_avgsec_var"] + + mcparams["tmpb_mc_set"] / out["tmpb_mc" + "_avgsec_var"] + ) * tmpw_var + + mcparams["tmpw" + "_mc_set"] = q # + + # out["tmpw"] = out["tmpw" + '_mc_set'].mean(dim='mc') + out["tmpw" + "_avgsec"] = ( + mcparams["tmpf_avgsec"] / out["tmpf_mc" + "_avgsec_var"] + + mcparams["tmpb_avgsec"] / out["tmpb_mc" + "_avgsec_var"] + ) * tmpw_var + + q = mcparams["tmpw" + "_mc_set"] - out["tmpw_avgsec"] + out["tmpw" + "_mc" + "_avgsec_var"] = q.var(dim="mc", ddof=1) + + if ci_avg_time_flag1: + out["tmpw" + "_avg1"] = out["tmpw" + "_avgsec"].mean(dim=time_dim2) + + out["tmpw" + "_mc_avg1_var"] = mcparams["tmpw" + "_mc_set"].var( + dim=["mc", time_dim2] + ) + + if conf_ints: + new_chunks_weighted = ((len(conf_ints),),) + (memchunk[1],) + avg_axis = mcparams["tmpw" + "_mc_set"].get_axis_num(["mc", time_dim2]) + q2 = mcparams["tmpw" + "_mc_set"].data.map_blocks( + lambda x: np.percentile(x, q=conf_ints, axis=avg_axis), + chunks=new_chunks_weighted, + # Explicitly define output chunks + drop_axis=avg_axis, # avg dimensions are dropped + new_axis=0, + dtype=float, + ) # The new CI dimension is added as + # first axis + out["tmpw" + "_mc_avg1"] = (("CI", x_dim2), q2) + + if ci_avg_time_flag2: + tmpw_var_avg2 = 1 / ( + 1 / out["tmpf_mc_avg2_var"] + 1 / out["tmpb_mc_avg2_var"] + ) + + q = ( + mcparams["tmpf_mc_avg2_set"] / out["tmpf_mc_avg2_var"] + + mcparams["tmpb_mc_avg2_set"] / out["tmpb_mc_avg2_var"] + ) * tmpw_var_avg2 + + mcparams["tmpw" + "_mc_avg2_set"] = q # + + out["tmpw" + "_avg2"] = ( + out["tmpf_avg2"] / out["tmpf_mc_avg2_var"] + + out["tmpb_avg2"] / out["tmpb_mc_avg2_var"] + ) * tmpw_var_avg2 + + out["tmpw" + "_mc_avg2_var"] = tmpw_var_avg2 + + if conf_ints: + # We first need to know the x-dim-chunk-size + new_chunks_weighted = ((len(conf_ints),),) + (memchunk[1],) + avg_axis_avg2 = mcparams["tmpw" + "_mc_avg2_set"].get_axis_num("mc") + q2 = mcparams["tmpw" + "_mc_avg2_set"].data.map_blocks( + lambda x: np.percentile(x, q=conf_ints, axis=avg_axis_avg2), + chunks=new_chunks_weighted, + # Explicitly define output chunks + drop_axis=avg_axis_avg2, # avg dimensions are dropped + new_axis=0, + dtype=float, + ) # The new CI dimension is added as firstax + out["tmpw" + "_mc_avg2"] = (("CI", x_dim2), q2) + + if ci_avg_x_flag1: + out["tmpw" + "_avgx1"] = out["tmpw" + "_avgsec"].mean(dim=x_dim2) + + out["tmpw" + "_mc_avgx1_var"] = mcparams["tmpw" + "_mc_set"].var(dim=x_dim2) + + if conf_ints: + new_chunks_weighted = ((len(conf_ints),),) + (memchunk[2],) + avg_axis = mcparams["tmpw" + "_mc_set"].get_axis_num(["mc", x_dim2]) + q2 = mcparams["tmpw" + "_mc_set"].data.map_blocks( + lambda x: np.percentile(x, q=conf_ints, axis=avg_axis), + chunks=new_chunks_weighted, + # Explicitly define output chunks + drop_axis=avg_axis, # avg dimensions are dropped + new_axis=0, + dtype=float, + ) # The new CI dimension is added as + # first axis + out["tmpw" + "_mc_avgx1"] = (("CI", time_dim2), q2) + + if ci_avg_x_flag2: + tmpw_var_avgx2 = 1 / ( + 1 / out["tmpf_mc_avgx2_var"] + 1 / out["tmpb_mc_avgx2_var"] + ) + + q = ( + mcparams["tmpf_mc_avgx2_set"] / out["tmpf_mc_avgx2_var"] + + mcparams["tmpb_mc_avgx2_set"] / out["tmpb_mc_avgx2_var"] + ) * tmpw_var_avgx2 + + mcparams["tmpw" + "_mc_avgx2_set"] = q # + + out["tmpw" + "_avgx2"] = ( + out["tmpf_avgx2"] / out["tmpf_mc_avgx2_var"] + + out["tmpb_avgx2"] / out["tmpb_mc_avgx2_var"] + ) * tmpw_var_avgx2 + + out["tmpw" + "_mc_avgx2_var"] = tmpw_var_avgx2 + + if conf_ints: + # We first need to know the x-dim-chunk-size + new_chunks_weighted = ((len(conf_ints),),) + (memchunk[2],) + avg_axis_avgx2 = mcparams["tmpw" + "_mc_avgx2_set"].get_axis_num("mc") + q2 = mcparams["tmpw" + "_mc_avgx2_set"].data.map_blocks( + lambda x: np.percentile(x, q=conf_ints, axis=avg_axis_avgx2), + chunks=new_chunks_weighted, + # Explicitly define output chunks + drop_axis=avg_axis_avgx2, # avg dimensions are dropped + new_axis=0, + dtype=float, + ) # The new CI dimension is added as firstax + out["tmpw" + "_mc_avgx2"] = (("CI", time_dim2), q2) + + # Clean up the garbage. All arrays with a Monte Carlo dimension. + if mc_remove_set_flag: + remove_mc_set = [ + "r_st", + "r_ast", + "r_rst", + "r_rast", + "gamma_mc", + "alpha_mc", + "df_mc", + "db_mc", + "x_avg", + "time_avg", + "mc", + ] + + for i in ["tmpf", "tmpb", "tmpw"]: + remove_mc_set.append(i + "_avgsec") + remove_mc_set.append(i + "_mc_set") + remove_mc_set.append(i + "_mc_avg2_set") + remove_mc_set.append(i + "_mc_avgx2_set") + remove_mc_set.append(i + "_mc_avgsec_var") + + if "trans_att" in mcparams and mcparams.trans_att.size: + remove_mc_set.append('talpha"_fw_mc') + remove_mc_set.append('talpha"_bw_mc') + + for k in remove_mc_set: + if k in out: + print(f"Removed from results: {k}") + del out[k] + + self.update(out) + pass diff --git a/tests/test_dtscalibration.py b/tests/test_dtscalibration.py index 1ff7a32a..4dd0fed3 100644 --- a/tests/test_dtscalibration.py +++ b/tests/test_dtscalibration.py @@ -58,689 +58,6 @@ def assert_almost_equal_verbose(actual, desired, verbose=False, **kwargs): pass -@pytest.mark.slow # Execution time ~20 seconds -def test_variance_input_types_single(): - import dask.array as da - - state = da.random.RandomState(0) - - stokes_m_var = 40.0 - cable_len = 100.0 - nt = 500 - time = np.arange(nt) - x = np.linspace(0.0, cable_len, 100) - ts_cold = np.ones(nt) * 4.0 - ts_warm = np.ones(nt) * 20.0 - - C_p = 15246 - C_m = 2400.0 - dalpha_r = 0.005284 - dalpha_m = 0.004961 - dalpha_p = 0.005607 - gamma = 482.6 - cold_mask = x < 0.5 * cable_len - warm_mask = np.invert(cold_mask) # == False - temp_real = np.ones((len(x), nt)) - temp_real[cold_mask] *= ts_cold + 273.15 - temp_real[warm_mask] *= ts_warm + 273.15 - - st = ( - C_p - * np.exp(-dalpha_r * x[:, None]) - * np.exp(-dalpha_p * x[:, None]) - * np.exp(-gamma / temp_real) - / (1 - np.exp(-gamma / temp_real)) - ) - ast = ( - C_m - * np.exp(-dalpha_r * x[:, None]) - * np.exp(-dalpha_m * x[:, None]) - / (1 - np.exp(-gamma / temp_real)) - ) - - st_m = st + stats.norm.rvs(size=st.shape, scale=stokes_m_var**0.5) - ast_m = ast + stats.norm.rvs(size=ast.shape, scale=1.1 * stokes_m_var**0.5) - - print("alphaint", cable_len * (dalpha_p - dalpha_m)) - print("alpha", dalpha_p - dalpha_m) - print("C", np.log(C_p / C_m)) - print("x0", x.max()) - - ds = Dataset( - { - "st": (["x", "time"], st_m), - "ast": (["x", "time"], ast_m), - "userAcquisitionTimeFW": (["time"], np.ones(nt)), - "cold": (["time"], ts_cold), - "warm": (["time"], ts_warm), - }, - coords={"x": x, "time": time}, - attrs={"isDoubleEnded": "0"}, - ) - - sections = { - "cold": [slice(0.0, 0.4 * cable_len)], - "warm": [slice(0.6 * cable_len, cable_len)], - } - - # Test float input - st_var = 5.0 - - ds.dts.calibrate_single_ended( - sections=sections, st_var=st_var, ast_var=st_var, method="wls", solver="sparse" - ) - - ds.dts.monte_carlo_single_ended( - st_var=st_var, - ast_var=st_var, - mc_sample_size=100, - da_random_state=state, - mc_remove_set_flag=False, - ) - - assert_almost_equal_verbose( - ds.tmpf_mc_var.sel(x=slice(0, 10)).mean(), 0.044361, decimal=2 - ) - assert_almost_equal_verbose( - ds.tmpf_mc_var.sel(x=slice(90, 100)).mean(), 0.242028, decimal=2 - ) - - # Test callable input - def callable_st_var(stokes): - slope = 0.01 - offset = 0 - return slope * stokes + offset - - ds.dts.calibrate_single_ended( - sections=sections, - st_var=callable_st_var, - ast_var=callable_st_var, - method="wls", - solver="sparse", - ) - - ds.monte_carlo_single_ended( - st_var=callable_st_var, - ast_var=callable_st_var, - mc_sample_size=100, - da_random_state=state, - ) - - assert_almost_equal_verbose( - ds.tmpf_mc_var.sel(x=slice(0, 10)).mean(), 0.184753, decimal=2 - ) - assert_almost_equal_verbose( - ds.tmpf_mc_var.sel(x=slice(90, 100)).mean(), 0.545186, decimal=2 - ) - - # Test input with shape of (ntime, nx) - st_var = ds.st.values * 0 + 20.0 - ds.dts.calibrate_single_ended( - sections=sections, st_var=st_var, ast_var=st_var, method="wls", solver="sparse" - ) - - ds.dts.monte_carlo_single_ended( - st_var=st_var, ast_var=st_var, mc_sample_size=100, da_random_state=state - ) - - assert_almost_equal_verbose(ds.tmpf_mc_var.mean(), 0.418098, decimal=2) - - # Test input with shape (nx, 1) - st_var = np.vstack( - ds.st.mean(dim="time").values * 0 + np.linspace(10, 50, num=ds.st.x.size) - ) - - ds.dts.calibrate_single_ended( - sections=sections, st_var=st_var, ast_var=st_var, method="wls", solver="sparse" - ) - - ds.dts.monte_carlo_single_ended( - st_var=st_var, ast_var=st_var, mc_sample_size=100, da_random_state=state - ) - - assert_almost_equal_verbose( - ds.tmpf_mc_var.sel(x=slice(0, 50)).mean().values, 0.2377, decimal=2 - ) - assert_almost_equal_verbose( - ds.tmpf_mc_var.sel(x=slice(50, 100)).mean().values, 1.3203, decimal=2 - ) - - # Test input with shape (ntime) - st_var = ds.st.mean(dim="x").values * 0 + np.linspace(5, 200, num=nt) - - ds.dts.calibrate_single_ended( - sections=sections, st_var=st_var, ast_var=st_var, method="wls", solver="sparse" - ) - - ds.dts.monte_carlo_single_ended( - st_var=st_var, ast_var=st_var, mc_sample_size=100, da_random_state=state - ) - - assert_almost_equal_verbose( - ds.tmpf_mc_var.sel(time=slice(0, nt // 2)).mean().values, 1.0908, decimal=2 - ) - assert_almost_equal_verbose( - ds.tmpf_mc_var.sel(time=slice(nt // 2, None)).mean().values, 3.0759, decimal=2 - ) - - pass - - -@pytest.mark.slow # Execution time ~0.5 minute -def test_variance_input_types_double(): - import dask.array as da - - state = da.random.RandomState(0) - - stokes_m_var = 40.0 - cable_len = 100.0 - nt = 500 - time = np.arange(nt) - x = np.linspace(0.0, cable_len, 100) - ts_cold = np.ones(nt) * 4.0 - ts_warm = np.ones(nt) * 20.0 - - C_p = 15246 - C_m = 2400.0 - dalpha_r = 0.005284 - dalpha_m = 0.004961 - dalpha_p = 0.005607 - gamma = 482.6 - cold_mask = x < 0.5 * cable_len - warm_mask = np.invert(cold_mask) # == False - temp_real = np.ones((len(x), nt)) - temp_real[cold_mask] *= ts_cold + 273.15 - temp_real[warm_mask] *= ts_warm + 273.15 - - st = ( - C_p - * np.exp(-dalpha_r * x[:, None]) - * np.exp(-dalpha_p * x[:, None]) - * np.exp(-gamma / temp_real) - / (1 - np.exp(-gamma / temp_real)) - ) - ast = ( - C_m - * np.exp(-dalpha_r * x[:, None]) - * np.exp(-dalpha_m * x[:, None]) - / (1 - np.exp(-gamma / temp_real)) - ) - rst = ( - C_p - * np.exp(-dalpha_r * (-x[:, None] + 100)) - * np.exp(-dalpha_p * (-x[:, None] + 100)) - * np.exp(-gamma / temp_real) - / (1 - np.exp(-gamma / temp_real)) - ) - rast = ( - C_m - * np.exp(-dalpha_r * (-x[:, None] + 100)) - * np.exp(-dalpha_m * (-x[:, None] + 100)) - / (1 - np.exp(-gamma / temp_real)) - ) - - st_m = st + stats.norm.rvs(size=st.shape, scale=stokes_m_var**0.5) - ast_m = ast + stats.norm.rvs(size=ast.shape, scale=1.1 * stokes_m_var**0.5) - rst_m = rst + stats.norm.rvs(size=rst.shape, scale=0.9 * stokes_m_var**0.5) - rast_m = rast + stats.norm.rvs(size=rast.shape, scale=0.8 * stokes_m_var**0.5) - - print("alphaint", cable_len * (dalpha_p - dalpha_m)) - print("alpha", dalpha_p - dalpha_m) - print("C", np.log(C_p / C_m)) - print("x0", x.max()) - - ds = Dataset( - { - "st": (["x", "time"], st_m), - "ast": (["x", "time"], ast_m), - "rst": (["x", "time"], rst_m), - "rast": (["x", "time"], rast_m), - "userAcquisitionTimeFW": (["time"], np.ones(nt)), - "userAcquisitionTimeBW": (["time"], np.ones(nt)), - "cold": (["time"], ts_cold), - "warm": (["time"], ts_warm), - }, - coords={"x": x, "time": time}, - attrs={"isDoubleEnded": "1"}, - ) - - sections = { - "cold": [slice(0.0, 0.4 * cable_len)], - "warm": [slice(0.6 * cable_len, cable_len)], - } - - # Test float input - st_var = 5.0 - - ds.dts.calibration_double_ended( - sections=sections, - st_var=st_var, - ast_var=st_var, - rst_var=st_var, - rast_var=st_var, - method="wls", - solver="sparse", - ) - - ds.conf_int_double_ended( - sections=sections, - st_var=st_var, - ast_var=st_var, - rst_var=st_var, - rast_var=st_var, - mc_sample_size=100, - da_random_state=state, - ) - - assert_almost_equal_verbose( - ds.tmpf_mc_var.sel(x=slice(0, 10)).mean(), 0.03584935, decimal=2 - ) - assert_almost_equal_verbose( - ds.tmpf_mc_var.sel(x=slice(90, 100)).mean(), 0.22982146, decimal=2 - ) - - # Test callable input - def st_var_callable(stokes): - slope = 0.01 - offset = 0 - return slope * stokes + offset - - ds.dts.calibration_double_ended( - sections=sections, - st_var=st_var_callable, - ast_var=st_var_callable, - rst_var=st_var_callable, - rast_var=st_var_callable, - method="wls", - solver="sparse", - ) - - ds.conf_int_double_ended( - sections=sections, - st_var=st_var_callable, - ast_var=st_var_callable, - rst_var=st_var_callable, - rast_var=st_var_callable, - mc_sample_size=100, - da_random_state=state, - ) - - assert_almost_equal_verbose( - ds.tmpf_mc_var.sel(x=slice(0, 10)).mean(), 0.18058514, decimal=2 - ) - assert_almost_equal_verbose( - ds.tmpf_mc_var.sel(x=slice(90, 100)).mean(), 0.53862813, decimal=2 - ) - - # Test input with shape of (ntime, nx) - st_var = ds.st.values * 0 + 20.0 - - ds.dts.calibration_double_ended( - sections=sections, - st_var=st_var, - ast_var=st_var, - rst_var=st_var, - rast_var=st_var, - method="wls", - solver="sparse", - ) - - ds.conf_int_double_ended( - sections=sections, - st_var=st_var, - ast_var=st_var, - rst_var=st_var, - rast_var=st_var, - mc_sample_size=100, - da_random_state=state, - ) - - assert_almost_equal_verbose(ds.tmpf_mc_var.mean(), 0.40725674, decimal=2) - - # Test input with shape (nx, 1) - st_var = np.vstack( - ds.st.mean(dim="time").values * 0 + np.linspace(10, 50, num=ds.st.x.size) - ) - - ds.dts.calibration_double_ended( - sections=sections, - st_var=st_var, - ast_var=st_var, - rst_var=st_var, - rast_var=st_var, - method="wls", - solver="sparse", - ) - - ds.conf_int_double_ended( - sections=sections, - st_var=st_var, - ast_var=st_var, - rst_var=st_var, - rast_var=st_var, - mc_sample_size=100, - da_random_state=state, - ) - - assert_almost_equal_verbose( - ds.tmpf_mc_var.sel(x=slice(0, 50)).mean().values, 0.21163704, decimal=2 - ) - assert_almost_equal_verbose( - ds.tmpf_mc_var.sel(x=slice(50, 100)).mean().values, 1.28247762, decimal=2 - ) - - # Test input with shape (ntime) - st_var = ds.st.mean(dim="x").values * 0 + np.linspace(5, 200, num=nt) - - ds.dts.calibration_double_ended( - sections=sections, - st_var=st_var, - ast_var=st_var, - rst_var=st_var, - rast_var=st_var, - method="wls", - solver="sparse", - ) - - ds.conf_int_double_ended( - sections=sections, - st_var=st_var, - ast_var=st_var, - rst_var=st_var, - rast_var=st_var, - mc_sample_size=100, - da_random_state=state, - ) - - assert_almost_equal_verbose( - ds.tmpf_mc_var.sel(time=slice(0, nt // 2)).mean().values, 1.090, decimal=2 - ) - assert_almost_equal_verbose( - ds.tmpf_mc_var.sel(time=slice(nt // 2, None)).mean().values, 3.06, decimal=2 - ) - - pass - - -@pytest.mark.slow # Execution time ~0.5 minute -def test_double_ended_variance_estimate_synthetic(): - import dask.array as da - - state = da.random.RandomState(0) - - stokes_m_var = 40.0 - cable_len = 100.0 - nt = 500 - time = np.arange(nt) - x = np.linspace(0.0, cable_len, 100) - ts_cold = np.ones(nt) * 4.0 - ts_warm = np.ones(nt) * 20.0 - - C_p = 15246 - C_m = 2400.0 - dalpha_r = 0.0005284 - dalpha_m = 0.0004961 - dalpha_p = 0.0005607 - gamma = 482.6 - cold_mask = x < 0.5 * cable_len - warm_mask = np.invert(cold_mask) # == False - temp_real = np.ones((len(x), nt)) - temp_real[cold_mask] *= ts_cold + 273.15 - temp_real[warm_mask] *= ts_warm + 273.15 - - st = ( - C_p - * np.exp(-dalpha_r * x[:, None]) - * np.exp(-dalpha_p * x[:, None]) - * np.exp(-gamma / temp_real) - / (1 - np.exp(-gamma / temp_real)) - ) - ast = ( - C_m - * np.exp(-dalpha_r * x[:, None]) - * np.exp(-dalpha_m * x[:, None]) - / (1 - np.exp(-gamma / temp_real)) - ) - rst = ( - C_p - * np.exp(-dalpha_r * (-x[:, None] + 100)) - * np.exp(-dalpha_p * (-x[:, None] + 100)) - * np.exp(-gamma / temp_real) - / (1 - np.exp(-gamma / temp_real)) - ) - rast = ( - C_m - * np.exp(-dalpha_r * (-x[:, None] + 100)) - * np.exp(-dalpha_m * (-x[:, None] + 100)) - / (1 - np.exp(-gamma / temp_real)) - ) - - st_m = st + stats.norm.rvs(size=st.shape, scale=stokes_m_var**0.5) - ast_m = ast + stats.norm.rvs(size=ast.shape, scale=1.1 * stokes_m_var**0.5) - rst_m = rst + stats.norm.rvs(size=rst.shape, scale=0.9 * stokes_m_var**0.5) - rast_m = rast + stats.norm.rvs(size=rast.shape, scale=0.8 * stokes_m_var**0.5) - - print("alphaint", cable_len * (dalpha_p - dalpha_m)) - print("alpha", dalpha_p - dalpha_m) - print("C", np.log(C_p / C_m)) - print("x0", x.max()) - - ds = Dataset( - { - "st": (["x", "time"], st_m), - "ast": (["x", "time"], ast_m), - "rst": (["x", "time"], rst_m), - "rast": (["x", "time"], rast_m), - "userAcquisitionTimeFW": (["time"], np.ones(nt)), - "userAcquisitionTimeBW": (["time"], np.ones(nt)), - "cold": (["time"], ts_cold), - "warm": (["time"], ts_warm), - }, - coords={"x": x, "time": time}, - attrs={"isDoubleEnded": "1"}, - ) - - sections = { - "cold": [slice(0.0, 0.5 * cable_len)], - "warm": [slice(0.5 * cable_len, cable_len)], - } - - mst_var, _ = ds.variance_stokes(st_label="st", sections=sections) - mast_var, _ = ds.variance_stokes(st_label="ast", sections=sections) - mrst_var, _ = ds.variance_stokes(st_label="rst", sections=sections) - mrast_var, _ = ds.variance_stokes(st_label="rast", sections=sections) - - mst_var = float(mst_var) - mast_var = float(mast_var) - mrst_var = float(mrst_var) - mrast_var = float(mrast_var) - - # MC variance - ds.dts.calibration_double_ended( - sections=sections, - st_var=mst_var, - ast_var=mast_var, - rst_var=mrst_var, - rast_var=mrast_var, - method="wls", - solver="sparse", - ) - - assert_almost_equal_verbose(ds.tmpf.mean(), 12.0, decimal=2) - assert_almost_equal_verbose(ds.tmpb.mean(), 12.0, decimal=3) - - ds.conf_int_double_ended( - sections=sections, - p_val="p_val", - p_cov="p_cov", - st_var=mst_var, - ast_var=mast_var, - rst_var=mrst_var, - rast_var=mrast_var, - conf_ints=[2.5, 50.0, 97.5], - mc_sample_size=100, - da_random_state=state, - ) - - # Calibrated variance - stdsf1 = ds.ufunc_per_section( - sections=sections, label="tmpf", func=np.std, temp_err=True, calc_per="stretch" - ) - stdsb1 = ds.ufunc_per_section( - sections=sections, label="tmpb", func=np.std, temp_err=True, calc_per="stretch" - ) - - # Use a single timestep to better check if the parameter uncertainties propagate - ds1 = ds.isel(time=1) - # Estimated VAR - stdsf2 = ds1.ufunc_per_section( - sections=sections, - label="tmpf_mc_var", - func=np.mean, - temp_err=False, - calc_per="stretch", - ) - stdsb2 = ds1.ufunc_per_section( - sections=sections, - label="tmpb_mc_var", - func=np.mean, - temp_err=False, - calc_per="stretch", - ) - - for (_, v1), (_, v2) in zip(stdsf1.items(), stdsf2.items()): - for v1i, v2i in zip(v1, v2): - print("Real VAR: ", v1i**2, "Estimated VAR: ", float(v2i)) - assert_almost_equal_verbose(v1i**2, v2i, decimal=2) - - for (_, v1), (_, v2) in zip(stdsb1.items(), stdsb2.items()): - for v1i, v2i in zip(v1, v2): - print("Real VAR: ", v1i**2, "Estimated VAR: ", float(v2i)) - assert_almost_equal_verbose(v1i**2, v2i, decimal=2) - - pass - - -def test_single_ended_variance_estimate_synthetic(): - import dask.array as da - - state = da.random.RandomState(0) - - stokes_m_var = 40.0 - astokes_m_var = 60.0 - cable_len = 100.0 - nt = 50 - time = np.arange(nt) - x = np.linspace(0.0, cable_len, 500) - ts_cold = np.ones(nt) * 4.0 - ts_warm = np.ones(nt) * 20.0 - - C_p = 15246 - C_m = 2400.0 - dalpha_r = 0.0005284 - dalpha_m = 0.0004961 - dalpha_p = 0.0005607 - gamma = 482.6 - cold_mask = x < 0.5 * cable_len - warm_mask = np.invert(cold_mask) # == False - temp_real = np.ones((len(x), nt)) - temp_real[cold_mask] *= ts_cold + 273.15 - temp_real[warm_mask] *= ts_warm + 273.15 - - st = ( - C_p - * np.exp(-dalpha_r * x[:, None]) - * np.exp(-dalpha_p * x[:, None]) - * np.exp(-gamma / temp_real) - / (1 - np.exp(-gamma / temp_real)) - ) - ast = ( - C_m - * np.exp(-dalpha_r * x[:, None]) - * np.exp(-dalpha_m * x[:, None]) - / (1 - np.exp(-gamma / temp_real)) - ) - st_m = st + stats.norm.rvs(size=st.shape, scale=stokes_m_var**0.5) - ast_m = ast + stats.norm.rvs(size=ast.shape, scale=astokes_m_var**0.5) - - print("alphaint", cable_len * (dalpha_p - dalpha_m)) - print("alpha", dalpha_p - dalpha_m) - print("C", np.log(C_p / C_m)) - print("x0", x.max()) - - ds = Dataset( - { - "st": (["x", "time"], st_m), - "ast": (["x", "time"], ast_m), - "userAcquisitionTimeFW": (["time"], np.ones(nt)), - "cold": (["time"], ts_cold), - "warm": (["time"], ts_warm), - }, - coords={"x": x, "time": time}, - attrs={"isDoubleEnded": "0"}, - ) - - sections = { - "cold": [slice(0.0, 0.5 * cable_len)], - "warm": [slice(0.5 * cable_len, cable_len)], - } - - st_label = "st" - ast_label = "ast" - - mst_var, _ = ds.variance_stokes(st_label=st_label, sections=sections) - mast_var, _ = ds.variance_stokes(st_label=ast_label, sections=sections) - mst_var = float(mst_var) - mast_var = float(mast_var) - - # MC variqnce - ds.dts.calibrate_single_ended( - sections=sections, - st_var=mst_var, - ast_var=mast_var, - method="wls", - solver="sparse", - ) - - ds.dts.monte_carlo_single_ended( - p_val="p_val", - p_cov="p_cov", - st_var=mst_var, - ast_var=mast_var, - conf_ints=[2.5, 50.0, 97.5], - mc_sample_size=50, - da_random_state=state, - ) - - # Calibrated variance - stdsf1 = ds.ufunc_per_section( - sections=sections, - label="tmpf", - func=np.std, - temp_err=True, - calc_per="stretch", - ddof=1, - ) - - # Use a single timestep to better check if the parameter uncertainties propagate - ds1 = ds.isel(time=1) - # Estimated VAR - stdsf2 = ds1.ufunc_per_section( - sections=sections, - label="tmpf_mc_var", - func=np.mean, - temp_err=False, - calc_per="stretch", - ) - - for (_, v1), (_, v2) in zip(stdsf1.items(), stdsf2.items()): - for v1i, v2i in zip(v1, v2): - print("Real VAR: ", v1i**2, "Estimated VAR: ", float(v2i)) - assert_almost_equal_verbose(v1i**2, v2i, decimal=2) - - pass - - def test_double_ended_wls_estimate_synthetic(): """Checks whether the coefficients are correctly defined by creating a synthetic measurement set, and derive the parameters from this set. @@ -813,7 +130,7 @@ def test_double_ended_wls_estimate_synthetic(): } # WLS - ds.dts.calibration_double_ended( + out = ds.dts.calibration_double_ended( sections=sections, st_var=1e-7, ast_var=1e-7, @@ -823,11 +140,11 @@ def test_double_ended_wls_estimate_synthetic(): solver="sparse", ) - assert_almost_equal_verbose(ds.gamma.values, gamma, decimal=10) - assert_almost_equal_verbose(ds.alpha.values, alpha, decimal=8) - assert_almost_equal_verbose(ds.tmpf.values, temp_real - 273.15, decimal=6) - assert_almost_equal_verbose(ds.tmpb.values, temp_real - 273.15, decimal=6) - assert_almost_equal_verbose(ds.tmpw.values, temp_real - 273.15, decimal=6) + assert_almost_equal_verbose(out["gamma"].values, gamma, decimal=10) + assert_almost_equal_verbose(out["alpha"].values, alpha, decimal=8) + assert_almost_equal_verbose(out["tmpf"].values, temp_real - 273.15, decimal=6) + assert_almost_equal_verbose(out["tmpb"].values, temp_real - 273.15, decimal=6) + assert_almost_equal_verbose(out["tmpw"].values, temp_real - 273.15, decimal=6) def test_double_ended_wls_estimate_synthetic_df_and_db_are_different(): @@ -927,7 +244,7 @@ def test_double_ended_wls_estimate_synthetic_df_and_db_are_different(): real_ans2 = np.concatenate(([gamma], df, db, E_real[:, 0])) - ds.dts.calibration_double_ended( + out = ds.dts.calibration_double_ended( sections=sections, st_var=1.5, ast_var=1.5, @@ -938,15 +255,15 @@ def test_double_ended_wls_estimate_synthetic_df_and_db_are_different(): fix_gamma=(gamma, 0.0), ) - assert_almost_equal_verbose(df, ds.df.values, decimal=14) - assert_almost_equal_verbose(db, ds.db.values, decimal=13) + assert_almost_equal_verbose(df, out["df"].values, decimal=14) + assert_almost_equal_verbose(db, out["db"].values, decimal=13) assert_almost_equal_verbose( - x * (dalpha_p - dalpha_m), ds.alpha.values - ds.alpha.values[0], decimal=13 + x * (dalpha_p - dalpha_m), out["alpha"].values - out["alpha"].values[0], decimal=13 ) - assert np.all(np.abs(real_ans2 - ds.p_val.values) < 1e-10) - assert_almost_equal_verbose(temp_real_celsius, ds.tmpf.values, decimal=10) - assert_almost_equal_verbose(temp_real_celsius, ds.tmpb.values, decimal=10) - assert_almost_equal_verbose(temp_real_celsius, ds.tmpw.values, decimal=10) + assert np.all(np.abs(real_ans2 - out["p_val"].values) < 1e-10) + assert_almost_equal_verbose(temp_real_celsius, out["tmpf"].values, decimal=10) + assert_almost_equal_verbose(temp_real_celsius, out["tmpb"].values, decimal=10) + assert_almost_equal_verbose(temp_real_celsius, out["tmpw"].values, decimal=10) pass @@ -1127,7 +444,6 @@ def test_double_ended_asymmetrical_attenuation(): "warm": (["time"], ts_warm), }, coords={"x": x, "time": time}, - attrs={"isDoubleEnded": "1"}, ) sections = { @@ -1138,37 +454,7 @@ def test_double_ended_asymmetrical_attenuation(): ], } - ds.dts.calibration_double_ended( - sections=sections, - st_var=1.5, - ast_var=1.5, - rst_var=1.0, - rast_var=1.0, - method="wls", - solver="sparse", - trans_att=[50.0], - ) - - assert_almost_equal_verbose(temp_real_celsius, ds.tmpf.values, decimal=7) - assert_almost_equal_verbose(temp_real_celsius, ds.tmpb.values, decimal=7) - assert_almost_equal_verbose(temp_real_celsius, ds.tmpw.values, decimal=7) - - # test `trans_att` related functions - - # Clear out old results - ds.set_trans_att([]) - - assert ds.trans_att.size == 0, "clear out trans_att config" - - del_keys = [] - for k, v in ds.data_vars.items(): - if "trans_att" in v.dims: - del_keys.append(k) - - assert len(del_keys) == 0, "clear out trans_att config" - - # About to be depreciated - ds.dts.calibration_double_ended( + out = ds.dts.calibration_double_ended( sections=sections, st_var=1.5, ast_var=1.5, @@ -1179,9 +465,9 @@ def test_double_ended_asymmetrical_attenuation(): trans_att=[50.0], ) - assert_almost_equal_verbose(temp_real_celsius, ds.tmpf.values, decimal=7) - assert_almost_equal_verbose(temp_real_celsius, ds.tmpb.values, decimal=7) - assert_almost_equal_verbose(temp_real_celsius, ds.tmpw.values, decimal=7) + assert_almost_equal_verbose(temp_real_celsius, out.tmpf.values, decimal=7) + assert_almost_equal_verbose(temp_real_celsius, out.tmpb.values, decimal=7) + assert_almost_equal_verbose(temp_real_celsius, out.tmpw.values, decimal=7) pass @@ -1271,7 +557,7 @@ def test_double_ended_one_matching_section_and_one_asym_att(): "warm": [slice(x[nx_per_sec], x[2 * nx_per_sec - 1])], } - ds.dts.calibration_double_ended( + out = ds.dts.calibration_double_ended( sections=sections, st_var=1.5, ast_var=1.5, @@ -1289,9 +575,9 @@ def test_double_ended_one_matching_section_and_one_asym_att(): ], ) - assert_almost_equal_verbose(temp_real_celsius, ds.tmpf.values, decimal=7) - assert_almost_equal_verbose(temp_real_celsius, ds.tmpb.values, decimal=7) - assert_almost_equal_verbose(temp_real_celsius, ds.tmpw.values, decimal=7) + assert_almost_equal_verbose(temp_real_celsius, out["tmpf"].values, decimal=7) + assert_almost_equal_verbose(temp_real_celsius, out["tmpb"].values, decimal=7) + assert_almost_equal_verbose(temp_real_celsius, out["tmpw"].values, decimal=7) def test_double_ended_two_matching_sections_and_two_asym_atts(): @@ -1401,7 +687,7 @@ def test_double_ended_two_matching_sections_and_two_asym_atts(): ), ] - ds.dts.calibration_double_ended( + out = ds.dts.calibration_double_ended( sections=sections, st_var=0.5, ast_var=0.5, @@ -1413,9 +699,9 @@ def test_double_ended_two_matching_sections_and_two_asym_atts(): matching_sections=ms, ) - assert_almost_equal_verbose(temp_real_celsius, ds.tmpf.values, decimal=7) - assert_almost_equal_verbose(temp_real_celsius, ds.tmpb.values, decimal=7) - assert_almost_equal_verbose(temp_real_celsius, ds.tmpw.values, decimal=7) + assert_almost_equal_verbose(temp_real_celsius, out["tmpf"].values, decimal=7) + assert_almost_equal_verbose(temp_real_celsius, out["tmpb"].values, decimal=7) + assert_almost_equal_verbose(temp_real_celsius, out["tmpw"].values, decimal=7) pass @@ -1497,7 +783,7 @@ def test_double_ended_wls_fix_gamma_estimate_synthetic(): } # WLS - ds.dts.calibration_double_ended( + out = ds.dts.calibration_double_ended( sections=sections, st_var=1e-12, ast_var=1e-12, @@ -1508,11 +794,11 @@ def test_double_ended_wls_fix_gamma_estimate_synthetic(): fix_gamma=(gamma, 0.0), ) - assert_almost_equal_verbose(ds.gamma.values, gamma, decimal=18) - assert_almost_equal_verbose(ds.alpha.values, alpha, decimal=9) - assert_almost_equal_verbose(ds.tmpf.values, temp_real - 273.15, decimal=6) - assert_almost_equal_verbose(ds.tmpb.values, temp_real - 273.15, decimal=6) - assert_almost_equal_verbose(ds.tmpw.values, temp_real - 273.15, decimal=6) + assert_almost_equal_verbose(out["gamma"].values, gamma, decimal=18) + assert_almost_equal_verbose(out["alpha"].values, alpha, decimal=9) + assert_almost_equal_verbose(out["tmpf"].values, temp_real - 273.15, decimal=6) + assert_almost_equal_verbose(out["tmpb"].values, temp_real - 273.15, decimal=6) + assert_almost_equal_verbose(out["tmpw"].values, temp_real - 273.15, decimal=6) pass @@ -1590,7 +876,7 @@ def test_double_ended_wls_fix_alpha_estimate_synthetic(): } # WLS - ds.dts.calibration_double_ended( + out = ds.dts.calibration_double_ended( sections=sections, st_var=1e-7, ast_var=1e-7, @@ -1601,11 +887,11 @@ def test_double_ended_wls_fix_alpha_estimate_synthetic(): fix_alpha=(alpha, np.zeros_like(alpha)), ) - assert_almost_equal_verbose(ds.gamma.values, gamma, decimal=8) - assert_almost_equal_verbose(ds.alpha.values, alpha, decimal=18) - assert_almost_equal_verbose(ds.tmpf.values, temp_real - 273.15, decimal=7) - assert_almost_equal_verbose(ds.tmpb.values, temp_real - 273.15, decimal=7) - assert_almost_equal_verbose(ds.tmpw.values, temp_real - 273.15, decimal=7) + assert_almost_equal_verbose(out["gamma"].values, gamma, decimal=8) + assert_almost_equal_verbose(out["alpha"].values, alpha, decimal=18) + assert_almost_equal_verbose(out["tmpf"].values, temp_real - 273.15, decimal=7) + assert_almost_equal_verbose(out["tmpb"].values, temp_real - 273.15, decimal=7) + assert_almost_equal_verbose(out["tmpw"].values, temp_real - 273.15, decimal=7) pass @@ -1683,7 +969,7 @@ def test_double_ended_wls_fix_alpha_fix_gamma_estimate_synthetic(): } # WLS - ds.dts.calibration_double_ended( + out = ds.dts.calibration_double_ended( sections=sections, st_var=1e-7, ast_var=1e-7, @@ -1695,11 +981,11 @@ def test_double_ended_wls_fix_alpha_fix_gamma_estimate_synthetic(): fix_alpha=(alpha, np.zeros_like(alpha)), ) - assert_almost_equal_verbose(ds.gamma.values, gamma, decimal=18) - assert_almost_equal_verbose(ds.alpha.values, alpha, decimal=18) - assert_almost_equal_verbose(ds.tmpf.values, temp_real - 273.15, decimal=11) - assert_almost_equal_verbose(ds.tmpb.values, temp_real - 273.15, decimal=11) - assert_almost_equal_verbose(ds.tmpw.values, temp_real - 273.15, decimal=11) + assert_almost_equal_verbose(out["gamma"].values, gamma, decimal=18) + assert_almost_equal_verbose(out["alpha"].values, alpha, decimal=18) + assert_almost_equal_verbose(out["tmpf"].values, temp_real - 273.15, decimal=11) + assert_almost_equal_verbose(out["tmpb"].values, temp_real - 273.15, decimal=11) + assert_almost_equal_verbose(out["tmpw"].values, temp_real - 273.15, decimal=11) pass @@ -1790,7 +1076,7 @@ def test_double_ended_fix_alpha_matching_sections_and_one_asym_att(): "warm": [slice(x[nx_per_sec], x[2 * nx_per_sec - 1])], } - ds.dts.calibration_double_ended( + out = ds.dts.calibration_double_ended( sections=sections, st_var=1.5, ast_var=1.5, @@ -1812,12 +1098,12 @@ def test_double_ended_fix_alpha_matching_sections_and_one_asym_att(): k = ["talpha_fw", "talpha_bw", "trans_att"] for ki in k: - del ds[ki] + del out[ki] - alpha_adj = ds.alpha.values.copy() - alpha_var_adj = ds.alpha_var.values.copy() + alpha_adj = out["alpha"].values.copy() + alpha_var_adj = out["alpha_var"].values.copy() - ds.dts.calibration_double_ended( + out = ds.dts.calibration_double_ended( sections=sections, st_var=1.5, ast_var=1.5, @@ -1836,9 +1122,9 @@ def test_double_ended_fix_alpha_matching_sections_and_one_asym_att(): ], ) - assert_almost_equal_verbose(temp_real_celsius, ds.tmpf.values, decimal=7) - assert_almost_equal_verbose(temp_real_celsius, ds.tmpb.values, decimal=7) - assert_almost_equal_verbose(temp_real_celsius, ds.tmpw.values, decimal=7) + assert_almost_equal_verbose(temp_real_celsius, out["tmpf"].values, decimal=7) + assert_almost_equal_verbose(temp_real_celsius, out["tmpb"].values, decimal=7) + assert_almost_equal_verbose(temp_real_celsius, out["tmpw"].values, decimal=7) pass @@ -1928,7 +1214,7 @@ def test_double_ended_fix_alpha_gamma_matching_sections_and_one_asym_att(): "warm": [slice(x[nx_per_sec], x[2 * nx_per_sec - 1])], } - ds.dts.calibration_double_ended( + out = ds.dts.calibration_double_ended( sections=sections, st_var=1.5, ast_var=1.5, @@ -1950,12 +1236,12 @@ def test_double_ended_fix_alpha_gamma_matching_sections_and_one_asym_att(): k = ["talpha_fw", "talpha_bw", "trans_att"] for ki in k: - del ds[ki] + del out[ki] - alpha_adj = ds.alpha.values.copy() - alpha_var_adj = ds.alpha_var.values.copy() + alpha_adj = out["alpha"].values.copy() + alpha_var_adj = out["alpha_var"].values.copy() - ds.dts.calibration_double_ended( + out = ds.dts.calibration_double_ended( sections=sections, st_var=1.5, ast_var=1.5, @@ -1975,9 +1261,9 @@ def test_double_ended_fix_alpha_gamma_matching_sections_and_one_asym_att(): ], ) - assert_almost_equal_verbose(temp_real_celsius, ds.tmpf.values, decimal=7) - assert_almost_equal_verbose(temp_real_celsius, ds.tmpb.values, decimal=7) - assert_almost_equal_verbose(temp_real_celsius, ds.tmpw.values, decimal=7) + assert_almost_equal_verbose(temp_real_celsius, out["tmpf"].values, decimal=7) + assert_almost_equal_verbose(temp_real_celsius, out["tmpb"].values, decimal=7) + assert_almost_equal_verbose(temp_real_celsius, out["tmpw"].values, decimal=7) pass @@ -2067,7 +1353,7 @@ def test_double_ended_fix_gamma_matching_sections_and_one_asym_att(): "warm": [slice(x[nx_per_sec], x[2 * nx_per_sec - 1])], } - ds.dts.calibration_double_ended( + out = ds.dts.calibration_double_ended( sections=sections, st_var=1.5, ast_var=1.5, @@ -2086,9 +1372,9 @@ def test_double_ended_fix_gamma_matching_sections_and_one_asym_att(): ], ) - assert_almost_equal_verbose(temp_real_celsius, ds.tmpf.values, decimal=7) - assert_almost_equal_verbose(temp_real_celsius, ds.tmpb.values, decimal=7) - assert_almost_equal_verbose(temp_real_celsius, ds.tmpw.values, decimal=7) + assert_almost_equal_verbose(temp_real_celsius, out["tmpf"].values, decimal=7) + assert_almost_equal_verbose(temp_real_celsius, out["tmpb"].values, decimal=7) + assert_almost_equal_verbose(temp_real_celsius, out["tmpw"].values, decimal=7) pass @@ -2188,7 +1474,7 @@ def test_double_ended_exponential_variance_estimate_synthetic(): rast_label = "rast" # MC variance - ds.dts.calibration_double_ended( + out = ds.dts.calibration_double_ended( sections=sections, st_label=st_label, ast_label=ast_label, @@ -2346,7 +1632,7 @@ def test_estimate_variance_of_temperature_estimate(): "warm": [slice(0.5 * cable_len, 0.75 * cable_len)], } # MC variance - ds.dts.calibration_double_ended( + out = ds.dts.calibration_double_ended( sections=sections, st_var=mst_var, ast_var=mast_var, @@ -2358,15 +1644,13 @@ def test_estimate_variance_of_temperature_estimate(): solver="stats", ) - ds.conf_int_double_ended( - sections=sections, - p_val="p_val", - p_cov="p_cov", + out2 = ds.dts.monte_carlo_double_ended( + result=out, st_var=mst_var, ast_var=mast_var, rst_var=mrst_var, rast_var=mrast_var, - # conf_ints=[20., 80.], + conf_ints=[], mc_sample_size=nmc, da_random_state=state, mc_remove_set_flag=False, @@ -2374,41 +1658,41 @@ def test_estimate_variance_of_temperature_estimate(): ) assert_almost_equal_verbose( - (ds.r_st - ds.st).var(dim=["mc", "time"]), mst_var, decimal=2 + (out2["r_st"] - ds["st"]).var(dim=["mc", "time"]), mst_var, decimal=2 ) assert_almost_equal_verbose( - (ds.r_ast - ds.ast).var(dim=["mc", "time"]), mast_var, decimal=2 + (out2["r_ast"] - ds["ast"]).var(dim=["mc", "time"]), mast_var, decimal=2 ) assert_almost_equal_verbose( - (ds.r_rst - ds.rst).var(dim=["mc", "time"]), mrst_var, decimal=2 + (out2["r_rst"] - ds["rst"]).var(dim=["mc", "time"]), mrst_var, decimal=2 ) assert_almost_equal_verbose( - (ds.r_rast - ds.rast).var(dim=["mc", "time"]), mrast_var, decimal=3 + (out2["r_rast"] - ds["rast"]).var(dim=["mc", "time"]), mrast_var, decimal=3 ) - assert_almost_equal_verbose(ds.gamma_mc.var(dim="mc"), 0.0, decimal=2) - assert_almost_equal_verbose(ds.alpha_mc.var(dim="mc"), 0.0, decimal=8) - assert_almost_equal_verbose(ds.df_mc.var(dim="mc"), ds.df_var, decimal=8) - assert_almost_equal_verbose(ds.db_mc.var(dim="mc"), ds.db_var, decimal=8) + assert_almost_equal_verbose(out2["gamma_mc"].var(dim="mc"), 0.0, decimal=2) + assert_almost_equal_verbose(out2["alpha_mc"].var(dim="mc"), 0.0, decimal=8) + assert_almost_equal_verbose(out2["df_mc"].var(dim="mc"), out["df_var"], decimal=8) + assert_almost_equal_verbose(out2["db_mc"].var(dim="mc"), out["db_var"], decimal=8) # tmpf temp_real2 = temp_real[:, 0] - 273.15 - actual = (ds.tmpf - temp_real2[:, None]).var(dim="time") - desire2 = ds.tmpf_var.mean(dim="time") + actual = (out["tmpf"] - temp_real2[:, None]).var(dim="time") + desire2 = out["tmpf_var"].mean(dim="time") # Validate on sections that were not used for calibration. assert_almost_equal_verbose(actual[16:32], desire2[16:32], decimal=2) # tmpb - actual = (ds.tmpb - temp_real2[:, None]).var(dim="time") - desire2 = ds.tmpb_var.mean(dim="time") + actual = (out["tmpb"] - temp_real2[:, None]).var(dim="time") + desire2 = out["tmpb_var"].mean(dim="time") # Validate on sections that were not used for calibration. assert_almost_equal_verbose(actual[16:32], desire2[16:32], decimal=2) # tmpw - actual = (ds.tmpw - temp_real2[:, None]).var(dim="time") - desire2 = ds.tmpw_var.mean(dim="time") + actual = (out["tmpw"] - temp_real2[:, None]).var(dim="time") + desire2 = out["tmpw_var"].mean(dim="time") assert_almost_equal_verbose(actual[16:32], desire2[16:32], decimal=3) pass @@ -2480,11 +1764,10 @@ def test_single_ended_wls_estimate_synthetic(): out = ds.dts.calibrate_single_ended( sections=sections, st_var=1.0, ast_var=1.0, method="wls", solver="sparse" ) - ds.update(out) - assert_almost_equal_verbose(ds.gamma.values, gamma, decimal=6) - assert_almost_equal_verbose(ds.dalpha.values, dalpha_p - dalpha_m, decimal=8) - assert_almost_equal_verbose(ds.tmpf.values, temp_real - 273.15, decimal=4) + assert_almost_equal_verbose(out["gamma"].values, gamma, decimal=6) + assert_almost_equal_verbose(out["dalpha"].values, dalpha_p - dalpha_m, decimal=8) + assert_almost_equal_verbose(out["tmpf"].values, temp_real - 273.15, decimal=4) pass @@ -3276,7 +2559,7 @@ def test_average_measurements_single_ended(): st_var, ast_var = 5.0, 5.0 - ds.dts.calibrate_single_ended( + out = ds.dts.calibrate_single_ended( sections=sections, st_var=st_var, ast_var=ast_var, method="wls", solver="sparse" ) ds.average_single_ended( @@ -3342,7 +2625,7 @@ def test_average_measurements_double_ended(): st_var, ast_var, rst_var, rast_var = 5.0, 5.0, 5.0, 5.0 - ds.dts.calibration_double_ended( + out = ds.dts.calibration_double_ended( sections=sections, st_var=st_var, ast_var=ast_var, diff --git a/tests/test_variance_stokes.py b/tests/test_variance_stokes.py index 6accabe3..d37cad45 100644 --- a/tests/test_variance_stokes.py +++ b/tests/test_variance_stokes.py @@ -4,8 +4,10 @@ import pytest from scipy import stats import xarray as xr +from xarray import Dataset from dtscalibration import read_silixa_files +from dtscalibration.datastore_accessor import DtsAccessor # noqa: F401 from dtscalibration.variance_stokes import variance_stokes_exponential from dtscalibration.variance_stokes import variance_stokes_constant from dtscalibration.variance_stokes import variance_stokes_linear @@ -54,6 +56,695 @@ def assert_almost_equal_verbose(actual, desired, verbose=False, **kwargs): np.testing.assert_almost_equal(actual, desired2, err_msg=m, **kwargs) pass +@pytest.mark.slow # Execution time ~20 seconds +def test_variance_input_types_single(): + import dask.array as da + + state = da.random.RandomState(0) + + stokes_m_var = 40.0 + cable_len = 100.0 + nt = 500 + time = np.arange(nt) + x = np.linspace(0.0, cable_len, 100) + ts_cold = np.ones(nt) * 4.0 + ts_warm = np.ones(nt) * 20.0 + + C_p = 15246 + C_m = 2400.0 + dalpha_r = 0.005284 + dalpha_m = 0.004961 + dalpha_p = 0.005607 + gamma = 482.6 + cold_mask = x < 0.5 * cable_len + warm_mask = np.invert(cold_mask) # == False + temp_real = np.ones((len(x), nt)) + temp_real[cold_mask] *= ts_cold + 273.15 + temp_real[warm_mask] *= ts_warm + 273.15 + + st = ( + C_p + * np.exp(-dalpha_r * x[:, None]) + * np.exp(-dalpha_p * x[:, None]) + * np.exp(-gamma / temp_real) + / (1 - np.exp(-gamma / temp_real)) + ) + ast = ( + C_m + * np.exp(-dalpha_r * x[:, None]) + * np.exp(-dalpha_m * x[:, None]) + / (1 - np.exp(-gamma / temp_real)) + ) + + st_m = st + stats.norm.rvs(size=st.shape, scale=stokes_m_var**0.5) + ast_m = ast + stats.norm.rvs(size=ast.shape, scale=1.1 * stokes_m_var**0.5) + + print("alphaint", cable_len * (dalpha_p - dalpha_m)) + print("alpha", dalpha_p - dalpha_m) + print("C", np.log(C_p / C_m)) + print("x0", x.max()) + + ds = Dataset( + { + "st": (["x", "time"], st_m), + "ast": (["x", "time"], ast_m), + "userAcquisitionTimeFW": (["time"], np.ones(nt)), + "cold": (["time"], ts_cold), + "warm": (["time"], ts_warm), + }, + coords={"x": x, "time": time}, + attrs={"isDoubleEnded": "0"}, + ) + + sections = { + "cold": [slice(0.0, 0.4 * cable_len)], + "warm": [slice(0.6 * cable_len, cable_len)], + } + + # Test float input + st_var = 5.0 + + out = ds.dts.calibrate_single_ended( + sections=sections, st_var=st_var, ast_var=st_var, method="wls", solver="sparse" + ) + + out2 = ds.dts.monte_carlo_single_ended( + result=out, + st_var=st_var, + ast_var=st_var, + mc_sample_size=100, + da_random_state=state, + mc_remove_set_flag=False, + ) + + assert_almost_equal_verbose( + out2["tmpf_mc_var"].sel(x=slice(0, 10)).mean(), 0.044361, decimal=2 + ) + assert_almost_equal_verbose( + out2["tmpf_mc_var"].sel(x=slice(90, 100)).mean(), 0.242028, decimal=2 + ) + + # Test callable input + def callable_st_var(stokes): + slope = 0.01 + offset = 0 + return slope * stokes + offset + + out = ds.dts.calibrate_single_ended( + sections=sections, + st_var=callable_st_var, + ast_var=callable_st_var, + method="wls", + solver="sparse", + ) + + out2 = ds.dts.monte_carlo_single_ended( + result=out, + st_var=callable_st_var, + ast_var=callable_st_var, + mc_sample_size=100, + da_random_state=state, + ) + + assert_almost_equal_verbose( + out2["tmpf_mc_var"].sel(x=slice(0, 10)).mean(), 0.184753, decimal=2 + ) + assert_almost_equal_verbose( + out2["tmpf_mc_var"].sel(x=slice(90, 100)).mean(), 0.545186, decimal=2 + ) + + # Test input with shape of (ntime, nx) + st_var = ds.st.values * 0 + 20.0 + out = ds.dts.calibrate_single_ended( + sections=sections, st_var=st_var, ast_var=st_var, method="wls", solver="sparse" + ) + + out2 = ds.dts.monte_carlo_single_ended(result=out, + st_var=st_var, ast_var=st_var, mc_sample_size=100, da_random_state=state + ) + + assert_almost_equal_verbose(out2["tmpf_mc_var"].mean(), 0.418098, decimal=2) + + # Test input with shape (nx, 1) + st_var = np.vstack( + ds.st.mean(dim="time").values * 0 + np.linspace(10, 50, num=ds.st.x.size) + ) + + out = ds.dts.calibrate_single_ended( + sections=sections, st_var=st_var, ast_var=st_var, method="wls", solver="sparse" + ) + + out2 = ds.dts.monte_carlo_single_ended(result=out, + st_var=st_var, ast_var=st_var, mc_sample_size=100, da_random_state=state + ) + + assert_almost_equal_verbose( + out2["tmpf_mc_var"].sel(x=slice(0, 50)).mean().values, 0.2377, decimal=2 + ) + assert_almost_equal_verbose( + out2["tmpf_mc_var"].sel(x=slice(50, 100)).mean().values, 1.3203, decimal=2 + ) + + # Test input with shape (ntime) + st_var = ds.st.mean(dim="x").values * 0 + np.linspace(5, 200, num=nt) + + out = ds.dts.calibrate_single_ended( + sections=sections, st_var=st_var, ast_var=st_var, method="wls", solver="sparse" + ) + + out2 = ds.dts.monte_carlo_single_ended(result=out, + st_var=st_var, ast_var=st_var, mc_sample_size=100, da_random_state=state + ) + + assert_almost_equal_verbose( + out2["tmpf_mc_var"].sel(time=slice(0, nt // 2)).mean().values, 1.0908, decimal=2 + ) + assert_almost_equal_verbose( + out2["tmpf_mc_var"].sel(time=slice(nt // 2, None)).mean().values, 3.0759, decimal=2 + ) + + pass + + +@pytest.mark.slow # Execution time ~0.5 minute +def test_variance_input_types_double(): + import dask.array as da + + state = da.random.RandomState(0) + + stokes_m_var = 40.0 + cable_len = 100.0 + nt = 500 + time = np.arange(nt) + x = np.linspace(0.0, cable_len, 100) + ts_cold = np.ones(nt) * 4.0 + ts_warm = np.ones(nt) * 20.0 + + C_p = 15246 + C_m = 2400.0 + dalpha_r = 0.005284 + dalpha_m = 0.004961 + dalpha_p = 0.005607 + gamma = 482.6 + cold_mask = x < 0.5 * cable_len + warm_mask = np.invert(cold_mask) # == False + temp_real = np.ones((len(x), nt)) + temp_real[cold_mask] *= ts_cold + 273.15 + temp_real[warm_mask] *= ts_warm + 273.15 + + st = ( + C_p + * np.exp(-dalpha_r * x[:, None]) + * np.exp(-dalpha_p * x[:, None]) + * np.exp(-gamma / temp_real) + / (1 - np.exp(-gamma / temp_real)) + ) + ast = ( + C_m + * np.exp(-dalpha_r * x[:, None]) + * np.exp(-dalpha_m * x[:, None]) + / (1 - np.exp(-gamma / temp_real)) + ) + rst = ( + C_p + * np.exp(-dalpha_r * (-x[:, None] + 100)) + * np.exp(-dalpha_p * (-x[:, None] + 100)) + * np.exp(-gamma / temp_real) + / (1 - np.exp(-gamma / temp_real)) + ) + rast = ( + C_m + * np.exp(-dalpha_r * (-x[:, None] + 100)) + * np.exp(-dalpha_m * (-x[:, None] + 100)) + / (1 - np.exp(-gamma / temp_real)) + ) + + st_m = st + stats.norm.rvs(size=st.shape, scale=stokes_m_var**0.5) + ast_m = ast + stats.norm.rvs(size=ast.shape, scale=1.1 * stokes_m_var**0.5) + rst_m = rst + stats.norm.rvs(size=rst.shape, scale=0.9 * stokes_m_var**0.5) + rast_m = rast + stats.norm.rvs(size=rast.shape, scale=0.8 * stokes_m_var**0.5) + + print("alphaint", cable_len * (dalpha_p - dalpha_m)) + print("alpha", dalpha_p - dalpha_m) + print("C", np.log(C_p / C_m)) + print("x0", x.max()) + + ds = Dataset( + { + "st": (["x", "time"], st_m), + "ast": (["x", "time"], ast_m), + "rst": (["x", "time"], rst_m), + "rast": (["x", "time"], rast_m), + "userAcquisitionTimeFW": (["time"], np.ones(nt)), + "userAcquisitionTimeBW": (["time"], np.ones(nt)), + "cold": (["time"], ts_cold), + "warm": (["time"], ts_warm), + }, + coords={"x": x, "time": time}, + attrs={"isDoubleEnded": "1"}, + ) + + sections = { + "cold": [slice(0.0, 0.4 * cable_len)], + "warm": [slice(0.6 * cable_len, cable_len)], + } + + # Test float input + st_var = 5.0 + + out = ds.dts.calibration_double_ended( + sections=sections, + st_var=st_var, + ast_var=st_var, + rst_var=st_var, + rast_var=st_var, + method="wls", + solver="sparse", + ) + + out2 = ds.dts.monte_carlo_double_ended( + result=out, + st_var=st_var, + ast_var=st_var, + rst_var=st_var, + rast_var=st_var, + conf_ints=[], + mc_sample_size=100, + da_random_state=state, + ) + + assert_almost_equal_verbose( + out2["tmpf_mc_var"].sel(x=slice(0, 10)).mean(), 0.03584935, decimal=2 + ) + assert_almost_equal_verbose( + out2["tmpf_mc_var"].sel(x=slice(90, 100)).mean(), 0.22982146, decimal=2 + ) + + # Test callable input + def st_var_callable(stokes): + slope = 0.01 + offset = 0 + return slope * stokes + offset + + out = ds.dts.calibration_double_ended( + sections=sections, + st_var=st_var_callable, + ast_var=st_var_callable, + rst_var=st_var_callable, + rast_var=st_var_callable, + method="wls", + solver="sparse", + ) + + out2 = ds.dts.monte_carlo_double_ended( + result=out, + st_var=st_var_callable, + ast_var=st_var_callable, + rst_var=st_var_callable, + rast_var=st_var_callable, + conf_ints=[], + mc_sample_size=100, + da_random_state=state, + ) + + assert_almost_equal_verbose( + out2["tmpf_mc_var"].sel(x=slice(0, 10)).mean(), 0.18058514, decimal=2 + ) + assert_almost_equal_verbose( + out2["tmpf_mc_var"].sel(x=slice(90, 100)).mean(), 0.53862813, decimal=2 + ) + + # Test input with shape of (ntime, nx) + st_var = ds.st.values * 0 + 20.0 + + out = ds.dts.calibration_double_ended( + sections=sections, + st_var=st_var, + ast_var=st_var, + rst_var=st_var, + rast_var=st_var, + method="wls", + solver="sparse", + ) + + out2 = ds.dts.monte_carlo_double_ended( + result=out, + st_var=st_var, + ast_var=st_var, + rst_var=st_var, + rast_var=st_var, + conf_ints=[], + mc_sample_size=100, + da_random_state=state, + ) + + assert_almost_equal_verbose(out2["tmpf_mc_var"].mean(), 0.40725674, decimal=2) + + # Test input with shape (nx, 1) + st_var = np.vstack( + ds.st.mean(dim="time").values * 0 + np.linspace(10, 50, num=ds.st.x.size) + ) + + out = ds.dts.calibration_double_ended( + sections=sections, + st_var=st_var, + ast_var=st_var, + rst_var=st_var, + rast_var=st_var, + method="wls", + solver="sparse", + ) + + out2 = ds.dts.monte_carlo_double_ended( + result=out, + st_var=st_var, + ast_var=st_var, + rst_var=st_var, + rast_var=st_var, + conf_ints=[], + mc_sample_size=100, + da_random_state=state, + ) + + assert_almost_equal_verbose( + out2["tmpf_mc_var"].sel(x=slice(0, 50)).mean().values, 0.21163704, decimal=2 + ) + assert_almost_equal_verbose( + out2["tmpf_mc_var"].sel(x=slice(50, 100)).mean().values, 1.28247762, decimal=2 + ) + + # Test input with shape (ntime) + st_var = ds.st.mean(dim="x").values * 0 + np.linspace(5, 200, num=nt) + + out = ds.dts.calibration_double_ended( + sections=sections, + st_var=st_var, + ast_var=st_var, + rst_var=st_var, + rast_var=st_var, + method="wls", + solver="sparse", + ) + + out2 = ds.dts.monte_carlo_double_ended( + result=out, + st_var=st_var, + ast_var=st_var, + rst_var=st_var, + rast_var=st_var, + conf_ints=[], + mc_sample_size=100, + da_random_state=state, + ) + + assert_almost_equal_verbose( + out2["tmpf_mc_var"].sel(time=slice(0, nt // 2)).mean().values, 1.090, decimal=2 + ) + assert_almost_equal_verbose( + out2["tmpf_mc_var"].sel(time=slice(nt // 2, None)).mean().values, 3.06, decimal=2 + ) + + pass + + +@pytest.mark.slow # Execution time ~0.5 minute +def test_double_ended_variance_estimate_synthetic(): + import dask.array as da + + state = da.random.RandomState(0) + + stokes_m_var = 40.0 + cable_len = 100.0 + nt = 500 + time = np.arange(nt) + x = np.linspace(0.0, cable_len, 100) + ts_cold = np.ones(nt) * 4.0 + ts_warm = np.ones(nt) * 20.0 + + C_p = 15246 + C_m = 2400.0 + dalpha_r = 0.0005284 + dalpha_m = 0.0004961 + dalpha_p = 0.0005607 + gamma = 482.6 + cold_mask = x < 0.5 * cable_len + warm_mask = np.invert(cold_mask) # == False + temp_real = np.ones((len(x), nt)) + temp_real[cold_mask] *= ts_cold + 273.15 + temp_real[warm_mask] *= ts_warm + 273.15 + + st = ( + C_p + * np.exp(-dalpha_r * x[:, None]) + * np.exp(-dalpha_p * x[:, None]) + * np.exp(-gamma / temp_real) + / (1 - np.exp(-gamma / temp_real)) + ) + ast = ( + C_m + * np.exp(-dalpha_r * x[:, None]) + * np.exp(-dalpha_m * x[:, None]) + / (1 - np.exp(-gamma / temp_real)) + ) + rst = ( + C_p + * np.exp(-dalpha_r * (-x[:, None] + 100)) + * np.exp(-dalpha_p * (-x[:, None] + 100)) + * np.exp(-gamma / temp_real) + / (1 - np.exp(-gamma / temp_real)) + ) + rast = ( + C_m + * np.exp(-dalpha_r * (-x[:, None] + 100)) + * np.exp(-dalpha_m * (-x[:, None] + 100)) + / (1 - np.exp(-gamma / temp_real)) + ) + + st_m = st + stats.norm.rvs(size=st.shape, scale=stokes_m_var**0.5) + ast_m = ast + stats.norm.rvs(size=ast.shape, scale=1.1 * stokes_m_var**0.5) + rst_m = rst + stats.norm.rvs(size=rst.shape, scale=0.9 * stokes_m_var**0.5) + rast_m = rast + stats.norm.rvs(size=rast.shape, scale=0.8 * stokes_m_var**0.5) + + print("alphaint", cable_len * (dalpha_p - dalpha_m)) + print("alpha", dalpha_p - dalpha_m) + print("C", np.log(C_p / C_m)) + print("x0", x.max()) + + ds = Dataset( + { + "st": (["x", "time"], st_m), + "ast": (["x", "time"], ast_m), + "rst": (["x", "time"], rst_m), + "rast": (["x", "time"], rast_m), + "userAcquisitionTimeFW": (["time"], np.ones(nt)), + "userAcquisitionTimeBW": (["time"], np.ones(nt)), + "cold": (["time"], ts_cold), + "warm": (["time"], ts_warm), + }, + coords={"x": x, "time": time}, + attrs={"isDoubleEnded": "1"}, + ) + + sections = { + "cold": [slice(0.0, 0.5 * cable_len)], + "warm": [slice(0.5 * cable_len, cable_len)], + } + + mst_var, _ = variance_stokes_constant(ds.dts.st, sections, ds.dts.acquisitiontime_fw, reshape_residuals=False) + mast_var, _ = variance_stokes_constant(ds.dts.ast, sections, ds.dts.acquisitiontime_fw, reshape_residuals=False) + mrst_var, _ = variance_stokes_constant(ds.dts.rst, sections, ds.dts.acquisitiontime_bw, reshape_residuals=False) + mrast_var, _ = variance_stokes_constant(ds.dts.rast, sections, ds.dts.acquisitiontime_bw, reshape_residuals=False) + + mst_var = float(mst_var) + mast_var = float(mast_var) + mrst_var = float(mrst_var) + mrast_var = float(mrast_var) + + # MC variance + out = ds.dts.calibration_double_ended( + sections=sections, + st_var=mst_var, + ast_var=mast_var, + rst_var=mrst_var, + rast_var=mrast_var, + method="wls", + solver="sparse", + ) + + assert_almost_equal_verbose(out["tmpf"].mean(), 12.0, decimal=2) + assert_almost_equal_verbose(out["tmpb"].mean(), 12.0, decimal=3) + + # Calibrated variance + stdsf1 = out.dts.ufunc_per_section( + sections=sections, label="tmpf", func=np.std, temp_err=True, calc_per="stretch" + ) + stdsb1 = out.dts.ufunc_per_section( + sections=sections, label="tmpb", func=np.std, temp_err=True, calc_per="stretch" + ) + + out2 = ds.dts.monte_carlo_double_ended( + sections=sections, + p_val="p_val", + p_cov="p_cov", + st_var=mst_var, + ast_var=mast_var, + rst_var=mrst_var, + rast_var=mrast_var, + conf_ints=[2.5, 50.0, 97.5], + mc_sample_size=100, + da_random_state=state, + ) + + # Use a single timestep to better check if the parameter uncertainties propagate + ds1 = out2.isel(time=1) + # Estimated VAR + stdsf2 = ds1.dts.ufunc_per_section( + sections=sections, + label="tmpf_mc_var", + func=np.mean, + temp_err=False, + calc_per="stretch", + ) + stdsb2 = ds1.dts.ufunc_per_section( + sections=sections, + label="tmpb_mc_var", + func=np.mean, + temp_err=False, + calc_per="stretch", + ) + + for (_, v1), (_, v2) in zip(stdsf1.items(), stdsf2.items()): + for v1i, v2i in zip(v1, v2): + print("Real VAR: ", v1i**2, "Estimated VAR: ", float(v2i)) + assert_almost_equal_verbose(v1i**2, v2i, decimal=2) + + for (_, v1), (_, v2) in zip(stdsb1.items(), stdsb2.items()): + for v1i, v2i in zip(v1, v2): + print("Real VAR: ", v1i**2, "Estimated VAR: ", float(v2i)) + assert_almost_equal_verbose(v1i**2, v2i, decimal=2) + + pass + + +def test_single_ended_variance_estimate_synthetic(): + import dask.array as da + + state = da.random.RandomState(0) + + stokes_m_var = 40.0 + astokes_m_var = 60.0 + cable_len = 100.0 + nt = 50 + time = np.arange(nt) + x = np.linspace(0.0, cable_len, 500) + ts_cold = np.ones(nt) * 4.0 + ts_warm = np.ones(nt) * 20.0 + + C_p = 15246 + C_m = 2400.0 + dalpha_r = 0.0005284 + dalpha_m = 0.0004961 + dalpha_p = 0.0005607 + gamma = 482.6 + cold_mask = x < 0.5 * cable_len + warm_mask = np.invert(cold_mask) # == False + temp_real = np.ones((len(x), nt)) + temp_real[cold_mask] *= ts_cold + 273.15 + temp_real[warm_mask] *= ts_warm + 273.15 + + st = ( + C_p + * np.exp(-dalpha_r * x[:, None]) + * np.exp(-dalpha_p * x[:, None]) + * np.exp(-gamma / temp_real) + / (1 - np.exp(-gamma / temp_real)) + ) + ast = ( + C_m + * np.exp(-dalpha_r * x[:, None]) + * np.exp(-dalpha_m * x[:, None]) + / (1 - np.exp(-gamma / temp_real)) + ) + st_m = st + stats.norm.rvs(size=st.shape, scale=stokes_m_var**0.5) + ast_m = ast + stats.norm.rvs(size=ast.shape, scale=astokes_m_var**0.5) + + print("alphaint", cable_len * (dalpha_p - dalpha_m)) + print("alpha", dalpha_p - dalpha_m) + print("C", np.log(C_p / C_m)) + print("x0", x.max()) + + ds = Dataset( + { + "st": (["x", "time"], st_m), + "ast": (["x", "time"], ast_m), + "userAcquisitionTimeFW": (["time"], np.ones(nt)), + "cold": (["time"], ts_cold), + "warm": (["time"], ts_warm), + }, + coords={"x": x, "time": time}, + attrs={"isDoubleEnded": "0"}, + ) + + sections = { + "cold": [slice(0.0, 0.5 * cable_len)], + "warm": [slice(0.5 * cable_len, cable_len)], + } + + st_label = "st" + ast_label = "ast" + + mst_var, _ = ds.variance_stokes(st_label=st_label, sections=sections) + mast_var, _ = ds.variance_stokes(st_label=ast_label, sections=sections) + mst_var = float(mst_var) + mast_var = float(mast_var) + + # MC variqnce + out = ds.dts.calibrate_single_ended( + sections=sections, + st_var=mst_var, + ast_var=mast_var, + method="wls", + solver="sparse", + ) + + out2 = ds.dts.monte_carlo_single_ended( + result=out, + st_var=mst_var, + ast_var=mast_var, + conf_ints=[2.5, 50.0, 97.5], + mc_sample_size=50, + da_random_state=state, + ) + + # Calibrated variance + stdsf1 = out.dts.ufunc_per_section( + sections=sections, + label="tmpf", + func=np.std, + temp_err=True, + calc_per="stretch", + ddof=1, + ) + + # Use a single timestep to better check if the parameter uncertainties propagate + ds1 = out2.isel(time=1) + # Estimated VAR + stdsf2 = ds1.dts.ufunc_per_section( + sections=sections, + label="tmpf_mc_var", + func=np.mean, + temp_err=False, + calc_per="stretch", + ) + + for (_, v1), (_, v2) in zip(stdsf1.items(), stdsf2.items()): + for v1i, v2i in zip(v1, v2): + print("Real VAR: ", v1i**2, "Estimated VAR: ", float(v2i)) + assert_almost_equal_verbose(v1i**2, v2i, decimal=2) + + pass + + @pytest.mark.skip(reason="Not enough measurements in time. Use exponential instead.") def test_variance_of_stokes(): @@ -226,7 +917,7 @@ def test_exponential_variance_of_stokes_synthetic(): y += stats.norm.rvs(size=y.size, scale=yvar**0.5).reshape(y.shape) - ds = DataStore( + ds = Dataset( { "st": (["x", "time"], y), "probe1Temperature": (["time"], range(nt)), From 068305e654b41f6937291b04e76dc7b22b86349d Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Sun, 15 Oct 2023 21:41:26 +0200 Subject: [PATCH 30/66] Got several variance_tests working --- src/dtscalibration/datastore_accessor.py | 149 ++++++++---------- src/dtscalibration/io.py | 31 ++-- tests/test_averaging.py | 192 +++++++++++++++++++++++ tests/test_dtscalibration.py | 145 ----------------- tests/test_variance_stokes.py | 15 +- 5 files changed, 286 insertions(+), 246 deletions(-) create mode 100644 tests/test_averaging.py diff --git a/src/dtscalibration/datastore_accessor.py b/src/dtscalibration/datastore_accessor.py index 089a4de3..255f9b5a 100644 --- a/src/dtscalibration/datastore_accessor.py +++ b/src/dtscalibration/datastore_accessor.py @@ -400,6 +400,10 @@ def ufunc_per_section( if not suppress_section_validation: validate_sections_definition(sections=sections) validate_no_overlapping_sections(sections=sections) + + if temp_err or ref_temp_broadcasted: + for k in sections: + assert k in self._obj, f"{k} is not in the Dataset but is in `sections` and is required to compute temp_err" if label is None: dataarray = None @@ -1912,8 +1916,9 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): return out - def average_single_ended( + def average_monte_carlo_single_ended( self, + result, st_var, ast_var, conf_ints=None, @@ -1938,17 +1943,9 @@ def average_single_ended( Parameters ---------- - p_val : array-like, optional - Define `p_val`, `p_var`, `p_cov` if you used an external function - for calibration. Has size 2 + `nt`. First value is :math:`\gamma`, - second is :math:`\Delta \\alpha`, others are :math:`C` for each - timestep. - If set to False, no uncertainty in the parameters is propagated - into the confidence intervals. Similar to the spec sheets of the DTS - manufacturers. And similar to passing an array filled with zeros - p_cov : array-like, optional - The covariances of `p_val`. - st_var, ast_var : float, callable, array-like, optional + result : xr.Dataset + The result from the `calibrate_single_ended()` method. + st_var, ast_var : float, callable, array-like The variance of the measurement noise of the Stokes signals in the forward direction. If `float` the variance of the noise from the Stokes detector is described with a single value. @@ -2045,12 +2042,15 @@ def average_single_ended( Returns ------- - """ - out = xr.Dataset() + """ + # out contains the state + out = xr.Dataset(coords={"x": self.x, "time": self.time, "trans_att": result["trans_att"]}).copy() + out.coords["x"].attrs = dim_attrs["x"] + out.coords["trans_att"].attrs = dim_attrs["trans_att"] + out.coords["CI"] = conf_ints - mcparams = self.conf_int_single_ended( - p_val=p_val, - p_cov=p_cov, + mcparams = self.monte_carlo_single_ended( + result=result, st_var=st_var, ast_var=ast_var, conf_ints=None, @@ -2059,7 +2059,7 @@ def average_single_ended( mc_remove_set_flag=False, reduce_memory_usage=reduce_memory_usage, ) - mcparams["tmpf"] = self["tmpf"] + mcparams["tmpf"] = result["tmpf"] if ci_avg_time_sel is not None: time_dim2 = "time" + "_avg" @@ -2252,18 +2252,15 @@ def average_single_ended( if k in out: del out[k] - self.update(out) - pass + return out - def average_double_ended( + def average_monte_carlo_double_ended( self, - sections=None, - p_val="p_val", - p_cov="p_cov", - st_var=None, - ast_var=None, - rst_var=None, - rast_var=None, + result, + st_var, + ast_var, + rst_var, + rast_var, conf_ints=None, mc_sample_size=100, ci_avg_time_flag1=False, @@ -2286,17 +2283,9 @@ def average_double_ended( Parameters ---------- - p_val : array-like, optional - Define `p_val`, `p_var`, `p_cov` if you used an external function - for calibration. Has size 2 + `nt`. First value is :math:`\\gamma`, - second is :math:`\\Delta \\alpha`, others are :math:`C` for each - timestep. - If set to False, no uncertainty in the parameters is propagated - into the confidence intervals. Similar to the spec sheets of the DTS - manufacturers. And similar to passing an array filled with zeros - p_cov : array-like, optional - The covariances of `p_val`. - st_var, ast_var, rst_var, rast_var : float, callable, array-like, optional + result : xr.Dataset + The result from the `calibrate_double_ended()` method. + st_var, ast_var, rst_var, rast_var : float, callable, array-like The variance of the measurement noise of the Stokes signals in the forward direction. If `float` the variance of the noise from the Stokes detector is described with a single value. @@ -2395,36 +2384,39 @@ def average_double_ended( """ - def create_da_ta2(no, i_splice, direction="fw", chunks=None): - """create mask array mc, o, nt""" - - if direction == "fw": - arr = da.concatenate( - ( - da.zeros((1, i_splice, 1), chunks=(1, i_splice, 1), dtype=bool), - da.ones( - (1, no - i_splice, 1), - chunks=(1, no - i_splice, 1), - dtype=bool, - ), - ), - axis=1, - ).rechunk((1, chunks[1], 1)) - else: - arr = da.concatenate( - ( - da.ones((1, i_splice, 1), chunks=(1, i_splice, 1), dtype=bool), - da.zeros( - (1, no - i_splice, 1), - chunks=(1, no - i_splice, 1), - dtype=bool, - ), - ), - axis=1, - ).rechunk((1, chunks[1], 1)) - return arr + # def create_da_ta2(no, i_splice, direction="fw", chunks=None): + # """create mask array mc, o, nt""" + + # if direction == "fw": + # arr = da.concatenate( + # ( + # da.zeros((1, i_splice, 1), chunks=(1, i_splice, 1), dtype=bool), + # da.ones( + # (1, no - i_splice, 1), + # chunks=(1, no - i_splice, 1), + # dtype=bool, + # ), + # ), + # axis=1, + # ).rechunk((1, chunks[1], 1)) + # else: + # arr = da.concatenate( + # ( + # da.ones((1, i_splice, 1), chunks=(1, i_splice, 1), dtype=bool), + # da.zeros( + # (1, no - i_splice, 1), + # chunks=(1, no - i_splice, 1), + # dtype=bool, + # ), + # ), + # axis=1, + # ).rechunk((1, chunks[1], 1)) + # return arr - check_deprecated_kwargs(kwargs) + out = xr.Dataset(coords={"x": self.x, "time": self.time, "trans_att": result["trans_att"]}).copy() + out.coords["x"].attrs = dim_attrs["x"] + out.coords["trans_att"].attrs = dim_attrs["trans_att"] + out.coords["CI"] = conf_ints if (ci_avg_x_flag1 or ci_avg_x_flag2) and ( ci_avg_time_flag1 or ci_avg_time_flag2 @@ -2441,12 +2433,8 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): else: pass - out = xr.Dataset() - - mcparams = self.conf_int_double_ended( - sections=sections, - p_val=p_val, - p_cov=p_cov, + mcparams = self.monte_carlo_double_ended( + result=result, st_var=st_var, ast_var=ast_var, rst_var=rst_var, @@ -2469,7 +2457,7 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): ) mcparams[label + "_avgsec"] = ( ("x", time_dim2), - self[label].sel(**{"time": ci_avg_time_sel}).data, + result[label].sel(**{"time": ci_avg_time_sel}).data, ) mcparams[label + "_mc_set"] = ( ("mc", "x", time_dim2), @@ -2485,7 +2473,7 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): ) mcparams[label + "_avgsec"] = ( ("x", time_dim2), - self[label].isel(**{"time": ci_avg_time_isel}).data, + result[label].isel(**{"time": ci_avg_time_isel}).data, ) mcparams[label + "_mc_set"] = ( ("mc", "x", time_dim2), @@ -2501,7 +2489,7 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): ) mcparams[label + "_avgsec"] = ( (x_dim2, "time"), - self[label].sel(x=ci_avg_x_sel).data, + result[label].sel(x=ci_avg_x_sel).data, ) mcparams[label + "_mc_set"] = ( ("mc", x_dim2, "time"), @@ -2517,14 +2505,14 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): ) mcparams[label + "_avgsec"] = ( (x_dim2, time_dim2), - self[label].isel(x=ci_avg_x_isel).data, + result[label].isel(x=ci_avg_x_isel).data, ) mcparams[label + "_mc_set"] = ( ("mc", x_dim2, time_dim2), mcparams[label + "_mc_set"].isel(x=ci_avg_x_isel).data, ) else: - mcparams[label + "_avgsec"] = self[label] + mcparams[label + "_avgsec"] = result[label] x_dim2 = "x" time_dim2 = "time" @@ -2797,5 +2785,4 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): print(f"Removed from results: {k}") del out[k] - self.update(out) - pass + return out diff --git a/src/dtscalibration/io.py b/src/dtscalibration/io.py index 0ee3483a..8bc312e0 100644 --- a/src/dtscalibration/io.py +++ b/src/dtscalibration/io.py @@ -6,6 +6,7 @@ import numpy as np import xarray as xr +from xarray import Dataset from dtscalibration.io_utils import apsensing_xml_version_check from dtscalibration.io_utils import read_apsensing_files_routine @@ -41,7 +42,7 @@ def open_datastore( Parameters ---------- - filename_or_obj : str, Path, file or xarray.backends.*DataStore + filename_or_obj : str, Path, file or xarray.backends.*Dataset Strings and Path objects are interpreted as a path to a netCDF file or an OpenDAP URL and opened with python-netCDF4, unless the filename ends with .gz, in which case the file is gunzipped and opened with @@ -137,7 +138,7 @@ def open_datastore( drop_variables=drop_variables, backend_kwargs=backend_kwargs, ) as ds_xr: - ds = DataStore( + ds = Dataset( data_vars=ds_xr.data_vars, coords=ds_xr.coords, attrs=ds_xr.attrs, @@ -187,7 +188,7 @@ def open_mf_datastore( assert paths, "No files match found with: " + path with open_mfdataset(paths=paths, combine=combine, **kwargs) as xds: - ds = DataStore(data_vars=xds.data_vars, coords=xds.coords, attrs=xds.attrs) + ds = Dataset(data_vars=xds.data_vars, coords=xds.coords, attrs=xds.attrs) # to support deprecated st_labels ds = ds.rename_labels(assertion=False) @@ -232,11 +233,11 @@ def read_silixa_files( load_in_memory : {'auto', True, False} If 'auto' the Stokes data is only loaded to memory for small files kwargs : dict-like, optional - keyword-arguments are passed to DataStore initialization + keyword-arguments are passed to Dataset initialization Returns ------- - datastore : DataStore + datastore : Dataset The newly created datastore. """ @@ -284,7 +285,7 @@ def read_silixa_files( "Silixa xml version " + f"{xml_version} not implemented" ) - ds = DataStore(data_vars=data_vars, coords=coords, attrs=attrs, **kwargs) + ds = Dataset(data_vars=data_vars, coords=coords, attrs=attrs, **kwargs) return ds @@ -310,11 +311,11 @@ def read_sensortran_files( silent : bool If set tot True, some verbose texts are not printed to stdout/screen kwargs : dict-like, optional - keyword-arguments are passed to DataStore initialization + keyword-arguments are passed to Dataset initialization Returns ------- - datastore : DataStore + datastore : Dataset The newly created datastore. """ @@ -349,7 +350,7 @@ def read_sensortran_files( "Sensortran binary version " + f"{version} not implemented" ) - ds = DataStore(data_vars=data_vars, coords=coords, attrs=attrs, **kwargs) + ds = Dataset(data_vars=data_vars, coords=coords, attrs=attrs, **kwargs) return ds @@ -386,7 +387,7 @@ def read_apsensing_files( load_in_memory : {'auto', True, False} If 'auto' the Stokes data is only loaded to memory for small files kwargs : dict-like, optional - keyword-arguments are passed to DataStore initialization + keyword-arguments are passed to Dataset initialization Notes ----- @@ -394,7 +395,7 @@ def read_apsensing_files( Returns ------- - datastore : DataStore + datastore : Dataset The newly created datastore. """ if not file_ext == "*.xml": @@ -435,7 +436,7 @@ def read_apsensing_files( load_in_memory=load_in_memory, ) - ds = DataStore(data_vars=data_vars, coords=coords, attrs=attrs, **kwargs) + ds = Dataset(data_vars=data_vars, coords=coords, attrs=attrs, **kwargs) return ds @@ -479,7 +480,7 @@ def read_sensornet_files( device. If left to `None`, it is approximated with `x[-1] - add_internal_fiber_length`. kwargs : dict-like, optional - keyword-arguments are passed to DataStore initialization + keyword-arguments are passed to Dataset initialization Notes ----- @@ -489,7 +490,7 @@ def read_sensornet_files( Returns ------- - datastore : DataStore + datastore : Dataset The newly created datastore. """ if filepathlist is None: @@ -557,5 +558,5 @@ def read_sensornet_files( flip_reverse_measurements=flip_reverse_measurements, ) - ds = DataStore(data_vars=data_vars, coords=coords, attrs=attrs, **kwargs) + ds = Dataset(data_vars=data_vars, coords=coords, attrs=attrs, **kwargs) return ds diff --git a/tests/test_averaging.py b/tests/test_averaging.py new file mode 100644 index 00000000..557e5907 --- /dev/null +++ b/tests/test_averaging.py @@ -0,0 +1,192 @@ +import os + +import numpy as np +from dtscalibration import read_silixa_files +from dtscalibration.datastore_accessor import DtsAccessor # noqa: F401 + +np.random.seed(0) + +fn = [ + "channel 1_20170921112245510.xml", + "channel 1_20170921112746818.xml", + "channel 1_20170921112746818.xml", +] +fn_single = [ + "channel 2_20180504132202074.xml", + "channel 2_20180504132232903.xml", + "channel 2_20180504132303723.xml", +] + +if 1: + # working dir is tests + wd = os.path.dirname(os.path.abspath(__file__)) + data_dir_single_ended = os.path.join(wd, "data", "single_ended") + data_dir_double_ended = os.path.join(wd, "data", "double_ended") + data_dir_double_ended2 = os.path.join(wd, "data", "double_ended2") + +else: + # working dir is src + data_dir_single_ended = os.path.join("..", "..", "tests", "data", "single_ended") + data_dir_double_ended = os.path.join("..", "..", "tests", "data", "double_ended") + data_dir_double_ended2 = os.path.join("..", "..", "tests", "data", "double_ended2") + + +def assert_almost_equal_verbose(actual, desired, verbose=False, **kwargs): + """Print the actual precision decimals""" + err = np.abs(actual - desired).max() + dec = -np.ceil(np.log10(err)) + + if not (np.isfinite(dec)): + dec = 18.0 + + m = "\n>>>>>The actual precision is: " + str(float(dec)) + + if verbose: + print(m) + + desired2 = np.broadcast_to(desired, actual.shape) + np.testing.assert_almost_equal(actual, desired2, err_msg=m, **kwargs) + pass + + +def test_average_measurements_single_ended(): + filepath = data_dir_single_ended + + ds_ = read_silixa_files(directory=filepath, timezone_netcdf="UTC", file_ext="*.xml") + + ds = ds_.sel(x=slice(0, 100)) # only calibrate parts of the fiber + sections = {"probe2Temperature": [slice(6.0, 14.0)]} # warm bath + + st_var, ast_var = 5.0, 5.0 + + out = ds.dts.calibrate_single_ended( + sections=sections, st_var=st_var, ast_var=ast_var, method="wls", solver="sparse" + ) + ds.dts.average_monte_carlo_single_ended( + result=out, + st_var=st_var, + ast_var=ast_var, + conf_ints=[2.5, 97.5], + mc_sample_size=50, # <- choose a much larger sample size + ci_avg_x_flag1=True, + ci_avg_x_sel=slice(6.0, 14.0), + ) + def get_section_indices(x_da, sec): + """Returns the x-indices of the section. `sec` is a slice.""" + xis = x_da.astype(int) * 0 + np.arange(x_da.size, dtype=int) + return xis.sel(x=sec).values + ix = get_section_indices(ds.x, slice(6, 14)) + ds.dts.average_monte_carlo_single_ended( + result=out, + st_var=st_var, + ast_var=ast_var, + conf_ints=[2.5, 97.5], + mc_sample_size=50, # <- choose a much larger sample size + ci_avg_x_flag2=True, + ci_avg_x_isel=ix, + ) + sl = slice( + np.datetime64("2018-05-04T12:22:17.710000000"), + np.datetime64("2018-05-04T12:22:47.702000000"), + ) + ds.dts.average_monte_carlo_single_ended( + result=out, + st_var=st_var, + ast_var=ast_var, + conf_ints=[2.5, 97.5], + mc_sample_size=50, # <- choose a much larger sample size + ci_avg_time_flag1=True, + ci_avg_time_flag2=False, + ci_avg_time_sel=sl, + ) + ds.dts.average_monte_carlo_single_ended( + result=out, + st_var=st_var, + ast_var=ast_var, + conf_ints=[2.5, 97.5], + mc_sample_size=50, # <- choose a much larger sample size + ci_avg_time_flag1=False, + ci_avg_time_flag2=True, + ci_avg_time_isel=range(3), + ) + pass + + +def test_average_measurements_double_ended(): + filepath = data_dir_double_ended2 + + ds_ = read_silixa_files(directory=filepath, timezone_netcdf="UTC", file_ext="*.xml") + + ds = ds_.sel(x=slice(0, 100)) # only calibrate parts of the fiber + sections = { + "probe1Temperature": [slice(7.5, 17.0), slice(70.0, 80.0)], # cold bath + "probe2Temperature": [slice(24.0, 34.0), slice(85.0, 95.0)], # warm bath + } + + st_var, ast_var, rst_var, rast_var = 5.0, 5.0, 5.0, 5.0 + + out = ds.dts.calibration_double_ended( + sections=sections, + st_var=st_var, + ast_var=ast_var, + rst_var=rst_var, + rast_var=rast_var, + method="wls", + solver="sparse", + ) + ds.dts.average_monte_carlo_double_ended( + result=out, + st_var=st_var, + ast_var=ast_var, + rst_var=rst_var, + rast_var=rast_var, + conf_ints=[2.5, 97.5], + mc_sample_size=50, # <- choose a much larger sample size + ci_avg_x_flag1=True, + ci_avg_x_sel=slice(6, 10), + ) + def get_section_indices(x_da, sec): + """Returns the x-indices of the section. `sec` is a slice.""" + xis = x_da.astype(int) * 0 + np.arange(x_da.size, dtype=int) + return xis.sel(x=sec).values + ix = get_section_indices(ds.x, slice(6, 10)) + ds.dts.average_monte_carlo_double_ended( + result=out, + st_var=st_var, + ast_var=ast_var, + rst_var=rst_var, + rast_var=rast_var, + conf_ints=[2.5, 97.5], + mc_sample_size=50, # <- choose a much larger sample size + ci_avg_x_flag2=True, + ci_avg_x_isel=ix, + ) + sl = slice( + np.datetime64("2018-03-28T00:40:54.097000000"), + np.datetime64("2018-03-28T00:41:12.084000000"), + ) + ds.dts.average_monte_carlo_double_ended( + result=out, + st_var=st_var, + ast_var=ast_var, + rst_var=rst_var, + rast_var=rast_var, + conf_ints=[2.5, 97.5], + mc_sample_size=50, # <- choose a much larger sample size + ci_avg_time_flag1=True, + ci_avg_time_flag2=False, + ci_avg_time_sel=sl, + ) + ds.dts.average_monte_carlo_double_ended( + result=out, + st_var=st_var, + ast_var=ast_var, + rst_var=rst_var, + rast_var=rast_var, + conf_ints=[2.5, 97.5], + mc_sample_size=50, # <- choose a much larger sample size + ci_avg_time_flag1=False, + ci_avg_time_flag2=True, + ci_avg_time_isel=range(3), + ) + pass diff --git a/tests/test_dtscalibration.py b/tests/test_dtscalibration.py index 4dd0fed3..6e141193 100644 --- a/tests/test_dtscalibration.py +++ b/tests/test_dtscalibration.py @@ -2549,148 +2549,3 @@ def test_calibrate_wls_solver_procedures(): pass -def test_average_measurements_single_ended(): - filepath = data_dir_single_ended - - ds_ = read_silixa_files(directory=filepath, timezone_netcdf="UTC", file_ext="*.xml") - - ds = ds_.sel(x=slice(0, 100)) # only calibrate parts of the fiber - sections = {"probe2Temperature": [slice(6.0, 14.0)]} # warm bath - - st_var, ast_var = 5.0, 5.0 - - out = ds.dts.calibrate_single_ended( - sections=sections, st_var=st_var, ast_var=ast_var, method="wls", solver="sparse" - ) - ds.average_single_ended( - p_val="p_val", - p_cov="p_cov", - st_var=st_var, - ast_var=ast_var, - conf_ints=[2.5, 97.5], - mc_sample_size=50, # <- choose a much larger sample size - ci_avg_x_flag1=True, - ci_avg_x_sel=slice(6.0, 14.0), - ) - ix = ds.get_section_indices(slice(6, 14)) - ds.average_single_ended( - p_val="p_val", - p_cov="p_cov", - st_var=st_var, - ast_var=ast_var, - conf_ints=[2.5, 97.5], - mc_sample_size=50, # <- choose a much larger sample size - ci_avg_x_flag2=True, - ci_avg_x_isel=ix, - ) - sl = slice( - np.datetime64("2018-05-04T12:22:17.710000000"), - np.datetime64("2018-05-04T12:22:47.702000000"), - ) - ds.average_single_ended( - p_val="p_val", - p_cov="p_cov", - st_var=st_var, - ast_var=ast_var, - conf_ints=[2.5, 97.5], - mc_sample_size=50, # <- choose a much larger sample size - ci_avg_time_flag1=True, - ci_avg_time_flag2=False, - ci_avg_time_sel=sl, - ) - ds.average_single_ended( - p_val="p_val", - p_cov="p_cov", - st_var=st_var, - ast_var=ast_var, - conf_ints=[2.5, 97.5], - mc_sample_size=50, # <- choose a much larger sample size - ci_avg_time_flag1=False, - ci_avg_time_flag2=True, - ci_avg_time_isel=range(3), - ) - pass - - -def test_average_measurements_double_ended(): - filepath = data_dir_double_ended2 - - ds_ = read_silixa_files(directory=filepath, timezone_netcdf="UTC", file_ext="*.xml") - - ds = ds_.sel(x=slice(0, 100)) # only calibrate parts of the fiber - sections = { - "probe1Temperature": [slice(7.5, 17.0), slice(70.0, 80.0)], # cold bath - "probe2Temperature": [slice(24.0, 34.0), slice(85.0, 95.0)], # warm bath - } - - st_var, ast_var, rst_var, rast_var = 5.0, 5.0, 5.0, 5.0 - - out = ds.dts.calibration_double_ended( - sections=sections, - st_var=st_var, - ast_var=ast_var, - rst_var=rst_var, - rast_var=rast_var, - method="wls", - solver="sparse", - ) - ds.average_double_ended( - sections=sections, - p_val="p_val", - p_cov="p_cov", - st_var=st_var, - ast_var=ast_var, - rst_var=rst_var, - rast_var=rast_var, - conf_ints=[2.5, 97.5], - mc_sample_size=50, # <- choose a much larger sample size - ci_avg_x_flag1=True, - ci_avg_x_sel=slice(6, 10), - ) - ix = ds.get_section_indices(slice(6, 10)) - ds.average_double_ended( - sections=sections, - p_val="p_val", - p_cov="p_cov", - st_var=st_var, - ast_var=ast_var, - rst_var=rst_var, - rast_var=rast_var, - conf_ints=[2.5, 97.5], - mc_sample_size=50, # <- choose a much larger sample size - ci_avg_x_flag2=True, - ci_avg_x_isel=ix, - ) - sl = slice( - np.datetime64("2018-03-28T00:40:54.097000000"), - np.datetime64("2018-03-28T00:41:12.084000000"), - ) - ds.average_double_ended( - sections=sections, - p_val="p_val", - p_cov="p_cov", - st_var=st_var, - ast_var=ast_var, - rst_var=rst_var, - rast_var=rast_var, - conf_ints=[2.5, 97.5], - mc_sample_size=50, # <- choose a much larger sample size - ci_avg_time_flag1=True, - ci_avg_time_flag2=False, - ci_avg_time_sel=sl, - ) - ds.average_double_ended( - sections=sections, - p_val="p_val", - p_cov="p_cov", - st_var=st_var, - ast_var=ast_var, - rst_var=rst_var, - rast_var=rast_var, - conf_ints=[2.5, 97.5], - mc_sample_size=50, # <- choose a much larger sample size - ci_avg_time_flag1=False, - ci_avg_time_flag2=True, - ci_avg_time_isel=range(3), - ) - pass diff --git a/tests/test_variance_stokes.py b/tests/test_variance_stokes.py index d37cad45..81a8e841 100644 --- a/tests/test_variance_stokes.py +++ b/tests/test_variance_stokes.py @@ -570,22 +570,22 @@ def test_double_ended_variance_estimate_synthetic(): method="wls", solver="sparse", ) + out["cold"] = ds.cold + out["warm"] = ds.warm assert_almost_equal_verbose(out["tmpf"].mean(), 12.0, decimal=2) assert_almost_equal_verbose(out["tmpb"].mean(), 12.0, decimal=3) # Calibrated variance stdsf1 = out.dts.ufunc_per_section( - sections=sections, label="tmpf", func=np.std, temp_err=True, calc_per="stretch" + sections=sections, label="tmpf", func=np.std, temp_err=True, calc_per="stretch", suppress_section_validation=True, ) stdsb1 = out.dts.ufunc_per_section( - sections=sections, label="tmpb", func=np.std, temp_err=True, calc_per="stretch" + sections=sections, label="tmpb", func=np.std, temp_err=True, calc_per="stretch", suppress_section_validation=True, ) out2 = ds.dts.monte_carlo_double_ended( - sections=sections, - p_val="p_val", - p_cov="p_cov", + result=out, st_var=mst_var, ast_var=mast_var, rst_var=mrst_var, @@ -595,6 +595,9 @@ def test_double_ended_variance_estimate_synthetic(): da_random_state=state, ) + out2["cold"] = ds.cold + out2["warm"] = ds.warm + # Use a single timestep to better check if the parameter uncertainties propagate ds1 = out2.isel(time=1) # Estimated VAR @@ -604,6 +607,7 @@ def test_double_ended_variance_estimate_synthetic(): func=np.mean, temp_err=False, calc_per="stretch", + suppress_section_validation=True, ) stdsb2 = ds1.dts.ufunc_per_section( sections=sections, @@ -611,6 +615,7 @@ def test_double_ended_variance_estimate_synthetic(): func=np.mean, temp_err=False, calc_per="stretch", + suppress_section_validation=True, ) for (_, v1), (_, v2) in zip(stdsf1.items(), stdsf2.items()): From 18250501c1916f7c6b9e519141611c866eabb4e8 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Mon, 16 Oct 2023 12:16:52 +0200 Subject: [PATCH 31/66] Passing all variance tests --- src/dtscalibration/variance_stokes.py | 2 +- tests/test_variance_stokes.py | 30 +++++++++++++++------------ 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/dtscalibration/variance_stokes.py b/src/dtscalibration/variance_stokes.py index eb09041a..7be69274 100644 --- a/src/dtscalibration/variance_stokes.py +++ b/src/dtscalibration/variance_stokes.py @@ -465,7 +465,7 @@ def variance_stokes_linear( assert st.dims[0] == "x", "Stokes are transposed" _, resid = variance_stokes_constant( - sections=sections, st=st, reshape_residuals=False + sections=sections, st=st, acquisitiontime=acquisitiontime, reshape_residuals=False ) ix_sec = ufunc_per_section_helper(sections=sections, x_coords=st.coords["x"], calc_per="all") diff --git a/tests/test_variance_stokes.py b/tests/test_variance_stokes.py index 81a8e841..a52eadf7 100644 --- a/tests/test_variance_stokes.py +++ b/tests/test_variance_stokes.py @@ -695,11 +695,10 @@ def test_single_ended_variance_estimate_synthetic(): "warm": [slice(0.5 * cable_len, cable_len)], } - st_label = "st" - ast_label = "ast" - - mst_var, _ = ds.variance_stokes(st_label=st_label, sections=sections) - mast_var, _ = ds.variance_stokes(st_label=ast_label, sections=sections) + mst_var, _ = variance_stokes_constant(ds.dts.st, sections, ds.dts.acquisitiontime_fw, reshape_residuals=False) + mast_var, _ = variance_stokes_constant(ds.dts.ast, sections, ds.dts.acquisitiontime_fw, reshape_residuals=False) + # mrst_var, _ = variance_stokes_constant(ds.dts.rst, sections, ds.dts.acquisitiontime_bw, reshape_residuals=False) + # mrast_var, _ = variance_stokes_constant(ds.dts.rast, sections, ds.dts.acquisitiontime_bw, reshape_residuals=False) mst_var = float(mst_var) mast_var = float(mast_var) @@ -711,6 +710,8 @@ def test_single_ended_variance_estimate_synthetic(): method="wls", solver="sparse", ) + out["cold"] = ds.cold + out["warm"] = ds.warm out2 = ds.dts.monte_carlo_single_ended( result=out, @@ -720,6 +721,8 @@ def test_single_ended_variance_estimate_synthetic(): mc_sample_size=50, da_random_state=state, ) + out2["cold"] = ds.cold + out2["warm"] = ds.warm # Calibrated variance stdsf1 = out.dts.ufunc_per_section( @@ -792,7 +795,7 @@ def test_variance_of_stokes_synthetic(): y += stats.norm.rvs(size=y.size, scale=yvar**0.5).reshape(y.shape) - ds = xr.Dataset( + ds = Dataset( { "st": (["x", "time"], y), "probe1Temperature": (["time"], range(nt)), @@ -803,7 +806,7 @@ def test_variance_of_stokes_synthetic(): ) sections = {"probe1Temperature": [slice(0.0, 20.0)]} - test_st_var, _ = variance_stokes_constant(st=ds["st"], sections=sections) + test_st_var, _ = variance_stokes_constant(ds.dts.st, sections, ds.dts.acquisitiontime_fw, reshape_residuals=False) assert_almost_equal_verbose(test_st_var, yvar, decimal=1) pass @@ -833,7 +836,7 @@ def test_variance_of_stokes_linear_synthetic(): # size=y.size, scale=(var_slope * c_no_noise) ** 0.5, ) - ds = DataStore( + ds = Dataset( { "st": (["x", "time"], c_no_noise), "c_lin_var_through_zero": (["x", "time"], c_lin_var_through_zero), @@ -845,7 +848,7 @@ def test_variance_of_stokes_linear_synthetic(): ) sections = {"probe1Temperature": [slice(0.0, 20.0)]} - test_st_var, _ = variance_stokes_constant(st=ds["st"], sections=sections) + test_st_var, _ = variance_stokes_constant(ds.dts.st, sections, ds.dts.acquisitiontime_fw, reshape_residuals=False) # If fit is forced through zero. Only Poisson distributed noise ( @@ -858,6 +861,7 @@ def test_variance_of_stokes_linear_synthetic(): ) = variance_stokes_linear( st=ds["c_lin_var_through_zero"], sections=sections, + acquisitiontime=ds.dts.acquisitiontime_fw, nbin=10, through_zero=True, plot_fit=False, @@ -873,7 +877,7 @@ def test_variance_of_stokes_linear_synthetic(): resid, var_fun, ) = variance_stokes_linear( - st=ds["c_lin_var_through_zero"], sections=sections, nbin=100, through_zero=False + st=ds["c_lin_var_through_zero"], sections=sections, acquisitiontime=ds.dts.acquisitiontime_fw, nbin=100, through_zero=False ) assert_almost_equal_verbose(slope, var_slope, decimal=3) assert_almost_equal_verbose(offset, 0.0, decimal=0) @@ -891,11 +895,11 @@ def test_exponential_variance_of_stokes(): "probe2Temperature": [slice(24.0, 34.0), slice(85.0, 95.0)], # warm bath } - I_var, _ = variance_stokes_exponential(st=ds["st"], sections=sections) + I_var, _ = variance_stokes_exponential(st=ds["st"], sections=sections, acquisitiontime=ds.dts.acquisitiontime_fw) assert_almost_equal_verbose(I_var, correct_var, decimal=5) ds_dask = ds.chunk(chunks={}) - I_var, _ = variance_stokes_exponential(st=ds_dask["st"], sections=sections) + I_var, _ = variance_stokes_exponential(st=ds_dask["st"], sections=sections, acquisitiontime=ds.dts.acquisitiontime_fw) assert_almost_equal_verbose(I_var, correct_var, decimal=5) pass @@ -933,7 +937,7 @@ def test_exponential_variance_of_stokes_synthetic(): ) sections = {"probe1Temperature": [slice(0.0, 20.0)]} - test_st_var, _ = variance_stokes_exponential(st=ds["st"], sections=sections) + test_st_var, _ = variance_stokes_exponential(st=ds["st"], sections=sections, acquisitiontime=ds.dts.acquisitiontime_fw) assert_almost_equal_verbose(test_st_var, yvar, decimal=1) pass From 74a0a8b072233fc0ca5e33e77ab8c146d1e647fa Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Mon, 16 Oct 2023 14:50:56 +0200 Subject: [PATCH 32/66] Passing all tests except lint and examples --- src/dtscalibration/__init__.py | 6 - src/dtscalibration/datastore_accessor.py | 8 +- src/dtscalibration/datastore_utils.py | 64 ++------ src/dtscalibration/io.py | 182 ----------------------- tests/test_datastore.py | 89 ++++------- 5 files changed, 47 insertions(+), 302 deletions(-) diff --git a/src/dtscalibration/__init__.py b/src/dtscalibration/__init__.py index 66fdb7ac..856a02b9 100644 --- a/src/dtscalibration/__init__.py +++ b/src/dtscalibration/__init__.py @@ -1,11 +1,8 @@ -from dtscalibration.datastore import DataStore from dtscalibration.datastore_utils import check_dims from dtscalibration.datastore_utils import get_netcdf_encoding from dtscalibration.datastore_utils import merge_double_ended from dtscalibration.datastore_utils import shift_double_ended from dtscalibration.datastore_utils import suggest_cable_shift_double_ended -from dtscalibration.io import open_datastore -from dtscalibration.io import open_mf_datastore from dtscalibration.io import read_apsensing_files from dtscalibration.io import read_sensornet_files from dtscalibration.io import read_sensortran_files @@ -18,9 +15,6 @@ __version__ = "2.0.0" __all__ = [ - "DataStore", - "open_datastore", - "open_mf_datastore", "read_apsensing_files", "read_sensornet_files", "read_sensortran_files", diff --git a/src/dtscalibration/datastore_accessor.py b/src/dtscalibration/datastore_accessor.py index 255f9b5a..3227ca74 100644 --- a/src/dtscalibration/datastore_accessor.py +++ b/src/dtscalibration/datastore_accessor.py @@ -46,7 +46,7 @@ def __repr__(self): # 'xarray' is prepended. so we remove it and add 'dtscalibration' s = xr.core.formatting.dataset_repr(self._obj) name_module = type(self._obj).__name__ - preamble_new = "" % name_module + preamble_new = f"" # Add sections to new preamble preamble_new += "\nSections:" @@ -268,6 +268,12 @@ def get_default_encoding(self, time_chunks_from_key=None): return encoding + def get_timeseries_keys(self): + """ + Returns a list of the keys of the time series variables. + """ + return [k for k, v in self._obj.data_vars.items() if v.dims == ("time",)] + def ufunc_per_section( self, sections=None, diff --git a/src/dtscalibration/datastore_utils.py b/src/dtscalibration/datastore_utils.py index 41709c32..25c86c9d 100644 --- a/src/dtscalibration/datastore_utils.py +++ b/src/dtscalibration/datastore_utils.py @@ -459,45 +459,8 @@ def check_deprecated_kwargs(kwargs): pass -# def check_timestep_allclose(ds: "DataStore", eps: float = 0.05) -> None: -# """ -# Check if all timesteps are of equal size. For now it is not possible to calibrate -# over timesteps if the acquisition time of timesteps varies, as the Stokes variance -# would change over time. - -# The acquisition time is stored for single ended measurements in userAcquisitionTime, -# for double ended measurements in userAcquisitionTimeFW and userAcquisitionTimeBW. - -# Parameters -# ---------- -# ds : DataStore -# eps : float -# Default accepts 1% of relative variation between min and max acquisition time. - -# Returns -# ------- -# """ -# dt = ds["userAcquisitionTimeFW"].data -# dtmin = dt.min() -# dtmax = dt.max() -# dtavg = (dtmin + dtmax) / 2 -# assert (dtmax - dtmin) / dtavg < eps, ( -# "Acquisition time is Forward channel not equal for all time steps" -# ) - -# if "userAcquisitionTimeBW" in ds: -# dt = ds["userAcquisitionTimeBW"].data -# dtmin = dt.min() -# dtmax = dt.max() -# dtavg = (dtmin + dtmax) / 2 -# assert (dtmax - dtmin) / dtavg < eps, ( -# "Acquisition time Backward channel is not equal for all time steps" -# ) -# pass - - def get_netcdf_encoding( - ds: "DataStore", zlib: bool = True, complevel: int = 5, **kwargs + ds: xr.Dataset, zlib: bool = True, complevel: int = 5, **kwargs ) -> dict: """Get default netcdf compression parameters. The same for each data variable. @@ -788,12 +751,12 @@ def get_params_from_pval_single_ended( def merge_double_ended( - ds_fw: "DataStore", - ds_bw: "DataStore", + ds_fw: xr.Dataset, + ds_bw: xr.Dataset, cable_length: float, plot_result: bool = True, verbose: bool = True, -) -> "DataStore": +) -> xr.Dataset: """ Some measurements are not set up on the DTS-device as double-ended meausurements. This means that the two channels have to be merged manually. @@ -858,11 +821,11 @@ def merge_double_ended( def merge_double_ended_times( - ds_fw: "DataStore", - ds_bw: "DataStore", + ds_fw: xr.Dataset, + ds_bw: xr.Dataset, verify_timedeltas: bool = True, verbose: bool = True, -) -> tuple["DataStore", "DataStore"]: +) -> tuple[xr.Dataset, xr.Dataset]: """Helper for `merge_double_ended()` to deal with missing measurements. The number of measurements of the forward and backward channels might get out of sync if the device shuts down before the measurement of the last channel @@ -997,8 +960,8 @@ def merge_double_ended_times( def shift_double_ended( - ds: "DataStore", i_shift: int, verbose: bool = True -) -> "DataStore": + ds: xr.Dataset, i_shift: int, verbose: bool = True +) -> xr.Dataset: """ The cable length was initially configured during the DTS measurement. For double ended measurements it is important to enter the correct length so that the forward channel and the @@ -1031,8 +994,6 @@ def shift_double_ended( ds2 : DataStore object With a shifted x-axis """ - from dtscalibration import DataStore - assert isinstance(i_shift, (int, np.integer)) nx = ds.x.size @@ -1074,11 +1035,11 @@ def shift_double_ended( if not_included and verbose: print("I dont know what to do with the following data", not_included) - return DataStore(data_vars=d2_data, coords=d2_coords, attrs=ds.attrs) + return xr.Dataset(data_vars=d2_data, coords=d2_coords, attrs=ds.attrs) def suggest_cable_shift_double_ended( - ds: "DataStore", + ds: xr.Dataset, irange: npt.NDArray[np.int_], plot_result: bool = True, **fig_kwargs, @@ -1107,8 +1068,7 @@ def suggest_cable_shift_double_ended( Parameters ---------- - ds : DataSore object - DataStore object that needs to be shifted + ds : Xarray Dataset irange : array-like a numpy array with data of type int. Containing all the shift index that are tested. diff --git a/src/dtscalibration/io.py b/src/dtscalibration/io.py index 8bc312e0..f06ca683 100644 --- a/src/dtscalibration/io.py +++ b/src/dtscalibration/io.py @@ -20,188 +20,6 @@ from dtscalibration.io_utils import ziphandle_to_filepathlist -def open_datastore( - filename_or_obj, - group=None, - decode_cf=True, - mask_and_scale=None, - decode_times=True, - concat_characters=True, - decode_coords=True, - engine=None, - chunks=None, - lock=None, - cache=None, - drop_variables=None, - backend_kwargs=None, - load_in_memory=False, - **kwargs, -): - """Load and decode a datastore from a file or file-like object. Most - arguments are passed to xarray.open_dataset(). - - Parameters - ---------- - filename_or_obj : str, Path, file or xarray.backends.*Dataset - Strings and Path objects are interpreted as a path to a netCDF file - or an OpenDAP URL and opened with python-netCDF4, unless the filename - ends with .gz, in which case the file is gunzipped and opened with - scipy.io.netcdf (only netCDF3 supported). File-like objects are opened - with scipy.io.netcdf (only netCDF3 supported). - group : str, optional - Path to the netCDF4 group in the given file to open (only works for - netCDF4 files). - decode_cf : bool, optional - Whether to decode these variables, assuming they were saved according - to CF conventions. - mask_and_scale : bool, optional - If True, replace array values equal to `_FillValue` with NA and scale - values according to the formula `original_values * scale_factor + - add_offset`, where `_FillValue`, `scale_factor` and `add_offset` are - taken from variable attributes (if they exist). If the `_FillValue` or - `missing_value` attribute contains multiple values a warning will be - issued and all array values matching one of the multiple values will - be replaced by NA. mask_and_scale defaults to True except for the - pseudonetcdf backend. - decode_times : bool, optional - If True, decode times encoded in the standard NetCDF datetime format - into datetime objects. Otherwise, leave them encoded as numbers. - concat_characters : bool, optional - If True, concatenate along the last dimension of character arrays to - form string arrays. Dimensions will only be concatenated over (and - removed) if they have no corresponding variable and if they are only - used as the last dimension of character arrays. - decode_coords : bool, optional - If True, decode the 'coordinates' attribute to identify coordinates in - the resulting dataset. - engine : {'netcdf4', 'scipy', 'pydap', 'h5netcdf', 'pynio', - 'pseudonetcdf'}, optional - Engine to use when reading files. If not provided, the default engine - is chosen based on available dependencies, with a preference for - 'netcdf4'. - chunks : int or dict, optional - If chunks is provided, it used to load the new dataset into dask - arrays. ``chunks={}`` loads the dataset with dask using a single - chunk for all arrays. - lock : False, True or threading.Lock, optional - If chunks is provided, this argument is passed on to - :py:func:`dask.array.from_array`. By default, a global lock is - used when reading data from netCDF files with the netcdf4 and h5netcdf - engines to avoid issues with concurrent access when using dask's - multithreaded backend. - cache : bool, optional - If True, cache data loaded from the underlying datastore in memory as - NumPy arrays when accessed to avoid reading from the underlying data- - store multiple times. Defaults to True unless you specify the `chunks` - argument to use dask, in which case it defaults to False. Does not - change the behavior of coordinates corresponding to dimensions, which - always load their data from disk into a ``pandas.Index``. - drop_variables: string or iterable, optional - A variable or list of variables to exclude from being parsed from the - dataset. This may be useful to drop variables with problems or - inconsistent values. - backend_kwargs: dictionary, optional - A dictionary of keyword arguments to pass on to the backend. This - may be useful when backend options would improve performance or - allow user control of dataset processing. - - Returns - ------- - dataset : Dataset - The newly created dataset. - - See Also - -------- - xarray.open_dataset - xarray.load_dataset - """ - - xr_kws = inspect.signature(xr.open_dataset).parameters.keys() - - ds_kwargs = {k: v for k, v in kwargs.items() if k not in xr_kws} - - if chunks is None: - chunks = {} - - with xr.open_dataset( - filename_or_obj, - group=group, - decode_cf=decode_cf, - mask_and_scale=mask_and_scale, - decode_times=decode_times, - concat_characters=concat_characters, - decode_coords=decode_coords, - engine=engine, - chunks=chunks, - lock=lock, - cache=cache, - drop_variables=drop_variables, - backend_kwargs=backend_kwargs, - ) as ds_xr: - ds = Dataset( - data_vars=ds_xr.data_vars, - coords=ds_xr.coords, - attrs=ds_xr.attrs, - **ds_kwargs, - ) - - # to support deprecated st_labels - ds = ds.rename_labels(assertion=False) - - if load_in_memory: - if "cache" in kwargs: - raise TypeError("cache has no effect in this context") - return ds.load() - - else: - return ds - - -def open_mf_datastore( - path=None, paths=None, combine="by_coords", load_in_memory=False, **kwargs -): - """ - Open a datastore from multiple netCDF files. This script assumes the - datastore was split along the time dimension. But only variables with a - time dimension should be concatenated in the time dimension. Other - options from xarray do not support this. - - Parameters - ---------- - combine : {'by_coords', 'nested'}, optional - Leave it at by_coords - path : str - A file path to the stored netcdf files with an asterisk in the - filename to list all. Ensure you have leading zeros in the file - numbering. - paths : list - Define you own list of file paths. - Returns - ------- - dataset : Dataset - The newly created dataset. - """ - from xarray.backends.api import open_mfdataset - - if paths is None: - paths = sorted(glob.glob(path)) - assert paths, "No files match found with: " + path - - with open_mfdataset(paths=paths, combine=combine, **kwargs) as xds: - ds = Dataset(data_vars=xds.data_vars, coords=xds.coords, attrs=xds.attrs) - - # to support deprecated st_labels - ds = ds.rename_labels(assertion=False) - - if load_in_memory: - if "cache" in kwargs: - raise TypeError("cache has no effect in this context") - return ds.load() - - else: - return ds - - def read_silixa_files( filepathlist=None, directory=None, diff --git a/tests/test_datastore.py b/tests/test_datastore.py index 5620f54b..8b7e5461 100644 --- a/tests/test_datastore.py +++ b/tests/test_datastore.py @@ -8,18 +8,21 @@ import dask.array as da import numpy as np import pytest +from xarray import Dataset +import xarray as xr -from dtscalibration import open_datastore -from dtscalibration import open_mf_datastore from dtscalibration import read_apsensing_files from dtscalibration import read_sensornet_files from dtscalibration import read_sensortran_files from dtscalibration import read_silixa_files from dtscalibration.calibration.section_utils import set_sections +from dtscalibration.calibration.section_utils import set_matching_sections from dtscalibration.datastore_utils import merge_double_ended from dtscalibration.datastore_utils import shift_double_ended from dtscalibration.datastore_utils import suggest_cable_shift_double_ended +from dtscalibration.datastore_accessor import DtsAccessor # noqa: F401 + np.random.seed(0) fn = [ @@ -120,24 +123,23 @@ def test_read_data_from_single_file_single_ended(): assert actual_hash == desired_hash, "The data is not read correctly" -def test_empty_construction(): - ds = DataStore() # noqa: F841 - - def test_repr(): - ds = DataStore() - assert "dtscalibration" in str(ds) - assert "Sections" in str(ds) - + ds = Dataset( + { + "st": (["x", "time"], np.ones((100, 5))), + "ast": (["x", "time"], np.ones((100, 5))), + "probe1Temperature": (["time"], range(5)), + "probe2Temperature": (["time"], range(5)), + }, + coords={"x": range(100), "time": range(5)}, + ) -def test_has_sectionattr_upon_creation(): - ds = DataStore() - assert hasattr(ds, "_sections") - assert isinstance(ds._sections, str) + assert "dtscalibration" in str(ds.dts) + assert "Sections" in str(ds.dts) def test_sections_property(): - ds = DataStore( + ds = Dataset( { "st": (["x", "time"], np.ones((100, 5))), "ast": (["x", "time"], np.ones((100, 5))), @@ -155,15 +157,15 @@ def test_sections_property(): "probe1Temperature": [slice(0.0, 17.0), slice(70.0, 80.0)], # cold bath "probe2Temperature": [slice(24.0, 34.0), slice(85.0, 95.0)], # warm bath } - ds = set_sections(ds, sections1) + set_sections(ds, sections1) assert isinstance(ds.attrs["_sections"], str) - assert ds.sections == sections1 - assert ds.sections != sections2 + assert ds.dts.sections == sections1 + assert ds.dts.sections != sections2 # test if accepts singleton numpy arrays - ds = set_sections( + set_sections( ds, { "probe1Temperature": [ @@ -175,7 +177,7 @@ def test_sections_property(): def test_io_sections_property(): - ds = DataStore( + ds = Dataset( { "st": (["x", "time"], np.ones((100, 5))), "ast": (["x", "time"], np.ones((100, 5))), @@ -191,7 +193,7 @@ def test_io_sections_property(): } ds["x"].attrs["units"] = "m" - ds = set_sections(ds, sections) + set_sections(ds, sections) # Create a temporary file to write data to. # 'with' method is used so the file is closed by tempfile @@ -203,14 +205,14 @@ def test_io_sections_property(): ds.to_netcdf(path=temppath) try: - ds2 = open_datastore(temppath) + ds2 = xr.open_dataset(temppath) except ValueError as e: if str(e) != "cannot guess the engine, try passing one explicitly": raise warnings.warn("Could not guess engine, defaulted to netcdf4") - ds2 = open_datastore(temppath, engine="netcdf4") + ds2 = xr.open_datastore(temppath, engine="netcdf4") - assert ds.sections == ds2.sections + assert ds.dts.sections == ds2.dts.sections # Close the datastore so the temp file can be removed ds2.close() @@ -493,41 +495,6 @@ def test_read_sensortran_files(): ) -def test_to_mf_netcdf_open_mf_datastore(): - filepath = data_dir_single_ended - ds = read_silixa_files(directory=filepath, file_ext="*.xml") - - with tempfile.TemporaryDirectory() as tmpdirname: - print("created temporary directory", tmpdirname) - - # work around the effects of deafault encoding. - path = os.path.join(tmpdirname, "ds_merged.nc") - - with read_silixa_files(directory=filepath, file_ext="*.xml") as ds: - ds.to_netcdf(path) - - time.sleep(5) # to ensure all is written on Windows and file released - - with open_datastore(path, load_in_memory=True) as ds1: - # Test saving - ds1 = ds1.chunk({"time": 1}) - ds1.to_mf_netcdf( - folder_path=tmpdirname, - filename_preamble="file_", - filename_extension=".nc", - ) - correct_val = float(ds1.st.sum()) - - time.sleep(2) # to ensure all is written on Windows and file released - - # Test loading - path = os.path.join(tmpdirname, "file_*.nc") - - with open_mf_datastore(path=path, load_in_memory=True) as ds2: - test_val = float(ds2.st.sum()) - np.testing.assert_equal(correct_val, test_val) - - def read_data_from_fp_numpy(fp): """ Read the data from a single Silixa xml file. Using a simple approach @@ -565,7 +532,7 @@ def test_resample_datastore(): ds = read_silixa_files(directory=filepath, timezone_netcdf="UTC", file_ext="*.xml") assert ds.time.size == 3 - ds_resampled = DataStore(ds.resample(time="47S").mean()) + ds_resampled = Dataset(ds.resample(time="47S").mean()) assert ds_resampled.time.size == 2 assert ds_resampled.st.dims == ("x", "time"), ( @@ -580,7 +547,7 @@ def test_timeseries_keys(): filepath = data_dir_single_ended ds = read_silixa_files(directory=filepath, timezone_netcdf="UTC", file_ext="*.xml") - k = ds.timeseries_keys + k = ds.dts.get_timeseries_keys() # no false positive for ki in k: From 68e0d46be59c514768237b5ae6efcafa6de8b748 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Mon, 16 Oct 2023 15:33:09 +0200 Subject: [PATCH 33/66] Undo certain merge changes --- src/dtscalibration/calibrate_utils.py | 20 +++++++++++--------- tests/test_examples.py | 2 +- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/dtscalibration/calibrate_utils.py b/src/dtscalibration/calibrate_utils.py index ddf68f38..e0590d7c 100644 --- a/src/dtscalibration/calibrate_utils.py +++ b/src/dtscalibration/calibrate_utils.py @@ -242,7 +242,7 @@ def calibration_single_ended_solver( # noqa: MC0001 """ # get ix_sec argsort so the sections are in order of increasing x - ix_sec = ds.ufunc_per_section(x_indices=True, calc_per="all") + ix_sec = ds.dts.ufunc_per_section(sections=sections, x_indices=True, calc_per="all") ds_sec = ds.isel(x=ix_sec) x_sec = ds_sec["x"].values @@ -261,8 +261,8 @@ def calibration_single_ended_solver( # noqa: MC0001 p0_est_alpha = np.asarray([485.0] + no * [0.0] + nt * [1.4] + nta * nt * [0.0]) # X \gamma # Eq.34 - cal_ref = ds.ufunc_per_section( - label="st", ref_temp_broadcasted=True, calc_per="all" + cal_ref = ds.dts.ufunc_per_section( + sections=sections, label="st", ref_temp_broadcasted=True, calc_per="all" ) # cal_ref = cal_ref # sort by increasing x data_gamma = 1 / (cal_ref.T.ravel() + 273.15) # gamma @@ -1195,7 +1195,7 @@ def calibration_double_ended_solver( # noqa: MC0001 ------- """ - ix_sec = ds.ufunc_per_section(x_indices=True, calc_per="all") + ix_sec = ds.dts.ufunc_per_section(sections=sections, x_indices=True, calc_per="all") ds_sec = ds.isel(x=ix_sec) ix_alpha_is_zero = ix_sec[0] # per definition of E @@ -1224,7 +1224,7 @@ def calibration_double_ended_solver( # noqa: MC0001 Zero_d, Z_TA_fw, Z_TA_bw, - ) = construct_submatrices(nt, nx_sec, ds, ds.trans_att.values, x_sec) + ) = construct_submatrices(sections, nt, nx_sec, ds, trans_att, x_sec) # y # Eq.41--45 y_F = np.log(ds_sec.st / ds_sec.ast).values.ravel() @@ -1859,7 +1859,9 @@ def construct_submatrices(sections, nt, nx, ds, trans_att, x_sec): # Z \gamma # Eq.47 cal_ref = np.array( - ds.ufunc_per_section(label="st", ref_temp_broadcasted=True, calc_per="all") + ds.dts.ufunc_per_section( + sections=sections, label="st", ref_temp_broadcasted=True, calc_per="all" + ) ) data_gamma = 1 / (cal_ref.ravel() + 273.15) # gamma coord_gamma_row = np.arange(nt * nx, dtype=int) @@ -2240,10 +2242,10 @@ def calc_df_db_double_est(ds, sections, ix_alpha_is_zero, gamma_est): Ibwx0 = np.log( ds.rst.isel(x=ix_alpha_is_zero) / ds.rast.isel(x=ix_alpha_is_zero) ).values - ref_temps_refs = ds.ufunc_per_section( - label="st", ref_temp_broadcasted=True, calc_per="all" + ref_temps_refs = ds.dts.ufunc_per_section( + sections=sections, label="st", ref_temp_broadcasted=True, calc_per="all" ) - ix_sec = ds.ufunc_per_section(x_indices=True, calc_per="all") + ix_sec = ds.dts.ufunc_per_section(sections=sections, x_indices=True, calc_per="all") ref_temps_x0 = ( ref_temps_refs[ix_sec == ix_alpha_is_zero].flatten().compute() + 273.15 ) diff --git a/tests/test_examples.py b/tests/test_examples.py index 0695492e..e2a8793c 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -64,7 +64,7 @@ def _notebook_run(path): def test_ipynb(): file_ext = "*.ipynb" wd = os.path.dirname(os.path.abspath(__file__)) - nb_dir = os.path.join(wd, "..", "examples", "notebooks", file_ext) + nb_dir = os.path.join(wd, "..", "docs", "notebooks", file_ext) filepathlist = glob.glob(nb_dir) for fp in filepathlist: From c207897d208c965eeffa66648beb0822c305cdba Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Mon, 16 Oct 2023 15:52:43 +0200 Subject: [PATCH 34/66] Minor cleanup --- src/dtscalibration/__init__.py | 19 +- src/dtscalibration/datastore.py | 4131 ----------------- src/dtscalibration/datastore_utils.py | 66 - ...{datastore_accessor.py => dts_accessor.py} | 2 +- src/dtscalibration/io/apsensing.py | 4 +- src/dtscalibration/io/datastore.py | 188 - src/dtscalibration/io/sensornet.py | 4 +- src/dtscalibration/io/sensortran.py | 6 +- src/dtscalibration/io/silixa.py | 4 +- tests/test_averaging.py | 2 +- tests/test_datastore.py | 2 +- tests/test_dtscalibration.py | 5 +- tests/test_variance_stokes.py | 2 +- 13 files changed, 17 insertions(+), 4418 deletions(-) delete mode 100644 src/dtscalibration/datastore.py rename src/dtscalibration/{datastore_accessor.py => dts_accessor.py} (99%) delete mode 100644 src/dtscalibration/io/datastore.py diff --git a/src/dtscalibration/__init__.py b/src/dtscalibration/__init__.py index 267f6f68..cc42cb77 100644 --- a/src/dtscalibration/__init__.py +++ b/src/dtscalibration/__init__.py @@ -1,3 +1,4 @@ +from dtscalibration import DtsAccessor from dtscalibration.datastore_utils import check_dims from dtscalibration.datastore_utils import get_netcdf_encoding from dtscalibration.datastore_utils import merge_double_ended @@ -15,6 +16,7 @@ __version__ = "2.0.0" __all__ = [ + "DTSAccessor", "read_apsensing_files", "read_sensornet_files", "read_sensortran_files", @@ -29,19 +31,4 @@ "plot_residuals_reference_sections", "plot_residuals_reference_sections_single", "plot_sigma_report", -] - -# filenames = ['datastore.py', 'datastore_utils.py', 'calibrate_utils.py', -# 'plot.py', 'io_utils.py'] -# filenames = ['plot.py'] -# -# for filename in filenames: -# with open(join(dirname(__file__), filename)) as file: -# node = ast.parse(file.read()) -# -# functions = [n for n in node.body if isinstance(n, ast.FunctionDef)] -# classes = [n for n in node.body if isinstance(n, ast.ClassDef)] -# __all__.extend([i.name for i in functions]) -# -# __all__.sort() -# print(__all__) +] \ No newline at end of file diff --git a/src/dtscalibration/datastore.py b/src/dtscalibration/datastore.py deleted file mode 100644 index e18756ca..00000000 --- a/src/dtscalibration/datastore.py +++ /dev/null @@ -1,4131 +0,0 @@ -import os -import warnings - -import dask -import dask.array as da -import numpy as np -import scipy.stats as sst -import xarray as xr -import yaml - -from dtscalibration.calibrate_utils import calibration_double_ended_helper -from dtscalibration.calibrate_utils import calibration_single_ended_helper -from dtscalibration.calibrate_utils import match_sections -from dtscalibration.calibrate_utils import parse_st_var -from dtscalibration.calibration.section_utils import set_sections -from dtscalibration.calibration.section_utils import validate_sections -from dtscalibration.datastore_utils import ParameterIndexDoubleEnded -from dtscalibration.datastore_utils import ParameterIndexSingleEnded -from dtscalibration.datastore_utils import check_deprecated_kwargs -from dtscalibration.datastore_utils import get_params_from_pval_double_ended -from dtscalibration.datastore_utils import get_params_from_pval_single_ended -from dtscalibration.datastore_utils import ufunc_per_section_helper -from dtscalibration.io.utils import _dim_attrs -from dtscalibration.variance_helpers import variance_stokes_constant_helper -from dtscalibration.variance_helpers import variance_stokes_exponential_helper -from dtscalibration.variance_helpers import variance_stokes_linear_helper - -dtsattr_namelist = ["double_ended_flag"] -dim_attrs = {k: v for kl, v in _dim_attrs.items() for k in kl} -warnings.filterwarnings( - "ignore", message="xarray subclass DataStore should explicitly define __slots__" -) - - -class DataStore(xr.Dataset): - """The data class that stores the measurements, contains calibration - methods to relate Stokes and anti-Stokes to temperature. The user should - never initiate this class directly, but use open_datastore - functions instead. - - Parameters - ---------- - data_vars : dict-like, optional - A mapping from variable names to :py:class:`~xarray.DataArray` - objects, :py:class:`~xarray.Variable` objects or tuples of the - form ``(dims, data[, attrs])`` which can be used as arguments to - create a new ``Variable``. Each dimension must have the same length - in all variables in which it appears. - coords : dict-like, optional - Another mapping in the same form as the `variables` argument, - except the each item is saved on the datastore as a "coordinate". - These variables have an associated meaning: they describe - constant/fixed/independent quantities, unlike the - varying/measured/dependent quantities that belong in `variables`. - Coordinates values may be given by 1-dimensional arrays or scalars, - in which case `dims` do not need to be supplied: 1D arrays will be - assumed to give index values along the dimension with the same - name. - attrs : dict-like, optional - Global attributes to save on this datastore. - sections : Dict[str, List[slice]], optional - Sections for calibration. The dictionary should contain key-var - couples in which the key is the name of the calibration temp time - series. And the var is a list of slice objects as 'slice(start, - stop)'; start and stop in meter (float). - compat : {'broadcast_equals', 'equals', 'identical'}, optional - String indicating how to compare variables of the same name for - potential conflicts when initializing this datastore: - - 'broadcast_equals': all values must be equal when variables are - broadcast against each other to ensure common dimensions. - - 'equals': all values and dimensions must be the same. - - 'identical': all values, dimensions and attributes must be the - same. - - See Also - -------- - dtscalibration.open_datastore : Load (calibrated) measurements from - netCDF-like file - """ - - def __init__(self, *args, autofill_dim_attrs=True, **kwargs): - with warnings.catch_warnings(): - # Filter out nanosecond precision warning: no good way to avoid ATM. - warnings.filterwarnings( - "ignore", - message="Converting non-nanosecond precision timedelta values to nanosecond precision.", - ) - super().__init__(*args, **kwargs) - - # check order of the dimensions of the data_vars - # first 'x' (if in initiated DataStore), then 'time', then the rest - ideal_dim = [] # perfect order dims - all_dim = list(self.dims) - - if all_dim: - if "x" in all_dim: - ideal_dim.append("x") - all_dim.pop(all_dim.index("x")) - - if "time": - if "time" in all_dim: - ideal_dim.append("time") - all_dim.pop(all_dim.index("time")) - - ideal_dim += all_dim - - for name, var in self._variables.items(): - var_dims = tuple( - dim for dim in ideal_dim if dim in (var.dims + (...,)) - ) - self._variables[name] = var.transpose(*var_dims) - - if "trans_att" not in self.coords: - self.set_trans_att(trans_att=[]) - - # Get attributes from dataset - for arg in args: - if isinstance(arg, xr.Dataset): - self.attrs = arg.attrs - - # Add attributes to loaded dimensions - if autofill_dim_attrs: - for name, data_arri in self.coords.items(): - if name in dim_attrs and not self.coords[name].attrs: - self.coords[name].attrs = dim_attrs[name] - - if "_sections" not in self.attrs: - self.attrs["_sections"] = yaml.dump(None) - - if "sections" in kwargs: - self = set_sections(self, kwargs["sections"]) - - pass - - def __repr__(self): - # __repr__ from xarray is used and edited. - # 'xarray' is prepended. so we remove it and add 'dtscalibration' - s = xr.core.formatting.dataset_repr(self) - name_module = type(self).__name__ - preamble_new = "" % name_module - - # Add sections to new preamble - preamble_new += "\nSections:" - if hasattr(self, "_sections") and self.sections: - preamble_new += "\n" - - if "units" in self.x: - unit = self.x.units - else: - unit = "" - - for k, v in self.sections.items(): - preamble_new += f" {k: <23}" - - # Compute statistics reference section timeseries - sec_stat = f"({float(self[k].mean()):6.2f}" - sec_stat += f" +/-{float(self[k].std()):5.2f}" - sec_stat += "\N{DEGREE SIGN}C)\t" - preamble_new += sec_stat - - # print sections - vl = [f"{vi.start:.2f}{unit} - {vi.stop:.2f}{unit}" for vi in v] - preamble_new += " and ".join(vl) + "\n" - - else: - preamble_new += 18 * " " + "()\n" - - # add new preamble to the remainder of the former __repr__ - len_preamble_old = 8 + len(name_module) + 2 - - # untill the attribute listing - attr_index = s.find("Attributes:") - - # abbreviate attribute listing - attr_list_all = s[attr_index:].split(sep="\n") - if len(attr_list_all) > 10: - s_too_many = ["\n.. and many more attributes. See: ds.attrs"] - attr_list = attr_list_all[:10] + s_too_many - else: - attr_list = attr_list_all - - s_out = preamble_new + s[len_preamble_old:attr_index] + "\n".join(attr_list) - - # return new __repr__ - return s_out - - # noinspection PyIncorrectDocstring - @property - def sections(self): - """ - Define calibration sections. Each section requires a reference - temperature time series, such as the temperature measured by an - external temperature sensor. They should already be part of the - DataStore object. - - Please look at the example notebook on `sections` if you encounter - difficulties. - - Parameters - ---------- - sections : Dict[str, List[slice]] - Sections are defined in a dictionary with its keywords of the - names of the reference - temperature time series. Its values are lists of slice objects, - where each slice object - is a stretch. - Returns - ------- - - """ - if "_sections" not in self.attrs: - self.attrs["_sections"] = yaml.dump(None) - - return yaml.load(self.attrs["_sections"], Loader=yaml.UnsafeLoader) - - @sections.deleter - def sections(self): - self.attrs["_sections"] = yaml.dump(None) - - @sections.setter - def sections(self, value): - msg = ( - "Not possible anymore. Instead, pass the sections as an argument to \n" - "ds.dts.calibrate_single_ended() or ds.dts.calibrate_double_ended()." - ) - raise DeprecationWarning(msg) - - def check_reference_section_values(self): - """ - Checks if the values of the used sections are of the right datatype - (floats), if there are finite number (no NaN/inf), and if the time - dimension corresponds with the time dimension of the st/ast data. - - Parameters - ---------- - - Returns - ------- - - """ - for key in self.sections.keys(): - if not np.issubdtype(self[key].dtype, np.floating): - raise ValueError( - 'Data of reference temperature "' - + key - + '" does not have a float data type. Please ensure that ' - "the data is of a valid type (e.g. np.float32)" - ) - - if np.any(~np.isfinite(self[key].values)): - raise ValueError( - 'NaN/inf value(s) found in reference temperature "' + key + '"' - ) - - if self[key].dims != ("time",): - raise ValueError( - "Time dimension of the reference temperature timeseries " - + key - + "is not the same as the time dimension" - + " of the Stokes measurement. See examples/notebooks/09" - + "Import_timeseries.ipynb for more info" - ) - - @property - def is_double_ended(self) -> float: - """ - Whether or not the data is loaded from a double-ended setup. - - Returns - ------- - - """ - if "isDoubleEnded" in self.attrs: - return bool(int(self.attrs["isDoubleEnded"])) - elif "customData:isDoubleEnded" in self.attrs: - # backward compatible to when only silixa files were supported - return bool(int(self.attrs["customData:isDoubleEnded"])) - else: - raise ValueError( - "Could not determine if the data was from a double-ended setup." - ) - - @is_double_ended.setter - def is_double_ended(self, flag: bool): - self.attrs["isDoubleEnded"] = flag - pass - - @property - def chfw(self) -> float: - """ - Zero based channel index of the forward measurements - - Returns - ------- - - """ - return int(self.attrs["forwardMeasurementChannel"]) - 1 # zero-based - - @property - def chbw(self): - """ - Zero based channel index of the backward measurements - - Returns - ------- - - """ - if self.is_double_ended: - return int(self.attrs["reverseMeasurementChannel"]) - 1 # zero-based - else: - return None - - @property - def channel_configuration(self): - """ - Renaming conversion dictionary - - Returns - ------- - - """ - d = { - "chfw": { - "st_label": "st", - "ast_label": "ast", - "acquisitiontime_label": "userAcquisitionTimeFW", - "time_start_label": "timeFWstart", - "time_label": "timeFW", - "time_end_label": "timeFWend", - }, - "chbw": { - "st_label": "rst", - "ast_label": "rast", - "acquisitiontime_label": "userAcquisitionTimeBW", - "time_start_label": "timeBWstart", - "time_label": "timeBW", - "time_end_label": "timeBWend", - }, - } - return d - - @property - def timeseries_keys(self): - """ - Returns the keys of all timeseires that can be used for calibration. - """ - return [k for k, v in self.data_vars.items() if v.dims == ("time",)] - - def to_netcdf( - self, - path=None, - mode="w", - format=None, - group=None, - engine=None, - encoding=None, - unlimited_dims=None, - compute=True, - ): - """Write datastore contents to a netCDF file. - - Parameters - ---------- - path : str, Path or file-like object, optional - Path to which to save this dataset. File-like objects are only - supported by the scipy engine. If no path is provided, this - function returns the resulting netCDF file as bytes; in this case, - we need to use scipy, which does not support netCDF version 4 (the - default format becomes NETCDF3_64BIT). - mode : {'w', 'a'}, optional - Write ('w') or append ('a') mode. If mode='w', any existing file at - this location will be overwritten. If mode='a', existing variables - will be overwritten. - format : {'NETCDF4', 'NETCDF4_CLASSIC', 'NETCDF3_64BIT', - 'NETCDF3_CLASSIC'}, optional - File format for the resulting netCDF file: - * NETCDF4: Data is stored in an HDF5 file, using netCDF4 API - features. - * NETCDF4_CLASSIC: Data is stored in an HDF5 file, using only - netCDF 3 compatible API features. - * NETCDF3_64BIT: 64-bit offset version of the netCDF 3 file format, - which fully supports 2+ GB files, but is only compatible with - clients linked against netCDF version 3.6.0 or later. - * NETCDF3_CLASSIC: The classic netCDF 3 file format. It does not - handle 2+ GB files very well. - All formats are supported by the netCDF4-python library. - scipy.io.netcdf only supports the last two formats. - The default format is NETCDF4 if you are saving a file to disk and - have the netCDF4-python library available. Otherwise, xarray falls - back to using scipy to write netCDF files and defaults to the - NETCDF3_64BIT format (scipy does not support netCDF4). - group : str, optional - Path to the netCDF4 group in the given file to open (only works for - format='NETCDF4'). The group(s) will be created if necessary. - engine : {'netcdf4', 'scipy', 'h5netcdf'}, optional - Engine to use when writing netCDF files. If not provided, the - default engine is chosen based on available dependencies, with a - preference for 'netcdf4' if writing to a file on disk. - encoding : dict, optional - defaults to reasonable compression. Use encoding={} to disable - encoding. - Nested dictionary with variable names as keys and dictionaries of - variable specific encodings as values, e.g., - ``{'my_variable': {'dtype': 'int16', 'scale_factor': 0.1, - 'zlib': True}, ...}`` - The `h5netcdf` engine supports both the NetCDF4-style compression - encoding parameters ``{'zlib': True, 'complevel': 9}`` and the h5py - ones ``{'compression': 'gzip', 'compression_opts': 9}``. - This allows using any compression plugin installed in the HDF5 - library, e.g. LZF. - unlimited_dims : sequence of str, optional - Dimension(s) that should be serialized as unlimited dimensions. - By default, no dimensions are treated as unlimited dimensions. - Note that unlimited_dims may also be set via - ``dataset.encoding['unlimited_dims']``. - compute: boolean - If true compute immediately, otherwise return a - ``dask.delayed.Delayed`` object that can be computed later. - """ - if encoding is None: - encoding = self.get_default_encoding() - - if engine is None: - engine = "netcdf4" - - # Fix Bart Schilperoort: netCDF doesn't like None's - for attribute, value in self.attrs.items(): - if value is None: - self.attrs[attribute] = "" - - return super().to_netcdf( - path, - mode, - format=format, - group=group, - engine=engine, - encoding=encoding, - unlimited_dims=unlimited_dims, - compute=compute, - ) - - def to_mf_netcdf( - self, - folder_path=None, - filename_preamble="file_", - filename_extension=".nc", - format="netCDF4", - engine="netcdf4", - encoding=None, - mode="w", - compute=True, - time_chunks_from_key="st", - ): - """Write DataStore to multiple netCDF files. - - Splits the DataStore along the time dimension using the chunks. It - first checks if all chunks in `ds` are time aligned. If this is not - the case, calculate optimal chunk sizes using the - `time_chunks_from_key` array. The files are written per time-chunk to - disk. - - Almost similar to xarray.save_mfdataset, - - Parameters - ---------- - folder_path : str, Path - Folder to place the files - filename_preamble : str - Filename is `filename_preamble + '0000' + filename_extension - filename_extension : str - Filename is `filename_preamble + '0000' + filename_extension - mode : {'w', 'a'}, optional - Write ('w') or append ('a') mode. If mode='w', any existing file at - these locations will be overwritten. - format : {'NETCDF4', 'NETCDF4_CLASSIC', 'NETCDF3_64BIT', - 'NETCDF3_CLASSIC'}, optional - File format for the resulting netCDF file: - * NETCDF4: Data is stored in an HDF5 file, using netCDF4 API - features. - * NETCDF4_CLASSIC: Data is stored in an HDF5 file, using only - netCDF 3 compatible API features. - * NETCDF3_64BIT: 64-bit offset version of the netCDF 3 file format, - which fully supports 2+ GB files, but is only compatible with - clients linked against netCDF version 3.6.0 or later. - * NETCDF3_CLASSIC: The classic netCDF 3 file format. It does not - handle 2+ GB files very well. - All formats are supported by the netCDF4-python library. - scipy.io.netcdf only supports the last two formats. - The default format is NETCDF4 if you are saving a file to disk and - have the netCDF4-python library available. Otherwise, xarray falls - back to using scipy to write netCDF files and defaults to the - NETCDF3_64BIT format (scipy does not support netCDF4). - engine : {'netcdf4', 'scipy', 'h5netcdf'}, optional - Engine to use when writing netCDF files. If not provided, the - default engine is chosen based on available dependencies, with a - preference for 'netcdf4' if writing to a file on disk. - See `Dataset.to_netcdf` for additional information. - encoding : list of dict, optional - Defaults to reasonable compression/encoding. - If you want to define your own encoding, you first needs to know the - time-chunk sizes this routine will write to disk. After which you - need to provide a list with the encoding specified for each chunk. - Use a list of empty dicts to disable encoding. - Nested dictionary with variable names as keys and dictionaries of - variable specific encodings as values, e.g., - ``{'my_variable': {'dtype': 'int16', 'scale_factor': 0.1, - 'zlib': True}, ...}`` - The `h5netcdf` engine supports both the NetCDF4-style compression - encoding parameters ``{'zlib': True, 'complevel': 9}`` and the h5py - ones ``{'compression': 'gzip', 'compression_opts': 9}``. - This allows using any compression plugin installed in the HDF5 - library, e.g. LZF. - compute: boolean - If true compute immediately, otherwise return a - ``dask.delayed.Delayed`` object that can be computed later. - time_chunks_from_key: str - - Examples - -------- - ds.to_mf_netcdf(folder_path='.') - - See Also - -------- - dtscalibration.open_mf_datastore - xarray.save_mfdataset - - """ - - try: - # This fails if not all chunks of the data_vars are time aligned. - # In case we let Dask estimate an optimal chunk size. - t_chunks = self.chunks["time"] - - except: # noqa: E722 - if self[time_chunks_from_key].dims == ("x", "time"): - _, t_chunks = da.ones( - self[time_chunks_from_key].shape, - chunks=(-1, "auto"), - dtype="float64", - ).chunks - - elif self[time_chunks_from_key].dims == ("time", "x"): - _, t_chunks = da.ones( - self[time_chunks_from_key].shape, - chunks=("auto", -1), - dtype="float64", - ).chunks - else: - assert 0, "something went wrong with your Stokes dimensions" - - bnds = np.cumsum((0,) + t_chunks) - x = [range(bu, bd) for bu, bd in zip(bnds[:-1], bnds[1:])] - - datasets = [self.isel(time=xi) for xi in x] - paths = [ - os.path.join( - folder_path, - filename_preamble + f"{ix:04d}" + filename_extension, - ) - for ix in range(len(x)) - ] - - encodings = [] - for ids, ds in enumerate(datasets): - if encoding is None: - encodings.append( - ds.get_default_encoding(time_chunks_from_key=time_chunks_from_key) - ) - - else: - encodings.append(encoding[ids]) - - writers, stores = zip( - *[ - xr.backends.api.to_netcdf( - ds, - path, - mode, - format, - None, - engine, - compute=compute, - multifile=True, - encoding=enc, - ) - for ds, path, enc in zip(datasets, paths, encodings) - ] - ) - - try: - writes = [w.sync(compute=compute) for w in writers] - finally: - if compute: - for store in stores: - store.close() - - if not compute: - - def _finalize_store(write, store): - """Finalize this store by explicitly syncing and closing""" - del write # ensure writing is done first - store.close() - pass - - return dask.delayed( - [dask.delayed(_finalize_store)(w, s) for w, s in zip(writes, stores)] - ) - - pass - - def get_default_encoding(self, time_chunks_from_key=None): - """ - Returns a dictionary with sensible compression setting for writing - netCDF files. - - Returns - ------- - - """ - # The following variables are stored with a sufficiently large - # precision in 32 bit - float32l = [ - "st", - "ast", - "rst", - "rast", - "time", - "timestart", - "tmp", - "timeend", - "acquisitionTime", - "x", - ] - int32l = [ - "filename_tstamp", - "acquisitiontimeFW", - "acquisitiontimeBW", - "userAcquisitionTimeFW", - "userAcquisitionTimeBW", - ] - - # default variable compression - compdata = dict( - zlib=True, complevel=6, shuffle=False - ) # , least_significant_digit=None - - # default coordinate compression - compcoords = dict(zlib=True, complevel=4) - - # construct encoding dict - encoding = {var: compdata.copy() for var in self.data_vars} - encoding.update({var: compcoords.copy() for var in self.coords}) - - for k, v in encoding.items(): - if k in float32l: - v["dtype"] = "float32" - - if k in int32l: - v["dtype"] = "int32" - # v['_FillValue'] = -9999 # Int does not support NaN - - if np.issubdtype(self[k].dtype, str) or np.issubdtype( - self[k].dtype, object - ): - # Compression not supported for variable length strings - # https://github.com/Unidata/netcdf4-python/issues/1205 - v["zlib"] = False - - if time_chunks_from_key is not None: - # obtain optimal chunk sizes in time and x dim - if self[time_chunks_from_key].dims == ("x", "time"): - x_chunk, t_chunk = da.ones( - self[time_chunks_from_key].shape, - chunks=(-1, "auto"), - dtype="float64", - ).chunks - - elif self[time_chunks_from_key].dims == ("time", "x"): - x_chunk, t_chunk = da.ones( - self[time_chunks_from_key].shape, - chunks=("auto", -1), - dtype="float64", - ).chunks - else: - assert 0, "something went wrong with your Stokes dimensions" - - for k, v in encoding.items(): - # By writing and compressing the data in chunks, some sort of - # parallism is possible. - if self[k].dims == ("x", "time"): - chunks = (x_chunk[0], t_chunk[0]) - - elif self[k].dims == ("time", "x"): - chunks = (t_chunk[0], x_chunk[0]) - - elif self[k].dims == ("x",): - chunks = (x_chunk[0],) - - elif self[k].dims == ("time",): - chunks = (t_chunk[0],) - - else: - continue - - v["chunksizes"] = chunks - - return encoding - - def get_section_indices(self, sec): - """Returns the x-indices of the section. `sec` is a slice.""" - xis = self.x.astype(int) * 0 + np.arange(self.x.size, dtype=int) - return xis.sel(x=sec).values - - def rename_labels(self, assertion=True): - """ - Renames the `ST` DataArrays (old convention) to `st` (new convention). - The new naming convention simplifies the notation of the reverse Stokes - `ds['REV-ST']` becomes `ds.rst`. Plus the parameter-naming convention in - Python in lowercase. - - Parameters - ---------- - assertion : bool - If set to `True`, raises an error if complications occur. - - Returns - ------- - - """ - re_dict = { - "ST": "st", - "AST": "ast", - "REV-ST": "rst", - "REV-AST": "rast", - "TMP": "tmp", - "TMPF": "tmpf", - "TMPB": "tmpb", - "TMPW": "tmpw", - } - - re_dict_err = { - k: v - for k, v in re_dict.items() - if k in self.data_vars and v in self.data_vars - } - - msg = ( - "Unable to rename the st_labels automagically. \n" - "Please manually rename ST->st and REV-ST->rst. The \n" - f"parameters {re_dict_err.values()} were already present" - ) - - if assertion: - assert len(re_dict_err) == 0, msg - elif len(re_dict_err) != 0: - print(msg) - for v in re_dict_err.values(): - print(f"Variable {v} was not renamed") - - re_dict2 = { - k: v - for k, v in re_dict.items() - if k in self.data_vars and v not in self.data_vars - } - - return self.rename(re_dict2) - - def variance_stokes(self, *args, **kwargs): - """Backwards compatibility. See `ds.variance_stokes_constant()`""" - return self.variance_stokes_constant(*args, **kwargs) - - def variance_stokes_constant(self, st_label, sections=None, reshape_residuals=True): - """ - Approximate the variance of the noise in Stokes intensity measurements - with one value, suitable for small setups. - - * `ds.variance_stokes_constant()` for small setups with small variations in\ - intensity. Variance of the Stokes measurements is assumed to be the same\ - along the entire fiber. - - * `ds.variance_stokes_exponential()` for small setups with very few time\ - steps. Too many degrees of freedom results in an under estimation of the\ - noise variance. Almost never the case, but use when calibrating pre time\ - step. - - * `ds.variance_stokes_linear()` for larger setups with more time steps.\ - Assumes Poisson distributed noise with the following model:: - - st_var = a * ds.st + b - - - where `a` and `b` are constants. Requires reference sections at - beginning and end of the fiber, to have residuals at high and low - intensity measurements. - - The Stokes and anti-Stokes intensities are measured with detectors, - which inherently introduce noise to the measurements. Knowledge of the - distribution of the measurement noise is needed for a calibration with - weighted observations (Sections 5 and 6 of [1]_) - and to project the associated uncertainty to the temperature confidence - intervals (Section 7 of [1]_). Two sources dominate the noise - in the Stokes and anti-Stokes intensity measurements - (Hartog, 2017, p.125). Close to the laser, noise from the conversion of - backscatter to electricity dominates the measurement noise. The - detecting component, an avalanche photodiode, produces Poisson- - distributed noise with a variance that increases linearly with the - intensity. The Stokes and anti-Stokes intensities are commonly much - larger than the standard deviation of the noise, so that the Poisson - distribution can be approximated with a Normal distribution with a mean - of zero and a variance that increases linearly with the intensity. At - the far-end of the fiber, noise from the electrical circuit dominates - the measurement noise. It produces Normal-distributed noise with a mean - of zero and a variance that is independent of the intensity. - - Calculates the variance between the measurements and a best fit - at each reference section. This fits a function to the nt * nx - measurements with ns * nt + nx parameters, where nx are the total - number of reference locations along all sections. The temperature is - constant along the reference sections, so the expression of the - Stokes power can be split in a time series per reference section and - a constant per observation location. - - Idea from Discussion at page 127 in Richter, P. H. (1995). Estimating - errors in least-squares fitting. - - The timeseries and the constant are, of course, highly correlated - (Equations 20 and 21 in [1]_), but that is not relevant here as only the - product is of interest. The residuals between the fitted product and the - Stokes intensity measurements are attributed to the - noise from the detector. The variance of the residuals is used as a - proxy for the variance of the noise in the Stokes and anti-Stokes - intensity measurements. A non-uniform temperature of - the reference sections results in an over estimation of the noise - variance estimate because all temperature variation is attributed to - the noise. - - Parameters - ---------- - reshape_residuals - st_label : str - label of the Stokes, anti-Stokes measurement. - E.g., st, ast, rst, rast - sections : Dict[str, List[slice]], optional - If `None` is supplied, `ds.sections` is used. Define calibration - sections. Each section requires a reference temperature time series, - such as the temperature measured by an external temperature sensor. - They should already be part of the DataStore object. `sections` - is defined with a dictionary with its keywords of the - names of the reference temperature time series. Its values are - lists of slice objects, where each slice object is a fiber stretch - that has the reference temperature. Afterwards, `sections` is stored - under `ds.sections`. - - Returns - ------- - I_var : float - Variance of the residuals between measured and best fit - resid : array_like - Residuals between measured and best fit - - Notes - ----- - - * Because there are a large number of unknowns, spend time on\ - calculating an initial estimate. Can be turned off by setting to False. - - * It is often not needed to use measurements from all time steps. If\ - your variance estimate does not change when including measurements from\ - more time steps, you have included enough measurements. - - References - ---------- - .. [1] des Tombe, B., Schilperoort, B., & Bakker, M. (2020). Estimation - of Temperature and Associated Uncertainty from Fiber-Optic Raman- - Spectrum Distributed Temperature Sensing. Sensors, 20(8), 2235. - https://doi.org/10.3390/s20082235 - - Examples - -------- - - `Example notebook 4: Calculate variance Stokes intensity measurements\ - `_ - """ - if sections is None: - sections = self.sections - else: - sections = validate_sections(self, sections) - - assert self[st_label].dims[0] == "x", f"{st_label} are transposed" - # check_timestep_allclose(self, eps=0.01) - - # should maybe be per section. But then residuals - # seem to be correlated between stretches. I don't know why.. BdT. - data_dict = da.compute( - self.ufunc_per_section( - sections=sections, label=st_label, calc_per="stretch" - ) - )[0] - - var_I, resid = variance_stokes_constant_helper(data_dict) - - if not reshape_residuals: - return var_I, resid - - else: - ix_resid = self.ufunc_per_section( - sections=sections, x_indices=True, calc_per="all" - ) - - resid_sorted = np.full(shape=self[st_label].shape, fill_value=np.nan) - resid_sorted[ix_resid, :] = resid - resid_da = xr.DataArray(data=resid_sorted, coords=self[st_label].coords) - - return var_I, resid_da - - def variance_stokes_exponential( - self, - st_label, - sections=None, - use_statsmodels=False, - suppress_info=True, - reshape_residuals=True, - ): - """ - Approximate the variance of the noise in Stokes intensity measurements - with one value, suitable for small setups with measurements from only - a few times. - - * `ds.variance_stokes_constant()` for small setups with small variations in\ - intensity. Variance of the Stokes measurements is assumed to be the same\ - along the entire fiber. - - * `ds.variance_stokes_exponential()` for small setups with very few time\ - steps. Too many degrees of freedom results in an under estimation of the\ - noise variance. Almost never the case, but use when calibrating pre time\ - step. - - * `ds.variance_stokes_linear()` for larger setups with more time steps.\ - Assumes Poisson distributed noise with the following model:: - - st_var = a * ds.st + b - - - where `a` and `b` are constants. Requires reference sections at - beginning and end of the fiber, to have residuals at high and low - intensity measurements. - - The Stokes and anti-Stokes intensities are measured with detectors, - which inherently introduce noise to the measurements. Knowledge of the - distribution of the measurement noise is needed for a calibration with - weighted observations (Sections 5 and 6 of [1]_) - and to project the associated uncertainty to the temperature confidence - intervals (Section 7 of [1]_). Two sources dominate the noise - in the Stokes and anti-Stokes intensity measurements - (Hartog, 2017, p.125). Close to the laser, noise from the conversion of - backscatter to electricity dominates the measurement noise. The - detecting component, an avalanche photodiode, produces Poisson- - distributed noise with a variance that increases linearly with the - intensity. The Stokes and anti-Stokes intensities are commonly much - larger than the standard deviation of the noise, so that the Poisson - distribution can be approximated with a Normal distribution with a mean - of zero and a variance that increases linearly with the intensity. At - the far-end of the fiber, noise from the electrical circuit dominates - the measurement noise. It produces Normal-distributed noise with a mean - of zero and a variance that is independent of the intensity. - - Calculates the variance between the measurements and a best fit - at each reference section. This fits a function to the nt * nx - measurements with ns * nt + nx parameters, where nx are the total - number of reference locations along all sections. The temperature is - constant along the reference sections. This fits a two-parameter - exponential to the stokes measurements. The temperature is constant - and there are no splices/sharp bends in each reference section. - Therefore all signal decrease is due to differential attenuation, - which is the same for each reference section. The scale of the - exponential does differ per reference section. - - Assumptions: 1) the temperature is the same along a reference - section. 2) no sharp bends and splices in the reference sections. 3) - Same type of optical cable in each reference section. - - Idea from discussion at page 127 in Richter, P. H. (1995). Estimating - errors in least-squares fitting. For weights used error propagation: - w^2 = 1/sigma(lny)^2 = y^2/sigma(y)^2 = y^2 - - The timeseries and the constant are, of course, highly correlated - (Equations 20 and 21 in [1]_), but that is not relevant here as only the - product is of interest. The residuals between the fitted product and the - Stokes intensity measurements are attributed to the - noise from the detector. The variance of the residuals is used as a - proxy for the variance of the noise in the Stokes and anti-Stokes - intensity measurements. A non-uniform temperature of - the reference sections results in an over estimation of the noise - variance estimate because all temperature variation is attributed to - the noise. - - Parameters - ---------- - suppress_info : bool, optional - Suppress print statements. - use_statsmodels : bool, optional - Use statsmodels to fit the exponential. If `False`, use scipy. - reshape_residuals : bool, optional - Reshape the residuals to the shape of the Stokes intensity - st_label : str - label of the Stokes, anti-Stokes measurement. - E.g., st, ast, rst, rast - sections : Dict[str, List[slice]], optional - If `None` is supplied, `ds.sections` is used. Define calibration - sections. Each section requires a reference temperature time series, - such as the temperature measured by an external temperature sensor. - They should already be part of the DataStore object. `sections` - is defined with a dictionary with its keywords of the - names of the reference temperature time series. Its values are - lists of slice objects, where each slice object is a fiber stretch - that has the reference temperature. Afterwards, `sections` is stored - under `ds.sections`. - - Returns - ------- - I_var : float - Variance of the residuals between measured and best fit - resid : array_like - Residuals between measured and best fit - - Notes - ----- - - * Because there are a large number of unknowns, spend time on\ - calculating an initial estimate. Can be turned off by setting to False. - - * It is often not needed to use measurements from all time steps. If\ - your variance estimate does not change when including measurements from\ - more time steps, you have included enough measurements. - - References - ---------- - .. [1] des Tombe, B., Schilperoort, B., & Bakker, M. (2020). Estimation - of Temperature and Associated Uncertainty from Fiber-Optic Raman- - Spectrum Distributed Temperature Sensing. Sensors, 20(8), 2235. - https://doi.org/10.3390/s20082235 - - Examples - -------- - - `Example notebook 4: Calculate variance Stokes intensity measurements\ - `_ - """ - if sections is None: - sections = self.sections - else: - sections = validate_sections(self, sections) - - assert self[st_label].dims[0] == "x", "Stokes are transposed" - - # check_timestep_allclose(self, eps=0.01) - - nt = self.time.size - - len_stretch_list = [] # number of reference points per section ( - # spatial) - y_list = [] # intensities of stokes - x_list = [] # length rel to start of section. for alpha - - for k, stretches in sections.items(): - for stretch in stretches: - y_list.append(self[st_label].sel(x=stretch).data.T.reshape(-1)) - _x = self.x.sel(x=stretch).data.copy() - _x -= _x[0] - x_list.append(da.tile(_x, nt)) - len_stretch_list.append(_x.size) - - x = np.concatenate(x_list) # coordinates are already in memory - y = np.concatenate(y_list) - - var_I, resid = variance_stokes_exponential_helper( - nt, x, y, len_stretch_list, use_statsmodels, suppress_info - ) - - if not reshape_residuals: - return var_I, resid - - else: - # restructure the residuals, such that they can be plotted and - # added to ds - resid_res = [] - for leni, lenis, lenie in zip( - len_stretch_list, - nt * np.cumsum([0] + len_stretch_list[:-1]), - nt * np.cumsum(len_stretch_list), - ): - try: - resid_res.append(resid[lenis:lenie].reshape((leni, nt), order="F")) - except: # noqa: E722 - # Dask array does not support order - resid_res.append(resid[lenis:lenie].T.reshape((nt, leni)).T) - - _resid = np.concatenate(resid_res) - _resid_x = self.ufunc_per_section( - sections=sections, label="x", calc_per="all" - ) - isort = np.argsort(_resid_x) - resid_x = _resid_x[isort] # get indices from ufunc directly - resid = _resid[isort, :] - - ix_resid = np.array([np.argmin(np.abs(ai - self.x.data)) for ai in resid_x]) - - resid_sorted = np.full(shape=self[st_label].shape, fill_value=np.nan) - resid_sorted[ix_resid, :] = resid - resid_da = xr.DataArray(data=resid_sorted, coords=self[st_label].coords) - - return var_I, resid_da - - def variance_stokes_linear( - self, st_label, sections=None, nbin=50, through_zero=False, plot_fit=False - ): - """ - Approximate the variance of the noise in Stokes intensity measurements - with a linear function of the intensity, suitable for large setups. - - * `ds.variance_stokes_constant()` for small setups with small variations in\ - intensity. Variance of the Stokes measurements is assumed to be the same\ - along the entire fiber. - - * `ds.variance_stokes_exponential()` for small setups with very few time\ - steps. Too many degrees of freedom results in an under estimation of the\ - noise variance. Almost never the case, but use when calibrating pre time\ - step. - - * `ds.variance_stokes_linear()` for larger setups with more time steps.\ - Assumes Poisson distributed noise with the following model:: - - st_var = a * ds.st + b - - - where `a` and `b` are constants. Requires reference sections at - beginning and end of the fiber, to have residuals at high and low - intensity measurements. - - The Stokes and anti-Stokes intensities are measured with detectors, - which inherently introduce noise to the measurements. Knowledge of the - distribution of the measurement noise is needed for a calibration with - weighted observations (Sections 5 and 6 of [1]_) - and to project the associated uncertainty to the temperature confidence - intervals (Section 7 of [1]_). Two sources dominate the noise - in the Stokes and anti-Stokes intensity measurements - (Hartog, 2017, p.125). Close to the laser, noise from the conversion of - backscatter to electricity dominates the measurement noise. The - detecting component, an avalanche photodiode, produces Poisson- - distributed noise with a variance that increases linearly with the - intensity. The Stokes and anti-Stokes intensities are commonly much - larger than the standard deviation of the noise, so that the Poisson - distribution can be approximated with a Normal distribution with a mean - of zero and a variance that increases linearly with the intensity. At - the far-end of the fiber, noise from the electrical circuit dominates - the measurement noise. It produces Normal-distributed noise with a mean - of zero and a variance that is independent of the intensity. - - Calculates the variance between the measurements and a best fit - at each reference section. This fits a function to the nt * nx - measurements with ns * nt + nx parameters, where nx are the total - number of reference locations along all sections. The temperature is - constant along the reference sections, so the expression of the - Stokes power can be split in a time series per reference section and - a constant per observation location. - - Idea from Discussion at page 127 in Richter, P. H. (1995). Estimating - errors in least-squares fitting. - - The timeseries and the constant are, of course, highly correlated - (Equations 20 and 21 in [1]_), but that is not relevant here as only the - product is of interest. The residuals between the fitted product and the - Stokes intensity measurements are attributed to the - noise from the detector. The variance of the residuals is used as a - proxy for the variance of the noise in the Stokes and anti-Stokes - intensity measurements. A non-uniform temperature of - the reference sections results in an over estimation of the noise - variance estimate because all temperature variation is attributed to - the noise. - - Notes - ----- - - * Because there are a large number of unknowns, spend time on\ - calculating an initial estimate. Can be turned off by setting to False. - - * It is often not needed to use measurements from all time steps. If\ - your variance estimate does not change when including measurements \ - from more time steps, you have included enough measurements. - - References - ---------- - .. [1] des Tombe, B., Schilperoort, B., & Bakker, M. (2020). Estimation - of Temperature and Associated Uncertainty from Fiber-Optic Raman- - Spectrum Distributed Temperature Sensing. Sensors, 20(8), 2235. - https://doi.org/10.3390/s20082235 - - Examples - -------- - - `Example notebook 4: Calculate variance Stokes intensity \ - measurements `_ - - Parameters - ---------- - st_label : str - Key under which the Stokes DataArray is stored. E.g., 'st', 'rst' - sections : dict, optional - Define sections. See documentation - nbin : int - Number of bins to compute the variance for, through which the - linear function is fitted. Make sure that that are at least 50 - residuals per bin to compute the variance from. - through_zero : bool - If True, the variance is computed as: VAR(Stokes) = slope * Stokes - If False, VAR(Stokes) = slope * Stokes + offset. - From what we can tell from our inital trails, is that the offset - seems relatively small, so that True seems a better option for - setups where a reference section with very low Stokes intensities - is missing. If data with low Stokes intensities available, it is - better to not fit through zero, but determine the offset from - the data. - plot_fit : bool - If True plot the variances for each bin and plot the fitted - linear function - """ - import matplotlib.pyplot as plt - - if sections is None: - sections = self.sections - else: - sections = validate_sections(self, sections) - - assert self[st_label].dims[0] == "x", "Stokes are transposed" - _, resid = self.variance_stokes( - sections=sections, st_label=st_label, reshape_residuals=False - ) - - ix_sec = self.ufunc_per_section( - sections=sections, x_indices=True, calc_per="all" - ) - - st = self[st_label].isel(x=ix_sec).values.ravel() - diff_st = resid.ravel() - - ( - slope, - offset, - st_sort_mean, - st_sort_var, - resid, - var_fun, - ) = variance_stokes_linear_helper(st, diff_st, nbin, through_zero) - - if plot_fit: - plt.figure() - plt.scatter(st_sort_mean, st_sort_var, marker=".", c="black") - plt.plot( - [0.0, st_sort_mean[-1]], - [var_fun(0.0), var_fun(st_sort_mean[-1])], - c="white", - lw=1.3, - ) - plt.plot( - [0.0, st_sort_mean[-1]], - [var_fun(0.0), var_fun(st_sort_mean[-1])], - c="black", - lw=0.8, - ) - plt.xlabel(st_label + " intensity") - plt.ylabel(st_label + " intensity variance") - - return slope, offset, st_sort_mean, st_sort_var, resid, var_fun - - def i_var(self, st_var, ast_var, st_label="st", ast_label="ast"): - r""" - Compute the variance of an observation given the stokes and anti-Stokes - intensities and their variance. - The variance, :math:`\sigma^2_{I_{m,n}}`, of the distribution of the - noise in the observation at location :math:`m`, time :math:`n`, is a - function of the variance of the noise in the Stokes and anti-Stokes - intensity measurements (:math:`\sigma_{P_+}^2` and - :math:`\sigma_{P_-}^2`), and is approximated with (Ku et al., 1966): - - .. math:: - - \sigma^2_{I_{m,n}} \\approx \left[\\frac{\partial I_{m,n}}{\partial\ - P_{m,n+}}\\right]^2\sigma^2_{P_{+}} + \left[\\frac{\partial\ - I_{m,n}}{\partial\ - P_{m,n-}}\\right]^2\sigma^2_{P_{-}} - - .. math:: - - \sigma^2_{I_{m,n}} \\approx \\frac{1}{P_{m,n+}^2}\sigma^2_{P_{+}} +\ - \\frac{1}{P_{m,n-}^2}\sigma^2_{P_{-}} - - The variance of the noise in the Stokes and anti-Stokes intensity - measurements is estimated directly from Stokes and anti-Stokes intensity - measurements using the steps outlined in Section 4. - - Parameters - ---------- - st_var, ast_var : float, callable, array-like, optional - The variance of the measurement noise of the Stokes signals in the - forward direction. If `float` the variance of the noise from the - Stokes detector is described with a single value. - If `callable` the variance of the noise from the Stokes detector is - a function of the intensity, as defined in the callable function. - Or manually define a variance with a DataArray of the shape - `ds.st.shape`, where the variance can be a function of time and/or - x. - st_label : {'st', 'rst'} - ast_label : {'ast', 'rast'} - - Returns - ------- - - """ - st = self[st_label] - ast = self[ast_label] - - if callable(st_var): - st_var = st_var(self[st_label]).values - else: - st_var = np.asarray(st_var, dtype=float) - - if callable(ast_var): - ast_var = ast_var(self[ast_label]).values - else: - ast_var = np.asarray(ast_var, dtype=float) - - return st**-2 * st_var + ast**-2 * ast_var - - def calibration_single_ended( - self, - sections, - st_var=None, - ast_var=None, - method="wls", - solver="sparse", - p_val=None, - p_var=None, - p_cov=None, - matching_sections=None, - trans_att=None, - fix_gamma=None, - fix_dalpha=None, - fix_alpha=None, - **kwargs, - ): - r""" - Calibrate the Stokes (`ds.st`) and anti-Stokes (`ds.ast`) data to - temperature using fiber sections with a known temperature - (`ds.sections`) for single-ended setups. The calibrated temperature is - stored under `ds.tmpf` and its variance under `ds.tmpf_var`. - - In single-ended setups, Stokes and anti-Stokes intensity is measured - from a single end of the fiber. The differential attenuation is assumed - constant along the fiber so that the integrated differential attenuation - may be written as (Hausner et al, 2011): - - .. math:: - - \int_0^x{\Delta\\alpha(x')\,\mathrm{d}x'} \\approx \Delta\\alpha x - - The temperature can now be written from Equation 10 [1]_ as: - - .. math:: - - T(x,t) \\approx \\frac{\gamma}{I(x,t) + C(t) + \Delta\\alpha x} - - where - - .. math:: - - I(x,t) = \ln{\left(\\frac{P_+(x,t)}{P_-(x,t)}\\right)} - - - .. math:: - - C(t) = \ln{\left(\\frac{\eta_-(t)K_-/\lambda_-^4}{\eta_+(t)K_+/\lambda_+^4}\\right)} - - where :math:`C` is the lumped effect of the difference in gain at - :math:`x=0` between Stokes and anti-Stokes intensity measurements and - the dependence of the scattering intensity on the wavelength. The - parameters :math:`P_+` and :math:`P_-` are the Stokes and anti-Stokes - intensity measurements, respectively. - The parameters :math:`\gamma`, :math:`C(t)`, and :math:`\Delta\\alpha` - must be estimated from calibration to reference sections, as discussed - in Section 5 [1]_. The parameter :math:`C` must be estimated - for each time and is constant along the fiber. :math:`T` in the listed - equations is in Kelvin, but is converted to Celsius after calibration. - - Parameters - ---------- - p_val : array-like, optional - Define `p_val`, `p_var`, `p_cov` if you used an external function - for calibration. Has size 2 + `nt`. First value is :math:`\gamma`, - second is :math:`\Delta \\alpha`, others are :math:`C` for each - timestep. - p_var : array-like, optional - Define `p_val`, `p_var`, `p_cov` if you used an external function - for calibration. Has size 2 + `nt`. First value is :math:`\gamma`, - second is :math:`\Delta \\alpha`, others are :math:`C` for each - timestep. - p_cov : array-like, optional - The covariances of `p_val`. - If set to False, no uncertainty in the parameters is propagated - into the confidence intervals. Similar to the spec sheets of the DTS - manufacturers. And similar to passing an array filled with zeros. - sections : Dict[str, List[slice]], optional - If `None` is supplied, `ds.sections` is used. Define calibration - sections. Each section requires a reference temperature time series, - such as the temperature measured by an external temperature sensor. - They should already be part of the DataStore object. `sections` - is defined with a dictionary with its keywords of the - names of the reference temperature time series. Its values are - lists of slice objects, where each slice object is a fiber stretch - that has the reference temperature. Afterwards, `sections` is stored - under `ds.sections`. - st_var, ast_var : float, callable, array-like, optional - The variance of the measurement noise of the Stokes signals in the - forward direction. If `float` the variance of the noise from the - Stokes detector is described with a single value. - If `callable` the variance of the noise from the Stokes detector is - a function of the intensity, as defined in the callable function. - Or manually define a variance with a DataArray of the shape - `ds.st.shape`, where the variance can be a function of time and/or - x. Required if method is wls. - method : {'wls',} - Use `'wls'` for weighted least - squares. - solver : {'sparse', 'stats'} - Either use the homemade weighted sparse solver or the weighted - dense matrix solver of statsmodels. The sparse solver uses much less - memory, is faster, and gives the same result as the statsmodels - solver. The statsmodels solver is mostly used to check the sparse - solver. `'stats'` is the default. - matching_sections : List[Tuple[slice, slice, bool]], optional - Provide a list of tuples. A tuple per matching section. Each tuple - has three items. The first two items are the slices of the sections - that are matched. The third item is a boolean and is True if the two - sections have a reverse direction ("J-configuration"). - trans_att : iterable, optional - Splices can cause jumps in differential attenuation. Normal single - ended calibration assumes these are not present. An additional loss - term is added in the 'shadow' of the splice. Each location - introduces an additional nt parameters to solve for. Requiring - either an additional calibration section or matching sections. - If multiple locations are defined, the losses are added. - fix_gamma : Tuple[float, float], optional - A tuple containing two floats. The first float is the value of - gamma, and the second item is the variance of the estimate of gamma. - Covariances between gamma and other parameters are not accounted - for. - fix_dalpha : Tuple[float, float], optional - A tuple containing two floats. The first float is the value of - dalpha (:math:`\Delta \\alpha` in [1]_), and the second item is the - variance of the estimate of dalpha. - Covariances between alpha and other parameters are not accounted - for. - fix_alpha : Tuple[array-like, array-like], optional - A tuple containing two array-likes. The first array-like is the integrated - differential attenuation of length x, and the second item is its variance. - - Returns - ------- - - References - ---------- - .. [1] des Tombe, B., Schilperoort, B., & Bakker, M. (2020). Estimation - of Temperature and Associated Uncertainty from Fiber-Optic Raman- - Spectrum Distributed Temperature Sensing. Sensors, 20(8), 2235. - https://doi.org/10.3390/s20082235 - - Examples - -------- - - `Example notebook 7: Calibrate single ended `_ - - """ - check_deprecated_kwargs(kwargs) - self.set_trans_att(trans_att=trans_att, **kwargs) - - self = set_sections(self, sections) # TODO: don't change object in-place. - - if method == "wls": - assert st_var is not None and ast_var is not None, "Set `st_var`" - - self.check_reference_section_values() - - nx = self.x.size - nt = self["time"].size - nta = self.trans_att.size - - assert self["st"].dims[0] == "x", "Stokes are transposed" - assert self.ast.dims[0] == "x", "Stokes are transposed" - - if matching_sections: - matching_indices = match_sections(self, matching_sections) - else: - matching_indices = None - - ix_sec = self.ufunc_per_section( - sections=sections, x_indices=True, calc_per="all" - ) - assert not np.any(self.st.isel(x=ix_sec) <= 0.0), ( - "There is uncontrolled noise in the ST signal. Are your sections" - "correctly defined?" - ) - assert not np.any(self.ast.isel(x=ix_sec) <= 0.0), ( - "There is uncontrolled noise in the AST signal. Are your sections" - "correctly defined?" - ) - - if method == "wls": - p_cov, p_val, p_var = calibration_single_ended_helper( - self, - sections, - st_var, - ast_var, - fix_alpha, - fix_dalpha, - fix_gamma, - matching_indices, - nt, - nta, - nx, - solver, - ) - - elif method == "external": - for input_item in [p_val, p_var, p_cov]: - assert ( - input_item is not None - ), "Define p_val, p_var, p_cov when using an external solver" - - else: - raise ValueError("Choose a valid method") - - # all below require the following solution sizes - if fix_alpha: - ip = ParameterIndexSingleEnded( - nt, nx, nta, includes_alpha=True, includes_dalpha=False - ) - else: - ip = ParameterIndexSingleEnded( - nt, nx, nta, includes_alpha=False, includes_dalpha=True - ) - - # npar = 1 + 1 + nt + nta * nt - assert p_val.size == ip.npar - assert p_var.size == ip.npar - assert p_cov.shape == (ip.npar, ip.npar) - - # store calibration parameters in DataStore - coords = {"x": self["x"], "time": self["time"], "trans_att": self["trans_att"]} - params, param_covs = get_params_from_pval_single_ended( - ip, coords, p_val=p_val, p_var=p_var, p_cov=p_cov, fix_alpha=fix_alpha - ) - - tmpf = params["gamma"] / ( - (np.log(self.st / self.ast) + (params["c"] + params["talpha_fw_full"])) - + params["alpha"] - ) - out = xr.Dataset({"tmpf": tmpf - 273.15}) - out["tmpf"].attrs.update(_dim_attrs[("tmpf",)]) - - # tmpf_var - deriv_dict = dict( - T_gamma_fw=tmpf / params["gamma"], - T_st_fw=-(tmpf**2) / (params["gamma"] * self.st), - T_ast_fw=tmpf**2 / (params["gamma"] * self.ast), - T_c_fw=-(tmpf**2) / params["gamma"], - T_dalpha_fw=-self.x * (tmpf**2) / params["gamma"], - T_alpha_fw=-(tmpf**2) / params["gamma"], - T_ta_fw=-(tmpf**2) / params["gamma"], - ) - deriv_ds = xr.Dataset(deriv_dict) - - var_fw_dict = dict( - dT_dst=deriv_ds.T_st_fw**2 * parse_st_var(self, st_var, st_label="st"), - dT_dast=deriv_ds.T_ast_fw**2 - * parse_st_var(self, ast_var, st_label="ast"), - dT_gamma=deriv_ds.T_gamma_fw**2 * param_covs["gamma"], - dT_dc=deriv_ds.T_c_fw**2 * param_covs["c"], - dT_ddalpha=deriv_ds.T_alpha_fw**2 - * param_covs["alpha"], # same as dT_dalpha - dT_dta=deriv_ds.T_ta_fw**2 * param_covs["talpha_fw_full"], - dgamma_dc=( - 2 * deriv_ds.T_gamma_fw * deriv_ds.T_c_fw * param_covs["gamma_c"] - ), - dta_dgamma=( - 2 * deriv_ds.T_ta_fw * deriv_ds.T_gamma_fw * param_covs["tafw_gamma"] - ), - dta_dc=(2 * deriv_ds.T_ta_fw * deriv_ds.T_c_fw * param_covs["tafw_c"]), - ) - - if not fix_alpha: - # These correlations don't exist in case of fix_alpha. Including them reduces tmpf_var. - var_fw_dict.update( - dict( - dgamma_ddalpha=( - 2 - * deriv_ds.T_gamma_fw - * deriv_ds.T_dalpha_fw - * param_covs["gamma_dalpha"] - ), - ddalpha_dc=( - 2 - * deriv_ds.T_dalpha_fw - * deriv_ds.T_c_fw - * param_covs["dalpha_c"] - ), - dta_ddalpha=( - 2 - * deriv_ds.T_ta_fw - * deriv_ds.T_dalpha_fw - * param_covs["tafw_dalpha"] - ), - ) - ) - - out["var_fw_da"] = xr.Dataset(var_fw_dict).to_array(dim="comp_fw") - out["tmpf_var"] = out["var_fw_da"].sum(dim="comp_fw") - out["tmpf_var"].attrs.update(_dim_attrs[("tmpf_var",)]) - - drop_vars = [ - k for k, v in self.items() if {"params1", "params2"}.intersection(v.dims) - ] - - for k in drop_vars: - del self[k] - - out["p_val"] = (("params1",), p_val) - out["p_cov"] = (("params1", "params2"), p_cov) - - out.update(params) - for key, dataarray in param_covs.data_vars.items(): - out[key + "_var"] = dataarray - - self.update(out) - pass - - def calibration_double_ended( - self, - sections, - st_var=None, - ast_var=None, - rst_var=None, - rast_var=None, - method="wls", - solver="sparse", - p_val=None, - p_var=None, - p_cov=None, - trans_att=None, - fix_gamma=None, - fix_alpha=None, - matching_sections=None, - matching_indices=None, - verbose=False, - **kwargs, - ): - r""" - See example notebook 8 for an explanation on how to use this function. - Calibrate the Stokes (`ds.st`) and anti-Stokes (`ds.ast`) of the forward - channel and from the backward channel (`ds.rst`, `ds.rast`) data to - temperature using fiber sections with a known temperature - (`ds.sections`) for double-ended setups. The calibrated temperature of - the forward channel is stored under `ds.tmpf` and its variance under - `ds.tmpf_var`, and that of the backward channel under `ds.tmpb` and - `ds.tmpb_var`. The inverse-variance weighted average of the forward and - backward channel is stored under `ds.tmpw` and `ds.tmpw_var`. - In double-ended setups, Stokes and anti-Stokes intensity is measured in - two directions from both ends of the fiber. The forward-channel - measurements are denoted with subscript F, and the backward-channel - measurements are denoted with subscript B. Both measurement channels - start at a different end of the fiber and have opposite directions, and - therefore have different spatial coordinates. - The first processing step - with double-ended measurements is to align the measurements of the two - measurement channels so that they have the same spatial coordinates. The - spatial coordinate :math:`x` (m) is defined here positive in the forward - direction, starting at 0 where the fiber is connected to the forward - channel of the DTS system; the length of the fiber is :math:`L`. - Consequently, the backward-channel measurements are flipped and shifted - to align with the forward-channel measurements. Alignment of the - measurements of the two channels is prone to error because it requires - the exact fiber length (McDaniel et al., 2018). Depending on the DTS system - used, the forward channel and backward channel are measured one after - another by making use of an optical switch, so that only a single - detector is needed. However, it is assumed in this package that the - forward channel and backward channel are measured simultaneously, so - that the temperature of both measurements is the same. This assumption - holds better for short acquisition times with respect to the timescale - of the temperature variation, and when there is no systematic difference - in temperature between the two channels. The temperature may be computed - from the forward-channel measurements (Equation 10 [1]_) with: - .. math:: - T_\mathrm{F} (x,t) = \\frac{\gamma}{I_\mathrm{F}(x,t) + \ - C_\mathrm{F}(t) + \int_0^x{\Delta\\alpha(x')\,\mathrm{d}x'}} - and from the backward-channel measurements with: - .. math:: - T_\mathrm{B} (x,t) = \\frac{\gamma}{I_\mathrm{B}(x,t) + \ - C_\mathrm{B}(t) + \int_x^L{\Delta\\alpha(x')\,\mathrm{d}x'}} - with - .. math:: - I(x,t) = \ln{\left(\\frac{P_+(x,t)}{P_-(x,t)}\\right)} - .. math:: - C(t) = \ln{\left(\\frac{\eta_-(t)K_-/\lambda_-^4}{\eta_+(t)K_+/\lambda_+^4}\\right)} - where :math:`C` is the lumped effect of the difference in gain at - :param mc_conf_ints: - :math:`x=0` between Stokes and anti-Stokes intensity measurements and - the dependence of the scattering intensity on the wavelength. The - parameters :math:`P_+` and :math:`P_-` are the Stokes and anti-Stokes - intensity measurements, respectively. - :math:`C_\mathrm{F}(t)` and :math:`C_\mathrm{B}(t)` are the - parameter :math:`C(t)` for the forward-channel and backward-channel - measurements, respectively. :math:`C_\mathrm{B}(t)` may be different - from :math:`C_\mathrm{F}(t)` due to differences in gain, and difference - in the attenuation between the detectors and the point the fiber end is - connected to the DTS system (:math:`\eta_+` and :math:`\eta_-` in - Equation~\\ref{eqn:c}). :math:`T` in the listed - equations is in Kelvin, but is converted to Celsius after calibration. - The calibration procedure presented in van de - Giesen et al. 2012 approximates :math:`C(t)` to be - the same for the forward and backward-channel measurements, but this - approximation is not made here. - Parameter :math:`A(x)` (`ds.alpha`) is introduced to simplify the notation of the - double-ended calibration procedure and represents the integrated - differential attenuation between locations :math:`x_1` and :math:`x` - along the fiber. Location :math:`x_1` is the first reference section - location (the smallest x-value of all used reference sections). - .. math:: - A(x) = \int_{x_1}^x{\Delta\\alpha(x')\,\mathrm{d}x'} - so that the expressions for temperature may be written as: - .. math:: - T_\mathrm{F} (x,t) = \\frac{\gamma}{I_\mathrm{F}(x,t) + D_\mathrm{F}(t) + A(x)}, - T_\mathrm{B} (x,t) = \\frac{\gamma}{I_\mathrm{B}(x,t) + D_\mathrm{B}(t) - A(x)} - where - .. math:: - D_{\mathrm{F}}(t) = C_{\mathrm{F}}(t) + \int_0^{x_1}{\Delta\\alpha(x')\,\mathrm{d}x'}, - D_{\mathrm{B}}(t) = C_{\mathrm{B}}(t) + \int_{x_1}^L{\Delta\\alpha(x')\,\mathrm{d}x'} - Parameters :math:`D_\mathrm{F}` (`ds.df`) and :math:`D_\mathrm{B}` - (`ds.db`) must be estimated for each time and are constant along the fiber, and parameter - :math:`A` must be estimated for each location and is constant over time. - The calibration procedure is discussed in Section 6. - :math:`T_\mathrm{F}` (`ds.tmpf`) and :math:`T_\mathrm{B}` (`ds.tmpb`) - are separate - approximations of the same temperature at the same time. The estimated - :math:`T_\mathrm{F}` is more accurate near :math:`x=0` because that is - where the signal is strongest. Similarly, the estimated - :math:`T_\mathrm{B}` is more accurate near :math:`x=L`. A single best - estimate of the temperature is obtained from the weighted average of - :math:`T_\mathrm{F}` and :math:`T_\mathrm{B}` as discussed in - Section 7.2 [1]_ . - - - Parameters - ---------- - p_val : array-like, optional - Define `p_val`, `p_var`, `p_cov` if you used an external function - for calibration. Has size `1 + 2 * nt + nx + 2 * nt * nta`. - First value is :math:`\gamma`, then `nt` times - :math:`D_\mathrm{F}`, then `nt` times - :math:`D_\mathrm{B}`, then for each location :math:`D_\mathrm{B}`, - then for each connector that introduces directional attenuation two - parameters per time step. - p_var : array-like, optional - Define `p_val`, `p_var`, `p_cov` if you used an external function - for calibration. Has size `1 + 2 * nt + nx + 2 * nt * nta`. - Is the variance of `p_val`. - p_cov : array-like, optional - The covariances of `p_val`. Square matrix. - If set to False, no uncertainty in the parameters is propagated - into the confidence intervals. Similar to the spec sheets of the DTS - manufacturers. And similar to passing an array filled with zeros. - sections : Dict[str, List[slice]], optional - If `None` is supplied, `ds.sections` is used. Define calibration - sections. Each section requires a reference temperature time series, - such as the temperature measured by an external temperature sensor. - They should already be part of the DataStore object. `sections` - is defined with a dictionary with its keywords of the - names of the reference temperature time series. Its values are - lists of slice objects, where each slice object is a fiber stretch - that has the reference temperature. Afterwards, `sections` is stored - under `ds.sections`. - st_var, ast_var, rst_var, rast_var : float, callable, array-like, optional - The variance of the measurement noise of the Stokes signals in the - forward direction. If `float` the variance of the noise from the - Stokes detector is described with a single value. - If `callable` the variance of the noise from the Stokes detector is - a function of the intensity, as defined in the callable function. - Or manually define a variance with a DataArray of the shape - `ds.st.shape`, where the variance can be a function of time and/or - x. Required if method is wls. - mc_sample_size : int, optional - If set, the variance is also computed using Monte Carlo sampling. - The number of Monte Carlo samples drawn used to estimate the - variance of the forward and backward channel temperature estimates - and estimate the inverse-variance weighted average temperature. - conf_ints : iterable object of float - A list with the confidence boundaries that are calculated. Valid - values are between [0, 1]. - mc_da_random_state - For testing purposes. Similar to random seed. The seed for dask. - Makes random not so random. To produce reproducable results for - testing environments. - mc_remove_set_flag : bool - Remove the monte carlo data set, from which the CI and the - variance are calculated. - variance_suffix : str, optional - String appended for storing the variance. Only used when method - is wls. - method : {'wls', 'external'} - Use `'wls'` for weighted least squares. - solver : {'sparse', 'stats'} - Either use the homemade weighted sparse solver or the weighted - dense matrix solver of statsmodels. The sparse solver uses much less - memory, is faster, and gives the same result as the statsmodels - solver. The statsmodels solver is mostly used to check the sparse - solver. `'stats'` is the default. - trans_att : iterable, optional - Splices can cause jumps in differential attenuation. Normal single - ended calibration assumes these are not present. An additional loss - term is added in the 'shadow' of the splice. Each location - introduces an additional nt parameters to solve for. Requiring - either an additional calibration section or matching sections. - If multiple locations are defined, the losses are added. - fix_gamma : Tuple[float, float], optional - A tuple containing two floats. The first float is the value of - gamma, and the second item is the variance of the estimate of gamma. - Covariances between gamma and other parameters are not accounted - for. - fix_alpha : Tuple[array-like, array-like], optional - A tuple containing two arrays. The first array contains the - values of integrated differential att (:math:`A` in paper), and the - second array contains the variance of the estimate of alpha. - Covariances (in-) between alpha and other parameters are not - accounted for. - matching_sections : List[Tuple[slice, slice, bool]] - Provide a list of tuples. A tuple per matching section. Each tuple - has three items. The first two items are the slices of the sections - that are matched. The third item is a boolean and is True if the two - sections have a reverse direction ("J-configuration"). - matching_indices : array - Provide an array of x-indices of size (npair, 2), where each pair - has the same temperature. Used to improve the estimate of the - integrated differential attenuation. - verbose : bool - Show additional calibration information - Returns - ------- - References - ---------- - .. [1] des Tombe, B., Schilperoort, B., & Bakker, M. (2020). Estimation - of Temperature and Associated Uncertainty from Fiber-Optic Raman- - Spectrum Distributed Temperature Sensing. Sensors, 20(8), 2235. - https://doi.org/10.3390/s20082235 - Examples - -------- - - `Example notebook 8: Calibrate double ended `_ - """ - # TODO: confidence intervals using the variance approximated by linear error propagation - check_deprecated_kwargs(kwargs) - - self.set_trans_att(trans_att=trans_att, **kwargs) - - self = set_sections(self, sections) # TODO: don't change object in-place. - - self.check_reference_section_values() - - nx = self.x.size - nt = self["time"].size - nta = self.trans_att.size - ix_sec = self.ufunc_per_section( - sections=sections, x_indices=True, calc_per="all" - ) - nx_sec = ix_sec.size - - assert self.st.dims[0] == "x", "Stokes are transposed" - assert self.ast.dims[0] == "x", "Stokes are transposed" - assert self.rst.dims[0] == "x", "Stokes are transposed" - assert self.rast.dims[0] == "x", "Stokes are transposed" - - assert not np.any(self.st.isel(x=ix_sec) <= 0.0), ( - "There is uncontrolled noise in the ST signal. Are your sections" - "correctly defined?" - ) - assert not np.any(self.ast.isel(x=ix_sec) <= 0.0), ( - "There is uncontrolled noise in the AST signal. Are your sections" - "correctly defined?" - ) - assert not np.any(self.rst.isel(x=ix_sec) <= 0.0), ( - "There is uncontrolled noise in the REV-ST signal. Are your " - "sections correctly defined?" - ) - assert not np.any(self.rast.isel(x=ix_sec) <= 0.0), ( - "There is uncontrolled noise in the REV-AST signal. Are your " - "sections correctly defined?" - ) - - if method == "wls": - for input_item in [st_var, ast_var, rst_var, rast_var]: - assert input_item is not None, ( - "For wls define all variances (`st_var`, `ast_var`," - + " `rst_var`, `rast_var`)" - ) - - if np.any(matching_indices): - assert ( - not matching_sections - ), "Either define `matching_sections` or `matching_indices`" - - if matching_sections: - assert ( - not matching_indices - ), "Either define `matching_sections` or `matching_indices" - matching_indices = match_sections(self, matching_sections) - - if method == "wls": - p_cov, p_val, p_var = calibration_double_ended_helper( - self, - sections, - st_var, - ast_var, - rst_var, - rast_var, - fix_alpha, - fix_gamma, - nt, - nta, - nx, - nx_sec, - ix_sec, - matching_indices, - solver, - verbose, - ) - - elif method == "external": - for input_item in [p_val, p_var, p_cov]: - assert input_item is not None - - elif method == "external_split": - raise ValueError("Not implemented yet") - - else: - raise ValueError("Choose a valid method") - - # all below require the following solution sizes - ip = ParameterIndexDoubleEnded(nt, nx, nta) - - # npar = 1 + 2 * nt + nx + 2 * nt * nta - assert p_val.size == ip.npar - assert p_var.size == ip.npar - assert p_cov.shape == (ip.npar, ip.npar) - - coords = {"x": self["x"], "time": self["time"], "trans_att": self["trans_att"]} - params = get_params_from_pval_double_ended(ip, coords, p_val=p_val) - param_covs = get_params_from_pval_double_ended( - ip, coords, p_val=p_var, p_cov=p_cov - ) - - out = xr.Dataset( - { - "tmpf": params["gamma"] - / ( - np.log(self.st / self.ast) - + params["df"] - + params["alpha"] - + params["talpha_fw_full"] - ) - - 273.15, - "tmpb": params["gamma"] - / ( - np.log(self.rst / self.rast) - + params["db"] - - params["alpha"] - + params["talpha_bw_full"] - ) - - 273.15, - } - ) - - tmpf = out["tmpf"] + 273.15 - tmpb = out["tmpb"] + 273.15 - - deriv_dict = dict( - T_gamma_fw=tmpf / params["gamma"], - T_st_fw=-(tmpf**2) / (params["gamma"] * self.st), - T_ast_fw=tmpf**2 / (params["gamma"] * self.ast), - T_df_fw=-(tmpf**2) / params["gamma"], - T_alpha_fw=-(tmpf**2) / params["gamma"], - T_ta_fw=-(tmpf**2) / params["gamma"], - T_gamma_bw=tmpb / params["gamma"], - T_rst_bw=-(tmpb**2) / (params["gamma"] * self.rst), - T_rast_bw=tmpb**2 / (params["gamma"] * self.rast), - T_db_bw=-(tmpb**2) / params["gamma"], - T_alpha_bw=tmpb**2 / params["gamma"], - T_ta_bw=-(tmpb**2) / params["gamma"], - ) - deriv_ds = xr.Dataset(deriv_dict) - out["deriv"] = deriv_ds.to_array(dim="com2") - - var_fw_dict = dict( - dT_dst=deriv_ds.T_st_fw**2 * parse_st_var(self, st_var, st_label="st"), - dT_dast=deriv_ds.T_ast_fw**2 - * parse_st_var(self, ast_var, st_label="ast"), - dT_gamma=deriv_ds.T_gamma_fw**2 * param_covs["gamma"], - dT_ddf=deriv_ds.T_df_fw**2 * param_covs["df"], - dT_dalpha=deriv_ds.T_alpha_fw**2 * param_covs["alpha"], - dT_dta=deriv_ds.T_ta_fw**2 * param_covs["talpha_fw_full"], - dgamma_ddf=( - 2 * deriv_ds.T_gamma_fw * deriv_ds.T_df_fw * param_covs["gamma_df"] - ), - dgamma_dalpha=( - 2 - * deriv_ds.T_gamma_fw - * deriv_ds.T_alpha_fw - * param_covs["gamma_alpha"] - ), - dalpha_ddf=( - 2 * deriv_ds.T_alpha_fw * deriv_ds.T_df_fw * param_covs["alpha_df"] - ), - dta_dgamma=( - 2 * deriv_ds.T_ta_fw * deriv_ds.T_gamma_fw * param_covs["tafw_gamma"] - ), - dta_ddf=(2 * deriv_ds.T_ta_fw * deriv_ds.T_df_fw * param_covs["tafw_df"]), - dta_dalpha=( - 2 * deriv_ds.T_ta_fw * deriv_ds.T_alpha_fw * param_covs["tafw_alpha"] - ), - ) - var_bw_dict = dict( - dT_drst=deriv_ds.T_rst_bw**2 - * parse_st_var(self, rst_var, st_label="rst"), - dT_drast=deriv_ds.T_rast_bw**2 - * parse_st_var(self, rast_var, st_label="rast"), - dT_gamma=deriv_ds.T_gamma_bw**2 * param_covs["gamma"], - dT_ddb=deriv_ds.T_db_bw**2 * param_covs["db"], - dT_dalpha=deriv_ds.T_alpha_bw**2 * param_covs["alpha"], - dT_dta=deriv_ds.T_ta_bw**2 * param_covs["talpha_bw_full"], - dgamma_ddb=( - 2 * deriv_ds.T_gamma_bw * deriv_ds.T_db_bw * param_covs["gamma_db"] - ), - dgamma_dalpha=( - 2 - * deriv_ds.T_gamma_bw - * deriv_ds.T_alpha_bw - * param_covs["gamma_alpha"] - ), - dalpha_ddb=( - 2 * deriv_ds.T_alpha_bw * deriv_ds.T_db_bw * param_covs["alpha_db"] - ), - dta_dgamma=( - 2 * deriv_ds.T_ta_bw * deriv_ds.T_gamma_bw * param_covs["tabw_gamma"] - ), - dta_ddb=(2 * deriv_ds.T_ta_bw * deriv_ds.T_db_bw * param_covs["tabw_db"]), - dta_dalpha=( - 2 * deriv_ds.T_ta_bw * deriv_ds.T_alpha_bw * param_covs["tabw_alpha"] - ), - ) - - out["var_fw_da"] = xr.Dataset(var_fw_dict).to_array(dim="comp_fw") - out["var_bw_da"] = xr.Dataset(var_bw_dict).to_array(dim="comp_bw") - - out["tmpf_var"] = out["var_fw_da"].sum(dim="comp_fw") - out["tmpb_var"] = out["var_bw_da"].sum(dim="comp_bw") - - # First estimate of tmpw_var - out["tmpw_var" + "_approx"] = 1 / (1 / out["tmpf_var"] + 1 / out["tmpb_var"]) - out["tmpw"] = ( - (tmpf / out["tmpf_var"] + tmpb / out["tmpb_var"]) - * out["tmpw_var" + "_approx"] - ) - 273.15 - - weightsf = out["tmpw_var" + "_approx"] / out["tmpf_var"] - weightsb = out["tmpw_var" + "_approx"] / out["tmpb_var"] - - deriv_dict2 = dict( - T_gamma_w=weightsf * deriv_dict["T_gamma_fw"] - + weightsb * deriv_dict["T_gamma_bw"], - T_st_w=weightsf * deriv_dict["T_st_fw"], - T_ast_w=weightsf * deriv_dict["T_ast_fw"], - T_rst_w=weightsb * deriv_dict["T_rst_bw"], - T_rast_w=weightsb * deriv_dict["T_rast_bw"], - T_df_w=weightsf * deriv_dict["T_df_fw"], - T_db_w=weightsb * deriv_dict["T_db_bw"], - T_alpha_w=weightsf * deriv_dict["T_alpha_fw"] - + weightsb * deriv_dict["T_alpha_bw"], - T_taf_w=weightsf * deriv_dict["T_ta_fw"], - T_tab_w=weightsb * deriv_dict["T_ta_bw"], - ) - deriv_ds2 = xr.Dataset(deriv_dict2) - - # TODO: sigma2_tafw_tabw - var_w_dict = dict( - dT_dst=deriv_ds2.T_st_w**2 * parse_st_var(self, st_var, st_label="st"), - dT_dast=deriv_ds2.T_ast_w**2 - * parse_st_var(self, ast_var, st_label="ast"), - dT_drst=deriv_ds2.T_rst_w**2 - * parse_st_var(self, rst_var, st_label="rst"), - dT_drast=deriv_ds2.T_rast_w**2 - * parse_st_var(self, rast_var, st_label="rast"), - dT_gamma=deriv_ds2.T_gamma_w**2 * param_covs["gamma"], - dT_ddf=deriv_ds2.T_df_w**2 * param_covs["df"], - dT_ddb=deriv_ds2.T_db_w**2 * param_covs["db"], - dT_dalpha=deriv_ds2.T_alpha_w**2 * param_covs["alpha"], - dT_dtaf=deriv_ds2.T_taf_w**2 * param_covs["talpha_fw_full"], - dT_dtab=deriv_ds2.T_tab_w**2 * param_covs["talpha_bw_full"], - dgamma_ddf=2 - * deriv_ds2.T_gamma_w - * deriv_ds2.T_df_w - * param_covs["gamma_df"], - dgamma_ddb=2 - * deriv_ds2.T_gamma_w - * deriv_ds2.T_db_w - * param_covs["gamma_db"], - dgamma_dalpha=2 - * deriv_ds2.T_gamma_w - * deriv_ds2.T_alpha_w - * param_covs["gamma_alpha"], - dgamma_dtaf=2 - * deriv_ds2.T_gamma_w - * deriv_ds2.T_taf_w - * param_covs["tafw_gamma"], - dgamma_dtab=2 - * deriv_ds2.T_gamma_w - * deriv_ds2.T_tab_w - * param_covs["tabw_gamma"], - ddf_ddb=2 * deriv_ds2.T_df_w * deriv_ds2.T_db_w * param_covs["df_db"], - ddf_dalpha=2 - * deriv_ds2.T_df_w - * deriv_ds2.T_alpha_w - * param_covs["alpha_df"], - ddf_dtaf=2 * deriv_ds2.T_df_w * deriv_ds2.T_taf_w * param_covs["tafw_df"], - ddf_dtab=2 * deriv_ds2.T_df_w * deriv_ds2.T_tab_w * param_covs["tabw_df"], - ddb_dalpha=2 - * deriv_ds2.T_db_w - * deriv_ds2.T_alpha_w - * param_covs["alpha_db"], - ddb_dtaf=2 * deriv_ds2.T_db_w * deriv_ds2.T_taf_w * param_covs["tafw_db"], - ddb_dtab=2 * deriv_ds2.T_db_w * deriv_ds2.T_tab_w * param_covs["tabw_db"], - # dtaf_dtab=2 * deriv_ds2.T_tab_w * deriv_ds2.T_tab_w * param_covs["tafw_tabw"], - ) - out["var_w_da"] = xr.Dataset(var_w_dict).to_array(dim="comp_w") - out["tmpw_var"] = out["var_w_da"].sum(dim="comp_w") - - # Compute uncertainty solely due to noise in Stokes signal, neglecting parameter uncenrtainty - tmpf_var_excl_par = ( - out["var_fw_da"].sel(comp_fw=["dT_dst", "dT_dast"]).sum(dim="comp_fw") - ) - tmpb_var_excl_par = ( - out["var_bw_da"].sel(comp_bw=["dT_drst", "dT_drast"]).sum(dim="comp_bw") - ) - out["tmpw_var" + "_lower"] = 1 / (1 / tmpf_var_excl_par + 1 / tmpb_var_excl_par) - - out["tmpf"].attrs.update(_dim_attrs[("tmpf",)]) - out["tmpb"].attrs.update(_dim_attrs[("tmpb",)]) - out["tmpw"].attrs.update(_dim_attrs[("tmpw",)]) - out["tmpf_var"].attrs.update(_dim_attrs[("tmpf_var",)]) - out["tmpb_var"].attrs.update(_dim_attrs[("tmpb_var",)]) - out["tmpw_var"].attrs.update(_dim_attrs[("tmpw_var",)]) - out["tmpw_var" + "_approx"].attrs.update(_dim_attrs[("tmpw_var_approx",)]) - out["tmpw_var" + "_lower"].attrs.update(_dim_attrs[("tmpw_var_lower",)]) - - drop_vars = [ - k for k, v in self.items() if {"params1", "params2"}.intersection(v.dims) - ] - - for k in drop_vars: - print(f"removing {k}") - del self[k] - - out["p_val"] = (("params1",), p_val) - out["p_cov"] = (("params1", "params2"), p_cov) - - out.update(params) - for key, dataarray in param_covs.data_vars.items(): - out[key + "_var"] = dataarray - - self.update(out) - pass - - def average_single_ended( - self, - p_val="p_val", - p_cov="p_cov", - st_var=None, - ast_var=None, - conf_ints=None, - mc_sample_size=100, - ci_avg_time_flag1=False, - ci_avg_time_flag2=False, - ci_avg_time_sel=None, - ci_avg_time_isel=None, - ci_avg_x_flag1=False, - ci_avg_x_flag2=False, - ci_avg_x_sel=None, - ci_avg_x_isel=None, - var_only_sections=None, - da_random_state=None, - mc_remove_set_flag=True, - reduce_memory_usage=False, - **kwargs, - ): - """ - Average temperatures from single-ended setups. - - Four types of averaging are implemented. Please see Example Notebook 16. - - - Parameters - ---------- - p_val : array-like, optional - Define `p_val`, `p_var`, `p_cov` if you used an external function - for calibration. Has size 2 + `nt`. First value is :math:`\\gamma`, - second is :math:`\\Delta \\alpha`, others are :math:`C` for each - timestep. - If set to False, no uncertainty in the parameters is propagated - into the confidence intervals. Similar to the spec sheets of the DTS - manufacturers. And similar to passing an array filled with zeros - p_cov : array-like, optional - The covariances of `p_val`. - st_var, ast_var : float, callable, array-like, optional - The variance of the measurement noise of the Stokes signals in the - forward direction. If `float` the variance of the noise from the - Stokes detector is described with a single value. - If `callable` the variance of the noise from the Stokes detector is - a function of the intensity, as defined in the callable function. - Or manually define a variance with a DataArray of the shape - `ds.st.shape`, where the variance can be a function of time and/or - x. Required if method is wls. - conf_ints : iterable object of float - A list with the confidence boundaries that are calculated. Valid - values are between - [0, 1]. - mc_sample_size : int - Size of the monte carlo parameter set used to calculate the - confidence interval - ci_avg_time_flag1 : bool - The confidence intervals differ each time step. Assumes the - temperature varies during the measurement period. Computes the - arithmic temporal mean. If you would like to know the confidence - interfal of: - (1) a single additional measurement. So you can state "if another - measurement were to be taken, it would have this ci" - (2) all measurements. So you can state "The temperature remained - during the entire measurement period between these ci bounds". - Adds "tmpw" + '_avg1' and "tmpw" + '_mc_avg1_var' to the - DataStore. If `conf_ints` are set, also the confidence intervals - `_mc_avg1` are added to the DataStore. Works independently of the - ci_avg_time_flag2 and ci_avg_x_flag. - ci_avg_time_flag2 : bool - The confidence intervals differ each time step. Assumes the - temperature remains constant during the measurement period. - Computes the inverse-variance-weighted-temporal-mean temperature - and its uncertainty. - If you would like to know the confidence interfal of: - (1) I want to estimate a background temperature with confidence - intervals. I hereby assume the temperature does not change over - time and average all measurements to get a better estimate of the - background temperature. - Adds "tmpw" + '_avg2' and "tmpw" + '_mc_avg2_var' to the - DataStore. If `conf_ints` are set, also the confidence intervals - `_mc_avg2` are added to the DataStore. Works independently of the - ci_avg_time_flag1 and ci_avg_x_flag. - ci_avg_time_sel : slice - Compute ci_avg_time_flag1 and ci_avg_time_flag2 using only a - selection of the data - ci_avg_time_isel : iterable of int - Compute ci_avg_time_flag1 and ci_avg_time_flag2 using only a - selection of the data - ci_avg_x_flag1 : bool - The confidence intervals differ at each location. Assumes the - temperature varies over `x` and over time. Computes the - arithmic spatial mean. If you would like to know the confidence - interfal of: - (1) a single additional measurement location. So you can state "if - another measurement location were to be taken, - it would have this ci" - (2) all measurement locations. So you can state "The temperature - along the fiber remained between these ci bounds". - Adds "tmpw" + '_avgx1' and "tmpw" + '_mc_avgx1_var' to the - DataStore. If `conf_ints` are set, also the confidence intervals - `_mc_avgx1` are added to the DataStore. Works independently of the - ci_avg_time_flag1, ci_avg_time_flag2 and ci_avg_x2_flag. - ci_avg_x_flag2 : bool - The confidence intervals differ at each location. Assumes the - temperature is the same at each location but varies over time. - Computes the inverse-variance-weighted-spatial-mean temperature - and its uncertainty. - If you would like to know the confidence interfal of: - (1) I have put a lot of fiber in water, and I know that the - temperature variation in the water is much smaller than along - other parts of the fiber. And I would like to average the - measurements from multiple locations to improve the estimated - temperature. - Adds "tmpw" + '_avg2' and "tmpw" + '_mc_avg2_var' to the - DataStore. If `conf_ints` are set, also the confidence intervals - `_mc_avg2` are added to the DataStore. Works independently of the - ci_avg_time_flag1 and ci_avg_x_flag. - ci_avg_x_sel : slice - Compute ci_avg_time_flag1 and ci_avg_time_flag2 using only a - selection of the data - ci_avg_x_isel : iterable of int - Compute ci_avg_time_flag1 and ci_avg_time_flag2 using only a - selection of the data - var_only_sections : bool - useful if using the ci_avg_x_flag. Only calculates the var over the - sections, so that the values can be compared with accuracy along the - reference sections. Where the accuracy is the variance of the - residuals between the estimated temperature and temperature of the - water baths. - da_random_state - For testing purposes. Similar to random seed. The seed for dask. - Makes random not so random. To produce reproducable results for - testing environments. - mc_remove_set_flag : bool - Remove the monte carlo data set, from which the CI and the - variance are calculated. - reduce_memory_usage : bool - Use less memory but at the expense of longer computation time - - Returns - ------- - - """ - check_deprecated_kwargs(kwargs) - - if var_only_sections is not None: - raise NotImplementedError() - - out = xr.Dataset() - - mcparams = self.conf_int_single_ended( - p_val=p_val, - p_cov=p_cov, - st_var=st_var, - ast_var=ast_var, - conf_ints=None, - mc_sample_size=mc_sample_size, - da_random_state=da_random_state, - mc_remove_set_flag=False, - reduce_memory_usage=reduce_memory_usage, - **kwargs, - ) - mcparams["tmpf"] = self["tmpf"] - - if ci_avg_time_sel is not None: - time_dim2 = "time" + "_avg" - x_dim2 = "x" - mcparams.coords[time_dim2] = ( - (time_dim2,), - mcparams["time"].sel(**{"time": ci_avg_time_sel}).data, - ) - mcparams["tmpf_avgsec"] = ( - ("x", time_dim2), - mcparams["tmpf"].sel(**{"time": ci_avg_time_sel}).data, - ) - mcparams["tmpf_mc_set"] = ( - ("mc", "x", time_dim2), - mcparams["tmpf" + "_mc_set"].sel(**{"time": ci_avg_time_sel}).data, - ) - - elif ci_avg_time_isel is not None: - time_dim2 = "time" + "_avg" - x_dim2 = "x" - mcparams.coords[time_dim2] = ( - (time_dim2,), - mcparams["time"].isel(**{"time": ci_avg_time_isel}).data, - ) - mcparams["tmpf_avgsec"] = ( - ("x", time_dim2), - mcparams["tmpf"].isel(**{"time": ci_avg_time_isel}).data, - ) - mcparams["tmpf_mc_set"] = ( - ("mc", "x", time_dim2), - mcparams["tmpf" + "_mc_set"].isel(**{"time": ci_avg_time_isel}).data, - ) - - elif ci_avg_x_sel is not None: - time_dim2 = "time" - x_dim2 = "x_avg" - mcparams.coords[x_dim2] = ((x_dim2,), mcparams.x.sel(x=ci_avg_x_sel).data) - mcparams["tmpf_avgsec"] = ( - (x_dim2, "time"), - mcparams["tmpf"].sel(x=ci_avg_x_sel).data, - ) - mcparams["tmpf_mc_set"] = ( - ("mc", x_dim2, "time"), - mcparams["tmpf_mc_set"].sel(x=ci_avg_x_sel).data, - ) - - elif ci_avg_x_isel is not None: - time_dim2 = "time" - x_dim2 = "x_avg" - mcparams.coords[x_dim2] = ((x_dim2,), mcparams.x.isel(x=ci_avg_x_isel).data) - mcparams["tmpf_avgsec"] = ( - (x_dim2, time_dim2), - mcparams["tmpf"].isel(x=ci_avg_x_isel).data, - ) - mcparams["tmpf_mc_set"] = ( - ("mc", x_dim2, time_dim2), - mcparams["tmpf_mc_set"].isel(x=ci_avg_x_isel).data, - ) - else: - mcparams["tmpf_avgsec"] = mcparams["tmpf"] - x_dim2 = "x" - time_dim2 = "time" - - # subtract the mean temperature - q = mcparams["tmpf_mc_set"] - mcparams["tmpf_avgsec"] - out["tmpf_mc" + "_avgsec_var"] = q.var(dim="mc", ddof=1) - - if ci_avg_x_flag1: - # unweighted mean - out["tmpf_avgx1"] = mcparams["tmpf" + "_avgsec"].mean(dim=x_dim2) - - q = mcparams["tmpf_mc_set"] - mcparams["tmpf_avgsec"] - qvar = q.var(dim=["mc", x_dim2], ddof=1) - out["tmpf_mc_avgx1_var"] = qvar - - if conf_ints: - new_chunks = (len(conf_ints), mcparams["tmpf_mc_set"].chunks[2]) - avg_axis = mcparams["tmpf_mc_set"].get_axis_num(["mc", x_dim2]) - q = mcparams["tmpf_mc_set"].data.map_blocks( - lambda x: np.percentile(x, q=conf_ints, axis=avg_axis), - chunks=new_chunks, # - drop_axis=avg_axis, - # avg dimensions are dropped from input arr - new_axis=0, - ) # The new CI dim is added as firsaxis - - out["tmpf_mc_avgx1"] = (("CI", time_dim2), q) - - if ci_avg_x_flag2: - q = mcparams["tmpf_mc_set"] - mcparams["tmpf_avgsec"] - - qvar = q.var(dim=["mc"], ddof=1) - - # Inverse-variance weighting - avg_x_var = 1 / (1 / qvar).sum(dim=x_dim2) - - out["tmpf_mc_avgx2_var"] = avg_x_var - - mcparams["tmpf" + "_mc_avgx2_set"] = (mcparams["tmpf_mc_set"] / qvar).sum( - dim=x_dim2 - ) * avg_x_var - out["tmpf" + "_avgx2"] = mcparams["tmpf" + "_mc_avgx2_set"].mean(dim="mc") - - if conf_ints: - new_chunks = (len(conf_ints), mcparams["tmpf_mc_set"].chunks[2]) - avg_axis_avgx = mcparams["tmpf_mc_set"].get_axis_num("mc") - - qq = mcparams["tmpf_mc_avgx2_set"].data.map_blocks( - lambda x: np.percentile(x, q=conf_ints, axis=avg_axis_avgx), - chunks=new_chunks, # - drop_axis=avg_axis_avgx, - # avg dimensions are dropped from input arr - new_axis=0, - dtype=float, - ) # The new CI dimension is added as - # firsaxis - out["tmpf_mc_avgx2"] = (("CI", time_dim2), qq) - - if ci_avg_time_flag1 is not None: - # unweighted mean - out["tmpf_avg1"] = mcparams["tmpf_avgsec"].mean(dim=time_dim2) - - q = mcparams["tmpf_mc_set"] - mcparams["tmpf_avgsec"] - qvar = q.var(dim=["mc", time_dim2], ddof=1) - out["tmpf_mc_avg1_var"] = qvar - - if conf_ints: - new_chunks = (len(conf_ints), mcparams["tmpf_mc_set"].chunks[1]) - avg_axis = mcparams["tmpf_mc_set"].get_axis_num(["mc", time_dim2]) - q = mcparams["tmpf_mc_set"].data.map_blocks( - lambda x: np.percentile(x, q=conf_ints, axis=avg_axis), - chunks=new_chunks, # - drop_axis=avg_axis, - # avg dimensions are dropped from input arr - new_axis=0, - ) # The new CI dim is added as firsaxis - - out["tmpf_mc_avg1"] = (("CI", x_dim2), q) - - if ci_avg_time_flag2: - q = mcparams["tmpf_mc_set"] - mcparams["tmpf_avgsec"] - - qvar = q.var(dim=["mc"], ddof=1) - - # Inverse-variance weighting - avg_time_var = 1 / (1 / qvar).sum(dim=time_dim2) - - out["tmpf_mc_avg2_var"] = avg_time_var - - mcparams["tmpf" + "_mc_avg2_set"] = (mcparams["tmpf_mc_set"] / qvar).sum( - dim=time_dim2 - ) * avg_time_var - out["tmpf_avg2"] = mcparams["tmpf" + "_mc_avg2_set"].mean(dim="mc") - - if conf_ints: - new_chunks = (len(conf_ints), mcparams["tmpf_mc_set"].chunks[1]) - avg_axis_avg2 = mcparams["tmpf_mc_set"].get_axis_num("mc") - - qq = mcparams["tmpf_mc_avg2_set"].data.map_blocks( - lambda x: np.percentile(x, q=conf_ints, axis=avg_axis_avg2), - chunks=new_chunks, # - drop_axis=avg_axis_avg2, - # avg dimensions are dropped from input arr - new_axis=0, - dtype=float, - ) # The new CI dimension is added as - # firsaxis - out["tmpf_mc_avg2"] = (("CI", x_dim2), qq) - - # Clean up the garbage. All arrays with a Monte Carlo dimension. - if mc_remove_set_flag: - remove_mc_set = [ - "r_st", - "r_ast", - "gamma_mc", - "dalpha_mc", - "c_mc", - "x_avg", - "time_avg", - "mc", - "ta_mc_arr", - ] - remove_mc_set.append("tmpf_avgsec") - remove_mc_set.append("tmpf_mc_set") - remove_mc_set.append("tmpf_mc_avg2_set") - remove_mc_set.append("tmpf_mc_avgx2_set") - remove_mc_set.append("tmpf_mc_avgsec_var") - - for k in remove_mc_set: - if k in out: - del out[k] - - self.update(out) - pass - - def average_double_ended( - self, - sections=None, - p_val="p_val", - p_cov="p_cov", - st_var=None, - ast_var=None, - rst_var=None, - rast_var=None, - conf_ints=None, - mc_sample_size=100, - ci_avg_time_flag1=False, - ci_avg_time_flag2=False, - ci_avg_time_sel=None, - ci_avg_time_isel=None, - ci_avg_x_flag1=False, - ci_avg_x_flag2=False, - ci_avg_x_sel=None, - ci_avg_x_isel=None, - da_random_state=None, - mc_remove_set_flag=True, - reduce_memory_usage=False, - **kwargs, - ): - """ - Average temperatures from double-ended setups. - - Four types of averaging are implemented. Please see Example Notebook 16. - - Parameters - ---------- - p_val : array-like, optional - Define `p_val`, `p_var`, `p_cov` if you used an external function - for calibration. Has size 2 + `nt`. First value is :math:`\\gamma`, - second is :math:`\\Delta \\alpha`, others are :math:`C` for each - timestep. - If set to False, no uncertainty in the parameters is propagated - into the confidence intervals. Similar to the spec sheets of the DTS - manufacturers. And similar to passing an array filled with zeros - p_cov : array-like, optional - The covariances of `p_val`. - st_var, ast_var, rst_var, rast_var : float, callable, array-like, optional - The variance of the measurement noise of the Stokes signals in the - forward direction. If `float` the variance of the noise from the - Stokes detector is described with a single value. - If `callable` the variance of the noise from the Stokes detector is - a function of the intensity, as defined in the callable function. - Or manually define a variance with a DataArray of the shape - `ds.st.shape`, where the variance can be a function of time and/or - x. Required if method is wls. - conf_ints : iterable object of float - A list with the confidence boundaries that are calculated. Valid - values are between - [0, 1]. - mc_sample_size : int - Size of the monte carlo parameter set used to calculate the - confidence interval - ci_avg_time_flag1 : bool - The confidence intervals differ each time step. Assumes the - temperature varies during the measurement period. Computes the - arithmic temporal mean. If you would like to know the confidence - interfal of: - (1) a single additional measurement. So you can state "if another - measurement were to be taken, it would have this ci" - (2) all measurements. So you can state "The temperature remained - during the entire measurement period between these ci bounds". - Adds "tmpw" + '_avg1' and "tmpw" + '_mc_avg1_var' to the - DataStore. If `conf_ints` are set, also the confidence intervals - `_mc_avg1` are added to the DataStore. Works independently of the - ci_avg_time_flag2 and ci_avg_x_flag. - ci_avg_time_flag2 : bool - The confidence intervals differ each time step. Assumes the - temperature remains constant during the measurement period. - Computes the inverse-variance-weighted-temporal-mean temperature - and its uncertainty. - If you would like to know the confidence interfal of: - (1) I want to estimate a background temperature with confidence - intervals. I hereby assume the temperature does not change over - time and average all measurements to get a better estimate of the - background temperature. - Adds "tmpw" + '_avg2' and "tmpw" + '_mc_avg2_var' to the - DataStore. If `conf_ints` are set, also the confidence intervals - `_mc_avg2` are added to the DataStore. Works independently of the - ci_avg_time_flag1 and ci_avg_x_flag. - ci_avg_time_sel : slice - Compute ci_avg_time_flag1 and ci_avg_time_flag2 using only a - selection of the data - ci_avg_time_isel : iterable of int - Compute ci_avg_time_flag1 and ci_avg_time_flag2 using only a - selection of the data - ci_avg_x_flag1 : bool - The confidence intervals differ at each location. Assumes the - temperature varies over `x` and over time. Computes the - arithmic spatial mean. If you would like to know the confidence - interfal of: - (1) a single additional measurement location. So you can state "if - another measurement location were to be taken, - it would have this ci" - (2) all measurement locations. So you can state "The temperature - along the fiber remained between these ci bounds". - Adds "tmpw" + '_avgx1' and "tmpw" + '_mc_avgx1_var' to the - DataStore. If `conf_ints` are set, also the confidence intervals - `_mc_avgx1` are added to the DataStore. Works independently of the - ci_avg_time_flag1, ci_avg_time_flag2 and ci_avg_x2_flag. - ci_avg_x_flag2 : bool - The confidence intervals differ at each location. Assumes the - temperature is the same at each location but varies over time. - Computes the inverse-variance-weighted-spatial-mean temperature - and its uncertainty. - If you would like to know the confidence interfal of: - (1) I have put a lot of fiber in water, and I know that the - temperature variation in the water is much smaller than along - other parts of the fiber. And I would like to average the - measurements from multiple locations to improve the estimated - temperature. - Adds "tmpw" + '_avg2' and "tmpw" + '_mc_avg2_var' to the - DataStore. If `conf_ints` are set, also the confidence intervals - `_mc_avg2` are added to the DataStore. Works independently of the - ci_avg_time_flag1 and ci_avg_x_flag. - ci_avg_x_sel : slice - Compute ci_avg_time_flag1 and ci_avg_time_flag2 using only a - selection of the data - ci_avg_x_isel : iterable of int - Compute ci_avg_time_flag1 and ci_avg_time_flag2 using only a - selection of the data - da_random_state - For testing purposes. Similar to random seed. The seed for dask. - Makes random not so random. To produce reproducable results for - testing environments. - mc_remove_set_flag : bool - Remove the monte carlo data set, from which the CI and the - variance are calculated. - reduce_memory_usage : bool - Use less memory but at the expense of longer computation time - - Returns - ------- - - """ - - def create_da_ta2(no, i_splice, direction="fw", chunks=None): - """create mask array mc, o, nt""" - - if direction == "fw": - arr = da.concatenate( - ( - da.zeros((1, i_splice, 1), chunks=(1, i_splice, 1), dtype=bool), - da.ones( - (1, no - i_splice, 1), - chunks=(1, no - i_splice, 1), - dtype=bool, - ), - ), - axis=1, - ).rechunk((1, chunks[1], 1)) - else: - arr = da.concatenate( - ( - da.ones((1, i_splice, 1), chunks=(1, i_splice, 1), dtype=bool), - da.zeros( - (1, no - i_splice, 1), - chunks=(1, no - i_splice, 1), - dtype=bool, - ), - ), - axis=1, - ).rechunk((1, chunks[1], 1)) - return arr - - check_deprecated_kwargs(kwargs) - - if (ci_avg_x_flag1 or ci_avg_x_flag2) and ( - ci_avg_time_flag1 or ci_avg_time_flag2 - ): - raise NotImplementedError( - "Incompatible flags. Can not pick " "the right chunks" - ) - - elif not ( - ci_avg_x_flag1 or ci_avg_x_flag2 or ci_avg_time_flag1 or ci_avg_time_flag2 - ): - raise NotImplementedError("Pick one of the averaging options") - - else: - pass - - out = xr.Dataset() - - mcparams = self.conf_int_double_ended( - sections=sections, - p_val=p_val, - p_cov=p_cov, - st_var=st_var, - ast_var=ast_var, - rst_var=rst_var, - rast_var=rast_var, - conf_ints=None, - mc_sample_size=mc_sample_size, - da_random_state=da_random_state, - mc_remove_set_flag=False, - reduce_memory_usage=reduce_memory_usage, - **kwargs, - ) - - for label in ["tmpf", "tmpb"]: - if ci_avg_time_sel is not None: - time_dim2 = "time" + "_avg" - x_dim2 = "x" - mcparams.coords[time_dim2] = ( - (time_dim2,), - mcparams["time"].sel(**{"time": ci_avg_time_sel}).data, - ) - mcparams[label + "_avgsec"] = ( - ("x", time_dim2), - self[label].sel(**{"time": ci_avg_time_sel}).data, - ) - mcparams[label + "_mc_set"] = ( - ("mc", "x", time_dim2), - mcparams[label + "_mc_set"].sel(**{"time": ci_avg_time_sel}).data, - ) - - elif ci_avg_time_isel is not None: - time_dim2 = "time" + "_avg" - x_dim2 = "x" - mcparams.coords[time_dim2] = ( - (time_dim2,), - mcparams["time"].isel(**{"time": ci_avg_time_isel}).data, - ) - mcparams[label + "_avgsec"] = ( - ("x", time_dim2), - self[label].isel(**{"time": ci_avg_time_isel}).data, - ) - mcparams[label + "_mc_set"] = ( - ("mc", "x", time_dim2), - mcparams[label + "_mc_set"].isel(**{"time": ci_avg_time_isel}).data, - ) - - elif ci_avg_x_sel is not None: - time_dim2 = "time" - x_dim2 = "x_avg" - mcparams.coords[x_dim2] = ( - (x_dim2,), - mcparams.x.sel(x=ci_avg_x_sel).data, - ) - mcparams[label + "_avgsec"] = ( - (x_dim2, "time"), - self[label].sel(x=ci_avg_x_sel).data, - ) - mcparams[label + "_mc_set"] = ( - ("mc", x_dim2, "time"), - mcparams[label + "_mc_set"].sel(x=ci_avg_x_sel).data, - ) - - elif ci_avg_x_isel is not None: - time_dim2 = "time" - x_dim2 = "x_avg" - mcparams.coords[x_dim2] = ( - (x_dim2,), - mcparams.x.isel(x=ci_avg_x_isel).data, - ) - mcparams[label + "_avgsec"] = ( - (x_dim2, time_dim2), - self[label].isel(x=ci_avg_x_isel).data, - ) - mcparams[label + "_mc_set"] = ( - ("mc", x_dim2, time_dim2), - mcparams[label + "_mc_set"].isel(x=ci_avg_x_isel).data, - ) - else: - mcparams[label + "_avgsec"] = self[label] - x_dim2 = "x" - time_dim2 = "time" - - memchunk = mcparams[label + "_mc_set"].chunks - - # subtract the mean temperature - q = mcparams[label + "_mc_set"] - mcparams[label + "_avgsec"] - out[label + "_mc" + "_avgsec_var"] = q.var(dim="mc", ddof=1) - - if ci_avg_x_flag1: - # unweighted mean - out[label + "_avgx1"] = mcparams[label + "_avgsec"].mean(dim=x_dim2) - - q = mcparams[label + "_mc_set"] - mcparams[label + "_avgsec"] - qvar = q.var(dim=["mc", x_dim2], ddof=1) - out[label + "_mc_avgx1_var"] = qvar - - if conf_ints: - new_chunks = (len(conf_ints), mcparams[label + "_mc_set"].chunks[2]) - avg_axis = mcparams[label + "_mc_set"].get_axis_num(["mc", x_dim2]) - q = mcparams[label + "_mc_set"].data.map_blocks( - lambda x: np.percentile(x, q=conf_ints, axis=avg_axis), - chunks=new_chunks, # - drop_axis=avg_axis, - # avg dimensions are dropped from input arr - new_axis=0, - ) # The new CI dim is added as firsaxis - - out[label + "_mc_avgx1"] = (("CI", time_dim2), q) - - if ci_avg_x_flag2: - q = mcparams[label + "_mc_set"] - mcparams[label + "_avgsec"] - - qvar = q.var(dim=["mc"], ddof=1) - - # Inverse-variance weighting - avg_x_var = 1 / (1 / qvar).sum(dim=x_dim2) - - out[label + "_mc_avgx2_var"] = avg_x_var - - mcparams[label + "_mc_avgx2_set"] = ( - mcparams[label + "_mc_set"] / qvar - ).sum(dim=x_dim2) * avg_x_var - out[label + "_avgx2"] = mcparams[label + "_mc_avgx2_set"].mean(dim="mc") - - if conf_ints: - new_chunks = (len(conf_ints), mcparams[label + "_mc_set"].chunks[2]) - avg_axis_avgx = mcparams[label + "_mc_set"].get_axis_num("mc") - - qq = mcparams[label + "_mc_avgx2_set"].data.map_blocks( - lambda x: np.percentile(x, q=conf_ints, axis=avg_axis_avgx), - chunks=new_chunks, # - drop_axis=avg_axis_avgx, - # avg dimensions are dropped from input arr - new_axis=0, - dtype=float, - ) # The new CI dimension is added as - # firsaxis - out[label + "_mc_avgx2"] = (("CI", time_dim2), qq) - - if ci_avg_time_flag1 is not None: - # unweighted mean - out[label + "_avg1"] = mcparams[label + "_avgsec"].mean(dim=time_dim2) - - q = mcparams[label + "_mc_set"] - mcparams[label + "_avgsec"] - qvar = q.var(dim=["mc", time_dim2], ddof=1) - out[label + "_mc_avg1_var"] = qvar - - if conf_ints: - new_chunks = (len(conf_ints), mcparams[label + "_mc_set"].chunks[1]) - avg_axis = mcparams[label + "_mc_set"].get_axis_num( - ["mc", time_dim2] - ) - q = mcparams[label + "_mc_set"].data.map_blocks( - lambda x: np.percentile(x, q=conf_ints, axis=avg_axis), - chunks=new_chunks, # - drop_axis=avg_axis, - # avg dimensions are dropped from input arr - new_axis=0, - ) # The new CI dim is added as firsaxis - - out[label + "_mc_avg1"] = (("CI", x_dim2), q) - - if ci_avg_time_flag2: - q = mcparams[label + "_mc_set"] - mcparams[label + "_avgsec"] - - qvar = q.var(dim=["mc"], ddof=1) - - # Inverse-variance weighting - avg_time_var = 1 / (1 / qvar).sum(dim=time_dim2) - - out[label + "_mc_avg2_var"] = avg_time_var - - mcparams[label + "_mc_avg2_set"] = ( - mcparams[label + "_mc_set"] / qvar - ).sum(dim=time_dim2) * avg_time_var - out[label + "_avg2"] = mcparams[label + "_mc_avg2_set"].mean(dim="mc") - - if conf_ints: - new_chunks = (len(conf_ints), mcparams[label + "_mc_set"].chunks[1]) - avg_axis_avg2 = mcparams[label + "_mc_set"].get_axis_num("mc") - - qq = mcparams[label + "_mc_avg2_set"].data.map_blocks( - lambda x: np.percentile(x, q=conf_ints, axis=avg_axis_avg2), - chunks=new_chunks, # - drop_axis=avg_axis_avg2, - # avg dimensions are dropped from input arr - new_axis=0, - dtype=float, - ) # The new CI dimension is added as - # firsaxis - out[label + "_mc_avg2"] = (("CI", x_dim2), qq) - - # Weighted mean of the forward and backward - tmpw_var = 1 / ( - 1 / out["tmpf_mc" + "_avgsec_var"] + 1 / out["tmpb_mc" + "_avgsec_var"] - ) - - q = ( - mcparams["tmpf_mc_set"] / out["tmpf_mc" + "_avgsec_var"] - + mcparams["tmpb_mc_set"] / out["tmpb_mc" + "_avgsec_var"] - ) * tmpw_var - - mcparams["tmpw" + "_mc_set"] = q # - - # out["tmpw"] = out["tmpw" + '_mc_set'].mean(dim='mc') - out["tmpw" + "_avgsec"] = ( - mcparams["tmpf_avgsec"] / out["tmpf_mc" + "_avgsec_var"] - + mcparams["tmpb_avgsec"] / out["tmpb_mc" + "_avgsec_var"] - ) * tmpw_var - - q = mcparams["tmpw" + "_mc_set"] - out["tmpw_avgsec"] - out["tmpw" + "_mc" + "_avgsec_var"] = q.var(dim="mc", ddof=1) - - if ci_avg_time_flag1: - out["tmpw" + "_avg1"] = out["tmpw" + "_avgsec"].mean(dim=time_dim2) - - out["tmpw" + "_mc_avg1_var"] = mcparams["tmpw" + "_mc_set"].var( - dim=["mc", time_dim2] - ) - - if conf_ints: - new_chunks_weighted = ((len(conf_ints),),) + (memchunk[1],) - avg_axis = mcparams["tmpw" + "_mc_set"].get_axis_num(["mc", time_dim2]) - q2 = mcparams["tmpw" + "_mc_set"].data.map_blocks( - lambda x: np.percentile(x, q=conf_ints, axis=avg_axis), - chunks=new_chunks_weighted, - # Explicitly define output chunks - drop_axis=avg_axis, # avg dimensions are dropped - new_axis=0, - dtype=float, - ) # The new CI dimension is added as - # first axis - out["tmpw" + "_mc_avg1"] = (("CI", x_dim2), q2) - - if ci_avg_time_flag2: - tmpw_var_avg2 = 1 / ( - 1 / out["tmpf_mc_avg2_var"] + 1 / out["tmpb_mc_avg2_var"] - ) - - q = ( - mcparams["tmpf_mc_avg2_set"] / out["tmpf_mc_avg2_var"] - + mcparams["tmpb_mc_avg2_set"] / out["tmpb_mc_avg2_var"] - ) * tmpw_var_avg2 - - mcparams["tmpw" + "_mc_avg2_set"] = q # - - out["tmpw" + "_avg2"] = ( - out["tmpf_avg2"] / out["tmpf_mc_avg2_var"] - + out["tmpb_avg2"] / out["tmpb_mc_avg2_var"] - ) * tmpw_var_avg2 - - out["tmpw" + "_mc_avg2_var"] = tmpw_var_avg2 - - if conf_ints: - # We first need to know the x-dim-chunk-size - new_chunks_weighted = ((len(conf_ints),),) + (memchunk[1],) - avg_axis_avg2 = mcparams["tmpw" + "_mc_avg2_set"].get_axis_num("mc") - q2 = mcparams["tmpw" + "_mc_avg2_set"].data.map_blocks( - lambda x: np.percentile(x, q=conf_ints, axis=avg_axis_avg2), - chunks=new_chunks_weighted, - # Explicitly define output chunks - drop_axis=avg_axis_avg2, # avg dimensions are dropped - new_axis=0, - dtype=float, - ) # The new CI dimension is added as firstax - out["tmpw" + "_mc_avg2"] = (("CI", x_dim2), q2) - - if ci_avg_x_flag1: - out["tmpw" + "_avgx1"] = out["tmpw" + "_avgsec"].mean(dim=x_dim2) - - out["tmpw" + "_mc_avgx1_var"] = mcparams["tmpw" + "_mc_set"].var(dim=x_dim2) - - if conf_ints: - new_chunks_weighted = ((len(conf_ints),),) + (memchunk[2],) - avg_axis = mcparams["tmpw" + "_mc_set"].get_axis_num(["mc", x_dim2]) - q2 = mcparams["tmpw" + "_mc_set"].data.map_blocks( - lambda x: np.percentile(x, q=conf_ints, axis=avg_axis), - chunks=new_chunks_weighted, - # Explicitly define output chunks - drop_axis=avg_axis, # avg dimensions are dropped - new_axis=0, - dtype=float, - ) # The new CI dimension is added as - # first axis - out["tmpw" + "_mc_avgx1"] = (("CI", time_dim2), q2) - - if ci_avg_x_flag2: - tmpw_var_avgx2 = 1 / ( - 1 / out["tmpf_mc_avgx2_var"] + 1 / out["tmpb_mc_avgx2_var"] - ) - - q = ( - mcparams["tmpf_mc_avgx2_set"] / out["tmpf_mc_avgx2_var"] - + mcparams["tmpb_mc_avgx2_set"] / out["tmpb_mc_avgx2_var"] - ) * tmpw_var_avgx2 - - mcparams["tmpw" + "_mc_avgx2_set"] = q # - - out["tmpw" + "_avgx2"] = ( - out["tmpf_avgx2"] / out["tmpf_mc_avgx2_var"] - + out["tmpb_avgx2"] / out["tmpb_mc_avgx2_var"] - ) * tmpw_var_avgx2 - - out["tmpw" + "_mc_avgx2_var"] = tmpw_var_avgx2 - - if conf_ints: - # We first need to know the x-dim-chunk-size - new_chunks_weighted = ((len(conf_ints),),) + (memchunk[2],) - avg_axis_avgx2 = mcparams["tmpw" + "_mc_avgx2_set"].get_axis_num("mc") - q2 = mcparams["tmpw" + "_mc_avgx2_set"].data.map_blocks( - lambda x: np.percentile(x, q=conf_ints, axis=avg_axis_avgx2), - chunks=new_chunks_weighted, - # Explicitly define output chunks - drop_axis=avg_axis_avgx2, # avg dimensions are dropped - new_axis=0, - dtype=float, - ) # The new CI dimension is added as firstax - out["tmpw" + "_mc_avgx2"] = (("CI", time_dim2), q2) - - # Clean up the garbage. All arrays with a Monte Carlo dimension. - if mc_remove_set_flag: - remove_mc_set = [ - "r_st", - "r_ast", - "r_rst", - "r_rast", - "gamma_mc", - "alpha_mc", - "df_mc", - "db_mc", - "x_avg", - "time_avg", - "mc", - ] - - for i in ["tmpf", "tmpb", "tmpw"]: - remove_mc_set.append(i + "_avgsec") - remove_mc_set.append(i + "_mc_set") - remove_mc_set.append(i + "_mc_avg2_set") - remove_mc_set.append(i + "_mc_avgx2_set") - remove_mc_set.append(i + "_mc_avgsec_var") - - if "trans_att" in mcparams and mcparams.trans_att.size: - remove_mc_set.append('talpha"_fw_mc') - remove_mc_set.append('talpha"_bw_mc') - - for k in remove_mc_set: - if k in out: - print(f"Removed from results: {k}") - del out[k] - - self.update(out) - pass - - def conf_int_single_ended( - self, - p_val="p_val", - p_cov="p_cov", - st_var=None, - ast_var=None, - conf_ints=None, - mc_sample_size=100, - da_random_state=None, - mc_remove_set_flag=True, - reduce_memory_usage=False, - **kwargs, - ): - r""" - Estimation of the confidence intervals for the temperatures measured - with a single-ended setup. It consists of five steps. First, the variances - of the Stokes and anti-Stokes intensity measurements are estimated - following the steps in Section 4 [1]_. A Normal - distribution is assigned to each intensity measurement that is centered - at the measurement and using the estimated variance. Second, a multi- - variate Normal distribution is assigned to the estimated parameters - using the covariance matrix from the calibration procedure presented in - Section 5 [1]_. Third, the distributions are sampled, and the - temperature is computed with Equation 12 [1]_. Fourth, step - three is repeated, e.g., 10,000 times for each location and for each - time. The resulting 10,000 realizations of the temperatures - approximate the probability density functions of the estimated - temperature at that location and time. Fifth, the standard uncertainties - are computed with the standard deviations of the realizations of the - temperatures, and the 95\% confidence intervals are computed from the - 2.5\% and 97.5\% percentiles of the realizations of the temperatures. - - - Parameters - ---------- - p_val : array-like, optional - Define `p_val`, `p_var`, `p_cov` if you used an external function - for calibration. Has size 2 + `nt`. First value is :math:`\gamma`, - second is :math:`\Delta \\alpha`, others are :math:`C` for each - timestep. - If set to False, no uncertainty in the parameters is propagated - into the confidence intervals. Similar to the spec sheets of the DTS - manufacturers. And similar to passing an array filled with zeros - p_cov : array-like, optional - The covariances of `p_val`. - st_var, ast_var : float, callable, array-like, optional - The variance of the measurement noise of the Stokes signals in the - forward direction. If `float` the variance of the noise from the - Stokes detector is described with a single value. - If `callable` the variance of the noise from the Stokes detector is - a function of the intensity, as defined in the callable function. - Or manually define a variance with a DataArray of the shape - `ds.st.shape`, where the variance can be a function of time and/or - x. Required if method is wls. - conf_ints : iterable object of float - A list with the confidence boundaries that are calculated. Valid - values are between - [0, 1]. - mc_sample_size : int - Size of the monte carlo parameter set used to calculate the - confidence interval - da_random_state - For testing purposes. Similar to random seed. The seed for dask. - Makes random not so random. To produce reproducable results for - testing environments. - mc_remove_set_flag : bool - Remove the monte carlo data set, from which the CI and the - variance are calculated. - reduce_memory_usage : bool - Use less memory but at the expense of longer computation time - - - References - ---------- - .. [1] des Tombe, B., Schilperoort, B., & Bakker, M. (2020). Estimation - of Temperature and Associated Uncertainty from Fiber-Optic Raman- - Spectrum Distributed Temperature Sensing. Sensors, 20(8), 2235. - https://doi.org/10.3390/s20082235 - """ - check_deprecated_kwargs(kwargs) - - out = xr.Dataset() - params = xr.Dataset() - - if da_random_state: - state = da_random_state - else: - state = da.random.RandomState() - - no, nt = self.st.data.shape - if "trans_att" in self.keys(): - nta = self.trans_att.size - - else: - nta = 0 - - assert isinstance(p_val, (str, np.ndarray, np.generic)) - if isinstance(p_val, str): - p_val = self[p_val].data - - npar = p_val.size - - # number of parameters - if npar == nt + 2 + nt * nta: - fixed_alpha = False - elif npar == 1 + no + nt + nt * nta: - fixed_alpha = True - else: - raise Exception("The size of `p_val` is not what I expected") - - params.coords["mc"] = range(mc_sample_size) - params.coords["x"] = self.x - params.coords["time"] = self.time - - if conf_ints: - out.coords["CI"] = conf_ints - params.coords["CI"] = conf_ints - - # WLS - if isinstance(p_cov, str): - p_cov = self[p_cov].data - assert p_cov.shape == (npar, npar) - - p_mc = sst.multivariate_normal.rvs(mean=p_val, cov=p_cov, size=mc_sample_size) - - if fixed_alpha: - params["alpha_mc"] = (("mc", "x"), p_mc[:, 1 : no + 1]) - params["c_mc"] = (("mc", "time"), p_mc[:, 1 + no : 1 + no + nt]) - else: - params["dalpha_mc"] = (("mc",), p_mc[:, 1]) - params["c_mc"] = (("mc", "time"), p_mc[:, 2 : nt + 2]) - - params["gamma_mc"] = (("mc",), p_mc[:, 0]) - if nta: - params["ta_mc"] = ( - ("mc", "trans_att", "time"), - np.reshape(p_mc[:, -nt * nta :], (mc_sample_size, nta, nt)), - ) - - rsize = (params.mc.size, params.x.size, params.time.size) - - if reduce_memory_usage: - memchunk = da.ones( - (mc_sample_size, no, nt), chunks={0: -1, 1: 1, 2: "auto"} - ).chunks - else: - memchunk = da.ones( - (mc_sample_size, no, nt), chunks={0: -1, 1: "auto", 2: "auto"} - ).chunks - - # Draw from the normal distributions for the Stokes intensities - for k, st_labeli, st_vari in zip( - ["r_st", "r_ast"], ["st", "ast"], [st_var, ast_var] - ): - # Load the mean as chunked Dask array, otherwise eats memory - if type(self[st_labeli].data) == da.core.Array: - loc = da.asarray(self[st_labeli].data, chunks=memchunk[1:]) - else: - loc = da.from_array(self[st_labeli].data, chunks=memchunk[1:]) - - # Make sure variance is of size (no, nt) - if np.size(st_vari) > 1: - if st_vari.shape == self[st_labeli].shape: - pass - else: - st_vari = np.broadcast_to(st_vari, (no, nt)) - else: - pass - - # Load variance as chunked Dask array, otherwise eats memory - if type(st_vari) == da.core.Array: - st_vari_da = da.asarray(st_vari, chunks=memchunk[1:]) - - elif callable(st_vari) and type(self[st_labeli].data) == da.core.Array: - st_vari_da = da.asarray( - st_vari(self[st_labeli]).data, chunks=memchunk[1:] - ) - - elif callable(st_vari) and type(self[st_labeli].data) != da.core.Array: - st_vari_da = da.from_array( - st_vari(self[st_labeli]).data, chunks=memchunk[1:] - ) - - else: - st_vari_da = da.from_array(st_vari, chunks=memchunk[1:]) - - params[k] = ( - ("mc", "x", "time"), - state.normal( - loc=loc, # has chunks=memchunk[1:] - scale=st_vari_da**0.5, - size=rsize, - chunks=memchunk, - ), - ) - - ta_arr = np.zeros((mc_sample_size, no, nt)) - - if nta: - for ii, ta in enumerate(params["ta_mc"]): - for tai, taxi in zip(ta.values, self.trans_att.values): - ta_arr[ii, self.x.values >= taxi] = ( - ta_arr[ii, self.x.values >= taxi] + tai - ) - params["ta_mc_arr"] = (("mc", "x", "time"), ta_arr) - - if fixed_alpha: - params["tmpf_mc_set"] = ( - params["gamma_mc"] - / ( - ( - np.log(params["r_st"]) - - np.log(params["r_ast"]) - + (params["c_mc"] + params["ta_mc_arr"]) - ) - + params["alpha_mc"] - ) - - 273.15 - ) - else: - params["tmpf_mc_set"] = ( - params["gamma_mc"] - / ( - ( - np.log(params["r_st"]) - - np.log(params["r_ast"]) - + (params["c_mc"] + params["ta_mc_arr"]) - ) - + (params["dalpha_mc"] * params.x) - ) - - 273.15 - ) - - avg_dims = ["mc"] - avg_axis = params["tmpf_mc_set"].get_axis_num(avg_dims) - out["tmpf_mc_var"] = (params["tmpf_mc_set"] - self["tmpf"]).var( - dim=avg_dims, ddof=1 - ) - - if conf_ints: - new_chunks = ((len(conf_ints),),) + params["tmpf_mc_set"].chunks[1:] - - qq = params["tmpf_mc_set"] - - q = qq.data.map_blocks( - lambda x: np.percentile(x, q=conf_ints, axis=avg_axis), - chunks=new_chunks, # - drop_axis=avg_axis, # avg dimesnions are dropped from input arr - new_axis=0, - ) # The new CI dimension is added as first axis - - out["tmpf_mc"] = (("CI", "x", "time"), q) - - if not mc_remove_set_flag: - out.update(params) - - self.update(out) - return out - - def conf_int_double_ended( - self, - sections=None, - p_val="p_val", - p_cov="p_cov", - st_var=None, - ast_var=None, - rst_var=None, - rast_var=None, - conf_ints=None, - mc_sample_size=100, - var_only_sections=False, - da_random_state=None, - mc_remove_set_flag=True, - reduce_memory_usage=False, - **kwargs, - ): - r""" - Estimation of the confidence intervals for the temperatures measured - with a double-ended setup. - Double-ended setups require four additional steps to estimate the - confidence intervals for the temperature. First, the variances of the - Stokes and anti-Stokes intensity measurements of the forward and - backward channels are estimated following the steps in - Section 4 [1]_. See `ds.variance_stokes_constant()`. - A Normal distribution is assigned to each - intensity measurement that is centered at the measurement and using the - estimated variance. Second, a multi-variate Normal distribution is - assigned to the estimated parameters using the covariance matrix from - the calibration procedure presented in Section 6 [1]_ (`p_cov`). Third, - Normal distributions are assigned for :math:`A` (`ds.alpha`) - for each location - outside of the reference sections. These distributions are centered - around :math:`A_p` and have variance :math:`\sigma^2\left[A_p\\right]` - given by Equations 44 and 45. Fourth, the distributions are sampled - and :math:`T_{\mathrm{F},m,n}` and :math:`T_{\mathrm{B},m,n}` are - computed with Equations 16 and 17, respectively. Fifth, step four is repeated to - compute, e.g., 10,000 realizations (`mc_sample_size`) of :math:`T_{\mathrm{F},m,n}` and - :math:`T_{\mathrm{B},m,n}` to approximate their probability density - functions. Sixth, the standard uncertainties of - :math:`T_{\mathrm{F},m,n}` and :math:`T_{\mathrm{B},m,n}` - (:math:`\sigma\left[T_{\mathrm{F},m,n}\\right]` and - :math:`\sigma\left[T_{\mathrm{B},m,n}\\right]`) are estimated with the - standard deviation of their realizations. Seventh, for each realization - :math:`i` the temperature :math:`T_{m,n,i}` is computed as the weighted - average of :math:`T_{\mathrm{F},m,n,i}` and - :math:`T_{\mathrm{B},m,n,i}`: - - .. math:: - - T_{m,n,i} =\ - \sigma^2\left[T_{m,n}\\right]\left({\\frac{T_{\mathrm{F},m,n,i}}{\ - \sigma^2\left[T_{\mathrm{F},m,n}\\right]} +\ - \\frac{T_{\mathrm{B},m,n,i}}{\ - \sigma^2\left[T_{\mathrm{B},m,n}\\right]}}\\right) - - where - - .. math:: - - \sigma^2\left[T_{m,n}\\right] = \\frac{1}{1 /\ - \sigma^2\left[T_{\mathrm{F},m,n}\\right] + 1 /\ - \sigma^2\left[T_{\mathrm{B},m,n}\\right]} - - The best estimate of the temperature :math:`T_{m,n}` is computed - directly from the best estimates of :math:`T_{\mathrm{F},m,n}` and - :math:`T_{\mathrm{B},m,n}` as: - - .. math:: - T_{m,n} =\ - \sigma^2\left[T_{m,n}\\right]\left({\\frac{T_{\mathrm{F},m,n}}{\ - \sigma^2\left[T_{\mathrm{F},m,n}\\right]} + \\frac{T_{\mathrm{B},m,n}}{\ - \sigma^2\left[T_{\mathrm{B},m,n}\\right]}}\\right) - - Alternatively, the best estimate of :math:`T_{m,n}` can be approximated - with the mean of the :math:`T_{m,n,i}` values. Finally, the 95\% - confidence interval for :math:`T_{m,n}` are estimated with the 2.5\% and - 97.5\% percentiles of :math:`T_{m,n,i}`. - - Assumes sections are set. - - Parameters - ---------- - p_val : array-like, optional - Define `p_val`, `p_var`, `p_cov` if you used an external function - for calibration. Has size `1 + 2 * nt + nx + 2 * nt * nta`. - First value is :math:`\gamma`, then `nt` times - :math:`D_\mathrm{F}`, then `nt` times - :math:`D_\mathrm{B}`, then for each location :math:`D_\mathrm{B}`, - then for each connector that introduces directional attenuation two - parameters per time step. - p_cov : array-like, optional - The covariances of `p_val`. Square matrix. - If set to False, no uncertainty in the parameters is propagated - into the confidence intervals. Similar to the spec sheets of the DTS - manufacturers. And similar to passing an array filled with zeros. - st_var, ast_var, rst_var, rast_var : float, callable, array-like, optional - The variance of the measurement noise of the Stokes signals in the - forward direction. If `float` the variance of the noise from the - Stokes detector is described with a single value. - If `callable` the variance of the noise from the Stokes detector is - a function of the intensity, as defined in the callable function. - Or manually define a variance with a DataArray of the shape - `ds.st.shape`, where the variance can be a function of time and/or - x. Required if method is wls. - conf_ints : iterable object of float - A list with the confidence boundaries that are calculated. Valid - values are between [0, 1]. - mc_sample_size : int - Size of the monte carlo parameter set used to calculate the - confidence interval - var_only_sections : bool - useful if using the ci_avg_x_flag. Only calculates the var over the - sections, so that the values can be compared with accuracy along the - reference sections. Where the accuracy is the variance of the - residuals between the estimated temperature and temperature of the - water baths. - da_random_state - For testing purposes. Similar to random seed. The seed for dask. - Makes random not so random. To produce reproducable results for - testing environments. - mc_remove_set_flag : bool - Remove the monte carlo data set, from which the CI and the - variance are calculated. - reduce_memory_usage : bool - Use less memory but at the expense of longer computation time - - Returns - ------- - - References - ---------- - .. [1] des Tombe, B., Schilperoort, B., & Bakker, M. (2020). Estimation - of Temperature and Associated Uncertainty from Fiber-Optic Raman- - Spectrum Distributed Temperature Sensing. Sensors, 20(8), 2235. - https://doi.org/10.3390/s20082235 - - """ - - def create_da_ta2(no, i_splice, direction="fw", chunks=None): - """create mask array mc, o, nt""" - - if direction == "fw": - arr = da.concatenate( - ( - da.zeros((1, i_splice, 1), chunks=(1, i_splice, 1), dtype=bool), - da.ones( - (1, no - i_splice, 1), - chunks=(1, no - i_splice, 1), - dtype=bool, - ), - ), - axis=1, - ).rechunk((1, chunks[1], 1)) - else: - arr = da.concatenate( - ( - da.ones((1, i_splice, 1), chunks=(1, i_splice, 1), dtype=bool), - da.zeros( - (1, no - i_splice, 1), - chunks=(1, no - i_splice, 1), - dtype=bool, - ), - ), - axis=1, - ).rechunk((1, chunks[1], 1)) - return arr - - check_deprecated_kwargs(kwargs) - - out = xr.Dataset() - params = xr.Dataset() - - if da_random_state: - # In testing environments - assert isinstance(da_random_state, da.random.RandomState) - state = da_random_state - else: - state = da.random.RandomState() - - if conf_ints: - assert "tmpw", ( - "Current implementation requires you to " - 'define "tmpw" when estimating confidence ' - "intervals" - ) - - no, nt = self.st.shape - nta = self.trans_att.size - npar = 1 + 2 * nt + no + nt * 2 * nta # number of parameters - - rsize = (mc_sample_size, no, nt) - - if reduce_memory_usage: - memchunk = da.ones( - (mc_sample_size, no, nt), chunks={0: -1, 1: 1, 2: "auto"} - ).chunks - else: - memchunk = da.ones( - (mc_sample_size, no, nt), chunks={0: -1, 1: "auto", 2: "auto"} - ).chunks - - params.coords["mc"] = range(mc_sample_size) - params.coords["x"] = self.x - params.coords["time"] = self.time - - if conf_ints: - self.coords["CI"] = conf_ints - params.coords["CI"] = conf_ints - - assert isinstance(p_val, (str, np.ndarray, np.generic)) - if isinstance(p_val, str): - p_val = self[p_val].values - assert p_val.shape == (npar,), ( - "Did you set 'talpha' as " - "keyword argument of the " - "conf_int_double_ended() function?" - ) - - assert isinstance(p_cov, (str, np.ndarray, np.generic, bool)) - - if isinstance(p_cov, bool) and not p_cov: - # Exclude parameter uncertainty if p_cov == False - gamma = p_val[0] - d_fw = p_val[1 : nt + 1] - d_bw = p_val[1 + nt : 2 * nt + 1] - alpha = p_val[2 * nt + 1 : 2 * nt + 1 + no] - - params["gamma_mc"] = (tuple(), gamma) - params["alpha_mc"] = (("x",), alpha) - params["df_mc"] = (("time",), d_fw) - params["db_mc"] = (("time",), d_bw) - - if nta: - ta = p_val[2 * nt + 1 + no :].reshape((nt, 2, nta), order="F") - ta_fw = ta[:, 0, :] - ta_bw = ta[:, 1, :] - - ta_fw_arr = np.zeros((no, nt)) - for tai, taxi in zip(ta_fw.T, params.coords["trans_att"].values): - ta_fw_arr[params.x.values >= taxi] = ( - ta_fw_arr[params.x.values >= taxi] + tai - ) - - ta_bw_arr = np.zeros((no, nt)) - for tai, taxi in zip(ta_bw.T, params.coords["trans_att"].values): - ta_bw_arr[params.x.values < taxi] = ( - ta_bw_arr[params.x.values < taxi] + tai - ) - - params["talpha_fw_mc"] = (("x", "time"), ta_fw_arr) - params["talpha_bw_mc"] = (("x", "time"), ta_bw_arr) - - elif isinstance(p_cov, bool) and p_cov: - raise NotImplementedError("Not an implemented option. Check p_cov argument") - - else: - # WLS - if isinstance(p_cov, str): - p_cov = self[p_cov].values - assert p_cov.shape == (npar, npar) - - assert sections is not None, "Define sections" - ix_sec = self.ufunc_per_section( - sections=sections, x_indices=True, calc_per="all" - ) - nx_sec = ix_sec.size - from_i = np.concatenate( - ( - np.arange(1 + 2 * nt), - 1 + 2 * nt + ix_sec, - np.arange(1 + 2 * nt + no, 1 + 2 * nt + no + nt * 2 * nta), - ) - ) - iox_sec1, iox_sec2 = np.meshgrid(from_i, from_i, indexing="ij") - po_val = p_val[from_i] - po_cov = p_cov[iox_sec1, iox_sec2] - - po_mc = sst.multivariate_normal.rvs( - mean=po_val, cov=po_cov, size=mc_sample_size - ) - - gamma = po_mc[:, 0] - d_fw = po_mc[:, 1 : nt + 1] - d_bw = po_mc[:, 1 + nt : 2 * nt + 1] - - params["gamma_mc"] = (("mc",), gamma) - params["df_mc"] = (("mc", "time"), d_fw) - params["db_mc"] = (("mc", "time"), d_bw) - - # calculate alpha seperately - alpha = np.zeros((mc_sample_size, no), dtype=float) - alpha[:, ix_sec] = po_mc[:, 1 + 2 * nt : 1 + 2 * nt + nx_sec] - - not_ix_sec = np.array([i for i in range(no) if i not in ix_sec]) - - if np.any(not_ix_sec): - not_alpha_val = p_val[2 * nt + 1 + not_ix_sec] - not_alpha_var = p_cov[2 * nt + 1 + not_ix_sec, 2 * nt + 1 + not_ix_sec] - - not_alpha_mc = np.random.normal( - loc=not_alpha_val, - scale=not_alpha_var**0.5, - size=(mc_sample_size, not_alpha_val.size), - ) - - alpha[:, not_ix_sec] = not_alpha_mc - - params["alpha_mc"] = (("mc", "x"), alpha) - - if nta: - ta = po_mc[:, 2 * nt + 1 + nx_sec :].reshape( - (mc_sample_size, nt, 2, nta), order="F" - ) - ta_fw = ta[:, :, 0, :] - ta_bw = ta[:, :, 1, :] - - ta_fw_arr = da.zeros( - (mc_sample_size, no, nt), chunks=memchunk, dtype=float - ) - for tai, taxi in zip( - ta_fw.swapaxes(0, 2), params.coords["trans_att"].values - ): - # iterate over the splices - i_splice = sum(params.x.values < taxi) - mask = create_da_ta2(no, i_splice, direction="fw", chunks=memchunk) - - ta_fw_arr += mask * tai.T[:, None, :] - - ta_bw_arr = da.zeros( - (mc_sample_size, no, nt), chunks=memchunk, dtype=float - ) - for tai, taxi in zip( - ta_bw.swapaxes(0, 2), params.coords["trans_att"].values - ): - i_splice = sum(params.x.values < taxi) - mask = create_da_ta2(no, i_splice, direction="bw", chunks=memchunk) - - ta_bw_arr += mask * tai.T[:, None, :] - - params["talpha_fw_mc"] = (("mc", "x", "time"), ta_fw_arr) - params["talpha_bw_mc"] = (("mc", "x", "time"), ta_bw_arr) - - # Draw from the normal distributions for the Stokes intensities - for k, st_labeli, st_vari in zip( - ["r_st", "r_ast", "r_rst", "r_rast"], - ["st", "ast", "rst", "rast"], - [st_var, ast_var, rst_var, rast_var], - ): - # Load the mean as chunked Dask array, otherwise eats memory - if type(self[st_labeli].data) == da.core.Array: - loc = da.asarray(self[st_labeli].data, chunks=memchunk[1:]) - else: - loc = da.from_array(self[st_labeli].data, chunks=memchunk[1:]) - - # Make sure variance is of size (no, nt) - if np.size(st_vari) > 1: - if st_vari.shape == self[st_labeli].shape: - pass - else: - st_vari = np.broadcast_to(st_vari, (no, nt)) - else: - pass - - # Load variance as chunked Dask array, otherwise eats memory - if type(st_vari) == da.core.Array: - st_vari_da = da.asarray(st_vari, chunks=memchunk[1:]) - - elif callable(st_vari) and type(self[st_labeli].data) == da.core.Array: - st_vari_da = da.asarray( - st_vari(self[st_labeli]).data, chunks=memchunk[1:] - ) - - elif callable(st_vari) and type(self[st_labeli].data) != da.core.Array: - st_vari_da = da.from_array( - st_vari(self[st_labeli]).data, chunks=memchunk[1:] - ) - - else: - st_vari_da = da.from_array(st_vari, chunks=memchunk[1:]) - - params[k] = ( - ("mc", "x", "time"), - state.normal( - loc=loc, # has chunks=memchunk[1:] - scale=st_vari_da**0.5, - size=rsize, - chunks=memchunk, - ), - ) - - for label in ["tmpf", "tmpb"]: - if "tmpw" or label: - if label == "tmpf": - if nta: - params["tmpf_mc_set"] = ( - params["gamma_mc"] - / ( - np.log(params["r_st"] / params["r_ast"]) - + params["df_mc"] - + params["alpha_mc"] - + params["talpha_fw_mc"] - ) - - 273.15 - ) - else: - params["tmpf_mc_set"] = ( - params["gamma_mc"] - / ( - np.log(params["r_st"] / params["r_ast"]) - + params["df_mc"] - + params["alpha_mc"] - ) - - 273.15 - ) - else: - if nta: - params["tmpb_mc_set"] = ( - params["gamma_mc"] - / ( - np.log(params["r_rst"] / params["r_rast"]) - + params["db_mc"] - - params["alpha_mc"] - + params["talpha_bw_mc"] - ) - - 273.15 - ) - else: - params["tmpb_mc_set"] = ( - params["gamma_mc"] - / ( - np.log(params["r_rst"] / params["r_rast"]) - + params["db_mc"] - - params["alpha_mc"] - ) - - 273.15 - ) - - if var_only_sections: - # sets the values outside the reference sections to NaN - xi = self.ufunc_per_section( - sections=sections, x_indices=True, calc_per="all" - ) - x_mask_ = [ - True if ix in xi else False for ix in range(params.x.size) - ] - x_mask = np.reshape(x_mask_, (1, -1, 1)) - params[label + "_mc_set"] = params[label + "_mc_set"].where(x_mask) - - # subtract the mean temperature - q = params[label + "_mc_set"] - self[label] - out[label + "_mc_var"] = q.var(dim="mc", ddof=1) - - if conf_ints: - new_chunks = list(params[label + "_mc_set"].chunks) - new_chunks[0] = (len(conf_ints),) - avg_axis = params[label + "_mc_set"].get_axis_num("mc") - q = params[label + "_mc_set"].data.map_blocks( - lambda x: np.percentile(x, q=conf_ints, axis=avg_axis), - chunks=new_chunks, # - drop_axis=avg_axis, - # avg dimensions are dropped from input arr - new_axis=0, - ) # The new CI dimension is added as firsaxis - - out[label + "_mc"] = (("CI", "x", "time"), q) - - # Weighted mean of the forward and backward - tmpw_var = 1 / (1 / out["tmpf_mc_var"] + 1 / out["tmpb_mc_var"]) - - q = ( - params["tmpf_mc_set"] / out["tmpf_mc_var"] - + params["tmpb_mc_set"] / out["tmpb_mc_var"] - ) * tmpw_var - - params["tmpw" + "_mc_set"] = q # - - out["tmpw"] = ( - self["tmpf"] / out["tmpf_mc_var"] + self["tmpb"] / out["tmpb_mc_var"] - ) * tmpw_var - - q = params["tmpw" + "_mc_set"] - self["tmpw"] - out["tmpw" + "_mc_var"] = q.var(dim="mc", ddof=1) - - # Calculate the CI of the weighted MC_set - if conf_ints: - new_chunks_weighted = ((len(conf_ints),),) + memchunk[1:] - avg_axis = params["tmpw" + "_mc_set"].get_axis_num("mc") - q2 = params["tmpw" + "_mc_set"].data.map_blocks( - lambda x: np.percentile(x, q=conf_ints, axis=avg_axis), - chunks=new_chunks_weighted, # Explicitly define output chunks - drop_axis=avg_axis, # avg dimensions are dropped - new_axis=0, - dtype=float, - ) # The new CI dimension is added as first axis - out["tmpw" + "_mc"] = (("CI", "x", "time"), q2) - - # Clean up the garbage. All arrays with a Monte Carlo dimension. - if mc_remove_set_flag: - remove_mc_set = [ - "r_st", - "r_ast", - "r_rst", - "r_rast", - "gamma_mc", - "alpha_mc", - "df_mc", - "db_mc", - ] - - for i in ["tmpf", "tmpb", "tmpw"]: - remove_mc_set.append(i + "_mc_set") - - if nta: - remove_mc_set.append('talpha"_fw_mc') - remove_mc_set.append('talpha"_bw_mc') - - for k in remove_mc_set: - if k in out: - del out[k] - - if not mc_remove_set_flag: - out.update(params) - - self.update(out) - return out - - def in_confidence_interval(self, ci_label, conf_ints=None, sections=None): - """ - Returns an array with bools wether the temperature of the reference - sections are within the confidence intervals - - Parameters - ---------- - sections : Dict[str, List[slice]] - ci_label : str - The label of the data containing the confidence intervals. - conf_ints : Tuple - A tuple containing two floats between 0 and 1, representing the - levels between which the reference temperature should lay. - - Returns - ------- - - """ - if sections is None: - sections = self.sections - else: - sections = validate_sections(self, sections) - - if conf_ints is None: - conf_ints = self[ci_label].values - - assert len(conf_ints) == 2, "Please define conf_ints" - - tmp_dn = self[ci_label].sel(CI=conf_ints[0], method="nearest") - tmp_up = self[ci_label].sel(CI=conf_ints[1], method="nearest") - - ref = self.ufunc_per_section( - sections=sections, label="st", ref_temp_broadcasted=True, calc_per="all" - ) - ix_resid = self.ufunc_per_section( - sections=sections, x_indices=True, calc_per="all" - ) - ref_sorted = np.full(shape=tmp_dn.shape, fill_value=np.nan) - ref_sorted[ix_resid, :] = ref - ref_da = xr.DataArray(data=ref_sorted, coords=tmp_dn.coords) - - mask_dn = ref_da >= tmp_dn - mask_up = ref_da <= tmp_up - - return np.logical_and(mask_dn, mask_up) - - def temperature_residuals(self, label=None, sections=None): - """ - Compute the temperature residuals, between the known temperature of the - reference sections and the DTS temperature. - - Parameters - ---------- - label : str - The key of the temperature DataArray - sections : Dict[str, List[slice]], optional - If `None` is supplied, `ds.sections` is used. Define calibration - sections. Each section requires a reference temperature time series, - such as the temperature measured by an external temperature sensor. - They should already be part of the DataStore object. `sections` - is defined with a dictionary with its keywords of the - names of the reference temperature time series. Its values are - lists of slice objects, where each slice object is a fiber stretch - that has the reference temperature. Afterwards, `sections` is stored - under `ds.sections`. - - Returns - ------- - resid_da : xarray.DataArray - The residuals as DataArray - """ - if sections is None: - sections = self.sections - else: - sections = validate_sections(self, sections) - - resid_temp = self.ufunc_per_section( - sections=sections, label=label, temp_err=True, calc_per="all" - ) - resid_x = self.ufunc_per_section(sections=sections, label="x", calc_per="all") - - resid_ix = np.array([np.argmin(np.abs(ai - self.x.data)) for ai in resid_x]) - - resid_sorted = np.full(shape=self[label].shape, fill_value=np.nan) - resid_sorted[resid_ix, :] = resid_temp - resid_da = xr.DataArray( - data=resid_sorted, - dims=("x", "time"), - coords={"x": self.x, "time": self.time}, - ) - return resid_da - - def ufunc_per_section( - self, - sections=None, - func=None, - label=None, - subtract_from_label=None, - temp_err=False, - x_indices=False, - ref_temp_broadcasted=False, - calc_per="stretch", - **func_kwargs, - ): - """ - User function applied to parts of the cable. Super useful, - many options and slightly - complicated. - - The function `func` is taken over all the timesteps and calculated - per `calc_per`. This - is returned as a dictionary - - Parameters - ---------- - sections : Dict[str, List[slice]], optional - If `None` is supplied, `ds.sections` is used. Define calibration - sections. Each section requires a reference temperature time series, - such as the temperature measured by an external temperature sensor. - They should already be part of the DataStore object. `sections` - is defined with a dictionary with its keywords of the - names of the reference temperature time series. Its values are - lists of slice objects, where each slice object is a fiber stretch - that has the reference temperature. Afterwards, `sections` is stored - under `ds.sections`. - func : callable, str - A numpy function, or lambda function to apple to each 'calc_per'. - label - subtract_from_label - temp_err : bool - The argument of the function is label minus the reference - temperature. - x_indices : bool - To retreive an integer array with the indices of the - x-coordinates in the section/stretch. The indices are sorted. - ref_temp_broadcasted : bool - calc_per : {'all', 'section', 'stretch'} - func_kwargs : dict - Dictionary with options that are passed to func - - TODO: Spend time on creating a slice instead of appendng everything\ - to a list and concatenating after. - - - Returns - ------- - - Examples - -------- - - 1. Calculate the variance of the residuals in the along ALL the\ - reference sections wrt the temperature of the water baths - - >>> tmpf_var = d.ufunc_per_section( - >>> sections=sections, - >>> func='var', - >>> calc_per='all', - >>> label='tmpf', - >>> temp_err=True) - - 2. Calculate the variance of the residuals in the along PER\ - reference section wrt the temperature of the water baths - - >>> tmpf_var = d.ufunc_per_section( - >>> sections=sections, - >>> func='var', - >>> calc_per='stretch', - >>> label='tmpf', - >>> temp_err=True) - - 3. Calculate the variance of the residuals in the along PER\ - water bath wrt the temperature of the water baths - - >>> tmpf_var = d.ufunc_per_section( - >>> sections=sections, - >>> func='var', - >>> calc_per='section', - >>> label='tmpf', - >>> temp_err=True) - - 4. Obtain the coordinates of the measurements per section - - >>> locs = d.ufunc_per_section( - >>> sections=sections, - >>> func=None, - >>> label='x', - >>> temp_err=False, - >>> ref_temp_broadcasted=False, - >>> calc_per='stretch') - - 5. Number of observations per stretch - - >>> nlocs = d.ufunc_per_section( - >>> sections=sections, - >>> func=len, - >>> label='x', - >>> temp_err=False, - >>> ref_temp_broadcasted=False, - >>> calc_per='stretch') - - 6. broadcast the temperature of the reference sections to\ - stretch/section/all dimensions. The value of the reference\ - temperature (a timeseries) is broadcasted to the shape of self[\ - label]. The self[label] is not used for anything else. - - >>> temp_ref = d.ufunc_per_section( - >>> label='st', - >>> ref_temp_broadcasted=True, - >>> calc_per='all') - - 7. x-coordinate index - - >>> ix_loc = d.ufunc_per_section(sections=sections, x_indices=True) - - - Note - ---- - If `self[label]` or `self[subtract_from_label]` is a Dask array, a Dask - array is returned else a numpy array is returned - """ - # if sections is None: - # sections = self.sections - if label is None: - dataarray = None - else: - dataarray = self[label] - - if x_indices: - x_coords = self.x - reference_dataset = None - - else: - sections = validate_sections(self, sections) - - x_coords = None - reference_dataset = {k: self[k] for k in sections} - - out = ufunc_per_section_helper( - x_coords=x_coords, - sections=sections, - func=func, - dataarray=dataarray, - subtract_from_dataarray=subtract_from_label, - reference_dataset=reference_dataset, - subtract_reference_from_dataarray=temp_err, - ref_temp_broadcasted=ref_temp_broadcasted, - calc_per=calc_per, - **func_kwargs, - ) - return out diff --git a/src/dtscalibration/datastore_utils.py b/src/dtscalibration/datastore_utils.py index ac8c147d..f8ca0eec 100644 --- a/src/dtscalibration/datastore_utils.py +++ b/src/dtscalibration/datastore_utils.py @@ -394,72 +394,6 @@ def get_taf_values(self, pval, x, trans_att, axis=""): return arr_out -def check_deprecated_kwargs(kwargs): - """ - Internal function that parses the `kwargs` for depreciated keyword - arguments. - - Depreciated keywords raise an error, pending to be depreciated do not. - But this requires that the code currently deals with those arguments. - - Parameters - ---------- - kwargs : Dict - A dictionary with keyword arguments. - - Returns - ------- - - """ - msg = """Previously, it was possible to manually set the label from - which the Stokes and anti-Stokes were read within the DataStore - object. To reduce the clutter in the code base and be able to - maintain it, this option was removed. - See: https://github.com/dtscalibration/python-dts-calibration/issues/81 - - The new **fixed** names are: st, ast, rst, rast. - - It is still possible to use the previous defaults, for example when - reading stored measurements from netCDF, by renaming the labels. The - old default labels were ST, AST, REV-ST, REV-AST. - - ``` - ds = open_datastore(path_to_old_file) - ds = ds.rename_labels() - ds.calibration_double_ended( - st_var=1.5, - ast_var=1.5, - rst_var=1., - rast_var=1., - method='wls') - ``` - - ds.tmpw.plot() - """ - list_of_depr = [ - "st_label", - "ast_label", - "rst_label", - "rast_label", - "transient_asym_att_x", - "transient_att_x", - ] - list_of_pending_depr = [] - - kwargs = {k: v for k, v in kwargs.items() if k not in list_of_pending_depr} - - for k in kwargs: - if k in list_of_depr: - raise NotImplementedError(msg) - - if len(kwargs) != 0: - raise NotImplementedError( - "The following keywords are not " + "supported: " + ", ".join(kwargs.keys()) - ) - - pass - - def get_netcdf_encoding( ds: xr.Dataset, zlib: bool = True, complevel: int = 5, **kwargs ) -> dict: diff --git a/src/dtscalibration/datastore_accessor.py b/src/dtscalibration/dts_accessor.py similarity index 99% rename from src/dtscalibration/datastore_accessor.py rename to src/dtscalibration/dts_accessor.py index 3227ca74..1a074ce6 100644 --- a/src/dtscalibration/datastore_accessor.py +++ b/src/dtscalibration/dts_accessor.py @@ -15,7 +15,7 @@ from dtscalibration.datastore_utils import get_params_from_pval_double_ended from dtscalibration.datastore_utils import get_params_from_pval_single_ended from dtscalibration.datastore_utils import ufunc_per_section_helper -from dtscalibration.io_utils import dim_attrs +from dtscalibration.io.utils import dim_attrs @xr.register_dataset_accessor("dts") diff --git a/src/dtscalibration/io/apsensing.py b/src/dtscalibration/io/apsensing.py index ec013bc6..f2164d9f 100644 --- a/src/dtscalibration/io/apsensing.py +++ b/src/dtscalibration/io/apsensing.py @@ -7,8 +7,8 @@ import dask.array as da import numpy as np import pandas as pd +import xarray as xr -from dtscalibration import DataStore from dtscalibration.io.utils import dim_attrs from dtscalibration.io.utils import get_xml_namespace from dtscalibration.io.utils import open_file @@ -103,7 +103,7 @@ def read_apsensing_files( load_in_memory=load_in_memory, ) - ds = DataStore(data_vars=data_vars, coords=coords, attrs=attrs, **kwargs) + ds = xr.Dataset(data_vars=data_vars, coords=coords, attrs=attrs, **kwargs) return ds diff --git a/src/dtscalibration/io/datastore.py b/src/dtscalibration/io/datastore.py deleted file mode 100644 index 174b84fd..00000000 --- a/src/dtscalibration/io/datastore.py +++ /dev/null @@ -1,188 +0,0 @@ -import glob -import inspect - -import xarray as xr - -from dtscalibration import DataStore - - -def open_datastore( - filename_or_obj, - group=None, - decode_cf=True, - mask_and_scale=None, - decode_times=True, - concat_characters=True, - decode_coords=True, - engine=None, - chunks=None, - lock=None, - cache=None, - drop_variables=None, - backend_kwargs=None, - load_in_memory=False, - **kwargs, -): - """Load and decode a datastore from a file or file-like object. Most - arguments are passed to xarray.open_dataset(). - - Parameters - ---------- - filename_or_obj : str, Path, file or xarray.backends.*DataStore - Strings and Path objects are interpreted as a path to a netCDF file - or an OpenDAP URL and opened with python-netCDF4, unless the filename - ends with .gz, in which case the file is gunzipped and opened with - scipy.io.netcdf (only netCDF3 supported). File-like objects are opened - with scipy.io.netcdf (only netCDF3 supported). - group : str, optional - Path to the netCDF4 group in the given file to open (only works for - netCDF4 files). - decode_cf : bool, optional - Whether to decode these variables, assuming they were saved according - to CF conventions. - mask_and_scale : bool, optional - If True, replace array values equal to `_FillValue` with NA and scale - values according to the formula `original_values * scale_factor + - add_offset`, where `_FillValue`, `scale_factor` and `add_offset` are - taken from variable attributes (if they exist). If the `_FillValue` or - `missing_value` attribute contains multiple values a warning will be - issued and all array values matching one of the multiple values will - be replaced by NA. mask_and_scale defaults to True except for the - pseudonetcdf backend. - decode_times : bool, optional - If True, decode times encoded in the standard NetCDF datetime format - into datetime objects. Otherwise, leave them encoded as numbers. - concat_characters : bool, optional - If True, concatenate along the last dimension of character arrays to - form string arrays. Dimensions will only be concatenated over (and - removed) if they have no corresponding variable and if they are only - used as the last dimension of character arrays. - decode_coords : bool, optional - If True, decode the 'coordinates' attribute to identify coordinates in - the resulting dataset. - engine : {'netcdf4', 'scipy', 'pydap', 'h5netcdf', 'pynio', - 'pseudonetcdf'}, optional - Engine to use when reading files. If not provided, the default engine - is chosen based on available dependencies, with a preference for - 'netcdf4'. - chunks : int or dict, optional - If chunks is provided, it used to load the new dataset into dask - arrays. ``chunks={}`` loads the dataset with dask using a single - chunk for all arrays. - lock : False, True or threading.Lock, optional - If chunks is provided, this argument is passed on to - :py:func:`dask.array.from_array`. By default, a global lock is - used when reading data from netCDF files with the netcdf4 and h5netcdf - engines to avoid issues with concurrent access when using dask's - multithreaded backend. - cache : bool, optional - If True, cache data loaded from the underlying datastore in memory as - NumPy arrays when accessed to avoid reading from the underlying data- - store multiple times. Defaults to True unless you specify the `chunks` - argument to use dask, in which case it defaults to False. Does not - change the behavior of coordinates corresponding to dimensions, which - always load their data from disk into a ``pandas.Index``. - drop_variables: string or iterable, optional - A variable or list of variables to exclude from being parsed from the - dataset. This may be useful to drop variables with problems or - inconsistent values. - backend_kwargs: dictionary, optional - A dictionary of keyword arguments to pass on to the backend. This - may be useful when backend options would improve performance or - allow user control of dataset processing. - - Returns - ------- - dataset : Dataset - The newly created dataset. - - See Also - -------- - xarray.open_dataset - xarray.load_dataset - """ - - xr_kws = inspect.signature(xr.open_dataset).parameters.keys() - - ds_kwargs = {k: v for k, v in kwargs.items() if k not in xr_kws} - - if chunks is None: - chunks = {} - - with xr.open_dataset( - filename_or_obj, - group=group, - decode_cf=decode_cf, - mask_and_scale=mask_and_scale, - decode_times=decode_times, - concat_characters=concat_characters, - decode_coords=decode_coords, - engine=engine, - chunks=chunks, - lock=lock, - cache=cache, - drop_variables=drop_variables, - backend_kwargs=backend_kwargs, - ) as ds_xr: - ds = DataStore( - data_vars=ds_xr.data_vars, - coords=ds_xr.coords, - attrs=ds_xr.attrs, - **ds_kwargs, - ) - - # to support deprecated st_labels - ds = ds.rename_labels(assertion=False) - - if load_in_memory: - if "cache" in kwargs: - raise TypeError("cache has no effect in this context") - return ds.load() - - else: - return ds - - -def open_mf_datastore( - path=None, paths=None, combine="by_coords", load_in_memory=False, **kwargs -): - """ - Open a datastore from multiple netCDF files. This script assumes the - datastore was split along the time dimension. But only variables with a - time dimension should be concatenated in the time dimension. Other - options from xarray do not support this. - - Parameters - ---------- - combine : {'by_coords', 'nested'}, optional - Leave it at by_coords - path : str - A file path to the stored netcdf files with an asterisk in the - filename to list all. Ensure you have leading zeros in the file - numbering. - paths : list - Define you own list of file paths. - Returns - ------- - dataset : Dataset - The newly created dataset. - """ - from xarray.backends.api import open_mfdataset - - if paths is None: - paths = sorted(glob.glob(path)) - assert paths, "No files match found with: " + path - - with open_mfdataset(paths=paths, combine=combine, **kwargs) as xds: - ds = DataStore(data_vars=xds.data_vars, coords=xds.coords, attrs=xds.attrs) - - # to support deprecated st_labels - ds = ds.rename_labels(assertion=False) - - if load_in_memory: - if "cache" in kwargs: - raise TypeError("cache has no effect in this context") - return ds.load() - - else: - return ds diff --git a/src/dtscalibration/io/sensornet.py b/src/dtscalibration/io/sensornet.py index e887f16b..ccac51c9 100644 --- a/src/dtscalibration/io/sensornet.py +++ b/src/dtscalibration/io/sensornet.py @@ -5,8 +5,8 @@ import numpy as np import pandas as pd +import xarray as xr -from dtscalibration import DataStore from dtscalibration.io.utils import coords_time from dtscalibration.io.utils import dim_attrs from dtscalibration.io.utils import open_file @@ -130,7 +130,7 @@ def read_sensornet_files( flip_reverse_measurements=flip_reverse_measurements, ) - ds = DataStore(data_vars=data_vars, coords=coords, attrs=attrs, **kwargs) + ds = xr.Dataset(data_vars=data_vars, coords=coords, attrs=attrs, **kwargs) return ds diff --git a/src/dtscalibration/io/sensortran.py b/src/dtscalibration/io/sensortran.py index 0d2ff728..b1077897 100644 --- a/src/dtscalibration/io/sensortran.py +++ b/src/dtscalibration/io/sensortran.py @@ -4,8 +4,8 @@ from typing import Union import numpy as np +import xarray as xr -from dtscalibration import DataStore from dtscalibration.io.utils import coords_time from dtscalibration.io.utils import dim_attrs @@ -16,7 +16,7 @@ def read_sensortran_files( timezone_netcdf: str = "UTC", silent: bool = False, **kwargs, -) -> DataStore: +) -> xr.Dataset: """Read a folder with measurement files from a device of the Sensortran brand. Each measurement file contains values for a single timestep. Remember to check which timezone you are working in. @@ -77,7 +77,7 @@ def read_sensortran_files( "Sensortran binary version " + f"{version} not implemented" ) - ds = DataStore(data_vars=data_vars, coords=coords, attrs=attrs, **kwargs) + ds = xr.Dataset(data_vars=data_vars, coords=coords, attrs=attrs, **kwargs) return ds diff --git a/src/dtscalibration/io/silixa.py b/src/dtscalibration/io/silixa.py index f89a672e..025b7b1f 100644 --- a/src/dtscalibration/io/silixa.py +++ b/src/dtscalibration/io/silixa.py @@ -6,8 +6,8 @@ import dask.array as da import numpy as np import pandas as pd +import xarray as xr -from dtscalibration import DataStore from dtscalibration.io.utils import coords_time from dtscalibration.io.utils import dim_attrs from dtscalibration.io.utils import get_xml_namespace @@ -98,7 +98,7 @@ def read_silixa_files( "Silixa xml version " + f"{xml_version} not implemented" ) - ds = DataStore(data_vars=data_vars, coords=coords, attrs=attrs, **kwargs) + ds = xr.Dataset(data_vars=data_vars, coords=coords, attrs=attrs, **kwargs) return ds diff --git a/tests/test_averaging.py b/tests/test_averaging.py index 557e5907..dc94931c 100644 --- a/tests/test_averaging.py +++ b/tests/test_averaging.py @@ -2,7 +2,7 @@ import numpy as np from dtscalibration import read_silixa_files -from dtscalibration.datastore_accessor import DtsAccessor # noqa: F401 +from dtscalibration import DtsAccessor # noqa: F401 np.random.seed(0) diff --git a/tests/test_datastore.py b/tests/test_datastore.py index 8b7e5461..f98291ba 100644 --- a/tests/test_datastore.py +++ b/tests/test_datastore.py @@ -21,7 +21,7 @@ from dtscalibration.datastore_utils import shift_double_ended from dtscalibration.datastore_utils import suggest_cable_shift_double_ended -from dtscalibration.datastore_accessor import DtsAccessor # noqa: F401 +from dtscalibration import DtsAccessor # noqa: F401 np.random.seed(0) diff --git a/tests/test_dtscalibration.py b/tests/test_dtscalibration.py index 8f070ec6..b3880b61 100644 --- a/tests/test_dtscalibration.py +++ b/tests/test_dtscalibration.py @@ -6,13 +6,10 @@ import scipy.sparse as sp from scipy import stats from xarray import Dataset -from dtscalibration import read_silixa_files from dtscalibration.calibrate_utils import wls_sparse from dtscalibration.calibrate_utils import wls_stats -from dtscalibration.datastore_accessor import DtsAccessor # noqa: F401 +from dtscalibration import DtsAccessor # noqa: F401 from dtscalibration.variance_stokes import variance_stokes_exponential -from dtscalibration.variance_stokes import variance_stokes_constant -from dtscalibration.variance_stokes import variance_stokes_linear np.random.seed(0) diff --git a/tests/test_variance_stokes.py b/tests/test_variance_stokes.py index a52eadf7..116c57e2 100644 --- a/tests/test_variance_stokes.py +++ b/tests/test_variance_stokes.py @@ -7,7 +7,7 @@ from xarray import Dataset from dtscalibration import read_silixa_files -from dtscalibration.datastore_accessor import DtsAccessor # noqa: F401 +from dtscalibration import DtsAccessor # noqa: F401 from dtscalibration.variance_stokes import variance_stokes_exponential from dtscalibration.variance_stokes import variance_stokes_constant from dtscalibration.variance_stokes import variance_stokes_linear From fdea611a80c21c376f8c4cbe75c580e63e0ed26a Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Mon, 16 Oct 2023 16:07:56 +0200 Subject: [PATCH 35/66] Circumventing circular import DTSaccessor --- src/dtscalibration/__init__.py | 2 -- tests/test_averaging.py | 2 +- tests/test_datastore.py | 2 +- tests/test_dtscalibration.py | 2 +- tests/test_variance_stokes.py | 2 +- 5 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/dtscalibration/__init__.py b/src/dtscalibration/__init__.py index cc42cb77..8f3d2365 100644 --- a/src/dtscalibration/__init__.py +++ b/src/dtscalibration/__init__.py @@ -1,4 +1,3 @@ -from dtscalibration import DtsAccessor from dtscalibration.datastore_utils import check_dims from dtscalibration.datastore_utils import get_netcdf_encoding from dtscalibration.datastore_utils import merge_double_ended @@ -16,7 +15,6 @@ __version__ = "2.0.0" __all__ = [ - "DTSAccessor", "read_apsensing_files", "read_sensornet_files", "read_sensortran_files", diff --git a/tests/test_averaging.py b/tests/test_averaging.py index dc94931c..63bb6350 100644 --- a/tests/test_averaging.py +++ b/tests/test_averaging.py @@ -2,7 +2,7 @@ import numpy as np from dtscalibration import read_silixa_files -from dtscalibration import DtsAccessor # noqa: F401 +from dtscalibration.dts_accessor import DtsAccessor # noqa: F401 np.random.seed(0) diff --git a/tests/test_datastore.py b/tests/test_datastore.py index f98291ba..bad2d30c 100644 --- a/tests/test_datastore.py +++ b/tests/test_datastore.py @@ -21,7 +21,7 @@ from dtscalibration.datastore_utils import shift_double_ended from dtscalibration.datastore_utils import suggest_cable_shift_double_ended -from dtscalibration import DtsAccessor # noqa: F401 +from dtscalibration.dts_accessor import DtsAccessor # noqa: F401 np.random.seed(0) diff --git a/tests/test_dtscalibration.py b/tests/test_dtscalibration.py index b3880b61..36f75e2c 100644 --- a/tests/test_dtscalibration.py +++ b/tests/test_dtscalibration.py @@ -8,7 +8,7 @@ from xarray import Dataset from dtscalibration.calibrate_utils import wls_sparse from dtscalibration.calibrate_utils import wls_stats -from dtscalibration import DtsAccessor # noqa: F401 +from dtscalibration.dts_accessor import DtsAccessor # noqa: F401 from dtscalibration.variance_stokes import variance_stokes_exponential np.random.seed(0) diff --git a/tests/test_variance_stokes.py b/tests/test_variance_stokes.py index 116c57e2..2d0219b5 100644 --- a/tests/test_variance_stokes.py +++ b/tests/test_variance_stokes.py @@ -7,7 +7,7 @@ from xarray import Dataset from dtscalibration import read_silixa_files -from dtscalibration import DtsAccessor # noqa: F401 +from dtscalibration.dts_accessor import DtsAccessor # noqa: F401 from dtscalibration.variance_stokes import variance_stokes_exponential from dtscalibration.variance_stokes import variance_stokes_constant from dtscalibration.variance_stokes import variance_stokes_linear From 3906555e64116e84c7ea930a4c33a00470ecaebf Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Mon, 16 Oct 2023 16:15:46 +0200 Subject: [PATCH 36/66] Update 03Define_sections.ipynb --- docs/notebooks/03Define_sections.ipynb | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/notebooks/03Define_sections.ipynb b/docs/notebooks/03Define_sections.ipynb index c342cd53..19f87af1 100644 --- a/docs/notebooks/03Define_sections.ipynb +++ b/docs/notebooks/03Define_sections.ipynb @@ -22,7 +22,6 @@ "outputs": [], "source": [ "import os\n", - "\n", "from dtscalibration import read_silixa_files" ] }, @@ -49,7 +48,7 @@ "source": [ "First we have a look at which temperature timeseries are available for calibration. Therefore we access `ds.data_vars` and we find `probe1Temperature` and `probe2Temperature` that refer to the temperature measurement timeseries of the two probes attached to the Ultima.\n", "\n", - "Alternatively, we can access the `ds.timeseries_keys` property to list all timeseries that can be used for calibration." + "Alternatively, we can access the `ds.dts.get_timeseries_keys()` function to list all timeseries that can be used for calibration." ] }, { @@ -65,9 +64,11 @@ }, "outputs": [], "source": [ - "print(ds.timeseries_keys) # list the available timeseeries\n", - "ds.probe1Temperature.plot(figsize=(12, 8))\n", - "# plot one of the timeseries" + "# The following command adds the .dts accessor to the xarray Dataset.\n", + "from dtscalibration.dts_accessor import DtsAccessor # noqa: E402\n", + "\n", + "print(ds.dts.get_timeseries_keys()) # list the available timeseeries\n", + "ds.probe1Temperature.plot(figsize=(12, 8));" ] }, { From 6a0f7ff4c8769a05f4fca97aad74b968bb0deb86 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Mon, 16 Oct 2023 16:18:37 +0200 Subject: [PATCH 37/66] Update 04Calculate_variance_Stokes.ipynb --- docs/notebooks/04Calculate_variance_Stokes.ipynb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/notebooks/04Calculate_variance_Stokes.ipynb b/docs/notebooks/04Calculate_variance_Stokes.ipynb index a167e999..cbf84ee4 100644 --- a/docs/notebooks/04Calculate_variance_Stokes.ipynb +++ b/docs/notebooks/04Calculate_variance_Stokes.ipynb @@ -35,6 +35,8 @@ "warnings.simplefilter(\"ignore\") # Hide warnings to avoid clutter in the notebook\n", "\n", "from dtscalibration import read_silixa_files\n", + "from dtscalibration.dts_accessor import DtsAccessor # noqa: E402\n", + "from dtscalibration.variance_stokes import variance_stokes_constant\n", "from matplotlib import pyplot as plt\n", "\n", "%matplotlib inline" @@ -119,7 +121,9 @@ }, "outputs": [], "source": [ - "I_var, residuals = ds.variance_stokes_constant(sections=sections, st_label=\"st\")\n", + "I_var, residuals = variance_stokes_constant(\n", + " ds.dts.st, sections, ds.dts.acquisitiontime_fw, reshape_residuals=True\n", + ")\n", "print(\n", " \"The variance of the Stokes signal along the reference sections \"\n", " \"is approximately {:.2f} on a {:.1f} sec acquisition time\".format(\n", @@ -213,7 +217,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.11" + "version": "3.10.10" } }, "nbformat": 4, From 1d348e37680b9287092c8654bc25d32d1f59c1af Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Mon, 16 Oct 2023 16:22:59 +0200 Subject: [PATCH 38/66] Update 07Calibrate_single_ended.ipynb --- docs/notebooks/07Calibrate_single_ended.ipynb | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/docs/notebooks/07Calibrate_single_ended.ipynb b/docs/notebooks/07Calibrate_single_ended.ipynb index 7edf0a8a..353de5dc 100644 --- a/docs/notebooks/07Calibrate_single_ended.ipynb +++ b/docs/notebooks/07Calibrate_single_ended.ipynb @@ -43,6 +43,8 @@ "import os\n", "\n", "from dtscalibration import read_silixa_files\n", + "from dtscalibration.dts_accessor import DtsAccessor # noqa: E402\n", + "from dtscalibration.variance_stokes import variance_stokes_constant\n", "import matplotlib.pyplot as plt\n", "\n", "%matplotlib inline" @@ -119,8 +121,12 @@ }, "outputs": [], "source": [ - "st_var, resid = ds.variance_stokes_constant(sections=sections, st_label=\"st\")\n", - "ast_var, _ = ds.variance_stokes_constant(sections=sections, st_label=\"ast\")" + "st_var, resid = variance_stokes_constant(\n", + " ds.dts.st, sections, ds.dts.acquisitiontime_fw, reshape_residuals=True\n", + ")\n", + "ast_var, _ = variance_stokes_constant(\n", + " ds.dts.ast, sections, ds.dts.acquisitiontime_fw, reshape_residuals=False\n", + ")" ] }, { @@ -160,7 +166,7 @@ }, "outputs": [], "source": [ - "ds.calibration_single_ended(sections=sections, st_var=st_var, ast_var=ast_var)" + "out = ds.dts.calibrate_single_ended(sections=sections, st_var=st_var, ast_var=ast_var)" ] }, { @@ -177,7 +183,7 @@ "metadata": {}, "outputs": [], "source": [ - "ds.tmpf.plot(figsize=(12, 4))" + "out.tmpf.plot(figsize=(12, 4))" ] }, { @@ -186,7 +192,7 @@ "metadata": {}, "outputs": [], "source": [ - "ds1 = ds.isel(time=0)\n", + "ds1 = out.isel(time=0)\n", "ds1.tmpf.plot(figsize=(12, 4))\n", "(ds1.tmpf_var**0.5).plot(figsize=(12, 4))\n", "plt.ylabel(\"$\\sigma$ ($^\\circ$C)\")" @@ -216,8 +222,8 @@ }, "outputs": [], "source": [ - "ds1.st.plot(figsize=(12, 8))\n", - "ds1.ast.plot()" + "ds.isel(time=0).st.plot(figsize=(12, 8))\n", + "ds.isel(time=0).ast.plot()" ] }, { From 8737e858e2b51001ad510ed54d8409b5a16fe40c Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Mon, 16 Oct 2023 16:40:10 +0200 Subject: [PATCH 39/66] Update 08Calibrate_double_ended.ipynb --- docs/notebooks/08Calibrate_double_ended.ipynb | 58 +++++++++++-------- 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/docs/notebooks/08Calibrate_double_ended.ipynb b/docs/notebooks/08Calibrate_double_ended.ipynb index 5f36c954..1eeecb53 100644 --- a/docs/notebooks/08Calibrate_double_ended.ipynb +++ b/docs/notebooks/08Calibrate_double_ended.ipynb @@ -45,6 +45,8 @@ " suggest_cable_shift_double_ended,\n", " shift_double_ended,\n", ")\n", + "from dtscalibration.dts_accessor import DtsAccessor # noqa: E402\n", + "from dtscalibration.variance_stokes import variance_stokes_constant\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", "\n", @@ -170,7 +172,9 @@ "## Estimate the variance of the noise in the Stokes and anti-Stokes measurements\n", "First calculate the variance in the measured Stokes and anti-Stokes signals, in the forward and backward direction. See Notebook 4 for more information.\n", "\n", - "The Stokes and anti-Stokes signals should follow a smooth decaying exponential. This function fits a decaying exponential to each reference section for each time step. The variance of the residuals between the measured Stokes and anti-Stokes signals and the fitted signals is used as an estimate of the variance in measured signals. This algorithm assumes that the temperature is the same for the entire section but may vary over time and differ per section." + "The Stokes and anti-Stokes signals should follow a smooth decaying exponential. This function fits a decaying exponential to each reference section for each time step. The variance of the residuals between the measured Stokes and anti-Stokes signals and the fitted signals is used as an estimate of the variance in measured signals. This algorithm assumes that the temperature is the same for the entire section but may vary over time and differ per section.\n", + "\n", + "Note that the acquisition time of the backward channel is passed to the variance_stokes function for the later two funciton calls." ] }, { @@ -186,10 +190,18 @@ }, "outputs": [], "source": [ - "st_var, resid = ds.variance_stokes_constant(sections=sections, st_label=\"st\")\n", - "ast_var, _ = ds.variance_stokes_constant(sections=sections, st_label=\"ast\")\n", - "rst_var, _ = ds.variance_stokes_constant(sections=sections, st_label=\"rst\")\n", - "rast_var, _ = ds.variance_stokes_constant(sections=sections, st_label=\"rast\")" + "st_var, resid = variance_stokes_constant(\n", + " ds.dts.st, sections, ds.dts.acquisitiontime_fw, reshape_residuals=True\n", + ")\n", + "ast_var, _ = variance_stokes_constant(\n", + " ds.dts.ast, sections, ds.dts.acquisitiontime_fw, reshape_residuals=False\n", + ")\n", + "rst_var, _ = variance_stokes_constant(\n", + " ds.dts.rst, sections, ds.dts.acquisitiontime_bw, reshape_residuals=False\n", + ")\n", + "rast_var, _ = variance_stokes_constant(\n", + " ds.dts.rast, sections, ds.dts.acquisitiontime_bw, reshape_residuals=False\n", + ")" ] }, { @@ -236,7 +248,7 @@ }, "outputs": [], "source": [ - "ds.calibration_double_ended(\n", + "out = ds.dts.calibrate_double_ended(\n", " sections=sections,\n", " st_var=st_var,\n", " ast_var=ast_var,\n", @@ -258,7 +270,7 @@ }, "outputs": [], "source": [ - "ds.tmpw.plot(figsize=(12, 4))" + "out.tmpw.plot(figsize=(12, 4))" ] }, { @@ -293,10 +305,10 @@ "metadata": {}, "outputs": [], "source": [ - "ds.tmpw_var.plot(figsize=(12, 4))\n", - "ds1 = ds.isel(time=-1) # take only the first timestep\n", + "out.tmpw_var.plot(figsize=(12, 4))\n", + "ds1 = out.isel(time=-1) # take only the first timestep\n", "(ds1.tmpw_var**0.5).plot(figsize=(12, 4))\n", - "plt.gca().set_ylabel(\"Standard error ($^\\circ$C)\")" + "plt.gca().set_ylabel(\"Standard error ($^\\circ$C)\");" ] }, { @@ -323,8 +335,8 @@ "metadata": {}, "outputs": [], "source": [ - "ds.conf_int_double_ended(\n", - " sections=sections,\n", + "out2 = ds.dts.monte_carlo_double_ended(\n", + " result=out,\n", " st_var=st_var,\n", " ast_var=ast_var,\n", " rst_var=rst_var,\n", @@ -333,7 +345,7 @@ " mc_sample_size=500,\n", ") # < increase sample size for better approximation\n", "\n", - "ds.tmpw_mc_var.plot(figsize=(12, 4))" + "out2.tmpw_mc_var.plot(figsize=(12, 4))" ] }, { @@ -349,10 +361,9 @@ }, "outputs": [], "source": [ - "ds1 = ds.isel(time=-1) # take only the first timestep\n", - "ds1.tmpw.plot(linewidth=0.7, figsize=(12, 4))\n", - "ds1.tmpw_mc.isel(CI=0).plot(linewidth=0.7, label=\"CI: 2.5%\")\n", - "ds1.tmpw_mc.isel(CI=1).plot(linewidth=0.7, label=\"CI: 97.5%\")\n", + "out.isel(time=-1).tmpw.plot(linewidth=0.7, figsize=(12, 4))\n", + "out2.isel(time=-1).tmpw_mc.isel(CI=0).plot(linewidth=0.7, label=\"CI: 2.5%\")\n", + "out2.isel(time=-1).tmpw_mc.isel(CI=1).plot(linewidth=0.7, label=\"CI: 97.5%\")\n", "plt.legend(fontsize=\"small\")" ] }, @@ -376,12 +387,13 @@ }, "outputs": [], "source": [ - "(ds1.tmpf_mc_var**0.5).plot(figsize=(12, 4))\n", - "(ds1.tmpf_var**0.5).plot()\n", - "(ds1.tmpb_mc_var**0.5).plot()\n", - "(ds1.tmpb_var**0.5).plot()\n", - "(ds1.tmpw_var**0.5).plot()\n", - "(ds1.tmpw_mc_var**0.5).plot()\n", + "\n", + "(out2.isel(time=-1).tmpf_mc_var**0.5).plot(figsize=(12, 4))\n", + "(out.isel(time=-1).tmpf_var**0.5).plot()\n", + "(out2.isel(time=-1).tmpb_mc_var**0.5).plot()\n", + "(out.isel(time=-1).tmpb_var**0.5).plot()\n", + "(out.isel(time=-1).tmpw_var**0.5).plot()\n", + "(out2.isel(time=-1).tmpw_mc_var**0.5).plot()\n", "plt.ylabel(\"$\\sigma$ ($^\\circ$C)\")" ] }, From b42110d872ae6b12a7b0092baa9e145332fa3ae8 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Mon, 16 Oct 2023 16:42:38 +0200 Subject: [PATCH 40/66] calibration_double_ended to calibrate_double_ended --- src/dtscalibration/calibrate_utils.py | 8 +++---- src/dtscalibration/dts_accessor.py | 6 ++--- tests/test_averaging.py | 2 +- tests/test_dtscalibration.py | 32 +++++++++++++-------------- tests/test_variance_stokes.py | 20 ++++++++--------- 5 files changed, 34 insertions(+), 34 deletions(-) diff --git a/src/dtscalibration/calibrate_utils.py b/src/dtscalibration/calibrate_utils.py index e0590d7c..6af751dc 100644 --- a/src/dtscalibration/calibrate_utils.py +++ b/src/dtscalibration/calibrate_utils.py @@ -490,7 +490,7 @@ def calibration_single_ended_solver( # noqa: MC0001 return (p_sol, p_var, p_cov) if calc_cov else (p_sol, p_var) -def calibration_double_ended_helper( +def calibrate_double_ended_helper( self, sections, st_var, @@ -519,7 +519,7 @@ def calibration_double_ended_helper( matching_indices = None if fix_alpha or fix_gamma: - split = calibration_double_ended_solver( + split = calibrate_double_ended_solver( self, sections, st_var, @@ -534,7 +534,7 @@ def calibration_double_ended_helper( verbose=verbose, ) else: - out = calibration_double_ended_solver( + out = calibrate_double_ended_solver( self, sections, st_var, @@ -1125,7 +1125,7 @@ def calibration_double_ended_helper( return p_cov, p_val, p_var -def calibration_double_ended_solver( # noqa: MC0001 +def calibrate_double_ended_solver( # noqa: MC0001 ds, sections=None, st_var=None, diff --git a/src/dtscalibration/dts_accessor.py b/src/dtscalibration/dts_accessor.py index 1a074ce6..651e91d3 100644 --- a/src/dtscalibration/dts_accessor.py +++ b/src/dtscalibration/dts_accessor.py @@ -5,7 +5,7 @@ import scipy.stats as sst from dtscalibration.calibration.section_utils import validate_sections, validate_sections_definition, validate_no_overlapping_sections -from dtscalibration.calibrate_utils import calibration_double_ended_helper +from dtscalibration.calibrate_utils import calibrate_double_ended_helper from dtscalibration.calibrate_utils import calibration_single_ended_helper from dtscalibration.calibrate_utils import parse_st_var from dtscalibration.calibration.section_utils import set_sections @@ -732,7 +732,7 @@ def calibrate_single_ended( return out - def calibration_double_ended( + def calibrate_double_ended( self, sections, st_var, @@ -991,7 +991,7 @@ def calibration_double_ended( ) if method == "wls": - p_cov, p_val, p_var = calibration_double_ended_helper( + p_cov, p_val, p_var = calibrate_double_ended_helper( self._obj, sections, st_var, diff --git a/tests/test_averaging.py b/tests/test_averaging.py index 63bb6350..6276db63 100644 --- a/tests/test_averaging.py +++ b/tests/test_averaging.py @@ -125,7 +125,7 @@ def test_average_measurements_double_ended(): st_var, ast_var, rst_var, rast_var = 5.0, 5.0, 5.0, 5.0 - out = ds.dts.calibration_double_ended( + out = ds.dts.calibrate_double_ended( sections=sections, st_var=st_var, ast_var=ast_var, diff --git a/tests/test_dtscalibration.py b/tests/test_dtscalibration.py index 36f75e2c..61d69b2e 100644 --- a/tests/test_dtscalibration.py +++ b/tests/test_dtscalibration.py @@ -132,7 +132,7 @@ def test_double_ended_wls_estimate_synthetic(): } # WLS - out = ds.dts.calibration_double_ended( + out = ds.dts.calibrate_double_ended( sections=sections, st_var=1e-7, ast_var=1e-7, @@ -246,7 +246,7 @@ def test_double_ended_wls_estimate_synthetic_df_and_db_are_different(): real_ans2 = np.concatenate(([gamma], df, db, E_real[:, 0])) - out = ds.dts.calibration_double_ended( + out = ds.dts.calibrate_double_ended( sections=sections, st_var=1.5, ast_var=1.5, @@ -349,7 +349,7 @@ def test_fail_if_st_labels_are_passed_to_calibration_function(): "warm": [slice(0.9 * cable_len, cable_len)], } - ds.dts.calibration_double_ended( + ds.dts.calibrate_double_ended( sections=sections, st_label="ST", ast_label="AST", @@ -456,7 +456,7 @@ def test_double_ended_asymmetrical_attenuation(): ], } - out = ds.dts.calibration_double_ended( + out = ds.dts.calibrate_double_ended( sections=sections, st_var=1.5, ast_var=1.5, @@ -559,7 +559,7 @@ def test_double_ended_one_matching_section_and_one_asym_att(): "warm": [slice(x[nx_per_sec], x[2 * nx_per_sec - 1])], } - out = ds.dts.calibration_double_ended( + out = ds.dts.calibrate_double_ended( sections=sections, st_var=1.5, ast_var=1.5, @@ -689,7 +689,7 @@ def test_double_ended_two_matching_sections_and_two_asym_atts(): ), ] - out = ds.dts.calibration_double_ended( + out = ds.dts.calibrate_double_ended( sections=sections, st_var=0.5, ast_var=0.5, @@ -785,7 +785,7 @@ def test_double_ended_wls_fix_gamma_estimate_synthetic(): } # WLS - out = ds.dts.calibration_double_ended( + out = ds.dts.calibrate_double_ended( sections=sections, st_var=1e-12, ast_var=1e-12, @@ -878,7 +878,7 @@ def test_double_ended_wls_fix_alpha_estimate_synthetic(): } # WLS - out = ds.dts.calibration_double_ended( + out = ds.dts.calibrate_double_ended( sections=sections, st_var=1e-7, ast_var=1e-7, @@ -971,7 +971,7 @@ def test_double_ended_wls_fix_alpha_fix_gamma_estimate_synthetic(): } # WLS - out = ds.dts.calibration_double_ended( + out = ds.dts.calibrate_double_ended( sections=sections, st_var=1e-7, ast_var=1e-7, @@ -1078,7 +1078,7 @@ def test_double_ended_fix_alpha_matching_sections_and_one_asym_att(): "warm": [slice(x[nx_per_sec], x[2 * nx_per_sec - 1])], } - out = ds.dts.calibration_double_ended( + out = ds.dts.calibrate_double_ended( sections=sections, st_var=1.5, ast_var=1.5, @@ -1105,7 +1105,7 @@ def test_double_ended_fix_alpha_matching_sections_and_one_asym_att(): alpha_adj = out["alpha"].values.copy() alpha_var_adj = out["alpha_var"].values.copy() - out = ds.dts.calibration_double_ended( + out = ds.dts.calibrate_double_ended( sections=sections, st_var=1.5, ast_var=1.5, @@ -1216,7 +1216,7 @@ def test_double_ended_fix_alpha_gamma_matching_sections_and_one_asym_att(): "warm": [slice(x[nx_per_sec], x[2 * nx_per_sec - 1])], } - out = ds.dts.calibration_double_ended( + out = ds.dts.calibrate_double_ended( sections=sections, st_var=1.5, ast_var=1.5, @@ -1243,7 +1243,7 @@ def test_double_ended_fix_alpha_gamma_matching_sections_and_one_asym_att(): alpha_adj = out["alpha"].values.copy() alpha_var_adj = out["alpha_var"].values.copy() - out = ds.dts.calibration_double_ended( + out = ds.dts.calibrate_double_ended( sections=sections, st_var=1.5, ast_var=1.5, @@ -1355,7 +1355,7 @@ def test_double_ended_fix_gamma_matching_sections_and_one_asym_att(): "warm": [slice(x[nx_per_sec], x[2 * nx_per_sec - 1])], } - out = ds.dts.calibration_double_ended( + out = ds.dts.calibrate_double_ended( sections=sections, st_var=1.5, ast_var=1.5, @@ -1476,7 +1476,7 @@ def test_double_ended_exponential_variance_estimate_synthetic(): rast_label = "rast" # MC variance - out = ds.dts.calibration_double_ended( + out = ds.dts.calibrate_double_ended( sections=sections, st_label=st_label, ast_label=ast_label, @@ -1634,7 +1634,7 @@ def test_estimate_variance_of_temperature_estimate(): "warm": [slice(0.5 * cable_len, 0.75 * cable_len)], } # MC variance - out = ds.dts.calibration_double_ended( + out = ds.dts.calibrate_double_ended( sections=sections, st_var=mst_var, ast_var=mast_var, diff --git a/tests/test_variance_stokes.py b/tests/test_variance_stokes.py index 2d0219b5..9a3344cb 100644 --- a/tests/test_variance_stokes.py +++ b/tests/test_variance_stokes.py @@ -279,10 +279,10 @@ def test_variance_input_types_double(): / (1 - np.exp(-gamma / temp_real)) ) - st_m = st + stats.norm.rvs(size=st.shape, scale=stokes_m_var**0.5) - ast_m = ast + stats.norm.rvs(size=ast.shape, scale=1.1 * stokes_m_var**0.5) - rst_m = rst + stats.norm.rvs(size=rst.shape, scale=0.9 * stokes_m_var**0.5) - rast_m = rast + stats.norm.rvs(size=rast.shape, scale=0.8 * stokes_m_var**0.5) + st_m = st + stats.norm.rvs(size=st.shape, scale=stokes_m_var**0.5, random_state=state) + ast_m = ast + stats.norm.rvs(size=ast.shape, scale=1.1 * stokes_m_var**0.5, random_state=state) + rst_m = rst + stats.norm.rvs(size=rst.shape, scale=0.9 * stokes_m_var**0.5, random_state=state) + rast_m = rast + stats.norm.rvs(size=rast.shape, scale=0.8 * stokes_m_var**0.5, random_state=state) print("alphaint", cable_len * (dalpha_p - dalpha_m)) print("alpha", dalpha_p - dalpha_m) @@ -312,7 +312,7 @@ def test_variance_input_types_double(): # Test float input st_var = 5.0 - out = ds.dts.calibration_double_ended( + out = ds.dts.calibrate_double_ended( sections=sections, st_var=st_var, ast_var=st_var, @@ -346,7 +346,7 @@ def st_var_callable(stokes): offset = 0 return slope * stokes + offset - out = ds.dts.calibration_double_ended( + out = ds.dts.calibrate_double_ended( sections=sections, st_var=st_var_callable, ast_var=st_var_callable, @@ -377,7 +377,7 @@ def st_var_callable(stokes): # Test input with shape of (ntime, nx) st_var = ds.st.values * 0 + 20.0 - out = ds.dts.calibration_double_ended( + out = ds.dts.calibrate_double_ended( sections=sections, st_var=st_var, ast_var=st_var, @@ -405,7 +405,7 @@ def st_var_callable(stokes): ds.st.mean(dim="time").values * 0 + np.linspace(10, 50, num=ds.st.x.size) ) - out = ds.dts.calibration_double_ended( + out = ds.dts.calibrate_double_ended( sections=sections, st_var=st_var, ast_var=st_var, @@ -436,7 +436,7 @@ def st_var_callable(stokes): # Test input with shape (ntime) st_var = ds.st.mean(dim="x").values * 0 + np.linspace(5, 200, num=nt) - out = ds.dts.calibration_double_ended( + out = ds.dts.calibrate_double_ended( sections=sections, st_var=st_var, ast_var=st_var, @@ -561,7 +561,7 @@ def test_double_ended_variance_estimate_synthetic(): mrast_var = float(mrast_var) # MC variance - out = ds.dts.calibration_double_ended( + out = ds.dts.calibrate_double_ended( sections=sections, st_var=mst_var, ast_var=mast_var, From 42c432488fc717b4cf1d9b1e0249a0cc17e05e04 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Mon, 16 Oct 2023 16:46:52 +0200 Subject: [PATCH 41/66] Update 12Datastore_from_numpy_arrays.ipynb --- .../12Datastore_from_numpy_arrays.ipynb | 90 ++++++++++++------- 1 file changed, 57 insertions(+), 33 deletions(-) diff --git a/docs/notebooks/12Datastore_from_numpy_arrays.ipynb b/docs/notebooks/12Datastore_from_numpy_arrays.ipynb index 9cd233a5..e47b93f4 100644 --- a/docs/notebooks/12Datastore_from_numpy_arrays.ipynb +++ b/docs/notebooks/12Datastore_from_numpy_arrays.ipynb @@ -4,13 +4,13 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# 12. Creating a DataStore from numpy arrays\n", + "# 12. Creating a Dataset from numpy arrays\n", "The goal of this notebook is to demonstrate how to create a `DataStore` from scratch. This can be useful if your device is not supported or if you would like to integrate the `dtscalibration` library in your current routine." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": { "execution": { "iopub.execute_input": "2022-04-06T08:11:05.700886Z", @@ -26,7 +26,9 @@ "import matplotlib.pyplot as plt\n", "import xarray as xr\n", "\n", - "from dtscalibration import DataStore, read_silixa_files" + "from dtscalibration import read_silixa_files\n", + "from dtscalibration.dts_accessor import DtsAccessor # noqa: E402\n", + "from dtscalibration.variance_stokes import variance_stokes_constant" ] }, { @@ -48,7 +50,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": { "execution": { "iopub.execute_input": "2022-04-06T08:11:07.173171Z", @@ -75,7 +77,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": { "execution": { "iopub.execute_input": "2022-04-06T08:11:07.394645Z", @@ -101,7 +103,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": { "execution": { "iopub.execute_input": "2022-04-06T08:11:07.400661Z", @@ -121,7 +123,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": { "execution": { "iopub.execute_input": "2022-04-06T08:11:07.409666Z", @@ -130,9 +132,23 @@ "shell.execute_reply": "2022-04-06T08:11:07.416796Z" } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Dimensions: (x: 1461, time: 3)\n", + "Coordinates:\n", + " * x (x) float64 -80.74 -80.62 -80.49 -80.36 ... 104.4 104.6 104.7 104.8\n", + " * time (time) datetime64[ns] 2018-05-04T12:22:17.710000 ... 2018-05-04T...\n", + "Data variables:\n", + " st (x, time) float64 -0.8058 0.4287 -0.513 ... 27.99 27.83 28.81\n", + " ast (x, time) float64 -0.2459 -0.5932 0.1111 0.3748 ... 36.2 35.7 35.16\n" + ] + } + ], "source": [ - "ds = DataStore(ds)\n", "print(ds)" ] }, @@ -153,7 +169,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": { "execution": { "iopub.execute_input": "2022-04-06T08:11:07.419863Z", @@ -181,7 +197,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": { "execution": { "iopub.execute_input": "2022-04-06T08:11:07.430326Z", @@ -190,7 +206,28 @@ "shell.execute_reply": "2022-04-06T08:11:07.508335Z" } }, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjYAAAHHCAYAAACskBIUAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB/3UlEQVR4nO3dd3hUVfoH8O/0SQ8JJAFC71U6RFARUVAsLIhlEcG6uuiqWFFZO6i4uu6KrLoIiqD+sHdX6UrvRXpvCSWkJ1PP74/k3tw7MzeZSTKZku/neXhM7ty5c2ZMZt6873vO0QkhBIiIiIiigD7UAyAiIiKqKwxsiIiIKGowsCEiIqKowcCGiIiIogYDGyIiIooaDGyIiIgoajCwISIioqjBwIaIiIiiBgMbIiIiihoMbEKodevWmDRpUqiHQUQUdEVFRbjzzjuRkZEBnU6HBx98EIcPH4ZOp8O8efPk85599lnodLrQDZQiHgObIFu1ahWeffZZ5OXlhXoo9Wr9+vW477770K1bN8TFxaFly5a44YYbsHfvXp/n79q1CyNHjkR8fDxSUlIwYcIEnDlzxuu8l156Cddeey3S09Oh0+nw7LPPao7h119/xaWXXorGjRsjOTkZAwYMwPz58/1+Dm63G6+++iratGkDq9WKnj174uOPP/Y6b9KkSdDpdF7/Onfu7Pdj2Ww2PP7442jWrBliYmIwcOBA/PLLL1XeJy8vD2lpadDpdPjss8/k477G4uvfsmXLAACzZ8/GuHHj0LJlS+h0Os1ge/Hixbj99tvRsWNHxMbGom3btrjzzjtx6tQpv57jsWPH8Nxzz2HAgAFo1KgRGjdujKFDh+LXX3+t88cK5HkNHTpU8zUymUzVPk5Vr/Hll1+uOjeQn98TJ07ghhtuQHJyMhITE3Hdddfh4MGDPs+dM2cOunTpAqvVig4dOuDf//53vV3TX9OnT8e8efNw7733Yv78+ZgwYUKtrkekxRjqAUS7VatW4bnnnsOkSZOQnJysum3Pnj3Q66MztnzllVfw+++/Y9y4cejZsyeys7Px1ltvoU+fPlizZg26d+8un3v8+HFcfPHFSEpKwvTp01FUVITXXnsN27dvx7p162A2m+Vzn376aWRkZKB37974+eefNR//m2++wejRo5GVlSX/Bfh///d/uPXWW3H27Fk89NBD1T6Hp556Ci+//DLuuusu9O/fH19//TX+/Oc/Q6fT4aabblKda7FY8N///ld1LCkpyd+XC5MmTcJnn32GBx98EB06dMC8efNw1VVXYenSpRgyZIjP+/z9739HSUmJ13HP4O3DDz/EL7/84nW8S5cuAMr/XxUWFmLAgAFVBg6PP/44cnNzMW7cOHTo0AEHDx7EW2+9he+++w5btmxBRkZGlc/x66+/xiuvvILRo0dj4sSJcDqd+PDDD3H55Zfj/fffx2233VZnjxXI83rqqadw5513qo4VFxfjnnvuwRVXXFHt4/gKljds2IA333zT6/7+/vwWFRXh0ksvRX5+Pp588kmYTCa88cYbuOSSS7BlyxakpqbK577zzju45557MHbsWEyZMgUrV67E3/72N5SUlODxxx8P6jUDsWTJEgwaNAjPPPOMfEwIgdLSUr8CSCK/CQqqmTNnCgDi0KFDoR5Kvfr999+FzWZTHdu7d6+wWCxi/PjxquP33nuviImJEUeOHJGP/fLLLwKAeOedd1TnSq/jmTNnBADxzDPP+Hz8yy+/XDRr1kyUlZXJxxwOh2jXrp3o2bNnteM/fvy4MJlMYvLkyfIxt9stLrroIpGZmSmcTqd8fOLEiSIuLq7aa2pZu3atACBmzpwpHystLRXt2rUTWVlZPu+zfft2YTQaxfPPPy8AiEWLFmlef/LkyaKqX/XDhw8Lt9sthBAiLi5OTJw40ed5y5cvFy6Xy+sYAPHUU09pXl+yY8cOcebMGdWxsrIy0blzZ5GZmVmnjyWE/8/Ll/nz5wsAYsGCBX7fR+mOO+4QOp1OHDt2THXc35/fV155RQAQ69atk4/t2rVLGAwGMXXqVPlYSUmJSE1NFaNGjVLdf/z48SIuLk7k5uYG9ZqBaNOmjdc1fXnmmWeq/Hklqg5/eoJI+gX1/Ce9ubVq1Ur1Zjt37lwBQKxcuVLcf//9onHjxiIpKUncfffdwmazifPnz4sJEyaI5ORkkZycLB599FH5jVvicrnEG2+8Ibp27SosFotIS0sTd999d43fjOpanz59RJ8+fVTH0tLSxLhx47zO7dixo7jssst8Xqe6D4aBAweKbt26+Tw+cODAasc5a9YsAUDs3LlTdXzhwoXy/yOJFNg4nU6Rn59f7bU9Pfroo8JgMHjdd/r06QKAOHr0qNd9hg0bJsaNGyeWLl1a68BGKdAAQAghUlJSxJgxY1THzpw5I3bt2iWKi4urvf+UKVMEAFFQUBC0xwr0eV155ZUiLi5OFBUV+X0fSVlZmUhOThZDhw7VPKe6n9/+/fuL/v37ex2/4oorRLt27eTvv//+ewFAfP/996rzVq1aJQCI+fPnB/Wa/pB+Rn29Dx46dEgAEHPnzpXP1wps5s+fL/r06SOsVqto1KiRuPHGG71+Ny655BLRrVs3sXPnTjF06FARExMjmjVrJl555RXVeTabTUybNk306dNHJCYmitjYWDFkyBCxZMkS1XnS+GbOnCneeust0aZNGxETEyMuv/xycfToUeF2u8Xzzz8vmjdvLqxWq7j22mvFuXPnvMb+ww8/iCFDhojY2FgRHx8vrrrqKrFjx46AXse6UFpaKp555hnRoUMHYbFYREZGhvjTn/4k9u/fX+9jCaborIOEiTFjxuDmm28GALzxxhuYP38+5s+fjyZNmlR5v/vvvx/79u3Dc889h2uvvRbvvvsupk2bhmuuuQYulwvTp0/HkCFDMHPmTK80+F/+8hc8+uijGDx4MN58803cdtttWLBgAUaMGAGHw1Hl49psNpw9e9avfzUhhEBOTg4aN24sHztx4gROnz6Nfv36eZ0/YMAAbN68uUaPNXToUOzcuRPTpk3D/v37ceDAAbzwwgvYsGEDHnvssWrvv3nzZsTFxcnlGuWYpNuVSkpKkJiYiKSkJKSkpGDy5MkoKirya6ybN29Gx44dkZiY6POxtmzZojq+aNEirFq1Cq+++qpf1w+moqIiFBUVqf6fAsBbb72FLl26YN26ddVeIzs7G7GxsYiNjQ36Y/njzJkz+OWXXzB69GjExcUFfP8ffvgBeXl5GD9+fI0e3+12Y9u2bZq/EwcOHEBhYSGAyp9Dz3P79u0LvV4v3x6Ma/qrS5cumD9/Pho3boxevXr5/T6o9NJLL+HWW29Fhw4d8Prrr+PBBx/E4sWLcfHFF3v1L54/fx4jR47EBRdcgH/84x/o3LkzHn/8cfz444/yOQUFBfjvf/+LoUOH4pVXXsGzzz6LM2fOYMSIEV6/bwCwYMECvP3227j//vvx8MMPY/ny5bjhhhvw9NNP46effsLjjz+Ou+++G99++y0eeeQR1X3nz5+PUaNGIT4+Hq+88gqmTZuGP/74A0OGDMHhw4erfN5ut9vv9+Tq3t9dLheuvvpqPPfcc+jbty/+8Y9/4IEHHkB+fj527NhR5X0jTqgjq2hXVSlKK2MzYsQIVSYmKytL6HQ6cc8998jHnE6nyMzMFJdccol8bOXKlT7T5z/99JNfaXXp8f35VxNSen/OnDnysfXr1wsA4sMPP/Q6/9FHHxUAVOUkSXV/8RYVFYkbbrhB6HQ6ecyxsbHiq6++8muso0aNEm3btvU6XlxcLACIJ554Qj72xBNPiMcff1x8+umn4uOPPxYTJ04UAMTgwYOFw+Go9rG6desmhg0b5nV8586dAoD4z3/+Ix8rKSkRLVu2lEsHoc7YvPDCCwKAWLx4seq49Ff30qVLq7z/vn37hNVqFRMmTAjqYwXyvP79738LAOKHH37w63xPY8eOFRaLRZw/f17znKp+fqXbnn/+ea/bpEzi7t27hRDl/28NBoPPx2jSpIm46aabgnbNQLVq1cqrFOVPxubw4cPCYDCIl156SXVfqRyrPH7JJZd4vZ/YbDaRkZEhxo4dKx9zOp1epfLz58+L9PR0cfvtt3uNr0mTJiIvL08+PnXqVAFAXHDBBarf8ZtvvlmYzWb5PauwsFAkJyeLu+66S/VY2dnZIikpyeu4J+nx/flX3e/a+++/LwCI119/3es2z8x/pGPzcBi64447VNMdBw4ciNWrV+OOO+6QjxkMBvTr1w8bN26Ujy1atAhJSUm4/PLLVVmVvn37Ij4+HkuXLsWf//xnzccdMWJEtTNxamr37t2YPHkysrKyMHHiRPl4aWkpgPLmW09Wq1U+x9ftVbFYLOjYsSOuv/56jBkzBi6XC++++y5uueUW/PLLLxg0aFCV99d6TOWYJDNmzFCdc9NNN6Fjx4546qmn8Nlnn3k1GtfmsV5++WU4HA48+eSTVV6zPqxYsQLPPfccbrjhBgwbNkx127PPPlvljB+gPMs1btw4xMTE4OWXXw7qYwVi4cKFaNKkideMJn8UFBTg+++/x1VXXeU1WcBf/v5OSP9VNtd7nqs8r66vWV+++OILuN1u3HDDDar3tYyMDHTo0AFLly5V/T7Ex8fjlltukb83m80YMGCAavaXwWCAwWAAUJ4VycvLg9vtRr9+/bBp0yavMYwbN041GWDgwIEAgFtuuQVGo1F1/OOPP8aJEyfQtm1b/PLLL8jLy8PNN9+sGrvBYMDAgQOxdOnSKp97RkaG3+/JF1xwQZW3f/7552jcuDHuv/9+r9uibXo9A5sw1LJlS9X30i9UixYtvI6fP39e/n7fvn3Iz89HWlqaz+uePn26ysdt2rQpmjZtWpMhVyk7OxujRo1CUlISPvvsM/kNBQBiYmIAlJfBPJWVlanOCcR9992HNWvWYNOmTfLMsxtuuAHdunXDAw88gLVr18pjU0pKSkJMTAxiYmJqNaaHHnoI06ZNw6+//oqbbroJLpfLa/p6SkoKzGaz3491+PBhzJw5E7NmzUJ8fLw/L0PQ7N69G3/605/QvXt3r9lg/nC5XLjpppvwxx9/4Mcff0SzZs2C9liBOHjwIFavXo377rtP9YHlr88//xxlZWU1LkMBgf1OxMTEwG63+7xOWVmZ6ry6vmZ92bdvH4QQ6NChg8/bPWdUZWZmen1QN2rUCNu2bVMd++CDD/CPf/wDu3fvVpVx2rRp4/UYgbwnA5Dfl/ft2wcAXsG4xLP87MlqtWL48OFVnuOvAwcOoFOnTjX6uY400f8MI5Dyg7+640II+Wu32420tDQsWLDA5/2rq2mXlpYiPz/frzH6M90WAPLz83HllVciLy8PK1eu9PoAkwIpX9NxT506hZSUlICzNXa7HXPmzMFjjz2mmk5vMplw5ZVX4q233oLdbofZbPYK5ObOnYtJkyahadOmWLp0KYQQqjdJaZxVfRAD5R8OqampyM3NBVC+hovnG+bSpUsxdOhQNG3aFCdOnPD5/JWP9fe//x3NmzfH0KFD5dq8FJidOXMGhw8fRsuWLYO+hMCxY8dwxRVXICkpCT/88AMSEhICvsZdd92F7777DgsWLNB806+rxwrEwoULAaDGgcmCBQuQlJSEq6++usZjkH7mtX4ngMqfiaZNm8LlcuH06dOqP2jsdjvOnTsnnxeMa9YXt9sNnU6HH3/80ed7oGeQr/X+qXyv/OijjzBp0iSMHj0ajz76KNLS0mAwGDBjxgwcOHDA676BvCcrH8vtdgMo77Px9Z5ZXZDh6w8iLdIfSsTAJujqM8XXrl07/Prrrxg8eHCN/qr69NNPVWuJVEX5JqGlrKwM11xzDfbu3Ytff/0VXbt29TqnefPmaNKkCTZs2OB127p169CrVy+/xqN07tw5OJ1OuFwur9scDgfcbrd8m2eat1u3bgCAXr164b///S927dqlGreU6aluXIWFhTh79qwcTPpKKUup4169emHp0qUoKChQ/QXn+VhHjx7F/v370bZtW6/H++tf/wqg/C/FmpZA/HHu3DlcccUVsNlsWLx4cY0yfI8++ijmzp2Lf/7zn3JzfbAeK1ALFy5Eu3btqi1V+nLq1CksXboUkyZNCjgYV9Lr9ejRo4fP34m1a9eibdu2coAn/Wxs2LABV111lXzehg0b4Ha75duDcc360q5dOwgh0KZNG3Ts2LFOrvnZZ5+hbdu2+OKLL1Tv0co1dupCu3btAABpaWk1yrz4+oNIi/SHUlVjWbt2LRwOR9SvG8TAJsikWRX1sfLwDTfcgLfffhsvvPACpk+frrrN6XSiqKioyg+9uuyxcblcuPHGG7F69Wp8/fXXyMrK0jx37Nix+OCDD3Ds2DE5tbt48WLs3bvXr4X0PKWlpSE5ORlffvklnn/+efmvmKKiInz77bfo3LmzHPhpvdlcd911eOihh/D222/jrbfeAlAezP3nP/9B8+bNceGFFwIoD94cDodXJuGFF16AEAIjR44EUHVK+frrr8drr72Gd999V55RYbPZMHfuXAwcOFB+TV588UWvGWk7duzAtGnT8NhjjyErK6tGs3j8VVxcjKuuugonTpzA0qVLNUsDAOSZGi1btlTNdpo5cyZee+01PPnkk3jggQeC+liB2rx5M3bt2oVp06ZpniP9NS99YCl98skncLvdtSpDSa6//no88cQT2LBhgzw7ac+ePViyZIlq1s2wYcOQkpKC2bNnq4KQ2bNnIzY2FqNGjQrqNevDmDFjMHXqVDz33HP46KOPVIGIEAK5ubmqxQX9IWValBnZtWvXYvXq1V5lp9oYMWIEEhMTMX36dFx66aVeAcWZM2eqzKTXZY/N2LFj8f333+Ott97yel9Vvg6+fsZPnTqF/Px8tGvXTn4O+fn5OHXqFJo2bRrQYqT1gYFNkPXt2xdA+eqmN910E0wmE6655pqgfABdcskl+Mtf/oIZM2Zgy5YtuOKKK2AymbBv3z4sWrQIb775Jq6//nrN+9dlj83DDz+Mb775Btdccw1yc3Px0UcfqW5XNvc9+eSTWLRoES699FI88MADKCoqwsyZM9GjRw+vDNL8+fNx5MgRecXdFStW4MUXXwQATJgwAa1atYLBYMAjjzyCp59+GoMGDcKtt94Kl8uFOXPm4Pjx415j8SUzMxMPPvggZs6cCYfDgf79++Orr77CypUrsWDBAvmNMTs7G71798bNN98sb6Hw888/44cffsDIkSNx3XXXVftYAwcOxLhx4zB16lScPn0a7du3xwcffIDDhw9jzpw58nm+ViCWAtX+/ftj9OjR1T6WL99++y22bt0KoDyjtW3bNvk1vfbaa9GzZ08A5eWZdevW4fbbb8euXbuwa9cu+Rrx8fGqx3/rrbfw3HPPqf6K/PLLL/HYY4+hQ4cO6NKli9f/h8svvxzp6el18liBPC+JVMKtKjC57LLLAMDnNN0FCxagWbNmVf7V7M/PL1CegXvvvfcwatQoPPLIIzCZTHj99deRnp6Ohx9+WL5eTEwMXnjhBUyePBnjxo3DiBEjsHLlSnz00Ud46aWXkJKSIp8bjGsuW7YMl156KZ555pk6beBWateuHV588UVMnToVhw8fxujRo5GQkIBDhw7hyy+/xN133+01xbo6V199Nb744gv86U9/wqhRo3Do0CH85z//QdeuXf1epsEfiYmJmD17NiZMmIA+ffrgpptuQpMmTXD06FF8//33GDx4sPyHky912WNz66234sMPP8SUKVOwbt06XHTRRSguLsavv/6Kv/71r/J7la+f8alTp+KDDz7AoUOH0Lp1awDlv8+33XabXL4PK6GZjNWwvPDCC6J58+ZCr9f7tUDf+vXrVfeXpj96rtqqteLtu+++K/r27StiYmJEQkKC6NGjh3jsscfEyZMn6/y5aZGmXWr987Rjxw5xxRVXiNjYWJGcnCzGjx8vsrOzA7qu53THBQsWiAEDBojk5GQRExMjBg4cKD777DO/n4PL5RLTp08XrVq1EmazWXTr1k189NFHqnPOnz8vbrnlFtG+fXsRGxsrLBaL6Natm5g+fbqw2+1+P1Zpaal45JFHREZGhrBYLKJ///7ip59+qvZ+dTHdW5qe7uufchpuq1atNM9r1aqV6pq+pmBrLVjp6/9fbR8rkOclRPn/6+bNm3stHumpVatWXo8vhBC7d+8WAMSUKVOqvH8gP7/Hjh0T119/vUhMTBTx8fHi6quvFvv27fN53XfffVd06tRJmM1m0a5dO/HGG2/4nMJb19f89ttvvZYk0FLT6d6Szz//XAwZMkTExcWJuLg40blzZzF58mSxZ88e+RxpgT5PEydOVP1/c7vd8u+2xWIRvXv3Ft99953XecoF+pS0fu+03sOXLl0qRowYIZKSkoTVahXt2rUTkyZNEhs2bNB8vYKhpKREPPXUU6JNmzbCZDKJjIwMcf3114sDBw7I5/j6GZd+l5TLlkjP1fN3KRzohPCjWYKIiMjDY489ho8//hj79++vVV8RUV3iysNERFQjS5cuxbRp0xjUUFhhxoaIiIiiBjM2REREFDUY2BAREVHUYGBDREREUYOBDREREUWNBrdAn9vtxsmTJ5GQkBB1O5oSERFFKyEECgsL0axZsyr3xGtwgc3Jkye9dmQlIiKiyHDs2DFkZmZq3t7gAhtpT59jx45Vu2U8ERERhYeCggK0aNHCa28+T2EV2Dz77LN47rnnVMc6deqE3bt3AyjfcPDhhx/GJ598ApvNhhEjRuDtt9+W95fxh1R+SkxMZGBDREQUYaprIwm75uFu3brh1KlT8r/ffvtNvu2hhx7Ct99+i0WLFmH58uU4efIkxowZE8LREhERUTgJq4wNABiNRmRkZHgdz8/Px5w5c7Bw4UIMGzYMADB37lx06dIFa9aswaBBg+p7qERERBRmwi5js2/fPjRr1gxt27bF+PHjcfToUQDAxo0b4XA4VFu4d+7cGS1btsTq1as1r2ez2VBQUKD6R0RERNEprAKbgQMHYt68efjpp58we/ZsHDp0CBdddBEKCwuRnZ0Ns9mM5ORk1X3S09ORnZ2tec0ZM2YgKSlJ/scZUURERNErrEpRV155pfx1z549MXDgQLRq1Qr/93//h5iYmBpdc+rUqZgyZYr8vdRVTURERNEnrDI2npKTk9GxY0fs378fGRkZsNvtyMvLU52Tk5PjsydHYrFY5BlQnAlFREQU3cI6sCkqKsKBAwfQtGlT9O3bFyaTCYsXL5Zv37NnD44ePYqsrKwQjpKIiIjCRViVoh555BFcc801aNWqFU6ePIlnnnkGBoMBN998M5KSknDHHXdgypQpSElJQWJiIu6//35kZWVxRhQREREBCLPA5vjx47j55ptx7tw5NGnSBEOGDMGaNWvQpEkTAMAbb7wBvV6PsWPHqhboIyIiIgIAnRBChHoQ9amgoABJSUnIz89nvw0REVGE8PfzO6x7bIiIiIgCwcCGiIiIogYDGyKKWKV2V6iHQERhhoENEUWkZ7/Zia7P/ITd2dwmhYgqMbAhoog0b9VhCAH8a/G+UA+FiMIIAxsiimhWoyHUQyCiMMLAhogimsXEwIaIKjGwIaKI43ZXLr9lNfFtjIgq8R2BiCJOoc0pf21lxoaIFBjYEFHEyS9xyF8rszfhTAiBv8zfgHvmb0QDW/CdqF6F1V5RRET+KLRVBjZlDhd+/SMHybEm9GudEsJRVS2vxIGfd+YAAM4V29E43hLiERFFJwY2RBRxHK7KjMfx86W488MNAIAD06+CQa8L1bCqpNdVjsvudIdwJETRjaUoIoo4DldlYHC22C5/fSy3JBTD8YtLUX5iYEMUPAxsiCjiOBSBQam9spF4/+miUAzHL0535ZjLnNwKgihYGNgQUcSxKTI2RWWVgc2p/NJQDMcviriGe1wRBREDGyKKOMqMjXLqtyuMZ0gpS1GlDgY2RMHCwIaIIo6yebhIEdg4wzmwUYy5jIENUdAwsCGiiKNsHlYuCeMO4/VhVBkbO5uHiYKFgQ0RRRy7y3dgENYZG0WTDUtRRMHDwIaIIo5DI7BRlnvqypebj+PZb3bWeoVjl6p52Kl9IhHVChfoI6KIo7UOjCsIpaiHPt0KABjSvjGGd02v8XWczNgQ1QtmbIgo4mhmbIJYisordVR/UhWU0725QB9R8DCwIaKIsunoeUz/YbfP24IZ2BhruVWDMpsUxq1ARBGPgQ0RRZQxb6/SvK2uAxtlX43nHlTrD+fixe/+8LnY3vliO2weqwsrm4fDefYWUaRjjw0RRY26nhVlU5SMPDM24/6zGgAQazZgyhWd5ONnCm3o/9KvaJESg5WPDZOPK6tnzNgQBQ8zNkQUNeo6Y6NcSE+vUYraf0a9P9Xv+88CAI7lqrd3UDYP13aGFRFpY2BDRFGjzgMbRTnJ3+qRVgDkVmVsGNgQBQsDGyKKGnU93bvMURmNKDMuSp4PqYxrhOJGNg8T1Q8GNkQUMaor4dT1An3KUpRT49regU1lZKPc04rNw0T1g4ENEUUMh0bWRFLXzcPKwEZr7RwB9WMqMzbKmVGq5mGmbIiChoENEUUMRzUZmZpkQn7cfgq7swt87ritLkX5l7EBKiMb5awqdcYm4GESkZ843ZuIIoZTI2si3x5gxLD24Dncu2CT/P3To7rgzovayt8rm4ere+zKMVSepw5sKs9hKYooeJixIaKIoczYXN83E41iTarbAy3xbD6Wp/r+xe93qb63qUpRGhkbrzEqAhtljw57bIjqBQMbIooYUnBgMujw2rgLYDbqfd7ur+r2bFIGM/7OinI4Kw8oMzZu1awoBjZEwcLAhogihjQzyWTQq/4rCXQdG89tDzwpr6fd36M+bnf5U4ryf4xEFBgGNkQUMaSgQdrewDNj4xnYbD+ej1UHzmper9Tuf8+Ov9O9tUpRyuZhwYwNUdAwsCGiiOGZsTEbPEtRAt9vO4Uftp8CAFzz1m/483trcSy3xOf1zpfYq3k83wv0VZUZUgY2ZRoZm2DuQk7U0DGwIaKIIQUNRkN5xsazFFVsc2Lywk3464JNOFNok49vPZ7n83q+SlHKY06NUpQyePFuHlb02GhkbBjXEAUPAxsiihhSoGHUSz026n2ZShXrzhxUbE55/Lx6Q0qJ3ekdYZTYlMGIshTle00bz0ZgZUOyuseGzcNE9YGBDRFFDCm4kHprPDM2ykzKMUUwc7rABl98zXRSrm6s6rFRZm+qmE2lHINL4/5ceZgoeLhAHxFFjOqah5XZkqOKvhqtqdq+GoKVx5RZGmXA4vDotymyOXHXBxtwWZc01XlaU7wZ1xAFDzM2RBQxpKDDqNE8rAwqjpwrVhz3HUn42v/JoVFyUgc8lV/bnW7MXrYfqw+ew4vf7/LY+FL5deVjsBRFFDwMbIgobNmdbsz8eTc2HskFoF6gr/y/2hmbI+cqMzZaG1j62oJBKzBRZmkcHpmcrcfyK8egkbFRT/f2ORwiqgMMbIgobL3+y17MWnoAY2evBlAZdEilKJNHKUrZrHuuuLKvRjOw8XHcqdFj49aYIeUSwGFldsiPRfk43ZsoeBjYEFHY+nzTcflrp8stByiVKw+rZ0UpsyX5JQ7FfbVKUT4yNk7fWRblqao+GrdQz35SZmk0e2wY2BAFCwMbIgpbyrVouj/7M5bsOg1Ae4E+ZSmq0OasPK5ZiqpmVpTLd8bG5fG1Mk5xa2R5lEkaJmyIgoeBDRFFhDKHG99uOwlAe4E+JaGRYVHynbHxXYryDGYk5dkXdWlKfZs0Ht9fE1HdYmBDRBFDCkQsFb01ntO9tWiXonz12GgEMxolpqoyNlrBkIuBDVHQMLAhoohjNhoAVJ2xUdIsRfkIeOwa+0MpsyxujyBFeRV/1q5hKYooeBjYEFHEMcs9NrpqziynPd3bR8ZGcx2a8q/tTre6FOUWqqBHa+0alqKI6gcDGyIKS1VNidbaUkFLILOi1KsNq3tnpn6xDRc89z+czC9VHPc3Y+M7+CGiusXAhojCUondqXmb1GPjuY6NlurWsfnpwYuQ1TYVgLoU5ZmZ+XjdMZQ6XJjz2yHFcXVpSqtHR12KYmBDFCwMbIgoLJXYXZq3WQLM2Gj12Dgqoo0Eq0kOklRbJ2gEKcq1bpxutypjo7UJJveKIqofDGyIKCwV27QzNlIpKiXO5Ne1qsvYmPQ6mCpWM1YHJr63R1BvggnlbG/VWjrqvhr4PE5EdYuBDRGFpcKyKgKbikxNeqLVr2v56rFxuYWcOTEa9HL2x6HMzLh8Z1ycHseVt9k1ViFmjw1R/WBgQ0Rh6bRi1WFPUsYmw8/AprpdvI0Gnbzon1OjlKQqRXmcowxTbBp7RWn12xBR3WJgQ0RhKaegTPM2qccmI8m/wEaZRZEo+2fMBr2cBXK4NFYermKvKK1VjrX6aliKIgoeBjZEFJZOVxHYSAv0xZqNuOaCZmiVGlvltZw+UiTKzIxRX5mxUU3xdqsDGIl6GrhQZWO0ZlUJjyDH7RaYMGctHv6/rVWOnYgCw8CGiMLKobPF+H7bKeQUVF+KAoB/39wb//eXrCqv6XC58cWm43j9f3vkAEMZgBj0OrnHRllK8uylUV5P4nIL1bVUPTYas6JcboEDZ4qwct9ZfL7pOGxO7RlgRBQYY6gHQESkdOlrywAAcebyrMyTV3XGifOl+GD1Efkczz2i9LqqVyB2uASmVGRGru3VHO3T4mFzlAcgMSYDdDod0hLKy1rZisX3VGUpl+/AxrN/RxnYaG+pIKAc8vliBzKSDFU+ByLyDzM2RBSWiivWsYkxGzF+UCvVbRavwMb/65ZWXLfMIV2/PKBo3bi8nHX4bIl8rjITU6bIqqhnS6mvrx3YqKd+lzkqzztbpJ2dIqLAMLAhorBmUTT2SjwzNoYAIhspWCmVAhtTRWCTGgcAOHyuWD7Xpgg+pEDI3+sDnntFVX7tEkJVfjpXbFdd4+CZIkxesAn7cgr9ekwiqsRSFBGFNbNRjzhL5VuVQa9Dj+ZJqnP0AQQ2UulIytxYTOVBkhTYnC60odjmRJzFqM7YOLxnVvm+vu9dwD2zN8rrnfPI2Az7x/Ly48U2fHJ31f1DRKTGjA0RhTWTQY/k2MoVhjMbxaBxvEV1TnU9NkpSr4xnxiYp1oRGFY9z5Fx5OUpZVqpJg69Lc7p3ZWAFAHklDp/3L7axqZgoUAxsiChs+FqR12zUq/aEMvrIzhgCCGzOFtnw276z8l5UUmADAK08ylE2VWDjX8ZGSZWx8cjeKHt2yjSCppQ4c8CPSdTQsRRFRGHDVx+LyaAOWnz10+gD+BPtwU+3AAAGtkkBUNk8DABNEsozQVIGRb3vk/+PIalqS4Uyh+8yl9C4DxH5hxkbIgobpT4CG+9GYe+3La1SlF7nO8MDAGsP5QIArIqMTWxFkFNiL9+nqrbry2itYyOE+rnaFF9r3YeI/MPAhojChrLvRCLNiEqryKYM75LmdY5WKcpk0Fc7Y0pZipK+LnO44HYLVSNwTQjV2jXKr4UqmFEGOU5V+apWD0/UIIVtYPPyyy9Dp9PhwQcflI+VlZVh8uTJSE1NRXx8PMaOHYucnJzQDZKI6pSvDImUsfn83gvxwnXdMPnS9l7naM2KMhv0mhkbiSqwkTM2LtWMqJrS3lJBqMpuZQ4Xlu89g5H/XIHNR/NU5xFRYMIysFm/fj3eeecd9OzZU3X8oYcewrfffotFixZh+fLlOHnyJMaMGROiURJRXSu1ewcTUmDTIiUWE7Jaq0pHSr7iF6NBV23GxmqqfBuUgpwSu6tGzcKetGZFudzqLE2Zw42J76/D7uxCTJy7Tj7OuIYocGEX2BQVFWH8+PF477330KhRI/l4fn4+5syZg9dffx3Dhg1D3759MXfuXKxatQpr1qwJ4YiJqK746rExGfx7m/IVwJgMehiruX/7tHj5a6nHpszh8rkjeKC01rERQiC/tHKKtzJ7o9prykdkY3e6UWxz1npsRNEq7AKbyZMnY9SoURg+fLjq+MaNG+FwOFTHO3fujJYtW2L16tX1PUwiCgKfpSg/Axudjz4bf3pshnaq7NmxqjI2tV9DRitj4xYC2fmVi/KVaQRRTh/lsOGvL0e3Z35GYZnvtW+IGrqwmu79ySefYNOmTVi/fr3XbdnZ2TCbzUhOTlYdT09PR3Z2tuY1bTYbbLbKN5CCgoI6Gy8R1S3PDSUB71lRWnw1EJsMOrhF1YFNorVy8b9Yc/lbYmkdZWxcGuvYuNwC2QWVm21qbdew9Xg+juWWoEVKrHzsaG754oFbjuXhog5Naj1GomgTNhmbY8eO4YEHHsCCBQtgtVrr7LozZsxAUlKS/K9FixZ1dm0iqlu+ZiH5m7Hxvb6NrtpVia1mRY9Nxdeldhe+3XrKr8etivDI0iiPZ+eXyd/bqtiH6qJXl/o87isIJKIwCmw2btyI06dPo0+fPjAajTAajVi+fDn+9a9/wWg0Ij09HXa7HXl5ear75eTkICMjQ/O6U6dORX5+vvzv2LFjQX4mRFRTTh+BjcnPjI3P+EWUNxBXdR9l4BRjKs/YFNmc+O/Kg349blW01qRxCaHa+NJXb1F17E52FhP5EjaBzWWXXYbt27djy5Yt8r9+/fph/Pjx8tcmkwmLFy+W77Nnzx4cPXoUWVnam8RZLBYkJiaq/hFRePKVhfBceViLr4yN0DguiTEZVL050nTv4+dLUFjRoLvwzoF+Pb4vWj02JXaXKptT4mP9Hl+U5SwnF7kh8ilsemwSEhLQvXt31bG4uDikpqbKx++44w5MmTIFKSkpSExMxP3334+srCwMGjQoFEMmojrmM7Dxc78EXz02Br3O5zRwSYzH1HFpVtTZovJsSqLViE4ZCX49vi9ujXVsPOVrbILpSbl4H0tRRL6FTWDjjzfeeAN6vR5jx46FzWbDiBEj8Pbbb4d6WERURzx7bPQ67cX3PPmaFWXQ6aq8v+eaOJ6BTrzFGNDO4Z60MjaeCv2cvq0sZzlYiiLyKawDm2XLlqm+t1qtmDVrFmbNmhWaARFRUHmWV4wB7G7pq8dYr9dVufKwcnE+QL0hJgDEWYx+B1a+aK1jI8lItOJ0YVmVQY90Hb1ep8rY1MXKyETRKGx6bIiIPKdYV7cGjepcn6Woqq9RXcYmzmIMaAyeqsvYJMYY0SjWXO11pCDG5WIpiqg6YZ2xIaKGxenx6V/dPk9KWqWoqq7hGbTEemRsEqxGzQ02/eF2AwvXHsXrv+zF+RK71+1xFiPcAqoZUr7YXW5YTQZVoMTAhsg3ZmyIKGw4PDM2fs6IAnxnZgz6qveK8rzFqxRlNiKAapgXtxB48svtOFtkU039lsRbjGilWHxPqWN65VYPUiZLWaqzORjYEPnCwIaIwoajFhkbrcCmqnVsPJkNetUsqjhL7TI2voIZpTizEZd2TvN5m1Gvl9fYkQIb5fVqsvYNUUPAwIaIwobn3kiB9Lf4ij/0Oh0MVaRcEhTbKZRfQydvqwAA8RZDrXtsqlqHx2TU4/Ku6T5vMxp08nYSvgKbMmZsiHxiYENEYcOzbySgWVEa69hUlfVJjjV5HVM2FMdZjNDpqi5nVcXtFj57f+Tx6YD0RN9byBj0isDG5R3YsMeGyDcGNkQUNjzXsQloVlQNemx8zUhSNhDHWYwBj0PJJUSVpSwpm+Tr+ka9zqsU5eTKw0TVYmBDRGFhw+FczFt1WHWstrOiBrdvXOU1WjeO8zqmDGziKwIbf8bROtW7Cdjtrjooktbe8RX8GPV6OWNjqwhslOviHM0twcWvLsW7Kw5UOzaihoSBDRGFhfsWbvY6FljGpvLrv1zcFi/9qTvuGNIGRo3dwUf1bIpbBrX0Op4SV5nFia8mY5PZKAYA0KZxHAa2SfW63S2E78055TFXjM3HOb56bJQZm9/3n8PR3BJM/2G39gMQNUBcx4aIwoKvWT6BBTaVAUybxnG4aUB50GL2EdgMbp+KWX/u4/M6jRSBjVSKMmkERw9f0REOp8DQTk3wxq97vW53uYXXooNKUibI19M0KEtRPnpsiMg3ZmyIKCx09rHZZCBTtZXlIuU2CBaT99ucr2BHkhLrf8YmwWLCDf1bIC3R6nNPKZdbyGUkX6Tr+sr2GPU6eey2iqCPgQ1R9RjYEFFYaJ4c43Wsqqna3udWBhbKnhVfQYxU4vGlkWKmVJylvN9Gq8fGpLiOr+Cn2F715pbSfWaO64lJF7b2uk3q95GyWZ4rMxORNwY2RBQWfG0SGUjzsPJcQ3UZG6PB65ikkY8eG63MkXKNGl8Zm+rWmpHGmZZgxbPXdlP14xgNenlNnaKK3b+1MjZVlbuIGhoGNkQUFnxlI2o63VsV2PjI2FiqyNhktUtFvMWIpBgTmlVkkbTW01Fmg3wFNoGMufxxdKqv4yoyNiW2qktRX2w6jpveXY1T+aUBj4Eo2jCwIaKwIH1oX92zqXwskIyNssFXGTD4KjtVVYrqnJGI9U8Nx8rHL612HRv1Y/o91Mr7eARDnsGZ9PhSSUsrsHnii+1YczAXD3yyJfBBEEUZzooiorAgZWwsijJRTTM2yuyJr+neVTUPA+WbYcagchyaPTbKjI1HtsWffhjvjI0egFu+hhzYVJSiqluUb92h3GofkyjaMWNDRGFBykZYFT0xddFj4+sKVZWifF5bo8fGbPQdTFWVEVJdV19VxkaPOLOUsambWVEbj+Ri89HztboGUbhjYENEYcF3xqZms6KqC4j8DTyqG4fZoBirIrDxN3DSV9djUzErq7ia5mF/lDlcGDt7Nf709iocOltc4+sQhTsGNkQUFlwVZRZLHWRslAGDr57eGLP2rKjqrq1kMvp+zLrI2BgNlTuNF1fTPOyPMsUCiP/bmV3j6xCFOwY2RBQWnBUbYFqVGZsAFuhTZlWq2ngSAPq1SglobP702BhqUIqqdlaUR8amNuvYOLkzODUQbB4morAgZSPqJmPj+5zLOqfB6Rbo26pRQGPTXsdGOd278nhVzcmxZgNKKnpmvAIb1Syryh6bkopZUb7W+vGXMttTi8sQhT0GNkQUFip7bKpezVeLMrujte7MnEn9azQ27R4b37OitPaWAgCrqTKw8QzcvDM26uZhKatVE8qMDRcwpmjGUhQRhYXKWVHVT7P2RT0rqu7GBQAmzVKU1mrH2j08qtWKq5wVVbfNwy6XMrBhZEPRi4ENEYUF3xmbms2KUk69HtY5HQDQJMFS47FpZY6MGqUoX6sd+xpbVc3DJlXzcEVgU4uARLkGjmBgQ1GMpSgiCgvyrChF83AguxRorWPTPi0eKx+7FCmKPaACpeyx+VPv5vhy8wk8cWVn1Tn+rmOjPM9zGwajKgOkl/eqKra7IITwah7W6/wvK7lYiqIGgoENEYUFp48F+gJZSE81K8ojE9IiJbZWY1Nee0yf5ph2dVfVLuCej1llYKO4ybMpWfk4Rr0OsRWlKJdbwOZ0w+Uxm6lDWgL25BT69RzUPTaMbCh6sRRFRGHB5WOBvtgA1pvRytjUBWWPjcVoQEqcGTqPbIsy+2KqYpq68jzPUpvnc4hV9OoU25zw7B3ulJHg3xMAMzbUcDCwIaKwIM34UU73jqmiCdeT0aDdu1Jb/mRj1HtF6TWDK1VgU8UmmEaDDkaDXs5gldhdcrlO0j4t3s9nwIwNNRwMbIgoLMizohQZmxiz/9VyZTATG8D9/Lq2ImjSWqPGoFMHJVqxlfJ4VQv0SbdJa9kU2Zyq4OTNm3oh0er9PJWNwSfySvHqT7uRU1CmztgwZUNRjIENEYUF6UNbuU1BIBkbZftJvI8P/NpQroujzCgpKeMdg17n1RgsUZeiqpgVVfGYUobI6RJyQHLzgBa4rldzmHxkj5Qxy70fbcTbyw7grws2sRRFDQYDGyIKC1KZRZm1iDH7/xZV5qzcCymujjM2yiZmrYyNzmMatz+lKM+SmXoDUJ3qv063Ww7+pGv4WghQOa172/F8AMDGI+dVx1mKomjGWVFEFBakD21lQ22Myf+3KOUmj3XdPJwUUzkDSmumltljOwRlaeq2wa3Rrkk8hrRvjHsXbJKPey7Qp9pOoqL8JQU/LreQsy7SMV9BlltjGyiHovOY69hQNGNgQ0RhwfNDGwAuaJHk9/3LHMHb2FHa2gBQZ1WUlCsmW4x6VdCSYDHilkGtAKh7bDwzNtYqMzaVgY0U/GlnbLzH+OAnm+WvWYqiaMbAhojCQmXGRoe1T16GglIHmibF+H1/myJjU9diFNPOtWZFKc+JNRs0t1ioqsfG6mMDUKm/R5WxkbI5PqaVK3tpEq1GFJSVr1p8vsQhH2cpiqIZAxsiCgvKjE1aohXpidaA7q/ssalryiZmrcAm1iOwUQYwMarApvI+noGNMhskBTTKjI1nj42vUpRy5pTnWjsSZmwomjGwIaKQE0JZZqlZf0wwS1HKMpPW+JTBS4zZqApglPdXr3ejnbExeGRlXG63V7nOVynq9V/2olVKLJolxyC/1OF1O8AeG4puDGyIKOSU5RNjABtfKj0+sjNWHziHOy9qU1fDkqUnVr+BpjJ48SxFKQMW9QrF6ueqzth49Ni4vIM/XyscL1x71OtYx/R47M0pkr9nKYqiGQMbIgo5ZfnEUMV2BFXplJGAbc9e4TOLUVu9WiTjvkvbI7ORds9PTUpRnmUtVcZG7rGpnBXl9Axs/NhLq3lyDPq1TlEFNq7gJbeIQo6BDRGFnDpjU/Op2sEIaoDyXpVHRnSq8pwY1awogyposWo0D3tnbLzvo+yxcXsENlpr6ij96+Ze+HXXadUxlqIomnGBPiIKOVXGpo7XoKkvyllROh0QZ1EEOhqlKK8F+hQBUJN4S8U5lbOinH702HhqmRKnWocHYCmKohsDGyIKOWXGxnNjyEjhuXCfcr8qVSlKcZpnKcrurKwRNUkoD2zU69i4Vceq2kVcYjXpfQQ21d6NKGIxsCGikJOW+9frvFfjjRSeWyrEKxb187cUVaCYxSTdp7LHxg1p8WBDABkbq8mgWmAQYMaGohsDGyIKucppzJH9ljSubya6Nk3EkA6NVc3EMRrTxT0X2LP62PTTV8ZG3lKhmuZho14Hk0GPWI/rMq6haMbmYSIKOaerdmvYhIuZ4y6Qv1ZuxKlaB0eRsfFs/v3zwJZYeygXV3bPkI9VrmMj5NdJ72fGRnpcZZAFMGND0Y2BDRGFnK99oiJdqWKLh5Q4s/y1stTmGZjEWYz478R+qmNSFsvhEnJAYvSzx0YKbGIY2FADEtl5XyKKClKPTU3XsAlH54pt8tfKkpGhir2ifFH22HjugF59xqb8dmUjM8DmYYpuDGyIKOQ8pzFHgyKb772rAm0j8r27d/lt1QU2MRqlKK5jQ9GMgQ0RhVy09NgoPXVVF5iNejx1VRfVcX2A09nlHhvVlgrqDTK1aJWiXEzZUBRjjw0RhVy0zIpSGtAmBTueHeE1cynQ4M3X7t7+ZrYqS1GePTYBDYEookTPuwgRRSzPPZCiha/p2AFnbBQrD0sBoL/XkDI2ViObh6nhYGBDRCEXjbOitAQa2PjqsfE/Y1Me0Hguesi4hqIZAxsiCjmnx1YB0SzQfTpVKw9LmS0/Z48pFwa8uGMT+WtmbCiaMbAhopBzRWkpype66LHxdz8tq2LzzXmT+uP1G8oXEGRgQ9GMgQ0RhZzcFBtF69ho0QXcY1O58rA7wABQvfmmDpaKXhs2D1M0Y2BDRCHncqmnMUezyzqnAQDizN77QvkivSZOt4CjomTnz+aXgPfeU1I8xHVsKJpxujcRhVw0LtCnZVjnNCy8ayA6pCX4db5yHRuHSwps/HudLJ6BTcXry4wNRTMGNkQUcg2px0an0+HCdo39Pl/ZY+Nwlr9O/mZsYrwyNlJgw8iGolf0532JKOxJs6IaQsYmUMpZUZUZG//euj1fT+lbN1M2FMUY2BBRyDWkjE2gVBmbAEtRuSV21feVGZs6HCBRmGFgQ0Qh15B6bAJlNFSuPOxwBVaKyi1SBzbShCyWoiiaMbAhopDz3NyRKhl9Zmyqfp1S48ww6HW4bUhr1XFmbKghCKh5+Jtvvgn4AS6//HLExMQEfD8iajiYsdEml6Jcbvl1qq4U9c+beqF3y0aIt6jf4qXAhtO9KZoFFNiMHj06oIvrdDrs27cPbdu2Deh+RNSwuCoyEf5uFdCQSMFemcMtHzP52FxTKcZk8ApqAEXzMAMbimIB532zs7Phdrv9+hcbGxuMMRNRlGHGRpuUsSl1uORjpmpKdp4L80l0LEVRAxBQYDNx4sSAykq33HILEhMTAx4UETUsge6B1JAYK4KYMmVgU01mK0ZjVWODnuvYUPQLKLCZO3cuEhL8Wy0TAGbPno3Gjf1fiIqIGiZnRSnKXE2JpSHyzNjodOpp8WN6NwcAJForS0+tUnxnyyu3VAjGSInCA1ceJqKQs7saziaYgZLKc6X28sDGZNCrNtKcMbYHbuzfAl2aJeI/yw7g6p7N5CninnRceZgagID/PFqyZAm6du2KgoICr9vy8/PRrVs3rFy5sk4GR0QNgzPAFXUbEqmhWsrYmDz6kCxGAwa2TUWi1YTHRnZG12ba5X/pri422VAUC/hd5J///Cfuuusun70zSUlJ+Mtf/oLXX3+9TgZHRA2DtD6LmYGNFyljY6uYFVXdjKiqVE73rv24iMJVwL8hW7duxciRIzVvv+KKK7Bx48YaDWb27Nno2bMnEhMTkZiYiKysLPz444/y7WVlZZg8eTJSU1MRHx+PsWPHIicnp0aPRUThw8FSlCapn8ZeB1ktboJJDUHAvyE5OTkwmUyatxuNRpw5c6ZGg8nMzMTLL7+MjRs3YsOGDRg2bBiuu+467Ny5EwDw0EMP4dtvv8WiRYuwfPlynDx5EmPGjKnRYxFR+Ah0c8eGxOgxtbs2WS1uqUANQcDNw82bN8eOHTvQvn17n7dv27YNTZs2rdFgrrnmGtX3L730EmbPno01a9YgMzMTc+bMwcKFCzFs2DAA5bO0unTpgjVr1mDQoEE1ekwiCj0GNto8NwatTVaLWypQQxDwu8hVV12FadOmoayszOu20tJSPPPMM7j66qtrPTCXy4VPPvkExcXFyMrKwsaNG+FwODB8+HD5nM6dO6Nly5ZYvXq15nVsNhsKCgpU/4govFRu7shSlCfPRQtrVYqquCu3VKBoFnDG5umnn8YXX3yBjh074r777kOnTp0AALt378asWbPgcrnw1FNP1XhA27dvR1ZWFsrKyhAfH48vv/wSXbt2xZYtW2A2m5GcnKw6Pz09HdnZ2ZrXmzFjBp577rkaj4eIgqfM4cKtc9Zh3eFcAMzY+OKZsalNKYoZG2oIAg5s0tPTsWrVKtx7772YOnWqHPnrdDqMGDECs2bNQnp6eo0H1KlTJ2zZsgX5+fn47LPPMHHiRCxfvrzG15s6dSqmTJkif19QUIAWLVrU+HpEVHd+3ZUjBzUANNdfacg8g70mCZYaX4vNw9QQ1GiBvlatWuGHH37A+fPnsX//fggh0KFDBzRq1KjWAzKbzXL/Tt++fbF+/Xq8+eabuPHGG2G325GXl6fK2uTk5CAjI0PzehaLBRZLzd8IiCh4PMssZpaivHhmbJomWWt8LXkTTKZsKIrV6s+jRo0aoX///hgwYECdBDW+uN1u2Gw29O3bFyaTCYsXL5Zv27NnD44ePYqsrKygPDYRBVecxw7ULEV58wz+MmoV2HAdG4p+YbWlwtSpU3HllVeiZcuWKCwsxMKFC7Fs2TL8/PPPSEpKwh133IEpU6YgJSUFiYmJuP/++5GVlcUZUUQRynMFXJaivHlmbNITax/YuBjZUBQLq8Dm9OnTuPXWW3Hq1CkkJSWhZ8+e+Pnnn3H55ZcDAN544w3o9XqMHTsWNpsNI0aMwNtvvx3iURNRTUmzoSQsRXnznN6dFKO9jlh1uI4NNQQBBTYOhwMffvghAGDChAkwm811Opg5c+ZUebvVasWsWbMwa9asOn1cIgoNaf0aCUtR3jwzNvGWmv89qtdzVhRFv4DeRR555BGkp6cjLS0Njz76aLDGREQNhGdgw1KUN8+Vh+OttQhsKmIkrmND0Syg3xC32w232w2XywW32139HYiIqmB3emZsWIry5JmxSahNxobr2FADENCfR//4xz+Qm5uLvLw8vPbaa8EaExE1EN49NszYePKcFVWbjA17bKghCOg3xGw2Y9KkSUEaChE1NCxFVc8zY+M5RT6gaymmewshoNMxQ0bRh+8iRBQynoFNrNkQopGEL88sVpy59qUogGvZUPRiYENEIWP3CGwax3OVcE96j4yNZwYnoGspAhuWoyhaBRTYbNu2LaCm4Z07d8LpdAY8KCJqGBxO9Ydrci3WaKHq6RTv+GwgpmgVUGDTu3dvnDt3zu/zs7KycPTo0YAHRVQbucV2/HflQWw7nhfqoVA1PEtRntkJqlvM2FBDEFCxVgiBadOmITY21q/z7XZ7jQZFVBuzl+3HeysPAQAOvzwqxKOhqniWoii4lHEjAxuKVgEFNhdffDH27Nnj9/lZWVmIiYkJeFBEtXHgTLH8tdstmAUIY8p1bKZc3jGEI2kY1BmbEA6EKIgCCmyWLVsWpGEQ1R3lxoqlDletpsdScEmlqAcu64C/XdYhxKOJfjpmbKgB4KwoijpORYN7sZ3N6+FMCmzMRr4V1QfVdG9WASlK8d2Eok5RWWUwU2p3hXAkVB1p5WFupVA/DGwepgaAgQ1FnUJbZWBTbGNgE86cFWVDz40eybeWKf5N3NDCUhQ1BHw3oahTqMzYOFiKCmeuirKhkRmbKn351wtxSccmmDOxX62uo9PpFPtF1cHAiMJQUBfoI6pvx3JLcLbIJn/PjE14kxq9a7OabkPQu2UjfHD7AHRIT6j1tfTyflGMbCg6BbxA39mzZwEAbdu2DWixPqJg23jkPC56dalqD5wS9tiENZdcimJgU1/0zNhQlAsosElOTsahQxULnx0+zOwNhZX3fzvkdayEs6LCmlPO2LAqXl+kHb1dzNhQlApogY+xY8fikksuQdOmTaHT6dCvXz8YDL534z148GCdDJDIH7nFdizZfdrreLGNgU04qyxFhXggDYicsWHKhqJUQIHNu+++izFjxmD//v3429/+hrvuugsJCbWv+RLV1qYj51Hq8C47FZQxsAlnThczNvWtsscmxAMhCpKAl2QdOXIkAGDjxo144IEHGNhQWNh1qsDn8YIyRz2PhAIhlUPYY1N/pMCG070pWtV4rfm5c+fKX0vd9Tod35y05Jc4YDHpYTX5Lt1R7Rw4UwQAeHREJ/RqkYwVe8/gnRUHUVDKjE0446yo+lfZPMzAhqJTrfK/c+bMQffu3WG1WmG1WtG9e3f897//rauxRY38UgeGvLoE4/6zOtRDiVrnist3ks9ItGJw+8bISLICYMYm3MnNw/yjqN5Im8KyxYaiVY0zNn//+9/x+uuv4/7770dWVhYAYPXq1XjooYdw9OhRPP/883U2yEi36ch5FJY5sf1EPs4X25Eca8L8NUfQKT0BA9umhnp4UeFsUXlgkxpvBgAkWk0A1Iv1UfiRFugzcIG+esN1bCja1TiwmT17Nt577z3cfPPN8rFrr70WPXv2xP3338/ARkG5YNyuUwXQ63X4+9c7AQCf33sh+rZqFKqhRY3c4vLXODXOAgBIjCkPbPJK7CEbE1VPah5mj0394To2FO1qXIpyOBzo1897ee++ffvC6eRfyUrHckvkr/edLsLhs8Xy92Nnr+JfTrUkhEBusTpjk5FYXorafiIfR8+VaN6XQkvq82CPTf3RsXmYolyNA5sJEyZg9uzZXsffffddjB8/vlaDimTL957BiDdW4PONx+Vjx86Xyl8v3n1a7geRHDhTDKq5glKnvEt0Slx5YNO9eSIuyEyCEMDvB86GcnhUBfbY1D82D1O0q3EpCihvHv7f//6HQYMGAQDWrl2Lo0eP4tZbb8WUKVPk815//fXajTKC3DFvPZxugVd+2o0xfZpDp9PhqCJjs2LvGazYe0Z1n9OFZWifFl/fQ40apwrKA8dGsSZ51plOp8OgdqnYejwfO0/mh3J4VAV5SwX22NQbebo3F46nKFXjwGbHjh3o06cPAODAgQMAgMaNG6Nx48bYsWOHfF5DmQL+1pJ9+Gzjcfkv0NOFNpwutCE90aoKbHwpKOXMndo4mVce2DRLjlEd75BWvsbSEZaiwhYX6Kt/XMeGol2NA5ulS5fW5TgiXk6BDYc9PkBL7C44XG6cKbT5vE+zJCtO5pchn4FNQArLHIgzG6HX61Bsc2LD4fMAvAObOHN59qbMx4rEFB64CWb907EURVGuVqWosrIybNu2DadPn1ZtiKnT6XDNNdfUenCRJNbivfCe3elWfahOurA15q06LH/fu2UjnNx+KioCGyEEjuWWwmjQoUmCBSU2F9YdzsXhs8W4bXBrON0C54rtaF4RfOSV2PH+b4dwYfvG6NOyEb7afAJfbz2B88UO/HlgS5wvtsNs1EOv06GgzIH2afEwG/Q4W2zHc9/shNMt0DE9HntziuQxtGgUqxqTtSKw8bXVAoUHaeVhfQPJ7IYDA9exoShX48Dmp59+woQJE3Du3Dmv23Q6HVyuhvVhEmf2fintTjfszsqAr03jOPnrt/7cW840hDKwWbbnNN5edgC3XdgaOp0OWW1TUWx3YtvxfJwpssHlcuPLLSeh1wHtm8SjR2YSHC6BBWuP4PDZYrRuHIcT50thc2oX7F/6YRd0uvK9aVLizLhjSBt8veUE9uYU4V9L9nud//RXO3xcxZsyqAGAYZ3TVN9bjVLGhs0E4Yo9NvWP69hQtKtxYHP//ffjhhtuwN///nekp6fX5ZgiUpzFR2DjcsHuKv9QNRl0uKpHU2w5loc+rRrh6p7N5A/mvJLgBjb5JQ78tPMUdpwowOjezdAsOQZlDjc+WnMEc347BABYdyi32utsPpqHRYrZXgBw0M8ZXdJ7aG6xHTN/3uPznCHtG6NN4zjMX3NEPtY8OQZni2yqwGliViukxlvgcguczCuFxaSHUa9HVjv1YocxUsbG3rCC7EjirPj94HTv+qPjOjYU5Woc2OTk5GDKlCkMaipI/RxKNocbtopsgdmgR5MEC964sZd8e5OE8sXklNPBq3LkXDE+WX8M91zcDkmx5QvQOVxubDh8Hu+uOIBYixHZ+WW4sX8LnC4ow5ebT3hNJVcGDVVJiTOjVWosth7L83oD7N48EXqdDglWI3pmJqNJvAXL9p7B3Re1xdkiGx5etBXX9GyKJ67sgm3H82B3udEo1gyjXocXvv8Dh84Uo0mCBR/ePhCp8WYcOluMFo1ikWA1QqcDBrdPRYuUWHRtmgghgNwSOw6eKUas2QC9ToeuzRL9eg5WU3lDqs3JwCYUluzOweajeXhweEfNwIU9NvWPzcMU7Woc2Fx//fVYtmwZ2rVrV5fjiVixPjI2NpdbzthYfGx+2btFMgBg89HzcLuFvIeLpNjmxC9/5KDQ5sQ7yw/geEUANHtZ+Sy0OLMBxT6yERuPnPd73PcObYeHhnfEwbNFOJ5bimPnSzCgTQq6NUsCUN4L464oIR0/X4KmSTE+P6RuH9JG/np413TEmQ3Q6XTISMpQnffd/RfB7nRDpwNMhvLAo3vzJNU5I7s3lb/W6YDG8RY0jrf4/ZwkMSZmbEKl2ObE7fM2AAA6ZyRiVM+mPs+TZhGyx6b+cB0binY1DmzeeustjBs3DitXrkSPHj1gMplUt//tb3+r9eAiSbxG87DUY2M2eE9n7ZyRALNRj8IyJ46dL8GOEwXo37oR0hKt+OWPHNz14YYqH9NXUONLgsWI8YNa4b5h7WFzuGA1GZBf6sCe7EIM7dQEOp0OnTMS0TnDOxOSHGuWv870aM7VEu8jyFMyG+tnaq+0pk2Z0w0hRINZeiDUTuaV4sKXl8jfbzp6XjOwkT5c2WNTfyp7bEI8EKIgqXFg8/HHH+N///sfrFYrli1bpvrQ0Ol0DS6widVoHpbKIL4+zI0GPVqnxmJvThEumbmsxo8dYzLg2/uHoE3jOOSXOmDQ6XAktxhFZU60SIlFi5TKgEQKOuIsRq/p0dFGCmxcbgGHS8Bs5IdnfZj7+yHV94fOavdhySsPsxRVb7ilAkW7Ggc2Tz31FJ577jk88cQT0HNxLc1ZUVLTq0UjS9GmcZzX7B5fxvRpjkkXtkb3ZklyyarM4cL+00Xo1ixRfrOSthToGZtck6cRVaQeGwAoc7rqLVPU0J2uWLcpOdaEvBIHCst8N8e73ULOGhj5HlJvpBjSxe5hilI1DmzsdjtuvPFGBjUVEqy+ZkVVBjZaH6qdMxLx884czet+c99g9MxM9nmb1WTw6k+hSmaDHnpd+eyPUrsLm46cR+eMRGQkWUM9tKhWVFa+Ce7g9o3x/bZTKCzzvSmuU/HByr2i6g9LURTtahyVTJw4EZ9++mldjiWipSd6f1gqe2y0MjaXeqy9Un4tC/52WQcsfvgSzaCGqqfT6eRy1PfbTmHS3PUY+hpXzA62Ilt5INO04ndCK7BRZgwM7LGpN2wepmhX44yNy+XCq6++ip9//hk9e/b0ah5uSBtfApVrpiipmoc1ApteLZLx75t74/ttp3BJpyZonRrntR4L1VyMyYASuwsr9pVvPMrF+oKv2F4eyEiZMa1SlEvxwcrp3vVHz5WHKcrVOLDZvn07evfuDQCqTS+BhrPxZXXUpSjvwEdyzQXNcM0FzeprWA2KlLFR9kA5XW4YfcxSo7ohlaKaJpU3pxfZnD5npblciowNA5t6w3VsKNrVOLD54IMPkJmZ6dVjI4TAsWPHaj2wSGQ26lVbKNj8KEVRcEkNxMrpxKcLbVE/IyyUimzlMwGbJpdnbNyifGkCz2UAnIr95dhjU3+kGJJbKlC0qvGnbZs2bXD27Fmv47m5uWjTpo2Pe0S/xVMuwStje2BiVisAUilKe7o3BZ+UsVH2eWQXlIVqOA1CcUWPTeM4i1xi8lWOkhavNOp1XotTUvBUTvcO8UCIgqTGn7Za0X5RURGs1oY566RFSixu7N9SXoVYNd2bpY+QkFYfVm40WmLjSsTB4nS55d3U461Gebagrwbi6vrPKDjYPEzRLuBS1JQpUwCUR/1///vfERtbufiby+XC2rVr0atXrzobYCSSyk42p6uyFGXim3coSE3deSV2+Zj0wUt1T7kadpzFgASrCedLHD4DG4e8QSx/N+qT1GPDdWwoWgUc2GzevBlAecZm+/btMJsrl9w3m8244IIL8Mgjj9TdCCOQ9Beo3Vm5V5SvLRUo+CxGKWNT+cFaYvc9/ZhqTypDmQw6WIwGua/GVymqujWeKDgMejYPU3QLOLBZurR8HZDbbrsNb775JhIT/dtpuSGRgph1h3Nx5FwJAN+bYFLwSRmbAkUpiptiBo8U2MRVBDR+laIY9NcrKbBxuhjYUHSq8TvK3LlzGdRokEpRUlAD8M07VKxS9sxVOQOnxEdg88J3f6DHMz9j0YaGOaOvrhRWBDbxcmBTvr6V71JU+QcrMzb1y8Dp3hTl+I4SBL7eqPnmHRq+Fk701WMz57dDKLQ58dr/9tTHsKJWsUdgk2jVLkUxYxMa0gw0F9eqpCjFd5Qg8BXEcB2b0LD6KAF6lqKkLQAAIKfAJje1UuACKkW5uBRCKEhT8F3M2FCU4jtKEJgN3h+mfPMODV+BjWcp6vj5EtX3V7yxIqhjimZSAONZiiqyOfHdtpOY+fNuOVNjd7IUFQpyxoYBPEWpGq88TNp8Z2zYPBwKiT52XS91qLMHx3NLVd8fOluMvTmF6JieENSxRSPPUlR8xeufW2zHfQvLZ1Qa9Xo8dHlHue/JxA0w65XUY8PeYYpW/FMpCNhjEz6SYkxexzwzNscqMjYD2qTIx77afCK4A4tS0jo2cZbyQF4qRUmbkALA/jNFAJQL9DHor0/ydG+uY0NRip+2QeCrn4aBTWgkx5q9jhV59HscP1+esbkgMwmz/twHAPD1lpNR8ca/5VgeyupxQcIiOWNTHlBKpai8EuXKz+XnOLjGU0gY2GNDUY7vKEHA5uHwkRzrnbHxbGSVpuW3SInFZV3SkGAx4kReKTYcOV8vYwyWLzcfx+hZv+P2eevrbcPDIrnHRp2xUZ1TEdhUZmxYiqpPBq48TFGOn7ZB4OsvUO5eHBq+SlEFiqnHNqcLaw+eAwB0a5YIq8mA4V3TAQA3vLMaqw54b/QaKd5ZfhAAsOrAORw6W1wvj+k5K8pXj5O0+zczNqFROd2bgQ1FJ76jBIGv7IzTzRkIoZDsK7BRrEL8+/6zKLQ5kZ5oQe8WjQAAI7qly7e/8lNkrWuzN6cQJ/NK4XC5cfBMZTBzJLekinvVnSKv6d7er3+Rrfz155YKoWFkYENRjrOigsDXG7WdUxBCItFHYCOVos4V2XD7vA0AgJHdMuS/ZK/omoF4ixFFNifyFZtnhrv1h3Mx7j+rAQDNk2NUqy0fq+fARipBJSoCm3ZN4nDgTLFcrpJKUdwEs34ZGNhQlOM7ShD4CmxYigoNi1EPvcdLX2hzwuUW+Mcve+VjV/ZoKn+t1+vwyd2DAFSWTSLBhsOVPUEn8tRT2OsrsJFLUebywCY90YLeLZMBAKN7NQdQGfzIpShmbOqVvLs3m4cpSvEdJQikNLxkQOsUDO+aFqLRNGw6nU7+kFXaf7oIC9celb/v3zpFdXvz5BgAwNkiW73OKqqNAxXTqJWk5un8Uu8tDYLBsxSl0+mw8M5BWHjnQNya1RpA+R5RNqdL0TzMt6H6JCXIomHWH5EvLEUFgTL93ijWhP+7JyuEo6E4i1HenFHy6GdbAQD9WjXConuyoPPIqCmDU5vD7XMF43BztGJ21/PXdcOq/eeQYDWifVo8Zvy4G8X1lHnyLEUB5ft1Xdi+MdxuAZ0OEKI80LKzeTgkDPry15ulKIpWDGyCTHoTodCJVWyEmZZgwelCG7Ydz4dBr8Mz13TzCmoA9Wq4kdL4XeYsD14yG8XgPxP6AoC8W3mRzXuvpmCQAijPrCVQXuJLijEhr8SBvBIHZ0WFiPRyOxnYUJTiO0qQtW0SF+ohNHgWRbalQ3q8/PXUKzujR2aSz/vodDq5yTJSPgAcFQ3qRkUwLW1tUFwPgY0QAsV2qRTlO8PVqGLBxLwSB2dFhYjU7+dmjw1FKb6jBMncSf3Rv3UjvDymR6iH0uApsy/9WpX30ozq2RR3DGlT5f2kabGRstu3qyKzZFR0S0uZk/rI2JTYXZA+KxMs3rPRgMqen/Mlds6KChGuY0PRjqWoILm0cxou7cyG4XBgUHzQ3zu0HS5sl4q+rRr5LEEpmQx62JxuOCNkqr40TqMiUKjPwEZ6DL0OsJp8ByuVGRs7Z0WFCNexoWjHdxSKeiZFacZqMmBg21TVh7+WylJUZGRsHFLGRpGhkpp466MUpZwRpRU0ShmbN3/dx1lRIcKMDUU7vqNQ1PO1SJ8/pBJWpPTYuOQeG+9SVH3MiiqWN8DUTgR3b1be03S60IbSimn0bB6uXwauY0NRLqzeUWbMmIH+/fsjISEBaWlpGD16NPbsUS9pX1ZWhsmTJyM1NRXx8fEYO3YscnJyQjRiigSD2qZUf5IPUhNupJSiHG4fzcMVa/jYXW45QxIslRtgagc2Ey9sDaA8WJR2VWfGpn5JmUiuY0PRKqzeUZYvX47JkydjzZo1+OWXX+BwOHDFFVeguLhyz5uHHnoI3377LRYtWoTly5fj5MmTGDNmTAhHTeFu/MBWuLBdKu65pF1A95NKOpHSPOx0Sc24yoxN5eykYJejPBfn88Wg1yGzUfnih3Jgw4xNvYq02X5EgQqr5uGffvpJ9f28efOQlpaGjRs34uKLL0Z+fj7mzJmDhQsXYtiwYQCAuXPnokuXLlizZg0GDRoUimFTmIsxG7DwrsB/NqTZOpHyASCNU9ksbTToYTGWN0EX2ZxoFGcO2uNLU72rytgA5as6S0ENwIxNfZMzNixFUZQK63eU/Px8AEBKSnkpYePGjXA4HBg+fLh8TufOndGyZUusXr06JGOk6GWIsOneUsnMc/p0fD3NjJL21aousPHM6HC6d/2S94qKkICdKFBhlbFRcrvdePDBBzF48GB0794dAJCdnQ2z2Yzk5GTVuenp6cjOzvZ5HZvNBpvNJn9fUFAQtDFTdIm0abFOH7OiACDeasS5YnvwS1Fl1ZeiAHhtSsqMTf2q/LkO8UCIgiRs31EmT56MHTt24JNPPqnVdWbMmIGkpCT5X4sWLepohBTt5FJUhDQP+ypFAZU7bQc7Y1M5K6rqfbX0HlPBLQxs6lXldG9GNhSdwvId5b777sN3332HpUuXIjMzUz6ekZEBu92OvLw81fk5OTnIyMjwea2pU6ciPz9f/nfs2LFgDp2iSCQ1D7vcQl7116T3XYoK9pRvKXCKt1aXsVEHNixF1a/K6d4hHghRkITVO4oQAvfddx++/PJLLFmyBG3aqJe879u3L0wmExYvXiwf27NnD44ePYqsLN87aFssFiQmJqr+EflDChAioXlYuYigZylKmhkVDrOiAO+MEktR9YvTvSnahVWPzeTJk7Fw4UJ8/fXXSEhIkPtmkpKSEBMTg6SkJNxxxx2YMmUKUlJSkJiYiPvvvx9ZWVmcEUV1LpKah5XlMqNHxkYKNArrrRRV9duK56LERs+mGwqqSFtRmyhQYRXYzJ49GwAwdOhQ1fG5c+di0qRJAIA33ngDer0eY8eOhc1mw4gRI/D222/X80ipIZAyH5HQPKwKbDwyNvW1rUKRn4GNZylKz8CmXlVmbEI8EKIgCavARvixroLVasWsWbMwa9asehgRNWSR1DysKkVpNA+HaymKYU390nNLBYpyLG4TaZACBEcE/GmrnBHluQFlfe3wnVtsBwCkVLMIoGcpyjODQ8EVacsYEAWKgQ2RhkjK2Eh9QL76VSpnRQU3sDlbWL5eVON4S5XnGTxLUYxr6pWBgQ1FOQY2RBoiaU8dl9t7Z29JZcYmeNO9S+xOFNvLr98koerAxitDw8CmXukZ2FCUY2BDpEFqwnVGwKwoR0VWyehjTRhpuneRzRG0xz9bWF6Gspr0iDNXs0Cf3jNjw8imPkkZM+4VRdGKgQ2Rhkhcx8Zk8A4SKmdFBS9jc7qwDEB5tsazx8eTZ1KJgU39YimKoh0DGyINBjljE/4fANIYPWccAfUzK+rg2WIAQKuUuGrP9QxkGNbULwY2FO0Y2BBpiKSUvVPusfH+lU6NL5+llFNQFrTVZg+cKQIAtGtSfWDjGXwxY1O/pGolp3tTtGJgQ6RB+vyNhMBGGqOvjE2r1DiYDXoU21147PNtmL/6MIpsTrjdAueKbMjOLy8jFZQ5MO2rHXjm6x04cq4YZQ4XVuw9A5uz+hLWjhP5AID2afHVnuvVO8x3oXolr2PDjA1FqbBaoI8onETS7BEpE+Nr6rTJoEenjARsP5GPzzYex2cbj2Pa1zthNuhhVzRGx1uM8lo3X205CZNBh7NF5U3Bwzqn4eYBLZGeaMG/Fu+DXqdDQZkDB84UlwdIFWvYDOnQpNqxek73Zr6mfklZvUj4uSaqCQY2RBoqS1EhHogfpDFqbU8w9arOeHTRNpzIK5WP2T1mexXZnEiwGhFjMuB0xZo0kiW7T2PJ7tNVjmFA6xS0aexHjw1LUSElVSsZ2FC0YmBDpEH6AI6kUpRWkHBhu8b4/YlhOF9sx+FzxTh2vhRCCOh1OizaeBxbj+XhtsGtcdvgNjDodfhk3VGUOVzo1zoFP+3IxrxVh1XXu6hDY7ROjcOuUwUQAFqmxOKJKzv7NVavvaIY2NQrQwT9XBPVBAMbIg2R1ItQGdhUfV6jODMaxZnRu2Uj+dg1FzTzOu/Oi9rKXw9qm4qnRnWBzenGgdNF6NE8qVYbV3relXFN/eKWChTtGNgQaYio5uGKqlKwsh8mgx4mgx4XtEiu9bW8pnszsKlX0usfCeszEdUE5yMQaZBT9hHwASAFX9UtjhcO2GMTWpH0c01UEwxsiDToI6p5WJruHeKB+MGrFBWaYTRYcok1AjKRRDURAW+DRKERSR8A0hAjIfvhvbt3+I85mkh7oLnDfws0ohphYEOkQcp+RELKPpJLUREw5KhikHtsGNlQdGJgQ6RBF0FbKriqWKAv3Hg3D0fAoKNI5TIGgIiAn22iQDGwIdJQuVlgiAfiBymp5FnmCUfK4CsSArFoo/wZiYBkJFHAGNgQaYikTTBFNQv0hRPlGCNhvNFG2rUeYDmKohMDGyIN0mduJAQ20l/ekRAnKHtsImG80UaVsWFcQ1GIgQ2RBkMErdDqiqiMTeXX7K+pf8od4CNhxh9RoBjYEGmQgoRIeO8X8jo24R8oKMcYAcONOqrAJgKCdqJAMbAh0qCPoIxN5XTvEA/ED8osjY7L89U7ZSkqEn62iQLFwIZIgyGCFugL9l5RdcmgY8YmlPTM2FCUY2BDpEF6/4+EtT5cfu7uHQ7U070jYMBRSN4vKgJ+tokCxcCGSEMklaIidbo3K1GhIQU23OGbohEDGyINkbUJZvl/PbcrCEd6VfNw+I83GslrNEXCDzdRgBjYEGmQ94qKgHS9O2JLUaEbR0MWSUsZEAWKgQ2RBnl37wh485czNhGQATGoFugL//FGI+l/QSQ0xhMFioENkQZ9BG2p4HZHTo+NjrOiQs5YkY6MhKCdKFAMbIg0yDNHImDZebkUFQGRgnK6NzM2oRFJ2UiiQDGwIdKgj6R1bORSVGjH4Q89J0WFnNQ/xsCGohEDGyIN0gdwJJSiImm6t467e4ecUV/+1h8JP9tEgWJgQ6ShshQV/m/+0l/ekRAncK+o0KuIa7iODUUlBjZEGiJxHRtDBEQ23N079EwVkY3TFQE/3EQBYmBDpCGSVh52R1ApSq+a7h3CgTRgpoomG4crAjrjiQLEwIZIgyGCpnvLPTYR8ButZ49NyBkN5a87AxuKRhHwNkgUGpHUPCx9PkVCaUddigrdOBqyyoxN+P9sEwWKgQ2RhsgsRYV4IH5gxib0zAapx4YZG4o+DGyINEgfuhGQsJFLUZHQPKxjxibkpFKUnYENRSEGNkQa5EXMIiCykZJKkVGKUjQPh3AcDZnJwFlRFL0Y2BBpiKRl5yNqVhRLUSFnYvMwRTEGNkQaIqkU5YqoHhvl1xEw4CjE6d4UzRjYEGkwRFDzsBR8GSIgstGpNsEM4UAaMCNnRVEUY2BDpEH60I2IHht5S4XwjxS48nDosRRF0YyBDZEGKfshIiGwiajdvblXVKjJWypEQDaSKFAMbIg0GNg8HBR6lqJCzmSsmO7tZMaGog8DGyINOnlLhRAPxA9yYBMBKRAdm4dDTp7u7WZgQ9GHgQ2RBqkU5Y6AyCaSVh7Wsccm5LilAkUzBjZEGvSR1Dws99iEf6DABfpCT2oeZimKohEDGyINerkUFQGBjTtyMjZsHg49o56lKIpeDGyINFSWokI8ED9IwVcklHa4QF/omY0VpShn+AftRIFiYEOkQd5SIRIyNlygjwJgrPg5cURC1E4UIGOoB0AUriqy9ZFRioqg5mEu0Bd6UvNwffbYlDlcWH84F9n5ZRjeJR3xViN+3JGNOb8dQu8WyRjUNhXDOqfJ2SSimmJgQ6RBuVeUECKsP4RFhDYPR0IgFo0SrOVv/UU2Z1AfZ8XeM/hqywmsO5SL4+dLNc/beiwP81YdBgCM7ZOJwe1T0STBgl//yMEDwzsiJc4c1HFSdGFgQ6TBoPgAdgvAEMYfwq6I2lJBOSsq/McbjZJjywOFvBJHnV7X5RbYejwPi3fl4FR+Gb7YdMKv+yVYjCisCLI+33Qcn286Lt/2weojAIA4swEzx12A3dmFaJUSi1E9m8JqMtTp+Ck6MLAh0qBc7M7lFmHdvyL1AYVz8CVRLdDHqkNIJMWYAAD5pXUX2Bw6W4ynv9qO3/efUx1vkRKD+4d1wPHzpbggMwnt0+JhNRlQbHPi8LliXNiuMawmA47lluDRz7ZizcFcn9cvtrvw1wWb5O9f/mk3nhjZGdf2agYAOHCmCJmNYhFvqfxY++NkAVxugR6ZSXX2PCn8MbAh0qCMY8K9z0aa7h3OwZdEGTBGQuksGiXHlgc2eSX2OrnexiPncct/16LU4QJQ3pyc1S4VvVsk4y+XtEOcxfdHTdsm8fLXLVJi8cndWRBCYMeJAny64Sj25hRh3aFcWE16lDnU/UBnCm14eNFWPLxoq3ysU3oCnruuG9YdysXFHZvgxndWQwhg6aND0Tw5pk6eK4U/BjZEGpRBQrgHNlIpKhK2VIiAIUa9ZEXGxu0WNf65OV9sx4p9Z/DoZ9tgd7rRt1UjvPSn7mjfJB5GQ83ScTqdDj0yk9AjsweAyv62w2eL8eOObFyQmQQBYO7vh/HrrhzVfffkFOKmd9cAAF7/Za98/B//2wOTXo9LO6dhRLd0FNmcKLa5kJFkrdEYKbwxsCHSoMwmhPtGmG65FBX+UYO6eTj8xxuNEisCG7cACsucSKrI4ARCCIEb3lmNfaeLAABtGsdh7m39kWgN/FpVkfrGWjeOw71D28nHB7dvjL05hbA73Zi97ADySx04kVeKQ2eLva4h9fp8uuGYfMxi1OPpq7viwOkiTLmiY52Pm0KHgQ2RBr1H83A4i6SMjXKEETDcqGQ1GZAca0JeiQPH80qQFBt4D8rKfWfloAYAHg5BcNAxPQEAMGt8HwBAYZkDv+07i76tGmHnqQIkWk24f+EmnMwv87qvzenGtK92AACO5pbgiSs7y9ejyMbAhkiDqhQV5pGNtJdhJGRs1Av0hf94o1X7JvHYcOQ89p8uQrdmgQU27/92CM9/9weA8kbkJ67sjKt7NgvGMAOSYDXhyh5NAQBpieVlpjdv7o0vNp3ATf1b4Oed2Xh72QGv+y3ZfRpLdp8GAFzYLhUv/akHWqfGwuUW+GbrSQxqm4pm7NGJGAxsiDQoswnhvvqwvFdUBMwyUi3QF7phNHgd0ssDm01HzuO6Xs39us/ag+dwz0cbcb5imrhRr8OKxy6VZ1mFo/6tU9C/dQoA4IIWybiqR1P8tv8sJma1xg/bT6majwFg1YFzuPS1ZapjA9uk4L5h7fHNlpO4ZVAr7DiZj4s7NEGLlNj6ehoUAAY2RBp0Oh10uvLF78K9ebhy5eHwDxUiYYwNwRVdM/DxumNYtPE4Hr+yM2LNVX8c7D9diDs+2KBa1O/9Sf3DOqjxpXvzJHRvXp6hGts3EyO7Z+BUfile+WkPluw+7bOfbu2hXKydsw4AsGhj5Ro7Rr0OM8f1RHKMGUM6NJZXdKbQYmBDVAW9TgeXEGG/EaYrkqZ7c6+osDC0UxO0So3FkXMleP7bP/DC6O6aH8wr953BXxdskoOaO4e0wVOjukRFKTHOYkT7tAS8d2s/AMCRc8XYfiIfm4/mISXOjO+2ncKuUwU+7+t0Czz0aXnGp1uzRPRonoQR3TPQMiUW247n4ZqezWo8O4xqjoENURUMOh1cEOFfioqgWVE6xft8NHwwRiqdToebB7TEyz/uxifrj+HbrSfxwujuGNMnUz7nj5MFeOabHdhw5DyEAPq2aoR3JvRF43hLCEceXK1S49AqNU7uGbrrorb4dP1R5BY7UGx34v3fDuHeoe1gd7rxzoqD8v12nizAzpMF+GR95cyrF7/bhZHdM1DmcOOiDo1xXa9m+HrLSaQlWtC9eRLOFtrQpnEcfw/qGAMboiro9QBcEdA8HEGzorhXVPiYdGFr7M0pxBebTqDY7sKU/9uKF7/fBbvTjYFtUrC4oqEWKN/DafqY7rAYG9Y2BmajHhOyWsvfPzqik5zZurhjE9z/8WakxplVM8Qk54rtWLD2KIDyrSIe/WwrHC71e8lVPTLQp2UjXN41HRlJVpwvdnB9nVpiYENUBelDONx7bCJpVpQymGG/TWhZTQa8fkMvTMxqjVd+2o1VB84ht7h8NWJlUPPJ3YMwqG1qqIYZVpTlusHtG2PTtMshhMCCtUdxIq8UzZJjMKhNChJjTLjm37/hdKFNPt8zqAGAH7Zn44ft2Xjx+13ysUdHdEJmoxh8tvE4Dp0txpg+mWiZEouremRU2wtFYRbYrFixAjNnzsTGjRtx6tQpfPnllxg9erR8uxACzzzzDN577z3k5eVh8ODBmD17Njp06BC6QVNUM8iBTYgHUo2I2lKBPTZh54IWyfjw9gHo+PSP8s/6xKxWWLrnDIZ2asKgpho6nQ63DGrldXzFY5fi0/XHkJ5owT0fle9z9ew1XdEhPQGrD5zDr7tycORcibwVhWTmz3tU3/9r8b7y+36zE12bJeLK7uVZHqNBhx0n8lFqd2F413RM/WI7Tpwvxdu39EGn9AQcyy1Fi5QYudQlreIc7cIqsCkuLsYFF1yA22+/HWPGjPG6/dVXX8W//vUvfPDBB2jTpg2mTZuGESNG4I8//oDVytQd1T2ptBPuKw9HUilKp5ruHf7jbSiMBj2mXN4R7644iKev7oob+rXAc6EeVISzmgyYeGFrAMC82/rjXJEdY/o0h06nw+D2jfHIiE4QQuDAmWK8uXgfjuaWYOuxPNU1DHqd/PtdZHNi3aFcrDvkvVHos9/+IX898p8r0SzJipP5Zbj74rYY2ycTL37/B1buO6u6T4/mSbA5XXh7fB/EmI0wGXR49ac92HosD+9P6o85vx1C31aNcHGHJli5/wxGdMuAyaCH0+XGwnVHEWs24k+9m8Og16HM4cLhc8W12k6jruiECM8cu06nU2VshBBo1qwZHn74YTzyyCMAgPz8fKSnp2PevHm46aab/LpuQUEBkpKSkJ+fj8TExGANn6JE7+f/h/MlDvzvoYvDelXSkf9cgd3ZhfjojoEY0qFxqIdTJYfLjQ5P/QgAuLpnU7z15z4hHhFR+LA5Xfh680m0SInFoLYpKHW48O3Wk7ioQxP8tv8svtlyEnanG1uO5cFk0KFjRgI2H82rl7FZjHq0SImFUa/D7uxC+fhPD16Exz/bhq3H8wEA7dPicd+l7TG6t3/rI/nL38/vsMrYVOXQoUPIzs7G8OHD5WNJSUkYOHAgVq9erRnY2Gw22GyVNc6CAt/T9oh8kUo7Yd9jE1EL9HHlYSItFqMBN/RvIX8fazbixv4tAQA39GuBG/qV3+ZwuaFDeabtdGEZPtt4HF0yEmE06PDbvrMotDnx+/6zOHKuRHEtA+4Y0gYHzhThh+3ZAY/N5nRjv48m6ZH/XKn6fv/popC+Z0ZMYJOdXf4/IT09XXU8PT1dvs2XGTNm4LnnmFClmpE+eMO+FBVJ070VX0dA5YwoLCmbmNMSrPjr0Pby9xd1aAIAyC2249uKLSE6pMV7lartTjcEBOb+fhhvLdkPm9OFbs2SMLJ7Bo6cK8awzukotjnxzoqDGNwuFfvPFOHgmWL0bpmMoZ2aoNTuxlNfbYcQgMmgw5XdmyKrXfn2E10yQpfhjpjApqamTp2KKVOmyN8XFBSgRYsWVdyDqJLcPBzmC/RFUvOwuseGiIIlJc4s9/j4YjaWB0f3XNIO91zSTvO8qkpKY/s2x5ajeejWPAnxlvAIKcJjFH7IyMgAAOTk5KBp06by8ZycHPTq1UvzfhaLBRZL9C4mRcEVMaUoEUnNw8p1bMJ/vESkzWI0YGCYzZqLgIp8uTZt2iAjIwOLFy+WjxUUFGDt2rXIysoK4cgomkmfu2G/8nBFRikSSlEqETZcIgp/YZWxKSoqwv79++XvDx06hC1btiAlJQUtW7bEgw8+iBdffBEdOnSQp3s3a9ZMtdYNUV2SMjZhOnlQFkl7RSkxY0NEdS2sApsNGzbg0ksvlb+XemMmTpyIefPm4bHHHkNxcTHuvvtu5OXlYciQIfjpp5+4hg0FjV5uHg7xQKrhiqDdvZUiLA4joggQVoHN0KFDq/zLWKfT4fnnn8fzzz9fj6Oihkz64A33WVGR1DysxAX6iKiuRUyPDVEoREwpSpruHWG/0ZGw7g4RRRa+rRBVQS5FhXtg447MUhS7h4morjGwIaqCnptgBlXExWFEFPYY2BBVQSqVuMM8smHzMBFROQY2RFUwRMiWCvI6NhEWKbB5mIjqGgMboiroI2zl4UgLbCJsuEQUARjYEFWhsscmzAObCG0e5u7eRFTXGNgQVcEQAQv0Kft/Ii1jw7iGiOoaAxuiKkgfvOGcsVFORY+0vaIiLcNEROGPgQ1RFSJhd29lY7Muwn6jGdYQUV2LsLdBovoVCYGNO5IzNhFWOiOi8MfAhqgK5oo9Csoc4dtk44rkHptQD4CIog4DG6IqJFjL94ktKnOGeCTa3IqYK9J6VjgriojqGgMboirEVwQ2hbbwDWxUzcORlrGJrOESUQRgYENUhQSrCQBQWOYI8Ui0KUtRERbXRNx4iSj8MbAhqkK8JfxLUcUV2aQ4syHiSjvcUoGI6hoDG6IqJFaUohZtPA5nmK7SV1gRdCXGmEI8ksBFWBxGRBGAgQ1RFWLMRvnrRRuPh3Ak2goqymRSo3MkuG1wazRJsGDSha1DPRQiijKR805IFALKNWL+tzMbw7ukY+OR8+jSNAGtUuPgdLlR6nAhwWpCid0Jp1sg0WpCTkEZ1hw8hyu7N4WAwLvLD+KPUwXokJ4Ai1GPBKsRY/pkIs5swJqDuWiaZEWJ3YViuxMxJgM2HT2Pkd0y4BICSTEmnCuyI7NRjFepyeZ04fvtpwAAidbIydg8c003TBvVlevYEFGd0wkRxiuPBUFBQQGSkpKQn5+PxMTEUA+HwlyZw4UR/1yBI+dKvG7r1iwRJ/JKYXe6MeXyjnjph10QAujbqhE2Hjlf7bWtJj1apsRib06RX2NplmTF9X0zEWsx4r8rD8EtBArLHHC4yn+FB7ZJwad/yQrsCRIRRQh/P78Z2BBVQwiBuz7ciF935dT6Wld2z0B+qQPbT+TLvTF1pXG8GRuevrxOr0lEFC78/fxmKYqoGjqdDu/d2hfL957B8r1n8OcBLfHHqQLsyS5EcqwJ/1q8H0U2J2LNBjx3bTfsPFmAFfvO4LbBbXC+2A6Hy41JF7ZGarxFvqbd6cYP20+hyObEyO4ZyCkow/liBzKSrDAZdGiZEovThTbEWYwoc7hw+Gwxpv+wC5uO5gEAjHod7rmkHQa2TcGEOesAAA3rTxQiIt+YsSGqpRN5pTiVV4o+LRsFvWfE6XJjye7T6NuqkRwo/bbvLJ74YhteHN0dQzulBfXxiYhChaUoDQxsiIiIIo+/n9+c7k1ERERRg4ENERERRQ0GNkRERBQ1GNgQERFR1GBgQ0RERFGDgQ0RERFFDQY2REREFDUY2BAREVHUYGBDREREUYOBDREREUUNBjZEREQUNRjYEBERUdRgYENERERRg4ENERERRQ1jqAdQ34QQAMq3PyciIqLIIH1uS5/jWhpcYFNYWAgAaNGiRYhHQkRERIEqLCxEUlKS5u06UV3oE2XcbjdOnjyJhIQE6HS6UA8nKAoKCtCiRQscO3YMiYmJoR5OWOBr4o2viRpfD298TbzxNfFWX6+JEAKFhYVo1qwZ9HrtTpoGl7HR6/XIzMwM9TDqRWJiIn/xPPA18cbXRI2vhze+Jt74mnirj9ekqkyNhM3DREREFDUY2BAREVHUYGAThSwWC5555hlYLJZQDyVs8DXxxtdEja+HN74m3viaeAu316TBNQ8TERFR9GLGhoiIiKIGAxsiIiKKGgxsiIiIKGowsCEiIqKowcAmihw+fBh33HEH2rRpg5iYGLRr1w7PPPMM7Ha76rxt27bhoosugtVqRYsWLfDqq6+GaMT1Y9asWWjdujWsVisGDhyIdevWhXpI9WbGjBno378/EhISkJaWhtGjR2PPnj2qc8rKyjB58mSkpqYiPj4eY8eORU5OTohGXL9efvll6HQ6PPjgg/Kxhvp6nDhxArfccgtSU1MRExODHj16YMOGDfLtQgj8/e9/R9OmTRETE4Phw4dj3759IRxxcLlcLkybNk31fvrCCy+o9imK9tdkxYoVuOaaa9CsWTPodDp89dVXqtv9ef65ubkYP348EhMTkZycjDvuuANFRUXBHbigqPHjjz+KSZMmiZ9//lkcOHBAfP311yItLU08/PDD8jn5+fkiPT1djB8/XuzYsUN8/PHHIiYmRrzzzjshHHnwfPLJJ8JsNov3339f7Ny5U9x1110iOTlZ5OTkhHpo9WLEiBFi7ty5YseOHWLLli3iqquuEi1bthRFRUXyOffcc49o0aKFWLx4sdiwYYMYNGiQuPDCC0M46vqxbt060bp1a9GzZ0/xwAMPyMcb4uuRm5srWrVqJSZNmiTWrl0rDh48KH7++Wexf/9++ZyXX35ZJCUlia+++kps3bpVXHvttaJNmzaitLQ0hCMPnpdeekmkpqaK7777Thw6dEgsWrRIxMfHizfffFM+J9pfkx9++EE89dRT4osvvhAAxJdffqm63Z/nP3LkSHHBBReINWvWiJUrV4r27duLm2++OajjZmAT5V599VXRpk0b+fu3335bNGrUSNhsNvnY448/Ljp16hSK4QXdgAEDxOTJk+XvXS6XaNasmZgxY0YIRxU6p0+fFgDE8uXLhRBC5OXlCZPJJBYtWiSfs2vXLgFArF69OlTDDLrCwkLRoUMH8csvv4hLLrlEDmwa6uvx+OOPiyFDhmje7na7RUZGhpg5c6Z8LC8vT1gsFvHxxx/XxxDr3ahRo8Ttt9+uOjZmzBgxfvx4IUTDe008Axt/nv8ff/whAIj169fL5/z4449Cp9OJEydOBG2sLEVFufz8fKSkpMjfr169GhdffDHMZrN8bMSIEdizZw/Onz8fiiEGjd1ux8aNGzF8+HD5mF6vx/Dhw7F69eoQjix08vPzAUD+mdi4cSMcDofqNercuTNatmwZ1a/R5MmTMWrUKNXzBhru6/HNN9+gX79+GDduHNLS0tC7d2+899578u2HDh1Cdna26nVJSkrCwIEDo/Z1ufDCC7F48WLs3bsXALB161b89ttvuPLKKwE0zNdEyZ/nv3r1aiQnJ6Nfv37yOcOHD4der8fatWuDNrYGtwlmQ7J//378+9//xmuvvSYfy87ORps2bVTnpaeny7c1atSoXscYTGfPnoXL5ZKfnyQ9PR27d+8O0ahCx+1248EHH8TgwYPRvXt3AOX/z81mM5KTk1XnpqenIzs7OwSjDL5PPvkEmzZtwvr1671ua4ivBwAcPHgQs2fPxpQpU/Dkk09i/fr1+Nvf/gaz2YyJEyfKz93X71K0vi5PPPEECgoK0LlzZxgMBrhcLrz00ksYP348ADTI10TJn+efnZ2NtLQ01e1GoxEpKSlBfY2YsYkATzzxBHQ6XZX/PD+oT5w4gZEjR2LcuHG46667QjRyCieTJ0/Gjh078Mknn4R6KCFz7NgxPPDAA1iwYAGsVmuohxM23G43+vTpg+nTp6N37964++67cdddd+E///lPqIcWMv/3f/+HBQsWYOHChdi0aRM++OADvPbaa/jggw9CPTSqBjM2EeDhhx/GpEmTqjynbdu28tcnT57EpZdeigsvvBDvvvuu6ryMjAyvGR7S9xkZGXUz4DDRuHFjGAwGn8832p5rde677z589913WLFiBTIzM+XjGRkZsNvtyMvLU2UpovU12rhxI06fPo0+ffrIx1wuF1asWIG33noLP//8c4N6PSRNmzZF165dVce6dOmCzz//HEDle0NOTg6aNm0qn5OTk4NevXrV2zjr06OPPoonnngCN910EwCgR48eOHLkCGbMmIGJEyc2yNdEyZ/nn5GRgdOnT6vu53Q6kZubG9TfJ2ZsIkCTJk3QuXPnKv9JPTMnTpzA0KFD0bdvX8ydOxd6vfp/cVZWFlasWAGHwyEf++WXX9CpU6eoKkMBgNlsRt++fbF48WL5mNvtxuLFi5GVlRXCkdUfIQTuu+8+fPnll1iyZIlXGbJv374wmUyq12jPnj04evRoVL5Gl112GbZv344tW7bI//r164fx48fLXzek10MyePBgr2UA9u7di1atWgEA2rRpg4yMDNXrUlBQgLVr10bt61JSUuL1/mkwGOB2uwE0zNdEyZ/nn5WVhby8PGzcuFE+Z8mSJXC73Rg4cGDwBhe0tmSqd8ePHxft27cXl112mTh+/Lg4deqU/E+Sl5cn0tPTxYQJE8SOHTvEJ598ImJjY6N6urfFYhHz5s0Tf/zxh7j77rtFcnKyyM7ODvXQ6sW9994rkpKSxLJly1Q/DyUlJfI599xzj2jZsqVYsmSJ2LBhg8jKyhJZWVkhHHX9Us6KEqJhvh7r1q0TRqNRvPTSS2Lfvn1iwYIFIjY2Vnz00UfyOS+//LJITk4WX3/9tdi2bZu47rrrompqs6eJEyeK5s2by9O9v/jiC9G4cWPx2GOPyedE+2tSWFgoNm/eLDZv3iwAiNdff11s3rxZHDlyRAjh3/MfOXKk6N27t1i7dq347bffRIcOHTjdm/w3d+5cAcDnP6WtW7eKIUOGCIvFIpo3by5efvnlEI24fvz73/8WLVu2FGazWQwYMECsWbMm1EOqN1o/D3PnzpXPKS0tFX/9619Fo0aNRGxsrPjTn/6kCoajnWdg01Bfj2+//VZ0795dWCwW0blzZ/Huu++qbne73WLatGkiPT1dWCwWcdlll4k9e/aEaLTBV1BQIB544AHRsmVLYbVaRdu2bcVTTz2lWioj2l+TpUuX+nz/mDhxohDCv+d/7tw5cfPNN4v4+HiRmJgobrvtNlFYWBjUceuEUCyjSERERBTB2GNDREREUYOBDREREUUNBjZEREQUNRjYEBERUdRgYENERERRg4ENERERRQ0GNkRERBQ1GNgQUUQ6fPiwvAlsXezNI13Lc2dvIoosDGyIKKL9+uuvqv1qaurUqVP45z//WfsBEVFIMbAhooiWmpqK1NTUWl8nIyMDSUlJdTAiIgolBjZEFHJnzpxBRkYGpk+fLh9btWoVzGZzwNmYSZMmYfTo0Zg+fTrS09ORnJyM559/Hk6nE48++ihSUlKQmZmJuXPn1vXTIKIwYAz1AIiImjRpgvfffx+jR4/GFVdcgU6dOmHChAm47777cNlllwV8vSVLliAzMxMrVqzA77//jjvuuAOrVq3CxRdfjLVr1+LTTz/FX/7yF1x++eXIzMwMwjMiolBhxoaIwsJVV12Fu+66C+PHj8c999yDuLg4zJgxo0bXSklJwb/+9S906tQJt99+Ozp16oSSkhI8+eST6NChA6ZOnQqz2Yzffvutjp8FEYUaMzZEFDZee+01dO/eHYsWLcLGjRthsVhqdJ1u3bpBr6/8uy09PR3du3eXvzcYDEhNTcXp06drPWYiCi/M2BBR2Dhw4ABOnjwJt9uNw4cP1/g6JpNJ9b1Op/N5zO121/gxiCg8MWNDRGHBbrfjlltuwY033ohOnTrhzjvvxPbt25GWlhbqoRFRBGHGhojCwlNPPYX8/Hz861//wuOPP46OHTvi9ttvD/WwiCjCMLAhopBbtmwZ/vnPf2L+/PlITEyEXq/H/PnzsXLlSsyePTvUwyOiCMJSFBGF3NChQ+FwOFTHWrdujfz8/ICvNW/ePK9jy5Yt8zpWmx4eIgpfDGyIKKJdeOGF6NWrF1atWlWr68THx8PpdMJqtdbRyIgoFBjYEFFEyszMxL59+wCgxtPClbZs2QKgfCo4EUUunRBChHoQRERERHWBzcNEREQUNRjYEBERUdRgYENERERRg4ENERERRQ0GNkRERBQ1GNgQERFR1GBgQ0RERFGDgQ0RERFFDQY2REREFDX+H/vbhRIF9oKdAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "ds = ds.sel(x=slice(-30, 101))\n", "sections = {\n", @@ -198,28 +235,15 @@ " \"temp2\": [slice(5.5, 15.5)], # cold bath\n", "}\n", "\n", - "st_var, resid = ds.variance_stokes_constant(sections=sections, st_label=\"st\")\n", - "ast_var, _ = ds.variance_stokes_constant(sections=sections, st_label=\"ast\")\n", - "ds.calibration_single_ended(sections=sections, st_var=st_var, ast_var=ast_var)\n", - "\n", - "ds.isel(time=0).tmpf.plot()" + "st_var, resid = variance_stokes_constant(\n", + " ds.dts.st, sections, ds.dts.acquisitiontime_fw, reshape_residuals=True\n", + ")\n", + "ast_var, _ = variance_stokes_constant(\n", + " ds.dts.ast, sections, ds.dts.acquisitiontime_fw, reshape_residuals=False\n", + ")\n", + "out = ds.dts.calibrate_single_ended(sections=sections, st_var=st_var, ast_var=ast_var)\n", + "out.isel(time=0).tmpf.plot()" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ds" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { From 70c0cd49a42f50870134a544ed60d7a4393f7143 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Mon, 16 Oct 2023 16:49:46 +0200 Subject: [PATCH 42/66] Update 13Fixed_parameter_calibration.ipynb --- .../13Fixed_parameter_calibration.ipynb | 29 +++++++++---------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/docs/notebooks/13Fixed_parameter_calibration.ipynb b/docs/notebooks/13Fixed_parameter_calibration.ipynb index fc2642ae..88407620 100644 --- a/docs/notebooks/13Fixed_parameter_calibration.ipynb +++ b/docs/notebooks/13Fixed_parameter_calibration.ipynb @@ -26,6 +26,8 @@ "import os\n", "\n", "from dtscalibration import read_silixa_files\n", + "from dtscalibration.dts_accessor import DtsAccessor # noqa: E402\n", + "from dtscalibration.variance_stokes import variance_stokes_constant\n", "import matplotlib.pyplot as plt\n", "\n", "%matplotlib inline\n", @@ -68,9 +70,13 @@ "fix_gamma = (481.9, 0) # (gamma value, gamma variance)\n", "fix_dalpha = (-2.014e-5, 0) # (alpha value, alpha variance)\n", "\n", - "st_var, resid = ds100.variance_stokes_constant(sections=sections, st_label=\"st\")\n", - "ast_var, _ = ds100.variance_stokes_constant(sections=sections, st_label=\"ast\")\n", - "ds100.calibration_single_ended(\n", + "st_var, resid = variance_stokes_constant(\n", + " ds.dts.st, sections, ds.dts.acquisitiontime_fw, reshape_residuals=True\n", + ")\n", + "ast_var, _ = variance_stokes_constant(\n", + " ds.dts.ast, sections, ds.dts.acquisitiontime_fw, reshape_residuals=False\n", + ")\n", + "out = ds100.dts.calibrate_single_ended(\n", " sections=sections,\n", " st_var=st_var,\n", " ast_var=ast_var,\n", @@ -99,8 +105,8 @@ }, "outputs": [], "source": [ - "print(\"gamma used in calibration:\", ds100.gamma.values)\n", - "print(\"dalpha used in calibration:\", ds100.dalpha.values)" + "print(\"gamma used in calibration:\", out.gamma.values)\n", + "print(\"dalpha used in calibration:\", out.dalpha.values)" ] }, { @@ -123,24 +129,15 @@ }, "outputs": [], "source": [ - "ds1 = ds100.isel(time=0) # take only the first timestep\n", - "\n", - "ds1.tmpf.plot(\n", + "out.isel(time=0).tmpf.plot(\n", " linewidth=1, figsize=(12, 8), label=\"User calibrated\"\n", ") # plot the temperature calibrated by us\n", - "ds1.tmp.plot(\n", + "ds100.isel(time=0).tmp.plot(\n", " linewidth=1, label=\"Device calibrated\"\n", ") # plot the temperature calibrated by the device\n", "plt.title(\"Temperature at the first time step\")\n", "plt.legend()" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { From a4b0f40b29255f4b86dd7a1743ef983d37ec9581 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Mon, 16 Oct 2023 16:53:12 +0200 Subject: [PATCH 43/66] Update 14Lossy_splices.ipynb --- docs/notebooks/14Lossy_splices.ipynb | 46 +++++++++++++++++++--------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/docs/notebooks/14Lossy_splices.ipynb b/docs/notebooks/14Lossy_splices.ipynb index 977969f3..09759eb0 100644 --- a/docs/notebooks/14Lossy_splices.ipynb +++ b/docs/notebooks/14Lossy_splices.ipynb @@ -44,6 +44,8 @@ "import os\n", "\n", "from dtscalibration import read_silixa_files\n", + "from dtscalibration.dts_accessor import DtsAccessor # noqa: E402\n", + "from dtscalibration.variance_stokes import variance_stokes_constant\n", "import matplotlib.pyplot as plt\n", "\n", "%matplotlib inline" @@ -144,14 +146,20 @@ }, "outputs": [], "source": [ - "ds_a = ds.copy(deep=True)\n", - "\n", - "st_var, resid = ds_a.variance_stokes(sections=sections, st_label=\"st\")\n", - "ast_var, _ = ds_a.variance_stokes(sections=sections, st_label=\"ast\")\n", - "rst_var, _ = ds_a.variance_stokes(sections=sections, st_label=\"rst\")\n", - "rast_var, _ = ds_a.variance_stokes(sections=sections, st_label=\"rast\")\n", + "st_var, resid = variance_stokes_constant(\n", + " ds.dts.st, sections, ds.dts.acquisitiontime_fw, reshape_residuals=True\n", + ")\n", + "ast_var, _ = variance_stokes_constant(\n", + " ds.dts.ast, sections, ds.dts.acquisitiontime_fw, reshape_residuals=False\n", + ")\n", + "rst_var, _ = variance_stokes_constant(\n", + " ds.dts.rst, sections, ds.dts.acquisitiontime_bw, reshape_residuals=False\n", + ")\n", + "rast_var, _ = variance_stokes_constant(\n", + " ds.dts.rast, sections, ds.dts.acquisitiontime_bw, reshape_residuals=False\n", + ")\n", "\n", - "ds_a.calibration_double_ended(\n", + "out = ds.dts.calibrate_double_ended(\n", " sections=sections,\n", " st_var=st_var,\n", " ast_var=ast_var,\n", @@ -159,7 +167,7 @@ " rast_var=rast_var,\n", ")\n", "\n", - "ds_a.isel(time=0).tmpw.plot(label=\"calibrated\")" + "out.isel(time=0).tmpw.plot(label=\"calibrated\")" ] }, { @@ -184,12 +192,20 @@ }, "outputs": [], "source": [ - "st_var, resid = ds.variance_stokes(sections=sections, st_label=\"st\")\n", - "ast_var, _ = ds.variance_stokes(sections=sections, st_label=\"ast\")\n", - "rst_var, _ = ds.variance_stokes(sections=sections, st_label=\"rst\")\n", - "rast_var, _ = ds.variance_stokes(sections=sections, st_label=\"rast\")\n", + "st_var, resid = variance_stokes_constant(\n", + " ds.dts.st, sections, ds.dts.acquisitiontime_fw, reshape_residuals=True\n", + ")\n", + "ast_var, _ = variance_stokes_constant(\n", + " ds.dts.ast, sections, ds.dts.acquisitiontime_fw, reshape_residuals=False\n", + ")\n", + "rst_var, _ = variance_stokes_constant(\n", + " ds.dts.rst, sections, ds.dts.acquisitiontime_bw, reshape_residuals=False\n", + ")\n", + "rast_var, _ = variance_stokes_constant(\n", + " ds.dts.rast, sections, ds.dts.acquisitiontime_bw, reshape_residuals=False\n", + ")\n", "\n", - "ds.calibration_double_ended(\n", + "out2 = ds.dts.calibrate_double_ended(\n", " sections=sections,\n", " st_var=st_var,\n", " ast_var=ast_var,\n", @@ -198,8 +214,8 @@ " trans_att=[50.0],\n", ")\n", "\n", - "ds_a.isel(time=0).tmpw.plot(label=\"no trans. att.\")\n", - "ds.isel(time=0).tmpw.plot(label=\"with trans. att.\")\n", + "out.isel(time=0).tmpw.plot(label=\"no trans. att.\")\n", + "out2.isel(time=0).tmpw.plot(label=\"with trans. att.\")\n", "plt.legend()" ] }, From 70abdd78eee7090c1e1d854d4ffb0c4a7e854539 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Mon, 16 Oct 2023 16:55:42 +0200 Subject: [PATCH 44/66] Update 16Averaging_temperatures.ipynb --- docs/notebooks/16Averaging_temperatures.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/notebooks/16Averaging_temperatures.ipynb b/docs/notebooks/16Averaging_temperatures.ipynb index 2d97bb85..ba606b2f 100644 --- a/docs/notebooks/16Averaging_temperatures.ipynb +++ b/docs/notebooks/16Averaging_temperatures.ipynb @@ -98,7 +98,7 @@ }, "outputs": [], "source": [ - "ds.calibration_double_ended(\n", + "ds.calibrate_double_ended(\n", " sections=sections,\n", " st_var=st_var,\n", " ast_var=ast_var,\n", From 3ff015b4f503d106f00a7f6b398ac2a65899cd87 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Mon, 16 Oct 2023 17:00:06 +0200 Subject: [PATCH 45/66] Update 17Temperature_uncertainty_single_ended.ipynb --- ...Temperature_uncertainty_single_ended.ipynb | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/docs/notebooks/17Temperature_uncertainty_single_ended.ipynb b/docs/notebooks/17Temperature_uncertainty_single_ended.ipynb index 301e4963..207f2f51 100644 --- a/docs/notebooks/17Temperature_uncertainty_single_ended.ipynb +++ b/docs/notebooks/17Temperature_uncertainty_single_ended.ipynb @@ -24,6 +24,8 @@ "import os\n", "\n", "from dtscalibration import read_silixa_files\n", + "from dtscalibration.dts_accessor import DtsAccessor # noqa: E402\n", + "from dtscalibration.variance_stokes import variance_stokes_constant\n", "import matplotlib.pyplot as plt\n", "\n", "%matplotlib inline\n", @@ -36,11 +38,14 @@ " \"probe1Temperature\": [slice(20, 25.5)], # warm bath\n", " \"probe2Temperature\": [slice(5.5, 15.5)], # cold bath\n", "}\n", + "st_var, resid = variance_stokes_constant(\n", + " ds.dts.st, sections, ds.dts.acquisitiontime_fw, reshape_residuals=True\n", + ")\n", + "ast_var, _ = variance_stokes_constant(\n", + " ds.dts.ast, sections, ds.dts.acquisitiontime_fw, reshape_residuals=False\n", + ")\n", "\n", - "st_var, resid = ds.variance_stokes_constant(sections=sections, st_label=\"st\")\n", - "ast_var, _ = ds.variance_stokes_constant(sections=sections, st_label=\"ast\")\n", - "\n", - "ds.calibration_single_ended(sections=sections, st_var=st_var, ast_var=ast_var)" + "out = ds.dts.calibrate_single_ended(sections=sections, st_var=st_var, ast_var=ast_var)" ] }, { @@ -59,7 +64,7 @@ "metadata": {}, "outputs": [], "source": [ - "ds1 = ds.isel(time=0)\n", + "ds1 = out.isel(time=0)\n", "\n", "# Uncertainty from the noise in (anti-) stokes measurements\n", "stast_var = ds1.var_fw_da.sel(comp_fw=[\"dT_dst\", \"dT_dast\"]).sum(dim=\"comp_fw\")\n", @@ -84,7 +89,8 @@ "outputs": [], "source": [ "# The effects of the parameter uncertainty can be further inspected\n", - "ds1.var_fw_da.plot(hue=\"comp_fw\", figsize=(12, 4))" + "# Note that the parameter uncertainty is not constant over the fiber and certain covariations can reduce to temperature uncertainty\n", + "ds1.var_fw_da.plot(hue=\"comp_fw\", figsize=(12, 4));" ] }, { @@ -119,7 +125,7 @@ "metadata": {}, "outputs": [], "source": [ - "ds.conf_int_single_ended(\n", + "out2 = ds.dts.monte_carlo_single_ended(result=out,\n", " st_var=st_var, ast_var=ast_var, conf_ints=[2.5, 97.5], mc_sample_size=500\n", ")" ] @@ -140,10 +146,10 @@ "metadata": {}, "outputs": [], "source": [ - "ds1 = ds.isel(time=0)\n", + "ds1 = out.isel(time=0)\n", "\n", - "(ds1.tmpf_mc_var**0.5).plot(figsize=(12, 4), label=\"Monte Carlo approx.\")\n", - "(ds1.tmpf_var**0.5).plot(label=\"Linear error approx.\")\n", + "(out2.isel(time=0).tmpf_mc_var**0.5).plot(figsize=(12, 4), label=\"Monte Carlo approx.\")\n", + "(out.isel(time=0).tmpf_var**0.5).plot(label=\"Linear error approx.\")\n", "plt.ylabel(\"$\\sigma$ ($^\\circ$C)\")\n", "plt.legend(fontsize=\"small\")" ] @@ -163,9 +169,9 @@ "metadata": {}, "outputs": [], "source": [ - "ds1.tmpf.plot(linewidth=0.7, figsize=(12, 4))\n", - "ds1.tmpf_mc.sel(CI=2.5).plot(linewidth=0.7, label=\"CI: 2.5%\")\n", - "ds1.tmpf_mc.sel(CI=97.5).plot(linewidth=0.7, label=\"CI: 97.5%\")\n", + "out.isel(time=0).tmpf.plot(linewidth=0.7, figsize=(12, 4))\n", + "out2.isel(time=0).tmpf_mc.sel(CI=2.5).plot(linewidth=0.7, label=\"CI: 2.5%\")\n", + "out2.isel(time=0).tmpf_mc.sel(CI=97.5).plot(linewidth=0.7, label=\"CI: 97.5%\")\n", "plt.legend(fontsize=\"small\")" ] }, From 7d053943e366ad64e877ca0833b9d4c837e240a2 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Mon, 16 Oct 2023 17:15:30 +0200 Subject: [PATCH 46/66] Formatting of the entire repository --- docs/notebooks/03Define_sections.ipynb | 2 +- .../04Calculate_variance_Stokes.ipynb | 2 +- docs/notebooks/07Calibrate_single_ended.ipynb | 2 +- docs/notebooks/08Calibrate_double_ended.ipynb | 15 +- .../12Datastore_from_numpy_arrays.ipynb | 2 +- .../13Fixed_parameter_calibration.ipynb | 2 +- docs/notebooks/14Lossy_splices.ipynb | 2 +- ...Temperature_uncertainty_single_ended.ipynb | 16 ++- src/dtscalibration/__init__.py | 2 +- src/dtscalibration/averaging_utils.py | 2 +- src/dtscalibration/calibrate_utils.py | 80 +++++------ .../calibration/section_utils.py | 14 +- src/dtscalibration/datastore_utils.py | 1 - src/dtscalibration/dts_accessor.py | 131 +++++++++--------- src/dtscalibration/variance_helpers.py | 6 +- src/dtscalibration/variance_stokes.py | 30 ++-- tests/test_averaging.py | 5 + tests/test_datastore.py | 5 +- tests/test_dtscalibration.py | 27 ++-- tests/test_variance_stokes.py | 119 ++++++++++++---- 20 files changed, 259 insertions(+), 206 deletions(-) diff --git a/docs/notebooks/03Define_sections.ipynb b/docs/notebooks/03Define_sections.ipynb index 19f87af1..74c56371 100644 --- a/docs/notebooks/03Define_sections.ipynb +++ b/docs/notebooks/03Define_sections.ipynb @@ -65,7 +65,7 @@ "outputs": [], "source": [ "# The following command adds the .dts accessor to the xarray Dataset.\n", - "from dtscalibration.dts_accessor import DtsAccessor # noqa: E402\n", + "from dtscalibration.dts_accessor import DtsAccessor # noqa: E402\n", "\n", "print(ds.dts.get_timeseries_keys()) # list the available timeseeries\n", "ds.probe1Temperature.plot(figsize=(12, 8));" diff --git a/docs/notebooks/04Calculate_variance_Stokes.ipynb b/docs/notebooks/04Calculate_variance_Stokes.ipynb index cbf84ee4..f4dd9647 100644 --- a/docs/notebooks/04Calculate_variance_Stokes.ipynb +++ b/docs/notebooks/04Calculate_variance_Stokes.ipynb @@ -35,7 +35,7 @@ "warnings.simplefilter(\"ignore\") # Hide warnings to avoid clutter in the notebook\n", "\n", "from dtscalibration import read_silixa_files\n", - "from dtscalibration.dts_accessor import DtsAccessor # noqa: E402\n", + "from dtscalibration.dts_accessor import DtsAccessor # noqa: E402\n", "from dtscalibration.variance_stokes import variance_stokes_constant\n", "from matplotlib import pyplot as plt\n", "\n", diff --git a/docs/notebooks/07Calibrate_single_ended.ipynb b/docs/notebooks/07Calibrate_single_ended.ipynb index 353de5dc..81aeae45 100644 --- a/docs/notebooks/07Calibrate_single_ended.ipynb +++ b/docs/notebooks/07Calibrate_single_ended.ipynb @@ -43,7 +43,7 @@ "import os\n", "\n", "from dtscalibration import read_silixa_files\n", - "from dtscalibration.dts_accessor import DtsAccessor # noqa: E402\n", + "from dtscalibration.dts_accessor import DtsAccessor # noqa: E402\n", "from dtscalibration.variance_stokes import variance_stokes_constant\n", "import matplotlib.pyplot as plt\n", "\n", diff --git a/docs/notebooks/08Calibrate_double_ended.ipynb b/docs/notebooks/08Calibrate_double_ended.ipynb index 1eeecb53..d0f1603d 100644 --- a/docs/notebooks/08Calibrate_double_ended.ipynb +++ b/docs/notebooks/08Calibrate_double_ended.ipynb @@ -45,7 +45,7 @@ " suggest_cable_shift_double_ended,\n", " shift_double_ended,\n", ")\n", - "from dtscalibration.dts_accessor import DtsAccessor # noqa: E402\n", + "from dtscalibration.dts_accessor import DtsAccessor # noqa: E402\n", "from dtscalibration.variance_stokes import variance_stokes_constant\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", @@ -387,13 +387,12 @@ }, "outputs": [], "source": [ - "\n", - "(out2.isel(time=-1).tmpf_mc_var**0.5).plot(figsize=(12, 4))\n", - "(out.isel(time=-1).tmpf_var**0.5).plot()\n", - "(out2.isel(time=-1).tmpb_mc_var**0.5).plot()\n", - "(out.isel(time=-1).tmpb_var**0.5).plot()\n", - "(out.isel(time=-1).tmpw_var**0.5).plot()\n", - "(out2.isel(time=-1).tmpw_mc_var**0.5).plot()\n", + "(out2.isel(time=-1).tmpf_mc_var ** 0.5).plot(figsize=(12, 4))\n", + "(out.isel(time=-1).tmpf_var ** 0.5).plot()\n", + "(out2.isel(time=-1).tmpb_mc_var ** 0.5).plot()\n", + "(out.isel(time=-1).tmpb_var ** 0.5).plot()\n", + "(out.isel(time=-1).tmpw_var ** 0.5).plot()\n", + "(out2.isel(time=-1).tmpw_mc_var ** 0.5).plot()\n", "plt.ylabel(\"$\\sigma$ ($^\\circ$C)\")" ] }, diff --git a/docs/notebooks/12Datastore_from_numpy_arrays.ipynb b/docs/notebooks/12Datastore_from_numpy_arrays.ipynb index e47b93f4..b2b9c702 100644 --- a/docs/notebooks/12Datastore_from_numpy_arrays.ipynb +++ b/docs/notebooks/12Datastore_from_numpy_arrays.ipynb @@ -27,7 +27,7 @@ "import xarray as xr\n", "\n", "from dtscalibration import read_silixa_files\n", - "from dtscalibration.dts_accessor import DtsAccessor # noqa: E402\n", + "from dtscalibration.dts_accessor import DtsAccessor # noqa: E402\n", "from dtscalibration.variance_stokes import variance_stokes_constant" ] }, diff --git a/docs/notebooks/13Fixed_parameter_calibration.ipynb b/docs/notebooks/13Fixed_parameter_calibration.ipynb index 88407620..8f4f3e0f 100644 --- a/docs/notebooks/13Fixed_parameter_calibration.ipynb +++ b/docs/notebooks/13Fixed_parameter_calibration.ipynb @@ -26,7 +26,7 @@ "import os\n", "\n", "from dtscalibration import read_silixa_files\n", - "from dtscalibration.dts_accessor import DtsAccessor # noqa: E402\n", + "from dtscalibration.dts_accessor import DtsAccessor # noqa: E402\n", "from dtscalibration.variance_stokes import variance_stokes_constant\n", "import matplotlib.pyplot as plt\n", "\n", diff --git a/docs/notebooks/14Lossy_splices.ipynb b/docs/notebooks/14Lossy_splices.ipynb index 09759eb0..318d3372 100644 --- a/docs/notebooks/14Lossy_splices.ipynb +++ b/docs/notebooks/14Lossy_splices.ipynb @@ -44,7 +44,7 @@ "import os\n", "\n", "from dtscalibration import read_silixa_files\n", - "from dtscalibration.dts_accessor import DtsAccessor # noqa: E402\n", + "from dtscalibration.dts_accessor import DtsAccessor # noqa: E402\n", "from dtscalibration.variance_stokes import variance_stokes_constant\n", "import matplotlib.pyplot as plt\n", "\n", diff --git a/docs/notebooks/17Temperature_uncertainty_single_ended.ipynb b/docs/notebooks/17Temperature_uncertainty_single_ended.ipynb index 207f2f51..a21be0bd 100644 --- a/docs/notebooks/17Temperature_uncertainty_single_ended.ipynb +++ b/docs/notebooks/17Temperature_uncertainty_single_ended.ipynb @@ -24,7 +24,7 @@ "import os\n", "\n", "from dtscalibration import read_silixa_files\n", - "from dtscalibration.dts_accessor import DtsAccessor # noqa: E402\n", + "from dtscalibration.dts_accessor import DtsAccessor # noqa: E402\n", "from dtscalibration.variance_stokes import variance_stokes_constant\n", "import matplotlib.pyplot as plt\n", "\n", @@ -125,8 +125,12 @@ "metadata": {}, "outputs": [], "source": [ - "out2 = ds.dts.monte_carlo_single_ended(result=out,\n", - " st_var=st_var, ast_var=ast_var, conf_ints=[2.5, 97.5], mc_sample_size=500\n", + "out2 = ds.dts.monte_carlo_single_ended(\n", + " result=out,\n", + " st_var=st_var,\n", + " ast_var=ast_var,\n", + " conf_ints=[2.5, 97.5],\n", + " mc_sample_size=500,\n", ")" ] }, @@ -148,8 +152,10 @@ "source": [ "ds1 = out.isel(time=0)\n", "\n", - "(out2.isel(time=0).tmpf_mc_var**0.5).plot(figsize=(12, 4), label=\"Monte Carlo approx.\")\n", - "(out.isel(time=0).tmpf_var**0.5).plot(label=\"Linear error approx.\")\n", + "(out2.isel(time=0).tmpf_mc_var ** 0.5).plot(\n", + " figsize=(12, 4), label=\"Monte Carlo approx.\"\n", + ")\n", + "(out.isel(time=0).tmpf_var ** 0.5).plot(label=\"Linear error approx.\")\n", "plt.ylabel(\"$\\sigma$ ($^\\circ$C)\")\n", "plt.legend(fontsize=\"small\")" ] diff --git a/src/dtscalibration/__init__.py b/src/dtscalibration/__init__.py index 8f3d2365..5bf2ca5b 100644 --- a/src/dtscalibration/__init__.py +++ b/src/dtscalibration/__init__.py @@ -29,4 +29,4 @@ "plot_residuals_reference_sections", "plot_residuals_reference_sections_single", "plot_sigma_report", -] \ No newline at end of file +] diff --git a/src/dtscalibration/averaging_utils.py b/src/dtscalibration/averaging_utils.py index d17b1f56..662510b5 100644 --- a/src/dtscalibration/averaging_utils.py +++ b/src/dtscalibration/averaging_utils.py @@ -7,7 +7,7 @@ def inverse_variance_weighted_mean( tmpw_store="tmpw", tmpw_var_store="tmpw_var", ): - """Compute inverse variance weighted average, and add result in-place. + """Compute inverse variance weighted average, and add result in-place. Parameters ---------- diff --git a/src/dtscalibration/calibrate_utils.py b/src/dtscalibration/calibrate_utils.py index 6af751dc..e1156157 100644 --- a/src/dtscalibration/calibrate_utils.py +++ b/src/dtscalibration/calibrate_utils.py @@ -34,13 +34,13 @@ def parse_st_var(st, st_var): else: st_var_sec = xr.ones_like(st) * st_var - assert np.all(np.isfinite(st_var_sec)), ( - "NaN/inf values detected in computed st_var. Please check input." - ) + assert np.all( + np.isfinite(st_var_sec) + ), "NaN/inf values detected in computed st_var. Please check input." - assert np.all(st_var_sec > 0.0), ( - "Negative values detected in computed st_var. Please check input." - ) + assert np.all( + st_var_sec > 0.0 + ), "Negative values detected in computed st_var. Please check input." return st_var_sec @@ -76,7 +76,7 @@ def calibration_single_ended_helper( calc_cov=calc_cov, solver="external_split", matching_indices=matching_indices, - trans_att=trans_att + trans_att=trans_att, ) y = split["y"] w = split["w"] @@ -84,12 +84,12 @@ def calibration_single_ended_helper( # Stack all X's if fix_alpha: assert not fix_dalpha, "Use either `fix_dalpha` or `fix_alpha`" - assert fix_alpha[0].size == nx, ( - "fix_alpha also needs to be defined outside the reference sections" - ) - assert fix_alpha[1].size == nx, ( - "fix_alpha also needs to be defined outside the reference sections" - ) + assert ( + fix_alpha[0].size == nx + ), "fix_alpha also needs to be defined outside the reference sections" + assert ( + fix_alpha[1].size == nx + ), "fix_alpha also needs to be defined outside the reference sections" p_val = split["p0_est_alpha"].copy() if np.any(matching_indices): @@ -405,24 +405,16 @@ def calibration_single_ended_solver( # noqa: MC0001 if np.any(matching_indices): st_var_ms0 = ( - parse_st_var(ds.st, st_var) - .isel(x=matching_indices[:, 0]) - .values + parse_st_var(ds.st, st_var).isel(x=matching_indices[:, 0]).values ) st_var_ms1 = ( - parse_st_var(ds.st, st_var) - .isel(x=matching_indices[:, 1]) - .values + parse_st_var(ds.st, st_var).isel(x=matching_indices[:, 1]).values ) ast_var_ms0 = ( - parse_st_var(ds.ast, ast_var) - .isel(x=matching_indices[:, 0]) - .values + parse_st_var(ds.ast, ast_var).isel(x=matching_indices[:, 0]).values ) ast_var_ms1 = ( - parse_st_var(ds.ast, ast_var) - .isel(x=matching_indices[:, 1]) - .values + parse_st_var(ds.ast, ast_var).isel(x=matching_indices[:, 1]).values ) w_ms = ( @@ -517,7 +509,7 @@ def calibrate_double_ended_helper( matching_indices = match_sections(self, matching_sections) else: matching_indices = None - + if fix_alpha or fix_gamma: split = calibrate_double_ended_solver( self, @@ -1213,7 +1205,7 @@ def calibrate_double_ended_solver( # noqa: MC0001 rst_var=rst_var, rast_var=rast_var, ix_alpha_is_zero=ix_alpha_is_zero, - trans_att=trans_att + trans_att=trans_att, ) df_est, db_est = calc_df_db_double_est(ds, sections, ix_alpha_is_zero, 485.0) @@ -1367,18 +1359,10 @@ def calibrate_double_ended_solver( # noqa: MC0001 rst_var_tix = parse_st_var(ds.rst, rst_var).isel(x=tix).values rast_var_tix = parse_st_var(ds.rast, rast_var).isel(x=tix).values - st_var_mnc = ( - parse_st_var(ds.st, st_var).isel(x=ix_match_not_cal).values - ) - ast_var_mnc = ( - parse_st_var(ds.ast, ast_var).isel(x=ix_match_not_cal).values - ) - rst_var_mnc = ( - parse_st_var(ds.rst, rst_var).isel(x=ix_match_not_cal).values - ) - rast_var_mnc = ( - parse_st_var(ds.rast, rast_var).isel(x=ix_match_not_cal).values - ) + st_var_mnc = parse_st_var(ds.st, st_var).isel(x=ix_match_not_cal).values + ast_var_mnc = parse_st_var(ds.ast, ast_var).isel(x=ix_match_not_cal).values + rst_var_mnc = parse_st_var(ds.rst, rst_var).isel(x=ix_match_not_cal).values + rast_var_mnc = parse_st_var(ds.rast, rast_var).isel(x=ix_match_not_cal).values w_eq1 = 1 / ( ( @@ -2174,9 +2158,7 @@ def calc_alpha_double( ta_arr_fw = np.zeros((ds.x.size, ds["time"].size)) ta_arr_fw_var = np.zeros((ds.x.size, ds["time"].size)) - for tai, taxi, tai_var in zip( - talpha_fw.T, trans_att, talpha_fw_var.T - ): + for tai, taxi, tai_var in zip(talpha_fw.T, trans_att, talpha_fw_var.T): ta_arr_fw[ds.x.values >= taxi] = ( ta_arr_fw[ds.x.values >= taxi] + tai ) @@ -2186,9 +2168,7 @@ def calc_alpha_double( ta_arr_bw = np.zeros((ds.x.size, ds["time"].size)) ta_arr_bw_var = np.zeros((ds.x.size, ds["time"].size)) - for tai, taxi, tai_var in zip( - talpha_bw.T, trans_att, talpha_bw_var.T - ): + for tai, taxi, tai_var in zip(talpha_bw.T, trans_att, talpha_bw_var.T): ta_arr_bw[ds.x.values < taxi] = ta_arr_bw[ds.x.values < taxi] + tai ta_arr_bw_var[ds.x.values < taxi] = ( ta_arr_bw_var[ds.x.values < taxi] + tai_var @@ -2289,13 +2269,19 @@ def match_sections(ds, matching_sections): ) hix = ds.dts.ufunc_per_section( - sections={0: [i[0] for i in matching_sections]}, x_indices=True, calc_per="all", suppress_section_validation=True + sections={0: [i[0] for i in matching_sections]}, + x_indices=True, + calc_per="all", + suppress_section_validation=True, ) tixl = [] for _, tslice, reverse_flag in matching_sections: ixi = ds.dts.ufunc_per_section( - sections={0: [tslice]}, x_indices=True, calc_per="all", suppress_section_validation=True + sections={0: [tslice]}, + x_indices=True, + calc_per="all", + suppress_section_validation=True, ) if reverse_flag: diff --git a/src/dtscalibration/calibration/section_utils.py b/src/dtscalibration/calibration/section_utils.py index 4333c8a1..e259739f 100644 --- a/src/dtscalibration/calibration/section_utils.py +++ b/src/dtscalibration/calibration/section_utils.py @@ -5,11 +5,13 @@ from dtscalibration.datastore_utils import ufunc_per_section_helper -def set_sections(ds: xr.Dataset, sections: dict[str, list[slice]] = None): +def set_sections(ds: xr.Dataset, sections: dict[str, list[slice]]): ds.attrs["_sections"] = yaml.dump(sections) -def set_matching_sections(ds: xr.Dataset, matching_sections: dict[str, list[slice]] = None): +def set_matching_sections( + ds: xr.Dataset, matching_sections: dict[str, list[slice]] +): ds.attrs["_matching_sections"] = yaml.dump(matching_sections) @@ -42,7 +44,7 @@ def validate_no_overlapping_sections(sections: dict[str, list[slice]]): all_start_stop = [[stretch.start, stretch.stop] for stretch in all_stretches] isorted_start = np.argsort([i[0] for i in all_start_stop]) all_start_stop_startsort = [all_start_stop[i] for i in isorted_start] - all_start_stop_startsort_flat = sum(all_start_stop_startsort, []) + all_start_stop_startsort_flat = sum(all_start_stop_startsort, []) # type: ignore assert all_start_stop_startsort_flat == sorted( all_start_stop_startsort_flat ), "Sections contains overlapping stretches" @@ -75,9 +77,7 @@ def validate_sections_definition(sections: dict[str, list[slice]]): for k, v in sections.items(): assert isinstance(k, str), ( - "The keys of the " - "sections-dictionary should " - "be strings" + "The keys of the " "sections-dictionary should " "be strings" ) assert isinstance(v, (list, tuple)), ( @@ -115,7 +115,7 @@ def validate_sections(ds: xr.Dataset, sections: dict[str, list[slice]]): """ validate_sections_definition(sections=sections) validate_no_overlapping_sections(sections=sections) - + for k, v in sections.items(): assert k in ds.data_vars, ( "The keys of the " diff --git a/src/dtscalibration/datastore_utils.py b/src/dtscalibration/datastore_utils.py index f8ca0eec..3fe3555a 100644 --- a/src/dtscalibration/datastore_utils.py +++ b/src/dtscalibration/datastore_utils.py @@ -1,5 +1,4 @@ import warnings -from typing import TYPE_CHECKING from typing import Optional from typing import Union diff --git a/src/dtscalibration/dts_accessor.py b/src/dtscalibration/dts_accessor.py index 651e91d3..10489b8a 100644 --- a/src/dtscalibration/dts_accessor.py +++ b/src/dtscalibration/dts_accessor.py @@ -1,15 +1,17 @@ import dask.array as da import numpy as np +import scipy.stats as sst import xarray as xr import yaml -import scipy.stats as sst -from dtscalibration.calibration.section_utils import validate_sections, validate_sections_definition, validate_no_overlapping_sections from dtscalibration.calibrate_utils import calibrate_double_ended_helper from dtscalibration.calibrate_utils import calibration_single_ended_helper from dtscalibration.calibrate_utils import parse_st_var -from dtscalibration.calibration.section_utils import set_sections from dtscalibration.calibration.section_utils import set_matching_sections +from dtscalibration.calibration.section_utils import set_sections +from dtscalibration.calibration.section_utils import validate_no_overlapping_sections +from dtscalibration.calibration.section_utils import validate_sections +from dtscalibration.calibration.section_utils import validate_sections_definition from dtscalibration.datastore_utils import ParameterIndexDoubleEnded from dtscalibration.datastore_utils import ParameterIndexSingleEnded from dtscalibration.datastore_utils import get_params_from_pval_double_ended @@ -34,7 +36,7 @@ def __init__(self, xarray_obj): # None if doesn't exist self.st = xarray_obj.get("st") self.ast = xarray_obj.get("ast") - self.rst = xarray_obj.get("rst") + self.rst = xarray_obj.get("rst") self.rast = xarray_obj.get("rast") self.acquisitiontime_fw = xarray_obj.get("userAcquisitionTimeFW") @@ -129,7 +131,7 @@ def sections(self, value): "ds.dts.calibrate_single_ended() or ds.dts.calibrate_double_ended()." ) raise NotImplementedError(msg) - + # noinspection PyIncorrectDocstring @property def matching_sections(self): @@ -156,7 +158,9 @@ def matching_sections(self): if "_matching_sections" not in self._obj.attrs: self._obj.attrs["_matching_sections"] = yaml.dump(None) - return yaml.load(self._obj.attrs["_matching_sections"], Loader=yaml.UnsafeLoader) + return yaml.load( + self._obj.attrs["_matching_sections"], Loader=yaml.UnsafeLoader + ) @matching_sections.deleter def matching_sections(self): @@ -267,13 +271,13 @@ def get_default_encoding(self, time_chunks_from_key=None): v["chunksizes"] = chunks return encoding - + def get_timeseries_keys(self): """ Returns a list of the keys of the time series variables. """ return [k for k, v in self._obj.data_vars.items() if v.dims == ("time",)] - + def ufunc_per_section( self, sections=None, @@ -409,8 +413,10 @@ def ufunc_per_section( if temp_err or ref_temp_broadcasted: for k in sections: - assert k in self._obj, f"{k} is not in the Dataset but is in `sections` and is required to compute temp_err" - + assert ( + k in self._obj + ), f"{k} is not in the Dataset but is in `sections` and is required to compute temp_err" + if label is None: dataarray = None else: @@ -586,7 +592,9 @@ def calibrate_single_ended( """ # out contains the state - out = xr.Dataset(coords={"x": self.x, "time": self.time, "trans_att": trans_att}).copy() + out = xr.Dataset( + coords={"x": self.x, "time": self.time, "trans_att": trans_att} + ).copy() out.coords["x"].attrs = dim_attrs["x"] out.coords["trans_att"].attrs = dim_attrs["trans_att"] @@ -677,8 +685,7 @@ def calibrate_single_ended( var_fw_dict = dict( dT_dst=deriv_ds.T_st_fw**2 * parse_st_var(self.st, st_var), - dT_dast=deriv_ds.T_ast_fw**2 - * parse_st_var(self.ast, ast_var), + dT_dast=deriv_ds.T_ast_fw**2 * parse_st_var(self.ast, ast_var), dT_gamma=deriv_ds.T_gamma_fw**2 * param_covs["gamma"], dT_dc=deriv_ds.T_c_fw**2 * param_covs["c"], dT_ddalpha=deriv_ds.T_alpha_fw**2 @@ -730,7 +737,6 @@ def calibrate_single_ended( out[key + "_var"] = dataarray return out - def calibrate_double_ended( self, @@ -953,7 +959,9 @@ def calibrate_double_ended( 08Calibrate_double_wls.ipynb>`_ """ # out contains the state - out = xr.Dataset(coords={"x": self.x, "time": self.time, "trans_att": trans_att}).copy() + out = xr.Dataset( + coords={"x": self.x, "time": self.time, "trans_att": trans_att} + ).copy() out.coords["x"].attrs = dim_attrs["x"] out.coords["trans_att"].attrs = dim_attrs["trans_att"] @@ -988,7 +996,7 @@ def calibrate_double_ended( assert not np.any(self.rast.isel(x=ix_sec) <= 0.0), ( "There is uncontrolled noise in the REV-AST signal. Are your " "sections correctly defined?" - ) + ) if method == "wls": p_cov, p_val, p_var = calibrate_double_ended_helper( @@ -1034,17 +1042,17 @@ def calibrate_double_ended( ) tmpf = params["gamma"] / ( - np.log(self.st / self.ast) - + params["df"] - + params["alpha"] - + params["talpha_fw_full"] - ) + np.log(self.st / self.ast) + + params["df"] + + params["alpha"] + + params["talpha_fw_full"] + ) tmpb = params["gamma"] / ( - np.log(self.rst / self.rast) - + params["db"] - - params["alpha"] - + params["talpha_bw_full"] - ) + np.log(self.rst / self.rast) + + params["db"] + - params["alpha"] + + params["talpha_bw_full"] + ) out["tmpf"] = tmpf - 273.15 out["tmpb"] = tmpb - 273.15 @@ -1067,8 +1075,7 @@ def calibrate_double_ended( var_fw_dict = dict( dT_dst=deriv_ds.T_st_fw**2 * parse_st_var(self.st, st_var), - dT_dast=deriv_ds.T_ast_fw**2 - * parse_st_var(self.ast, ast_var), + dT_dast=deriv_ds.T_ast_fw**2 * parse_st_var(self.ast, ast_var), dT_gamma=deriv_ds.T_gamma_fw**2 * param_covs["gamma"], dT_ddf=deriv_ds.T_df_fw**2 * param_covs["df"], dT_dalpha=deriv_ds.T_alpha_fw**2 * param_covs["alpha"], @@ -1094,10 +1101,8 @@ def calibrate_double_ended( ), ) var_bw_dict = dict( - dT_drst=deriv_ds.T_rst_bw**2 - * parse_st_var(self.rst, rst_var), - dT_drast=deriv_ds.T_rast_bw**2 - * parse_st_var(self.rast, rast_var), + dT_drst=deriv_ds.T_rst_bw**2 * parse_st_var(self.rst, rst_var), + dT_drast=deriv_ds.T_rast_bw**2 * parse_st_var(self.rast, rast_var), dT_gamma=deriv_ds.T_gamma_bw**2 * param_covs["gamma"], dT_ddb=deriv_ds.T_db_bw**2 * param_covs["db"], dT_dalpha=deriv_ds.T_alpha_bw**2 * param_covs["alpha"], @@ -1158,12 +1163,9 @@ def calibrate_double_ended( # TODO: sigma2_tafw_tabw var_w_dict = dict( dT_dst=deriv_ds2.T_st_w**2 * parse_st_var(self.st, st_var), - dT_dast=deriv_ds2.T_ast_w**2 - * parse_st_var(self.ast, ast_var), - dT_drst=deriv_ds2.T_rst_w**2 - * parse_st_var(self.rst, rst_var), - dT_drast=deriv_ds2.T_rast_w**2 - * parse_st_var(self.rast, rast_var), + dT_dast=deriv_ds2.T_ast_w**2 * parse_st_var(self.ast, ast_var), + dT_drst=deriv_ds2.T_rst_w**2 * parse_st_var(self.rst, rst_var), + dT_drast=deriv_ds2.T_rast_w**2 * parse_st_var(self.rast, rast_var), dT_gamma=deriv_ds2.T_gamma_w**2 * param_covs["gamma"], dT_ddf=deriv_ds2.T_df_w**2 * param_covs["df"], dT_ddb=deriv_ds2.T_db_w**2 * param_covs["db"], @@ -1244,9 +1246,10 @@ def monte_carlo_single_ended( mc_sample_size=100, da_random_state=None, reduce_memory_usage=False, - mc_remove_set_flag=True): + mc_remove_set_flag=True, + ): """The result object is what comes out of the single_ended_calibration routine) - + TODO: Use get_params_from_pval_single_ended() to extract parameter sets from mc """ assert self.st.dims[0] == "x", "Stokes are transposed" @@ -1258,7 +1261,9 @@ def monte_carlo_single_ended( state = da.random.RandomState() # out contains the state - out = xr.Dataset(coords={"x": self.x, "time": self.time, "trans_att": result["trans_att"]}).copy() + out = xr.Dataset( + coords={"x": self.x, "time": self.time, "trans_att": result["trans_att"]} + ).copy() out.coords["x"].attrs = dim_attrs["x"] out.coords["trans_att"].attrs = dim_attrs["trans_att"] out.coords["CI"] = conf_ints @@ -1312,10 +1317,10 @@ def monte_carlo_single_ended( (mc_sample_size, no, nt), chunks={0: -1, 1: "auto", 2: "auto"} ).chunks - # Draw from the normal distributions for the Stokes intensities - for key_mc, sti, st_vari in zip(["r_st", "r_ast"], [self.st, self.ast], - [st_var, ast_var]): + for key_mc, sti, st_vari in zip( + ["r_st", "r_ast"], [self.st, self.ast], [st_var, ast_var] + ): # Load the mean as chunked Dask array, otherwise eats memory if type(sti.data) == da.core.Array: loc = da.asarray(sti.data, chunks=memchunk[1:]) @@ -1336,14 +1341,10 @@ def monte_carlo_single_ended( st_vari_da = da.asarray(st_vari, chunks=memchunk[1:]) elif callable(st_vari) and type(sti.data) == da.core.Array: - st_vari_da = da.asarray( - st_vari(sti).data, chunks=memchunk[1:] - ) + st_vari_da = da.asarray(st_vari(sti).data, chunks=memchunk[1:]) elif callable(st_vari) and type(sti.data) != da.core.Array: - st_vari_da = da.from_array( - st_vari(sti).data, chunks=memchunk[1:] - ) + st_vari_da = da.from_array(st_vari(sti).data, chunks=memchunk[1:]) else: st_vari_da = da.from_array(st_vari, chunks=memchunk[1:]) @@ -1433,7 +1434,7 @@ def monte_carlo_double_ended( exclude_parameter_uncertainty=False, da_random_state=None, mc_remove_set_flag=True, - reduce_memory_usage=False + reduce_memory_usage=False, ): r""" Estimation of the confidence intervals for the temperatures measured @@ -1594,7 +1595,9 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): else: state = da.random.RandomState() - out = xr.Dataset(coords={"x": self.x, "time": self.time, "trans_att": result["trans_att"]}).copy() + out = xr.Dataset( + coords={"x": self.x, "time": self.time, "trans_att": result["trans_att"]} + ).copy() out.coords["x"].attrs = dim_attrs["x"] out.coords["trans_att"].attrs = dim_attrs["trans_att"] out.coords["CI"] = conf_ints @@ -1766,14 +1769,10 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): st_vari_da = da.asarray(st_vari, chunks=memchunk[1:]) elif callable(st_vari) and type(sti.data) == da.core.Array: - st_vari_da = da.asarray( - st_vari(sti).data, chunks=memchunk[1:] - ) + st_vari_da = da.asarray(st_vari(sti).data, chunks=memchunk[1:]) elif callable(st_vari) and type(sti.data) != da.core.Array: - st_vari_da = da.from_array( - st_vari(sti).data, chunks=memchunk[1:] - ) + st_vari_da = da.from_array(st_vari(sti).data, chunks=memchunk[1:]) else: st_vari_da = da.from_array(st_vari, chunks=memchunk[1:]) @@ -1839,9 +1838,7 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): xi = self.ufunc_per_section( sections=sections, x_indices=True, calc_per="all" ) - x_mask_ = [ - True if ix in xi else False for ix in range(params.x.size) - ] + x_mask_ = [True if ix in xi else False for ix in range(params.x.size)] x_mask = np.reshape(x_mask_, (1, -1, 1)) params[label + "_mc_set"] = params[label + "_mc_set"].where(x_mask) @@ -1921,7 +1918,7 @@ def create_da_ta2(no, i_splice, direction="fw", chunks=None): out.update(params) return out - + def average_monte_carlo_single_ended( self, result, @@ -2048,9 +2045,11 @@ def average_monte_carlo_single_ended( Returns ------- - """ + """ # out contains the state - out = xr.Dataset(coords={"x": self.x, "time": self.time, "trans_att": result["trans_att"]}).copy() + out = xr.Dataset( + coords={"x": self.x, "time": self.time, "trans_att": result["trans_att"]} + ).copy() out.coords["x"].attrs = dim_attrs["x"] out.coords["trans_att"].attrs = dim_attrs["trans_att"] out.coords["CI"] = conf_ints @@ -2419,7 +2418,9 @@ def average_monte_carlo_double_ended( # ).rechunk((1, chunks[1], 1)) # return arr - out = xr.Dataset(coords={"x": self.x, "time": self.time, "trans_att": result["trans_att"]}).copy() + out = xr.Dataset( + coords={"x": self.x, "time": self.time, "trans_att": result["trans_att"]} + ).copy() out.coords["x"].attrs = dim_attrs["x"] out.coords["trans_att"].attrs = dim_attrs["trans_att"] out.coords["CI"] = conf_ints diff --git a/src/dtscalibration/variance_helpers.py b/src/dtscalibration/variance_helpers.py index 6298810a..6420e5a5 100644 --- a/src/dtscalibration/variance_helpers.py +++ b/src/dtscalibration/variance_helpers.py @@ -190,7 +190,7 @@ def check_allclose_acquisitiontime(acquisitiontime, eps: float = 0.05) -> None: dtmin = acquisitiontime.min() dtmax = acquisitiontime.max() dtavg = (dtmin + dtmax) / 2 - assert (dtmax - dtmin) / dtavg < eps, ( - "Acquisition time is Forward channel not equal for all time steps" - ) + assert ( + dtmax - dtmin + ) / dtavg < eps, "Acquisition time is Forward channel not equal for all time steps" pass diff --git a/src/dtscalibration/variance_stokes.py b/src/dtscalibration/variance_stokes.py index 7be69274..0d37db1b 100644 --- a/src/dtscalibration/variance_stokes.py +++ b/src/dtscalibration/variance_stokes.py @@ -2,13 +2,13 @@ import numpy as np import xarray as xr +from dtscalibration.calibration.section_utils import validate_no_overlapping_sections +from dtscalibration.calibration.section_utils import validate_sections_definition from dtscalibration.datastore_utils import ufunc_per_section_helper +from dtscalibration.variance_helpers import check_allclose_acquisitiontime from dtscalibration.variance_helpers import variance_stokes_constant_helper from dtscalibration.variance_helpers import variance_stokes_exponential_helper from dtscalibration.variance_helpers import variance_stokes_linear_helper -from dtscalibration.calibration.section_utils import validate_sections_definition -from dtscalibration.calibration.section_utils import validate_no_overlapping_sections -from dtscalibration.variance_helpers import check_allclose_acquisitiontime def variance_stokes_constant(st, sections, acquisitiontime, reshape_residuals=True): @@ -124,11 +124,7 @@ def variance_stokes_constant(st, sections, acquisitiontime, reshape_residuals=Tr # should maybe be per section. But then residuals # seem to be correlated between stretches. I don't know why.. BdT. data_dict = da.compute( - ufunc_per_section_helper( - sections=sections, - dataarray=st, - calc_per="stretch" - ) + ufunc_per_section_helper(sections=sections, dataarray=st, calc_per="stretch") )[0] var_I, resid = variance_stokes_constant_helper(data_dict) @@ -331,12 +327,16 @@ def variance_stokes_exponential( # _resid_x = self.ufunc_per_section( # sections=sections, label="x", calc_per="all" # ) - _resid_x = ufunc_per_section_helper(sections=sections, dataarray=st.coords["x"], calc_per="all") + _resid_x = ufunc_per_section_helper( + sections=sections, dataarray=st.coords["x"], calc_per="all" + ) isort = np.argsort(_resid_x) resid_x = _resid_x[isort] # get indices from ufunc directly resid = _resid[isort, :] - ix_resid = np.array([np.argmin(np.abs(ai - st.coords["x"].data)) for ai in resid_x]) + ix_resid = np.array( + [np.argmin(np.abs(ai - st.coords["x"].data)) for ai in resid_x] + ) resid_sorted = np.full(shape=st.shape, fill_value=np.nan) resid_sorted[ix_resid, :] = resid @@ -465,9 +465,14 @@ def variance_stokes_linear( assert st.dims[0] == "x", "Stokes are transposed" _, resid = variance_stokes_constant( - sections=sections, st=st, acquisitiontime=acquisitiontime, reshape_residuals=False + sections=sections, + st=st, + acquisitiontime=acquisitiontime, + reshape_residuals=False, + ) + ix_sec = ufunc_per_section_helper( + sections=sections, x_coords=st.coords["x"], calc_per="all" ) - ix_sec = ufunc_per_section_helper(sections=sections, x_coords=st.coords["x"], calc_per="all") st = st.isel(x=ix_sec).values.ravel() diff_st = resid.ravel() @@ -483,6 +488,7 @@ def variance_stokes_linear( if plot_fit: import matplotlib.pyplot as plt + plt.figure() plt.scatter(st_sort_mean, st_sort_var, marker=".", c="black") plt.plot( diff --git a/tests/test_averaging.py b/tests/test_averaging.py index 6276db63..36ecf408 100644 --- a/tests/test_averaging.py +++ b/tests/test_averaging.py @@ -1,6 +1,7 @@ import os import numpy as np + from dtscalibration import read_silixa_files from dtscalibration.dts_accessor import DtsAccessor # noqa: F401 @@ -71,10 +72,12 @@ def test_average_measurements_single_ended(): ci_avg_x_flag1=True, ci_avg_x_sel=slice(6.0, 14.0), ) + def get_section_indices(x_da, sec): """Returns the x-indices of the section. `sec` is a slice.""" xis = x_da.astype(int) * 0 + np.arange(x_da.size, dtype=int) return xis.sel(x=sec).values + ix = get_section_indices(ds.x, slice(6, 14)) ds.dts.average_monte_carlo_single_ended( result=out, @@ -145,10 +148,12 @@ def test_average_measurements_double_ended(): ci_avg_x_flag1=True, ci_avg_x_sel=slice(6, 10), ) + def get_section_indices(x_da, sec): """Returns the x-indices of the section. `sec` is a slice.""" xis = x_da.astype(int) * 0 + np.arange(x_da.size, dtype=int) return xis.sel(x=sec).values + ix = get_section_indices(ds.x, slice(6, 10)) ds.dts.average_monte_carlo_double_ended( result=out, diff --git a/tests/test_datastore.py b/tests/test_datastore.py index bad2d30c..03fa60f3 100644 --- a/tests/test_datastore.py +++ b/tests/test_datastore.py @@ -1,26 +1,23 @@ import hashlib import os import tempfile -import time import warnings from zipfile import ZipFile import dask.array as da import numpy as np import pytest -from xarray import Dataset import xarray as xr +from xarray import Dataset from dtscalibration import read_apsensing_files from dtscalibration import read_sensornet_files from dtscalibration import read_sensortran_files from dtscalibration import read_silixa_files from dtscalibration.calibration.section_utils import set_sections -from dtscalibration.calibration.section_utils import set_matching_sections from dtscalibration.datastore_utils import merge_double_ended from dtscalibration.datastore_utils import shift_double_ended from dtscalibration.datastore_utils import suggest_cable_shift_double_ended - from dtscalibration.dts_accessor import DtsAccessor # noqa: F401 np.random.seed(0) diff --git a/tests/test_dtscalibration.py b/tests/test_dtscalibration.py index 61d69b2e..ec951835 100644 --- a/tests/test_dtscalibration.py +++ b/tests/test_dtscalibration.py @@ -6,6 +6,7 @@ import scipy.sparse as sp from scipy import stats from xarray import Dataset + from dtscalibration.calibrate_utils import wls_sparse from dtscalibration.calibrate_utils import wls_stats from dtscalibration.dts_accessor import DtsAccessor # noqa: F401 @@ -260,7 +261,9 @@ def test_double_ended_wls_estimate_synthetic_df_and_db_are_different(): assert_almost_equal_verbose(df, out["df"].values, decimal=14) assert_almost_equal_verbose(db, out["db"].values, decimal=13) assert_almost_equal_verbose( - x * (dalpha_p - dalpha_m), out["alpha"].values - out["alpha"].values[0], decimal=13 + x * (dalpha_p - dalpha_m), + out["alpha"].values - out["alpha"].values[0], + decimal=13, ) assert np.all(np.abs(real_ans2 - out["p_val"].values) < 1e-10) assert_almost_equal_verbose(temp_real_celsius, out["tmpf"].values, decimal=10) @@ -1476,7 +1479,7 @@ def test_double_ended_exponential_variance_estimate_synthetic(): rast_label = "rast" # MC variance - out = ds.dts.calibrate_double_ended( + ds.dts.calibrate_double_ended( sections=sections, st_label=st_label, ast_label=ast_label, @@ -1847,12 +1850,8 @@ def test_single_ended_wls_fix_dalpha_synthetic(): fix_dalpha=(dalpha_p - dalpha_m, 0.0), ) assert_almost_equal_verbose(out.gamma.values, gamma, decimal=12) - assert_almost_equal_verbose( - out.dalpha.values, dalpha_p - dalpha_m, decimal=14 - ) - assert_almost_equal_verbose( - out.alpha.values, x * (dalpha_p - dalpha_m), decimal=14 - ) + assert_almost_equal_verbose(out.dalpha.values, dalpha_p - dalpha_m, decimal=14) + assert_almost_equal_verbose(out.alpha.values, x * (dalpha_p - dalpha_m), decimal=14) assert_almost_equal_verbose(out.tmpf.values, temp_real - 273.15, decimal=10) # Test fix_alpha @@ -1867,9 +1866,7 @@ def test_single_ended_wls_fix_dalpha_synthetic(): ) assert_almost_equal_verbose(out.gamma.values, gamma, decimal=12) - assert_almost_equal_verbose( - out.alpha.values, x * (dalpha_p - dalpha_m), decimal=14 - ) + assert_almost_equal_verbose(out.alpha.values, x * (dalpha_p - dalpha_m), decimal=14) assert_almost_equal_verbose(out.tmpf.values, temp_real - 273.15, decimal=10) pass @@ -2114,7 +2111,7 @@ def test_single_ended_trans_att_synthetic(): st_var=1.0, ast_var=1.0, method="wls", - trans_att=[40., 60.], + trans_att=[40.0, 60.0], solver="sparse", ) @@ -2136,7 +2133,7 @@ def test_single_ended_trans_att_synthetic(): ast_var=1.0, method="wls", fix_gamma=(482.6, 0), - trans_att=[40., 60.], + trans_att=[40.0, 60.0], solver="sparse", ) @@ -2483,12 +2480,10 @@ def test_single_ended_exponential_variance_estimate_synthetic(): conf_ints=[2.5, 50.0, 97.5], mc_sample_size=50, da_random_state=state, - mc_remove_set_flag=False + mc_remove_set_flag=False, ) ds2.update(out_ci) - - # Use a single timestep to better check if the parameter uncertainties # propagate # Estimated VAR diff --git a/tests/test_variance_stokes.py b/tests/test_variance_stokes.py index 9a3344cb..256ef986 100644 --- a/tests/test_variance_stokes.py +++ b/tests/test_variance_stokes.py @@ -3,13 +3,12 @@ import numpy as np import pytest from scipy import stats -import xarray as xr from xarray import Dataset from dtscalibration import read_silixa_files from dtscalibration.dts_accessor import DtsAccessor # noqa: F401 -from dtscalibration.variance_stokes import variance_stokes_exponential from dtscalibration.variance_stokes import variance_stokes_constant +from dtscalibration.variance_stokes import variance_stokes_exponential from dtscalibration.variance_stokes import variance_stokes_linear np.random.seed(0) @@ -56,6 +55,7 @@ def assert_almost_equal_verbose(actual, desired, verbose=False, **kwargs): np.testing.assert_almost_equal(actual, desired2, err_msg=m, **kwargs) pass + @pytest.mark.slow # Execution time ~20 seconds def test_variance_input_types_single(): import dask.array as da @@ -179,8 +179,12 @@ def callable_st_var(stokes): sections=sections, st_var=st_var, ast_var=st_var, method="wls", solver="sparse" ) - out2 = ds.dts.monte_carlo_single_ended(result=out, - st_var=st_var, ast_var=st_var, mc_sample_size=100, da_random_state=state + out2 = ds.dts.monte_carlo_single_ended( + result=out, + st_var=st_var, + ast_var=st_var, + mc_sample_size=100, + da_random_state=state, ) assert_almost_equal_verbose(out2["tmpf_mc_var"].mean(), 0.418098, decimal=2) @@ -194,8 +198,12 @@ def callable_st_var(stokes): sections=sections, st_var=st_var, ast_var=st_var, method="wls", solver="sparse" ) - out2 = ds.dts.monte_carlo_single_ended(result=out, - st_var=st_var, ast_var=st_var, mc_sample_size=100, da_random_state=state + out2 = ds.dts.monte_carlo_single_ended( + result=out, + st_var=st_var, + ast_var=st_var, + mc_sample_size=100, + da_random_state=state, ) assert_almost_equal_verbose( @@ -212,15 +220,21 @@ def callable_st_var(stokes): sections=sections, st_var=st_var, ast_var=st_var, method="wls", solver="sparse" ) - out2 = ds.dts.monte_carlo_single_ended(result=out, - st_var=st_var, ast_var=st_var, mc_sample_size=100, da_random_state=state + out2 = ds.dts.monte_carlo_single_ended( + result=out, + st_var=st_var, + ast_var=st_var, + mc_sample_size=100, + da_random_state=state, ) assert_almost_equal_verbose( out2["tmpf_mc_var"].sel(time=slice(0, nt // 2)).mean().values, 1.0908, decimal=2 ) assert_almost_equal_verbose( - out2["tmpf_mc_var"].sel(time=slice(nt // 2, None)).mean().values, 3.0759, decimal=2 + out2["tmpf_mc_var"].sel(time=slice(nt // 2, None)).mean().values, + 3.0759, + decimal=2, ) pass @@ -279,10 +293,18 @@ def test_variance_input_types_double(): / (1 - np.exp(-gamma / temp_real)) ) - st_m = st + stats.norm.rvs(size=st.shape, scale=stokes_m_var**0.5, random_state=state) - ast_m = ast + stats.norm.rvs(size=ast.shape, scale=1.1 * stokes_m_var**0.5, random_state=state) - rst_m = rst + stats.norm.rvs(size=rst.shape, scale=0.9 * stokes_m_var**0.5, random_state=state) - rast_m = rast + stats.norm.rvs(size=rast.shape, scale=0.8 * stokes_m_var**0.5, random_state=state) + st_m = st + stats.norm.rvs( + size=st.shape, scale=stokes_m_var**0.5, random_state=state + ) + ast_m = ast + stats.norm.rvs( + size=ast.shape, scale=1.1 * stokes_m_var**0.5, random_state=state + ) + rst_m = rst + stats.norm.rvs( + size=rst.shape, scale=0.9 * stokes_m_var**0.5, random_state=state + ) + rast_m = rast + stats.norm.rvs( + size=rast.shape, scale=0.8 * stokes_m_var**0.5, random_state=state + ) print("alphaint", cable_len * (dalpha_p - dalpha_m)) print("alpha", dalpha_p - dalpha_m) @@ -461,7 +483,9 @@ def st_var_callable(stokes): out2["tmpf_mc_var"].sel(time=slice(0, nt // 2)).mean().values, 1.090, decimal=2 ) assert_almost_equal_verbose( - out2["tmpf_mc_var"].sel(time=slice(nt // 2, None)).mean().values, 3.06, decimal=2 + out2["tmpf_mc_var"].sel(time=slice(nt // 2, None)).mean().values, + 3.06, + decimal=2, ) pass @@ -550,10 +574,18 @@ def test_double_ended_variance_estimate_synthetic(): "warm": [slice(0.5 * cable_len, cable_len)], } - mst_var, _ = variance_stokes_constant(ds.dts.st, sections, ds.dts.acquisitiontime_fw, reshape_residuals=False) - mast_var, _ = variance_stokes_constant(ds.dts.ast, sections, ds.dts.acquisitiontime_fw, reshape_residuals=False) - mrst_var, _ = variance_stokes_constant(ds.dts.rst, sections, ds.dts.acquisitiontime_bw, reshape_residuals=False) - mrast_var, _ = variance_stokes_constant(ds.dts.rast, sections, ds.dts.acquisitiontime_bw, reshape_residuals=False) + mst_var, _ = variance_stokes_constant( + ds.dts.st, sections, ds.dts.acquisitiontime_fw, reshape_residuals=False + ) + mast_var, _ = variance_stokes_constant( + ds.dts.ast, sections, ds.dts.acquisitiontime_fw, reshape_residuals=False + ) + mrst_var, _ = variance_stokes_constant( + ds.dts.rst, sections, ds.dts.acquisitiontime_bw, reshape_residuals=False + ) + mrast_var, _ = variance_stokes_constant( + ds.dts.rast, sections, ds.dts.acquisitiontime_bw, reshape_residuals=False + ) mst_var = float(mst_var) mast_var = float(mast_var) @@ -575,13 +607,23 @@ def test_double_ended_variance_estimate_synthetic(): assert_almost_equal_verbose(out["tmpf"].mean(), 12.0, decimal=2) assert_almost_equal_verbose(out["tmpb"].mean(), 12.0, decimal=3) - + # Calibrated variance stdsf1 = out.dts.ufunc_per_section( - sections=sections, label="tmpf", func=np.std, temp_err=True, calc_per="stretch", suppress_section_validation=True, + sections=sections, + label="tmpf", + func=np.std, + temp_err=True, + calc_per="stretch", + suppress_section_validation=True, ) stdsb1 = out.dts.ufunc_per_section( - sections=sections, label="tmpb", func=np.std, temp_err=True, calc_per="stretch", suppress_section_validation=True, + sections=sections, + label="tmpb", + func=np.std, + temp_err=True, + calc_per="stretch", + suppress_section_validation=True, ) out2 = ds.dts.monte_carlo_double_ended( @@ -695,8 +737,12 @@ def test_single_ended_variance_estimate_synthetic(): "warm": [slice(0.5 * cable_len, cable_len)], } - mst_var, _ = variance_stokes_constant(ds.dts.st, sections, ds.dts.acquisitiontime_fw, reshape_residuals=False) - mast_var, _ = variance_stokes_constant(ds.dts.ast, sections, ds.dts.acquisitiontime_fw, reshape_residuals=False) + mst_var, _ = variance_stokes_constant( + ds.dts.st, sections, ds.dts.acquisitiontime_fw, reshape_residuals=False + ) + mast_var, _ = variance_stokes_constant( + ds.dts.ast, sections, ds.dts.acquisitiontime_fw, reshape_residuals=False + ) # mrst_var, _ = variance_stokes_constant(ds.dts.rst, sections, ds.dts.acquisitiontime_bw, reshape_residuals=False) # mrast_var, _ = variance_stokes_constant(ds.dts.rast, sections, ds.dts.acquisitiontime_bw, reshape_residuals=False) mst_var = float(mst_var) @@ -753,7 +799,6 @@ def test_single_ended_variance_estimate_synthetic(): pass - @pytest.mark.skip(reason="Not enough measurements in time. Use exponential instead.") def test_variance_of_stokes(): correct_var = 9.045 @@ -806,7 +851,9 @@ def test_variance_of_stokes_synthetic(): ) sections = {"probe1Temperature": [slice(0.0, 20.0)]} - test_st_var, _ = variance_stokes_constant(ds.dts.st, sections, ds.dts.acquisitiontime_fw, reshape_residuals=False) + test_st_var, _ = variance_stokes_constant( + ds.dts.st, sections, ds.dts.acquisitiontime_fw, reshape_residuals=False + ) assert_almost_equal_verbose(test_st_var, yvar, decimal=1) pass @@ -848,7 +895,9 @@ def test_variance_of_stokes_linear_synthetic(): ) sections = {"probe1Temperature": [slice(0.0, 20.0)]} - test_st_var, _ = variance_stokes_constant(ds.dts.st, sections, ds.dts.acquisitiontime_fw, reshape_residuals=False) + test_st_var, _ = variance_stokes_constant( + ds.dts.st, sections, ds.dts.acquisitiontime_fw, reshape_residuals=False + ) # If fit is forced through zero. Only Poisson distributed noise ( @@ -877,7 +926,11 @@ def test_variance_of_stokes_linear_synthetic(): resid, var_fun, ) = variance_stokes_linear( - st=ds["c_lin_var_through_zero"], sections=sections, acquisitiontime=ds.dts.acquisitiontime_fw, nbin=100, through_zero=False + st=ds["c_lin_var_through_zero"], + sections=sections, + acquisitiontime=ds.dts.acquisitiontime_fw, + nbin=100, + through_zero=False, ) assert_almost_equal_verbose(slope, var_slope, decimal=3) assert_almost_equal_verbose(offset, 0.0, decimal=0) @@ -895,11 +948,15 @@ def test_exponential_variance_of_stokes(): "probe2Temperature": [slice(24.0, 34.0), slice(85.0, 95.0)], # warm bath } - I_var, _ = variance_stokes_exponential(st=ds["st"], sections=sections, acquisitiontime=ds.dts.acquisitiontime_fw) + I_var, _ = variance_stokes_exponential( + st=ds["st"], sections=sections, acquisitiontime=ds.dts.acquisitiontime_fw + ) assert_almost_equal_verbose(I_var, correct_var, decimal=5) ds_dask = ds.chunk(chunks={}) - I_var, _ = variance_stokes_exponential(st=ds_dask["st"], sections=sections, acquisitiontime=ds.dts.acquisitiontime_fw) + I_var, _ = variance_stokes_exponential( + st=ds_dask["st"], sections=sections, acquisitiontime=ds.dts.acquisitiontime_fw + ) assert_almost_equal_verbose(I_var, correct_var, decimal=5) pass @@ -937,7 +994,9 @@ def test_exponential_variance_of_stokes_synthetic(): ) sections = {"probe1Temperature": [slice(0.0, 20.0)]} - test_st_var, _ = variance_stokes_exponential(st=ds["st"], sections=sections, acquisitiontime=ds.dts.acquisitiontime_fw) + test_st_var, _ = variance_stokes_exponential( + st=ds["st"], sections=sections, acquisitiontime=ds.dts.acquisitiontime_fw + ) assert_almost_equal_verbose(test_st_var, yvar, decimal=1) pass From 635d29dd53e1cf2d203081bc600203d132fffa21 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Wed, 18 Oct 2023 21:16:17 +0200 Subject: [PATCH 47/66] Docs cleanup --- docs/api/dtscalibration.DataStore.rst | 7 ------- docs/api/dtscalibration.check_dims.rst | 6 ------ docs/api/dtscalibration.check_timestep_allclose.rst | 6 ------ docs/api/dtscalibration.get_netcdf_encoding.rst | 6 ------ docs/api/dtscalibration.merge_double_ended.rst | 6 ------ docs/api/dtscalibration.open_datastore.rst | 6 ------ docs/api/dtscalibration.open_mf_datastore.rst | 6 ------ docs/api/dtscalibration.plot_accuracy.rst | 6 ------ ...tscalibration.plot_location_residuals_double_ended.rst | 6 ------ .../dtscalibration.plot_residuals_reference_sections.rst | 6 ------ ...libration.plot_residuals_reference_sections_single.rst | 6 ------ docs/api/dtscalibration.plot_sigma_report.rst | 6 ------ docs/api/dtscalibration.read_apsensing_files.rst | 6 ------ docs/api/dtscalibration.read_sensornet_files.rst | 6 ------ docs/api/dtscalibration.read_sensortran_files.rst | 6 ------ docs/api/dtscalibration.read_silixa_files.rst | 6 ------ docs/api/dtscalibration.shift_double_ended.rst | 6 ------ .../dtscalibration.suggest_cable_shift_double_ended.rst | 6 ------ docs/notebooks/01Load_xml_measurement_files.ipynb | 2 +- ..._DataStore_functions_slice_mean_max_std_resample.ipynb | 8 +++----- 20 files changed, 4 insertions(+), 115 deletions(-) delete mode 100644 docs/api/dtscalibration.DataStore.rst delete mode 100644 docs/api/dtscalibration.check_dims.rst delete mode 100644 docs/api/dtscalibration.check_timestep_allclose.rst delete mode 100644 docs/api/dtscalibration.get_netcdf_encoding.rst delete mode 100644 docs/api/dtscalibration.merge_double_ended.rst delete mode 100644 docs/api/dtscalibration.open_datastore.rst delete mode 100644 docs/api/dtscalibration.open_mf_datastore.rst delete mode 100644 docs/api/dtscalibration.plot_accuracy.rst delete mode 100644 docs/api/dtscalibration.plot_location_residuals_double_ended.rst delete mode 100644 docs/api/dtscalibration.plot_residuals_reference_sections.rst delete mode 100644 docs/api/dtscalibration.plot_residuals_reference_sections_single.rst delete mode 100644 docs/api/dtscalibration.plot_sigma_report.rst delete mode 100644 docs/api/dtscalibration.read_apsensing_files.rst delete mode 100644 docs/api/dtscalibration.read_sensornet_files.rst delete mode 100644 docs/api/dtscalibration.read_sensortran_files.rst delete mode 100644 docs/api/dtscalibration.read_silixa_files.rst delete mode 100644 docs/api/dtscalibration.shift_double_ended.rst delete mode 100644 docs/api/dtscalibration.suggest_cable_shift_double_ended.rst diff --git a/docs/api/dtscalibration.DataStore.rst b/docs/api/dtscalibration.DataStore.rst deleted file mode 100644 index 8ed756d9..00000000 --- a/docs/api/dtscalibration.DataStore.rst +++ /dev/null @@ -1,7 +0,0 @@ -DataStore -========= - -.. currentmodule:: dtscalibration - -.. autoclass:: DataStore - :show-inheritance: diff --git a/docs/api/dtscalibration.check_dims.rst b/docs/api/dtscalibration.check_dims.rst deleted file mode 100644 index 1dc68e7e..00000000 --- a/docs/api/dtscalibration.check_dims.rst +++ /dev/null @@ -1,6 +0,0 @@ -check_dims -========== - -.. currentmodule:: dtscalibration - -.. autofunction:: check_dims diff --git a/docs/api/dtscalibration.check_timestep_allclose.rst b/docs/api/dtscalibration.check_timestep_allclose.rst deleted file mode 100644 index d655bfe4..00000000 --- a/docs/api/dtscalibration.check_timestep_allclose.rst +++ /dev/null @@ -1,6 +0,0 @@ -check_timestep_allclose -======================= - -.. currentmodule:: dtscalibration - -.. autofunction:: check_timestep_allclose diff --git a/docs/api/dtscalibration.get_netcdf_encoding.rst b/docs/api/dtscalibration.get_netcdf_encoding.rst deleted file mode 100644 index 820b0a0f..00000000 --- a/docs/api/dtscalibration.get_netcdf_encoding.rst +++ /dev/null @@ -1,6 +0,0 @@ -get_netcdf_encoding -=================== - -.. currentmodule:: dtscalibration - -.. autofunction:: get_netcdf_encoding diff --git a/docs/api/dtscalibration.merge_double_ended.rst b/docs/api/dtscalibration.merge_double_ended.rst deleted file mode 100644 index c8e512b5..00000000 --- a/docs/api/dtscalibration.merge_double_ended.rst +++ /dev/null @@ -1,6 +0,0 @@ -merge_double_ended -================== - -.. currentmodule:: dtscalibration - -.. autofunction:: merge_double_ended diff --git a/docs/api/dtscalibration.open_datastore.rst b/docs/api/dtscalibration.open_datastore.rst deleted file mode 100644 index 95a987bd..00000000 --- a/docs/api/dtscalibration.open_datastore.rst +++ /dev/null @@ -1,6 +0,0 @@ -open_datastore -============== - -.. currentmodule:: dtscalibration - -.. autofunction:: open_datastore diff --git a/docs/api/dtscalibration.open_mf_datastore.rst b/docs/api/dtscalibration.open_mf_datastore.rst deleted file mode 100644 index edab4ebe..00000000 --- a/docs/api/dtscalibration.open_mf_datastore.rst +++ /dev/null @@ -1,6 +0,0 @@ -open_mf_datastore -================= - -.. currentmodule:: dtscalibration - -.. autofunction:: open_mf_datastore diff --git a/docs/api/dtscalibration.plot_accuracy.rst b/docs/api/dtscalibration.plot_accuracy.rst deleted file mode 100644 index a2e41fc2..00000000 --- a/docs/api/dtscalibration.plot_accuracy.rst +++ /dev/null @@ -1,6 +0,0 @@ -plot_accuracy -============= - -.. currentmodule:: dtscalibration - -.. autofunction:: plot_accuracy diff --git a/docs/api/dtscalibration.plot_location_residuals_double_ended.rst b/docs/api/dtscalibration.plot_location_residuals_double_ended.rst deleted file mode 100644 index ad0d16db..00000000 --- a/docs/api/dtscalibration.plot_location_residuals_double_ended.rst +++ /dev/null @@ -1,6 +0,0 @@ -plot_location_residuals_double_ended -==================================== - -.. currentmodule:: dtscalibration - -.. autofunction:: plot_location_residuals_double_ended diff --git a/docs/api/dtscalibration.plot_residuals_reference_sections.rst b/docs/api/dtscalibration.plot_residuals_reference_sections.rst deleted file mode 100644 index 45f2529b..00000000 --- a/docs/api/dtscalibration.plot_residuals_reference_sections.rst +++ /dev/null @@ -1,6 +0,0 @@ -plot_residuals_reference_sections -================================= - -.. currentmodule:: dtscalibration - -.. autofunction:: plot_residuals_reference_sections diff --git a/docs/api/dtscalibration.plot_residuals_reference_sections_single.rst b/docs/api/dtscalibration.plot_residuals_reference_sections_single.rst deleted file mode 100644 index 725d77a0..00000000 --- a/docs/api/dtscalibration.plot_residuals_reference_sections_single.rst +++ /dev/null @@ -1,6 +0,0 @@ -plot_residuals_reference_sections_single -======================================== - -.. currentmodule:: dtscalibration - -.. autofunction:: plot_residuals_reference_sections_single diff --git a/docs/api/dtscalibration.plot_sigma_report.rst b/docs/api/dtscalibration.plot_sigma_report.rst deleted file mode 100644 index b047cdeb..00000000 --- a/docs/api/dtscalibration.plot_sigma_report.rst +++ /dev/null @@ -1,6 +0,0 @@ -plot_sigma_report -================= - -.. currentmodule:: dtscalibration - -.. autofunction:: plot_sigma_report diff --git a/docs/api/dtscalibration.read_apsensing_files.rst b/docs/api/dtscalibration.read_apsensing_files.rst deleted file mode 100644 index 04bd67d6..00000000 --- a/docs/api/dtscalibration.read_apsensing_files.rst +++ /dev/null @@ -1,6 +0,0 @@ -read_apsensing_files -==================== - -.. currentmodule:: dtscalibration - -.. autofunction:: read_apsensing_files diff --git a/docs/api/dtscalibration.read_sensornet_files.rst b/docs/api/dtscalibration.read_sensornet_files.rst deleted file mode 100644 index 541f00cb..00000000 --- a/docs/api/dtscalibration.read_sensornet_files.rst +++ /dev/null @@ -1,6 +0,0 @@ -read_sensornet_files -==================== - -.. currentmodule:: dtscalibration - -.. autofunction:: read_sensornet_files diff --git a/docs/api/dtscalibration.read_sensortran_files.rst b/docs/api/dtscalibration.read_sensortran_files.rst deleted file mode 100644 index 35d92c94..00000000 --- a/docs/api/dtscalibration.read_sensortran_files.rst +++ /dev/null @@ -1,6 +0,0 @@ -read_sensortran_files -===================== - -.. currentmodule:: dtscalibration - -.. autofunction:: read_sensortran_files diff --git a/docs/api/dtscalibration.read_silixa_files.rst b/docs/api/dtscalibration.read_silixa_files.rst deleted file mode 100644 index d5adee72..00000000 --- a/docs/api/dtscalibration.read_silixa_files.rst +++ /dev/null @@ -1,6 +0,0 @@ -read_silixa_files -================= - -.. currentmodule:: dtscalibration - -.. autofunction:: read_silixa_files diff --git a/docs/api/dtscalibration.shift_double_ended.rst b/docs/api/dtscalibration.shift_double_ended.rst deleted file mode 100644 index 0a879ea1..00000000 --- a/docs/api/dtscalibration.shift_double_ended.rst +++ /dev/null @@ -1,6 +0,0 @@ -shift_double_ended -================== - -.. currentmodule:: dtscalibration - -.. autofunction:: shift_double_ended diff --git a/docs/api/dtscalibration.suggest_cable_shift_double_ended.rst b/docs/api/dtscalibration.suggest_cable_shift_double_ended.rst deleted file mode 100644 index 14a135c0..00000000 --- a/docs/api/dtscalibration.suggest_cable_shift_double_ended.rst +++ /dev/null @@ -1,6 +0,0 @@ -suggest_cable_shift_double_ended -================================ - -.. currentmodule:: dtscalibration - -.. autofunction:: suggest_cable_shift_double_ended diff --git a/docs/notebooks/01Load_xml_measurement_files.ipynb b/docs/notebooks/01Load_xml_measurement_files.ipynb index 04611154..c25396aa 100644 --- a/docs/notebooks/01Load_xml_measurement_files.ipynb +++ b/docs/notebooks/01Load_xml_measurement_files.ipynb @@ -157,7 +157,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.5" + "version": "3.10.10" } }, "nbformat": 4, diff --git a/docs/notebooks/02Common_DataStore_functions_slice_mean_max_std_resample.ipynb b/docs/notebooks/02Common_DataStore_functions_slice_mean_max_std_resample.ipynb index ad563848..79f14247 100644 --- a/docs/notebooks/02Common_DataStore_functions_slice_mean_max_std_resample.ipynb +++ b/docs/notebooks/02Common_DataStore_functions_slice_mean_max_std_resample.ipynb @@ -28,7 +28,7 @@ "outputs": [], "source": [ "import os\n", - "\n", + "import xarray as xr\n", "from dtscalibration import read_silixa_files" ] }, @@ -94,7 +94,7 @@ }, "outputs": [], "source": [ - "ds[\"tmp\"].plot(figsize=(12, 8));" + "ds[\"tmp\"].plot(figsize=(12, 8))" ] }, { @@ -283,9 +283,7 @@ "outputs": [], "source": [ "# We use the logic from xarray to resample. However, it returns an xarray dataset type. Therefore we convert it back to the dtscalibration Datastore type.\n", - "from dtscalibration import DataStore\n", - "\n", - "ds_resampled = DataStore(ds.resample(time=\"47S\").mean())" + "ds_resampled = xr.Dataset(ds.resample(time=\"47S\").mean())" ] }, { From c15d2d01957648fd9ee37ef3c3b87e1583da9f83 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Wed, 18 Oct 2023 21:25:39 +0200 Subject: [PATCH 48/66] Fixed variance test --- tests/test_variance_stokes.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_variance_stokes.py b/tests/test_variance_stokes.py index 256ef986..acf01ede 100644 --- a/tests/test_variance_stokes.py +++ b/tests/test_variance_stokes.py @@ -11,7 +11,7 @@ from dtscalibration.variance_stokes import variance_stokes_exponential from dtscalibration.variance_stokes import variance_stokes_linear -np.random.seed(0) +state = np.random.seed(0) fn = [ "channel 1_20170921112245510.xml", @@ -242,9 +242,7 @@ def callable_st_var(stokes): @pytest.mark.slow # Execution time ~0.5 minute def test_variance_input_types_double(): - import dask.array as da - - state = da.random.RandomState(0) + state = np.random.seed(0) stokes_m_var = 40.0 cable_len = 100.0 From 0ba58048d4ec4a93090ed60b0613b067fd105a1e Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Wed, 18 Oct 2023 21:54:43 +0200 Subject: [PATCH 49/66] Update test_variance_stokes.py --- tests/test_variance_stokes.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/test_variance_stokes.py b/tests/test_variance_stokes.py index acf01ede..50143b91 100644 --- a/tests/test_variance_stokes.py +++ b/tests/test_variance_stokes.py @@ -60,7 +60,8 @@ def assert_almost_equal_verbose(actual, desired, verbose=False, **kwargs): def test_variance_input_types_single(): import dask.array as da - state = da.random.RandomState(0) + state_da = da.random.RandomState(0) + state_np = np.random.RandomState(0) stokes_m_var = 40.0 cable_len = 100.0 @@ -96,8 +97,8 @@ def test_variance_input_types_single(): / (1 - np.exp(-gamma / temp_real)) ) - st_m = st + stats.norm.rvs(size=st.shape, scale=stokes_m_var**0.5) - ast_m = ast + stats.norm.rvs(size=ast.shape, scale=1.1 * stokes_m_var**0.5) + st_m = st + stats.norm.rvs(size=st.shape, scale=stokes_m_var**0.5, random_state=state_np) + ast_m = ast + stats.norm.rvs(size=ast.shape, scale=1.1 * stokes_m_var**0.5, random_state=state_np) print("alphaint", cable_len * (dalpha_p - dalpha_m)) print("alpha", dalpha_p - dalpha_m) @@ -133,7 +134,7 @@ def test_variance_input_types_single(): st_var=st_var, ast_var=st_var, mc_sample_size=100, - da_random_state=state, + da_random_state=state_da, mc_remove_set_flag=False, ) @@ -163,7 +164,7 @@ def callable_st_var(stokes): st_var=callable_st_var, ast_var=callable_st_var, mc_sample_size=100, - da_random_state=state, + da_random_state=state_da, ) assert_almost_equal_verbose( From 6e97a0ed1818f78915087a25091e5b872a80b13c Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Wed, 18 Oct 2023 21:54:48 +0200 Subject: [PATCH 50/66] Update 15Matching_sections.ipynb --- docs/notebooks/15Matching_sections.ipynb | 38 ++++++++++++++---------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/docs/notebooks/15Matching_sections.ipynb b/docs/notebooks/15Matching_sections.ipynb index 3017c04b..26c017a9 100644 --- a/docs/notebooks/15Matching_sections.ipynb +++ b/docs/notebooks/15Matching_sections.ipynb @@ -40,6 +40,9 @@ "import os\n", "\n", "from dtscalibration import read_silixa_files\n", + "from dtscalibration.dts_accessor import DtsAccessor # noqa: E402\n", + "from dtscalibration.variance_stokes import variance_stokes_constant\n", + "\n", "import matplotlib.pyplot as plt\n", "\n", "%matplotlib inline" @@ -120,14 +123,20 @@ }, "outputs": [], "source": [ - "ds_a = ds.copy(deep=True)\n", - "\n", - "st_var, resid = ds_a.variance_stokes(sections=sections, st_label=\"st\")\n", - "ast_var, _ = ds_a.variance_stokes(sections=sections, st_label=\"ast\")\n", - "rst_var, _ = ds_a.variance_stokes(sections=sections, st_label=\"rst\")\n", - "rast_var, _ = ds_a.variance_stokes(sections=sections, st_label=\"rast\")\n", + "st_var, resid = variance_stokes_constant(\n", + " ds.dts.st, sections, ds.dts.acquisitiontime_fw, reshape_residuals=True\n", + ")\n", + "ast_var, _ = variance_stokes_constant(\n", + " ds.dts.ast, sections, ds.dts.acquisitiontime_fw, reshape_residuals=False\n", + ")\n", + "rst_var, _ = variance_stokes_constant(\n", + " ds.dts.rst, sections, ds.dts.acquisitiontime_bw, reshape_residuals=False\n", + ")\n", + "rast_var, _ = variance_stokes_constant(\n", + " ds.dts.rast, sections, ds.dts.acquisitiontime_bw, reshape_residuals=False\n", + ")\n", "\n", - "ds_a.calibration_double_ended(\n", + "out = ds.dts.calibrate_double_ended(\n", " sections=sections,\n", " st_var=st_var,\n", " ast_var=ast_var,\n", @@ -135,7 +144,7 @@ " rast_var=rast_var,\n", ")\n", "\n", - "ds_a.isel(time=0).tmpw.plot(label=\"calibrated\")" + "out.isel(time=0).tmpw.plot(label=\"calibrated\")" ] }, { @@ -168,12 +177,7 @@ "source": [ "matching_sections = [(slice(7.5, 17.6), slice(69, 79.1), False)]\n", "\n", - "st_var, resid = ds.variance_stokes(sections=sections, st_label=\"st\")\n", - "ast_var, _ = ds.variance_stokes(sections=sections, st_label=\"ast\")\n", - "rst_var, _ = ds.variance_stokes(sections=sections, st_label=\"rst\")\n", - "rast_var, _ = ds.variance_stokes(sections=sections, st_label=\"rast\")\n", - "\n", - "ds.calibration_double_ended(\n", + "out2 = ds.dts.calibrate_double_ended(\n", " sections=sections,\n", " st_var=st_var,\n", " ast_var=ast_var,\n", @@ -183,8 +187,10 @@ " matching_sections=matching_sections,\n", ")\n", "\n", - "ds_a.isel(time=0).tmpw.plot(label=\"normal calibration\")\n", - "ds.isel(time=0).tmpw.plot(label=\"matching sections\")\n", + "out2.isel(time=0).tmpw.plot(label=\"calibrated\")\n", + "\n", + "out.isel(time=0).tmpw.plot(label=\"normal calibration\")\n", + "out2.isel(time=0).tmpw.plot(label=\"matching sections\")\n", "plt.legend()" ] }, From a97988db80e78f90b868f5049199d4c54373d5a5 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Thu, 19 Oct 2023 22:20:26 +0200 Subject: [PATCH 51/66] Updated 16Averaging notebook --- docs/notebooks/16Averaging_temperatures.ipynb | 51 ++++++++++--------- tests/test_examples.py | 1 + 2 files changed, 28 insertions(+), 24 deletions(-) diff --git a/docs/notebooks/16Averaging_temperatures.ipynb b/docs/notebooks/16Averaging_temperatures.ipynb index ba606b2f..2bdd41e8 100644 --- a/docs/notebooks/16Averaging_temperatures.ipynb +++ b/docs/notebooks/16Averaging_temperatures.ipynb @@ -37,6 +37,8 @@ "import os\n", "\n", "from dtscalibration import read_silixa_files\n", + "from dtscalibration.dts_accessor import DtsAccessor # noqa: E402\n", + "from dtscalibration.variance_stokes import variance_stokes_constant\n", "import matplotlib.pyplot as plt\n", "\n", "%matplotlib inline" @@ -79,10 +81,18 @@ }, "outputs": [], "source": [ - "st_var, resid = ds.variance_stokes(sections=sections, st_label=\"st\")\n", - "ast_var, _ = ds.variance_stokes(sections=sections, st_label=\"ast\")\n", - "rst_var, _ = ds.variance_stokes(sections=sections, st_label=\"rst\")\n", - "rast_var, _ = ds.variance_stokes(sections=sections, st_label=\"rast\")" + "st_var, resid = variance_stokes_constant(\n", + " ds.dts.st, sections, ds.dts.acquisitiontime_fw, reshape_residuals=True\n", + ")\n", + "ast_var, _ = variance_stokes_constant(\n", + " ds.dts.ast, sections, ds.dts.acquisitiontime_fw, reshape_residuals=False\n", + ")\n", + "rst_var, _ = variance_stokes_constant(\n", + " ds.dts.rst, sections, ds.dts.acquisitiontime_bw, reshape_residuals=False\n", + ")\n", + "rast_var, _ = variance_stokes_constant(\n", + " ds.dts.rast, sections, ds.dts.acquisitiontime_bw, reshape_residuals=False\n", + ")" ] }, { @@ -98,7 +108,7 @@ }, "outputs": [], "source": [ - "ds.calibrate_double_ended(\n", + "out = ds.dts.calibrate_double_ended(\n", " sections=sections,\n", " st_var=st_var,\n", " ast_var=ast_var,\n", @@ -178,8 +188,8 @@ }, "outputs": [], "source": [ - "ds.average_double_ended(\n", - " sections=sections,\n", + "out_avg = ds.dts.average_monte_carlo_double_ended(\n", + " result=out,\n", " st_var=st_var,\n", " ast_var=ast_var,\n", " rst_var=rst_var,\n", @@ -191,7 +201,7 @@ " ci_avg_time_isel=[0, 1, 2, 3, 4, 5],\n", " ci_avg_time_sel=None,\n", ")\n", - "ds.tmpw_mc_avg1.plot(hue=\"CI\", linewidth=0.8)" + "out_avg.tmpw_mc_avg1.plot(hue=\"CI\", linewidth=0.8)" ] }, { @@ -239,8 +249,8 @@ }, "outputs": [], "source": [ - "ds.average_double_ended(\n", - " sections=sections,\n", + "out_avg = ds.dts.average_monte_carlo_double_ended(\n", + " result=out,\n", " st_var=st_var,\n", " ast_var=ast_var,\n", " rst_var=rst_var,\n", @@ -252,7 +262,7 @@ " ci_avg_time_isel=[0, 1, 2, 3, 4, 5],\n", " ci_avg_time_sel=None,\n", ")\n", - "ds.tmpw_mc_avg2.plot(hue=\"CI\", linewidth=0.8)" + "out_avg.tmpw_mc_avg2.plot(hue=\"CI\", linewidth=0.8)" ] }, { @@ -300,8 +310,8 @@ }, "outputs": [], "source": [ - "ds.average_double_ended(\n", - " sections=sections,\n", + "out_avg = ds.dts.average_monte_carlo_double_ended(\n", + " result=out,\n", " st_var=st_var,\n", " ast_var=ast_var,\n", " rst_var=rst_var,\n", @@ -313,7 +323,7 @@ " ci_avg_x_sel=slice(7.5, 17.0),\n", " ci_avg_x_isel=None,\n", ")\n", - "ds.tmpw_mc_avgx1.plot(hue=\"CI\", linewidth=0.8)" + "out_avg.tmpw_mc_avgx1.plot(hue=\"CI\", linewidth=0.8)" ] }, { @@ -361,8 +371,8 @@ }, "outputs": [], "source": [ - "ds.average_double_ended(\n", - " sections=sections,\n", + "out_avg = ds.dts.average_monte_carlo_double_ended(\n", + " result=out,\n", " st_var=st_var,\n", " ast_var=ast_var,\n", " rst_var=rst_var,\n", @@ -374,15 +384,8 @@ " ci_avg_x_sel=slice(7.5, 17.0),\n", " ci_avg_x_isel=None,\n", ")\n", - "ds.tmpw_mc_avgx2.plot(hue=\"CI\", linewidth=0.8)" + "out_avg.tmpw_mc_avgx2.plot(hue=\"CI\", linewidth=0.8)" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/tests/test_examples.py b/tests/test_examples.py index e2a8793c..8cc689bd 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -68,5 +68,6 @@ def test_ipynb(): filepathlist = glob.glob(nb_dir) for fp in filepathlist: + print("Running: {fp}") _, errors = _notebook_run(fp) assert errors == [] From 6ab648ad0c879317d8c8525f1aa6341a5d79e52b Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Fri, 20 Oct 2023 10:34:51 +0200 Subject: [PATCH 52/66] Update test_examples.py --- tests/test_examples.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_examples.py b/tests/test_examples.py index 8cc689bd..7a3b2171 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -68,6 +68,6 @@ def test_ipynb(): filepathlist = glob.glob(nb_dir) for fp in filepathlist: - print("Running: {fp}") + print(f"Running: {fp}") _, errors = _notebook_run(fp) assert errors == [] From c70b2d0cd2539e5f699f7f680481570cb8e47cfe Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Fri, 20 Oct 2023 10:59:22 +0200 Subject: [PATCH 53/66] Update test_examples.py --- tests/test_examples.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_examples.py b/tests/test_examples.py index 7a3b2171..1cb97248 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -70,4 +70,5 @@ def test_ipynb(): for fp in filepathlist: print(f"Running: {fp}") _, errors = _notebook_run(fp) - assert errors == [] + assert errors == [], f"Errors: {errors}" + print("No errors found in nb") From 1245571bf0bfbeb65728b9aab87e862e0aebe335 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Fri, 20 Oct 2023 11:18:41 +0200 Subject: [PATCH 54/66] Fixing weirdly failing notebooks --- ...eries.ipynb => 09Import_timeseries2.ipynb} | 72 +++++++++---------- tests/test_examples.py | 1 + 2 files changed, 37 insertions(+), 36 deletions(-) rename docs/notebooks/{09Import_timeseries.ipynb => 09Import_timeseries2.ipynb} (70%) diff --git a/docs/notebooks/09Import_timeseries.ipynb b/docs/notebooks/09Import_timeseries2.ipynb similarity index 70% rename from docs/notebooks/09Import_timeseries.ipynb rename to docs/notebooks/09Import_timeseries2.ipynb index 39e06c53..c12234c9 100644 --- a/docs/notebooks/09Import_timeseries.ipynb +++ b/docs/notebooks/09Import_timeseries2.ipynb @@ -22,10 +22,10 @@ "execution_count": null, "metadata": { "execution": { - "iopub.execute_input": "2022-04-06T08:10:31.444824Z", - "iopub.status.busy": "2022-04-06T08:10:31.444098Z", - "iopub.status.idle": "2022-04-06T08:10:32.898656Z", - "shell.execute_reply": "2022-04-06T08:10:32.897961Z" + "iopub.execute_input": "2023-10-20T09:16:48.371460Z", + "iopub.status.busy": "2023-10-20T09:16:48.371128Z", + "iopub.status.idle": "2023-10-20T09:16:49.350990Z", + "shell.execute_reply": "2023-10-20T09:16:49.350668Z" } }, "outputs": [], @@ -48,10 +48,10 @@ "execution_count": null, "metadata": { "execution": { - "iopub.execute_input": "2022-04-06T08:10:32.901423Z", - "iopub.status.busy": "2022-04-06T08:10:32.901062Z", - "iopub.status.idle": "2022-04-06T08:10:32.905451Z", - "shell.execute_reply": "2022-04-06T08:10:32.904987Z" + "iopub.execute_input": "2023-10-20T09:16:49.353033Z", + "iopub.status.busy": "2023-10-20T09:16:49.352815Z", + "iopub.status.idle": "2023-10-20T09:16:49.355625Z", + "shell.execute_reply": "2023-10-20T09:16:49.355358Z" } }, "outputs": [], @@ -77,10 +77,10 @@ "execution_count": null, "metadata": { "execution": { - "iopub.execute_input": "2022-04-06T08:10:32.928616Z", - "iopub.status.busy": "2022-04-06T08:10:32.927992Z", - "iopub.status.idle": "2022-04-06T08:10:33.076019Z", - "shell.execute_reply": "2022-04-06T08:10:33.075543Z" + "iopub.execute_input": "2023-10-20T09:16:49.375672Z", + "iopub.status.busy": "2023-10-20T09:16:49.375532Z", + "iopub.status.idle": "2023-10-20T09:16:49.399183Z", + "shell.execute_reply": "2023-10-20T09:16:49.398877Z" } }, "outputs": [], @@ -96,10 +96,10 @@ "execution_count": null, "metadata": { "execution": { - "iopub.execute_input": "2022-04-06T08:10:33.078306Z", - "iopub.status.busy": "2022-04-06T08:10:33.078143Z", - "iopub.status.idle": "2022-04-06T08:10:33.088386Z", - "shell.execute_reply": "2022-04-06T08:10:33.087913Z" + "iopub.execute_input": "2023-10-20T09:16:49.401036Z", + "iopub.status.busy": "2023-10-20T09:16:49.400906Z", + "iopub.status.idle": "2023-10-20T09:16:49.405335Z", + "shell.execute_reply": "2023-10-20T09:16:49.405054Z" } }, "outputs": [], @@ -119,10 +119,10 @@ "execution_count": null, "metadata": { "execution": { - "iopub.execute_input": "2022-04-06T08:10:33.090805Z", - "iopub.status.busy": "2022-04-06T08:10:33.090642Z", - "iopub.status.idle": "2022-04-06T08:10:33.301302Z", - "shell.execute_reply": "2022-04-06T08:10:33.300636Z" + "iopub.execute_input": "2023-10-20T09:16:49.407001Z", + "iopub.status.busy": "2023-10-20T09:16:49.406895Z", + "iopub.status.idle": "2023-10-20T09:16:49.587342Z", + "shell.execute_reply": "2023-10-20T09:16:49.587037Z" } }, "outputs": [], @@ -145,10 +145,10 @@ "execution_count": null, "metadata": { "execution": { - "iopub.execute_input": "2022-04-06T08:10:33.303735Z", - "iopub.status.busy": "2022-04-06T08:10:33.303562Z", - "iopub.status.idle": "2022-04-06T08:10:33.308068Z", - "shell.execute_reply": "2022-04-06T08:10:33.307543Z" + "iopub.execute_input": "2023-10-20T09:16:49.589047Z", + "iopub.status.busy": "2023-10-20T09:16:49.588934Z", + "iopub.status.idle": "2023-10-20T09:16:49.591317Z", + "shell.execute_reply": "2023-10-20T09:16:49.591061Z" } }, "outputs": [], @@ -168,10 +168,10 @@ "execution_count": null, "metadata": { "execution": { - "iopub.execute_input": "2022-04-06T08:10:33.310439Z", - "iopub.status.busy": "2022-04-06T08:10:33.310255Z", - "iopub.status.idle": "2022-04-06T08:10:33.315622Z", - "shell.execute_reply": "2022-04-06T08:10:33.315088Z" + "iopub.execute_input": "2023-10-20T09:16:49.592925Z", + "iopub.status.busy": "2023-10-20T09:16:49.592798Z", + "iopub.status.idle": "2023-10-20T09:16:49.594944Z", + "shell.execute_reply": "2023-10-20T09:16:49.594703Z" } }, "outputs": [], @@ -192,10 +192,10 @@ "execution_count": null, "metadata": { "execution": { - "iopub.execute_input": "2022-04-06T08:10:33.318123Z", - "iopub.status.busy": "2022-04-06T08:10:33.317912Z", - "iopub.status.idle": "2022-04-06T08:10:33.328112Z", - "shell.execute_reply": "2022-04-06T08:10:33.327546Z" + "iopub.execute_input": "2023-10-20T09:16:49.596483Z", + "iopub.status.busy": "2023-10-20T09:16:49.596379Z", + "iopub.status.idle": "2023-10-20T09:16:49.716179Z", + "shell.execute_reply": "2023-10-20T09:16:49.715795Z" } }, "outputs": [], @@ -208,10 +208,10 @@ "execution_count": null, "metadata": { "execution": { - "iopub.execute_input": "2022-04-06T08:10:33.330579Z", - "iopub.status.busy": "2022-04-06T08:10:33.330405Z", - "iopub.status.idle": "2022-04-06T08:10:33.342398Z", - "shell.execute_reply": "2022-04-06T08:10:33.341846Z" + "iopub.execute_input": "2023-10-20T09:16:49.718112Z", + "iopub.status.busy": "2023-10-20T09:16:49.717968Z", + "iopub.status.idle": "2023-10-20T09:16:49.722423Z", + "shell.execute_reply": "2023-10-20T09:16:49.721693Z" }, "scrolled": true }, diff --git a/tests/test_examples.py b/tests/test_examples.py index 1cb97248..141f4f9c 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -35,6 +35,7 @@ def _notebook_run(path): path, "--output", out_path, + "--debug", "--to", "notebook", "--execute", From 975466ac654bebd47a21acf821e2a88d04447788 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Fri, 20 Oct 2023 12:50:48 +0200 Subject: [PATCH 55/66] Update test_examples.py --- tests/test_examples.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/tests/test_examples.py b/tests/test_examples.py index 141f4f9c..f671db66 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -1,4 +1,5 @@ import glob +import json import os import shutil import subprocess @@ -35,7 +36,6 @@ def _notebook_run(path): path, "--output", out_path, - "--debug", "--to", "notebook", "--execute", @@ -44,20 +44,24 @@ def _notebook_run(path): subprocess.check_call(args) assert os.path.exists(nbpath), "nbconvert used different output filename" + + with open(nbpath) as f: + _ = json.load(f) + print("Basic JSON validation successful.") - nb = nbformat.read(nbpath, nbformat.current_nbformat) + try: + nb = nbformat.read(nbpath, nbformat.current_nbformat) + except Exception as e: + print(f"Error reading notebook: {e}") + assert 0, "nbformat.read failed" - errors = [ - output - for cell in nb.cells - if "outputs" in cell - for output in cell["outputs"] - if output.output_type == "error" - ] - - # Remove the temp file once the test is done - if os.path.exists(nbpath): - os.remove(nbpath) + errors = [ + output + for cell in nb.cells + if "outputs" in cell + for output in cell["outputs"] + if output.output_type == "error" + ] return nb, errors From 21ecdf9bac3e3bb608aa209a3f9b2de03174d628 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Sat, 21 Oct 2023 09:43:59 +0200 Subject: [PATCH 56/66] Rewritten test_notebooks routine --- .../calibration/section_utils.py | 4 +- tests/data/docs_notebooks/01Not_working.ipynb | 34 +++++ tests/test_examples.py | 119 ++++++++---------- tests/test_variance_stokes.py | 8 +- 4 files changed, 96 insertions(+), 69 deletions(-) create mode 100644 tests/data/docs_notebooks/01Not_working.ipynb diff --git a/src/dtscalibration/calibration/section_utils.py b/src/dtscalibration/calibration/section_utils.py index e259739f..df9edcb7 100644 --- a/src/dtscalibration/calibration/section_utils.py +++ b/src/dtscalibration/calibration/section_utils.py @@ -9,9 +9,7 @@ def set_sections(ds: xr.Dataset, sections: dict[str, list[slice]]): ds.attrs["_sections"] = yaml.dump(sections) -def set_matching_sections( - ds: xr.Dataset, matching_sections: dict[str, list[slice]] -): +def set_matching_sections(ds: xr.Dataset, matching_sections: dict[str, list[slice]]): ds.attrs["_matching_sections"] = yaml.dump(matching_sections) diff --git a/tests/data/docs_notebooks/01Not_working.ipynb b/tests/data/docs_notebooks/01Not_working.ipynb new file mode 100644 index 00000000..cf95764e --- /dev/null +++ b/tests/data/docs_notebooks/01Not_working.ipynb @@ -0,0 +1,34 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import not_existing_package" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.10" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/test_examples.py b/tests/test_examples.py index f671db66..6ca2761c 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -1,79 +1,70 @@ -import glob -import json import os -import shutil -import subprocess -import tempfile +from collections import namedtuple +from pathlib import Path +from typing import Optional import nbformat +import pytest +from nbconvert.preprocessors import CellExecutionError +from nbconvert.preprocessors import ExecutePreprocessor +NBError = namedtuple("NBError", "title, errname, errvalue, exception") -def _notebook_run(path): - """Execute a notebook via nbconvert and collect output. - :returns (parsed nb object, execution errors) - """ - dirname, __ = os.path.split(path) - os.chdir(dirname) +wd = os.path.dirname(os.path.abspath(__file__)) +src_dir = os.path.join(wd, "..", "docs", "notebooks") - # Create a temporary file to write the notebook to. - # 'with' method is used so the file is closed by tempfile - # and free to be overwritten. - # with tempfile.NamedTemporaryFile('w', suffix=".ipynb") as fout: - with tempfile.NamedTemporaryFile( - "w", suffix=".nbconvert.ipynb", delete=False - ) as fout: - nbpath = fout.name - jupyter_exec = shutil.which("jupyter") +@pytest.mark.parametrize( + "src_path", Path(src_dir).glob("*.ipynb"), ids=lambda x: x.name +) +def test_docs_notebook(src_path): + _test_notebook(src_path, "python3") + + +@pytest.mark.xfail +def test_identify_not_working_docs_notebook(): + fp = Path(os.path.join(wd, "data", "docs_notebooks", "01Not_working.ipynb")) + error = _test_notebook(fp, "python3") - # recent version (~7.3.1) requires output without extension - out_path = os.path.join( - os.path.dirname(nbpath), os.path.basename(nbpath).split(".", 1)[0] - ) - args = [ - jupyter_exec, - "nbconvert", - path, - "--output", - out_path, - "--to", - "notebook", - "--execute", - "--ExecutePreprocessor.timeout=60", - ] - subprocess.check_call(args) - assert os.path.exists(nbpath), "nbconvert used different output filename" - - with open(nbpath) as f: - _ = json.load(f) - print("Basic JSON validation successful.") +def _test_notebook(notebook_file, kernel) -> Optional[NBError]: + """ + Test single notebook. - try: - nb = nbformat.read(nbpath, nbformat.current_nbformat) - except Exception as e: - print(f"Error reading notebook: {e}") - assert 0, "nbformat.read failed" + Parameters + ---------- + notebook_file : str + Source path for notebook + kernel : str, optional + Notebook kernel name, by default "python3" - errors = [ - output - for cell in nb.cells - if "outputs" in cell - for output in cell["outputs"] - if output.output_type == "error" - ] + Returns + ------- + NBError + Error tuple - return nb, errors + """ + print(f"\nTesting notebook {notebook_file.name}") + with open(notebook_file, "rb") as file_handle: + nb_bytes = file_handle.read() + nb_text = nb_bytes.decode("utf-8") + nb_content = nbformat.reads(nb_text, as_version=4) + exec_processor = ExecutePreprocessor(timeout=600, kernel_name=kernel) + try: + print(f"{notebook_file.name} - running notebook: ...") + exec_processor.preprocess( + nb_content, {"metadata": {"path": str(notebook_file.parent)}} + ) -def test_ipynb(): - file_ext = "*.ipynb" - wd = os.path.dirname(os.path.abspath(__file__)) - nb_dir = os.path.join(wd, "..", "docs", "notebooks", file_ext) - filepathlist = glob.glob(nb_dir) + except CellExecutionError as cell_err: + msg = f"Error executing the notebook '{notebook_file.absolute()}" + print(msg) + return NBError( + f"Error while running notebook {notebook_file}", + cell_err.ename, + cell_err.evalue, + cell_err.args[0], + ) - for fp in filepathlist: - print(f"Running: {fp}") - _, errors = _notebook_run(fp) - assert errors == [], f"Errors: {errors}" - print("No errors found in nb") + return None diff --git a/tests/test_variance_stokes.py b/tests/test_variance_stokes.py index 50143b91..02f56568 100644 --- a/tests/test_variance_stokes.py +++ b/tests/test_variance_stokes.py @@ -97,8 +97,12 @@ def test_variance_input_types_single(): / (1 - np.exp(-gamma / temp_real)) ) - st_m = st + stats.norm.rvs(size=st.shape, scale=stokes_m_var**0.5, random_state=state_np) - ast_m = ast + stats.norm.rvs(size=ast.shape, scale=1.1 * stokes_m_var**0.5, random_state=state_np) + st_m = st + stats.norm.rvs( + size=st.shape, scale=stokes_m_var**0.5, random_state=state_np + ) + ast_m = ast + stats.norm.rvs( + size=ast.shape, scale=1.1 * stokes_m_var**0.5, random_state=state_np + ) print("alphaint", cable_len * (dalpha_p - dalpha_m)) print("alpha", dalpha_p - dalpha_m) From 59fd0fd73c32a45fffa279c4676127f4cfb7bcec Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Sat, 21 Oct 2023 10:04:08 +0200 Subject: [PATCH 57/66] Removed unused variable --- tests/test_examples.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_examples.py b/tests/test_examples.py index 6ca2761c..3e5d348b 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -24,7 +24,7 @@ def test_docs_notebook(src_path): @pytest.mark.xfail def test_identify_not_working_docs_notebook(): fp = Path(os.path.join(wd, "data", "docs_notebooks", "01Not_working.ipynb")) - error = _test_notebook(fp, "python3") + _test_notebook(fp, "python3") def _test_notebook(notebook_file, kernel) -> Optional[NBError]: From bca953d873be0e62992d5e1eef78e2109b9d5fb6 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Sat, 21 Oct 2023 11:34:44 +0200 Subject: [PATCH 58/66] Remove metadata from notebook 12 --- ...eries2.ipynb => 09Import_timeseries.ipynb} | 0 .../12Datastore_from_numpy_arrays.ipynb | 54 ++++--------------- 2 files changed, 9 insertions(+), 45 deletions(-) rename docs/notebooks/{09Import_timeseries2.ipynb => 09Import_timeseries.ipynb} (100%) diff --git a/docs/notebooks/09Import_timeseries2.ipynb b/docs/notebooks/09Import_timeseries.ipynb similarity index 100% rename from docs/notebooks/09Import_timeseries2.ipynb rename to docs/notebooks/09Import_timeseries.ipynb diff --git a/docs/notebooks/12Datastore_from_numpy_arrays.ipynb b/docs/notebooks/12Datastore_from_numpy_arrays.ipynb index b2b9c702..83748210 100644 --- a/docs/notebooks/12Datastore_from_numpy_arrays.ipynb +++ b/docs/notebooks/12Datastore_from_numpy_arrays.ipynb @@ -10,7 +10,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2022-04-06T08:11:05.700886Z", @@ -50,7 +50,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2022-04-06T08:11:07.173171Z", @@ -77,7 +77,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2022-04-06T08:11:07.394645Z", @@ -103,7 +103,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2022-04-06T08:11:07.400661Z", @@ -123,7 +123,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2022-04-06T08:11:07.409666Z", @@ -132,22 +132,7 @@ "shell.execute_reply": "2022-04-06T08:11:07.416796Z" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Dimensions: (x: 1461, time: 3)\n", - "Coordinates:\n", - " * x (x) float64 -80.74 -80.62 -80.49 -80.36 ... 104.4 104.6 104.7 104.8\n", - " * time (time) datetime64[ns] 2018-05-04T12:22:17.710000 ... 2018-05-04T...\n", - "Data variables:\n", - " st (x, time) float64 -0.8058 0.4287 -0.513 ... 27.99 27.83 28.81\n", - " ast (x, time) float64 -0.2459 -0.5932 0.1111 0.3748 ... 36.2 35.7 35.16\n" - ] - } - ], + "outputs": [], "source": [ "print(ds)" ] @@ -169,7 +154,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2022-04-06T08:11:07.419863Z", @@ -197,7 +182,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2022-04-06T08:11:07.430326Z", @@ -206,28 +191,7 @@ "shell.execute_reply": "2022-04-06T08:11:07.508335Z" } }, - "outputs": [ - { - "data": { - "text/plain": [ - "[]" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjYAAAHHCAYAAACskBIUAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB/3UlEQVR4nO3dd3hUVfoH8O/0SQ8JJAFC71U6RFARUVAsLIhlEcG6uuiqWFFZO6i4uu6KrLoIiqD+sHdX6UrvRXpvCSWkJ1PP74/k3tw7MzeZSTKZku/neXhM7ty5c2ZMZt6873vO0QkhBIiIiIiigD7UAyAiIiKqKwxsiIiIKGowsCEiIqKowcCGiIiIogYDGyIiIooaDGyIiIgoajCwISIioqjBwIaIiIiiBgMbIiIiihoMbEKodevWmDRpUqiHQUQUdEVFRbjzzjuRkZEBnU6HBx98EIcPH4ZOp8O8efPk85599lnodLrQDZQiHgObIFu1ahWeffZZ5OXlhXoo9Wr9+vW477770K1bN8TFxaFly5a44YYbsHfvXp/n79q1CyNHjkR8fDxSUlIwYcIEnDlzxuu8l156Cddeey3S09Oh0+nw7LPPao7h119/xaWXXorGjRsjOTkZAwYMwPz58/1+Dm63G6+++iratGkDq9WKnj174uOPP/Y6b9KkSdDpdF7/Onfu7Pdj2Ww2PP7442jWrBliYmIwcOBA/PLLL1XeJy8vD2lpadDpdPjss8/k477G4uvfsmXLAACzZ8/GuHHj0LJlS+h0Os1ge/Hixbj99tvRsWNHxMbGom3btrjzzjtx6tQpv57jsWPH8Nxzz2HAgAFo1KgRGjdujKFDh+LXX3+t88cK5HkNHTpU8zUymUzVPk5Vr/Hll1+uOjeQn98TJ07ghhtuQHJyMhITE3Hdddfh4MGDPs+dM2cOunTpAqvVig4dOuDf//53vV3TX9OnT8e8efNw7733Yv78+ZgwYUKtrkekxRjqAUS7VatW4bnnnsOkSZOQnJysum3Pnj3Q66MztnzllVfw+++/Y9y4cejZsyeys7Px1ltvoU+fPlizZg26d+8un3v8+HFcfPHFSEpKwvTp01FUVITXXnsN27dvx7p162A2m+Vzn376aWRkZKB37974+eefNR//m2++wejRo5GVlSX/Bfh///d/uPXWW3H27Fk89NBD1T6Hp556Ci+//DLuuusu9O/fH19//TX+/Oc/Q6fT4aabblKda7FY8N///ld1LCkpyd+XC5MmTcJnn32GBx98EB06dMC8efNw1VVXYenSpRgyZIjP+/z9739HSUmJ13HP4O3DDz/EL7/84nW8S5cuAMr/XxUWFmLAgAFVBg6PP/44cnNzMW7cOHTo0AEHDx7EW2+9he+++w5btmxBRkZGlc/x66+/xiuvvILRo0dj4sSJcDqd+PDDD3H55Zfj/fffx2233VZnjxXI83rqqadw5513qo4VFxfjnnvuwRVXXFHt4/gKljds2IA333zT6/7+/vwWFRXh0ksvRX5+Pp588kmYTCa88cYbuOSSS7BlyxakpqbK577zzju45557MHbsWEyZMgUrV67E3/72N5SUlODxxx8P6jUDsWTJEgwaNAjPPPOMfEwIgdLSUr8CSCK/CQqqmTNnCgDi0KFDoR5Kvfr999+FzWZTHdu7d6+wWCxi/PjxquP33nuviImJEUeOHJGP/fLLLwKAeOedd1TnSq/jmTNnBADxzDPP+Hz8yy+/XDRr1kyUlZXJxxwOh2jXrp3o2bNnteM/fvy4MJlMYvLkyfIxt9stLrroIpGZmSmcTqd8fOLEiSIuLq7aa2pZu3atACBmzpwpHystLRXt2rUTWVlZPu+zfft2YTQaxfPPPy8AiEWLFmlef/LkyaKqX/XDhw8Lt9sthBAiLi5OTJw40ed5y5cvFy6Xy+sYAPHUU09pXl+yY8cOcebMGdWxsrIy0blzZ5GZmVmnjyWE/8/Ll/nz5wsAYsGCBX7fR+mOO+4QOp1OHDt2THXc35/fV155RQAQ69atk4/t2rVLGAwGMXXqVPlYSUmJSE1NFaNGjVLdf/z48SIuLk7k5uYG9ZqBaNOmjdc1fXnmmWeq/Hklqg5/eoJI+gX1/Ce9ubVq1Ur1Zjt37lwBQKxcuVLcf//9onHjxiIpKUncfffdwmazifPnz4sJEyaI5ORkkZycLB599FH5jVvicrnEG2+8Ibp27SosFotIS0sTd999d43fjOpanz59RJ8+fVTH0tLSxLhx47zO7dixo7jssst8Xqe6D4aBAweKbt26+Tw+cODAasc5a9YsAUDs3LlTdXzhwoXy/yOJFNg4nU6Rn59f7bU9Pfroo8JgMHjdd/r06QKAOHr0qNd9hg0bJsaNGyeWLl1a68BGKdAAQAghUlJSxJgxY1THzpw5I3bt2iWKi4urvf+UKVMEAFFQUBC0xwr0eV155ZUiLi5OFBUV+X0fSVlZmUhOThZDhw7VPKe6n9/+/fuL/v37ex2/4oorRLt27eTvv//+ewFAfP/996rzVq1aJQCI+fPnB/Wa/pB+Rn29Dx46dEgAEHPnzpXP1wps5s+fL/r06SOsVqto1KiRuPHGG71+Ny655BLRrVs3sXPnTjF06FARExMjmjVrJl555RXVeTabTUybNk306dNHJCYmitjYWDFkyBCxZMkS1XnS+GbOnCneeust0aZNGxETEyMuv/xycfToUeF2u8Xzzz8vmjdvLqxWq7j22mvFuXPnvMb+ww8/iCFDhojY2FgRHx8vrrrqKrFjx46AXse6UFpaKp555hnRoUMHYbFYREZGhvjTn/4k9u/fX+9jCaborIOEiTFjxuDmm28GALzxxhuYP38+5s+fjyZNmlR5v/vvvx/79u3Dc889h2uvvRbvvvsupk2bhmuuuQYulwvTp0/HkCFDMHPmTK80+F/+8hc8+uijGDx4MN58803cdtttWLBgAUaMGAGHw1Hl49psNpw9e9avfzUhhEBOTg4aN24sHztx4gROnz6Nfv36eZ0/YMAAbN68uUaPNXToUOzcuRPTpk3D/v37ceDAAbzwwgvYsGEDHnvssWrvv3nzZsTFxcnlGuWYpNuVSkpKkJiYiKSkJKSkpGDy5MkoKirya6ybN29Gx44dkZiY6POxtmzZojq+aNEirFq1Cq+++qpf1w+moqIiFBUVqf6fAsBbb72FLl26YN26ddVeIzs7G7GxsYiNjQ36Y/njzJkz+OWXXzB69GjExcUFfP8ffvgBeXl5GD9+fI0e3+12Y9u2bZq/EwcOHEBhYSGAyp9Dz3P79u0LvV4v3x6Ma/qrS5cumD9/Pho3boxevXr5/T6o9NJLL+HWW29Fhw4d8Prrr+PBBx/E4sWLcfHFF3v1L54/fx4jR47EBRdcgH/84x/o3LkzHn/8cfz444/yOQUFBfjvf/+LoUOH4pVXXsGzzz6LM2fOYMSIEV6/bwCwYMECvP3227j//vvx8MMPY/ny5bjhhhvw9NNP46effsLjjz+Ou+++G99++y0eeeQR1X3nz5+PUaNGIT4+Hq+88gqmTZuGP/74A0OGDMHhw4erfN5ut9vv9+Tq3t9dLheuvvpqPPfcc+jbty/+8Y9/4IEHHkB+fj527NhR5X0jTqgjq2hXVSlKK2MzYsQIVSYmKytL6HQ6cc8998jHnE6nyMzMFJdccol8bOXKlT7T5z/99JNfaXXp8f35VxNSen/OnDnysfXr1wsA4sMPP/Q6/9FHHxUAVOUkSXV/8RYVFYkbbrhB6HQ6ecyxsbHiq6++8muso0aNEm3btvU6XlxcLACIJ554Qj72xBNPiMcff1x8+umn4uOPPxYTJ04UAMTgwYOFw+Go9rG6desmhg0b5nV8586dAoD4z3/+Ix8rKSkRLVu2lEsHoc7YvPDCCwKAWLx4seq49Ff30qVLq7z/vn37hNVqFRMmTAjqYwXyvP79738LAOKHH37w63xPY8eOFRaLRZw/f17znKp+fqXbnn/+ea/bpEzi7t27hRDl/28NBoPPx2jSpIm46aabgnbNQLVq1cqrFOVPxubw4cPCYDCIl156SXVfqRyrPH7JJZd4vZ/YbDaRkZEhxo4dKx9zOp1epfLz58+L9PR0cfvtt3uNr0mTJiIvL08+PnXqVAFAXHDBBarf8ZtvvlmYzWb5PauwsFAkJyeLu+66S/VY2dnZIikpyeu4J+nx/flX3e/a+++/LwCI119/3es2z8x/pGPzcBi64447VNMdBw4ciNWrV+OOO+6QjxkMBvTr1w8bN26Ujy1atAhJSUm4/PLLVVmVvn37Ij4+HkuXLsWf//xnzccdMWJEtTNxamr37t2YPHkysrKyMHHiRPl4aWkpgPLmW09Wq1U+x9ftVbFYLOjYsSOuv/56jBkzBi6XC++++y5uueUW/PLLLxg0aFCV99d6TOWYJDNmzFCdc9NNN6Fjx4546qmn8Nlnn3k1GtfmsV5++WU4HA48+eSTVV6zPqxYsQLPPfccbrjhBgwbNkx127PPPlvljB+gPMs1btw4xMTE4OWXXw7qYwVi4cKFaNKkideMJn8UFBTg+++/x1VXXeU1WcBf/v5OSP9VNtd7nqs8r66vWV+++OILuN1u3HDDDar3tYyMDHTo0AFLly5V/T7Ex8fjlltukb83m80YMGCAavaXwWCAwWAAUJ4VycvLg9vtRr9+/bBp0yavMYwbN041GWDgwIEAgFtuuQVGo1F1/OOPP8aJEyfQtm1b/PLLL8jLy8PNN9+sGrvBYMDAgQOxdOnSKp97RkaG3+/JF1xwQZW3f/7552jcuDHuv/9+r9uibXo9A5sw1LJlS9X30i9UixYtvI6fP39e/n7fvn3Iz89HWlqaz+uePn26ysdt2rQpmjZtWpMhVyk7OxujRo1CUlISPvvsM/kNBQBiYmIAlJfBPJWVlanOCcR9992HNWvWYNOmTfLMsxtuuAHdunXDAw88gLVr18pjU0pKSkJMTAxiYmJqNaaHHnoI06ZNw6+//oqbbroJLpfLa/p6SkoKzGaz3491+PBhzJw5E7NmzUJ8fLw/L0PQ7N69G3/605/QvXt3r9lg/nC5XLjpppvwxx9/4Mcff0SzZs2C9liBOHjwIFavXo377rtP9YHlr88//xxlZWU1LkMBgf1OxMTEwG63+7xOWVmZ6ry6vmZ92bdvH4QQ6NChg8/bPWdUZWZmen1QN2rUCNu2bVMd++CDD/CPf/wDu3fvVpVx2rRp4/UYgbwnA5Dfl/ft2wcAXsG4xLP87MlqtWL48OFVnuOvAwcOoFOnTjX6uY400f8MI5Dyg7+640II+Wu32420tDQsWLDA5/2rq2mXlpYiPz/frzH6M90WAPLz83HllVciLy8PK1eu9PoAkwIpX9NxT506hZSUlICzNXa7HXPmzMFjjz2mmk5vMplw5ZVX4q233oLdbofZbPYK5ObOnYtJkyahadOmWLp0KYQQqjdJaZxVfRAD5R8OqampyM3NBVC+hovnG+bSpUsxdOhQNG3aFCdOnPD5/JWP9fe//x3NmzfH0KFD5dq8FJidOXMGhw8fRsuWLYO+hMCxY8dwxRVXICkpCT/88AMSEhICvsZdd92F7777DgsWLNB806+rxwrEwoULAaDGgcmCBQuQlJSEq6++usZjkH7mtX4ngMqfiaZNm8LlcuH06dOqP2jsdjvOnTsnnxeMa9YXt9sNnU6HH3/80ed7oGeQr/X+qXyv/OijjzBp0iSMHj0ajz76KNLS0mAwGDBjxgwcOHDA676BvCcrH8vtdgMo77Px9Z5ZXZDh6w8iLdIfSsTAJujqM8XXrl07/Prrrxg8eHCN/qr69NNPVWuJVEX5JqGlrKwM11xzDfbu3Ytff/0VXbt29TqnefPmaNKkCTZs2OB127p169CrVy+/xqN07tw5OJ1OuFwur9scDgfcbrd8m2eat1u3bgCAXr164b///S927dqlGreU6aluXIWFhTh79qwcTPpKKUup4169emHp0qUoKChQ/QXn+VhHjx7F/v370bZtW6/H++tf/wqg/C/FmpZA/HHu3DlcccUVsNlsWLx4cY0yfI8++ijmzp2Lf/7zn3JzfbAeK1ALFy5Eu3btqi1V+nLq1CksXboUkyZNCjgYV9Lr9ejRo4fP34m1a9eibdu2coAn/Wxs2LABV111lXzehg0b4Ha75duDcc360q5dOwgh0KZNG3Ts2LFOrvnZZ5+hbdu2+OKLL1Tv0co1dupCu3btAABpaWk1yrz4+oNIi/SHUlVjWbt2LRwOR9SvG8TAJsikWRX1sfLwDTfcgLfffhsvvPACpk+frrrN6XSiqKioyg+9uuyxcblcuPHGG7F69Wp8/fXXyMrK0jx37Nix+OCDD3Ds2DE5tbt48WLs3bvXr4X0PKWlpSE5ORlffvklnn/+efmvmKKiInz77bfo3LmzHPhpvdlcd911eOihh/D222/jrbfeAlAezP3nP/9B8+bNceGFFwIoD94cDodXJuGFF16AEAIjR44EUHVK+frrr8drr72Gd999V55RYbPZMHfuXAwcOFB+TV588UWvGWk7duzAtGnT8NhjjyErK6tGs3j8VVxcjKuuugonTpzA0qVLNUsDAOSZGi1btlTNdpo5cyZee+01PPnkk3jggQeC+liB2rx5M3bt2oVp06ZpniP9NS99YCl98skncLvdtSpDSa6//no88cQT2LBhgzw7ac+ePViyZIlq1s2wYcOQkpKC2bNnq4KQ2bNnIzY2FqNGjQrqNevDmDFjMHXqVDz33HP46KOPVIGIEAK5ubmqxQX9IWValBnZtWvXYvXq1V5lp9oYMWIEEhMTMX36dFx66aVeAcWZM2eqzKTXZY/N2LFj8f333+Ott97yel9Vvg6+fsZPnTqF/Px8tGvXTn4O+fn5OHXqFJo2bRrQYqT1gYFNkPXt2xdA+eqmN910E0wmE6655pqgfABdcskl+Mtf/oIZM2Zgy5YtuOKKK2AymbBv3z4sWrQIb775Jq6//nrN+9dlj83DDz+Mb775Btdccw1yc3Px0UcfqW5XNvc9+eSTWLRoES699FI88MADKCoqwsyZM9GjRw+vDNL8+fNx5MgRecXdFStW4MUXXwQATJgwAa1atYLBYMAjjzyCp59+GoMGDcKtt94Kl8uFOXPm4Pjx415j8SUzMxMPPvggZs6cCYfDgf79++Orr77CypUrsWDBAvmNMTs7G71798bNN98sb6Hw888/44cffsDIkSNx3XXXVftYAwcOxLhx4zB16lScPn0a7du3xwcffIDDhw9jzpw58nm+ViCWAtX+/ftj9OjR1T6WL99++y22bt0KoDyjtW3bNvk1vfbaa9GzZ08A5eWZdevW4fbbb8euXbuwa9cu+Rrx8fGqx3/rrbfw3HPPqf6K/PLLL/HYY4+hQ4cO6NKli9f/h8svvxzp6el18liBPC+JVMKtKjC57LLLAMDnNN0FCxagWbNmVf7V7M/PL1CegXvvvfcwatQoPPLIIzCZTHj99deRnp6Ohx9+WL5eTEwMXnjhBUyePBnjxo3DiBEjsHLlSnz00Ud46aWXkJKSIp8bjGsuW7YMl156KZ555pk6beBWateuHV588UVMnToVhw8fxujRo5GQkIBDhw7hyy+/xN133+01xbo6V199Nb744gv86U9/wqhRo3Do0CH85z//QdeuXf1epsEfiYmJmD17NiZMmIA+ffrgpptuQpMmTXD06FF8//33GDx4sPyHky912WNz66234sMPP8SUKVOwbt06XHTRRSguLsavv/6Kv/71r/J7la+f8alTp+KDDz7AoUOH0Lp1awDlv8+33XabXL4PK6GZjNWwvPDCC6J58+ZCr9f7tUDf+vXrVfeXpj96rtqqteLtu+++K/r27StiYmJEQkKC6NGjh3jsscfEyZMn6/y5aZGmXWr987Rjxw5xxRVXiNjYWJGcnCzGjx8vsrOzA7qu53THBQsWiAEDBojk5GQRExMjBg4cKD777DO/n4PL5RLTp08XrVq1EmazWXTr1k189NFHqnPOnz8vbrnlFtG+fXsRGxsrLBaL6Natm5g+fbqw2+1+P1Zpaal45JFHREZGhrBYLKJ///7ip59+qvZ+dTHdW5qe7uufchpuq1atNM9r1aqV6pq+pmBrLVjp6/9fbR8rkOclRPn/6+bNm3stHumpVatWXo8vhBC7d+8WAMSUKVOqvH8gP7/Hjh0T119/vUhMTBTx8fHi6quvFvv27fN53XfffVd06tRJmM1m0a5dO/HGG2/4nMJb19f89ttvvZYk0FLT6d6Szz//XAwZMkTExcWJuLg40blzZzF58mSxZ88e+RxpgT5PEydOVP1/c7vd8u+2xWIRvXv3Ft99953XecoF+pS0fu+03sOXLl0qRowYIZKSkoTVahXt2rUTkyZNEhs2bNB8vYKhpKREPPXUU6JNmzbCZDKJjIwMcf3114sDBw7I5/j6GZd+l5TLlkjP1fN3KRzohPCjWYKIiMjDY489ho8//hj79++vVV8RUV3iysNERFQjS5cuxbRp0xjUUFhhxoaIiIiiBjM2REREFDUY2BAREVHUYGBDREREUYOBDREREUWNBrdAn9vtxsmTJ5GQkBB1O5oSERFFKyEECgsL0axZsyr3xGtwgc3Jkye9dmQlIiKiyHDs2DFkZmZq3t7gAhtpT59jx45Vu2U8ERERhYeCggK0aNHCa28+T2EV2Dz77LN47rnnVMc6deqE3bt3AyjfcPDhhx/GJ598ApvNhhEjRuDtt9+W95fxh1R+SkxMZGBDREQUYaprIwm75uFu3brh1KlT8r/ffvtNvu2hhx7Ct99+i0WLFmH58uU4efIkxowZE8LREhERUTgJq4wNABiNRmRkZHgdz8/Px5w5c7Bw4UIMGzYMADB37lx06dIFa9aswaBBg+p7qERERBRmwi5js2/fPjRr1gxt27bF+PHjcfToUQDAxo0b4XA4VFu4d+7cGS1btsTq1as1r2ez2VBQUKD6R0RERNEprAKbgQMHYt68efjpp58we/ZsHDp0CBdddBEKCwuRnZ0Ns9mM5ORk1X3S09ORnZ2tec0ZM2YgKSlJ/scZUURERNErrEpRV155pfx1z549MXDgQLRq1Qr/93//h5iYmBpdc+rUqZgyZYr8vdRVTURERNEnrDI2npKTk9GxY0fs378fGRkZsNvtyMvLU52Tk5PjsydHYrFY5BlQnAlFREQU3cI6sCkqKsKBAwfQtGlT9O3bFyaTCYsXL5Zv37NnD44ePYqsrKwQjpKIiIjCRViVoh555BFcc801aNWqFU6ePIlnnnkGBoMBN998M5KSknDHHXdgypQpSElJQWJiIu6//35kZWVxRhQREREBCLPA5vjx47j55ptx7tw5NGnSBEOGDMGaNWvQpEkTAMAbb7wBvV6PsWPHqhboIyIiIgIAnRBChHoQ9amgoABJSUnIz89nvw0REVGE8PfzO6x7bIiIiIgCwcCGiIiIogYDGyKKWKV2V6iHQERhhoENEUWkZ7/Zia7P/ITd2dwmhYgqMbAhoog0b9VhCAH8a/G+UA+FiMIIAxsiimhWoyHUQyCiMMLAhogimsXEwIaIKjGwIaKI43ZXLr9lNfFtjIgq8R2BiCJOoc0pf21lxoaIFBjYEFHEyS9xyF8rszfhTAiBv8zfgHvmb0QDW/CdqF6F1V5RRET+KLRVBjZlDhd+/SMHybEm9GudEsJRVS2vxIGfd+YAAM4V29E43hLiERFFJwY2RBRxHK7KjMfx86W488MNAIAD06+CQa8L1bCqpNdVjsvudIdwJETRjaUoIoo4DldlYHC22C5/fSy3JBTD8YtLUX5iYEMUPAxsiCjiOBSBQam9spF4/+miUAzHL0535ZjLnNwKgihYGNgQUcSxKTI2RWWVgc2p/NJQDMcviriGe1wRBREDGyKKOMqMjXLqtyuMZ0gpS1GlDgY2RMHCwIaIIo6yebhIEdg4wzmwUYy5jIENUdAwsCGiiKNsHlYuCeMO4/VhVBkbO5uHiYKFgQ0RRRy7y3dgENYZG0WTDUtRRMHDwIaIIo5DI7BRlnvqypebj+PZb3bWeoVjl6p52Kl9IhHVChfoI6KIo7UOjCsIpaiHPt0KABjSvjGGd02v8XWczNgQ1QtmbIgo4mhmbIJYisordVR/UhWU0725QB9R8DCwIaKIsunoeUz/YbfP24IZ2BhruVWDMpsUxq1ARBGPgQ0RRZQxb6/SvK2uAxtlX43nHlTrD+fixe/+8LnY3vliO2weqwsrm4fDefYWUaRjjw0RRY26nhVlU5SMPDM24/6zGgAQazZgyhWd5ONnCm3o/9KvaJESg5WPDZOPK6tnzNgQBQ8zNkQUNeo6Y6NcSE+vUYraf0a9P9Xv+88CAI7lqrd3UDYP13aGFRFpY2BDRFGjzgMbRTnJ3+qRVgDkVmVsGNgQBQsDGyKKGnU93bvMURmNKDMuSp4PqYxrhOJGNg8T1Q8GNkQUMaor4dT1An3KUpRT49regU1lZKPc04rNw0T1g4ENEUUMh0bWRFLXzcPKwEZr7RwB9WMqMzbKmVGq5mGmbIiChoENEUUMRzUZmZpkQn7cfgq7swt87ritLkX5l7EBKiMb5awqdcYm4GESkZ843ZuIIoZTI2si3x5gxLD24Dncu2CT/P3To7rgzovayt8rm4ere+zKMVSepw5sKs9hKYooeJixIaKIoczYXN83E41iTarbAy3xbD6Wp/r+xe93qb63qUpRGhkbrzEqAhtljw57bIjqBQMbIooYUnBgMujw2rgLYDbqfd7ur+r2bFIGM/7OinI4Kw8oMzZu1awoBjZEwcLAhogihjQzyWTQq/4rCXQdG89tDzwpr6fd36M+bnf5U4ryf4xEFBgGNkQUMaSgQdrewDNj4xnYbD+ej1UHzmper9Tuf8+Ov9O9tUpRyuZhwYwNUdAwsCGiiOGZsTEbPEtRAt9vO4Uftp8CAFzz1m/483trcSy3xOf1zpfYq3k83wv0VZUZUgY2ZRoZm2DuQk7U0DGwIaKIIQUNRkN5xsazFFVsc2Lywk3464JNOFNok49vPZ7n83q+SlHKY06NUpQyePFuHlb02GhkbBjXEAUPAxsiihhSoGHUSz026n2ZShXrzhxUbE55/Lx6Q0qJ3ekdYZTYlMGIshTle00bz0ZgZUOyuseGzcNE9YGBDRFFDCm4kHprPDM2ykzKMUUwc7rABl98zXRSrm6s6rFRZm+qmE2lHINL4/5ceZgoeLhAHxFFjOqah5XZkqOKvhqtqdq+GoKVx5RZGmXA4vDotymyOXHXBxtwWZc01XlaU7wZ1xAFDzM2RBQxpKDDqNE8rAwqjpwrVhz3HUn42v/JoVFyUgc8lV/bnW7MXrYfqw+ew4vf7/LY+FL5deVjsBRFFDwMbIgobNmdbsz8eTc2HskFoF6gr/y/2hmbI+cqMzZaG1j62oJBKzBRZmkcHpmcrcfyK8egkbFRT/f2ORwiqgMMbIgobL3+y17MWnoAY2evBlAZdEilKJNHKUrZrHuuuLKvRjOw8XHcqdFj49aYIeUSwGFldsiPRfk43ZsoeBjYEFHY+nzTcflrp8stByiVKw+rZ0UpsyX5JQ7FfbVKUT4yNk7fWRblqao+GrdQz35SZmk0e2wY2BAFCwMbIgpbyrVouj/7M5bsOg1Ae4E+ZSmq0OasPK5ZiqpmVpTLd8bG5fG1Mk5xa2R5lEkaJmyIgoeBDRFFhDKHG99uOwlAe4E+JaGRYVHynbHxXYryDGYk5dkXdWlKfZs0Ht9fE1HdYmBDRBFDCkQsFb01ntO9tWiXonz12GgEMxolpqoyNlrBkIuBDVHQMLAhoohjNhoAVJ2xUdIsRfkIeOwa+0MpsyxujyBFeRV/1q5hKYooeBjYEFHEMcs9NrpqziynPd3bR8ZGcx2a8q/tTre6FOUWqqBHa+0alqKI6gcDGyIKS1VNidbaUkFLILOi1KsNq3tnpn6xDRc89z+czC9VHPc3Y+M7+CGiusXAhojCUondqXmb1GPjuY6NlurWsfnpwYuQ1TYVgLoU5ZmZ+XjdMZQ6XJjz2yHFcXVpSqtHR12KYmBDFCwMbIgoLJXYXZq3WQLM2Gj12Dgqoo0Eq0kOklRbJ2gEKcq1bpxutypjo7UJJveKIqofDGyIKCwV27QzNlIpKiXO5Ne1qsvYmPQ6mCpWM1YHJr63R1BvggnlbG/VWjrqvhr4PE5EdYuBDRGFpcKyKgKbikxNeqLVr2v56rFxuYWcOTEa9HL2x6HMzLh8Z1ycHseVt9k1ViFmjw1R/WBgQ0Rh6bRi1WFPUsYmw8/AprpdvI0Gnbzon1OjlKQqRXmcowxTbBp7RWn12xBR3WJgQ0RhKaegTPM2qccmI8m/wEaZRZEo+2fMBr2cBXK4NFYermKvKK1VjrX6aliKIgoeBjZEFJZOVxHYSAv0xZqNuOaCZmiVGlvltZw+UiTKzIxRX5mxUU3xdqsDGIl6GrhQZWO0ZlUJjyDH7RaYMGctHv6/rVWOnYgCw8CGiMLKobPF+H7bKeQUVF+KAoB/39wb//eXrCqv6XC58cWm43j9f3vkAEMZgBj0OrnHRllK8uylUV5P4nIL1bVUPTYas6JcboEDZ4qwct9ZfL7pOGxO7RlgRBQYY6gHQESkdOlrywAAcebyrMyTV3XGifOl+GD1Efkczz2i9LqqVyB2uASmVGRGru3VHO3T4mFzlAcgMSYDdDod0hLKy1rZisX3VGUpl+/AxrN/RxnYaG+pIKAc8vliBzKSDFU+ByLyDzM2RBSWiivWsYkxGzF+UCvVbRavwMb/65ZWXLfMIV2/PKBo3bi8nHX4bIl8rjITU6bIqqhnS6mvrx3YqKd+lzkqzztbpJ2dIqLAMLAhorBmUTT2SjwzNoYAIhspWCmVAhtTRWCTGgcAOHyuWD7Xpgg+pEDI3+sDnntFVX7tEkJVfjpXbFdd4+CZIkxesAn7cgr9ekwiqsRSFBGFNbNRjzhL5VuVQa9Dj+ZJqnP0AQQ2UulIytxYTOVBkhTYnC60odjmRJzFqM7YOLxnVvm+vu9dwD2zN8rrnfPI2Az7x/Ly48U2fHJ31f1DRKTGjA0RhTWTQY/k2MoVhjMbxaBxvEV1TnU9NkpSr4xnxiYp1oRGFY9z5Fx5OUpZVqpJg69Lc7p3ZWAFAHklDp/3L7axqZgoUAxsiChs+FqR12zUq/aEMvrIzhgCCGzOFtnw276z8l5UUmADAK08ylE2VWDjX8ZGSZWx8cjeKHt2yjSCppQ4c8CPSdTQsRRFRGHDVx+LyaAOWnz10+gD+BPtwU+3AAAGtkkBUNk8DABNEsozQVIGRb3vk/+PIalqS4Uyh+8yl9C4DxH5hxkbIgobpT4CG+9GYe+3La1SlF7nO8MDAGsP5QIArIqMTWxFkFNiL9+nqrbry2itYyOE+rnaFF9r3YeI/MPAhojChrLvRCLNiEqryKYM75LmdY5WKcpk0Fc7Y0pZipK+LnO44HYLVSNwTQjV2jXKr4UqmFEGOU5V+apWD0/UIIVtYPPyyy9Dp9PhwQcflI+VlZVh8uTJSE1NRXx8PMaOHYucnJzQDZKI6pSvDImUsfn83gvxwnXdMPnS9l7naM2KMhv0mhkbiSqwkTM2LtWMqJrS3lJBqMpuZQ4Xlu89g5H/XIHNR/NU5xFRYMIysFm/fj3eeecd9OzZU3X8oYcewrfffotFixZh+fLlOHnyJMaMGROiURJRXSu1ewcTUmDTIiUWE7Jaq0pHSr7iF6NBV23GxmqqfBuUgpwSu6tGzcKetGZFudzqLE2Zw42J76/D7uxCTJy7Tj7OuIYocGEX2BQVFWH8+PF477330KhRI/l4fn4+5syZg9dffx3Dhg1D3759MXfuXKxatQpr1qwJ4YiJqK746rExGfx7m/IVwJgMehiruX/7tHj5a6nHpszh8rkjeKC01rERQiC/tHKKtzJ7o9prykdkY3e6UWxz1npsRNEq7AKbyZMnY9SoURg+fLjq+MaNG+FwOFTHO3fujJYtW2L16tX1PUwiCgKfpSg/Axudjz4bf3pshnaq7NmxqjI2tV9DRitj4xYC2fmVi/KVaQRRTh/lsOGvL0e3Z35GYZnvtW+IGrqwmu79ySefYNOmTVi/fr3XbdnZ2TCbzUhOTlYdT09PR3Z2tuY1bTYbbLbKN5CCgoI6Gy8R1S3PDSUB71lRWnw1EJsMOrhF1YFNorVy8b9Yc/lbYmkdZWxcGuvYuNwC2QWVm21qbdew9Xg+juWWoEVKrHzsaG754oFbjuXhog5Naj1GomgTNhmbY8eO4YEHHsCCBQtgtVrr7LozZsxAUlKS/K9FixZ1dm0iqlu+ZiH5m7Hxvb6NrtpVia1mRY9Nxdeldhe+3XrKr8etivDI0iiPZ+eXyd/bqtiH6qJXl/o87isIJKIwCmw2btyI06dPo0+fPjAajTAajVi+fDn+9a9/wWg0Ij09HXa7HXl5ear75eTkICMjQ/O6U6dORX5+vvzv2LFjQX4mRFRTTh+BjcnPjI3P+EWUNxBXdR9l4BRjKs/YFNmc+O/Kg349blW01qRxCaHa+NJXb1F17E52FhP5EjaBzWWXXYbt27djy5Yt8r9+/fph/Pjx8tcmkwmLFy+W77Nnzx4cPXoUWVnam8RZLBYkJiaq/hFRePKVhfBceViLr4yN0DguiTEZVL050nTv4+dLUFjRoLvwzoF+Pb4vWj02JXaXKptT4mP9Hl+U5SwnF7kh8ilsemwSEhLQvXt31bG4uDikpqbKx++44w5MmTIFKSkpSExMxP3334+srCwMGjQoFEMmojrmM7Dxc78EXz02Br3O5zRwSYzH1HFpVtTZovJsSqLViE4ZCX49vi9ujXVsPOVrbILpSbl4H0tRRL6FTWDjjzfeeAN6vR5jx46FzWbDiBEj8Pbbb4d6WERURzx7bPQ67cX3PPmaFWXQ6aq8v+eaOJ6BTrzFGNDO4Z60MjaeCv2cvq0sZzlYiiLyKawDm2XLlqm+t1qtmDVrFmbNmhWaARFRUHmWV4wB7G7pq8dYr9dVufKwcnE+QL0hJgDEWYx+B1a+aK1jI8lItOJ0YVmVQY90Hb1ep8rY1MXKyETRKGx6bIiIPKdYV7cGjepcn6Woqq9RXcYmzmIMaAyeqsvYJMYY0SjWXO11pCDG5WIpiqg6YZ2xIaKGxenx6V/dPk9KWqWoqq7hGbTEemRsEqxGzQ02/eF2AwvXHsXrv+zF+RK71+1xFiPcAqoZUr7YXW5YTQZVoMTAhsg3ZmyIKGw4PDM2fs6IAnxnZgz6qveK8rzFqxRlNiKAapgXtxB48svtOFtkU039lsRbjGilWHxPqWN65VYPUiZLWaqzORjYEPnCwIaIwoajFhkbrcCmqnVsPJkNetUsqjhL7TI2voIZpTizEZd2TvN5m1Gvl9fYkQIb5fVqsvYNUUPAwIaIwobn3kiB9Lf4ij/0Oh0MVaRcEhTbKZRfQydvqwAA8RZDrXtsqlqHx2TU4/Ku6T5vMxp08nYSvgKbMmZsiHxiYENEYcOzbySgWVEa69hUlfVJjjV5HVM2FMdZjNDpqi5nVcXtFj57f+Tx6YD0RN9byBj0isDG5R3YsMeGyDcGNkQUNjzXsQloVlQNemx8zUhSNhDHWYwBj0PJJUSVpSwpm+Tr+ka9zqsU5eTKw0TVYmBDRGFhw+FczFt1WHWstrOiBrdvXOU1WjeO8zqmDGziKwIbf8bROtW7Cdjtrjooktbe8RX8GPV6OWNjqwhslOviHM0twcWvLsW7Kw5UOzaihoSBDRGFhfsWbvY6FljGpvLrv1zcFi/9qTvuGNIGRo3dwUf1bIpbBrX0Op4SV5nFia8mY5PZKAYA0KZxHAa2SfW63S2E78055TFXjM3HOb56bJQZm9/3n8PR3BJM/2G39gMQNUBcx4aIwoKvWT6BBTaVAUybxnG4aUB50GL2EdgMbp+KWX/u4/M6jRSBjVSKMmkERw9f0REOp8DQTk3wxq97vW53uYXXooNKUibI19M0KEtRPnpsiMg3ZmyIKCx09rHZZCBTtZXlIuU2CBaT99ucr2BHkhLrf8YmwWLCDf1bIC3R6nNPKZdbyGUkX6Tr+sr2GPU6eey2iqCPgQ1R9RjYEFFYaJ4c43Wsqqna3udWBhbKnhVfQYxU4vGlkWKmVJylvN9Gq8fGpLiOr+Cn2F715pbSfWaO64lJF7b2uk3q95GyWZ4rMxORNwY2RBQWfG0SGUjzsPJcQ3UZG6PB65ikkY8eG63MkXKNGl8Zm+rWmpHGmZZgxbPXdlP14xgNenlNnaKK3b+1MjZVlbuIGhoGNkQUFnxlI2o63VsV2PjI2FiqyNhktUtFvMWIpBgTmlVkkbTW01Fmg3wFNoGMufxxdKqv4yoyNiW2qktRX2w6jpveXY1T+aUBj4Eo2jCwIaKwIH1oX92zqXwskIyNssFXGTD4KjtVVYrqnJGI9U8Nx8rHL612HRv1Y/o91Mr7eARDnsGZ9PhSSUsrsHnii+1YczAXD3yyJfBBEEUZzooiorAgZWwsijJRTTM2yuyJr+neVTUPA+WbYcagchyaPTbKjI1HtsWffhjvjI0egFu+hhzYVJSiqluUb92h3GofkyjaMWNDRGFBykZYFT0xddFj4+sKVZWifF5bo8fGbPQdTFWVEVJdV19VxkaPOLOUsambWVEbj+Ri89HztboGUbhjYENEYcF3xqZms6KqC4j8DTyqG4fZoBirIrDxN3DSV9djUzErq7ia5mF/lDlcGDt7Nf709iocOltc4+sQhTsGNkQUFlwVZRZLHWRslAGDr57eGLP2rKjqrq1kMvp+zLrI2BgNlTuNF1fTPOyPMsUCiP/bmV3j6xCFOwY2RBQWnBUbYFqVGZsAFuhTZlWq2ngSAPq1SglobP702BhqUIqqdlaUR8amNuvYOLkzODUQbB4morAgZSPqJmPj+5zLOqfB6Rbo26pRQGPTXsdGOd278nhVzcmxZgNKKnpmvAIb1Syryh6bkopZUb7W+vGXMttTi8sQhT0GNkQUFip7bKpezVeLMrujte7MnEn9azQ27R4b37OitPaWAgCrqTKw8QzcvDM26uZhKatVE8qMDRcwpmjGUhQRhYXKWVHVT7P2RT0rqu7GBQAmzVKU1mrH2j08qtWKq5wVVbfNwy6XMrBhZEPRi4ENEYUF3xmbms2KUk69HtY5HQDQJMFS47FpZY6MGqUoX6sd+xpbVc3DJlXzcEVgU4uARLkGjmBgQ1GMpSgiCgvyrChF83AguxRorWPTPi0eKx+7FCmKPaACpeyx+VPv5vhy8wk8cWVn1Tn+rmOjPM9zGwajKgOkl/eqKra7IITwah7W6/wvK7lYiqIGgoENEYUFp48F+gJZSE81K8ojE9IiJbZWY1Nee0yf5ph2dVfVLuCej1llYKO4ybMpWfk4Rr0OsRWlKJdbwOZ0w+Uxm6lDWgL25BT69RzUPTaMbCh6sRRFRGHB5WOBvtgA1pvRytjUBWWPjcVoQEqcGTqPbIsy+2KqYpq68jzPUpvnc4hV9OoU25zw7B3ulJHg3xMAMzbUcDCwIaKwIM34UU73jqmiCdeT0aDdu1Jb/mRj1HtF6TWDK1VgU8UmmEaDDkaDXs5gldhdcrlO0j4t3s9nwIwNNRwMbIgoLMizohQZmxiz/9VyZTATG8D9/Lq2ImjSWqPGoFMHJVqxlfJ4VQv0SbdJa9kU2Zyq4OTNm3oh0er9PJWNwSfySvHqT7uRU1CmztgwZUNRjIENEYUF6UNbuU1BIBkbZftJvI8P/NpQroujzCgpKeMdg17n1RgsUZeiqpgVVfGYUobI6RJyQHLzgBa4rldzmHxkj5Qxy70fbcTbyw7grws2sRRFDQYDGyIKC1KZRZm1iDH7/xZV5qzcCymujjM2yiZmrYyNzmMatz+lKM+SmXoDUJ3qv063Ww7+pGv4WghQOa172/F8AMDGI+dVx1mKomjGWVFEFBakD21lQ22Myf+3KOUmj3XdPJwUUzkDSmumltljOwRlaeq2wa3Rrkk8hrRvjHsXbJKPey7Qp9pOoqL8JQU/LreQsy7SMV9BlltjGyiHovOY69hQNGNgQ0RhwfNDGwAuaJHk9/3LHMHb2FHa2gBQZ1WUlCsmW4x6VdCSYDHilkGtAKh7bDwzNtYqMzaVgY0U/GlnbLzH+OAnm+WvWYqiaMbAhojCQmXGRoe1T16GglIHmibF+H1/myJjU9diFNPOtWZFKc+JNRs0t1ioqsfG6mMDUKm/R5WxkbI5PqaVK3tpEq1GFJSVr1p8vsQhH2cpiqIZAxsiCgvKjE1aohXpidaA7q/ssalryiZmrcAm1iOwUQYwMarApvI+noGNMhskBTTKjI1nj42vUpRy5pTnWjsSZmwomjGwIaKQE0JZZqlZf0wwS1HKMpPW+JTBS4zZqApglPdXr3ejnbExeGRlXG63V7nOVynq9V/2olVKLJolxyC/1OF1O8AeG4puDGyIKOSU5RNjABtfKj0+sjNWHziHOy9qU1fDkqUnVr+BpjJ48SxFKQMW9QrF6ueqzth49Ni4vIM/XyscL1x71OtYx/R47M0pkr9nKYqiGQMbIgo5ZfnEUMV2BFXplJGAbc9e4TOLUVu9WiTjvkvbI7ORds9PTUpRnmUtVcZG7rGpnBXl9Axs/NhLq3lyDPq1TlEFNq7gJbeIQo6BDRGFnDpjU/Op2sEIaoDyXpVHRnSq8pwY1awogyposWo0D3tnbLzvo+yxcXsENlpr6ij96+Ze+HXXadUxlqIomnGBPiIKOVXGpo7XoKkvyllROh0QZ1EEOhqlKK8F+hQBUJN4S8U5lbOinH702HhqmRKnWocHYCmKohsDGyIKOWXGxnNjyEjhuXCfcr8qVSlKcZpnKcrurKwRNUkoD2zU69i4Vceq2kVcYjXpfQQ21d6NKGIxsCGikJOW+9frvFfjjRSeWyrEKxb187cUVaCYxSTdp7LHxg1p8WBDABkbq8mgWmAQYMaGohsDGyIKucppzJH9ljSubya6Nk3EkA6NVc3EMRrTxT0X2LP62PTTV8ZG3lKhmuZho14Hk0GPWI/rMq6haMbmYSIKOaerdmvYhIuZ4y6Qv1ZuxKlaB0eRsfFs/v3zwJZYeygXV3bPkI9VrmMj5NdJ72fGRnpcZZAFMGND0Y2BDRGFnK99oiJdqWKLh5Q4s/y1stTmGZjEWYz478R+qmNSFsvhEnJAYvSzx0YKbGIY2FADEtl5XyKKClKPTU3XsAlH54pt8tfKkpGhir2ifFH22HjugF59xqb8dmUjM8DmYYpuDGyIKOQ8pzFHgyKb772rAm0j8r27d/lt1QU2MRqlKK5jQ9GMgQ0RhVy09NgoPXVVF5iNejx1VRfVcX2A09nlHhvVlgrqDTK1aJWiXEzZUBRjjw0RhVy0zIpSGtAmBTueHeE1cynQ4M3X7t7+ZrYqS1GePTYBDYEookTPuwgRRSzPPZCiha/p2AFnbBQrD0sBoL/XkDI2ViObh6nhYGBDRCEXjbOitAQa2PjqsfE/Y1Me0Hguesi4hqIZAxsiCjmnx1YB0SzQfTpVKw9LmS0/Z48pFwa8uGMT+WtmbCiaMbAhopBzRWkpype66LHxdz8tq2LzzXmT+uP1G8oXEGRgQ9GMgQ0RhZzcFBtF69ho0QXcY1O58rA7wABQvfmmDpaKXhs2D1M0Y2BDRCHncqmnMUezyzqnAQDizN77QvkivSZOt4CjomTnz+aXgPfeU1I8xHVsKJpxujcRhVw0LtCnZVjnNCy8ayA6pCX4db5yHRuHSwps/HudLJ6BTcXry4wNRTMGNkQUcg2px0an0+HCdo39Pl/ZY+Nwlr9O/mZsYrwyNlJgw8iGolf0532JKOxJs6IaQsYmUMpZUZUZG//euj1fT+lbN1M2FMUY2BBRyDWkjE2gVBmbAEtRuSV21feVGZs6HCBRmGFgQ0Qh15B6bAJlNFSuPOxwBVaKyi1SBzbShCyWoiiaMbAhopDz3NyRKhl9Zmyqfp1S48ww6HW4bUhr1XFmbKghCKh5+Jtvvgn4AS6//HLExMQEfD8iajiYsdEml6Jcbvl1qq4U9c+beqF3y0aIt6jf4qXAhtO9KZoFFNiMHj06oIvrdDrs27cPbdu2Deh+RNSwuCoyEf5uFdCQSMFemcMtHzP52FxTKcZk8ApqAEXzMAMbimIB532zs7Phdrv9+hcbGxuMMRNRlGHGRpuUsSl1uORjpmpKdp4L80l0LEVRAxBQYDNx4sSAykq33HILEhMTAx4UETUsge6B1JAYK4KYMmVgU01mK0ZjVWODnuvYUPQLKLCZO3cuEhL8Wy0TAGbPno3Gjf1fiIqIGiZnRSnKXE2JpSHyzNjodOpp8WN6NwcAJForS0+tUnxnyyu3VAjGSInCA1ceJqKQs7saziaYgZLKc6X28sDGZNCrNtKcMbYHbuzfAl2aJeI/yw7g6p7N5CninnRceZgagID/PFqyZAm6du2KgoICr9vy8/PRrVs3rFy5sk4GR0QNgzPAFXUbEqmhWsrYmDz6kCxGAwa2TUWi1YTHRnZG12ba5X/pri422VAUC/hd5J///Cfuuusun70zSUlJ+Mtf/oLXX3+9TgZHRA2DtD6LmYGNFyljY6uYFVXdjKiqVE73rv24iMJVwL8hW7duxciRIzVvv+KKK7Bx48YaDWb27Nno2bMnEhMTkZiYiKysLPz444/y7WVlZZg8eTJSU1MRHx+PsWPHIicnp0aPRUThw8FSlCapn8ZeB1ktboJJDUHAvyE5OTkwmUyatxuNRpw5c6ZGg8nMzMTLL7+MjRs3YsOGDRg2bBiuu+467Ny5EwDw0EMP4dtvv8WiRYuwfPlynDx5EmPGjKnRYxFR+Ah0c8eGxOgxtbs2WS1uqUANQcDNw82bN8eOHTvQvn17n7dv27YNTZs2rdFgrrnmGtX3L730EmbPno01a9YgMzMTc+bMwcKFCzFs2DAA5bO0unTpgjVr1mDQoEE1ekwiCj0GNto8NwatTVaLWypQQxDwu8hVV12FadOmoayszOu20tJSPPPMM7j66qtrPTCXy4VPPvkExcXFyMrKwsaNG+FwODB8+HD5nM6dO6Nly5ZYvXq15nVsNhsKCgpU/4govFRu7shSlCfPRQtrVYqquCu3VKBoFnDG5umnn8YXX3yBjh074r777kOnTp0AALt378asWbPgcrnw1FNP1XhA27dvR1ZWFsrKyhAfH48vv/wSXbt2xZYtW2A2m5GcnKw6Pz09HdnZ2ZrXmzFjBp577rkaj4eIgqfM4cKtc9Zh3eFcAMzY+OKZsalNKYoZG2oIAg5s0tPTsWrVKtx7772YOnWqHPnrdDqMGDECs2bNQnp6eo0H1KlTJ2zZsgX5+fn47LPPMHHiRCxfvrzG15s6dSqmTJkif19QUIAWLVrU+HpEVHd+3ZUjBzUANNdfacg8g70mCZYaX4vNw9QQ1GiBvlatWuGHH37A+fPnsX//fggh0KFDBzRq1KjWAzKbzXL/Tt++fbF+/Xq8+eabuPHGG2G325GXl6fK2uTk5CAjI0PzehaLBRZLzd8IiCh4PMssZpaivHhmbJomWWt8LXkTTKZsKIrV6s+jRo0aoX///hgwYECdBDW+uN1u2Gw29O3bFyaTCYsXL5Zv27NnD44ePYqsrKygPDYRBVecxw7ULEV58wz+MmoV2HAdG4p+YbWlwtSpU3HllVeiZcuWKCwsxMKFC7Fs2TL8/PPPSEpKwh133IEpU6YgJSUFiYmJuP/++5GVlcUZUUQRynMFXJaivHlmbNITax/YuBjZUBQLq8Dm9OnTuPXWW3Hq1CkkJSWhZ8+e+Pnnn3H55ZcDAN544w3o9XqMHTsWNpsNI0aMwNtvvx3iURNRTUmzoSQsRXnznN6dFKO9jlh1uI4NNQQBBTYOhwMffvghAGDChAkwm811Opg5c+ZUebvVasWsWbMwa9asOn1cIgoNaf0aCUtR3jwzNvGWmv89qtdzVhRFv4DeRR555BGkp6cjLS0Njz76aLDGREQNhGdgw1KUN8+Vh+OttQhsKmIkrmND0Syg3xC32w232w2XywW32139HYiIqmB3emZsWIry5JmxSahNxobr2FADENCfR//4xz+Qm5uLvLw8vPbaa8EaExE1EN49NszYePKcFVWbjA17bKghCOg3xGw2Y9KkSUEaChE1NCxFVc8zY+M5RT6gaymmewshoNMxQ0bRh+8iRBQynoFNrNkQopGEL88sVpy59qUogGvZUPRiYENEIWP3CGwax3OVcE96j4yNZwYnoGspAhuWoyhaBRTYbNu2LaCm4Z07d8LpdAY8KCJqGBxO9Ydrci3WaKHq6RTv+GwgpmgVUGDTu3dvnDt3zu/zs7KycPTo0YAHRVQbucV2/HflQWw7nhfqoVA1PEtRntkJqlvM2FBDEFCxVgiBadOmITY21q/z7XZ7jQZFVBuzl+3HeysPAQAOvzwqxKOhqniWoii4lHEjAxuKVgEFNhdffDH27Nnj9/lZWVmIiYkJeFBEtXHgTLH8tdstmAUIY8p1bKZc3jGEI2kY1BmbEA6EKIgCCmyWLVsWpGEQ1R3lxoqlDletpsdScEmlqAcu64C/XdYhxKOJfjpmbKgB4KwoijpORYN7sZ3N6+FMCmzMRr4V1QfVdG9WASlK8d2Eok5RWWUwU2p3hXAkVB1p5WFupVA/DGwepgaAgQ1FnUJbZWBTbGNgE86cFWVDz40eybeWKf5N3NDCUhQ1BHw3oahTqMzYOFiKCmeuirKhkRmbKn351wtxSccmmDOxX62uo9PpFPtF1cHAiMJQUBfoI6pvx3JLcLbIJn/PjE14kxq9a7OabkPQu2UjfHD7AHRIT6j1tfTyflGMbCg6BbxA39mzZwEAbdu2DWixPqJg23jkPC56dalqD5wS9tiENZdcimJgU1/0zNhQlAsosElOTsahQxULnx0+zOwNhZX3fzvkdayEs6LCmlPO2LAqXl+kHb1dzNhQlApogY+xY8fikksuQdOmTaHT6dCvXz8YDL534z148GCdDJDIH7nFdizZfdrreLGNgU04qyxFhXggDYicsWHKhqJUQIHNu+++izFjxmD//v3429/+hrvuugsJCbWv+RLV1qYj51Hq8C47FZQxsAlnThczNvWtsscmxAMhCpKAl2QdOXIkAGDjxo144IEHGNhQWNh1qsDn8YIyRz2PhAIhlUPYY1N/pMCG070pWtV4rfm5c+fKX0vd9Tod35y05Jc4YDHpYTX5Lt1R7Rw4UwQAeHREJ/RqkYwVe8/gnRUHUVDKjE0446yo+lfZPMzAhqJTrfK/c+bMQffu3WG1WmG1WtG9e3f897//rauxRY38UgeGvLoE4/6zOtRDiVrnist3ks9ItGJw+8bISLICYMYm3MnNw/yjqN5Im8KyxYaiVY0zNn//+9/x+uuv4/7770dWVhYAYPXq1XjooYdw9OhRPP/883U2yEi36ch5FJY5sf1EPs4X25Eca8L8NUfQKT0BA9umhnp4UeFsUXlgkxpvBgAkWk0A1Iv1UfiRFugzcIG+esN1bCja1TiwmT17Nt577z3cfPPN8rFrr70WPXv2xP3338/ARkG5YNyuUwXQ63X4+9c7AQCf33sh+rZqFKqhRY3c4vLXODXOAgBIjCkPbPJK7CEbE1VPah5mj0394To2FO1qXIpyOBzo1897ee++ffvC6eRfyUrHckvkr/edLsLhs8Xy92Nnr+JfTrUkhEBusTpjk5FYXorafiIfR8+VaN6XQkvq82CPTf3RsXmYolyNA5sJEyZg9uzZXsffffddjB8/vlaDimTL957BiDdW4PONx+Vjx86Xyl8v3n1a7geRHDhTDKq5glKnvEt0Slx5YNO9eSIuyEyCEMDvB86GcnhUBfbY1D82D1O0q3EpCihvHv7f//6HQYMGAQDWrl2Lo0eP4tZbb8WUKVPk815//fXajTKC3DFvPZxugVd+2o0xfZpDp9PhqCJjs2LvGazYe0Z1n9OFZWifFl/fQ40apwrKA8dGsSZ51plOp8OgdqnYejwfO0/mh3J4VAV5SwX22NQbebo3F46nKFXjwGbHjh3o06cPAODAgQMAgMaNG6Nx48bYsWOHfF5DmQL+1pJ9+Gzjcfkv0NOFNpwutCE90aoKbHwpKOXMndo4mVce2DRLjlEd75BWvsbSEZaiwhYX6Kt/XMeGol2NA5ulS5fW5TgiXk6BDYc9PkBL7C44XG6cKbT5vE+zJCtO5pchn4FNQArLHIgzG6HX61Bsc2LD4fMAvAObOHN59qbMx4rEFB64CWb907EURVGuVqWosrIybNu2DadPn1ZtiKnT6XDNNdfUenCRJNbivfCe3elWfahOurA15q06LH/fu2UjnNx+KioCGyEEjuWWwmjQoUmCBSU2F9YdzsXhs8W4bXBrON0C54rtaF4RfOSV2PH+b4dwYfvG6NOyEb7afAJfbz2B88UO/HlgS5wvtsNs1EOv06GgzIH2afEwG/Q4W2zHc9/shNMt0DE9HntziuQxtGgUqxqTtSKw8bXVAoUHaeVhfQPJ7IYDA9exoShX48Dmp59+woQJE3Du3Dmv23Q6HVyuhvVhEmf2fintTjfszsqAr03jOPnrt/7cW840hDKwWbbnNN5edgC3XdgaOp0OWW1TUWx3YtvxfJwpssHlcuPLLSeh1wHtm8SjR2YSHC6BBWuP4PDZYrRuHIcT50thc2oX7F/6YRd0uvK9aVLizLhjSBt8veUE9uYU4V9L9nud//RXO3xcxZsyqAGAYZ3TVN9bjVLGhs0E4Yo9NvWP69hQtKtxYHP//ffjhhtuwN///nekp6fX5ZgiUpzFR2DjcsHuKv9QNRl0uKpHU2w5loc+rRrh6p7N5A/mvJLgBjb5JQ78tPMUdpwowOjezdAsOQZlDjc+WnMEc347BABYdyi32utsPpqHRYrZXgBw0M8ZXdJ7aG6xHTN/3uPznCHtG6NN4zjMX3NEPtY8OQZni2yqwGliViukxlvgcguczCuFxaSHUa9HVjv1YocxUsbG3rCC7EjirPj94HTv+qPjOjYU5Woc2OTk5GDKlCkMaipI/RxKNocbtopsgdmgR5MEC964sZd8e5OE8sXklNPBq3LkXDE+WX8M91zcDkmx5QvQOVxubDh8Hu+uOIBYixHZ+WW4sX8LnC4ow5ebT3hNJVcGDVVJiTOjVWosth7L83oD7N48EXqdDglWI3pmJqNJvAXL9p7B3Re1xdkiGx5etBXX9GyKJ67sgm3H82B3udEo1gyjXocXvv8Dh84Uo0mCBR/ePhCp8WYcOluMFo1ikWA1QqcDBrdPRYuUWHRtmgghgNwSOw6eKUas2QC9ToeuzRL9eg5WU3lDqs3JwCYUluzOweajeXhweEfNwIU9NvWPzcMU7Woc2Fx//fVYtmwZ2rVrV5fjiVixPjI2NpdbzthYfGx+2btFMgBg89HzcLuFvIeLpNjmxC9/5KDQ5sQ7yw/geEUANHtZ+Sy0OLMBxT6yERuPnPd73PcObYeHhnfEwbNFOJ5bimPnSzCgTQq6NUsCUN4L464oIR0/X4KmSTE+P6RuH9JG/np413TEmQ3Q6XTISMpQnffd/RfB7nRDpwNMhvLAo3vzJNU5I7s3lb/W6YDG8RY0jrf4/ZwkMSZmbEKl2ObE7fM2AAA6ZyRiVM+mPs+TZhGyx6b+cB0binY1DmzeeustjBs3DitXrkSPHj1gMplUt//tb3+r9eAiSbxG87DUY2M2eE9n7ZyRALNRj8IyJ46dL8GOEwXo37oR0hKt+OWPHNz14YYqH9NXUONLgsWI8YNa4b5h7WFzuGA1GZBf6sCe7EIM7dQEOp0OnTMS0TnDOxOSHGuWv870aM7VEu8jyFMyG+tnaq+0pk2Z0w0hRINZeiDUTuaV4sKXl8jfbzp6XjOwkT5c2WNTfyp7bEI8EKIgqXFg8/HHH+N///sfrFYrli1bpvrQ0Ol0DS6widVoHpbKIL4+zI0GPVqnxmJvThEumbmsxo8dYzLg2/uHoE3jOOSXOmDQ6XAktxhFZU60SIlFi5TKgEQKOuIsRq/p0dFGCmxcbgGHS8Bs5IdnfZj7+yHV94fOavdhySsPsxRVb7ilAkW7Ggc2Tz31FJ577jk88cQT0HNxLc1ZUVLTq0UjS9GmcZzX7B5fxvRpjkkXtkb3ZklyyarM4cL+00Xo1ixRfrOSthToGZtck6cRVaQeGwAoc7rqLVPU0J2uWLcpOdaEvBIHCst8N8e73ULOGhj5HlJvpBjSxe5hilI1DmzsdjtuvPFGBjUVEqy+ZkVVBjZaH6qdMxLx884czet+c99g9MxM9nmb1WTw6k+hSmaDHnpd+eyPUrsLm46cR+eMRGQkWUM9tKhWVFa+Ce7g9o3x/bZTKCzzvSmuU/HByr2i6g9LURTtahyVTJw4EZ9++mldjiWipSd6f1gqe2y0MjaXeqy9Un4tC/52WQcsfvgSzaCGqqfT6eRy1PfbTmHS3PUY+hpXzA62Ilt5INO04ndCK7BRZgwM7LGpN2wepmhX44yNy+XCq6++ip9//hk9e/b0ah5uSBtfApVrpiipmoc1ApteLZLx75t74/ttp3BJpyZonRrntR4L1VyMyYASuwsr9pVvPMrF+oKv2F4eyEiZMa1SlEvxwcrp3vVHz5WHKcrVOLDZvn07evfuDQCqTS+BhrPxZXXUpSjvwEdyzQXNcM0FzeprWA2KlLFR9kA5XW4YfcxSo7ohlaKaJpU3pxfZnD5npblciowNA5t6w3VsKNrVOLD54IMPkJmZ6dVjI4TAsWPHaj2wSGQ26lVbKNj8KEVRcEkNxMrpxKcLbVE/IyyUimzlMwGbJpdnbNyifGkCz2UAnIr95dhjU3+kGJJbKlC0qvGnbZs2bXD27Fmv47m5uWjTpo2Pe0S/xVMuwStje2BiVisAUilKe7o3BZ+UsVH2eWQXlIVqOA1CcUWPTeM4i1xi8lWOkhavNOp1XotTUvBUTvcO8UCIgqTGn7Za0X5RURGs1oY566RFSixu7N9SXoVYNd2bpY+QkFYfVm40WmLjSsTB4nS55d3U461Gebagrwbi6vrPKDjYPEzRLuBS1JQpUwCUR/1///vfERtbufiby+XC2rVr0atXrzobYCSSyk42p6uyFGXim3coSE3deSV2+Zj0wUt1T7kadpzFgASrCedLHD4DG4e8QSx/N+qT1GPDdWwoWgUc2GzevBlAecZm+/btMJsrl9w3m8244IIL8Mgjj9TdCCOQ9Beo3Vm5V5SvLRUo+CxGKWNT+cFaYvc9/ZhqTypDmQw6WIwGua/GVymqujWeKDgMejYPU3QLOLBZurR8HZDbbrsNb775JhIT/dtpuSGRgph1h3Nx5FwJAN+bYFLwSRmbAkUpiptiBo8U2MRVBDR+laIY9NcrKbBxuhjYUHSq8TvK3LlzGdRokEpRUlAD8M07VKxS9sxVOQOnxEdg88J3f6DHMz9j0YaGOaOvrhRWBDbxcmBTvr6V71JU+QcrMzb1y8Dp3hTl+I4SBL7eqPnmHRq+Fk701WMz57dDKLQ58dr/9tTHsKJWsUdgk2jVLkUxYxMa0gw0F9eqpCjFd5Qg8BXEcB2b0LD6KAF6lqKkLQAAIKfAJje1UuACKkW5uBRCKEhT8F3M2FCU4jtKEJgN3h+mfPMODV+BjWcp6vj5EtX3V7yxIqhjimZSAONZiiqyOfHdtpOY+fNuOVNjd7IUFQpyxoYBPEWpGq88TNp8Z2zYPBwKiT52XS91qLMHx3NLVd8fOluMvTmF6JieENSxRSPPUlR8xeufW2zHfQvLZ1Qa9Xo8dHlHue/JxA0w65XUY8PeYYpW/FMpCNhjEz6SYkxexzwzNscqMjYD2qTIx77afCK4A4tS0jo2cZbyQF4qRUmbkALA/jNFAJQL9DHor0/ydG+uY0NRip+2QeCrn4aBTWgkx5q9jhV59HscP1+esbkgMwmz/twHAPD1lpNR8ca/5VgeyupxQcIiOWNTHlBKpai8EuXKz+XnOLjGU0gY2GNDUY7vKEHA5uHwkRzrnbHxbGSVpuW3SInFZV3SkGAx4kReKTYcOV8vYwyWLzcfx+hZv+P2eevrbcPDIrnHRp2xUZ1TEdhUZmxYiqpPBq48TFGOn7ZB4OsvUO5eHBq+SlEFiqnHNqcLaw+eAwB0a5YIq8mA4V3TAQA3vLMaqw54b/QaKd5ZfhAAsOrAORw6W1wvj+k5K8pXj5O0+zczNqFROd2bgQ1FJ76jBIGv7IzTzRkIoZDsK7BRrEL8+/6zKLQ5kZ5oQe8WjQAAI7qly7e/8lNkrWuzN6cQJ/NK4XC5cfBMZTBzJLekinvVnSKv6d7er3+Rrfz155YKoWFkYENRjrOigsDXG7WdUxBCItFHYCOVos4V2XD7vA0AgJHdMuS/ZK/omoF4ixFFNifyFZtnhrv1h3Mx7j+rAQDNk2NUqy0fq+fARipBJSoCm3ZN4nDgTLFcrpJKUdwEs34ZGNhQlOM7ShD4CmxYigoNi1EPvcdLX2hzwuUW+Mcve+VjV/ZoKn+t1+vwyd2DAFSWTSLBhsOVPUEn8tRT2OsrsJFLUebywCY90YLeLZMBAKN7NQdQGfzIpShmbOqVvLs3m4cpSvEdJQikNLxkQOsUDO+aFqLRNGw6nU7+kFXaf7oIC9celb/v3zpFdXvz5BgAwNkiW73OKqqNAxXTqJWk5un8Uu8tDYLBsxSl0+mw8M5BWHjnQNya1RpA+R5RNqdL0TzMt6H6JCXIomHWH5EvLEUFgTL93ijWhP+7JyuEo6E4i1HenFHy6GdbAQD9WjXConuyoPPIqCmDU5vD7XMF43BztGJ21/PXdcOq/eeQYDWifVo8Zvy4G8X1lHnyLEUB5ft1Xdi+MdxuAZ0OEKI80LKzeTgkDPry15ulKIpWDGyCTHoTodCJVWyEmZZgwelCG7Ydz4dBr8Mz13TzCmoA9Wq4kdL4XeYsD14yG8XgPxP6AoC8W3mRzXuvpmCQAijPrCVQXuJLijEhr8SBvBIHZ0WFiPRyOxnYUJTiO0qQtW0SF+ohNHgWRbalQ3q8/PXUKzujR2aSz/vodDq5yTJSPgAcFQ3qRkUwLW1tUFwPgY0QAsV2qRTlO8PVqGLBxLwSB2dFhYjU7+dmjw1FKb6jBMncSf3Rv3UjvDymR6iH0uApsy/9WpX30ozq2RR3DGlT5f2kabGRstu3qyKzZFR0S0uZk/rI2JTYXZA+KxMs3rPRgMqen/Mlds6KChGuY0PRjqWoILm0cxou7cyG4XBgUHzQ3zu0HS5sl4q+rRr5LEEpmQx62JxuOCNkqr40TqMiUKjPwEZ6DL0OsJp8ByuVGRs7Z0WFCNexoWjHdxSKeiZFacZqMmBg21TVh7+WylJUZGRsHFLGRpGhkpp466MUpZwRpRU0ShmbN3/dx1lRIcKMDUU7vqNQ1PO1SJ8/pBJWpPTYuOQeG+9SVH3MiiqWN8DUTgR3b1be03S60IbSimn0bB6uXwauY0NRLqzeUWbMmIH+/fsjISEBaWlpGD16NPbsUS9pX1ZWhsmTJyM1NRXx8fEYO3YscnJyQjRiigSD2qZUf5IPUhNupJSiHG4fzcMVa/jYXW45QxIslRtgagc2Ey9sDaA8WJR2VWfGpn5JmUiuY0PRKqzeUZYvX47JkydjzZo1+OWXX+BwOHDFFVeguLhyz5uHHnoI3377LRYtWoTly5fj5MmTGDNmTAhHTeFu/MBWuLBdKu65pF1A95NKOpHSPOx0Sc24yoxN5eykYJejPBfn88Wg1yGzUfnih3Jgw4xNvYq02X5EgQqr5uGffvpJ9f28efOQlpaGjRs34uKLL0Z+fj7mzJmDhQsXYtiwYQCAuXPnokuXLlizZg0GDRoUimFTmIsxG7DwrsB/NqTZOpHyASCNU9ksbTToYTGWN0EX2ZxoFGcO2uNLU72rytgA5as6S0ENwIxNfZMzNixFUZQK63eU/Px8AEBKSnkpYePGjXA4HBg+fLh8TufOndGyZUusXr06JGOk6GWIsOneUsnMc/p0fD3NjJL21aousPHM6HC6d/2S94qKkICdKFBhlbFRcrvdePDBBzF48GB0794dAJCdnQ2z2Yzk5GTVuenp6cjOzvZ5HZvNBpvNJn9fUFAQtDFTdIm0abFOH7OiACDeasS5YnvwS1Fl1ZeiAHhtSsqMTf2q/LkO8UCIgiRs31EmT56MHTt24JNPPqnVdWbMmIGkpCT5X4sWLepohBTt5FJUhDQP+ypFAZU7bQc7Y1M5K6rqfbX0HlPBLQxs6lXldG9GNhSdwvId5b777sN3332HpUuXIjMzUz6ekZEBu92OvLw81fk5OTnIyMjwea2pU6ciPz9f/nfs2LFgDp2iSCQ1D7vcQl7116T3XYoK9pRvKXCKt1aXsVEHNixF1a/K6d4hHghRkITVO4oQAvfddx++/PJLLFmyBG3aqJe879u3L0wmExYvXiwf27NnD44ePYqsLN87aFssFiQmJqr+EflDChAioXlYuYigZylKmhkVDrOiAO+MEktR9YvTvSnahVWPzeTJk7Fw4UJ8/fXXSEhIkPtmkpKSEBMTg6SkJNxxxx2YMmUKUlJSkJiYiPvvvx9ZWVmcEUV1LpKah5XlMqNHxkYKNArrrRRV9duK56LERs+mGwqqSFtRmyhQYRXYzJ49GwAwdOhQ1fG5c+di0qRJAIA33ngDer0eY8eOhc1mw4gRI/D222/X80ipIZAyH5HQPKwKbDwyNvW1rUKRn4GNZylKz8CmXlVmbEI8EKIgCavARvixroLVasWsWbMwa9asehgRNWSR1DysKkVpNA+HaymKYU390nNLBYpyLG4TaZACBEcE/GmrnBHluQFlfe3wnVtsBwCkVLMIoGcpyjODQ8EVacsYEAWKgQ2RhkjK2Eh9QL76VSpnRQU3sDlbWL5eVON4S5XnGTxLUYxr6pWBgQ1FOQY2RBoiaU8dl9t7Z29JZcYmeNO9S+xOFNvLr98koerAxitDw8CmXukZ2FCUY2BDpEFqwnVGwKwoR0VWyehjTRhpuneRzRG0xz9bWF6Gspr0iDNXs0Cf3jNjw8imPkkZM+4VRdGKgQ2Rhkhcx8Zk8A4SKmdFBS9jc7qwDEB5tsazx8eTZ1KJgU39YimKoh0DGyINBjljE/4fANIYPWccAfUzK+rg2WIAQKuUuGrP9QxkGNbULwY2FO0Y2BBpiKSUvVPusfH+lU6NL5+llFNQFrTVZg+cKQIAtGtSfWDjGXwxY1O/pGolp3tTtGJgQ6RB+vyNhMBGGqOvjE2r1DiYDXoU21147PNtmL/6MIpsTrjdAueKbMjOLy8jFZQ5MO2rHXjm6x04cq4YZQ4XVuw9A5uz+hLWjhP5AID2afHVnuvVO8x3oXolr2PDjA1FqbBaoI8onETS7BEpE+Nr6rTJoEenjARsP5GPzzYex2cbj2Pa1zthNuhhVzRGx1uM8lo3X205CZNBh7NF5U3Bwzqn4eYBLZGeaMG/Fu+DXqdDQZkDB84UlwdIFWvYDOnQpNqxek73Zr6mfklZvUj4uSaqCQY2RBoqS1EhHogfpDFqbU8w9arOeHTRNpzIK5WP2T1mexXZnEiwGhFjMuB0xZo0kiW7T2PJ7tNVjmFA6xS0aexHjw1LUSElVSsZ2FC0YmBDpEH6AI6kUpRWkHBhu8b4/YlhOF9sx+FzxTh2vhRCCOh1OizaeBxbj+XhtsGtcdvgNjDodfhk3VGUOVzo1zoFP+3IxrxVh1XXu6hDY7ROjcOuUwUQAFqmxOKJKzv7NVavvaIY2NQrQwT9XBPVBAMbIg2R1ItQGdhUfV6jODMaxZnRu2Uj+dg1FzTzOu/Oi9rKXw9qm4qnRnWBzenGgdNF6NE8qVYbV3relXFN/eKWChTtGNgQaYio5uGKqlKwsh8mgx4mgx4XtEiu9bW8pnszsKlX0usfCeszEdUE5yMQaZBT9hHwASAFX9UtjhcO2GMTWpH0c01UEwxsiDToI6p5WJruHeKB+MGrFBWaYTRYcok1AjKRRDURAW+DRKERSR8A0hAjIfvhvbt3+I85mkh7oLnDfws0ohphYEOkQcp+RELKPpJLUREw5KhikHtsGNlQdGJgQ6RBF0FbKriqWKAv3Hg3D0fAoKNI5TIGgIiAn22iQDGwIdJQuVlgiAfiBymp5FnmCUfK4CsSArFoo/wZiYBkJFHAGNgQaYikTTBFNQv0hRPlGCNhvNFG2rUeYDmKohMDGyIN0mduJAQ20l/ekRAnKHtsImG80UaVsWFcQ1GIgQ2RBkMErdDqiqiMTeXX7K+pf8od4CNhxh9RoBjYEGmQgoRIeO8X8jo24R8oKMcYAcONOqrAJgKCdqJAMbAh0qCPoIxN5XTvEA/ED8osjY7L89U7ZSkqEn62iQLFwIZIgyGCFugL9l5RdcmgY8YmlPTM2FCUY2BDpEF6/4+EtT5cfu7uHQ7U070jYMBRSN4vKgJ+tokCxcCGSEMklaIidbo3K1GhIQU23OGbohEDGyINkbUJZvl/PbcrCEd6VfNw+I83GslrNEXCDzdRgBjYEGmQ94qKgHS9O2JLUaEbR0MWSUsZEAWKgQ2RBnl37wh485czNhGQATGoFugL//FGI+l/QSQ0xhMFioENkQZ9BG2p4HZHTo+NjrOiQs5YkY6MhKCdKFAMbIg0yDNHImDZebkUFQGRgnK6NzM2oRFJ2UiiQDGwIdKgj6R1bORSVGjH4Q89J0WFnNQ/xsCGohEDGyIN0gdwJJSiImm6t467e4ecUV/+1h8JP9tEgWJgQ6ShshQV/m/+0l/ekRAncK+o0KuIa7iODUUlBjZEGiJxHRtDBEQ23N079EwVkY3TFQE/3EQBYmBDpCGSVh52R1ApSq+a7h3CgTRgpoomG4crAjrjiQLEwIZIgyGCpnvLPTYR8ButZ49NyBkN5a87AxuKRhHwNkgUGpHUPCx9PkVCaUddigrdOBqyyoxN+P9sEwWKgQ2RhsgsRYV4IH5gxib0zAapx4YZG4o+DGyINEgfuhGQsJFLUZHQPKxjxibkpFKUnYENRSEGNkQa5EXMIiCykZJKkVGKUjQPh3AcDZnJwFlRFL0Y2BBpiKRl5yNqVhRLUSFnYvMwRTEGNkQaIqkU5YqoHhvl1xEw4CjE6d4UzRjYEGkwRFDzsBR8GSIgstGpNsEM4UAaMCNnRVEUY2BDpEH60I2IHht5S4XwjxS48nDosRRF0YyBDZEGKfshIiGwiajdvblXVKjJWypEQDaSKFAMbIg0GNg8HBR6lqJCzmSsmO7tZMaGog8DGyINOnlLhRAPxA9yYBMBKRAdm4dDTp7u7WZgQ9GHgQ2RBqkU5Y6AyCaSVh7Wsccm5LilAkUzBjZEGvSR1Dws99iEf6DABfpCT2oeZimKohEDGyINerkUFQGBjTtyMjZsHg49o56lKIpeDGyINFSWokI8ED9IwVcklHa4QF/omY0VpShn+AftRIFiYEOkQd5SIRIyNlygjwJgrPg5cURC1E4UIGOoB0AUriqy9ZFRioqg5mEu0Bd6UvNwffbYlDlcWH84F9n5ZRjeJR3xViN+3JGNOb8dQu8WyRjUNhXDOqfJ2SSimmJgQ6RBuVeUECKsP4RFhDYPR0IgFo0SrOVv/UU2Z1AfZ8XeM/hqywmsO5SL4+dLNc/beiwP81YdBgCM7ZOJwe1T0STBgl//yMEDwzsiJc4c1HFSdGFgQ6TBoPgAdgvAEMYfwq6I2lJBOSsq/McbjZJjywOFvBJHnV7X5RbYejwPi3fl4FR+Gb7YdMKv+yVYjCisCLI+33Qcn286Lt/2weojAIA4swEzx12A3dmFaJUSi1E9m8JqMtTp+Ck6MLAh0qBc7M7lFmHdvyL1AYVz8CVRLdDHqkNIJMWYAAD5pXUX2Bw6W4ynv9qO3/efUx1vkRKD+4d1wPHzpbggMwnt0+JhNRlQbHPi8LliXNiuMawmA47lluDRz7ZizcFcn9cvtrvw1wWb5O9f/mk3nhjZGdf2agYAOHCmCJmNYhFvqfxY++NkAVxugR6ZSXX2PCn8MbAh0qCMY8K9z0aa7h3OwZdEGTBGQuksGiXHlgc2eSX2OrnexiPncct/16LU4QJQ3pyc1S4VvVsk4y+XtEOcxfdHTdsm8fLXLVJi8cndWRBCYMeJAny64Sj25hRh3aFcWE16lDnU/UBnCm14eNFWPLxoq3ysU3oCnruuG9YdysXFHZvgxndWQwhg6aND0Tw5pk6eK4U/BjZEGpRBQrgHNlIpKhK2VIiAIUa9ZEXGxu0WNf65OV9sx4p9Z/DoZ9tgd7rRt1UjvPSn7mjfJB5GQ83ScTqdDj0yk9AjsweAyv62w2eL8eOObFyQmQQBYO7vh/HrrhzVfffkFOKmd9cAAF7/Za98/B//2wOTXo9LO6dhRLd0FNmcKLa5kJFkrdEYKbwxsCHSoMwmhPtGmG65FBX+UYO6eTj8xxuNEisCG7cACsucSKrI4ARCCIEb3lmNfaeLAABtGsdh7m39kWgN/FpVkfrGWjeOw71D28nHB7dvjL05hbA73Zi97ADySx04kVeKQ2eLva4h9fp8uuGYfMxi1OPpq7viwOkiTLmiY52Pm0KHgQ2RBr1H83A4i6SMjXKEETDcqGQ1GZAca0JeiQPH80qQFBt4D8rKfWfloAYAHg5BcNAxPQEAMGt8HwBAYZkDv+07i76tGmHnqQIkWk24f+EmnMwv87qvzenGtK92AACO5pbgiSs7y9ejyMbAhkiDqhQV5pGNtJdhJGRs1Av0hf94o1X7JvHYcOQ89p8uQrdmgQU27/92CM9/9weA8kbkJ67sjKt7NgvGMAOSYDXhyh5NAQBpieVlpjdv7o0vNp3ATf1b4Oed2Xh72QGv+y3ZfRpLdp8GAFzYLhUv/akHWqfGwuUW+GbrSQxqm4pm7NGJGAxsiDQoswnhvvqwvFdUBMwyUi3QF7phNHgd0ssDm01HzuO6Xs39us/ag+dwz0cbcb5imrhRr8OKxy6VZ1mFo/6tU9C/dQoA4IIWybiqR1P8tv8sJma1xg/bT6majwFg1YFzuPS1ZapjA9uk4L5h7fHNlpO4ZVAr7DiZj4s7NEGLlNj6ehoUAAY2RBp0Oh10uvLF78K9ebhy5eHwDxUiYYwNwRVdM/DxumNYtPE4Hr+yM2LNVX8c7D9diDs+2KBa1O/9Sf3DOqjxpXvzJHRvXp6hGts3EyO7Z+BUfile+WkPluw+7bOfbu2hXKydsw4AsGhj5Ro7Rr0OM8f1RHKMGUM6NJZXdKbQYmBDVAW9TgeXEGG/EaYrkqZ7c6+osDC0UxO0So3FkXMleP7bP/DC6O6aH8wr953BXxdskoOaO4e0wVOjukRFKTHOYkT7tAS8d2s/AMCRc8XYfiIfm4/mISXOjO+2ncKuUwU+7+t0Czz0aXnGp1uzRPRonoQR3TPQMiUW247n4ZqezWo8O4xqjoENURUMOh1cEOFfioqgWVE6xft8NHwwRiqdToebB7TEyz/uxifrj+HbrSfxwujuGNMnUz7nj5MFeOabHdhw5DyEAPq2aoR3JvRF43hLCEceXK1S49AqNU7uGbrrorb4dP1R5BY7UGx34v3fDuHeoe1gd7rxzoqD8v12nizAzpMF+GR95cyrF7/bhZHdM1DmcOOiDo1xXa9m+HrLSaQlWtC9eRLOFtrQpnEcfw/qGAMboiro9QBcEdA8HEGzorhXVPiYdGFr7M0pxBebTqDY7sKU/9uKF7/fBbvTjYFtUrC4oqEWKN/DafqY7rAYG9Y2BmajHhOyWsvfPzqik5zZurhjE9z/8WakxplVM8Qk54rtWLD2KIDyrSIe/WwrHC71e8lVPTLQp2UjXN41HRlJVpwvdnB9nVpiYENUBelDONx7bCJpVpQymGG/TWhZTQa8fkMvTMxqjVd+2o1VB84ht7h8NWJlUPPJ3YMwqG1qqIYZVpTlusHtG2PTtMshhMCCtUdxIq8UzZJjMKhNChJjTLjm37/hdKFNPt8zqAGAH7Zn44ft2Xjx+13ysUdHdEJmoxh8tvE4Dp0txpg+mWiZEouremRU2wtFYRbYrFixAjNnzsTGjRtx6tQpfPnllxg9erR8uxACzzzzDN577z3k5eVh8ODBmD17Njp06BC6QVNUM8iBTYgHUo2I2lKBPTZh54IWyfjw9gHo+PSP8s/6xKxWWLrnDIZ2asKgpho6nQ63DGrldXzFY5fi0/XHkJ5owT0fle9z9ew1XdEhPQGrD5zDr7tycORcibwVhWTmz3tU3/9r8b7y+36zE12bJeLK7uVZHqNBhx0n8lFqd2F413RM/WI7Tpwvxdu39EGn9AQcyy1Fi5QYudQlreIc7cIqsCkuLsYFF1yA22+/HWPGjPG6/dVXX8W//vUvfPDBB2jTpg2mTZuGESNG4I8//oDVytQd1T2ptBPuKw9HUilKp5ruHf7jbSiMBj2mXN4R7644iKev7oob+rXAc6EeVISzmgyYeGFrAMC82/rjXJEdY/o0h06nw+D2jfHIiE4QQuDAmWK8uXgfjuaWYOuxPNU1DHqd/PtdZHNi3aFcrDvkvVHos9/+IX898p8r0SzJipP5Zbj74rYY2ycTL37/B1buO6u6T4/mSbA5XXh7fB/EmI0wGXR49ac92HosD+9P6o85vx1C31aNcHGHJli5/wxGdMuAyaCH0+XGwnVHEWs24k+9m8Og16HM4cLhc8W12k6jruiECM8cu06nU2VshBBo1qwZHn74YTzyyCMAgPz8fKSnp2PevHm46aab/LpuQUEBkpKSkJ+fj8TExGANn6JE7+f/h/MlDvzvoYvDelXSkf9cgd3ZhfjojoEY0qFxqIdTJYfLjQ5P/QgAuLpnU7z15z4hHhFR+LA5Xfh680m0SInFoLYpKHW48O3Wk7ioQxP8tv8svtlyEnanG1uO5cFk0KFjRgI2H82rl7FZjHq0SImFUa/D7uxC+fhPD16Exz/bhq3H8wEA7dPicd+l7TG6t3/rI/nL38/vsMrYVOXQoUPIzs7G8OHD5WNJSUkYOHAgVq9erRnY2Gw22GyVNc6CAt/T9oh8kUo7Yd9jE1EL9HHlYSItFqMBN/RvIX8fazbixv4tAQA39GuBG/qV3+ZwuaFDeabtdGEZPtt4HF0yEmE06PDbvrMotDnx+/6zOHKuRHEtA+4Y0gYHzhThh+3ZAY/N5nRjv48m6ZH/XKn6fv/popC+Z0ZMYJOdXf4/IT09XXU8PT1dvs2XGTNm4LnnmFClmpE+eMO+FBVJ070VX0dA5YwoLCmbmNMSrPjr0Pby9xd1aAIAyC2249uKLSE6pMV7lartTjcEBOb+fhhvLdkPm9OFbs2SMLJ7Bo6cK8awzukotjnxzoqDGNwuFfvPFOHgmWL0bpmMoZ2aoNTuxlNfbYcQgMmgw5XdmyKrXfn2E10yQpfhjpjApqamTp2KKVOmyN8XFBSgRYsWVdyDqJLcPBzmC/RFUvOwuseGiIIlJc4s9/j4YjaWB0f3XNIO91zSTvO8qkpKY/s2x5ajeejWPAnxlvAIKcJjFH7IyMgAAOTk5KBp06by8ZycHPTq1UvzfhaLBRZL9C4mRcEVMaUoEUnNw8p1bMJ/vESkzWI0YGCYzZqLgIp8uTZt2iAjIwOLFy+WjxUUFGDt2rXIysoK4cgomkmfu2G/8nBFRikSSlEqETZcIgp/YZWxKSoqwv79++XvDx06hC1btiAlJQUtW7bEgw8+iBdffBEdOnSQp3s3a9ZMtdYNUV2SMjZhOnlQFkl7RSkxY0NEdS2sApsNGzbg0ksvlb+XemMmTpyIefPm4bHHHkNxcTHuvvtu5OXlYciQIfjpp5+4hg0FjV5uHg7xQKrhiqDdvZUiLA4joggQVoHN0KFDq/zLWKfT4fnnn8fzzz9fj6Oihkz64A33WVGR1DysxAX6iKiuRUyPDVEoREwpSpruHWG/0ZGw7g4RRRa+rRBVQS5FhXtg447MUhS7h4morjGwIaqCnptgBlXExWFEFPYY2BBVQSqVuMM8smHzMBFROQY2RFUwRMiWCvI6NhEWKbB5mIjqGgMboiroI2zl4UgLbCJsuEQUARjYEFWhsscmzAObCG0e5u7eRFTXGNgQVcEQAQv0Kft/Ii1jw7iGiOoaAxuiKkgfvOGcsVFORY+0vaIiLcNEROGPgQ1RFSJhd29lY7Muwn6jGdYQUV2LsLdBovoVCYGNO5IzNhFWOiOi8MfAhqgK5oo9Csoc4dtk44rkHptQD4CIog4DG6IqJFjL94ktKnOGeCTa3IqYK9J6VjgriojqGgMboirEVwQ2hbbwDWxUzcORlrGJrOESUQRgYENUhQSrCQBQWOYI8Ui0KUtRERbXRNx4iSj8MbAhqkK8JfxLUcUV2aQ4syHiSjvcUoGI6hoDG6IqJFaUohZtPA5nmK7SV1gRdCXGmEI8ksBFWBxGRBGAgQ1RFWLMRvnrRRuPh3Ak2goqymRSo3MkuG1wazRJsGDSha1DPRQiijKR805IFALKNWL+tzMbw7ukY+OR8+jSNAGtUuPgdLlR6nAhwWpCid0Jp1sg0WpCTkEZ1hw8hyu7N4WAwLvLD+KPUwXokJ4Ai1GPBKsRY/pkIs5swJqDuWiaZEWJ3YViuxMxJgM2HT2Pkd0y4BICSTEmnCuyI7NRjFepyeZ04fvtpwAAidbIydg8c003TBvVlevYEFGd0wkRxiuPBUFBQQGSkpKQn5+PxMTEUA+HwlyZw4UR/1yBI+dKvG7r1iwRJ/JKYXe6MeXyjnjph10QAujbqhE2Hjlf7bWtJj1apsRib06RX2NplmTF9X0zEWsx4r8rD8EtBArLHHC4yn+FB7ZJwad/yQrsCRIRRQh/P78Z2BBVQwiBuz7ciF935dT6Wld2z0B+qQPbT+TLvTF1pXG8GRuevrxOr0lEFC78/fxmKYqoGjqdDu/d2hfL957B8r1n8OcBLfHHqQLsyS5EcqwJ/1q8H0U2J2LNBjx3bTfsPFmAFfvO4LbBbXC+2A6Hy41JF7ZGarxFvqbd6cYP20+hyObEyO4ZyCkow/liBzKSrDAZdGiZEovThTbEWYwoc7hw+Gwxpv+wC5uO5gEAjHod7rmkHQa2TcGEOesAAA3rTxQiIt+YsSGqpRN5pTiVV4o+LRsFvWfE6XJjye7T6NuqkRwo/bbvLJ74YhteHN0dQzulBfXxiYhChaUoDQxsiIiIIo+/n9+c7k1ERERRg4ENERERRQ0GNkRERBQ1GNgQERFR1GBgQ0RERFGDgQ0RERFFDQY2REREFDUY2BAREVHUYGBDREREUYOBDREREUUNBjZEREQUNRjYEBERUdRgYENERERRg4ENERERRQ1jqAdQ34QQAMq3PyciIqLIIH1uS5/jWhpcYFNYWAgAaNGiRYhHQkRERIEqLCxEUlKS5u06UV3oE2XcbjdOnjyJhIQE6HS6UA8nKAoKCtCiRQscO3YMiYmJoR5OWOBr4o2viRpfD298TbzxNfFWX6+JEAKFhYVo1qwZ9HrtTpoGl7HR6/XIzMwM9TDqRWJiIn/xPPA18cbXRI2vhze+Jt74mnirj9ekqkyNhM3DREREFDUY2BAREVHUYGAThSwWC5555hlYLJZQDyVs8DXxxtdEja+HN74m3viaeAu316TBNQ8TERFR9GLGhoiIiKIGAxsiIiKKGgxsiIiIKGowsCEiIqKowcAmihw+fBh33HEH2rRpg5iYGLRr1w7PPPMM7Ha76rxt27bhoosugtVqRYsWLfDqq6+GaMT1Y9asWWjdujWsVisGDhyIdevWhXpI9WbGjBno378/EhISkJaWhtGjR2PPnj2qc8rKyjB58mSkpqYiPj4eY8eORU5OTohGXL9efvll6HQ6PPjgg/Kxhvp6nDhxArfccgtSU1MRExODHj16YMOGDfLtQgj8/e9/R9OmTRETE4Phw4dj3759IRxxcLlcLkybNk31fvrCCy+o9imK9tdkxYoVuOaaa9CsWTPodDp89dVXqtv9ef65ubkYP348EhMTkZycjDvuuANFRUXBHbigqPHjjz+KSZMmiZ9//lkcOHBAfP311yItLU08/PDD8jn5+fkiPT1djB8/XuzYsUN8/PHHIiYmRrzzzjshHHnwfPLJJ8JsNov3339f7Ny5U9x1110iOTlZ5OTkhHpo9WLEiBFi7ty5YseOHWLLli3iqquuEi1bthRFRUXyOffcc49o0aKFWLx4sdiwYYMYNGiQuPDCC0M46vqxbt060bp1a9GzZ0/xwAMPyMcb4uuRm5srWrVqJSZNmiTWrl0rDh48KH7++Wexf/9++ZyXX35ZJCUlia+++kps3bpVXHvttaJNmzaitLQ0hCMPnpdeekmkpqaK7777Thw6dEgsWrRIxMfHizfffFM+J9pfkx9++EE89dRT4osvvhAAxJdffqm63Z/nP3LkSHHBBReINWvWiJUrV4r27duLm2++OajjZmAT5V599VXRpk0b+fu3335bNGrUSNhsNvnY448/Ljp16hSK4QXdgAEDxOTJk+XvXS6XaNasmZgxY0YIRxU6p0+fFgDE8uXLhRBC5OXlCZPJJBYtWiSfs2vXLgFArF69OlTDDLrCwkLRoUMH8csvv4hLLrlEDmwa6uvx+OOPiyFDhmje7na7RUZGhpg5c6Z8LC8vT1gsFvHxxx/XxxDr3ahRo8Ttt9+uOjZmzBgxfvx4IUTDe008Axt/nv8ff/whAIj169fL5/z4449Cp9OJEydOBG2sLEVFufz8fKSkpMjfr169GhdffDHMZrN8bMSIEdizZw/Onz8fiiEGjd1ux8aNGzF8+HD5mF6vx/Dhw7F69eoQjix08vPzAUD+mdi4cSMcDofqNercuTNatmwZ1a/R5MmTMWrUKNXzBhru6/HNN9+gX79+GDduHNLS0tC7d2+899578u2HDh1Cdna26nVJSkrCwIEDo/Z1ufDCC7F48WLs3bsXALB161b89ttvuPLKKwE0zNdEyZ/nv3r1aiQnJ6Nfv37yOcOHD4der8fatWuDNrYGtwlmQ7J//378+9//xmuvvSYfy87ORps2bVTnpaeny7c1atSoXscYTGfPnoXL5ZKfnyQ9PR27d+8O0ahCx+1248EHH8TgwYPRvXt3AOX/z81mM5KTk1XnpqenIzs7OwSjDL5PPvkEmzZtwvr1671ua4ivBwAcPHgQs2fPxpQpU/Dkk09i/fr1+Nvf/gaz2YyJEyfKz93X71K0vi5PPPEECgoK0LlzZxgMBrhcLrz00ksYP348ADTI10TJn+efnZ2NtLQ01e1GoxEpKSlBfY2YsYkATzzxBHQ6XZX/PD+oT5w4gZEjR2LcuHG46667QjRyCieTJ0/Gjh078Mknn4R6KCFz7NgxPPDAA1iwYAGsVmuohxM23G43+vTpg+nTp6N37964++67cdddd+E///lPqIcWMv/3f/+HBQsWYOHChdi0aRM++OADvPbaa/jggw9CPTSqBjM2EeDhhx/GpEmTqjynbdu28tcnT57EpZdeigsvvBDvvvuu6ryMjAyvGR7S9xkZGXUz4DDRuHFjGAwGn8832p5rde677z589913WLFiBTIzM+XjGRkZsNvtyMvLU2UpovU12rhxI06fPo0+ffrIx1wuF1asWIG33noLP//8c4N6PSRNmzZF165dVce6dOmCzz//HEDle0NOTg6aNm0qn5OTk4NevXrV2zjr06OPPoonnngCN910EwCgR48eOHLkCGbMmIGJEyc2yNdEyZ/nn5GRgdOnT6vu53Q6kZubG9TfJ2ZsIkCTJk3QuXPnKv9JPTMnTpzA0KFD0bdvX8ydOxd6vfp/cVZWFlasWAGHwyEf++WXX9CpU6eoKkMBgNlsRt++fbF48WL5mNvtxuLFi5GVlRXCkdUfIQTuu+8+fPnll1iyZIlXGbJv374wmUyq12jPnj04evRoVL5Gl112GbZv344tW7bI//r164fx48fLXzek10MyePBgr2UA9u7di1atWgEA2rRpg4yMDNXrUlBQgLVr10bt61JSUuL1/mkwGOB2uwE0zNdEyZ/nn5WVhby8PGzcuFE+Z8mSJXC73Rg4cGDwBhe0tmSqd8ePHxft27cXl112mTh+/Lg4deqU/E+Sl5cn0tPTxYQJE8SOHTvEJ598ImJjY6N6urfFYhHz5s0Tf/zxh7j77rtFcnKyyM7ODvXQ6sW9994rkpKSxLJly1Q/DyUlJfI599xzj2jZsqVYsmSJ2LBhg8jKyhJZWVkhHHX9Us6KEqJhvh7r1q0TRqNRvPTSS2Lfvn1iwYIFIjY2Vnz00UfyOS+//LJITk4WX3/9tdi2bZu47rrrompqs6eJEyeK5s2by9O9v/jiC9G4cWPx2GOPyedE+2tSWFgoNm/eLDZv3iwAiNdff11s3rxZHDlyRAjh3/MfOXKk6N27t1i7dq347bffRIcOHTjdm/w3d+5cAcDnP6WtW7eKIUOGCIvFIpo3by5efvnlEI24fvz73/8WLVu2FGazWQwYMECsWbMm1EOqN1o/D3PnzpXPKS0tFX/9619Fo0aNRGxsrPjTn/6kCoajnWdg01Bfj2+//VZ0795dWCwW0blzZ/Huu++qbne73WLatGkiPT1dWCwWcdlll4k9e/aEaLTBV1BQIB544AHRsmVLYbVaRdu2bcVTTz2lWioj2l+TpUuX+nz/mDhxohDCv+d/7tw5cfPNN4v4+HiRmJgobrvtNlFYWBjUceuEUCyjSERERBTB2GNDREREUYOBDREREUUNBjZEREQUNRjYEBERUdRgYENERERRg4ENERERRQ0GNkRERBQ1GNgQUUQ6fPiwvAlsXezNI13Lc2dvIoosDGyIKKL9+uuvqv1qaurUqVP45z//WfsBEVFIMbAhooiWmpqK1NTUWl8nIyMDSUlJdTAiIgolBjZEFHJnzpxBRkYGpk+fLh9btWoVzGZzwNmYSZMmYfTo0Zg+fTrS09ORnJyM559/Hk6nE48++ihSUlKQmZmJuXPn1vXTIKIwYAz1AIiImjRpgvfffx+jR4/GFVdcgU6dOmHChAm47777cNlllwV8vSVLliAzMxMrVqzA77//jjvuuAOrVq3CxRdfjLVr1+LTTz/FX/7yF1x++eXIzMwMwjMiolBhxoaIwsJVV12Fu+66C+PHj8c999yDuLg4zJgxo0bXSklJwb/+9S906tQJt99+Ozp16oSSkhI8+eST6NChA6ZOnQqz2Yzffvutjp8FEYUaMzZEFDZee+01dO/eHYsWLcLGjRthsVhqdJ1u3bpBr6/8uy09PR3du3eXvzcYDEhNTcXp06drPWYiCi/M2BBR2Dhw4ABOnjwJt9uNw4cP1/g6JpNJ9b1Op/N5zO121/gxiCg8MWNDRGHBbrfjlltuwY033ohOnTrhzjvvxPbt25GWlhbqoRFRBGHGhojCwlNPPYX8/Hz861//wuOPP46OHTvi9ttvD/WwiCjCMLAhopBbtmwZ/vnPf2L+/PlITEyEXq/H/PnzsXLlSsyePTvUwyOiCMJSFBGF3NChQ+FwOFTHWrdujfz8/ICvNW/ePK9jy5Yt8zpWmx4eIgpfDGyIKKJdeOGF6NWrF1atWlWr68THx8PpdMJqtdbRyIgoFBjYEFFEyszMxL59+wCgxtPClbZs2QKgfCo4EUUunRBChHoQRERERHWBzcNEREQUNRjYEBERUdRgYENERERRg4ENERERRQ0GNkRERBQ1GNgQERFR1GBgQ0RERFGDgQ0RERFFDQY2REREFDX+H/vbhRIF9oKdAAAAAElFTkSuQmCC", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "ds = ds.sel(x=slice(-30, 101))\n", "sections = {\n", From 19d5044a57e83c4a7adc9960d6712af7171c6af5 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Fri, 27 Oct 2023 14:44:31 +0200 Subject: [PATCH 59/66] Delete 02Common_DataStore_functions_slice_mean_max_std_resample.ipynb --- ...unctions_slice_mean_max_std_resample.ipynb | 371 ------------------ 1 file changed, 371 deletions(-) delete mode 100644 docs/notebooks/02Common_DataStore_functions_slice_mean_max_std_resample.ipynb diff --git a/docs/notebooks/02Common_DataStore_functions_slice_mean_max_std_resample.ipynb b/docs/notebooks/02Common_DataStore_functions_slice_mean_max_std_resample.ipynb deleted file mode 100644 index 79f14247..00000000 --- a/docs/notebooks/02Common_DataStore_functions_slice_mean_max_std_resample.ipynb +++ /dev/null @@ -1,371 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# 2. Common DataStore functions\n", - "Examples of how to do some of the more commonly used functions:\n", - "\n", - "1. mean, min, max, std\n", - "2. Selecting\n", - "3. Selecting by index\n", - "4. Downsample (time dimension)\n", - "5. Upsample / Interpolation (length and time dimension)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "execution": { - "iopub.execute_input": "2022-04-06T08:08:57.302425Z", - "iopub.status.busy": "2022-04-06T08:08:57.301918Z", - "iopub.status.idle": "2022-04-06T08:08:58.945453Z", - "shell.execute_reply": "2022-04-06T08:08:58.944983Z" - } - }, - "outputs": [], - "source": [ - "import os\n", - "import xarray as xr\n", - "from dtscalibration import read_silixa_files" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First we load the raw measurements into a `DataStore` object, as we learned from the previous notebook." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "execution": { - "iopub.execute_input": "2022-04-06T08:08:58.948063Z", - "iopub.status.busy": "2022-04-06T08:08:58.947893Z", - "iopub.status.idle": "2022-04-06T08:08:59.145387Z", - "shell.execute_reply": "2022-04-06T08:08:59.144710Z" - } - }, - "outputs": [], - "source": [ - "filepath = os.path.join(\"..\", \"..\", \"tests\", \"data\", \"single_ended\")\n", - "\n", - "ds = read_silixa_files(directory=filepath, timezone_netcdf=\"UTC\", file_ext=\"*.xml\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 0 Access the data\n", - "The implemented read routines try to read as much data from the raw DTS files as possible. Usually they would have coordinates (time and space) and Stokes and anti Stokes measurements. We can access the data by key. It is presented as a DataArray. More examples are found at http://xarray.pydata.org/en/stable/indexing.html" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "execution": { - "iopub.execute_input": "2022-04-06T08:08:59.171097Z", - "iopub.status.busy": "2022-04-06T08:08:59.170926Z", - "iopub.status.idle": "2022-04-06T08:08:59.201341Z", - "shell.execute_reply": "2022-04-06T08:08:59.200765Z" - } - }, - "outputs": [], - "source": [ - "ds[\"st\"] # is the data stored, presented as a DataArray" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "execution": { - "iopub.execute_input": "2022-04-06T08:08:59.203775Z", - "iopub.status.busy": "2022-04-06T08:08:59.203591Z", - "iopub.status.idle": "2022-04-06T08:08:59.265177Z", - "shell.execute_reply": "2022-04-06T08:08:59.264679Z" - } - }, - "outputs": [], - "source": [ - "ds[\"tmp\"].plot(figsize=(12, 8))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 1 mean, min, max\n", - "The first argument is the dimension. The function is taken along that dimension. `dim` can be any dimension (e.g., `time`, `x`). The returned `DataStore` does not contain that dimension anymore.\n", - "\n", - "Normally, you would like to keep the attributes (the informative texts from the loaded files), so set `keep_attrs` to `True`. They don't take any space compared to your Stokes data, so keep them.\n", - "\n", - "Note that also the sections are stored as attribute. If you delete the attributes, you would have to redefine the sections." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "execution": { - "iopub.execute_input": "2022-04-06T08:08:59.267698Z", - "iopub.status.busy": "2022-04-06T08:08:59.267493Z", - "iopub.status.idle": "2022-04-06T08:08:59.273319Z", - "shell.execute_reply": "2022-04-06T08:08:59.272886Z" - } - }, - "outputs": [], - "source": [ - "ds_min = ds.mean(\n", - " dim=\"time\", keep_attrs=True\n", - ") # take the minimum of all data variables (e.g., Stokes, Temperature) along the time dimension" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "execution": { - "iopub.execute_input": "2022-04-06T08:08:59.275670Z", - "iopub.status.busy": "2022-04-06T08:08:59.275507Z", - "iopub.status.idle": "2022-04-06T08:08:59.279270Z", - "shell.execute_reply": "2022-04-06T08:08:59.278851Z" - } - }, - "outputs": [], - "source": [ - "ds_max = ds.max(\n", - " dim=\"x\", keep_attrs=True\n", - ") # Take the maximum of all data variables (e.g., Stokes, Temperature) along the x dimension" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "execution": { - "iopub.execute_input": "2022-04-06T08:08:59.281530Z", - "iopub.status.busy": "2022-04-06T08:08:59.281321Z", - "iopub.status.idle": "2022-04-06T08:08:59.287525Z", - "shell.execute_reply": "2022-04-06T08:08:59.286991Z" - } - }, - "outputs": [], - "source": [ - "ds_std = ds.std(\n", - " dim=\"time\", keep_attrs=True\n", - ") # Calculate the standard deviation along the time dimension" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 2 Selecting\n", - "What if you would like to get the maximum temperature between $x >= 20$ m and $x < 35$ m over time? We first have to select a section along the cable." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "execution": { - "iopub.execute_input": "2022-04-06T08:08:59.290091Z", - "iopub.status.busy": "2022-04-06T08:08:59.289877Z", - "iopub.status.idle": "2022-04-06T08:08:59.293824Z", - "shell.execute_reply": "2022-04-06T08:08:59.293152Z" - } - }, - "outputs": [], - "source": [ - "section = slice(20.0, 35.0)\n", - "section_of_interest = ds.sel(x=section)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "execution": { - "iopub.execute_input": "2022-04-06T08:08:59.296109Z", - "iopub.status.busy": "2022-04-06T08:08:59.295850Z", - "iopub.status.idle": "2022-04-06T08:08:59.299802Z", - "shell.execute_reply": "2022-04-06T08:08:59.299282Z" - } - }, - "outputs": [], - "source": [ - "section_of_interest_max = section_of_interest.max(dim=\"x\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "What if you would like to have the measurement at approximately $x=20$ m?" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "execution": { - "iopub.execute_input": "2022-04-06T08:08:59.302128Z", - "iopub.status.busy": "2022-04-06T08:08:59.301950Z", - "iopub.status.idle": "2022-04-06T08:08:59.306081Z", - "shell.execute_reply": "2022-04-06T08:08:59.305484Z" - } - }, - "outputs": [], - "source": [ - "point_of_interest = ds.sel(x=20.0, method=\"nearest\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 3 Selecting by index\n", - "What if you would like to see what the values on the first timestep are? We can use isel (index select) " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "execution": { - "iopub.execute_input": "2022-04-06T08:08:59.308603Z", - "iopub.status.busy": "2022-04-06T08:08:59.308281Z", - "iopub.status.idle": "2022-04-06T08:08:59.312353Z", - "shell.execute_reply": "2022-04-06T08:08:59.311877Z" - }, - "scrolled": true - }, - "outputs": [], - "source": [ - "section_of_interest = ds.isel(time=slice(0, 2)) # The first two time steps" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "execution": { - "iopub.execute_input": "2022-04-06T08:08:59.314626Z", - "iopub.status.busy": "2022-04-06T08:08:59.314411Z", - "iopub.status.idle": "2022-04-06T08:08:59.317904Z", - "shell.execute_reply": "2022-04-06T08:08:59.317392Z" - } - }, - "outputs": [], - "source": [ - "section_of_interest = ds.isel(x=0)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 4 Downsample (time dimension)\n", - "We currently have measurements at 3 time steps, with 30.001 seconds inbetween. For our next exercise we would like to down sample the measurements to 2 time steps with 47 seconds inbetween. The calculated variances are not valid anymore. We use the function `resample` from xarray." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# We use the logic from xarray to resample. However, it returns an xarray dataset type. Therefore we convert it back to the dtscalibration Datastore type.\n", - "ds_resampled = xr.Dataset(ds.resample(time=\"47S\").mean())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 5 Upsample / Interpolation (length and time dimension)\n", - "So we have measurements every 0.12 cm starting at $x=0$ m. What if we would like to change our coordinate system to have a value every 12 cm starting at $x=0.05$ m. We use (linear) interpolation, extrapolation is not supported. The calculated variances are not valid anymore." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "execution": { - "iopub.execute_input": "2022-04-06T08:08:59.344388Z", - "iopub.status.busy": "2022-04-06T08:08:59.344165Z", - "iopub.status.idle": "2022-04-06T08:08:59.353186Z", - "shell.execute_reply": "2022-04-06T08:08:59.352734Z" - } - }, - "outputs": [], - "source": [ - "x_old = ds.x.data\n", - "x_new = x_old[:-1] + 0.05 # no extrapolation\n", - "ds_xinterped = ds.interp(coords={\"x\": x_new})" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can do the same in the time dimension" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "execution": { - "iopub.execute_input": "2022-04-06T08:08:59.355702Z", - "iopub.status.busy": "2022-04-06T08:08:59.355479Z", - "iopub.status.idle": "2022-04-06T08:08:59.371585Z", - "shell.execute_reply": "2022-04-06T08:08:59.371063Z" - } - }, - "outputs": [], - "source": [ - "import numpy as np\n", - "\n", - "time_old = ds.time.data\n", - "time_new = time_old + np.timedelta64(10, \"s\")\n", - "ds_tinterped = ds.interp(coords={\"time\": time_new})" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.10" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} From 4b86bdbb8bad85ed51cafdff4bc4e701e5b12ccc Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Fri, 27 Oct 2023 14:59:07 +0200 Subject: [PATCH 60/66] Remove old reference to DataStore in notebooks --- .../01Load_xml_measurement_files.ipynb | 22 ++++--------------- docs/notebooks/03Define_sections.ipynb | 4 ++-- .../04Calculate_variance_Stokes.ipynb | 4 ++-- .../10Align_double_ended_measurements.ipynb | 1 - .../12Datastore_from_numpy_arrays.ipynb | 8 +++---- docs/notebooks/A2Load_sensornet_files.ipynb | 2 +- docs/notebooks/A3Load_ap_sensing_files.ipynb | 2 +- docs/notebooks/A4Load_sensortran_files.ipynb | 2 +- 8 files changed, 15 insertions(+), 30 deletions(-) diff --git a/docs/notebooks/01Load_xml_measurement_files.ipynb b/docs/notebooks/01Load_xml_measurement_files.ipynb index c25396aa..506249f6 100644 --- a/docs/notebooks/01Load_xml_measurement_files.ipynb +++ b/docs/notebooks/01Load_xml_measurement_files.ipynb @@ -7,13 +7,13 @@ "source": [ "# 1. Load your first measurement files\n", "\n", - "The goal of this notebook is to show the different options of loading measurements from raw DTS files. These files are loaded into a `DataStore` object. This object has various methods for calibration, plotting. The current supported devices are:\n", + "The goal of this notebook is to show the different options of loading measurements from raw DTS files. These files are loaded into a `xarray.Dataset` object. This object has various methods for calibration, plotting. Both single-ended and double-ended measurements are supported. The current supported devices are:\n", "- Silixa\n", "- Sensornet\n", + "- AP Sensing\n", + "- Sensortran\n", "\n", - "This example loads Silixa files. Both single-ended and double-ended measurements are supported. The first step is to load the correct read routine from `dtscalibration`.\n", - "- Silixa -> `dtscalibration.read_silixa_files`\n", - "- Sensornet -> `dtscalibration.read_sensornet_files`" + "See notebooks A2, A3, and A4." ] }, { @@ -125,20 +125,6 @@ "source": [ "print(ds)" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/docs/notebooks/03Define_sections.ipynb b/docs/notebooks/03Define_sections.ipynb index 74c56371..6e390a1f 100644 --- a/docs/notebooks/03Define_sections.ipynb +++ b/docs/notebooks/03Define_sections.ipynb @@ -5,7 +5,7 @@ "metadata": {}, "source": [ "# 3. Define calibration sections\n", - "The goal of this notebook is to show how you can define calibration sections. That means that we define certain parts of the fiber to a timeseries of temperature measurements. Here, we assume the temperature timeseries is already part of the `DataStore` object." + "The goal of this notebook is to show how you can define calibration sections. That means that we define certain parts of the fiber to a timeseries of temperature measurements. Here, we assume the temperature timeseries is already part of the `xarray.Dataset` object." ] }, { @@ -68,7 +68,7 @@ "from dtscalibration.dts_accessor import DtsAccessor # noqa: E402\n", "\n", "print(ds.dts.get_timeseries_keys()) # list the available timeseeries\n", - "ds.probe1Temperature.plot(figsize=(12, 8));" + "ds.probe1Temperature.plot(figsize=(12, 8))" ] }, { diff --git a/docs/notebooks/04Calculate_variance_Stokes.ipynb b/docs/notebooks/04Calculate_variance_Stokes.ipynb index f4dd9647..ec8cbb78 100644 --- a/docs/notebooks/04Calculate_variance_Stokes.ipynb +++ b/docs/notebooks/04Calculate_variance_Stokes.ipynb @@ -92,10 +92,10 @@ "source": [ "The variance in the Stokes signal will vary along the length of the fiber. There are multiple ways to approach this, each has its own pros and cons. **It is important to consider which model you use for your setup, as this will impact the calibration weights and predicted uncertainty.**\n", "\n", - "- In small setups with small variations in Stokes intensity, `ds.variance_stokes_constant` can be used. This function determines a single (constant) value for the variance. This method is not recommended for larger setups (e.g., >300 m) due to the signal strength dependency of the variance.\n", + "- In small setups with small variations in Stokes intensity, `variance_stokes_constant` can be used. This function determines a single (constant) value for the variance. This method is not recommended for larger setups (e.g., >300 m) due to the signal strength dependency of the variance.\n", "\n", "\n", - "- For larger setups `ds.variance_stokes_linear` should be used. This function assumes a linear relationship between the Stokes signal strength and variance. Tests on Silixa and Sensornet devices indicate this relationship is linear, and (approximately) goes through the origin; i.e. at 0 Stokes intensity, the signal variance is very close to 0.\n", + "- For larger setups `variance_stokes_linear` should be used. This function assumes a linear relationship between the Stokes signal strength and variance. Tests on Silixa and Sensornet devices indicate this relationship is linear, and (approximately) goes through the origin; i.e. at 0 Stokes intensity, the signal variance is very close to 0.\n", "\n", "\n", "- `variance_stokes_exponential` can be used for small setups with very few time steps. Too many degrees of freedom results in an under estimation of the noise variance. Almost never the case, but use when calibrating e.g. a single time step." diff --git a/docs/notebooks/10Align_double_ended_measurements.ipynb b/docs/notebooks/10Align_double_ended_measurements.ipynb index d80d9f02..911f2dea 100644 --- a/docs/notebooks/10Align_double_ended_measurements.ipynb +++ b/docs/notebooks/10Align_double_ended_measurements.ipynb @@ -38,7 +38,6 @@ " shift_double_ended,\n", ")\n", "import numpy as np\n", - "import matplotlib.pyplot as plt\n", "\n", "%matplotlib inline" ] diff --git a/docs/notebooks/12Datastore_from_numpy_arrays.ipynb b/docs/notebooks/12Datastore_from_numpy_arrays.ipynb index 83748210..1d17574c 100644 --- a/docs/notebooks/12Datastore_from_numpy_arrays.ipynb +++ b/docs/notebooks/12Datastore_from_numpy_arrays.ipynb @@ -5,7 +5,7 @@ "metadata": {}, "source": [ "# 12. Creating a Dataset from numpy arrays\n", - "The goal of this notebook is to demonstrate how to create a `DataStore` from scratch. This can be useful if your device is not supported or if you would like to integrate the `dtscalibration` library in your current routine." + "The goal of this notebook is to demonstrate how to create a `xarray.Dataset` from scratch. This can be useful if your device is not supported or if you would like to integrate the `dtscalibration` library in your current routine." ] }, { @@ -35,7 +35,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "For a `DataStore` object, a few things are needed:\n", + "For a `xarray.Dataset` object, a few things are needed:\n", "\n", "- timestamps\n", "\n", @@ -70,7 +70,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We will get all the numpy arrays from this `DataStore` to create a new one from 'scratch'.\n", + "We will get all the numpy arrays from this `xarray.Dataset` to create a new one from 'scratch'.\n", "\n", "Let's start with the most basic data:" ] @@ -149,7 +149,7 @@ "\n", "- a double ended flag\n", "\n", - "We'll put these into the custom `DataStore`:" + "We'll put these into the custom `xarray.Dataset`:" ] }, { diff --git a/docs/notebooks/A2Load_sensornet_files.ipynb b/docs/notebooks/A2Load_sensornet_files.ipynb index 58069ca1..88fff3f7 100644 --- a/docs/notebooks/A2Load_sensornet_files.ipynb +++ b/docs/notebooks/A2Load_sensornet_files.ipynb @@ -98,7 +98,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The object tries to gather as much metadata from the measurement files as possible (temporal and spatial coordinates, filenames, temperature probes measurements). All other configuration settings are loaded from the first files and stored as attributes of the `DataStore`." + "The object tries to gather as much metadata from the measurement files as possible (temporal and spatial coordinates, filenames, temperature probes measurements). All other configuration settings are loaded from the first files and stored as attributes of the `xarray.Dataset`." ] }, { diff --git a/docs/notebooks/A3Load_ap_sensing_files.ipynb b/docs/notebooks/A3Load_ap_sensing_files.ipynb index 810fc920..91a6e7b6 100644 --- a/docs/notebooks/A3Load_ap_sensing_files.ipynb +++ b/docs/notebooks/A3Load_ap_sensing_files.ipynb @@ -98,7 +98,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The object tries to gather as much metadata from the measurement files as possible (temporal and spatial coordinates, filenames, temperature probes measurements). All other configuration settings are loaded from the first files and stored as attributes of the `DataStore`.\n", + "The object tries to gather as much metadata from the measurement files as possible (temporal and spatial coordinates, filenames, temperature probes measurements). All other configuration settings are loaded from the first files and stored as attributes of the `xarray.Dataset`.\n", "\n", "Calibration follows as usual (see the other notebooks)." ] diff --git a/docs/notebooks/A4Load_sensortran_files.ipynb b/docs/notebooks/A4Load_sensortran_files.ipynb index 1a045a41..1db6488f 100644 --- a/docs/notebooks/A4Load_sensortran_files.ipynb +++ b/docs/notebooks/A4Load_sensortran_files.ipynb @@ -103,7 +103,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The object tries to gather as much metadata from the measurement files as possible (temporal and spatial coordinates, filenames, temperature probes measurements). All other configuration settings are loaded from the first files and stored as attributes of the `DataStore`. Sensortran's data files contain less information than the other manufacturer's devices, one being the acquisition time. The acquisition time is needed for estimating variances, and is set a constant 1s." + "The object tries to gather as much metadata from the measurement files as possible (temporal and spatial coordinates, filenames, temperature probes measurements). All other configuration settings are loaded from the first files and stored as attributes of the `xarray.Dataset`. Sensortran's data files contain less information than the other manufacturer's devices, one being the acquisition time. The acquisition time is needed for estimating variances, and is set a constant 1s." ] }, { From cba03b67d9fa25bb38b1d691994972b4515ddd50 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Fri, 27 Oct 2023 15:08:54 +0200 Subject: [PATCH 61/66] Removed references to ds.variance_*() from docs --- src/dtscalibration/dts_accessor.py | 2 +- src/dtscalibration/plot.py | 6 +++--- src/dtscalibration/variance_helpers.py | 2 +- src/dtscalibration/variance_stokes.py | 18 +++++++++--------- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/dtscalibration/dts_accessor.py b/src/dtscalibration/dts_accessor.py index 10489b8a..ed33cabb 100644 --- a/src/dtscalibration/dts_accessor.py +++ b/src/dtscalibration/dts_accessor.py @@ -1443,7 +1443,7 @@ def monte_carlo_double_ended( confidence intervals for the temperature. First, the variances of the Stokes and anti-Stokes intensity measurements of the forward and backward channels are estimated following the steps in - Section 4 [1]_. See `ds.variance_stokes_constant()`. + Section 4 [1]_. See `variance_stokes_constant()`. A Normal distribution is assigned to each intensity measurement that is centered at the measurement and using the estimated variance. Second, a multi-variate Normal distribution is diff --git a/src/dtscalibration/plot.py b/src/dtscalibration/plot.py index 867abe4f..d864cafe 100644 --- a/src/dtscalibration/plot.py +++ b/src/dtscalibration/plot.py @@ -26,7 +26,7 @@ def plot_residuals_reference_sections( plot_avg_std resid : DataArray The residuals of the fit to estimate the noise in the measured - Stokes signal. is returned by `ds.variance_stokes` + Stokes signal. is returned by `variance_stokes_*()` sections : Dict[str, List[slice]] The sections obj is normally used to set DataStore.sections, now is used toobtain the @@ -254,7 +254,7 @@ def plot_residuals_reference_sections_single( plot_avg_std resid : DataArray The residuals of the fit to estimate the noise in the measured - Stokes signal. is returned by `ds.variance_stokes` + Stokes signal. is returned by `variance_stokes_*()` fig : Figurehandle, optional title : str, optional Adds a title to the plot @@ -389,7 +389,7 @@ def plot_accuracy( plot_avg_std resid : DataArray The residuals of the fit to estimate the noise in the measured - Stokes signal. is returned by `ds.variance_stokes` + Stokes signal. is returned by `variance_stokes_*()` fig : Figurehandle, optional title : str, optional Adds a title to the plot diff --git a/src/dtscalibration/variance_helpers.py b/src/dtscalibration/variance_helpers.py index 6420e5a5..9ea6c834 100644 --- a/src/dtscalibration/variance_helpers.py +++ b/src/dtscalibration/variance_helpers.py @@ -158,7 +158,7 @@ def variance_stokes_linear_helper(st_sec, resid_sec, nbin, through_zero): "not possible. Most likely, your Stokes intensities do " "not vary enough to fit a linear curve. Either " "use `through_zero` option or use " - "`ds.variance_stokes_constant()`. Another reason " + "`variance_stokes_constant()`. Another reason " "could be that your sections are defined to be " "wider than they actually are." ) diff --git a/src/dtscalibration/variance_stokes.py b/src/dtscalibration/variance_stokes.py index 0d37db1b..59819b87 100644 --- a/src/dtscalibration/variance_stokes.py +++ b/src/dtscalibration/variance_stokes.py @@ -16,16 +16,16 @@ def variance_stokes_constant(st, sections, acquisitiontime, reshape_residuals=Tr Approximate the variance of the noise in Stokes intensity measurements with one value, suitable for small setups. - * `ds.variance_stokes_constant()` for small setups with small variations in\ + * `variance_stokes_constant()` for small setups with small variations in\ intensity. Variance of the Stokes measurements is assumed to be the same\ along the entire fiber. - * `ds.variance_stokes_exponential()` for small setups with very few time\ + * `variance_stokes_exponential()` for small setups with very few time\ steps. Too many degrees of freedom results in an under estimation of the\ noise variance. Almost never the case, but use when calibrating pre time\ step. - * `ds.variance_stokes_linear()` for larger setups with more time steps.\ + * `variance_stokes_linear()` for larger setups with more time steps.\ Assumes Poisson distributed noise with the following model:: st_var = a * ds.st + b @@ -157,16 +157,16 @@ def variance_stokes_exponential( with one value, suitable for small setups with measurements from only a few times. - * `ds.variance_stokes_constant()` for small setups with small variations in\ + * `variance_stokes_constant()` for small setups with small variations in\ intensity. Variance of the Stokes measurements is assumed to be the same\ along the entire fiber. - * `ds.variance_stokes_exponential()` for small setups with very few time\ + * `variance_stokes_exponential()` for small setups with very few time\ steps. Too many degrees of freedom results in an under estimation of the\ noise variance. Almost never the case, but use when calibrating pre time\ step. - * `ds.variance_stokes_linear()` for larger setups with more time steps.\ + * `variance_stokes_linear()` for larger setups with more time steps.\ Assumes Poisson distributed noise with the following model:: st_var = a * ds.st + b @@ -352,16 +352,16 @@ def variance_stokes_linear( Approximate the variance of the noise in Stokes intensity measurements with a linear function of the intensity, suitable for large setups. - * `ds.variance_stokes_constant()` for small setups with small variations in\ + * `variance_stokes_constant()` for small setups with small variations in\ intensity. Variance of the Stokes measurements is assumed to be the same\ along the entire fiber. - * `ds.variance_stokes_exponential()` for small setups with very few time\ + * `variance_stokes_exponential()` for small setups with very few time\ steps. Too many degrees of freedom results in an under estimation of the\ noise variance. Almost never the case, but use when calibrating pre time\ step. - * `ds.variance_stokes_linear()` for larger setups with more time steps.\ + * `variance_stokes_linear()` for larger setups with more time steps.\ Assumes Poisson distributed noise with the following model:: st_var = a * ds.st + b From 98636eae1bd3349aef86bfa684fd2e404283c0e0 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Fri, 27 Oct 2023 15:11:47 +0200 Subject: [PATCH 62/66] Update src/dtscalibration/dts_accessor.py Co-authored-by: Bart Schilperoort --- src/dtscalibration/dts_accessor.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/dtscalibration/dts_accessor.py b/src/dtscalibration/dts_accessor.py index ed33cabb..b841b4c3 100644 --- a/src/dtscalibration/dts_accessor.py +++ b/src/dtscalibration/dts_accessor.py @@ -41,7 +41,6 @@ def __init__(self, xarray_obj): self.acquisitiontime_fw = xarray_obj.get("userAcquisitionTimeFW") self.acquisitiontime_bw = xarray_obj.get("userAcquisitionTimeBW") - pass def __repr__(self): # __repr__ from xarray is used and edited. From 4da18d7932fcd2677c886753d6a4585923b62cd1 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Fri, 27 Oct 2023 15:43:59 +0200 Subject: [PATCH 63/66] Update test_examples.py --- tests/test_examples.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_examples.py b/tests/test_examples.py index 3e5d348b..e0efbf14 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -18,7 +18,7 @@ "src_path", Path(src_dir).glob("*.ipynb"), ids=lambda x: x.name ) def test_docs_notebook(src_path): - _test_notebook(src_path, "python3") + print(_test_notebook(src_path, "python3")) @pytest.mark.xfail From 1f40022c4408a2acabb2eefcf39370a3e11e7b6b Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Fri, 27 Oct 2023 15:44:56 +0200 Subject: [PATCH 64/66] Update docs --- docs/conf.py | 6 +++++- docs/reference/index.rst | 44 +++++++++++++++++++++++++++++++++++----- 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 6e28eadd..6ef238c1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -2,6 +2,9 @@ from datetime import date import os +from dtscalibration.dts_accessor import DtsAccessor # noqa: E402 +import sphinx_autosummary_accessors + extensions = [ "sphinx_rtd_theme", @@ -15,6 +18,7 @@ "sphinx.ext.todo", "sphinx.ext.viewcode", "sphinx.ext.autosectionlabel", + "sphinx_autosummary_accessors", "nbsphinx", "sphinx.ext.mathjax", "sphinx.ext.intersphinx", @@ -45,7 +49,7 @@ version = release = "2.0.0" pygments_style = "trac" -templates_path = ["."] +templates_path = [".", sphinx_autosummary_accessors.templates_path] extlinks = { "issue": ( "https://github.com/dtscalibration/python-dts-calibration/issues" "/%s", diff --git a/docs/reference/index.rst b/docs/reference/index.rst index 9d9e4821..a8df48bd 100644 --- a/docs/reference/index.rst +++ b/docs/reference/index.rst @@ -1,10 +1,44 @@ Reference ========= -.. toctree:: - :glob: +Load the data +------------- -.. automodapi:: dtscalibration - :skip: plot_dask +.. automodule:: dtscalibration.io :members: - :no-inheritance-diagram: + :nosignatures: + +Compute the variance in the Stokes measurements +----------------------------------------------- + +.. automodule:: dtscalibration.variance_stokes + :members: + :nosignatures: + +The DTS Accessor +---------------- + +.. currentmodule:: xarray +.. autosummary:: + :toctree: generated/ + :template: autosummary/accessor_method.rst + :nosignatures: + + Dataset.dts.sections + Dataset.dts.calibrate_single_ended + Dataset.dts.calibrate_double_ended + Dataset.dts.monte_carlo_single_ended + Dataset.dts.monte_carlo_double_ended + Dataset.dts.average_monte_carlo_single_ended + Dataset.dts.average_monte_carlo_double_ended + Dataset.dts.get_default_encoding + Dataset.dts.get_timeseries_keys + Dataset.dts.matching_sections + Dataset.dts.ufunc_per_section + +Plot the results +---------------- + +.. automodule:: dtscalibration.plot + :members: + :nosignatures: \ No newline at end of file From b56918b81d8da67d06298c73366019df7b510473 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Fri, 27 Oct 2023 16:25:54 +0200 Subject: [PATCH 65/66] Minimal attempt to fix docs --- .gitignore | 1 + docs/conf.py | 11 ++++++----- docs/reference/index.rst | 9 ++++++++- pyproject.toml | 3 ++- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index f06bd184..dd531a85 100644 --- a/.gitignore +++ b/.gitignore @@ -57,6 +57,7 @@ output/*/index.html # Sphinx docs/_build +**/generated/**/* .DS_Store *~ diff --git a/docs/conf.py b/docs/conf.py index 6ef238c1..d69bb728 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,10 +1,11 @@ -# -*- coding: utf-8 -*- -from datetime import date import os +from datetime import date -from dtscalibration.dts_accessor import DtsAccessor # noqa: E402 +from xarray import Dataset # noqa: E402 import sphinx_autosummary_accessors +import dtscalibration # noqa: E402 +from dtscalibration.dts_accessor import DtsAccessor # noqa: E402 extensions = [ "sphinx_rtd_theme", @@ -45,7 +46,7 @@ project = "dtscalibration" year = str(date.today().year) author = "Bas des Tombe and Bart Schilperoort" -copyright = "{0}, {1}".format(year, author) +copyright = f"{year}, {author}" version = release = "2.0.0" pygments_style = "trac" @@ -67,7 +68,7 @@ html_sidebars = { "**": ["searchbox.html", "globaltoc.html", "sourcelink.html"], } -html_short_title = "%s-%s" % (project, version) +html_short_title = f"{project}-{version}" napoleon_use_ivar = True napoleon_use_rtype = False diff --git a/docs/reference/index.rst b/docs/reference/index.rst index a8df48bd..73c3961c 100644 --- a/docs/reference/index.rst +++ b/docs/reference/index.rst @@ -4,20 +4,27 @@ Reference Load the data ------------- +See example notebooks 01, A2, A3, and A4. + .. automodule:: dtscalibration.io - :members: + :members: dtscalibration.read_apsensing_files :nosignatures: Compute the variance in the Stokes measurements ----------------------------------------------- +See example notebooks 04 and have a look at the docstring of the dtscalibration.variance_stokes funcitons. + .. automodule:: dtscalibration.variance_stokes :members: :nosignatures: + The DTS Accessor ---------------- +See example natebooks 07, 08, and 17. + .. currentmodule:: xarray .. autosummary:: :toctree: generated/ diff --git a/pyproject.toml b/pyproject.toml index 682d0c3e..4fae6751 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -91,6 +91,7 @@ docs = [ # Required for ReadTheDocs "sphinx_rtd_theme", "sphinx-autoapi", "sphinx-automodapi", + "sphinx-autosummary-accessors", "coverage[toml]", "nbsphinx", "ipykernel", @@ -122,7 +123,7 @@ features = ["docs"] [tool.hatch.envs.docs.scripts] build = [ - "sphinx-build -c docs -b html docs dist/docs", #"python docs/nb_examples_to_docs.py", + "sphinx-build -E -c docs -b html docs dist/docs", #"python docs/nb_examples_to_docs.py", ] [tool.hatch.envs.matrix_test] From 2cf247c4e990559db212e4b2d11a605c3b6a270e Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Fri, 27 Oct 2023 16:41:00 +0200 Subject: [PATCH 66/66] Update README.rst --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index 3c645cb9..03f01ee2 100644 --- a/README.rst +++ b/README.rst @@ -93,6 +93,7 @@ Devices currently supported Documentation ============= +* A full calibration procedure for single-ended setups is presented in notebook `07Calibrate_single_ended.ipynb `_ and for double-ended setups in `08Calibrate_double_ended.ipynb `_. * Documentation at https://python-dts-calibration.readthedocs.io/ . * Example notebooks that work within the browser can be viewed `here `_.