From 62cf7819bf5bde3952fe1786b2c491abeaca6b64 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Fri, 21 Jul 2023 11:32:43 +0200 Subject: [PATCH 1/5] Also verify timedeltas between forward and backward when merging single ended to double ended --- src/dtscalibration/datastore_utils.py | 36 ++++++++++++++++++++++++--- tests/test_datastore.py | 2 +- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/src/dtscalibration/datastore_utils.py b/src/dtscalibration/datastore_utils.py index 1e1d3a79..692a322e 100644 --- a/src/dtscalibration/datastore_utils.py +++ b/src/dtscalibration/datastore_utils.py @@ -156,7 +156,7 @@ def merge_double_ended(ds_fw, ds_bw, cable_length, plot_result=True, verbose=Tru return ds -def merge_double_ended_times(ds_fw, ds_bw, verbose=True): +def merge_double_ended_times(ds_fw, ds_bw, verify_timedeltas=True, verbose=True): """Helper for `merge_double_ended()` to deal with missing measurements. The number of measurements of the forward and backward channels might get out of sync if the device shuts down before the measurement of the last channel @@ -189,6 +189,10 @@ def merge_double_ended_times(ds_fw, ds_bw, verbose=True): DataStore object representing the forward measurement channel ds_bw : DataSore object DataStore object representing the backward measurement channel + verify_timedeltas : bool + Check whether times between forward and backward measurements are similar to those of neighboring measurements + verbose : bool + Print additional information to screen Returns ------- @@ -205,8 +209,16 @@ def merge_double_ended_times(ds_fw, ds_bw, verbose=True): assert ds_fw.attrs['forwardMeasurementChannel'] < ds_bw.attrs['forwardMeasurementChannel'], \ "ds_fw and ds_bw are swapped" + # Are all dt's within 1.5 seconds from one another? if (ds_bw.time.size == ds_fw.time.size) and np.all(ds_bw.time.values > ds_fw.time.values): - return ds_fw, ds_bw + if verify_timedeltas: + dt_ori = (ds_bw.time.values - ds_fw.time.values) / np.array(1000000000, dtype='timedelta64[ns]') + dt_all_close = np.allclose(dt_ori, dt_ori[0], atol=1.5, rtol=0.) + else: + dt_all_close = True + + if dt_all_close: + return ds_fw, ds_bw iuse_chfw = list() iuse_chbw = list() @@ -232,7 +244,25 @@ def merge_double_ended_times(ds_fw, ds_bw, verbose=True): if verbose: print(f"Missing forward measurement beween {ds_bw.time.values[ind]} and {ds_bw.time.values[ind_next]}") - return ds_fw.isel(time=iuse_chfw), ds_bw.isel(time=iuse_chbw) + # throw out is dt differs from its neighbors + if verify_timedeltas: + dt = (ds_bw.isel(time=iuse_chbw).time.values - ds_fw.isel(time=iuse_chfw).time.values) /\ + np.array(1000000000, dtype='timedelta64[ns]') + leaveout = np.zeros_like(dt, dtype=bool) + leaveout[1:-1] = np.isclose(dt[:-2], dt[2:], atol=1.5, rtol=0.) * ~np.isclose(dt[:-2], dt[1:-1], atol=1.5, rtol=0.) + iuse_chfw2 = np.array(iuse_chfw)[~leaveout] + iuse_chbw2 = np.array(iuse_chbw)[~leaveout] + + if verbose: + for itfw, itbw in zip(np.array(iuse_chfw)[leaveout], np.array(iuse_chbw)[leaveout]): + print(f"FW: {ds_fw.isel(time=itfw).time.values} and BW: {ds_bw.isel(time=itbw).time.values} do not " + f"belong together as their timedelta is larger than their neighboring timedeltas. Thrown out.") + + else: + iuse_chfw2 = iuse_chfw + iuse_chbw2 = iuse_chbw + + return ds_fw.isel(time=iuse_chfw2), ds_bw.isel(time=iuse_chbw2) # pylint: disable=too-many-locals diff --git a/tests/test_datastore.py b/tests/test_datastore.py index 8f1e3534..11aaf66a 100644 --- a/tests/test_datastore.py +++ b/tests/test_datastore.py @@ -649,7 +649,7 @@ def test_merge_double_ended(): ([1, 2], [], [1, 2]), ([], [1, 2], [1, 2]), ([1], [2], [1, 2]), - pytest.param([2], [1], [1, 2], marks=pytest.mark.xfail) + ([2], [1], [1, 2]) ]) def test_merge_double_ended_times(inotinfw, inotinbw, inotinout): """ From 17d796335ca93eec21fbffe75f3cdb8a8a1c9c5e Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Mon, 24 Jul 2023 19:41:53 +0200 Subject: [PATCH 2/5] Minor update to verification_timedeltas in merge_double_ended_times timedelta64[ns] to timedelta64[s] Co-authored-by: Bart Schilperoort --- src/dtscalibration/datastore_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dtscalibration/datastore_utils.py b/src/dtscalibration/datastore_utils.py index 692a322e..20e87227 100644 --- a/src/dtscalibration/datastore_utils.py +++ b/src/dtscalibration/datastore_utils.py @@ -247,7 +247,7 @@ def merge_double_ended_times(ds_fw, ds_bw, verify_timedeltas=True, verbose=True) # throw out is dt differs from its neighbors if verify_timedeltas: dt = (ds_bw.isel(time=iuse_chbw).time.values - ds_fw.isel(time=iuse_chfw).time.values) /\ - np.array(1000000000, dtype='timedelta64[ns]') + np.timedelta64(1, "s") leaveout = np.zeros_like(dt, dtype=bool) leaveout[1:-1] = np.isclose(dt[:-2], dt[2:], atol=1.5, rtol=0.) * ~np.isclose(dt[:-2], dt[1:-1], atol=1.5, rtol=0.) iuse_chfw2 = np.array(iuse_chfw)[~leaveout] From 3606e85eed6de8b0d4275a8fb5831ab33a87ad7a Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Mon, 24 Jul 2023 19:48:52 +0200 Subject: [PATCH 3/5] Minor update to verification_timedeltas in merge_double_ended_times Rearrange print statements. Suggested by @BSchilperoort Co-authored-by: Bart Schilperoort --- src/dtscalibration/datastore_utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/dtscalibration/datastore_utils.py b/src/dtscalibration/datastore_utils.py index 20e87227..7be2a056 100644 --- a/src/dtscalibration/datastore_utils.py +++ b/src/dtscalibration/datastore_utils.py @@ -255,8 +255,11 @@ def merge_double_ended_times(ds_fw, ds_bw, verify_timedeltas=True, verbose=True) if verbose: for itfw, itbw in zip(np.array(iuse_chfw)[leaveout], np.array(iuse_chbw)[leaveout]): - print(f"FW: {ds_fw.isel(time=itfw).time.values} and BW: {ds_bw.isel(time=itbw).time.values} do not " - f"belong together as their timedelta is larger than their neighboring timedeltas. Thrown out.") + print( + "The following measurements do not belong together, as the time difference\n" + "between the\forward and backward measurements is more than 1.5 seconds\n" + "larger than the neighboring measurements.\n" + f"FW: {ds_fw.isel(time=itfw).time.values} and BW: {ds_bw.isel(time=itbw).time.values}" else: iuse_chfw2 = iuse_chfw From 17ee04d74ef999aef4427c47ad81b8a9e4e765c4 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Mon, 24 Jul 2023 19:52:47 +0200 Subject: [PATCH 4/5] Use brackets instead of linebreaks --- src/dtscalibration/datastore_utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/dtscalibration/datastore_utils.py b/src/dtscalibration/datastore_utils.py index 7be2a056..282cf435 100644 --- a/src/dtscalibration/datastore_utils.py +++ b/src/dtscalibration/datastore_utils.py @@ -246,8 +246,9 @@ def merge_double_ended_times(ds_fw, ds_bw, verify_timedeltas=True, verbose=True) # throw out is dt differs from its neighbors if verify_timedeltas: - dt = (ds_bw.isel(time=iuse_chbw).time.values - ds_fw.isel(time=iuse_chfw).time.values) /\ - np.timedelta64(1, "s") + dt = ( + (ds_bw.isel(time=iuse_chbw).time.values - ds_fw.isel(time=iuse_chfw).time.values) / + np.timedelta64(1, "s")) leaveout = np.zeros_like(dt, dtype=bool) leaveout[1:-1] = np.isclose(dt[:-2], dt[2:], atol=1.5, rtol=0.) * ~np.isclose(dt[:-2], dt[1:-1], atol=1.5, rtol=0.) iuse_chfw2 = np.array(iuse_chfw)[~leaveout] From a6b374d8c34add0cf0d79451e27e59f13f477fdf Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Mon, 24 Jul 2023 19:54:52 +0200 Subject: [PATCH 5/5] Forgotten closing bracket in print statement --- src/dtscalibration/datastore_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dtscalibration/datastore_utils.py b/src/dtscalibration/datastore_utils.py index 282cf435..f5cb92c7 100644 --- a/src/dtscalibration/datastore_utils.py +++ b/src/dtscalibration/datastore_utils.py @@ -260,7 +260,7 @@ def merge_double_ended_times(ds_fw, ds_bw, verify_timedeltas=True, verbose=True) "The following measurements do not belong together, as the time difference\n" "between the\forward and backward measurements is more than 1.5 seconds\n" "larger than the neighboring measurements.\n" - f"FW: {ds_fw.isel(time=itfw).time.values} and BW: {ds_bw.isel(time=itbw).time.values}" + f"FW: {ds_fw.isel(time=itfw).time.values} and BW: {ds_bw.isel(time=itbw).time.values}") else: iuse_chfw2 = iuse_chfw