Refactor datastore (#202)

* User-friendly error when setting manually ds.sections

Solves #200

* User-friendly error when setting manually ds.sections

Solves #200

* Revert accidental double implementation of sections.setter

* Update documentation of datastore class

* Removed numbagg from dependencies

llvmlite is a pain to install with pip. Alternatively, it could be installed with conda

* Explicitly pass sections to more functions

* Format

* Moving away from calibrate_double_ended() inplace

self.update(out) is still in place at the end

* Refactored calibrate_double_ended covariance calculation

* Parallel testing

* refactored variance functions

* Include pyupgrade flag of ruff

* Refactored single ended routine p_val to params and moved averaging routines

* Update .gitignore

* Refactored calibration routines

In preparation of the accessor

* Please ruff

* Refactored Datastore class

* Updated sections example notebook

* Updated example notebooks

* Add api to docs

* Unfortenate formatting of notebooks

* Fixes to minor comments from Bart

.github/workflows/python-publish-dry-run.yml

@@ -34,5 +34,5 @@ jobs:
         pip install build
     - name: Build package
       run: python -m build
-    - name: Test build
+    - name: Check long description for PyPi
       run: twine check --strict dist/*

.gitignore

@@ -72,3 +72,4 @@ docs/_build
 .appveyor.token
 *.bak
 .vscode/settings.json
+*.code-workspace

docs/api/dtscalibration.DataStore.rst

@@ -0,0 +1,7 @@
+DataStore
+=========
+
+.. currentmodule:: dtscalibration
+
+.. autoclass:: DataStore
+   :show-inheritance:

docs/api/dtscalibration.check_dims.rst

@@ -0,0 +1,6 @@
+check_dims
+==========
+
+.. currentmodule:: dtscalibration
+
+.. autofunction:: check_dims

docs/api/dtscalibration.check_timestep_allclose.rst

@@ -0,0 +1,6 @@
+check_timestep_allclose
+=======================
+
+.. currentmodule:: dtscalibration
+
+.. autofunction:: check_timestep_allclose

docs/api/dtscalibration.get_netcdf_encoding.rst

@@ -0,0 +1,6 @@
+get_netcdf_encoding
+===================
+
+.. currentmodule:: dtscalibration
+
+.. autofunction:: get_netcdf_encoding

docs/api/dtscalibration.merge_double_ended.rst

@@ -0,0 +1,6 @@
+merge_double_ended
+==================
+
+.. currentmodule:: dtscalibration
+
+.. autofunction:: merge_double_ended

docs/api/dtscalibration.open_datastore.rst

@@ -0,0 +1,6 @@
+open_datastore
+==============
+
+.. currentmodule:: dtscalibration
+
+.. autofunction:: open_datastore

docs/api/dtscalibration.open_mf_datastore.rst

@@ -0,0 +1,6 @@
+open_mf_datastore
+=================
+
+.. currentmodule:: dtscalibration
+
+.. autofunction:: open_mf_datastore

docs/api/dtscalibration.plot_accuracy.rst

@@ -0,0 +1,6 @@
+plot_accuracy
+=============
+
+.. currentmodule:: dtscalibration
+
+.. autofunction:: plot_accuracy

docs/api/dtscalibration.plot_location_residuals_double_ended.rst

@@ -0,0 +1,6 @@
+plot_location_residuals_double_ended
+====================================
+
+.. currentmodule:: dtscalibration
+
+.. autofunction:: plot_location_residuals_double_ended

docs/api/dtscalibration.plot_residuals_reference_sections.rst

@@ -0,0 +1,6 @@
+plot_residuals_reference_sections
+=================================
+
+.. currentmodule:: dtscalibration
+
+.. autofunction:: plot_residuals_reference_sections

docs/api/dtscalibration.plot_residuals_reference_sections_single.rst

@@ -0,0 +1,6 @@
+plot_residuals_reference_sections_single
+========================================
+
+.. currentmodule:: dtscalibration
+
+.. autofunction:: plot_residuals_reference_sections_single

docs/api/dtscalibration.plot_sigma_report.rst

@@ -0,0 +1,6 @@
+plot_sigma_report
+=================
+
+.. currentmodule:: dtscalibration
+
+.. autofunction:: plot_sigma_report

docs/api/dtscalibration.read_apsensing_files.rst

@@ -0,0 +1,6 @@
+read_apsensing_files
+====================
+
+.. currentmodule:: dtscalibration
+
+.. autofunction:: read_apsensing_files

docs/api/dtscalibration.read_sensornet_files.rst

@@ -0,0 +1,6 @@
+read_sensornet_files
+====================
+
+.. currentmodule:: dtscalibration
+
+.. autofunction:: read_sensornet_files

docs/api/dtscalibration.read_sensortran_files.rst

@@ -0,0 +1,6 @@
+read_sensortran_files
+=====================
+
+.. currentmodule:: dtscalibration
+
+.. autofunction:: read_sensortran_files

docs/api/dtscalibration.read_silixa_files.rst

@@ -0,0 +1,6 @@
+read_silixa_files
+=================
+
+.. currentmodule:: dtscalibration
+
+.. autofunction:: read_silixa_files

docs/api/dtscalibration.shift_double_ended.rst

@@ -0,0 +1,6 @@
+shift_double_ended
+==================
+
+.. currentmodule:: dtscalibration
+
+.. autofunction:: shift_double_ended

docs/api/dtscalibration.suggest_cable_shift_double_ended.rst

@@ -0,0 +1,6 @@
+suggest_cable_shift_double_ended
+================================
+
+.. currentmodule:: dtscalibration
+
+.. autofunction:: suggest_cable_shift_double_ended

docs/notebooks/03Define_sections.ipynb

@@ -66,7 +66,8 @@
    "outputs": [],
    "source": [
     "print(ds.timeseries_keys)  # list the available timeseeries\n",
-    "ds.probe1Temperature.plot(figsize=(12, 8));  # plot one of the timeseries"
+    "ds.probe1Temperature.plot(figsize=(12, 8))\n",
+    "# plot one of the timeseries"
    ]
   },
   {
@@ -115,24 +116,7 @@
     "sections = {\n",
     "    \"probe1Temperature\": [slice(7.5, 17.0), slice(70.0, 80.0)],  # cold bath\n",
     "    \"probe2Temperature\": [slice(24.0, 34.0), slice(85.0, 95.0)],  # warm bath\n",
-    "}\n",
-    "ds.sections = sections"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "execution": {
-     "iopub.execute_input": "2022-04-06T08:09:10.309314Z",
-     "iopub.status.busy": "2022-04-06T08:09:10.308985Z",
-     "iopub.status.idle": "2022-04-06T08:09:10.323444Z",
-     "shell.execute_reply": "2022-04-06T08:09:10.322874Z"
-    }
-   },
-   "outputs": [],
-   "source": [
-    "ds.sections"
+    "}"
    ]
   },
   {
@@ -141,13 +125,6 @@
    "source": [
     "NetCDF files do not support reading/writing python dictionaries. Internally the sections dictionary is stored in `ds._sections` as a string encoded with yaml, which can be saved to a netCDF file. Each time the sections dictionary is requested, yaml decodes the string and evaluates it to the Python dictionary. "
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {
@@ -166,7 +143,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.11"
+   "version": "3.10.10"
   }
  },
  "nbformat": 4,

docs/notebooks/04Calculate_variance_Stokes.ipynb

@@ -81,8 +81,7 @@
     "sections = {\n",
     "    \"probe1Temperature\": [slice(7.5, 17.0), slice(70.0, 80.0)],  # cold bath\n",
     "    \"probe2Temperature\": [slice(24.0, 34.0), slice(85.0, 95.0)],  # warm bath\n",
-    "}\n",
-    "ds.sections = sections"
+    "}"
    ]
   },
   {
@@ -120,7 +119,7 @@
    },
    "outputs": [],
    "source": [
-    "I_var, residuals = ds.variance_stokes_constant(st_label=\"st\")\n",
+    "I_var, residuals = ds.variance_stokes_constant(sections=sections, st_label=\"st\")\n",
     "print(\n",
     "    \"The variance of the Stokes signal along the reference sections \"\n",
     "    \"is approximately {:.2f} on a {:.1f} sec acquisition time\".format(\n",
@@ -187,7 +186,7 @@
     "x = np.linspace(mean - 3 * sigma, mean + 3 * sigma, 100)\n",
     "approximated_normal_fit = scipy.stats.norm.pdf(x, mean, sigma)\n",
     "residuals.plot.hist(bins=50, figsize=(12, 8), density=True)\n",
-    "plt.plot(x, approximated_normal_fit);"
+    "plt.plot(x, approximated_normal_fit)"
    ]
   },
   {

docs/notebooks/07Calibrate_single_ended.ipynb

@@ -90,7 +90,7 @@
    "outputs": [],
    "source": [
     "ds = ds.sel(x=slice(-30, 101))  # dismiss parts of the fiber that are not interesting\n",
-    "ds.sections = {\n",
+    "sections = {\n",
     "    \"probe1Temperature\": [slice(20, 25.5)],  # warm bath\n",
     "    \"probe2Temperature\": [slice(5.5, 15.5)],  # cold bath\n",
     "}"
@@ -119,8 +119,8 @@
    },
    "outputs": [],
    "source": [
-    "st_var, resid = ds.variance_stokes_constant(st_label=\"st\")\n",
-    "ast_var, _ = ds.variance_stokes_constant(st_label=\"ast\")"
+    "st_var, resid = ds.variance_stokes_constant(sections=sections, st_label=\"st\")\n",
+    "ast_var, _ = ds.variance_stokes_constant(sections=sections, st_label=\"ast\")"
    ]
   },
   {
@@ -136,7 +136,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "resid.plot(figsize=(12, 4));"
+    "resid.plot(figsize=(12, 4))"
    ]
   },
   {
@@ -160,7 +160,7 @@
    },
    "outputs": [],
    "source": [
-    "ds.calibration_single_ended(st_var=st_var, ast_var=ast_var)"
+    "ds.calibration_single_ended(sections=sections, st_var=st_var, ast_var=ast_var)"
    ]
   },
   {
@@ -177,7 +177,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "ds.tmpf.plot(figsize=(12, 4));"
+    "ds.tmpf.plot(figsize=(12, 4))"
    ]
   },
   {
@@ -189,7 +189,7 @@
     "ds1 = ds.isel(time=0)\n",
     "ds1.tmpf.plot(figsize=(12, 4))\n",
     "(ds1.tmpf_var**0.5).plot(figsize=(12, 4))\n",
-    "plt.ylabel(\"$\\sigma$ ($^\\circ$C)\");"
+    "plt.ylabel(\"$\\sigma$ ($^\\circ$C)\")"
    ]
   },
   {
@@ -217,7 +217,7 @@
    "outputs": [],
    "source": [
     "ds1.st.plot(figsize=(12, 8))\n",
-    "ds1.ast.plot();"
+    "ds1.ast.plot()"
    ]
   },
   {

docs/notebooks/08Calibrate_double_ended.ipynb

@@ -157,7 +157,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "ds.sections = {\n",
+    "sections = {\n",
     "    \"probe1Temperature\": [slice(7.5, 17.0), slice(70.0, 80.0)],  # cold bath\n",
     "    \"probe2Temperature\": [slice(24.0, 34.0), slice(85.0, 95.0)],  # warm bath\n",
     "}"
@@ -186,10 +186,10 @@
    },
    "outputs": [],
    "source": [
-    "st_var, resid = ds.variance_stokes_constant(st_label=\"st\")\n",
-    "ast_var, _ = ds.variance_stokes_constant(st_label=\"ast\")\n",
-    "rst_var, _ = ds.variance_stokes_constant(st_label=\"rst\")\n",
-    "rast_var, _ = ds.variance_stokes_constant(st_label=\"rast\")"
+    "st_var, resid = ds.variance_stokes_constant(sections=sections, st_label=\"st\")\n",
+    "ast_var, _ = ds.variance_stokes_constant(sections=sections, st_label=\"ast\")\n",
+    "rst_var, _ = ds.variance_stokes_constant(sections=sections, st_label=\"rst\")\n",
+    "rast_var, _ = ds.variance_stokes_constant(sections=sections, st_label=\"rast\")"
    ]
   },
   {
@@ -212,7 +212,7 @@
    },
    "outputs": [],
    "source": [
-    "resid.plot(figsize=(12, 4));"
+    "resid.plot(figsize=(12, 4))"
    ]
   },
   {
@@ -237,6 +237,7 @@
    "outputs": [],
    "source": [
     "ds.calibration_double_ended(\n",
+    "    sections=sections,\n",
     "    st_var=st_var,\n",
     "    ast_var=ast_var,\n",
     "    rst_var=rst_var,\n",
@@ -257,7 +258,7 @@
    },
    "outputs": [],
    "source": [
-    "ds.tmpw.plot(figsize=(12, 4));"
+    "ds.tmpw.plot(figsize=(12, 4))"
    ]
   },
   {
@@ -295,7 +296,7 @@
     "ds.tmpw_var.plot(figsize=(12, 4))\n",
     "ds1 = ds.isel(time=-1)  # take only the first timestep\n",
     "(ds1.tmpw_var**0.5).plot(figsize=(12, 4))\n",
-    "plt.gca().set_ylabel(\"Standard error ($^\\circ$C)\");"
+    "plt.gca().set_ylabel(\"Standard error ($^\\circ$C)\")"
    ]
   },
   {
@@ -323,6 +324,7 @@
    "outputs": [],
    "source": [
     "ds.conf_int_double_ended(\n",
+    "    sections=sections,\n",
     "    st_var=st_var,\n",
     "    ast_var=ast_var,\n",
     "    rst_var=rst_var,\n",
@@ -331,7 +333,7 @@
     "    mc_sample_size=500,\n",
     ")  # < increase sample size for better approximation\n",
     "\n",
-    "ds.tmpw_mc_var.plot(figsize=(12, 4));"
+    "ds.tmpw_mc_var.plot(figsize=(12, 4))"
    ]
   },
   {
@@ -380,7 +382,7 @@
     "(ds1.tmpb_var**0.5).plot()\n",
     "(ds1.tmpw_var**0.5).plot()\n",
     "(ds1.tmpw_mc_var**0.5).plot()\n",
-    "plt.ylabel(\"$\\sigma$ ($^\\circ$C)\");"
+    "plt.ylabel(\"$\\sigma$ ($^\\circ$C)\")"
    ]
   },
   {

docs/notebooks/12Datastore_from_numpy_arrays.ipynb

@@ -197,13 +197,12 @@
     "    \"temp1\": [slice(20, 25.5)],  # warm bath\n",
     "    \"temp2\": [slice(5.5, 15.5)],  # cold bath\n",
     "}\n",
-    "ds.sections = sections\n",
     "\n",
-    "st_var, resid = ds.variance_stokes_constant(st_label=\"st\")\n",
-    "ast_var, _ = ds.variance_stokes_constant(st_label=\"ast\")\n",
-    "ds.calibration_single_ended(st_var=st_var, ast_var=ast_var)\n",
+    "st_var, resid = ds.variance_stokes_constant(sections=sections, st_label=\"st\")\n",
+    "ast_var, _ = ds.variance_stokes_constant(sections=sections, st_label=\"ast\")\n",
+    "ds.calibration_single_ended(sections=sections, st_var=st_var, ast_var=ast_var)\n",
     "\n",
-    "ds.isel(time=0).tmpf.plot();"
+    "ds.isel(time=0).tmpf.plot()"
    ]
   },
   {