Merge branch 'main' into default_config

# Conflicts: # .github/workflows/ci.yml # ctapipe/io/tests/test_simteleventsource.py # docs/conf.py # docs/developer-guide/index.rst # docs/examples/Tools.ipynb # docs/getting_started/index.rst # docs/index.rst # environment.yml # setup.cfg
cta-observatory · Nov 24, 2023 · 8ea1cc0 · 8ea1cc0
2 parents d163d07 + 4a45dcd
commit 8ea1cc0
Show file tree

Hide file tree

Showing 209 changed files with 6,447 additions and 8,346 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -24,6 +24,10 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: "3.9"
+      - name: Check README syntax
+        run: |
+          pip install restructuredtext-lint
+          restructuredtext-lint README.rst
       - uses: pre-commit/[email protected]
         with:
           extra_args: --files $(git diff origin/main --name-only)
@@ -75,25 +79,28 @@ jobs:
           key: ctapipe-test-data
 
       - name: Prepare mamba installation
-        if: matrix.install-method == 'mamba'
+        if: matrix.install-method == 'mamba' &&  contains(github.event.pull_request.labels.*.name, 'documentation-only') == false
         env:
           PYTHON_VERSION: ${{ matrix.python-version }}
         run: |
           # setup correct python version
           sed -i -e "s/- python=.*/- python=$PYTHON_VERSION/g" environment.yml
 
       - name: mamba setup
-        if: matrix.install-method == 'mamba'
-        uses: mamba-org/provision-with-micromamba@v14
+        if: matrix.install-method == 'mamba' && contains(github.event.pull_request.labels.*.name, 'documentation-only') == false
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-file: environment.yml
+          cache-downloads: true
 
       - name: Python setup
-        if: matrix.install-method == 'pip'
+        if: matrix.install-method == 'pip' && contains(github.event.pull_request.labels.*.name, 'documentation-only') == false 
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
           check-latest: true
 
-      - if: ${{ matrix.install-method == 'pip' && runner.os == 'macOS' }}
+      - if: matrix.install-method == 'pip' && runner.os == 'macOS' && contains(github.event.pull_request.labels.*.name, 'documentation-only') == false
         name: Fix Python PATH on macOS
         # See https://github.com/actions/setup-python/issues/132 and
         # https://github.com/actions/setup-python/issues/132#issuecomment-779406058
@@ -104,34 +111,37 @@ jobs:
           tee -a ~/.bash_profile <<<'export PATH="$pythonLocation/bin:$PATH"'
 
       - name: Install dependencies
+        if: contains(github.event.pull_request.labels.*.name, 'documentation-only') == false
+        env:
+          PYTHON_VERSION: ${{ matrix.python-version }}
         run: |
-          python --version
-          pip install pytest-cov restructuredtext-lint pytest-xdist 'coverage!=6.3.0'
-          pip install .[all]
+          python --version | grep "Python ${PYTHON_VERSION}"
+          pip install pytest-cov pytest-xdist 'coverage!=6.3.0'
+          pip install -e .[all]
           pip install ./test_plugin
           pip freeze
 
-      - name: Static codechecks
-        run: |
-          restructuredtext-lint README.rst
-
       - name: ctapipe-info
+        if: contains(github.event.pull_request.labels.*.name, 'documentation-only') == false
         run: |
           ctapipe-info --all
 
       - name: Tests
+        if: contains(github.event.pull_request.labels.*.name, 'documentation-only') == false
         run: |
           cd $(mktemp -d)
           pytest -n auto --dist loadscope \
-            --cov --cov-report=xml \
+            --cov --cov-report=xml:$GITHUB_WORKSPACE/coverage.xml \
             --doctest-modules --doctest-glob='*.rst' \
-            --ignore=$HOME/work/ctapipe/ctapipe/docs/conf.py \
-            --pyargs ctapipe $HOME/work/ctapipe/ctapipe/docs
+            --ignore=$GITHUB_WORKSPACE/docs/conf.py \
+            --pyargs ctapipe $GITHUB_WORKSPACE/docs
 
           ctapipe-info --version
 
       - uses: codecov/codecov-action@v3
-        if: contains(matrix.extra-args, 'codecov')
+        if: contains(matrix.extra-args, 'codecov') && contains(github.event.pull_request.labels.*.name, 'documentation-only') == false
+        with:
+          fail_ci_if_error: true
 
   docs:
     runs-on: ubuntu-latest

diff --git a/.gitignore b/.gitignore
@@ -12,6 +12,7 @@ __pycache__
 ctapipe/_version_cache.py
 ctapipe/_version.py
 
+
 # Ignore .c files by default to avoid including generated code. If you want to
 # add a non-generated .c extension, use `git add -f filename.c`.
 *.c
@@ -25,6 +26,7 @@ MANIFEST
 # Sphinx
 docs/api
 docs/_build
+docs/auto_examples
 
 # Editors and IDEs
 
@@ -86,7 +88,4 @@ distribute-*.tar.gz
 target
 .mypy_cache
 
-examples/notebooks/*.html
-examples/notebooks/*.png
-
 provenance.log
diff --git a/CHANGES.rst b/CHANGES.rst
@@ -1,3 +1,65 @@
+ctapipe v0.20.0 (2023-09-11)
+============================
+
+
+API Changes
+-----------
+
+- The ``ctapipe-dump-triggers`` tool was removed, since it wrote a custom data format
+  not compatble with e.g. the output of the ``DataWriter`` and ``ctapipe-process``.
+  If you only want to store trigger and simulation information from simulated / DL0
+  input files into the ctapipe format HDF5 files, you can now use
+  ``ctapipe-process -i <input> -o <output> --no-write-parameters``. [`#2375 <https://github.com/cta-observatory/ctapipe/pull/2375>`__]
+
+- Change the fill value for invalid telescope ids in ``SubarrayDescription.tel_index_array``
+  from ``-1`` to ``np.iinfo(int).minval`` to prevent ``-1`` being used as an index resulting in the last element being used for invalid telescope ids. [`#2376 <https://github.com/cta-observatory/ctapipe/pull/2376>`__]
+
+- Remove ``EventSource.from_config``, simply use ``EventSource(config=config)`` or 
+  ``EventSource(parent=parent)``. [`#2384 <https://github.com/cta-observatory/ctapipe/pull/2384>`__]
+
+
+Data Model Changes
+------------------
+
+- Added missing fields defined in the CTAO R1 and DL0 data models to the corresponding containers. [`#2338 <https://github.com/cta-observatory/ctapipe/pull/2338>`__]
+
+- Remove the ``injection_height`` field from the ``SimulationConfigContainer``,
+  this field was always empty and is never filled by ``sim_telarray``.
+
+  Add the corresponding ``starting_grammage`` field to the ``SimulatedShowerContainer``,
+  where it is actually available. [`#2343 <https://github.com/cta-observatory/ctapipe/pull/2343>`__]
+
+- Added new fields to the ``MuonEfficiencyContainer`` - ``is_valid`` to check if fit converged successfully, ``parameters_at_limit`` to check if parameters were fitted close to a bound and ``likelihood_value`` which represents cost function value atthe minimum. These fields were added to the output of the ``MuonIntensityFitter``. [`#2381 <https://github.com/cta-observatory/ctapipe/pull/2381>`__]
+
+
+New Features
+------------
+
+- Remove writing the full provenance information to the log  and instead simply refer the reader to the actual provenance file. [`#2328 <https://github.com/cta-observatory/ctapipe/pull/2328>`__]
+
+- Add support for including r1 and r0 waveforms in the ``ctapipe-merge`` tool. [`#2386 <https://github.com/cta-observatory/ctapipe/pull/2386>`__]
+
+
+Bug Fixes
+---------
+
+- The ```HillasIntersection``` method used to fail when individual events were reconstructed to originate from a FoV offset of more than 90 degrees.
+  This is now fixed by returning an INVALID container for a reconstructed offset of larger than 45 degrees. [`#2265 <https://github.com/cta-observatory/ctapipe/pull/2265>`__]
+
+- Fixed a bug in the calculation of the full numeric pixel likelihood and the corresponding tests. [`#2388 <https://github.com/cta-observatory/ctapipe/pull/2388>`__]
+
+
+Maintenance
+-----------
+
+- Drop support for python 3.8 in accordance with the NEP 29 schedule. [`#2342 <https://github.com/cta-observatory/ctapipe/pull/2342>`__]
+
+- * Switched to ``PyData`` theme for docs
+  * Updated ``Sphinx`` to version 6.2.1
+  * Updated front page of docs [`#2373 <https://github.com/cta-observatory/ctapipe/pull/2373>`__]
+
+
+
 ctapipe 0.19.3 (2023-06-20)
 ===========================
 

diff --git a/README.rst b/README.rst
@@ -42,16 +42,17 @@ cite using this bibtex entry:
 
 .. code::
 
-  @inproceedings{ctapipe-icrc-2021,
-      author = {Nöthe, Maximilian  and  Kosack, Karl  and  Nickel, Lukas  and  Peresano, Michele},
-      title = {Prototype Open Event Reconstruction Pipeline for the Cherenkov Telescope Array},
-      doi = {10.22323/1.395.0744},
-      booktitle = {Proceedings, 37th International Cosmic Ray Conference},
-      year=2021,
-      volume={395},
-      number={744},
-      location={Berlin, Germany},
-    }
+   @inproceedings{ctapipe-icrc-2023,
+     author = {Linhoff, Maximilian and Beiske, Lukas and Biederbeck, Noah and Fröse, Stefan and Kosack, Karl and Nickel, Lukas},
+     title = {ctapipe -- Prototype Open Event Reconstruction Pipeline for the Cherenkov Telescope Array},
+     usera = {for the CTA Consortium and Observatory},
+     doi = {10.22323/1.444.0703},
+     booktitle = {Proceedings, 38th International Cosmic Ray Conference},
+     year=2023,
+     volume={444},
+     number={703},
+     location={Nagoya, Japan},
+   }
 
 
 Installation for Users
@@ -71,11 +72,11 @@ or via::
 
   pip install ctapipe
 
-**Note**: to install a specific version of ctapipe take look at the documentation `here <https://ctapipe.readthedocs.org/en/latest/getting_started_users/>`__.
+**Note**: to install a specific version of ctapipe take look at the documentation `here <https://ctapipe.readthedocs.io/en/latest/user-guide/index.html>`__.
 
 **Note**: ``mamba`` is a C++ reimplementation of conda and can be found `here <https://github.com/mamba-org/mamba>`__.
 
 Note this is *pre-alpha* software and is not yet stable enough for end-users (expect large API changes until the first stable 1.0 release).
 
 Developers should follow the development install instructions found in the
-`documentation <https://ctapipe.readthedocs.org/en/latest/getting_started/>`__.
+`documentation <https://ctapipe.readthedocs.io/en/latest/developer-guide/getting-started.html>`__.
diff --git a/ctapipe/calib/__init__.py b/ctapipe/calib/__init__.py
@@ -1,7 +1,10 @@
 # Licensed under a 3-clause BSD style license - see LICENSE.rst
 """
-Calibration
+Module for calibration code
 """
 from .camera import CameraCalibrator, GainSelector
 
-__all__ = ["CameraCalibrator", "GainSelector"]
+__all__ = [
+    "CameraCalibrator",
+    "GainSelector",
+]
diff --git a/ctapipe/calib/camera/calibrator.py b/ctapipe/calib/camera/calibrator.py
@@ -3,13 +3,11 @@
 calibration and image extraction, as well as supporting algorithms.
 """
 
-import warnings
-
 import astropy.units as u
 import numpy as np
 from numba import float32, float64, guvectorize, int64
 
-from ctapipe.containers import DL1CameraContainer
+from ctapipe.containers import DL0CameraContainer, DL1CameraContainer, PixelStatus
 from ctapipe.core import TelescopeComponent
 from ctapipe.core.traits import (
     BoolTelescopeParameter,
@@ -129,7 +127,7 @@ def __init__(
         self.image_extractors = {}
 
         if image_extractor is None:
-            for (_, _, name) in self.image_extractor_type:
+            for _, _, name in self.image_extractor_type:
                 self.image_extractors[name] = ImageExtractor.from_name(
                     name, subarray=self.subarray, parent=self
                 )
@@ -156,7 +154,7 @@ def __init__(
     def _check_r1_empty(self, waveforms):
         if waveforms is None:
             if not self._r1_empty_warn:
-                warnings.warn(
+                self.log.debug(
                     "Encountered an event with no R1 data. "
                     "DL0 is unchanged in this circumstance."
                 )
@@ -168,7 +166,7 @@ def _check_r1_empty(self, waveforms):
     def _check_dl0_empty(self, waveforms):
         if waveforms is None:
             if not self._dl0_empty_warn:
-                warnings.warn(
+                self.log.warning(
                     "Encountered an event with no DL0 data. "
                     "DL1 is unchanged in this circumstance."
                 )
@@ -178,19 +176,35 @@ def _check_dl0_empty(self, waveforms):
             return False
 
     def _calibrate_dl0(self, event, tel_id):
-        waveforms = event.r1.tel[tel_id].waveform
-        selected_gain_channel = event.r1.tel[tel_id].selected_gain_channel
-        if self._check_r1_empty(waveforms):
+        r1 = event.r1.tel[tel_id]
+
+        if self._check_r1_empty(r1.waveform):
             return
 
-        reduced_waveforms_mask = self.data_volume_reducer(
-            waveforms, tel_id=tel_id, selected_gain_channel=selected_gain_channel
+        signal_pixels = self.data_volume_reducer(
+            r1.waveform,
+            tel_id=tel_id,
+            selected_gain_channel=r1.selected_gain_channel,
         )
 
-        waveforms_copy = waveforms.copy()
-        waveforms_copy[~reduced_waveforms_mask] = 0
-        event.dl0.tel[tel_id].waveform = waveforms_copy
-        event.dl0.tel[tel_id].selected_gain_channel = selected_gain_channel
+        dl0_waveform = r1.waveform.copy()
+        dl0_waveform[~signal_pixels] = 0
+
+        dl0_pixel_status = r1.pixel_status.copy()
+        # set dvr pixel bit in pixel_status for pixels kept by DVR
+        dl0_pixel_status[signal_pixels] |= PixelStatus.DVR_STORED_AS_SIGNAL
+        # unset dvr bits for removed pixels
+        dl0_pixel_status[~signal_pixels] &= ~np.uint8(PixelStatus.DVR_STATUS)
+
+        event.dl0.tel[tel_id] = DL0CameraContainer(
+            event_type=r1.event_type,
+            event_time=r1.event_time,
+            waveform=dl0_waveform,
+            selected_gain_channel=r1.selected_gain_channel,
+            pixel_status=dl0_pixel_status,
+            first_cell_id=r1.first_cell_id,
+            calibration_monitoring_id=r1.calibration_monitoring_id,
+        )
 
     def _calibrate_dl1(self, event, tel_id):
         waveforms = event.dl0.tel[tel_id].waveform
@@ -229,7 +243,6 @@ def _calibrate_dl1(self, event, tel_id):
                 is_valid=True,
             )
         else:
-
             # shift waveforms if time_shift calibration is available
             if time_shift is not None:
                 if self.apply_waveform_time_shift.tel[tel_id]:

diff --git a/ctapipe/calib/camera/tests/test_calibrator.py b/ctapipe/calib/camera/tests/test_calibrator.py
@@ -5,7 +5,6 @@
 
 import astropy.units as u
 import numpy as np
-import pytest
 from scipy.stats import norm
 from traitlets.config import Config
 
@@ -101,11 +100,6 @@ def test_check_r1_empty(example_event, example_subarray):
     calibrator = CameraCalibrator(subarray=example_subarray)
     tel_id = list(example_event.r0.tel)[0]
     waveform = example_event.r1.tel[tel_id].waveform.copy()
-    with pytest.warns(UserWarning):
-        example_event.r1.tel[tel_id].waveform = None
-        calibrator._calibrate_dl0(example_event, tel_id)
-        assert example_event.dl0.tel[tel_id].waveform is None
-
     assert calibrator._check_r1_empty(None) is True
     assert calibrator._check_r1_empty(waveform) is False
 
@@ -115,8 +109,7 @@ def test_check_r1_empty(example_event, example_subarray):
     )
     event = ArrayEventContainer()
     event.dl0.tel[tel_id].waveform = np.full((2048, 128), 2)
-    with pytest.warns(UserWarning):
-        calibrator(event)
+    calibrator(event)
     assert (event.dl0.tel[tel_id].waveform == 2).all()
     assert (event.dl1.tel[tel_id].image == 2 * 128).all()
 
@@ -126,19 +119,13 @@ def test_check_dl0_empty(example_event, example_subarray):
     tel_id = list(example_event.r0.tel)[0]
     calibrator._calibrate_dl0(example_event, tel_id)
     waveform = example_event.dl0.tel[tel_id].waveform.copy()
-    with pytest.warns(UserWarning):
-        example_event.dl0.tel[tel_id].waveform = None
-        calibrator._calibrate_dl1(example_event, tel_id)
-        assert example_event.dl1.tel[tel_id].image is None
-
     assert calibrator._check_dl0_empty(None) is True
     assert calibrator._check_dl0_empty(waveform) is False
 
     calibrator = CameraCalibrator(subarray=example_subarray)
     event = ArrayEventContainer()
     event.dl1.tel[tel_id].image = np.full(2048, 2)
-    with pytest.warns(UserWarning):
-        calibrator(event)
+    calibrator(event)
     assert (event.dl1.tel[tel_id].image == 2).all()
 
 
@@ -267,7 +254,6 @@ def test_shift_waveforms():
 
 
 def test_invalid_pixels(example_event, example_subarray):
-
     # switching off the corrections makes it easier to test for
     # the exact value of 1.0
     config = Config(