Skip to content

Commit

Permalink
removed base class CalibrationCalculator and only have one Statistics…
Browse files Browse the repository at this point in the history
…Calculator class
  • Loading branch information
TjarkMiener committed Aug 27, 2024
1 parent 0ed515d commit 48c686b
Show file tree
Hide file tree
Showing 4 changed files with 37 additions and 58 deletions.
2 changes: 1 addition & 1 deletion docs/api-reference/monitoring/index.rst
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ Monitoring data are time-series used to monitor the status or quality of hardwar

This module provides some code to help to generate monitoring data from processed event data, particularly for the purposes of calibration and data quality assessment.

Currently, code related to :ref:`stats_aggregator`, :ref:`calibration_calculator`, and :ref:`outlier_detector` is implemented here.
Code related to :ref:`stats_aggregator`, :ref:`calibration_calculator`, and :ref:`outlier_detector` is implemented here.


Submodules
Expand Down
2 changes: 1 addition & 1 deletion src/ctapipe/monitoring/aggregator.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ def __call__(
and call the relevant function of the particular aggregator to compute aggregated statistic values.
The chunks are generated in a way that ensures they do not overflow the bounds of the table.
- If ``chunk_shift`` is None, chunks will not overlap, but the last chunk is ensured to be
of size `chunk_size`, even if it means the last two chunks will overlap.
of size ``chunk_size``, even if it means the last two chunks will overlap.
- If ``chunk_shift`` is provided, it will determine the number of samples to shift between the start
of consecutive chunks resulting in an overlap of chunks. Chunks that overflows the bounds
of the table are not considered.
Expand Down
86 changes: 33 additions & 53 deletions src/ctapipe/monitoring/calculator.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
"""
Definition of the ``CalibrationCalculator`` classes, providing all steps needed to
Definition of the ``StatisticsCalculator`` class, providing all steps needed to
calculate the montoring data for the camera calibration.
"""

Expand All @@ -19,26 +19,22 @@
from ctapipe.monitoring.outlier import OutlierDetector

__all__ = [
"CalibrationCalculator",
"StatisticsCalculator",
]


class CalibrationCalculator(TelescopeComponent):
class StatisticsCalculator(TelescopeComponent):
"""
Base component for calibration calculators.
This class provides the foundational methods and attributes for
calculating camera-related monitoring data. It is designed
to be extended by specific calibration calculators that implement
the required methods for different types of calibration.
Component to calculate statistics from calibration events.
Attributes
----------
stats_aggregator_type : ctapipe.core.traits.TelescopeParameter
The type of StatisticsAggregator to be used for aggregating statistics.
outlier_detector_list : list of dict
List of dictionaries containing the apply to, the name of the OutlierDetector subclass to be used, and the validity range of the detector.
The ``StatisticsCalculator`` is responsible for calculating various statistics from
calibration events, such as pedestal and flat-field data. It aggregates statistics,
detects outliers, and handles faulty data periods.
This class holds two functions to conduct two different passes over the data with and without
overlapping aggregation chunks. The first pass is conducted with non-overlapping chunks,
while overlapping chunks can be set by the ``chunk_shift`` parameter for the second pass.
The second pass over the data is only conducted in regions of trouble with a high percentage
of faulty pixels exceeding the threshold ``faulty_pixels_threshold``.
"""

stats_aggregator_type = TelescopeParameter(
Expand All @@ -56,7 +52,27 @@ class CalibrationCalculator(TelescopeComponent):
help=(
"List of dicts containing the name of the OutlierDetector subclass to be used, "
"the aggregated statistic value to which the detector should be applied, "
"and the validity range of the detector."
"and the validity range of the detector. "
"E.g. ``[{'apply_to': 'std', 'name': 'RangeOutlierDetector', 'validity_range': [2.0, 8.0]},]``."
),
).tag(config=True)

chunk_shift = Int(
default_value=None,
allow_none=True,
help=(
"Number of samples to shift the aggregation chunk for the calculation "
"of the statistical values. Only used in the second_pass(), since the "
"first_pass() is conducted with non-overlapping chunks (chunk_shift=None)."
),
).tag(config=True)

faulty_pixels_threshold = Float(
default_value=10.0,
allow_none=True,
help=(
"Threshold in percentage of faulty pixels over the camera "
"to identify regions of trouble."
),
).tag(config=True)

Expand Down Expand Up @@ -114,42 +130,6 @@ def __init__(
parent=self,
)


class StatisticsCalculator(CalibrationCalculator):
"""
Component to calculate statistics from calibration events.
This class inherits from ``CalibrationCalculator`` and is responsible for
calculating various statistics from calibration events, such as pedestal
and flat-field data. It aggregates statistics, detects outliers,
handles faulty data chunks.
The ``StatisticsCalculator`` holds two functions to conduct two different passes
over the data with and without overlapping chunks. The first pass is conducted
with non-overlapping, while overlapping chunks can be set by the ``chunk_shift``
parameter for the second pass. The second pass over the data is only conducted
in regions of trouble with a high percentage of faulty pixels exceeding
the threshold ``faulty_pixels_threshold``.
"""

chunk_shift = Int(
default_value=None,
allow_none=True,
help=(
"Number of samples to shift the aggregation chunk for the calculation "
"of the statistical values. Only used in the second_pass(), since the "
"first_pass() is conducted without overlapping chunks (chunk_shift=None)."
),
).tag(config=True)

faulty_pixels_threshold = Float(
default_value=10.0,
allow_none=True,
help=(
"Threshold in percentage of faulty pixels over the camera "
"to identify regions of trouble."
),
).tag(config=True)

def first_pass(
self,
table,
Expand Down Expand Up @@ -252,7 +232,7 @@ def second_pass(
faulty_chunks_indices = np.where(~valid_chunks)[0]
for index in faulty_chunks_indices:
# Log information of the faulty chunks
self.log.warning(
self.log.info(
f"Faulty chunk detected in the first pass at index '{index}'."
)
# Calculate the start of the slice depending on whether the previous chunk was faulty or not
Expand Down
5 changes: 2 additions & 3 deletions src/ctapipe/monitoring/tests/test_calculator.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from traitlets.config.loader import Config

from ctapipe.monitoring.aggregator import PlainAggregator
from ctapipe.monitoring.calculator import CalibrationCalculator, StatisticsCalculator
from ctapipe.monitoring.calculator import StatisticsCalculator


def test_statistics_calculator(example_subarray):
Expand All @@ -28,8 +28,7 @@ def test_statistics_calculator(example_subarray):
)
# Initialize the aggregator and calculator
aggregator = PlainAggregator(subarray=example_subarray, chunk_size=1000)
calculator = CalibrationCalculator.from_name(
name="StatisticsCalculator",
calculator = StatisticsCalculator(
subarray=example_subarray,
stats_aggregator=aggregator,
chunk_shift=100,
Expand Down

0 comments on commit 48c686b

Please sign in to comment.