From c522715d4b27e55857c7dc2cd37af24d7dac3b98 Mon Sep 17 00:00:00 2001 From: Maximilian Linhoff Date: Fri, 7 Jul 2023 12:22:18 +0200 Subject: [PATCH] Start adapting code-base to new container structure --- ctapipe/calib/camera/calibrator.py | 4 +- ctapipe/calib/camera/flatfield.py | 8 +- ctapipe/calib/camera/pedestals.py | 8 +- ctapipe/calib/camera/tests/test_calibrator.py | 28 +- ctapipe/calib/camera/tests/test_flatfield.py | 4 +- ctapipe/calib/camera/tests/test_pedestals.py | 4 +- ctapipe/containers.py | 134 +++---- ctapipe/image/extractor.py | 42 +-- ctapipe/image/image_processor.py | 85 +++-- ctapipe/image/muon/processor.py | 31 +- ctapipe/image/reducer.py | 4 +- .../tests/test_sliding_window_correction.py | 4 +- ctapipe/instrument/tests/test_trigger.py | 14 +- ctapipe/instrument/trigger.py | 18 +- ctapipe/io/datawriter.py | 354 +++++++++--------- ctapipe/io/eventsource.py | 4 +- ctapipe/io/hdf5eventsource.py | 201 +++++----- ctapipe/io/simteleventsource.py | 38 +- ctapipe/io/tableloader.py | 4 +- ctapipe/io/tests/test_datawriter.py | 27 +- ctapipe/io/tests/test_event_source.py | 4 +- ctapipe/io/tests/test_hdf5.py | 8 +- ctapipe/io/tests/test_hdf5eventsource.py | 71 ++-- ctapipe/io/tests/test_simteleventsource.py | 10 +- ctapipe/io/tests/test_table_loader.py | 8 +- ctapipe/io/toymodel.py | 8 +- ctapipe/reco/hillas_reconstructor.py | 15 +- ctapipe/reco/preprocessing.py | 4 +- ctapipe/reco/reconstructor.py | 21 +- ctapipe/reco/shower_processor.py | 4 +- ctapipe/reco/sklearn.py | 10 +- ctapipe/reco/stereo_combination.py | 6 +- ctapipe/reco/tests/test_stereo_combination.py | 12 +- ctapipe/utils/tests/test_event_filter.py | 8 +- ctapipe/visualization/tests/test_mpl.py | 4 +- test_plugin/ctapipe_test_plugin/__init__.py | 6 +- 36 files changed, 611 insertions(+), 604 deletions(-) diff --git a/ctapipe/calib/camera/calibrator.py b/ctapipe/calib/camera/calibrator.py index 546be83814d..5383e0d81ce 100644 --- a/ctapipe/calib/camera/calibrator.py +++ b/ctapipe/calib/camera/calibrator.py @@ -9,7 +9,7 @@ import numpy as np from numba import float32, float64, guvectorize, int64 -from ctapipe.containers import TelescopeDL1Container, TelescopeEventContainer +from ctapipe.containers import DL1TelescopeContainer, TelescopeEventContainer from ctapipe.core import TelescopeComponent from ctapipe.core.traits import ( BoolTelescopeParameter, @@ -229,7 +229,7 @@ def dl0_to_dl1(self, tel_event: TelescopeEventContainer): # - Read into dl1 container directly? # - Don't do anything if dl1 container already filled # - Update on SST review decision - dl1 = TelescopeDL1Container( + dl1 = DL1TelescopeContainer( image=waveforms[..., 0].astype(np.float32), peak_time=np.zeros(n_pixels, dtype=np.float32), is_valid=True, diff --git a/ctapipe/calib/camera/flatfield.py b/ctapipe/calib/camera/flatfield.py index 21497781550..d70c6304c54 100644 --- a/ctapipe/calib/camera/flatfield.py +++ b/ctapipe/calib/camera/flatfield.py @@ -7,7 +7,7 @@ import numpy as np from astropy import units as u -from ctapipe.containers import TelescopeDL1Container +from ctapipe.containers import DL1TelescopeContainer from ctapipe.core import Component from ctapipe.core.traits import Int, List, Unicode from ctapipe.image.extractor import ImageExtractor @@ -169,7 +169,7 @@ def __init__(self, **kwargs): self.arrival_times = None # arrival time per event in sample self.sample_masked_pixels = None # masked pixels per event in sample - def _extract_charge(self, event) -> TelescopeDL1Container: + def _extract_charge(self, event) -> DL1TelescopeContainer: """ Extract the charge and the time from a calibration event @@ -195,7 +195,7 @@ def _extract_charge(self, event) -> TelescopeDL1Container: waveforms, self.tel_id, selected_gain_channel, broken_pixels ) else: - return TelescopeDL1Container(image=0, peak_pos=0, is_valid=False) + return DL1TelescopeContainer(image=0, peak_pos=0, is_valid=False) def calculate_relative_gain(self, event): """ @@ -237,7 +237,7 @@ def calculate_relative_gain(self, event): # extract the charge of the event and # the peak position (assumed as time for the moment) - dl1: TelescopeDL1Container = self._extract_charge(event) + dl1: DL1TelescopeContainer = self._extract_charge(event) if not dl1.is_valid: return False diff --git a/ctapipe/calib/camera/pedestals.py b/ctapipe/calib/camera/pedestals.py index a815b9ea3e5..b6511f5028e 100644 --- a/ctapipe/calib/camera/pedestals.py +++ b/ctapipe/calib/camera/pedestals.py @@ -7,7 +7,7 @@ import numpy as np from astropy import units as u -from ctapipe.containers import TelescopeDL1Container +from ctapipe.containers import DL1TelescopeContainer from ctapipe.core import Component from ctapipe.core.traits import Int, List, Unicode from ctapipe.image.extractor import ImageExtractor @@ -197,7 +197,7 @@ def __init__(self, **kwargs): self.charges = None # charge per event in sample self.sample_masked_pixels = None # pixels tp be masked per event in sample - def _extract_charge(self, event) -> TelescopeDL1Container: + def _extract_charge(self, event) -> DL1TelescopeContainer: """ Extract the charge and the time from a pedestal event @@ -224,7 +224,7 @@ def _extract_charge(self, event) -> TelescopeDL1Container: waveforms, self.tel_id, selected_gain_channel, broken_pixels ) else: - return TelescopeDL1Container(image=0, peak_pos=0, is_valid=False) + return DL1TelescopeContainer(image=0, peak_pos=0, is_valid=False) def calculate_pedestals(self, event): """ @@ -258,7 +258,7 @@ def calculate_pedestals(self, event): # extract the charge of the event and # the peak position (assumed as time for the moment) - dl1: TelescopeDL1Container = self._extract_charge(event) + dl1: DL1TelescopeContainer = self._extract_charge(event) if not dl1.is_valid: return False diff --git a/ctapipe/calib/camera/tests/test_calibrator.py b/ctapipe/calib/camera/tests/test_calibrator.py index fc75faa0ee6..dcaf10e9cd2 100644 --- a/ctapipe/calib/camera/tests/test_calibrator.py +++ b/ctapipe/calib/camera/tests/test_calibrator.py @@ -11,12 +11,12 @@ from ctapipe.calib.camera.calibrator import CameraCalibrator from ctapipe.containers import ( - ArrayEventContainer, - TelescopeDL0Container, - TelescopeDL1Container, + DL0TelescopeContainer, + DL1TelescopeContainer, + R1TelescopeContainer, + SubarrayEventContainer, TelescopeEventContainer, TelescopeEventIndexContainer, - TelescopeR1Container, ) from ctapipe.image.extractor import ( FullWaveformSum, @@ -114,17 +114,17 @@ def test_check_r1_empty(example_event, example_subarray): assert tel_event.dl0.waveform is None assert calibrator._check_r1_empty(None) is True - assert calibrator._check_r1_empty(TelescopeR1Container(waveform=None)) is True - assert calibrator._check_r1_empty(TelescopeR1Container(waveform=waveform)) is False + assert calibrator._check_r1_empty(R1TelescopeContainer(waveform=None)) is True + assert calibrator._check_r1_empty(R1TelescopeContainer(waveform=waveform)) is False calibrator = CameraCalibrator( subarray=example_subarray, image_extractor=FullWaveformSum(subarray=example_subarray), ) - event = ArrayEventContainer() + event = SubarrayEventContainer() event.tel[tel_id] = TelescopeEventContainer( index=TelescopeEventIndexContainer(obs_id=1, event_id=1, tel_id=tel_id), - dl0=TelescopeDL0Container(waveform=np.full((2048, 128), 2)), + dl0=DL0TelescopeContainer(waveform=np.full((2048, 128), 2)), ) with pytest.warns(UserWarning): calibrator(event) @@ -145,16 +145,16 @@ def test_check_dl0_empty(example_event, example_subarray): assert tel_event.dl1.image is None assert calibrator._check_dl0_empty(None) is True - assert calibrator._check_dl0_empty(TelescopeDL0Container(waveform=None)) is True + assert calibrator._check_dl0_empty(DL0TelescopeContainer(waveform=None)) is True assert ( - calibrator._check_dl0_empty(TelescopeDL0Container(waveform=waveform)) is False + calibrator._check_dl0_empty(DL0TelescopeContainer(waveform=waveform)) is False ) calibrator = CameraCalibrator(subarray=example_subarray) - event = ArrayEventContainer() + event = SubarrayEventContainer() tel_event = TelescopeEventContainer( index=TelescopeEventIndexContainer(obs_id=1, event_id=1, tel_id=tel_id), - dl1=TelescopeDL1Container(image=np.full(2048, 2)), + dl1=DL1TelescopeContainer(image=np.full(2048, 2)), ) event.tel[tel_id] = tel_event with pytest.warns(UserWarning): @@ -198,10 +198,10 @@ def test_dl1_charge_calib(example_subarray): pedestal = rng.uniform(-4, 4, n_pixels) y += pedestal[:, np.newaxis] - event = ArrayEventContainer() + event = SubarrayEventContainer() event.tel[tel_id] = TelescopeEventContainer( index=TelescopeEventIndexContainer(obs_id=1, event_id=1, tel_id=tel_id), - dl0=TelescopeDL0Container( + dl0=DL0TelescopeContainer( waveform=y, selected_gain_channel=np.zeros(len(y), dtype=int), ), diff --git a/ctapipe/calib/camera/tests/test_flatfield.py b/ctapipe/calib/camera/tests/test_flatfield.py index 632dce7b414..05b678bb172 100644 --- a/ctapipe/calib/camera/tests/test_flatfield.py +++ b/ctapipe/calib/camera/tests/test_flatfield.py @@ -6,7 +6,7 @@ from traitlets.config import Config from ctapipe.calib.camera.flatfield import FlasherFlatFieldCalculator -from ctapipe.containers import ArrayEventContainer +from ctapipe.containers import SubarrayEventContainer from ctapipe.instrument import SubarrayDescription @@ -37,7 +37,7 @@ def test_flasherflatfieldcalculator(prod5_sst): config=config, ) # create one event - data = ArrayEventContainer() + data = SubarrayEventContainer() data.meta["origin"] = "test" data.trigger.time = Time.now() diff --git a/ctapipe/calib/camera/tests/test_pedestals.py b/ctapipe/calib/camera/tests/test_pedestals.py index fe47ca74a84..52a689a407f 100644 --- a/ctapipe/calib/camera/tests/test_pedestals.py +++ b/ctapipe/calib/camera/tests/test_pedestals.py @@ -11,7 +11,7 @@ PedestalIntegrator, calc_pedestals_from_traces, ) -from ctapipe.containers import ArrayEventContainer +from ctapipe.containers import SubarrayEventContainer from ctapipe.instrument import SubarrayDescription @@ -39,7 +39,7 @@ def test_pedestal_integrator(prod5_sst): tel_id=tel_id, ) # create one event - data = ArrayEventContainer() + data = SubarrayEventContainer() data.meta["origin"] = "test" data.trigger.time = Time.now() diff --git a/ctapipe/containers.py b/ctapipe/containers.py index d8e3a72d5db..493ee62b498 100644 --- a/ctapipe/containers.py +++ b/ctapipe/containers.py @@ -12,12 +12,12 @@ from .core import Container, Field, Map __all__ = [ - "ArrayEventContainer", + "SubarrayEventContainer", "ConcentrationContainer", - "TelescopeDL0Container", + "DL0TelescopeContainer", "DL1CameraCalibrationContainer", - "TelescopeDL1Container", - "ArrayDL2Container", + "DL1TelescopeContainer", + "DL2SubarrayContainer", "TelescopeCalibrationContainer", "ArrayEventIndexContainer", "EventType", @@ -34,19 +34,19 @@ "ParticleClassificationContainer", "PedestalContainer", "PixelStatusContainer", - "TelescopeR0Container", - "TelescopeR1Container", + "R0TelescopeContainer", + "R1TelescopeContainer", "ReconstructedEnergyContainer", "ReconstructedGeometryContainer", "DispContainer", - "TelescopeSimulationContainer", + "SimulationTelescopeContainer", "SimulatedShowerContainer", "SimulatedShowerDistribution", "SimulationConfigContainer", "TelescopeEventIndexContainer", "BaseTimingParametersContainer", "TimingParametersContainer", - "ArrayTriggerContainer", + "SubarrayTriggerContainer", "WaveformCalibrationContainer", "StatisticsContainer", "IntensityStatisticsContainer", @@ -202,6 +202,25 @@ class TelescopeEventIndexContainer(Container): tel_id = tel_id_field() +class TelescopeTriggerContainer(Container): + default_prefix = "" + time = Field(NAN_TIME, description="Telescope trigger time") + n_trigger_pixels = Field( + -1, description="Number of trigger groups (sectors) listed" + ) + trigger_pixels = Field(None, description="pixels involved in the camera trigger") + event_type = Field(TelescopeEventType.SUBARRAY, description="Event type") + + +class SubarrayTriggerContainer(Container): + default_prefix = "" + time = Field(NAN_TIME, description="central average time stamp") + tels_with_trigger = Field( + None, description="List of telescope ids that triggered the array event" + ) + event_type = Field(EventType.SUBARRAY, description="Event type") + + class BaseHillasParametersContainer(Container): """ Base container for hillas parameters to @@ -416,7 +435,7 @@ class ImageParametersContainer(Container): ) -class TelescopeDL1Container(Container): +class DL1TelescopeContainer(Container): """ Storage of output of camera calibration e.g the final calibrated image in intensity units and the pulse time. @@ -485,7 +504,7 @@ class DL1CameraCalibrationContainer(Container): ) -class TelescopeR0Container(Container): +class R0TelescopeContainer(Container): """ Storage of raw data from a single telescope """ @@ -495,7 +514,7 @@ class TelescopeR0Container(Container): ) -class TelescopeR1Container(Container): +class R1TelescopeContainer(Container): """ Storage of r1 calibrated data from a single telescope """ @@ -516,11 +535,16 @@ class TelescopeR1Container(Container): ) -class TelescopeDL0Container(Container): +class DL0TelescopeContainer(Container): """ - Storage of data volume reduced dl0 data from a single telescope + DL0 data from a single telescope """ + trigger = Field( + default_factory=TelescopeTriggerContainer, + description="telescope trigger information", + ) + waveform = Field( None, ( @@ -540,6 +564,17 @@ class TelescopeDL0Container(Container): ) +class DL0SubarrayContainer(Container): + """ + DL0 data at subarray level + """ + + trigger = Field( + default_factory=SubarrayTriggerContainer, + description="subarray trigger information", + ) + + class TelescopeImpactParameterContainer(Container): """ Impact Parameter computed from reconstructed shower geometry @@ -572,7 +607,7 @@ class SimulatedShowerContainer(Container): ) -class TelescopeSimulationContainer(Container): +class SimulationTelescopeContainer(Container): """ True images and parameters derived from them, analgous to the `DL1CameraContainer` but for simulated data. @@ -603,7 +638,7 @@ class TelescopeSimulationContainer(Container): ) -class ArraySimulationContainer(Container): +class SimulationSubarrayContainer(Container): shower = Field( default_factory=SimulatedShowerContainer, description="True event information", @@ -696,25 +731,6 @@ class SimulationConfigContainer(Container): ) -class TelescopeTriggerContainer(Container): - default_prefix = "" - time = Field(NAN_TIME, description="Telescope trigger time") - n_trigger_pixels = Field( - -1, description="Number of trigger groups (sectors) listed" - ) - trigger_pixels = Field(None, description="pixels involved in the camera trigger") - event_type = Field(TelescopeEventType.SUBARRAY, description="Event type") - - -class ArrayTriggerContainer(Container): - default_prefix = "" - time = Field(NAN_TIME, description="central average time stamp") - tels_with_trigger = Field( - None, description="List of telescope ids that triggered the array event" - ) - event_type = Field(EventType.SUBARRAY, description="Event type") - - class ReconstructedGeometryContainer(Container): """ Standard output of algorithms reconstructing shower geometry @@ -831,11 +847,12 @@ class DispContainer(Container): is_valid = Field(False, "true if the predictions are valid") -class ArrayDL2Container(Container): +class DL2SubarrayContainer(Container): """Reconstructed Shower information for a given reconstruction algorithm, including optionally both per-telescope mono reconstruction and per-shower stereo reconstructions """ + geometry = Field( default_factory=partial(Map, ReconstructedGeometryContainer), description="map of algorithm to reconstructed shower parameters", @@ -850,8 +867,9 @@ class ArrayDL2Container(Container): ) -class TelescopeDL2Container(ArrayDL2Container): +class DL2TelescopeContainer(DL2SubarrayContainer): """Telescope-wise reconstructed quantities""" + impact = Field( default_factory=partial(Map, TelescopeImpactParameterContainer), description="map of algorithm to impact parameter info", @@ -874,7 +892,7 @@ class TelescopePointingContainer(Container): altitude = Field(nan * u.rad, "Altitude", unit=u.rad) -class ArrayPointingContainer(Container): +class SubarrayPointingContainer(Container): azimuth = Field(nan * u.rad, "Array pointing azimuth", unit=u.rad) altitude = Field(nan * u.rad, "Array pointing altitude", unit=u.rad) ra = Field(nan * u.rad, "Array pointing right ascension", unit=u.rad) @@ -937,7 +955,7 @@ class MuonParametersContainer(Container): ) -class MuonTelescopeContainer(Container): +class MuonContainer(Container): """ Container for muon analysis """ @@ -951,15 +969,6 @@ class MuonTelescopeContainer(Container): ) -class MuonContainer(Container): - """Root container for muon parameters""" - - tel = Field( - default_factory=partial(Map, MuonTelescopeContainer), - description="map of tel_id to MuonTelescopeContainer", - ) - - class FlatFieldContainer(Container): """ Container for flat-field parameters obtained from a set of @@ -1163,26 +1172,22 @@ class TelescopeEventContainer(Container): simulation = Field( None, description="Simulated Event Information", - type=TelescopeSimulationContainer, + type=SimulationTelescopeContainer, ) - r0 = Field(default_factory=TelescopeR0Container, description="Raw Data") - r1 = Field(default_factory=TelescopeR1Container, description="R1 Calibrated Data") + r0 = Field(default_factory=R0TelescopeContainer, description="Raw Data") + r1 = Field(default_factory=R1TelescopeContainer, description="R1 Calibrated Data") dl0 = Field( - default_factory=TelescopeDL0Container, + default_factory=DL0TelescopeContainer, description="DL0 Data Volume Reduced Data", ) dl1 = Field( - default_factory=TelescopeDL1Container, description="DL1 images and parameters" + default_factory=DL1TelescopeContainer, description="DL1 images and parameters" ) dl2 = Field( - default_factory=TelescopeDL2Container, + default_factory=DL2TelescopeContainer, description="DL2 reconstructed event properties", ) - trigger = Field( - default_factory=TelescopeTriggerContainer, - description="central trigger information", - ) count = Field(0, description="number of events processed") pointing = Field( default_factory=TelescopePointingContainer, @@ -1201,7 +1206,7 @@ class TelescopeEventContainer(Container): ) -class ArrayEventContainer(Container): +class SubarrayEventContainer(Container): """Top-level container for all event information""" count = Field(0, description="number of events processed") @@ -1212,20 +1217,23 @@ class ArrayEventContainer(Container): ) simulation = Field( - None, description="Simulated Event Information", type=ArraySimulationContainer + None, + description="Simulated Event Information", + type=SimulationSubarrayContainer, ) - trigger = Field( - default_factory=ArrayTriggerContainer, description="central trigger information" + dl0 = Field( + default_factory=DL0SubarrayContainer, + description="DL0 subarray wide information", ) dl2 = Field( - default_factory=ArrayDL2Container, + default_factory=DL2SubarrayContainer, description="DL2 reconstructed event properties", ) pointing = Field( - default_factory=ArrayPointingContainer, + default_factory=SubarrayPointingContainer, description="Array and telescope pointing positions", ) diff --git a/ctapipe/image/extractor.py b/ctapipe/image/extractor.py index cd12d37bd22..21564d8a154 100644 --- a/ctapipe/image/extractor.py +++ b/ctapipe/image/extractor.py @@ -33,7 +33,7 @@ from scipy.ndimage import convolve1d from traitlets import Bool, Int -from ctapipe.containers import TelescopeDL1Container +from ctapipe.containers import DL1TelescopeContainer from ctapipe.core import TelescopeComponent from ctapipe.core.traits import ( BoolTelescopeParameter, @@ -387,7 +387,7 @@ def __init__(self, subarray, config=None, parent=None, **kwargs): @abstractmethod def __call__( self, waveforms, tel_id, selected_gain_channel, broken_pixels - ) -> TelescopeDL1Container: + ) -> DL1TelescopeContainer: """ Call the relevant functions to fully extract the charge and time for the particular extractor. @@ -407,7 +407,7 @@ def __call__( Returns ------- - DL1CameraContainer: + DL1TelescopeContainer: extracted images and validity flags """ @@ -419,11 +419,11 @@ class FullWaveformSum(ImageExtractor): def __call__( self, waveforms, tel_id, selected_gain_channel, broken_pixels - ) -> TelescopeDL1Container: + ) -> DL1TelescopeContainer: charge, peak_time = extract_around_peak( waveforms, 0, waveforms.shape[-1], 0, self.sampling_rate_ghz[tel_id] ) - return TelescopeDL1Container(image=charge, peak_time=peak_time, is_valid=True) + return DL1TelescopeContainer(image=charge, peak_time=peak_time, is_valid=True) class FixedWindowSum(ImageExtractor): @@ -478,7 +478,7 @@ def _calculate_correction(self, tel_id): def __call__( self, waveforms, tel_id, selected_gain_channel, broken_pixels - ) -> TelescopeDL1Container: + ) -> DL1TelescopeContainer: charge, peak_time = extract_around_peak( waveforms, self.peak_index.tel[tel_id], @@ -488,7 +488,7 @@ def __call__( ) if self.apply_integration_correction.tel[tel_id]: charge *= self._calculate_correction(tel_id=tel_id)[selected_gain_channel] - return TelescopeDL1Container(image=charge, peak_time=peak_time, is_valid=True) + return DL1TelescopeContainer(image=charge, peak_time=peak_time, is_valid=True) class GlobalPeakWindowSum(ImageExtractor): @@ -557,7 +557,7 @@ def _calculate_correction(self, tel_id): def __call__( self, waveforms, tel_id, selected_gain_channel, broken_pixels - ) -> TelescopeDL1Container: + ) -> DL1TelescopeContainer: if self.pixel_fraction.tel[tel_id] == 1.0: # average over pixels then argmax over samples peak_index = waveforms[~broken_pixels].mean(axis=-2).argmax() @@ -579,7 +579,7 @@ def __call__( ) if self.apply_integration_correction.tel[tel_id]: charge *= self._calculate_correction(tel_id=tel_id)[selected_gain_channel] - return TelescopeDL1Container(image=charge, peak_time=peak_time, is_valid=True) + return DL1TelescopeContainer(image=charge, peak_time=peak_time, is_valid=True) class LocalPeakWindowSum(ImageExtractor): @@ -633,7 +633,7 @@ def _calculate_correction(self, tel_id): def __call__( self, waveforms, tel_id, selected_gain_channel, broken_pixels - ) -> TelescopeDL1Container: + ) -> DL1TelescopeContainer: peak_index = waveforms.argmax(axis=-1).astype(np.int64) charge, peak_time = extract_around_peak( waveforms, @@ -644,7 +644,7 @@ def __call__( ) if self.apply_integration_correction.tel[tel_id]: charge *= self._calculate_correction(tel_id=tel_id)[selected_gain_channel] - return TelescopeDL1Container(image=charge, peak_time=peak_time, is_valid=True) + return DL1TelescopeContainer(image=charge, peak_time=peak_time, is_valid=True) class SlidingWindowMaxSum(ImageExtractor): @@ -714,13 +714,13 @@ def _calculate_correction(self, tel_id): def __call__( self, waveforms, tel_id, selected_gain_channel, broken_pixels - ) -> TelescopeDL1Container: + ) -> DL1TelescopeContainer: charge, peak_time = extract_sliding_window( waveforms, self.window_width.tel[tel_id], self.sampling_rate_ghz[tel_id] ) if self.apply_integration_correction.tel[tel_id]: charge *= self._calculate_correction(tel_id=tel_id)[selected_gain_channel] - return TelescopeDL1Container(image=charge, peak_time=peak_time, is_valid=True) + return DL1TelescopeContainer(image=charge, peak_time=peak_time, is_valid=True) class NeighborPeakWindowSum(ImageExtractor): @@ -780,7 +780,7 @@ def _calculate_correction(self, tel_id): def __call__( self, waveforms, tel_id, selected_gain_channel, broken_pixels - ) -> TelescopeDL1Container: + ) -> DL1TelescopeContainer: neighbors = self.subarray.tel[tel_id].camera.geometry.neighbor_matrix_sparse peak_index = neighbor_average_maximum( waveforms, @@ -798,7 +798,7 @@ def __call__( ) if self.apply_integration_correction.tel[tel_id]: charge *= self._calculate_correction(tel_id=tel_id)[selected_gain_channel] - return TelescopeDL1Container(image=charge, peak_time=peak_time, is_valid=True) + return DL1TelescopeContainer(image=charge, peak_time=peak_time, is_valid=True) class BaselineSubtractedNeighborPeakWindowSum(NeighborPeakWindowSum): @@ -814,7 +814,7 @@ class BaselineSubtractedNeighborPeakWindowSum(NeighborPeakWindowSum): def __call__( self, waveforms, tel_id, selected_gain_channel, broken_pixels - ) -> TelescopeDL1Container: + ) -> DL1TelescopeContainer: baseline_corrected = subtract_baseline( waveforms, self.baseline_start, self.baseline_end ) @@ -1263,12 +1263,12 @@ def _apply_second_pass( def __call__( self, waveforms, tel_id, selected_gain_channel, broken_pixels - ) -> TelescopeDL1Container: + ) -> DL1TelescopeContainer: charge1, pulse_time1, correction1 = self._apply_first_pass(waveforms, tel_id) # FIXME: properly make sure that output is 32Bit instead of downcasting here if self.disable_second_pass: - return TelescopeDL1Container( + return DL1TelescopeContainer( image=(charge1 * correction1[selected_gain_channel]).astype("float32"), peak_time=pulse_time1.astype("float32"), is_valid=True, @@ -1284,7 +1284,7 @@ def __call__( broken_pixels, ) # FIXME: properly make sure that output is 32Bit instead of downcasting here - return TelescopeDL1Container( + return DL1TelescopeContainer( image=charge2.astype("float32"), peak_time=pulse_time2.astype("float32"), is_valid=is_valid, @@ -1642,7 +1642,7 @@ def clip(x, lo=0.0, hi=np.inf): def __call__( self, waveforms, tel_id, selected_gain_channel, broken_pixels - ) -> DL1CameraContainer: + ) -> DL1TelescopeContainer: upsampling = self.upsampling.tel[tel_id] integration_window_width = self.window_width.tel[tel_id] integration_window_shift = self.window_shift.tel[tel_id] @@ -1695,4 +1695,4 @@ def __call__( if shift != 0: peak_time -= shift - return DL1CameraContainer(image=charge, peak_time=peak_time, is_valid=True) + return DL1TelescopeContainer(image=charge, peak_time=peak_time, is_valid=True) diff --git a/ctapipe/image/image_processor.py b/ctapipe/image/image_processor.py index 6fc5e4bd8e5..f2b37518911 100644 --- a/ctapipe/image/image_processor.py +++ b/ctapipe/image/image_processor.py @@ -8,12 +8,12 @@ from ctapipe.coordinates import TelescopeFrame from ..containers import ( - ArrayEventContainer, CameraHillasParametersContainer, CameraTimingParametersContainer, ImageParametersContainer, IntensityStatisticsContainer, PeakTimeStatisticsContainer, + SubarrayEventContainer, TimingParametersContainer, ) from ..core import QualityQuery, TelescopeComponent @@ -116,8 +116,9 @@ def __init__( for tel_id in self.subarray.tel } - def __call__(self, event: ArrayEventContainer): - self._process_telescope_event(event) + def __call__(self, event: SubarrayEventContainer): + for tel_event in event.tel.values(): + self._process_telescope_event(tel_event) def _parameterize_image( self, @@ -209,51 +210,49 @@ def _parameterize_image( # parameterization return default - def _process_telescope_event(self, event): + def _process_telescope_event(self, tel_event): """ Loop over telescopes and process the calibrated images into parameters """ - for tel_id, dl1_camera in event.dl1.tel.items(): + tel_id = tel_event.index.tel_id + dl1 = tel_event.dl1 - if self.apply_image_modifier.tel[tel_id]: - dl1_camera.image = self.modify(tel_id=tel_id, image=dl1_camera.image) + if self.apply_image_modifier.tel[tel_id]: + dl1.image = self.modify(tel_id=tel_id, image=dl1.image) - dl1_camera.image_mask = self.clean( - tel_id=tel_id, - image=dl1_camera.image, - arrival_times=dl1_camera.peak_time, - ) + dl1.image_mask = self.clean( + tel_id=tel_id, + image=dl1.image, + arrival_times=dl1.peak_time, + ) + + dl1.parameters = self._parameterize_image( + tel_id=tel_id, + image=dl1.image, + signal_pixels=dl1.image_mask, + peak_time=dl1.peak_time, + default=self.default_image_container, + ) - dl1_camera.parameters = self._parameterize_image( - tel_id=tel_id, - image=dl1_camera.image, - signal_pixels=dl1_camera.image_mask, - peak_time=dl1_camera.peak_time, - default=self.default_image_container, + self.log.debug("params: %s", dl1.parameters.as_dict(recursive=True)) + + if ( + tel_event.simulation is not None + and tel_event.simulation.true_image is not None + ): + simulation = tel_event.simulation + simulation.true_parameters = self._parameterize_image( + tel_id, + image=simulation.true_image, + signal_pixels=simulation.true_image > 0, + peak_time=None, # true image from simulation has no peak time + default=DEFAULT_TRUE_IMAGE_PARAMETERS, ) + for container in simulation.true_parameters.values(): + if not container.prefix.startswith("true_"): + container.prefix = f"true_{container.prefix}" - self.log.debug("params: %s", dl1_camera.parameters.as_dict(recursive=True)) - - if ( - event.simulation is not None - and tel_id in event.simulation.tel - and event.simulation.tel[tel_id].true_image is not None - ): - sim_camera = event.simulation.tel[tel_id] - sim_camera.true_parameters = self._parameterize_image( - tel_id, - image=sim_camera.true_image, - signal_pixels=sim_camera.true_image > 0, - peak_time=None, # true image from simulation has no peak time - default=DEFAULT_TRUE_IMAGE_PARAMETERS, - ) - for container in sim_camera.true_parameters.values(): - if not container.prefix.startswith("true_"): - container.prefix = f"true_{container.prefix}" - - self.log.debug( - "sim params: %s", - event.simulation.tel[tel_id].true_parameters.as_dict( - recursive=True - ), - ) + self.log.debug( + "true image parameters: %s", + tel_event.simulation.true_parameters.as_dict(recursive=True), + ) diff --git a/ctapipe/image/muon/processor.py b/ctapipe/image/muon/processor.py index 30b382aef2c..1fac1270f6f 100644 --- a/ctapipe/image/muon/processor.py +++ b/ctapipe/image/muon/processor.py @@ -4,9 +4,9 @@ import numpy as np from ctapipe.containers import ( - ArrayEventContainer, + MuonContainer, MuonParametersContainer, - MuonTelescopeContainer, + SubarrayEventContainer, ) from ctapipe.coordinates import TelescopeFrame from ctapipe.core import QualityQuery, TelescopeComponent @@ -21,7 +21,7 @@ from .intensity_fitter import MuonIntensityFitter from .ring_fitter import MuonRingFitter -INVALID = MuonTelescopeContainer() +INVALID = MuonContainer() INVALID_PARAMETERS = MuonParametersContainer() __all__ = ["MuonProcessor"] @@ -114,11 +114,11 @@ def __init__(self, subarray, **kwargs): self.intensity_fitter = MuonIntensityFitter(subarray=subarray, parent=self) - def __call__(self, event: ArrayEventContainer): - for tel_id in event.dl1.tel: - self._process_telescope_event(event, tel_id) + def __call__(self, event: SubarrayEventContainer): + for tel_event in event.tel.values(): + self._process_telescope_event(tel_event) - def _process_telescope_event(self, event, tel_id): + def _process_telescope_event(self, tel_event): """ Extract and process a ring from a single image. @@ -129,8 +129,9 @@ def _process_telescope_event(self, event, tel_id): tel_id: int Telescope ID of the instrument that has measured the image """ - event_index = event.index - event_id = event_index.event_id + index = tel_event.index + tel_id = index.tel_id + event_id = index.event_id if self.subarray.tel[tel_id].optics.n_mirrors != 1: self.log.warning( @@ -139,11 +140,11 @@ def _process_telescope_event(self, event, tel_id): " not supported. Exclude dual mirror telescopes via setting" " 'EventSource.allowed_tels'." ) - event.muon.tel[tel_id] = INVALID + tel_event.muon = INVALID return self.log.debug(f"Processing event {event_id}, telescope {tel_id}") - dl1 = event.dl1.tel[tel_id] + dl1 = tel_event.dl1 image = dl1.image mask = dl1.image_mask if mask is None: @@ -152,7 +153,7 @@ def _process_telescope_event(self, event, tel_id): checks = self.dl1_query(dl1_params=dl1.parameters) if not all(checks): - event.muon.tel[tel_id] = INVALID + tel_event.muon = INVALID return geometry = self.geometries[tel_id] @@ -176,9 +177,7 @@ def _process_telescope_event(self, event, tel_id): checks = self.ring_query(parameters=parameters, ring=ring, mask=mask) if not all(checks): - event.muon.tel[tel_id] = MuonTelescopeContainer( - parameters=parameters, ring=ring - ) + tel_event.muon = MuonContainer(parameters=parameters, ring=ring) return efficiency = self.intensity_fitter( @@ -197,7 +196,7 @@ def _process_telescope_event(self, event, tel_id): f", efficiency={efficiency.optical_efficiency:.2%}" ) - event.muon.tel[tel_id] = MuonTelescopeContainer( + tel_event.muon = MuonContainer( ring=ring, efficiency=efficiency, parameters=parameters ) diff --git a/ctapipe/image/reducer.py b/ctapipe/image/reducer.py index 530a68c8c4c..c1dcc3c6c7c 100644 --- a/ctapipe/image/reducer.py +++ b/ctapipe/image/reducer.py @@ -5,7 +5,7 @@ import numpy as np -from ctapipe.containers import TelescopeDL1Container +from ctapipe.containers import DL1TelescopeContainer from ctapipe.core import TelescopeComponent from ctapipe.core.traits import ( BoolTelescopeParameter, @@ -183,7 +183,7 @@ def select_pixels(self, waveforms, tel_id=None, selected_gain_channel=None): extractor = self.image_extractors[self.image_extractor_type.tel[tel_id]] # do not treat broken pixels in data volume reduction broken_pixels = np.zeros(camera_geom.n_pixels, dtype=bool) - dl1: TelescopeDL1Container = extractor( + dl1: DL1TelescopeContainer = extractor( waveforms, tel_id=tel_id, selected_gain_channel=selected_gain_channel, diff --git a/ctapipe/image/tests/test_sliding_window_correction.py b/ctapipe/image/tests/test_sliding_window_correction.py index bb17daaf1f3..f38edaeccb6 100644 --- a/ctapipe/image/tests/test_sliding_window_correction.py +++ b/ctapipe/image/tests/test_sliding_window_correction.py @@ -8,7 +8,7 @@ from numpy.testing import assert_allclose from traitlets.config.loader import Config -from ctapipe.containers import TelescopeDL1Container +from ctapipe.containers import DL1TelescopeContainer from ctapipe.image.extractor import ImageExtractor, SlidingWindowMaxSum from ctapipe.image.toymodel import WaveformModel from ctapipe.instrument import SubarrayDescription @@ -53,7 +53,7 @@ def test_sw_pulse_lst(prod5_lst): ) broken_pixels = np.zeros(n_pixels, dtype=bool) - dl1: TelescopeDL1Container = extractor( + dl1: DL1TelescopeContainer = extractor( waveform, tel_id, selected_gain_channel, broken_pixels ) print(dl1.image / charge_true) diff --git a/ctapipe/instrument/tests/test_trigger.py b/ctapipe/instrument/tests/test_trigger.py index fec25900e6e..5739ffa44c5 100644 --- a/ctapipe/instrument/tests/test_trigger.py +++ b/ctapipe/instrument/tests/test_trigger.py @@ -4,7 +4,7 @@ import pytest from numpy.testing import assert_equal -from ctapipe.containers import ArrayEventContainer +from ctapipe.containers import SubarrayEventContainer from ctapipe.io import EventSource @@ -37,38 +37,38 @@ def test_software_trigger(subarray_prod5_paranal, data_type): ) # only one telescope, no SWAT - event = ArrayEventContainer() + event = SubarrayEventContainer() event.trigger.tels_with_trigger = data_type([5]) assert trigger(event) == False assert_equal(event.trigger.tels_with_trigger, data_type([])) # 1 LST + 1 MST, 1 LST would not have triggered LST hardware trigger # and after LST is removed, we only have 1 telescope, so no SWAT either - event = ArrayEventContainer() + event = SubarrayEventContainer() event.trigger.tels_with_trigger = data_type([1, 6]) assert trigger(event) == False assert_equal(event.trigger.tels_with_trigger, data_type([])) # two MSTs and 1 LST, -> remove single LST - event = ArrayEventContainer() + event = SubarrayEventContainer() event.trigger.tels_with_trigger = data_type([1, 5, 6]) assert trigger(event) == True assert_equal(event.trigger.tels_with_trigger, data_type([5, 6])) # two MSTs, nothing to change - event = ArrayEventContainer() + event = SubarrayEventContainer() event.trigger.tels_with_trigger = data_type([5, 6]) assert trigger(event) == True assert_equal(event.trigger.tels_with_trigger, data_type([5, 6])) # three LSTs, nothing to change - event = ArrayEventContainer() + event = SubarrayEventContainer() event.trigger.tels_with_trigger = data_type([1, 2, 3]) assert trigger(event) == True assert_equal(event.trigger.tels_with_trigger, data_type([1, 2, 3])) # thee LSTs, plus MSTs, nothing to change - event = ArrayEventContainer() + event = SubarrayEventContainer() event.trigger.tels_with_trigger = data_type([1, 2, 3, 5, 6, 7]) assert trigger(event) == True assert_equal(event.trigger.tels_with_trigger, data_type([1, 2, 3, 5, 6, 7])) diff --git a/ctapipe/instrument/trigger.py b/ctapipe/instrument/trigger.py index 29e79428f3f..cd1f9f9946c 100644 --- a/ctapipe/instrument/trigger.py +++ b/ctapipe/instrument/trigger.py @@ -1,6 +1,6 @@ import numpy as np -from ctapipe.containers import ArrayEventContainer +from ctapipe.containers import SubarrayEventContainer from ctapipe.core import TelescopeComponent from ctapipe.core.traits import Integer, IntTelescopeParameter @@ -64,7 +64,7 @@ def __init__(self, subarray, *args, **kwargs): for type in self.subarray.telescope_types } - def __call__(self, event: ArrayEventContainer) -> bool: + def __call__(self, event: SubarrayEventContainer) -> bool: """ Remove telescope events that have not the required number of telescopes of a given type from the subarray event and decide if the event would @@ -87,7 +87,7 @@ def __call__(self, event: ArrayEventContainer) -> bool: if min_tels == 0: continue - tels_with_trigger = set(event.trigger.tels_with_trigger) + tels_with_trigger = set(event.dl0.trigger.tels_with_trigger) tel_ids = self._ids_by_type[tel_type_str] tels_in_event = tels_with_trigger.intersection(tel_ids) @@ -103,20 +103,16 @@ def __call__(self, event: ArrayEventContainer) -> bool: tels_removed.add(tel_id) # remove any related data - for container in event.values(): - if hasattr(container, "tel"): - tel_map = container.tel - if tel_id in tel_map: - del tel_map[tel_id] + del event.tel[tel_id] if len(tels_removed) > 0: # convert to array with correct dtype to have setdiff1d work correctly tels_removed = np.fromiter(tels_removed, np.uint16, len(tels_removed)) - event.trigger.tels_with_trigger = np.setdiff1d( - event.trigger.tels_with_trigger, tels_removed, assume_unique=True + event.dl0.trigger.tels_with_trigger = np.setdiff1d( + event.dl0.trigger.tels_with_trigger, tels_removed, assume_unique=True ) - if len(event.trigger.tels_with_trigger) < self.min_telescopes: + if len(event.dl0.trigger.tels_with_trigger) < self.min_telescopes: event.trigger.tels_with_trigger = [] # remove any related data for container in event.values(): diff --git a/ctapipe/io/datawriter.py b/ctapipe/io/datawriter.py index f9a0122af91..2fbabd2a7f1 100644 --- a/ctapipe/io/datawriter.py +++ b/ctapipe/io/datawriter.py @@ -14,9 +14,9 @@ from traitlets import Dict, Instance from ..containers import ( - ArrayEventContainer, SimulatedShowerDistribution, - TelescopeEventIndexContainer, + SubarrayEventContainer, + TelescopeEventContainer, ) from ..core import Component, Container, Field, Provenance, ToolConfigurationError from ..core.traits import Bool, CaselessStrEnum, Float, Int, Path, Unicode @@ -34,14 +34,6 @@ tables.parameters.NODE_CACHE_SLOTS = 3000 # fixes problem with too many datasets -def _get_tel_index(event, tel_id): - return TelescopeEventIndexContainer( - obs_id=event.index.obs_id, - event_id=event.index.event_id, - tel_id=np.int16(tel_id), - ) - - # define the version of the data model written here. This should be updated # when necessary: # - increase the major number if there is a breaking change to the model @@ -293,46 +285,194 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, exc_traceback): self.finish() - def __call__(self, event: ArrayEventContainer): + def __call__(self, event: SubarrayEventContainer): """ Write a single event to the output file. """ self._at_least_one_event = True self.log.debug("WRITING EVENT %s", event.index) - self._write_subarray_pointing(event) - self._write_trigger(event) + self._write_simulation_subarray(event) + self._write_dl0_subarray(event) + self._write_dl2_subarray(event) + + for tel_id, tel_event in event.tel.items(): + telescope = self._subarray.tel[tel_id] + self.log.debug("WRITING TELESCOPE %s: %s", tel_id, telescope) + self._write_telescope_event(tel_event) + def _write_simulation_subarray(self, event): if event.simulation is not None and event.simulation.shower is not None: self._writer.write( table_name="simulation/event/subarray/shower", containers=[event.index, event.simulation.shower], ) - for tel_id, sim in event.simulation.tel.items(): - table_name = self.table_name(tel_id) - tel_index = _get_tel_index(event, tel_id) - self._writer.write( - f"simulation/event/telescope/impact/{table_name}", - [tel_index, sim.impact], - ) + def _write_dl0_subarray(self, event): + self._writer.write( + table_name="dl0/event/subarray/trigger", + containers=[event.index, event.dl0.trigger], + ) - if self.write_waveforms: - self._write_r1_telescope_events(event) + def _write_telescope_event(self, tel_event): + table_name = self.table_name(tel_event.index.tel_id) + self._writer.write( + f"simulation/event/telescope/impact/{table_name}", + [tel_event.index, tel_event.simulation.impact], + ) + + if tel_event.simulation is not None: + self._write_simulation_telescope(tel_event) if self.write_raw_waveforms: - self._write_r0_telescope_events(event) + self._write_r0_telescope(tel_event) + + if self.write_waveforms: + self._write_r1_telescope(tel_event) + + # always write dl0, contains needed trigger info etc. + self._write_dl0_telescope(tel_event) + self._write_pointing_telescope(tel_event) - # write telescope event data - self._write_dl1_telescope_events(event) + if self.write_images or self.write_parameters: + self._write_dl1_telescope(tel_event) - # write DL2 info if requested if self.write_showers: - self._write_dl2_telescope_events(event) - self._write_dl2_stereo_event(event) + self._write_dl2_telescope(tel_event) if self.write_muon_parameters: self._write_muon_telescope_events(event) + self._write_muon_telescope(tel_event) + + def _write_r0_telescope(self, tel_event: TelescopeEventContainer): + table_name = self.table_name(tel_event.index.tel_id) + tel_event.r0.prefix = "" + self._writer.write( + f"r0/event/telescope/{table_name}", + [tel_event.index, tel_event.r0], + ) + + def _write_r1_telescope(self, tel_event: TelescopeEventContainer): + table_name = self.table_name(tel_event.index.tel_id) + tel_event.r1.prefix = "" + self._writer.write( + f"r1/event/telescope/{table_name}", + [tel_event.index, tel_event.r1], + ) + + def _write_dl0_telescope(self, tel_event: TelescopeEventContainer): + self._writer.write( + "dl0/event/telescope/trigger", + [tel_event.index, tel_event.dl0.trigger], + ) + + def _write_pointing_telescope(self, tel_event: TelescopeEventContainer): + tel_index = tel_event.index + tel_id = tel_index.tel_id + current_pointing = (tel_event.pointing.azimuth, tel_event.pointing.altitude) + if current_pointing != self._last_pointing_tel[tel_id]: + table_name = self.table_name(tel_id) + tel_event.pointing.prefix = "" + self._writer.write( + f"dl1/monitoring/telescope/pointing/{table_name}", + [tel_event.dl0.trigger, tel_event.pointing], + ) + self._last_pointing_tel[tel_id] = current_pointing + + def _write_dl1_telescope(self, tel_event: TelescopeEventContainer): + tel_index = tel_event.index + tel_id = tel_index.tel_id + table_name = self.table_name(tel_id) + tel_event.dl1.prefix = "" # don't want a prefix for this container + + if self.write_parameters: + self._writer.write( + table_name=f"dl1/event/telescope/parameters/{table_name}", + containers=[tel_index, *tel_event.dl1.parameters.values()], + ) + + if self.write_images: + if tel_event.dl1.image is None: + raise ValueError( + "DataWriter.write_images is True but event does not contain image" + ) + + self._writer.write( + table_name=f"dl1/event/telescope/images/{table_name}", + containers=[tel_index, tel_event.dl1], + ) + + def _write_simulation_telescope(self, tel_event: TelescopeEventContainer): + tel_index = tel_event.index + tel_id = tel_index.tel_id + table_name = self.table_name(tel_id) + + # always write this, so that at least the sum is included + self._writer.write( + f"simulation/event/telescope/images/{table_name}", + [tel_index, tel_event.simulation], + ) + + has_sim_image = ( + tel_event.simulation is not None + and tel_event.simulation.true_image is not None + ) + if self.write_parameters and has_sim_image: + true_parameters = tel_event.simulation.true_parameters + # only write the available containers, no peak time related + # features for true image available. + self._writer.write( + f"simulation/event/telescope/parameters/{table_name}", + [ + tel_index, + true_parameters.hillas, + true_parameters.leakage, + true_parameters.concentration, + true_parameters.morphology, + true_parameters.intensity_statistics, + ], + ) + + def _write_muon_telescope(self, tel_event: TelescopeEventContainer): + table_name = self.table_name(tel_event.index.tel_id) + muon = tel_event.muon + self._writer.write( + f"dl1/event/telescope/muon/{table_name}", + [tel_event.index, muon.ring, muon.parameters, muon.efficiency], + ) + + def _write_dl2_telescope(self, tel_event: TelescopeEventContainer): + """ + write per-telescope DL2 shower information. + + Currently this writes to a single table per type of shower + reconstruction and per algorithm, with all telescopes combined. + """ + + table_name = self.table_name(tel_event.index.tel_id) + + for container_name, algorithm_map in tel_event.dl2.items(): + for algorithm, container in algorithm_map.items(): + name = f"dl2/event/telescope/{container_name}/{algorithm}/{table_name}" + + self._writer.write( + table_name=name, containers=[tel_event.index, container] + ) + + def _write_dl2_subarray(self, event: SubarrayEventContainer): + """ + write per-telescope DL2 shower information to e.g. + `/dl2/event/stereo/{geometry,energy,classification}/` + """ + for container_name, algorithm_map in event.dl2.items(): + for algorithm, container in algorithm_map.items(): + # note this will only write info if the particular algorithm + # generated it (otherwise the algorithm map is empty, and no + # data will be written) + self._writer.write( + table_name=f"dl2/event/subarray/{container_name}/{algorithm}", + containers=[event.index, container], + ) def finish(self): """called after all events are done""" @@ -431,7 +571,7 @@ def _setup_writer(self): tr_tel_list_to_mask = TelListToMaskTransform(self._subarray) writer.add_column_transform( - table_name="dl1/event/subarray/trigger", + table_name="dl0/event/subarray/trigger", col_name="tels_with_trigger", transform=tr_tel_list_to_mask, ) @@ -442,7 +582,7 @@ def _setup_writer(self): writer.exclude("dl1/monitoring/subarray/pointing", "event_type") writer.exclude("dl1/monitoring/subarray/pointing", "tels_with_trigger") writer.exclude("dl1/monitoring/subarray/pointing", "n_trigger_pixels") - writer.exclude("/dl1/event/telescope/trigger", "trigger_pixels") + writer.exclude("/dl0/event/telescope/trigger", "trigger_pixels") writer.exclude("/dl1/monitoring/telescope/pointing/.*", "n_trigger_pixels") writer.exclude("/dl1/monitoring/telescope/pointing/.*", "trigger_pixels") writer.exclude("/dl1/monitoring/event/pointing/.*", "event_type") @@ -506,10 +646,10 @@ def _setup_writer(self): def _write_subarray_pointing(self, event: ArrayEventContainer): """store subarray pointing info in a monitoring table""" pnt = event.pointing - current_pointing = (pnt.array_azimuth, pnt.array_altitude) + current_pointing = (pnt.azimuth, pnt.altitude) if current_pointing != self._last_pointing: pnt.prefix = "" - self._writer.write("dl1/monitoring/subarray/pointing", [event.trigger, pnt]) + self._writer.write("dl1/monitoring/subarray/pointing", [event.dl0.trigger, pnt]) self._last_pointing = current_pointing def _write_scheduling_and_observation_blocks(self): @@ -612,156 +752,6 @@ def table_name(self, tel_id): """construct dataset table names depending on chosen split method""" return f"tel_{tel_id:03d}" - def _write_trigger(self, event: ArrayEventContainer): - """ - Write trigger information - """ - self._writer.write( - table_name="dl1/event/subarray/trigger", - containers=[event.index, event.trigger], - ) - - for tel_id, trigger in event.trigger.tel.items(): - self._writer.write( - "dl1/event/telescope/trigger", (_get_tel_index(event, tel_id), trigger) - ) - - def _write_r1_telescope_events(self, event: ArrayEventContainer): - for tel_id, r1_tel in event.r1.tel.items(): - - tel_index = _get_tel_index(event, tel_id) - table_name = self.table_name(tel_id) - - r1_tel.prefix = "" - self._writer.write(f"r1/event/telescope/{table_name}", [tel_index, r1_tel]) - - def _write_r0_telescope_events(self, event: ArrayEventContainer): - for tel_id, r0_tel in event.r0.tel.items(): - - tel_index = _get_tel_index(event, tel_id) - table_name = self.table_name(tel_id) - - r0_tel.prefix = "" - self._writer.write(f"r0/event/telescope/{table_name}", [tel_index, r0_tel]) - - def _write_dl1_telescope_events(self, event: ArrayEventContainer): - """ - add entries to the event/telescope tables for each telescope in a single - event - """ - - # pointing info - for tel_id, pnt in event.pointing.tel.items(): - current_pointing = (pnt.azimuth, pnt.altitude) - if current_pointing != self._last_pointing_tel[tel_id]: - pnt.prefix = "" - self._writer.write( - f"dl1/monitoring/telescope/pointing/tel_{tel_id:03d}", - [event.trigger.tel[tel_id], pnt], - ) - self._last_pointing_tel[tel_id] = current_pointing - - for tel_id, dl1_camera in event.dl1.tel.items(): - tel_index = _get_tel_index(event, tel_id) - - dl1_camera.prefix = "" # don't want a prefix for this container - telescope = self._subarray.tel[tel_id] - self.log.debug("WRITING TELESCOPE %s: %s", tel_id, telescope) - - table_name = self.table_name(tel_id) - - if self.write_parameters: - self._writer.write( - table_name=f"dl1/event/telescope/parameters/{table_name}", - containers=[tel_index, *dl1_camera.parameters.values()], - ) - - if self.write_images: - if dl1_camera.image is None: - raise ValueError( - "DataWriter.write_images is True but event does not contain image" - ) - - self._writer.write( - table_name=f"dl1/event/telescope/images/{table_name}", - containers=[tel_index, dl1_camera], - ) - - if self._is_simulation: - # always write this, so that at least the sum is included - self._writer.write( - f"simulation/event/telescope/images/{table_name}", - [tel_index, event.simulation.tel[tel_id]], - ) - - has_sim_image = ( - tel_id in event.simulation.tel - and event.simulation.tel[tel_id].true_image is not None - ) - if self.write_parameters and has_sim_image: - true_parameters = event.simulation.tel[tel_id].true_parameters - # only write the available containers, no peak time related - # features for true image available. - self._writer.write( - f"simulation/event/telescope/parameters/{table_name}", - [ - tel_index, - true_parameters.hillas, - true_parameters.leakage, - true_parameters.concentration, - true_parameters.morphology, - true_parameters.intensity_statistics, - ], - ) - - def _write_muon_telescope_events(self, event: ArrayEventContainer): - - for tel_id, muon in event.muon.tel.items(): - table_name = self.table_name(tel_id) - tel_index = _get_tel_index(event, tel_id) - self._writer.write( - f"dl1/event/telescope/muon/{table_name}", - [tel_index, muon.ring, muon.parameters, muon.efficiency], - ) - - def _write_dl2_telescope_events(self, event: ArrayEventContainer): - """ - write per-telescope DL2 shower information. - - Currently this writes to a single table per type of shower - reconstruction and per algorithm, with all telescopes combined. - """ - - for tel_id, dl2_tel in event.dl2.tel.items(): - table_name = self.table_name(tel_id) - - tel_index = _get_tel_index(event, tel_id) - for container_name, algorithm_map in dl2_tel.items(): - for algorithm, container in algorithm_map.items(): - name = ( - f"dl2/event/telescope/{container_name}/{algorithm}/{table_name}" - ) - - self._writer.write( - table_name=name, containers=[tel_index, container] - ) - - def _write_dl2_stereo_event(self, event: ArrayEventContainer): - """ - write per-telescope DL2 shower information to e.g. - `/dl2/event/stereo/{geometry,energy,classification}/` - """ - # pylint: disable=no-self-use - for container_name, algorithm_map in event.dl2.stereo.items(): - for algorithm, container in algorithm_map.items(): - # note this will only write info if the particular algorithm - # generated it (otherwise the algorithm map is empty, and no - # data will be written) - self._writer.write( - table_name=f"dl2/event/subarray/{container_name}/{algorithm}", - containers=[event.index, container], - ) - def _generate_table_indices(self, h5file, start_node): """helper to generate PyTables index tabnles for common columns""" for node in h5file.iter_nodes(start_node): diff --git a/ctapipe/io/eventsource.py b/ctapipe/io/eventsource.py index 40de7dd7891..10e3d4bf1e5 100644 --- a/ctapipe/io/eventsource.py +++ b/ctapipe/io/eventsource.py @@ -10,10 +10,10 @@ from ctapipe.atmosphere import AtmosphereDensityProfile from ..containers import ( - ArrayEventContainer, ObservationBlockContainer, SchedulingBlockContainer, SimulationConfigContainer, + SubarrayEventContainer, ) from ..core import Provenance, ToolConfigurationError from ..core.component import Component, find_config_in_hierarchy @@ -300,7 +300,7 @@ def atmosphere_density_profile(self) -> AtmosphereDensityProfile: return None @abstractmethod - def _generator(self) -> Generator[ArrayEventContainer, None, None]: + def _generator(self) -> Generator[SubarrayEventContainer, None, None]: """ Abstract method to be defined in child class. diff --git a/ctapipe/io/hdf5eventsource.py b/ctapipe/io/hdf5eventsource.py index 9f32f27b644..0ac58da6513 100644 --- a/ctapipe/io/hdf5eventsource.py +++ b/ctapipe/io/hdf5eventsource.py @@ -12,34 +12,40 @@ from ctapipe.instrument.optics import FocalLengthKind from ..containers import ( - ArrayEventContainer, ArrayEventIndexContainer, - ArraySimulationContainer, - ArrayTriggerContainer, CameraHillasParametersContainer, CameraTimingParametersContainer, ConcentrationContainer, + DL0SubarrayContainer, + DL0TelescopeContainer, + DL1TelescopeContainer, + DL2TelescopeContainer, HillasParametersContainer, ImageParametersContainer, IntensityStatisticsContainer, LeakageContainer, MorphologyContainer, + MuonContainer, MuonEfficiencyContainer, MuonParametersContainer, MuonRingContainer, - MuonTelescopeContainer, ObservationBlockContainer, ParticleClassificationContainer, PeakTimeStatisticsContainer, + R1TelescopeContainer, ReconstructedEnergyContainer, ReconstructedGeometryContainer, SchedulingBlockContainer, SimulatedShowerContainer, SimulationConfigContainer, - TelescopeDL1Container, + SimulationSubarrayContainer, + SimulationTelescopeContainer, + SubarrayEventContainer, + SubarrayTriggerContainer, + TelescopeEventContainer, TelescopeEventIndexContainer, TelescopeImpactParameterContainer, - TelescopeR1Container, + TelescopePointingContainer, TelescopeTriggerContainer, TimingParametersContainer, ) @@ -201,13 +207,12 @@ def __init__(self, input_url=None, config=None, parent=None, **kwargs): if self.allowed_tels: self._subarray = self._full_subarray.select_subarray(self.allowed_tels) + self._allowed_tels_array = np.array(list(self.allowed_tels)) else: self._subarray = self._full_subarray + self._simulation_configs = self._parse_simulation_configs() - ( - self._scheduling_block, - self._observation_block, - ) = self._parse_sb_and_ob_configs() + self._scheduling_block, self._observation_block = self._parse_sb_and_ob() version = self.file_.root._v_attrs["CTA PRODUCT DATA MODEL VERSION"] self.datamodel_version = tuple(map(int, version.lstrip("v").split("."))) @@ -326,7 +331,7 @@ def simulation_config(self) -> Dict[int, SimulationConfigContainer]: return self._simulation_configs def __len__(self): - n_events = len(self.file_.root.dl1.event.subarray.trigger) + n_events = len(self.file_.root.dl0.event.subarray.trigger) if self.max_events is not None: return min(n_events, self.max_events) return n_events @@ -356,7 +361,7 @@ class ObsIdContainer(Container): else: return {} - def _parse_sb_and_ob_configs(self): + def _parse_sb_and_ob(self): """read Observation and Scheduling block configurations""" sb_reader = HDF5TableReader(self.file_).read( @@ -395,7 +400,7 @@ def _generator(self): if DataLevel.R1 in self.datalevels: waveform_readers = { table.name: self.reader.read( - f"/r1/event/telescope/{table.name}", TelescopeR1Container + f"/r1/event/telescope/{table.name}", R1TelescopeContainer ) for table in self.file_.root.r1.event.telescope } @@ -410,7 +415,7 @@ def _generator(self): image_readers = { table.name: self.reader.read( f"/dl1/event/telescope/images/{table.name}", - TelescopeDL1Container, + DL1TelescopeContainer, ignore_columns=ignore_columns, ) for table in self.file_.root.dl1.event.telescope.images @@ -556,12 +561,12 @@ def _generator(self): # Setup iterators for the array events events = HDF5TableReader(self.file_).read( - "/dl1/event/subarray/trigger", - [ArrayTriggerContainer, ArrayEventIndexContainer], + "/dl0/event/subarray/trigger", + [SubarrayTriggerContainer, ArrayEventIndexContainer], ignore_columns={"tel"}, ) telescope_trigger_reader = HDF5TableReader(self.file_).read( - "/dl1/event/telescope/trigger", + "/dl0/event/telescope/trigger", [TelescopeEventIndexContainer, TelescopeTriggerContainer], ignore_columns={"trigger_pixels"}, ) @@ -577,27 +582,42 @@ def _generator(self): counter = 0 for trigger, index in events: - data = ArrayEventContainer( - trigger=trigger, + event = SubarrayEventContainer( + dl0=DL0SubarrayContainer(trigger=trigger), count=counter, index=index, - simulation=ArraySimulationContainer() if self.is_simulation else None, + simulation=SimulationSubarrayContainer() + if self.is_simulation + else None, ) # Maybe take some other metadata, but there are still some 'unknown' # written out by the stage1 tool - data.meta["origin"] = self.file_.root._v_attrs["CTA PROCESS TYPE"] - data.meta["input_url"] = self.input_url - data.meta["max_events"] = self.max_events + event.meta["origin"] = self.file_.root._v_attrs["CTA PROCESS TYPE"] + event.meta["input_url"] = self.input_url + event.meta["max_events"] = self.max_events - data.trigger.tels_with_trigger = self._full_subarray.tel_mask_to_tel_ids( - data.trigger.tels_with_trigger + event.dl0.trigger.tels_with_trigger = ( + self._full_subarray.tel_mask_to_tel_ids( + event.dl0.trigger.tels_with_trigger + ) ) - full_tels_with_trigger = data.trigger.tels_with_trigger.copy() + full_tels_with_trigger = event.dl0.trigger.tels_with_trigger.copy() if self.allowed_tels: - data.trigger.tels_with_trigger = np.intersect1d( - data.trigger.tels_with_trigger, np.array(list(self.allowed_tels)) + event.dl0.trigger.tels_with_trigger = np.intersect1d( + event.dl0.trigger.tels_with_trigger, + self._allowed_tels_array, ) + if self.is_simulation: + event.simulation.shower = next(mc_shower_reader) + + for kind, readers in dl2_readers.items(): + c = getattr(event.dl2, kind) + for algorithm, reader in readers.items(): + c[algorithm] = next(reader) + + self._fill_array_pointing(event, array_pointing_finder) + # the telescope trigger table contains triggers for all telescopes # that participated in the event, so we need to read a row for each # of them, ignoring the ones not in allowed_tels after reading @@ -607,53 +627,43 @@ def _generator(self): if self.allowed_tels and tel_id not in self.allowed_tels: continue - data.trigger.tel[tel_index.tel_id] = tel_trigger - - if self.is_simulation: - data.simulation.shower = next(mc_shower_reader) - - for kind, readers in dl2_readers.items(): - c = getattr(data.dl2.stereo, kind) - for algorithm, reader in readers.items(): - c[algorithm] = next(reader) - - # this needs to stay *after* reading the telescope trigger table - # and after reading all subarray event information, so that we don't - # go out of sync - if len(data.trigger.tels_with_trigger) == 0: - continue - - self._fill_array_pointing(data, array_pointing_finder) - self._fill_telescope_pointing(data, tel_pointing_finder) - - for tel_id in data.trigger.tel.keys(): key = f"tel_{tel_id:03d}" - if self.allowed_tels and tel_id not in self.allowed_tels: - continue + containers = { + "index": tel_index, + "dl0": DL0TelescopeContainer(trigger=tel_trigger), + } - if key in true_impact_readers: - data.simulation.tel[tel_id].impact = next(true_impact_readers[key]) + containers["pointing"] = self._fill_telescope_pointing( + tel_id, tel_trigger.time, tel_pointing_finder + ) if DataLevel.R1 in self.datalevels: - data.r1.tel[tel_id] = next(waveform_readers[key]) + containers["r1"] = next(waveform_readers[key]) + + if self.is_simulation: + containers["simulation"] = SimulationTelescopeContainer() - if self.has_simulated_dl1: - simulated = data.simulation.tel[tel_id] + if key in true_impact_readers: + containers["simulation"].impact = next(true_impact_readers[key]) if DataLevel.DL1_IMAGES in self.datalevels: - data.dl1.tel[tel_id] = next(image_readers[key]) + containers["dl1"] = next(image_readers[key]) if self.has_simulated_dl1: simulated_image_row = next(simulated_image_iterators[key]) - simulated.true_image = simulated_image_row["true_image"] + containers["simulation"].true_image = simulated_image_row[ + "true_image" + ] + else: + containers["dl1"] = DL1TelescopeContainer() if DataLevel.DL1_PARAMETERS in self.datalevels: # Is there a smarter way to unpack this? # Best would probbaly be if we could directly read # into the ImageParametersContainer params = next(param_readers[key]) - data.dl1.tel[tel_id].parameters = ImageParametersContainer( + containers["dl1"].parameters = ImageParametersContainer( hillas=params[0], timing=params[1], leakage=params[2], @@ -664,7 +674,9 @@ def _generator(self): ) if self.has_simulated_dl1: simulated_params = next(simulated_param_readers[key]) - simulated.true_parameters = ImageParametersContainer( + containers[ + "simulation" + ].true_parameters = ImageParametersContainer( hillas=simulated_params[0], leakage=simulated_params[1], concentration=simulated_params[2], @@ -674,14 +686,15 @@ def _generator(self): if self.has_muon_parameters: ring, parameters, efficiency = next(muon_readers[key]) - data.muon.tel[tel_id] = MuonTelescopeContainer( + containers["muon"] = MuonContainer( ring=ring, parameters=parameters, efficiency=efficiency, ) + dl2 = DL2TelescopeContainer() for kind, algorithms in dl2_tel_readers.items(): - c = getattr(data.dl2.tel[tel_id], kind) + c = getattr(dl2, kind) for algorithm, readers in algorithms.items(): c[algorithm] = next(readers[key]) @@ -690,7 +703,15 @@ def _generator(self): prefix = f"{algorithm}_tel_{c[algorithm].default_prefix}" c[algorithm].prefix = prefix - yield data + event.tel[tel_id] = TelescopeEventContainer(**containers, dl2=dl2) + + # this needs to stay *after* reading the telescope trigger table + # and after reading all subarray event information, so that we don't + # go out of sync + if len(event.tel) == 0: + continue + + yield event counter += 1 @lazyproperty @@ -703,57 +724,53 @@ def _telescope_pointing_attrs(self, tel_id): pointing_group = self.file_.root.dl1.monitoring.telescope.pointing return get_column_attrs(pointing_group[f"tel_{tel_id:03d}"]) - def _fill_array_pointing(self, data, array_pointing_finder): + def _fill_array_pointing(self, event, array_pointing_finder): """ Fill the array pointing information of a given event """ # Only unique pointings are stored, so reader.read() wont work as easily # Thats why we match the pointings based on trigger time - closest_time_index = array_pointing_finder.closest(data.trigger.time.mjd) + closest_time_index = array_pointing_finder.closest(event.dl0.trigger.time.mjd) table = self.file_.root.dl1.monitoring.subarray.pointing array_pointing = table[closest_time_index] - data.pointing.array_azimuth = u.Quantity( - array_pointing["array_azimuth"], - self._subarray_pointing_attrs["array_azimuth"]["UNIT"], + event.pointing.azimuth = u.Quantity( + array_pointing["azimuth"], + self._subarray_pointing_attrs["azimuth"]["UNIT"], ) - data.pointing.array_altitude = u.Quantity( - array_pointing["array_altitude"], - self._subarray_pointing_attrs["array_altitude"]["UNIT"], + event.pointing.altitude = u.Quantity( + array_pointing["altitude"], + self._subarray_pointing_attrs["altitude"]["UNIT"], ) - data.pointing.array_ra = u.Quantity( - array_pointing["array_ra"], - self._subarray_pointing_attrs["array_ra"]["UNIT"], + event.pointing.ra = u.Quantity( + array_pointing["ra"], + self._subarray_pointing_attrs["ra"]["UNIT"], ) - data.pointing.array_dec = u.Quantity( - array_pointing["array_dec"], - self._subarray_pointing_attrs["array_dec"]["UNIT"], + event.pointing.dec = u.Quantity( + array_pointing["dec"], + self._subarray_pointing_attrs["dec"]["UNIT"], ) - def _fill_telescope_pointing(self, data, tel_pointing_finder): + def _fill_telescope_pointing(self, tel_id, time, tel_pointing_finder): """ Fill the telescope pointing information of a given event """ # Same comments as to _fill_array_pointing apply pointing_group = self.file_.root.dl1.monitoring.telescope.pointing - for tel_id in data.trigger.tel.keys(): - key = f"tel_{tel_id:03d}" - - if self.allowed_tels and tel_id not in self.allowed_tels: - continue + key = f"tel_{tel_id:03d}" - tel_pointing_table = pointing_group[key] - closest_time_index = tel_pointing_finder[key].closest( - data.trigger.tel[tel_id].time.mjd - ) + tel_pointing_table = pointing_group[key] + closest_time_index = tel_pointing_finder[key].closest(time.mjd) - pointing_telescope = tel_pointing_table[closest_time_index] - attrs = self._telescope_pointing_attrs(tel_id) - data.pointing.tel[tel_id].azimuth = u.Quantity( + pointing_telescope = tel_pointing_table[closest_time_index] + attrs = self._telescope_pointing_attrs(tel_id) + return TelescopePointingContainer( + azimuth=u.Quantity( pointing_telescope["azimuth"], attrs["azimuth"]["UNIT"], - ) - data.pointing.tel[tel_id].altitude = u.Quantity( + ), + altitude=u.Quantity( pointing_telescope["altitude"], attrs["altitude"]["UNIT"], - ) + ), + ) diff --git a/ctapipe/io/simteleventsource.py b/ctapipe/io/simteleventsource.py index 7590f180855..a7153278cb1 100644 --- a/ctapipe/io/simteleventsource.py +++ b/ctapipe/io/simteleventsource.py @@ -22,12 +22,10 @@ ) from ..calib.camera.gainselection import GainSelector from ..containers import ( - ArrayEventContainer, ArrayEventIndexContainer, - ArrayPointingContainer, - ArraySimulationContainer, - ArrayTriggerContainer, CoordinateFrameType, + DL0SubarrayContainer, + DL0TelescopeContainer, DL1CameraCalibrationContainer, EventType, ObservationBlockContainer, @@ -35,19 +33,23 @@ ObservingMode, PixelStatusContainer, PointingMode, + R0TelescopeContainer, + R1TelescopeContainer, SchedulingBlockContainer, SchedulingBlockType, SimulatedShowerContainer, SimulationConfigContainer, + SimulationSubarrayContainer, + SimulationTelescopeContainer, + SubarrayEventContainer, + SubarrayPointingContainer, + SubarrayTriggerContainer, TelescopeCalibrationContainer, TelescopeEventContainer, TelescopeEventIndexContainer, TelescopeImpactParameterContainer, TelescopeMonitoringContainer, TelescopePointingContainer, - TelescopeR0Container, - TelescopeR1Container, - TelescopeSimulationContainer, TelescopeTriggerContainer, ) from ..coordinates import CameraFrame, shower_impact_distance @@ -733,12 +735,12 @@ def _generate_events(self): else: shower = None - event = ArrayEventContainer( - simulation=ArraySimulationContainer(shower=shower), + event = SubarrayEventContainer( + count=counter, + simulation=SimulationSubarrayContainer(shower=shower), + dl0=DL0SubarrayContainer(trigger=array_trigger), pointing=self._fill_array_pointing(), index=ArrayEventIndexContainer(obs_id=obs_id, event_id=event_id), - count=counter, - trigger=array_trigger, ) event.meta["origin"] = "hessio" event.meta["input_url"] = self.input_url @@ -788,7 +790,7 @@ def _generate_events(self): prefix="true_impact", ) - simulation = TelescopeSimulationContainer( + simulation = SimulationTelescopeContainer( true_image_sum=true_image_sums[ self.telescope_indices_original[tel_id] ], @@ -798,7 +800,7 @@ def _generate_events(self): else: simulation = None - r0 = TelescopeR0Container(waveform=adc_samples) + r0 = R0TelescopeContainer(waveform=adc_samples) cam_mon = array_event["camera_monitorings"][tel_id] pedestal = cam_mon["pedestal"] / cam_mon["n_ped_slices"] @@ -819,7 +821,7 @@ def _generate_events(self): self.calib_scale, self.calib_shift, ) - r1 = TelescopeR1Container( + r1 = R1TelescopeContainer( waveform=r1_waveform, selected_gain_channel=selected_gain_channel, ) @@ -842,10 +844,10 @@ def _generate_events(self): ), r0=r0, r1=r1, + dl0=DL0TelescopeContainer(trigger=tel_trigger[tel_id]), simulation=simulation, mon=mon, pointing=self._fill_event_pointing(tracking_positions[tel_id]), - trigger=tel_trigger[tel_id], calibration=calibration, ) @@ -917,7 +919,7 @@ def _fill_trigger_info(self, array_event): central_time = parse_simtel_time(trigger["gps_time"]) - array_trigger = ArrayTriggerContainer( + array_trigger = SubarrayTriggerContainer( event_type=event_type, time=central_time, tels_with_trigger=tels_with_trigger, @@ -961,13 +963,13 @@ def _fill_trigger_info(self, array_event): def _fill_array_pointing(self): if self.file_.header["tracking_mode"] == 0: az, alt = self.file_.header["direction"] - return ArrayPointingContainer( + return SubarrayPointingContainer( altitude=u.Quantity(alt, u.rad), azimuth=u.Quantity(az, u.rad), ) else: ra, dec = self.file_.header["direction"] - return ArrayPointingContainer( + return SubarrayPointingContainer( ra=u.Quantity(ra, u.rad), dec=u.Quantity(dec, u.rad), ) diff --git a/ctapipe/io/tableloader.py b/ctapipe/io/tableloader.py index f7077969e8e..a70f9b1d4a3 100644 --- a/ctapipe/io/tableloader.py +++ b/ctapipe/io/tableloader.py @@ -23,7 +23,7 @@ PARAMETERS_GROUP = "/dl1/event/telescope/parameters" IMAGES_GROUP = "/dl1/event/telescope/images" MUON_GROUP = "/dl1/event/telescope/muon" -TRIGGER_TABLE = "/dl1/event/subarray/trigger" +TRIGGER_TABLE = "/dl0/event/subarray/trigger" SHOWER_TABLE = "/simulation/event/subarray/shower" TRUE_IMAGES_GROUP = "/simulation/event/telescope/images" TRUE_PARAMETERS_GROUP = "/simulation/event/telescope/parameters" @@ -462,7 +462,7 @@ def _read_telescope_events_for_id(self, tel_id, start=None, stop=None): table = read_table( self.h5file, - "/dl1/event/telescope/trigger", + "/dl0/event/telescope/trigger", condition=f"tel_id == {tel_id}", start=trigger_start, stop=trigger_stop, diff --git a/ctapipe/io/tests/test_datawriter.py b/ctapipe/io/tests/test_datawriter.py index abb68cce42c..351a1f42618 100644 --- a/ctapipe/io/tests/test_datawriter.py +++ b/ctapipe/io/tests/test_datawriter.py @@ -28,37 +28,34 @@ def generate_dummy_dl2(event): algos = ["HillasReconstructor", "ImPACTReconstructor"] for algo in algos: - for tel_id in event.dl1.tel: - event.dl2.tel[tel_id].geometry[algo] = ReconstructedGeometryContainer( + for tel_id, tel_event in event.tel.items(): + tel_event.dl2.geometry[algo] = ReconstructedGeometryContainer( alt=70 * u.deg, az=120 * u.deg, prefix=f"{algo}_tel", ) - event.dl2.tel[tel_id].energy[algo] = ReconstructedEnergyContainer( + tel_event.dl2.energy[algo] = ReconstructedEnergyContainer( energy=10 * u.TeV, prefix=f"{algo}_tel", ) - event.dl2.tel[tel_id].classification[ - algo - ] = ParticleClassificationContainer( + tel_event.dl2.classification[algo] = ParticleClassificationContainer( prediction=0.9, prefix=f"{algo}_tel", ) - event.dl2.stereo.geometry[algo] = ReconstructedGeometryContainer( + event.dl2.geometry[algo] = ReconstructedGeometryContainer( alt=72 * u.deg, az=121 * u.deg, telescopes=[1, 2, 4], prefix=algo, ) - - event.dl2.stereo.energy[algo] = ReconstructedEnergyContainer( + event.dl2.energy[algo] = ReconstructedEnergyContainer( energy=10 * u.TeV, telescopes=[1, 2, 4], prefix=algo, ) - event.dl2.stereo.classification[algo] = ParticleClassificationContainer( + event.dl2.classification[algo] = ParticleClassificationContainer( prediction=0.9, telescopes=[1, 2, 4], prefix=algo, @@ -201,13 +198,13 @@ def test_roundtrip(tmpdir: Path): for event in EventSource(output_path): - for tel_id, dl1 in event.dl1.tel.items(): - original_image = events[event.count].dl1.tel[tel_id].image - read_image = dl1.image + for tel_id, tel_event in event.tel.items(): + original_image = events[event.count].tel[tel_id].dl1.image + read_image = tel_event.dl1.image assert np.allclose(original_image, read_image, atol=0.1) - original_peaktime = events[event.count].dl1.tel[tel_id].peak_time - read_peaktime = dl1.peak_time + original_peaktime = events[event.count].tel[tel_id].dl1.peak_time + read_peaktime = tel_event.dl1.peak_time assert np.allclose(original_peaktime, read_peaktime, atol=0.01) diff --git a/ctapipe/io/tests/test_event_source.py b/ctapipe/io/tests/test_event_source.py index 639d3547181..1abfc843a41 100644 --- a/ctapipe/io/tests/test_event_source.py +++ b/ctapipe/io/tests/test_event_source.py @@ -2,7 +2,7 @@ from traitlets import TraitError from traitlets.config.loader import Config -from ctapipe.containers import ArrayEventContainer +from ctapipe.containers import SubarrayEventContainer from ctapipe.core import Component from ctapipe.io import DataLevel, EventSource, SimTelEventSource from ctapipe.utils import get_dataset_path @@ -23,7 +23,7 @@ class DummyEventSource(EventSource): def _generator(self): for i in range(5): - yield ArrayEventContainer(count=i) + yield SubarrayEventContainer(count=i) @staticmethod def is_compatible(file_path): diff --git a/ctapipe/io/tests/test_hdf5.py b/ctapipe/io/tests/test_hdf5.py index da0b679c7a9..7612cad0e34 100644 --- a/ctapipe/io/tests/test_hdf5.py +++ b/ctapipe/io/tests/test_hdf5.py @@ -11,9 +11,9 @@ from ctapipe.containers import ( HillasParametersContainer, LeakageContainer, + R0TelescopeContainer, SimulatedShowerContainer, TelescopeEventIndexContainer, - TelescopeR0Container, ) from ctapipe.core.container import Container, Field from ctapipe.io import read_table @@ -25,7 +25,7 @@ def test_h5_file(tmp_path_factory): """Test hdf5 file with some tables for the reader tests""" path = tmp_path_factory.mktemp("hdf5") / "test.h5" - r0 = TelescopeR0Container() + r0 = R0TelescopeContainer() shower = SimulatedShowerContainer() r0.waveform = np.random.uniform(size=(50, 10)) r0.meta["test_attribute"] = 3.14159 @@ -381,8 +381,8 @@ def test_read_container(test_h5_file): # test supplying a single container as well as an # iterable with one entry only simtab = reader.read("/R0/sim_shower", (SimulatedShowerContainer,)) - r0tab1 = reader.read("/R0/tel_001", TelescopeR0Container) - r0tab2 = reader.read("/R0/tel_002", TelescopeR0Container) + r0tab1 = reader.read("/R0/tel_001", R0TelescopeContainer) + r0tab2 = reader.read("/R0/tel_002", R0TelescopeContainer) # read all 3 tables in sync for _ in range(3): diff --git a/ctapipe/io/tests/test_hdf5eventsource.py b/ctapipe/io/tests/test_hdf5eventsource.py index bae1eb1359b..4d7b6b23a62 100644 --- a/ctapipe/io/tests/test_hdf5eventsource.py +++ b/ctapipe/io/tests/test_hdf5eventsource.py @@ -77,14 +77,14 @@ def test_simulation_info(dl1_file): with HDF5EventSource(input_url=dl1_file) as source: for event in source: assert np.isfinite(event.simulation.shower.energy) - for tel in event.simulation.tel: - assert tel in event.simulation.tel - assert event.simulation.tel[tel].true_image is not None + for tel_event in event.tel.values(): + assert tel_event.simulation is not None + assert tel_event.simulation.true_image is not None reco_lons.append( - event.simulation.tel[tel].true_parameters.hillas.fov_lon.value + tel_event.simulation.true_parameters.hillas.fov_lon.value ) reco_concentrations.append( - event.simulation.tel[tel].true_parameters.concentration.core + tel_event.simulation.true_parameters.concentration.core ) assert not np.isnan(reco_lons).all() assert sum(np.isnan(reco_lons)) == sum(np.isnan(reco_concentrations)) @@ -94,8 +94,8 @@ def test_dl1_a_only_data(dl1_image_file): with HDF5EventSource(input_url=dl1_image_file) as source: assert source.datalevels == (DataLevel.DL1_IMAGES,) for event in source: - for tel in event.dl1.tel: - assert event.dl1.tel[tel].image.any() + for tel_event in event.tel.values(): + assert tel_event.dl1.image.any() def test_dl1_b_only_data(dl1_parameters_file): @@ -104,12 +104,12 @@ def test_dl1_b_only_data(dl1_parameters_file): with HDF5EventSource(input_url=dl1_parameters_file) as source: assert source.datalevels == (DataLevel.DL1_PARAMETERS,) for event in source: - for tel in event.dl1.tel: + for tel_event in event.tel.values(): reco_lons.append( - event.simulation.tel[tel].true_parameters.hillas.fov_lon.value + tel_event.simulation.true_parameters.hillas.fov_lon.value ) reco_concentrations.append( - event.simulation.tel[tel].true_parameters.concentration.core + tel_event.simulation.true_parameters.concentration.core ) assert not np.isnan(reco_lons).all() assert sum(np.isnan(reco_lons)) == sum(np.isnan(reco_concentrations)) @@ -120,14 +120,15 @@ def test_dl1_data(dl1_file): reco_concentrations = [] with HDF5EventSource(input_url=dl1_file) as source: for event in source: - for tel in event.dl1.tel: - assert event.dl1.tel[tel].image.any() + for tel_event in event.tel.values(): + assert tel_event.dl1.image.any() reco_lons.append( - event.simulation.tel[tel].true_parameters.hillas.fov_lon.value + tel_event.simulation.true_parameters.hillas.fov_lon.value ) reco_concentrations.append( - event.simulation.tel[tel].true_parameters.concentration.core + tel_event.simulation.true_parameters.concentration.core ) + assert not np.isnan(reco_lons).all() assert sum(np.isnan(reco_lons)) == sum(np.isnan(reco_concentrations)) @@ -135,23 +136,22 @@ def test_dl1_data(dl1_file): def test_pointing(dl1_file): with HDF5EventSource(input_url=dl1_file) as source: for event in source: - assert np.isclose(event.pointing.array_azimuth.to_value(u.deg), 0) - assert np.isclose(event.pointing.array_altitude.to_value(u.deg), 70) - assert event.pointing.tel - for tel in event.pointing.tel: - assert np.isclose(event.pointing.tel[tel].azimuth.to_value(u.deg), 0) - assert np.isclose(event.pointing.tel[tel].altitude.to_value(u.deg), 70) + assert np.isclose(event.pointing.azimuth.to_value(u.deg), 0) + assert np.isclose(event.pointing.altitude.to_value(u.deg), 70) + for tel_event in event.tel.values(): + assert np.isclose(tel_event.pointing.azimuth.to_value(u.deg), 0) + assert np.isclose(tel_event.pointing.altitude.to_value(u.deg), 70) def test_read_r1(r1_hdf5_file): - print(r1_hdf5_file) with HDF5EventSource(input_url=r1_hdf5_file) as source: e = None assert source.datalevels == (DataLevel.R1,) for e in source: - pass + for tel_event in e.tel.values(): + assert tel_event.r1.waveform is not None assert e is not None assert e.count == 3 @@ -165,8 +165,8 @@ def test_trigger_allowed_tels(dl1_proton_file): i = 0 for i, e in enumerate(s): assert e.count == i - assert set(e.trigger.tels_with_trigger) == e.trigger.tel.keys() - assert len(e.trigger.tels_with_trigger) > 1 + assert set(e.dl0.trigger.tels_with_trigger) == e.tel.keys() + assert len(e.dl0.trigger.tels_with_trigger) > 1 assert i == 1 @@ -182,18 +182,18 @@ def test_read_dl2(dl2_shower_geometry_file): ) e = next(iter(s)) - assert algorithm in e.dl2.stereo.geometry - assert e.dl2.stereo.geometry[algorithm].alt is not None - assert e.dl2.stereo.geometry[algorithm].az is not None - assert e.dl2.stereo.geometry[algorithm].telescopes is not None - assert e.dl2.stereo.geometry[algorithm].prefix == algorithm + assert algorithm in e.dl2.geometry + assert e.dl2.geometry[algorithm].alt is not None + assert e.dl2.geometry[algorithm].az is not None + assert e.dl2.geometry[algorithm].telescopes is not None + assert e.dl2.geometry[algorithm].prefix == algorithm - tel_mask = e.dl2.stereo.geometry[algorithm].telescopes + tel_mask = e.dl2.geometry[algorithm].telescopes tel_ids = s.subarray.tel_mask_to_tel_ids(tel_mask) for tel_id in tel_ids: - assert tel_id in e.dl2.tel - assert algorithm in e.dl2.tel[tel_id].impact - impact = e.dl2.tel[tel_id].impact[algorithm] + assert tel_id in e.tel + assert algorithm in e.tel[tel_id].dl2.impact + impact = e.tel[tel_id].dl2.impact[algorithm] assert impact.prefix == algorithm + "_tel_impact" assert impact.distance is not None @@ -202,7 +202,8 @@ def test_dl1_camera_frame(dl1_camera_frame_file): with HDF5EventSource(dl1_camera_frame_file) as s: tel_id = None for e in s: - for tel_id, dl1 in e.dl1.tel.items(): + for tel_id, tel_event in e.tel.items(): + dl1 = tel_event.dl1 assert isinstance( dl1.parameters.hillas, CameraHillasParametersContainer ) @@ -211,7 +212,7 @@ def test_dl1_camera_frame(dl1_camera_frame_file): ) assert dl1.parameters.hillas.intensity is not None - for tel_id, sim in e.simulation.tel.items(): + sim = tel_event.simulation assert isinstance( sim.true_parameters.hillas, CameraHillasParametersContainer ) diff --git a/ctapipe/io/tests/test_simteleventsource.py b/ctapipe/io/tests/test_simteleventsource.py index 9a8ecab125c..e3d34911191 100644 --- a/ctapipe/io/tests/test_simteleventsource.py +++ b/ctapipe/io/tests/test_simteleventsource.py @@ -195,7 +195,7 @@ def test_allowed_telescopes(): assert not allowed_tels.symmetric_difference(reader.subarray.tel_ids) for event in reader: assert set(event.tel).issubset(allowed_tels) - assert set(event.trigger.tels_with_trigger).issubset(allowed_tels) + assert set(event.dl0.trigger.tels_with_trigger).issubset(allowed_tels) def test_calibration_events(): @@ -224,7 +224,7 @@ def test_calibration_events(): for event, expected_type, expected_id in zip_longest( reader, expected_types, expected_ids ): - assert event.trigger.event_type is expected_type + assert event.dl0.trigger.event_type is expected_type assert event.index.event_id == expected_id @@ -238,12 +238,14 @@ def test_trigger_times(): t1 = Time("2020-05-06T15:40:00") for array_event in source: - assert t0 <= array_event.trigger.time <= t1 + assert t0 <= array_event.dl0.trigger.time <= t1 for tel_event in array_event.tel.values(): # test single telescope events triggered within 50 ns assert ( 0 - <= (tel_event.trigger.time - array_event.trigger.time).to_value(u.ns) + <= (tel_event.dl0.trigger.time - array_event.dl0.trigger.time).to_value( + u.ns + ) <= 50 ) diff --git a/ctapipe/io/tests/test_table_loader.py b/ctapipe/io/tests/test_table_loader.py index 864a53ba9f7..5824f7b2600 100644 --- a/ctapipe/io/tests/test_table_loader.py +++ b/ctapipe/io/tests/test_table_loader.py @@ -183,7 +183,7 @@ def test_table_loader_keeps_original_order(dl2_merged_file): from ctapipe.io.tableloader import TableLoader # check that the order is the same as in the file itself - trigger = read_table(dl2_merged_file, "/dl1/event/subarray/trigger") + trigger = read_table(dl2_merged_file, "/dl0/event/subarray/trigger") # check we actually have unsorted input assert not np.all(np.diff(trigger["obs_id"]) >= 0) @@ -278,7 +278,7 @@ def test_chunked(dl2_shower_geometry_file): """Test chunked reading""" from ctapipe.io.tableloader import TableLoader, read_table - trigger = read_table(dl2_shower_geometry_file, "/dl1/event/subarray/trigger") + trigger = read_table(dl2_shower_geometry_file, "/dl0/event/subarray/trigger") n_events = len(trigger) n_read = 0 @@ -415,8 +415,8 @@ def test_order_merged(): path = get_dataset_path("gamma_diffuse_dl2_train_small.dl2.h5") - trigger = read_table(path, "/dl1/event/subarray/trigger") - tel_trigger = read_table(path, "/dl1/event/telescope/trigger") + trigger = read_table(path, "/dl0/event/subarray/trigger") + tel_trigger = read_table(path, "/dl0/event/telescope/trigger") with TableLoader( path, load_dl1_parameters=True, diff --git a/ctapipe/io/toymodel.py b/ctapipe/io/toymodel.py index 558028749a3..04e46ebef35 100644 --- a/ctapipe/io/toymodel.py +++ b/ctapipe/io/toymodel.py @@ -9,11 +9,11 @@ import numpy as np from ..containers import ( - ArrayEventContainer, ArrayEventIndexContainer, + DL1TelescopeContainer, ObservationBlockContainer, SchedulingBlockContainer, - TelescopeDL1Container, + SubarrayEventContainer, ) from ..core import TelescopeComponent, traits from ..image import toymodel @@ -98,7 +98,7 @@ def _generator(self): def generate_event(self): - event = ArrayEventContainer( + event = SubarrayEventContainer( index=ArrayEventIndexContainer(obs_id=1, event_id=self.event_id), trigger=None, r0=None, @@ -150,6 +150,6 @@ def generate_event(self): ) image, _, _ = model.generate_image(cam, intensity) - event.dl1.tel[tel_id] = TelescopeDL1Container(image=image) + event.dl1.tel[tel_id] = DL1TelescopeContainer(image=image) return event diff --git a/ctapipe/reco/hillas_reconstructor.py b/ctapipe/reco/hillas_reconstructor.py index 1fbd305d7c2..588029157d8 100644 --- a/ctapipe/reco/hillas_reconstructor.py +++ b/ctapipe/reco/hillas_reconstructor.py @@ -144,7 +144,7 @@ def __call__(self, event): try: hillas_dict = self._create_hillas_dict(event) except (TooFewTelescopesException, InvalidWidthException): - event.dl2.stereo.geometry[self.__class__.__name__] = INVALID + event.dl2.geometry[self.__class__.__name__] = INVALID self._store_impact_parameter(event) return @@ -165,7 +165,7 @@ def __call__(self, event): # store core corrected psi values for tel_id, psi in zip(tel_ids, corrected_psi): - event.dl1.tel[tel_id].parameters.core.psi = u.Quantity( + event.tel[tel_id].dl1.parameters.core.psi = u.Quantity( np.rad2deg(psi), u.deg ) @@ -186,9 +186,7 @@ def __call__(self, event): # az is clockwise, lon counter-clockwise, make sure it stays in [0, 2pi) az = Longitude(-lon) - event.dl2.stereo.geometry[ - self.__class__.__name__ - ] = ReconstructedGeometryContainer( + event.dl2.geometry[self.__class__.__name__] = ReconstructedGeometryContainer( alt=lat, az=az, core_x=core_pos_ground.x, @@ -230,10 +228,9 @@ def initialize_arrays(self, event, hillas_dict): # get one telescope id to check what frame to use altaz = AltAz() - # Due to tracking the pointing of the array will never be a constant array_pointing = SkyCoord( - az=event.pointing.array_azimuth, - alt=event.pointing.array_altitude, + az=event.pointing.azimuth, + alt=event.pointing.altitude, frame=altaz, ) @@ -254,7 +251,7 @@ def initialize_arrays(self, event, hillas_dict): for i, (tel_id, hillas) in enumerate(hillas_dict.items()): tel_ids[i] = tel_id - pointing = event.pointing.tel[tel_id] + pointing = event.tel[tel_id].pointing alt[i] = pointing.altitude.to_value(u.rad) az[i] = pointing.azimuth.to_value(u.rad) diff --git a/ctapipe/reco/preprocessing.py b/ctapipe/reco/preprocessing.py index c815dfb44ba..9d7aae5df1c 100644 --- a/ctapipe/reco/preprocessing.py +++ b/ctapipe/reco/preprocessing.py @@ -17,7 +17,7 @@ from ctapipe.coordinates import MissingFrameAttributeWarning, TelescopeFrame -from ..containers import ArrayEventContainer +from ..containers import SubarrayEventContainer LOG = logging.getLogger(__name__) @@ -69,7 +69,7 @@ def table_to_X(table: Table, features: List[str], log=LOG): def collect_features( - event: ArrayEventContainer, tel_id: int, subarray_table=None + event: SubarrayEventContainer, tel_id: int, subarray_table=None ) -> Table: """Loop over all containers with features. diff --git a/ctapipe/reco/reconstructor.py b/ctapipe/reco/reconstructor.py index 28402449ee4..96efc5ba384 100644 --- a/ctapipe/reco/reconstructor.py +++ b/ctapipe/reco/reconstructor.py @@ -6,7 +6,7 @@ import numpy as np from astropy.coordinates import AltAz, SkyCoord -from ctapipe.containers import ArrayEventContainer, TelescopeImpactParameterContainer +from ctapipe.containers import SubarrayEventContainer, TelescopeImpactParameterContainer from ctapipe.core import Provenance, QualityQuery, TelescopeComponent from ctapipe.core.traits import List @@ -78,7 +78,7 @@ def __init__(self, subarray, **kwargs): self.quality_query = StereoQualityQuery(parent=self) @abstractmethod - def __call__(self, event: ArrayEventContainer): + def __call__(self, event: SubarrayEventContainer): """ Perform stereo reconstruction on event. @@ -149,9 +149,9 @@ class HillasGeometryReconstructor(Reconstructor): def _create_hillas_dict(self, event): hillas_dict = { - tel_id: dl1.parameters.hillas - for tel_id, dl1 in event.dl1.tel.items() - if all(self.quality_query(parameters=dl1.parameters)) + tel_id: tel_event.dl1.parameters.hillas + for tel_id, tel_event in event.tel.items() + if all(self.quality_query(parameters=tel_event.dl1.parameters)) } if len(hillas_dict) < 2: @@ -183,7 +183,8 @@ def _get_telescope_pointings(event): def _store_impact_parameter(self, event): """Compute and store the impact parameter for each reconstruction.""" - geometry = event.dl2.stereo.geometry[self.__class__.__name__] + key = self.__class__.__name__ + geometry = event.dl2.geometry[key] if geometry.is_valid: impact_distances = shower_impact_distance( @@ -195,12 +196,10 @@ def _store_impact_parameter(self, event): impact_distances = u.Quantity(np.full(n_tels, np.nan), u.m) default_prefix = TelescopeImpactParameterContainer.default_prefix - prefix = f"{self.__class__.__name__}_tel_{default_prefix}" - for tel_id in event.trigger.tels_with_trigger: + prefix = f"{key}_tel_{default_prefix}" + for tel_id, tel_event in event.tel.items(): tel_index = self.subarray.tel_indices[tel_id] - event.dl2.tel[tel_id].impact[ - self.__class__.__name__ - ] = TelescopeImpactParameterContainer( + tel_event.dl2.impact[key] = TelescopeImpactParameterContainer( distance=impact_distances[tel_index], prefix=prefix, ) diff --git a/ctapipe/reco/shower_processor.py b/ctapipe/reco/shower_processor.py index af4011b19ad..86e3b267d37 100644 --- a/ctapipe/reco/shower_processor.py +++ b/ctapipe/reco/shower_processor.py @@ -1,7 +1,7 @@ """ High level processing of showers. """ -from ..containers import ArrayEventContainer +from ..containers import SubarrayEventContainer from ..core import Component, traits from ..instrument import SubarrayDescription from .reconstructor import Reconstructor @@ -69,7 +69,7 @@ def __init__( for reco_type in self.reconstructor_types ] - def __call__(self, event: ArrayEventContainer): + def __call__(self, event: SubarrayEventContainer): """ Apply all configured stereo reconstructors to the given event. diff --git a/ctapipe/reco/sklearn.py b/ctapipe/reco/sklearn.py index 976ecb6079e..8818963efcf 100644 --- a/ctapipe/reco/sklearn.py +++ b/ctapipe/reco/sklearn.py @@ -21,11 +21,11 @@ from ctapipe.exceptions import TooFewEvents from ..containers import ( - ArrayEventContainer, DispContainer, ParticleClassificationContainer, ReconstructedEnergyContainer, ReconstructedGeometryContainer, + SubarrayEventContainer, ) from ..coordinates import TelescopeFrame from ..core import Component, FeatureGenerator, Provenance, QualityQuery, traits @@ -154,7 +154,7 @@ def __init__(self, subarray=None, models=None, **kwargs): self.prefix = self.model_cls @abstractmethod - def __call__(self, event: ArrayEventContainer) -> None: + def __call__(self, event: SubarrayEventContainer) -> None: """Event-wise prediction for the EventSource-Loop. Fills the event.dl2.[name] container. @@ -364,7 +364,7 @@ class EnergyRegressor(SKLearnRegressionReconstructor): target = "true_energy" property = ReconstructionProperty.ENERGY - def __call__(self, event: ArrayEventContainer) -> None: + def __call__(self, event: SubarrayEventContainer) -> None: """ Apply model for a single event and fill result into the event container """ @@ -436,7 +436,7 @@ class ParticleClassifier(SKLearnClassificationReconstructor): property = ReconstructionProperty.PARTICLE_TYPE - def __call__(self, event: ArrayEventContainer) -> None: + def __call__(self, event: SubarrayEventContainer) -> None: for tel_id in event.trigger.tels_with_trigger: table = collect_features(event, tel_id, self.instrument_table) table = self.feature_generator(table, subarray=self.subarray) @@ -669,7 +669,7 @@ def _predict(self, key, table): return prediction, valid - def __call__(self, event: ArrayEventContainer) -> None: + def __call__(self, event: SubarrayEventContainer) -> None: """Event-wise prediction for the EventSource-Loop. Fills the event.dl2.tel[tel_id].disp[prefix] container diff --git a/ctapipe/reco/stereo_combination.py b/ctapipe/reco/stereo_combination.py index f36e1230891..4875b92aca7 100644 --- a/ctapipe/reco/stereo_combination.py +++ b/ctapipe/reco/stereo_combination.py @@ -11,10 +11,10 @@ from ctapipe.reco.reconstructor import ReconstructionProperty from ..containers import ( - ArrayEventContainer, ParticleClassificationContainer, ReconstructedEnergyContainer, ReconstructedGeometryContainer, + SubarrayEventContainer, ) from .utils import add_defaults_and_meta @@ -76,7 +76,7 @@ class StereoCombiner(Component): ).tag(config=True) @abstractmethod - def __call__(self, event: ArrayEventContainer) -> None: + def __call__(self, event: SubarrayEventContainer) -> None: """ Fill event container with stereo predictions """ @@ -263,7 +263,7 @@ def _combine_altaz(self, event): prefix=self.prefix, ) - def __call__(self, event: ArrayEventContainer) -> None: + def __call__(self, event: SubarrayEventContainer) -> None: """ Calculate the mean prediction for a single array event. """ diff --git a/ctapipe/reco/tests/test_stereo_combination.py b/ctapipe/reco/tests/test_stereo_combination.py index 49616fc63a8..d44b0bbe145 100644 --- a/ctapipe/reco/tests/test_stereo_combination.py +++ b/ctapipe/reco/tests/test_stereo_combination.py @@ -5,13 +5,13 @@ from numpy.testing import assert_allclose, assert_array_equal from ctapipe.containers import ( - ArrayDL2Container, - ArrayEventContainer, + DL2SubarrayContainer, HillasParametersContainer, ImageParametersContainer, ParticleClassificationContainer, ReconstructedEnergyContainer, ReconstructedGeometryContainer, + SubarrayEventContainer, ) from ctapipe.reco.reconstructor import ReconstructionProperty from ctapipe.reco.stereo_combination import StereoMeanCombiner @@ -152,7 +152,7 @@ def test_predict_mean_disp(mono_table): @pytest.mark.parametrize("weights", ["konrad", "intensity", "none"]) def test_mean_prediction_single_event(weights): - event = ArrayEventContainer() + event = SubarrayEventContainer() for tel_id, intensity in zip((25, 125, 130), (100, 200, 400)): event.dl1.tel[tel_id].parameters = ImageParametersContainer( @@ -163,7 +163,7 @@ def test_mean_prediction_single_event(weights): ) ) - event.dl2.tel[25] = ArrayDL2Container( + event.dl2.tel[25] = DL2SubarrayContainer( energy={ "dummy": ReconstructedEnergyContainer(energy=10 * u.GeV, is_valid=True) }, @@ -176,7 +176,7 @@ def test_mean_prediction_single_event(weights): ) }, ) - event.dl2.tel[125] = ArrayDL2Container( + event.dl2.tel[125] = DL2SubarrayContainer( energy={ "dummy": ReconstructedEnergyContainer(energy=20 * u.GeV, is_valid=True) }, @@ -189,7 +189,7 @@ def test_mean_prediction_single_event(weights): ) }, ) - event.dl2.tel[130] = ArrayDL2Container( + event.dl2.tel[130] = DL2SubarrayContainer( energy={ "dummy": ReconstructedEnergyContainer(energy=0.04 * u.TeV, is_valid=True) }, diff --git a/ctapipe/utils/tests/test_event_filter.py b/ctapipe/utils/tests/test_event_filter.py index 6a0fc5990e8..99b9fab2b9b 100644 --- a/ctapipe/utils/tests/test_event_filter.py +++ b/ctapipe/utils/tests/test_event_filter.py @@ -1,6 +1,6 @@ from traitlets.config import Config -from ctapipe.containers import ArrayEventContainer, EventType +from ctapipe.containers import EventType, SubarrayEventContainer def test_event_filter(): @@ -10,7 +10,7 @@ def test_event_filter(): allowed_types={EventType.SUBARRAY, EventType.FLATFIELD} ) - e = ArrayEventContainer() + e = SubarrayEventContainer() e.trigger.event_type = EventType.SUBARRAY assert event_filter(e) e.trigger.event_type = EventType.FLATFIELD @@ -25,7 +25,7 @@ def test_event_filter_none(): event_filter = EventTypeFilter(allowed_types=None) # all event types should pass - e = ArrayEventContainer() + e = SubarrayEventContainer() for value in EventType: e.trigger.event_type = value assert event_filter(e) @@ -53,7 +53,7 @@ def test_event_filter_config(): EventType.SINGLE_PE, } - e = ArrayEventContainer() + e = SubarrayEventContainer() e.trigger.event_type = EventType.DARK_PEDESTAL assert not event_filter(e) diff --git a/ctapipe/visualization/tests/test_mpl.py b/ctapipe/visualization/tests/test_mpl.py index 31867d5d158..6017c6f6027 100644 --- a/ctapipe/visualization/tests/test_mpl.py +++ b/ctapipe/visualization/tests/test_mpl.py @@ -157,11 +157,11 @@ def test_camera_display_multiple(prod5_lst_cam, tmp_path): def test_array_display(prod5_mst_nectarcam): """check that we can do basic array display functionality""" from ctapipe.containers import ( - ArrayEventContainer, CoreParametersContainer, DL1CameraContainer, DL1Container, ImageParametersContainer, + SubarrayEventContainer, ) from ctapipe.image import timing_parameters from ctapipe.visualization.mpl_array import ArrayDisplay @@ -179,7 +179,7 @@ def test_array_display(prod5_mst_nectarcam): # Create a fake event containing telescope-wise information about # the image directions projected on the ground - event = ArrayEventContainer() + event = SubarrayEventContainer() event.dl1 = DL1Container() event.dl1.tel = {1: DL1CameraContainer(), 2: DL1CameraContainer()} event.dl1.tel[1].parameters = ImageParametersContainer() diff --git a/test_plugin/ctapipe_test_plugin/__init__.py b/test_plugin/ctapipe_test_plugin/__init__.py index 060c8109d8a..0f747b7d1e8 100644 --- a/test_plugin/ctapipe_test_plugin/__init__.py +++ b/test_plugin/ctapipe_test_plugin/__init__.py @@ -19,7 +19,7 @@ TelescopeDescription, ) from ctapipe.io import DataLevel, EventSource -from ctapipe.io.datawriter import ArrayEventContainer +from ctapipe.io.datawriter import SubarrayEventContainer from ctapipe.reco import Reconstructor __all__ = [ @@ -92,7 +92,7 @@ def is_compatible(cls, path): def _generator(self): """Foo""" for i in range(10): - yield ArrayEventContainer(count=i) + yield SubarrayEventContainer(count=i) class PluginReconstructor(Reconstructor): @@ -102,6 +102,6 @@ class PluginReconstructor(Reconstructor): True, help="example traitlet to see that it is included in --help" ).tag(config=True) - def __call__(self, event: ArrayEventContainer): + def __call__(self, event: SubarrayEventContainer): """Foo""" event.dl2.geometry["PluginReconstructor"] = ReconstructedGeometryContainer()