Skip to content

Commit

Permalink
Prevent some warnings for input files where we know that no reference…
Browse files Browse the repository at this point in the history
… metadata is available
  • Loading branch information
maxnoe committed Jul 24, 2024
1 parent a8ff9e2 commit 819dc87
Show file tree
Hide file tree
Showing 5 changed files with 24 additions and 12 deletions.
22 changes: 14 additions & 8 deletions src/ctapipe/core/provenance.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ def _get_current_or_start_activity(self):

return self.current_activity

def add_input_file(self, filename, role=None):
def add_input_file(self, filename, role=None, add_meta=True):
"""register an input to the current activity
Parameters
Expand All @@ -107,14 +107,14 @@ def add_input_file(self, filename, role=None):
role this input file satisfies (optional)
"""
activity = self._get_current_or_start_activity()
activity.register_input(abspath(filename), role=role)
activity.register_input(abspath(filename), role=role, add_meta=add_meta)
log.debug(
"added input entity '%s' to activity: '%s'",
filename,
activity.name,
)

def add_output_file(self, filename, role=None):
def add_output_file(self, filename, role=None, add_meta=True):
"""
register an output to the current activity
Expand All @@ -127,7 +127,7 @@ def add_output_file(self, filename, role=None):
"""
activity = self._get_current_or_start_activity()
activity.register_output(abspath(filename), role=role)
activity.register_output(abspath(filename), role=role, add_meta=add_meta)
log.debug(
"added output entity '%s' to activity: '%s'",
filename,
Expand Down Expand Up @@ -244,7 +244,7 @@ def start(self):
self._prov["start"].update(_sample_cpu_and_memory())
self._prov["system"].update(_get_system_provenance())

def register_input(self, url, role=None):
def register_input(self, url, role=None, add_meta=True):
"""
Add a URL of a file to the list of inputs (can be a filename or full
url, if no URL specifier is given, assume 'file://')
Expand All @@ -255,13 +255,16 @@ def register_input(self, url, role=None):
filename or url of input file
role: str
role name that this input satisfies
add_meta: bool
If true, try to load reference metadata from input file
and add to provenance.
"""
reference_meta = self._get_reference_meta(url=url)
reference_meta = self._get_reference_meta(url=url) if add_meta else None
self._prov["input"].append(
dict(url=url, role=role, reference_meta=reference_meta)
)

def register_output(self, url, role=None):
def register_output(self, url, role=None, add_meta=True):
"""
Add a URL of a file to the list of outputs (can be a filename or full
url, if no URL specifier is given, assume 'file://')
Expand All @@ -275,8 +278,11 @@ def register_output(self, url, role=None):
filename or url of output file
role: str
role name that this output satisfies
add_meta: bool
If true, try to load reference metadata from input file
and add to provenance.
"""
reference_meta = self._get_reference_meta(url=url)
reference_meta = self._get_reference_meta(url=url) if add_meta else None
self._prov["output"].append(
dict(url=url, role=role, reference_meta=reference_meta)
)
Expand Down
2 changes: 1 addition & 1 deletion src/ctapipe/core/tool.py
Original file line number Diff line number Diff line change
Expand Up @@ -283,7 +283,7 @@ def load_config_file(self, path: str | pathlib.Path) -> None:
# fall back to traitlets.config.Application's implementation
super().load_config_file(str(path))

Provenance().add_input_file(path, role="Tool Configuration")
Provenance().add_input_file(path, role="Tool Configuration", add_meta=False)

def update_logging_config(self):
"""Update the configuration of loggers."""
Expand Down
6 changes: 5 additions & 1 deletion src/ctapipe/io/simteleventsource.py
Original file line number Diff line number Diff line change
Expand Up @@ -345,8 +345,12 @@ def read_atmosphere_profile_from_simtel(

if isinstance(simtelfile, str | Path):
context_manager = SimTelFile(simtelfile)
# FIXME: simtel files currently do not have CTAO reference

Check warning on line 348 in src/ctapipe/io/simteleventsource.py

View check run for this annotation

CTAO-DPPS-SonarQube / ctapipe Sonarqube Results

src/ctapipe/io/simteleventsource.py#L348

Take the required action to fix the issue indicated by this "FIXME" comment.
# metadata, should be set to True once we store metadata
Provenance().add_input_file(
filename=simtelfile, role="ctapipe.atmosphere.AtmosphereDensityProfile"
filename=simtelfile,
role="ctapipe.atmosphere.AtmosphereDensityProfile",
add_meta=False,
)

else:
Expand Down
3 changes: 2 additions & 1 deletion src/ctapipe/reco/reconstructor.py
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,8 @@ def read(cls, path, parent=None, subarray=None, **kwargs):
for attr, value in kwargs.items():
setattr(instance, attr, value)

Provenance().add_input_file(path, role="reconstructor")
# FIXME: we currently don't store metadata in the joblib / pickle files, see #2603

Check warning on line 153 in src/ctapipe/reco/reconstructor.py

View check run for this annotation

CTAO-DPPS-SonarQube / ctapipe Sonarqube Results

src/ctapipe/reco/reconstructor.py#L153

Take the required action to fix the issue indicated by this "FIXME" comment.
Provenance().add_input_file(path, role="reconstructor", add_meta=False)
return instance


Expand Down
3 changes: 2 additions & 1 deletion src/ctapipe/reco/sklearn.py
Original file line number Diff line number Diff line change
Expand Up @@ -670,7 +670,8 @@ def read(cls, path, **kwargs):
f"{path} did not contain an instance of {cls}, got {instance}"
)

Provenance().add_input_file(path, role="ml-models")
# FIXME: we currently don't store metadata in the joblib / pickle files, see #2603

Check warning on line 673 in src/ctapipe/reco/sklearn.py

View check run for this annotation

CTAO-DPPS-SonarQube / ctapipe Sonarqube Results

src/ctapipe/reco/sklearn.py#L673

Take the required action to fix the issue indicated by this "FIXME" comment.
Provenance().add_input_file(path, role="ml-models", add_meta=False)
return instance

@lazyproperty
Expand Down

0 comments on commit 819dc87

Please sign in to comment.