From 77f53272666887c634964625fe7c1f89e958a162 Mon Sep 17 00:00:00 2001 From: juacrumar Date: Mon, 15 Apr 2024 17:51:49 +0200 Subject: [PATCH 1/2] use results instead of creating a new object when pdf errors are requested --- validphys2/src/validphys/covmats.py | 21 +++++++++++++++------ validphys2/src/validphys/results.py | 22 ++++++++++++++++++++-- 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/validphys2/src/validphys/covmats.py b/validphys2/src/validphys/covmats.py index 1a4f050eca..6a1f2ffdc0 100644 --- a/validphys2/src/validphys/covmats.py +++ b/validphys2/src/validphys/covmats.py @@ -1,8 +1,9 @@ """Module for handling logic and manipulation of covariance and correlation matrices on different levels of abstraction """ -import logging + import functools +import logging import numpy as np import pandas as pd @@ -670,7 +671,7 @@ def groups_corrmat(groups_covmat): @check_pdf_is_montecarlo_or_hessian -def pdferr_plus_covmat(dataset, pdf, covmat_t0_considered): +def pdferr_plus_covmat(results_without_covmat, pdf, covmat_t0_considered): """For a given `dataset`, returns the sum of the covariance matrix given by `covmat_t0_considered` and the PDF error: - If the PDF error_type is 'replicas', a covariance matrix is estimated from @@ -713,7 +714,7 @@ def pdferr_plus_covmat(dataset, pdf, covmat_t0_considered): >>> np.allclose(a == b) True """ - th = ThPredictionsResult.from_convolution(pdf, dataset) + _, th = results_without_covmat if pdf.error_type == 'replicas': pdf_cov = np.cov(th.error_members, rowvar=True) @@ -754,18 +755,26 @@ def reorder_thcovmat_as_expcovmat(fitthcovmat, data): return tmp.reindex(index=bb, columns=bb, level=0) -def pdferr_plus_dataset_inputs_covmat(data, pdf, dataset_inputs_covmat_t0_considered, fitthcovmat): +def pdferr_plus_dataset_inputs_covmat( + dataset_inputs_results_without_covmat, + data, + pdf, + dataset_inputs_covmat_t0_considered, + fitthcovmat, +): """Like `pdferr_plus_covmat` except for an experiment""" # do checks get performed here? if fitthcovmat is not None: # change ordering according to exp_covmat (so according to runcard order) return pdferr_plus_covmat( - data, + dataset_inputs_results_without_covmat, pdf, dataset_inputs_covmat_t0_considered + reorder_thcovmat_as_expcovmat(fitthcovmat, data).values, ) - return pdferr_plus_covmat(data, pdf, dataset_inputs_covmat_t0_considered) + return pdferr_plus_covmat( + dataset_inputs_results_without_covmat, pdf, dataset_inputs_covmat_t0_considered + ) def dataset_inputs_sqrt_covmat(dataset_inputs_covariance_matrix): diff --git a/validphys2/src/validphys/results.py b/validphys2/src/validphys/results.py index 6b45019215..6c9e15a4b9 100644 --- a/validphys2/src/validphys/results.py +++ b/validphys2/src/validphys/results.py @@ -3,6 +3,7 @@ Tools to obtain theory predictions and basic statistical estimators. """ + from __future__ import generator_stop from collections import OrderedDict, namedtuple @@ -536,7 +537,7 @@ def procs_corrmat(procs_covmat): return groups_corrmat(procs_covmat) -def results(dataset: (DataSetSpec), pdf: PDF, covariance_matrix, sqrt_covmat): +def results(dataset: DataSetSpec, pdf: PDF, covariance_matrix, sqrt_covmat): """Tuple of data and theory results for a single pdf. The data will have an associated covariance matrix, which can include a contribution from the theory covariance matrix which is constructed from scale variation. The inclusion of this covariance matrix by default is used @@ -553,7 +554,7 @@ def results(dataset: (DataSetSpec), pdf: PDF, covariance_matrix, sqrt_covmat): ) -def results_central(dataset: (DataSetSpec), pdf: PDF, covariance_matrix, sqrt_covmat): +def results_central(dataset: DataSetSpec, pdf: PDF, covariance_matrix, sqrt_covmat): """Same as :py:func:`results` but only calculates the prediction for replica0.""" return ( DataResult(dataset, covariance_matrix, sqrt_covmat), @@ -561,6 +562,18 @@ def results_central(dataset: (DataSetSpec), pdf: PDF, covariance_matrix, sqrt_co ) +def results_without_covmat(dataset: DataSetSpec, pdf: PDF): + """Return a results object with a diagonal covmat so that it can be used to generate + results-depending covmats elsewhere. Uses :py:funct:`results` under the hook""" + loaded_cd = dataset.load_commondata() + if isinstance(loaded_cd, list): + ndata = np.sum([cd.ndata for cd in loaded_cd]) + else: + ndata = loaded_cd.ndata + diag = np.eye(ndata) + return results(dataset, pdf, diag, diag) + + def results_with_theory_covmat(dataset, results, theory_covmat_dataset): """Returns results with a modfy ``DataResult`` such that the covariance matrix includes also the theory covmat. @@ -602,6 +615,11 @@ def results_with_scale_variations(results, theory_covmat_dataset): return (data_result, theory_error_result) +def dataset_inputs_results_without_covmat(data, pdf: PDF): + """Like `dataset_inputs_results` but skipping the computation of the covmat""" + return results_without_covmat(data, pdf) + + def dataset_inputs_results_central( data, pdf: PDF, dataset_inputs_covariance_matrix, dataset_inputs_sqrt_covmat ): From eeadf60bb57da6fc8d96aa3a0285cbd0385389f4 Mon Sep 17 00:00:00 2001 From: RoyStegeman Date: Mon, 15 Apr 2024 18:50:09 +0100 Subject: [PATCH 2/2] fix docstring + cleanup unused im ports --- validphys2/src/validphys/covmats.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/validphys2/src/validphys/covmats.py b/validphys2/src/validphys/covmats.py index 6a1f2ffdc0..dec41cd2ba 100644 --- a/validphys2/src/validphys/covmats.py +++ b/validphys2/src/validphys/covmats.py @@ -11,20 +11,15 @@ from reportengine import collect from reportengine.table import table -from validphys.calcutils import get_df_block, regularize_covmat +from validphys.calcutils import regularize_covmat from validphys.checks import ( check_cuts_considered, - check_data_cuts_match_theorycovmat, - check_dataset_cuts_match_theorycovmat, check_norm_threshold, check_pdf_is_montecarlo_or_hessian, check_speclabels_different, ) -from validphys.commondata import loaded_commondata_with_cuts from validphys.convolution import central_predictions -from validphys.core import PDF, DataGroupSpec, DataSetSpec from validphys.covmats_utils import construct_covmat, systematics_matrix -from validphys.results import ThPredictionsResult log = logging.getLogger(__name__) @@ -702,16 +697,19 @@ def pdferr_plus_covmat(results_without_covmat, pdf, covmat_t0_considered): `use_pdferr` makes this action be used for `covariance_matrix` >>> from validphys.api import API - >>> from import numpy as np + >>> import numpy as np >>> inp = { - 'dataset_input': {'dataset' : 'ATLASTTBARTOT'}, - 'theoryid': 53, - 'pdf': 'NNPDF31_nlo_as_0118', - 'use_cuts': 'nocuts' + 'dataset_input': { + 'dataset': 'ATLAS_TTBAR_8TEV_LJ_DIF_YTTBAR-NORM', + 'variant': 'legacy', + }, + 'theoryid': 700, + 'pdf': 'NNPDF40_nlo_as_01180', + 'use_cuts': 'internal', } >>> a = API.covariance_matrix(**inp, use_pdferr=True) >>> b = API.pdferr_plus_covmat(**inp) - >>> np.allclose(a == b) + >>> (a == b).all() True """ _, th = results_without_covmat