From fe04ff28ec074a5a85c95f1d548ade0f2f897f5f Mon Sep 17 00:00:00 2001 From: oloapinivad Date: Mon, 22 Jan 2024 06:20:30 +0100 Subject: [PATCH 01/12] add Fraction --- ecmean/libs/units.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecmean/libs/units.py b/ecmean/libs/units.py index eacf4c8..f33b714 100644 --- a/ecmean/libs/units.py +++ b/ecmean/libs/units.py @@ -158,7 +158,7 @@ def units_extra_definition(): # needed to work with metpy 1.4.0 see # https://github.com/Unidata/MetPy/issues/2884 units._on_redefinition = 'ignore' - units.define('fraction = [] = frac') + units.define('Fraction = fraction = [] = frac') units.define('psu = 1e-3 frac') units.define('PSU = 1e-3 frac') units.define('million = 1e6 = M') From 713c9d1c0a4d16bf7fa87641d6cfc739a8a60dbd Mon Sep 17 00:00:00 2001 From: oloapinivad Date: Mon, 22 Jan 2024 06:29:41 +0100 Subject: [PATCH 02/12] fix --- ecmean/libs/units.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ecmean/libs/units.py b/ecmean/libs/units.py index f33b714..1d71708 100644 --- a/ecmean/libs/units.py +++ b/ecmean/libs/units.py @@ -158,7 +158,8 @@ def units_extra_definition(): # needed to work with metpy 1.4.0 see # https://github.com/Unidata/MetPy/issues/2884 units._on_redefinition = 'ignore' - units.define('Fraction = fraction = [] = frac') + units.define('fraction = [] = frac') + units.define('Fraction = [] = frac') units.define('psu = 1e-3 frac') units.define('PSU = 1e-3 frac') units.define('million = 1e6 = M') From d7abf95144e2496528d7025c7a6e05fb815c3796 Mon Sep 17 00:00:00 2001 From: oloapinivad Date: Mon, 22 Jan 2024 07:26:22 +0100 Subject: [PATCH 03/12] adding possibility for plotting nan and new colorscale --- ecmean/global_mean.py | 3 ++- ecmean/libs/plotting.py | 27 +++++++++++++++++++-------- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/ecmean/global_mean.py b/ecmean/global_mean.py index b915ae0..cd17c1b 100755 --- a/ecmean/global_mean.py +++ b/ecmean/global_mean.py @@ -324,7 +324,8 @@ def global_mean(exp, year1, year2, f'global_mean_{diag.expname}_{diag.modelname}_r1i1p1f1_{diag.year1}_{diag.year2}.pdf' loggy.info('Figure file is: %s', mapfile) - heatmap_comparison_gm(data_table, mean_table, std_table, diag, mapfile) + heatmap_comparison_gm(data_table, mean_table, std_table, + diag, mapfile, add_nan=False) # Print appending one line to table (for tuning) if diag.ftable: diff --git a/ecmean/libs/plotting.py b/ecmean/libs/plotting.py index e9be213..a966a62 100644 --- a/ecmean/libs/plotting.py +++ b/ecmean/libs/plotting.py @@ -12,10 +12,11 @@ import matplotlib.pyplot as plt from matplotlib.colors import ListedColormap import seaborn as sns +import numpy as np -def heatmap_comparison_pi(relative_table, diag, filemap): +def heatmap_comparison_pi(relative_table, diag, filemap, size_model = 14): """Function to produce a heatmap - seaborn based - for Performance Indices based on CMIP6 ratio""" @@ -42,7 +43,8 @@ def heatmap_comparison_pi(relative_table, diag, filemap): chart = sns.heatmap(myfield, norm=divnorm, cmap=pal, cbar_kws={"ticks": tictoc, 'label': title}, ax=axs, annot=True, linewidth=0.5, fmt='.2f', - annot_kws={'fontsize': 14, 'fontweight': 'bold'}) + annot_kws={'fontsize': size_model, 'fontweight': 'bold'}) + chart = chart.set_facecolor('whitesmoke') axs.set_title(f'{title} {diag.modelname} {diag.expname} {diag.year1} {diag.year2}', fontsize=25) axs.vlines(list(range(sss, tot + sss, sss)), ymin=-1, ymax=len(myfield.index), colors='k') @@ -60,36 +62,45 @@ def heatmap_comparison_pi(relative_table, diag, filemap): plt.close() -def heatmap_comparison_gm(data_table, mean_table, std_table, diag, filemap): +def heatmap_comparison_gm(data_table, mean_table, std_table, diag, filemap, add_nan = True, + size_model = 14, size_obs = 8): """Function to produce a heatmap - seaborn based - for Global Mean based on season-averaged standard deviation ratio""" # define array ratio = (data_table - mean_table) / std_table - mask = ratio[('ALL', 'Global')].notna() + if add_nan: + mask = data_table[('ALL', 'Global')].notna() + else: + mask = ratio[('ALL', 'Global')].notna() clean = ratio[mask] # for dimension of plots xfig = len(clean.columns) yfig = len(clean.index) - _, axs = plt.subplots(1, 1, sharey=True, tight_layout=True, figsize=(xfig+5, yfig+2)) + fig, axs = plt.subplots(1, 1, sharey=True, tight_layout=True, figsize=(xfig+5, yfig+2)) title = 'GLOBAL MEAN' # set color range and palette thr = 10 tictoc = range(-thr, thr + 1) - pal = ListedColormap(sns.color_palette("seismic", n_colors=21)) + pal = ListedColormap(sns.color_palette("vlag", n_colors=21)) tot = len(clean.columns) sss = len(set([tup[1] for tup in clean.columns])) chart = sns.heatmap(clean, annot=data_table[mask], vmin=-thr - 0.5, vmax=thr + 0.5, center=0, - annot_kws={'va': 'bottom', 'fontsize': 14}, + annot_kws={'va': 'bottom', 'fontsize': size_model}, cbar_kws={'ticks': tictoc, 'label': 'Model Bias \n (standard deviation of interannual variability from observations)'}, fmt='.2f', cmap=pal) + if add_nan: + chart = sns.heatmap(np.where(clean.isna(), 0, np.nan), annot=data_table[mask], fmt='.2f', + vmin=-thr - 0.5, vmax=thr + 0.5, center=0, + annot_kws={'va': 'bottom', 'fontsize': size_model, 'color':'dimgrey'}, cbar=False, + cmap=ListedColormap(['none'])) chart = sns.heatmap(clean, annot=mean_table[mask], vmin=-thr - 0.5, vmax=thr + 0.5, center=0, - annot_kws={'va': 'top', 'fontsize': 8, 'fontstyle': 'italic'}, + annot_kws={'va': 'top', 'fontsize': size_obs, 'fontstyle': 'italic'}, fmt='.2f', cmap=pal, cbar=False) chart = chart.set_facecolor('whitesmoke') From ef1938e6dc488049920331d9b45ae10360af03f7 Mon Sep 17 00:00:00 2001 From: oloapinivad Date: Mon, 22 Jan 2024 07:27:37 +0100 Subject: [PATCH 04/12] test py12 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0647399..e0918f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ authors = [ ] description = "ECmean4 Global Climate Model lightweight evaluation tool" readme = "README.md" -requires-python = ">=3.9,<3.12" +requires-python = ">=3.9" classifiers = [ "Programming Language :: Python :: 3", "Intended Audience :: Science/Research", From a9838fe18c8965f2bd6bd741d54a075dc487798f Mon Sep 17 00:00:00 2001 From: oloapinivad Date: Mon, 22 Jan 2024 07:28:03 +0100 Subject: [PATCH 05/12] test --- .github/workflows/mambatest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mambatest.yml b/.github/workflows/mambatest.yml index 4f0ab7f..5f4de90 100644 --- a/.github/workflows/mambatest.yml +++ b/.github/workflows/mambatest.yml @@ -21,7 +21,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] #python-version: ["3.11"] defaults: run: From 421a7807815729f479945404adfc50c2bfe4c1c5 Mon Sep 17 00:00:00 2001 From: oloapinivad Date: Mon, 22 Jan 2024 07:30:09 +0100 Subject: [PATCH 06/12] env --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 1b4d307..e90348e 100644 --- a/environment.yml +++ b/environment.yml @@ -7,7 +7,7 @@ name : ecmean channels: - conda-forge dependencies: - - python>=3.9,<3.12 + - python>=3.9 - libnetcdf - xesmf - eccodes From f743d742acfdc9ad990e69d93b21d31b5a4ed389 Mon Sep 17 00:00:00 2001 From: oloapinivad Date: Mon, 22 Jan 2024 07:55:56 +0100 Subject: [PATCH 07/12] roll back --- .github/workflows/mambatest.yml | 2 +- environment.yml | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/mambatest.yml b/.github/workflows/mambatest.yml index 5f4de90..4f0ab7f 100644 --- a/.github/workflows/mambatest.yml +++ b/.github/workflows/mambatest.yml @@ -21,7 +21,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11"] #python-version: ["3.11"] defaults: run: diff --git a/environment.yml b/environment.yml index e90348e..1b4d307 100644 --- a/environment.yml +++ b/environment.yml @@ -7,7 +7,7 @@ name : ecmean channels: - conda-forge dependencies: - - python>=3.9 + - python>=3.9,<3.12 - libnetcdf - xesmf - eccodes diff --git a/pyproject.toml b/pyproject.toml index e0918f7..0647399 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ authors = [ ] description = "ECmean4 Global Climate Model lightweight evaluation tool" readme = "README.md" -requires-python = ">=3.9" +requires-python = ">=3.9,<3.12" classifiers = [ "Programming Language :: Python :: 3", "Intended Audience :: Science/Research", From 6665ed5da8fffdfbed0be6744b9618d0e6beb78d Mon Sep 17 00:00:00 2001 From: oloapinivad Date: Mon, 22 Jan 2024 10:01:41 +0100 Subject: [PATCH 08/12] better add nan and some nice logging --- ecmean/global_mean.py | 14 +++++++++----- ecmean/libs/plotting.py | 6 ++++-- ecmean/performance_indices.py | 19 ++++++++----------- 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/ecmean/global_mean.py b/ecmean/global_mean.py index cd17c1b..0abbe4f 100755 --- a/ecmean/global_mean.py +++ b/ecmean/global_mean.py @@ -139,7 +139,7 @@ def gm_worker(util, ref, face, diag, varmean, vartrend, varlist): if len(avg) == len(diag.years_joined): trend[season][region] = np.polyfit(diag.years_joined, avg, 1)[0] if season == 'ALL' and region == 'Global': - print('Average:', var, season, region, result[season][region]) + logging.info('Average:', var, season, region, result[season][region]) # nested dictionary, to be redifend as a dict to remove lambdas varmean[var] = result @@ -151,6 +151,7 @@ def global_mean(exp, year1, year2, loglevel='WARNING', numproc=1, interface=None, model=None, ensemble='r1i1p1f1', + add_nan=True, silent=None, trend=None, line=None, outputdir=None, xdataset=None): """The main ECmean4 global mean function @@ -164,6 +165,7 @@ def global_mean(exp, year1, year2, :param interface: interface file to be used, optional (default as specifified in config file) :param model: model to be analyzed, optional (default as specifified in config file) :param ensemble: ensemble member to be analyzed, optional (default as 'r1i1p1f1') + :param add_nan: add to the final plots also fields which cannot be compared against observations :param silent: do not print anything to std output, optional :param trend: compute yearly trends, optional :param line: appends also single line to a table, optional @@ -207,6 +209,7 @@ def global_mean(exp, year1, year2, # get file info inifiles = get_inifiles(face, diag) + # add missing unit definition units_extra_definition() @@ -240,7 +243,7 @@ def global_mean(exp, year1, year2, toc = time() # evaluate tic-toc time of execution - loggy.warning('Analysis done in {:.4f} seconds'.format(toc - tic)) + loggy.info('Analysis done in {:.4f} seconds'.format(toc - tic)) tic = time() @@ -324,8 +327,8 @@ def global_mean(exp, year1, year2, f'global_mean_{diag.expname}_{diag.modelname}_r1i1p1f1_{diag.year1}_{diag.year2}.pdf' loggy.info('Figure file is: %s', mapfile) - heatmap_comparison_gm(data_table, mean_table, std_table, - diag, mapfile, add_nan=False) + heatmap_comparison_gm(data_table, mean_table, std_table, + diag, mapfile, add_nan=add_nan) # Print appending one line to table (for tuning) if diag.ftable: @@ -334,7 +337,8 @@ def global_mean(exp, year1, year2, toc = time() # evaluate tic-toc time of postprocessing - loggy.warning(f"Postproc done in {toc - tic:.4f} seconds") + loggy.info(f"Postproc done in {toc - tic:.4f} seconds") + print('ECmean4 Global Mean succesfully computed!') diff --git a/ecmean/libs/plotting.py b/ecmean/libs/plotting.py index a966a62..77507c8 100644 --- a/ecmean/libs/plotting.py +++ b/ecmean/libs/plotting.py @@ -91,11 +91,13 @@ def heatmap_comparison_gm(data_table, mean_table, std_table, diag, filemap, add_ chart = sns.heatmap(clean, annot=data_table[mask], vmin=-thr - 0.5, vmax=thr + 0.5, center=0, annot_kws={'va': 'bottom', 'fontsize': size_model}, - cbar_kws={'ticks': tictoc, + cbar_kws={'ticks': tictoc, "shrink": .5, 'label': 'Model Bias \n (standard deviation of interannual variability from observations)'}, fmt='.2f', cmap=pal) if add_nan: - chart = sns.heatmap(np.where(clean.isna(), 0, np.nan), annot=data_table[mask], fmt='.2f', + empty = np.where(clean.isna(), 0, np.nan) + empty = np.where(data_table[mask]==0, np.nan, empty) + chart = sns.heatmap(empty, annot=data_table[mask], fmt='.2f', vmin=-thr - 0.5, vmax=thr + 0.5, center=0, annot_kws={'va': 'bottom', 'fontsize': size_model, 'color':'dimgrey'}, cbar=False, cmap=ListedColormap(['none'])) diff --git a/ecmean/performance_indices.py b/ecmean/performance_indices.py index abe14e6..e5c6367 100755 --- a/ecmean/performance_indices.py +++ b/ecmean/performance_indices.py @@ -99,7 +99,7 @@ def pi_worker(util, piclim, face, diag, field_3d, varstat, varlist): # mean over time and fixing of the units for season in diag.seasons: - loggy.info(season) + loggy.debug(season) # copy of the full field tmean = outfield.copy(deep=True) @@ -202,7 +202,7 @@ def pi_worker(util, piclim, face, diag, field_3d, varstat, varlist): # diagnostic if region == 'Global': - print('PI for', region, season, var, result[season][region]) + logging.info('PI for', region, season, var, result[season][region]) # nested dictionary, to be redifend as a dict to remove lambdas varstat[var] = result @@ -291,7 +291,7 @@ def performance_indices(exp, year1, year2, processes = [] toc = time() - loggy.warning('Preproc in {:.4f} seconds'.format(toc - tic)) + loggy.info('Preproc in {:.4f} seconds'.format(toc - tic)) tic = time() @@ -313,7 +313,7 @@ def performance_indices(exp, year1, year2, toc = time() # evaluate tic-toc time of execution - loggy.warning('Done in {:.4f} seconds'.format(toc - tic) + ' with ' + str(diag.numproc) + ' processors') + loggy.info('Done in {:.4f} seconds'.format(toc - tic) + ' with ' + str(diag.numproc) + ' processors') tic = time() @@ -349,7 +349,7 @@ def performance_indices(exp, year1, year2, # convert output dictionary to pandas dataframe data_table = dict_to_dataframe(plotted) - loggy.info(data_table) + loggy.debug(data_table) # relative pi with re-ordering of rows @@ -361,7 +361,7 @@ def performance_indices(exp, year1, year2, # reordering columns lll = [(x, y) for x in diag.seasons for y in diag.regions] cmip6_table = cmip6_table[lll] - loggy.info(cmip6_table) + loggy.debug(cmip6_table) # call the heatmap routine for a plot @@ -372,7 +372,8 @@ def performance_indices(exp, year1, year2, toc = time() # evaluate tic-toc time of postprocessing - loggy.warning('Postproc done in {:.4f} seconds'.format(toc - tic)) + loggy.info('Postproc done in {:.4f} seconds'.format(toc - tic)) + print('ECmean4 Performance Indices succesfully computed!') def pi_entry_point(): @@ -391,7 +392,3 @@ def pi_entry_point(): model=args.model, ensemble=args.ensemble, outputdir=args.outputdir) - -#if __name__ == '__main__': -# -# sys.exit(pi_entry_point()) From 63768a668acea686d6cf47af071583c0fdeeb7b1 Mon Sep 17 00:00:00 2001 From: oloapinivad Date: Mon, 22 Jan 2024 11:37:57 +0100 Subject: [PATCH 09/12] various updates and new version --- CHANGELOG.md | 6 ++++++ ecmean/__init__.py | 2 +- ecmean/global_mean.py | 10 ++++------ ecmean/libs/areas.py | 2 +- ecmean/libs/diagnostic.py | 1 + ecmean/libs/files.py | 8 ++++---- ecmean/libs/parser.py | 2 ++ ecmean/libs/plotting.py | 6 +++--- ecmean/libs/support.py | 2 +- ecmean/libs/units.py | 18 +++++++++--------- 10 files changed, 32 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c4c2e6..495409e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) Unreleased is the current development version, which is currently lying in `main` branch. +## [v0.1.7] + +- global_mean can plot also values where observation are missing through the `--addnan` option (#95) +- Verbosity level reduced in both global mean and performance indices (#95) +- New colorscale `vlag` for global_mean (#95) + ## [v0.1.6] - Remove support for python 3.8 diff --git a/ecmean/__init__.py b/ecmean/__init__.py index 50e24ef..a028bfc 100644 --- a/ecmean/__init__.py +++ b/ecmean/__init__.py @@ -1,6 +1,6 @@ """ECmean4 module""" -__version__ = '0.1.6' +__version__ = '0.1.7' # functions to be accessible everywhere from ecmean.libs.diagnostic import Diagnostic diff --git a/ecmean/global_mean.py b/ecmean/global_mean.py index 0abbe4f..28a8988 100755 --- a/ecmean/global_mean.py +++ b/ecmean/global_mean.py @@ -139,7 +139,7 @@ def gm_worker(util, ref, face, diag, varmean, vartrend, varlist): if len(avg) == len(diag.years_joined): trend[season][region] = np.polyfit(diag.years_joined, avg, 1)[0] if season == 'ALL' and region == 'Global': - logging.info('Average:', var, season, region, result[season][region]) + loggy.info('Average: %s %s %s %s', var, season, region, result[season][region]) # nested dictionary, to be redifend as a dict to remove lambdas varmean[var] = result @@ -151,7 +151,7 @@ def global_mean(exp, year1, year2, loglevel='WARNING', numproc=1, interface=None, model=None, ensemble='r1i1p1f1', - add_nan=True, + addnan=False, silent=None, trend=None, line=None, outputdir=None, xdataset=None): """The main ECmean4 global mean function @@ -328,7 +328,7 @@ def global_mean(exp, year1, year2, loggy.info('Figure file is: %s', mapfile) heatmap_comparison_gm(data_table, mean_table, std_table, - diag, mapfile, add_nan=add_nan) + diag, mapfile, addnan=diag.addnan) # Print appending one line to table (for tuning) if diag.ftable: @@ -356,8 +356,6 @@ def gm_entry_point(): loglevel=args.loglevel, interface=args.interface, config=args.config, model=args.model, ensemble=args.ensemble, + addnan=args.addnan, outputdir=args.outputdir) -#if __name__ == "__main__": -# -# gm_entry_point() diff --git a/ecmean/libs/areas.py b/ecmean/libs/areas.py index 0a451c3..41fd686 100644 --- a/ecmean/libs/areas.py +++ b/ecmean/libs/areas.py @@ -195,7 +195,7 @@ def area_cell(xfield, gridtype=None, formula='triangles'): outfield = xr.DataArray(area, dims=area_dims, coords=xfield.coords, name='area') # check the total area - loggy.info('Total Earth Surface: %s Km2', + loggy.debug('Total Earth Surface: %s Km2', str(outfield.sum().values / 10**6)) return outfield diff --git a/ecmean/libs/diagnostic.py b/ecmean/libs/diagnostic.py index 888ff4b..62c684f 100644 --- a/ecmean/libs/diagnostic.py +++ b/ecmean/libs/diagnostic.py @@ -41,6 +41,7 @@ def __init__(self, args): self.interface = getattr(args, 'interface', '') self.resolution = getattr(args, 'resolution', '') self.ensemble = getattr(args, 'ensemble', 'r1i1p1f1') + self.addnan = getattr(args, 'addnan', False) self.funcname = args.funcname.split(".")[1] self.version = __version__ if self.year1 == self.year2: diff --git a/ecmean/libs/files.py b/ecmean/libs/files.py index 0b67dac..c5b9ec7 100644 --- a/ecmean/libs/files.py +++ b/ecmean/libs/files.py @@ -85,7 +85,7 @@ def var_is_there(flist, var, face): if not varunit: varunit = units_avail.get(var_req[0]) if len(var_req) > 1: - loggy.info('%s is a derived var, assuming unit ' + loggy.debug('%s is a derived var, assuming unit ' 'as the first of its term', var) else: @@ -207,7 +207,7 @@ def _filter_filename_by_year(template, filenames, year): if not filternames and len(filenames) > 0: loggy.warning('Data for year %s has not been found!', str(year)) - loggy.info('Filtered filenames: %s', filternames) + loggy.debug('Filtered filenames: %s', filternames) return filternames @@ -230,7 +230,7 @@ def _create_filepath(cmorname, face, diag): Path(face['model']['basedir']) / \ Path(face['filetype'][filetype]['dir']) / \ Path(face['filetype'][filetype]['filename']) - loggy.info('Filepath: %s', filepath) + loggy.debug('Filepath: %s', filepath) return filepath @@ -265,7 +265,7 @@ def make_input_filename(cmorname, face, diag): filename1 = filename1 + fname filename = filename + filename1 filename = list(dict.fromkeys(filename)) # Filter unique ones - loggy.info("Filenames: %s", filename) + loggy.debug("Filenames: %s", filename) return filename diff --git a/ecmean/libs/parser.py b/ecmean/libs/parser.py index d3454c3..a5f9bb8 100644 --- a/ecmean/libs/parser.py +++ b/ecmean/libs/parser.py @@ -34,6 +34,8 @@ def parse_arguments(args, script): help='variant label (ripf number for cmor)') parser.add_argument('-s', '--silent', action='store_true', help='do not print anything to std output') + parser.add_argument('--addnan', action='store_true', + help='provide figures also where observations are missing') parser.add_argument('-v', '--loglevel', type=str, default='WARNING', help='define the level of logging.') parser.add_argument('-o', '--outputdir', type=str, diff --git a/ecmean/libs/plotting.py b/ecmean/libs/plotting.py index 77507c8..b136d3d 100644 --- a/ecmean/libs/plotting.py +++ b/ecmean/libs/plotting.py @@ -62,14 +62,14 @@ def heatmap_comparison_pi(relative_table, diag, filemap, size_model = 14): plt.close() -def heatmap_comparison_gm(data_table, mean_table, std_table, diag, filemap, add_nan = True, +def heatmap_comparison_gm(data_table, mean_table, std_table, diag, filemap, addnan = True, size_model = 14, size_obs = 8): """Function to produce a heatmap - seaborn based - for Global Mean based on season-averaged standard deviation ratio""" # define array ratio = (data_table - mean_table) / std_table - if add_nan: + if addnan: mask = data_table[('ALL', 'Global')].notna() else: mask = ratio[('ALL', 'Global')].notna() @@ -94,7 +94,7 @@ def heatmap_comparison_gm(data_table, mean_table, std_table, diag, filemap, add_ cbar_kws={'ticks': tictoc, "shrink": .5, 'label': 'Model Bias \n (standard deviation of interannual variability from observations)'}, fmt='.2f', cmap=pal) - if add_nan: + if addnan: empty = np.where(clean.isna(), 0, np.nan) empty = np.where(data_table[mask]==0, np.nan, empty) chart = sns.heatmap(empty, annot=data_table[mask], fmt='.2f', diff --git a/ecmean/libs/support.py b/ecmean/libs/support.py index 57f3cf3..f9608b3 100644 --- a/ecmean/libs/support.py +++ b/ecmean/libs/support.py @@ -31,7 +31,7 @@ def __init__(self, component, atmdict, ocedict, areas=True, remap=False, targetg """Class for masks, areas and interpolation (xESMF-based) for both atmospheric and oceanic component""" - loggy.info('Running with xesmf version %s', xe.__version__) + loggy.debug('Running with xesmf version %s', xe.__version__) # define the basics self.atmareafile = inifiles_priority(atmdict) diff --git a/ecmean/libs/units.py b/ecmean/libs/units.py index 1d71708..f785814 100644 --- a/ecmean/libs/units.py +++ b/ecmean/libs/units.py @@ -54,7 +54,7 @@ def __init__(self, var=None, org_units=None, tgt_units=None, if face and clim: self.init_ecmean(clim, face) - loggy.info(vars(self)) + loggy.debug(vars(self)) # if units are defined and convert called if convert and self.org_units and self.tgt_units: @@ -101,12 +101,12 @@ def units_converter(self): It will not work if BOTH factor and offset are required""" units_relation = (self.org_units / self.tgt_units).to_base_units() - loggy.info('Original vs Target units relation is %s', units_relation) + loggy.debug('Original vs Target units relation is %s', units_relation) if units_relation.units == units('dimensionless'): if units_relation.magnitude != 1: - loggy.info('Unit conversion required...') + loggy.debug('Unit conversion required...') offset_standard = 0 * self.org_units offset = offset_standard.to(self.tgt_units).magnitude if offset == 0: @@ -119,16 +119,16 @@ def units_converter(self): factor = 1. elif units_relation.units == units('kg / m^3'): - loggy.info("Assuming this as a water flux! Am I correct?") - loggy.info("Dividing by water density...") + loggy.debug("Assuming this as a water flux! Am I correct?") + loggy.debug("Dividing by water density...") density_water = units('kg / m^3') * 1000 offset = 0. factor_standard = 1 * self.org_units factor = (factor_standard / density_water).to(self.tgt_units).magnitude elif units_relation.units == units('s'): - loggy.info("Assuming this is a cumulated flux...") - loggy.info("Dividing by cumulation time (expressed in seconds)...") + loggy.debug("Assuming this is a cumulated flux...") + loggy.debug("Dividing by cumulation time (expressed in seconds)...") if self.cumulation_time: cumtime = units('s') * self.cumulation_time offset = 0. @@ -145,8 +145,8 @@ def units_converter(self): if self.org_direction != self.tgt_direction: factor = -1. * factor - loggy.info('Offset is %s', str(offset)) - loggy.info('Factor is %s', str(factor)) + loggy.debug('Offset is %s', str(offset)) + loggy.debug('Factor is %s', str(factor)) return offset, factor From dc7732917411b7ad898345b0eea4ac994174a3cb Mon Sep 17 00:00:00 2001 From: oloapinivad Date: Mon, 22 Jan 2024 11:44:49 +0100 Subject: [PATCH 10/12] add doc --- docs/sphinx/source/globalmean.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/sphinx/source/globalmean.rst b/docs/sphinx/source/globalmean.rst index 4612982..94a4359 100644 --- a/docs/sphinx/source/globalmean.rst +++ b/docs/sphinx/source/globalmean.rst @@ -70,6 +70,10 @@ Optional Arguments Specify the model name, overriding the configuration specified in ``config.yml``. +.. option:: --addnan + + Activate to plot also in the heatmap also fields which does not have a comparison against observations. Default is False. + .. option:: -v LOGLEVEL, --loglevel LOGLEVEL Define the level of logging. The default is 'warning'. From 49fc13886fd181e0d805da85e906451a3962cb96 Mon Sep 17 00:00:00 2001 From: oloapinivad Date: Mon, 22 Jan 2024 11:46:01 +0100 Subject: [PATCH 11/12] extend test --- tests/test_global_mean.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_global_mean.py b/tests/test_global_mean.py index 5c21bf8..b1a8f78 100644 --- a/tests/test_global_mean.py +++ b/tests/test_global_mean.py @@ -61,7 +61,7 @@ def test_global_mean_amip(): if os.path.isfile(file1): os.remove(file1) global_mean(exp='amip', year1=1990, year2=1990, numproc=1, config='tests/config.yml', - line=True) + line=True, addnan=True) data1 = load_gm_txt_files(file1) data2 = load_gm_txt_files(file2) From 1cf46d26b378c7f105eb0f8f5b6104ed97ee3594 Mon Sep 17 00:00:00 2001 From: oloapinivad Date: Mon, 22 Jan 2024 12:04:35 +0100 Subject: [PATCH 12/12] store also basic observations --- ecmean/global_mean.py | 4 ++++ ecmean/libs/plotting.py | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/ecmean/global_mean.py b/ecmean/global_mean.py index 28a8988..191bdff 100755 --- a/ecmean/global_mean.py +++ b/ecmean/global_mean.py @@ -264,11 +264,15 @@ def global_mean(exp, year1, year2, # extract from yaml table for obs mean and standard deviation mmm = init_mydict(diag.seasons, diag.regions) sss = init_mydict(diag.seasons, diag.regions) + # if we have all the obs/std available if isinstance(gamma['obs'], dict): for season in diag.seasons: for region in diag.regions: mmm[season][region] = gamma['obs'][season][region]['mean'] sss[season][region] = gamma['obs'][season][region]['std'] + # if only global observation is available + else: + mmm['ALL']['Global'] = gamma['obs'] obsmean[gamma['longname']] = mmm obsstd[gamma['longname']] = sss diff --git a/ecmean/libs/plotting.py b/ecmean/libs/plotting.py index b136d3d..1004d87 100644 --- a/ecmean/libs/plotting.py +++ b/ecmean/libs/plotting.py @@ -104,6 +104,12 @@ def heatmap_comparison_gm(data_table, mean_table, std_table, diag, filemap, addn chart = sns.heatmap(clean, annot=mean_table[mask], vmin=-thr - 0.5, vmax=thr + 0.5, center=0, annot_kws={'va': 'top', 'fontsize': size_obs, 'fontstyle': 'italic'}, fmt='.2f', cmap=pal, cbar=False) + if addnan: + empty = np.where(clean.isna(), 0, np.nan) + empty = np.where(mean_table[mask].isna(), np.nan, empty) + chart = sns.heatmap(empty, annot=mean_table[mask], vmin=-thr - 0.5, vmax=thr + 0.5, center=0, + annot_kws={'va': 'top', 'fontsize': size_obs, 'fontstyle': 'italic', 'color':'dimgrey'}, + fmt='.2f', cmap=ListedColormap(['none']), cbar=False) chart = chart.set_facecolor('whitesmoke') axs.set_title(f'{title} {diag.modelname} {diag.expname} {diag.year1} {diag.year2}', fontsize=25)