Merge pull request #95 from oloapinivad/units-and-al

Units and other updates

CHANGELOG.md

@@ -7,6 +7,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 Unreleased is the current development version, which is currently lying in `main` branch.
 
+## [v0.1.7]
+
+- global_mean can plot also values where observation are missing through the `--addnan` option (#95)
+- Verbosity level reduced in both global mean and performance indices (#95)
+- New colorscale `vlag` for global_mean (#95)
+
 ## [v0.1.6]
 
 - Remove support for python 3.8

docs/sphinx/source/globalmean.rst

@@ -70,6 +70,10 @@ Optional Arguments
 
    Specify the model name, overriding the configuration specified in ``config.yml``.
 
+.. option:: --addnan
+
+   Activate to plot also in the heatmap also fields which does not have a comparison against observations. Default is False.
+
 .. option:: -v LOGLEVEL, --loglevel LOGLEVEL
 
    Define the level of logging. The default is 'warning'.

ecmean/__init__.py

@@ -1,6 +1,6 @@
 """ECmean4 module"""
 
-__version__ = '0.1.6'
+__version__ = '0.1.7'
 
 # functions to be accessible everywhere
 from ecmean.libs.diagnostic import Diagnostic

ecmean/global_mean.py

@@ -139,7 +139,7 @@ def gm_worker(util, ref, face, diag, varmean, vartrend, varlist):
                             if len(avg) == len(diag.years_joined):
                                 trend[season][region] = np.polyfit(diag.years_joined, avg, 1)[0]
                         if season == 'ALL' and region == 'Global':
-                            print('Average:', var, season, region, result[season][region])
+                            loggy.info('Average: %s %s %s %s', var, season, region, result[season][region])
 
         # nested dictionary, to be redifend as a dict to remove lambdas
         varmean[var] = result
@@ -151,6 +151,7 @@ def global_mean(exp, year1, year2,
                 loglevel='WARNING',
                 numproc=1,
                 interface=None, model=None, ensemble='r1i1p1f1',
+                addnan=False,
                 silent=None, trend=None, line=None,
                 outputdir=None, xdataset=None):
     """The main ECmean4 global mean function
@@ -164,6 +165,7 @@ def global_mean(exp, year1, year2,
     :param interface: interface file to be used, optional (default as specifified in config file)
     :param model: model to be analyzed, optional (default as specifified in config file)
     :param ensemble: ensemble member to be analyzed, optional (default as 'r1i1p1f1')
+    :param add_nan: add to the final plots also fields which cannot be compared against observations
     :param silent: do not print anything to std output, optional
     :param trend: compute yearly trends, optional
     :param line: appends also single line to a table, optional
@@ -207,6 +209,7 @@ def global_mean(exp, year1, year2,
     # get file info
     inifiles = get_inifiles(face, diag)
 
+
     # add missing unit definition
     units_extra_definition()
 
@@ -240,7 +243,7 @@ def global_mean(exp, year1, year2,
     toc = time()
 
     # evaluate tic-toc time  of execution
-    loggy.warning('Analysis done in {:.4f} seconds'.format(toc - tic))
+    loggy.info('Analysis done in {:.4f} seconds'.format(toc - tic))
 
     tic = time()
 
@@ -261,11 +264,15 @@ def global_mean(exp, year1, year2,
         # extract from yaml table for obs mean and standard deviation
         mmm = init_mydict(diag.seasons, diag.regions)
         sss = init_mydict(diag.seasons, diag.regions)
+        # if we have all the obs/std available
         if isinstance(gamma['obs'], dict):
             for season in diag.seasons:
                 for region in diag.regions:
                     mmm[season][region] = gamma['obs'][season][region]['mean']
                     sss[season][region] = gamma['obs'][season][region]['std']
+        # if only global observation is available
+        else:
+            mmm['ALL']['Global'] = gamma['obs']
         obsmean[gamma['longname']] = mmm
         obsstd[gamma['longname']] = sss
 
@@ -324,7 +331,8 @@ def global_mean(exp, year1, year2,
         f'global_mean_{diag.expname}_{diag.modelname}_r1i1p1f1_{diag.year1}_{diag.year2}.pdf'
     loggy.info('Figure file is: %s', mapfile)
 
-    heatmap_comparison_gm(data_table, mean_table, std_table, diag, mapfile)
+    heatmap_comparison_gm(data_table, mean_table, std_table,
+                          diag, mapfile, addnan=diag.addnan)
 
     # Print appending one line to table (for tuning)
     if diag.ftable:
@@ -333,7 +341,8 @@ def global_mean(exp, year1, year2,
 
     toc = time()
     # evaluate tic-toc time of postprocessing
-    loggy.warning(f"Postproc done in {toc - tic:.4f} seconds")
+    loggy.info(f"Postproc done in {toc - tic:.4f} seconds")
+    print('ECmean4 Global Mean succesfully computed!')
 
 
 
@@ -351,8 +360,6 @@ def gm_entry_point():
                 loglevel=args.loglevel,
                 interface=args.interface, config=args.config,
                 model=args.model, ensemble=args.ensemble,
+                addnan=args.addnan,
                 outputdir=args.outputdir)
 
-#if __name__ == "__main__":
-#
-#    gm_entry_point()

ecmean/libs/areas.py

@@ -195,7 +195,7 @@ def area_cell(xfield, gridtype=None, formula='triangles'):
     outfield = xr.DataArray(area, dims=area_dims, coords=xfield.coords, name='area')
 
     # check the total area
-    loggy.info('Total Earth Surface: %s Km2',
+    loggy.debug('Total Earth Surface: %s Km2',
                  str(outfield.sum().values / 10**6))
 
     return outfield

ecmean/libs/diagnostic.py

@@ -41,6 +41,7 @@ def __init__(self, args):
         self.interface = getattr(args, 'interface', '')
         self.resolution = getattr(args, 'resolution', '')
         self.ensemble = getattr(args, 'ensemble', 'r1i1p1f1')
+        self.addnan = getattr(args, 'addnan', False)
         self.funcname = args.funcname.split(".")[1]
         self.version = __version__
         if self.year1 == self.year2:

ecmean/libs/files.py

@@ -85,7 +85,7 @@ def var_is_there(flist, var, face):
         if not varunit:
             varunit = units_avail.get(var_req[0])
             if len(var_req) > 1:
-                loggy.info('%s is a derived var, assuming unit '
+                loggy.debug('%s is a derived var, assuming unit '
                              'as the first of its term', var)
 
     else:
@@ -207,7 +207,7 @@ def _filter_filename_by_year(template, filenames, year):
     if not filternames and len(filenames) > 0:
         loggy.warning('Data for year %s has not been found!', str(year))
 
-    loggy.info('Filtered filenames: %s', filternames)
+    loggy.debug('Filtered filenames: %s', filternames)
     return filternames
 
 
@@ -230,7 +230,7 @@ def _create_filepath(cmorname, face, diag):
         Path(face['model']['basedir']) / \
         Path(face['filetype'][filetype]['dir']) / \
         Path(face['filetype'][filetype]['filename'])
-    loggy.info('Filepath: %s', filepath)
+    loggy.debug('Filepath: %s', filepath)
 
     return filepath
 
@@ -265,7 +265,7 @@ def make_input_filename(cmorname, face, diag):
                 filename1 = filename1 + fname
             filename = filename + filename1
         filename = list(dict.fromkeys(filename))  # Filter unique ones
-        loggy.info("Filenames: %s", filename)
+        loggy.debug("Filenames: %s", filename)
         return filename
 
 

ecmean/libs/parser.py

@@ -34,6 +34,8 @@ def parse_arguments(args, script):
                         help='variant label (ripf number for cmor)')
     parser.add_argument('-s', '--silent', action='store_true',
                         help='do not print anything to std output')
+    parser.add_argument('--addnan', action='store_true',
+                        help='provide figures also where observations are missing')
     parser.add_argument('-v', '--loglevel', type=str, default='WARNING',
                         help='define the level of logging.')
     parser.add_argument('-o', '--outputdir', type=str,

ecmean/libs/plotting.py

@@ -12,10 +12,11 @@
 import matplotlib.pyplot as plt
 from matplotlib.colors import ListedColormap
 import seaborn as sns
+import numpy as np
 
 
 
-def heatmap_comparison_pi(relative_table, diag, filemap):
+def heatmap_comparison_pi(relative_table, diag, filemap, size_model = 14):
     """Function to produce a heatmap - seaborn based - for Performance Indices
     based on CMIP6 ratio"""
 
@@ -42,7 +43,8 @@ def heatmap_comparison_pi(relative_table, diag, filemap):
     chart = sns.heatmap(myfield, norm=divnorm, cmap=pal,
                         cbar_kws={"ticks": tictoc, 'label': title},
                         ax=axs, annot=True, linewidth=0.5, fmt='.2f',
-                        annot_kws={'fontsize': 14, 'fontweight': 'bold'})
+                        annot_kws={'fontsize': size_model, 'fontweight': 'bold'})
+
     chart = chart.set_facecolor('whitesmoke')
     axs.set_title(f'{title} {diag.modelname} {diag.expname} {diag.year1} {diag.year2}', fontsize=25)
     axs.vlines(list(range(sss, tot + sss, sss)), ymin=-1, ymax=len(myfield.index), colors='k')
@@ -60,37 +62,54 @@ def heatmap_comparison_pi(relative_table, diag, filemap):
     plt.close()
 
 
-def heatmap_comparison_gm(data_table, mean_table, std_table, diag, filemap):
+def heatmap_comparison_gm(data_table, mean_table, std_table, diag, filemap, addnan = True,
+                          size_model = 14, size_obs = 8):
     """Function to produce a heatmap - seaborn based - for Global Mean
     based on season-averaged standard deviation ratio"""
 
     # define array
     ratio = (data_table - mean_table) / std_table
-    mask = ratio[('ALL', 'Global')].notna()
+    if addnan:
+        mask = data_table[('ALL', 'Global')].notna()
+    else:
+        mask = ratio[('ALL', 'Global')].notna()
     clean = ratio[mask]
 
     # for dimension of plots
     xfig = len(clean.columns)
     yfig = len(clean.index)
-    _, axs = plt.subplots(1, 1, sharey=True, tight_layout=True, figsize=(xfig+5, yfig+2))
+    fig, axs = plt.subplots(1, 1, sharey=True, tight_layout=True, figsize=(xfig+5, yfig+2))
 
     title = 'GLOBAL MEAN'
 
     # set color range and palette
     thr = 10
     tictoc = range(-thr, thr + 1)
-    pal = ListedColormap(sns.color_palette("seismic", n_colors=21))
+    pal = ListedColormap(sns.color_palette("vlag", n_colors=21))
     tot = len(clean.columns)
     sss = len(set([tup[1] for tup in clean.columns]))
 
     chart = sns.heatmap(clean, annot=data_table[mask], vmin=-thr - 0.5, vmax=thr + 0.5, center=0,
-                        annot_kws={'va': 'bottom', 'fontsize': 14},
-                        cbar_kws={'ticks': tictoc,
+                        annot_kws={'va': 'bottom', 'fontsize': size_model},
+                        cbar_kws={'ticks': tictoc, "shrink": .5,
                                   'label': 'Model Bias \n (standard deviation of interannual variability from observations)'},
                         fmt='.2f', cmap=pal)
+    if addnan:
+        empty = np.where(clean.isna(), 0, np.nan)
+        empty = np.where(data_table[mask]==0, np.nan, empty)
+        chart = sns.heatmap(empty, annot=data_table[mask], fmt='.2f',
+                            vmin=-thr - 0.5, vmax=thr + 0.5, center=0,
+                            annot_kws={'va': 'bottom', 'fontsize': size_model, 'color':'dimgrey'}, cbar=False,
+                            cmap=ListedColormap(['none']))
     chart = sns.heatmap(clean, annot=mean_table[mask], vmin=-thr - 0.5, vmax=thr + 0.5, center=0,
-                        annot_kws={'va': 'top', 'fontsize': 8, 'fontstyle': 'italic'},
+                        annot_kws={'va': 'top', 'fontsize': size_obs, 'fontstyle': 'italic'},
                         fmt='.2f', cmap=pal, cbar=False)
+    if addnan:
+        empty = np.where(clean.isna(), 0, np.nan)
+        empty = np.where(mean_table[mask].isna(), np.nan, empty)
+        chart = sns.heatmap(empty, annot=mean_table[mask], vmin=-thr - 0.5, vmax=thr + 0.5, center=0,
+                        annot_kws={'va': 'top', 'fontsize': size_obs, 'fontstyle': 'italic', 'color':'dimgrey'},
+                        fmt='.2f', cmap=ListedColormap(['none']), cbar=False)
 
     chart = chart.set_facecolor('whitesmoke')
     axs.set_title(f'{title} {diag.modelname} {diag.expname} {diag.year1} {diag.year2}', fontsize=25)

ecmean/libs/support.py

@@ -31,7 +31,7 @@ def __init__(self, component, atmdict, ocedict, areas=True, remap=False, targetg
         """Class for masks, areas and interpolation (xESMF-based)
         for both atmospheric and oceanic component"""
 
-        loggy.info('Running with xesmf version %s', xe.__version__)
+        loggy.debug('Running with xesmf version %s', xe.__version__)
 
         # define the basics
         self.atmareafile = inifiles_priority(atmdict)

ecmean/libs/units.py

@@ -54,7 +54,7 @@ def __init__(self, var=None, org_units=None, tgt_units=None,
         if face and clim:
             self.init_ecmean(clim, face)
 
-        loggy.info(vars(self))
+        loggy.debug(vars(self))
         # if units are defined and convert called
         if convert and self.org_units and self.tgt_units:
 
@@ -101,12 +101,12 @@ def units_converter(self):
         It will not work if BOTH factor and offset are required"""
 
         units_relation = (self.org_units / self.tgt_units).to_base_units()
-        loggy.info('Original vs Target units relation is %s', units_relation)
+        loggy.debug('Original vs Target units relation is %s', units_relation)
 
         if units_relation.units == units('dimensionless'):
 
             if units_relation.magnitude != 1:
-                loggy.info('Unit conversion required...')
+                loggy.debug('Unit conversion required...')
                 offset_standard = 0 * self.org_units
                 offset = offset_standard.to(self.tgt_units).magnitude
                 if offset == 0:
@@ -119,16 +119,16 @@ def units_converter(self):
                 factor = 1.
 
         elif units_relation.units == units('kg / m^3'):
-            loggy.info("Assuming this as a water flux! Am I correct?")
-            loggy.info("Dividing by water density...")
+            loggy.debug("Assuming this as a water flux! Am I correct?")
+            loggy.debug("Dividing by water density...")
             density_water = units('kg / m^3') * 1000
             offset = 0.
             factor_standard = 1 * self.org_units
             factor = (factor_standard / density_water).to(self.tgt_units).magnitude
 
         elif units_relation.units == units('s'):
-            loggy.info("Assuming this is a cumulated flux...")
-            loggy.info("Dividing by cumulation time (expressed in seconds)...")
+            loggy.debug("Assuming this is a cumulated flux...")
+            loggy.debug("Dividing by cumulation time (expressed in seconds)...")
             if self.cumulation_time:
                 cumtime = units('s') * self.cumulation_time
                 offset = 0.
@@ -145,8 +145,8 @@ def units_converter(self):
         if self.org_direction != self.tgt_direction:
             factor = -1. * factor
 
-        loggy.info('Offset is %s', str(offset))
-        loggy.info('Factor is %s', str(factor))
+        loggy.debug('Offset is %s', str(offset))
+        loggy.debug('Factor is %s', str(factor))
         return offset, factor
 
 
@@ -159,6 +159,7 @@ def units_extra_definition():
     # https://github.com/Unidata/MetPy/issues/2884
     units._on_redefinition = 'ignore'
     units.define('fraction = [] = frac')
+    units.define('Fraction = [] = frac')
     units.define('psu = 1e-3 frac')
     units.define('PSU = 1e-3 frac')
     units.define('million = 1e6 = M')