diff --git a/Release/dreqPy/docs/CMIP6_MIP_tables.xlsx b/Release/dreqPy/docs/CMIP6_MIP_tables.xlsx
index c7598ce..b5eb5c8 100644
Binary files a/Release/dreqPy/docs/CMIP6_MIP_tables.xlsx and b/Release/dreqPy/docs/CMIP6_MIP_tables.xlsx differ
diff --git a/Release/dreqPy/docs/dreq.xml b/Release/dreqPy/docs/dreq.xml
index b873a87..0e2592f 100644
--- a/Release/dreqPy/docs/dreq.xml
+++ b/Release/dreqPy/docs/dreq.xml
@@ -1,11 +1,11 @@
-Draft CMIP6 Data Request [01.00.27]
+Draft CMIP6 Data Request [01.00.29]
The CMIP6 Data Request will specify the variables requested for the CMIP6 archive, and the detail the experiments and time slices for which they are required.
Martin Juckes
-2018-07-25
+2018-12-11
CF Standard Name table; CMIP6 Controlled Vocabularies; ESDOC CMIP6 Experiment Documentation
-01.00.27
+01.00.29
@@ -15,10 +15,8 @@
-%s
+%%s
-"""
+""" % dreqMonitoring
def getHtmlItemStyle(self, sect):
"""Get the styling method associated with a given section."""
diff --git a/Release/dreqPy/extensions/collect.py b/Release/dreqPy/extensions/collect.py
index c75c2bd..a042962 100644
--- a/Release/dreqPy/extensions/collect.py
+++ b/Release/dreqPy/extensions/collect.py
@@ -3,13 +3,23 @@
import collections
+def _expt__CMORvar(self):
+ """Return set of CMORvar item identifiers for CMORvars requested for this experiment"""
+ cmv = set()
+ for u in self._get__requestItem():
+ ri = self._inx.uid[u]
+ rl = self._inx.uid[ri.rlid]
+ for i in rl._get__CMORvar():
+ cmv.add(i)
+ return cmv
+
def _expt__requestItem(self):
"""Return set of request item item identifiers for request items linked directly or indirectly to this experiment"""
s = set()
for u in [self.uid,self.mip,self.egid]:
- if 'requestItem' in self._inx.uid[u].a:
- for x in self._inx.uid[u].a['requestItem']:
+ if 'requestItem' in self._inx.iref_by_sect[u].a:
+ for x in self._inx.iref_by_sect[u].a['requestItem']:
s.add(x)
return s
@@ -245,6 +255,7 @@ def add(dq):
"""Add extensions to data request section classes."""
dq.coll['mip'].items[0].__class__._get__expt = _mip__expt
dq.coll['experiment'].items[0].__class__._get__requestItem = _expt__requestItem
+ dq.coll['experiment'].items[0].__class__._get__CMORvar = _expt__CMORvar
dq.coll['requestItem'].items[0].__class__._get__expt = _requestItem__expt
dq.coll['requestLink'].items[0].__class__._get__expt = _requestLink__expt
dq.coll['requestLink'].items[0].__class__._get__CMORvar = _requestLink__CMORvar
diff --git a/Release/dreqPy/extensions/versions.py b/Release/dreqPy/extensions/versions.py
index 9ef813d..80f443c 100644
--- a/Release/dreqPy/extensions/versions.py
+++ b/Release/dreqPy/extensions/versions.py
@@ -13,13 +13,13 @@ def compare(self, oth,count=True,lst='short',sect='CMORvar' ):
if k not in this._uidDict:
n0 += 1
if lst != None:
- print 'Only in %s: %s [%s]' % (oth.version, k, that._uidDict[k].label )
+ print ( 'Only in %s: %s [%s]' % (oth.version, k, that._uidDict[k].label ) )
for k in sorted( this._uidDict.keys() ):
if k not in that._uidDict:
n1 += 1
if lst != None:
- print 'Only in %s: %s [%s]' % (self.version, k, this._uidDict[k].label )
+ print ( 'Only in %s: %s [%s]' % (self.version, k, this._uidDict[k].label ) )
elif this._uidDict[k].__dict__ != that._uidDict[k].__dict__:
n2 += 1
@@ -29,16 +29,16 @@ def compare(self, oth,count=True,lst='short',sect='CMORvar' ):
for i in s1 + s2 + s3:
cc[i] += 1
if lst == 'long':
- print 'Changed in %s -- %s: %s [%s], {%s|%s|%s}' % (self.version,oth.version, k, this._uidDict[k].label, str(s1),str(s2),str(s3) )
+ print ('Changed in %s -- %s: %s [%s], {%s|%s|%s}' % (self.version,oth.version, k, this._uidDict[k].label, str(s1),str(s2),str(s3) ) )
if n0 == 0 and n1 == 0:
- print '%s (%s --> %s):: No additons or removals' % (sect,oth.version,self.version)
+ print ('%s (%s --> %s):: No additons or removals' % (sect,oth.version,self.version) )
else:
- print '%s (%s --> %s):: Added %s, Removed %s' % (sect,oth.version,self.version,n1,n0)
- print '%s:: Number changed in %s -- %s: %s' % (sect, self.version,oth.version,n2)
+ print ('%s (%s --> %s):: Added %s, Removed %s' % (sect,oth.version,self.version,n1,n0) )
+ print ('%s:: Number changed in %s -- %s: %s' % (sect, self.version,oth.version,n2) )
if n2 > 0:
msg = '; '.join( ['%s: %s' % (k,cc[k]) for k in sorted(cc.keys())] )
- print '%s:: Attribute changes: %s' % (sect,msg)
+ print ('%s:: Attribute changes: %s' % (sect,msg) )
def fetch(self,version=None, vdir=None):
files = ['dreq.xml', 'dreq2Defn.xml', 'dreqSupp.xml', 'dreqSuppDefn.xml']
@@ -53,7 +53,7 @@ def fetch(self,version=None, vdir=None):
u = utmpl % (version,f)
page = requests.get(u)
md5 = hashlib.md5(page.text.encode('utf-8')).hexdigest()
- print version, f, md5
+ print ( (version, f, md5 ) )
oo = open( '%s/%s/%s' % (vdir,version,f), 'w' )
oo.write( page.text.encode('utf-8') )
oo.close()
diff --git a/Release/dreqPy/makeTables.py b/Release/dreqPy/makeTables.py
index 7d82891..c379c0f 100644
--- a/Release/dreqPy/makeTables.py
+++ b/Release/dreqPy/makeTables.py
@@ -215,6 +215,12 @@ def stidLink01(self,a,targ,frm='',ann=''):
else:
return '%s [%s]: %s [%s]' % ( targ._h.title, a, targ.__href__(label=targ.title), targ.label )
+ def baseLink01(self,targ,frm='',ann=''):
+ if targ._h.label == 'remarks':
+ return 'Broken link [%s]' % ( targ.__href__() )
+ else:
+ return '%s: %s' % ( targ.label, targ.__href__(odir='../u/',label=targ.title) )
+
def rqlLink02(self,targ,frm='',ann=''):
t2 = targ._inx.uid[targ.refid]
if t2._h.label == 'remarks':
@@ -230,7 +236,11 @@ def rqiLink02(self,targ,frm='',ann=''):
if t2._h.label == 'remarks':
return '%s: %s' % ( targ.__href__(odir='../u/', label=targ.title), "Link to request link broken" )
else:
- t3 = t2._inx.uid[t2.refid]
+ try:
+ t3 = t2._inx.uid[t2.refid]
+ except:
+ print t2.uid, t2.__dict__
+ raise
if t3._h.label == 'remarks':
return '%s [%s]: %s' % ( targ.__href__(odir='../u/', label=targ.title), t2.__href__(odir='../u/', label=t2.title),"Link to request group broken" )
else:
@@ -258,7 +268,7 @@ def cmvLink(self,targ,frm='',ann=''):
return '%s {%s}: %s [%s: %s] (%s)' % ( targ.__href__(odir='../u/', label=targ.label), targ.mipTable, targ.title, targ.frequency, t2.title, nrq )
def objLink(self,targ,frm='',ann=''):
- return '%s: %s' % ( targ.label, targ.__href__(odir='../u/', label=targ.title,title=targ.description) )
+ return '%s [%s]: %s' % ( targ.label, targ.mip, targ.__href__(odir='../u/', label=targ.title,title=targ.description) )
def unitLink(self,targ,frm='',ann=''):
return '%s [%s]: %s' % ( targ.text, targ.label, targ.__href__(odir='../u/', label=targ.title) )
@@ -287,14 +297,19 @@ def cmLink(self,targ,frm='',ann=''):
return '%s [%s]: %s {%s/%s}' % ( targ.cell_methods,targ.label, targ.__href__(odir='../u/', label=targ.title),sz0,sz1 )
def objLnkLink(self,targ,frm='',ann=''):
+ lab1 = targ.title
+ if lab1 == '':
+ lab1 = targ.label
+
if frm == 'objective':
t2 = targ._inx.uid[targ.rid]
t3 = targ._inx.uid[t2.refid]
thislab = '%s (%s)' % (t2.mip,t3.label)
- return '%s: %s' % ( t2.title, t2.__href__(odir='../u/',label=thislab) )
+ ##return '%s: %s' % ( t2.title, t2.__href__(odir='../u/',label=thislab) )
+ return '%s: %s' % ( targ.__href__(odir='../u/',label=lab1), t2.__href__(odir='../u/',label=thislab, title=t2.title) )
else:
t2 = targ._inx.uid[targ.oid]
- return '%s: %s' % ( t2.label, t2.__href__(odir='../u/',label=t2.title) )
+ return '%s: %s' % ( targ.__href__(odir='../u/',label=lab1), t2.__href__(odir='../u/',label=t2.label, title=t2.title) )
def labTtl(self,targ,frm='',ann=''):
return '%s: %s' % ( targ.__href__(odir='../u/', label=targ.label), targ.title )
@@ -376,6 +391,7 @@ def run():
dq.itemStyles['structure'] = styls.strLink
dq.itemStyles['cellMethods'] = styls.cmLink
dq.itemStyles['remarks'] = styls.remarkLink
+ dq.itemStyles['exptgroup'] = styls.baseLink01
dq.itemStyles['objectiveLink'] = styls.objLnkLink
dq.itemStyles['requestVarGroup'] = styls.vgrpLink
dq.itemStyles['miptable'] = styls.miptableLink
diff --git a/Release/dreqPy/misc_utils.py b/Release/dreqPy/misc_utils.py
index 9bc0040..30215c3 100644
--- a/Release/dreqPy/misc_utils.py
+++ b/Release/dreqPy/misc_utils.py
@@ -1,4 +1,4 @@
-import collections, string , os, sys
+import collections, os, sys
import logging
import time
import difflib
@@ -125,9 +125,9 @@ def rankCMORvars(dq):
if i.title != ic.title:
print ( '%s: %s, %s' % (kk, ic.title, i.title) )
kd += 1
- if string.find( ic.modeling_realm, ' ' ) != -1:
- for x in string.split( ic.modeling_realm ):
- r.add( string.strip( x ) )
+ if ic.modeling_realm.find( ' ' ) != -1:
+ for x in ic.modeling_realm.split( ):
+ r.add( x.strip( ) )
elif ic.modeling_realm not in ['__unset__','']:
r.add( ic.modeling_realm )
if 'requestVar' in dq.inx.iref_by_sect[ic.uid].a:
@@ -160,9 +160,9 @@ def rankVars(dq):
if 'CMORvar' in dq.inx.iref_by_sect[i.uid].a:
for cmv in dq.inx.iref_by_sect[i.uid].a['CMORvar']:
ic = dq.inx.uid[cmv]
- if string.find( ic.modeling_realm, ' ' ) != -1:
- for x in string.split( ic.modeling_realm ):
- r.add( string.strip( x ) )
+ if ic.modeling_realm.find( ' ' ) != -1:
+ for x in ic.modeling_realm.split( ):
+ r.add( x.strip( ) )
elif ic.modeling_realm not in ['__unset__','']:
r.add( ic.modeling_realm )
if 'requestVar' in dq.inx.iref_by_sect[cmv].a:
@@ -363,7 +363,7 @@ def getExptByThisMip(self,mip,rqi,tiermax=1):
self.expts[ex.mip].add(id)
self.exrqi[id].add( i.uid )
ks = sorted( list( self.expts.keys() ) )
- xx = string.join( ['%s: %s' % (k,len(self.expts[k])) for k in ks], ', ' )
+ xx = ', '.join( ['%s: %s' % (k,len(self.expts[k])) for k in ks] )
print ( '%s:: %s' % (mip,xx) )
class c1(object):
diff --git a/Release/dreqPy/overviewTabs.py b/Release/dreqPy/overviewTabs.py
index 84abe01..0bf3fce 100644
--- a/Release/dreqPy/overviewTabs.py
+++ b/Release/dreqPy/overviewTabs.py
@@ -1,15 +1,19 @@
-
-import collections, string, os
+import collections, os
import xlsxwriter
try:
import dreq
+ import scope_utils
+ import table_utils
except:
import dreqPy.dreq as dreq
+ import dreqPy.scope_utils as scope_utils
+ import dreqPy.table_utils as table_utils
jsh='''
-'''
+%s
+''' % dreq.dreqMonitoring
def vfmt( x ):
if x < 1.e9:
@@ -44,7 +48,7 @@ def __init__(self,sc,mt_tables,tiermax=1,pmax=1,only=False,vols=None,fnm='new',m
self.fnm = fnm
assert vols == None or type(vols) == type( () ), 'vols argument must be none or tuple of length 2: %s' % type(vols)
self.dq = sc.dq
- self.mips = ['CMIP','AerChemMIP', 'C4MIP', 'CFMIP', 'DAMIP', 'DCPP', 'FAFMIP', 'GeoMIP', 'GMMIP', 'HighResMIP', 'ISMIP6', 'LS3MIP', 'LUMIP', 'OMIP', 'PAMIP', 'PMIP', 'RFMIP', 'ScenarioMIP', 'VolMIP', 'CORDEX', 'DynVar', 'SIMIP', 'VIACSAB']
+ self.mips = ['CMIP'] + scope_utils.mips
self.mipsp = self.mips[:-4]
self.sc = sc
self.pmax=pmax
@@ -138,10 +142,10 @@ def makeMMhtml(self,m,m2):
##print ('INFO.mmhtml.00001: %s, %s' % (kc,len( self.cc[kc].a.keys() ) ) )
if len( self.cc[kc].a.keys() ) == 0:
return
- if not os.path.isdir( 'tabs03' ):
+ if not os.path.isdir( 'html/tabs03' ):
print ( 'WARNING.makeMMhtml: creating directory for html files: tabs03' )
- os.mkdir( 'tabs03' )
- oo = open( 'tabs03/%s' % fss, 'w' )
+ os.mkdir( 'html/tabs03' )
+ oo = open( 'html/tabs03/%s' % fss, 'w' )
ttl = 'Data requested by %s from %s experiments (tier %s, priority %s)' % (m,m2,self.tiermax,self.pmax)
jsh = ''
pream = '%s
\n' % ttl
@@ -300,5 +304,5 @@ def writeMips(self,sss=False):
import dreqPy.scope as scope
import dreqPy.makeTables as makeTables
sc = scope.dreqQuery()
- r = r1( sc, makeTables.tables, tiermax=1, pmax=1 )
- r = r1( sc, makeTables.tables, tiermax=3, pmax=3 )
+ r = r1( sc, table_utils.tables, tiermax=1, pmax=1 )
+ r = r1( sc, table_utils.tables, tiermax=3, pmax=3 )
diff --git a/Release/dreqPy/packageConfig.py b/Release/dreqPy/packageConfig.py
index eeb4fa9..93d0387 100644
--- a/Release/dreqPy/packageConfig.py
+++ b/Release/dreqPy/packageConfig.py
@@ -16,7 +16,7 @@
VERSION_DEFAULT_DIR = os.path.join(HOME, '.dreqPy')
VERSION_DIR = os.environ.get('DRQ_VERSION_DIR', VERSION_DEFAULT_DIR)
-__version__ = "01.00.27"
+__version__ = "01.00.29"
__versionComment__ = "Version %s" % __version__
__title__ = "dreqPy"
__description__ = "CMIP6 Data Request Python API"
diff --git a/Release/dreqPy/scope.py b/Release/dreqPy/scope.py
index af61ff3..59d096a 100644
--- a/Release/dreqPy/scope.py
+++ b/Release/dreqPy/scope.py
@@ -17,15 +17,17 @@ def __init__(self):
if imm == 1:
from utilities import cmvFilter, gridOptionSort
import misc_utils
+ import scope_utils
import fgrid
import volsum
else:
+ import dreqPy.scope_utils as scope_utils
import dreqPy.volsum as volsum
import dreqPy.fgrid as fgrid
from dreqPy.utilities import cmvFilter, gridOptionSort
import dreqPy.misc_utils as misc_utils
-import collections, string, operator
+import collections, operator
import sys, os
def intdict():
@@ -247,8 +249,9 @@ def __init__(self,dq=None,tierMax=1):
self.exptFilterBlack = None
self.uniqueRequest = False
- self.mips = set( [x.label for x in self.dq.coll['mip'].items ] )
- self.mips = ['CMIP','AerChemMIP', 'C4MIP', 'CFMIP', 'DAMIP', 'DCPP', 'FAFMIP', 'GeoMIP', 'GMMIP', 'HighResMIP', 'ISMIP6', 'LS3MIP', 'LUMIP', 'OMIP', 'PAMIP', 'PMIP', 'RFMIP', 'ScenarioMIP', 'VolMIP', 'CORDEX', 'DynVar', 'SIMIP', 'VIACSAB']
+ ##self.mips = set( [x.label for x in self.dq.coll['mip'].items ] )
+ ##self.mips = ['CMIP','AerChemMIP', 'C4MIP', 'CFMIP', 'DAMIP', 'DCPP', 'FAFMIP', 'GeoMIP', 'GMMIP', 'HighResMIP', 'ISMIP6', 'LS3MIP', 'LUMIP', 'OMIP', 'PAMIP', 'PMIP', 'RFMIP', 'ScenarioMIP', 'VolMIP', 'CORDEX', 'DynVar', 'SIMIP', 'VIACSAB']
+ self.mips = ['CMIP'] + scope_utils.mips
self.mipsp = self.mips[:-4]
self.cmvGridId, i4 = fgrid.fgrid( self.dq )
assert len(i4) == 0
@@ -1007,8 +1010,8 @@ def rqiByMip( self, mip):
l2.append( self.dq.inx.uid[u] )
l20 = self.rqiByMip0( mip )
- for i in l20:
- assert i in l2
+ ##for i in l20:
+ ##assert i in l2
return l2
@@ -1061,7 +1064,6 @@ def checkDir(self,odir,msg):
print ('Created directory %s for: %s' % (odir,msg) )
def xlsByMipExpt(self,m,ex,pmax,odir='xls',xls=True,txt=False,txtOpts=None):
- import scope_utils
mxls = scope_utils.xlsTabs(self,tiermax=self.tierMax,pmax=pmax,xls=xls, txt=txt, txtOpts=txtOpts,odir=odir)
mlab = misc_utils.setMlab( m )
mxls.run( m, mlab=mlab )
@@ -1598,7 +1600,7 @@ class dreqUI(object):
-------------------------
-v : print version and exit;
--unitTest : run some simple tests;
- -m : MIP of list of MIPs (comma separated; for objective selection see note [1] below);
+ -m : MIP of list of MIPs (comma separated; use '_all_' for all; for objective selection see note [1] below);
-l : List for options:
o: objectives
e: experiments
@@ -1621,10 +1623,12 @@ class dreqUI(object):
--omitCmip : omit the CMIP core data request (included by default);
--allgrd : When a variable is requested on multiple grids, archive all grids requested (default: only the finest resolution);
--unique : List only variables which are requested uniquely by this MIP, for at least one experiment;
- --esm : include ESM experiments (default is to omit esm-hist etc from volume estimates);
- --txt : Create text file with requested variables;
+ --esm : include ESM experiments (default is to omit esm-hist etc from volume estimates; over-ridden by --mcat);
+ --txt : Create text (tab seperated variables) file with requested variables; the files are placed in the same directory as xls files;
--mcfg : Model configuration: 7 integers, comma separated, 'nho','nlo','nha','nla','nlas','nls','nh1'
default: 259200,60,64800,40,20,5,100
+ --mcat [none]: Source types inlcuded in model, as comma separated list: only experiments with all required source types are included in volume estimates. Set to 'none' to turn off filtering;
+ --mcat-strict : if present, the experiments are filtered to those with the specified configuration, not allowing components to be switched off;
--txtOpts : options for content of text file: (v|c)[(+|-)att1[,att2[...]]]
--xlsDir : Directory in which to place variable listing [xls];
--xmlVersion : version number of XML document [only with extension enabled -- not stable];
@@ -1641,7 +1645,7 @@ class dreqUI(object):
drq -m HighResMIP:Ocean.DiurnalCycle
"""
def __init__(self,args):
- self.adict = {}
+ self.adict = {'mcatStrict':False}
self.knownargs = {'-m':('m',True), '-p':('p',True), '-e':('e',True), '-t':('t',True), \
'-h':('h',False), '--printLinesMax':('plm',True), \
'-l':('l',True),
@@ -1661,6 +1665,8 @@ def __init__(self,args):
'--allgrd':('allgrd',False), \
'--unique':('unique',False), \
'--mcfg':('mcfg',True), \
+ '--mcat':('mcat',True), \
+ '--mcatStrict':('mcatStrict',False), \
'--txtOpts':('txtOpts',True), \
'--xmlVersion':('xmlVersion',True), \
'--xlsDir':('xlsdir',True), '--xls':('xls',False) \
@@ -1681,6 +1687,9 @@ def __init__(self,args):
assert self.checkArgs( notKnownArgs ), 'FATAL ERROR 001: Arguments not recognised: %s' % (str(notKnownArgs) )
+ if self.adict.get('mcat','none') != 'none':
+ self.adict['esm'] = True
+
if 'm' in self.adict:
if self.adict['m'] == '_all_':
pass
@@ -1699,6 +1708,14 @@ def __init__(self,args):
if 'omitcmip' not in self.adict and 'CMIP' not in self.adict['m']:
self.adict['m'].add( 'CMIP' )
+ if self.adict.get('mcat','none') != 'none':
+ stys = self.adict['mcat'].split(',')
+ stys_pp = stys[:]
+ if 'AOGCM' in stys:
+ stys_pp.append( 'AGCM' )
+ if 'AGCM' in stys_pp:
+ stys_pp += ['LAND','RAD']
+ self.adict['_mcat'] = (stys,stys_pp)
if 'grdpol' in self.adict:
assert self.adict['grdpol'] in ['native','1deg'], 'Grid policy argument --grdpol must be native or 1deg : %s' % self.adict['grdpol']
@@ -1784,13 +1801,14 @@ def run(self, dq=None):
return
if 'mcfg' in self.adict:
- ll = string.split( self.adict['mcfg'], ',' )
+ ll = self.adict['mcfg'].split( ',' )
assert len(ll) == 7, 'Length of model configuration argument must be 7 comma separated integers: %s' % self.adict['mcfg']
lli = [ int(x) for x in ll]
self.sc = dreqQuery( dq=self.dq )
self.sc.intersection = self.intersection
+
if 'grdforce' in self.adict:
self.sc.gridPolicyForce = self.adict['grdforce']
if 'grdpol' in self.adict:
@@ -1817,6 +1835,7 @@ def run(self, dq=None):
if doSf:
self.adict['sf'] = True
assert not ('legacy' in self.adict and 'sf' in self.adict), "Conflicting command line argument, 'legacy' and 'sf': use only one of these"
+
if makeXls or makeTxt or doSf:
xlsOdir = self.adict.get( 'xlsdir', 'xls' )
self.sc.checkDir( xlsOdir, 'xls files' )
@@ -1893,10 +1912,54 @@ def run(self, dq=None):
for e in ['esm-hist','esm-hist-ext','esm-piControl','piControl-spinup','esm-piControl-spinup']:
ss.add( self.sc.exptByLabel[ e ] )
self.sc.exptFilterBlack = ss
+
+ if self.sc.exptFilterBlack != None and self.sc.exptFilter != None:
+ ss = [x for x in self.sc.exptFilter if x not in self.sc.exptFilterBlack]
+ if len(ss) == 0:
+ print ( """WARNING: filter settings give no experiments: try using --esm flag: by default esm experiments are filtered out""" )
+ return
+
+
+ makeTxt = self.adict.get( 'txt', False )
makeXls = self.adict.get( 'xls', False )
+ if 'txtOpts' in self.adict:
+ if self.adict['txtOpts'][0] == 'v':
+ txtOpts = NT_txtopts( 'var' )
+ else:
+ txtOpts = NT_txtopts( 'cmv' )
+ else:
+ txtOpts=None
+
+ exptFilters = collections.defaultdict( set )
+ for i in self.dq.coll['experiment'].items:
+## required
+ tt = tuple( i.mcfg.split( '|' )[0].strip().split(' ') )
+## allowed
+ uu = tuple( i.mcfg.split( '|' )[1].strip().split(' ') )
+ exptFilters[(tt,uu)].add(i.uid)
+## NB this is the default##
if 'sf' in self.adict:
- vs = volsum.vsum( self.sc, odsz, npy, odir=xlsOdir, tabByFreqRealm=tabByFreqRealm )
+ if self.adict.get('mcat','none') != 'none':
+ thisFilter = set()
+ ##self.sc.exptFilter = set()
+ for tt,uu in exptFilters:
+ t1 = all( [x in self.adict['_mcat'][1] for x in tt] )
+ if self.adict['mcatStrict']:
+ t1 = t1 and all( [x in (tt + uu) for x in self.adict['_mcat'][0] ] )
+ if t1:
+ thisFilter = thisFilter.union( exptFilters[(tt,uu)] )
+ if self.sc.exptFilter == None:
+ self.sc.exptFilter = thisFilter
+ else:
+ self.sc.exptFilter = thisFilter.intersection( self.sc.exptFilter )
+ if len( self.sc.exptFilter ) == 0:
+ print ( 'WARNING: filter settings give no experiments' )
+ return
+
+ ##vs = volsum.vsum( self.sc, odsz, npy, odir=xlsOdir, tabByFreqRealm=tabByFreqRealm, txt=makeTxt,txtOpts=txtOpts, exptFilter=exptFilters['AOGCM'] )
+ vs = volsum.vsum( self.sc, odsz, npy, odir=xlsOdir, tabByFreqRealm=tabByFreqRealm, txt=makeTxt,txtOpts=txtOpts )
+ self.vs = vs
vs.run( self.adict['m'], '%s/requestVol_%s_%s_%s' % (xlsOdir,mlab,tierMax,pmax), pmax=pmax, doxlsx=makeXls )
totalOnly = False
if len( self.adict['m'] ) == 1 or totalOnly:
@@ -1935,18 +1998,10 @@ def run(self, dq=None):
adsCount = self.adict.get( 'count', False )
self.getVolByMip(pmax,eid,adsCount)
- makeTxt = self.adict.get( 'txt', False )
+
if makeXls or makeTxt:
mips = self.adict['m']
- if 'txtOpts' in self.adict:
- if self.adict['txtOpts'][0] == 'v':
- txtOpts = NT_txtopts( 'var' )
- else:
- txtOpts = NT_txtopts( 'cmv' )
- else:
- txtOpts=None
-
self.sc.xlsByMipExpt(mips,eid,pmax,odir=xlsOdir,xls=makeXls,txt=makeTxt,txtOpts=txtOpts)
def printListCc(self,cc):
diff --git a/Release/dreqPy/scope_utils.py b/Release/dreqPy/scope_utils.py
index 13c5aa4..61e538a 100644
--- a/Release/dreqPy/scope_utils.py
+++ b/Release/dreqPy/scope_utils.py
@@ -1,4 +1,4 @@
-import collections, string
+import collections
try:
import table_utils
except:
@@ -6,6 +6,8 @@
##NT_txtopts = collections.namedtuple( 'txtopts', ['mode'] )
+mips = ['AerChemMIP', 'CDRMIP', 'C4MIP', 'CFMIP', 'DAMIP', 'DCPP', 'FAFMIP', 'GeoMIP', 'GMMIP', 'HighResMIP', 'ISMIP6', 'LS3MIP', 'LUMIP', 'OMIP', 'PAMIP', 'PMIP', 'RFMIP', 'ScenarioMIP', 'VolMIP', 'CORDEX', 'DynVar', 'SIMIP', 'VIACSAB']
+
class c1(object):
def __init__(self):
self.a = collections.defaultdict( int )
@@ -22,7 +24,7 @@ def __init__(self,sc,tiermax=1,pmax=1,xls=True, txt=False, txtOpts=None, odir='x
self.doXls = xls
self.doTxt = txt
- self.mips = ['AerChemMIP', 'C4MIP', 'CFMIP', 'DAMIP', 'DCPP', 'FAFMIP', 'GeoMIP', 'GMMIP', 'HighResMIP', 'ISMIP6', 'LS3MIP', 'LUMIP', 'OMIP', 'PAMIP', 'PMIP', 'RFMIP', 'ScenarioMIP', 'VolMIP', 'CORDEX', 'DynVar', 'SIMIP', 'VIACSAB']
+ self.mips = mips
self.mipsp = ['DECK','CMIP6',] + self.mips[:-4]
self.tabs = table_utils.tables( sc, xls=xls, txt=txt, txtOpts=txtOpts, odir=odir )
diff --git a/Release/dreqPy/simpleCheck.py b/Release/dreqPy/simpleCheck.py
index d6e5879..b4a6240 100644
--- a/Release/dreqPy/simpleCheck.py
+++ b/Release/dreqPy/simpleCheck.py
@@ -9,7 +9,9 @@
##from . import __init__
##DOC_DIR = __init__.DOC_DIR
-import string, os, sys, collections
+import os, sys, collections
+
+usingPython3 = sys.version_info >= (3,0)
try:
import pkgutil
@@ -144,14 +146,42 @@ def _clear_ch04(self):
os.unlink( '.simpleCheck_check2_err.txt' )
os.unlink( '.simpleCheck_check2.txt' )
- def _ch04_checkCmd(self):
- import os
+ def _getCmd(self):
if self.entryPoint == 'drq':
- cmd = 'drq'
+ self.cmd = 'drq'
else:
- cmd = 'python dreqCmdl.py'
+ if usingPython3:
+ self.cmd = 'python3 dreqCmdl.py'
+ else:
+ self.cmd = 'python dreqCmdl.py'
+
+ def _ch05_checkMcfg(self):
+ self._getCmd()
+ thisCmd = '%s -m CMIP -e historical --mcfg 259200,60,64800,40,20,5,100' % self.cmd
+ os.popen( '%s 2> .simpleCheck_check5_err.txt 1>.simpleCheck_check5.txt' % thisCmd ).readlines()
+
+ ii = open( '.simpleCheck_check5_err.txt' ).readlines()
+ if len(ii) > 0:
+ print ( 'WARNING[005]: failed to get volume est. with command line call' )
+ self.ok = False
+ ##self._clear_ch04()
+ return
+
+ ii = open( '.simpleCheck_check5.txt' ).readlines()
+ if len(ii) < 1:
+ print ( 'WARNING[006]: failed to get get volume est. with command line call' )
+ self.ok = False
+ ##self._clear_ch04()
+ return
+
+ self.ok = True
+ return
+
+ def _ch04_checkCmd(self):
+ import os
+ self._getCmd()
- os.popen( '%s -v 2> .simpleCheck_check2_err.txt 1>.simpleCheck_check2.txt' % cmd ).readlines()
+ os.popen( '%s -v 2> .simpleCheck_check2_err.txt 1>.simpleCheck_check2.txt' % self.cmd ).readlines()
ii = open( '.simpleCheck_check2_err.txt' ).readlines()
if len(ii) > 0:
diff --git a/Release/dreqPy/table_utils.py b/Release/dreqPy/table_utils.py
index 4747ae5..9169d55 100644
--- a/Release/dreqPy/table_utils.py
+++ b/Release/dreqPy/table_utils.py
@@ -146,7 +146,7 @@ def cmvtabrec(self,j,t,orec):
raise
if self.txt:
- self.oo.write( '\t'.join( [t,] + orec) + '\n' )
+ self.oo.write( '\t'.join( [t,] + [x.replace('"',"'") for x in orec]) + '\n' )
def varrec(self,j,orec):
if self.xls:
@@ -158,7 +158,7 @@ def varrec(self,j,orec):
def var(self):
if self.xls:
- self.sht = self.newSheet( 'var' )
+ self.sht = self.newSheet( 'var' )
j = 0
hrec = ['Long name', 'units', 'description', 'Variable Name', 'CF Standard Name' ]
if self.xls:
@@ -366,10 +366,19 @@ def __init__(self, sc, subset=None, mcfgNote=None, dest='tables/test', skipped=s
thisp = str( min( pdict[cmv.uid] ) )
else:
thisp = str(cmv.defaultPriority)
- print 'ERROR.priority.0101: ',cmv.label,dest
+ print ('ERROR.priority.0101: %s, %s ' % (cmv.label,dest) )
else:
thisp = str(cmv.defaultPriority)
- orec = [thisp,var.title, var.units, var.description, cmv.description, var.label, var.sn, strc.cell_methods, cmv.positive, cmv.type, dims, cmv.label, cmv.modeling_realm, cmv.frequency, strc.cell_measures, cmv.prov,cmv.provNote,str(cmv.rowIndex),cmv.uid,cmv.vid,cmv.stid,strc.title, valid_min, valid_max, ok_min_mean_abs, ok_max_mean_abs]
+#
+# avoid duplication of information (added in 01.00.29)
+##
+ if cmv.description != var.description:
+ cmmt = cmv.description
+ else:
+ cmmt = ''
+ orec = [thisp,var.title, var.units, var.description, cmmt, var.label, var.sn, strc.cell_methods, cmv.positive, cmv.type, dims, cmv.label, cmv.modeling_realm, cmv.frequency, strc.cell_measures, cmv.prov,cmv.provNote,str(cmv.rowIndex),cmv.uid,cmv.vid,cmv.stid,strc.title, valid_min, valid_max, ok_min_mean_abs, ok_max_mean_abs]
+
+##
except:
print ('FAILED TO CONSTRUCT RECORD: %s [%s], %s [%s]' % (cmv.uid,cmv.label,var.uid,var.label) )
raise
@@ -417,7 +426,6 @@ def __init__(self, sc, subset=None, mcfgNote=None, dest='tables/test', skipped=s
if type( priority ) != type(1):
thisp = priority
priority = thisp[1]
- ##print 'ERROR in priority type[2]: ',priority, tslice[cmv.uid]
orec[0] = '%s' % priority
if tsmode[:4] in ['simp','bran']:
@@ -437,7 +445,7 @@ def __init__(self, sc, subset=None, mcfgNote=None, dest='tables/test', skipped=s
orec.append( grid )
if orec[0] in [0,'0',None]:
- print 'ERROR.priority.006: ',orec,dest
+ print ('ERROR.priority.006: %s, %s ' % (orec,dest))
if withoo:
oo.write( '\t'.join(orec ) + '\n' )
j+=1
diff --git a/Release/dreqPy/volsum.py b/Release/dreqPy/volsum.py
index e3f13ef..53bec73 100644
--- a/Release/dreqPy/volsum.py
+++ b/Release/dreqPy/volsum.py
@@ -58,15 +58,21 @@ def close(self):
self.wb.close()
class vsum(object):
- def __init__(self,sc,odsz,npy,exptFilter=None, odir='xls', tabByFreqRealm=False):
+ def __init__(self,sc,odsz,npy,exptFilter=None, odir='xls', tabByFreqRealm=False,txt=False,txtOpts=None):
self.tabByFreqRealm = tabByFreqRealm
+ self.doTxt = txt
+ self.txtOpts = txtOpts
idir = dreq.DOC_DIR
if 'collect' not in sc.dq._extensions_:
extCollect.add(sc.dq)
self.sc = sc
self.exptMipRql = collections.defaultdict( set )
+ if exptFilter != None:
+ assert type( exptFilter ) == type( set() ), 'FATAl.vsum.001: exptFilter argument must be type set: %s' % type( exptFilter )
for i in self.sc.dq.coll['requestLink'].items:
expts = requestLink__expt(i,rql=[i.uid,])
+ if exptFilter != None:
+ expts = exptFilter.intersection( expts )
for e in expts:
self.exptMipRql[ (i.mip,self.sc.dq.inx.uid[e].label) ].add( i.uid)
self.odsz=odsz
@@ -76,6 +82,7 @@ def __init__(self,sc,odsz,npy,exptFilter=None, odir='xls', tabByFreqRealm=False)
self.accum = False
self.odir = odir
self.efnsfx = ''
+ self.accPdict = collections.defaultdict( set )
if sc.gridPolicyForce == 'native':
self.efnsfx = '_fn'
elif sc.gridPolicyForce == '1deg':
@@ -109,20 +116,27 @@ def analAll(self,pmax,mips=None,html=True,makeTabs=True):
if mips == None:
theseMips = ['TOTAL',] + self.sc.mips
else:
- theseMips = mips
+ theseMips = list(mips)
+
+## move *TOTAL to end of list.
+ if '*TOTAL' in theseMips:
+ theseMips.remove( '*TOTAL' )
+ theseMips.append( '*TOTAL' )
self.rres = {}
self.rresu = {}
for m in theseMips:
olab = m
+ useAccPdict = False
if m == '*TOTAL':
- thism = theseMips.copy()
+ thism = theseMips[:]
if type( thism ) == type( set() ):
thism.remove( '*TOTAL' )
else:
- thism.pop( '*TOTAL' )
+ thism.remove( '*TOTAL' )
olab = misc_utils.setMlab( thism )
+ useAccPdict = True
elif type( theseMips ) == type( dict() ):
thism = {m:theseMips[m]}
else:
@@ -134,7 +148,7 @@ def analAll(self,pmax,mips=None,html=True,makeTabs=True):
self.run( thism, '%s/requestVol_%s_%s_%s' % (self.odir,olab,self.sc.tierMax,pmax), pmax=pmax,doxlsx=makeTabs )
- self.anal(olab=olab,doUnique='TOTAL' in theseMips, makeTabs=makeTabs)
+ self.anal(olab=olab,doUnique='TOTAL' in theseMips, makeTabs=makeTabs, useAccPdict=useAccPdict)
ttl = sum( [x for k,x in self.res['vu'].items()] )*2.*1.e-12
volsmm[m] = self.res['vm']
volsmmt[m] = self.res['vmt']
@@ -189,7 +203,7 @@ def xlsDest(self,mode,olab,lab2):
else:
return '%s/%s_%s_%s_%s_%s%s' % (self.odir,self.xlsPrefixM,olab,lab2,self.sc.tierMax,self.pmax,self.efnsfx)
- def anal(self,olab=None,doUnique=False,makeTabs=False,mode='full'):
+ def anal(self,olab=None,doUnique=False,makeTabs=False,mode='full',useAccPdict=False):
vmt = collections.defaultdict( int )
vm = collections.defaultdict( int )
ve = collections.defaultdict( int )
@@ -249,12 +263,12 @@ def anal(self,olab=None,doUnique=False,makeTabs=False,mode='full'):
ss = ss.union( lm[m] )
if makeTabs:
##table_utils.makeTab(self.sc.dq, subset=lm[m], dest=self.xlsDest('m',olab,m), collected=cc[m],exptUid=self.sc.exptByLabel.get(m,m) )
- table_utils.makeTab(self.sc, subset=lm[m], dest=self.xlsDest('m',olab,m), collected=cc[m] )
+ table_utils.makeTab(self.sc, subset=lm[m], dest=self.xlsDest('m',olab,m), collected=cc[m], txt=self.doTxt, txtOpts=self.txtOpts )
if olab != None and makeTabs:
- table_utils.makeTab(self.sc, subset=ss, dest=self.xlsDest('m',olab,'TOTAL'), collected=cct)
+ table_utils.makeTab(self.sc, subset=ss, dest=self.xlsDest('m',olab,'TOTAL'), collected=cct, txt=self.doTxt, txtOpts=self.txtOpts )
if olab != 'TOTAL' and doUnique:
- table_utils.makeTab(self.sc, subset=s_lm, dest=self.xlsDest('m',olab,'Unique'), collected=s_cc)
+ table_utils.makeTab(self.sc, subset=s_lm, dest=self.xlsDest('m',olab,'Unique'), collected=s_cc, txt=self.doTxt, txtOpts=self.txtOpts )
cc = collections.defaultdict( dict )
ucc = collections.defaultdict( dict )
@@ -265,7 +279,12 @@ def anal(self,olab=None,doUnique=False,makeTabs=False,mode='full'):
if olab != None and makeTabs:
el = self.sc.dq.inx.uid[e].label
- if olab in ['Total','TOTAL']:
+ if useAccPdict:
+ pdict = collections.defaultdict( set )
+ for vid, p in self.accPdict[e]:
+ pdict[vid].add( p )
+
+ elif olab in ['Total','TOTAL']:
pdict = None
elif (olab,el) in self.exptMipRql:
pdict = collections.defaultdict( set )
@@ -274,8 +293,9 @@ def anal(self,olab=None,doUnique=False,makeTabs=False,mode='full'):
for rqvid in self.sc.dq.inx.iref_by_sect[rql.refid].a['requestVar']:
rqv = self.sc.dq.inx.uid[rqvid]
pdict[rqv.vid].add( rqv.priority )
+ self.accPdict[e].add( (rqv.vid,rqv.priority) )
else:
- print 'INFO.00201: olab,e not found:',olab,el
+ print ( 'INFO.00201: olab,e not found:',olab,el )
pdict = None
tslice = {}
@@ -284,7 +304,7 @@ def anal(self,olab=None,doUnique=False,makeTabs=False,mode='full'):
tslice[v] = self.sc.cmvts[v][e]
dest = self.xlsDest('e',olab,el)
mode ='e'
- table_utils.makeTab(self.sc, subset=lex[e], dest=self.xlsDest(mode,olab,el), collected=cc[e],byFreqRealm=self.tabByFreqRealm, tslice=tslice, exptUid=e, tabMode=mode, pdict=pdict)
+ table_utils.makeTab(self.sc, subset=lex[e], dest=self.xlsDest(mode,olab,el), collected=cc[e],byFreqRealm=self.tabByFreqRealm, tslice=tslice, exptUid=e, tabMode=mode, pdict=pdict, txt=self.doTxt, txtOpts=self.txtOpts )
if olab != 'TOTAL' and doUnique:
for e,t in s_vet:
@@ -292,7 +312,7 @@ def anal(self,olab=None,doUnique=False,makeTabs=False,mode='full'):
for e in sorted( uve.keys() ):
if olab != None and makeTabs:
el = self.sc.dq.inx.uid[e].label
- table_utils.makeTab(self.sc, subset=s_lex[e], dest=self.xlsDest('u',olab,el), collected=ucc[e])
+ table_utils.makeTab(self.sc, subset=s_lex[e], dest=self.xlsDest('u',olab,el), collected=ucc[e], txt=self.doTxt, txtOpts=self.txtOpts)
self.res = { 'vmt':vmt, 'vet':vet, 'vm':vm, 'uve':uve, 've':ve, 'lm':lm, 'lex':lex, 'vu':vu, 'cc':cc, 'cct':cct, 'vf':vf}
diff --git a/Release/dreqPy/vrev.py b/Release/dreqPy/vrev.py
index 4158458..d3469d4 100644
--- a/Release/dreqPy/vrev.py
+++ b/Release/dreqPy/vrev.py
@@ -23,9 +23,13 @@ def chk2(self,vn,byExpt=False, byBoth=False):
cc = {}
l = dq.inx.iref_by_sect[v.uid].a['CMORvar']
for i in l:
- r = dq.inx.uid[i]
- kk = '%s.%s' % (r.mipTable, r.label )
- cc[i] = (kk,self.chkCmv( i, byExpt=byExpt, byBoth=byBoth ) )
+ try:
+ r = dq.inx.uid[i]
+ kk = '%s.%s' % (r.mipTable, r.label )
+ cc[i] = (kk,self.chkCmv( i, byExpt=byExpt, byBoth=byBoth ) )
+ except:
+ print ( 'failed at uid = %s' % i )
+ raise
return cc
@@ -110,7 +114,7 @@ def chkCmv(self,cmvid, byExpt=False, byBoth=False,expt=None):
if byExpt or byBoth:
s3 = set()
- if expt != None:
+ if expt != None and expt in self.sc.rqLinkByExpt:
for i in s2:
if i in self.sc.rqLinkByExpt[expt]:
s3.add(i)