diff --git a/Release/MANIFEST.in b/Release/MANIFEST.in index a642231..259fa1f 100644 --- a/Release/MANIFEST.in +++ b/Release/MANIFEST.in @@ -2,6 +2,8 @@ recursive-include docs * include LICENSE +include dreqPy/utilP2/__init__.py +include dreqPy/utilP2/util.py include dreqPy/docs/BlockSchema.csv include dreqPy/docs/CMIP6_MIP_tables.xlsx include dreqPy/docs/dc1.xsd @@ -26,5 +28,3 @@ include dreqPy/docs/dreqSupp.xml include dreqPy/docs/dreqManifest.txt include dreqPy/docs/md5Manifest.txt include dreqPy/docs/sfheadings.csv -include dreqPy/utilP2/__init__.py -include dreqPy/utilP2/util.py diff --git a/Release/dreqPy/__init__.py b/Release/dreqPy/__init__.py index 490bef8..ba2baf3 100644 --- a/Release/dreqPy/__init__.py +++ b/Release/dreqPy/__init__.py @@ -5,13 +5,3 @@ except: #from dreqPy import packageConfig from .packageConfig import * - -try: - import utilP2 -except: - from dreqPy import utilP2 - -##try: - ##import utilP2.util -##except: - ##from dreqPy.utilP2 import util diff --git a/Release/dreqPy/docs/CMIP6_MIP_tables.xlsx b/Release/dreqPy/docs/CMIP6_MIP_tables.xlsx index 2c95e40..f0e2876 100644 Binary files a/Release/dreqPy/docs/CMIP6_MIP_tables.xlsx and b/Release/dreqPy/docs/CMIP6_MIP_tables.xlsx differ diff --git a/Release/dreqPy/docs/dreq.xml b/Release/dreqPy/docs/dreq.xml index 461bc20..cc3daae 100644 --- a/Release/dreqPy/docs/dreq.xml +++ b/Release/dreqPy/docs/dreq.xml @@ -1,11 +1,11 @@ -Draft CMIP6 Data Request [01.00.31beta] +Draft CMIP6 Data Request [01.00.32] The CMIP6 Data Request will specify the variables requested for the CMIP6 archive, and the detail the experiments and time slices for which they are required. Martin Juckes -2019-05-20 +2020-04-03 CF Standard Name table; CMIP6 Controlled Vocabularies; ESDOC CMIP6 Experiment Documentation -01.00.31beta +01.00.32
@@ -112,9 +112,7 @@ - - @@ -251,7 +249,6 @@ - @@ -285,7 +282,6 @@ - @@ -293,6 +289,7 @@ + @@ -303,6 +300,7 @@ + @@ -318,6 +316,7 @@ + @@ -326,6 +325,7 @@ + @@ -343,6 +343,7 @@ + @@ -360,6 +361,7 @@ + @@ -376,6 +378,8 @@ + + @@ -392,6 +396,7 @@ + @@ -409,6 +414,7 @@ + @@ -426,6 +432,7 @@ + @@ -435,6 +442,7 @@ + @@ -454,6 +462,7 @@ + @@ -474,6 +483,7 @@ + @@ -505,7 +515,6 @@ - @@ -558,7 +567,6 @@ - @@ -572,7 +580,6 @@ - @@ -1039,14 +1046,11 @@ - - - - + @@ -1057,9 +1061,9 @@ + - @@ -1242,13 +1246,11 @@ - - @@ -1256,25 +1258,21 @@ - - - - @@ -1542,7 +1540,6 @@ - @@ -1554,7 +1551,6 @@ - @@ -1615,169 +1611,169 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1787,7 +1783,6 @@ - @@ -1797,7 +1792,6 @@ - @@ -1815,15 +1809,12 @@ - - - @@ -1893,7 +1884,6 @@ - @@ -1905,7 +1895,6 @@ - @@ -1915,7 +1904,6 @@ - @@ -1925,7 +1913,6 @@ - @@ -2417,7 +2404,6 @@ - @@ -2435,17 +2421,14 @@ - - - @@ -3268,7 +3251,6 @@ - @@ -3278,7 +3260,6 @@ - @@ -3321,7 +3302,6 @@ - @@ -3332,7 +3312,6 @@ - @@ -3473,6 +3452,7 @@ + @@ -3482,6 +3462,7 @@ + @@ -3648,11 +3629,11 @@ - + - + @@ -3663,8 +3644,8 @@ - - + + @@ -3691,7 +3672,7 @@ - + @@ -3867,7 +3848,7 @@ - + @@ -3961,26 +3942,26 @@ - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - + + @@ -4124,7 +4105,7 @@ - + @@ -4180,12 +4161,12 @@ - - - - - - + + + + + + @@ -4197,7 +4178,7 @@ - + @@ -4219,7 +4200,7 @@ - + @@ -4380,13 +4361,13 @@ - + - + @@ -4482,7 +4463,7 @@ - + @@ -4778,7 +4759,6 @@ - @@ -4981,7 +4961,7 @@ - + @@ -5058,7 +5038,7 @@ - + @@ -5195,7 +5175,7 @@ - + @@ -5228,7 +5208,7 @@ - + @@ -5928,9 +5908,7 @@ - - @@ -6174,7 +6152,7 @@ - + @@ -6384,8 +6362,6 @@ - - @@ -6906,11 +6882,6 @@ - - - - - @@ -6918,300 +6889,318 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -9364,14 +9353,14 @@ - + - + @@ -9444,7 +9433,6 @@ - @@ -9578,7 +9566,7 @@ - + @@ -9802,7 +9790,7 @@ - + @@ -10738,7 +10726,6 @@ - @@ -10748,6 +10735,7 @@ + @@ -10825,6 +10813,7 @@ + @@ -10872,7 +10861,6 @@ - @@ -11883,7 +11871,7 @@ - + @@ -11940,7 +11928,6 @@ - @@ -12358,7 +12345,7 @@ - + @@ -12511,7 +12498,6 @@ - @@ -12589,7 +12575,7 @@ - + @@ -12611,7 +12597,7 @@ - + @@ -12650,7 +12636,6 @@ - @@ -12855,7 +12840,7 @@ - + @@ -12870,7 +12855,7 @@ - + @@ -12882,7 +12867,7 @@ - + @@ -12896,7 +12881,7 @@ - + @@ -12916,14 +12901,14 @@ - + - + @@ -13044,7 +13029,7 @@ - + @@ -13117,7 +13102,7 @@ - + @@ -13209,12 +13194,12 @@ - + - + @@ -13238,7 +13223,7 @@ - + @@ -13520,7 +13505,7 @@ - + @@ -13604,8 +13589,11 @@ + - + + + @@ -13615,6 +13603,8 @@ + + @@ -13628,20 +13618,19 @@ - - + - - - - - + + + + + @@ -13665,14 +13654,14 @@ - - + + - + @@ -13692,27 +13681,27 @@ - + - - - - - - - + + + + + + + - - + + - + @@ -13729,38 +13718,39 @@ - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + - + - + @@ -13774,7 +13764,7 @@ - + @@ -13782,14 +13772,15 @@ + - - + + @@ -13817,21 +13808,21 @@ - + - - - - - - - + + + + + + + - + @@ -13843,36 +13834,36 @@ - - - - - + + + + + - + - - + + - - - - + + + + - + - + @@ -13889,7 +13880,7 @@ - + @@ -13920,15 +13911,16 @@ - - - - - + + + + + - + + - + @@ -13941,6 +13933,8 @@ + + @@ -13948,24 +13942,24 @@ - + - + - + - - + + @@ -13976,12 +13970,12 @@ - + - + - + @@ -13989,20 +13983,23 @@ - + + + - - + + - + + - - + + @@ -14018,6 +14015,7 @@ + @@ -14027,12 +14025,14 @@ + - + - + + @@ -14043,7 +14043,7 @@ - + @@ -14058,7 +14058,11 @@ + + + + @@ -14094,7 +14098,7 @@ - + @@ -14108,33 +14112,34 @@ + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - - @@ -14142,22 +14147,25 @@ + - + - + - + + + @@ -14180,14 +14188,15 @@ - - + + + - + - + @@ -14209,19 +14218,19 @@ - - + + - - + + - - - + + + @@ -14606,10 +14615,10 @@ - + - + @@ -14618,11 +14627,11 @@ - - + + - + @@ -14659,8 +14668,8 @@ - - + + @@ -14669,9 +14678,10 @@ + - + @@ -14714,35 +14724,39 @@ - + + + + - + - + - - - - - - - + + + + + + + - - - + + + - - - - + + + + + @@ -14753,35 +14767,35 @@ - - + + - - - - - + + + + + - + - - + + - - + + - - + + @@ -14789,11 +14803,11 @@ - - + + - + @@ -14811,11 +14825,11 @@ - + - - + + @@ -14837,17 +14851,18 @@ - + + - + @@ -14884,98 +14899,99 @@ - - - - - - + + + + + + - - - - - - - - - - - + + + + + + + + + + + - - + + - - + + - + - - - - - - + + + + + + - - + + - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - + + + + + + - - + + - - - - + + + + - - - + + + - - - + + + @@ -15004,33 +15020,32 @@ - - - + + + - - + - + - - - + + + - + - + @@ -15078,9 +15093,9 @@ - + - + @@ -15088,83 +15103,88 @@ + + - + - + - + - + - - - - - - - - - + + + + + + + + + - + - - + + - + - + + - - + + + - - + + - + - - - - - + + + + + - + - - + + + - - - - + + + + @@ -15172,15 +15192,15 @@ - + - - + + - + @@ -15189,103 +15209,118 @@ - - + + - - - - + + + + - + + - - - - - - + + + + + + - + - + - + - - + + - - + + - + - - + + - - + + - - - - - - + + + + + + - - - + + + + + - + - + - - - - - - - - + + + + + + + + + - + + + + + + + + + + + + @@ -15295,65 +15330,73 @@ - + - - + + - - + + - + - + - + + - - + + + + + + - + - + + - - + + + + - + - - - - - - - - + + + + + + + + - + @@ -15367,7 +15410,7 @@ - + @@ -15379,7 +15422,7 @@ - + @@ -15411,7 +15454,9 @@ + + @@ -15423,27 +15468,30 @@ + - - + + - + - + + + @@ -15453,7 +15501,7 @@ - + @@ -15476,17 +15524,17 @@ - + - - - - - + + + + + @@ -15497,21 +15545,21 @@ - - - - + + + + - + - + - + - + @@ -15521,8 +15569,8 @@ - - + + @@ -15530,24 +15578,52 @@ - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -15558,35 +15634,37 @@ - + + - - + + + - + - - - + + - - - - + + + + + @@ -15958,18 +16036,20 @@ - + + - + + - + @@ -15981,10 +16061,10 @@ - - + + - + @@ -15995,7 +16075,7 @@ - + @@ -16038,12 +16118,16 @@ - + + + + + @@ -16051,27 +16135,33 @@ + + + + - + + + @@ -16088,10 +16178,17 @@ + + + + + + + @@ -16099,6 +16196,7 @@ + @@ -16117,7 +16215,6 @@ - @@ -16126,32 +16223,40 @@ - - - + + + - + + - + - + - + + + + + - + + + + @@ -16162,7 +16267,7 @@ - + @@ -16172,7 +16277,7 @@ - + @@ -16234,6 +16339,7 @@ + @@ -16247,13 +16353,13 @@ - + - - + + @@ -16281,18 +16387,19 @@ - - - - + + + + + - + @@ -16307,12 +16414,12 @@ - - + + - + @@ -16366,7 +16473,7 @@ - + @@ -16387,11 +16494,11 @@ - - - - - + + + + + @@ -16764,7 +16871,7 @@ - + @@ -16814,7 +16921,7 @@ - + @@ -16836,20 +16943,21 @@ + - + - + @@ -16866,7 +16974,7 @@ - + @@ -16951,13 +17059,13 @@ - - - - - - - + + + + + + + @@ -16976,7 +17084,7 @@ - + @@ -17032,46 +17140,46 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + @@ -17082,9 +17190,9 @@ - - + + @@ -17117,7 +17225,7 @@ - + @@ -17268,7 +17376,6 @@ - @@ -17277,6 +17384,7 @@ + @@ -17291,18 +17399,18 @@ - - - - - - - - - - - - + + + + + + + + + + + + @@ -17328,11 +17436,11 @@ - - + + - - + + @@ -17348,21 +17456,21 @@ - + - - - - - - - - + + + + + + + + - + @@ -17374,36 +17482,36 @@ - - - - - + + + + + - + - - + + - - - - - + + + + + - + - + @@ -17421,12 +17529,12 @@ - + - + @@ -17466,86 +17574,86 @@ - - - - + + + + - - + + - + - - - + + + - - - + + + - + - + - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + - + - - - - - - - - - - - - - + + + + + + + + + + + + + - - - - - - - + + + + + + + - + @@ -17557,26 +17665,26 @@ - - - - + + + + - + - - - + + + - + @@ -17588,7 +17696,7 @@ - + @@ -17604,23 +17712,23 @@ - - - + + + - + + - + - @@ -17632,8 +17740,8 @@ + - @@ -17647,12 +17755,12 @@ - + - + @@ -17661,8 +17769,8 @@ - - + + @@ -17686,26 +17794,27 @@ + - - - + + + - + - + - - - - + + + + @@ -17719,6 +17828,7 @@ + @@ -17729,22 +17839,22 @@ - + - - - - - + + + + + - - - + + + @@ -17754,8 +17864,8 @@ - - + + @@ -17796,12 +17906,13 @@ - + + @@ -17821,18 +17932,18 @@ - + - - - - - - - - - - + + + + + + + + + + @@ -17863,8 +17974,8 @@ - - + + @@ -17942,7 +18053,7 @@ - + @@ -17953,26 +18064,26 @@ - - + + - + - + - + @@ -17984,8 +18095,8 @@ - - + + @@ -18075,7 +18186,7 @@ - + @@ -18144,16 +18255,16 @@ - - - - - - - - - - + + + + + + + + + + @@ -18161,8 +18272,8 @@ - - + + @@ -18256,7 +18367,7 @@ - + @@ -18278,7 +18389,7 @@ - + @@ -18323,7 +18434,7 @@ - + @@ -18339,7 +18450,7 @@ - + @@ -18373,7 +18484,7 @@ - + @@ -18400,17 +18511,17 @@ - + - + - + @@ -18433,12 +18544,12 @@ - + - + @@ -18503,7 +18614,7 @@ - + @@ -18554,8 +18665,8 @@ - - + + @@ -18581,8 +18692,8 @@ - - + + @@ -18590,7 +18701,7 @@ - + @@ -18627,8 +18738,8 @@ - - + + @@ -18652,7 +18763,7 @@ - + @@ -18695,7 +18806,7 @@ - + @@ -18703,18 +18814,18 @@ - - - - - - - - - - - - + + + + + + + + + + + + @@ -18787,7 +18898,7 @@ - + @@ -18804,7 +18915,7 @@ - + @@ -18821,7 +18932,7 @@ - + @@ -18933,13 +19044,13 @@ - + - + @@ -18957,7 +19068,7 @@ - + @@ -19058,7 +19169,7 @@ - + @@ -19075,7 +19186,7 @@ - + @@ -19101,11 +19212,11 @@ - + - + @@ -19463,8 +19574,8 @@ - - + + @@ -19500,7 +19611,7 @@ - + diff --git a/Release/dreqPy/docs/dreq2Defn.xml b/Release/dreqPy/docs/dreq2Defn.xml index 260a2c9..4033de6 100644 --- a/Release/dreqPy/docs/dreq2Defn.xml +++ b/Release/dreqPy/docs/dreq2Defn.xml @@ -127,7 +127,7 @@ which case a primary realm is assigned as the first listed and other relev - + diff --git a/Release/dreqPy/docs/dreq2Sample.xml b/Release/dreqPy/docs/dreq2Sample.xml index b7c263b..a6df102 100644 --- a/Release/dreqPy/docs/dreq2Sample.xml +++ b/Release/dreqPy/docs/dreq2Sample.xml @@ -6,126 +6,126 @@ xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:pav="http://purl.org/pav/2.3" xmlns="urn:w3id.org:cmip6.dreq.dreq:a"> -Draft CMIP6 Data Request [01.00.31] +Draft CMIP6 Data Request [01.00.32] The CMIP6 Data Request will specify the variables requested for the CMIP6 archive, and the detail the experiments and time slices for which they are required. Martin Juckes -2019-07-22 +2020-04-03 CF Standard Name table; CMIP6 Controlled Vocabularies; ESDOC CMIP6 Experiment Documentation -01.00.31 +01.00.32
- + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - +
- + - + - + diff --git a/Release/dreqPy/docs/dreqSupp.xml b/Release/dreqPy/docs/dreqSupp.xml index 1e7c859..e53d7cc 100644 --- a/Release/dreqPy/docs/dreqSupp.xml +++ b/Release/dreqPy/docs/dreqSupp.xml @@ -1,127 +1,127 @@ -CMIP6 Data Request Supplement [01.00.31beta] +CMIP6 Data Request Supplement [01.00.32] The CMIP6 Data Request will specify the variables requested for the CMIP6 archive, and the detail the experiments and time slices for which they are required. Martin Juckes -2019-07-22 +2020-04-03 CF Standard Name table; CMIP6 Controlled Vocabularies; ESDOC CMIP6 Experiment Documentation -01.00.31beta +01.00.32
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -132,72 +132,72 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - - - - + + + + + + + + + + + + + + + + + + + + + + - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
diff --git a/Release/dreqPy/dreq.py b/Release/dreqPy/dreq.py index e12bef2..dc53806 100644 --- a/Release/dreqPy/dreq.py +++ b/Release/dreqPy/dreq.py @@ -384,15 +384,23 @@ def dictInit( self, idict ): def mdInit( self, el, etree=False ): __doc__ = """Initialisation from a mindom XML element. The list of attributes must be set by the class factory before the class is initialised""" - deferredHandling=False - nw1 = 0 tvtl = [] if etree: ks = set( el.keys() ) for a in self._a.keys(): if a in ks: aa = '%s%s' % (self.ns,a) - tvtl.append( (a,True, str( el.get( a ) ) ) ) +# +# exception trapping to provide more diagnostics when a character code issue arises +# added in 01.00.32beta +# + this = el.get( a ) + try: + tvtl.append( (a,True, str( this ) ) ) + except: + print( 'ERROR: FAILED to str %s: %s' % (a,this) ) + tvtl.append( (a,True, this.encode( 'utf-8' ) ) ) + elif self._a[a].__dict__.get( 'required', True ) in [False,'false',u'false']: tvtl.append( (a,True,None) ) else: @@ -408,18 +416,82 @@ def mdInit( self, el, etree=False ): ## elif self._a[a].__dict__.get( 'required', True ) not in [False,'false',u'false']: tvtl.append( (a,False,None) ) - + + self._tvtlInit( tvtl ) + self._contentInitialised = True + + def _tvtlInit( self, tvtl ): + """Coercing string values into a python objects appropriate to declared types: looping over tuples containing tags and values. + the code sets the default if appropriate, otherwise call _avInit to coerce the string value to an appropriate python object.""" + + nw1 = 0 for a,tv,v in tvtl: if a == 'uid': uid = v for a,tv,v in tvtl: if tv: - erase = False if v == None: - pass - erase = True - elif self._a[a].type == u'xs:float': + self.__dict__[a] = '__tmp__' + delattr( self, a ) + ### need to overwrite attribute (which is inherited from parent class) before deleting it. + ### this may not be needed in python3 + else: + self._avInit( a, v ) + else: + if a in ['uid',]: + thissect = '%s' % (self._h.title) + print ( 'ERROR.020.0001: missing uid: %s: a,tv,v: %s, %s, %s\n %s' % (thissect,a,tv,v,str( self.__dict__ )) ) + ##if etree: + ##p = self.parent_map( el ) + ##print ( ks ) + raise + ##import sys + ##sys.exit(0) + self.__dict__[a] = self._d.defaults.get( a, self._d.glob ) + + + def _avInit( self, a, v ): + """Coercing a string value into a python object appropriate to the declared type: non-numeric data types: + a: the tag identifying the data element; + v: the string serialisation of the data value: + + Approach + -------- + (1) Check whether type is numric -- if true call _avInitNumeric; + (2) Otherwise, deal with boolean or string list types, or default which is to take string value as is.""" + + numericTypes = [u'xs:float',u'aa:st__floatList', u'aa:st__floatListMonInc',u'aa:st__integerList', u'aa:st__integerListMonInc',u'xs:integer'] + + if self._a[a].type in numericTypes: + return self._avInitNumeric( a,v ) + elif self._a[a].type == u'xs:boolean': + v = v in ['true','1'] + elif self._a[a].type == u'aa:st__stringList': + if v.find(' ') != -1: + v = tuple( v.split() ) + else: + v = (v,) + + elif self._a[a].type not in [u'xs:string', u'aa:st__uid', 'aa:st__fortranType', 'aa:st__configurationType','aa:st__sliceType']: + print ('ERROR: Type %s not recognised [%s:%s]' % (self._a[a].type,self._h.label,a) ) + + self.__dict__[a] = v + + def _avInitNumeric( self, a, v ): + """Coercing a value into a python object appropriate to the declared type: numeric data types. + a: the tag identifying the data element; + v: the string serialisation of the data value: + + list types are split into elements; + integers coerced with the python int function; + floats coerced with the python float function; + -- if integer coercion fails, indirect coercion is attempted via int (float (v) ) + -- if specified in the type, monotonicity is checked. + -- if type is integer, a blank is taken as zero. + """ + msg = '' + if self._a[a].type == u'xs:float': if v == '': v = None else: @@ -457,41 +529,10 @@ def mdInit( self, el, etree=False ): print ( 'WARN: input integer non-compliant: %s: %s: %s [%s] %s' % (thissect,a,v0,v,uid) ) except: msg = 'ERROR: failed to convert integer: %s: %s: %s, %s' % (thissect,a,v,type(v)) - deferredHandling=True - elif self._a[a].type == u'xs:boolean': - v = v in ['true','1'] - elif self._a[a].type == u'aa:st__stringList': - if v.find(' ') != -1: - v = tuple( v.split() ) - else: - v = (v,) - elif self._a[a].type not in [u'xs:string', u'aa:st__uid', 'aa:st__fortranType', 'aa:st__configurationType','aa:st__sliceType']: - print ('ERROR: Type %s not recognised [%s:%s]' % (self._a[a].type,self._h.label,a) ) - - if erase: - ### need to overwrite attribute (which is inherited from parent class) before deleting it. - ### this may not be needed in python3 - self.__dict__[a] = '__tmp__' - delattr( self, a ) + print ( msg ) else: - self.__dict__[a] = v - else: - if a in ['uid',]: - thissect = '%s' % (self._h.title) - print ( 'ERROR.020.0001: missing uid: %s: a,tv,v: %s, %s, %s\n %s' % (thissect,a,tv,v,str( self.__dict__ )) ) - if etree: - ##p = self.parent_map( el ) - print ( ks ) - raise - import sys - sys.exit(0) - self.__dict__[a] = self._d.defaults.get( a, self._d.glob ) - - if deferredHandling: - print ( msg ) - - self._contentInitialised = True - + raise + self.__dict__[a] = v class config(object): """Read in a vocabulary collection configuration document and a vocabulary document""" @@ -565,6 +606,7 @@ def __read__(self, thisdoc, configdoc): ## self.etree = False self.etree = True + root=None self.ns = None if not self.configOnly: if self.etree: @@ -660,11 +702,19 @@ def __read__(self, thisdoc, configdoc): ## self.coll['__core__'] = self.ntf( self._t0.header, self._t0.attributes, [self.tt0[k] for k in self.tt0] ) + self._read_vl(vl,tables) + + for k in tables: + self.recordAttributeDefn[k] = tables[k] + + if not self.configOnly: + self.__read_tables__(root,tables) + + def _read_vl(self,vl,tables): ec = {} for i in self.coll['__core__'].items: ec[i.label] = i - for v in vl: t = self.parsevcfg(v) tables[t[0].label] = t @@ -688,11 +738,7 @@ def __read__(self, thisdoc, configdoc): assert k in ec, 'Key %s [%s] not found' % (k,sct) self.coll[sct].attDefn[k] = ec[k] - for k in tables: - self.recordAttributeDefn[k] = tables[k] - - if self.configOnly: - return + def __read_tables__(self,root,tables): for k in tables.keys(): if self.etree: vl = root.findall( './/%s%s' % (self.ns,k) ) diff --git a/Release/dreqPy/dreqCmdl.py b/Release/dreqPy/dreqCmdl.py index fa8e935..45c7af9 100644 --- a/Release/dreqPy/dreqCmdl.py +++ b/Release/dreqPy/dreqCmdl.py @@ -34,12 +34,20 @@ def main_entry(): import simpleCheck else: from . import simpleCheck + if '--strict' in sys.argv: + assert simpleCheck.all, 'Errors detected in simpleCheck' print( "Starting test suite 2" ) if scr: import examples.ex203 as ex203 else: from .examples import ex203 ex203.main( scope ) + print( "Starting test suite 3" ) + if scr: + import utilP2.info as info + else: + from .utilP2 import info + print( info.__doc__ ) print( "Tests completed" ) elif sys.argv[1] == '--makeTables': print( "Making web page tables" ) diff --git a/Release/dreqPy/extensions/collect.py b/Release/dreqPy/extensions/collect.py index dd7f2be..ed224f5 100644 --- a/Release/dreqPy/extensions/collect.py +++ b/Release/dreqPy/extensions/collect.py @@ -32,6 +32,42 @@ def _isLinked(self): return True return False +### +### can generalise here +### +### _inx.iref_by_sect[ ].a[ ] +### +### +### resolveRequestPath( target=, ) +### +### "path" is a given by "spine" plus - objective - link. 3 elements. once origin and target are known, it reduces to a line: this logic sits in +### instantiation. +### Alternatively, if we instantiate only on the start, the target becomes a call argument: the path should then be derived from the links. +#### +### E.g. assign each branch a name (mip,var,expt) and number sections (varPack = var.1), then X.n --> Y.m goes X.n-1, .... C, y.1, ..Y.m +### Gives a simple alogoithm to establish path from current location. +### Not much benefit gained from instantiation on start ... but it gives some transparency. + +### to modify doc strings, need to use a class factory rather than instantiation: cannot write to __doc__ attribute of a method +### You can modify the class .... but that doesn't help .. +### unless all the generic stuff is in a base class .. +### + +def manufacture_resolve_requestPath(start): + _startId = get_sectionId(start) + def _resolve_requestPath(self,target,filter=None,fmt=None): + startId = _startId + targetId = get_sectionId(target) + + + +class ResolveRequestPathMaster(object): + def __init__( start='requestLink'): + self.start = start + + def __call__(self,target,filter=None,fmt=None): + """Return set of target objects linked on request path ...""" +### def _var__mips(self): """Return set of mip item identifiers for MIPs requesting this var""" mips = set() diff --git a/Release/dreqPy/makeTables.py b/Release/dreqPy/makeTables.py index 454529a..dab345f 100644 --- a/Release/dreqPy/makeTables.py +++ b/Release/dreqPy/makeTables.py @@ -379,7 +379,7 @@ def run(): assert os.path.isdir( 'html/index' ), 'Before running this script you need to create "html", "html/index" and "html/u" sub-directories, or edit the call to dq.makeHtml, and refernces to "u" in style lines below' assert os.path.isdir( 'tables' ), 'Before running this script you need to create a "tables" sub-directory, or edit the table_utils.makeTab class' - dq = dreq.loadDreq( htmlStyles=htmlStyle, manifest='docs/dreqManifest.txt' ) + dq = dreq.loadDreq( htmlStyles=htmlStyle, manifest='out/dreqManifest.txt' ) ## ## add special styles to dq object "itemStyle" dictionary. ## diff --git a/Release/dreqPy/overviewTabs.py b/Release/dreqPy/overviewTabs.py index 9eb4348..bd4a534 100644 --- a/Release/dreqPy/overviewTabs.py +++ b/Release/dreqPy/overviewTabs.py @@ -53,8 +53,8 @@ def __init__(self,sc,mt_tables,tiermax=1,pmax=1,only=False,vols=None,fnm='new',m assert vols == None or type(vols) == type( () ), 'vols argument must be none or tuple of length 2: %s' % type(vols) self.dq = sc.dq self.mips = ['CMIP'] + scope_utils.mips - self.mipsp = [x for x in self.mips if x not in scope_utils.mipsdiag] - + self.mipsp = self.mips[:-3] + self.mipsp.remove( 'VIACSAB' ) self.sc = sc self.pmax=pmax self.efnsfx = '' diff --git a/Release/dreqPy/packageConfig.py b/Release/dreqPy/packageConfig.py index 642d4c4..320d8a7 100644 --- a/Release/dreqPy/packageConfig.py +++ b/Release/dreqPy/packageConfig.py @@ -16,7 +16,7 @@ VERSION_DEFAULT_DIR = os.path.join(HOME, '.dreqPy') VERSION_DIR = os.environ.get('DRQ_VERSION_DIR', VERSION_DEFAULT_DIR) -__version__ = "01.00.31.post6" +__version__ = "01.00.32" __versionComment__ = "Version %s" % __version__ __title__ = "dreqPy" __description__ = "CMIP6 Data Request Python API" diff --git a/Release/dreqPy/scope.py b/Release/dreqPy/scope.py index 68939e8..a5885b9 100644 --- a/Release/dreqPy/scope.py +++ b/Release/dreqPy/scope.py @@ -254,10 +254,8 @@ def __init__(self,dq=None,tierMax=1): ##self.mips = set( [x.label for x in self.dq.coll['mip'].items ] ) ##self.mips = ['CMIP','AerChemMIP', 'C4MIP', 'CFMIP', 'DAMIP', 'DCPP', 'FAFMIP', 'GeoMIP', 'GMMIP', 'HighResMIP', 'ISMIP6', 'LS3MIP', 'LUMIP', 'OMIP', 'PAMIP', 'PMIP', 'RFMIP', 'ScenarioMIP', 'VolMIP', 'CORDEX', 'DynVar', 'DynVarMIP', 'SIMIP', 'VIACSAB'] - self.mips = ['CMIP'] + scope_utils.mips - self.mipsp = [x for x in self.mips if x not in scope_utils.mipsdiag] - + self.mipsp = ['CMIP'] + scope_utils.mipsp self.cmvGridId, i4 = fgrid.fgrid( self.dq ) assert len(i4) == 0 @@ -1622,7 +1620,7 @@ class dreqUI(object): : definition of an attribute. -h : help: print help text; -e : experiment; - -t maxmum tier; + -t maximum tier; -p maximum priority; --xls : Create Excel file with requested variables; --sf : Print summary of variable count by structure and frequency [default]; @@ -1836,6 +1834,7 @@ def run(self, dq=None): if 'mcfg' in self.adict: self.sc.setMcfg( lli ) + self.tierByDefault = 't' not in self.adict tierMax = self.adict.get( 't', 1 ) self.sc.setTierMax( tierMax ) pmax = self.adict.get( 'p', 1 ) @@ -1901,9 +1900,11 @@ def run(self, dq=None): if ex in self.sc.mipsp: eid = set( self.dq.inx.iref_by_sect[ex].a['experiment'] ) self.sc.exptFilter = eid + self.exptMode = 'mip' elif self.adict['e'] in self.sc.exptByLabel: eid = self.sc.exptByLabel[ self.adict['e'] ] self.sc.exptFilter = set( [eid,] ) + self.exptMode = 'expt' else: try: ns = 0 @@ -1936,6 +1937,11 @@ def run(self, dq=None): print ( """WARNING: filter settings give no experiments: try using --esm flag: by default esm experiments are filtered out""" ) return + if self.exptMode == 'expt' and self.tierByDefault: + tier = min(self.dq.inx.uid[eid].tier) + if tier > tierMax: + print ( """WARNING: default is to filter experiment tier > %s, default tier for %s is %s""" % (tierMax, self.dq.inx.uid[eid].label, tier) ) + print ( """Resubmit with '-t %s' to avoid this""" % tier ) makeTxt = self.adict.get( 'txt', False ) makeXls = self.adict.get( 'xls', False ) @@ -1984,7 +1990,7 @@ def run(self, dq=None): vsmode='full' else: vsmode='short' - vs.anal(olab=mlab,doUnique=True, mode=vsmode, makeTabs=makeXls) + vs.anal(olab=mlab,doUnique=False, mode=vsmode, makeTabs=makeXls) for f in sorted( vs.res['vf'].keys() ): mlg.prnt ( 'Frequency: %s: %s' % (f, vs.res['vf'][f]*2.*1.e-12 ) ) ttl = sum( [x for k,x in vs.res['vu'].items()] )*2.*1.e-12 diff --git a/Release/dreqPy/scope_utils.py b/Release/dreqPy/scope_utils.py index 058fdc0..9cd512c 100644 --- a/Release/dreqPy/scope_utils.py +++ b/Release/dreqPy/scope_utils.py @@ -7,7 +7,8 @@ ##NT_txtopts = collections.namedtuple( 'txtopts', ['mode'] ) mips = ['ScenarioMIP', 'VIACSAB', 'AerChemMIP', 'CDRMIP', 'C4MIP', 'CFMIP', 'DAMIP', 'DCPP', 'FAFMIP', 'GeoMIP', 'GMMIP', 'HighResMIP', 'ISMIP6', 'LS3MIP', 'LUMIP', 'OMIP', 'PAMIP', 'PMIP', 'RFMIP', 'VolMIP', 'CORDEX', 'DynVarMIP', 'SIMIP' ] -mipsdiag = ['VolMIP', 'CORDEX', 'DynVarMIP', 'SIMIP' ] + +mipsp = [x for x in mips if x not in ['VIACSAB','CORDEX', 'DynVarMIP', 'SIMIP' ] ] class c1(object): def __init__(self): @@ -26,7 +27,7 @@ def __init__(self,sc,tiermax=1,pmax=1,xls=True, txt=False, txtOpts=None, odir='x self.doTxt = txt self.mips = mips - self.mipsp = ['DECK','CMIP6',] + [x for x in mips if x not in mipsdiag] + self.mipsp = ['DECK','CMIP6',] + self.mips[:-4] self.tabs = table_utils.tables( sc, xls=xls, txt=txt, txtOpts=txtOpts, odir=odir ) diff --git a/Release/dreqPy/simpleCheck.py b/Release/dreqPy/simpleCheck.py index bbb8c95..fa46762 100644 --- a/Release/dreqPy/simpleCheck.py +++ b/Release/dreqPy/simpleCheck.py @@ -55,7 +55,9 @@ class checkbase(object): def __init__(self,lab): self.lab = lab self.ok = True + self.entryPath = sys.argv[0] self.entryPoint = sys.argv[0].split( '/' )[-1] + self.entryDir = self.entryPath.replace( self.entryPoint, '' ) ##if os.path.isdir( 'out' ): ##self.docdir = 'out' ##elif os.path.isdir( 'docs' ): @@ -147,13 +149,13 @@ def _clear_ch04(self): os.unlink( '.simpleCheck_check2.txt' ) def _getCmd(self): - if self.entryPoint == 'drq': - self.cmd = 'drq' + if self.entryPoint in ['drq', 'drqTest']: + self.cmd = self.entryPoint else: if usingPython3: - self.cmd = 'python3 dreqCmdl.py' + self.cmd = 'python3 %sdreqCmdl.py' % self.entryDir else: - self.cmd = 'python dreqCmdl.py' + self.cmd = 'python2.7 %sdreqCmdl.py' % self.entryDir def _ch05_checkMcfg(self): self._getCmd() @@ -162,7 +164,7 @@ def _ch05_checkMcfg(self): ii = open( '.simpleCheck_check5_err.txt' ).readlines() if len(ii) > 0: - print ( 'WARNING[005]: failed to get volume est. with command line call' ) + print ( 'WARNING[005]: failed to get volume est. with command line call: %s' % self.cmd ) self.ok = False ##self._clear_ch04() return diff --git a/Release/dreqPy/utilP2/info.py b/Release/dreqPy/utilP2/info.py new file mode 100644 index 0000000..d238da3 --- /dev/null +++ b/Release/dreqPy/utilP2/info.py @@ -0,0 +1 @@ +"""Dummy file to ease package management across python versions""" diff --git a/Release/pypiUpload.sh b/Release/pypiUpload.sh index 3ca70d9..d4e52cd 100644 --- a/Release/pypiUpload.sh +++ b/Release/pypiUpload.sh @@ -1,9 +1,16 @@ #################### post pypi upgrade.... +## +## for local install: + +# sudo python setup.py install --force + + # compile egg etc (inc. wheel) -/usr/bin/python3 setup.py sdist bdist_wheel +python3 setup.py sdist bdist_wheel -/usr/bin/python3 -m twine upload --repository-url https://upload.pypi.org/legacy/ dist/* -/usr/bin/python3 -m twine upload dist/* +python3 -m twine upload --repository-url https://test.pypi.org/legacy/ dist/* +python3 -m twine upload --repository-url https://upload.pypi.org/legacy/ dist/* +python3 -m twine upload dist/* #python setup.py sdist upload -r https://test.pypi.org/legacy/ #python setup.py sdist upload -r https://upload.pypi.org/legacy