diff --git a/src/Applications/GEOSdas_App/edhist.py b/src/Applications/GEOSdas_App/edhist.py new file mode 100755 index 00000000..602b0e38 --- /dev/null +++ b/src/Applications/GEOSdas_App/edhist.py @@ -0,0 +1,833 @@ +#!/usr/local/other/python/GEOSpyD/4.10.3_py3.9/2022-01-14/bin/python3 +import argparse +import os +import re +import sys + +from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser +from history_rc import HistoryRc + +#....................................................................... +def main(args): + + addflag = args.addflag + arcflag = args.arcflag + infile = args.infile + listflag = args.listflag + mnthlyRC = args.mnthlyRC + plotHIST = args.plotHIST + quiet = args.quiet + sumflag = args.sumflag + + strSubs = string_substitution_requests(args) + hist_rc = HistoryRc(infile, strSubs, quiet) + apply_runtime_choices(hist_rc, args) + + if plotHIST: + plot_edit(hist_rc, args) + + if listflag: + print_list_to_stdout(hist_rc, args) + exit() + + if sumflag: + sum_slices(hist_rc, args) + exit() + + if addflag: + add_silo_mstorage_traits(hist_rc, args) + + if arcflag: + write_silo_mstorage_arc(hist_rc, args) + exit() + + write_HISTfile(hist_rc, args) + +#....................................................................... +def add_silo_mstorage_traits(hist_rc, args): + """ + Add the silo and mstorage trits to each collection, if they are not + already present. + """ + fcstflag = args.fcstflag + + tmpl_info = [ { "id": "prog", + "labels": ["prog", "traj", "ptrj"] }, + + { "id": "ana", + "labels": ["vtx", ".eta", ".sfc", ".prs"] }, + + { "id": "chem", + "labels": ["_adg_", "_aer_", "_ctm_", "_chm_", + "_gas_", "_nav_", "_tag_", + "adg_", "aer_", "ctm_", "chm_", + "gas_", "nav_", "tag_"] }, + + { "id": "diag", + "labels": ["_asm_", "_cld_", "_csp_", "_dyn_", + "_ext_", "_flx_", "_glc_", "_hwl_", + "_int_", "_lfo_", "_lnd_", "_lsf_", + "_met_", "_mst_", "_ocn_", "_odt_", + "_qdt_", "_rad_", "_slv_", "_tdt_", + "_tmp_", "_trb_", "_udt_", "_wnd_", + "asm_", "cld_", "csp_", "dyn_", + "ext_", "flx_", "glc_", "hwl_", + "int_", "lfo_", "lnd_", "lsf_", + "met_", "mst_", "ocn_", "odt_", + "qdt_", "rad_", "slv_", "tdt_", + "tmp_", "trb_", "udt_", "wnd_"] } ] + + # add silo trait + #--------------- + trait = "silo" + for cname in hist_rc.collection_names(): + + # skip restart collections + #------------------------- + if cname.endswith("_rst") and cname not in ["traj_lcv_rst", "bkg_clcv_rst"]: + continue + + # get tmpl id + #------------ + for tmpl in tmpl_info: + for label in tmpl["labels"]: + if label in cname: + id = tmpl["id"] + break + else: + continue + break + else: + id = "other" + + # set value for silo trait + #------------------------- + if fcstflag and id not in ["prog", "other"]: + val = "'prog/Y%y4/M%m2/D%d2/H%h2'" + else: + val = f"'{id}/Y%y4/M%m2'" + + # add silo trait after template trait + #------------------------------------ + index1 = hist_rc.trait_index(cname, "template") + 1 + hist_rc.add_trait(cname, trait, val, index=index1, overwrite=False) + + # add mstorage trait + #------------------- + trait = "mstorage" + val = "'yes'" + for cname in hist_rc.collection_names(): + + # skip restart collections + #------------------------- + if cname.endswith("_rst") and cname != "traj_lcv_rst": + continue + + # add mstorage trait after silo trait + #------------------------------------ + index1 = hist_rc.trait_index(cname, "silo") + 1 + hist_rc.add_trait(cname, trait, val, index=index1, overwrite=False) + +#....................................................................... +def apply_runtime_choices(hist_rc, args): + """ + Turn collections on and off by commenting or uncommenting them. + """ + excDEP = args.excDEP + excPM = args.excPM + exclude = args.exclude + incPM = args.incPM + include = args.include + + excDASH = args.excDASH + excPLUS = args.excPLUS + excludeAll = args.excludeAll + incDASH = args.incDASH + incPLUS = args.incPLUS + includeAll = args.includeAll + rmDASH = args.rmDASH + rmPLUS = args.rmPLUS + + includePM = [] + excludePM = [] + + dashlist = [] + pluslist = [] + + if rmDASH and rmPLUS: + raise Exception(f"Cannot choose both -rD and -rP") + + if rmDASH: excPLUS = True + if rmPLUS: excDASH = True + + # pattern match inclusions and exclusions + #---------------------------------------- + for cname in hist_rc.collection_names(): + for pm in incPM: + if cname.find(pm) >= 0: + includePM.append(cname) + for pm in excPM: + if cname.find(pm) >= 0: + excludePM.append(cname) + + if cname.find("+-") >= 0: + pluslist.append(cname) + elif cname.find("-") >= 0: + dashlist.append(cname) + + # check for contradicting input flags + #------------------------------------ + intersectX = set.intersection(set(include), set(exclude)) + if intersectX: + raise Exception(f"-I and -X conflict: {intersectX}") + + intersectX = set.intersection(set(incPM), set(excPM)) + if intersectX: + raise Exception(f"-Ipm and -Xpm conflict: {intersectX}") + + # global exclusion/inclusion + #--------------------------- + if excludeAll: + for cname in hist_rc.collection_names(): + hist_rc.comment_collection(cname) + + if includeAll: + for cname in hist_rc.collection_names(): + hist_rc.comment_collection(cname, flag=False) + + for cname in hist_rc.collection_names(): + + # pattern match exclusion/inclusion + #---------------------------------- + if cname in excludePM: + hist_rc.comment_collection(cname) + + if cname in includePM: + hist_rc.comment_collection(cname, flag=False) + + # ".-"/".+-" exclusion/inclusion + #------------------------------- + if excDASH: + if cname in dashlist: + hist_rc.comment_collection(cname) + + if incDASH: + if cname in dashlist: + hist_rc.comment_collection(cname, flag=False) + + if excPLUS: + if cname in pluslist: + hist_rc.comment_collection(cname) + + if incPLUS: + if cname in pluslist: + hist_rc.comment_collection(cname, flag=False) + + # file-specified exclusion/inclusion + #----------------------------------- + if cname in exclude: + hist_rc.comment_collection(cname) + + if cname in include: + hist_rc.comment_collection(cname, flag=False) + + # exclude collections with specified dependencies + #------------------------------------------------ + if excDEP: + for dep in excDEP: + hist_rc.comment_collection(dep=dep) + + # remove "-"/"+-" labels + #----------------------- + if rmDASH: + for cname in dashlist: + hist_rc.strip_dashplus(cname) + + if rmPLUS: + for cname in pluslist: + hist_rc.strip_dashplus(cname) + +#....................................................................... +def plot_edit(hist_rc, args): + """ + Edit the HISTORY information for outputting the HISTORY.rc_tmpl for + monthly_plots. + """ + mnthlyRC = args.mnthlyRC + plotHISTdir = args.plotHISTdir + + includeList = [] + for cname in hist_rc.collection_names(): + clean = cname.replace("_NCKS", "") + skip = True + if clean[-2:] in ["Cp", "Np", "Nx"]: skip = False + if clean[-3:] in ["slv", "p42"]: skip = False + if clean[-3:] in ["v72", "v73"]: skip = False + if skip: continue + includeList.append(cname) + + # get names from monthly rcfile + #------------------------------ + plotList = [] + with open(mnthlyRC, mode="r") as mrcfile: + for line in mrcfile: + cleanline = line.strip() + if cleanline == "": continue + + # remove following line to include commented entries in output + if cleanline.startswith("#"): continue + + vals = cleanline.split() + if len(vals) > 3: continue + + while len(vals) < 3: + vals.append("") + (template, processflags, htype) = vals + + if "P" not in processflags: continue + cname = template.split(".")[1] + if cname not in includeList: continue + + # comment in collections list if commented in mnthlyRC + #----------------------------------------------------- + plotList.append(cname) + if cleanline.startswith("#"): + hist_rc.comment_collection(cname) + + # write datarc template file + #--------------------------- + if plotHISTdir: + write_datarc(template, cname, plotHISTdir) + + # only include collections which are to be plotted + #------------------------------------------------- + hist_rc.trim_collections(plotList) + + # adjust traits as necessary + #--------------------------- + for cname in includeList: + hist_rc.delete_trait(cname, "end_date") + hist_rc.delete_trait(cname, "end_time") + + trait = "grads_ddf" + val = f"'>>>PLOTSDIR<<= 0: + if rmPLUS: cname = cname.replace("+-", "") + + elif cname.find("-") >= 0: + if rmDASH: cname = cname.replace("-", "") + + return cname + +#....................................................................... +def split(list): + """ + Split entries in a list on commas and colons. + """ + newlist = [] + for str in list: + newlist.extend(str.split(",")) + + list = [] + for str in newlist: + list.extend(str.split(":")) + + return list + +#....................................................................... +def string_substitution_requests(args): + """ + Gather the string substitution requests for the HISTORY.rc.tmpl file + """ + str1str2list = args.str1str2list + str1list = args.str1list + str2list = args.str2list + + strSubs = {} + for s1s2 in str1str2list: + s12 = s1s2.split("=") + if len(s12) == 2: + (str1, str2) = s12 + strSubs[str1] = str2 + else: + raise Exception(f"ERROR. ill-formatted string substitution: {s1s2}") + + if str1list and len(str1list) == len(str2list): + strSubs = strSubs | dict(zip(str1list, str2list)) + + return strSubs + +#....................................................................... +def sum_slices(hist_rc, args): + """ + """ + sumflag = args.sumflag + print("The -sum option has not yet been implemented.") + +#....................................................................... +def write_HISTfile(hist_rc, args): + """ + Write the output HISTORY template file. + """ + orderflag = args.orderflag + outfile = args.outfile + + if os.path.isfile(outfile): + outtilde = outfile + "~" + if not os.path.exists(outtilde): + os.rename(outfile, outtilde) + + hist_rc.hwrite(outfile, orderflag) + +#....................................................................... +def write_datarc(template, cname, plotHISTdir): + """ + Write data rc template file. + """ + + # write rc file for const data + #----------------------------- + if cname.startswith("const"): + dirlocSTR = r"#*(\w+)/" + dirlocPTR = re.compile(dirlocSTR) + + if not dirlocPTR.match(template): + raise Exception(f"ERROR. Unable to write datarc template for {cname}") + + dirloc = "/"+dirlocPTR.match(template).group(1) + datarc = f"{plotHISTdir}/{cname}.tmpl" + with open(datarc, mode="w") as drc: + name_tmpl = f"{cname}.nc4" + drc.write(f"DSET >>>FVHOME<<<{dirloc}>>>EXPID<<<.{name_tmpl}\n") + drc.write(f"TITLE >>>EXPID<<<\n") + drc.write(f"OPTIONS template\n") + drc.write(f"TDEF time 1 LINEAR 00:00Z01>>>TDEF<<< 1mo\n") + return + + # write rc file for all other data + #--------------------------------- + dirlocSTR = r"#*(.*M%m2/)" + dirlocPTR = re.compile(dirlocSTR) + + if not dirlocPTR.match(template): + raise Exception(f"ERROR. Unable to write datarc template for {cname}") + + dirloc = "/"+dirlocPTR.match(template).group(1) + datarc = f"{plotHISTdir}/{cname}.tmpl" + with open(datarc, mode="w") as drc: + name_tmpl = f"{cname}.monthly.%y4%m2.nc4" + drc.write(f"DSET >>>FVHOME<<<{dirloc}>>>EXPID<<<.{name_tmpl}\n") + drc.write(f"TITLE >>>EXPID<<<\n") + drc.write(f"OPTIONS template\n") + drc.write(f"TDEF time >>>NFILES<<< LINEAR 00:00Z01>>>TDEF<<< 1mo\n") + +#....................................................................... +def write_silo_mstorage_arc(hist_rc, args): + """ + Write arc files, silo_arc and mstorage_arc, based on info in HISTORY file + """ + appendflag = args.appendflag + exclude = args.exclude + fcstflag = args.fcstflag + infile = args.infile + mstorage_arc = args.mstorage_arc + silo_arc = args.silo_arc + + if appendflag: modeflag = "a" + else: modeflag = "w" + + for arcfile in (silo_arc, mstorage_arc): + with open(arcfile, mode=modeflag) as arcf: + arcf.write("#\n" + + "# -------------------\n" + + "# HISTORY COLLECTIONS\n" + + "# -------------------\n" + + f"# {infile}\n" + + "#\n") + + for cname in hist_rc.collection_names(): + + # write the collection comments + #------------------------------ + cmtLIST = hist_rc.collections_comments(cname) + if cmtLIST: + arcf.write("#\n") + for comment in cmtLIST: + arcf.write(comment+"\n") + arcf.write("#\n") + + # write template + #--------------- + silo = hist_rc.trait_value(cname, "silo") + msflag = hist_rc.trait_value(cname, "mstorage") + template = hist_rc.trait_value(cname, "template") + + (dt, punkt, ext) = template.partition(".") + if fcstflag and ext: + if "h2%n2z" in dt: + dt1 = dt.replace("%n2", "") + else: + dt1 = dt + template = f"{dt1}+{dt}.{ext}" + name1 = rm_dash_plus(cname, args) + + pestoroot = "${PESTOROOT}" + silo_path = f"{pestoroot}%s/{silo}/%s.{name1}.{template}\n" + silo_path = silo_path.replace("'", "") + silo_path = silo_path.replace(">>>NCSUFFIX<<<", "nc4") + if cname in exclude: + silo_path = "#" + silo_path + + if arcf == mstorage_arc: + if msflag.lower() != "'yes'": + silo_path = "#--" + silo_path + arcf.write(silo_path) + +#....................................................................... +if __name__ == "__main__": + """ + Edit the HISTORY.rc.tmpl file. + """ + script = os.path.basename(__file__) + + help_I = "names to include in list of collections" + help_ID = 'include collections which end with "-"' + help_IP = 'include collections which end with "+-"' + help_Iall = "include all collections except those excluded by other flags" + help_Ipm = ("match patterns used to determine which names\n"+ + "to include in list of collections") + help_X = "names to exclude from list of collections" + help_XD = 'exclude collections which end with "-"' + help_XP = 'exclude collections which end wit "+-"' + help_Xall = "exclude all collections except those included by other flags" + help_Xdep = "exclude collections which include data from source, e.g. GOCART" + help_Xpm = ("match patterns usedto determine which names\n"+ + "to exclude from list of collections") + help_db = "print extra info for debugging purposes" + help_arc = ("write arc files: silo.hist.arc and mstorage.hist.arc"+ + "(see -silo and -mstorage flags)") + help_add = ("add silo and mstorage traits to collections that do not have\n"+ + "them; if the -arc flag is not called, then the HISTORY.rc file\n"+ + "will be written with silo and mstorage traits for all collections") + help_append = ("with -arc flag; append to arc files if they already exist,"+ + "rather then overwriting") + help_fcst = "with -arc flag; write arc entries for forecast output" + help_silo= "with -arc flag; use alternate name for silohist.arc output" + help_mstorage = "with -arc flag; use alternate name for mstorage.hist.arc output" + help_i = 'edit input file "in place"; i.e. overwrite input with output' + help_infile = "input HISTORY file (default: HISTORY.rc.tmpl)" + help_list = ("all list all collections\n"+ + "inc list only the included collections\n"+ + "exc list only the excluded collections\n\n"+ + "Add ':' or ',' to end of list option to get output as\n"+ + "single string with names separated by colons or commas;\n\n"+ + f"Ex: {script} -list inc (print included collections, "+ + "one per line)\n"+ + f" {script} -list inc: (print collections "+ + "in string, separated by colons)\n"+ + f" {script} -list inc, (print collections "+ + "in string, separated by commas)\n\n") + help_o = ("name of output HISTORY template file; defaults to STDOUT;\n"+ + "if outfile is directory name, then the output will be written\n"+ + "in the directory with the same name as the infile") + help_order = "print definitions in same order as listed in top COLLECTIONS" + help_plot = ("name or directory location of monthly.rc file to\n"+ + "use for writin the monthly_plots HISTORY file;\n") + help_q = ("quiet mode; turns off some of the WARNINGs\n"+ + " n==1 turn off INFO messages\n"+ + " n==2 turn off WARNING messages\n"+ + " n==3 turn off INFO and WARNING messages\n"+ + "if flag is given without value, then defaults to 3\n\n") + help_rD = 'remove "-" from collection name endings' + help_rP = 'remove "+-" from collection name endings' + help_s = "substitute str1 with str2" + help_s1 = "substitute str1 with str2" + help_s2 = "substitute str1 with str2" + help_sum = ("sum the number of output slices per 24-hour period\n"+ + "i.e. sum of variables times levels times frequency\n"+ + " for each output collection\n\n"+ + "if n=0, then print only the final value\n"+ + "if n=1, then print details grouped by number of atmoshere levels\n"+ + "if n=2, then print details grouped by data frequency\n"+ + "* defaults to n=1 if 'n' is excluded.\n\n"+ + "You may want to use the -q flag with -sum to turn off WARNINGs") + + class CustomFormatter(argparse.RawTextHelpFormatter, + argparse.RawDescriptionHelpFormatter): + pass + + parser = argparse.ArgumentParser(description='test\ntest\ntest.', + epilog='test\ntest\ntest.', + formatter_class=CustomFormatter) + + parser = ArgumentParser(description=__doc__, formatter_class=CustomFormatter) + + group1 = parser.add_mutually_exclusive_group() + group2 = parser.add_mutually_exclusive_group() + group3 = parser.add_mutually_exclusive_group() + group4 = parser.add_mutually_exclusive_group() + + parser.add_argument("infile", + default="./HISTORY.rc.tmpl", + nargs="?", + help=help_infile) + + group1.add_argument("-o", dest="outfile", + help=help_o) + + group1.add_argument("-i", dest="inplace", + action="store_true", + help=help_i) + + group2.add_argument("-Iall", dest="includeAll", + action="store_true", + help=help_Iall) + + group2.add_argument("-Xall", dest="excludeAll", + action="store_true", + help=help_Xall) + + parser.add_argument("-I", dest="include", + action="extend", + default=[], + metavar="names", + nargs="+", + help=help_I) + + parser.add_argument("-X", dest="exclude", + action="extend", + default=[], + metavar="names", + nargs="+", + help=help_X) + + parser.add_argument("-Ipm", dest="incPM", + action="extend", + default=[], + metavar="patterns", + nargs="+", + help=help_Ipm) + + parser.add_argument("-Xpm", dest="excPM", + action="extend", + default=[], + metavar="patterns", + nargs="+", + help=help_Xpm) + + parser.add_argument("-Xdep", dest="excDEP", + action="extend", + default=[], + metavar="source", + nargs="+", + help=help_Xdep) + + group3.add_argument("-ID", dest="incDASH", + action="store_true", + help=help_ID) + + group3.add_argument("-XD", dest="excDASH", + action="store_true", + help=help_XD) + + group4.add_argument("-IP", dest="incPLUS", + action="store_true", + help=help_IP) + + group4.add_argument("-XP", dest="excPLUS", + action="store_true", + help=help_XP) + + parser.add_argument("-rD", dest="rmDASH", + action="store_true", + help=help_rD) + + parser.add_argument("-rP", dest="rmPLUS", + action="store_true", + help=help_rP) + + parser.add_argument("-s", dest="str1str2list", + action="extend", + default=[], + metavar="str1=str2", + nargs='+', + help=help_s) + + parser.add_argument("-str1", "-s1", dest="str1list", + action="extend", + default=[], + metavar="str1", + nargs="+", + help=help_s1) + + parser.add_argument("-str2", "-s2", dest="str2list", + action="extend", + default=[], + metavar="str2", + nargs="+", + help=help_s2) + + parser.add_argument("-q", dest="quiet", + choices=[1,2,3], + default=0, + nargs="?", + type=int, + help=help_q) + + parser.add_argument("-order", dest="orderflag", + action="store_true", + help=help_order) + + parser.add_argument("-db", "-debug", + action="store_true", + help=help_db) + + parser.add_argument("-plot", dest="mnthlyRC", + default=False, + nargs="?", + help=help_plot) + + parser.add_argument("-list", dest="listflag", + help=help_list) + + parser.add_argument("-sum", dest="sumflag", + default=False, + nargs="?", + help=help_sum) + + # arc file options + #----------------- + parser.add_argument("-arc", dest="arcflag", + action="store_true", + help=help_arc) + + parser.add_argument("-add", dest="addflag", + action="store_true", + help=help_add) + + parser.add_argument("-append", dest="appendflag", + action="store_true", + help=help_append) + + parser.add_argument("-fcst", dest="fcstflag", + action="store_true", + help=help_fcst) + + parser.add_argument("-silo", dest="silo_arc", + metavar="name", + help=help_silo) + + parser.add_argument("-mstorage", dest="mstorage_arc", + metavar="name", + help=help_mstorage) + + args = parser.parse_args() + + # flag defaults and dependencies + #------------------------------- + if args.sumflag is None: + args.sumflag = True + + if args.appendflag: + args.arcflag = True + + if args.arcflag: + args.mnthlyRC is None + args.addflag = True + + if not args.silo_arc: + args.silo_arc = "silo.hist.arc" + if not args.mstorage_arc: + args.mstorage_arc = "mstorage.hist.arc" + + args.plotHIST = False + if args.mnthlyRC is not False: + if args.mnthlyRC is None: + args.mnthlyRC = "monthly.rc" + + if os.path.isdir(args.mnthlyRC): + args.mnthlyRC += "/monthly.rc" + + if not os.path.isfile(args.mnthlyRC): + raise Exception(f"ERROR. Cannot find {mnthlyRC}") + + args.plotHIST = True + args.orderflag = True + + if args.quiet is None: + args.quiet = 3 + + if not args.outfile: + if args.inplace: + args.outfile = args.infile + elif args.plotHIST: + args.outfile = "HISTORY.rc_tmpl" + + if args.outfile: + if os.path.isdir(args.outfile): + if args.plotHIST: + args.outfile += "/HISTORY.rc_tmpl" + else: + args.outfile += f"/{args.infile}" + + if args.plotHIST: + args.plotHISTdir = os.path.dirname(args.outfile) + + args.excDEP = split(args.excDEP) + args.excPM = split(args.excPM) + args.exclude = split(args.exclude) + args.incPM = split(args.incPM) + args.include = split(args.include) + + args.str1str2list = split(args.str1str2list) + args.str1list = split(args.str1list) + args.str2list = split(args.str2list) + + len1 = len(args.str1list) + len2 = len(args.str2list) + if len1 != len2: + msg = f"str1list/str2list count mismatch ({len1} != {len2})" + raise Exception(f"ERROR. {msg}") + + main(args) diff --git a/src/Applications/GEOSdas_App/fvsetup b/src/Applications/GEOSdas_App/fvsetup index 66ad8704..18adb9ca 100755 --- a/src/Applications/GEOSdas_App/fvsetup +++ b/src/Applications/GEOSdas_App/fvsetup @@ -1508,7 +1508,7 @@ sub ed_g5prog_rc_new { if($cubed) { $fvtraj = "#"; $fv3traj = " " } else { $fvtraj = " "; $fv3traj = "#" } - $cmd = "$fvbin/edhist.pl $fdfn -$iflag" + $cmd = "$fvbin/edhist.pl $fdfn $iflag" . " -s \@EXPID=$expid" . " -s \@EXPDSC=$expdsc" . " -s \@EXPSRC=$cvstag" diff --git a/src/Applications/GEOSdas_App/history_rc.py b/src/Applications/GEOSdas_App/history_rc.py new file mode 100644 index 00000000..7ad81dfd --- /dev/null +++ b/src/Applications/GEOSdas_App/history_rc.py @@ -0,0 +1,1032 @@ +import os +import re + +class HistoryRc(object): + + #....................................................................... + def __init__(self, filename="HISTORY.rc.tmpl", strSubs={}, quietflag=0): + """ + Initialize HISTORY.rc.tmpl instance. + + Note: + self._info = { "prolog" : prolog, + "header" : header, + "gridlabels" : gridlabels, + "gridLIST" : gridLIST, + "collections" : collections, + "collectLIST" : collectLIST } + where + + prolog: comments at the top of the file + header: variable definitions following the prolog + gridlabels: list of grid labels + gridLIST: list of grid definitions + collections: list of collection names + collectLIST: list of collection definitions + + prolog = [ cmtLIST ] + + header = [ ("VERSION", VERSION), + ("EXPID", EXPID), + ("EXPDSC", EXPDSC) ] + + gridlabels = [ gridlabels ] + + gridLIST = [ (gridlabel, gridDEF), ... ] + gridDEF = [ ("GRID_TYPE", grid_type), + ("IM_WORLD", im_world), + ("JM_WORLD", jm_world), + ("POLE", pole), + ("DATELINE", dateline), + ("LM", lm) ] + + collections = [ (cname, cmt3) } + + collectLIST = [ (cname, (cmtLIST, collectDEF)), ... ] + + collectDEF = [ ("format", (format, cmtflag)), + ("descr", (descr, cmtflag)), + ("nbits", (nbits, cmtflag)), + ("template", (template, cmtflag)), + ("mode", (mode, cmtflag)), + ("grid_label", (gridlabel, cmtflag)), + ("frequency", (frequency, cmtflag)), + ("duration", (duration, cmtflag)), + ("end_date", (end_date, cmtflag)), + ("end_time", (end_time, cmtflag)), + ("vscale", (vscale, cmtflag)), + ("vunit", (vunit, cmtflag)), + ("vvars", (vvars, cmtflag)), + ("levels", (levelLIST, cmtflag)), + ("fields", (fieldLIST, cmtflag)) ] + fieldLIST = [ (field, cmt4), ... ] + field = [ fname, source, nname, max/min ] + + cmt3 = { "list": cmtLIST, # comment lines prior to item + "flag": cmtflag, # True, if item is commented + "at_label": at_label } # inline @ label + + cmt4 = { "list": cmtLIST, # comment lines prior to item + "flag": cmtflag, # True, if item is commented + "inline": inline, # inline comment + "at_label": at_label } # inline @ label + + """ + if not os.path.isfile(filename): + raise ValueError(f"File not found: {filename}") + self.filename = filename + self.quietflag = quietflag + self._info = {} + self.__read(strSubs) + + #....................................................................... + def __read(self, strSubs): + """ + Read information from HISTORY.rc.tmpl file. + """ + with open(self.filename, mode='r') as history_rc: + + # create RE pattern objects + #-------------------------- + headerSTR = r"\s*(\S+)\s*:\s*(\S+)" + headerPTR = re.compile(headerSTR) + + gdefSTR = r"\s*(\S+)\.(\S+)\s*:\s*(\S+)" + gdefPTR = re.compile(gdefSTR) + + stringSTR = r"\s*(\S+)" + stringPTR = re.compile(stringSTR) + + collectionsSTR = r"\s*COLLECTIONS:" + collectionsPTR = re.compile(collectionsSTR) + + cnameSTR = r"\s*(['\"])(\S+)\1" + cnamePTR = re.compile(cnameSTR) + + atSTR = r"(@\w+)\b" + atPTR = re.compile(atSTR) + + cdefSTR = r"\s*(\S+)\.(\S+)\s*:(.+)$" + cdefPTR = re.compile(cdefSTR) + + fieldSTR = r"\s*(['\"])(\S+)\1" + fieldPTR = re.compile(fieldSTR) + + blankSTR = r"\s*$" + blankPTR = re.compile(blankSTR) + + commentSTR = r"\s*#" + commentPTR = re.compile(commentSTR) + + incommentSTR = r"\s*#([\w\s]*)(['\"])\S+\2" + incommentPTR = re.compile(incommentSTR) + + endSTR = r"::" + endPTR = re.compile(endSTR) + + fieldendSTR = r"^::|\W::" + fieldendPTR = re.compile(fieldendSTR) + + look4prolog = True + look4header = False + look4glabels = False + look4gdefs = False + look4cname = False + look4clist = False + look4fields = False + + prolog = [] + header = [] + gridlabels = [] + dummy = [] + gridLIST = [] + gridDEF = [] + gridlabel_curr = "" + collections = [] + collectLIST = [] + collectDEF = [] + fieldLIST = [] + cmtLIST = [] + + startcdef = True + + duplicate = [] + conflict = [] + + # loop through lines of file + #--------------------------- + for line in history_rc: + line = line.rstrip() + if blankPTR.match(line): continue + + # string substitutions + #--------------------- + for str1 in strSubs: + str2 = strSubs[str1] + line = line.replace(str1, str2) + + # read prolog + #------------ + if look4prolog: + + if headerPTR.match(line): + look4prolog = False + look4header = True + + else: + prolog.append(line) + continue + + # extract header lines + #--------------------- + if look4header: + + if headerPTR.match(line): + (var, val) = headerPTR.match(line).groups() + + if var == "GRID_LABELS": + look4header = False + look4glabels = True + + else: + header.append((var, val)) + continue + + # extract grid labels + #-------------------- + # NOTE: the gridlabels info is read into a dummy variable now. + # the real info will be extracted later from gridLIST. + #----------------------------------------------------------- + if look4glabels: + + if headerPTR.match(line): + (var, val) = headerPTR.match(line).groups() + if var != "GRID_LABELS": + errmsg = "New variable found in GRID_LABELS list: {}" + raise Exception(errmsg.format(var)) + else: + dummy.append(val.strip()) + + elif stringPTR.search(line): + val = stringPTR.search(line).group(0) + if val != "::": + dummy.append(val.strip()) + + if endPTR.search(line): + look4glabels = False + look4gdefs = True + + continue + + # extract grid definitions + #------------------------- + if look4gdefs: + + if gdefPTR.match(line): + (gridlabel, var, val) = gdefPTR.match(line).groups() + + if gridlabel_curr == "": + gridlabel_curr = gridlabel + + if gridlabel in dict(gridLIST): + gridDEF = dict(gridLIST)[gridlabel] + + if var in dict(gridDEF): + val0 = dict(gridDEF)[var] + if val == val0: + duplicate.append(line) + else: + line = f"{gridlabel}.{var}: {val0} != {val}" + conflict.append(line) + else: + index = gridLIST.index((gridlabel, gridDEF)) + gridLIST.pop(index) + gridDEF.append((var, val)) + gridLIST.insert(index, (gridlabel, gridDEF)) + + else: + gridDEF= [] + gridDEF.append((var, val)) + gridLIST.append((gridlabel, gridDEF)) + + elif collectionsPTR.match(line): + if gridlabel_curr not in dict(gridLIST): + gridLIST.append((gridlabel_curr, gridDEF)) + + look4gdefs = False + look4cname = True + cmtLIST = [] + + if duplicate: + for ll in duplicate: + msg = f"WARNING. Duplicate grid variable: {ll}" + self.print(msg) + + if conflict: + for ll in conflict: + msg = f"WARNING. grid variable conflict: {ll}" + self.print(msg) + + # extract collections list + #------------------------- + if look4cname: + + if cnamePTR.search(line): + cname = cnamePTR.search(line).group(2) + if commentPTR.match(line): + cmtflag = True + else: + cmtflag = False + if atPTR.search(line): + at_label = atPTR.search(line).group(1) + else: + at_label = "" + + cmt3 = { "list": cmtLIST, + "flag": cmtflag, + "at_label": at_label } + + self.__update_collections(collections, cname, cmt3) + cmtLIST = [] + + elif commentPTR.match(line): + cmtLIST.append(line) + + if endPTR.search(line): + look4cname = False + look4clist = True + + continue + + # extract collection definitions + #------------------------------- + if look4clist: + + if startcdef: + if commentPTR.match(line): + cmtLIST.append(line) + + if cdefPTR.search(line): + (cname, var, val) = cdefPTR.search(line).groups() + val = val.strip("\t ,") + + if commentPTR.match(line): + cmtflag = True + else: + cmtflag = False + + if startcdef: + cname_curr = cname + startcdef = False + cmtLIST_ = cmtLIST + cmtLIST = [] + else: + if cname != cname_curr: + warn = "WARNING. Name inconsistency in {} def" + msg = "WARNING. Changing {0}.{1} to {2}.{1}" + self.print(warn.format(cname_curr)) + self.print(msg.format(cname, var, cname_curr)) + cname = cname_curr + + if var == "fields": + look4clist = False + look4fields = True + firstfield = True + + else: + collectDEF.append((var, (val, cmtflag))) + + # extract collection fields + #-------------------------- + if look4fields: + + # if multiple "fields", then store and start over + #------------------------------------------------ + if cdefPTR.search(line): + if fieldLIST: + collectDEF.append(("fields", (fieldLIST, cmtflag))) + fieldLIST = [] + + (cname, var, val) = cdefPTR.search(line).groups() + if var != "fields": + raise Exception(f"ERROR; var = {var}") + val = val.strip("\t ,") + + if fieldPTR.search(line): + field = [fld for indx, fld in fieldPTR.findall(line)] + + while len(field) < 4: + field.append("") + + cmtflag = False + inline = "" + at_label = "" + if commentPTR.match(line): + cmtflag = True + if incommentPTR.match(line): + inline = incommentPTR.match(line).group(1).strip() + if atPTR.search(line): + at_label = atPTR.search(line).group(1) + + cmt4 = { "list": cmtLIST, + "flag": cmtflag, + "inline": inline, + "at_label": at_label } + + fieldLIST.append((field, cmt4)) + cmtLIST = [] + + else: + if commentPTR.match(line): + cmtLIST.append(line) + + if fieldendPTR.search(line): + collectDEF.append(("fields", (fieldLIST, cmtflag))) + collectLIST.append((cname, (cmtLIST_, collectDEF))) + collectDEF = [] + fieldLIST = [] + cmtLIST = [] + look4fields = False + look4clist = True + startcdef = True + continue + + # disregard previous gridlabels list; + # redefine based on grid defs + #------------------------------------ + gridlabels = [ glabel for glabel, gdef in gridLIST ] + + # store extracted values + #----------------------- + self.__check_grids(gridlabels, gridLIST) + self.__check_collections(collections, collectLIST, gridlabels) + + self._info["prolog"] = prolog + self._info["header"] = header + self._info["gridlabels"] = gridlabels + self._info["gridLIST"] = gridLIST + self._info["collections"] = collections + self._info["collectLIST"] = collectLIST + + #....................................................................... + def __update_collections(self, collections, cname, cmt3): + """ + Add a new collection to the list; if a previous record exists with + the same collection name, then replace the previous record if the + previous record is commented and the new one isn't. Otherwise, discard + the new record (i.e. first record read takes precedence). + """ + appendflag = True + + # check for duplicate entry + #-------------------------- + if cname in dict(collections): + warn = "WARNING. Duplicate name in COLLECTIONS: {}" + + cmt3_ = dict(collections)[cname] + cmtflag_ = cmt3_["flag"] + cmtflag = cmt3["flag"] + + # replace if previous entry commented and new one isn't + #------------------------------------------------------ + if cmtflag_ and not cmtflag: + index = collections.index((cname, cmt3_)) + collections.pop(index) + warn += "; replaced" + else: + warn += "; discarded" + appendflag = False + + self.print(warn.format(cname)) + + # append record + #-------------- + if appendflag: + collections.append((cname, cmt3)) + + #....................................................................... + def __check_grids(self, gridlabels, gridLIST): + """ + Check for errors in the gridLIST. + """ + gvars = ["GRID_TYPE", "IM_WORLD", "JM_WORLD", "POLE", "DATELINE", "LM"] + + errcnt = 0 + for glabel, gdef in gridLIST: + for var, val in gdef: + if var not in gvars: + self.print(f"WARNING. Unexpected grid variable: {glabel}.{var}") + errcnt += 1 + + for var in gvars: + if var not in dict(gdef): + warn = "WARNING. Expected grid variable not found: {}.{}" + msg = "WARNING. Adding grid variable, {}.{}" + self.print(warn.format(glabel, var)) + self.print(msg.format(glabel, var)) + errcnt += 1 + + # set "LM" value equal to 72, if missing + #--------------------------------------- + if var == "LM": + gridDEF = dict(gridLIST)[glabel] + index = gridLIST.index((glabel, gridDEF)) + gridLIST.pop(index) + + gridDEF.append(("LM", 72)) + gridLIST.insert(index, (glabel, gridDEF)) + + #....................................................................... + def __check_collections(self, collections, collectLIST, gridlabels): + """ + Check for errors in collections and collectLIST. + """ + script = os.path.basename(__file__) + + # check that each collection definition is listed in COLLECTIONS list + # if not, then add it to the list and comment it + #----------------------------------------------- + notfound = [] + for cname, (cmtLIST, cdef) in collectLIST: + if cname not in dict(collections): + notfound.append(cname) + + if notfound: + first = True + + for cname in notfound: + self.print(f"WARNING. Added to COLLECTIONS: {cname}") + + if first: + msg1 = f"# Added by {script} script" + len1 = len(msg1)-2 + msg2 = f"# {'!'*len1}" + + cmtLIST = [ msg2, msg1, msg2 ] + first = False + else: + cmtLIST = [] + at_label = "" + + cmt3 = { "list": cmtLIST, "flag": True, "at_label": at_label } + collections.append((cname, cmt3)) + + # check that each name in COLLECTIONS list has a definition + # if not, then move name to end of the list and comment it + #--------------------------------------------------------- + notfound = [] + for cname, cmt3 in collections: + if cname not in dict(collectLIST): + index = collections.index((cname, cmt3)) + notfound.append((index, cname)) + + if notfound: + first = True + for index, cname in notfound: + self.print(f"WARNING. No definition: {cname}") + if first: + msg0 = "#" + msg1 = "# Collection name(s) without definition" + msg2 = "# -------------------------------------" + cmtLIST = [ msg0, msg1, msg2 ] + first = False + else: + cmtLIST = [] + at_label = "" + + cmt3 = { "list": cmtLIST, "flag": True, "at_label": at_label } + + collections.pop(index) + collections.append((cname, cmt3)) + + # check grid_label in each collectDEF is also in gridlabels + #---------------------------------------------------------- + errcnt = 0 + for cname, (cmtLIST, cdef) in collectLIST: + var = "grid_label" + if var in dict(cdef): + glabel = dict(cdef)[var][0] + if glabel not in gridlabels: + errmsg = "ERROR. grid_label not defined: {}.{}: {}" + print(errmsg.format(cname, var, glabel)) + errcnt += 1 + + #....................................................................... + def add_trait(self, cname, trait, val, cmtflag=False, index=None, + overwrite=False): + """ + Add a trait for collection name to collectLIST. + input parameters: + => cname: collection into which to insert trait + => trait: trait name + => val: value of trait + => cmtflag: if True, then trait is to be commented, default = False + => index: index indicating where to insert trait into cdef array; + default = append to end + => overwrite: if True, then overwrite trait value if one exists + """ + + # get collectLIST information + #---------------------------- + collectLIST = self._info["collectLIST"] + if cname not in dict(collectLIST): + self.print(f"WARNING. collection {cname} not found for add_grads_ddf()") + return + + (cmtLIST, cdef) = dict(collectLIST)[cname] + indexCL = collectLIST.index((cname, (cmtLIST, cdef))) + + # is trait already in collection? + #-------------------------------- + if trait in dict(cdef): + info = f"{trait} trait already in {cname} collection" + + # do not overwrite + #----------------- + if not overwrite: + self.print(f"INFO. {info}.") + return + + # collection already has specified trait and value + #------------------------------------------------- + (val_, cmtflag_) = dict(cdef)[trait] + if (val, cmtflag) == (val_, cmtflag_): + info += f", with (val, cmtflag) = ({val}, {cmtflag})\n" + self.print(f"INFO. {info}") + return + + # remove conflicting trait and value + #----------------------------------- + info += (". Replacing (val, cmtflag): " + + f"({val_}, {cmtflag_}) => ({val}, {cmtflag})\n") + index_ = cdef.index((trait, (val_, cmtflag_))) + cdef.pop(index_) + cdef.insert(index_, (trait, (val, cmtflag))) + + # add new trait and value + #------------------------ + elif index: + cdef.insert(index, (trait, (val, cmtflag))) + info = f"{trait} trait inserted into {cname} collection" + else: + cdef.append((trait, (val, cmtflag))) + info = f"{trait} trait appended to {cname} collection" + self.print(f"INFO. {info}") + + # replace old cname record with modified record + #---------------------------------------------- + collectLIST.pop(indexCL) + collectLIST.insert(indexCL, (cname, (cmtLIST, cdef))) + self._info["collectLIST"] = collectLIST + + #....................................................................... + def collectLIST_comments(self, cname): + """ + Return the commentLIST comments for a specified collection. + """ + collectLIST = self._info["collectLIST"] + (cmtLIST, cdef) = dict(collectLIST)[cname] + return cmtLIST + + #....................................................................... + def collection_names(self, flag=1): + """ + Generator function returns collection names one at a time. + if flag == 1, then return cnames in the order from "collections" + if flag != 1, then return cnames in the order from "collectLIST" + """ + if flag == 1: + for cname in dict(self._info["collections"]): yield cname + else: + for cname in dict(self._info["collectLIST"]): yield cname + + #....................................................................... + def collections_comments(self, cname): + """ + Return the collections comments for a specified collection. + """ + collections = self._info["collections"] + cmt3 = dict(collections)[cname] + return cmt3["list"] + + #....................................................................... + def comment_collection(self, cname=None, dep=None, flag=True): + """ + if cname is given: + comment cname collection, if flag is True + uncomment cname collection, if flag is False. + + if dep is given: + comment all collections which have dep as a field source + """ + if cname: + collections = self._info["collections"] + cmt3 = dict(collections)[cname] + + flag_ = cmt3["flag"] + if flag != flag_: + cmt3["flag"] = flag + index = collections.index((cname, cmt3)) + self._info["collections"].pop(index) + self._info["collections"].insert(index, (cname, cmt3)) + + if dep: + collections = self._info["collections"] + collectLIST = self._info["collectLIST"] + + #--------------------------------------- + # cmtflag1 = collection name cmtflag + # cmtflag2 = collection variable cmtflag + # cmtflag3 = collection field cmtflag + #--------------------------------------- + for cname, (cmtLIST, collectDEF) in collectLIST: + cmt3 = dict(collections)[cname] + cmtflag1 = cmt3["flag"] + + if not cmtflag1: + (fieldLIST, cmtflag2) = dict(collectDEF)["fields"] + if not cmtflag2: + for (field, cmt4) in fieldLIST: + cmtflag3 = cmt4["flag"] + if not cmtflag3: + [fname, source, nname, maxmin] = field + if source.upper() == dep.upper(): + self.comment_collection(cname) + + #....................................................................... + def commented(self, cname): + """ + Return True if cname is commented in COLLECTIONS list + """ + collections = self._info["collections"] + cmt3 = dict(collections)[cname] + return cmt3["flag"] + + #....................................................................... + def delete_trait(self, cname, trait): + """ + Delete trait from specified collection (cname) + """ + collectLIST = self._info["collectLIST"] + + # return if cname not found + #-------------------------- + if cname not in dict(collectLIST): + self.print(f"WARNING. collection {cname} not found for delete {trait}") + return + + # return if cname record does not contain trait + #---------------------------------------------- + (cmtLIST, cdef) = dict(collectLIST)[cname] + if trait not in dict(cdef): + self.print(f"WARNING. trait {trait} not found in collection {cname}") + return + + # find index of cname record + #--------------------------- + index = collectLIST.index((cname, (cmtLIST, cdef))) + + # remove trait from cname record + #------------------------------- + (val, cmtflag) = dict(cdef)[trait] + cdef.remove((trait, (val, cmtflag))) + + # replace old cname record and insert modified record + #---------------------------------------------------- + collectLIST.pop(index) + collectLIST.insert(index, (cname, (cmtLIST, cdef))) + + self._info["collectLIST"] = collectLIST + + #....................................................................... + def hwrite(self, newname="default", orderflag=False): + """ + Write data from self._info to output file. + + input parameters: + => newname: name of output file + """ + if newname == "default": + newname = self.filename + ".new" + + if os.path.isfile(newname): + os.remove(newname) + with open(newname, mode='w') as new: + + # write prolog + #------------- + prolog = self._info["prolog"] + for line in prolog: + new.write(line+"\n") + new.write("\n") + + # write header + #------------- + header = self._info["header"] + for var, val in header: + line = f"{var}: {val}\n" + new.write(line) + new.write("\n") + + # write gridlabels + #----------------- + gridlabels = self._info["gridlabels"] + first = True + for glabel in gridlabels: + cmtflag = False + if glabel[0:1] == "#": + glabel = glabel[1:] + cmtflag = True + if first: + if cmtflag: + new.write("GRID_LABELS:\n") + line = f" {glabel}" + else: + line = f"GRID_LABELS: {glabel}" + first = False + else: + line = f" {glabel}" + if cmtflag: + line = "#" + line[1:] + new.write(line+"\n") + new.write(" ::\n\n") + + # write gridLIST + #--------------- + gridLIST = self._info["gridLIST"] + for glab, gdef in gridLIST: + for var, val in gdef: + new.write(f"{glab}.{var}: {val}\n") + new.write("\n") + + # write collections + #------------------ + collections = self._info["collections"] + first = True + for cname, cmt3 in collections: + for line in cmt3["list"]: + new.write(line+"\n") + + if first: + if cmt3["flag"]: + new.write("COLLECTIONS:\n") + line = f" '{cname}'\n" + else: + line = f"COLLECTIONS: '{cname}'\n" + first = False + else: + line = f" '{cname}'\n" + + front = cmt3["at_label"] + if cmt3["flag"]: + front = "#" + front + line = front + line[len(front):] + new.write(line) + new.write(" ::\n\n") + + # write collection definitions + #----------------------------- + collectLIST = self._info["collectLIST"] + + if orderflag: + cnameLIST = [cname for (cname, cmt3) in collections] + else: + cnameLIST = [cname for (cname, (cmtLIST, cdef)) in collectLIST] + + for cname in cnameLIST: + (cmtLIST, cdef) = dict(collectLIST)[cname] + if cmtLIST: + for line in cmtLIST: + new.write(line+"\n") + new.write("\n") + + maxlen = max([len(f"{cname}.{vvc[0]}:") for vvc in cdef]) + for var, (val, cmtflag) in cdef: + cnamevar = "{}.{}:".format(cname, var) + front = f" {cnamevar:{maxlen}}" + front_ = f" {'':{maxlen}}" + + if var == "fields": + fldLIST = val + first = True + + maxLEN = [] + for ii in range(4): + maxii = max([len(flds[ii]) for flds, cmt4 in fldLIST]) + maxLEN.append(maxii+3) + + for flds, cmt4 in fldLIST: + cmtLIST = cmt4["list"] + cmtflag = cmt4["flag"] + inline = cmt4["inline"] + at_label = cmt4["at_label"] + + if cmtLIST: + for line in cmtLIST: + new.write(line+"\n") + + line = "" + if cmtflag: + line = "#" + if inline: + line = line + " " + inline + " " + if at_label: + line = line + at_label + " " + + if first: + start = min([1, len(line)]) + line += front[start:] + first = False + else: + start = len(line) + line += front_[start:] + + for ii in range(4): + if flds[ii] == "": continue + str = f" '{flds[ii]}'" + line = line + f" {str:{maxLEN[ii]}} ," + + line += "\n" + new.write(line) + + else: + if cmtflag: + front = "#" + front[1:] + new.write(f"{front} {val} ,\n") + + new.write(f"{'':{maxlen+4}}::\n\n") + + #....................................................................... + def print(self, msg): + """ + Print a message to standard output, turning off some messages if the + self.quietflag is non-zero. + if self.quietflag = 1, turn off INFO messages + if self.quietflag = 2, turn off WARNING messages + if self.quietflag = 3, turn off INFO messages + """ + if self.quietflag == 1: + if msg.startswith("INFO"): + return + + elif self.quietflag == 2: + if msg.startswith("WARNING"): + return + + elif self.quietflag == 3: + if msg.startswith("INFO"): + return + if msg.startswith("WARNING"): + return + + print(msg) + + #....................................................................... + def strip_dashplus(self, cname): + """ + Strip "-" or "+-" from collection name in collections and collectLIST. + """ + collections = self._info["collections"] + collectLIST = self._info["collectLIST"] + + # remove "+-" or "-" from end of cname + #------------------------------------- + if cname.find("+-") >= 0: + cname_ = cname.replace("+-", "") + + elif cname.find("-") >= 0: + cname_ = cname.replace("-", "") + + # check for cname conflict + #------------------------- + if cname_ in dict(collections): + self.print(f"WARNING. Name conflict; cannot rename {cname} to {cname_}.") + return + + # replace cname in "collections" + #------------------------------- + cmt3 = dict(collections)[cname] + index = collections.index((cname, cmt3)) + self._info["collections"].pop(index) + self._info["collections"].insert(index, (cname_, cmt3)) + + # replace cname in "collectLIST" + #------------------------------- + (cmtLIST, cdef) = dict(collectLIST)[cname] + index = collectLIST.index((cname, (cmtLIST, cdef))) + self._info["collectLIST"].pop(index) + self._info["collectLIST"].insert(index, (cname_, (cmtLIST, cdef))) + + #....................................................................... + def trait_index(self, cname, trait): + """ + Return the index for trait in the cname collectDEF list. + """ + collectLIST = self._info["collectLIST"] + if cname not in dict(collectLIST): + raise Exception(f"Error. collection {cname} not found.") + + (cmtLIST, collectDEF) = dict(collectLIST)[cname] + if trait in dict(collectDEF): + (val, cmtflag) = dict(collectDEF)[trait] + index = collectDEF.index((trait, (val, cmtflag))) + else: + index = -1 + + return index + + #....................................................................... + def trait_value(self, cname, trait): + """ + Return the value of a collection trait. + """ + collectLIST = self._info["collectLIST"] + if cname not in dict(collectLIST): + raise Exception(f"Error. collection {cname} not found.") + + (cmtLIST, collectDEF) = dict(collectLIST)[cname] + if trait in dict(collectDEF): + (val, cmtflag) = dict(collectDEF)[trait] + else: + val = None + + return val + + #....................................................................... + def trim_collections(self, cnameLIST, incflag=True): + """ + Remove collections from the data: + + input parameters: + => cnameLIST: list of collection names + => incflag: + if true, then remove all collections except those in cnameLIST + if false, then remove cnameLIST collections + """ + collections = self._info["collections"] + collectLIST = self._info["collectLIST"] + + for cname in dict(collections): + if cname in cnameLIST: + if incflag: + continue + else: + if not incflag: + continue + + cmt3 = dict(collections)[cname] + collections.remove((cname, cmt3)) + + for cname in dict(collectLIST): + if cname in cnameLIST: + if incflag: + continue + else: + if not incflag: + continue + + (cmtLIST, cdef) = dict(collectLIST)[cname] + collectLIST.remove((cname, (cmtLIST, cdef))) + + self._info["collections"] = collections + self._info["collectLIST"] = collectLIST