diff --git a/.gitmodules b/.gitmodules index 721ab25ea1..9e80e983eb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -102,73 +102,66 @@ [submodule "cism"] path = components/cism url = https://github.com/ESCOMP/CISM-wrapper -fxtag = cismwrap_2_2_001 +fxtag = cismwrap_2_2_002 fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/ESCOMP/CISM-wrapper [submodule "rtm"] path = components/rtm url = https://github.com/ESCOMP/RTM -fxtag = rtm1_0_79 +fxtag = rtm1_0_80 fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/ESCOMP/RTM [submodule "mosart"] path = components/mosart url = https://github.com/ESCOMP/MOSART -fxtag = mosart1_0_49 +fxtag = mosart1.1.02 fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/ESCOMP/MOSART [submodule "mizuRoute"] path = components/mizuRoute url = https://github.com/ESCOMP/mizuRoute -fxtag = cesm-coupling.n02_v2.1.2 +fxtag = cesm-coupling.n02_v2.1.3 fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/ESCOMP/mizuRoute [submodule "ccs_config"] path = ccs_config url = https://github.com/ESMCI/ccs_config_cesm.git -fxtag = ccs_config_cesm0.0.106 +fxtag = ccs_config_cesm1.0.0 fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/ESMCI/ccs_config_cesm.git [submodule "cime"] path = cime url = https://github.com/ESMCI/cime -fxtag = cime6.0.246 +fxtag = cime6.1.0 fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/ESMCI/cime [submodule "cmeps"] path = components/cmeps url = https://github.com/ESCOMP/CMEPS.git -fxtag = cmeps0.14.67 +fxtag = cmeps1.0.2 fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/ESCOMP/CMEPS.git [submodule "cdeps"] path = components/cdeps url = https://github.com/ESCOMP/CDEPS.git -fxtag = cdeps1.0.34 +fxtag = cdeps1.0.43 fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/ESCOMP/CDEPS.git [submodule "share"] path = share url = https://github.com/ESCOMP/CESM_share -fxtag = share1.0.19 +fxtag = share1.1.2 fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/ESCOMP/CESM_share -[submodule "mct"] -path = libraries/mct -url = https://github.com/MCSclimate/MCT -fxtag = MCT_2.11.0 -fxrequired = ToplevelRequired -fxDONOTUSEurl = https://github.com/MCSclimate/MCT - [submodule "parallelio"] path = libraries/parallelio url = https://github.com/NCAR/ParallelIO @@ -186,7 +179,7 @@ fxDONOTUSEurl = https://github.com/NCAR/ParallelIO [submodule "clm"] path = components/clm url = https://github.com/ESCOMP/CTSM -fxtag = ctsm5.2.007 +fxtag = ctsm5.2.009 fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/ESCOMP/CTSM diff --git a/.lib/git-fleximod/CODE_OF_CONDUCT.md b/.lib/git-fleximod/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000..84f2925bba --- /dev/null +++ b/.lib/git-fleximod/CODE_OF_CONDUCT.md @@ -0,0 +1,107 @@ +# Contributor Code of Conduct +_The Contributor Code of Conduct is for participants in our software projects and community._ + +## Our Pledge +We, as contributors, creators, stewards, and maintainers (participants), of **git-fleximod** pledge to make participation in +our software, system or hardware project and community a safe, productive, welcoming and inclusive experience for everyone. +All participants are required to abide by this Code of Conduct. +This includes respectful treatment of everyone regardless of age, body size, disability, ethnicity, gender identity or expression, +level of experience, nationality, political affiliation, veteran status, pregnancy, genetic information, physical appearance, race, +religion, or sexual orientation, as well as any other characteristic protected under applicable US federal or state law. + +## Our Standards +Examples of behaviors that contribute to a positive environment include: + +* All participants are treated with respect and consideration, valuing a diversity of views and opinions +* Be considerate, respectful, and collaborative +* Communicate openly with respect for others, critiquing ideas rather than individuals and gracefully accepting criticism +* Acknowledging the contributions of others +* Avoid personal attacks directed toward other participants +* Be mindful of your surroundings and of your fellow participants +* Alert UCAR staff and suppliers/vendors if you notice a dangerous situation or someone in distress +* Respect the rules and policies of the project and venue + +Examples of unacceptable behavior include, but are not limited to: + +* Harassment, intimidation, or discrimination in any form +* Physical, verbal, or written abuse by anyone to anyone, including repeated use of pronouns other than those requested +* Unwelcome sexual attention or advances +* Personal attacks directed at other guests, members, participants, etc. +* Publishing others' private information, such as a physical or electronic address, without explicit permission +* Alarming, intimidating, threatening, or hostile comments or conduct +* Inappropriate use of nudity and/or sexual images +* Threatening or stalking anyone, including a participant +* Other conduct which could reasonably be considered inappropriate in a professional setting + +## Scope +This Code of Conduct applies to all spaces managed by the Project whether they be physical, online or face-to-face. +This includes project code, code repository, associated web pages, documentation, mailing lists, project websites and wiki pages, +issue tracker, meetings, telecons, events, project social media accounts, and any other forums created by the project team which the +community uses for communication. +In addition, violations of this Code of Conduct outside these spaces may affect a person's ability to participate within them. +Representation of a project may be further defined and clarified by project maintainers. + +## Community Responsibilities +Everyone in the community is empowered to respond to people who are showing unacceptable behavior. +They can talk to them privately or publicly. +Anyone requested to stop unacceptable behavior is expected to comply immediately. +If the behavior continues concerns may be brought to the project administrators or to any other party listed in the +[Reporting](#reporting) section below. + +## Project Administrator Responsibilities +Project administrators are responsible for clarifying the standards of acceptable behavior and are encouraged to model appropriate +behavior and provide support when people in the community point out inappropriate behavior. +Project administrator(s) are normally the ones that would be tasked to carry out the actions in the [Consequences](#consequences) +section below. + +Project administrators are also expected to keep this Code of Conduct updated with the main one housed at UCAR, as listed below in +the [Attribution](#attribution) section. + +## Reporting +Instances of unacceptable behavior can be brought to the attention of the project administrator(s) who may take any action as +outlined in the [Consequences](#consequences) section below. +However, making a report to a project administrator is not considered an 'official report' to UCAR. + +Instances of unacceptable behavior may also be reported directly to UCAR pursuant to [UCAR's Harassment Reporting and Complaint +Procedure](https://www2.fin.ucar.edu/procedures/hr/harassment-reporting-and-complaint-procedure), or anonymously through [UCAR's +EthicsPoint Hotline](https://www2.fin.ucar.edu/ethics/anonymous-reporting). + +Complaints received by UCAR will be handled pursuant to the procedures outlined in UCAR's Harassment Reporting and Complaint +Procedure. +Complaints to UCAR will be held as confidential as practicable under the circumstances, and retaliation against a person who +initiates a complaint or an inquiry about inappropriate behavior will not be tolerated. + +Any Contributor can use these reporting methods even if they are not directly affiliated with UCAR. +The Frequently Asked Questions (FAQ) page for reporting is [here](https://www2.fin.ucar.edu/procedures/hr/reporting-faqs). + +## Consequences +Upon receipt of a complaint, the project administrator(s) may take any action deemed necessary and appropriate under the +circumstances. +Such action can include things such as: removing, editing, or rejecting comments, commits, code, wiki edits, email, issues, and +other contributions that are not aligned to this Code of Conduct, or banning temporarily or permanently any contributor for other +behaviors that are deemed inappropriate, threatening, offensive, or harmful. +Project administrators also have the right to report violations to UCAR HR and/or UCAR's Office of Diversity, Equity and Inclusion +(ODEI), as well as a participant's home institution and/or law enforcement. +In the event an incident is reported to UCAR, UCAR will follow its Harassment Reporting and Complaint Procedure. + +## Process for Changes +All UCAR managed projects are required to adopt this Contributor Code of Conduct. +Adoption is assumed even if not expressly stated in the repository. +Projects should fill in sections where prompted with project-specific information, including, project name and adoption date. + +Projects that adopt this Code of Conduct need to stay up to date with UCAR's Contributor Code of Conduct, linked with a DOI in the +[Attribution](#attribution) section below. +Projects can make limited substantive changes to the Code of Conduct, however, the changes must be limited in scope and may not +contradict the UCAR Contributor Code of Conduct. + +## Attribution +This Code of Conduct was originally adapted from the [Contributor Covenant](http://contributor-covenant.org/version/1/4), version +1.4. +We then aligned it with the UCAR Participant Code of Conduct, which also borrows from the American Geophysical Union (AGU) Code of +Conduct. +The UCAR Participant Code of Conduct applies to both UCAR employees as well as participants in activities run by UCAR. +The original version of this for all software projects that have strong management from UCAR or UCAR staff is available on the UCAR +website at https://doi.org/10.5065/6w2c-a132. +The date that it was adopted by this project was **Feb/13/2018**. +When responding to complaints, UCAR HR and ODEI will do so based on the latest published version. +Therefore, any project-specific changes should follow the [Process for Changes](#process-for-changes) section above. diff --git a/.lib/git-fleximod/git_fleximod/cli.py b/.lib/git-fleximod/git_fleximod/cli.py index bc099fcbc0..fdcf102a19 100644 --- a/.lib/git-fleximod/git_fleximod/cli.py +++ b/.lib/git-fleximod/git_fleximod/cli.py @@ -2,7 +2,7 @@ import argparse from git_fleximod import utils -__version__ = "0.7.7" +__version__ = "0.8.2" def find_root_dir(filename=".gitmodules"): """ finds the highest directory in tree @@ -26,7 +26,7 @@ def find_root_dir(filename=".gitmodules"): def get_parser(): description = """ - %(prog)s manages checking out groups of gitsubmodules with addtional support for Earth System Models + %(prog)s manages checking out groups of gitsubmodules with additional support for Earth System Models """ parser = argparse.ArgumentParser( description=description, formatter_class=argparse.RawDescriptionHelpFormatter diff --git a/.lib/git-fleximod/git_fleximod/git_fleximod.py b/.lib/git-fleximod/git_fleximod/git_fleximod.py index ca5f90622d..4595cd2ab2 100755 --- a/.lib/git-fleximod/git_fleximod/git_fleximod.py +++ b/.lib/git-fleximod/git_fleximod/git_fleximod.py @@ -13,14 +13,14 @@ from git_fleximod import cli from git_fleximod.gitinterface import GitInterface from git_fleximod.gitmodules import GitModules -from configparser import NoOptionError +from git_fleximod.submodule import Submodule # logger variable is global logger = None def fxrequired_allowed_values(): - return ["ToplevelRequired", "ToplevelOptional", "AlwaysRequired", "AlwaysOptional"] + return ["ToplevelRequired", "ToplevelOptional", "AlwaysRequired", "AlwaysOptional", "TopLevelRequired", "TopLevelOptional"] def commandline_arguments(args=None): @@ -33,14 +33,9 @@ def commandline_arguments(args=None): # explicitly listing a component overrides the optional flag if options.optional or options.components: - fxrequired = [ - "ToplevelRequired", - "ToplevelOptional", - "AlwaysRequired", - "AlwaysOptional", - ] + fxrequired = fxrequired_allowed_values() else: - fxrequired = ["ToplevelRequired", "AlwaysRequired"] + fxrequired = ["ToplevelRequired", "AlwaysRequired", "TopLevelRequired"] action = options.action if not action: @@ -98,7 +93,8 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master """ logger.info("Called sparse_checkout for {}".format(name)) rgit = GitInterface(root_dir, logger) - superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree") + superroot = git_toplevelroot(root_dir, logger) + if superroot: gitroot = superroot.strip() else: @@ -154,6 +150,8 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master if os.path.isdir(os.path.join(root_dir, path, ".git")): with utils.pushd(sprep_repo): + if os.path.isdir(os.path.join(topgit,".git")): + shutil.rmtree(os.path.join(topgit,".git")) shutil.move(".git", topgit) with open(".git", "w") as f: f.write("gitdir: " + os.path.relpath(topgit)) @@ -166,7 +164,9 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master return with utils.pushd(sprep_repo): - shutil.copy(sparsefile, gitsparse) + if os.path.isfile(sparsefile): + shutil.copy(sparsefile, gitsparse) + # Finally checkout the repo sprepo_git.git_operation("fetch", "origin", "--tags") @@ -176,283 +176,81 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master rgit.config_set_value(f'submodule "{name}"', "active", "true") rgit.config_set_value(f'submodule "{name}"', "url", url) - -def single_submodule_checkout( - root, name, path, url=None, tag=None, force=False, optional=False -): - """ - This function checks out a single git submodule. - - Parameters: - root (str): The root directory for the git operation. - name (str): The name of the submodule. - path (str): The path to the submodule. - url (str, optional): The URL of the submodule. Defaults to None. - tag (str, optional): The tag to checkout. Defaults to None. - force (bool, optional): If set to True, forces the checkout operation. Defaults to False. - optional (bool, optional): If set to True, the submodule is considered optional. Defaults to False. - - Returns: - None - """ - # function implementation... - git = GitInterface(root, logger) - repodir = os.path.join(root, path) - logger.info("Checkout {} into {}/{}".format(name, root, path)) - # if url is provided update to the new url - tmpurl = None - repo_exists = False - if os.path.exists(os.path.join(repodir, ".git")): - logger.info("Submodule {} already checked out".format(name)) - repo_exists = True - # Look for a .gitmodules file in the newly checkedout repo - if not repo_exists and url: - # ssh urls cause problems for those who dont have git accounts with ssh keys defined - # but cime has one since e3sm prefers ssh to https, because the .gitmodules file was - # opened with a GitModules object we don't need to worry about restoring the file here - # it will be done by the GitModules class - if url.startswith("git@"): - tmpurl = url - url = url.replace("git@github.com:", "https://github.com/") - git.git_operation("clone", url, path) - smgit = GitInterface(repodir, logger) - if not tag: - tag = smgit.git_operation("describe", "--tags", "--always").rstrip() - smgit.git_operation("checkout", tag) - # Now need to move the .git dir to the submodule location - rootdotgit = os.path.join(root, ".git") - if os.path.isfile(rootdotgit): - with open(rootdotgit) as f: - line = f.readline() - if line.startswith("gitdir: "): - rootdotgit = line[8:].rstrip() - - newpath = os.path.abspath(os.path.join(root, rootdotgit, "modules", name)) - if os.path.exists(newpath): - shutil.rmtree(os.path.join(repodir, ".git")) - else: - shutil.move(os.path.join(repodir, ".git"), newpath) - - with open(os.path.join(repodir, ".git"), "w") as f: - f.write("gitdir: " + os.path.relpath(newpath, start=repodir)) - - if not os.path.exists(repodir): - parent = os.path.dirname(repodir) - if not os.path.isdir(parent): - os.makedirs(parent) - git.git_operation("submodule", "add", "--name", name, "--", url, path) - - if not repo_exists or not tmpurl: - git.git_operation("submodule", "update", "--init", "--", path) - - if os.path.exists(os.path.join(repodir, ".gitmodules")): - # recursively handle this checkout - print(f"Recursively checking out submodules of {name}") - gitmodules = GitModules(logger, confpath=repodir) - requiredlist = ["AlwaysRequired"] - if optional: - requiredlist.append("AlwaysOptional") - submodules_checkout(gitmodules, repodir, requiredlist, force=force) - if not os.path.exists(os.path.join(repodir, ".git")): - utils.fatal_error( - f"Failed to checkout {name} {repo_exists} {tmpurl} {repodir} {path}" - ) - - if tmpurl: - print(git.git_operation("restore", ".gitmodules")) - - return - -def add_remote(git, url): - remotes = git.git_operation("remote", "-v") - newremote = "newremote.00" - if url in remotes: - for line in remotes: - if url in line and "fetch" in line: - newremote = line.split()[0] - break - else: - i = 0 - while "newremote" in remotes: - i = i + 1 - newremote = f"newremote.{i:02d}" - git.git_operation("remote", "add", newremote, url) - return newremote - -def submodules_status(gitmodules, root_dir, toplevel=False): +def init_submodule_from_gitmodules(gitmodules, name, root_dir, logger): + path = gitmodules.get(name, "path") + url = gitmodules.get(name, "url") + assert path and url, f"Malformed .gitmodules file {path} {url}" + tag = gitmodules.get(name, "fxtag") + fxurl = gitmodules.get(name, "fxDONOTUSEurl") + fxsparse = gitmodules.get(name, "fxsparse") + fxrequired = gitmodules.get(name, "fxrequired") + return Submodule(root_dir, name, path, url, fxtag=tag, fxurl=fxurl, fxsparse=fxsparse, fxrequired=fxrequired, logger=logger) + +def submodules_status(gitmodules, root_dir, toplevel=False, depth=0): testfails = 0 localmods = 0 needsupdate = 0 + wrapper = textwrap.TextWrapper(initial_indent=' '*(depth*10), width=120,subsequent_indent=' '*(depth*20)) for name in gitmodules.sections(): - path = gitmodules.get(name, "path") - tag = gitmodules.get(name, "fxtag") - url = gitmodules.get(name, "url") - required = gitmodules.get(name, "fxrequired") - level = required and "Toplevel" in required - if not path: - utils.fatal_error("No path found in .gitmodules for {}".format(name)) - newpath = os.path.join(root_dir, path) - logger.debug("newpath is {}".format(newpath)) - if not os.path.exists(os.path.join(newpath, ".git")): - rootgit = GitInterface(root_dir, logger) - # submodule commands use path, not name - url = url.replace("git@github.com:", "https://github.com/") - tags = rootgit.git_operation("ls-remote", "--tags", url) - result = rootgit.git_operation("submodule","status",newpath).split() - ahash = None - if result: - ahash = result[0][1:] - hhash = None - atag = None - needsupdate += 1 - if not toplevel and level: - continue - for htag in tags.split("\n"): - if htag.endswith('^{}'): - htag = htag[:-3] - if ahash and not atag and ahash in htag: - atag = (htag.split()[1])[10:] - if tag and not hhash and htag.endswith(tag): - hhash = htag.split()[0] - if hhash and atag: - break - if tag and (ahash == hhash or atag == tag): - print(f"e {name:>20} not checked out, aligned at tag {tag}") - elif tag: - ahash = rootgit.git_operation( - "submodule", "status", "{}".format(path) - ).rstrip() - ahash = ahash[1 : len(tag) + 1] - if tag == ahash: - print(f"e {name:>20} not checked out, aligned at hash {ahash}") - else: - print( - f"e {name:>20} not checked out, out of sync at tag {atag}, expected tag is {tag}" - ) - testfails += 1 - else: - print(f"e {name:>20} has no fxtag defined in .gitmodules") - testfails += 1 - else: - with utils.pushd(newpath): - git = GitInterface(newpath, logger) - atag = git.git_operation("describe", "--tags", "--always").rstrip() - ahash = git.git_operation("rev-list", "HEAD").partition("\n")[0] - rurl = git.git_operation("ls-remote","--get-url").rstrip() - if rurl != url: - remote = add_remote(git, url) - git.git_operation("fetch", remote) - if tag and atag == tag: - print(f" {name:>20} at tag {tag}") - elif tag and ahash[: len(tag)] == tag: - print(f" {name:>20} at hash {ahash}") - elif atag == ahash: - print(f" {name:>20} at hash {ahash}") - elif tag: - print( - f"s {name:>20} {atag} {ahash} is out of sync with .gitmodules {tag}" - ) - testfails += 1 - needsupdate += 1 - else: - print( - f"e {name:>20} has no fxtag defined in .gitmodules, module at {atag}" - ) - testfails += 1 - - status = git.git_operation("status", "--ignore-submodules", "-uno") - if "nothing to commit" not in status: - localmods = localmods + 1 - print("M" + textwrap.indent(status, " ")) - + submod = init_submodule_from_gitmodules(gitmodules, name, root_dir, logger) + + result,n,l,t = submod.status() + testfails += t + localmods += l + needsupdate += n + if toplevel or not submod.toplevel(): + print(wrapper.fill(result)) + subdir = os.path.join(root_dir, submod.path) + if os.path.exists(os.path.join(subdir, ".gitmodules")): + submod = GitModules(logger, confpath=subdir) + t,l,n = submodules_status(submod, subdir, depth=depth+1) + testfails += t + localmods += l + needsupdate += n + return testfails, localmods, needsupdate +def git_toplevelroot(root_dir, logger): + rgit = GitInterface(root_dir, logger) + superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree") + return superroot def submodules_update(gitmodules, root_dir, requiredlist, force): - _, localmods, needsupdate = submodules_status(gitmodules, root_dir) - - if localmods and not force: - local_mods_output() - return - if needsupdate == 0: - return - for name in gitmodules.sections(): - fxtag = gitmodules.get(name, "fxtag") - path = gitmodules.get(name, "path") - url = gitmodules.get(name, "url") - logger.info( - "name={} path={} url={} fxtag={} requiredlist={} ".format( - name, os.path.join(root_dir, path), url, fxtag, requiredlist - ) - ) - - fxrequired = gitmodules.get(name, "fxrequired") - assert fxrequired in fxrequired_allowed_values() - rgit = GitInterface(root_dir, logger) - superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree") - - fxsparse = gitmodules.get(name, "fxsparse") - + submod = init_submodule_from_gitmodules(gitmodules, name, root_dir, logger) + + _, needsupdate, localmods, testfails = submod.status() + if not submod.fxrequired: + submod.fxrequired = "AlwaysRequired" + fxrequired = submod.fxrequired + allowedvalues = fxrequired_allowed_values() + assert fxrequired in allowedvalues + + superroot = git_toplevelroot(root_dir, logger) + if ( fxrequired - and (superroot and "Toplevel" in fxrequired) - or fxrequired not in requiredlist + and ((superroot and "Toplevel" in fxrequired) + or fxrequired not in requiredlist) ): - if "ToplevelOptional" == fxrequired: - print("Skipping optional component {}".format(name)) - continue - if fxsparse: - logger.debug( - "Callng submodule_sparse_checkout({}, {}, {}, {}, {}, {}".format( - root_dir, name, url, path, fxsparse, fxtag - ) - ) - submodule_sparse_checkout(root_dir, name, url, path, fxsparse, tag=fxtag) - else: - logger.info( - "Calling submodule_checkout({},{},{},{})".format( - root_dir, name, path, url - ) - ) - - single_submodule_checkout( - root_dir, - name, - path, - url=url, - tag=fxtag, - force=force, - optional=("AlwaysOptional" in requiredlist), - ) - - if os.path.exists(os.path.join(path, ".git")): - submoddir = os.path.join(root_dir, path) - with utils.pushd(submoddir): - git = GitInterface(submoddir, logger) - # first make sure the url is correct - upstream = git.git_operation("ls-remote", "--get-url").rstrip() - newremote = "origin" - if upstream != url: - add_remote(git, url) - - tags = git.git_operation("tag", "-l") - if fxtag and fxtag not in tags: - git.git_operation("fetch", newremote, "--tags") - atag = git.git_operation("describe", "--tags", "--always").rstrip() - if fxtag and fxtag != atag: - try: - git.git_operation("checkout", fxtag) - print(f"{name:>20} updated to {fxtag}") - except Exception as error: - print(error) - elif not fxtag: - print(f"No fxtag found for submodule {name:>20}") - else: - print(f"{name:>20} up to date.") - + if "Optional" in fxrequired and "Optional" not in requiredlist: + if fxrequired.startswith("Always"): + print(f"Skipping optional component {name:>20}") + continue + optional = "AlwaysOptional" in requiredlist + if fxrequired in requiredlist: + submod.update() + repodir = os.path.join(root_dir, submod.path) + if os.path.exists(os.path.join(repodir, ".gitmodules")): + # recursively handle this checkout + print(f"Recursively checking out submodules of {name}") + gitsubmodules = GitModules(submod.logger, confpath=repodir) + newrequiredlist = ["AlwaysRequired"] + if optional: + newrequiredlist.append("AlwaysOptional") + submodules_update(gitsubmodules, repodir, newrequiredlist, force=force) def local_mods_output(): text = """\ @@ -467,62 +265,6 @@ def local_mods_output(): """ print(text) - -# checkout is done by update if required so this function may be depricated -def submodules_checkout(gitmodules, root_dir, requiredlist, force=False): - """ - This function checks out all git submodules based on the provided parameters. - - Parameters: - gitmodules (ConfigParser): The gitmodules configuration. - root_dir (str): The root directory for the git operation. - requiredlist (list): The list of required modules. - force (bool, optional): If set to True, forces the checkout operation. Defaults to False. - - Returns: - None - """ - # function implementation... - print("") - _, localmods, needsupdate = submodules_status(gitmodules, root_dir) - if localmods and not force: - local_mods_output() - return - if not needsupdate: - return - for name in gitmodules.sections(): - fxrequired = gitmodules.get(name, "fxrequired") - fxsparse = gitmodules.get(name, "fxsparse") - fxtag = gitmodules.get(name, "fxtag") - path = gitmodules.get(name, "path") - url = gitmodules.get(name, "url") - if fxrequired and fxrequired not in requiredlist: - if "Optional" in fxrequired: - print("Skipping optional component {}".format(name)) - continue - - if fxsparse: - logger.debug( - "Callng submodule_sparse_checkout({}, {}, {}, {}, {}, {}".format( - root_dir, name, url, path, fxsparse, fxtag - ) - ) - submodule_sparse_checkout(root_dir, name, url, path, fxsparse, tag=fxtag) - else: - logger.debug( - "Calling submodule_checkout({},{},{})".format(root_dir, name, path) - ) - single_submodule_checkout( - root_dir, - name, - path, - url=url, - tag=fxtag, - force=force, - optional="AlwaysOptional" in requiredlist, - ) - - def submodules_test(gitmodules, root_dir): """ This function tests the git submodules based on the provided parameters. @@ -545,7 +287,7 @@ def submodules_test(gitmodules, root_dir): # and that sparse checkout files exist for name in gitmodules.sections(): url = gitmodules.get(name, "url") - fxurl = gitmodules.get(name, "fxDONOTMODIFYurl") + fxurl = gitmodules.get(name, "fxDONOTUSEurl") fxsparse = gitmodules.get(name, "fxsparse") path = gitmodules.get(name, "path") fxurl = fxurl[:-4] if fxurl.endswith(".git") else fxurl diff --git a/.lib/git-fleximod/git_fleximod/gitinterface.py b/.lib/git-fleximod/git_fleximod/gitinterface.py index 93ae38ecde..5831201446 100644 --- a/.lib/git-fleximod/git_fleximod/gitinterface.py +++ b/.lib/git-fleximod/git_fleximod/gitinterface.py @@ -49,8 +49,14 @@ def _init_git_repo(self): # pylint: disable=unused-argument def git_operation(self, operation, *args, **kwargs): - command = self._git_command(operation, *args) - self.logger.info(command) + newargs = [] + for a in args: + # Do not use ssh interface + if isinstance(a, str): + a = a.replace("git@github.com:", "https://github.com/") + newargs.append(a) + + command = self._git_command(operation, *newargs) if isinstance(command, list): try: return utils.execute_subprocess(command, output_to_caller=True) @@ -62,7 +68,11 @@ def git_operation(self, operation, *args, **kwargs): def config_get_value(self, section, name): if self._use_module: config = self.repo.config_reader() - return config.get_value(section, name) + try: + val = config.get_value(section, name) + except: + val = None + return val else: cmd = ("git", "-C", str(self.repo_path), "config", "--get", f"{section}.{name}") output = utils.execute_subprocess(cmd, output_to_caller=True) diff --git a/.lib/git-fleximod/git_fleximod/gitmodules.py b/.lib/git-fleximod/git_fleximod/gitmodules.py index 7e4e05394a..cf8b350dd6 100644 --- a/.lib/git-fleximod/git_fleximod/gitmodules.py +++ b/.lib/git-fleximod/git_fleximod/gitmodules.py @@ -1,4 +1,4 @@ -import shutil +import shutil, os from pathlib import Path from configparser import RawConfigParser, ConfigParser from .lstripreader import LstripReader diff --git a/.lib/git-fleximod/git_fleximod/submodule.py b/.lib/git-fleximod/git_fleximod/submodule.py new file mode 100644 index 0000000000..48657272ee --- /dev/null +++ b/.lib/git-fleximod/git_fleximod/submodule.py @@ -0,0 +1,403 @@ +import os +import textwrap +import shutil +import string +from configparser import NoOptionError +from git_fleximod import utils +from git_fleximod.gitinterface import GitInterface + +class Submodule(): + """ + Represents a Git submodule with enhanced features for flexible management. + + Attributes: + name (str): The name of the submodule. + root_dir (str): The root directory of the main project. + path (str): The relative path from the root directory to the submodule. + url (str): The URL of the submodule repository. + fxurl (str): The URL for flexible submodule management (optional). + fxtag (str): The tag for flexible submodule management (optional). + fxsparse (str): Path to the sparse checkout file relative to the submodule path, see git-sparse-checkout for details (optional). + fxrequired (str): Indicates if the submodule is optional or required (optional). + logger (logging.Logger): Logger instance for logging (optional). + """ + def __init__(self, root_dir, name, path, url, fxtag=None, fxurl=None, fxsparse=None, fxrequired=None, logger=None): + """ + Initializes a new Submodule instance with the provided attributes. + """ + self.name = name + self.root_dir = root_dir + self.path = path + self.url = url + self.fxurl = fxurl + self.fxtag = fxtag + self.fxsparse = fxsparse + if fxrequired: + self.fxrequired = fxrequired + else: + self.fxrequired = "AlwaysRequired" + self.logger = logger + + def status(self): + """ + Checks the status of the submodule and returns 4 parameters: + - result (str): The status of the submodule. + - needsupdate (bool): An indicator if the submodule needs to be updated. + - localmods (bool): An indicator if the submodule has local modifications. + - testfails (bool): An indicator if the submodule has failed a test, this is used for testing purposes. + """ + + smpath = os.path.join(self.root_dir, self.path) + testfails = False + localmods = False + needsupdate = False + ahash = None + optional = "" + if "Optional" in self.fxrequired: + optional = " (optional)" + required = None + level = None + if not os.path.exists(os.path.join(smpath, ".git")): + rootgit = GitInterface(self.root_dir, self.logger) + # submodule commands use path, not name + tags = rootgit.git_operation("ls-remote", "--tags", self.url) + result = rootgit.git_operation("submodule","status",smpath).split() + + if result: + ahash = result[0][1:] + hhash = None + atag = None + for htag in tags.split("\n"): + if htag.endswith('^{}'): + htag = htag[:-3] + if ahash and not atag and ahash in htag: + atag = (htag.split()[1])[10:] + if self.fxtag and not hhash and htag.endswith(self.fxtag): + hhash = htag.split()[0] + if hhash and atag: + break + if self.fxtag and (ahash == hhash or atag == self.fxtag): + result = f"e {self.name:>20} not checked out, aligned at tag {self.fxtag}{optional}" + needsupdate = True + elif self.fxtag: + ahash = rootgit.git_operation( + "submodule", "status", "{}".format(self.path) + ).rstrip() + ahash = ahash[1 : len(self.fxtag) + 1] + if self.fxtag == ahash: + result = f"e {self.name:>20} not checked out, aligned at hash {ahash}{optional}" + else: + result = f"e {self.name:>20} not checked out, out of sync at tag {atag}, expected tag is {self.fxtag}{optional}" + testfails = True + needsupdate = True + else: + result = f"e {self.name:>20} has no fxtag defined in .gitmodules{optional}" + testfails = True + else: + with utils.pushd(smpath): + git = GitInterface(smpath, self.logger) + remote = git.git_operation("remote").rstrip() + if remote == '': + result = f"e {self.name:>20} has no associated remote" + testfails = True + needsupdate = True + return result, needsupdate, localmods, testfails + rurl = git.git_operation("ls-remote","--get-url").rstrip() + line = git.git_operation("log", "--pretty=format:\"%h %d").partition('\n')[0] + parts = line.split() + ahash = parts[0][1:] + if len(parts) > 3: + atag = parts[3][:-1] + else: + atag = None + + #print(f"line is {line} ahash is {ahash} atag is {atag}") + # atag = git.git_operation("describe", "--tags", "--always").rstrip() + # ahash = git.git_operation("rev-list", "HEAD").partition("\n")[0] + + recurse = False + if rurl != self.url: + remote = self._add_remote(git) + git.git_operation("fetch", remote) + if self.fxtag and atag == self.fxtag: + result = f" {self.name:>20} at tag {self.fxtag}" + recurse = True + elif self.fxtag and ahash[: len(self.fxtag)] == self.fxtag: + result = f" {self.name:>20} at hash {ahash}" + recurse = True + elif atag == ahash: + result = f" {self.name:>20} at hash {ahash}" + recurse = True + elif self.fxtag: + result = f"s {self.name:>20} {atag} {ahash} is out of sync with .gitmodules {self.fxtag}" + testfails = True + needsupdate = True + else: + result = f"e {self.name:>20} has no fxtag defined in .gitmodules, module at {atag}" + testfails = True + + status = git.git_operation("status", "--ignore-submodules", "-uno") + if "nothing to commit" not in status: + localmods = True + result = "M" + textwrap.indent(status, " ") + + return result, needsupdate, localmods, testfails + + + def _add_remote(self, git): + """ + Adds a new remote to the submodule if it does not already exist. + + This method checks the existing remotes of the submodule. If the submodule's URL is not already listed as a remote, + it attempts to add a new remote. The name for the new remote is generated dynamically to avoid conflicts. If no + remotes exist, it defaults to naming the new remote 'origin'. + + Args: + git (GitInterface): An instance of GitInterface to perform git operations. + + Returns: + str: The name of the new remote if added, or the name of the existing remote that matches the submodule's URL. + """ + remotes = git.git_operation("remote", "-v").splitlines() + upstream = None + if remotes: + upstream = git.git_operation("ls-remote", "--get-url").rstrip() + newremote = "newremote.00" + tmpurl = self.url.replace("git@github.com:", "https://github.com/") + line = next((s for s in remotes if self.url in s or tmpurl in s), None) + if line: + newremote = line.split()[0] + return newremote + else: + i = 0 + while "newremote" in remotes: + i = i + 1 + newremote = f"newremote.{i:02d}" + else: + newremote = "origin" + git.git_operation("remote", "add", newremote, self.url) + return newremote + + def toplevel(self): + """ + Returns True if the submodule is Toplevel (either Required or Optional) + """ + return True if "Top" in self.fxrequired else False + + def sparse_checkout(self): + """ + Performs a sparse checkout of the submodule. + + This method optimizes the checkout process by only checking out files specified in the submodule's sparse-checkout configuration, + rather than the entire submodule content. It achieves this by first ensuring the `.git/info/sparse-checkout` file is created and + configured in the submodule's directory. Then, it proceeds to checkout the desired tag. If the submodule has already been checked out, + this method will not perform the checkout again. + + This approach is particularly beneficial for submodules with a large number of files, as it significantly reduces the time and disk space + required for the checkout process by avoiding the unnecessary checkout and subsequent removal of unneeded files. + + Returns: + None + """ + self.logger.info("Called sparse_checkout for {}".format(self.name)) + rgit = GitInterface(self.root_dir, self.logger) + superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree") + if superroot: + gitroot = superroot.strip() + else: + gitroot = self.root_dir.strip() + assert os.path.isdir(os.path.join(gitroot, ".git")) + # first create the module directory + if not os.path.isdir(os.path.join(self.root_dir, self.path)): + os.makedirs(os.path.join(self.root_dir, self.path)) + + # initialize a new git repo and set the sparse checkout flag + sprep_repo = os.path.join(self.root_dir, self.path) + sprepo_git = GitInterface(sprep_repo, self.logger) + if os.path.exists(os.path.join(sprep_repo, ".git")): + try: + self.logger.info("Submodule {} found".format(self.name)) + chk = sprepo_git.config_get_value("core", "sparseCheckout") + if chk == "true": + self.logger.info("Sparse submodule {} already checked out".format(self.name)) + return + except (NoOptionError): + self.logger.debug("Sparse submodule {} not present".format(self.name)) + except Exception as e: + utils.fatal_error("Unexpected error {} occured.".format(e)) + + sprepo_git.config_set_value("core", "sparseCheckout", "true") + + # set the repository remote + + self.logger.info("Setting remote origin in {}/{}".format(self.root_dir, self.path)) + status = sprepo_git.git_operation("remote", "-v") + if self.url not in status: + sprepo_git.git_operation("remote", "add", "origin", self.url) + + topgit = os.path.join(gitroot, ".git") + + if gitroot != self.root_dir and os.path.isfile(os.path.join(self.root_dir, ".git")): + with open(os.path.join(self.root_dir, ".git")) as f: + gitpath = os.path.relpath( + os.path.join(self.root_dir, f.read().split()[1]), + start=os.path.join(self.root_dir, self.path), + ) + topgit = os.path.join(gitpath, "modules") + else: + topgit = os.path.relpath( + os.path.join(self.root_dir, ".git", "modules"), + start=os.path.join(self.root_dir, self.path), + ) + + with utils.pushd(sprep_repo): + if not os.path.isdir(topgit): + os.makedirs(topgit) + topgit += os.sep + self.name + + if os.path.isdir(os.path.join(self.root_dir, self.path, ".git")): + with utils.pushd(sprep_repo): + if os.path.isdir(os.path.join(topgit,".git")): + shutil.rmtree(os.path.join(topgit,".git")) + shutil.move(".git", topgit) + with open(".git", "w") as f: + f.write("gitdir: " + os.path.relpath(topgit)) + # assert(os.path.isdir(os.path.relpath(topgit, start=sprep_repo))) + gitsparse = os.path.abspath(os.path.join(topgit, "info", "sparse-checkout")) + if os.path.isfile(gitsparse): + self.logger.warning( + "submodule {} is already initialized {}".format(self.name, topgit) + ) + return + + with utils.pushd(sprep_repo): + if os.path.isfile(self.fxsparse): + shutil.copy(self.fxsparse, gitsparse) + + + # Finally checkout the repo + sprepo_git.git_operation("fetch", "origin", "--tags") + sprepo_git.git_operation("checkout", self.fxtag) + + print(f"Successfully checked out {self.name:>20} at {self.fxtag}") + rgit.config_set_value(f'submodule "{self.name}"', "active", "true") + rgit.config_set_value(f'submodule "{self.name}"', "url", self.url) + rgit.config_set_value(f'submodule "{self.name}"', "path", self.path) + + def update(self): + """ + Updates the submodule to the latest or specified version. + + This method handles the update process of the submodule, including checking out the submodule into the specified path, + handling sparse checkouts if configured, and updating the submodule's URL if necessary. It supports both SSH and HTTPS URLs, + automatically converting SSH URLs to HTTPS to avoid issues for users without SSH keys. + + The update process involves the following steps: + 1. If the submodule is configured for sparse checkout, it performs a sparse checkout. + 2. If the submodule is not already checked out, it clones the submodule using the provided URL. + 3. If a specific tag or hash is provided, it checks out that tag; otherwise, it checks out the latest version. + 4. If the root `.git` is a file (indicating a submodule or a worktree), additional steps are taken to integrate the submodule properly. + + Args: + None + Note: + - SSH URLs are automatically converted to HTTPS to accommodate users without SSH keys. + + Returns: + None + """ + git = GitInterface(self.root_dir, self.logger) + repodir = os.path.join(self.root_dir, self.path) + self.logger.info("Checkout {} into {}/{}".format(self.name, self.root_dir, self.path)) + # if url is provided update to the new url + tag = None + repo_exists = False + if os.path.exists(os.path.join(repodir, ".git")): + self.logger.info("Submodule {} already checked out".format(self.name)) + repo_exists = True + # Look for a .gitmodules file in the newly checkedout repo + if self.fxsparse: + print(f"Sparse checkout {self.name} fxsparse {self.fxsparse}") + self.sparse_checkout() + else: + if not repo_exists and self.url: + # ssh urls cause problems for those who dont have git accounts with ssh keys defined + # but cime has one since e3sm prefers ssh to https, because the .gitmodules file was + # opened with a GitModules object we don't need to worry about restoring the file here + # it will be done by the GitModules class + if self.url.startswith("git@"): + git.git_operation("clone", self.url, self.path) + smgit = GitInterface(repodir, self.logger) + if not tag: + tag = smgit.git_operation("describe", "--tags", "--always").rstrip() + smgit.git_operation("checkout", tag) + # Now need to move the .git dir to the submodule location + rootdotgit = os.path.join(self.root_dir, ".git") + if os.path.isfile(rootdotgit): + with open(rootdotgit) as f: + line = f.readline() + if line.startswith("gitdir: "): + rootdotgit = line[8:].rstrip() + + newpath = os.path.abspath(os.path.join(self.root_dir, rootdotgit, "modules", self.name)) + if os.path.exists(newpath): + shutil.rmtree(os.path.join(repodir, ".git")) + else: + shutil.move(os.path.join(repodir, ".git"), newpath) + + with open(os.path.join(repodir, ".git"), "w") as f: + f.write("gitdir: " + os.path.relpath(newpath, start=repodir)) + + if not os.path.exists(repodir): + parent = os.path.dirname(repodir) + if not os.path.isdir(parent): + os.makedirs(parent) + git.git_operation("submodule", "add", "--name", self.name, "--", self.url, self.path) + + if not repo_exists: + git.git_operation("submodule", "update", "--init", "--", self.path) + + if self.fxtag: + smgit = GitInterface(repodir, self.logger) + newremote = self._add_remote(smgit) + # Trying to distingush a tag from a hash + allowed = set(string.digits + 'abcdef') + if not set(self.fxtag) <= allowed: + # This is a tag + tag = f"refs/tags/{self.fxtag}:refs/tags/{self.fxtag}" + smgit.git_operation("fetch", newremote, tag) + smgit.git_operation("checkout", self.fxtag) + + if not os.path.exists(os.path.join(repodir, ".git")): + utils.fatal_error( + f"Failed to checkout {self.name} {repo_exists} {repodir} {self.path}" + ) + + + if os.path.exists(os.path.join(self.path, ".git")): + submoddir = os.path.join(self.root_dir, self.path) + with utils.pushd(submoddir): + git = GitInterface(submoddir, self.logger) + # first make sure the url is correct + newremote = self._add_remote(git) + tags = git.git_operation("tag", "-l") + fxtag = self.fxtag + if fxtag and fxtag not in tags: + git.git_operation("fetch", newremote, "--tags") + atag = git.git_operation("describe", "--tags", "--always").rstrip() + if fxtag and fxtag != atag: + try: + git.git_operation("checkout", fxtag) + print(f"{self.name:>20} updated to {fxtag}") + except Exception as error: + print(error) + + + elif not fxtag: + print(f"No fxtag found for submodule {self.name:>20}") + else: + print(f"{self.name:>20} up to date.") + + + + return diff --git a/.lib/git-fleximod/git_fleximod/utils.py b/.lib/git-fleximod/git_fleximod/utils.py index 7cc1de38cc..1a2d5ccf2f 100644 --- a/.lib/git-fleximod/git_fleximod/utils.py +++ b/.lib/git-fleximod/git_fleximod/utils.py @@ -241,12 +241,12 @@ def _hanging_msg(working_directory, command): has taken {hanging_sec} seconds. It may be hanging. The command will continue to run, but you may want to abort -manage_externals with ^C and investigate. A possible cause of hangs is -when svn or git require authentication to access a private -repository. On some systems, svn and git requests for authentication -information will not be displayed to the user. In this case, the program -will appear to hang. Ensure you can run svn and git manually and access -all repositories without entering your authentication information. +git-fleximod with ^C and investigate. A possible cause of hangs is git +requires authentication to access a private repository. On some +systems, git requests for authentication information will not +be displayed to the user. In this case, the program will appear to +hang. Ensure you can run git manually and access all +repositories without entering your authentication information. """.format( command=command, diff --git a/.lib/git-fleximod/poetry.lock b/.lib/git-fleximod/poetry.lock index b59ed3942c..3a74effcd1 100644 --- a/.lib/git-fleximod/poetry.lock +++ b/.lib/git-fleximod/poetry.lock @@ -13,13 +13,13 @@ files = [ [[package]] name = "babel" -version = "2.14.0" +version = "2.15.0" description = "Internationalization utilities" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "Babel-2.14.0-py3-none-any.whl", hash = "sha256:efb1a25b7118e67ce3a259bed20545c29cb68be8ad2c784c83689981b7a57287"}, - {file = "Babel-2.14.0.tar.gz", hash = "sha256:6919867db036398ba21eb5c7a0f6b28ab8cbc3ae7a73a44ebe34ae74a4e7d363"}, + {file = "Babel-2.15.0-py3-none-any.whl", hash = "sha256:08706bdad8d0a3413266ab61bd6c34d0c28d6e1e7badf40a2cebe67644e2e1fb"}, + {file = "babel-2.15.0.tar.gz", hash = "sha256:8daf0e265d05768bc6c7a314cf1321e9a123afc328cc635c18622a2f30a04413"}, ] [package.dependencies] @@ -30,13 +30,13 @@ dev = ["freezegun (>=1.0,<2.0)", "pytest (>=6.0)", "pytest-cov"] [[package]] name = "certifi" -version = "2024.2.2" +version = "2024.6.2" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, - {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, + {file = "certifi-2024.6.2-py3-none-any.whl", hash = "sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56"}, + {file = "certifi-2024.6.2.tar.gz", hash = "sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516"}, ] [[package]] @@ -162,13 +162,13 @@ files = [ [[package]] name = "exceptiongroup" -version = "1.2.0" +version = "1.2.1" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" files = [ - {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, - {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, + {file = "exceptiongroup-1.2.1-py3-none-any.whl", hash = "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad"}, + {file = "exceptiongroup-1.2.1.tar.gz", hash = "sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16"}, ] [package.extras] @@ -225,30 +225,31 @@ smmap = ">=3.0.1,<6" [[package]] name = "gitpython" -version = "3.1.41" +version = "3.1.43" description = "GitPython is a Python library used to interact with Git repositories" optional = false python-versions = ">=3.7" files = [ - {file = "GitPython-3.1.41-py3-none-any.whl", hash = "sha256:c36b6634d069b3f719610175020a9aed919421c87552185b085e04fbbdb10b7c"}, - {file = "GitPython-3.1.41.tar.gz", hash = "sha256:ed66e624884f76df22c8e16066d567aaa5a37d5b5fa19db2c6df6f7156db9048"}, + {file = "GitPython-3.1.43-py3-none-any.whl", hash = "sha256:eec7ec56b92aad751f9912a73404bc02ba212a23adb2c7098ee668417051a1ff"}, + {file = "GitPython-3.1.43.tar.gz", hash = "sha256:35f314a9f878467f5453cc1fee295c3e18e52f1b99f10f6cf5b1682e968a9e7c"}, ] [package.dependencies] gitdb = ">=4.0.1,<5" [package.extras] -test = ["black", "coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "sumtypes"] +doc = ["sphinx (==4.3.2)", "sphinx-autodoc-typehints", "sphinx-rtd-theme", "sphinxcontrib-applehelp (>=1.0.2,<=1.0.4)", "sphinxcontrib-devhelp (==1.0.2)", "sphinxcontrib-htmlhelp (>=2.0.0,<=2.0.1)", "sphinxcontrib-qthelp (==1.0.3)", "sphinxcontrib-serializinghtml (==1.1.5)"] +test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions"] [[package]] name = "idna" -version = "3.6" +version = "3.7" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.5" files = [ - {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, - {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, + {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"}, + {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, ] [[package]] @@ -264,22 +265,22 @@ files = [ [[package]] name = "importlib-metadata" -version = "7.0.1" +version = "8.0.0" description = "Read metadata from Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "importlib_metadata-7.0.1-py3-none-any.whl", hash = "sha256:4805911c3a4ec7c3966410053e9ec6a1fecd629117df5adee56dfc9432a1081e"}, - {file = "importlib_metadata-7.0.1.tar.gz", hash = "sha256:f238736bb06590ae52ac1fab06a3a9ef1d8dce2b7a35b5ab329371d6c8f5d2cc"}, + {file = "importlib_metadata-8.0.0-py3-none-any.whl", hash = "sha256:15584cf2b1bf449d98ff8a6ff1abef57bf20f3ac6454f431736cd3e660921b2f"}, + {file = "importlib_metadata-8.0.0.tar.gz", hash = "sha256:188bd24e4c346d3f0a933f275c2fec67050326a856b9a359881d7c2a697e8812"}, ] [package.dependencies] zipp = ">=0.5" [package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] perf = ["ipython"] -testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)", "pytest-ruff"] +test = ["flufl.flake8", "importlib-resources (>=1.3)", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-perf (>=0.9.2)", "pytest-ruff (>=0.2.1)"] [[package]] name = "iniconfig" @@ -294,13 +295,13 @@ files = [ [[package]] name = "jinja2" -version = "3.1.3" +version = "3.1.4" description = "A very fast and expressive template engine." optional = false python-versions = ">=3.7" files = [ - {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, - {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, + {file = "jinja2-3.1.4-py3-none-any.whl", hash = "sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d"}, + {file = "jinja2-3.1.4.tar.gz", hash = "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369"}, ] [package.dependencies] @@ -380,24 +381,24 @@ files = [ [[package]] name = "packaging" -version = "23.2" +version = "24.1" description = "Core utilities for Python packages" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, - {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, + {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"}, + {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, ] [[package]] name = "pluggy" -version = "1.4.0" +version = "1.5.0" description = "plugin and hook calling mechanisms for python" optional = false python-versions = ">=3.8" files = [ - {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, - {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, + {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, + {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, ] [package.extras] @@ -406,39 +407,38 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "pyfakefs" -version = "5.3.5" +version = "5.5.0" description = "pyfakefs implements a fake file system that mocks the Python file system modules." optional = false python-versions = ">=3.7" files = [ - {file = "pyfakefs-5.3.5-py3-none-any.whl", hash = "sha256:751015c1de94e1390128c82b48cdedc3f088bbdbe4bc713c79d02a27f0f61e69"}, - {file = "pyfakefs-5.3.5.tar.gz", hash = "sha256:7cdc500b35a214cb7a614e1940543acc6650e69a94ac76e30f33c9373bd9cf90"}, + {file = "pyfakefs-5.5.0-py3-none-any.whl", hash = "sha256:8dbf203ab7bef1529f11f7d41b9478b898e95bf9f3b71262163aac07a518cd76"}, + {file = "pyfakefs-5.5.0.tar.gz", hash = "sha256:7448aaa07142f892d0a4eb52a5ed3206a9f02c6599e686cd97d624c18979c154"}, ] [[package]] name = "pygments" -version = "2.17.2" +version = "2.18.0" description = "Pygments is a syntax highlighting package written in Python." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "pygments-2.17.2-py3-none-any.whl", hash = "sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c"}, - {file = "pygments-2.17.2.tar.gz", hash = "sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367"}, + {file = "pygments-2.18.0-py3-none-any.whl", hash = "sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a"}, + {file = "pygments-2.18.0.tar.gz", hash = "sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199"}, ] [package.extras] -plugins = ["importlib-metadata"] windows-terminal = ["colorama (>=0.4.6)"] [[package]] name = "pytest" -version = "8.0.0" +version = "8.2.2" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.8" files = [ - {file = "pytest-8.0.0-py3-none-any.whl", hash = "sha256:50fb9cbe836c3f20f0dfa99c565201fb75dc54c8d76373cd1bde06b06657bdb6"}, - {file = "pytest-8.0.0.tar.gz", hash = "sha256:249b1b0864530ba251b7438274c4d251c58d868edaaec8762893ad4a0d71c36c"}, + {file = "pytest-8.2.2-py3-none-any.whl", hash = "sha256:c434598117762e2bd304e526244f67bf66bbd7b5d6cf22138be51ff661980343"}, + {file = "pytest-8.2.2.tar.gz", hash = "sha256:de4bb8104e201939ccdc688b27a89a7be2079b22e2bd2b07f806b6ba71117977"}, ] [package.dependencies] @@ -446,11 +446,11 @@ colorama = {version = "*", markers = "sys_platform == \"win32\""} exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} iniconfig = "*" packaging = "*" -pluggy = ">=1.3.0,<2.0" -tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} +pluggy = ">=1.5,<2.0" +tomli = {version = ">=1", markers = "python_version < \"3.11\""} [package.extras] -testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] +dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] [[package]] name = "pytz" @@ -465,13 +465,13 @@ files = [ [[package]] name = "requests" -version = "2.31.0" +version = "2.32.3" description = "Python HTTP for Humans." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, - {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, + {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, + {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, ] [package.dependencies] @@ -643,13 +643,13 @@ files = [ [[package]] name = "urllib3" -version = "2.2.0" +version = "2.2.2" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.8" files = [ - {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, - {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, + {file = "urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472"}, + {file = "urllib3-2.2.2.tar.gz", hash = "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"}, ] [package.extras] @@ -674,18 +674,18 @@ test = ["pytest (>=6.0.0)", "setuptools (>=65)"] [[package]] name = "zipp" -version = "3.17.0" +version = "3.19.2" description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.8" files = [ - {file = "zipp-3.17.0-py3-none-any.whl", hash = "sha256:0e923e726174922dce09c53c59ad483ff7bbb8e572e00c7f7c46b88556409f31"}, - {file = "zipp-3.17.0.tar.gz", hash = "sha256:84e64a1c28cf7e91ed2078bb8cc8c259cb19b76942096c8d7b84947690cabaf0"}, + {file = "zipp-3.19.2-py3-none-any.whl", hash = "sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c"}, + {file = "zipp-3.19.2.tar.gz", hash = "sha256:bf1dcf6450f873a13e952a29504887c89e6de7506209e5b1bcc3460135d4de19"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy (>=0.9.1)", "pytest-ruff"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] [metadata] lock-version = "2.0" diff --git a/.lib/git-fleximod/pyproject.toml b/.lib/git-fleximod/pyproject.toml index a316914bf5..9cff1423c8 100644 --- a/.lib/git-fleximod/pyproject.toml +++ b/.lib/git-fleximod/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "git-fleximod" -version = "0.7.7" +version = "0.8.2" description = "Extended support for git-submodule and git-sparse-checkout" authors = ["Jim Edwards "] maintainers = ["Jim Edwards "] diff --git a/.lib/git-fleximod/tbump.toml b/.lib/git-fleximod/tbump.toml index c22637ccda..b4eed7d40c 100644 --- a/.lib/git-fleximod/tbump.toml +++ b/.lib/git-fleximod/tbump.toml @@ -2,7 +2,7 @@ github_url = "https://github.com/jedwards4b/git-fleximod/" [version] -current = "0.7.7" +current = "0.8.2" # Example of a semver regexp. # Make sure this matches current_version before diff --git a/.lib/git-fleximod/tests/conftest.py b/.lib/git-fleximod/tests/conftest.py index 1cc008eb1d..81edbe713e 100644 --- a/.lib/git-fleximod/tests/conftest.py +++ b/.lib/git-fleximod/tests/conftest.py @@ -32,7 +32,7 @@ def logger(): "submodule_name": "test_optional", "status1" : "test_optional MPIserial_2.5.0-3-gd82ce7c is out of sync with .gitmodules MPIserial_2.4.0", "status2" : "test_optional at tag MPIserial_2.4.0", - "status3" : "test_optional not checked out, out of sync at tag None, expected tag is MPIserial_2.4.0", + "status3" : "test_optional not checked out, out of sync at tag None, expected tag is MPIserial_2.4.0 (optional)", "status4" : "test_optional at tag MPIserial_2.4.0", "gitmodules_content": """ [submodule "test_optional"] @@ -119,8 +119,20 @@ def complex_repo(tmp_path, logger): str_path = str(test_dir) gitp = GitInterface(str_path, logger) gitp.git_operation("remote", "add", "origin", "https://github.com/jedwards4b/fleximod-test2") - gitp.git_operation("fetch", "origin", "main") - gitp.git_operation("checkout", "main") + gitp.git_operation("fetch", "origin") + gitp.git_operation("checkout", "v0.0.1") + return test_dir + +@pytest.fixture +def complex_update(tmp_path, logger): + test_dir = tmp_path / "testcomplex" + test_dir.mkdir() + str_path = str(test_dir) + gitp = GitInterface(str_path, logger) + gitp.git_operation("remote", "add", "origin", "https://github.com/jedwards4b/fleximod-test2") + gitp.git_operation("fetch", "origin") + gitp.git_operation("checkout", "v0.0.2") + return test_dir @pytest.fixture diff --git a/.lib/git-fleximod/tests/test_e_complex_update.py b/.lib/git-fleximod/tests/test_e_complex_update.py new file mode 100644 index 0000000000..0c3ab4c6a6 --- /dev/null +++ b/.lib/git-fleximod/tests/test_e_complex_update.py @@ -0,0 +1,69 @@ +import pytest +from pathlib import Path +from git_fleximod.gitinterface import GitInterface + +def test_complex_update(git_fleximod, complex_update, logger): + status = git_fleximod(complex_update, "status") + assert("ToplevelOptional not checked out, aligned at tag v5.3.2" in status.stdout) + assert("ToplevelRequired not checked out, aligned at tag MPIserial_2.5.0" in status.stdout) + assert("AlwaysRequired not checked out, aligned at tag MPIserial_2.4.0" in status.stdout) + assert("Complex not checked out, out of sync at tag testtag02, expected tag is testtag3" in status.stdout) + assert("AlwaysOptional not checked out, out of sync at tag None, expected tag is MPIserial_2.3.0" in status.stdout) + + # This should checkout and update test_submodule and complex_sub + result = git_fleximod(complex_update, "update") + assert result.returncode == 0 + + status = git_fleximod(complex_update, "status") + assert("ToplevelOptional not checked out, aligned at tag v5.3.2" in status.stdout) + assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) + assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) + assert("Complex at tag testtag3" in status.stdout) + + # now check the complex_sub + root = (complex_update / "modules" / "complex") + assert(not (root / "libraries" / "gptl" / ".git").exists()) + assert(not (root / "libraries" / "mpi-serial" / ".git").exists()) + assert((root / "modules" / "mpi-serialAR" / ".git").exists()) + assert((root / "modules" / "mpi-serialSAR" / ".git").exists()) + assert(not (root / "modules" / "mpi-serial2" / ".git").exists()) + assert((root / "modules" / "mpi-sparse" / ".git").exists()) + assert((root / "modules" / "mpi-sparse" / "m4").exists()) + assert(not (root / "modules" / "mpi-sparse" / "README").exists()) + + # update a single optional submodule + + result = git_fleximod(complex_update, "update ToplevelOptional") + assert result.returncode == 0 + + status = git_fleximod(complex_update, "status") + assert("ToplevelOptional at tag v5.3.2" in status.stdout) + assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) + assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) + assert("Complex at tag testtag3" in status.stdout) + assert("AlwaysOptional not checked out, out of sync at tag None, expected tag is MPIserial_2.3.0" in status.stdout) + + # Finally update optional + result = git_fleximod(complex_update, "update --optional") + assert result.returncode == 0 + + status = git_fleximod(complex_update, "status") + assert("ToplevelOptional at tag v5.3.2" in status.stdout) + assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) + assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) + assert("Complex at tag testtag3" in status.stdout) + assert("AlwaysOptional at tag MPIserial_2.3.0" in status.stdout) + + # now check the complex_sub + root = (complex_update / "modules" / "complex" ) + assert(not (root / "libraries" / "gptl" / ".git").exists()) + assert(not (root / "libraries" / "mpi-serial" / ".git").exists()) + assert(not (root / "modules" / "mpi-serial" / ".git").exists()) + assert((root / "modules" / "mpi-serialAR" / ".git").exists()) + assert((root / "modules" / "mpi-serialSAR" / ".git").exists()) + assert((root / "modules" / "mpi-sparse" / ".git").exists()) + assert((root / "modules" / "mpi-serial2" / ".git").exists()) + assert((root / "modules" / "mpi-sparse" / "m4").exists()) + assert(not (root / "modules" / "mpi-sparse" / "README").exists()) + + diff --git a/ccs_config b/ccs_config index 2ff978f92a..69a958581e 160000 --- a/ccs_config +++ b/ccs_config @@ -1 +1 @@ -Subproject commit 2ff978f92a5ac9a6ab243e5c14d06a7e2d2f5799 +Subproject commit 69a958581ecd2d32ee9cb1c38bcd3847b8b920bf diff --git a/cime b/cime index 422ddaa770..fcb9c6ec1e 160000 --- a/cime +++ b/cime @@ -1 +1 @@ -Subproject commit 422ddaa770a3cea6e83a60c9700ebce77acaceed +Subproject commit fcb9c6ec1e15f2f33995cf247aef3f8ef9f121eb diff --git a/cime_config/buildcpp b/cime_config/buildcpp index e7df81cecb..9143f2bcb6 100644 --- a/cime_config/buildcpp +++ b/cime_config/buildcpp @@ -73,15 +73,6 @@ def buildcpp(case): case.set_value("EPS_AAREA", "1.0e-04") case.set_value("EPS_AGRID", "1.0e-05") - # The vector mapping (in the mediator) needs to be 'cart3d' for SE - # NB: This is currently the default, is it working by conincidence for - # other unstructured dycores? - # For cmeps/nuopc cart3d is always the default option for all grids - match = re.match(r'ne[0-9]', atm_grid) - if match: - if (comp_interface == 'mct'): - case.set_value('VECT_MAP', 'cart3d') - # if need to build - then construct configure command config_opts = ["-s", "-fc_type", compiler, "-dyn", cam_dycore, "-hgrid", atm_grid, "-cpl", comp_interface, diff --git a/cime_config/buildlib b/cime_config/buildlib index b8cc333601..0328e84f2f 100755 --- a/cime_config/buildlib +++ b/cime_config/buildlib @@ -52,7 +52,7 @@ def _build_fms(caseroot, libroot, bldroot): mpilib = case.get_value("MPILIB") sharedpath = os.path.join(case.get_value("COMPILER"), mpilib, - strdebug, strthread, "nuopc") + strdebug, strthread) slr = os.path.abspath(case.get_value("SHAREDLIBROOT")) fmsbuildroot = os.path.join(slr, sharedpath) fmsinstallpath = os.path.join(fmsbuildroot, "FMS") @@ -108,7 +108,7 @@ def _build_cam(caseroot, libroot, bldroot): threaded = "threads" if case.get_value("BUILD_THREADED") or case.get_value("FORCE_BUILD_SMP") else "nothreads" comp_interface = case.get_value("COMP_INTERFACE") fmsbuilddir = os.path.join( - slr, compiler, mpilib, debug, threaded, comp_interface, "FMS") + slr, compiler, mpilib, debug, threaded, "FMS") user_incldir = '"-I{} -I{} -I{}"'.format( os.path.join(srcroot, "libraries", "FMS", "src", "include"), os.path.join(srcroot, "libraries", "FMS", "src", "mpp", "include"), diff --git a/cime_config/config_compsets.xml b/cime_config/config_compsets.xml index 3a4bb4bad4..240587f3a8 100644 --- a/cime_config/config_compsets.xml +++ b/cime_config/config_compsets.xml @@ -63,22 +63,22 @@ FLTHIST - HIST_CAM70%LT_CLM51%SP_CICE%PRES_DOCN%DOM_MOSART_SGLC_SWAV + HIST_CAM70%LT_CLM60%SP_CICE%PRES_DOCN%DOM_MOSART_SGLC_SWAV FMTHIST - HIST_CAM70%MT_CLM51%SP_CICE%PRES_DOCN%DOM_MOSART_SGLC_SWAV + HIST_CAM70%MT_CLM60%SP_CICE%PRES_DOCN%DOM_MOSART_SGLC_SWAV FLT1850_TESTINGONLY_v0c - 1850_CAM70%LT_CLM51%SP_CICE%PRES_DOCN%DOM_MOSART_SGLC_SWAV + 1850_CAM70%LT_CLM60%SP_CICE%PRES_DOCN%DOM_MOSART_SGLC_SWAV FMT1850_TESTINGONLY_v0c - 1850_CAM70%MT_CLM51%SP_CICE%PRES_DOCN%DOM_MOSART_SGLC_SWAV + 1850_CAM70%MT_CLM60%SP_CICE%PRES_DOCN%DOM_MOSART_SGLC_SWAV @@ -282,7 +282,7 @@ F2000dev - 2000_CAM70_CLM51%SP_CICE%PRES_DOCN%DOM_MOSART_SGLC_SWAV + 2000_CAM70_CLM60%SP_CICE%PRES_DOCN%DOM_MOSART_SGLC_SWAV @@ -344,11 +344,11 @@ FCLTHIST - HIST_CAM70%LT%CCTS1_CLM51%SP_CICE%PRES_DOCN%DOM_MOSART_SGLC_SWAV + HIST_CAM70%LT%CCTS1_CLM60%SP_CICE%PRES_DOCN%DOM_MOSART_SGLC_SWAV FCMTHIST - HIST_CAM70%MT%CCTS1_CLM51%SP_CICE%PRES_DOCN%DOM_MOSART_SGLC_SWAV + HIST_CAM70%MT%CCTS1_CLM60%SP_CICE%PRES_DOCN%DOM_MOSART_SGLC_SWAV FCvbsxHIST diff --git a/components/cdeps b/components/cdeps index 7476950699..453a9d175a 160000 --- a/components/cdeps +++ b/components/cdeps @@ -1 +1 @@ -Subproject commit 7476950699909813d1938a34bd8d71bf5bfbf1e9 +Subproject commit 453a9d175a5739d9cca5c4ec7b96f45b201decec diff --git a/components/cism b/components/cism index c05dd5c4fc..c84cc9f5b3 160000 --- a/components/cism +++ b/components/cism @@ -1 +1 @@ -Subproject commit c05dd5c4fc85327e76523aaea9cfe1e388748928 +Subproject commit c84cc9f5b3103766a35d0a7ddd5e9dbd7deae762 diff --git a/components/clm b/components/clm index a9433779f0..e04b7e2ee9 160000 --- a/components/clm +++ b/components/clm @@ -1 +1 @@ -Subproject commit a9433779f0ae499d60ad118d2ec331628f0eaaa8 +Subproject commit e04b7e2ee974aaef93117776a96fd7ce1e774b4d diff --git a/components/cmeps b/components/cmeps index 90f815ba9e..1b8920c3bf 160000 --- a/components/cmeps +++ b/components/cmeps @@ -1 +1 @@ -Subproject commit 90f815ba9e7493d71043b5d8e627a3f20bd5dc78 +Subproject commit 1b8920c3bf6c64056d6c1b1b88393617de2fefa3 diff --git a/components/mizuRoute b/components/mizuRoute index 81c720c7ee..2ff305a029 160000 --- a/components/mizuRoute +++ b/components/mizuRoute @@ -1 +1 @@ -Subproject commit 81c720c7ee51f9c69f2934f696078c42f4493565 +Subproject commit 2ff305a0292cb06789de6cfea7ad3cc0d6173493 diff --git a/components/mosart b/components/mosart index 8c682b1b7f..e2ffe00004 160000 --- a/components/mosart +++ b/components/mosart @@ -1 +1 @@ -Subproject commit 8c682b1b7f15d146816de302e0d359da3e957056 +Subproject commit e2ffe00004cc416cfc8bcfae2a949474075c1d1f diff --git a/components/rtm b/components/rtm index 88503adbc2..b3dfcfbba5 160000 --- a/components/rtm +++ b/components/rtm @@ -1 +1 @@ -Subproject commit 88503adbc275fb2ccbb6b598e460deaeb140e515 +Subproject commit b3dfcfbba58c151ac5a6ab513b3515ef3deff798 diff --git a/doc/ChangeLog b/doc/ChangeLog index 0edfda4a52..46bed8a4e5 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,4 +1,201 @@ +=============================================================== + +Tag name: cam6_4_011 +Originator(s): jedwards, eaton, cacraig +Date: July 19, 2024 +One-line Summary: Update submodules, git-fleximod; fix fv3 build; remove mct reference +Github PR URL: https://github.com/ESCOMP/CAM/pull/1089 + +Purpose of changes (include the issue number and title text for each relevant GitHub issue): + +This PR replaces PR #1067 and #1075 by merging them with other updates. + +. update submodules to be consistent with cesm3_0_alpha02a + +. merge in PR #1067 - fix the path to fms for fv3 build, remove mct reference: https://github.com/ESCOMP/CAM/issues/1068 + +. merge in PR #1075 - Git fleximod update0.8.2: https://github.com/ESCOMP/CAM/issues/1076 + +. Fix CLM regression errors due to their upgrade and older version no longer working with CAM7 runs: https://github.com/ESCOMP/CAM/issues/1091 + +. resolves #1076 +. resolves #1068 +. resolves #1091 + +Describe any changes made to build system: none + +Describe any changes made to the namelist: none + +List any changes to the defaults for the boundary datasets: none + +Describe any substantial timing or memory changes: none + +Code reviewed by: cacraig + +List all files eliminated: none + +List all files added and what they do: none + +List all existing files that have been modified, and describe the changes: + +.gitmodules +. remove submodule "mct" +. cismwrap_2_2_001 -> cismwrap_2_2_002 +. rtm1_0_79 -> rtm1_0_80 +. mosart1_0_49 -> mosart1.1.02 +. cesm-coupling.n02_v2.1.2 -> cesm-coupling.n02_v2.1.3 +. ccs_config_cesm0.0.106 -> ccs_config_cesm1.0.0 +. cime6.0.246 -> cime6.1.0 +. cmeps0.14.67 -> cmeps1.0.2 +. cdeps1.0.34 -> cdeps1.0.43 +. share1.0.19 -> share1.1.2 +. ctsm5.2.007 -> ctsm5.2.009 + +cime_config/buildcpp +. remove mct conditional + +cime_config/buildlib +. fix sharedpath and fmsbuilddir + +cime_config/config_compsets.xml +. Change from CLM51 to CLM60 for all CAM7 compsets + +If there were any failures reported from running test_driver.sh on any test +platform, and checkin with these failures has been OK'd by the gatekeeper, +then copy the lines from the td.*.status files for the failed tests to the +appropriate machine below. All failed tests must be justified. + +derecho/intel/aux_cam: + ERP_Ln9.f09_f09_mg17.FCSD_HCO.derecho_intel.cam-outfrq9s (Overall: PEND) details: + PEND ERP_Ln9.f09_f09_mg17.FCSD_HCO.derecho_intel.cam-outfrq9s RUN + FAIL ERP_Ln9.f09_f09_mg17.FCSD_HCO.derecho_intel.cam-outfrq9s COMPARE_base_rest + FAIL ERP_Ln9.f09_f09_mg17.FCSD_HCO.derecho_intel.cam-outfrq9s GENERATE exception + - pre-existing failure due to HEMCO not having reproducible results issues #1018 and #856 + + SMS_D_Ln9_P1280x1.ne0ARCTICne30x4_ne0ARCTICne30x4_mt12.FHIST.derecho_intel.cam-outfrq9s (Overall: PEND) details: + PEND SMS_D_Ln9_P1280x1.ne0ARCTICne30x4_ne0ARCTICne30x4_mt12.FHIST.derecho_intel.cam-outfrq9s SHAREDLIB_BUILD + SMS_D_Ln9_P1280x1.ne0CONUSne30x8_ne0CONUSne30x8_mt12.FCHIST.derecho_intel.cam-outfrq9s (Overall: PEND) details: + PEND SMS_D_Ln9_P1280x1.ne0CONUSne30x8_ne0CONUSne30x8_mt12.FCHIST.derecho_intel.cam-outfrq9s SHAREDLIB_BUILD + - pre-existing failures - need fix in CLM external + + SMS_D_Ln9.T42_T42.FSCAM.derecho_intel.cam-outfrq9s (Overall: FAIL) details: + FAIL SMS_D_Ln9.T42_T42.FSCAM.derecho_intel.cam-outfrq9s RUN time=65 + - Bug in med.F90 - Will go away when CICE external is updated post git-fleximod + + ERC_D_Ln9.f19_f19_mg17.QPC6.derecho_intel.cam-outfrq3s_cosp (Overall: DIFF) details: + ERC_D_Ln9.f19_f19_mg17.QPMOZ.derecho_intel.cam-outfrq3s (Overall: DIFF) details: + ERC_D_Ln9.f19_f19_mg17.QPX2000.derecho_intel.cam-outfrq3s (Overall: DIFF) details: + ERC_D_Ln9.ne16_ne16_mg17.QPC5HIST.derecho_intel.cam-outfrq3s_usecase (Overall: DIFF) details: + ERC_D_Ln9_P144x1.ne16pg3_ne16pg3_mg17.QPC6HIST.derecho_intel.cam-outfrq3s_ttrac_usecase (Overall: DIFF) details: + ERI_D_Ln18.f45_f45_mg37.QPC41850.derecho_intel.cam-co2rmp_usecase (Overall: DIFF) details: + ERP_D_Ln9.f19_f19_mg17.QPC6.derecho_intel.cam-outfrq9s (Overall: DIFF) details: + ERP_D_Ln9.ne30pg3_ne30pg3_mg17.FLTHIST.derecho_intel.cam-outfrq9s (Overall: DIFF) details: + ERP_D_Ln9.ne30pg3_ne30pg3_mg17.FLTHIST.derecho_intel.cam-outfrq9s_rrtmgp (Overall: DIFF) details: + ERP_D_Ln9_P64x2.f09_f09_mg17.QSC6.derecho_intel.cam-outfrq9s (Overall: DIFF) details: + ERP_Ld3.f09_f09_mg17.FWHIST.derecho_intel.cam-reduced_hist1d (Overall: DIFF) details: + ERP_Lh12.f19_f19_mg17.FW4madSD.derecho_intel.cam-outfrq3h (Overall: DIFF) details: + ERP_Ln9.C96_C96_mg17.F2000climo.derecho_intel.cam-outfrq9s_mg3 (Overall: DIFF) details: + ERP_Ln9.f09_f09_mg17.F1850.derecho_intel.cam-outfrq9s (Overall: DIFF) details: + ERP_Ln9.f09_f09_mg17.F2000climo.derecho_intel.cam-outfrq9s (Overall: DIFF) details: + ERP_Ln9.f09_f09_mg17.F2010climo.derecho_intel.cam-outfrq9s (Overall: DIFF) details: + ERP_Ln9.f09_f09_mg17.FHIST_BDRD.derecho_intel.cam-outfrq9s (Overall: DIFF) details: + ERP_Ln9.f19_f19_mg17.FWsc1850.derecho_intel.cam-outfrq9s (Overall: DIFF) details: + ERP_Ln9.ne30pg3_ne30pg3_mg17.FCnudged.derecho_intel.cam-outfrq9s (Overall: DIFF) details: + ERP_Ln9.ne30pg3_ne30pg3_mg17.FW2000climo.derecho_intel.cam-outfrq9s (Overall: DIFF) details: + ERP_Ln9_P24x3.f45_f45_mg37.QPWmaC6.derecho_intel.cam-outfrq9s_mee_fluxes (Overall: DIFF) details: + ERS_Ld3.f10_f10_mg37.F1850.derecho_intel.cam-outfrq1d_14dec_ghg_cam7 (Overall: DIFF) details: + ERS_Ln9.f09_f09_mg17.FX2000.derecho_intel.cam-outfrq9s (Overall: DIFF) details: + ERS_Ln9.f19_f19_mg17.FSPCAMS.derecho_intel.cam-outfrq9s (Overall: DIFF) details: + ERS_Ln9.f19_f19_mg17.FXSD.derecho_intel.cam-outfrq9s (Overall: DIFF) details: + ERS_Ln9_P288x1.mpasa120_mpasa120.F2000climo.derecho_intel.cam-outfrq9s_mpasa120 (Overall: DIFF) details: + ERS_Ln9_P36x1.mpasa480_mpasa480.F2000climo.derecho_intel.cam-outfrq9s_mpasa480 (Overall: DIFF) details: + SCT_D_Ln7.T42_T42_mg17.QPC5.derecho_intel.cam-scm_prep (Overall: DIFF) details: + SMS_D_Ld2.f19_f19_mg17.QPC5HIST.derecho_intel.cam-volc_usecase (Overall: DIFF) details: + SMS_D_Ln9.f09_f09_mg17.FCts2nudged.derecho_intel.cam-outfrq9s_leapday (Overall: DIFF) details: + SMS_D_Ln9.f09_f09_mg17.FCvbsxHIST.derecho_intel.cam-outfrq9s (Overall: DIFF) details: + SMS_D_Ln9.f09_f09_mg17.FSD.derecho_intel.cam-outfrq9s (Overall: DIFF) details: + SMS_D_Ln9.f19_f19_mg17.FWma2000climo.derecho_intel.cam-outfrq9s (Overall: DIFF) details: + SMS_D_Ln9.f19_f19_mg17.FWma2000climo.derecho_intel.cam-outfrq9s_waccm_ma_mam4 (Overall: DIFF) details: + SMS_D_Ln9.f19_f19_mg17.FXHIST.derecho_intel.cam-outfrq9s_amie (Overall: DIFF) details: + SMS_D_Ln9.f19_f19_mg17.QPC2000climo.derecho_intel.cam-outfrq3s_usecase (Overall: DIFF) details: + SMS_D_Ln9.f19_f19_mg17.QPC5M7.derecho_intel.cam-outfrq9s (Overall: DIFF) details: + SMS_D_Ln9.ne16_ne16_mg17.QPX2000.derecho_intel.cam-outfrq9s (Overall: DIFF) details: + SMS_D_Ln9.ne16pg3_ne16pg3_mg17.FX2000.derecho_intel.cam-outfrq9s (Overall: DIFF) details: + SMS_D_Ln9.ne30pg3_ne30pg3_mg17.FMTHIST.derecho_intel.cam-outfrq9s (Overall: DIFF) details: + SMS_D_Ln9_P1280x1.ne30pg3_ne30pg3_mg17.FCLTHIST.derecho_intel.cam-outfrq9s (Overall: DIFF) details: + SMS_Ld1.f09_f09_mg17.FCHIST_GC.derecho_intel.cam-outfrq1d (Overall: DIFF) details: + SMS_Ld1.f09_f09_mg17.FW2000climo.derecho_intel.cam-outfrq1d (Overall: DIFF) details: + SMS_Ld1.ne30pg3_ne30pg3_mg17.FC2010climo.derecho_intel.cam-outfrq1d (Overall: DIFF) details: + SMS_Lh12.f09_f09_mg17.FCSD_HCO.derecho_intel.cam-outfrq3h (Overall: DIFF) details: + SMS_Lm13.f10_f10_mg37.F2000climo.derecho_intel.cam-outfrq1m (Overall: DIFF) details: + SMS_Ln9.f09_f09_mg17.F2010climo.derecho_intel.cam-nudging (Overall: DIFF) details: + SMS_Ln9.f09_f09_mg17.FW1850.derecho_intel.cam-reduced_hist3s (Overall: DIFF) details: + SMS_Ln9.f19_f19.F2000climo.derecho_intel.cam-silhs (Overall: DIFF) details: + SMS_Ln9.f19_f19_mg17.FHIST.derecho_intel.cam-outfrq9s_nochem (Overall: DIFF) details: + SMS_Ln9.ne30pg3_ne30pg3_mg17.FW2000climo.derecho_intel.cam-outfrq9s_rrtmgp (Overall: DIFF) details: + - Answer changes due to external updates - FIELDLIST differ only +izumi/nag/aux_cam: + DAE.f45_f45_mg37.FHS94.izumi_nag.cam-dae (Overall: FAIL) details: + FAIL DAE.f45_f45_mg37.FHS94.izumi_nag.cam-dae RUN time=10 + PEND DAE.f45_f45_mg37.FHS94.izumi_nag.cam-dae COMPARE_base_da + - pre-existing failure - issue #670 + + ERC_D_Ln9.f10_f10_mg37.QPC5.izumi_nag.cam-carma_sea_salt (Overall: DIFF) details: + ERC_D_Ln9.f10_f10_mg37.QPC5.izumi_nag.cam-outfrq3s_cosp (Overall: DIFF) details: + ERC_D_Ln9.f10_f10_mg37.QPC5.izumi_nag.cam-outfrq3s_subcol (Overall: DIFF) details: + ERC_D_Ln9.f10_f10_mg37.QPC6.izumi_nag.cam-outfrq3s_am (Overall: DIFF) details: + ERC_D_Ln9.f10_f10_mg37.QPC6.izumi_nag.cam-outfrq3s_cospsathist (Overall: DIFF) details: + ERC_D_Ln9.f10_f10_mg37.QPC6.izumi_nag.cam-outfrq3s (Overall: DIFF) details: + ERC_D_Ln9.f10_f10_mg37.QPWmaC6.izumi_nag.cam-outfrq3s (Overall: DIFF) details: + ERC_D_Ln9.f10_f10_mg37.QSPCAMS.izumi_nag.cam-outfrq3s (Overall: DIFF) details: + ERC_D_Ln9.ne16_ne16_mg17.QPC4.izumi_nag.cam-outfrq3s_usecase (Overall: DIFF) details: + ERC_D_Ln9.ne16pg3_ne16pg3_mg17.QPC4.izumi_nag.cam-outfrq3s_usecase (Overall: DIFF) details: + ERC_D_Ln9.ne5_ne5_mg37.QPC5.izumi_nag.cam-outfrq3s_ttrac (Overall: DIFF) details: + ERC_D_Ln9.T5_T5_mg37.QPC4.izumi_nag.cam-outfrq3s_usecase (Overall: DIFF) details: + ERI_D_Ln18.f10_f10_mg37.QPC5.izumi_nag.cam-outfrq3s_eoyttrac (Overall: DIFF) details: + ERI_D_Ln18.f19_f19_mg17.QPC6.izumi_nag.cam-ghgrmp_e8 (Overall: DIFF) details: + ERP_Ln9.ne5pg3_ne5pg3_mg37.QPC6.izumi_nag.cam-outfrq9s_clubbmf (Overall: DIFF) details: + PLB_D_Ln9.f10_f10_mg37.QPC5.izumi_nag.cam-ttrac_loadbal0 (Overall: DIFF) details: + PLB_D_Ln9.f10_f10_mg37.QPC5.izumi_nag.cam-ttrac_loadbal1 (Overall: DIFF) details: + PLB_D_Ln9.f10_f10_mg37.QPC5.izumi_nag.cam-ttrac_loadbal3 (Overall: DIFF) details: + PLB_D_Ln9.ne5_ne5_mg37.QPC5.izumi_nag.cam-ttrac_loadbal0 (Overall: DIFF) details: + PLB_D_Ln9.ne5_ne5_mg37.QPC5.izumi_nag.cam-ttrac_loadbal1 (Overall: DIFF) details: + PLB_D_Ln9.ne5_ne5_mg37.QPC5.izumi_nag.cam-ttrac_loadbal3 (Overall: DIFF) details: + SMS_D_Ln3.ne5pg3_ne5pg3_mg37.QPX2000.izumi_nag.cam-outfrq3s (Overall: DIFF) details: + SMS_D_Ln6.ne5_ne5_mg37.QPWmaC4.izumi_nag.cam-outfrq3s_physgrid_tem (Overall: DIFF) details: + SMS_D_Ln7.T42_T42_mg17.QPSCAMC5.izumi_nag.cam-scmarm (Overall: DIFF) details: + SMS_D_Ln9.f10_f10_mg37.QPC5.izumi_nag.cam-rad_diag_mam (Overall: DIFF) details: + SMS_D_Ln9.f10_f10_mg37.QPC6.izumi_nag.cam-outfrq3s_ba (Overall: DIFF) details: + SMS_P48x1_D_Ln3.f09_f09_mg17.QPC6HIST.izumi_nag.cam-outfrq3s_co2cycle_usecase (Overall: DIFF) details: + SUB_D_Ln9.f10_f10_mg37.QPC5.izumi_nag.cam-outfrq3s (Overall: DIFF) details: + - Answer changes due to external updates - FIELDLIST differ only + +izumi/gnu/aux_cam: + ERC_D_Ln9.f10_f10_mg37.QPC4.izumi_gnu.cam-outfrq3s_diags (Overall: DIFF) details: + ERC_D_Ln9.f10_f10_mg37.QPC5.izumi_gnu.cam-outfrq3s_unicon (Overall: DIFF) details: + ERC_D_Ln9.f10_f10_mg37.QPC5.izumi_gnu.cam-rad_diag (Overall: DIFF) details: + ERC_D_Ln9.f10_f10_mg37.QPSPCAMM.izumi_gnu.cam-outfrq3s (Overall: DIFF) details: + ERC_D_Ln9.ne5_ne5_mg37.QPC4.izumi_gnu.cam-outfrq3s_nudging_ne5_L26 (Overall: DIFF) details: + ERC_D_Ln9.ne5_ne5_mg37.QPC5.izumi_gnu.cam-outfrq3s_ba (Overall: DIFF) details: + ERI_D_Ln18.T5_T5_mg37.QPC4.izumi_gnu.cam-co2rmp (Overall: DIFF) details: + ERP_D_Ln9.C48_C48_mg17.QPC6.izumi_gnu.cam-outfrq9s (Overall: DIFF) details: + ERP_D_Ln9.ne3pg3_ne3pg3_mg37.QPC6.izumi_gnu.cam-outfrq9s_rrtmgp (Overall: DIFF) details: + ERP_Ln9.ne5_ne5_mg37.QPC5.izumi_gnu.cam-outfrq9s (Overall: DIFF) details: + PLB_D_Ln9.ne5pg3_ne5pg3_mg37.QPC5.izumi_gnu.cam-ttrac_loadbal0 (Overall: DIFF) details: + PLB_D_Ln9.ne5pg3_ne5pg3_mg37.QPC5.izumi_gnu.cam-ttrac_loadbal1 (Overall: DIFF) details: + PLB_D_Ln9.ne5pg3_ne5pg3_mg37.QPC5.izumi_gnu.cam-ttrac_loadbal3 (Overall: DIFF) details: + SCT_D_Ln7.T42_T42_mg17.QPC4.izumi_gnu.cam-scm_prep (Overall: DIFF) details: + SCT_D_Ln7.T42_T42_mg17.QPC6.izumi_gnu.cam-scm_prep_c6 (Overall: DIFF) details: + SMS_D_Ln3.f10_f10_mg37.QPMOZ.izumi_gnu.cam-outfrq3s_chemproc (Overall: DIFF) details: + SMS_D_Ln9.f10_f10_mg37.QPWmaC4.izumi_gnu.cam-outfrq9s_apmee (Overall: DIFF) details: + SMS_D_Ln9.ne5pg3_ne5pg3_mg37.QPC5.izumi_gnu.cam-outfrq3s_ttrac (Overall: DIFF) details: + SMS_P48x1_D_Ln9.f19_f19_mg17.FW4madSD.izumi_gnu.cam-outfrq9s (Overall: DIFF) details: + - Answer changes due to external updates - FIELDLIST differ only + +Summarize any changes to answers: FIELDLIST differences only + +=============================================================== =============================================================== Tag name: cam6_4_010 diff --git a/share b/share index 4b9dc4871a..f6f31fd61c 160000 --- a/share +++ b/share @@ -1 +1 @@ -Subproject commit 4b9dc4871a259f00f35bb47708d876cb7dcdf75c +Subproject commit f6f31fd61cb8f80aee97311fcca64b3e26b0202c