Merge pull request #77 from GEOS-ESM/develop

Merge Develop into master for release

mepo.d/cmdline/parser.py

@@ -39,7 +39,7 @@ def __init(self):
             'init',
             description = 'Initialize mepo based on <config-file>')
         init.add_argument(
-            'config_file',
+            '--config',
             metavar = 'config-file',
             nargs = '?',
             default = 'components.yaml',
@@ -48,7 +48,18 @@ def __init(self):
     def __clone(self):
         clone = self.subparsers.add_parser(
             'clone',
-            description = "Clone repositories. Command 'mepo init' should have already been run")
+            description = "Clone repositories.")
+        clone.add_argument(
+            'repo_url',
+            nargs = '?',
+            default = None,
+            help = 'URL to clone')
+        clone.add_argument(
+            '--config',
+            metavar = 'config-file',
+            nargs = '?',
+            default = 'components.yaml',
+            help = 'Configuration file (ignored if init already called, default: %(default)s)')
 
     def __list(self):
         listcomps = self.subparsers.add_parser(
@@ -90,6 +101,8 @@ def __fetch(self):
             'Specifying --all causes all remotes to be fetched.')
         fetch.add_argument('comp_name', metavar = 'comp-name', nargs = '+')
         fetch.add_argument('--all', action = 'store_true', help = 'Fetch all remotes.')
+        fetch.add_argument('--prune','-p', action = 'store_true', help = 'Prune remote branches.')
+        fetch.add_argument('--tags','-t', action = 'store_true', help = 'Fetch tags.')
 
     def __branch(self):
         branch = self.subparsers.add_parser('branch')

mepo.d/command/clone/clone.py

@@ -1,8 +1,36 @@
-from state.state import MepoState
+from state.state    import MepoState, StateDoesNotExistError
 from repository.git import GitRepository
+from command.init   import init as mepo_init
+from utilities      import shellcmd
+from urllib.parse import urlparse
+
+import os
+import pathlib
 
 def run(args):
-    allcomps = MepoState.read_state()
+
+    if args.repo_url:
+        p = urlparse(args.repo_url)
+        last_url_node = p.path.rsplit('/')[-1]
+        url_suffix = pathlib.Path(last_url_node).suffix
+        if url_suffix == '.git':
+            git_url_directory = pathlib.Path(last_url_node).stem
+        else:
+            git_url_directory = last_url_node
+
+        local_clone(args.repo_url)
+        os.chdir(git_url_directory)
+
+    # This tries to read the state and if not, calls init,
+    # loops back, and reads the state
+    while True:
+        try:
+            allcomps = MepoState.read_state()
+        except StateDoesNotExistError:
+            mepo_init.run(args)
+            continue
+        break
+
     max_namelen = len(max([comp.name for comp in allcomps], key=len))
     for comp in allcomps:
         git = GitRepository(comp.remote, comp.local)
@@ -16,3 +44,8 @@ def run(args):
 def print_clone_info(comp, name_width):
     ver_name_type = '({}) {}'.format(comp.version.type, comp.version.name)
     print('{:<{width}} | {:<s}'.format(comp.name, ver_name_type, width = name_width))
+
+def local_clone(url):
+    cmd = 'git clone '
+    cmd += '--quiet {}'.format(url)
+    shellcmd.run(cmd.split())

mepo.d/command/init/init.py

@@ -1,5 +1,5 @@
 from state.state import MepoState
 
 def run(args):
-    allcomps = MepoState.initialize(args.config_file)
-    print('Initialized mepo!')
+    allcomps = MepoState.initialize(args.config)
+    print('Initializing mepo using {}'.format(args.config))

mepo.d/config/config_file.py

@@ -1,5 +1,7 @@
 import pathlib
 
+from state.exceptions import SuffixNotRecognizedError
+
 class ConfigFile(object):
 
     __slots__ = ['__filename', '__filetype']
@@ -11,7 +13,7 @@ def __init__(self, filename):
         if file_suffix in SUFFIX_LIST:
             self.__filetype = file_suffix[1:]
         else:
-            raise RuntimeError('suffix {} not supported'.format(file_suffix))
+            raise SuffixNotRecognizedError('suffix {} not supported'.format(file_suffix))
 
     def read_file(self):
         '''Call read_yaml, read_json etc. using dispatch pattern'''

mepo.d/repository/git.py

@@ -4,6 +4,7 @@
 
 from utilities import shellcmd
 from utilities import colors
+from urllib.parse import urljoin
 
 class GitRepository(object):
     """
@@ -26,7 +27,13 @@ def clone(self, recurse):
         cmd = 'git clone '
         if recurse:
             cmd += '--recurse-submodules '
-        cmd += '--quiet {} {}'.format(self.__remote, self.__local)
+        if self.__remote.startswith('..'):
+            rel_remote = os.path.basename(self.__remote)
+            fixture_url = get_current_remote_url()
+            remote = urljoin(fixture_url,rel_remote)
+        else:
+            remote = self.__remote
+        cmd += '--quiet {} {}'.format(remote, self.__local)
         shellcmd.run(cmd.split())
 
     def checkout(self, version):
@@ -59,6 +66,10 @@ def fetch(self, args=None):
         cmd = self.__git + ' fetch'
         if args.all:
             cmd += ' --all'
+        if args.prune:
+            cmd += ' --prune'
+        if args.tags:
+            cmd += ' --tags'
         return shellcmd.run(cmd.split(), output=True)
 
     def create_branch(self, branch_name):
@@ -228,4 +239,15 @@ def get_version(self):
             else:
                 name = tmp
                 tYpe = 'b'
+        elif output.startswith('HEAD'): # Assume hash
+            cmd = self.__git + ' rev-parse --short HEAD'
+            hash_out = shellcmd.run(cmd.split(), output=True)
+            detached = True
+            name = hash_out.rstrip()
+            tYpe = 'h'
         return (name, tYpe, detached)
+
+def get_current_remote_url():
+    cmd = 'git remote get-url origin'
+    output = shellcmd.run(cmd.split(), output=True).strip()
+    return output

mepo.d/state/component.py

@@ -24,6 +24,10 @@ def __set_original_version(self, comp_details):
             # SPECIAL HANDLING of 'detached head' branches
             ver_name = 'origin/' + comp_details['branch']
             ver_type = 'b'
+        elif comp_details.get('hash', None):
+            # Hashes don't have to exist
+            ver_name = comp_details['hash'][:7]
+            ver_type = 'h'
         else:
             ver_name = comp_details['tag'] # 'tag' key has to exist
             ver_type = 't'

mepo.d/state/exceptions.py

@@ -0,0 +1,15 @@
+class StateDoesNotExistError(Exception):
+    """Raised when the mepo state does not exist"""
+    pass
+
+class StateAlreadyInitializedError(Exception):
+    """Raised when the mepo state has already been initialized"""
+    pass
+
+class ConfigFileNotFoundError(FileNotFoundError):
+    """Raised when the config file is not found"""
+    pass
+
+class SuffixNotRecognizedError(RuntimeError):
+    """Raised when the config suffix is not recognized"""
+    pass

mepo.d/state/state.py

@@ -8,7 +8,7 @@
 from state.component import MepoComponent
 from utilities import shellcmd
 from pathlib import Path
-from urllib.parse import urljoin
+from state.exceptions import StateDoesNotExistError, StateAlreadyInitializedError
 
 class MepoState(object):
 
@@ -55,24 +55,17 @@ def exists(cls):
     @classmethod
     def initialize(cls, project_config_file):
         if cls.exists():
-            raise Exception('mepo state already exists')
+            raise StateAlreadyInitializedError('mepo state already exists')
         input_components = ConfigFile(project_config_file).read_file()
         complist = list()
         for name, comp in input_components.items():
-            for key, value in comp.items():
-                if key == "remote":
-                    if comp[key].startswith('..'):
-                        rel_remote = os.path.basename(comp[key])
-                        fixture_url = get_current_remote_url()
-                        resolved_remote = urljoin(fixture_url,rel_remote)
-                        comp[key] = resolved_remote
             complist.append(MepoComponent().to_component(name, comp))
         cls.write_state(complist)
 
     @classmethod
     def read_state(cls):
         if not cls.exists():
-            raise Exception('mepo state does not exist')
+            raise StateDoesNotExistError('mepo state does not exist')
         with open(cls.get_file(), 'rb') as fin:
             allcomps = pickle.load(fin)
         return allcomps
@@ -96,8 +89,3 @@ def write_state(cls, state_details):
         if os.path.isfile(state_fileptr):
             os.remove(state_fileptr)
         os.symlink(new_state_file, state_fileptr)
-
-def get_current_remote_url():
-    cmd = 'git remote get-url origin'
-    output = shellcmd.run(cmd.split(), output=True).strip()
-    return output

mepo.d/utest/test_mepo_commands.py

@@ -39,10 +39,11 @@ def setUpClass(cls):
             shutil.rmtree(cls.fixture_dir)
         cls.__checkout_fixture()
         cls.__copy_config_file()
-        args.config_file = 'components.yaml'
+        args.config = 'components.yaml'
         os.chdir(cls.fixture_dir)
         mepo_init.run(args)
         args.config = None
+        args.repo_url = None
         mepo_clone.run(args)
 
     def setUp(self):