Merge pull request #802 from ARMmbed/hash-support

Git and GitHub support rework

docs/reference/commands.md

@@ -534,7 +534,7 @@ example:
 
 
 ## <a href="#yotta-uninstall" name="yotta-uninstall">#</a> yotta uninstall
-Synonyms: `yotta unlink`, `yotta rm`
+Synonyms: `yotta unlink`, `yotta rm`, `yotta un`
 #### Synopsis
 
 ```

docs/reference/module.md

@@ -198,6 +198,10 @@ To specify a dependency on a github module, use one of the following forms:
 
     Uses the latest committed version on the specified branch.
 
+ * `"usefulmodule": "username/repositoryname#commit-id"`
+
+    Uses the specified commit ID.
+
 #### Depending on git Modules
 To specify a module available from a non-Github git server as a dependency, use
 a git URL:
@@ -206,8 +210,9 @@ a git URL:
  * `"usefulmodule": "git+ssh://somwhere.com/anything/anywhere#<version specification>"`
  * `"usefulmodule": "git+ssh://somwhere.com/anything/anywhere#<branch name>"`
  * `"usefulmodule": "git+ssh://somwhere.com/anything/anywhere#<tag name>"`
+ * `"usefulmodule": "git+ssh://somwhere.com/anything/anywhere#<commit id>"`
  * `"usefulmodule": "<anything>://somwhere.git"`
- * `"usefulmodule": "<anything>://somwhere.git#<version spec, tag, or branch name>"`
+ * `"usefulmodule": "<anything>://somwhere.git#<version spec, tag, branch name or commit id>"`
 
 #### Depending on hg Modules
 To specify a module available from a mercurial server as a dependency, use

docs/tutorial/privaterepos.md

@@ -24,18 +24,33 @@ Sometimes it may not be appropriate publish a module to the public package regis
 
 The shorthand GitHub URL is formed of two parts: `<username>/<reponame>` where `<username>` is the GitHub user or organisation name of the repository owner and `<reponame>` is the name of the repositiry. e.g. the `yotta` repositry can be found at `ARMmbed/yotta`.
 
-You can specify a particular branch or tag to use by providing it in the URL. The supported GitHub URL formats are:
+You can specify a particular branch, tag or commit to use by providing it in the URL. The supported GitHub URL formats are:
 
 ```
 username/reponame
 username/reponame#<versionspec>
 username/reponame#<branchname>
 username/reponame#<tagname>
+username/reponame#<commit>
+https://github.com/username/reponame
+https://github.com/username/reponame#<branchname>
+https://github.com/username/reponame#<tagname>
+https://github.com/username/reponame#<commit>
 ```
 
+If the GitHub repository is public, the dependency will simply be downloaded. If the GitHub repository is private and this is the first time you are downloading from a private GitHub repository, you will be prompted to log in to GitHub using a URL.
+
+If you have a private GitHub repository and you would prefer to download it using SSH keys, you can use the following dependency form:
+
+```
+[email protected]:username/reponame.git
+[email protected]:username/reponame.git#<branchname>
+[email protected]:username/reponame.git#<tagname>
+[email protected]:username/reponame.git#<commit>
+```
 
 ###Other ways to depend on private repositories 
-Using shorthand GitHub URLs is the easiest and reccomneded method of working with private repositories, however as not all projects are hosted on GitHub, `yotta` supports using git and hg URLs directly as well.
+Using shorthand GitHub URLs is the easiest and recommended method of working with private repositories, however as not all projects are hosted on GitHub, `yotta` supports using git and hg URLs directly as well.
 
 For example, to include a privately hosted git repository from example.com:
 
@@ -47,13 +62,13 @@ For example, to include a privately hosted git repository from example.com:
 ...
 ```
 
-Git URLs support branch, version and tags specifications:
+Git URLs support branch, version, tag and commit specifications:
 
 ```
 git+ssh://example.com/path/to/repo
-git+ssh://example.com/path/to/repo#<versionspec, branch or tag>
+git+ssh://example.com/path/to/repo#<versionspec, branch, tag or commit>
 anything://example.com/path/to/repo.git
-anything://example.com/path/to/repo.git#<versionspec, branch or tag>
+anything://example.com/path/to/repo.git#<versionspec, branch, tag or commit>
 ```
 
 Currently, mercurial URLs only support a version specification:

yotta/install.py

@@ -167,7 +167,12 @@ def installComponentAsDependency(args, current_component):
     # (if it is not already present), and write that back to disk. Without
     # writing to disk the dependency wouldn't be usable.
     if installed and not current_component.hasDependency(component_name):
-        saved_spec = current_component.saveDependency(installed)
+        vs = sourceparse.parseSourceURL(component_spec)
+        if vs.source_type == 'registry':
+            saved_spec = current_component.saveDependency(installed)
+        else:
+            saved_spec = current_component.saveDependency(installed, component_spec)
+
         current_component.writeDescription()
         logging.info('dependency %s: %s written to module.json', component_name, saved_spec)
     else:

yotta/lib/access.py

@@ -147,8 +147,14 @@ def latestSuitableVersion(name, version_required, registry='modules', quiet=Fals
             )
             if v:
                 return v
+
+            # we have passed a specific commit ID:
+            v = remote_component.commitVersion()
+            if v:
+                return v
+
             raise access_common.Unavailable(
-                'Github repository "%s" does not have any tags or branches matching "%s"' % (
+                'Github repository "%s" does not have any tags, branches or commits matching "%s"' % (
                     version_required, remote_component.tagOrBranchSpec()
                 )
             )
@@ -189,8 +195,14 @@ def latestSuitableVersion(name, version_required, registry='modules', quiet=Fals
             )
             if v:
                 return v
+
+            # we have passed a specific commit ID:
+            v = local_clone.commitVersion(remote_component.tagOrBranchSpec())
+            if v:
+                return v
+
             raise access_common.Unavailable(
-                '%s repository "%s" does not have any tags or branches matching "%s"' % (
+                '%s repository "%s" does not have any tags, branches or commits matching "%s"' % (
                     clone_type, version_required, spec
                 )
             )

yotta/lib/git_access.py

@@ -75,6 +75,16 @@ def availableBranches(self):
     def tipVersion(self):
         return GitCloneVersion('', '', self)
 
+    def commitVersion(self, spec):
+        ''' return a GithubComponentVersion object for a specific commit if valid
+        '''
+        import re
+
+        commit_match = re.match('^[a-f0-9]{7,40}$', spec, re.I)
+        if commit_match:
+            return GitCloneVersion('', spec, self)
+
+        return None
 
 class GitComponent(access_common.RemoteComponent):
     def __init__(self, url, tag_or_branch=None, semantic_spec=None):

yotta/lib/github_access.py

@@ -141,6 +141,12 @@ def _getTipArchiveURL(repo):
     repo = g.get_repo(repo)
     return repo.get_archive_link('tarball')
 
+@_handleAuth
+def _getCommitArchiveURL(repo, commit):
+    ''' return a string containing a tarball url '''
+    g = Github(settings.getProperty('github', 'authtoken'))
+    repo = g.get_repo(repo)
+    return repo.get_archive_link('tarball', commit)
 
 @_handleAuth
 def _getTarball(url, into_directory, cache_key, origin_info=None):
@@ -283,6 +289,19 @@ def tipVersion(self):
             '', '', _getTipArchiveURL(self.repo), self.name, cache_key=None
         )
 
+    def commitVersion(self):
+        ''' return a GithubComponentVersion object for a specific commit if valid
+        '''
+        import re
+
+        commit_match = re.match('^[a-f0-9]{7,40}$', self.tagOrBranchSpec(), re.I)
+        if commit_match:
+            return GithubComponentVersion(
+                '', '', _getCommitArchiveURL(self.repo, self.tagOrBranchSpec()), self.name, cache_key=None
+            )
+
+        return None
+
     @classmethod
     def remoteType(cls):
         return 'github'

yotta/lib/sourceparse.py

@@ -51,39 +51,55 @@ def semanticSpecMatches(self, v):
             return self.semantic_spec.match(v)
 
 
-def _splitFragment(url):
-    parsed = urlsplit(url)
-    if '#' in url:
-        return url[:url.index('#')], parsed.fragment
-    else:
-        return url, None
-
-def _getGithubRef(source_url):
+def _getNonRegistryRef(source_url):
     import re
+
     # something/something#spec = github
-    defragmented, fragment = _splitFragment(source_url)
-    github_match = re.match('^[a-z0-9_-]+/([a-z0-9_-]+)$', defragmented, re.I)
+    # something/something@spec = github
+    # something/something spec = github
+    github_match = re.match('^([.a-z0-9_-]+/([.a-z0-9_-]+)) *[@#]?([.a-z0-9_\-\*\^\~\>\<\=]*)$', source_url, re.I)
     if github_match:
-        return github_match.group(1), VersionSource('github', defragmented, fragment)
+        return github_match.group(2), VersionSource('github', github_match.group(1), github_match.group(3))
 
-    # something/something@spec = github
-    alternate_github_match = re.match('([a-z0-9_-]+/([a-z0-9_-]+)) *@?([~^><=.0-9a-z\*-]*)$', source_url, re.I)
-    if alternate_github_match:
-        return alternate_github_match.group(2), VersionSource('github', alternate_github_match.group(1), alternate_github_match.group(3))
+    parsed = urlsplit(source_url)
+
+    # github
+    if parsed.netloc.endswith('github.com'):
+        # any URL onto github should be fetched over the github API, even if it
+        # would parse as a valid git URL
+        name_match = re.match('^/([.a-z0-9_-]+/([.a-z0-9_-]+?))(.git)?$', parsed.path, re.I)
+        if name_match:
+            return name_match.group(2), VersionSource('github', name_match.group(1), parsed.fragment)
+
+    if '#' in source_url:
+        without_fragment = source_url[:source_url.index('#')]
+    else:
+        without_fragment = source_url
+
+    # git
+    if parsed.scheme.startswith('git+') or parsed.path.endswith('.git'):
+        # git+anything://anything or anything.git is a git repo:
+        name_match = re.match('^.*?([.a-z0-9_-]+?)(.git)?$', parsed.path, re.I)
+        if name_match:
+            return name_match.group(1), VersionSource('git', without_fragment, parsed.fragment)
+
+    # mercurial
+    if parsed.scheme.startswith('hg+') or parsed.path.endswith('.hg'):
+        # hg+anything://anything or anything.hg is a hg repo:
+        name_match = re.match('^.*?([.a-z0-9_-]+?)(.hg)?$', parsed.path, re.I)
+        if name_match:
+            return name_match.group(1), VersionSource('hg', without_fragment, parsed.fragment)
 
     return None, None
 
+
 def parseSourceURL(source_url):
     ''' Parse the specified version source URL (or version spec), and return an
         instance of VersionSource
     '''
-    import re
-    parsed = urlsplit(source_url)
-
-    if '#' in source_url:
-        without_fragment = source_url[:source_url.index('#')]
-    else:
-        without_fragment = source_url
+    name, spec = _getNonRegistryRef(source_url)
+    if spec:
+        return spec
 
     try:
         url_is_spec = version.Spec(source_url)
@@ -94,22 +110,6 @@ def parseSourceURL(source_url):
         # if the url is an unadorned version specification (including an empty
         # string) then the source is the module registry:
         return VersionSource('registry', '', source_url)
-    elif parsed.netloc.endswith('github.com'):
-        # any URL onto github should be fetched over the github API, even if it
-        # would parse as a valid git URL
-        return VersionSource('github', parsed.path, parsed.fragment)
-    elif parsed.scheme.startswith('git+') or parsed.path.endswith('.git'):
-        # git+anything://anything or anything.git is a git repo:
-        return VersionSource('git', without_fragment, parsed.fragment)
-    elif parsed.scheme.startswith('hg+') or parsed.path.endswith('.hg'):
-        # hg+anything://anything or anything.hg is a hg repo:
-        return VersionSource('hg', without_fragment, parsed.fragment)
-
-    # something/something@spec = github
-    # something/something#spec = github
-    module_name, github_match = _getGithubRef(source_url)
-    if github_match:
-        return github_match
 
     raise InvalidVersionSpec("Invalid version specification: \"%s\"" % (source_url))
 
@@ -143,8 +143,8 @@ def parseTargetNameAndSpec(target_name_and_spec):
     import re
     # fist check if this is a raw github specification that we can get the
     # target name from:
-    name, spec = _getGithubRef(target_name_and_spec)
-    if name and spec:
+    name, spec = _getNonRegistryRef(target_name_and_spec)
+    if name:
         return name, target_name_and_spec
 
     # next split at the first @ or , if any
@@ -178,8 +178,8 @@ def parseModuleNameAndSpec(module_name_and_spec):
     import re
     # fist check if this is a raw github specification that we can get the
     # module name from:
-    name, spec = _getGithubRef(module_name_and_spec)
-    if name and spec:
+    name, spec = _getNonRegistryRef(module_name_and_spec)
+    if name:
         return name, module_name_and_spec
 
     # next split at the first @, if any

yotta/main.py

@@ -201,6 +201,7 @@ def onParserAdded(parser):
     short_commands = {
                 'up':subparser.choices['update'],
                 'in':subparser.choices['install'],
+                'un':subparser.choices['uninstall'],
                 'ln':subparser.choices['link'],
                  'v':subparser.choices['version'],
                 'ls':subparser.choices['list'],