Add a GitLab manifest provider

src/rosdistro/distribution.py

@@ -34,14 +34,15 @@
 from .manifest_provider.bitbucket import bitbucket_manifest_provider
 from .manifest_provider.git import git_manifest_provider, git_source_manifest_provider
 from .manifest_provider.github import github_manifest_provider, github_source_manifest_provider
+from .manifest_provider.gitlab import gitlab_manifest_provider, gitlab_source_manifest_provider
 from .manifest_provider.tar import tar_manifest_provider, tar_source_manifest_provider
 from .package import Package
 
 
 class Distribution(object):
 
-    default_manifest_providers = [github_manifest_provider, bitbucket_manifest_provider, git_manifest_provider, tar_manifest_provider]
-    default_source_manifest_providers = [github_source_manifest_provider, git_source_manifest_provider, tar_source_manifest_provider]
+    default_manifest_providers = [github_manifest_provider, gitlab_manifest_provider, bitbucket_manifest_provider, git_manifest_provider, tar_manifest_provider]
+    default_source_manifest_providers = [github_source_manifest_provider, gitlab_source_manifest_provider, git_source_manifest_provider, tar_source_manifest_provider]
 
     def __init__(self, distribution_file, manifest_providers=None, source_manifest_providers=None):
         self._distribution_file = distribution_file

src/rosdistro/manifest_provider/gitlab.py

@@ -0,0 +1,137 @@
+# Software License Agreement (BSD License)
+#
+# Copyright (c) 2021, Open Source Robotics Foundation, Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above
+#    copyright notice, this list of conditions and the following
+#    disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of Open Source Robotics Foundation, Inc. nor
+#    the names of its contributors may be used to endorse or promote
+#    products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+try:
+    from urllib.request import urlopen, Request
+    from urllib.error import URLError
+    from urllib.parse import quote as urlquote
+except ImportError:
+    from urllib2 import urlopen, Request
+    from urllib2 import URLError
+    from urllib2.parse import quote as urlquote
+
+import json
+import os
+import re
+
+from catkin_pkg.package import parse_package_string
+
+from rosdistro.source_repository_cache import SourceRepositoryCache
+from rosdistro import logger
+
+
+def _gitlab_paged_api_query(path, resource, attrs):
+    _attrs = {'per_page': 50}
+    _attrs.update(attrs)
+    _attrs.pop('pagination', None)
+    _attrs.pop('page', None)
+
+    url = 'https://gitlab.com/api/v4/projects/%s/%s?pagination=keyset' % (urlquote(path, safe=''), resource)
+    for k, v in _attrs.items():
+        url += '&%s=%s' % (k, urlquote(str(v), safe=''))
+
+    while True:
+        with urlopen(url) as res:
+            for result in json.loads(res.read().decode('utf-8')):
+                yield result
+
+            # Get the URL to the next page
+            links = res.getheader('Link')
+            if not links:
+                break
+            match = re.match(r'.*<([^>]*)>; rel="next"', links)
+            if not match:
+                break
+            url = match.group(1)
+
+
+def gitlab_manifest_provider(_dist_name, repo, pkg_name):
+    assert repo.version
+    server, path = repo.get_url_parts()
+    if not server.endswith('gitlab.com'):
+        logger.debug('Skip non-gitlab url "%s"' % repo.url)
+        raise RuntimeError('can not handle non gitlab urls')
+
+    release_tag = repo.get_release_tag(pkg_name)
+
+    if not repo.has_remote_tag(release_tag):
+        raise RuntimeError('specified tag "%s" is not a git tag' % release_tag)
+
+    url = 'https://gitlab.com/%s/-/raw/%s/package.xml' % (path, release_tag)
+    try:
+        logger.debug('Load package.xml file from url "%s"' % url)
+        return urlopen(url).read().decode('utf-8')
+    except URLError as e:
+        logger.debug('- failed (%s), trying "%s"' % (e, url))
+        raise RuntimeError()
+
+
+def gitlab_source_manifest_provider(repo):
+    assert repo.version
+    server, path = repo.get_url_parts()
+    if not server.endswith('gitlab.com'):
+        logger.debug('Skip non-gitlab url "%s"' % repo.url)
+        raise RuntimeError('can not handle non gitlab urls')
+
+    # Resolve the version ref to a sha
+    sha = next(_gitlab_paged_api_query(path, 'repository/commits', {'per_page': 1, 'ref_name': repo.version}))['id']
+
+    # Look for package.xml files in the tree
+    package_xml_paths = set()
+    for obj in _gitlab_paged_api_query(path, 'repository/tree', {'recursive': 'true', 'ref': sha}):
+        if obj['path'].split('/')[-1] == 'package.xml':
+            package_xml_paths.add(os.path.dirname(obj['path']))
+
+    # Filter out ones that are inside other packages (eg, part of tests)
+    def package_xml_in_parent(path):
+        if path == '':
+            return True
+        parent = path
+        while True:
+            parent = os.path.dirname(parent)
+            if parent in package_xml_paths:
+                return False
+            if parent == '':
+                return True
+    package_xml_paths = list(filter(package_xml_in_parent, package_xml_paths))
+
+    cache = SourceRepositoryCache.from_ref(sha)
+    for package_xml_path in package_xml_paths:
+        url = 'https://gitlab.com/%s/-/raw/%s/%s' % \
+            (path, sha, package_xml_path + '/package.xml' if package_xml_path else 'package.xml')
+        logger.debug('- load package.xml from %s' % url)
+        package_xml = urlopen(url).read().decode('utf-8')
+        name = parse_package_string(package_xml).name
+        cache.add(name, package_xml_path, package_xml)
+
+    return cache

src/rosdistro/release.py

@@ -34,11 +34,12 @@
 from .manifest_provider.bitbucket import bitbucket_manifest_provider
 from .manifest_provider.git import git_manifest_provider
 from .manifest_provider.github import github_manifest_provider
+from .manifest_provider.gitlab import gitlab_manifest_provider
 
 
 class Release(object):
 
-    default_manifest_providers = [github_manifest_provider, bitbucket_manifest_provider, git_manifest_provider]
+    default_manifest_providers = [github_manifest_provider, gitlab_manifest_provider, bitbucket_manifest_provider, git_manifest_provider]
 
     def __init__(self, rel_file, manifest_providers=None):
         self._rel_file = rel_file

test/test_manifest_providers.py

@@ -8,6 +8,7 @@
 from rosdistro.manifest_provider.bitbucket import bitbucket_manifest_provider
 from rosdistro.manifest_provider.cache import CachedManifestProvider, sanitize_xml
 from rosdistro.manifest_provider.git import git_manifest_provider, git_source_manifest_provider
+from rosdistro.manifest_provider.gitlab import gitlab_manifest_provider, gitlab_source_manifest_provider
 from rosdistro.release_repository_specification import ReleaseRepositorySpecification
 from rosdistro.source_repository_specification import SourceRepositorySpecification
 
@@ -16,6 +17,10 @@ def test_bitbucket():
     assert '</package>' in bitbucket_manifest_provider('indigo', _rospeex_release_repo(), 'rospeex_msgs')
 
 
+def test_gitlab():
+    assert '</package>' in gitlab_manifest_provider('foxy', _tracetools_analysis_release_repo(), 'tracetools_analysis')
+
+
 def test_cached():
     class FakeDistributionCache(object):
         def __init__(self):
@@ -102,6 +107,17 @@ def test_github_source():
     assert '<version>0.5.11</version>' in package_xml
 
 
+def test_gitlab_source():
+    repo_cache = gitlab_source_manifest_provider(_tracetools_analysis_source_repo())
+
+    # This hash corresponds to the 1.0.3 tag.
+    assert repo_cache.ref() == 'cd30853005ef3a591cb8594b4aa49f9ef400d30f'
+
+    package_path, package_xml = repo_cache['ros2trace_analysis']
+    assert 'ros2trace_analysis' == package_path
+    assert '<version>1.0.3</version>' in package_xml
+
+
 def test_git_source_multi():
     repo_cache = git_source_manifest_provider(_ros_source_repo())
     assert repo_cache.ref()
@@ -175,3 +191,19 @@ def _rospeex_release_repo():
         'url': 'https://bitbucket.org/rospeex/rospeex-release.git',
         'version': '2.14.7-0'
     })
+
+
+def _tracetools_analysis_release_repo():
+    return ReleaseRepositorySpecification('tracetools_analysis', {
+        'packages': ['ros2trace_analysis', 'tracetools_analysis'],
+        'tags': {'release': 'release/foxy/{package}/{version}'},
+        'url': 'https://gitlab.com/ros-tracing/tracetools_analysis-release.git',
+        'version': '1.0.3-1'
+    })
+
+
+def _tracetools_analysis_source_repo():
+    return SourceRepositorySpecification('tracetools_analysis', {
+        'url': 'https://gitlab.com/ros-tracing/tracetools_analysis.git',
+        'version': '1.0.3'
+    })