Merge pull request #5 from JonathonReinhart/use-python-gitlab-and-upd…

…ate-for-v4-api

Use python-gitlab and update for v4 API

README.md

@@ -15,23 +15,23 @@
     ```yaml
     - project: kosma/foobar-documentation
       ref: branches/stable
-      build: doc
+      job: doc
       install:
         build/apidoc/html/: docs/api/
         VERSION: docs/VERSION
     - project: kosma/foobar-firmware
       ref: tags/1.4.0
-      build: firmware-8051
+      job: firmware-8051
       install:
         build/8051/release/firmware.bin: blobs/firmware-8051.blob
     - project: kosma/foobar-icons
       ref: 69881ebc852f5e02b8328c6b9da615e90b7184b2
-      build: icons
+      job: icons
       install:
         .: icons/
     ```
 
-3. Run `art update` to automatically determine latest versions and build numbers
+3. Run `art update` to automatically determine latest versions and job numbers
    of needed projects and save them into `artifacts.lock.yml`. Commit both files
    to version control system.
 
@@ -41,7 +41,7 @@
 ## The Lockfile
 
 The `artifacts.lock.yml` is conceptually similar to Ruby's `Gemfile.lock`: it
-allows locking to exact revisions and builds while still semantically tracking
+allows locking to exact revisions and jobs while still semantically tracking
 tags or branches and allowing easy updates when needs arise. The following good
 practices should be followed:
 
@@ -68,7 +68,7 @@ cache:
 
 `art` uses [appdirs](https://github.com/ActiveState/appdirs) to store configuration
 and cache files. When running under CI environment, the default cache directory is
-automatically set to `.art-cache` so it can be preserved across builds.
+automatically set to `.art-cache` so it can be preserved across jobs.
 
 ## Bugs and limitations
 

art/__main__.py

@@ -0,0 +1,2 @@
+from .command_line import main
+main()

art/_cache.py

@@ -1,24 +1,34 @@
 # -*- coding: utf-8 -*-
 
 from __future__ import absolute_import
+from contextlib import contextmanager
 
 import errno
 import os
 from . import _paths
 
 
-def save(filename, content):
-    path = os.path.join(_paths.cache_dir, filename)
+def cache_path(filename):
+    return os.path.join(_paths.cache_dir, filename)
+
+@contextmanager
+def save_file(filename):
+    path = cache_path(filename)
     path_tmp = path + '.tmp'
     _paths.mkdirs(os.path.dirname(path))
     with open(path_tmp, 'wb') as stream:
-        stream.write(content)
+        yield stream
     os.rename(path_tmp, path)
 
 
+def save(filename, content):
+    with save_file(filename) as f:
+        f.write(content)
+
+
 def get(filename):
     try:
-        return open(os.path.join(_paths.cache_dir, filename), 'rb')
+        return open(cache_path(filename), 'rb')
     except IOError as exc:
         # translate "No such file or directory" into KeyError
         if exc.errno == errno.ENOENT:

art/_config.py

@@ -11,7 +11,7 @@
 def save(gitlab_url, private_token):
     config = {
         'gitlab_url': gitlab_url,
-        'auth_header': {'PRIVATE-TOKEN': private_token}
+        'private_token': private_token,
     }
     _paths.mkdirs(os.path.dirname(_paths.config_file))
     _yaml.save(_paths.config_file, config)

art/command_line.py

@@ -7,12 +7,33 @@
 import sys
 import zipfile
 import click
+from gitlab import Gitlab
 from . import _cache
 from . import _config
-from . import _gitlab
 from . import _paths
 from . import _yaml
 
+def get_gitlab():
+    config = _config.load()
+    return Gitlab(config['gitlab_url'], private_token=config['private_token'])
+
+
+def get_commit_last_successful_job(project, commit, job_name):
+    pipelines = project.pipelines.list(as_list=False, sha=commit, order_by='id', sort='desc')
+    for pipeline in pipelines:
+        jobs = pipeline.jobs.list(as_list=False, scope='success')
+        for job in jobs:
+            if job.name == job_name:
+                # Turn ProjectPipelineJob into ProjectJob
+                return project.jobs.get(job.id, lazy=True)
+
+    raise Exception("Could not find latest successful '{}' job for commit {}".format(
+        job_name, commit))
+
+
+def zip_name(project, job_id):
+    return os.path.join(project, '{}.zip'.format(job_id))
+
 
 @click.group()
 @click.option('--cache', '-c', help='Download cache directory.')
@@ -40,15 +61,15 @@ def configure(**kwargs):
 def update():
     """Update latest tag/branch commits."""
 
-    config = _config.load()
-    gitlab = _gitlab.Gitlab(**config)
+    gitlab = get_gitlab()
     artifacts = _yaml.load(_paths.artifacts_file)
 
     for entry in artifacts:
-        entry['commit'] = gitlab.get_ref_commit(entry['project'], entry['ref'])
-        entry['build_id'] = gitlab.get_commit_last_successful_build(entry['project'], entry['commit'], entry['build'])
+        proj = gitlab.projects.get(entry['project'])
+        entry['commit'] = proj.commits.get(entry['ref']).id
+        entry['job_id'] = get_commit_last_successful_job(proj, entry['commit'], entry['job']).id
         click.echo('* %s: %s => %s => %s' % (
-            entry['project'], entry['ref'], entry['commit'], entry['build_id']), sys.stderr)
+            entry['project'], entry['ref'], entry['commit'], entry['job_id']), sys.stderr)
 
     _yaml.save(_paths.artifacts_lock_file, artifacts)
 
@@ -57,21 +78,22 @@ def update():
 def download():
     """Download artifacts to local cache."""
 
-    config = _config.load()
-    gitlab = _gitlab.Gitlab(**config)
+    gitlab = get_gitlab()
     artifacts_lock = _yaml.load(_paths.artifacts_lock_file)
 
     for entry in artifacts_lock:
-        filename = '%s/%s.zip' % (entry['project'], entry['build_id'])
+        filename = zip_name(entry['project'], entry['job_id'])
         try:
             _cache.get(filename)
         except KeyError:
-            click.echo('* %s: %s => downloading...' % (entry['project'], entry['build_id']))
-            artifacts_zip = gitlab.get_artifacts_zip(entry['project'], entry['build_id'])
-            _cache.save(filename, artifacts_zip)
-            click.echo('* %s: %s => downloaded.' % (entry['project'], entry['build_id']))
+            click.echo('* %s: %s => downloading...' % (entry['project'], entry['job_id']))
+            proj = gitlab.projects.get(entry['project'])
+            job = proj.jobs.get(entry['job_id'], lazy=True)
+            with _cache.save_file(filename) as f:
+                job.artifacts(streamed=True, action=f.write)
+            click.echo('* %s: %s => downloaded.' % (entry['project'], entry['job_id']))
         else:
-            click.echo('* %s: %s => present' % (entry['project'], entry['build_id']))
+            click.echo('* %s: %s => present' % (entry['project'], entry['job_id']))
 
 
 @main.command()
@@ -108,7 +130,7 @@ def install():
         del source, destination # pylint: disable=undefined-loop-variable
 
         # open the artifacts.zip archive
-        filename = '%s/%s.zip' % (entry['project'], entry['build_id'])
+        filename = zip_name(entry['project'], entry['job_id'])
         archive_file = _cache.get(filename)
         archive = zipfile.ZipFile(archive_file)
 

setup.py

@@ -13,7 +13,7 @@
         'PyYAML',
         'appdirs',
         'click',
-        'requests',
+        'python-gitlab',
     ],
     entry_points={
         'console_scripts': [