Merge pull request #396 from indigo-dc/devel3

Devel3

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## udocker (1.3.9)
+
+* add support to access non-config metadata from containers
+* added support for multiplatform manifests and indices solves #392
+
 ## udocker (1.3.8)
 
 * build udockertools 1.2.9 and set it as default

codemeta.json

@@ -6,7 +6,7 @@
     "@type": "SoftwareSourceCode",
     "identifier": "udocker",
     "name": "udocker",
-    "version": "1.3.8",
+    "version": "1.3.9",
     "description": "A basic user tool to execute simple docker containers in batch or interactive systems without root privileges",
     "license": "Apache Software License 2.0, OSI Approved :: Apache Software License",
     "author": [

docs/installation_manual.md

@@ -32,18 +32,18 @@ udocker requires:
 Download a release tarball from <https://github.com/indigo-dc/udocker/releases>:
 
 ```bash
-wget https://github.com/indigo-dc/udocker/releases/download/1.3.8/udocker-1.3.8.tar.gz
-tar zxvf udocker-1.3.8.tar.gz
-export PATH=`pwd`/udocker-1.3.8/udocker:$PATH
+wget https://github.com/indigo-dc/udocker/releases/download/1.3.9/udocker-1.3.9.tar.gz
+tar zxvf udocker-1.3.9.tar.gz
+export PATH=`pwd`/udocker-1.3.9/udocker:$PATH
 ```
 
 Alternatively use `curl` instead of `wget` as follows:
 
 ```bash
-curl -L https://github.com/indigo-dc/udocker/releases/download/1.3.8/udocker-1.3.8.tar.gz \
-  > udocker-1.3.8.tar.gz
-tar zxvf udocker-1.3.8.tar.gz
-export PATH=`pwd`/udocker-1.3.8/udocker:$PATH
+curl -L https://github.com/indigo-dc/udocker/releases/download/1.3.9/udocker-1.3.9.tar.gz \
+  > udocker-1.3.9.tar.gz
+tar zxvf udocker-1.3.9.tar.gz
+export PATH=`pwd`/udocker-1.3.9/udocker:$PATH
 ```
 
 udocker executes containers using external tools and libraries that
@@ -345,8 +345,8 @@ The udocker tool should be installed as shown in section 2.1:
 
 ```bash
 cd /sw
-wget https://github.com/indigo-dc/udocker/releases/download/1.3.8/udocker-1.3.8.tar.gz
-tar zxvf udocker-1.3.8.tar.gz
+wget https://github.com/indigo-dc/udocker/releases/download/1.3.9/udocker-1.3.9.tar.gz
+tar zxvf udocker-1.3.9.tar.gz
 ```
 
 Directing users to the central udocker installation can be done using the

udocker/__init__.py

@@ -31,5 +31,5 @@
     "Singularity http://singularity.lbl.gov"
     ]
 __license__ = "Licensed under the Apache License, Version 2.0"
-__version__ = "1.3.8"
+__version__ = "1.3.9"
 __date__ = "2023"

udocker/cli.py

@@ -532,6 +532,7 @@ def do_pull(self, cmdp):
         --httpproxy=socks5h://host:port                 :use http proxy
         --index=https://index.docker.io/v1              :docker index
         --registry=https://registry-1.docker.io         :docker registry
+        --platform=os/arch                              :docker platform
 
         Examples:
           pull fedora:latest
@@ -540,14 +541,15 @@ def do_pull(self, cmdp):
         index_url = cmdp.get("--index=")
         registry_url = cmdp.get("--registry=")
         http_proxy = cmdp.get("--httpproxy=")
+        platform = cmdp.get("--platform=")
         (imagerepo, tag) = self._check_imagespec(cmdp.get("P1"))
         if (not imagerepo) or cmdp.missing_options():    # syntax error
             return self.STATUS_ERROR
 
         self._set_repository(registry_url, index_url, imagerepo, http_proxy)
         v2_auth_token = self.keystore.get(self.dockerioapi.registry_url)
         self.dockerioapi.set_v2_login_token(v2_auth_token)
-        if self.dockerioapi.get(imagerepo, tag):
+        if self.dockerioapi.get(imagerepo, tag, platform):
             return self.STATUS_OK
 
         Msg().err("Error: no files downloaded")
@@ -690,6 +692,10 @@ def _get_run_options(self, cmdp, exec_engine=None):
             "nobanner": {
                 "fl": ("--nobanner",), "act": 'R',
                 "p2": "CMD_OPT", "p3": False
+            },
+            "platform": {
+                "fl": ("--platform=",), "act": 'R',
+                "p2": "CMD_OPT", "p3": False
             }
         }
         for option, cmdp_args in list(cmd_options.items()):

udocker/container/structure.py

@@ -4,7 +4,6 @@
 import os
 import subprocess
 
-from udocker.genstr import is_genstr
 from udocker.config import Config
 from udocker.msg import Msg
 from udocker.helper.unique import Unique
@@ -45,33 +44,39 @@ def get_container_attr(self):
 
         return (container_dir, container_json)
 
-    def get_container_meta(self, param, default, container_json):
-        """Get the container metadata from the container"""
-        confidx = ""
-        if "config" in container_json:
-            confidx = "config"
-        elif "container_config" in container_json:
-            confidx = "container_config"
-        if container_json[confidx] and param in container_json[confidx]:
-            if container_json[confidx][param] is None:
-                pass
-            elif (is_genstr(container_json[confidx][param]) and
-                  (isinstance(default, (list, tuple)))):
-                return container_json[confidx][param].strip().split()
-            elif (is_genstr(default) and (
-                    isinstance(container_json[confidx][param], (list, tuple)))):
-                return " ".join(container_json[confidx][param])
-            elif (is_genstr(default) and (
-                    isinstance(container_json[confidx][param], dict))):
-                return self._dict_to_str(container_json[confidx][param])
-            elif (isinstance(default, list) and (
-                    isinstance(container_json[confidx][param], dict))):
-                return self._dict_to_list(container_json[confidx][param])
-            else:
-                return container_json[confidx][param]
+    def get_container_meta(self, param, default, cntjson):
+        """Get the metadata configuration from the container"""
+        cidx = ""
+        if "config" in cntjson:
+            cidx = "config"
+        elif "container_config" in cntjson:
+            cidx = "container_config"
+
+        meta_item = None
+        if cntjson[cidx] and param in cntjson[cidx]:
+            meta_item = cntjson[cidx][param]
+        elif param in cntjson:
+            meta_item = cntjson[param]
+
+        if meta_item is None:
+            pass
+        elif (isinstance(meta_item, str) and
+              (isinstance(default, (list, tuple)))):
+            return meta_item.strip().split()
+        elif (isinstance(default, str) and
+              (isinstance(meta_item, (list, tuple)))):
+            return " ".join(meta_item)
+        elif (isinstance(default, str) and
+              (isinstance(meta_item, dict))):
+            return self._dict_to_str(meta_item)
+        elif (isinstance(default, list) and
+              (isinstance(meta_item, dict))):
+            return self._dict_to_list(meta_item)
+        else:
+            return meta_item
+
         return default
 
-    # DEBUG
     def _dict_to_str(self, in_dict):
         """Convert dict to str"""
         out_str = ""

udocker/docker.py

@@ -14,6 +14,7 @@
 from udocker.utils.fileutil import FileUtil
 from udocker.utils.curl import GetURL
 from udocker.utils.chksum import ChkSUM
+from udocker.helper.hostinfo import HostInfo
 
 
 class DockerIoAPI(object):
@@ -96,7 +97,11 @@ def _get_url(self, *args, **kwargs):
                                                     kwargs["RETRY"])
                 if "/v1/" in url:
                     auth_header = self._get_v1_auth(www_authenticate)
-                auth_kwargs.update({"header": [auth_header]})
+                # OCI and multiplatform, prevent removal of header attributes
+                try:
+                    auth_kwargs["header"].append(auth_header)
+                except KeyError:
+                    auth_kwargs.update({"header": [auth_header]})
         (hdr, buf) = self._get_url(*args, **auth_kwargs)
         return (hdr, buf)
 
@@ -356,18 +361,71 @@ def get_v2_image_tags(self, imagerepo, tags_only=False):
         except (IOError, OSError, AttributeError, ValueError, TypeError):
             return []
 
-    def get_v2_image_manifest(self, imagerepo, tag):
-        """Get the image manifest which contains JSON metadata
+    def _get_v2_digest_from_image_index(self, image_index, platform):
+        """Get OCI or docker manifest from an image index"""
+        if isinstance(image_index, dict):
+            index_list = image_index
+        else:
+            try:
+                index_list = json.loads(image_index.decode())
+            except (OSError, AttributeError, ValueError, TypeError):
+                return ""
+        (p_os, p_architecture, p_variant) = HostInfo().parse_platform(platform)
+        try:
+            for manifest in index_list["manifests"]:
+                manifest_p = manifest["platform"]
+                if (p_os and
+                    (manifest_p["os"]).lower() != p_os):
+                    continue
+                if (p_architecture and
+                    (manifest_p["architecture"]).lower() != p_architecture):
+                    continue
+                if (p_variant and
+                    (manifest_p["variant"]).lower() != p_variant):
+                    continue
+                return manifest["digest"]
+        except (KeyError, AttributeError, ValueError, TypeError):
+            pass
+        return ""
+
+    def get_v2_image_manifest(self, imagerepo, tag, platform=""):
+        """API v2 Get the image manifest which contains JSON metadata
         that is common to all layers in this image tag
         """
-        url = self.registry_url + "/v2/" + imagerepo + \
-            "/manifests/" + tag
+        reqhdr = [
+            'Accept: application/vnd.docker.distribution.manifest.v2+json',
+            'Accept: application/vnd.docker.distribution.manifest.v1+prettyjws',
+            'Accept: application/json',
+            'Accept: application/vnd.docker.distribution.manifest.list.v2+json',
+            'Accept: application/vnd.oci.image.manifest.v1+json',
+            'Accept: application/vnd.oci.image.index.v1+json',
+        ]
+        url = self.registry_url + "/v2/" + imagerepo + "/manifests/" + tag
         Msg().out("Debug: manifest url", url, l=Msg.DBG)
-        (hdr, buf) = self._get_url(url)
+        (hdr, buf) = self._get_url(url, header=reqhdr)
+
         try:
-            return (hdr.data, json.loads(buf.getvalue().decode()))
-        except (IOError, OSError, AttributeError, ValueError, TypeError):
-            return (hdr.data, [])
+            content_type = hdr.data['content-type']
+            if "docker.distribution.manifest.v1" in content_type:
+                return (hdr.data, json.loads(buf.getvalue().decode()))
+            if "docker.distribution.manifest.v2" in content_type:
+                return (hdr.data, json.loads(buf.getvalue().decode()))
+            if "oci.image.manifest.v1+json" in content_type:
+                return (hdr.data, json.loads(buf.getvalue().decode()))
+            if ("docker.distribution.manifest.list.v2" in content_type
+                or "oci.image.index.v1+json" in content_type):
+                image_index = json.loads(buf.getvalue().decode())
+                digest = self._get_v2_digest_from_image_index(image_index,
+                                                              platform)
+                if not digest:
+                    Msg().err("no image found in manifest for platform (%s)" %
+                              HostInfo().platform_to_str(platform))
+                else:
+                    return self.get_v2_image_manifest(imagerepo,
+                                                      digest, platform)
+        except (OSError, KeyError, AttributeError, ValueError, TypeError):
+            pass
+        return (hdr.data, [])
 
     def get_v2_image_layer(self, imagerepo, layer_id):
         """Get one image layer data file (tarball)"""
@@ -396,17 +454,21 @@ def get_v2_layers_all(self, imagerepo, fslayers):
                 files.append(blob)
         return files
 
-    def get_v2(self, imagerepo, tag):
+    def get_v2(self, imagerepo, tag, platform=""):
         """Pull container with v2 API"""
         files = []
-        (hdr_data, manifest) = self.get_v2_image_manifest(imagerepo, tag)
+        (hdr_data, manifest) = self.get_v2_image_manifest(imagerepo, tag,
+                                                          platform)
         status = self.curl.get_status_code(hdr_data["X-ND-HTTPSTATUS"])
         if status == 401:
             Msg().err("Error: manifest not found or not authorized")
             return []
         if status != 200:
             Msg().err("Error: pulling manifest:")
             return []
+        if not manifest:
+            Msg().err("no manifest for given image and platform")
+            return []
         try:
             if not (self.localrepo.setup_tag(tag) and
                     self.localrepo.set_version("v2")):
@@ -518,7 +580,7 @@ def _parse_imagerepo(self, imagerepo):
                 self.index_url = index_url
         return (imagerepo, remoterepo)
 
-    def get(self, imagerepo, tag):
+    def get(self, imagerepo, tag, platform=""):
         """Pull a docker image from a v2 registry or v1 index"""
         Msg().out("Debug: get imagerepo: %s tag: %s" % (imagerepo, tag), l=Msg.DBG)
         (imagerepo, remoterepo) = self._parse_imagerepo(imagerepo)
@@ -528,7 +590,9 @@ def get(self, imagerepo, tag):
             self.localrepo.setup_imagerepo(imagerepo)
             new_repo = True
         if self.is_v2():
-            files = self.get_v2(remoterepo, tag)  # try v2
+            if not platform:
+                platform = HostInfo().platform()
+            files = self.get_v2(remoterepo, tag, platform)  # try v2
         else:
             files = self.get_v1(remoterepo, tag)  # try v1
         if new_repo and not files:

udocker/helper/hostinfo.py

@@ -78,6 +78,57 @@ def oskernel_isgreater(self, version):
 
         return True
 
+    def parse_platform(self, platform_in):
+        """Convert a platform string or dict into (os, architecture, variant)"""
+        if isinstance(platform_in, dict):
+            p_os = ""
+            p_architecture = ""
+            p_variant = ""
+            for (key, val) in platform_in.items():
+                if key == "os":
+                    p_os = val.lower()
+                elif key == "architecture":
+                    p_architecture = val.lower()
+                elif key == "variant":
+                    p_variant = val.lower()
+            return (p_os, p_architecture, p_variant)
+        if isinstance(platform_in, str):
+            try:
+                (p_os, p_architecture, p_variant) = platform_in.lower().split("/")
+                return (p_os, p_architecture, p_variant)
+            except ValueError:
+                try:
+                    (p_os, p_architecture) = platform_in.split("/")
+                    return (p_os, p_architecture, "")
+                except ValueError:
+                    return (platform_in.strip(), "", "")
+        return ("", "", "")
+
+    def platform_to_str(self, platform_in):
+        """Parse platform and return a string with os/architecture/variant"""
+        parsed_platform = self.parse_platform(platform_in)
+        if parsed_platform[2]:
+            return "%s/%s/%s" % parsed_platform
+        if parsed_platform[1]:
+            return "%s/%s" % parsed_platform[0:2]
+        return parsed_platform[0]
+
+    def platform(self, return_str=True):
+        """get docker platform os/architecture/variant"""
+        translate_architecture = {"i386":"386", "i486":"486", "i586":"586", "i686":"686"}
+        architecture = self.arch()
+        host_platform = self.osversion() + "/" + \
+                translate_architecture.get(architecture, architecture)
+        if return_str:
+            return host_platform.lower()
+        return self.parse_platform(host_platform)
+
+    def is_same_platform(self, platform_in):
+        """Compare some platform against the host platform"""
+        if self.parse_platform(platform_in) == self.platform(return_str=False):
+            return True
+        return False
+
     def cmd_has_option(self, executable, search_option, arg=None):
         """Check if executable has a given cli option"""
         if not executable:

udocker/tools.py

@@ -276,7 +276,7 @@ def _install_logic(self, force=False):
     def install(self, force=False):
         """Get the udocker tools tarball and install the binaries"""
         if self.is_available() and not force:
-            Msg().out("Info: already installed, installation skipped", l=Msg.INF)
+            Msg().out("Debug: already installed, installation skipped", l=Msg.DBG)
             return True
 
         if not self._autoinstall and not force: