Support PKCS#11 for mutual TLS on Unix platforms (#323)

Add bindings for new PKCS#11 functionality.

Related: awslabs/aws-c-io#451

.builder/actions/aws_crt_python.py

@@ -5,6 +5,7 @@
 import os.path
 import pathlib
 import subprocess
+import re
 import sys
 import tempfile
 
@@ -45,6 +46,8 @@ def run(self, env):
         # enable S3 tests
         env.shell.setenv('AWS_TEST_S3', '1')
 
+        self._try_setup_pkcs11()
+
     def _get_secret(self, secret_id):
         """get string from secretsmanager"""
 
@@ -80,6 +83,147 @@ def _setenv_tmpfile_from_secret(self, env_var_name, secret_name, file_name):
         file_path = self._tmpfile_from_secret(secret_name, file_name)
         self.env.shell.setenv(env_var_name, file_path)
 
+    def _try_setup_pkcs11(self):
+        """Attempt to setup for PKCS#11 tests, but bail out if we can't get SoftHSM2 installed"""
+
+        # currently, we only support PKCS#11 on unix
+        if sys.platform == 'darwin' or sys.platform == 'win32':
+            print(f"PKCS#11 on '{sys.platform}' is not currently supported. PKCS#11 tests are disabled")
+            return
+
+        # try to install SoftHSM2, so we can run PKCS#11 tests
+        try:
+            softhsm2_install_action = Builder.InstallPackages(['softhsm'])
+            softhsm2_install_action.run(self.env)
+        except Exception:
+            print("WARNING: SoftHSM2 could not be installed. PKCS#11 tests are disabled")
+            return
+
+        softhsm2_lib = self._find_softhsm2_lib()
+        if softhsm2_lib is None:
+            print("WARNING: libsofthsm2.so not found. PKCS#11 tests are disabled")
+            return
+
+        # put SoftHSM2 config file and token directory under the temp dir.
+        softhsm2_dir = os.path.join(tempfile.gettempdir(), 'softhsm2')
+        conf_path = os.path.join(softhsm2_dir, 'softhsm2.conf')
+        token_dir = os.path.join(softhsm2_dir, 'tokens')
+        if os.path.exists(token_dir):
+            self.env.shell.rm(token_dir)
+        self.env.shell.mkdir(token_dir)
+        self.env.shell.setenv('SOFTHSM2_CONF', conf_path)
+        pathlib.Path(conf_path).write_text(f"directories.tokendir = {token_dir}\n")
+
+        # print SoftHSM2 version
+        self._exec_softhsm2_util('--version')
+
+        # create token
+        token_label = 'my-token'
+        pin = '0000'
+        init_token_result = self._exec_softhsm2_util('--init-token', '--free', '--label', token_label,
+                                                     '--pin', pin, '--so-pin', '0000')
+
+        # figure out which slot the token ended up in.
+        #
+        # older versions of SoftHSM2 (ex: 2.1.0) make us pass --slot number to the --import command.
+        # (newer versions let us pass --label name instead)
+        #
+        # to learn the slot of our new token examine the output of the --show-slots command.
+        # we can't just examine the output of --init-token because some versions
+        # of SoftHSM2 (ex: 2.2.0) reassign tokens to random slots without printing out where they went.
+        token_slot = self._find_sofhsm2_token_slot()
+
+        # add private key to token
+        # key must be in PKCS#8 format
+        # we have this stored in secretsmanager
+        key_path = self._tmpfile_from_secret('unit-test/privatekey-p8', 'privatekey.p8.pem')
+        key_label = 'my-key'
+        self._exec_softhsm2_util('--import', key_path, '--slot', token_slot,
+                                 '--label', key_label, '--id', 'BEEFCAFE', '--pin', pin)
+
+        # set env vars for tests
+        self.env.shell.setenv('AWS_TEST_PKCS11_LIB', softhsm2_lib)
+        self.env.shell.setenv('AWS_TEST_PKCS11_PIN', pin)
+        self.env.shell.setenv('AWS_TEST_PKCS11_TOKEN_LABEL', token_label)
+        self.env.shell.setenv('AWS_TEST_PKCS11_KEY_LABEL', key_label)
+
+    def _find_softhsm2_lib(self):
+        """Return path to SoftHSM2 shared lib, or None if not found"""
+
+        # note: not using `ldconfig --print-cache` to find it because
+        # some installers put it in weird places where ldconfig doesn't look
+        # (like in a subfolder under lib/)
+
+        for lib_dir in ['lib64', 'lib']:  # search lib64 before lib
+            for base_dir in ['/usr/local', '/usr', '/', ]:
+                search_dir = os.path.join(base_dir, lib_dir)
+                for root, dirs, files in os.walk(search_dir):
+                    for file_name in files:
+                        if 'libsofthsm2.so' in file_name:
+                            return os.path.join(root, file_name)
+        return None
+
+    def _exec_softhsm2_util(self, *args, **kwargs):
+        if 'check' not in kwargs:
+            kwargs['check'] = True
+
+        result = self.env.shell.exec('softhsm2-util', *args, **kwargs)
+
+        # older versions of softhsm2-util (2.1.0 is a known offender)
+        # return error code 0 and print the help if invalid args are passed.
+        # This should be an error.
+        #
+        # invalid args can happen because newer versions of softhsm2-util
+        # support more args than older versions, so what works on your
+        # machine might not work on some ancient docker image.
+        if 'Usage: softhsm2-util' in result.output:
+            raise Exception('softhsm2-util failed')
+
+        return result
+
+    def _find_sofhsm2_token_slot(self):
+        """Return slot ID of first initialized token"""
+
+        output = self._exec_softhsm2_util('--show-slots').output
+
+        # --- output looks like ---
+        # Available slots:
+        # Slot 0
+        #    Slot info:
+        #        ...
+        #        Token present:    yes
+        #    Token info:
+        #        ...
+        #        Initialized:      yes
+        current_slot = None
+        current_info_block = None
+        for line in output.splitlines():
+            # check for start of "Slot <ID>" block
+            m = re.match(r"Slot ([0-9]+)", line)
+            if m:
+                current_slot = m.group(1)
+                current_info_block = None
+                continue
+
+            if current_slot is None:
+                continue
+
+            # check for start of next indented block, like "Token info"
+            m = re.match(r"    ([^ ].*)", line)
+            if m:
+                current_info_block = m.group(1)
+                continue
+
+            if current_info_block is None:
+                continue
+
+            # if we're in token block, check for "Initialized: yes"
+            if "Token info" in current_info_block:
+                if re.match(r" *Initialized: *yes", line):
+                    return current_slot
+
+        raise Exception('No initialized tokens found')
+
 
 class AWSCrtPython(Builder.Action):
 

.github/workflows/ci.yml

@@ -7,7 +7,7 @@ on:
       - '!main'
 
 env:
-  BUILDER_VERSION: v0.9.5
+  BUILDER_VERSION: v0.9.10
   BUILDER_SOURCE: releases
   BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net
   PACKAGE_NAME: aws-crt-python

.lgtm.yml

@@ -3,3 +3,17 @@ extraction:
     index:
       # not sure why cpp builds are using python 2, but this should stop it
       build_command: "python3 setup.py build"
+
+# add tags for folders and files that we don't want alerts about
+# LGTM already has defaults tagging folders like "test/", so we're just adding non-obvious things here
+path_classifiers:
+  library:
+    # ignore alerts in libraries that the Common Runtime team doesn't own
+    - crt/s2n
+    - crt/aws-lc
+  test:
+    - codebuild
+    - continuous-delivery
+    - elasticurl.py
+    - mqtt_test.py
+    - s3_benchmark.py

MANIFEST.in

@@ -14,7 +14,6 @@ prune crt/aws-c-auth/tests/fuzz/corpus
 prune crt/aws-c-s3/benchmarks
 prune crt/s2n/tests
 # s2n's cmake relies on a some files under test/ for compile time feature tests
-include crt/s2n/tests/unit/s2n_pq_asm_noop_test.c
 graft crt/s2n/tests/features
 exclude crt/aws-lc/**/*test*.go
 exclude crt/aws-lc/**/*test*.json

README.md

@@ -60,6 +60,13 @@ MQTT
 * AWS_TEST_TLS_KEY_PATH - file path to the private key used to initialize the TLS context of the MQTT connection
 * AWS_TEST_TLS_ROOT_CERT_PATH - file path to the root CA used to initialize the TLS context of the MQTT connection
 
+PKCS11
+* AWS_TEST_PKCS11_LIB - path to PKCS#11 library
+* AWS_TEST_PKCS11_PIN - user PIN for logging into PKCS#11 token
+* AWS_TEST_PKCS11_TOKEN_LABEL - label of PKCS#11 token
+* AWS_TEST_PKCS11_KEY_LABEL - label of private key on PKCS#11 token,
+  which must correspond to the cert at AWS_TEST_TLS_CERT_PATH.
+
 PROXY
 * AWS_TEST_HTTP_PROXY_HOST - host address of the proxy to use for tests that make open connections to the proxy
 * AWS_TEST_HTTP_PROXY_PORT - port to use for tests that make open connections to the proxy

awscrt/io.py

@@ -219,9 +219,23 @@ class TlsContextOptions:
             via :meth:`TlsConnectionOptions.set_alpn_list()`.
     """
     __slots__ = (
-        'min_tls_ver', 'ca_dirpath', 'ca_buffer', 'alpn_list',
-        'certificate_buffer', 'private_key_buffer',
-        'pkcs12_filepath', 'pkcs12_password', 'verify_peer')
+        'min_tls_ver',
+        'ca_dirpath',
+        'ca_buffer',
+        'alpn_list',
+        'certificate_buffer',
+        'private_key_buffer',
+        'pkcs12_filepath',
+        'pkcs12_password',
+        'verify_peer',
+        '_pkcs11_lib',
+        '_pkcs11_user_pin',
+        '_pkcs11_slot_id',
+        '_pkcs11_token_label',
+        '_pkcs11_private_key_label',
+        '_pkcs11_cert_file_path',
+        '_pkcs11_cert_file_contents',
+    )
 
     def __init__(self):
 
@@ -276,7 +290,65 @@ def create_client_with_mtls(cert_buffer, key_buffer):
         opt.certificate_buffer = cert_buffer
         opt.private_key_buffer = key_buffer
 
-        opt.verify_peer = True
+        return opt
+
+    @staticmethod
+    def create_client_with_mtls_pkcs11(*,
+                                       pkcs11_lib: 'Pkcs11Lib',
+                                       user_pin: str,
+                                       slot_id: int = None,
+                                       token_label: str = None,
+                                       private_key_label: str = None,
+                                       cert_file_path: str = None,
+                                       cert_file_contents=None):
+        """
+        Create options configured for use with mutual TLS in client mode,
+        using a PKCS#11 library for private key operations.
+
+        NOTE: This configuration only works on Unix devices.
+
+        Keyword Args:
+            pkcs11_lib (Pkcs11Lib): Use this PKCS#11 library
+
+            user_pin (Optional[str]): User PIN, for logging into the PKCS#11 token.
+                Pass `None` to log into a token with a "protected authentication path".
+
+            slot_id (Optional[int]): ID of slot containing PKCS#11 token.
+                If not specified, the token will be chosen based on other criteria (such as token label).
+
+            token_label (Optional[str]): Label of the PKCS#11 token to use.
+                If not specified, the token will be chosen based on other criteria (such as slot ID).
+
+            private_key_label (Optional[str]): Label of private key object on PKCS#11 token.
+                If not specified, the key will be chosen based on other criteria
+                (such as being the only available private key on the token).
+
+            cert_file_path (Optional[str]): Use this X.509 certificate (file on disk).
+                The certificate must be PEM-formatted. The certificate may be
+                specified by other means instead (ex: `cert_file_contents`)
+
+            cert_file_contents (Optional[bytes-like object]):
+                Use this X.509 certificate (contents in memory).
+                The certificate must be PEM-formatted. The certificate may be
+                specified by other means instead (ex: `cert_file_path`)
+        """
+
+        assert isinstance(pkcs11_lib, Pkcs11Lib)
+        assert isinstance(user_pin, str) or user_pin is None
+        assert isinstance(slot_id, int) or slot_id is None
+        assert isinstance(token_label, str) or token_label is None
+        assert isinstance(private_key_label, str) or private_key_label is None
+        assert isinstance(cert_file_path, str) or cert_file_path is None
+        # note: not validating cert_file_contents, because "bytes-like object" isn't a strict type
+
+        opt = TlsContextOptions()
+        opt._pkcs11_lib = pkcs11_lib
+        opt._pkcs11_user_pin = user_pin
+        opt._pkcs11_slot_id = slot_id
+        opt._pkcs11_token_label = token_label
+        opt._pkcs11_private_key_label = private_key_label
+        opt._pkcs11_cert_file_path = cert_file_path
+        opt._pkcs11_cert_file_contents = cert_file_contents
         return opt
 
     @staticmethod
@@ -301,7 +373,6 @@ def create_client_with_mtls_pkcs12(pkcs12_filepath, pkcs12_password):
         opt = TlsContextOptions()
         opt.pkcs12_filepath = pkcs12_filepath
         opt.pkcs12_password = pkcs12_password
-        opt.verify_peer = True
         return opt
 
     @staticmethod
@@ -430,7 +501,14 @@ def __init__(self, options):
             options.private_key_buffer,
             options.pkcs12_filepath,
             options.pkcs12_password,
-            options.verify_peer
+            options.verify_peer,
+            options._pkcs11_lib,
+            options._pkcs11_user_pin,
+            options._pkcs11_slot_id,
+            options._pkcs11_token_label,
+            options._pkcs11_private_key_label,
+            options._pkcs11_cert_file_path,
+            options._pkcs11_cert_file_contents,
         )
 
     def new_connection_options(self):
@@ -565,3 +643,59 @@ def wrap(cls, stream, allow_none=False):
         if isinstance(stream, InputStream):
             return stream
         return cls(stream)
+
+
+class Pkcs11Lib(NativeResource):
+    """
+    Handle to a loaded PKCS#11 library.
+
+    For most use cases, a single instance of :class:`Pkcs11Lib` should be used for the
+    lifetime of your application.
+
+    Keyword Args:
+        file (str): Path to PKCS#11 library.
+        behavior (Optional[InitializeFinalizeBehavior]):
+            Specifies how `C_Initialize()` and `C_Finalize()` will be called
+            on the PKCS#11 library (default is :attr:`InitializeFinalizeBehavior.DEFAULT`)
+    """
+
+    class InitializeFinalizeBehavior(IntEnum):
+        """
+        An enumeration.
+
+        Controls how `C_Initialize()` and `C_Finalize()` are called on the PKCS#11 library.
+        """
+
+        DEFAULT = 0
+        """
+        Relaxed behavior that accommodates most use cases.
+
+        `C_Initialize()` is called on creation, and "already-initialized"
+        errors are ignored. `C_Finalize()` is never called, just in case
+        another part of your application is still using the PKCS#11 library.
+        """
+
+        OMIT = 1
+        """
+        Skip calling `C_Initialize()` and `C_Finalize()`.
+
+        Use this if your application has already initialized the PKCS#11 library, and
+        you do not want `C_Initialize()` called again.
+        """
+
+        STRICT = 2
+        """
+        `C_Initialize()` is called on creation and `C_Finalize()` is
+        called on cleanup.
+
+        If `C_Initialize()` reports that's it's already initialized, this is
+        treated as an error. Use this if you need perfect cleanup (ex: running
+        valgrind with --leak-check).
+        """
+
+    def __init__(self, *, file: str, behavior: InitializeFinalizeBehavior = None):
+        super().__init__()
+        if behavior is None:
+            behavior = Pkcs11Lib.InitializeFinalizeBehavior.DEFAULT
+
+        self._binding = _awscrt.pkcs11_lib_new(file, behavior)