From 90d0d501d1fb581210f8e7bfb5d6c613a515ef4b Mon Sep 17 00:00:00 2001 From: Michael Graeb Date: Thu, 6 Jan 2022 17:11:21 -0800 Subject: [PATCH] Support PKCS#11 for mutual TLS on Unix platforms (#323) Add bindings for new PKCS#11 functionality. Related: awslabs/aws-c-io#451 --- .builder/actions/aws_crt_python.py | 144 ++++++++++++++++++++++++++++ .github/workflows/ci.yml | 2 +- .lgtm.yml | 14 +++ MANIFEST.in | 1 - README.md | 7 ++ awscrt/io.py | 146 +++++++++++++++++++++++++++-- builder.json | 6 ++ crt/aws-c-common | 2 +- crt/aws-c-io | 2 +- crt/aws-c-mqtt | 2 +- crt/aws-c-s3 | 2 +- crt/s2n | 2 +- setup.py | 2 +- source/io.c | 93 ++++++++++++++---- source/io.h | 6 ++ source/module.c | 1 + source/pkcs11_lib.c | 45 +++++++++ test/test_io.py | 55 ++++++++++- test/test_mqtt.py | 84 +++++++++-------- 19 files changed, 541 insertions(+), 75 deletions(-) create mode 100644 source/pkcs11_lib.c diff --git a/.builder/actions/aws_crt_python.py b/.builder/actions/aws_crt_python.py index 54b639129..7a27d18cb 100644 --- a/.builder/actions/aws_crt_python.py +++ b/.builder/actions/aws_crt_python.py @@ -5,6 +5,7 @@ import os.path import pathlib import subprocess +import re import sys import tempfile @@ -45,6 +46,8 @@ def run(self, env): # enable S3 tests env.shell.setenv('AWS_TEST_S3', '1') + self._try_setup_pkcs11() + def _get_secret(self, secret_id): """get string from secretsmanager""" @@ -80,6 +83,147 @@ def _setenv_tmpfile_from_secret(self, env_var_name, secret_name, file_name): file_path = self._tmpfile_from_secret(secret_name, file_name) self.env.shell.setenv(env_var_name, file_path) + def _try_setup_pkcs11(self): + """Attempt to setup for PKCS#11 tests, but bail out if we can't get SoftHSM2 installed""" + + # currently, we only support PKCS#11 on unix + if sys.platform == 'darwin' or sys.platform == 'win32': + print(f"PKCS#11 on '{sys.platform}' is not currently supported. PKCS#11 tests are disabled") + return + + # try to install SoftHSM2, so we can run PKCS#11 tests + try: + softhsm2_install_action = Builder.InstallPackages(['softhsm']) + softhsm2_install_action.run(self.env) + except Exception: + print("WARNING: SoftHSM2 could not be installed. PKCS#11 tests are disabled") + return + + softhsm2_lib = self._find_softhsm2_lib() + if softhsm2_lib is None: + print("WARNING: libsofthsm2.so not found. PKCS#11 tests are disabled") + return + + # put SoftHSM2 config file and token directory under the temp dir. + softhsm2_dir = os.path.join(tempfile.gettempdir(), 'softhsm2') + conf_path = os.path.join(softhsm2_dir, 'softhsm2.conf') + token_dir = os.path.join(softhsm2_dir, 'tokens') + if os.path.exists(token_dir): + self.env.shell.rm(token_dir) + self.env.shell.mkdir(token_dir) + self.env.shell.setenv('SOFTHSM2_CONF', conf_path) + pathlib.Path(conf_path).write_text(f"directories.tokendir = {token_dir}\n") + + # print SoftHSM2 version + self._exec_softhsm2_util('--version') + + # create token + token_label = 'my-token' + pin = '0000' + init_token_result = self._exec_softhsm2_util('--init-token', '--free', '--label', token_label, + '--pin', pin, '--so-pin', '0000') + + # figure out which slot the token ended up in. + # + # older versions of SoftHSM2 (ex: 2.1.0) make us pass --slot number to the --import command. + # (newer versions let us pass --label name instead) + # + # to learn the slot of our new token examine the output of the --show-slots command. + # we can't just examine the output of --init-token because some versions + # of SoftHSM2 (ex: 2.2.0) reassign tokens to random slots without printing out where they went. + token_slot = self._find_sofhsm2_token_slot() + + # add private key to token + # key must be in PKCS#8 format + # we have this stored in secretsmanager + key_path = self._tmpfile_from_secret('unit-test/privatekey-p8', 'privatekey.p8.pem') + key_label = 'my-key' + self._exec_softhsm2_util('--import', key_path, '--slot', token_slot, + '--label', key_label, '--id', 'BEEFCAFE', '--pin', pin) + + # set env vars for tests + self.env.shell.setenv('AWS_TEST_PKCS11_LIB', softhsm2_lib) + self.env.shell.setenv('AWS_TEST_PKCS11_PIN', pin) + self.env.shell.setenv('AWS_TEST_PKCS11_TOKEN_LABEL', token_label) + self.env.shell.setenv('AWS_TEST_PKCS11_KEY_LABEL', key_label) + + def _find_softhsm2_lib(self): + """Return path to SoftHSM2 shared lib, or None if not found""" + + # note: not using `ldconfig --print-cache` to find it because + # some installers put it in weird places where ldconfig doesn't look + # (like in a subfolder under lib/) + + for lib_dir in ['lib64', 'lib']: # search lib64 before lib + for base_dir in ['/usr/local', '/usr', '/', ]: + search_dir = os.path.join(base_dir, lib_dir) + for root, dirs, files in os.walk(search_dir): + for file_name in files: + if 'libsofthsm2.so' in file_name: + return os.path.join(root, file_name) + return None + + def _exec_softhsm2_util(self, *args, **kwargs): + if 'check' not in kwargs: + kwargs['check'] = True + + result = self.env.shell.exec('softhsm2-util', *args, **kwargs) + + # older versions of softhsm2-util (2.1.0 is a known offender) + # return error code 0 and print the help if invalid args are passed. + # This should be an error. + # + # invalid args can happen because newer versions of softhsm2-util + # support more args than older versions, so what works on your + # machine might not work on some ancient docker image. + if 'Usage: softhsm2-util' in result.output: + raise Exception('softhsm2-util failed') + + return result + + def _find_sofhsm2_token_slot(self): + """Return slot ID of first initialized token""" + + output = self._exec_softhsm2_util('--show-slots').output + + # --- output looks like --- + # Available slots: + # Slot 0 + # Slot info: + # ... + # Token present: yes + # Token info: + # ... + # Initialized: yes + current_slot = None + current_info_block = None + for line in output.splitlines(): + # check for start of "Slot " block + m = re.match(r"Slot ([0-9]+)", line) + if m: + current_slot = m.group(1) + current_info_block = None + continue + + if current_slot is None: + continue + + # check for start of next indented block, like "Token info" + m = re.match(r" ([^ ].*)", line) + if m: + current_info_block = m.group(1) + continue + + if current_info_block is None: + continue + + # if we're in token block, check for "Initialized: yes" + if "Token info" in current_info_block: + if re.match(r" *Initialized: *yes", line): + return current_slot + + raise Exception('No initialized tokens found') + class AWSCrtPython(Builder.Action): diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1f499cad1..2dd1791f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,7 @@ on: - '!main' env: - BUILDER_VERSION: v0.9.5 + BUILDER_VERSION: v0.9.10 BUILDER_SOURCE: releases BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net PACKAGE_NAME: aws-crt-python diff --git a/.lgtm.yml b/.lgtm.yml index d267832eb..ad56377d6 100644 --- a/.lgtm.yml +++ b/.lgtm.yml @@ -3,3 +3,17 @@ extraction: index: # not sure why cpp builds are using python 2, but this should stop it build_command: "python3 setup.py build" + +# add tags for folders and files that we don't want alerts about +# LGTM already has defaults tagging folders like "test/", so we're just adding non-obvious things here +path_classifiers: + library: + # ignore alerts in libraries that the Common Runtime team doesn't own + - crt/s2n + - crt/aws-lc + test: + - codebuild + - continuous-delivery + - elasticurl.py + - mqtt_test.py + - s3_benchmark.py diff --git a/MANIFEST.in b/MANIFEST.in index 145f57c29..66e5037d3 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -14,7 +14,6 @@ prune crt/aws-c-auth/tests/fuzz/corpus prune crt/aws-c-s3/benchmarks prune crt/s2n/tests # s2n's cmake relies on a some files under test/ for compile time feature tests -include crt/s2n/tests/unit/s2n_pq_asm_noop_test.c graft crt/s2n/tests/features exclude crt/aws-lc/**/*test*.go exclude crt/aws-lc/**/*test*.json diff --git a/README.md b/README.md index 8474c1e93..8974274b3 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,13 @@ MQTT * AWS_TEST_TLS_KEY_PATH - file path to the private key used to initialize the TLS context of the MQTT connection * AWS_TEST_TLS_ROOT_CERT_PATH - file path to the root CA used to initialize the TLS context of the MQTT connection +PKCS11 +* AWS_TEST_PKCS11_LIB - path to PKCS#11 library +* AWS_TEST_PKCS11_PIN - user PIN for logging into PKCS#11 token +* AWS_TEST_PKCS11_TOKEN_LABEL - label of PKCS#11 token +* AWS_TEST_PKCS11_KEY_LABEL - label of private key on PKCS#11 token, + which must correspond to the cert at AWS_TEST_TLS_CERT_PATH. + PROXY * AWS_TEST_HTTP_PROXY_HOST - host address of the proxy to use for tests that make open connections to the proxy * AWS_TEST_HTTP_PROXY_PORT - port to use for tests that make open connections to the proxy diff --git a/awscrt/io.py b/awscrt/io.py index 0cfb5b794..d8d052492 100644 --- a/awscrt/io.py +++ b/awscrt/io.py @@ -219,9 +219,23 @@ class TlsContextOptions: via :meth:`TlsConnectionOptions.set_alpn_list()`. """ __slots__ = ( - 'min_tls_ver', 'ca_dirpath', 'ca_buffer', 'alpn_list', - 'certificate_buffer', 'private_key_buffer', - 'pkcs12_filepath', 'pkcs12_password', 'verify_peer') + 'min_tls_ver', + 'ca_dirpath', + 'ca_buffer', + 'alpn_list', + 'certificate_buffer', + 'private_key_buffer', + 'pkcs12_filepath', + 'pkcs12_password', + 'verify_peer', + '_pkcs11_lib', + '_pkcs11_user_pin', + '_pkcs11_slot_id', + '_pkcs11_token_label', + '_pkcs11_private_key_label', + '_pkcs11_cert_file_path', + '_pkcs11_cert_file_contents', + ) def __init__(self): @@ -276,7 +290,65 @@ def create_client_with_mtls(cert_buffer, key_buffer): opt.certificate_buffer = cert_buffer opt.private_key_buffer = key_buffer - opt.verify_peer = True + return opt + + @staticmethod + def create_client_with_mtls_pkcs11(*, + pkcs11_lib: 'Pkcs11Lib', + user_pin: str, + slot_id: int = None, + token_label: str = None, + private_key_label: str = None, + cert_file_path: str = None, + cert_file_contents=None): + """ + Create options configured for use with mutual TLS in client mode, + using a PKCS#11 library for private key operations. + + NOTE: This configuration only works on Unix devices. + + Keyword Args: + pkcs11_lib (Pkcs11Lib): Use this PKCS#11 library + + user_pin (Optional[str]): User PIN, for logging into the PKCS#11 token. + Pass `None` to log into a token with a "protected authentication path". + + slot_id (Optional[int]): ID of slot containing PKCS#11 token. + If not specified, the token will be chosen based on other criteria (such as token label). + + token_label (Optional[str]): Label of the PKCS#11 token to use. + If not specified, the token will be chosen based on other criteria (such as slot ID). + + private_key_label (Optional[str]): Label of private key object on PKCS#11 token. + If not specified, the key will be chosen based on other criteria + (such as being the only available private key on the token). + + cert_file_path (Optional[str]): Use this X.509 certificate (file on disk). + The certificate must be PEM-formatted. The certificate may be + specified by other means instead (ex: `cert_file_contents`) + + cert_file_contents (Optional[bytes-like object]): + Use this X.509 certificate (contents in memory). + The certificate must be PEM-formatted. The certificate may be + specified by other means instead (ex: `cert_file_path`) + """ + + assert isinstance(pkcs11_lib, Pkcs11Lib) + assert isinstance(user_pin, str) or user_pin is None + assert isinstance(slot_id, int) or slot_id is None + assert isinstance(token_label, str) or token_label is None + assert isinstance(private_key_label, str) or private_key_label is None + assert isinstance(cert_file_path, str) or cert_file_path is None + # note: not validating cert_file_contents, because "bytes-like object" isn't a strict type + + opt = TlsContextOptions() + opt._pkcs11_lib = pkcs11_lib + opt._pkcs11_user_pin = user_pin + opt._pkcs11_slot_id = slot_id + opt._pkcs11_token_label = token_label + opt._pkcs11_private_key_label = private_key_label + opt._pkcs11_cert_file_path = cert_file_path + opt._pkcs11_cert_file_contents = cert_file_contents return opt @staticmethod @@ -301,7 +373,6 @@ def create_client_with_mtls_pkcs12(pkcs12_filepath, pkcs12_password): opt = TlsContextOptions() opt.pkcs12_filepath = pkcs12_filepath opt.pkcs12_password = pkcs12_password - opt.verify_peer = True return opt @staticmethod @@ -430,7 +501,14 @@ def __init__(self, options): options.private_key_buffer, options.pkcs12_filepath, options.pkcs12_password, - options.verify_peer + options.verify_peer, + options._pkcs11_lib, + options._pkcs11_user_pin, + options._pkcs11_slot_id, + options._pkcs11_token_label, + options._pkcs11_private_key_label, + options._pkcs11_cert_file_path, + options._pkcs11_cert_file_contents, ) def new_connection_options(self): @@ -565,3 +643,59 @@ def wrap(cls, stream, allow_none=False): if isinstance(stream, InputStream): return stream return cls(stream) + + +class Pkcs11Lib(NativeResource): + """ + Handle to a loaded PKCS#11 library. + + For most use cases, a single instance of :class:`Pkcs11Lib` should be used for the + lifetime of your application. + + Keyword Args: + file (str): Path to PKCS#11 library. + behavior (Optional[InitializeFinalizeBehavior]): + Specifies how `C_Initialize()` and `C_Finalize()` will be called + on the PKCS#11 library (default is :attr:`InitializeFinalizeBehavior.DEFAULT`) + """ + + class InitializeFinalizeBehavior(IntEnum): + """ + An enumeration. + + Controls how `C_Initialize()` and `C_Finalize()` are called on the PKCS#11 library. + """ + + DEFAULT = 0 + """ + Relaxed behavior that accommodates most use cases. + + `C_Initialize()` is called on creation, and "already-initialized" + errors are ignored. `C_Finalize()` is never called, just in case + another part of your application is still using the PKCS#11 library. + """ + + OMIT = 1 + """ + Skip calling `C_Initialize()` and `C_Finalize()`. + + Use this if your application has already initialized the PKCS#11 library, and + you do not want `C_Initialize()` called again. + """ + + STRICT = 2 + """ + `C_Initialize()` is called on creation and `C_Finalize()` is + called on cleanup. + + If `C_Initialize()` reports that's it's already initialized, this is + treated as an error. Use this if you need perfect cleanup (ex: running + valgrind with --leak-check). + """ + + def __init__(self, *, file: str, behavior: InitializeFinalizeBehavior = None): + super().__init__() + if behavior is None: + behavior = Pkcs11Lib.InitializeFinalizeBehavior.DEFAULT + + self._binding = _awscrt.pkcs11_lib_new(file, behavior) diff --git a/builder.json b/builder.json index fe75cca35..1668a5ad9 100644 --- a/builder.json +++ b/builder.json @@ -3,6 +3,12 @@ "!cmake_args": [ "-DS2N_NO_PQ_ASM=ON" ], + "hosts": { + "manylinux": { + "_comment": "Use existing compiler on manylinux. These are the images we use for release. We want to be sure things work with the defaults.", + "needs_compiler": false + } + }, "targets": { "android": { "enabled": false, diff --git a/crt/aws-c-common b/crt/aws-c-common index 4639b27bf..d9031cf4f 160000 --- a/crt/aws-c-common +++ b/crt/aws-c-common @@ -1 +1 @@ -Subproject commit 4639b27bf04559b0414e029bb6c22856f356da5a +Subproject commit d9031cf4f02fe4d8c24a53bb6c8df67ce8be9f0c diff --git a/crt/aws-c-io b/crt/aws-c-io index 797b0b7ba..866765e8d 160000 --- a/crt/aws-c-io +++ b/crt/aws-c-io @@ -1 +1 @@ -Subproject commit 797b0b7ba320d6bbe05afe6611236b604db31796 +Subproject commit 866765e8dea76f67fd867c5ff6c05e7ba04c674d diff --git a/crt/aws-c-mqtt b/crt/aws-c-mqtt index 60f9a179c..6168e32bf 160000 --- a/crt/aws-c-mqtt +++ b/crt/aws-c-mqtt @@ -1 +1 @@ -Subproject commit 60f9a179c1a599cbf217edcccc9dd008c8e9b787 +Subproject commit 6168e32bf9f745dec40df633b78baa03420b7f83 diff --git a/crt/aws-c-s3 b/crt/aws-c-s3 index 8655417a9..76f49fbfa 160000 --- a/crt/aws-c-s3 +++ b/crt/aws-c-s3 @@ -1 +1 @@ -Subproject commit 8655417a9c6940e6373cbcb4fb93b823317bccb9 +Subproject commit 76f49fbfa9deabd464842900744fad10a44314b4 diff --git a/crt/s2n b/crt/s2n index ab9a3be5a..404111ca6 160000 --- a/crt/s2n +++ b/crt/s2n @@ -1 +1 @@ -Subproject commit ab9a3be5a8abcbc5bd7bdba662497a717737c838 +Subproject commit 404111ca673c05beafec79a0836d0e5084dd421e diff --git a/setup.py b/setup.py index c44d807a2..ca1b455c3 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ def determine_generator_args(): vs_version = 14 vs_year = 2015 assert(vs_version and vs_year) - except BaseException: + except Exception: raise RuntimeError('No supported version of MSVC compiler could be found!') print('Using Visual Studio', vs_version, vs_year) diff --git a/source/io.c b/source/io.c index 428afe752..15fa0a97b 100644 --- a/source/io.c +++ b/source/io.c @@ -412,9 +412,22 @@ PyObject *aws_py_client_tls_ctx_new(PyObject *self, PyObject *args) { const char *pkcs12_filepath; const char *pkcs12_password; uint8_t verify_peer; + PyObject *py_pkcs11_lib; + const char *pkcs11_user_pin; + Py_ssize_t pkcs11_user_pin_len; + PyObject *py_pkcs11_slot_id; + const char *pkcs11_token_label; + Py_ssize_t pkcs11_token_label_len; + const char *pkcs11_priv_key_label; + Py_ssize_t pkcs11_priv_key_label_len; + const char *pkcs11_cert_file_path; + Py_ssize_t pkcs11_cert_file_path_len; + const char *pkcs11_cert_file_contents; + Py_ssize_t pkcs11_cert_file_contents_len; + if (!PyArg_ParseTuple( args, - "bzz#zz#z#zzb", + "bzz#zz#z#zzbOz#Oz#z#z#z#", &min_tls_version, &ca_dirpath, &ca_buffer, @@ -426,20 +439,77 @@ PyObject *aws_py_client_tls_ctx_new(PyObject *self, PyObject *args) { &private_key_buffer_len, &pkcs12_filepath, &pkcs12_password, - &verify_peer)) { + &verify_peer, + &py_pkcs11_lib, + &pkcs11_user_pin, + &pkcs11_user_pin_len, + &py_pkcs11_slot_id, + &pkcs11_token_label, + &pkcs11_token_label_len, + &pkcs11_priv_key_label, + &pkcs11_priv_key_label_len, + &pkcs11_cert_file_path, + &pkcs11_cert_file_path_len, + &pkcs11_cert_file_contents, + &pkcs11_cert_file_contents_len)) { return NULL; } struct aws_tls_ctx_options ctx_options; AWS_ZERO_STRUCT(ctx_options); - if (certificate_buffer_len > 0 && private_key_buffer_len > 0) { + if (certificate_buffer != NULL) { + /* mTLS with certificate and private key*/ struct aws_byte_cursor cert = aws_byte_cursor_from_array(certificate_buffer, certificate_buffer_len); struct aws_byte_cursor key = aws_byte_cursor_from_array(private_key_buffer, private_key_buffer_len); if (aws_tls_ctx_options_init_client_mtls(&ctx_options, allocator, &cert, &key)) { PyErr_SetAwsLastError(); return NULL; } + } else if (py_pkcs11_lib != Py_None) { + /* mTLS with PKCS#11 */ + struct aws_pkcs11_lib *pkcs11_lib = aws_py_get_pkcs11_lib(py_pkcs11_lib); + if (pkcs11_lib == NULL) { + return NULL; + } + + bool has_slot_id = false; + uint64_t slot_id_value = 0; + if (py_pkcs11_slot_id != Py_None) { + has_slot_id = true; + slot_id_value = PyLong_AsUnsignedLongLong(py_pkcs11_slot_id); + if ((slot_id_value == (uint64_t)-1) && PyErr_Occurred()) { + PyErr_SetString(PyExc_ValueError, "PKCS#11 slot_id is not a valid int"); + return NULL; + } + } + + struct aws_tls_ctx_pkcs11_options pkcs11_options = { + .pkcs11_lib = pkcs11_lib, + .user_pin = aws_byte_cursor_from_array(pkcs11_user_pin, pkcs11_user_pin_len), + .slot_id = has_slot_id ? &slot_id_value : NULL, + .token_label = aws_byte_cursor_from_array(pkcs11_token_label, pkcs11_token_label_len), + .private_key_object_label = aws_byte_cursor_from_array(pkcs11_priv_key_label, pkcs11_priv_key_label_len), + .cert_file_path = aws_byte_cursor_from_array(pkcs11_cert_file_path, pkcs11_cert_file_path_len), + .cert_file_contents = aws_byte_cursor_from_array(pkcs11_cert_file_contents, pkcs11_cert_file_contents_len), + }; + + if (aws_tls_ctx_options_init_client_mtls_with_pkcs11(&ctx_options, allocator, &pkcs11_options)) { + return PyErr_AwsLastError(); + } + } else if (pkcs12_filepath != NULL) { + /* mTLS with PKCS#12 */ +#ifdef __APPLE__ + struct aws_byte_cursor password = aws_byte_cursor_from_c_str(pkcs12_password); + if (aws_tls_ctx_options_init_client_mtls_pkcs12_from_path( + &ctx_options, allocator, pkcs12_filepath, &password)) { + return PyErr_AwsLastError(); + } +#else + PyErr_SetString(PyExc_NotImplementedError, "PKCS#12 is currently only supported on Apple devices"); + return NULL; +#endif } else { + /* no mTLS */ aws_tls_ctx_options_init_default_client(&ctx_options, allocator); } @@ -447,13 +517,14 @@ PyObject *aws_py_client_tls_ctx_new(PyObject *self, PyObject *args) { ctx_options.minimum_tls_version = min_tls_version; - if (ca_dirpath) { + if (ca_dirpath != NULL) { if (aws_tls_ctx_options_override_default_trust_store_from_path(&ctx_options, ca_dirpath, NULL)) { PyErr_SetAwsLastError(); goto ctx_options_failure; } } - if (ca_buffer_len > 0) { + + if (ca_buffer != NULL) { struct aws_byte_cursor ca = aws_byte_cursor_from_array(ca_buffer, ca_buffer_len); if (aws_tls_ctx_options_override_default_trust_store(&ctx_options, &ca)) { @@ -462,23 +533,13 @@ PyObject *aws_py_client_tls_ctx_new(PyObject *self, PyObject *args) { } } - if (alpn_list) { + if (alpn_list != NULL) { if (aws_tls_ctx_options_set_alpn_list(&ctx_options, alpn_list)) { PyErr_SetAwsLastError(); goto ctx_options_failure; } } -#ifdef __APPLE__ - if (pkcs12_filepath && pkcs12_password) { - struct aws_byte_cursor password = aws_byte_cursor_from_c_str(pkcs12_password); - if (aws_tls_ctx_options_init_client_mtls_pkcs12_from_path( - &ctx_options, allocator, pkcs12_filepath, &password)) { - PyErr_SetAwsLastError(); - goto ctx_options_failure; - } - } -#endif ctx_options.verify_peer = (bool)verify_peer; struct aws_tls_ctx *tls_ctx = aws_tls_client_ctx_new(allocator, &ctx_options); if (!tls_ctx) { diff --git a/source/io.h b/source/io.h index d6c81e40e..fbad8e2e1 100644 --- a/source/io.h +++ b/source/io.h @@ -59,6 +59,11 @@ PyObject *aws_py_tls_connection_options_set_server_name(PyObject *self, PyObject */ PyObject *aws_py_input_stream_new(PyObject *self, PyObject *args); +/** + * Create a new aws_pkcs11_lib to be managed by a Python capsule. + */ +PyObject *aws_py_pkcs11_lib_new(PyObject *self, PyObject *args); + /* Given a python object, return a pointer to its underlying native type. * If NULL is returned, a python error has been set */ @@ -68,5 +73,6 @@ struct aws_client_bootstrap *aws_py_get_client_bootstrap(PyObject *client_bootst struct aws_tls_ctx *aws_py_get_tls_ctx(PyObject *tls_ctx); struct aws_tls_connection_options *aws_py_get_tls_connection_options(PyObject *tls_connection_options); struct aws_input_stream *aws_py_get_input_stream(PyObject *input_stream); +struct aws_pkcs11_lib *aws_py_get_pkcs11_lib(PyObject *pkcs11_lib); #endif /* AWS_CRT_PYTHON_IO_H */ diff --git a/source/module.c b/source/module.c index 5ab4f9ffd..d7ad5ab92 100644 --- a/source/module.c +++ b/source/module.c @@ -551,6 +551,7 @@ static PyMethodDef s_module_methods[] = { AWS_PY_METHOD_DEF(tls_connection_options_set_server_name, METH_VARARGS), AWS_PY_METHOD_DEF(init_logging, METH_VARARGS), AWS_PY_METHOD_DEF(input_stream_new, METH_VARARGS), + AWS_PY_METHOD_DEF(pkcs11_lib_new, METH_VARARGS), /* MQTT Client */ AWS_PY_METHOD_DEF(mqtt_client_new, METH_VARARGS), diff --git a/source/pkcs11_lib.c b/source/pkcs11_lib.c new file mode 100644 index 000000000..793e39335 --- /dev/null +++ b/source/pkcs11_lib.c @@ -0,0 +1,45 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include "io.h" + +#include + +static const char *s_capsule_name = "aws_pkcs11_lib"; + +static void s_pkcs11_lib_capsule_destructor(PyObject *capsule) { + struct aws_pkcs11_lib *pkcs11_lib = PyCapsule_GetPointer(capsule, s_capsule_name); + aws_pkcs11_lib_release(pkcs11_lib); +} + +struct aws_pkcs11_lib *aws_py_get_pkcs11_lib(PyObject *pkcs11_lib) { + return aws_py_get_binding(pkcs11_lib, s_capsule_name, "Pkcs11Lib"); +} + +PyObject *aws_py_pkcs11_lib_new(PyObject *self, PyObject *args) { + (void)self; + + struct aws_byte_cursor filename; + int behavior; + if (!PyArg_ParseTuple(args, "s#i", &filename.ptr, &filename.len, &behavior)) { + return NULL; + } + + struct aws_pkcs11_lib_options options = { + .filename = filename, + .initialize_finalize_behavior = behavior, + }; + struct aws_pkcs11_lib *pkcs11_lib = aws_pkcs11_lib_new(aws_py_get_allocator(), &options); + if (pkcs11_lib == NULL) { + return PyErr_AwsLastError(); + } + + PyObject *capsule = PyCapsule_New(pkcs11_lib, s_capsule_name, s_pkcs11_lib_capsule_destructor); + if (capsule == NULL) { + aws_pkcs11_lib_release(pkcs11_lib); /* cleanup due to error */ + return NULL; + } + + return capsule; +} diff --git a/test/test_io.py b/test/test_io.py index 6682894e7..a967c0d26 100644 --- a/test/test_io.py +++ b/test/test_io.py @@ -1,9 +1,11 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0. -from awscrt.io import ClientBootstrap, ClientTlsContext, DefaultHostResolver, EventLoopGroup, InputStream, TlsConnectionOptions, TlsContextOptions +from awscrt.io import * from test import NativeResourceTest, TIMEOUT import io +import os +import sys import unittest @@ -53,9 +55,12 @@ def test_with_mtls_from_path(self): ctx = ClientTlsContext(opt) def test_with_mtls_pkcs12(self): - opt = TlsContextOptions.create_client_with_mtls_pkcs12( - 'test/resources/unittest.p12', '1234') - ctx = ClientTlsContext(opt) + try: + opt = TlsContextOptions.create_client_with_mtls_pkcs12( + 'test/resources/unittest.p12', '1234') + ctx = ClientTlsContext(opt) + except NotImplementedError: + raise unittest.SkipTest(f'PKCS#12 not supported on this platform ({sys.platform})') def test_override_default_trust_store_dir(self): opt = TlsContextOptions() @@ -143,5 +148,47 @@ def test_read_duck_typed_io(self): self._test(python_stream, src_data) +class Pkcs11LibTest(NativeResourceTest): + def _lib_path(self): + name = 'AWS_TEST_PKCS11_LIB' + val = os.environ.get(name) + if not val: + raise unittest.SkipTest(f"test requires env var: {name}") + return val + + def test_init(self): + # sanity check that we can create/destroy + lib_path = self._lib_path() + pcks11_lib = Pkcs11Lib(file=lib_path, behavior=Pkcs11Lib.InitializeFinalizeBehavior.STRICT) + + def test_exceptions(self): + # check that initialization errors bubble up as exceptions + with self.assertRaises(Exception): + pkcs11_lib = Pkcs11Lib(file='obviously-invalid-path.so') + + with self.assertRaises(Exception): + with open(self._lib_path()) as literal_open_file: + # a filepath str should passed, not a literal open file + pkcs11_lib = Pkcs11Lib(file=literal_open_file) + + def test_strict_behavior(self): + lib_path = self._lib_path() + lib1 = Pkcs11Lib(file=lib_path, behavior=Pkcs11Lib.InitializeFinalizeBehavior.STRICT) + # InitializeFinalizeBehavior.STRICT behavior should fail if the PKCS#11 lib is already loaded + with self.assertRaises(Exception): + lib2 = Pkcs11Lib(file=lib_path, behavior=Pkcs11Lib.InitializeFinalizeBehavior.STRICT) + + def test_omit_behavior(self): + lib_path = self._lib_path() + # InitializeFinalizeBehavior.OMIT should fail unless another instance of the PKCS#11 lib is already loaded + with self.assertRaises(Exception): + lib = Pkcs11Lib(file=lib_path, behavior=Pkcs11Lib.InitializeFinalizeBehavior.OMIT) + + # InitializeFinalizeBehavior.OMIT behavior should be fine when another + # instance of the PKCS#11 lib is already loaded + strict_lib = Pkcs11Lib(file=lib_path, behavior=Pkcs11Lib.InitializeFinalizeBehavior.STRICT) + omit_lib = Pkcs11Lib(file=lib_path, behavior=Pkcs11Lib.InitializeFinalizeBehavior.OMIT) + + if __name__ == '__main__': unittest.main() diff --git a/test/test_mqtt.py b/test/test_mqtt.py index 292d99bda..25a49d74c 100644 --- a/test/test_mqtt.py +++ b/test/test_mqtt.py @@ -1,15 +1,14 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0. -from awscrt.auth import AwsCredentialsProvider -from awscrt.http import HttpProxyOptions -from awscrt.io import ClientBootstrap, ClientTlsContext, DefaultHostResolver, EventLoopGroup, TlsConnectionOptions, TlsContextOptions, LogLevel, init_logging +from awscrt.io import ClientBootstrap, ClientTlsContext, DefaultHostResolver, EventLoopGroup, Pkcs11Lib, TlsContextOptions, LogLevel, init_logging from awscrt.mqtt import Client, Connection, QoS from test import NativeResourceTest from concurrent.futures import Future import enum import os import pathlib +import sys import unittest import uuid @@ -30,14 +29,14 @@ def test_lifetime(self): class Config: def __init__(self, auth_type): self.endpoint = self._get_env('AWS_TEST_IOT_MQTT_ENDPOINT') + self.cert_path = self._get_env('AWS_TEST_TLS_CERT_PATH') + self.cert = pathlib.Path(self.cert_path).read_text().encode('utf-8') if auth_type == AuthType.CERT_AND_KEY: - self.cert_path = self._get_env('AWS_TEST_TLS_CERT_PATH') - self.cert = pathlib.Path(self.cert_path).read_text().encode('utf-8') self.key_path = self._get_env('AWS_TEST_TLS_KEY_PATH') self.key = pathlib.Path(self.key_path).read_text().encode('utf-8') - if auth_type == AuthType.PKCS11: + elif auth_type == AuthType.PKCS11: self.pkcs11_lib_path = self._get_env('AWS_TEST_PKCS11_LIB') self.pkcs11_pin = self._get_env('AWS_TEST_PKCS11_PIN') self.pkcs11_token_label = self._get_env('AWS_TEST_PKCS11_TOKEN_LABEL') @@ -58,14 +57,37 @@ class MqttConnectionTest(NativeResourceTest): TEST_TOPIC = '/test/me/senpai' TEST_MSG = 'NOTICE ME!'.encode('utf8') - def _test_connection(self): - config = Config(AuthType.CERT_AND_KEY) + def _create_connection(self, auth_type=AuthType.CERT_AND_KEY): + config = Config(auth_type) elg = EventLoopGroup() resolver = DefaultHostResolver(elg) bootstrap = ClientBootstrap(elg, resolver) - tls_opts = TlsContextOptions.create_client_with_mtls(config.cert, config.key) - tls = ClientTlsContext(tls_opts) + if auth_type == AuthType.CERT_AND_KEY: + tls_opts = TlsContextOptions.create_client_with_mtls_from_path(config.cert_path, config.key_path) + tls = ClientTlsContext(tls_opts) + + elif auth_type == AuthType.PKCS11: + try: + pkcs11_lib = Pkcs11Lib( + file=config.pkcs11_lib_path, + behavior=Pkcs11Lib.InitializeFinalizeBehavior.STRICT) + + tls_opts = TlsContextOptions.create_client_with_mtls_pkcs11( + pkcs11_lib=pkcs11_lib, + user_pin=config.pkcs11_pin, + token_label=config.pkcs11_token_label, + private_key_label=config.pkcs11_key_label, + cert_file_path=config.cert_path) + + tls = ClientTlsContext(tls_opts) + + except Exception as e: + if 'AWS_ERROR_UNIMPLEMENTED' in str(e): + raise unittest.SkipTest(f'TLS with PKCS#11 not supported on this platform ({sys.platform})') + else: + # re-raise exception + raise client = Client(bootstrap, tls) connection = Connection( @@ -73,15 +95,21 @@ def _test_connection(self): client_id=create_client_id(), host_name=config.endpoint, port=8883) - connection.connect().result(TIMEOUT) return connection def test_connect_disconnect(self): - connection = self._test_connection() + connection = self._create_connection() + connection.connect().result(TIMEOUT) + connection.disconnect().result(TIMEOUT) + + def test_pkcs11(self): + connection = self._create_connection(AuthType.PKCS11) + connection.connect().result(TIMEOUT) connection.disconnect().result(TIMEOUT) def test_pub_sub(self): - connection = self._test_connection() + connection = self._create_connection() + connection.connect().result(TIMEOUT) received = Future() def on_message(**kwargs): @@ -116,20 +144,7 @@ def on_message(**kwargs): connection.disconnect().result(TIMEOUT) def test_on_message(self): - config = Config(AuthType.CERT_AND_KEY) - elg = EventLoopGroup() - resolver = DefaultHostResolver(elg) - bootstrap = ClientBootstrap(elg, resolver) - - tls_opts = TlsContextOptions.create_client_with_mtls(config.cert, config.key) - tls = ClientTlsContext(tls_opts) - - client = Client(bootstrap, tls) - connection = Connection( - client=client, - client_id=create_client_id(), - host_name=config.endpoint, - port=8883) + connection = self._create_connection() received = Future() def on_message(**kwargs): @@ -160,20 +175,7 @@ def on_message(**kwargs): def test_on_message_old_fn_signature(self): # ensure that message-received callbacks with the old function signature still work - config = Config(AuthType.CERT_AND_KEY) - elg = EventLoopGroup() - resolver = DefaultHostResolver(elg) - bootstrap = ClientBootstrap(elg, resolver) - - tls_opts = TlsContextOptions.create_client_with_mtls(config.cert, config.key) - tls = ClientTlsContext(tls_opts) - - client = Client(bootstrap, tls) - connection = Connection( - client=client, - client_id=create_client_id(), - host_name=config.endpoint, - port=8883) + connection = self._create_connection() any_received = Future() sub_received = Future()