From ca061e4e374d7aeee3b0e4a3b476a33de7e1f464 Mon Sep 17 00:00:00 2001 From: Michael Graeb Date: Thu, 6 Jan 2022 16:16:43 -0800 Subject: [PATCH] Fix CI failures (#326) ### ISSUE 1: Fix Windows x86. I discovered that the "windows-vc14 (x86)" CI run was neither x86, nor using vc14. Change it so we actually use x86 python for the x86 runs. I can't figure out how to force an older compiler though, so let's just use whatever python picks by default ### ISSUE 2: Fix occasional failures involving boto3. Some environments failed due to installing `boto3` and then trying to use it, without restarting the python application. So use AWS CLI instead, since we know that's installed everywhere that the builder runs. ### ISSUE 3: Fix manylinux2014-aarch64 failing to link in pthreads library. Usually python automatically sets the `-pthreads` linker flag for you, but this week it wasn't doing that on our manylinux2014-aarch64 docker images. I didn't 100% get to the bottom of why this is suddenly happening, but googling around found evidence of other python extensions [tackling this same issue](https://github.com/scipy/scipy/issues/11323). Simplest fix is to always set `-pthreads` linker flag on Linux. --- .builder/actions/aws_crt_python.py | 77 ++++++++++++++++++------------ .github/workflows/ci.yml | 16 +------ setup.py | 4 ++ 3 files changed, 51 insertions(+), 46 deletions(-) diff --git a/.builder/actions/aws_crt_python.py b/.builder/actions/aws_crt_python.py index f1ab01303..54b639129 100644 --- a/.builder/actions/aws_crt_python.py +++ b/.builder/actions/aws_crt_python.py @@ -1,6 +1,7 @@ import Builder import argparse +import json import os.path import pathlib import subprocess @@ -11,7 +12,7 @@ class InstallPythonReqs(Builder.Action): def __init__(self, trust_hosts=False, deps=[], python=sys.executable): self.trust_hosts = trust_hosts - self.core = ('pip', 'setuptools') + self.core = ('pip', 'setuptools', 'wheel') self.deps = deps self.python = python @@ -25,7 +26,8 @@ def run(self, env): # package database in time for the subsequent install and pip fails steps = [] for deps in (self.core, self.deps): - steps.append([self.python, '-m', 'pip', 'install', *trusted_hosts, *deps]) + if deps: + steps.append([self.python, '-m', 'pip', 'install', '--upgrade', *trusted_hosts, *deps]) return Builder.Script(steps, name='install-python-reqs') @@ -33,38 +35,51 @@ def run(self, env): class SetupForTests(Builder.Action): def run(self, env): - # we want to use boto3 to pull data from secretsmanager - # boto3 might need to be installed first... - try: - import boto3 - except BaseException: - subprocess.check_call([sys.executable, '-m', 'pip', 'install', 'boto3']) - import boto3 - - secrets = boto3.client('secretsmanager') - - # get string from secretsmanager and store in environment variable - def setenv_from_secret(env_var_name, secret_name): - response = secrets.get_secret_value(SecretId=secret_name) - env.shell.setenv(env_var_name, response['SecretString']) - - # get file contents from secretsmanager, store as file under /tmp - # and store path in environment variable - def setenv_tmpfile_from_secret(env_var_name, secret_name, file_name): - response = secrets.get_secret_value(SecretId=secret_name) - file_contents = response['SecretString'] - file_path = os.path.join(tempfile.gettempdir(), file_name) - pathlib.Path(file_path).write_text(file_contents) - env.shell.setenv(env_var_name, file_path) - - setenv_from_secret('AWS_TEST_IOT_MQTT_ENDPOINT', 'unit-test/endpoint') - - setenv_tmpfile_from_secret('AWS_TEST_TLS_CERT_PATH', 'unit-test/certificate', 'certificate.pem') - setenv_tmpfile_from_secret('AWS_TEST_TLS_KEY_PATH', 'unit-test/privatekey', 'privatekey.pem') + self.env = env + + self._setenv_from_secret('AWS_TEST_IOT_MQTT_ENDPOINT', 'unit-test/endpoint') + + self._setenv_tmpfile_from_secret('AWS_TEST_TLS_CERT_PATH', 'unit-test/certificate', 'certificate.pem') + self._setenv_tmpfile_from_secret('AWS_TEST_TLS_KEY_PATH', 'unit-test/privatekey', 'privatekey.pem') # enable S3 tests env.shell.setenv('AWS_TEST_S3', '1') + def _get_secret(self, secret_id): + """get string from secretsmanager""" + + # NOTE: using AWS CLI instead of boto3 because we know CLI is already + # installed wherever builder is run. Once upon a time we tried using + # boto3 by installing it while the builder was running but this didn't + # work in some rare scenarios. + + cmd = ['aws', 'secretsmanager', 'get-secret-value', '--secret-id', secret_id] + # NOTE: print command args, but use "quiet" mode so that output isn't printed. + # we don't want secrets leaked to the build log + print('>', subprocess.list2cmdline(cmd)) + result = self.env.shell.exec(*cmd, check=True, quiet=True) + secret_value = json.loads(result.output) + return secret_value['SecretString'] + + def _tmpfile_from_secret(self, secret_name, file_name): + """get file contents from secretsmanager, store as file under /tmp, return file path""" + file_contents = self._get_secret(secret_name) + file_path = os.path.join(tempfile.gettempdir(), file_name) + print(f"Writing to: {file_path}") + pathlib.Path(file_path).write_text(file_contents) + return file_path + + def _setenv_from_secret(self, env_var_name, secret_name): + """get string from secretsmanager and store in environment variable""" + + secret_value = self._get_secret(secret_name) + self.env.shell.setenv(env_var_name, secret_value) + + def _setenv_tmpfile_from_secret(self, env_var_name, secret_name, file_name): + """get file contents from secretsmanager, store as file under /tmp, and store path in environment variable""" + file_path = self._tmpfile_from_secret(secret_name, file_name) + self.env.shell.setenv(env_var_name, file_path) + class AWSCrtPython(Builder.Action): @@ -76,7 +91,7 @@ def run(self, env): python = args.python if args.python else sys.executable actions = [ - InstallPythonReqs(deps=['boto3'], python=python), + InstallPythonReqs(deps=[], python=python), SetupForTests(), [python, '-m', 'pip', 'install', '--verbose', '.'], # "--failfast" because, given how our leak-detection in tests currently works, diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 71de7c73d..1f499cad1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,29 +83,15 @@ jobs: ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build -p ${{ env.PACKAGE_NAME }} --compiler=${{ matrix.compiler }} windows: - runs-on: windows-latest - steps: - - name: Build ${{ env.PACKAGE_NAME }} + consumers - run: | - python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" - python builder.pyz build -p ${{ env.PACKAGE_NAME }} - - windows-vc14: runs-on: windows-latest strategy: matrix: arch: [x86, x64] steps: - - uses: ilammy/msvc-dev-cmd@v1 - with: - toolset: 14.0 - arch: ${{ matrix.arch }} - uwp: false - spectre: true - name: Build ${{ env.PACKAGE_NAME }} + consumers run: | python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" - python builder.pyz build -p ${{ env.PACKAGE_NAME }} + python builder.pyz build -p ${{ env.PACKAGE_NAME }} --python "C:\\hostedtoolcache\\windows\\Python\\3.7.9\\${{ matrix.arch }}\\python.exe" osx: runs-on: macos-latest diff --git a/setup.py b/setup.py index fdaf83b5d..c44d807a2 100644 --- a/setup.py +++ b/setup.py @@ -258,6 +258,10 @@ def awscrt_ext(): libraries = [':lib{}.a'.format(x) for x in libraries] libraries += ['rt'] + # python usually adds -pthread automatically, but we've observed + # rare cases where that didn't happen, so let's be explicit. + extra_link_args += ['-pthread'] + if distutils.ccompiler.get_default_compiler() != 'msvc': extra_compile_args += ['-Wextra', '-Werror', '-Wno-strict-aliasing', '-std=gnu99']