Fix CI failures (#326)

### ISSUE 1: Fix Windows x86.
I discovered that the "windows-vc14 (x86)" CI run was neither x86, nor using vc14. Change it so we actually use x86 python for the x86 runs. I can't figure out how to force an older compiler though, so let's just use whatever python picks by default

### ISSUE 2: Fix occasional failures involving boto3. 
Some environments failed due to installing `boto3` and then trying to use it, without restarting the python application. So use AWS CLI instead, since we know that's installed everywhere that the builder runs.

### ISSUE 3: Fix manylinux2014-aarch64 failing to link in pthreads library.
Usually python automatically sets the `-pthreads` linker flag for you, but this week it wasn't doing that on our manylinux2014-aarch64 docker images. I didn't 100% get to the bottom of why this is suddenly happening, but googling around found evidence of other python extensions [tackling this same issue](scipy/scipy#11323). Simplest fix is to always set `-pthreads` linker flag on Linux.

.builder/actions/aws_crt_python.py

@@ -1,6 +1,7 @@
 
 import Builder
 import argparse
+import json
 import os.path
 import pathlib
 import subprocess
@@ -11,7 +12,7 @@
 class InstallPythonReqs(Builder.Action):
     def __init__(self, trust_hosts=False, deps=[], python=sys.executable):
         self.trust_hosts = trust_hosts
-        self.core = ('pip', 'setuptools')
+        self.core = ('pip', 'setuptools', 'wheel')
         self.deps = deps
         self.python = python
 
@@ -25,46 +26,60 @@ def run(self, env):
         # package database in time for the subsequent install and pip fails
         steps = []
         for deps in (self.core, self.deps):
-            steps.append([self.python, '-m', 'pip', 'install', *trusted_hosts, *deps])
+            if deps:
+                steps.append([self.python, '-m', 'pip', 'install', '--upgrade', *trusted_hosts, *deps])
 
         return Builder.Script(steps, name='install-python-reqs')
 
 
 class SetupForTests(Builder.Action):
 
     def run(self, env):
-        # we want to use boto3 to pull data from secretsmanager
-        # boto3 might need to be installed first...
-        try:
-            import boto3
-        except BaseException:
-            subprocess.check_call([sys.executable, '-m', 'pip', 'install', 'boto3'])
-            import boto3
-
-        secrets = boto3.client('secretsmanager')
-
-        # get string from secretsmanager and store in environment variable
-        def setenv_from_secret(env_var_name, secret_name):
-            response = secrets.get_secret_value(SecretId=secret_name)
-            env.shell.setenv(env_var_name, response['SecretString'])
-
-        # get file contents from secretsmanager, store as file under /tmp
-        # and store path in environment variable
-        def setenv_tmpfile_from_secret(env_var_name, secret_name, file_name):
-            response = secrets.get_secret_value(SecretId=secret_name)
-            file_contents = response['SecretString']
-            file_path = os.path.join(tempfile.gettempdir(), file_name)
-            pathlib.Path(file_path).write_text(file_contents)
-            env.shell.setenv(env_var_name, file_path)
-
-        setenv_from_secret('AWS_TEST_IOT_MQTT_ENDPOINT', 'unit-test/endpoint')
-
-        setenv_tmpfile_from_secret('AWS_TEST_TLS_CERT_PATH', 'unit-test/certificate', 'certificate.pem')
-        setenv_tmpfile_from_secret('AWS_TEST_TLS_KEY_PATH', 'unit-test/privatekey', 'privatekey.pem')
+        self.env = env
+
+        self._setenv_from_secret('AWS_TEST_IOT_MQTT_ENDPOINT', 'unit-test/endpoint')
+
+        self._setenv_tmpfile_from_secret('AWS_TEST_TLS_CERT_PATH', 'unit-test/certificate', 'certificate.pem')
+        self._setenv_tmpfile_from_secret('AWS_TEST_TLS_KEY_PATH', 'unit-test/privatekey', 'privatekey.pem')
 
         # enable S3 tests
         env.shell.setenv('AWS_TEST_S3', '1')
 
+    def _get_secret(self, secret_id):
+        """get string from secretsmanager"""
+
+        # NOTE: using AWS CLI instead of boto3 because we know CLI is already
+        # installed wherever builder is run. Once upon a time we tried using
+        # boto3 by installing it while the builder was running but this didn't
+        # work in some rare scenarios.
+
+        cmd = ['aws', 'secretsmanager', 'get-secret-value', '--secret-id', secret_id]
+        # NOTE: print command args, but use "quiet" mode so that output isn't printed.
+        # we don't want secrets leaked to the build log
+        print('>', subprocess.list2cmdline(cmd))
+        result = self.env.shell.exec(*cmd, check=True, quiet=True)
+        secret_value = json.loads(result.output)
+        return secret_value['SecretString']
+
+    def _tmpfile_from_secret(self, secret_name, file_name):
+        """get file contents from secretsmanager, store as file under /tmp, return file path"""
+        file_contents = self._get_secret(secret_name)
+        file_path = os.path.join(tempfile.gettempdir(), file_name)
+        print(f"Writing to: {file_path}")
+        pathlib.Path(file_path).write_text(file_contents)
+        return file_path
+
+    def _setenv_from_secret(self, env_var_name, secret_name):
+        """get string from secretsmanager and store in environment variable"""
+
+        secret_value = self._get_secret(secret_name)
+        self.env.shell.setenv(env_var_name, secret_value)
+
+    def _setenv_tmpfile_from_secret(self, env_var_name, secret_name, file_name):
+        """get file contents from secretsmanager, store as file under /tmp, and store path in environment variable"""
+        file_path = self._tmpfile_from_secret(secret_name, file_name)
+        self.env.shell.setenv(env_var_name, file_path)
+
 
 class AWSCrtPython(Builder.Action):
 
@@ -76,7 +91,7 @@ def run(self, env):
         python = args.python if args.python else sys.executable
 
         actions = [
-            InstallPythonReqs(deps=['boto3'], python=python),
+            InstallPythonReqs(deps=[], python=python),
             SetupForTests(),
             [python, '-m', 'pip', 'install', '--verbose', '.'],
             # "--failfast" because, given how our leak-detection in tests currently works,

.github/workflows/ci.yml

@@ -83,29 +83,15 @@ jobs:
           ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build -p ${{ env.PACKAGE_NAME }} --compiler=${{ matrix.compiler }}
 
   windows:
-    runs-on: windows-latest
-    steps:
-    - name: Build ${{ env.PACKAGE_NAME }} + consumers
-      run: |
-        python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')"
-        python builder.pyz build -p ${{ env.PACKAGE_NAME }}
-
-  windows-vc14:
     runs-on: windows-latest
     strategy:
       matrix:
         arch: [x86, x64]
     steps:
-    - uses: ilammy/msvc-dev-cmd@v1
-      with:
-        toolset: 14.0
-        arch: ${{ matrix.arch }}
-        uwp: false
-        spectre: true
     - name: Build ${{ env.PACKAGE_NAME }} + consumers
       run: |
         python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')"
-        python builder.pyz build -p ${{ env.PACKAGE_NAME }}
+        python builder.pyz build -p ${{ env.PACKAGE_NAME }} --python "C:\\hostedtoolcache\\windows\\Python\\3.7.9\\${{ matrix.arch }}\\python.exe"
 
   osx:
     runs-on: macos-latest

setup.py

@@ -258,6 +258,10 @@ def awscrt_ext():
         libraries = [':lib{}.a'.format(x) for x in libraries]
         libraries += ['rt']
 
+        # python usually adds -pthread automatically, but we've observed
+        # rare cases where that didn't happen, so let's be explicit.
+        extra_link_args += ['-pthread']
+
     if distutils.ccompiler.get_default_compiler() != 'msvc':
         extra_compile_args += ['-Wextra', '-Werror', '-Wno-strict-aliasing', '-std=gnu99']