Actually use manylinux2014 docker image. (#194)

Actually use the manylinux2014 docker images, instead of just running on an EC2 instance that's similar enough that we happen to pass the wheel audit.

Also, setup.py improvements:
- accept `cmake3` as well as `cmake`.
- slight cleanup of how AWS_LIBCRYPTO_INSTALL is used.

.gitignore

@@ -83,6 +83,7 @@ lib64/
 parts/
 sdist/
 var/
+wheelhouse/
 wheels/
 *.egg-info/
 .installed.cfg

codebuild/cd/manylinux-x64-build.yml

@@ -19,6 +19,10 @@ phases:
       - auditwheel repair --plat manylinux1_x86_64 dist/awscrt-*cp36-cp36m-linux_x86_64.whl
       - /opt/python/cp37-cp37m/bin/python setup.py sdist bdist_wheel
       - auditwheel repair --plat manylinux1_x86_64 dist/awscrt-*cp37-cp37m-linux_x86_64.whl
+      - /opt/python/cp38-cp38/bin/python setup.py sdist bdist_wheel
+      - auditwheel repair --plat manylinux1_x86_64 dist/awscrt-*cp38-cp38-linux_x86_64.whl
+      - /opt/python/cp39-cp39/bin/python setup.py sdist bdist_wheel
+      - auditwheel repair --plat manylinux1_x86_64 dist/awscrt-*cp39-cp39-linux_x86_64.whl
       - cp -r wheelhouse ../dist
       - cp dist/*.tar.gz ../dist/
   post_build:

codebuild/cd/manylinux-x86-build.yml

@@ -19,6 +19,10 @@ phases:
       - auditwheel repair --plat manylinux1_i686 dist/awscrt-*cp36-cp36m-linux_i686.whl
       - /opt/python/cp37-cp37m/bin/python setup.py sdist bdist_wheel
       - auditwheel repair --plat manylinux1_i686 dist/awscrt-*cp37-cp37m-linux_i686.whl
+      - /opt/python/cp38-cp38/bin/python setup.py sdist bdist_wheel
+      - auditwheel repair --plat manylinux1_i686 dist/awscrt-*cp38-cp38-linux_i686.whl
+      - /opt/python/cp39-cp39/bin/python setup.py sdist bdist_wheel
+      - auditwheel repair --plat manylinux1_i686 dist/awscrt-*cp39-cp39-linux_i686.whl
       - cp -r wheelhouse ../dist
   post_build:
     commands:

continuous-delivery/build-wheels-manylinux2014-aarch64-jenkins.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+#run build-wheels script in manylinux2014 docker image
+set -ex
+
+DOCKER_IMAGE=123124136734.dkr.ecr.us-east-1.amazonaws.com/aws-crt/manylinux2014-aarch64:latest
+
+$(aws --region us-east-1 ecr get-login --no-include-email)
+
+docker pull $DOCKER_IMAGE
+
+docker run \
+    --mount type=bind,source=`pwd`,target=/aws-crt-python \
+    --workdir /aws-crt-python \
+    --entrypoint /aws-crt-python/continuous-delivery/build-wheels-manylinux2014-aarch64.sh \
+    $DOCKER_IMAGE

continuous-delivery/build-wheels-manylinux2014-aarch64.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+#assumes image based on manylinux2014 + extras (cmake3, libcrypto, etc)
+set -ex
+
+/opt/python/cp38-cp38/bin/python ./continuous-delivery/update-version.py
+
+export AWS_LIBCRYPTO_INSTALL=/opt/openssl
+
+/opt/python/cp35-cp35m/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp35*.whl
+
+/opt/python/cp36-cp36m/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp36*.whl
+
+/opt/python/cp37-cp37m/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp37*.whl
+
+/opt/python/cp38-cp38/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp38*.whl
+
+/opt/python/cp39-cp39/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp39*.whl
+
+rm dist/*.whl
+cp -rv wheelhouse/* dist/
+
+#now you just need to run twine (that's in a different script)

continuous-delivery/build-wheels-manylinux2014-x86_64-jenkins.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+#run build-wheels script in manylinux2014 docker image
+set -ex
+
+DOCKER_IMAGE=123124136734.dkr.ecr.us-east-1.amazonaws.com/aws-crt/manylinux2014-x64:latest
+
+$(aws --region us-east-1 ecr get-login --no-include-email)
+
+docker pull $DOCKER_IMAGE
+
+docker run \
+    --mount type=bind,source=`pwd`,target=/aws-crt-python \
+    --workdir /aws-crt-python \
+    --entrypoint /aws-crt-python/continuous-delivery/build-wheels-manylinux2014-x86_64.sh \
+    $DOCKER_IMAGE

continuous-delivery/build-wheels-manylinux2014-x86_64.sh

@@ -1,24 +1,26 @@
 #!/bin/bash
-#before running this, you'll need cmake3 and a compiler. These python versions are just
-#using the default python installers from python.org. Each version needs updated pip, wheel, and setuptools
+#assumes image based on manylinux2014 + extras (cmake3, libcrypto, etc)
 set -ex
 
-/usr/local/bin/python3.8 ./continuous-delivery/update-version.py
+/opt/python/cp38-cp38/bin/python ./continuous-delivery/update-version.py
 
 export AWS_LIBCRYPTO_INSTALL=/opt/openssl
 
-/usr/local/bin/python3.5m setup.py sdist bdist_wheel
+/opt/python/cp35-cp35m/bin/python setup.py sdist bdist_wheel
 auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp35*.whl
 
-/usr/local/bin/python3.6m setup.py sdist bdist_wheel
+/opt/python/cp36-cp36m/bin/python setup.py sdist bdist_wheel
 auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp36*.whl
 
-/usr/local/bin/python3.7m setup.py sdist bdist_wheel
+/opt/python/cp37-cp37m/bin/python setup.py sdist bdist_wheel
 auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp37*.whl
 
-/usr/local/bin/python3.8 setup.py sdist bdist_wheel
+/opt/python/cp38-cp38/bin/python setup.py sdist bdist_wheel
 auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp38*.whl
 
+/opt/python/cp39-cp39/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp39*.whl
+
 rm dist/*.whl
 cp -rv wheelhouse/* dist/
 

setup.py

@@ -9,6 +9,7 @@
 import platform
 import setuptools
 import setuptools.command.build_ext
+import shutil
 import subprocess
 import sys
 
@@ -73,17 +74,54 @@ def determine_generator_args():
     return []
 
 
-cmake_found = False
+cmake_found = None
 
 
-def check_cmake_installed():
+def get_cmake_path():
     global cmake_found
-    if not cmake_found:
-        try:
-            subprocess.check_call(['cmake', '--version'])
-            cmake_found = True
-        except Exception:
-            raise Exception("'cmake' not found. cmake must be installed to build from source.")
+    if cmake_found:
+        return cmake_found
+
+    for cmake_alias in ['cmake3', 'cmake']:
+        cmake_found = shutil.which(cmake_alias)
+        if cmake_found:
+            return cmake_found
+
+    raise Exception("CMake must be installed to build from source.")
+
+
+def get_libcrypto_static_library(libcrypto_dir):
+    lib_path = os.path.join(libcrypto_dir, 'lib64', 'libcrypt')
+    if is_64bit() and os.path.exists(lib_path):
+        return lib_path
+
+    lib_path = os.path.join(libcrypto_dir, 'lib32', 'libcrypto.a')
+    if is_32bit() and os.path.exists(lib_path):
+        return lib_path
+
+    lib_path = os.path.join(libcrypto_dir, 'lib', 'libcrypto.a')
+    if os.path.exists(lib_path):
+        return lib_path
+
+    raise Exception('Bad AWS_LIBCRYPTO_INSTALL, file not found: ' + lib_path)
+
+
+def get_libcrypto_paths():
+    # return None if not using libcrypto
+    if sys.platform == 'darwin' or sys.platform == 'win32':
+        return None
+    libcrypto_dir = os.environ.get('AWS_LIBCRYPTO_INSTALL')
+    if not libcrypto_dir:
+        return None
+
+    # find include dir
+    include_dir = os.path.join(libcrypto_dir, 'include')
+    expected_file = os.path.join(include_dir, 'openssl', 'crypto.h')
+    if not os.path.exists(expected_file):
+        raise Exception('Bad AWS_LIBCRYPTO_INSTALL, file not found: ' + expected_file)
+
+    static_library = get_libcrypto_static_library(libcrypto_dir)
+    return {'include_dir': include_dir, 'static_library': static_library}
 
 
 class AwsLib:
@@ -110,14 +148,10 @@ def __init__(self, name, extra_cmake_args=[]):
 DEP_BUILD_DIR = os.path.join(PROJECT_DIR, 'build', 'deps')
 DEP_INSTALL_PATH = os.environ.get('AWS_C_INSTALL', os.path.join(DEP_BUILD_DIR, 'install'))
 
-AWS_LIBCRYPTO_INSTALL = None
-if sys.platform != 'darwin' and sys.platform != 'win32':
-    AWS_LIBCRYPTO_INSTALL = os.environ.get('AWS_LIBCRYPTO_INSTALL', os.path.join(DEP_BUILD_DIR, 'libcrypto'))
-
 
 class awscrt_build_ext(setuptools.command.build_ext.build_ext):
-    def _build_dependency(self, aws_lib):
-        check_cmake_installed()
+    def _build_dependency(self, aws_lib, libcrypto_paths):
+        cmake = get_cmake_path()
 
         prev_cwd = os.getcwd()  # restore cwd at end of function
         lib_source_dir = os.path.join(PROJECT_DIR, 'crt', aws_lib.name)
@@ -137,7 +171,7 @@ def _build_dependency(self, aws_lib):
         os.chdir(lib_build_dir)
 
         # cmake configure
-        cmake_args = ['cmake']
+        cmake_args = [cmake]
         cmake_args.extend(determine_generator_args())
         cmake_args.extend(determine_cross_compile_args())
         cmake_args.extend([
@@ -151,10 +185,9 @@ def _build_dependency(self, aws_lib):
             cmake_args.append('-DCMAKE_INCLUDE_PATH="{}"'.format(';'.join(self.include_dirs)))
         if self.library_dirs:
             cmake_args.append('-DCMAKE_LIBRARY_PATH="{}"'.format(';'.join(self.library_dirs)))
-        if AWS_LIBCRYPTO_INSTALL:
-            cmake_args.append('-DLibCrypto_INCLUDE_DIR={}/include'.format(AWS_LIBCRYPTO_INSTALL))
-            cmake_args.append('-DLibCrypto_STATIC_LIBRARY={}/lib/libcrypto.a'.format(AWS_LIBCRYPTO_INSTALL))
-            self.library_dirs.append('{}/lib'.format(AWS_LIBCRYPTO_INSTALL))
+        if libcrypto_paths:
+            cmake_args.append('-DLibCrypto_INCLUDE_DIR={}'.format(libcrypto_paths['include_dir']))
+            cmake_args.append('-DLibCrypto_STATIC_LIBRARY={}'.format(libcrypto_paths['static_library']))
         cmake_args.extend(aws_lib.extra_cmake_args)
         cmake_args.append(lib_source_dir)
 
@@ -163,7 +196,7 @@ def _build_dependency(self, aws_lib):
 
         # cmake build/install
         build_cmd = [
-            'cmake',
+            cmake,
             '--build', './',
             '--config', build_type,
             '--target', 'install',
@@ -174,9 +207,13 @@ def _build_dependency(self, aws_lib):
         os.chdir(prev_cwd)
 
     def run(self):
+        libcrypto_paths = get_libcrypto_paths()
+        if libcrypto_paths:
+            self.library_dirs.append(os.path.dirname(libcrypto_paths['static_library']))
+
         # build dependencies
         for lib in AWS_LIBS:
-            self._build_dependency(lib)
+            self._build_dependency(lib, libcrypto_paths)
 
         # update paths so awscrt_ext can access dependencies
         self.include_dirs.append(os.path.join(DEP_INSTALL_PATH, 'include'))