Builder2 (#119)

* Use python-fixes version of builder * Updated aws-crt-python and manylinux jobs to match new module API * Disable PQ ASM in setup * Re-enabled clang/gcc matrix testing, got rid of superfluous config * Removed manylinux1 moniker * Got rid of additional autopep runs, there's already a lint job * Use v0.4 (latest release)
awslabs · Feb 14, 2020 · f093628 · f093628
1 parent 0f950ea
commit f093628
Show file tree

Hide file tree

Showing 7 changed files with 229 additions and 108 deletions.
diff --git a/.builder/actions/aws_crt_python.py b/.builder/actions/aws_crt_python.py
@@ -0,0 +1,46 @@
+
+import Builder
+import os
+import sys
+
+class InstallPythonReqs(Builder.Action):
+    def __init__(self, trust_hosts=False, deps=[], python=sys.executable):
+        self.trust_hosts = trust_hosts
+        self.core = ('pip', 'setuptools', 'virtualenv')
+        self.deps = deps
+        self.python = python
+
+    def run(self, env):
+        trusted_hosts = []
+        # These are necessary for older hosts with out of date certs or pip
+        if self.trust_hosts:
+            trusted_hosts = ['--trusted-host', 'pypi.org', '--trusted-host', 'files.pythonhosted.org']
+
+        # setuptools must be installed before packages that need it, or else it won't be in the
+        # package database in time for the subsequent install and pip fails
+        steps = []
+        for deps in (self.core, self.deps):
+            steps.append([self.python, '-m', 'pip', 'install', '--upgrade', *trusted_hosts, *deps])
+
+        return Builder.Script(steps, name='install-python-reqs')
+
+
+class AWSCrtPython(Builder.Action):
+    def run(self, env):        
+        # Once the virtualenv is set up, we must use that python, so that the venv is used
+        python = sys.executable
+
+        install_options = []
+        if 'linux' == Builder.Host.current_platform():
+            install_options = [
+                '--install-option=--include-dirs={openssl_include}',
+                '--install-option=--library-dirs={openssl_lib}']
+
+        actions = [
+            InstallPythonReqs(deps=['boto3']),
+            [python, '-m', 'pip', 'install', '.', '--install-option=--verbose', '--install-option=build_ext', *install_options],
+            [python, '-m', 'unittest', 'discover', '--verbose'],
+            [python, 'aws-common-runtime/aws-c-http/integration-testing/http_client_test.py', python, 'elasticurl.py'],
+        ]
+
+        return Builder.Script(actions, name='aws-crt-python')
diff --git a/.builder/actions/manylinux-ci.py b/.builder/actions/manylinux-ci.py
@@ -0,0 +1,86 @@
+
+import Builder
+import os
+import sys
+from aws_crt_python import InstallPythonReqs
+
+pythons = (
+    'cp35-cp35m',
+    'cp36-cp36m',
+    'cp37-cp37m',
+    'cp27-cp27m',
+    'cp27-cp27mu',
+)
+
+
+def python_path(version):
+    return '/opt/python/{}/bin/python'.format(version)
+
+
+def default_python():
+    return sys.executable
+
+
+class ManyLinuxPackage(Builder.Action):
+    def run(self, env):
+        steps = []
+        for version in pythons:
+            python = python_path(version)
+            if not os.path.isfile(python):
+                print('Skipping {} as it is not installed'.format(python))
+                continue
+
+            install_options = []
+            if 'linux' == Builder.Host.current_platform():
+                install_options = [
+                    '--install-option=--include-dirs={openssl_include}',
+                    '--install-option=--library-dirs={openssl_lib}']
+
+            actions = [
+                InstallPythonReqs(python=python),
+                [python,'-m', 'pip', 'install', '.',
+                    '--install-option=--verbose',
+                    '--install-option=sdist',
+                    '--install-option=bdist_wheel',
+                    *install_options],
+                ['auditwheel', 'repair', '--plat', 'manylinux1_x86_64',
+                    'dist/awscrt-*{}-linux_x86_64.whl'.format(python)],
+            ]
+            steps.append(Builder.Script(actions, name=python))
+
+        copy_steps = [
+            ['cp', '-r', 'wheelhouse' '../dist']
+            ['cp', 'dist/*.tar.gz', '../dist/']
+        ]
+
+        steps += copy_steps
+
+        return Builder.Script(steps, name='manylinux-package')
+
+
+class ManyLinuxCI(Builder.Action):
+    def run(self, env):
+        python3 = default_python()
+
+        steps = []
+        for version in pythons:
+            python = python_path(version)
+            if not os.path.isfile(python):
+                print('Skipping {} as it is not installed'.format(python))
+                continue
+
+            install_options = []
+            if 'linux' == Builder.Host.current_platform():
+                install_options = [
+                    '--install-option=--include-dirs={openssl_include}',
+                    '--install-option=--library-dirs={openssl_lib}']
+
+            actions = [
+                InstallPythonReqs(python=python, deps=['boto3'], trust_hosts=True),
+                [python, '-m', 'pip', 'install', '.',
+                    '--install-option=--verbose', '--install-option=build_ext', *install_options],
+                [python3, 'aws-common-runtime/aws-c-http/integration-testing/http_client_test.py', python, 'elasticurl.py'],
+            ]
+            steps.append(Builder.Script(actions, name=python))
+
+        return Builder.Script(steps, name='manylinux-ci')
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -0,0 +1,90 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - '*'
+      - '!master'
+
+env:
+  BUILDER_VERSION: v0.4
+  BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net
+  PACKAGE_NAME: aws-crt-python
+  LINUX_BASE_IMAGE: ubuntu-16-x64
+  RUN: ${{ github.run_id }}-${{ github.run_number }}
+
+jobs:
+  linux-compat:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        image:
+          - manylinux1-x64
+          - manylinux1-x86
+          - manylinux2014-x64
+          - manylinux2014-x86
+    steps:
+        # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages
+    - name: Build ${{ env.PACKAGE_NAME }}
+      run: |
+        echo "${{ secrets.GITHUB_TOKEN }}" | docker login docker.pkg.github.com -u awslabs --password-stdin
+        export DOCKER_IMAGE=docker.pkg.github.com/awslabs/aws-crt-builder/aws-crt-${{ matrix.image }}:${{ env.BUILDER_VERSION }}
+        docker pull $DOCKER_IMAGE
+        docker run --env GITHUB_REF $DOCKER_IMAGE -p ${{ env.PACKAGE_NAME }} run manylinux-ci
+
+  al2:
+    runs-on: ubuntu-latest
+    steps:
+        # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages
+    - name: Build ${{ env.PACKAGE_NAME }} + consumers
+      run: |
+        echo "${{ secrets.GITHUB_TOKEN }}" | docker login docker.pkg.github.com -u awslabs --password-stdin
+        export DOCKER_IMAGE=docker.pkg.github.com/awslabs/aws-crt-builder/aws-crt-al2-x64:${{ env.BUILDER_VERSION }}
+        docker pull $DOCKER_IMAGE
+        docker run --env GITHUB_REF $DOCKER_IMAGE -p ${{ env.PACKAGE_NAME }} run aws-crt-python
+      
+  clang-compat:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        version: [3, 6, 8, 9]
+    steps:
+        # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages
+    - name: Build ${{ env.PACKAGE_NAME }}
+      run: |
+        echo "${{ secrets.GITHUB_TOKEN }}" | docker login docker.pkg.github.com -u awslabs --password-stdin
+        export DOCKER_IMAGE=docker.pkg.github.com/awslabs/aws-crt-builder/aws-crt-${{ env.LINUX_BASE_IMAGE }}:${{ env.BUILDER_VERSION }}
+        docker pull $DOCKER_IMAGE
+        docker run --env GITHUB_REF $DOCKER_IMAGE -p ${{ env.PACKAGE_NAME }} run aws-crt-python --compiler=clang-${{ matrix.version }}
+
+  gcc-compat:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        version: [5, 6, 7, 8] # no 4.8 on ubuntu due to the -Wdate-time missing command line options
+    steps:
+        # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages
+    - name: Build ${{ env.PACKAGE_NAME }}
+      run: |
+        echo "${{ secrets.GITHUB_TOKEN }}" | docker login docker.pkg.github.com -u awslabs --password-stdin
+        export DOCKER_IMAGE=docker.pkg.github.com/awslabs/aws-crt-builder/aws-crt-${{ env.LINUX_BASE_IMAGE }}:${{ env.BUILDER_VERSION }}
+        docker pull $DOCKER_IMAGE
+        docker run --env GITHUB_REF $DOCKER_IMAGE -p ${{ env.PACKAGE_NAME }} run aws-crt-python --compiler=gcc-${{ matrix.version }}
+
+  windows:
+    runs-on: windows-latest
+    steps:
+    - name: Build ${{ env.PACKAGE_NAME }} + consumers
+      run: |
+        python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_VERSION }}/builder?run=${{ env.RUN }}', 'builder.pyz')"
+        python builder.pyz -p ${{ env.PACKAGE_NAME }} run aws-crt-python
+
+  osx:
+    runs-on: macos-latest
+    steps:
+    - name: Build ${{ env.PACKAGE_NAME }} + consumers
+      run: |
+        python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_VERSION }}/builder?run=${{ env.RUN }}', 'builder')"
+        chmod a+x builder
+        ./builder -p ${{ env.PACKAGE_NAME }} run aws-crt-python
+
diff --git a/.github/workflows/mac-osx-ci.yml b/.github/workflows/mac-osx-ci.yml
diff --git a/aws-common-runtime/s2n b/aws-common-runtime/s2n
diff --git a/builder.json b/builder.json
@@ -1,96 +1,17 @@
 {
     "name": "aws-crt-python",
-    "hosts": {
-        "linux": {
-            "architectures": {
-                "x64": {
-                    "apt_packages": ["build-essential"],
-                    "_comment": "on ubuntu 16, python was using /bin/ld to link, which wasn't fully installed"
-                }
-            },
-            "variables": {
-                "openssl_include": "=/opt/openssl/include",
-                "openssl_lib": "=/opt/openssl/lib"
-            }
-        },
-        "al2012": {
-            "enabled": false,
-            "_comment": "al2012 can't currently pull builder.py from github. SSL: CERTIFICATE_VERIFY_FAILED",
-            "variables": {
-                "openssl_include": "=/opt/openssl/include",
-                "openssl_lib": "=/opt/openssl/lib"
-            }
-        },
-        "manylinux": {
-            "variables": {
-                "openssl_include": "=/opt/openssl/include",
-                "openssl_lib": "=/opt/openssl/lib"
-            },
-            "post_build_steps": [
-                ["echo", "------ Python 3.6 ------"],
-                ["/opt/python/cp36-cp36m/bin/python", "-m", "pip", "install", "--upgrade", "--trusted-host", "pypi.org", "--trusted-host", "files.pythonhosted.org", "pip", "setuptools", "boto3", "autopep8"],
-                ["/opt/python/cp36-cp36m/bin/python", "setup.py", "--verbose", "build_ext", "--include-dirs{openssl_include}", "--library-dirs{openssl_lib}", "install"],
-                ["/opt/python/cp36-cp36m/bin/python", "-m", "unittest", "discover", "--verbose"],
-                ["/opt/python/cp37-cp37m/bin/python", "aws-common-runtime/aws-c-http/integration-testing/http_client_test.py", "/opt/python/cp36-cp36m/bin/python", "elasticurl.py"],
-                ["echo", "------ Python 3.5 ------"],
-                ["/opt/python/cp35-cp35m/bin/python", "-m", "pip", "install", "--upgrade", "--trusted-host", "pypi.org", "--trusted-host", "files.pythonhosted.org", "pip", "setuptools", "boto3", "autopep8"],
-                ["/opt/python/cp35-cp35m/bin/python", "setup.py", "--verbose", "build_ext", "--include-dirs{openssl_include}", "--library-dirs{openssl_lib}", "install"],
-                ["/opt/python/cp35-cp35m/bin/python", "-m", "unittest", "discover", "--verbose"],
-                ["/opt/python/cp37-cp37m/bin/python", "aws-common-runtime/aws-c-http/integration-testing/http_client_test.py", "/opt/python/cp35-cp35m/bin/python", "elasticurl.py"],
-                ["echo", "------ Python 3.4 ------"],
-                ["/opt/python/cp34-cp34m/bin/python", "-m", "pip", "install", "--upgrade", "--trusted-host", "pypi.org", "--trusted-host", "files.pythonhosted.org", "pip", "setuptools", "boto3", "autopep8"],
-                ["/opt/python/cp34-cp34m/bin/python", "setup.py", "--verbose", "build_ext", "--include-dirs{openssl_include}", "--library-dirs{openssl_lib}", "install"],
-                ["/opt/python/cp34-cp34m/bin/python", "-m", "unittest", "discover", "--verbose"],
-                ["/opt/python/cp37-cp37m/bin/python", "aws-common-runtime/aws-c-http/integration-testing/http_client_test.py", "/opt/python/cp34-cp34m/bin/python", "elasticurl.py"],
-                ["echo", "------ Python 2.7 narrow-unicode ------"],
-                ["/opt/python/cp27-cp27m/bin/python", "-m", "pip", "install", "--upgrade", "--trusted-host", "pypi.org", "--trusted-host", "files.pythonhosted.org", "pip", "setuptools", "boto3", "autopep8"],
-                ["/opt/python/cp27-cp27m/bin/python", "setup.py", "--verbose", "build_ext", "--include-dirs{openssl_include}", "--library-dirs{openssl_lib}", "install"],
-                ["/opt/python/cp27-cp27m/bin/python", "-m", "unittest", "discover", "--verbose"],
-                ["/opt/python/cp37-cp37m/bin/python", "aws-common-runtime/aws-c-http/integration-testing/http_client_test.py", "/opt/python/cp27-cp27m/bin/python", "elasticurl.py"],
-                ["echo", "------ Python 2.7 wide-unicode ------"],
-                ["/opt/python/cp27-cp27mu/bin/python", "-m", "pip", "install", "--upgrade", "--trusted-host", "pypi.org", "--trusted-host", "files.pythonhosted.org", "pip", "setuptools", "boto3", "autopep8"],
-                ["/opt/python/cp27-cp27mu/bin/python", "setup.py", "--verbose", "build_ext", "--include-dirs{openssl_include}", "--library-dirs{openssl_lib}", "install"],
-                ["/opt/python/cp27-cp27mu/bin/python", "-m", "unittest", "discover", "--verbose"],
-                ["/opt/python/cp37-cp37m/bin/python", "aws-common-runtime/aws-c-http/integration-testing/http_client_test.py", "/opt/python/cp27-cp27mu/bin/python", "elasticurl.py"]
-            ]
-        }
-    },
     "targets": {
         "android": {
             "enabled": false,
             "_comment": "disabled until we have a reason to support python on android"
-        }
-    },
-    "compilers": {
-        "clang": {
-            "!post_build_steps": [],
-            "_comment": "don't run clang-tidy until we find a way to generate compile_commands.json.",
-            "versions": {
-                "6": {
-                    "_comment": "nuking post_build_steps also nuked the format check. It's better as a pre-build step anyway.",
-                    "pre_build_steps": ["./format-check.sh"]
-                }
-            }
         },
-        "gcc": {
-            "versions": {
-                "4.8": {
-                    "enabled": false,
-                    "_comment": "on ubuntu 16, python was passing unrecognized flag '-Wdate-time' to gcc 4 (flag doesn't exist until gcc 5). This flag is coming from 'PY_CORE_CFLAGS' defined in /usr/lib/python3.5/plat-x86_64-linux-gnu/_sysconfigdata_m.py"
-                }
+        "linux": {
+            "variables": {
+                "openssl_include": "/opt/openssl/include",
+                "openssl_lib": "/opt/openssl/lib"
             }
         }
     },
     "upstream": [],
-    "downstream": [],
-    "build": [
-        ["git", "submodule", "update", "--init", "--recursive"],
-        ["{python}", "setup.py", "--verbose", "build_ext", "--include-dirs{openssl_include}", "--library-dirs{openssl_lib}", "install"]
-    ],
-    "test": [
-        ["{python}", "-m", "pip", "install", "--upgrade", "--trusted-host", "pypi.org", "--trusted-host", "files.pythonhosted.org", "pip", "setuptools", "boto3", "autopep8"],
-        ["{python}", "-m", "unittest", "discover", "--verbose"],
-        ["{python}", "aws-common-runtime/aws-c-http/integration-testing/http_client_test.py", "{python}", "elasticurl.py"],
-        ["{python}", "-m", "autopep8", "--exit-code", "--diff", "--recursive", "awscrt", "test", "setup.py"]
-    ]
+    "downstream": []
 }
diff --git a/setup.py b/setup.py
@@ -137,6 +137,7 @@ def _build_dependency(self, aws_lib):
             '-DBUILD_SHARED_LIBS=OFF',
             '-DCMAKE_BUILD_TYPE={}'.format(build_type),
             '-DBUILD_TESTING=OFF',
+            '-DS2N_NO_PQ_ASM=ON',
         ])
         if self.include_dirs:
             cmake_args.append('-DCMAKE_INCLUDE_PATH={}'.format(';'.join(self.include_dirs)))