Update Dependencies. Misc improvements (#130)

- Update setup.py version to 0.5.7
- Update dependencies
- Small improvements
  - Reduce default MQTT keep-alive values (see Issue #128)
  - Simplify README
  - Better error message if `cmake` not installed (see [forum issue](https://forums.aws.amazon.com/thread.jspa?threadID=316634&tstart=0))
- CI improvements
  - Set up Github secrets so MQTT tests will run again
  - Alter credentials tests so they can't accidentally leak secrets
  - `ManyLinuxCI` action uses `AWSCrtPython` action.

.builder/actions/aws_crt_python.py

@@ -3,6 +3,7 @@
 import os
 import sys
 
+
 class InstallPythonReqs(Builder.Action):
     def __init__(self, trust_hosts=False, deps=[], python=sys.executable):
         self.trust_hosts = trust_hosts
@@ -26,21 +27,30 @@ def run(self, env):
 
 
 class AWSCrtPython(Builder.Action):
-    def run(self, env):        
-        # Once the virtualenv is set up, we must use that python, so that the venv is used
-        python = sys.executable
+    def __init__(self, custom_python=None, name=None):
+        """
+        Prefer the current python3 executable (so that venv is used).
+        But allow a custom python to be used for installing and testing awscrt on.
+        """
+        self.python3 = sys.executable
+        self.custom_python = custom_python if custom_python else self.python3
+        self.name = name if name else 'aws-crt-python'
 
+    def run(self, env):
         install_options = []
         if 'linux' == Builder.Host.current_platform():
             install_options = [
                 '--install-option=--include-dirs={openssl_include}',
                 '--install-option=--library-dirs={openssl_lib}']
 
         actions = [
-            InstallPythonReqs(deps=['boto3']),
-            [python, '-m', 'pip', 'install', '.', '--install-option=--verbose', '--install-option=build_ext', *install_options],
-            [python, '-m', 'unittest', 'discover', '--verbose'],
-            [python, 'aws-common-runtime/aws-c-http/integration-testing/http_client_test.py', python, 'elasticurl.py'],
+            InstallPythonReqs(deps=['boto3'], python=self.custom_python),
+            [self.custom_python, '-m', 'pip', 'install', '.', '--install-option=--verbose',
+                '--install-option=build_ext', *install_options],
+            [self.custom_python, '-m', 'unittest', 'discover', '--verbose'],
+            # http_client_test.py is python3-only. It launches external processes using the extra args
+            [self.python3, 'aws-common-runtime/aws-c-http/integration-testing/http_client_test.py',
+                self.custom_python, 'elasticurl.py'],
         ]
 
-        return Builder.Script(actions, name='aws-crt-python')
+        return Builder.Script(actions, name=self.name)

.builder/actions/manylinux-ci.py

@@ -2,7 +2,7 @@
 import Builder
 import os
 import sys
-from aws_crt_python import InstallPythonReqs
+from aws_crt_python import AWSCrtPython, InstallPythonReqs
 
 pythons = (
     'cp35-cp35m',
@@ -17,10 +17,6 @@ def python_path(version):
     return '/opt/python/{}/bin/python'.format(version)
 
 
-def default_python():
-    return sys.executable
-
-
 class ManyLinuxPackage(Builder.Action):
     def run(self, env):
         steps = []
@@ -38,7 +34,7 @@ def run(self, env):
 
             actions = [
                 InstallPythonReqs(python=python),
-                [python,'-m', 'pip', 'install', '.',
+                [python, '-m', 'pip', 'install', '.',
                     '--install-option=--verbose',
                     '--install-option=sdist',
                     '--install-option=bdist_wheel',
@@ -60,27 +56,14 @@ def run(self, env):
 
 class ManyLinuxCI(Builder.Action):
     def run(self, env):
-        python3 = default_python()
-
         steps = []
         for version in pythons:
             python = python_path(version)
             if not os.path.isfile(python):
                 print('Skipping {} as it is not installed'.format(python))
                 continue
 
-            install_options = []
-            if 'linux' == Builder.Host.current_platform():
-                install_options = [
-                    '--install-option=--include-dirs={openssl_include}',
-                    '--install-option=--library-dirs={openssl_lib}']
-
-            actions = [
-                InstallPythonReqs(python=python, deps=['boto3'], trust_hosts=True),
-                [python, '-m', 'pip', 'install', '.',
-                    '--install-option=--verbose', '--install-option=build_ext', *install_options],
-                [python3, 'aws-common-runtime/aws-c-http/integration-testing/http_client_test.py', python, 'elasticurl.py'],
-            ]
-            steps.append(Builder.Script(actions, name=python))
+            # Run the usual AWSCrtPython build and test steps for this python
+            steps.append(AWSCrtPython(custom_python=python, name=python))
 
         return Builder.Script(steps, name='manylinux-ci')

.github/workflows/ci.yml

@@ -12,6 +12,9 @@ env:
   PACKAGE_NAME: aws-crt-python
   LINUX_BASE_IMAGE: ubuntu-16-x64
   RUN: ${{ github.run_id }}-${{ github.run_number }}
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
 
 jobs:
   linux-compat:
@@ -30,7 +33,7 @@ jobs:
         echo "${{ secrets.GITHUB_TOKEN }}" | docker login docker.pkg.github.com -u awslabs --password-stdin
         export DOCKER_IMAGE=docker.pkg.github.com/awslabs/aws-crt-builder/aws-crt-${{ matrix.image }}:${{ env.BUILDER_VERSION }}
         docker pull $DOCKER_IMAGE
-        docker run --env GITHUB_REF $DOCKER_IMAGE -p ${{ env.PACKAGE_NAME }} run manylinux-ci
+        docker run --env GITHUB_REF --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --env AWS_DEFAULT_REGION $DOCKER_IMAGE -p ${{ env.PACKAGE_NAME }} run manylinux-ci
 
   al2:
     runs-on: ubuntu-latest
@@ -41,8 +44,8 @@ jobs:
         echo "${{ secrets.GITHUB_TOKEN }}" | docker login docker.pkg.github.com -u awslabs --password-stdin
         export DOCKER_IMAGE=docker.pkg.github.com/awslabs/aws-crt-builder/aws-crt-al2-x64:${{ env.BUILDER_VERSION }}
         docker pull $DOCKER_IMAGE
-        docker run --env GITHUB_REF $DOCKER_IMAGE -p ${{ env.PACKAGE_NAME }} run aws-crt-python
-      
+        docker run --env GITHUB_REF --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --env AWS_DEFAULT_REGION $DOCKER_IMAGE -p ${{ env.PACKAGE_NAME }} run aws-crt-python
+
   clang-compat:
     runs-on: ubuntu-latest
     strategy:
@@ -55,7 +58,7 @@ jobs:
         echo "${{ secrets.GITHUB_TOKEN }}" | docker login docker.pkg.github.com -u awslabs --password-stdin
         export DOCKER_IMAGE=docker.pkg.github.com/awslabs/aws-crt-builder/aws-crt-${{ env.LINUX_BASE_IMAGE }}:${{ env.BUILDER_VERSION }}
         docker pull $DOCKER_IMAGE
-        docker run --env GITHUB_REF $DOCKER_IMAGE -p ${{ env.PACKAGE_NAME }} run aws-crt-python --compiler=clang-${{ matrix.version }}
+        docker run --env GITHUB_REF --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --env AWS_DEFAULT_REGION $DOCKER_IMAGE -p ${{ env.PACKAGE_NAME }} run aws-crt-python --compiler=clang-${{ matrix.version }}
 
   gcc-compat:
     runs-on: ubuntu-latest
@@ -69,7 +72,7 @@ jobs:
         echo "${{ secrets.GITHUB_TOKEN }}" | docker login docker.pkg.github.com -u awslabs --password-stdin
         export DOCKER_IMAGE=docker.pkg.github.com/awslabs/aws-crt-builder/aws-crt-${{ env.LINUX_BASE_IMAGE }}:${{ env.BUILDER_VERSION }}
         docker pull $DOCKER_IMAGE
-        docker run --env GITHUB_REF $DOCKER_IMAGE -p ${{ env.PACKAGE_NAME }} run aws-crt-python --compiler=gcc-${{ matrix.version }}
+        docker run --env GITHUB_REF --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --env AWS_DEFAULT_REGION $DOCKER_IMAGE -p ${{ env.PACKAGE_NAME }} run aws-crt-python --compiler=gcc-${{ matrix.version }}
 
   windows:
     runs-on: windows-latest

.github/workflows/lint.yml

@@ -0,0 +1,31 @@
+name: Lint
+
+on: [push]
+
+jobs:
+  clang-format:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout Sources
+      uses: actions/checkout@v2
+
+    - name: clang-format lint
+      uses: DoozyX/[email protected]
+      with:
+        # List of extensions to check
+        extensions: c,h
+
+  autopep8:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout Source
+      uses: actions/checkout@v2
+
+    - name: Build and Test
+      run: |
+        python3 -m pip install --upgrade setuptools
+        python3 -m pip install --upgrade autopep8
+        python3 -m autopep8 --exit-code --diff --recursive awscrt test .builder setup.py

README.md

@@ -1,61 +1,24 @@
-## AWS Crt Python
+## AWS CRT Python
 
 Python bindings for the AWS Common Runtime
 
 ## License
 
-This library is licensed under the Apache 2.0 License. 
+This library is licensed under the Apache 2.0 License.
 
-## Building the Wheel
+## Installation
 
-This builds the wheel itself
+To install from pip:
 ````bash
-<path to your python binary> setup.py sdist bdist_wheel
+python -m pip install awscrt
 ````
 
-You'll want to do this for every version of Python:
-2.7m, 2.7mu, 3.4m, 3.5m, 3.6m, and 3.7m
-
-Don't worry, it will only do a full build on the first run. All the others will just reuse the artifacts.
-
-### Intel Linux
-On linux, assuming you built on an ancient distro (GLIBC is <= 2.5), you'll need to have python inspect the artifact
-and ensure it's compatible:
-
-````bash
-auditwheel show dist/<your output wheel>
-````
-
-If everything is good to go, you'll see a line that says:
-
-following platform tag: "manylinux1_\<arch\>".
-
-One thing to keep in mind, you'll need a version of libcrypto build as a static lib with position independent code (-fPIC).
-DO NOT use the shared lib version of libcrypto or the wheel will not pass this test.
-
-Now you'll need to have the wheel renamed to something compatible with pypi: 
-
+To install from Github:
 ````bash
-auditwheel repair --plat manylinux1_x86_64 -w dist dist/<your image name>
+git clone https://github.com/awslabs/aws-crt-python.git
+cd aws-crt-python
+git submodule update --init
+python -m pip install .
 ````
 
-After doing this, I like to delete all of the old wheel images for making the next step easier:
-
-### ARM Linux
-The process is similar to Intel, but we haven't built the wheels yet. We'll update this document when we finally get that finished.
-
-## Publishing
-Upload to pypi:
-
-````bash
-<path to your python binary> -m twine upload --repository-url <the repo> dist/*
-````
-
-## Using From Your Python Application
-
-Normally, you just declare `aws_crt` as a dependency in your setup.py file.
-
-### Installing from pip
-````bash
-pip install aws_crt
-````
+To use from your Python application, declare `awscrt` as a dependency in your `setup.py` file.

awscrt/awsiot_mqtt_connection_builder.py

@@ -40,11 +40,11 @@
     keep_alive_secs (int): The keep alive value, in seconds, to send in CONNECT packet.
                 A PING will automatically be sent at this interval.
                 The server will assume the connection is lost if no PING is received after 1.5X this value.
-                Default is 3600 (1 hour). This value must be higher than ping_timeout_ms.
+                Default is 1200sec (20 minutes). This duration must be longer than ping_timeout_ms.
 
     ping_timeout_ms (int): Milliseconds to wait for ping response before client assumes
                 the connection is invalid and attempts to reconnect.
-                Default is 3000ms (3 seconds). This value must be less than keep_alive.
+                Default is 3000ms (3 seconds). This duration must be shorter than keep_alive_secs.
                 Alternatively, TCP keep-alive may accomplish this in a more efficient (low-power) scenario,
                 but keep-alive options may not work the same way on every platform and OS version.
 
@@ -182,7 +182,7 @@ def _builder(
         clean_session=kwargs.get('clean_session', False),
         reconnect_min_timeout_secs=kwargs.get('reconnect_min_timeout_secs', 5),
         reconnect_max_timeout_secs=kwargs.get('reconnect_max_timeout_secs', 60),
-        keep_alive_secs=kwargs.get('keep_alive_secs', 3600),
+        keep_alive_secs=kwargs.get('keep_alive_secs', 1200),
         ping_timeout_ms=kwargs.get('ping_timeout_ms', 3000),
         will=kwargs.get('will'),
         username=username,

awscrt/mqtt.py

@@ -77,7 +77,7 @@ def __init__(self,
                  on_connection_resumed=None,
                  reconnect_min_timeout_secs=5,
                  reconnect_max_timeout_secs=60,
-                 keep_alive_secs=3600,
+                 keep_alive_secs=1200,
                  ping_timeout_ms=3000,
                  will=None,
                  username=None,
@@ -132,11 +132,11 @@ def __init__(self,
             keep_alive_secs (int): The keep alive value, in seconds, to send in CONNECT packet.
                         A PING will automatically be sent at this interval.
                         The server will assume the connection is lost if no PING is received after 1.5X this value.
-                        This value must be higher than ping_timeout_ms.
+                        This duration must be longer than ping_timeout_ms.
 
             ping_timeout_ms (int): Milliseconds to wait for ping response before client assumes
                         the connection is invalid and attempts to reconnect.
-                        This value must be less than keep_alive.
+                        This duration must be shorter than keep_alive_secs.
                         Alternatively, TCP keep-alive may accomplish this in a more efficient (low-power) scenario,
                         but keep-alive options may not work the same way on every platform and OS version.
 
@@ -173,6 +173,9 @@ def __init__(self,
         assert isinstance(websocket_proxy_options, HttpProxyOptions) or websocket_proxy_options is None
         assert callable(websocket_handshake_transform) or websocket_handshake_transform is None
 
+        if keep_alive_secs * 1000 <= ping_timeout_ms:
+            raise ValueError("'keep_alive_secs' duration must be longer than 'ping_timeout_ms'")
+
         super(Connection, self).__init__()
 
         # init-only

setup.py

@@ -83,6 +83,19 @@ def determine_generator_args():
     return []
 
 
+cmake_found = False
+
+
+def check_cmake_installed():
+    global cmake_found
+    if not cmake_found:
+        try:
+            subprocess.check_call(['cmake', '--version'])
+            cmake_found = True
+        except Exception:
+            raise Exception("'cmake' not found. cmake must be installed to build from source.")
+
+
 class AwsLib(object):
     def __init__(self, name, extra_cmake_args=[]):
         self.name = name
@@ -110,6 +123,8 @@ def __init__(self, name, extra_cmake_args=[]):
 
 class awscrt_build_ext(setuptools.command.build_ext.build_ext):
     def _build_dependency(self, aws_lib):
+        check_cmake_installed()
+
         prev_cwd = os.getcwd()  # restore cwd at end of function
         lib_source_dir = os.path.join(PROJECT_DIR, 'aws-common-runtime', aws_lib.name)
 
@@ -229,7 +244,7 @@ def awscrt_ext():
 
 setuptools.setup(
     name="awscrt",
-    version="0.5.6",
+    version="0.5.7",
     author="Amazon Web Services, Inc",
     author_email="[email protected]",
     description="A common runtime for AWS Python projects",