Merge branch 'main' into prebuild-aws-lc

awslabs · Oct 8, 2024 · 7811488 · 7811488
2 parents 6d9e9b9 + 49959e3
commit 7811488
Show file tree

Hide file tree

Showing 39 changed files with 478 additions and 276 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
diff --git a/README.md b/README.md
@@ -13,7 +13,7 @@ This library is licensed under the Apache 2.0 License.
 
 ## Minimum Requirements:
 
-*   Python 3.7+
+*   Python 3.8+
 
 ## Installation
 
@@ -45,10 +45,10 @@ To simplify installation, aws-crt-python has its own copy of libcrypto.
 This lets you install a wheel from PyPI without having OpenSSL installed.
 Unix wheels on PyPI come with libcrypto statically compiled in.
 Code to build libcrypto comes from [AWS-LC](https://github.com/aws/aws-lc).
-AWS-LC's code is included in the PyPI source package, 
+AWS-LC's code is included in the PyPI source package,
 and the git repository includes it as a submodule.
 
-If you need aws-crt-python to use the libcrypto included on your system, 
+If you need aws-crt-python to use the libcrypto included on your system,
 set environment variable `AWS_CRT_BUILD_USE_SYSTEM_LIBCRYPTO=1` while building from source:
 
 ```sh
@@ -59,6 +59,19 @@ AWS_CRT_BUILD_USE_SYSTEM_LIBCRYPTO=1 python3 -m pip install --no-binary :all: --
 You can ignore all this on Windows and Apple platforms, where aws-crt-python
 uses the OS's default libraries for TLS and cryptography math.
 
+### AWS_CRT_BUILD_USE_SYSTEM_LIBS ###
+
+aws-crt-python depends on several C libraries that make up the AWS Common Runtime (libaws-c-common, libaws-c-s3, etc).
+By default, these libraries are built along with aws-crt-python and statically compiled in
+(their source code is under [crt/](crt/)).
+
+To skip building these dependencies, because they're already available on your system,
+set environment variable `AWS_CRT_BUILD_USE_SYSTEM_LIBS=1` while building from source:
+
+```sh
+AWS_CRT_BUILD_USE_SYSTEM_LIBS=1 python3 -m pip install .
+```
+
 ## Mac-Only TLS Behavior
 
 Please note that on Mac, once a private key is used with a certificate, that certificate-key pair is imported into the Mac Keychain. All subsequent uses of that certificate will use the stored private key and ignore anything passed in programmatically. Beginning in v0.6.2, when a stored private key from the Keychain is used, the following will be logged at the "info" log level:

diff --git a/awscrt/auth.py b/awscrt/auth.py
@@ -507,7 +507,7 @@ class AwsSignedBodyHeaderType(IntEnum):
     """Do not add a header."""
 
     X_AMZ_CONTENT_SHA_256 = 1
-    """Add the "x-amz-content-sha-256" header with the canonical request's signed body value"""
+    """Add the "x-amz-content-sha256" header with the canonical request's signed body value"""
 
 
 class AwsSigningConfig(NativeResource):

diff --git a/awscrt/checksums.py b/awscrt/checksums.py
@@ -21,3 +21,12 @@ def crc32c(input: bytes, previous_crc32c: int = 0) -> int:
     Returns an unsigned 32-bit integer.
     """
     return _awscrt.checksums_crc32c(input, previous_crc32c)
+
+
+def crc64nvme(input: bytes, previous_crc64nvme: int = 0) -> int:
+    """
+    Perform a CRC64 NVME computation.
+    If continuing to update a running CRC, pass its value into `previous_crc64nvme`.
+    Returns an unsigned 64-bit integer.
+    """
+    return _awscrt.checksums_crc64nvme(input, previous_crc64nvme)
diff --git a/codebuild/cd/manylinux-x64-build.yml b/codebuild/cd/manylinux-x64-build.yml
@@ -12,8 +12,6 @@ phases:
   build:
     commands:
       - echo Build started on `date`
-      - /opt/python/cp37-cp37m/bin/python setup.py sdist bdist_wheel
-      - auditwheel repair --plat manylinux1_x86_64 dist/awscrt-*cp37-cp37m-linux_x86_64.whl
       - /opt/python/cp38-cp38/bin/python setup.py sdist bdist_wheel
       - auditwheel repair --plat manylinux1_x86_64 dist/awscrt-*cp38-cp38-linux_x86_64.whl
       - /opt/python/cp39-cp39/bin/python setup.py sdist bdist_wheel

diff --git a/codebuild/cd/manylinux-x86-build.yml b/codebuild/cd/manylinux-x86-build.yml
@@ -12,8 +12,6 @@ phases:
   build:
     commands:
       - echo Build started on `date`
-      - /opt/python/cp37-cp37m/bin/python setup.py sdist bdist_wheel
-      - auditwheel repair --plat manylinux1_i686 dist/awscrt-*cp37-cp37m-linux_i686.whl
       - /opt/python/cp38-cp38/bin/python setup.py sdist bdist_wheel
       - auditwheel repair --plat manylinux1_i686 dist/awscrt-*cp38-cp38-linux_i686.whl
       - /opt/python/cp39-cp39/bin/python setup.py sdist bdist_wheel

diff --git a/continuous-delivery/build-wheels-manylinux2014-aarch64.sh b/continuous-delivery/build-wheels-manylinux2014-aarch64.sh
@@ -4,9 +4,6 @@ set -ex
 
 /opt/python/cp39-cp39/bin/python ./continuous-delivery/update-version.py
 
-/opt/python/cp37-cp37m/bin/python setup.py sdist bdist_wheel
-auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp37*.whl
-
 /opt/python/cp38-cp38/bin/python setup.py sdist bdist_wheel
 auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp38*.whl
 
@@ -19,9 +16,17 @@ auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp310*.whl
 /opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel
 auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp311*.whl
 
-# Don't need to build wheels for Python 3.12 and later.
+# Don't need to build wheels for Python 3.12.
 # The 3.11 wheel uses the stable ABI, so it works with newer versions too.
 
+# We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions.
+# Manylinux images don't contain setuptools from Python 3.13, so we need to install it.
+# Install in a custom location due to access issues.
+/opt/python/cp313-cp313/bin/python -m pip install --target ./local -r requirements-dev.txt
+export PYTHONPATH=./local:$PYTHONPATH
+/opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp313*.whl
+
 rm dist/*.whl
 cp -rv wheelhouse/* dist/
 

diff --git a/continuous-delivery/build-wheels-manylinux2014-x86_64.sh b/continuous-delivery/build-wheels-manylinux2014-x86_64.sh
@@ -4,9 +4,6 @@ set -ex
 
 /opt/python/cp39-cp39/bin/python ./continuous-delivery/update-version.py
 
-/opt/python/cp37-cp37m/bin/python setup.py sdist bdist_wheel
-auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp37*.whl
-
 /opt/python/cp38-cp38/bin/python setup.py sdist bdist_wheel
 auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp38*.whl
 
@@ -19,9 +16,17 @@ auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp310*.whl
 /opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel
 auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp311*.whl
 
-# Don't need to build wheels for Python 3.12 and later.
+# Don't need to build wheels for Python 3.12.
 # The 3.11 wheel uses the stable ABI, so it works with newer versions too.
 
+# We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions.
+# Manylinux images don't contain setuptools from Python 3.13, so we need to install it.
+# Install in a custom location due to access issues.
+/opt/python/cp313-cp313/bin/python -m pip install --target ./local -r requirements-dev.txt
+export PYTHONPATH=./local:$PYTHONPATH
+/opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp313*.whl
+
 rm dist/*.whl
 cp -rv wheelhouse/* dist/
 

diff --git a/continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh b/continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh
@@ -4,9 +4,6 @@ set -ex
 
 /opt/python/cp39-cp39/bin/python ./continuous-delivery/update-version.py
 
-/opt/python/cp37-cp37m/bin/python setup.py sdist bdist_wheel
-auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp37*.whl
-
 /opt/python/cp38-cp38/bin/python setup.py sdist bdist_wheel
 auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp38*.whl
 
@@ -19,9 +16,17 @@ auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp310*.whl
 /opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel
 auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp311*.whl
 
-# Don't need to build wheels for Python 3.12 and later.
+# Don't need to build wheels for Python 3.12.
 # The 3.11 wheel uses the stable ABI, so it works with newer versions too.
 
+# We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions.
+# Manylinux images don't contain setuptools from Python 3.13, so we need to install it.
+# Install in a custom location due to access issues.
+/opt/python/cp313-cp313/bin/python -m pip install --target ./local -r requirements-dev.txt
+export PYTHONPATH=./local:$PYTHONPATH
+/opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp313*.whl
+
 rm dist/*.whl
 cp -rv wheelhouse/* dist/
 

diff --git a/continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh b/continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh
@@ -4,9 +4,6 @@ set -ex
 
 /opt/python/cp39-cp39/bin/python ./continuous-delivery/update-version.py
 
-/opt/python/cp37-cp37m/bin/python setup.py sdist bdist_wheel
-auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp37*.whl
-
 /opt/python/cp38-cp38/bin/python setup.py sdist bdist_wheel
 auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp38*.whl
 
@@ -19,9 +16,17 @@ auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp310*.whl
 /opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel
 auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp311*.whl
 
-# Don't need to build wheels for Python 3.12 and later.
+# Don't need to build wheels for Python 3.12.
 # The 3.11 wheel uses the stable ABI, so it works with newer versions too.
 
+# We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions.
+# Manylinux images don't contain setuptools from Python 3.13, so we need to install it.
+# Install in a custom location due to access issues.
+/opt/python/cp313-cp313/bin/python -m pip install --target ./local -r requirements-dev.txt
+export PYTHONPATH=./local:$PYTHONPATH
+/opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp313*.whl
+
 rm dist/*.whl
 cp -rv wheelhouse/* dist/
 

diff --git a/continuous-delivery/build-wheels-osx.sh b/continuous-delivery/build-wheels-osx.sh
@@ -5,13 +5,15 @@ set -ex
 
 /Library/Frameworks/Python.framework/Versions/3.9/bin/python3 ./continuous-delivery/update-version.py
 
-/Library/Frameworks/Python.framework/Versions/3.7/bin/python3 setup.py sdist bdist_wheel
 /Library/Frameworks/Python.framework/Versions/3.8/bin/python3 setup.py sdist bdist_wheel
 /Library/Frameworks/Python.framework/Versions/3.9/bin/python3 setup.py sdist bdist_wheel
 /Library/Frameworks/Python.framework/Versions/3.10/bin/python3 setup.py sdist bdist_wheel
 /Library/Frameworks/Python.framework/Versions/3.11/bin/python3 setup.py sdist bdist_wheel
 
-# Don't need to build wheels for Python 3.12 and later.
+# Don't need to build wheels for Python 3.12.
 # The 3.11 wheel uses the stable ABI, so it works with newer versions too.
 
+# We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions.
+/Library/Frameworks/Python.framework/Versions/3.13/bin/python3 setup.py sdist bdist_wheel
+
 #now you just need to run twine (that's in a different script)
diff --git a/continuous-delivery/build-wheels-win32.bat b/continuous-delivery/build-wheels-win32.bat
@@ -1,12 +1,17 @@
 
 "C:\Program Files (x86)\Python39-32\python.exe" .\continuous-delivery\update-version.py || goto error
 
-"C:\Program Files (x86)\Python37-32\python.exe" setup.py sdist bdist_wheel || goto error
 "C:\Program Files (x86)\Python38-32\python.exe" setup.py sdist bdist_wheel || goto error
 "C:\Program Files (x86)\Python39-32\python.exe" setup.py sdist bdist_wheel || goto error
 "C:\Program Files (x86)\Python310-32\python.exe" setup.py sdist bdist_wheel || goto error
 "C:\Program Files (x86)\Python311-32\python.exe" setup.py sdist bdist_wheel || goto error
 
+:: Don't need to build wheels for Python 3.12.
+:: The 3.11 wheel uses the stable ABI, so it works with newer versions too.
+
+:: We are using the 3.13 stable ABI from 3.13 onwards because of deprecated functions.
+"C:\Program Files (x86)\Python313-32\python.exe" setup.py sdist bdist_wheel || goto error
+
 goto :EOF
 
 :error

diff --git a/continuous-delivery/build-wheels-win64.bat b/continuous-delivery/build-wheels-win64.bat
@@ -1,11 +1,16 @@
 "C:\Program Files\Python39\python.exe" continuous-delivery\update-version.py || goto error
 
-"C:\Program Files\Python37\python.exe" setup.py sdist bdist_wheel || goto error
 "C:\Program Files\Python38\python.exe" setup.py sdist bdist_wheel || goto error
 "C:\Program Files\Python39\python.exe" setup.py sdist bdist_wheel || goto error
 "C:\Program Files\Python310\python.exe" setup.py sdist bdist_wheel || goto error
 "C:\Program Files\Python311\python.exe" setup.py sdist bdist_wheel || goto error
 
+:: Don't need to build wheels for Python 3.12.
+:: The 3.11 wheel uses the stable ABI, so it works with newer versions too.
+
+:: We are using the 3.13 stable ABI from 3.13 onwards because of deprecated functions.
+"C:\Program Files\Python313\python.exe" setup.py sdist bdist_wheel || goto error
+
 goto :EOF
 
 :error

diff --git a/crt/aws-c-auth b/crt/aws-c-auth
diff --git a/crt/aws-c-cal b/crt/aws-c-cal
diff --git a/crt/aws-c-common b/crt/aws-c-common
diff --git a/crt/aws-c-compression b/crt/aws-c-compression
diff --git a/crt/aws-c-event-stream b/crt/aws-c-event-stream
diff --git a/crt/aws-c-http b/crt/aws-c-http
diff --git a/crt/aws-c-io b/crt/aws-c-io
diff --git a/crt/aws-c-mqtt b/crt/aws-c-mqtt
diff --git a/crt/aws-c-s3 b/crt/aws-c-s3
diff --git a/crt/aws-c-sdkutils b/crt/aws-c-sdkutils
diff --git a/crt/aws-checksums b/crt/aws-checksums
diff --git a/crt/aws-lc b/crt/aws-lc
diff --git a/crt/s2n b/crt/s2n
diff --git a/docsrc/source/api/checksums.rst b/docsrc/source/api/checksums.rst
@@ -0,0 +1,5 @@
+awscrt.checksums
+================
+
+.. automodule:: awscrt.checksums
+    :members:
diff --git a/docsrc/source/index.rst b/docsrc/source/index.rst
@@ -12,6 +12,7 @@ API Reference
    :maxdepth: 2
 
    api/auth
+   api/checksums
    api/common
    api/crypto
    api/exceptions
+15 −3		.github/workflows/ci.yml
+4 −6		.github/workflows/clang-format.yml
+1 −0		.gitignore
+47 −0		format-check.py
+0 −24		format-check.sh
+1 −0		include/aws/auth/auth.h
+15 −0		include/aws/auth/credentials.h
+8 −8		include/aws/auth/private/credentials_utils.h
+3 −0		source/auth.c
+25 −0		source/aws_signing.c
+3 −1		source/credentials_provider_cached.c
+1 −16		source/credentials_provider_default_chain.c
+104 −5		source/credentials_provider_ecs.c
+17 −2		source/credentials_provider_process.c
+44 −5		source/credentials_provider_profile.c
+37 −12		source/credentials_provider_sts.c
+1 −0		source/credentials_provider_sts_web_identity.c
+49 −7		source/credentials_utils.c
+17 −9		tests/CMakeLists.txt
+92 −17		tests/credentials_provider_ecs_tests.c
+75 −26		tests/credentials_provider_process_tests.c
+273 −108		tests/credentials_provider_sts_tests.c
+42 −0		tests/credentials_provider_sts_web_identity_tests.c
+58 −0		tests/credentials_utils_tests.c
+2 −2		.github/workflows/proof_ci_resources/config.yaml
+19 −4		cmake/AwsCheckHeaders.cmake
+1 −1		cmake/AwsPrebuildDependency.cmake
+7 −1		cmake/AwsSIMD.cmake
+1 −1		include/aws/common/array_list.inl
+8 −1		include/aws/common/math.inl
+212 −183		verification/cbmc/proofs/Makefile.common
+12 −3		.github/workflows/ci.yml
+4 −6		.github/workflows/clang-format.yml
+47 −0		format-check.py
+0 −24		format-check.sh
+1 −1		source/compression.c
+12 −3		.github/workflows/ci.yml
+4 −8		.github/workflows/clang-format.yml
+7 −4		.github/workflows/stale_issue.yml
+47 −0		format-check.py
+0 −24		format-check.sh
+4 −2		include/aws/event-stream/event_stream.h
+15 −5		.github/workflows/ci.yml
+3 −0		.github/workflows/stale_issue.yml
+15 −0		include/aws/http/connection_manager.h
+2 −0		include/aws/http/http.h
+14 −2		include/aws/http/private/h1_connection.h
+1 −1		integration-testing/http_client_test.py
+156 −47		source/connection_manager.c
+123 −23		source/h1_connection.c
+6 −0		source/http.c
+61 −59		source/websocket.c
+9 −0		tests/CMakeLists.txt
+0 −3		tests/test_connection.c
+131 −2		tests/test_connection_manager.c
+214 −0		tests/test_h1_client.c
+135 −36		tests/test_tls.c
+32 −0		tests/test_websocket_handler.c
+30 −3		.github/workflows/ci.yml
+0 −4		CMakeLists.txt
+6 −0		include/aws/io/private/tls_channel_handler_shared.h
+1 −0		include/aws/io/socket.h
+2 −2		source/channel.c
+123 −52		source/darwin/secure_transport_tls_channel_handler.c
+1 −0		source/posix/socket.c
+115 −53		source/s2n/s2n_tls_channel_handler.c
+1 −2		source/socket_channel_handler.c
+151 −86		source/windows/secure_channel_tls_handler.c
+2 −0		tests/CMakeLists.txt
+2 −2		tests/socket_test.c
+341 −115		tests/tls_handler_test.c
+16 −3		.github/workflows/ci.yml
+5 −5		include/aws/s3/private/s3_meta_request_impl.h
+11 −0		include/aws/s3/private/s3_util.h
+2 −0		include/aws/s3/s3.h
+62 −10		include/aws/s3/s3_client.h
+2 −0		source/s3.c
+20 −10		source/s3_auto_ranged_get.c
+4 −10		source/s3_auto_ranged_put.c
+1 −0		source/s3_buffer_pool.c
+5 −16		source/s3_copy_object.c
+21 −10		source/s3_default_meta_request.c
+187 −184		source/s3_endpoint_resolver/aws_s3_endpoint_resolver_partition.c
+186 −158		source/s3_meta_request.c
+56 −0		source/s3_util.c
+7 −0		tests/CMakeLists.txt
+1 −0		tests/s3_buffer_pool_tests.c
+256 −7		tests/s3_data_plane_tests.c
+65 −17		tests/s3_tester.c
+7 −0		tests/s3_tester.h
+105 −15		.github/workflows/ci.yml
+4 −6		.github/workflows/clang-format.yml
+60 −49		CMakeLists.txt
+29 −0		bin/benchmark/CMakeLists.txt
+133 −0		bin/benchmark/main.c
+4 −0		builder.json
+47 −0		format-check.py
+0 −24		format-check.sh
+42 −3		include/aws/checksums/crc.h
+51 −0		include/aws/checksums/private/crc64_priv.h
+28 −11		include/aws/checksums/private/crc_priv.h
+24 −0		include/aws/checksums/private/crc_util.h
+4 −4		source/arm/crc32c_arm.c
+208 −0		source/arm/crc64_arm.c
+35 −9		source/crc.c
+131 −0		source/crc64.c
+579 −0		source/crc64_sw.c
+0 −18		source/generic/crc32c_null.c
+3 −14		source/intel/asm/crc32c_sse42_asm.c
+245 −0		source/intel/intrin/crc32c_sse42_avx512.c
+130 −0		source/intel/intrin/crc64nvme_avx512.c
+147 −0		source/intel/intrin/crc64nvme_clmul.c
+0 −77		source/intel/visualc/visualc_crc32c_sse42.c
+3 −0		tests/CMakeLists.txt
+144 −0		tests/crc64_test.c
+129 −43		tests/crc_test.c