Add Python 3.13 CI (#587)

Co-authored-by: Michael Graeb <[email protected]>

continuous-delivery/build-wheels-manylinux2014-aarch64.sh

@@ -19,9 +19,13 @@ auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp310*.whl
 /opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel
 auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp311*.whl
 
-# Don't need to build wheels for Python 3.12 and later.
+# Don't need to build wheels for Python 3.12.
 # The 3.11 wheel uses the stable ABI, so it works with newer versions too.
 
+# We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions.
+/opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp313*.whl
+
 rm dist/*.whl
 cp -rv wheelhouse/* dist/
 

continuous-delivery/build-wheels-manylinux2014-x86_64.sh

@@ -19,9 +19,13 @@ auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp310*.whl
 /opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel
 auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp311*.whl
 
-# Don't need to build wheels for Python 3.12 and later.
+# Don't need to build wheels for Python 3.12.
 # The 3.11 wheel uses the stable ABI, so it works with newer versions too.
 
+# We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions.
+/opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp313*.whl
+
 rm dist/*.whl
 cp -rv wheelhouse/* dist/
 

continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh

@@ -19,9 +19,13 @@ auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp310*.whl
 /opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel
 auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp311*.whl
 
-# Don't need to build wheels for Python 3.12 and later.
+# Don't need to build wheels for Python 3.12.
 # The 3.11 wheel uses the stable ABI, so it works with newer versions too.
 
+# We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions.
+/opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp313*.whl
+
 rm dist/*.whl
 cp -rv wheelhouse/* dist/
 

continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh

@@ -19,9 +19,13 @@ auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp310*.whl
 /opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel
 auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp311*.whl
 
-# Don't need to build wheels for Python 3.12 and later.
+# Don't need to build wheels for Python 3.12.
 # The 3.11 wheel uses the stable ABI, so it works with newer versions too.
 
+# We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions.
+/opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp313*.whl
+
 rm dist/*.whl
 cp -rv wheelhouse/* dist/
 

continuous-delivery/build-wheels-osx.sh

@@ -11,7 +11,10 @@ set -ex
 /Library/Frameworks/Python.framework/Versions/3.10/bin/python3 setup.py sdist bdist_wheel
 /Library/Frameworks/Python.framework/Versions/3.11/bin/python3 setup.py sdist bdist_wheel
 
-# Don't need to build wheels for Python 3.12 and later.
+# Don't need to build wheels for Python 3.12.
 # The 3.11 wheel uses the stable ABI, so it works with newer versions too.
 
+# We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions.
+/Library/Frameworks/Python.framework/Versions/3.13/bin/python3 setup.py sdist bdist_wheel
+
 #now you just need to run twine (that's in a different script)

continuous-delivery/build-wheels-win32.bat

@@ -7,6 +7,12 @@
 "C:\Program Files (x86)\Python310-32\python.exe" setup.py sdist bdist_wheel || goto error
 "C:\Program Files (x86)\Python311-32\python.exe" setup.py sdist bdist_wheel || goto error
 
+:: Don't need to build wheels for Python 3.12.
+:: The 3.11 wheel uses the stable ABI, so it works with newer versions too.
+
+:: We are using the 3.13 stable ABI from 3.13 onwards because of deprecated functions.
+"C:\Program Files (x86)\Python313-32\python.exe" setup.py sdist bdist_wheel || goto error
+
 goto :EOF
 
 :error

continuous-delivery/build-wheels-win64.bat

@@ -6,6 +6,12 @@
 "C:\Program Files\Python310\python.exe" setup.py sdist bdist_wheel || goto error
 "C:\Program Files\Python311\python.exe" setup.py sdist bdist_wheel || goto error
 
+:: Don't need to build wheels for Python 3.12.
+:: The 3.11 wheel uses the stable ABI, so it works with newer versions too.
+
+:: We are using the 3.13 stable ABI from 3.13 onwards because of deprecated functions.
+"C:\Program Files\Python313\python.exe" setup.py sdist bdist_wheel || goto error
+
 goto :EOF
 
 :error

setup.py

@@ -403,7 +403,13 @@ def awscrt_ext():
                 else:
                     extra_link_args += ['-Wl,--fatal-warnings']
 
-    if sys.version_info >= (3, 11):
+    # prefer building with stable ABI, so a wheel can work with multiple major versions
+    if sys.version_info >= (3, 13):
+        # 3.13 deprecates PyWeakref_GetObject(), adds alternative
+        define_macros.append(('Py_LIMITED_API', '0x030D0000'))
+        py_limited_api = True
+    elif sys.version_info >= (3, 11):
+        # 3.11 is the first stable ABI that has everything we need
         define_macros.append(('Py_LIMITED_API', '0x030B0000'))
         py_limited_api = True
 

source/http_stream.c

@@ -203,7 +203,10 @@ static void s_on_stream_complete(struct aws_http_stream *native_stream, int erro
     }
 
     /* DECREF python self, we don't need to force it to stay alive any longer. */
-    Py_DECREF(PyWeakref_GetObject(stream->self_proxy));
+    PyObject *self = aws_py_weakref_get_ref(stream->self_proxy);
+    /* DECREF twice because `aws_py_weakref_get_ref` returns a strong reference */
+    Py_XDECREF(self);
+    Py_XDECREF(self);
 
     PyGILState_Release(state);
     /*************** GIL RELEASE ***************/

source/module.c

@@ -516,6 +516,39 @@ PyObject *aws_py_memory_view_from_byte_buffer(struct aws_byte_buf *buf) {
     return PyMemoryView_FromMemory(mem_start, mem_size, PyBUF_WRITE);
 }
 
+PyObject *aws_py_weakref_get_ref(PyObject *ref) {
+    /* If Python >= 3.13 */
+#if PY_VERSION_HEX >= 0x030D0000
+    /* Use PyWeakref_GetRef() (new in Python 3.13), which gets you:
+     * a new strong reference,
+     * or NULL because ref is dead,
+     * or -1 because you called it wrong */
+    PyObject *obj = NULL;
+    if (PyWeakref_GetRef(ref, &obj) == -1) {
+        PyErr_WriteUnraisable(PyErr_Occurred());
+        AWS_ASSERT(0 && "expected a weakref");
+    }
+    return obj;
+
+#else
+    /* Use PyWeakref_GetObject() (deprecated as of Python 3.13), which gets you:
+     * a borrowed reference,
+     * or Py_None because ref is dead,
+     * or NULL because you called it wrong */
+    PyObject *obj = PyWeakref_GetObject(ref); /* borrowed reference */
+    if (obj == NULL) {
+        PyErr_WriteUnraisable(PyErr_Occurred());
+        AWS_ASSERT(0 && "expected a weakref");
+    } else if (obj == Py_None) {
+        obj = NULL;
+    } else {
+        /* Be like PyWeakref_GetRef() and make it new strong reference */
+        Py_INCREF(obj);
+    }
+    return obj;
+#endif
+}
+
 int aws_py_gilstate_ensure(PyGILState_STATE *out_state) {
     if (AWS_LIKELY(Py_IsInitialized())) {
         *out_state = PyGILState_Ensure();

source/module.h

@@ -106,6 +106,27 @@ PyObject *aws_py_get_error_message(PyObject *self, PyObject *args);
 /* Create a write-only memoryview from the remaining free space in an aws_byte_buf */
 PyObject *aws_py_memory_view_from_byte_buffer(struct aws_byte_buf *buf);
 
+/* Python 3.13+ changed the function to get a reference from WeakRef. This function is an abstraction over two different
+ * APIs since we support Python versions before 3.13. Returns a strong reference if non-null, which you must release. */
+
+/**
+ * Given a weak reference, returns a NEW strong reference to the referenced object,
+ * or NULL if the reference is dead (this function NEVER raises a python exception or AWS Error).
+ *
+ * You MUST NOT call this if ref came from a user, or ref is NULL.
+ *
+ * This is a simplified version of PyWeakref_GetRef() / PyWeakref_GetObject().
+ * Simpler because:
+ * - Python 3.13 adds PyWeakref_GetRef() and deprecates PyWeakref_GetObject().
+ *   This function calls the appropriate one.
+ *
+ * - This functions has 2 outcomes instead of 3:
+ *   The 3rd being a Python exception for calling it incorrectly.
+ *   If that happens, this function calls PyErr_WriteUnraisable() to clear the exception,
+ *   which is what you would have done anyway.
+ */
+PyObject *aws_py_weakref_get_ref(PyObject *ref);
+
 /* Allocator that calls into PyObject_[Malloc|Free|Realloc] */
 struct aws_allocator *aws_py_get_allocator(void);
 

source/mqtt_client_connection.c

@@ -140,15 +140,16 @@ static void s_on_connection_success(
         return; /* Python has shut down. Nothing matters anymore, but don't crash */
     }
 
-    PyObject *self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */
-    if (self != Py_None) {
+    PyObject *self = aws_py_weakref_get_ref(py_connection->self_proxy); /* new reference */
+    if (self != NULL) {
         PyObject *success_result =
             PyObject_CallMethod(self, "_on_connection_success", "(iN)", return_code, PyBool_FromLong(session_present));
         if (success_result) {
             Py_DECREF(success_result);
         } else {
             PyErr_WriteUnraisable(PyErr_Occurred());
         }
+        Py_DECREF(self);
     }
 
     PyGILState_Release(state);
@@ -167,14 +168,15 @@ static void s_on_connection_failure(struct aws_mqtt_client_connection *connectio
         return; /* Python has shut down. Nothing matters anymore, but don't crash */
     }
 
-    PyObject *self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */
-    if (self != Py_None) {
+    PyObject *self = aws_py_weakref_get_ref(py_connection->self_proxy); /* new reference */
+    if (self != NULL) {
         PyObject *success_result = PyObject_CallMethod(self, "_on_connection_failure", "(i)", error_code);
         if (success_result) {
             Py_DECREF(success_result);
         } else {
             PyErr_WriteUnraisable(PyErr_Occurred());
         }
+        Py_DECREF(self);
     }
 
     PyGILState_Release(state);
@@ -194,14 +196,15 @@ static void s_on_connection_interrupted(struct aws_mqtt_client_connection *conne
     }
 
     /* Ensure that python class is still alive */
-    PyObject *self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */
-    if (self != Py_None) {
+    PyObject *self = aws_py_weakref_get_ref(py_connection->self_proxy); /* new reference */
+    if (self != NULL) {
         PyObject *result = PyObject_CallMethod(self, "_on_connection_interrupted", "(i)", error_code);
         if (result) {
             Py_DECREF(result);
         } else {
             PyErr_WriteUnraisable(PyErr_Occurred());
         }
+        Py_DECREF(self);
     }
 
     PyGILState_Release(state);
@@ -227,15 +230,16 @@ static void s_on_connection_resumed(
     }
 
     /* Ensure that python class is still alive */
-    PyObject *self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */
-    if (self != Py_None) {
+    PyObject *self = aws_py_weakref_get_ref(py_connection->self_proxy); /* new reference */
+    if (self != NULL) {
         PyObject *result =
             PyObject_CallMethod(self, "_on_connection_resumed", "(iN)", return_code, PyBool_FromLong(session_present));
         if (result) {
             Py_DECREF(result);
         } else {
             PyErr_WriteUnraisable(PyErr_Occurred());
         }
+        Py_DECREF(self);
     }
 
     PyGILState_Release(state);
@@ -258,14 +262,15 @@ static void s_on_connection_closed(
 
     struct mqtt_connection_binding *py_connection = userdata;
     /* Ensure that python class is still alive */
-    PyObject *self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */
-    if (self != Py_None) {
+    PyObject *self = aws_py_weakref_get_ref(py_connection->self_proxy); /* new reference */
+    if (self != NULL) {
         PyObject *result = PyObject_CallMethod(self, "_on_connection_closed", "()");
         if (result) {
             Py_DECREF(result);
         } else {
             PyErr_WriteUnraisable(PyErr_Occurred());
         }
+        Py_DECREF(self);
     }
 
     PyGILState_Release(state);
@@ -535,8 +540,9 @@ static void s_ws_handshake_transform(
     }
 
     /* Ensure python mqtt connection object is still alive */
-    PyObject *connection_py = PyWeakref_GetObject(connection_binding->self_proxy); /* borrowed reference */
-    if (connection_py == Py_None) {
+
+    PyObject *connection_py = aws_py_weakref_get_ref(connection_binding->self_proxy); /* new reference */
+    if (connection_py == NULL) {
         aws_raise_error(AWS_ERROR_INVALID_STATE);
         goto done;
     }
@@ -593,6 +599,7 @@ static void s_ws_handshake_transform(
 done:;
     /* Save off error code, so it doesn't got stomped before we pass it to callback*/
     int error_code = aws_last_error();
+    Py_XDECREF(connection_py);
 
     if (ws_transform_capsule) {
         Py_DECREF(ws_transform_capsule);

test/test_mqtt5_credentials.py

@@ -197,40 +197,6 @@ def test_mqtt5_ws_cred_static(self):
             input_role_secret_access_key,
             input_role_session_token
         )
-        credentials = auth.AwsCredentialsProvider.new_default_chain()
-
-        def sign_function(transform_args, **kwargs):
-            signing_config = auth.AwsSigningConfig(
-                algorithm=auth.AwsSigningAlgorithm.V4,
-                signature_type=auth.AwsSignatureType.HTTP_REQUEST_QUERY_PARAMS,
-                credentials_provider=credentials,
-                region=input_region,
-                service="iotdevicegateway",
-                omit_session_token=True
-            )
-            signing_future = auth.aws_sign_request(
-                http_request=transform_args.http_request,
-                signing_config=signing_config)
-            signing_future.add_done_callback(lambda x: transform_args.set_done(x.exception()))
-        client_options.websocket_handshake_transform = sign_function
-        client_options.tls_ctx = io.ClientTlsContext(io.TlsContextOptions())
-
-        callbacks = Mqtt5TestCallbacks()
-        client = self._create_client(client_options=client_options, callbacks=callbacks)
-        client.start()
-        callbacks.future_connection_success.result(TIMEOUT)
-        client.stop()
-        callbacks.future_stopped.result(TIMEOUT)
-
-    def test_mqtt5_ws_cred_default(self):
-        input_host_name = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_HOST")
-        input_region = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_REGION")
-
-        client_options = mqtt5.ClientOptions(
-            host_name=input_host_name,
-            port=443
-        )
-        credentials = auth.AwsCredentialsProvider.new_default_chain()
 
         def sign_function(transform_args, **kwargs):
             signing_config = auth.AwsSigningConfig(
@@ -380,6 +346,12 @@ def sign_function(transform_args, **kwargs):
         callbacks.future_stopped.result(TIMEOUT)
 
     def test_mqtt5_ws_cred_environment(self):
+        self._test_mqtt5_ws_cred_environment(use_default_chain=False)
+
+    def test_mqtt5_ws_cred_default_chain(self):
+        self._test_mqtt5_ws_cred_environment(use_default_chain=True)
+
+    def _test_mqtt5_ws_cred_environment(self, use_default_chain):
         input_host_name = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_HOST")
         input_access_key = _get_env_variable("AWS_TEST_MQTT5_ROLE_CREDENTIAL_ACCESS_KEY")
         input_secret_access_key = _get_env_variable("AWS_TEST_MQTT5_ROLE_CREDENTIAL_SECRET_ACCESS_KEY")
@@ -399,7 +371,10 @@ def test_mqtt5_ws_cred_environment(self):
         os.environ["AWS_SECRET_ACCESS_KEY"] = input_secret_access_key
         os.environ["AWS_SESSION_TOKEN"] = input_session_token
         # This should load the environment variables we just set
-        credentials = auth.AwsCredentialsProvider.new_environment()
+        if use_default_chain:
+            credentials = auth.AwsCredentialsProvider.new_default_chain()
+        else:
+            credentials = auth.AwsCredentialsProvider.new_environment()
 
         def sign_function(transform_args, **kwargs):
             signing_config = auth.AwsSigningConfig(