apply http stream activation refactor. (#136)

apply http stream activation refactor.
Numerous bug fixes. Reworked Mac OS trust store evaluation.

.github/workflows/ci.yml

@@ -7,7 +7,7 @@ on:
       - '!master'
 
 env:
-  BUILDER_VERSION: v0.5.3
+  BUILDER_VERSION: v0.6.0
   BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net
   PACKAGE_NAME: aws-crt-python
   LINUX_BASE_IMAGE: ubuntu-16-x64
@@ -79,7 +79,7 @@ jobs:
     steps:
     - name: Build ${{ env.PACKAGE_NAME }} + consumers
       run: |
-        python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')"
+        python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/releases/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')"
         python builder.pyz build -p ${{ env.PACKAGE_NAME }}
 
   windows-vc14:
@@ -96,15 +96,15 @@ jobs:
         spectre: true
     - name: Build ${{ env.PACKAGE_NAME }} + consumers
       run: |
-        python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')"
-        python builder.pyz -p ${{ env.PACKAGE_NAME }} build
+        python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/releases/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')"
+        python builder.pyz build -p ${{ env.PACKAGE_NAME }}
 
   osx:
     runs-on: macos-latest
     steps:
     - name: Build ${{ env.PACKAGE_NAME }} + consumers
       run: |
-        python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
+        python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/releases/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
         chmod a+x builder
         ./builder build -p ${{ env.PACKAGE_NAME }}
 

.github/workflows/lint.yml

@@ -27,5 +27,6 @@ jobs:
     - name: Build and Test
       run: |
         python3 -m pip install --upgrade setuptools
+        python3 -m pip install --upgrade wheel
         python3 -m pip install --upgrade autopep8
-        python3 -m autopep8 --exit-code --diff --recursive awscrt test .builder setup.py
+        python3 -m autopep8 --exit-code --diff --recursive awscrt test .builder setup.py

.gitignore

@@ -525,9 +525,9 @@ ASALocalRun/
 # End of https://www.gitignore.io/api/git,c++,cmake,python,visualstudio,visualstudiocode
 
 # credentials
-*.key
+.key
 *.pem
-*.crt
+.crt
 
 # deps from build-deps.sh
 deps/

awscrt/http.py

@@ -174,12 +174,15 @@ def __init__(self, connection, request, on_response=None, on_body=None):
         self._on_response_cb = on_response
         self._response_status_code = None
 
-        _awscrt.http_client_stream_new(self, connection, request)
+        self._binding = _awscrt.http_client_stream_new(self, connection, request)
 
     @property
     def response_status_code(self):
         return self._response_status_code
 
+    def activate(self):
+        _awscrt.http_client_stream_activate(self)
+
     def _on_response(self, status_code, name_value_pairs):
         self._response_status_code = status_code
 

elasticurl.py

@@ -240,6 +240,7 @@ def response_received_cb(http_stream, status_code, headers, **kwargs):
 
 # make the request
 stream = connection.request(request, response_received_cb, on_incoming_body)
+stream.activate()
 
 # wait until the full response is finished
 stream.completion_future.result()

setup.py

@@ -244,7 +244,7 @@ def awscrt_ext():
 
 setuptools.setup(
     name="awscrt",
-    version="0.5.11",
+    version="0.5.12",
     author="Amazon Web Services, Inc",
     author_email="[email protected]",
     description="A common runtime for AWS Python projects",

source/http.h

@@ -48,6 +48,8 @@ PyObject *aws_py_http_client_connection_new(PyObject *self, PyObject *args);
 
 PyObject *aws_py_http_client_stream_new(PyObject *self, PyObject *args);
 
+PyObject *aws_py_http_client_stream_activate(PyObject *self, PyObject *args);
+
 /* Create capsule around new request-style aws_http_message struct */
 PyObject *aws_py_http_message_new_request(PyObject *self, PyObject *args);
 

source/http_stream.c

@@ -27,7 +27,7 @@ struct http_stream_binding {
 
     /* Weak reference proxy to python self.
      * NOTE: The python self is forced to stay alive until on_complete fires.
-     * We do this by INCREFing when setup is successful, and DECREFing when on_complete fires. */
+     * We do this by INCREFing when activate() is called, and DECREFing when on_complete fires. */
     PyObject *self_proxy;
 
     /* Buffer up headers as they come in via repeated on_headers callacks.
@@ -100,9 +100,6 @@ static int s_on_incoming_header_block_done(
         return AWS_OP_ERR; /* Python has shut down. Nothing matters anymore, but don't crash */
     }
 
-    /* Set this just in case callback fires before aws_http_connection_make_request() has even returned */
-    stream->native = native_stream;
-
     /* Build up a list of (name,value) tuples,
      * extracting values from buffer of [name,value,name,value,...] null-terminated strings */
     PyObject *header_list = PyList_New(num_headers);
@@ -204,9 +201,6 @@ static void s_on_stream_complete(struct aws_http_stream *native_stream, int erro
         return; /* Python has shut down. Nothing matters anymore, but don't crash */
     }
 
-    /* Set this just in case callback fires before aws_http_connection_make_request() has even returned */
-    stream->native = native_stream;
-
     PyObject *result = PyObject_CallMethod(stream->self_proxy, "_on_complete", "(i)", error_code);
     if (result) {
         Py_DECREF(result);
@@ -289,12 +283,6 @@ PyObject *aws_py_http_client_stream_new(PyObject *self, PyObject *args) {
         goto error;
     }
 
-    /* NOTE: Callbacks might start firing before aws_http_connection_make_request() can even return.
-     * Therefore, we set `HttpClientStream._binding = capsule` now, instead of returning capsule to caller */
-    if (PyObject_SetAttrString(py_stream, "_binding", capsule) == -1) {
-        goto error;
-    }
-
     struct aws_http_make_request_options request_options = {
         .self_size = sizeof(request_options),
         .request = native_request,
@@ -311,13 +299,7 @@ PyObject *aws_py_http_client_stream_new(PyObject *self, PyObject *args) {
         goto error;
     }
 
-    /* From hereon, nothing will fail */
-
-    /* Force python self to stay alive until on_complete callback */
-    Py_INCREF(py_stream);
-
-    Py_DECREF(capsule);
-    Py_RETURN_NONE;
+    return capsule;
 
 error:
     if (capsule) {
@@ -327,3 +309,26 @@ PyObject *aws_py_http_client_stream_new(PyObject *self, PyObject *args) {
     }
     return NULL;
 }
+
+PyObject *aws_py_http_client_stream_activate(PyObject *self, PyObject *args) {
+    (void)self;
+
+    PyObject *py_stream = NULL;
+    if (!PyArg_ParseTuple(args, "O", &py_stream)) {
+        return NULL;
+    }
+
+    struct aws_http_stream *native_stream = aws_py_get_http_stream(py_stream);
+    if (!native_stream) {
+        return NULL;
+    }
+
+    if (aws_http_stream_activate(native_stream)) {
+        return PyErr_AwsLastError();
+    }
+
+    /* Force python self to stay alive until on_complete callback */
+    Py_INCREF(py_stream);
+
+    Py_RETURN_NONE;
+}

source/module.c

@@ -528,6 +528,7 @@ static PyMethodDef s_module_methods[] = {
     AWS_PY_METHOD_DEF(http_connection_is_open, METH_VARARGS),
     AWS_PY_METHOD_DEF(http_client_connection_new, METH_VARARGS),
     AWS_PY_METHOD_DEF(http_client_stream_new, METH_VARARGS),
+    AWS_PY_METHOD_DEF(http_client_stream_activate, METH_VARARGS),
     AWS_PY_METHOD_DEF(http_message_new_request, METH_VARARGS),
     AWS_PY_METHOD_DEF(http_message_get_request_method, METH_VARARGS),
     AWS_PY_METHOD_DEF(http_message_set_request_method, METH_VARARGS),

test/appexit_http.py

@@ -108,6 +108,7 @@ def on_incoming_body(http_stream, chunk):
         receiving_body_event.set()
 
     http_stream = http_connection.request(request, on_body=on_incoming_body)
+    http_stream.activate()
     set_stage(Stage.HttpStreamStart)
 
     # HttpStreamReceivingBodyMainThread: Exit from main thread while receiving body

test/resources/ca.crt

@@ -0,0 +1,24 @@
+-----BEGIN CERTIFICATE-----
+MIIEADCCAugCCQCtEgLxN71/TDANBgkqhkiG9w0BAQsFADCBwTELMAkGA1UEBhMC
+VVMxEzARBgNVBAgMCldhc2hpbmd0b24xEDAOBgNVBAcMB1NlYXR0bGUxITAfBgNV
+BAoMGEFtYXpvbiBXZWIgU2VydmljZXMsIEluYzEbMBkGA1UECwwSQXdzIGNvbW1v
+biBydW50aW1lMRkwFwYDVQQDDBBwa2kuaXMudGhlLndvcnN0MTAwLgYJKoZIhvcN
+AQkBFiFhd3Mtc2RrLWNvbW1vbi1ydW50aW1lQGFtYXpvbi5jb20wHhcNMjAwMzI4
+MDMwNTI4WhcNNDcwODE0MDMwNTI4WjCBwTELMAkGA1UEBhMCVVMxEzARBgNVBAgM
+Cldhc2hpbmd0b24xEDAOBgNVBAcMB1NlYXR0bGUxITAfBgNVBAoMGEFtYXpvbiBX
+ZWIgU2VydmljZXMsIEluYzEbMBkGA1UECwwSQXdzIGNvbW1vbiBydW50aW1lMRkw
+FwYDVQQDDBBwa2kuaXMudGhlLndvcnN0MTAwLgYJKoZIhvcNAQkBFiFhd3Mtc2Rr
+LWNvbW1vbi1ydW50aW1lQGFtYXpvbi5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IB
+DwAwggEKAoIBAQDCUFDcwnUkyK/7M2RFXmWGryCfUM/Y4sPV5dm8Q52wZcPATaAO
+Ucnh03qhPbxHyiF56EYNxF6tIXqMifO6CyiC1Dcvkq4yYVAYBH2GpwpscBraVrNX
+KUeBZVv3tAFIFWMgGSz4SYB+6pdTEHxFA3iZjH4WIMgMMAZWiz7ktqz3b+OqOSHT
+i89rShb0QVrAdHzSeKXwyRA5X4vTMLQEFwNU/6D57+1IdSlo0svdrF70GYuxpR7G
+2VOIZqbVSyoL0dkuFCh1XjbuIyvNqshWXV644BtEwoP5P36tSONUiLbhLaR++pO0
+RB2WgniJSBGUe0gSfl9djaaAg7KdbfIoIosRAgMBAAEwDQYJKoZIhvcNAQELBQAD
+ggEBAKdg2Z2DL2sow8ckf90uqrwNpuCKlydg1XGS2hBSWyhiFd3Ztj9pRKdpsjA8
+mF9x4U+WwmekzV6CZKXPlO5qlCvU1h7dFVRHZtG4O1bIRQNcgycoiL7Q9Sp+crP1
+Y8pF1vjCuoMlGFSSMXazRVXcJ/FVeBD4rS8AmyR6B4W8fykVQAYNcH4gXwQ0WP7O
+bjJrI1kTB+3JQU1KHHDTi3/gMvCUanzDc8HJ0CEpch8AY9S4suAYJmM6HpWzczoy
+jUH/yQT0ehur9wRy566BwUdTVfi+2cQQfYNPqs9iJdYZ7loSBDRH7vO7ivE4Y/Zz
+ilbJgpP/vnoxwEVSYD0W+1ctaEg=
+-----END CERTIFICATE-----

test/resources/crt.unittests.crt

@@ -0,0 +1,24 @@
+-----BEGIN CERTIFICATE-----
+MIID+TCCAuECCQDufZzOyUBvoTANBgkqhkiG9w0BAQUFADCBwTELMAkGA1UEBhMC
+VVMxEzARBgNVBAgMCldhc2hpbmd0b24xEDAOBgNVBAcMB1NlYXR0bGUxITAfBgNV
+BAoMGEFtYXpvbiBXZWIgU2VydmljZXMsIEluYzEbMBkGA1UECwwSQXdzIGNvbW1v
+biBydW50aW1lMRkwFwYDVQQDDBBwa2kuaXMudGhlLndvcnN0MTAwLgYJKoZIhvcN
+AQkBFiFhd3Mtc2RrLWNvbW1vbi1ydW50aW1lQGFtYXpvbi5jb20wHhcNMjAwMzI4
+MDMwODU3WhcNMzAwMzI2MDMwODU3WjCBujELMAkGA1UEBhMCVVMxEzARBgNVBAgM
+Cldhc2hpbmd0b24xEDAOBgNVBAcMB1NlYXR0bGUxITAfBgNVBAoMGEFtYXpvbiBX
+ZWIgU2VydmljZXMsIEluYzEbMBkGA1UECwwSQXdzIGNvbW1vbiBydW50aW1lMRIw
+EAYDVQQDDAlsb2NhbGhvc3QxMDAuBgkqhkiG9w0BCQEWIWF3cy1zZGstY29tbW9u
+LXJ1bnRpbWVAYW1hem9uLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC
+ggEBAJ/WE+6CF8V/UJ4uL79LvnEfXWlfUA/KMiopiDJOKyn4dP4McKSqzZsuaiBK
+XJcT1O25MRIdyacHht0+c4+zlLChqhiGs5Ld1M+AHr5bA55dBojUfbPQJIN36TiH
+SzyhvENn4XGK140HynhlW2omnnNoUEqNi45qJTgPy4+IqfH5e8zpiI9bs7GXcewz
+PCNiX0mxeNq0wlyrCjfY35XY+fKCqo80kdJ6e12RM7+a2JO1BAiqhs0+TngNfXSP
+eo5jdeC8GhREssR7/JVyB2yWDx5VDmdWAxDwWYy51t9MlsSLWXB2kA51NC8ER6YX
+aFMwiU48G2xveaXp/0lVDR0ohD0CAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAOljE
+Z5yomuomyQSzsmGc3gbGsSKY6waSTpns6A6teqEqi99Xcvxk6rpUjkqXavedpaEu
+5CiLJQXbXLiqoz2aVKj4eN94R/PGXYHWZ3l8MXGpUssUyee6jveHLNclliHNfgyX
+hecS7EAeGks88bY33RF5zh80TTurWs7Eaddw8CoxZmLPUFq+2SeMirxS2HVgjytV
+ex/MS3XkWEvBcTpzZNYSP2H4nHlpzRtB/scRBf3rIcEuLiLyaRuvSZAkxrdFRnSa
+tj1mArGWt9CRvRANL18TzhHTIxZkQ4gnxTXJHAWsqexQqucWSOblm9ouFkooetoO
+TI4h04F8jszodmLKUA==
+-----END CERTIFICATE-----

test/resources/crt.unittests.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpQIBAAKCAQEAn9YT7oIXxX9Qni4vv0u+cR9daV9QD8oyKimIMk4rKfh0/gxw
+pKrNmy5qIEpclxPU7bkxEh3JpweG3T5zj7OUsKGqGIazkt3Uz4AevlsDnl0GiNR9
+s9Akg3fpOIdLPKG8Q2fhcYrXjQfKeGVbaiaec2hQSo2LjmolOA/Lj4ip8fl7zOmI
+j1uzsZdx7DM8I2JfSbF42rTCXKsKN9jfldj58oKqjzSR0np7XZEzv5rYk7UECKqG
+zT5OeA19dI96jmN14LwaFESyxHv8lXIHbJYPHlUOZ1YDEPBZjLnW30yWxItZcHaQ
+DnU0LwRHphdoUzCJTjwbbG95pen/SVUNHSiEPQIDAQABAoIBAQCC3+mGG+OXBT5f
+pvXVrxGK0uHLzA1kQG1+9G7esf7J6PTyqAD79BdSf1jKg9j4o3ZwCHlTmixppdIa
+Cm7ZNXQEhPsgBfu3wAfmt8N0nlWCMC2PkF41A7NCKsc+UBmc1dmvX7nLHKKOidiR
+aHzHmEOZv8ouoWOqAuPi2ZwWNRLiW9gUw4xuCa13u8CJstrB83KqLN9qycfzzI0z
+i1VjoxicQ5rvp/btTVAHNIfaNLdSP8pydO0Z9GuYn0HbtlTpPhTV+jE1iFwG9Ft1
+03kTv+6EtlbUeJohjYRtSXw7DkRGfzOuSFsZgLvn0bbbUCz3IfgV/5/eQrOMR4zQ
+fxGDWc8BAoGBANO8AS0p8F3ooELhHEojo70SJHtGjfMnzZ/d3NYesSGnJ94I6c/t
+7YkgWX9eYhgefCdHBou+OITMXenHMto9Oy5bVs85WRkbHryTzqMpgGLY1UdVH2IF
+aSZXOUrz+WhXJq2VwIs/G2Y+0+xRYLaw1Sg+CsLY2tZnG2kFUXrSVuDhAoGBAMFA
+fcuQhDE513ogK0aC2W6U8TaZOzlauNRmyUhp5o2jMxK7dfnsFkQ0g2slxIrNmNMN
+RGFE9hg6WfOqprXwfZ2SqZ7u/M4c4yY3NShnfTQHua0CoDMZ3FYWeX0F2NtWwSTP
+hN2O4bZPw9KGNV8AS5BF9XTd4A/RzZR2UNOXGqLdAoGAGPlJ4jvaIBl6HalTnuMJ
+QxnKt8egopeFR1/x1keUePWwXB4NBft4Uv4/0CgfxdyCpFdaTE1JeVBpm1NlihxC
+yLr5SEPpYG60d2Wvr8sLW51buqgtpXAh8OXE2jRgrI/q/fp11XbCEpNCh9MpZyMA
+YVjACC9v0G4k64Q3EheRK8ECgYEAgOF8w5TmxFmZ/eWUVtsPHTORxuLcX3zEhUrj
+dDrEOrObbSO+I2Yy11vaRUwTzqodAKv5bZFrOm837ExkEDSJ4duc6dTinqcfSi5J
+cl/KXTfFub9ljY7ypiso7JtIq8co2l7XcldG6qo5o55D2867lkVB1KXgN0ypeRiL
+AQHmDrUCgYEAvKFJYxA8j6we3xzKSKb627nodAjW4tX4ohJSaEwICm7YNUkC1Bfa
+SD9O2fBm2mtLoX1h7vicmaOnZ8V4wf2JGYp2DgPBsZK8sxhi9xzGP7iYMpTMq2TM
+KO9KBGl1MvZNdq7edqlSoqFX1XlPk21o+iHC3lm2w1njUWo4O/avzc8=
+-----END RSA PRIVATE KEY-----