Performance optimization Updates (#531)

awscrt/auth.py

@@ -455,6 +455,9 @@ class AwsSigningAlgorithm(IntEnum):
     V4_ASYMMETRIC = 1
     """Signature Version 4 - Asymmetric"""
 
+    V4_S3EXPRESS = 2
+    """Signature Version 4 - S3 Express"""
+
 
 class AwsSignatureType(IntEnum):
     """Which sort of signature should be computed from the signable."""
@@ -595,11 +598,11 @@ class AwsSigningConfig(NativeResource):
     )
 
     def __init__(self,
-                 algorithm,
-                 signature_type,
-                 credentials_provider,
-                 region,
-                 service,
+                 algorithm=AwsSigningAlgorithm.V4,
+                 signature_type=AwsSignatureType.HTTP_REQUEST_HEADERS,
+                 credentials_provider=None,
+                 region="",
+                 service="",
                  date=None,
                  should_sign_header=None,
                  use_double_uri_encode=True,
@@ -612,7 +615,7 @@ def __init__(self,
 
         assert isinstance(algorithm, AwsSigningAlgorithm)
         assert isinstance(signature_type, AwsSignatureType)
-        assert isinstance(credentials_provider, AwsCredentialsProvider)
+        assert isinstance(credentials_provider, AwsCredentialsProvider) or credentials_provider is None
         assert isinstance(region, str)
         assert isinstance(service, str)
         assert callable(should_sign_header) or should_sign_header is None

awscrt/s3.py

@@ -10,11 +10,12 @@
 from awscrt import NativeResource
 from awscrt.http import HttpRequest
 from awscrt.io import ClientBootstrap, TlsConnectionOptions
+from awscrt.auth import AwsCredentials, AwsCredentialsProvider, AwsSignatureType, AwsSignedBodyHeaderType, AwsSignedBodyValue, AwsSigningAlgorithm, AwsSigningConfig
 from awscrt.auth import AwsCredentialsProvider, AwsSignatureType, AwsSignedBodyHeaderType, AwsSignedBodyValue, \
     AwsSigningAlgorithm, AwsSigningConfig
 import awscrt.exceptions
-from dataclasses import dataclass
 import threading
+from dataclasses import dataclass
 from typing import List, Optional, Tuple
 from enum import IntEnum
 
@@ -156,9 +157,22 @@ class S3Client(NativeResource):
             If this is :attr:`S3RequestTlsMode.DISABLED`:
                 No TLS options will be used, regardless of `tls_connection_options` value.
 
-        signing_config (Optional[AwsSigningConfig]):
-             Configuration for signing of the client. Use :func:`create_default_s3_signing_config()` to create the default config.
-             If None is provided, the request will not be signed.
+        signing_config (Optional[AwsSigningConfig]): Configuration for signing of the client.
+            Use :func:`create_default_s3_signing_config()` to create the default config.
+
+            If not set, a default config will be used with anonymous credentials and skip signing the request.
+
+            If set:
+                Credentials provider is required. Other configs are all optional, and will be default to what
+                    needs to sign the request for S3, only overrides when Non-zero/Not-empty is set.
+
+                S3 Client will derive the right config for signing process based on this.
+
+            Notes:
+
+                1. For SIGV4_S3EXPRESS, S3 client will use the credentials in the config to derive the S3 Express
+                    credentials that are used in the signing process.
+                2. Client may make modifications to signing config before passing it on to signer.
 
         credential_provider (Optional[AwsCredentialsProvider]): Deprecated, prefer `signing_config` instead.
             Credentials providers source the :class:`~awscrt.auth.AwsCredentials` needed to sign an authenticated AWS request.
@@ -181,6 +195,11 @@ class S3Client(NativeResource):
             You can also use `get_recommended_throughput_target_gbps()` to get recommended value for your system.
             10.0 Gbps by default (may change in future)
 
+        enable_s3express (Optional[bool]): To enable S3 Express support for the client.
+            The typical usage for a S3 Express request is to set this to true and let the request to be
+            signed with `AwsSigningAlgorithm.V4_S3EXPRESS`, either from the client-level `signing_config`
+            or the request-level override.
+
         memory_limit (Optional[int]): Memory limit, in bytes, of how much memory
             client can use for buffering data for requests.
             Default values scale with target throughput and are currently
@@ -201,6 +220,7 @@ def __init__(
             part_size=None,
             multipart_upload_threshold=None,
             throughput_target_gbps=None,
+            enable_s3express=False,
             memory_limit=None):
         assert isinstance(bootstrap, ClientBootstrap) or bootstrap is None
         assert isinstance(region, str)
@@ -213,6 +233,7 @@ def __init__(
             int) or isinstance(
             throughput_target_gbps,
             float) or throughput_target_gbps is None
+        assert isinstance(enable_s3express, bool) or enable_s3express is None
 
         if credential_provider and signing_config:
             raise ValueError("'credential_provider' has been deprecated in favor of 'signing_config'.  "
@@ -231,7 +252,11 @@ def on_shutdown():
         if not bootstrap:
             bootstrap = ClientBootstrap.get_or_create_static_default()
 
-        s3_client_core = _S3ClientCore(bootstrap, credential_provider, signing_config, tls_connection_options)
+        s3_client_core = _S3ClientCore(
+            bootstrap,
+            credential_provider,
+            signing_config,
+            tls_connection_options)
 
         # C layer uses 0 to indicate defaults
         if tls_mode is None:
@@ -256,6 +281,7 @@ def on_shutdown():
             part_size,
             multipart_upload_threshold,
             throughput_target_gbps,
+            enable_s3express,
             memory_limit,
             s3_client_core)
 
@@ -300,10 +326,21 @@ def make_request(
                 request's `body_stream` is ignored. This should give better
                 performance than reading a file from a stream.
 
-            signing_config (Optional[AwsSigningConfig]):
-                Configuration for signing of the request to override the configuration from client. Use :func:`create_default_s3_signing_config()` to create the default config.
+            signing_config (Optional[AwsSigningConfig]): Configuration for signing of the request to override the configuration from client.
+                Use :func:`create_default_s3_signing_config()` to create the default config.
+
                 If None is provided, the client configuration will be used.
 
+                If set:
+                    All fields are optional. The credentials will be resolve from client if not set.
+                    S3 Client will derive the right config for signing process based on this.
+
+                Notes:
+
+                    1. For SIGV4_S3EXPRESS, S3 client will use the credentials in the config to derive the S3 Express
+                        credentials that are used in the signing process.
+                    2. Client may make modifications to signing config before passing it on to signer.
+
             credential_provider (Optional[AwsCredentialsProvider]):  Deprecated, prefer `signing_config` instead.
                 Credentials providers source the :class:`~awscrt.auth.AwsCredentials` needed to sign an authenticated AWS request, for this request only.
                 If None is provided, the client configuration will be used.

source/auth.h

@@ -47,5 +47,6 @@ PyObject *aws_py_sign_request_aws(PyObject *self, PyObject *args);
 struct aws_credentials *aws_py_get_credentials(PyObject *credentials);
 struct aws_credentials_provider *aws_py_get_credentials_provider(PyObject *credentials_provider);
 struct aws_signing_config_aws *aws_py_get_signing_config(PyObject *signing_config);
+PyObject *aws_py_credentials_new_request_from_native(struct aws_credentials *credentials);
 
 #endif // AWS_CRT_PYTHON_AUTH_H

source/auth_credentials.c

@@ -23,6 +23,15 @@ static void s_credentials_capsule_destructor(PyObject *capsule) {
     aws_credentials_release(credentials);
 }
 
+PyObject *aws_py_credentials_new_request_from_native(struct aws_credentials *credentials) {
+    PyObject *capsule = PyCapsule_New(credentials, s_capsule_name_credentials, s_credentials_capsule_destructor);
+    if (!capsule) {
+        return NULL;
+    }
+    aws_credentials_acquire(credentials);
+    return capsule;
+}
+
 PyObject *aws_py_credentials_new(PyObject *self, PyObject *args) {
     (void)self;
 

source/auth_signing_config.c

@@ -72,7 +72,7 @@ PyObject *aws_py_signing_config_new(PyObject *self, PyObject *args) {
 
     int algorithm;
     int signature_type;
-    PyObject *py_credentials_provider;
+    PyObject *py_credentials_provider = NULL;
     struct aws_byte_cursor region;
     struct aws_byte_cursor service;
     PyObject *py_date;
@@ -136,13 +136,14 @@ PyObject *aws_py_signing_config_new(PyObject *self, PyObject *args) {
     binding->native.flags.omit_session_token = PyObject_IsTrue(py_omit_session_token);
 
     /* credentials_provider */
-    binding->native.credentials_provider = aws_py_get_credentials_provider(py_credentials_provider);
-    if (!binding->native.credentials_provider) {
-        goto error;
+    if (py_credentials_provider != Py_None) {
+        binding->native.credentials_provider = aws_py_get_credentials_provider(py_credentials_provider);
+        if (!binding->native.credentials_provider) {
+            goto error;
+        }
+        binding->py_credentials_provider = py_credentials_provider;
+        Py_INCREF(binding->py_credentials_provider);
     }
-    binding->py_credentials_provider = py_credentials_provider;
-    Py_INCREF(binding->py_credentials_provider);
-
     /* backup strings */
     if (aws_byte_buf_init_cache_and_update_cursors(
             &binding->string_storage,
@@ -220,8 +221,9 @@ PyObject *aws_py_signing_config_get_credentials_provider(PyObject *self, PyObjec
     if (!binding) {
         return NULL;
     }
-
-    Py_INCREF(binding->py_credentials_provider);
+    if (binding->py_credentials_provider) {
+        Py_INCREF(binding->py_credentials_provider);
+    }
     return binding->py_credentials_provider;
 }
 

source/s3_client.c

@@ -6,7 +6,11 @@
 
 #include "auth.h"
 #include "io.h"
+
+#include <aws/auth/credentials.h>
 #include <aws/common/cross_process_lock.h>
+#include <aws/common/hash_table.h>
+#include <aws/common/string.h>
 #include <aws/s3/s3_client.h>
 
 static const char *s_capsule_name_s3_client = "aws_s3_client";
@@ -158,6 +162,8 @@ PyObject *aws_py_s3_cross_process_lock_acquire(PyObject *self, PyObject *args) {
 
 PyObject *aws_py_s3_cross_process_lock_release(PyObject *self, PyObject *args) {
     (void)self;
+    (void)args;
+
     PyObject *lock_capsule; /* O */
 
     if (!PyArg_ParseTuple(args, "O", &lock_capsule)) {
@@ -249,11 +255,12 @@ PyObject *aws_py_s3_client_new(PyObject *self, PyObject *args) {
     uint64_t part_size;                  /* K */
     uint64_t multipart_upload_threshold; /* K */
     double throughput_target_gbps;       /* d */
+    int enable_s3express;                /* p */
     uint64_t mem_limit;                  /* K */
     PyObject *py_core;                   /* O */
     if (!PyArg_ParseTuple(
             args,
-            "OOOOOs#iKKdKO",
+            "OOOOOs#iKKdpKO",
             &bootstrap_py,
             &signing_config_py,
             &credential_provider_py,
@@ -265,6 +272,7 @@ PyObject *aws_py_s3_client_new(PyObject *self, PyObject *args) {
             &part_size,
             &multipart_upload_threshold,
             &throughput_target_gbps,
+            &enable_s3express,
             &mem_limit,
             &py_core)) {
         return NULL;
@@ -282,20 +290,6 @@ PyObject *aws_py_s3_client_new(PyObject *self, PyObject *args) {
             return NULL;
         }
     }
-    struct aws_signing_config_aws *signing_config = NULL;
-    if (signing_config_py != Py_None) {
-        signing_config = aws_py_get_signing_config(signing_config_py);
-        if (!signing_config) {
-            return NULL;
-        }
-    }
-    struct aws_signing_config_aws signing_config_from_credentials_provider;
-    AWS_ZERO_STRUCT(signing_config_from_credentials_provider);
-
-    if (credential_provider) {
-        aws_s3_init_default_signing_config(&signing_config_from_credentials_provider, region, credential_provider);
-        signing_config = &signing_config_from_credentials_provider;
-    }
 
     struct aws_tls_connection_options *tls_options = NULL;
     if (tls_options_py != Py_None) {
@@ -305,15 +299,33 @@ PyObject *aws_py_s3_client_new(PyObject *self, PyObject *args) {
         }
     }
 
-    struct s3_client_binding *s3_client = aws_mem_calloc(allocator, 1, sizeof(struct s3_client_binding));
-    if (!s3_client) {
-        return PyErr_AwsLastError();
+    struct aws_signing_config_aws default_signing_config;
+    AWS_ZERO_STRUCT(default_signing_config);
+
+    struct aws_signing_config_aws *signing_config = NULL;
+    struct aws_credentials *anonymous_credentials = NULL;
+    if (signing_config_py != Py_None) {
+        signing_config = aws_py_get_signing_config(signing_config_py);
+        if (!signing_config) {
+            return NULL;
+        }
+    } else if (credential_provider) {
+        aws_s3_init_default_signing_config(&default_signing_config, region, credential_provider);
+        signing_config = &default_signing_config;
+    } else {
+        /* Default to use a signing config with anonymous credentials */
+        anonymous_credentials = aws_credentials_new_anonymous(allocator);
+        default_signing_config.credentials = anonymous_credentials;
+        signing_config = &default_signing_config;
     }
 
+    struct s3_client_binding *s3_client = aws_mem_calloc(allocator, 1, sizeof(struct s3_client_binding));
+
     /* From hereon, we need to clean up if errors occur */
 
     PyObject *capsule = PyCapsule_New(s3_client, s_capsule_name_s3_client, s_s3_client_capsule_destructor);
     if (!capsule) {
+        aws_credentials_release(anonymous_credentials);
         aws_mem_release(allocator, s3_client);
         return NULL;
     }
@@ -336,17 +348,19 @@ PyObject *aws_py_s3_client_new(PyObject *self, PyObject *args) {
         .throughput_target_gbps = throughput_target_gbps,
         .shutdown_callback = s_s3_client_shutdown,
         .shutdown_callback_user_data = s3_client,
+        .enable_s3express = enable_s3express,
     };
 
     s3_client->native = aws_s3_client_new(allocator, &s3_config);
     if (s3_client->native == NULL) {
         PyErr_SetAwsLastError();
         goto error;
     }
-
+    aws_credentials_release(anonymous_credentials);
     return capsule;
 
 error:
+    aws_credentials_release(anonymous_credentials);
     Py_DECREF(capsule);
     return NULL;
 }