Skip to content

Commit

Permalink
Add checksum settings for S3 (#512)
Browse files Browse the repository at this point in the history
  • Loading branch information
graebm authored Oct 18, 2023
1 parent 965cf27 commit 2ea0145
Show file tree
Hide file tree
Showing 7 changed files with 200 additions and 66 deletions.
112 changes: 93 additions & 19 deletions awscrt/s3.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,9 @@
from awscrt.io import ClientBootstrap, TlsConnectionOptions
from awscrt.auth import AwsCredentialsProvider, AwsSignatureType, AwsSignedBodyHeaderType, AwsSignedBodyValue, AwsSigningAlgorithm, AwsSigningConfig
import awscrt.exceptions
from dataclasses import dataclass
import threading
from typing import Optional
from enum import IntEnum


Expand Down Expand Up @@ -50,6 +52,60 @@ class S3RequestTlsMode(IntEnum):
"""


class S3ChecksumAlgorithm(IntEnum):
"""
Checksum algorithm used to verify object integrity.
https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
"""

CRC32C = 1
"""CRC32C"""

CRC32 = 2
"""CRC32"""

SHA1 = 3
"""SHA-1"""

SHA256 = 4
"""SHA-256"""


class S3ChecksumLocation(IntEnum):
"""Where to put the checksum."""

HEADER = 1
"""
Add checksum as a request header field.
The checksum is calculated before any part of the request is sent to the server.
"""

TRAILER = 2
"""
Add checksum as a request trailer field.
The checksum is calculated as the body is streamed to the server, then
added as a trailer field. This may be more efficient than HEADER, but
can only be used with "streaming" requests that support it.
"""


@dataclass
class S3ChecksumConfig:
"""Configures how the S3Client calculates and verifies checksums."""

algorithm: Optional[S3ChecksumAlgorithm] = None
"""
If set, the S3Client will calculate a checksum using this algorithm
and add it to the request. If you set this, you must also set `location`.
"""

location: Optional[S3ChecksumLocation] = None
"""Where to put the request checksum."""

validate_response: bool = False
"""Whether to retrieve and validate response checksums."""


class S3Client(NativeResource):
"""S3 client
Expand Down Expand Up @@ -153,12 +209,13 @@ def on_shutdown():
def make_request(
self,
*,
request,
type,
signing_config=None,
credential_provider=None,
request,
recv_filepath=None,
send_filepath=None,
signing_config=None,
credential_provider=None,
checksum_config=None,
on_headers=None,
on_body=None,
on_done=None,
Expand All @@ -168,19 +225,11 @@ def make_request(
requests under the hood for acceleration.
Keyword Args:
request (HttpRequest): The overall outgoing API request for S3 operation.
If the request body is a file, set send_filepath for better performance.
type (S3RequestType): The type of S3 request passed in,
:attr:`~S3RequestType.GET_OBJECT`/:attr:`~S3RequestType.PUT_OBJECT` can be accelerated
signing_config (Optional[AwsSigningConfig]):
Configuration for signing of the request to override the configuration from client. Use :func:`create_default_s3_signing_config()` to create the default config.
If None is provided, the client configuration will be used.
credential_provider (Optional[AwsCredentialsProvider]): Deprecated, prefer `signing_config` instead.
Credentials providers source the :class:`~awscrt.auth.AwsCredentials` needed to sign an authenticated AWS request, for this request only.
If None is provided, the client configuration will be used.
request (HttpRequest): The overall outgoing API request for S3 operation.
If the request body is a file, set send_filepath for better performance.
recv_filepath (Optional[str]): Optional file path. If set, the
response body is written directly to a file and the
Expand All @@ -192,6 +241,16 @@ def make_request(
request's `body_stream` is ignored. This should give better
performance than reading a file from a stream.
signing_config (Optional[AwsSigningConfig]):
Configuration for signing of the request to override the configuration from client. Use :func:`create_default_s3_signing_config()` to create the default config.
If None is provided, the client configuration will be used.
credential_provider (Optional[AwsCredentialsProvider]): Deprecated, prefer `signing_config` instead.
Credentials providers source the :class:`~awscrt.auth.AwsCredentials` needed to sign an authenticated AWS request, for this request only.
If None is provided, the client configuration will be used.
checksum_config (Optional[S3ChecksumConfig]): Optional checksum settings.
on_headers: Optional callback invoked as the response received, and even the API request
has been split into multiple parts, this callback will only be invoked once as
it's just making one API request to S3.
Expand Down Expand Up @@ -244,12 +303,13 @@ def make_request(
"""
return S3Request(
client=self,
request=request,
type=type,
signing_config=signing_config,
credential_provider=credential_provider,
request=request,
recv_filepath=recv_filepath,
send_filepath=send_filepath,
signing_config=signing_config,
credential_provider=credential_provider,
checksum_config=checksum_config,
on_headers=on_headers,
on_body=on_body,
on_done=on_done,
Expand Down Expand Up @@ -277,12 +337,13 @@ def __init__(
self,
*,
client,
request,
type,
signing_config=None,
credential_provider=None,
request,
recv_filepath=None,
send_filepath=None,
signing_config=None,
credential_provider=None,
checksum_config=None,
on_headers=None,
on_body=None,
on_done=None,
Expand All @@ -299,6 +360,16 @@ def __init__(
self._finished_future = Future()
self.shutdown_event = threading.Event()

checksum_algorithm = 0 # 0 means NONE in C
checksum_location = 0 # 0 means NONE in C
validate_response_checksum = False
if checksum_config is not None:
if checksum_config.algorithm is not None:
checksum_algorithm = checksum_config.algorithm.value
if checksum_config.location is not None:
checksum_location = checksum_config.location.value
validate_response_checksum = checksum_config.validate_response

s3_request_core = _S3RequestCore(
request,
self._finished_future,
Expand All @@ -320,6 +391,9 @@ def __init__(
recv_filepath,
send_filepath,
region,
checksum_algorithm,
checksum_location,
validate_response_checksum,
s3_request_core)

@property
Expand Down
2 changes: 1 addition & 1 deletion crt/aws-c-cal
2 changes: 1 addition & 1 deletion crt/s2n
Submodule s2n updated from 4654fe to 3526e6
32 changes: 14 additions & 18 deletions source/s3_client.c
Original file line number Diff line number Diff line change
Expand Up @@ -71,17 +71,16 @@ PyObject *aws_py_s3_client_new(PyObject *self, PyObject *args) {

struct aws_allocator *allocator = aws_py_get_allocator();

PyObject *bootstrap_py = NULL;
PyObject *signing_config_py = NULL;
PyObject *credential_provider_py = NULL;
PyObject *tls_options_py = NULL;
PyObject *on_shutdown_py = NULL;
PyObject *py_core = NULL;
const char *region;
Py_ssize_t region_len;
uint64_t part_size = 0;
double throughput_target_gbps = 0;
int tls_mode;
PyObject *bootstrap_py; /* O */
PyObject *signing_config_py; /* O */
PyObject *credential_provider_py; /* O */
PyObject *tls_options_py; /* O */
PyObject *on_shutdown_py; /* O */
struct aws_byte_cursor region; /* s# */
int tls_mode; /* i */
uint64_t part_size; /* K */
double throughput_target_gbps; /* d */
PyObject *py_core; /* O */
if (!PyArg_ParseTuple(
args,
"OOOOOs#iKdO",
Expand All @@ -90,8 +89,8 @@ PyObject *aws_py_s3_client_new(PyObject *self, PyObject *args) {
&credential_provider_py,
&tls_options_py,
&on_shutdown_py,
&region,
&region_len,
&region.ptr,
&region.len,
&tls_mode,
&part_size,
&throughput_target_gbps,
Expand Down Expand Up @@ -121,11 +120,8 @@ PyObject *aws_py_s3_client_new(PyObject *self, PyObject *args) {
struct aws_signing_config_aws signing_config_from_credentials_provider;
AWS_ZERO_STRUCT(signing_config_from_credentials_provider);

struct aws_byte_cursor region_cursor = aws_byte_cursor_from_array((const uint8_t *)region, region_len);

if (credential_provider) {
aws_s3_init_default_signing_config(
&signing_config_from_credentials_provider, region_cursor, credential_provider);
aws_s3_init_default_signing_config(&signing_config_from_credentials_provider, region, credential_provider);
signing_config = &signing_config_from_credentials_provider;
}

Expand Down Expand Up @@ -157,7 +153,7 @@ PyObject *aws_py_s3_client_new(PyObject *self, PyObject *args) {
Py_INCREF(s3_client->py_core);

struct aws_s3_client_config s3_config = {
.region = aws_byte_cursor_from_array((const uint8_t *)region, region_len),
.region = region,
.client_bootstrap = bootstrap,
.tls_mode = tls_mode,
.signing_config = signing_config,
Expand Down
45 changes: 28 additions & 17 deletions source/s3_meta_request.c
Original file line number Diff line number Diff line change
Expand Up @@ -319,6 +319,7 @@ static void s_s3_request_on_progress(
const struct aws_s3_meta_request_progress *progress,
void *user_data) {

(void)meta_request;
struct s3_meta_request_binding *request_binding = user_data;

bool report_progress = false;
Expand Down Expand Up @@ -346,20 +347,22 @@ PyObject *aws_py_s3_client_make_meta_request(PyObject *self, PyObject *args) {

struct aws_allocator *allocator = aws_py_get_allocator();

PyObject *py_s3_request = NULL;
PyObject *s3_client_py = NULL;
PyObject *http_request_py = NULL;
int type;
PyObject *signing_config_py = NULL;
PyObject *credential_provider_py = NULL;
const char *recv_filepath;
const char *send_filepath;
const char *region;
Py_ssize_t region_len;
PyObject *py_core = NULL;
PyObject *py_s3_request; /* O */
PyObject *s3_client_py; /* O */
PyObject *http_request_py; /* O */
int type; /* i */
PyObject *signing_config_py; /* O */
PyObject *credential_provider_py; /* O */
const char *recv_filepath; /* z */
const char *send_filepath; /* z */
struct aws_byte_cursor region; /* s# */
enum aws_s3_checksum_algorithm checksum_algorithm; /* i */
enum aws_s3_checksum_location checksum_location; /* i */
int validate_response_checksum; /* p - boolean predicate */
PyObject *py_core; /* O */
if (!PyArg_ParseTuple(
args,
"OOOiOOzzs#O",
"OOOiOOzzs#iipO",
&py_s3_request,
&s3_client_py,
&http_request_py,
Expand All @@ -368,8 +371,11 @@ PyObject *aws_py_s3_client_make_meta_request(PyObject *self, PyObject *args) {
&credential_provider_py,
&recv_filepath,
&send_filepath,
&region,
&region_len,
&region.ptr,
&region.len,
&checksum_algorithm,
&checksum_location,
&validate_response_checksum,
&py_core)) {
return NULL;
}
Expand Down Expand Up @@ -402,12 +408,16 @@ PyObject *aws_py_s3_client_make_meta_request(PyObject *self, PyObject *args) {
struct aws_signing_config_aws signing_config_from_credentials_provider;
AWS_ZERO_STRUCT(signing_config_from_credentials_provider);
if (credential_provider) {
struct aws_byte_cursor region_cursor = aws_byte_cursor_from_array((const uint8_t *)region, region_len);
aws_s3_init_default_signing_config(
&signing_config_from_credentials_provider, region_cursor, credential_provider);
aws_s3_init_default_signing_config(&signing_config_from_credentials_provider, region, credential_provider);
signing_config = &signing_config_from_credentials_provider;
}

struct aws_s3_checksum_config checksum_config = {
.checksum_algorithm = checksum_algorithm,
.location = checksum_location,
.validate_response_checksum = validate_response_checksum != 0,
};

struct s3_meta_request_binding *meta_request = aws_mem_calloc(allocator, 1, sizeof(struct s3_meta_request_binding));
if (!meta_request) {
return PyErr_AwsLastError();
Expand Down Expand Up @@ -438,6 +448,7 @@ PyObject *aws_py_s3_client_make_meta_request(PyObject *self, PyObject *args) {
.type = type,
.message = http_request,
.signing_config = signing_config,
.checksum_config = &checksum_config,
.send_filepath = aws_byte_cursor_from_c_str(send_filepath),
.headers_callback = s_s3_request_on_headers,
.body_callback = s_s3_request_on_body,
Expand Down
Loading

0 comments on commit 2ea0145

Please sign in to comment.