From 337155f6c07d39e61234e705ed6e58c31d4841eb Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Fri, 6 Dec 2024 09:18:41 -0800 Subject: [PATCH] Support full object checksum (#468) Co-authored-by: Michael Graeb --- include/aws/s3/private/s3_checksums.h | 53 +++- include/aws/s3/private/s3_meta_request_impl.h | 2 +- include/aws/s3/private/s3_request_messages.h | 14 +- include/aws/s3/private/s3_util.h | 43 +-- include/aws/s3/s3_client.h | 7 +- source/s3_auto_ranged_put.c | 76 ++++- source/s3_checksum_stream.c | 2 +- source/s3_checksums.c | 288 +++++++++++------- source/s3_chunk_stream.c | 12 +- source/s3_client.c | 22 -- source/s3_meta_request.c | 25 +- source/s3_request_messages.c | 116 ++++--- source/s3_util.c | 32 +- tests/CMakeLists.txt | 13 +- tests/s3_checksum_stream_test.c | 25 +- tests/s3_checksums_crc32_tests.c | 47 +-- tests/s3_checksums_crc32c_tests.c | 47 +-- tests/s3_checksums_crc64nvme_tests.c | 148 +++++++++ tests/s3_checksums_sha1_tests.c | 44 +-- tests/s3_checksums_sha256_tests.c | 60 +--- tests/s3_checksums_test_case_helper.h | 4 +- tests/s3_data_plane_tests.c | 47 ++- tests/s3_list_objects_tests.c | 5 +- tests/s3_request_messages_tests.c | 5 + tests/s3_tester.c | 64 +++- tests/s3_tester.h | 1 + 26 files changed, 730 insertions(+), 472 deletions(-) create mode 100644 tests/s3_checksums_crc64nvme_tests.c diff --git a/include/aws/s3/private/s3_checksums.h b/include/aws/s3/private/s3_checksums.h index 62155614e..e54b1f7cc 100644 --- a/include/aws/s3/private/s3_checksums.h +++ b/include/aws/s3/private/s3_checksums.h @@ -11,26 +11,45 @@ struct aws_s3_checksum; +/* List to check the checksum algorithm to use based on the priority. */ +static const enum aws_s3_checksum_algorithm s_checksum_algo_priority_list[] = { + AWS_SCA_CRC64NVME, + AWS_SCA_CRC32C, + AWS_SCA_CRC32, + AWS_SCA_SHA1, + AWS_SCA_SHA256, +}; +AWS_STATIC_ASSERT(AWS_ARRAY_SIZE(s_checksum_algo_priority_list) == (AWS_SCA_END - AWS_SCA_INIT + 1)); + struct aws_checksum_vtable { void (*destroy)(struct aws_s3_checksum *checksum); int (*update)(struct aws_s3_checksum *checksum, const struct aws_byte_cursor *buf); - int (*finalize)(struct aws_s3_checksum *checksum, struct aws_byte_buf *out, size_t truncate_to); + int (*finalize)(struct aws_s3_checksum *checksum, struct aws_byte_buf *out); }; struct aws_s3_checksum { struct aws_allocator *allocator; struct aws_checksum_vtable *vtable; - void *impl; size_t digest_size; enum aws_s3_checksum_algorithm algorithm; bool good; + union { + struct aws_hash *hash; + uint32_t crc_val_32bit; + uint64_t crc_val_64bit; + } impl; }; -struct checksum_config { +struct checksum_config_storage { + struct aws_allocator *allocator; + struct aws_byte_buf full_object_checksum; + bool has_full_object_checksum; + enum aws_s3_checksum_location location; enum aws_s3_checksum_algorithm checksum_algorithm; bool validate_response_checksum; struct { + bool crc64nvme; bool crc32c; bool crc32; bool sha1; @@ -85,25 +104,26 @@ struct aws_input_stream *aws_chunk_stream_new( * Get the size of the checksum output corresponding to the aws_s3_checksum_algorithm enum value. */ AWS_S3_API -size_t aws_get_digest_size_from_algorithm(enum aws_s3_checksum_algorithm algorithm); +size_t aws_get_digest_size_from_checksum_algorithm(enum aws_s3_checksum_algorithm algorithm); /** - * Get the header name corresponding to the aws_s3_checksum_algorithm enum value. + * Get header name to use for algorithm (e.g. "x-amz-checksum-crc32") */ AWS_S3_API -const struct aws_byte_cursor *aws_get_http_header_name_from_algorithm(enum aws_s3_checksum_algorithm algorithm); +struct aws_byte_cursor aws_get_http_header_name_from_checksum_algorithm(enum aws_s3_checksum_algorithm algorithm); /** - * Get the multipart upload header name corresponding to the aws_s3_checksum_algorithm enum value. + * Get algorithm's name (e.g. "CRC32"), to be used as the value of headers like `x-amz-checksum-algorithm` */ AWS_S3_API -const struct aws_byte_cursor *aws_get_create_mpu_header_name_from_algorithm(enum aws_s3_checksum_algorithm algorithm); +struct aws_byte_cursor aws_get_checksum_algorithm_name(enum aws_s3_checksum_algorithm algorithm); /** - * Get the complete multipart upload name corresponding to the aws_s3_checksum_algorithm enum value. + * Get the name of checksum algorithm to be used as the details of the parts were uploaded. Referring to + * https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompletedPart.html#AmazonS3-Type-CompletedPart */ AWS_S3_API -const struct aws_byte_cursor *aws_get_complete_mpu_name_from_algorithm(enum aws_s3_checksum_algorithm algorithm); +struct aws_byte_cursor aws_get_completed_part_name_from_checksum_algorithm(enum aws_s3_checksum_algorithm algorithm); /** * create a new aws_checksum corresponding to the aws_s3_checksum_algorithm enum value. @@ -121,8 +141,7 @@ int aws_checksum_compute( struct aws_allocator *allocator, enum aws_s3_checksum_algorithm algorithm, const struct aws_byte_cursor *input, - struct aws_byte_buf *output, - size_t truncate_to); + struct aws_byte_buf *output); /** * Cleans up and deallocates checksum. @@ -141,9 +160,15 @@ int aws_checksum_update(struct aws_s3_checksum *checksum, const struct aws_byte_ * Allocation of output is the caller's responsibility. */ AWS_S3_API -int aws_checksum_finalize(struct aws_s3_checksum *checksum, struct aws_byte_buf *output, size_t truncate_to); +int aws_checksum_finalize(struct aws_s3_checksum *checksum, struct aws_byte_buf *output); + +AWS_S3_API +void aws_checksum_config_storage_init( + struct aws_allocator *allocator, + struct checksum_config_storage *internal_config, + const struct aws_s3_checksum_config *config); AWS_S3_API -void checksum_config_init(struct checksum_config *internal_config, const struct aws_s3_checksum_config *config); +void aws_checksum_config_storage_cleanup(struct checksum_config_storage *internal_config); #endif /* AWS_S3_CHECKSUMS_H */ diff --git a/include/aws/s3/private/s3_meta_request_impl.h b/include/aws/s3/private/s3_meta_request_impl.h index 024680c8f..593acca67 100644 --- a/include/aws/s3/private/s3_meta_request_impl.h +++ b/include/aws/s3/private/s3_meta_request_impl.h @@ -272,7 +272,7 @@ struct aws_s3_meta_request { const bool should_compute_content_md5; /* deep copy of the checksum config. */ - struct checksum_config checksum_config; + struct checksum_config_storage checksum_config; /* checksum found in either a default get request, or in the initial head request of a multipart get */ struct aws_byte_buf meta_request_level_response_header_checksum; diff --git a/include/aws/s3/private/s3_request_messages.h b/include/aws/s3/private/s3_request_messages.h index 2c1da1127..eb6ee129e 100644 --- a/include/aws/s3/private/s3_request_messages.h +++ b/include/aws/s3/private/s3_request_messages.h @@ -17,7 +17,7 @@ struct aws_byte_buf; struct aws_byte_cursor; struct aws_string; struct aws_array_list; -struct checksum_config; +struct checksum_config_storage; AWS_EXTERN_C_BEGIN @@ -52,13 +52,9 @@ struct aws_input_stream *aws_s3_message_util_assign_body( struct aws_allocator *allocator, struct aws_byte_buf *byte_buf, struct aws_http_message *out_message, - const struct checksum_config *checksum_config, + const struct checksum_config_storage *checksum_config, struct aws_byte_buf *out_checksum); -/* Return true if checksum headers has been set. */ -AWS_S3_API -bool aws_s3_message_util_check_checksum_header(struct aws_http_message *message); - /* Create an HTTP request for an S3 Ranged Get Object Request, using the given request as a basis */ AWS_S3_API struct aws_http_message *aws_s3_ranged_get_object_message_new( @@ -80,7 +76,7 @@ AWS_S3_API struct aws_http_message *aws_s3_create_multipart_upload_message_new( struct aws_allocator *allocator, struct aws_http_message *base_message, - const struct checksum_config *checksum_config); + const struct checksum_config_storage *checksum_config); /* Create an HTTP request for an S3 Put Object request, using the original request as a basis. Creates and assigns a * body stream using the passed in buffer. If multipart is not needed, part number and upload_id can be 0 and NULL, @@ -93,7 +89,7 @@ struct aws_http_message *aws_s3_upload_part_message_new( uint32_t part_number, const struct aws_string *upload_id, bool should_compute_content_md5, - const struct checksum_config *checksum_config, + const struct checksum_config_storage *checksum_config, struct aws_byte_buf *encoded_checksum_output); /* Create an HTTP request for an S3 UploadPartCopy request, using the original request as a basis. @@ -120,7 +116,7 @@ struct aws_http_message *aws_s3_complete_multipart_message_new( struct aws_byte_buf *body_buffer, const struct aws_string *upload_id, const struct aws_array_list *parts, - const struct checksum_config *checksum_config); + const struct checksum_config_storage *checksum_config); AWS_S3_API struct aws_http_message *aws_s3_abort_multipart_upload_message_new( diff --git a/include/aws/s3/private/s3_util.h b/include/aws/s3/private/s3_util.h index 19f3e737f..9373e3299 100644 --- a/include/aws/s3/private/s3_util.h +++ b/include/aws/s3/private/s3_util.h @@ -61,44 +61,17 @@ extern const struct aws_byte_cursor g_request_validation_mode; AWS_S3_API extern const struct aws_byte_cursor g_enabled; +/** + * The checksum-algorithm header name used for CopyObject and CreateMultipartUpload + */ AWS_S3_API -extern const struct aws_byte_cursor g_create_mpu_checksum_header_name; - -AWS_S3_API -extern const struct aws_byte_cursor g_crc32c_header_name; - -AWS_S3_API -extern const struct aws_byte_cursor g_crc32_header_name; - -AWS_S3_API -extern const struct aws_byte_cursor g_sha1_header_name; - -AWS_S3_API -extern const struct aws_byte_cursor g_sha256_header_name; - -AWS_S3_API -extern const struct aws_byte_cursor g_crc32c_create_mpu_header_name; - -AWS_S3_API -extern const struct aws_byte_cursor g_crc32_create_mpu_header_name; - -AWS_S3_API -extern const struct aws_byte_cursor g_sha1_create_mpu_header_name; - -AWS_S3_API -extern const struct aws_byte_cursor g_sha256_create_mpu_header_name; - -AWS_S3_API -extern const struct aws_byte_cursor g_crc32c_complete_mpu_name; - -AWS_S3_API -extern const struct aws_byte_cursor g_crc32_complete_mpu_name; - -AWS_S3_API -extern const struct aws_byte_cursor g_sha1_complete_mpu_name; +extern const struct aws_byte_cursor g_checksum_algorithm_header_name; +/** + * The checksum-algorithm header name used for PutObject, UploadParts and PutObject* + */ AWS_S3_API -extern const struct aws_byte_cursor g_sha256_complete_mpu_name; +extern const struct aws_byte_cursor g_sdk_checksum_algorithm_header_name; AWS_S3_API extern const struct aws_byte_cursor g_s3_client_version; diff --git a/include/aws/s3/s3_client.h b/include/aws/s3/s3_client.h index c765d79da..daab68f2a 100644 --- a/include/aws/s3/s3_client.h +++ b/include/aws/s3/s3_client.h @@ -241,7 +241,8 @@ enum aws_s3_checksum_algorithm { AWS_SCA_CRC32, AWS_SCA_SHA1, AWS_SCA_SHA256, - AWS_SCA_END = AWS_SCA_SHA256, + AWS_SCA_CRC64NVME, + AWS_SCA_END = AWS_SCA_CRC64NVME, }; enum aws_s3_checksum_location { @@ -559,7 +560,7 @@ struct aws_s3_checksum_config { /** * The location of client added checksum header. * - * If AWS_SCL_NONE. No request payload checksum will be calculated or added. + * If AWS_SCL_NONE. No request payload checksum will be added. * * If AWS_SCL_HEADER, the client will calculate the checksum and add it to the headers. * @@ -592,7 +593,7 @@ struct aws_s3_checksum_config { * * The list of algorithms for user to pick up when validate the checksum. Client will pick up the algorithm from the * list with the priority based on performance, and the algorithm sent by server. The priority based on performance - * is [CRC32C, CRC32, SHA1, SHA256]. + * is [CRC64NVME, CRC32C, CRC32, SHA1, SHA256]. * * If the response checksum was validated by client, the result will indicate which algorithm was picked. */ diff --git a/source/s3_auto_ranged_put.c b/source/s3_auto_ranged_put.c index 68c901313..d45edd8b3 100644 --- a/source/s3_auto_ranged_put.c +++ b/source/s3_auto_ranged_put.c @@ -306,6 +306,71 @@ static struct aws_s3_meta_request_vtable s_s3_auto_ranged_put_vtable = { .pause = s_s3_auto_ranged_put_pause, }; +static int s_init_and_verify_checksum_config_from_headers( + struct checksum_config_storage *checksum_config, + const struct aws_http_message *message, + const void *log_id) { + /* Check if the checksum header was set from the message */ + struct aws_http_headers *headers = aws_http_message_get_headers(message); + enum aws_s3_checksum_algorithm header_algo = AWS_SCA_NONE; + struct aws_byte_cursor header_value; + AWS_ZERO_STRUCT(header_value); + + for (size_t i = 0; i < AWS_ARRAY_SIZE(s_checksum_algo_priority_list); i++) { + enum aws_s3_checksum_algorithm algorithm = s_checksum_algo_priority_list[i]; + const struct aws_byte_cursor algorithm_header_name = + aws_get_http_header_name_from_checksum_algorithm(algorithm); + if (aws_http_headers_get(headers, algorithm_header_name, &header_value) == AWS_OP_SUCCESS) { + if (header_algo == AWS_SCA_NONE) { + header_algo = algorithm; + } else { + /* If there are multiple checksum headers set, it's malformed request */ + AWS_LOGF_ERROR( + AWS_LS_S3_META_REQUEST, + "id=%p Could not create auto-ranged-put meta request; multiple checksum headers has been set", + log_id); + return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); + } + } + } + if (header_algo == AWS_SCA_NONE) { + /* No checksum header found, done */ + return AWS_OP_SUCCESS; + } + + /* Found the full object checksum from the header, check if it matches the explicit setting from config */ + if (checksum_config->checksum_algorithm != AWS_SCA_NONE && checksum_config->checksum_algorithm != header_algo) { + AWS_LOGF_ERROR( + AWS_LS_S3_META_REQUEST, + "id=%p Could not create auto-ranged-put meta request; checksum config mismatch the checksum from header.", + log_id); + return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); + } + AWS_ASSERT(!checksum_config->has_full_object_checksum); + + AWS_LOGF_DEBUG( + AWS_LS_S3_META_REQUEST, + "id=%p Setting the full-object checksum from header; algorithm: " PRInSTR ", value: " PRInSTR ".", + log_id, + AWS_BYTE_CURSOR_PRI(aws_get_checksum_algorithm_name(header_algo)), + AWS_BYTE_CURSOR_PRI(header_value)); + /* Set algo */ + checksum_config->checksum_algorithm = header_algo; + if (checksum_config->location == AWS_SCL_NONE) { + /* Set the checksum location to trailer for the parts, complete MPU will still have the checksum in the header. + * But to keep the data integrity for the parts, we need to set the checksum location to trailer to send the + * parts level checksums. + */ + checksum_config->location = AWS_SCL_TRAILER; + } + + /* Set full object checksum from the header value. */ + aws_byte_buf_init_copy_from_cursor( + &checksum_config->full_object_checksum, checksum_config->allocator, header_value); + checksum_config->has_full_object_checksum = true; + return AWS_OP_SUCCESS; +} + /* Allocate a new auto-ranged put meta request */ struct aws_s3_meta_request *aws_s3_meta_request_auto_ranged_put_new( struct aws_allocator *allocator, @@ -363,6 +428,11 @@ struct aws_s3_meta_request *aws_s3_meta_request_auto_ranged_put_new( goto error_clean_up; } + if (s_init_and_verify_checksum_config_from_headers( + &auto_ranged_put->base.checksum_config, options->message, (void *)&auto_ranged_put->base)) { + goto error_clean_up; + } + AWS_LOGF_DEBUG( AWS_LS_S3_META_REQUEST, "id=%p Created new Auto-Ranged Put Meta Request.", (void *)&auto_ranged_put->base); @@ -767,7 +837,7 @@ static int s_verify_part_matches_checksum( } struct aws_byte_buf checksum; - if (aws_byte_buf_init(&checksum, allocator, aws_get_digest_size_from_algorithm(algorithm))) { + if (aws_byte_buf_init(&checksum, allocator, aws_get_digest_size_from_checksum_algorithm(algorithm))) { return AWS_OP_ERR; } @@ -776,14 +846,14 @@ static int s_verify_part_matches_checksum( int return_status = AWS_OP_SUCCESS; size_t encoded_len = 0; - if (aws_base64_compute_encoded_len(aws_get_digest_size_from_algorithm(algorithm), &encoded_len)) { + if (aws_base64_compute_encoded_len(aws_get_digest_size_from_checksum_algorithm(algorithm), &encoded_len)) { AWS_LOGF_ERROR( AWS_LS_S3_META_REQUEST, "Failed to resume upload. Unable to determine length of encoded checksum."); return_status = aws_raise_error(AWS_ERROR_S3_RESUME_FAILED); goto on_done; } - if (aws_checksum_compute(allocator, algorithm, &body_cur, &checksum, 0)) { + if (aws_checksum_compute(allocator, algorithm, &body_cur, &checksum)) { AWS_LOGF_ERROR( AWS_LS_S3_META_REQUEST, "Failed to resume upload. Unable to compute checksum for the skipped part."); return_status = aws_raise_error(AWS_ERROR_S3_RESUME_FAILED); diff --git a/source/s3_checksum_stream.c b/source/s3_checksum_stream.c index 7c0a2a3a5..2b21351af 100644 --- a/source/s3_checksum_stream.c +++ b/source/s3_checksum_stream.c @@ -24,7 +24,7 @@ static int s_finalize_checksum(struct aws_checksum_stream *impl) { return AWS_OP_SUCCESS; } - if (aws_checksum_finalize(impl->checksum, &impl->checksum_result, 0) != AWS_OP_SUCCESS) { + if (aws_checksum_finalize(impl->checksum, &impl->checksum_result) != AWS_OP_SUCCESS) { AWS_LOGF_ERROR( AWS_LS_S3_CLIENT, "Failed to calculate checksum with error code %d (%s).", diff --git a/source/s3_checksums.c b/source/s3_checksums.c index 0bfdc0181..b1bef1981 100644 --- a/source/s3_checksums.c +++ b/source/s3_checksums.c @@ -3,11 +3,43 @@ #include #include -#define AWS_CRC32_LEN 4 -#define AWS_CRC32C_LEN 4 +#define AWS_CRC32_LEN sizeof(uint32_t) +#define AWS_CRC32C_LEN sizeof(uint32_t) +#define AWS_CRC64_LEN sizeof(uint64_t) -size_t aws_get_digest_size_from_algorithm(enum aws_s3_checksum_algorithm algorithm) { +static const struct aws_byte_cursor s_crc64nvme_algorithm_value = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("CRC64NVME"); +static const struct aws_byte_cursor s_crc32c_algorithm_value = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("CRC32C"); +static const struct aws_byte_cursor s_crc32_algorithm_value = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("CRC32"); +static const struct aws_byte_cursor s_sha1_algorithm_value = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("SHA1"); +static const struct aws_byte_cursor s_sha256_algorithm_value = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("SHA256"); + +static const struct aws_byte_cursor s_crc64nvme_header_name = + AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-crc64nvme"); +static const struct aws_byte_cursor s_crc32c_header_name = + AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-crc32c"); +static const struct aws_byte_cursor s_crc32_header_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-crc32"); +static const struct aws_byte_cursor s_sha1_header_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-sha1"); +static const struct aws_byte_cursor s_sha256_header_name = + AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-sha256"); + +static const struct aws_byte_cursor s_crc64nvme_completed_part_name = + AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("ChecksumCRC64NVME"); +static const struct aws_byte_cursor s_crc32c_completed_part_name = + AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("ChecksumCRC32C"); +static const struct aws_byte_cursor s_crc32_completed_part_name = + AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("ChecksumCRC32"); +static const struct aws_byte_cursor s_sha1_completed_part_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("ChecksumSHA1"); +static const struct aws_byte_cursor s_sha256_completed_part_name = + AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("ChecksumSHA256"); +static const struct aws_byte_cursor s_empty_cursor = { + .len = 0, + .ptr = NULL, +}; + +size_t aws_get_digest_size_from_checksum_algorithm(enum aws_s3_checksum_algorithm algorithm) { switch (algorithm) { + case AWS_SCA_CRC64NVME: + return AWS_CRC64_LEN; case AWS_SCA_CRC32C: return AWS_CRC32C_LEN; case AWS_SCA_CRC32: @@ -21,121 +53,128 @@ size_t aws_get_digest_size_from_algorithm(enum aws_s3_checksum_algorithm algorit } } -const struct aws_byte_cursor *aws_get_http_header_name_from_algorithm(enum aws_s3_checksum_algorithm algorithm) { +struct aws_byte_cursor aws_get_http_header_name_from_checksum_algorithm(enum aws_s3_checksum_algorithm algorithm) { switch (algorithm) { + case AWS_SCA_CRC64NVME: + return s_crc64nvme_header_name; case AWS_SCA_CRC32C: - return &g_crc32c_header_name; + return s_crc32c_header_name; case AWS_SCA_CRC32: - return &g_crc32_header_name; + return s_crc32_header_name; case AWS_SCA_SHA1: - return &g_sha1_header_name; + return s_sha1_header_name; case AWS_SCA_SHA256: - return &g_sha256_header_name; + return s_sha256_header_name; default: - return NULL; + return s_empty_cursor; } } -const struct aws_byte_cursor *aws_get_create_mpu_header_name_from_algorithm(enum aws_s3_checksum_algorithm algorithm) { + +struct aws_byte_cursor aws_get_checksum_algorithm_name(enum aws_s3_checksum_algorithm algorithm) { switch (algorithm) { + case AWS_SCA_CRC64NVME: + return s_crc64nvme_algorithm_value; case AWS_SCA_CRC32C: - return &g_crc32c_create_mpu_header_name; + return s_crc32c_algorithm_value; case AWS_SCA_CRC32: - return &g_crc32_create_mpu_header_name; + return s_crc32_algorithm_value; case AWS_SCA_SHA1: - return &g_sha1_create_mpu_header_name; + return s_sha1_algorithm_value; case AWS_SCA_SHA256: - return &g_sha256_create_mpu_header_name; + return s_sha256_algorithm_value; default: - return NULL; + return s_empty_cursor; } } -const struct aws_byte_cursor *aws_get_complete_mpu_name_from_algorithm(enum aws_s3_checksum_algorithm algorithm) { +struct aws_byte_cursor aws_get_completed_part_name_from_checksum_algorithm(enum aws_s3_checksum_algorithm algorithm) { switch (algorithm) { + case AWS_SCA_CRC64NVME: + return s_crc64nvme_completed_part_name; case AWS_SCA_CRC32C: - return &g_crc32c_complete_mpu_name; + return s_crc32c_completed_part_name; case AWS_SCA_CRC32: - return &g_crc32_complete_mpu_name; + return s_crc32_completed_part_name; case AWS_SCA_SHA1: - return &g_sha1_complete_mpu_name; + return s_sha1_completed_part_name; case AWS_SCA_SHA256: - return &g_sha256_complete_mpu_name; + return s_sha256_completed_part_name; default: - return NULL; + return s_empty_cursor; } } void s3_hash_destroy(struct aws_s3_checksum *checksum) { - struct aws_hash *hash = (struct aws_hash *)checksum->impl; + struct aws_hash *hash = checksum->impl.hash; aws_hash_destroy(hash); aws_mem_release(checksum->allocator, checksum); } int s3_hash_update(struct aws_s3_checksum *checksum, const struct aws_byte_cursor *to_checksum) { - struct aws_hash *hash = (struct aws_hash *)checksum->impl; - return aws_hash_update(hash, to_checksum); + return aws_hash_update(checksum->impl.hash, to_checksum); } -int s3_hash_finalize(struct aws_s3_checksum *checksum, struct aws_byte_buf *output, size_t truncate_to) { - struct aws_hash *hash = (struct aws_hash *)checksum->impl; +int s3_hash_finalize(struct aws_s3_checksum *checksum, struct aws_byte_buf *output) { checksum->good = false; - return aws_hash_finalize(hash, output, truncate_to); + return aws_hash_finalize(checksum->impl.hash, output, 0); } -typedef uint32_t (*crc_fn)(const uint8_t *, int, uint32_t); - -uint32_t aws_crc32_common(uint32_t previous, const struct aws_byte_cursor *buf, crc_fn checksum_fn) { - - size_t length = buf->len; - uint8_t *buffer = buf->ptr; - uint32_t val = previous; - while (length > INT_MAX) { - val = checksum_fn(buffer, INT_MAX, val); - buffer += (size_t)INT_MAX; - length -= (size_t)INT_MAX; - } - return checksum_fn(buffer, (int)length, val); -} +static int s_crc_finalize_helper(struct aws_s3_checksum *checksum, struct aws_byte_buf *out) { + AWS_PRECONDITION(aws_byte_buf_is_valid(out)); -int aws_crc_finalize(struct aws_s3_checksum *checksum, struct aws_byte_buf *out, size_t truncate_to) { if (!checksum->good) { return aws_raise_error(AWS_ERROR_INVALID_STATE); } checksum->good = false; - size_t available_buffer = out->capacity - out->len; size_t len = checksum->digest_size; - if (truncate_to && truncate_to < len) { - len = truncate_to; - } - if (available_buffer < len) { + if (out->capacity - out->len < len) { return aws_raise_error(AWS_ERROR_SHORT_BUFFER); } - AWS_PRECONDITION(aws_byte_buf_is_valid(out)); - uint32_t tmp = aws_hton32(*(uint32_t *)checksum->impl); - if (aws_byte_buf_write(out, (uint8_t *)&tmp, len)) { - return AWS_OP_SUCCESS; + if (checksum->digest_size == AWS_CRC32_LEN) { + if (aws_byte_buf_write_be32(out, checksum->impl.crc_val_32bit)) { + return AWS_OP_SUCCESS; + } + } else { + if (aws_byte_buf_write_be64(out, checksum->impl.crc_val_64bit)) { + return AWS_OP_SUCCESS; + } } return aws_raise_error(AWS_ERROR_INVALID_BUFFER_SIZE); } -int aws_crc32_checksum_update(struct aws_s3_checksum *checksum, const struct aws_byte_cursor *buf) { +static int s_crc32_finalize(struct aws_s3_checksum *checksum, struct aws_byte_buf *out) { + return s_crc_finalize_helper(checksum, out); +} + +static int s_crc64_finalize(struct aws_s3_checksum *checksum, struct aws_byte_buf *out) { + return s_crc_finalize_helper(checksum, out); +} + +static int s_crc32_checksum_update(struct aws_s3_checksum *checksum, const struct aws_byte_cursor *buf) { + if (!checksum->good) { + return aws_raise_error(AWS_ERROR_INVALID_STATE); + } + checksum->impl.crc_val_32bit = aws_checksums_crc32_ex(buf->ptr, buf->len, checksum->impl.crc_val_32bit); + return AWS_OP_SUCCESS; +} + +static int s_crc32c_checksum_update(struct aws_s3_checksum *checksum, const struct aws_byte_cursor *buf) { if (!checksum->good) { return aws_raise_error(AWS_ERROR_INVALID_STATE); } - *(uint32_t *)checksum->impl = aws_crc32_common(*(uint32_t *)checksum->impl, buf, aws_checksums_crc32); + checksum->impl.crc_val_32bit = aws_checksums_crc32c_ex(buf->ptr, buf->len, checksum->impl.crc_val_32bit); return AWS_OP_SUCCESS; } -int aws_crc32c_checksum_update(struct aws_s3_checksum *checksum, const struct aws_byte_cursor *buf) { +static int s_crc64nvme_checksum_update(struct aws_s3_checksum *checksum, const struct aws_byte_cursor *buf) { if (!checksum->good) { return aws_raise_error(AWS_ERROR_INVALID_STATE); } - *(uint32_t *)checksum->impl = aws_crc32_common(*(uint32_t *)checksum->impl, buf, aws_checksums_crc32c); + checksum->impl.crc_val_64bit = aws_checksums_crc64nvme_ex(buf->ptr, buf->len, checksum->impl.crc_val_64bit); return AWS_OP_SUCCESS; } -void aws_crc_destroy(struct aws_s3_checksum *checksum) { - aws_mem_release(checksum->allocator, checksum->impl); +static void s_crc_destroy(struct aws_s3_checksum *checksum) { aws_mem_release(checksum->allocator, checksum); } @@ -146,20 +185,30 @@ static struct aws_checksum_vtable hash_vtable = { }; static struct aws_checksum_vtable crc32_vtable = { - .update = aws_crc32_checksum_update, - .finalize = aws_crc_finalize, - .destroy = aws_crc_destroy, + .update = s_crc32_checksum_update, + .finalize = s_crc32_finalize, + .destroy = s_crc_destroy, }; static struct aws_checksum_vtable crc32c_vtable = { - .update = aws_crc32c_checksum_update, - .finalize = aws_crc_finalize, - .destroy = aws_crc_destroy, + .update = s_crc32c_checksum_update, + .finalize = s_crc32_finalize, + .destroy = s_crc_destroy, +}; +static struct aws_checksum_vtable crc64nvme_vtable = { + .update = s_crc64nvme_checksum_update, + .finalize = s_crc64_finalize, + .destroy = s_crc_destroy, }; struct aws_s3_checksum *aws_hash_new(struct aws_allocator *allocator, aws_hash_new_fn hash_fn) { - struct aws_s3_checksum *checksum = aws_mem_acquire(allocator, sizeof(struct aws_s3_checksum)); + struct aws_s3_checksum *checksum = aws_mem_calloc(allocator, 1, sizeof(struct aws_s3_checksum)); struct aws_hash *hash = hash_fn(allocator); - checksum->impl = (void *)hash; + if (!hash) { + aws_mem_release(allocator, checksum); + aws_raise_error(aws_last_error_or_unknown()); + return NULL; + } + checksum->impl.hash = hash; checksum->allocator = allocator; checksum->vtable = &hash_vtable; checksum->good = true; @@ -167,39 +216,48 @@ struct aws_s3_checksum *aws_hash_new(struct aws_allocator *allocator, aws_hash_n return checksum; } -struct aws_s3_checksum *aws_crc32_checksum_new(struct aws_allocator *allocator) { - struct aws_s3_checksum *checksum = aws_mem_acquire(allocator, sizeof(struct aws_s3_checksum)); - uint32_t *crc_val = aws_mem_acquire(allocator, sizeof(uint32_t)); - *crc_val = 0; +static struct aws_s3_checksum *s_crc32_checksum_new(struct aws_allocator *allocator) { + struct aws_s3_checksum *checksum = aws_mem_calloc(allocator, 1, sizeof(struct aws_s3_checksum)); checksum->vtable = &crc32_vtable; checksum->allocator = allocator; - checksum->impl = crc_val; + checksum->impl.crc_val_32bit = 0; checksum->good = true; checksum->digest_size = AWS_CRC32_LEN; return checksum; } -struct aws_s3_checksum *aws_crc32c_checksum_new(struct aws_allocator *allocator) { - struct aws_s3_checksum *checksum = aws_mem_acquire(allocator, sizeof(struct aws_s3_checksum)); - uint32_t *crc_val = aws_mem_acquire(allocator, sizeof(uint32_t)); - *crc_val = 0; +static struct aws_s3_checksum *s_crc32c_checksum_new(struct aws_allocator *allocator) { + struct aws_s3_checksum *checksum = aws_mem_calloc(allocator, 1, sizeof(struct aws_s3_checksum)); checksum->vtable = &crc32c_vtable; checksum->allocator = allocator; - checksum->impl = crc_val; + checksum->impl.crc_val_32bit = 0; checksum->good = true; - checksum->digest_size = AWS_CRC32_LEN; + checksum->digest_size = AWS_CRC32C_LEN; + return checksum; +} + +static struct aws_s3_checksum *s_crc64nvme_checksum_new(struct aws_allocator *allocator) { + struct aws_s3_checksum *checksum = aws_mem_calloc(allocator, 1, sizeof(struct aws_s3_checksum)); + checksum->vtable = &crc64nvme_vtable; + checksum->allocator = allocator; + checksum->impl.crc_val_64bit = 0; + checksum->good = true; + checksum->digest_size = AWS_CRC64_LEN; return checksum; } struct aws_s3_checksum *aws_checksum_new(struct aws_allocator *allocator, enum aws_s3_checksum_algorithm algorithm) { struct aws_s3_checksum *checksum = NULL; switch (algorithm) { + case AWS_SCA_CRC64NVME: + checksum = s_crc64nvme_checksum_new(allocator); + break; case AWS_SCA_CRC32C: - checksum = aws_crc32c_checksum_new(allocator); + checksum = s_crc32c_checksum_new(allocator); break; case AWS_SCA_CRC32: - checksum = aws_crc32_checksum_new(allocator); + checksum = s_crc32_checksum_new(allocator); break; case AWS_SCA_SHA1: checksum = aws_hash_new(allocator, aws_sha1_new); @@ -210,22 +268,39 @@ struct aws_s3_checksum *aws_checksum_new(struct aws_allocator *allocator, enum a default: return NULL; } - checksum->algorithm = algorithm; + if (checksum != NULL) { + checksum->algorithm = algorithm; + } return checksum; } -int aws_checksum_compute_fn( +void aws_checksum_destroy(struct aws_s3_checksum *checksum) { + if (checksum != NULL) { + checksum->vtable->destroy(checksum); + } +} + +int aws_checksum_update(struct aws_s3_checksum *checksum, const struct aws_byte_cursor *to_checksum) { + AWS_PRECONDITION(checksum); + return checksum->vtable->update(checksum, to_checksum); +} + +int aws_checksum_finalize(struct aws_s3_checksum *checksum, struct aws_byte_buf *output) { + AWS_PRECONDITION(checksum); + return checksum->vtable->finalize(checksum, output); +} + +static int s_checksum_compute_fn( struct aws_allocator *allocator, const struct aws_byte_cursor *input, struct aws_byte_buf *output, - struct aws_s3_checksum *(*aws_crc_new)(struct aws_allocator *), - size_t truncate_to) { - struct aws_s3_checksum *checksum = aws_crc_new(allocator); + struct aws_s3_checksum *(*s_crc_new)(struct aws_allocator *)) { + struct aws_s3_checksum *checksum = s_crc_new(allocator); if (aws_checksum_update(checksum, input)) { aws_checksum_destroy(checksum); return AWS_OP_ERR; } - if (aws_checksum_finalize(checksum, output, truncate_to)) { + if (aws_checksum_finalize(checksum, output)) { aws_checksum_destroy(checksum); return AWS_OP_ERR; } @@ -233,43 +308,36 @@ int aws_checksum_compute_fn( return AWS_OP_SUCCESS; } -void aws_checksum_destroy(struct aws_s3_checksum *checksum) { - if (checksum != NULL) { - checksum->vtable->destroy(checksum); - } -} - -int aws_checksum_update(struct aws_s3_checksum *checksum, const struct aws_byte_cursor *to_checksum) { - return checksum->vtable->update(checksum, to_checksum); -} - -int aws_checksum_finalize(struct aws_s3_checksum *checksum, struct aws_byte_buf *output, size_t truncate_to) { - return checksum->vtable->finalize(checksum, output, truncate_to); -} - int aws_checksum_compute( struct aws_allocator *allocator, enum aws_s3_checksum_algorithm algorithm, const struct aws_byte_cursor *input, - struct aws_byte_buf *output, - size_t truncate_to) { + struct aws_byte_buf *output) { switch (algorithm) { case AWS_SCA_SHA1: - return aws_sha1_compute(allocator, input, output, truncate_to); + return aws_sha1_compute(allocator, input, output, 0); case AWS_SCA_SHA256: - return aws_sha256_compute(allocator, input, output, truncate_to); + return aws_sha256_compute(allocator, input, output, 0); + case AWS_SCA_CRC64NVME: + return s_checksum_compute_fn(allocator, input, output, s_crc64nvme_checksum_new); case AWS_SCA_CRC32: - return aws_checksum_compute_fn(allocator, input, output, aws_crc32_checksum_new, truncate_to); + return s_checksum_compute_fn(allocator, input, output, s_crc32_checksum_new); case AWS_SCA_CRC32C: - return aws_checksum_compute_fn(allocator, input, output, aws_crc32c_checksum_new, truncate_to); + return s_checksum_compute_fn(allocator, input, output, s_crc32c_checksum_new); default: return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); } } -void checksum_config_init(struct checksum_config *internal_config, const struct aws_s3_checksum_config *config) { +void aws_checksum_config_storage_init( + struct aws_allocator *allocator, + struct checksum_config_storage *internal_config, + const struct aws_s3_checksum_config *config) { AWS_ZERO_STRUCT(*internal_config); + /* Zero out the struct and set the allocator regardless. */ + internal_config->allocator = allocator; + if (!config) { return; } @@ -283,6 +351,9 @@ void checksum_config_init(struct checksum_config *internal_config, const struct enum aws_s3_checksum_algorithm algorithm; aws_array_list_get_at(config->validate_checksum_algorithms, &algorithm, i); switch (algorithm) { + case AWS_SCA_CRC64NVME: + internal_config->response_checksum_algorithms.crc64nvme = true; + break; case AWS_SCA_CRC32C: internal_config->response_checksum_algorithms.crc32c = true; break; @@ -301,9 +372,16 @@ void checksum_config_init(struct checksum_config *internal_config, const struct } } else if (config->validate_response_checksum) { + internal_config->response_checksum_algorithms.crc64nvme = true; internal_config->response_checksum_algorithms.crc32 = true; internal_config->response_checksum_algorithms.crc32c = true; internal_config->response_checksum_algorithms.sha1 = true; internal_config->response_checksum_algorithms.sha256 = true; } } + +void aws_checksum_config_storage_cleanup(struct checksum_config_storage *internal_config) { + if (internal_config->has_full_object_checksum) { + aws_byte_buf_clean_up(&internal_config->full_object_checksum); + } +} diff --git a/source/s3_chunk_stream.c b/source/s3_chunk_stream.c index 40b9e80bf..46896af6a 100644 --- a/source/s3_chunk_stream.c +++ b/source/s3_chunk_stream.c @@ -32,7 +32,7 @@ struct aws_chunk_stream { struct aws_byte_buf *checksum_result_output; struct aws_byte_buf pre_chunk_buffer; struct aws_byte_buf post_chunk_buffer; - const struct aws_byte_cursor *checksum_header_name; + struct aws_byte_cursor checksum_header_name; int64_t length; set_stream_fn *set_current_stream_fn; }; @@ -76,12 +76,12 @@ static int s_set_post_chunk_stream(struct aws_chunk_stream *parent_stream) { if (aws_byte_buf_init( &parent_stream->post_chunk_buffer, parent_stream->allocator, - final_chunk_cursor.len + parent_stream->checksum_header_name->len + colon_cursor.len + + final_chunk_cursor.len + parent_stream->checksum_header_name.len + colon_cursor.len + checksum_result_cursor.len + post_trailer_cursor.len)) { goto error; } if (aws_byte_buf_append(&parent_stream->post_chunk_buffer, &final_chunk_cursor) || - aws_byte_buf_append(&parent_stream->post_chunk_buffer, parent_stream->checksum_header_name) || + aws_byte_buf_append(&parent_stream->post_chunk_buffer, &parent_stream->checksum_header_name) || aws_byte_buf_append(&parent_stream->post_chunk_buffer, &colon_cursor) || aws_byte_buf_append(&parent_stream->post_chunk_buffer, &checksum_result_cursor) || aws_byte_buf_append(&parent_stream->post_chunk_buffer, &post_trailer_cursor)) { @@ -219,7 +219,7 @@ struct aws_input_stream *aws_chunk_stream_new( goto error; } - size_t checksum_len = aws_get_digest_size_from_algorithm(algorithm); + size_t checksum_len = aws_get_digest_size_from_checksum_algorithm(algorithm); size_t encoded_checksum_len = 0; if (aws_base64_compute_encoded_len(checksum_len, &encoded_checksum_len)) { @@ -254,14 +254,14 @@ struct aws_input_stream *aws_chunk_stream_new( impl->set_current_stream_fn = s_set_post_chunk_stream; } - impl->checksum_header_name = aws_get_http_header_name_from_algorithm(algorithm); + impl->checksum_header_name = aws_get_http_header_name_from_checksum_algorithm(algorithm); if (aws_input_stream_get_length(impl->current_stream, &prechunk_stream_len)) { goto error; } /* we subtract one since aws_base64_compute_encoded_len accounts for the null terminator which won't show up in our * stream */ - impl->length = prechunk_stream_len + stream_length + final_chunk_len + impl->checksum_header_name->len + colon_len + + impl->length = prechunk_stream_len + stream_length + final_chunk_len + impl->checksum_header_name.len + colon_len + encoded_checksum_len + post_trailer_len - 1; AWS_ASSERT(impl->current_stream); diff --git a/source/s3_client.c b/source/s3_client.c index 2b1e82274..b16c11634 100644 --- a/source/s3_client.c +++ b/source/s3_client.c @@ -973,16 +973,6 @@ struct aws_s3_meta_request *aws_s3_client_make_meta_request( aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); return NULL; } - if (options->checksum_config->checksum_algorithm != AWS_SCA_NONE && - options->checksum_config->location == AWS_SCL_NONE && options->upload_review_callback == NULL) { - AWS_LOGF_ERROR( - AWS_LS_S3_CLIENT, - "id=%p Cannot create meta s3 request; checksum algorithm is set, but no checksum location selected " - "and no upload review callback set, so checksums will be unused", - (void *)client); - aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); - return NULL; - } } if (s_apply_endpoint_override(client, message_headers, options->endpoint)) { @@ -1347,19 +1337,7 @@ static struct aws_s3_meta_request *s_s3_client_meta_request_factory_default( client->compute_content_md5 == AWS_MR_CONTENT_MD5_ENABLED && !aws_http_headers_has(initial_message_headers, g_content_md5_header_name), options); - } else { - if (aws_s3_message_util_check_checksum_header(options->message)) { - /* The checksum header has been set and the request will be split. We fail the request */ - AWS_LOGF_ERROR( - AWS_LS_S3_META_REQUEST, - "Could not create auto-ranged-put meta request; checksum headers has been set for " - "auto-ranged-put that will be split. Pre-calculated checksums are only supported for " - "single part upload."); - aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); - return NULL; - } } - return aws_s3_meta_request_auto_ranged_put_new( client->allocator, client, part_size, content_length_found, content_length, num_parts, options); } else { /* else using resume token */ diff --git a/source/s3_meta_request.c b/source/s3_meta_request.c index 2982bb40d..121ac0cf5 100644 --- a/source/s3_meta_request.c +++ b/source/s3_meta_request.c @@ -114,7 +114,7 @@ static bool s_validate_checksum( aws_byte_buf_init(&encoded_response_body_sum, checksum_to_validate->allocator, encoded_checksum_len); aws_byte_buf_init(&response_body_sum, checksum_to_validate->allocator, checksum_to_validate->digest_size); - if (aws_checksum_finalize(checksum_to_validate, &response_body_sum, 0)) { + if (aws_checksum_finalize(checksum_to_validate, &response_body_sum)) { goto done; } struct aws_byte_cursor response_body_sum_cursor = aws_byte_cursor_from_buf(&response_body_sum); @@ -218,7 +218,7 @@ int aws_s3_meta_request_init_base( *((size_t *)&meta_request->part_size) = part_size; *((bool *)&meta_request->should_compute_content_md5) = should_compute_content_md5; - checksum_config_init(&meta_request->checksum_config, options->checksum_config); + aws_checksum_config_storage_init(meta_request->allocator, &meta_request->checksum_config, options->checksum_config); if (options->signing_config) { meta_request->cached_signing_config = aws_cached_signing_config_new(client, options->signing_config); @@ -481,6 +481,7 @@ static void s_s3_meta_request_destroy(void *user_data) { AWS_LOGF_DEBUG(AWS_LS_S3_META_REQUEST, "id=%p Cleaning up meta request", (void *)meta_request); /* Clean up our initial http message */ + aws_checksum_config_storage_cleanup(&meta_request->checksum_config); meta_request->request_body_async_stream = aws_async_input_stream_release(meta_request->request_body_async_stream); meta_request->initial_request_message = aws_http_message_release(meta_request->initial_request_message); @@ -1203,10 +1204,10 @@ static int s_s3_meta_request_error_code_from_response_status(int response_status static bool s_header_value_from_list( const struct aws_http_header *headers, size_t headers_count, - const struct aws_byte_cursor *name, + const struct aws_byte_cursor name, struct aws_byte_cursor *out_value) { for (size_t i = 0; i < headers_count; ++i) { - if (aws_byte_cursor_eq(&headers[i].name, name)) { + if (aws_byte_cursor_eq(&headers[i].name, &name)) { *out_value = headers[i].value; return true; } @@ -1219,20 +1220,24 @@ static void s_get_part_response_headers_checksum_helper( struct aws_s3_meta_request *meta_request, const struct aws_http_header *headers, size_t headers_count) { - for (int i = AWS_SCA_INIT; i <= AWS_SCA_END; i++) { - if (!aws_s3_meta_request_checksum_config_has_algorithm(meta_request, i)) { + for (size_t i = 0; i < AWS_ARRAY_SIZE(s_checksum_algo_priority_list); i++) { + enum aws_s3_checksum_algorithm algorithm = s_checksum_algo_priority_list[i]; + if (!aws_s3_meta_request_checksum_config_has_algorithm(meta_request, algorithm)) { /* If user doesn't select this algorithm, skip */ continue; } - const struct aws_byte_cursor *algorithm_header_name = aws_get_http_header_name_from_algorithm(i); + const struct aws_byte_cursor algorithm_header_name = + aws_get_http_header_name_from_checksum_algorithm(algorithm); struct aws_byte_cursor header_sum; if (s_header_value_from_list(headers, headers_count, algorithm_header_name, &header_sum)) { size_t encoded_len = 0; - aws_base64_compute_encoded_len(aws_get_digest_size_from_algorithm(i), &encoded_len); + aws_base64_compute_encoded_len(aws_get_digest_size_from_checksum_algorithm(algorithm), &encoded_len); if (header_sum.len == encoded_len - 1) { aws_byte_buf_init_copy_from_cursor( &connection->request->request_level_response_header_checksum, meta_request->allocator, header_sum); - connection->request->request_level_running_response_sum = aws_checksum_new(meta_request->allocator, i); + connection->request->request_level_running_response_sum = + aws_checksum_new(meta_request->allocator, algorithm); + AWS_ASSERT(connection->request->request_level_running_response_sum != NULL); } break; } @@ -2498,6 +2503,8 @@ bool aws_s3_meta_request_checksum_config_has_algorithm( AWS_PRECONDITION(meta_request); switch (algorithm) { + case AWS_SCA_CRC64NVME: + return meta_request->checksum_config.response_checksum_algorithms.crc64nvme; case AWS_SCA_CRC32C: return meta_request->checksum_config.response_checksum_algorithms.crc32c; case AWS_SCA_CRC32: diff --git a/source/s3_request_messages.c b/source/s3_request_messages.c index 663305477..6f1057276 100644 --- a/source/s3_request_messages.c +++ b/source/s3_request_messages.c @@ -20,6 +20,11 @@ const struct aws_byte_cursor g_s3_create_multipart_upload_excluded_headers[] = { AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("Content-MD5"), AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-copy-source"), AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-copy-source-range"), + AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-crc64nvme"), + AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-crc32c"), + AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-crc32"), + AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-sha1"), + AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-sha256"), AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("if-none-match"), }; @@ -50,6 +55,11 @@ const struct aws_byte_cursor g_s3_upload_part_excluded_headers[] = { AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-object-lock-mode"), AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-object-lock-retain-until-date"), AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-object-lock-legal-hold"), + AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-crc64nvme"), + AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-crc32c"), + AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-crc32"), + AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-sha1"), + AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-sha256"), AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("if-none-match"), }; @@ -84,6 +94,7 @@ const struct aws_byte_cursor g_s3_complete_multipart_upload_excluded_headers[] = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-object-lock-legal-hold"), AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-copy-source"), AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-copy-source-range"), + AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-mp-object-size"), }; const size_t g_s3_complete_multipart_upload_excluded_headers_count = @@ -118,6 +129,7 @@ const struct aws_byte_cursor g_s3_complete_multipart_upload_with_checksum_exclud AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-copy-source"), AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-copy-source-range"), AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-sdk-checksum-algorithm"), + AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-mp-object-size"), }; const struct aws_byte_cursor g_s3_list_parts_excluded_headers[] = { @@ -218,6 +230,10 @@ const struct aws_byte_cursor g_s3_abort_multipart_upload_excluded_headers[] = { static const struct aws_byte_cursor s_x_amz_meta_prefix = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-meta-"); +static const struct aws_byte_cursor s_checksum_type_header = + AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-type"); +static const struct aws_byte_cursor s_checksum_type_full_object = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("full_object"); + const size_t g_s3_abort_multipart_upload_excluded_headers_count = AWS_ARRAY_SIZE(g_s3_abort_multipart_upload_excluded_headers); @@ -252,7 +268,7 @@ struct aws_http_message *aws_s3_ranged_get_object_message_new( struct aws_http_message *aws_s3_create_multipart_upload_message_new( struct aws_allocator *allocator, struct aws_http_message *base_message, - const struct checksum_config *checksum_config) { + const struct checksum_config_storage *checksum_config) { AWS_PRECONDITION(allocator); /* For multipart upload, some headers should ONLY be in the initial create-multipart request. @@ -285,12 +301,20 @@ struct aws_http_message *aws_s3_create_multipart_upload_message_new( goto error_clean_up; } } - if (checksum_config && checksum_config->checksum_algorithm && checksum_config->location != AWS_SCL_NONE) { - if (aws_http_headers_set( - headers, - g_create_mpu_checksum_header_name, - *aws_get_create_mpu_header_name_from_algorithm(checksum_config->checksum_algorithm))) { - goto error_clean_up; + if (checksum_config && checksum_config->location != AWS_SCL_NONE) { + if (checksum_config->checksum_algorithm) { + if (aws_http_headers_set( + headers, + g_checksum_algorithm_header_name, + aws_get_checksum_algorithm_name(checksum_config->checksum_algorithm))) { + goto error_clean_up; + } + } + if (checksum_config->has_full_object_checksum) { + /* Request S3 to store the full object checksum as it's set from user. */ + if (aws_http_headers_set(headers, s_checksum_type_header, s_checksum_type_full_object)) { + goto error_clean_up; + } } } @@ -318,7 +342,7 @@ struct aws_http_message *aws_s3_upload_part_message_new( uint32_t part_number, const struct aws_string *upload_id, bool should_compute_content_md5, - const struct checksum_config *checksum_config, + const struct checksum_config_storage *checksum_config, struct aws_byte_buf *encoded_checksum_output) { AWS_PRECONDITION(allocator); AWS_PRECONDITION(base_message); @@ -561,27 +585,30 @@ struct aws_http_message *aws_s3_complete_multipart_message_new( struct aws_byte_buf *body_buffer, const struct aws_string *upload_id, const struct aws_array_list *parts, - const struct checksum_config *checksum_config) { + const struct checksum_config_storage *checksum_config) { AWS_PRECONDITION(allocator); AWS_PRECONDITION(base_message); AWS_PRECONDITION(body_buffer); AWS_PRECONDITION(upload_id); AWS_PRECONDITION(parts); - const struct aws_byte_cursor *mpu_algorithm_checksum_name = NULL; + struct aws_byte_cursor mpu_algorithm_checksum_name; + AWS_ZERO_STRUCT(mpu_algorithm_checksum_name); struct aws_http_message *message = NULL; - - if (checksum_config && checksum_config->location != AWS_SCL_NONE) { - mpu_algorithm_checksum_name = aws_get_complete_mpu_name_from_algorithm(checksum_config->checksum_algorithm); + bool set_checksums = checksum_config && checksum_config->location != AWS_SCL_NONE; + const struct aws_http_headers *initial_message_headers = aws_http_message_get_headers(base_message); + AWS_ASSERT(initial_message_headers); + if (set_checksums) { + mpu_algorithm_checksum_name = + aws_get_completed_part_name_from_checksum_algorithm(checksum_config->checksum_algorithm); message = aws_s3_message_util_copy_http_message_no_body_filter_headers( allocator, base_message, g_s3_complete_multipart_upload_with_checksum_excluded_headers, AWS_ARRAY_SIZE(g_s3_complete_multipart_upload_with_checksum_excluded_headers), true /*exclude_x_amz_meta*/); + } else { - /* We don't need to worry about the pre-calculated checksum from user as for multipart upload, only way to - * calculate checksum is from client. */ message = aws_s3_message_util_copy_http_message_no_body_filter_headers( allocator, base_message, @@ -607,6 +634,28 @@ struct aws_http_message *aws_s3_complete_multipart_message_new( if (headers == NULL) { goto error_clean_up; } + if (set_checksums && checksum_config->has_full_object_checksum) { + /* Set the full object checksum header. */ + AWS_ASSERT(checksum_config->checksum_algorithm != AWS_SCA_NONE); + if (aws_http_headers_set( + headers, + aws_get_http_header_name_from_checksum_algorithm(checksum_config->checksum_algorithm), + aws_byte_cursor_from_buf(&checksum_config->full_object_checksum))) { + goto error_clean_up; + } + /* Request S3 to store the full object checksum as it's set from user. */ + if (aws_http_headers_set(headers, s_checksum_type_header, s_checksum_type_full_object)) { + goto error_clean_up; + } + } + struct aws_byte_cursor content_length_cursor; + if (aws_http_headers_get(initial_message_headers, g_content_length_header_name, &content_length_cursor) == + AWS_OP_SUCCESS) { + /* Set content-length from base message as x-amz-mp-object-size. */ + if (aws_http_headers_set(headers, aws_byte_cursor_from_c_str("x-amz-mp-object-size"), content_length_cursor)) { + goto error_clean_up; + } + } /* Create XML payload with all the etags of finished parts */ { @@ -650,13 +699,13 @@ struct aws_http_message *aws_s3_complete_multipart_message_new( goto error_clean_up; } - if (mpu_algorithm_checksum_name) { + if (mpu_algorithm_checksum_name.len) { struct aws_byte_cursor checksum = aws_byte_cursor_from_buf(&part->checksum_base64); if (aws_byte_buf_append_dynamic(body_buffer, &s_open_start_bracket)) { goto error_clean_up; } - if (aws_byte_buf_append_dynamic(body_buffer, mpu_algorithm_checksum_name)) { + if (aws_byte_buf_append_dynamic(body_buffer, &mpu_algorithm_checksum_name)) { goto error_clean_up; } if (aws_byte_buf_append_dynamic(body_buffer, &s_close_bracket)) { @@ -668,7 +717,7 @@ struct aws_http_message *aws_s3_complete_multipart_message_new( if (aws_byte_buf_append_dynamic(body_buffer, &s_open_end_bracket)) { goto error_clean_up; } - if (aws_byte_buf_append_dynamic(body_buffer, mpu_algorithm_checksum_name)) { + if (aws_byte_buf_append_dynamic(body_buffer, &mpu_algorithm_checksum_name)) { goto error_clean_up; } if (aws_byte_buf_append_dynamic(body_buffer, &s_close_bracket_new_line)) { @@ -684,6 +733,8 @@ struct aws_http_message *aws_s3_complete_multipart_message_new( goto error_clean_up; } + AWS_LOGF_TRACE( + AWS_LS_S3_GENERAL, "Payload for Complete MPU is:\n" PRInSTR "\n", AWS_BYTE_BUF_PRI(*body_buffer)); aws_s3_message_util_assign_body( allocator, body_buffer, message, NULL /* checksum_config */, NULL /* out_checksum */); } @@ -740,14 +791,14 @@ struct aws_http_message *aws_s3_abort_multipart_upload_message_new( static int s_calculate_in_memory_checksum_helper( struct aws_allocator *allocator, struct aws_byte_cursor data, - const struct checksum_config *checksum_config, + const struct checksum_config_storage *checksum_config, struct aws_byte_buf *out_checksum) { AWS_ASSERT(checksum_config->checksum_algorithm != AWS_SCA_NONE); AWS_ASSERT(out_checksum != NULL); AWS_ZERO_STRUCT(*out_checksum); int ret_code = AWS_OP_ERR; - size_t digest_size = aws_get_digest_size_from_algorithm(checksum_config->checksum_algorithm); + size_t digest_size = aws_get_digest_size_from_checksum_algorithm(checksum_config->checksum_algorithm); size_t encoded_checksum_len = 0; if (aws_base64_compute_encoded_len(digest_size, &encoded_checksum_len)) { return AWS_OP_ERR; @@ -758,7 +809,7 @@ static int s_calculate_in_memory_checksum_helper( struct aws_byte_buf raw_checksum; aws_byte_buf_init(&raw_checksum, allocator, digest_size); - if (aws_checksum_compute(allocator, checksum_config->checksum_algorithm, &data, &raw_checksum, 0 /*truncate_to*/)) { + if (aws_checksum_compute(allocator, checksum_config->checksum_algorithm, &data, &raw_checksum)) { goto done; } struct aws_byte_cursor raw_checksum_cursor = aws_byte_cursor_from_buf(&raw_checksum); @@ -783,7 +834,7 @@ static int s_calculate_in_memory_checksum_helper( static int s_calculate_and_add_checksum_to_header_helper( struct aws_allocator *allocator, struct aws_byte_cursor data, - const struct checksum_config *checksum_config, + const struct checksum_config_storage *checksum_config, struct aws_http_message *out_message, struct aws_byte_buf *out_checksum) { AWS_ASSERT(checksum_config->checksum_algorithm != AWS_SCA_NONE); @@ -803,11 +854,11 @@ static int s_calculate_and_add_checksum_to_header_helper( } /* Add the encoded checksum to header. */ - const struct aws_byte_cursor *header_name = - aws_get_http_header_name_from_algorithm(checksum_config->checksum_algorithm); + const struct aws_byte_cursor header_name = + aws_get_http_header_name_from_checksum_algorithm(checksum_config->checksum_algorithm); struct aws_byte_cursor encoded_checksum_val = aws_byte_cursor_from_buf(local_encoded_checksum); struct aws_http_headers *headers = aws_http_message_get_headers(out_message); - if (aws_http_headers_set(headers, *header_name, encoded_checksum_val)) { + if (aws_http_headers_set(headers, header_name, encoded_checksum_val)) { goto done; } @@ -826,7 +877,7 @@ struct aws_input_stream *aws_s3_message_util_assign_body( struct aws_allocator *allocator, struct aws_byte_buf *byte_buf, struct aws_http_message *out_message, - const struct checksum_config *checksum_config, + const struct checksum_config_storage *checksum_config, struct aws_byte_buf *out_checksum) { AWS_PRECONDITION(allocator); AWS_PRECONDITION(out_message); @@ -880,7 +931,7 @@ struct aws_input_stream *aws_s3_message_util_assign_body( if (aws_http_headers_set( headers, g_trailer_header_name, - *aws_get_http_header_name_from_algorithm(checksum_config->checksum_algorithm))) { + aws_get_http_header_name_from_checksum_algorithm(checksum_config->checksum_algorithm))) { goto error_clean_up; } /* set x-amz-decoded-content-length header */ @@ -943,17 +994,6 @@ struct aws_input_stream *aws_s3_message_util_assign_body( return NULL; } -bool aws_s3_message_util_check_checksum_header(struct aws_http_message *message) { - struct aws_http_headers *headers = aws_http_message_get_headers(message); - for (int algorithm = AWS_SCA_INIT; algorithm <= AWS_SCA_END; algorithm++) { - const struct aws_byte_cursor *algorithm_header_name = aws_get_http_header_name_from_algorithm(algorithm); - if (aws_http_headers_has(headers, *algorithm_header_name)) { - return true; - } - } - return false; -} - /* Add a content-md5 header. */ int aws_s3_message_util_add_content_md5_header( struct aws_allocator *allocator, diff --git a/source/s3_util.c b/source/s3_util.c index 93d51e93a..d2dc46d7e 100644 --- a/source/s3_util.c +++ b/source/s3_util.c @@ -43,20 +43,10 @@ const struct aws_byte_cursor g_trailer_header_name = AWS_BYTE_CUR_INIT_FROM_STRI const struct aws_byte_cursor g_request_validation_mode = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-mode"); const struct aws_byte_cursor g_enabled = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("enabled"); -const struct aws_byte_cursor g_create_mpu_checksum_header_name = +const struct aws_byte_cursor g_checksum_algorithm_header_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-algorithm"); -const struct aws_byte_cursor g_crc32c_header_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-crc32c"); -const struct aws_byte_cursor g_crc32_header_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-crc32"); -const struct aws_byte_cursor g_sha1_header_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-sha1"); -const struct aws_byte_cursor g_sha256_header_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-sha256"); -const struct aws_byte_cursor g_crc32c_create_mpu_header_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("CRC32C"); -const struct aws_byte_cursor g_crc32_create_mpu_header_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("CRC32"); -const struct aws_byte_cursor g_sha1_create_mpu_header_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("SHA1"); -const struct aws_byte_cursor g_sha256_create_mpu_header_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("SHA256"); -const struct aws_byte_cursor g_crc32c_complete_mpu_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("ChecksumCRC32C"); -const struct aws_byte_cursor g_crc32_complete_mpu_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("ChecksumCRC32"); -const struct aws_byte_cursor g_sha1_complete_mpu_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("ChecksumSHA1"); -const struct aws_byte_cursor g_sha256_complete_mpu_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("ChecksumSHA256"); +const struct aws_byte_cursor g_sdk_checksum_algorithm_header_name = + AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-sdk-checksum-algorithm"); const struct aws_byte_cursor g_accept_ranges_header_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("accept-ranges"); const struct aws_byte_cursor g_acl_header_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-acl"); const struct aws_byte_cursor g_mp_parts_count_header_name = @@ -738,26 +728,28 @@ int aws_s3_check_headers_for_checksum( return AWS_OP_SUCCESS; } - for (int i = AWS_SCA_INIT; i <= AWS_SCA_END; i++) { - if (!aws_s3_meta_request_checksum_config_has_algorithm(meta_request, i)) { + for (size_t i = 0; i < AWS_ARRAY_SIZE(s_checksum_algo_priority_list); i++) { + enum aws_s3_checksum_algorithm algorithm = s_checksum_algo_priority_list[i]; + if (!aws_s3_meta_request_checksum_config_has_algorithm(meta_request, algorithm)) { /* If user doesn't select this algorithm, skip */ continue; } - const struct aws_byte_cursor *algorithm_header_name = aws_get_http_header_name_from_algorithm(i); + const struct aws_byte_cursor algorithm_header_name = + aws_get_http_header_name_from_checksum_algorithm(algorithm); struct aws_byte_cursor checksum_value; - if (aws_http_headers_get(headers, *algorithm_header_name, &checksum_value) == AWS_OP_SUCCESS) { + if (aws_http_headers_get(headers, algorithm_header_name, &checksum_value) == AWS_OP_SUCCESS) { /* Found the checksum header, keep the header value and initialize the running checksum */ size_t encoded_len = 0; - aws_base64_compute_encoded_len(aws_get_digest_size_from_algorithm(i), &encoded_len); + aws_base64_compute_encoded_len(aws_get_digest_size_from_checksum_algorithm(algorithm), &encoded_len); if (checksum_value.len == encoded_len - 1) { /* encoded_len includes the nullptr length. -1 is the expected length. */ aws_byte_buf_init_copy_from_cursor(out_checksum_buffer, meta_request->allocator, checksum_value); - *out_checksum = aws_checksum_new(meta_request->allocator, i); + *out_checksum = aws_checksum_new(meta_request->allocator, algorithm); if (!*out_checksum) { AWS_LOGF_ERROR( AWS_LS_S3_META_REQUEST, "Could not create checksum for algorithm: %d, due to error code %d (%s)", - i, + algorithm, aws_last_error_or_unknown(), aws_error_str(aws_last_error_or_unknown())); return AWS_OP_ERR; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 353b07318..b531e93f8 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -186,6 +186,8 @@ add_net_test_case(test_s3_round_trip_with_filepath) add_net_test_case(test_s3_round_trip_mpu_with_filepath) add_net_test_case(test_s3_round_trip_with_filepath_no_content_length) add_net_test_case(test_s3_round_trip_mpu_with_filepath_no_content_length) +add_net_test_case(test_s3_round_trip_mpu_multipart_get_full_object_checksum_fc) +add_net_test_case(test_s3_round_trip_mpu_multipart_get_full_object_checksum_fc_header) add_net_test_case(test_s3_chunked_then_unchunked) add_net_test_case(test_s3_cancel_mpu_one_part_completed_fc) @@ -247,7 +249,6 @@ add_net_test_case(sha1_nist_test_case_2) add_net_test_case(sha1_nist_test_case_3) add_net_test_case(sha1_nist_test_case_4) add_net_test_case(sha1_nist_test_case_5) -add_net_test_case(sha1_nist_test_case_5_truncated) add_net_test_case(sha1_nist_test_case_6) add_net_test_case(sha1_test_invalid_buffer) add_net_test_case(sha1_test_oneshot) @@ -258,18 +259,23 @@ add_net_test_case(sha256_nist_test_case_2) add_net_test_case(sha256_nist_test_case_3) add_net_test_case(sha256_nist_test_case_4) add_net_test_case(sha256_nist_test_case_5) -add_net_test_case(sha256_nist_test_case_5_truncated) add_net_test_case(sha256_nist_test_case_6) add_net_test_case(sha256_test_invalid_buffer) add_net_test_case(sha256_test_oneshot) add_net_test_case(sha256_test_invalid_state) +add_test_case(crc64nvme_nist_test_case_1) +add_test_case(crc64nvme_nist_test_case_2) +add_test_case(crc64nvme_nist_test_case_3) +add_test_case(crc64nvme_nist_test_case_4) +add_test_case(crc64nvme_test_invalid_buffer) +add_test_case(crc64nvme_test_invalid_state) + add_test_case(crc32_nist_test_case_1) add_test_case(crc32_nist_test_case_2) add_test_case(crc32_nist_test_case_3) add_test_case(crc32_nist_test_case_4) add_test_case(crc32_nist_test_case_5) -add_test_case(crc32_nist_test_case_5_truncated) add_test_case(crc32_nist_test_case_6) add_test_case(crc32_test_invalid_buffer) add_test_case(crc32_test_oneshot) @@ -280,7 +286,6 @@ add_test_case(crc32c_nist_test_case_2) add_test_case(crc32c_nist_test_case_3) add_test_case(crc32c_nist_test_case_4) add_test_case(crc32c_nist_test_case_5) -add_test_case(crc32c_nist_test_case_5_truncated) add_test_case(crc32c_nist_test_case_6) add_test_case(crc32c_test_invalid_buffer) add_test_case(crc32c_test_oneshot) diff --git a/tests/s3_checksum_stream_test.c b/tests/s3_checksum_stream_test.c index d3d6f689b..e4fb3eac7 100644 --- a/tests/s3_checksum_stream_test.c +++ b/tests/s3_checksum_stream_test.c @@ -17,12 +17,13 @@ static int compare_checksum_stream(struct aws_allocator *allocator, struct aws_b struct aws_byte_buf read_buf; size_t encoded_len = 0; aws_byte_buf_init(&read_buf, allocator, buffer_size); - for (int algorithm = AWS_SCA_INIT; algorithm <= AWS_SCA_END; algorithm++) { - aws_base64_compute_encoded_len(aws_get_digest_size_from_algorithm(algorithm), &encoded_len); - aws_byte_buf_init(&compute_checksum_output, allocator, aws_get_digest_size_from_algorithm(algorithm)); + for (size_t i = 0; i < AWS_ARRAY_SIZE(s_checksum_algo_priority_list); i++) { + enum aws_s3_checksum_algorithm algorithm = s_checksum_algo_priority_list[i]; + aws_base64_compute_encoded_len(aws_get_digest_size_from_checksum_algorithm(algorithm), &encoded_len); + aws_byte_buf_init(&compute_checksum_output, allocator, aws_get_digest_size_from_checksum_algorithm(algorithm)); aws_byte_buf_init(&stream_checksum_output, allocator, encoded_len); aws_byte_buf_init(&compute_encoded_checksum_output, allocator, encoded_len); - aws_checksum_compute(allocator, algorithm, input, &compute_checksum_output, 0); + aws_checksum_compute(allocator, algorithm, input, &compute_checksum_output); struct aws_byte_cursor checksum_result_cursor = aws_byte_cursor_from_buf(&compute_checksum_output); aws_base64_encode(&checksum_result_cursor, &compute_encoded_checksum_output); struct aws_input_stream *cursor_stream = aws_input_stream_new_from_cursor(allocator, input); @@ -64,11 +65,11 @@ static int s_compute_chunk_stream( struct aws_byte_buf *encoded_checksum_output) { struct aws_byte_cursor pre_chunk_cursor = aws_byte_cursor_from_string(pre_chunk); struct aws_byte_cursor final_chunk = aws_byte_cursor_from_string(s_final_chunk); - const struct aws_byte_cursor *checksum_header_name = aws_get_http_header_name_from_algorithm(algorithm); + struct aws_byte_cursor checksum_header_name = aws_get_http_header_name_from_checksum_algorithm(algorithm); struct aws_byte_cursor colon = aws_byte_cursor_from_string(s_colon); struct aws_byte_cursor post_trailer = aws_byte_cursor_from_string(s_post_trailer); struct aws_byte_buf checksum_result; - aws_byte_buf_init(&checksum_result, allocator, aws_get_digest_size_from_algorithm(algorithm)); + aws_byte_buf_init(&checksum_result, allocator, aws_get_digest_size_from_checksum_algorithm(algorithm)); if (aws_byte_buf_append(output, &pre_chunk_cursor)) { return AWS_OP_ERR; } @@ -80,13 +81,13 @@ static int s_compute_chunk_stream( return AWS_OP_ERR; } } - if (aws_byte_buf_append(output, checksum_header_name)) { + if (aws_byte_buf_append(output, &checksum_header_name)) { return AWS_OP_ERR; } if (aws_byte_buf_append(output, &colon)) { return AWS_OP_ERR; } - if (aws_checksum_compute(allocator, algorithm, input, &checksum_result, 0)) { + if (aws_checksum_compute(allocator, algorithm, input, &checksum_result)) { return AWS_OP_ERR; } struct aws_byte_cursor checksum_result_cursor = aws_byte_cursor_from_buf(&checksum_result); @@ -141,9 +142,11 @@ static int compare_chunk_stream( size_t encoded_len = 0; struct aws_byte_buf read_buf; aws_byte_buf_init(&read_buf, allocator, buffer_size); - for (int algorithm = AWS_SCA_INIT; algorithm <= AWS_SCA_END; algorithm++) { - aws_base64_compute_encoded_len(aws_get_digest_size_from_algorithm(algorithm), &encoded_len); - size_t total_len = len_no_checksum + encoded_len + aws_get_http_header_name_from_algorithm(algorithm)->len; + for (size_t i = 0; i < AWS_ARRAY_SIZE(s_checksum_algo_priority_list); i++) { + enum aws_s3_checksum_algorithm algorithm = s_checksum_algo_priority_list[i]; + aws_base64_compute_encoded_len(aws_get_digest_size_from_checksum_algorithm(algorithm), &encoded_len); + size_t total_len = + len_no_checksum + encoded_len + aws_get_http_header_name_from_checksum_algorithm(algorithm).len; aws_byte_buf_init(&computed_encoded_checksum, allocator, encoded_len); aws_byte_buf_init(&compute_chunk_output, allocator, total_len); aws_byte_buf_init(&stream_chunk_output, allocator, total_len); diff --git a/tests/s3_checksums_crc32_tests.c b/tests/s3_checksums_crc32_tests.c index 3c78f049e..134858d60 100644 --- a/tests/s3_checksums_crc32_tests.c +++ b/tests/s3_checksums_crc32_tests.c @@ -9,10 +9,6 @@ #include #define AWS_CRC32_LEN 4 -/* - * these are the NIST test vectors, as compiled here: - * https://www.di-mgt.com.au/sha_testvectors.html - */ static int s_crc32_nist_test_case_1_fn(struct aws_allocator *allocator, void *ctx) { (void)ctx; @@ -81,7 +77,7 @@ static int s_crc32_nist_test_case_5_fn(struct aws_allocator *allocator, void *ct uint8_t output[AWS_CRC32_LEN] = {0}; struct aws_byte_buf output_buf = aws_byte_buf_from_array(output, sizeof(output)); output_buf.len = 0; - ASSERT_SUCCESS(aws_checksum_finalize(checksum, &output_buf, 0)); + ASSERT_SUCCESS(aws_checksum_finalize(checksum, &output_buf)); uint8_t expected[] = {0xdc, 0x25, 0xbf, 0xbc}; struct aws_byte_cursor expected_buf = aws_byte_cursor_from_array(expected, sizeof(expected)); @@ -96,37 +92,6 @@ static int s_crc32_nist_test_case_5_fn(struct aws_allocator *allocator, void *ct AWS_TEST_CASE(crc32_nist_test_case_5, s_crc32_nist_test_case_5_fn) -static int s_crc32_nist_test_case_5_truncated_fn(struct aws_allocator *allocator, void *ctx) { - (void)ctx; - - aws_s3_library_init(allocator); - - struct aws_s3_checksum *checksum = aws_checksum_new(allocator, AWS_SCA_CRC32); - ASSERT_NOT_NULL(checksum); - struct aws_byte_cursor input = aws_byte_cursor_from_c_str("a"); - - for (size_t i = 0; i < 1000000; ++i) { - ASSERT_SUCCESS(aws_checksum_update(checksum, &input)); - } - - uint8_t expected[] = {0xdc, 0x25}; - struct aws_byte_cursor expected_buf = aws_byte_cursor_from_array(expected, sizeof(expected)); - uint8_t output[AWS_CRC32_LEN] = {0}; - struct aws_byte_buf output_buf = aws_byte_buf_from_array(output, expected_buf.len); - output_buf.len = 0; - ASSERT_SUCCESS(aws_checksum_finalize(checksum, &output_buf, 2)); - - ASSERT_BIN_ARRAYS_EQUALS(expected_buf.ptr, expected_buf.len, output_buf.buffer, output_buf.len); - - aws_checksum_destroy(checksum); - - aws_s3_library_clean_up(); - - return AWS_OP_SUCCESS; -} - -AWS_TEST_CASE(crc32_nist_test_case_5_truncated, s_crc32_nist_test_case_5_truncated_fn) - static int s_crc32_nist_test_case_6_fn(struct aws_allocator *allocator, void *ctx) { (void)ctx; @@ -144,7 +109,7 @@ static int s_crc32_nist_test_case_6_fn(struct aws_allocator *allocator, void *ct uint8_t output[AWS_CRC32_LEN] = {0}; struct aws_byte_buf output_buf = aws_byte_buf_from_array(output, sizeof(output)); output_buf.len = 0; - ASSERT_SUCCESS(aws_checksum_finalize(checksum, &output_buf, 0)); + ASSERT_SUCCESS(aws_checksum_finalize(checksum, &output_buf)); uint8_t expected[] = {0x55, 0x1c, 0xbc, 0x00}; @@ -172,7 +137,7 @@ static int s_crc32_test_invalid_buffer_fn(struct aws_allocator *allocator, void struct aws_byte_buf output_buf = aws_byte_buf_from_array(output, sizeof(output)); output_buf.len = 1; - ASSERT_ERROR(AWS_ERROR_SHORT_BUFFER, aws_checksum_compute(allocator, AWS_SCA_CRC32, &input, &output_buf, 0)); + ASSERT_ERROR(AWS_ERROR_SHORT_BUFFER, aws_checksum_compute(allocator, AWS_SCA_CRC32, &input, &output_buf)); aws_s3_library_clean_up(); @@ -195,7 +160,7 @@ static int s_crc32_test_oneshot_fn(struct aws_allocator *allocator, void *ctx) { struct aws_byte_buf output_buf = aws_byte_buf_from_array(output, sizeof(output)); output_buf.len = 0; - ASSERT_SUCCESS(aws_checksum_compute(allocator, AWS_SCA_CRC32, &input, &output_buf, 0)); + ASSERT_SUCCESS(aws_checksum_compute(allocator, AWS_SCA_CRC32, &input, &output_buf)); ASSERT_BIN_ARRAYS_EQUALS(expected, sizeof(expected), output_buf.buffer, output_buf.len); aws_s3_library_clean_up(); @@ -222,9 +187,9 @@ static int s_crc32_test_invalid_state_fn(struct aws_allocator *allocator, void * output_buf.len = 0; ASSERT_SUCCESS(aws_checksum_update(checksum, &input)); - ASSERT_SUCCESS(aws_checksum_finalize(checksum, &output_buf, 0)); + ASSERT_SUCCESS(aws_checksum_finalize(checksum, &output_buf)); ASSERT_ERROR(AWS_ERROR_INVALID_STATE, aws_checksum_update(checksum, &input)); - ASSERT_ERROR(AWS_ERROR_INVALID_STATE, aws_checksum_finalize(checksum, &output_buf, 0)); + ASSERT_ERROR(AWS_ERROR_INVALID_STATE, aws_checksum_finalize(checksum, &output_buf)); aws_checksum_destroy(checksum); diff --git a/tests/s3_checksums_crc32c_tests.c b/tests/s3_checksums_crc32c_tests.c index c97dc2dae..47d29a373 100644 --- a/tests/s3_checksums_crc32c_tests.c +++ b/tests/s3_checksums_crc32c_tests.c @@ -9,10 +9,6 @@ #include #define AWS_CRC32C_LEN 4 -/* - * these are the NIST test vectors, as compiled here: - * https://www.di-mgt.com.au/sha_testvectors.html - */ static int s_crc32c_nist_test_case_1_fn(struct aws_allocator *allocator, void *ctx) { (void)ctx; @@ -81,7 +77,7 @@ static int s_crc32c_nist_test_case_5_fn(struct aws_allocator *allocator, void *c uint8_t output[AWS_CRC32C_LEN] = {0}; struct aws_byte_buf output_buf = aws_byte_buf_from_array(output, sizeof(output)); output_buf.len = 0; - ASSERT_SUCCESS(aws_checksum_finalize(checksum, &output_buf, 0)); + ASSERT_SUCCESS(aws_checksum_finalize(checksum, &output_buf)); uint8_t expected[] = {0x43, 0x6f, 0xe2, 0x40}; struct aws_byte_cursor expected_buf = aws_byte_cursor_from_array(expected, sizeof(expected)); @@ -96,37 +92,6 @@ static int s_crc32c_nist_test_case_5_fn(struct aws_allocator *allocator, void *c AWS_TEST_CASE(crc32c_nist_test_case_5, s_crc32c_nist_test_case_5_fn) -static int s_crc32c_nist_test_case_5_truncated_fn(struct aws_allocator *allocator, void *ctx) { - (void)ctx; - - aws_s3_library_init(allocator); - - struct aws_s3_checksum *checksum = aws_checksum_new(allocator, AWS_SCA_CRC32C); - ASSERT_NOT_NULL(checksum); - struct aws_byte_cursor input = aws_byte_cursor_from_c_str("a"); - - for (size_t i = 0; i < 1000000; ++i) { - ASSERT_SUCCESS(aws_checksum_update(checksum, &input)); - } - - uint8_t expected[] = {0x43, 0x6f}; - struct aws_byte_cursor expected_buf = aws_byte_cursor_from_array(expected, sizeof(expected)); - uint8_t output[AWS_CRC32C_LEN] = {0}; - struct aws_byte_buf output_buf = aws_byte_buf_from_array(output, expected_buf.len); - output_buf.len = 0; - ASSERT_SUCCESS(aws_checksum_finalize(checksum, &output_buf, 2)); - - ASSERT_BIN_ARRAYS_EQUALS(expected_buf.ptr, expected_buf.len, output_buf.buffer, output_buf.len); - - aws_checksum_destroy(checksum); - - aws_s3_library_clean_up(); - - return AWS_OP_SUCCESS; -} - -AWS_TEST_CASE(crc32c_nist_test_case_5_truncated, s_crc32c_nist_test_case_5_truncated_fn) - static int s_crc32c_nist_test_case_6_fn(struct aws_allocator *allocator, void *ctx) { (void)ctx; @@ -144,7 +109,7 @@ static int s_crc32c_nist_test_case_6_fn(struct aws_allocator *allocator, void *c uint8_t output[AWS_CRC32C_LEN] = {0}; struct aws_byte_buf output_buf = aws_byte_buf_from_array(output, sizeof(output)); output_buf.len = 0; - ASSERT_SUCCESS(aws_checksum_finalize(checksum, &output_buf, 0)); + ASSERT_SUCCESS(aws_checksum_finalize(checksum, &output_buf)); uint8_t expected[] = {0x0d, 0xcd, 0x03, 0xc6}; @@ -172,7 +137,7 @@ static int s_crc32c_test_invalid_buffer_fn(struct aws_allocator *allocator, void struct aws_byte_buf output_buf = aws_byte_buf_from_array(output, sizeof(output)); output_buf.len = 1; - ASSERT_ERROR(AWS_ERROR_SHORT_BUFFER, aws_checksum_compute(allocator, AWS_SCA_CRC32C, &input, &output_buf, 0)); + ASSERT_ERROR(AWS_ERROR_SHORT_BUFFER, aws_checksum_compute(allocator, AWS_SCA_CRC32C, &input, &output_buf)); aws_s3_library_clean_up(); @@ -195,7 +160,7 @@ static int s_crc32c_test_oneshot_fn(struct aws_allocator *allocator, void *ctx) struct aws_byte_buf output_buf = aws_byte_buf_from_array(output, sizeof(output)); output_buf.len = 0; - ASSERT_SUCCESS(aws_checksum_compute(allocator, AWS_SCA_CRC32C, &input, &output_buf, 0)); + ASSERT_SUCCESS(aws_checksum_compute(allocator, AWS_SCA_CRC32C, &input, &output_buf)); ASSERT_BIN_ARRAYS_EQUALS(expected, sizeof(expected), output_buf.buffer, output_buf.len); aws_s3_library_clean_up(); @@ -222,9 +187,9 @@ static int s_crc32c_test_invalid_state_fn(struct aws_allocator *allocator, void output_buf.len = 0; ASSERT_SUCCESS(aws_checksum_update(checksum, &input)); - ASSERT_SUCCESS(aws_checksum_finalize(checksum, &output_buf, 0)); + ASSERT_SUCCESS(aws_checksum_finalize(checksum, &output_buf)); ASSERT_ERROR(AWS_ERROR_INVALID_STATE, aws_checksum_update(checksum, &input)); - ASSERT_ERROR(AWS_ERROR_INVALID_STATE, aws_checksum_finalize(checksum, &output_buf, 0)); + ASSERT_ERROR(AWS_ERROR_INVALID_STATE, aws_checksum_finalize(checksum, &output_buf)); aws_checksum_destroy(checksum); diff --git a/tests/s3_checksums_crc64nvme_tests.c b/tests/s3_checksums_crc64nvme_tests.c new file mode 100644 index 000000000..e7e574d8f --- /dev/null +++ b/tests/s3_checksums_crc64nvme_tests.c @@ -0,0 +1,148 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include "aws/s3/private/s3_checksums.h" +#include +#include + +#include +#define AWS_CRC64_LEN sizeof(uint64_t) + +static int s_crc64nvme_nist_test_case_1_fn(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + + struct aws_byte_cursor input = aws_byte_cursor_from_c_str("aaaaaaaaaa"); + uint8_t expected[] = {0x0C, 0x1A, 0x80, 0x03, 0x6D, 0x65, 0xC5, 0x55}; + struct aws_byte_cursor expected_buf = aws_byte_cursor_from_array(expected, sizeof(expected)); + + return s_verify_checksum_test_case(allocator, &input, &expected_buf, aws_checksum_new, AWS_SCA_CRC64NVME); +} + +AWS_TEST_CASE(crc64nvme_nist_test_case_1, s_crc64nvme_nist_test_case_1_fn) + +static int s_crc64nvme_nist_test_case_2_fn(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + + struct aws_byte_cursor input = aws_byte_cursor_from_c_str(""); + uint8_t expected[] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; + struct aws_byte_cursor expected_buf = aws_byte_cursor_from_array(expected, sizeof(expected)); + + return s_verify_checksum_test_case(allocator, &input, &expected_buf, aws_checksum_new, AWS_SCA_CRC64NVME); +} + +AWS_TEST_CASE(crc64nvme_nist_test_case_2, s_crc64nvme_nist_test_case_2_fn) + +static int s_crc64nvme_nist_test_case_3_fn(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + + aws_s3_library_init(allocator); + + struct aws_s3_checksum *checksum = aws_checksum_new(allocator, AWS_SCA_CRC64NVME); + ASSERT_NOT_NULL(checksum); + struct aws_byte_cursor input = aws_byte_cursor_from_c_str("a"); + + for (size_t i = 0; i < 10; ++i) { + ASSERT_SUCCESS(aws_checksum_update(checksum, &input)); + } + + uint8_t output[AWS_CRC64_LEN] = {0}; + struct aws_byte_buf output_buf = aws_byte_buf_from_array(output, sizeof(output)); + output_buf.len = 0; + ASSERT_SUCCESS(aws_checksum_finalize(checksum, &output_buf)); + + uint8_t expected[] = {0x0C, 0x1A, 0x80, 0x03, 0x6D, 0x65, 0xC5, 0x55}; + struct aws_byte_cursor expected_buf = aws_byte_cursor_from_array(expected, sizeof(expected)); + ASSERT_BIN_ARRAYS_EQUALS(expected_buf.ptr, expected_buf.len, output_buf.buffer, output_buf.len); + + aws_checksum_destroy(checksum); + + aws_s3_library_clean_up(); + + return AWS_OP_SUCCESS; +} + +AWS_TEST_CASE(crc64nvme_nist_test_case_3, s_crc64nvme_nist_test_case_3_fn) + +static int s_crc64nvme_nist_test_case_4_fn(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + + aws_s3_library_init(allocator); + + struct aws_s3_checksum *checksum = aws_checksum_new(allocator, AWS_SCA_CRC64NVME); + ASSERT_NOT_NULL(checksum); + struct aws_byte_cursor input = aws_byte_cursor_from_c_str("aa"); + + for (size_t i = 0; i < 5; ++i) { + ASSERT_SUCCESS(aws_checksum_update(checksum, &input)); + } + + uint8_t output[AWS_CRC64_LEN] = {0}; + struct aws_byte_buf output_buf = aws_byte_buf_from_array(output, sizeof(output)); + output_buf.len = 0; + ASSERT_SUCCESS(aws_checksum_finalize(checksum, &output_buf)); + + uint8_t expected[] = {0x0C, 0x1A, 0x80, 0x03, 0x6D, 0x65, 0xC5, 0x55}; + + struct aws_byte_cursor expected_buf = aws_byte_cursor_from_array(expected, sizeof(expected)); + ASSERT_BIN_ARRAYS_EQUALS(expected_buf.ptr, expected_buf.len, output_buf.buffer, output_buf.len); + + aws_checksum_destroy(checksum); + + aws_s3_library_clean_up(); + + return AWS_OP_SUCCESS; +} + +AWS_TEST_CASE(crc64nvme_nist_test_case_4, s_crc64nvme_nist_test_case_4_fn) + +static int s_crc64nvme_test_invalid_buffer_fn(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + + aws_s3_library_init(allocator); + + struct aws_byte_cursor input = aws_byte_cursor_from_c_str("abcdefghbcdefghicdefghijdefghijkefghijklfghij" + "klmghijklmnhijklmnoijklmnopjklmnopqklm" + "nopqrlmnopqrsmnopqrstnopqrstu"); + uint8_t output[AWS_CRC64_LEN] = {0}; + struct aws_byte_buf output_buf = aws_byte_buf_from_array(output, sizeof(output)); + output_buf.len = 1; + + ASSERT_ERROR(AWS_ERROR_SHORT_BUFFER, aws_checksum_compute(allocator, AWS_SCA_CRC64NVME, &input, &output_buf)); + + aws_s3_library_clean_up(); + + return AWS_OP_SUCCESS; +} + +AWS_TEST_CASE(crc64nvme_test_invalid_buffer, s_crc64nvme_test_invalid_buffer_fn) + +static int s_crc64nvme_test_invalid_state_fn(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + + aws_s3_library_init(allocator); + + struct aws_byte_cursor input = aws_byte_cursor_from_c_str("abcdefghbcdefghicdefghijdefghijkefghijklfghij" + "klmghijklmnhijklmnoijklmnopjklmnopqklm" + "nopqrlmnopqrsmnopqrstnopqrstu"); + + struct aws_s3_checksum *checksum = aws_checksum_new(allocator, AWS_SCA_CRC64NVME); + ASSERT_NOT_NULL(checksum); + + uint8_t output[AWS_CRC64_LEN] = {0}; + struct aws_byte_buf output_buf = aws_byte_buf_from_array(output, sizeof(output)); + output_buf.len = 0; + + ASSERT_SUCCESS(aws_checksum_update(checksum, &input)); + ASSERT_SUCCESS(aws_checksum_finalize(checksum, &output_buf)); + ASSERT_ERROR(AWS_ERROR_INVALID_STATE, aws_checksum_update(checksum, &input)); + ASSERT_ERROR(AWS_ERROR_INVALID_STATE, aws_checksum_finalize(checksum, &output_buf)); + + aws_checksum_destroy(checksum); + + aws_s3_library_clean_up(); + + return AWS_OP_SUCCESS; +} + +AWS_TEST_CASE(crc64nvme_test_invalid_state, s_crc64nvme_test_invalid_state_fn) diff --git a/tests/s3_checksums_sha1_tests.c b/tests/s3_checksums_sha1_tests.c index ed6713ad4..8863f7b7f 100644 --- a/tests/s3_checksums_sha1_tests.c +++ b/tests/s3_checksums_sha1_tests.c @@ -92,7 +92,7 @@ static int s_sha1_nist_test_case_5_fn(struct aws_allocator *allocator, void *ctx uint8_t output[AWS_SHA1_LEN] = {0}; struct aws_byte_buf output_buf = aws_byte_buf_from_array(output, sizeof(output)); output_buf.len = 0; - ASSERT_SUCCESS(aws_checksum_finalize(checksum, &output_buf, 0)); + ASSERT_SUCCESS(aws_checksum_finalize(checksum, &output_buf)); uint8_t expected[] = { 0x34, 0xaa, 0x97, 0x3c, 0xd4, 0xc4, 0xda, 0xa4, 0xf6, 0x1e, @@ -110,38 +110,6 @@ static int s_sha1_nist_test_case_5_fn(struct aws_allocator *allocator, void *ctx AWS_TEST_CASE(sha1_nist_test_case_5, s_sha1_nist_test_case_5_fn) -static int s_sha1_nist_test_case_5_truncated_fn(struct aws_allocator *allocator, void *ctx) { - (void)ctx; - - aws_s3_library_init(allocator); - - struct aws_s3_checksum *checksum = aws_checksum_new(allocator, AWS_SCA_SHA1); - ASSERT_NOT_NULL(checksum); - struct aws_byte_cursor input = aws_byte_cursor_from_c_str("a"); - - for (size_t i = 0; i < 1000000; ++i) { - ASSERT_SUCCESS(aws_checksum_update(checksum, &input)); - } - - uint8_t expected[] = { - 0x34, 0xaa, 0x97, 0x3c, 0xd4, 0xc4, 0xda, 0xa4, 0xf6, 0x1e, 0xeb, 0x2b, 0xdb, 0xad, 0x27, 0x31}; - struct aws_byte_cursor expected_buf = aws_byte_cursor_from_array(expected, sizeof(expected)); - uint8_t output[AWS_SHA1_LEN] = {0}; - struct aws_byte_buf output_buf = aws_byte_buf_from_array(output, expected_buf.len); - output_buf.len = 0; - ASSERT_SUCCESS(aws_checksum_finalize(checksum, &output_buf, 16)); - - ASSERT_BIN_ARRAYS_EQUALS(expected_buf.ptr, expected_buf.len, output_buf.buffer, output_buf.len); - - aws_checksum_destroy(checksum); - - aws_s3_library_clean_up(); - - return AWS_OP_SUCCESS; -} - -AWS_TEST_CASE(sha1_nist_test_case_5_truncated, s_sha1_nist_test_case_5_truncated_fn) - static int s_sha1_nist_test_case_6_fn(struct aws_allocator *allocator, void *ctx) { (void)ctx; @@ -159,7 +127,7 @@ static int s_sha1_nist_test_case_6_fn(struct aws_allocator *allocator, void *ctx uint8_t output[AWS_SHA1_LEN] = {0}; struct aws_byte_buf output_buf = aws_byte_buf_from_array(output, sizeof(output)); output_buf.len = 0; - ASSERT_SUCCESS(aws_checksum_finalize(checksum, &output_buf, 0)); + ASSERT_SUCCESS(aws_checksum_finalize(checksum, &output_buf)); uint8_t expected[] = { 0x77, 0x89, 0xf0, 0xc9, 0xef, 0x7b, 0xfc, 0x40, 0xd9, 0x33, @@ -190,7 +158,7 @@ static int s_sha1_test_invalid_buffer_fn(struct aws_allocator *allocator, void * struct aws_byte_buf output_buf = aws_byte_buf_from_array(output, sizeof(output)); output_buf.len = 1; - ASSERT_ERROR(AWS_ERROR_SHORT_BUFFER, aws_checksum_compute(allocator, AWS_SCA_SHA1, &input, &output_buf, 0)); + ASSERT_ERROR(AWS_ERROR_SHORT_BUFFER, aws_checksum_compute(allocator, AWS_SCA_SHA1, &input, &output_buf)); aws_s3_library_clean_up(); @@ -216,7 +184,7 @@ static int s_sha1_test_oneshot_fn(struct aws_allocator *allocator, void *ctx) { struct aws_byte_buf output_buf = aws_byte_buf_from_array(output, sizeof(output)); output_buf.len = 0; - ASSERT_SUCCESS(aws_checksum_compute(allocator, AWS_SCA_SHA1, &input, &output_buf, 0)); + ASSERT_SUCCESS(aws_checksum_compute(allocator, AWS_SCA_SHA1, &input, &output_buf)); ASSERT_BIN_ARRAYS_EQUALS(expected, sizeof(expected), output_buf.buffer, output_buf.len); aws_s3_library_clean_up(); @@ -243,9 +211,9 @@ static int s_sha1_test_invalid_state_fn(struct aws_allocator *allocator, void *c output_buf.len = 0; ASSERT_SUCCESS(aws_checksum_update(checksum, &input)); - ASSERT_SUCCESS(aws_checksum_finalize(checksum, &output_buf, 0)); + ASSERT_SUCCESS(aws_checksum_finalize(checksum, &output_buf)); ASSERT_ERROR(AWS_ERROR_INVALID_STATE, aws_checksum_update(checksum, &input)); - ASSERT_ERROR(AWS_ERROR_INVALID_STATE, aws_checksum_finalize(checksum, &output_buf, 0)); + ASSERT_ERROR(AWS_ERROR_INVALID_STATE, aws_checksum_finalize(checksum, &output_buf)); aws_checksum_destroy(checksum); diff --git a/tests/s3_checksums_sha256_tests.c b/tests/s3_checksums_sha256_tests.c index bde11ff01..f067d150c 100644 --- a/tests/s3_checksums_sha256_tests.c +++ b/tests/s3_checksums_sha256_tests.c @@ -92,7 +92,7 @@ static int s_sha256_nist_test_case_5_fn(struct aws_allocator *allocator, void *c uint8_t output[AWS_SHA256_LEN] = {0}; struct aws_byte_buf output_buf = aws_byte_buf_from_array(output, sizeof(output)); output_buf.len = 0; - ASSERT_SUCCESS(aws_checksum_finalize(checksum, &output_buf, 0)); + ASSERT_SUCCESS(aws_checksum_finalize(checksum, &output_buf)); uint8_t expected[] = { 0xcd, 0xc7, 0x6e, 0x5c, 0x99, 0x14, 0xfb, 0x92, 0x81, 0xa1, 0xc7, 0xe2, 0x84, 0xd7, 0x3e, 0x67, @@ -110,54 +110,6 @@ static int s_sha256_nist_test_case_5_fn(struct aws_allocator *allocator, void *c AWS_TEST_CASE(sha256_nist_test_case_5, s_sha256_nist_test_case_5_fn) -static int s_sha256_nist_test_case_5_truncated_fn(struct aws_allocator *allocator, void *ctx) { - (void)ctx; - - aws_s3_library_init(allocator); - - struct aws_s3_checksum *checksum = aws_checksum_new(allocator, AWS_SCA_SHA256); - ASSERT_NOT_NULL(checksum); - struct aws_byte_cursor input = aws_byte_cursor_from_c_str("a"); - - for (size_t i = 0; i < 1000000; ++i) { - ASSERT_SUCCESS(aws_checksum_update(checksum, &input)); - } - - uint8_t expected[] = { - 0xcd, - 0xc7, - 0x6e, - 0x5c, - 0x99, - 0x14, - 0xfb, - 0x92, - 0x81, - 0xa1, - 0xc7, - 0xe2, - 0x84, - 0xd7, - 0x3e, - 0x67, - }; - struct aws_byte_cursor expected_buf = aws_byte_cursor_from_array(expected, sizeof(expected)); - uint8_t output[AWS_SHA256_LEN] = {0}; - struct aws_byte_buf output_buf = aws_byte_buf_from_array(output, expected_buf.len); - output_buf.len = 0; - ASSERT_SUCCESS(aws_checksum_finalize(checksum, &output_buf, 16)); - - ASSERT_BIN_ARRAYS_EQUALS(expected_buf.ptr, expected_buf.len, output_buf.buffer, output_buf.len); - - aws_checksum_destroy(checksum); - - aws_s3_library_clean_up(); - - return AWS_OP_SUCCESS; -} - -AWS_TEST_CASE(sha256_nist_test_case_5_truncated, s_sha256_nist_test_case_5_truncated_fn) - static int s_sha256_nist_test_case_6_fn(struct aws_allocator *allocator, void *ctx) { (void)ctx; @@ -175,7 +127,7 @@ static int s_sha256_nist_test_case_6_fn(struct aws_allocator *allocator, void *c uint8_t output[AWS_SHA256_LEN] = {0}; struct aws_byte_buf output_buf = aws_byte_buf_from_array(output, sizeof(output)); output_buf.len = 0; - ASSERT_SUCCESS(aws_checksum_finalize(checksum, &output_buf, 0)); + ASSERT_SUCCESS(aws_checksum_finalize(checksum, &output_buf)); uint8_t expected[] = { 0x50, 0xe7, 0x2a, 0x0e, 0x26, 0x44, 0x2f, 0xe2, 0x55, 0x2d, 0xc3, 0x93, 0x8a, 0xc5, 0x86, 0x58, @@ -206,7 +158,7 @@ static int s_sha256_test_invalid_buffer_fn(struct aws_allocator *allocator, void struct aws_byte_buf output_buf = aws_byte_buf_from_array(output, sizeof(output)); output_buf.len = 1; - ASSERT_ERROR(AWS_ERROR_SHORT_BUFFER, aws_checksum_compute(allocator, AWS_SCA_SHA256, &input, &output_buf, 0)); + ASSERT_ERROR(AWS_ERROR_SHORT_BUFFER, aws_checksum_compute(allocator, AWS_SCA_SHA256, &input, &output_buf)); aws_s3_library_clean_up(); @@ -232,7 +184,7 @@ static int s_sha256_test_oneshot_fn(struct aws_allocator *allocator, void *ctx) struct aws_byte_buf output_buf = aws_byte_buf_from_array(output, sizeof(output)); output_buf.len = 0; - ASSERT_SUCCESS(aws_checksum_compute(allocator, AWS_SCA_SHA256, &input, &output_buf, 0)); + ASSERT_SUCCESS(aws_checksum_compute(allocator, AWS_SCA_SHA256, &input, &output_buf)); ASSERT_BIN_ARRAYS_EQUALS(expected, sizeof(expected), output_buf.buffer, output_buf.len); aws_s3_library_clean_up(); @@ -259,9 +211,9 @@ static int s_sha256_test_invalid_state_fn(struct aws_allocator *allocator, void output_buf.len = 0; ASSERT_SUCCESS(aws_checksum_update(checksum, &input)); - ASSERT_SUCCESS(aws_checksum_finalize(checksum, &output_buf, 0)); + ASSERT_SUCCESS(aws_checksum_finalize(checksum, &output_buf)); ASSERT_ERROR(AWS_ERROR_INVALID_STATE, aws_checksum_update(checksum, &input)); - ASSERT_ERROR(AWS_ERROR_INVALID_STATE, aws_checksum_finalize(checksum, &output_buf, 0)); + ASSERT_ERROR(AWS_ERROR_INVALID_STATE, aws_checksum_finalize(checksum, &output_buf)); aws_checksum_destroy(checksum); diff --git a/tests/s3_checksums_test_case_helper.h b/tests/s3_checksums_test_case_helper.h index bf2432785..e4625492a 100644 --- a/tests/s3_checksums_test_case_helper.h +++ b/tests/s3_checksums_test_case_helper.h @@ -37,9 +37,7 @@ static inline int s_verify_checksum_test_case( aws_byte_cursor_advance(&input_cpy, max_advance); } - size_t truncation_size = checksum->digest_size - expected->len; - - ASSERT_SUCCESS(aws_checksum_finalize(checksum, &output_buf, truncation_size)); + ASSERT_SUCCESS(aws_checksum_finalize(checksum, &output_buf)); ASSERT_BIN_ARRAYS_EQUALS(expected->ptr, expected->len, output_buf.buffer, output_buf.len); aws_checksum_destroy(checksum); diff --git a/tests/s3_data_plane_tests.c b/tests/s3_data_plane_tests.c index c2ab1c62d..ea7218f1d 100644 --- a/tests/s3_data_plane_tests.c +++ b/tests/s3_data_plane_tests.c @@ -3252,6 +3252,7 @@ static int s_test_s3_put_object_sse_aes256_multipart(struct aws_allocator *alloc AWS_TEST_CASE(test_s3_put_object_sse_c_aes256_multipart, s_test_s3_put_object_sse_c_aes256_multipart) static int s_test_s3_put_object_sse_c_aes256_multipart(struct aws_allocator *allocator, void *ctx) { (void)ctx; + (void)allocator; struct aws_s3_tester tester; ASSERT_SUCCESS(aws_s3_tester_init(allocator, &tester)); @@ -3932,7 +3933,8 @@ static int s_test_s3_round_trip_default_get_fc_helper(struct aws_allocator *allo struct aws_byte_buf path_buf; AWS_ZERO_STRUCT(path_buf); - for (int algorithm = AWS_SCA_INIT; algorithm <= AWS_SCA_END; ++algorithm) { + for (size_t i = 0; i < AWS_ARRAY_SIZE(s_checksum_algo_priority_list); i++) { + enum aws_s3_checksum_algorithm algorithm = s_checksum_algo_priority_list[i]; char object_path_sprintf_buffer[128] = ""; snprintf( object_path_sprintf_buffer, @@ -4022,7 +4024,7 @@ static int s_test_s3_round_trip_multipart_get_fc_helper(struct aws_allocator *al .allocator = allocator, .meta_request_type = AWS_S3_META_REQUEST_TYPE_PUT_OBJECT, .client = client, - .checksum_algorithm = AWS_SCA_CRC32, + .checksum_algorithm = AWS_SCA_CRC64NVME, .validate_get_response_checksum = false, .checksum_via_header = via_header, .put_options = @@ -4042,7 +4044,7 @@ static int s_test_s3_round_trip_multipart_get_fc_helper(struct aws_allocator *al .validate_type = AWS_S3_TESTER_VALIDATE_TYPE_EXPECT_SUCCESS, .client = client, .validate_get_response_checksum = true, - .expected_validate_checksum_alg = AWS_SCA_CRC32, + .expected_validate_checksum_alg = AWS_SCA_CRC64NVME, .get_options = { .object_path = object_path, @@ -4074,7 +4076,8 @@ static int s_test_s3_round_trip_multipart_get_fc_header(struct aws_allocator *al static int s_test_s3_round_trip_mpu_multipart_get_fc_helper( struct aws_allocator *allocator, void *ctx, - bool via_header) { + bool via_header, + bool add_full_object_checksum_via_header) { (void)ctx; struct aws_s3_tester tester; @@ -4088,9 +4091,12 @@ static int s_test_s3_round_trip_mpu_multipart_get_fc_helper( struct aws_byte_buf path_buf; AWS_ZERO_STRUCT(path_buf); + struct aws_byte_cursor object_name = + add_full_object_checksum_via_header + ? aws_byte_cursor_from_c_str("/prefix/round_trip/test_mpu_fc_full_object.txt") + : aws_byte_cursor_from_c_str("/prefix/round_trip/test_mpu_fc.txt"); - ASSERT_SUCCESS(aws_s3_tester_upload_file_path_init( - allocator, &path_buf, aws_byte_cursor_from_c_str("/prefix/round_trip/test_mpu_fc.txt"))); + ASSERT_SUCCESS(aws_s3_tester_upload_file_path_init(allocator, &path_buf, object_name)); struct aws_byte_cursor object_path = aws_byte_cursor_from_buf(&path_buf); @@ -4105,6 +4111,7 @@ static int s_test_s3_round_trip_mpu_multipart_get_fc_helper( { .object_size_mb = 10, .object_path_override = object_path, + .add_full_object_checksum_via_header = add_full_object_checksum_via_header, }, }; @@ -4137,12 +4144,27 @@ static int s_test_s3_round_trip_mpu_multipart_get_fc_helper( AWS_TEST_CASE(test_s3_round_trip_mpu_multipart_get_fc, s_test_s3_round_trip_mpu_multipart_get_fc) static int s_test_s3_round_trip_mpu_multipart_get_fc(struct aws_allocator *allocator, void *ctx) { - return s_test_s3_round_trip_mpu_multipart_get_fc_helper(allocator, ctx, false); + return s_test_s3_round_trip_mpu_multipart_get_fc_helper(allocator, ctx, false, false); } AWS_TEST_CASE(test_s3_round_trip_mpu_multipart_get_fc_header, s_test_s3_round_trip_mpu_multipart_get_fc_header) static int s_test_s3_round_trip_mpu_multipart_get_fc_header(struct aws_allocator *allocator, void *ctx) { - return s_test_s3_round_trip_mpu_multipart_get_fc_helper(allocator, ctx, true); + return s_test_s3_round_trip_mpu_multipart_get_fc_helper(allocator, ctx, true, false); +} + +AWS_TEST_CASE( + test_s3_round_trip_mpu_multipart_get_full_object_checksum_fc, + s_test_s3_round_trip_mpu_multipart_get_full_object_checksum_fc) +static int s_test_s3_round_trip_mpu_multipart_get_full_object_checksum_fc(struct aws_allocator *allocator, void *ctx) { + return s_test_s3_round_trip_mpu_multipart_get_fc_helper(allocator, ctx, false, true); +} +AWS_TEST_CASE( + test_s3_round_trip_mpu_multipart_get_full_object_checksum_fc_header, + s_test_s3_round_trip_mpu_multipart_get_full_object_checksum_fc_header) +static int s_test_s3_round_trip_mpu_multipart_get_full_object_checksum_fc_header( + struct aws_allocator *allocator, + void *ctx) { + return s_test_s3_round_trip_mpu_multipart_get_fc_helper(allocator, ctx, true, true); } static int s_test_s3_download_empty_file_with_checksum_helper( @@ -6968,7 +6990,7 @@ static int s_pause_resume_upload_review_callback( struct aws_byte_buf checksum_buf; aws_byte_buf_init(&checksum_buf, allocator, 128); ASSERT_SUCCESS( - aws_checksum_compute(allocator, review->checksum_algorithm, &reread_part_cursor, &checksum_buf, 0)); + aws_checksum_compute(allocator, review->checksum_algorithm, &reread_part_cursor, &checksum_buf)); struct aws_byte_cursor checksum_cursor = aws_byte_cursor_from_buf(&checksum_buf); struct aws_byte_buf encoded_checksum_buf; @@ -7747,19 +7769,20 @@ static int s_test_s3_upload_review_checksum_location_none(struct aws_allocator * ASSERT_STR_EQUALS("7/xUXw==", aws_string_c_str(test_results.upload_review.part_checksums_array[0])); ASSERT_STR_EQUALS("PCOjcw==", aws_string_c_str(test_results.upload_review.part_checksums_array[1])); - /* Get the file, which should not have checksums present to validate */ + /* S3 will store the crc64 checksum for the whole object, and we can still have validate the checksum, but the algo + * be validated will be crc64, instead of the crc32 we get from the client. */ struct aws_s3_tester_meta_request_options get_options = { .allocator = allocator, .meta_request_type = AWS_S3_META_REQUEST_TYPE_GET_OBJECT, .validate_type = AWS_S3_TESTER_VALIDATE_TYPE_EXPECT_SUCCESS, .client = client, - .expected_validate_checksum_alg = AWS_SCA_CRC32, + .expected_validate_checksum_alg = AWS_SCA_CRC64NVME, .validate_get_response_checksum = true, .get_options = { .object_path = object_path, }, - .finish_callback = s_s3_test_no_validate_checksum, + .finish_callback = s_s3_test_validate_checksum, }; ASSERT_SUCCESS(aws_s3_tester_send_meta_request_with_options(&tester, &get_options, NULL)); diff --git a/tests/s3_list_objects_tests.c b/tests/s3_list_objects_tests.c index 436fdecf0..a4a8df099 100644 --- a/tests/s3_list_objects_tests.c +++ b/tests/s3_list_objects_tests.c @@ -151,7 +151,10 @@ static int s_test_s3_list_bucket_valid(struct aws_allocator *allocator, void *ct ASSERT_SUCCESS(aws_array_list_init_dynamic(&test_data.entries_found, allocator, 16, sizeof(struct aws_string *))); - struct aws_byte_cursor endpoint = aws_byte_cursor_from_c_str("s3.us-west-2.amazonaws.com"); + char endpoint_buff[128] = ""; + snprintf( + endpoint_buff, sizeof(endpoint_buff), "s3." PRInSTR ".amazonaws.com", AWS_BYTE_CURSOR_PRI(g_test_s3_region)); + struct aws_byte_cursor endpoint = aws_byte_cursor_from_c_str(endpoint_buff); struct aws_s3_list_objects_params params = { .client = client, diff --git a/tests/s3_request_messages_tests.c b/tests/s3_request_messages_tests.c index 01219d16a..eef4d4cf2 100644 --- a/tests/s3_request_messages_tests.c +++ b/tests/s3_request_messages_tests.c @@ -176,6 +176,10 @@ static const struct aws_http_header s_put_object_test_headers[] = { .name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-expected-bucket-owner"), .value = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("ExpectedBucketOwnerValue"), }, + { + .name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-checksum-crc32"), + .value = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("AAAAAA=="), + }, }; static int s_fill_byte_buf(struct aws_byte_buf *buffer, struct aws_allocator *allocator, size_t buffer_size) { @@ -862,6 +866,7 @@ static int s_test_s3_complete_multipart_message_new(struct aws_allocator *alloca const struct aws_byte_cursor header_exclude_exceptions[] = { AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("Content-Length"), + AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-mp-object-size"), }; struct aws_byte_cursor path = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL(TEST_PATH); diff --git a/tests/s3_tester.c b/tests/s3_tester.c index d879f3857..e8b3caef8 100644 --- a/tests/s3_tester.c +++ b/tests/s3_tester.c @@ -10,6 +10,7 @@ #include "aws/s3/private/s3_meta_request_impl.h" #include "aws/s3/private/s3_util.h" #include +#include #include #include #include @@ -1301,6 +1302,43 @@ struct aws_http_message *aws_s3_test_put_object_request_new_without_body( return NULL; } +/** + * Calculate the in memory checksum based on the checksum config. Initialize and set the out_checksum to the encoded + * checksum result + */ +static int s_calculate_in_memory_checksum_helper( + struct aws_allocator *allocator, + struct aws_byte_cursor data, + enum aws_s3_checksum_algorithm algo, + struct aws_byte_buf *out_encoded_checksum) { + AWS_ASSERT(out_encoded_checksum != NULL); + + int ret_code = AWS_OP_ERR; + size_t digest_size = aws_get_digest_size_from_checksum_algorithm(algo); + size_t encoded_checksum_len = 0; + if (aws_base64_compute_encoded_len(digest_size, &encoded_checksum_len)) { + return ret_code; + } + + aws_byte_buf_init(out_encoded_checksum, allocator, encoded_checksum_len); + + struct aws_byte_buf raw_checksum; + aws_byte_buf_init(&raw_checksum, allocator, digest_size); + + if (aws_checksum_compute(allocator, algo, &data, &raw_checksum)) { + goto done; + } + struct aws_byte_cursor raw_checksum_cursor = aws_byte_cursor_from_buf(&raw_checksum); + if (aws_base64_encode(&raw_checksum_cursor, out_encoded_checksum)) { + goto done; + } + + ret_code = AWS_OP_SUCCESS; +done: + aws_byte_buf_clean_up(&raw_checksum); + return ret_code; +} + struct aws_http_message *aws_s3_test_put_object_request_new( struct aws_allocator *allocator, struct aws_byte_cursor *host, @@ -1696,6 +1734,29 @@ int aws_s3_tester_send_meta_request_with_options( aws_http_message_add_header(message, content_encoding_header); } + if (options->put_options.add_full_object_checksum_via_header) { + struct aws_http_headers *headers = aws_http_message_get_headers(message); + ASSERT_NOT_NULL(input_stream); + struct aws_byte_buf data; + int64_t out_length = 0; + aws_input_stream_get_length(input_stream, &out_length); + aws_byte_buf_init(&data, allocator, (size_t)out_length); + /* Read everything into the buf */ + aws_input_stream_read(input_stream, &data); + /* Seek back to beginning for upload. */ + aws_input_stream_seek(input_stream, 0, AWS_SSB_BEGIN); + /* Get the checksum from the buf */ + struct aws_byte_buf out_encoded_checksum; + ASSERT_SUCCESS(s_calculate_in_memory_checksum_helper( + allocator, aws_byte_cursor_from_buf(&data), options->checksum_algorithm, &out_encoded_checksum)); + /* Set the header */ + const struct aws_byte_cursor header_name = + aws_get_http_header_name_from_checksum_algorithm(options->checksum_algorithm); + ASSERT_SUCCESS( + aws_http_headers_set(headers, header_name, aws_byte_cursor_from_buf(&out_encoded_checksum))); + aws_byte_buf_clean_up(&data); + aws_byte_buf_clean_up(&out_encoded_checksum); + } if (options->put_options.if_none_match_header.ptr != NULL) { struct aws_http_header if_none_match_header = { .name = aws_byte_cursor_from_c_str("if-none-match"), @@ -1766,7 +1827,7 @@ int aws_s3_tester_send_meta_request_with_options( ASSERT_SUCCESS(aws_s3_tester_validate_put_object_results(out_results, options->sse_type)); /* Expected number of bytes should have been read from stream, and reported via progress callbacks */ - if (input_stream != NULL) { + if (input_stream != NULL && !options->put_options.add_full_object_checksum_via_header) { ASSERT_UINT_EQUALS(upload_size_bytes, aws_input_stream_tester_total_bytes_read(input_stream)); } else if (async_stream != NULL) { ASSERT_UINT_EQUALS(upload_size_bytes, aws_async_input_stream_tester_total_bytes_read(async_stream)); @@ -2382,6 +2443,7 @@ int aws_test_s3_copy_object_from_x_amz_copy_source( struct aws_s3_client_config client_config; AWS_ZERO_STRUCT(client_config); client_config.enable_s3express = s3express; + struct aws_byte_cursor region_cursor = g_test_s3_region; client_config.region = region_cursor; ASSERT_SUCCESS(aws_s3_tester_bind_client(&tester, &client_config, AWS_S3_TESTER_BIND_CLIENT_SIGNING)); diff --git a/tests/s3_tester.h b/tests/s3_tester.h index 03b1f1f6c..72aa164c0 100644 --- a/tests/s3_tester.h +++ b/tests/s3_tester.h @@ -219,6 +219,7 @@ struct aws_s3_tester_meta_request_options { size_t content_length; bool skip_content_length; struct aws_byte_cursor content_encoding; + bool add_full_object_checksum_via_header; struct aws_byte_cursor if_none_match_header; } put_options;