diff --git a/include/aws/s3/s3_client.h b/include/aws/s3/s3_client.h index 3609b343..c765d79d 100644 --- a/include/aws/s3/s3_client.h +++ b/include/aws/s3/s3_client.h @@ -559,19 +559,19 @@ struct aws_s3_checksum_config { /** * The location of client added checksum header. * - * If AWS_SCL_NONE. No request payload checksum will be add and calculated. + * If AWS_SCL_NONE. No request payload checksum will be calculated or added. * - * If AWS_SCL_HEADER, the checksum will be calculated by client and added related header to the request sent. + * If AWS_SCL_HEADER, the client will calculate the checksum and add it to the headers. * - * If AWS_SCL_TRAILER, the payload will be aws_chunked encoded, The checksum will be calculate while reading the - * payload by client. Related header will be added to the trailer part of the encoded payload. Note the payload of - * the original request cannot be aws-chunked encoded already. Otherwise, error will be raised. + * If AWS_SCL_TRAILER, the payload will be aws_chunked encoded, The client will calculate the checksum and add it to + * the trailer. Note the payload of the original request cannot be aws-chunked encoded already, this will cause an + * error. */ enum aws_s3_checksum_location location; /** * The checksum algorithm used. - * Must be set if location is not AWS_SCL_NONE. Must be AWS_SCA_NONE if location is AWS_SCL_NONE. + * Must be set if location is not AWS_SCL_NONE. */ enum aws_s3_checksum_algorithm checksum_algorithm; diff --git a/source/s3_client.c b/source/s3_client.c index 85066027..c1f2fcf7 100644 --- a/source/s3_client.c +++ b/source/s3_client.c @@ -963,12 +963,6 @@ struct aws_s3_meta_request *aws_s3_client_make_meta_request( } } - if (options->checksum_config->location == AWS_SCL_HEADER) { - /* TODO: support calculate checksum to add to header */ - aws_raise_error(AWS_ERROR_UNSUPPORTED_OPERATION); - return NULL; - } - if (options->checksum_config->location != AWS_SCL_NONE && options->checksum_config->checksum_algorithm == AWS_SCA_NONE) { AWS_LOGF_ERROR( diff --git a/source/s3_request_messages.c b/source/s3_request_messages.c index 0551b3a9..4b678df2 100644 --- a/source/s3_request_messages.c +++ b/source/s3_request_messages.c @@ -730,6 +730,94 @@ struct aws_http_message *aws_s3_abort_multipart_upload_message_new( return NULL; } +/** + * Calculate the in memory checksum based on the checksum config. Initialize and set the out_checksum to the encoded + * checksum result + */ +static int s_calculate_in_memory_checksum_helper( + struct aws_allocator *allocator, + struct aws_byte_cursor data, + const struct checksum_config *checksum_config, + struct aws_byte_buf *out_checksum) { + AWS_ASSERT(checksum_config->checksum_algorithm != AWS_SCA_NONE); + AWS_ASSERT(out_checksum != NULL); + AWS_ZERO_STRUCT(*out_checksum); + + int ret_code = AWS_OP_ERR; + size_t digest_size = aws_get_digest_size_from_algorithm(checksum_config->checksum_algorithm); + size_t encoded_checksum_len = 0; + if (aws_base64_compute_encoded_len(digest_size, &encoded_checksum_len)) { + return AWS_OP_ERR; + } + + aws_byte_buf_init(out_checksum, allocator, encoded_checksum_len); + + struct aws_byte_buf raw_checksum; + aws_byte_buf_init(&raw_checksum, allocator, digest_size); + + if (aws_checksum_compute(allocator, checksum_config->checksum_algorithm, &data, &raw_checksum, 0 /*truncate_to*/)) { + goto done; + } + struct aws_byte_cursor raw_checksum_cursor = aws_byte_cursor_from_buf(&raw_checksum); + if (aws_base64_encode(&raw_checksum_cursor, out_checksum)) { + goto done; + } + + ret_code = AWS_OP_SUCCESS; +done: + if (ret_code) { + aws_byte_buf_clean_up(out_checksum); + } + aws_byte_buf_clean_up(&raw_checksum); + return ret_code; +} + +/** + * Calculate the in memory checksum based on the checksum config. + * If out_checksum set, initialize and set it to the encoded checksum result. + * Set the corresponding header in out_message to the encoded checksum result. + */ +static int s_calculate_and_add_checksum_to_header_helper( + struct aws_allocator *allocator, + struct aws_byte_cursor data, + const struct checksum_config *checksum_config, + struct aws_http_message *out_message, + struct aws_byte_buf *out_checksum) { + AWS_ASSERT(checksum_config->checksum_algorithm != AWS_SCA_NONE); + AWS_ASSERT(out_message != NULL); + int ret_code = AWS_OP_ERR; + + struct aws_byte_buf local_encoded_checksum_buf; + struct aws_byte_buf *local_encoded_checksum; + if (out_checksum == NULL) { + local_encoded_checksum = &local_encoded_checksum_buf; + } else { + local_encoded_checksum = out_checksum; + } + AWS_ZERO_STRUCT(*local_encoded_checksum); + if (s_calculate_in_memory_checksum_helper(allocator, data, checksum_config, local_encoded_checksum)) { + goto done; + } + + /* Add the encoded checksum to header. */ + const struct aws_byte_cursor *header_name = + aws_get_http_header_name_from_algorithm(checksum_config->checksum_algorithm); + struct aws_byte_cursor encoded_checksum_val = aws_byte_cursor_from_buf(local_encoded_checksum); + struct aws_http_headers *headers = aws_http_message_get_headers(out_message); + if (aws_http_headers_set(headers, *header_name, encoded_checksum_val)) { + goto done; + } + + ret_code = AWS_OP_SUCCESS; +done: + if (ret_code || out_checksum == NULL) { + /* In case of error happen or out_checksum is not set, clean up the encoded checksum. Otherwise, the caller will + * own the encoded checksum. */ + aws_byte_buf_clean_up(local_encoded_checksum); + } + return ret_code; +} + /* Assign a buffer to an HTTP message, creating a stream and setting the content-length header */ struct aws_input_stream *aws_s3_message_util_assign_body( struct aws_allocator *allocator, @@ -812,25 +900,19 @@ struct aws_input_stream *aws_s3_message_util_assign_body( } aws_input_stream_release(input_stream); input_stream = chunk_stream; - } else if ( - checksum_config->checksum_algorithm != AWS_SCA_NONE && checksum_config->location == AWS_SCL_NONE && - out_checksum != NULL) { - /* The checksum won't be uploaded, but we still need it for the upload review callback */ - size_t checksum_len = aws_get_digest_size_from_algorithm(checksum_config->checksum_algorithm); - size_t encoded_checksum_len = 0; - if (aws_base64_compute_encoded_len(checksum_len, &encoded_checksum_len)) { + } else if (checksum_config->location == AWS_SCL_HEADER) { + /* Calculate the checksum directly from memory and add it to the header. */ + if (s_calculate_and_add_checksum_to_header_helper( + allocator, buffer_byte_cursor, checksum_config, out_message, out_checksum)) { goto error_clean_up; } - if (aws_byte_buf_init(out_checksum, allocator, encoded_checksum_len)) { - goto error_clean_up; - } - struct aws_input_stream *checksum_stream = - aws_checksum_stream_new(allocator, input_stream, checksum_config->checksum_algorithm, out_checksum); - if (!checksum_stream) { + + } else if (checksum_config->checksum_algorithm != AWS_SCA_NONE && out_checksum != NULL) { + /* In case checksums still wanted, and we can calculate it directly from the buffer in memory to + * out_checksum */ + if (s_calculate_in_memory_checksum_helper(allocator, buffer_byte_cursor, checksum_config, out_checksum)) { goto error_clean_up; } - aws_input_stream_release(input_stream); - input_stream = checksum_stream; } } int64_t stream_length = 0; @@ -892,9 +974,7 @@ int aws_s3_message_util_add_content_md5_header( return AWS_OP_ERR; } struct aws_byte_buf base64_output_buf; - if (aws_byte_buf_init(&base64_output_buf, allocator, base64_output_size)) { - return AWS_OP_ERR; - } + aws_byte_buf_init(&base64_output_buf, allocator, base64_output_size); if (aws_base64_encode(&base64_input, &base64_output_buf)) { goto error_clean_up; } @@ -1056,9 +1136,7 @@ int aws_s3_message_util_set_multipart_request_path( return AWS_OP_ERR; } - if (aws_byte_buf_init(&request_path_buf, allocator, request_path.len)) { - return AWS_OP_ERR; - } + aws_byte_buf_init(&request_path_buf, allocator, request_path.len); if (aws_byte_buf_append_dynamic(&request_path_buf, &request_path)) { goto error_clean_up; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d3403244..b3466fb9 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -142,6 +142,7 @@ add_net_test_case(test_s3_put_object_async_no_content_length_2parts) add_net_test_case(test_s3_put_object_async_fail_reading) add_net_test_case(test_s3_many_async_uploads_without_data) add_net_test_case(test_s3_download_empty_file_with_checksum) +add_net_test_case(test_s3_download_empty_file_with_checksum_header) add_net_test_case(test_s3_download_single_part_file_with_checksum) add_net_test_case(test_s3_download_multipart_file_with_checksum) add_net_test_case(test_s3_asyncwrite_empty_file) @@ -176,6 +177,9 @@ add_net_test_case(test_s3_round_trip_default_get_fc) add_net_test_case(test_s3_round_trip_mpu_multipart_get_fc) add_net_test_case(test_s3_round_trip_mpu_multipart_get_with_list_algorithm_fc) add_net_test_case(test_s3_round_trip_mpu_default_get_fc) +add_net_test_case(test_s3_round_trip_default_get_fc_header) +add_net_test_case(test_s3_round_trip_multipart_get_fc_header) +add_net_test_case(test_s3_round_trip_mpu_multipart_get_fc_header) add_net_test_case(test_s3_round_trip_with_filepath) add_net_test_case(test_s3_round_trip_mpu_with_filepath) add_net_test_case(test_s3_round_trip_with_filepath_no_content_length) diff --git a/tests/s3_data_plane_tests.c b/tests/s3_data_plane_tests.c index 953fead0..4154f515 100644 --- a/tests/s3_data_plane_tests.c +++ b/tests/s3_data_plane_tests.c @@ -3830,8 +3830,7 @@ void s_s3_test_no_validate_checksum( } /* TODO: maybe refactor the fc -> flexible checksum tests to be less copy/paste */ -AWS_TEST_CASE(test_s3_round_trip_default_get_fc, s_test_s3_round_trip_default_get_fc) -static int s_test_s3_round_trip_default_get_fc(struct aws_allocator *allocator, void *ctx) { +static int s_test_s3_round_trip_default_get_fc_helper(struct aws_allocator *allocator, void *ctx, bool via_header) { (void)ctx; struct aws_s3_tester tester; @@ -3865,6 +3864,7 @@ static int s_test_s3_round_trip_default_get_fc(struct aws_allocator *allocator, .client = client, .checksum_algorithm = algorithm, .validate_get_response_checksum = false, + .checksum_via_header = via_header, .put_options = { .object_size_mb = 1, @@ -3901,8 +3901,17 @@ static int s_test_s3_round_trip_default_get_fc(struct aws_allocator *allocator, return 0; } -AWS_TEST_CASE(test_s3_round_trip_multipart_get_fc, s_test_s3_round_trip_multipart_get_fc) -static int s_test_s3_round_trip_multipart_get_fc(struct aws_allocator *allocator, void *ctx) { +AWS_TEST_CASE(test_s3_round_trip_default_get_fc, s_test_s3_round_trip_default_get_fc) +static int s_test_s3_round_trip_default_get_fc(struct aws_allocator *allocator, void *ctx) { + return s_test_s3_round_trip_default_get_fc_helper(allocator, ctx, false); +} + +AWS_TEST_CASE(test_s3_round_trip_default_get_fc_header, s_test_s3_round_trip_default_get_fc_header) +static int s_test_s3_round_trip_default_get_fc_header(struct aws_allocator *allocator, void *ctx) { + return s_test_s3_round_trip_default_get_fc_helper(allocator, ctx, true); +} + +static int s_test_s3_round_trip_multipart_get_fc_helper(struct aws_allocator *allocator, void *ctx, bool via_header) { (void)ctx; struct aws_s3_tester tester; @@ -3928,6 +3937,7 @@ static int s_test_s3_round_trip_multipart_get_fc(struct aws_allocator *allocator .client = client, .checksum_algorithm = AWS_SCA_CRC32, .validate_get_response_checksum = false, + .checksum_via_header = via_header, .put_options = { .object_size_mb = 1, @@ -3963,10 +3973,21 @@ static int s_test_s3_round_trip_multipart_get_fc(struct aws_allocator *allocator return 0; } +AWS_TEST_CASE(test_s3_round_trip_multipart_get_fc, s_test_s3_round_trip_multipart_get_fc) +static int s_test_s3_round_trip_multipart_get_fc(struct aws_allocator *allocator, void *ctx) { + return s_test_s3_round_trip_multipart_get_fc_helper(allocator, ctx, false); +} +AWS_TEST_CASE(test_s3_round_trip_multipart_get_fc_header, s_test_s3_round_trip_multipart_get_fc_header) +static int s_test_s3_round_trip_multipart_get_fc_header(struct aws_allocator *allocator, void *ctx) { + return s_test_s3_round_trip_multipart_get_fc_helper(allocator, ctx, true); +} + /* Test the multipart uploaded object was downloaded with same part size, which will download the object matches all the * parts and validate the parts checksum. */ -AWS_TEST_CASE(test_s3_round_trip_mpu_multipart_get_fc, s_test_s3_round_trip_mpu_multipart_get_fc) -static int s_test_s3_round_trip_mpu_multipart_get_fc(struct aws_allocator *allocator, void *ctx) { +static int s_test_s3_round_trip_mpu_multipart_get_fc_helper( + struct aws_allocator *allocator, + void *ctx, + bool via_header) { (void)ctx; struct aws_s3_tester tester; @@ -3992,6 +4013,7 @@ static int s_test_s3_round_trip_mpu_multipart_get_fc(struct aws_allocator *alloc .client = client, .checksum_algorithm = AWS_SCA_CRC32, .validate_get_response_checksum = false, + .checksum_via_header = via_header, .put_options = { .object_size_mb = 10, @@ -4026,8 +4048,20 @@ static int s_test_s3_round_trip_mpu_multipart_get_fc(struct aws_allocator *alloc return 0; } -AWS_TEST_CASE(test_s3_download_empty_file_with_checksum, s_test_s3_download_empty_file_with_checksum) -static int s_test_s3_download_empty_file_with_checksum(struct aws_allocator *allocator, void *ctx) { +AWS_TEST_CASE(test_s3_round_trip_mpu_multipart_get_fc, s_test_s3_round_trip_mpu_multipart_get_fc) +static int s_test_s3_round_trip_mpu_multipart_get_fc(struct aws_allocator *allocator, void *ctx) { + return s_test_s3_round_trip_mpu_multipart_get_fc_helper(allocator, ctx, false); +} + +AWS_TEST_CASE(test_s3_round_trip_mpu_multipart_get_fc_header, s_test_s3_round_trip_mpu_multipart_get_fc_header) +static int s_test_s3_round_trip_mpu_multipart_get_fc_header(struct aws_allocator *allocator, void *ctx) { + return s_test_s3_round_trip_mpu_multipart_get_fc_helper(allocator, ctx, true); +} + +static int s_test_s3_download_empty_file_with_checksum_helper( + struct aws_allocator *allocator, + void *ctx, + bool via_header) { (void)ctx; /* Upload the file */ @@ -4053,6 +4087,7 @@ static int s_test_s3_download_empty_file_with_checksum(struct aws_allocator *all .meta_request_type = AWS_S3_META_REQUEST_TYPE_PUT_OBJECT, .client = client, .checksum_algorithm = AWS_SCA_CRC32, + .checksum_via_header = via_header, .put_options = { .object_size_mb = 0, @@ -4092,6 +4127,15 @@ static int s_test_s3_download_empty_file_with_checksum(struct aws_allocator *all return 0; } +AWS_TEST_CASE(test_s3_download_empty_file_with_checksum, s_test_s3_download_empty_file_with_checksum) +static int s_test_s3_download_empty_file_with_checksum(struct aws_allocator *allocator, void *ctx) { + return s_test_s3_download_empty_file_with_checksum_helper(allocator, ctx, false); +} +AWS_TEST_CASE(test_s3_download_empty_file_with_checksum_header, s_test_s3_download_empty_file_with_checksum_header) +static int s_test_s3_download_empty_file_with_checksum_header(struct aws_allocator *allocator, void *ctx) { + return s_test_s3_download_empty_file_with_checksum_helper(allocator, ctx, true); +} + AWS_TEST_CASE(test_s3_download_single_part_file_with_checksum, s_test_s3_download_single_part_file_with_checksum) static int s_test_s3_download_single_part_file_with_checksum(struct aws_allocator *allocator, void *ctx) { (void)ctx; diff --git a/tests/s3_tester.c b/tests/s3_tester.c index 2a071122..d0283094 100644 --- a/tests/s3_tester.c +++ b/tests/s3_tester.c @@ -1452,9 +1452,11 @@ int aws_s3_tester_send_meta_request_with_options( struct aws_s3_checksum_config checksum_config = { .checksum_algorithm = options->checksum_algorithm, .validate_response_checksum = options->validate_get_response_checksum, - .location = disable_trailing_checksum ? AWS_SCL_NONE : AWS_SCL_TRAILER, .validate_checksum_algorithms = options->validate_checksum_algorithms, }; + if (!disable_trailing_checksum) { + checksum_config.location = options->checksum_via_header ? AWS_SCL_HEADER : AWS_SCL_TRAILER; + } struct aws_s3_meta_request_options meta_request_options = { .type = options->meta_request_type, diff --git a/tests/s3_tester.h b/tests/s3_tester.h index e939b541..5c1c41e8 100644 --- a/tests/s3_tester.h +++ b/tests/s3_tester.h @@ -164,6 +164,7 @@ struct aws_s3_tester_meta_request_options { struct aws_array_list *validate_checksum_algorithms; enum aws_s3_checksum_algorithm expected_validate_checksum_alg; bool disable_put_trailing_checksum; + bool checksum_via_header; /* override client signing config */ struct aws_signing_config_aws *signing_config;