Support header checksum (#454)

include/aws/s3/s3_client.h

@@ -559,19 +559,19 @@ struct aws_s3_checksum_config {
     /**
      * The location of client added checksum header.
      *
-     * If AWS_SCL_NONE. No request payload checksum will be add and calculated.
+     * If AWS_SCL_NONE. No request payload checksum will be calculated or added.
      *
-     * If AWS_SCL_HEADER, the checksum will be calculated by client and added related header to the request sent.
+     * If AWS_SCL_HEADER, the client will calculate the checksum and add it to the headers.
      *
-     * If AWS_SCL_TRAILER, the payload will be aws_chunked encoded, The checksum will be calculate while reading the
-     * payload by client. Related header will be added to the trailer part of the encoded payload. Note the payload of
-     * the original request cannot be aws-chunked encoded already. Otherwise, error will be raised.
+     * If AWS_SCL_TRAILER, the payload will be aws_chunked encoded, The client will calculate the checksum and add it to
+     * the trailer. Note the payload of the original request cannot be aws-chunked encoded already, this will cause an
+     * error.
      */
     enum aws_s3_checksum_location location;
 
     /**
      * The checksum algorithm used.
-     * Must be set if location is not AWS_SCL_NONE. Must be AWS_SCA_NONE if location is AWS_SCL_NONE.
+     * Must be set if location is not AWS_SCL_NONE.
      */
     enum aws_s3_checksum_algorithm checksum_algorithm;
 

source/s3_client.c

@@ -963,12 +963,6 @@ struct aws_s3_meta_request *aws_s3_client_make_meta_request(
             }
         }
 
-        if (options->checksum_config->location == AWS_SCL_HEADER) {
-            /* TODO: support calculate checksum to add to header */
-            aws_raise_error(AWS_ERROR_UNSUPPORTED_OPERATION);
-            return NULL;
-        }
-
         if (options->checksum_config->location != AWS_SCL_NONE &&
             options->checksum_config->checksum_algorithm == AWS_SCA_NONE) {
             AWS_LOGF_ERROR(

source/s3_request_messages.c

@@ -730,6 +730,94 @@ struct aws_http_message *aws_s3_abort_multipart_upload_message_new(
     return NULL;
 }
 
+/**
+ * Calculate the in memory checksum based on the checksum config. Initialize and set the out_checksum to the encoded
+ * checksum result
+ */
+static int s_calculate_in_memory_checksum_helper(
+    struct aws_allocator *allocator,
+    struct aws_byte_cursor data,
+    const struct checksum_config *checksum_config,
+    struct aws_byte_buf *out_checksum) {
+    AWS_ASSERT(checksum_config->checksum_algorithm != AWS_SCA_NONE);
+    AWS_ASSERT(out_checksum != NULL);
+    AWS_ZERO_STRUCT(*out_checksum);
+
+    int ret_code = AWS_OP_ERR;
+    size_t digest_size = aws_get_digest_size_from_algorithm(checksum_config->checksum_algorithm);
+    size_t encoded_checksum_len = 0;
+    if (aws_base64_compute_encoded_len(digest_size, &encoded_checksum_len)) {
+        return AWS_OP_ERR;
+    }
+
+    aws_byte_buf_init(out_checksum, allocator, encoded_checksum_len);
+
+    struct aws_byte_buf raw_checksum;
+    aws_byte_buf_init(&raw_checksum, allocator, digest_size);
+
+    if (aws_checksum_compute(allocator, checksum_config->checksum_algorithm, &data, &raw_checksum, 0 /*truncate_to*/)) {
+        goto done;
+    }
+    struct aws_byte_cursor raw_checksum_cursor = aws_byte_cursor_from_buf(&raw_checksum);
+    if (aws_base64_encode(&raw_checksum_cursor, out_checksum)) {
+        goto done;
+    }
+
+    ret_code = AWS_OP_SUCCESS;
+done:
+    if (ret_code) {
+        aws_byte_buf_clean_up(out_checksum);
+    }
+    aws_byte_buf_clean_up(&raw_checksum);
+    return ret_code;
+}
+
+/**
+ * Calculate the in memory checksum based on the checksum config.
+ * If out_checksum set, initialize and set it to the encoded checksum result.
+ * Set the corresponding header in out_message to the encoded checksum result.
+ */
+static int s_calculate_and_add_checksum_to_header_helper(
+    struct aws_allocator *allocator,
+    struct aws_byte_cursor data,
+    const struct checksum_config *checksum_config,
+    struct aws_http_message *out_message,
+    struct aws_byte_buf *out_checksum) {
+    AWS_ASSERT(checksum_config->checksum_algorithm != AWS_SCA_NONE);
+    AWS_ASSERT(out_message != NULL);
+    int ret_code = AWS_OP_ERR;
+
+    struct aws_byte_buf local_encoded_checksum_buf;
+    struct aws_byte_buf *local_encoded_checksum;
+    if (out_checksum == NULL) {
+        local_encoded_checksum = &local_encoded_checksum_buf;
+    } else {
+        local_encoded_checksum = out_checksum;
+    }
+    AWS_ZERO_STRUCT(*local_encoded_checksum);
+    if (s_calculate_in_memory_checksum_helper(allocator, data, checksum_config, local_encoded_checksum)) {
+        goto done;
+    }
+
+    /* Add the encoded checksum to header. */
+    const struct aws_byte_cursor *header_name =
+        aws_get_http_header_name_from_algorithm(checksum_config->checksum_algorithm);
+    struct aws_byte_cursor encoded_checksum_val = aws_byte_cursor_from_buf(local_encoded_checksum);
+    struct aws_http_headers *headers = aws_http_message_get_headers(out_message);
+    if (aws_http_headers_set(headers, *header_name, encoded_checksum_val)) {
+        goto done;
+    }
+
+    ret_code = AWS_OP_SUCCESS;
+done:
+    if (ret_code || out_checksum == NULL) {
+        /* In case of error happen or out_checksum is not set, clean up the encoded checksum. Otherwise, the caller will
+         * own the encoded checksum. */
+        aws_byte_buf_clean_up(local_encoded_checksum);
+    }
+    return ret_code;
+}
+
 /* Assign a buffer to an HTTP message, creating a stream and setting the content-length header */
 struct aws_input_stream *aws_s3_message_util_assign_body(
     struct aws_allocator *allocator,
@@ -812,25 +900,19 @@ struct aws_input_stream *aws_s3_message_util_assign_body(
             }
             aws_input_stream_release(input_stream);
             input_stream = chunk_stream;
-        } else if (
-            checksum_config->checksum_algorithm != AWS_SCA_NONE && checksum_config->location == AWS_SCL_NONE &&
-            out_checksum != NULL) {
-            /* The checksum won't be uploaded, but we still need it for the upload review callback */
-            size_t checksum_len = aws_get_digest_size_from_algorithm(checksum_config->checksum_algorithm);
-            size_t encoded_checksum_len = 0;
-            if (aws_base64_compute_encoded_len(checksum_len, &encoded_checksum_len)) {
+        } else if (checksum_config->location == AWS_SCL_HEADER) {
+            /* Calculate the checksum directly from memory and add it to the header. */
+            if (s_calculate_and_add_checksum_to_header_helper(
+                    allocator, buffer_byte_cursor, checksum_config, out_message, out_checksum)) {
                 goto error_clean_up;
             }
-            if (aws_byte_buf_init(out_checksum, allocator, encoded_checksum_len)) {
-                goto error_clean_up;
-            }
-            struct aws_input_stream *checksum_stream =
-                aws_checksum_stream_new(allocator, input_stream, checksum_config->checksum_algorithm, out_checksum);
-            if (!checksum_stream) {
+
+        } else if (checksum_config->checksum_algorithm != AWS_SCA_NONE && out_checksum != NULL) {
+            /* In case checksums still wanted, and we can calculate it directly from the buffer in memory to
+             * out_checksum */
+            if (s_calculate_in_memory_checksum_helper(allocator, buffer_byte_cursor, checksum_config, out_checksum)) {
                 goto error_clean_up;
             }
-            aws_input_stream_release(input_stream);
-            input_stream = checksum_stream;
         }
     }
     int64_t stream_length = 0;
@@ -892,9 +974,7 @@ int aws_s3_message_util_add_content_md5_header(
         return AWS_OP_ERR;
     }
     struct aws_byte_buf base64_output_buf;
-    if (aws_byte_buf_init(&base64_output_buf, allocator, base64_output_size)) {
-        return AWS_OP_ERR;
-    }
+    aws_byte_buf_init(&base64_output_buf, allocator, base64_output_size);
     if (aws_base64_encode(&base64_input, &base64_output_buf)) {
         goto error_clean_up;
     }
@@ -1056,9 +1136,7 @@ int aws_s3_message_util_set_multipart_request_path(
         return AWS_OP_ERR;
     }
 
-    if (aws_byte_buf_init(&request_path_buf, allocator, request_path.len)) {
-        return AWS_OP_ERR;
-    }
+    aws_byte_buf_init(&request_path_buf, allocator, request_path.len);
 
     if (aws_byte_buf_append_dynamic(&request_path_buf, &request_path)) {
         goto error_clean_up;

tests/CMakeLists.txt

@@ -142,6 +142,7 @@ add_net_test_case(test_s3_put_object_async_no_content_length_2parts)
 add_net_test_case(test_s3_put_object_async_fail_reading)
 add_net_test_case(test_s3_many_async_uploads_without_data)
 add_net_test_case(test_s3_download_empty_file_with_checksum)
+add_net_test_case(test_s3_download_empty_file_with_checksum_header)
 add_net_test_case(test_s3_download_single_part_file_with_checksum)
 add_net_test_case(test_s3_download_multipart_file_with_checksum)
 add_net_test_case(test_s3_asyncwrite_empty_file)
@@ -176,6 +177,9 @@ add_net_test_case(test_s3_round_trip_default_get_fc)
 add_net_test_case(test_s3_round_trip_mpu_multipart_get_fc)
 add_net_test_case(test_s3_round_trip_mpu_multipart_get_with_list_algorithm_fc)
 add_net_test_case(test_s3_round_trip_mpu_default_get_fc)
+add_net_test_case(test_s3_round_trip_default_get_fc_header)
+add_net_test_case(test_s3_round_trip_multipart_get_fc_header)
+add_net_test_case(test_s3_round_trip_mpu_multipart_get_fc_header)
 add_net_test_case(test_s3_round_trip_with_filepath)
 add_net_test_case(test_s3_round_trip_mpu_with_filepath)
 add_net_test_case(test_s3_round_trip_with_filepath_no_content_length)

tests/s3_data_plane_tests.c

@@ -3830,8 +3830,7 @@ void s_s3_test_no_validate_checksum(
 }
 
 /* TODO: maybe refactor the fc -> flexible checksum tests to be less copy/paste */
-AWS_TEST_CASE(test_s3_round_trip_default_get_fc, s_test_s3_round_trip_default_get_fc)
-static int s_test_s3_round_trip_default_get_fc(struct aws_allocator *allocator, void *ctx) {
+static int s_test_s3_round_trip_default_get_fc_helper(struct aws_allocator *allocator, void *ctx, bool via_header) {
     (void)ctx;
 
     struct aws_s3_tester tester;
@@ -3865,6 +3864,7 @@ static int s_test_s3_round_trip_default_get_fc(struct aws_allocator *allocator,
             .client = client,
             .checksum_algorithm = algorithm,
             .validate_get_response_checksum = false,
+            .checksum_via_header = via_header,
             .put_options =
                 {
                     .object_size_mb = 1,
@@ -3901,8 +3901,17 @@ static int s_test_s3_round_trip_default_get_fc(struct aws_allocator *allocator,
     return 0;
 }
 
-AWS_TEST_CASE(test_s3_round_trip_multipart_get_fc, s_test_s3_round_trip_multipart_get_fc)
-static int s_test_s3_round_trip_multipart_get_fc(struct aws_allocator *allocator, void *ctx) {
+AWS_TEST_CASE(test_s3_round_trip_default_get_fc, s_test_s3_round_trip_default_get_fc)
+static int s_test_s3_round_trip_default_get_fc(struct aws_allocator *allocator, void *ctx) {
+    return s_test_s3_round_trip_default_get_fc_helper(allocator, ctx, false);
+}
+
+AWS_TEST_CASE(test_s3_round_trip_default_get_fc_header, s_test_s3_round_trip_default_get_fc_header)
+static int s_test_s3_round_trip_default_get_fc_header(struct aws_allocator *allocator, void *ctx) {
+    return s_test_s3_round_trip_default_get_fc_helper(allocator, ctx, true);
+}
+
+static int s_test_s3_round_trip_multipart_get_fc_helper(struct aws_allocator *allocator, void *ctx, bool via_header) {
     (void)ctx;
 
     struct aws_s3_tester tester;
@@ -3928,6 +3937,7 @@ static int s_test_s3_round_trip_multipart_get_fc(struct aws_allocator *allocator
         .client = client,
         .checksum_algorithm = AWS_SCA_CRC32,
         .validate_get_response_checksum = false,
+        .checksum_via_header = via_header,
         .put_options =
             {
                 .object_size_mb = 1,
@@ -3963,10 +3973,21 @@ static int s_test_s3_round_trip_multipart_get_fc(struct aws_allocator *allocator
     return 0;
 }
 
+AWS_TEST_CASE(test_s3_round_trip_multipart_get_fc, s_test_s3_round_trip_multipart_get_fc)
+static int s_test_s3_round_trip_multipart_get_fc(struct aws_allocator *allocator, void *ctx) {
+    return s_test_s3_round_trip_multipart_get_fc_helper(allocator, ctx, false);
+}
+AWS_TEST_CASE(test_s3_round_trip_multipart_get_fc_header, s_test_s3_round_trip_multipart_get_fc_header)
+static int s_test_s3_round_trip_multipart_get_fc_header(struct aws_allocator *allocator, void *ctx) {
+    return s_test_s3_round_trip_multipart_get_fc_helper(allocator, ctx, true);
+}
+
 /* Test the multipart uploaded object was downloaded with same part size, which will download the object matches all the
  * parts and validate the parts checksum. */
-AWS_TEST_CASE(test_s3_round_trip_mpu_multipart_get_fc, s_test_s3_round_trip_mpu_multipart_get_fc)
-static int s_test_s3_round_trip_mpu_multipart_get_fc(struct aws_allocator *allocator, void *ctx) {
+static int s_test_s3_round_trip_mpu_multipart_get_fc_helper(
+    struct aws_allocator *allocator,
+    void *ctx,
+    bool via_header) {
     (void)ctx;
 
     struct aws_s3_tester tester;
@@ -3992,6 +4013,7 @@ static int s_test_s3_round_trip_mpu_multipart_get_fc(struct aws_allocator *alloc
         .client = client,
         .checksum_algorithm = AWS_SCA_CRC32,
         .validate_get_response_checksum = false,
+        .checksum_via_header = via_header,
         .put_options =
             {
                 .object_size_mb = 10,
@@ -4026,8 +4048,20 @@ static int s_test_s3_round_trip_mpu_multipart_get_fc(struct aws_allocator *alloc
     return 0;
 }
 
-AWS_TEST_CASE(test_s3_download_empty_file_with_checksum, s_test_s3_download_empty_file_with_checksum)
-static int s_test_s3_download_empty_file_with_checksum(struct aws_allocator *allocator, void *ctx) {
+AWS_TEST_CASE(test_s3_round_trip_mpu_multipart_get_fc, s_test_s3_round_trip_mpu_multipart_get_fc)
+static int s_test_s3_round_trip_mpu_multipart_get_fc(struct aws_allocator *allocator, void *ctx) {
+    return s_test_s3_round_trip_mpu_multipart_get_fc_helper(allocator, ctx, false);
+}
+
+AWS_TEST_CASE(test_s3_round_trip_mpu_multipart_get_fc_header, s_test_s3_round_trip_mpu_multipart_get_fc_header)
+static int s_test_s3_round_trip_mpu_multipart_get_fc_header(struct aws_allocator *allocator, void *ctx) {
+    return s_test_s3_round_trip_mpu_multipart_get_fc_helper(allocator, ctx, true);
+}
+
+static int s_test_s3_download_empty_file_with_checksum_helper(
+    struct aws_allocator *allocator,
+    void *ctx,
+    bool via_header) {
     (void)ctx;
 
     /* Upload the file */
@@ -4053,6 +4087,7 @@ static int s_test_s3_download_empty_file_with_checksum(struct aws_allocator *all
         .meta_request_type = AWS_S3_META_REQUEST_TYPE_PUT_OBJECT,
         .client = client,
         .checksum_algorithm = AWS_SCA_CRC32,
+        .checksum_via_header = via_header,
         .put_options =
             {
                 .object_size_mb = 0,
@@ -4092,6 +4127,15 @@ static int s_test_s3_download_empty_file_with_checksum(struct aws_allocator *all
     return 0;
 }
 
+AWS_TEST_CASE(test_s3_download_empty_file_with_checksum, s_test_s3_download_empty_file_with_checksum)
+static int s_test_s3_download_empty_file_with_checksum(struct aws_allocator *allocator, void *ctx) {
+    return s_test_s3_download_empty_file_with_checksum_helper(allocator, ctx, false);
+}
+AWS_TEST_CASE(test_s3_download_empty_file_with_checksum_header, s_test_s3_download_empty_file_with_checksum_header)
+static int s_test_s3_download_empty_file_with_checksum_header(struct aws_allocator *allocator, void *ctx) {
+    return s_test_s3_download_empty_file_with_checksum_helper(allocator, ctx, true);
+}
+
 AWS_TEST_CASE(test_s3_download_single_part_file_with_checksum, s_test_s3_download_single_part_file_with_checksum)
 static int s_test_s3_download_single_part_file_with_checksum(struct aws_allocator *allocator, void *ctx) {
     (void)ctx;

tests/s3_tester.c

@@ -1452,9 +1452,11 @@ int aws_s3_tester_send_meta_request_with_options(
     struct aws_s3_checksum_config checksum_config = {
         .checksum_algorithm = options->checksum_algorithm,
         .validate_response_checksum = options->validate_get_response_checksum,
-        .location = disable_trailing_checksum ? AWS_SCL_NONE : AWS_SCL_TRAILER,
         .validate_checksum_algorithms = options->validate_checksum_algorithms,
     };
+    if (!disable_trailing_checksum) {
+        checksum_config.location = options->checksum_via_header ? AWS_SCL_HEADER : AWS_SCL_TRAILER;
+    }
 
     struct aws_s3_meta_request_options meta_request_options = {
         .type = options->meta_request_type,

tests/s3_tester.h

@@ -164,6 +164,7 @@ struct aws_s3_tester_meta_request_options {
     struct aws_array_list *validate_checksum_algorithms;
     enum aws_s3_checksum_algorithm expected_validate_checksum_alg;
     bool disable_put_trailing_checksum;
+    bool checksum_via_header;
 
     /* override client signing config */
     struct aws_signing_config_aws *signing_config;