HTTP Header Exclusion Lists (#109)

* Adding exclusion lists for headers when deriving messages. (ie: PutObject->CreateMultipartUpload, PutObject->UploadPart, etc..) This is cleaner than the special case stuff that we're doing now, and lets us more easily keep in any custom headers that the end user passed in. * Now more carefully appending query parameters to account for possible pre-existing query parameters. * Doing a small amount of clean up for related code.
awslabs · Apr 9, 2021 · 53870cd · 53870cd
1 parent 651171f
commit 53870cd
Show file tree

Hide file tree

Showing 10 changed files with 1,245 additions and 180 deletions.
diff --git a/include/aws/s3/private/s3_request_messages.h b/include/aws/s3/private/s3_request_messages.h
@@ -5,7 +5,7 @@
  * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
  * SPDX-License-Identifier: Apache-2.0.
  */
-
+#include "aws/s3/s3.h"
 #include <inttypes.h>
 #include <stdbool.h>
 #include <stddef.h>
@@ -17,67 +17,96 @@ struct aws_byte_cursor;
 struct aws_string;
 struct aws_array_list;
 
-enum aws_s3_copy_http_message_flags {
-    AWS_S3_COPY_MESSAGE_INCLUDE_SSE = 0x00000001,
-    /* For multipart upload complete and abort, only host and two payer related headers are needed */
-    AWS_S3_COPY_MESSAGE_MULTIPART_UPLOAD_OPS = 0x00000002,
-    /* For ranged put, acl should not be there */
-    AWS_S3_COPY_MESSAGE_WITHOUT_ACL = 0x00000004,
-};
+AWS_EXTERN_C_BEGIN
+
+AWS_S3_API
+struct aws_http_message *aws_s3_message_util_copy_http_message(
+    struct aws_allocator *allocator,
+    struct aws_http_message *message,
+    const struct aws_byte_cursor *excluded_headers_arrays,
+    size_t excluded_headers_size);
+
+AWS_S3_API
+struct aws_input_stream *aws_s3_message_util_assign_body(
+    struct aws_allocator *allocator,
+    struct aws_byte_buf *byte_buf,
+    struct aws_http_message *out_message);
 
 /* Create an HTTP request for an S3 Get Object Request, using the original request as a basis. If multipart is not
  * needed, part_number and part_size can be 0. */
+AWS_S3_API
 struct aws_http_message *aws_s3_get_object_message_new(
     struct aws_allocator *allocator,
     struct aws_http_message *base_message,
     uint32_t part_number,
-    size_t part_size,
-    bool has_range);
+    size_t part_size);
 
-/* Create an HTTP request for an S3 Put Object request, using the original request as a basis.  Creates and assigns a
- * body stream using the passed in buffer.  If multipart is not needed, part number and upload_id can be 0 and NULL,
- * respectively. */
-struct aws_http_message *aws_s3_put_object_message_new(
+AWS_S3_API
+int aws_s3_message_util_set_multipart_request_path(
     struct aws_allocator *allocator,
-    struct aws_http_message *base_message,
-    struct aws_byte_buf *buffer,
+    const struct aws_string *upload_id,
     uint32_t part_number,
-    const struct aws_string *upload_id);
+    bool append_uploads_suffix,
+    struct aws_http_message *message);
 
 /* Create an HTTP request for an S3 Create-Multipart-Upload request. */
+AWS_S3_API
 struct aws_http_message *aws_s3_create_multipart_upload_message_new(
     struct aws_allocator *allocator,
     struct aws_http_message *base_message);
 
-/* Given a response body from a multipart upload, try to extract the upload id. */
-struct aws_string *aws_s3_create_multipart_upload_get_upload_id(
+/* Create an HTTP request for an S3 Put Object request, using the original request as a basis.  Creates and assigns a
+ * body stream using the passed in buffer.  If multipart is not needed, part number and upload_id can be 0 and NULL,
+ * respectively. */
+AWS_S3_API
+struct aws_http_message *aws_s3_upload_part_message_new(
     struct aws_allocator *allocator,
-    struct aws_byte_cursor *response_body);
+    struct aws_http_message *base_message,
+    struct aws_byte_buf *buffer,
+    uint32_t part_number,
+    const struct aws_string *upload_id);
 
 /* Create an HTTP request for an S3 Complete-Multipart-Upload request. Creates the necessary XML payload using the
  * passed in array list of ETags.  (Each ETag is assumed to be an aws_string*)  Buffer passed in will be used to store
  * said XML payload, which will be used as the body. */
+AWS_S3_API
 struct aws_http_message *aws_s3_complete_multipart_message_new(
     struct aws_allocator *allocator,
     struct aws_http_message *base_message,
     struct aws_byte_buf *body_buffer,
     const struct aws_string *upload_id,
     const struct aws_array_list *etags);
 
+AWS_S3_API
 struct aws_http_message *aws_s3_abort_multipart_upload_message_new(
     struct aws_allocator *allocator,
     struct aws_http_message *base_message,
     const struct aws_string *upload_id);
 
-/* TODO: maybe set a list of the headers we want */
-struct aws_http_message *aws_s3_message_util_copy_http_message(
-    struct aws_allocator *allocator,
-    struct aws_http_message *message,
-    uint32_t flags);
+AWS_S3_API
+extern const struct aws_byte_cursor g_s3_create_multipart_upload_excluded_headers[];
 
-struct aws_input_stream *aws_s3_message_util_assign_body(
-    struct aws_allocator *allocator,
-    struct aws_byte_buf *byte_buf,
-    struct aws_http_message *out_message);
+AWS_S3_API
+extern const size_t g_s3_create_multipart_upload_excluded_headers_count;
+
+AWS_S3_API
+extern const struct aws_byte_cursor g_s3_upload_part_excluded_headers[];
+
+AWS_S3_API
+extern const size_t g_s3_upload_part_excluded_headers_count;
+
+AWS_S3_API
+extern const struct aws_byte_cursor g_s3_complete_multipart_upload_excluded_headers[];
+
+AWS_S3_API
+extern const size_t g_s3_complete_multipart_upload_excluded_headers_count;
+
+AWS_S3_API
+extern const struct aws_byte_cursor g_s3_abort_multipart_upload_excluded_headers[];
+
+AWS_S3_API
+extern const size_t g_s3_abort_multipart_upload_excluded_headers_count;
+
+AWS_EXTERN_C_END
 
 #endif /* AWS_S3_REQUEST_H */
diff --git a/include/aws/s3/private/s3_util.h b/include/aws/s3/private/s3_util.h
@@ -39,14 +39,6 @@ struct aws_cached_signing_config_aws {
     struct aws_signing_config_aws config;
 };
 
-extern const struct aws_byte_cursor g_s3_service_name;
-extern const struct aws_byte_cursor g_range_header_name;
-extern const struct aws_byte_cursor g_content_range_header_name;
-extern const struct aws_byte_cursor g_accept_ranges_header_name;
-extern const struct aws_byte_cursor g_post_method;
-extern const struct aws_byte_cursor g_delete_method;
-extern const uint32_t g_s3_max_num_upload_parts;
-
 AWS_EXTERN_C_BEGIN
 
 AWS_S3_API
@@ -76,6 +68,27 @@ extern const struct aws_byte_cursor g_etag_header_name;
 AWS_S3_API
 extern const size_t g_s3_min_upload_part_size;
 
+AWS_S3_API
+extern const struct aws_byte_cursor g_s3_service_name;
+
+AWS_S3_API
+extern const struct aws_byte_cursor g_range_header_name;
+
+AWS_S3_API
+extern const struct aws_byte_cursor g_content_range_header_name;
+
+AWS_S3_API
+extern const struct aws_byte_cursor g_accept_ranges_header_name;
+
+AWS_S3_API
+extern const struct aws_byte_cursor g_post_method;
+
+AWS_S3_API
+extern const struct aws_byte_cursor g_delete_method;
+
+AWS_S3_API
+extern const uint32_t g_s3_max_num_upload_parts;
+
 struct aws_cached_signing_config_aws *aws_cached_signing_config_new(
     struct aws_allocator *allocator,
     const struct aws_signing_config_aws *signing_config);

diff --git a/source/s3_auto_ranged_get.c b/source/s3_auto_ranged_get.c
@@ -200,7 +200,7 @@ static bool s_s3_auto_ranged_get_update(
             request = aws_s3_request_new(
                 meta_request,
                 AWS_S3_AUTO_RANGE_GET_REQUEST_TYPE_PART_WITHOUT_RANGE,
-                1,
+                0,
                 AWS_S3_REQUEST_DESC_RECORD_RESPONSE_HEADERS);
 
             auto_ranged_get->synced_data.get_without_range_sent = true;
@@ -297,11 +297,7 @@ static int s_s3_auto_ranged_get_prepare_request(
 
     /* Generate a new ranged get request based on the original message. */
     struct aws_http_message *message = aws_s3_get_object_message_new(
-        meta_request->allocator,
-        meta_request->initial_request_message,
-        request->part_number,
-        meta_request->part_size,
-        request->request_tag != AWS_S3_AUTO_RANGE_GET_REQUEST_TYPE_PART_WITHOUT_RANGE);
+        meta_request->allocator, meta_request->initial_request_message, request->part_number, meta_request->part_size);
 
     if (message == NULL) {
         AWS_LOGF_ERROR(
@@ -344,7 +340,9 @@ static void s_s3_auto_ranged_get_request_finished(
 
     uint32_t num_parts = 0;
 
-    if (error_code == AWS_ERROR_SUCCESS && request->part_number == 1) {
+    /* Check if this was the first part and if it was successful. For a ranged-get request, the first part number will
+     * be 1. For an empty file request, the part number will be 0.*/
+    if (error_code == AWS_ERROR_SUCCESS && request->part_number <= 1) {
         uint64_t total_object_size = 0;
         if (request->request_tag == AWS_S3_AUTO_RANGE_GET_REQUEST_TYPE_PART) {
             struct aws_byte_cursor content_range_header_value;

diff --git a/source/s3_auto_ranged_put.c b/source/s3_auto_ranged_put.c
@@ -368,7 +368,7 @@ static int s_s3_auto_ranged_put_prepare_request(
             }
 
             /* Create a new put-object message to upload a part. */
-            message = aws_s3_put_object_message_new(
+            message = aws_s3_upload_part_message_new(
                 meta_request->allocator,
                 meta_request->initial_request_message,
                 &request->request_body,

diff --git a/source/s3_default_meta_request.c b/source/s3_default_meta_request.c
@@ -226,8 +226,8 @@ static int s_s3_meta_request_default_prepare_request(
         }
     }
 
-    struct aws_http_message *message = aws_s3_message_util_copy_http_message(
-        meta_request->allocator, meta_request->initial_request_message, AWS_S3_COPY_MESSAGE_INCLUDE_SSE);
+    struct aws_http_message *message =
+        aws_s3_message_util_copy_http_message(meta_request->allocator, meta_request->initial_request_message, NULL, 0);
 
     aws_s3_message_util_assign_body(meta_request->allocator, &request->request_body, message);
 

diff --git a/source/s3_meta_request.c b/source/s3_meta_request.c
@@ -571,12 +571,21 @@ static void s_s3_meta_request_request_on_signed(
         aws_apply_signing_result_to_http_request(request->send_data.message, meta_request->allocator, signing_result)) {
 
         error_code = aws_last_error_or_unknown();
+
         goto finish;
     }
 
 finish:
 
     if (error_code != AWS_ERROR_SUCCESS) {
+
+        AWS_LOGF_ERROR(
+            AWS_LS_S3_META_REQUEST,
+            "id=%p Meta request could not sign TTP request due to error code %d (%s)",
+            (void *)meta_request,
+            error_code,
+            aws_error_str(error_code));
+
         aws_s3_meta_request_lock_synced_data(meta_request);
         aws_s3_meta_request_set_fail_synced(meta_request, request, error_code);
         aws_s3_meta_request_unlock_synced_data(meta_request);