From 50e505c250ac586bd0ebe2d15a9bb3c04b08e547 Mon Sep 17 00:00:00 2001 From: Ryan Carper <51676630+rccarper@users.noreply.github.com> Date: Wed, 10 Mar 2021 12:38:09 -0500 Subject: [PATCH] Adding user-agent information to request headers (#102) --- include/aws/s3/private/s3_client_impl.h | 3 + include/aws/s3/private/s3_util.h | 24 ++- source/s3_meta_request.c | 2 + source/s3_request_messages.c | 3 +- source/s3_util.c | 59 ++++++ tests/CMakeLists.txt | 4 + tests/s3_data_plane_tests.c | 266 ++++++++++++++++++++++++ 7 files changed, 354 insertions(+), 7 deletions(-) diff --git a/include/aws/s3/private/s3_client_impl.h b/include/aws/s3/private/s3_client_impl.h index 675d56ed3..4035831ad 100644 --- a/include/aws/s3/private/s3_client_impl.h +++ b/include/aws/s3/private/s3_client_impl.h @@ -16,6 +16,9 @@ #include #include +/* TODO automate this value in the future to prevent it from becoming out-of-sync. */ +#define AWS_S3_CLIENT_VERSION "0.1.x" + struct aws_http_connection; struct aws_http_connection_manager; diff --git a/include/aws/s3/private/s3_util.h b/include/aws/s3/private/s3_util.h index d86875360..e1b16586f 100644 --- a/include/aws/s3/private/s3_util.h +++ b/include/aws/s3/private/s3_util.h @@ -19,6 +19,7 @@ struct aws_allocator; struct aws_http_stream; struct aws_http_headers; +struct aws_http_message; struct aws_event_loop; enum aws_s3_response_status { @@ -38,20 +39,28 @@ struct aws_cached_signing_config_aws { struct aws_signing_config_aws config; }; -AWS_EXTERN_C_BEGIN - extern const struct aws_byte_cursor g_s3_service_name; extern const struct aws_byte_cursor g_range_header_name; extern const struct aws_byte_cursor g_content_range_header_name; extern const struct aws_byte_cursor g_accept_ranges_header_name; - -AWS_S3_API -extern const struct aws_byte_cursor g_acl_header_name; - extern const struct aws_byte_cursor g_post_method; extern const struct aws_byte_cursor g_delete_method; extern const uint32_t g_s3_max_num_upload_parts; +AWS_EXTERN_C_BEGIN + +AWS_S3_API +extern const struct aws_byte_cursor g_s3_client_version; + +AWS_S3_API +extern const struct aws_byte_cursor g_user_agent_header_name; + +AWS_S3_API +extern const struct aws_byte_cursor g_user_agent_header_product_name; + +AWS_S3_API +extern const struct aws_byte_cursor g_acl_header_name; + AWS_S3_API extern const struct aws_byte_cursor g_host_header_name; @@ -88,6 +97,9 @@ void replace_quote_entities(struct aws_allocator *allocator, struct aws_string * /* TODO could be moved to aws-c-common. */ int aws_last_error_or_unknown(void); +AWS_S3_API +void aws_s3_add_user_agent_header(struct aws_allocator *allocator, struct aws_http_message *message); + AWS_EXTERN_C_END #endif /* AWS_S3_UTIL_H */ diff --git a/source/s3_meta_request.c b/source/s3_meta_request.c index 3d3767f7a..a283aa6d0 100644 --- a/source/s3_meta_request.c +++ b/source/s3_meta_request.c @@ -123,6 +123,8 @@ int aws_s3_meta_request_init_base( meta_request->initial_request_message = options->message; aws_http_message_acquire(options->message); + aws_s3_add_user_agent_header(meta_request->allocator, meta_request->initial_request_message); + if (aws_mutex_init(&meta_request->synced_data.lock)) { AWS_LOGF_ERROR( AWS_LS_S3_META_REQUEST, "id=%p Could not initialize mutex for meta request", (void *)meta_request); diff --git a/source/s3_request_messages.c b/source/s3_request_messages.c index e3a2faa4c..c0324dcf4 100644 --- a/source/s3_request_messages.c +++ b/source/s3_request_messages.c @@ -427,7 +427,8 @@ struct aws_http_message *aws_s3_message_util_copy_http_message( if (multipart_upload_ops) { if (aws_byte_cursor_eq_c_str_ignore_case(&header.name, "host") || aws_byte_cursor_eq_c_str_ignore_case(&header.name, "x-amz-request-payer") || - aws_byte_cursor_eq_c_str_ignore_case(&header.name, "x-amz-expected-bucket-owner")) { + aws_byte_cursor_eq_c_str_ignore_case(&header.name, "x-amz-expected-bucket-owner") || + aws_byte_cursor_eq_ignore_case(&header.name, &g_user_agent_header_name)) { if (aws_http_message_add_header(message, header)) { goto error_clean_up; } diff --git a/source/s3_util.c b/source/s3_util.c index e5e6ab8e0..81c6693c8 100644 --- a/source/s3_util.c +++ b/source/s3_util.c @@ -4,6 +4,7 @@ */ #include "aws/s3/private/s3_util.h" +#include "aws/s3/private/s3_client_impl.h" #include #include #include @@ -11,6 +12,7 @@ #include #include +const struct aws_byte_cursor g_s3_client_version = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL(AWS_S3_CLIENT_VERSION); const struct aws_byte_cursor g_s3_service_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("s3"); const struct aws_byte_cursor g_host_header_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("Host"); const struct aws_byte_cursor g_range_header_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("Range"); @@ -23,6 +25,10 @@ const struct aws_byte_cursor g_acl_header_name = AWS_BYTE_CUR_INIT_FROM_STRING_L const struct aws_byte_cursor g_post_method = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("POST"); const struct aws_byte_cursor g_delete_method = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("DELETE"); +const struct aws_byte_cursor g_user_agent_header_name = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("User-Agent"); +const struct aws_byte_cursor g_user_agent_header_product_name = + AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("CRTS3NativeClient"); + const uint32_t g_s3_max_num_upload_parts = 10000; const size_t g_s3_min_upload_part_size = MB_TO_BYTES(5); @@ -249,3 +255,56 @@ int aws_last_error_or_unknown() { return error; } + +void aws_s3_add_user_agent_header(struct aws_allocator *allocator, struct aws_http_message *message) { + AWS_PRECONDITION(allocator); + AWS_PRECONDITION(message); + + const struct aws_byte_cursor space_delimeter = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL(" "); + const struct aws_byte_cursor forward_slash = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("/"); + + const size_t user_agent_product_version_length = + g_user_agent_header_product_name.len + forward_slash.len + g_s3_client_version.len; + + struct aws_http_headers *headers = aws_http_message_get_headers(message); + AWS_ASSERT(headers != NULL); + + struct aws_byte_cursor current_user_agent_header; + AWS_ZERO_STRUCT(current_user_agent_header); + + struct aws_byte_buf user_agent_buffer; + AWS_ZERO_STRUCT(user_agent_buffer); + + if (!aws_http_headers_get(headers, g_user_agent_header_name, ¤t_user_agent_header)) { + /* If the header was found, then create a buffer with the total size we'll need, and append the curent user + * agent header with a trailing space. */ + aws_byte_buf_init( + &user_agent_buffer, + allocator, + current_user_agent_header.len + space_delimeter.len + user_agent_product_version_length); + + aws_byte_buf_append_dynamic(&user_agent_buffer, ¤t_user_agent_header); + + aws_byte_buf_append_dynamic(&user_agent_buffer, &space_delimeter); + + } else { + AWS_ASSERT(aws_last_error() == AWS_ERROR_HTTP_HEADER_NOT_FOUND); + + /* If the header was not found, then create a buffer with just the size of the user agent string that is about + * to be appended to the buffer. */ + aws_byte_buf_init(&user_agent_buffer, allocator, user_agent_product_version_length); + } + + /* Append the client's user-agent string. */ + { + aws_byte_buf_append_dynamic(&user_agent_buffer, &g_user_agent_header_product_name); + aws_byte_buf_append_dynamic(&user_agent_buffer, &forward_slash); + aws_byte_buf_append_dynamic(&user_agent_buffer, &g_s3_client_version); + } + + /* Apply the updated header. */ + aws_http_headers_set(headers, g_user_agent_header_name, aws_byte_cursor_from_buf(&user_agent_buffer)); + + /* Clean up the scratch buffer. */ + aws_byte_buf_clean_up(&user_agent_buffer); +} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 304053daf..3dbfebcaf 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -64,6 +64,10 @@ add_net_test_case(test_s3_bad_endpoint) add_net_test_case(test_s3_put_object_clamp_part_size) add_net_test_case(test_s3_replace_quote_entities) add_net_test_case(test_s3_different_endpoints) +add_test_case(test_add_user_agent_header) +add_net_test_case(test_s3_auto_ranged_get_sending_user_agent) +add_net_test_case(test_s3_auto_ranged_put_sending_user_agent) +add_net_test_case(test_s3_default_sending_meta_request) add_test_case(test_get_existing_compute_platform_info) add_test_case(test_get_nonexistent_compute_platform_info) diff --git a/tests/s3_data_plane_tests.c b/tests/s3_data_plane_tests.c index 933e32315..c2bdfd401 100644 --- a/tests/s3_data_plane_tests.c +++ b/tests/s3_data_plane_tests.c @@ -1569,3 +1569,269 @@ static int s_test_s3_replace_quote_entities(struct aws_allocator *allocator, voi return 0; } + +static int s_get_expected_user_agent(struct aws_allocator *allocator, struct aws_byte_buf *dest) { + AWS_ASSERT(allocator); + AWS_ASSERT(dest); + + const struct aws_byte_cursor forward_slash = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("/"); + + ASSERT_SUCCESS(aws_byte_buf_init(dest, allocator, 32)); + ASSERT_SUCCESS(aws_byte_buf_append_dynamic(dest, &g_user_agent_header_product_name)); + ASSERT_SUCCESS(aws_byte_buf_append_dynamic(dest, &forward_slash)); + ASSERT_SUCCESS(aws_byte_buf_append_dynamic(dest, &g_s3_client_version)); + + return AWS_OP_SUCCESS; +} + +AWS_TEST_CASE(test_add_user_agent_header, s_test_add_user_agent_header) +static int s_test_add_user_agent_header(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + + struct aws_s3_tester tester; + AWS_ZERO_STRUCT(tester); + ASSERT_SUCCESS(aws_s3_tester_init(allocator, &tester)); + + const struct aws_byte_cursor forward_slash = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("/"); + const struct aws_byte_cursor single_space = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL(" "); + + struct aws_byte_buf expected_user_agent_value_buf; + s_get_expected_user_agent(allocator, &expected_user_agent_value_buf); + + struct aws_byte_cursor expected_user_agent_value = aws_byte_cursor_from_buf(&expected_user_agent_value_buf); + + { + struct aws_byte_cursor user_agent_value; + AWS_ZERO_STRUCT(user_agent_value); + + struct aws_http_message *message = aws_http_message_new_request(allocator); + + aws_s3_add_user_agent_header(allocator, message); + + struct aws_http_headers *headers = aws_http_message_get_headers(message); + + ASSERT_TRUE(headers != NULL); + ASSERT_SUCCESS(aws_http_headers_get(headers, g_user_agent_header_name, &user_agent_value)); + ASSERT_TRUE(aws_byte_cursor_eq(&user_agent_value, &expected_user_agent_value)); + + aws_http_message_release(message); + } + + { + const struct aws_byte_cursor dummy_agent_header_value = + AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("dummy_user_agent_product/dummy_user_agent_value"); + + struct aws_byte_buf total_expected_user_agent_value_buf; + aws_byte_buf_init(&total_expected_user_agent_value_buf, allocator, 64); + aws_byte_buf_append_dynamic(&total_expected_user_agent_value_buf, &dummy_agent_header_value); + aws_byte_buf_append_dynamic(&total_expected_user_agent_value_buf, &single_space); + + aws_byte_buf_append_dynamic(&total_expected_user_agent_value_buf, &g_user_agent_header_product_name); + aws_byte_buf_append_dynamic(&total_expected_user_agent_value_buf, &forward_slash); + aws_byte_buf_append_dynamic(&total_expected_user_agent_value_buf, &g_s3_client_version); + + struct aws_byte_cursor total_expected_user_agent_value = + aws_byte_cursor_from_buf(&total_expected_user_agent_value_buf); + + struct aws_http_message *message = aws_http_message_new_request(allocator); + struct aws_http_headers *headers = aws_http_message_get_headers(message); + ASSERT_TRUE(headers != NULL); + + ASSERT_SUCCESS(aws_http_headers_add(headers, g_user_agent_header_name, dummy_agent_header_value)); + + aws_s3_add_user_agent_header(allocator, message); + + { + struct aws_byte_cursor user_agent_value; + AWS_ZERO_STRUCT(user_agent_value); + ASSERT_SUCCESS(aws_http_headers_get(headers, g_user_agent_header_name, &user_agent_value)); + ASSERT_TRUE(aws_byte_cursor_eq(&user_agent_value, &total_expected_user_agent_value)); + } + + aws_byte_buf_clean_up(&total_expected_user_agent_value_buf); + aws_http_message_release(message); + } + + aws_byte_buf_clean_up(&expected_user_agent_value_buf); + aws_s3_tester_clean_up(&tester); + + return 0; +} + +static int s_s3_test_user_agent_meta_request_prepare_request( + struct aws_s3_meta_request *meta_request, + struct aws_s3_client *client, + struct aws_s3_vip_connection *vip_connection, + bool is_initial_prepare) { + + AWS_ASSERT(meta_request != NULL); + + struct aws_s3_meta_request_test_results *results = meta_request->user_data; + AWS_ASSERT(results != NULL); + + struct aws_s3_tester *tester = results->tester; + AWS_ASSERT(tester != NULL); + + struct aws_s3_meta_request_vtable *original_meta_request_vtable = + aws_s3_tester_get_meta_request_vtable_patch(tester, 0)->original_vtable; + + if (original_meta_request_vtable->prepare_request(meta_request, client, vip_connection, is_initial_prepare)) { + return AWS_OP_ERR; + } + + struct aws_byte_buf expected_user_agent_value_buf; + s_get_expected_user_agent(meta_request->allocator, &expected_user_agent_value_buf); + + const struct aws_byte_cursor null_terminator = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("\0"); + ASSERT_SUCCESS(aws_byte_buf_append_dynamic(&expected_user_agent_value_buf, &null_terminator)); + + struct aws_byte_cursor expected_user_agent_value = aws_byte_cursor_from_buf(&expected_user_agent_value_buf); + + struct aws_s3_request *request = vip_connection->request; + struct aws_http_message *message = request->send_data.message; + struct aws_http_headers *headers = aws_http_message_get_headers(message); + + struct aws_byte_cursor user_agent_value; + AWS_ZERO_STRUCT(user_agent_value); + + ASSERT_SUCCESS(aws_http_headers_get(headers, g_user_agent_header_name, &user_agent_value)); + + const char *find_result = strstr((const char *)user_agent_value.ptr, (const char *)expected_user_agent_value.ptr); + + ASSERT_TRUE(find_result != NULL); + ASSERT_TRUE(find_result < (const char *)(user_agent_value.ptr + user_agent_value.len)); + + aws_byte_buf_clean_up(&expected_user_agent_value_buf); + return AWS_OP_SUCCESS; +} + +static struct aws_s3_meta_request *s_s3_meta_request_factory_override_prepare_request( + struct aws_s3_client *client, + const struct aws_s3_meta_request_options *options) { + AWS_ASSERT(client != NULL); + + struct aws_s3_tester *tester = client->shutdown_callback_user_data; + AWS_ASSERT(tester != NULL); + + struct aws_s3_client_vtable *original_client_vtable = + aws_s3_tester_get_client_vtable_patch(tester, 0)->original_vtable; + + struct aws_s3_meta_request *meta_request = original_client_vtable->meta_request_factory(client, options); + + struct aws_s3_meta_request_vtable *patched_meta_request_vtable = + aws_s3_tester_patch_meta_request_vtable(tester, meta_request, NULL); + patched_meta_request_vtable->prepare_request = s_s3_test_user_agent_meta_request_prepare_request; + + return meta_request; +} + +int s_s3_test_sending_user_agent_create_client(struct aws_s3_tester *tester, struct aws_s3_client **client) { + AWS_ASSERT(tester); + + struct aws_s3_tester_client_options client_options; + AWS_ZERO_STRUCT(client_options); + + ASSERT_SUCCESS(aws_s3_tester_client_new(tester, &client_options, client)); + + struct aws_s3_client_vtable *patched_client_vtable = aws_s3_tester_patch_client_vtable(tester, *client, NULL); + patched_client_vtable->meta_request_factory = s_s3_meta_request_factory_override_prepare_request; + + return AWS_OP_SUCCESS; +} + +AWS_TEST_CASE(test_s3_auto_ranged_get_sending_user_agent, s_test_s3_auto_ranged_get_sending_user_agent) +static int s_test_s3_auto_ranged_get_sending_user_agent(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + + struct aws_s3_tester tester; + ASSERT_SUCCESS(aws_s3_tester_init(allocator, &tester)); + + struct aws_s3_client *client = NULL; + ASSERT_SUCCESS(s_s3_test_sending_user_agent_create_client(&tester, &client)); + + { + struct aws_s3_tester_meta_request_options options = { + .allocator = allocator, + .client = client, + .meta_request_type = AWS_S3_META_REQUEST_TYPE_GET_OBJECT, + .validate_type = AWS_S3_TESTER_VALIDATE_TYPE_EXPECT_SUCCESS, + .get_options = + { + .object_path = g_s3_path_get_object_test_1MB, + }, + }; + + ASSERT_SUCCESS(aws_s3_tester_send_meta_request_with_options(&tester, &options, NULL)); + } + + aws_s3_client_release(client); + aws_s3_tester_clean_up(&tester); + + return 0; +} + +AWS_TEST_CASE(test_s3_auto_ranged_put_sending_user_agent, s_test_s3_auto_ranged_put_sending_user_agent) +static int s_test_s3_auto_ranged_put_sending_user_agent(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + + struct aws_s3_tester tester; + ASSERT_SUCCESS(aws_s3_tester_init(allocator, &tester)); + + struct aws_s3_client *client = NULL; + ASSERT_SUCCESS(s_s3_test_sending_user_agent_create_client(&tester, &client)); + + { + struct aws_s3_tester_meta_request_options options = { + .allocator = allocator, + .client = client, + .meta_request_type = AWS_S3_META_REQUEST_TYPE_PUT_OBJECT, + .validate_type = AWS_S3_TESTER_VALIDATE_TYPE_EXPECT_SUCCESS, + .put_options = + { + .ensure_multipart = true, + }, + }; + + ASSERT_SUCCESS(aws_s3_tester_send_meta_request_with_options(&tester, &options, NULL)); + } + + aws_s3_client_release(client); + aws_s3_tester_clean_up(&tester); + + return 0; +} + +AWS_TEST_CASE(test_s3_default_sending_meta_request, s_test_s3_default_sending_meta_request) +static int s_test_s3_default_sending_meta_request(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + + struct aws_s3_tester tester; + ASSERT_SUCCESS(aws_s3_tester_init(allocator, &tester)); + + struct aws_s3_client *client = NULL; + ASSERT_SUCCESS(s_s3_test_sending_user_agent_create_client(&tester, &client)); + + { + struct aws_s3_tester_meta_request_options options = { + .allocator = allocator, + .client = client, + .meta_request_type = AWS_S3_META_REQUEST_TYPE_DEFAULT, + .validate_type = AWS_S3_TESTER_VALIDATE_TYPE_EXPECT_SUCCESS, + .default_type_options = + { + .mode = AWS_S3_TESTER_DEFAULT_TYPE_MODE_GET, + }, + .get_options = + { + .object_path = g_s3_path_get_object_test_1MB, + }, + }; + + ASSERT_SUCCESS(aws_s3_tester_send_meta_request_with_options(&tester, &options, NULL)); + } + + aws_s3_client_release(client); + aws_s3_tester_clean_up(&tester); + + return 0; +}