From 8a149ee02c8931b0abcdc2499d36585e0aa5a0a8 Mon Sep 17 00:00:00 2001 From: Ryan Carper <51676630+rccarper@users.noreply.github.com> Date: Tue, 18 May 2021 17:38:14 -0400 Subject: [PATCH] BYO Crypto fix + BYO Crypto CI job (#124) --- .github/workflows/ci.yml | 10 +++++ include/aws/s3/s3_client.h | 3 +- source/s3_client.c | 15 ++++++++ tests/CMakeLists.txt | 9 +++++ tests/s3_data_plane_tests.c | 75 ++++++++++++++++++++++++++++++++++--- tests/s3_tester.c | 25 ++++++++++++- 6 files changed, 128 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 009ad50f2..596927ede 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -124,3 +124,13 @@ jobs: python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" chmod a+x builder ./builder build -p ${{ env.PACKAGE_NAME }} downstream + byo-crypto: + runs-on: ubuntu-latest + steps: + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: Build ${{ env.PACKAGE_NAME }} + run: | + echo "${{ secrets.GITHUB_TOKEN }}" | docker login docker.pkg.github.com -u awslabs --password-stdin + export DOCKER_IMAGE=docker.pkg.github.com/awslabs/aws-crt-builder/aws-crt-${{ env.LINUX_BASE_IMAGE }}:${{ env.BUILDER_VERSION }} + docker pull $DOCKER_IMAGE + docker run --mount type=bind,source=$(pwd),target=/root/${{ env.PACKAGE_NAME }} --env GITHUB_REF --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --env AWS_DEFAULT_REGION $DOCKER_IMAGE build -p ${{ env.PACKAGE_NAME }} --cmake-extra=-DBYO_CRYPTO=ON downstream diff --git a/include/aws/s3/s3_client.h b/include/aws/s3/s3_client.h index 304b6bcf0..d20582614 100644 --- a/include/aws/s3/s3_client.h +++ b/include/aws/s3/s3_client.h @@ -80,7 +80,8 @@ struct aws_s3_client_config { */ enum aws_s3_meta_request_tls_mode tls_mode; - /* TLS Options to be used for each connection, if tls_mode is ENABLED */ + /* TLS Options to be used for each connection, if tls_mode is ENABLED. When compiling with BYO_CRYPTO, and tls_mode + * is ENABLED, this is required. Otherwise, this is optional. */ struct aws_tls_connection_options *tls_connection_options; /* Signing options to be used for each request. Specify NULL to not sign requests. */ diff --git a/source/s3_client.c b/source/s3_client.c index e809c1247..93da05ef5 100644 --- a/source/s3_client.c +++ b/source/s3_client.c @@ -227,6 +227,17 @@ struct aws_s3_client *aws_s3_client_new( return NULL; } +#ifdef BYO_CRYPTO + if (client_config->tls_mode == AWS_MR_TLS_ENABLED && client_config->tls_connection_options == NULL) { + AWS_LOGF_ERROR( + AWS_LS_S3_CLIENT, + "Cannot create client from client_config; when using BYO_CRYPTO, tls_connection_options can not be " + "NULL when TLS is enabled."); + aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); + return NULL; + } +#endif + struct aws_s3_client *client = aws_mem_calloc(allocator, 1, sizeof(struct aws_s3_client)); client->allocator = allocator; @@ -320,6 +331,9 @@ struct aws_s3_client *aws_s3_client_new( if (client_config->tls_connection_options != NULL) { aws_tls_connection_options_copy(client->tls_connection_options, client_config->tls_connection_options); } else { +#ifdef BYO_CRYPTO + AWS_ASSERT(false); +#else struct aws_tls_ctx_options default_tls_ctx_options; AWS_ZERO_STRUCT(default_tls_ctx_options); @@ -334,6 +348,7 @@ struct aws_s3_client *aws_s3_client_new( aws_tls_ctx_release(default_tls_ctx); aws_tls_ctx_options_clean_up(&default_tls_ctx_options); +#endif } } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c545a405e..fa2839da7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,6 +1,15 @@ include(AwsTestHarness) enable_testing() +option(BYO_CRYPTO "Don't build a tls implementation or link against a crypto interface. This feature is only for unix builds currently." OFF) + +if(BYO_CRYPTO) + set(ENABLE_NET_TESTS OFF) + + add_test_case(test_s3_client_byo_crypto_no_options) + add_test_case(test_s3_client_byo_crypto_with_options) +endif() + file(GLOB TEST_SRC "*.c") file(GLOB TEST_HDRS "*.h") file(GLOB TESTS ${TEST_HDRS} ${TEST_SRC}) diff --git a/tests/s3_data_plane_tests.c b/tests/s3_data_plane_tests.c index 2c2012361..b7150d8fb 100644 --- a/tests/s3_data_plane_tests.c +++ b/tests/s3_data_plane_tests.c @@ -49,7 +49,6 @@ static int s_test_s3_client_create_destroy(struct aws_allocator *allocator, void AWS_TEST_CASE(test_s3_client_max_active_connections_override, s_test_s3_client_max_active_connections_override) static int s_test_s3_client_max_active_connections_override(struct aws_allocator *allocator, void *ctx) { - (void)allocator; (void)ctx; struct aws_s3_tester tester; @@ -72,6 +71,56 @@ static int s_test_s3_client_max_active_connections_override(struct aws_allocator return 0; } +AWS_TEST_CASE(test_s3_client_byo_crypto_no_options, s_test_s3_client_byo_crypto_no_options) +static int s_test_s3_client_byo_crypto_no_options(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + + struct aws_s3_tester tester; + AWS_ZERO_STRUCT(tester); + ASSERT_SUCCESS(aws_s3_tester_init(allocator, &tester)); + + struct aws_s3_client_config client_config = { + .tls_mode = AWS_MR_TLS_ENABLED, + }; + + struct aws_s3_client *client = aws_s3_client_new(allocator, &client_config); + + ASSERT_TRUE(aws_last_error() == AWS_ERROR_INVALID_ARGUMENT); + ASSERT_TRUE(client == NULL); + + aws_s3_tester_clean_up(&tester); + + return 0; +} + +AWS_TEST_CASE(test_s3_client_byo_crypto_with_options, s_test_s3_client_byo_crypto_with_options) +static int s_test_s3_client_byo_crypto_with_options(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + + struct aws_s3_tester tester; + AWS_ZERO_STRUCT(tester); + ASSERT_SUCCESS(aws_s3_tester_init(allocator, &tester)); + + struct aws_tls_connection_options tls_conn_options; + AWS_ZERO_STRUCT(tls_conn_options); + + struct aws_s3_client_config client_config; + AWS_ZERO_STRUCT(client_config); + ASSERT_SUCCESS(aws_s3_tester_bind_client(&tester, &client_config, 0)); + + client_config.tls_mode = AWS_MR_TLS_ENABLED; + client_config.tls_connection_options = &tls_conn_options; + + struct aws_s3_client *client = aws_s3_client_new(allocator, &client_config); + + ASSERT_TRUE(client != NULL); + + aws_s3_client_release(client); + aws_s3_tester_clean_up(&tester); + + return 0; +} + AWS_TEST_CASE(test_s3_client_get_max_active_connections, s_test_s3_client_get_max_active_connections) static int s_test_s3_client_get_max_active_connections(struct aws_allocator *allocator, void *ctx) { (void)ctx; @@ -1211,12 +1260,16 @@ static int s_test_s3_get_object_helper( .part_size = 64 * 1024, }; + struct aws_tls_connection_options tls_connection_options; + AWS_ZERO_STRUCT(tls_connection_options); + +#ifndef BYO_CRYPTO struct aws_tls_ctx_options tls_context_options; aws_tls_ctx_options_init_default_client(&tls_context_options, allocator); - struct aws_tls_ctx *context = aws_tls_client_ctx_new(allocator, &tls_context_options); - struct aws_tls_connection_options tls_connection_options; + struct aws_tls_ctx *context = aws_tls_client_ctx_new(allocator, &tls_context_options); aws_tls_connection_options_init_from_ctx(&tls_connection_options, context); +#endif struct aws_string *endpoint = aws_s3_tester_build_endpoint_string(allocator, &g_test_bucket_name, &g_test_s3_region); @@ -1245,9 +1298,12 @@ static int s_test_s3_get_object_helper( ASSERT_SUCCESS(aws_s3_tester_send_get_object_meta_request(&tester, client, s3_path, flags, NULL)); aws_string_destroy(endpoint); + +#ifndef BYO_CRYPTO aws_tls_ctx_release(context); aws_tls_connection_options_clean_up(&tls_connection_options); aws_tls_ctx_options_clean_up(&tls_context_options); +#endif aws_s3_client_release(client); client = NULL; @@ -1279,7 +1335,7 @@ AWS_TEST_CASE(test_s3_get_object_tls_default, s_test_s3_get_object_tls_default) static int s_test_s3_get_object_tls_default(struct aws_allocator *allocator, void *ctx) { (void)ctx; - ASSERT_SUCCESS(s_test_s3_get_object_helper(allocator, AWS_S3_TLS_ENABLED, 0, g_s3_path_get_object_test_1MB)); + ASSERT_SUCCESS(s_test_s3_get_object_helper(allocator, AWS_S3_TLS_DEFAULT, 0, g_s3_path_get_object_test_1MB)); return 0; } @@ -1583,12 +1639,16 @@ static int s_test_s3_put_object_helper( AWS_ZERO_STRUCT(tester); ASSERT_SUCCESS(aws_s3_tester_init(allocator, &tester)); + struct aws_tls_connection_options tls_connection_options; + AWS_ZERO_STRUCT(tls_connection_options); + +#ifndef BYO_CRYPTO struct aws_tls_ctx_options tls_context_options; aws_tls_ctx_options_init_default_client(&tls_context_options, allocator); - struct aws_tls_ctx *context = aws_tls_client_ctx_new(allocator, &tls_context_options); - struct aws_tls_connection_options tls_connection_options; + struct aws_tls_ctx *context = aws_tls_client_ctx_new(allocator, &tls_context_options); aws_tls_connection_options_init_from_ctx(&tls_connection_options, context); +#endif struct aws_string *endpoint = aws_s3_tester_build_endpoint_string(allocator, &g_test_bucket_name, &g_test_s3_region); @@ -1621,9 +1681,12 @@ static int s_test_s3_put_object_helper( &tester, client, 10, AWS_S3_TESTER_SEND_META_REQUEST_EXPECT_SUCCESS | extra_meta_request_flag, NULL)); aws_string_destroy(endpoint); + +#ifndef BYO_CRYPTO aws_tls_ctx_release(context); aws_tls_connection_options_clean_up(&tls_connection_options); aws_tls_ctx_options_clean_up(&tls_context_options); +#endif aws_s3_client_release(client); client = NULL; diff --git a/tests/s3_tester.c b/tests/s3_tester.c index 3e868f85e..d94ebf2bd 100644 --- a/tests/s3_tester.c +++ b/tests/s3_tester.c @@ -33,6 +33,18 @@ const struct aws_byte_cursor g_s3_path_get_object_test_1MB = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("/get_object_test_1MB.txt"); const struct aws_byte_cursor g_s3_sse_header = AWS_BYTE_CUR_INIT_FROM_STRING_LITERAL("x-amz-server-side-encryption"); +#ifdef BYO_CRYPTO +/* Under BYO_CRYPTO, this function currently needs to be defined by the user. Defining a null implementation here so + * that tests build, but it is not currently meant to be used by any tests. */ +struct aws_byte_buf aws_tls_handler_protocol(struct aws_channel_handler *handler) { + (void)handler; + AWS_FATAL_ASSERT(false); + struct aws_byte_buf byte_buf; + AWS_ZERO_STRUCT(byte_buf); + return byte_buf; +} +#endif + static int s_s3_test_meta_request_header_callback( struct aws_s3_meta_request *meta_request, const struct aws_http_headers *headers, @@ -203,6 +215,7 @@ int aws_s3_tester_init(struct aws_allocator *allocator, struct aws_s3_tester *te tester->client_bootstrap = aws_client_bootstrap_new(allocator, &bootstrap_options); } +#ifndef BYO_CRYPTO /* Setup the credentials provider */ { struct aws_credentials_provider_chain_default_options credentials_config; @@ -210,6 +223,7 @@ int aws_s3_tester_init(struct aws_allocator *allocator, struct aws_s3_tester *te credentials_config.bootstrap = tester->client_bootstrap; tester->credentials_provider = aws_credentials_provider_new_chain_default(allocator, &credentials_config); } +#endif aws_s3_init_default_signing_config(&tester->default_signing_config, g_test_s3_region, tester->credentials_provider); @@ -976,12 +990,16 @@ int aws_s3_tester_client_new( .max_part_size = options->max_part_size, }; + struct aws_tls_connection_options tls_connection_options; + AWS_ZERO_STRUCT(tls_connection_options); + +#ifndef BYO_CRYPTO struct aws_tls_ctx_options tls_context_options; aws_tls_ctx_options_init_default_client(&tls_context_options, tester->allocator); - struct aws_tls_ctx *context = aws_tls_client_ctx_new(tester->allocator, &tls_context_options); - struct aws_tls_connection_options tls_connection_options; + struct aws_tls_ctx *context = aws_tls_client_ctx_new(tester->allocator, &tls_context_options); aws_tls_connection_options_init_from_ctx(&tls_connection_options, context); +#endif struct aws_string *endpoint = aws_s3_tester_build_endpoint_string(tester->allocator, &g_test_bucket_name, &g_test_s3_region); @@ -1007,9 +1025,12 @@ int aws_s3_tester_client_new( *out_client = aws_s3_client_new(tester->allocator, &client_config); aws_string_destroy(endpoint); + +#ifndef BYO_CRYPTO aws_tls_ctx_release(context); aws_tls_connection_options_clean_up(&tls_connection_options); aws_tls_ctx_options_clean_up(&tls_context_options); +#endif return AWS_OP_SUCCESS; }