From 257c9ec5f30d0aadfdf0064aa22265a9cdb97adf Mon Sep 17 00:00:00 2001 From: Bret Ambrose Date: Tue, 12 Nov 2024 08:57:20 -0800 Subject: [PATCH 1/7] Unlink shutdown callback from ref count (#1166) Co-authored-by: Bret Ambrose --- include/aws/common/ref_count.h | 8 +------ include/aws/common/shutdown_types.h | 34 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 7 deletions(-) create mode 100644 include/aws/common/shutdown_types.h diff --git a/include/aws/common/ref_count.h b/include/aws/common/ref_count.h index 5166eb3f3..43fbde5e5 100644 --- a/include/aws/common/ref_count.h +++ b/include/aws/common/ref_count.h @@ -8,11 +8,10 @@ #include #include +#include AWS_PUSH_SANE_WARNING_LEVEL -typedef void(aws_simple_completion_callback)(void *); - /* * A utility type for making ref-counted types, reminiscent of std::shared_ptr in C++ */ @@ -22,11 +21,6 @@ struct aws_ref_count { aws_simple_completion_callback *on_zero_fn; }; -struct aws_shutdown_callback_options { - aws_simple_completion_callback *shutdown_callback_fn; - void *shutdown_callback_user_data; -}; - AWS_EXTERN_C_BEGIN /** diff --git a/include/aws/common/shutdown_types.h b/include/aws/common/shutdown_types.h new file mode 100644 index 000000000..4ade53a0c --- /dev/null +++ b/include/aws/common/shutdown_types.h @@ -0,0 +1,34 @@ +#ifndef AWS_COMMON_SHUTDOWN_TYPES_H +#define AWS_COMMON_SHUTDOWN_TYPES_H + +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +#include + +AWS_PUSH_SANE_WARNING_LEVEL + +typedef void(aws_simple_completion_callback)(void *); + +/** + * Configuration for a callback to invoke when something has been completely + * cleaned up. Primarily used in async cleanup control flows. + */ +struct aws_shutdown_callback_options { + + /** + * Function to invoke when the associated object is fully destroyed. + */ + aws_simple_completion_callback *shutdown_callback_fn; + + /** + * User data to invoke the shutdown callback with. + */ + void *shutdown_callback_user_data; +}; + +AWS_POP_SANE_WARNING_LEVEL + +#endif /* AWS_COMMON_SHUTDOWN_TYPES_H */ From f88218867ba85b463d9f6f00561295e7d90f159e Mon Sep 17 00:00:00 2001 From: Dmitriy Musatkin <63878209+DmitriyMusatkin@users.noreply.github.com> Date: Tue, 12 Nov 2024 10:01:13 -0800 Subject: [PATCH 2/7] fix empty xml node handling (#1168) --- include/aws/common/private/xml_parser_impl.h | 1 + source/xml_parser.c | 10 ++++++ tests/CMakeLists.txt | 1 + tests/xml_parser_test.c | 36 ++++++++++++++++++++ 4 files changed, 48 insertions(+) diff --git a/include/aws/common/private/xml_parser_impl.h b/include/aws/common/private/xml_parser_impl.h index eea061b1e..d9763e88e 100644 --- a/include/aws/common/private/xml_parser_impl.h +++ b/include/aws/common/private/xml_parser_impl.h @@ -14,6 +14,7 @@ struct aws_xml_node { struct aws_array_list attributes; struct aws_byte_cursor doc_at_body; bool processed; + bool is_empty; }; struct aws_xml_parser { diff --git a/source/xml_parser.c b/source/xml_parser.c index e1b580740..7675bff30 100644 --- a/source/xml_parser.c +++ b/source/xml_parser.c @@ -40,6 +40,8 @@ static int s_load_node_decl( AWS_PRECONDITION(decl_body); AWS_PRECONDITION(node); + node->is_empty = decl_body->ptr[decl_body->len - 1] == '/'; + struct aws_array_list splits; AWS_ZERO_STRUCT(splits); @@ -158,6 +160,14 @@ int s_advance_to_closing_tag( AWS_PRECONDITION(parser); AWS_PRECONDITION(node); + if (node->is_empty) { + if (out_body) { + out_body->ptr = NULL; + out_body->len = 0; + } + return AWS_OP_SUCCESS; + } + /* currently the max node name is 256 characters. This is arbitrary, but should be enough * for our uses. If we ever generalize this, we'll have to come back and rethink this. */ uint8_t name_close[MAX_NAME_LEN + NODE_CLOSE_OVERHEAD] = {0}; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d3556c5f7..36feaa367 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -467,6 +467,7 @@ add_test_case(xml_parser_nested_node_same_name_test) add_test_case(xml_parser_nested_node_deep_recursion_test) add_test_case(xml_parser_too_many_attributes_test) add_test_case(xml_parser_name_too_long_test) +add_test_case(xml_parser_child_empty_tag) add_test_case(test_thread_scheduler_ordering) add_test_case(test_thread_scheduler_happy_path_cancellation) diff --git a/tests/xml_parser_test.c b/tests/xml_parser_test.c index 9a3b33041..06d01107d 100644 --- a/tests/xml_parser_test.c +++ b/tests/xml_parser_test.c @@ -165,6 +165,42 @@ static int s_xml_parser_siblings_with_text_test(struct aws_allocator *allocator, AWS_TEST_CASE(xml_parser_siblings_with_text, s_xml_parser_siblings_with_text_test) +const char *siblings_with_empty_tag = "TestBody2"; + +static int s_xml_parser_child_empty_tag(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + struct sibling_text_capture capture; + AWS_ZERO_STRUCT(capture); + + capture.capture1 = aws_byte_cursor_from_c_str("random values"); + + struct aws_xml_parser_options options = { + .doc = aws_byte_cursor_from_c_str(siblings_with_empty_tag), + .on_root_encountered = s_root_with_child_siblings, + .user_data = &capture, + }; + ASSERT_SUCCESS(aws_xml_parse(allocator, &options)); + + const char expected_name1[] = "child1"; + + const char expected_name2[] = "child2"; + const char expected_value2[] = "TestBody2"; + + ASSERT_BIN_ARRAYS_EQUALS( + expected_name1, sizeof(expected_name1) - 1, capture.node_name1.ptr, capture.node_name1.len); + ASSERT_PTR_EQUALS(capture.capture1.ptr, NULL); + ASSERT_UINT_EQUALS(capture.capture1.len, 0); + + ASSERT_BIN_ARRAYS_EQUALS( + expected_name2, sizeof(expected_name2) - 1, capture.node_name2.ptr, capture.node_name2.len); + ASSERT_BIN_ARRAYS_EQUALS(expected_value2, sizeof(expected_value2) - 1, capture.capture2.ptr, capture.capture2.len); + + return AWS_OP_SUCCESS; +} + +AWS_TEST_CASE(xml_parser_child_empty_tag, s_xml_parser_child_empty_tag) + const char *preamble_and_attributes = "\n" " Date: Wed, 13 Nov 2024 09:32:39 -0800 Subject: [PATCH 3/7] disable visibility hidden on old gcc (#1167) --- cmake/AwsCFlags.cmake | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmake/AwsCFlags.cmake b/cmake/AwsCFlags.cmake index 18ebb712c..93c4c8163 100644 --- a/cmake/AwsCFlags.cmake +++ b/cmake/AwsCFlags.cmake @@ -257,6 +257,10 @@ function(aws_set_common_properties target) # We do this so that backtraces are more likely to show function names. # We mostly use backtraces to diagnose memory leaks. if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug") - set_target_properties(${target} PROPERTIES C_VISIBILITY_PRESET hidden CXX_VISIBILITY_PRESET hidden VISIBILITY_INLINES_HIDDEN ON) + # And dont hide symbols on anything pre GCC 5.0 (Visibility support was not great on older compilers and some libraries didnt annotate visibility - + # looking at you jni, which does not annotate on gcc less than 4.2. Mixing no annotation and hidden symbols leads to unexpected failures.). + if (NOT (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS "5.0")) + set_target_properties(${target} PROPERTIES C_VISIBILITY_PRESET hidden CXX_VISIBILITY_PRESET hidden VISIBILITY_INLINES_HIDDEN ON) + endif () endif () endfunction() From 63187b976a482309e23296c5f967fc19c4131746 Mon Sep 17 00:00:00 2001 From: Dmitriy Musatkin <63878209+DmitriyMusatkin@users.noreply.github.com> Date: Wed, 13 Nov 2024 11:56:24 -0800 Subject: [PATCH 4/7] switch c compiler check to different cmake variable (#1169) --- cmake/AwsCFlags.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/AwsCFlags.cmake b/cmake/AwsCFlags.cmake index 93c4c8163..b9d2f6446 100644 --- a/cmake/AwsCFlags.cmake +++ b/cmake/AwsCFlags.cmake @@ -259,7 +259,7 @@ function(aws_set_common_properties target) if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug") # And dont hide symbols on anything pre GCC 5.0 (Visibility support was not great on older compilers and some libraries didnt annotate visibility - # looking at you jni, which does not annotate on gcc less than 4.2. Mixing no annotation and hidden symbols leads to unexpected failures.). - if (NOT (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS "5.0")) + if (NOT (CMAKE_C_COMPILER_ID STREQUAL "GNU" AND CMAKE_C_COMPILER_VERSION VERSION_LESS "5.0")) set_target_properties(${target} PROPERTIES C_VISIBILITY_PRESET hidden CXX_VISIBILITY_PRESET hidden VISIBILITY_INLINES_HIDDEN ON) endif () endif () From 45230f04b98911e83eaffeb22f3f2b157898bde3 Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Mon, 25 Nov 2024 09:54:03 -0800 Subject: [PATCH 5/7] Doc fix for cbor (#1171) --- include/aws/common/cbor.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/aws/common/cbor.h b/include/aws/common/cbor.h index 1c220ecb7..4760108ee 100644 --- a/include/aws/common/cbor.h +++ b/include/aws/common/cbor.h @@ -327,7 +327,7 @@ int aws_cbor_decoder_peek_type(struct aws_cbor_decoder *decoder, enum aws_cbor_t * @brief Consume the next data item, includes all the content within the data item. * * As an example for the following cbor, this function will consume all the data - * as it's only one cbor data item, an indefinite map with 2 key, value pair: + * as it's only one cbor data item, an indefinite map with 2 pair: * 0xbf6346756ef563416d7421ff * BF -- Start indefinite-length map * 63 -- First key, UTF-8 string length 3 @@ -374,7 +374,7 @@ int aws_cbor_decoder_consume_next_single_element(struct aws_cbor_decoder *decode * Specifically: * AWS_CBOR_TYPE_UINT - aws_cbor_decoder_pop_next_unsigned_int_val * AWS_CBOR_TYPE_NEGINT - aws_cbor_decoder_pop_next_negative_int_val, it represents (-1 - *out) - * AWS_CBOR_TYPE_FLOAT - aws_cbor_decoder_pop_next_double_val + * AWS_CBOR_TYPE_FLOAT - aws_cbor_decoder_pop_next_float_val * AWS_CBOR_TYPE_BYTES - aws_cbor_decoder_pop_next_bytes_val * AWS_CBOR_TYPE_TEXT - aws_cbor_decoder_pop_next_text_val * From 252e8c25227c6dcd0ea382f0e8f465201357ce30 Mon Sep 17 00:00:00 2001 From: Dmitriy Musatkin <63878209+DmitriyMusatkin@users.noreply.github.com> Date: Mon, 25 Nov 2024 16:16:53 -0800 Subject: [PATCH 6/7] Remove reliance on hardcoded user in ci (#1170) --- .github/workflows/ci.yml | 202 ++++++++++++++++++++++++++++----------- tests/hash_table_test.c | 2 +- 2 files changed, 145 insertions(+), 59 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f8b49cca..c0f31fbe1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,19 +6,21 @@ on: - 'main' env: - BUILDER_VERSION: v0.9.62 + BUILDER_VERSION: v0.9.72 BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net BUILDER_SOURCE: releases PACKAGE_NAME: aws-c-common LINUX_BASE_IMAGE: ubuntu-18-x64 RUN: ${{ github.run_id }}-${{ github.run_number }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: us-east-1 + CRT_CI_ROLE: ${{ secrets.CRT_CI_ROLE_ARN }} + AWS_DEFAULT_REGION: us-east-1 + +permissions: + id-token: write # This is required for requesting the JWT jobs: linux-compat: - runs-on: ubuntu-20.04 # latest + runs-on: ubuntu-24.04 # latest strategy: fail-fast: false matrix: @@ -33,7 +35,10 @@ jobs: - rhel8-x64 - al2-x64 steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Build ${{ env.PACKAGE_NAME }} run: | aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh @@ -41,7 +46,7 @@ jobs: ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} linux-compiler-compat: - runs-on: ubuntu-20.04 # latest + runs-on: ubuntu-24.04 # latest strategy: matrix: compiler: @@ -51,55 +56,73 @@ jobs: - clang-9 - clang-10 - clang-11 + - clang-15 + - clang-17 - gcc-4.8 - gcc-5 - gcc-6 - gcc-7 - gcc-8 - gcc-11 + - gcc-13 steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - - name: Build ${{ env.PACKAGE_NAME }} - run: | - aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build -p ${{ env.PACKAGE_NAME }} --compiler=${{ matrix.compiler }} + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build -p ${{ env.PACKAGE_NAME }} --compiler=${{ matrix.compiler }} # Test downstream repos. # This should not be required because we can run into a chicken and egg problem if there is a change that needs some fix in a downstream repo. downstream: - runs-on: ubuntu-22.04 # latest + runs-on: ubuntu-24.04 # latest steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - - name: Build ${{ env.PACKAGE_NAME }} - run: | - aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build downstream -p ${{ env.PACKAGE_NAME }} + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build downstream -p ${{ env.PACKAGE_NAME }} clang-sanitizers: - runs-on: ubuntu-20.04 # latest + runs-on: ubuntu-24.04 # latest strategy: matrix: sanitizers: [",thread", ",address,undefined"] steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - - name: Build ${{ env.PACKAGE_NAME }} - run: | - aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build -p ${{ env.PACKAGE_NAME }} --compiler=clang-11 --cmake-extra=-DENABLE_SANITIZERS=ON --cmake-extra=-DSANITIZERS="${{ matrix.sanitizers }}" + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build -p ${{ env.PACKAGE_NAME }} --compiler=clang-11 --cmake-extra=-DENABLE_SANITIZERS=ON --cmake-extra=-DSANITIZERS="${{ matrix.sanitizers }}" linux-shared-libs: - runs-on: ubuntu-20.04 # latest + runs-on: ubuntu-24.04 # latest steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Build ${{ env.PACKAGE_NAME }} run: | aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build -p ${{ env.PACKAGE_NAME }} --cmake-extra=-DBUILD_SHARED_LIBS=ON linux-no-cpu-extensions: - runs-on: ubuntu-20.04 # latest + runs-on: ubuntu-24.04 # latest steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Build ${{ env.PACKAGE_NAME }} run: | aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh @@ -108,6 +131,10 @@ jobs: windows: runs-on: windows-2022 # latest steps: + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Build ${{ env.PACKAGE_NAME }} + consumers run: | python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" @@ -119,6 +146,10 @@ jobs: matrix: arch: [x86, x64] steps: + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Build ${{ env.PACKAGE_NAME }} + consumers run: | python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" @@ -130,6 +161,10 @@ jobs: matrix: arch: [x86, x64] steps: + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Build ${{ env.PACKAGE_NAME }} + consumers run: | python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" @@ -141,6 +176,10 @@ jobs: matrix: arch: [x86, x64] steps: + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Build ${{ env.PACKAGE_NAME }} + consumers run: | python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" @@ -149,6 +188,10 @@ jobs: windows-shared-libs: runs-on: windows-2022 # latest steps: + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Build ${{ env.PACKAGE_NAME }} + consumers run: | python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" @@ -157,6 +200,10 @@ jobs: windows-no-cpu-extensions: runs-on: windows-2022 # latest steps: + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Build ${{ env.PACKAGE_NAME }} + consumers run: | python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" @@ -165,6 +212,10 @@ jobs: windows-app-verifier: runs-on: windows-2022 # latest steps: + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Build ${{ env.PACKAGE_NAME }} + consumers run: | python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" @@ -177,6 +228,10 @@ jobs: macos-x64: runs-on: macos-14-large # latest steps: + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Build ${{ env.PACKAGE_NAME }} + consumers run: | python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" @@ -186,6 +241,10 @@ jobs: macos: runs-on: macos-14 # latest steps: + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Build ${{ env.PACKAGE_NAME }} + consumers run: | python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" @@ -195,6 +254,10 @@ jobs: macos-no-cpu-extensions: runs-on: macos-14 # latest steps: + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Build ${{ env.PACKAGE_NAME }} + consumers run: | python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" @@ -202,46 +265,57 @@ jobs: ./builder build -p ${{ env.PACKAGE_NAME }} --cmake-extra=-DUSE_CPU_EXTENSIONS=OFF openbsd: - runs-on: ubuntu-latest # unit tests hang on macos; use ubuntu instead + runs-on: ubuntu-24.04 # unit tests hang on macos; use ubuntu instead steps: - - uses: actions/checkout@v3 - - name: Build ${{ env.PACKAGE_NAME }} + consumers - uses: cross-platform-actions/action@v0.20.0 - with: - operating_system: openbsd - version: '7.2' - shell: bash - run: | - sudo pkg_add py3-urllib3 - python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')" - chmod a+x builder - ./builder build -p ${{ env.PACKAGE_NAME }} + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - uses: actions/checkout@v4 + - name: Build ${{ env.PACKAGE_NAME }} + consumers + uses: cross-platform-actions/action@v0.25.0 + with: + operating_system: openbsd + version: '7.5' + shell: bash + run: | + sudo pkg_add py3-urllib3 + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')" + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} freebsd: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v3 - with: - submodules: true - - name: Build ${{ env.PACKAGE_NAME }} + consumers - uses: cross-platform-actions/action@v0.20.0 - with: - operating_system: freebsd - version: '13.2' - run: | - sudo pkg install -y python3 devel/py-pip net/py-urllib3 cmake - python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')" - chmod a+x builder - ./builder build -p ${{ env.PACKAGE_NAME }} + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - uses: actions/checkout@v4 + with: + submodules: true + - name: Build ${{ env.PACKAGE_NAME }} + consumers + uses: cross-platform-actions/action@v0.25.0 + with: + operating_system: freebsd + version: '14.1' + run: | + sudo pkg install -y python3 devel/py-pip net/py-urllib3 cmake + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')" + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} cross_compile: name: Cross Compile ${{matrix.arch}} - runs-on: ubuntu-20.04 # latest + runs-on: ubuntu-24.04 # latest strategy: matrix: arch: [linux-armv6, linux-armv7, linux-arm64, android-armv7] - steps: + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Build ${{ env.PACKAGE_NAME }} run: | python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" @@ -249,8 +323,12 @@ jobs: ./builder build -p ${{ env.PACKAGE_NAME }} --target=${{matrix.arch}} linux-debug: - runs-on: ubuntu-20.04 # latest + runs-on: ubuntu-24.04 # latest steps: + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Build ${{ env.PACKAGE_NAME }} run: | aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh @@ -259,6 +337,10 @@ jobs: windows-debug: runs-on: windows-2022 # latest steps: + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Build ${{ env.PACKAGE_NAME }} + consumers run: | python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" @@ -267,6 +349,10 @@ jobs: macos-debug: runs-on: macos-14 # latest steps: + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Build ${{ env.PACKAGE_NAME }} + consumers run: | python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" diff --git a/tests/hash_table_test.c b/tests/hash_table_test.c index 9d9237d79..2a867ba3f 100644 --- a/tests/hash_table_test.c +++ b/tests/hash_table_test.c @@ -1084,7 +1084,7 @@ static int s_test_hash_churn_fn(struct aws_allocator *allocator, void *ctx) { int was_present; err_code = aws_hash_table_remove(&hash_table, e->key, NULL, &was_present); ASSERT_SUCCESS(err_code, "Unexpected failure removing element"); - if (i == 0 && entries[i - 1].key == e->key && entries[i - 1].is_removed) { + if (i != 0 && entries[i - 1].key == e->key && entries[i - 1].is_removed) { ASSERT_INT_EQUALS(0, was_present, "Expected item to be missing"); } else { ASSERT_INT_EQUALS(1, was_present, "Expected item to be present"); From 4f3c2eef2fdfa503f864bd509fff63cfe0cb9a9b Mon Sep 17 00:00:00 2001 From: Igor Abdrakhimov Date: Tue, 26 Nov 2024 10:31:58 -0800 Subject: [PATCH 7/7] Forward CMake variables to prebuilding dependencies (#1161) Co-authored-by: Igor Abdrakhimov Co-authored-by: Michael Graeb --- cmake/AwsPrebuildDependency.cmake | 80 +++++++++++++++++++++++++++++-- 1 file changed, 77 insertions(+), 3 deletions(-) diff --git a/cmake/AwsPrebuildDependency.cmake b/cmake/AwsPrebuildDependency.cmake index 0a52f69a8..2637f9594 100644 --- a/cmake/AwsPrebuildDependency.cmake +++ b/cmake/AwsPrebuildDependency.cmake @@ -13,11 +13,11 @@ function(aws_prebuild_dependency) set(multiValueArgs CMAKE_ARGUMENTS) cmake_parse_arguments(AWS_PREBUILD "" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - if(NOT AWS_PREBUILD_DEPENDENCY_NAME) + if (NOT AWS_PREBUILD_DEPENDENCY_NAME) message(FATAL_ERROR "Missing DEPENDENCY_NAME argument in prebuild_dependency function") endif() - if(NOT AWS_PREBUILD_SOURCE_DIR) + if (NOT AWS_PREBUILD_SOURCE_DIR) message(FATAL_ERROR "Missing SOURCE_DIR argument in prebuild_dependency function") endif() @@ -29,18 +29,31 @@ function(aws_prebuild_dependency) string(REPLACE ";" "\\\\;" ESCAPED_PREFIX_PATH "${CMAKE_PREFIX_PATH}") # For execute_process to accept a dynamically constructed command, it should be passed in a list format. set(cmakeCommand "${CMAKE_COMMAND}") + + # Get the list of optional and platform-specific variables that may affect build process. + set(cmakeOptionalVariables "") + aws_get_variables_for_prebuild_dependency(cmakeOptionalVariables) + list(APPEND cmakeCommand ${cmakeOptionalVariables}) + + # The following variables should always be used. list(APPEND cmakeCommand ${AWS_PREBUILD_SOURCE_DIR}) list(APPEND cmakeCommand -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}) list(APPEND cmakeCommand -DCMAKE_PREFIX_PATH=${ESCAPED_PREFIX_PATH}) list(APPEND cmakeCommand -DCMAKE_INSTALL_PREFIX=${depInstallDir}) list(APPEND cmakeCommand -DCMAKE_INSTALL_RPATH=${CMAKE_INSTALL_RPATH}) list(APPEND cmakeCommand -DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS}) + # In case a custom generator was provided via -G option. If we don't propagate it, the default value might + # conflict with other cmake options (e.g. CMAKE_MAKE_PROGRAM) or no make program could be found at all. + if (CMAKE_GENERATOR) + list(APPEND cmakeCommand -G${CMAKE_GENERATOR}) + endif() # Append provided arguments to CMake command. - if(AWS_PREBUILD_CMAKE_ARGUMENTS) + if (AWS_PREBUILD_CMAKE_ARGUMENTS) list(APPEND cmakeCommand ${AWS_PREBUILD_CMAKE_ARGUMENTS}) endif() + message(STATUS "cmake command for dependency ${AWS_PREBUILD_DEPENDENCY_NAME}: ${cmakeCommand}") # Configure dependency project. execute_process( COMMAND ${cmakeCommand} @@ -65,6 +78,10 @@ function(aws_prebuild_dependency) # Make the installation visible for others. list(INSERT CMAKE_PREFIX_PATH 0 ${depInstallDir}/) set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} PARENT_SCOPE) + if (CMAKE_CROSSCOMPILING) + list(INSERT CMAKE_FIND_ROOT_PATH 0 ${depInstallDir}) + set(CMAKE_FIND_ROOT_PATH ${CMAKE_FIND_ROOT_PATH} PARENT_SCOPE) + endif() set(${AWS_PREBUILD_DEPENDENCY_NAME}_PREBUILT TRUE CACHE INTERNAL "Indicate that dependency is built and can be used") @@ -75,3 +92,60 @@ function(aws_prebuild_dependency) DESTINATION ${CMAKE_INSTALL_PREFIX} ) endfunction() + +# Get list of optional or platform-specific variables that may affect build process. +function(aws_get_variables_for_prebuild_dependency AWS_CMAKE_PREBUILD_ARGS) + set(variables "") + + # The CMake variables below were chosen for Linux, BSD, and Android platforms. If you want to use the prebuild logic + # on other platforms, the chances are you have to handle additional variables (like CMAKE_OSX_SYSROOT). Refer to + # https://cmake.org/cmake/help/latest/manual/cmake-toolchains.7.html to update the list of handled variables, and + # then you can enable a new platform here. + if ((NOT UNIX) OR APPLE) + message(FATAL_ERROR "aws_get_variables_for_prebuild_dependency is called for unsupported platform") + endif() + + get_cmake_property(vars CACHE_VARIABLES) + foreach(var ${vars}) + # Variables in this block make sense only in cross-compiling mode. The variable list is created from the CMake + # documentation on toolchains: https://cmake.org/cmake/help/latest/manual/cmake-toolchains.7.html + # NOTE: Some variables are missed here (e.g. CMAKE_SYSROOT) because they can be set via toolchain file only. + if (CMAKE_CROSSCOMPILING AND ( + var STREQUAL "CMAKE_TOOLCHAIN_FILE" + OR var STREQUAL "CMAKE_SYSTEM_NAME" + OR var STREQUAL "CMAKE_SYSTEM_VERSION" + OR var STREQUAL "CMAKE_SYSTEM_PROCESSOR" + # Android-specific variables. + OR var MATCHES "^(CMAKE_)?ANDROID_")) + # To store a list within another list, it needs to be escaped first. + string(REPLACE ";" "\\\\;" escapedVar "${${var}}") + if (escapedVar) + list(APPEND variables "-D${var}=${escapedVar}") + endif() + endif() + + # Other optional variables applicable both in cross-compiling and non-cross-compiling modes. + # Refer to https://cmake.org/cmake/help/latest/manual/cmake-variables.7.html for each variable description. + if (var STREQUAL "CMAKE_C_COMPILER" + OR var MATCHES "^CMAKE_C_FLAGS(_DEBUG|_RELEASE|_RELWITHDEBINFO|_MINSIZEREL)?" + OR var STREQUAL "CMAKE_CXX_COMPILER" + OR var MATCHES "^CMAKE_CXX_FLAGS(_DEBUG|_RELEASE|_RELWITHDEBINFO|_MINSIZEREL)?" + OR var STREQUAL "CMAKE_LINKER_TYPE" + OR var MATCHES "^CMAKE_EXE_LINKER_FLAGS(_DEBUG|_RELEASE|_RELWITHDEBINFO|_MINSIZEREL)?" + OR var MATCHES "^CMAKE_MODULE_LINKER_FLAGS(_DEBUG|_RELEASE|_RELWITHDEBINFO|_MINSIZEREL)?" + OR var MATCHES "^CMAKE_STATIC_LINKER_FLAGS(_DEBUG|_RELEASE|_RELWITHDEBINFO|_MINSIZEREL)?" + OR var MATCHES "^CMAKE_SHARED_LINKER_FLAGS(_DEBUG|_RELEASE|_RELWITHDEBINFO|_MINSIZEREL)?" + OR var STREQUAL "CMAKE_MAKE_PROGRAM" + OR var MATCHES "^CMAKE_RUNTIME_OUTPUT_DIRECTORY" + OR var MATCHES "^CMAKE_ARCHIVE_OUTPUT_DIRECTORY" + OR var MATCHES "^CMAKE_LIBRARY_OUTPUT_DIRECTORY") + # To store a list within another list, it needs to be escaped first. + string(REPLACE ";" "\\\\;" escapedVar "${${var}}") + if (escapedVar) + list(APPEND variables "-D${var}=${escapedVar}") + endif() + endif() + endforeach() + + set(${AWS_CMAKE_PREBUILD_ARGS} ${variables} PARENT_SCOPE) +endfunction()