From f676897244ae06180b5b61cb78d2e016088384f6 Mon Sep 17 00:00:00 2001 From: Dmitriy Musatkin <63878209+DmitriyMusatkin@users.noreply.github.com> Date: Mon, 23 Oct 2023 15:08:00 -0700 Subject: [PATCH 1/3] Basic process mem usage functions (#1065) --- CMakeLists.txt | 5 +++- include/aws/common/system_resource_util.h | 30 +++++++++++++++++++++ source/posix/system_resource_utils.c | 32 +++++++++++++++++++++++ source/windows/system_resource_utils.c | 31 ++++++++++++++++++++++ tests/CMakeLists.txt | 2 ++ tests/system_resource_util_test.c | 22 ++++++++++++++++ 6 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 include/aws/common/system_resource_util.h create mode 100644 source/posix/system_resource_utils.c create mode 100644 source/windows/system_resource_utils.c create mode 100644 tests/system_resource_util_test.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 0ff1d9e7e..09803ffa9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -82,7 +82,10 @@ if (WIN32) endif () list(APPEND PLATFORM_DEFINES WINDOWS_KERNEL_LIB=${WINDOWS_KERNEL_LIB}) - list(APPEND PLATFORM_LIBS BCrypt ${WINDOWS_KERNEL_LIB} Ws2_32 Shlwapi) + # PSAPI_VERSION=1 is needed to support GetProcessMemoryInfo on both pre and + # post Win7 OS's. + list(APPEND PLATFORM_DEFINES PSAPI_VERSION=1) + list(APPEND PLATFORM_LIBS BCrypt ${WINDOWS_KERNEL_LIB} Ws2_32 Shlwapi Psapi) else () file(GLOB AWS_COMMON_OS_HEADERS "include/aws/common/posix/*" diff --git a/include/aws/common/system_resource_util.h b/include/aws/common/system_resource_util.h new file mode 100644 index 000000000..ffe8d53e4 --- /dev/null +++ b/include/aws/common/system_resource_util.h @@ -0,0 +1,30 @@ +#ifndef AWS_COMMON_SYSTEM_RESOURCE_UTIL_H +#define AWS_COMMON_SYSTEM_RESOURCE_UTIL_H + +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include + +AWS_PUSH_SANE_WARNING_LEVEL + +AWS_EXTERN_C_BEGIN + +struct aws_memory_usage_stats { + size_t maxrss; /* max resident set size in kilobytes since program start */ + size_t page_faults; /* num of page faults since program start */ + + size_t _reserved[8]; +}; + +/* + * Get memory usage for current process. + * Raises AWS_ERROR_SYS_CALL_FAILURE on failure. + */ +AWS_COMMON_API int aws_init_memory_usage_for_current_process(struct aws_memory_usage_stats *memory_usage); + +AWS_EXTERN_C_END +AWS_POP_SANE_WARNING_LEVEL + +#endif /* AWS_COMMON_SYSTEM_RESOURCE_UTIL_H */ diff --git a/source/posix/system_resource_utils.c b/source/posix/system_resource_utils.c new file mode 100644 index 000000000..68165072b --- /dev/null +++ b/source/posix/system_resource_utils.c @@ -0,0 +1,32 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +#include + +#include + +int aws_init_memory_usage_for_current_process(struct aws_memory_usage_stats *memory_usage) { + AWS_PRECONDITION(memory_usage); + + AWS_ZERO_STRUCT(*memory_usage); + struct rusage usage; + + if (getrusage(RUSAGE_SELF, &usage)) { + return aws_raise_error(AWS_ERROR_SYS_CALL_FAILURE); + } + +#if defined(AWS_OS_APPLE) + /* + * For some reason Apple switched to reporting this in bytes instead of KB + * around MacOS 10.6. + * Make it back to KB. Result might be slightly off due to rounding. + */ + memory_usage->maxrss = usage.ru_maxrss / 1024; +#else + memory_usage->maxrss = usage.ru_maxrss; +#endif + memory_usage->page_faults = usage.ru_majflt; + return AWS_OP_SUCCESS; +} diff --git a/source/windows/system_resource_utils.c b/source/windows/system_resource_utils.c new file mode 100644 index 000000000..618a9e25b --- /dev/null +++ b/source/windows/system_resource_utils.c @@ -0,0 +1,31 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +#include + +#include + +#include + +int aws_init_memory_usage_for_current_process(struct aws_memory_usage_stats *memory_usage) { + AWS_PRECONDITION(memory_usage); + + AWS_ZERO_STRUCT(*memory_usage); + HANDLE hProcess = GetCurrentProcess(); + + PROCESS_MEMORY_COUNTERS pmc; + + BOOL ret = GetProcessMemoryInfo(hProcess, &pmc, sizeof(pmc)); + CloseHandle(hProcess); + + if (!ret) { + return aws_raise_error(AWS_ERROR_SYS_CALL_FAILURE); + } + + memory_usage->maxrss = pmc.PeakWorkingSetSize; + memory_usage->page_faults = pmc.PageFaultCount; + + return AWS_OP_SUCCESS; +} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4681e04c3..de072868a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -402,6 +402,8 @@ add_test_case(ring_buffer_acquire_up_to_multi_threaded_test) add_test_case(string_to_log_level_success_test) add_test_case(string_to_log_level_failure_test) +add_test_case(test_memory_usage_maxrss) + if(NOT ANDROID) add_test_case(test_logging_filter_at_AWS_LL_NONE_s_logf_all_levels) add_test_case(test_logging_filter_at_AWS_LL_FATAL_s_logf_all_levels) diff --git a/tests/system_resource_util_test.c b/tests/system_resource_util_test.c new file mode 100644 index 000000000..ba5945d93 --- /dev/null +++ b/tests/system_resource_util_test.c @@ -0,0 +1,22 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +#include + +#include + +static int s_test_memory_usage_maxrss(struct aws_allocator *allocator, void *ctx) { + (void)allocator; + (void)ctx; + + struct aws_memory_usage_stats mu; + ASSERT_SUCCESS(aws_init_memory_usage_for_current_process(&mu)); + + ASSERT_TRUE(mu.maxrss > 0); + + return 0; +} + +AWS_TEST_CASE(test_memory_usage_maxrss, s_test_memory_usage_maxrss) From 0ce8c560872bbd494ae09b97a787f647cf86f041 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Wed, 25 Oct 2023 16:21:07 -0500 Subject: [PATCH 2/3] Fix loading of libnuma.so (#1068) --- source/common.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/common.c b/source/common.c index 9a6f56cf4..f734c16e6 100644 --- a/source/common.c +++ b/source/common.c @@ -323,15 +323,15 @@ void aws_common_library_init(struct aws_allocator *allocator) { assumptions due to the way loaders and dlload are often implemented and those symbols are defined by things like libpthread.so on some unix distros. Sorry about the memory usage here, but it's our only safe choice. Also, please don't do numa configurations if memory is your economic bottleneck. */ - g_libnuma_handle = dlopen("libnuma.so", RTLD_LOCAL); + g_libnuma_handle = dlopen("libnuma.so", RTLD_LAZY | RTLD_LOCAL); /* turns out so versioning is really inconsistent these days */ if (!g_libnuma_handle) { - g_libnuma_handle = dlopen("libnuma.so.1", RTLD_LOCAL); + g_libnuma_handle = dlopen("libnuma.so.1", RTLD_LAZY | RTLD_LOCAL); } if (!g_libnuma_handle) { - g_libnuma_handle = dlopen("libnuma.so.2", RTLD_LOCAL); + g_libnuma_handle = dlopen("libnuma.so.2", RTLD_LAZY | RTLD_LOCAL); } if (g_libnuma_handle) { From e381a7beeacb070f1816989dcb0e2c0ae6eccaea Mon Sep 17 00:00:00 2001 From: Alfred G <28123637+alfred2g@users.noreply.github.com> Date: Thu, 26 Oct 2023 09:45:07 -0700 Subject: [PATCH 3/3] Comment: fix (#1069) --- include/aws/common/hash_table.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/aws/common/hash_table.h b/include/aws/common/hash_table.h index 195c9bdff..a23c4f966 100644 --- a/include/aws/common/hash_table.h +++ b/include/aws/common/hash_table.h @@ -276,6 +276,7 @@ int aws_hash_table_create( * * Returns AWS_OP_SUCCESS if an item was found or created. * Raises AWS_ERROR_OOM if hash table expansion was required and memory + * allocation failed. */ AWS_COMMON_API int aws_hash_table_put(struct aws_hash_table *map, const void *key, void *value, int *was_created);