diff --git a/include/aws/common/allocator.h b/include/aws/common/allocator.h index c0e1c85af..8d092055f 100644 --- a/include/aws/common/allocator.h +++ b/include/aws/common/allocator.h @@ -12,6 +12,22 @@ AWS_PUSH_SANE_WARNING_LEVEL AWS_EXTERN_C_BEGIN +/* + * Quick guide to allocators: + * CRT offers several flavours of allocators: + * - default: basic allocator that invokes system one directly. + * - aligned: basic allocator that aligns small allocations on 8 byte + * boundary and big buffers on 32/64 byte (system dependent) boundary. + * Aligned mem can improve perf on some operations, like memcpy or hashes. + * Depending on a system, can result in higher peak memory count in heavy + * acquire/free scenarios (ex. s3), due to memory fragmentation related to how + * aligned allocators work (over allocate, find aligned offset, release extra memory) + * - wrapped_cf: wraps MacOS's Security Framework allocator. + * - mem_tracer: wraps any allocator and provides tracing functionality to allocations + * - small_block_allocator: pools smaller allocations into preallocated buckets. + * Not actively maintained. Avoid if possible. + */ + /* Allocator structure. An instance of this will be passed around for anything needing memory allocation */ struct aws_allocator { void *(*mem_acquire)(struct aws_allocator *allocator, size_t size); @@ -32,6 +48,13 @@ bool aws_allocator_is_valid(const struct aws_allocator *alloc); AWS_COMMON_API struct aws_allocator *aws_default_allocator(void); +/* + * Allocator that align small allocations on 8 byte boundary and big allocations + * on 32/64 byte boundary. + */ +AWS_COMMON_API +struct aws_allocator *aws_aligned_allocator(void); + #ifdef __MACH__ /* Avoid pulling in CoreFoundation headers in a header file. */ struct __CFAllocator; /* NOLINT(bugprone-reserved-identifier) */ diff --git a/source/allocator.c b/source/allocator.c index e444d282a..02847b1ae 100644 --- a/source/allocator.c +++ b/source/allocator.c @@ -34,7 +34,7 @@ bool aws_allocator_is_valid(const struct aws_allocator *alloc) { return alloc && AWS_OBJECT_PTR_IS_READABLE(alloc) && alloc->mem_acquire && alloc->mem_release; } -static void *s_default_malloc(struct aws_allocator *allocator, size_t size) { +static void *s_aligned_malloc(struct aws_allocator *allocator, size_t size) { (void)allocator; /* larger allocations should be aligned so that AVX and friends can avoid * the extra preamble during unaligned versions of memcpy/memset on big buffers @@ -62,7 +62,7 @@ static void *s_default_malloc(struct aws_allocator *allocator, size_t size) { #endif } -static void s_default_free(struct aws_allocator *allocator, void *ptr) { +static void s_aligned_free(struct aws_allocator *allocator, void *ptr) { (void)allocator; #if !defined(_WIN32) free(ptr); @@ -71,7 +71,7 @@ static void s_default_free(struct aws_allocator *allocator, void *ptr) { #endif } -static void *s_default_realloc(struct aws_allocator *allocator, void *ptr, size_t oldsize, size_t newsize) { +static void *s_aligned_realloc(struct aws_allocator *allocator, void *ptr, size_t oldsize, size_t newsize) { (void)allocator; (void)oldsize; AWS_FATAL_PRECONDITION(newsize); @@ -82,12 +82,12 @@ static void *s_default_realloc(struct aws_allocator *allocator, void *ptr, size_ } /* newsize is > oldsize, need more memory */ - void *new_mem = s_default_malloc(allocator, newsize); - AWS_PANIC_OOM(new_mem, "Unhandled OOM encountered in s_default_malloc"); + void *new_mem = s_aligned_malloc(allocator, newsize); + AWS_PANIC_OOM(new_mem, "Unhandled OOM encountered in s_aligned_malloc"); if (ptr) { memcpy(new_mem, ptr, oldsize); - s_default_free(allocator, ptr); + s_aligned_free(allocator, ptr); } return new_mem; @@ -99,24 +99,75 @@ static void *s_default_realloc(struct aws_allocator *allocator, void *ptr, size_ #endif } -static void *s_default_calloc(struct aws_allocator *allocator, size_t num, size_t size) { - void *mem = s_default_malloc(allocator, num * size); - AWS_PANIC_OOM(mem, "Unhandled OOM encountered in s_default_malloc"); +static void *s_aligned_calloc(struct aws_allocator *allocator, size_t num, size_t size) { + void *mem = s_aligned_malloc(allocator, num * size); + AWS_PANIC_OOM(mem, "Unhandled OOM encountered in s_aligned_calloc"); memset(mem, 0, num * size); return mem; } +static void *s_non_aligned_malloc(struct aws_allocator *allocator, size_t size) { + (void)allocator; + void *result = malloc(size); + AWS_PANIC_OOM(result, "malloc failed to allocate memory"); + return result; +} + +static void s_non_aligned_free(struct aws_allocator *allocator, void *ptr) { + (void)allocator; + free(ptr); +} + +static void *s_non_aligned_realloc(struct aws_allocator *allocator, void *ptr, size_t oldsize, size_t newsize) { + (void)allocator; + (void)oldsize; + AWS_FATAL_PRECONDITION(newsize); + + if (newsize <= oldsize) { + return ptr; + } + + /* newsize is > oldsize, need more memory */ + void *new_mem = s_non_aligned_malloc(allocator, newsize); + AWS_PANIC_OOM(new_mem, "Unhandled OOM encountered in s_non_aligned_realloc"); + + if (ptr) { + memcpy(new_mem, ptr, oldsize); + s_non_aligned_free(allocator, ptr); + } + + return new_mem; +} + +static void *s_non_aligned_calloc(struct aws_allocator *allocator, size_t num, size_t size) { + (void)allocator; + void *mem = calloc(num, size); + AWS_PANIC_OOM(mem, "Unhandled OOM encountered in s_non_aligned_calloc"); + return mem; +} + static struct aws_allocator default_allocator = { - .mem_acquire = s_default_malloc, - .mem_release = s_default_free, - .mem_realloc = s_default_realloc, - .mem_calloc = s_default_calloc, + .mem_acquire = s_non_aligned_malloc, + .mem_release = s_non_aligned_free, + .mem_realloc = s_non_aligned_realloc, + .mem_calloc = s_non_aligned_calloc, }; struct aws_allocator *aws_default_allocator(void) { return &default_allocator; } +static struct aws_allocator aligned_allocator = { + .mem_acquire = s_aligned_malloc, + .mem_release = s_aligned_free, + .mem_realloc = s_aligned_realloc, + .mem_calloc = s_aligned_calloc, +}; + +struct aws_allocator *aws_aligned_allocator(void) { + return &aligned_allocator; +} + void *aws_mem_acquire(struct aws_allocator *allocator, size_t size) { AWS_FATAL_PRECONDITION(allocator != NULL); AWS_FATAL_PRECONDITION(allocator->mem_acquire != NULL); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 623eaf4c9..660c840ab 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -321,6 +321,8 @@ add_test_case(sba_churn) add_test_case(sba_metrics) add_test_case(default_threaded_reallocs) add_test_case(default_threaded_allocs_and_frees) +add_test_case(aligned_threaded_reallocs) +add_test_case(aligned_threaded_allocs_and_frees) add_test_case(test_memtrace_none) add_test_case(test_memtrace_count) diff --git a/tests/alloc_test.c b/tests/alloc_test.c index 66ff95999..364b3f319 100644 --- a/tests/alloc_test.c +++ b/tests/alloc_test.c @@ -48,10 +48,12 @@ static int s_test_alloc_nothing_fn(struct aws_allocator *allocator, void *ctx) { (void)allocator; (void)ctx; - struct aws_allocator test_allocator = {.mem_acquire = s_test_alloc_acquire, - .mem_release = s_test_alloc_release, - .mem_realloc = s_test_realloc, - .mem_calloc = s_test_calloc}; + struct aws_allocator test_allocator = { + .mem_acquire = s_test_alloc_acquire, + .mem_release = s_test_alloc_release, + .mem_realloc = s_test_realloc, + .mem_calloc = s_test_calloc, + }; /* realloc should handle the case correctly, return null, and free the memory */ void *p = aws_mem_acquire(&test_allocator, 12); @@ -346,3 +348,36 @@ static int s_default_threaded_allocs_and_frees(struct aws_allocator *allocator, return 0; } AWS_TEST_CASE(default_threaded_allocs_and_frees, s_default_threaded_allocs_and_frees) + +/* + * No align allocator tests. + */ +static int s_aligned_threaded_reallocs(struct aws_allocator *allocator, void *ctx) { + (void)allocator; + (void)ctx; + srand(15); + + struct aws_allocator *alloc = aws_mem_tracer_new(aws_aligned_allocator(), NULL, AWS_MEMTRACE_STACKS, 8); + + s_thread_test(alloc, s_threaded_realloc_worker, alloc); + + aws_mem_tracer_destroy(alloc); + + return 0; +} +AWS_TEST_CASE(aligned_threaded_reallocs, s_aligned_threaded_reallocs) + +static int s_aligned_threaded_allocs_and_frees(struct aws_allocator *allocator, void *ctx) { + (void)allocator; + (void)ctx; + srand(99); + + struct aws_allocator *alloc = aws_mem_tracer_new(aws_aligned_allocator(), NULL, AWS_MEMTRACE_STACKS, 8); + + s_thread_test(alloc, s_threaded_alloc_worker, alloc); + + aws_mem_tracer_destroy(alloc); + + return 0; +} +AWS_TEST_CASE(aligned_threaded_allocs_and_frees, s_aligned_threaded_allocs_and_frees)