Merge branch 'main' into mem_freebsd

include/aws/common/allocator.h

@@ -12,6 +12,22 @@
 AWS_PUSH_SANE_WARNING_LEVEL
 AWS_EXTERN_C_BEGIN
 
+/*
+ * Quick guide to allocators:
+ *  CRT offers several flavours of allocators:
+ * - default: basic allocator that invokes system one directly.
+ * - aligned: basic allocator that aligns small allocations on 8 byte
+ *   boundary and big buffers on 32/64 byte (system dependent) boundary.
+ *   Aligned mem can improve perf on some operations, like memcpy or hashes.
+ *   Depending on a system, can result in higher peak memory count in heavy
+ *   acquire/free scenarios (ex. s3), due to memory fragmentation related to how
+ *   aligned allocators work (over allocate, find aligned offset, release extra memory)
+ * - wrapped_cf: wraps MacOS's Security Framework allocator.
+ * - mem_tracer: wraps any allocator and provides tracing functionality to allocations
+ * - small_block_allocator: pools smaller allocations into preallocated buckets.
+ *   Not actively maintained. Avoid if possible.
+ */
+
 /* Allocator structure. An instance of this will be passed around for anything needing memory allocation */
 struct aws_allocator {
     void *(*mem_acquire)(struct aws_allocator *allocator, size_t size);
@@ -32,6 +48,13 @@ bool aws_allocator_is_valid(const struct aws_allocator *alloc);
 AWS_COMMON_API
 struct aws_allocator *aws_default_allocator(void);
 
+/*
+ * Allocator that align small allocations on 8 byte boundary and big allocations
+ * on 32/64 byte boundary.
+ */
+AWS_COMMON_API
+struct aws_allocator *aws_aligned_allocator(void);
+
 #ifdef __MACH__
 /* Avoid pulling in CoreFoundation headers in a header file. */
 struct __CFAllocator; /* NOLINT(bugprone-reserved-identifier) */

source/allocator.c

@@ -34,7 +34,7 @@ bool aws_allocator_is_valid(const struct aws_allocator *alloc) {
     return alloc && AWS_OBJECT_PTR_IS_READABLE(alloc) && alloc->mem_acquire && alloc->mem_release;
 }
 
-static void *s_default_malloc(struct aws_allocator *allocator, size_t size) {
+static void *s_aligned_malloc(struct aws_allocator *allocator, size_t size) {
     (void)allocator;
     /* larger allocations should be aligned so that AVX and friends can avoid
      * the extra preamble during unaligned versions of memcpy/memset on big buffers
@@ -62,7 +62,7 @@ static void *s_default_malloc(struct aws_allocator *allocator, size_t size) {
 #endif
 }
 
-static void s_default_free(struct aws_allocator *allocator, void *ptr) {
+static void s_aligned_free(struct aws_allocator *allocator, void *ptr) {
     (void)allocator;
 #if !defined(_WIN32)
     free(ptr);
@@ -71,7 +71,7 @@ static void s_default_free(struct aws_allocator *allocator, void *ptr) {
 #endif
 }
 
-static void *s_default_realloc(struct aws_allocator *allocator, void *ptr, size_t oldsize, size_t newsize) {
+static void *s_aligned_realloc(struct aws_allocator *allocator, void *ptr, size_t oldsize, size_t newsize) {
     (void)allocator;
     (void)oldsize;
     AWS_FATAL_PRECONDITION(newsize);
@@ -82,12 +82,12 @@ static void *s_default_realloc(struct aws_allocator *allocator, void *ptr, size_
     }
 
     /* newsize is > oldsize, need more memory */
-    void *new_mem = s_default_malloc(allocator, newsize);
-    AWS_PANIC_OOM(new_mem, "Unhandled OOM encountered in s_default_malloc");
+    void *new_mem = s_aligned_malloc(allocator, newsize);
+    AWS_PANIC_OOM(new_mem, "Unhandled OOM encountered in s_aligned_malloc");
 
     if (ptr) {
         memcpy(new_mem, ptr, oldsize);
-        s_default_free(allocator, ptr);
+        s_aligned_free(allocator, ptr);
     }
 
     return new_mem;
@@ -99,24 +99,75 @@ static void *s_default_realloc(struct aws_allocator *allocator, void *ptr, size_
 #endif
 }
 
-static void *s_default_calloc(struct aws_allocator *allocator, size_t num, size_t size) {
-    void *mem = s_default_malloc(allocator, num * size);
-    AWS_PANIC_OOM(mem, "Unhandled OOM encountered in s_default_malloc");
+static void *s_aligned_calloc(struct aws_allocator *allocator, size_t num, size_t size) {
+    void *mem = s_aligned_malloc(allocator, num * size);
+    AWS_PANIC_OOM(mem, "Unhandled OOM encountered in s_aligned_calloc");
     memset(mem, 0, num * size);
     return mem;
 }
 
+static void *s_non_aligned_malloc(struct aws_allocator *allocator, size_t size) {
+    (void)allocator;
+    void *result = malloc(size);
+    AWS_PANIC_OOM(result, "malloc failed to allocate memory");
+    return result;
+}
+
+static void s_non_aligned_free(struct aws_allocator *allocator, void *ptr) {
+    (void)allocator;
+    free(ptr);
+}
+
+static void *s_non_aligned_realloc(struct aws_allocator *allocator, void *ptr, size_t oldsize, size_t newsize) {
+    (void)allocator;
+    (void)oldsize;
+    AWS_FATAL_PRECONDITION(newsize);
+
+    if (newsize <= oldsize) {
+        return ptr;
+    }
+
+    /* newsize is > oldsize, need more memory */
+    void *new_mem = s_non_aligned_malloc(allocator, newsize);
+    AWS_PANIC_OOM(new_mem, "Unhandled OOM encountered in s_non_aligned_realloc");
+
+    if (ptr) {
+        memcpy(new_mem, ptr, oldsize);
+        s_non_aligned_free(allocator, ptr);
+    }
+
+    return new_mem;
+}
+
+static void *s_non_aligned_calloc(struct aws_allocator *allocator, size_t num, size_t size) {
+    (void)allocator;
+    void *mem = calloc(num, size);
+    AWS_PANIC_OOM(mem, "Unhandled OOM encountered in s_non_aligned_calloc");
+    return mem;
+}
+
 static struct aws_allocator default_allocator = {
-    .mem_acquire = s_default_malloc,
-    .mem_release = s_default_free,
-    .mem_realloc = s_default_realloc,
-    .mem_calloc = s_default_calloc,
+    .mem_acquire = s_non_aligned_malloc,
+    .mem_release = s_non_aligned_free,
+    .mem_realloc = s_non_aligned_realloc,
+    .mem_calloc = s_non_aligned_calloc,
 };
 
 struct aws_allocator *aws_default_allocator(void) {
     return &default_allocator;
 }
 
+static struct aws_allocator aligned_allocator = {
+    .mem_acquire = s_aligned_malloc,
+    .mem_release = s_aligned_free,
+    .mem_realloc = s_aligned_realloc,
+    .mem_calloc = s_aligned_calloc,
+};
+
+struct aws_allocator *aws_aligned_allocator(void) {
+    return &aligned_allocator;
+}
+
 void *aws_mem_acquire(struct aws_allocator *allocator, size_t size) {
     AWS_FATAL_PRECONDITION(allocator != NULL);
     AWS_FATAL_PRECONDITION(allocator->mem_acquire != NULL);

tests/CMakeLists.txt

@@ -321,6 +321,8 @@ add_test_case(sba_churn)
 add_test_case(sba_metrics)
 add_test_case(default_threaded_reallocs)
 add_test_case(default_threaded_allocs_and_frees)
+add_test_case(aligned_threaded_reallocs)
+add_test_case(aligned_threaded_allocs_and_frees)
 
 add_test_case(test_memtrace_none)
 add_test_case(test_memtrace_count)

tests/alloc_test.c

@@ -48,10 +48,12 @@ static int s_test_alloc_nothing_fn(struct aws_allocator *allocator, void *ctx) {
     (void)allocator;
     (void)ctx;
 
-    struct aws_allocator test_allocator = {.mem_acquire = s_test_alloc_acquire,
-                                           .mem_release = s_test_alloc_release,
-                                           .mem_realloc = s_test_realloc,
-                                           .mem_calloc = s_test_calloc};
+    struct aws_allocator test_allocator = {
+        .mem_acquire = s_test_alloc_acquire,
+        .mem_release = s_test_alloc_release,
+        .mem_realloc = s_test_realloc,
+        .mem_calloc = s_test_calloc,
+    };
 
     /* realloc should handle the case correctly, return null, and free the memory */
     void *p = aws_mem_acquire(&test_allocator, 12);
@@ -346,3 +348,36 @@ static int s_default_threaded_allocs_and_frees(struct aws_allocator *allocator,
     return 0;
 }
 AWS_TEST_CASE(default_threaded_allocs_and_frees, s_default_threaded_allocs_and_frees)
+
+/*
+ * No align allocator tests.
+ */
+static int s_aligned_threaded_reallocs(struct aws_allocator *allocator, void *ctx) {
+    (void)allocator;
+    (void)ctx;
+    srand(15);
+
+    struct aws_allocator *alloc = aws_mem_tracer_new(aws_aligned_allocator(), NULL, AWS_MEMTRACE_STACKS, 8);
+
+    s_thread_test(alloc, s_threaded_realloc_worker, alloc);
+
+    aws_mem_tracer_destroy(alloc);
+
+    return 0;
+}
+AWS_TEST_CASE(aligned_threaded_reallocs, s_aligned_threaded_reallocs)
+
+static int s_aligned_threaded_allocs_and_frees(struct aws_allocator *allocator, void *ctx) {
+    (void)allocator;
+    (void)ctx;
+    srand(99);
+
+    struct aws_allocator *alloc = aws_mem_tracer_new(aws_aligned_allocator(), NULL, AWS_MEMTRACE_STACKS, 8);
+
+    s_thread_test(alloc, s_threaded_alloc_worker, alloc);
+
+    aws_mem_tracer_destroy(alloc);
+
+    return 0;
+}
+AWS_TEST_CASE(aligned_threaded_allocs_and_frees, s_aligned_threaded_allocs_and_frees)