diff --git a/core/heap.c b/core/heap.c index 49618d9d04b..162264f5483 100644 --- a/core/heap.c +++ b/core/heap.c @@ -1,5 +1,5 @@ /* ********************************************************** - * Copyright (c) 2010-2023 Google, Inc. All rights reserved. + * Copyright (c) 2010-2024 Google, Inc. All rights reserved. * Copyright (c) 2001-2010 VMware, Inc. All rights reserved. * **********************************************************/ @@ -782,7 +782,7 @@ report_w_xor_x_fatal_error_and_exit(void) } static void -vmm_place_vmcode(vm_heap_t *vmh, size_t size, heap_error_code_t *error_code) +vmm_place_vmcode(vm_heap_t *vmh, /*INOUT*/ size_t *size, heap_error_code_t *error_code) { ptr_uint_t preferred = 0; #ifdef X64 @@ -818,7 +818,7 @@ vmm_place_vmcode(vm_heap_t *vmh, size_t size, heap_error_code_t *error_code) } vmh->alloc_start = os_heap_reserve_in_region( (void *)ALIGN_FORWARD(reach_base, PAGE_SIZE), - (void *)ALIGN_BACKWARD(reach_end, PAGE_SIZE), size + add_for_align, + (void *)ALIGN_BACKWARD(reach_end, PAGE_SIZE), *size + add_for_align, error_code, true /*+x*/); if (vmh->alloc_start != NULL) { vmh->start_addr = (heap_pc)ALIGN_FORWARD( @@ -851,11 +851,11 @@ vmm_place_vmcode(vm_heap_t *vmh, size_t size, heap_error_code_t *error_code) DYNAMO_OPTION(vmm_block_size)); preferred = ALIGN_FORWARD(preferred, OS_ALLOC_GRANULARITY); /* overflow check: w/ vm_base shouldn't happen so debug-only check */ - ASSERT(!POINTER_OVERFLOW_ON_ADD(preferred, size)); + ASSERT(!POINTER_OVERFLOW_ON_ADD(preferred, *size)); /* let's assume a single chunk is sufficient to reserve */ #ifdef X64 if ((byte *)preferred < heap_allowable_region_start || - (byte *)preferred + size > heap_allowable_region_end) { + (byte *)preferred + *size > heap_allowable_region_end) { *error_code = HEAP_ERROR_NOT_AT_PREFERRED; LOG(GLOBAL, LOG_HEAP, 1, "vmm_heap_unit_init preferred=" PFX " too far from " PFX "-" PFX "\n", @@ -863,7 +863,7 @@ vmm_place_vmcode(vm_heap_t *vmh, size_t size, heap_error_code_t *error_code) } else { #endif vmh->alloc_start = - os_heap_reserve((void *)preferred, size, error_code, true /*+x*/); + os_heap_reserve((void *)preferred, *size, error_code, true /*+x*/); vmh->start_addr = vmh->alloc_start; LOG(GLOBAL, LOG_HEAP, 1, "vmm_heap_unit_init preferred=" PFX " got start_addr=" PFX "\n", @@ -877,29 +877,30 @@ vmm_place_vmcode(vm_heap_t *vmh, size_t size, heap_error_code_t *error_code) * syslog or assert here */ /* need extra size to ensure alignment */ - vmh->alloc_size = size + DYNAMO_OPTION(vmm_block_size); + vmh->alloc_size = *size + DYNAMO_OPTION(vmm_block_size); #ifdef X64 /* PR 215395, make sure allocation satisfies heap reachability contraints */ vmh->alloc_start = os_heap_reserve_in_region( (void *)ALIGN_FORWARD(heap_allowable_region_start, PAGE_SIZE), (void *)ALIGN_BACKWARD(heap_allowable_region_end, PAGE_SIZE), - size + DYNAMO_OPTION(vmm_block_size), error_code, true /*+x*/); + *size + DYNAMO_OPTION(vmm_block_size), error_code, true /*+x*/); #else vmh->alloc_start = (heap_pc)os_heap_reserve( - NULL, size + DYNAMO_OPTION(vmm_block_size), error_code, true /*+x*/); + NULL, *size + DYNAMO_OPTION(vmm_block_size), error_code, true /*+x*/); #endif vmh->start_addr = (heap_pc)ALIGN_FORWARD(vmh->alloc_start, DYNAMO_OPTION(vmm_block_size)); LOG(GLOBAL, LOG_HEAP, 1, "vmm_heap_unit_init unable to allocate at preferred=" PFX " letting OS place sz=%dM addr=" PFX "\n", - preferred, size / (1024 * 1024), vmh->start_addr); + preferred, *size / (1024 * 1024), vmh->start_addr); if (vmh->alloc_start == NULL && DYNAMO_OPTION(vm_allow_smaller)) { /* Just a little smaller might fit */ size_t sub = (size_t)ALIGN_FORWARD(size / 16, 1024 * 1024); SYSLOG_INTERNAL_WARNING_ONCE("Full size vmm heap allocation failed"); - if (size > sub) - size -= sub; + /* Don't go too small. */ + if (*size > 2 * sub) + *size -= sub; else break; } else @@ -913,7 +914,7 @@ vmm_place_vmcode(vm_heap_t *vmh, size_t size, heap_error_code_t *error_code) * on top. TODO i#3566: We need a different strategy on Windows. */ /* Ensure os_map_file ignores vmcode: */ - ASSERT(!is_vmm_reserved_address(vmh->start_addr, size, NULL, NULL)); + ASSERT(!is_vmm_reserved_address(vmh->start_addr, *size, NULL, NULL)); size_t map_size = vmh->alloc_size; byte *map_base = os_map_file(heapmgt->dual_map_file, &map_size, 0, vmh->alloc_start, @@ -926,9 +927,9 @@ vmm_place_vmcode(vm_heap_t *vmh, size_t size, heap_error_code_t *error_code) /* ensure future out-of-block heap allocations are reachable from this allocation */ if (vmh->start_addr != NULL) { ASSERT(vmh->start_addr >= heap_allowable_region_start && - !POINTER_OVERFLOW_ON_ADD(vmh->start_addr, size) && - vmh->start_addr + size <= heap_allowable_region_end); - request_region_be_heap_reachable(vmh->start_addr, size); + !POINTER_OVERFLOW_ON_ADD(vmh->start_addr, *size) && + vmh->start_addr + *size <= heap_allowable_region_end); + request_region_be_heap_reachable(vmh->start_addr, *size); } #endif ASSERT(ALIGNED(vmh->start_addr, DYNAMO_OPTION(vmm_block_size))); @@ -977,7 +978,7 @@ vmm_heap_unit_init(vm_heap_t *vmh, size_t size, bool is_vmcode, const char *name ASSERT_NOT_REACHED(); } } - vmm_place_vmcode(vmh, size, &error_code); + vmm_place_vmcode(vmh, &size, &error_code); if (DYNAMO_OPTION(satisfy_w_xor_x)) { size_t map_size = vmh->alloc_size; heapmgt->vmcode_writable_alloc = diff --git a/core/unix/os.c b/core/unix/os.c index e3b35471e8d..4c30f71a86b 100644 --- a/core/unix/os.c +++ b/core/unix/os.c @@ -3480,11 +3480,30 @@ os_heap_reserve(void *preferred, size_t size, heap_error_code_t *error_code, if (executable) os_flags |= MAP_JIT; #endif +#if defined(LINUX) && !defined(ANDROID) + if (preferred != NULL) { + /* We fail if we don't get the preferred address, so we use the 4.17+ + * fixed-but-no-clobber flag to ensure the kernel actually tries for our hint. + */ + os_flags |= MAP_FIXED_NOREPLACE; + } +#endif - /* FIXME: note that this memory is in fact still committed - see man mmap */ - /* FIXME: case 2347 on Linux or -vm_reserve should be set to false */ - /* FIXME: Need to actually get a mmap-ing with |MAP_NORESERVE */ + /* We could try for |MAP_NORESERVE but usually overcommit is set on the + * system and pages aren't actually committed until we touch them. + */ p = mmap_syscall(preferred, size, prot, os_flags, -1, 0); +#if defined(LINUX) && !defined(ANDROID) + if (preferred != NULL && p == (void *)(-EINVAL)) { + /* We're probably on an old pre-4.17 kernel. + * We could have a global var but we live w/ doing this every time. + */ + SYSLOG_INTERNAL_WARNING_ONCE( + "Got EINVAL on mmap: removing MAP_FIXED_NOREPLACE\n"); + os_flags &= ~MAP_FIXED_NOREPLACE; + p = mmap_syscall(preferred, size, prot, os_flags, -1, 0); + } +#endif if (!mmap_syscall_succeeded(p)) { *error_code = -(heap_error_code_t)(ptr_int_t)p; LOG(GLOBAL, LOG_HEAP, 4, "os_heap_reserve %d bytes failed " PFX "\n", size, p); @@ -3497,8 +3516,8 @@ os_heap_reserve(void *preferred, size_t size, heap_error_code_t *error_code, os_heap_free(p, size, &dummy); ASSERT(dummy == HEAP_ERROR_SUCCESS); LOG(GLOBAL, LOG_HEAP, 4, - "os_heap_reserve %d bytes at " PFX " not preferred " PFX "\n", size, - preferred, p); + "os_heap_reserve %d bytes at " PFX " not preferred " PFX "\n", size, p, + preferred); return NULL; } else { *error_code = HEAP_ERROR_SUCCESS; @@ -3564,7 +3583,8 @@ os_heap_reserve_in_region(void *start, void *end, size_t size, end); /* if no restriction on location use regular os_heap_reserve() */ - if (start == (void *)PTR_UINT_0 && end == (void *)POINTER_MAX) + if (start == (void *)PTR_UINT_0 && + end == (void *)ALIGN_BACKWARD(POINTER_MAX, PAGE_SIZE)) return os_heap_reserve(NULL, size, error_code, executable); /* loop to handle races */