diff options
Diffstat (limited to 'mm/execmem.c')
| -rw-r--r-- | mm/execmem.c | 148 |
1 files changed, 77 insertions, 71 deletions
diff --git a/mm/execmem.c b/mm/execmem.c index 349cadd87486..74a178a87e75 100644 --- a/mm/execmem.c +++ b/mm/execmem.c @@ -51,8 +51,7 @@ static void *execmem_vmalloc(struct execmem_range *range, size_t size, } if (!p) { - if (!(vm_flags & VM_REQUIRE_HUGE_VMAP)) - pr_warn_ratelimited("unable to allocate memory\n"); + pr_warn_ratelimited("unable to allocate memory\n"); return NULL; } @@ -114,6 +113,28 @@ static inline unsigned long mas_range_len(struct ma_state *mas) return mas->last - mas->index + 1; } +static int execmem_set_direct_map_valid(struct vm_struct *vm, bool valid) +{ + unsigned int nr = (1 << get_vm_area_page_order(vm)); + unsigned int updated = 0; + int err = 0; + + for (int i = 0; i < vm->nr_pages; i += nr) { + err = set_direct_map_valid_noflush(vm->pages[i], nr, valid); + if (err) + goto err_restore; + updated += nr; + } + + return 0; + +err_restore: + for (int i = 0; i < updated; i += nr) + set_direct_map_valid_noflush(vm->pages[i], nr, !valid); + + return err; +} + static int execmem_force_rw(void *ptr, size_t size) { unsigned int nr = PAGE_ALIGN(size) >> PAGE_SHIFT; @@ -138,30 +159,24 @@ int execmem_restore_rox(void *ptr, size_t size) static void execmem_cache_clean(struct work_struct *work) { struct maple_tree *free_areas = &execmem_cache.free_areas; + struct mutex *mutex = &execmem_cache.mutex; MA_STATE(mas, free_areas, 0, ULONG_MAX); void *area; - guard(mutex)(&execmem_cache.mutex); + mutex_lock(mutex); mas_for_each(&mas, area, ULONG_MAX) { - struct vm_struct *vm = find_vm_area(area); size_t size = mas_range_len(&mas); - if (vm && get_vm_area_size(vm) == size) { - VM_WARN_ON_ONCE(!IS_ALIGNED(mas.index, PMD_SIZE) || - !IS_ALIGNED(size, PMD_SIZE)); + if (IS_ALIGNED(size, PMD_SIZE) && + IS_ALIGNED(mas.index, PMD_SIZE)) { + struct vm_struct *vm = find_vm_area(area); - /* - * Preallocate to ensure mas_store does not fail - * If there is no memory for the tree update, bail out, - * next execmem_free() might be more lucky - */ - if (mas_preallocate(&mas, NULL, GFP_KERNEL)) - break; - - mas_store_prealloc(&mas, NULL); + execmem_set_direct_map_valid(vm, true); + mas_store_gfp(&mas, NULL, GFP_KERNEL); vfree(area); } } + mutex_unlock(mutex); } static DECLARE_WORK(execmem_cache_clean_work, execmem_cache_clean); @@ -227,34 +242,30 @@ static void *execmem_cache_alloc_locked(struct execmem_range *range, size_t size addr = mas_free.index; last = mas_free.last; - mas_set_range(&mas_free, addr, addr + size - 1); - if (mas_preallocate(&mas_free, NULL, GFP_KERNEL)) - return NULL; - /* insert allocated size to busy_areas at range [addr, addr + size) */ mas_set_range(&mas_busy, addr, addr + size - 1); err = mas_store_gfp(&mas_busy, (void *)addr, GFP_KERNEL); if (err) - goto err_destroy_mas_free; + return NULL; - mas_store_prealloc(&mas_free, NULL); + mas_store_gfp(&mas_free, NULL, GFP_KERNEL); if (area_size > size) { + void *ptr = (void *)(addr + size); + /* * re-insert remaining free size to free_areas at range * [addr + size, last] - * the range matches an existing entry, so this cannot allocate */ - ptr = (void *)(addr + size); mas_set_range(&mas_free, addr + size, last); - mas_store_gfp(&mas_free, ptr, GFP_KERNEL); + err = mas_store_gfp(&mas_free, ptr, GFP_KERNEL); + if (err) { + mas_store_gfp(&mas_busy, NULL, GFP_KERNEL); + return NULL; + } } ptr = (void *)addr; return ptr; - -err_destroy_mas_free: - mas_destroy(&mas_free); - return NULL; } static void *__execmem_cache_alloc(struct execmem_range *range, size_t size) @@ -264,55 +275,58 @@ static void *__execmem_cache_alloc(struct execmem_range *range, size_t size) return execmem_cache_alloc_locked(range, size); } -static void *execmem_vmalloc_rox(struct execmem_range *range, size_t size, - unsigned long vm_flags) +static void *execmem_cache_populate_alloc(struct execmem_range *range, size_t size) { - void *p __free(vfree) = execmem_vmalloc(range, size, PAGE_KERNEL, vm_flags); - int err; + unsigned long vm_flags = VM_ALLOW_HUGE_VMAP; + struct mutex *mutex = &execmem_cache.mutex; + struct vm_struct *vm; + size_t alloc_size; + int err = -ENOMEM; + void *p; - if (!p) - return NULL; + alloc_size = round_up(size, PMD_SIZE); + p = execmem_vmalloc(range, alloc_size, PAGE_KERNEL, vm_flags); + if (!p) { + alloc_size = size; + p = execmem_vmalloc(range, alloc_size, PAGE_KERNEL, vm_flags); + } - /* fill memory with instructions that will trap */ - execmem_fill_trapping_insns(p, size); - set_vm_flush_reset_perms(p); - err = set_memory_rox((unsigned long)p, size >> PAGE_SHIFT); - if (err) + if (!p) return NULL; - return no_free_ptr(p); -} + vm = find_vm_area(p); + if (!vm) + goto err_free_mem; -static void *execmem_cache_populate_alloc(struct execmem_range *range, size_t size) -{ - unsigned long vm_flags = VM_REQUIRE_HUGE_VMAP; - size_t alloc_size = round_up(size, PMD_SIZE); - void *p __free(vfree) = NULL; - int err; + /* fill memory with instructions that will trap */ + execmem_fill_trapping_insns(p, alloc_size); - p = execmem_vmalloc_rox(range, alloc_size, vm_flags); - if (!p) - return NULL; + err = set_memory_rox((unsigned long)p, vm->nr_pages); + if (err) + goto err_free_mem; /* * New memory blocks must be allocated and added to the cache * as an atomic operation, otherwise they may be consumed * by a parallel call to the execmem_cache_alloc function. */ - guard(mutex)(&execmem_cache.mutex); + mutex_lock(mutex); err = execmem_cache_add_locked(p, alloc_size, GFP_KERNEL); if (err) - return NULL; + goto err_reset_direct_map; - /* the chunk belongs to the cache now */ - retain_and_null_ptr(p); + p = execmem_cache_alloc_locked(range, size); - return execmem_cache_alloc_locked(range, size); -} + mutex_unlock(mutex); -static void *execmem_alloc_rox(struct execmem_range *range, size_t size) -{ - return execmem_vmalloc_rox(range, size, 0); + return p; + +err_reset_direct_map: + mutex_unlock(mutex); + execmem_set_direct_map_valid(vm, true); +err_free_mem: + vfree(p); + return NULL; } static void *execmem_cache_alloc(struct execmem_range *range, size_t size) @@ -442,11 +456,6 @@ static void *execmem_cache_alloc(struct execmem_range *range, size_t size) return NULL; } -static void *execmem_alloc_rox(struct execmem_range *range, size_t size) -{ - return NULL; -} - static bool execmem_cache_free(void *ptr) { return false; @@ -456,20 +465,17 @@ static bool execmem_cache_free(void *ptr) void *execmem_alloc(enum execmem_type type, size_t size) { struct execmem_range *range = &execmem_info->ranges[type]; - bool use_rox_cache = range->flags & EXECMEM_ROX_CACHE; + bool use_cache = range->flags & EXECMEM_ROX_CACHE; unsigned long vm_flags = VM_FLUSH_RESET_PERMS; pgprot_t pgprot = range->pgprot; void *p = NULL; size = PAGE_ALIGN(size); - if (use_rox_cache) { + if (use_cache) p = execmem_cache_alloc(range, size); - if (!p) - p = execmem_alloc_rox(range, size); - } else { + else p = execmem_vmalloc(range, size, pgprot, vm_flags); - } return kasan_reset_tag(p); } |