diff options
Diffstat (limited to 'mm/vmalloc.c')
| -rw-r--r-- | mm/vmalloc.c | 40 |
1 files changed, 31 insertions, 9 deletions
diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 021fc2526886..60025b682bc0 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -43,6 +43,7 @@ #include <asm/tlbflush.h> #include <asm/shmparam.h> #include <linux/page_owner.h> +#include <linux/cleanup.h> #define CREATE_TRACE_POINTS #include <trace/events/vmalloc.h> @@ -155,10 +156,21 @@ static int vmap_try_huge_pmd(pmd_t *pmd, unsigned long addr, unsigned long end, if (!IS_ALIGNED(phys_addr, PMD_SIZE)) return 0; - if (pmd_present(*pmd) && !pmd_free_pte_page(pmd, addr)) - return 0; + if (!pmd_present(*pmd)) + return pmd_set_huge(pmd, phys_addr, prot); - return pmd_set_huge(pmd, phys_addr, prot); + /* + * Acquire the mmap read lock to exclude ptdump, which walks + * kernel page tables it does not own under the mmap write lock. + * + * Concurrent read lock holders are safe: each exclusively owns + * the range it operates on and cannot reach this page table. + */ + scoped_cond_guard(mmap_read_lock_try, return 0, &init_mm) { + if (!pmd_free_pte_page(pmd, addr)) + return 0; + return pmd_set_huge(pmd, phys_addr, prot); + } } static int vmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, @@ -205,10 +217,15 @@ static int vmap_try_huge_pud(pud_t *pud, unsigned long addr, unsigned long end, if (!IS_ALIGNED(phys_addr, PUD_SIZE)) return 0; - if (pud_present(*pud) && !pud_free_pmd_page(pud, addr)) - return 0; + if (!pud_present(*pud)) + return pud_set_huge(pud, phys_addr, prot); - return pud_set_huge(pud, phys_addr, prot); + /* See comment in vmap_try_huge_pmd(). */ + scoped_cond_guard(mmap_read_lock_try, return 0, &init_mm) { + if (!pud_free_pmd_page(pud, addr)) + return 0; + return pud_set_huge(pud, phys_addr, prot); + } } static int vmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, @@ -256,10 +273,15 @@ static int vmap_try_huge_p4d(p4d_t *p4d, unsigned long addr, unsigned long end, if (!IS_ALIGNED(phys_addr, P4D_SIZE)) return 0; - if (p4d_present(*p4d) && !p4d_free_pud_page(p4d, addr)) - return 0; + if (!p4d_present(*p4d)) + return p4d_set_huge(p4d, phys_addr, prot); - return p4d_set_huge(p4d, phys_addr, prot); + /* See comment in vmap_try_huge_pmd(). */ + scoped_cond_guard(mmap_read_lock_try, return 0, &init_mm) { + if (!p4d_free_pud_page(p4d, addr)) + return 0; + return p4d_set_huge(p4d, phys_addr, prot); + } } static int vmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end, |