diff options
Diffstat (limited to 'mm')
| -rw-r--r-- | mm/hugetlb.c | 95 | ||||
| -rw-r--r-- | mm/hugetlb_cma.c | 13 | ||||
| -rw-r--r-- | mm/hugetlb_cma.h | 8 | ||||
| -rw-r--r-- | mm/internal.h | 9 |
4 files changed, 54 insertions, 71 deletions
diff --git a/mm/hugetlb.c b/mm/hugetlb.c index df2f8b996b18..cb37380849d1 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -3039,79 +3039,58 @@ out_end_reservation: static __init void *alloc_bootmem(struct hstate *h, int nid, bool node_exact) { - struct huge_bootmem_page *m; - int listnode = nid; - if (hugetlb_early_cma(h)) - m = hugetlb_cma_alloc_bootmem(h, &listnode, node_exact); - else { - if (node_exact) - m = memblock_alloc_exact_nid_raw(huge_page_size(h), + return hugetlb_cma_alloc_bootmem(h, nid, node_exact); + + if (node_exact) + return memblock_alloc_exact_nid_raw(huge_page_size(h), huge_page_size(h), 0, MEMBLOCK_ALLOC_ACCESSIBLE, nid); - else { - m = memblock_alloc_try_nid_raw(huge_page_size(h), + + return memblock_alloc_try_nid_raw(huge_page_size(h), huge_page_size(h), 0, MEMBLOCK_ALLOC_ACCESSIBLE, nid); - /* - * For pre-HVO to work correctly, pages need to be on - * the list for the node they were actually allocated - * from. That node may be different in the case of - * fallback by memblock_alloc_try_nid_raw. So, - * extract the actual node first. - */ - if (m) - listnode = early_pfn_to_nid(PHYS_PFN(__pa(m))); - } - - if (m) { - m->flags = 0; - m->cma = NULL; - } - } - - if (m) { - /* - * Use the beginning of the huge page to store the - * huge_bootmem_page struct (until gather_bootmem - * puts them into the mem_map). - * - * Put them into a private list first because mem_map - * is not up yet. - */ - INIT_LIST_HEAD(&m->list); - list_add(&m->list, &huge_boot_pages[listnode]); - m->hstate = h; - } - - return m; } -int alloc_bootmem_huge_page(struct hstate *h, int nid) +void *__init arch_alloc_bootmem_huge_page(struct hstate *h, int nid) __attribute__ ((weak, alias("__alloc_bootmem_huge_page"))); -int __alloc_bootmem_huge_page(struct hstate *h, int nid) +void *__init __alloc_bootmem_huge_page(struct hstate *h, int nid) { - struct huge_bootmem_page *m = NULL; /* initialize for clang */ int nr_nodes, node = nid; /* do node specific alloc */ - if (nid != NUMA_NO_NODE) { - m = alloc_bootmem(h, node, true); - if (!m) - return 0; - goto found; - } + if (nid != NUMA_NO_NODE) + return alloc_bootmem(h, node, true); /* allocate from next node when distributing huge pages */ for_each_node_mask_to_alloc(&h->next_nid_to_alloc, nr_nodes, node, - &hugetlb_bootmem_nodes) { - m = alloc_bootmem(h, node, false); - if (!m) - return 0; - goto found; - } + &hugetlb_bootmem_nodes) + return alloc_bootmem(h, node, false); -found: + return NULL; +} + +static bool __init alloc_bootmem_huge_page(struct hstate *h, int nid) +{ + struct huge_bootmem_page *m = arch_alloc_bootmem_huge_page(h, nid); + + if (!m) + return false; + + nid = early_pfn_to_nid(PHYS_PFN(__pa(m))); + /* + * Use the beginning of the huge page to store the huge_bootmem_page + * struct (until gather_bootmem puts them into the mem_map). + * + * Put them into a private list first because mem_map is not up yet. + */ + INIT_LIST_HEAD(&m->list); + list_add(&m->list, &huge_boot_pages[nid]); + m->hstate = h; + if (!hugetlb_early_cma(h)) { + m->cma = NULL; + m->flags = 0; + } /* * Only initialize the head struct page in memmap_init_reserved_pages, @@ -3123,7 +3102,7 @@ found: memblock_reserved_mark_noinit(__pa((void *)m + PAGE_SIZE), huge_page_size(h) - PAGE_SIZE); - return 1; + return true; } /* Initialize [start_page:end_page_number] tail struct pages of a hugepage */ diff --git a/mm/hugetlb_cma.c b/mm/hugetlb_cma.c index 665d910b0f13..ae21b24e45b0 100644 --- a/mm/hugetlb_cma.c +++ b/mm/hugetlb_cma.c @@ -56,14 +56,13 @@ struct folio *hugetlb_cma_alloc_frozen_folio(int order, gfp_t gfp_mask, return folio; } -struct huge_bootmem_page * __init -hugetlb_cma_alloc_bootmem(struct hstate *h, int *nid, bool node_exact) +void * __init hugetlb_cma_alloc_bootmem(struct hstate *h, int nid, bool node_exact) { struct cma *cma; struct huge_bootmem_page *m; - int node = *nid; + int node; - cma = hugetlb_cma[*nid]; + cma = hugetlb_cma[nid]; m = cma_reserve_early(cma, huge_page_size(h)); if (!m) { if (node_exact) @@ -71,13 +70,11 @@ hugetlb_cma_alloc_bootmem(struct hstate *h, int *nid, bool node_exact) for_each_node_mask(node, hugetlb_bootmem_nodes) { cma = hugetlb_cma[node]; - if (!cma || node == *nid) + if (!cma || node == nid) continue; m = cma_reserve_early(cma, huge_page_size(h)); - if (m) { - *nid = node; + if (m) break; - } } } diff --git a/mm/hugetlb_cma.h b/mm/hugetlb_cma.h index c619c394b1ae..3aa483573d17 100644 --- a/mm/hugetlb_cma.h +++ b/mm/hugetlb_cma.h @@ -6,8 +6,7 @@ void hugetlb_cma_free_frozen_folio(struct folio *folio); struct folio *hugetlb_cma_alloc_frozen_folio(int order, gfp_t gfp_mask, int nid, nodemask_t *nodemask); -struct huge_bootmem_page *hugetlb_cma_alloc_bootmem(struct hstate *h, int *nid, - bool node_exact); +void *hugetlb_cma_alloc_bootmem(struct hstate *h, int nid, bool node_exact); bool hugetlb_cma_exclusive_alloc(void); unsigned long hugetlb_cma_total_size(void); void hugetlb_cma_validate_params(void); @@ -23,9 +22,8 @@ static inline struct folio *hugetlb_cma_alloc_frozen_folio(int order, return NULL; } -static inline -struct huge_bootmem_page *hugetlb_cma_alloc_bootmem(struct hstate *h, int *nid, - bool node_exact) +static inline void *hugetlb_cma_alloc_bootmem(struct hstate *h, int nid, + bool node_exact) { return NULL; } diff --git a/mm/internal.h b/mm/internal.h index 09efb9f4d126..3401759924d9 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -23,6 +23,15 @@ #include "vma.h" struct folio_batch; +struct hstate; +struct cma; + +struct huge_bootmem_page { + struct list_head list; + struct hstate *hstate; + unsigned long flags; + struct cma *cma; +}; /* * Maintains state across a page table move. The operation assumes both source |