From e384abeb559d10d6505aec053ede9368d81d4c71 Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Thu, 27 Aug 2026 20:55:57 +0100 Subject: mm/huge_memory: bypass THP tuneables for huge pfnmap mappings The sysfs THP tuneables at /sys/kernel/mm/transparent_huge_pages/ rather confusingly only control the behaviour of THP in some instances. They are not applicable to MADV_COLLAPSE operations, nor to DAX mappings. Long-term, THP is predicated upon compaction being able to obtain large folios to populate THP ranges. However, vm_normal_folio() returns NULL for PFN map mappings, thus their reference count is maintained by the driver, not core mm. As a consequence, the folios are not subject to reclaim nor compaction, so are not truly part of the THP mechanism at all. However, since commit 5dd40721f147 ("mm: allow THP orders for PFNMAPs") introduced the ability to establish huge PFN maps, they have been subject to THP tuneables. This is incorrect - if a huge PFN map is available (defined by vma->vm_ops->huge_fault being non-NULL for a VMA_PFNMAP_BIT VMA), then it should be mapped huge upon fault-in. Correct this by explicitly checking for this while ensuring that smaps continues to accurately report THPeligible statistics. While here, abstract the entire file-backed THP check in vma_can_map_huge_file(), with sensible separation of logic into helper functions. Note that drm_gem_shmem_mmap() and panthor_gem_mmap() establish huge PFN maps of shmem folios, however they are marked unevictable in drm_gem_get_pages(), and in any case would fail the reference check in __remove_mapping() even if they weren't. Failing to map huge PFN maps has resulted in significant real-world performance degradation, see links for details. [ziy@nvidia.com: rename some functions] Link: https://lore.kernel.org/DL1HIHWYJ7TB.1CY76SJS0V03L@nvidia.com Link: https://lore.kernel.org/20260827-hugepfn-allowable-orders-v1-1-94819c8807c8@kernel.org Fixes: 5dd40721f147 ("mm: allow THP orders for PFNMAPs") Signed-off-by: Lorenzo Stoakes (ARM) Signed-off-by: Zi Yan Reported-by: Cedric Le Goater Closes: https://lore.kernel.org/linux-mm/20260805055544.1568534-1-clg@redhat.com/ Reported-by: Saravanan D Closes: https://lore.kernel.org/linux-mm/20260821070520.25759-1-saravanand@crusoe.ai/ Reviewed-by: Zi Yan Tested-by: Saravanan D Tested-by: Lance Yang Reviewed-by: SJ Park Reviewed-by: Baolin Wang Cc: Barry Song Cc: David Hildenbrand Cc: Dev Jain Cc: Jason Gunthorpe Cc: Liam R. Howlett Cc: Peter Xu Cc: Ryan Roberts Cc: Signed-off-by: Andrew Morton --- mm/huge_memory.c | 86 +++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 64 insertions(+), 22 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index afbb5974bd22..1e5d68acf62a 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -92,7 +92,7 @@ unsigned long huge_anon_orders_madvise __read_mostly; unsigned long huge_anon_orders_inherit __read_mostly; static bool anon_orders_configured __initdata; -static inline bool file_thp_enabled(struct vm_area_struct *vma) +static inline bool file_thp_enabled(const struct vm_area_struct *vma) { struct inode *inode; @@ -118,6 +118,67 @@ static bool vma_is_special_huge(const struct vm_area_struct *vma) return vma_test_any(vma, VMA_PFNMAP_BIT, VMA_MIXEDMAP_BIT); } +static bool vma_file_bypass_thp_tuneables(const struct vm_area_struct *vma, + enum tva_type type) +{ + const bool has_huge_fault = vma->vm_ops->huge_fault; + + /* MADV_COLLAPSE ignores tuneables. */ + if (type == TVA_FORCED_COLLAPSE) + return true; + /* Huge PFN mappings are uncompactable so the policy doesn't apply. */ + if (vma_test(vma, VMA_PFNMAP_BIT) && has_huge_fault) + return true; + return false; +} + +static bool vma_file_allow_thp_tuneables(vm_flags_t vm_flags) +{ + /* THP=always? */ + if (hugepage_global_always()) + return true; + /* THP=madvise and marked MADV_HUGEPAGE? */ + if (hugepage_global_enabled() && (vm_flags & VM_HUGEPAGE)) + return true; + return false; +} + +static bool vma_file_check_thp_tuneables(const struct vm_area_struct *vma, + vm_flags_t vm_flags, enum tva_type type) +{ + return vma_file_bypass_thp_tuneables(vma, type) || + vma_file_allow_thp_tuneables(vm_flags); +} + +static bool vma_can_map_huge_file(const struct vm_area_struct *vma, + vm_flags_t vm_flags, enum tva_type type) +{ + const bool has_huge_fault = vma->vm_ops->huge_fault; + + /* + * Enforce THP collapse requirements as necessary. Anonymous vmas + * were already handled in thp_vma_allowable_orders(). + */ + if (!vma_file_check_thp_tuneables(vma, vm_flags, type)) + return false; + + switch (type) { + case TVA_PAGEFAULT: + /* + * Trust that ->huge_fault() handlers know what they are doing + * in fault path. + */ + return has_huge_fault; + case TVA_SMAPS: + if (has_huge_fault) + return true; + fallthrough; + default: + /* Only regular file is valid in collapse path. */ + return file_thp_enabled(vma); + } +} + unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma, vm_flags_t vm_flags, enum tva_type type, @@ -190,27 +251,8 @@ unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma, vma, vma_start_pgoff(vma), 0, forced_collapse); - if (!vma_is_anonymous(vma)) { - /* - * Enforce THP collapse requirements as necessary. Anonymous vmas - * were already handled in thp_vma_allowable_orders(). - */ - if (!forced_collapse && - (!hugepage_global_enabled() || (!(vm_flags & VM_HUGEPAGE) && - !hugepage_global_always()))) - return 0; - - /* - * Trust that ->huge_fault() handlers know what they are doing - * in fault path. - */ - if (((in_pf || smaps)) && vma->vm_ops->huge_fault) - return orders; - /* Only regular file is valid in collapse path */ - if (((!in_pf || smaps)) && file_thp_enabled(vma)) - return orders; - return 0; - } + if (!vma_is_anonymous(vma)) + return vma_can_map_huge_file(vma, vm_flags, type) ? orders : 0; if (vma_is_temporary_stack(vma)) return 0; -- cgit v1.2.3 From 397432cab17bccb600fd6c16ed593f1149042268 Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Fri, 28 Aug 2026 12:20:37 +0100 Subject: mm/mremap: account mm->locked_vm correctly for MREMAP_DONTUNMAP When a VMA is mremap()'d with MREMAP_DONTUNMAP set, that results in the VMA being copied, but the source VMA not being unmapped. If the VMA is mlock()'d this is a legal operation, though the source VMA has its VMA_LOCKED_BIT cleared. However this is done in dontunmap_complete(), after mm->locked_vm was incremented via vrm_stat_account(), resulting in double-counting. Worse, this is not even corrected when source VMA is unmapped, due to the VMA_LOCKED_BIT flag having been cleared. This all works fine in the usual mremap() case (without MREMAP_DONTUNMAP), as the source VMA is unmapped with VMA_LOCKED_BIT intact, at which time mm->locked_vm is decremented accordingly. Resolve the issue by invoking vrm_stat_account() only after dontunmap_complete() has run. Note that MREMAP_DONTUNMAP requires old_len == new_len, so no need to account for a delta in size in this case. The bug was introduced by commit b714ccb02a76 ("mm/mremap: complete refactor of move_vma()") which incorrectly reordered the accounting and the clearing of the VMA_LOCKED_BIT flag. Link: https://lore.kernel.org/20260828-mremap-fix-locked-vm-v1-1-c80be7505d1e@kernel.org Fixes: b714ccb02a76 ("mm/mremap: complete refactor of move_vma()") Signed-off-by: Lorenzo Stoakes (ARM) Reported-by: sashiko-bot Closes: https://sashiko.dev/#/patchset/20260825-fix-mremap-dontunmap-pgoff-v1-1-39a40b2c98b3@kernel.org Reported-by: Kunwu Chan Closes: https://lore.kernel.org/all/20260828094823.594279-1-kunwu.chan@linux.dev/ Acked-by: Vlastimil Babka (SUSE) Tested-by: Kunwu Chan Reviewed-by: Kunwu Chan Cc: Jann Horn Cc: Liam R. Howlett Cc: Pedro Falcato Cc: Signed-off-by: Andrew Morton --- mm/mremap.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/mm/mremap.c b/mm/mremap.c index 2b4b523a86b8..7c368440fafe 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -1355,12 +1355,11 @@ static void dontunmap_complete(struct vma_remap_struct *vrm, if (vma_is_anonymous(vma) && !vma->vm_file) vma_set_pgoff(vma, pgoff_unfaulted); } - - /* Because we won't unmap we don't need to touch locked_vm. */ } static unsigned long move_vma(struct vma_remap_struct *vrm) { + const bool is_dontunmap = vrm->flags & MREMAP_DONTUNMAP; struct mm_struct *mm = current->mm; struct vm_area_struct *new_vma; unsigned long hiwater_vm; @@ -1401,10 +1400,10 @@ static unsigned long move_vma(struct vma_remap_struct *vrm) */ hiwater_vm = mm->hiwater_vm; - vrm_stat_account(vrm, vrm->new_len); - if (unlikely(!err && (vrm->flags & MREMAP_DONTUNMAP))) + if (unlikely(is_dontunmap && !err)) dontunmap_complete(vrm, new_vma); - else + vrm_stat_account(vrm, vrm->new_len); + if (!is_dontunmap || err) unmap_source_vma(vrm); mm->hiwater_vm = hiwater_vm; -- cgit v1.2.3 From e1d56f046507befa20a5e0837d8075abdf5848fd Mon Sep 17 00:00:00 2001 From: Coiby Xu Date: Fri, 28 Aug 2026 16:41:06 +0800 Subject: mailmap: map Coiby Xu's address Point to my gmail address as I've left Red Hat. Link: https://lore.kernel.org/20260828084106.1494733-1-coiby.xu@gmail.com Signed-off-by: Coiby Xu Signed-off-by: Andrew Morton --- .mailmap | 1 + 1 file changed, 1 insertion(+) diff --git a/.mailmap b/.mailmap index 37aad399e4e0..b355f1c85958 100644 --- a/.mailmap +++ b/.mailmap @@ -222,6 +222,7 @@ Chuck Lever Chuck Lever Chuck Lever Claudiu Beznea +Coiby Xu Colin Ian King Corey Minyard Damian Hobson-Garcia -- cgit v1.2.3 From 12e9ac7bc5b254048f886bf421e3a15491106c1f Mon Sep 17 00:00:00 2001 From: Nhat Pham Date: Fri, 28 Aug 2026 12:14:33 -0700 Subject: mm, swap: fix SWAP_USAGE_OFFLIST_BIT collision with real usage count SWAP_USAGE_OFFLIST_BIT is embedded in the si->inuse_pages usage counter, and is meant to sit above any value that counter can reach. However, it is defined from BITS_PER_TYPE(atomic_t), so it is bit 30. On a system with 4 KiB pages the flag collides with the usage count once that count reaches 4 TiB. swap_usage_in_pages() masks bit 30 out, so whenever the real count has that bit set, every caller of it reads 4 TiB low: * /proc/swaps understates Used by 4 TiB. * A raw count of exactly 2^30 masks to zero, so try_to_unuse() takes its "if (!swap_usage_in_pages(si)) goto success;" early exit and swapoff tears the device down while pages are still swapped out. Nothing in the rest of swapoff aborts the teardown, so those pages are lost. Independently of swapoff, the collision also corrupts the counter and the plist. On a device in normal use, a free that leaves bit 30 set in the count makes swap_usage_sub() see the flag where there is only count, and call add_to_avail_list(). It clears the bit with fetch_and(~SWAP_USAGE_OFFLIST_BIT), leaving the stored count 4 TiB below the real one, and calls plist_add() on a device that is already listed, tripping the WARN_ON(!plist_node_empty(node)) in plist_add() and linking the node a second time. Change the definition of SWAP_USAGE_OFFLIST_BIT to be based on atomic_long_t instead. Note that the usage counter field itself is of this same type, so it is still a valid bit. Link: https://lore.kernel.org/20260828191433.3304458-1-nphamcs@gmail.com Fixes: b228386cf237 ("mm, swap: clean up plist removal and adding") Signed-off-by: Nhat Pham Reported-by: Sashiko Closes: https://sashiko.dev/#/patchset/20260825153238.2695446-1-nphamcs%40gmail.com Suggested-by: Andrew Morton Reviewed-by: Andrew Morton Acked-by: Kairui Song Cc: Baoquan He Cc: Barry Song Cc: Chris Li Cc: Gregory Price Cc: Johannes Weiner Cc: Joshua Hahn Cc: Kemeng Shi Cc: Shakeel Butt Cc: Youngjun Park Cc: Signed-off-by: Andrew Morton --- mm/swapfile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/swapfile.c b/mm/swapfile.c index 53bf01d5f7f1..601979b97f95 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -156,7 +156,7 @@ static struct swap_info_struct *swap_entry_to_info(swp_entry_t entry) * This bit will be set if the device is not on the plist and not * usable, will be cleared if the device is on the plist. */ -#define SWAP_USAGE_OFFLIST_BIT (1UL << (BITS_PER_TYPE(atomic_t) - 2)) +#define SWAP_USAGE_OFFLIST_BIT (1UL << (BITS_PER_TYPE(atomic_long_t) - 2)) #define SWAP_USAGE_COUNTER_MASK (~SWAP_USAGE_OFFLIST_BIT) static long swap_usage_in_pages(struct swap_info_struct *si) { -- cgit v1.2.3 From 6e673d0879ef78c395cfe0d3ba316690a60055d8 Mon Sep 17 00:00:00 2001 From: Shakeel Butt Date: Fri, 28 Aug 2026 19:32:51 -0700 Subject: memcg: avoid charging the root memcg from obj_cgroup_charge_pages() obj_cgroup_charge_pages() resolves the objcg to its memcg and calls try_charge_memcg(), which does not short circuit the root memcg. That memcg can be the root memcg: obj_cgroup_is_root() reflects the memcg the objcg was created for and is never updated, while memcg_reparent_objcgs() does redirect objcg->memcg to the parent on rmdir. An objcg of a dying child of root therefore passes every obj_cgroup_is_root() filter but resolves to the root memcg. Folios keep the objcg they were charged with, so this is easy to reach through zswap: allocate anon memory in a cgroup, move the task out, remove the cgroup, then write to the root cgroup's memory.reclaim. The reclaimed folios are charged through the reparented objcg and end up in refill_stock() with the root memcg: WARNING: mm/memcontrol.c:2198 at refill_stock+0x644/0x940 refill_stock+0x644/0x940 try_charge_memcg+0x12d6/0x1570 __obj_cgroup_charge+0x35/0xf0 obj_cgroup_charge+0x1de/0x210 obj_cgroup_charge_zswap+0x83/0x270 zswap_store+0x1620/0x2000 swap_writeout+0x94c/0x14c0 shrink_folio_list+0x3388/0x52b0 [...] try_to_free_mem_cgroup_pages+0x30d/0x830 user_proactive_reclaim+0x504/0x840 memory_reclaim+0x1f/0x30 Beyond the warning, the charge is asymmetric: obj_cgroup_uncharge_pages() skips refill_stock() for the root memcg, so the root's page counter grows and is never uncharged. It is not user visible, since memory.current is not exposed on the root, but it is a leak. Use try_charge(), which returns early for the root memcg, restoring the symmetry with obj_cgroup_uncharge_pages(). The above sequence was scripted into a standalone reproducer (zswap on, swap on a virtio disk, 512MB of anon memory faulted in inside a child of the root cgroup, the task then migrated to the root cgroup, the child removed, followed by "echo 600M swappiness=max > memory.reclaim" on the root) and run in a CONFIG_DEBUG_VM=y VM. It reproduces the splat on the first zswap store of a reparented folio, with the same call chain as the report. With this patch applied the splat is gone while the zswap store count over the run is unchanged, so the same path is still exercised. cgroup selftests test_zswap, test_kmem and test_memcontrol show no new failures. Link: https://lore.kernel.org/20260829023251.474083-1-shakeel.butt@linux.dev Fixes: 20d6c1725228 ("memcg: avoid refill_stock for root memcg") Signed-off-by: Shakeel Butt Reported-by: Farhad Alemi Closes: https://lore.kernel.org/all/CA+0ovCgWzUMK+nNbbtH7eV65Ca=fDN4Ozu7iASgryjvv8Tk8zQ@mail.gmail.com/ Reviewed-by: Muchun Song Reviewed-by: Johannes Weiner Cc: Michal Hocko Cc: Roman Gushchin Cc: Signed-off-by: Andrew Morton --- mm/memcontrol.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 1271d390b617..856a7d07586c 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -3158,7 +3158,7 @@ static int obj_cgroup_charge_pages(struct obj_cgroup *objcg, gfp_t gfp, memcg = get_mem_cgroup_from_objcg(objcg); - ret = try_charge_memcg(memcg, gfp, nr_pages); + ret = try_charge(memcg, gfp, nr_pages); if (ret) goto out; -- cgit v1.2.3 From e1d469a8d6c63032a5aed3679a13a086b61bd5cc Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 29 Aug 2026 12:28:23 +0100 Subject: mailmap: update entry for Christopher Obbard I have changed employer; update my mailmap entry to point at my new email address. Link: https://lore.kernel.org/20260829-update-mail-oss-qualcomm-v2-1-1670c515f225@oss.qualcomm.com Signed-off-by: Christopher Obbard Signed-off-by: Andrew Morton --- .mailmap | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.mailmap b/.mailmap index b355f1c85958..90ff4831f592 100644 --- a/.mailmap +++ b/.mailmap @@ -211,7 +211,8 @@ Christophe Leroy Christophe Leroy Christophe Leroy Christophe Ricard -Christopher Obbard +Christopher Obbard +Christopher Obbard Christoph Hellwig Christoph Manszewski Christoph Paasch -- cgit v1.2.3 From 848d2ce2fce15fbdc083fbf9691bfa72911033c4 Mon Sep 17 00:00:00 2001 From: Wenjie Qi Date: Sun, 30 Aug 2026 01:36:12 +0800 Subject: mm: filemap: retain mapped dropbehind folios Fault-around can map ready dropbehind folios without going through the normal page-cache lookup that clears dropbehind. A mapping represents a competing cached user, so retain the folio instead of forcibly unmapping it when writeback completes. For a mapped folio, folio_unmap_invalidate() can call unmap_mapping_folio(), which takes i_mmap_rwsem and may sleep. Retaining mapped folios avoids this path when folio_end_dropbehind() runs in non-preemptible task context. Tal was able to trigger a sleeping-in-atomic warning due to this [1]. Unmapped dropbehind folios continue through the existing invalidation path. Link: https://lore.kernel.org/4aba05e1a2c3b61cb337d373eb9b7a8db4ddd822.1788024049.git.qiwenjie@xiaomi.com Link: https://lore.kernel.org/076bb01b-6fcf-4691-be8c-0e8507c9fe64@columbia.edu [1] Fixes: fb7d3bc41493 ("mm/filemap: drop streaming/uncached pages when writeback completes") Signed-off-by: Wenjie Qi Reviewed-by: Matthew Wilcox (Oracle) Reviewed-by: Tal Zussman Tested-by: Tal Zussman Cc: Barry Song Cc: Jan Kara Cc: Jens Axboe Cc: Trond Myklebust Cc: Signed-off-by: Andrew Morton --- mm/filemap.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/filemap.c b/mm/filemap.c index 6afec636881f..00fd89cf6f55 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -1616,7 +1616,7 @@ static void filemap_end_dropbehind(struct folio *folio) return; if (!folio_test_clear_dropbehind(folio)) return; - if (mapping) + if (mapping && !folio_mapped(folio)) folio_unmap_invalidate(mapping, folio, 0); } -- cgit v1.2.3 From 641aade99f06df0037e52b5c81645461b7132947 Mon Sep 17 00:00:00 2001 From: Qi Zheng Date: Mon, 17 Aug 2026 17:03:25 +0800 Subject: fs: fix missed removal of super_fs_objects_eligible() Commit 0ef8faff490be ("fs: push nr_cached_objects memcg gating into individual filesystems") was meant to drop the blanket memcg gate in fs/super.c and let each ->nr_cached_objects() implementation decide for itself whether it is meaningful in per-memcg reclaim. However, when that patch was applied the removal of super_fs_objects_eligible() and its two call sites in super_cache_scan() / super_cache_count() was lost, so the helper is still gating every ->nr_cached_objects() hook and 0ef8faff490be is effectively a no-op. Consequences of the leftover gate: - XFS's inode-reclaim hook, which is intentionally driven from per-memcg contexts to free memcg-charged slab, is still short-circuited in fs/super.c exactly the regression from commit 0baad6f9b997 ("fs/super: skip non-memcg-aware nr_cached_objects in memcg slab shrink") that 0ef8faff490be was written to undo. Memcg-charged XFS inode slab therefore keeps piling up under per-memcg pressure until global reclaim kicks in. - Any future ->nr_cached_objects()/->free_cached_objects() that grows memcg awareness is likewise blocked before it can run, so filesystems cannot opt in to per-memcg reclaim on their own defeating the whole point of pushing the gating decision down into the callbacks. Drop the leftover helper and its call sites so the intent of 0ef8faff490be actually takes effect. Link: https://lore.kernel.org/cover.1786955972.git.zhengqi.arch@bytedance.com Link: https://lore.kernel.org/3b038d373c70ebac7cdabfb0035bb91d1d6e6cfe.1786955972.git.zhengqi.arch@bytedance.com Link: https://lore.kernel.org/all/20260715103516.2410175-1-usama.arif@linux.dev/ [0] Fixes: 0ef8faff490b ("fs: push nr_cached_objects memcg gating into individual filesystems") Signed-off-by: Qi Zheng Acked-by: Usama Arif Cc: Baolin Wang Cc: Christian Brauner Cc: David Hildenbrand Cc: Hugh Dickins Cc: Johannes Weiner Cc: Michal Hocko Cc: Muchun Song Cc: Roman Gushchin Cc: Shakeel Butt Cc: Signed-off-by: Andrew Morton --- fs/super.c | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/fs/super.c b/fs/super.c index 05e443173038..3ecce24328f6 100644 --- a/fs/super.c +++ b/fs/super.c @@ -171,19 +171,6 @@ static void super_wake(struct super_block *sb, unsigned int flag) wake_up_var(&sb->s_flags); } -/* - * The s_op->nr_cached_objects hooks (used for example by btrfs and xfs) - * operate on filesystem-global state and ignore sc->memcg. Driving them - * from per-memcg shrink_slab_memcg() invocations only burns CPU walking - * per-cpu counters and queueing duplicate work: the actual reclaim happens on - * the global path (kswapd or root direct reclaim) regardless. Restrict them - * to that path. - */ -static inline bool super_fs_objects_eligible(struct shrink_control *sc) -{ - return !sc->memcg || mem_cgroup_is_root(sc->memcg); -} - /* * One thing we have to be careful of with a per-sb shrinker is that we don't * drop the last active reference to the superblock from within the shrinker. @@ -213,7 +200,7 @@ static unsigned long super_cache_scan(struct shrinker *shrink, if (!super_trylock_shared(sb)) return SHRINK_STOP; - if (sb->s_op->nr_cached_objects && super_fs_objects_eligible(sc)) + if (sb->s_op->nr_cached_objects) fs_objects = sb->s_op->nr_cached_objects(sb, sc); inodes = list_lru_shrink_count(&sb->s_inode_lru, sc); @@ -274,8 +261,7 @@ static unsigned long super_cache_count(struct shrinker *shrink, return 0; smp_rmb(); - if (sb->s_op && sb->s_op->nr_cached_objects && - super_fs_objects_eligible(sc)) + if (sb->s_op && sb->s_op->nr_cached_objects) total_objects = sb->s_op->nr_cached_objects(sb, sc); total_objects += list_lru_shrink_count(&sb->s_dentry_lru, sc); -- cgit v1.2.3 From 8e2b8614039853e68d5338e37821e8bcee9fc05f Mon Sep 17 00:00:00 2001 From: Seunguk Shin Date: Mon, 3 Aug 2026 13:34:55 +0100 Subject: fs/dax: check zero or empty entry before converting xarray entry Calling dax_to_folio() with empty entry causes kernel panic below when booting a VM with DAX enabled storage. This patch checks empty entry before calling dax_to_folio() on dax_associate_entry(), dax_disassociate_entry(), and dax_busy_page(). Commit 98c183a4fccf ("fs/dax: don't disassociate zero page entries") added guards in the associate and disassociate paths, but the guards still come after dax_to_folio(), and dax_busy_page() still has the same problem. [ 0.737679] EXT4-fs (pmem0p1): mounted filesystem 79676804-7c8b-491a-b2a6-9bae3c72af70 ro with ordered data mode. Quota mode: disabled. [ 0.737891] VFS: Mounted root (ext4 filesystem) readonly on device 259:1. [ 0.739119] devtmpfs: mounted [ 0.739476] Freeing unused kernel memory: 1920K [ 0.740156] Run /sbin/init as init process [ 0.740229] with arguments: [ 0.740286] /sbin/init [ 0.740321] with environment: [ 0.740369] HOME=/ [ 0.740400] TERM=linux [ 0.743162] Unable to handle kernel paging request at virtual address fffffdffbf000008 [ 0.743285] Mem abort info: [ 0.743316] ESR = 0x0000000096000006 [ 0.743371] EC = 0x25: DABT (current EL), IL = 32 bits [ 0.743444] SET = 0, FnV = 0 [ 0.743489] EA = 0, S1PTW = 0 [ 0.743545] FSC = 0x06: level 2 translation fault [ 0.743610] Data abort info: [ 0.743656] ISV = 0, ISS = 0x00000006, ISS2 = 0x00000000 [ 0.743720] CM = 0, WnR = 0, TnD = 0, TagAccess = 0 [ 0.743785] GCS = 0, Overlay = 0, DirtyBit = 0, Xs = 0 [ 0.743848] swapper pgtable: 4k pages, 48-bit VAs, pgdp=00000000b9d17000 [ 0.743931] [fffffdffbf000008] pgd=10000000bfa3d403, p4d=10000000bfa3d403, pud=1000000040bfe403, pmd=0000000000000000 [ 0.744070] Internal error: Oops: 0000000096000006 [#1] SMP [ 0.748888] CPU: 0 UID: 0 PID: 1 Comm: init Not tainted 6.18.4 #1 NONE [ 0.749421] pstate: 004000c5 (nzcv daIF +PAN -UAO -TCO -DIT -SSBS BTYPE=--) [ 0.749969] pc : dax_disassociate_entry.constprop.0+0x20/0x50 [ 0.750444] lr : dax_insert_entry+0xcc/0x408 [ 0.750802] sp : ffff80008000b9e0 [ 0.751083] x29: ffff80008000b9e0 x28: 0000000000000000 x27: 0000000000000000 [ 0.751682] x26: 0000000001963d01 x25: ffff0000004f7d90 x24: 0000000000000000 [ 0.752264] x23: 0000000000000000 x22: ffff80008000bcc8 x21: 0000000000000011 [ 0.752836] x20: ffff80008000ba90 x19: 0000000001963d01 x18: 0000000000000000 [ 0.753407] x17: 0000000000000000 x16: 0000000000000000 x15: 0000000000000000 [ 0.753970] x14: ffffbf3154b9ae70 x13: 0000000000000000 x12: ffffbf3154b9ae70 [ 0.754548] x11: ffffffffffffffff x10: 0000000000000000 x9 : 0000000000000000 [ 0.755122] x8 : 000000000000000d x7 : 000000000000001f x6 : 0000000000000000 [ 0.755707] x5 : 0000000000000000 x4 : 0000000000000000 x3 : fffffdffc0000000 [ 0.756287] x2 : 0000000000000008 x1 : 0000000040000000 x0 : fffffdffbf000000 [ 0.756871] Call trace: [ 0.757107] dax_disassociate_entry.constprop.0+0x20/0x50 (P) [ 0.757592] dax_iomap_pte_fault+0x4fc/0x808 [ 0.757951] dax_iomap_fault+0x28/0x30 [ 0.758258] ext4_dax_huge_fault+0x80/0x2dc [ 0.758594] ext4_dax_fault+0x10/0x3c [ 0.758892] __do_fault+0x38/0x12c [ 0.759175] __handle_mm_fault+0x530/0xcf0 [ 0.759518] handle_mm_fault+0xe4/0x230 [ 0.759833] do_page_fault+0x17c/0x4dc [ 0.760144] do_translation_fault+0x30/0x38 [ 0.760483] do_mem_abort+0x40/0x8c [ 0.760771] el0_ia+0x4c/0x170 [ 0.761032] el0t_64_sync_handler+0xd8/0xdc [ 0.761371] el0t_64_sync+0x168/0x16c [ 0.761677] Code: f9453021 f2dfbfe3 cb813080 8b001860 (f9400401) [ 0.762168] ---[ end trace 0000000000000000 ]--- [ 0.762550] note: init[1] exited with irqs disabled [ 0.762631] Kernel panic - not syncing: Attempted to kill init! exitcode=0x0000000b Link: https://lore.kernel.org/m2y0enxtzk.fsf@arm.com Fixes: 38607c62b34b ("fs/dax: properly refcount fs dax pages") Signed-off-by: Seunguk Shin Reviewed-by: Jan Kara Reviewed-by: Alistair Popple Reported-by: Kiara Grouwstra Cc: Al Viro Cc: Christian Brauner Cc: Matthew Wilcox (Oracle) Cc: Signed-off-by: Andrew Morton --- fs/dax.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/fs/dax.c b/fs/dax.c index 6ba50142eeb2..1fbba0d21c13 100644 --- a/fs/dax.c +++ b/fs/dax.c @@ -480,11 +480,12 @@ static void dax_associate_entry(void *entry, struct address_space *mapping, unsigned long address, bool shared) { unsigned long size = dax_entry_size(entry), index; - struct folio *folio = dax_to_folio(entry); + struct folio *folio; if (dax_is_zero_entry(entry) || dax_is_empty_entry(entry)) return; + folio = dax_to_folio(entry); index = linear_page_index(vma, address & ~(size - 1)); if (shared && (folio->mapping || dax_folio_is_shared(folio))) { if (folio->mapping) @@ -505,21 +506,23 @@ static void dax_associate_entry(void *entry, struct address_space *mapping, static void dax_disassociate_entry(void *entry, struct address_space *mapping, bool trunc) { - struct folio *folio = dax_to_folio(entry); + struct folio *folio; if (dax_is_zero_entry(entry) || dax_is_empty_entry(entry)) return; + folio = dax_to_folio(entry); dax_folio_put(folio); } static struct page *dax_busy_page(void *entry) { - struct folio *folio = dax_to_folio(entry); + struct folio *folio; if (dax_is_zero_entry(entry) || dax_is_empty_entry(entry)) return NULL; + folio = dax_to_folio(entry); if (folio_ref_count(folio) - folio_mapcount(folio)) return &folio->page; else -- cgit v1.2.3 From 0791a234b35d4b72187c497e05817f2c9019c3ab Mon Sep 17 00:00:00 2001 From: Andrew Morton Date: Tue, 1 Sep 2026 13:09:29 -0700 Subject: remove old lib/alloc_tag.c This was moved into mm/, but the original lib/ file somehow remained. Remove it. Reported-by: Suren Baghdasaryan Cc: Lorenzo Stoakes Signed-off-by: Andrew Morton --- lib/alloc_tag.c | 1029 ------------------------------------------------------- 1 file changed, 1029 deletions(-) delete mode 100644 lib/alloc_tag.c diff --git a/lib/alloc_tag.c b/lib/alloc_tag.c deleted file mode 100644 index e5b218176c5a..000000000000 --- a/lib/alloc_tag.c +++ /dev/null @@ -1,1029 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define ALLOCINFO_FILE_NAME "allocinfo" -#define MODULE_ALLOC_TAG_VMAP_SIZE (100000UL * sizeof(struct alloc_tag)) -#define SECTION_START(NAME) (CODETAG_SECTION_START_PREFIX NAME) -#define SECTION_STOP(NAME) (CODETAG_SECTION_STOP_PREFIX NAME) - -#ifdef CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT -static bool mem_profiling_support = true; -#else -static bool mem_profiling_support; -#endif - -/* - * Memory allocation profiling is permanently disabled and cannot be enabled. - * Must be called after setup_early_mem_profiling(). - */ -bool mem_alloc_profiling_permanently_disabled(void) -{ - return !mem_profiling_support; -} - -static struct codetag_type *alloc_tag_cttype; - -#ifdef CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU -DEFINE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag); -EXPORT_SYMBOL(_shared_alloc_tag); -#endif - -DEFINE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT, - mem_alloc_profiling_key); -EXPORT_SYMBOL(mem_alloc_profiling_key); - -DEFINE_STATIC_KEY_FALSE(mem_profiling_compressed); - -struct alloc_tag_kernel_section kernel_tags = { NULL, 0 }; -unsigned long alloc_tag_ref_mask; -int alloc_tag_ref_offs; - -struct allocinfo_private { - struct codetag_iterator iter; - struct codetag_iterator reported_iter; - bool print_header; -}; - -static void *allocinfo_start(struct seq_file *m, loff_t *pos) -{ - struct allocinfo_private *priv; - loff_t node = *pos; - - priv = (struct allocinfo_private *)m->private; - codetag_lock_module_list(alloc_tag_cttype); - if (node == 0) { - priv->print_header = true; - priv->iter = codetag_get_ct_iter(alloc_tag_cttype); - } else { - priv->iter = priv->reported_iter; - } - codetag_next_ct(&priv->iter); - return priv->iter.ct ? priv : NULL; -} - -static void *allocinfo_next(struct seq_file *m, void *arg, loff_t *pos) -{ - struct allocinfo_private *priv = (struct allocinfo_private *)arg; - struct codetag *ct; - - priv->reported_iter = priv->iter; - ct = codetag_next_ct(&priv->iter); - (*pos)++; - if (!ct) - return NULL; - - return priv; -} - -static void allocinfo_stop(struct seq_file *m, void *arg) -{ - codetag_unlock_module_list(alloc_tag_cttype); -} - -static void print_allocinfo_header(struct seq_buf *buf) -{ - /* Output format version, so we can change it. */ - seq_buf_printf(buf, "allocinfo - version: 2.0\n"); - seq_buf_printf(buf, "# \n"); -} - -static void alloc_tag_to_text(struct seq_buf *out, struct codetag *ct) -{ - struct alloc_tag *tag = ct_to_alloc_tag(ct); - struct alloc_tag_counters counter = alloc_tag_read(tag); - s64 bytes = counter.bytes; - - seq_buf_printf(out, "%12lli %8llu ", bytes, counter.calls); - codetag_to_text(out, ct); - if (unlikely(alloc_tag_is_inaccurate(tag))) - seq_buf_printf(out, " accurate:no"); - seq_buf_putc(out, ' '); - seq_buf_putc(out, '\n'); -} - -static int allocinfo_show(struct seq_file *m, void *arg) -{ - struct allocinfo_private *priv = (struct allocinfo_private *)arg; - char *bufp; - size_t n = seq_get_buf(m, &bufp); - struct seq_buf buf; - - seq_buf_init(&buf, bufp, n); - if (priv->print_header) { - print_allocinfo_header(&buf); - priv->print_header = false; - } - alloc_tag_to_text(&buf, priv->iter.ct); - seq_commit(m, seq_buf_used(&buf)); - return 0; -} - -static const struct seq_operations allocinfo_seq_op = { - .start = allocinfo_start, - .next = allocinfo_next, - .stop = allocinfo_stop, - .show = allocinfo_show, -}; - -size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sleep) -{ - struct codetag_iterator iter; - struct codetag *ct; - struct codetag_bytes n; - unsigned int i, nr = 0; - - if (IS_ERR_OR_NULL(alloc_tag_cttype)) - return 0; - - if (can_sleep) - codetag_lock_module_list(alloc_tag_cttype); - else if (!codetag_trylock_module_list(alloc_tag_cttype)) - return 0; - - iter = codetag_get_ct_iter(alloc_tag_cttype); - while ((ct = codetag_next_ct(&iter))) { - struct alloc_tag_counters counter = alloc_tag_read(ct_to_alloc_tag(ct)); - - n.ct = ct; - n.bytes = counter.bytes; - - for (i = 0; i < nr; i++) - if (n.bytes > tags[i].bytes) - break; - - if (i < count) { - nr -= nr == count; - memmove(&tags[i + 1], - &tags[i], - sizeof(tags[0]) * (nr - i)); - nr++; - tags[i] = n; - } - } - - codetag_unlock_module_list(alloc_tag_cttype); - - return nr; -} - -void pgalloc_tag_split(struct folio *folio, int old_order, int new_order) -{ - int i; - struct alloc_tag *tag; - unsigned int nr_pages = 1 << new_order; - - if (!mem_alloc_profiling_enabled()) - return; - - tag = __pgalloc_tag_get(&folio->page); - if (!tag) - return; - - for (i = nr_pages; i < (1 << old_order); i += nr_pages) { - union pgtag_ref_handle handle; - union codetag_ref ref; - - if (get_page_tag_ref(folio_page(folio, i), &ref, &handle)) { - /* Set new reference to point to the original tag */ - alloc_tag_ref_set(&ref, tag); - update_page_tag_ref(handle, &ref); - put_page_tag_ref(handle); - } - } -} - -void pgalloc_tag_swap(struct folio *new, struct folio *old) -{ - union pgtag_ref_handle handle_old, handle_new; - union codetag_ref ref_old, ref_new; - struct alloc_tag *tag_old, *tag_new; - - if (!mem_alloc_profiling_enabled()) - return; - - tag_old = __pgalloc_tag_get(&old->page); - if (!tag_old) - return; - tag_new = __pgalloc_tag_get(&new->page); - if (!tag_new) - return; - - if (!get_page_tag_ref(&old->page, &ref_old, &handle_old)) - return; - if (!get_page_tag_ref(&new->page, &ref_new, &handle_new)) { - put_page_tag_ref(handle_old); - return; - } - - /* - * Clear tag references to avoid debug warning when using - * __alloc_tag_ref_set() with non-empty reference. - */ - set_codetag_empty(&ref_old); - set_codetag_empty(&ref_new); - - /* swap tags */ - __alloc_tag_ref_set(&ref_old, tag_new); - update_page_tag_ref(handle_old, &ref_old); - __alloc_tag_ref_set(&ref_new, tag_old); - update_page_tag_ref(handle_new, &ref_new); - - put_page_tag_ref(handle_old); - put_page_tag_ref(handle_new); -} - -static void shutdown_mem_profiling(bool remove_file) -{ - if (mem_alloc_profiling_enabled()) - static_branch_disable(&mem_alloc_profiling_key); - - if (!mem_profiling_support) - return; - - if (remove_file) - remove_proc_entry(ALLOCINFO_FILE_NAME, NULL); - mem_profiling_support = false; -} - -void __init alloc_tag_sec_init(void) -{ - struct alloc_tag *last_codetag; - - if (!mem_profiling_support) - return; - - if (!static_key_enabled(&mem_profiling_compressed)) - return; - - kernel_tags.first_tag = (struct alloc_tag *)kallsyms_lookup_name( - SECTION_START(ALLOC_TAG_SECTION_NAME)); - last_codetag = (struct alloc_tag *)kallsyms_lookup_name( - SECTION_STOP(ALLOC_TAG_SECTION_NAME)); - kernel_tags.count = last_codetag - kernel_tags.first_tag; - - /* Check if kernel tags fit into page flags */ - if (kernel_tags.count > (1UL << NR_UNUSED_PAGEFLAG_BITS)) { - shutdown_mem_profiling(false); /* allocinfo file does not exist yet */ - pr_err("%lu allocation tags cannot be references using %d available page flag bits. Memory allocation profiling is disabled!\n", - kernel_tags.count, NR_UNUSED_PAGEFLAG_BITS); - return; - } - - alloc_tag_ref_offs = (LRU_REFS_PGOFF - NR_UNUSED_PAGEFLAG_BITS); - alloc_tag_ref_mask = ((1UL << NR_UNUSED_PAGEFLAG_BITS) - 1); - pr_debug("Memory allocation profiling compression is using %d page flag bits!\n", - NR_UNUSED_PAGEFLAG_BITS); -} - -#ifdef CONFIG_MODULES - -static struct maple_tree mod_area_mt = MTREE_INIT(mod_area_mt, MT_FLAGS_ALLOC_RANGE); -static struct vm_struct *vm_module_tags; -/* A dummy object used to indicate an unloaded module */ -static struct module unloaded_mod; -/* A dummy object used to indicate a module prepended area */ -static struct module prepend_mod; - -struct alloc_tag_module_section module_tags; - -static inline unsigned long alloc_tag_align(unsigned long val) -{ - if (!static_key_enabled(&mem_profiling_compressed)) { - /* No alignment requirements when we are not indexing the tags */ - return val; - } - - if (val % sizeof(struct alloc_tag) == 0) - return val; - return ((val / sizeof(struct alloc_tag)) + 1) * sizeof(struct alloc_tag); -} - -static bool ensure_alignment(unsigned long align, unsigned int *prepend) -{ - if (!static_key_enabled(&mem_profiling_compressed)) { - /* No alignment requirements when we are not indexing the tags */ - return true; - } - - /* - * If alloc_tag size is not a multiple of required alignment, tag - * indexing does not work. - */ - if (!IS_ALIGNED(sizeof(struct alloc_tag), align)) - return false; - - /* Ensure prepend consumes multiple of alloc_tag-sized blocks */ - if (*prepend) - *prepend = alloc_tag_align(*prepend); - - return true; -} - -static inline bool tags_addressable(void) -{ - unsigned long tag_idx_count; - - if (!static_key_enabled(&mem_profiling_compressed)) - return true; /* with page_ext tags are always addressable */ - - tag_idx_count = CODETAG_ID_FIRST + kernel_tags.count + - module_tags.size / sizeof(struct alloc_tag); - - return tag_idx_count < (1UL << NR_UNUSED_PAGEFLAG_BITS); -} - -static bool needs_section_mem(struct module *mod, unsigned long size) -{ - if (!mem_profiling_support) - return false; - - return size >= sizeof(struct alloc_tag); -} - -static bool clean_unused_counters(struct alloc_tag *start_tag, - struct alloc_tag *end_tag) -{ - struct alloc_tag *tag; - bool ret = true; - - for (tag = start_tag; tag <= end_tag; tag++) { - struct alloc_tag_counters counter; - - if (!tag->counters) - continue; - - counter = alloc_tag_read(tag); - if (!counter.bytes) { - free_percpu(tag->counters); - tag->counters = NULL; - } else { - ret = false; - } - } - - return ret; -} - -/* Called with mod_area_mt locked */ -static void clean_unused_module_areas_locked(void) -{ - MA_STATE(mas, &mod_area_mt, 0, module_tags.size); - struct module *val; - - mas_for_each(&mas, val, module_tags.size) { - struct alloc_tag *start_tag; - struct alloc_tag *end_tag; - - if (val != &unloaded_mod) - continue; - - /* Release area if all tags are unused */ - start_tag = (struct alloc_tag *)(module_tags.start_addr + mas.index); - end_tag = (struct alloc_tag *)(module_tags.start_addr + mas.last); - if (clean_unused_counters(start_tag, end_tag)) - mas_erase(&mas); - } -} - -/* Called with mod_area_mt locked */ -static bool find_aligned_area(struct ma_state *mas, unsigned long section_size, - unsigned long size, unsigned int prepend, unsigned long align) -{ - bool cleanup_done = false; - -repeat: - /* Try finding exact size and hope the start is aligned */ - if (!mas_empty_area(mas, 0, section_size - 1, prepend + size)) { - if (IS_ALIGNED(mas->index + prepend, align)) - return true; - - /* Try finding larger area to align later */ - mas_reset(mas); - if (!mas_empty_area(mas, 0, section_size - 1, - size + prepend + align - 1)) - return true; - } - - /* No free area, try cleanup stale data and repeat the search once */ - if (!cleanup_done) { - clean_unused_module_areas_locked(); - cleanup_done = true; - mas_reset(mas); - goto repeat; - } - - return false; -} - -static int vm_module_tags_populate(void) -{ - unsigned long phys_end = ALIGN_DOWN(module_tags.start_addr, PAGE_SIZE) + - (vm_module_tags->nr_pages << PAGE_SHIFT); - unsigned long new_end = module_tags.start_addr + module_tags.size; - - if (phys_end < new_end) { - struct page **next_page = vm_module_tags->pages + vm_module_tags->nr_pages; - unsigned long old_shadow_end = ALIGN(phys_end, MODULE_ALIGN); - unsigned long new_shadow_end = ALIGN(new_end, MODULE_ALIGN); - unsigned long more_pages; - unsigned long nr = 0; - - more_pages = ALIGN(new_end - phys_end, PAGE_SIZE) >> PAGE_SHIFT; - while (nr < more_pages) { - unsigned long allocated; - - allocated = alloc_pages_bulk_node(GFP_KERNEL | __GFP_NOWARN, - NUMA_NO_NODE, more_pages - nr, next_page + nr); - - if (!allocated) - break; - nr += allocated; - } - - if (nr < more_pages || - vmap_pages_range(phys_end, phys_end + (nr << PAGE_SHIFT), PAGE_KERNEL, - next_page, PAGE_SHIFT) < 0) { - release_pages_arg arg = { .pages = next_page }; - - /* Clean up and error out */ - release_pages(arg, nr); - return -ENOMEM; - } - - vm_module_tags->nr_pages += nr; - - /* - * Kasan allocates 1 byte of shadow for every 8 bytes of data. - * When kasan_alloc_module_shadow allocates shadow memory, - * its unit of allocation is a page. - * Therefore, here we need to align to MODULE_ALIGN. - */ - if (old_shadow_end < new_shadow_end) - kasan_alloc_module_shadow((void *)old_shadow_end, - new_shadow_end - old_shadow_end, - GFP_KERNEL); - } - - /* - * Mark the pages as accessible, now that they are mapped. - * With hardware tag-based KASAN, marking is skipped for - * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc(). - */ - kasan_unpoison_vmalloc((void *)module_tags.start_addr, - new_end - module_tags.start_addr, - KASAN_VMALLOC_PROT_NORMAL); - - return 0; -} - -static void *reserve_module_tags(struct module *mod, unsigned long size, - unsigned int prepend, unsigned long align) -{ - unsigned long section_size = module_tags.end_addr - module_tags.start_addr; - MA_STATE(mas, &mod_area_mt, 0, section_size - 1); - unsigned long offset; - void *ret = NULL; - - /* If no tags return error */ - if (size < sizeof(struct alloc_tag)) - return ERR_PTR(-EINVAL); - - /* - * align is always power of 2, so we can use IS_ALIGNED and ALIGN. - * align 0 or 1 means no alignment, to simplify set to 1. - */ - if (!align) - align = 1; - - if (!ensure_alignment(align, &prepend)) { - shutdown_mem_profiling(true); - pr_err("%s: alignment %lu is incompatible with allocation tag indexing. Memory allocation profiling is disabled!\n", - mod->name, align); - return ERR_PTR(-EINVAL); - } - - mas_lock(&mas); - if (!find_aligned_area(&mas, section_size, size, prepend, align)) { - ret = ERR_PTR(-ENOMEM); - goto unlock; - } - - /* Mark found area as reserved */ - offset = mas.index; - offset += prepend; - offset = ALIGN(offset, align); - if (offset != mas.index) { - unsigned long pad_start = mas.index; - - mas.last = offset - 1; - mas_store(&mas, &prepend_mod); - if (mas_is_err(&mas)) { - ret = ERR_PTR(xa_err(mas.node)); - goto unlock; - } - mas.index = offset; - mas.last = offset + size - 1; - mas_store(&mas, mod); - if (mas_is_err(&mas)) { - mas.index = pad_start; - mas_erase(&mas); - ret = ERR_PTR(xa_err(mas.node)); - } - } else { - mas.last = offset + size - 1; - mas_store(&mas, mod); - if (mas_is_err(&mas)) - ret = ERR_PTR(xa_err(mas.node)); - } -unlock: - mas_unlock(&mas); - - if (IS_ERR(ret)) - return ret; - - if (module_tags.size < offset + size) { - int grow_res; - - module_tags.size = offset + size; - if (mem_alloc_profiling_enabled() && !tags_addressable()) { - shutdown_mem_profiling(true); - pr_warn("With module %s there are too many tags to fit in %d page flag bits. Memory allocation profiling is disabled!\n", - mod->name, NR_UNUSED_PAGEFLAG_BITS); - } - - grow_res = vm_module_tags_populate(); - if (grow_res) { - shutdown_mem_profiling(true); - pr_err("Failed to allocate memory for allocation tags in the module %s. Memory allocation profiling is disabled!\n", - mod->name); - return ERR_PTR(grow_res); - } - } - - return (struct alloc_tag *)(module_tags.start_addr + offset); -} - -static void release_module_tags(struct module *mod, bool used) -{ - MA_STATE(mas, &mod_area_mt, module_tags.size, module_tags.size); - struct alloc_tag *start_tag; - struct alloc_tag *end_tag; - struct module *val; - - mas_lock(&mas); - mas_for_each_rev(&mas, val, 0) - if (val == mod) - break; - - if (!val) /* module not found */ - goto out; - - if (!used) - goto release_area; - - start_tag = (struct alloc_tag *)(module_tags.start_addr + mas.index); - end_tag = (struct alloc_tag *)(module_tags.start_addr + mas.last); - if (!clean_unused_counters(start_tag, end_tag)) { - struct alloc_tag *tag; - - for (tag = start_tag; tag <= end_tag; tag++) { - struct alloc_tag_counters counter; - - if (!tag->counters) - continue; - - counter = alloc_tag_read(tag); - pr_info("%s:%u module %s func:%s has %llu allocated at module unload\n", - tag->ct.filename, tag->ct.lineno, tag->ct.modname, - tag->ct.function, counter.bytes); - } - } else { - used = false; - } -release_area: - mas_store(&mas, used ? &unloaded_mod : NULL); - val = mas_prev_range(&mas, 0); - if (val == &prepend_mod) - mas_store(&mas, NULL); -out: - mas_unlock(&mas); -} - -static int load_module(struct module *mod, struct codetag *start, struct codetag *stop) -{ - /* Allocate module alloc_tag percpu counters */ - struct alloc_tag *start_tag; - struct alloc_tag *stop_tag; - struct alloc_tag *tag; - - /* percpu counters for core allocations are already statically allocated */ - if (!mod) - return 0; - - start_tag = ct_to_alloc_tag(start); - stop_tag = ct_to_alloc_tag(stop); - for (tag = start_tag; tag < stop_tag; tag++) { - WARN_ON(tag->counters); - tag->counters = alloc_percpu(struct alloc_tag_counters); - if (!tag->counters) { - while (--tag >= start_tag) { - free_percpu(tag->counters); - tag->counters = NULL; - } - pr_err("Failed to allocate memory for allocation tag percpu counters in the module %s\n", - mod->name); - return -ENOMEM; - } - - /* - * Avoid a kmemleak false positive. The pointer to the counters is stored - * in the alloc_tag section of the module and cannot be directly accessed. - */ - kmemleak_ignore_percpu(tag->counters); - } - return 0; -} - -static void replace_module(struct module *mod, struct module *new_mod) -{ - MA_STATE(mas, &mod_area_mt, 0, module_tags.size); - struct module *val; - - mas_lock(&mas); - mas_for_each(&mas, val, module_tags.size) { - if (val != mod) - continue; - - mas_store_gfp(&mas, new_mod, GFP_KERNEL); - break; - } - mas_unlock(&mas); -} - -static int __init alloc_mod_tags_mem(void) -{ - /* Map space to copy allocation tags */ - vm_module_tags = execmem_vmap(MODULE_ALLOC_TAG_VMAP_SIZE); - if (!vm_module_tags) { - pr_err("Failed to map %lu bytes for module allocation tags\n", - MODULE_ALLOC_TAG_VMAP_SIZE); - module_tags.start_addr = 0; - return -ENOMEM; - } - - vm_module_tags->pages = kmalloc_objs(struct page *, - get_vm_area_size(vm_module_tags) >> PAGE_SHIFT, - GFP_KERNEL | __GFP_ZERO); - if (!vm_module_tags->pages) { - free_vm_area(vm_module_tags); - return -ENOMEM; - } - - module_tags.start_addr = (unsigned long)vm_module_tags->addr; - module_tags.end_addr = module_tags.start_addr + MODULE_ALLOC_TAG_VMAP_SIZE; - /* Ensure the base is alloc_tag aligned when required for indexing */ - module_tags.start_addr = alloc_tag_align(module_tags.start_addr); - - return 0; -} - -static void __init free_mod_tags_mem(void) -{ - release_pages_arg arg = { .pages = vm_module_tags->pages }; - - module_tags.start_addr = 0; - release_pages(arg, vm_module_tags->nr_pages); - kfree(vm_module_tags->pages); - free_vm_area(vm_module_tags); -} - -#else /* CONFIG_MODULES */ - -static inline int alloc_mod_tags_mem(void) { return 0; } -static inline void free_mod_tags_mem(void) {} - -#endif /* CONFIG_MODULES */ - -/* See: Documentation/mm/allocation-profiling.rst */ -static int __init setup_early_mem_profiling(char *str) -{ - bool compressed = false; - bool enable; - - if (!str || !str[0]) - return -EINVAL; - - if (!strncmp(str, "never", 5)) { - enable = false; - mem_profiling_support = false; - pr_info("Memory allocation profiling is disabled!\n"); - } else { - char *token = strsep(&str, ","); - - if (kstrtobool(token, &enable)) - return -EINVAL; - - if (str) { - - if (strcmp(str, "compressed")) - return -EINVAL; - - compressed = true; - } - mem_profiling_support = true; - pr_info("Memory allocation profiling is enabled %s compression and is turned %s!\n", - compressed ? "with" : "without", str_on_off(enable)); - } - - if (enable != mem_alloc_profiling_enabled()) { - if (enable) - static_branch_enable(&mem_alloc_profiling_key); - else - static_branch_disable(&mem_alloc_profiling_key); - } - if (compressed != static_key_enabled(&mem_profiling_compressed)) { - if (compressed) - static_branch_enable(&mem_profiling_compressed); - else - static_branch_disable(&mem_profiling_compressed); - } - - return 0; -} -early_param("sysctl.vm.mem_profiling", setup_early_mem_profiling); - -static __init bool need_page_alloc_tagging(void) -{ - if (static_key_enabled(&mem_profiling_compressed)) - return false; - - return mem_profiling_support; -} - -#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG -/* - * Track page allocations before page_ext is initialized. - * Some pages are allocated before page_ext becomes available, leaving - * their codetag uninitialized. Track these early PFNs so we can clear - * their codetag refs later to avoid warnings when they are freed. - * - * Each page is cast to a pfn_pool: the first few bytes hold metadata - * (next pointer and slot count), the remainder stores PFNs. - */ -struct pfn_pool { - struct pfn_pool *next; - atomic_t count; - unsigned long pfns[]; -}; - -#define PFN_POOL_SIZE ((PAGE_SIZE - offsetof(struct pfn_pool, pfns)) / \ - sizeof(unsigned long)) - -/* - * Skip early PFN recording for a page allocation. Reuses the - * %__GFP_NO_OBJ_EXT bit. Used by __alloc_tag_add_early_pfn() to avoid - * recursion when allocating pages for the early PFN tracking list - * itself. - * - * Codetags of the pages allocated with __GFP_NO_CODETAG should be - * cleared (via clear_page_tag_ref()) before freeing the pages to prevent - * alloc_tag_sub_check() from triggering a warning. - */ -#define __GFP_NO_CODETAG __GFP_NO_OBJ_EXT - -static struct pfn_pool *current_pfn_pool __initdata; - -static void __init __alloc_tag_add_early_pfn(unsigned long pfn) -{ - struct pfn_pool *pool; - int idx; - - do { - pool = READ_ONCE(current_pfn_pool); - if (!pool || atomic_read(&pool->count) >= PFN_POOL_SIZE) { - struct page *new_page = alloc_page(__GFP_HIGH | __GFP_NO_CODETAG); - struct pfn_pool *new; - - if (!new_page) { - pr_warn_once("early PFN tracking page allocation failed\n"); - return; - } - new = page_address(new_page); - new->next = pool; - atomic_set(&new->count, 0); - if (cmpxchg(¤t_pfn_pool, pool, new) != pool) { - clear_page_tag_ref(new_page); - __free_page(new_page); - continue; - } - pool = new; - } - idx = atomic_read(&pool->count); - if (idx >= PFN_POOL_SIZE) - continue; - if (atomic_cmpxchg(&pool->count, idx, idx + 1) == idx) - break; - } while (1); - - pool->pfns[idx] = pfn; -} - -typedef void alloc_tag_add_func(unsigned long pfn); -static alloc_tag_add_func __rcu *alloc_tag_add_early_pfn_ptr __refdata = - RCU_INITIALIZER(__alloc_tag_add_early_pfn); - -void alloc_tag_add_early_pfn(unsigned long pfn, gfp_t gfp_flags) -{ - alloc_tag_add_func *alloc_tag_add; - - if (static_key_enabled(&mem_profiling_compressed)) - return; - - /* Skip allocations for the tracking list itself to avoid recursion. */ - if (gfp_flags & __GFP_NO_CODETAG) - return; - - rcu_read_lock(); - alloc_tag_add = rcu_dereference(alloc_tag_add_early_pfn_ptr); - if (alloc_tag_add) - alloc_tag_add(pfn); - rcu_read_unlock(); -} - -static void __init clear_early_alloc_pfn_tag_refs(void) -{ - struct pfn_pool *pool, *next; - struct page *page; - int i; - - if (static_key_enabled(&mem_profiling_compressed)) - return; - - rcu_assign_pointer(alloc_tag_add_early_pfn_ptr, NULL); - /* Make sure we are not racing with __alloc_tag_add_early_pfn() */ - synchronize_rcu(); - - for (pool = current_pfn_pool; pool; pool = next) { - int nr_pfns = atomic_read(&pool->count); - - for (i = 0; i < nr_pfns; i++) { - unsigned long pfn = pool->pfns[i]; - - if (pfn_valid(pfn)) { - union pgtag_ref_handle handle; - union codetag_ref ref; - - if (get_page_tag_ref(pfn_to_page(pfn), &ref, &handle)) { - /* - * An early-allocated page could be freed and reallocated - * after its page_ext is initialized but before we clear it. - * In that case, it already has a valid tag set. - * We should not overwrite that valid tag - * with CODETAG_EMPTY. - * - * Note: there is still a small race window between checking - * ref.ct and calling set_codetag_empty(). We accept this - * race as it's unlikely and the extra complexity of atomic - * cmpxchg is not worth it for this debug-only code path. - */ - if (ref.ct) { - put_page_tag_ref(handle); - continue; - } - - set_codetag_empty(&ref); - update_page_tag_ref(handle, &ref); - put_page_tag_ref(handle); - } - } - } - - next = pool->next; - page = virt_to_page(pool); - clear_page_tag_ref(page); - __free_page(page); - } -} -#else /* !CONFIG_MEM_ALLOC_PROFILING_DEBUG */ -static inline void __init clear_early_alloc_pfn_tag_refs(void) {} -#endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */ - -static __init void init_page_alloc_tagging(void) -{ - clear_early_alloc_pfn_tag_refs(); -} - -struct page_ext_operations page_alloc_tagging_ops = { - .size = sizeof(union codetag_ref), - .need = need_page_alloc_tagging, - .init = init_page_alloc_tagging, -}; -EXPORT_SYMBOL(page_alloc_tagging_ops); - -#ifdef CONFIG_SYSCTL -/* - * Not using proc_do_static_key() directly to prevent enabling profiling - * after it was shut down. - */ -static int proc_mem_profiling_handler(const struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos) -{ - if (write) { - /* - * Call from do_sysctl_args() which is a no-op since the same - * value was already set by setup_early_mem_profiling. - * Return success to avoid warnings from do_sysctl_args(). - */ - if (!current->mm) - return 0; - -#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG - /* User can't toggle profiling while debugging */ - return -EACCES; -#endif - if (!mem_profiling_support) - return -EINVAL; - } - - return proc_do_static_key(table, write, buffer, lenp, ppos); -} - - -static const struct ctl_table memory_allocation_profiling_sysctls[] = { - { - .procname = "mem_profiling", - .data = &mem_alloc_profiling_key, - .mode = 0644, - .proc_handler = proc_mem_profiling_handler, - }, -}; - -static void __init sysctl_init(void) -{ - register_sysctl_init("vm", memory_allocation_profiling_sysctls); -} -#else /* CONFIG_SYSCTL */ -static inline void sysctl_init(void) {} -#endif /* CONFIG_SYSCTL */ - -static int __init alloc_tag_init(void) -{ - const struct codetag_type_desc desc = { - .section = ALLOC_TAG_SECTION_NAME, - .tag_size = sizeof(struct alloc_tag), -#ifdef CONFIG_MODULES - .needs_section_mem = needs_section_mem, - .alloc_section_mem = reserve_module_tags, - .free_section_mem = release_module_tags, - .module_load = load_module, - .module_replaced = replace_module, -#endif - }; - int res; - - sysctl_init(); - - if (!mem_profiling_support) { - pr_info("Memory allocation profiling is not supported!\n"); - return 0; - } - - if (!proc_create_seq_private(ALLOCINFO_FILE_NAME, 0400, NULL, &allocinfo_seq_op, - sizeof(struct allocinfo_private), NULL)) { - pr_err("Failed to create %s file\n", ALLOCINFO_FILE_NAME); - shutdown_mem_profiling(false); - return -ENOMEM; - } - - res = alloc_mod_tags_mem(); - if (res) { - pr_err("Failed to reserve address space for module tags, errno = %d\n", res); - shutdown_mem_profiling(true); - return res; - } - - alloc_tag_cttype = codetag_register_type(&desc); - if (IS_ERR(alloc_tag_cttype)) { - pr_err("Allocation tags registration failed, errno = %pe\n", alloc_tag_cttype); - free_mod_tags_mem(); - shutdown_mem_profiling(true); - return PTR_ERR(alloc_tag_cttype); - } - - return 0; -} -module_init(alloc_tag_init); -- cgit v1.2.3 From e14a3454806468b086fe2e4ca2e1bff95b528531 Mon Sep 17 00:00:00 2001 From: Shakeel Butt Date: Tue, 1 Sep 2026 11:01:09 -0700 Subject: mm/mlock: use the IRQ-safe accessor for NR_MLOCK in __munlock_folio() NR_MLOCK is updated from interrupt context. __free_pages_prepare() clears a stray PG_mlocked and adjusts NR_MLOCK, and a folio can reach it with the flag still set from a bio completion handler: __free_pages_ok+0x6af/0x7a0 __bio_release_pages+0xde/0x260 __iomap_dio_bio_end_io+0x16e/0x1a0 blk_update_request+0x14b/0x3d0 blk_mq_end_request+0x18/0x30 blk_done_softirq+0x49/0x60 The folio gets there like this. A MAP_SHARED file mapping is mlocked, so its page cache folios carry PG_mlocked, and an O_DIRECT write sourced from that mapping GUP-pins those same folios. munlock() then runs mlock_vma_pages_range(), which clears VM_LOCKED before walking the page tables to munlock each folio. A concurrent hole punch reaches the folio through the rmap (i_mmap_rwsem, not mmap_lock) and can land inside that window: __folio_remove_rmap() -> munlock_vma_folio() sees VM_LOCKED already clear, so it neither queues the folio on the mlock batch nor takes a reference, and the pte it clears makes the pending mlock_pte_range() walk skip the folio at its !pte_present() check. filemap_remove_folio() then drops the page cache reference, leaving the bio's pin as the last one, released from the completion handler above. So __zone_stat_mod_folio() here needs interrupts disabled, not merely preemption, and __munlock_folio() has a path where they are not: when the folio has already been taken off the LRU by somebody else the function jumps straight to the counter update without taking the lruvec lock. The read-modify-write of the per-CPU NR_MLOCK diff can then be interrupted by the softirq above, and one of the two decrements is lost, leaving Mlocked in /proc/meminfo permanently overstated. Use zone_stat_mod_folio(). mod_zone_state()'s this_cpu_try_cmpxchg() is atomic against a same-CPU interrupt and retries, and on the path where the lruvec lock is held its cost is negligible next to the lock itself. The UNEVICTABLE_PG* events are deliberately left on the __ accessors: they occupy different vm_event_states slots from the UNEVICTABLE_PGCLEARED that __free_pages_prepare() bumps, and nothing updates those two from interrupt context. Link: https://lore.kernel.org/20260901180109.3797944-1-shakeel.butt@linux.dev Fixes: 2fbb0c10d1e8 ("mm/munlock: mlock_page() munlock_page() batch by pagevec") Signed-off-by: Shakeel Butt Reported-by: syzbot+cd2073ee6d958a8d0fcd@syzkaller.appspotmail.com Closes: https://lore.kernel.org/linux-mm/6a931c5a.08e933ee.dbf97.0093.GAE@google.com/ Acked-by: Hugh Dickins Cc: Jann Horn Cc: Liam R. Howlett Cc: Lorenzo Stoakes Cc: Matthew Wilcox (Oracle) Cc: Pedro Falcato Cc: Vlastimil Babka Cc: Signed-off-by: Andrew Morton --- mm/mlock.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/mlock.c b/mm/mlock.c index efa6716e4dfb..39215a3eab1f 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -141,7 +141,7 @@ static struct lruvec *__munlock_folio(struct folio *folio, struct lruvec *lruvec munlock: if (folio_test_clear_mlocked(folio)) { - __zone_stat_mod_folio(folio, NR_MLOCK, -nr_pages); + zone_stat_mod_folio(folio, NR_MLOCK, -nr_pages); if (isolated || !folio_test_unevictable(folio)) __count_vm_events(UNEVICTABLE_PGMUNLOCKED, nr_pages); else -- cgit v1.2.3 From 6cc27d82196385fe06853319f74312a7d8019726 Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Wed, 2 Sep 2026 19:08:08 +0100 Subject: mm/vma: correctly unaccount on mmap_prepare() failure __mmap_setup() accounts memory for relevant mappings via: security_vm_enough_memory_mm() -> __vm_enough_memory() -> vm_acct_memory() If __mmap_setup() fails, this indicates that this accounting did not take place, and thus it's appropriate for __mmap_region() to jump to abort_munmap. However if call_mmap_prepare() fails, it also jumps there and any accounted memory is not correctly unaccounted. Fix this by handling each error separately. Link: https://lore.kernel.org/20260902-fix-unaccount-mmap_prepare-v1-1-ea070189fdfb@kernel.org Fixes: c84bf6dd2b83 ("mm: introduce new .mmap_prepare() file callback") Signed-off-by: Lorenzo Stoakes (ARM) Cc: Jann Horn Cc: Liam R. Howlett Cc: Pedro Falcato Cc: Vlastimil Babka Cc: Signed-off-by: Andrew Morton --- mm/vma.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mm/vma.c b/mm/vma.c index 35e7a64855fa..f29abb30956b 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -2859,10 +2859,12 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, map.check_ksm_early = can_set_ksm_flags_early(&map); error = __mmap_setup(&map, &desc, uf); - if (!error && have_mmap_prepare) - error = call_mmap_prepare(&map, &desc); if (error) goto abort_munmap; + if (have_mmap_prepare) + error = call_mmap_prepare(&map, &desc); + if (error) + goto unacct_error; if (map.check_ksm_early) update_ksm_flags(&map); -- cgit v1.2.3 From 932cfb25e7ce98d1f93895671ec186a3087e4f80 Mon Sep 17 00:00:00 2001 From: Jiayuan Chen Date: Wed, 2 Sep 2026 15:37:59 +0800 Subject: mm/shrinker: fix bogus set_shrinker_bit() with cgroup.memory=nokmem With cgroup.memory=nokmem, shrinker_memcg_alloc() bails out early and never allocates an id, so shrinker->id keeps the 0 it got from the kzalloc() in shrinker_alloc(). __list_lru_init() then copies that 0 into lru->shrinker_id, where it looks like a valid bit index. Nothing calls expand_shrinker_info() on nokmem either, so shrinker_nr_max stays 0 and every memcg ends up with an empty map (map_nr_max == 0). deferred_split_folio() hands a real memcg to __list_lru_add() regardless of whether the lru is memcg aware, so the first THP queued in a cgroup does set_shrinker_bit(memcg, nid, 0) and trips the bounds check: WARNING: mm/shrinker.c:212 at set_shrinker_bit+0x7d/0x90, CPU#126 Call Trace: deferred_split_folio+0x18c/0x220 map_anon_folio_pmd_nopf+0xdd/0x130 map_anon_folio_pmd_pf+0x14/0xb0 do_huge_pmd_anonymous_page+0x1a1/0x620 __handle_mm_fault+0xea9/0x10d0 handle_mm_fault+0xe5/0x320 do_user_addr_fault+0x1cc/0x870 exc_page_fault+0x81/0x1b0 asm_exc_page_fault+0x27/0x30 Harmless, the WARN_ON_ONCE() is what keeps the out of bounds unit[] read from happening, but the id should not look valid in the first place. Clear it before returning. Two other spots could paper over this: drop the id in __list_lru_init() when nokmem turns memcg_aware off, or make deferred_split_folio() pass NULL like list_lru_add_obj() does. Both leave shrinker->id lying around for the next caller, so fix it where the id is handed out. Link: https://lore.kernel.org/20260902073800.305481-1-jiayuan.chen@linux.dev Fixes: fafaeceb89a5 ("mm: switch deferred split shrinker to list_lru") Signed-off-by: Jiayuan Chen Acked-by: Shakeel Butt Cc: Usama Arif Cc: Dave Chinner Cc: Johannes Weiner Cc: Kairui Song Cc: Muchun Song Cc: Roman Gushchin Cc: Signed-off-by: Andrew Morton --- mm/shrinker.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mm/shrinker.c b/mm/shrinker.c index a70aab124a0e..7ec2a9704f6f 100644 --- a/mm/shrinker.c +++ b/mm/shrinker.c @@ -227,6 +227,8 @@ static int shrinker_memcg_alloc(struct shrinker *shrinker) { int id; + shrinker->id = -1; + if (mem_cgroup_disabled()) return -ENOSYS; if (mem_cgroup_kmem_disabled() && !(shrinker->flags & SHRINKER_NONSLAB)) -- cgit v1.2.3 From 7891fbb9512f127826e1d5dbf380ee212bd15eb0 Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Tue, 1 Sep 2026 20:38:40 -0700 Subject: mm/folio: EXPORT_SYMBOL_FOR_KVM(lru_cache_drain_for_folio) To simplify independent development in the KVM and MM subsystems, now export to KVM the lru_cache_drain_for_folio() which MM added in 7.3-rc1. Link: https://lore.kernel.org/lkml/bd6c9c74-e374-a9d3-ba1f-8b6f430894fc@google.com/T/#u Link: https://lore.kernel.org/02876cea-5727-2ca4-bead-73659ea6fec4@google.com Signed-off-by: Ackerley Tng Signed-off-by: Hugh Dickins Acked-by: Vlastimil Babka (SUSE) Suggested-by: David Hildenbrand Reviewed-by: Fuad Tabba Reviewed-by: Binbin Wu Cc: Matthew Wilcox (Oracle) Cc: Sean Christopherson Signed-off-by: Andrew Morton --- mm/folio.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mm/folio.c b/mm/folio.c index c02dcea9c03c..50a6dbe55998 100644 --- a/mm/folio.c +++ b/mm/folio.c @@ -33,6 +33,7 @@ #include #include #include +#include #include "internal.h" #include "page_alloc.h" @@ -926,6 +927,7 @@ void lru_cache_drain_for_folio(const struct folio *folio, *drained = LRU_CACHE_DRAINED_ALL; } } +EXPORT_SYMBOL_FOR_KVM(lru_cache_drain_for_folio); atomic_t lru_disable_count = ATOMIC_INIT(0); -- cgit v1.2.3