From 51943a18ad4bd6ff8baea2da7b8cce2f86f1a959 Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Thu, 13 Aug 2026 18:32:19 +0100 Subject: mm: provide vma_[flags_]is_cow_mapping() and remove is_cow_mapping() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All remaining callers of is_cow_mapping() are invoking it in the form of is_cow_mapping(vma->vm_flags) or an indirected version of this. Therefore, provide a helper - vma_is_cow_mapping() to directly test the VMA. Additionally provide a new helper vma_flags_is_cow_mapping() which performs the check using the new vma_flags_t type, and share this logic between vma_is_cow_mapping() and vma_desc_is_cow_mapping(). With these changes, no callers of is_cow_mapping() remain, so remove it. Also update the userland VMA tests to reflect the change. No functional change intended. [akpm@linux-foundation.org: fix kerneldoc comment typo, per Lorenzo] Link: https://lore.kernel.org/aob1goSSPH6sTN9y@gremlin Link: https://lore.kernel.org/20260813-b4-scalable-cow-virt-pgoff-v5-2-c21581c0c3c8@kernel.org Signed-off-by: Lorenzo Stoakes (ARM) Acked-by: David Hildenbrand (Arm) Cc: Adrian Hunter Cc: Alexander Deucher Cc: Alexander Gordeev Cc: Alexander Shishkin Cc: Alistair Popple Cc: Arnaldo Carvalho de Melo Cc: Arnd Bergmann Cc: Baolin Wang Cc: Baoquan He Cc: Barry Song Cc: Boris Brezillon Cc: Byungchul Park Cc: Chengming Zhou Cc: Chris Li Cc: Christan König Cc: Christian Borntraeger Cc: Claudio Imbrenda Cc: Dave Airlie Cc: Dev Jain Cc: Gerald Schaefer Cc: Greg Kroah-Hartman Cc: Gregory Price (Meta) Cc: Harry Yoo Cc: Heiko Carstens Cc: Huang Ray Cc: "Huang, Ying" Cc: Ian Rogers Cc: Ingo Molnar Cc: James Clark Cc: Jan Kara Cc: Jann Horn Cc: Janosch Frank Cc: Jason Gunthorpe Cc: Jiri Olsa Cc: John Hubbard Cc: Joshua Hahn Cc: Kairui Song Cc: Kees Cook Cc: Kemeng Shi Cc: Lance Yang Cc: Liam R. Howlett Cc: Liviu Dudau Cc: Maarten Lankhorst Cc: Marc Rutland Cc: "Masami Hiramatsu (Google)" Cc: Matthew Auld Cc: Matthew Brost Cc: Matthew Wilcox (Oracle) Cc: Maxime Ripard Cc: Miaohe Lin Cc: Michal Hocko Cc: Mike Rapoport Cc: Muchun Song Cc: Namhyung kim Cc: Naoya Horiguchi Cc: Nhat Pham Cc: Nico Pache Cc: Oleg Nesterov Cc: Oscar Salvador Cc: Pedro Falcato Cc: Peter Xu Cc: Peter Zijlstra Cc: Rakie Kim Cc: Rik van Riel Cc: Rodrigo Vivi Cc: Ryan Roberts Cc: Steven Price Cc: Suren Baghdasaryan Cc: Sven Schnelle Cc: Thomas Hellström Cc: Thomas Zimemrmann Cc: Vasily Gorbik Cc: Vlastimil Babka Cc: xu xin Cc: Zi Yan Signed-off-by: Andrew Morton --- drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c | 4 ++-- drivers/gpu/drm/drm_gem_shmem_helper.c | 2 +- drivers/gpu/drm/panthor/panthor_gem.c | 2 +- drivers/gpu/drm/ttm/ttm_bo_vm.c | 2 +- drivers/gpu/drm/xe/xe_device.c | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) (limited to 'drivers') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c index 6a0699746fbc..0c7309080a7a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c @@ -377,9 +377,9 @@ static int amdgpu_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_str /* Workaround for Thunk bug creating PROT_NONE,MAP_PRIVATE mappings * for debugger access to invisible VRAM. Should have used MAP_SHARED * instead. Clearing VM_MAYWRITE prevents the mapping from ever - * becoming writable and makes is_cow_mapping(vm_flags) false. + * becoming writable and makes vma_is_cow_mapping(vma) false. */ - if (is_cow_mapping(vma->vm_flags) && + if (vma_is_cow_mapping(vma) && !(vma->vm_flags & VM_ACCESS_FLAGS)) vm_flags_clear(vma, VM_MAYWRITE); diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c index 06d019d51d3e..177d0e0b9334 100644 --- a/drivers/gpu/drm/drm_gem_shmem_helper.c +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c @@ -753,7 +753,7 @@ int drm_gem_shmem_mmap(struct drm_gem_shmem_object *shmem, struct vm_area_struct return ret; } - if (is_cow_mapping(vma->vm_flags)) + if (vma_is_cow_mapping(vma)) return -EINVAL; dma_resv_lock(shmem->base.resv, NULL); diff --git a/drivers/gpu/drm/panthor/panthor_gem.c b/drivers/gpu/drm/panthor/panthor_gem.c index 770556353968..d2eec46f7abe 100644 --- a/drivers/gpu/drm/panthor/panthor_gem.c +++ b/drivers/gpu/drm/panthor/panthor_gem.c @@ -761,7 +761,7 @@ static int panthor_gem_mmap(struct drm_gem_object *obj, struct vm_area_struct *v return ret; } - if (is_cow_mapping(vma->vm_flags)) + if (vma_is_cow_mapping(vma)) return -EINVAL; if (!refcount_inc_not_zero(&bo->cmap.mmap_count)) { diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c index 88babf435ac2..872bf444b1f0 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_vm.c +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c @@ -489,7 +489,7 @@ static const struct vm_operations_struct ttm_bo_vm_ops = { int ttm_bo_mmap_obj(struct vm_area_struct *vma, struct ttm_buffer_object *bo) { /* Enforce no COW since would have really strange behavior with it. */ - if (is_cow_mapping(vma->vm_flags)) + if (vma_is_cow_mapping(vma)) return -EINVAL; drm_gem_object_get(&bo->base); diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index 9d119c95a569..de5fdf49d729 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -330,7 +330,7 @@ static int xe_pci_barrier_mmap(struct file *filp, if (vma->vm_end - vma->vm_start > SZ_4K) return -EINVAL; - if (is_cow_mapping(vma->vm_flags)) + if (vma_is_cow_mapping(vma)) return -EINVAL; if (vma->vm_flags & (VM_READ | VM_EXEC)) -- cgit v1.2.3 From dde75313eed0b014c437f48dd75c0308b592cbf9 Mon Sep 17 00:00:00 2001 From: Sergey Senozhatsky Date: Wed, 29 Jul 2026 13:57:35 +0900 Subject: zram: set default primary compressor in zram_destroy_comps() Patch series "zram: fix zram issues reported by sashiko". Sashiko drove by and reported [1] a couple of zram issues: a possible BUG_ON() in zlib code due to missing winbits range validation and one possible NULL-ptr dereference in zcomp. Both are low risk yet still worth fixing. This patch (of 2): zram_destroy_comps() resets all compressors and leaves them set to NULL, including the primary one, which is invalid device state, as now comp_algorithm_show()->strcmp() can be called on a NULL compressor. Set default primary compressor in zram_destroy_comps(). Link: https://lore.kernel.org/20260729045745.775973-2-senozhatsky@chromium.org Fixes: 486fd58af7ac ("zram: don't free statically defined names") Link: https://sashiko.dev/#/patchset/20260728092935.31139-1-haoqinhuang7@gmail.com [1] Signed-off-by: Sergey Senozhatsky Cc: Minchan Kim Cc: Haoqin Huang Cc: Signed-off-by: Andrew Morton --- drivers/block/zram/zram_drv.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index 4bfe63a5225d..cfa98846ac48 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -2828,6 +2828,7 @@ static void zram_destroy_comps(struct zram *zram) zram->comp_algs[prio] = NULL; zram_comp_params_reset(zram); + comp_algorithm_set(zram, ZRAM_PRIMARY_COMP, default_compressor); } static void zram_reset_device(struct zram *zram) @@ -2845,8 +2846,6 @@ static void zram_reset_device(struct zram *zram) zram_destroy_comps(zram); memset(&zram->stats, 0, sizeof(zram->stats)); reset_bdev(zram); - - comp_algorithm_set(zram, ZRAM_PRIMARY_COMP, default_compressor); } static ssize_t disksize_store(struct device *dev, struct device_attribute *attr, -- cgit v1.2.3 From ec7607ac4717ff521c9c1e9d8271c26293345513 Mon Sep 17 00:00:00 2001 From: Sergey Senozhatsky Date: Wed, 29 Jul 2026 13:57:36 +0900 Subject: zram: validate deflate params We must validate user-supplied deflate winbits before we pass it to zlib_deflate_workspacesize(), which triggers BUG_ON() if winbits value is outside of valid ranges. Link: https://lore.kernel.org/20260729045745.775973-3-senozhatsky@chromium.org Fixes: dc75a0d93bd5 ("zram: support deflate-specific params") Link: https://sashiko.dev/#/patchset/20260728092935.31139-1-haoqinhuang7@gmail.com Signed-off-by: Sergey Senozhatsky Cc: Minchan Kim Cc: Haoqin Huang Cc: Signed-off-by: Andrew Morton --- drivers/block/zram/backend_deflate.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/block/zram/backend_deflate.c b/drivers/block/zram/backend_deflate.c index f92a52a720d1..b3f7d08b49d9 100644 --- a/drivers/block/zram/backend_deflate.c +++ b/drivers/block/zram/backend_deflate.c @@ -24,8 +24,16 @@ static int deflate_setup_params(struct zcomp_params *params) { if (params->level == ZCOMP_PARAM_NOT_SET) params->level = Z_DEFAULT_COMPRESSION; - if (params->deflate.winbits == ZCOMP_PARAM_NOT_SET) + if (params->deflate.winbits == ZCOMP_PARAM_NOT_SET) { params->deflate.winbits = DEFLATE_DEF_WINBITS; + } else { + s32 wb = params->deflate.winbits; + + if ((wb < -15 || wb > -9) && (wb < 9 || wb > 15)) { + pr_err("invalid deflate winbits: %d\n", wb); + return -EINVAL; + } + } return 0; } -- cgit v1.2.3 From 45214458d6b50124afef3187f6352adeddf74d6f Mon Sep 17 00:00:00 2001 From: Haoqin Huang Date: Tue, 4 Aug 2026 17:38:37 +0800 Subject: zram: do not release zstd global params from error paths Patch series "zram: fix zstd error paths and add parameter validation", v6, Patch 1 removes zstd_release_params() from both zstd_create() and zstd_setup_params() error paths -- the former is a layering violation in a per-CPU callback, the latter is redundant as zcomp_init() already calls release_params() on setup failure. Patch 2 rejects zero-size dictionaries and prints distinct error messages for sz < 0 (returns the original error code) and sz == 0 ("empty file"). Currently errors are silently swallowed. Patch 3 adds pr_fmt to each backend file so that pr_err() messages are auto-prefixed with the algorithm name. Patch 4 validates dict and level parameters in each backend's .setup_params(), rejecting unsupported combinations and out-of-range levels. Patch 5 resets per-priority params on algorithm change before init. This patch (of 5): zstd_setup_params() creates global cdict and ddict stored in params->drv_data, shared across all per-CPU contexts. The per-CPU zstd_create() error path called zstd_release_params(), which freed those globally-shared objects. This is a layering violation: a per-CPU callback should only clean up its own context, not release resources owned by the compression lifecycle. zstd_setup_params() called zstd_release_params() on its own error path as well, but zcomp_init() already calls release_params() when setup fails, so this is redundant. Remove zstd_release_params() from both error paths. Link: https://lore.kernel.org/20260804093841.67920-1-haoqinhuang7@gmail.com Link: https://lore.kernel.org/20260804093841.67920-2-haoqinhuang7@gmail.com Signed-off-by: Haoqin Huang Signed-off-by: Rongwei Wang Reviewed-by: Sergey Senozhatsky Tested-by: Sergey Senozhatsky Cc: David Sterba Cc: Jens Axboe Cc: Minchan Kim Cc: Nick Terrell Signed-off-by: Andrew Morton --- drivers/block/zram/backend_zstd.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'drivers') diff --git a/drivers/block/zram/backend_zstd.c b/drivers/block/zram/backend_zstd.c index d00b548056dc..5fabc3e7e975 100644 --- a/drivers/block/zram/backend_zstd.c +++ b/drivers/block/zram/backend_zstd.c @@ -85,7 +85,6 @@ static int zstd_setup_params(struct zcomp_params *params) return 0; error: - zstd_release_params(params); return -EINVAL; } @@ -161,7 +160,6 @@ static int zstd_create(struct zcomp_params *params, struct zcomp_ctx *ctx) return 0; error: - zstd_release_params(params); zstd_destroy(ctx); return -EINVAL; } -- cgit v1.2.3 From 6dc404d433adf09045565054aecf85714db95b46 Mon Sep 17 00:00:00 2001 From: Haoqin Huang Date: Tue, 4 Aug 2026 17:38:38 +0800 Subject: zram: reject zero-size dictionary kernel_read_file_from_path() already rejects empty files (i_size <= 0) and returns -EINVAL, but the current implementation only checks for sz < 0 without logging any information. Use sz == 0 to reject the zero-size case and print distinct error messages for each failure type. Link: https://lore.kernel.org/20260804093841.67920-3-haoqinhuang7@gmail.com Signed-off-by: Haoqin Huang Signed-off-by: Rongwei Wang Reviewed-by: Sergey Senozhatsky Tested-by: Sergey Senozhatsky Cc: David Sterba Cc: Jens Axboe Cc: Minchan Kim Cc: Nick Terrell Signed-off-by: Andrew Morton --- drivers/block/zram/zram_drv.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index cfa98846ac48..f73e30b61067 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -1700,8 +1700,16 @@ static int comp_params_store(struct zram *zram, u32 prio, s32 level, INT_MAX, NULL, READING_POLICY); - if (sz < 0) + if (sz < 0) { + pr_err("failed to load dictionary %s (err=%zd)\n", + dict_path, sz); + return sz; + } + if (sz == 0) { + pr_err("failed to load dictionary %s (empty file)\n", + dict_path); return -EINVAL; + } } zram->params[prio].dict_sz = sz; -- cgit v1.2.3 From 70922d5ef84a5863ac80d4b13f574cb9e461a716 Mon Sep 17 00:00:00 2001 From: Haoqin Huang Date: Tue, 4 Aug 2026 17:38:39 +0800 Subject: zram: add pr_fmt to backend files Add pr_fmt to each backend so that pr_err() messages are auto-prefixed with the algorithm name. While at it, tweak the deflate winbits pr_err to avoid a duplicated "deflate" prefix. Link: https://lore.kernel.org/20260804093841.67920-4-haoqinhuang7@gmail.com Signed-off-by: Haoqin Huang Signed-off-by: Rongwei Wang Reviewed-by: Sergey Senozhatsky Tested-by: Sergey Senozhatsky Cc: David Sterba Cc: Jens Axboe Cc: Minchan Kim Cc: Nick Terrell Signed-off-by: Andrew Morton --- drivers/block/zram/backend_842.c | 2 ++ drivers/block/zram/backend_deflate.c | 4 +++- drivers/block/zram/backend_lz4.c | 4 ++++ drivers/block/zram/backend_lz4hc.c | 4 ++++ drivers/block/zram/backend_lzo.c | 2 ++ drivers/block/zram/backend_lzorle.c | 2 ++ drivers/block/zram/backend_zstd.c | 2 ++ 7 files changed, 19 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/block/zram/backend_842.c b/drivers/block/zram/backend_842.c index 10d9d5c60f53..d9b8a6bba2cb 100644 --- a/drivers/block/zram/backend_842.c +++ b/drivers/block/zram/backend_842.c @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later +#define pr_fmt(fmt) "842: " fmt + #include #include #include diff --git a/drivers/block/zram/backend_deflate.c b/drivers/block/zram/backend_deflate.c index b3f7d08b49d9..ee26e6c9282f 100644 --- a/drivers/block/zram/backend_deflate.c +++ b/drivers/block/zram/backend_deflate.c @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later +#define pr_fmt(fmt) "deflate: " fmt + #include #include #include @@ -30,7 +32,7 @@ static int deflate_setup_params(struct zcomp_params *params) s32 wb = params->deflate.winbits; if ((wb < -15 || wb > -9) && (wb < 9 || wb > 15)) { - pr_err("invalid deflate winbits: %d\n", wb); + pr_err("invalid winbits %d\n", wb); return -EINVAL; } } diff --git a/drivers/block/zram/backend_lz4.c b/drivers/block/zram/backend_lz4.c index c449d511ba86..6d58956ed5b2 100644 --- a/drivers/block/zram/backend_lz4.c +++ b/drivers/block/zram/backend_lz4.c @@ -1,3 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +#define pr_fmt(fmt) "lz4: " fmt + #include #include #include diff --git a/drivers/block/zram/backend_lz4hc.c b/drivers/block/zram/backend_lz4hc.c index f6a336acfe20..c0c3715087c8 100644 --- a/drivers/block/zram/backend_lz4hc.c +++ b/drivers/block/zram/backend_lz4hc.c @@ -1,3 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +#define pr_fmt(fmt) "lz4hc: " fmt + #include #include #include diff --git a/drivers/block/zram/backend_lzo.c b/drivers/block/zram/backend_lzo.c index 4c906beaae6b..84330dea6af5 100644 --- a/drivers/block/zram/backend_lzo.c +++ b/drivers/block/zram/backend_lzo.c @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later +#define pr_fmt(fmt) "lzo: " fmt + #include #include #include diff --git a/drivers/block/zram/backend_lzorle.c b/drivers/block/zram/backend_lzorle.c index 10640c96cbfc..b3b03a008b64 100644 --- a/drivers/block/zram/backend_lzorle.c +++ b/drivers/block/zram/backend_lzorle.c @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later +#define pr_fmt(fmt) "lzo-rle: " fmt + #include #include #include diff --git a/drivers/block/zram/backend_zstd.c b/drivers/block/zram/backend_zstd.c index 5fabc3e7e975..fb61acdaef67 100644 --- a/drivers/block/zram/backend_zstd.c +++ b/drivers/block/zram/backend_zstd.c @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later +#define pr_fmt(fmt) "zstd: " fmt + #include #include #include -- cgit v1.2.3 From 7b0f677c7bd539bd5695b14f9a195e257c3b4463 Mon Sep 17 00:00:00 2001 From: Haoqin Huang Date: Tue, 4 Aug 2026 17:38:40 +0800 Subject: zram: validate parameters in each backend's setup_params Dict and level parameters are silently accepted even for backends that do not support them. Validate these parameters in each backend's .setup_params() to reject unsupported combinations and out-of-range levels with a specific error message. Link: https://lore.kernel.org/20260804093841.67920-5-haoqinhuang7@gmail.com Signed-off-by: Haoqin Huang Signed-off-by: Rongwei Wang Reviewed-by: Sergey Senozhatsky Tested-by: Sergey Senozhatsky Cc: David Sterba Cc: Jens Axboe Cc: Minchan Kim Cc: Nick Terrell Signed-off-by: Andrew Morton --- drivers/block/zram/backend_842.c | 8 ++++++++ drivers/block/zram/backend_deflate.c | 13 ++++++++++++- drivers/block/zram/backend_lz4.c | 6 +++++- drivers/block/zram/backend_lz4hc.c | 12 +++++++++++- drivers/block/zram/backend_lzo.c | 8 ++++++++ drivers/block/zram/backend_lzorle.c | 8 ++++++++ drivers/block/zram/backend_zstd.c | 7 ++++++- 7 files changed, 58 insertions(+), 4 deletions(-) (limited to 'drivers') diff --git a/drivers/block/zram/backend_842.c b/drivers/block/zram/backend_842.c index d9b8a6bba2cb..3846a04c69d7 100644 --- a/drivers/block/zram/backend_842.c +++ b/drivers/block/zram/backend_842.c @@ -15,6 +15,14 @@ static void release_params_842(struct zcomp_params *params) static int setup_params_842(struct zcomp_params *params) { + if (params->dict_sz) { + pr_err("dictionary is not supported\n"); + return -EOPNOTSUPP; + } + if (params->level != ZCOMP_PARAM_NOT_SET) { + pr_err("compression level is not supported\n"); + return -EOPNOTSUPP; + } return 0; } diff --git a/drivers/block/zram/backend_deflate.c b/drivers/block/zram/backend_deflate.c index ee26e6c9282f..f71b11bcac78 100644 --- a/drivers/block/zram/backend_deflate.c +++ b/drivers/block/zram/backend_deflate.c @@ -24,8 +24,19 @@ static void deflate_release_params(struct zcomp_params *params) static int deflate_setup_params(struct zcomp_params *params) { - if (params->level == ZCOMP_PARAM_NOT_SET) + if (params->dict_sz) { + pr_err("dictionary is not supported\n"); + return -EOPNOTSUPP; + } + + if (params->level == ZCOMP_PARAM_NOT_SET) { params->level = Z_DEFAULT_COMPRESSION; + } else if (params->level < Z_DEFAULT_COMPRESSION || + params->level > Z_BEST_COMPRESSION) { + pr_err("invalid compression level %d\n", params->level); + return -EINVAL; + } + if (params->deflate.winbits == ZCOMP_PARAM_NOT_SET) { params->deflate.winbits = DEFLATE_DEF_WINBITS; } else { diff --git a/drivers/block/zram/backend_lz4.c b/drivers/block/zram/backend_lz4.c index 6d58956ed5b2..1e28104ad964 100644 --- a/drivers/block/zram/backend_lz4.c +++ b/drivers/block/zram/backend_lz4.c @@ -32,8 +32,12 @@ static int lz4_setup_params(struct zcomp_params *params) LZ4_stream_t *dict_stream; int ret; - if (params->level == ZCOMP_PARAM_NOT_SET) + if (params->level == ZCOMP_PARAM_NOT_SET) { params->level = LZ4_ACCELERATION_DEFAULT; + } else if (params->level < LZ4_ACCELERATION_DEFAULT) { + pr_err("invalid compression level %d\n", params->level); + return -EINVAL; + } if (!params->dict || !params->dict_sz) return 0; diff --git a/drivers/block/zram/backend_lz4hc.c b/drivers/block/zram/backend_lz4hc.c index c0c3715087c8..d8aa01bb258f 100644 --- a/drivers/block/zram/backend_lz4hc.c +++ b/drivers/block/zram/backend_lz4hc.c @@ -22,8 +22,18 @@ static void lz4hc_release_params(struct zcomp_params *params) static int lz4hc_setup_params(struct zcomp_params *params) { - if (params->level == ZCOMP_PARAM_NOT_SET) + if (params->level == ZCOMP_PARAM_NOT_SET) { params->level = LZ4HC_DEFAULT_CLEVEL; + } else if (params->level < 1 || params->level > LZ4HC_MAX_CLEVEL) { + /* + * Use < 1 rather than < LZ4HC_MIN_CLEVEL here because + * LZ4HC_compress_generic() only clamps levels below 1 + * (levels 1 and 2 are valid). LZ4HC_MIN_CLEVEL (3) is + * advisory and not enforced by the library. + */ + pr_err("invalid compression level %d\n", params->level); + return -EINVAL; + } return 0; } diff --git a/drivers/block/zram/backend_lzo.c b/drivers/block/zram/backend_lzo.c index 84330dea6af5..d83f92cf757c 100644 --- a/drivers/block/zram/backend_lzo.c +++ b/drivers/block/zram/backend_lzo.c @@ -14,6 +14,14 @@ static void lzo_release_params(struct zcomp_params *params) static int lzo_setup_params(struct zcomp_params *params) { + if (params->dict_sz) { + pr_err("dictionary is not supported\n"); + return -EOPNOTSUPP; + } + if (params->level != ZCOMP_PARAM_NOT_SET) { + pr_err("compression level is not supported\n"); + return -EOPNOTSUPP; + } return 0; } diff --git a/drivers/block/zram/backend_lzorle.c b/drivers/block/zram/backend_lzorle.c index b3b03a008b64..1b120d062c92 100644 --- a/drivers/block/zram/backend_lzorle.c +++ b/drivers/block/zram/backend_lzorle.c @@ -14,6 +14,14 @@ static void lzorle_release_params(struct zcomp_params *params) static int lzorle_setup_params(struct zcomp_params *params) { + if (params->dict_sz) { + pr_err("dictionary is not supported\n"); + return -EOPNOTSUPP; + } + if (params->level != ZCOMP_PARAM_NOT_SET) { + pr_err("compression level is not supported\n"); + return -EOPNOTSUPP; + } return 0; } diff --git a/drivers/block/zram/backend_zstd.c b/drivers/block/zram/backend_zstd.c index fb61acdaef67..08da3810cffd 100644 --- a/drivers/block/zram/backend_zstd.c +++ b/drivers/block/zram/backend_zstd.c @@ -60,8 +60,13 @@ static int zstd_setup_params(struct zcomp_params *params) return -ENOMEM; params->drv_data = zp; - if (params->level == ZCOMP_PARAM_NOT_SET) + if (params->level == ZCOMP_PARAM_NOT_SET) { params->level = zstd_default_clevel(); + } else if (params->level < zstd_min_clevel() || + params->level > zstd_max_clevel()) { + pr_err("invalid compression level %d\n", params->level); + goto error; + } zp->cprm = zstd_get_params(params->level, PAGE_SIZE); -- cgit v1.2.3 From 702c5a799db20e49fe67cdfa27bac65374ad00ab Mon Sep 17 00:00:00 2001 From: Haoqin Huang Date: Tue, 4 Aug 2026 17:38:41 +0800 Subject: zram: reset per-priority params when changing algorithm before init Parameters validated against one algorithm may be invalid for another (e.g. lz4 accepts level=65535 but zstd does not). Although algorithm changes are blocked after disksize is set, they are allowed before device initialization. Reset per-priority params on algorithm change so that stale parameters do not silently carry over. Link: https://lore.kernel.org/20260804093841.67920-6-haoqinhuang7@gmail.com Signed-off-by: Haoqin Huang Signed-off-by: Rongwei Wang Reviewed-by: Sergey Senozhatsky Tested-by: Sergey Senozhatsky Cc: David Sterba Cc: Jens Axboe Cc: Minchan Kim Cc: Nick Terrell Signed-off-by: Andrew Morton --- drivers/block/zram/zram_drv.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) (limited to 'drivers') diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index f73e30b61067..56183c827e1b 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -1652,6 +1652,17 @@ static void comp_algorithm_set(struct zram *zram, u32 prio, const char *alg) zram->comp_algs[prio] = alg; } +static void comp_params_reset(struct zram *zram, u32 prio) +{ + struct zcomp_params *params = &zram->params[prio]; + + vfree(params->dict); + params->level = ZCOMP_PARAM_NOT_SET; + params->deflate.winbits = ZCOMP_PARAM_NOT_SET; + params->dict_sz = 0; + params->dict = NULL; +} + static int __comp_algorithm_store(struct zram *zram, u32 prio, const char *buf) { const char *alg; @@ -1672,20 +1683,10 @@ static int __comp_algorithm_store(struct zram *zram, u32 prio, const char *buf) } comp_algorithm_set(zram, prio, alg); + comp_params_reset(zram, prio); return 0; } -static void comp_params_reset(struct zram *zram, u32 prio) -{ - struct zcomp_params *params = &zram->params[prio]; - - vfree(params->dict); - params->level = ZCOMP_PARAM_NOT_SET; - params->deflate.winbits = ZCOMP_PARAM_NOT_SET; - params->dict_sz = 0; - params->dict = NULL; -} - static int comp_params_store(struct zram *zram, u32 prio, s32 level, const char *dict_path, struct deflate_params *deflate_params) -- cgit v1.2.3 From 894913e2d35c46ff19a77530907771ae57862b96 Mon Sep 17 00:00:00 2001 From: Longlong Xia Date: Tue, 4 Aug 2026 14:59:18 +0800 Subject: zram: fix out-of-bounds access in writeback_store() Patch series "zram: fix stale scan bounds after reinitialization". Both writeback_store() and read_block_state() derive their table scan bounds from zram->disksize before acquiring dev_lock. If the device is reset and reinitialized with a smaller disksize between that read and lock acquisition, the bound can describe the old table while the scan operates on the new one. This can lead to out-of-bounds slot accesses. Move both bound calculations under dev_lock so each bound remains consistent with the table throughout its scan. Keep the fixes separate because the affected interfaces originate from different commits and can be backported independently. This patch (of 2): writeback_store() calculates the table scan bounds before taking dev_lock. A reset followed by reconfiguration with a smaller disksize can therefore replace zram->table while writeback_store() is waiting for the lock. Once it acquires the lock, it sees an initialized device but scans the new table using the old upper bound, resulting in an out-of-bounds access. Calculate the number of pages while holding dev_lock so the scan bound matches the table protected by the lock. Link: https://lore.kernel.org/20260804065919.3970386-1-xialonglong2025@163.com Link: https://lore.kernel.org/20260804065919.3970386-2-xialonglong2025@163.com Fixes: a939888ec38b ("zram: support idle/huge page writeback") Assisted-by: Codex:gpt-5.6-sol Signed-off-by: Longlong Xia Reviewed-by: Sergey Senozhatsky Cc: Jens Axboe Cc: Minchan Kim Cc: Signed-off-by: Andrew Morton --- drivers/block/zram/zram_drv.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index 56183c827e1b..2be5c20e3f14 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -1235,8 +1235,8 @@ static ssize_t writeback_store(struct device *dev, const char *buf, size_t len) { struct zram *zram = dev_to_zram(dev); - u64 nr_pages = zram->disksize >> PAGE_SHIFT; - unsigned long lo = 0, hi = nr_pages; + u64 nr_pages; + unsigned long lo = 0, hi; struct zram_pp_ctl *pp_ctl = NULL; struct zram_wb_ctl *wb_ctl = NULL; char *args, *param, *val; @@ -1250,6 +1250,9 @@ static ssize_t writeback_store(struct device *dev, if (!zram->backing_dev) return -ENODEV; + nr_pages = zram->disksize >> PAGE_SHIFT; + hi = nr_pages; + pp_ctl = init_pp_ctl(); if (!pp_ctl) return -ENOMEM; -- cgit v1.2.3 From 391f057f44a51cc9418da5cba78b014324174264 Mon Sep 17 00:00:00 2001 From: Longlong Xia Date: Tue, 4 Aug 2026 14:59:19 +0800 Subject: zram: fix out-of-bounds access in read_block_state() read_block_state() calculates nr_pages before taking dev_lock. If the device is reset and reinitialized with a smaller disksize before lock acquisition, nr_pages still describes the old table. The subsequent loop can then call slot_lock() past the end of the newly allocated table. Read disksize after acquiring dev_lock and checking that the device is initialized. The read lock then keeps the table and its bound stable for the duration of the scan. Link: https://lore.kernel.org/20260804065919.3970386-3-xialonglong2025@163.com Fixes: c0265342bff4 ("zram: introduce zram memory tracking") Assisted-by: Codex:gpt-5.6-sol Signed-off-by: Longlong Xia Reviewed-by: Sergey Senozhatsky Cc: Jens Axboe Cc: Minchan Kim Cc: Signed-off-by: Andrew Morton --- drivers/block/zram/zram_drv.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index 2be5c20e3f14..82b78e6e55b2 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -1543,7 +1543,7 @@ static ssize_t read_block_state(struct file *file, char __user *buf, char *kbuf; ssize_t index, written = 0; struct zram *zram = file->private_data; - unsigned long nr_pages = zram->disksize >> PAGE_SHIFT; + unsigned long nr_pages; kbuf = kvmalloc(count, GFP_KERNEL); if (!kbuf) @@ -1555,6 +1555,8 @@ static ssize_t read_block_state(struct file *file, char __user *buf, return -EINVAL; } + nr_pages = zram->disksize >> PAGE_SHIFT; + for (index = *ppos; index < nr_pages; index++) { int copied; -- cgit v1.2.3 From 184bf187c45ba6c1141aa7fe10bf10f85d5a7634 Mon Sep 17 00:00:00 2001 From: Sergey Senozhatsky Date: Thu, 6 Aug 2026 12:16:32 +0900 Subject: zram: switch to unsigned long indexing zram has always used "unsigned int" for (page) index calculations, which unnecessarily limited max zram disksize. Switch to "unsigned long" and permit much larger zram devices. Link: https://lore.kernel.org/20260806031640.536615-1-senozhatsky@chromium.org Signed-off-by: Sergey Senozhatsky Suggested-by: Andrew Morton Co-developed-by: Longlong Xia Cc: Minchan Kim Signed-off-by: Andrew Morton --- drivers/block/zram/zram_drv.c | 137 ++++++++++++++++++++++++------------------ 1 file changed, 77 insertions(+), 60 deletions(-) (limited to 'drivers') diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index 82b78e6e55b2..d09fdca49cbd 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -56,7 +56,7 @@ static size_t huge_class_size; static const struct block_device_operations zram_devops; -static void slot_free(struct zram *zram, u32 index); +static void slot_free(struct zram *zram, unsigned long index); /* * entry locking rules: @@ -70,7 +70,7 @@ static void slot_free(struct zram *zram, u32 index); * 4) Use TRY lock variant when in atomic context * - must check return value and handle locking failers */ -static __must_check bool slot_trylock(struct zram *zram, u32 index) +static __must_check bool slot_trylock(struct zram *zram, unsigned long index) { unsigned long *lock = &zram->table[index].__lock; @@ -83,7 +83,7 @@ static __must_check bool slot_trylock(struct zram *zram, u32 index) return false; } -static void slot_lock(struct zram *zram, u32 index) +static void slot_lock(struct zram *zram, unsigned long index) { unsigned long *lock = &zram->table[index].__lock; @@ -92,7 +92,7 @@ static void slot_lock(struct zram *zram, u32 index) lock_acquired(&zram->table_lock_map, _RET_IP_); } -static void slot_unlock(struct zram *zram, u32 index) +static void slot_unlock(struct zram *zram, unsigned long index) { unsigned long *lock = &zram->table[index].__lock; @@ -110,55 +110,56 @@ static inline struct zram *dev_to_zram(struct device *dev) return (struct zram *)dev_to_disk(dev)->private_data; } -static unsigned long get_slot_handle(struct zram *zram, u32 index) +static unsigned long get_slot_handle(struct zram *zram, unsigned long index) { return zram->table[index].handle; } -static void set_slot_handle(struct zram *zram, u32 index, unsigned long handle) +static void set_slot_handle(struct zram *zram, unsigned long index, + unsigned long handle) { zram->table[index].handle = handle; } -static bool test_slot_flag(struct zram *zram, u32 index, +static bool test_slot_flag(struct zram *zram, unsigned long index, enum zram_pageflags flag) { return zram->table[index].attr.flags & BIT(flag); } -static void set_slot_flag(struct zram *zram, u32 index, +static void set_slot_flag(struct zram *zram, unsigned long index, enum zram_pageflags flag) { zram->table[index].attr.flags |= BIT(flag); } -static void clear_slot_flag(struct zram *zram, u32 index, +static void clear_slot_flag(struct zram *zram, unsigned long index, enum zram_pageflags flag) { zram->table[index].attr.flags &= ~BIT(flag); } -static size_t get_slot_size(struct zram *zram, u32 index) +static size_t get_slot_size(struct zram *zram, unsigned long index) { return zram->table[index].attr.flags & (BIT(ZRAM_FLAG_SHIFT) - 1); } -static void set_slot_size(struct zram *zram, u32 index, size_t size) +static void set_slot_size(struct zram *zram, unsigned long index, size_t size) { unsigned long flags = zram->table[index].attr.flags >> ZRAM_FLAG_SHIFT; zram->table[index].attr.flags = (flags << ZRAM_FLAG_SHIFT) | size; } -static inline bool slot_allocated(struct zram *zram, u32 index) +static inline bool slot_allocated(struct zram *zram, unsigned long index) { return get_slot_size(zram, index) || test_slot_flag(zram, index, ZRAM_SAME) || test_slot_flag(zram, index, ZRAM_WB); } -static inline void set_slot_comp_priority(struct zram *zram, u32 index, - u32 prio) +static inline void set_slot_comp_priority(struct zram *zram, + unsigned long index, u32 prio) { prio &= ZRAM_COMP_PRIORITY_MASK; /* @@ -170,14 +171,14 @@ static inline void set_slot_comp_priority(struct zram *zram, u32 index, zram->table[index].attr.flags |= (prio << ZRAM_COMP_PRIORITY_BIT1); } -static inline u32 get_slot_comp_priority(struct zram *zram, u32 index) +static inline u32 get_slot_comp_priority(struct zram *zram, unsigned long index) { u32 prio = zram->table[index].attr.flags >> ZRAM_COMP_PRIORITY_BIT1; return prio & ZRAM_COMP_PRIORITY_MASK; } -static void mark_slot_accessed(struct zram *zram, u32 index) +static void mark_slot_accessed(struct zram *zram, unsigned long index) { clear_slot_flag(zram, index, ZRAM_IDLE); clear_slot_flag(zram, index, ZRAM_PP_SLOT); @@ -284,7 +285,7 @@ static void release_pp_ctl(struct zram *zram, struct zram_pp_ctl *ctl) } static bool place_pp_slot(struct zram *zram, struct zram_pp_ctl *ctl, - u32 index) + unsigned long index) { struct zram_pp_slot *pps; u32 bid; @@ -418,7 +419,7 @@ static void mark_idle(struct zram *zram, ktime_t cutoff) { int is_idle = 1; unsigned long nr_pages = zram->disksize >> PAGE_SHIFT; - int index; + unsigned long index; for (index = 0; index < nr_pages; index++) { /* @@ -485,8 +486,9 @@ static ssize_t idle_store(struct device *dev, struct device_attribute *attr, #define INVALID_BDEV_BLOCK (~0UL) static int read_from_zspool_raw(struct zram *zram, struct page *page, - u32 index); -static int read_from_zspool(struct zram *zram, struct page *page, u32 index); + unsigned long index); +static int read_from_zspool(struct zram *zram, struct page *page, + unsigned long index); struct zram_wb_ctl { /* idle list is accessed only by the writeback task, no concurency */ @@ -522,7 +524,7 @@ struct zram_rb_req { /* error status (sync read) */ int error; }; - u32 index; + unsigned long index; }; #define FOUR_K(x) ((x) * (1 << (PAGE_SHIFT - 12))) @@ -910,7 +912,7 @@ static void zram_account_writeback_submit(struct zram *zram) static int zram_writeback_complete(struct zram *zram, struct zram_wb_req *req) { - u32 index = req->pps->index; + unsigned long index = req->pps->index; int err; err = blk_status_to_errno(req->bio.bi_status); @@ -1032,7 +1034,7 @@ static int zram_writeback_slots(struct zram *zram, struct zram_wb_req *req = NULL; struct zram_pp_slot *pps; int ret = 0, err = 0; - u32 index = 0; + unsigned long index = 0; while ((pps = select_pp_slot(ctl))) { if (zram->wb_limit_enable && !zram->bd_wb_limit) { @@ -1198,7 +1200,7 @@ static void scan_slots_for_writeback(struct zram *zram, u32 mode, unsigned long lo, unsigned long hi, struct zram_pp_ctl *ctl) { - u32 index = lo; + unsigned long index = lo; while (index < hi) { bool ok = true; @@ -1235,7 +1237,7 @@ static ssize_t writeback_store(struct device *dev, const char *buf, size_t len) { struct zram *zram = dev_to_zram(dev); - u64 nr_pages; + unsigned long nr_pages; unsigned long lo = 0, hi; struct zram_pp_ctl *pp_ctl = NULL; struct zram_wb_ctl *wb_ctl = NULL; @@ -1336,7 +1338,8 @@ out: return ret; } -static int decompress_bdev_page(struct zram *zram, struct page *page, u32 index) +static int decompress_bdev_page(struct zram *zram, struct page *page, + unsigned long index) { struct zcomp_strm *zstrm; unsigned int size; @@ -1378,7 +1381,7 @@ static void zram_deferred_decompress(struct work_struct *w) struct zram_rb_req *req = container_of(w, struct zram_rb_req, work); struct page *page = bio_first_page_all(req->bio); struct zram *zram = req->zram; - u32 index = req->index; + unsigned long index = req->index; int ret; ret = decompress_bdev_page(zram, page, index); @@ -1429,7 +1432,7 @@ static void zram_async_read_endio(struct bio *bio) } static int read_from_bdev_async(struct zram *zram, struct page *page, - u32 index, unsigned long blk_idx, + unsigned long index, unsigned long blk_idx, struct bio *parent) { struct zram_rb_req *req; @@ -1479,8 +1482,8 @@ static void zram_sync_read(struct work_struct *w) * chained IO with parent IO in same context, it's a deadlock. To avoid that, * use a worker thread context. */ -static int read_from_bdev_sync(struct zram *zram, struct page *page, u32 index, - unsigned long blk_idx) +static int read_from_bdev_sync(struct zram *zram, struct page *page, + unsigned long index, unsigned long blk_idx) { struct zram_rb_req req; @@ -1499,8 +1502,9 @@ static int read_from_bdev_sync(struct zram *zram, struct page *page, u32 index, return decompress_bdev_page(zram, page, index); } -static int read_from_bdev(struct zram *zram, struct page *page, u32 index, - unsigned long blk_idx, struct bio *parent) +static int read_from_bdev(struct zram *zram, struct page *page, + unsigned long index, unsigned long blk_idx, + struct bio *parent) { atomic64_inc(&zram->stats.bd_reads); if (!parent) { @@ -1512,8 +1516,9 @@ static int read_from_bdev(struct zram *zram, struct page *page, u32 index, } #else static inline void reset_bdev(struct zram *zram) {}; -static int read_from_bdev(struct zram *zram, struct page *page, u32 index, - unsigned long blk_idx, struct bio *parent) +static int read_from_bdev(struct zram *zram, struct page *page, + unsigned long index, unsigned long blk_idx, + struct bio *parent) { return -EIO; } @@ -1541,7 +1546,8 @@ static ssize_t read_block_state(struct file *file, char __user *buf, size_t count, loff_t *ppos) { char *kbuf; - ssize_t index, written = 0; + unsigned long index; + ssize_t written = 0; struct zram *zram = file->private_data; unsigned long nr_pages; @@ -1565,7 +1571,7 @@ static ssize_t read_block_state(struct file *file, char __user *buf, goto next; copied = snprintf(kbuf + written, count, - "%12zd %12u.%06d %c%c%c%c%c%c\n", + "%12lu %12u.%06d %c%c%c%c%c%c\n", index, zram->table[index].attr.ac_time, 0, test_slot_flag(zram, index, ZRAM_SAME) ? 's' : '.', test_slot_flag(zram, index, ZRAM_WB) ? 'w' : '.', @@ -1972,8 +1978,8 @@ static ssize_t debug_stat_show(struct device *dev, static void zram_meta_free(struct zram *zram, u64 disksize) { - size_t num_pages = disksize >> PAGE_SHIFT; - size_t index; + unsigned long num_pages = disksize >> PAGE_SHIFT; + unsigned long index; if (!zram->table) return; @@ -1990,7 +1996,7 @@ static void zram_meta_free(struct zram *zram, u64 disksize) static bool zram_meta_alloc(struct zram *zram, u64 disksize) { - size_t num_pages; + unsigned long num_pages; num_pages = disksize >> PAGE_SHIFT; zram->table = vzalloc(array_size(num_pages, sizeof(*zram->table))); @@ -2013,7 +2019,7 @@ static bool zram_meta_alloc(struct zram *zram, u64 disksize) return true; } -static void slot_free(struct zram *zram, u32 index) +static void slot_free(struct zram *zram, unsigned long index) { unsigned long handle; @@ -2067,7 +2073,7 @@ out: } static int read_same_filled_page(struct zram *zram, struct page *page, - u32 index) + unsigned long index) { void *mem; @@ -2078,7 +2084,7 @@ static int read_same_filled_page(struct zram *zram, struct page *page, } static int read_incompressible_page(struct zram *zram, struct page *page, - u32 index) + unsigned long index) { unsigned long handle; void *src, *dst; @@ -2093,7 +2099,8 @@ static int read_incompressible_page(struct zram *zram, struct page *page, return 0; } -static int read_compressed_page(struct zram *zram, struct page *page, u32 index) +static int read_compressed_page(struct zram *zram, struct page *page, + unsigned long index) { struct zcomp_strm *zstrm; unsigned long handle; @@ -2118,7 +2125,8 @@ static int read_compressed_page(struct zram *zram, struct page *page, u32 index) } #if defined CONFIG_ZRAM_WRITEBACK -static int read_from_zspool_raw(struct zram *zram, struct page *page, u32 index) +static int read_from_zspool_raw(struct zram *zram, struct page *page, + unsigned long index) { struct zcomp_strm *zstrm; unsigned long handle; @@ -2150,7 +2158,8 @@ static int read_from_zspool_raw(struct zram *zram, struct page *page, u32 index) * Reads (decompresses if needed) a page from zspool (zsmalloc). * Corresponding ZRAM slot should be locked. */ -static int read_from_zspool(struct zram *zram, struct page *page, u32 index) +static int read_from_zspool(struct zram *zram, struct page *page, + unsigned long index) { if (test_slot_flag(zram, index, ZRAM_SAME) || !get_slot_handle(zram, index)) @@ -2162,8 +2171,8 @@ static int read_from_zspool(struct zram *zram, struct page *page, u32 index) return read_incompressible_page(zram, page, index); } -static int zram_read_page(struct zram *zram, struct page *page, u32 index, - struct bio *parent) +static int zram_read_page(struct zram *zram, struct page *page, + unsigned long index, struct bio *parent) { int ret; @@ -2185,7 +2194,7 @@ static int zram_read_page(struct zram *zram, struct page *page, u32 index, /* Should NEVER happen. Return bio error if it does. */ if (WARN_ON(ret < 0)) - pr_err("Decompression failed! err=%d, page=%u\n", ret, index); + pr_err("Decompression failed! err=%d, page=%lu\n", ret, index); return ret; } @@ -2195,7 +2204,7 @@ static int zram_read_page(struct zram *zram, struct page *page, u32 index, * always expects a full page for the output. */ static int zram_bvec_read_partial(struct zram *zram, struct bio_vec *bvec, - u32 index, int offset) + unsigned long index, int offset) { struct page *page = alloc_page(GFP_NOIO); int ret; @@ -2210,7 +2219,7 @@ static int zram_bvec_read_partial(struct zram *zram, struct bio_vec *bvec, } static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec, - u32 index, int offset, struct bio *bio) + unsigned long index, int offset, struct bio *bio) { if (is_partial_io(bvec)) return zram_bvec_read_partial(zram, bvec, index, offset); @@ -2218,7 +2227,7 @@ static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec, } static int write_same_filled_page(struct zram *zram, unsigned long fill, - u32 index) + unsigned long index) { slot_lock(zram, index); slot_free(zram, index); @@ -2233,7 +2242,7 @@ static int write_same_filled_page(struct zram *zram, unsigned long fill, } static int write_incompressible_page(struct zram *zram, struct page *page, - u32 index) + unsigned long index) { unsigned long handle; void *src; @@ -2273,7 +2282,8 @@ static int write_incompressible_page(struct zram *zram, struct page *page, return 0; } -static int zram_write_page(struct zram *zram, struct page *page, u32 index) +static int zram_write_page(struct zram *zram, struct page *page, + unsigned long index) { int ret = 0; unsigned long handle; @@ -2340,7 +2350,7 @@ static int zram_write_page(struct zram *zram, struct page *page, u32 index) * This is a partial IO. Read the full page before writing the changes. */ static int zram_bvec_write_partial(struct zram *zram, struct bio_vec *bvec, - u32 index, int offset) + unsigned long index, int offset) { struct page *page = alloc_page(GFP_NOIO); int ret; @@ -2358,7 +2368,7 @@ static int zram_bvec_write_partial(struct zram *zram, struct bio_vec *bvec, } static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, - u32 index, int offset) + unsigned long index, int offset) { if (is_partial_io(bvec)) return zram_bvec_write_partial(zram, bvec, index, offset); @@ -2426,8 +2436,9 @@ next: * * Corresponding ZRAM slot should be locked. */ -static int recompress_slot(struct zram *zram, u32 index, struct page *page, - u64 *num_recomp_pages, u32 threshold, u32 prio) +static int recompress_slot(struct zram *zram, unsigned long index, + struct page *page, u64 *num_recomp_pages, + u32 threshold, u32 prio) { struct zcomp_strm *zstrm = NULL; unsigned long handle_old; @@ -2679,7 +2690,7 @@ out: static void zram_bio_discard(struct zram *zram, struct bio *bio) { size_t n = bio->bi_iter.bi_size; - u32 index = bio->bi_iter.bi_sector >> SECTORS_PER_PAGE_SHIFT; + unsigned long index = bio->bi_iter.bi_sector >> SECTORS_PER_PAGE_SHIFT; u32 offset = (bio->bi_iter.bi_sector & (SECTORS_PER_PAGE - 1)) << SECTOR_SHIFT; @@ -2720,7 +2731,7 @@ static void zram_bio_read(struct zram *zram, struct bio *bio) struct bvec_iter iter = bio->bi_iter; do { - u32 index = iter.bi_sector >> SECTORS_PER_PAGE_SHIFT; + unsigned long index = iter.bi_sector >> SECTORS_PER_PAGE_SHIFT; u32 offset = (iter.bi_sector & (SECTORS_PER_PAGE - 1)) << SECTOR_SHIFT; struct bio_vec bv = bio_iter_iovec(bio, iter); @@ -2751,7 +2762,7 @@ static void zram_bio_write(struct zram *zram, struct bio *bio) struct bvec_iter iter = bio->bi_iter; do { - u32 index = iter.bi_sector >> SECTORS_PER_PAGE_SHIFT; + unsigned long index = iter.bi_sector >> SECTORS_PER_PAGE_SHIFT; u32 offset = (iter.bi_sector & (SECTORS_PER_PAGE - 1)) << SECTOR_SHIFT; struct bio_vec bv = bio_iter_iovec(bio, iter); @@ -2865,6 +2876,7 @@ static void zram_reset_device(struct zram *zram) static ssize_t disksize_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t len) { + unsigned long num_pages; u64 disksize; struct zcomp *comp; struct zram *zram = dev_to_zram(dev); @@ -2882,6 +2894,11 @@ static ssize_t disksize_store(struct device *dev, struct device_attribute *attr, } disksize = PAGE_ALIGN(disksize); + num_pages = disksize >> PAGE_SHIFT; + /* Slots are addressed by an unsigned long index */ + if (!num_pages || ((u64)num_pages << PAGE_SHIFT) != disksize) + return -EINVAL; + if (!zram_meta_alloc(zram, disksize)) return -ENOMEM; -- cgit v1.2.3 From 3774c56cc38b9ddcdf46dd717d4de954b40d8986 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Thu, 6 Aug 2026 09:58:38 +0300 Subject: drivers/base, mm: move arch_numa.c to mm/ arch_numa.c implements boot time discovery and initialization of NUMA topology on architectures that select GENERIC_ARCH_NUMA (currently arm64 and riscv). Since this is step in the initialization of the memory management subsystem, it's logical to have arch_numa.c in mm/ alongside numa.c, numa_memblks.c and numa_emulation.c. Move arch_numa.c to mm/ and add its F: entry to "MEMBLOCK AND MEMORY MANAGEMENT INITIALIZATION" in MAINTAINERS. Link: https://lore.kernel.org/20260806-arch-numa-v1-1-968ec128121e@kernel.org Signed-off-by: Mike Rapoport (Microsoft) Acked-by: Greg Kroah-Hartman Acked-by: Lorenzo Stoakes (ARM) Acked-by: Vlastimil Babka (SUSE) Acked-by: David Hildenbrand (Arm) Acked-by: Danilo Krummrich Cc: Albert Ou Cc: Alexandre Ghiti Cc: Catalin Marinas Cc: Liam R. Howlett Cc: Michal Hocko Cc: Palmer Dabbelt Cc: "Rafael J. Wysocki" Cc: Suren Baghdasaryan Cc: Will Deacon Signed-off-by: Andrew Morton --- MAINTAINERS | 1 + drivers/base/Kconfig | 7 - drivers/base/Makefile | 1 - drivers/base/arch_numa.c | 375 ----------------------------------------------- mm/Kconfig | 7 + mm/Makefile | 1 + mm/arch_numa.c | 375 +++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 384 insertions(+), 383 deletions(-) delete mode 100644 drivers/base/arch_numa.c create mode 100644 mm/arch_numa.c (limited to 'drivers') diff --git a/MAINTAINERS b/MAINTAINERS index e0ea1b915305..06271e742d32 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -16880,6 +16880,7 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/rppt/memblock.git fixes F: Documentation/core-api/boot-time-mm.rst F: include/linux/kho/abi/memblock.h F: include/linux/memblock.h +F: mm/arch_numa.c F: mm/memblock.c F: mm/memtest.c F: mm/mm_init.c diff --git a/drivers/base/Kconfig b/drivers/base/Kconfig index f7d385cbd3ba..1d93bfc34490 100644 --- a/drivers/base/Kconfig +++ b/drivers/base/Kconfig @@ -239,13 +239,6 @@ config GENERIC_ARCH_TOPOLOGY appropriate scaling, sysfs interface for reading capacity values at runtime. -config GENERIC_ARCH_NUMA - bool - select NUMA_MEMBLKS - help - Enable support for generic NUMA implementation. Currently, RISC-V - and ARM64 use it. - config FW_DEVLINK_SYNC_STATE_TIMEOUT bool "sync_state() behavior defaults to timeout instead of strict" help diff --git a/drivers/base/Makefile b/drivers/base/Makefile index 8074a10183dc..435710f643a5 100644 --- a/drivers/base/Makefile +++ b/drivers/base/Makefile @@ -25,7 +25,6 @@ obj-$(CONFIG_PINCTRL) += pinctrl.o obj-$(CONFIG_DEV_COREDUMP) += devcoredump.o obj-$(CONFIG_GENERIC_MSI_IRQ) += platform-msi.o obj-$(CONFIG_GENERIC_ARCH_TOPOLOGY) += arch_topology.o -obj-$(CONFIG_GENERIC_ARCH_NUMA) += arch_numa.o obj-$(CONFIG_ACPI) += physical_location.o obj-y += test/ diff --git a/drivers/base/arch_numa.c b/drivers/base/arch_numa.c deleted file mode 100644 index 442ea239bba7..000000000000 --- a/drivers/base/arch_numa.c +++ /dev/null @@ -1,375 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * NUMA support, based on the x86 implementation. - * - * Copyright (C) 2015 Cavium Inc. - * Author: Ganapatrao Kulkarni - */ - -#define pr_fmt(fmt) "NUMA: " fmt - -#include -#include -#include -#include -#include - -#include - -static int cpu_to_node_map[NR_CPUS] = { [0 ... NR_CPUS-1] = NUMA_NO_NODE }; - -bool numa_off; - -static __init int numa_parse_early_param(char *opt) -{ - if (!opt) - return -EINVAL; - if (str_has_prefix(opt, "off")) - numa_off = true; - if (!strncmp(opt, "fake=", 5)) - return numa_emu_cmdline(opt + 5); - - return 0; -} -early_param("numa", numa_parse_early_param); - -cpumask_var_t node_to_cpumask_map[MAX_NUMNODES]; -EXPORT_SYMBOL(node_to_cpumask_map); - -#ifdef CONFIG_DEBUG_PER_CPU_MAPS - -/* - * Returns a pointer to the bitmask of CPUs on Node 'node'. - */ -const struct cpumask *cpumask_of_node(int node) -{ - - if (node == NUMA_NO_NODE) - return cpu_all_mask; - - if (WARN_ON(node < 0 || node >= nr_node_ids)) - return cpu_none_mask; - - if (WARN_ON(node_to_cpumask_map[node] == NULL)) - return cpu_online_mask; - - return node_to_cpumask_map[node]; -} -EXPORT_SYMBOL(cpumask_of_node); - -#endif - -#ifndef CONFIG_NUMA_EMU -static void numa_update_cpu(unsigned int cpu, bool remove) -{ - int nid = cpu_to_node(cpu); - - if (nid == NUMA_NO_NODE) - return; - - if (remove) - cpumask_clear_cpu(cpu, node_to_cpumask_map[nid]); - else - cpumask_set_cpu(cpu, node_to_cpumask_map[nid]); -} - -void numa_add_cpu(unsigned int cpu) -{ - numa_update_cpu(cpu, false); -} - -void numa_remove_cpu(unsigned int cpu) -{ - numa_update_cpu(cpu, true); -} -#endif - -void numa_clear_node(unsigned int cpu) -{ - numa_remove_cpu(cpu); - set_cpu_numa_node(cpu, NUMA_NO_NODE); -} - -/* - * Allocate node_to_cpumask_map based on number of available nodes - * Requires node_possible_map to be valid. - * - * Note: cpumask_of_node() is not valid until after this is done. - * (Use CONFIG_DEBUG_PER_CPU_MAPS to check this.) - */ -static void __init setup_node_to_cpumask_map(void) -{ - int node; - - /* setup nr_node_ids if not done yet */ - if (nr_node_ids == MAX_NUMNODES) - setup_nr_node_ids(); - - /* allocate and clear the mapping */ - for (node = 0; node < nr_node_ids; node++) { - alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]); - cpumask_clear(node_to_cpumask_map[node]); - } - - /* cpumask_of_node() will now work */ - pr_debug("Node to cpumask map for %u nodes\n", nr_node_ids); -} - -/* - * Set the cpu to node and mem mapping - */ -void numa_store_cpu_info(unsigned int cpu) -{ - set_cpu_numa_node(cpu, cpu_to_node_map[cpu]); -} - -void __init early_map_cpu_to_node(unsigned int cpu, int nid) -{ - /* fallback to node 0 */ - if (nid < 0 || nid >= MAX_NUMNODES || numa_off) - nid = 0; - - cpu_to_node_map[cpu] = nid; - - /* - * We should set the numa node of cpu0 as soon as possible, because it - * has already been set up online before. cpu_to_node(0) will soon be - * called. - */ - if (!cpu) - set_cpu_numa_node(cpu, nid); -} - -#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA -unsigned long __per_cpu_offset[NR_CPUS] __read_mostly; -EXPORT_SYMBOL(__per_cpu_offset); - -int early_cpu_to_node(int cpu) -{ - return cpu_to_node_map[cpu]; -} - -static int __init pcpu_cpu_distance(unsigned int from, unsigned int to) -{ - return node_distance(early_cpu_to_node(from), early_cpu_to_node(to)); -} - -void __init setup_per_cpu_areas(void) -{ - unsigned long delta; - unsigned int cpu; - int rc = -EINVAL; - - if (pcpu_chosen_fc != PCPU_FC_PAGE) { - /* - * Always reserve area for module percpu variables. That's - * what the legacy allocator did. - */ - rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE, - PERCPU_DYNAMIC_RESERVE, PAGE_SIZE, - pcpu_cpu_distance, - early_cpu_to_node); -#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK - if (rc < 0) - pr_warn("PERCPU: %s allocator failed (%d), falling back to page size\n", - pcpu_fc_names[pcpu_chosen_fc], rc); -#endif - } - -#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK - if (rc < 0) - rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, early_cpu_to_node); -#endif - if (rc < 0) - panic("Failed to initialize percpu areas (err=%d).", rc); - - delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start; - for_each_possible_cpu(cpu) - __per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu]; -} -#endif - -/* - * Initialize NODE_DATA for a node on the local memory - */ -static void __init setup_node_data(int nid, u64 start_pfn, u64 end_pfn) -{ - if (start_pfn >= end_pfn) - pr_info("Initmem setup node %d []\n", nid); - - alloc_node_data(nid); - - NODE_DATA(nid)->node_id = nid; - NODE_DATA(nid)->node_start_pfn = start_pfn; - NODE_DATA(nid)->node_spanned_pages = end_pfn - start_pfn; -} - -static int __init numa_register_nodes(void) -{ - int nid; - - /* Check the validity of the memblock/node mapping */ - if (!memblock_validate_numa_coverage(0)) - return -EINVAL; - - /* Finally register nodes. */ - for_each_node_mask(nid, numa_nodes_parsed) { - unsigned long start_pfn, end_pfn; - - get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); - setup_node_data(nid, start_pfn, end_pfn); - node_set_online(nid); - } - - /* Setup online nodes to actual nodes*/ - node_possible_map = numa_nodes_parsed; - - return 0; -} - -static int __init numa_init(int (*init_func)(void)) -{ - int ret; - - ret = numa_memblks_init(init_func, /* memblock_force_top_down */ false); - if (ret < 0) - goto out_free_distance; - - if (nodes_empty(numa_nodes_parsed)) { - pr_info("No NUMA configuration found\n"); - ret = -EINVAL; - goto out_free_distance; - } - - ret = numa_register_nodes(); - if (ret < 0) - goto out_free_distance; - - setup_node_to_cpumask_map(); - - return 0; -out_free_distance: - numa_reset_distance(); - return ret; -} - -/** - * dummy_numa_init() - Fallback dummy NUMA init - * - * Used if there's no underlying NUMA architecture, NUMA initialization - * fails, or NUMA is disabled on the command line. - * - * Must online at least one node (node 0) and add memory blocks that cover all - * allowed memory. It is unlikely that this function fails. - * - * Return: 0 on success, -errno on failure. - */ -static int __init dummy_numa_init(void) -{ - phys_addr_t start = memblock_start_of_DRAM(); - phys_addr_t end = memblock_end_of_DRAM() - 1; - int ret; - - if (numa_off) - pr_info("NUMA disabled\n"); /* Forced off on command line. */ - pr_info("Faking a node at [mem %pap-%pap]\n", &start, &end); - - ret = numa_add_memblk(0, start, end + 1); - if (ret) { - pr_err("NUMA init failed\n"); - return ret; - } - node_set(0, numa_nodes_parsed); - - numa_off = true; - return 0; -} - -#ifdef CONFIG_ACPI_NUMA -static int __init arch_acpi_numa_init(void) -{ - int ret; - - ret = acpi_numa_init(); - if (ret) { - pr_debug("Failed to initialise from firmware\n"); - return ret; - } - - return srat_disabled() ? -EINVAL : 0; -} -#else -static int __init arch_acpi_numa_init(void) -{ - return -EOPNOTSUPP; -} -#endif - -/** - * arch_numa_init() - Initialize NUMA - * - * Try each configured NUMA initialization method until one succeeds. The - * last fallback is dummy single node config encompassing whole memory. - */ -void __init arch_numa_init(void) -{ - if (!numa_off) { - if (!acpi_disabled && !numa_init(arch_acpi_numa_init)) - return; - if (acpi_disabled && !numa_init(of_numa_init)) - return; - } - - numa_init(dummy_numa_init); -} - -#ifdef CONFIG_NUMA_EMU -void __init numa_emu_update_cpu_to_node(int *emu_nid_to_phys, - unsigned int nr_emu_nids) -{ - int i, j; - - /* - * Transform cpu_to_node_map table to use emulated nids by - * reverse-mapping phys_nid. The maps should always exist but fall - * back to zero just in case. - */ - for (i = 0; i < ARRAY_SIZE(cpu_to_node_map); i++) { - if (cpu_to_node_map[i] == NUMA_NO_NODE) - continue; - for (j = 0; j < nr_emu_nids; j++) - if (cpu_to_node_map[i] == emu_nid_to_phys[j]) - break; - cpu_to_node_map[i] = j < nr_emu_nids ? j : 0; - } -} - -u64 __init numa_emu_dma_end(void) -{ - return memblock_start_of_DRAM() + SZ_4G; -} - -void debug_cpumask_set_cpu(unsigned int cpu, int node, bool enable) -{ - struct cpumask *mask; - - if (node == NUMA_NO_NODE) - return; - - mask = node_to_cpumask_map[node]; - if (!cpumask_available(mask)) { - pr_err("node_to_cpumask_map[%i] NULL\n", node); - dump_stack(); - return; - } - - if (enable) - cpumask_set_cpu(cpu, mask); - else - cpumask_clear_cpu(cpu, mask); - - pr_debug("%s cpu %d node %d: mask now %*pbl\n", - enable ? "numa_add_cpu" : "numa_remove_cpu", - cpu, node, cpumask_pr_args(mask)); -} -#endif /* CONFIG_NUMA_EMU */ diff --git a/mm/Kconfig b/mm/Kconfig index 331daf7fcfab..8a24c130d008 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -1463,6 +1463,13 @@ config NUMA_EMU into virtual nodes when booted with "numa=fake=N", where N is the number of nodes. This is only useful for debugging. +config GENERIC_ARCH_NUMA + bool + select NUMA_MEMBLKS + help + Enable support for generic NUMA implementation. Currently, RISC-V + and ARM64 use it. + config ARCH_HAS_USER_SHADOW_STACK bool help diff --git a/mm/Makefile b/mm/Makefile index ab37ef428d98..e7245cb88c66 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -122,6 +122,7 @@ obj-$(CONFIG_CMA) += cma.o obj-$(CONFIG_NUMA) += numa.o obj-$(CONFIG_NUMA_MEMBLKS) += numa_memblks.o obj-$(CONFIG_NUMA_EMU) += numa_emulation.o +obj-$(CONFIG_GENERIC_ARCH_NUMA) += arch_numa.o obj-$(CONFIG_BALLOON) += balloon.o obj-$(CONFIG_PAGE_EXTENSION) += page_ext.o obj-$(CONFIG_PAGE_TABLE_CHECK) += page_table_check.o diff --git a/mm/arch_numa.c b/mm/arch_numa.c new file mode 100644 index 000000000000..442ea239bba7 --- /dev/null +++ b/mm/arch_numa.c @@ -0,0 +1,375 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * NUMA support, based on the x86 implementation. + * + * Copyright (C) 2015 Cavium Inc. + * Author: Ganapatrao Kulkarni + */ + +#define pr_fmt(fmt) "NUMA: " fmt + +#include +#include +#include +#include +#include + +#include + +static int cpu_to_node_map[NR_CPUS] = { [0 ... NR_CPUS-1] = NUMA_NO_NODE }; + +bool numa_off; + +static __init int numa_parse_early_param(char *opt) +{ + if (!opt) + return -EINVAL; + if (str_has_prefix(opt, "off")) + numa_off = true; + if (!strncmp(opt, "fake=", 5)) + return numa_emu_cmdline(opt + 5); + + return 0; +} +early_param("numa", numa_parse_early_param); + +cpumask_var_t node_to_cpumask_map[MAX_NUMNODES]; +EXPORT_SYMBOL(node_to_cpumask_map); + +#ifdef CONFIG_DEBUG_PER_CPU_MAPS + +/* + * Returns a pointer to the bitmask of CPUs on Node 'node'. + */ +const struct cpumask *cpumask_of_node(int node) +{ + + if (node == NUMA_NO_NODE) + return cpu_all_mask; + + if (WARN_ON(node < 0 || node >= nr_node_ids)) + return cpu_none_mask; + + if (WARN_ON(node_to_cpumask_map[node] == NULL)) + return cpu_online_mask; + + return node_to_cpumask_map[node]; +} +EXPORT_SYMBOL(cpumask_of_node); + +#endif + +#ifndef CONFIG_NUMA_EMU +static void numa_update_cpu(unsigned int cpu, bool remove) +{ + int nid = cpu_to_node(cpu); + + if (nid == NUMA_NO_NODE) + return; + + if (remove) + cpumask_clear_cpu(cpu, node_to_cpumask_map[nid]); + else + cpumask_set_cpu(cpu, node_to_cpumask_map[nid]); +} + +void numa_add_cpu(unsigned int cpu) +{ + numa_update_cpu(cpu, false); +} + +void numa_remove_cpu(unsigned int cpu) +{ + numa_update_cpu(cpu, true); +} +#endif + +void numa_clear_node(unsigned int cpu) +{ + numa_remove_cpu(cpu); + set_cpu_numa_node(cpu, NUMA_NO_NODE); +} + +/* + * Allocate node_to_cpumask_map based on number of available nodes + * Requires node_possible_map to be valid. + * + * Note: cpumask_of_node() is not valid until after this is done. + * (Use CONFIG_DEBUG_PER_CPU_MAPS to check this.) + */ +static void __init setup_node_to_cpumask_map(void) +{ + int node; + + /* setup nr_node_ids if not done yet */ + if (nr_node_ids == MAX_NUMNODES) + setup_nr_node_ids(); + + /* allocate and clear the mapping */ + for (node = 0; node < nr_node_ids; node++) { + alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]); + cpumask_clear(node_to_cpumask_map[node]); + } + + /* cpumask_of_node() will now work */ + pr_debug("Node to cpumask map for %u nodes\n", nr_node_ids); +} + +/* + * Set the cpu to node and mem mapping + */ +void numa_store_cpu_info(unsigned int cpu) +{ + set_cpu_numa_node(cpu, cpu_to_node_map[cpu]); +} + +void __init early_map_cpu_to_node(unsigned int cpu, int nid) +{ + /* fallback to node 0 */ + if (nid < 0 || nid >= MAX_NUMNODES || numa_off) + nid = 0; + + cpu_to_node_map[cpu] = nid; + + /* + * We should set the numa node of cpu0 as soon as possible, because it + * has already been set up online before. cpu_to_node(0) will soon be + * called. + */ + if (!cpu) + set_cpu_numa_node(cpu, nid); +} + +#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA +unsigned long __per_cpu_offset[NR_CPUS] __read_mostly; +EXPORT_SYMBOL(__per_cpu_offset); + +int early_cpu_to_node(int cpu) +{ + return cpu_to_node_map[cpu]; +} + +static int __init pcpu_cpu_distance(unsigned int from, unsigned int to) +{ + return node_distance(early_cpu_to_node(from), early_cpu_to_node(to)); +} + +void __init setup_per_cpu_areas(void) +{ + unsigned long delta; + unsigned int cpu; + int rc = -EINVAL; + + if (pcpu_chosen_fc != PCPU_FC_PAGE) { + /* + * Always reserve area for module percpu variables. That's + * what the legacy allocator did. + */ + rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE, + PERCPU_DYNAMIC_RESERVE, PAGE_SIZE, + pcpu_cpu_distance, + early_cpu_to_node); +#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK + if (rc < 0) + pr_warn("PERCPU: %s allocator failed (%d), falling back to page size\n", + pcpu_fc_names[pcpu_chosen_fc], rc); +#endif + } + +#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK + if (rc < 0) + rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, early_cpu_to_node); +#endif + if (rc < 0) + panic("Failed to initialize percpu areas (err=%d).", rc); + + delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start; + for_each_possible_cpu(cpu) + __per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu]; +} +#endif + +/* + * Initialize NODE_DATA for a node on the local memory + */ +static void __init setup_node_data(int nid, u64 start_pfn, u64 end_pfn) +{ + if (start_pfn >= end_pfn) + pr_info("Initmem setup node %d []\n", nid); + + alloc_node_data(nid); + + NODE_DATA(nid)->node_id = nid; + NODE_DATA(nid)->node_start_pfn = start_pfn; + NODE_DATA(nid)->node_spanned_pages = end_pfn - start_pfn; +} + +static int __init numa_register_nodes(void) +{ + int nid; + + /* Check the validity of the memblock/node mapping */ + if (!memblock_validate_numa_coverage(0)) + return -EINVAL; + + /* Finally register nodes. */ + for_each_node_mask(nid, numa_nodes_parsed) { + unsigned long start_pfn, end_pfn; + + get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); + setup_node_data(nid, start_pfn, end_pfn); + node_set_online(nid); + } + + /* Setup online nodes to actual nodes*/ + node_possible_map = numa_nodes_parsed; + + return 0; +} + +static int __init numa_init(int (*init_func)(void)) +{ + int ret; + + ret = numa_memblks_init(init_func, /* memblock_force_top_down */ false); + if (ret < 0) + goto out_free_distance; + + if (nodes_empty(numa_nodes_parsed)) { + pr_info("No NUMA configuration found\n"); + ret = -EINVAL; + goto out_free_distance; + } + + ret = numa_register_nodes(); + if (ret < 0) + goto out_free_distance; + + setup_node_to_cpumask_map(); + + return 0; +out_free_distance: + numa_reset_distance(); + return ret; +} + +/** + * dummy_numa_init() - Fallback dummy NUMA init + * + * Used if there's no underlying NUMA architecture, NUMA initialization + * fails, or NUMA is disabled on the command line. + * + * Must online at least one node (node 0) and add memory blocks that cover all + * allowed memory. It is unlikely that this function fails. + * + * Return: 0 on success, -errno on failure. + */ +static int __init dummy_numa_init(void) +{ + phys_addr_t start = memblock_start_of_DRAM(); + phys_addr_t end = memblock_end_of_DRAM() - 1; + int ret; + + if (numa_off) + pr_info("NUMA disabled\n"); /* Forced off on command line. */ + pr_info("Faking a node at [mem %pap-%pap]\n", &start, &end); + + ret = numa_add_memblk(0, start, end + 1); + if (ret) { + pr_err("NUMA init failed\n"); + return ret; + } + node_set(0, numa_nodes_parsed); + + numa_off = true; + return 0; +} + +#ifdef CONFIG_ACPI_NUMA +static int __init arch_acpi_numa_init(void) +{ + int ret; + + ret = acpi_numa_init(); + if (ret) { + pr_debug("Failed to initialise from firmware\n"); + return ret; + } + + return srat_disabled() ? -EINVAL : 0; +} +#else +static int __init arch_acpi_numa_init(void) +{ + return -EOPNOTSUPP; +} +#endif + +/** + * arch_numa_init() - Initialize NUMA + * + * Try each configured NUMA initialization method until one succeeds. The + * last fallback is dummy single node config encompassing whole memory. + */ +void __init arch_numa_init(void) +{ + if (!numa_off) { + if (!acpi_disabled && !numa_init(arch_acpi_numa_init)) + return; + if (acpi_disabled && !numa_init(of_numa_init)) + return; + } + + numa_init(dummy_numa_init); +} + +#ifdef CONFIG_NUMA_EMU +void __init numa_emu_update_cpu_to_node(int *emu_nid_to_phys, + unsigned int nr_emu_nids) +{ + int i, j; + + /* + * Transform cpu_to_node_map table to use emulated nids by + * reverse-mapping phys_nid. The maps should always exist but fall + * back to zero just in case. + */ + for (i = 0; i < ARRAY_SIZE(cpu_to_node_map); i++) { + if (cpu_to_node_map[i] == NUMA_NO_NODE) + continue; + for (j = 0; j < nr_emu_nids; j++) + if (cpu_to_node_map[i] == emu_nid_to_phys[j]) + break; + cpu_to_node_map[i] = j < nr_emu_nids ? j : 0; + } +} + +u64 __init numa_emu_dma_end(void) +{ + return memblock_start_of_DRAM() + SZ_4G; +} + +void debug_cpumask_set_cpu(unsigned int cpu, int node, bool enable) +{ + struct cpumask *mask; + + if (node == NUMA_NO_NODE) + return; + + mask = node_to_cpumask_map[node]; + if (!cpumask_available(mask)) { + pr_err("node_to_cpumask_map[%i] NULL\n", node); + dump_stack(); + return; + } + + if (enable) + cpumask_set_cpu(cpu, mask); + else + cpumask_clear_cpu(cpu, mask); + + pr_debug("%s cpu %d node %d: mask now %*pbl\n", + enable ? "numa_add_cpu" : "numa_remove_cpu", + cpu, node, cpumask_pr_args(mask)); +} +#endif /* CONFIG_NUMA_EMU */ -- cgit v1.2.3 From a8efc69a65fbef61bed0923c0de3425d74817c1c Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 13 Jul 2026 11:33:38 +0200 Subject: shmem: provide a shmem_write_folio wrapper Patch series "better block swap batching and a different take on swap_ops v5". This series makes use of the swap_iocb for block as well so that it doesn't do inefficient single-bio I/O, and then rebases the swap_ops from Baoquan on top of the now very different method structure. When running doing kernels builds, which is a workload that doesn't really do much THP anonymous memory it still gets 2x clustering for writeout and 1.2x for reading back swap in. The overall times do not actually change, though. This patch (of 7): Provide a wrapper for the shmem abuses in drm to prepare for swap I/O refactoring by keeping swap_iocb handling entirely contained in mm/. Link: https://lore.kernel.org/20260713093350.2154226-1-hch@lst.de Link: https://lore.kernel.org/20260713093350.2154226-2-hch@lst.de Signed-off-by: Christoph Hellwig Reviewed-by: Baoquan He Reviewed-by: Nhat Pham Reviewed-by: Baolin Wang Acked-by: Chris Li Reviewed-by: Kairui Song Cc: Kemeng Shi Cc: Barry Song Cc: Youngjun Park Signed-off-by: Andrew Morton --- drivers/gpu/drm/i915/gem/i915_gem_shmem.c | 2 +- drivers/gpu/drm/ttm/ttm_backup.c | 2 +- include/linux/shmem_fs.h | 5 +---- mm/shmem.c | 7 ++++++- mm/swap.h | 4 ++++ 5 files changed, 13 insertions(+), 7 deletions(-) (limited to 'drivers') diff --git a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c index 06543ae60706..ef9440166295 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c @@ -325,7 +325,7 @@ void __shmem_writeback(size_t size, struct address_space *mapping) if (folio_mapped(folio)) folio_redirty_for_writepage(&wbc, folio); else - error = shmem_writeout(folio, NULL, NULL); + error = shmem_write_folio(folio); } } diff --git a/drivers/gpu/drm/ttm/ttm_backup.c b/drivers/gpu/drm/ttm/ttm_backup.c index 3c067aadc52d..0c2d53a13b2a 100644 --- a/drivers/gpu/drm/ttm/ttm_backup.c +++ b/drivers/gpu/drm/ttm/ttm_backup.c @@ -160,7 +160,7 @@ ttm_backup_backup_folio(struct file *backup, struct folio *folio, if (writeback && !folio_mapped(to_folio) && folio_clear_dirty_for_io(to_folio)) { folio_set_reclaim(to_folio); - ret = shmem_writeout(to_folio, NULL, NULL); + ret = shmem_write_folio(to_folio); if (!folio_test_writeback(to_folio)) folio_clear_reclaim(to_folio); if (ret == AOP_WRITEPAGE_ACTIVATE) diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h index e729b9b0e38d..5663dff53186 100644 --- a/include/linux/shmem_fs.h +++ b/include/linux/shmem_fs.h @@ -12,8 +12,6 @@ #include #include -struct swap_iocb; - /* inode in-kernel data */ #ifdef CONFIG_TMPFS_QUOTA @@ -123,8 +121,7 @@ static inline bool shmem_mapping(const struct address_space *mapping) void shmem_unlock_mapping(struct address_space *mapping); struct page *shmem_read_mapping_page_gfp(struct address_space *mapping, pgoff_t index, gfp_t gfp_mask); -int shmem_writeout(struct folio *folio, struct swap_iocb **plug, - struct list_head *folio_list); +int shmem_write_folio(struct folio *folio); void shmem_truncate_range(struct inode *inode, loff_t start, uoff_t end); int shmem_unuse(unsigned int type); diff --git a/mm/shmem.c b/mm/shmem.c index 8ea776e52823..d245e01416e9 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -1751,7 +1751,12 @@ redirty: folio_mark_dirty(folio); return AOP_WRITEPAGE_ACTIVATE; /* Return with folio locked */ } -EXPORT_SYMBOL_GPL(shmem_writeout); + +int shmem_write_folio(struct folio *folio) +{ + return shmem_writeout(folio, NULL, NULL); +} +EXPORT_SYMBOL_GPL(shmem_write_folio); #if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS) static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol) diff --git a/mm/swap.h b/mm/swap.h index 4e4c291bbfde..276b7975a9dc 100644 --- a/mm/swap.h +++ b/mm/swap.h @@ -486,4 +486,8 @@ static inline unsigned int folio_swap_flags(struct folio *folio) } #endif /* CONFIG_SWAP */ + +int shmem_writeout(struct folio *folio, struct swap_iocb **plug, + struct list_head *folio_list); + #endif /* _MM_SWAP_H */ -- cgit v1.2.3 From a8b5875741d416703e19ad8eeac6fce8a12bd6e4 Mon Sep 17 00:00:00 2001 From: David Carlier Date: Mon, 10 Aug 2026 21:22:37 +0100 Subject: zram: fix slot lock bit position on big-endian 64-bit The slot lock is a bit operation on the whole __lock word, which flags and ac_time alias as two u32s. On little-endian the lock bit lands in the position ZRAM_ENTRY_LOCK reserves in flags, so the aliasing works out. On 64-bit big-endian it lands in ac_time instead: with ZRAM_TRACK_ENTRY_ACTIME enabled, storing the access time from mark_slot_accessed() or slot_free() wipes out the held lock bit, letting another CPU take the same slot lock; an access time value with that bit set makes the slot look locked forever. Shift the lock bit into the flags half of the word on big-endian 64-bit. Link: https://lore.kernel.org/20260810202241.2436603-1-devnexen@gmail.com Fixes: 2e8ff2f51dde ("zram: use u32 for entry ac_time tracking") Signed-off-by: David Carlier Reviewed-by: Sergey Senozhatsky Cc: Minchan Kim Cc: Signed-off-by: Andrew Morton --- drivers/block/zram/zram_drv.c | 6 +++--- drivers/block/zram/zram_drv.h | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) (limited to 'drivers') diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index d09fdca49cbd..a9b3bb1d3bef 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -74,7 +74,7 @@ static __must_check bool slot_trylock(struct zram *zram, unsigned long index) { unsigned long *lock = &zram->table[index].__lock; - if (!test_and_set_bit_lock(ZRAM_ENTRY_LOCK, lock)) { + if (!test_and_set_bit_lock(ZRAM_ENTRY_LOCK_BIT, lock)) { mutex_acquire(&zram->table_lock_map, 0, 1, _RET_IP_); lock_acquired(&zram->table_lock_map, _RET_IP_); return true; @@ -88,7 +88,7 @@ static void slot_lock(struct zram *zram, unsigned long index) unsigned long *lock = &zram->table[index].__lock; mutex_acquire(&zram->table_lock_map, 0, 0, _RET_IP_); - wait_on_bit_lock(lock, ZRAM_ENTRY_LOCK, TASK_UNINTERRUPTIBLE); + wait_on_bit_lock(lock, ZRAM_ENTRY_LOCK_BIT, TASK_UNINTERRUPTIBLE); lock_acquired(&zram->table_lock_map, _RET_IP_); } @@ -97,7 +97,7 @@ static void slot_unlock(struct zram *zram, unsigned long index) unsigned long *lock = &zram->table[index].__lock; mutex_release(&zram->table_lock_map, _RET_IP_); - clear_and_wake_up_bit(ZRAM_ENTRY_LOCK, lock); + clear_and_wake_up_bit(ZRAM_ENTRY_LOCK_BIT, lock); } static inline bool init_done(struct zram *zram) diff --git a/drivers/block/zram/zram_drv.h b/drivers/block/zram/zram_drv.h index 4fddc582f3b8..7a55d751417e 100644 --- a/drivers/block/zram/zram_drv.h +++ b/drivers/block/zram/zram_drv.h @@ -15,6 +15,7 @@ #ifndef _ZRAM_DRV_H_ #define _ZRAM_DRV_H_ +#include #include #include @@ -57,6 +58,19 @@ enum zram_pageflags { __NR_ZRAM_PAGEFLAGS, }; +/* + * The slot lock is a bit-wait lock on the whole __lock word, while + * flags and ac_time alias that word as two u32s. The lock bit must + * land in the slot that ZRAM_ENTRY_LOCK reserves in attr.flags; on + * 64-bit big-endian the flags word maps to the upper half of __lock, + * so the bit position has to be shifted up. + */ +#if defined(CONFIG_64BIT) && defined(__BIG_ENDIAN) +#define ZRAM_ENTRY_LOCK_BIT (ZRAM_ENTRY_LOCK + 32) +#else +#define ZRAM_ENTRY_LOCK_BIT ZRAM_ENTRY_LOCK +#endif + /* * Allocated for each disk page. We use bit-lock (ZRAM_ENTRY_LOCK bit * of flags) to save memory. There can be plenty of entries and standard -- cgit v1.2.3