mm/slab: simplify freeing remote objects in free_to_pcs_bulk()

We have been moving remote objects to an on-stack array and flushing it
when full. Instead, we can swap them towards the beginning of the
supplied array and bulk-free it just once.

Also add a comment to explain the rationale of freeing remote objects
last, because now it would appear to be simpler to free them first.

Reviewed-by: Pedro Falcato <pfalcato@suse.de>
Reviewed-by: Shengming Hu <hu.shengming@zte.com.cn>
Reviewed-by: Harry Yoo (Oracle) <harry@kernel.org>
Reviewed-by: Hao Li <hao.li@linux.dev>
Link: https://patch.msgid.link/20260713-bulk_free_remote-v2-1-24ee24771c2f@kernel.org
Signed-off-by: Vlastimil Babka (SUSE) <vbabka@kernel.org>

1 files changed, 20 insertions, 10 deletions

diff --git a/mm/slub.c b/mm/slub.c
index 22045dc919ef..b650c1729437 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -6255,9 +6255,17 @@ no_empty:
 static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p)
 {
 	bool init = slab_want_init_on_free(s);
-	void *remote_objects[PCS_BATCH_MAX];
+	void **remote_objects = p;
 	unsigned int remote_nr = 0;
 
+	/*
+	 * Process the free hooks and separate out remote objects by
+	 * partitioning the 'p' array in place:
+	 *
+	 * [0, remote_nr) - processed remote objects
+	 * [remote_nr, i) - processed local objects
+	 * [i, size)      - unprocessed objects
+	 */
 	for (unsigned int i = 0; i < size;) {
 		struct slab *slab = virt_to_slab(p[i]);
 
@@ -6270,19 +6278,17 @@ static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p)
 		}
 
 		if (unlikely(!can_free_to_pcs(slab))) {
-			remote_objects[remote_nr] = p[i];
-			p[i] = p[--size];
-			if (++remote_nr >= PCS_BATCH_MAX) {
-				__kmem_cache_free_bulk(s, remote_nr, &remote_objects[0]);
-				stat_add(s, FREE_SLOWPATH, remote_nr);
-				remote_nr = 0;
-			}
-			continue;
+			if (i != remote_nr)
+				swap(remote_objects[remote_nr], p[i]);
+			remote_nr++;
 		}
 
 		i++;
 	}
 
+	p += remote_nr;
+	size -= remote_nr;
+
 	while (size) {
 		unsigned int batch_freed = __free_to_pcs_batch(s, size, p);
 
@@ -6296,8 +6302,12 @@ static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p)
 		size -= batch_freed;
 	}
 
+	/*
+	 * Processing remote objects last decreases the chances of cpu migration
+	 * while freeing to sheaves and compromising object locality
+	 */
 	if (remote_nr) {
-		__kmem_cache_free_bulk(s, remote_nr, &remote_objects[0]);
+		__kmem_cache_free_bulk(s, remote_nr, remote_objects);
 		stat_add(s, FREE_SLOWPATH, remote_nr);
 	}
 }