From 6648f54f3459c5d069b7ce294170d721ce9bac2f Mon Sep 17 00:00:00 2001 From: Miklos Szeredi Date: Mon, 6 Jul 2026 15:44:58 +0200 Subject: fuse: move "epoch" from dentry.d_time to fuse_dentry.epoch ...in hope of removing d_time one day. Fixes: 2396356a945b ("fuse: add more control over cache invalidation behaviour") Signed-off-by: Miklos Szeredi --- fs/fuse/dir.c | 23 ++++++++++++++++------- fs/fuse/fuse_i.h | 2 ++ fs/fuse/readdir.c | 2 +- 3 files changed, 19 insertions(+), 8 deletions(-) (limited to 'fs') diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index 0e2a1039fa43..b689503bc880 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -96,6 +96,7 @@ static void fuse_advise_use_readdirplus(struct inode *dir) struct fuse_dentry { u64 time; + u64 epoch; union { struct rcu_head rcu; struct rb_node node; @@ -236,6 +237,13 @@ void fuse_dentry_tree_cleanup(void) WARN_ON_ONCE(!RB_EMPTY_ROOT(&dentry_hash[i].tree)); } +void fuse_dentry_set_epoch(struct dentry *dentry, u64 epoch) +{ + struct fuse_dentry *fd = dentry->d_fsdata; + + fd->epoch = epoch; +} + static inline void __fuse_dentry_settime(struct dentry *dentry, u64 time) { ((struct fuse_dentry *) dentry->d_fsdata)->time = time; @@ -387,10 +395,11 @@ static int fuse_dentry_revalidate(struct inode *dir, const struct qstr *name, struct fuse_mount *fm; struct fuse_conn *fc; struct fuse_inode *fi; + struct fuse_dentry *fd = entry->d_fsdata; int ret; fc = get_fuse_conn_super(dir->i_sb); - if (entry->d_time < atomic_read(&fc->epoch)) + if (fd->epoch < atomic_read(&fc->epoch)) goto invalid; inode = d_inode_rcu(entry); @@ -480,10 +489,10 @@ static int fuse_dentry_init(struct dentry *dentry) RB_CLEAR_NODE(&fd->node); dentry->d_fsdata = fd; /* - * Initialising d_time (epoch) to '0' ensures the dentry is invalid + * Initialising epoch to '0' ensures the dentry is invalid * if compared to fc->epoch, which is initialized to '1'. */ - dentry->d_time = 0; + fuse_dentry_set_epoch(dentry, 0); return 0; } @@ -641,7 +650,7 @@ static struct dentry *fuse_lookup(struct inode *dir, struct dentry *entry, goto out_err; entry = newent ? newent : entry; - entry->d_time = epoch; + fuse_dentry_set_epoch(entry, epoch); if (outarg_valid) fuse_change_entry_timeout(entry, &outarg); else @@ -898,7 +907,7 @@ static int fuse_create_open(struct mnt_idmap *idmap, struct inode *dir, } kfree(forget); d_instantiate(entry, inode); - entry->d_time = epoch; + fuse_dentry_set_epoch(entry, epoch); fuse_change_entry_timeout(entry, &outentry); fuse_dir_changed(dir); err = generic_file_open(inode, file); @@ -1028,10 +1037,10 @@ static struct dentry *create_new_entry(struct mnt_idmap *idmap, struct fuse_moun return d; if (d) { - d->d_time = epoch; + fuse_dentry_set_epoch(d, epoch); fuse_change_entry_timeout(d, &outarg); } else { - entry->d_time = epoch; + fuse_dentry_set_epoch(entry, epoch); fuse_change_entry_timeout(entry, &outarg); } fuse_dir_changed(dir); diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 85f738c53122..c8d4c5f3af7e 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -1054,6 +1054,8 @@ u64 fuse_time_to_jiffies(u64 sec, u32 nsec); void fuse_change_entry_timeout(struct dentry *entry, struct fuse_entry_out *o); +void fuse_dentry_set_epoch(struct dentry *dentry, u64 epoch); + /* * Initialize fuse_conn */ diff --git a/fs/fuse/readdir.c b/fs/fuse/readdir.c index 0e1321491747..5ca87151d70d 100644 --- a/fs/fuse/readdir.c +++ b/fs/fuse/readdir.c @@ -260,7 +260,7 @@ retry: } if (fc->readdirplus_auto) set_bit(FUSE_I_INIT_RDPLUS, &get_fuse_inode(inode)->state); - dentry->d_time = epoch; + fuse_dentry_set_epoch(dentry, epoch); fuse_change_entry_timeout(dentry, o); dput(dentry); -- cgit v1.2.3 From cc6f804e785f9a7c04333cb55372723da482ae3f Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Tue, 7 Jul 2026 15:04:48 -0700 Subject: fuse: don't clear folio uptodate on writethrough errors In the writethrough path (fuse_send_write_pages()), if the write to the server failed or was a short write, the uptodate flag on the folios are cleared. As explained by Matthew in [1], this is dangerous because the folio may be mapped into userspace. The mm code has the invariant that a non-uptodate folio must never be visible to userspace (to avoid potentially leaking confidental information to userspace) and has checks in place for this that if violated can bring down the whole machine. Practically speaking, the effect of this change for the fuse writethrough error path is that if an application does a write and then the server fails to persist the data or only services a short write, the page cache folio keeps the data the application wrote instead of being reverted to the server's contents on the next read. The failure is still reported to the application synchronously through the short count / error return of the write() syscall. Folios that were only partially written are unaffected since they were never marked uptodate in the first place (fuse_fill_write_page() only marks a folio as uptodate if the whole folio was written to). [1] https://lore.kernel.org/linux-fsdevel/ajtPMgO65FA1TXhi@casper.infradead.org/ Suggested-by: Matthew Wilcox Reviewed-by: Darrick J. Wong Signed-off-by: Joanne Koong Reviewed-by: Christoph Hellwig Signed-off-by: Miklos Szeredi --- fs/fuse/file.c | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) (limited to 'fs') diff --git a/fs/fuse/file.c b/fs/fuse/file.c index ceada75310b8..f5bcbfa8b6ae 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1219,8 +1219,7 @@ static ssize_t fuse_send_write_pages(struct fuse_io_args *ia, struct file *file = iocb->ki_filp; struct fuse_file *ff = file->private_data; struct fuse_mount *fm = ff->fm; - unsigned int offset, i; - bool short_write; + unsigned int i; int err; for (i = 0; i < ap->num_folios; i++) @@ -1235,24 +1234,9 @@ static ssize_t fuse_send_write_pages(struct fuse_io_args *ia, if (!err && ia->write.out.size > count) err = -EIO; - short_write = ia->write.out.size < count; - offset = ap->descs[0].offset; - count = ia->write.out.size; for (i = 0; i < ap->num_folios; i++) { struct folio *folio = ap->folios[i]; - if (err) { - folio_clear_uptodate(folio); - } else { - if (count >= folio_size(folio) - offset) - count -= folio_size(folio) - offset; - else { - if (short_write) - folio_clear_uptodate(folio); - count = 0; - } - offset = 0; - } if (ia->write.folio_locked && (i == ap->num_folios - 1)) folio_unlock(folio); folio_put(folio); -- cgit v1.2.3 From 03e1dd35c206f24b6bc987198ac58e5138176cb7 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Tue, 7 Jul 2026 15:04:49 -0700 Subject: iomap: add helper to mark folio uptodate Add an exported helper iomap_folio_mark_uptodate() to mark a folio as uptodate and update its uptodate bitmap if the folio has iomap state data attached. This is needed because there are some filesystems (eg fuse) that have paths outside of conventional iomap calls that need to mark a folio as uptodate (eg writing server-pushed data directly into the page cache) and need the iomap-internal uptodate bitmap to be in sync with the uptodate state of the folio. Reviewed-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Joanne Koong Signed-off-by: Miklos Szeredi --- fs/iomap/buffered-io.c | 6 ++++++ include/linux/iomap.h | 1 + 2 files changed, 7 insertions(+) (limited to 'fs') diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c index 276720bc18dc..af415293e265 100644 --- a/fs/iomap/buffered-io.c +++ b/fs/iomap/buffered-io.c @@ -105,6 +105,12 @@ static void iomap_set_range_uptodate(struct folio *folio, size_t off, folio_mark_uptodate(folio); } +void iomap_folio_mark_uptodate(struct folio *folio) +{ + iomap_set_range_uptodate(folio, 0, folio_size(folio)); +} +EXPORT_SYMBOL_GPL(iomap_folio_mark_uptodate); + /* * Find the next dirty block in the folio. end_blk is inclusive. * If no dirty block is found, this will return end_blk + 1. diff --git a/include/linux/iomap.h b/include/linux/iomap.h index 56b43d594e6e..40aa3476a351 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -365,6 +365,7 @@ struct folio *iomap_get_folio(struct iomap_iter *iter, loff_t pos, size_t len); bool iomap_release_folio(struct folio *folio, gfp_t gfp_flags); void iomap_invalidate_folio(struct folio *folio, size_t offset, size_t len); bool iomap_dirty_folio(struct address_space *mapping, struct folio *folio); +void iomap_folio_mark_uptodate(struct folio *folio); int iomap_file_unshare(struct inode *inode, loff_t pos, loff_t len, const struct iomap_ops *ops, const struct iomap_write_ops *write_ops); -- cgit v1.2.3 From 16f4be93c65ab3fc9c4c831849722581f8f3ca14 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Tue, 7 Jul 2026 15:04:50 -0700 Subject: fuse: use iomap helper to mark folio uptodate When fuse enables large folios, a large folio will be backed by iomap_folio_state that keeps track of uptodate and dirty state in an internal bitmap. Fuse writethrough and notify store paths currently set folio uptodate state with folio_mark_uptodate(), which touches only the folio-level flag, but on an iomap-backed folio, that leaves the uptodate bitmap out of sync. Use the iomap_folio_mark_uptodate() helper to update both the folio uptodate state and the iomap uptodate bitmap. Reviewed-by: Darrick J. Wong Signed-off-by: Joanne Koong Reviewed-by: Christoph Hellwig Signed-off-by: Miklos Szeredi --- fs/fuse/file.c | 2 +- fs/fuse/notify.c | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/fuse/file.c b/fs/fuse/file.c index f5bcbfa8b6ae..da5859e8159d 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1311,7 +1311,7 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, /* If we copied full folio, mark it uptodate */ if (tmp == folio_size(folio)) - folio_mark_uptodate(folio); + iomap_folio_mark_uptodate(folio); if (folio_test_uptodate(folio)) { folio_unlock(folio); diff --git a/fs/fuse/notify.c b/fs/fuse/notify.c index 29578104ae6c..1ba763705d91 100644 --- a/fs/fuse/notify.c +++ b/fs/fuse/notify.c @@ -2,6 +2,8 @@ #include "dev.h" #include "fuse_i.h" + +#include #include static int fuse_notify_poll(struct fuse_conn *fc, unsigned int size, @@ -192,7 +194,7 @@ static int fuse_notify_store(struct fuse_conn *fc, unsigned int size, if (!folio_test_uptodate(folio) && !err && folio_offset == 0 && (nr_bytes == folio_size(folio) || file_size == end)) { folio_zero_segment(folio, nr_bytes, folio_size(folio)); - folio_mark_uptodate(folio); + iomap_folio_mark_uptodate(folio); } folio_unlock(folio); folio_put(folio); -- cgit v1.2.3 From ed9c881f3b498383f73c42712b359419da42a7b0 Mon Sep 17 00:00:00 2001 From: Miklos Szeredi Date: Thu, 9 Jul 2026 08:37:05 +0200 Subject: fuse: fix race between interrupt and resend After commit f8fce75fedf7 ("fuse: clear intr_entry in fuse_resend and fuse_remove_pending_req") the WARN_ON(!list_empty(&req->intr_entry)) in fuse_request_free() still triggers due to the following race: In request_wait_answer() if (test_bit(FR_SENT, &req->flags)) -> returns true In fuse_chan_resend() clear_bit(FR_SENT, &req->flags) In request_wait_answer() queue_interrupt(req) Fix by: - move clearing FR_SENT inside fpq->lock - move setting FR_PENDING inside fiq->lock - recheck FR_SENT after acquiring fiq->lock in fuse_dev_queue_interrupt() Reported-by: zdi-disclosures@trendmicro.com Fixes: f8fce75fedf7 ("fuse: clear intr_entry in fuse_resend and fuse_remove_pending_req") Cc: stable@vger.kernel.org # 6.9 Signed-off-by: Miklos Szeredi --- fs/fuse/dev.c | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) (limited to 'fs') diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 5763a7cd3b37..7e81dea4f1c7 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -240,7 +240,8 @@ void fuse_dev_queue_forget(struct fuse_iqueue *fiq, void fuse_dev_queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req) { spin_lock(&fiq->lock); - if (list_empty(&req->intr_entry)) { + /* Repeat FR_SENT test after obtaining the lock to prevent race with fuse_resend() */ + if (list_empty(&req->intr_entry) && test_bit(FR_SENT, &req->flags)) { list_add_tail(&req->intr_entry, &fiq->interrupts); /* * Pairs with smp_mb() implied by test_and_set_bit() @@ -1760,7 +1761,7 @@ out: void fuse_chan_resend(struct fuse_chan *fch) { struct fuse_dev *fud; - struct fuse_req *req, *next; + struct fuse_req *req; struct fuse_iqueue *fiq = &fch->iq; LIST_HEAD(to_queue); unsigned int i; @@ -1775,24 +1776,20 @@ void fuse_chan_resend(struct fuse_chan *fch) struct fuse_pqueue *fpq = &fud->pq; spin_lock(&fpq->lock); - for (i = 0; i < FUSE_PQ_HASH_SIZE; i++) - list_splice_tail_init(&fpq->processing[i], &to_queue); + for (i = 0; i < FUSE_PQ_HASH_SIZE; i++) { + struct list_head *this_queue = &fpq->processing[i]; + + list_for_each_entry(req, this_queue, list) + clear_bit(FR_SENT, &req->flags); + list_splice_tail_init(this_queue, &to_queue); + } spin_unlock(&fpq->lock); } spin_unlock(&fch->lock); - list_for_each_entry_safe(req, next, &to_queue, list) { - set_bit(FR_PENDING, &req->flags); - clear_bit(FR_SENT, &req->flags); - /* mark the request as resend request */ - req->in.h.unique |= FUSE_UNIQUE_RESEND; - } - spin_lock(&fiq->lock); if (!fiq->connected) { spin_unlock(&fiq->lock); - list_for_each_entry(req, &to_queue, list) - clear_bit(FR_PENDING, &req->flags); fuse_dev_end_requests(&to_queue); return; } @@ -1801,6 +1798,10 @@ void fuse_chan_resend(struct fuse_chan *fch) * intr_entry on fiq->interrupts after the request is re-queued. */ list_for_each_entry(req, &to_queue, list) { + set_bit(FR_PENDING, &req->flags); + /* mark the request as resend request */ + req->in.h.unique |= FUSE_UNIQUE_RESEND; + if (test_bit(FR_INTERRUPTED, &req->flags)) list_del_init(&req->intr_entry); } -- cgit v1.2.3 From edb310bc27f0ad83e7fd558a3caf1a94ca511654 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Thu, 16 Jul 2026 11:31:42 -0700 Subject: fuse: fix missing barrier when checking io-uring readiness fuse_block_alloc() reads fch->initialized and then fch->io_uring. fch->io_uring is set before fch->initialized, ordered by the smp_wmb() in fuse_chan_set_intialized(), but fuse_block_alloc() has no matching read barrier between the two loads. This may lead a CPU to observe fch->initialized=1 but fch->io_uring=0, and skip the check that blocks request allocation until the io-uring queues are ready. This can reintroduce the lock-order inversion deadlock that commit 3393ff964e0f prevents. Add an smp_rmb() barrier to pair with the smp_wmb() in fuse_chan_set_initialized() to prevent this. Fixes: 3393ff964e0f ("fuse: block request allocation until io-uring init is complete") Cc: stable@vger.kernel.org Reviewed-by: Bernd Schubert Signed-off-by: Joanne Koong Signed-off-by: Miklos Szeredi --- fs/fuse/dev.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 7e81dea4f1c7..a27ea64d763a 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -85,7 +85,13 @@ void fuse_chan_set_initialized(struct fuse_chan *fch, struct fuse_chan_param *pa static bool fuse_block_alloc(struct fuse_chan *fch, bool for_background) { - return !fch->initialized || (for_background && fch->blocked) || + if (!fch->initialized) + return true; + + /* Pairs with smp_wmb() in fuse_chan_set_initialized() */ + smp_rmb(); + + return (for_background && fch->blocked) || (fch->io_uring && fch->connected && !fuse_uring_ready(fch)); } @@ -120,9 +126,6 @@ static struct fuse_req *fuse_get_req(struct fuse_chan *fch, bool for_background) goto out; } - /* Matches smp_wmb() in fuse_chan_set_initialized() */ - smp_rmb(); - err = -ENOTCONN; if (!fch->connected) goto out; -- cgit v1.2.3 From 4ef7c8cc9894fccc7aa5fdaf6b39faa45c58c23e Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Thu, 16 Jul 2026 11:31:43 -0700 Subject: fuse: use release/acquire for fch->initialized fuse_chan_set_initialized() sets values for the connection state and then sets fch->initialized to true, but lockless readers read fch->initialized and if true, go to read the connection state values, without using any barriers. There are a few instances where this happens (fuse_uring_cmd() before dispatching register / commit-and-fetch cmds, fuse_dev_do_wriite() for handling notify retrieves, etc). To make this as simple as possible, use release/acquire semantics for writing/reading fch->initialized. Add the missing read barriers. This is not marked for stable as these are not realistically reachable on a well-behaved server, and buggy/malicious servers who trigger this path fail benignly rather than crash or deadlock the kernel. Signed-off-by: Joanne Koong Signed-off-by: Miklos Szeredi --- fs/fuse/cuse.c | 3 ++- fs/fuse/dev.c | 14 ++++++-------- fs/fuse/dev_uring.c | 4 +++- 3 files changed, 11 insertions(+), 10 deletions(-) (limited to 'fs') diff --git a/fs/fuse/cuse.c b/fs/fuse/cuse.c index 3c15b5ba16d7..96d57735a79f 100644 --- a/fs/fuse/cuse.c +++ b/fs/fuse/cuse.c @@ -530,7 +530,8 @@ static int cuse_channel_open(struct inode *inode, struct file *file) INIT_LIST_HEAD(&cc->list); - cc->fc.chan->initialized = 1; + /* Pairs with smp_load_acquire() readers of fch->initialized */ + smp_store_release(&cc->fc.chan->initialized, 1); rc = cuse_send_init(cc); if (rc) { fuse_dev_put(fud); diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index a27ea64d763a..27dafda2a841 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -77,20 +77,17 @@ void fuse_chan_set_initialized(struct fuse_chan *fch, struct fuse_chan_param *pa fch->max_pages = param->max_pages; } - /* Make sure stores before this are seen on another CPU */ - smp_wmb(); - fch->initialized = 1; + /* Pairs with smp_load_acquire() readers of fch->initialized */ + smp_store_release(&fch->initialized, 1); wake_up_all(&fch->blocked_waitq); } static bool fuse_block_alloc(struct fuse_chan *fch, bool for_background) { - if (!fch->initialized) + /* Pairs with smp_store_release() in fuse_chan_set_initialized() */ + if (!smp_load_acquire(&fch->initialized)) return true; - /* Pairs with smp_wmb() in fuse_chan_set_initialized() */ - smp_rmb(); - return (for_background && fch->blocked) || (fch->io_uring && fch->connected && !fuse_uring_ready(fch)); } @@ -1892,7 +1889,8 @@ static ssize_t fuse_dev_do_write(struct fuse_dev *fud, * initialized and connected state */ err = -EINVAL; - if (!fch->initialized || !fch->connected) + /* Pairs with smp_store_release() in fuse_chan_set_initialized() */ + if (!smp_load_acquire(&fch->initialized) || !fch->connected) goto copy_finish; /* Don't try to move folios (yet) */ diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 77c8cec43d9c..51f985154aa1 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -1251,8 +1251,10 @@ int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) /* * fuse_uring_register() needs the ring to be initialized, * we need to know the max payload size + * + * Pairs with smp_store_release() in fuse_chan_set_initialized() */ - if (!fch->initialized) + if (!smp_load_acquire(&fch->initialized)) return -EAGAIN; switch (cmd_op) { -- cgit v1.2.3 From 42df916e5a5f8fb4b60c8cefb54318d1ec02c580 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Thu, 16 Jul 2026 11:31:44 -0700 Subject: fuse: publish io-uring queues with release semantics fuse_uring_create_queue() initializes a fuse_ring_queue and then publishes the pointer into ring->queues[qid] with WRITE_ONCE() under the fch->lock. There are several readers that may concurrently be fetching that pointer locklessly and then deferencing it. WRITE_ONCE() doesn't ensure ordering of the queue's field initialization before the ring->queues[qid] pointer assignment. The queue must be published with smp_store_release() so the field initialization is guaranteed to happen before. Readers in paths where the read may happen concurrently with the store need to use READ_ONCE() because any race involving a plain access is undefined. Fixes: 24fe962c86f5 ("fuse: {io-uring} Handle SQEs - register commands") Cc: stable@vger.kernel.org Reviewed-by: Bernd Schubert Signed-off-by: Joanne Koong Signed-off-by: Miklos Szeredi --- fs/fuse/dev_uring.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) (limited to 'fs') diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 51f985154aa1..c8488ebc1d1f 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -204,7 +204,7 @@ void fuse_uring_destruct(struct fuse_chan *fch) return; for (qid = 0; qid < ring->nr_queues; qid++) { - struct fuse_ring_queue *queue = ring->queues[qid]; + struct fuse_ring_queue *queue = READ_ONCE(ring->queues[qid]); struct fuse_ring_ent *ent, *next; if (!queue) @@ -223,7 +223,7 @@ void fuse_uring_destruct(struct fuse_chan *fch) kfree(queue->fpq.processing); kfree(queue); - ring->queues[qid] = NULL; + WRITE_ONCE(ring->queues[qid], NULL); } kfree(ring->queues); @@ -321,9 +321,11 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, } /* - * write_once and lock as the caller mostly doesn't take the lock at all + * fch->lock serializes concurrent creators for this qid. + * smp_store_release() are for the lockless readers who must see a + * fully initialized queue after &ring->queues[qid] is set */ - WRITE_ONCE(ring->queues[qid], queue); + smp_store_release(&ring->queues[qid], queue); spin_unlock(&fch->lock); return queue; @@ -434,7 +436,7 @@ static void fuse_uring_log_ent_state(struct fuse_ring *ring) struct fuse_ring_ent *ent; for (qid = 0; qid < ring->nr_queues; qid++) { - struct fuse_ring_queue *queue = ring->queues[qid]; + struct fuse_ring_queue *queue = READ_ONCE(ring->queues[qid]); if (!queue) continue; @@ -967,7 +969,7 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, if (qid >= ring->nr_queues) return -EINVAL; - queue = ring->queues[qid]; + queue = READ_ONCE(ring->queues[qid]); if (!queue) return err; fpq = &queue->fpq; @@ -1035,7 +1037,7 @@ static bool is_ring_ready(struct fuse_ring *ring, int current_qid) if (current_qid == qid) continue; - queue = ring->queues[qid]; + queue = READ_ONCE(ring->queues[qid]); if (!queue) { ready = false; break; @@ -1191,7 +1193,7 @@ static int fuse_uring_register(struct io_uring_cmd *cmd, return -EINVAL; } - queue = ring->queues[qid]; + queue = READ_ONCE(ring->queues[qid]); if (!queue) { queue = fuse_uring_create_queue(ring, qid); if (!queue) @@ -1332,7 +1334,7 @@ static struct fuse_ring_queue *fuse_uring_task_to_queue(struct fuse_ring *ring) ring->nr_queues)) qid = 0; - queue = ring->queues[qid]; + queue = READ_ONCE(ring->queues[qid]); WARN_ONCE(!queue, "Missing queue for qid %d\n", qid); return queue; -- cgit v1.2.3 From 98b4ca2378e1f6b6c06a74f699623ebecfb3549d Mon Sep 17 00:00:00 2001 From: Jim Harris Date: Mon, 22 Jun 2026 17:42:27 -0700 Subject: fuse: allow larger read requests by setting bdi->io_pages A FUSE server that advertises a large max_pages and max_write (e.g. max_pages=256, max_write=1MB) cannot currently obtain matching FUSE_READ request sizes from the kernel. Buffered sequential writes arrive at the server at the negotiated max_write size, but a large buffered read() is split into several smaller FUSE_READ requests. For a buffered read, filemap_get_pages() -> page_cache_sync_ra() sizes the read against ractl_max_pages(): max_pages = ractl->ra->ra_pages; if (req_size > max_pages && bdi->io_pages > max_pages) max_pages = min(req_size, bdi->io_pages); fuse leaves bdi->io_pages at the default VM_READAHEAD_PAGES (128KB), so a 1MB read() (req_size = 256 pages) is clamped to the readahead window (128KB, or 256KB for POSIX_FADV_SEQUENTIAL), producing four 256KB FUSE_READ round-trips instead of one. Set bdi->io_pages to fc->max_pages after feature negotiation. As the code above shows, io_pages only raises the limit when the request size already exceeds the readahead window, so it enlarges explicitly requested reads without enlarging the speculative readahead window. This avoids increasing speculative page-cache readahead on behalf of an unprivileged server. NFS does the same, setting io_pages from rpages while leaving ra_pages at the default. fc->max_pages is already bounded by fc->max_pages_limit (and, for virtio-fs, by the virtqueue descriptor count), so io_pages inherits the same bound. Suggested-by: Joanne Koong Signed-off-by: Jim Harris Assisted-by: Cursor:claude-opus-4.8 Reviewed-by: Joanne Koong Signed-off-by: Miklos Szeredi --- fs/fuse/inode.c | 1 + 1 file changed, 1 insertion(+) (limited to 'fs') diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index d975073c6029..f7a0a0860a04 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -1416,6 +1416,7 @@ static void process_init_reply(struct fuse_args *args, int error) fm->sb->s_bdi->ra_pages = min(fm->sb->s_bdi->ra_pages, ra_pages); + fm->sb->s_bdi->io_pages = fc->max_pages; fc->minor = arg->minor; fc->max_write = arg->minor < 5 ? 4096 : arg->max_write; fc->max_write = max_t(unsigned, 4096, fc->max_write); -- cgit v1.2.3 From 6330b1f61ed1d17850fc61bdb8920ca1056e2cf9 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Fri, 14 Aug 2026 11:59:41 -0700 Subject: fuse: decouple fuse_ring creation from ent registration Currently, the connection's fuse_ring is created lazily on the first FUSE_IO_URING_CMD_REGISTER command. A server registers entries from one thread per queue (one per CPU) and those threads issue their first REGISTER command concurrently. They then race to create the single per-connection fuse_ring, which required open-coded handling in fuse_uring_create() to detect and protect against concurrent creations. Decouple fuse_ring creation from ent registration and move it to FUSE_INIT reply processing after a server has negotiated and set FUSE_OVER_IO_URING. The ring is published before the connection is marked initialized. fuse_uring_register() no longer creates the ring and it instead uses the ring set up at init time. Reviewed-by: Bernd Schubert Signed-off-by: Joanne Koong Signed-off-by: Miklos Szeredi --- fs/fuse/dev.c | 8 +++----- fs/fuse/dev.h | 2 +- fs/fuse/dev_uring.c | 26 ++++++++++---------------- fs/fuse/dev_uring_i.h | 5 +++++ fs/fuse/inode.c | 4 +++- 5 files changed, 22 insertions(+), 23 deletions(-) (limited to 'fs') diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 27dafda2a841..d8f97943e973 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -75,6 +75,9 @@ void fuse_chan_set_initialized(struct fuse_chan *fch, struct fuse_chan_param *pa fch->minor = param->minor; fch->max_write = param->max_write; fch->max_pages = param->max_pages; + + if (param->io_uring_enabled) + fuse_uring_conn_init(fch); } /* Pairs with smp_load_acquire() readers of fch->initialized */ @@ -412,11 +415,6 @@ void fuse_chan_set_fc(struct fuse_chan *fch, struct fuse_conn *fc) fch->conn = fc; } -void fuse_chan_io_uring_enable(struct fuse_chan *fch) -{ - fch->io_uring = 1; -} - void fuse_pqueue_init(struct fuse_pqueue *fpq) { spin_lock_init(&fpq->lock); diff --git a/fs/fuse/dev.h b/fs/fuse/dev.h index aed69fd14c41..8d25378c0918 100644 --- a/fs/fuse/dev.h +++ b/fs/fuse/dev.h @@ -22,6 +22,7 @@ struct fuse_chan_param { unsigned int minor; unsigned int max_write; unsigned int max_pages; + bool io_uring_enabled; }; struct fuse_chan *fuse_chan_new(void); @@ -34,7 +35,6 @@ void fuse_chan_max_background_set(struct fuse_chan *fch, unsigned int val); unsigned int fuse_chan_num_waiting(struct fuse_chan *fch); void fuse_chan_set_fc(struct fuse_chan *fch, struct fuse_conn *fc); void fuse_chan_set_initialized(struct fuse_chan *fch, struct fuse_chan_param *param); -void fuse_chan_io_uring_enable(struct fuse_chan *fch); ssize_t fuse_chan_send(struct fuse_chan *fch, struct fuse_args *args); int fuse_chan_send_bg(struct fuse_chan *fch, struct fuse_args *args, gfp_t gfp_flags); int fuse_chan_send_notify_reply(struct fuse_chan *fch, struct fuse_args *args, u64 unique); diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index c8488ebc1d1f..9616778505ba 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -238,7 +238,6 @@ static struct fuse_ring *fuse_uring_create(struct fuse_chan *fch) { struct fuse_ring *ring; size_t nr_queues = num_possible_cpus(); - struct fuse_ring *res = NULL; size_t max_payload_size; ring = kzalloc_obj(*ring, GFP_KERNEL_ACCOUNT); @@ -258,12 +257,6 @@ static struct fuse_ring *fuse_uring_create(struct fuse_chan *fch) spin_unlock(&fch->lock); goto out_err; } - if (fch->ring) { - /* race, another thread created the ring in the meantime */ - spin_unlock(&fch->lock); - res = fch->ring; - goto out_err; - } init_waitqueue_head(&ring->stop_waitq); @@ -278,7 +271,13 @@ static struct fuse_ring *fuse_uring_create(struct fuse_chan *fch) out_err: kfree(ring->queues); kfree(ring); - return res; + return NULL; +} + +void fuse_uring_conn_init(struct fuse_chan *fch) +{ + if (fuse_uring_create(fch)) + fch->io_uring = 1; } static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, @@ -1178,15 +1177,10 @@ static int fuse_uring_register(struct io_uring_cmd *cmd, struct fuse_ring *ring = smp_load_acquire(&fch->ring); struct fuse_ring_queue *queue; struct fuse_ring_ent *ent; - int err; unsigned int qid = READ_ONCE(cmd_req->qid); - err = -ENOMEM; - if (!ring) { - ring = fuse_uring_create(fch); - if (!ring) - return err; - } + if (!ring) + return -EINVAL; if (qid >= ring->nr_queues) { pr_info_ratelimited("fuse: Invalid ring qid %u\n", qid); @@ -1197,7 +1191,7 @@ static int fuse_uring_register(struct io_uring_cmd *cmd, if (!queue) { queue = fuse_uring_create_queue(ring, qid); if (!queue) - return err; + return -ENOMEM; } /* diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h index 55f8d04e4b0b..d721a4fc0215 100644 --- a/fs/fuse/dev_uring_i.h +++ b/fs/fuse/dev_uring_i.h @@ -135,6 +135,7 @@ struct fuse_ring { bool ready; }; +void fuse_uring_conn_init(struct fuse_chan *fch); void fuse_uring_stop_queues(struct fuse_ring *ring); void fuse_uring_abort_end_requests(struct fuse_ring *ring); int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags); @@ -174,6 +175,10 @@ static inline bool fuse_uring_ready(struct fuse_chan *fch) #else /* CONFIG_FUSE_IO_URING */ +static inline void fuse_uring_conn_init(struct fuse_chan *fch) +{ +} + static inline void fuse_uring_abort(struct fuse_chan *fch) { } diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index f7a0a0860a04..33773c7d129a 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -1272,6 +1272,7 @@ static void process_init_reply(struct fuse_args *args, int error) struct fuse_mount *fm = ia->fm; struct fuse_conn *fc = fm->fc; struct fuse_init_out *arg = &ia->out; + bool io_uring_enabled = false; bool ok = true; if (error || arg->major != FUSE_KERNEL_VERSION) @@ -1402,7 +1403,7 @@ static void process_init_reply(struct fuse_args *args, int error) ok = false; } if (flags & FUSE_OVER_IO_URING && fuse_uring_enabled()) - fuse_chan_io_uring_enable(fc->chan); + io_uring_enabled = true; if (flags & FUSE_REQUEST_TIMEOUT) timeout = arg->request_timeout; @@ -1433,6 +1434,7 @@ static void process_init_reply(struct fuse_args *args, int error) .minor = fc->minor, .max_write = fc->max_write, .max_pages = fc->max_pages, + .io_uring_enabled = io_uring_enabled, }; fuse_chan_set_initialized(fc->chan, &cp); } -- cgit v1.2.3 From ebed9ea5b469588c6074f3ed5b8d8ec63c4ccf48 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Fri, 14 Aug 2026 11:59:42 -0700 Subject: fuse: add FUSE_IO_URING_CMD_ADD_QUEUE fuse-over-io-uring queues are currently created lazily, as a side effect of the first FUSE_IO_URING_CMD_REGISTER command for a given qid. This ties queue creation to entry registration. Add a FUSE_IO_URING_CMD_ADD_QUEUE command so a server can create a queue explicitly, decoupling queue setup from entry registration. This is additionally a prerequisite for FUSE_IO_URING_CMD_ADD_BUFPOOL, which attaches a buffer pool to an existing queue and therefore needs the queue to have been created first. Reviewed-by: Bernd Schubert Signed-off-by: Joanne Koong Signed-off-by: Miklos Szeredi --- fs/fuse/dev_uring.c | 45 ++++++++++++++++++++++++++++++++++++++------- include/uapi/linux/fuse.h | 8 +++++++- 2 files changed, 45 insertions(+), 8 deletions(-) (limited to 'fs') diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 9616778505ba..3b9fd0daef66 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -281,7 +281,8 @@ void fuse_uring_conn_init(struct fuse_chan *fch) } static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, - int qid) + int qid, + bool fail_if_exists) { struct fuse_chan *fch = ring->chan; struct fuse_ring_queue *queue; @@ -289,11 +290,11 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, queue = kzalloc_obj(*queue, GFP_KERNEL_ACCOUNT); if (!queue) - return NULL; + return ERR_PTR(-ENOMEM); pq = fuse_pqueue_alloc(); if (!pq) { kfree(queue); - return NULL; + return ERR_PTR(-ENOMEM); } queue->qid = qid; @@ -316,7 +317,7 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, spin_unlock(&fch->lock); kfree(queue->fpq.processing); kfree(queue); - return ring->queues[qid]; + return fail_if_exists ? ERR_PTR(-EEXIST) : ring->queues[qid]; } /* @@ -1189,9 +1190,9 @@ static int fuse_uring_register(struct io_uring_cmd *cmd, queue = READ_ONCE(ring->queues[qid]); if (!queue) { - queue = fuse_uring_create_queue(ring, qid); - if (!queue) - return -ENOMEM; + queue = fuse_uring_create_queue(ring, qid, false); + if (IS_ERR(queue)) + return PTR_ERR(queue); } /* @@ -1206,6 +1207,30 @@ static int fuse_uring_register(struct io_uring_cmd *cmd, return fuse_uring_do_register(ent, cmd, issue_flags); } +static int fuse_uring_add_queue(struct io_uring_cmd *cmd, struct fuse_chan *fch) +{ + const struct fuse_uring_cmd_req *cmd_req = + io_uring_sqe128_cmd(cmd->sqe, struct fuse_uring_cmd_req); + struct fuse_ring *ring = smp_load_acquire(&fch->ring); + unsigned int qid = READ_ONCE(cmd_req->qid); + uint64_t flags = READ_ONCE(cmd_req->flags); + struct fuse_ring_queue *queue; + + if (!ring || flags) + return -EINVAL; + + if (qid >= ring->nr_queues) { + pr_info_ratelimited("fuse: Invalid ring qid %u\n", qid); + return -EINVAL; + } + + queue = fuse_uring_create_queue(ring, qid, true); + if (IS_ERR(queue)) + return PTR_ERR(queue); + + return 0; +} + /* * Entry function from io_uring to handle the given passthrough command * (op code IORING_OP_URING_CMD) @@ -1272,6 +1297,12 @@ int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) return err; } break; + case FUSE_IO_URING_CMD_ADD_QUEUE: + err = fuse_uring_add_queue(cmd, fch); + if (err) + pr_info_once("FUSE_IO_URING_CMD_ADD_QUEUE failed err=%d\n", + err); + return err; default: return -EINVAL; } diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h index c13e1f9a2f12..cfb055c0c764 100644 --- a/include/uapi/linux/fuse.h +++ b/include/uapi/linux/fuse.h @@ -240,6 +240,9 @@ * - add FUSE_COPY_FILE_RANGE_64 * - add struct fuse_copy_file_range_out * - add FUSE_NOTIFY_PRUNE + * + * 7.46 + * - add FUSE_IO_URING_CMD_ADD_QUEUE */ #ifndef _LINUX_FUSE_H @@ -275,7 +278,7 @@ #define FUSE_KERNEL_VERSION 7 /** Minor version number of this interface */ -#define FUSE_KERNEL_MINOR_VERSION 45 +#define FUSE_KERNEL_MINOR_VERSION 46 /** The node ID of the root inode */ #define FUSE_ROOT_ID 1 @@ -1292,6 +1295,9 @@ enum fuse_uring_cmd { /* commit fuse request result and fetch next request */ FUSE_IO_URING_CMD_COMMIT_AND_FETCH = 2, + + /* add a queue */ + FUSE_IO_URING_CMD_ADD_QUEUE = 3, }; /** -- cgit v1.2.3 From b45aaabc628bc7356e21bd2eb0c2ae9bdfa13894 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Fri, 14 Aug 2026 11:59:43 -0700 Subject: fuse: add io-uring buffer pools Right now, ents and buffers are tightly coupled in fuse io-uring where each entry has its own dedicated payload buffer, requiring N buffers for N entries where each buffer must be large enough to accomodate the maximum payload size. This is suboptimal as most request types (lookup, open, release, getattr, etc) require vastly less bytes than the maximum payload size and some requests (unlink, rmdir, fsync, flush, etc) do not require payload buffers at all. Instead of requiring a 1:1 coupling between ents and payload buffers, allow the server to pass in a buffer pool (a contiguous chunk of memory) that the kernel will use as it wishes for servicing ents/requests. Entries only reserve a "buffer" from the pool while actively processing a request that requires a payload buffer. This decoupling and letting the kernel delegate memory from the pool for requests allows the kernel to optimize memory usage and reduces the memory usage requirements needed to use fuse-over-io-uring. A pool is registered per queue with the new FUSE_IO_URING_CMD_ADD_BUFPOOL command. The server passes the pool's base address and length in fuse_uring_cmd_req.bufpool.{uaddr,len}. Internally, the kernel splits the region into buffers of ring->max_payload_sz bytes each (nr_bufs = pool len / max_payload_sz). A queue commits to a payload mode on first use: registering an entry that carries its own payload selects the legacy per-entry mode, while ADD_BUFPOOL selects pool mode. The two are mutually exclusive, so ADD_BUFPOOL must be issued before any payload-carrying entries are registered on that queue. The queue must have been created before the bufpool is added, through the FUSE_IO_URING_CMD_ADD_QUEUE command. The kernel tracks free buffers with a bitmap (a set bit marks a free buffer). On dispatch, a request that needs a payload claims a free buffer (find_first_bit + clear). A request that needs none claims nothing. The buffer's byte offset within the pool is reported to the server in the new fuse_uring_ent_in_out.offset field so that the server can locate the payload. On completion the buffer is returned to the pool or reused directly if the next request on that entry also has a payload. The FUSE_HAS_IO_URING_BUFPOOL flag advertises kernel support to the server for bufpools. Buffer pool request flow ~~~~~~~~~~~~~~~~~~~~~~~~ | Kernel | FUSE daemon | | | [request arrives] | | [claim a free pool buffer] | | >fuse_uring_select_buffer() | | [copy headers to ring] | | [copy payload to buffer] | | [report buffer offset in ent_in_out] | | >io_uring_cmd_done() | | | [read headers] | | [read/write payload at offset] | | [process request] | | >io_uring_submit() | | COMMIT_AND_FETCH | >fuse_uring_commit_fetch() | | [copy reply from ring] | | [return buffer to the pool] | | >fuse_uring_recycle_buffer() | Reviewed-by: Bernd Schubert Signed-off-by: Joanne Koong Signed-off-by: Miklos Szeredi --- fs/fuse/dev_uring.c | 239 ++++++++++++++++++++++++++++++++++++++++------ fs/fuse/dev_uring_i.h | 37 ++++++- fs/fuse/inode.c | 2 +- include/uapi/linux/fuse.h | 21 +++- 4 files changed, 269 insertions(+), 30 deletions(-) (limited to 'fs') diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 3b9fd0daef66..d300c7f441c4 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -9,6 +9,7 @@ #include "dev_uring_i.h" #include "fuse_trace.h" +#include #include #include @@ -41,6 +42,11 @@ enum fuse_uring_header_type { FUSE_URING_HEADER_RING_ENT, }; +static inline bool bufpool_enabled(struct fuse_ring_queue *queue) +{ + return queue->payload_mode == FUSE_PAYLOAD_BUFPOOL; +} + static void uring_cmd_set_ring_ent(struct io_uring_cmd *cmd, struct fuse_ring_ent *ring_ent) { @@ -222,6 +228,7 @@ void fuse_uring_destruct(struct fuse_chan *fch) } kfree(queue->fpq.processing); + kfree(queue->bufpool); kfree(queue); WRITE_ONCE(ring->queues[qid], NULL); } @@ -316,6 +323,7 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, if (ring->queues[qid]) { spin_unlock(&fch->lock); kfree(queue->fpq.processing); + kfree(queue->bufpool); kfree(queue); return fail_if_exists ? ERR_PTR(-EEXIST) : ring->queues[qid]; } @@ -646,13 +654,14 @@ static int copy_header_from_ring(struct fuse_ring_ent *ent, } static int setup_fuse_copy_state(struct fuse_copy_state *cs, - struct fuse_ring *ring, struct fuse_req *req, + struct fuse_req *req, struct fuse_ring_ent *ent, int dir, struct iov_iter *iter) { int err; - err = import_ubuf(dir, ent->payload, ring->max_payload_sz, iter); + err = import_ubuf(dir, ent->payload.iov_base, ent->payload.iov_len, + iter); if (err) { pr_info_ratelimited("fuse: Import of user buffer failed\n"); return err; @@ -666,8 +675,7 @@ static int setup_fuse_copy_state(struct fuse_copy_state *cs, return 0; } -static int fuse_uring_copy_from_ring(struct fuse_ring *ring, - struct fuse_req *req, +static int fuse_uring_copy_from_ring(struct fuse_req *req, struct fuse_ring_ent *ent) { struct fuse_copy_state cs; @@ -681,7 +689,7 @@ static int fuse_uring_copy_from_ring(struct fuse_ring *ring, if (err) return err; - err = setup_fuse_copy_state(&cs, ring, req, ent, ITER_SOURCE, &iter); + err = setup_fuse_copy_state(&cs, req, ent, ITER_SOURCE, &iter); if (err) return err; @@ -693,7 +701,7 @@ static int fuse_uring_copy_from_ring(struct fuse_ring *ring, /* * Copy data from the req to the ring buffer */ -static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, +static int fuse_uring_args_to_ring(struct fuse_req *req, struct fuse_ring_ent *ent) { struct fuse_copy_state cs; @@ -707,7 +715,7 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, .commit_id = req->in.h.unique, }; - err = setup_fuse_copy_state(&cs, ring, req, ent, ITER_DEST, &iter); + err = setup_fuse_copy_state(&cs, req, ent, ITER_DEST, &iter); if (err) return err; @@ -737,6 +745,10 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, } ent_in_out.payload_sz = cs.ring.copied_sz; + if (bufpool_enabled(ent->queue) && ent->payload.iov_base) + ent_in_out.offset = + (uintptr_t)ent->payload.iov_base - ent->queue->bufpool->base_uaddr; + return copy_header_to_ring(ent, FUSE_URING_HEADER_RING_ENT, &ent_in_out, sizeof(ent_in_out)); } @@ -745,7 +757,6 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, struct fuse_req *req) { struct fuse_ring_queue *queue = ent->queue; - struct fuse_ring *ring = queue->ring; int err; err = -EIO; @@ -760,7 +771,7 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, return err; /* copy the request */ - err = fuse_uring_args_to_ring(ring, req, ent); + err = fuse_uring_args_to_ring(req, ent); if (unlikely(err)) { pr_info_ratelimited("Copy to ring failed: %d\n", err); return err; @@ -771,6 +782,91 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, sizeof(req->in.h)); } +static bool fuse_uring_req_has_payload(struct fuse_req *req) +{ + struct fuse_args *args = req->args; + + return args->in_numargs > 1 || args->out_numargs; +} + +static int fuse_uring_select_buffer(struct fuse_ring_ent *ent) +{ + struct fuse_ring_queue *queue = ent->queue; + struct fuse_bufpool *pool = queue->bufpool; + unsigned int id; + + lockdep_assert_held(&queue->lock); + + id = find_first_bit(pool->free_map, pool->nr_bufs); + if (id >= pool->nr_bufs) + return -ENOBUFS; + + WARN_ON_ONCE(ent->payload.iov_base); + __clear_bit(id, pool->free_map); + + ent->buf_id = id; + ent->payload.iov_base = + (void __user *)(pool->base_uaddr + id * pool->buf_size); + ent->payload.iov_len = pool->buf_size; + + return 0; +} + +static void fuse_uring_recycle_buffer(struct fuse_ring_ent *ent) +{ + struct iovec *ent_payload = &ent->payload; + struct fuse_ring_queue *queue = ent->queue; + struct fuse_bufpool *pool; + + lockdep_assert_held(&queue->lock); + + if (!bufpool_enabled(queue) || !ent_payload->iov_base) + return; + + pool = queue->bufpool; + + /* a buffer should never be recycled twice */ + WARN_ON_ONCE(test_bit(ent->buf_id, pool->free_map)); + __set_bit(ent->buf_id, pool->free_map); + + memset(ent_payload, 0, sizeof(*ent_payload)); + ent->buf_id = 0; +} + +static int fuse_uring_next_req_update_buffer(struct fuse_ring_ent *ent, + struct fuse_req *req) +{ + bool buffer_selected; + bool has_payload; + + if (!bufpool_enabled(ent->queue)) + return 0; + + buffer_selected = !!ent->payload.iov_base; + has_payload = fuse_uring_req_has_payload(req); + + if (has_payload && !buffer_selected) + return fuse_uring_select_buffer(ent); + + if (!has_payload && buffer_selected) + fuse_uring_recycle_buffer(ent); + + return 0; +} + +static int fuse_uring_prep_buffer(struct fuse_ring_ent *ent, + struct fuse_req *req) +{ + if (!bufpool_enabled(ent->queue)) + return 0; + + /* no payload to copy, can skip selecting a buffer */ + if (!fuse_uring_req_has_payload(req)) + return 0; + + return fuse_uring_select_buffer(ent); +} + static int fuse_uring_prepare_send(struct fuse_ring_ent *ent, struct fuse_req *req) { @@ -858,9 +954,12 @@ static struct fuse_req *fuse_uring_ent_assign_req(struct fuse_ring_ent *ent) /* get and assign the next entry while it is still holding the lock */ req = list_first_entry_or_null(req_queue, struct fuse_req, list); - if (req) - fuse_uring_add_req_to_ring_ent(ent, req); + if (!req || fuse_uring_next_req_update_buffer(ent, req)) { + fuse_uring_recycle_buffer(ent); + return NULL; + } + fuse_uring_add_req_to_ring_ent(ent, req); return req; } @@ -872,7 +971,6 @@ static struct fuse_req *fuse_uring_ent_assign_req(struct fuse_ring_ent *ent) static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req, unsigned int issue_flags) { - struct fuse_ring *ring = ent->queue->ring; ssize_t err = -EFAULT; if (copy_header_from_ring(ent, FUSE_URING_HEADER_IN_OUT, &req->out.h, @@ -885,7 +983,7 @@ static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req, goto out; } - err = fuse_uring_copy_from_ring(ring, req, ent); + err = fuse_uring_copy_from_ring(req, ent); out: fuse_uring_req_end(ent, req, err); } @@ -1004,6 +1102,7 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, if (err != 0) { pr_info_ratelimited("qid=%d commit_id %llu state %d", queue->qid, commit_id, ent->state); + fuse_uring_recycle_buffer(ent); spin_unlock(&queue->lock); fuse_uring_req_end(ent, req, err); return err; @@ -1021,6 +1120,11 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, * fuse requests would otherwise not get processed - committing * and fetching is done in one step vs legacy fuse, which has separated * read (fetch request) and write (commit result). + * + * If there is no next request or if all buffers are busy (if using a + * bufpool), the cmd is not returned to userspace. The entry is left + * available and the cmd only returns to userspace when there's a + * next request and an available buffer. */ if (fuse_uring_get_next_fuse_req(ent, queue)) fuse_uring_send(ent, cmd, 0, issue_flags); @@ -1145,11 +1249,23 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, } payload = &iov[FUSE_URING_IOV_PAYLOAD]; - if (payload->iov_len < ring->max_payload_sz) { - pr_info_ratelimited("Invalid req payload len %zu\n", - payload->iov_len); - return ERR_PTR(err); + + spin_lock(&queue->lock); + if (bufpool_enabled(queue)) { + if (payload->iov_base || payload->iov_len) { + spin_unlock(&queue->lock); + return ERR_PTR(err); + } + } else { + if (payload->iov_len < ring->max_payload_sz) { + pr_info_ratelimited("Invalid req payload len %zu\n", + payload->iov_len); + spin_unlock(&queue->lock); + return ERR_PTR(err); + } + queue->payload_mode = FUSE_PAYLOAD_PER_ENT; } + spin_unlock(&queue->lock); err = -ENOMEM; ent = kzalloc_obj(*ent, GFP_KERNEL_ACCOUNT); @@ -1160,7 +1276,8 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, ent->queue = queue; ent->headers = headers->iov_base; - ent->payload = payload->iov_base; + if (queue->payload_mode == FUSE_PAYLOAD_PER_ENT) + ent->payload = *payload; atomic_inc(&ring->queue_refs); return ent; @@ -1231,6 +1348,67 @@ static int fuse_uring_add_queue(struct io_uring_cmd *cmd, struct fuse_chan *fch) return 0; } +static int fuse_uring_add_bufpool(struct io_uring_cmd *cmd, + struct fuse_chan *fch) +{ + const struct fuse_uring_cmd_req *cmd_req = + io_uring_sqe128_cmd(cmd->sqe, struct fuse_uring_cmd_req); + unsigned int qid = READ_ONCE(cmd_req->qid); + uint64_t flags = READ_ONCE(cmd_req->flags); + /* paired with the smp_store_release() in fuse_uring_create */ + struct fuse_ring *ring = smp_load_acquire(&fch->ring); + struct fuse_ring_queue *queue; + struct fuse_bufpool *pool; + uintptr_t pool_uaddr; + unsigned int pool_len, nr_bufs; + size_t pool_size, buf_size; + + if (!ring || qid >= ring->nr_queues || flags) + return -EINVAL; + + /* reserved for future use, must be zero */ + if (READ_ONCE(cmd_req->bufpool.reserved)) + return -EINVAL; + + /* Pairs with smp_store_release() in fuse_uring_create_queue() */ + queue = smp_load_acquire(&ring->queues[qid]); + if (!queue) + return -EINVAL; + + pool_uaddr = READ_ONCE(cmd_req->bufpool.uaddr); + pool_len = READ_ONCE(cmd_req->bufpool.len); + + /* each buffer holds the max payload size */ + buf_size = queue->ring->max_payload_sz; + + nr_bufs = pool_len / buf_size; + if (!nr_bufs) + return -EINVAL; + + pool_size = struct_size(pool, free_map, BITS_TO_LONGS(nr_bufs)); + pool = kzalloc(pool_size, GFP_KERNEL_ACCOUNT); + if (!pool) + return -ENOMEM; + + pool->base_uaddr = pool_uaddr; + pool->buf_size = buf_size; + pool->nr_bufs = nr_bufs; + /* all buffers are free */ + bitmap_set(pool->free_map, 0, nr_bufs); + + spin_lock(&queue->lock); + if (queue->payload_mode != FUSE_PAYLOAD_UNSET) { + spin_unlock(&queue->lock); + kfree(pool); + return -EINVAL; + } + queue->bufpool = pool; + queue->payload_mode = FUSE_PAYLOAD_BUFPOOL; + spin_unlock(&queue->lock); + + return 0; +} + /* * Entry function from io_uring to handle the given passthrough command * (op code IORING_OP_URING_CMD) @@ -1303,6 +1481,12 @@ int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) pr_info_once("FUSE_IO_URING_CMD_ADD_QUEUE failed err=%d\n", err); return err; + case FUSE_IO_URING_CMD_ADD_BUFPOOL: + err = fuse_uring_add_bufpool(cmd, fch); + if (err) + pr_info_once("FUSE_IO_URING_ADD_BUFPOOL failed err=%d\n", + err); + return err; default: return -EINVAL; } @@ -1336,6 +1520,7 @@ static void fuse_uring_send_in_task(struct io_tw_req tw_req, io_tw_token_t tw) spin_lock(&queue->lock); list_del_init(&ent->list); + fuse_uring_recycle_buffer(ent); spin_unlock(&queue->lock); io_uring_cmd_done(cmd, err, issue_flags); @@ -1397,15 +1582,16 @@ void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req) req->ring_queue = queue; ent = list_first_entry_or_null(&queue->ent_avail_queue, struct fuse_ring_ent, list); - if (ent) - fuse_uring_add_req_to_ring_ent(ent, req); - else - list_add_tail(&req->list, &queue->fuse_req_queue); - spin_unlock(&queue->lock); - if (ent) - fuse_uring_dispatch_ent(ent); + if (!ent || fuse_uring_prep_buffer(ent, req)) { + list_add_tail(&req->list, &queue->fuse_req_queue); + spin_unlock(&queue->lock); + return; + } + fuse_uring_add_req_to_ring_ent(ent, req); + spin_unlock(&queue->lock); + fuse_uring_dispatch_ent(ent); return; err_unlock: @@ -1453,10 +1639,9 @@ bool fuse_uring_queue_bq_req(struct fuse_req *req) */ req = list_first_entry_or_null(&queue->fuse_req_queue, struct fuse_req, list); - if (ent && req) { + if (ent && req && !fuse_uring_prep_buffer(ent, req)) { fuse_uring_add_req_to_ring_ent(ent, req); spin_unlock(&queue->lock); - fuse_uring_dispatch_ent(ent); } else { spin_unlock(&queue->lock); diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h index d721a4fc0215..cdf56f8b38b5 100644 --- a/fs/fuse/dev_uring_i.h +++ b/fs/fuse/dev_uring_i.h @@ -7,6 +7,8 @@ #ifndef _FS_FUSE_DEV_URING_I_H #define _FS_FUSE_DEV_URING_I_H +#include + #include "fuse_dev_i.h" #ifdef CONFIG_FUSE_IO_URING @@ -36,11 +38,38 @@ enum fuse_ring_req_state { FRRS_RELEASED, }; +/* how a queue's payload buffers are provided */ +enum fuse_queue_payload_mode { + /* not yet committed (a bufpool may still be added) */ + FUSE_PAYLOAD_UNSET = 0, + /* each entry registers its own payload buffer */ + FUSE_PAYLOAD_PER_ENT, + /* each entry's payload buffer is assigned from a bufpool */ + FUSE_PAYLOAD_BUFPOOL, +}; + +struct fuse_bufpool { + /* starting uaddr of the bufpool */ + uintptr_t base_uaddr; + + /* size of each buffer in the pool */ + size_t buf_size; + + /* total number of buffers in the pool */ + unsigned int nr_bufs; + + /* bitmap tracking which buffers are free */ + unsigned long free_map[]; +}; + /** A fuse ring entry, part of the ring queue */ struct fuse_ring_ent { /* userspace buffer */ struct fuse_uring_req_header __user *headers; - void __user *payload; + struct iovec payload; + + /* buffer id in the pool, if bufpools are used. ignored otherwise */ + unsigned int buf_id; /* the ring queue that owns the request */ struct fuse_ring_queue *queue; @@ -99,6 +128,12 @@ struct fuse_ring_queue { unsigned int active_background; bool stopped; + + /* how this queue's payload buffers are provided */ + enum fuse_queue_payload_mode payload_mode; + + /* only allocated when payload_mode == FUSE_PAYLOAD_BUFPOOL */ + struct fuse_bufpool *bufpool; }; /* diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 33773c7d129a..9779adc98593 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -1482,7 +1482,7 @@ static struct fuse_init_args *fuse_new_init(struct fuse_mount *fm) * the reply - server is either sending IORING_OP_URING_CMD or not. */ if (fuse_uring_enabled()) - flags |= FUSE_OVER_IO_URING; + flags |= FUSE_OVER_IO_URING | FUSE_HAS_IO_URING_BUFPOOL; ia->in.flags = flags; ia->in.flags2 = flags >> 32; diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h index cfb055c0c764..538d844da099 100644 --- a/include/uapi/linux/fuse.h +++ b/include/uapi/linux/fuse.h @@ -243,6 +243,9 @@ * * 7.46 * - add FUSE_IO_URING_CMD_ADD_QUEUE + * - add FUSE_HAS_IO_URING_BUFPOOL + * - add fuse_uring_cmd_req bufpool struct + * - add bufpool offset field to fuse_uring_ent_in_out struct */ #ifndef _LINUX_FUSE_H @@ -451,6 +454,7 @@ struct fuse_file_lock { * FUSE_OVER_IO_URING: Indicate that client supports io-uring * FUSE_REQUEST_TIMEOUT: kernel supports timing out requests. * init_out.request_timeout contains the timeout (in secs) + * FUSE_HAS_IO_URING_BUFPOOL: kernel supports io-uring buffer pools */ #define FUSE_ASYNC_READ (1 << 0) #define FUSE_POSIX_LOCKS (1 << 1) @@ -498,6 +502,7 @@ struct fuse_file_lock { #define FUSE_ALLOW_IDMAP (1ULL << 40) #define FUSE_OVER_IO_URING (1ULL << 41) #define FUSE_REQUEST_TIMEOUT (1ULL << 42) +#define FUSE_HAS_IO_URING_BUFPOOL (1ULL << 43) /** * CUSE INIT request/reply flags @@ -1266,7 +1271,9 @@ struct fuse_uring_ent_in_out { /* size of user payload buffer */ uint32_t payload_sz; - uint32_t padding; + + /* Offset into the bufpool, if bufpools are used */ + uint32_t offset; uint64_t reserved; }; @@ -1298,6 +1305,9 @@ enum fuse_uring_cmd { /* add a queue */ FUSE_IO_URING_CMD_ADD_QUEUE = 3, + + /* add a bufpool to a queue */ + FUSE_IO_URING_CMD_ADD_BUFPOOL = 4, }; /** @@ -1312,6 +1322,15 @@ struct fuse_uring_cmd_req { /* queue the command is for (queue index) */ uint16_t qid; uint8_t padding[6]; + + union { + struct { + /* base address of bufpool */ + uint64_t uaddr; + uint32_t len; + uint32_t reserved; + } bufpool; + }; }; #endif /* _LINUX_FUSE_H */ -- cgit v1.2.3 From 96caf2496e15b3b12e1e4f3ac592648291333ecb Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Fri, 14 Aug 2026 11:59:44 -0700 Subject: fuse: support registered buffer pools in io-uring Allow servers to use a buffer pool that is also registered through io-uring. When the server registers a buffer pool with io-uring, the pages backing the pool are pinned upfront. This eliminates the overhead of pinning/unpinning user pages and translating virtual addresses per i/o request. This also allows servers to use the same registered memory for subsequent backing store I/O (eg read_fixed/write_fixed), keeping data in the same pinned pages without additional pinning or mapping overhead required. To use this, the server needs to set the FUSE_URING_REGISTERED_BUFPOOL flag when adding a bufpool through the FUSE_IO_URING_CMD_ADD_BUFPOOL cmd. For every sqe submitted (including the one for adding the bufpool), it should set sqe->uring_cmd_flags to include IORING_URING_CMD_FIXED, and pass in the index where the registered bufpool resides to sqe->buf_index. Benchmarked with passthrough_hp (--nopassthrough, q_depth=8) on a 2-socket Intel Xeon Gold 6138 (40 cores / 80 threads), using fio (sync engine, bs=1M, O_DIRECT, numjobs=2, 30s run + 10s ramp, 3 runs) where direct-I/O throughput is against a RAM-backed (tmpfs) source (backing I/O is not the bottleneck): baseline registered buffers direct read ~5.1 GB/s ~5.4 GB/s (+~5%) direct write ~3.4 GB/s ~4.8 GB/s (+~45%) Registered buffers bring up the write path speed up closer to speed of reads. There isn't much improvement for reads because it is already fast enough where it's at the copy-bound ceiling (surpassing that requires doing zero-copy). On a device-bound NVMe though, the differences are within noise, as backing I/O dominates per-request latency. Signed-off-by: Joanne Koong Reviewed-by: Bernd Schubert Signed-off-by: Miklos Szeredi --- fs/fuse/dev_uring.c | 112 ++++++++++++++++++++++++++++++++++++++++---------- fs/fuse/dev_uring_i.h | 8 ++++ 2 files changed, 99 insertions(+), 21 deletions(-) (limited to 'fs') diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index d300c7f441c4..17806da93039 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -47,6 +47,27 @@ static inline bool bufpool_enabled(struct fuse_ring_queue *queue) return queue->payload_mode == FUSE_PAYLOAD_BUFPOOL; } +static inline bool bufpool_registered(struct fuse_ring_queue *queue) +{ + return queue->bufpool && queue->bufpool->registered; +} + +/* + * For a registered bufpool, every sqe that drives a payload import (REGISTER, + * COMMIT_AND_FETCH) must carry the registered buffer index of the pool. + * This also must be called from the command's issue handler, where cmd->sqe is + * still valid + */ +static inline bool fuse_uring_cmd_index_ok(struct io_uring_cmd *cmd, + struct fuse_ring_queue *queue) +{ + if (!bufpool_registered(queue)) + return true; + + return (cmd->flags & IORING_URING_CMD_FIXED) && + READ_ONCE(cmd->sqe->buf_index) == queue->bufpool->registered_index; +} + static void uring_cmd_set_ring_ent(struct io_uring_cmd *cmd, struct fuse_ring_ent *ring_ent) { @@ -653,19 +674,42 @@ static int copy_header_from_ring(struct fuse_ring_ent *ent, return 0; } +static int fuse_uring_import_payload(struct fuse_ring_ent *ent, int dir, + struct iov_iter *iter, + unsigned int issue_flags) +{ + void __user *base = ent->payload.iov_base; + size_t len = ent->payload.iov_len; + int err = 0; + + if (!base) { + memset(iter, 0, sizeof(*iter)); + return 0; + } + + if (bufpool_registered(ent->queue)) + err = io_uring_cmd_import_fixed((u64)(uintptr_t)base, len, dir, + iter, ent->cmd, issue_flags); + else + err = import_ubuf(dir, base, len, iter); + + if (err) + pr_info_ratelimited("fuse: Import of user buffer failed\n"); + + return err; +} + static int setup_fuse_copy_state(struct fuse_copy_state *cs, struct fuse_req *req, struct fuse_ring_ent *ent, int dir, - struct iov_iter *iter) + struct iov_iter *iter, + unsigned int issue_flags) { int err; - err = import_ubuf(dir, ent->payload.iov_base, ent->payload.iov_len, - iter); - if (err) { - pr_info_ratelimited("fuse: Import of user buffer failed\n"); + err = fuse_uring_import_payload(ent, dir, iter, issue_flags); + if (err) return err; - } fuse_copy_init(cs, dir == ITER_DEST, iter); @@ -676,7 +720,8 @@ static int setup_fuse_copy_state(struct fuse_copy_state *cs, } static int fuse_uring_copy_from_ring(struct fuse_req *req, - struct fuse_ring_ent *ent) + struct fuse_ring_ent *ent, + unsigned int issue_flags) { struct fuse_copy_state cs; struct fuse_args *args = req->args; @@ -689,7 +734,8 @@ static int fuse_uring_copy_from_ring(struct fuse_req *req, if (err) return err; - err = setup_fuse_copy_state(&cs, req, ent, ITER_SOURCE, &iter); + err = setup_fuse_copy_state(&cs, req, ent, ITER_SOURCE, &iter, + issue_flags); if (err) return err; @@ -702,7 +748,8 @@ static int fuse_uring_copy_from_ring(struct fuse_req *req, * Copy data from the req to the ring buffer */ static int fuse_uring_args_to_ring(struct fuse_req *req, - struct fuse_ring_ent *ent) + struct fuse_ring_ent *ent, + unsigned int issue_flags) { struct fuse_copy_state cs; struct fuse_args *args = req->args; @@ -715,7 +762,8 @@ static int fuse_uring_args_to_ring(struct fuse_req *req, .commit_id = req->in.h.unique, }; - err = setup_fuse_copy_state(&cs, req, ent, ITER_DEST, &iter); + err = setup_fuse_copy_state(&cs, req, ent, ITER_DEST, &iter, + issue_flags); if (err) return err; @@ -754,7 +802,8 @@ static int fuse_uring_args_to_ring(struct fuse_req *req, } static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, - struct fuse_req *req) + struct fuse_req *req, + unsigned int issue_flags) { struct fuse_ring_queue *queue = ent->queue; int err; @@ -771,7 +820,7 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, return err; /* copy the request */ - err = fuse_uring_args_to_ring(req, ent); + err = fuse_uring_args_to_ring(req, ent, issue_flags); if (unlikely(err)) { pr_info_ratelimited("Copy to ring failed: %d\n", err); return err; @@ -868,11 +917,12 @@ static int fuse_uring_prep_buffer(struct fuse_ring_ent *ent, } static int fuse_uring_prepare_send(struct fuse_ring_ent *ent, - struct fuse_req *req) + struct fuse_req *req, + unsigned int issue_flags) { int err; - err = fuse_uring_copy_to_ring(ent, req); + err = fuse_uring_copy_to_ring(ent, req, issue_flags); if (!err) { set_bit(FR_SENT, &req->flags); trace_fuse_request_sent(req); @@ -983,7 +1033,7 @@ static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req, goto out; } - err = fuse_uring_copy_from_ring(req, ent); + err = fuse_uring_copy_from_ring(req, ent, issue_flags); out: fuse_uring_req_end(ent, req, err); } @@ -995,7 +1045,8 @@ out: * Else, there is no next fuse request and this returns false. */ static bool fuse_uring_get_next_fuse_req(struct fuse_ring_ent *ent, - struct fuse_ring_queue *queue) + struct fuse_ring_queue *queue, + unsigned int issue_flags) { int err; struct fuse_req *req; @@ -1007,7 +1058,7 @@ retry: spin_unlock(&queue->lock); if (req) { - err = fuse_uring_prepare_send(ent, req); + err = fuse_uring_prepare_send(ent, req, issue_flags); if (err) goto retry; } @@ -1081,6 +1132,11 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, return err; } + if (!fuse_uring_cmd_index_ok(cmd, queue)) { + spin_unlock(&queue->lock); + return -EINVAL; + } + /* Find a request based on the unique ID of the fuse request * This should get revised, as it needs a hash calculation and list * search. And full struct fuse_pqueue is needed (memory overhead). @@ -1126,7 +1182,7 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, * available and the cmd only returns to userspace when there's a * next request and an available buffer. */ - if (fuse_uring_get_next_fuse_req(ent, queue)) + if (fuse_uring_get_next_fuse_req(ent, queue, issue_flags)) fuse_uring_send(ent, cmd, 0, issue_flags); return 0; } @@ -1252,7 +1308,8 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, spin_lock(&queue->lock); if (bufpool_enabled(queue)) { - if (payload->iov_base || payload->iov_len) { + if (payload->iov_base || payload->iov_len || + !fuse_uring_cmd_index_ok(cmd, queue)) { spin_unlock(&queue->lock); return ERR_PTR(err); } @@ -1362,6 +1419,7 @@ static int fuse_uring_add_bufpool(struct io_uring_cmd *cmd, uintptr_t pool_uaddr; unsigned int pool_len, nr_bufs; size_t pool_size, buf_size; + bool registered = cmd->flags & IORING_URING_CMD_FIXED; if (!ring || qid >= ring->nr_queues || flags) return -EINVAL; @@ -1396,6 +1454,17 @@ static int fuse_uring_add_bufpool(struct io_uring_cmd *cmd, /* all buffers are free */ bitmap_set(pool->free_map, 0, nr_bufs); + /* + * A registered bufpool is reached through an io_uring fixed buffer, so + * the pool is registered iff this command was submitted with + * IORING_URING_CMD_FIXED. The registered buffer index is taken from + * sqe->buf_index. + */ + if (registered) { + pool->registered = true; + pool->registered_index = READ_ONCE(cmd->sqe->buf_index); + } + spin_lock(&queue->lock); if (queue->payload_mode != FUSE_PAYLOAD_UNSET) { spin_unlock(&queue->lock); @@ -1508,9 +1577,10 @@ static void fuse_uring_send_in_task(struct io_tw_req tw_req, io_tw_token_t tw) int err; if (!tw.cancel) { - err = fuse_uring_prepare_send(ent, ent->fuse_req); + err = fuse_uring_prepare_send(ent, ent->fuse_req, issue_flags); if (err) { - if (!fuse_uring_get_next_fuse_req(ent, queue)) + if (!fuse_uring_get_next_fuse_req(ent, queue, + issue_flags)) return; err = 0; } diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h index cdf56f8b38b5..e142cae43022 100644 --- a/fs/fuse/dev_uring_i.h +++ b/fs/fuse/dev_uring_i.h @@ -49,6 +49,14 @@ enum fuse_queue_payload_mode { }; struct fuse_bufpool { + bool registered; + + /* + * io_uring registered buffer table index for this pool, bound at + * ADD_BUFPOOL time. Only valid if the bufpool is registered + */ + u16 registered_index; + /* starting uaddr of the bufpool */ uintptr_t base_uaddr; -- cgit v1.2.3 From 43f8343858eb942d7f7c49964b31c54dcc314890 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Fri, 14 Aug 2026 11:59:45 -0700 Subject: fuse: add zero-copy over io-uring Implement zero-copy in fuse io-uring to eliminate memory copies between the application, kernel, and server for read/write operations. The server can directly access client pages or page cache folios without copying data through an intermediary buffer. When a fuse request arrives, the kernel registers the relevant pages into a sparse slot in the server's io_uring registered buffer table. The server can then operate on these pages directly using io-uring fixed buffer operations (eg read_fixed/write_fixed) and the kernel unregisters these pages when the request completes. Non-page-backed args (eg op out headers) will go through the payload buffer as normal. The server can specify which open files should have their reads/writes go through zero-copy, by setting the FOPEN_IO_URING_ZERO_COPY flag when servicing opens. This requires CAP_SYS_ADMIN and bufpools. This is gated behind CAP_SYS_ADMIN because zero-copy allows the server direct access to the client's underlying pages, rather than operating on an intermediary buffer that the contents of the client's pages were copied into or on page cache folios. The request flow for the zero-copy direct-io write path (client writes data, server reads it) is as follows: ======================================================================= | Kernel | FUSE server | | | "write(fd, buf, 1MB)" | | | | >sys_write() | | >fuse_file_write_iter() | | >fuse_send_one() | | [req->args->in_pages = true] | | [folios hold client write data] | | | | >fuse_uring_copy_to_ring() | | >copy_header_to_ring(IN_OUT) | | [memcpy fuse_in_header] | | >copy_header_to_ring(OP) | | [memcpy write_in header] | | | | >fuse_uring_args_to_ring() | | >setup_fuse_copy_state() | | [skip_folio_copy = true] | | | | >fuse_uring_set_up_zero_copy() | | [folio_get for each client folio] | | [build bio_vec array from folios] | | >io_buffer_register_bvec() | | [register pages at ent->zero_copy_index] | | [ent->zero_copied = true] | | | | >fuse_copy_args() | | [skip_folio_copy => return 0 | | for page arg, skip data copy] | | | | >copy_header_to_ring(RING_ENT) | | [memcpy ent_in_out] | | >io_uring_cmd_done() | | | | | [CQE received] | | | | [issue io_uring READ at | | ent->zero_copy_index] | | [reads directly from | |client's pages (ZERO_COPY)] | | | | [write data to backing | | store] | | [submit COMMIT AND FETCH] | | | >fuse_uring_commit_fetch() | | >fuse_uring_commit() | | >fuse_uring_copy_from_ring() | | >fuse_uring_req_end() | | >io_buffer_unregister(ent->zero_copy_index) | | [unregister pages from index] | | >fuse_zero_copy_release() | | [folio_put for each folio] | | [ent->zero_copied = false] | | >fuse_request_end() | | [wake up client] | The zero-copy read path is analogous. Some requests may have both page-backed args and non-page-backed args. For these requests, the page-backed args are zero-copied while the non-page-backed args are copied to the buffer selected from the buffer pool: zero-copy: pages registered via io_buffer_register_bvec() non-page-backed: copied to payload buffer via fuse_copy_args() For a request whose payload is zero-copied, the registration/unregistration path looks like: register: fuse_uring_set_up_zero_copy() folio_get() for each folio io_buffer_register_bvec(ent->zero_copy_index) unregister: fuse_uring_req_end() io_buffer_unregister(ent->zero_copy_index) -> fuse_zero_copy_release() callback folio_put() for each folio Please note that on abort for in-flight zero-copied requests that have been sent to userspace, the registered bvec slot remains occupied and its folios remain pinned until the io-uring ring is destroyed, at which point io-uring unregisters all buffers and the fuse_zero_copy_release() callback drops the folio references. Unregistering at teardown would require operating on the ring context directly, whose validity is hard to ascertain; this is deemed not worth the complexity for the abort race, since everything is freed when the ring is torn down. The throughput improvement from zero-copy depends on how much of the per-request latency is spent on data copying vs backing I/O. The gain comes from eliminating the payload-buffer memcpy, but accessing the zero-copied pages requires the server to issue the read/write as an IORING_OP_READ/WRITE_FIXED operation. The benefit is largest when the mempcy is a meaningful fraction of per-request latency while backing i/o is still noticable enough that the extra io-uring op's overhead doesn't dominate. Benchmarked with passthrough_hp (--nopassthrough, q_depth=8) on a 2-socket Intel Xeon Gold 6138 (40 cores / 80 threads), using fio (sync engine, bs=1M, O_DIRECT, numjobs=2, 30s run + 10s ramp, 3 runs) where direct-I/O throughput is against a RAM-backed (tmpfs) source (backing I/O is not the bottleneck): baseline registered-buf zero-copy (zc vs base) direct read ~5.1 GB/s ~5.4 GB/s ~8.9 GB/s (+75%) direct write ~3.4 GB/s ~4.8 GB/s ~5.1 GB/s (+50%) Reads end up higher than writes because the backing store reads faster than it writes (the baseline shows the same read>write gap, and the raw device does too). On a device-bound NVMe (~2 GB/s reads) the read gain shrinks to ~10-16% (and no measurable gains for writes), as backing I/O rather than the eliminated copy dominates latency. The benefit overall scales with how much of the per-request latency is the data copy versus backing I/O. Signed-off-by: Joanne Koong Reviewed-by: Bernd Schubert Signed-off-by: Miklos Szeredi --- fs/fuse/args.h | 2 + fs/fuse/dev.c | 24 +++++- fs/fuse/dev_uring.c | 182 ++++++++++++++++++++++++++++++++++++++++++---- fs/fuse/dev_uring_i.h | 6 ++ fs/fuse/file.c | 2 + fs/fuse/fuse_dev_i.h | 2 + include/uapi/linux/fuse.h | 34 +++++++++ 7 files changed, 236 insertions(+), 16 deletions(-) (limited to 'fs') diff --git a/fs/fuse/args.h b/fs/fuse/args.h index ecfe51a192af..5173264a1261 100644 --- a/fs/fuse/args.h +++ b/fs/fuse/args.h @@ -42,6 +42,8 @@ struct fuse_args { bool is_pinned:1; bool invalidate_vmap:1; bool abort_on_kill:1; + /* server requested io-uring zero-copy for this op */ + bool zero_copy:1; struct fuse_in_arg in_args[4]; struct fuse_arg out_args[2]; void (*end)(struct fuse_args *args, int error); diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index d8f97943e973..90dceb7da571 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -1248,11 +1248,25 @@ int fuse_copy_folio(struct fuse_copy_state *cs, struct folio **foliop, if (folio) { size = folio_size(folio); - if (zeroing && count < size) - folio_zero_range(folio, 0, size); + if (zeroing && count < size) { + /* + * When the copy is skipped the folio already holds the + * payload, so only the bytes outside [offset, offset + + * count) may be zeroed. + * + * Otherwise, the whole folio is cleared first so that a + * failed copy leaves zeros rather than stale folio + * contents. + */ + if (cs->skip_folio_copy) + folio_zero_segments(folio, 0, offset, + offset + count, size); + else + folio_zero_range(folio, 0, size); + } } - while (count) { + while (!cs->skip_folio_copy && count) { if (cs->write && cs->pipebufs && folio) { /* * Can't control lifetime of pipe buffers, so always @@ -1345,6 +1359,10 @@ int fuse_copy_args(struct fuse_copy_state *cs, unsigned numargs, for (i = 0; !err && i < numargs; i++) { struct fuse_arg *arg = &args[i]; if (i == numargs - 1 && argpages) + /* + * if cs->skip_folio_copy is set, this just does any + * needed zeroing. No copying is involved. + */ err = fuse_copy_folios(cs, arg->size, zeroing); else err = fuse_copy_one(cs, arg->value, arg->size); diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 17806da93039..1547a4f0d9b9 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -22,6 +22,8 @@ MODULE_PARM_DESC(enable_uring, #define FUSE_URING_IOV_HEADERS 0 #define FUSE_URING_IOV_PAYLOAD 1 +#define FUSE_URING_ADD_QUEUE_FLAGS (FUSE_URING_ZERO_COPY) + bool fuse_uring_enabled(void) { return enable_uring; @@ -31,6 +33,11 @@ struct fuse_uring_pdu { struct fuse_ring_ent *ent; }; +struct fuse_zero_copy_bvs { + unsigned int nr_bvs; + struct bio_vec bvs[]; +}; + static const struct fuse_iqueue_ops fuse_io_uring_ops; enum fuse_uring_header_type { @@ -113,8 +120,36 @@ static void fuse_uring_flush_bg(struct fuse_ring_queue *queue) } } +static bool can_zero_copy_req(struct fuse_ring_ent *ent, struct fuse_req *req) +{ + struct fuse_args *args = req->args; + + if (!ent->queue->zero_copy || !args->zero_copy) + return false; + + if (args->opcode != FUSE_READ && args->opcode != FUSE_WRITE) + return false; + + return args->in_pages || args->out_pages; +} + +static void zero_copy_unregister(struct io_uring_cmd *cmd, + struct fuse_ring_ent *ent, + unsigned int issue_flags) +{ + if (ent->zero_copied) { + int err = io_buffer_unregister(cmd, ent->zero_copy_index, + issue_flags); + + if (err) + pr_warn_ratelimited("qid=%d zero-copy unregister failed: %d\n", + ent->queue->qid, err); + ent->zero_copied = false; + } +} + static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req, - int error) + int error, unsigned int issue_flags) { struct fuse_ring_queue *queue = ent->queue; struct fuse_ring *ring = queue->ring; @@ -134,6 +169,8 @@ static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req, spin_unlock(&queue->lock); + zero_copy_unregister(ent->cmd, ent, issue_flags); + if (error) req->out.h.error = error; @@ -309,7 +346,7 @@ void fuse_uring_conn_init(struct fuse_chan *fch) } static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, - int qid, + int qid, bool zero_copy, bool fail_if_exists) { struct fuse_chan *fch = ring->chan; @@ -328,6 +365,7 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, queue->qid = qid; queue->ring = ring; spin_lock_init(&queue->lock); + queue->zero_copy = zero_copy; INIT_LIST_HEAD(&queue->ent_avail_queue); INIT_LIST_HEAD(&queue->ent_commit_queue); @@ -713,6 +751,9 @@ static int setup_fuse_copy_state(struct fuse_copy_state *cs, fuse_copy_init(cs, dir == ITER_DEST, iter); + if (ent->zero_copied) + cs->skip_folio_copy = true; + cs->is_uring = true; cs->req = req; @@ -744,6 +785,62 @@ static int fuse_uring_copy_from_ring(struct fuse_req *req, return err; } +static void fuse_zero_copy_release(void *priv) +{ + struct fuse_zero_copy_bvs *zc_bvs = priv; + unsigned int i; + + for (i = 0; i < zc_bvs->nr_bvs; i++) + folio_put(page_folio(zc_bvs->bvs[i].bv_page)); + + kvfree(zc_bvs); +} + +static int fuse_uring_set_up_zero_copy(struct fuse_ring_ent *ent, + struct fuse_req *req, + unsigned int issue_flags) +{ + struct fuse_args_pages *ap; + int err, i, ddir = 0; + struct fuse_zero_copy_bvs *zc_bvs; + struct bio_vec *bvs; + + /* out_pages indicates a read, in_pages indicates a write */ + if (req->args->out_pages) + ddir |= IO_BUF_DEST; + if (req->args->in_pages) + ddir |= IO_BUF_SOURCE; + + ap = container_of(req->args, typeof(*ap), args); + + zc_bvs = kvmalloc_flex(*zc_bvs, bvs, ap->num_folios, + GFP_KERNEL_ACCOUNT); + if (!zc_bvs) + return -ENOMEM; + + zc_bvs->nr_bvs = ap->num_folios; + bvs = zc_bvs->bvs; + for (i = 0; i < ap->num_folios; i++) { + bvs[i].bv_page = folio_page(ap->folios[i], 0); + bvs[i].bv_offset = ap->descs[i].offset; + bvs[i].bv_len = ap->descs[i].length; + folio_get(ap->folios[i]); + } + + err = io_buffer_register_bvec(ent->cmd, bvs, ap->num_folios, + fuse_zero_copy_release, zc_bvs, + ddir, ent->zero_copy_index, + issue_flags); + if (err) { + fuse_zero_copy_release(zc_bvs); + return err; + } + + ent->zero_copied = true; + + return 0; +} + /* * Copy data from the req to the ring buffer */ @@ -762,6 +859,13 @@ static int fuse_uring_args_to_ring(struct fuse_req *req, .commit_id = req->in.h.unique, }; + if (can_zero_copy_req(ent, req)) { + ent_in_out.flags |= FUSE_URING_ENT_ZERO_COPY; + err = fuse_uring_set_up_zero_copy(ent, req, issue_flags); + if (err) + return err; + } + err = setup_fuse_copy_state(&cs, req, ent, ITER_DEST, &iter, issue_flags); if (err) @@ -793,6 +897,18 @@ static int fuse_uring_args_to_ring(struct fuse_req *req, } ent_in_out.payload_sz = cs.ring.copied_sz; + /* + * on a zero-copied write the pages are registered for the server to + * read via a fixed-buffer op rather than copied into the payload + * buffer, so copied_sz does not account for it. The server still needs + * the total inbound size to know how many bytes to read from the + * registered buffer, so add the page arg (always the last in-arg) back + * in + */ + if (cs.skip_folio_copy && args->in_pages) + ent_in_out.payload_sz += + args->in_args[args->in_numargs - 1].size; + if (bufpool_enabled(ent->queue) && ent->payload.iov_base) ent_in_out.offset = (uintptr_t)ent->payload.iov_base - ent->queue->bufpool->base_uaddr; @@ -831,11 +947,25 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, sizeof(req->in.h)); } -static bool fuse_uring_req_has_payload(struct fuse_req *req) +static bool fuse_uring_req_has_copyable_payload(struct fuse_ring_ent *ent, + struct fuse_req *req) { struct fuse_args *args = req->args; - return args->in_numargs > 1 || args->out_numargs; + if (!can_zero_copy_req(ent, req)) + return args->in_numargs > 1 || args->out_numargs; + + /* + * the asymmetry between in_numargs > 2 and out_numargs > 1 is because + * the per-op header is extracted before fuse_copy_args() for inargs but + * not for outargs + */ + if ((args->in_numargs > 1) && (!args->in_pages || args->in_numargs > 2)) + return true; + if (args->out_numargs && (!args->out_pages || args->out_numargs > 1)) + return true; + + return false; } static int fuse_uring_select_buffer(struct fuse_ring_ent *ent) @@ -892,7 +1022,7 @@ static int fuse_uring_next_req_update_buffer(struct fuse_ring_ent *ent, return 0; buffer_selected = !!ent->payload.iov_base; - has_payload = fuse_uring_req_has_payload(req); + has_payload = fuse_uring_req_has_copyable_payload(ent, req); if (has_payload && !buffer_selected) return fuse_uring_select_buffer(ent); @@ -910,7 +1040,7 @@ static int fuse_uring_prep_buffer(struct fuse_ring_ent *ent, return 0; /* no payload to copy, can skip selecting a buffer */ - if (!fuse_uring_req_has_payload(req)) + if (!fuse_uring_req_has_copyable_payload(ent, req)) return 0; return fuse_uring_select_buffer(ent); @@ -936,7 +1066,7 @@ static int fuse_uring_prepare_send(struct fuse_ring_ent *ent, ent->state = FRRS_INVALID; spin_unlock(&ent->queue->lock); - fuse_uring_req_end(ent, req, err); + fuse_uring_req_end(ent, req, err, issue_flags); } return err; @@ -1035,7 +1165,7 @@ static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req, err = fuse_uring_copy_from_ring(req, ent, issue_flags); out: - fuse_uring_req_end(ent, req, err); + fuse_uring_req_end(ent, req, err, issue_flags); } /* @@ -1160,7 +1290,12 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, queue->qid, commit_id, ent->state); fuse_uring_recycle_buffer(ent); spin_unlock(&queue->lock); - fuse_uring_req_end(ent, req, err); + /* + * Unregister any zero copyable pages since ent->cmd is null + * when it hits fuse_uring_req_end() in this path + */ + zero_copy_unregister(cmd, ent, issue_flags); + fuse_uring_req_end(ent, req, err, issue_flags); return err; } @@ -1284,10 +1419,14 @@ static struct fuse_ring_ent * fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, struct fuse_ring_queue *queue) { + const struct fuse_uring_cmd_req *cmd_req = + io_uring_sqe128_cmd(cmd->sqe, struct fuse_uring_cmd_req); struct fuse_ring *ring = queue->ring; struct fuse_ring_ent *ent; struct iovec iov[FUSE_URING_IOV_SEGS]; struct iovec *headers, *payload; + unsigned int zero_copy_index; + int err; err = fuse_uring_get_iovec_from_sqe(cmd->sqe, iov); @@ -1297,6 +1436,10 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, return ERR_PTR(err); } + zero_copy_index = READ_ONCE(cmd_req->ent_zero_copy_buf_index); + if (zero_copy_index && !queue->zero_copy) + return ERR_PTR(-EINVAL); + err = -EINVAL; headers = &iov[FUSE_URING_IOV_HEADERS]; if (headers->iov_len < sizeof(struct fuse_uring_req_header)) { @@ -1315,9 +1458,14 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, } } else { if (payload->iov_len < ring->max_payload_sz) { + spin_unlock(&queue->lock); pr_info_ratelimited("Invalid req payload len %zu\n", payload->iov_len); + return ERR_PTR(err); + } + if (queue->zero_copy) { spin_unlock(&queue->lock); + pr_info_ratelimited("Can only use zero copy with bufpools\n"); return ERR_PTR(err); } queue->payload_mode = FUSE_PAYLOAD_PER_ENT; @@ -1335,6 +1483,7 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, ent->headers = headers->iov_base; if (queue->payload_mode == FUSE_PAYLOAD_PER_ENT) ent->payload = *payload; + ent->zero_copy_index = zero_copy_index; atomic_inc(&ring->queue_refs); return ent; @@ -1364,7 +1513,7 @@ static int fuse_uring_register(struct io_uring_cmd *cmd, queue = READ_ONCE(ring->queues[qid]); if (!queue) { - queue = fuse_uring_create_queue(ring, qid, false); + queue = fuse_uring_create_queue(ring, qid, false, false); if (IS_ERR(queue)) return PTR_ERR(queue); } @@ -1389,8 +1538,9 @@ static int fuse_uring_add_queue(struct io_uring_cmd *cmd, struct fuse_chan *fch) unsigned int qid = READ_ONCE(cmd_req->qid); uint64_t flags = READ_ONCE(cmd_req->flags); struct fuse_ring_queue *queue; + bool zero_copy = flags & FUSE_URING_ZERO_COPY; - if (!ring || flags) + if (!ring) return -EINVAL; if (qid >= ring->nr_queues) { @@ -1398,7 +1548,13 @@ static int fuse_uring_add_queue(struct io_uring_cmd *cmd, struct fuse_chan *fch) return -EINVAL; } - queue = fuse_uring_create_queue(ring, qid, true); + if (flags & ~FUSE_URING_ADD_QUEUE_FLAGS) + return -EINVAL; + + if (zero_copy && !capable(CAP_SYS_ADMIN)) + return -EPERM; + + queue = fuse_uring_create_queue(ring, qid, zero_copy, true); if (IS_ERR(queue)) return PTR_ERR(queue); @@ -1595,7 +1751,7 @@ static void fuse_uring_send_in_task(struct io_tw_req tw_req, io_tw_token_t tw) io_uring_cmd_done(cmd, err, issue_flags); - fuse_uring_req_end(ent, ent->fuse_req, err); + fuse_uring_req_end(ent, ent->fuse_req, err, issue_flags); kfree(ent); if (atomic_dec_and_test(&queue->ring->queue_refs)) wake_up_all(&queue->ring->stop_waitq); diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h index e142cae43022..263d0f8b9714 100644 --- a/fs/fuse/dev_uring_i.h +++ b/fs/fuse/dev_uring_i.h @@ -79,6 +79,10 @@ struct fuse_ring_ent { /* buffer id in the pool, if bufpools are used. ignored otherwise */ unsigned int buf_id; + /* true if the request's pages are being zero-copied */ + bool zero_copied; + unsigned int zero_copy_index; + /* the ring queue that owns the request */ struct fuse_ring_queue *queue; @@ -142,6 +146,8 @@ struct fuse_ring_queue { /* only allocated when payload_mode == FUSE_PAYLOAD_BUFPOOL */ struct fuse_bufpool *bufpool; + + bool zero_copy; }; /* diff --git a/fs/fuse/file.c b/fs/fuse/file.c index da5859e8159d..7b883cf170ac 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -605,6 +605,7 @@ void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos, args->out_argvar = true; args->out_numargs = 1; args->out_args[0].size = count; + args->zero_copy = ff->open_flags & FOPEN_IO_URING_ZERO_COPY; } static void fuse_release_user_pages(struct fuse_args_pages *ap, ssize_t nres, @@ -1151,6 +1152,7 @@ static void fuse_write_args_fill(struct fuse_io_args *ia, struct fuse_file *ff, args->out_numargs = 1; args->out_args[0].size = sizeof(ia->write.out); args->out_args[0].value = &ia->write.out; + args->zero_copy = ff->open_flags & FOPEN_IO_URING_ZERO_COPY; } static unsigned int fuse_write_flags(struct kiocb *iocb) diff --git a/fs/fuse/fuse_dev_i.h b/fs/fuse/fuse_dev_i.h index 668c8391d61c..f41749f484df 100644 --- a/fs/fuse/fuse_dev_i.h +++ b/fs/fuse/fuse_dev_i.h @@ -325,6 +325,8 @@ struct fuse_copy_state { bool write:1; bool move_folios:1; bool is_uring:1; + /* set when the payload is zero-copied. folios are filled in place */ + bool skip_folio_copy:1; struct { unsigned int copied_sz; /* copied size into the user buffer */ } ring; diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h index 538d844da099..7435e09c87fe 100644 --- a/include/uapi/linux/fuse.h +++ b/include/uapi/linux/fuse.h @@ -246,6 +246,8 @@ * - add FUSE_HAS_IO_URING_BUFPOOL * - add fuse_uring_cmd_req bufpool struct * - add bufpool offset field to fuse_uring_ent_in_out struct + * - add FUSE_URING_ZERO_COPY, FUSE_URING_ENT_ZERO_COPY, and + * FOPEN_IO_URING_ZERO_COPY flag */ #ifndef _LINUX_FUSE_H @@ -389,6 +391,12 @@ struct fuse_file_lock { * FOPEN_NOFLUSH: don't flush data cache on close (unless FUSE_WRITEBACK_CACHE) * FOPEN_PARALLEL_DIRECT_WRITES: Allow concurrent direct writes on the same inode * FOPEN_PASSTHROUGH: passthrough read/write io for this open file + * FOPEN_IO_URING_ZERO_COPY: use io-uring zero-copy for reads/writes on this + * open file. Honored only when the serving io-uring + * queue was set up for zero-copy + * (FUSE_URING_ZERO_COPY) and the request carries page + * payload. Otherwise reads/writes fall back to + * copying. */ #define FOPEN_DIRECT_IO (1 << 0) #define FOPEN_KEEP_CACHE (1 << 1) @@ -398,6 +406,7 @@ struct fuse_file_lock { #define FOPEN_NOFLUSH (1 << 5) #define FOPEN_PARALLEL_DIRECT_WRITES (1 << 6) #define FOPEN_PASSTHROUGH (1 << 7) +#define FOPEN_IO_URING_ZERO_COPY (1 << 8) /** * INIT request/reply flags @@ -1259,6 +1268,13 @@ struct fuse_supp_groups { #define FUSE_URING_IN_OUT_HEADER_SZ 128 #define FUSE_URING_OP_IN_OUT_SZ 128 +/** + * fuse_uring_ent_in_out flags + * + * FUSE_URING_ENT_ZERO_COPY: Set if the ent's payload is zero-copied + */ +#define FUSE_URING_ENT_ZERO_COPY (1 << 0) + /* Used as part of the fuse_uring_req_header */ struct fuse_uring_ent_in_out { uint64_t flags; @@ -1310,6 +1326,14 @@ enum fuse_uring_cmd { FUSE_IO_URING_CMD_ADD_BUFPOOL = 4, }; +/* + * fuse_uring_cmd_req flags for FUSE_IO_URING_CMD_ADD_QUEUE + * + * FUSE_URING_ZERO_COPY is only supported for queues with bufpools on privileged + * servers + */ +#define FUSE_URING_ZERO_COPY (1 << 0) + /** * In the 80B command area of the SQE. */ @@ -1330,6 +1354,16 @@ struct fuse_uring_cmd_req { uint32_t len; uint32_t reserved; } bufpool; + + /* + * Index of this entry's slot in the server's io_uring + * registered buffer table, where the kernel registers the + * request's pages for zero-copy. Set for + * FUSE_IO_URING_CMD_REGISTER cmds only, and only on queues + * created with FUSE_URING_ZERO_COPY. On a non-zero-copy queue + * this must be 0 + */ + uint16_t ent_zero_copy_buf_index; }; }; -- cgit v1.2.3 From 6e64df0f73f1c070db816ef71ada96c847bca1c9 Mon Sep 17 00:00:00 2001 From: Ben Dooks Date: Tue, 23 Jun 2026 10:42:06 +0100 Subject: fuse: make dentry_tree_work static The dentry_tree_work is not exported, so make it static to remove the followign sparse warning: fs/fuse/dir.c:37:21: warning: symbol 'dentry_tree_work' was not declared. Should it be static? Signed-off-by: Ben Dooks Signed-off-by: Miklos Szeredi --- fs/fuse/dir.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index b689503bc880..d73e2bf65634 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -34,7 +34,7 @@ struct dentry_bucket { #define FUSE_HASH_BITS 5 #define FUSE_HASH_SIZE (1 << FUSE_HASH_BITS) static struct dentry_bucket dentry_hash[FUSE_HASH_SIZE]; -struct delayed_work dentry_tree_work; +static struct delayed_work dentry_tree_work; /* Minimum invalidation work queue frequency */ #define FUSE_DENTRY_INVAL_FREQ_MIN 5 -- cgit v1.2.3 From 64b0b5cacbd2fea88001464cb712c9dfc795b26e Mon Sep 17 00:00:00 2001 From: Rochan Avlur Date: Wed, 12 Aug 2026 20:58:36 -0700 Subject: fuse: wait for FR_FINISHED on abort_on_kill to prevent use-after-free The abort_on_kill path in request_wait_answer() calls fuse_abort_conn() and returns without waiting for FR_FINISHED. If fuse_dev_do_write() is concurrently processing the same request (FR_LOCKED set), the caller frees req->args while it is still being accessed, causing a use-after-free. Fix this by jumping to the existing wait_event(FR_FINISHED) instead of returning early. The wait will not hang because fuse_abort_conn() ensures all requests are ended. Reported-by: syzbot+d6540a3fa1626e11360d@syzkaller.appspotmail.com Fixes: 204aa22a686b ("fuse: abort on fatal signal during sync init") Cc: stable@vger.kernel.org Assisted-by: Claude:claude-opus-4-6 Signed-off-by: Rochan Avlur Signed-off-by: Miklos Szeredi --- fs/fuse/dev.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 90dceb7da571..34106f6e66a0 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -724,7 +724,7 @@ static void request_wait_answer(struct fuse_req *req) if (req->args->abort_on_kill) { fuse_chan_abort(fch, false); - return; + goto wait_for_finish; } if (test_bit(FR_URING, &req->flags)) @@ -735,6 +735,7 @@ static void request_wait_answer(struct fuse_req *req) return; } +wait_for_finish: /* * Either request is already in userspace, or it was forced. * Wait it out. -- cgit v1.2.3 From 9afeca0d569c9fc89d758fe7a9339d1e8afb1546 Mon Sep 17 00:00:00 2001 From: Baokun Li Date: Mon, 17 Aug 2026 23:18:00 +0800 Subject: fuse: fix invalidate lock leak on setattr writeback failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fuse_do_setattr() takes filemap_invalidate_lock() for a DAX truncate (fault_blocked = true) and releases it at the out:/error: labels. But when a writeback flush is also needed, a write_inode_now() failure returns directly and leaks the lock, so any later fault or truncate on the file stalls on the stale rwsem. For example, truncate(2) on a setuid file reaches fuse_do_setattr() with both ATTR_SIZE and ATTR_MODE set: truncate(2) └─ do_truncate() ├─ dentry_needs_remove_privs() # S_ISUID └─ notify_change() # KILL_SUID -> ATTR_MODE └─ fuse_setattr() # no killpriv: │ # ia_valid |= ATTR_MODE └─ fuse_do_setattr() ├─ filemap_invalidate_lock() # IS_DAX && is_truncate └─ write_inode_now() # is_wb && ATTR_MODE └─ if (err) # e.g. daemon -> -EIO return err # <- lock leaked Fix this by adding an unlock label that releases the lock before returning the error, and use it for the fuse_dax_break_layouts() failure path as well. Fixes: 6ae330cad6ef ("virtiofs: serialize truncate/punch_hole and dax fault path") Cc: stable@vger.kernel.org # v5.10+ Signed-off-by: Baokun Li Signed-off-by: Miklos Szeredi --- fs/fuse/dir.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'fs') diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index d73e2bf65634..172261bad71e 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -2170,10 +2170,8 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, filemap_invalidate_lock(mapping); fault_blocked = true; err = fuse_dax_break_layouts(inode, 0, -1); - if (err) { - filemap_invalidate_unlock(mapping); - return err; - } + if (err) + goto unlock; } if (attr->ia_valid & ATTR_OPEN) { @@ -2200,7 +2198,7 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, ATTR_TIMES_SET)) { err = write_inode_now(inode, true); if (err) - return err; + goto unlock; fuse_set_nowrite(inode); fuse_release_nowrite(inode); @@ -2308,6 +2306,7 @@ error: clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); +unlock: if (fault_blocked) filemap_invalidate_unlock(mapping); return err; -- cgit v1.2.3 From a927f1867e61b78f39f9da0bbba3c98c2ca151fe Mon Sep 17 00:00:00 2001 From: Baokun Li Date: Mon, 17 Aug 2026 23:18:01 +0800 Subject: fuse: fix invalidate lock leak on open O_TRUNC DAX failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fuse_open() takes filemap_invalidate_lock() for a DAX truncate (dax_truncate = true) and releases it before the out_inode_unlock label. But when fuse_dax_break_layouts() fails, the goto out_inode_unlock skips the unlock and leaks the rwsem, so any later fault or truncate on the file stalls on the stale lock. fuse_dax_break_layouts() can fail with -ERESTARTSYS when a signal interrupts the wait for busy DAX pages to drain: open("file", O_RDWR | O_TRUNC) └─ fuse_open() ├─ filemap_invalidate_lock() # dax_truncate └─ fuse_dax_break_layouts() └─ dax_break_layout() └─ wait_page_idle() # TASK_INTERRUPTIBLE └─ fuse_wait_dax_page() # unlock, schedule, re-lock └─ signal → -ERESTARTSYS goto out_inode_unlock # <- lock leaked Fix this by moving filemap_invalidate_unlock() below the label so that all error paths release the lock, and rename the label to out_unlock as it now covers more than just the inode lock. Fixes: 2fdbb8dd0155 ("fuse: fix deadlock between atomic O_TRUNC and page invalidation") Cc: stable@vger.kernel.org # v6.0+ Signed-off-by: Baokun Li Signed-off-by: Miklos Szeredi --- fs/fuse/file.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 7b883cf170ac..3f67348d0f7e 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -272,7 +272,7 @@ static int fuse_open(struct inode *inode, struct file *file) filemap_invalidate_lock(inode->i_mapping); err = fuse_dax_break_layouts(inode, 0, -1); if (err) - goto out_inode_unlock; + goto out_unlock; } if (is_wb_truncate || dax_truncate) @@ -296,9 +296,9 @@ static int fuse_open(struct inode *inode, struct file *file) else if (!(ff->open_flags & FOPEN_KEEP_CACHE)) invalidate_inode_pages2(inode->i_mapping); } +out_unlock: if (dax_truncate) filemap_invalidate_unlock(inode->i_mapping); -out_inode_unlock: if (is_wb_truncate || dax_truncate) inode_unlock(inode); -- cgit v1.2.3 From 4deb3edead0c0e172cc7349e8855d741d3c5e162 Mon Sep 17 00:00:00 2001 From: Baokun Li Date: Fri, 14 Aug 2026 21:40:15 +0800 Subject: cuse: wait for pending RCU callbacks on module exit Since commit 053fc4f755ad ("fuse: fix UAF in rcu pathwalks"), fuse_conn_put() frees the fuse_conn through call_rcu() rather than synchronously. For cuse, fc->release is cuse_fc_release(), which lives in the cuse module. If the module is removed before the RCU grace period ends, the callback jumps into freed module memory: userspace / module unload | RCU softirq ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ close(/dev/cuse) | cuse_channel_release() | fuse_dev_release() | fuse_conn_put(fch->conn) | call_rcu(delayed_release) ------+---> callback queued | rmmod cuse | cuse_exit() | cuse_channel_destroy() | ... | return | | | | rcu_do_batch() | delayed_release() | fc->release() | -> cuse_fc_release() | ^^^ freed text! The freed module text is unmapped by vfree(), so the jump into the stale callback triggers a page-fault Oops. If the virtual address is subsequently reused, the callback could execute unrelated code (undefined behaviour). Fix this by calling rcu_barrier() in cuse_exit() so that any pending fuse_conn release callback completes before the module is removed. Fixes: 053fc4f755ad ("fuse: fix UAF in rcu pathwalks") Signed-off-by: Baokun Li Signed-off-by: Miklos Szeredi --- fs/fuse/cuse.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'fs') diff --git a/fs/fuse/cuse.c b/fs/fuse/cuse.c index 96d57735a79f..4079cf8e5974 100644 --- a/fs/fuse/cuse.c +++ b/fs/fuse/cuse.c @@ -654,6 +654,11 @@ static void __exit cuse_exit(void) { misc_deregister(&cuse_miscdev); class_destroy(cuse_class); + /* + * Wait for pending call_rcu() callbacks that call back into + * this module via fc->release (cuse_fc_release). + */ + rcu_barrier(); } module_init(cuse_init); -- cgit v1.2.3 From ed976994939f05ffbb6e9e5482adc788e679c0ea Mon Sep 17 00:00:00 2001 From: Baokun Li Date: Tue, 11 Aug 2026 12:53:50 +0800 Subject: fuse: reject a duplicate fd= mount option fuse_opt_fd() stored the fuse device in ctx->fud and bumped its refcount unconditionally: ctx->fud = fuse_dev_grab(file); If fd= is given twice (two fsconfig FSCONFIG_SET_FD calls), the second call overwrites ctx->fud and grabs the new device, while the reference taken on the first device is never released - a permanent refcount leak that pins the first fuse_dev until reboot. Reject a second fd= outright. ctx is zeroed on allocation, so a non-NULL ctx->fud reliably means the option was already processed. Fixes: d42eb23b2ef9 ("fuse: don't require /dev/fuse fd to be kept open during mount") Signed-off-by: Baokun Li Reviewed-by: Jingbo Xu Signed-off-by: Miklos Szeredi --- fs/fuse/inode.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'fs') diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 9779adc98593..9d877b2e5c12 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -791,6 +791,9 @@ static int fuse_opt_fd(struct fs_context *fsc, struct file *file) { struct fuse_fs_context *ctx = fsc->fs_private; + if (ctx->fud) + return invalfc(fsc, "Multiple fd specified"); + if (file->f_op != &fuse_dev_operations) return invalfc(fsc, "fd is not a fuse device"); /* -- cgit v1.2.3 From 928f659a3e3650978a5b4829cc982324f72b474b Mon Sep 17 00:00:00 2001 From: Baokun Li Date: Sat, 8 Aug 2026 12:13:39 +0800 Subject: fuse: check for NULL root inode in fuse_fill_super_submount fuse_iget() can return NULL when its inode allocation fails, but fuse_fill_super_submount() passed the result straight to get_fuse_inode() and decremented fi->nlookup without checking it: root = fuse_iget(sb, parent_fi->nodeid, ...); fi = get_fuse_inode(root); fi->nlookup--; Inside fuse_iget() the inode allocation can fail and return NULL. The submount root takes the iget5_locked() path, whose alloc_inode() can fail under memory pressure (the auto-submount branch can fail the same way in new_inode() or fuse_alloc_submount_lookup()): inode = iget5_locked(sb, nodeid, fuse_inode_eq, fuse_inode_set, &nodeid); if (!inode) return NULL; A NULL root makes get_fuse_inode() a container_of() on NULL and the nlookup decrement a write to a bogus address, oopsing the mount. With CONFIG_KASAN the following null pointer dereference is reported when the root inode allocation of an auto-submount fails (e.g. under memory pressure): ================================================================== BUG: KASAN: null-ptr-deref in fuse_get_tree_submount+0x656/0x8b0 Read of size 8 at addr 00000000000002b0 by task ls/942 CPU: 0 PID: 942 Comm: ls Tainted: G W 6.6 #15 Call Trace: fuse_get_tree_submount+0x656/0x8b0 vfs_get_tree+0x48/0x140 fc_mount+0x13/0x50 fuse_dentry_automount+0x7a/0xb0 __traverse_mounts+0xca/0x330 step_into+0x339/0xac0 path_lookupat+0xc5/0x2f0 filename_lookup+0x163/0x2a0 vfs_statx+0xd5/0x200 do_statx+0x83/0xd0 __x64_sys_statx+0xa0/0xc0 do_syscall_64+0x37/0x90 entry_SYSCALL_64_after_hwframe+0x78/0xe2 ================================================================== Return -ENOMEM instead; the caller tears down the partially built superblock on error, matching the other error returns in this function. Fixes: 1866d779d5d2 ("fuse: Allow fuse_fill_super_common() for submounts") Signed-off-by: Baokun Li Reviewed-by: Jingbo Xu Signed-off-by: Miklos Szeredi --- fs/fuse/inode.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'fs') diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 9d877b2e5c12..1c6ee01c6796 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -1645,6 +1645,8 @@ static int fuse_fill_super_submount(struct super_block *sb, fuse_fill_attr_from_inode(&root_attr, parent_fi); root = fuse_iget(sb, parent_fi->nodeid, 0, &root_attr, 0, 0, fuse_get_evict_ctr(fm->fc)); + if (!root) + return -ENOMEM; /* * This inode is just a duplicate, so it is not looked up and * its nlookup should not be incremented. fuse_iget() does -- cgit v1.2.3 From 4332cf75e4dafbe0981356d1b898d23104acd5db Mon Sep 17 00:00:00 2001 From: Xuewen Yan Date: Fri, 31 Jul 2026 15:01:02 +0800 Subject: fuse: give wakeup hints to the scheduler for synchronous requests When a synchronous FUSE request is sent, the in-kernel client queues it on fiq->pending and wakes the userspace daemon sleeping in fuse_dev_do_read()->wait_event_interruptible_exclusive(fiq->waitq, ...). The client then blocks in request_wait_answer() waiting for the reply, so the waker is about to go to sleep: this is exactly the pattern that WF_SYNC is meant to optimise. As Peter Zijlstra explained in the earlier discussion [1], WF_SYNC is a hint that the waker is about to sleep and the waker and wakee share data, so stacking the woken thread on the current CPU is beneficial for cache locality instead of searching for an idle one. Add a wake_up_sync() wrapper for task on the synchronous request path. Performance: On an Android big.LITTLE device where the FUSE daemon (MediaProvider) runs as a background service on the little cores while foreground applications run on the big cores, the synchronous wakeup hint lets the scheduler pull the daemon thread onto the big core that is issuing the request, where the request data is cache-hot. Measured by qixiaoyu [2] on a 2000-picture zip decompression to /sdcard: ------------------------------------------ | Default | patched | Improvement | ------------------------------------------ | 13.0 s | 7.0 s | 46% | ------------------------------------------ Server thread wall duration: 3583 ms -> 1276 ms Server runs on big core: 5% -> 79% The original 4K-file copy/compress/decompress workload [1] on the same kind of device showed a ~28% improvement (13.8s -> 9.9s). Note: Miklos reported [2] that on his test box he could not observe an actual migration from wake_up_interruptible_sync(); the benefit appears to be most visible on asymmetric topologies (big.LITTLE, where the daemon normally lives on a little core) and on workloads dominated by small synchronous requests. No regression was reported on the symmetric- SMP test setups tried. The earlier version of this change [1] added a `bool sync` argument to all three hooks of `struct fuse_iqueue_ops` and threaded it through virtio_fs as well. Miklos questioned the interface churn, and the patch has been stalled since. Re-work it so the exported interface is left alone. The hint is carried in a new FR_SYNC_WAKEUP bit of the existing `fuse_req->flags` bitfield (an `unsigned long`, so no layout change): - __fuse_request_send() sets the flag before fuse_send_one(). - fuse_dev_queue_req() consumes it with test_and_clear_bit() and forwards the result to fuse_dev_wake_and_unlock(), which then picks wake_up_sync() or wake_up(). - The forget, interrupt and resend paths pass `false` explicitly, preserving their original wake_up() behaviour. Only /dev/fuse ever wakes fiq->waitq; virtio_fs and fuse_uring dispatch through their own transport and never call wake_up(), so threading `sync` through their ops would just add an unused argument. test_and_clear_bit() makes the flag a one-shot hint that cannot leak into a future requeue, and no extra cleanup is needed in fuse_request_end()/fuse_put_request(). [1] https://lore.kernel.org/lkml/1638780405-38026-1-git-send-email-quic_pragalla@quicinc.com/ [2] https://lore.kernel.org/lkml/20221222093407.GA1141@mi-HP-ProDesk-680-G4-MT/ This work is based on "Pradeep P V K " and "Pavankumar Kondeti " Assisted-by: TRAE:GLM-5.2 Signed-off-by: Xuewen Yan Signed-off-by: Miklos Szeredi --- fs/fuse/dev.c | 22 ++++++++++++++++------ fs/fuse/fuse_dev_i.h | 3 +++ 2 files changed, 19 insertions(+), 6 deletions(-) (limited to 'fs') diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 34106f6e66a0..3f498630f421 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -213,10 +213,13 @@ EXPORT_SYMBOL_GPL(fuse_req_hash); /* * A new request is available, wake fiq->waitq */ -static void fuse_dev_wake_and_unlock(struct fuse_iqueue *fiq) +static void fuse_dev_wake_and_unlock(struct fuse_iqueue *fiq, bool sync) __releases(fiq->lock) { - wake_up(&fiq->waitq); + if (sync) + wake_up_sync(&fiq->waitq); + else + wake_up(&fiq->waitq); kill_fasync(&fiq->fasync, SIGIO, POLL_IN); spin_unlock(&fiq->lock); } @@ -233,7 +236,7 @@ void fuse_dev_queue_forget(struct fuse_iqueue *fiq, if (fiq->connected) { fiq->forget_list_tail->next = forget; fiq->forget_list_tail = forget; - fuse_dev_wake_and_unlock(fiq); + fuse_dev_wake_and_unlock(fiq, false); } else { kfree(forget); spin_unlock(&fiq->lock); @@ -255,7 +258,7 @@ void fuse_dev_queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req) list_del_init(&req->intr_entry); spin_unlock(&fiq->lock); } else { - fuse_dev_wake_and_unlock(fiq); + fuse_dev_wake_and_unlock(fiq, false); } } else { spin_unlock(&fiq->lock); @@ -285,11 +288,13 @@ EXPORT_SYMBOL_GPL(fuse_request_assign_unique); static void fuse_dev_queue_req(struct fuse_iqueue *fiq, struct fuse_req *req) { + bool sync = test_and_clear_bit(FR_SYNC_WAKEUP, &req->flags); + spin_lock(&fiq->lock); if (fiq->connected) { fuse_request_assign_unique_locked(fiq, req); list_add_tail(&req->list, &fiq->pending); - fuse_dev_wake_and_unlock(fiq); + fuse_dev_wake_and_unlock(fiq, sync); } else { spin_unlock(&fiq->lock); req->out.h.error = -ENOTCONN; @@ -752,6 +757,11 @@ static void __fuse_request_send(struct fuse_req *req) /* acquire extra reference, since request is still needed after fuse_request_end() */ __fuse_get_request(req); + /* + * This is a synchronous request: the caller will block waiting for + * the answer. Hint the scheduler via wake_up_sync(). + */ + set_bit(FR_SYNC_WAKEUP, &req->flags); fuse_send_one(fiq, req); request_wait_answer(req); @@ -1824,7 +1834,7 @@ void fuse_chan_resend(struct fuse_chan *fch) } /* iq and pq requests are both oldest to newest */ list_splice(&to_queue, &fiq->pending); - fuse_dev_wake_and_unlock(fiq); + fuse_dev_wake_and_unlock(fiq, false); } /* Look up request on processing list by unique ID */ diff --git a/fs/fuse/fuse_dev_i.h b/fs/fuse/fuse_dev_i.h index f41749f484df..4b412a76225f 100644 --- a/fs/fuse/fuse_dev_i.h +++ b/fs/fuse/fuse_dev_i.h @@ -38,6 +38,8 @@ struct fuse_iqueue; * @FR_PRIVATE: request is on private list * @FR_ASYNC: request is asynchronous * @FR_URING: request is handled through fuse-io-uring + * @FR_SYNC_WAKEUP: use synchronous wakeup when queueing this request to + * give the scheduler a hint about the waker task */ enum fuse_req_flag { FR_ISREPLY, @@ -53,6 +55,7 @@ enum fuse_req_flag { FR_PRIVATE, FR_ASYNC, FR_URING, + FR_SYNC_WAKEUP, }; /** -- cgit v1.2.3 From fd10f40af314f07b6d6e028b1ca25c8b49903aab Mon Sep 17 00:00:00 2001 From: Xiang Mei Date: Mon, 27 Jul 2026 16:37:04 -0700 Subject: fuse: copy request headers via a stack buffer for io-uring The fuse-io-uring transport copies req->in.h out to the ring in fuse_uring_copy_to_ring() and req->out.h back in fuse_uring_commit(). Both headers live inside the fuse_request slab object, whose cache (fuse_req_cachep) is created without a usercopy whitelist, so copying them directly to/from userspace trips CONFIG_HARDENED_USERCOPY and panics: usercopy: Kernel memory exposure attempt detected from SLUB object 'fuse_request' (offset 56, size 40)! kernel BUG at mm/usercopy.c:102! Oops: invalid opcode: 0000 [#1] SMP KASAN NOPTI RIP: 0010:usercopy_abort (mm/usercopy.c:90) Call Trace: __check_heap_object (mm/slub.c:8268) __check_object_size (mm/usercopy.c:197 mm/usercopy.c:258 mm/usercopy.c:223) copy_header_to_ring (fs/fuse/dev_uring.c:618) fuse_uring_prepare_send (fs/fuse/dev_uring.c:776 fs/fuse/dev_uring.c:785) fuse_uring_send_in_task (fs/fuse/dev_uring.c:1306) tctx_task_work_run (io_uring/tw.c:96) task_work_run (kernel/task_work.c:233) io_run_task_work (io_uring/tw.h:84) io_cqring_wait (io_uring/wait.c:278) __do_sys_io_uring_enter (io_uring/io_uring.c:2685) entry_SYSCALL_64_after_hwframe (arch/x86/entry/entry_64.S:121) Bounce both headers through an on-stack copy so the usercopy touches stack memory, not the slab object. Fixes: c090c8abae4b ("fuse: Add io-uring sqe commit and fetch support") Cc: stable@vger.kernel.org Reported-by: Weiming Shi Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Xiang Mei Reviewed-by: Bernd Schubert Reviewed-by: Joanne Koong Signed-off-by: Miklos Szeredi --- fs/fuse/dev_uring.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'fs') diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 1547a4f0d9b9..e22a48c9a678 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -922,6 +922,7 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, unsigned int issue_flags) { struct fuse_ring_queue *queue = ent->queue; + struct fuse_in_header in_header; int err; err = -EIO; @@ -943,8 +944,9 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, } /* copy fuse_in_header */ - return copy_header_to_ring(ent, FUSE_URING_HEADER_IN_OUT, &req->in.h, - sizeof(req->in.h)); + in_header = req->in.h; + return copy_header_to_ring(ent, FUSE_URING_HEADER_IN_OUT, &in_header, + sizeof(in_header)); } static bool fuse_uring_req_has_copyable_payload(struct fuse_ring_ent *ent, @@ -1151,11 +1153,13 @@ static struct fuse_req *fuse_uring_ent_assign_req(struct fuse_ring_ent *ent) static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req, unsigned int issue_flags) { + struct fuse_out_header out_header; ssize_t err = -EFAULT; - if (copy_header_from_ring(ent, FUSE_URING_HEADER_IN_OUT, &req->out.h, - sizeof(req->out.h))) + if (copy_header_from_ring(ent, FUSE_URING_HEADER_IN_OUT, &out_header, + sizeof(out_header))) goto out; + req->out.h = out_header; err = fuse_uring_out_header_has_err(&req->out.h, req); if (err) { -- cgit v1.2.3 From b77334654027ef56583e257e26e6d8a6ed8f9830 Mon Sep 17 00:00:00 2001 From: Sang-Heon Jeon Date: Sat, 1 Aug 2026 00:44:36 +0900 Subject: fuse: use min_not_zero() in fuse_init_server_timeout() fuse_init_server_timeout() limits timeout to fuse_max_req_timeout with the same logic as min_not_zero(), and returns early exactly when the computed timeout would be zero. So use min_not_zero() instead and return when the computed timeout is zero. No functional change. Signed-off-by: Sang-Heon Jeon Reviewed-by: Joanne Koong Signed-off-by: Miklos Szeredi --- fs/fuse/req_timeout.c | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) (limited to 'fs') diff --git a/fs/fuse/req_timeout.c b/fs/fuse/req_timeout.c index 6cc6fc491343..95a1acd7bc08 100644 --- a/fs/fuse/req_timeout.c +++ b/fs/fuse/req_timeout.c @@ -128,18 +128,12 @@ static void set_request_timeout(struct fuse_chan *fch, unsigned int timeout) void fuse_init_server_timeout(struct fuse_chan *fch, unsigned int timeout) { - if (!timeout && !fuse_max_req_timeout && !fuse_default_req_timeout) - return; - if (!timeout) timeout = fuse_default_req_timeout; - if (fuse_max_req_timeout) { - if (timeout) - timeout = min(fuse_max_req_timeout, timeout); - else - timeout = fuse_max_req_timeout; - } + timeout = min_not_zero(timeout, fuse_max_req_timeout); + if (!timeout) + return; timeout = max(FUSE_TIMEOUT_TIMER_FREQ, timeout); -- cgit v1.2.3 From d1dbc59200b54944f00251ca4dfbb2b318beca13 Mon Sep 17 00:00:00 2001 From: Baokun Li Date: Sat, 1 Aug 2026 16:24:51 +0800 Subject: fuse: wake one waiter per freed slot when raising max_background MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fuse_get_req() parks background allocations on fch->blocked_waitq via wait_event_state_exclusive(), so each wakeup releases exactly one waiter. fuse_chan_max_background_set() clears fch->blocked when the new limit exceeds num_background, but the accompanying wake_up() releases a single waiter regardless of how many slots just became available. Raising max_background from 10 to 100 therefore admits one request instead of ninety. The remaining waiters are not permanently stranded — the "else if (!fch->blocked)" branch in fuse_request_end() wakes one more per completion — but that only helps while requests keep completing. Consider a fixed pool of threads doing readahead or async direct I/O with the quota exhausted: every thread is either in flight or parked, and each completion wakes one waiter while freeing one slot, a net change of zero. num_background oscillates around the old limit and the added quota is never taken up. Waking one waiter per freed slot also preserves submission order: once fch->blocked is clear, new callers of fuse_get_req() skip the waitqueue entirely, overtaking waiters that parked before the limit was raised. Use wake_up_nr() with the number of slots that just became available. Since the wakeup is guarded by !fch->blocked, num_background is strictly below max_background, so the count is at least 1 and never degenerates into wake_up_all(). Signed-off-by: Baokun Li Reviewed-By: Horst Birthelmer Signed-off-by: Miklos Szeredi --- fs/fuse/dev.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 3f498630f421..4fec31fc0b84 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -406,7 +406,8 @@ void fuse_chan_max_background_set(struct fuse_chan *fch, unsigned int val) fch->max_background = val; fch->blocked = fch->num_background >= fch->max_background; if (!fch->blocked) - wake_up(&fch->blocked_waitq); + wake_up_nr(&fch->blocked_waitq, + fch->max_background - fch->num_background); spin_unlock(&fch->bg_lock); } -- cgit v1.2.3 From 26d7e1f5c407b5859122b5cd47d7ebbf4b4c1cd2 Mon Sep 17 00:00:00 2001 From: Baokun Li Date: Wed, 19 Aug 2026 17:07:28 +0800 Subject: fuse: invalidate the correct range after O_APPEND direct write fuse_direct_write_iter() captures pos before generic_write_checks(), which moves ki_pos to EOF for O_APPEND writes: fuse_direct_write_iter() { pos = iocb->ki_pos; /* 0 (user-supplied) */ generic_write_checks(); /* ki_pos -> EOF */ fuse_direct_io(); /* writes at EOF, correct */ invalidate(pos, pos + res); /* [0, res) -- wrong */ } The post-write invalidation targets a stale range instead of the actual written range at EOF. This can cause data inconsistency when the file size is not page-aligned. The tail page straddling EOF has a valid portion before EOF that concurrent readers can fault back in during the DIO write window: Tail page (file size X not page-aligned): page_start X (EOF) page_end |--- valid data ----|-- stale --| CPU0 (O_APPEND DIO writer) CPU1 (buffered reader) -------------------------- ---------------------- invalidate [X, X+len) tail page evicted FUSE_WRITE in flight ... read [page_start, X) tail page re-faulted [X, page_end) = stale FUSE_WRITE completes i_size = X + len invalidate [0, len) <- WRONG tail page still cached read [X, X+len) hits stale tail page returns old data Fix by reading pos back from iocb->ki_pos after generic_write_checks(), as generic_file_direct_write() does. Also fix a typo in the comment ("may have" -> "may have competed"). Fixes: 2b0408d0284f ("fuse: invalidate page cache after DIO and async DIO writes") Signed-off-by: Baokun Li Reviewed-by: Bernd Schubert Signed-off-by: Miklos Szeredi --- fs/fuse/file.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 3f67348d0f7e..5b421d619e16 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1787,13 +1787,14 @@ static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) { struct inode *inode = file_inode(iocb->ki_filp); struct address_space *mapping = inode->i_mapping; - loff_t pos = iocb->ki_pos; ssize_t res; bool exclusive; fuse_dio_lock(iocb, from, &exclusive); res = generic_write_checks(iocb, from); if (res > 0) { + loff_t pos = iocb->ki_pos; + task_io_account_write(res); if (!is_sync_kiocb(iocb)) { res = fuse_direct_IO(iocb, from); @@ -1808,7 +1809,7 @@ static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) /* * As in generic_file_direct_write(), invalidate after * write, to invalidate read-ahead cache that may have - * with the write. + * competed with the write. */ invalidate_inode_pages2_range(mapping, pos >> PAGE_SHIFT, -- cgit v1.2.3 From 1f59015e958174e89be58cc8db16d70a60d17255 Mon Sep 17 00:00:00 2001 From: Bernd Schubert Date: Fri, 21 Aug 2026 18:19:09 +0200 Subject: fuse: Fix the condition to enable over-io-uring The existing condition in fuse_uring_cmd() is there only to avoid disabling io-uring for connections that already run with it, missing was a condition to refuse any IORING_OP_URING_CMD if the connection/channel didn't get enabled because of missing FUSE_INIT reply flag FUSE_OVER_IO_URING. Without the reply flag the barrier in fuse_uring_ready() doesn't work and IO could already be going on and cause deadlock states (at a minimum one between fch->bg_lock and queue->lock). The change itself is trivial, but brings behavior change, FUSE_OVER_IO_URING has to be set in the FUSE_INIT_REPLY by fuse servers to accept any IORING_OP_URING_CMD. Libfuse does that and the only non-libfuse implementation I found (fractal-fuse) also does it. Qemu patches for fuse-io-uring are not merged yet, as far as I know. Moved up is the smp_load_acquire(&fch->initialized) check, as a fuse-server implementation might try to setup io-uring before FUSE_INIT is processed and might have gotten -EOPNOTSUPP instead of -EAGAIN. Also fixed is a stale comment that explains the handling of the FUSE_OVER_IO_URING flag in early RFC versions. If there should be a report from any library or application we probably need to revert this commit. Fixes: 3393ff964e0f ("fuse: block request allocation until io-uring init is complete") Signed-off-by: Bernd Schubert Signed-off-by: Miklos Szeredi --- fs/fuse/dev_uring.c | 31 ++++++++++++++++++------------- fs/fuse/inode.c | 4 ---- 2 files changed, 18 insertions(+), 17 deletions(-) (limited to 'fs') diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index e22a48c9a678..c6dd420c4034 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -1665,26 +1665,31 @@ int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) } fch = fud->chan; - /* Once a connection has io-uring enabled on it, it can't be disabled */ - if (!enable_uring && !fch->io_uring) { - pr_info_ratelimited("fuse-io-uring is disabled\n"); - return -EOPNOTSUPP; - } - - if (fch->abort_with_err) - return -ECONNABORTED; - if (!fch->connected) - return -ENOTCONN; - /* - * fuse_uring_register() needs the ring to be initialized, - * we need to know the max payload size + * The ring is sized from values negotiated by FUSE_INIT * * Pairs with smp_store_release() in fuse_chan_set_initialized() */ if (!smp_load_acquire(&fch->initialized)) return -EAGAIN; + if (fch->abort_with_err) + return -ECONNABORTED; + if (!fch->connected) + return -ENOTCONN; + + /* Once a connection has io-uring enabled on it, it can't be disabled */ + if (!enable_uring && !fch->io_uring) { + pr_info_ratelimited("fuse-io-uring is disabled by module parameter\n"); + return -EOPNOTSUPP; + } + + if (!fch->io_uring) { + pr_info_ratelimited( + "fuse-io-uring not enabled on this connection\n"); + return -EOPNOTSUPP; + } + switch (cmd_op) { case FUSE_IO_URING_CMD_REGISTER: err = fuse_uring_register(cmd, issue_flags, fch); diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 1c6ee01c6796..e9552be3637b 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -1480,10 +1480,6 @@ static struct fuse_init_args *fuse_new_init(struct fuse_mount *fm) if (IS_ENABLED(CONFIG_FUSE_PASSTHROUGH)) flags |= FUSE_PASSTHROUGH; - /* - * This is just an information flag for fuse server. No need to check - * the reply - server is either sending IORING_OP_URING_CMD or not. - */ if (fuse_uring_enabled()) flags |= FUSE_OVER_IO_URING | FUSE_HAS_IO_URING_BUFPOOL; -- cgit v1.2.3 From 34b5c4a6e4fb9dbb3f9d87f3b0fb0372105c8302 Mon Sep 17 00:00:00 2001 From: Jimmy Zuber Date: Mon, 24 Aug 2026 14:30:51 +0000 Subject: fuse: zero the partial EOF page when extending a file Extending a fuse file past a non-page-aligned EOF does not zero the tail of the old last page. When that page is cached and has been mmap-dirtied beyond the old EOF, the now in-bounds tail is served to later reads as stale data rather than zeros, which violates POSIX file-extension semantics. Some file systems get this zeroing automatically at writeback time (block_write_full_folio() / iomap_writeback_handle_eof() zero the tail of the folio straddling i_size). A non-writeback caching fuse file system uses neither path, so it has to zero the tail itself from the size-extending paths, like XFS (xfs_file_write_zero_eof()) and ext4 (ext4_block_zero_eof()) do. Call truncate_pagecache_range() over the newly-exposed range up front from the three paths that extend a file, before the new size is published: - a buffered write whose position is past the old EOF (fuse_perform_write()); - a size-extending setattr/truncate (fuse_do_setattr()); - a size-extending fallocate (fuse_file_fallocate()). This unmaps the stale mappings and zeroes the partial tail of the old EOF folio, so a later read returns zeros. Truncating [old EOF, write start) before a buffered write keeps the dropped range disjoint from the written data, so a write that lands inside the old EOF folio is preserved. writeback_cache connections are unaffected, as their writes go through iomap_file_buffered_write(), which zeroes post-EOF folios. The bug is observable on a non-writeback_cache server that returns FOPEN_KEEP_CACHE on writable files (without FOPEN_DIRECT_IO), and is caught by the new write_extend_eof fuse selftest. Signed-off-by: Jimmy Zuber Signed-off-by: Miklos Szeredi --- fs/fuse/dir.c | 3 +++ fs/fuse/file.c | 9 +++++++++ 2 files changed, 12 insertions(+) (limited to 'fs') diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index 172261bad71e..d3907aa3642b 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -2289,6 +2289,9 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, */ if ((is_truncate || !is_wb) && S_ISREG(inode->i_mode) && oldsize != outarg.attr.size) { + if (outarg.attr.size > oldsize) + truncate_pagecache_range(inode, oldsize, + outarg.attr.size - 1); truncate_pagecache(inode, outarg.attr.size); invalidate_inode_pages2(mapping); } diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 5b421d619e16..e9b46c8e75b3 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1346,9 +1346,13 @@ static ssize_t fuse_perform_write(struct kiocb *iocb, struct iov_iter *ii) struct fuse_conn *fc = get_fuse_conn(inode); struct fuse_inode *fi = get_fuse_inode(inode); loff_t pos = iocb->ki_pos; + loff_t old_size = i_size_read(inode); int err = 0; ssize_t res = 0; + if (pos > old_size) + truncate_pagecache_range(inode, old_size, pos - 1); + if (inode->i_size < pos + iov_iter_count(ii)) set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); @@ -2895,6 +2899,11 @@ static long fuse_file_fallocate(struct file *file, int mode, loff_t offset, /* we could have extended the file */ if (!(mode & FALLOC_FL_KEEP_SIZE)) { + loff_t oldsize = i_size_read(inode); + + if (offset + length > oldsize) + truncate_pagecache_range(inode, oldsize, + offset + length - 1); if (fuse_write_update_attr(inode, offset + length, length)) file_update_time(file); } -- cgit v1.2.3