From ebed9ea5b469588c6074f3ed5b8d8ec63c4ccf48 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Fri, 14 Aug 2026 11:59:42 -0700 Subject: fuse: add FUSE_IO_URING_CMD_ADD_QUEUE fuse-over-io-uring queues are currently created lazily, as a side effect of the first FUSE_IO_URING_CMD_REGISTER command for a given qid. This ties queue creation to entry registration. Add a FUSE_IO_URING_CMD_ADD_QUEUE command so a server can create a queue explicitly, decoupling queue setup from entry registration. This is additionally a prerequisite for FUSE_IO_URING_CMD_ADD_BUFPOOL, which attaches a buffer pool to an existing queue and therefore needs the queue to have been created first. Reviewed-by: Bernd Schubert Signed-off-by: Joanne Koong Signed-off-by: Miklos Szeredi --- include/uapi/linux/fuse.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'include/uapi') diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h index c13e1f9a2f12..cfb055c0c764 100644 --- a/include/uapi/linux/fuse.h +++ b/include/uapi/linux/fuse.h @@ -240,6 +240,9 @@ * - add FUSE_COPY_FILE_RANGE_64 * - add struct fuse_copy_file_range_out * - add FUSE_NOTIFY_PRUNE + * + * 7.46 + * - add FUSE_IO_URING_CMD_ADD_QUEUE */ #ifndef _LINUX_FUSE_H @@ -275,7 +278,7 @@ #define FUSE_KERNEL_VERSION 7 /** Minor version number of this interface */ -#define FUSE_KERNEL_MINOR_VERSION 45 +#define FUSE_KERNEL_MINOR_VERSION 46 /** The node ID of the root inode */ #define FUSE_ROOT_ID 1 @@ -1292,6 +1295,9 @@ enum fuse_uring_cmd { /* commit fuse request result and fetch next request */ FUSE_IO_URING_CMD_COMMIT_AND_FETCH = 2, + + /* add a queue */ + FUSE_IO_URING_CMD_ADD_QUEUE = 3, }; /** -- cgit v1.2.3 From b45aaabc628bc7356e21bd2eb0c2ae9bdfa13894 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Fri, 14 Aug 2026 11:59:43 -0700 Subject: fuse: add io-uring buffer pools Right now, ents and buffers are tightly coupled in fuse io-uring where each entry has its own dedicated payload buffer, requiring N buffers for N entries where each buffer must be large enough to accomodate the maximum payload size. This is suboptimal as most request types (lookup, open, release, getattr, etc) require vastly less bytes than the maximum payload size and some requests (unlink, rmdir, fsync, flush, etc) do not require payload buffers at all. Instead of requiring a 1:1 coupling between ents and payload buffers, allow the server to pass in a buffer pool (a contiguous chunk of memory) that the kernel will use as it wishes for servicing ents/requests. Entries only reserve a "buffer" from the pool while actively processing a request that requires a payload buffer. This decoupling and letting the kernel delegate memory from the pool for requests allows the kernel to optimize memory usage and reduces the memory usage requirements needed to use fuse-over-io-uring. A pool is registered per queue with the new FUSE_IO_URING_CMD_ADD_BUFPOOL command. The server passes the pool's base address and length in fuse_uring_cmd_req.bufpool.{uaddr,len}. Internally, the kernel splits the region into buffers of ring->max_payload_sz bytes each (nr_bufs = pool len / max_payload_sz). A queue commits to a payload mode on first use: registering an entry that carries its own payload selects the legacy per-entry mode, while ADD_BUFPOOL selects pool mode. The two are mutually exclusive, so ADD_BUFPOOL must be issued before any payload-carrying entries are registered on that queue. The queue must have been created before the bufpool is added, through the FUSE_IO_URING_CMD_ADD_QUEUE command. The kernel tracks free buffers with a bitmap (a set bit marks a free buffer). On dispatch, a request that needs a payload claims a free buffer (find_first_bit + clear). A request that needs none claims nothing. The buffer's byte offset within the pool is reported to the server in the new fuse_uring_ent_in_out.offset field so that the server can locate the payload. On completion the buffer is returned to the pool or reused directly if the next request on that entry also has a payload. The FUSE_HAS_IO_URING_BUFPOOL flag advertises kernel support to the server for bufpools. Buffer pool request flow ~~~~~~~~~~~~~~~~~~~~~~~~ | Kernel | FUSE daemon | | | [request arrives] | | [claim a free pool buffer] | | >fuse_uring_select_buffer() | | [copy headers to ring] | | [copy payload to buffer] | | [report buffer offset in ent_in_out] | | >io_uring_cmd_done() | | | [read headers] | | [read/write payload at offset] | | [process request] | | >io_uring_submit() | | COMMIT_AND_FETCH | >fuse_uring_commit_fetch() | | [copy reply from ring] | | [return buffer to the pool] | | >fuse_uring_recycle_buffer() | Reviewed-by: Bernd Schubert Signed-off-by: Joanne Koong Signed-off-by: Miklos Szeredi --- fs/fuse/dev_uring.c | 239 ++++++++++++++++++++++++++++++++++++++++------ fs/fuse/dev_uring_i.h | 37 ++++++- fs/fuse/inode.c | 2 +- include/uapi/linux/fuse.h | 21 +++- 4 files changed, 269 insertions(+), 30 deletions(-) (limited to 'include/uapi') diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 3b9fd0daef66..d300c7f441c4 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -9,6 +9,7 @@ #include "dev_uring_i.h" #include "fuse_trace.h" +#include #include #include @@ -41,6 +42,11 @@ enum fuse_uring_header_type { FUSE_URING_HEADER_RING_ENT, }; +static inline bool bufpool_enabled(struct fuse_ring_queue *queue) +{ + return queue->payload_mode == FUSE_PAYLOAD_BUFPOOL; +} + static void uring_cmd_set_ring_ent(struct io_uring_cmd *cmd, struct fuse_ring_ent *ring_ent) { @@ -222,6 +228,7 @@ void fuse_uring_destruct(struct fuse_chan *fch) } kfree(queue->fpq.processing); + kfree(queue->bufpool); kfree(queue); WRITE_ONCE(ring->queues[qid], NULL); } @@ -316,6 +323,7 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, if (ring->queues[qid]) { spin_unlock(&fch->lock); kfree(queue->fpq.processing); + kfree(queue->bufpool); kfree(queue); return fail_if_exists ? ERR_PTR(-EEXIST) : ring->queues[qid]; } @@ -646,13 +654,14 @@ static int copy_header_from_ring(struct fuse_ring_ent *ent, } static int setup_fuse_copy_state(struct fuse_copy_state *cs, - struct fuse_ring *ring, struct fuse_req *req, + struct fuse_req *req, struct fuse_ring_ent *ent, int dir, struct iov_iter *iter) { int err; - err = import_ubuf(dir, ent->payload, ring->max_payload_sz, iter); + err = import_ubuf(dir, ent->payload.iov_base, ent->payload.iov_len, + iter); if (err) { pr_info_ratelimited("fuse: Import of user buffer failed\n"); return err; @@ -666,8 +675,7 @@ static int setup_fuse_copy_state(struct fuse_copy_state *cs, return 0; } -static int fuse_uring_copy_from_ring(struct fuse_ring *ring, - struct fuse_req *req, +static int fuse_uring_copy_from_ring(struct fuse_req *req, struct fuse_ring_ent *ent) { struct fuse_copy_state cs; @@ -681,7 +689,7 @@ static int fuse_uring_copy_from_ring(struct fuse_ring *ring, if (err) return err; - err = setup_fuse_copy_state(&cs, ring, req, ent, ITER_SOURCE, &iter); + err = setup_fuse_copy_state(&cs, req, ent, ITER_SOURCE, &iter); if (err) return err; @@ -693,7 +701,7 @@ static int fuse_uring_copy_from_ring(struct fuse_ring *ring, /* * Copy data from the req to the ring buffer */ -static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, +static int fuse_uring_args_to_ring(struct fuse_req *req, struct fuse_ring_ent *ent) { struct fuse_copy_state cs; @@ -707,7 +715,7 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, .commit_id = req->in.h.unique, }; - err = setup_fuse_copy_state(&cs, ring, req, ent, ITER_DEST, &iter); + err = setup_fuse_copy_state(&cs, req, ent, ITER_DEST, &iter); if (err) return err; @@ -737,6 +745,10 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, } ent_in_out.payload_sz = cs.ring.copied_sz; + if (bufpool_enabled(ent->queue) && ent->payload.iov_base) + ent_in_out.offset = + (uintptr_t)ent->payload.iov_base - ent->queue->bufpool->base_uaddr; + return copy_header_to_ring(ent, FUSE_URING_HEADER_RING_ENT, &ent_in_out, sizeof(ent_in_out)); } @@ -745,7 +757,6 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, struct fuse_req *req) { struct fuse_ring_queue *queue = ent->queue; - struct fuse_ring *ring = queue->ring; int err; err = -EIO; @@ -760,7 +771,7 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, return err; /* copy the request */ - err = fuse_uring_args_to_ring(ring, req, ent); + err = fuse_uring_args_to_ring(req, ent); if (unlikely(err)) { pr_info_ratelimited("Copy to ring failed: %d\n", err); return err; @@ -771,6 +782,91 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, sizeof(req->in.h)); } +static bool fuse_uring_req_has_payload(struct fuse_req *req) +{ + struct fuse_args *args = req->args; + + return args->in_numargs > 1 || args->out_numargs; +} + +static int fuse_uring_select_buffer(struct fuse_ring_ent *ent) +{ + struct fuse_ring_queue *queue = ent->queue; + struct fuse_bufpool *pool = queue->bufpool; + unsigned int id; + + lockdep_assert_held(&queue->lock); + + id = find_first_bit(pool->free_map, pool->nr_bufs); + if (id >= pool->nr_bufs) + return -ENOBUFS; + + WARN_ON_ONCE(ent->payload.iov_base); + __clear_bit(id, pool->free_map); + + ent->buf_id = id; + ent->payload.iov_base = + (void __user *)(pool->base_uaddr + id * pool->buf_size); + ent->payload.iov_len = pool->buf_size; + + return 0; +} + +static void fuse_uring_recycle_buffer(struct fuse_ring_ent *ent) +{ + struct iovec *ent_payload = &ent->payload; + struct fuse_ring_queue *queue = ent->queue; + struct fuse_bufpool *pool; + + lockdep_assert_held(&queue->lock); + + if (!bufpool_enabled(queue) || !ent_payload->iov_base) + return; + + pool = queue->bufpool; + + /* a buffer should never be recycled twice */ + WARN_ON_ONCE(test_bit(ent->buf_id, pool->free_map)); + __set_bit(ent->buf_id, pool->free_map); + + memset(ent_payload, 0, sizeof(*ent_payload)); + ent->buf_id = 0; +} + +static int fuse_uring_next_req_update_buffer(struct fuse_ring_ent *ent, + struct fuse_req *req) +{ + bool buffer_selected; + bool has_payload; + + if (!bufpool_enabled(ent->queue)) + return 0; + + buffer_selected = !!ent->payload.iov_base; + has_payload = fuse_uring_req_has_payload(req); + + if (has_payload && !buffer_selected) + return fuse_uring_select_buffer(ent); + + if (!has_payload && buffer_selected) + fuse_uring_recycle_buffer(ent); + + return 0; +} + +static int fuse_uring_prep_buffer(struct fuse_ring_ent *ent, + struct fuse_req *req) +{ + if (!bufpool_enabled(ent->queue)) + return 0; + + /* no payload to copy, can skip selecting a buffer */ + if (!fuse_uring_req_has_payload(req)) + return 0; + + return fuse_uring_select_buffer(ent); +} + static int fuse_uring_prepare_send(struct fuse_ring_ent *ent, struct fuse_req *req) { @@ -858,9 +954,12 @@ static struct fuse_req *fuse_uring_ent_assign_req(struct fuse_ring_ent *ent) /* get and assign the next entry while it is still holding the lock */ req = list_first_entry_or_null(req_queue, struct fuse_req, list); - if (req) - fuse_uring_add_req_to_ring_ent(ent, req); + if (!req || fuse_uring_next_req_update_buffer(ent, req)) { + fuse_uring_recycle_buffer(ent); + return NULL; + } + fuse_uring_add_req_to_ring_ent(ent, req); return req; } @@ -872,7 +971,6 @@ static struct fuse_req *fuse_uring_ent_assign_req(struct fuse_ring_ent *ent) static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req, unsigned int issue_flags) { - struct fuse_ring *ring = ent->queue->ring; ssize_t err = -EFAULT; if (copy_header_from_ring(ent, FUSE_URING_HEADER_IN_OUT, &req->out.h, @@ -885,7 +983,7 @@ static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req, goto out; } - err = fuse_uring_copy_from_ring(ring, req, ent); + err = fuse_uring_copy_from_ring(req, ent); out: fuse_uring_req_end(ent, req, err); } @@ -1004,6 +1102,7 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, if (err != 0) { pr_info_ratelimited("qid=%d commit_id %llu state %d", queue->qid, commit_id, ent->state); + fuse_uring_recycle_buffer(ent); spin_unlock(&queue->lock); fuse_uring_req_end(ent, req, err); return err; @@ -1021,6 +1120,11 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, * fuse requests would otherwise not get processed - committing * and fetching is done in one step vs legacy fuse, which has separated * read (fetch request) and write (commit result). + * + * If there is no next request or if all buffers are busy (if using a + * bufpool), the cmd is not returned to userspace. The entry is left + * available and the cmd only returns to userspace when there's a + * next request and an available buffer. */ if (fuse_uring_get_next_fuse_req(ent, queue)) fuse_uring_send(ent, cmd, 0, issue_flags); @@ -1145,11 +1249,23 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, } payload = &iov[FUSE_URING_IOV_PAYLOAD]; - if (payload->iov_len < ring->max_payload_sz) { - pr_info_ratelimited("Invalid req payload len %zu\n", - payload->iov_len); - return ERR_PTR(err); + + spin_lock(&queue->lock); + if (bufpool_enabled(queue)) { + if (payload->iov_base || payload->iov_len) { + spin_unlock(&queue->lock); + return ERR_PTR(err); + } + } else { + if (payload->iov_len < ring->max_payload_sz) { + pr_info_ratelimited("Invalid req payload len %zu\n", + payload->iov_len); + spin_unlock(&queue->lock); + return ERR_PTR(err); + } + queue->payload_mode = FUSE_PAYLOAD_PER_ENT; } + spin_unlock(&queue->lock); err = -ENOMEM; ent = kzalloc_obj(*ent, GFP_KERNEL_ACCOUNT); @@ -1160,7 +1276,8 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, ent->queue = queue; ent->headers = headers->iov_base; - ent->payload = payload->iov_base; + if (queue->payload_mode == FUSE_PAYLOAD_PER_ENT) + ent->payload = *payload; atomic_inc(&ring->queue_refs); return ent; @@ -1231,6 +1348,67 @@ static int fuse_uring_add_queue(struct io_uring_cmd *cmd, struct fuse_chan *fch) return 0; } +static int fuse_uring_add_bufpool(struct io_uring_cmd *cmd, + struct fuse_chan *fch) +{ + const struct fuse_uring_cmd_req *cmd_req = + io_uring_sqe128_cmd(cmd->sqe, struct fuse_uring_cmd_req); + unsigned int qid = READ_ONCE(cmd_req->qid); + uint64_t flags = READ_ONCE(cmd_req->flags); + /* paired with the smp_store_release() in fuse_uring_create */ + struct fuse_ring *ring = smp_load_acquire(&fch->ring); + struct fuse_ring_queue *queue; + struct fuse_bufpool *pool; + uintptr_t pool_uaddr; + unsigned int pool_len, nr_bufs; + size_t pool_size, buf_size; + + if (!ring || qid >= ring->nr_queues || flags) + return -EINVAL; + + /* reserved for future use, must be zero */ + if (READ_ONCE(cmd_req->bufpool.reserved)) + return -EINVAL; + + /* Pairs with smp_store_release() in fuse_uring_create_queue() */ + queue = smp_load_acquire(&ring->queues[qid]); + if (!queue) + return -EINVAL; + + pool_uaddr = READ_ONCE(cmd_req->bufpool.uaddr); + pool_len = READ_ONCE(cmd_req->bufpool.len); + + /* each buffer holds the max payload size */ + buf_size = queue->ring->max_payload_sz; + + nr_bufs = pool_len / buf_size; + if (!nr_bufs) + return -EINVAL; + + pool_size = struct_size(pool, free_map, BITS_TO_LONGS(nr_bufs)); + pool = kzalloc(pool_size, GFP_KERNEL_ACCOUNT); + if (!pool) + return -ENOMEM; + + pool->base_uaddr = pool_uaddr; + pool->buf_size = buf_size; + pool->nr_bufs = nr_bufs; + /* all buffers are free */ + bitmap_set(pool->free_map, 0, nr_bufs); + + spin_lock(&queue->lock); + if (queue->payload_mode != FUSE_PAYLOAD_UNSET) { + spin_unlock(&queue->lock); + kfree(pool); + return -EINVAL; + } + queue->bufpool = pool; + queue->payload_mode = FUSE_PAYLOAD_BUFPOOL; + spin_unlock(&queue->lock); + + return 0; +} + /* * Entry function from io_uring to handle the given passthrough command * (op code IORING_OP_URING_CMD) @@ -1303,6 +1481,12 @@ int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) pr_info_once("FUSE_IO_URING_CMD_ADD_QUEUE failed err=%d\n", err); return err; + case FUSE_IO_URING_CMD_ADD_BUFPOOL: + err = fuse_uring_add_bufpool(cmd, fch); + if (err) + pr_info_once("FUSE_IO_URING_ADD_BUFPOOL failed err=%d\n", + err); + return err; default: return -EINVAL; } @@ -1336,6 +1520,7 @@ static void fuse_uring_send_in_task(struct io_tw_req tw_req, io_tw_token_t tw) spin_lock(&queue->lock); list_del_init(&ent->list); + fuse_uring_recycle_buffer(ent); spin_unlock(&queue->lock); io_uring_cmd_done(cmd, err, issue_flags); @@ -1397,15 +1582,16 @@ void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req) req->ring_queue = queue; ent = list_first_entry_or_null(&queue->ent_avail_queue, struct fuse_ring_ent, list); - if (ent) - fuse_uring_add_req_to_ring_ent(ent, req); - else - list_add_tail(&req->list, &queue->fuse_req_queue); - spin_unlock(&queue->lock); - if (ent) - fuse_uring_dispatch_ent(ent); + if (!ent || fuse_uring_prep_buffer(ent, req)) { + list_add_tail(&req->list, &queue->fuse_req_queue); + spin_unlock(&queue->lock); + return; + } + fuse_uring_add_req_to_ring_ent(ent, req); + spin_unlock(&queue->lock); + fuse_uring_dispatch_ent(ent); return; err_unlock: @@ -1453,10 +1639,9 @@ bool fuse_uring_queue_bq_req(struct fuse_req *req) */ req = list_first_entry_or_null(&queue->fuse_req_queue, struct fuse_req, list); - if (ent && req) { + if (ent && req && !fuse_uring_prep_buffer(ent, req)) { fuse_uring_add_req_to_ring_ent(ent, req); spin_unlock(&queue->lock); - fuse_uring_dispatch_ent(ent); } else { spin_unlock(&queue->lock); diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h index d721a4fc0215..cdf56f8b38b5 100644 --- a/fs/fuse/dev_uring_i.h +++ b/fs/fuse/dev_uring_i.h @@ -7,6 +7,8 @@ #ifndef _FS_FUSE_DEV_URING_I_H #define _FS_FUSE_DEV_URING_I_H +#include + #include "fuse_dev_i.h" #ifdef CONFIG_FUSE_IO_URING @@ -36,11 +38,38 @@ enum fuse_ring_req_state { FRRS_RELEASED, }; +/* how a queue's payload buffers are provided */ +enum fuse_queue_payload_mode { + /* not yet committed (a bufpool may still be added) */ + FUSE_PAYLOAD_UNSET = 0, + /* each entry registers its own payload buffer */ + FUSE_PAYLOAD_PER_ENT, + /* each entry's payload buffer is assigned from a bufpool */ + FUSE_PAYLOAD_BUFPOOL, +}; + +struct fuse_bufpool { + /* starting uaddr of the bufpool */ + uintptr_t base_uaddr; + + /* size of each buffer in the pool */ + size_t buf_size; + + /* total number of buffers in the pool */ + unsigned int nr_bufs; + + /* bitmap tracking which buffers are free */ + unsigned long free_map[]; +}; + /** A fuse ring entry, part of the ring queue */ struct fuse_ring_ent { /* userspace buffer */ struct fuse_uring_req_header __user *headers; - void __user *payload; + struct iovec payload; + + /* buffer id in the pool, if bufpools are used. ignored otherwise */ + unsigned int buf_id; /* the ring queue that owns the request */ struct fuse_ring_queue *queue; @@ -99,6 +128,12 @@ struct fuse_ring_queue { unsigned int active_background; bool stopped; + + /* how this queue's payload buffers are provided */ + enum fuse_queue_payload_mode payload_mode; + + /* only allocated when payload_mode == FUSE_PAYLOAD_BUFPOOL */ + struct fuse_bufpool *bufpool; }; /* diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 33773c7d129a..9779adc98593 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -1482,7 +1482,7 @@ static struct fuse_init_args *fuse_new_init(struct fuse_mount *fm) * the reply - server is either sending IORING_OP_URING_CMD or not. */ if (fuse_uring_enabled()) - flags |= FUSE_OVER_IO_URING; + flags |= FUSE_OVER_IO_URING | FUSE_HAS_IO_URING_BUFPOOL; ia->in.flags = flags; ia->in.flags2 = flags >> 32; diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h index cfb055c0c764..538d844da099 100644 --- a/include/uapi/linux/fuse.h +++ b/include/uapi/linux/fuse.h @@ -243,6 +243,9 @@ * * 7.46 * - add FUSE_IO_URING_CMD_ADD_QUEUE + * - add FUSE_HAS_IO_URING_BUFPOOL + * - add fuse_uring_cmd_req bufpool struct + * - add bufpool offset field to fuse_uring_ent_in_out struct */ #ifndef _LINUX_FUSE_H @@ -451,6 +454,7 @@ struct fuse_file_lock { * FUSE_OVER_IO_URING: Indicate that client supports io-uring * FUSE_REQUEST_TIMEOUT: kernel supports timing out requests. * init_out.request_timeout contains the timeout (in secs) + * FUSE_HAS_IO_URING_BUFPOOL: kernel supports io-uring buffer pools */ #define FUSE_ASYNC_READ (1 << 0) #define FUSE_POSIX_LOCKS (1 << 1) @@ -498,6 +502,7 @@ struct fuse_file_lock { #define FUSE_ALLOW_IDMAP (1ULL << 40) #define FUSE_OVER_IO_URING (1ULL << 41) #define FUSE_REQUEST_TIMEOUT (1ULL << 42) +#define FUSE_HAS_IO_URING_BUFPOOL (1ULL << 43) /** * CUSE INIT request/reply flags @@ -1266,7 +1271,9 @@ struct fuse_uring_ent_in_out { /* size of user payload buffer */ uint32_t payload_sz; - uint32_t padding; + + /* Offset into the bufpool, if bufpools are used */ + uint32_t offset; uint64_t reserved; }; @@ -1298,6 +1305,9 @@ enum fuse_uring_cmd { /* add a queue */ FUSE_IO_URING_CMD_ADD_QUEUE = 3, + + /* add a bufpool to a queue */ + FUSE_IO_URING_CMD_ADD_BUFPOOL = 4, }; /** @@ -1312,6 +1322,15 @@ struct fuse_uring_cmd_req { /* queue the command is for (queue index) */ uint16_t qid; uint8_t padding[6]; + + union { + struct { + /* base address of bufpool */ + uint64_t uaddr; + uint32_t len; + uint32_t reserved; + } bufpool; + }; }; #endif /* _LINUX_FUSE_H */ -- cgit v1.2.3 From 43f8343858eb942d7f7c49964b31c54dcc314890 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Fri, 14 Aug 2026 11:59:45 -0700 Subject: fuse: add zero-copy over io-uring Implement zero-copy in fuse io-uring to eliminate memory copies between the application, kernel, and server for read/write operations. The server can directly access client pages or page cache folios without copying data through an intermediary buffer. When a fuse request arrives, the kernel registers the relevant pages into a sparse slot in the server's io_uring registered buffer table. The server can then operate on these pages directly using io-uring fixed buffer operations (eg read_fixed/write_fixed) and the kernel unregisters these pages when the request completes. Non-page-backed args (eg op out headers) will go through the payload buffer as normal. The server can specify which open files should have their reads/writes go through zero-copy, by setting the FOPEN_IO_URING_ZERO_COPY flag when servicing opens. This requires CAP_SYS_ADMIN and bufpools. This is gated behind CAP_SYS_ADMIN because zero-copy allows the server direct access to the client's underlying pages, rather than operating on an intermediary buffer that the contents of the client's pages were copied into or on page cache folios. The request flow for the zero-copy direct-io write path (client writes data, server reads it) is as follows: ======================================================================= | Kernel | FUSE server | | | "write(fd, buf, 1MB)" | | | | >sys_write() | | >fuse_file_write_iter() | | >fuse_send_one() | | [req->args->in_pages = true] | | [folios hold client write data] | | | | >fuse_uring_copy_to_ring() | | >copy_header_to_ring(IN_OUT) | | [memcpy fuse_in_header] | | >copy_header_to_ring(OP) | | [memcpy write_in header] | | | | >fuse_uring_args_to_ring() | | >setup_fuse_copy_state() | | [skip_folio_copy = true] | | | | >fuse_uring_set_up_zero_copy() | | [folio_get for each client folio] | | [build bio_vec array from folios] | | >io_buffer_register_bvec() | | [register pages at ent->zero_copy_index] | | [ent->zero_copied = true] | | | | >fuse_copy_args() | | [skip_folio_copy => return 0 | | for page arg, skip data copy] | | | | >copy_header_to_ring(RING_ENT) | | [memcpy ent_in_out] | | >io_uring_cmd_done() | | | | | [CQE received] | | | | [issue io_uring READ at | | ent->zero_copy_index] | | [reads directly from | |client's pages (ZERO_COPY)] | | | | [write data to backing | | store] | | [submit COMMIT AND FETCH] | | | >fuse_uring_commit_fetch() | | >fuse_uring_commit() | | >fuse_uring_copy_from_ring() | | >fuse_uring_req_end() | | >io_buffer_unregister(ent->zero_copy_index) | | [unregister pages from index] | | >fuse_zero_copy_release() | | [folio_put for each folio] | | [ent->zero_copied = false] | | >fuse_request_end() | | [wake up client] | The zero-copy read path is analogous. Some requests may have both page-backed args and non-page-backed args. For these requests, the page-backed args are zero-copied while the non-page-backed args are copied to the buffer selected from the buffer pool: zero-copy: pages registered via io_buffer_register_bvec() non-page-backed: copied to payload buffer via fuse_copy_args() For a request whose payload is zero-copied, the registration/unregistration path looks like: register: fuse_uring_set_up_zero_copy() folio_get() for each folio io_buffer_register_bvec(ent->zero_copy_index) unregister: fuse_uring_req_end() io_buffer_unregister(ent->zero_copy_index) -> fuse_zero_copy_release() callback folio_put() for each folio Please note that on abort for in-flight zero-copied requests that have been sent to userspace, the registered bvec slot remains occupied and its folios remain pinned until the io-uring ring is destroyed, at which point io-uring unregisters all buffers and the fuse_zero_copy_release() callback drops the folio references. Unregistering at teardown would require operating on the ring context directly, whose validity is hard to ascertain; this is deemed not worth the complexity for the abort race, since everything is freed when the ring is torn down. The throughput improvement from zero-copy depends on how much of the per-request latency is spent on data copying vs backing I/O. The gain comes from eliminating the payload-buffer memcpy, but accessing the zero-copied pages requires the server to issue the read/write as an IORING_OP_READ/WRITE_FIXED operation. The benefit is largest when the mempcy is a meaningful fraction of per-request latency while backing i/o is still noticable enough that the extra io-uring op's overhead doesn't dominate. Benchmarked with passthrough_hp (--nopassthrough, q_depth=8) on a 2-socket Intel Xeon Gold 6138 (40 cores / 80 threads), using fio (sync engine, bs=1M, O_DIRECT, numjobs=2, 30s run + 10s ramp, 3 runs) where direct-I/O throughput is against a RAM-backed (tmpfs) source (backing I/O is not the bottleneck): baseline registered-buf zero-copy (zc vs base) direct read ~5.1 GB/s ~5.4 GB/s ~8.9 GB/s (+75%) direct write ~3.4 GB/s ~4.8 GB/s ~5.1 GB/s (+50%) Reads end up higher than writes because the backing store reads faster than it writes (the baseline shows the same read>write gap, and the raw device does too). On a device-bound NVMe (~2 GB/s reads) the read gain shrinks to ~10-16% (and no measurable gains for writes), as backing I/O rather than the eliminated copy dominates latency. The benefit overall scales with how much of the per-request latency is the data copy versus backing I/O. Signed-off-by: Joanne Koong Reviewed-by: Bernd Schubert Signed-off-by: Miklos Szeredi --- fs/fuse/args.h | 2 + fs/fuse/dev.c | 24 +++++- fs/fuse/dev_uring.c | 182 ++++++++++++++++++++++++++++++++++++++++++---- fs/fuse/dev_uring_i.h | 6 ++ fs/fuse/file.c | 2 + fs/fuse/fuse_dev_i.h | 2 + include/uapi/linux/fuse.h | 34 +++++++++ 7 files changed, 236 insertions(+), 16 deletions(-) (limited to 'include/uapi') diff --git a/fs/fuse/args.h b/fs/fuse/args.h index ecfe51a192af..5173264a1261 100644 --- a/fs/fuse/args.h +++ b/fs/fuse/args.h @@ -42,6 +42,8 @@ struct fuse_args { bool is_pinned:1; bool invalidate_vmap:1; bool abort_on_kill:1; + /* server requested io-uring zero-copy for this op */ + bool zero_copy:1; struct fuse_in_arg in_args[4]; struct fuse_arg out_args[2]; void (*end)(struct fuse_args *args, int error); diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index d8f97943e973..90dceb7da571 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -1248,11 +1248,25 @@ int fuse_copy_folio(struct fuse_copy_state *cs, struct folio **foliop, if (folio) { size = folio_size(folio); - if (zeroing && count < size) - folio_zero_range(folio, 0, size); + if (zeroing && count < size) { + /* + * When the copy is skipped the folio already holds the + * payload, so only the bytes outside [offset, offset + + * count) may be zeroed. + * + * Otherwise, the whole folio is cleared first so that a + * failed copy leaves zeros rather than stale folio + * contents. + */ + if (cs->skip_folio_copy) + folio_zero_segments(folio, 0, offset, + offset + count, size); + else + folio_zero_range(folio, 0, size); + } } - while (count) { + while (!cs->skip_folio_copy && count) { if (cs->write && cs->pipebufs && folio) { /* * Can't control lifetime of pipe buffers, so always @@ -1345,6 +1359,10 @@ int fuse_copy_args(struct fuse_copy_state *cs, unsigned numargs, for (i = 0; !err && i < numargs; i++) { struct fuse_arg *arg = &args[i]; if (i == numargs - 1 && argpages) + /* + * if cs->skip_folio_copy is set, this just does any + * needed zeroing. No copying is involved. + */ err = fuse_copy_folios(cs, arg->size, zeroing); else err = fuse_copy_one(cs, arg->value, arg->size); diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 17806da93039..1547a4f0d9b9 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -22,6 +22,8 @@ MODULE_PARM_DESC(enable_uring, #define FUSE_URING_IOV_HEADERS 0 #define FUSE_URING_IOV_PAYLOAD 1 +#define FUSE_URING_ADD_QUEUE_FLAGS (FUSE_URING_ZERO_COPY) + bool fuse_uring_enabled(void) { return enable_uring; @@ -31,6 +33,11 @@ struct fuse_uring_pdu { struct fuse_ring_ent *ent; }; +struct fuse_zero_copy_bvs { + unsigned int nr_bvs; + struct bio_vec bvs[]; +}; + static const struct fuse_iqueue_ops fuse_io_uring_ops; enum fuse_uring_header_type { @@ -113,8 +120,36 @@ static void fuse_uring_flush_bg(struct fuse_ring_queue *queue) } } +static bool can_zero_copy_req(struct fuse_ring_ent *ent, struct fuse_req *req) +{ + struct fuse_args *args = req->args; + + if (!ent->queue->zero_copy || !args->zero_copy) + return false; + + if (args->opcode != FUSE_READ && args->opcode != FUSE_WRITE) + return false; + + return args->in_pages || args->out_pages; +} + +static void zero_copy_unregister(struct io_uring_cmd *cmd, + struct fuse_ring_ent *ent, + unsigned int issue_flags) +{ + if (ent->zero_copied) { + int err = io_buffer_unregister(cmd, ent->zero_copy_index, + issue_flags); + + if (err) + pr_warn_ratelimited("qid=%d zero-copy unregister failed: %d\n", + ent->queue->qid, err); + ent->zero_copied = false; + } +} + static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req, - int error) + int error, unsigned int issue_flags) { struct fuse_ring_queue *queue = ent->queue; struct fuse_ring *ring = queue->ring; @@ -134,6 +169,8 @@ static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req, spin_unlock(&queue->lock); + zero_copy_unregister(ent->cmd, ent, issue_flags); + if (error) req->out.h.error = error; @@ -309,7 +346,7 @@ void fuse_uring_conn_init(struct fuse_chan *fch) } static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, - int qid, + int qid, bool zero_copy, bool fail_if_exists) { struct fuse_chan *fch = ring->chan; @@ -328,6 +365,7 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, queue->qid = qid; queue->ring = ring; spin_lock_init(&queue->lock); + queue->zero_copy = zero_copy; INIT_LIST_HEAD(&queue->ent_avail_queue); INIT_LIST_HEAD(&queue->ent_commit_queue); @@ -713,6 +751,9 @@ static int setup_fuse_copy_state(struct fuse_copy_state *cs, fuse_copy_init(cs, dir == ITER_DEST, iter); + if (ent->zero_copied) + cs->skip_folio_copy = true; + cs->is_uring = true; cs->req = req; @@ -744,6 +785,62 @@ static int fuse_uring_copy_from_ring(struct fuse_req *req, return err; } +static void fuse_zero_copy_release(void *priv) +{ + struct fuse_zero_copy_bvs *zc_bvs = priv; + unsigned int i; + + for (i = 0; i < zc_bvs->nr_bvs; i++) + folio_put(page_folio(zc_bvs->bvs[i].bv_page)); + + kvfree(zc_bvs); +} + +static int fuse_uring_set_up_zero_copy(struct fuse_ring_ent *ent, + struct fuse_req *req, + unsigned int issue_flags) +{ + struct fuse_args_pages *ap; + int err, i, ddir = 0; + struct fuse_zero_copy_bvs *zc_bvs; + struct bio_vec *bvs; + + /* out_pages indicates a read, in_pages indicates a write */ + if (req->args->out_pages) + ddir |= IO_BUF_DEST; + if (req->args->in_pages) + ddir |= IO_BUF_SOURCE; + + ap = container_of(req->args, typeof(*ap), args); + + zc_bvs = kvmalloc_flex(*zc_bvs, bvs, ap->num_folios, + GFP_KERNEL_ACCOUNT); + if (!zc_bvs) + return -ENOMEM; + + zc_bvs->nr_bvs = ap->num_folios; + bvs = zc_bvs->bvs; + for (i = 0; i < ap->num_folios; i++) { + bvs[i].bv_page = folio_page(ap->folios[i], 0); + bvs[i].bv_offset = ap->descs[i].offset; + bvs[i].bv_len = ap->descs[i].length; + folio_get(ap->folios[i]); + } + + err = io_buffer_register_bvec(ent->cmd, bvs, ap->num_folios, + fuse_zero_copy_release, zc_bvs, + ddir, ent->zero_copy_index, + issue_flags); + if (err) { + fuse_zero_copy_release(zc_bvs); + return err; + } + + ent->zero_copied = true; + + return 0; +} + /* * Copy data from the req to the ring buffer */ @@ -762,6 +859,13 @@ static int fuse_uring_args_to_ring(struct fuse_req *req, .commit_id = req->in.h.unique, }; + if (can_zero_copy_req(ent, req)) { + ent_in_out.flags |= FUSE_URING_ENT_ZERO_COPY; + err = fuse_uring_set_up_zero_copy(ent, req, issue_flags); + if (err) + return err; + } + err = setup_fuse_copy_state(&cs, req, ent, ITER_DEST, &iter, issue_flags); if (err) @@ -793,6 +897,18 @@ static int fuse_uring_args_to_ring(struct fuse_req *req, } ent_in_out.payload_sz = cs.ring.copied_sz; + /* + * on a zero-copied write the pages are registered for the server to + * read via a fixed-buffer op rather than copied into the payload + * buffer, so copied_sz does not account for it. The server still needs + * the total inbound size to know how many bytes to read from the + * registered buffer, so add the page arg (always the last in-arg) back + * in + */ + if (cs.skip_folio_copy && args->in_pages) + ent_in_out.payload_sz += + args->in_args[args->in_numargs - 1].size; + if (bufpool_enabled(ent->queue) && ent->payload.iov_base) ent_in_out.offset = (uintptr_t)ent->payload.iov_base - ent->queue->bufpool->base_uaddr; @@ -831,11 +947,25 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, sizeof(req->in.h)); } -static bool fuse_uring_req_has_payload(struct fuse_req *req) +static bool fuse_uring_req_has_copyable_payload(struct fuse_ring_ent *ent, + struct fuse_req *req) { struct fuse_args *args = req->args; - return args->in_numargs > 1 || args->out_numargs; + if (!can_zero_copy_req(ent, req)) + return args->in_numargs > 1 || args->out_numargs; + + /* + * the asymmetry between in_numargs > 2 and out_numargs > 1 is because + * the per-op header is extracted before fuse_copy_args() for inargs but + * not for outargs + */ + if ((args->in_numargs > 1) && (!args->in_pages || args->in_numargs > 2)) + return true; + if (args->out_numargs && (!args->out_pages || args->out_numargs > 1)) + return true; + + return false; } static int fuse_uring_select_buffer(struct fuse_ring_ent *ent) @@ -892,7 +1022,7 @@ static int fuse_uring_next_req_update_buffer(struct fuse_ring_ent *ent, return 0; buffer_selected = !!ent->payload.iov_base; - has_payload = fuse_uring_req_has_payload(req); + has_payload = fuse_uring_req_has_copyable_payload(ent, req); if (has_payload && !buffer_selected) return fuse_uring_select_buffer(ent); @@ -910,7 +1040,7 @@ static int fuse_uring_prep_buffer(struct fuse_ring_ent *ent, return 0; /* no payload to copy, can skip selecting a buffer */ - if (!fuse_uring_req_has_payload(req)) + if (!fuse_uring_req_has_copyable_payload(ent, req)) return 0; return fuse_uring_select_buffer(ent); @@ -936,7 +1066,7 @@ static int fuse_uring_prepare_send(struct fuse_ring_ent *ent, ent->state = FRRS_INVALID; spin_unlock(&ent->queue->lock); - fuse_uring_req_end(ent, req, err); + fuse_uring_req_end(ent, req, err, issue_flags); } return err; @@ -1035,7 +1165,7 @@ static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req, err = fuse_uring_copy_from_ring(req, ent, issue_flags); out: - fuse_uring_req_end(ent, req, err); + fuse_uring_req_end(ent, req, err, issue_flags); } /* @@ -1160,7 +1290,12 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, queue->qid, commit_id, ent->state); fuse_uring_recycle_buffer(ent); spin_unlock(&queue->lock); - fuse_uring_req_end(ent, req, err); + /* + * Unregister any zero copyable pages since ent->cmd is null + * when it hits fuse_uring_req_end() in this path + */ + zero_copy_unregister(cmd, ent, issue_flags); + fuse_uring_req_end(ent, req, err, issue_flags); return err; } @@ -1284,10 +1419,14 @@ static struct fuse_ring_ent * fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, struct fuse_ring_queue *queue) { + const struct fuse_uring_cmd_req *cmd_req = + io_uring_sqe128_cmd(cmd->sqe, struct fuse_uring_cmd_req); struct fuse_ring *ring = queue->ring; struct fuse_ring_ent *ent; struct iovec iov[FUSE_URING_IOV_SEGS]; struct iovec *headers, *payload; + unsigned int zero_copy_index; + int err; err = fuse_uring_get_iovec_from_sqe(cmd->sqe, iov); @@ -1297,6 +1436,10 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, return ERR_PTR(err); } + zero_copy_index = READ_ONCE(cmd_req->ent_zero_copy_buf_index); + if (zero_copy_index && !queue->zero_copy) + return ERR_PTR(-EINVAL); + err = -EINVAL; headers = &iov[FUSE_URING_IOV_HEADERS]; if (headers->iov_len < sizeof(struct fuse_uring_req_header)) { @@ -1315,9 +1458,14 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, } } else { if (payload->iov_len < ring->max_payload_sz) { + spin_unlock(&queue->lock); pr_info_ratelimited("Invalid req payload len %zu\n", payload->iov_len); + return ERR_PTR(err); + } + if (queue->zero_copy) { spin_unlock(&queue->lock); + pr_info_ratelimited("Can only use zero copy with bufpools\n"); return ERR_PTR(err); } queue->payload_mode = FUSE_PAYLOAD_PER_ENT; @@ -1335,6 +1483,7 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, ent->headers = headers->iov_base; if (queue->payload_mode == FUSE_PAYLOAD_PER_ENT) ent->payload = *payload; + ent->zero_copy_index = zero_copy_index; atomic_inc(&ring->queue_refs); return ent; @@ -1364,7 +1513,7 @@ static int fuse_uring_register(struct io_uring_cmd *cmd, queue = READ_ONCE(ring->queues[qid]); if (!queue) { - queue = fuse_uring_create_queue(ring, qid, false); + queue = fuse_uring_create_queue(ring, qid, false, false); if (IS_ERR(queue)) return PTR_ERR(queue); } @@ -1389,8 +1538,9 @@ static int fuse_uring_add_queue(struct io_uring_cmd *cmd, struct fuse_chan *fch) unsigned int qid = READ_ONCE(cmd_req->qid); uint64_t flags = READ_ONCE(cmd_req->flags); struct fuse_ring_queue *queue; + bool zero_copy = flags & FUSE_URING_ZERO_COPY; - if (!ring || flags) + if (!ring) return -EINVAL; if (qid >= ring->nr_queues) { @@ -1398,7 +1548,13 @@ static int fuse_uring_add_queue(struct io_uring_cmd *cmd, struct fuse_chan *fch) return -EINVAL; } - queue = fuse_uring_create_queue(ring, qid, true); + if (flags & ~FUSE_URING_ADD_QUEUE_FLAGS) + return -EINVAL; + + if (zero_copy && !capable(CAP_SYS_ADMIN)) + return -EPERM; + + queue = fuse_uring_create_queue(ring, qid, zero_copy, true); if (IS_ERR(queue)) return PTR_ERR(queue); @@ -1595,7 +1751,7 @@ static void fuse_uring_send_in_task(struct io_tw_req tw_req, io_tw_token_t tw) io_uring_cmd_done(cmd, err, issue_flags); - fuse_uring_req_end(ent, ent->fuse_req, err); + fuse_uring_req_end(ent, ent->fuse_req, err, issue_flags); kfree(ent); if (atomic_dec_and_test(&queue->ring->queue_refs)) wake_up_all(&queue->ring->stop_waitq); diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h index e142cae43022..263d0f8b9714 100644 --- a/fs/fuse/dev_uring_i.h +++ b/fs/fuse/dev_uring_i.h @@ -79,6 +79,10 @@ struct fuse_ring_ent { /* buffer id in the pool, if bufpools are used. ignored otherwise */ unsigned int buf_id; + /* true if the request's pages are being zero-copied */ + bool zero_copied; + unsigned int zero_copy_index; + /* the ring queue that owns the request */ struct fuse_ring_queue *queue; @@ -142,6 +146,8 @@ struct fuse_ring_queue { /* only allocated when payload_mode == FUSE_PAYLOAD_BUFPOOL */ struct fuse_bufpool *bufpool; + + bool zero_copy; }; /* diff --git a/fs/fuse/file.c b/fs/fuse/file.c index da5859e8159d..7b883cf170ac 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -605,6 +605,7 @@ void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos, args->out_argvar = true; args->out_numargs = 1; args->out_args[0].size = count; + args->zero_copy = ff->open_flags & FOPEN_IO_URING_ZERO_COPY; } static void fuse_release_user_pages(struct fuse_args_pages *ap, ssize_t nres, @@ -1151,6 +1152,7 @@ static void fuse_write_args_fill(struct fuse_io_args *ia, struct fuse_file *ff, args->out_numargs = 1; args->out_args[0].size = sizeof(ia->write.out); args->out_args[0].value = &ia->write.out; + args->zero_copy = ff->open_flags & FOPEN_IO_URING_ZERO_COPY; } static unsigned int fuse_write_flags(struct kiocb *iocb) diff --git a/fs/fuse/fuse_dev_i.h b/fs/fuse/fuse_dev_i.h index 668c8391d61c..f41749f484df 100644 --- a/fs/fuse/fuse_dev_i.h +++ b/fs/fuse/fuse_dev_i.h @@ -325,6 +325,8 @@ struct fuse_copy_state { bool write:1; bool move_folios:1; bool is_uring:1; + /* set when the payload is zero-copied. folios are filled in place */ + bool skip_folio_copy:1; struct { unsigned int copied_sz; /* copied size into the user buffer */ } ring; diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h index 538d844da099..7435e09c87fe 100644 --- a/include/uapi/linux/fuse.h +++ b/include/uapi/linux/fuse.h @@ -246,6 +246,8 @@ * - add FUSE_HAS_IO_URING_BUFPOOL * - add fuse_uring_cmd_req bufpool struct * - add bufpool offset field to fuse_uring_ent_in_out struct + * - add FUSE_URING_ZERO_COPY, FUSE_URING_ENT_ZERO_COPY, and + * FOPEN_IO_URING_ZERO_COPY flag */ #ifndef _LINUX_FUSE_H @@ -389,6 +391,12 @@ struct fuse_file_lock { * FOPEN_NOFLUSH: don't flush data cache on close (unless FUSE_WRITEBACK_CACHE) * FOPEN_PARALLEL_DIRECT_WRITES: Allow concurrent direct writes on the same inode * FOPEN_PASSTHROUGH: passthrough read/write io for this open file + * FOPEN_IO_URING_ZERO_COPY: use io-uring zero-copy for reads/writes on this + * open file. Honored only when the serving io-uring + * queue was set up for zero-copy + * (FUSE_URING_ZERO_COPY) and the request carries page + * payload. Otherwise reads/writes fall back to + * copying. */ #define FOPEN_DIRECT_IO (1 << 0) #define FOPEN_KEEP_CACHE (1 << 1) @@ -398,6 +406,7 @@ struct fuse_file_lock { #define FOPEN_NOFLUSH (1 << 5) #define FOPEN_PARALLEL_DIRECT_WRITES (1 << 6) #define FOPEN_PASSTHROUGH (1 << 7) +#define FOPEN_IO_URING_ZERO_COPY (1 << 8) /** * INIT request/reply flags @@ -1259,6 +1268,13 @@ struct fuse_supp_groups { #define FUSE_URING_IN_OUT_HEADER_SZ 128 #define FUSE_URING_OP_IN_OUT_SZ 128 +/** + * fuse_uring_ent_in_out flags + * + * FUSE_URING_ENT_ZERO_COPY: Set if the ent's payload is zero-copied + */ +#define FUSE_URING_ENT_ZERO_COPY (1 << 0) + /* Used as part of the fuse_uring_req_header */ struct fuse_uring_ent_in_out { uint64_t flags; @@ -1310,6 +1326,14 @@ enum fuse_uring_cmd { FUSE_IO_URING_CMD_ADD_BUFPOOL = 4, }; +/* + * fuse_uring_cmd_req flags for FUSE_IO_URING_CMD_ADD_QUEUE + * + * FUSE_URING_ZERO_COPY is only supported for queues with bufpools on privileged + * servers + */ +#define FUSE_URING_ZERO_COPY (1 << 0) + /** * In the 80B command area of the SQE. */ @@ -1330,6 +1354,16 @@ struct fuse_uring_cmd_req { uint32_t len; uint32_t reserved; } bufpool; + + /* + * Index of this entry's slot in the server's io_uring + * registered buffer table, where the kernel registers the + * request's pages for zero-copy. Set for + * FUSE_IO_URING_CMD_REGISTER cmds only, and only on queues + * created with FUSE_URING_ZERO_COPY. On a non-zero-copy queue + * this must be 0 + */ + uint16_t ent_zero_copy_buf_index; }; }; -- cgit v1.2.3