fuse: add io-uring buffer pools

Right now, ents and buffers are tightly coupled in fuse io-uring where
each entry has its own dedicated payload buffer, requiring N buffers for
N entries where each buffer must be large enough to accomodate the
maximum payload size. This is suboptimal as most request types (lookup,
open, release, getattr, etc) require vastly less bytes than the maximum
payload size and some requests (unlink, rmdir, fsync, flush, etc) do not
require payload buffers at all.

Instead of requiring a 1:1 coupling between ents and payload buffers,
allow the server to pass in a buffer pool (a contiguous chunk of memory)
that the kernel will use as it wishes for servicing ents/requests.
Entries only reserve a "buffer" from the pool while actively processing
a request that requires a payload buffer. This decoupling and letting
the kernel delegate memory from the pool for requests allows the kernel to
optimize memory usage and reduces the memory usage requirements needed
to use fuse-over-io-uring.

A pool is registered per queue with the new
FUSE_IO_URING_CMD_ADD_BUFPOOL command. The server passes the pool's base
address and length in fuse_uring_cmd_req.bufpool.{uaddr,len}.

Internally, the kernel splits the region into buffers of
ring->max_payload_sz bytes each (nr_bufs = pool len / max_payload_sz). A
queue commits to a payload mode on first use: registering an entry that
carries its own payload selects the legacy per-entry mode, while
ADD_BUFPOOL selects pool mode. The two are mutually exclusive, so
ADD_BUFPOOL must be issued before any payload-carrying entries are
registered on that queue. The queue must have been created before the
bufpool is added, through the FUSE_IO_URING_CMD_ADD_QUEUE command.

The kernel tracks free buffers with a bitmap (a set bit marks a free
buffer). On dispatch, a request that needs a payload claims a free
buffer (find_first_bit + clear). A request that needs none claims
nothing. The buffer's byte offset within the pool is reported to the
server in the new fuse_uring_ent_in_out.offset field so that the server
can locate the payload. On completion the buffer is returned to the pool
or reused directly if the next request on that entry also has a payload.

The FUSE_HAS_IO_URING_BUFPOOL flag advertises kernel support to the
server for bufpools.

Buffer pool request flow
~~~~~~~~~~~~~~~~~~~~~~~~
|  Kernel                                  |  FUSE daemon
|                                          |
|  [request arrives]                       |
|    [claim a free pool buffer]            |
|    >fuse_uring_select_buffer()           |
|    [copy headers to ring]                |
|    [copy payload to buffer]              |
|    [report buffer offset in ent_in_out]  |
|    >io_uring_cmd_done()                  |
|                                          |  [read headers]
|                                          |  [read/write payload at offset]
|                                          |  [process request]
|                                          |  >io_uring_submit()
|                                          |   COMMIT_AND_FETCH
|  >fuse_uring_commit_fetch()              |
|    [copy reply from ring]                |
|    [return buffer to the pool]           |
|    >fuse_uring_recycle_buffer()          |

Reviewed-by: Bernd Schubert <bernd@bsbernd.com>
Signed-off-by: Joanne Koong <joannelkoong@gmail.com>
Signed-off-by: Miklos Szeredi <mszeredi@redhat.com>

1 files changed, 20 insertions, 1 deletions

diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h
index cfb055c0c764..538d844da099 100644
--- a/include/uapi/linux/fuse.h
+++ b/include/uapi/linux/fuse.h
@@ -243,6 +243,9 @@
  *
  *  7.46
  *  - add FUSE_IO_URING_CMD_ADD_QUEUE
+ *  - add FUSE_HAS_IO_URING_BUFPOOL
+ *  - add fuse_uring_cmd_req bufpool struct
+ *  - add bufpool offset field to fuse_uring_ent_in_out struct
  */
 
 #ifndef _LINUX_FUSE_H
@@ -451,6 +454,7 @@ struct fuse_file_lock {
  * FUSE_OVER_IO_URING: Indicate that client supports io-uring
  * FUSE_REQUEST_TIMEOUT: kernel supports timing out requests.
  *			 init_out.request_timeout contains the timeout (in secs)
+ * FUSE_HAS_IO_URING_BUFPOOL: kernel supports io-uring buffer pools
  */
 #define FUSE_ASYNC_READ		(1 << 0)
 #define FUSE_POSIX_LOCKS	(1 << 1)
@@ -498,6 +502,7 @@ struct fuse_file_lock {
 #define FUSE_ALLOW_IDMAP	(1ULL << 40)
 #define FUSE_OVER_IO_URING	(1ULL << 41)
 #define FUSE_REQUEST_TIMEOUT	(1ULL << 42)
+#define FUSE_HAS_IO_URING_BUFPOOL (1ULL << 43)
 
 /**
  * CUSE INIT request/reply flags
@@ -1266,7 +1271,9 @@ struct fuse_uring_ent_in_out {
 
 	/* size of user payload buffer */
 	uint32_t payload_sz;
-	uint32_t padding;
+
+	/* Offset into the bufpool, if bufpools are used */
+	uint32_t offset;
 
 	uint64_t reserved;
 };
@@ -1298,6 +1305,9 @@ enum fuse_uring_cmd {
 
 	/* add a queue */
 	FUSE_IO_URING_CMD_ADD_QUEUE = 3,
+
+	/* add a bufpool to a queue */
+	FUSE_IO_URING_CMD_ADD_BUFPOOL = 4,
 };
 
 /**
@@ -1312,6 +1322,15 @@ struct fuse_uring_cmd_req {
 	/* queue the command is for (queue index) */
 	uint16_t qid;
 	uint8_t padding[6];
+
+	union {
+		struct {
+			/* base address of bufpool */
+			uint64_t uaddr;
+			uint32_t len;
+			uint32_t reserved;
+		} bufpool;
+	};
 };
 
 #endif /* _LINUX_FUSE_H */