ublk: zc register/unregister bvec

Provide new operations for the user to request mapping an active request
to an io uring instance's buf_table. The user has to provide the index
it wants to install the buffer.

A reference count is taken on the request to ensure it can't be
completed while it is active in a ring's buf_table.

Signed-off-by: Keith Busch <kbusch@kernel.org>
Link: https://lore.kernel.org/r/20250227223916.143006-6-kbusch@meta.com
Reviewed-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c
index 5290851..b5cf92b 100644 (file)
--- a/drivers/block/ublk_drv.c
+++ b/drivers/block/ublk_drv.c
@@ -51,6 +51,9 @@
 /* private ioctl command mirror */
 #define UBLK_CMD_DEL_DEV_ASYNC _IOC_NR(UBLK_U_CMD_DEL_DEV_ASYNC)
 
+#define UBLK_IO_REGISTER_IO_BUF                _IOC_NR(UBLK_U_IO_REGISTER_IO_BUF)
+#define UBLK_IO_UNREGISTER_IO_BUF      _IOC_NR(UBLK_U_IO_UNREGISTER_IO_BUF)
+
 /* All UBLK_F_* have to be included into UBLK_F_ALL */
 #define UBLK_F_ALL (UBLK_F_SUPPORT_ZERO_COPY \
                | UBLK_F_URING_CMD_COMP_IN_TASK \
@@ -196,12 +199,14 @@ struct ublk_params_header {
 
 static bool ublk_abort_requests(struct ublk_device *ub, struct ublk_queue *ubq);
 
+static inline struct request *__ublk_check_and_get_req(struct ublk_device *ub,
+               struct ublk_queue *ubq, int tag, size_t offset);
 static inline unsigned int ublk_req_build_flags(struct request *req);
 static inline struct ublksrv_io_desc *ublk_get_iod(struct ublk_queue *ubq,
                                                   int tag);
 static inline bool ublk_dev_is_user_copy(const struct ublk_device *ub)
 {
-       return ub->dev_info.flags & UBLK_F_USER_COPY;
+       return ub->dev_info.flags & (UBLK_F_USER_COPY | UBLK_F_SUPPORT_ZERO_COPY);
 }
 
 static inline bool ublk_dev_is_zoned(const struct ublk_device *ub)
@@ -581,7 +586,7 @@ static void ublk_apply_params(struct ublk_device *ub)
 
 static inline bool ublk_support_user_copy(const struct ublk_queue *ubq)
 {
-       return ubq->flags & UBLK_F_USER_COPY;
+       return ubq->flags & (UBLK_F_USER_COPY | UBLK_F_SUPPORT_ZERO_COPY);
 }
 
 static inline bool ublk_need_req_ref(const struct ublk_queue *ubq)
@@ -1747,6 +1752,45 @@ static inline void ublk_prep_cancel(struct io_uring_cmd *cmd,
        io_uring_cmd_mark_cancelable(cmd, issue_flags);
 }
 
+static void ublk_io_release(void *priv)
+{
+       struct request *rq = priv;
+       struct ublk_queue *ubq = rq->mq_hctx->driver_data;
+
+       ublk_put_req_ref(ubq, rq);
+}
+
+static int ublk_register_io_buf(struct io_uring_cmd *cmd,
+                               struct ublk_queue *ubq, unsigned int tag,
+                               const struct ublksrv_io_cmd *ub_cmd,
+                               unsigned int issue_flags)
+{
+       struct ublk_device *ub = cmd->file->private_data;
+       int index = (int)ub_cmd->addr, ret;
+       struct request *req;
+
+       req = __ublk_check_and_get_req(ub, ubq, tag, 0);
+       if (!req)
+               return -EINVAL;
+
+       ret = io_buffer_register_bvec(cmd, req, ublk_io_release, index,
+                                     issue_flags);
+       if (ret) {
+               ublk_put_req_ref(ubq, req);
+               return ret;
+       }
+
+       return 0;
+}
+
+static int ublk_unregister_io_buf(struct io_uring_cmd *cmd,
+                                 const struct ublksrv_io_cmd *ub_cmd,
+                                 unsigned int issue_flags)
+{
+       io_buffer_unregister_bvec(cmd, ub_cmd->addr, issue_flags);
+       return 0;
+}
+
 static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd,
                               unsigned int issue_flags,
                               const struct ublksrv_io_cmd *ub_cmd)
@@ -1798,6 +1842,10 @@ static int __ublk_ch_uring_cmd(struct io_uring_cmd *cmd,
 
        ret = -EINVAL;
        switch (_IOC_NR(cmd_op)) {
+       case UBLK_IO_REGISTER_IO_BUF:
+               return ublk_register_io_buf(cmd, ubq, tag, ub_cmd, issue_flags);
+       case UBLK_IO_UNREGISTER_IO_BUF:
+               return ublk_unregister_io_buf(cmd, ub_cmd, issue_flags);
        case UBLK_IO_FETCH_REQ:
                /* UBLK_IO_FETCH_REQ is only allowed before queue is setup */
                if (ublk_queue_ready(ubq)) {
@@ -2459,7 +2507,7 @@ static int ublk_ctrl_add_dev(struct io_uring_cmd *cmd)
                 * buffer by pwrite() to ublk char device, which can't be
                 * used for unprivileged device
                 */
-               if (info.flags & UBLK_F_USER_COPY)
+               if (info.flags & (UBLK_F_USER_COPY | UBLK_F_SUPPORT_ZERO_COPY))
                        return -EINVAL;
        }
 
@@ -2527,9 +2575,6 @@ static int ublk_ctrl_add_dev(struct io_uring_cmd *cmd)
                goto out_free_dev_number;
        }
 
-       /* We are not ready to support zero copy */
-       ub->dev_info.flags &= ~UBLK_F_SUPPORT_ZERO_COPY;
-
        ub->dev_info.nr_hw_queues = min_t(unsigned int,
                        ub->dev_info.nr_hw_queues, nr_cpu_ids);
        ublk_align_max_io_size(ub);
@@ -2860,7 +2905,7 @@ static int ublk_ctrl_get_features(struct io_uring_cmd *cmd)
 {
        const struct ublksrv_ctrl_cmd *header = io_uring_sqe_cmd(cmd->sqe);
        void __user *argp = (void __user *)(unsigned long)header->addr;
-       u64 features = UBLK_F_ALL & ~UBLK_F_SUPPORT_ZERO_COPY;
+       u64 features = UBLK_F_ALL;
 
        if (header->len != UBLK_FEATURES_LEN || !header->addr)
                return -EINVAL;

diff --git a/include/uapi/linux/ublk_cmd.h b/include/uapi/linux/ublk_cmd.h
index a8bc98b..74246c9 100644 (file)
--- a/include/uapi/linux/ublk_cmd.h
+++ b/include/uapi/linux/ublk_cmd.h
@@ -94,6 +94,10 @@
        _IOWR('u', UBLK_IO_COMMIT_AND_FETCH_REQ, struct ublksrv_io_cmd)
 #define        UBLK_U_IO_NEED_GET_DATA         \
        _IOWR('u', UBLK_IO_NEED_GET_DATA, struct ublksrv_io_cmd)
+#define        UBLK_U_IO_REGISTER_IO_BUF       \
+       _IOWR('u', 0x23, struct ublksrv_io_cmd)
+#define        UBLK_U_IO_UNREGISTER_IO_BUF     \
+       _IOWR('u', 0x24, struct ublksrv_io_cmd)
 
 /* only ABORT means that no re-fetch */
 #define UBLK_IO_RES_OK                 0