io_uring/uring_cmd.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <linux/kernel.h>
   3 #include <linux/errno.h>
   4 #include <linux/file.h>
   5 #include <linux/io_uring.h>
   6
   7 #include <uapi/linux/io_uring.h>
   8
   9 #include "io_uring.h"
  10 #include "uring_cmd.h"
  11
  12 static void io_uring_cmd_work(struct io_kiocb *req, bool *locked)
  13 {
  14         struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req);
  15
  16         ioucmd->task_work_cb(ioucmd);
  17 }
  18
  19 void io_uring_cmd_complete_in_task(struct io_uring_cmd *ioucmd,
  20                         void (*task_work_cb)(struct io_uring_cmd *))
  21 {
  22         struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
  23
  24         ioucmd->task_work_cb = task_work_cb;
  25         req->io_task_work.func = io_uring_cmd_work;
  26         io_req_task_work_add(req);
  27 }
  28 EXPORT_SYMBOL_GPL(io_uring_cmd_complete_in_task);
  29
  30 static inline void io_req_set_cqe32_extra(struct io_kiocb *req,
  31                                           u64 extra1, u64 extra2)
  32 {
  33         req->extra1 = extra1;
  34         req->extra2 = extra2;
  35         req->flags |= REQ_F_CQE32_INIT;
  36 }
  37
  38 /*
  39  * Called by consumers of io_uring_cmd, if they originally returned
  40  * -EIOCBQUEUED upon receiving the command.
  41  */
  42 void io_uring_cmd_done(struct io_uring_cmd *ioucmd, ssize_t ret, ssize_t res2)
  43 {
  44         struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
  45
  46         if (ret < 0)
  47                 req_set_fail(req);
  48
  49         io_req_set_res(req, 0, ret);
  50         if (req->ctx->flags & IORING_SETUP_CQE32)
  51                 io_req_set_cqe32_extra(req, res2, 0);
  52         __io_req_complete(req, 0);
  53 }
  54 EXPORT_SYMBOL_GPL(io_uring_cmd_done);
  55
  56 int io_uring_cmd_prep_async(struct io_kiocb *req)
  57 {
  58         struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req);
  59         size_t cmd_size;
  60
  61         cmd_size = uring_cmd_pdu_size(req->ctx->flags & IORING_SETUP_SQE128);
  62
  63         memcpy(req->async_data, ioucmd->cmd, cmd_size);
  64         return 0;
  65 }
  66
  67 int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
  68 {
  69         struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req);
  70
  71         if (sqe->rw_flags || sqe->__pad1)
  72                 return -EINVAL;
  73         ioucmd->cmd = sqe->cmd;
  74         ioucmd->cmd_op = READ_ONCE(sqe->cmd_op);
  75         return 0;
  76 }
  77
  78 int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags)
  79 {
  80         struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req);
  81         struct io_ring_ctx *ctx = req->ctx;
  82         struct file *file = req->file;
  83         int ret;
  84
  85         if (!req->file->f_op->uring_cmd)
  86                 return -EOPNOTSUPP;
  87
  88         if (ctx->flags & IORING_SETUP_SQE128)
  89                 issue_flags |= IO_URING_F_SQE128;
  90         if (ctx->flags & IORING_SETUP_CQE32)
  91                 issue_flags |= IO_URING_F_CQE32;
  92         if (ctx->flags & IORING_SETUP_IOPOLL)
  93                 issue_flags |= IO_URING_F_IOPOLL;
  94
  95         if (req_has_async_data(req))
  96                 ioucmd->cmd = req->async_data;
  97
  98         ret = file->f_op->uring_cmd(ioucmd, issue_flags);
  99         if (ret == -EAGAIN) {
 100                 if (!req_has_async_data(req)) {
 101                         if (io_alloc_async_data(req))
 102                                 return -ENOMEM;
 103                         io_uring_cmd_prep_async(req);
 104                 }
 105                 return -EAGAIN;
 106         }
 107
 108         if (ret != -EIOCBQUEUED) {
 109                 io_uring_cmd_done(ioucmd, ret, 0);
 110                 return IOU_OK;
 111         }
 112
 113         return IOU_ISSUE_SKIP_COMPLETE;
 114 }