1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/kernel.h>
3 #include <linux/errno.h>
5 #include <linux/file.h>
7 #include <linux/slab.h>
8 #include <linux/namei.h>
9 #include <linux/nospec.h>
10 #include <linux/io_uring.h>
12 #include <uapi/linux/io_uring.h>
30 #define CANCEL_FLAGS (IORING_ASYNC_CANCEL_ALL | IORING_ASYNC_CANCEL_FD | \
31 IORING_ASYNC_CANCEL_ANY | IORING_ASYNC_CANCEL_FD_FIXED | \
32 IORING_ASYNC_CANCEL_USERDATA | IORING_ASYNC_CANCEL_OP)
35 * Returns true if the request matches the criteria outlined by 'cd'.
37 bool io_cancel_req_match(struct io_kiocb *req, struct io_cancel_data *cd)
39 bool match_user_data = cd->flags & IORING_ASYNC_CANCEL_USERDATA;
41 if (req->ctx != cd->ctx)
44 if (!(cd->flags & (IORING_ASYNC_CANCEL_FD | IORING_ASYNC_CANCEL_OP)))
45 match_user_data = true;
47 if (cd->flags & IORING_ASYNC_CANCEL_ANY)
49 if (cd->flags & IORING_ASYNC_CANCEL_FD) {
50 if (req->file != cd->file)
53 if (cd->flags & IORING_ASYNC_CANCEL_OP) {
54 if (req->opcode != cd->opcode)
57 if (match_user_data && req->cqe.user_data != cd->data)
59 if (cd->flags & IORING_ASYNC_CANCEL_ALL) {
61 if (cd->seq == req->work.cancel_seq)
63 req->work.cancel_seq = cd->seq;
69 static bool io_cancel_cb(struct io_wq_work *work, void *data)
71 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
72 struct io_cancel_data *cd = data;
74 return io_cancel_req_match(req, cd);
77 static int io_async_cancel_one(struct io_uring_task *tctx,
78 struct io_cancel_data *cd)
80 enum io_wq_cancel cancel_ret;
84 if (!tctx || !tctx->io_wq)
87 all = cd->flags & (IORING_ASYNC_CANCEL_ALL|IORING_ASYNC_CANCEL_ANY);
88 cancel_ret = io_wq_cancel_cb(tctx->io_wq, io_cancel_cb, cd, all);
93 case IO_WQ_CANCEL_RUNNING:
96 case IO_WQ_CANCEL_NOTFOUND:
104 int io_try_cancel(struct io_uring_task *tctx, struct io_cancel_data *cd,
105 unsigned issue_flags)
107 struct io_ring_ctx *ctx = cd->ctx;
110 WARN_ON_ONCE(!io_wq_current_is_worker() && tctx != current->io_uring);
112 ret = io_async_cancel_one(tctx, cd);
114 * Fall-through even for -EALREADY, as we may have poll armed
115 * that need unarming.
120 ret = io_poll_cancel(ctx, cd, issue_flags);
124 ret = io_waitid_cancel(ctx, cd, issue_flags);
128 ret = io_futex_cancel(ctx, cd, issue_flags);
132 spin_lock(&ctx->completion_lock);
133 if (!(cd->flags & IORING_ASYNC_CANCEL_FD))
134 ret = io_timeout_cancel(ctx, cd);
135 spin_unlock(&ctx->completion_lock);
139 int io_async_cancel_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
141 struct io_cancel *cancel = io_kiocb_to_cmd(req, struct io_cancel);
143 if (unlikely(req->flags & REQ_F_BUFFER_SELECT))
145 if (sqe->off || sqe->splice_fd_in)
148 cancel->addr = READ_ONCE(sqe->addr);
149 cancel->flags = READ_ONCE(sqe->cancel_flags);
150 if (cancel->flags & ~CANCEL_FLAGS)
152 if (cancel->flags & IORING_ASYNC_CANCEL_FD) {
153 if (cancel->flags & IORING_ASYNC_CANCEL_ANY)
155 cancel->fd = READ_ONCE(sqe->fd);
157 if (cancel->flags & IORING_ASYNC_CANCEL_OP) {
158 if (cancel->flags & IORING_ASYNC_CANCEL_ANY)
160 cancel->opcode = READ_ONCE(sqe->len);
166 static int __io_async_cancel(struct io_cancel_data *cd,
167 struct io_uring_task *tctx,
168 unsigned int issue_flags)
170 bool all = cd->flags & (IORING_ASYNC_CANCEL_ALL|IORING_ASYNC_CANCEL_ANY);
171 struct io_ring_ctx *ctx = cd->ctx;
172 struct io_tctx_node *node;
176 ret = io_try_cancel(tctx, cd, issue_flags);
184 /* slow path, try all io-wq's */
185 io_ring_submit_lock(ctx, issue_flags);
187 list_for_each_entry(node, &ctx->tctx_list, ctx_node) {
188 struct io_uring_task *tctx = node->task->io_uring;
190 ret = io_async_cancel_one(tctx, cd);
191 if (ret != -ENOENT) {
197 io_ring_submit_unlock(ctx, issue_flags);
198 return all ? nr : ret;
201 int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags)
203 struct io_cancel *cancel = io_kiocb_to_cmd(req, struct io_cancel);
204 struct io_cancel_data cd = {
206 .data = cancel->addr,
207 .flags = cancel->flags,
208 .opcode = cancel->opcode,
209 .seq = atomic_inc_return(&req->ctx->cancel_seq),
211 struct io_uring_task *tctx = req->task->io_uring;
214 if (cd.flags & IORING_ASYNC_CANCEL_FD) {
215 if (req->flags & REQ_F_FIXED_FILE ||
216 cd.flags & IORING_ASYNC_CANCEL_FD_FIXED) {
217 req->flags |= REQ_F_FIXED_FILE;
218 req->file = io_file_get_fixed(req, cancel->fd,
221 req->file = io_file_get_normal(req, cancel->fd);
230 ret = __io_async_cancel(&cd, tctx, issue_flags);
234 io_req_set_res(req, ret, 0);
238 void init_hash_table(struct io_hash_table *table, unsigned size)
242 for (i = 0; i < size; i++) {
243 spin_lock_init(&table->hbs[i].lock);
244 INIT_HLIST_HEAD(&table->hbs[i].list);
248 static int __io_sync_cancel(struct io_uring_task *tctx,
249 struct io_cancel_data *cd, int fd)
251 struct io_ring_ctx *ctx = cd->ctx;
253 /* fixed must be grabbed every time since we drop the uring_lock */
254 if ((cd->flags & IORING_ASYNC_CANCEL_FD) &&
255 (cd->flags & IORING_ASYNC_CANCEL_FD_FIXED)) {
256 if (unlikely(fd >= ctx->nr_user_files))
258 fd = array_index_nospec(fd, ctx->nr_user_files);
259 cd->file = io_file_from_index(&ctx->file_table, fd);
264 return __io_async_cancel(cd, tctx, 0);
267 int io_sync_cancel(struct io_ring_ctx *ctx, void __user *arg)
268 __must_hold(&ctx->uring_lock)
270 struct io_cancel_data cd = {
272 .seq = atomic_inc_return(&ctx->cancel_seq),
274 ktime_t timeout = KTIME_MAX;
275 struct io_uring_sync_cancel_reg sc;
280 if (copy_from_user(&sc, arg, sizeof(sc)))
282 if (sc.flags & ~CANCEL_FLAGS)
284 for (i = 0; i < ARRAY_SIZE(sc.pad); i++)
287 for (i = 0; i < ARRAY_SIZE(sc.pad2); i++)
293 cd.opcode = sc.opcode;
295 /* we can grab a normal file descriptor upfront */
296 if ((cd.flags & IORING_ASYNC_CANCEL_FD) &&
297 !(cd.flags & IORING_ASYNC_CANCEL_FD_FIXED)) {
304 ret = __io_sync_cancel(current->io_uring, &cd, sc.fd);
306 /* found something, done! */
307 if (ret != -EALREADY)
310 if (sc.timeout.tv_sec != -1UL || sc.timeout.tv_nsec != -1UL) {
311 struct timespec64 ts = {
312 .tv_sec = sc.timeout.tv_sec,
313 .tv_nsec = sc.timeout.tv_nsec
316 timeout = ktime_add_ns(timespec64_to_ktime(ts), ktime_get_ns());
320 * Keep looking until we get -ENOENT. we'll get woken everytime
321 * every time a request completes and will retry the cancelation.
324 cd.seq = atomic_inc_return(&ctx->cancel_seq);
326 prepare_to_wait(&ctx->cq_wait, &wait, TASK_INTERRUPTIBLE);
328 ret = __io_sync_cancel(current->io_uring, &cd, sc.fd);
330 mutex_unlock(&ctx->uring_lock);
331 if (ret != -EALREADY)
334 ret = io_run_task_work_sig(ctx);
337 ret = schedule_hrtimeout(&timeout, HRTIMER_MODE_ABS);
342 mutex_lock(&ctx->uring_lock);
345 finish_wait(&ctx->cq_wait, &wait);
346 mutex_lock(&ctx->uring_lock);
348 if (ret == -ENOENT || ret > 0)