1 // SPDX-License-Identifier: GPL-2.0
3 * Shared application/kernel submission and completion ring pairs, for
4 * supporting fast/efficient IO.
6 * A note on the read/write ordering memory barriers that are matched between
7 * the application and kernel side.
9 * After the application reads the CQ ring tail, it must use an
10 * appropriate smp_rmb() to pair with the smp_wmb() the kernel uses
11 * before writing the tail (using smp_load_acquire to read the tail will
12 * do). It also needs a smp_mb() before updating CQ head (ordering the
13 * entry load(s) with the head store), pairing with an implicit barrier
14 * through a control-dependency in io_get_cqe (smp_store_release to
15 * store head will do). Failure to do so could lead to reading invalid
18 * Likewise, the application must use an appropriate smp_wmb() before
19 * writing the SQ tail (ordering SQ entry stores with the tail store),
20 * which pairs with smp_load_acquire in io_get_sqring (smp_store_release
21 * to store the tail will do). And it needs a barrier ordering the SQ
22 * head load before writing new SQ entries (smp_load_acquire to read
25 * When using the SQ poll thread (IORING_SETUP_SQPOLL), the application
26 * needs to check the SQ flags for IORING_SQ_NEED_WAKEUP *after*
27 * updating the SQ tail; a full memory barrier smp_mb() is needed
30 * Also see the examples in the liburing library:
32 * git://git.kernel.dk/liburing
34 * io_uring also uses READ/WRITE_ONCE() for _any_ store or load that happens
35 * from data shared between the kernel and application. This is done both
36 * for ordering purposes, but also to ensure that once a value is loaded from
37 * data that the application could potentially modify, it remains stable.
39 * Copyright (C) 2018-2019 Jens Axboe
40 * Copyright (c) 2018-2019 Christoph Hellwig
42 #include <linux/kernel.h>
43 #include <linux/init.h>
44 #include <linux/errno.h>
45 #include <linux/syscalls.h>
46 #include <linux/compat.h>
47 #include <net/compat.h>
48 #include <linux/refcount.h>
49 #include <linux/uio.h>
50 #include <linux/bits.h>
52 #include <linux/sched/signal.h>
54 #include <linux/file.h>
55 #include <linux/fdtable.h>
57 #include <linux/mman.h>
58 #include <linux/percpu.h>
59 #include <linux/slab.h>
60 #include <linux/blkdev.h>
61 #include <linux/bvec.h>
62 #include <linux/net.h>
64 #include <net/af_unix.h>
66 #include <linux/anon_inodes.h>
67 #include <linux/sched/mm.h>
68 #include <linux/uaccess.h>
69 #include <linux/nospec.h>
70 #include <linux/sizes.h>
71 #include <linux/hugetlb.h>
72 #include <linux/highmem.h>
73 #include <linux/namei.h>
74 #include <linux/fsnotify.h>
75 #include <linux/fadvise.h>
76 #include <linux/eventpoll.h>
77 #include <linux/splice.h>
78 #include <linux/task_work.h>
79 #include <linux/pagemap.h>
80 #include <linux/io_uring.h>
82 #define CREATE_TRACE_POINTS
83 #include <trace/events/io_uring.h>
85 #include <uapi/linux/io_uring.h>
90 #define IORING_MAX_ENTRIES 32768
91 #define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES)
92 #define IORING_SQPOLL_CAP_ENTRIES_VALUE 8
95 * Shift of 9 is 512 entries, or exactly one page on 64-bit archs
97 #define IORING_FILE_TABLE_SHIFT 9
98 #define IORING_MAX_FILES_TABLE (1U << IORING_FILE_TABLE_SHIFT)
99 #define IORING_FILE_TABLE_MASK (IORING_MAX_FILES_TABLE - 1)
100 #define IORING_MAX_FIXED_FILES (64 * IORING_MAX_FILES_TABLE)
101 #define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \
102 IORING_REGISTER_LAST + IORING_OP_LAST)
104 #define IO_RSRC_TAG_TABLE_SHIFT 9
105 #define IO_RSRC_TAG_TABLE_MAX (1U << IO_RSRC_TAG_TABLE_SHIFT)
106 #define IO_RSRC_TAG_TABLE_MASK (IO_RSRC_TAG_TABLE_MAX - 1)
108 #define IORING_MAX_REG_BUFFERS (1U << 14)
110 #define SQE_VALID_FLAGS (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK| \
111 IOSQE_IO_HARDLINK | IOSQE_ASYNC | \
113 #define IO_REQ_CLEAN_FLAGS (REQ_F_BUFFER_SELECTED | REQ_F_NEED_CLEANUP | \
114 REQ_F_POLLED | REQ_F_INFLIGHT | REQ_F_CREDS)
116 #define IO_TCTX_REFS_CACHE_NR (1U << 10)
119 u32 head ____cacheline_aligned_in_smp;
120 u32 tail ____cacheline_aligned_in_smp;
124 * This data is shared with the application through the mmap at offsets
125 * IORING_OFF_SQ_RING and IORING_OFF_CQ_RING.
127 * The offsets to the member fields are published through struct
128 * io_sqring_offsets when calling io_uring_setup.
132 * Head and tail offsets into the ring; the offsets need to be
133 * masked to get valid indices.
135 * The kernel controls head of the sq ring and the tail of the cq ring,
136 * and the application controls tail of the sq ring and the head of the
139 struct io_uring sq, cq;
141 * Bitmasks to apply to head and tail offsets (constant, equals
144 u32 sq_ring_mask, cq_ring_mask;
145 /* Ring sizes (constant, power of 2) */
146 u32 sq_ring_entries, cq_ring_entries;
148 * Number of invalid entries dropped by the kernel due to
149 * invalid index stored in array
151 * Written by the kernel, shouldn't be modified by the
152 * application (i.e. get number of "new events" by comparing to
155 * After a new SQ head value was read by the application this
156 * counter includes all submissions that were dropped reaching
157 * the new SQ head (and possibly more).
163 * Written by the kernel, shouldn't be modified by the
166 * The application needs a full memory barrier before checking
167 * for IORING_SQ_NEED_WAKEUP after updating the sq tail.
173 * Written by the application, shouldn't be modified by the
178 * Number of completion events lost because the queue was full;
179 * this should be avoided by the application by making sure
180 * there are not more requests pending than there is space in
181 * the completion queue.
183 * Written by the kernel, shouldn't be modified by the
184 * application (i.e. get number of "new events" by comparing to
187 * As completion events come in out of order this counter is not
188 * ordered with any other data.
192 * Ring buffer of completion events.
194 * The kernel writes completion events fresh every time they are
195 * produced, so the application is allowed to modify pending
198 struct io_uring_cqe cqes[] ____cacheline_aligned_in_smp;
201 enum io_uring_cmd_flags {
202 IO_URING_F_NONBLOCK = 1,
203 IO_URING_F_COMPLETE_DEFER = 2,
206 struct io_mapped_ubuf {
209 unsigned int nr_bvecs;
210 unsigned long acct_pages;
211 struct bio_vec bvec[];
216 struct io_overflow_cqe {
217 struct io_uring_cqe cqe;
218 struct list_head list;
221 struct io_fixed_file {
222 /* file * with additional FFS_* flags */
223 unsigned long file_ptr;
227 struct list_head list;
232 struct io_mapped_ubuf *buf;
236 struct io_file_table {
237 /* two level table */
238 struct io_fixed_file **files;
241 struct io_rsrc_node {
242 struct percpu_ref refs;
243 struct list_head node;
244 struct list_head rsrc_list;
245 struct io_rsrc_data *rsrc_data;
246 struct llist_node llist;
250 typedef void (rsrc_put_fn)(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc);
252 struct io_rsrc_data {
253 struct io_ring_ctx *ctx;
259 struct completion done;
264 struct list_head list;
270 struct io_restriction {
271 DECLARE_BITMAP(register_op, IORING_REGISTER_LAST);
272 DECLARE_BITMAP(sqe_op, IORING_OP_LAST);
273 u8 sqe_flags_allowed;
274 u8 sqe_flags_required;
279 IO_SQ_THREAD_SHOULD_STOP = 0,
280 IO_SQ_THREAD_SHOULD_PARK,
285 atomic_t park_pending;
288 /* ctx's that are using this sqd */
289 struct list_head ctx_list;
291 struct task_struct *thread;
292 struct wait_queue_head wait;
294 unsigned sq_thread_idle;
300 struct completion exited;
303 #define IO_IOPOLL_BATCH 8
304 #define IO_COMPL_BATCH 32
305 #define IO_REQ_CACHE_SIZE 32
306 #define IO_REQ_ALLOC_BATCH 8
308 struct io_comp_state {
309 struct io_kiocb *reqs[IO_COMPL_BATCH];
311 /* inline/task_work completion list, under ->uring_lock */
312 struct list_head free_list;
315 struct io_submit_link {
316 struct io_kiocb *head;
317 struct io_kiocb *last;
320 struct io_submit_state {
321 struct blk_plug plug;
322 struct io_submit_link link;
325 * io_kiocb alloc cache
327 void *reqs[IO_REQ_CACHE_SIZE];
328 unsigned int free_reqs;
333 * Batch completion logic
335 struct io_comp_state comp;
338 * File reference cache
342 unsigned int file_refs;
343 unsigned int ios_left;
347 /* const or read-mostly hot data */
349 struct percpu_ref refs;
351 struct io_rings *rings;
353 unsigned int compat: 1;
354 unsigned int drain_next: 1;
355 unsigned int eventfd_async: 1;
356 unsigned int restricted: 1;
357 unsigned int off_timeout_used: 1;
358 unsigned int drain_active: 1;
359 } ____cacheline_aligned_in_smp;
361 /* submission data */
363 struct mutex uring_lock;
366 * Ring buffer of indices into array of io_uring_sqe, which is
367 * mmapped by the application using the IORING_OFF_SQES offset.
369 * This indirection could e.g. be used to assign fixed
370 * io_uring_sqe entries to operations and only submit them to
371 * the queue when needed.
373 * The kernel modifies neither the indices array nor the entries
377 struct io_uring_sqe *sq_sqes;
378 unsigned cached_sq_head;
380 struct list_head defer_list;
383 * Fixed resources fast path, should be accessed only under
384 * uring_lock, and updated through io_uring_register(2)
386 struct io_rsrc_node *rsrc_node;
387 struct io_file_table file_table;
388 unsigned nr_user_files;
389 unsigned nr_user_bufs;
390 struct io_mapped_ubuf **user_bufs;
392 struct io_submit_state submit_state;
393 struct list_head timeout_list;
394 struct list_head cq_overflow_list;
395 struct xarray io_buffers;
396 struct xarray personalities;
398 unsigned sq_thread_idle;
399 } ____cacheline_aligned_in_smp;
401 /* IRQ completion list, under ->completion_lock */
402 struct list_head locked_free_list;
403 unsigned int locked_free_nr;
405 const struct cred *sq_creds; /* cred used for __io_sq_thread() */
406 struct io_sq_data *sq_data; /* if using sq thread polling */
408 struct wait_queue_head sqo_sq_wait;
409 struct list_head sqd_list;
411 unsigned long check_cq_overflow;
414 unsigned cached_cq_tail;
416 struct eventfd_ctx *cq_ev_fd;
417 struct wait_queue_head poll_wait;
418 struct wait_queue_head cq_wait;
420 atomic_t cq_timeouts;
421 struct fasync_struct *cq_fasync;
422 unsigned cq_last_tm_flush;
423 } ____cacheline_aligned_in_smp;
426 spinlock_t completion_lock;
429 * ->iopoll_list is protected by the ctx->uring_lock for
430 * io_uring instances that don't use IORING_SETUP_SQPOLL.
431 * For SQPOLL, only the single threaded io_sq_thread() will
432 * manipulate the list, hence no extra locking is needed there.
434 struct list_head iopoll_list;
435 struct hlist_head *cancel_hash;
436 unsigned cancel_hash_bits;
437 bool poll_multi_queue;
438 } ____cacheline_aligned_in_smp;
440 struct io_restriction restrictions;
442 /* slow path rsrc auxilary data, used by update/register */
444 struct io_rsrc_node *rsrc_backup_node;
445 struct io_mapped_ubuf *dummy_ubuf;
446 struct io_rsrc_data *file_data;
447 struct io_rsrc_data *buf_data;
449 struct delayed_work rsrc_put_work;
450 struct llist_head rsrc_put_llist;
451 struct list_head rsrc_ref_list;
452 spinlock_t rsrc_ref_lock;
455 /* Keep this last, we don't need it for the fast path */
457 #if defined(CONFIG_UNIX)
458 struct socket *ring_sock;
460 /* hashed buffered write serialization */
461 struct io_wq_hash *hash_map;
463 /* Only used for accounting purposes */
464 struct user_struct *user;
465 struct mm_struct *mm_account;
467 /* ctx exit and cancelation */
468 struct llist_head fallback_llist;
469 struct delayed_work fallback_work;
470 struct work_struct exit_work;
471 struct list_head tctx_list;
472 struct completion ref_comp;
476 struct io_uring_task {
477 /* submission side */
480 struct wait_queue_head wait;
481 const struct io_ring_ctx *last;
483 struct percpu_counter inflight;
484 atomic_t inflight_tracked;
487 spinlock_t task_lock;
488 struct io_wq_work_list task_list;
489 unsigned long task_state;
490 struct callback_head task_work;
494 * First field must be the file pointer in all the
495 * iocb unions! See also 'struct kiocb' in <linux/fs.h>
497 struct io_poll_iocb {
499 struct wait_queue_head *head;
503 struct wait_queue_entry wait;
506 struct io_poll_update {
512 bool update_user_data;
520 struct io_timeout_data {
521 struct io_kiocb *req;
522 struct hrtimer timer;
523 struct timespec64 ts;
524 enum hrtimer_mode mode;
529 struct sockaddr __user *addr;
530 int __user *addr_len;
532 unsigned long nofile;
552 struct list_head list;
553 /* head of the link, used by linked timeouts only */
554 struct io_kiocb *head;
557 struct io_timeout_rem {
562 struct timespec64 ts;
567 /* NOTE: kiocb has the file as the first member, so don't do it here */
575 struct sockaddr __user *addr;
582 struct compat_msghdr __user *umsg_compat;
583 struct user_msghdr __user *umsg;
589 struct io_buffer *kbuf;
595 struct filename *filename;
597 unsigned long nofile;
600 struct io_rsrc_update {
626 struct epoll_event event;
630 struct file *file_out;
631 struct file *file_in;
638 struct io_provide_buf {
652 const char __user *filename;
653 struct statx __user *buffer;
665 struct filename *oldpath;
666 struct filename *newpath;
674 struct filename *filename;
677 struct io_completion {
679 struct list_head list;
683 struct io_async_connect {
684 struct sockaddr_storage address;
687 struct io_async_msghdr {
688 struct iovec fast_iov[UIO_FASTIOV];
689 /* points to an allocated iov, if NULL we use fast_iov instead */
690 struct iovec *free_iov;
691 struct sockaddr __user *uaddr;
693 struct sockaddr_storage addr;
697 struct iovec fast_iov[UIO_FASTIOV];
698 const struct iovec *free_iovec;
699 struct iov_iter iter;
701 struct wait_page_queue wpq;
705 REQ_F_FIXED_FILE_BIT = IOSQE_FIXED_FILE_BIT,
706 REQ_F_IO_DRAIN_BIT = IOSQE_IO_DRAIN_BIT,
707 REQ_F_LINK_BIT = IOSQE_IO_LINK_BIT,
708 REQ_F_HARDLINK_BIT = IOSQE_IO_HARDLINK_BIT,
709 REQ_F_FORCE_ASYNC_BIT = IOSQE_ASYNC_BIT,
710 REQ_F_BUFFER_SELECT_BIT = IOSQE_BUFFER_SELECT_BIT,
712 /* first byte is taken by user flags, shift it to not overlap */
717 REQ_F_LINK_TIMEOUT_BIT,
718 REQ_F_NEED_CLEANUP_BIT,
720 REQ_F_BUFFER_SELECTED_BIT,
721 REQ_F_LTIMEOUT_ACTIVE_BIT,
722 REQ_F_COMPLETE_INLINE_BIT,
724 REQ_F_DONT_REISSUE_BIT,
726 /* keep async read/write and isreg together and in order */
727 REQ_F_ASYNC_READ_BIT,
728 REQ_F_ASYNC_WRITE_BIT,
731 /* not a real bit, just to check we're not overflowing the space */
737 REQ_F_FIXED_FILE = BIT(REQ_F_FIXED_FILE_BIT),
738 /* drain existing IO first */
739 REQ_F_IO_DRAIN = BIT(REQ_F_IO_DRAIN_BIT),
741 REQ_F_LINK = BIT(REQ_F_LINK_BIT),
742 /* doesn't sever on completion < 0 */
743 REQ_F_HARDLINK = BIT(REQ_F_HARDLINK_BIT),
745 REQ_F_FORCE_ASYNC = BIT(REQ_F_FORCE_ASYNC_BIT),
746 /* IOSQE_BUFFER_SELECT */
747 REQ_F_BUFFER_SELECT = BIT(REQ_F_BUFFER_SELECT_BIT),
749 /* fail rest of links */
750 REQ_F_FAIL = BIT(REQ_F_FAIL_BIT),
751 /* on inflight list, should be cancelled and waited on exit reliably */
752 REQ_F_INFLIGHT = BIT(REQ_F_INFLIGHT_BIT),
753 /* read/write uses file position */
754 REQ_F_CUR_POS = BIT(REQ_F_CUR_POS_BIT),
755 /* must not punt to workers */
756 REQ_F_NOWAIT = BIT(REQ_F_NOWAIT_BIT),
757 /* has or had linked timeout */
758 REQ_F_LINK_TIMEOUT = BIT(REQ_F_LINK_TIMEOUT_BIT),
760 REQ_F_NEED_CLEANUP = BIT(REQ_F_NEED_CLEANUP_BIT),
761 /* already went through poll handler */
762 REQ_F_POLLED = BIT(REQ_F_POLLED_BIT),
763 /* buffer already selected */
764 REQ_F_BUFFER_SELECTED = BIT(REQ_F_BUFFER_SELECTED_BIT),
765 /* linked timeout is active, i.e. prepared by link's head */
766 REQ_F_LTIMEOUT_ACTIVE = BIT(REQ_F_LTIMEOUT_ACTIVE_BIT),
767 /* completion is deferred through io_comp_state */
768 REQ_F_COMPLETE_INLINE = BIT(REQ_F_COMPLETE_INLINE_BIT),
769 /* caller should reissue async */
770 REQ_F_REISSUE = BIT(REQ_F_REISSUE_BIT),
771 /* don't attempt request reissue, see io_rw_reissue() */
772 REQ_F_DONT_REISSUE = BIT(REQ_F_DONT_REISSUE_BIT),
773 /* supports async reads */
774 REQ_F_ASYNC_READ = BIT(REQ_F_ASYNC_READ_BIT),
775 /* supports async writes */
776 REQ_F_ASYNC_WRITE = BIT(REQ_F_ASYNC_WRITE_BIT),
778 REQ_F_ISREG = BIT(REQ_F_ISREG_BIT),
779 /* has creds assigned */
780 REQ_F_CREDS = BIT(REQ_F_CREDS_BIT),
784 struct io_poll_iocb poll;
785 struct io_poll_iocb *double_poll;
788 typedef void (*io_req_tw_func_t)(struct io_kiocb *req);
790 struct io_task_work {
792 struct io_wq_work_node node;
793 struct llist_node fallback_node;
795 io_req_tw_func_t func;
799 IORING_RSRC_FILE = 0,
800 IORING_RSRC_BUFFER = 1,
804 * NOTE! Each of the iocb union members has the file pointer
805 * as the first entry in their struct definition. So you can
806 * access the file pointer through any of the sub-structs,
807 * or directly as just 'ki_filp' in this struct.
813 struct io_poll_iocb poll;
814 struct io_poll_update poll_update;
815 struct io_accept accept;
817 struct io_cancel cancel;
818 struct io_timeout timeout;
819 struct io_timeout_rem timeout_rem;
820 struct io_connect connect;
821 struct io_sr_msg sr_msg;
823 struct io_close close;
824 struct io_rsrc_update rsrc_update;
825 struct io_fadvise fadvise;
826 struct io_madvise madvise;
827 struct io_epoll epoll;
828 struct io_splice splice;
829 struct io_provide_buf pbuf;
830 struct io_statx statx;
831 struct io_shutdown shutdown;
832 struct io_rename rename;
833 struct io_unlink unlink;
834 /* use only after cleaning per-op data, see io_clean_op() */
835 struct io_completion compl;
838 /* opcode allocated if it needs to store data for async defer */
841 /* polled IO has completed */
847 struct io_ring_ctx *ctx;
850 struct task_struct *task;
853 struct io_kiocb *link;
854 struct percpu_ref *fixed_rsrc_refs;
856 /* used with ctx->iopoll_list with reads/writes */
857 struct list_head inflight_entry;
858 struct io_task_work io_task_work;
859 /* for polled requests, i.e. IORING_OP_POLL_ADD and async armed poll */
860 struct hlist_node hash_node;
861 struct async_poll *apoll;
862 struct io_wq_work work;
863 const struct cred *creds;
865 /* store used ubuf, so we can prevent reloading */
866 struct io_mapped_ubuf *imu;
869 struct io_tctx_node {
870 struct list_head ctx_node;
871 struct task_struct *task;
872 struct io_ring_ctx *ctx;
875 struct io_defer_entry {
876 struct list_head list;
877 struct io_kiocb *req;
882 /* needs req->file assigned */
883 unsigned needs_file : 1;
884 /* hash wq insertion if file is a regular file */
885 unsigned hash_reg_file : 1;
886 /* unbound wq insertion if file is a non-regular file */
887 unsigned unbound_nonreg_file : 1;
888 /* opcode is not supported by this kernel */
889 unsigned not_supported : 1;
890 /* set if opcode supports polled "wait" */
892 unsigned pollout : 1;
893 /* op supports buffer selection */
894 unsigned buffer_select : 1;
895 /* do prep async if is going to be punted */
896 unsigned needs_async_setup : 1;
897 /* should block plug */
899 /* size of async data needed, if any */
900 unsigned short async_size;
903 static const struct io_op_def io_op_defs[] = {
904 [IORING_OP_NOP] = {},
905 [IORING_OP_READV] = {
907 .unbound_nonreg_file = 1,
910 .needs_async_setup = 1,
912 .async_size = sizeof(struct io_async_rw),
914 [IORING_OP_WRITEV] = {
917 .unbound_nonreg_file = 1,
919 .needs_async_setup = 1,
921 .async_size = sizeof(struct io_async_rw),
923 [IORING_OP_FSYNC] = {
926 [IORING_OP_READ_FIXED] = {
928 .unbound_nonreg_file = 1,
931 .async_size = sizeof(struct io_async_rw),
933 [IORING_OP_WRITE_FIXED] = {
936 .unbound_nonreg_file = 1,
939 .async_size = sizeof(struct io_async_rw),
941 [IORING_OP_POLL_ADD] = {
943 .unbound_nonreg_file = 1,
945 [IORING_OP_POLL_REMOVE] = {},
946 [IORING_OP_SYNC_FILE_RANGE] = {
949 [IORING_OP_SENDMSG] = {
951 .unbound_nonreg_file = 1,
953 .needs_async_setup = 1,
954 .async_size = sizeof(struct io_async_msghdr),
956 [IORING_OP_RECVMSG] = {
958 .unbound_nonreg_file = 1,
961 .needs_async_setup = 1,
962 .async_size = sizeof(struct io_async_msghdr),
964 [IORING_OP_TIMEOUT] = {
965 .async_size = sizeof(struct io_timeout_data),
967 [IORING_OP_TIMEOUT_REMOVE] = {
968 /* used by timeout updates' prep() */
970 [IORING_OP_ACCEPT] = {
972 .unbound_nonreg_file = 1,
975 [IORING_OP_ASYNC_CANCEL] = {},
976 [IORING_OP_LINK_TIMEOUT] = {
977 .async_size = sizeof(struct io_timeout_data),
979 [IORING_OP_CONNECT] = {
981 .unbound_nonreg_file = 1,
983 .needs_async_setup = 1,
984 .async_size = sizeof(struct io_async_connect),
986 [IORING_OP_FALLOCATE] = {
989 [IORING_OP_OPENAT] = {},
990 [IORING_OP_CLOSE] = {},
991 [IORING_OP_FILES_UPDATE] = {},
992 [IORING_OP_STATX] = {},
995 .unbound_nonreg_file = 1,
999 .async_size = sizeof(struct io_async_rw),
1001 [IORING_OP_WRITE] = {
1003 .unbound_nonreg_file = 1,
1006 .async_size = sizeof(struct io_async_rw),
1008 [IORING_OP_FADVISE] = {
1011 [IORING_OP_MADVISE] = {},
1012 [IORING_OP_SEND] = {
1014 .unbound_nonreg_file = 1,
1017 [IORING_OP_RECV] = {
1019 .unbound_nonreg_file = 1,
1023 [IORING_OP_OPENAT2] = {
1025 [IORING_OP_EPOLL_CTL] = {
1026 .unbound_nonreg_file = 1,
1028 [IORING_OP_SPLICE] = {
1031 .unbound_nonreg_file = 1,
1033 [IORING_OP_PROVIDE_BUFFERS] = {},
1034 [IORING_OP_REMOVE_BUFFERS] = {},
1038 .unbound_nonreg_file = 1,
1040 [IORING_OP_SHUTDOWN] = {
1043 [IORING_OP_RENAMEAT] = {},
1044 [IORING_OP_UNLINKAT] = {},
1047 static bool io_disarm_next(struct io_kiocb *req);
1048 static void io_uring_del_tctx_node(unsigned long index);
1049 static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
1050 struct task_struct *task,
1052 static void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd);
1053 static struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx);
1055 static bool io_cqring_fill_event(struct io_ring_ctx *ctx, u64 user_data,
1056 long res, unsigned int cflags);
1057 static void io_put_req(struct io_kiocb *req);
1058 static void io_put_req_deferred(struct io_kiocb *req, int nr);
1059 static void io_dismantle_req(struct io_kiocb *req);
1060 static void io_put_task(struct task_struct *task, int nr);
1061 static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req);
1062 static void io_queue_linked_timeout(struct io_kiocb *req);
1063 static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,
1064 struct io_uring_rsrc_update2 *up,
1066 static void io_clean_op(struct io_kiocb *req);
1067 static struct file *io_file_get(struct io_submit_state *state,
1068 struct io_kiocb *req, int fd, bool fixed);
1069 static void __io_queue_sqe(struct io_kiocb *req);
1070 static void io_rsrc_put_work(struct work_struct *work);
1072 static void io_req_task_queue(struct io_kiocb *req);
1073 static void io_submit_flush_completions(struct io_ring_ctx *ctx);
1074 static bool io_poll_remove_waitqs(struct io_kiocb *req);
1075 static int io_req_prep_async(struct io_kiocb *req);
1077 static void io_fallback_req_func(struct work_struct *unused);
1079 static struct kmem_cache *req_cachep;
1081 static const struct file_operations io_uring_fops;
1083 struct sock *io_uring_get_socket(struct file *file)
1085 #if defined(CONFIG_UNIX)
1086 if (file->f_op == &io_uring_fops) {
1087 struct io_ring_ctx *ctx = file->private_data;
1089 return ctx->ring_sock->sk;
1094 EXPORT_SYMBOL(io_uring_get_socket);
1096 #define io_for_each_link(pos, head) \
1097 for (pos = (head); pos; pos = pos->link)
1099 static inline void io_req_set_rsrc_node(struct io_kiocb *req)
1101 struct io_ring_ctx *ctx = req->ctx;
1103 if (!req->fixed_rsrc_refs) {
1104 req->fixed_rsrc_refs = &ctx->rsrc_node->refs;
1105 percpu_ref_get(req->fixed_rsrc_refs);
1109 static void io_refs_resurrect(struct percpu_ref *ref, struct completion *compl)
1111 bool got = percpu_ref_tryget(ref);
1113 /* already at zero, wait for ->release() */
1115 wait_for_completion(compl);
1116 percpu_ref_resurrect(ref);
1118 percpu_ref_put(ref);
1121 static bool io_match_task(struct io_kiocb *head, struct task_struct *task,
1124 struct io_kiocb *req;
1126 if (task && head->task != task)
1131 io_for_each_link(req, head) {
1132 if (req->flags & REQ_F_INFLIGHT)
1138 static inline void req_set_fail(struct io_kiocb *req)
1140 req->flags |= REQ_F_FAIL;
1143 static void io_ring_ctx_ref_free(struct percpu_ref *ref)
1145 struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs);
1147 complete(&ctx->ref_comp);
1150 static inline bool io_is_timeout_noseq(struct io_kiocb *req)
1152 return !req->timeout.off;
1155 static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
1157 struct io_ring_ctx *ctx;
1160 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
1165 * Use 5 bits less than the max cq entries, that should give us around
1166 * 32 entries per hash list if totally full and uniformly spread.
1168 hash_bits = ilog2(p->cq_entries);
1172 ctx->cancel_hash_bits = hash_bits;
1173 ctx->cancel_hash = kmalloc((1U << hash_bits) * sizeof(struct hlist_head),
1175 if (!ctx->cancel_hash)
1177 __hash_init(ctx->cancel_hash, 1U << hash_bits);
1179 ctx->dummy_ubuf = kzalloc(sizeof(*ctx->dummy_ubuf), GFP_KERNEL);
1180 if (!ctx->dummy_ubuf)
1182 /* set invalid range, so io_import_fixed() fails meeting it */
1183 ctx->dummy_ubuf->ubuf = -1UL;
1185 if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free,
1186 PERCPU_REF_ALLOW_REINIT, GFP_KERNEL))
1189 ctx->flags = p->flags;
1190 init_waitqueue_head(&ctx->sqo_sq_wait);
1191 INIT_LIST_HEAD(&ctx->sqd_list);
1192 init_waitqueue_head(&ctx->poll_wait);
1193 INIT_LIST_HEAD(&ctx->cq_overflow_list);
1194 init_completion(&ctx->ref_comp);
1195 xa_init_flags(&ctx->io_buffers, XA_FLAGS_ALLOC1);
1196 xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1);
1197 mutex_init(&ctx->uring_lock);
1198 init_waitqueue_head(&ctx->cq_wait);
1199 spin_lock_init(&ctx->completion_lock);
1200 INIT_LIST_HEAD(&ctx->iopoll_list);
1201 INIT_LIST_HEAD(&ctx->defer_list);
1202 INIT_LIST_HEAD(&ctx->timeout_list);
1203 spin_lock_init(&ctx->rsrc_ref_lock);
1204 INIT_LIST_HEAD(&ctx->rsrc_ref_list);
1205 INIT_DELAYED_WORK(&ctx->rsrc_put_work, io_rsrc_put_work);
1206 init_llist_head(&ctx->rsrc_put_llist);
1207 INIT_LIST_HEAD(&ctx->tctx_list);
1208 INIT_LIST_HEAD(&ctx->submit_state.comp.free_list);
1209 INIT_LIST_HEAD(&ctx->locked_free_list);
1210 INIT_DELAYED_WORK(&ctx->fallback_work, io_fallback_req_func);
1213 kfree(ctx->dummy_ubuf);
1214 kfree(ctx->cancel_hash);
1219 static void io_account_cq_overflow(struct io_ring_ctx *ctx)
1221 struct io_rings *r = ctx->rings;
1223 WRITE_ONCE(r->cq_overflow, READ_ONCE(r->cq_overflow) + 1);
1227 static bool req_need_defer(struct io_kiocb *req, u32 seq)
1229 if (unlikely(req->flags & REQ_F_IO_DRAIN)) {
1230 struct io_ring_ctx *ctx = req->ctx;
1232 return seq + READ_ONCE(ctx->cq_extra) != ctx->cached_cq_tail;
1238 static void io_req_track_inflight(struct io_kiocb *req)
1240 if (!(req->flags & REQ_F_INFLIGHT)) {
1241 req->flags |= REQ_F_INFLIGHT;
1242 atomic_inc(¤t->io_uring->inflight_tracked);
1246 static void io_prep_async_work(struct io_kiocb *req)
1248 const struct io_op_def *def = &io_op_defs[req->opcode];
1249 struct io_ring_ctx *ctx = req->ctx;
1251 if (!(req->flags & REQ_F_CREDS)) {
1252 req->flags |= REQ_F_CREDS;
1253 req->creds = get_current_cred();
1256 req->work.list.next = NULL;
1257 req->work.flags = 0;
1258 if (req->flags & REQ_F_FORCE_ASYNC)
1259 req->work.flags |= IO_WQ_WORK_CONCURRENT;
1261 if (req->flags & REQ_F_ISREG) {
1262 if (def->hash_reg_file || (ctx->flags & IORING_SETUP_IOPOLL))
1263 io_wq_hash_work(&req->work, file_inode(req->file));
1264 } else if (!req->file || !S_ISBLK(file_inode(req->file)->i_mode)) {
1265 if (def->unbound_nonreg_file)
1266 req->work.flags |= IO_WQ_WORK_UNBOUND;
1269 switch (req->opcode) {
1270 case IORING_OP_SPLICE:
1272 if (!S_ISREG(file_inode(req->splice.file_in)->i_mode))
1273 req->work.flags |= IO_WQ_WORK_UNBOUND;
1278 static void io_prep_async_link(struct io_kiocb *req)
1280 struct io_kiocb *cur;
1282 io_for_each_link(cur, req)
1283 io_prep_async_work(cur);
1286 static void io_queue_async_work(struct io_kiocb *req)
1288 struct io_ring_ctx *ctx = req->ctx;
1289 struct io_kiocb *link = io_prep_linked_timeout(req);
1290 struct io_uring_task *tctx = req->task->io_uring;
1293 BUG_ON(!tctx->io_wq);
1295 /* init ->work of the whole link before punting */
1296 io_prep_async_link(req);
1299 * Not expected to happen, but if we do have a bug where this _can_
1300 * happen, catch it here and ensure the request is marked as
1301 * canceled. That will make io-wq go through the usual work cancel
1302 * procedure rather than attempt to run this request (or create a new
1305 if (WARN_ON_ONCE(!same_thread_group(req->task, current)))
1306 req->work.flags |= IO_WQ_WORK_CANCEL;
1308 trace_io_uring_queue_async_work(ctx, io_wq_is_hashed(&req->work), req,
1309 &req->work, req->flags);
1310 io_wq_enqueue(tctx->io_wq, &req->work);
1312 io_queue_linked_timeout(link);
1315 static void io_kill_timeout(struct io_kiocb *req, int status)
1316 __must_hold(&req->ctx->completion_lock)
1318 struct io_timeout_data *io = req->async_data;
1320 if (hrtimer_try_to_cancel(&io->timer) != -1) {
1321 atomic_set(&req->ctx->cq_timeouts,
1322 atomic_read(&req->ctx->cq_timeouts) + 1);
1323 list_del_init(&req->timeout.list);
1324 io_cqring_fill_event(req->ctx, req->user_data, status, 0);
1325 io_put_req_deferred(req, 1);
1329 static void io_queue_deferred(struct io_ring_ctx *ctx)
1331 while (!list_empty(&ctx->defer_list)) {
1332 struct io_defer_entry *de = list_first_entry(&ctx->defer_list,
1333 struct io_defer_entry, list);
1335 if (req_need_defer(de->req, de->seq))
1337 list_del_init(&de->list);
1338 io_req_task_queue(de->req);
1343 static void io_flush_timeouts(struct io_ring_ctx *ctx)
1345 u32 seq = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts);
1347 while (!list_empty(&ctx->timeout_list)) {
1348 u32 events_needed, events_got;
1349 struct io_kiocb *req = list_first_entry(&ctx->timeout_list,
1350 struct io_kiocb, timeout.list);
1352 if (io_is_timeout_noseq(req))
1356 * Since seq can easily wrap around over time, subtract
1357 * the last seq at which timeouts were flushed before comparing.
1358 * Assuming not more than 2^31-1 events have happened since,
1359 * these subtractions won't have wrapped, so we can check if
1360 * target is in [last_seq, current_seq] by comparing the two.
1362 events_needed = req->timeout.target_seq - ctx->cq_last_tm_flush;
1363 events_got = seq - ctx->cq_last_tm_flush;
1364 if (events_got < events_needed)
1367 list_del_init(&req->timeout.list);
1368 io_kill_timeout(req, 0);
1370 ctx->cq_last_tm_flush = seq;
1373 static void __io_commit_cqring_flush(struct io_ring_ctx *ctx)
1375 if (ctx->off_timeout_used)
1376 io_flush_timeouts(ctx);
1377 if (ctx->drain_active)
1378 io_queue_deferred(ctx);
1381 static inline void io_commit_cqring(struct io_ring_ctx *ctx)
1383 if (unlikely(ctx->off_timeout_used || ctx->drain_active))
1384 __io_commit_cqring_flush(ctx);
1385 /* order cqe stores with ring update */
1386 smp_store_release(&ctx->rings->cq.tail, ctx->cached_cq_tail);
1389 static inline bool io_sqring_full(struct io_ring_ctx *ctx)
1391 struct io_rings *r = ctx->rings;
1393 return READ_ONCE(r->sq.tail) - ctx->cached_sq_head == ctx->sq_entries;
1396 static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx)
1398 return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head);
1401 static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx)
1403 struct io_rings *rings = ctx->rings;
1404 unsigned tail, mask = ctx->cq_entries - 1;
1407 * writes to the cq entry need to come after reading head; the
1408 * control dependency is enough as we're using WRITE_ONCE to
1411 if (__io_cqring_events(ctx) == ctx->cq_entries)
1414 tail = ctx->cached_cq_tail++;
1415 return &rings->cqes[tail & mask];
1418 static inline bool io_should_trigger_evfd(struct io_ring_ctx *ctx)
1420 if (likely(!ctx->cq_ev_fd))
1422 if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED)
1424 return !ctx->eventfd_async || io_wq_current_is_worker();
1427 static void io_cqring_ev_posted(struct io_ring_ctx *ctx)
1429 /* see waitqueue_active() comment */
1432 if (waitqueue_active(&ctx->cq_wait))
1433 wake_up(&ctx->cq_wait);
1434 if (ctx->sq_data && waitqueue_active(&ctx->sq_data->wait))
1435 wake_up(&ctx->sq_data->wait);
1436 if (io_should_trigger_evfd(ctx))
1437 eventfd_signal(ctx->cq_ev_fd, 1);
1438 if (waitqueue_active(&ctx->poll_wait)) {
1439 wake_up_interruptible(&ctx->poll_wait);
1440 kill_fasync(&ctx->cq_fasync, SIGIO, POLL_IN);
1444 static void io_cqring_ev_posted_iopoll(struct io_ring_ctx *ctx)
1446 /* see waitqueue_active() comment */
1449 if (ctx->flags & IORING_SETUP_SQPOLL) {
1450 if (waitqueue_active(&ctx->cq_wait))
1451 wake_up(&ctx->cq_wait);
1453 if (io_should_trigger_evfd(ctx))
1454 eventfd_signal(ctx->cq_ev_fd, 1);
1455 if (waitqueue_active(&ctx->poll_wait)) {
1456 wake_up_interruptible(&ctx->poll_wait);
1457 kill_fasync(&ctx->cq_fasync, SIGIO, POLL_IN);
1461 /* Returns true if there are no backlogged entries after the flush */
1462 static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)
1464 unsigned long flags;
1465 bool all_flushed, posted;
1467 if (!force && __io_cqring_events(ctx) == ctx->cq_entries)
1471 spin_lock_irqsave(&ctx->completion_lock, flags);
1472 while (!list_empty(&ctx->cq_overflow_list)) {
1473 struct io_uring_cqe *cqe = io_get_cqe(ctx);
1474 struct io_overflow_cqe *ocqe;
1478 ocqe = list_first_entry(&ctx->cq_overflow_list,
1479 struct io_overflow_cqe, list);
1481 memcpy(cqe, &ocqe->cqe, sizeof(*cqe));
1483 io_account_cq_overflow(ctx);
1486 list_del(&ocqe->list);
1490 all_flushed = list_empty(&ctx->cq_overflow_list);
1492 clear_bit(0, &ctx->check_cq_overflow);
1493 ctx->rings->sq_flags &= ~IORING_SQ_CQ_OVERFLOW;
1497 io_commit_cqring(ctx);
1498 spin_unlock_irqrestore(&ctx->completion_lock, flags);
1500 io_cqring_ev_posted(ctx);
1504 static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)
1508 if (test_bit(0, &ctx->check_cq_overflow)) {
1509 /* iopoll syncs against uring_lock, not completion_lock */
1510 if (ctx->flags & IORING_SETUP_IOPOLL)
1511 mutex_lock(&ctx->uring_lock);
1512 ret = __io_cqring_overflow_flush(ctx, force);
1513 if (ctx->flags & IORING_SETUP_IOPOLL)
1514 mutex_unlock(&ctx->uring_lock);
1521 * Shamelessly stolen from the mm implementation of page reference checking,
1522 * see commit f958d7b528b1 for details.
1524 #define req_ref_zero_or_close_to_overflow(req) \
1525 ((unsigned int) atomic_read(&(req->refs)) + 127u <= 127u)
1527 static inline bool req_ref_inc_not_zero(struct io_kiocb *req)
1529 return atomic_inc_not_zero(&req->refs);
1532 static inline bool req_ref_sub_and_test(struct io_kiocb *req, int refs)
1534 WARN_ON_ONCE(req_ref_zero_or_close_to_overflow(req));
1535 return atomic_sub_and_test(refs, &req->refs);
1538 static inline bool req_ref_put_and_test(struct io_kiocb *req)
1540 WARN_ON_ONCE(req_ref_zero_or_close_to_overflow(req));
1541 return atomic_dec_and_test(&req->refs);
1544 static inline void req_ref_put(struct io_kiocb *req)
1546 WARN_ON_ONCE(req_ref_put_and_test(req));
1549 static inline void req_ref_get(struct io_kiocb *req)
1551 WARN_ON_ONCE(req_ref_zero_or_close_to_overflow(req));
1552 atomic_inc(&req->refs);
1555 static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data,
1556 long res, unsigned int cflags)
1558 struct io_overflow_cqe *ocqe;
1560 ocqe = kmalloc(sizeof(*ocqe), GFP_ATOMIC | __GFP_ACCOUNT);
1563 * If we're in ring overflow flush mode, or in task cancel mode,
1564 * or cannot allocate an overflow entry, then we need to drop it
1567 io_account_cq_overflow(ctx);
1570 if (list_empty(&ctx->cq_overflow_list)) {
1571 set_bit(0, &ctx->check_cq_overflow);
1572 ctx->rings->sq_flags |= IORING_SQ_CQ_OVERFLOW;
1574 ocqe->cqe.user_data = user_data;
1575 ocqe->cqe.res = res;
1576 ocqe->cqe.flags = cflags;
1577 list_add_tail(&ocqe->list, &ctx->cq_overflow_list);
1581 static inline bool __io_cqring_fill_event(struct io_ring_ctx *ctx, u64 user_data,
1582 long res, unsigned int cflags)
1584 struct io_uring_cqe *cqe;
1586 trace_io_uring_complete(ctx, user_data, res, cflags);
1589 * If we can't get a cq entry, userspace overflowed the
1590 * submission (by quite a lot). Increment the overflow count in
1593 cqe = io_get_cqe(ctx);
1595 WRITE_ONCE(cqe->user_data, user_data);
1596 WRITE_ONCE(cqe->res, res);
1597 WRITE_ONCE(cqe->flags, cflags);
1600 return io_cqring_event_overflow(ctx, user_data, res, cflags);
1603 /* not as hot to bloat with inlining */
1604 static noinline bool io_cqring_fill_event(struct io_ring_ctx *ctx, u64 user_data,
1605 long res, unsigned int cflags)
1607 return __io_cqring_fill_event(ctx, user_data, res, cflags);
1610 static void io_req_complete_post(struct io_kiocb *req, long res,
1611 unsigned int cflags)
1613 struct io_ring_ctx *ctx = req->ctx;
1614 unsigned long flags;
1616 spin_lock_irqsave(&ctx->completion_lock, flags);
1617 __io_cqring_fill_event(ctx, req->user_data, res, cflags);
1619 * If we're the last reference to this request, add to our locked
1622 if (req_ref_put_and_test(req)) {
1623 if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) {
1624 if (req->flags & (REQ_F_LINK_TIMEOUT | REQ_F_FAIL))
1625 io_disarm_next(req);
1627 io_req_task_queue(req->link);
1631 io_dismantle_req(req);
1632 io_put_task(req->task, 1);
1633 list_add(&req->compl.list, &ctx->locked_free_list);
1634 ctx->locked_free_nr++;
1636 if (!percpu_ref_tryget(&ctx->refs))
1639 io_commit_cqring(ctx);
1640 spin_unlock_irqrestore(&ctx->completion_lock, flags);
1643 io_cqring_ev_posted(ctx);
1644 percpu_ref_put(&ctx->refs);
1648 static inline bool io_req_needs_clean(struct io_kiocb *req)
1650 return req->flags & IO_REQ_CLEAN_FLAGS;
1653 static void io_req_complete_state(struct io_kiocb *req, long res,
1654 unsigned int cflags)
1656 if (io_req_needs_clean(req))
1659 req->compl.cflags = cflags;
1660 req->flags |= REQ_F_COMPLETE_INLINE;
1663 static inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags,
1664 long res, unsigned cflags)
1666 if (issue_flags & IO_URING_F_COMPLETE_DEFER)
1667 io_req_complete_state(req, res, cflags);
1669 io_req_complete_post(req, res, cflags);
1672 static inline void io_req_complete(struct io_kiocb *req, long res)
1674 __io_req_complete(req, 0, res, 0);
1677 static void io_req_complete_failed(struct io_kiocb *req, long res)
1681 io_req_complete_post(req, res, 0);
1684 static void io_flush_cached_locked_reqs(struct io_ring_ctx *ctx,
1685 struct io_comp_state *cs)
1687 spin_lock_irq(&ctx->completion_lock);
1688 list_splice_init(&ctx->locked_free_list, &cs->free_list);
1689 ctx->locked_free_nr = 0;
1690 spin_unlock_irq(&ctx->completion_lock);
1693 /* Returns true IFF there are requests in the cache */
1694 static bool io_flush_cached_reqs(struct io_ring_ctx *ctx)
1696 struct io_submit_state *state = &ctx->submit_state;
1697 struct io_comp_state *cs = &state->comp;
1701 * If we have more than a batch's worth of requests in our IRQ side
1702 * locked cache, grab the lock and move them over to our submission
1705 if (READ_ONCE(ctx->locked_free_nr) > IO_COMPL_BATCH)
1706 io_flush_cached_locked_reqs(ctx, cs);
1708 nr = state->free_reqs;
1709 while (!list_empty(&cs->free_list)) {
1710 struct io_kiocb *req = list_first_entry(&cs->free_list,
1711 struct io_kiocb, compl.list);
1713 list_del(&req->compl.list);
1714 state->reqs[nr++] = req;
1715 if (nr == ARRAY_SIZE(state->reqs))
1719 state->free_reqs = nr;
1723 static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx)
1725 struct io_submit_state *state = &ctx->submit_state;
1727 BUILD_BUG_ON(ARRAY_SIZE(state->reqs) < IO_REQ_ALLOC_BATCH);
1729 if (!state->free_reqs) {
1730 gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
1733 if (io_flush_cached_reqs(ctx))
1736 ret = kmem_cache_alloc_bulk(req_cachep, gfp, IO_REQ_ALLOC_BATCH,
1740 * Bulk alloc is all-or-nothing. If we fail to get a batch,
1741 * retry single alloc to be on the safe side.
1743 if (unlikely(ret <= 0)) {
1744 state->reqs[0] = kmem_cache_alloc(req_cachep, gfp);
1745 if (!state->reqs[0])
1751 * Don't initialise the fields below on every allocation, but
1752 * do that in advance and keep valid on free.
1754 for (i = 0; i < ret; i++) {
1755 struct io_kiocb *req = state->reqs[i];
1759 req->async_data = NULL;
1760 /* not necessary, but safer to zero */
1763 state->free_reqs = ret;
1767 return state->reqs[state->free_reqs];
1770 static inline void io_put_file(struct file *file)
1776 static void io_dismantle_req(struct io_kiocb *req)
1778 unsigned int flags = req->flags;
1780 if (io_req_needs_clean(req))
1782 if (!(flags & REQ_F_FIXED_FILE))
1783 io_put_file(req->file);
1784 if (req->fixed_rsrc_refs)
1785 percpu_ref_put(req->fixed_rsrc_refs);
1786 if (req->async_data) {
1787 kfree(req->async_data);
1788 req->async_data = NULL;
1792 /* must to be called somewhat shortly after putting a request */
1793 static inline void io_put_task(struct task_struct *task, int nr)
1795 struct io_uring_task *tctx = task->io_uring;
1797 percpu_counter_sub(&tctx->inflight, nr);
1798 if (unlikely(atomic_read(&tctx->in_idle)))
1799 wake_up(&tctx->wait);
1800 put_task_struct_many(task, nr);
1803 static void __io_free_req(struct io_kiocb *req)
1805 struct io_ring_ctx *ctx = req->ctx;
1807 io_dismantle_req(req);
1808 io_put_task(req->task, 1);
1810 kmem_cache_free(req_cachep, req);
1811 percpu_ref_put(&ctx->refs);
1814 static inline void io_remove_next_linked(struct io_kiocb *req)
1816 struct io_kiocb *nxt = req->link;
1818 req->link = nxt->link;
1822 static bool io_kill_linked_timeout(struct io_kiocb *req)
1823 __must_hold(&req->ctx->completion_lock)
1825 struct io_kiocb *link = req->link;
1828 * Can happen if a linked timeout fired and link had been like
1829 * req -> link t-out -> link t-out [-> ...]
1831 if (link && (link->flags & REQ_F_LTIMEOUT_ACTIVE)) {
1832 struct io_timeout_data *io = link->async_data;
1834 io_remove_next_linked(req);
1835 link->timeout.head = NULL;
1836 if (hrtimer_try_to_cancel(&io->timer) != -1) {
1837 io_cqring_fill_event(link->ctx, link->user_data,
1839 io_put_req_deferred(link, 1);
1846 static void io_fail_links(struct io_kiocb *req)
1847 __must_hold(&req->ctx->completion_lock)
1849 struct io_kiocb *nxt, *link = req->link;
1856 trace_io_uring_fail_link(req, link);
1857 io_cqring_fill_event(link->ctx, link->user_data, -ECANCELED, 0);
1858 io_put_req_deferred(link, 2);
1863 static bool io_disarm_next(struct io_kiocb *req)
1864 __must_hold(&req->ctx->completion_lock)
1866 bool posted = false;
1868 if (likely(req->flags & REQ_F_LINK_TIMEOUT))
1869 posted = io_kill_linked_timeout(req);
1870 if (unlikely((req->flags & REQ_F_FAIL) &&
1871 !(req->flags & REQ_F_HARDLINK))) {
1872 posted |= (req->link != NULL);
1878 static struct io_kiocb *__io_req_find_next(struct io_kiocb *req)
1880 struct io_kiocb *nxt;
1883 * If LINK is set, we have dependent requests in this chain. If we
1884 * didn't fail this request, queue the first one up, moving any other
1885 * dependencies to the next request. In case of failure, fail the rest
1888 if (req->flags & (REQ_F_LINK_TIMEOUT | REQ_F_FAIL)) {
1889 struct io_ring_ctx *ctx = req->ctx;
1890 unsigned long flags;
1893 spin_lock_irqsave(&ctx->completion_lock, flags);
1894 posted = io_disarm_next(req);
1896 io_commit_cqring(req->ctx);
1897 spin_unlock_irqrestore(&ctx->completion_lock, flags);
1899 io_cqring_ev_posted(ctx);
1906 static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req)
1908 if (likely(!(req->flags & (REQ_F_LINK|REQ_F_HARDLINK))))
1910 return __io_req_find_next(req);
1913 static void ctx_flush_and_put(struct io_ring_ctx *ctx)
1917 if (ctx->submit_state.comp.nr) {
1918 mutex_lock(&ctx->uring_lock);
1919 io_submit_flush_completions(ctx);
1920 mutex_unlock(&ctx->uring_lock);
1922 percpu_ref_put(&ctx->refs);
1925 static void tctx_task_work(struct callback_head *cb)
1927 struct io_ring_ctx *ctx = NULL;
1928 struct io_uring_task *tctx = container_of(cb, struct io_uring_task,
1932 struct io_wq_work_node *node;
1934 spin_lock_irq(&tctx->task_lock);
1935 node = tctx->task_list.first;
1936 INIT_WQ_LIST(&tctx->task_list);
1937 spin_unlock_irq(&tctx->task_lock);
1940 struct io_wq_work_node *next = node->next;
1941 struct io_kiocb *req = container_of(node, struct io_kiocb,
1944 if (req->ctx != ctx) {
1945 ctx_flush_and_put(ctx);
1947 percpu_ref_get(&ctx->refs);
1949 req->io_task_work.func(req);
1952 if (wq_list_empty(&tctx->task_list)) {
1953 clear_bit(0, &tctx->task_state);
1954 if (wq_list_empty(&tctx->task_list))
1956 /* another tctx_task_work() is enqueued, yield */
1957 if (test_and_set_bit(0, &tctx->task_state))
1963 ctx_flush_and_put(ctx);
1966 static void io_req_task_work_add(struct io_kiocb *req)
1968 struct task_struct *tsk = req->task;
1969 struct io_uring_task *tctx = tsk->io_uring;
1970 enum task_work_notify_mode notify;
1971 struct io_wq_work_node *node;
1972 unsigned long flags;
1974 WARN_ON_ONCE(!tctx);
1976 spin_lock_irqsave(&tctx->task_lock, flags);
1977 wq_list_add_tail(&req->io_task_work.node, &tctx->task_list);
1978 spin_unlock_irqrestore(&tctx->task_lock, flags);
1980 /* task_work already pending, we're done */
1981 if (test_bit(0, &tctx->task_state) ||
1982 test_and_set_bit(0, &tctx->task_state))
1984 if (unlikely(tsk->flags & PF_EXITING))
1988 * SQPOLL kernel thread doesn't need notification, just a wakeup. For
1989 * all other cases, use TWA_SIGNAL unconditionally to ensure we're
1990 * processing task_work. There's no reliable way to tell if TWA_RESUME
1993 notify = (req->ctx->flags & IORING_SETUP_SQPOLL) ? TWA_NONE : TWA_SIGNAL;
1994 if (!task_work_add(tsk, &tctx->task_work, notify)) {
1995 wake_up_process(tsk);
1999 clear_bit(0, &tctx->task_state);
2000 spin_lock_irqsave(&tctx->task_lock, flags);
2001 node = tctx->task_list.first;
2002 INIT_WQ_LIST(&tctx->task_list);
2003 spin_unlock_irqrestore(&tctx->task_lock, flags);
2006 req = container_of(node, struct io_kiocb, io_task_work.node);
2008 if (llist_add(&req->io_task_work.fallback_node,
2009 &req->ctx->fallback_llist))
2010 schedule_delayed_work(&req->ctx->fallback_work, 1);
2014 static void io_req_task_cancel(struct io_kiocb *req)
2016 struct io_ring_ctx *ctx = req->ctx;
2018 /* ctx is guaranteed to stay alive while we hold uring_lock */
2019 mutex_lock(&ctx->uring_lock);
2020 io_req_complete_failed(req, req->result);
2021 mutex_unlock(&ctx->uring_lock);
2024 static void io_req_task_submit(struct io_kiocb *req)
2026 struct io_ring_ctx *ctx = req->ctx;
2028 /* ctx stays valid until unlock, even if we drop all ours ctx->refs */
2029 mutex_lock(&ctx->uring_lock);
2030 if (!(req->task->flags & PF_EXITING) && !req->task->in_execve)
2031 __io_queue_sqe(req);
2033 io_req_complete_failed(req, -EFAULT);
2034 mutex_unlock(&ctx->uring_lock);
2037 static void io_req_task_queue_fail(struct io_kiocb *req, int ret)
2040 req->io_task_work.func = io_req_task_cancel;
2041 io_req_task_work_add(req);
2044 static void io_req_task_queue(struct io_kiocb *req)
2046 req->io_task_work.func = io_req_task_submit;
2047 io_req_task_work_add(req);
2050 static inline void io_queue_next(struct io_kiocb *req)
2052 struct io_kiocb *nxt = io_req_find_next(req);
2055 io_req_task_queue(nxt);
2058 static void io_free_req(struct io_kiocb *req)
2065 struct task_struct *task;
2070 static inline void io_init_req_batch(struct req_batch *rb)
2077 static void io_req_free_batch_finish(struct io_ring_ctx *ctx,
2078 struct req_batch *rb)
2081 io_put_task(rb->task, rb->task_refs);
2083 percpu_ref_put_many(&ctx->refs, rb->ctx_refs);
2086 static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req,
2087 struct io_submit_state *state)
2090 io_dismantle_req(req);
2092 if (req->task != rb->task) {
2094 io_put_task(rb->task, rb->task_refs);
2095 rb->task = req->task;
2101 if (state->free_reqs != ARRAY_SIZE(state->reqs))
2102 state->reqs[state->free_reqs++] = req;
2104 list_add(&req->compl.list, &state->comp.free_list);
2107 static void io_submit_flush_completions(struct io_ring_ctx *ctx)
2109 struct io_comp_state *cs = &ctx->submit_state.comp;
2111 struct req_batch rb;
2113 spin_lock_irq(&ctx->completion_lock);
2114 for (i = 0; i < nr; i++) {
2115 struct io_kiocb *req = cs->reqs[i];
2117 __io_cqring_fill_event(ctx, req->user_data, req->result,
2120 io_commit_cqring(ctx);
2121 spin_unlock_irq(&ctx->completion_lock);
2122 io_cqring_ev_posted(ctx);
2124 io_init_req_batch(&rb);
2125 for (i = 0; i < nr; i++) {
2126 struct io_kiocb *req = cs->reqs[i];
2128 /* submission and completion refs */
2129 if (req_ref_sub_and_test(req, 2))
2130 io_req_free_batch(&rb, req, &ctx->submit_state);
2133 io_req_free_batch_finish(ctx, &rb);
2138 * Drop reference to request, return next in chain (if there is one) if this
2139 * was the last reference to this request.
2141 static inline struct io_kiocb *io_put_req_find_next(struct io_kiocb *req)
2143 struct io_kiocb *nxt = NULL;
2145 if (req_ref_put_and_test(req)) {
2146 nxt = io_req_find_next(req);
2152 static inline void io_put_req(struct io_kiocb *req)
2154 if (req_ref_put_and_test(req))
2158 static void io_free_req_deferred(struct io_kiocb *req)
2160 req->io_task_work.func = io_free_req;
2161 io_req_task_work_add(req);
2164 static inline void io_put_req_deferred(struct io_kiocb *req, int refs)
2166 if (req_ref_sub_and_test(req, refs))
2167 io_free_req_deferred(req);
2170 static unsigned io_cqring_events(struct io_ring_ctx *ctx)
2172 /* See comment at the top of this file */
2174 return __io_cqring_events(ctx);
2177 static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx)
2179 struct io_rings *rings = ctx->rings;
2181 /* make sure SQ entry isn't read before tail */
2182 return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head;
2185 static unsigned int io_put_kbuf(struct io_kiocb *req, struct io_buffer *kbuf)
2187 unsigned int cflags;
2189 cflags = kbuf->bid << IORING_CQE_BUFFER_SHIFT;
2190 cflags |= IORING_CQE_F_BUFFER;
2191 req->flags &= ~REQ_F_BUFFER_SELECTED;
2196 static inline unsigned int io_put_rw_kbuf(struct io_kiocb *req)
2198 struct io_buffer *kbuf;
2200 kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
2201 return io_put_kbuf(req, kbuf);
2204 static inline bool io_run_task_work(void)
2206 if (current->task_works) {
2207 __set_current_state(TASK_RUNNING);
2216 * Find and free completed poll iocbs
2218 static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,
2219 struct list_head *done, bool resubmit)
2221 struct req_batch rb;
2222 struct io_kiocb *req;
2224 /* order with ->result store in io_complete_rw_iopoll() */
2227 io_init_req_batch(&rb);
2228 while (!list_empty(done)) {
2231 req = list_first_entry(done, struct io_kiocb, inflight_entry);
2232 list_del(&req->inflight_entry);
2234 if (READ_ONCE(req->result) == -EAGAIN && resubmit &&
2235 !(req->flags & REQ_F_DONT_REISSUE)) {
2236 req->iopoll_completed = 0;
2238 io_queue_async_work(req);
2242 if (req->flags & REQ_F_BUFFER_SELECTED)
2243 cflags = io_put_rw_kbuf(req);
2245 __io_cqring_fill_event(ctx, req->user_data, req->result, cflags);
2248 if (req_ref_put_and_test(req))
2249 io_req_free_batch(&rb, req, &ctx->submit_state);
2252 io_commit_cqring(ctx);
2253 io_cqring_ev_posted_iopoll(ctx);
2254 io_req_free_batch_finish(ctx, &rb);
2257 static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
2258 long min, bool resubmit)
2260 struct io_kiocb *req, *tmp;
2266 * Only spin for completions if we don't have multiple devices hanging
2267 * off our complete list, and we're under the requested amount.
2269 spin = !ctx->poll_multi_queue && *nr_events < min;
2272 list_for_each_entry_safe(req, tmp, &ctx->iopoll_list, inflight_entry) {
2273 struct kiocb *kiocb = &req->rw.kiocb;
2276 * Move completed and retryable entries to our local lists.
2277 * If we find a request that requires polling, break out
2278 * and complete those lists first, if we have entries there.
2280 if (READ_ONCE(req->iopoll_completed)) {
2281 list_move_tail(&req->inflight_entry, &done);
2284 if (!list_empty(&done))
2287 ret = kiocb->ki_filp->f_op->iopoll(kiocb, spin);
2291 /* iopoll may have completed current req */
2292 if (READ_ONCE(req->iopoll_completed))
2293 list_move_tail(&req->inflight_entry, &done);
2300 if (!list_empty(&done))
2301 io_iopoll_complete(ctx, nr_events, &done, resubmit);
2307 * We can't just wait for polled events to come to us, we have to actively
2308 * find and complete them.
2310 static void io_iopoll_try_reap_events(struct io_ring_ctx *ctx)
2312 if (!(ctx->flags & IORING_SETUP_IOPOLL))
2315 mutex_lock(&ctx->uring_lock);
2316 while (!list_empty(&ctx->iopoll_list)) {
2317 unsigned int nr_events = 0;
2319 io_do_iopoll(ctx, &nr_events, 0, false);
2321 /* let it sleep and repeat later if can't complete a request */
2325 * Ensure we allow local-to-the-cpu processing to take place,
2326 * in this case we need to ensure that we reap all events.
2327 * Also let task_work, etc. to progress by releasing the mutex
2329 if (need_resched()) {
2330 mutex_unlock(&ctx->uring_lock);
2332 mutex_lock(&ctx->uring_lock);
2335 mutex_unlock(&ctx->uring_lock);
2338 static int io_iopoll_check(struct io_ring_ctx *ctx, long min)
2340 unsigned int nr_events = 0;
2344 * We disallow the app entering submit/complete with polling, but we
2345 * still need to lock the ring to prevent racing with polled issue
2346 * that got punted to a workqueue.
2348 mutex_lock(&ctx->uring_lock);
2350 * Don't enter poll loop if we already have events pending.
2351 * If we do, we can potentially be spinning for commands that
2352 * already triggered a CQE (eg in error).
2354 if (test_bit(0, &ctx->check_cq_overflow))
2355 __io_cqring_overflow_flush(ctx, false);
2356 if (io_cqring_events(ctx))
2360 * If a submit got punted to a workqueue, we can have the
2361 * application entering polling for a command before it gets
2362 * issued. That app will hold the uring_lock for the duration
2363 * of the poll right here, so we need to take a breather every
2364 * now and then to ensure that the issue has a chance to add
2365 * the poll to the issued list. Otherwise we can spin here
2366 * forever, while the workqueue is stuck trying to acquire the
2369 if (list_empty(&ctx->iopoll_list)) {
2370 u32 tail = ctx->cached_cq_tail;
2372 mutex_unlock(&ctx->uring_lock);
2374 mutex_lock(&ctx->uring_lock);
2376 /* some requests don't go through iopoll_list */
2377 if (tail != ctx->cached_cq_tail ||
2378 list_empty(&ctx->iopoll_list))
2381 ret = io_do_iopoll(ctx, &nr_events, min, true);
2382 } while (!ret && nr_events < min && !need_resched());
2384 mutex_unlock(&ctx->uring_lock);
2388 static void kiocb_end_write(struct io_kiocb *req)
2391 * Tell lockdep we inherited freeze protection from submission
2394 if (req->flags & REQ_F_ISREG) {
2395 struct super_block *sb = file_inode(req->file)->i_sb;
2397 __sb_writers_acquired(sb, SB_FREEZE_WRITE);
2403 static bool io_resubmit_prep(struct io_kiocb *req)
2405 struct io_async_rw *rw = req->async_data;
2408 return !io_req_prep_async(req);
2409 /* may have left rw->iter inconsistent on -EIOCBQUEUED */
2410 iov_iter_revert(&rw->iter, req->result - iov_iter_count(&rw->iter));
2414 static bool io_rw_should_reissue(struct io_kiocb *req)
2416 umode_t mode = file_inode(req->file)->i_mode;
2417 struct io_ring_ctx *ctx = req->ctx;
2419 if (!S_ISBLK(mode) && !S_ISREG(mode))
2421 if ((req->flags & REQ_F_NOWAIT) || (io_wq_current_is_worker() &&
2422 !(ctx->flags & IORING_SETUP_IOPOLL)))
2425 * If ref is dying, we might be running poll reap from the exit work.
2426 * Don't attempt to reissue from that path, just let it fail with
2429 if (percpu_ref_is_dying(&ctx->refs))
2434 static bool io_resubmit_prep(struct io_kiocb *req)
2438 static bool io_rw_should_reissue(struct io_kiocb *req)
2444 static void io_fallback_req_func(struct work_struct *work)
2446 struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx,
2447 fallback_work.work);
2448 struct llist_node *node = llist_del_all(&ctx->fallback_llist);
2449 struct io_kiocb *req, *tmp;
2451 llist_for_each_entry_safe(req, tmp, node, io_task_work.fallback_node)
2452 req->io_task_work.func(req);
2455 static void __io_complete_rw(struct io_kiocb *req, long res, long res2,
2456 unsigned int issue_flags)
2460 if (req->rw.kiocb.ki_flags & IOCB_WRITE)
2461 kiocb_end_write(req);
2462 if (res != req->result) {
2463 if ((res == -EAGAIN || res == -EOPNOTSUPP) &&
2464 io_rw_should_reissue(req)) {
2465 req->flags |= REQ_F_REISSUE;
2470 if (req->flags & REQ_F_BUFFER_SELECTED)
2471 cflags = io_put_rw_kbuf(req);
2472 __io_req_complete(req, issue_flags, res, cflags);
2475 static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
2477 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
2479 __io_complete_rw(req, res, res2, 0);
2482 static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)
2484 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
2486 if (kiocb->ki_flags & IOCB_WRITE)
2487 kiocb_end_write(req);
2488 if (unlikely(res != req->result)) {
2489 if (!(res == -EAGAIN && io_rw_should_reissue(req) &&
2490 io_resubmit_prep(req))) {
2492 req->flags |= REQ_F_DONT_REISSUE;
2496 WRITE_ONCE(req->result, res);
2497 /* order with io_iopoll_complete() checking ->result */
2499 WRITE_ONCE(req->iopoll_completed, 1);
2503 * After the iocb has been issued, it's safe to be found on the poll list.
2504 * Adding the kiocb to the list AFTER submission ensures that we don't
2505 * find it from a io_do_iopoll() thread before the issuer is done
2506 * accessing the kiocb cookie.
2508 static void io_iopoll_req_issued(struct io_kiocb *req)
2510 struct io_ring_ctx *ctx = req->ctx;
2511 const bool in_async = io_wq_current_is_worker();
2513 /* workqueue context doesn't hold uring_lock, grab it now */
2514 if (unlikely(in_async))
2515 mutex_lock(&ctx->uring_lock);
2518 * Track whether we have multiple files in our lists. This will impact
2519 * how we do polling eventually, not spinning if we're on potentially
2520 * different devices.
2522 if (list_empty(&ctx->iopoll_list)) {
2523 ctx->poll_multi_queue = false;
2524 } else if (!ctx->poll_multi_queue) {
2525 struct io_kiocb *list_req;
2526 unsigned int queue_num0, queue_num1;
2528 list_req = list_first_entry(&ctx->iopoll_list, struct io_kiocb,
2531 if (list_req->file != req->file) {
2532 ctx->poll_multi_queue = true;
2534 queue_num0 = blk_qc_t_to_queue_num(list_req->rw.kiocb.ki_cookie);
2535 queue_num1 = blk_qc_t_to_queue_num(req->rw.kiocb.ki_cookie);
2536 if (queue_num0 != queue_num1)
2537 ctx->poll_multi_queue = true;
2542 * For fast devices, IO may have already completed. If it has, add
2543 * it to the front so we find it first.
2545 if (READ_ONCE(req->iopoll_completed))
2546 list_add(&req->inflight_entry, &ctx->iopoll_list);
2548 list_add_tail(&req->inflight_entry, &ctx->iopoll_list);
2550 if (unlikely(in_async)) {
2552 * If IORING_SETUP_SQPOLL is enabled, sqes are either handle
2553 * in sq thread task context or in io worker task context. If
2554 * current task context is sq thread, we don't need to check
2555 * whether should wake up sq thread.
2557 if ((ctx->flags & IORING_SETUP_SQPOLL) &&
2558 wq_has_sleeper(&ctx->sq_data->wait))
2559 wake_up(&ctx->sq_data->wait);
2561 mutex_unlock(&ctx->uring_lock);
2565 static inline void io_state_file_put(struct io_submit_state *state)
2567 if (state->file_refs) {
2568 fput_many(state->file, state->file_refs);
2569 state->file_refs = 0;
2574 * Get as many references to a file as we have IOs left in this submission,
2575 * assuming most submissions are for one file, or at least that each file
2576 * has more than one submission.
2578 static struct file *__io_file_get(struct io_submit_state *state, int fd)
2583 if (state->file_refs) {
2584 if (state->fd == fd) {
2588 io_state_file_put(state);
2590 state->file = fget_many(fd, state->ios_left);
2591 if (unlikely(!state->file))
2595 state->file_refs = state->ios_left - 1;
2599 static bool io_bdev_nowait(struct block_device *bdev)
2601 return !bdev || blk_queue_nowait(bdev_get_queue(bdev));
2605 * If we tracked the file through the SCM inflight mechanism, we could support
2606 * any file. For now, just ensure that anything potentially problematic is done
2609 static bool __io_file_supports_async(struct file *file, int rw)
2611 umode_t mode = file_inode(file)->i_mode;
2613 if (S_ISBLK(mode)) {
2614 if (IS_ENABLED(CONFIG_BLOCK) &&
2615 io_bdev_nowait(I_BDEV(file->f_mapping->host)))
2621 if (S_ISREG(mode)) {
2622 if (IS_ENABLED(CONFIG_BLOCK) &&
2623 io_bdev_nowait(file->f_inode->i_sb->s_bdev) &&
2624 file->f_op != &io_uring_fops)
2629 /* any ->read/write should understand O_NONBLOCK */
2630 if (file->f_flags & O_NONBLOCK)
2633 if (!(file->f_mode & FMODE_NOWAIT))
2637 return file->f_op->read_iter != NULL;
2639 return file->f_op->write_iter != NULL;
2642 static bool io_file_supports_async(struct io_kiocb *req, int rw)
2644 if (rw == READ && (req->flags & REQ_F_ASYNC_READ))
2646 else if (rw == WRITE && (req->flags & REQ_F_ASYNC_WRITE))
2649 return __io_file_supports_async(req->file, rw);
2652 static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe)
2654 struct io_ring_ctx *ctx = req->ctx;
2655 struct kiocb *kiocb = &req->rw.kiocb;
2656 struct file *file = req->file;
2660 if (!(req->flags & REQ_F_ISREG) && S_ISREG(file_inode(file)->i_mode))
2661 req->flags |= REQ_F_ISREG;
2663 kiocb->ki_pos = READ_ONCE(sqe->off);
2664 if (kiocb->ki_pos == -1 && !(file->f_mode & FMODE_STREAM)) {
2665 req->flags |= REQ_F_CUR_POS;
2666 kiocb->ki_pos = file->f_pos;
2668 kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));
2669 kiocb->ki_flags = iocb_flags(kiocb->ki_filp);
2670 ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags));
2674 /* don't allow async punt for O_NONBLOCK or RWF_NOWAIT */
2675 if ((kiocb->ki_flags & IOCB_NOWAIT) || (file->f_flags & O_NONBLOCK))
2676 req->flags |= REQ_F_NOWAIT;
2678 ioprio = READ_ONCE(sqe->ioprio);
2680 ret = ioprio_check_cap(ioprio);
2684 kiocb->ki_ioprio = ioprio;
2686 kiocb->ki_ioprio = get_current_ioprio();
2688 if (ctx->flags & IORING_SETUP_IOPOLL) {
2689 if (!(kiocb->ki_flags & IOCB_DIRECT) ||
2690 !kiocb->ki_filp->f_op->iopoll)
2693 kiocb->ki_flags |= IOCB_HIPRI;
2694 kiocb->ki_complete = io_complete_rw_iopoll;
2695 req->iopoll_completed = 0;
2697 if (kiocb->ki_flags & IOCB_HIPRI)
2699 kiocb->ki_complete = io_complete_rw;
2702 if (req->opcode == IORING_OP_READ_FIXED ||
2703 req->opcode == IORING_OP_WRITE_FIXED) {
2705 io_req_set_rsrc_node(req);
2708 req->rw.addr = READ_ONCE(sqe->addr);
2709 req->rw.len = READ_ONCE(sqe->len);
2710 req->buf_index = READ_ONCE(sqe->buf_index);
2714 static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret)
2720 case -ERESTARTNOINTR:
2721 case -ERESTARTNOHAND:
2722 case -ERESTART_RESTARTBLOCK:
2724 * We can't just restart the syscall, since previously
2725 * submitted sqes may already be in progress. Just fail this
2731 kiocb->ki_complete(kiocb, ret, 0);
2735 static void kiocb_done(struct kiocb *kiocb, ssize_t ret,
2736 unsigned int issue_flags)
2738 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
2739 struct io_async_rw *io = req->async_data;
2740 bool check_reissue = kiocb->ki_complete == io_complete_rw;
2742 /* add previously done IO, if any */
2743 if (io && io->bytes_done > 0) {
2745 ret = io->bytes_done;
2747 ret += io->bytes_done;
2750 if (req->flags & REQ_F_CUR_POS)
2751 req->file->f_pos = kiocb->ki_pos;
2752 if (ret >= 0 && check_reissue)
2753 __io_complete_rw(req, ret, 0, issue_flags);
2755 io_rw_done(kiocb, ret);
2757 if (check_reissue && (req->flags & REQ_F_REISSUE)) {
2758 req->flags &= ~REQ_F_REISSUE;
2759 if (io_resubmit_prep(req)) {
2761 io_queue_async_work(req);
2766 if (req->flags & REQ_F_BUFFER_SELECTED)
2767 cflags = io_put_rw_kbuf(req);
2768 __io_req_complete(req, issue_flags, ret, cflags);
2773 static int __io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter,
2774 struct io_mapped_ubuf *imu)
2776 size_t len = req->rw.len;
2777 u64 buf_end, buf_addr = req->rw.addr;
2780 if (unlikely(check_add_overflow(buf_addr, (u64)len, &buf_end)))
2782 /* not inside the mapped region */
2783 if (unlikely(buf_addr < imu->ubuf || buf_end > imu->ubuf_end))
2787 * May not be a start of buffer, set size appropriately
2788 * and advance us to the beginning.
2790 offset = buf_addr - imu->ubuf;
2791 iov_iter_bvec(iter, rw, imu->bvec, imu->nr_bvecs, offset + len);
2795 * Don't use iov_iter_advance() here, as it's really slow for
2796 * using the latter parts of a big fixed buffer - it iterates
2797 * over each segment manually. We can cheat a bit here, because
2800 * 1) it's a BVEC iter, we set it up
2801 * 2) all bvecs are PAGE_SIZE in size, except potentially the
2802 * first and last bvec
2804 * So just find our index, and adjust the iterator afterwards.
2805 * If the offset is within the first bvec (or the whole first
2806 * bvec, just use iov_iter_advance(). This makes it easier
2807 * since we can just skip the first segment, which may not
2808 * be PAGE_SIZE aligned.
2810 const struct bio_vec *bvec = imu->bvec;
2812 if (offset <= bvec->bv_len) {
2813 iov_iter_advance(iter, offset);
2815 unsigned long seg_skip;
2817 /* skip first vec */
2818 offset -= bvec->bv_len;
2819 seg_skip = 1 + (offset >> PAGE_SHIFT);
2821 iter->bvec = bvec + seg_skip;
2822 iter->nr_segs -= seg_skip;
2823 iter->count -= bvec->bv_len + offset;
2824 iter->iov_offset = offset & ~PAGE_MASK;
2831 static int io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter)
2833 struct io_ring_ctx *ctx = req->ctx;
2834 struct io_mapped_ubuf *imu = req->imu;
2835 u16 index, buf_index = req->buf_index;
2838 if (unlikely(buf_index >= ctx->nr_user_bufs))
2840 index = array_index_nospec(buf_index, ctx->nr_user_bufs);
2841 imu = READ_ONCE(ctx->user_bufs[index]);
2844 return __io_import_fixed(req, rw, iter, imu);
2847 static void io_ring_submit_unlock(struct io_ring_ctx *ctx, bool needs_lock)
2850 mutex_unlock(&ctx->uring_lock);
2853 static void io_ring_submit_lock(struct io_ring_ctx *ctx, bool needs_lock)
2856 * "Normal" inline submissions always hold the uring_lock, since we
2857 * grab it from the system call. Same is true for the SQPOLL offload.
2858 * The only exception is when we've detached the request and issue it
2859 * from an async worker thread, grab the lock for that case.
2862 mutex_lock(&ctx->uring_lock);
2865 static struct io_buffer *io_buffer_select(struct io_kiocb *req, size_t *len,
2866 int bgid, struct io_buffer *kbuf,
2869 struct io_buffer *head;
2871 if (req->flags & REQ_F_BUFFER_SELECTED)
2874 io_ring_submit_lock(req->ctx, needs_lock);
2876 lockdep_assert_held(&req->ctx->uring_lock);
2878 head = xa_load(&req->ctx->io_buffers, bgid);
2880 if (!list_empty(&head->list)) {
2881 kbuf = list_last_entry(&head->list, struct io_buffer,
2883 list_del(&kbuf->list);
2886 xa_erase(&req->ctx->io_buffers, bgid);
2888 if (*len > kbuf->len)
2891 kbuf = ERR_PTR(-ENOBUFS);
2894 io_ring_submit_unlock(req->ctx, needs_lock);
2899 static void __user *io_rw_buffer_select(struct io_kiocb *req, size_t *len,
2902 struct io_buffer *kbuf;
2905 kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
2906 bgid = req->buf_index;
2907 kbuf = io_buffer_select(req, len, bgid, kbuf, needs_lock);
2910 req->rw.addr = (u64) (unsigned long) kbuf;
2911 req->flags |= REQ_F_BUFFER_SELECTED;
2912 return u64_to_user_ptr(kbuf->addr);
2915 #ifdef CONFIG_COMPAT
2916 static ssize_t io_compat_import(struct io_kiocb *req, struct iovec *iov,
2919 struct compat_iovec __user *uiov;
2920 compat_ssize_t clen;
2924 uiov = u64_to_user_ptr(req->rw.addr);
2925 if (!access_ok(uiov, sizeof(*uiov)))
2927 if (__get_user(clen, &uiov->iov_len))
2933 buf = io_rw_buffer_select(req, &len, needs_lock);
2935 return PTR_ERR(buf);
2936 iov[0].iov_base = buf;
2937 iov[0].iov_len = (compat_size_t) len;
2942 static ssize_t __io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,
2945 struct iovec __user *uiov = u64_to_user_ptr(req->rw.addr);
2949 if (copy_from_user(iov, uiov, sizeof(*uiov)))
2952 len = iov[0].iov_len;
2955 buf = io_rw_buffer_select(req, &len, needs_lock);
2957 return PTR_ERR(buf);
2958 iov[0].iov_base = buf;
2959 iov[0].iov_len = len;
2963 static ssize_t io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,
2966 if (req->flags & REQ_F_BUFFER_SELECTED) {
2967 struct io_buffer *kbuf;
2969 kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
2970 iov[0].iov_base = u64_to_user_ptr(kbuf->addr);
2971 iov[0].iov_len = kbuf->len;
2974 if (req->rw.len != 1)
2977 #ifdef CONFIG_COMPAT
2978 if (req->ctx->compat)
2979 return io_compat_import(req, iov, needs_lock);
2982 return __io_iov_buffer_select(req, iov, needs_lock);
2985 static int io_import_iovec(int rw, struct io_kiocb *req, struct iovec **iovec,
2986 struct iov_iter *iter, bool needs_lock)
2988 void __user *buf = u64_to_user_ptr(req->rw.addr);
2989 size_t sqe_len = req->rw.len;
2990 u8 opcode = req->opcode;
2993 if (opcode == IORING_OP_READ_FIXED || opcode == IORING_OP_WRITE_FIXED) {
2995 return io_import_fixed(req, rw, iter);
2998 /* buffer index only valid with fixed read/write, or buffer select */
2999 if (req->buf_index && !(req->flags & REQ_F_BUFFER_SELECT))
3002 if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE) {
3003 if (req->flags & REQ_F_BUFFER_SELECT) {
3004 buf = io_rw_buffer_select(req, &sqe_len, needs_lock);
3006 return PTR_ERR(buf);
3007 req->rw.len = sqe_len;
3010 ret = import_single_range(rw, buf, sqe_len, *iovec, iter);
3015 if (req->flags & REQ_F_BUFFER_SELECT) {
3016 ret = io_iov_buffer_select(req, *iovec, needs_lock);
3018 iov_iter_init(iter, rw, *iovec, 1, (*iovec)->iov_len);
3023 return __import_iovec(rw, buf, sqe_len, UIO_FASTIOV, iovec, iter,
3027 static inline loff_t *io_kiocb_ppos(struct kiocb *kiocb)
3029 return (kiocb->ki_filp->f_mode & FMODE_STREAM) ? NULL : &kiocb->ki_pos;
3033 * For files that don't have ->read_iter() and ->write_iter(), handle them
3034 * by looping over ->read() or ->write() manually.
3036 static ssize_t loop_rw_iter(int rw, struct io_kiocb *req, struct iov_iter *iter)
3038 struct kiocb *kiocb = &req->rw.kiocb;
3039 struct file *file = req->file;
3043 * Don't support polled IO through this interface, and we can't
3044 * support non-blocking either. For the latter, this just causes
3045 * the kiocb to be handled from an async context.
3047 if (kiocb->ki_flags & IOCB_HIPRI)
3049 if (kiocb->ki_flags & IOCB_NOWAIT)
3052 while (iov_iter_count(iter)) {
3056 if (!iov_iter_is_bvec(iter)) {
3057 iovec = iov_iter_iovec(iter);
3059 iovec.iov_base = u64_to_user_ptr(req->rw.addr);
3060 iovec.iov_len = req->rw.len;
3064 nr = file->f_op->read(file, iovec.iov_base,
3065 iovec.iov_len, io_kiocb_ppos(kiocb));
3067 nr = file->f_op->write(file, iovec.iov_base,
3068 iovec.iov_len, io_kiocb_ppos(kiocb));
3077 if (nr != iovec.iov_len)
3081 iov_iter_advance(iter, nr);
3087 static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec,
3088 const struct iovec *fast_iov, struct iov_iter *iter)
3090 struct io_async_rw *rw = req->async_data;
3092 memcpy(&rw->iter, iter, sizeof(*iter));
3093 rw->free_iovec = iovec;
3095 /* can only be fixed buffers, no need to do anything */
3096 if (iov_iter_is_bvec(iter))
3099 unsigned iov_off = 0;
3101 rw->iter.iov = rw->fast_iov;
3102 if (iter->iov != fast_iov) {
3103 iov_off = iter->iov - fast_iov;
3104 rw->iter.iov += iov_off;
3106 if (rw->fast_iov != fast_iov)
3107 memcpy(rw->fast_iov + iov_off, fast_iov + iov_off,
3108 sizeof(struct iovec) * iter->nr_segs);
3110 req->flags |= REQ_F_NEED_CLEANUP;
3114 static inline int io_alloc_async_data(struct io_kiocb *req)
3116 WARN_ON_ONCE(!io_op_defs[req->opcode].async_size);
3117 req->async_data = kmalloc(io_op_defs[req->opcode].async_size, GFP_KERNEL);
3118 return req->async_data == NULL;
3121 static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec,
3122 const struct iovec *fast_iov,
3123 struct iov_iter *iter, bool force)
3125 if (!force && !io_op_defs[req->opcode].needs_async_setup)
3127 if (!req->async_data) {
3128 if (io_alloc_async_data(req)) {
3133 io_req_map_rw(req, iovec, fast_iov, iter);
3138 static inline int io_rw_prep_async(struct io_kiocb *req, int rw)
3140 struct io_async_rw *iorw = req->async_data;
3141 struct iovec *iov = iorw->fast_iov;
3144 ret = io_import_iovec(rw, req, &iov, &iorw->iter, false);
3145 if (unlikely(ret < 0))
3148 iorw->bytes_done = 0;
3149 iorw->free_iovec = iov;
3151 req->flags |= REQ_F_NEED_CLEANUP;
3155 static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3157 if (unlikely(!(req->file->f_mode & FMODE_READ)))
3159 return io_prep_rw(req, sqe);
3163 * This is our waitqueue callback handler, registered through lock_page_async()
3164 * when we initially tried to do the IO with the iocb armed our waitqueue.
3165 * This gets called when the page is unlocked, and we generally expect that to
3166 * happen when the page IO is completed and the page is now uptodate. This will
3167 * queue a task_work based retry of the operation, attempting to copy the data
3168 * again. If the latter fails because the page was NOT uptodate, then we will
3169 * do a thread based blocking retry of the operation. That's the unexpected
3172 static int io_async_buf_func(struct wait_queue_entry *wait, unsigned mode,
3173 int sync, void *arg)
3175 struct wait_page_queue *wpq;
3176 struct io_kiocb *req = wait->private;
3177 struct wait_page_key *key = arg;
3179 wpq = container_of(wait, struct wait_page_queue, wait);
3181 if (!wake_page_match(wpq, key))
3184 req->rw.kiocb.ki_flags &= ~IOCB_WAITQ;
3185 list_del_init(&wait->entry);
3187 /* submit ref gets dropped, acquire a new one */
3189 io_req_task_queue(req);
3194 * This controls whether a given IO request should be armed for async page
3195 * based retry. If we return false here, the request is handed to the async
3196 * worker threads for retry. If we're doing buffered reads on a regular file,
3197 * we prepare a private wait_page_queue entry and retry the operation. This
3198 * will either succeed because the page is now uptodate and unlocked, or it
3199 * will register a callback when the page is unlocked at IO completion. Through
3200 * that callback, io_uring uses task_work to setup a retry of the operation.
3201 * That retry will attempt the buffered read again. The retry will generally
3202 * succeed, or in rare cases where it fails, we then fall back to using the
3203 * async worker threads for a blocking retry.
3205 static bool io_rw_should_retry(struct io_kiocb *req)
3207 struct io_async_rw *rw = req->async_data;
3208 struct wait_page_queue *wait = &rw->wpq;
3209 struct kiocb *kiocb = &req->rw.kiocb;
3211 /* never retry for NOWAIT, we just complete with -EAGAIN */
3212 if (req->flags & REQ_F_NOWAIT)
3215 /* Only for buffered IO */
3216 if (kiocb->ki_flags & (IOCB_DIRECT | IOCB_HIPRI))
3220 * just use poll if we can, and don't attempt if the fs doesn't
3221 * support callback based unlocks
3223 if (file_can_poll(req->file) || !(req->file->f_mode & FMODE_BUF_RASYNC))
3226 wait->wait.func = io_async_buf_func;
3227 wait->wait.private = req;
3228 wait->wait.flags = 0;
3229 INIT_LIST_HEAD(&wait->wait.entry);
3230 kiocb->ki_flags |= IOCB_WAITQ;
3231 kiocb->ki_flags &= ~IOCB_NOWAIT;
3232 kiocb->ki_waitq = wait;
3236 static inline int io_iter_do_read(struct io_kiocb *req, struct iov_iter *iter)
3238 if (req->file->f_op->read_iter)
3239 return call_read_iter(req->file, &req->rw.kiocb, iter);
3240 else if (req->file->f_op->read)
3241 return loop_rw_iter(READ, req, iter);
3246 static int io_read(struct io_kiocb *req, unsigned int issue_flags)
3248 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
3249 struct kiocb *kiocb = &req->rw.kiocb;
3250 struct iov_iter __iter, *iter = &__iter;
3251 struct io_async_rw *rw = req->async_data;
3252 ssize_t io_size, ret, ret2;
3253 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
3259 ret = io_import_iovec(READ, req, &iovec, iter, !force_nonblock);
3263 io_size = iov_iter_count(iter);
3264 req->result = io_size;
3266 /* Ensure we clear previously set non-block flag */
3267 if (!force_nonblock)
3268 kiocb->ki_flags &= ~IOCB_NOWAIT;
3270 kiocb->ki_flags |= IOCB_NOWAIT;
3272 /* If the file doesn't support async, just async punt */
3273 if (force_nonblock && !io_file_supports_async(req, READ)) {
3274 ret = io_setup_async_rw(req, iovec, inline_vecs, iter, true);
3275 return ret ?: -EAGAIN;
3278 ret = rw_verify_area(READ, req->file, io_kiocb_ppos(kiocb), io_size);
3279 if (unlikely(ret)) {
3284 ret = io_iter_do_read(req, iter);
3286 if (ret == -EAGAIN || (req->flags & REQ_F_REISSUE)) {
3287 req->flags &= ~REQ_F_REISSUE;
3288 /* IOPOLL retry should happen for io-wq threads */
3289 if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL))
3291 /* no retry on NONBLOCK nor RWF_NOWAIT */
3292 if (req->flags & REQ_F_NOWAIT)
3294 /* some cases will consume bytes even on error returns */
3295 iov_iter_revert(iter, io_size - iov_iter_count(iter));
3297 } else if (ret == -EIOCBQUEUED) {
3299 } else if (ret <= 0 || ret == io_size || !force_nonblock ||
3300 (req->flags & REQ_F_NOWAIT) || !(req->flags & REQ_F_ISREG)) {
3301 /* read all, failed, already did sync or don't want to retry */
3305 ret2 = io_setup_async_rw(req, iovec, inline_vecs, iter, true);
3310 rw = req->async_data;
3311 /* now use our persistent iterator, if we aren't already */
3316 rw->bytes_done += ret;
3317 /* if we can retry, do so with the callbacks armed */
3318 if (!io_rw_should_retry(req)) {
3319 kiocb->ki_flags &= ~IOCB_WAITQ;
3324 * Now retry read with the IOCB_WAITQ parts set in the iocb. If
3325 * we get -EIOCBQUEUED, then we'll get a notification when the
3326 * desired page gets unlocked. We can also get a partial read
3327 * here, and if we do, then just retry at the new offset.
3329 ret = io_iter_do_read(req, iter);
3330 if (ret == -EIOCBQUEUED)
3332 /* we got some bytes, but not all. retry. */
3333 kiocb->ki_flags &= ~IOCB_WAITQ;
3334 } while (ret > 0 && ret < io_size);
3336 kiocb_done(kiocb, ret, issue_flags);
3338 /* it's faster to check here then delegate to kfree */
3344 static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3346 if (unlikely(!(req->file->f_mode & FMODE_WRITE)))
3348 return io_prep_rw(req, sqe);
3351 static int io_write(struct io_kiocb *req, unsigned int issue_flags)
3353 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
3354 struct kiocb *kiocb = &req->rw.kiocb;
3355 struct iov_iter __iter, *iter = &__iter;
3356 struct io_async_rw *rw = req->async_data;
3357 ssize_t ret, ret2, io_size;
3358 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
3364 ret = io_import_iovec(WRITE, req, &iovec, iter, !force_nonblock);
3368 io_size = iov_iter_count(iter);
3369 req->result = io_size;
3371 /* Ensure we clear previously set non-block flag */
3372 if (!force_nonblock)
3373 kiocb->ki_flags &= ~IOCB_NOWAIT;
3375 kiocb->ki_flags |= IOCB_NOWAIT;
3377 /* If the file doesn't support async, just async punt */
3378 if (force_nonblock && !io_file_supports_async(req, WRITE))
3381 /* file path doesn't support NOWAIT for non-direct_IO */
3382 if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT) &&
3383 (req->flags & REQ_F_ISREG))
3386 ret = rw_verify_area(WRITE, req->file, io_kiocb_ppos(kiocb), io_size);
3391 * Open-code file_start_write here to grab freeze protection,
3392 * which will be released by another thread in
3393 * io_complete_rw(). Fool lockdep by telling it the lock got
3394 * released so that it doesn't complain about the held lock when
3395 * we return to userspace.
3397 if (req->flags & REQ_F_ISREG) {
3398 sb_start_write(file_inode(req->file)->i_sb);
3399 __sb_writers_release(file_inode(req->file)->i_sb,
3402 kiocb->ki_flags |= IOCB_WRITE;
3404 if (req->file->f_op->write_iter)
3405 ret2 = call_write_iter(req->file, kiocb, iter);
3406 else if (req->file->f_op->write)
3407 ret2 = loop_rw_iter(WRITE, req, iter);
3411 if (req->flags & REQ_F_REISSUE) {
3412 req->flags &= ~REQ_F_REISSUE;
3417 * Raw bdev writes will return -EOPNOTSUPP for IOCB_NOWAIT. Just
3418 * retry them without IOCB_NOWAIT.
3420 if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT))
3422 /* no retry on NONBLOCK nor RWF_NOWAIT */
3423 if (ret2 == -EAGAIN && (req->flags & REQ_F_NOWAIT))
3425 if (!force_nonblock || ret2 != -EAGAIN) {
3426 /* IOPOLL retry should happen for io-wq threads */
3427 if ((req->ctx->flags & IORING_SETUP_IOPOLL) && ret2 == -EAGAIN)
3430 kiocb_done(kiocb, ret2, issue_flags);
3433 /* some cases will consume bytes even on error returns */
3434 iov_iter_revert(iter, io_size - iov_iter_count(iter));
3435 ret = io_setup_async_rw(req, iovec, inline_vecs, iter, false);
3436 return ret ?: -EAGAIN;
3439 /* it's reportedly faster than delegating the null check to kfree() */
3445 static int io_renameat_prep(struct io_kiocb *req,
3446 const struct io_uring_sqe *sqe)
3448 struct io_rename *ren = &req->rename;
3449 const char __user *oldf, *newf;
3451 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3453 if (sqe->ioprio || sqe->buf_index)
3455 if (unlikely(req->flags & REQ_F_FIXED_FILE))
3458 ren->old_dfd = READ_ONCE(sqe->fd);
3459 oldf = u64_to_user_ptr(READ_ONCE(sqe->addr));
3460 newf = u64_to_user_ptr(READ_ONCE(sqe->addr2));
3461 ren->new_dfd = READ_ONCE(sqe->len);
3462 ren->flags = READ_ONCE(sqe->rename_flags);
3464 ren->oldpath = getname(oldf);
3465 if (IS_ERR(ren->oldpath))
3466 return PTR_ERR(ren->oldpath);
3468 ren->newpath = getname(newf);
3469 if (IS_ERR(ren->newpath)) {
3470 putname(ren->oldpath);
3471 return PTR_ERR(ren->newpath);
3474 req->flags |= REQ_F_NEED_CLEANUP;
3478 static int io_renameat(struct io_kiocb *req, unsigned int issue_flags)
3480 struct io_rename *ren = &req->rename;
3483 if (issue_flags & IO_URING_F_NONBLOCK)
3486 ret = do_renameat2(ren->old_dfd, ren->oldpath, ren->new_dfd,
3487 ren->newpath, ren->flags);
3489 req->flags &= ~REQ_F_NEED_CLEANUP;
3492 io_req_complete(req, ret);
3496 static int io_unlinkat_prep(struct io_kiocb *req,
3497 const struct io_uring_sqe *sqe)
3499 struct io_unlink *un = &req->unlink;
3500 const char __user *fname;
3502 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3504 if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index)
3506 if (unlikely(req->flags & REQ_F_FIXED_FILE))
3509 un->dfd = READ_ONCE(sqe->fd);
3511 un->flags = READ_ONCE(sqe->unlink_flags);
3512 if (un->flags & ~AT_REMOVEDIR)
3515 fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
3516 un->filename = getname(fname);
3517 if (IS_ERR(un->filename))
3518 return PTR_ERR(un->filename);
3520 req->flags |= REQ_F_NEED_CLEANUP;
3524 static int io_unlinkat(struct io_kiocb *req, unsigned int issue_flags)
3526 struct io_unlink *un = &req->unlink;
3529 if (issue_flags & IO_URING_F_NONBLOCK)
3532 if (un->flags & AT_REMOVEDIR)
3533 ret = do_rmdir(un->dfd, un->filename);
3535 ret = do_unlinkat(un->dfd, un->filename);
3537 req->flags &= ~REQ_F_NEED_CLEANUP;
3540 io_req_complete(req, ret);
3544 static int io_shutdown_prep(struct io_kiocb *req,
3545 const struct io_uring_sqe *sqe)
3547 #if defined(CONFIG_NET)
3548 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3550 if (sqe->ioprio || sqe->off || sqe->addr || sqe->rw_flags ||
3554 req->shutdown.how = READ_ONCE(sqe->len);
3561 static int io_shutdown(struct io_kiocb *req, unsigned int issue_flags)
3563 #if defined(CONFIG_NET)
3564 struct socket *sock;
3567 if (issue_flags & IO_URING_F_NONBLOCK)
3570 sock = sock_from_file(req->file);
3571 if (unlikely(!sock))
3574 ret = __sys_shutdown_sock(sock, req->shutdown.how);
3577 io_req_complete(req, ret);
3584 static int __io_splice_prep(struct io_kiocb *req,
3585 const struct io_uring_sqe *sqe)
3587 struct io_splice *sp = &req->splice;
3588 unsigned int valid_flags = SPLICE_F_FD_IN_FIXED | SPLICE_F_ALL;
3590 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3594 sp->len = READ_ONCE(sqe->len);
3595 sp->flags = READ_ONCE(sqe->splice_flags);
3597 if (unlikely(sp->flags & ~valid_flags))
3600 sp->file_in = io_file_get(NULL, req, READ_ONCE(sqe->splice_fd_in),
3601 (sp->flags & SPLICE_F_FD_IN_FIXED));
3604 req->flags |= REQ_F_NEED_CLEANUP;
3608 static int io_tee_prep(struct io_kiocb *req,
3609 const struct io_uring_sqe *sqe)
3611 if (READ_ONCE(sqe->splice_off_in) || READ_ONCE(sqe->off))
3613 return __io_splice_prep(req, sqe);
3616 static int io_tee(struct io_kiocb *req, unsigned int issue_flags)
3618 struct io_splice *sp = &req->splice;
3619 struct file *in = sp->file_in;
3620 struct file *out = sp->file_out;
3621 unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
3624 if (issue_flags & IO_URING_F_NONBLOCK)
3627 ret = do_tee(in, out, sp->len, flags);
3629 if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
3631 req->flags &= ~REQ_F_NEED_CLEANUP;
3635 io_req_complete(req, ret);
3639 static int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3641 struct io_splice *sp = &req->splice;
3643 sp->off_in = READ_ONCE(sqe->splice_off_in);
3644 sp->off_out = READ_ONCE(sqe->off);
3645 return __io_splice_prep(req, sqe);
3648 static int io_splice(struct io_kiocb *req, unsigned int issue_flags)
3650 struct io_splice *sp = &req->splice;
3651 struct file *in = sp->file_in;
3652 struct file *out = sp->file_out;
3653 unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
3654 loff_t *poff_in, *poff_out;
3657 if (issue_flags & IO_URING_F_NONBLOCK)
3660 poff_in = (sp->off_in == -1) ? NULL : &sp->off_in;
3661 poff_out = (sp->off_out == -1) ? NULL : &sp->off_out;
3664 ret = do_splice(in, poff_in, out, poff_out, sp->len, flags);
3666 if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
3668 req->flags &= ~REQ_F_NEED_CLEANUP;
3672 io_req_complete(req, ret);
3677 * IORING_OP_NOP just posts a completion event, nothing else.
3679 static int io_nop(struct io_kiocb *req, unsigned int issue_flags)
3681 struct io_ring_ctx *ctx = req->ctx;
3683 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
3686 __io_req_complete(req, issue_flags, 0, 0);
3690 static int io_fsync_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3692 struct io_ring_ctx *ctx = req->ctx;
3697 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
3699 if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
3702 req->sync.flags = READ_ONCE(sqe->fsync_flags);
3703 if (unlikely(req->sync.flags & ~IORING_FSYNC_DATASYNC))
3706 req->sync.off = READ_ONCE(sqe->off);
3707 req->sync.len = READ_ONCE(sqe->len);
3711 static int io_fsync(struct io_kiocb *req, unsigned int issue_flags)
3713 loff_t end = req->sync.off + req->sync.len;
3716 /* fsync always requires a blocking context */
3717 if (issue_flags & IO_URING_F_NONBLOCK)
3720 ret = vfs_fsync_range(req->file, req->sync.off,
3721 end > 0 ? end : LLONG_MAX,
3722 req->sync.flags & IORING_FSYNC_DATASYNC);
3725 io_req_complete(req, ret);
3729 static int io_fallocate_prep(struct io_kiocb *req,
3730 const struct io_uring_sqe *sqe)
3732 if (sqe->ioprio || sqe->buf_index || sqe->rw_flags)
3734 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3737 req->sync.off = READ_ONCE(sqe->off);
3738 req->sync.len = READ_ONCE(sqe->addr);
3739 req->sync.mode = READ_ONCE(sqe->len);
3743 static int io_fallocate(struct io_kiocb *req, unsigned int issue_flags)
3747 /* fallocate always requiring blocking context */
3748 if (issue_flags & IO_URING_F_NONBLOCK)
3750 ret = vfs_fallocate(req->file, req->sync.mode, req->sync.off,
3754 io_req_complete(req, ret);
3758 static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3760 const char __user *fname;
3763 if (unlikely(sqe->ioprio || sqe->buf_index))
3765 if (unlikely(req->flags & REQ_F_FIXED_FILE))
3768 /* open.how should be already initialised */
3769 if (!(req->open.how.flags & O_PATH) && force_o_largefile())
3770 req->open.how.flags |= O_LARGEFILE;
3772 req->open.dfd = READ_ONCE(sqe->fd);
3773 fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
3774 req->open.filename = getname(fname);
3775 if (IS_ERR(req->open.filename)) {
3776 ret = PTR_ERR(req->open.filename);
3777 req->open.filename = NULL;
3780 req->open.nofile = rlimit(RLIMIT_NOFILE);
3781 req->flags |= REQ_F_NEED_CLEANUP;
3785 static int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3789 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3791 mode = READ_ONCE(sqe->len);
3792 flags = READ_ONCE(sqe->open_flags);
3793 req->open.how = build_open_how(flags, mode);
3794 return __io_openat_prep(req, sqe);
3797 static int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3799 struct open_how __user *how;
3803 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3805 how = u64_to_user_ptr(READ_ONCE(sqe->addr2));
3806 len = READ_ONCE(sqe->len);
3807 if (len < OPEN_HOW_SIZE_VER0)
3810 ret = copy_struct_from_user(&req->open.how, sizeof(req->open.how), how,
3815 return __io_openat_prep(req, sqe);
3818 static int io_openat2(struct io_kiocb *req, unsigned int issue_flags)
3820 struct open_flags op;
3823 bool resolve_nonblock;
3826 ret = build_open_flags(&req->open.how, &op);
3829 nonblock_set = op.open_flag & O_NONBLOCK;
3830 resolve_nonblock = req->open.how.resolve & RESOLVE_CACHED;
3831 if (issue_flags & IO_URING_F_NONBLOCK) {
3833 * Don't bother trying for O_TRUNC, O_CREAT, or O_TMPFILE open,
3834 * it'll always -EAGAIN
3836 if (req->open.how.flags & (O_TRUNC | O_CREAT | O_TMPFILE))
3838 op.lookup_flags |= LOOKUP_CACHED;
3839 op.open_flag |= O_NONBLOCK;
3842 ret = __get_unused_fd_flags(req->open.how.flags, req->open.nofile);
3846 file = do_filp_open(req->open.dfd, req->open.filename, &op);
3849 * We could hang on to this 'fd' on retrying, but seems like
3850 * marginal gain for something that is now known to be a slower
3851 * path. So just put it, and we'll get a new one when we retry.
3855 ret = PTR_ERR(file);
3856 /* only retry if RESOLVE_CACHED wasn't already set by application */
3857 if (ret == -EAGAIN &&
3858 (!resolve_nonblock && (issue_flags & IO_URING_F_NONBLOCK)))
3863 if ((issue_flags & IO_URING_F_NONBLOCK) && !nonblock_set)
3864 file->f_flags &= ~O_NONBLOCK;
3865 fsnotify_open(file);
3866 fd_install(ret, file);
3868 putname(req->open.filename);
3869 req->flags &= ~REQ_F_NEED_CLEANUP;
3872 __io_req_complete(req, issue_flags, ret, 0);
3876 static int io_openat(struct io_kiocb *req, unsigned int issue_flags)
3878 return io_openat2(req, issue_flags);
3881 static int io_remove_buffers_prep(struct io_kiocb *req,
3882 const struct io_uring_sqe *sqe)
3884 struct io_provide_buf *p = &req->pbuf;
3887 if (sqe->ioprio || sqe->rw_flags || sqe->addr || sqe->len || sqe->off)
3890 tmp = READ_ONCE(sqe->fd);
3891 if (!tmp || tmp > USHRT_MAX)
3894 memset(p, 0, sizeof(*p));
3896 p->bgid = READ_ONCE(sqe->buf_group);
3900 static int __io_remove_buffers(struct io_ring_ctx *ctx, struct io_buffer *buf,
3901 int bgid, unsigned nbufs)
3905 /* shouldn't happen */
3909 /* the head kbuf is the list itself */
3910 while (!list_empty(&buf->list)) {
3911 struct io_buffer *nxt;
3913 nxt = list_first_entry(&buf->list, struct io_buffer, list);
3914 list_del(&nxt->list);
3921 xa_erase(&ctx->io_buffers, bgid);
3926 static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags)
3928 struct io_provide_buf *p = &req->pbuf;
3929 struct io_ring_ctx *ctx = req->ctx;
3930 struct io_buffer *head;
3932 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
3934 io_ring_submit_lock(ctx, !force_nonblock);
3936 lockdep_assert_held(&ctx->uring_lock);
3939 head = xa_load(&ctx->io_buffers, p->bgid);
3941 ret = __io_remove_buffers(ctx, head, p->bgid, p->nbufs);
3945 /* complete before unlock, IOPOLL may need the lock */
3946 __io_req_complete(req, issue_flags, ret, 0);
3947 io_ring_submit_unlock(ctx, !force_nonblock);
3951 static int io_provide_buffers_prep(struct io_kiocb *req,
3952 const struct io_uring_sqe *sqe)
3954 unsigned long size, tmp_check;
3955 struct io_provide_buf *p = &req->pbuf;
3958 if (sqe->ioprio || sqe->rw_flags)
3961 tmp = READ_ONCE(sqe->fd);
3962 if (!tmp || tmp > USHRT_MAX)
3965 p->addr = READ_ONCE(sqe->addr);
3966 p->len = READ_ONCE(sqe->len);
3968 if (check_mul_overflow((unsigned long)p->len, (unsigned long)p->nbufs,
3971 if (check_add_overflow((unsigned long)p->addr, size, &tmp_check))
3974 size = (unsigned long)p->len * p->nbufs;
3975 if (!access_ok(u64_to_user_ptr(p->addr), size))
3978 p->bgid = READ_ONCE(sqe->buf_group);
3979 tmp = READ_ONCE(sqe->off);
3980 if (tmp > USHRT_MAX)
3986 static int io_add_buffers(struct io_provide_buf *pbuf, struct io_buffer **head)
3988 struct io_buffer *buf;
3989 u64 addr = pbuf->addr;
3990 int i, bid = pbuf->bid;
3992 for (i = 0; i < pbuf->nbufs; i++) {
3993 buf = kmalloc(sizeof(*buf), GFP_KERNEL);
3998 buf->len = min_t(__u32, pbuf->len, MAX_RW_COUNT);
4003 INIT_LIST_HEAD(&buf->list);
4006 list_add_tail(&buf->list, &(*head)->list);
4010 return i ? i : -ENOMEM;
4013 static int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags)
4015 struct io_provide_buf *p = &req->pbuf;
4016 struct io_ring_ctx *ctx = req->ctx;
4017 struct io_buffer *head, *list;
4019 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
4021 io_ring_submit_lock(ctx, !force_nonblock);
4023 lockdep_assert_held(&ctx->uring_lock);
4025 list = head = xa_load(&ctx->io_buffers, p->bgid);
4027 ret = io_add_buffers(p, &head);
4028 if (ret >= 0 && !list) {
4029 ret = xa_insert(&ctx->io_buffers, p->bgid, head, GFP_KERNEL);
4031 __io_remove_buffers(ctx, head, p->bgid, -1U);
4035 /* complete before unlock, IOPOLL may need the lock */
4036 __io_req_complete(req, issue_flags, ret, 0);
4037 io_ring_submit_unlock(ctx, !force_nonblock);
4041 static int io_epoll_ctl_prep(struct io_kiocb *req,
4042 const struct io_uring_sqe *sqe)
4044 #if defined(CONFIG_EPOLL)
4045 if (sqe->ioprio || sqe->buf_index)
4047 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4050 req->epoll.epfd = READ_ONCE(sqe->fd);
4051 req->epoll.op = READ_ONCE(sqe->len);
4052 req->epoll.fd = READ_ONCE(sqe->off);
4054 if (ep_op_has_event(req->epoll.op)) {
4055 struct epoll_event __user *ev;
4057 ev = u64_to_user_ptr(READ_ONCE(sqe->addr));
4058 if (copy_from_user(&req->epoll.event, ev, sizeof(*ev)))
4068 static int io_epoll_ctl(struct io_kiocb *req, unsigned int issue_flags)
4070 #if defined(CONFIG_EPOLL)
4071 struct io_epoll *ie = &req->epoll;
4073 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
4075 ret = do_epoll_ctl(ie->epfd, ie->op, ie->fd, &ie->event, force_nonblock);
4076 if (force_nonblock && ret == -EAGAIN)
4081 __io_req_complete(req, issue_flags, ret, 0);
4088 static int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4090 #if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
4091 if (sqe->ioprio || sqe->buf_index || sqe->off)
4093 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4096 req->madvise.addr = READ_ONCE(sqe->addr);
4097 req->madvise.len = READ_ONCE(sqe->len);
4098 req->madvise.advice = READ_ONCE(sqe->fadvise_advice);
4105 static int io_madvise(struct io_kiocb *req, unsigned int issue_flags)
4107 #if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
4108 struct io_madvise *ma = &req->madvise;
4111 if (issue_flags & IO_URING_F_NONBLOCK)
4114 ret = do_madvise(current->mm, ma->addr, ma->len, ma->advice);
4117 io_req_complete(req, ret);
4124 static int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4126 if (sqe->ioprio || sqe->buf_index || sqe->addr)
4128 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4131 req->fadvise.offset = READ_ONCE(sqe->off);
4132 req->fadvise.len = READ_ONCE(sqe->len);
4133 req->fadvise.advice = READ_ONCE(sqe->fadvise_advice);
4137 static int io_fadvise(struct io_kiocb *req, unsigned int issue_flags)
4139 struct io_fadvise *fa = &req->fadvise;
4142 if (issue_flags & IO_URING_F_NONBLOCK) {
4143 switch (fa->advice) {
4144 case POSIX_FADV_NORMAL:
4145 case POSIX_FADV_RANDOM:
4146 case POSIX_FADV_SEQUENTIAL:
4153 ret = vfs_fadvise(req->file, fa->offset, fa->len, fa->advice);
4156 __io_req_complete(req, issue_flags, ret, 0);
4160 static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4162 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4164 if (sqe->ioprio || sqe->buf_index)
4166 if (req->flags & REQ_F_FIXED_FILE)
4169 req->statx.dfd = READ_ONCE(sqe->fd);
4170 req->statx.mask = READ_ONCE(sqe->len);
4171 req->statx.filename = u64_to_user_ptr(READ_ONCE(sqe->addr));
4172 req->statx.buffer = u64_to_user_ptr(READ_ONCE(sqe->addr2));
4173 req->statx.flags = READ_ONCE(sqe->statx_flags);
4178 static int io_statx(struct io_kiocb *req, unsigned int issue_flags)
4180 struct io_statx *ctx = &req->statx;
4183 if (issue_flags & IO_URING_F_NONBLOCK)
4186 ret = do_statx(ctx->dfd, ctx->filename, ctx->flags, ctx->mask,
4191 io_req_complete(req, ret);
4195 static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4197 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4199 if (sqe->ioprio || sqe->off || sqe->addr || sqe->len ||
4200 sqe->rw_flags || sqe->buf_index)
4202 if (req->flags & REQ_F_FIXED_FILE)
4205 req->close.fd = READ_ONCE(sqe->fd);
4209 static int io_close(struct io_kiocb *req, unsigned int issue_flags)
4211 struct files_struct *files = current->files;
4212 struct io_close *close = &req->close;
4213 struct fdtable *fdt;
4214 struct file *file = NULL;
4217 spin_lock(&files->file_lock);
4218 fdt = files_fdtable(files);
4219 if (close->fd >= fdt->max_fds) {
4220 spin_unlock(&files->file_lock);
4223 file = fdt->fd[close->fd];
4224 if (!file || file->f_op == &io_uring_fops) {
4225 spin_unlock(&files->file_lock);
4230 /* if the file has a flush method, be safe and punt to async */
4231 if (file->f_op->flush && (issue_flags & IO_URING_F_NONBLOCK)) {
4232 spin_unlock(&files->file_lock);
4236 ret = __close_fd_get_file(close->fd, &file);
4237 spin_unlock(&files->file_lock);
4244 /* No ->flush() or already async, safely close from here */
4245 ret = filp_close(file, current->files);
4251 __io_req_complete(req, issue_flags, ret, 0);
4255 static int io_sfr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4257 struct io_ring_ctx *ctx = req->ctx;
4259 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
4261 if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
4264 req->sync.off = READ_ONCE(sqe->off);
4265 req->sync.len = READ_ONCE(sqe->len);
4266 req->sync.flags = READ_ONCE(sqe->sync_range_flags);
4270 static int io_sync_file_range(struct io_kiocb *req, unsigned int issue_flags)
4274 /* sync_file_range always requires a blocking context */
4275 if (issue_flags & IO_URING_F_NONBLOCK)
4278 ret = sync_file_range(req->file, req->sync.off, req->sync.len,
4282 io_req_complete(req, ret);
4286 #if defined(CONFIG_NET)
4287 static int io_setup_async_msg(struct io_kiocb *req,
4288 struct io_async_msghdr *kmsg)
4290 struct io_async_msghdr *async_msg = req->async_data;
4294 if (io_alloc_async_data(req)) {
4295 kfree(kmsg->free_iov);
4298 async_msg = req->async_data;
4299 req->flags |= REQ_F_NEED_CLEANUP;
4300 memcpy(async_msg, kmsg, sizeof(*kmsg));
4301 async_msg->msg.msg_name = &async_msg->addr;
4302 /* if were using fast_iov, set it to the new one */
4303 if (!async_msg->free_iov)
4304 async_msg->msg.msg_iter.iov = async_msg->fast_iov;
4309 static int io_sendmsg_copy_hdr(struct io_kiocb *req,
4310 struct io_async_msghdr *iomsg)
4312 iomsg->msg.msg_name = &iomsg->addr;
4313 iomsg->free_iov = iomsg->fast_iov;
4314 return sendmsg_copy_msghdr(&iomsg->msg, req->sr_msg.umsg,
4315 req->sr_msg.msg_flags, &iomsg->free_iov);
4318 static int io_sendmsg_prep_async(struct io_kiocb *req)
4322 ret = io_sendmsg_copy_hdr(req, req->async_data);
4324 req->flags |= REQ_F_NEED_CLEANUP;
4328 static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4330 struct io_sr_msg *sr = &req->sr_msg;
4332 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4335 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
4336 sr->len = READ_ONCE(sqe->len);
4337 sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;
4338 if (sr->msg_flags & MSG_DONTWAIT)
4339 req->flags |= REQ_F_NOWAIT;
4341 #ifdef CONFIG_COMPAT
4342 if (req->ctx->compat)
4343 sr->msg_flags |= MSG_CMSG_COMPAT;
4348 static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)
4350 struct io_async_msghdr iomsg, *kmsg;
4351 struct socket *sock;
4356 sock = sock_from_file(req->file);
4357 if (unlikely(!sock))
4360 kmsg = req->async_data;
4362 ret = io_sendmsg_copy_hdr(req, &iomsg);
4368 flags = req->sr_msg.msg_flags;
4369 if (issue_flags & IO_URING_F_NONBLOCK)
4370 flags |= MSG_DONTWAIT;
4371 if (flags & MSG_WAITALL)
4372 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
4374 ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
4375 if ((issue_flags & IO_URING_F_NONBLOCK) && ret == -EAGAIN)
4376 return io_setup_async_msg(req, kmsg);
4377 if (ret == -ERESTARTSYS)
4380 /* fast path, check for non-NULL to avoid function call */
4382 kfree(kmsg->free_iov);
4383 req->flags &= ~REQ_F_NEED_CLEANUP;
4386 __io_req_complete(req, issue_flags, ret, 0);
4390 static int io_send(struct io_kiocb *req, unsigned int issue_flags)
4392 struct io_sr_msg *sr = &req->sr_msg;
4395 struct socket *sock;
4400 sock = sock_from_file(req->file);
4401 if (unlikely(!sock))
4404 ret = import_single_range(WRITE, sr->buf, sr->len, &iov, &msg.msg_iter);
4408 msg.msg_name = NULL;
4409 msg.msg_control = NULL;
4410 msg.msg_controllen = 0;
4411 msg.msg_namelen = 0;
4413 flags = req->sr_msg.msg_flags;
4414 if (issue_flags & IO_URING_F_NONBLOCK)
4415 flags |= MSG_DONTWAIT;
4416 if (flags & MSG_WAITALL)
4417 min_ret = iov_iter_count(&msg.msg_iter);
4419 msg.msg_flags = flags;
4420 ret = sock_sendmsg(sock, &msg);
4421 if ((issue_flags & IO_URING_F_NONBLOCK) && ret == -EAGAIN)
4423 if (ret == -ERESTARTSYS)
4428 __io_req_complete(req, issue_flags, ret, 0);
4432 static int __io_recvmsg_copy_hdr(struct io_kiocb *req,
4433 struct io_async_msghdr *iomsg)
4435 struct io_sr_msg *sr = &req->sr_msg;
4436 struct iovec __user *uiov;
4440 ret = __copy_msghdr_from_user(&iomsg->msg, sr->umsg,
4441 &iomsg->uaddr, &uiov, &iov_len);
4445 if (req->flags & REQ_F_BUFFER_SELECT) {
4448 if (copy_from_user(iomsg->fast_iov, uiov, sizeof(*uiov)))
4450 sr->len = iomsg->fast_iov[0].iov_len;
4451 iomsg->free_iov = NULL;
4453 iomsg->free_iov = iomsg->fast_iov;
4454 ret = __import_iovec(READ, uiov, iov_len, UIO_FASTIOV,
4455 &iomsg->free_iov, &iomsg->msg.msg_iter,
4464 #ifdef CONFIG_COMPAT
4465 static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req,
4466 struct io_async_msghdr *iomsg)
4468 struct io_sr_msg *sr = &req->sr_msg;
4469 struct compat_iovec __user *uiov;
4474 ret = __get_compat_msghdr(&iomsg->msg, sr->umsg_compat, &iomsg->uaddr,
4479 uiov = compat_ptr(ptr);
4480 if (req->flags & REQ_F_BUFFER_SELECT) {
4481 compat_ssize_t clen;
4485 if (!access_ok(uiov, sizeof(*uiov)))
4487 if (__get_user(clen, &uiov->iov_len))
4492 iomsg->free_iov = NULL;
4494 iomsg->free_iov = iomsg->fast_iov;
4495 ret = __import_iovec(READ, (struct iovec __user *)uiov, len,
4496 UIO_FASTIOV, &iomsg->free_iov,
4497 &iomsg->msg.msg_iter, true);
4506 static int io_recvmsg_copy_hdr(struct io_kiocb *req,
4507 struct io_async_msghdr *iomsg)
4509 iomsg->msg.msg_name = &iomsg->addr;
4511 #ifdef CONFIG_COMPAT
4512 if (req->ctx->compat)
4513 return __io_compat_recvmsg_copy_hdr(req, iomsg);
4516 return __io_recvmsg_copy_hdr(req, iomsg);
4519 static struct io_buffer *io_recv_buffer_select(struct io_kiocb *req,
4522 struct io_sr_msg *sr = &req->sr_msg;
4523 struct io_buffer *kbuf;
4525 kbuf = io_buffer_select(req, &sr->len, sr->bgid, sr->kbuf, needs_lock);
4530 req->flags |= REQ_F_BUFFER_SELECTED;
4534 static inline unsigned int io_put_recv_kbuf(struct io_kiocb *req)
4536 return io_put_kbuf(req, req->sr_msg.kbuf);
4539 static int io_recvmsg_prep_async(struct io_kiocb *req)
4543 ret = io_recvmsg_copy_hdr(req, req->async_data);
4545 req->flags |= REQ_F_NEED_CLEANUP;
4549 static int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4551 struct io_sr_msg *sr = &req->sr_msg;
4553 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4556 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
4557 sr->len = READ_ONCE(sqe->len);
4558 sr->bgid = READ_ONCE(sqe->buf_group);
4559 sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;
4560 if (sr->msg_flags & MSG_DONTWAIT)
4561 req->flags |= REQ_F_NOWAIT;
4563 #ifdef CONFIG_COMPAT
4564 if (req->ctx->compat)
4565 sr->msg_flags |= MSG_CMSG_COMPAT;
4570 static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
4572 struct io_async_msghdr iomsg, *kmsg;
4573 struct socket *sock;
4574 struct io_buffer *kbuf;
4577 int ret, cflags = 0;
4578 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
4580 sock = sock_from_file(req->file);
4581 if (unlikely(!sock))
4584 kmsg = req->async_data;
4586 ret = io_recvmsg_copy_hdr(req, &iomsg);
4592 if (req->flags & REQ_F_BUFFER_SELECT) {
4593 kbuf = io_recv_buffer_select(req, !force_nonblock);
4595 return PTR_ERR(kbuf);
4596 kmsg->fast_iov[0].iov_base = u64_to_user_ptr(kbuf->addr);
4597 kmsg->fast_iov[0].iov_len = req->sr_msg.len;
4598 iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->fast_iov,
4599 1, req->sr_msg.len);
4602 flags = req->sr_msg.msg_flags;
4604 flags |= MSG_DONTWAIT;
4605 if (flags & MSG_WAITALL)
4606 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
4608 ret = __sys_recvmsg_sock(sock, &kmsg->msg, req->sr_msg.umsg,
4609 kmsg->uaddr, flags);
4610 if (force_nonblock && ret == -EAGAIN)
4611 return io_setup_async_msg(req, kmsg);
4612 if (ret == -ERESTARTSYS)
4615 if (req->flags & REQ_F_BUFFER_SELECTED)
4616 cflags = io_put_recv_kbuf(req);
4617 /* fast path, check for non-NULL to avoid function call */
4619 kfree(kmsg->free_iov);
4620 req->flags &= ~REQ_F_NEED_CLEANUP;
4621 if (ret < min_ret || ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))))
4623 __io_req_complete(req, issue_flags, ret, cflags);
4627 static int io_recv(struct io_kiocb *req, unsigned int issue_flags)
4629 struct io_buffer *kbuf;
4630 struct io_sr_msg *sr = &req->sr_msg;
4632 void __user *buf = sr->buf;
4633 struct socket *sock;
4637 int ret, cflags = 0;
4638 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
4640 sock = sock_from_file(req->file);
4641 if (unlikely(!sock))
4644 if (req->flags & REQ_F_BUFFER_SELECT) {
4645 kbuf = io_recv_buffer_select(req, !force_nonblock);
4647 return PTR_ERR(kbuf);
4648 buf = u64_to_user_ptr(kbuf->addr);
4651 ret = import_single_range(READ, buf, sr->len, &iov, &msg.msg_iter);
4655 msg.msg_name = NULL;
4656 msg.msg_control = NULL;
4657 msg.msg_controllen = 0;
4658 msg.msg_namelen = 0;
4659 msg.msg_iocb = NULL;
4662 flags = req->sr_msg.msg_flags;
4664 flags |= MSG_DONTWAIT;
4665 if (flags & MSG_WAITALL)
4666 min_ret = iov_iter_count(&msg.msg_iter);
4668 ret = sock_recvmsg(sock, &msg, flags);
4669 if (force_nonblock && ret == -EAGAIN)
4671 if (ret == -ERESTARTSYS)
4674 if (req->flags & REQ_F_BUFFER_SELECTED)
4675 cflags = io_put_recv_kbuf(req);
4676 if (ret < min_ret || ((flags & MSG_WAITALL) && (msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))))
4678 __io_req_complete(req, issue_flags, ret, cflags);
4682 static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4684 struct io_accept *accept = &req->accept;
4686 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4688 if (sqe->ioprio || sqe->len || sqe->buf_index)
4691 accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
4692 accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2));
4693 accept->flags = READ_ONCE(sqe->accept_flags);
4694 accept->nofile = rlimit(RLIMIT_NOFILE);
4698 static int io_accept(struct io_kiocb *req, unsigned int issue_flags)
4700 struct io_accept *accept = &req->accept;
4701 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
4702 unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0;
4705 if (req->file->f_flags & O_NONBLOCK)
4706 req->flags |= REQ_F_NOWAIT;
4708 ret = __sys_accept4_file(req->file, file_flags, accept->addr,
4709 accept->addr_len, accept->flags,
4711 if (ret == -EAGAIN && force_nonblock)
4714 if (ret == -ERESTARTSYS)
4718 __io_req_complete(req, issue_flags, ret, 0);
4722 static int io_connect_prep_async(struct io_kiocb *req)
4724 struct io_async_connect *io = req->async_data;
4725 struct io_connect *conn = &req->connect;
4727 return move_addr_to_kernel(conn->addr, conn->addr_len, &io->address);
4730 static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4732 struct io_connect *conn = &req->connect;
4734 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4736 if (sqe->ioprio || sqe->len || sqe->buf_index || sqe->rw_flags)
4739 conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
4740 conn->addr_len = READ_ONCE(sqe->addr2);
4744 static int io_connect(struct io_kiocb *req, unsigned int issue_flags)
4746 struct io_async_connect __io, *io;
4747 unsigned file_flags;
4749 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
4751 if (req->async_data) {
4752 io = req->async_data;
4754 ret = move_addr_to_kernel(req->connect.addr,
4755 req->connect.addr_len,
4762 file_flags = force_nonblock ? O_NONBLOCK : 0;
4764 ret = __sys_connect_file(req->file, &io->address,
4765 req->connect.addr_len, file_flags);
4766 if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) {
4767 if (req->async_data)
4769 if (io_alloc_async_data(req)) {
4773 memcpy(req->async_data, &__io, sizeof(__io));
4776 if (ret == -ERESTARTSYS)
4781 __io_req_complete(req, issue_flags, ret, 0);
4784 #else /* !CONFIG_NET */
4785 #define IO_NETOP_FN(op) \
4786 static int io_##op(struct io_kiocb *req, unsigned int issue_flags) \
4788 return -EOPNOTSUPP; \
4791 #define IO_NETOP_PREP(op) \
4793 static int io_##op##_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) \
4795 return -EOPNOTSUPP; \
4798 #define IO_NETOP_PREP_ASYNC(op) \
4800 static int io_##op##_prep_async(struct io_kiocb *req) \
4802 return -EOPNOTSUPP; \
4805 IO_NETOP_PREP_ASYNC(sendmsg);
4806 IO_NETOP_PREP_ASYNC(recvmsg);
4807 IO_NETOP_PREP_ASYNC(connect);
4808 IO_NETOP_PREP(accept);
4811 #endif /* CONFIG_NET */
4813 struct io_poll_table {
4814 struct poll_table_struct pt;
4815 struct io_kiocb *req;
4820 static int __io_async_wake(struct io_kiocb *req, struct io_poll_iocb *poll,
4821 __poll_t mask, io_req_tw_func_t func)
4823 /* for instances that support it check for an event match first: */
4824 if (mask && !(mask & poll->events))
4827 trace_io_uring_task_add(req->ctx, req->opcode, req->user_data, mask);
4829 list_del_init(&poll->wait.entry);
4832 req->io_task_work.func = func;
4835 * If this fails, then the task is exiting. When a task exits, the
4836 * work gets canceled, so just cancel this request as well instead
4837 * of executing it. We can't safely execute it anyway, as we may not
4838 * have the needed state needed for it anyway.
4840 io_req_task_work_add(req);
4844 static bool io_poll_rewait(struct io_kiocb *req, struct io_poll_iocb *poll)
4845 __acquires(&req->ctx->completion_lock)
4847 struct io_ring_ctx *ctx = req->ctx;
4849 if (unlikely(req->task->flags & PF_EXITING))
4850 WRITE_ONCE(poll->canceled, true);
4852 if (!req->result && !READ_ONCE(poll->canceled)) {
4853 struct poll_table_struct pt = { ._key = poll->events };
4855 req->result = vfs_poll(req->file, &pt) & poll->events;
4858 spin_lock_irq(&ctx->completion_lock);
4859 if (!req->result && !READ_ONCE(poll->canceled)) {
4860 add_wait_queue(poll->head, &poll->wait);
4867 static struct io_poll_iocb *io_poll_get_double(struct io_kiocb *req)
4869 /* pure poll stashes this in ->async_data, poll driven retry elsewhere */
4870 if (req->opcode == IORING_OP_POLL_ADD)
4871 return req->async_data;
4872 return req->apoll->double_poll;
4875 static struct io_poll_iocb *io_poll_get_single(struct io_kiocb *req)
4877 if (req->opcode == IORING_OP_POLL_ADD)
4879 return &req->apoll->poll;
4882 static void io_poll_remove_double(struct io_kiocb *req)
4883 __must_hold(&req->ctx->completion_lock)
4885 struct io_poll_iocb *poll = io_poll_get_double(req);
4887 lockdep_assert_held(&req->ctx->completion_lock);
4889 if (poll && poll->head) {
4890 struct wait_queue_head *head = poll->head;
4892 spin_lock(&head->lock);
4893 list_del_init(&poll->wait.entry);
4894 if (poll->wait.private)
4897 spin_unlock(&head->lock);
4901 static bool io_poll_complete(struct io_kiocb *req, __poll_t mask)
4902 __must_hold(&req->ctx->completion_lock)
4904 struct io_ring_ctx *ctx = req->ctx;
4905 unsigned flags = IORING_CQE_F_MORE;
4908 if (READ_ONCE(req->poll.canceled)) {
4910 req->poll.events |= EPOLLONESHOT;
4912 error = mangle_poll(mask);
4914 if (req->poll.events & EPOLLONESHOT)
4916 if (!io_cqring_fill_event(ctx, req->user_data, error, flags)) {
4917 io_poll_remove_waitqs(req);
4918 req->poll.done = true;
4921 if (flags & IORING_CQE_F_MORE)
4924 io_commit_cqring(ctx);
4925 return !(flags & IORING_CQE_F_MORE);
4928 static void io_poll_task_func(struct io_kiocb *req)
4930 struct io_ring_ctx *ctx = req->ctx;
4931 struct io_kiocb *nxt;
4933 if (io_poll_rewait(req, &req->poll)) {
4934 spin_unlock_irq(&ctx->completion_lock);
4938 done = io_poll_complete(req, req->result);
4940 hash_del(&req->hash_node);
4943 add_wait_queue(req->poll.head, &req->poll.wait);
4945 spin_unlock_irq(&ctx->completion_lock);
4946 io_cqring_ev_posted(ctx);
4949 nxt = io_put_req_find_next(req);
4951 io_req_task_submit(nxt);
4956 static int io_poll_double_wake(struct wait_queue_entry *wait, unsigned mode,
4957 int sync, void *key)
4959 struct io_kiocb *req = wait->private;
4960 struct io_poll_iocb *poll = io_poll_get_single(req);
4961 __poll_t mask = key_to_poll(key);
4963 /* for instances that support it check for an event match first: */
4964 if (mask && !(mask & poll->events))
4966 if (!(poll->events & EPOLLONESHOT))
4967 return poll->wait.func(&poll->wait, mode, sync, key);
4969 list_del_init(&wait->entry);
4974 spin_lock(&poll->head->lock);
4975 done = list_empty(&poll->wait.entry);
4977 list_del_init(&poll->wait.entry);
4978 /* make sure double remove sees this as being gone */
4979 wait->private = NULL;
4980 spin_unlock(&poll->head->lock);
4982 /* use wait func handler, so it matches the rq type */
4983 poll->wait.func(&poll->wait, mode, sync, key);
4990 static void io_init_poll_iocb(struct io_poll_iocb *poll, __poll_t events,
4991 wait_queue_func_t wake_func)
4995 poll->canceled = false;
4996 #define IO_POLL_UNMASK (EPOLLERR|EPOLLHUP|EPOLLNVAL|EPOLLRDHUP)
4997 /* mask in events that we always want/need */
4998 poll->events = events | IO_POLL_UNMASK;
4999 INIT_LIST_HEAD(&poll->wait.entry);
5000 init_waitqueue_func_entry(&poll->wait, wake_func);
5003 static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt,
5004 struct wait_queue_head *head,
5005 struct io_poll_iocb **poll_ptr)
5007 struct io_kiocb *req = pt->req;
5010 * The file being polled uses multiple waitqueues for poll handling
5011 * (e.g. one for read, one for write). Setup a separate io_poll_iocb
5014 if (unlikely(pt->nr_entries)) {
5015 struct io_poll_iocb *poll_one = poll;
5017 /* already have a 2nd entry, fail a third attempt */
5019 pt->error = -EINVAL;
5023 * Can't handle multishot for double wait for now, turn it
5024 * into one-shot mode.
5026 if (!(poll_one->events & EPOLLONESHOT))
5027 poll_one->events |= EPOLLONESHOT;
5028 /* double add on the same waitqueue head, ignore */
5029 if (poll_one->head == head)
5031 poll = kmalloc(sizeof(*poll), GFP_ATOMIC);
5033 pt->error = -ENOMEM;
5036 io_init_poll_iocb(poll, poll_one->events, io_poll_double_wake);
5038 poll->wait.private = req;
5045 if (poll->events & EPOLLEXCLUSIVE)
5046 add_wait_queue_exclusive(head, &poll->wait);
5048 add_wait_queue(head, &poll->wait);
5051 static void io_async_queue_proc(struct file *file, struct wait_queue_head *head,
5052 struct poll_table_struct *p)
5054 struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
5055 struct async_poll *apoll = pt->req->apoll;
5057 __io_queue_proc(&apoll->poll, pt, head, &apoll->double_poll);
5060 static void io_async_task_func(struct io_kiocb *req)
5062 struct async_poll *apoll = req->apoll;
5063 struct io_ring_ctx *ctx = req->ctx;
5065 trace_io_uring_task_run(req->ctx, req, req->opcode, req->user_data);
5067 if (io_poll_rewait(req, &apoll->poll)) {
5068 spin_unlock_irq(&ctx->completion_lock);
5072 hash_del(&req->hash_node);
5073 io_poll_remove_double(req);
5074 spin_unlock_irq(&ctx->completion_lock);
5076 if (!READ_ONCE(apoll->poll.canceled))
5077 io_req_task_submit(req);
5079 io_req_complete_failed(req, -ECANCELED);
5082 static int io_async_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
5085 struct io_kiocb *req = wait->private;
5086 struct io_poll_iocb *poll = &req->apoll->poll;
5088 trace_io_uring_poll_wake(req->ctx, req->opcode, req->user_data,
5091 return __io_async_wake(req, poll, key_to_poll(key), io_async_task_func);
5094 static void io_poll_req_insert(struct io_kiocb *req)
5096 struct io_ring_ctx *ctx = req->ctx;
5097 struct hlist_head *list;
5099 list = &ctx->cancel_hash[hash_long(req->user_data, ctx->cancel_hash_bits)];
5100 hlist_add_head(&req->hash_node, list);
5103 static __poll_t __io_arm_poll_handler(struct io_kiocb *req,
5104 struct io_poll_iocb *poll,
5105 struct io_poll_table *ipt, __poll_t mask,
5106 wait_queue_func_t wake_func)
5107 __acquires(&ctx->completion_lock)
5109 struct io_ring_ctx *ctx = req->ctx;
5110 bool cancel = false;
5112 INIT_HLIST_NODE(&req->hash_node);
5113 io_init_poll_iocb(poll, mask, wake_func);
5114 poll->file = req->file;
5115 poll->wait.private = req;
5117 ipt->pt._key = mask;
5120 ipt->nr_entries = 0;
5122 mask = vfs_poll(req->file, &ipt->pt) & poll->events;
5123 if (unlikely(!ipt->nr_entries) && !ipt->error)
5124 ipt->error = -EINVAL;
5126 spin_lock_irq(&ctx->completion_lock);
5128 io_poll_remove_double(req);
5129 if (likely(poll->head)) {
5130 spin_lock(&poll->head->lock);
5131 if (unlikely(list_empty(&poll->wait.entry))) {
5137 if ((mask && (poll->events & EPOLLONESHOT)) || ipt->error)
5138 list_del_init(&poll->wait.entry);
5140 WRITE_ONCE(poll->canceled, true);
5141 else if (!poll->done) /* actually waiting for an event */
5142 io_poll_req_insert(req);
5143 spin_unlock(&poll->head->lock);
5155 static int io_arm_poll_handler(struct io_kiocb *req)
5157 const struct io_op_def *def = &io_op_defs[req->opcode];
5158 struct io_ring_ctx *ctx = req->ctx;
5159 struct async_poll *apoll;
5160 struct io_poll_table ipt;
5161 __poll_t ret, mask = EPOLLONESHOT | POLLERR | POLLPRI;
5164 if (!req->file || !file_can_poll(req->file))
5165 return IO_APOLL_ABORTED;
5166 if (req->flags & REQ_F_POLLED)
5167 return IO_APOLL_ABORTED;
5168 if (!def->pollin && !def->pollout)
5169 return IO_APOLL_ABORTED;
5173 mask |= POLLIN | POLLRDNORM;
5175 /* If reading from MSG_ERRQUEUE using recvmsg, ignore POLLIN */
5176 if ((req->opcode == IORING_OP_RECVMSG) &&
5177 (req->sr_msg.msg_flags & MSG_ERRQUEUE))
5181 mask |= POLLOUT | POLLWRNORM;
5184 /* if we can't nonblock try, then no point in arming a poll handler */
5185 if (!io_file_supports_async(req, rw))
5186 return IO_APOLL_ABORTED;
5188 apoll = kmalloc(sizeof(*apoll), GFP_ATOMIC);
5189 if (unlikely(!apoll))
5190 return IO_APOLL_ABORTED;
5191 apoll->double_poll = NULL;
5193 req->flags |= REQ_F_POLLED;
5194 ipt.pt._qproc = io_async_queue_proc;
5196 ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask,
5198 if (ret || ipt.error) {
5199 io_poll_remove_double(req);
5200 spin_unlock_irq(&ctx->completion_lock);
5202 return IO_APOLL_READY;
5203 return IO_APOLL_ABORTED;
5205 spin_unlock_irq(&ctx->completion_lock);
5206 trace_io_uring_poll_arm(ctx, req, req->opcode, req->user_data,
5207 mask, apoll->poll.events);
5211 static bool __io_poll_remove_one(struct io_kiocb *req,
5212 struct io_poll_iocb *poll, bool do_cancel)
5213 __must_hold(&req->ctx->completion_lock)
5215 bool do_complete = false;
5219 spin_lock(&poll->head->lock);
5221 WRITE_ONCE(poll->canceled, true);
5222 if (!list_empty(&poll->wait.entry)) {
5223 list_del_init(&poll->wait.entry);
5226 spin_unlock(&poll->head->lock);
5227 hash_del(&req->hash_node);
5231 static bool io_poll_remove_waitqs(struct io_kiocb *req)
5232 __must_hold(&req->ctx->completion_lock)
5236 io_poll_remove_double(req);
5237 do_complete = __io_poll_remove_one(req, io_poll_get_single(req), true);
5239 if (req->opcode != IORING_OP_POLL_ADD && do_complete) {
5240 /* non-poll requests have submit ref still */
5246 static bool io_poll_remove_one(struct io_kiocb *req)
5247 __must_hold(&req->ctx->completion_lock)
5251 do_complete = io_poll_remove_waitqs(req);
5253 io_cqring_fill_event(req->ctx, req->user_data, -ECANCELED, 0);
5254 io_commit_cqring(req->ctx);
5256 io_put_req_deferred(req, 1);
5263 * Returns true if we found and killed one or more poll requests
5265 static bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk,
5268 struct hlist_node *tmp;
5269 struct io_kiocb *req;
5272 spin_lock_irq(&ctx->completion_lock);
5273 for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
5274 struct hlist_head *list;
5276 list = &ctx->cancel_hash[i];
5277 hlist_for_each_entry_safe(req, tmp, list, hash_node) {
5278 if (io_match_task(req, tsk, cancel_all))
5279 posted += io_poll_remove_one(req);
5282 spin_unlock_irq(&ctx->completion_lock);
5285 io_cqring_ev_posted(ctx);
5290 static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, __u64 sqe_addr,
5292 __must_hold(&ctx->completion_lock)
5294 struct hlist_head *list;
5295 struct io_kiocb *req;
5297 list = &ctx->cancel_hash[hash_long(sqe_addr, ctx->cancel_hash_bits)];
5298 hlist_for_each_entry(req, list, hash_node) {
5299 if (sqe_addr != req->user_data)
5301 if (poll_only && req->opcode != IORING_OP_POLL_ADD)
5308 static int io_poll_cancel(struct io_ring_ctx *ctx, __u64 sqe_addr,
5310 __must_hold(&ctx->completion_lock)
5312 struct io_kiocb *req;
5314 req = io_poll_find(ctx, sqe_addr, poll_only);
5317 if (io_poll_remove_one(req))
5323 static __poll_t io_poll_parse_events(const struct io_uring_sqe *sqe,
5328 events = READ_ONCE(sqe->poll32_events);
5330 events = swahw32(events);
5332 if (!(flags & IORING_POLL_ADD_MULTI))
5333 events |= EPOLLONESHOT;
5334 return demangle_poll(events) | (events & (EPOLLEXCLUSIVE|EPOLLONESHOT));
5337 static int io_poll_update_prep(struct io_kiocb *req,
5338 const struct io_uring_sqe *sqe)
5340 struct io_poll_update *upd = &req->poll_update;
5343 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
5345 if (sqe->ioprio || sqe->buf_index)
5347 flags = READ_ONCE(sqe->len);
5348 if (flags & ~(IORING_POLL_UPDATE_EVENTS | IORING_POLL_UPDATE_USER_DATA |
5349 IORING_POLL_ADD_MULTI))
5351 /* meaningless without update */
5352 if (flags == IORING_POLL_ADD_MULTI)
5355 upd->old_user_data = READ_ONCE(sqe->addr);
5356 upd->update_events = flags & IORING_POLL_UPDATE_EVENTS;
5357 upd->update_user_data = flags & IORING_POLL_UPDATE_USER_DATA;
5359 upd->new_user_data = READ_ONCE(sqe->off);
5360 if (!upd->update_user_data && upd->new_user_data)
5362 if (upd->update_events)
5363 upd->events = io_poll_parse_events(sqe, flags);
5364 else if (sqe->poll32_events)
5370 static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
5373 struct io_kiocb *req = wait->private;
5374 struct io_poll_iocb *poll = &req->poll;
5376 return __io_async_wake(req, poll, key_to_poll(key), io_poll_task_func);
5379 static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head,
5380 struct poll_table_struct *p)
5382 struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
5384 __io_queue_proc(&pt->req->poll, pt, head, (struct io_poll_iocb **) &pt->req->async_data);
5387 static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
5389 struct io_poll_iocb *poll = &req->poll;
5392 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
5394 if (sqe->ioprio || sqe->buf_index || sqe->off || sqe->addr)
5396 flags = READ_ONCE(sqe->len);
5397 if (flags & ~IORING_POLL_ADD_MULTI)
5400 poll->events = io_poll_parse_events(sqe, flags);
5404 static int io_poll_add(struct io_kiocb *req, unsigned int issue_flags)
5406 struct io_poll_iocb *poll = &req->poll;
5407 struct io_ring_ctx *ctx = req->ctx;
5408 struct io_poll_table ipt;
5411 ipt.pt._qproc = io_poll_queue_proc;
5413 mask = __io_arm_poll_handler(req, &req->poll, &ipt, poll->events,
5416 if (mask) { /* no async, we'd stolen it */
5418 io_poll_complete(req, mask);
5420 spin_unlock_irq(&ctx->completion_lock);
5423 io_cqring_ev_posted(ctx);
5424 if (poll->events & EPOLLONESHOT)
5430 static int io_poll_update(struct io_kiocb *req, unsigned int issue_flags)
5432 struct io_ring_ctx *ctx = req->ctx;
5433 struct io_kiocb *preq;
5437 spin_lock_irq(&ctx->completion_lock);
5438 preq = io_poll_find(ctx, req->poll_update.old_user_data, true);
5444 if (!req->poll_update.update_events && !req->poll_update.update_user_data) {
5446 ret = io_poll_remove_one(preq) ? 0 : -EALREADY;
5451 * Don't allow racy completion with singleshot, as we cannot safely
5452 * update those. For multishot, if we're racing with completion, just
5453 * let completion re-add it.
5455 completing = !__io_poll_remove_one(preq, &preq->poll, false);
5456 if (completing && (preq->poll.events & EPOLLONESHOT)) {
5460 /* we now have a detached poll request. reissue. */
5464 spin_unlock_irq(&ctx->completion_lock);
5466 io_req_complete(req, ret);
5469 /* only mask one event flags, keep behavior flags */
5470 if (req->poll_update.update_events) {
5471 preq->poll.events &= ~0xffff;
5472 preq->poll.events |= req->poll_update.events & 0xffff;
5473 preq->poll.events |= IO_POLL_UNMASK;
5475 if (req->poll_update.update_user_data)
5476 preq->user_data = req->poll_update.new_user_data;
5477 spin_unlock_irq(&ctx->completion_lock);
5479 /* complete update request, we're done with it */
5480 io_req_complete(req, ret);
5483 ret = io_poll_add(preq, issue_flags);
5486 io_req_complete(preq, ret);
5492 static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer)
5494 struct io_timeout_data *data = container_of(timer,
5495 struct io_timeout_data, timer);
5496 struct io_kiocb *req = data->req;
5497 struct io_ring_ctx *ctx = req->ctx;
5498 unsigned long flags;
5500 spin_lock_irqsave(&ctx->completion_lock, flags);
5501 list_del_init(&req->timeout.list);
5502 atomic_set(&req->ctx->cq_timeouts,
5503 atomic_read(&req->ctx->cq_timeouts) + 1);
5505 io_cqring_fill_event(ctx, req->user_data, -ETIME, 0);
5506 io_commit_cqring(ctx);
5507 spin_unlock_irqrestore(&ctx->completion_lock, flags);
5509 io_cqring_ev_posted(ctx);
5512 return HRTIMER_NORESTART;
5515 static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx,
5517 __must_hold(&ctx->completion_lock)
5519 struct io_timeout_data *io;
5520 struct io_kiocb *req;
5523 list_for_each_entry(req, &ctx->timeout_list, timeout.list) {
5524 found = user_data == req->user_data;
5529 return ERR_PTR(-ENOENT);
5531 io = req->async_data;
5532 if (hrtimer_try_to_cancel(&io->timer) == -1)
5533 return ERR_PTR(-EALREADY);
5534 list_del_init(&req->timeout.list);
5538 static int io_timeout_cancel(struct io_ring_ctx *ctx, __u64 user_data)
5539 __must_hold(&ctx->completion_lock)
5541 struct io_kiocb *req = io_timeout_extract(ctx, user_data);
5544 return PTR_ERR(req);
5547 io_cqring_fill_event(ctx, req->user_data, -ECANCELED, 0);
5548 io_put_req_deferred(req, 1);
5552 static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,
5553 struct timespec64 *ts, enum hrtimer_mode mode)
5554 __must_hold(&ctx->completion_lock)
5556 struct io_kiocb *req = io_timeout_extract(ctx, user_data);
5557 struct io_timeout_data *data;
5560 return PTR_ERR(req);
5562 req->timeout.off = 0; /* noseq */
5563 data = req->async_data;
5564 list_add_tail(&req->timeout.list, &ctx->timeout_list);
5565 hrtimer_init(&data->timer, CLOCK_MONOTONIC, mode);
5566 data->timer.function = io_timeout_fn;
5567 hrtimer_start(&data->timer, timespec64_to_ktime(*ts), mode);
5571 static int io_timeout_remove_prep(struct io_kiocb *req,
5572 const struct io_uring_sqe *sqe)
5574 struct io_timeout_rem *tr = &req->timeout_rem;
5576 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
5578 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
5580 if (sqe->ioprio || sqe->buf_index || sqe->len)
5583 tr->addr = READ_ONCE(sqe->addr);
5584 tr->flags = READ_ONCE(sqe->timeout_flags);
5585 if (tr->flags & IORING_TIMEOUT_UPDATE) {
5586 if (tr->flags & ~(IORING_TIMEOUT_UPDATE|IORING_TIMEOUT_ABS))
5588 if (get_timespec64(&tr->ts, u64_to_user_ptr(sqe->addr2)))
5590 } else if (tr->flags) {
5591 /* timeout removal doesn't support flags */
5598 static inline enum hrtimer_mode io_translate_timeout_mode(unsigned int flags)
5600 return (flags & IORING_TIMEOUT_ABS) ? HRTIMER_MODE_ABS
5605 * Remove or update an existing timeout command
5607 static int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags)
5609 struct io_timeout_rem *tr = &req->timeout_rem;
5610 struct io_ring_ctx *ctx = req->ctx;
5613 spin_lock_irq(&ctx->completion_lock);
5614 if (!(req->timeout_rem.flags & IORING_TIMEOUT_UPDATE))
5615 ret = io_timeout_cancel(ctx, tr->addr);
5617 ret = io_timeout_update(ctx, tr->addr, &tr->ts,
5618 io_translate_timeout_mode(tr->flags));
5620 io_cqring_fill_event(ctx, req->user_data, ret, 0);
5621 io_commit_cqring(ctx);
5622 spin_unlock_irq(&ctx->completion_lock);
5623 io_cqring_ev_posted(ctx);
5630 static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
5631 bool is_timeout_link)
5633 struct io_timeout_data *data;
5635 u32 off = READ_ONCE(sqe->off);
5637 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
5639 if (sqe->ioprio || sqe->buf_index || sqe->len != 1)
5641 if (off && is_timeout_link)
5643 flags = READ_ONCE(sqe->timeout_flags);
5644 if (flags & ~IORING_TIMEOUT_ABS)
5647 req->timeout.off = off;
5648 if (unlikely(off && !req->ctx->off_timeout_used))
5649 req->ctx->off_timeout_used = true;
5651 if (!req->async_data && io_alloc_async_data(req))
5654 data = req->async_data;
5657 if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr)))
5660 data->mode = io_translate_timeout_mode(flags);
5661 hrtimer_init(&data->timer, CLOCK_MONOTONIC, data->mode);
5662 if (is_timeout_link)
5663 io_req_track_inflight(req);
5667 static int io_timeout(struct io_kiocb *req, unsigned int issue_flags)
5669 struct io_ring_ctx *ctx = req->ctx;
5670 struct io_timeout_data *data = req->async_data;
5671 struct list_head *entry;
5672 u32 tail, off = req->timeout.off;
5674 spin_lock_irq(&ctx->completion_lock);
5677 * sqe->off holds how many events that need to occur for this
5678 * timeout event to be satisfied. If it isn't set, then this is
5679 * a pure timeout request, sequence isn't used.
5681 if (io_is_timeout_noseq(req)) {
5682 entry = ctx->timeout_list.prev;
5686 tail = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts);
5687 req->timeout.target_seq = tail + off;
5689 /* Update the last seq here in case io_flush_timeouts() hasn't.
5690 * This is safe because ->completion_lock is held, and submissions
5691 * and completions are never mixed in the same ->completion_lock section.
5693 ctx->cq_last_tm_flush = tail;
5696 * Insertion sort, ensuring the first entry in the list is always
5697 * the one we need first.
5699 list_for_each_prev(entry, &ctx->timeout_list) {
5700 struct io_kiocb *nxt = list_entry(entry, struct io_kiocb,
5703 if (io_is_timeout_noseq(nxt))
5705 /* nxt.seq is behind @tail, otherwise would've been completed */
5706 if (off >= nxt->timeout.target_seq - tail)
5710 list_add(&req->timeout.list, entry);
5711 data->timer.function = io_timeout_fn;
5712 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode);
5713 spin_unlock_irq(&ctx->completion_lock);
5717 struct io_cancel_data {
5718 struct io_ring_ctx *ctx;
5722 static bool io_cancel_cb(struct io_wq_work *work, void *data)
5724 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
5725 struct io_cancel_data *cd = data;
5727 return req->ctx == cd->ctx && req->user_data == cd->user_data;
5730 static int io_async_cancel_one(struct io_uring_task *tctx, u64 user_data,
5731 struct io_ring_ctx *ctx)
5733 struct io_cancel_data data = { .ctx = ctx, .user_data = user_data, };
5734 enum io_wq_cancel cancel_ret;
5737 if (!tctx || !tctx->io_wq)
5740 cancel_ret = io_wq_cancel_cb(tctx->io_wq, io_cancel_cb, &data, false);
5741 switch (cancel_ret) {
5742 case IO_WQ_CANCEL_OK:
5745 case IO_WQ_CANCEL_RUNNING:
5748 case IO_WQ_CANCEL_NOTFOUND:
5756 static void io_async_find_and_cancel(struct io_ring_ctx *ctx,
5757 struct io_kiocb *req, __u64 sqe_addr,
5760 unsigned long flags;
5763 ret = io_async_cancel_one(req->task->io_uring, sqe_addr, ctx);
5764 spin_lock_irqsave(&ctx->completion_lock, flags);
5767 ret = io_timeout_cancel(ctx, sqe_addr);
5770 ret = io_poll_cancel(ctx, sqe_addr, false);
5774 io_cqring_fill_event(ctx, req->user_data, ret, 0);
5775 io_commit_cqring(ctx);
5776 spin_unlock_irqrestore(&ctx->completion_lock, flags);
5777 io_cqring_ev_posted(ctx);
5783 static int io_async_cancel_prep(struct io_kiocb *req,
5784 const struct io_uring_sqe *sqe)
5786 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
5788 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
5790 if (sqe->ioprio || sqe->off || sqe->len || sqe->cancel_flags)
5793 req->cancel.addr = READ_ONCE(sqe->addr);
5797 static int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags)
5799 struct io_ring_ctx *ctx = req->ctx;
5800 u64 sqe_addr = req->cancel.addr;
5801 struct io_tctx_node *node;
5804 /* tasks should wait for their io-wq threads, so safe w/o sync */
5805 ret = io_async_cancel_one(req->task->io_uring, sqe_addr, ctx);
5806 spin_lock_irq(&ctx->completion_lock);
5809 ret = io_timeout_cancel(ctx, sqe_addr);
5812 ret = io_poll_cancel(ctx, sqe_addr, false);
5815 spin_unlock_irq(&ctx->completion_lock);
5817 /* slow path, try all io-wq's */
5818 io_ring_submit_lock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));
5820 list_for_each_entry(node, &ctx->tctx_list, ctx_node) {
5821 struct io_uring_task *tctx = node->task->io_uring;
5823 ret = io_async_cancel_one(tctx, req->cancel.addr, ctx);
5827 io_ring_submit_unlock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));
5829 spin_lock_irq(&ctx->completion_lock);
5831 io_cqring_fill_event(ctx, req->user_data, ret, 0);
5832 io_commit_cqring(ctx);
5833 spin_unlock_irq(&ctx->completion_lock);
5834 io_cqring_ev_posted(ctx);
5842 static int io_rsrc_update_prep(struct io_kiocb *req,
5843 const struct io_uring_sqe *sqe)
5845 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
5847 if (sqe->ioprio || sqe->rw_flags)
5850 req->rsrc_update.offset = READ_ONCE(sqe->off);
5851 req->rsrc_update.nr_args = READ_ONCE(sqe->len);
5852 if (!req->rsrc_update.nr_args)
5854 req->rsrc_update.arg = READ_ONCE(sqe->addr);
5858 static int io_files_update(struct io_kiocb *req, unsigned int issue_flags)
5860 struct io_ring_ctx *ctx = req->ctx;
5861 struct io_uring_rsrc_update2 up;
5864 if (issue_flags & IO_URING_F_NONBLOCK)
5867 up.offset = req->rsrc_update.offset;
5868 up.data = req->rsrc_update.arg;
5873 mutex_lock(&ctx->uring_lock);
5874 ret = __io_register_rsrc_update(ctx, IORING_RSRC_FILE,
5875 &up, req->rsrc_update.nr_args);
5876 mutex_unlock(&ctx->uring_lock);
5880 __io_req_complete(req, issue_flags, ret, 0);
5884 static int io_req_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
5886 switch (req->opcode) {
5889 case IORING_OP_READV:
5890 case IORING_OP_READ_FIXED:
5891 case IORING_OP_READ:
5892 return io_read_prep(req, sqe);
5893 case IORING_OP_WRITEV:
5894 case IORING_OP_WRITE_FIXED:
5895 case IORING_OP_WRITE:
5896 return io_write_prep(req, sqe);
5897 case IORING_OP_POLL_ADD:
5898 return io_poll_add_prep(req, sqe);
5899 case IORING_OP_POLL_REMOVE:
5900 return io_poll_update_prep(req, sqe);
5901 case IORING_OP_FSYNC:
5902 return io_fsync_prep(req, sqe);
5903 case IORING_OP_SYNC_FILE_RANGE:
5904 return io_sfr_prep(req, sqe);
5905 case IORING_OP_SENDMSG:
5906 case IORING_OP_SEND:
5907 return io_sendmsg_prep(req, sqe);
5908 case IORING_OP_RECVMSG:
5909 case IORING_OP_RECV:
5910 return io_recvmsg_prep(req, sqe);
5911 case IORING_OP_CONNECT:
5912 return io_connect_prep(req, sqe);
5913 case IORING_OP_TIMEOUT:
5914 return io_timeout_prep(req, sqe, false);
5915 case IORING_OP_TIMEOUT_REMOVE:
5916 return io_timeout_remove_prep(req, sqe);
5917 case IORING_OP_ASYNC_CANCEL:
5918 return io_async_cancel_prep(req, sqe);
5919 case IORING_OP_LINK_TIMEOUT:
5920 return io_timeout_prep(req, sqe, true);
5921 case IORING_OP_ACCEPT:
5922 return io_accept_prep(req, sqe);
5923 case IORING_OP_FALLOCATE:
5924 return io_fallocate_prep(req, sqe);
5925 case IORING_OP_OPENAT:
5926 return io_openat_prep(req, sqe);
5927 case IORING_OP_CLOSE:
5928 return io_close_prep(req, sqe);
5929 case IORING_OP_FILES_UPDATE:
5930 return io_rsrc_update_prep(req, sqe);
5931 case IORING_OP_STATX:
5932 return io_statx_prep(req, sqe);
5933 case IORING_OP_FADVISE:
5934 return io_fadvise_prep(req, sqe);
5935 case IORING_OP_MADVISE:
5936 return io_madvise_prep(req, sqe);
5937 case IORING_OP_OPENAT2:
5938 return io_openat2_prep(req, sqe);
5939 case IORING_OP_EPOLL_CTL:
5940 return io_epoll_ctl_prep(req, sqe);
5941 case IORING_OP_SPLICE:
5942 return io_splice_prep(req, sqe);
5943 case IORING_OP_PROVIDE_BUFFERS:
5944 return io_provide_buffers_prep(req, sqe);
5945 case IORING_OP_REMOVE_BUFFERS:
5946 return io_remove_buffers_prep(req, sqe);
5948 return io_tee_prep(req, sqe);
5949 case IORING_OP_SHUTDOWN:
5950 return io_shutdown_prep(req, sqe);
5951 case IORING_OP_RENAMEAT:
5952 return io_renameat_prep(req, sqe);
5953 case IORING_OP_UNLINKAT:
5954 return io_unlinkat_prep(req, sqe);
5957 printk_once(KERN_WARNING "io_uring: unhandled opcode %d\n",
5962 static int io_req_prep_async(struct io_kiocb *req)
5964 if (!io_op_defs[req->opcode].needs_async_setup)
5966 if (WARN_ON_ONCE(req->async_data))
5968 if (io_alloc_async_data(req))
5971 switch (req->opcode) {
5972 case IORING_OP_READV:
5973 return io_rw_prep_async(req, READ);
5974 case IORING_OP_WRITEV:
5975 return io_rw_prep_async(req, WRITE);
5976 case IORING_OP_SENDMSG:
5977 return io_sendmsg_prep_async(req);
5978 case IORING_OP_RECVMSG:
5979 return io_recvmsg_prep_async(req);
5980 case IORING_OP_CONNECT:
5981 return io_connect_prep_async(req);
5983 printk_once(KERN_WARNING "io_uring: prep_async() bad opcode %d\n",
5988 static u32 io_get_sequence(struct io_kiocb *req)
5990 u32 seq = req->ctx->cached_sq_head;
5992 /* need original cached_sq_head, but it was increased for each req */
5993 io_for_each_link(req, req)
5998 static bool io_drain_req(struct io_kiocb *req)
6000 struct io_kiocb *pos;
6001 struct io_ring_ctx *ctx = req->ctx;
6002 struct io_defer_entry *de;
6007 * If we need to drain a request in the middle of a link, drain the
6008 * head request and the next request/link after the current link.
6009 * Considering sequential execution of links, IOSQE_IO_DRAIN will be
6010 * maintained for every request of our link.
6012 if (ctx->drain_next) {
6013 req->flags |= REQ_F_IO_DRAIN;
6014 ctx->drain_next = false;
6016 /* not interested in head, start from the first linked */
6017 io_for_each_link(pos, req->link) {
6018 if (pos->flags & REQ_F_IO_DRAIN) {
6019 ctx->drain_next = true;
6020 req->flags |= REQ_F_IO_DRAIN;
6025 /* Still need defer if there is pending req in defer list. */
6026 if (likely(list_empty_careful(&ctx->defer_list) &&
6027 !(req->flags & REQ_F_IO_DRAIN))) {
6028 ctx->drain_active = false;
6032 seq = io_get_sequence(req);
6033 /* Still a chance to pass the sequence check */
6034 if (!req_need_defer(req, seq) && list_empty_careful(&ctx->defer_list))
6037 ret = io_req_prep_async(req);
6040 io_prep_async_link(req);
6041 de = kmalloc(sizeof(*de), GFP_KERNEL);
6045 io_req_complete_failed(req, ret);
6049 spin_lock_irq(&ctx->completion_lock);
6050 if (!req_need_defer(req, seq) && list_empty(&ctx->defer_list)) {
6051 spin_unlock_irq(&ctx->completion_lock);
6053 io_queue_async_work(req);
6057 trace_io_uring_defer(ctx, req, req->user_data);
6060 list_add_tail(&de->list, &ctx->defer_list);
6061 spin_unlock_irq(&ctx->completion_lock);
6065 static void io_clean_op(struct io_kiocb *req)
6067 if (req->flags & REQ_F_BUFFER_SELECTED) {
6068 switch (req->opcode) {
6069 case IORING_OP_READV:
6070 case IORING_OP_READ_FIXED:
6071 case IORING_OP_READ:
6072 kfree((void *)(unsigned long)req->rw.addr);
6074 case IORING_OP_RECVMSG:
6075 case IORING_OP_RECV:
6076 kfree(req->sr_msg.kbuf);
6081 if (req->flags & REQ_F_NEED_CLEANUP) {
6082 switch (req->opcode) {
6083 case IORING_OP_READV:
6084 case IORING_OP_READ_FIXED:
6085 case IORING_OP_READ:
6086 case IORING_OP_WRITEV:
6087 case IORING_OP_WRITE_FIXED:
6088 case IORING_OP_WRITE: {
6089 struct io_async_rw *io = req->async_data;
6091 kfree(io->free_iovec);
6094 case IORING_OP_RECVMSG:
6095 case IORING_OP_SENDMSG: {
6096 struct io_async_msghdr *io = req->async_data;
6098 kfree(io->free_iov);
6101 case IORING_OP_SPLICE:
6103 if (!(req->splice.flags & SPLICE_F_FD_IN_FIXED))
6104 io_put_file(req->splice.file_in);
6106 case IORING_OP_OPENAT:
6107 case IORING_OP_OPENAT2:
6108 if (req->open.filename)
6109 putname(req->open.filename);
6111 case IORING_OP_RENAMEAT:
6112 putname(req->rename.oldpath);
6113 putname(req->rename.newpath);
6115 case IORING_OP_UNLINKAT:
6116 putname(req->unlink.filename);
6120 if ((req->flags & REQ_F_POLLED) && req->apoll) {
6121 kfree(req->apoll->double_poll);
6125 if (req->flags & REQ_F_INFLIGHT) {
6126 struct io_uring_task *tctx = req->task->io_uring;
6128 atomic_dec(&tctx->inflight_tracked);
6130 if (req->flags & REQ_F_CREDS)
6131 put_cred(req->creds);
6133 req->flags &= ~IO_REQ_CLEAN_FLAGS;
6136 static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)
6138 struct io_ring_ctx *ctx = req->ctx;
6139 const struct cred *creds = NULL;
6142 if ((req->flags & REQ_F_CREDS) && req->creds != current_cred())
6143 creds = override_creds(req->creds);
6145 switch (req->opcode) {
6147 ret = io_nop(req, issue_flags);
6149 case IORING_OP_READV:
6150 case IORING_OP_READ_FIXED:
6151 case IORING_OP_READ:
6152 ret = io_read(req, issue_flags);
6154 case IORING_OP_WRITEV:
6155 case IORING_OP_WRITE_FIXED:
6156 case IORING_OP_WRITE:
6157 ret = io_write(req, issue_flags);
6159 case IORING_OP_FSYNC:
6160 ret = io_fsync(req, issue_flags);
6162 case IORING_OP_POLL_ADD:
6163 ret = io_poll_add(req, issue_flags);
6165 case IORING_OP_POLL_REMOVE:
6166 ret = io_poll_update(req, issue_flags);
6168 case IORING_OP_SYNC_FILE_RANGE:
6169 ret = io_sync_file_range(req, issue_flags);
6171 case IORING_OP_SENDMSG:
6172 ret = io_sendmsg(req, issue_flags);
6174 case IORING_OP_SEND:
6175 ret = io_send(req, issue_flags);
6177 case IORING_OP_RECVMSG:
6178 ret = io_recvmsg(req, issue_flags);
6180 case IORING_OP_RECV:
6181 ret = io_recv(req, issue_flags);
6183 case IORING_OP_TIMEOUT:
6184 ret = io_timeout(req, issue_flags);
6186 case IORING_OP_TIMEOUT_REMOVE:
6187 ret = io_timeout_remove(req, issue_flags);
6189 case IORING_OP_ACCEPT:
6190 ret = io_accept(req, issue_flags);
6192 case IORING_OP_CONNECT:
6193 ret = io_connect(req, issue_flags);
6195 case IORING_OP_ASYNC_CANCEL:
6196 ret = io_async_cancel(req, issue_flags);
6198 case IORING_OP_FALLOCATE:
6199 ret = io_fallocate(req, issue_flags);
6201 case IORING_OP_OPENAT:
6202 ret = io_openat(req, issue_flags);
6204 case IORING_OP_CLOSE:
6205 ret = io_close(req, issue_flags);
6207 case IORING_OP_FILES_UPDATE:
6208 ret = io_files_update(req, issue_flags);
6210 case IORING_OP_STATX:
6211 ret = io_statx(req, issue_flags);
6213 case IORING_OP_FADVISE:
6214 ret = io_fadvise(req, issue_flags);
6216 case IORING_OP_MADVISE:
6217 ret = io_madvise(req, issue_flags);
6219 case IORING_OP_OPENAT2:
6220 ret = io_openat2(req, issue_flags);
6222 case IORING_OP_EPOLL_CTL:
6223 ret = io_epoll_ctl(req, issue_flags);
6225 case IORING_OP_SPLICE:
6226 ret = io_splice(req, issue_flags);
6228 case IORING_OP_PROVIDE_BUFFERS:
6229 ret = io_provide_buffers(req, issue_flags);
6231 case IORING_OP_REMOVE_BUFFERS:
6232 ret = io_remove_buffers(req, issue_flags);
6235 ret = io_tee(req, issue_flags);
6237 case IORING_OP_SHUTDOWN:
6238 ret = io_shutdown(req, issue_flags);
6240 case IORING_OP_RENAMEAT:
6241 ret = io_renameat(req, issue_flags);
6243 case IORING_OP_UNLINKAT:
6244 ret = io_unlinkat(req, issue_flags);
6252 revert_creds(creds);
6255 /* If the op doesn't have a file, we're not polling for it */
6256 if ((ctx->flags & IORING_SETUP_IOPOLL) && req->file)
6257 io_iopoll_req_issued(req);
6262 static void io_wq_submit_work(struct io_wq_work *work)
6264 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
6265 struct io_kiocb *timeout;
6268 timeout = io_prep_linked_timeout(req);
6270 io_queue_linked_timeout(timeout);
6272 if (work->flags & IO_WQ_WORK_CANCEL)
6277 ret = io_issue_sqe(req, 0);
6279 * We can get EAGAIN for polled IO even though we're
6280 * forcing a sync submission from here, since we can't
6281 * wait for request slots on the block side.
6289 /* avoid locking problems by failing it from a clean context */
6291 /* io-wq is going to take one down */
6293 io_req_task_queue_fail(req, ret);
6297 #define FFS_ASYNC_READ 0x1UL
6298 #define FFS_ASYNC_WRITE 0x2UL
6300 #define FFS_ISREG 0x4UL
6302 #define FFS_ISREG 0x0UL
6304 #define FFS_MASK ~(FFS_ASYNC_READ|FFS_ASYNC_WRITE|FFS_ISREG)
6306 static inline struct io_fixed_file *io_fixed_file_slot(struct io_file_table *table,
6309 struct io_fixed_file *table_l2;
6311 table_l2 = table->files[i >> IORING_FILE_TABLE_SHIFT];
6312 return &table_l2[i & IORING_FILE_TABLE_MASK];
6315 static inline struct file *io_file_from_index(struct io_ring_ctx *ctx,
6318 struct io_fixed_file *slot = io_fixed_file_slot(&ctx->file_table, index);
6320 return (struct file *) (slot->file_ptr & FFS_MASK);
6323 static void io_fixed_file_set(struct io_fixed_file *file_slot, struct file *file)
6325 unsigned long file_ptr = (unsigned long) file;
6327 if (__io_file_supports_async(file, READ))
6328 file_ptr |= FFS_ASYNC_READ;
6329 if (__io_file_supports_async(file, WRITE))
6330 file_ptr |= FFS_ASYNC_WRITE;
6331 if (S_ISREG(file_inode(file)->i_mode))
6332 file_ptr |= FFS_ISREG;
6333 file_slot->file_ptr = file_ptr;
6336 static struct file *io_file_get(struct io_submit_state *state,
6337 struct io_kiocb *req, int fd, bool fixed)
6339 struct io_ring_ctx *ctx = req->ctx;
6343 unsigned long file_ptr;
6345 if (unlikely((unsigned int)fd >= ctx->nr_user_files))
6347 fd = array_index_nospec(fd, ctx->nr_user_files);
6348 file_ptr = io_fixed_file_slot(&ctx->file_table, fd)->file_ptr;
6349 file = (struct file *) (file_ptr & FFS_MASK);
6350 file_ptr &= ~FFS_MASK;
6351 /* mask in overlapping REQ_F and FFS bits */
6352 req->flags |= (file_ptr << REQ_F_ASYNC_READ_BIT);
6353 io_req_set_rsrc_node(req);
6355 trace_io_uring_file_get(ctx, fd);
6356 file = __io_file_get(state, fd);
6358 /* we don't allow fixed io_uring files */
6359 if (file && unlikely(file->f_op == &io_uring_fops))
6360 io_req_track_inflight(req);
6366 static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
6368 struct io_timeout_data *data = container_of(timer,
6369 struct io_timeout_data, timer);
6370 struct io_kiocb *prev, *req = data->req;
6371 struct io_ring_ctx *ctx = req->ctx;
6372 unsigned long flags;
6374 spin_lock_irqsave(&ctx->completion_lock, flags);
6375 prev = req->timeout.head;
6376 req->timeout.head = NULL;
6379 * We don't expect the list to be empty, that will only happen if we
6380 * race with the completion of the linked work.
6383 io_remove_next_linked(prev);
6384 if (!req_ref_inc_not_zero(prev))
6387 spin_unlock_irqrestore(&ctx->completion_lock, flags);
6390 io_async_find_and_cancel(ctx, req, prev->user_data, -ETIME);
6391 io_put_req_deferred(prev, 1);
6392 io_put_req_deferred(req, 1);
6394 io_req_complete_post(req, -ETIME, 0);
6396 return HRTIMER_NORESTART;
6399 static void io_queue_linked_timeout(struct io_kiocb *req)
6401 struct io_ring_ctx *ctx = req->ctx;
6403 spin_lock_irq(&ctx->completion_lock);
6405 * If the back reference is NULL, then our linked request finished
6406 * before we got a chance to setup the timer
6408 if (req->timeout.head) {
6409 struct io_timeout_data *data = req->async_data;
6411 data->timer.function = io_link_timeout_fn;
6412 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts),
6415 spin_unlock_irq(&ctx->completion_lock);
6416 /* drop submission reference */
6420 static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req)
6422 struct io_kiocb *nxt = req->link;
6424 if (!nxt || (req->flags & REQ_F_LINK_TIMEOUT) ||
6425 nxt->opcode != IORING_OP_LINK_TIMEOUT)
6428 nxt->timeout.head = req;
6429 nxt->flags |= REQ_F_LTIMEOUT_ACTIVE;
6430 req->flags |= REQ_F_LINK_TIMEOUT;
6434 static void __io_queue_sqe(struct io_kiocb *req)
6436 struct io_kiocb *linked_timeout = io_prep_linked_timeout(req);
6440 ret = io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_COMPLETE_DEFER);
6443 * We async punt it if the file wasn't marked NOWAIT, or if the file
6444 * doesn't support non-blocking read/write attempts
6447 /* drop submission reference */
6448 if (req->flags & REQ_F_COMPLETE_INLINE) {
6449 struct io_ring_ctx *ctx = req->ctx;
6450 struct io_comp_state *cs = &ctx->submit_state.comp;
6452 cs->reqs[cs->nr++] = req;
6453 if (cs->nr == ARRAY_SIZE(cs->reqs))
6454 io_submit_flush_completions(ctx);
6458 } else if (ret == -EAGAIN && !(req->flags & REQ_F_NOWAIT)) {
6459 switch (io_arm_poll_handler(req)) {
6460 case IO_APOLL_READY:
6462 case IO_APOLL_ABORTED:
6464 * Queued up for async execution, worker will release
6465 * submit reference when the iocb is actually submitted.
6467 io_queue_async_work(req);
6471 io_req_complete_failed(req, ret);
6474 io_queue_linked_timeout(linked_timeout);
6477 static inline void io_queue_sqe(struct io_kiocb *req)
6479 if (unlikely(req->ctx->drain_active) && io_drain_req(req))
6482 if (likely(!(req->flags & REQ_F_FORCE_ASYNC))) {
6483 __io_queue_sqe(req);
6485 int ret = io_req_prep_async(req);
6488 io_req_complete_failed(req, ret);
6490 io_queue_async_work(req);
6495 * Check SQE restrictions (opcode and flags).
6497 * Returns 'true' if SQE is allowed, 'false' otherwise.
6499 static inline bool io_check_restriction(struct io_ring_ctx *ctx,
6500 struct io_kiocb *req,
6501 unsigned int sqe_flags)
6503 if (likely(!ctx->restricted))
6506 if (!test_bit(req->opcode, ctx->restrictions.sqe_op))
6509 if ((sqe_flags & ctx->restrictions.sqe_flags_required) !=
6510 ctx->restrictions.sqe_flags_required)
6513 if (sqe_flags & ~(ctx->restrictions.sqe_flags_allowed |
6514 ctx->restrictions.sqe_flags_required))
6520 static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
6521 const struct io_uring_sqe *sqe)
6523 struct io_submit_state *state;
6524 unsigned int sqe_flags;
6525 int personality, ret = 0;
6527 req->opcode = READ_ONCE(sqe->opcode);
6528 /* same numerical values with corresponding REQ_F_*, safe to copy */
6529 req->flags = sqe_flags = READ_ONCE(sqe->flags);
6530 req->user_data = READ_ONCE(sqe->user_data);
6532 req->fixed_rsrc_refs = NULL;
6533 /* one is dropped after submission, the other at completion */
6534 atomic_set(&req->refs, 2);
6535 req->task = current;
6537 /* enforce forwards compatibility on users */
6538 if (unlikely(sqe_flags & ~SQE_VALID_FLAGS))
6540 if (unlikely(req->opcode >= IORING_OP_LAST))
6542 if (!io_check_restriction(ctx, req, sqe_flags))
6545 if ((sqe_flags & IOSQE_BUFFER_SELECT) &&
6546 !io_op_defs[req->opcode].buffer_select)
6548 if (unlikely(sqe_flags & IOSQE_IO_DRAIN))
6549 ctx->drain_active = true;
6551 personality = READ_ONCE(sqe->personality);
6553 req->creds = xa_load(&ctx->personalities, personality);
6556 get_cred(req->creds);
6557 req->flags |= REQ_F_CREDS;
6559 state = &ctx->submit_state;
6562 * Plug now if we have more than 1 IO left after this, and the target
6563 * is potentially a read/write to block based storage.
6565 if (!state->plug_started && state->ios_left > 1 &&
6566 io_op_defs[req->opcode].plug) {
6567 blk_start_plug(&state->plug);
6568 state->plug_started = true;
6571 if (io_op_defs[req->opcode].needs_file) {
6572 bool fixed = req->flags & REQ_F_FIXED_FILE;
6574 req->file = io_file_get(state, req, READ_ONCE(sqe->fd), fixed);
6575 if (unlikely(!req->file))
6583 static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
6584 const struct io_uring_sqe *sqe)
6586 struct io_submit_link *link = &ctx->submit_state.link;
6589 ret = io_init_req(ctx, req, sqe);
6590 if (unlikely(ret)) {
6593 /* fail even hard links since we don't submit */
6594 req_set_fail(link->head);
6595 io_req_complete_failed(link->head, -ECANCELED);
6598 io_req_complete_failed(req, ret);
6602 ret = io_req_prep(req, sqe);
6606 /* don't need @sqe from now on */
6607 trace_io_uring_submit_sqe(ctx, req, req->opcode, req->user_data,
6609 ctx->flags & IORING_SETUP_SQPOLL);
6612 * If we already have a head request, queue this one for async
6613 * submittal once the head completes. If we don't have a head but
6614 * IOSQE_IO_LINK is set in the sqe, start a new head. This one will be
6615 * submitted sync once the chain is complete. If none of those
6616 * conditions are true (normal request), then just queue it.
6619 struct io_kiocb *head = link->head;
6621 ret = io_req_prep_async(req);
6624 trace_io_uring_link(ctx, req, head);
6625 link->last->link = req;
6628 /* last request of a link, enqueue the link */
6629 if (!(req->flags & (REQ_F_LINK | REQ_F_HARDLINK))) {
6634 if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) {
6646 * Batched submission is done, ensure local IO is flushed out.
6648 static void io_submit_state_end(struct io_submit_state *state,
6649 struct io_ring_ctx *ctx)
6651 if (state->link.head)
6652 io_queue_sqe(state->link.head);
6654 io_submit_flush_completions(ctx);
6655 if (state->plug_started)
6656 blk_finish_plug(&state->plug);
6657 io_state_file_put(state);
6661 * Start submission side cache.
6663 static void io_submit_state_start(struct io_submit_state *state,
6664 unsigned int max_ios)
6666 state->plug_started = false;
6667 state->ios_left = max_ios;
6668 /* set only head, no need to init link_last in advance */
6669 state->link.head = NULL;
6672 static void io_commit_sqring(struct io_ring_ctx *ctx)
6674 struct io_rings *rings = ctx->rings;
6677 * Ensure any loads from the SQEs are done at this point,
6678 * since once we write the new head, the application could
6679 * write new data to them.
6681 smp_store_release(&rings->sq.head, ctx->cached_sq_head);
6685 * Fetch an sqe, if one is available. Note this returns a pointer to memory
6686 * that is mapped by userspace. This means that care needs to be taken to
6687 * ensure that reads are stable, as we cannot rely on userspace always
6688 * being a good citizen. If members of the sqe are validated and then later
6689 * used, it's important that those reads are done through READ_ONCE() to
6690 * prevent a re-load down the line.
6692 static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx)
6694 unsigned head, mask = ctx->sq_entries - 1;
6695 unsigned sq_idx = ctx->cached_sq_head++ & mask;
6698 * The cached sq head (or cq tail) serves two purposes:
6700 * 1) allows us to batch the cost of updating the user visible
6702 * 2) allows the kernel side to track the head on its own, even
6703 * though the application is the one updating it.
6705 head = READ_ONCE(ctx->sq_array[sq_idx]);
6706 if (likely(head < ctx->sq_entries))
6707 return &ctx->sq_sqes[head];
6709 /* drop invalid entries */
6711 WRITE_ONCE(ctx->rings->sq_dropped,
6712 READ_ONCE(ctx->rings->sq_dropped) + 1);
6716 static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)
6718 struct io_uring_task *tctx;
6721 /* make sure SQ entry isn't read before tail */
6722 nr = min3(nr, ctx->sq_entries, io_sqring_entries(ctx));
6723 if (!percpu_ref_tryget_many(&ctx->refs, nr))
6726 tctx = current->io_uring;
6727 tctx->cached_refs -= nr;
6728 if (unlikely(tctx->cached_refs < 0)) {
6729 unsigned int refill = -tctx->cached_refs + IO_TCTX_REFS_CACHE_NR;
6731 percpu_counter_add(&tctx->inflight, refill);
6732 refcount_add(refill, ¤t->usage);
6733 tctx->cached_refs += refill;
6735 io_submit_state_start(&ctx->submit_state, nr);
6737 while (submitted < nr) {
6738 const struct io_uring_sqe *sqe;
6739 struct io_kiocb *req;
6741 req = io_alloc_req(ctx);
6742 if (unlikely(!req)) {
6744 submitted = -EAGAIN;
6747 sqe = io_get_sqe(ctx);
6748 if (unlikely(!sqe)) {
6749 kmem_cache_free(req_cachep, req);
6752 /* will complete beyond this point, count as submitted */
6754 if (io_submit_sqe(ctx, req, sqe))
6758 if (unlikely(submitted != nr)) {
6759 int ref_used = (submitted == -EAGAIN) ? 0 : submitted;
6760 int unused = nr - ref_used;
6762 current->io_uring->cached_refs += unused;
6763 percpu_ref_put_many(&ctx->refs, unused);
6766 io_submit_state_end(&ctx->submit_state, ctx);
6767 /* Commit SQ ring head once we've consumed and submitted all SQEs */
6768 io_commit_sqring(ctx);
6773 static inline bool io_sqd_events_pending(struct io_sq_data *sqd)
6775 return READ_ONCE(sqd->state);
6778 static inline void io_ring_set_wakeup_flag(struct io_ring_ctx *ctx)
6780 /* Tell userspace we may need a wakeup call */
6781 spin_lock_irq(&ctx->completion_lock);
6782 ctx->rings->sq_flags |= IORING_SQ_NEED_WAKEUP;
6783 spin_unlock_irq(&ctx->completion_lock);
6786 static inline void io_ring_clear_wakeup_flag(struct io_ring_ctx *ctx)
6788 spin_lock_irq(&ctx->completion_lock);
6789 ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
6790 spin_unlock_irq(&ctx->completion_lock);
6793 static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries)
6795 unsigned int to_submit;
6798 to_submit = io_sqring_entries(ctx);
6799 /* if we're handling multiple rings, cap submit size for fairness */
6800 if (cap_entries && to_submit > IORING_SQPOLL_CAP_ENTRIES_VALUE)
6801 to_submit = IORING_SQPOLL_CAP_ENTRIES_VALUE;
6803 if (!list_empty(&ctx->iopoll_list) || to_submit) {
6804 unsigned nr_events = 0;
6805 const struct cred *creds = NULL;
6807 if (ctx->sq_creds != current_cred())
6808 creds = override_creds(ctx->sq_creds);
6810 mutex_lock(&ctx->uring_lock);
6811 if (!list_empty(&ctx->iopoll_list))
6812 io_do_iopoll(ctx, &nr_events, 0, true);
6815 * Don't submit if refs are dying, good for io_uring_register(),
6816 * but also it is relied upon by io_ring_exit_work()
6818 if (to_submit && likely(!percpu_ref_is_dying(&ctx->refs)) &&
6819 !(ctx->flags & IORING_SETUP_R_DISABLED))
6820 ret = io_submit_sqes(ctx, to_submit);
6821 mutex_unlock(&ctx->uring_lock);
6823 if (to_submit && wq_has_sleeper(&ctx->sqo_sq_wait))
6824 wake_up(&ctx->sqo_sq_wait);
6826 revert_creds(creds);
6832 static void io_sqd_update_thread_idle(struct io_sq_data *sqd)
6834 struct io_ring_ctx *ctx;
6835 unsigned sq_thread_idle = 0;
6837 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
6838 sq_thread_idle = max(sq_thread_idle, ctx->sq_thread_idle);
6839 sqd->sq_thread_idle = sq_thread_idle;
6842 static bool io_sqd_handle_event(struct io_sq_data *sqd)
6844 bool did_sig = false;
6845 struct ksignal ksig;
6847 if (test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state) ||
6848 signal_pending(current)) {
6849 mutex_unlock(&sqd->lock);
6850 if (signal_pending(current))
6851 did_sig = get_signal(&ksig);
6853 mutex_lock(&sqd->lock);
6855 return did_sig || test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state);
6858 static int io_sq_thread(void *data)
6860 struct io_sq_data *sqd = data;
6861 struct io_ring_ctx *ctx;
6862 unsigned long timeout = 0;
6863 char buf[TASK_COMM_LEN];
6866 snprintf(buf, sizeof(buf), "iou-sqp-%d", sqd->task_pid);
6867 set_task_comm(current, buf);
6869 if (sqd->sq_cpu != -1)
6870 set_cpus_allowed_ptr(current, cpumask_of(sqd->sq_cpu));
6872 set_cpus_allowed_ptr(current, cpu_online_mask);
6873 current->flags |= PF_NO_SETAFFINITY;
6875 mutex_lock(&sqd->lock);
6877 bool cap_entries, sqt_spin = false;
6879 if (io_sqd_events_pending(sqd) || signal_pending(current)) {
6880 if (io_sqd_handle_event(sqd))
6882 timeout = jiffies + sqd->sq_thread_idle;
6885 cap_entries = !list_is_singular(&sqd->ctx_list);
6886 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) {
6887 int ret = __io_sq_thread(ctx, cap_entries);
6889 if (!sqt_spin && (ret > 0 || !list_empty(&ctx->iopoll_list)))
6892 if (io_run_task_work())
6895 if (sqt_spin || !time_after(jiffies, timeout)) {
6898 timeout = jiffies + sqd->sq_thread_idle;
6902 prepare_to_wait(&sqd->wait, &wait, TASK_INTERRUPTIBLE);
6903 if (!io_sqd_events_pending(sqd) && !current->task_works) {
6904 bool needs_sched = true;
6906 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) {
6907 io_ring_set_wakeup_flag(ctx);
6909 if ((ctx->flags & IORING_SETUP_IOPOLL) &&
6910 !list_empty_careful(&ctx->iopoll_list)) {
6911 needs_sched = false;
6914 if (io_sqring_entries(ctx)) {
6915 needs_sched = false;
6921 mutex_unlock(&sqd->lock);
6923 mutex_lock(&sqd->lock);
6925 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
6926 io_ring_clear_wakeup_flag(ctx);
6929 finish_wait(&sqd->wait, &wait);
6930 timeout = jiffies + sqd->sq_thread_idle;
6933 io_uring_cancel_generic(true, sqd);
6935 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
6936 io_ring_set_wakeup_flag(ctx);
6938 mutex_unlock(&sqd->lock);
6940 complete(&sqd->exited);
6944 struct io_wait_queue {
6945 struct wait_queue_entry wq;
6946 struct io_ring_ctx *ctx;
6948 unsigned nr_timeouts;
6951 static inline bool io_should_wake(struct io_wait_queue *iowq)
6953 struct io_ring_ctx *ctx = iowq->ctx;
6956 * Wake up if we have enough events, or if a timeout occurred since we
6957 * started waiting. For timeouts, we always want to return to userspace,
6958 * regardless of event count.
6960 return io_cqring_events(ctx) >= iowq->to_wait ||
6961 atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts;
6964 static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode,
6965 int wake_flags, void *key)
6967 struct io_wait_queue *iowq = container_of(curr, struct io_wait_queue,
6971 * Cannot safely flush overflowed CQEs from here, ensure we wake up
6972 * the task, and the next invocation will do it.
6974 if (io_should_wake(iowq) || test_bit(0, &iowq->ctx->check_cq_overflow))
6975 return autoremove_wake_function(curr, mode, wake_flags, key);
6979 static int io_run_task_work_sig(void)
6981 if (io_run_task_work())
6983 if (!signal_pending(current))
6985 if (test_thread_flag(TIF_NOTIFY_SIGNAL))
6986 return -ERESTARTSYS;
6990 /* when returns >0, the caller should retry */
6991 static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx,
6992 struct io_wait_queue *iowq,
6993 signed long *timeout)
6997 /* make sure we run task_work before checking for signals */
6998 ret = io_run_task_work_sig();
6999 if (ret || io_should_wake(iowq))
7001 /* let the caller flush overflows, retry */
7002 if (test_bit(0, &ctx->check_cq_overflow))
7005 *timeout = schedule_timeout(*timeout);
7006 return !*timeout ? -ETIME : 1;
7010 * Wait until events become available, if we don't already have some. The
7011 * application must reap them itself, as they reside on the shared cq ring.
7013 static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
7014 const sigset_t __user *sig, size_t sigsz,
7015 struct __kernel_timespec __user *uts)
7017 struct io_wait_queue iowq = {
7020 .func = io_wake_function,
7021 .entry = LIST_HEAD_INIT(iowq.wq.entry),
7024 .to_wait = min_events,
7026 struct io_rings *rings = ctx->rings;
7027 signed long timeout = MAX_SCHEDULE_TIMEOUT;
7031 io_cqring_overflow_flush(ctx, false);
7032 if (io_cqring_events(ctx) >= min_events)
7034 if (!io_run_task_work())
7039 #ifdef CONFIG_COMPAT
7040 if (in_compat_syscall())
7041 ret = set_compat_user_sigmask((const compat_sigset_t __user *)sig,
7045 ret = set_user_sigmask(sig, sigsz);
7052 struct timespec64 ts;
7054 if (get_timespec64(&ts, uts))
7056 timeout = timespec64_to_jiffies(&ts);
7059 iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts);
7060 trace_io_uring_cqring_wait(ctx, min_events);
7062 /* if we can't even flush overflow, don't wait for more */
7063 if (!io_cqring_overflow_flush(ctx, false)) {
7067 prepare_to_wait_exclusive(&ctx->cq_wait, &iowq.wq,
7068 TASK_INTERRUPTIBLE);
7069 ret = io_cqring_wait_schedule(ctx, &iowq, &timeout);
7070 finish_wait(&ctx->cq_wait, &iowq.wq);
7074 restore_saved_sigmask_unless(ret == -EINTR);
7076 return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0;
7079 static void io_free_page_table(void **table, size_t size)
7081 unsigned i, nr_tables = DIV_ROUND_UP(size, PAGE_SIZE);
7083 for (i = 0; i < nr_tables; i++)
7088 static void **io_alloc_page_table(size_t size)
7090 unsigned i, nr_tables = DIV_ROUND_UP(size, PAGE_SIZE);
7091 size_t init_size = size;
7094 table = kcalloc(nr_tables, sizeof(*table), GFP_KERNEL);
7098 for (i = 0; i < nr_tables; i++) {
7099 unsigned int this_size = min_t(size_t, size, PAGE_SIZE);
7101 table[i] = kzalloc(this_size, GFP_KERNEL);
7103 io_free_page_table(table, init_size);
7111 static inline void io_rsrc_ref_lock(struct io_ring_ctx *ctx)
7113 spin_lock_bh(&ctx->rsrc_ref_lock);
7116 static inline void io_rsrc_ref_unlock(struct io_ring_ctx *ctx)
7118 spin_unlock_bh(&ctx->rsrc_ref_lock);
7121 static void io_rsrc_node_destroy(struct io_rsrc_node *ref_node)
7123 percpu_ref_exit(&ref_node->refs);
7127 static void io_rsrc_node_switch(struct io_ring_ctx *ctx,
7128 struct io_rsrc_data *data_to_kill)
7130 WARN_ON_ONCE(!ctx->rsrc_backup_node);
7131 WARN_ON_ONCE(data_to_kill && !ctx->rsrc_node);
7134 struct io_rsrc_node *rsrc_node = ctx->rsrc_node;
7136 rsrc_node->rsrc_data = data_to_kill;
7137 io_rsrc_ref_lock(ctx);
7138 list_add_tail(&rsrc_node->node, &ctx->rsrc_ref_list);
7139 io_rsrc_ref_unlock(ctx);
7141 atomic_inc(&data_to_kill->refs);
7142 percpu_ref_kill(&rsrc_node->refs);
7143 ctx->rsrc_node = NULL;
7146 if (!ctx->rsrc_node) {
7147 ctx->rsrc_node = ctx->rsrc_backup_node;
7148 ctx->rsrc_backup_node = NULL;
7152 static int io_rsrc_node_switch_start(struct io_ring_ctx *ctx)
7154 if (ctx->rsrc_backup_node)
7156 ctx->rsrc_backup_node = io_rsrc_node_alloc(ctx);
7157 return ctx->rsrc_backup_node ? 0 : -ENOMEM;
7160 static int io_rsrc_ref_quiesce(struct io_rsrc_data *data, struct io_ring_ctx *ctx)
7164 /* As we may drop ->uring_lock, other task may have started quiesce */
7168 data->quiesce = true;
7170 ret = io_rsrc_node_switch_start(ctx);
7173 io_rsrc_node_switch(ctx, data);
7175 /* kill initial ref, already quiesced if zero */
7176 if (atomic_dec_and_test(&data->refs))
7178 flush_delayed_work(&ctx->rsrc_put_work);
7179 ret = wait_for_completion_interruptible(&data->done);
7183 atomic_inc(&data->refs);
7184 /* wait for all works potentially completing data->done */
7185 flush_delayed_work(&ctx->rsrc_put_work);
7186 reinit_completion(&data->done);
7188 mutex_unlock(&ctx->uring_lock);
7189 ret = io_run_task_work_sig();
7190 mutex_lock(&ctx->uring_lock);
7192 data->quiesce = false;
7197 static u64 *io_get_tag_slot(struct io_rsrc_data *data, unsigned int idx)
7199 unsigned int off = idx & IO_RSRC_TAG_TABLE_MASK;
7200 unsigned int table_idx = idx >> IO_RSRC_TAG_TABLE_SHIFT;
7202 return &data->tags[table_idx][off];
7205 static void io_rsrc_data_free(struct io_rsrc_data *data)
7207 size_t size = data->nr * sizeof(data->tags[0][0]);
7210 io_free_page_table((void **)data->tags, size);
7214 static int io_rsrc_data_alloc(struct io_ring_ctx *ctx, rsrc_put_fn *do_put,
7215 u64 __user *utags, unsigned nr,
7216 struct io_rsrc_data **pdata)
7218 struct io_rsrc_data *data;
7222 data = kzalloc(sizeof(*data), GFP_KERNEL);
7225 data->tags = (u64 **)io_alloc_page_table(nr * sizeof(data->tags[0][0]));
7233 data->do_put = do_put;
7236 for (i = 0; i < nr; i++) {
7237 u64 *tag_slot = io_get_tag_slot(data, i);
7239 if (copy_from_user(tag_slot, &utags[i],
7245 atomic_set(&data->refs, 1);
7246 init_completion(&data->done);
7250 io_rsrc_data_free(data);
7254 static bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files)
7256 size_t size = nr_files * sizeof(struct io_fixed_file);
7258 table->files = (struct io_fixed_file **)io_alloc_page_table(size);
7259 return !!table->files;
7262 static void io_free_file_tables(struct io_file_table *table, unsigned nr_files)
7264 size_t size = nr_files * sizeof(struct io_fixed_file);
7266 io_free_page_table((void **)table->files, size);
7267 table->files = NULL;
7270 static void __io_sqe_files_unregister(struct io_ring_ctx *ctx)
7272 #if defined(CONFIG_UNIX)
7273 if (ctx->ring_sock) {
7274 struct sock *sock = ctx->ring_sock->sk;
7275 struct sk_buff *skb;
7277 while ((skb = skb_dequeue(&sock->sk_receive_queue)) != NULL)
7283 for (i = 0; i < ctx->nr_user_files; i++) {
7286 file = io_file_from_index(ctx, i);
7291 io_free_file_tables(&ctx->file_table, ctx->nr_user_files);
7292 io_rsrc_data_free(ctx->file_data);
7293 ctx->file_data = NULL;
7294 ctx->nr_user_files = 0;
7297 static int io_sqe_files_unregister(struct io_ring_ctx *ctx)
7301 if (!ctx->file_data)
7303 ret = io_rsrc_ref_quiesce(ctx->file_data, ctx);
7305 __io_sqe_files_unregister(ctx);
7309 static void io_sq_thread_unpark(struct io_sq_data *sqd)
7310 __releases(&sqd->lock)
7312 WARN_ON_ONCE(sqd->thread == current);
7315 * Do the dance but not conditional clear_bit() because it'd race with
7316 * other threads incrementing park_pending and setting the bit.
7318 clear_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state);
7319 if (atomic_dec_return(&sqd->park_pending))
7320 set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state);
7321 mutex_unlock(&sqd->lock);
7324 static void io_sq_thread_park(struct io_sq_data *sqd)
7325 __acquires(&sqd->lock)
7327 WARN_ON_ONCE(sqd->thread == current);
7329 atomic_inc(&sqd->park_pending);
7330 set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state);
7331 mutex_lock(&sqd->lock);
7333 wake_up_process(sqd->thread);
7336 static void io_sq_thread_stop(struct io_sq_data *sqd)
7338 WARN_ON_ONCE(sqd->thread == current);
7339 WARN_ON_ONCE(test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state));
7341 set_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state);
7342 mutex_lock(&sqd->lock);
7344 wake_up_process(sqd->thread);
7345 mutex_unlock(&sqd->lock);
7346 wait_for_completion(&sqd->exited);
7349 static void io_put_sq_data(struct io_sq_data *sqd)
7351 if (refcount_dec_and_test(&sqd->refs)) {
7352 WARN_ON_ONCE(atomic_read(&sqd->park_pending));
7354 io_sq_thread_stop(sqd);
7359 static void io_sq_thread_finish(struct io_ring_ctx *ctx)
7361 struct io_sq_data *sqd = ctx->sq_data;
7364 io_sq_thread_park(sqd);
7365 list_del_init(&ctx->sqd_list);
7366 io_sqd_update_thread_idle(sqd);
7367 io_sq_thread_unpark(sqd);
7369 io_put_sq_data(sqd);
7370 ctx->sq_data = NULL;
7374 static struct io_sq_data *io_attach_sq_data(struct io_uring_params *p)
7376 struct io_ring_ctx *ctx_attach;
7377 struct io_sq_data *sqd;
7380 f = fdget(p->wq_fd);
7382 return ERR_PTR(-ENXIO);
7383 if (f.file->f_op != &io_uring_fops) {
7385 return ERR_PTR(-EINVAL);
7388 ctx_attach = f.file->private_data;
7389 sqd = ctx_attach->sq_data;
7392 return ERR_PTR(-EINVAL);
7394 if (sqd->task_tgid != current->tgid) {
7396 return ERR_PTR(-EPERM);
7399 refcount_inc(&sqd->refs);
7404 static struct io_sq_data *io_get_sq_data(struct io_uring_params *p,
7407 struct io_sq_data *sqd;
7410 if (p->flags & IORING_SETUP_ATTACH_WQ) {
7411 sqd = io_attach_sq_data(p);
7416 /* fall through for EPERM case, setup new sqd/task */
7417 if (PTR_ERR(sqd) != -EPERM)
7421 sqd = kzalloc(sizeof(*sqd), GFP_KERNEL);
7423 return ERR_PTR(-ENOMEM);
7425 atomic_set(&sqd->park_pending, 0);
7426 refcount_set(&sqd->refs, 1);
7427 INIT_LIST_HEAD(&sqd->ctx_list);
7428 mutex_init(&sqd->lock);
7429 init_waitqueue_head(&sqd->wait);
7430 init_completion(&sqd->exited);
7434 #if defined(CONFIG_UNIX)
7436 * Ensure the UNIX gc is aware of our file set, so we are certain that
7437 * the io_uring can be safely unregistered on process exit, even if we have
7438 * loops in the file referencing.
7440 static int __io_sqe_files_scm(struct io_ring_ctx *ctx, int nr, int offset)
7442 struct sock *sk = ctx->ring_sock->sk;
7443 struct scm_fp_list *fpl;
7444 struct sk_buff *skb;
7447 fpl = kzalloc(sizeof(*fpl), GFP_KERNEL);
7451 skb = alloc_skb(0, GFP_KERNEL);
7460 fpl->user = get_uid(current_user());
7461 for (i = 0; i < nr; i++) {
7462 struct file *file = io_file_from_index(ctx, i + offset);
7466 fpl->fp[nr_files] = get_file(file);
7467 unix_inflight(fpl->user, fpl->fp[nr_files]);
7472 fpl->max = SCM_MAX_FD;
7473 fpl->count = nr_files;
7474 UNIXCB(skb).fp = fpl;
7475 skb->destructor = unix_destruct_scm;
7476 refcount_add(skb->truesize, &sk->sk_wmem_alloc);
7477 skb_queue_head(&sk->sk_receive_queue, skb);
7479 for (i = 0; i < nr_files; i++)
7490 * If UNIX sockets are enabled, fd passing can cause a reference cycle which
7491 * causes regular reference counting to break down. We rely on the UNIX
7492 * garbage collection to take care of this problem for us.
7494 static int io_sqe_files_scm(struct io_ring_ctx *ctx)
7496 unsigned left, total;
7500 left = ctx->nr_user_files;
7502 unsigned this_files = min_t(unsigned, left, SCM_MAX_FD);
7504 ret = __io_sqe_files_scm(ctx, this_files, total);
7508 total += this_files;
7514 while (total < ctx->nr_user_files) {
7515 struct file *file = io_file_from_index(ctx, total);
7525 static int io_sqe_files_scm(struct io_ring_ctx *ctx)
7531 static void io_rsrc_file_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc)
7533 struct file *file = prsrc->file;
7534 #if defined(CONFIG_UNIX)
7535 struct sock *sock = ctx->ring_sock->sk;
7536 struct sk_buff_head list, *head = &sock->sk_receive_queue;
7537 struct sk_buff *skb;
7540 __skb_queue_head_init(&list);
7543 * Find the skb that holds this file in its SCM_RIGHTS. When found,
7544 * remove this entry and rearrange the file array.
7546 skb = skb_dequeue(head);
7548 struct scm_fp_list *fp;
7550 fp = UNIXCB(skb).fp;
7551 for (i = 0; i < fp->count; i++) {
7554 if (fp->fp[i] != file)
7557 unix_notinflight(fp->user, fp->fp[i]);
7558 left = fp->count - 1 - i;
7560 memmove(&fp->fp[i], &fp->fp[i + 1],
7561 left * sizeof(struct file *));
7568 __skb_queue_tail(&list, skb);
7578 __skb_queue_tail(&list, skb);
7580 skb = skb_dequeue(head);
7583 if (skb_peek(&list)) {
7584 spin_lock_irq(&head->lock);
7585 while ((skb = __skb_dequeue(&list)) != NULL)
7586 __skb_queue_tail(head, skb);
7587 spin_unlock_irq(&head->lock);
7594 static void __io_rsrc_put_work(struct io_rsrc_node *ref_node)
7596 struct io_rsrc_data *rsrc_data = ref_node->rsrc_data;
7597 struct io_ring_ctx *ctx = rsrc_data->ctx;
7598 struct io_rsrc_put *prsrc, *tmp;
7600 list_for_each_entry_safe(prsrc, tmp, &ref_node->rsrc_list, list) {
7601 list_del(&prsrc->list);
7604 bool lock_ring = ctx->flags & IORING_SETUP_IOPOLL;
7606 io_ring_submit_lock(ctx, lock_ring);
7607 spin_lock_irq(&ctx->completion_lock);
7608 io_cqring_fill_event(ctx, prsrc->tag, 0, 0);
7610 io_commit_cqring(ctx);
7611 spin_unlock_irq(&ctx->completion_lock);
7612 io_cqring_ev_posted(ctx);
7613 io_ring_submit_unlock(ctx, lock_ring);
7616 rsrc_data->do_put(ctx, prsrc);
7620 io_rsrc_node_destroy(ref_node);
7621 if (atomic_dec_and_test(&rsrc_data->refs))
7622 complete(&rsrc_data->done);
7625 static void io_rsrc_put_work(struct work_struct *work)
7627 struct io_ring_ctx *ctx;
7628 struct llist_node *node;
7630 ctx = container_of(work, struct io_ring_ctx, rsrc_put_work.work);
7631 node = llist_del_all(&ctx->rsrc_put_llist);
7634 struct io_rsrc_node *ref_node;
7635 struct llist_node *next = node->next;
7637 ref_node = llist_entry(node, struct io_rsrc_node, llist);
7638 __io_rsrc_put_work(ref_node);
7643 static void io_rsrc_node_ref_zero(struct percpu_ref *ref)
7645 struct io_rsrc_node *node = container_of(ref, struct io_rsrc_node, refs);
7646 struct io_ring_ctx *ctx = node->rsrc_data->ctx;
7647 bool first_add = false;
7649 io_rsrc_ref_lock(ctx);
7652 while (!list_empty(&ctx->rsrc_ref_list)) {
7653 node = list_first_entry(&ctx->rsrc_ref_list,
7654 struct io_rsrc_node, node);
7655 /* recycle ref nodes in order */
7658 list_del(&node->node);
7659 first_add |= llist_add(&node->llist, &ctx->rsrc_put_llist);
7661 io_rsrc_ref_unlock(ctx);
7664 mod_delayed_work(system_wq, &ctx->rsrc_put_work, HZ);
7667 static struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx)
7669 struct io_rsrc_node *ref_node;
7671 ref_node = kzalloc(sizeof(*ref_node), GFP_KERNEL);
7675 if (percpu_ref_init(&ref_node->refs, io_rsrc_node_ref_zero,
7680 INIT_LIST_HEAD(&ref_node->node);
7681 INIT_LIST_HEAD(&ref_node->rsrc_list);
7682 ref_node->done = false;
7686 static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
7687 unsigned nr_args, u64 __user *tags)
7689 __s32 __user *fds = (__s32 __user *) arg;
7698 if (nr_args > IORING_MAX_FIXED_FILES)
7700 ret = io_rsrc_node_switch_start(ctx);
7703 ret = io_rsrc_data_alloc(ctx, io_rsrc_file_put, tags, nr_args,
7709 if (!io_alloc_file_tables(&ctx->file_table, nr_args))
7712 for (i = 0; i < nr_args; i++, ctx->nr_user_files++) {
7713 if (copy_from_user(&fd, &fds[i], sizeof(fd))) {
7717 /* allow sparse sets */
7720 if (unlikely(*io_get_tag_slot(ctx->file_data, i)))
7727 if (unlikely(!file))
7731 * Don't allow io_uring instances to be registered. If UNIX
7732 * isn't enabled, then this causes a reference cycle and this
7733 * instance can never get freed. If UNIX is enabled we'll
7734 * handle it just fine, but there's still no point in allowing
7735 * a ring fd as it doesn't support regular read/write anyway.
7737 if (file->f_op == &io_uring_fops) {
7741 io_fixed_file_set(io_fixed_file_slot(&ctx->file_table, i), file);
7744 ret = io_sqe_files_scm(ctx);
7746 __io_sqe_files_unregister(ctx);
7750 io_rsrc_node_switch(ctx, NULL);
7753 for (i = 0; i < ctx->nr_user_files; i++) {
7754 file = io_file_from_index(ctx, i);
7758 io_free_file_tables(&ctx->file_table, nr_args);
7759 ctx->nr_user_files = 0;
7761 io_rsrc_data_free(ctx->file_data);
7762 ctx->file_data = NULL;
7766 static int io_sqe_file_register(struct io_ring_ctx *ctx, struct file *file,
7769 #if defined(CONFIG_UNIX)
7770 struct sock *sock = ctx->ring_sock->sk;
7771 struct sk_buff_head *head = &sock->sk_receive_queue;
7772 struct sk_buff *skb;
7775 * See if we can merge this file into an existing skb SCM_RIGHTS
7776 * file set. If there's no room, fall back to allocating a new skb
7777 * and filling it in.
7779 spin_lock_irq(&head->lock);
7780 skb = skb_peek(head);
7782 struct scm_fp_list *fpl = UNIXCB(skb).fp;
7784 if (fpl->count < SCM_MAX_FD) {
7785 __skb_unlink(skb, head);
7786 spin_unlock_irq(&head->lock);
7787 fpl->fp[fpl->count] = get_file(file);
7788 unix_inflight(fpl->user, fpl->fp[fpl->count]);
7790 spin_lock_irq(&head->lock);
7791 __skb_queue_head(head, skb);
7796 spin_unlock_irq(&head->lock);
7803 return __io_sqe_files_scm(ctx, 1, index);
7809 static int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx,
7810 struct io_rsrc_node *node, void *rsrc)
7812 struct io_rsrc_put *prsrc;
7814 prsrc = kzalloc(sizeof(*prsrc), GFP_KERNEL);
7818 prsrc->tag = *io_get_tag_slot(data, idx);
7820 list_add(&prsrc->list, &node->rsrc_list);
7824 static int __io_sqe_files_update(struct io_ring_ctx *ctx,
7825 struct io_uring_rsrc_update2 *up,
7828 u64 __user *tags = u64_to_user_ptr(up->tags);
7829 __s32 __user *fds = u64_to_user_ptr(up->data);
7830 struct io_rsrc_data *data = ctx->file_data;
7831 struct io_fixed_file *file_slot;
7835 bool needs_switch = false;
7837 if (!ctx->file_data)
7839 if (up->offset + nr_args > ctx->nr_user_files)
7842 for (done = 0; done < nr_args; done++) {
7845 if ((tags && copy_from_user(&tag, &tags[done], sizeof(tag))) ||
7846 copy_from_user(&fd, &fds[done], sizeof(fd))) {
7850 if ((fd == IORING_REGISTER_FILES_SKIP || fd == -1) && tag) {
7854 if (fd == IORING_REGISTER_FILES_SKIP)
7857 i = array_index_nospec(up->offset + done, ctx->nr_user_files);
7858 file_slot = io_fixed_file_slot(&ctx->file_table, i);
7860 if (file_slot->file_ptr) {
7861 file = (struct file *)(file_slot->file_ptr & FFS_MASK);
7862 err = io_queue_rsrc_removal(data, up->offset + done,
7863 ctx->rsrc_node, file);
7866 file_slot->file_ptr = 0;
7867 needs_switch = true;
7876 * Don't allow io_uring instances to be registered. If
7877 * UNIX isn't enabled, then this causes a reference
7878 * cycle and this instance can never get freed. If UNIX
7879 * is enabled we'll handle it just fine, but there's
7880 * still no point in allowing a ring fd as it doesn't
7881 * support regular read/write anyway.
7883 if (file->f_op == &io_uring_fops) {
7888 *io_get_tag_slot(data, up->offset + done) = tag;
7889 io_fixed_file_set(file_slot, file);
7890 err = io_sqe_file_register(ctx, file, i);
7892 file_slot->file_ptr = 0;
7900 io_rsrc_node_switch(ctx, data);
7901 return done ? done : err;
7904 static struct io_wq_work *io_free_work(struct io_wq_work *work)
7906 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
7908 req = io_put_req_find_next(req);
7909 return req ? &req->work : NULL;
7912 static struct io_wq *io_init_wq_offload(struct io_ring_ctx *ctx,
7913 struct task_struct *task)
7915 struct io_wq_hash *hash;
7916 struct io_wq_data data;
7917 unsigned int concurrency;
7919 mutex_lock(&ctx->uring_lock);
7920 hash = ctx->hash_map;
7922 hash = kzalloc(sizeof(*hash), GFP_KERNEL);
7924 mutex_unlock(&ctx->uring_lock);
7925 return ERR_PTR(-ENOMEM);
7927 refcount_set(&hash->refs, 1);
7928 init_waitqueue_head(&hash->wait);
7929 ctx->hash_map = hash;
7931 mutex_unlock(&ctx->uring_lock);
7935 data.free_work = io_free_work;
7936 data.do_work = io_wq_submit_work;
7938 /* Do QD, or 4 * CPUS, whatever is smallest */
7939 concurrency = min(ctx->sq_entries, 4 * num_online_cpus());
7941 return io_wq_create(concurrency, &data);
7944 static int io_uring_alloc_task_context(struct task_struct *task,
7945 struct io_ring_ctx *ctx)
7947 struct io_uring_task *tctx;
7950 tctx = kzalloc(sizeof(*tctx), GFP_KERNEL);
7951 if (unlikely(!tctx))
7954 ret = percpu_counter_init(&tctx->inflight, 0, GFP_KERNEL);
7955 if (unlikely(ret)) {
7960 tctx->io_wq = io_init_wq_offload(ctx, task);
7961 if (IS_ERR(tctx->io_wq)) {
7962 ret = PTR_ERR(tctx->io_wq);
7963 percpu_counter_destroy(&tctx->inflight);
7969 init_waitqueue_head(&tctx->wait);
7970 atomic_set(&tctx->in_idle, 0);
7971 atomic_set(&tctx->inflight_tracked, 0);
7972 task->io_uring = tctx;
7973 spin_lock_init(&tctx->task_lock);
7974 INIT_WQ_LIST(&tctx->task_list);
7975 init_task_work(&tctx->task_work, tctx_task_work);
7979 void __io_uring_free(struct task_struct *tsk)
7981 struct io_uring_task *tctx = tsk->io_uring;
7983 WARN_ON_ONCE(!xa_empty(&tctx->xa));
7984 WARN_ON_ONCE(tctx->io_wq);
7985 WARN_ON_ONCE(tctx->cached_refs);
7987 percpu_counter_destroy(&tctx->inflight);
7989 tsk->io_uring = NULL;
7992 static int io_sq_offload_create(struct io_ring_ctx *ctx,
7993 struct io_uring_params *p)
7997 /* Retain compatibility with failing for an invalid attach attempt */
7998 if ((ctx->flags & (IORING_SETUP_ATTACH_WQ | IORING_SETUP_SQPOLL)) ==
7999 IORING_SETUP_ATTACH_WQ) {
8002 f = fdget(p->wq_fd);
8005 if (f.file->f_op != &io_uring_fops) {
8011 if (ctx->flags & IORING_SETUP_SQPOLL) {
8012 struct task_struct *tsk;
8013 struct io_sq_data *sqd;
8016 sqd = io_get_sq_data(p, &attached);
8022 ctx->sq_creds = get_current_cred();
8024 ctx->sq_thread_idle = msecs_to_jiffies(p->sq_thread_idle);
8025 if (!ctx->sq_thread_idle)
8026 ctx->sq_thread_idle = HZ;
8028 io_sq_thread_park(sqd);
8029 list_add(&ctx->sqd_list, &sqd->ctx_list);
8030 io_sqd_update_thread_idle(sqd);
8031 /* don't attach to a dying SQPOLL thread, would be racy */
8032 ret = (attached && !sqd->thread) ? -ENXIO : 0;
8033 io_sq_thread_unpark(sqd);
8040 if (p->flags & IORING_SETUP_SQ_AFF) {
8041 int cpu = p->sq_thread_cpu;
8044 if (cpu >= nr_cpu_ids || !cpu_online(cpu))
8051 sqd->task_pid = current->pid;
8052 sqd->task_tgid = current->tgid;
8053 tsk = create_io_thread(io_sq_thread, sqd, NUMA_NO_NODE);
8060 ret = io_uring_alloc_task_context(tsk, ctx);
8061 wake_up_new_task(tsk);
8064 } else if (p->flags & IORING_SETUP_SQ_AFF) {
8065 /* Can't have SQ_AFF without SQPOLL */
8072 complete(&ctx->sq_data->exited);
8074 io_sq_thread_finish(ctx);
8078 static inline void __io_unaccount_mem(struct user_struct *user,
8079 unsigned long nr_pages)
8081 atomic_long_sub(nr_pages, &user->locked_vm);
8084 static inline int __io_account_mem(struct user_struct *user,
8085 unsigned long nr_pages)
8087 unsigned long page_limit, cur_pages, new_pages;
8089 /* Don't allow more pages than we can safely lock */
8090 page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
8093 cur_pages = atomic_long_read(&user->locked_vm);
8094 new_pages = cur_pages + nr_pages;
8095 if (new_pages > page_limit)
8097 } while (atomic_long_cmpxchg(&user->locked_vm, cur_pages,
8098 new_pages) != cur_pages);
8103 static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
8106 __io_unaccount_mem(ctx->user, nr_pages);
8108 if (ctx->mm_account)
8109 atomic64_sub(nr_pages, &ctx->mm_account->pinned_vm);
8112 static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
8117 ret = __io_account_mem(ctx->user, nr_pages);
8122 if (ctx->mm_account)
8123 atomic64_add(nr_pages, &ctx->mm_account->pinned_vm);
8128 static void io_mem_free(void *ptr)
8135 page = virt_to_head_page(ptr);
8136 if (put_page_testzero(page))
8137 free_compound_page(page);
8140 static void *io_mem_alloc(size_t size)
8142 gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_COMP |
8143 __GFP_NORETRY | __GFP_ACCOUNT;
8145 return (void *) __get_free_pages(gfp_flags, get_order(size));
8148 static unsigned long rings_size(unsigned sq_entries, unsigned cq_entries,
8151 struct io_rings *rings;
8152 size_t off, sq_array_size;
8154 off = struct_size(rings, cqes, cq_entries);
8155 if (off == SIZE_MAX)
8159 off = ALIGN(off, SMP_CACHE_BYTES);
8167 sq_array_size = array_size(sizeof(u32), sq_entries);
8168 if (sq_array_size == SIZE_MAX)
8171 if (check_add_overflow(off, sq_array_size, &off))
8177 static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf **slot)
8179 struct io_mapped_ubuf *imu = *slot;
8182 if (imu != ctx->dummy_ubuf) {
8183 for (i = 0; i < imu->nr_bvecs; i++)
8184 unpin_user_page(imu->bvec[i].bv_page);
8185 if (imu->acct_pages)
8186 io_unaccount_mem(ctx, imu->acct_pages);
8192 static void io_rsrc_buf_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc)
8194 io_buffer_unmap(ctx, &prsrc->buf);
8198 static void __io_sqe_buffers_unregister(struct io_ring_ctx *ctx)
8202 for (i = 0; i < ctx->nr_user_bufs; i++)
8203 io_buffer_unmap(ctx, &ctx->user_bufs[i]);
8204 kfree(ctx->user_bufs);
8205 io_rsrc_data_free(ctx->buf_data);
8206 ctx->user_bufs = NULL;
8207 ctx->buf_data = NULL;
8208 ctx->nr_user_bufs = 0;
8211 static int io_sqe_buffers_unregister(struct io_ring_ctx *ctx)
8218 ret = io_rsrc_ref_quiesce(ctx->buf_data, ctx);
8220 __io_sqe_buffers_unregister(ctx);
8224 static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst,
8225 void __user *arg, unsigned index)
8227 struct iovec __user *src;
8229 #ifdef CONFIG_COMPAT
8231 struct compat_iovec __user *ciovs;
8232 struct compat_iovec ciov;
8234 ciovs = (struct compat_iovec __user *) arg;
8235 if (copy_from_user(&ciov, &ciovs[index], sizeof(ciov)))
8238 dst->iov_base = u64_to_user_ptr((u64)ciov.iov_base);
8239 dst->iov_len = ciov.iov_len;
8243 src = (struct iovec __user *) arg;
8244 if (copy_from_user(dst, &src[index], sizeof(*dst)))
8250 * Not super efficient, but this is just a registration time. And we do cache
8251 * the last compound head, so generally we'll only do a full search if we don't
8254 * We check if the given compound head page has already been accounted, to
8255 * avoid double accounting it. This allows us to account the full size of the
8256 * page, not just the constituent pages of a huge page.
8258 static bool headpage_already_acct(struct io_ring_ctx *ctx, struct page **pages,
8259 int nr_pages, struct page *hpage)
8263 /* check current page array */
8264 for (i = 0; i < nr_pages; i++) {
8265 if (!PageCompound(pages[i]))
8267 if (compound_head(pages[i]) == hpage)
8271 /* check previously registered pages */
8272 for (i = 0; i < ctx->nr_user_bufs; i++) {
8273 struct io_mapped_ubuf *imu = ctx->user_bufs[i];
8275 for (j = 0; j < imu->nr_bvecs; j++) {
8276 if (!PageCompound(imu->bvec[j].bv_page))
8278 if (compound_head(imu->bvec[j].bv_page) == hpage)
8286 static int io_buffer_account_pin(struct io_ring_ctx *ctx, struct page **pages,
8287 int nr_pages, struct io_mapped_ubuf *imu,
8288 struct page **last_hpage)
8292 imu->acct_pages = 0;
8293 for (i = 0; i < nr_pages; i++) {
8294 if (!PageCompound(pages[i])) {
8299 hpage = compound_head(pages[i]);
8300 if (hpage == *last_hpage)
8302 *last_hpage = hpage;
8303 if (headpage_already_acct(ctx, pages, i, hpage))
8305 imu->acct_pages += page_size(hpage) >> PAGE_SHIFT;
8309 if (!imu->acct_pages)
8312 ret = io_account_mem(ctx, imu->acct_pages);
8314 imu->acct_pages = 0;
8318 static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,
8319 struct io_mapped_ubuf **pimu,
8320 struct page **last_hpage)
8322 struct io_mapped_ubuf *imu = NULL;
8323 struct vm_area_struct **vmas = NULL;
8324 struct page **pages = NULL;
8325 unsigned long off, start, end, ubuf;
8327 int ret, pret, nr_pages, i;
8329 if (!iov->iov_base) {
8330 *pimu = ctx->dummy_ubuf;
8334 ubuf = (unsigned long) iov->iov_base;
8335 end = (ubuf + iov->iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
8336 start = ubuf >> PAGE_SHIFT;
8337 nr_pages = end - start;
8342 pages = kvmalloc_array(nr_pages, sizeof(struct page *), GFP_KERNEL);
8346 vmas = kvmalloc_array(nr_pages, sizeof(struct vm_area_struct *),
8351 imu = kvmalloc(struct_size(imu, bvec, nr_pages), GFP_KERNEL);
8356 mmap_read_lock(current->mm);
8357 pret = pin_user_pages(ubuf, nr_pages, FOLL_WRITE | FOLL_LONGTERM,
8359 if (pret == nr_pages) {
8360 /* don't support file backed memory */
8361 for (i = 0; i < nr_pages; i++) {
8362 struct vm_area_struct *vma = vmas[i];
8364 if (vma_is_shmem(vma))
8367 !is_file_hugepages(vma->vm_file)) {
8373 ret = pret < 0 ? pret : -EFAULT;
8375 mmap_read_unlock(current->mm);
8378 * if we did partial map, or found file backed vmas,
8379 * release any pages we did get
8382 unpin_user_pages(pages, pret);
8386 ret = io_buffer_account_pin(ctx, pages, pret, imu, last_hpage);
8388 unpin_user_pages(pages, pret);
8392 off = ubuf & ~PAGE_MASK;
8393 size = iov->iov_len;
8394 for (i = 0; i < nr_pages; i++) {
8397 vec_len = min_t(size_t, size, PAGE_SIZE - off);
8398 imu->bvec[i].bv_page = pages[i];
8399 imu->bvec[i].bv_len = vec_len;
8400 imu->bvec[i].bv_offset = off;
8404 /* store original address for later verification */
8406 imu->ubuf_end = ubuf + iov->iov_len;
8407 imu->nr_bvecs = nr_pages;
8418 static int io_buffers_map_alloc(struct io_ring_ctx *ctx, unsigned int nr_args)
8420 ctx->user_bufs = kcalloc(nr_args, sizeof(*ctx->user_bufs), GFP_KERNEL);
8421 return ctx->user_bufs ? 0 : -ENOMEM;
8424 static int io_buffer_validate(struct iovec *iov)
8426 unsigned long tmp, acct_len = iov->iov_len + (PAGE_SIZE - 1);
8429 * Don't impose further limits on the size and buffer
8430 * constraints here, we'll -EINVAL later when IO is
8431 * submitted if they are wrong.
8434 return iov->iov_len ? -EFAULT : 0;
8438 /* arbitrary limit, but we need something */
8439 if (iov->iov_len > SZ_1G)
8442 if (check_add_overflow((unsigned long)iov->iov_base, acct_len, &tmp))
8448 static int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,
8449 unsigned int nr_args, u64 __user *tags)
8451 struct page *last_hpage = NULL;
8452 struct io_rsrc_data *data;
8458 if (!nr_args || nr_args > IORING_MAX_REG_BUFFERS)
8460 ret = io_rsrc_node_switch_start(ctx);
8463 ret = io_rsrc_data_alloc(ctx, io_rsrc_buf_put, tags, nr_args, &data);
8466 ret = io_buffers_map_alloc(ctx, nr_args);
8468 io_rsrc_data_free(data);
8472 for (i = 0; i < nr_args; i++, ctx->nr_user_bufs++) {
8473 ret = io_copy_iov(ctx, &iov, arg, i);
8476 ret = io_buffer_validate(&iov);
8479 if (!iov.iov_base && *io_get_tag_slot(data, i)) {
8484 ret = io_sqe_buffer_register(ctx, &iov, &ctx->user_bufs[i],
8490 WARN_ON_ONCE(ctx->buf_data);
8492 ctx->buf_data = data;
8494 __io_sqe_buffers_unregister(ctx);
8496 io_rsrc_node_switch(ctx, NULL);
8500 static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
8501 struct io_uring_rsrc_update2 *up,
8502 unsigned int nr_args)
8504 u64 __user *tags = u64_to_user_ptr(up->tags);
8505 struct iovec iov, __user *iovs = u64_to_user_ptr(up->data);
8506 struct page *last_hpage = NULL;
8507 bool needs_switch = false;
8513 if (up->offset + nr_args > ctx->nr_user_bufs)
8516 for (done = 0; done < nr_args; done++) {
8517 struct io_mapped_ubuf *imu;
8518 int offset = up->offset + done;
8521 err = io_copy_iov(ctx, &iov, iovs, done);
8524 if (tags && copy_from_user(&tag, &tags[done], sizeof(tag))) {
8528 err = io_buffer_validate(&iov);
8531 if (!iov.iov_base && tag) {
8535 err = io_sqe_buffer_register(ctx, &iov, &imu, &last_hpage);
8539 i = array_index_nospec(offset, ctx->nr_user_bufs);
8540 if (ctx->user_bufs[i] != ctx->dummy_ubuf) {
8541 err = io_queue_rsrc_removal(ctx->buf_data, offset,
8542 ctx->rsrc_node, ctx->user_bufs[i]);
8543 if (unlikely(err)) {
8544 io_buffer_unmap(ctx, &imu);
8547 ctx->user_bufs[i] = NULL;
8548 needs_switch = true;
8551 ctx->user_bufs[i] = imu;
8552 *io_get_tag_slot(ctx->buf_data, offset) = tag;
8556 io_rsrc_node_switch(ctx, ctx->buf_data);
8557 return done ? done : err;
8560 static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg)
8562 __s32 __user *fds = arg;
8568 if (copy_from_user(&fd, fds, sizeof(*fds)))
8571 ctx->cq_ev_fd = eventfd_ctx_fdget(fd);
8572 if (IS_ERR(ctx->cq_ev_fd)) {
8573 int ret = PTR_ERR(ctx->cq_ev_fd);
8575 ctx->cq_ev_fd = NULL;
8582 static int io_eventfd_unregister(struct io_ring_ctx *ctx)
8584 if (ctx->cq_ev_fd) {
8585 eventfd_ctx_put(ctx->cq_ev_fd);
8586 ctx->cq_ev_fd = NULL;
8593 static void io_destroy_buffers(struct io_ring_ctx *ctx)
8595 struct io_buffer *buf;
8596 unsigned long index;
8598 xa_for_each(&ctx->io_buffers, index, buf)
8599 __io_remove_buffers(ctx, buf, index, -1U);
8602 static void io_req_cache_free(struct list_head *list, struct task_struct *tsk)
8604 struct io_kiocb *req, *nxt;
8606 list_for_each_entry_safe(req, nxt, list, compl.list) {
8607 if (tsk && req->task != tsk)
8609 list_del(&req->compl.list);
8610 kmem_cache_free(req_cachep, req);
8614 static void io_req_caches_free(struct io_ring_ctx *ctx)
8616 struct io_submit_state *submit_state = &ctx->submit_state;
8617 struct io_comp_state *cs = &ctx->submit_state.comp;
8619 mutex_lock(&ctx->uring_lock);
8621 if (submit_state->free_reqs) {
8622 kmem_cache_free_bulk(req_cachep, submit_state->free_reqs,
8623 submit_state->reqs);
8624 submit_state->free_reqs = 0;
8627 io_flush_cached_locked_reqs(ctx, cs);
8628 io_req_cache_free(&cs->free_list, NULL);
8629 mutex_unlock(&ctx->uring_lock);
8632 static bool io_wait_rsrc_data(struct io_rsrc_data *data)
8636 if (!atomic_dec_and_test(&data->refs))
8637 wait_for_completion(&data->done);
8641 static void io_ring_ctx_free(struct io_ring_ctx *ctx)
8643 io_sq_thread_finish(ctx);
8645 if (ctx->mm_account) {
8646 mmdrop(ctx->mm_account);
8647 ctx->mm_account = NULL;
8650 mutex_lock(&ctx->uring_lock);
8651 if (io_wait_rsrc_data(ctx->buf_data))
8652 __io_sqe_buffers_unregister(ctx);
8653 if (io_wait_rsrc_data(ctx->file_data))
8654 __io_sqe_files_unregister(ctx);
8656 __io_cqring_overflow_flush(ctx, true);
8657 mutex_unlock(&ctx->uring_lock);
8658 io_eventfd_unregister(ctx);
8659 io_destroy_buffers(ctx);
8661 put_cred(ctx->sq_creds);
8663 /* there are no registered resources left, nobody uses it */
8665 io_rsrc_node_destroy(ctx->rsrc_node);
8666 if (ctx->rsrc_backup_node)
8667 io_rsrc_node_destroy(ctx->rsrc_backup_node);
8668 flush_delayed_work(&ctx->rsrc_put_work);
8670 WARN_ON_ONCE(!list_empty(&ctx->rsrc_ref_list));
8671 WARN_ON_ONCE(!llist_empty(&ctx->rsrc_put_llist));
8673 #if defined(CONFIG_UNIX)
8674 if (ctx->ring_sock) {
8675 ctx->ring_sock->file = NULL; /* so that iput() is called */
8676 sock_release(ctx->ring_sock);
8680 io_mem_free(ctx->rings);
8681 io_mem_free(ctx->sq_sqes);
8683 percpu_ref_exit(&ctx->refs);
8684 free_uid(ctx->user);
8685 io_req_caches_free(ctx);
8687 io_wq_put_hash(ctx->hash_map);
8688 kfree(ctx->cancel_hash);
8689 kfree(ctx->dummy_ubuf);
8693 static __poll_t io_uring_poll(struct file *file, poll_table *wait)
8695 struct io_ring_ctx *ctx = file->private_data;
8698 poll_wait(file, &ctx->poll_wait, wait);
8700 * synchronizes with barrier from wq_has_sleeper call in
8704 if (!io_sqring_full(ctx))
8705 mask |= EPOLLOUT | EPOLLWRNORM;
8708 * Don't flush cqring overflow list here, just do a simple check.
8709 * Otherwise there could possible be ABBA deadlock:
8712 * lock(&ctx->uring_lock);
8714 * lock(&ctx->uring_lock);
8717 * Users may get EPOLLIN meanwhile seeing nothing in cqring, this
8718 * pushs them to do the flush.
8720 if (io_cqring_events(ctx) || test_bit(0, &ctx->check_cq_overflow))
8721 mask |= EPOLLIN | EPOLLRDNORM;
8726 static int io_uring_fasync(int fd, struct file *file, int on)
8728 struct io_ring_ctx *ctx = file->private_data;
8730 return fasync_helper(fd, file, on, &ctx->cq_fasync);
8733 static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id)
8735 const struct cred *creds;
8737 creds = xa_erase(&ctx->personalities, id);
8746 struct io_tctx_exit {
8747 struct callback_head task_work;
8748 struct completion completion;
8749 struct io_ring_ctx *ctx;
8752 static void io_tctx_exit_cb(struct callback_head *cb)
8754 struct io_uring_task *tctx = current->io_uring;
8755 struct io_tctx_exit *work;
8757 work = container_of(cb, struct io_tctx_exit, task_work);
8759 * When @in_idle, we're in cancellation and it's racy to remove the
8760 * node. It'll be removed by the end of cancellation, just ignore it.
8762 if (!atomic_read(&tctx->in_idle))
8763 io_uring_del_tctx_node((unsigned long)work->ctx);
8764 complete(&work->completion);
8767 static bool io_cancel_ctx_cb(struct io_wq_work *work, void *data)
8769 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
8771 return req->ctx == data;
8774 static void io_ring_exit_work(struct work_struct *work)
8776 struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, exit_work);
8777 unsigned long timeout = jiffies + HZ * 60 * 5;
8778 struct io_tctx_exit exit;
8779 struct io_tctx_node *node;
8783 * If we're doing polled IO and end up having requests being
8784 * submitted async (out-of-line), then completions can come in while
8785 * we're waiting for refs to drop. We need to reap these manually,
8786 * as nobody else will be looking for them.
8789 io_uring_try_cancel_requests(ctx, NULL, true);
8791 struct io_sq_data *sqd = ctx->sq_data;
8792 struct task_struct *tsk;
8794 io_sq_thread_park(sqd);
8796 if (tsk && tsk->io_uring && tsk->io_uring->io_wq)
8797 io_wq_cancel_cb(tsk->io_uring->io_wq,
8798 io_cancel_ctx_cb, ctx, true);
8799 io_sq_thread_unpark(sqd);
8802 WARN_ON_ONCE(time_after(jiffies, timeout));
8803 } while (!wait_for_completion_timeout(&ctx->ref_comp, HZ/20));
8805 init_completion(&exit.completion);
8806 init_task_work(&exit.task_work, io_tctx_exit_cb);
8809 * Some may use context even when all refs and requests have been put,
8810 * and they are free to do so while still holding uring_lock or
8811 * completion_lock, see io_req_task_submit(). Apart from other work,
8812 * this lock/unlock section also waits them to finish.
8814 mutex_lock(&ctx->uring_lock);
8815 while (!list_empty(&ctx->tctx_list)) {
8816 WARN_ON_ONCE(time_after(jiffies, timeout));
8818 node = list_first_entry(&ctx->tctx_list, struct io_tctx_node,
8820 /* don't spin on a single task if cancellation failed */
8821 list_rotate_left(&ctx->tctx_list);
8822 ret = task_work_add(node->task, &exit.task_work, TWA_SIGNAL);
8823 if (WARN_ON_ONCE(ret))
8825 wake_up_process(node->task);
8827 mutex_unlock(&ctx->uring_lock);
8828 wait_for_completion(&exit.completion);
8829 mutex_lock(&ctx->uring_lock);
8831 mutex_unlock(&ctx->uring_lock);
8832 spin_lock_irq(&ctx->completion_lock);
8833 spin_unlock_irq(&ctx->completion_lock);
8835 io_ring_ctx_free(ctx);
8838 /* Returns true if we found and killed one or more timeouts */
8839 static bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk,
8842 struct io_kiocb *req, *tmp;
8845 spin_lock_irq(&ctx->completion_lock);
8846 list_for_each_entry_safe(req, tmp, &ctx->timeout_list, timeout.list) {
8847 if (io_match_task(req, tsk, cancel_all)) {
8848 io_kill_timeout(req, -ECANCELED);
8853 io_commit_cqring(ctx);
8854 spin_unlock_irq(&ctx->completion_lock);
8856 io_cqring_ev_posted(ctx);
8857 return canceled != 0;
8860 static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
8862 unsigned long index;
8863 struct creds *creds;
8865 mutex_lock(&ctx->uring_lock);
8866 percpu_ref_kill(&ctx->refs);
8868 __io_cqring_overflow_flush(ctx, true);
8869 xa_for_each(&ctx->personalities, index, creds)
8870 io_unregister_personality(ctx, index);
8871 mutex_unlock(&ctx->uring_lock);
8873 io_kill_timeouts(ctx, NULL, true);
8874 io_poll_remove_all(ctx, NULL, true);
8876 /* if we failed setting up the ctx, we might not have any rings */
8877 io_iopoll_try_reap_events(ctx);
8879 INIT_WORK(&ctx->exit_work, io_ring_exit_work);
8881 * Use system_unbound_wq to avoid spawning tons of event kworkers
8882 * if we're exiting a ton of rings at the same time. It just adds
8883 * noise and overhead, there's no discernable change in runtime
8884 * over using system_wq.
8886 queue_work(system_unbound_wq, &ctx->exit_work);
8889 static int io_uring_release(struct inode *inode, struct file *file)
8891 struct io_ring_ctx *ctx = file->private_data;
8893 file->private_data = NULL;
8894 io_ring_ctx_wait_and_kill(ctx);
8898 struct io_task_cancel {
8899 struct task_struct *task;
8903 static bool io_cancel_task_cb(struct io_wq_work *work, void *data)
8905 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
8906 struct io_task_cancel *cancel = data;
8909 if (!cancel->all && (req->flags & REQ_F_LINK_TIMEOUT)) {
8910 unsigned long flags;
8911 struct io_ring_ctx *ctx = req->ctx;
8913 /* protect against races with linked timeouts */
8914 spin_lock_irqsave(&ctx->completion_lock, flags);
8915 ret = io_match_task(req, cancel->task, cancel->all);
8916 spin_unlock_irqrestore(&ctx->completion_lock, flags);
8918 ret = io_match_task(req, cancel->task, cancel->all);
8923 static bool io_cancel_defer_files(struct io_ring_ctx *ctx,
8924 struct task_struct *task, bool cancel_all)
8926 struct io_defer_entry *de;
8929 spin_lock_irq(&ctx->completion_lock);
8930 list_for_each_entry_reverse(de, &ctx->defer_list, list) {
8931 if (io_match_task(de->req, task, cancel_all)) {
8932 list_cut_position(&list, &ctx->defer_list, &de->list);
8936 spin_unlock_irq(&ctx->completion_lock);
8937 if (list_empty(&list))
8940 while (!list_empty(&list)) {
8941 de = list_first_entry(&list, struct io_defer_entry, list);
8942 list_del_init(&de->list);
8943 io_req_complete_failed(de->req, -ECANCELED);
8949 static bool io_uring_try_cancel_iowq(struct io_ring_ctx *ctx)
8951 struct io_tctx_node *node;
8952 enum io_wq_cancel cret;
8955 mutex_lock(&ctx->uring_lock);
8956 list_for_each_entry(node, &ctx->tctx_list, ctx_node) {
8957 struct io_uring_task *tctx = node->task->io_uring;
8960 * io_wq will stay alive while we hold uring_lock, because it's
8961 * killed after ctx nodes, which requires to take the lock.
8963 if (!tctx || !tctx->io_wq)
8965 cret = io_wq_cancel_cb(tctx->io_wq, io_cancel_ctx_cb, ctx, true);
8966 ret |= (cret != IO_WQ_CANCEL_NOTFOUND);
8968 mutex_unlock(&ctx->uring_lock);
8973 static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
8974 struct task_struct *task,
8977 struct io_task_cancel cancel = { .task = task, .all = cancel_all, };
8978 struct io_uring_task *tctx = task ? task->io_uring : NULL;
8981 enum io_wq_cancel cret;
8985 ret |= io_uring_try_cancel_iowq(ctx);
8986 } else if (tctx && tctx->io_wq) {
8988 * Cancels requests of all rings, not only @ctx, but
8989 * it's fine as the task is in exit/exec.
8991 cret = io_wq_cancel_cb(tctx->io_wq, io_cancel_task_cb,
8993 ret |= (cret != IO_WQ_CANCEL_NOTFOUND);
8996 /* SQPOLL thread does its own polling */
8997 if ((!(ctx->flags & IORING_SETUP_SQPOLL) && cancel_all) ||
8998 (ctx->sq_data && ctx->sq_data->thread == current)) {
8999 while (!list_empty_careful(&ctx->iopoll_list)) {
9000 io_iopoll_try_reap_events(ctx);
9005 ret |= io_cancel_defer_files(ctx, task, cancel_all);
9006 ret |= io_poll_remove_all(ctx, task, cancel_all);
9007 ret |= io_kill_timeouts(ctx, task, cancel_all);
9009 ret |= io_run_task_work();
9016 static int __io_uring_add_tctx_node(struct io_ring_ctx *ctx)
9018 struct io_uring_task *tctx = current->io_uring;
9019 struct io_tctx_node *node;
9022 if (unlikely(!tctx)) {
9023 ret = io_uring_alloc_task_context(current, ctx);
9026 tctx = current->io_uring;
9028 if (!xa_load(&tctx->xa, (unsigned long)ctx)) {
9029 node = kmalloc(sizeof(*node), GFP_KERNEL);
9033 node->task = current;
9035 ret = xa_err(xa_store(&tctx->xa, (unsigned long)ctx,
9042 mutex_lock(&ctx->uring_lock);
9043 list_add(&node->ctx_node, &ctx->tctx_list);
9044 mutex_unlock(&ctx->uring_lock);
9051 * Note that this task has used io_uring. We use it for cancelation purposes.
9053 static inline int io_uring_add_tctx_node(struct io_ring_ctx *ctx)
9055 struct io_uring_task *tctx = current->io_uring;
9057 if (likely(tctx && tctx->last == ctx))
9059 return __io_uring_add_tctx_node(ctx);
9063 * Remove this io_uring_file -> task mapping.
9065 static void io_uring_del_tctx_node(unsigned long index)
9067 struct io_uring_task *tctx = current->io_uring;
9068 struct io_tctx_node *node;
9072 node = xa_erase(&tctx->xa, index);
9076 WARN_ON_ONCE(current != node->task);
9077 WARN_ON_ONCE(list_empty(&node->ctx_node));
9079 mutex_lock(&node->ctx->uring_lock);
9080 list_del(&node->ctx_node);
9081 mutex_unlock(&node->ctx->uring_lock);
9083 if (tctx->last == node->ctx)
9088 static void io_uring_clean_tctx(struct io_uring_task *tctx)
9090 struct io_wq *wq = tctx->io_wq;
9091 struct io_tctx_node *node;
9092 unsigned long index;
9094 xa_for_each(&tctx->xa, index, node)
9095 io_uring_del_tctx_node(index);
9098 * Must be after io_uring_del_task_file() (removes nodes under
9099 * uring_lock) to avoid race with io_uring_try_cancel_iowq().
9102 io_wq_put_and_exit(wq);
9106 static s64 tctx_inflight(struct io_uring_task *tctx, bool tracked)
9109 return atomic_read(&tctx->inflight_tracked);
9110 return percpu_counter_sum(&tctx->inflight);
9113 static void io_uring_drop_tctx_refs(struct task_struct *task)
9115 struct io_uring_task *tctx = task->io_uring;
9116 unsigned int refs = tctx->cached_refs;
9118 tctx->cached_refs = 0;
9119 percpu_counter_sub(&tctx->inflight, refs);
9120 put_task_struct_many(task, refs);
9124 * Find any io_uring ctx that this task has registered or done IO on, and cancel
9125 * requests. @sqd should be not-null IIF it's an SQPOLL thread cancellation.
9127 static void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd)
9129 struct io_uring_task *tctx = current->io_uring;
9130 struct io_ring_ctx *ctx;
9134 WARN_ON_ONCE(sqd && sqd->thread != current);
9136 if (!current->io_uring)
9139 io_wq_exit_start(tctx->io_wq);
9141 io_uring_drop_tctx_refs(current);
9142 atomic_inc(&tctx->in_idle);
9144 /* read completions before cancelations */
9145 inflight = tctx_inflight(tctx, !cancel_all);
9150 struct io_tctx_node *node;
9151 unsigned long index;
9153 xa_for_each(&tctx->xa, index, node) {
9154 /* sqpoll task will cancel all its requests */
9155 if (node->ctx->sq_data)
9157 io_uring_try_cancel_requests(node->ctx, current,
9161 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
9162 io_uring_try_cancel_requests(ctx, current,
9166 prepare_to_wait(&tctx->wait, &wait, TASK_UNINTERRUPTIBLE);
9168 * If we've seen completions, retry without waiting. This
9169 * avoids a race where a completion comes in before we did
9170 * prepare_to_wait().
9172 if (inflight == tctx_inflight(tctx, !cancel_all))
9174 finish_wait(&tctx->wait, &wait);
9176 atomic_dec(&tctx->in_idle);
9178 io_uring_clean_tctx(tctx);
9180 /* for exec all current's requests should be gone, kill tctx */
9181 __io_uring_free(current);
9185 void __io_uring_cancel(struct files_struct *files)
9187 io_uring_cancel_generic(!files, NULL);
9190 static void *io_uring_validate_mmap_request(struct file *file,
9191 loff_t pgoff, size_t sz)
9193 struct io_ring_ctx *ctx = file->private_data;
9194 loff_t offset = pgoff << PAGE_SHIFT;
9199 case IORING_OFF_SQ_RING:
9200 case IORING_OFF_CQ_RING:
9203 case IORING_OFF_SQES:
9207 return ERR_PTR(-EINVAL);
9210 page = virt_to_head_page(ptr);
9211 if (sz > page_size(page))
9212 return ERR_PTR(-EINVAL);
9219 static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
9221 size_t sz = vma->vm_end - vma->vm_start;
9225 ptr = io_uring_validate_mmap_request(file, vma->vm_pgoff, sz);
9227 return PTR_ERR(ptr);
9229 pfn = virt_to_phys(ptr) >> PAGE_SHIFT;
9230 return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot);
9233 #else /* !CONFIG_MMU */
9235 static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
9237 return vma->vm_flags & (VM_SHARED | VM_MAYSHARE) ? 0 : -EINVAL;
9240 static unsigned int io_uring_nommu_mmap_capabilities(struct file *file)
9242 return NOMMU_MAP_DIRECT | NOMMU_MAP_READ | NOMMU_MAP_WRITE;
9245 static unsigned long io_uring_nommu_get_unmapped_area(struct file *file,
9246 unsigned long addr, unsigned long len,
9247 unsigned long pgoff, unsigned long flags)
9251 ptr = io_uring_validate_mmap_request(file, pgoff, len);
9253 return PTR_ERR(ptr);
9255 return (unsigned long) ptr;
9258 #endif /* !CONFIG_MMU */
9260 static int io_sqpoll_wait_sq(struct io_ring_ctx *ctx)
9265 if (!io_sqring_full(ctx))
9267 prepare_to_wait(&ctx->sqo_sq_wait, &wait, TASK_INTERRUPTIBLE);
9269 if (!io_sqring_full(ctx))
9272 } while (!signal_pending(current));
9274 finish_wait(&ctx->sqo_sq_wait, &wait);
9278 static int io_get_ext_arg(unsigned flags, const void __user *argp, size_t *argsz,
9279 struct __kernel_timespec __user **ts,
9280 const sigset_t __user **sig)
9282 struct io_uring_getevents_arg arg;
9285 * If EXT_ARG isn't set, then we have no timespec and the argp pointer
9286 * is just a pointer to the sigset_t.
9288 if (!(flags & IORING_ENTER_EXT_ARG)) {
9289 *sig = (const sigset_t __user *) argp;
9295 * EXT_ARG is set - ensure we agree on the size of it and copy in our
9296 * timespec and sigset_t pointers if good.
9298 if (*argsz != sizeof(arg))
9300 if (copy_from_user(&arg, argp, sizeof(arg)))
9302 *sig = u64_to_user_ptr(arg.sigmask);
9303 *argsz = arg.sigmask_sz;
9304 *ts = u64_to_user_ptr(arg.ts);
9308 SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
9309 u32, min_complete, u32, flags, const void __user *, argp,
9312 struct io_ring_ctx *ctx;
9319 if (unlikely(flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP |
9320 IORING_ENTER_SQ_WAIT | IORING_ENTER_EXT_ARG)))
9324 if (unlikely(!f.file))
9328 if (unlikely(f.file->f_op != &io_uring_fops))
9332 ctx = f.file->private_data;
9333 if (unlikely(!percpu_ref_tryget(&ctx->refs)))
9337 if (unlikely(ctx->flags & IORING_SETUP_R_DISABLED))
9341 * For SQ polling, the thread will do all submissions and completions.
9342 * Just return the requested submit count, and wake the thread if
9346 if (ctx->flags & IORING_SETUP_SQPOLL) {
9347 io_cqring_overflow_flush(ctx, false);
9350 if (unlikely(ctx->sq_data->thread == NULL))
9352 if (flags & IORING_ENTER_SQ_WAKEUP)
9353 wake_up(&ctx->sq_data->wait);
9354 if (flags & IORING_ENTER_SQ_WAIT) {
9355 ret = io_sqpoll_wait_sq(ctx);
9359 submitted = to_submit;
9360 } else if (to_submit) {
9361 ret = io_uring_add_tctx_node(ctx);
9364 mutex_lock(&ctx->uring_lock);
9365 submitted = io_submit_sqes(ctx, to_submit);
9366 mutex_unlock(&ctx->uring_lock);
9368 if (submitted != to_submit)
9371 if (flags & IORING_ENTER_GETEVENTS) {
9372 const sigset_t __user *sig;
9373 struct __kernel_timespec __user *ts;
9375 ret = io_get_ext_arg(flags, argp, &argsz, &ts, &sig);
9379 min_complete = min(min_complete, ctx->cq_entries);
9382 * When SETUP_IOPOLL and SETUP_SQPOLL are both enabled, user
9383 * space applications don't need to do io completion events
9384 * polling again, they can rely on io_sq_thread to do polling
9385 * work, which can reduce cpu usage and uring_lock contention.
9387 if (ctx->flags & IORING_SETUP_IOPOLL &&
9388 !(ctx->flags & IORING_SETUP_SQPOLL)) {
9389 ret = io_iopoll_check(ctx, min_complete);
9391 ret = io_cqring_wait(ctx, min_complete, sig, argsz, ts);
9396 percpu_ref_put(&ctx->refs);
9399 return submitted ? submitted : ret;
9402 #ifdef CONFIG_PROC_FS
9403 static int io_uring_show_cred(struct seq_file *m, unsigned int id,
9404 const struct cred *cred)
9406 struct user_namespace *uns = seq_user_ns(m);
9407 struct group_info *gi;
9412 seq_printf(m, "%5d\n", id);
9413 seq_put_decimal_ull(m, "\tUid:\t", from_kuid_munged(uns, cred->uid));
9414 seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->euid));
9415 seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->suid));
9416 seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->fsuid));
9417 seq_put_decimal_ull(m, "\n\tGid:\t", from_kgid_munged(uns, cred->gid));
9418 seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->egid));
9419 seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->sgid));
9420 seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->fsgid));
9421 seq_puts(m, "\n\tGroups:\t");
9422 gi = cred->group_info;
9423 for (g = 0; g < gi->ngroups; g++) {
9424 seq_put_decimal_ull(m, g ? " " : "",
9425 from_kgid_munged(uns, gi->gid[g]));
9427 seq_puts(m, "\n\tCapEff:\t");
9428 cap = cred->cap_effective;
9429 CAP_FOR_EACH_U32(__capi)
9430 seq_put_hex_ll(m, NULL, cap.cap[CAP_LAST_U32 - __capi], 8);
9435 static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m)
9437 struct io_sq_data *sq = NULL;
9442 * Avoid ABBA deadlock between the seq lock and the io_uring mutex,
9443 * since fdinfo case grabs it in the opposite direction of normal use
9444 * cases. If we fail to get the lock, we just don't iterate any
9445 * structures that could be going away outside the io_uring mutex.
9447 has_lock = mutex_trylock(&ctx->uring_lock);
9449 if (has_lock && (ctx->flags & IORING_SETUP_SQPOLL)) {
9455 seq_printf(m, "SqThread:\t%d\n", sq ? task_pid_nr(sq->thread) : -1);
9456 seq_printf(m, "SqThreadCpu:\t%d\n", sq ? task_cpu(sq->thread) : -1);
9457 seq_printf(m, "UserFiles:\t%u\n", ctx->nr_user_files);
9458 for (i = 0; has_lock && i < ctx->nr_user_files; i++) {
9459 struct file *f = io_file_from_index(ctx, i);
9462 seq_printf(m, "%5u: %s\n", i, file_dentry(f)->d_iname);
9464 seq_printf(m, "%5u: <none>\n", i);
9466 seq_printf(m, "UserBufs:\t%u\n", ctx->nr_user_bufs);
9467 for (i = 0; has_lock && i < ctx->nr_user_bufs; i++) {
9468 struct io_mapped_ubuf *buf = ctx->user_bufs[i];
9469 unsigned int len = buf->ubuf_end - buf->ubuf;
9471 seq_printf(m, "%5u: 0x%llx/%u\n", i, buf->ubuf, len);
9473 if (has_lock && !xa_empty(&ctx->personalities)) {
9474 unsigned long index;
9475 const struct cred *cred;
9477 seq_printf(m, "Personalities:\n");
9478 xa_for_each(&ctx->personalities, index, cred)
9479 io_uring_show_cred(m, index, cred);
9481 seq_printf(m, "PollList:\n");
9482 spin_lock_irq(&ctx->completion_lock);
9483 for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
9484 struct hlist_head *list = &ctx->cancel_hash[i];
9485 struct io_kiocb *req;
9487 hlist_for_each_entry(req, list, hash_node)
9488 seq_printf(m, " op=%d, task_works=%d\n", req->opcode,
9489 req->task->task_works != NULL);
9491 spin_unlock_irq(&ctx->completion_lock);
9493 mutex_unlock(&ctx->uring_lock);
9496 static void io_uring_show_fdinfo(struct seq_file *m, struct file *f)
9498 struct io_ring_ctx *ctx = f->private_data;
9500 if (percpu_ref_tryget(&ctx->refs)) {
9501 __io_uring_show_fdinfo(ctx, m);
9502 percpu_ref_put(&ctx->refs);
9507 static const struct file_operations io_uring_fops = {
9508 .release = io_uring_release,
9509 .mmap = io_uring_mmap,
9511 .get_unmapped_area = io_uring_nommu_get_unmapped_area,
9512 .mmap_capabilities = io_uring_nommu_mmap_capabilities,
9514 .poll = io_uring_poll,
9515 .fasync = io_uring_fasync,
9516 #ifdef CONFIG_PROC_FS
9517 .show_fdinfo = io_uring_show_fdinfo,
9521 static int io_allocate_scq_urings(struct io_ring_ctx *ctx,
9522 struct io_uring_params *p)
9524 struct io_rings *rings;
9525 size_t size, sq_array_offset;
9527 /* make sure these are sane, as we already accounted them */
9528 ctx->sq_entries = p->sq_entries;
9529 ctx->cq_entries = p->cq_entries;
9531 size = rings_size(p->sq_entries, p->cq_entries, &sq_array_offset);
9532 if (size == SIZE_MAX)
9535 rings = io_mem_alloc(size);
9540 ctx->sq_array = (u32 *)((char *)rings + sq_array_offset);
9541 rings->sq_ring_mask = p->sq_entries - 1;
9542 rings->cq_ring_mask = p->cq_entries - 1;
9543 rings->sq_ring_entries = p->sq_entries;
9544 rings->cq_ring_entries = p->cq_entries;
9546 size = array_size(sizeof(struct io_uring_sqe), p->sq_entries);
9547 if (size == SIZE_MAX) {
9548 io_mem_free(ctx->rings);
9553 ctx->sq_sqes = io_mem_alloc(size);
9554 if (!ctx->sq_sqes) {
9555 io_mem_free(ctx->rings);
9563 static int io_uring_install_fd(struct io_ring_ctx *ctx, struct file *file)
9567 fd = get_unused_fd_flags(O_RDWR | O_CLOEXEC);
9571 ret = io_uring_add_tctx_node(ctx);
9576 fd_install(fd, file);
9581 * Allocate an anonymous fd, this is what constitutes the application
9582 * visible backing of an io_uring instance. The application mmaps this
9583 * fd to gain access to the SQ/CQ ring details. If UNIX sockets are enabled,
9584 * we have to tie this fd to a socket for file garbage collection purposes.
9586 static struct file *io_uring_get_file(struct io_ring_ctx *ctx)
9589 #if defined(CONFIG_UNIX)
9592 ret = sock_create_kern(&init_net, PF_UNIX, SOCK_RAW, IPPROTO_IP,
9595 return ERR_PTR(ret);
9598 file = anon_inode_getfile("[io_uring]", &io_uring_fops, ctx,
9599 O_RDWR | O_CLOEXEC);
9600 #if defined(CONFIG_UNIX)
9602 sock_release(ctx->ring_sock);
9603 ctx->ring_sock = NULL;
9605 ctx->ring_sock->file = file;
9611 static int io_uring_create(unsigned entries, struct io_uring_params *p,
9612 struct io_uring_params __user *params)
9614 struct io_ring_ctx *ctx;
9620 if (entries > IORING_MAX_ENTRIES) {
9621 if (!(p->flags & IORING_SETUP_CLAMP))
9623 entries = IORING_MAX_ENTRIES;
9627 * Use twice as many entries for the CQ ring. It's possible for the
9628 * application to drive a higher depth than the size of the SQ ring,
9629 * since the sqes are only used at submission time. This allows for
9630 * some flexibility in overcommitting a bit. If the application has
9631 * set IORING_SETUP_CQSIZE, it will have passed in the desired number
9632 * of CQ ring entries manually.
9634 p->sq_entries = roundup_pow_of_two(entries);
9635 if (p->flags & IORING_SETUP_CQSIZE) {
9637 * If IORING_SETUP_CQSIZE is set, we do the same roundup
9638 * to a power-of-two, if it isn't already. We do NOT impose
9639 * any cq vs sq ring sizing.
9643 if (p->cq_entries > IORING_MAX_CQ_ENTRIES) {
9644 if (!(p->flags & IORING_SETUP_CLAMP))
9646 p->cq_entries = IORING_MAX_CQ_ENTRIES;
9648 p->cq_entries = roundup_pow_of_two(p->cq_entries);
9649 if (p->cq_entries < p->sq_entries)
9652 p->cq_entries = 2 * p->sq_entries;
9655 ctx = io_ring_ctx_alloc(p);
9658 ctx->compat = in_compat_syscall();
9659 if (!capable(CAP_IPC_LOCK))
9660 ctx->user = get_uid(current_user());
9663 * This is just grabbed for accounting purposes. When a process exits,
9664 * the mm is exited and dropped before the files, hence we need to hang
9665 * on to this mm purely for the purposes of being able to unaccount
9666 * memory (locked/pinned vm). It's not used for anything else.
9668 mmgrab(current->mm);
9669 ctx->mm_account = current->mm;
9671 ret = io_allocate_scq_urings(ctx, p);
9675 ret = io_sq_offload_create(ctx, p);
9678 /* always set a rsrc node */
9679 ret = io_rsrc_node_switch_start(ctx);
9682 io_rsrc_node_switch(ctx, NULL);
9684 memset(&p->sq_off, 0, sizeof(p->sq_off));
9685 p->sq_off.head = offsetof(struct io_rings, sq.head);
9686 p->sq_off.tail = offsetof(struct io_rings, sq.tail);
9687 p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask);
9688 p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries);
9689 p->sq_off.flags = offsetof(struct io_rings, sq_flags);
9690 p->sq_off.dropped = offsetof(struct io_rings, sq_dropped);
9691 p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings;
9693 memset(&p->cq_off, 0, sizeof(p->cq_off));
9694 p->cq_off.head = offsetof(struct io_rings, cq.head);
9695 p->cq_off.tail = offsetof(struct io_rings, cq.tail);
9696 p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask);
9697 p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries);
9698 p->cq_off.overflow = offsetof(struct io_rings, cq_overflow);
9699 p->cq_off.cqes = offsetof(struct io_rings, cqes);
9700 p->cq_off.flags = offsetof(struct io_rings, cq_flags);
9702 p->features = IORING_FEAT_SINGLE_MMAP | IORING_FEAT_NODROP |
9703 IORING_FEAT_SUBMIT_STABLE | IORING_FEAT_RW_CUR_POS |
9704 IORING_FEAT_CUR_PERSONALITY | IORING_FEAT_FAST_POLL |
9705 IORING_FEAT_POLL_32BITS | IORING_FEAT_SQPOLL_NONFIXED |
9706 IORING_FEAT_EXT_ARG | IORING_FEAT_NATIVE_WORKERS |
9707 IORING_FEAT_RSRC_TAGS;
9709 if (copy_to_user(params, p, sizeof(*p))) {
9714 file = io_uring_get_file(ctx);
9716 ret = PTR_ERR(file);
9721 * Install ring fd as the very last thing, so we don't risk someone
9722 * having closed it before we finish setup
9724 ret = io_uring_install_fd(ctx, file);
9726 /* fput will clean it up */
9731 trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags);
9734 io_ring_ctx_wait_and_kill(ctx);
9739 * Sets up an aio uring context, and returns the fd. Applications asks for a
9740 * ring size, we return the actual sq/cq ring sizes (among other things) in the
9741 * params structure passed in.
9743 static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
9745 struct io_uring_params p;
9748 if (copy_from_user(&p, params, sizeof(p)))
9750 for (i = 0; i < ARRAY_SIZE(p.resv); i++) {
9755 if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |
9756 IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE |
9757 IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ |
9758 IORING_SETUP_R_DISABLED))
9761 return io_uring_create(entries, &p, params);
9764 SYSCALL_DEFINE2(io_uring_setup, u32, entries,
9765 struct io_uring_params __user *, params)
9767 return io_uring_setup(entries, params);
9770 static int io_probe(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)
9772 struct io_uring_probe *p;
9776 size = struct_size(p, ops, nr_args);
9777 if (size == SIZE_MAX)
9779 p = kzalloc(size, GFP_KERNEL);
9784 if (copy_from_user(p, arg, size))
9787 if (memchr_inv(p, 0, size))
9790 p->last_op = IORING_OP_LAST - 1;
9791 if (nr_args > IORING_OP_LAST)
9792 nr_args = IORING_OP_LAST;
9794 for (i = 0; i < nr_args; i++) {
9796 if (!io_op_defs[i].not_supported)
9797 p->ops[i].flags = IO_URING_OP_SUPPORTED;
9802 if (copy_to_user(arg, p, size))
9809 static int io_register_personality(struct io_ring_ctx *ctx)
9811 const struct cred *creds;
9815 creds = get_current_cred();
9817 ret = xa_alloc_cyclic(&ctx->personalities, &id, (void *)creds,
9818 XA_LIMIT(0, USHRT_MAX), &ctx->pers_next, GFP_KERNEL);
9825 static int io_register_restrictions(struct io_ring_ctx *ctx, void __user *arg,
9826 unsigned int nr_args)
9828 struct io_uring_restriction *res;
9832 /* Restrictions allowed only if rings started disabled */
9833 if (!(ctx->flags & IORING_SETUP_R_DISABLED))
9836 /* We allow only a single restrictions registration */
9837 if (ctx->restrictions.registered)
9840 if (!arg || nr_args > IORING_MAX_RESTRICTIONS)
9843 size = array_size(nr_args, sizeof(*res));
9844 if (size == SIZE_MAX)
9847 res = memdup_user(arg, size);
9849 return PTR_ERR(res);
9853 for (i = 0; i < nr_args; i++) {
9854 switch (res[i].opcode) {
9855 case IORING_RESTRICTION_REGISTER_OP:
9856 if (res[i].register_op >= IORING_REGISTER_LAST) {
9861 __set_bit(res[i].register_op,
9862 ctx->restrictions.register_op);
9864 case IORING_RESTRICTION_SQE_OP:
9865 if (res[i].sqe_op >= IORING_OP_LAST) {
9870 __set_bit(res[i].sqe_op, ctx->restrictions.sqe_op);
9872 case IORING_RESTRICTION_SQE_FLAGS_ALLOWED:
9873 ctx->restrictions.sqe_flags_allowed = res[i].sqe_flags;
9875 case IORING_RESTRICTION_SQE_FLAGS_REQUIRED:
9876 ctx->restrictions.sqe_flags_required = res[i].sqe_flags;
9885 /* Reset all restrictions if an error happened */
9887 memset(&ctx->restrictions, 0, sizeof(ctx->restrictions));
9889 ctx->restrictions.registered = true;
9895 static int io_register_enable_rings(struct io_ring_ctx *ctx)
9897 if (!(ctx->flags & IORING_SETUP_R_DISABLED))
9900 if (ctx->restrictions.registered)
9901 ctx->restricted = 1;
9903 ctx->flags &= ~IORING_SETUP_R_DISABLED;
9904 if (ctx->sq_data && wq_has_sleeper(&ctx->sq_data->wait))
9905 wake_up(&ctx->sq_data->wait);
9909 static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,
9910 struct io_uring_rsrc_update2 *up,
9918 if (check_add_overflow(up->offset, nr_args, &tmp))
9920 err = io_rsrc_node_switch_start(ctx);
9925 case IORING_RSRC_FILE:
9926 return __io_sqe_files_update(ctx, up, nr_args);
9927 case IORING_RSRC_BUFFER:
9928 return __io_sqe_buffers_update(ctx, up, nr_args);
9933 static int io_register_files_update(struct io_ring_ctx *ctx, void __user *arg,
9936 struct io_uring_rsrc_update2 up;
9940 memset(&up, 0, sizeof(up));
9941 if (copy_from_user(&up, arg, sizeof(struct io_uring_rsrc_update)))
9943 return __io_register_rsrc_update(ctx, IORING_RSRC_FILE, &up, nr_args);
9946 static int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg,
9947 unsigned size, unsigned type)
9949 struct io_uring_rsrc_update2 up;
9951 if (size != sizeof(up))
9953 if (copy_from_user(&up, arg, sizeof(up)))
9955 if (!up.nr || up.resv)
9957 return __io_register_rsrc_update(ctx, type, &up, up.nr);
9960 static int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg,
9961 unsigned int size, unsigned int type)
9963 struct io_uring_rsrc_register rr;
9965 /* keep it extendible */
9966 if (size != sizeof(rr))
9969 memset(&rr, 0, sizeof(rr));
9970 if (copy_from_user(&rr, arg, size))
9972 if (!rr.nr || rr.resv || rr.resv2)
9976 case IORING_RSRC_FILE:
9977 return io_sqe_files_register(ctx, u64_to_user_ptr(rr.data),
9978 rr.nr, u64_to_user_ptr(rr.tags));
9979 case IORING_RSRC_BUFFER:
9980 return io_sqe_buffers_register(ctx, u64_to_user_ptr(rr.data),
9981 rr.nr, u64_to_user_ptr(rr.tags));
9986 static int io_register_iowq_aff(struct io_ring_ctx *ctx, void __user *arg,
9989 struct io_uring_task *tctx = current->io_uring;
9990 cpumask_var_t new_mask;
9993 if (!tctx || !tctx->io_wq)
9996 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
9999 cpumask_clear(new_mask);
10000 if (len > cpumask_size())
10001 len = cpumask_size();
10003 if (copy_from_user(new_mask, arg, len)) {
10004 free_cpumask_var(new_mask);
10008 ret = io_wq_cpu_affinity(tctx->io_wq, new_mask);
10009 free_cpumask_var(new_mask);
10013 static int io_unregister_iowq_aff(struct io_ring_ctx *ctx)
10015 struct io_uring_task *tctx = current->io_uring;
10017 if (!tctx || !tctx->io_wq)
10020 return io_wq_cpu_affinity(tctx->io_wq, NULL);
10023 static bool io_register_op_must_quiesce(int op)
10026 case IORING_REGISTER_BUFFERS:
10027 case IORING_UNREGISTER_BUFFERS:
10028 case IORING_REGISTER_FILES:
10029 case IORING_UNREGISTER_FILES:
10030 case IORING_REGISTER_FILES_UPDATE:
10031 case IORING_REGISTER_PROBE:
10032 case IORING_REGISTER_PERSONALITY:
10033 case IORING_UNREGISTER_PERSONALITY:
10034 case IORING_REGISTER_FILES2:
10035 case IORING_REGISTER_FILES_UPDATE2:
10036 case IORING_REGISTER_BUFFERS2:
10037 case IORING_REGISTER_BUFFERS_UPDATE:
10038 case IORING_REGISTER_IOWQ_AFF:
10039 case IORING_UNREGISTER_IOWQ_AFF:
10046 static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
10047 void __user *arg, unsigned nr_args)
10048 __releases(ctx->uring_lock)
10049 __acquires(ctx->uring_lock)
10054 * We're inside the ring mutex, if the ref is already dying, then
10055 * someone else killed the ctx or is already going through
10056 * io_uring_register().
10058 if (percpu_ref_is_dying(&ctx->refs))
10061 if (ctx->restricted) {
10062 if (opcode >= IORING_REGISTER_LAST)
10064 opcode = array_index_nospec(opcode, IORING_REGISTER_LAST);
10065 if (!test_bit(opcode, ctx->restrictions.register_op))
10069 if (io_register_op_must_quiesce(opcode)) {
10070 percpu_ref_kill(&ctx->refs);
10073 * Drop uring mutex before waiting for references to exit. If
10074 * another thread is currently inside io_uring_enter() it might
10075 * need to grab the uring_lock to make progress. If we hold it
10076 * here across the drain wait, then we can deadlock. It's safe
10077 * to drop the mutex here, since no new references will come in
10078 * after we've killed the percpu ref.
10080 mutex_unlock(&ctx->uring_lock);
10082 ret = wait_for_completion_interruptible(&ctx->ref_comp);
10085 ret = io_run_task_work_sig();
10089 mutex_lock(&ctx->uring_lock);
10092 io_refs_resurrect(&ctx->refs, &ctx->ref_comp);
10098 case IORING_REGISTER_BUFFERS:
10099 ret = io_sqe_buffers_register(ctx, arg, nr_args, NULL);
10101 case IORING_UNREGISTER_BUFFERS:
10103 if (arg || nr_args)
10105 ret = io_sqe_buffers_unregister(ctx);
10107 case IORING_REGISTER_FILES:
10108 ret = io_sqe_files_register(ctx, arg, nr_args, NULL);
10110 case IORING_UNREGISTER_FILES:
10112 if (arg || nr_args)
10114 ret = io_sqe_files_unregister(ctx);
10116 case IORING_REGISTER_FILES_UPDATE:
10117 ret = io_register_files_update(ctx, arg, nr_args);
10119 case IORING_REGISTER_EVENTFD:
10120 case IORING_REGISTER_EVENTFD_ASYNC:
10124 ret = io_eventfd_register(ctx, arg);
10127 if (opcode == IORING_REGISTER_EVENTFD_ASYNC)
10128 ctx->eventfd_async = 1;
10130 ctx->eventfd_async = 0;
10132 case IORING_UNREGISTER_EVENTFD:
10134 if (arg || nr_args)
10136 ret = io_eventfd_unregister(ctx);
10138 case IORING_REGISTER_PROBE:
10140 if (!arg || nr_args > 256)
10142 ret = io_probe(ctx, arg, nr_args);
10144 case IORING_REGISTER_PERSONALITY:
10146 if (arg || nr_args)
10148 ret = io_register_personality(ctx);
10150 case IORING_UNREGISTER_PERSONALITY:
10154 ret = io_unregister_personality(ctx, nr_args);
10156 case IORING_REGISTER_ENABLE_RINGS:
10158 if (arg || nr_args)
10160 ret = io_register_enable_rings(ctx);
10162 case IORING_REGISTER_RESTRICTIONS:
10163 ret = io_register_restrictions(ctx, arg, nr_args);
10165 case IORING_REGISTER_FILES2:
10166 ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_FILE);
10168 case IORING_REGISTER_FILES_UPDATE2:
10169 ret = io_register_rsrc_update(ctx, arg, nr_args,
10172 case IORING_REGISTER_BUFFERS2:
10173 ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_BUFFER);
10175 case IORING_REGISTER_BUFFERS_UPDATE:
10176 ret = io_register_rsrc_update(ctx, arg, nr_args,
10177 IORING_RSRC_BUFFER);
10179 case IORING_REGISTER_IOWQ_AFF:
10181 if (!arg || !nr_args)
10183 ret = io_register_iowq_aff(ctx, arg, nr_args);
10185 case IORING_UNREGISTER_IOWQ_AFF:
10187 if (arg || nr_args)
10189 ret = io_unregister_iowq_aff(ctx);
10196 if (io_register_op_must_quiesce(opcode)) {
10197 /* bring the ctx back to life */
10198 percpu_ref_reinit(&ctx->refs);
10199 reinit_completion(&ctx->ref_comp);
10204 SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
10205 void __user *, arg, unsigned int, nr_args)
10207 struct io_ring_ctx *ctx;
10216 if (f.file->f_op != &io_uring_fops)
10219 ctx = f.file->private_data;
10221 io_run_task_work();
10223 mutex_lock(&ctx->uring_lock);
10224 ret = __io_uring_register(ctx, opcode, arg, nr_args);
10225 mutex_unlock(&ctx->uring_lock);
10226 trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs,
10227 ctx->cq_ev_fd != NULL, ret);
10233 static int __init io_uring_init(void)
10235 #define __BUILD_BUG_VERIFY_ELEMENT(stype, eoffset, etype, ename) do { \
10236 BUILD_BUG_ON(offsetof(stype, ename) != eoffset); \
10237 BUILD_BUG_ON(sizeof(etype) != sizeof_field(stype, ename)); \
10240 #define BUILD_BUG_SQE_ELEM(eoffset, etype, ename) \
10241 __BUILD_BUG_VERIFY_ELEMENT(struct io_uring_sqe, eoffset, etype, ename)
10242 BUILD_BUG_ON(sizeof(struct io_uring_sqe) != 64);
10243 BUILD_BUG_SQE_ELEM(0, __u8, opcode);
10244 BUILD_BUG_SQE_ELEM(1, __u8, flags);
10245 BUILD_BUG_SQE_ELEM(2, __u16, ioprio);
10246 BUILD_BUG_SQE_ELEM(4, __s32, fd);
10247 BUILD_BUG_SQE_ELEM(8, __u64, off);
10248 BUILD_BUG_SQE_ELEM(8, __u64, addr2);
10249 BUILD_BUG_SQE_ELEM(16, __u64, addr);
10250 BUILD_BUG_SQE_ELEM(16, __u64, splice_off_in);
10251 BUILD_BUG_SQE_ELEM(24, __u32, len);
10252 BUILD_BUG_SQE_ELEM(28, __kernel_rwf_t, rw_flags);
10253 BUILD_BUG_SQE_ELEM(28, /* compat */ int, rw_flags);
10254 BUILD_BUG_SQE_ELEM(28, /* compat */ __u32, rw_flags);
10255 BUILD_BUG_SQE_ELEM(28, __u32, fsync_flags);
10256 BUILD_BUG_SQE_ELEM(28, /* compat */ __u16, poll_events);
10257 BUILD_BUG_SQE_ELEM(28, __u32, poll32_events);
10258 BUILD_BUG_SQE_ELEM(28, __u32, sync_range_flags);
10259 BUILD_BUG_SQE_ELEM(28, __u32, msg_flags);
10260 BUILD_BUG_SQE_ELEM(28, __u32, timeout_flags);
10261 BUILD_BUG_SQE_ELEM(28, __u32, accept_flags);
10262 BUILD_BUG_SQE_ELEM(28, __u32, cancel_flags);
10263 BUILD_BUG_SQE_ELEM(28, __u32, open_flags);
10264 BUILD_BUG_SQE_ELEM(28, __u32, statx_flags);
10265 BUILD_BUG_SQE_ELEM(28, __u32, fadvise_advice);
10266 BUILD_BUG_SQE_ELEM(28, __u32, splice_flags);
10267 BUILD_BUG_SQE_ELEM(32, __u64, user_data);
10268 BUILD_BUG_SQE_ELEM(40, __u16, buf_index);
10269 BUILD_BUG_SQE_ELEM(40, __u16, buf_group);
10270 BUILD_BUG_SQE_ELEM(42, __u16, personality);
10271 BUILD_BUG_SQE_ELEM(44, __s32, splice_fd_in);
10273 BUILD_BUG_ON(sizeof(struct io_uring_files_update) !=
10274 sizeof(struct io_uring_rsrc_update));
10275 BUILD_BUG_ON(sizeof(struct io_uring_rsrc_update) >
10276 sizeof(struct io_uring_rsrc_update2));
10277 /* should fit into one byte */
10278 BUILD_BUG_ON(SQE_VALID_FLAGS >= (1 << 8));
10280 BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST);
10281 BUILD_BUG_ON(__REQ_F_LAST_BIT >= 8 * sizeof(int));
10283 req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC |
10287 __initcall(io_uring_init);