1 #include <linux/kernel.h>
2 #include <linux/errno.h>
3 #include <linux/file.h>
4 #include <linux/slab.h>
6 #include <linux/io_uring.h>
12 static void __io_notif_complete_tw(struct io_kiocb *notif, bool *locked)
14 struct io_notif_data *nd = io_notif_to_data(notif);
15 struct io_ring_ctx *ctx = notif->ctx;
17 if (nd->account_pages && ctx->user) {
18 __io_unaccount_mem(ctx->user, nd->account_pages);
19 nd->account_pages = 0;
21 io_req_task_complete(notif, locked);
24 static inline void io_notif_complete(struct io_kiocb *notif)
25 __must_hold(¬if->ctx->uring_lock)
29 __io_notif_complete_tw(notif, &locked);
32 static void io_uring_tx_zerocopy_callback(struct sk_buff *skb,
33 struct ubuf_info *uarg,
36 struct io_notif_data *nd = container_of(uarg, struct io_notif_data, uarg);
37 struct io_kiocb *notif = cmd_to_io_kiocb(nd);
39 if (refcount_dec_and_test(&uarg->refcnt)) {
40 notif->io_task_work.func = __io_notif_complete_tw;
41 io_req_task_work_add(notif);
45 struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx,
46 struct io_notif_slot *slot)
47 __must_hold(&ctx->uring_lock)
49 struct io_kiocb *notif;
50 struct io_notif_data *nd;
52 if (unlikely(!io_alloc_req_refill(ctx)))
54 notif = io_alloc_req(ctx);
55 notif->opcode = IORING_OP_NOP;
58 notif->task = current;
60 notif->rsrc_node = NULL;
61 io_req_set_rsrc_node(notif, ctx, 0);
62 notif->cqe.user_data = slot->tag;
63 notif->cqe.flags = slot->seq++;
66 nd = io_notif_to_data(notif);
67 nd->account_pages = 0;
68 nd->uarg.flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN;
69 nd->uarg.callback = io_uring_tx_zerocopy_callback;
70 /* master ref owned by io_notif_slot, will be dropped on flush */
71 refcount_set(&nd->uarg.refcnt, 1);
75 void io_notif_slot_flush(struct io_notif_slot *slot)
76 __must_hold(&slot->notif->ctx->uring_lock)
78 struct io_kiocb *notif = slot->notif;
79 struct io_notif_data *nd = io_notif_to_data(notif);
83 /* drop slot's master ref */
84 if (refcount_dec_and_test(&nd->uarg.refcnt)) {
85 notif->io_task_work.func = __io_notif_complete_tw;
86 io_req_task_work_add(notif);