struct io_comp_state {
struct io_kiocb *reqs[IO_COMPL_BATCH];
unsigned int nr;
- unsigned int locked_free_nr;
/* inline/task_work completion list, under ->uring_lock */
struct list_head free_list;
- /* IRQ completion list, under ->completion_lock */
- struct list_head locked_free_list;
};
struct io_submit_link {
} ____cacheline_aligned_in_smp;
struct io_submit_state submit_state;
+ /* IRQ completion list, under ->completion_lock */
+ struct list_head locked_free_list;
+ unsigned int locked_free_nr;
struct io_rings *rings;
init_llist_head(&ctx->rsrc_put_llist);
INIT_LIST_HEAD(&ctx->tctx_list);
INIT_LIST_HEAD(&ctx->submit_state.comp.free_list);
- INIT_LIST_HEAD(&ctx->submit_state.comp.locked_free_list);
+ INIT_LIST_HEAD(&ctx->locked_free_list);
return ctx;
err:
kfree(ctx->dummy_ubuf);
* free_list cache.
*/
if (req_ref_put_and_test(req)) {
- struct io_comp_state *cs = &ctx->submit_state.comp;
-
if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) {
if (req->flags & (REQ_F_LINK_TIMEOUT | REQ_F_FAIL))
io_disarm_next(req);
}
io_dismantle_req(req);
io_put_task(req->task, 1);
- list_add(&req->compl.list, &cs->locked_free_list);
- cs->locked_free_nr++;
+ list_add(&req->compl.list, &ctx->locked_free_list);
+ ctx->locked_free_nr++;
} else {
if (!percpu_ref_tryget(&ctx->refs))
req = NULL;
struct io_comp_state *cs)
{
spin_lock_irq(&ctx->completion_lock);
- list_splice_init(&cs->locked_free_list, &cs->free_list);
- cs->locked_free_nr = 0;
+ list_splice_init(&ctx->locked_free_list, &cs->free_list);
+ ctx->locked_free_nr = 0;
spin_unlock_irq(&ctx->completion_lock);
}
* locked cache, grab the lock and move them over to our submission
* side cache.
*/
- if (READ_ONCE(cs->locked_free_nr) > IO_COMPL_BATCH)
+ if (READ_ONCE(ctx->locked_free_nr) > IO_COMPL_BATCH)
io_flush_cached_locked_reqs(ctx, cs);
nr = state->free_reqs;