virtio-blk: support completion batching for the IRQ path

[linux-2.6-microblaze.git] / drivers / block / virtio_blk.c

diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c
index f2fb49a..23d4fa3 100644 (file)
--- a/drivers/block/virtio_blk.c
+++ b/drivers/block/virtio_blk.c
@@ -334,33 +334,63 @@ static inline void virtblk_request_done(struct request *req)
        blk_mq_end_request(req, status);
 }
 
+static void virtblk_complete_batch(struct io_comp_batch *iob)
+{
+       struct request *req;
+
+       rq_list_for_each(&iob->req_list, req) {
+               virtblk_unmap_data(req, blk_mq_rq_to_pdu(req));
+               virtblk_cleanup_cmd(req);
+       }
+       blk_mq_end_request_batch(iob);
+}
+
+static int virtblk_handle_req(struct virtio_blk_vq *vq,
+                             struct io_comp_batch *iob)
+{
+       struct virtblk_req *vbr;
+       int req_done = 0;
+       unsigned int len;
+
+       while ((vbr = virtqueue_get_buf(vq->vq, &len)) != NULL) {
+               struct request *req = blk_mq_rq_from_pdu(vbr);
+
+               if (likely(!blk_should_fake_timeout(req->q)) &&
+                   !blk_mq_complete_request_remote(req) &&
+                   !blk_mq_add_to_batch(req, iob, vbr->status,
+                                        virtblk_complete_batch))
+                       virtblk_request_done(req);
+               req_done++;
+       }
+
+       return req_done;
+}
+
 static void virtblk_done(struct virtqueue *vq)
 {
        struct virtio_blk *vblk = vq->vdev->priv;
-       bool req_done = false;
-       int qid = vq->index;
-       struct virtblk_req *vbr;
+       struct virtio_blk_vq *vblk_vq = &vblk->vqs[vq->index];
+       int req_done = 0;
        unsigned long flags;
-       unsigned int len;
+       DEFINE_IO_COMP_BATCH(iob);
 
-       spin_lock_irqsave(&vblk->vqs[qid].lock, flags);
+       spin_lock_irqsave(&vblk_vq->lock, flags);
        do {
                virtqueue_disable_cb(vq);
-               while ((vbr = virtqueue_get_buf(vblk->vqs[qid].vq, &len)) != NULL) {
-                       struct request *req = blk_mq_rq_from_pdu(vbr);
+               req_done += virtblk_handle_req(vblk_vq, &iob);
 
-                       if (likely(!blk_should_fake_timeout(req->q)))
-                               blk_mq_complete_request(req);
-                       req_done = true;
-               }
                if (unlikely(virtqueue_is_broken(vq)))
                        break;
        } while (!virtqueue_enable_cb(vq));
 
-       /* In case queue is stopped waiting for more buffers. */
-       if (req_done)
+       if (req_done) {
+               if (!rq_list_empty(iob.req_list))
+                       iob.complete(&iob);
+
+               /* In case queue is stopped waiting for more buffers. */
                blk_mq_start_stopped_hw_queues(vblk->disk->queue, true);
-       spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags);
+       }
+       spin_unlock_irqrestore(&vblk_vq->lock, flags);
 }
 
 static void virtio_commit_rqs(struct blk_mq_hw_ctx *hctx)
@@ -1176,37 +1206,15 @@ static void virtblk_map_queues(struct blk_mq_tag_set *set)
        }
 }
 
-static void virtblk_complete_batch(struct io_comp_batch *iob)
-{
-       struct request *req;
-
-       rq_list_for_each(&iob->req_list, req) {
-               virtblk_unmap_data(req, blk_mq_rq_to_pdu(req));
-               virtblk_cleanup_cmd(req);
-       }
-       blk_mq_end_request_batch(iob);
-}
-
 static int virtblk_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
 {
        struct virtio_blk *vblk = hctx->queue->queuedata;
        struct virtio_blk_vq *vq = get_virtio_blk_vq(hctx);
-       struct virtblk_req *vbr;
        unsigned long flags;
-       unsigned int len;
        int found = 0;
 
        spin_lock_irqsave(&vq->lock, flags);
-
-       while ((vbr = virtqueue_get_buf(vq->vq, &len)) != NULL) {
-               struct request *req = blk_mq_rq_from_pdu(vbr);
-
-               found++;
-               if (!blk_mq_complete_request_remote(req) &&
-                   !blk_mq_add_to_batch(req, iob, vbr->status,
-                                               virtblk_complete_batch))
-                       virtblk_request_done(req);
-       }
+       found = virtblk_handle_req(vq, iob);
 
        if (found)
                blk_mq_start_stopped_hw_queues(vblk->disk->queue, true);