if (starget->can_queue > 0)
atomic_dec(&starget->target_busy);
- atomic_dec(&sdev->device_busy);
+ sbitmap_put(&sdev->budget_map, cmd->budget_token);
+ cmd->budget_token = -1;
}
static void scsi_kick_queue(struct request_queue *q)
static inline bool scsi_device_is_busy(struct scsi_device *sdev)
{
- if (atomic_read(&sdev->device_busy) >= sdev->queue_depth)
+ if (scsi_device_busy(sdev) >= sdev->queue_depth)
return true;
if (atomic_read(&sdev->device_blocked) > 0)
return true;
}
/**
- * scsi_alloc_sgtables - allocate S/G tables for a command
- * @cmd: command descriptor we wish to initialize
+ * scsi_alloc_sgtables - Allocate and initialize data and integrity scatterlists
+ * @cmd: SCSI command data structure to initialize.
+ *
+ * Initializes @cmd->sdb and also @cmd->prot_sdb if data integrity is enabled
+ * for @cmd.
*
* Returns:
* * BLK_STS_OK - on success
unsigned long jiffies_at_alloc;
int retries, to_clear;
bool in_flight;
+ int budget_token = cmd->budget_token;
if (!blk_rq_is_scsi(rq) && !(flags & SCMD_INITIALIZED)) {
flags |= SCMD_INITIALIZED;
cmd->retries = retries;
if (in_flight)
__set_bit(SCMD_STATE_INFLIGHT, &cmd->state);
+ cmd->budget_token = budget_token;
}
}
/*
- * scsi_dev_queue_ready: if we can send requests to sdev, return 1 else
- * return 0.
- *
- * Called with the queue_lock held.
+ * scsi_dev_queue_ready: if we can send requests to sdev, assign one token
+ * and return the token else return -1.
*/
static inline int scsi_dev_queue_ready(struct request_queue *q,
struct scsi_device *sdev)
{
- unsigned int busy;
+ int token;
- busy = atomic_inc_return(&sdev->device_busy) - 1;
+ token = sbitmap_get(&sdev->budget_map);
if (atomic_read(&sdev->device_blocked)) {
- if (busy)
+ if (token < 0)
+ goto out;
+
+ if (scsi_device_busy(sdev) > 1)
goto out_dec;
/*
"unblocking device at zero depth\n"));
}
- if (busy >= sdev->queue_depth)
- goto out_dec;
-
- return 1;
+ return token;
out_dec:
- atomic_dec(&sdev->device_busy);
- return 0;
+ if (token >= 0)
+ sbitmap_put(&sdev->budget_map, token);
+out:
+ return -1;
}
/*
return false;
}
-static void scsi_softirq_done(struct request *rq)
+/*
+ * Block layer request completion callback. May be called from interrupt
+ * context.
+ */
+static void scsi_complete(struct request *rq)
{
struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);
- int disposition;
+ enum scsi_disposition disposition;
INIT_LIST_HEAD(&cmd->eh_entry);
blk_mq_complete_request(cmd->request);
}
-static void scsi_mq_put_budget(struct request_queue *q)
+static void scsi_mq_put_budget(struct request_queue *q, int budget_token)
{
struct scsi_device *sdev = q->queuedata;
- atomic_dec(&sdev->device_busy);
+ sbitmap_put(&sdev->budget_map, budget_token);
}
-static bool scsi_mq_get_budget(struct request_queue *q)
+static int scsi_mq_get_budget(struct request_queue *q)
{
struct scsi_device *sdev = q->queuedata;
+ int token = scsi_dev_queue_ready(q, sdev);
- if (scsi_dev_queue_ready(q, sdev))
- return true;
+ if (token >= 0)
+ return token;
atomic_inc(&sdev->restarts);
* the .restarts flag, and the request queue will be run for handling
* this request, see scsi_end_request().
*/
- if (unlikely(atomic_read(&sdev->device_busy) == 0 &&
+ if (unlikely(scsi_device_busy(sdev) == 0 &&
!scsi_device_blocked(sdev)))
blk_mq_delay_run_hw_queues(sdev->request_queue, SCSI_QUEUE_DELAY);
- return false;
+ return -1;
+}
+
+static void scsi_mq_set_rq_budget_token(struct request *req, int token)
+{
+ struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
+
+ cmd->budget_token = token;
+}
+
+static int scsi_mq_get_rq_budget_token(struct request *req)
+{
+ struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
+
+ return cmd->budget_token;
}
static blk_status_t scsi_queue_rq(struct blk_mq_hw_ctx *hctx,
blk_status_t ret;
int reason;
+ WARN_ON_ONCE(cmd->budget_token < 0);
+
/*
* If the device is not in running state we will reject some or all
* commands.
if (scsi_target(sdev)->can_queue > 0)
atomic_dec(&scsi_target(sdev)->target_busy);
out_put_budget:
- scsi_mq_put_budget(q);
+ scsi_mq_put_budget(q, cmd->budget_token);
+ cmd->budget_token = -1;
switch (ret) {
case BLK_STS_OK:
break;
cmd->sense_buffer);
}
+
+static int scsi_mq_poll(struct blk_mq_hw_ctx *hctx)
+{
+ struct Scsi_Host *shost = hctx->driver_data;
+
+ if (shost->hostt->mq_poll)
+ return shost->hostt->mq_poll(shost, hctx->queue_num);
+
+ return 0;
+}
+
+static int scsi_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
+ unsigned int hctx_idx)
+{
+ struct Scsi_Host *shost = data;
+
+ hctx->driver_data = shost;
+ return 0;
+}
+
static int scsi_map_queues(struct blk_mq_tag_set *set)
{
struct Scsi_Host *shost = container_of(set, struct Scsi_Host, tag_set);
.get_budget = scsi_mq_get_budget,
.put_budget = scsi_mq_put_budget,
.queue_rq = scsi_queue_rq,
- .complete = scsi_softirq_done,
+ .complete = scsi_complete,
.timeout = scsi_timeout,
#ifdef CONFIG_BLK_DEBUG_FS
.show_rq = scsi_show_rq,
.cleanup_rq = scsi_cleanup_rq,
.busy = scsi_mq_lld_busy,
.map_queues = scsi_map_queues,
+ .init_hctx = scsi_init_hctx,
+ .poll = scsi_mq_poll,
+ .set_rq_budget_token = scsi_mq_set_rq_budget_token,
+ .get_rq_budget_token = scsi_mq_get_rq_budget_token,
};
static void scsi_commit_rqs(struct blk_mq_hw_ctx *hctx)
{
- struct request_queue *q = hctx->queue;
- struct scsi_device *sdev = q->queuedata;
- struct Scsi_Host *shost = sdev->host;
+ struct Scsi_Host *shost = hctx->driver_data;
shost->hostt->commit_rqs(shost, hctx->queue_num);
}
.put_budget = scsi_mq_put_budget,
.queue_rq = scsi_queue_rq,
.commit_rqs = scsi_commit_rqs,
- .complete = scsi_softirq_done,
+ .complete = scsi_complete,
.timeout = scsi_timeout,
#ifdef CONFIG_BLK_DEBUG_FS
.show_rq = scsi_show_rq,
.cleanup_rq = scsi_cleanup_rq,
.busy = scsi_mq_lld_busy,
.map_queues = scsi_map_queues,
+ .init_hctx = scsi_init_hctx,
+ .poll = scsi_mq_poll,
+ .set_rq_budget_token = scsi_mq_set_rq_budget_token,
+ .get_rq_budget_token = scsi_mq_get_rq_budget_token,
};
struct request_queue *scsi_mq_alloc_queue(struct scsi_device *sdev)
else
tag_set->ops = &scsi_mq_ops_no_commit;
tag_set->nr_hw_queues = shost->nr_hw_queues ? : 1;
+ tag_set->nr_maps = shost->nr_maps ? : 1;
tag_set->queue_depth = shost->can_queue;
tag_set->cmd_size = cmd_size;
tag_set->numa_node = NUMA_NO_NODE;
projects / linux-2.6-microblaze.git / blobdiff