summary |
shortlog |
log |
commit | commitdiff |
tree
raw |
patch |
inline | side by side (from parent 1:
1c0706a)
Introduce pointers for the blk_mq_tags regular and reserved bitmap tags,
with the goal of later being able to use a common shared tag bitmap across
all HW contexts in a set.
Signed-off-by: John Garry <john.garry@huawei.com>
Tested-by: Don Brace<don.brace@microsemi.com> #SCSI resv cmds patches used
Tested-by: Douglas Gilbert <dgilbert@interlog.com>
Reviewed-by: Hannes Reinecke <hare@suse.de>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
struct blk_mq_tags *tags = hctx->sched_tags;
unsigned int min_shallow;
struct blk_mq_tags *tags = hctx->sched_tags;
unsigned int min_shallow;
- min_shallow = bfq_update_depths(bfqd, &tags->bitmap_tags);
- sbitmap_queue_min_shallow_depth(&tags->bitmap_tags, min_shallow);
+ min_shallow = bfq_update_depths(bfqd, tags->bitmap_tags);
+ sbitmap_queue_min_shallow_depth(tags->bitmap_tags, min_shallow);
}
static int bfq_init_hctx(struct blk_mq_hw_ctx *hctx, unsigned int index)
}
static int bfq_init_hctx(struct blk_mq_hw_ctx *hctx, unsigned int index)
atomic_read(&tags->active_queues));
seq_puts(m, "\nbitmap_tags:\n");
atomic_read(&tags->active_queues));
seq_puts(m, "\nbitmap_tags:\n");
- sbitmap_queue_show(&tags->bitmap_tags, m);
+ sbitmap_queue_show(tags->bitmap_tags, m);
if (tags->nr_reserved_tags) {
seq_puts(m, "\nbreserved_tags:\n");
if (tags->nr_reserved_tags) {
seq_puts(m, "\nbreserved_tags:\n");
- sbitmap_queue_show(&tags->breserved_tags, m);
+ sbitmap_queue_show(tags->breserved_tags, m);
if (res)
goto out;
if (hctx->tags)
if (res)
goto out;
if (hctx->tags)
- sbitmap_bitmap_show(&hctx->tags->bitmap_tags.sb, m);
+ sbitmap_bitmap_show(&hctx->tags->bitmap_tags->sb, m);
mutex_unlock(&q->sysfs_lock);
out:
mutex_unlock(&q->sysfs_lock);
out:
if (res)
goto out;
if (hctx->sched_tags)
if (res)
goto out;
if (hctx->sched_tags)
- sbitmap_bitmap_show(&hctx->sched_tags->bitmap_tags.sb, m);
+ sbitmap_bitmap_show(&hctx->sched_tags->bitmap_tags->sb, m);
mutex_unlock(&q->sysfs_lock);
out:
mutex_unlock(&q->sysfs_lock);
out:
*/
void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags, bool include_reserve)
{
*/
void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags, bool include_reserve)
{
- sbitmap_queue_wake_all(&tags->bitmap_tags);
+ sbitmap_queue_wake_all(tags->bitmap_tags);
- sbitmap_queue_wake_all(&tags->breserved_tags);
+ sbitmap_queue_wake_all(tags->breserved_tags);
WARN_ON_ONCE(1);
return BLK_MQ_NO_TAG;
}
WARN_ON_ONCE(1);
return BLK_MQ_NO_TAG;
}
- bt = &tags->breserved_tags;
+ bt = tags->breserved_tags;
- bt = &tags->bitmap_tags;
+ bt = tags->bitmap_tags;
tag_offset = tags->nr_reserved_tags;
}
tag_offset = tags->nr_reserved_tags;
}
data->ctx);
tags = blk_mq_tags_from_data(data);
if (data->flags & BLK_MQ_REQ_RESERVED)
data->ctx);
tags = blk_mq_tags_from_data(data);
if (data->flags & BLK_MQ_REQ_RESERVED)
- bt = &tags->breserved_tags;
+ bt = tags->breserved_tags;
- bt = &tags->bitmap_tags;
+ bt = tags->bitmap_tags;
/*
* If destination hw queue is changed, fake wake up on
/*
* If destination hw queue is changed, fake wake up on
const int real_tag = tag - tags->nr_reserved_tags;
BUG_ON(real_tag >= tags->nr_tags);
const int real_tag = tag - tags->nr_reserved_tags;
BUG_ON(real_tag >= tags->nr_tags);
- sbitmap_queue_clear(&tags->bitmap_tags, real_tag, ctx->cpu);
+ sbitmap_queue_clear(tags->bitmap_tags, real_tag, ctx->cpu);
} else {
BUG_ON(tag >= tags->nr_reserved_tags);
} else {
BUG_ON(tag >= tags->nr_reserved_tags);
- sbitmap_queue_clear(&tags->breserved_tags, tag, ctx->cpu);
+ sbitmap_queue_clear(tags->breserved_tags, tag, ctx->cpu);
WARN_ON_ONCE(flags & BT_TAG_ITER_RESERVED);
if (tags->nr_reserved_tags)
WARN_ON_ONCE(flags & BT_TAG_ITER_RESERVED);
if (tags->nr_reserved_tags)
- bt_tags_for_each(tags, &tags->breserved_tags, fn, priv,
+ bt_tags_for_each(tags, tags->breserved_tags, fn, priv,
flags | BT_TAG_ITER_RESERVED);
flags | BT_TAG_ITER_RESERVED);
- bt_tags_for_each(tags, &tags->bitmap_tags, fn, priv, flags);
+ bt_tags_for_each(tags, tags->bitmap_tags, fn, priv, flags);
continue;
if (tags->nr_reserved_tags)
continue;
if (tags->nr_reserved_tags)
- bt_for_each(hctx, &tags->breserved_tags, fn, priv, true);
- bt_for_each(hctx, &tags->bitmap_tags, fn, priv, false);
+ bt_for_each(hctx, tags->breserved_tags, fn, priv, true);
+ bt_for_each(hctx, tags->bitmap_tags, fn, priv, false);
unsigned int depth = tags->nr_tags - tags->nr_reserved_tags;
bool round_robin = alloc_policy == BLK_TAG_ALLOC_RR;
unsigned int depth = tags->nr_tags - tags->nr_reserved_tags;
bool round_robin = alloc_policy == BLK_TAG_ALLOC_RR;
- if (bt_alloc(&tags->bitmap_tags, depth, round_robin, node))
+ if (bt_alloc(&tags->__bitmap_tags, depth, round_robin, node))
- if (bt_alloc(&tags->breserved_tags, tags->nr_reserved_tags, round_robin,
- node))
+ if (bt_alloc(&tags->__breserved_tags, tags->nr_reserved_tags,
+ round_robin, node))
+ tags->bitmap_tags = &tags->__bitmap_tags;
+ tags->breserved_tags = &tags->__breserved_tags;
+
return 0;
free_bitmap_tags:
return 0;
free_bitmap_tags:
- sbitmap_queue_free(&tags->bitmap_tags);
+ sbitmap_queue_free(&tags->__bitmap_tags);
void blk_mq_free_tags(struct blk_mq_tags *tags, unsigned int flags)
{
void blk_mq_free_tags(struct blk_mq_tags *tags, unsigned int flags)
{
- sbitmap_queue_free(&tags->bitmap_tags);
- sbitmap_queue_free(&tags->breserved_tags);
+ sbitmap_queue_free(&tags->__bitmap_tags);
+ sbitmap_queue_free(&tags->__breserved_tags);
* Don't need (or can't) update reserved tags here, they
* remain static and should never need resizing.
*/
* Don't need (or can't) update reserved tags here, they
* remain static and should never need resizing.
*/
- sbitmap_queue_resize(&tags->bitmap_tags,
+ sbitmap_queue_resize(tags->bitmap_tags,
tdepth - tags->nr_reserved_tags);
}
tdepth - tags->nr_reserved_tags);
}
- struct sbitmap_queue bitmap_tags;
- struct sbitmap_queue breserved_tags;
+ struct sbitmap_queue *bitmap_tags;
+ struct sbitmap_queue *breserved_tags;
+
+ struct sbitmap_queue __bitmap_tags;
+ struct sbitmap_queue __breserved_tags;
struct request **rqs;
struct request **static_rqs;
struct request **rqs;
struct request **static_rqs;
static bool __blk_mq_get_driver_tag(struct request *rq)
{
static bool __blk_mq_get_driver_tag(struct request *rq)
{
- struct sbitmap_queue *bt = &rq->mq_hctx->tags->bitmap_tags;
+ struct sbitmap_queue *bt = rq->mq_hctx->tags->bitmap_tags;
unsigned int tag_offset = rq->mq_hctx->tags->nr_reserved_tags;
int tag;
blk_mq_tag_busy(rq->mq_hctx);
if (blk_mq_tag_is_reserved(rq->mq_hctx->sched_tags, rq->internal_tag)) {
unsigned int tag_offset = rq->mq_hctx->tags->nr_reserved_tags;
int tag;
blk_mq_tag_busy(rq->mq_hctx);
if (blk_mq_tag_is_reserved(rq->mq_hctx->sched_tags, rq->internal_tag)) {
- bt = &rq->mq_hctx->tags->breserved_tags;
+ bt = rq->mq_hctx->tags->breserved_tags;
struct sbitmap_queue *sbq;
list_del_init(&wait->entry);
struct sbitmap_queue *sbq;
list_del_init(&wait->entry);
- sbq = &hctx->tags->bitmap_tags;
+ sbq = hctx->tags->bitmap_tags;
atomic_dec(&sbq->ws_active);
}
spin_unlock(&hctx->dispatch_wait_lock);
atomic_dec(&sbq->ws_active);
}
spin_unlock(&hctx->dispatch_wait_lock);
static bool blk_mq_mark_tag_wait(struct blk_mq_hw_ctx *hctx,
struct request *rq)
{
static bool blk_mq_mark_tag_wait(struct blk_mq_hw_ctx *hctx,
struct request *rq)
{
- struct sbitmap_queue *sbq = &hctx->tags->bitmap_tags;
+ struct sbitmap_queue *sbq = hctx->tags->bitmap_tags;
struct wait_queue_head *wq;
wait_queue_entry_t *wait;
bool ret;
struct wait_queue_head *wq;
wait_queue_entry_t *wait;
bool ret;
* All of the hardware queues have the same depth, so we can just grab
* the shift of the first one.
*/
* All of the hardware queues have the same depth, so we can just grab
* the shift of the first one.
*/
- return q->queue_hw_ctx[0]->sched_tags->bitmap_tags.sb.shift;
+ return q->queue_hw_ctx[0]->sched_tags->bitmap_tags->sb.shift;
}
static struct kyber_queue_data *kyber_queue_data_alloc(struct request_queue *q)
}
static struct kyber_queue_data *kyber_queue_data_alloc(struct request_queue *q)
khd->batching = 0;
hctx->sched_data = khd;
khd->batching = 0;
hctx->sched_data = khd;
- sbitmap_queue_min_shallow_depth(&hctx->sched_tags->bitmap_tags,
+ sbitmap_queue_min_shallow_depth(hctx->sched_tags->bitmap_tags,
kqd->async_depth);
return 0;
kqd->async_depth);
return 0;