Commit 602380d2 authored by Ming Lei's avatar Ming Lei Committed by Jens Axboe

blk-mq: add blk_mq_all_tag_iter

Add a new blk_mq_all_tag_iter function to iterate over all allocated
scheduler tags and driver tags.  This is more flexible than the existing
blk_mq_all_tag_busy_iter function as it allows the callers to do whatever
they want on allocated request instead of being limited to started
requests.

It will be used to implement draining allocated requests on specified
hctx in this patchset.

[hch: switch from the two booleans to a more readable flags field and
 consolidate the tags iter functions]
Signed-off-by: default avatarMing Lei <ming.lei@redhat.com>
Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
Reviewed-by: default avatarHannes Reinecke <hare@suse.de>
Reviewed-by: default avatarDaniel Wagner <dwagner@suse.de>
Reviewed-by: default avatarBart van Assche <bvanassche@acm.org>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 600c3b0c
...@@ -256,14 +256,17 @@ struct bt_tags_iter_data { ...@@ -256,14 +256,17 @@ struct bt_tags_iter_data {
struct blk_mq_tags *tags; struct blk_mq_tags *tags;
busy_tag_iter_fn *fn; busy_tag_iter_fn *fn;
void *data; void *data;
bool reserved; unsigned int flags;
}; };
#define BT_TAG_ITER_RESERVED (1 << 0)
#define BT_TAG_ITER_STARTED (1 << 1)
static bool bt_tags_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data) static bool bt_tags_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data)
{ {
struct bt_tags_iter_data *iter_data = data; struct bt_tags_iter_data *iter_data = data;
struct blk_mq_tags *tags = iter_data->tags; struct blk_mq_tags *tags = iter_data->tags;
bool reserved = iter_data->reserved; bool reserved = iter_data->flags & BT_TAG_ITER_RESERVED;
struct request *rq; struct request *rq;
if (!reserved) if (!reserved)
...@@ -274,10 +277,12 @@ static bool bt_tags_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data) ...@@ -274,10 +277,12 @@ static bool bt_tags_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data)
* test and set the bit before assining ->rqs[]. * test and set the bit before assining ->rqs[].
*/ */
rq = tags->rqs[bitnr]; rq = tags->rqs[bitnr];
if (rq && blk_mq_request_started(rq)) if (!rq)
return iter_data->fn(rq, iter_data->data, reserved); return true;
if ((iter_data->flags & BT_TAG_ITER_STARTED) &&
return true; !blk_mq_request_started(rq))
return true;
return iter_data->fn(rq, iter_data->data, reserved);
} }
/** /**
...@@ -290,39 +295,47 @@ static bool bt_tags_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data) ...@@ -290,39 +295,47 @@ static bool bt_tags_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data)
* @reserved) where rq is a pointer to a request. Return true * @reserved) where rq is a pointer to a request. Return true
* to continue iterating tags, false to stop. * to continue iterating tags, false to stop.
* @data: Will be passed as second argument to @fn. * @data: Will be passed as second argument to @fn.
* @reserved: Indicates whether @bt is the breserved_tags member or the * @flags: BT_TAG_ITER_*
* bitmap_tags member of struct blk_mq_tags.
*/ */
static void bt_tags_for_each(struct blk_mq_tags *tags, struct sbitmap_queue *bt, static void bt_tags_for_each(struct blk_mq_tags *tags, struct sbitmap_queue *bt,
busy_tag_iter_fn *fn, void *data, bool reserved) busy_tag_iter_fn *fn, void *data, unsigned int flags)
{ {
struct bt_tags_iter_data iter_data = { struct bt_tags_iter_data iter_data = {
.tags = tags, .tags = tags,
.fn = fn, .fn = fn,
.data = data, .data = data,
.reserved = reserved, .flags = flags,
}; };
if (tags->rqs) if (tags->rqs)
sbitmap_for_each_set(&bt->sb, bt_tags_iter, &iter_data); sbitmap_for_each_set(&bt->sb, bt_tags_iter, &iter_data);
} }
static void __blk_mq_all_tag_iter(struct blk_mq_tags *tags,
busy_tag_iter_fn *fn, void *priv, unsigned int flags)
{
WARN_ON_ONCE(flags & BT_TAG_ITER_RESERVED);
if (tags->nr_reserved_tags)
bt_tags_for_each(tags, &tags->breserved_tags, fn, priv,
flags | BT_TAG_ITER_RESERVED);
bt_tags_for_each(tags, &tags->bitmap_tags, fn, priv, flags);
}
/** /**
* blk_mq_all_tag_busy_iter - iterate over all started requests in a tag map * blk_mq_all_tag_iter - iterate over all requests in a tag map
* @tags: Tag map to iterate over. * @tags: Tag map to iterate over.
* @fn: Pointer to the function that will be called for each started * @fn: Pointer to the function that will be called for each
* request. @fn will be called as follows: @fn(rq, @priv, * request. @fn will be called as follows: @fn(rq, @priv,
* reserved) where rq is a pointer to a request. 'reserved' * reserved) where rq is a pointer to a request. 'reserved'
* indicates whether or not @rq is a reserved request. Return * indicates whether or not @rq is a reserved request. Return
* true to continue iterating tags, false to stop. * true to continue iterating tags, false to stop.
* @priv: Will be passed as second argument to @fn. * @priv: Will be passed as second argument to @fn.
*/ */
static void blk_mq_all_tag_busy_iter(struct blk_mq_tags *tags, void blk_mq_all_tag_iter(struct blk_mq_tags *tags, busy_tag_iter_fn *fn,
busy_tag_iter_fn *fn, void *priv) void *priv)
{ {
if (tags->nr_reserved_tags) return __blk_mq_all_tag_iter(tags, fn, priv, 0);
bt_tags_for_each(tags, &tags->breserved_tags, fn, priv, true);
bt_tags_for_each(tags, &tags->bitmap_tags, fn, priv, false);
} }
/** /**
...@@ -342,7 +355,8 @@ void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset, ...@@ -342,7 +355,8 @@ void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset,
for (i = 0; i < tagset->nr_hw_queues; i++) { for (i = 0; i < tagset->nr_hw_queues; i++) {
if (tagset->tags && tagset->tags[i]) if (tagset->tags && tagset->tags[i])
blk_mq_all_tag_busy_iter(tagset->tags[i], fn, priv); __blk_mq_all_tag_iter(tagset->tags[i], fn, priv,
BT_TAG_ITER_STARTED);
} }
} }
EXPORT_SYMBOL(blk_mq_tagset_busy_iter); EXPORT_SYMBOL(blk_mq_tagset_busy_iter);
......
...@@ -34,6 +34,8 @@ extern int blk_mq_tag_update_depth(struct blk_mq_hw_ctx *hctx, ...@@ -34,6 +34,8 @@ extern int blk_mq_tag_update_depth(struct blk_mq_hw_ctx *hctx,
extern void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags, bool); extern void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags, bool);
void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_iter_fn *fn, void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_iter_fn *fn,
void *priv); void *priv);
void blk_mq_all_tag_iter(struct blk_mq_tags *tags, busy_tag_iter_fn *fn,
void *priv);
static inline struct sbq_wait_state *bt_wait_ptr(struct sbitmap_queue *bt, static inline struct sbq_wait_state *bt_wait_ptr(struct sbitmap_queue *bt,
struct blk_mq_hw_ctx *hctx) struct blk_mq_hw_ctx *hctx)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment