Commit be94f058 authored by Jens Axboe's avatar Jens Axboe

blk-mq: use bd->last == true for list inserts

If we are issuing a list of requests, we know if we're at the last one.
If we fail issuing, ensure that we call ->commits_rqs() to flush any
potential previous requests.
Reviewed-by: default avatarOmar Sandoval <osandov@fb.com>
Reviewed-by: default avatarMing Lei <ming.lei@redhat.com>
Reviewed-by: default avatarChristoph Hellwig <hch@lst.de>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 80ff2040
...@@ -1334,7 +1334,7 @@ blk_status_t blk_insert_cloned_request(struct request_queue *q, struct request * ...@@ -1334,7 +1334,7 @@ blk_status_t blk_insert_cloned_request(struct request_queue *q, struct request *
* bypass a potential scheduler on the bottom device for * bypass a potential scheduler on the bottom device for
* insert. * insert.
*/ */
return blk_mq_request_issue_directly(rq); return blk_mq_request_issue_directly(rq, true);
} }
EXPORT_SYMBOL_GPL(blk_insert_cloned_request); EXPORT_SYMBOL_GPL(blk_insert_cloned_request);
......
...@@ -1743,12 +1743,12 @@ static blk_qc_t request_to_qc_t(struct blk_mq_hw_ctx *hctx, struct request *rq) ...@@ -1743,12 +1743,12 @@ static blk_qc_t request_to_qc_t(struct blk_mq_hw_ctx *hctx, struct request *rq)
static blk_status_t __blk_mq_issue_directly(struct blk_mq_hw_ctx *hctx, static blk_status_t __blk_mq_issue_directly(struct blk_mq_hw_ctx *hctx,
struct request *rq, struct request *rq,
blk_qc_t *cookie) blk_qc_t *cookie, bool last)
{ {
struct request_queue *q = rq->q; struct request_queue *q = rq->q;
struct blk_mq_queue_data bd = { struct blk_mq_queue_data bd = {
.rq = rq, .rq = rq,
.last = true, .last = last,
}; };
blk_qc_t new_cookie; blk_qc_t new_cookie;
blk_status_t ret; blk_status_t ret;
...@@ -1783,7 +1783,7 @@ static blk_status_t __blk_mq_issue_directly(struct blk_mq_hw_ctx *hctx, ...@@ -1783,7 +1783,7 @@ static blk_status_t __blk_mq_issue_directly(struct blk_mq_hw_ctx *hctx,
static blk_status_t __blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx, static blk_status_t __blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
struct request *rq, struct request *rq,
blk_qc_t *cookie, blk_qc_t *cookie,
bool bypass_insert) bool bypass_insert, bool last)
{ {
struct request_queue *q = rq->q; struct request_queue *q = rq->q;
bool run_queue = true; bool run_queue = true;
...@@ -1812,7 +1812,7 @@ static blk_status_t __blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx, ...@@ -1812,7 +1812,7 @@ static blk_status_t __blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
goto insert; goto insert;
} }
return __blk_mq_issue_directly(hctx, rq, cookie); return __blk_mq_issue_directly(hctx, rq, cookie, last);
insert: insert:
if (bypass_insert) if (bypass_insert)
return BLK_STS_RESOURCE; return BLK_STS_RESOURCE;
...@@ -1831,7 +1831,7 @@ static void blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx, ...@@ -1831,7 +1831,7 @@ static void blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
hctx_lock(hctx, &srcu_idx); hctx_lock(hctx, &srcu_idx);
ret = __blk_mq_try_issue_directly(hctx, rq, cookie, false); ret = __blk_mq_try_issue_directly(hctx, rq, cookie, false, true);
if (ret == BLK_STS_RESOURCE || ret == BLK_STS_DEV_RESOURCE) if (ret == BLK_STS_RESOURCE || ret == BLK_STS_DEV_RESOURCE)
blk_mq_sched_insert_request(rq, false, true, false); blk_mq_sched_insert_request(rq, false, true, false);
else if (ret != BLK_STS_OK) else if (ret != BLK_STS_OK)
...@@ -1840,7 +1840,7 @@ static void blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx, ...@@ -1840,7 +1840,7 @@ static void blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
hctx_unlock(hctx, srcu_idx); hctx_unlock(hctx, srcu_idx);
} }
blk_status_t blk_mq_request_issue_directly(struct request *rq) blk_status_t blk_mq_request_issue_directly(struct request *rq, bool last)
{ {
blk_status_t ret; blk_status_t ret;
int srcu_idx; int srcu_idx;
...@@ -1848,7 +1848,7 @@ blk_status_t blk_mq_request_issue_directly(struct request *rq) ...@@ -1848,7 +1848,7 @@ blk_status_t blk_mq_request_issue_directly(struct request *rq)
struct blk_mq_hw_ctx *hctx = rq->mq_hctx; struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
hctx_lock(hctx, &srcu_idx); hctx_lock(hctx, &srcu_idx);
ret = __blk_mq_try_issue_directly(hctx, rq, &unused_cookie, true); ret = __blk_mq_try_issue_directly(hctx, rq, &unused_cookie, true, last);
hctx_unlock(hctx, srcu_idx); hctx_unlock(hctx, srcu_idx);
return ret; return ret;
...@@ -1863,7 +1863,7 @@ void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx, ...@@ -1863,7 +1863,7 @@ void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,
queuelist); queuelist);
list_del_init(&rq->queuelist); list_del_init(&rq->queuelist);
ret = blk_mq_request_issue_directly(rq); ret = blk_mq_request_issue_directly(rq, list_empty(list));
if (ret != BLK_STS_OK) { if (ret != BLK_STS_OK) {
if (ret == BLK_STS_RESOURCE || if (ret == BLK_STS_RESOURCE ||
ret == BLK_STS_DEV_RESOURCE) { ret == BLK_STS_DEV_RESOURCE) {
......
...@@ -69,7 +69,7 @@ void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx, ...@@ -69,7 +69,7 @@ void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
struct list_head *list); struct list_head *list);
/* Used by blk_insert_cloned_request() to issue request directly */ /* Used by blk_insert_cloned_request() to issue request directly */
blk_status_t blk_mq_request_issue_directly(struct request *rq); blk_status_t blk_mq_request_issue_directly(struct request *rq, bool last);
void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx, void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,
struct list_head *list); struct list_head *list);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment