Commit 6f816b4b authored by Tejun Heo's avatar Tejun Heo Committed by Jens Axboe

blk-mq: add optional request->alloc_time_ns

There are currently two start time timestamps - start_time_ns and
io_start_time_ns.  The former marks the request allocation and and the
second issue-to-device time.  The planned io.weight controller needs
to measure the total time bios take to execute after it leaves rq_qos
including the time spent waiting for request to become available,
which can easily dominate on saturated devices.

This patch adds request->alloc_time_ns which records when the request
allocation attempt started.  As it isn't used for the usual stats,
make it optional behind CONFIG_BLK_RQ_ALLOC_TIME and
QUEUE_FLAG_RQ_ALLOC_TIME so that it can be compiled out when there are
no users and it's active only on queues which need it even when
compiled in.

v2: s/pre_start_time/alloc_time/ and add CONFIG_BLK_RQ_ALLOC_TIME
    gating as suggested by Jens.
Signed-off-by: default avatarTejun Heo <tj@kernel.org>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent beab17fc
...@@ -26,6 +26,9 @@ menuconfig BLOCK ...@@ -26,6 +26,9 @@ menuconfig BLOCK
if BLOCK if BLOCK
config BLK_RQ_ALLOC_TIME
bool
config BLK_SCSI_REQUEST config BLK_SCSI_REQUEST
bool bool
......
...@@ -291,7 +291,7 @@ static inline bool blk_mq_need_time_stamp(struct request *rq) ...@@ -291,7 +291,7 @@ static inline bool blk_mq_need_time_stamp(struct request *rq)
} }
static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data, static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
unsigned int tag, unsigned int op) unsigned int tag, unsigned int op, u64 alloc_time_ns)
{ {
struct blk_mq_tags *tags = blk_mq_tags_from_data(data); struct blk_mq_tags *tags = blk_mq_tags_from_data(data);
struct request *rq = tags->static_rqs[tag]; struct request *rq = tags->static_rqs[tag];
...@@ -325,6 +325,9 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data, ...@@ -325,6 +325,9 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
RB_CLEAR_NODE(&rq->rb_node); RB_CLEAR_NODE(&rq->rb_node);
rq->rq_disk = NULL; rq->rq_disk = NULL;
rq->part = NULL; rq->part = NULL;
#ifdef CONFIG_BLK_RQ_ALLOC_TIME
rq->alloc_time_ns = alloc_time_ns;
#endif
if (blk_mq_need_time_stamp(rq)) if (blk_mq_need_time_stamp(rq))
rq->start_time_ns = ktime_get_ns(); rq->start_time_ns = ktime_get_ns();
else else
...@@ -356,8 +359,14 @@ static struct request *blk_mq_get_request(struct request_queue *q, ...@@ -356,8 +359,14 @@ static struct request *blk_mq_get_request(struct request_queue *q,
struct request *rq; struct request *rq;
unsigned int tag; unsigned int tag;
bool clear_ctx_on_error = false; bool clear_ctx_on_error = false;
u64 alloc_time_ns = 0;
blk_queue_enter_live(q); blk_queue_enter_live(q);
/* alloc_time includes depth and tag waits */
if (blk_queue_rq_alloc_time(q))
alloc_time_ns = ktime_get_ns();
data->q = q; data->q = q;
if (likely(!data->ctx)) { if (likely(!data->ctx)) {
data->ctx = blk_mq_get_ctx(q); data->ctx = blk_mq_get_ctx(q);
...@@ -393,7 +402,7 @@ static struct request *blk_mq_get_request(struct request_queue *q, ...@@ -393,7 +402,7 @@ static struct request *blk_mq_get_request(struct request_queue *q,
return NULL; return NULL;
} }
rq = blk_mq_rq_ctx_init(data, tag, data->cmd_flags); rq = blk_mq_rq_ctx_init(data, tag, data->cmd_flags, alloc_time_ns);
if (!op_is_flush(data->cmd_flags)) { if (!op_is_flush(data->cmd_flags)) {
rq->elv.icq = NULL; rq->elv.icq = NULL;
if (e && e->type->ops.prepare_request) { if (e && e->type->ops.prepare_request) {
......
...@@ -194,7 +194,11 @@ struct request { ...@@ -194,7 +194,11 @@ struct request {
struct gendisk *rq_disk; struct gendisk *rq_disk;
struct hd_struct *part; struct hd_struct *part;
/* Time that I/O was submitted to the kernel. */ #ifdef CONFIG_BLK_RQ_ALLOC_TIME
/* Time that the first bio started allocating this request. */
u64 alloc_time_ns;
#endif
/* Time that this request was allocated for this IO. */
u64 start_time_ns; u64 start_time_ns;
/* Time that I/O was submitted to the device. */ /* Time that I/O was submitted to the device. */
u64 io_start_time_ns; u64 io_start_time_ns;
...@@ -609,6 +613,7 @@ struct request_queue { ...@@ -609,6 +613,7 @@ struct request_queue {
#define QUEUE_FLAG_QUIESCED 24 /* queue has been quiesced */ #define QUEUE_FLAG_QUIESCED 24 /* queue has been quiesced */
#define QUEUE_FLAG_PCI_P2PDMA 25 /* device supports PCI p2p requests */ #define QUEUE_FLAG_PCI_P2PDMA 25 /* device supports PCI p2p requests */
#define QUEUE_FLAG_ZONE_RESETALL 26 /* supports Zone Reset All */ #define QUEUE_FLAG_ZONE_RESETALL 26 /* supports Zone Reset All */
#define QUEUE_FLAG_RQ_ALLOC_TIME 27 /* record rq->alloc_time_ns */
#define QUEUE_FLAG_MQ_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \ #define QUEUE_FLAG_MQ_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \
(1 << QUEUE_FLAG_SAME_COMP)) (1 << QUEUE_FLAG_SAME_COMP))
...@@ -637,6 +642,12 @@ bool blk_queue_flag_test_and_set(unsigned int flag, struct request_queue *q); ...@@ -637,6 +642,12 @@ bool blk_queue_flag_test_and_set(unsigned int flag, struct request_queue *q);
test_bit(QUEUE_FLAG_SCSI_PASSTHROUGH, &(q)->queue_flags) test_bit(QUEUE_FLAG_SCSI_PASSTHROUGH, &(q)->queue_flags)
#define blk_queue_pci_p2pdma(q) \ #define blk_queue_pci_p2pdma(q) \
test_bit(QUEUE_FLAG_PCI_P2PDMA, &(q)->queue_flags) test_bit(QUEUE_FLAG_PCI_P2PDMA, &(q)->queue_flags)
#ifdef CONFIG_BLK_RQ_ALLOC_TIME
#define blk_queue_rq_alloc_time(q) \
test_bit(QUEUE_FLAG_RQ_ALLOC_TIME, &(q)->queue_flags)
#else
#define blk_queue_rq_alloc_time(q) false
#endif
#define blk_noretry_request(rq) \ #define blk_noretry_request(rq) \
((rq)->cmd_flags & (REQ_FAILFAST_DEV|REQ_FAILFAST_TRANSPORT| \ ((rq)->cmd_flags & (REQ_FAILFAST_DEV|REQ_FAILFAST_TRANSPORT| \
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment