Commit 6c3b7af1 authored by Omar Sandoval's avatar Omar Sandoval Committed by Jens Axboe

kyber: add tracepoints

When debugging Kyber, it's really useful to know what latencies we've
been having, how the domain depths have been adjusted, and if we've
actually been throttling. Add three tracepoints, kyber_latency,
kyber_adjust, and kyber_throttled, to record that.
Signed-off-by: default avatarOmar Sandoval <osandov@fb.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 6e25cb01
...@@ -30,6 +30,9 @@ ...@@ -30,6 +30,9 @@
#include "blk-mq-sched.h" #include "blk-mq-sched.h"
#include "blk-mq-tag.h" #include "blk-mq-tag.h"
#define CREATE_TRACE_POINTS
#include <trace/events/kyber.h>
/* /*
* Scheduling domains: the device is divided into multiple domains based on the * Scheduling domains: the device is divided into multiple domains based on the
* request type. * request type.
...@@ -42,6 +45,13 @@ enum { ...@@ -42,6 +45,13 @@ enum {
KYBER_NUM_DOMAINS, KYBER_NUM_DOMAINS,
}; };
static const char *kyber_domain_names[] = {
[KYBER_READ] = "READ",
[KYBER_WRITE] = "WRITE",
[KYBER_DISCARD] = "DISCARD",
[KYBER_OTHER] = "OTHER",
};
enum { enum {
/* /*
* In order to prevent starvation of synchronous requests by a flood of * In order to prevent starvation of synchronous requests by a flood of
...@@ -122,6 +132,11 @@ enum { ...@@ -122,6 +132,11 @@ enum {
KYBER_IO_LATENCY, KYBER_IO_LATENCY,
}; };
static const char *kyber_latency_type_names[] = {
[KYBER_TOTAL_LATENCY] = "total",
[KYBER_IO_LATENCY] = "I/O",
};
/* /*
* Per-cpu latency histograms: total latency and I/O latency for each scheduling * Per-cpu latency histograms: total latency and I/O latency for each scheduling
* domain except for KYBER_OTHER. * domain except for KYBER_OTHER.
...@@ -144,6 +159,8 @@ struct kyber_ctx_queue { ...@@ -144,6 +159,8 @@ struct kyber_ctx_queue {
} ____cacheline_aligned_in_smp; } ____cacheline_aligned_in_smp;
struct kyber_queue_data { struct kyber_queue_data {
struct request_queue *q;
/* /*
* Each scheduling domain has a limited number of in-flight requests * Each scheduling domain has a limited number of in-flight requests
* device-wide, limited by these tokens. * device-wide, limited by these tokens.
...@@ -249,6 +266,10 @@ static int calculate_percentile(struct kyber_queue_data *kqd, ...@@ -249,6 +266,10 @@ static int calculate_percentile(struct kyber_queue_data *kqd,
} }
memset(buckets, 0, sizeof(kqd->latency_buckets[sched_domain][type])); memset(buckets, 0, sizeof(kqd->latency_buckets[sched_domain][type]));
trace_kyber_latency(kqd->q, kyber_domain_names[sched_domain],
kyber_latency_type_names[type], percentile,
bucket + 1, 1 << KYBER_LATENCY_SHIFT, samples);
return bucket; return bucket;
} }
...@@ -256,8 +277,11 @@ static void kyber_resize_domain(struct kyber_queue_data *kqd, ...@@ -256,8 +277,11 @@ static void kyber_resize_domain(struct kyber_queue_data *kqd,
unsigned int sched_domain, unsigned int depth) unsigned int sched_domain, unsigned int depth)
{ {
depth = clamp(depth, 1U, kyber_depth[sched_domain]); depth = clamp(depth, 1U, kyber_depth[sched_domain]);
if (depth != kqd->domain_tokens[sched_domain].sb.depth) if (depth != kqd->domain_tokens[sched_domain].sb.depth) {
sbitmap_queue_resize(&kqd->domain_tokens[sched_domain], depth); sbitmap_queue_resize(&kqd->domain_tokens[sched_domain], depth);
trace_kyber_adjust(kqd->q, kyber_domain_names[sched_domain],
depth);
}
} }
static void kyber_timer_fn(struct timer_list *t) static void kyber_timer_fn(struct timer_list *t)
...@@ -360,6 +384,8 @@ static struct kyber_queue_data *kyber_queue_data_alloc(struct request_queue *q) ...@@ -360,6 +384,8 @@ static struct kyber_queue_data *kyber_queue_data_alloc(struct request_queue *q)
if (!kqd) if (!kqd)
goto err; goto err;
kqd->q = q;
kqd->cpu_latency = alloc_percpu_gfp(struct kyber_cpu_latency, kqd->cpu_latency = alloc_percpu_gfp(struct kyber_cpu_latency,
GFP_KERNEL | __GFP_ZERO); GFP_KERNEL | __GFP_ZERO);
if (!kqd->cpu_latency) if (!kqd->cpu_latency)
...@@ -756,6 +782,9 @@ kyber_dispatch_cur_domain(struct kyber_queue_data *kqd, ...@@ -756,6 +782,9 @@ kyber_dispatch_cur_domain(struct kyber_queue_data *kqd,
rq_set_domain_token(rq, nr); rq_set_domain_token(rq, nr);
list_del_init(&rq->queuelist); list_del_init(&rq->queuelist);
return rq; return rq;
} else {
trace_kyber_throttled(kqd->q,
kyber_domain_names[khd->cur_domain]);
} }
} else if (sbitmap_any_bit_set(&khd->kcq_map[khd->cur_domain])) { } else if (sbitmap_any_bit_set(&khd->kcq_map[khd->cur_domain])) {
nr = kyber_get_domain_token(kqd, khd, hctx); nr = kyber_get_domain_token(kqd, khd, hctx);
...@@ -766,6 +795,9 @@ kyber_dispatch_cur_domain(struct kyber_queue_data *kqd, ...@@ -766,6 +795,9 @@ kyber_dispatch_cur_domain(struct kyber_queue_data *kqd,
rq_set_domain_token(rq, nr); rq_set_domain_token(rq, nr);
list_del_init(&rq->queuelist); list_del_init(&rq->queuelist);
return rq; return rq;
} else {
trace_kyber_throttled(kqd->q,
kyber_domain_names[khd->cur_domain]);
} }
} }
...@@ -944,23 +976,7 @@ static int kyber_cur_domain_show(void *data, struct seq_file *m) ...@@ -944,23 +976,7 @@ static int kyber_cur_domain_show(void *data, struct seq_file *m)
struct blk_mq_hw_ctx *hctx = data; struct blk_mq_hw_ctx *hctx = data;
struct kyber_hctx_data *khd = hctx->sched_data; struct kyber_hctx_data *khd = hctx->sched_data;
switch (khd->cur_domain) { seq_printf(m, "%s\n", kyber_domain_names[khd->cur_domain]);
case KYBER_READ:
seq_puts(m, "READ\n");
break;
case KYBER_WRITE:
seq_puts(m, "WRITE\n");
break;
case KYBER_DISCARD:
seq_puts(m, "DISCARD\n");
break;
case KYBER_OTHER:
seq_puts(m, "OTHER\n");
break;
default:
seq_printf(m, "%u\n", khd->cur_domain);
break;
}
return 0; return 0;
} }
......
/* SPDX-License-Identifier: GPL-2.0 */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM kyber
#if !defined(_TRACE_KYBER_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_KYBER_H
#include <linux/blkdev.h>
#include <linux/tracepoint.h>
#define DOMAIN_LEN 16
#define LATENCY_TYPE_LEN 8
TRACE_EVENT(kyber_latency,
TP_PROTO(struct request_queue *q, const char *domain, const char *type,
unsigned int percentile, unsigned int numerator,
unsigned int denominator, unsigned int samples),
TP_ARGS(q, domain, type, percentile, numerator, denominator, samples),
TP_STRUCT__entry(
__field( dev_t, dev )
__array( char, domain, DOMAIN_LEN )
__array( char, type, LATENCY_TYPE_LEN )
__field( u8, percentile )
__field( u8, numerator )
__field( u8, denominator )
__field( unsigned int, samples )
),
TP_fast_assign(
__entry->dev = disk_devt(dev_to_disk(kobj_to_dev(q->kobj.parent)));
strlcpy(__entry->domain, domain, DOMAIN_LEN);
strlcpy(__entry->type, type, DOMAIN_LEN);
__entry->percentile = percentile;
__entry->numerator = numerator;
__entry->denominator = denominator;
__entry->samples = samples;
),
TP_printk("%d,%d %s %s p%u %u/%u samples=%u",
MAJOR(__entry->dev), MINOR(__entry->dev), __entry->domain,
__entry->type, __entry->percentile, __entry->numerator,
__entry->denominator, __entry->samples)
);
TRACE_EVENT(kyber_adjust,
TP_PROTO(struct request_queue *q, const char *domain,
unsigned int depth),
TP_ARGS(q, domain, depth),
TP_STRUCT__entry(
__field( dev_t, dev )
__array( char, domain, DOMAIN_LEN )
__field( unsigned int, depth )
),
TP_fast_assign(
__entry->dev = disk_devt(dev_to_disk(kobj_to_dev(q->kobj.parent)));
strlcpy(__entry->domain, domain, DOMAIN_LEN);
__entry->depth = depth;
),
TP_printk("%d,%d %s %u",
MAJOR(__entry->dev), MINOR(__entry->dev), __entry->domain,
__entry->depth)
);
TRACE_EVENT(kyber_throttled,
TP_PROTO(struct request_queue *q, const char *domain),
TP_ARGS(q, domain),
TP_STRUCT__entry(
__field( dev_t, dev )
__array( char, domain, DOMAIN_LEN )
),
TP_fast_assign(
__entry->dev = disk_devt(dev_to_disk(kobj_to_dev(q->kobj.parent)));
strlcpy(__entry->domain, domain, DOMAIN_LEN);
),
TP_printk("%d,%d %s", MAJOR(__entry->dev), MINOR(__entry->dev),
__entry->domain)
);
#define _TRACE_KYBER_H
#endif /* _TRACE_KYBER_H */
/* This part must be outside protection */
#include <trace/define_trace.h>
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment