Commit 9aff93d7 authored by Vlad Buslov's avatar Vlad Buslov Committed by Saeed Mahameed

net/mlx5: Store flow counters in a list

In order to improve performance of flow counter stats query loop that
traverses all configured flow counters, replace rb_tree with double-linked
list. This change improves performance of traversing flow counters by
removing the tree traversal. (profiling data showed that call to rb_next
was most top CPU consumer)

However, lookup of flow flow counter in list becomes linear, instead of
logarithmic. This problem is fixed by next patch in series, which adds idr
for fast lookup. Idr is to be used because it is not an intrusive data
structure and doesn't require adding any new members to struct mlx5_fc,
which allows its control data part to stay <= 1 cache line in size.
Signed-off-by: default avatarVlad Buslov <vladbu@mellanox.com>
Acked-by: default avatarAmir Vadai <amir@vadai.me>
Reviewed-by: default avatarPaul Blakey <paulb@mellanox.com>
Signed-off-by: default avatarSaeed Mahameed <saeedm@mellanox.com>
parent 6e5e2283
...@@ -139,7 +139,7 @@ struct mlx5_fc_cache { ...@@ -139,7 +139,7 @@ struct mlx5_fc_cache {
}; };
struct mlx5_fc { struct mlx5_fc {
struct rb_node node; struct list_head list;
struct llist_node addlist; struct llist_node addlist;
struct llist_node dellist; struct llist_node dellist;
......
...@@ -73,36 +73,38 @@ ...@@ -73,36 +73,38 @@
* elapsed, the thread will actually query the hardware. * elapsed, the thread will actually query the hardware.
*/ */
static void mlx5_fc_stats_insert(struct rb_root *root, struct mlx5_fc *counter) static struct list_head *mlx5_fc_counters_lookup_next(struct mlx5_core_dev *dev,
u32 id)
{ {
struct rb_node **new = &root->rb_node; struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats;
struct rb_node *parent = NULL; struct mlx5_fc *counter;
while (*new) { list_for_each_entry(counter, &fc_stats->counters, list)
struct mlx5_fc *this = rb_entry(*new, struct mlx5_fc, node); if (counter->id > id)
int result = counter->id - this->id; return &counter->list;
parent = *new; return &fc_stats->counters;
if (result < 0) }
new = &((*new)->rb_left);
else static void mlx5_fc_stats_insert(struct mlx5_core_dev *dev,
new = &((*new)->rb_right); struct mlx5_fc *counter)
} {
struct list_head *next = mlx5_fc_counters_lookup_next(dev, counter->id);
/* Add new node and rebalance tree. */ list_add_tail(&counter->list, next);
rb_link_node(&counter->node, parent, new);
rb_insert_color(&counter->node, root);
} }
/* The function returns the last node that was queried so the caller /* The function returns the last counter that was queried so the caller
* function can continue calling it till all counters are queried. * function can continue calling it till all counters are queried.
*/ */
static struct rb_node *mlx5_fc_stats_query(struct mlx5_core_dev *dev, static struct mlx5_fc *mlx5_fc_stats_query(struct mlx5_core_dev *dev,
struct mlx5_fc *first, struct mlx5_fc *first,
u32 last_id) u32 last_id)
{ {
struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats;
struct mlx5_fc *counter = NULL;
struct mlx5_cmd_fc_bulk *b; struct mlx5_cmd_fc_bulk *b;
struct rb_node *node = NULL; bool more = false;
u32 afirst_id; u32 afirst_id;
int num; int num;
int err; int err;
...@@ -132,14 +134,16 @@ static struct rb_node *mlx5_fc_stats_query(struct mlx5_core_dev *dev, ...@@ -132,14 +134,16 @@ static struct rb_node *mlx5_fc_stats_query(struct mlx5_core_dev *dev,
goto out; goto out;
} }
for (node = &first->node; node; node = rb_next(node)) { counter = first;
struct mlx5_fc *counter = rb_entry(node, struct mlx5_fc, node); list_for_each_entry_from(counter, &fc_stats->counters, list) {
struct mlx5_fc_cache *c = &counter->cache; struct mlx5_fc_cache *c = &counter->cache;
u64 packets; u64 packets;
u64 bytes; u64 bytes;
if (counter->id > last_id) if (counter->id > last_id) {
more = true;
break; break;
}
mlx5_cmd_fc_bulk_get(dev, b, mlx5_cmd_fc_bulk_get(dev, b,
counter->id, &packets, &bytes); counter->id, &packets, &bytes);
...@@ -155,7 +159,7 @@ static struct rb_node *mlx5_fc_stats_query(struct mlx5_core_dev *dev, ...@@ -155,7 +159,7 @@ static struct rb_node *mlx5_fc_stats_query(struct mlx5_core_dev *dev,
out: out:
mlx5_cmd_fc_bulk_free(b); mlx5_cmd_fc_bulk_free(b);
return node; return more ? counter : NULL;
} }
static void mlx5_free_fc(struct mlx5_core_dev *dev, static void mlx5_free_fc(struct mlx5_core_dev *dev,
...@@ -173,33 +177,30 @@ static void mlx5_fc_stats_work(struct work_struct *work) ...@@ -173,33 +177,30 @@ static void mlx5_fc_stats_work(struct work_struct *work)
struct llist_node *tmplist = llist_del_all(&fc_stats->addlist); struct llist_node *tmplist = llist_del_all(&fc_stats->addlist);
struct mlx5_fc *counter = NULL, *last = NULL, *tmp; struct mlx5_fc *counter = NULL, *last = NULL, *tmp;
unsigned long now = jiffies; unsigned long now = jiffies;
struct rb_node *node;
if (tmplist || !RB_EMPTY_ROOT(&fc_stats->counters)) if (tmplist || !list_empty(&fc_stats->counters))
queue_delayed_work(fc_stats->wq, &fc_stats->work, queue_delayed_work(fc_stats->wq, &fc_stats->work,
fc_stats->sampling_interval); fc_stats->sampling_interval);
llist_for_each_entry(counter, tmplist, addlist) llist_for_each_entry(counter, tmplist, addlist)
mlx5_fc_stats_insert(&fc_stats->counters, counter); mlx5_fc_stats_insert(dev, counter);
tmplist = llist_del_all(&fc_stats->dellist); tmplist = llist_del_all(&fc_stats->dellist);
llist_for_each_entry_safe(counter, tmp, tmplist, dellist) { llist_for_each_entry_safe(counter, tmp, tmplist, dellist) {
rb_erase(&counter->node, &fc_stats->counters); list_del(&counter->list);
mlx5_free_fc(dev, counter); mlx5_free_fc(dev, counter);
} }
node = rb_last(&fc_stats->counters); if (time_before(now, fc_stats->next_query) ||
if (time_before(now, fc_stats->next_query) || !node) list_empty(&fc_stats->counters))
return; return;
last = rb_entry(node, struct mlx5_fc, node); last = list_last_entry(&fc_stats->counters, struct mlx5_fc, list);
node = rb_first(&fc_stats->counters);
while (node) {
counter = rb_entry(node, struct mlx5_fc, node);
node = mlx5_fc_stats_query(dev, counter, last->id); counter = list_first_entry(&fc_stats->counters, struct mlx5_fc,
} list);
while (counter)
counter = mlx5_fc_stats_query(dev, counter, last->id);
fc_stats->next_query = now + fc_stats->sampling_interval; fc_stats->next_query = now + fc_stats->sampling_interval;
} }
...@@ -257,7 +258,7 @@ int mlx5_init_fc_stats(struct mlx5_core_dev *dev) ...@@ -257,7 +258,7 @@ int mlx5_init_fc_stats(struct mlx5_core_dev *dev)
{ {
struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats; struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats;
fc_stats->counters = RB_ROOT; INIT_LIST_HEAD(&fc_stats->counters);
init_llist_head(&fc_stats->addlist); init_llist_head(&fc_stats->addlist);
init_llist_head(&fc_stats->dellist); init_llist_head(&fc_stats->dellist);
...@@ -277,7 +278,6 @@ void mlx5_cleanup_fc_stats(struct mlx5_core_dev *dev) ...@@ -277,7 +278,6 @@ void mlx5_cleanup_fc_stats(struct mlx5_core_dev *dev)
struct llist_node *tmplist; struct llist_node *tmplist;
struct mlx5_fc *counter; struct mlx5_fc *counter;
struct mlx5_fc *tmp; struct mlx5_fc *tmp;
struct rb_node *node;
cancel_delayed_work_sync(&dev->priv.fc_stats.work); cancel_delayed_work_sync(&dev->priv.fc_stats.work);
destroy_workqueue(dev->priv.fc_stats.wq); destroy_workqueue(dev->priv.fc_stats.wq);
...@@ -287,16 +287,8 @@ void mlx5_cleanup_fc_stats(struct mlx5_core_dev *dev) ...@@ -287,16 +287,8 @@ void mlx5_cleanup_fc_stats(struct mlx5_core_dev *dev)
llist_for_each_entry_safe(counter, tmp, tmplist, addlist) llist_for_each_entry_safe(counter, tmp, tmplist, addlist)
mlx5_free_fc(dev, counter); mlx5_free_fc(dev, counter);
node = rb_first(&fc_stats->counters); list_for_each_entry_safe(counter, tmp, &fc_stats->counters, list)
while (node) {
counter = rb_entry(node, struct mlx5_fc, node);
node = rb_next(node);
rb_erase(&counter->node, &fc_stats->counters);
mlx5_free_fc(dev, counter); mlx5_free_fc(dev, counter);
}
} }
int mlx5_fc_query(struct mlx5_core_dev *dev, struct mlx5_fc *counter, int mlx5_fc_query(struct mlx5_core_dev *dev, struct mlx5_fc *counter,
......
...@@ -583,7 +583,7 @@ struct mlx5_irq_info { ...@@ -583,7 +583,7 @@ struct mlx5_irq_info {
}; };
struct mlx5_fc_stats { struct mlx5_fc_stats {
struct rb_root counters; struct list_head counters;
struct llist_head addlist; struct llist_head addlist;
struct llist_head dellist; struct llist_head dellist;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment