Commit 3b47d303 authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller

net: gro: add a per device gro flush timer

Tuning coalescing parameters on NIC can be really hard.

Servers can handle both bulk and RPC like traffic, with conflicting
goals : bulk flows want as big GRO packets as possible, RPC want minimal
latencies.

To reach big GRO packets on 10Gbe NIC, one can use :

ethtool -C eth0 rx-usecs 4 rx-frames 44

But this penalizes rpc sessions, with an increase of latencies, up to
50% in some cases, as NICs generally do not force an interrupt when
a packet with TCP Push flag is received.

Some NICs do not have an absolute timer, only a timer rearmed for every
incoming packet.

This patch uses a different strategy : Let GRO stack decides what do do,
based on traffic pattern.

Packets with Push flag wont be delayed.
Packets without Push flag might be held in GRO engine, if we keep
receiving data.

This new mechanism is off by default, and shall be enabled by setting
/sys/class/net/ethX/gro_flush_timeout to a value in nanosecond.

To fully enable this mechanism, drivers should use napi_complete_done()
instead of napi_complete().

Tested:
 Ran 200 netperf TCP_STREAM from A to B (10Gbe mlx4 link, 8 RX queues)

Without this feature, we send back about 305,000 ACK per second.

GRO aggregation ratio is low (811/305 = 2.65 segments per GRO packet)

Setting a timer of 2000 nsec is enough to increase GRO packet sizes
and reduce number of ACK packets. (811/19.2 = 42)

Receiver performs less calls to upper stacks, less wakes up.
This also reduces cpu usage on the sender, as it receives less ACK
packets.

Note that reducing number of wakes up increases cpu efficiency, but can
decrease QPS, as applications wont have the chance to warmup cpu caches
doing a partial read of RPC requests/answers if they fit in one skb.

B:~# sar -n DEV 1 10 | grep eth0 | tail -1
Average:         eth0 811269.80 305732.30 1199462.57  19705.72      0.00
0.00      0.50

B:~# echo 2000 >/sys/class/net/eth0/gro_flush_timeout

B:~# sar -n DEV 1 10 | grep eth0 | tail -1
Average:         eth0 811577.30  19230.80 1199916.51   1239.80      0.00
0.00      0.50
Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent be955b29
...@@ -314,6 +314,7 @@ struct napi_struct { ...@@ -314,6 +314,7 @@ struct napi_struct {
struct net_device *dev; struct net_device *dev;
struct sk_buff *gro_list; struct sk_buff *gro_list;
struct sk_buff *skb; struct sk_buff *skb;
struct hrtimer timer;
struct list_head dev_list; struct list_head dev_list;
struct hlist_node napi_hash_node; struct hlist_node napi_hash_node;
unsigned int napi_id; unsigned int napi_id;
...@@ -443,14 +444,19 @@ static inline bool napi_reschedule(struct napi_struct *napi) ...@@ -443,14 +444,19 @@ static inline bool napi_reschedule(struct napi_struct *napi)
return false; return false;
} }
void __napi_complete(struct napi_struct *n);
void napi_complete_done(struct napi_struct *n, int work_done);
/** /**
* napi_complete - NAPI processing complete * napi_complete - NAPI processing complete
* @n: napi context * @n: napi context
* *
* Mark NAPI processing as complete. * Mark NAPI processing as complete.
* Consider using napi_complete_done() instead.
*/ */
void __napi_complete(struct napi_struct *n); static inline void napi_complete(struct napi_struct *n)
void napi_complete(struct napi_struct *n); {
return napi_complete_done(n, 0);
}
/** /**
* napi_by_id - lookup a NAPI by napi_id * napi_by_id - lookup a NAPI by napi_id
...@@ -485,14 +491,7 @@ void napi_hash_del(struct napi_struct *napi); ...@@ -485,14 +491,7 @@ void napi_hash_del(struct napi_struct *napi);
* Stop NAPI from being scheduled on this context. * Stop NAPI from being scheduled on this context.
* Waits till any outstanding processing completes. * Waits till any outstanding processing completes.
*/ */
static inline void napi_disable(struct napi_struct *n) void napi_disable(struct napi_struct *n);
{
might_sleep();
set_bit(NAPI_STATE_DISABLE, &n->state);
while (test_and_set_bit(NAPI_STATE_SCHED, &n->state))
msleep(1);
clear_bit(NAPI_STATE_DISABLE, &n->state);
}
/** /**
* napi_enable - enable NAPI scheduling * napi_enable - enable NAPI scheduling
...@@ -1603,6 +1602,7 @@ struct net_device { ...@@ -1603,6 +1602,7 @@ struct net_device {
#endif #endif
unsigned long gro_flush_timeout;
rx_handler_func_t __rcu *rx_handler; rx_handler_func_t __rcu *rx_handler;
void __rcu *rx_handler_data; void __rcu *rx_handler_data;
......
...@@ -134,6 +134,7 @@ ...@@ -134,6 +134,7 @@
#include <linux/vmalloc.h> #include <linux/vmalloc.h>
#include <linux/if_macvlan.h> #include <linux/if_macvlan.h>
#include <linux/errqueue.h> #include <linux/errqueue.h>
#include <linux/hrtimer.h>
#include "net-sysfs.h" #include "net-sysfs.h"
...@@ -4412,7 +4413,6 @@ EXPORT_SYMBOL(__napi_schedule_irqoff); ...@@ -4412,7 +4413,6 @@ EXPORT_SYMBOL(__napi_schedule_irqoff);
void __napi_complete(struct napi_struct *n) void __napi_complete(struct napi_struct *n)
{ {
BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state)); BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
BUG_ON(n->gro_list);
list_del_init(&n->poll_list); list_del_init(&n->poll_list);
smp_mb__before_atomic(); smp_mb__before_atomic();
...@@ -4420,7 +4420,7 @@ void __napi_complete(struct napi_struct *n) ...@@ -4420,7 +4420,7 @@ void __napi_complete(struct napi_struct *n)
} }
EXPORT_SYMBOL(__napi_complete); EXPORT_SYMBOL(__napi_complete);
void napi_complete(struct napi_struct *n) void napi_complete_done(struct napi_struct *n, int work_done)
{ {
unsigned long flags; unsigned long flags;
...@@ -4431,8 +4431,18 @@ void napi_complete(struct napi_struct *n) ...@@ -4431,8 +4431,18 @@ void napi_complete(struct napi_struct *n)
if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state))) if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))
return; return;
napi_gro_flush(n, false); if (n->gro_list) {
unsigned long timeout = 0;
if (work_done)
timeout = n->dev->gro_flush_timeout;
if (timeout)
hrtimer_start(&n->timer, ns_to_ktime(timeout),
HRTIMER_MODE_REL_PINNED);
else
napi_gro_flush(n, false);
}
if (likely(list_empty(&n->poll_list))) { if (likely(list_empty(&n->poll_list))) {
WARN_ON_ONCE(!test_and_clear_bit(NAPI_STATE_SCHED, &n->state)); WARN_ON_ONCE(!test_and_clear_bit(NAPI_STATE_SCHED, &n->state));
} else { } else {
...@@ -4442,7 +4452,7 @@ void napi_complete(struct napi_struct *n) ...@@ -4442,7 +4452,7 @@ void napi_complete(struct napi_struct *n)
local_irq_restore(flags); local_irq_restore(flags);
} }
} }
EXPORT_SYMBOL(napi_complete); EXPORT_SYMBOL(napi_complete_done);
/* must be called under rcu_read_lock(), as we dont take a reference */ /* must be called under rcu_read_lock(), as we dont take a reference */
struct napi_struct *napi_by_id(unsigned int napi_id) struct napi_struct *napi_by_id(unsigned int napi_id)
...@@ -4496,10 +4506,23 @@ void napi_hash_del(struct napi_struct *napi) ...@@ -4496,10 +4506,23 @@ void napi_hash_del(struct napi_struct *napi)
} }
EXPORT_SYMBOL_GPL(napi_hash_del); EXPORT_SYMBOL_GPL(napi_hash_del);
static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
{
struct napi_struct *napi;
napi = container_of(timer, struct napi_struct, timer);
if (napi->gro_list)
napi_schedule(napi);
return HRTIMER_NORESTART;
}
void netif_napi_add(struct net_device *dev, struct napi_struct *napi, void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
int (*poll)(struct napi_struct *, int), int weight) int (*poll)(struct napi_struct *, int), int weight)
{ {
INIT_LIST_HEAD(&napi->poll_list); INIT_LIST_HEAD(&napi->poll_list);
hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
napi->timer.function = napi_watchdog;
napi->gro_count = 0; napi->gro_count = 0;
napi->gro_list = NULL; napi->gro_list = NULL;
napi->skb = NULL; napi->skb = NULL;
...@@ -4518,6 +4541,20 @@ void netif_napi_add(struct net_device *dev, struct napi_struct *napi, ...@@ -4518,6 +4541,20 @@ void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
} }
EXPORT_SYMBOL(netif_napi_add); EXPORT_SYMBOL(netif_napi_add);
void napi_disable(struct napi_struct *n)
{
might_sleep();
set_bit(NAPI_STATE_DISABLE, &n->state);
while (test_and_set_bit(NAPI_STATE_SCHED, &n->state))
msleep(1);
hrtimer_cancel(&n->timer);
clear_bit(NAPI_STATE_DISABLE, &n->state);
}
EXPORT_SYMBOL(napi_disable);
void netif_napi_del(struct napi_struct *napi) void netif_napi_del(struct napi_struct *napi)
{ {
list_del_init(&napi->dev_list); list_del_init(&napi->dev_list);
......
...@@ -325,6 +325,23 @@ static ssize_t tx_queue_len_store(struct device *dev, ...@@ -325,6 +325,23 @@ static ssize_t tx_queue_len_store(struct device *dev,
} }
NETDEVICE_SHOW_RW(tx_queue_len, fmt_ulong); NETDEVICE_SHOW_RW(tx_queue_len, fmt_ulong);
static int change_gro_flush_timeout(struct net_device *dev, unsigned long val)
{
dev->gro_flush_timeout = val;
return 0;
}
static ssize_t gro_flush_timeout_store(struct device *dev,
struct device_attribute *attr,
const char *buf, size_t len)
{
if (!capable(CAP_NET_ADMIN))
return -EPERM;
return netdev_store(dev, attr, buf, len, change_gro_flush_timeout);
}
NETDEVICE_SHOW_RW(gro_flush_timeout, fmt_ulong);
static ssize_t ifalias_store(struct device *dev, struct device_attribute *attr, static ssize_t ifalias_store(struct device *dev, struct device_attribute *attr,
const char *buf, size_t len) const char *buf, size_t len)
{ {
...@@ -422,6 +439,7 @@ static struct attribute *net_class_attrs[] = { ...@@ -422,6 +439,7 @@ static struct attribute *net_class_attrs[] = {
&dev_attr_mtu.attr, &dev_attr_mtu.attr,
&dev_attr_flags.attr, &dev_attr_flags.attr,
&dev_attr_tx_queue_len.attr, &dev_attr_tx_queue_len.attr,
&dev_attr_gro_flush_timeout.attr,
&dev_attr_phys_port_id.attr, &dev_attr_phys_port_id.attr,
NULL, NULL,
}; };
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment