Commit dc54a08c authored by stephen hemminger's avatar stephen hemminger Committed by David S. Miller

netvsc: optimize receive path

Do manual optimizations of receive path:
  - remove checks for impossible conditions (but keep checks
    for bad data from host)
  - pass argument down, rather than having callee recompute what
    is already known
  - remove indirection about receive buffer datalength
  - remove dependence on VLAN_TAG_PRESENCE
  - use _hot/_cold and likely/unlikely
Signed-off-by: default avatarStephen Hemminger <sthemmin@microsoft.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent b8b835a8
...@@ -119,6 +119,7 @@ struct ndis_recv_scale_param { /* NDIS_RECEIVE_SCALE_PARAMETERS */ ...@@ -119,6 +119,7 @@ struct ndis_recv_scale_param { /* NDIS_RECEIVE_SCALE_PARAMETERS */
/* Fwd declaration */ /* Fwd declaration */
struct ndis_tcp_ip_checksum_info; struct ndis_tcp_ip_checksum_info;
struct ndis_pkt_8021q_info;
/* /*
* Represent netvsc packet which contains 1 RNDIS and 1 ethernet frame * Represent netvsc packet which contains 1 RNDIS and 1 ethernet frame
...@@ -186,12 +187,11 @@ int netvsc_send(struct hv_device *device, ...@@ -186,12 +187,11 @@ int netvsc_send(struct hv_device *device,
struct sk_buff *skb); struct sk_buff *skb);
void netvsc_linkstatus_callback(struct hv_device *device_obj, void netvsc_linkstatus_callback(struct hv_device *device_obj,
struct rndis_message *resp); struct rndis_message *resp);
int netvsc_recv_callback(struct hv_device *device_obj, int netvsc_recv_callback(struct net_device *net,
struct hv_netvsc_packet *packet,
void **data,
struct ndis_tcp_ip_checksum_info *csum_info,
struct vmbus_channel *channel, struct vmbus_channel *channel,
u16 vlan_tci); void *data, u32 len,
const struct ndis_tcp_ip_checksum_info *csum_info,
const struct ndis_pkt_8021q_info *vlan);
void netvsc_channel_cb(void *context); void netvsc_channel_cb(void *context);
int rndis_filter_open(struct netvsc_device *nvdev); int rndis_filter_open(struct netvsc_device *nvdev);
int rndis_filter_close(struct netvsc_device *nvdev); int rndis_filter_close(struct netvsc_device *nvdev);
...@@ -200,10 +200,11 @@ int rndis_filter_device_add(struct hv_device *dev, ...@@ -200,10 +200,11 @@ int rndis_filter_device_add(struct hv_device *dev,
void rndis_filter_device_remove(struct hv_device *dev); void rndis_filter_device_remove(struct hv_device *dev);
int rndis_filter_set_rss_param(struct rndis_device *rdev, int rndis_filter_set_rss_param(struct rndis_device *rdev,
const u8 *key, int num_queue); const u8 *key, int num_queue);
int rndis_filter_receive(struct hv_device *dev, int rndis_filter_receive(struct net_device *ndev,
struct hv_netvsc_packet *pkt, struct netvsc_device *net_dev,
void **data, struct hv_device *dev,
struct vmbus_channel *channel); struct vmbus_channel *channel,
void *data, u32 buflen);
int rndis_filter_set_packet_filter(struct rndis_device *dev, u32 new_filter); int rndis_filter_set_packet_filter(struct rndis_device *dev, u32 new_filter);
int rndis_filter_set_device_mac(struct net_device *ndev, char *mac); int rndis_filter_set_device_mac(struct net_device *ndev, char *mac);
......
...@@ -1095,50 +1095,34 @@ static inline struct recv_comp_data *get_recv_comp_slot( ...@@ -1095,50 +1095,34 @@ static inline struct recv_comp_data *get_recv_comp_slot(
return rcd; return rcd;
} }
static void netvsc_receive(struct netvsc_device *net_device, static void netvsc_receive(struct net_device *ndev,
struct vmbus_channel *channel, struct netvsc_device *net_device,
struct net_device_context *net_device_ctx,
struct hv_device *device, struct hv_device *device,
struct vmpacket_descriptor *packet) struct vmbus_channel *channel,
struct vmtransfer_page_packet_header *vmxferpage_packet,
struct nvsp_message *nvsp)
{ {
struct vmtransfer_page_packet_header *vmxferpage_packet; char *recv_buf = net_device->recv_buf;
struct nvsp_message *nvsp_packet;
struct hv_netvsc_packet nv_pkt;
struct hv_netvsc_packet *netvsc_packet = &nv_pkt;
u32 status = NVSP_STAT_SUCCESS; u32 status = NVSP_STAT_SUCCESS;
int i; int i;
int count = 0; int count = 0;
struct net_device *ndev = hv_get_drvdata(device);
void *data;
int ret; int ret;
struct recv_comp_data *rcd; struct recv_comp_data *rcd;
u16 q_idx = channel->offermsg.offer.sub_channel_index; u16 q_idx = channel->offermsg.offer.sub_channel_index;
/*
* All inbound packets other than send completion should be xfer page
* packet
*/
if (packet->type != VM_PKT_DATA_USING_XFER_PAGES) {
netdev_err(ndev, "Unknown packet type received - %d\n",
packet->type);
return;
}
nvsp_packet = (struct nvsp_message *)((unsigned long)packet +
(packet->offset8 << 3));
/* Make sure this is a valid nvsp packet */ /* Make sure this is a valid nvsp packet */
if (nvsp_packet->hdr.msg_type != if (unlikely(nvsp->hdr.msg_type != NVSP_MSG1_TYPE_SEND_RNDIS_PKT)) {
NVSP_MSG1_TYPE_SEND_RNDIS_PKT) { netif_err(net_device_ctx, rx_err, ndev,
netdev_err(ndev, "Unknown nvsp packet type received-" "Unknown nvsp packet type received %u\n",
" %d\n", nvsp_packet->hdr.msg_type); nvsp->hdr.msg_type);
return; return;
} }
vmxferpage_packet = (struct vmtransfer_page_packet_header *)packet; if (unlikely(vmxferpage_packet->xfer_pageset_id != NETVSC_RECEIVE_BUFFER_ID)) {
netif_err(net_device_ctx, rx_err, ndev,
if (vmxferpage_packet->xfer_pageset_id != NETVSC_RECEIVE_BUFFER_ID) { "Invalid xfer page set id - expecting %x got %x\n",
netdev_err(ndev, "Invalid xfer page set id - " NETVSC_RECEIVE_BUFFER_ID,
"expecting %x got %x\n", NETVSC_RECEIVE_BUFFER_ID,
vmxferpage_packet->xfer_pageset_id); vmxferpage_packet->xfer_pageset_id);
return; return;
} }
...@@ -1147,15 +1131,13 @@ static void netvsc_receive(struct netvsc_device *net_device, ...@@ -1147,15 +1131,13 @@ static void netvsc_receive(struct netvsc_device *net_device,
/* Each range represents 1 RNDIS pkt that contains 1 ethernet frame */ /* Each range represents 1 RNDIS pkt that contains 1 ethernet frame */
for (i = 0; i < count; i++) { for (i = 0; i < count; i++) {
/* Initialize the netvsc packet */ void *data = recv_buf
data = (void *)((unsigned long)net_device-> + vmxferpage_packet->ranges[i].byte_offset;
recv_buf + vmxferpage_packet->ranges[i].byte_offset); u32 buflen = vmxferpage_packet->ranges[i].byte_count;
netvsc_packet->total_data_buflen =
vmxferpage_packet->ranges[i].byte_count;
/* Pass it to the upper layer */ /* Pass it to the upper layer */
status = rndis_filter_receive(device, netvsc_packet, &data, status = rndis_filter_receive(ndev, net_device, device,
channel); channel, data, buflen);
} }
if (!net_device->chan_table[q_idx].mrc.buf) { if (!net_device->chan_table[q_idx].mrc.buf) {
...@@ -1234,11 +1216,10 @@ static void netvsc_process_raw_pkt(struct hv_device *device, ...@@ -1234,11 +1216,10 @@ static void netvsc_process_raw_pkt(struct hv_device *device,
u64 request_id, u64 request_id,
struct vmpacket_descriptor *desc) struct vmpacket_descriptor *desc)
{ {
struct nvsp_message *nvmsg;
struct net_device_context *net_device_ctx = netdev_priv(ndev); struct net_device_context *net_device_ctx = netdev_priv(ndev);
struct nvsp_message *nvmsg
nvmsg = (struct nvsp_message *)((unsigned long) = (struct nvsp_message *)((unsigned long)desc
desc + (desc->offset8 << 3)); + (desc->offset8 << 3));
switch (desc->type) { switch (desc->type) {
case VM_PKT_COMP: case VM_PKT_COMP:
...@@ -1246,7 +1227,10 @@ static void netvsc_process_raw_pkt(struct hv_device *device, ...@@ -1246,7 +1227,10 @@ static void netvsc_process_raw_pkt(struct hv_device *device,
break; break;
case VM_PKT_DATA_USING_XFER_PAGES: case VM_PKT_DATA_USING_XFER_PAGES:
netvsc_receive(net_device, channel, device, desc); netvsc_receive(ndev, net_device, net_device_ctx,
device, channel,
(struct vmtransfer_page_packet_header *)desc,
nvmsg);
break; break;
case VM_PKT_DATA_INBAND: case VM_PKT_DATA_INBAND:
......
...@@ -596,13 +596,13 @@ void netvsc_linkstatus_callback(struct hv_device *device_obj, ...@@ -596,13 +596,13 @@ void netvsc_linkstatus_callback(struct hv_device *device_obj,
} }
static struct sk_buff *netvsc_alloc_recv_skb(struct net_device *net, static struct sk_buff *netvsc_alloc_recv_skb(struct net_device *net,
struct hv_netvsc_packet *packet, const struct ndis_tcp_ip_checksum_info *csum_info,
struct ndis_tcp_ip_checksum_info *csum_info, const struct ndis_pkt_8021q_info *vlan,
void *data, u16 vlan_tci) void *data, u32 buflen)
{ {
struct sk_buff *skb; struct sk_buff *skb;
skb = netdev_alloc_skb_ip_align(net, packet->total_data_buflen); skb = netdev_alloc_skb_ip_align(net, buflen);
if (!skb) if (!skb)
return skb; return skb;
...@@ -610,8 +610,7 @@ static struct sk_buff *netvsc_alloc_recv_skb(struct net_device *net, ...@@ -610,8 +610,7 @@ static struct sk_buff *netvsc_alloc_recv_skb(struct net_device *net,
* Copy to skb. This copy is needed here since the memory pointed by * Copy to skb. This copy is needed here since the memory pointed by
* hv_netvsc_packet cannot be deallocated * hv_netvsc_packet cannot be deallocated
*/ */
memcpy(skb_put(skb, packet->total_data_buflen), data, memcpy(skb_put(skb, buflen), data, buflen);
packet->total_data_buflen);
skb->protocol = eth_type_trans(skb, net); skb->protocol = eth_type_trans(skb, net);
...@@ -628,9 +627,12 @@ static struct sk_buff *netvsc_alloc_recv_skb(struct net_device *net, ...@@ -628,9 +627,12 @@ static struct sk_buff *netvsc_alloc_recv_skb(struct net_device *net,
skb->ip_summed = CHECKSUM_UNNECESSARY; skb->ip_summed = CHECKSUM_UNNECESSARY;
} }
if (vlan_tci & VLAN_TAG_PRESENT) if (vlan) {
u16 vlan_tci = vlan->vlanid | (vlan->pri << VLAN_PRIO_SHIFT);
__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
vlan_tci); vlan_tci);
}
return skb; return skb;
} }
...@@ -639,14 +641,12 @@ static struct sk_buff *netvsc_alloc_recv_skb(struct net_device *net, ...@@ -639,14 +641,12 @@ static struct sk_buff *netvsc_alloc_recv_skb(struct net_device *net,
* netvsc_recv_callback - Callback when we receive a packet from the * netvsc_recv_callback - Callback when we receive a packet from the
* "wire" on the specified device. * "wire" on the specified device.
*/ */
int netvsc_recv_callback(struct hv_device *device_obj, int netvsc_recv_callback(struct net_device *net,
struct hv_netvsc_packet *packet,
void **data,
struct ndis_tcp_ip_checksum_info *csum_info,
struct vmbus_channel *channel, struct vmbus_channel *channel,
u16 vlan_tci) void *data, u32 len,
const struct ndis_tcp_ip_checksum_info *csum_info,
const struct ndis_pkt_8021q_info *vlan)
{ {
struct net_device *net = hv_get_drvdata(device_obj);
struct net_device_context *net_device_ctx = netdev_priv(net); struct net_device_context *net_device_ctx = netdev_priv(net);
struct net_device *vf_netdev; struct net_device *vf_netdev;
struct sk_buff *skb; struct sk_buff *skb;
...@@ -668,7 +668,7 @@ int netvsc_recv_callback(struct hv_device *device_obj, ...@@ -668,7 +668,7 @@ int netvsc_recv_callback(struct hv_device *device_obj,
net = vf_netdev; net = vf_netdev;
/* Allocate a skb - TODO direct I/O to pages? */ /* Allocate a skb - TODO direct I/O to pages? */
skb = netvsc_alloc_recv_skb(net, packet, csum_info, *data, vlan_tci); skb = netvsc_alloc_recv_skb(net, csum_info, vlan, data, len);
if (unlikely(!skb)) { if (unlikely(!skb)) {
++net->stats.rx_dropped; ++net->stats.rx_dropped;
rcu_read_unlock(); rcu_read_unlock();
...@@ -687,7 +687,7 @@ int netvsc_recv_callback(struct hv_device *device_obj, ...@@ -687,7 +687,7 @@ int netvsc_recv_callback(struct hv_device *device_obj,
rx_stats = this_cpu_ptr(net_device_ctx->rx_stats); rx_stats = this_cpu_ptr(net_device_ctx->rx_stats);
u64_stats_update_begin(&rx_stats->syncp); u64_stats_update_begin(&rx_stats->syncp);
rx_stats->packets++; rx_stats->packets++;
rx_stats->bytes += packet->total_data_buflen; rx_stats->bytes += len;
if (skb->pkt_type == PACKET_BROADCAST) if (skb->pkt_type == PACKET_BROADCAST)
++rx_stats->broadcast; ++rx_stats->broadcast;
......
...@@ -132,7 +132,7 @@ static void put_rndis_request(struct rndis_device *dev, ...@@ -132,7 +132,7 @@ static void put_rndis_request(struct rndis_device *dev,
} }
static void dump_rndis_message(struct hv_device *hv_dev, static void dump_rndis_message(struct hv_device *hv_dev,
struct rndis_message *rndis_msg) const struct rndis_message *rndis_msg)
{ {
struct net_device *netdev = hv_get_drvdata(hv_dev); struct net_device *netdev = hv_get_drvdata(hv_dev);
...@@ -347,102 +347,78 @@ static inline void *rndis_get_ppi(struct rndis_packet *rpkt, u32 type) ...@@ -347,102 +347,78 @@ static inline void *rndis_get_ppi(struct rndis_packet *rpkt, u32 type)
return NULL; return NULL;
} }
static int rndis_filter_receive_data(struct rndis_device *dev, static int rndis_filter_receive_data(struct net_device *ndev,
struct rndis_device *dev,
struct rndis_message *msg, struct rndis_message *msg,
struct hv_netvsc_packet *pkt, struct vmbus_channel *channel,
void **data, void *data, u32 data_buflen)
struct vmbus_channel *channel)
{ {
struct rndis_packet *rndis_pkt; struct rndis_packet *rndis_pkt = &msg->msg.pkt;
const struct ndis_tcp_ip_checksum_info *csum_info;
const struct ndis_pkt_8021q_info *vlan;
u32 data_offset; u32 data_offset;
struct ndis_pkt_8021q_info *vlan;
struct ndis_tcp_ip_checksum_info *csum_info;
u16 vlan_tci = 0;
struct net_device_context *net_device_ctx = netdev_priv(dev->ndev);
rndis_pkt = &msg->msg.pkt;
/* Remove the rndis header and pass it back up the stack */ /* Remove the rndis header and pass it back up the stack */
data_offset = RNDIS_HEADER_SIZE + rndis_pkt->data_offset; data_offset = RNDIS_HEADER_SIZE + rndis_pkt->data_offset;
pkt->total_data_buflen -= data_offset; data_buflen -= data_offset;
/* /*
* Make sure we got a valid RNDIS message, now total_data_buflen * Make sure we got a valid RNDIS message, now total_data_buflen
* should be the data packet size plus the trailer padding size * should be the data packet size plus the trailer padding size
*/ */
if (pkt->total_data_buflen < rndis_pkt->data_len) { if (unlikely(data_buflen < rndis_pkt->data_len)) {
netdev_err(dev->ndev, "rndis message buffer " netdev_err(dev->ndev, "rndis message buffer "
"overflow detected (got %u, min %u)" "overflow detected (got %u, min %u)"
"...dropping this message!\n", "...dropping this message!\n",
pkt->total_data_buflen, rndis_pkt->data_len); data_buflen, rndis_pkt->data_len);
return NVSP_STAT_FAIL; return NVSP_STAT_FAIL;
} }
vlan = rndis_get_ppi(rndis_pkt, IEEE_8021Q_INFO);
/* /*
* Remove the rndis trailer padding from rndis packet message * Remove the rndis trailer padding from rndis packet message
* rndis_pkt->data_len tell us the real data length, we only copy * rndis_pkt->data_len tell us the real data length, we only copy
* the data packet to the stack, without the rndis trailer padding * the data packet to the stack, without the rndis trailer padding
*/ */
pkt->total_data_buflen = rndis_pkt->data_len; data = (void *)((unsigned long)data + data_offset);
*data = (void *)((unsigned long)(*data) + data_offset);
vlan = rndis_get_ppi(rndis_pkt, IEEE_8021Q_INFO);
if (vlan) {
vlan_tci = VLAN_TAG_PRESENT | vlan->vlanid |
(vlan->pri << VLAN_PRIO_SHIFT);
}
csum_info = rndis_get_ppi(rndis_pkt, TCPIP_CHKSUM_PKTINFO); csum_info = rndis_get_ppi(rndis_pkt, TCPIP_CHKSUM_PKTINFO);
return netvsc_recv_callback(net_device_ctx->device_ctx, pkt, data, return netvsc_recv_callback(ndev, channel,
csum_info, channel, vlan_tci); data, rndis_pkt->data_len,
csum_info, vlan);
} }
int rndis_filter_receive(struct hv_device *dev, int rndis_filter_receive(struct net_device *ndev,
struct hv_netvsc_packet *pkt, struct netvsc_device *net_dev,
void **data, struct hv_device *dev,
struct vmbus_channel *channel) struct vmbus_channel *channel,
void *data, u32 buflen)
{ {
struct net_device *ndev = hv_get_drvdata(dev);
struct net_device_context *net_device_ctx = netdev_priv(ndev); struct net_device_context *net_device_ctx = netdev_priv(ndev);
struct netvsc_device *net_dev = net_device_ctx->nvdev; struct rndis_device *rndis_dev = net_dev->extension;
struct rndis_device *rndis_dev; struct rndis_message *rndis_msg = data;
struct rndis_message *rndis_msg;
int ret = 0;
if (!net_dev) {
ret = NVSP_STAT_FAIL;
goto exit;
}
/* Make sure the rndis device state is initialized */ /* Make sure the rndis device state is initialized */
if (!net_dev->extension) { if (unlikely(!rndis_dev)) {
netdev_err(ndev, "got rndis message but no rndis device - " netif_err(net_device_ctx, rx_err, ndev,
"dropping this message!\n"); "got rndis message but no rndis device!\n");
ret = NVSP_STAT_FAIL; return NVSP_STAT_FAIL;
goto exit;
} }
rndis_dev = (struct rndis_device *)net_dev->extension; if (unlikely(rndis_dev->state == RNDIS_DEV_UNINITIALIZED)) {
if (rndis_dev->state == RNDIS_DEV_UNINITIALIZED) { netif_err(net_device_ctx, rx_err, ndev,
netdev_err(ndev, "got rndis message but rndis device " "got rndis message uninitialized\n");
"uninitialized...dropping this message!\n"); return NVSP_STAT_FAIL;
ret = NVSP_STAT_FAIL;
goto exit;
} }
rndis_msg = *data; if (netif_msg_rx_status(net_device_ctx))
if (netif_msg_rx_err(net_device_ctx))
dump_rndis_message(dev, rndis_msg); dump_rndis_message(dev, rndis_msg);
switch (rndis_msg->ndis_msg_type) { switch (rndis_msg->ndis_msg_type) {
case RNDIS_MSG_PACKET: case RNDIS_MSG_PACKET:
/* data msg */ return rndis_filter_receive_data(ndev, rndis_dev, rndis_msg,
ret = rndis_filter_receive_data(rndis_dev, rndis_msg, pkt, channel, data, buflen);
data, channel);
break;
case RNDIS_MSG_INIT_C: case RNDIS_MSG_INIT_C:
case RNDIS_MSG_QUERY_C: case RNDIS_MSG_QUERY_C:
case RNDIS_MSG_SET_C: case RNDIS_MSG_SET_C:
...@@ -462,8 +438,7 @@ int rndis_filter_receive(struct hv_device *dev, ...@@ -462,8 +438,7 @@ int rndis_filter_receive(struct hv_device *dev,
break; break;
} }
exit: return 0;
return ret;
} }
static int rndis_filter_query_device(struct rndis_device *dev, u32 oid, static int rndis_filter_query_device(struct rndis_device *dev, u32 oid,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment