virtio_net.c 49.6 KB
Newer Older
1
/* A network driver using virtio.
Rusty Russell's avatar
Rusty Russell committed
2 3 4 5 6 7 8 9 10 11 12 13 14 15
 *
 * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
16
 * along with this program; if not, see <http://www.gnu.org/licenses/>.
Rusty Russell's avatar
Rusty Russell committed
17 18 19 20
 */
//#define DEBUG
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
21
#include <linux/ethtool.h>
Rusty Russell's avatar
Rusty Russell committed
22 23 24 25
#include <linux/module.h>
#include <linux/virtio.h>
#include <linux/virtio_net.h>
#include <linux/scatterlist.h>
26
#include <linux/if_vlan.h>
27
#include <linux/slab.h>
28
#include <linux/cpu.h>
29
#include <linux/average.h>
30
#include <net/busy_poll.h>
Rusty Russell's avatar
Rusty Russell committed
31

32
static int napi_weight = NAPI_POLL_WEIGHT;
33 34
module_param(napi_weight, int, 0444);

35
static bool csum = true, gso = true;
36 37 38
module_param(csum, bool, 0444);
module_param(gso, bool, 0444);

Rusty Russell's avatar
Rusty Russell committed
39
/* FIXME: MTU in config. */
40
#define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
41
#define GOOD_COPY_LEN	128
Rusty Russell's avatar
Rusty Russell committed
42

43 44 45 46 47 48 49 50 51 52
/* Weight used for the RX packet size EWMA. The average packet size is used to
 * determine the packet buffer size when refilling RX rings. As the entire RX
 * ring may be refilled at once, the weight is chosen so that the EWMA will be
 * insensitive to short-term, transient changes in packet size.
 */
#define RECEIVE_AVG_WEIGHT 64

/* Minimum alignment for mergeable packet buffers. */
#define MERGEABLE_BUFFER_ALIGN max(L1_CACHE_BYTES, 256)

53
#define VIRTNET_DRIVER_VERSION "1.0.0"
54

55
struct virtnet_stats {
56 57
	struct u64_stats_sync tx_syncp;
	struct u64_stats_sync rx_syncp;
58 59 60 61 62 63 64
	u64 tx_bytes;
	u64 tx_packets;

	u64 rx_bytes;
	u64 rx_packets;
};

65 66 67 68 69 70 71
/* Internal representation of a send virtqueue */
struct send_queue {
	/* Virtqueue associated with this send _queue */
	struct virtqueue *vq;

	/* TX: fragments + linear part + virtio header */
	struct scatterlist sg[MAX_SKB_FRAGS + 2];
Jason Wang's avatar
Jason Wang committed
72 73 74

	/* Name of the send queue: output.$index */
	char name[40];
75 76 77 78 79 80 81
};

/* Internal representation of a receive virtqueue */
struct receive_queue {
	/* Virtqueue associated with this receive_queue */
	struct virtqueue *vq;

Rusty Russell's avatar
Rusty Russell committed
82 83
	struct napi_struct napi;

84 85 86
	/* Chain pages by the private ptr. */
	struct page *pages;

87 88 89
	/* Average packet length for mergeable receive buffers. */
	struct ewma mrg_avg_pkt_len;

90 91 92
	/* Page frag for packet buffer allocation. */
	struct page_frag alloc_frag;

93 94
	/* RX: fragments + linear part + virtio header */
	struct scatterlist sg[MAX_SKB_FRAGS + 2];
Jason Wang's avatar
Jason Wang committed
95 96 97

	/* Name of this receive queue: input.$index */
	char name[40];
98 99 100 101 102 103
};

struct virtnet_info {
	struct virtio_device *vdev;
	struct virtqueue *cvq;
	struct net_device *dev;
Jason Wang's avatar
Jason Wang committed
104 105
	struct send_queue *sq;
	struct receive_queue *rq;
106 107
	unsigned int status;

Jason Wang's avatar
Jason Wang committed
108 109 110 111 112 113
	/* Max # of queue pairs supported by the device */
	u16 max_queue_pairs;

	/* # of queue pairs currently used by the driver */
	u16 curr_queue_pairs;

114 115 116
	/* I like... big packets and I cannot lie! */
	bool big_packets;

117 118 119
	/* Host will merge rx buffers for big packets (shake it! shake it!) */
	bool mergeable_rx_bufs;

Jason Wang's avatar
Jason Wang committed
120 121 122
	/* Has control virtqueue */
	bool has_cvq;

123 124 125
	/* Host can handle any s/g split between our header and packet data */
	bool any_header_sg;

126 127 128
	/* Active statistics */
	struct virtnet_stats __percpu *stats;

129 130 131
	/* Work struct for refilling if we run low on memory. */
	struct delayed_work refill;

132 133 134
	/* Work struct for config space updates */
	struct work_struct config_work;

Jason Wang's avatar
Jason Wang committed
135 136
	/* Does the affinity hint is set for virtqueues? */
	bool affinity_hint_set;
137

138 139
	/* CPU hot plug notifier */
	struct notifier_block nb;
Rusty Russell's avatar
Rusty Russell committed
140 141
};

142 143 144 145 146 147 148
struct skb_vnet_hdr {
	union {
		struct virtio_net_hdr hdr;
		struct virtio_net_hdr_mrg_rxbuf mhdr;
	};
};

149 150 151 152 153 154 155 156 157 158
struct padded_vnet_hdr {
	struct virtio_net_hdr hdr;
	/*
	 * virtio_net_hdr should be in a separated sg buffer because of a
	 * QEMU bug, and data sg buffer shares same page with this header sg.
	 * This padding makes next sg 16 byte aligned after virtio_net_hdr.
	 */
	char padding[6];
};

Jason Wang's avatar
Jason Wang committed
159 160 161 162 163
/* Converting between virtqueue no. and kernel tx/rx queue no.
 * 0:rx0 1:tx0 2:rx1 3:tx1 ... 2N:rxN 2N+1:txN 2N+2:cvq
 */
static int vq2txq(struct virtqueue *vq)
{
164
	return (vq->index - 1) / 2;
Jason Wang's avatar
Jason Wang committed
165 166 167 168 169 170 171 172 173
}

static int txq2vq(int txq)
{
	return txq * 2 + 1;
}

static int vq2rxq(struct virtqueue *vq)
{
174
	return vq->index / 2;
Jason Wang's avatar
Jason Wang committed
175 176 177 178 179 180 181
}

static int rxq2vq(int rxq)
{
	return rxq * 2;
}

182
static inline struct skb_vnet_hdr *skb_vnet_hdr(struct sk_buff *skb)
Rusty Russell's avatar
Rusty Russell committed
183
{
184
	return (struct skb_vnet_hdr *)skb->cb;
Rusty Russell's avatar
Rusty Russell committed
185 186
}

187 188 189 190
/*
 * private is used to chain pages for big packets, put the whole
 * most recent used list in the beginning for reuse
 */
191
static void give_pages(struct receive_queue *rq, struct page *page)
192
{
193
	struct page *end;
194

195
	/* Find end of list, sew whole thing into vi->rq.pages. */
196
	for (end = page; end->private; end = (struct page *)end->private);
197 198
	end->private = (unsigned long)rq->pages;
	rq->pages = page;
199 200
}

201
static struct page *get_a_page(struct receive_queue *rq, gfp_t gfp_mask)
202
{
203
	struct page *p = rq->pages;
204

205
	if (p) {
206
		rq->pages = (struct page *)p->private;
207 208 209
		/* clear private here, it is used to chain pages */
		p->private = 0;
	} else
210 211 212 213
		p = alloc_page(gfp_mask);
	return p;
}

214
static void skb_xmit_done(struct virtqueue *vq)
Rusty Russell's avatar
Rusty Russell committed
215
{
216
	struct virtnet_info *vi = vq->vdev->priv;
Rusty Russell's avatar
Rusty Russell committed
217

218
	/* Suppress further interrupts. */
219
	virtqueue_disable_cb(vq);
220

221
	/* We were probably waiting for more output buffers. */
Jason Wang's avatar
Jason Wang committed
222
	netif_wake_subqueue(vi->dev, vq2txq(vq));
Rusty Russell's avatar
Rusty Russell committed
223 224
}

225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242
static unsigned int mergeable_ctx_to_buf_truesize(unsigned long mrg_ctx)
{
	unsigned int truesize = mrg_ctx & (MERGEABLE_BUFFER_ALIGN - 1);
	return (truesize + 1) * MERGEABLE_BUFFER_ALIGN;
}

static void *mergeable_ctx_to_buf_address(unsigned long mrg_ctx)
{
	return (void *)(mrg_ctx & -MERGEABLE_BUFFER_ALIGN);

}

static unsigned long mergeable_buf_to_ctx(void *buf, unsigned int truesize)
{
	unsigned int size = truesize / MERGEABLE_BUFFER_ALIGN;
	return (unsigned long)buf | (size - 1);
}

243
/* Called from bottom half context */
244
static struct sk_buff *page_to_skb(struct receive_queue *rq,
245 246
				   struct page *page, unsigned int offset,
				   unsigned int len, unsigned int truesize)
247
{
248
	struct virtnet_info *vi = rq->vq->vdev->priv;
249 250
	struct sk_buff *skb;
	struct skb_vnet_hdr *hdr;
251
	unsigned int copy, hdr_len, hdr_padded_len;
252
	char *p;
253

254
	p = page_address(page) + offset;
255

256 257 258 259
	/* copy small packet so we can reuse these pages for small data */
	skb = netdev_alloc_skb_ip_align(vi->dev, GOOD_COPY_LEN);
	if (unlikely(!skb))
		return NULL;
260

261
	hdr = skb_vnet_hdr(skb);
262

263 264
	if (vi->mergeable_rx_bufs) {
		hdr_len = sizeof hdr->mhdr;
265
		hdr_padded_len = sizeof hdr->mhdr;
266 267
	} else {
		hdr_len = sizeof hdr->hdr;
268
		hdr_padded_len = sizeof(struct padded_vnet_hdr);
269
	}
270

271
	memcpy(hdr, p, hdr_len);
272

273
	len -= hdr_len;
274 275
	offset += hdr_padded_len;
	p += hdr_padded_len;
276

277 278 279 280
	copy = len;
	if (copy > skb_tailroom(skb))
		copy = skb_tailroom(skb);
	memcpy(skb_put(skb, copy), p, copy);
281

282 283
	len -= copy;
	offset += copy;
284

285 286 287 288 289 290 291 292
	if (vi->mergeable_rx_bufs) {
		if (len)
			skb_add_rx_frag(skb, 0, page, offset, len, truesize);
		else
			put_page(page);
		return skb;
	}

293 294 295 296 297 298 299
	/*
	 * Verify that we can indeed put this data into a skb.
	 * This is here to handle cases when the device erroneously
	 * tries to receive more than is possible. This is usually
	 * the case of a broken device.
	 */
	if (unlikely(len > MAX_SKB_FRAGS * PAGE_SIZE)) {
300
		net_dbg_ratelimited("%s: too much data\n", skb->dev->name);
301 302 303
		dev_kfree_skb(skb);
		return NULL;
	}
304
	BUG_ON(offset >= PAGE_SIZE);
305
	while (len) {
306 307 308 309
		unsigned int frag_size = min((unsigned)PAGE_SIZE - offset, len);
		skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, offset,
				frag_size, truesize);
		len -= frag_size;
310 311 312
		page = (struct page *)page->private;
		offset = 0;
	}
313

314
	if (page)
315
		give_pages(rq, page);
316

317 318
	return skb;
}
319

320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348
static struct sk_buff *receive_small(void *buf, unsigned int len)
{
	struct sk_buff * skb = buf;

	len -= sizeof(struct virtio_net_hdr);
	skb_trim(skb, len);

	return skb;
}

static struct sk_buff *receive_big(struct net_device *dev,
				   struct receive_queue *rq,
				   void *buf,
				   unsigned int len)
{
	struct page *page = buf;
	struct sk_buff *skb = page_to_skb(rq, page, 0, len, PAGE_SIZE);

	if (unlikely(!skb))
		goto err;

	return skb;

err:
	dev->stats.rx_dropped++;
	give_pages(rq, page);
	return NULL;
}

349 350
static struct sk_buff *receive_mergeable(struct net_device *dev,
					 struct receive_queue *rq,
351
					 unsigned long ctx,
352
					 unsigned int len)
353
{
354
	void *buf = mergeable_ctx_to_buf_address(ctx);
355 356 357 358
	struct skb_vnet_hdr *hdr = buf;
	int num_buf = hdr->mhdr.num_buffers;
	struct page *page = virt_to_head_page(buf);
	int offset = buf - page_address(page);
359 360
	unsigned int truesize = max(len, mergeable_ctx_to_buf_truesize(ctx));

361
	struct sk_buff *head_skb = page_to_skb(rq, page, offset, len, truesize);
362
	struct sk_buff *curr_skb = head_skb;
363

364 365
	if (unlikely(!curr_skb))
		goto err_skb;
366
	while (--num_buf) {
367 368
		int num_skb_frags;

369 370
		ctx = (unsigned long)virtqueue_get_buf(rq->vq, &len);
		if (unlikely(!ctx)) {
371 372 373 374
			pr_debug("%s: rx error: %d buffers out of %d missing\n",
				 dev->name, num_buf, hdr->mhdr.num_buffers);
			dev->stats.rx_length_errors++;
			goto err_buf;
375
		}
376

377
		buf = mergeable_ctx_to_buf_address(ctx);
378 379 380
		page = virt_to_head_page(buf);

		num_skb_frags = skb_shinfo(curr_skb)->nr_frags;
381 382
		if (unlikely(num_skb_frags == MAX_SKB_FRAGS)) {
			struct sk_buff *nskb = alloc_skb(0, GFP_ATOMIC);
383 384 385

			if (unlikely(!nskb))
				goto err_skb;
386 387 388 389 390 391 392 393
			if (curr_skb == head_skb)
				skb_shinfo(curr_skb)->frag_list = nskb;
			else
				curr_skb->next = nskb;
			curr_skb = nskb;
			head_skb->truesize += nskb->truesize;
			num_skb_frags = 0;
		}
394
		truesize = max(len, mergeable_ctx_to_buf_truesize(ctx));
395 396 397
		if (curr_skb != head_skb) {
			head_skb->data_len += len;
			head_skb->len += len;
398
			head_skb->truesize += truesize;
399
		}
400
		offset = buf - page_address(page);
401 402 403
		if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) {
			put_page(page);
			skb_coalesce_rx_frag(curr_skb, num_skb_frags - 1,
404
					     len, truesize);
405 406
		} else {
			skb_add_rx_frag(curr_skb, num_skb_frags, page,
407
					offset, len, truesize);
408
		}
409 410
	}

411
	ewma_add(&rq->mrg_avg_pkt_len, head_skb->len);
412 413 414 415 416
	return head_skb;

err_skb:
	put_page(page);
	while (--num_buf) {
417 418
		ctx = (unsigned long)virtqueue_get_buf(rq->vq, &len);
		if (unlikely(!ctx)) {
419 420 421 422 423
			pr_debug("%s: rx error: %d buffers missing\n",
				 dev->name, num_buf);
			dev->stats.rx_length_errors++;
			break;
		}
424
		page = virt_to_head_page(mergeable_ctx_to_buf_address(ctx));
425
		put_page(page);
426
	}
427 428 429 430
err_buf:
	dev->stats.rx_dropped++;
	dev_kfree_skb(head_skb);
	return NULL;
431 432
}

433
static void receive_buf(struct receive_queue *rq, void *buf, unsigned int len)
434
{
435 436
	struct virtnet_info *vi = rq->vq->vdev->priv;
	struct net_device *dev = vi->dev;
437
	struct virtnet_stats *stats = this_cpu_ptr(vi->stats);
438 439
	struct sk_buff *skb;
	struct skb_vnet_hdr *hdr;
440

441 442 443
	if (unlikely(len < sizeof(struct virtio_net_hdr) + ETH_HLEN)) {
		pr_debug("%s: short packet %i\n", dev->name, len);
		dev->stats.rx_length_errors++;
444 445 446 447 448
		if (vi->mergeable_rx_bufs) {
			unsigned long ctx = (unsigned long)buf;
			void *base = mergeable_ctx_to_buf_address(ctx);
			put_page(virt_to_head_page(base));
		} else if (vi->big_packets) {
449
			give_pages(rq, buf);
450
		} else {
451
			dev_kfree_skb(buf);
452
		}
453 454
		return;
	}
455

456
	if (vi->mergeable_rx_bufs)
457
		skb = receive_mergeable(dev, rq, (unsigned long)buf, len);
458 459 460 461 462 463 464
	else if (vi->big_packets)
		skb = receive_big(dev, rq, buf, len);
	else
		skb = receive_small(buf, len);

	if (unlikely(!skb))
		return;
465

466
	hdr = skb_vnet_hdr(skb);
467

468
	u64_stats_update_begin(&stats->rx_syncp);
469 470
	stats->rx_bytes += skb->len;
	stats->rx_packets++;
471
	u64_stats_update_end(&stats->rx_syncp);
Rusty Russell's avatar
Rusty Russell committed
472

473
	if (hdr->hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
Rusty Russell's avatar
Rusty Russell committed
474
		pr_debug("Needs csum!\n");
475 476 477
		if (!skb_partial_csum_set(skb,
					  hdr->hdr.csum_start,
					  hdr->hdr.csum_offset))
Rusty Russell's avatar
Rusty Russell committed
478
			goto frame_err;
479 480
	} else if (hdr->hdr.flags & VIRTIO_NET_HDR_F_DATA_VALID) {
		skb->ip_summed = CHECKSUM_UNNECESSARY;
Rusty Russell's avatar
Rusty Russell committed
481 482
	}

483 484 485 486
	skb->protocol = eth_type_trans(skb, dev);
	pr_debug("Receiving skb proto 0x%04x len %i type %i\n",
		 ntohs(skb->protocol), skb->len, skb->pkt_type);

487
	if (hdr->hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
Rusty Russell's avatar
Rusty Russell committed
488
		pr_debug("GSO!\n");
489
		switch (hdr->hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
Rusty Russell's avatar
Rusty Russell committed
490
		case VIRTIO_NET_HDR_GSO_TCPV4:
491
			skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
Rusty Russell's avatar
Rusty Russell committed
492 493
			break;
		case VIRTIO_NET_HDR_GSO_UDP:
494
			skb_shinfo(skb)->gso_type = SKB_GSO_UDP;
Rusty Russell's avatar
Rusty Russell committed
495 496
			break;
		case VIRTIO_NET_HDR_GSO_TCPV6:
497
			skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
Rusty Russell's avatar
Rusty Russell committed
498 499
			break;
		default:
500 501
			net_warn_ratelimited("%s: bad gso type %u.\n",
					     dev->name, hdr->hdr.gso_type);
Rusty Russell's avatar
Rusty Russell committed
502 503 504
			goto frame_err;
		}

505
		if (hdr->hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
506
			skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;
507

508
		skb_shinfo(skb)->gso_size = hdr->hdr.gso_size;
Rusty Russell's avatar
Rusty Russell committed
509
		if (skb_shinfo(skb)->gso_size == 0) {
510
			net_warn_ratelimited("%s: zero gso size.\n", dev->name);
Rusty Russell's avatar
Rusty Russell committed
511 512 513 514 515 516 517 518
			goto frame_err;
		}

		/* Header must be checked, and gso_segs computed. */
		skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
		skb_shinfo(skb)->gso_segs = 0;
	}

519 520
	skb_mark_napi_id(skb, &rq->napi);

Rusty Russell's avatar
Rusty Russell committed
521 522 523 524 525 526 527 528
	netif_receive_skb(skb);
	return;

frame_err:
	dev->stats.rx_frame_errors++;
	dev_kfree_skb(skb);
}

529
static int add_recvbuf_small(struct receive_queue *rq, gfp_t gfp)
Rusty Russell's avatar
Rusty Russell committed
530
{
531
	struct virtnet_info *vi = rq->vq->vdev->priv;
Rusty Russell's avatar
Rusty Russell committed
532
	struct sk_buff *skb;
533 534
	struct skb_vnet_hdr *hdr;
	int err;
535

536
	skb = __netdev_alloc_skb_ip_align(vi->dev, GOOD_PACKET_LEN, gfp);
537 538
	if (unlikely(!skb))
		return -ENOMEM;
Rusty Russell's avatar
Rusty Russell committed
539

540
	skb_put(skb, GOOD_PACKET_LEN);
541

542
	hdr = skb_vnet_hdr(skb);
543
	sg_set_buf(rq->sg, &hdr->hdr, sizeof hdr->hdr);
544

545
	skb_to_sgvec(skb, rq->sg + 1, 0, skb->len);
546

547
	err = virtqueue_add_inbuf(rq->vq, rq->sg, 2, skb, gfp);
548 549
	if (err < 0)
		dev_kfree_skb(skb);
550

551 552
	return err;
}
553

554
static int add_recvbuf_big(struct receive_queue *rq, gfp_t gfp)
555 556 557 558 559
{
	struct page *first, *list = NULL;
	char *p;
	int i, err, offset;

560
	/* page in rq->sg[MAX_SKB_FRAGS + 1] is list tail */
561
	for (i = MAX_SKB_FRAGS + 1; i > 1; --i) {
562
		first = get_a_page(rq, gfp);
563 564
		if (!first) {
			if (list)
565
				give_pages(rq, list);
566
			return -ENOMEM;
567
		}
568
		sg_set_buf(&rq->sg[i], page_address(first), PAGE_SIZE);
569

570 571 572 573
		/* chain new page in list head to match sg */
		first->private = (unsigned long)list;
		list = first;
	}
Rusty Russell's avatar
Rusty Russell committed
574

575
	first = get_a_page(rq, gfp);
576
	if (!first) {
577
		give_pages(rq, list);
578 579 580 581
		return -ENOMEM;
	}
	p = page_address(first);

582 583 584
	/* rq->sg[0], rq->sg[1] share the same page */
	/* a separated rq->sg[0] for virtio_net_hdr only due to QEMU bug */
	sg_set_buf(&rq->sg[0], p, sizeof(struct virtio_net_hdr));
585

586
	/* rq->sg[1] for data packet, from offset */
587
	offset = sizeof(struct padded_vnet_hdr);
588
	sg_set_buf(&rq->sg[1], p + offset, PAGE_SIZE - offset);
589 590 591

	/* chain first in list head */
	first->private = (unsigned long)list;
592 593
	err = virtqueue_add_inbuf(rq->vq, rq->sg, MAX_SKB_FRAGS + 2,
				  first, gfp);
594
	if (err < 0)
595
		give_pages(rq, first);
596 597

	return err;
Rusty Russell's avatar
Rusty Russell committed
598 599
}

600
static unsigned int get_mergeable_buf_len(struct ewma *avg_pkt_len)
601
{
602
	const size_t hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
603 604 605 606 607 608 609 610 611
	unsigned int len;

	len = hdr_len + clamp_t(unsigned int, ewma_read(avg_pkt_len),
			GOOD_PACKET_LEN, PAGE_SIZE - hdr_len);
	return ALIGN(len, MERGEABLE_BUFFER_ALIGN);
}

static int add_recvbuf_mergeable(struct receive_queue *rq, gfp_t gfp)
{
612 613
	struct page_frag *alloc_frag = &rq->alloc_frag;
	char *buf;
614
	unsigned long ctx;
615
	int err;
616
	unsigned int len, hole;
617

618
	len = get_mergeable_buf_len(&rq->mrg_avg_pkt_len);
619
	if (unlikely(!skb_page_frag_refill(len, alloc_frag, gfp)))
620
		return -ENOMEM;
621

622
	buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
623
	ctx = mergeable_buf_to_ctx(buf, len);
624 625 626
	get_page(alloc_frag->page);
	alloc_frag->offset += len;
	hole = alloc_frag->size - alloc_frag->offset;
627 628 629 630 631 632
	if (hole < len) {
		/* To avoid internal fragmentation, if there is very likely not
		 * enough space for another buffer, add the remaining space to
		 * the current buffer. This extra space is not included in
		 * the truesize stored in ctx.
		 */
633 634 635
		len += hole;
		alloc_frag->offset += hole;
	}
636

637
	sg_init_one(rq->sg, buf, len);
638
	err = virtqueue_add_inbuf(rq->vq, rq->sg, 1, (void *)ctx, gfp);
639
	if (err < 0)
640
		put_page(virt_to_head_page(buf));
641

642 643
	return err;
}
644

645 646 647 648 649 650 651
/*
 * Returns false if we couldn't fill entirely (OOM).
 *
 * Normally run in the receive path, but can also be run from ndo_open
 * before we're receiving packets, or from refill_work which is
 * careful to disable receiving (using napi_disable).
 */
652
static bool try_fill_recv(struct receive_queue *rq, gfp_t gfp)
653
{
654
	struct virtnet_info *vi = rq->vq->vdev->priv;
655
	int err;
656
	bool oom;
657

658
	gfp |= __GFP_COLD;
659 660
	do {
		if (vi->mergeable_rx_bufs)
661
			err = add_recvbuf_mergeable(rq, gfp);
662
		else if (vi->big_packets)
663
			err = add_recvbuf_big(rq, gfp);
664
		else
665
			err = add_recvbuf_small(rq, gfp);
666

667
		oom = err == -ENOMEM;
668
		if (err)
669
			break;
670
	} while (rq->vq->num_free);
671
	virtqueue_kick(rq->vq);
672
	return !oom;
673 674
}

675
static void skb_recv_done(struct virtqueue *rvq)
Rusty Russell's avatar
Rusty Russell committed
676 677
{
	struct virtnet_info *vi = rvq->vdev->priv;
Jason Wang's avatar
Jason Wang committed
678
	struct receive_queue *rq = &vi->rq[vq2rxq(rvq)];
679

680
	/* Schedule NAPI, Suppress further interrupts if successful. */
681
	if (napi_schedule_prep(&rq->napi)) {
682
		virtqueue_disable_cb(rvq);
683
		__napi_schedule(&rq->napi);
684
	}
Rusty Russell's avatar
Rusty Russell committed
685 686
}

687
static void virtnet_napi_enable(struct receive_queue *rq)
688
{
689
	napi_enable(&rq->napi);
690 691 692 693 694

	/* If all buffers were filled by other side before we napi_enabled, we
	 * won't get another interrupt, so process any outstanding packets
	 * now.  virtnet_poll wants re-enable the queue, so we disable here.
	 * We synchronize against interrupts via NAPI_STATE_SCHED */
695 696
	if (napi_schedule_prep(&rq->napi)) {
		virtqueue_disable_cb(rq->vq);
697
		local_bh_disable();
698
		__napi_schedule(&rq->napi);
699
		local_bh_enable();
700 701 702
	}
}

703 704
static void refill_work(struct work_struct *work)
{
705 706
	struct virtnet_info *vi =
		container_of(work, struct virtnet_info, refill.work);
707
	bool still_empty;
Jason Wang's avatar
Jason Wang committed
708 709
	int i;

710
	for (i = 0; i < vi->curr_queue_pairs; i++) {
Jason Wang's avatar
Jason Wang committed
711
		struct receive_queue *rq = &vi->rq[i];
712

Jason Wang's avatar
Jason Wang committed
713 714 715
		napi_disable(&rq->napi);
		still_empty = !try_fill_recv(rq, GFP_KERNEL);
		virtnet_napi_enable(rq);
716

Jason Wang's avatar
Jason Wang committed
717 718 719 720 721 722
		/* In theory, this can happen: if we don't get any buffers in
		 * we will *never* try to fill again.
		 */
		if (still_empty)
			schedule_delayed_work(&vi->refill, HZ/2);
	}
723 724
}

725
static int virtnet_receive(struct receive_queue *rq, int budget)
Rusty Russell's avatar
Rusty Russell committed
726
{
727
	struct virtnet_info *vi = rq->vq->vdev->priv;
728
	unsigned int len, received = 0;
729
	void *buf;
Rusty Russell's avatar
Rusty Russell committed
730 731

	while (received < budget &&
732 733
	       (buf = virtqueue_get_buf(rq->vq, &len)) != NULL) {
		receive_buf(rq, buf, len);
Rusty Russell's avatar
Rusty Russell committed
734 735 736
		received++;
	}

737
	if (rq->vq->num_free > virtqueue_get_vring_size(rq->vq) / 2) {
738
		if (!try_fill_recv(rq, GFP_ATOMIC))
739
			schedule_delayed_work(&vi->refill, 0);
740
	}
Rusty Russell's avatar
Rusty Russell committed
741

742 743 744 745 746 747 748 749 750 751 752 753
	return received;
}

static int virtnet_poll(struct napi_struct *napi, int budget)
{
	struct receive_queue *rq =
		container_of(napi, struct receive_queue, napi);
	unsigned int r, received = 0;

again:
	received += virtnet_receive(rq, budget - received);

754 755
	/* Out of packets? */
	if (received < budget) {
756
		r = virtqueue_enable_cb_prepare(rq->vq);
757
		napi_complete(napi);
758
		if (unlikely(virtqueue_poll(rq->vq, r)) &&
759
		    napi_schedule_prep(napi)) {
760
			virtqueue_disable_cb(rq->vq);
761
			__napi_schedule(napi);
Rusty Russell's avatar
Rusty Russell committed
762
			goto again;
763
		}
Rusty Russell's avatar
Rusty Russell committed
764 765 766 767 768
	}

	return received;
}

769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805
#ifdef CONFIG_NET_RX_BUSY_POLL
/* must be called with local_bh_disable()d */
static int virtnet_busy_poll(struct napi_struct *napi)
{
	struct receive_queue *rq =
		container_of(napi, struct receive_queue, napi);
	struct virtnet_info *vi = rq->vq->vdev->priv;
	int r, received = 0, budget = 4;

	if (!(vi->status & VIRTIO_NET_S_LINK_UP))
		return LL_FLUSH_FAILED;

	if (!napi_schedule_prep(napi))
		return LL_FLUSH_BUSY;

	virtqueue_disable_cb(rq->vq);

again:
	received += virtnet_receive(rq, budget);

	r = virtqueue_enable_cb_prepare(rq->vq);
	clear_bit(NAPI_STATE_SCHED, &napi->state);
	if (unlikely(virtqueue_poll(rq->vq, r)) &&
	    napi_schedule_prep(napi)) {
		virtqueue_disable_cb(rq->vq);
		if (received < budget) {
			budget -= received;
			goto again;
		} else {
			__napi_schedule(napi);
		}
	}

	return received;
}
#endif	/* CONFIG_NET_RX_BUSY_POLL */

Jason Wang's avatar
Jason Wang committed
806 807 808 809 810
static int virtnet_open(struct net_device *dev)
{
	struct virtnet_info *vi = netdev_priv(dev);
	int i;

811 812 813 814 815
	for (i = 0; i < vi->max_queue_pairs; i++) {
		if (i < vi->curr_queue_pairs)
			/* Make sure we have some buffers: if oom use wq. */
			if (!try_fill_recv(&vi->rq[i], GFP_KERNEL))
				schedule_delayed_work(&vi->refill, 0);
Jason Wang's avatar
Jason Wang committed
816 817 818 819 820 821
		virtnet_napi_enable(&vi->rq[i]);
	}

	return 0;
}

822
static void free_old_xmit_skbs(struct send_queue *sq)
Rusty Russell's avatar
Rusty Russell committed
823 824
{
	struct sk_buff *skb;
825
	unsigned int len;
826
	struct virtnet_info *vi = sq->vq->vdev->priv;
827
	struct virtnet_stats *stats = this_cpu_ptr(vi->stats);
Rusty Russell's avatar
Rusty Russell committed
828

829
	while ((skb = virtqueue_get_buf(sq->vq, &len)) != NULL) {
Rusty Russell's avatar
Rusty Russell committed
830
		pr_debug("Sent skb %p\n", skb);
831

832
		u64_stats_update_begin(&stats->tx_syncp);
833 834
		stats->tx_bytes += skb->len;
		stats->tx_packets++;
835
		u64_stats_update_end(&stats->tx_syncp);
836

837
		dev_kfree_skb_any(skb);
Rusty Russell's avatar
Rusty Russell committed
838 839 840
	}
}

841
static int xmit_skb(struct send_queue *sq, struct sk_buff *skb)
Rusty Russell's avatar
Rusty Russell committed
842
{
843
	struct skb_vnet_hdr *hdr;
Rusty Russell's avatar
Rusty Russell committed
844
	const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest;
845
	struct virtnet_info *vi = sq->vq->vdev->priv;
846
	unsigned num_sg;
847 848
	unsigned hdr_len;
	bool can_push;
Rusty Russell's avatar
Rusty Russell committed
849

850
	pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest);
851 852 853 854 855 856 857 858 859 860 861 862 863 864
	if (vi->mergeable_rx_bufs)
		hdr_len = sizeof hdr->mhdr;
	else
		hdr_len = sizeof hdr->hdr;

	can_push = vi->any_header_sg &&
		!((unsigned long)skb->data & (__alignof__(*hdr) - 1)) &&
		!skb_header_cloned(skb) && skb_headroom(skb) >= hdr_len;
	/* Even if we can, don't push here yet as this would skew
	 * csum_start offset below. */
	if (can_push)
		hdr = (struct skb_vnet_hdr *)(skb->data - hdr_len);
	else
		hdr = skb_vnet_hdr(skb);
Rusty Russell's avatar
Rusty Russell committed
865 866

	if (skb->ip_summed == CHECKSUM_PARTIAL) {
867
		hdr->hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
868
		hdr->hdr.csum_start = skb_checksum_start_offset(skb);
869
		hdr->hdr.csum_offset = skb->csum_offset;
Rusty Russell's avatar
Rusty Russell committed
870
	} else {
871 872
		hdr->hdr.flags = 0;
		hdr->hdr.csum_offset = hdr->hdr.csum_start = 0;
Rusty Russell's avatar
Rusty Russell committed
873 874 875
	}

	if (skb_is_gso(skb)) {
876 877
		hdr->hdr.hdr_len = skb_headlen(skb);
		hdr->hdr.gso_size = skb_shinfo(skb)->gso_size;
878
		if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
879
			hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
Rusty Russell's avatar
Rusty Russell committed
880
		else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
881
			hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
Rusty Russell's avatar
Rusty Russell committed
882
		else if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP)
883
			hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
Rusty Russell's avatar
Rusty Russell committed
884 885
		else
			BUG();
886
		if (skb_shinfo(skb)->gso_type & SKB_GSO_TCP_ECN)
887
			hdr->hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
Rusty Russell's avatar
Rusty Russell committed
888
	} else {
889 890
		hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
		hdr->hdr.gso_size = hdr->hdr.hdr_len = 0;
Rusty Russell's avatar
Rusty Russell committed
891 892
	}

893
	if (vi->mergeable_rx_bufs)
894
		hdr->mhdr.num_buffers = 0;
895

896 897 898 899 900 901 902 903 904
	if (can_push) {
		__skb_push(skb, hdr_len);
		num_sg = skb_to_sgvec(skb, sq->sg, 0, skb->len);
		/* Pull header back to avoid skew in tx bytes calculations. */
		__skb_pull(skb, hdr_len);
	} else {
		sg_set_buf(sq->sg, hdr, hdr_len);
		num_sg = skb_to_sgvec(skb, sq->sg + 1, 0, skb->len) + 1;
	}
905
	return virtqueue_add_outbuf(sq->vq, sq->sg, num_sg, skb, GFP_ATOMIC);
906 907
}

908
static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
909 910
{
	struct virtnet_info *vi = netdev_priv(dev);
Jason Wang's avatar
Jason Wang committed
911 912
	int qnum = skb_get_queue_mapping(skb);
	struct send_queue *sq = &vi->sq[qnum];
913
	int err;
914 915

	/* Free up any pending old buffers before queueing new ones. */
916
	free_old_xmit_skbs(sq);
917

918
	/* Try to transmit */
919
	err = xmit_skb(sq, skb);
920

921
	/* This should not happen! */
922
	if (unlikely(err)) {
923 924 925
		dev->stats.tx_fifo_errors++;
		if (net_ratelimit())
			dev_warn(&dev->dev,
926
				 "Unexpected TXQ (%d) queue failure: %d\n", qnum, err);
927
		dev->stats.tx_dropped++;
928
		dev_kfree_skb_any(skb);
929
		return NETDEV_TX_OK;
Rusty Russell's avatar
Rusty Russell committed
930
	}
931
	virtqueue_kick(sq->vq);
932

933 934 935 936 937 938
	/* Don't wait up for transmitted skbs to be freed. */
	skb_orphan(skb);
	nf_reset(skb);

	/* Apparently nice girls don't return TX_BUSY; stop the queue
	 * before it gets out of hand.  Naturally, this wastes entries. */
939
	if (sq->vq->num_free < 2+MAX_SKB_FRAGS) {
Jason Wang's avatar
Jason Wang committed
940
		netif_stop_subqueue(dev, qnum);
941
		if (unlikely(!virtqueue_enable_cb_delayed(sq->vq))) {
942
			/* More just got used, free them then recheck. */
943 944
			free_old_xmit_skbs(sq);
			if (sq->vq->num_free >= 2+MAX_SKB_FRAGS) {
Jason Wang's avatar
Jason Wang committed
945
				netif_start_subqueue(dev, qnum);
946
				virtqueue_disable_cb(sq->vq);
947 948
			}
		}
949
	}
950 951

	return NETDEV_TX_OK;
Rusty Russell's avatar
Rusty Russell committed
952 953
}

954 955 956
/*
 * Send command via the control virtqueue and check status.  Commands
 * supported by the hypervisor, as indicated by feature bits, should
957
 * never fail unless improperly formatted.
958 959
 */
static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd,
960
				 struct scatterlist *out)
961
{
962
	struct scatterlist *sgs[4], hdr, stat;
963 964
	struct virtio_net_ctrl_hdr ctrl;
	virtio_net_ctrl_ack status = ~0;
965
	unsigned out_num = 0, tmp;
966 967

	/* Caller should know better */
968
	BUG_ON(!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ));
969 970 971

	ctrl.class = class;
	ctrl.cmd = cmd;
972 973 974
	/* Add header */
	sg_init_one(&hdr, &ctrl, sizeof(ctrl));
	sgs[out_num++] = &hdr;
975

976 977
	if (out)
		sgs[out_num++] = out;
978

979 980
	/* Add return status. */
	sg_init_one(&stat, &status, sizeof(status));
981
	sgs[out_num] = &stat;
982

983
	BUG_ON(out_num + 1 > ARRAY_SIZE(sgs));
984
	virtqueue_add_sgs(vi->cvq, sgs, out_num, 1, vi, GFP_ATOMIC);
985

986 987
	if (unlikely(!virtqueue_kick(vi->cvq)))
		return status == VIRTIO_NET_OK;
988 989 990 991

	/* Spin for a response, the kick causes an ioport write, trapping
	 * into the hypervisor, so the request should be handled immediately.
	 */
992 993
	while (!virtqueue_get_buf(vi->cvq, &tmp) &&
	       !virtqueue_is_broken(vi->cvq))
994 995 996 997 998
		cpu_relax();

	return status == VIRTIO_NET_OK;
}

999 1000 1001 1002
static int virtnet_set_mac_address(struct net_device *dev, void *p)
{
	struct virtnet_info *vi = netdev_priv(dev);
	struct virtio_device *vdev = vi->vdev;
1003
	int ret;
1004 1005
	struct sockaddr *addr = p;
	struct scatterlist sg;
1006

1007
	ret = eth_prepare_mac_addr_change(dev, p);
1008 1009
	if (ret)
		return ret;
1010

1011 1012 1013
	if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
		sg_init_one(&sg, addr->sa_data, dev->addr_len);
		if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
1014
					  VIRTIO_NET_CTRL_MAC_ADDR_SET, &sg)) {
1015 1016 1017 1018 1019
			dev_warn(&vdev->dev,
				 "Failed to set mac address by vq command.\n");
			return -EINVAL;
		}
	} else if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC)) {
1020 1021 1022 1023 1024 1025 1026
		unsigned int i;

		/* Naturally, this has an atomicity problem. */
		for (i = 0; i < dev->addr_len; i++)
			virtio_cwrite8(vdev,
				       offsetof(struct virtio_net_config, mac) +
				       i, addr->sa_data[i]);
1027 1028 1029
	}

	eth_commit_mac_addr_change(dev, p);
1030 1031 1032 1033

	return 0;
}

1034 1035 1036 1037 1038 1039 1040 1041
static struct rtnl_link_stats64 *virtnet_stats(struct net_device *dev,
					       struct rtnl_link_stats64 *tot)
{
	struct virtnet_info *vi = netdev_priv(dev);
	int cpu;
	unsigned int start;

	for_each_possible_cpu(cpu) {
1042
		struct virtnet_stats *stats = per_cpu_ptr(vi->stats, cpu);
1043 1044 1045
		u64 tpackets, tbytes, rpackets, rbytes;

		do {
1046
			start = u64_stats_fetch_begin_irq(&stats->tx_syncp);
1047 1048
			tpackets = stats->tx_packets;
			tbytes   = stats->tx_bytes;
1049
		} while (u64_stats_fetch_retry_irq(&stats->tx_syncp, start));
1050 1051

		do {
1052
			start = u64_stats_fetch_begin_irq(&stats->rx_syncp);
1053 1054
			rpackets = stats->rx_packets;
			rbytes   = stats->rx_bytes;
1055
		} while (u64_stats_fetch_retry_irq(&stats->rx_syncp, start));
1056 1057 1058 1059 1060 1061 1062 1063

		tot->rx_packets += rpackets;
		tot->tx_packets += tpackets;
		tot->rx_bytes   += rbytes;
		tot->tx_bytes   += tbytes;
	}

	tot->tx_dropped = dev->stats.tx_dropped;
1064
	tot->tx_fifo_errors = dev->stats.tx_fifo_errors;
1065 1066 1067 1068 1069 1070 1071
	tot->rx_dropped = dev->stats.rx_dropped;
	tot->rx_length_errors = dev->stats.rx_length_errors;
	tot->rx_frame_errors = dev->stats.rx_frame_errors;

	return tot;
}

1072 1073 1074 1075
#ifdef CONFIG_NET_POLL_CONTROLLER
static void virtnet_netpoll(struct net_device *dev)
{
	struct virtnet_info *vi = netdev_priv(dev);
Jason Wang's avatar
Jason Wang committed
1076
	int i;
1077

Jason Wang's avatar
Jason Wang committed
1078 1079
	for (i = 0; i < vi->curr_queue_pairs; i++)
		napi_schedule(&vi->rq[i].napi);
1080 1081 1082
}
#endif

1083 1084 1085 1086
static void virtnet_ack_link_announce(struct virtnet_info *vi)
{
	rtnl_lock();
	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_ANNOUNCE,
1087
				  VIRTIO_NET_CTRL_ANNOUNCE_ACK, NULL))
1088 1089 1090 1091
		dev_warn(&vi->dev->dev, "Failed to ack link announce.\n");
	rtnl_unlock();
}

Jason Wang's avatar
Jason Wang committed
1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104
static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)
{
	struct scatterlist sg;
	struct virtio_net_ctrl_mq s;
	struct net_device *dev = vi->dev;

	if (!vi->has_cvq || !virtio_has_feature(vi->vdev, VIRTIO_NET_F_MQ))
		return 0;

	s.virtqueue_pairs = queue_pairs;
	sg_init_one(&sg, &s, sizeof(s));

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ,
1105
				  VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, &sg)) {
Jason Wang's avatar
Jason Wang committed
1106 1107 1108
		dev_warn(&dev->dev, "Fail to set num of queue pairs to %d\n",
			 queue_pairs);
		return -EINVAL;
1109
	} else {
Jason Wang's avatar
Jason Wang committed
1110
		vi->curr_queue_pairs = queue_pairs;
1111 1112 1113
		/* virtnet_open() will refill when device is going to up. */
		if (dev->flags & IFF_UP)
			schedule_delayed_work(&vi->refill, 0);
1114
	}
Jason Wang's avatar
Jason Wang committed
1115 1116 1117 1118

	return 0;
}

Rusty Russell's avatar
Rusty Russell committed
1119 1120 1121
static int virtnet_close(struct net_device *dev)
{
	struct virtnet_info *vi = netdev_priv(dev);
Jason Wang's avatar
Jason Wang committed
1122
	int i;
Rusty Russell's avatar
Rusty Russell committed
1123

1124 1125
	/* Make sure refill_work doesn't re-enable napi! */
	cancel_delayed_work_sync(&vi->refill);
Jason Wang's avatar
Jason Wang committed
1126 1127 1128

	for (i = 0; i < vi->max_queue_pairs; i++)
		napi_disable(&vi->rq[i].napi);
Rusty Russell's avatar
Rusty Russell committed
1129 1130 1131 1132

	return 0;
}

1133 1134 1135
static void virtnet_set_rx_mode(struct net_device *dev)
{
	struct virtnet_info *vi = netdev_priv(dev);
1136
	struct scatterlist sg[2];
1137
	u8 promisc, allmulti;
1138
	struct virtio_net_ctrl_mac *mac_data;
Jiri Pirko's avatar
Jiri Pirko committed
1139
	struct netdev_hw_addr *ha;
1140
	int uc_count;
1141
	int mc_count;
1142 1143
	void *buf;
	int i;
1144

1145
	/* We can't dynamically set ndo_set_rx_mode, so return gracefully */
1146 1147 1148
	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX))
		return;

1149 1150
	promisc = ((dev->flags & IFF_PROMISC) != 0);
	allmulti = ((dev->flags & IFF_ALLMULTI) != 0);
1151

1152
	sg_init_one(sg, &promisc, sizeof(promisc));
1153 1154

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
1155
				  VIRTIO_NET_CTRL_RX_PROMISC, sg))
1156 1157 1158
		dev_warn(&dev->dev, "Failed to %sable promisc mode.\n",
			 promisc ? "en" : "dis");

1159
	sg_init_one(sg, &allmulti, sizeof(allmulti));
1160 1161

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
1162
				  VIRTIO_NET_CTRL_RX_ALLMULTI, sg))
1163 1164
		dev_warn(&dev->dev, "Failed to %sable allmulti mode.\n",
			 allmulti ? "en" : "dis");
1165

1166
	uc_count = netdev_uc_count(dev);
1167
	mc_count = netdev_mc_count(dev);
1168
	/* MAC filter - use one buffer for both lists */
1169 1170 1171
	buf = kzalloc(((uc_count + mc_count) * ETH_ALEN) +
		      (2 * sizeof(mac_data->entries)), GFP_ATOMIC);
	mac_data = buf;
1172
	if (!buf)
1173 1174
		return;

1175 1176
	sg_init_table(sg, 2);

1177
	/* Store the unicast list and count in the front of the buffer */
1178
	mac_data->entries = uc_count;
Jiri Pirko's avatar
Jiri Pirko committed
1179
	i = 0;
1180
	netdev_for_each_uc_addr(ha, dev)
Jiri Pirko's avatar
Jiri Pirko committed
1181
		memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
1182 1183

	sg_set_buf(&sg[0], mac_data,
1184
		   sizeof(mac_data->entries) + (uc_count * ETH_ALEN));
1185 1186

	/* multicast list and count fill the end */
1187
	mac_data = (void *)&mac_data->macs[uc_count][0];
1188

1189
	mac_data->entries = mc_count;
1190
	i = 0;
1191 1192
	netdev_for_each_mc_addr(ha, dev)
		memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
1193 1194

	sg_set_buf(&sg[1], mac_data,
1195
		   sizeof(mac_data->entries) + (mc_count * ETH_ALEN));
1196 1197

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
1198
				  VIRTIO_NET_CTRL_MAC_TABLE_SET, sg))
1199
		dev_warn(&dev->dev, "Failed to set MAC filter table.\n");
1200 1201

	kfree(buf);
1202 1203
}

1204 1205
static int virtnet_vlan_rx_add_vid(struct net_device *dev,
				   __be16 proto, u16 vid)
1206 1207 1208 1209
{
	struct virtnet_info *vi = netdev_priv(dev);
	struct scatterlist sg;

1210
	sg_init_one(&sg, &vid, sizeof(vid));
1211 1212

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
1213
				  VIRTIO_NET_CTRL_VLAN_ADD, &sg))
1214
		dev_warn(&dev->dev, "Failed to add VLAN ID %d.\n", vid);
1215
	return 0;
1216 1217
}

1218 1219
static int virtnet_vlan_rx_kill_vid(struct net_device *dev,
				    __be16 proto, u16 vid)
1220 1221 1222 1223
{
	struct virtnet_info *vi = netdev_priv(dev);
	struct scatterlist sg;

1224
	sg_init_one(&sg, &vid, sizeof(vid));
1225 1226

	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
1227
				  VIRTIO_NET_CTRL_VLAN_DEL, &sg))
1228
		dev_warn(&dev->dev, "Failed to kill VLAN ID %d.\n", vid);
1229
	return 0;
1230 1231
}

1232
static void virtnet_clean_affinity(struct virtnet_info *vi, long hcpu)
Jason Wang's avatar
Jason Wang committed
1233 1234 1235
{
	int i;

1236 1237
	if (vi->affinity_hint_set) {
		for (i = 0; i < vi->max_queue_pairs; i++) {
1238 1239 1240 1241
			virtqueue_set_affinity(vi->rq[i].vq, -1);
			virtqueue_set_affinity(vi->sq[i].vq, -1);
		}

1242 1243 1244
		vi->affinity_hint_set = false;
	}
}
1245

1246 1247 1248 1249
static void virtnet_set_affinity(struct virtnet_info *vi)
{
	int i;
	int cpu;
Jason Wang's avatar
Jason Wang committed
1250 1251 1252 1253 1254

	/* In multiqueue mode, when the number of cpu is equal to the number of
	 * queue pairs, we let the queue pairs to be private to one cpu by
	 * setting the affinity hint to eliminate the contention.
	 */
1255 1256 1257 1258
	if (vi->curr_queue_pairs == 1 ||
	    vi->max_queue_pairs != num_online_cpus()) {
		virtnet_clean_affinity(vi, -1);
		return;
Jason Wang's avatar
Jason Wang committed
1259 1260
	}

1261 1262
	i = 0;
	for_each_online_cpu(cpu) {
Jason Wang's avatar
Jason Wang committed
1263 1264
		virtqueue_set_affinity(vi->rq[i].vq, cpu);
		virtqueue_set_affinity(vi->sq[i].vq, cpu);
1265
		netif_set_xps_queue(vi->dev, cpumask_of(cpu), i);
1266
		i++;
Jason Wang's avatar
Jason Wang committed
1267 1268
	}

1269
	vi->affinity_hint_set = true;
Jason Wang's avatar
Jason Wang committed
1270 1271
}

1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288
static int virtnet_cpu_callback(struct notifier_block *nfb,
			        unsigned long action, void *hcpu)
{
	struct virtnet_info *vi = container_of(nfb, struct virtnet_info, nb);

	switch(action & ~CPU_TASKS_FROZEN) {
	case CPU_ONLINE:
	case CPU_DOWN_FAILED:
	case CPU_DEAD:
		virtnet_set_affinity(vi);
		break;
	case CPU_DOWN_PREPARE:
		virtnet_clean_affinity(vi, (long)hcpu);
		break;
	default:
		break;
	}
1289

1290
	return NOTIFY_OK;
Jason Wang's avatar
Jason Wang committed
1291 1292
}

1293 1294 1295 1296 1297
static void virtnet_get_ringparam(struct net_device *dev,
				struct ethtool_ringparam *ring)
{
	struct virtnet_info *vi = netdev_priv(dev);

Jason Wang's avatar
Jason Wang committed
1298 1299
	ring->rx_max_pending = virtqueue_get_vring_size(vi->rq[0].vq);
	ring->tx_max_pending = virtqueue_get_vring_size(vi->sq[0].vq);
1300 1301 1302 1303
	ring->rx_pending = ring->rx_max_pending;
	ring->tx_pending = ring->tx_max_pending;
}

1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316

static void virtnet_get_drvinfo(struct net_device *dev,
				struct ethtool_drvinfo *info)
{
	struct virtnet_info *vi = netdev_priv(dev);
	struct virtio_device *vdev = vi->vdev;

	strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
	strlcpy(info->version, VIRTNET_DRIVER_VERSION, sizeof(info->version));
	strlcpy(info->bus_info, virtio_bus_name(vdev), sizeof(info->bus_info));

}

1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330
/* TODO: Eliminate OOO packets during switching */
static int virtnet_set_channels(struct net_device *dev,
				struct ethtool_channels *channels)
{
	struct virtnet_info *vi = netdev_priv(dev);
	u16 queue_pairs = channels->combined_count;
	int err;

	/* We don't support separate rx/tx channels.
	 * We don't allow setting 'other' channels.
	 */
	if (channels->rx_count || channels->tx_count || channels->other_count)
		return -EINVAL;

1331
	if (queue_pairs > vi->max_queue_pairs || queue_pairs == 0)
1332 1333
		return -EINVAL;

1334
	get_online_cpus();
1335 1336 1337 1338 1339
	err = virtnet_set_queues(vi, queue_pairs);
	if (!err) {
		netif_set_real_num_tx_queues(dev, queue_pairs);
		netif_set_real_num_rx_queues(dev, queue_pairs);

1340
		virtnet_set_affinity(vi);
1341
	}
1342
	put_online_cpus();
1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359

	return err;
}

static void virtnet_get_channels(struct net_device *dev,
				 struct ethtool_channels *channels)
{
	struct virtnet_info *vi = netdev_priv(dev);

	channels->combined_count = vi->curr_queue_pairs;
	channels->max_combined = vi->max_queue_pairs;
	channels->max_other = 0;
	channels->rx_count = 0;
	channels->tx_count = 0;
	channels->other_count = 0;
}

1360
static const struct ethtool_ops virtnet_ethtool_ops = {
1361
	.get_drvinfo = virtnet_get_drvinfo,
1362
	.get_link = ethtool_op_get_link,
1363
	.get_ringparam = virtnet_get_ringparam,
1364 1365
	.set_channels = virtnet_set_channels,
	.get_channels = virtnet_get_channels,
1366 1367
};

1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378
#define MIN_MTU 68
#define MAX_MTU 65535

static int virtnet_change_mtu(struct net_device *dev, int new_mtu)
{
	if (new_mtu < MIN_MTU || new_mtu > MAX_MTU)
		return -EINVAL;
	dev->mtu = new_mtu;
	return 0;
}

1379 1380 1381 1382 1383
static const struct net_device_ops virtnet_netdev = {
	.ndo_open            = virtnet_open,
	.ndo_stop   	     = virtnet_close,
	.ndo_start_xmit      = start_xmit,
	.ndo_validate_addr   = eth_validate_addr,
1384
	.ndo_set_mac_address = virtnet_set_mac_address,
1385
	.ndo_set_rx_mode     = virtnet_set_rx_mode,
1386
	.ndo_change_mtu	     = virtnet_change_mtu,
1387
	.ndo_get_stats64     = virtnet_stats,
1388 1389
	.ndo_vlan_rx_add_vid = virtnet_vlan_rx_add_vid,
	.ndo_vlan_rx_kill_vid = virtnet_vlan_rx_kill_vid,
1390 1391 1392
#ifdef CONFIG_NET_POLL_CONTROLLER
	.ndo_poll_controller = virtnet_netpoll,
#endif
1393 1394 1395
#ifdef CONFIG_NET_RX_BUSY_POLL
	.ndo_busy_poll		= virtnet_busy_poll,
#endif
1396 1397
};

1398
static void virtnet_config_changed_work(struct work_struct *work)
1399
{
1400 1401
	struct virtnet_info *vi =
		container_of(work, struct virtnet_info, config_work);
1402 1403
	u16 v;

1404 1405
	if (virtio_cread_feature(vi->vdev, VIRTIO_NET_F_STATUS,
				 struct virtio_net_config, status, &v) < 0)
1406
		return;
1407 1408

	if (v & VIRTIO_NET_S_ANNOUNCE) {
1409
		netdev_notify_peers(vi->dev);
1410 1411
		virtnet_ack_link_announce(vi);
	}
1412 1413 1414 1415 1416

	/* Ignore unknown (future) status bits */
	v &= VIRTIO_NET_S_LINK_UP;

	if (vi->status == v)
1417
		return;
1418 1419 1420 1421 1422

	vi->status = v;

	if (vi->status & VIRTIO_NET_S_LINK_UP) {
		netif_carrier_on(vi->dev);
Jason Wang's avatar
Jason Wang committed
1423
		netif_tx_wake_all_queues(vi->dev);
1424 1425
	} else {
		netif_carrier_off(vi->dev);
Jason Wang's avatar
Jason Wang committed
1426
		netif_tx_stop_all_queues(vi->dev);
1427 1428 1429 1430 1431 1432 1433
	}
}

static void virtnet_config_changed(struct virtio_device *vdev)
{
	struct virtnet_info *vi = vdev->priv;

1434
	schedule_work(&vi->config_work);
1435 1436
}

Jason Wang's avatar
Jason Wang committed
1437 1438
static void virtnet_free_queues(struct virtnet_info *vi)
{
1439 1440 1441 1442 1443
	int i;

	for (i = 0; i < vi->max_queue_pairs; i++)
		netif_napi_del(&vi->rq[i].napi);

Jason Wang's avatar
Jason Wang committed
1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457
	kfree(vi->rq);
	kfree(vi->sq);
}

static void free_receive_bufs(struct virtnet_info *vi)
{
	int i;

	for (i = 0; i < vi->max_queue_pairs; i++) {
		while (vi->rq[i].pages)
			__free_pages(get_a_page(&vi->rq[i], GFP_KERNEL), 0);
	}
}

1458 1459 1460 1461 1462 1463 1464 1465
static void free_receive_page_frags(struct virtnet_info *vi)
{
	int i;
	for (i = 0; i < vi->max_queue_pairs; i++)
		if (vi->rq[i].alloc_frag.page)
			put_page(vi->rq[i].alloc_frag.page);
}

Jason Wang's avatar
Jason Wang committed
1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480
static void free_unused_bufs(struct virtnet_info *vi)
{
	void *buf;
	int i;

	for (i = 0; i < vi->max_queue_pairs; i++) {
		struct virtqueue *vq = vi->sq[i].vq;
		while ((buf = virtqueue_detach_unused_buf(vq)) != NULL)
			dev_kfree_skb(buf);
	}

	for (i = 0; i < vi->max_queue_pairs; i++) {
		struct virtqueue *vq = vi->rq[i].vq;

		while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) {
1481 1482 1483 1484 1485
			if (vi->mergeable_rx_bufs) {
				unsigned long ctx = (unsigned long)buf;
				void *base = mergeable_ctx_to_buf_address(ctx);
				put_page(virt_to_head_page(base));
			} else if (vi->big_packets) {
1486
				give_pages(&vi->rq[i], buf);
1487
			} else {
Jason Wang's avatar
Jason Wang committed
1488
				dev_kfree_skb(buf);
1489
			}
Jason Wang's avatar
Jason Wang committed
1490 1491 1492 1493
		}
	}
}

1494 1495 1496 1497
static void virtnet_del_vqs(struct virtnet_info *vi)
{
	struct virtio_device *vdev = vi->vdev;

1498
	virtnet_clean_affinity(vi, -1);
Jason Wang's avatar
Jason Wang committed
1499

1500
	vdev->config->del_vqs(vdev);
Jason Wang's avatar
Jason Wang committed
1501 1502

	virtnet_free_queues(vi);
1503 1504
}

Jason Wang's avatar
Jason Wang committed
1505
static int virtnet_find_vqs(struct virtnet_info *vi)
1506
{
Jason Wang's avatar
Jason Wang committed
1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535
	vq_callback_t **callbacks;
	struct virtqueue **vqs;
	int ret = -ENOMEM;
	int i, total_vqs;
	const char **names;

	/* We expect 1 RX virtqueue followed by 1 TX virtqueue, followed by
	 * possible N-1 RX/TX queue pairs used in multiqueue mode, followed by
	 * possible control vq.
	 */
	total_vqs = vi->max_queue_pairs * 2 +
		    virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ);

	/* Allocate space for find_vqs parameters */
	vqs = kzalloc(total_vqs * sizeof(*vqs), GFP_KERNEL);
	if (!vqs)
		goto err_vq;
	callbacks = kmalloc(total_vqs * sizeof(*callbacks), GFP_KERNEL);
	if (!callbacks)
		goto err_callback;
	names = kmalloc(total_vqs * sizeof(*names), GFP_KERNEL);
	if (!names)
		goto err_names;

	/* Parameters for control virtqueue, if any */
	if (vi->has_cvq) {
		callbacks[total_vqs - 1] = NULL;
		names[total_vqs - 1] = "control";
	}
1536

Jason Wang's avatar
Jason Wang committed
1537 1538 1539 1540 1541 1542 1543 1544 1545
	/* Allocate/initialize parameters for send/receive virtqueues */
	for (i = 0; i < vi->max_queue_pairs; i++) {
		callbacks[rxq2vq(i)] = skb_recv_done;
		callbacks[txq2vq(i)] = skb_xmit_done;
		sprintf(vi->rq[i].name, "input.%d", i);
		sprintf(vi->sq[i].name, "output.%d", i);
		names[rxq2vq(i)] = vi->rq[i].name;
		names[txq2vq(i)] = vi->sq[i].name;
	}
1546

Jason Wang's avatar
Jason Wang committed
1547 1548 1549 1550
	ret = vi->vdev->config->find_vqs(vi->vdev, total_vqs, vqs, callbacks,
					 names);
	if (ret)
		goto err_find;
1551

Jason Wang's avatar
Jason Wang committed
1552 1553
	if (vi->has_cvq) {
		vi->cvq = vqs[total_vqs - 1];
1554
		if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN))
1555
			vi->dev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
1556
	}
Jason Wang's avatar
Jason Wang committed
1557 1558 1559 1560 1561 1562 1563 1564 1565 1566

	for (i = 0; i < vi->max_queue_pairs; i++) {
		vi->rq[i].vq = vqs[rxq2vq(i)];
		vi->sq[i].vq = vqs[txq2vq(i)];
	}

	kfree(names);
	kfree(callbacks);
	kfree(vqs);

1567
	return 0;
Jason Wang's avatar
Jason Wang committed
1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586

err_find:
	kfree(names);
err_names:
	kfree(callbacks);
err_callback:
	kfree(vqs);
err_vq:
	return ret;
}

static int virtnet_alloc_queues(struct virtnet_info *vi)
{
	int i;

	vi->sq = kzalloc(sizeof(*vi->sq) * vi->max_queue_pairs, GFP_KERNEL);
	if (!vi->sq)
		goto err_sq;
	vi->rq = kzalloc(sizeof(*vi->rq) * vi->max_queue_pairs, GFP_KERNEL);
1587
	if (!vi->rq)
Jason Wang's avatar
Jason Wang committed
1588 1589 1590 1591 1592 1593 1594
		goto err_rq;

	INIT_DELAYED_WORK(&vi->refill, refill_work);
	for (i = 0; i < vi->max_queue_pairs; i++) {
		vi->rq[i].pages = NULL;
		netif_napi_add(vi->dev, &vi->rq[i].napi, virtnet_poll,
			       napi_weight);
1595
		napi_hash_add(&vi->rq[i].napi);
Jason Wang's avatar
Jason Wang committed
1596 1597

		sg_init_table(vi->rq[i].sg, ARRAY_SIZE(vi->rq[i].sg));
1598
		ewma_init(&vi->rq[i].mrg_avg_pkt_len, 1, RECEIVE_AVG_WEIGHT);
Jason Wang's avatar
Jason Wang committed
1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622
		sg_init_table(vi->sq[i].sg, ARRAY_SIZE(vi->sq[i].sg));
	}

	return 0;

err_rq:
	kfree(vi->sq);
err_sq:
	return -ENOMEM;
}

static int init_vqs(struct virtnet_info *vi)
{
	int ret;

	/* Allocate send & receive queues */
	ret = virtnet_alloc_queues(vi);
	if (ret)
		goto err;

	ret = virtnet_find_vqs(vi);
	if (ret)
		goto err_free;

1623
	get_online_cpus();
1624
	virtnet_set_affinity(vi);
1625 1626
	put_online_cpus();

Jason Wang's avatar
Jason Wang committed
1627 1628 1629 1630 1631 1632
	return 0;

err_free:
	virtnet_free_queues(vi);
err:
	return ret;
1633 1634
}

1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661
#ifdef CONFIG_SYSFS
static ssize_t mergeable_rx_buffer_size_show(struct netdev_rx_queue *queue,
		struct rx_queue_attribute *attribute, char *buf)
{
	struct virtnet_info *vi = netdev_priv(queue->dev);
	unsigned int queue_index = get_netdev_rx_queue_index(queue);
	struct ewma *avg;

	BUG_ON(queue_index >= vi->max_queue_pairs);
	avg = &vi->rq[queue_index].mrg_avg_pkt_len;
	return sprintf(buf, "%u\n", get_mergeable_buf_len(avg));
}

static struct rx_queue_attribute mergeable_rx_buffer_size_attribute =
	__ATTR_RO(mergeable_rx_buffer_size);

static struct attribute *virtio_net_mrg_rx_attrs[] = {
	&mergeable_rx_buffer_size_attribute.attr,
	NULL
};

static const struct attribute_group virtio_net_mrg_rx_group = {
	.name = "virtio_net",
	.attrs = virtio_net_mrg_rx_attrs
};
#endif

Rusty Russell's avatar
Rusty Russell committed
1662 1663
static int virtnet_probe(struct virtio_device *vdev)
{
Jason Wang's avatar
Jason Wang committed
1664
	int i, err;
Rusty Russell's avatar
Rusty Russell committed
1665 1666
	struct net_device *dev;
	struct virtnet_info *vi;
Jason Wang's avatar
Jason Wang committed
1667 1668 1669
	u16 max_queue_pairs;

	/* Find if host supports multiqueue virtio_net device */
1670 1671 1672
	err = virtio_cread_feature(vdev, VIRTIO_NET_F_MQ,
				   struct virtio_net_config,
				   max_virtqueue_pairs, &max_queue_pairs);
Jason Wang's avatar
Jason Wang committed
1673 1674 1675 1676 1677 1678

	/* We need at least 2 queue's */
	if (err || max_queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN ||
	    max_queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX ||
	    !virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
		max_queue_pairs = 1;
Rusty Russell's avatar
Rusty Russell committed
1679 1680

	/* Allocate ourselves a network device with room for our info */
Jason Wang's avatar
Jason Wang committed
1681
	dev = alloc_etherdev_mq(sizeof(struct virtnet_info), max_queue_pairs);
Rusty Russell's avatar
Rusty Russell committed
1682 1683 1684 1685
	if (!dev)
		return -ENOMEM;

	/* Set up network device as normal. */
1686
	dev->priv_flags |= IFF_UNICAST_FLT | IFF_LIVE_ADDR_CHANGE;
1687
	dev->netdev_ops = &virtnet_netdev;
Rusty Russell's avatar
Rusty Russell committed
1688
	dev->features = NETIF_F_HIGHDMA;
1689

1690
	dev->ethtool_ops = &virtnet_ethtool_ops;
Rusty Russell's avatar
Rusty Russell committed
1691 1692 1693
	SET_NETDEV_DEV(dev, &vdev->dev);

	/* Do we support "hardware" checksums? */
1694
	if (virtio_has_feature(vdev, VIRTIO_NET_F_CSUM)) {
Rusty Russell's avatar
Rusty Russell committed
1695
		/* This opens up the world of extra features. */
1696 1697 1698 1699 1700 1701
		dev->hw_features |= NETIF_F_HW_CSUM|NETIF_F_SG|NETIF_F_FRAGLIST;
		if (csum)
			dev->features |= NETIF_F_HW_CSUM|NETIF_F_SG|NETIF_F_FRAGLIST;

		if (virtio_has_feature(vdev, VIRTIO_NET_F_GSO)) {
			dev->hw_features |= NETIF_F_TSO | NETIF_F_UFO
1702 1703
				| NETIF_F_TSO_ECN | NETIF_F_TSO6;
		}
1704
		/* Individual feature bits: what can host handle? */
1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716
		if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO4))
			dev->hw_features |= NETIF_F_TSO;
		if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO6))
			dev->hw_features |= NETIF_F_TSO6;
		if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_ECN))
			dev->hw_features |= NETIF_F_TSO_ECN;
		if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_UFO))
			dev->hw_features |= NETIF_F_UFO;

		if (gso)
			dev->features |= dev->hw_features & (NETIF_F_ALL_TSO|NETIF_F_UFO);
		/* (!csum && gso) case will be fixed by register_netdev() */
Rusty Russell's avatar
Rusty Russell committed
1717
	}
1718 1719
	if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_CSUM))
		dev->features |= NETIF_F_RXCSUM;
Rusty Russell's avatar
Rusty Russell committed
1720

1721 1722
	dev->vlan_features = dev->features;

Rusty Russell's avatar
Rusty Russell committed
1723
	/* Configuration may specify what MAC to use.  Otherwise random. */
1724 1725 1726 1727 1728
	if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC))
		virtio_cread_bytes(vdev,
				   offsetof(struct virtio_net_config, mac),
				   dev->dev_addr, dev->addr_len);
	else
1729
		eth_hw_addr_random(dev);
Rusty Russell's avatar
Rusty Russell committed
1730 1731 1732 1733 1734

	/* Set up our device-specific information */
	vi = netdev_priv(dev);
	vi->dev = dev;
	vi->vdev = vdev;
1735
	vdev->priv = vi;
1736 1737 1738 1739 1740
	vi->stats = alloc_percpu(struct virtnet_stats);
	err = -ENOMEM;
	if (vi->stats == NULL)
		goto free;

1741 1742 1743 1744 1745 1746 1747
	for_each_possible_cpu(i) {
		struct virtnet_stats *virtnet_stats;
		virtnet_stats = per_cpu_ptr(vi->stats, i);
		u64_stats_init(&virtnet_stats->tx_syncp);
		u64_stats_init(&virtnet_stats->rx_syncp);
	}

1748
	INIT_WORK(&vi->config_work, virtnet_config_changed_work);
Rusty Russell's avatar
Rusty Russell committed
1749

1750
	/* If we can receive ANY GSO packets, we must allocate large ones. */
1751 1752
	if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) ||
	    virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6) ||
1753 1754
	    virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_ECN) ||
	    virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_UFO))
1755 1756
		vi->big_packets = true;

1757 1758 1759
	if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF))
		vi->mergeable_rx_bufs = true;

1760 1761 1762
	if (virtio_has_feature(vdev, VIRTIO_F_ANY_LAYOUT))
		vi->any_header_sg = true;

Jason Wang's avatar
Jason Wang committed
1763 1764 1765
	if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
		vi->has_cvq = true;

1766 1767 1768 1769 1770 1771 1772
	if (vi->any_header_sg) {
		if (vi->mergeable_rx_bufs)
			dev->needed_headroom = sizeof(struct virtio_net_hdr_mrg_rxbuf);
		else
			dev->needed_headroom = sizeof(struct virtio_net_hdr);
	}

Jason Wang's avatar
Jason Wang committed
1773 1774 1775 1776 1777
	/* Use single tx/rx queue pair as default */
	vi->curr_queue_pairs = 1;
	vi->max_queue_pairs = max_queue_pairs;

	/* Allocate/initialize the rx/tx queues, and invoke find_vqs */
1778
	err = init_vqs(vi);
1779
	if (err)
1780
		goto free_stats;
Rusty Russell's avatar
Rusty Russell committed
1781

1782 1783 1784 1785
#ifdef CONFIG_SYSFS
	if (vi->mergeable_rx_bufs)
		dev->sysfs_rx_queue_group = &virtio_net_mrg_rx_group;
#endif
1786 1787
	netif_set_real_num_tx_queues(dev, vi->curr_queue_pairs);
	netif_set_real_num_rx_queues(dev, vi->curr_queue_pairs);
Jason Wang's avatar
Jason Wang committed
1788

Rusty Russell's avatar
Rusty Russell committed
1789 1790 1791
	err = register_netdev(dev);
	if (err) {
		pr_debug("virtio_net: registering device failed\n");
1792
		goto free_vqs;
Rusty Russell's avatar
Rusty Russell committed
1793
	}
1794

1795 1796
	virtio_device_ready(vdev);

1797
	/* Last of all, set up some receive buffers. */
1798
	for (i = 0; i < vi->curr_queue_pairs; i++) {
Jason Wang's avatar
Jason Wang committed
1799 1800 1801
		try_fill_recv(&vi->rq[i], GFP_KERNEL);

		/* If we didn't even get one input buffer, we're useless. */
1802 1803
		if (vi->rq[i].vq->num_free ==
		    virtqueue_get_vring_size(vi->rq[i].vq)) {
Jason Wang's avatar
Jason Wang committed
1804 1805 1806 1807
			free_unused_bufs(vi);
			err = -ENOMEM;
			goto free_recv_bufs;
		}
1808 1809
	}

1810 1811 1812 1813 1814 1815 1816
	vi->nb.notifier_call = &virtnet_cpu_callback;
	err = register_hotcpu_notifier(&vi->nb);
	if (err) {
		pr_debug("virtio_net: registering cpu notifier failed\n");
		goto free_recv_bufs;
	}

1817 1818 1819 1820
	/* Assume link up if device can't report link status,
	   otherwise get link status from config. */
	if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) {
		netif_carrier_off(dev);
1821
		schedule_work(&vi->config_work);
1822 1823 1824 1825
	} else {
		vi->status = VIRTIO_NET_S_LINK_UP;
		netif_carrier_on(dev);
	}
1826

Jason Wang's avatar
Jason Wang committed
1827 1828 1829
	pr_debug("virtnet: registered device %s with %d RX and TX vq's\n",
		 dev->name, max_queue_pairs);

Rusty Russell's avatar
Rusty Russell committed
1830 1831
	return 0;

Jason Wang's avatar
Jason Wang committed
1832
free_recv_bufs:
1833 1834
	vi->vdev->config->reset(vdev);

Jason Wang's avatar
Jason Wang committed
1835
	free_receive_bufs(vi);
1836
	unregister_netdev(dev);
1837
free_vqs:
Jason Wang's avatar
Jason Wang committed
1838
	cancel_delayed_work_sync(&vi->refill);
1839
	free_receive_page_frags(vi);
1840
	virtnet_del_vqs(vi);
1841 1842
free_stats:
	free_percpu(vi->stats);
Rusty Russell's avatar
Rusty Russell committed
1843 1844 1845 1846 1847
free:
	free_netdev(dev);
	return err;
}

1848
static void remove_vq_common(struct virtnet_info *vi)
Rusty Russell's avatar
Rusty Russell committed
1849
{
1850
	vi->vdev->config->reset(vi->vdev);
Shirley Ma's avatar
Shirley Ma committed
1851 1852

	/* Free unused buffers in both send and recv, if any. */
1853
	free_unused_bufs(vi);
1854

Jason Wang's avatar
Jason Wang committed
1855
	free_receive_bufs(vi);
1856

1857 1858
	free_receive_page_frags(vi);

Jason Wang's avatar
Jason Wang committed
1859
	virtnet_del_vqs(vi);
1860 1861
}

1862
static void virtnet_remove(struct virtio_device *vdev)
1863 1864 1865
{
	struct virtnet_info *vi = vdev->priv;

1866 1867
	unregister_hotcpu_notifier(&vi->nb);

1868 1869
	/* Make sure no work handler is accessing the device. */
	flush_work(&vi->config_work);
1870

1871 1872 1873
	unregister_netdev(vi->dev);

	remove_vq_common(vi);
1874

1875
	free_percpu(vi->stats);
1876
	free_netdev(vi->dev);
Rusty Russell's avatar
Rusty Russell committed
1877 1878
}

1879
#ifdef CONFIG_PM_SLEEP
1880 1881 1882
static int virtnet_freeze(struct virtio_device *vdev)
{
	struct virtnet_info *vi = vdev->priv;
Jason Wang's avatar
Jason Wang committed
1883
	int i;
1884

1885 1886
	unregister_hotcpu_notifier(&vi->nb);

1887 1888
	/* Make sure no work handler is accessing the device */
	flush_work(&vi->config_work);
1889

1890 1891 1892
	netif_device_detach(vi->dev);
	cancel_delayed_work_sync(&vi->refill);

1893
	if (netif_running(vi->dev)) {
Jason Wang's avatar
Jason Wang committed
1894 1895
		for (i = 0; i < vi->max_queue_pairs; i++) {
			napi_disable(&vi->rq[i].napi);
1896
			napi_hash_del(&vi->rq[i].napi);
Jason Wang's avatar
Jason Wang committed
1897 1898
			netif_napi_del(&vi->rq[i].napi);
		}
1899
	}
1900 1901 1902 1903 1904 1905 1906 1907 1908

	remove_vq_common(vi);

	return 0;
}

static int virtnet_restore(struct virtio_device *vdev)
{
	struct virtnet_info *vi = vdev->priv;
Jason Wang's avatar
Jason Wang committed
1909
	int err, i;
1910 1911 1912 1913 1914

	err = init_vqs(vi);
	if (err)
		return err;

1915 1916 1917 1918 1919
	if (netif_running(vi->dev)) {
		for (i = 0; i < vi->curr_queue_pairs; i++)
			if (!try_fill_recv(&vi->rq[i], GFP_KERNEL))
				schedule_delayed_work(&vi->refill, 0);

Jason Wang's avatar
Jason Wang committed
1920 1921
		for (i = 0; i < vi->max_queue_pairs; i++)
			virtnet_napi_enable(&vi->rq[i]);
1922
	}
1923 1924 1925

	netif_device_attach(vi->dev);

1926
	rtnl_lock();
Jason Wang's avatar
Jason Wang committed
1927
	virtnet_set_queues(vi, vi->curr_queue_pairs);
1928
	rtnl_unlock();
Jason Wang's avatar
Jason Wang committed
1929

1930 1931 1932 1933
	err = register_hotcpu_notifier(&vi->nb);
	if (err)
		return err;

1934 1935 1936 1937
	return 0;
}
#endif

Rusty Russell's avatar
Rusty Russell committed
1938 1939 1940 1941 1942
static struct virtio_device_id id_table[] = {
	{ VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID },
	{ 0 },
};

1943
static unsigned int features[] = {
1944 1945
	VIRTIO_NET_F_CSUM, VIRTIO_NET_F_GUEST_CSUM,
	VIRTIO_NET_F_GSO, VIRTIO_NET_F_MAC,
1946
	VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_UFO, VIRTIO_NET_F_HOST_TSO6,
1947
	VIRTIO_NET_F_HOST_ECN, VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6,
1948
	VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_UFO,
1949
	VIRTIO_NET_F_MRG_RXBUF, VIRTIO_NET_F_STATUS, VIRTIO_NET_F_CTRL_VQ,
1950
	VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN,
Jason Wang's avatar
Jason Wang committed
1951
	VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_MQ,
1952
	VIRTIO_NET_F_CTRL_MAC_ADDR,
1953
	VIRTIO_F_ANY_LAYOUT,
1954 1955
};

1956
static struct virtio_driver virtio_net_driver = {
1957 1958
	.feature_table = features,
	.feature_table_size = ARRAY_SIZE(features),
Rusty Russell's avatar
Rusty Russell committed
1959 1960 1961 1962
	.driver.name =	KBUILD_MODNAME,
	.driver.owner =	THIS_MODULE,
	.id_table =	id_table,
	.probe =	virtnet_probe,
1963
	.remove =	virtnet_remove,
1964
	.config_changed = virtnet_config_changed,
1965
#ifdef CONFIG_PM_SLEEP
1966 1967 1968
	.freeze =	virtnet_freeze,
	.restore =	virtnet_restore,
#endif
Rusty Russell's avatar
Rusty Russell committed
1969 1970
};

1971
module_virtio_driver(virtio_net_driver);
Rusty Russell's avatar
Rusty Russell committed
1972 1973 1974 1975

MODULE_DEVICE_TABLE(virtio, id_table);
MODULE_DESCRIPTION("Virtio network driver");
MODULE_LICENSE("GPL");