bnx2x_cmn.c 93.6 KB
Newer Older
1 2
/* bnx2x_cmn.c: Broadcom Everest network driver.
 *
3
 * Copyright (c) 2007-2012 Broadcom Corporation
4 5 6 7 8 9 10 11 12 13 14 15 16 17
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation.
 *
 * Maintained by: Eilon Greenstein <eilong@broadcom.com>
 * Written by: Eliezer Tamir
 * Based on code from Michael Chan's bnx2 driver
 * UDP CSUM errata workaround by Arik Gendelman
 * Slowpath and fastpath rework by Vladislav Zolotarov
 * Statistics and Link management by Yitchak Gertner
 *
 */

18 19
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

20
#include <linux/etherdevice.h>
21
#include <linux/if_vlan.h>
22
#include <linux/interrupt.h>
23
#include <linux/ip.h>
Dmitry Kravkov's avatar
Dmitry Kravkov committed
24
#include <net/ipv6.h>
25
#include <net/ip6_checksum.h>
26
#include <linux/firmware.h>
27
#include <linux/prefetch.h>
28
#include "bnx2x_cmn.h"
29
#include "bnx2x_init.h"
30
#include "bnx2x_sp.h"
31

32

33

34 35 36 37 38 39 40 41
/**
 * bnx2x_move_fp - move content of the fastpath structure.
 *
 * @bp:		driver handle
 * @from:	source FP index
 * @to:		destination FP index
 *
 * Makes sure the contents of the bp->fp[to].napi is kept
42 43 44
 * intact. This is done by first copying the napi struct from
 * the target to the source, and then mem copying the entire
 * source onto the target
45 46 47 48 49
 */
static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
{
	struct bnx2x_fastpath *from_fp = &bp->fp[from];
	struct bnx2x_fastpath *to_fp = &bp->fp[to];
50 51 52 53

	/* Copy the NAPI object as it has been already initialized */
	from_fp->napi = to_fp->napi;

54 55 56 57 58
	/* Move bnx2x_fastpath contents */
	memcpy(to_fp, from_fp, sizeof(*to_fp));
	to_fp->index = to;
}

59 60
int load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */

61 62 63
/* free skb in the packet ring at pos idx
 * return idx of last bd freed
 */
64
static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
65 66
			     u16 idx, unsigned int *pkts_compl,
			     unsigned int *bytes_compl)
67
{
68
	struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
69 70 71 72 73 74 75 76 77
	struct eth_tx_start_bd *tx_start_bd;
	struct eth_tx_bd *tx_data_bd;
	struct sk_buff *skb = tx_buf->skb;
	u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
	int nbd;

	/* prefetch skb end pointer to speedup dev_kfree_skb() */
	prefetch(&skb->end);

78
	DP(BNX2X_MSG_FP, "fp[%d]: pkt_idx %d  buff @(%p)->skb %p\n",
79
	   txdata->txq_index, idx, tx_buf, skb);
80 81 82

	/* unmap first bd */
	DP(BNX2X_MSG_OFF, "free bd_idx %d\n", bd_idx);
83
	tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
84
	dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
85
			 BD_UNMAP_LEN(tx_start_bd), DMA_TO_DEVICE);
86

87

88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113
	nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
#ifdef BNX2X_STOP_ON_ERROR
	if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
		BNX2X_ERR("BAD nbd!\n");
		bnx2x_panic();
	}
#endif
	new_cons = nbd + tx_buf->first_bd;

	/* Get the next bd */
	bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));

	/* Skip a parse bd... */
	--nbd;
	bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));

	/* ...and the TSO split header bd since they have no mapping */
	if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
		--nbd;
		bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
	}

	/* now free frags */
	while (nbd > 0) {

		DP(BNX2X_MSG_OFF, "free frag bd_idx %d\n", bd_idx);
114
		tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
115 116 117 118 119 120 121 122
		dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
			       BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
		if (--nbd)
			bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
	}

	/* release skb */
	WARN_ON(!skb);
123 124 125 126
	if (skb) {
		(*pkts_compl)++;
		(*bytes_compl) += skb->len;
	}
127
	dev_kfree_skb_any(skb);
128 129 130 131 132 133
	tx_buf->first_bd = 0;
	tx_buf->skb = NULL;

	return new_cons;
}

134
int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
135 136
{
	struct netdev_queue *txq;
137
	u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
138
	unsigned int pkts_compl = 0, bytes_compl = 0;
139 140 141 142 143 144

#ifdef BNX2X_STOP_ON_ERROR
	if (unlikely(bp->panic))
		return -1;
#endif

145 146 147
	txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
	hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
	sw_cons = txdata->tx_pkt_cons;
148 149 150 151 152 153

	while (sw_cons != hw_cons) {
		u16 pkt_cons;

		pkt_cons = TX_BD(sw_cons);

Dmitry Kravkov's avatar
Dmitry Kravkov committed
154 155
		DP(NETIF_MSG_TX_DONE, "queue[%d]: hw_cons %u  sw_cons %u "
				      " pkt_cons %u\n",
156
		   txdata->txq_index, hw_cons, sw_cons, pkt_cons);
157

158 159 160
		bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons,
		    &pkts_compl, &bytes_compl);

161 162 163
		sw_cons++;
	}

164 165
	netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);

166 167
	txdata->tx_pkt_cons = sw_cons;
	txdata->tx_bd_cons = bd_cons;
168 169 170 171 172 173

	/* Need to make the tx_bd_cons update visible to start_xmit()
	 * before checking for netif_tx_queue_stopped().  Without the
	 * memory barrier, there is a small possibility that
	 * start_xmit() will miss it and cause the queue to be stopped
	 * forever.
174 175 176
	 * On the other hand we need an rmb() here to ensure the proper
	 * ordering of bit testing in the following
	 * netif_tx_queue_stopped(txq) call.
177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194
	 */
	smp_mb();

	if (unlikely(netif_tx_queue_stopped(txq))) {
		/* Taking tx_lock() is needed to prevent reenabling the queue
		 * while it's empty. This could have happen if rx_action() gets
		 * suspended in bnx2x_tx_int() after the condition before
		 * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
		 *
		 * stops the queue->sees fresh tx_bd_cons->releases the queue->
		 * sends some packets consuming the whole queue again->
		 * stops the queue
		 */

		__netif_tx_lock(txq, smp_processor_id());

		if ((netif_tx_queue_stopped(txq)) &&
		    (bp->state == BNX2X_STATE_OPEN) &&
195
		    (bnx2x_tx_avail(bp, txdata) >= MAX_SKB_FRAGS + 3))
196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227
			netif_tx_wake_queue(txq);

		__netif_tx_unlock(txq);
	}
	return 0;
}

static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
					     u16 idx)
{
	u16 last_max = fp->last_max_sge;

	if (SUB_S16(idx, last_max) > 0)
		fp->last_max_sge = idx;
}

static void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
				  struct eth_fast_path_rx_cqe *fp_cqe)
{
	struct bnx2x *bp = fp->bp;
	u16 sge_len = SGE_PAGE_ALIGN(le16_to_cpu(fp_cqe->pkt_len) -
				     le16_to_cpu(fp_cqe->len_on_bd)) >>
		      SGE_PAGE_SHIFT;
	u16 last_max, last_elem, first_elem;
	u16 delta = 0;
	u16 i;

	if (!sge_len)
		return;

	/* First mark all used pages */
	for (i = 0; i < sge_len; i++)
228
		BIT_VEC64_CLEAR_BIT(fp->sge_mask,
229
			RX_SGE(le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[i])));
230 231

	DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
232
	   sge_len - 1, le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[sge_len - 1]));
233 234 235

	/* Here we assume that the last SGE index is the biggest */
	prefetch((void *)(fp->sge_mask));
236 237
	bnx2x_update_last_max_sge(fp,
		le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[sge_len - 1]));
238 239

	last_max = RX_SGE(fp->last_max_sge);
240 241
	last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
	first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
242 243 244 245 246 247 248 249 250 251

	/* If ring is not full */
	if (last_elem + 1 != first_elem)
		last_elem++;

	/* Now update the prod */
	for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
		if (likely(fp->sge_mask[i]))
			break;

252 253
		fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
		delta += BIT_VEC64_ELEM_SZ;
254 255 256 257 258 259 260 261 262 263 264 265 266
	}

	if (delta > 0) {
		fp->rx_sge_prod += delta;
		/* clear page-end entries */
		bnx2x_clear_sge_mask_next_elems(fp);
	}

	DP(NETIF_MSG_RX_STATUS,
	   "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
	   fp->last_max_sge, fp->rx_sge_prod);
}

267 268 269 270 271 272 273 274 275 276 277 278 279
/* Set Toeplitz hash value in the skb using the value from the
 * CQE (calculated by HW).
 */
static u32 bnx2x_get_rxhash(const struct bnx2x *bp,
			    const struct eth_fast_path_rx_cqe *cqe)
{
	/* Set Toeplitz hash from CQE */
	if ((bp->dev->features & NETIF_F_RXHASH) &&
	    (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG))
		return le32_to_cpu(cqe->rss_hash_result);
	return 0;
}

280
static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
281
			    u16 cons, u16 prod,
282
			    struct eth_fast_path_rx_cqe *cqe)
283 284 285 286 287 288
{
	struct bnx2x *bp = fp->bp;
	struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
	struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
	struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
	dma_addr_t mapping;
289 290
	struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
	struct sw_rx_bd *first_buf = &tpa_info->first_buf;
291

292 293
	/* print error if current state != stop */
	if (tpa_info->tpa_state != BNX2X_TPA_STOP)
294 295
		BNX2X_ERR("start of bin not in stop [%d]\n", queue);

296
	/* Try to map an empty data buffer from the aggregation info  */
297
	mapping = dma_map_single(&bp->pdev->dev,
298
				 first_buf->data + NET_SKB_PAD,
299 300 301 302 303 304 305 306 307
				 fp->rx_buf_size, DMA_FROM_DEVICE);
	/*
	 *  ...if it fails - move the skb from the consumer to the producer
	 *  and set the current aggregation state as ERROR to drop it
	 *  when TPA_STOP arrives.
	 */

	if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
		/* Move the BD from the consumer to the producer */
308
		bnx2x_reuse_rx_data(fp, cons, prod);
309 310 311
		tpa_info->tpa_state = BNX2X_TPA_ERROR;
		return;
	}
312

313 314
	/* move empty data from pool to prod */
	prod_rx_buf->data = first_buf->data;
315
	dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
316
	/* point prod_bd to new data */
317 318 319
	prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
	prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));

320 321 322 323 324 325 326 327 328 329
	/* move partial skb from cons to pool (don't unmap yet) */
	*first_buf = *cons_rx_buf;

	/* mark bin state as START */
	tpa_info->parsing_flags =
		le16_to_cpu(cqe->pars_flags.flags);
	tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
	tpa_info->tpa_state = BNX2X_TPA_START;
	tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
	tpa_info->placement_offset = cqe->placement_offset;
330
	tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe);
331

332 333 334 335 336 337 338 339 340 341 342
#ifdef BNX2X_STOP_ON_ERROR
	fp->tpa_queue_used |= (1 << queue);
#ifdef _ASM_GENERIC_INT_L64_H
	DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
#else
	DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
#endif
	   fp->tpa_queue_used);
#endif
}

343 344 345 346 347 348
/* Timestamp option length allowed for TPA aggregation:
 *
 *		nop nop kind length echo val
 */
#define TPA_TSTAMP_OPT_LEN	12
/**
349
 * bnx2x_set_lro_mss - calculate the approximate value of the MSS
350
 *
351 352 353 354 355 356 357
 * @bp:			driver handle
 * @parsing_flags:	parsing flags from the START CQE
 * @len_on_bd:		total length of the first packet for the
 *			aggregation.
 *
 * Approximate value of the MSS for this aggregation calculated using
 * the first packet of it.
358 359 360 361
 */
static inline u16 bnx2x_set_lro_mss(struct bnx2x *bp, u16 parsing_flags,
				    u16 len_on_bd)
{
362 363 364
	/*
	 * TPA arrgregation won't have either IP options or TCP options
	 * other than timestamp or IPv6 extension headers.
365
	 */
366 367 368 369 370 371 372
	u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);

	if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
	    PRS_FLAG_OVERETH_IPV6)
		hdrs_len += sizeof(struct ipv6hdr);
	else /* IPv4 */
		hdrs_len += sizeof(struct iphdr);
373 374 375 376 377 378 379 380 381 382 383 384 385


	/* Check if there was a TCP timestamp, if there is it's will
	 * always be 12 bytes length: nop nop kind length echo val.
	 *
	 * Otherwise FW would close the aggregation.
	 */
	if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
		hdrs_len += TPA_TSTAMP_OPT_LEN;

	return len_on_bd - hdrs_len;
}

386
static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
387 388 389
			       u16 queue, struct sk_buff *skb,
			       struct eth_end_agg_rx_cqe *cqe,
			       u16 cqe_idx)
390 391 392 393 394
{
	struct sw_rx_page *rx_pg, old_rx_pg;
	u32 i, frag_len, frag_size, pages;
	int err;
	int j;
395 396
	struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
	u16 len_on_bd = tpa_info->len_on_bd;
397

398
	frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
399 400 401 402
	pages = SGE_PAGE_ALIGN(frag_size) >> SGE_PAGE_SHIFT;

	/* This is needed in order to enable forwarding support */
	if (frag_size)
403 404
		skb_shinfo(skb)->gso_size = bnx2x_set_lro_mss(bp,
					tpa_info->parsing_flags, len_on_bd);
405 406 407 408 409

#ifdef BNX2X_STOP_ON_ERROR
	if (pages > min_t(u32, 8, MAX_SKB_FRAGS)*SGE_PAGE_SIZE*PAGES_PER_SGE) {
		BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
			  pages, cqe_idx);
410
		BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
411 412 413 414 415 416 417
		bnx2x_panic();
		return -EINVAL;
	}
#endif

	/* Run through the SGL and compose the fragmented skb */
	for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
418
		u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442

		/* FW gives the indices of the SGE as if the ring is an array
		   (meaning that "next" element will consume 2 indices) */
		frag_len = min(frag_size, (u32)(SGE_PAGE_SIZE*PAGES_PER_SGE));
		rx_pg = &fp->rx_page_ring[sge_idx];
		old_rx_pg = *rx_pg;

		/* If we fail to allocate a substitute page, we simply stop
		   where we are and drop the whole packet */
		err = bnx2x_alloc_rx_sge(bp, fp, sge_idx);
		if (unlikely(err)) {
			fp->eth_q_stats.rx_skb_alloc_failed++;
			return err;
		}

		/* Unmap the page as we r going to pass it to the stack */
		dma_unmap_page(&bp->pdev->dev,
			       dma_unmap_addr(&old_rx_pg, mapping),
			       SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE);

		/* Add one frag and update the appropriate fields in the skb */
		skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);

		skb->data_len += frag_len;
443
		skb->truesize += SGE_PAGE_SIZE * PAGES_PER_SGE;
444 445 446 447 448 449 450 451 452
		skb->len += frag_len;

		frag_size -= frag_len;
	}

	return 0;
}

static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
453
			   u16 queue, struct eth_end_agg_rx_cqe *cqe,
454 455
			   u16 cqe_idx)
{
456 457
	struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
	struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
458
	u32 pad = tpa_info->placement_offset;
459
	u16 len = tpa_info->len_on_bd;
460 461
	struct sk_buff *skb = NULL;
	u8 *data = rx_buf->data;
462
	/* alloc new skb */
463
	u8 *new_data;
464 465 466 467 468 469 470 471 472 473
	u8 old_tpa_state = tpa_info->tpa_state;

	tpa_info->tpa_state = BNX2X_TPA_STOP;

	/* If we there was an error during the handling of the TPA_START -
	 * drop this aggregation.
	 */
	if (old_tpa_state == BNX2X_TPA_ERROR)
		goto drop;

474 475
	/* Try to allocate the new data */
	new_data = kmalloc(fp->rx_buf_size + NET_SKB_PAD, GFP_ATOMIC);
476 477 478 479 480

	/* Unmap skb in the pool anyway, as we are going to change
	   pool entry status to BNX2X_TPA_STOP even if new skb allocation
	   fails. */
	dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
481
			 fp->rx_buf_size, DMA_FROM_DEVICE);
482 483
	if (likely(new_data))
		skb = build_skb(data);
484

485
	if (likely(skb)) {
486
#ifdef BNX2X_STOP_ON_ERROR
487
		if (pad + len > fp->rx_buf_size) {
488 489
			BNX2X_ERR("skb_put is about to fail...  "
				  "pad %d  len %d  rx_buf_size %d\n",
490
				  pad, len, fp->rx_buf_size);
491 492 493 494 495
			bnx2x_panic();
			return;
		}
#endif

496
		skb_reserve(skb, pad + NET_SKB_PAD);
497
		skb_put(skb, len);
498
		skb->rxhash = tpa_info->rxhash;
499 500 501 502

		skb->protocol = eth_type_trans(skb, bp->dev);
		skb->ip_summed = CHECKSUM_UNNECESSARY;

503 504 505
		if (!bnx2x_fill_frag_skb(bp, fp, queue, skb, cqe, cqe_idx)) {
			if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
				__vlan_hwaccel_put_tag(skb, tpa_info->vlan_tag);
506
			napi_gro_receive(&fp->napi, skb);
507 508 509
		} else {
			DP(NETIF_MSG_RX_STATUS, "Failed to allocate new pages"
			   " - dropping packet!\n");
510
			dev_kfree_skb_any(skb);
511 512 513
		}


514 515
		/* put new data in bin */
		rx_buf->data = new_data;
516

517
		return;
518
	}
519
	kfree(new_data);
520 521 522 523 524
drop:
	/* drop the packet and keep the buffer in the bin */
	DP(NETIF_MSG_RX_STATUS,
	   "Failed to allocate or map a new skb - dropping packet!\n");
	fp->eth_q_stats.rx_skb_alloc_failed++;
525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564
}


int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
{
	struct bnx2x *bp = fp->bp;
	u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
	u16 hw_comp_cons, sw_comp_cons, sw_comp_prod;
	int rx_pkt = 0;

#ifdef BNX2X_STOP_ON_ERROR
	if (unlikely(bp->panic))
		return 0;
#endif

	/* CQ "next element" is of the size of the regular element,
	   that's why it's ok here */
	hw_comp_cons = le16_to_cpu(*fp->rx_cons_sb);
	if ((hw_comp_cons & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT)
		hw_comp_cons++;

	bd_cons = fp->rx_bd_cons;
	bd_prod = fp->rx_bd_prod;
	bd_prod_fw = bd_prod;
	sw_comp_cons = fp->rx_comp_cons;
	sw_comp_prod = fp->rx_comp_prod;

	/* Memory barrier necessary as speculative reads of the rx
	 * buffer can be ahead of the index in the status block
	 */
	rmb();

	DP(NETIF_MSG_RX_STATUS,
	   "queue[%d]:  hw_comp_cons %u  sw_comp_cons %u\n",
	   fp->index, hw_comp_cons, sw_comp_cons);

	while (sw_comp_cons != hw_comp_cons) {
		struct sw_rx_bd *rx_buf = NULL;
		struct sk_buff *skb;
		union eth_rx_cqe *cqe;
565
		struct eth_fast_path_rx_cqe *cqe_fp;
566
		u8 cqe_fp_flags;
567
		enum eth_rx_cqe_type cqe_fp_type;
568
		u16 len, pad;
569
		u8 *data;
570

571 572 573 574 575
#ifdef BNX2X_STOP_ON_ERROR
		if (unlikely(bp->panic))
			return 0;
#endif

576 577 578 579 580
		comp_ring_cons = RCQ_BD(sw_comp_cons);
		bd_prod = RX_BD(bd_prod);
		bd_cons = RX_BD(bd_cons);

		cqe = &fp->rx_comp_ring[comp_ring_cons];
581 582 583
		cqe_fp = &cqe->fast_path_cqe;
		cqe_fp_flags = cqe_fp->type_error_flags;
		cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
584 585 586

		DP(NETIF_MSG_RX_STATUS, "CQE type %x  err %x  status %x"
		   "  queue %x  vlan %x  len %u\n", CQE_TYPE(cqe_fp_flags),
587 588 589
		   cqe_fp_flags, cqe_fp->status_flags,
		   le32_to_cpu(cqe_fp->rss_hash_result),
		   le16_to_cpu(cqe_fp->vlan_tag), le16_to_cpu(cqe_fp->pkt_len));
590 591

		/* is this a slowpath msg? */
592
		if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
593 594
			bnx2x_sp_event(fp, cqe);
			goto next_cqe;
595 596 597
		}
		rx_buf = &fp->rx_buf_ring[bd_cons];
		data = rx_buf->data;
598

599
		if (!CQE_TYPE_FAST(cqe_fp_type)) {
600
#ifdef BNX2X_STOP_ON_ERROR
601 602 603 604 605 606 607
			/* sanity check */
			if (fp->disable_tpa &&
			    (CQE_TYPE_START(cqe_fp_type) ||
			     CQE_TYPE_STOP(cqe_fp_type)))
				BNX2X_ERR("START/STOP packet while "
					  "disable_tpa type %x\n",
					  CQE_TYPE(cqe_fp_type));
608
#endif
609

610 611 612 613 614
			if (CQE_TYPE_START(cqe_fp_type)) {
				u16 queue = cqe_fp->queue_index;
				DP(NETIF_MSG_RX_STATUS,
				   "calling tpa_start on queue %d\n",
				   queue);
615

616 617 618 619 620 621 622 623 624 625 626 627 628 629
				bnx2x_tpa_start(fp, queue,
						bd_cons, bd_prod,
						cqe_fp);
				goto next_rx;
			} else {
				u16 queue =
					cqe->end_agg_cqe.queue_index;
				DP(NETIF_MSG_RX_STATUS,
				   "calling tpa_stop on queue %d\n",
				   queue);

				bnx2x_tpa_stop(bp, fp, queue,
					       &cqe->end_agg_cqe,
					       comp_ring_cons);
630
#ifdef BNX2X_STOP_ON_ERROR
631 632
				if (bp->panic)
					return 0;
633 634
#endif

635 636
				bnx2x_update_sge_prod(fp, cqe_fp);
				goto next_cqe;
637
			}
638 639 640 641 642
		}
		/* non TPA */
		len = le16_to_cpu(cqe_fp->pkt_len);
		pad = cqe_fp->placement_offset;
		dma_sync_single_for_cpu(&bp->pdev->dev,
643
					dma_unmap_addr(rx_buf, mapping),
644 645 646 647 648 649 650 651 652 653 654 655
					pad + RX_COPY_THRESH,
					DMA_FROM_DEVICE);
		pad += NET_SKB_PAD;
		prefetch(data + pad); /* speedup eth_type_trans() */
		/* is this an error packet? */
		if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
			DP(NETIF_MSG_RX_ERR,
			   "ERROR  flags %x  rx packet %u\n",
			   cqe_fp_flags, sw_comp_cons);
			fp->eth_q_stats.rx_err_discard_pkt++;
			goto reuse_rx;
		}
656

657 658 659 660 661 662 663
		/* Since we don't have a jumbo ring
		 * copy small packets if mtu > 1500
		 */
		if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
		    (len <= RX_COPY_THRESH)) {
			skb = netdev_alloc_skb_ip_align(bp->dev, len);
			if (skb == NULL) {
664
				DP(NETIF_MSG_RX_ERR,
665 666
				   "ERROR  packet dropped because of alloc failure\n");
				fp->eth_q_stats.rx_skb_alloc_failed++;
667 668
				goto reuse_rx;
			}
669 670 671 672
			memcpy(skb->data, data + pad, len);
			bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
		} else {
			if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod) == 0)) {
673
				dma_unmap_single(&bp->pdev->dev,
674
						 dma_unmap_addr(rx_buf, mapping),
675
						 fp->rx_buf_size,
676
						 DMA_FROM_DEVICE);
677 678 679 680 681 682
				skb = build_skb(data);
				if (unlikely(!skb)) {
					kfree(data);
					fp->eth_q_stats.rx_skb_alloc_failed++;
					goto next_rx;
				}
683 684 685 686 687 688 689
				skb_reserve(skb, pad);
			} else {
				DP(NETIF_MSG_RX_ERR,
				   "ERROR  packet dropped because "
				   "of alloc failure\n");
				fp->eth_q_stats.rx_skb_alloc_failed++;
reuse_rx:
690
				bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
691 692
				goto next_rx;
			}
693
		}
694

695 696
		skb_put(skb, len);
		skb->protocol = eth_type_trans(skb, bp->dev);
697

698 699
		/* Set Toeplitz hash for a none-LRO skb */
		skb->rxhash = bnx2x_get_rxhash(bp, cqe_fp);
700

701
		skb_checksum_none_assert(skb);
Dmitry Kravkov's avatar
Dmitry Kravkov committed
702

703
		if (bp->dev->features & NETIF_F_RXCSUM) {
704

705 706 707 708
			if (likely(BNX2X_RX_CSUM_OK(cqe)))
				skb->ip_summed = CHECKSUM_UNNECESSARY;
			else
				fp->eth_q_stats.hw_csum_err++;
709 710
		}

711
		skb_record_rx_queue(skb, fp->rx_queue);
712

713 714
		if (le16_to_cpu(cqe_fp->pars_flags.flags) &
		    PARSING_FLAGS_VLAN)
715
			__vlan_hwaccel_put_tag(skb,
716
					       le16_to_cpu(cqe_fp->vlan_tag));
717
		napi_gro_receive(&fp->napi, skb);
718 719 720


next_rx:
721
		rx_buf->data = NULL;
722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753

		bd_cons = NEXT_RX_IDX(bd_cons);
		bd_prod = NEXT_RX_IDX(bd_prod);
		bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
		rx_pkt++;
next_cqe:
		sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
		sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);

		if (rx_pkt == budget)
			break;
	} /* while */

	fp->rx_bd_cons = bd_cons;
	fp->rx_bd_prod = bd_prod_fw;
	fp->rx_comp_cons = sw_comp_cons;
	fp->rx_comp_prod = sw_comp_prod;

	/* Update producers */
	bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
			     fp->rx_sge_prod);

	fp->rx_pkt += rx_pkt;
	fp->rx_calls++;

	return rx_pkt;
}

static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
{
	struct bnx2x_fastpath *fp = fp_cookie;
	struct bnx2x *bp = fp->bp;
754
	u8 cos;
755

756 757 758 759
	DP(BNX2X_MSG_FP, "got an MSI-X interrupt on IDX:SB "
			 "[fp %d fw_sd %d igusb %d]\n",
	   fp->index, fp->fw_sb_id, fp->igu_sb_id);
	bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
760 761 762 763 764 765 766 767

#ifdef BNX2X_STOP_ON_ERROR
	if (unlikely(bp->panic))
		return IRQ_HANDLED;
#endif

	/* Handle Rx and Tx according to MSI-X vector */
	prefetch(fp->rx_cons_sb);
768 769 770 771

	for_each_cos_in_tx_queue(fp, cos)
		prefetch(fp->txdata[cos].tx_cons_sb);

772
	prefetch(&fp->sb_running_index[SM_RX_ID]);
773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794
	napi_schedule(&bnx2x_fp(bp, fp->index, napi));

	return IRQ_HANDLED;
}

/* HW Lock for shared dual port PHYs */
void bnx2x_acquire_phy_lock(struct bnx2x *bp)
{
	mutex_lock(&bp->port.phy_mutex);

	if (bp->port.need_hw_lock)
		bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
}

void bnx2x_release_phy_lock(struct bnx2x *bp)
{
	if (bp->port.need_hw_lock)
		bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);

	mutex_unlock(&bp->port.phy_mutex);
}

795 796 797 798 799
/* calculates MF speed according to current linespeed and MF configuration */
u16 bnx2x_get_mf_speed(struct bnx2x *bp)
{
	u16 line_speed = bp->link_vars.line_speed;
	if (IS_MF(bp)) {
800 801 802 803 804
		u16 maxCfg = bnx2x_extract_max_cfg(bp,
						   bp->mf_config[BP_VN(bp)]);

		/* Calculate the current MAX line speed limit for the MF
		 * devices
805
		 */
806 807 808
		if (IS_MF_SI(bp))
			line_speed = (line_speed * maxCfg) / 100;
		else { /* SD mode */
809 810 811 812
			u16 vn_max_rate = maxCfg * 100;

			if (vn_max_rate < line_speed)
				line_speed = vn_max_rate;
813
		}
814 815 816 817 818
	}

	return line_speed;
}

819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864
/**
 * bnx2x_fill_report_data - fill link report data to report
 *
 * @bp:		driver handle
 * @data:	link state to update
 *
 * It uses a none-atomic bit operations because is called under the mutex.
 */
static inline void bnx2x_fill_report_data(struct bnx2x *bp,
					  struct bnx2x_link_report_data *data)
{
	u16 line_speed = bnx2x_get_mf_speed(bp);

	memset(data, 0, sizeof(*data));

	/* Fill the report data: efective line speed */
	data->line_speed = line_speed;

	/* Link is down */
	if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
		__set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
			  &data->link_report_flags);

	/* Full DUPLEX */
	if (bp->link_vars.duplex == DUPLEX_FULL)
		__set_bit(BNX2X_LINK_REPORT_FD, &data->link_report_flags);

	/* Rx Flow Control is ON */
	if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
		__set_bit(BNX2X_LINK_REPORT_RX_FC_ON, &data->link_report_flags);

	/* Tx Flow Control is ON */
	if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
		__set_bit(BNX2X_LINK_REPORT_TX_FC_ON, &data->link_report_flags);
}

/**
 * bnx2x_link_report - report link status to OS.
 *
 * @bp:		driver handle
 *
 * Calls the __bnx2x_link_report() under the same locking scheme
 * as a link/PHY state managing code to ensure a consistent link
 * reporting.
 */

865 866
void bnx2x_link_report(struct bnx2x *bp)
{
867 868 869 870
	bnx2x_acquire_phy_lock(bp);
	__bnx2x_link_report(bp);
	bnx2x_release_phy_lock(bp);
}
871

872 873 874 875 876 877 878 879 880 881 882
/**
 * __bnx2x_link_report - report link status to OS.
 *
 * @bp:		driver handle
 *
 * None atomic inmlementation.
 * Should be called under the phy_lock.
 */
void __bnx2x_link_report(struct bnx2x *bp)
{
	struct bnx2x_link_report_data cur_data;
883

884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899
	/* reread mf_cfg */
	if (!CHIP_IS_E1(bp))
		bnx2x_read_mf_cfg(bp);

	/* Read the current link report info */
	bnx2x_fill_report_data(bp, &cur_data);

	/* Don't report link down or exactly the same link status twice */
	if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
	    (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
		      &bp->last_reported_link.link_report_flags) &&
	     test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
		      &cur_data.link_report_flags)))
		return;

	bp->link_cnt++;
900

901 902 903 904
	/* We are going to report a new link parameters now -
	 * remember the current data for the next time.
	 */
	memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
905

906 907 908 909 910 911
	if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
		     &cur_data.link_report_flags)) {
		netif_carrier_off(bp->dev);
		netdev_err(bp->dev, "NIC Link is Down\n");
		return;
	} else {
912 913 914
		const char *duplex;
		const char *flow;

915
		netif_carrier_on(bp->dev);
916

917 918
		if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
				       &cur_data.link_report_flags))
919
			duplex = "full";
920
		else
921
			duplex = "half";
922

923 924 925 926 927 928 929 930 931
		/* Handle the FC at the end so that only these flags would be
		 * possibly set. This way we may easily check if there is no FC
		 * enabled.
		 */
		if (cur_data.link_report_flags) {
			if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
				     &cur_data.link_report_flags)) {
				if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
				     &cur_data.link_report_flags))
932 933 934
					flow = "ON - receive & transmit";
				else
					flow = "ON - receive";
935
			} else {
936
				flow = "ON - transmit";
937
			}
938 939
		} else {
			flow = "none";
940
		}
941 942
		netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
			    cur_data.line_speed, duplex, flow);
943 944 945 946 947 948
	}
}

void bnx2x_init_rx_rings(struct bnx2x *bp)
{
	int func = BP_FUNC(bp);
949
	u16 ring_prod;
950
	int i, j;
951

952
	/* Allocate TPA resources */
Vladislav Zolotarov's avatar
Vladislav Zolotarov committed
953
	for_each_rx_queue(bp, j) {
954
		struct bnx2x_fastpath *fp = &bp->fp[j];
955

956 957 958
		DP(NETIF_MSG_IFUP,
		   "mtu %d  rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);

959
		if (!fp->disable_tpa) {
960
			/* Fill the per-aggregtion pool */
961
			for (i = 0; i < MAX_AGG_QS(bp); i++) {
962 963 964 965 966
				struct bnx2x_agg_info *tpa_info =
					&fp->tpa_info[i];
				struct sw_rx_bd *first_buf =
					&tpa_info->first_buf;

967 968 969
				first_buf->data = kmalloc(fp->rx_buf_size + NET_SKB_PAD,
							  GFP_ATOMIC);
				if (!first_buf->data) {
970 971 972 973 974 975 976 977
					BNX2X_ERR("Failed to allocate TPA "
						  "skb pool for queue[%d] - "
						  "disabling TPA on this "
						  "queue!\n", j);
					bnx2x_free_tpa_pool(bp, fp, i);
					fp->disable_tpa = 1;
					break;
				}
978 979
				dma_unmap_addr_set(first_buf, mapping, 0);
				tpa_info->tpa_state = BNX2X_TPA_STOP;
980
			}
981 982 983 984 985 986 987 988 989 990 991 992 993 994

			/* "next page" elements initialization */
			bnx2x_set_next_page_sgl(fp);

			/* set SGEs bit mask */
			bnx2x_init_sge_ring_bit_mask(fp);

			/* Allocate SGEs and initialize the ring elements */
			for (i = 0, ring_prod = 0;
			     i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {

				if (bnx2x_alloc_rx_sge(bp, fp, ring_prod) < 0) {
					BNX2X_ERR("was only able to allocate "
						  "%d rx sges\n", i);
995 996
					BNX2X_ERR("disabling TPA for "
						  "queue[%d]\n", j);
997
					/* Cleanup already allocated elements */
998 999 1000
					bnx2x_free_rx_sge_range(bp, fp,
								ring_prod);
					bnx2x_free_tpa_pool(bp, fp,
1001
							    MAX_AGG_QS(bp));
1002 1003 1004 1005 1006 1007 1008 1009
					fp->disable_tpa = 1;
					ring_prod = 0;
					break;
				}
				ring_prod = NEXT_SGE_IDX(ring_prod);
			}

			fp->rx_sge_prod = ring_prod;
1010 1011 1012
		}
	}

Vladislav Zolotarov's avatar
Vladislav Zolotarov committed
1013
	for_each_rx_queue(bp, j) {
1014 1015 1016 1017
		struct bnx2x_fastpath *fp = &bp->fp[j];

		fp->rx_bd_cons = 0;

1018 1019 1020 1021 1022 1023 1024
		/* Activate BD ring */
		/* Warning!
		 * this will generate an interrupt (to the TSTORM)
		 * must only be done after chip is initialized
		 */
		bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
				     fp->rx_sge_prod);
1025 1026 1027 1028

		if (j != 0)
			continue;

1029
		if (CHIP_IS_E1(bp)) {
Dmitry Kravkov's avatar
Dmitry Kravkov committed
1030 1031 1032 1033 1034 1035 1036
			REG_WR(bp, BAR_USTRORM_INTMEM +
			       USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
			       U64_LO(fp->rx_comp_mapping));
			REG_WR(bp, BAR_USTRORM_INTMEM +
			       USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
			       U64_HI(fp->rx_comp_mapping));
		}
1037 1038
	}
}
Dmitry Kravkov's avatar
Dmitry Kravkov committed
1039

1040 1041 1042
static void bnx2x_free_tx_skbs(struct bnx2x *bp)
{
	int i;
1043
	u8 cos;
1044

Vladislav Zolotarov's avatar
Vladislav Zolotarov committed
1045
	for_each_tx_queue(bp, i) {
1046
		struct bnx2x_fastpath *fp = &bp->fp[i];
1047 1048
		for_each_cos_in_tx_queue(fp, cos) {
			struct bnx2x_fp_txdata *txdata = &fp->txdata[cos];
1049
			unsigned pkts_compl = 0, bytes_compl = 0;
1050

1051 1052
			u16 sw_prod = txdata->tx_pkt_prod;
			u16 sw_cons = txdata->tx_pkt_cons;
1053

1054
			while (sw_cons != sw_prod) {
1055 1056
				bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons),
				    &pkts_compl, &bytes_compl);
1057 1058
				sw_cons++;
			}
1059 1060
			netdev_tx_reset_queue(
			    netdev_get_tx_queue(bp->dev, txdata->txq_index));
1061 1062 1063 1064
		}
	}
}

1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075
static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
{
	struct bnx2x *bp = fp->bp;
	int i;

	/* ring wasn't allocated */
	if (fp->rx_buf_ring == NULL)
		return;

	for (i = 0; i < NUM_RX_BD; i++) {
		struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1076
		u8 *data = rx_buf->data;
1077

1078
		if (data == NULL)
1079 1080 1081 1082 1083
			continue;
		dma_unmap_single(&bp->pdev->dev,
				 dma_unmap_addr(rx_buf, mapping),
				 fp->rx_buf_size, DMA_FROM_DEVICE);

1084 1085
		rx_buf->data = NULL;
		kfree(data);
1086 1087 1088
	}
}

1089 1090
static void bnx2x_free_rx_skbs(struct bnx2x *bp)
{
1091
	int j;
1092

Vladislav Zolotarov's avatar
Vladislav Zolotarov committed
1093
	for_each_rx_queue(bp, j) {
1094 1095
		struct bnx2x_fastpath *fp = &bp->fp[j];

1096
		bnx2x_free_rx_bds(fp);
1097 1098

		if (!fp->disable_tpa)
1099
			bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp));
1100 1101 1102 1103 1104 1105 1106 1107 1108
	}
}

void bnx2x_free_skbs(struct bnx2x *bp)
{
	bnx2x_free_tx_skbs(bp);
	bnx2x_free_rx_skbs(bp);
}

1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125
void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
{
	/* load old values */
	u32 mf_cfg = bp->mf_config[BP_VN(bp)];

	if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
		/* leave all but MAX value */
		mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;

		/* set new MAX value */
		mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
				& FUNC_MF_CFG_MAX_BW_MASK;

		bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
	}
}

1126 1127 1128 1129 1130 1131 1132
/**
 * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
 *
 * @bp:		driver handle
 * @nvecs:	number of vectors to be released
 */
static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
1133
{
1134
	int i, offset = 0;
1135

1136 1137 1138
	if (nvecs == offset)
		return;
	free_irq(bp->msix_table[offset].vector, bp->dev);
1139
	DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1140 1141
	   bp->msix_table[offset].vector);
	offset++;
1142
#ifdef BCM_CNIC
1143 1144
	if (nvecs == offset)
		return;
1145 1146
	offset++;
#endif
1147

Vladislav Zolotarov's avatar
Vladislav Zolotarov committed
1148
	for_each_eth_queue(bp, i) {
1149 1150 1151 1152
		if (nvecs == offset)
			return;
		DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d "
		   "irq\n", i, bp->msix_table[offset].vector);
1153

1154
		free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
1155 1156 1157
	}
}

1158
void bnx2x_free_irq(struct bnx2x *bp)
1159
{
1160
	if (bp->flags & USING_MSIX_FLAG)
1161
		bnx2x_free_msix_irqs(bp, BNX2X_NUM_ETH_QUEUES(bp) +
1162
				     CNIC_PRESENT + 1);
1163 1164 1165
	else if (bp->flags & USING_MSI_FLAG)
		free_irq(bp->pdev->irq, bp->dev);
	else
1166 1167 1168
		free_irq(bp->pdev->irq, bp->dev);
}

1169
int bnx2x_enable_msix(struct bnx2x *bp)
1170
{
1171
	int msix_vec = 0, i, rc, req_cnt;
1172

1173 1174 1175 1176
	bp->msix_table[msix_vec].entry = msix_vec;
	DP(NETIF_MSG_IFUP, "msix_table[0].entry = %d (slowpath)\n",
	   bp->msix_table[0].entry);
	msix_vec++;
1177 1178

#ifdef BCM_CNIC
1179 1180 1181 1182
	bp->msix_table[msix_vec].entry = msix_vec;
	DP(NETIF_MSG_IFUP, "msix_table[%d].entry = %d (CNIC)\n",
	   bp->msix_table[msix_vec].entry, bp->msix_table[msix_vec].entry);
	msix_vec++;
1183
#endif
1184
	/* We need separate vectors for ETH queues only (not FCoE) */
Vladislav Zolotarov's avatar
Vladislav Zolotarov committed
1185
	for_each_eth_queue(bp, i) {
1186
		bp->msix_table[msix_vec].entry = msix_vec;
1187
		DP(NETIF_MSG_IFUP, "msix_table[%d].entry = %d "
1188 1189
		   "(fastpath #%u)\n", msix_vec, msix_vec, i);
		msix_vec++;
1190 1191
	}

1192
	req_cnt = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_PRESENT + 1;
1193 1194

	rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], req_cnt);
1195 1196 1197 1198 1199 1200

	/*
	 * reconfigure number of tx/rx queues according to available
	 * MSI-X vectors
	 */
	if (rc >= BNX2X_MIN_MSIX_VEC_CNT) {
1201 1202
		/* how less vectors we will have? */
		int diff = req_cnt - rc;
1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213

		DP(NETIF_MSG_IFUP,
		   "Trying to use less MSI-X vectors: %d\n", rc);

		rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);

		if (rc) {
			DP(NETIF_MSG_IFUP,
			   "MSI-X is not attainable  rc %d\n", rc);
			return rc;
		}
1214 1215 1216 1217
		/*
		 * decrease number of queues by number of unallocated entries
		 */
		bp->num_queues -= diff;
1218 1219 1220 1221

		DP(NETIF_MSG_IFUP, "New queue configuration set: %d\n",
				  bp->num_queues);
	} else if (rc) {
1222 1223 1224
		/* fall to INTx if not enough memory */
		if (rc == -ENOMEM)
			bp->flags |= DISABLE_MSI_FLAG;
1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235
		DP(NETIF_MSG_IFUP, "MSI-X is not attainable  rc %d\n", rc);
		return rc;
	}

	bp->flags |= USING_MSIX_FLAG;

	return 0;
}

static int bnx2x_req_msix_irqs(struct bnx2x *bp)
{
1236
	int i, rc, offset = 0;
1237

1238 1239
	rc = request_irq(bp->msix_table[offset++].vector,
			 bnx2x_msix_sp_int, 0,
1240 1241 1242 1243 1244 1245 1246 1247 1248
			 bp->dev->name, bp->dev);
	if (rc) {
		BNX2X_ERR("request sp irq failed\n");
		return -EBUSY;
	}

#ifdef BCM_CNIC
	offset++;
#endif
Vladislav Zolotarov's avatar
Vladislav Zolotarov committed
1249
	for_each_eth_queue(bp, i) {
1250 1251 1252 1253
		struct bnx2x_fastpath *fp = &bp->fp[i];
		snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
			 bp->dev->name, i);

1254
		rc = request_irq(bp->msix_table[offset].vector,
1255 1256
				 bnx2x_msix_fp_int, 0, fp->name, fp);
		if (rc) {
1257 1258 1259
			BNX2X_ERR("request fp #%d irq (%d) failed  rc %d\n", i,
			      bp->msix_table[offset].vector, rc);
			bnx2x_free_msix_irqs(bp, offset);
1260 1261 1262
			return -EBUSY;
		}

1263
		offset++;
1264 1265
	}

Vladislav Zolotarov's avatar
Vladislav Zolotarov committed
1266
	i = BNX2X_NUM_ETH_QUEUES(bp);
1267
	offset = 1 + CNIC_PRESENT;
1268 1269 1270 1271 1272 1273 1274 1275 1276
	netdev_info(bp->dev, "using MSI-X  IRQs: sp %d  fp[%d] %d"
	       " ... fp[%d] %d\n",
	       bp->msix_table[0].vector,
	       0, bp->msix_table[offset].vector,
	       i - 1, bp->msix_table[offset + i - 1].vector);

	return 0;
}

1277
int bnx2x_enable_msi(struct bnx2x *bp)
1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305
{
	int rc;

	rc = pci_enable_msi(bp->pdev);
	if (rc) {
		DP(NETIF_MSG_IFUP, "MSI is not attainable\n");
		return -1;
	}
	bp->flags |= USING_MSI_FLAG;

	return 0;
}

static int bnx2x_req_irq(struct bnx2x *bp)
{
	unsigned long flags;
	int rc;

	if (bp->flags & USING_MSI_FLAG)
		flags = 0;
	else
		flags = IRQF_SHARED;

	rc = request_irq(bp->pdev->irq, bnx2x_interrupt, flags,
			 bp->dev->name, bp->dev);
	return rc;
}

1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330
static inline int bnx2x_setup_irqs(struct bnx2x *bp)
{
	int rc = 0;
	if (bp->flags & USING_MSIX_FLAG) {
		rc = bnx2x_req_msix_irqs(bp);
		if (rc)
			return rc;
	} else {
		bnx2x_ack_int(bp);
		rc = bnx2x_req_irq(bp);
		if (rc) {
			BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
			return rc;
		}
		if (bp->flags & USING_MSI_FLAG) {
			bp->dev->irq = bp->pdev->irq;
			netdev_info(bp->dev, "using MSI  IRQ %d\n",
			       bp->pdev->irq);
		}
	}

	return 0;
}

static inline void bnx2x_napi_enable(struct bnx2x *bp)
1331 1332 1333
{
	int i;

1334
	for_each_rx_queue(bp, i)
1335 1336 1337
		napi_enable(&bnx2x_fp(bp, i, napi));
}

1338
static inline void bnx2x_napi_disable(struct bnx2x *bp)
1339 1340 1341
{
	int i;

1342
	for_each_rx_queue(bp, i)
1343 1344 1345 1346 1347
		napi_disable(&bnx2x_fp(bp, i, napi));
}

void bnx2x_netif_start(struct bnx2x *bp)
{
1348 1349 1350 1351 1352
	if (netif_running(bp->dev)) {
		bnx2x_napi_enable(bp);
		bnx2x_int_enable(bp);
		if (bp->state == BNX2X_STATE_OPEN)
			netif_tx_wake_all_queues(bp->dev);
1353 1354 1355 1356 1357 1358 1359 1360 1361
	}
}

void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
{
	bnx2x_int_disable_sync(bp, disable_hw);
	bnx2x_napi_disable(bp);
}

1362 1363 1364
u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb)
{
	struct bnx2x *bp = netdev_priv(dev);
1365

1366
#ifdef BCM_CNIC
1367
	if (!NO_FCOE(bp)) {
1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380
		struct ethhdr *hdr = (struct ethhdr *)skb->data;
		u16 ether_type = ntohs(hdr->h_proto);

		/* Skip VLAN tag if present */
		if (ether_type == ETH_P_8021Q) {
			struct vlan_ethhdr *vhdr =
				(struct vlan_ethhdr *)skb->data;

			ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
		}

		/* If ethertype is FCoE or FIP - use FCoE ring */
		if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1381
			return bnx2x_fcoe_tx(bp, txq_index);
1382 1383
	}
#endif
1384
	/* select a non-FCoE queue */
1385
	return __skb_tx_hash(dev, skb, BNX2X_NUM_ETH_QUEUES(bp));
1386 1387
}

1388 1389 1390 1391
void bnx2x_set_num_queues(struct bnx2x *bp)
{
	switch (bp->multi_mode) {
	case ETH_RSS_MODE_DISABLED:
1392
		bp->num_queues = 1;
1393 1394 1395
		break;
	case ETH_RSS_MODE_REGULAR:
		bp->num_queues = bnx2x_calc_num_queues(bp);
1396
		break;
Dmitry Kravkov's avatar
Dmitry Kravkov committed
1397

1398
	default:
1399
		bp->num_queues = 1;
1400 1401
		break;
	}
Vladislav Zolotarov's avatar
Vladislav Zolotarov committed
1402

Dmitry Kravkov's avatar
Dmitry Kravkov committed
1403 1404 1405 1406 1407
#ifdef BCM_CNIC
	/* override in ISCSI SD mod */
	if (IS_MF_ISCSI_SD(bp))
		bp->num_queues = 1;
#endif
Vladislav Zolotarov's avatar
Vladislav Zolotarov committed
1408
	/* Add special queues */
1409
	bp->num_queues += NON_ETH_CONTEXT_USE;
Vladislav Zolotarov's avatar
Vladislav Zolotarov committed
1410 1411
}

1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433
/**
 * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues
 *
 * @bp:		Driver handle
 *
 * We currently support for at most 16 Tx queues for each CoS thus we will
 * allocate a multiple of 16 for ETH L2 rings according to the value of the
 * bp->max_cos.
 *
 * If there is an FCoE L2 queue the appropriate Tx queue will have the next
 * index after all ETH L2 indices.
 *
 * If the actual number of Tx queues (for each CoS) is less than 16 then there
 * will be the holes at the end of each group of 16 ETh L2 indices (0..15,
 * 16..31,...) with indicies that are not coupled with any real Tx queue.
 *
 * The proper configuration of skb->queue_mapping is handled by
 * bnx2x_select_queue() and __skb_tx_hash().
 *
 * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash()
 * will return a proper Tx index if TC is enabled (netdev->num_tc > 0).
 */
Vladislav Zolotarov's avatar
Vladislav Zolotarov committed
1434 1435
static inline int bnx2x_set_real_num_queues(struct bnx2x *bp)
{
1436
	int rc, tx, rx;
Vladislav Zolotarov's avatar
Vladislav Zolotarov committed
1437

1438 1439
	tx = MAX_TXQS_PER_COS * bp->max_cos;
	rx = BNX2X_NUM_ETH_QUEUES(bp);
Vladislav Zolotarov's avatar
Vladislav Zolotarov committed
1440

1441 1442 1443 1444 1445 1446
/* account for fcoe queue */
#ifdef BCM_CNIC
	if (!NO_FCOE(bp)) {
		rx += FCOE_PRESENT;
		tx += FCOE_PRESENT;
	}
Vladislav Zolotarov's avatar
Vladislav Zolotarov committed
1447
#endif
1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462

	rc = netif_set_real_num_tx_queues(bp->dev, tx);
	if (rc) {
		BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
		return rc;
	}
	rc = netif_set_real_num_rx_queues(bp->dev, rx);
	if (rc) {
		BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
		return rc;
	}

	DP(NETIF_MSG_DRV, "Setting real num queues to (tx, rx) (%d, %d)\n",
			  tx, rx);

Vladislav Zolotarov's avatar
Vladislav Zolotarov committed
1463 1464 1465
	return rc;
}

1466 1467 1468 1469 1470 1471
static inline void bnx2x_set_rx_buf_size(struct bnx2x *bp)
{
	int i;

	for_each_queue(bp, i) {
		struct bnx2x_fastpath *fp = &bp->fp[i];
1472
		u32 mtu;
1473 1474 1475 1476 1477 1478 1479 1480 1481

		/* Always use a mini-jumbo MTU for the FCoE L2 ring */
		if (IS_FCOE_IDX(i))
			/*
			 * Although there are no IP frames expected to arrive to
			 * this ring we still want to add an
			 * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
			 * overrun attack.
			 */
1482
			mtu = BNX2X_FCOE_MINI_JUMBO_MTU;
1483
		else
1484 1485 1486 1487 1488 1489 1490
			mtu = bp->dev->mtu;
		fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START +
				  IP_HEADER_ALIGNMENT_PADDING +
				  ETH_OVREHEAD +
				  mtu +
				  BNX2X_FW_RX_ALIGN_END;
		/* Note : rx_buf_size doesnt take into account NET_SKB_PAD */
1491 1492 1493
	}
}

1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506
static inline int bnx2x_init_rss_pf(struct bnx2x *bp)
{
	int i;
	u8 ind_table[T_ETH_INDIRECTION_TABLE_SIZE] = {0};
	u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);

	/*
	 * Prepare the inital contents fo the indirection table if RSS is
	 * enabled
	 */
	if (bp->multi_mode != ETH_RSS_MODE_DISABLED) {
		for (i = 0; i < sizeof(ind_table); i++)
			ind_table[i] =
1507 1508
				bp->fp->cl_id +
				ethtool_rxfh_indir_default(i, num_eth_queues);
1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671
	}

	/*
	 * For 57710 and 57711 SEARCHER configuration (rss_keys) is
	 * per-port, so if explicit configuration is needed , do it only
	 * for a PMF.
	 *
	 * For 57712 and newer on the other hand it's a per-function
	 * configuration.
	 */
	return bnx2x_config_rss_pf(bp, ind_table,
				   bp->port.pmf || !CHIP_IS_E1x(bp));
}

int bnx2x_config_rss_pf(struct bnx2x *bp, u8 *ind_table, bool config_hash)
{
	struct bnx2x_config_rss_params params = {0};
	int i;

	/* Although RSS is meaningless when there is a single HW queue we
	 * still need it enabled in order to have HW Rx hash generated.
	 *
	 * if (!is_eth_multi(bp))
	 *      bp->multi_mode = ETH_RSS_MODE_DISABLED;
	 */

	params.rss_obj = &bp->rss_conf_obj;

	__set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);

	/* RSS mode */
	switch (bp->multi_mode) {
	case ETH_RSS_MODE_DISABLED:
		__set_bit(BNX2X_RSS_MODE_DISABLED, &params.rss_flags);
		break;
	case ETH_RSS_MODE_REGULAR:
		__set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
		break;
	case ETH_RSS_MODE_VLAN_PRI:
		__set_bit(BNX2X_RSS_MODE_VLAN_PRI, &params.rss_flags);
		break;
	case ETH_RSS_MODE_E1HOV_PRI:
		__set_bit(BNX2X_RSS_MODE_E1HOV_PRI, &params.rss_flags);
		break;
	case ETH_RSS_MODE_IP_DSCP:
		__set_bit(BNX2X_RSS_MODE_IP_DSCP, &params.rss_flags);
		break;
	default:
		BNX2X_ERR("Unknown multi_mode: %d\n", bp->multi_mode);
		return -EINVAL;
	}

	/* If RSS is enabled */
	if (bp->multi_mode != ETH_RSS_MODE_DISABLED) {
		/* RSS configuration */
		__set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
		__set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
		__set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
		__set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);

		/* Hash bits */
		params.rss_result_mask = MULTI_MASK;

		memcpy(params.ind_table, ind_table, sizeof(params.ind_table));

		if (config_hash) {
			/* RSS keys */
			for (i = 0; i < sizeof(params.rss_key) / 4; i++)
				params.rss_key[i] = random32();

			__set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
		}
	}

	return bnx2x_config_rss(bp, &params);
}

static inline int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
{
	struct bnx2x_func_state_params func_params = {0};

	/* Prepare parameters for function state transitions */
	__set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);

	func_params.f_obj = &bp->func_obj;
	func_params.cmd = BNX2X_F_CMD_HW_INIT;

	func_params.params.hw_init.load_phase = load_code;

	return bnx2x_func_state_change(bp, &func_params);
}

/*
 * Cleans the object that have internal lists without sending
 * ramrods. Should be run when interrutps are disabled.
 */
static void bnx2x_squeeze_objects(struct bnx2x *bp)
{
	int rc;
	unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
	struct bnx2x_mcast_ramrod_params rparam = {0};
	struct bnx2x_vlan_mac_obj *mac_obj = &bp->fp->mac_obj;

	/***************** Cleanup MACs' object first *************************/

	/* Wait for completion of requested */
	__set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
	/* Perform a dry cleanup */
	__set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);

	/* Clean ETH primary MAC */
	__set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
	rc = mac_obj->delete_all(bp, &bp->fp->mac_obj, &vlan_mac_flags,
				 &ramrod_flags);
	if (rc != 0)
		BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);

	/* Cleanup UC list */
	vlan_mac_flags = 0;
	__set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
	rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
				 &ramrod_flags);
	if (rc != 0)
		BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);

	/***************** Now clean mcast object *****************************/
	rparam.mcast_obj = &bp->mcast_obj;
	__set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);

	/* Add a DEL command... */
	rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
	if (rc < 0)
		BNX2X_ERR("Failed to add a new DEL command to a multi-cast "
			  "object: %d\n", rc);

	/* ...and wait until all pending commands are cleared */
	rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
	while (rc != 0) {
		if (rc < 0) {
			BNX2X_ERR("Failed to clean multi-cast object: %d\n",
				  rc);
			return;
		}

		rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
	}
}

#ifndef BNX2X_STOP_ON_ERROR
#define LOAD_ERROR_EXIT(bp, label) \
	do { \
		(bp)->state = BNX2X_STATE_ERROR; \
		goto label; \
	} while (0)
#else
#define LOAD_ERROR_EXIT(bp, label) \
	do { \
		(bp)->state = BNX2X_STATE_ERROR; \
		(bp)->panic = 1; \
		return -EBUSY; \
	} while (0)
#endif

1672 1673 1674
/* must be called with rtnl_lock */
int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
{
1675
	int port = BP_PORT(bp);
1676 1677 1678 1679 1680 1681 1682 1683 1684 1685
	u32 load_code;
	int i, rc;

#ifdef BNX2X_STOP_ON_ERROR
	if (unlikely(bp->panic))
		return -EPERM;
#endif

	bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;

1686 1687 1688 1689 1690 1691 1692
	/* Set the initial link reported state to link down */
	bnx2x_acquire_phy_lock(bp);
	memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
	__set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
		&bp->last_reported_link.link_report_flags);
	bnx2x_release_phy_lock(bp);

1693 1694 1695
	/* must be called before memory allocation and HW init */
	bnx2x_ilt_set_info(bp);

1696 1697 1698 1699
	/*
	 * Zero fastpath structures preserving invariants like napi, which are
	 * allocated only once, fp index, max_cos, bp pointer.
	 * Also set fp->disable_tpa.
1700 1701 1702 1703
	 */
	for_each_queue(bp, i)
		bnx2x_bz_fp(bp, i);

1704

1705 1706 1707
	/* Set the receive queues buffer size */
	bnx2x_set_rx_buf_size(bp);

1708
	if (bnx2x_alloc_mem(bp))
1709
		return -ENOMEM;
1710

1711 1712 1713 1714
	/* As long as bnx2x_alloc_mem() may possibly update
	 * bp->num_queues, bnx2x_set_real_num_queues() should always
	 * come after it.
	 */
Vladislav Zolotarov's avatar
Vladislav Zolotarov committed
1715
	rc = bnx2x_set_real_num_queues(bp);
1716
	if (rc) {
Vladislav Zolotarov's avatar
Vladislav Zolotarov committed
1717
		BNX2X_ERR("Unable to set real_num_queues\n");
1718
		LOAD_ERROR_EXIT(bp, load_error0);
1719 1720
	}

1721 1722 1723 1724 1725 1726
	/* configure multi cos mappings in kernel.
	 * this configuration may be overriden by a multi class queue discipline
	 * or by a dcbx negotiation result.
	 */
	bnx2x_setup_tc(bp->dev, bp->max_cos);

1727 1728
	bnx2x_napi_enable(bp);

1729 1730 1731
	/* set pf load just before approaching the MCP */
	bnx2x_set_pf_load(bp);

1732
	/* Send LOAD_REQUEST command to MCP
1733 1734 1735 1736
	 * Returns the type of LOAD command:
	 * if it is the first port to be initialized
	 * common blocks should be initialized, otherwise - not
	 */
1737
	if (!BP_NOMCP(bp)) {
Ariel Elior's avatar
Ariel Elior committed
1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749
		/* init fw_seq */
		bp->fw_seq =
			(SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) &
			 DRV_MSG_SEQ_NUMBER_MASK);
		BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);

		/* Get current FW pulse sequence */
		bp->fw_drv_pulse_wr_seq =
			(SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) &
			 DRV_PULSE_SEQ_MASK);
		BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq);

Yaniv Rosner's avatar
Yaniv Rosner committed
1750
		load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, 0);
1751 1752 1753
		if (!load_code) {
			BNX2X_ERR("MCP response failure, aborting\n");
			rc = -EBUSY;
1754
			LOAD_ERROR_EXIT(bp, load_error1);
1755 1756 1757
		}
		if (load_code == FW_MSG_CODE_DRV_LOAD_REFUSED) {
			rc = -EBUSY; /* other port in diagnostic mode */
1758
			LOAD_ERROR_EXIT(bp, load_error1);
1759
		}
1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782
		if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP &&
		    load_code != FW_MSG_CODE_DRV_LOAD_COMMON) {
			/* build FW version dword */
			u32 my_fw = (BCM_5710_FW_MAJOR_VERSION) +
					(BCM_5710_FW_MINOR_VERSION << 8) +
					(BCM_5710_FW_REVISION_VERSION << 16) +
					(BCM_5710_FW_ENGINEERING_VERSION << 24);

			/* read loaded FW from chip */
			u32 loaded_fw = REG_RD(bp, XSEM_REG_PRAM);

			DP(BNX2X_MSG_SP, "loaded fw %x, my fw %x",
			   loaded_fw, my_fw);

			/* abort nic load if version mismatch */
			if (my_fw != loaded_fw) {
				BNX2X_ERR("bnx2x with FW %x already loaded, "
					  "which mismatches my %x FW. aborting",
					  loaded_fw, my_fw);
				rc = -EBUSY;
				LOAD_ERROR_EXIT(bp, load_error2);
			}
		}
1783 1784

	} else {
Dmitry Kravkov's avatar
Dmitry Kravkov committed
1785
		int path = BP_PATH(bp);
1786

Dmitry Kravkov's avatar
Dmitry Kravkov committed
1787 1788 1789 1790 1791 1792 1793 1794 1795
		DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d]      %d, %d, %d\n",
		   path, load_count[path][0], load_count[path][1],
		   load_count[path][2]);
		load_count[path][0]++;
		load_count[path][1 + port]++;
		DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d]  %d, %d, %d\n",
		   path, load_count[path][0], load_count[path][1],
		   load_count[path][2]);
		if (load_count[path][0] == 1)
1796
			load_code = FW_MSG_CODE_DRV_LOAD_COMMON;
Dmitry Kravkov's avatar
Dmitry Kravkov committed
1797
		else if (load_count[path][1 + port] == 1)
1798 1799 1800 1801 1802 1803
			load_code = FW_MSG_CODE_DRV_LOAD_PORT;
		else
			load_code = FW_MSG_CODE_DRV_LOAD_FUNCTION;
	}

	if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
Dmitry Kravkov's avatar
Dmitry Kravkov committed
1804
	    (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
1805
	    (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
1806
		bp->port.pmf = 1;
1807 1808 1809 1810 1811 1812 1813 1814
		/*
		 * We need the barrier to ensure the ordering between the
		 * writing to bp->port.pmf here and reading it from the
		 * bnx2x_periodic_task().
		 */
		smp_mb();
		queue_delayed_work(bnx2x_wq, &bp->period_task, 0);
	} else
1815
		bp->port.pmf = 0;
1816

1817 1818
	DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);

1819 1820 1821
	/* Init Function state controlling object */
	bnx2x__init_func_obj(bp);

1822 1823 1824 1825
	/* Initialize HW */
	rc = bnx2x_init_hw(bp, load_code);
	if (rc) {
		BNX2X_ERR("HW init failed, aborting\n");
Yaniv Rosner's avatar
Yaniv Rosner committed
1826
		bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1827
		LOAD_ERROR_EXIT(bp, load_error2);
1828 1829
	}

1830 1831
	/* Connect to IRQs */
	rc = bnx2x_setup_irqs(bp);
1832 1833
	if (rc) {
		bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1834
		LOAD_ERROR_EXIT(bp, load_error2);
1835 1836
	}

1837 1838 1839
	/* Setup NIC internals and enable interrupts */
	bnx2x_nic_init(bp, load_code);

1840 1841 1842
	/* Init per-function objects */
	bnx2x_init_bp_objs(bp);

Dmitry Kravkov's avatar
Dmitry Kravkov committed
1843 1844
	if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
	    (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855
	    (bp->common.shmem2_base)) {
		if (SHMEM2_HAS(bp, dcc_support))
			SHMEM2_WR(bp, dcc_support,
				  (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
				   SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
	}

	bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
	rc = bnx2x_func_start(bp);
	if (rc) {
		BNX2X_ERR("Function start failed!\n");
1856
		bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1857 1858
		LOAD_ERROR_EXIT(bp, load_error3);
	}
1859 1860 1861

	/* Send LOAD_DONE command to MCP */
	if (!BP_NOMCP(bp)) {
Yaniv Rosner's avatar
Yaniv Rosner committed
1862
		load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1863 1864 1865
		if (!load_code) {
			BNX2X_ERR("MCP response failure, aborting\n");
			rc = -EBUSY;
1866
			LOAD_ERROR_EXIT(bp, load_error3);
1867 1868 1869
		}
	}

1870
	rc = bnx2x_setup_leading(bp);
1871 1872
	if (rc) {
		BNX2X_ERR("Setup leading failed!\n");
1873
		LOAD_ERROR_EXIT(bp, load_error3);
Dmitry Kravkov's avatar
Dmitry Kravkov committed
1874
	}
1875 1876

#ifdef BCM_CNIC
1877
	/* Enable Timer scan */
1878
	REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
1879
#endif
Dmitry Kravkov's avatar
Dmitry Kravkov committed
1880

1881
	for_each_nondefault_queue(bp, i) {
1882
		rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
1883
		if (rc)
1884
			LOAD_ERROR_EXIT(bp, load_error4);
1885 1886
	}

1887 1888 1889 1890
	rc = bnx2x_init_rss_pf(bp);
	if (rc)
		LOAD_ERROR_EXIT(bp, load_error4);

1891 1892 1893
	/* Now when Clients are configured we are ready to work */
	bp->state = BNX2X_STATE_OPEN;

1894 1895 1896 1897
	/* Configure a ucast MAC */
	rc = bnx2x_set_eth_mac(bp, true);
	if (rc)
		LOAD_ERROR_EXIT(bp, load_error4);
1898

1899 1900 1901 1902 1903
	if (bp->pending_max) {
		bnx2x_update_max_mf_config(bp, bp->pending_max);
		bp->pending_max = 0;
	}

1904 1905 1906
	if (bp->port.pmf)
		bnx2x_initial_phy_init(bp, load_mode);

1907 1908 1909 1910
	/* Start fast path */

	/* Initialize Rx filter. */
	netif_addr_lock_bh(bp->dev);
1911
	bnx2x_set_rx_mode(bp->dev);
1912
	netif_addr_unlock_bh(bp->dev);
1913

1914
	/* Start the Tx */
1915 1916
	switch (load_mode) {
	case LOAD_NORMAL:
1917 1918
		/* Tx queue should be only reenabled */
		netif_tx_wake_all_queues(bp->dev);
1919 1920 1921 1922
		break;

	case LOAD_OPEN:
		netif_tx_start_all_queues(bp->dev);
1923
		smp_mb__after_clear_bit();
1924 1925 1926 1927 1928 1929 1930 1931 1932 1933
		break;

	case LOAD_DIAG:
		bp->state = BNX2X_STATE_DIAG;
		break;

	default:
		break;
	}

1934 1935 1936
	if (bp->port.pmf)
		bnx2x_update_drv_flags(bp, DRV_FLAGS_DCB_CONFIGURED, 0);
	else
1937 1938 1939 1940 1941 1942
		bnx2x__link_status_update(bp);

	/* start the timer */
	mod_timer(&bp->timer, jiffies + bp->current_interval);

#ifdef BCM_CNIC
1943 1944
	/* re-read iscsi info */
	bnx2x_get_iscsi_info(bp);
1945 1946 1947 1948 1949
	bnx2x_setup_cnic_irq_info(bp);
	if (bp->state == BNX2X_STATE_OPEN)
		bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
#endif

1950 1951 1952 1953 1954 1955
	/* Wait for all pending SP commands to complete */
	if (!bnx2x_wait_sp_comp(bp, ~0x0UL)) {
		BNX2X_ERR("Timeout waiting for SP elements to complete\n");
		bnx2x_nic_unload(bp, UNLOAD_CLOSE);
		return -EBUSY;
	}
1956

1957
	bnx2x_dcbx_init(bp);
1958 1959
	return 0;

1960
#ifndef BNX2X_STOP_ON_ERROR
1961
load_error4:
1962
#ifdef BCM_CNIC
1963
	/* Disable Timer scan */
1964
	REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
1965 1966 1967
#endif
load_error3:
	bnx2x_int_disable_sync(bp, 1);
1968

1969 1970 1971
	/* Clean queueable objects */
	bnx2x_squeeze_objects(bp);

1972 1973
	/* Free SKBs, SGEs, TPA pool and driver internals */
	bnx2x_free_skbs(bp);
Vladislav Zolotarov's avatar
Vladislav Zolotarov committed
1974
	for_each_rx_queue(bp, i)
1975
		bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
1976

1977
	/* Release IRQs */
1978 1979 1980 1981 1982 1983 1984 1985
	bnx2x_free_irq(bp);
load_error2:
	if (!BP_NOMCP(bp)) {
		bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
		bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
	}

	bp->port.pmf = 0;
1986 1987
load_error1:
	bnx2x_napi_disable(bp);
1988 1989
	/* clear pf_load status, as it was already set */
	bnx2x_clear_pf_load(bp);
1990
load_error0:
1991 1992 1993
	bnx2x_free_mem(bp);

	return rc;
1994
#endif /* ! BNX2X_STOP_ON_ERROR */
1995 1996 1997 1998 1999 2000
}

/* must be called with rtnl_lock */
int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode)
{
	int i;
2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011
	bool global = false;

	if ((bp->state == BNX2X_STATE_CLOSED) ||
	    (bp->state == BNX2X_STATE_ERROR)) {
		/* We can get here if the driver has been unloaded
		 * during parity error recovery and is either waiting for a
		 * leader to complete or for other functions to unload and
		 * then ifdown has been issued. In this case we want to
		 * unload and let other functions to complete a recovery
		 * process.
		 */
2012 2013
		bp->recovery_state = BNX2X_RECOVERY_DONE;
		bp->is_leader = 0;
2014 2015 2016 2017
		bnx2x_release_leader_lock(bp);
		smp_mb();

		DP(NETIF_MSG_HW, "Releasing a leadership...\n");
2018 2019 2020 2021

		return -EINVAL;
	}

2022 2023 2024 2025 2026 2027 2028 2029
	/*
	 * It's important to set the bp->state to the value different from
	 * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
	 * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
	 */
	bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
	smp_mb();

2030 2031 2032
	/* Stop Tx */
	bnx2x_tx_disable(bp);

2033 2034 2035 2036 2037 2038 2039
#ifdef BCM_CNIC
	bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
#endif

	bp->rx_mode = BNX2X_RX_MODE_NONE;

	del_timer_sync(&bp->timer);
Dmitry Kravkov's avatar
Dmitry Kravkov committed
2040

2041 2042 2043 2044
	/* Set ALWAYS_ALIVE bit in shmem */
	bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;

	bnx2x_drv_pulse(bp);
2045

Dmitry Kravkov's avatar
Dmitry Kravkov committed
2046
	bnx2x_stats_handle(bp, STATS_EVENT_STOP);
2047
	bnx2x_save_statistics(bp);
2048 2049 2050 2051

	/* Cleanup the chip if needed */
	if (unload_mode != UNLOAD_RECOVERY)
		bnx2x_chip_cleanup(bp, unload_mode);
2052
	else {
2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066
		/* Send the UNLOAD_REQUEST to the MCP */
		bnx2x_send_unload_req(bp, unload_mode);

		/*
		 * Prevent transactions to host from the functions on the
		 * engine that doesn't reset global blocks in case of global
		 * attention once gloabl blocks are reset and gates are opened
		 * (the engine which leader will perform the recovery
		 * last).
		 */
		if (!CHIP_IS_E1x(bp))
			bnx2x_pf_disable(bp);

		/* Disable HW interrupts, NAPI */
2067 2068 2069
		bnx2x_netif_stop(bp, 1);

		/* Release IRQs */
2070
		bnx2x_free_irq(bp);
2071 2072 2073

		/* Report UNLOAD_DONE to MCP */
		bnx2x_send_unload_done(bp);
2074
	}
2075

2076 2077 2078 2079 2080 2081
	/*
	 * At this stage no more interrupts will arrive so we may safly clean
	 * the queueable objects here in case they failed to get cleaned so far.
	 */
	bnx2x_squeeze_objects(bp);

2082 2083 2084
	/* There should be no more pending SP commands at this stage */
	bp->sp_state = 0;

2085 2086 2087 2088
	bp->port.pmf = 0;

	/* Free SKBs, SGEs, TPA pool and driver internals */
	bnx2x_free_skbs(bp);
Vladislav Zolotarov's avatar
Vladislav Zolotarov committed
2089
	for_each_rx_queue(bp, i)
2090
		bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2091

2092 2093 2094 2095
	bnx2x_free_mem(bp);

	bp->state = BNX2X_STATE_CLOSED;

2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107
	/* Check if there are pending parity attentions. If there are - set
	 * RECOVERY_IN_PROGRESS.
	 */
	if (bnx2x_chk_parity_attn(bp, &global, false)) {
		bnx2x_set_reset_in_progress(bp);

		/* Set RESET_IS_GLOBAL if needed */
		if (global)
			bnx2x_set_reset_global(bp);
	}


2108 2109 2110
	/* The last driver must disable a "close the gate" if there is no
	 * parity attention or "process kill" pending.
	 */
2111
	if (!bnx2x_clear_pf_load(bp) && bnx2x_reset_is_done(bp, BP_PATH(bp)))
2112 2113 2114 2115
		bnx2x_disable_close_the_gate(bp);

	return 0;
}
Dmitry Kravkov's avatar
Dmitry Kravkov committed
2116

2117 2118 2119 2120
int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
{
	u16 pmcsr;

2121 2122 2123 2124 2125 2126
	/* If there is no power capability, silently succeed */
	if (!bp->pm_cap) {
		DP(NETIF_MSG_HW, "No power capability. Breaking.\n");
		return 0;
	}

2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171
	pci_read_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, &pmcsr);

	switch (state) {
	case PCI_D0:
		pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
				      ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
				       PCI_PM_CTRL_PME_STATUS));

		if (pmcsr & PCI_PM_CTRL_STATE_MASK)
			/* delay required during transition out of D3hot */
			msleep(20);
		break;

	case PCI_D3hot:
		/* If there are other clients above don't
		   shut down the power */
		if (atomic_read(&bp->pdev->enable_cnt) != 1)
			return 0;
		/* Don't shut down the power for emulation and FPGA */
		if (CHIP_REV_IS_SLOW(bp))
			return 0;

		pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
		pmcsr |= 3;

		if (bp->wol)
			pmcsr |= PCI_PM_CTRL_PME_ENABLE;

		pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
				      pmcsr);

		/* No more memory access after this point until
		* device is brought back to D0.
		*/
		break;

	default:
		return -EINVAL;
	}
	return 0;
}

/*
 * net_device service functions
 */
2172
int bnx2x_poll(struct napi_struct *napi, int budget)
2173 2174
{
	int work_done = 0;
2175
	u8 cos;
2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187
	struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
						 napi);
	struct bnx2x *bp = fp->bp;

	while (1) {
#ifdef BNX2X_STOP_ON_ERROR
		if (unlikely(bp->panic)) {
			napi_complete(napi);
			return 0;
		}
#endif

2188 2189 2190 2191
		for_each_cos_in_tx_queue(fp, cos)
			if (bnx2x_tx_queue_has_work(&fp->txdata[cos]))
				bnx2x_tx_int(bp, &fp->txdata[cos]);

2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202

		if (bnx2x_has_rx_work(fp)) {
			work_done += bnx2x_rx_int(fp, budget - work_done);

			/* must not complete if we consumed full budget */
			if (work_done >= budget)
				break;
		}

		/* Fall out from the NAPI loop if needed */
		if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
Vladislav Zolotarov's avatar
Vladislav Zolotarov committed
2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213
#ifdef BCM_CNIC
			/* No need to update SB for FCoE L2 ring as long as
			 * it's connected to the default SB and the SB
			 * has been updated when NAPI was scheduled.
			 */
			if (IS_FCOE_FP(fp)) {
				napi_complete(napi);
				break;
			}
#endif

2214
			bnx2x_update_fpsb_idx(fp);
Dmitry Kravkov's avatar
Dmitry Kravkov committed
2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227
			/* bnx2x_has_rx_work() reads the status block,
			 * thus we need to ensure that status block indices
			 * have been actually read (bnx2x_update_fpsb_idx)
			 * prior to this check (bnx2x_has_rx_work) so that
			 * we won't write the "newer" value of the status block
			 * to IGU (if there was a DMA right after
			 * bnx2x_has_rx_work and if there is no rmb, the memory
			 * reading (bnx2x_update_fpsb_idx) may be postponed
			 * to right before bnx2x_ack_sb). In this case there
			 * will never be another interrupt until there is
			 * another update of the status block, while there
			 * is still unhandled work.
			 */
2228 2229 2230 2231 2232
			rmb();

			if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
				napi_complete(napi);
				/* Re-enable interrupts */
2233 2234 2235 2236
				DP(NETIF_MSG_HW,
				   "Update index to %d\n", fp->fp_hc_idx);
				bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
					     le16_to_cpu(fp->fp_hc_idx),
2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252
					     IGU_INT_ENABLE, 1);
				break;
			}
		}
	}

	return work_done;
}

/* we split the first BD into headers and data BDs
 * to ease the pain of our fellow microcode engineers
 * we use one mapping for both BDs
 * So far this has only been observed to happen
 * in Other Operating Systems(TM)
 */
static noinline u16 bnx2x_tx_split(struct bnx2x *bp,
2253
				   struct bnx2x_fp_txdata *txdata,
2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273
				   struct sw_tx_bd *tx_buf,
				   struct eth_tx_start_bd **tx_bd, u16 hlen,
				   u16 bd_prod, int nbd)
{
	struct eth_tx_start_bd *h_tx_bd = *tx_bd;
	struct eth_tx_bd *d_tx_bd;
	dma_addr_t mapping;
	int old_len = le16_to_cpu(h_tx_bd->nbytes);

	/* first fix first BD */
	h_tx_bd->nbd = cpu_to_le16(nbd);
	h_tx_bd->nbytes = cpu_to_le16(hlen);

	DP(NETIF_MSG_TX_QUEUED,	"TSO split header size is %d "
	   "(%x:%x) nbd %d\n", h_tx_bd->nbytes, h_tx_bd->addr_hi,
	   h_tx_bd->addr_lo, h_tx_bd->nbd);

	/* now get a new data BD
	 * (after the pbd) and fill it */
	bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2274
	d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316

	mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
			   le32_to_cpu(h_tx_bd->addr_lo)) + hlen;

	d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
	d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
	d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);

	/* this marks the BD as one that has no individual mapping */
	tx_buf->flags |= BNX2X_TSO_SPLIT_BD;

	DP(NETIF_MSG_TX_QUEUED,
	   "TSO split data size is %d (%x:%x)\n",
	   d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);

	/* update tx_bd */
	*tx_bd = (struct eth_tx_start_bd *)d_tx_bd;

	return bd_prod;
}

static inline u16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
{
	if (fix > 0)
		csum = (u16) ~csum_fold(csum_sub(csum,
				csum_partial(t_header - fix, fix, 0)));

	else if (fix < 0)
		csum = (u16) ~csum_fold(csum_add(csum,
				csum_partial(t_header, -fix, 0)));

	return swab16(csum);
}

static inline u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
{
	u32 rc;

	if (skb->ip_summed != CHECKSUM_PARTIAL)
		rc = XMIT_PLAIN;

	else {
2317
		if (vlan_get_protocol(skb) == htons(ETH_P_IPV6)) {
2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328
			rc = XMIT_CSUM_V6;
			if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
				rc |= XMIT_CSUM_TCP;

		} else {
			rc = XMIT_CSUM_V4;
			if (ip_hdr(skb)->protocol == IPPROTO_TCP)
				rc |= XMIT_CSUM_TCP;
		}
	}

2329 2330 2331 2332
	if (skb_is_gso_v6(skb))
		rc |= XMIT_GSO_V6 | XMIT_CSUM_TCP | XMIT_CSUM_V6;
	else if (skb_is_gso(skb))
		rc |= XMIT_GSO_V4 | XMIT_CSUM_V4 | XMIT_CSUM_TCP;
2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373

	return rc;
}

#if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
/* check if packet requires linearization (packet is too fragmented)
   no need to check fragmentation if page size > 8K (there will be no
   violation to FW restrictions) */
static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
			     u32 xmit_type)
{
	int to_copy = 0;
	int hlen = 0;
	int first_bd_sz = 0;

	/* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
	if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {

		if (xmit_type & XMIT_GSO) {
			unsigned short lso_mss = skb_shinfo(skb)->gso_size;
			/* Check if LSO packet needs to be copied:
			   3 = 1 (for headers BD) + 2 (for PBD and last BD) */
			int wnd_size = MAX_FETCH_BD - 3;
			/* Number of windows to check */
			int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
			int wnd_idx = 0;
			int frag_idx = 0;
			u32 wnd_sum = 0;

			/* Headers length */
			hlen = (int)(skb_transport_header(skb) - skb->data) +
				tcp_hdrlen(skb);

			/* Amount of data (w/o headers) on linear part of SKB*/
			first_bd_sz = skb_headlen(skb) - hlen;

			wnd_sum  = first_bd_sz;

			/* Calculate the first sum - it's special */
			for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
				wnd_sum +=
2374
					skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]);
2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389

			/* If there was data on linear skb data - check it */
			if (first_bd_sz > 0) {
				if (unlikely(wnd_sum < lso_mss)) {
					to_copy = 1;
					goto exit_lbl;
				}

				wnd_sum -= first_bd_sz;
			}

			/* Others are easier: run through the frag list and
			   check all windows */
			for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
				wnd_sum +=
2390
			  skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]);
2391 2392 2393 2394 2395 2396

				if (unlikely(wnd_sum < lso_mss)) {
					to_copy = 1;
					break;
				}
				wnd_sum -=
2397
					skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]);
2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417
			}
		} else {
			/* in non-LSO too fragmented packet should always
			   be linearized */
			to_copy = 1;
		}
	}

exit_lbl:
	if (unlikely(to_copy))
		DP(NETIF_MSG_TX_QUEUED,
		   "Linearization IS REQUIRED for %s packet. "
		   "num_frags %d  hlen %d  first_bd_sz %d\n",
		   (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
		   skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);

	return to_copy;
}
#endif

2418 2419
static inline void bnx2x_set_pbd_gso_e2(struct sk_buff *skb, u32 *parsing_data,
					u32 xmit_type)
Dmitry Kravkov's avatar
Dmitry Kravkov committed
2420
{
2421 2422 2423
	*parsing_data |= (skb_shinfo(skb)->gso_size <<
			      ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
			      ETH_TX_PARSE_BD_E2_LSO_MSS;
Dmitry Kravkov's avatar
Dmitry Kravkov committed
2424 2425
	if ((xmit_type & XMIT_GSO_V6) &&
	    (ipv6_hdr(skb)->nexthdr == NEXTHDR_IPV6))
2426
		*parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
Dmitry Kravkov's avatar
Dmitry Kravkov committed
2427 2428 2429
}

/**
2430
 * bnx2x_set_pbd_gso - update PBD in GSO case.
Dmitry Kravkov's avatar
Dmitry Kravkov committed
2431
 *
2432 2433 2434
 * @skb:	packet skb
 * @pbd:	parse BD
 * @xmit_type:	xmit flags
Dmitry Kravkov's avatar
Dmitry Kravkov committed
2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458
 */
static inline void bnx2x_set_pbd_gso(struct sk_buff *skb,
				     struct eth_tx_parse_bd_e1x *pbd,
				     u32 xmit_type)
{
	pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
	pbd->tcp_send_seq = swab32(tcp_hdr(skb)->seq);
	pbd->tcp_flags = pbd_tcp_flags(skb);

	if (xmit_type & XMIT_GSO_V4) {
		pbd->ip_id = swab16(ip_hdr(skb)->id);
		pbd->tcp_pseudo_csum =
			swab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
						  ip_hdr(skb)->daddr,
						  0, IPPROTO_TCP, 0));

	} else
		pbd->tcp_pseudo_csum =
			swab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
						&ipv6_hdr(skb)->daddr,
						0, IPPROTO_TCP, 0));

	pbd->global_data |= ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN;
}
Dmitry Kravkov's avatar
Dmitry Kravkov committed
2459

Dmitry Kravkov's avatar
Dmitry Kravkov committed
2460
/**
2461
 * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
Dmitry Kravkov's avatar
Dmitry Kravkov committed
2462
 *
2463 2464 2465 2466
 * @bp:			driver handle
 * @skb:		packet skb
 * @parsing_data:	data to be updated
 * @xmit_type:		xmit flags
Dmitry Kravkov's avatar
Dmitry Kravkov committed
2467
 *
2468
 * 57712 related
Dmitry Kravkov's avatar
Dmitry Kravkov committed
2469 2470
 */
static inline  u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
2471
	u32 *parsing_data, u32 xmit_type)
Dmitry Kravkov's avatar
Dmitry Kravkov committed
2472
{
2473 2474 2475 2476
	*parsing_data |=
			((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
			ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W_SHIFT) &
			ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W;
Dmitry Kravkov's avatar
Dmitry Kravkov committed
2477

2478 2479 2480 2481
	if (xmit_type & XMIT_CSUM_TCP) {
		*parsing_data |= ((tcp_hdrlen(skb) / 4) <<
			ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
			ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
Dmitry Kravkov's avatar
Dmitry Kravkov committed
2482

2483 2484 2485 2486 2487 2488 2489
		return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
	} else
		/* We support checksum offload for TCP and UDP only.
		 * No need to pass the UDP header length - it's a constant.
		 */
		return skb_transport_header(skb) +
				sizeof(struct udphdr) - skb->data;
Dmitry Kravkov's avatar
Dmitry Kravkov committed
2490 2491
}

2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507
static inline void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,
	struct eth_tx_start_bd *tx_start_bd, u32 xmit_type)
{
	tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;

	if (xmit_type & XMIT_CSUM_V4)
		tx_start_bd->bd_flags.as_bitfield |=
					ETH_TX_BD_FLAGS_IP_CSUM;
	else
		tx_start_bd->bd_flags.as_bitfield |=
					ETH_TX_BD_FLAGS_IPV6;

	if (!(xmit_type & XMIT_CSUM_TCP))
		tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
}

Dmitry Kravkov's avatar
Dmitry Kravkov committed
2508
/**
2509
 * bnx2x_set_pbd_csum - update PBD with checksum and return header length
Dmitry Kravkov's avatar
Dmitry Kravkov committed
2510
 *
2511 2512 2513 2514
 * @bp:		driver handle
 * @skb:	packet skb
 * @pbd:	parse BD to be updated
 * @xmit_type:	xmit flags
Dmitry Kravkov's avatar
Dmitry Kravkov committed
2515 2516 2517 2518 2519
 */
static inline u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
	struct eth_tx_parse_bd_e1x *pbd,
	u32 xmit_type)
{
2520
	u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
Dmitry Kravkov's avatar
Dmitry Kravkov committed
2521 2522 2523 2524 2525 2526 2527

	/* for now NS flag is not used in Linux */
	pbd->global_data =
		(hlen | ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
			 ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));

	pbd->ip_hlen_w = (skb_transport_header(skb) -
2528
			skb_network_header(skb)) >> 1;
Dmitry Kravkov's avatar
Dmitry Kravkov committed
2529

2530 2531 2532 2533 2534 2535 2536
	hlen += pbd->ip_hlen_w;

	/* We support checksum offload for TCP and UDP only */
	if (xmit_type & XMIT_CSUM_TCP)
		hlen += tcp_hdrlen(skb) / 2;
	else
		hlen += sizeof(struct udphdr) / 2;
Dmitry Kravkov's avatar
Dmitry Kravkov committed
2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561

	pbd->total_hlen_w = cpu_to_le16(hlen);
	hlen = hlen*2;

	if (xmit_type & XMIT_CSUM_TCP) {
		pbd->tcp_pseudo_csum = swab16(tcp_hdr(skb)->check);

	} else {
		s8 fix = SKB_CS_OFF(skb); /* signed! */

		DP(NETIF_MSG_TX_QUEUED,
		   "hlen %d  fix %d  csum before fix %x\n",
		   le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));

		/* HW bug: fixup the CSUM */
		pbd->tcp_pseudo_csum =
			bnx2x_csum_fix(skb_transport_header(skb),
				       SKB_CS(skb), fix);

		DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
		   pbd->tcp_pseudo_csum);
	}

	return hlen;
}
Dmitry Kravkov's avatar
Dmitry Kravkov committed
2562

2563 2564 2565 2566 2567 2568 2569
/* called with netif_tx_lock
 * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
 * netif_wake_queue()
 */
netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
{
	struct bnx2x *bp = netdev_priv(dev);
2570

2571 2572
	struct bnx2x_fastpath *fp;
	struct netdev_queue *txq;
2573
	struct bnx2x_fp_txdata *txdata;
2574
	struct sw_tx_bd *tx_buf;
2575
	struct eth_tx_start_bd *tx_start_bd, *first_bd;
2576
	struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
2577
	struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
Dmitry Kravkov's avatar
Dmitry Kravkov committed
2578
	struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
2579
	u32 pbd_e2_parsing_data = 0;
2580
	u16 pkt_prod, bd_prod;
2581
	int nbd, txq_index, fp_index, txdata_index;
2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594
	dma_addr_t mapping;
	u32 xmit_type = bnx2x_xmit_type(bp, skb);
	int i;
	u8 hlen = 0;
	__le16 pkt_size = 0;
	struct ethhdr *eth;
	u8 mac_type = UNICAST_ADDRESS;

#ifdef BNX2X_STOP_ON_ERROR
	if (unlikely(bp->panic))
		return NETDEV_TX_BUSY;
#endif

2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617
	txq_index = skb_get_queue_mapping(skb);
	txq = netdev_get_tx_queue(dev, txq_index);

	BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + FCOE_PRESENT);

	/* decode the fastpath index and the cos index from the txq */
	fp_index = TXQ_TO_FP(txq_index);
	txdata_index = TXQ_TO_COS(txq_index);

#ifdef BCM_CNIC
	/*
	 * Override the above for the FCoE queue:
	 *   - FCoE fp entry is right after the ETH entries.
	 *   - FCoE L2 queue uses bp->txdata[0] only.
	 */
	if (unlikely(!NO_FCOE(bp) && (txq_index ==
				      bnx2x_fcoe_tx(bp, txq_index)))) {
		fp_index = FCOE_IDX;
		txdata_index = 0;
	}
#endif

	/* enable this debug print to view the transmission queue being used
2618
	DP(BNX2X_MSG_FP, "indices: txq %d, fp %d, txdata %d\n",
2619
	   txq_index, fp_index, txdata_index); */
2620

2621
	/* locate the fastpath and the txdata */
2622
	fp = &bp->fp[fp_index];
2623 2624 2625 2626
	txdata = &fp->txdata[txdata_index];

	/* enable this debug print to view the tranmission details
	DP(BNX2X_MSG_FP,"transmitting packet cid %d fp index %d txdata_index %d"
2627
			" tx_data ptr %p fp pointer %p\n",
2628
	   txdata->cid, fp_index, txdata_index, txdata, fp); */
2629

2630 2631
	if (unlikely(bnx2x_tx_avail(bp, txdata) <
		     (skb_shinfo(skb)->nr_frags + 3))) {
2632 2633 2634 2635 2636 2637
		fp->eth_q_stats.driver_xoff++;
		netif_tx_stop_queue(txq);
		BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
		return NETDEV_TX_BUSY;
	}

Dmitry Kravkov's avatar
Dmitry Kravkov committed
2638 2639
	DP(NETIF_MSG_TX_QUEUED, "queue[%d]: SKB: summed %x  protocol %x  "
				"protocol(%x,%x) gso type %x  xmit_type %x\n",
2640
	   txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667
	   ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type);

	eth = (struct ethhdr *)skb->data;

	/* set flag according to packet type (UNICAST_ADDRESS is default)*/
	if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
		if (is_broadcast_ether_addr(eth->h_dest))
			mac_type = BROADCAST_ADDRESS;
		else
			mac_type = MULTICAST_ADDRESS;
	}

#if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
	/* First, check if we need to linearize the skb (due to FW
	   restrictions). No need to check fragmentation if page size > 8K
	   (there will be no violation to FW restrictions) */
	if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
		/* Statistics of linearization */
		bp->lin_cnt++;
		if (skb_linearize(skb) != 0) {
			DP(NETIF_MSG_TX_QUEUED, "SKB linearization failed - "
			   "silently dropping this SKB\n");
			dev_kfree_skb_any(skb);
			return NETDEV_TX_OK;
		}
	}
#endif
2668 2669 2670 2671 2672 2673 2674 2675 2676
	/* Map skb linear data for DMA */
	mapping = dma_map_single(&bp->pdev->dev, skb->data,
				 skb_headlen(skb), DMA_TO_DEVICE);
	if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
		DP(NETIF_MSG_TX_QUEUED, "SKB mapping failed - "
		   "silently dropping this SKB\n");
		dev_kfree_skb_any(skb);
		return NETDEV_TX_OK;
	}
2677 2678 2679 2680 2681 2682 2683 2684 2685
	/*
	Please read carefully. First we use one BD which we mark as start,
	then we have a parsing info BD (used for TSO or xsum),
	and only then we have the rest of the TSO BDs.
	(don't forget to mark the last one as last,
	and to unmap only AFTER you write to the BD ...)
	And above all, all pdb sizes are in words - NOT DWORDS!
	*/

2686 2687 2688
	/* get current pkt produced now - advance it just before sending packet
	 * since mapping of pages may fail and cause packet to be dropped
	 */
2689 2690
	pkt_prod = txdata->tx_pkt_prod;
	bd_prod = TX_BD(txdata->tx_bd_prod);
2691

2692 2693 2694 2695
	/* get a tx_buf and first BD
	 * tx_start_bd may be changed during SPLIT,
	 * but first_bd will always stay first
	 */
2696 2697
	tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)];
	tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd;
2698
	first_bd = tx_start_bd;
2699 2700

	tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
Dmitry Kravkov's avatar
Dmitry Kravkov committed
2701 2702 2703
	SET_FLAG(tx_start_bd->general_data, ETH_TX_START_BD_ETH_ADDR_TYPE,
		 mac_type);

2704
	/* header nbd */
Dmitry Kravkov's avatar
Dmitry Kravkov committed
2705
	SET_FLAG(tx_start_bd->general_data, ETH_TX_START_BD_HDR_NBDS, 1);
2706 2707

	/* remember the first BD of the packet */
2708
	tx_buf->first_bd = txdata->tx_bd_prod;
2709 2710 2711 2712 2713
	tx_buf->skb = skb;
	tx_buf->flags = 0;

	DP(NETIF_MSG_TX_QUEUED,
	   "sending pkt %u @%p  next_idx %u  bd %u @%p\n",
2714
	   pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd);
2715

2716
	if (vlan_tx_tag_present(skb)) {
2717 2718 2719 2720
		tx_start_bd->vlan_or_ethertype =
		    cpu_to_le16(vlan_tx_tag_get(skb));
		tx_start_bd->bd_flags.as_bitfield |=
		    (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
2721
	} else
2722
		tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
2723 2724 2725 2726

	/* turn on parsing and get a BD */
	bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));

2727 2728
	if (xmit_type & XMIT_CSUM)
		bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type);
2729

2730
	if (!CHIP_IS_E1x(bp)) {
2731
		pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2;
Dmitry Kravkov's avatar
Dmitry Kravkov committed
2732 2733 2734
		memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
		/* Set PBD in checksum offload case */
		if (xmit_type & XMIT_CSUM)
2735 2736 2737
			hlen = bnx2x_set_pbd_csum_e2(bp, skb,
						     &pbd_e2_parsing_data,
						     xmit_type);
2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751
		if (IS_MF_SI(bp)) {
			/*
			 * fill in the MAC addresses in the PBD - for local
			 * switching
			 */
			bnx2x_set_fw_mac_addr(&pbd_e2->src_mac_addr_hi,
					      &pbd_e2->src_mac_addr_mid,
					      &pbd_e2->src_mac_addr_lo,
					      eth->h_source);
			bnx2x_set_fw_mac_addr(&pbd_e2->dst_mac_addr_hi,
					      &pbd_e2->dst_mac_addr_mid,
					      &pbd_e2->dst_mac_addr_lo,
					      eth->h_dest);
		}
Dmitry Kravkov's avatar
Dmitry Kravkov committed
2752
	} else {
2753
		pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x;
Dmitry Kravkov's avatar
Dmitry Kravkov committed
2754 2755 2756 2757
		memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
		/* Set PBD in checksum offload case */
		if (xmit_type & XMIT_CSUM)
			hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
2758 2759 2760

	}

Dmitry Kravkov's avatar
Dmitry Kravkov committed
2761
	/* Setup the data pointer of the first BD of the packet */
2762 2763
	tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
	tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2764
	nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */
2765 2766 2767 2768 2769 2770 2771
	tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
	pkt_size = tx_start_bd->nbytes;

	DP(NETIF_MSG_TX_QUEUED, "first bd @%p  addr (%x:%x)  nbd %d"
	   "  nbytes %d  flags %x  vlan %x\n",
	   tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
	   le16_to_cpu(tx_start_bd->nbd), le16_to_cpu(tx_start_bd->nbytes),
2772 2773
	   tx_start_bd->bd_flags.as_bitfield,
	   le16_to_cpu(tx_start_bd->vlan_or_ethertype));
2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784

	if (xmit_type & XMIT_GSO) {

		DP(NETIF_MSG_TX_QUEUED,
		   "TSO packet len %d  hlen %d  total len %d  tso size %d\n",
		   skb->len, hlen, skb_headlen(skb),
		   skb_shinfo(skb)->gso_size);

		tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;

		if (unlikely(skb_headlen(skb) > hlen))
2785 2786 2787
			bd_prod = bnx2x_tx_split(bp, txdata, tx_buf,
						 &tx_start_bd, hlen,
						 bd_prod, ++nbd);
2788
		if (!CHIP_IS_E1x(bp))
2789 2790
			bnx2x_set_pbd_gso_e2(skb, &pbd_e2_parsing_data,
					     xmit_type);
Dmitry Kravkov's avatar
Dmitry Kravkov committed
2791 2792
		else
			bnx2x_set_pbd_gso(skb, pbd_e1x, xmit_type);
2793
	}
2794 2795 2796 2797 2798 2799 2800

	/* Set the PBD's parsing_data field if not zero
	 * (for the chips newer than 57711).
	 */
	if (pbd_e2_parsing_data)
		pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);

2801 2802
	tx_data_bd = (struct eth_tx_bd *)tx_start_bd;

Dmitry Kravkov's avatar
Dmitry Kravkov committed
2803
	/* Handle fragmented skb */
2804 2805 2806
	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];

2807 2808
		mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 0,
					   skb_frag_size(frag), DMA_TO_DEVICE);
2809
		if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
2810
			unsigned int pkts_compl = 0, bytes_compl = 0;
2811 2812 2813 2814 2815 2816 2817 2818 2819 2820

			DP(NETIF_MSG_TX_QUEUED, "Unable to map page - "
						"dropping packet...\n");

			/* we need unmap all buffers already mapped
			 * for this SKB;
			 * first_bd->nbd need to be properly updated
			 * before call to bnx2x_free_tx_pkt
			 */
			first_bd->nbd = cpu_to_le16(nbd);
2821
			bnx2x_free_tx_pkt(bp, txdata,
2822 2823
					  TX_BD(txdata->tx_pkt_prod),
					  &pkts_compl, &bytes_compl);
2824 2825 2826
			return NETDEV_TX_OK;
		}

2827
		bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2828
		tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
2829
		if (total_pkt_bd == NULL)
2830
			total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
2831 2832 2833

		tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
		tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2834 2835
		tx_data_bd->nbytes = cpu_to_le16(skb_frag_size(frag));
		le16_add_cpu(&pkt_size, skb_frag_size(frag));
2836
		nbd++;
2837 2838 2839 2840 2841 2842 2843 2844 2845

		DP(NETIF_MSG_TX_QUEUED,
		   "frag %d  bd @%p  addr (%x:%x)  nbytes %d\n",
		   i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
		   le16_to_cpu(tx_data_bd->nbytes));
	}

	DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);

2846 2847 2848
	/* update with actual num BDs */
	first_bd->nbd = cpu_to_le16(nbd);

2849 2850 2851 2852 2853 2854 2855 2856
	bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));

	/* now send a tx doorbell, counting the next BD
	 * if the packet contains or ends with it
	 */
	if (TX_BD_POFF(bd_prod) < nbd)
		nbd++;

2857 2858 2859 2860 2861 2862 2863
	/* total_pkt_bytes should be set on the first data BD if
	 * it's not an LSO packet and there is more than one
	 * data BD. In this case pkt_size is limited by an MTU value.
	 * However we prefer to set it for an LSO packet (while we don't
	 * have to) in order to save some CPU cycles in a none-LSO
	 * case, when we much more care about them.
	 */
2864 2865 2866
	if (total_pkt_bd != NULL)
		total_pkt_bd->total_pkt_bytes = pkt_size;

2867
	if (pbd_e1x)
2868
		DP(NETIF_MSG_TX_QUEUED,
2869
		   "PBD (E1X) @%p  ip_data %x  ip_hlen %u  ip_id %u  lso_mss %u"
2870
		   "  tcp_flags %x  xsum %x  seq %u  hlen %u\n",
2871 2872 2873 2874
		   pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
		   pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
		   pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
		    le16_to_cpu(pbd_e1x->total_hlen_w));
Dmitry Kravkov's avatar
Dmitry Kravkov committed
2875 2876 2877 2878 2879 2880 2881
	if (pbd_e2)
		DP(NETIF_MSG_TX_QUEUED,
		   "PBD (E2) @%p  dst %x %x %x src %x %x %x parsing_data %x\n",
		   pbd_e2, pbd_e2->dst_mac_addr_hi, pbd_e2->dst_mac_addr_mid,
		   pbd_e2->dst_mac_addr_lo, pbd_e2->src_mac_addr_hi,
		   pbd_e2->src_mac_addr_mid, pbd_e2->src_mac_addr_lo,
		   pbd_e2->parsing_data);
2882 2883
	DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d  bd %u\n", nbd, bd_prod);

2884 2885
	netdev_tx_sent_queue(txq, skb->len);

2886
	txdata->tx_pkt_prod++;
2887 2888 2889 2890 2891 2892 2893 2894 2895
	/*
	 * Make sure that the BD data is updated before updating the producer
	 * since FW might read the BD right after the producer is updated.
	 * This is only applicable for weak-ordered memory model archs such
	 * as IA-64. The following barrier is also mandatory since FW will
	 * assumes packets must have BDs.
	 */
	wmb();

2896
	txdata->tx_db.data.prod += nbd;
2897
	barrier();
Dmitry Kravkov's avatar
Dmitry Kravkov committed
2898

2899
	DOORBELL(bp, txdata->cid, txdata->tx_db.raw);
2900 2901 2902

	mmiowb();

2903
	txdata->tx_bd_prod += nbd;
2904

2905
	if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_SKB_FRAGS + 3)) {
2906 2907 2908 2909 2910 2911 2912 2913
		netif_tx_stop_queue(txq);

		/* paired memory barrier is in bnx2x_tx_int(), we have to keep
		 * ordering of set_bit() in netif_tx_stop_queue() and read of
		 * fp->bd_tx_cons */
		smp_mb();

		fp->eth_q_stats.driver_xoff++;
2914
		if (bnx2x_tx_avail(bp, txdata) >= MAX_SKB_FRAGS + 3)
2915 2916
			netif_tx_wake_queue(txq);
	}
2917
	txdata->tx_pkt++;
2918 2919 2920

	return NETDEV_TX_OK;
}
Dmitry Kravkov's avatar
Dmitry Kravkov committed
2921

2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946
/**
 * bnx2x_setup_tc - routine to configure net_device for multi tc
 *
 * @netdev: net device to configure
 * @tc: number of traffic classes to enable
 *
 * callback connected to the ndo_setup_tc function pointer
 */
int bnx2x_setup_tc(struct net_device *dev, u8 num_tc)
{
	int cos, prio, count, offset;
	struct bnx2x *bp = netdev_priv(dev);

	/* setup tc must be called under rtnl lock */
	ASSERT_RTNL();

	/* no traffic classes requested. aborting */
	if (!num_tc) {
		netdev_reset_tc(dev);
		return 0;
	}

	/* requested to support too many traffic classes */
	if (num_tc > bp->max_cos) {
		DP(NETIF_MSG_TX_ERR, "support for too many traffic classes"
2947
				     " requested: %d. max supported is %d\n",
2948 2949 2950 2951 2952 2953
				     num_tc, bp->max_cos);
		return -EINVAL;
	}

	/* declare amount of supported traffic classes */
	if (netdev_set_num_tc(dev, num_tc)) {
2954
		DP(NETIF_MSG_TX_ERR, "failed to declare %d traffic classes\n",
2955 2956 2957 2958 2959 2960 2961
				     num_tc);
		return -EINVAL;
	}

	/* configure priority to traffic class mapping */
	for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) {
		netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[prio]);
2962
		DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n",
2963 2964 2965 2966 2967 2968 2969 2970
		   prio, bp->prio_to_cos[prio]);
	}


	/* Use this configuration to diffrentiate tc0 from other COSes
	   This can be used for ets or pfc, and save the effort of setting
	   up a multio class queue disc or negotiating DCBX with a switch
	netdev_set_prio_tc_map(dev, 0, 0);
2971
	DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", 0, 0);
2972 2973
	for (prio = 1; prio < 16; prio++) {
		netdev_set_prio_tc_map(dev, prio, 1);
2974
		DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", prio, 1);
2975 2976 2977 2978 2979 2980 2981
	} */

	/* configure traffic class to transmission queue mapping */
	for (cos = 0; cos < bp->max_cos; cos++) {
		count = BNX2X_NUM_ETH_QUEUES(bp);
		offset = cos * MAX_TXQS_PER_COS;
		netdev_set_tc_queue(dev, cos, count, offset);
2982
		DP(BNX2X_MSG_SP, "mapping tc %d to offset %d count %d\n",
2983 2984 2985 2986 2987 2988
		   cos, offset, count);
	}

	return 0;
}

2989 2990 2991 2992 2993
/* called with rtnl_lock */
int bnx2x_change_mac_addr(struct net_device *dev, void *p)
{
	struct sockaddr *addr = p;
	struct bnx2x *bp = netdev_priv(dev);
2994
	int rc = 0;
2995

Dmitry Kravkov's avatar
Dmitry Kravkov committed
2996 2997 2998 2999 3000
	if (!bnx2x_is_valid_ether_addr(bp, addr->sa_data))
		return -EINVAL;

#ifdef BCM_CNIC
	if (IS_MF_ISCSI_SD(bp) && !is_zero_ether_addr(addr->sa_data))
3001
		return -EINVAL;
Dmitry Kravkov's avatar
Dmitry Kravkov committed
3002
#endif
3003

3004 3005 3006 3007 3008 3009
	if (netif_running(dev))  {
		rc = bnx2x_set_eth_mac(bp, false);
		if (rc)
			return rc;
	}

3010
	dev->addr_assign_type &= ~NET_ADDR_RANDOM;
3011
	memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
3012

3013
	if (netif_running(dev))
3014
		rc = bnx2x_set_eth_mac(bp, true);
3015

3016
	return rc;
3017 3018
}

3019 3020 3021 3022
static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index)
{
	union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk);
	struct bnx2x_fastpath *fp = &bp->fp[fp_index];
3023
	u8 cos;
3024 3025 3026 3027 3028 3029 3030 3031 3032 3033

	/* Common */
#ifdef BCM_CNIC
	if (IS_FCOE_IDX(fp_index)) {
		memset(sb, 0, sizeof(union host_hc_status_block));
		fp->status_blk_mapping = 0;

	} else {
#endif
		/* status blocks */
3034
		if (!CHIP_IS_E1x(bp))
3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071
			BNX2X_PCI_FREE(sb->e2_sb,
				       bnx2x_fp(bp, fp_index,
						status_blk_mapping),
				       sizeof(struct host_hc_status_block_e2));
		else
			BNX2X_PCI_FREE(sb->e1x_sb,
				       bnx2x_fp(bp, fp_index,
						status_blk_mapping),
				       sizeof(struct host_hc_status_block_e1x));
#ifdef BCM_CNIC
	}
#endif
	/* Rx */
	if (!skip_rx_queue(bp, fp_index)) {
		bnx2x_free_rx_bds(fp);

		/* fastpath rx rings: rx_buf rx_desc rx_comp */
		BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring));
		BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring),
			       bnx2x_fp(bp, fp_index, rx_desc_mapping),
			       sizeof(struct eth_rx_bd) * NUM_RX_BD);

		BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring),
			       bnx2x_fp(bp, fp_index, rx_comp_mapping),
			       sizeof(struct eth_fast_path_rx_cqe) *
			       NUM_RCQ_BD);

		/* SGE ring */
		BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring));
		BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring),
			       bnx2x_fp(bp, fp_index, rx_sge_mapping),
			       BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
	}

	/* Tx */
	if (!skip_tx_queue(bp, fp_index)) {
		/* fastpath tx rings: tx_buf tx_desc */
3072 3073 3074 3075
		for_each_cos_in_tx_queue(fp, cos) {
			struct bnx2x_fp_txdata *txdata = &fp->txdata[cos];

			DP(BNX2X_MSG_SP,
3076
			   "freeing tx memory of fp %d cos %d cid %d\n",
3077 3078 3079 3080 3081 3082 3083
			   fp_index, cos, txdata->cid);

			BNX2X_FREE(txdata->tx_buf_ring);
			BNX2X_PCI_FREE(txdata->tx_desc_ring,
				txdata->tx_desc_mapping,
				sizeof(union eth_tx_bd_types) * NUM_TX_BD);
		}
3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097
	}
	/* end of fastpath */
}

void bnx2x_free_fp_mem(struct bnx2x *bp)
{
	int i;
	for_each_queue(bp, i)
		bnx2x_free_fp_mem_at(bp, i);
}

static inline void set_sb_shortcuts(struct bnx2x *bp, int index)
{
	union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk);
3098
	if (!CHIP_IS_E1x(bp)) {
3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115
		bnx2x_fp(bp, index, sb_index_values) =
			(__le16 *)status_blk.e2_sb->sb.index_values;
		bnx2x_fp(bp, index, sb_running_index) =
			(__le16 *)status_blk.e2_sb->sb.running_index;
	} else {
		bnx2x_fp(bp, index, sb_index_values) =
			(__le16 *)status_blk.e1x_sb->sb.index_values;
		bnx2x_fp(bp, index, sb_running_index) =
			(__le16 *)status_blk.e1x_sb->sb.running_index;
	}
}

static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index)
{
	union host_hc_status_block *sb;
	struct bnx2x_fastpath *fp = &bp->fp[index];
	int ring_size = 0;
3116
	u8 cos;
3117
	int rx_ring_size = 0;
3118

Dmitry Kravkov's avatar
Dmitry Kravkov committed
3119
#ifdef BCM_CNIC
3120
	if (!bp->rx_ring_size && IS_MF_ISCSI_SD(bp)) {
Dmitry Kravkov's avatar
Dmitry Kravkov committed
3121 3122 3123 3124
		rx_ring_size = MIN_RX_SIZE_NONTPA;
		bp->rx_ring_size = rx_ring_size;
	} else
#endif
3125
	if (!bp->rx_ring_size) {
3126 3127
		u32 cfg = SHMEM_RD(bp,
			     dev_info.port_hw_config[BP_PORT(bp)].default_cfg);
3128

3129 3130
		rx_ring_size = MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp);

3131 3132 3133 3134 3135
		/* Dercease ring size for 1G functions */
		if ((cfg & PORT_HW_CFG_NET_SERDES_IF_MASK) ==
		    PORT_HW_CFG_NET_SERDES_IF_SGMII)
			rx_ring_size /= 10;

3136 3137 3138 3139 3140
		/* allocate at least number of buffers required by FW */
		rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA :
				     MIN_RX_SIZE_TPA, rx_ring_size);

		bp->rx_ring_size = rx_ring_size;
Dmitry Kravkov's avatar
Dmitry Kravkov committed
3141
	} else /* if rx_ring_size specified - use it */
3142
		rx_ring_size = bp->rx_ring_size;
3143 3144 3145 3146 3147 3148 3149

	/* Common */
	sb = &bnx2x_fp(bp, index, status_blk);
#ifdef BCM_CNIC
	if (!IS_FCOE_IDX(index)) {
#endif
		/* status blocks */
3150
		if (!CHIP_IS_E1x(bp))
3151 3152 3153 3154 3155 3156 3157 3158 3159 3160
			BNX2X_PCI_ALLOC(sb->e2_sb,
				&bnx2x_fp(bp, index, status_blk_mapping),
				sizeof(struct host_hc_status_block_e2));
		else
			BNX2X_PCI_ALLOC(sb->e1x_sb,
				&bnx2x_fp(bp, index, status_blk_mapping),
			    sizeof(struct host_hc_status_block_e1x));
#ifdef BCM_CNIC
	}
#endif
3161 3162 3163 3164 3165 3166

	/* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to
	 * set shortcuts for it.
	 */
	if (!IS_FCOE_IDX(index))
		set_sb_shortcuts(bp, index);
3167 3168 3169 3170

	/* Tx */
	if (!skip_tx_queue(bp, index)) {
		/* fastpath tx rings: tx_buf tx_desc */
3171 3172 3173 3174
		for_each_cos_in_tx_queue(fp, cos) {
			struct bnx2x_fp_txdata *txdata = &fp->txdata[cos];

			DP(BNX2X_MSG_SP, "allocating tx memory of "
3175
					 "fp %d cos %d\n",
3176 3177 3178
			   index, cos);

			BNX2X_ALLOC(txdata->tx_buf_ring,
3179
				sizeof(struct sw_tx_bd) * NUM_TX_BD);
3180 3181
			BNX2X_PCI_ALLOC(txdata->tx_desc_ring,
				&txdata->tx_desc_mapping,
3182
				sizeof(union eth_tx_bd_types) * NUM_TX_BD);
3183
		}
3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225
	}

	/* Rx */
	if (!skip_rx_queue(bp, index)) {
		/* fastpath rx rings: rx_buf rx_desc rx_comp */
		BNX2X_ALLOC(bnx2x_fp(bp, index, rx_buf_ring),
				sizeof(struct sw_rx_bd) * NUM_RX_BD);
		BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_desc_ring),
				&bnx2x_fp(bp, index, rx_desc_mapping),
				sizeof(struct eth_rx_bd) * NUM_RX_BD);

		BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_comp_ring),
				&bnx2x_fp(bp, index, rx_comp_mapping),
				sizeof(struct eth_fast_path_rx_cqe) *
				NUM_RCQ_BD);

		/* SGE ring */
		BNX2X_ALLOC(bnx2x_fp(bp, index, rx_page_ring),
				sizeof(struct sw_rx_page) * NUM_RX_SGE);
		BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_sge_ring),
				&bnx2x_fp(bp, index, rx_sge_mapping),
				BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
		/* RX BD ring */
		bnx2x_set_next_page_rx_bd(fp);

		/* CQ ring */
		bnx2x_set_next_page_rx_cq(fp);

		/* BDs */
		ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size);
		if (ring_size < rx_ring_size)
			goto alloc_mem_err;
	}

	return 0;

/* handles low memory cases */
alloc_mem_err:
	BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n",
						index, ring_size);
	/* FW will drop all packets if queue is not big enough,
	 * In these cases we disable the queue
3226
	 * Min size is different for OOO, TPA and non-TPA queues
3227 3228
	 */
	if (ring_size < (fp->disable_tpa ?
3229
				MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) {
3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243
			/* release memory allocated for this queue */
			bnx2x_free_fp_mem_at(bp, index);
			return -ENOMEM;
	}
	return 0;
}

int bnx2x_alloc_fp_mem(struct bnx2x *bp)
{
	int i;

	/**
	 * 1. Allocate FP for leading - fatal if error
	 * 2. {CNIC} Allocate FCoE FP - fatal if error
3244 3245
	 * 3. {CNIC} Allocate OOO + FWD - disable OOO if error
	 * 4. Allocate RSS - fix number of queues if error
3246 3247 3248 3249 3250
	 */

	/* leading */
	if (bnx2x_alloc_fp_mem_at(bp, 0))
		return -ENOMEM;
3251

3252
#ifdef BCM_CNIC
3253 3254 3255 3256 3257 3258 3259
	if (!NO_FCOE(bp))
		/* FCoE */
		if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX))
			/* we will fail load process instead of mark
			 * NO_FCOE_FLAG
			 */
			return -ENOMEM;
3260
#endif
3261

3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278
	/* RSS */
	for_each_nondefault_eth_queue(bp, i)
		if (bnx2x_alloc_fp_mem_at(bp, i))
			break;

	/* handle memory failures */
	if (i != BNX2X_NUM_ETH_QUEUES(bp)) {
		int delta = BNX2X_NUM_ETH_QUEUES(bp) - i;

		WARN_ON(delta < 0);
#ifdef BCM_CNIC
		/**
		 * move non eth FPs next to last eth FP
		 * must be done in that order
		 * FCOE_IDX < FWD_IDX < OOO_IDX
		 */

3279
		/* move FCoE fp even NO_FCOE_FLAG is on */
3280 3281 3282 3283 3284 3285 3286 3287 3288
		bnx2x_move_fp(bp, FCOE_IDX, FCOE_IDX - delta);
#endif
		bp->num_queues -= delta;
		BNX2X_ERR("Adjusted num of queues from %d to %d\n",
			  bp->num_queues + delta, bp->num_queues);
	}

	return 0;
}
3289

3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301
void bnx2x_free_mem_bp(struct bnx2x *bp)
{
	kfree(bp->fp);
	kfree(bp->msix_table);
	kfree(bp->ilt);
}

int __devinit bnx2x_alloc_mem_bp(struct bnx2x *bp)
{
	struct bnx2x_fastpath *fp;
	struct msix_entry *tbl;
	struct bnx2x_ilt *ilt;
3302 3303 3304 3305 3306 3307 3308
	int msix_table_size = 0;

	/*
	 * The biggest MSI-X table we might need is as a maximum number of fast
	 * path IGU SBs plus default SB (for PF).
	 */
	msix_table_size = bp->igu_sb_cnt + 1;
3309

3310
	/* fp array: RSS plus CNIC related L2 queues */
3311
	fp = kcalloc(BNX2X_MAX_RSS_COUNT(bp) + NON_ETH_CONTEXT_USE,
3312
		     sizeof(*fp), GFP_KERNEL);
3313 3314 3315 3316 3317
	if (!fp)
		goto alloc_err;
	bp->fp = fp;

	/* msix table */
3318
	tbl = kcalloc(msix_table_size, sizeof(*tbl), GFP_KERNEL);
3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335
	if (!tbl)
		goto alloc_err;
	bp->msix_table = tbl;

	/* ilt */
	ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
	if (!ilt)
		goto alloc_err;
	bp->ilt = ilt;

	return 0;
alloc_err:
	bnx2x_free_mem_bp(bp);
	return -ENOMEM;

}

3336
int bnx2x_reload_if_running(struct net_device *dev)
3337 3338 3339 3340 3341 3342 3343 3344 3345 3346
{
	struct bnx2x *bp = netdev_priv(dev);

	if (unlikely(!netif_running(dev)))
		return 0;

	bnx2x_nic_unload(bp, UNLOAD_NORMAL);
	return bnx2x_nic_load(bp, LOAD_NORMAL);
}

Yaniv Rosner's avatar
Yaniv Rosner committed
3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395
int bnx2x_get_cur_phy_idx(struct bnx2x *bp)
{
	u32 sel_phy_idx = 0;
	if (bp->link_params.num_phys <= 1)
		return INT_PHY;

	if (bp->link_vars.link_up) {
		sel_phy_idx = EXT_PHY1;
		/* In case link is SERDES, check if the EXT_PHY2 is the one */
		if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) &&
		    (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE))
			sel_phy_idx = EXT_PHY2;
	} else {

		switch (bnx2x_phy_selection(&bp->link_params)) {
		case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT:
		case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY:
		case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY:
		       sel_phy_idx = EXT_PHY1;
		       break;
		case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY:
		case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY:
		       sel_phy_idx = EXT_PHY2;
		       break;
		}
	}

	return sel_phy_idx;

}
int bnx2x_get_link_cfg_idx(struct bnx2x *bp)
{
	u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp);
	/*
	 * The selected actived PHY is always after swapping (in case PHY
	 * swapping is enabled). So when swapping is enabled, we need to reverse
	 * the configuration
	 */

	if (bp->link_params.multi_phy_config &
	    PORT_HW_CFG_PHY_SWAPPED_ENABLED) {
		if (sel_phy_idx == EXT_PHY1)
			sel_phy_idx = EXT_PHY2;
		else if (sel_phy_idx == EXT_PHY2)
			sel_phy_idx = EXT_PHY1;
	}
	return LINK_CONFIG_IDX(sel_phy_idx);
}

3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418
#if defined(NETDEV_FCOE_WWNN) && defined(BCM_CNIC)
int bnx2x_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type)
{
	struct bnx2x *bp = netdev_priv(dev);
	struct cnic_eth_dev *cp = &bp->cnic_eth_dev;

	switch (type) {
	case NETDEV_FCOE_WWNN:
		*wwn = HILO_U64(cp->fcoe_wwn_node_name_hi,
				cp->fcoe_wwn_node_name_lo);
		break;
	case NETDEV_FCOE_WWPN:
		*wwn = HILO_U64(cp->fcoe_wwn_port_name_hi,
				cp->fcoe_wwn_port_name_lo);
		break;
	default:
		return -EINVAL;
	}

	return 0;
}
#endif

3419 3420 3421 3422 3423 3424
/* called with rtnl_lock */
int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
{
	struct bnx2x *bp = netdev_priv(dev);

	if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
Ariel Elior's avatar
Ariel Elior committed
3425
		netdev_err(dev, "Handling parity error recovery. Try again later\n");
3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438
		return -EAGAIN;
	}

	if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
	    ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE))
		return -EINVAL;

	/* This does not race with packet allocation
	 * because the actual alloc size is
	 * only updated as part of load
	 */
	dev->mtu = new_mtu;

3439 3440 3441
	return bnx2x_reload_if_running(dev);
}

3442 3443
netdev_features_t bnx2x_fix_features(struct net_device *dev,
	netdev_features_t features)
3444 3445 3446 3447 3448 3449 3450 3451 3452 3453
{
	struct bnx2x *bp = netdev_priv(dev);

	/* TPA requires Rx CSUM offloading */
	if (!(features & NETIF_F_RXCSUM) || bp->disable_tpa)
		features &= ~NETIF_F_LRO;

	return features;
}

3454
int bnx2x_set_features(struct net_device *dev, netdev_features_t features)
3455 3456 3457
{
	struct bnx2x *bp = netdev_priv(dev);
	u32 flags = bp->flags;
3458
	bool bnx2x_reload = false;
3459 3460 3461 3462 3463 3464

	if (features & NETIF_F_LRO)
		flags |= TPA_ENABLE_FLAG;
	else
		flags &= ~TPA_ENABLE_FLAG;

3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476
	if (features & NETIF_F_LOOPBACK) {
		if (bp->link_params.loopback_mode != LOOPBACK_BMAC) {
			bp->link_params.loopback_mode = LOOPBACK_BMAC;
			bnx2x_reload = true;
		}
	} else {
		if (bp->link_params.loopback_mode != LOOPBACK_NONE) {
			bp->link_params.loopback_mode = LOOPBACK_NONE;
			bnx2x_reload = true;
		}
	}

3477 3478
	if (flags ^ bp->flags) {
		bp->flags = flags;
3479 3480
		bnx2x_reload = true;
	}
3481

3482
	if (bnx2x_reload) {
3483 3484 3485
		if (bp->recovery_state == BNX2X_RECOVERY_DONE)
			return bnx2x_reload_if_running(dev);
		/* else: bnx2x_nic_load() will be called at end of recovery */
3486 3487
	}

3488
	return 0;
3489 3490 3491 3492 3493 3494 3495 3496 3497 3498
}

void bnx2x_tx_timeout(struct net_device *dev)
{
	struct bnx2x *bp = netdev_priv(dev);

#ifdef BNX2X_STOP_ON_ERROR
	if (!bp->panic)
		bnx2x_panic();
#endif
3499 3500 3501 3502 3503

	smp_mb__before_clear_bit();
	set_bit(BNX2X_SP_RTNL_TX_TIMEOUT, &bp->sp_rtnl_state);
	smp_mb__after_clear_bit();

3504
	/* This allows the netif to be shutdown gracefully before resetting */
3505
	schedule_delayed_work(&bp->sp_rtnl_task, 0);
3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551
}

int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
{
	struct net_device *dev = pci_get_drvdata(pdev);
	struct bnx2x *bp;

	if (!dev) {
		dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
		return -ENODEV;
	}
	bp = netdev_priv(dev);

	rtnl_lock();

	pci_save_state(pdev);

	if (!netif_running(dev)) {
		rtnl_unlock();
		return 0;
	}

	netif_device_detach(dev);

	bnx2x_nic_unload(bp, UNLOAD_CLOSE);

	bnx2x_set_power_state(bp, pci_choose_state(pdev, state));

	rtnl_unlock();

	return 0;
}

int bnx2x_resume(struct pci_dev *pdev)
{
	struct net_device *dev = pci_get_drvdata(pdev);
	struct bnx2x *bp;
	int rc;

	if (!dev) {
		dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
		return -ENODEV;
	}
	bp = netdev_priv(dev);

	if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
Ariel Elior's avatar
Ariel Elior committed
3552
		netdev_err(dev, "Handling parity error recovery. Try again later\n");
3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573
		return -EAGAIN;
	}

	rtnl_lock();

	pci_restore_state(pdev);

	if (!netif_running(dev)) {
		rtnl_unlock();
		return 0;
	}

	bnx2x_set_power_state(bp, PCI_D0);
	netif_device_attach(dev);

	rc = bnx2x_nic_load(bp, LOAD_OPEN);

	rtnl_unlock();

	return rc;
}
3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627


void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt,
			      u32 cid)
{
	/* ustorm cxt validation */
	cxt->ustorm_ag_context.cdu_usage =
		CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
			CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE);
	/* xcontext validation */
	cxt->xstorm_ag_context.cdu_reserved =
		CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
			CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE);
}

static inline void storm_memset_hc_timeout(struct bnx2x *bp, u8 port,
					     u8 fw_sb_id, u8 sb_index,
					     u8 ticks)
{

	u32 addr = BAR_CSTRORM_INTMEM +
		   CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index);
	REG_WR8(bp, addr, ticks);
	DP(NETIF_MSG_HW, "port %x fw_sb_id %d sb_index %d ticks %d\n",
			  port, fw_sb_id, sb_index, ticks);
}

static inline void storm_memset_hc_disable(struct bnx2x *bp, u8 port,
					     u16 fw_sb_id, u8 sb_index,
					     u8 disable)
{
	u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT);
	u32 addr = BAR_CSTRORM_INTMEM +
		   CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index);
	u16 flags = REG_RD16(bp, addr);
	/* clear and set */
	flags &= ~HC_INDEX_DATA_HC_ENABLED;
	flags |= enable_flag;
	REG_WR16(bp, addr, flags);
	DP(NETIF_MSG_HW, "port %x fw_sb_id %d sb_index %d disable %d\n",
			  port, fw_sb_id, sb_index, disable);
}

void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id,
				    u8 sb_index, u8 disable, u16 usec)
{
	int port = BP_PORT(bp);
	u8 ticks = usec / BNX2X_BTR;

	storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks);

	disable = disable ? 1 : (usec ? 0 : 1);
	storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable);
}