iwl-trans-pcie.c 55.6 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62
/******************************************************************************
 *
 * This file is provided under a dual BSD/GPLv2 license.  When using or
 * redistributing this file, you may do so under either license.
 *
 * GPL LICENSE SUMMARY
 *
 * Copyright(c) 2007 - 2011 Intel Corporation. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of version 2 of the GNU General Public License as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110,
 * USA
 *
 * The full GNU General Public License is included in this distribution
 * in the file called LICENSE.GPL.
 *
 * Contact Information:
 *  Intel Linux Wireless <ilw@linux.intel.com>
 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
 *
 * BSD LICENSE
 *
 * Copyright(c) 2005 - 2011 Intel Corporation. All rights reserved.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 *  * Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 *  * Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 *  * Neither the name Intel Corporation nor the names of its
 *    contributors may be used to endorse or promote products derived
 *    from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 *****************************************************************************/
63
#include <linux/interrupt.h>
64
#include <linux/debugfs.h>
65 66
#include <linux/bitops.h>
#include <linux/gfp.h>
67

68
#include "iwl-trans.h"
69
#include "iwl-trans-pcie-int.h"
70 71
#include "iwl-csr.h"
#include "iwl-prph.h"
72
#include "iwl-shared.h"
73
#include "iwl-eeprom.h"
74
#include "iwl-agn-hw.h"
75

76
static int iwl_trans_rx_alloc(struct iwl_trans *trans)
77
{
78 79 80 81
	struct iwl_trans_pcie *trans_pcie =
		IWL_TRANS_GET_PCIE_TRANS(trans);
	struct iwl_rx_queue *rxq = &trans_pcie->rxq;
	struct device *dev = bus(trans)->dev;
82

83
	memset(&trans_pcie->rxq, 0, sizeof(trans_pcie->rxq));
84 85 86 87 88 89 90 91 92

	spin_lock_init(&rxq->lock);
	INIT_LIST_HEAD(&rxq->rx_free);
	INIT_LIST_HEAD(&rxq->rx_used);

	if (WARN_ON(rxq->bd || rxq->rb_stts))
		return -EINVAL;

	/* Allocate the circular buffer of Read Buffer Descriptors (RBDs) */
93 94
	rxq->bd = dma_alloc_coherent(dev, sizeof(__le32) * RX_QUEUE_SIZE,
				     &rxq->bd_dma, GFP_KERNEL);
95 96
	if (!rxq->bd)
		goto err_bd;
97
	memset(rxq->bd, 0, sizeof(__le32) * RX_QUEUE_SIZE);
98 99 100 101 102 103 104 105 106 107 108

	/*Allocate the driver's pointer to receive buffer status */
	rxq->rb_stts = dma_alloc_coherent(dev, sizeof(*rxq->rb_stts),
					  &rxq->rb_stts_dma, GFP_KERNEL);
	if (!rxq->rb_stts)
		goto err_rb_stts;
	memset(rxq->rb_stts, 0, sizeof(*rxq->rb_stts));

	return 0;

err_rb_stts:
109 110
	dma_free_coherent(dev, sizeof(__le32) * RX_QUEUE_SIZE,
			rxq->bd, rxq->bd_dma);
111 112 113 114 115 116
	memset(&rxq->bd_dma, 0, sizeof(rxq->bd_dma));
	rxq->bd = NULL;
err_bd:
	return -ENOMEM;
}

117
static void iwl_trans_rxq_free_rx_bufs(struct iwl_trans *trans)
118
{
119 120 121
	struct iwl_trans_pcie *trans_pcie =
		IWL_TRANS_GET_PCIE_TRANS(trans);
	struct iwl_rx_queue *rxq = &trans_pcie->rxq;
122
	int i;
123 124 125 126 127 128

	/* Fill the rx_used queue with _all_ of the Rx buffers */
	for (i = 0; i < RX_FREE_BUFFERS + RX_QUEUE_SIZE; i++) {
		/* In the reset function, these buffers may have been allocated
		 * to an SKB, so we need to unmap and free potential storage */
		if (rxq->pool[i].page != NULL) {
129 130
			dma_unmap_page(bus(trans)->dev, rxq->pool[i].page_dma,
				PAGE_SIZE << hw_params(trans).rx_page_order,
131
				DMA_FROM_DEVICE);
132 133
			__free_pages(rxq->pool[i].page,
				     hw_params(trans).rx_page_order);
134 135 136 137
			rxq->pool[i].page = NULL;
		}
		list_add_tail(&rxq->pool[i].list, &rxq->rx_used);
	}
138 139
}

140
static void iwl_trans_rx_hw_init(struct iwl_trans *trans,
141 142 143 144
				 struct iwl_rx_queue *rxq)
{
	u32 rb_size;
	const u32 rfdnlog = RX_QUEUE_SIZE_LOG; /* 256 RBDs */
145
	u32 rb_timeout = RX_RB_TIMEOUT; /* FIXME: RX_RB_TIMEOUT for all devices? */
146 147 148 149 150 151 152

	if (iwlagn_mod_params.amsdu_size_8K)
		rb_size = FH_RCSR_RX_CONFIG_REG_VAL_RB_SIZE_8K;
	else
		rb_size = FH_RCSR_RX_CONFIG_REG_VAL_RB_SIZE_4K;

	/* Stop Rx DMA */
153
	iwl_write_direct32(bus(trans), FH_MEM_RCSR_CHNL0_CONFIG_REG, 0);
154 155

	/* Reset driver's Rx queue write index */
156
	iwl_write_direct32(bus(trans), FH_RSCSR_CHNL0_RBDCB_WPTR_REG, 0);
157 158

	/* Tell device where to find RBD circular buffer in DRAM */
159
	iwl_write_direct32(bus(trans), FH_RSCSR_CHNL0_RBDCB_BASE_REG,
160 161 162
			   (u32)(rxq->bd_dma >> 8));

	/* Tell device where in DRAM to update its Rx status */
163
	iwl_write_direct32(bus(trans), FH_RSCSR_CHNL0_STTS_WPTR_REG,
164 165 166 167 168 169 170 171 172 173
			   rxq->rb_stts_dma >> 4);

	/* Enable Rx DMA
	 * FH_RCSR_CHNL0_RX_IGNORE_RXF_EMPTY is set because of HW bug in
	 *      the credit mechanism in 5000 HW RX FIFO
	 * Direct rx interrupts to hosts
	 * Rx buffer size 4 or 8k
	 * RB timeout 0x10
	 * 256 RBDs
	 */
174
	iwl_write_direct32(bus(trans), FH_MEM_RCSR_CHNL0_CONFIG_REG,
175 176 177 178 179 180 181 182 183
			   FH_RCSR_RX_CONFIG_CHNL_EN_ENABLE_VAL |
			   FH_RCSR_CHNL0_RX_IGNORE_RXF_EMPTY |
			   FH_RCSR_CHNL0_RX_CONFIG_IRQ_DEST_INT_HOST_VAL |
			   FH_RCSR_CHNL0_RX_CONFIG_SINGLE_FRAME_MSK |
			   rb_size|
			   (rb_timeout << FH_RCSR_RX_CONFIG_REG_IRQ_RBTH_POS)|
			   (rfdnlog << FH_RCSR_RX_CONFIG_RBDCB_SIZE_POS));

	/* Set interrupt coalescing timer to default (2048 usecs) */
184
	iwl_write8(bus(trans), CSR_INT_COALESCING, IWL_HOST_INT_TIMEOUT_DEF);
185 186
}

187
static int iwl_rx_init(struct iwl_trans *trans)
188
{
189 190 191 192
	struct iwl_trans_pcie *trans_pcie =
		IWL_TRANS_GET_PCIE_TRANS(trans);
	struct iwl_rx_queue *rxq = &trans_pcie->rxq;

193 194 195 196
	int i, err;
	unsigned long flags;

	if (!rxq->bd) {
197
		err = iwl_trans_rx_alloc(trans);
198 199 200 201 202 203 204 205
		if (err)
			return err;
	}

	spin_lock_irqsave(&rxq->lock, flags);
	INIT_LIST_HEAD(&rxq->rx_free);
	INIT_LIST_HEAD(&rxq->rx_used);

206
	iwl_trans_rxq_free_rx_bufs(trans);
207 208 209 210 211 212 213 214 215 216 217

	for (i = 0; i < RX_QUEUE_SIZE; i++)
		rxq->queue[i] = NULL;

	/* Set us so that we have processed and used all buffers, but have
	 * not restocked the Rx queue with fresh buffers */
	rxq->read = rxq->write = 0;
	rxq->write_actual = 0;
	rxq->free_count = 0;
	spin_unlock_irqrestore(&rxq->lock, flags);

218
	iwlagn_rx_replenish(trans);
219

220
	iwl_trans_rx_hw_init(trans, rxq);
221

222
	spin_lock_irqsave(&trans->shrd->lock, flags);
223
	rxq->need_update = 1;
224 225
	iwl_rx_queue_update_write_ptr(trans, rxq);
	spin_unlock_irqrestore(&trans->shrd->lock, flags);
226

227 228 229
	return 0;
}

230
static void iwl_trans_pcie_rx_free(struct iwl_trans *trans)
231
{
232 233 234 235
	struct iwl_trans_pcie *trans_pcie =
		IWL_TRANS_GET_PCIE_TRANS(trans);
	struct iwl_rx_queue *rxq = &trans_pcie->rxq;

236 237 238 239 240
	unsigned long flags;

	/*if rxq->bd is NULL, it means that nothing has been allocated,
	 * exit now */
	if (!rxq->bd) {
241
		IWL_DEBUG_INFO(trans, "Free NULL rx context\n");
242 243 244 245
		return;
	}

	spin_lock_irqsave(&rxq->lock, flags);
246
	iwl_trans_rxq_free_rx_bufs(trans);
247 248
	spin_unlock_irqrestore(&rxq->lock, flags);

249
	dma_free_coherent(bus(trans)->dev, sizeof(__le32) * RX_QUEUE_SIZE,
250 251 252 253 254
			  rxq->bd, rxq->bd_dma);
	memset(&rxq->bd_dma, 0, sizeof(rxq->bd_dma));
	rxq->bd = NULL;

	if (rxq->rb_stts)
255
		dma_free_coherent(bus(trans)->dev,
256 257 258
				  sizeof(struct iwl_rb_status),
				  rxq->rb_stts, rxq->rb_stts_dma);
	else
259
		IWL_DEBUG_INFO(trans, "Free rxq->rb_stts which is NULL\n");
260 261 262 263
	memset(&rxq->rb_stts_dma, 0, sizeof(rxq->rb_stts_dma));
	rxq->rb_stts = NULL;
}

264
static int iwl_trans_rx_stop(struct iwl_trans *trans)
265 266 267
{

	/* stop Rx DMA */
268 269
	iwl_write_direct32(bus(trans), FH_MEM_RCSR_CHNL0_CONFIG_REG, 0);
	return iwl_poll_direct_bit(bus(trans), FH_MEM_RSSR_RX_STATUS_REG,
270 271 272
			    FH_RSSR_CHNL0_RX_STATUS_CHNL_IDLE, 1000);
}

273
static inline int iwlagn_alloc_dma_ptr(struct iwl_trans *trans,
274 275 276 277 278
				    struct iwl_dma_ptr *ptr, size_t size)
{
	if (WARN_ON(ptr->addr))
		return -EINVAL;

279
	ptr->addr = dma_alloc_coherent(bus(trans)->dev, size,
280 281 282 283 284 285 286
				       &ptr->dma, GFP_KERNEL);
	if (!ptr->addr)
		return -ENOMEM;
	ptr->size = size;
	return 0;
}

287
static inline void iwlagn_free_dma_ptr(struct iwl_trans *trans,
288 289 290 291 292
				    struct iwl_dma_ptr *ptr)
{
	if (unlikely(!ptr->addr))
		return;

293
	dma_free_coherent(bus(trans)->dev, ptr->size, ptr->addr, ptr->dma);
294 295 296
	memset(ptr, 0, sizeof(*ptr));
}

297 298 299
static int iwl_trans_txq_alloc(struct iwl_trans *trans,
				struct iwl_tx_queue *txq, int slots_num,
				u32 txq_id)
300
{
301
	size_t tfd_sz = sizeof(struct iwl_tfd) * TFD_QUEUE_SIZE_MAX;
302 303
	int i;

304
	if (WARN_ON(txq->meta || txq->cmd || txq->skbs || txq->tfds))
305 306
		return -EINVAL;

307 308
	txq->q.n_window = slots_num;

309 310
	txq->meta = kzalloc(sizeof(txq->meta[0]) * slots_num, GFP_KERNEL);
	txq->cmd = kzalloc(sizeof(txq->cmd[0]) * slots_num, GFP_KERNEL);
311 312 313 314

	if (!txq->meta || !txq->cmd)
		goto error;

315 316 317 318 319 320 321
	if (txq_id == trans->shrd->cmd_queue)
		for (i = 0; i < slots_num; i++) {
			txq->cmd[i] = kmalloc(sizeof(struct iwl_device_cmd),
						GFP_KERNEL);
			if (!txq->cmd[i])
				goto error;
		}
322 323 324 325

	/* Alloc driver data array and TFD circular buffer */
	/* Driver private data, only for Tx (not command) queues,
	 * not shared with device. */
326
	if (txq_id != trans->shrd->cmd_queue) {
327
		txq->skbs = kzalloc(sizeof(txq->skbs[0]) *
328
				   TFD_QUEUE_SIZE_MAX, GFP_KERNEL);
329
		if (!txq->skbs) {
330
			IWL_ERR(trans, "kmalloc for auxiliary BD "
331 332 333 334
				  "structures failed\n");
			goto error;
		}
	} else {
335
		txq->skbs = NULL;
336 337 338 339
	}

	/* Circular buffer of transmit frame descriptors (TFDs),
	 * shared with device */
340 341
	txq->tfds = dma_alloc_coherent(bus(trans)->dev, tfd_sz,
				       &txq->q.dma_addr, GFP_KERNEL);
342
	if (!txq->tfds) {
343
		IWL_ERR(trans, "dma_alloc_coherent(%zd) failed\n", tfd_sz);
344 345 346 347 348 349
		goto error;
	}
	txq->q.id = txq_id;

	return 0;
error:
350 351
	kfree(txq->skbs);
	txq->skbs = NULL;
352 353
	/* since txq->cmd has been zeroed,
	 * all non allocated cmd[i] will be NULL */
354
	if (txq->cmd && txq_id == trans->shrd->cmd_queue)
355 356 357 358 359 360 361 362 363 364 365
		for (i = 0; i < slots_num; i++)
			kfree(txq->cmd[i]);
	kfree(txq->meta);
	kfree(txq->cmd);
	txq->meta = NULL;
	txq->cmd = NULL;

	return -ENOMEM;

}

366
static int iwl_trans_txq_init(struct iwl_trans *trans, struct iwl_tx_queue *txq,
367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386
		      int slots_num, u32 txq_id)
{
	int ret;

	txq->need_update = 0;
	memset(txq->meta, 0, sizeof(txq->meta[0]) * slots_num);

	/*
	 * For the default queues 0-3, set up the swq_id
	 * already -- all others need to get one later
	 * (if they need one at all).
	 */
	if (txq_id < 4)
		iwl_set_swq_id(txq, txq_id, txq_id);

	/* TFD_QUEUE_SIZE_MAX must be power-of-two size, otherwise
	 * iwl_queue_inc_wrap and iwl_queue_dec_wrap are broken. */
	BUILD_BUG_ON(TFD_QUEUE_SIZE_MAX & (TFD_QUEUE_SIZE_MAX - 1));

	/* Initialize queue's high/low-water marks, and head/tail indexes */
387
	ret = iwl_queue_init(&txq->q, TFD_QUEUE_SIZE_MAX, slots_num,
388 389 390 391 392 393 394 395
			txq_id);
	if (ret)
		return ret;

	/*
	 * Tell nic where to find circular buffer of Tx Frame Descriptors for
	 * given Tx queue, and enable the DMA channel used for that queue.
	 * Circular buffer (TFD queue in DRAM) physical base address */
396
	iwl_write_direct32(bus(trans), FH_MEM_CBBC_QUEUE(txq_id),
397 398 399 400 401
			     txq->q.dma_addr >> 8);

	return 0;
}

402 403 404
/**
 * iwl_tx_queue_unmap -  Unmap any remaining DMA mappings and free skb's
 */
405
static void iwl_tx_queue_unmap(struct iwl_trans *trans, int txq_id)
406
{
407 408
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
	struct iwl_tx_queue *txq = &trans_pcie->txq[txq_id];
409
	struct iwl_queue *q = &txq->q;
410
	enum dma_data_direction dma_dir;
411 412 413 414

	if (!q->n_bd)
		return;

415 416 417 418 419 420 421 422
	/* In the command queue, all the TBs are mapped as BIDI
	 * so unmap them as such.
	 */
	if (txq_id == trans->shrd->cmd_queue)
		dma_dir = DMA_BIDIRECTIONAL;
	else
		dma_dir = DMA_TO_DEVICE;

423 424
	while (q->write_ptr != q->read_ptr) {
		/* The read_ptr needs to bound by q->n_window */
425 426
		iwlagn_txq_free_tfd(trans, txq, get_cmd_index(q, q->read_ptr),
				    dma_dir);
427 428 429 430
		q->read_ptr = iwl_queue_inc_wrap(q->read_ptr, q->n_bd);
	}
}

431 432 433 434 435 436 437 438
/**
 * iwl_tx_queue_free - Deallocate DMA queue.
 * @txq: Transmit queue to deallocate.
 *
 * Empty queue by removing and destroying all BD's.
 * Free all buffers.
 * 0-fill, but do not free "txq" descriptor structure.
 */
439
static void iwl_tx_queue_free(struct iwl_trans *trans, int txq_id)
440
{
441 442
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
	struct iwl_tx_queue *txq = &trans_pcie->txq[txq_id];
443
	struct device *dev = bus(trans)->dev;
444 445 446 447
	int i;
	if (WARN_ON(!txq))
		return;

448
	iwl_tx_queue_unmap(trans, txq_id);
449 450

	/* De-alloc array of command/tx buffers */
451 452 453 454

	if (txq_id == trans->shrd->cmd_queue)
		for (i = 0; i < txq->q.n_window; i++)
			kfree(txq->cmd[i]);
455 456 457

	/* De-alloc circular buffer of TFDs */
	if (txq->q.n_bd) {
458
		dma_free_coherent(dev, sizeof(struct iwl_tfd) *
459 460 461 462 463
				  txq->q.n_bd, txq->tfds, txq->q.dma_addr);
		memset(&txq->q.dma_addr, 0, sizeof(txq->q.dma_addr));
	}

	/* De-alloc array of per-TFD driver data */
464 465
	kfree(txq->skbs);
	txq->skbs = NULL;
466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481

	/* deallocate arrays */
	kfree(txq->cmd);
	kfree(txq->meta);
	txq->cmd = NULL;
	txq->meta = NULL;

	/* 0-fill queue descriptor structure */
	memset(txq, 0, sizeof(*txq));
}

/**
 * iwl_trans_tx_free - Free TXQ Context
 *
 * Destroy all TX DMA queues and structures
 */
482
static void iwl_trans_pcie_tx_free(struct iwl_trans *trans)
483 484
{
	int txq_id;
485
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
486 487

	/* Tx queues */
488
	if (trans_pcie->txq) {
489
		for (txq_id = 0;
490 491
		     txq_id < hw_params(trans).max_txq_num; txq_id++)
			iwl_tx_queue_free(trans, txq_id);
492 493
	}

494 495
	kfree(trans_pcie->txq);
	trans_pcie->txq = NULL;
496

497
	iwlagn_free_dma_ptr(trans, &trans_pcie->kw);
498

499
	iwlagn_free_dma_ptr(trans, &trans_pcie->scd_bc_tbls);
500 501
}

502 503 504 505 506 507 508
/**
 * iwl_trans_tx_alloc - allocate TX context
 * Allocate all Tx DMA structures and initialize them
 *
 * @param priv
 * @return error code
 */
509
static int iwl_trans_tx_alloc(struct iwl_trans *trans)
510 511 512
{
	int ret;
	int txq_id, slots_num;
513
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
514

515
	u16 scd_bc_tbls_size = hw_params(trans).max_txq_num *
516 517
			sizeof(struct iwlagn_scd_bc_tbl);

518 519
	/*It is not allowed to alloc twice, so warn when this happens.
	 * We cannot rely on the previous allocation, so free and fail */
520
	if (WARN_ON(trans_pcie->txq)) {
521 522 523 524
		ret = -EINVAL;
		goto error;
	}

525
	ret = iwlagn_alloc_dma_ptr(trans, &trans_pcie->scd_bc_tbls,
526
				   scd_bc_tbls_size);
527
	if (ret) {
528
		IWL_ERR(trans, "Scheduler BC Table allocation failed\n");
529 530 531 532
		goto error;
	}

	/* Alloc keep-warm buffer */
533
	ret = iwlagn_alloc_dma_ptr(trans, &trans_pcie->kw, IWL_KW_SIZE);
534
	if (ret) {
535
		IWL_ERR(trans, "Keep Warm allocation failed\n");
536 537 538
		goto error;
	}

539
	trans_pcie->txq = kzalloc(sizeof(struct iwl_tx_queue) *
540
			hw_params(trans).max_txq_num, GFP_KERNEL);
541
	if (!trans_pcie->txq) {
542
		IWL_ERR(trans, "Not enough memory for txq\n");
543 544 545 546 547
		ret = ENOMEM;
		goto error;
	}

	/* Alloc and init all Tx queues, including the command queue (#4/#9) */
548 549
	for (txq_id = 0; txq_id < hw_params(trans).max_txq_num; txq_id++) {
		slots_num = (txq_id == trans->shrd->cmd_queue) ?
550
					TFD_CMD_SLOTS : TFD_TX_CMD_SLOTS;
551 552
		ret = iwl_trans_txq_alloc(trans, &trans_pcie->txq[txq_id],
					  slots_num, txq_id);
553
		if (ret) {
554
			IWL_ERR(trans, "Tx %d queue alloc failed\n", txq_id);
555 556 557 558 559 560 561
			goto error;
		}
	}

	return 0;

error:
562
	iwl_trans_pcie_tx_free(trans);
563 564 565

	return ret;
}
566
static int iwl_tx_init(struct iwl_trans *trans)
567 568 569 570 571
{
	int ret;
	int txq_id, slots_num;
	unsigned long flags;
	bool alloc = false;
572
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
573

574
	if (!trans_pcie->txq) {
575
		ret = iwl_trans_tx_alloc(trans);
576 577 578 579 580
		if (ret)
			goto error;
		alloc = true;
	}

581
	spin_lock_irqsave(&trans->shrd->lock, flags);
582 583

	/* Turn off all Tx DMA fifos */
584
	iwl_write_prph(bus(trans), SCD_TXFACT, 0);
585 586

	/* Tell NIC where to find the "keep warm" buffer */
587 588
	iwl_write_direct32(bus(trans), FH_KW_MEM_ADDR_REG,
			   trans_pcie->kw.dma >> 4);
589

590
	spin_unlock_irqrestore(&trans->shrd->lock, flags);
591 592

	/* Alloc and init all Tx queues, including the command queue (#4/#9) */
593 594
	for (txq_id = 0; txq_id < hw_params(trans).max_txq_num; txq_id++) {
		slots_num = (txq_id == trans->shrd->cmd_queue) ?
595
					TFD_CMD_SLOTS : TFD_TX_CMD_SLOTS;
596 597
		ret = iwl_trans_txq_init(trans, &trans_pcie->txq[txq_id],
					 slots_num, txq_id);
598
		if (ret) {
599
			IWL_ERR(trans, "Tx %d queue init failed\n", txq_id);
600 601 602 603 604 605 606 607
			goto error;
		}
	}

	return 0;
error:
	/*Upon error, free only if we allocated something */
	if (alloc)
608
		iwl_trans_pcie_tx_free(trans);
609 610 611
	return ret;
}

612
static void iwl_set_pwr_vmain(struct iwl_trans *trans)
613 614 615 616 617 618
{
/*
 * (for documentation purposes)
 * to set power to V_AUX, do:

		if (pci_pme_capable(priv->pci_dev, PCI_D3cold))
619
			iwl_set_bits_mask_prph(bus(trans), APMG_PS_CTRL_REG,
620 621 622 623
					       APMG_PS_CTRL_VAL_PWR_SRC_VAUX,
					       ~APMG_PS_CTRL_MSK_PWR_SRC);
 */

624
	iwl_set_bits_mask_prph(bus(trans), APMG_PS_CTRL_REG,
625 626 627 628
			       APMG_PS_CTRL_VAL_PWR_SRC_VMAIN,
			       ~APMG_PS_CTRL_MSK_PWR_SRC);
}

629
static int iwl_nic_init(struct iwl_trans *trans)
630 631 632 633
{
	unsigned long flags;

	/* nic_init */
634
	spin_lock_irqsave(&trans->shrd->lock, flags);
635
	iwl_apm_init(priv(trans));
636 637

	/* Set interrupt coalescing calibration timer to default (512 usecs) */
638 639
	iwl_write8(bus(trans), CSR_INT_COALESCING,
		IWL_HOST_INT_CALIB_TIMEOUT_DEF);
640

641
	spin_unlock_irqrestore(&trans->shrd->lock, flags);
642

643
	iwl_set_pwr_vmain(trans);
644

645
	iwl_nic_config(priv(trans));
646 647

	/* Allocate the RX queue, or reset if it is already allocated */
648
	iwl_rx_init(trans);
649 650

	/* Allocate or reset and init all Tx and Command queues */
651
	if (iwl_tx_init(trans))
652 653
		return -ENOMEM;

654
	if (hw_params(trans).shadow_reg_enable) {
655
		/* enable shadow regs in HW */
656
		iwl_set_bit(bus(trans), CSR_MAC_SHADOW_REG_CTRL,
657 658 659
			0x800FFFFF);
	}

660
	set_bit(STATUS_INIT, &trans->shrd->status);
661 662 663 664 665 666 667

	return 0;
}

#define HW_READY_TIMEOUT (50)

/* Note: returns poll_bit return value, which is >= 0 if success */
668
static int iwl_set_hw_ready(struct iwl_trans *trans)
669 670 671
{
	int ret;

672
	iwl_set_bit(bus(trans), CSR_HW_IF_CONFIG_REG,
673 674 675
		CSR_HW_IF_CONFIG_REG_BIT_NIC_READY);

	/* See if we got it */
676
	ret = iwl_poll_bit(bus(trans), CSR_HW_IF_CONFIG_REG,
677 678 679 680
				CSR_HW_IF_CONFIG_REG_BIT_NIC_READY,
				CSR_HW_IF_CONFIG_REG_BIT_NIC_READY,
				HW_READY_TIMEOUT);

681
	IWL_DEBUG_INFO(trans, "hardware%s ready\n", ret < 0 ? " not" : "");
682 683 684 685
	return ret;
}

/* Note: returns standard 0/-ERROR code */
686
static int iwl_trans_pcie_prepare_card_hw(struct iwl_trans *trans)
687 688 689
{
	int ret;

690
	IWL_DEBUG_INFO(trans, "iwl_trans_prepare_card_hw enter\n");
691

692
	ret = iwl_set_hw_ready(trans);
693 694 695 696
	if (ret >= 0)
		return 0;

	/* If HW is not ready, prepare the conditions to check again */
697
	iwl_set_bit(bus(trans), CSR_HW_IF_CONFIG_REG,
698 699
			CSR_HW_IF_CONFIG_REG_PREPARE);

700
	ret = iwl_poll_bit(bus(trans), CSR_HW_IF_CONFIG_REG,
701 702 703 704 705 706 707
			~CSR_HW_IF_CONFIG_REG_BIT_NIC_PREPARE_DONE,
			CSR_HW_IF_CONFIG_REG_BIT_NIC_PREPARE_DONE, 150000);

	if (ret < 0)
		return ret;

	/* HW should be ready by now, check again. */
708
	ret = iwl_set_hw_ready(trans);
709 710 711 712 713
	if (ret >= 0)
		return 0;
	return ret;
}

714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766
#define IWL_AC_UNSET -1

struct queue_to_fifo_ac {
	s8 fifo, ac;
};

static const struct queue_to_fifo_ac iwlagn_default_queue_to_tx_fifo[] = {
	{ IWL_TX_FIFO_VO, IEEE80211_AC_VO, },
	{ IWL_TX_FIFO_VI, IEEE80211_AC_VI, },
	{ IWL_TX_FIFO_BE, IEEE80211_AC_BE, },
	{ IWL_TX_FIFO_BK, IEEE80211_AC_BK, },
	{ IWLAGN_CMD_FIFO_NUM, IWL_AC_UNSET, },
	{ IWL_TX_FIFO_UNUSED, IWL_AC_UNSET, },
	{ IWL_TX_FIFO_UNUSED, IWL_AC_UNSET, },
	{ IWL_TX_FIFO_UNUSED, IWL_AC_UNSET, },
	{ IWL_TX_FIFO_UNUSED, IWL_AC_UNSET, },
	{ IWL_TX_FIFO_UNUSED, IWL_AC_UNSET, },
	{ IWL_TX_FIFO_UNUSED, IWL_AC_UNSET, },
};

static const struct queue_to_fifo_ac iwlagn_ipan_queue_to_tx_fifo[] = {
	{ IWL_TX_FIFO_VO, IEEE80211_AC_VO, },
	{ IWL_TX_FIFO_VI, IEEE80211_AC_VI, },
	{ IWL_TX_FIFO_BE, IEEE80211_AC_BE, },
	{ IWL_TX_FIFO_BK, IEEE80211_AC_BK, },
	{ IWL_TX_FIFO_BK_IPAN, IEEE80211_AC_BK, },
	{ IWL_TX_FIFO_BE_IPAN, IEEE80211_AC_BE, },
	{ IWL_TX_FIFO_VI_IPAN, IEEE80211_AC_VI, },
	{ IWL_TX_FIFO_VO_IPAN, IEEE80211_AC_VO, },
	{ IWL_TX_FIFO_BE_IPAN, 2, },
	{ IWLAGN_CMD_FIFO_NUM, IWL_AC_UNSET, },
	{ IWL_TX_FIFO_AUX, IWL_AC_UNSET, },
};

static const u8 iwlagn_bss_ac_to_fifo[] = {
	IWL_TX_FIFO_VO,
	IWL_TX_FIFO_VI,
	IWL_TX_FIFO_BE,
	IWL_TX_FIFO_BK,
};
static const u8 iwlagn_bss_ac_to_queue[] = {
	0, 1, 2, 3,
};
static const u8 iwlagn_pan_ac_to_fifo[] = {
	IWL_TX_FIFO_VO_IPAN,
	IWL_TX_FIFO_VI_IPAN,
	IWL_TX_FIFO_BE_IPAN,
	IWL_TX_FIFO_BK_IPAN,
};
static const u8 iwlagn_pan_ac_to_queue[] = {
	7, 6, 5, 4,
};

767
static int iwl_trans_pcie_start_device(struct iwl_trans *trans)
768 769
{
	int ret;
770 771
	struct iwl_trans_pcie *trans_pcie =
		IWL_TRANS_GET_PCIE_TRANS(trans);
772

773
	trans->shrd->ucode_owner = IWL_OWNERSHIP_DRIVER;
774 775 776 777 778 779 780 781
	trans_pcie->ac_to_queue[IWL_RXON_CTX_BSS] = iwlagn_bss_ac_to_queue;
	trans_pcie->ac_to_queue[IWL_RXON_CTX_PAN] = iwlagn_pan_ac_to_queue;

	trans_pcie->ac_to_fifo[IWL_RXON_CTX_BSS] = iwlagn_bss_ac_to_fifo;
	trans_pcie->ac_to_fifo[IWL_RXON_CTX_PAN] = iwlagn_pan_ac_to_fifo;

	trans_pcie->mcast_queue[IWL_RXON_CTX_BSS] = 0;
	trans_pcie->mcast_queue[IWL_RXON_CTX_PAN] = IWL_IPAN_MCAST_QUEUE;
782

783
	if ((hw_params(trans).sku & EEPROM_SKU_CAP_AMT_ENABLE) &&
784 785
	     iwl_trans_pcie_prepare_card_hw(trans)) {
		IWL_WARN(trans, "Exit HW not ready\n");
786 787 788 789
		return -EIO;
	}

	/* If platform's RF_KILL switch is NOT set to KILL */
790
	if (iwl_read32(bus(trans), CSR_GP_CNTRL) &
791
			CSR_GP_CNTRL_REG_FLAG_HW_RF_KILL_SW)
792
		clear_bit(STATUS_RF_KILL_HW, &trans->shrd->status);
793
	else
794
		set_bit(STATUS_RF_KILL_HW, &trans->shrd->status);
795

796
	if (iwl_is_rfkill(trans->shrd)) {
797
		iwl_set_hw_rfkill_state(priv(trans), true);
798
		iwl_enable_interrupts(trans);
799 800 801
		return -ERFKILL;
	}

802
	iwl_write32(bus(trans), CSR_INT, 0xFFFFFFFF);
803

804
	ret = iwl_nic_init(trans);
805
	if (ret) {
806
		IWL_ERR(trans, "Unable to init nic\n");
807 808 809 810
		return ret;
	}

	/* make sure rfkill handshake bits are cleared */
811 812
	iwl_write32(bus(trans), CSR_UCODE_DRV_GP1_CLR, CSR_UCODE_SW_BIT_RFKILL);
	iwl_write32(bus(trans), CSR_UCODE_DRV_GP1_CLR,
813 814 815
		    CSR_UCODE_DRV_GP1_BIT_CMD_BLOCKED);

	/* clear (again), then enable host interrupts */
816
	iwl_write32(bus(trans), CSR_INT, 0xFFFFFFFF);
817
	iwl_enable_interrupts(trans);
818 819

	/* really make sure rfkill handshake bits are cleared */
820 821
	iwl_write32(bus(trans), CSR_UCODE_DRV_GP1_CLR, CSR_UCODE_SW_BIT_RFKILL);
	iwl_write32(bus(trans), CSR_UCODE_DRV_GP1_CLR, CSR_UCODE_SW_BIT_RFKILL);
822 823 824 825

	return 0;
}

826 827
/*
 * Activate/Deactivate Tx DMA/FIFO channels according tx fifos mask
828
 * must be called under priv->shrd->lock and mac access
829
 */
830
static void iwl_trans_txq_set_sched(struct iwl_trans *trans, u32 mask)
831
{
832
	iwl_write_prph(bus(trans), SCD_TXFACT, mask);
833 834
}

835
static void iwl_trans_pcie_tx_start(struct iwl_trans *trans)
836 837
{
	const struct queue_to_fifo_ac *queue_to_fifo;
838 839
	struct iwl_trans_pcie *trans_pcie =
		IWL_TRANS_GET_PCIE_TRANS(trans);
840 841 842 843 844
	u32 a;
	unsigned long flags;
	int i, chan;
	u32 reg_val;

845
	spin_lock_irqsave(&trans->shrd->lock, flags);
846

847 848
	trans_pcie->scd_base_addr =
		iwl_read_prph(bus(trans), SCD_SRAM_BASE_ADDR);
849
	a = trans_pcie->scd_base_addr + SCD_CONTEXT_MEM_LOWER_BOUND;
850
	/* reset conext data memory */
851
	for (; a < trans_pcie->scd_base_addr + SCD_CONTEXT_MEM_UPPER_BOUND;
852
		a += 4)
853
		iwl_write_targ_mem(bus(trans), a, 0);
854
	/* reset tx status memory */
855
	for (; a < trans_pcie->scd_base_addr + SCD_TX_STTS_MEM_UPPER_BOUND;
856
		a += 4)
857
		iwl_write_targ_mem(bus(trans), a, 0);
858
	for (; a < trans_pcie->scd_base_addr +
859
	       SCD_TRANS_TBL_OFFSET_QUEUE(hw_params(trans).max_txq_num);
860
	       a += 4)
861
		iwl_write_targ_mem(bus(trans), a, 0);
862

863
	iwl_write_prph(bus(trans), SCD_DRAM_BASE_ADDR,
864
		       trans_pcie->scd_bc_tbls.dma >> 10);
865 866 867

	/* Enable DMA channel */
	for (chan = 0; chan < FH_TCSR_CHNL_NUM ; chan++)
868
		iwl_write_direct32(bus(trans), FH_TCSR_CHNL_TX_CONFIG_REG(chan),
869 870 871 872
				FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE |
				FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_ENABLE);

	/* Update FH chicken bits */
873 874
	reg_val = iwl_read_direct32(bus(trans), FH_TX_CHICKEN_BITS_REG);
	iwl_write_direct32(bus(trans), FH_TX_CHICKEN_BITS_REG,
875 876
			   reg_val | FH_TX_CHICKEN_BITS_SCD_AUTO_RETRY_EN);

877
	iwl_write_prph(bus(trans), SCD_QUEUECHAIN_SEL,
878
		SCD_QUEUECHAIN_SEL_ALL(trans));
879
	iwl_write_prph(bus(trans), SCD_AGGR_SEL, 0);
880 881

	/* initiate the queues */
882
	for (i = 0; i < hw_params(trans).max_txq_num; i++) {
883 884 885
		iwl_write_prph(bus(trans), SCD_QUEUE_RDPTR(i), 0);
		iwl_write_direct32(bus(trans), HBUS_TARG_WRPTR, 0 | (i << 8));
		iwl_write_targ_mem(bus(trans), trans_pcie->scd_base_addr +
886
				SCD_CONTEXT_QUEUE_OFFSET(i), 0);
887
		iwl_write_targ_mem(bus(trans), trans_pcie->scd_base_addr +
888 889 890 891 892 893 894 895 896 897
				SCD_CONTEXT_QUEUE_OFFSET(i) +
				sizeof(u32),
				((SCD_WIN_SIZE <<
				SCD_QUEUE_CTX_REG2_WIN_SIZE_POS) &
				SCD_QUEUE_CTX_REG2_WIN_SIZE_MSK) |
				((SCD_FRAME_LIMIT <<
				SCD_QUEUE_CTX_REG2_FRAME_LIMIT_POS) &
				SCD_QUEUE_CTX_REG2_FRAME_LIMIT_MSK));
	}

898
	iwl_write_prph(bus(trans), SCD_INTERRUPT_MASK,
899
			IWL_MASK(0, hw_params(trans).max_txq_num));
900 901

	/* Activate all Tx DMA/FIFO channels */
902
	iwl_trans_txq_set_sched(trans, IWL_MASK(0, 7));
903 904

	/* map queues to FIFOs */
905
	if (trans->shrd->valid_contexts != BIT(IWL_RXON_CTX_BSS))
906 907 908 909
		queue_to_fifo = iwlagn_ipan_queue_to_tx_fifo;
	else
		queue_to_fifo = iwlagn_default_queue_to_tx_fifo;

910
	iwl_trans_set_wr_ptrs(trans, trans->shrd->cmd_queue, 0);
911 912

	/* make sure all queue are not stopped */
913 914
	memset(&trans_pcie->queue_stopped[0], 0,
		sizeof(trans_pcie->queue_stopped));
915
	for (i = 0; i < 4; i++)
916
		atomic_set(&trans_pcie->queue_stop_count[i], 0);
917 918

	/* reset to 0 to enable all the queue first */
919
	trans_pcie->txq_ctx_active_msk = 0;
920

921
	BUILD_BUG_ON(ARRAY_SIZE(iwlagn_default_queue_to_tx_fifo) <
922
						IWLAGN_FIRST_AMPDU_QUEUE);
923
	BUILD_BUG_ON(ARRAY_SIZE(iwlagn_ipan_queue_to_tx_fifo) <
924
						IWLAGN_FIRST_AMPDU_QUEUE);
925

926
	for (i = 0; i < IWLAGN_FIRST_AMPDU_QUEUE; i++) {
927 928 929
		int fifo = queue_to_fifo[i].fifo;
		int ac = queue_to_fifo[i].ac;

930
		iwl_txq_ctx_activate(trans_pcie, i);
931 932 933 934 935

		if (fifo == IWL_TX_FIFO_UNUSED)
			continue;

		if (ac != IWL_AC_UNSET)
936 937 938
			iwl_set_swq_id(&trans_pcie->txq[i], ac, i);
		iwl_trans_tx_queue_set_status(trans, &trans_pcie->txq[i],
					      fifo, 0);
939 940
	}

941
	spin_unlock_irqrestore(&trans->shrd->lock, flags);
942 943

	/* Enable L1-Active */
944
	iwl_clear_bits_prph(bus(trans), APMG_PCIDEV_STT_REG,
945 946 947
			  APMG_PCIDEV_STT_VAL_L1_ACT_DIS);
}

948 949 950
/**
 * iwlagn_txq_ctx_stop - Stop all Tx DMA channels
 */
951
static int iwl_trans_tx_stop(struct iwl_trans *trans)
952 953 954
{
	int ch, txq_id;
	unsigned long flags;
955
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
956 957

	/* Turn off all Tx DMA fifos */
958
	spin_lock_irqsave(&trans->shrd->lock, flags);
959

960
	iwl_trans_txq_set_sched(trans, 0);
961 962

	/* Stop each Tx DMA channel, and wait for it to be idle */
963
	for (ch = 0; ch < FH_TCSR_CHNL_NUM; ch++) {
964
		iwl_write_direct32(bus(trans),
965
				   FH_TCSR_CHNL_TX_CONFIG_REG(ch), 0x0);
966
		if (iwl_poll_direct_bit(bus(trans), FH_TSSR_TX_STATUS_REG,
967 968
				    FH_TSSR_TX_STATUS_REG_MSK_CHNL_IDLE(ch),
				    1000))
969
			IWL_ERR(trans, "Failing on timeout while stopping"
970
			    " DMA channel %d [0x%08x]", ch,
971
			    iwl_read_direct32(bus(trans),
972
					      FH_TSSR_TX_STATUS_REG));
973
	}
974
	spin_unlock_irqrestore(&trans->shrd->lock, flags);
975

976
	if (!trans_pcie->txq) {
977
		IWL_WARN(trans, "Stopping tx queues that aren't allocated...");
978 979 980 981
		return 0;
	}

	/* Unmap DMA from host system and free skb's */
982 983
	for (txq_id = 0; txq_id < hw_params(trans).max_txq_num; txq_id++)
		iwl_tx_queue_unmap(trans, txq_id);
984 985 986 987

	return 0;
}

988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002
static void iwl_trans_pcie_disable_sync_irq(struct iwl_trans *trans)
{
	unsigned long flags;
	struct iwl_trans_pcie *trans_pcie =
		IWL_TRANS_GET_PCIE_TRANS(trans);

	spin_lock_irqsave(&trans->shrd->lock, flags);
	iwl_disable_interrupts(trans);
	spin_unlock_irqrestore(&trans->shrd->lock, flags);

	/* wait to make sure we flush pending tasklet*/
	synchronize_irq(bus(trans)->irq);
	tasklet_kill(&trans_pcie->irq_tasklet);
}

1003
static void iwl_trans_pcie_stop_device(struct iwl_trans *trans)
1004 1005
{
	/* stop and reset the on-board processor */
1006
	iwl_write32(bus(trans), CSR_RESET, CSR_RESET_REG_FLAG_NEVO_RESET);
1007 1008

	/* tell the device to stop sending interrupts */
1009
	iwl_trans_pcie_disable_sync_irq(trans);
1010 1011

	/* device going down, Stop using ICT table */
1012
	iwl_disable_ict(trans);
1013 1014 1015 1016 1017 1018 1019 1020

	/*
	 * If a HW restart happens during firmware loading,
	 * then the firmware loading might call this function
	 * and later it might be called again due to the
	 * restart. So don't process again if the device is
	 * already dead.
	 */
1021 1022 1023
	if (test_bit(STATUS_DEVICE_ENABLED, &trans->shrd->status)) {
		iwl_trans_tx_stop(trans);
		iwl_trans_rx_stop(trans);
1024 1025

		/* Power-down device's busmaster DMA clocks */
1026
		iwl_write_prph(bus(trans), APMG_CLK_DIS_REG,
1027 1028 1029 1030 1031
			       APMG_CLK_VAL_DMA_CLK_RQT);
		udelay(5);
	}

	/* Make sure (redundant) we've released our request to stay awake */
1032
	iwl_clear_bit(bus(trans), CSR_GP_CNTRL,
1033
			CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
1034 1035

	/* Stop the device, and put it in low power state */
1036
	iwl_apm_stop(priv(trans));
1037 1038
}

1039
static int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb,
1040 1041
		struct iwl_device_cmd *dev_cmd, enum iwl_rxon_context_id ctx,
		u8 sta_id)
1042
{
1043 1044 1045
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
1046
	struct iwl_tx_cmd *tx_cmd = &dev_cmd->cmd.tx;
1047
	struct iwl_cmd_meta *out_meta;
1048 1049
	struct iwl_tx_queue *txq;
	struct iwl_queue *q;
1050 1051 1052 1053 1054

	dma_addr_t phys_addr = 0;
	dma_addr_t txcmd_phys;
	dma_addr_t scratch_phys;
	u16 len, firstlen, secondlen;
1055
	u16 seq_number = 0;
1056
	u8 wait_write_ptr = 0;
1057 1058 1059 1060
	u8 txq_id;
	u8 tid = 0;
	bool is_agg = false;
	__le16 fc = hdr->frame_control;
1061 1062
	u8 hdr_len = ieee80211_hdrlen(fc);

1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105
	/*
	 * Send this frame after DTIM -- there's a special queue
	 * reserved for this for contexts that support AP mode.
	 */
	if (info->flags & IEEE80211_TX_CTL_SEND_AFTER_DTIM) {
		txq_id = trans_pcie->mcast_queue[ctx];

		/*
		 * The microcode will clear the more data
		 * bit in the last frame it transmits.
		 */
		hdr->frame_control |=
			cpu_to_le16(IEEE80211_FCTL_MOREDATA);
	} else if (info->flags & IEEE80211_TX_CTL_TX_OFFCHAN)
		txq_id = IWL_AUX_QUEUE;
	else
		txq_id =
		    trans_pcie->ac_to_queue[ctx][skb_get_queue_mapping(skb)];

	if (ieee80211_is_data_qos(fc)) {
		u8 *qc = NULL;
		struct iwl_tid_data *tid_data;
		qc = ieee80211_get_qos_ctl(hdr);
		tid = qc[0] & IEEE80211_QOS_CTL_TID_MASK;
		tid_data = &trans->shrd->tid_data[sta_id][tid];

		if (WARN_ON_ONCE(tid >= IWL_MAX_TID_COUNT))
			return -1;

		seq_number = tid_data->seq_number;
		seq_number &= IEEE80211_SCTL_SEQ;
		hdr->seq_ctrl = hdr->seq_ctrl &
				cpu_to_le16(IEEE80211_SCTL_FRAG);
		hdr->seq_ctrl |= cpu_to_le16(seq_number);
		seq_number += 0x10;
		/* aggregation is on for this <sta,tid> */
		if (info->flags & IEEE80211_TX_CTL_AMPDU &&
		    tid_data->agg.state == IWL_AGG_ON) {
			txq_id = tid_data->agg.txq_id;
			is_agg = true;
		}
	}

1106
	txq = &trans_pcie->txq[txq_id];
1107 1108
	q = &txq->q;

1109
	/* Set up driver data for this TFD */
1110
	txq->skbs[q->write_ptr] = skb;
1111 1112 1113 1114 1115
	txq->cmd[q->write_ptr] = dev_cmd;

	dev_cmd->hdr.cmd = REPLY_TX;
	dev_cmd->hdr.sequence = cpu_to_le16((u16)(QUEUE_TO_SEQ(txq_id) |
				INDEX_TO_SEQ(q->write_ptr)));
1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138

	/* Set up first empty entry in queue's array of Tx/cmd buffers */
	out_meta = &txq->meta[q->write_ptr];

	/*
	 * Use the first empty entry in this queue's command buffer array
	 * to contain the Tx command and MAC header concatenated together
	 * (payload data will be in another buffer).
	 * Size of this varies, due to varying MAC header length.
	 * If end is not dword aligned, we'll have 2 extra bytes at the end
	 * of the MAC header (device reads on dword boundaries).
	 * We'll tell device about this padding later.
	 */
	len = sizeof(struct iwl_tx_cmd) +
		sizeof(struct iwl_cmd_header) + hdr_len;
	firstlen = (len + 3) & ~3;

	/* Tell NIC about any 2-byte padding after MAC header */
	if (firstlen != len)
		tx_cmd->tx_flags |= TX_CMD_FLG_MH_PAD_MSK;

	/* Physical address of this Tx command's header (not MAC header!),
	 * within command buffer array. */
1139
	txcmd_phys = dma_map_single(bus(trans)->dev,
1140 1141
				    &dev_cmd->hdr, firstlen,
				    DMA_BIDIRECTIONAL);
1142
	if (unlikely(dma_mapping_error(bus(trans)->dev, txcmd_phys)))
1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157
		return -1;
	dma_unmap_addr_set(out_meta, mapping, txcmd_phys);
	dma_unmap_len_set(out_meta, len, firstlen);

	if (!ieee80211_has_morefrags(fc)) {
		txq->need_update = 1;
	} else {
		wait_write_ptr = 1;
		txq->need_update = 0;
	}

	/* Set up TFD's 2nd entry to point directly to remainder of skb,
	 * if any (802.11 null frames have no payload). */
	secondlen = skb->len - hdr_len;
	if (secondlen > 0) {
1158
		phys_addr = dma_map_single(bus(trans)->dev, skb->data + hdr_len,
1159
					   secondlen, DMA_TO_DEVICE);
1160 1161
		if (unlikely(dma_mapping_error(bus(trans)->dev, phys_addr))) {
			dma_unmap_single(bus(trans)->dev,
1162 1163 1164 1165 1166 1167 1168 1169
					 dma_unmap_addr(out_meta, mapping),
					 dma_unmap_len(out_meta, len),
					 DMA_BIDIRECTIONAL);
			return -1;
		}
	}

	/* Attach buffers to TFD */
1170
	iwlagn_txq_attach_buf_to_tfd(trans, txq, txcmd_phys, firstlen, 1);
1171
	if (secondlen > 0)
1172
		iwlagn_txq_attach_buf_to_tfd(trans, txq, phys_addr,
1173 1174 1175 1176 1177 1178
					     secondlen, 0);

	scratch_phys = txcmd_phys + sizeof(struct iwl_cmd_header) +
				offsetof(struct iwl_tx_cmd, scratch);

	/* take back ownership of DMA buffer to enable update */
1179
	dma_sync_single_for_cpu(bus(trans)->dev, txcmd_phys, firstlen,
1180 1181 1182 1183
			DMA_BIDIRECTIONAL);
	tx_cmd->dram_lsb_ptr = cpu_to_le32(scratch_phys);
	tx_cmd->dram_msb_ptr = iwl_get_dma_hi_addr(scratch_phys);

1184
	IWL_DEBUG_TX(trans, "sequence nr = 0X%x\n",
1185
		     le16_to_cpu(dev_cmd->hdr.sequence));
1186 1187 1188
	IWL_DEBUG_TX(trans, "tx_flags = 0X%x\n", le32_to_cpu(tx_cmd->tx_flags));
	iwl_print_hex_dump(trans, IWL_DL_TX, (u8 *)tx_cmd, sizeof(*tx_cmd));
	iwl_print_hex_dump(trans, IWL_DL_TX, (u8 *)tx_cmd->hdr, hdr_len);
1189 1190

	/* Set up entry for this TFD in Tx byte-count array */
1191 1192
	if (is_agg)
		iwl_trans_txq_update_byte_cnt_tbl(trans, txq,
1193 1194
					       le16_to_cpu(tx_cmd->len));

1195
	dma_sync_single_for_device(bus(trans)->dev, txcmd_phys, firstlen,
1196 1197
			DMA_BIDIRECTIONAL);

1198
	trace_iwlwifi_dev_tx(priv(trans),
1199 1200 1201 1202 1203 1204 1205
			     &((struct iwl_tfd *)txq->tfds)[txq->q.write_ptr],
			     sizeof(struct iwl_tfd),
			     &dev_cmd->hdr, firstlen,
			     skb->data + hdr_len, secondlen);

	/* Tell device the write index *just past* this latest filled TFD */
	q->write_ptr = iwl_queue_inc_wrap(q->write_ptr, q->n_bd);
1206 1207 1208 1209 1210 1211 1212 1213
	iwl_txq_update_write_ptr(trans, txq);

	if (ieee80211_is_data_qos(fc)) {
		trans->shrd->tid_data[sta_id][tid].tfds_in_queue++;
		if (!ieee80211_has_morefrags(fc))
			trans->shrd->tid_data[sta_id][tid].seq_number =
				seq_number;
	}
1214 1215 1216 1217 1218 1219 1220

	/*
	 * At this point the frame is "transmitted" successfully
	 * and we will get a TX status notification eventually,
	 * regardless of the value of ret. "ret" only indicates
	 * whether or not we should update the write pointer.
	 */
1221
	if (iwl_queue_space(q) < q->high_mark) {
1222 1223
		if (wait_write_ptr) {
			txq->need_update = 1;
1224
			iwl_txq_update_write_ptr(trans, txq);
1225
		} else {
1226
			iwl_stop_queue(trans, txq);
1227 1228 1229 1230 1231
		}
	}
	return 0;
}

1232
static void iwl_trans_pcie_kick_nic(struct iwl_trans *trans)
1233 1234
{
	/* Remove all resets to allow NIC to operate */
1235
	iwl_write32(bus(trans), CSR_RESET, 0);
1236 1237
}

1238 1239
static int iwl_trans_pcie_request_irq(struct iwl_trans *trans)
{
1240 1241
	struct iwl_trans_pcie *trans_pcie =
		IWL_TRANS_GET_PCIE_TRANS(trans);
1242 1243
	int err;

1244 1245 1246 1247
	trans_pcie->inta_mask = CSR_INI_SET_MASK;

	tasklet_init(&trans_pcie->irq_tasklet, (void (*)(unsigned long))
		iwl_irq_tasklet, (unsigned long)trans);
1248

1249
	iwl_alloc_isr_ict(trans);
1250 1251

	err = request_irq(bus(trans)->irq, iwl_isr_ict, IRQF_SHARED,
1252
		DRV_NAME, trans);
1253
	if (err) {
1254 1255
		IWL_ERR(trans, "Error allocating IRQ %d\n", bus(trans)->irq);
		iwl_free_isr_ict(trans);
1256 1257 1258
		return err;
	}

1259
	INIT_WORK(&trans_pcie->rx_replenish, iwl_bg_rx_replenish);
1260 1261 1262
	return 0;
}

1263 1264
static int iwlagn_txq_check_empty(struct iwl_trans *trans,
			   int sta_id, u8 tid, int txq_id)
1265
{
1266 1267
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
	struct iwl_queue *q = &trans_pcie->txq[txq_id].q;
1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279
	struct iwl_tid_data *tid_data = &trans->shrd->tid_data[sta_id][tid];

	lockdep_assert_held(&trans->shrd->sta_lock);

	switch (trans->shrd->tid_data[sta_id][tid].agg.state) {
	case IWL_EMPTYING_HW_QUEUE_DELBA:
		/* We are reclaiming the last packet of the */
		/* aggregated HW queue */
		if ((txq_id  == tid_data->agg.txq_id) &&
		    (q->read_ptr == q->write_ptr)) {
			IWL_DEBUG_HT(trans,
				"HW queue empty: continue DELBA flow\n");
1280
			iwl_trans_pcie_txq_agg_disable(trans, txq_id);
1281 1282 1283 1284
			tid_data->agg.state = IWL_AGG_OFF;
			iwl_stop_tx_ba_trans_ready(priv(trans),
						   NUM_IWL_RXON_CTX,
						   sta_id, tid);
1285
			iwl_wake_queue(trans, &trans_pcie->txq[txq_id]);
1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298
		}
		break;
	case IWL_EMPTYING_HW_QUEUE_ADDBA:
		/* We are reclaiming the last packet of the queue */
		if (tid_data->tfds_in_queue == 0) {
			IWL_DEBUG_HT(trans,
				"HW queue empty: continue ADDBA flow\n");
			tid_data->agg.state = IWL_AGG_ON;
			iwl_start_tx_ba_trans_ready(priv(trans),
						    NUM_IWL_RXON_CTX,
						    sta_id, tid);
		}
		break;
1299 1300
	default:
		break;
1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324
	}

	return 0;
}

static void iwl_free_tfds_in_queue(struct iwl_trans *trans,
			    int sta_id, int tid, int freed)
{
	lockdep_assert_held(&trans->shrd->sta_lock);

	if (trans->shrd->tid_data[sta_id][tid].tfds_in_queue >= freed)
		trans->shrd->tid_data[sta_id][tid].tfds_in_queue -= freed;
	else {
		IWL_DEBUG_TX(trans, "free more than tfds_in_queue (%u:%d)\n",
			trans->shrd->tid_data[sta_id][tid].tfds_in_queue,
			freed);
		trans->shrd->tid_data[sta_id][tid].tfds_in_queue = 0;
	}
}

static void iwl_trans_pcie_reclaim(struct iwl_trans *trans, int sta_id, int tid,
		      int txq_id, int ssn, u32 status,
		      struct sk_buff_head *skbs)
{
1325 1326
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
	struct iwl_tx_queue *txq = &trans_pcie->txq[txq_id];
1327
	enum iwl_agg_state agg_state;
1328 1329
	/* n_bd is usually 256 => n_bd - 1 = 0xff */
	int tfd_num = ssn & (txq->q.n_bd - 1);
1330
	int freed = 0;
1331 1332
	bool cond;

1333 1334
	txq->time_stamp = jiffies;

1335 1336
	if (txq->sched_retry) {
		agg_state =
1337
			trans->shrd->tid_data[txq->sta_id][txq->tid].agg.state;
1338 1339 1340 1341 1342 1343 1344 1345 1346
		cond = (agg_state != IWL_EMPTYING_HW_QUEUE_DELBA);
	} else {
		cond = (status != TX_STATUS_FAIL_PASSIVE_NO_RX);
	}

	if (txq->q.read_ptr != tfd_num) {
		IWL_DEBUG_TX_REPLY(trans, "Retry scheduler reclaim "
				"scd_ssn=%d idx=%d txq=%d swq=%d\n",
				ssn , tfd_num, txq_id, txq->swq_id);
1347
		freed = iwl_tx_queue_reclaim(trans, txq_id, tfd_num, skbs);
1348
		if (iwl_queue_space(&txq->q) > txq->q.low_mark && cond)
1349
			iwl_wake_queue(trans, txq);
1350
	}
1351 1352 1353

	iwl_free_tfds_in_queue(trans, sta_id, tid, freed);
	iwlagn_txq_check_empty(trans, sta_id, tid, txq_id);
1354 1355
}

1356
static void iwl_trans_pcie_free(struct iwl_trans *trans)
1357
{
1358 1359
	iwl_trans_pcie_tx_free(trans);
	iwl_trans_pcie_rx_free(trans);
1360 1361 1362 1363
	free_irq(bus(trans)->irq, trans);
	iwl_free_isr_ict(trans);
	trans->shrd->trans = NULL;
	kfree(trans);
1364 1365
}

1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389
#ifdef CONFIG_PM

static int iwl_trans_pcie_suspend(struct iwl_trans *trans)
{
	/*
	 * This function is called when system goes into suspend state
	 * mac80211 will call iwl_mac_stop() from the mac80211 suspend function
	 * first but since iwl_mac_stop() has no knowledge of who the caller is,
	 * it will not call apm_ops.stop() to stop the DMA operation.
	 * Calling apm_ops.stop here to make sure we stop the DMA.
	 *
	 * But of course ... if we have configured WoWLAN then we did other
	 * things already :-)
	 */
	if (!trans->shrd->wowlan)
		iwl_apm_stop(priv(trans));

	return 0;
}

static int iwl_trans_pcie_resume(struct iwl_trans *trans)
{
	bool hw_rfkill = false;

1390
	iwl_enable_interrupts(trans);
1391

1392
	if (!(iwl_read32(bus(trans), CSR_GP_CNTRL) &
1393 1394 1395 1396 1397 1398 1399 1400
				CSR_GP_CNTRL_REG_FLAG_HW_RF_KILL_SW))
		hw_rfkill = true;

	if (hw_rfkill)
		set_bit(STATUS_RF_KILL_HW, &trans->shrd->status);
	else
		clear_bit(STATUS_RF_KILL_HW, &trans->shrd->status);

1401
	iwl_set_hw_rfkill_state(priv(trans), hw_rfkill);
1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413

	return 0;
}
#else /* CONFIG_PM */
static int iwl_trans_pcie_suspend(struct iwl_trans *trans)
{ return 0; }

static int iwl_trans_pcie_resume(struct iwl_trans *trans)
{ return 0; }

#endif /* CONFIG_PM */

1414
static void iwl_trans_pcie_wake_any_queue(struct iwl_trans *trans,
1415
					  enum iwl_rxon_context_id ctx)
1416 1417 1418 1419 1420 1421 1422 1423 1424
{
	u8 ac, txq_id;
	struct iwl_trans_pcie *trans_pcie =
		IWL_TRANS_GET_PCIE_TRANS(trans);

	for (ac = 0; ac < AC_NUM; ac++) {
		txq_id = trans_pcie->ac_to_queue[ctx][ac];
		IWL_DEBUG_INFO(trans, "Queue Status: Q[%d] %s\n",
			ac,
1425
			(atomic_read(&trans_pcie->queue_stop_count[ac]) > 0)
1426
			      ? "stopped" : "awake");
1427
		iwl_wake_queue(trans, &trans_pcie->txq[txq_id]);
1428 1429 1430
	}
}

1431
const struct iwl_trans_ops trans_ops_pcie;
1432

1433 1434 1435 1436 1437 1438
static struct iwl_trans *iwl_trans_pcie_alloc(struct iwl_shared *shrd)
{
	struct iwl_trans *iwl_trans = kzalloc(sizeof(struct iwl_trans) +
					      sizeof(struct iwl_trans_pcie),
					      GFP_KERNEL);
	if (iwl_trans) {
1439 1440
		struct iwl_trans_pcie *trans_pcie =
			IWL_TRANS_GET_PCIE_TRANS(iwl_trans);
1441 1442
		iwl_trans->ops = &trans_ops_pcie;
		iwl_trans->shrd = shrd;
1443
		trans_pcie->trans = iwl_trans;
1444
		spin_lock_init(&iwl_trans->hcmd_lock);
1445
	}
1446

1447 1448
	return iwl_trans;
}
1449

1450 1451
static void iwl_trans_pcie_stop_queue(struct iwl_trans *trans, int txq_id)
{
1452 1453 1454
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);

	iwl_stop_queue(trans, &trans_pcie->txq[txq_id]);
1455 1456
}

1457 1458 1459 1460
#define IWL_FLUSH_WAIT_MS	2000

static int iwl_trans_pcie_wait_tx_queue_empty(struct iwl_trans *trans)
{
1461
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1462 1463 1464 1465 1466 1467 1468 1469 1470 1471
	struct iwl_tx_queue *txq;
	struct iwl_queue *q;
	int cnt;
	unsigned long now = jiffies;
	int ret = 0;

	/* waiting for all the tx frames complete might take a while */
	for (cnt = 0; cnt < hw_params(trans).max_txq_num; cnt++) {
		if (cnt == trans->shrd->cmd_queue)
			continue;
1472
		txq = &trans_pcie->txq[cnt];
1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486
		q = &txq->q;
		while (q->read_ptr != q->write_ptr && !time_after(jiffies,
		       now + msecs_to_jiffies(IWL_FLUSH_WAIT_MS)))
			msleep(1);

		if (q->read_ptr != q->write_ptr) {
			IWL_ERR(trans, "fail to flush all tx fifo queues\n");
			ret = -ETIMEDOUT;
			break;
		}
	}
	return ret;
}

1487 1488 1489 1490 1491 1492
/*
 * On every watchdog tick we check (latest) time stamp. If it does not
 * change during timeout period and queue is not empty we reset firmware.
 */
static int iwl_trans_pcie_check_stuck_queue(struct iwl_trans *trans, int cnt)
{
1493 1494
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
	struct iwl_tx_queue *txq = &trans_pcie->txq[cnt];
1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508
	struct iwl_queue *q = &txq->q;
	unsigned long timeout;

	if (q->read_ptr == q->write_ptr) {
		txq->time_stamp = jiffies;
		return 0;
	}

	timeout = txq->time_stamp +
		  msecs_to_jiffies(hw_params(trans).wd_timeout);

	if (time_after(jiffies, timeout)) {
		IWL_ERR(trans, "Queue %d stuck for %u ms.\n", q->id,
			hw_params(trans).wd_timeout);
1509 1510
		IWL_ERR(trans, "Current read_ptr %d write_ptr %d\n",
			q->read_ptr, q->write_ptr);
1511 1512 1513 1514 1515 1516
		return 1;
	}

	return 0;
}

1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646
static const char *get_fh_string(int cmd)
{
	switch (cmd) {
	IWL_CMD(FH_RSCSR_CHNL0_STTS_WPTR_REG);
	IWL_CMD(FH_RSCSR_CHNL0_RBDCB_BASE_REG);
	IWL_CMD(FH_RSCSR_CHNL0_WPTR);
	IWL_CMD(FH_MEM_RCSR_CHNL0_CONFIG_REG);
	IWL_CMD(FH_MEM_RSSR_SHARED_CTRL_REG);
	IWL_CMD(FH_MEM_RSSR_RX_STATUS_REG);
	IWL_CMD(FH_MEM_RSSR_RX_ENABLE_ERR_IRQ2DRV);
	IWL_CMD(FH_TSSR_TX_STATUS_REG);
	IWL_CMD(FH_TSSR_TX_ERROR_REG);
	default:
		return "UNKNOWN";
	}
}

int iwl_dump_fh(struct iwl_trans *trans, char **buf, bool display)
{
	int i;
#ifdef CONFIG_IWLWIFI_DEBUG
	int pos = 0;
	size_t bufsz = 0;
#endif
	static const u32 fh_tbl[] = {
		FH_RSCSR_CHNL0_STTS_WPTR_REG,
		FH_RSCSR_CHNL0_RBDCB_BASE_REG,
		FH_RSCSR_CHNL0_WPTR,
		FH_MEM_RCSR_CHNL0_CONFIG_REG,
		FH_MEM_RSSR_SHARED_CTRL_REG,
		FH_MEM_RSSR_RX_STATUS_REG,
		FH_MEM_RSSR_RX_ENABLE_ERR_IRQ2DRV,
		FH_TSSR_TX_STATUS_REG,
		FH_TSSR_TX_ERROR_REG
	};
#ifdef CONFIG_IWLWIFI_DEBUG
	if (display) {
		bufsz = ARRAY_SIZE(fh_tbl) * 48 + 40;
		*buf = kmalloc(bufsz, GFP_KERNEL);
		if (!*buf)
			return -ENOMEM;
		pos += scnprintf(*buf + pos, bufsz - pos,
				"FH register values:\n");
		for (i = 0; i < ARRAY_SIZE(fh_tbl); i++) {
			pos += scnprintf(*buf + pos, bufsz - pos,
				"  %34s: 0X%08x\n",
				get_fh_string(fh_tbl[i]),
				iwl_read_direct32(bus(trans), fh_tbl[i]));
		}
		return pos;
	}
#endif
	IWL_ERR(trans, "FH register values:\n");
	for (i = 0; i <  ARRAY_SIZE(fh_tbl); i++) {
		IWL_ERR(trans, "  %34s: 0X%08x\n",
			get_fh_string(fh_tbl[i]),
			iwl_read_direct32(bus(trans), fh_tbl[i]));
	}
	return 0;
}

static const char *get_csr_string(int cmd)
{
	switch (cmd) {
	IWL_CMD(CSR_HW_IF_CONFIG_REG);
	IWL_CMD(CSR_INT_COALESCING);
	IWL_CMD(CSR_INT);
	IWL_CMD(CSR_INT_MASK);
	IWL_CMD(CSR_FH_INT_STATUS);
	IWL_CMD(CSR_GPIO_IN);
	IWL_CMD(CSR_RESET);
	IWL_CMD(CSR_GP_CNTRL);
	IWL_CMD(CSR_HW_REV);
	IWL_CMD(CSR_EEPROM_REG);
	IWL_CMD(CSR_EEPROM_GP);
	IWL_CMD(CSR_OTP_GP_REG);
	IWL_CMD(CSR_GIO_REG);
	IWL_CMD(CSR_GP_UCODE_REG);
	IWL_CMD(CSR_GP_DRIVER_REG);
	IWL_CMD(CSR_UCODE_DRV_GP1);
	IWL_CMD(CSR_UCODE_DRV_GP2);
	IWL_CMD(CSR_LED_REG);
	IWL_CMD(CSR_DRAM_INT_TBL_REG);
	IWL_CMD(CSR_GIO_CHICKEN_BITS);
	IWL_CMD(CSR_ANA_PLL_CFG);
	IWL_CMD(CSR_HW_REV_WA_REG);
	IWL_CMD(CSR_DBG_HPET_MEM_REG);
	default:
		return "UNKNOWN";
	}
}

void iwl_dump_csr(struct iwl_trans *trans)
{
	int i;
	static const u32 csr_tbl[] = {
		CSR_HW_IF_CONFIG_REG,
		CSR_INT_COALESCING,
		CSR_INT,
		CSR_INT_MASK,
		CSR_FH_INT_STATUS,
		CSR_GPIO_IN,
		CSR_RESET,
		CSR_GP_CNTRL,
		CSR_HW_REV,
		CSR_EEPROM_REG,
		CSR_EEPROM_GP,
		CSR_OTP_GP_REG,
		CSR_GIO_REG,
		CSR_GP_UCODE_REG,
		CSR_GP_DRIVER_REG,
		CSR_UCODE_DRV_GP1,
		CSR_UCODE_DRV_GP2,
		CSR_LED_REG,
		CSR_DRAM_INT_TBL_REG,
		CSR_GIO_CHICKEN_BITS,
		CSR_ANA_PLL_CFG,
		CSR_HW_REV_WA_REG,
		CSR_DBG_HPET_MEM_REG
	};
	IWL_ERR(trans, "CSR values:\n");
	IWL_ERR(trans, "(2nd byte of CSR_INT_COALESCING is "
		"CSR_INT_PERIODIC_REG)\n");
	for (i = 0; i <  ARRAY_SIZE(csr_tbl); i++) {
		IWL_ERR(trans, "  %25s: 0X%08x\n",
			get_csr_string(csr_tbl[i]),
			iwl_read32(bus(trans), csr_tbl[i]));
	}
}

1647 1648 1649
#ifdef CONFIG_IWLWIFI_DEBUGFS
/* create and remove of files */
#define DEBUGFS_ADD_FILE(name, parent, mode) do {			\
1650
	if (!debugfs_create_file(#name, mode, parent, trans,		\
1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680
				 &iwl_dbgfs_##name##_ops))		\
		return -ENOMEM;						\
} while (0)

/* file operation */
#define DEBUGFS_READ_FUNC(name)                                         \
static ssize_t iwl_dbgfs_##name##_read(struct file *file,               \
					char __user *user_buf,          \
					size_t count, loff_t *ppos);

#define DEBUGFS_WRITE_FUNC(name)                                        \
static ssize_t iwl_dbgfs_##name##_write(struct file *file,              \
					const char __user *user_buf,    \
					size_t count, loff_t *ppos);


static int iwl_dbgfs_open_file_generic(struct inode *inode, struct file *file)
{
	file->private_data = inode->i_private;
	return 0;
}

#define DEBUGFS_READ_FILE_OPS(name)					\
	DEBUGFS_READ_FUNC(name);					\
static const struct file_operations iwl_dbgfs_##name##_ops = {		\
	.read = iwl_dbgfs_##name##_read,				\
	.open = iwl_dbgfs_open_file_generic,				\
	.llseek = generic_file_llseek,					\
};

1681 1682 1683 1684 1685 1686 1687 1688
#define DEBUGFS_WRITE_FILE_OPS(name)                                    \
	DEBUGFS_WRITE_FUNC(name);                                       \
static const struct file_operations iwl_dbgfs_##name##_ops = {          \
	.write = iwl_dbgfs_##name##_write,                              \
	.open = iwl_dbgfs_open_file_generic,				\
	.llseek = generic_file_llseek,					\
};

1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700
#define DEBUGFS_READ_WRITE_FILE_OPS(name)				\
	DEBUGFS_READ_FUNC(name);					\
	DEBUGFS_WRITE_FUNC(name);					\
static const struct file_operations iwl_dbgfs_##name##_ops = {		\
	.write = iwl_dbgfs_##name##_write,				\
	.read = iwl_dbgfs_##name##_read,				\
	.open = iwl_dbgfs_open_file_generic,				\
	.llseek = generic_file_llseek,					\
};

static ssize_t iwl_dbgfs_tx_queue_read(struct file *file,
						char __user *user_buf,
1701 1702
						size_t count, loff_t *ppos)
{
1703
	struct iwl_trans *trans = file->private_data;
1704
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1705 1706 1707 1708 1709 1710
	struct iwl_tx_queue *txq;
	struct iwl_queue *q;
	char *buf;
	int pos = 0;
	int cnt;
	int ret;
1711
	const size_t bufsz = sizeof(char) * 64 * hw_params(trans).max_txq_num;
1712

1713
	if (!trans_pcie->txq) {
1714
		IWL_ERR(trans, "txq not ready\n");
1715 1716 1717 1718 1719 1720
		return -EAGAIN;
	}
	buf = kzalloc(bufsz, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;

1721
	for (cnt = 0; cnt < hw_params(trans).max_txq_num; cnt++) {
1722
		txq = &trans_pcie->txq[cnt];
1723 1724 1725 1726 1727
		q = &txq->q;
		pos += scnprintf(buf + pos, bufsz - pos,
				"hwq %.2d: read=%u write=%u stop=%d"
				" swq_id=%#.2x (ac %d/hwq %d)\n",
				cnt, q->read_ptr, q->write_ptr,
1728
				!!test_bit(cnt, trans_pcie->queue_stopped),
1729 1730 1731 1732 1733 1734
				txq->swq_id, txq->swq_id & 3,
				(txq->swq_id >> 2) & 0x1f);
		if (cnt >= 4)
			continue;
		/* for the ACs, display the stop count too */
		pos += scnprintf(buf + pos, bufsz - pos,
1735 1736
			"        stop-count: %d\n",
			atomic_read(&trans_pcie->queue_stop_count[cnt]));
1737 1738 1739 1740 1741 1742 1743 1744 1745
	}
	ret = simple_read_from_buffer(user_buf, count, ppos, buf, pos);
	kfree(buf);
	return ret;
}

static ssize_t iwl_dbgfs_rx_queue_read(struct file *file,
						char __user *user_buf,
						size_t count, loff_t *ppos) {
1746 1747 1748 1749
	struct iwl_trans *trans = file->private_data;
	struct iwl_trans_pcie *trans_pcie =
		IWL_TRANS_GET_PCIE_TRANS(trans);
	struct iwl_rx_queue *rxq = &trans_pcie->rxq;
1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769
	char buf[256];
	int pos = 0;
	const size_t bufsz = sizeof(buf);

	pos += scnprintf(buf + pos, bufsz - pos, "read: %u\n",
						rxq->read);
	pos += scnprintf(buf + pos, bufsz - pos, "write: %u\n",
						rxq->write);
	pos += scnprintf(buf + pos, bufsz - pos, "free_count: %u\n",
						rxq->free_count);
	if (rxq->rb_stts) {
		pos += scnprintf(buf + pos, bufsz - pos, "closed_rb_num: %u\n",
			 le16_to_cpu(rxq->rb_stts->closed_rb_num) &  0x0FFF);
	} else {
		pos += scnprintf(buf + pos, bufsz - pos,
					"closed_rb_num: Not Allocated\n");
	}
	return simple_read_from_buffer(user_buf, count, ppos, buf, pos);
}

1770 1771 1772 1773 1774 1775 1776 1777 1778
static ssize_t iwl_dbgfs_log_event_read(struct file *file,
					 char __user *user_buf,
					 size_t count, loff_t *ppos)
{
	struct iwl_trans *trans = file->private_data;
	char *buf;
	int pos = 0;
	ssize_t ret = -ENOMEM;

1779
	ret = pos = iwl_dump_nic_event_log(trans, true, &buf, true);
1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802
	if (buf) {
		ret = simple_read_from_buffer(user_buf, count, ppos, buf, pos);
		kfree(buf);
	}
	return ret;
}

static ssize_t iwl_dbgfs_log_event_write(struct file *file,
					const char __user *user_buf,
					size_t count, loff_t *ppos)
{
	struct iwl_trans *trans = file->private_data;
	u32 event_log_flag;
	char buf[8];
	int buf_size;

	memset(buf, 0, sizeof(buf));
	buf_size = min(count, sizeof(buf) -  1);
	if (copy_from_user(buf, user_buf, buf_size))
		return -EFAULT;
	if (sscanf(buf, "%d", &event_log_flag) != 1)
		return -EFAULT;
	if (event_log_flag == 1)
1803
		iwl_dump_nic_event_log(trans, true, NULL, false);
1804 1805 1806 1807

	return count;
}

1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893
static ssize_t iwl_dbgfs_interrupt_read(struct file *file,
					char __user *user_buf,
					size_t count, loff_t *ppos) {

	struct iwl_trans *trans = file->private_data;
	struct iwl_trans_pcie *trans_pcie =
		IWL_TRANS_GET_PCIE_TRANS(trans);
	struct isr_statistics *isr_stats = &trans_pcie->isr_stats;

	int pos = 0;
	char *buf;
	int bufsz = 24 * 64; /* 24 items * 64 char per item */
	ssize_t ret;

	buf = kzalloc(bufsz, GFP_KERNEL);
	if (!buf) {
		IWL_ERR(trans, "Can not allocate Buffer\n");
		return -ENOMEM;
	}

	pos += scnprintf(buf + pos, bufsz - pos,
			"Interrupt Statistics Report:\n");

	pos += scnprintf(buf + pos, bufsz - pos, "HW Error:\t\t\t %u\n",
		isr_stats->hw);
	pos += scnprintf(buf + pos, bufsz - pos, "SW Error:\t\t\t %u\n",
		isr_stats->sw);
	if (isr_stats->sw || isr_stats->hw) {
		pos += scnprintf(buf + pos, bufsz - pos,
			"\tLast Restarting Code:  0x%X\n",
			isr_stats->err_code);
	}
#ifdef CONFIG_IWLWIFI_DEBUG
	pos += scnprintf(buf + pos, bufsz - pos, "Frame transmitted:\t\t %u\n",
		isr_stats->sch);
	pos += scnprintf(buf + pos, bufsz - pos, "Alive interrupt:\t\t %u\n",
		isr_stats->alive);
#endif
	pos += scnprintf(buf + pos, bufsz - pos,
		"HW RF KILL switch toggled:\t %u\n", isr_stats->rfkill);

	pos += scnprintf(buf + pos, bufsz - pos, "CT KILL:\t\t\t %u\n",
		isr_stats->ctkill);

	pos += scnprintf(buf + pos, bufsz - pos, "Wakeup Interrupt:\t\t %u\n",
		isr_stats->wakeup);

	pos += scnprintf(buf + pos, bufsz - pos,
		"Rx command responses:\t\t %u\n", isr_stats->rx);

	pos += scnprintf(buf + pos, bufsz - pos, "Tx/FH interrupt:\t\t %u\n",
		isr_stats->tx);

	pos += scnprintf(buf + pos, bufsz - pos, "Unexpected INTA:\t\t %u\n",
		isr_stats->unhandled);

	ret = simple_read_from_buffer(user_buf, count, ppos, buf, pos);
	kfree(buf);
	return ret;
}

static ssize_t iwl_dbgfs_interrupt_write(struct file *file,
					 const char __user *user_buf,
					 size_t count, loff_t *ppos)
{
	struct iwl_trans *trans = file->private_data;
	struct iwl_trans_pcie *trans_pcie =
		IWL_TRANS_GET_PCIE_TRANS(trans);
	struct isr_statistics *isr_stats = &trans_pcie->isr_stats;

	char buf[8];
	int buf_size;
	u32 reset_flag;

	memset(buf, 0, sizeof(buf));
	buf_size = min(count, sizeof(buf) -  1);
	if (copy_from_user(buf, user_buf, buf_size))
		return -EFAULT;
	if (sscanf(buf, "%x", &reset_flag) != 1)
		return -EFAULT;
	if (reset_flag == 0)
		memset(isr_stats, 0, sizeof(*isr_stats));

	return count;
}

1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933
static ssize_t iwl_dbgfs_csr_write(struct file *file,
					 const char __user *user_buf,
					 size_t count, loff_t *ppos)
{
	struct iwl_trans *trans = file->private_data;
	char buf[8];
	int buf_size;
	int csr;

	memset(buf, 0, sizeof(buf));
	buf_size = min(count, sizeof(buf) -  1);
	if (copy_from_user(buf, user_buf, buf_size))
		return -EFAULT;
	if (sscanf(buf, "%d", &csr) != 1)
		return -EFAULT;

	iwl_dump_csr(trans);

	return count;
}

static ssize_t iwl_dbgfs_fh_reg_read(struct file *file,
					 char __user *user_buf,
					 size_t count, loff_t *ppos)
{
	struct iwl_trans *trans = file->private_data;
	char *buf;
	int pos = 0;
	ssize_t ret = -EFAULT;

	ret = pos = iwl_dump_fh(trans, &buf, true);
	if (buf) {
		ret = simple_read_from_buffer(user_buf,
					      count, ppos, buf, pos);
		kfree(buf);
	}

	return ret;
}

1934
DEBUGFS_READ_WRITE_FILE_OPS(log_event);
1935
DEBUGFS_READ_WRITE_FILE_OPS(interrupt);
1936
DEBUGFS_READ_FILE_OPS(fh_reg);
1937 1938
DEBUGFS_READ_FILE_OPS(rx_queue);
DEBUGFS_READ_FILE_OPS(tx_queue);
1939
DEBUGFS_WRITE_FILE_OPS(csr);
1940 1941 1942 1943 1944 1945 1946 1947 1948 1949

/*
 * Create the debugfs files and directories
 *
 */
static int iwl_trans_pcie_dbgfs_register(struct iwl_trans *trans,
					struct dentry *dir)
{
	DEBUGFS_ADD_FILE(rx_queue, dir, S_IRUSR);
	DEBUGFS_ADD_FILE(tx_queue, dir, S_IRUSR);
1950
	DEBUGFS_ADD_FILE(log_event, dir, S_IWUSR | S_IRUSR);
1951
	DEBUGFS_ADD_FILE(interrupt, dir, S_IWUSR | S_IRUSR);
1952 1953
	DEBUGFS_ADD_FILE(csr, dir, S_IWUSR);
	DEBUGFS_ADD_FILE(fh_reg, dir, S_IRUSR);
1954 1955 1956 1957 1958 1959 1960 1961 1962
	return 0;
}
#else
static int iwl_trans_pcie_dbgfs_register(struct iwl_trans *trans,
					struct dentry *dir)
{ return 0; }

#endif /*CONFIG_IWLWIFI_DEBUGFS */

1963 1964 1965 1966 1967 1968
const struct iwl_trans_ops trans_ops_pcie = {
	.alloc = iwl_trans_pcie_alloc,
	.request_irq = iwl_trans_pcie_request_irq,
	.start_device = iwl_trans_pcie_start_device,
	.prepare_card_hw = iwl_trans_pcie_prepare_card_hw,
	.stop_device = iwl_trans_pcie_stop_device,
1969

1970
	.tx_start = iwl_trans_pcie_tx_start,
1971
	.wake_any_queue = iwl_trans_pcie_wake_any_queue,
1972

1973 1974
	.send_cmd = iwl_trans_pcie_send_cmd,
	.send_cmd_pdu = iwl_trans_pcie_send_cmd_pdu,
1975

1976
	.tx = iwl_trans_pcie_tx,
1977
	.reclaim = iwl_trans_pcie_reclaim,
1978

1979
	.tx_agg_disable = iwl_trans_pcie_tx_agg_disable,
1980
	.tx_agg_alloc = iwl_trans_pcie_tx_agg_alloc,
1981
	.tx_agg_setup = iwl_trans_pcie_tx_agg_setup,
1982

1983
	.kick_nic = iwl_trans_pcie_kick_nic,
1984

1985
	.free = iwl_trans_pcie_free,
1986
	.stop_queue = iwl_trans_pcie_stop_queue,
1987 1988

	.dbgfs_register = iwl_trans_pcie_dbgfs_register,
1989 1990

	.wait_tx_queue_empty = iwl_trans_pcie_wait_tx_queue_empty,
1991
	.check_stuck_queue = iwl_trans_pcie_check_stuck_queue,
1992

1993 1994
	.suspend = iwl_trans_pcie_suspend,
	.resume = iwl_trans_pcie_resume,
1995
};