qp.c 26.4 KB
Newer Older
1 2 3
/*
 * Copyright (c) 2004 Topspin Communications.  All rights reserved.
 * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved.
4
 * Copyright (c) 2005, 2006, 2007, 2008 Mellanox Technologies. All rights reserved.
5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35
 * Copyright (c) 2004 Voltaire, Inc. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

36
#include <linux/gfp.h>
37
#include <linux/export.h>
38

39 40 41 42 43 44
#include <linux/mlx4/cmd.h>
#include <linux/mlx4/qp.h>

#include "mlx4.h"
#include "icm.h"

45 46 47 48
/* QP to support BF should have bits 6,7 cleared */
#define MLX4_BF_QP_SKIP_MASK	0xc0
#define MLX4_MAX_BF_QP_RANGE	0x40

49 50 51 52 53 54 55 56 57 58 59 60 61 62
void mlx4_qp_event(struct mlx4_dev *dev, u32 qpn, int event_type)
{
	struct mlx4_qp_table *qp_table = &mlx4_priv(dev)->qp_table;
	struct mlx4_qp *qp;

	spin_lock(&qp_table->lock);

	qp = __mlx4_qp_lookup(dev, qpn);
	if (qp)
		atomic_inc(&qp->refcount);

	spin_unlock(&qp_table->lock);

	if (!qp) {
63
		mlx4_dbg(dev, "Async event for none existent QP %08x\n", qpn);
64 65 66 67 68 69 70 71 72
		return;
	}

	qp->event(qp, event_type);

	if (atomic_dec_and_test(&qp->refcount))
		complete(&qp->free);
}

73
/* used for INIT/CLOSE port logic */
74
static int is_master_qp0(struct mlx4_dev *dev, struct mlx4_qp *qp, int *real_qp0, int *proxy_qp0)
75
{
76
	/* this procedure is called after we already know we are on the master */
77
	/* qp0 is either the proxy qp0, or the real qp0 */
78 79
	u32 pf_proxy_offset = dev->phys_caps.base_proxy_sqpn + 8 * mlx4_master_func_num(dev);
	*proxy_qp0 = qp->qpn >= pf_proxy_offset && qp->qpn <= pf_proxy_offset + 1;
80

81 82
	*real_qp0 = qp->qpn >= dev->phys_caps.base_sqpn &&
		qp->qpn <= dev->phys_caps.base_sqpn + 1;
83 84

	return *real_qp0 || *proxy_qp0;
85 86 87 88 89 90 91
}

static int __mlx4_qp_modify(struct mlx4_dev *dev, struct mlx4_mtt *mtt,
		     enum mlx4_qp_state cur_state, enum mlx4_qp_state new_state,
		     struct mlx4_qp_context *context,
		     enum mlx4_qp_optpar optpar,
		     int sqd_event, struct mlx4_qp *qp, int native)
92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132
{
	static const u16 op[MLX4_QP_NUM_STATE][MLX4_QP_NUM_STATE] = {
		[MLX4_QP_STATE_RST] = {
			[MLX4_QP_STATE_RST]	= MLX4_CMD_2RST_QP,
			[MLX4_QP_STATE_ERR]	= MLX4_CMD_2ERR_QP,
			[MLX4_QP_STATE_INIT]	= MLX4_CMD_RST2INIT_QP,
		},
		[MLX4_QP_STATE_INIT]  = {
			[MLX4_QP_STATE_RST]	= MLX4_CMD_2RST_QP,
			[MLX4_QP_STATE_ERR]	= MLX4_CMD_2ERR_QP,
			[MLX4_QP_STATE_INIT]	= MLX4_CMD_INIT2INIT_QP,
			[MLX4_QP_STATE_RTR]	= MLX4_CMD_INIT2RTR_QP,
		},
		[MLX4_QP_STATE_RTR]   = {
			[MLX4_QP_STATE_RST]	= MLX4_CMD_2RST_QP,
			[MLX4_QP_STATE_ERR]	= MLX4_CMD_2ERR_QP,
			[MLX4_QP_STATE_RTS]	= MLX4_CMD_RTR2RTS_QP,
		},
		[MLX4_QP_STATE_RTS]   = {
			[MLX4_QP_STATE_RST]	= MLX4_CMD_2RST_QP,
			[MLX4_QP_STATE_ERR]	= MLX4_CMD_2ERR_QP,
			[MLX4_QP_STATE_RTS]	= MLX4_CMD_RTS2RTS_QP,
			[MLX4_QP_STATE_SQD]	= MLX4_CMD_RTS2SQD_QP,
		},
		[MLX4_QP_STATE_SQD] = {
			[MLX4_QP_STATE_RST]	= MLX4_CMD_2RST_QP,
			[MLX4_QP_STATE_ERR]	= MLX4_CMD_2ERR_QP,
			[MLX4_QP_STATE_RTS]	= MLX4_CMD_SQD2RTS_QP,
			[MLX4_QP_STATE_SQD]	= MLX4_CMD_SQD2SQD_QP,
		},
		[MLX4_QP_STATE_SQER] = {
			[MLX4_QP_STATE_RST]	= MLX4_CMD_2RST_QP,
			[MLX4_QP_STATE_ERR]	= MLX4_CMD_2ERR_QP,
			[MLX4_QP_STATE_RTS]	= MLX4_CMD_SQERR2RTS_QP,
		},
		[MLX4_QP_STATE_ERR] = {
			[MLX4_QP_STATE_RST]	= MLX4_CMD_2RST_QP,
			[MLX4_QP_STATE_ERR]	= MLX4_CMD_2ERR_QP,
		}
	};

133
	struct mlx4_priv *priv = mlx4_priv(dev);
134 135
	struct mlx4_cmd_mailbox *mailbox;
	int ret = 0;
136 137
	int real_qp0 = 0;
	int proxy_qp0 = 0;
138
	u8 port;
139

140
	if (cur_state >= MLX4_QP_NUM_STATE || new_state >= MLX4_QP_NUM_STATE ||
141 142 143
	    !op[cur_state][new_state])
		return -EINVAL;

144 145 146 147 148
	if (op[cur_state][new_state] == MLX4_CMD_2RST_QP) {
		ret = mlx4_cmd(dev, 0, qp->qpn, 2,
			MLX4_CMD_2RST_QP, MLX4_CMD_TIME_CLASS_A, native);
		if (mlx4_is_master(dev) && cur_state != MLX4_QP_STATE_ERR &&
		    cur_state != MLX4_QP_STATE_RST &&
149
		    is_master_qp0(dev, qp, &real_qp0, &proxy_qp0)) {
150
			port = (qp->qpn & 1) + 1;
151 152 153 154
			if (proxy_qp0)
				priv->mfunc.master.qp0_state[port].proxy_qp0_active = 0;
			else
				priv->mfunc.master.qp0_state[port].qp0_active = 0;
155 156 157
		}
		return ret;
	}
158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175

	mailbox = mlx4_alloc_cmd_mailbox(dev);
	if (IS_ERR(mailbox))
		return PTR_ERR(mailbox);

	if (cur_state == MLX4_QP_STATE_RST && new_state == MLX4_QP_STATE_INIT) {
		u64 mtt_addr = mlx4_mtt_addr(dev, mtt);
		context->mtt_base_addr_h = mtt_addr >> 32;
		context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff);
		context->log_page_size   = mtt->page_shift - MLX4_ICM_PAGE_SHIFT;
	}

	*(__be32 *) mailbox->buf = cpu_to_be32(optpar);
	memcpy(mailbox->buf + 8, context, sizeof *context);

	((struct mlx4_qp_context *) (mailbox->buf + 8))->local_qpn =
		cpu_to_be32(qp->qpn);

176
	ret = mlx4_cmd(dev, mailbox->dma,
177
		       qp->qpn | (!!sqd_event << 31),
178
		       new_state == MLX4_QP_STATE_RST ? 2 : 0,
179
		       op[cur_state][new_state], MLX4_CMD_TIME_CLASS_C, native);
180

181
	if (mlx4_is_master(dev) && is_master_qp0(dev, qp, &real_qp0, &proxy_qp0)) {
182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197
		port = (qp->qpn & 1) + 1;
		if (cur_state != MLX4_QP_STATE_ERR &&
		    cur_state != MLX4_QP_STATE_RST &&
		    new_state == MLX4_QP_STATE_ERR) {
			if (proxy_qp0)
				priv->mfunc.master.qp0_state[port].proxy_qp0_active = 0;
			else
				priv->mfunc.master.qp0_state[port].qp0_active = 0;
		} else if (new_state == MLX4_QP_STATE_RTR) {
			if (proxy_qp0)
				priv->mfunc.master.qp0_state[port].proxy_qp0_active = 1;
			else
				priv->mfunc.master.qp0_state[port].qp0_active = 1;
		}
	}

198 199 200
	mlx4_free_cmd_mailbox(dev, mailbox);
	return ret;
}
201 202 203 204 205 206 207 208 209 210

int mlx4_qp_modify(struct mlx4_dev *dev, struct mlx4_mtt *mtt,
		   enum mlx4_qp_state cur_state, enum mlx4_qp_state new_state,
		   struct mlx4_qp_context *context,
		   enum mlx4_qp_optpar optpar,
		   int sqd_event, struct mlx4_qp *qp)
{
	return __mlx4_qp_modify(dev, mtt, cur_state, new_state, context,
				optpar, sqd_event, qp, 0);
}
211 212
EXPORT_SYMBOL_GPL(mlx4_qp_modify);

213
int __mlx4_qp_reserve_range(struct mlx4_dev *dev, int cnt, int align,
214
			    int *base, u8 flags)
215
{
216
	u32 uid;
217 218
	int bf_qp = !!(flags & (u8)MLX4_RESERVE_ETH_BF_QP);

219 220 221
	struct mlx4_priv *priv = mlx4_priv(dev);
	struct mlx4_qp_table *qp_table = &priv->qp_table;

222 223 224
	if (cnt > MLX4_MAX_BF_QP_RANGE && bf_qp)
		return -ENOMEM;

225 226 227 228 229 230 231 232 233 234
	uid = MLX4_QP_TABLE_ZONE_GENERAL;
	if (flags & (u8)MLX4_RESERVE_A0_QP) {
		if (bf_qp)
			uid = MLX4_QP_TABLE_ZONE_RAW_ETH;
		else
			uid = MLX4_QP_TABLE_ZONE_RSS;
	}

	*base = mlx4_zone_alloc_entries(qp_table->zones, uid, cnt, align,
					bf_qp ? MLX4_BF_QP_SKIP_MASK : 0, NULL);
235
	if (*base == -1)
236 237 238 239
		return -ENOMEM;

	return 0;
}
240

241 242
int mlx4_qp_reserve_range(struct mlx4_dev *dev, int cnt, int align,
			  int *base, u8 flags)
243
{
244
	u64 in_param = 0;
245 246 247
	u64 out_param;
	int err;

248 249 250
	/* Turn off all unsupported QP allocation flags */
	flags &= dev->caps.alloc_res_qp_mask;

251
	if (mlx4_is_mfunc(dev)) {
252
		set_param_l(&in_param, (((u32)flags) << 24) | (u32)cnt);
253 254 255 256 257 258 259 260 261 262 263
		set_param_h(&in_param, align);
		err = mlx4_cmd_imm(dev, in_param, &out_param,
				   RES_QP, RES_OP_RESERVE,
				   MLX4_CMD_ALLOC_RES,
				   MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
		if (err)
			return err;

		*base = get_param_l(&out_param);
		return 0;
	}
264
	return __mlx4_qp_reserve_range(dev, cnt, align, base, flags);
265
}
266 267
EXPORT_SYMBOL_GPL(mlx4_qp_reserve_range);

268
void __mlx4_qp_release_range(struct mlx4_dev *dev, int base_qpn, int cnt)
269 270 271 272
{
	struct mlx4_priv *priv = mlx4_priv(dev);
	struct mlx4_qp_table *qp_table = &priv->qp_table;

273 274
	if (mlx4_is_qp_reserved(dev, (u32) base_qpn))
		return;
275
	mlx4_zone_free_entries_unique(qp_table->zones, base_qpn, cnt);
276
}
277 278 279

void mlx4_qp_release_range(struct mlx4_dev *dev, int base_qpn, int cnt)
{
280
	u64 in_param = 0;
281 282
	int err;

283 284 285
	if (!cnt)
		return;

286 287 288 289 290 291 292
	if (mlx4_is_mfunc(dev)) {
		set_param_l(&in_param, base_qpn);
		set_param_h(&in_param, cnt);
		err = mlx4_cmd(dev, in_param, RES_QP, RES_OP_RESERVE,
			       MLX4_CMD_FREE_RES,
			       MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
		if (err) {
293 294
			mlx4_warn(dev, "Failed to release qp range base:%d cnt:%d\n",
				  base_qpn, cnt);
295 296 297 298
		}
	} else
		 __mlx4_qp_release_range(dev, base_qpn, cnt);
}
299 300
EXPORT_SYMBOL_GPL(mlx4_qp_release_range);

301
int __mlx4_qp_alloc_icm(struct mlx4_dev *dev, int qpn, gfp_t gfp)
302 303 304 305 306
{
	struct mlx4_priv *priv = mlx4_priv(dev);
	struct mlx4_qp_table *qp_table = &priv->qp_table;
	int err;

307
	err = mlx4_table_get(dev, &qp_table->qp_table, qpn, gfp);
308 309 310
	if (err)
		goto err_out;

311
	err = mlx4_table_get(dev, &qp_table->auxc_table, qpn, gfp);
312 313 314
	if (err)
		goto err_put_qp;

315
	err = mlx4_table_get(dev, &qp_table->altc_table, qpn, gfp);
316 317 318
	if (err)
		goto err_put_auxc;

319
	err = mlx4_table_get(dev, &qp_table->rdmarc_table, qpn, gfp);
320 321 322
	if (err)
		goto err_put_altc;

323
	err = mlx4_table_get(dev, &qp_table->cmpt_table, qpn, gfp);
324 325 326 327 328 329
	if (err)
		goto err_put_rdmarc;

	return 0;

err_put_rdmarc:
330
	mlx4_table_put(dev, &qp_table->rdmarc_table, qpn);
331 332

err_put_altc:
333
	mlx4_table_put(dev, &qp_table->altc_table, qpn);
334 335

err_put_auxc:
336
	mlx4_table_put(dev, &qp_table->auxc_table, qpn);
337 338

err_put_qp:
339
	mlx4_table_put(dev, &qp_table->qp_table, qpn);
340 341 342 343

err_out:
	return err;
}
344

345
static int mlx4_qp_alloc_icm(struct mlx4_dev *dev, int qpn, gfp_t gfp)
346
{
347
	u64 param = 0;
348 349 350 351 352 353 354

	if (mlx4_is_mfunc(dev)) {
		set_param_l(&param, qpn);
		return mlx4_cmd_imm(dev, param, &param, RES_QP, RES_OP_MAP_ICM,
				    MLX4_CMD_ALLOC_RES, MLX4_CMD_TIME_CLASS_A,
				    MLX4_CMD_WRAPPED);
	}
355
	return __mlx4_qp_alloc_icm(dev, qpn, gfp);
356 357
}

358
void __mlx4_qp_free_icm(struct mlx4_dev *dev, int qpn)
359 360 361 362 363 364 365 366 367 368 369 370 371
{
	struct mlx4_priv *priv = mlx4_priv(dev);
	struct mlx4_qp_table *qp_table = &priv->qp_table;

	mlx4_table_put(dev, &qp_table->cmpt_table, qpn);
	mlx4_table_put(dev, &qp_table->rdmarc_table, qpn);
	mlx4_table_put(dev, &qp_table->altc_table, qpn);
	mlx4_table_put(dev, &qp_table->auxc_table, qpn);
	mlx4_table_put(dev, &qp_table->qp_table, qpn);
}

static void mlx4_qp_free_icm(struct mlx4_dev *dev, int qpn)
{
372
	u64 in_param = 0;
373 374 375 376 377 378 379 380 381 382 383

	if (mlx4_is_mfunc(dev)) {
		set_param_l(&in_param, qpn);
		if (mlx4_cmd(dev, in_param, RES_QP, RES_OP_MAP_ICM,
			     MLX4_CMD_FREE_RES, MLX4_CMD_TIME_CLASS_A,
			     MLX4_CMD_WRAPPED))
			mlx4_warn(dev, "Failed to free icm of qp:%d\n", qpn);
	} else
		__mlx4_qp_free_icm(dev, qpn);
}

384 385 386 387 388
struct mlx4_qp *mlx4_qp_lookup(struct mlx4_dev *dev, u32 qpn)
{
	struct mlx4_qp_table *qp_table = &mlx4_priv(dev)->qp_table;
	struct mlx4_qp *qp;

389
	spin_lock_irq(&qp_table->lock);
390 391 392

	qp = __mlx4_qp_lookup(dev, qpn);

393
	spin_unlock_irq(&qp_table->lock);
394 395 396
	return qp;
}

397
int mlx4_qp_alloc(struct mlx4_dev *dev, int qpn, struct mlx4_qp *qp, gfp_t gfp)
398 399 400 401 402 403 404 405 406 407
{
	struct mlx4_priv *priv = mlx4_priv(dev);
	struct mlx4_qp_table *qp_table = &priv->qp_table;
	int err;

	if (!qpn)
		return -EINVAL;

	qp->qpn = qpn;

408
	err = mlx4_qp_alloc_icm(dev, qpn, gfp);
409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428
	if (err)
		return err;

	spin_lock_irq(&qp_table->lock);
	err = radix_tree_insert(&dev->qp_table_tree, qp->qpn &
				(dev->caps.num_qps - 1), qp);
	spin_unlock_irq(&qp_table->lock);
	if (err)
		goto err_icm;

	atomic_set(&qp->refcount, 1);
	init_completion(&qp->free);

	return 0;

err_icm:
	mlx4_qp_free_icm(dev, qpn);
	return err;
}

429 430
EXPORT_SYMBOL_GPL(mlx4_qp_alloc);

431
int mlx4_update_qp(struct mlx4_dev *dev, u32 qpn,
432 433 434 435 436 437
		   enum mlx4_update_qp_attr attr,
		   struct mlx4_update_qp_params *params)
{
	struct mlx4_cmd_mailbox *mailbox;
	struct mlx4_update_qp_context *cmd;
	u64 pri_addr_path_mask = 0;
438
	u64 qp_mask = 0;
439 440
	int err = 0;

441 442 443
	if (!attr || (attr & ~MLX4_UPDATE_QP_SUPPORTED_ATTRS))
		return -EINVAL;

444 445 446 447 448 449 450 451 452 453 454
	mailbox = mlx4_alloc_cmd_mailbox(dev);
	if (IS_ERR(mailbox))
		return PTR_ERR(mailbox);

	cmd = (struct mlx4_update_qp_context *)mailbox->buf;

	if (attr & MLX4_UPDATE_QP_SMAC) {
		pri_addr_path_mask |= 1ULL << MLX4_UPD_QP_PATH_MASK_MAC_INDEX;
		cmd->qp_context.pri_path.grh_mylmc = params->smac_index;
	}

455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471
	if (attr & MLX4_UPDATE_QP_ETH_SRC_CHECK_MC_LB) {
		if (!(dev->caps.flags2
		      & MLX4_DEV_CAP_FLAG2_UPDATE_QP_SRC_CHECK_LB)) {
			mlx4_warn(dev,
				  "Trying to set src check LB, but it isn't supported\n");
			err = -ENOTSUPP;
			goto out;
		}
		pri_addr_path_mask |=
			1ULL << MLX4_UPD_QP_PATH_MASK_ETH_SRC_CHECK_MC_LB;
		if (params->flags &
		    MLX4_UPDATE_QP_PARAMS_FLAGS_ETH_CHECK_MC_LB) {
			cmd->qp_context.pri_path.fl |=
				MLX4_FL_ETH_SRC_CHECK_MC_LB;
		}
	}

472 473 474 475 476 477
	if (attr & MLX4_UPDATE_QP_VSD) {
		qp_mask |= 1ULL << MLX4_UPD_QP_MASK_VSD;
		if (params->flags & MLX4_UPDATE_QP_PARAMS_FLAGS_VSD_ENABLE)
			cmd->qp_context.param3 |= cpu_to_be32(MLX4_STRIP_VLAN);
	}

478 479 480 481 482
	if (attr & MLX4_UPDATE_QP_RATE_LIMIT) {
		qp_mask |= 1ULL << MLX4_UPD_QP_MASK_RATE_LIMIT;
		cmd->qp_context.rate_limit_params = cpu_to_be16((params->rate_unit << 14) | params->rate_val);
	}

483
	if (attr & MLX4_UPDATE_QP_QOS_VPORT) {
484 485 486 487 488 489
		if (!(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_QOS_VPP)) {
			mlx4_warn(dev, "Granular QoS per VF is not enabled\n");
			err = -EOPNOTSUPP;
			goto out;
		}

490 491 492 493
		qp_mask |= 1ULL << MLX4_UPD_QP_MASK_QOS_VPP;
		cmd->qp_context.qos_vport = params->qos_vport;
	}

494
	cmd->primary_addr_path_mask = cpu_to_be64(pri_addr_path_mask);
495
	cmd->qp_mask = cpu_to_be64(qp_mask);
496

497
	err = mlx4_cmd(dev, mailbox->dma, qpn & 0xffffff, 0,
498 499
		       MLX4_CMD_UPDATE_QP, MLX4_CMD_TIME_CLASS_A,
		       MLX4_CMD_NATIVE);
500
out:
501 502 503 504 505
	mlx4_free_cmd_mailbox(dev, mailbox);
	return err;
}
EXPORT_SYMBOL_GPL(mlx4_update_qp);

506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522
void mlx4_qp_remove(struct mlx4_dev *dev, struct mlx4_qp *qp)
{
	struct mlx4_qp_table *qp_table = &mlx4_priv(dev)->qp_table;
	unsigned long flags;

	spin_lock_irqsave(&qp_table->lock, flags);
	radix_tree_delete(&dev->qp_table_tree, qp->qpn & (dev->caps.num_qps - 1));
	spin_unlock_irqrestore(&qp_table->lock, flags);
}
EXPORT_SYMBOL_GPL(mlx4_qp_remove);

void mlx4_qp_free(struct mlx4_dev *dev, struct mlx4_qp *qp)
{
	if (atomic_dec_and_test(&qp->refcount))
		complete(&qp->free);
	wait_for_completion(&qp->free);

523
	mlx4_qp_free_icm(dev, qp->qpn);
524 525 526 527 528 529
}
EXPORT_SYMBOL_GPL(mlx4_qp_free);

static int mlx4_CONF_SPECIAL_QP(struct mlx4_dev *dev, u32 base_qpn)
{
	return mlx4_cmd(dev, 0, base_qpn, 0, MLX4_CMD_CONF_SPECIAL_QP,
530
			MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE);
531 532
}

533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753
#define MLX4_QP_TABLE_RSS_ETH_PRIORITY 2
#define MLX4_QP_TABLE_RAW_ETH_PRIORITY 1
#define MLX4_QP_TABLE_RAW_ETH_SIZE     256

static int mlx4_create_zones(struct mlx4_dev *dev,
			     u32 reserved_bottom_general,
			     u32 reserved_top_general,
			     u32 reserved_bottom_rss,
			     u32 start_offset_rss,
			     u32 max_table_offset)
{
	struct mlx4_qp_table *qp_table = &mlx4_priv(dev)->qp_table;
	struct mlx4_bitmap (*bitmap)[MLX4_QP_TABLE_ZONE_NUM] = NULL;
	int bitmap_initialized = 0;
	u32 last_offset;
	int k;
	int err;

	qp_table->zones = mlx4_zone_allocator_create(MLX4_ZONE_ALLOC_FLAGS_NO_OVERLAP);

	if (NULL == qp_table->zones)
		return -ENOMEM;

	bitmap = kmalloc(sizeof(*bitmap), GFP_KERNEL);

	if (NULL == bitmap) {
		err = -ENOMEM;
		goto free_zone;
	}

	err = mlx4_bitmap_init(*bitmap + MLX4_QP_TABLE_ZONE_GENERAL, dev->caps.num_qps,
			       (1 << 23) - 1, reserved_bottom_general,
			       reserved_top_general);

	if (err)
		goto free_bitmap;

	++bitmap_initialized;

	err = mlx4_zone_add_one(qp_table->zones, *bitmap + MLX4_QP_TABLE_ZONE_GENERAL,
				MLX4_ZONE_FALLBACK_TO_HIGHER_PRIO |
				MLX4_ZONE_USE_RR, 0,
				0, qp_table->zones_uids + MLX4_QP_TABLE_ZONE_GENERAL);

	if (err)
		goto free_bitmap;

	err = mlx4_bitmap_init(*bitmap + MLX4_QP_TABLE_ZONE_RSS,
			       reserved_bottom_rss,
			       reserved_bottom_rss - 1,
			       dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],
			       reserved_bottom_rss - start_offset_rss);

	if (err)
		goto free_bitmap;

	++bitmap_initialized;

	err = mlx4_zone_add_one(qp_table->zones, *bitmap + MLX4_QP_TABLE_ZONE_RSS,
				MLX4_ZONE_ALLOW_ALLOC_FROM_LOWER_PRIO |
				MLX4_ZONE_ALLOW_ALLOC_FROM_EQ_PRIO |
				MLX4_ZONE_USE_RR, MLX4_QP_TABLE_RSS_ETH_PRIORITY,
				0, qp_table->zones_uids + MLX4_QP_TABLE_ZONE_RSS);

	if (err)
		goto free_bitmap;

	last_offset = dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW];
	/*  We have a single zone for the A0 steering QPs area of the FW. This area
	 *  needs to be split into subareas. One set of subareas is for RSS QPs
	 *  (in which qp number bits 6 and/or 7 are set); the other set of subareas
	 *  is for RAW_ETH QPs, which require that both bits 6 and 7 are zero.
	 *  Currently, the values returned by the FW (A0 steering area starting qp number
	 *  and A0 steering area size) are such that there are only two subareas -- one
	 *  for RSS and one for RAW_ETH.
	 */
	for (k = MLX4_QP_TABLE_ZONE_RSS + 1; k < sizeof(*bitmap)/sizeof((*bitmap)[0]);
	     k++) {
		int size;
		u32 offset = start_offset_rss;
		u32 bf_mask;
		u32 requested_size;

		/* Assuming MLX4_BF_QP_SKIP_MASK is consecutive ones, this calculates
		 * a mask of all LSB bits set until (and not including) the first
		 * set bit of  MLX4_BF_QP_SKIP_MASK. For example, if MLX4_BF_QP_SKIP_MASK
		 * is 0xc0, bf_mask will be 0x3f.
		 */
		bf_mask = (MLX4_BF_QP_SKIP_MASK & ~(MLX4_BF_QP_SKIP_MASK - 1)) - 1;
		requested_size = min((u32)MLX4_QP_TABLE_RAW_ETH_SIZE, bf_mask + 1);

		if (((last_offset & MLX4_BF_QP_SKIP_MASK) &&
		     ((int)(max_table_offset - last_offset)) >=
		     roundup_pow_of_two(MLX4_BF_QP_SKIP_MASK)) ||
		    (!(last_offset & MLX4_BF_QP_SKIP_MASK) &&
		     !((last_offset + requested_size - 1) &
		       MLX4_BF_QP_SKIP_MASK)))
			size = requested_size;
		else {
			u32 candidate_offset =
				(last_offset | MLX4_BF_QP_SKIP_MASK | bf_mask) + 1;

			if (last_offset & MLX4_BF_QP_SKIP_MASK)
				last_offset = candidate_offset;

			/* From this point, the BF bits are 0 */

			if (last_offset > max_table_offset) {
				/* need to skip */
				size = -1;
			} else {
				size = min3(max_table_offset - last_offset,
					    bf_mask - (last_offset & bf_mask),
					    requested_size);
				if (size < requested_size) {
					int candidate_size;

					candidate_size = min3(
						max_table_offset - candidate_offset,
						bf_mask - (last_offset & bf_mask),
						requested_size);

					/*  We will not take this path if last_offset was
					 *  already set above to candidate_offset
					 */
					if (candidate_size > size) {
						last_offset = candidate_offset;
						size = candidate_size;
					}
				}
			}
		}

		if (size > 0) {
			/* mlx4_bitmap_alloc_range will find a contiguous range of "size"
			 * QPs in which both bits 6 and 7 are zero, because we pass it the
			 * MLX4_BF_SKIP_MASK).
			 */
			offset = mlx4_bitmap_alloc_range(
					*bitmap + MLX4_QP_TABLE_ZONE_RSS,
					size, 1,
					MLX4_BF_QP_SKIP_MASK);

			if (offset == (u32)-1) {
				err = -ENOMEM;
				break;
			}

			last_offset = offset + size;

			err = mlx4_bitmap_init(*bitmap + k, roundup_pow_of_two(size),
					       roundup_pow_of_two(size) - 1, 0,
					       roundup_pow_of_two(size) - size);
		} else {
			/* Add an empty bitmap, we'll allocate from different zones (since
			 * at least one is reserved)
			 */
			err = mlx4_bitmap_init(*bitmap + k, 1,
					       MLX4_QP_TABLE_RAW_ETH_SIZE - 1, 0,
					       0);
			mlx4_bitmap_alloc_range(*bitmap + k, 1, 1, 0);
		}

		if (err)
			break;

		++bitmap_initialized;

		err = mlx4_zone_add_one(qp_table->zones, *bitmap + k,
					MLX4_ZONE_ALLOW_ALLOC_FROM_LOWER_PRIO |
					MLX4_ZONE_ALLOW_ALLOC_FROM_EQ_PRIO |
					MLX4_ZONE_USE_RR, MLX4_QP_TABLE_RAW_ETH_PRIORITY,
					offset, qp_table->zones_uids + k);

		if (err)
			break;
	}

	if (err)
		goto free_bitmap;

	qp_table->bitmap_gen = *bitmap;

	return err;

free_bitmap:
	for (k = 0; k < bitmap_initialized; k++)
		mlx4_bitmap_cleanup(*bitmap + k);
	kfree(bitmap);
free_zone:
	mlx4_zone_allocator_destroy(qp_table->zones);
	return err;
}

static void mlx4_cleanup_qp_zones(struct mlx4_dev *dev)
{
	struct mlx4_qp_table *qp_table = &mlx4_priv(dev)->qp_table;

	if (qp_table->zones) {
		int i;

		for (i = 0;
		     i < sizeof(qp_table->zones_uids)/sizeof(qp_table->zones_uids[0]);
		     i++) {
			struct mlx4_bitmap *bitmap =
				mlx4_zone_get_bitmap(qp_table->zones,
						     qp_table->zones_uids[i]);

			mlx4_zone_remove_one(qp_table->zones, qp_table->zones_uids[i]);
			if (NULL == bitmap)
				continue;

			mlx4_bitmap_cleanup(bitmap);
		}
		mlx4_zone_allocator_destroy(qp_table->zones);
		kfree(qp_table->bitmap_gen);
		qp_table->bitmap_gen = NULL;
		qp_table->zones = NULL;
	}
}

754
int mlx4_init_qp_table(struct mlx4_dev *dev)
755 756 757
{
	struct mlx4_qp_table *qp_table = &mlx4_priv(dev)->qp_table;
	int err;
758
	int reserved_from_top = 0;
759
	int reserved_from_bot;
760
	int k;
761 762
	int fixed_reserved_from_bot_rv = 0;
	int bottom_reserved_for_rss_bitmap;
763 764
	u32 max_table_offset = dev->caps.dmfs_high_rate_qpn_base +
			dev->caps.dmfs_high_rate_qpn_range;
765 766 767

	spin_lock_init(&qp_table->lock);
	INIT_RADIX_TREE(&dev->qp_table_tree, GFP_ATOMIC);
768 769
	if (mlx4_is_slave(dev))
		return 0;
770

771
	/* We reserve 2 extra QPs per port for the special QPs.  The
772 773
	 * block of special QPs must be aligned to a multiple of 8, so
	 * round up.
774 775 776
	 *
	 * We also reserve the MSB of the 24-bit QP number to indicate
	 * that a QP is an XRC QP.
777
	 */
778 779 780 781 782 783 784 785 786 787
	for (k = 0; k <= MLX4_QP_REGION_BOTTOM; k++)
		fixed_reserved_from_bot_rv += dev->caps.reserved_qps_cnt[k];

	if (fixed_reserved_from_bot_rv < max_table_offset)
		fixed_reserved_from_bot_rv = max_table_offset;

	/* We reserve at least 1 extra for bitmaps that we don't have enough space for*/
	bottom_reserved_for_rss_bitmap =
		roundup_pow_of_two(fixed_reserved_from_bot_rv + 1);
	dev->phys_caps.base_sqpn = ALIGN(bottom_reserved_for_rss_bitmap, 8);
788 789 790

	{
		int sort[MLX4_NUM_QP_REGION];
791
		int i, j;
792 793 794 795 796
		int last_base = dev->caps.num_qps;

		for (i = 1; i < MLX4_NUM_QP_REGION; ++i)
			sort[i] = i;

797 798
		for (i = MLX4_NUM_QP_REGION; i > MLX4_QP_REGION_BOTTOM; --i) {
			for (j = MLX4_QP_REGION_BOTTOM + 2; j < i; ++j) {
799
				if (dev->caps.reserved_qps_cnt[sort[j]] >
800 801
				    dev->caps.reserved_qps_cnt[sort[j - 1]])
					swap(sort[j], sort[j - 1]);
802 803 804
			}
		}

805
		for (i = MLX4_QP_REGION_BOTTOM + 1; i < MLX4_NUM_QP_REGION; ++i) {
806 807 808 809 810 811 812
			last_base -= dev->caps.reserved_qps_cnt[sort[i]];
			dev->caps.reserved_qps_base[sort[i]] = last_base;
			reserved_from_top +=
				dev->caps.reserved_qps_cnt[sort[i]];
		}
	}

813 814 815 816 817 818 819 820 821 822
       /* Reserve 8 real SQPs in both native and SRIOV modes.
	* In addition, in SRIOV mode, reserve 8 proxy SQPs per function
	* (for all PFs and VFs), and 8 corresponding tunnel QPs.
	* Each proxy SQP works opposite its own tunnel QP.
	*
	* The QPs are arranged as follows:
	* a. 8 real SQPs
	* b. All the proxy SQPs (8 per function)
	* c. All the tunnel QPs (8 per function)
	*/
823 824 825 826 827
	reserved_from_bot = mlx4_num_reserved_sqps(dev);
	if (reserved_from_bot + reserved_from_top > dev->caps.num_qps) {
		mlx4_err(dev, "Number of reserved QPs is higher than number of QPs\n");
		return -EINVAL;
	}
828

829 830 831 832 833
	err = mlx4_create_zones(dev, reserved_from_bot, reserved_from_bot,
				bottom_reserved_for_rss_bitmap,
				fixed_reserved_from_bot_rv,
				max_table_offset);

834 835
	if (err)
		return err;
836 837

	if (mlx4_is_mfunc(dev)) {
838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853
		/* for PPF use */
		dev->phys_caps.base_proxy_sqpn = dev->phys_caps.base_sqpn + 8;
		dev->phys_caps.base_tunnel_sqpn = dev->phys_caps.base_sqpn + 8 + 8 * MLX4_MFUNC_MAX;

		/* In mfunc, calculate proxy and tunnel qp offsets for the PF here,
		 * since the PF does not call mlx4_slave_caps */
		dev->caps.qp0_tunnel = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL);
		dev->caps.qp0_proxy = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL);
		dev->caps.qp1_tunnel = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL);
		dev->caps.qp1_proxy = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL);

		if (!dev->caps.qp0_tunnel || !dev->caps.qp0_proxy ||
		    !dev->caps.qp1_tunnel || !dev->caps.qp1_proxy) {
			err = -ENOMEM;
			goto err_mem;
		}
854

855 856 857 858 859 860 861 862 863 864 865 866
		for (k = 0; k < dev->caps.num_ports; k++) {
			dev->caps.qp0_proxy[k] = dev->phys_caps.base_proxy_sqpn +
				8 * mlx4_master_func_num(dev) + k;
			dev->caps.qp0_tunnel[k] = dev->caps.qp0_proxy[k] + 8 * MLX4_MFUNC_MAX;
			dev->caps.qp1_proxy[k] = dev->phys_caps.base_proxy_sqpn +
				8 * mlx4_master_func_num(dev) + MLX4_MAX_PORTS + k;
			dev->caps.qp1_tunnel[k] = dev->caps.qp1_proxy[k] + 8 * MLX4_MFUNC_MAX;
		}
	}


	err = mlx4_CONF_SPECIAL_QP(dev, dev->phys_caps.base_sqpn);
867
	if (err)
868
		goto err_mem;
869 870

	return err;
871

872 873 874 875 876 877 878
err_mem:
	kfree(dev->caps.qp0_tunnel);
	kfree(dev->caps.qp0_proxy);
	kfree(dev->caps.qp1_tunnel);
	kfree(dev->caps.qp1_proxy);
	dev->caps.qp0_tunnel = dev->caps.qp0_proxy =
		dev->caps.qp1_tunnel = dev->caps.qp1_proxy = NULL;
879
	mlx4_cleanup_qp_zones(dev);
880
	return err;
881 882 883 884
}

void mlx4_cleanup_qp_table(struct mlx4_dev *dev)
{
885 886 887
	if (mlx4_is_slave(dev))
		return;

888
	mlx4_CONF_SPECIAL_QP(dev, 0);
889 890

	mlx4_cleanup_qp_zones(dev);
891
}
892 893 894 895 896 897 898 899 900 901 902 903

int mlx4_qp_query(struct mlx4_dev *dev, struct mlx4_qp *qp,
		  struct mlx4_qp_context *context)
{
	struct mlx4_cmd_mailbox *mailbox;
	int err;

	mailbox = mlx4_alloc_cmd_mailbox(dev);
	if (IS_ERR(mailbox))
		return PTR_ERR(mailbox);

	err = mlx4_cmd_box(dev, 0, mailbox->dma, qp->qpn, 0,
904 905
			   MLX4_CMD_QUERY_QP, MLX4_CMD_TIME_CLASS_A,
			   MLX4_CMD_WRAPPED);
906 907 908 909 910 911 912 913
	if (!err)
		memcpy(context, mailbox->buf + 8, sizeof *context);

	mlx4_free_cmd_mailbox(dev, mailbox);
	return err;
}
EXPORT_SYMBOL_GPL(mlx4_qp_query);

914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929
int mlx4_qp_to_ready(struct mlx4_dev *dev, struct mlx4_mtt *mtt,
		     struct mlx4_qp_context *context,
		     struct mlx4_qp *qp, enum mlx4_qp_state *qp_state)
{
	int err;
	int i;
	enum mlx4_qp_state states[] = {
		MLX4_QP_STATE_RST,
		MLX4_QP_STATE_INIT,
		MLX4_QP_STATE_RTR,
		MLX4_QP_STATE_RTS
	};

	for (i = 0; i < ARRAY_SIZE(states) - 1; i++) {
		context->flags &= cpu_to_be32(~(0xf << 28));
		context->flags |= cpu_to_be32(states[i + 1] << 28);
930 931
		if (states[i + 1] != MLX4_QP_STATE_RTR)
			context->params2 &= ~MLX4_QP_BIT_FPP;
932 933 934
		err = mlx4_qp_modify(dev, mtt, states[i], states[i + 1],
				     context, 0, 0, qp);
		if (err) {
935
			mlx4_err(dev, "Failed to bring QP to state: %d with error: %d\n",
936 937 938 939 940 941 942 943 944 945
				 states[i + 1], err);
			return err;
		}

		*qp_state = states[i + 1];
	}

	return 0;
}
EXPORT_SYMBOL_GPL(mlx4_qp_to_ready);