ib_isert.c 71.3 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0-or-later
2 3 4
/*******************************************************************************
 * This file contains iSCSI extentions for RDMA (iSER) Verbs
 *
5
 * (c) Copyright 2013 Datera, Inc.
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
 *
 * Nicholas A. Bellinger <nab@linux-iscsi.org>
 *
 ****************************************************************************/

#include <linux/string.h>
#include <linux/module.h>
#include <linux/scatterlist.h>
#include <linux/socket.h>
#include <linux/in.h>
#include <linux/in6.h>
#include <rdma/ib_verbs.h>
#include <rdma/rdma_cm.h>
#include <target/target_core_base.h>
#include <target/target_core_fabric.h>
#include <target/iscsi/iscsi_transport.h>
22
#include <linux/semaphore.h>
23 24 25 26 27

#include "ib_isert.h"

#define	ISERT_MAX_CONN		8
#define ISER_MAX_RX_CQ_LEN	(ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN)
28 29
#define ISER_MAX_TX_CQ_LEN \
	((ISERT_QP_MAX_REQ_DTOS + ISCSI_DEF_XMIT_CMDS_MAX) * ISERT_MAX_CONN)
30 31
#define ISER_MAX_CQ_LEN		(ISER_MAX_RX_CQ_LEN + ISER_MAX_TX_CQ_LEN + \
				 ISERT_MAX_CONN)
32

33
static int isert_debug_level;
34 35 36
module_param_named(debug_level, isert_debug_level, int, 0644);
MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0 (default:0)");

37 38 39
static DEFINE_MUTEX(device_list_mutex);
static LIST_HEAD(device_list);
static struct workqueue_struct *isert_comp_wq;
40
static struct workqueue_struct *isert_release_wq;
41

42 43
static int
isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd);
44
static int
45
isert_login_post_recv(struct isert_conn *isert_conn);
46 47
static int
isert_rdma_accept(struct isert_conn *isert_conn);
48
struct rdma_cm_id *isert_setup_id(struct isert_np *isert_np);
49

50
static void isert_release_work(struct work_struct *work);
51 52 53 54
static void isert_recv_done(struct ib_cq *cq, struct ib_wc *wc);
static void isert_send_done(struct ib_cq *cq, struct ib_wc *wc);
static void isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc);
static void isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc);
55

56 57 58
static inline bool
isert_prot_cmd(struct isert_conn *conn, struct se_cmd *cmd)
{
59
	return (conn->pi_support &&
60 61 62 63
		cmd->prot_op != TARGET_PROT_NORMAL);
}


64 65 66
static void
isert_qp_event_callback(struct ib_event *e, void *context)
{
67
	struct isert_conn *isert_conn = context;
68

69 70 71
	isert_err("%s (%d): conn %p\n",
		  ib_event_msg(e->event), e->event, isert_conn);

72 73
	switch (e->event) {
	case IB_EVENT_COMM_EST:
74
		rdma_notify(isert_conn->cm_id, IB_EVENT_COMM_EST);
75 76
		break;
	case IB_EVENT_QP_LAST_WQE_REACHED:
77
		isert_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED\n");
78 79 80 81 82 83
		break;
	default:
		break;
	}
}

84 85
static struct isert_comp *
isert_comp_get(struct isert_conn *isert_conn)
86
{
87
	struct isert_device *device = isert_conn->device;
88
	struct isert_comp *comp;
89
	int i, min = 0;
90 91

	mutex_lock(&device_list_mutex);
92 93 94 95 96 97
	for (i = 0; i < device->comps_used; i++)
		if (device->comps[i].active_qps <
		    device->comps[min].active_qps)
			min = i;
	comp = &device->comps[min];
	comp->active_qps++;
98 99
	mutex_unlock(&device_list_mutex);

100
	isert_info("conn %p, using comp %p min_index: %d\n",
101
		   isert_conn, comp, min);
102 103 104 105 106 107 108 109 110

	return comp;
}

static void
isert_comp_put(struct isert_comp *comp)
{
	mutex_lock(&device_list_mutex);
	comp->active_qps--;
111
	mutex_unlock(&device_list_mutex);
112 113 114 115 116 117 118
}

static struct ib_qp *
isert_create_qp(struct isert_conn *isert_conn,
		struct isert_comp *comp,
		struct rdma_cm_id *cma_id)
{
119
	struct isert_device *device = isert_conn->device;
120 121
	struct ib_qp_init_attr attr;
	int ret;
122 123 124 125

	memset(&attr, 0, sizeof(struct ib_qp_init_attr));
	attr.event_handler = isert_qp_event_callback;
	attr.qp_context = isert_conn;
126 127
	attr.send_cq = comp->cq;
	attr.recv_cq = comp->cq;
128
	attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS + 1;
129
	attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1;
130
	attr.cap.max_rdma_ctxs = ISCSI_DEF_XMIT_CMDS_MAX;
131
	attr.cap.max_send_sge = device->ib_device->attrs.max_send_sge;
132 133 134
	attr.cap.max_recv_sge = 1;
	attr.sq_sig_type = IB_SIGNAL_REQ_WR;
	attr.qp_type = IB_QPT_RC;
135
	if (device->pi_capable)
136
		attr.create_flags |= IB_QP_CREATE_INTEGRITY_EN;
137

138
	ret = rdma_create_qp(cma_id, device->pd, &attr);
139
	if (ret) {
140
		isert_err("rdma_create_qp failed for cma_id %d\n", ret);
141 142 143 144 145 146 147 148 149 150 151 152 153
		return ERR_PTR(ret);
	}

	return cma_id->qp;
}

static int
isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id)
{
	struct isert_comp *comp;
	int ret;

	comp = isert_comp_get(isert_conn);
154 155 156
	isert_conn->qp = isert_create_qp(isert_conn, comp, cma_id);
	if (IS_ERR(isert_conn->qp)) {
		ret = PTR_ERR(isert_conn->qp);
157
		goto err;
158 159 160
	}

	return 0;
161
err:
162
	isert_comp_put(comp);
163
	return ret;
164 165 166 167 168
}

static int
isert_alloc_rx_descriptors(struct isert_conn *isert_conn)
{
169
	struct isert_device *device = isert_conn->device;
170
	struct ib_device *ib_dev = device->ib_device;
171 172 173 174 175
	struct iser_rx_desc *rx_desc;
	struct ib_sge *rx_sg;
	u64 dma_addr;
	int i, j;

176 177 178
	isert_conn->rx_descs = kcalloc(ISERT_QP_MAX_RECV_DTOS,
				       sizeof(struct iser_rx_desc),
				       GFP_KERNEL);
179
	if (!isert_conn->rx_descs)
180
		return -ENOMEM;
181

182
	rx_desc = isert_conn->rx_descs;
183 184 185 186 187 188 189 190 191 192 193 194

	for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++)  {
		dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc,
					ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
		if (ib_dma_mapping_error(ib_dev, dma_addr))
			goto dma_map_fail;

		rx_desc->dma_addr = dma_addr;

		rx_sg = &rx_desc->rx_sg;
		rx_sg->addr = rx_desc->dma_addr;
		rx_sg->length = ISER_RX_PAYLOAD_SIZE;
195
		rx_sg->lkey = device->pd->local_dma_lkey;
196
		rx_desc->rx_cqe.done = isert_recv_done;
197 198 199 200 201
	}

	return 0;

dma_map_fail:
202
	rx_desc = isert_conn->rx_descs;
203 204 205 206
	for (j = 0; j < i; j++, rx_desc++) {
		ib_dma_unmap_single(ib_dev, rx_desc->dma_addr,
				    ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
	}
207 208
	kfree(isert_conn->rx_descs);
	isert_conn->rx_descs = NULL;
209
	isert_err("conn %p failed to allocate rx descriptors\n", isert_conn);
210 211 212 213 214 215
	return -ENOMEM;
}

static void
isert_free_rx_descriptors(struct isert_conn *isert_conn)
{
216
	struct ib_device *ib_dev = isert_conn->device->ib_device;
217 218 219
	struct iser_rx_desc *rx_desc;
	int i;

220
	if (!isert_conn->rx_descs)
221 222
		return;

223
	rx_desc = isert_conn->rx_descs;
224 225 226 227 228
	for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++)  {
		ib_dma_unmap_single(ib_dev, rx_desc->dma_addr,
				    ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
	}

229 230
	kfree(isert_conn->rx_descs);
	isert_conn->rx_descs = NULL;
231 232
}

233 234
static void
isert_free_comps(struct isert_device *device)
235
{
236
	int i;
237

238 239
	for (i = 0; i < device->comps_used; i++) {
		struct isert_comp *comp = &device->comps[i];
240

241 242
		if (comp->cq)
			ib_free_cq(comp->cq);
243
	}
244 245
	kfree(device->comps);
}
246

247
static int
248
isert_alloc_comps(struct isert_device *device)
249 250
{
	int i, max_cqe, ret = 0;
251

252
	device->comps_used = min(ISERT_MAX_CQ, min_t(int, num_online_cpus(),
253 254
				 device->ib_device->num_comp_vectors));

255
	isert_info("Using %d CQs, %s supports %d vectors support "
256
		   "pi_capable %d\n",
257
		   device->comps_used, dev_name(&device->ib_device->dev),
258
		   device->ib_device->num_comp_vectors,
259 260 261 262
		   device->pi_capable);

	device->comps = kcalloc(device->comps_used, sizeof(struct isert_comp),
				GFP_KERNEL);
263
	if (!device->comps)
264
		return -ENOMEM;
265

266
	max_cqe = min(ISER_MAX_CQ_LEN, device->ib_device->attrs.max_cqe);
267

268 269 270 271
	for (i = 0; i < device->comps_used; i++) {
		struct isert_comp *comp = &device->comps[i];

		comp->device = device;
272 273
		comp->cq = ib_alloc_cq(device->ib_device, comp, max_cqe, i,
				IB_POLL_WORKQUEUE);
274
		if (IS_ERR(comp->cq)) {
275
			isert_err("Unable to allocate cq\n");
276 277
			ret = PTR_ERR(comp->cq);
			comp->cq = NULL;
278
			goto out_cq;
279
		}
280 281
	}

282 283 284 285 286 287 288 289 290
	return 0;
out_cq:
	isert_free_comps(device);
	return ret;
}

static int
isert_create_device_ib_res(struct isert_device *device)
{
291
	struct ib_device *ib_dev = device->ib_device;
292
	int ret;
293

294 295
	isert_dbg("devattr->max_send_sge: %d devattr->max_recv_sge %d\n",
		  ib_dev->attrs.max_send_sge, ib_dev->attrs.max_recv_sge);
296
	isert_dbg("devattr->max_sge_rd: %d\n", ib_dev->attrs.max_sge_rd);
297

298
	ret = isert_alloc_comps(device);
299
	if (ret)
300
		goto out;
301

302
	device->pd = ib_alloc_pd(ib_dev, 0);
303 304 305 306 307 308 309
	if (IS_ERR(device->pd)) {
		ret = PTR_ERR(device->pd);
		isert_err("failed to allocate pd, device %p, ret=%d\n",
			  device, ret);
		goto out_cq;
	}

310
	/* Check signature cap */
311
	device->pi_capable = ib_dev->attrs.device_cap_flags &
312
			     IB_DEVICE_INTEGRITY_HANDOVER ? true : false;
313

314 315 316
	return 0;

out_cq:
317
	isert_free_comps(device);
318 319 320
out:
	if (ret > 0)
		ret = -EINVAL;
321 322 323 324 325 326
	return ret;
}

static void
isert_free_device_ib_res(struct isert_device *device)
{
327
	isert_info("device %p\n", device);
328

329
	ib_dealloc_pd(device->pd);
330
	isert_free_comps(device);
331 332 333
}

static void
334
isert_device_put(struct isert_device *device)
335 336 337
{
	mutex_lock(&device_list_mutex);
	device->refcount--;
338
	isert_info("device %p refcount %d\n", device, device->refcount);
339 340 341 342 343 344 345 346 347
	if (!device->refcount) {
		isert_free_device_ib_res(device);
		list_del(&device->dev_node);
		kfree(device);
	}
	mutex_unlock(&device_list_mutex);
}

static struct isert_device *
348
isert_device_get(struct rdma_cm_id *cma_id)
349 350 351 352 353 354 355 356
{
	struct isert_device *device;
	int ret;

	mutex_lock(&device_list_mutex);
	list_for_each_entry(device, &device_list, dev_node) {
		if (device->ib_device->node_guid == cma_id->device->node_guid) {
			device->refcount++;
357 358
			isert_info("Found iser device %p refcount %d\n",
				   device, device->refcount);
359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381
			mutex_unlock(&device_list_mutex);
			return device;
		}
	}

	device = kzalloc(sizeof(struct isert_device), GFP_KERNEL);
	if (!device) {
		mutex_unlock(&device_list_mutex);
		return ERR_PTR(-ENOMEM);
	}

	INIT_LIST_HEAD(&device->dev_node);

	device->ib_device = cma_id->device;
	ret = isert_create_device_ib_res(device);
	if (ret) {
		kfree(device);
		mutex_unlock(&device_list_mutex);
		return ERR_PTR(ret);
	}

	device->refcount++;
	list_add_tail(&device->dev_node, &device_list);
382 383
	isert_info("Created a new iser device %p refcount %d\n",
		   device, device->refcount);
384 385 386 387 388
	mutex_unlock(&device_list_mutex);

	return device;
}

389 390
static void
isert_init_conn(struct isert_conn *isert_conn)
391 392
{
	isert_conn->state = ISER_CONN_INIT;
393
	INIT_LIST_HEAD(&isert_conn->node);
394
	init_completion(&isert_conn->login_comp);
395
	init_completion(&isert_conn->login_req_comp);
396
	init_waitqueue_head(&isert_conn->rem_wait);
397 398
	kref_init(&isert_conn->kref);
	mutex_init(&isert_conn->mutex);
399
	INIT_WORK(&isert_conn->release_work, isert_release_work);
400
}
401

402 403 404
static void
isert_free_login_buf(struct isert_conn *isert_conn)
{
405
	struct ib_device *ib_dev = isert_conn->device->ib_device;
406 407

	ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma,
408
			    ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE);
409 410
	kfree(isert_conn->login_rsp_buf);

411
	ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma,
412
			    ISER_RX_PAYLOAD_SIZE,
413
			    DMA_FROM_DEVICE);
414
	kfree(isert_conn->login_req_buf);
415 416 417 418 419 420 421
}

static int
isert_alloc_login_buf(struct isert_conn *isert_conn,
		      struct ib_device *ib_dev)
{
	int ret;
422

423 424
	isert_conn->login_req_buf = kzalloc(sizeof(*isert_conn->login_req_buf),
			GFP_KERNEL);
425
	if (!isert_conn->login_req_buf)
426
		return -ENOMEM;
427 428

	isert_conn->login_req_dma = ib_dma_map_single(ib_dev,
429 430
				isert_conn->login_req_buf,
				ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
431 432
	ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma);
	if (ret) {
433
		isert_err("login_req_dma mapping error: %d\n", ret);
434
		isert_conn->login_req_dma = 0;
435 436 437 438 439
		goto out_free_login_req_buf;
	}

	isert_conn->login_rsp_buf = kzalloc(ISER_RX_PAYLOAD_SIZE, GFP_KERNEL);
	if (!isert_conn->login_rsp_buf) {
440
		ret = -ENOMEM;
441
		goto out_unmap_login_req_buf;
442 443 444
	}

	isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev,
445
					isert_conn->login_rsp_buf,
446
					ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE);
447 448
	ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma);
	if (ret) {
449
		isert_err("login_rsp_dma mapping error: %d\n", ret);
450
		isert_conn->login_rsp_dma = 0;
451
		goto out_free_login_rsp_buf;
452 453
	}

454 455
	return 0;

456 457 458
out_free_login_rsp_buf:
	kfree(isert_conn->login_rsp_buf);
out_unmap_login_req_buf:
459
	ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma,
460 461 462
			    ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
out_free_login_req_buf:
	kfree(isert_conn->login_req_buf);
463 464 465
	return ret;
}

466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491
static void
isert_set_nego_params(struct isert_conn *isert_conn,
		      struct rdma_conn_param *param)
{
	struct ib_device_attr *attr = &isert_conn->device->ib_device->attrs;

	/* Set max inflight RDMA READ requests */
	isert_conn->initiator_depth = min_t(u8, param->initiator_depth,
				attr->max_qp_init_rd_atom);
	isert_dbg("Using initiator_depth: %u\n", isert_conn->initiator_depth);

	if (param->private_data) {
		u8 flags = *(u8 *)param->private_data;

		/*
		 * use remote invalidation if the both initiator
		 * and the HCA support it
		 */
		isert_conn->snd_w_inv = !(flags & ISER_SEND_W_INV_NOT_SUP) &&
					  (attr->device_cap_flags &
					   IB_DEVICE_MEM_MGT_EXTENSIONS);
		if (isert_conn->snd_w_inv)
			isert_info("Using remote invalidation\n");
	}
}

492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516
static int
isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
{
	struct isert_np *isert_np = cma_id->context;
	struct iscsi_np *np = isert_np->np;
	struct isert_conn *isert_conn;
	struct isert_device *device;
	int ret = 0;

	spin_lock_bh(&np->np_thread_lock);
	if (!np->enabled) {
		spin_unlock_bh(&np->np_thread_lock);
		isert_dbg("iscsi_np is not enabled, reject connect request\n");
		return rdma_reject(cma_id, NULL, 0);
	}
	spin_unlock_bh(&np->np_thread_lock);

	isert_dbg("cma_id: %p, portal: %p\n",
		 cma_id, cma_id->context);

	isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL);
	if (!isert_conn)
		return -ENOMEM;

	isert_init_conn(isert_conn);
517
	isert_conn->cm_id = cma_id;
518 519 520 521 522

	ret = isert_alloc_login_buf(isert_conn, cma_id->device);
	if (ret)
		goto out;

523
	device = isert_device_get(cma_id);
524 525 526 527
	if (IS_ERR(device)) {
		ret = PTR_ERR(device);
		goto out_rsp_dma_map;
	}
528
	isert_conn->device = device;
529

530
	isert_set_nego_params(isert_conn, &event->param.conn);
531

532
	ret = isert_conn_setup_qp(isert_conn, cma_id);
533 534 535
	if (ret)
		goto out_conn_dev;

536
	ret = isert_login_post_recv(isert_conn);
537 538 539 540 541 542 543
	if (ret)
		goto out_conn_dev;

	ret = isert_rdma_accept(isert_conn);
	if (ret)
		goto out_conn_dev;

544
	mutex_lock(&isert_np->mutex);
545
	list_add_tail(&isert_conn->node, &isert_np->accepted);
546
	mutex_unlock(&isert_np->mutex);
547 548 549 550

	return 0;

out_conn_dev:
551
	isert_device_put(device);
552
out_rsp_dma_map:
553
	isert_free_login_buf(isert_conn);
554 555
out:
	kfree(isert_conn);
556
	rdma_reject(cma_id, NULL, 0);
557 558 559 560 561 562
	return ret;
}

static void
isert_connect_release(struct isert_conn *isert_conn)
{
563
	struct isert_device *device = isert_conn->device;
564

565
	isert_dbg("conn %p\n", isert_conn);
566

567 568
	BUG_ON(!device);

569
	isert_free_rx_descriptors(isert_conn);
570 571
	if (isert_conn->cm_id &&
	    !isert_conn->dev_removed)
572
		rdma_destroy_id(isert_conn->cm_id);
573

574 575
	if (isert_conn->qp) {
		struct isert_comp *comp = isert_conn->qp->recv_cq->cq_context;
576

577
		isert_comp_put(comp);
578
		ib_destroy_qp(isert_conn->qp);
579 580
	}

581
	if (isert_conn->login_req_buf)
582 583
		isert_free_login_buf(isert_conn);

584
	isert_device_put(device);
585

586 587 588 589
	if (isert_conn->dev_removed)
		wake_up_interruptible(&isert_conn->rem_wait);
	else
		kfree(isert_conn);
590 591 592 593 594
}

static void
isert_connected_handler(struct rdma_cm_id *cma_id)
{
595
	struct isert_conn *isert_conn = cma_id->qp->qp_context;
596
	struct isert_np *isert_np = cma_id->context;
597

598
	isert_info("conn %p\n", isert_conn);
599

600
	mutex_lock(&isert_conn->mutex);
601 602
	isert_conn->state = ISER_CONN_UP;
	kref_get(&isert_conn->kref);
603
	mutex_unlock(&isert_conn->mutex);
604 605 606 607 608 609 610

	mutex_lock(&isert_np->mutex);
	list_move_tail(&isert_conn->node, &isert_np->pending);
	mutex_unlock(&isert_np->mutex);

	isert_info("np %p: Allow accept_np to continue\n", isert_np);
	up(&isert_np->sem);
611 612 613
}

static void
614
isert_release_kref(struct kref *kref)
615 616
{
	struct isert_conn *isert_conn = container_of(kref,
617
				struct isert_conn, kref);
618

619 620
	isert_info("conn %p final kref %s/%d\n", isert_conn, current->comm,
		   current->pid);
621 622 623 624 625 626 627

	isert_connect_release(isert_conn);
}

static void
isert_put_conn(struct isert_conn *isert_conn)
{
628
	kref_put(&isert_conn->kref, isert_release_kref);
629 630
}

631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648
static void
isert_handle_unbound_conn(struct isert_conn *isert_conn)
{
	struct isert_np *isert_np = isert_conn->cm_id->context;

	mutex_lock(&isert_np->mutex);
	if (!list_empty(&isert_conn->node)) {
		/*
		 * This means iscsi doesn't know this connection
		 * so schedule a cleanup ourselves
		 */
		list_del_init(&isert_conn->node);
		isert_put_conn(isert_conn);
		queue_work(isert_release_wq, &isert_conn->release_work);
	}
	mutex_unlock(&isert_np->mutex);
}

649 650 651 652 653
/**
 * isert_conn_terminate() - Initiate connection termination
 * @isert_conn: isert connection struct
 *
 * Notes:
654
 * In case the connection state is BOUND, move state
655
 * to TEMINATING and start teardown sequence (rdma_disconnect).
656
 * In case the connection state is UP, complete flush as well.
657
 *
658
 * This routine must be called with mutex held. Thus it is
659 660 661 662 663 664 665
 * safe to call multiple times.
 */
static void
isert_conn_terminate(struct isert_conn *isert_conn)
{
	int err;

666 667 668 669 670 671 672 673 674 675
	if (isert_conn->state >= ISER_CONN_TERMINATING)
		return;

	isert_info("Terminating conn %p state %d\n",
		   isert_conn, isert_conn->state);
	isert_conn->state = ISER_CONN_TERMINATING;
	err = rdma_disconnect(isert_conn->cm_id);
	if (err)
		isert_warn("Failed rdma_disconnect isert_conn %p\n",
			   isert_conn);
676 677
}

678
static int
679 680
isert_np_cma_handler(struct isert_np *isert_np,
		     enum rdma_cm_event_type event)
681
{
682 683
	isert_dbg("%s (%d): isert np %p\n",
		  rdma_event_msg(event), event, isert_np);
684

685 686
	switch (event) {
	case RDMA_CM_EVENT_DEVICE_REMOVAL:
687
		isert_np->cm_id = NULL;
688 689
		break;
	case RDMA_CM_EVENT_ADDR_CHANGE:
690 691
		isert_np->cm_id = isert_setup_id(isert_np);
		if (IS_ERR(isert_np->cm_id)) {
692
			isert_err("isert np %p setup id failed: %ld\n",
693 694
				  isert_np, PTR_ERR(isert_np->cm_id));
			isert_np->cm_id = NULL;
695 696 697
		}
		break;
	default:
698
		isert_err("isert np %p Unexpected event %d\n",
699
			  isert_np, event);
700 701
	}

702 703 704 705 706 707 708
	return -1;
}

static int
isert_disconnected_handler(struct rdma_cm_id *cma_id,
			   enum rdma_cm_event_type event)
{
709
	struct isert_conn *isert_conn = cma_id->qp->qp_context;
710

711
	mutex_lock(&isert_conn->mutex);
712 713 714 715 716
	switch (isert_conn->state) {
	case ISER_CONN_TERMINATING:
		break;
	case ISER_CONN_UP:
		isert_conn_terminate(isert_conn);
717
		ib_drain_qp(isert_conn->qp);
718 719 720 721 722 723 724
		isert_handle_unbound_conn(isert_conn);
		break;
	case ISER_CONN_BOUND:
	case ISER_CONN_FULL_FEATURE: /* FALLTHRU */
		iscsit_cause_connection_reinstatement(isert_conn->conn, 0);
		break;
	default:
725
		isert_warn("conn %p terminating in state %d\n",
726
			   isert_conn, isert_conn->state);
727
	}
728
	mutex_unlock(&isert_conn->mutex);
729

730
	return 0;
731 732
}

733
static int
734 735
isert_connect_error(struct rdma_cm_id *cma_id)
{
736
	struct isert_conn *isert_conn = cma_id->qp->qp_context;
737

738
	ib_drain_qp(isert_conn->qp);
739
	list_del_init(&isert_conn->node);
740
	isert_conn->cm_id = NULL;
741
	isert_put_conn(isert_conn);
742 743

	return -1;
744 745
}

746 747 748
static int
isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
{
749
	struct isert_np *isert_np = cma_id->context;
750
	struct isert_conn *isert_conn;
751 752
	int ret = 0;

753 754
	isert_info("%s (%d): status %d id %p np %p\n",
		   rdma_event_msg(event->event), event->event,
755
		   event->status, cma_id, cma_id->context);
756

757 758 759
	if (isert_np->cm_id == cma_id)
		return isert_np_cma_handler(cma_id->context, event->event);

760 761 762
	switch (event->event) {
	case RDMA_CM_EVENT_CONNECT_REQUEST:
		ret = isert_connect_request(cma_id, event);
763
		if (ret)
764
			isert_err("failed handle connect request %d\n", ret);
765 766 767 768
		break;
	case RDMA_CM_EVENT_ESTABLISHED:
		isert_connected_handler(cma_id);
		break;
769 770 771
	case RDMA_CM_EVENT_ADDR_CHANGE:    /* FALLTHRU */
	case RDMA_CM_EVENT_DISCONNECTED:   /* FALLTHRU */
	case RDMA_CM_EVENT_TIMEWAIT_EXIT:  /* FALLTHRU */
772
		ret = isert_disconnected_handler(cma_id, event->event);
773
		break;
774 775 776 777 778 779 780 781 782 783 784 785
	case RDMA_CM_EVENT_DEVICE_REMOVAL:
		isert_conn = cma_id->qp->qp_context;
		isert_conn->dev_removed = true;
		isert_disconnected_handler(cma_id, event->event);
		wait_event_interruptible(isert_conn->rem_wait,
					 isert_conn->state == ISER_CONN_DOWN);
		kfree(isert_conn);
		/*
		 * return non-zero from the callback to destroy
		 * the rdma cm id
		 */
		return 1;
786
	case RDMA_CM_EVENT_REJECTED:
787 788
		isert_info("Connection rejected: %s\n",
			   rdma_reject_msg(cma_id, event->status));
789 790
		/* fall through */
	case RDMA_CM_EVENT_UNREACHABLE:
791
	case RDMA_CM_EVENT_CONNECT_ERROR:
792
		ret = isert_connect_error(cma_id);
793
		break;
794
	default:
795
		isert_err("Unhandled RDMA CMA event: %d\n", event->event);
796 797 798 799 800 801 802
		break;
	}

	return ret;
}

static int
803
isert_post_recvm(struct isert_conn *isert_conn, u32 count)
804
{
805
	struct ib_recv_wr *rx_wr;
806 807 808
	int i, ret;
	struct iser_rx_desc *rx_desc;

809
	for (rx_wr = isert_conn->rx_wr, i = 0; i < count; i++, rx_wr++) {
810
		rx_desc = &isert_conn->rx_descs[i];
811 812

		rx_wr->wr_cqe = &rx_desc->rx_cqe;
813 814 815
		rx_wr->sg_list = &rx_desc->rx_sg;
		rx_wr->num_sge = 1;
		rx_wr->next = rx_wr + 1;
816
		rx_desc->in_use = false;
817 818 819 820
	}
	rx_wr--;
	rx_wr->next = NULL; /* mark end of work requests list */

821
	ret = ib_post_recv(isert_conn->qp, isert_conn->rx_wr, NULL);
822
	if (ret)
823
		isert_err("ib_post_recv() failed with ret: %d\n", ret);
824 825 826 827 828 829 830

	return ret;
}

static int
isert_post_recv(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc)
{
831
	struct ib_recv_wr rx_wr;
832 833
	int ret;

834 835 836 837 838 839 840 841 842
	if (!rx_desc->in_use) {
		/*
		 * if the descriptor is not in-use we already reposted it
		 * for recv, so just silently return
		 */
		return 0;
	}

	rx_desc->in_use = false;
843
	rx_wr.wr_cqe = &rx_desc->rx_cqe;
844 845 846 847
	rx_wr.sg_list = &rx_desc->rx_sg;
	rx_wr.num_sge = 1;
	rx_wr.next = NULL;

848
	ret = ib_post_recv(isert_conn->qp, &rx_wr, NULL);
849
	if (ret)
850 851
		isert_err("ib_post_recv() failed with ret: %d\n", ret);

852 853 854 855
	return ret;
}

static int
856
isert_login_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc)
857
{
858
	struct ib_device *ib_dev = isert_conn->cm_id->device;
859
	struct ib_send_wr send_wr;
860 861 862 863 864
	int ret;

	ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr,
				      ISER_HEADERS_LEN, DMA_TO_DEVICE);

865 866
	tx_desc->tx_cqe.done = isert_login_send_done;

867
	send_wr.next	= NULL;
868
	send_wr.wr_cqe	= &tx_desc->tx_cqe;
869 870 871 872 873
	send_wr.sg_list	= tx_desc->tx_sg;
	send_wr.num_sge	= tx_desc->num_sge;
	send_wr.opcode	= IB_WR_SEND;
	send_wr.send_flags = IB_SEND_SIGNALED;

874
	ret = ib_post_send(isert_conn->qp, &send_wr, NULL);
875
	if (ret)
876
		isert_err("ib_post_send() failed, ret: %d\n", ret);
877 878 879 880 881

	return ret;
}

static void
882 883
__isert_create_send_desc(struct isert_device *device,
			 struct iser_tx_desc *tx_desc)
884 885
{

886 887
	memset(&tx_desc->iser_header, 0, sizeof(struct iser_ctrl));
	tx_desc->iser_header.flags = ISCSI_CTRL;
888 889 890

	tx_desc->num_sge = 1;

891 892
	if (tx_desc->tx_sg[0].lkey != device->pd->local_dma_lkey) {
		tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey;
893
		isert_dbg("tx_desc %p lkey mismatch, fixing\n", tx_desc);
894 895 896
	}
}

897 898 899 900 901 902 903 904 905 906 907 908 909 910
static void
isert_create_send_desc(struct isert_conn *isert_conn,
		       struct isert_cmd *isert_cmd,
		       struct iser_tx_desc *tx_desc)
{
	struct isert_device *device = isert_conn->device;
	struct ib_device *ib_dev = device->ib_device;

	ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr,
				   ISER_HEADERS_LEN, DMA_TO_DEVICE);

	__isert_create_send_desc(device, tx_desc);
}

911 912 913 914
static int
isert_init_tx_hdrs(struct isert_conn *isert_conn,
		   struct iser_tx_desc *tx_desc)
{
915
	struct isert_device *device = isert_conn->device;
916
	struct ib_device *ib_dev = device->ib_device;
917 918 919 920 921
	u64 dma_addr;

	dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc,
			ISER_HEADERS_LEN, DMA_TO_DEVICE);
	if (ib_dma_mapping_error(ib_dev, dma_addr)) {
922
		isert_err("ib_dma_mapping_error() failed\n");
923 924 925 926 927 928
		return -ENOMEM;
	}

	tx_desc->dma_addr = dma_addr;
	tx_desc->tx_sg[0].addr	= tx_desc->dma_addr;
	tx_desc->tx_sg[0].length = ISER_HEADERS_LEN;
929
	tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey;
930

931 932 933
	isert_dbg("Setup tx_sg[0].addr: 0x%llx length: %u lkey: 0x%x\n",
		  tx_desc->tx_sg[0].addr, tx_desc->tx_sg[0].length,
		  tx_desc->tx_sg[0].lkey);
934 935 936 937 938

	return 0;
}

static void
939
isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
940
		   struct ib_send_wr *send_wr)
941
{
942 943
	struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc;

944 945
	tx_desc->tx_cqe.done = isert_send_done;
	send_wr->wr_cqe = &tx_desc->tx_cqe;
946 947 948 949 950 951 952 953

	if (isert_conn->snd_w_inv && isert_cmd->inv_rkey) {
		send_wr->opcode  = IB_WR_SEND_WITH_INV;
		send_wr->ex.invalidate_rkey = isert_cmd->inv_rkey;
	} else {
		send_wr->opcode = IB_WR_SEND;
	}

954
	send_wr->sg_list = &tx_desc->tx_sg[0];
955
	send_wr->num_sge = isert_cmd->tx_desc.num_sge;
956
	send_wr->send_flags = IB_SEND_SIGNALED;
957 958 959
}

static int
960
isert_login_post_recv(struct isert_conn *isert_conn)
961
{
962
	struct ib_recv_wr rx_wr;
963 964 965 966 967
	struct ib_sge sge;
	int ret;

	memset(&sge, 0, sizeof(struct ib_sge));
	sge.addr = isert_conn->login_req_dma;
968
	sge.length = ISER_RX_PAYLOAD_SIZE;
969
	sge.lkey = isert_conn->device->pd->local_dma_lkey;
970

971
	isert_dbg("Setup sge: addr: %llx length: %d 0x%08x\n",
972 973
		sge.addr, sge.length, sge.lkey);

974 975
	isert_conn->login_req_buf->rx_cqe.done = isert_login_recv_done;

976
	memset(&rx_wr, 0, sizeof(struct ib_recv_wr));
977
	rx_wr.wr_cqe = &isert_conn->login_req_buf->rx_cqe;
978 979 980
	rx_wr.sg_list = &sge;
	rx_wr.num_sge = 1;

981
	ret = ib_post_recv(isert_conn->qp, &rx_wr, NULL);
982
	if (ret)
983
		isert_err("ib_post_recv() failed: %d\n", ret);
984 985 986 987 988 989 990 991 992

	return ret;
}

static int
isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login,
		   u32 length)
{
	struct isert_conn *isert_conn = conn->context;
993
	struct isert_device *device = isert_conn->device;
994
	struct ib_device *ib_dev = device->ib_device;
995
	struct iser_tx_desc *tx_desc = &isert_conn->login_tx_desc;
996 997
	int ret;

998
	__isert_create_send_desc(device, tx_desc);
999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017

	memcpy(&tx_desc->iscsi_header, &login->rsp[0],
	       sizeof(struct iscsi_hdr));

	isert_init_tx_hdrs(isert_conn, tx_desc);

	if (length > 0) {
		struct ib_sge *tx_dsg = &tx_desc->tx_sg[1];

		ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma,
					   length, DMA_TO_DEVICE);

		memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length);

		ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma,
					      length, DMA_TO_DEVICE);

		tx_dsg->addr	= isert_conn->login_rsp_dma;
		tx_dsg->length	= length;
1018
		tx_dsg->lkey	= isert_conn->device->pd->local_dma_lkey;
1019 1020 1021 1022 1023 1024 1025 1026
		tx_desc->num_sge = 2;
	}
	if (!login->login_failed) {
		if (login->login_complete) {
			ret = isert_alloc_rx_descriptors(isert_conn);
			if (ret)
				return ret;

1027 1028
			ret = isert_post_recvm(isert_conn,
					       ISERT_QP_MAX_RECV_DTOS);
1029 1030 1031
			if (ret)
				return ret;

1032
			/* Now we are in FULL_FEATURE phase */
1033
			mutex_lock(&isert_conn->mutex);
1034
			isert_conn->state = ISER_CONN_FULL_FEATURE;
1035
			mutex_unlock(&isert_conn->mutex);
1036 1037 1038
			goto post_send;
		}

1039
		ret = isert_login_post_recv(isert_conn);
1040 1041 1042 1043
		if (ret)
			return ret;
	}
post_send:
1044
	ret = isert_login_post_send(isert_conn, tx_desc);
1045 1046 1047 1048 1049 1050 1051
	if (ret)
		return ret;

	return 0;
}

static void
1052
isert_rx_login_req(struct isert_conn *isert_conn)
1053
{
1054
	struct iser_rx_desc *rx_desc = isert_conn->login_req_buf;
1055
	int rx_buflen = isert_conn->login_req_len;
1056 1057 1058 1059
	struct iscsi_conn *conn = isert_conn->conn;
	struct iscsi_login *login = conn->conn_login;
	int size;

1060
	isert_info("conn %p\n", isert_conn);
1061 1062

	WARN_ON_ONCE(!login);
1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087

	if (login->first_request) {
		struct iscsi_login_req *login_req =
			(struct iscsi_login_req *)&rx_desc->iscsi_header;
		/*
		 * Setup the initial iscsi_login values from the leading
		 * login request PDU.
		 */
		login->leading_connection = (!login_req->tsih) ? 1 : 0;
		login->current_stage =
			(login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK)
			 >> 2;
		login->version_min	= login_req->min_version;
		login->version_max	= login_req->max_version;
		memcpy(login->isid, login_req->isid, 6);
		login->cmd_sn		= be32_to_cpu(login_req->cmdsn);
		login->init_task_tag	= login_req->itt;
		login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn);
		login->cid		= be16_to_cpu(login_req->cid);
		login->tsih		= be16_to_cpu(login_req->tsih);
	}

	memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN);

	size = min(rx_buflen, MAX_KEY_VALUE_PAIRS);
1088 1089 1090
	isert_dbg("Using login payload size: %d, rx_buflen: %d "
		  "MAX_KEY_VALUE_PAIRS: %d\n", size, rx_buflen,
		  MAX_KEY_VALUE_PAIRS);
1091 1092
	memcpy(login->req_buf, &rx_desc->data[0], size);

1093
	if (login->first_request) {
1094
		complete(&isert_conn->login_comp);
1095 1096 1097
		return;
	}
	schedule_delayed_work(&conn->login_work, 0);
1098 1099 1100
}

static struct iscsi_cmd
1101
*isert_allocate_cmd(struct iscsi_conn *conn, struct iser_rx_desc *rx_desc)
1102
{
1103
	struct isert_conn *isert_conn = conn->context;
1104
	struct isert_cmd *isert_cmd;
1105
	struct iscsi_cmd *cmd;
1106

1107
	cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
1108
	if (!cmd) {
1109
		isert_err("Unable to allocate iscsi_cmd + isert_cmd\n");
1110 1111
		return NULL;
	}
1112
	isert_cmd = iscsit_priv_cmd(cmd);
1113
	isert_cmd->conn = isert_conn;
1114
	isert_cmd->iscsi_cmd = cmd;
1115
	isert_cmd->rx_desc = rx_desc;
1116

1117
	return cmd;
1118 1119 1120 1121
}

static int
isert_handle_scsi_cmd(struct isert_conn *isert_conn,
1122 1123
		      struct isert_cmd *isert_cmd, struct iscsi_cmd *cmd,
		      struct iser_rx_desc *rx_desc, unsigned char *buf)
1124 1125 1126 1127 1128
{
	struct iscsi_conn *conn = isert_conn->conn;
	struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf;
	int imm_data, imm_data_len, unsol_data, sg_nents, rc;
	bool dump_payload = false;
1129
	unsigned int data_len;
1130 1131 1132 1133 1134 1135 1136 1137

	rc = iscsit_setup_scsi_cmd(conn, cmd, buf);
	if (rc < 0)
		return rc;

	imm_data = cmd->immediate_data;
	imm_data_len = cmd->first_burst_len;
	unsol_data = cmd->unsolicited_data;
1138
	data_len = cmd->se_cmd.data_length;
1139

1140 1141
	if (imm_data && imm_data_len == data_len)
		cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC;
1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152
	rc = iscsit_process_scsi_cmd(conn, cmd, hdr);
	if (rc < 0) {
		return 0;
	} else if (rc > 0) {
		dump_payload = true;
		goto sequence_cmd;
	}

	if (!imm_data)
		return 0;

1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166
	if (imm_data_len != data_len) {
		sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE));
		sg_copy_from_buffer(cmd->se_cmd.t_data_sg, sg_nents,
				    &rx_desc->data[0], imm_data_len);
		isert_dbg("Copy Immediate sg_nents: %u imm_data_len: %d\n",
			  sg_nents, imm_data_len);
	} else {
		sg_init_table(&isert_cmd->sg, 1);
		cmd->se_cmd.t_data_sg = &isert_cmd->sg;
		cmd->se_cmd.t_data_nents = 1;
		sg_set_buf(&isert_cmd->sg, &rx_desc->data[0], imm_data_len);
		isert_dbg("Transfer Immediate imm_data_len: %d\n",
			  imm_data_len);
	}
1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177

	cmd->write_data_done += imm_data_len;

	if (cmd->write_data_done == cmd->se_cmd.data_length) {
		spin_lock_bh(&cmd->istate_lock);
		cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
		cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
		spin_unlock_bh(&cmd->istate_lock);
	}

sequence_cmd:
1178
	rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn);
1179 1180

	if (!rc && dump_payload == false && unsol_data)
1181
		iscsit_set_unsolicited_dataout(cmd);
1182
	else if (dump_payload && imm_data)
1183
		target_put_sess_cmd(&cmd->se_cmd);
1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207

	return 0;
}

static int
isert_handle_iscsi_dataout(struct isert_conn *isert_conn,
			   struct iser_rx_desc *rx_desc, unsigned char *buf)
{
	struct scatterlist *sg_start;
	struct iscsi_conn *conn = isert_conn->conn;
	struct iscsi_cmd *cmd = NULL;
	struct iscsi_data *hdr = (struct iscsi_data *)buf;
	u32 unsol_data_len = ntoh24(hdr->dlength);
	int rc, sg_nents, sg_off, page_off;

	rc = iscsit_check_dataout_hdr(conn, buf, &cmd);
	if (rc < 0)
		return rc;
	else if (!cmd)
		return 0;
	/*
	 * FIXME: Unexpected unsolicited_data out
	 */
	if (!cmd->unsolicited_data) {
1208
		isert_err("Received unexpected solicited data payload\n");
1209 1210 1211 1212
		dump_stack();
		return -1;
	}

1213 1214 1215 1216
	isert_dbg("Unsolicited DataOut unsol_data_len: %u, "
		  "write_data_done: %u, data_length: %u\n",
		  unsol_data_len,  cmd->write_data_done,
		  cmd->se_cmd.data_length);
1217 1218 1219 1220 1221 1222 1223 1224 1225

	sg_off = cmd->write_data_done / PAGE_SIZE;
	sg_start = &cmd->se_cmd.t_data_sg[sg_off];
	sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE));
	page_off = cmd->write_data_done % PAGE_SIZE;
	/*
	 * FIXME: Non page-aligned unsolicited_data out
	 */
	if (page_off) {
1226
		isert_err("unexpected non-page aligned data payload\n");
1227 1228 1229
		dump_stack();
		return -1;
	}
1230 1231 1232
	isert_dbg("Copying DataOut: sg_start: %p, sg_off: %u "
		  "sg_nents: %u from %p %u\n", sg_start, sg_off,
		  sg_nents, &rx_desc->data[0], unsol_data_len);
1233 1234 1235 1236 1237 1238 1239 1240

	sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0],
			    unsol_data_len);

	rc = iscsit_check_dataout_payload(cmd, hdr, false);
	if (rc < 0)
		return rc;

1241 1242 1243 1244 1245 1246 1247 1248 1249
	/*
	 * multiple data-outs on the same command can arrive -
	 * so post the buffer before hand
	 */
	rc = isert_post_recv(isert_conn, rx_desc);
	if (rc) {
		isert_err("ib_post_recv failed with %d\n", rc);
		return rc;
	}
1250 1251 1252
	return 0;
}

1253 1254
static int
isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
1255 1256
		     struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc,
		     unsigned char *buf)
1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271
{
	struct iscsi_conn *conn = isert_conn->conn;
	struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf;
	int rc;

	rc = iscsit_setup_nop_out(conn, cmd, hdr);
	if (rc < 0)
		return rc;
	/*
	 * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload
	 */

	return iscsit_process_nop_out(conn, cmd, hdr);
}

1272 1273
static int
isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
1274 1275
		      struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc,
		      struct iscsi_text *hdr)
1276 1277 1278 1279
{
	struct iscsi_conn *conn = isert_conn->conn;
	u32 payload_length = ntoh24(hdr->dlength);
	int rc;
1280
	unsigned char *text_in = NULL;
1281 1282 1283 1284 1285

	rc = iscsit_setup_text_cmd(conn, cmd, hdr);
	if (rc < 0)
		return rc;

1286 1287
	if (payload_length) {
		text_in = kzalloc(payload_length, GFP_KERNEL);
1288
		if (!text_in)
1289
			return -ENOMEM;
1290 1291 1292 1293 1294 1295 1296 1297
	}
	cmd->text_in_ptr = text_in;

	memcpy(cmd->text_in_ptr, &rx_desc->data[0], payload_length);

	return iscsit_process_text_cmd(conn, cmd, hdr);
}

1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309
static int
isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc,
		uint32_t read_stag, uint64_t read_va,
		uint32_t write_stag, uint64_t write_va)
{
	struct iscsi_hdr *hdr = &rx_desc->iscsi_header;
	struct iscsi_conn *conn = isert_conn->conn;
	struct iscsi_cmd *cmd;
	struct isert_cmd *isert_cmd;
	int ret = -EINVAL;
	u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK);

1310
	if (conn->sess->sess_ops->SessionType &&
1311
	   (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) {
1312
		isert_err("Got illegal opcode: 0x%02x in SessionType=Discovery,"
1313
			  " ignoring\n", opcode);
1314 1315 1316
		return 0;
	}

1317 1318
	switch (opcode) {
	case ISCSI_OP_SCSI_CMD:
1319
		cmd = isert_allocate_cmd(conn, rx_desc);
1320 1321 1322
		if (!cmd)
			break;

1323
		isert_cmd = iscsit_priv_cmd(cmd);
1324 1325 1326 1327
		isert_cmd->read_stag = read_stag;
		isert_cmd->read_va = read_va;
		isert_cmd->write_stag = write_stag;
		isert_cmd->write_va = write_va;
1328
		isert_cmd->inv_rkey = read_stag ? read_stag : write_stag;
1329

1330
		ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd,
1331 1332 1333
					rx_desc, (unsigned char *)hdr);
		break;
	case ISCSI_OP_NOOP_OUT:
1334
		cmd = isert_allocate_cmd(conn, rx_desc);
1335 1336 1337
		if (!cmd)
			break;

1338 1339
		isert_cmd = iscsit_priv_cmd(cmd);
		ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd,
1340
					   rx_desc, (unsigned char *)hdr);
1341 1342 1343 1344 1345 1346
		break;
	case ISCSI_OP_SCSI_DATA_OUT:
		ret = isert_handle_iscsi_dataout(isert_conn, rx_desc,
						(unsigned char *)hdr);
		break;
	case ISCSI_OP_SCSI_TMFUNC:
1347
		cmd = isert_allocate_cmd(conn, rx_desc);
1348 1349 1350 1351 1352 1353 1354
		if (!cmd)
			break;

		ret = iscsit_handle_task_mgt_cmd(conn, cmd,
						(unsigned char *)hdr);
		break;
	case ISCSI_OP_LOGOUT:
1355
		cmd = isert_allocate_cmd(conn, rx_desc);
1356 1357 1358 1359 1360
		if (!cmd)
			break;

		ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr);
		break;
1361
	case ISCSI_OP_TEXT:
1362
		if (be32_to_cpu(hdr->ttt) != 0xFFFFFFFF)
1363
			cmd = iscsit_find_cmd_from_itt(conn, hdr->itt);
1364 1365 1366 1367 1368
		else
			cmd = isert_allocate_cmd(conn, rx_desc);

		if (!cmd)
			break;
1369

1370 1371
		isert_cmd = iscsit_priv_cmd(cmd);
		ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd,
1372 1373
					    rx_desc, (struct iscsi_text *)hdr);
		break;
1374
	default:
1375
		isert_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode);
1376 1377 1378 1379 1380 1381 1382 1383
		dump_stack();
		break;
	}

	return ret;
}

static void
1384
isert_print_wc(struct ib_wc *wc, const char *type)
1385
{
1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401
	if (wc->status != IB_WC_WR_FLUSH_ERR)
		isert_err("%s failure: %s (%d) vend_err %x\n", type,
			  ib_wc_status_msg(wc->status), wc->status,
			  wc->vendor_err);
	else
		isert_dbg("%s failure: %s (%d)\n", type,
			  ib_wc_status_msg(wc->status), wc->status);
}

static void
isert_recv_done(struct ib_cq *cq, struct ib_wc *wc)
{
	struct isert_conn *isert_conn = wc->qp->qp_context;
	struct ib_device *ib_dev = isert_conn->cm_id->device;
	struct iser_rx_desc *rx_desc = cqe_to_rx_desc(wc->wr_cqe);
	struct iscsi_hdr *hdr = &rx_desc->iscsi_header;
1402
	struct iser_ctrl *iser_ctrl = &rx_desc->iser_header;
1403 1404 1405
	uint64_t read_va = 0, write_va = 0;
	uint32_t read_stag = 0, write_stag = 0;

1406 1407 1408 1409 1410 1411 1412
	if (unlikely(wc->status != IB_WC_SUCCESS)) {
		isert_print_wc(wc, "recv");
		if (wc->status != IB_WC_WR_FLUSH_ERR)
			iscsit_cause_connection_reinstatement(isert_conn->conn, 0);
		return;
	}

1413 1414
	rx_desc->in_use = true;

1415 1416 1417 1418 1419 1420 1421
	ib_dma_sync_single_for_cpu(ib_dev, rx_desc->dma_addr,
			ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);

	isert_dbg("DMA: 0x%llx, iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n",
		 rx_desc->dma_addr, hdr->opcode, hdr->itt, hdr->flags,
		 (int)(wc->byte_len - ISER_HEADERS_LEN));

1422
	switch (iser_ctrl->flags & 0xF0) {
1423
	case ISCSI_CTRL:
1424 1425 1426
		if (iser_ctrl->flags & ISER_RSV) {
			read_stag = be32_to_cpu(iser_ctrl->read_stag);
			read_va = be64_to_cpu(iser_ctrl->read_va);
1427 1428
			isert_dbg("ISER_RSV: read_stag: 0x%x read_va: 0x%llx\n",
				  read_stag, (unsigned long long)read_va);
1429
		}
1430 1431 1432
		if (iser_ctrl->flags & ISER_WSV) {
			write_stag = be32_to_cpu(iser_ctrl->write_stag);
			write_va = be64_to_cpu(iser_ctrl->write_va);
1433 1434
			isert_dbg("ISER_WSV: write_stag: 0x%x write_va: 0x%llx\n",
				  write_stag, (unsigned long long)write_va);
1435 1436
		}

1437
		isert_dbg("ISER ISCSI_CTRL PDU\n");
1438 1439
		break;
	case ISER_HELLO:
1440
		isert_err("iSER Hello message\n");
1441 1442
		break;
	default:
1443
		isert_warn("Unknown iSER hdr flags: 0x%02x\n", iser_ctrl->flags);
1444 1445 1446
		break;
	}

1447 1448
	isert_rx_opcode(isert_conn, rx_desc,
			read_stag, read_va, write_stag, write_va);
1449 1450 1451

	ib_dma_sync_single_for_device(ib_dev, rx_desc->dma_addr,
			ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
1452 1453 1454
}

static void
1455
isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc)
1456
{
1457
	struct isert_conn *isert_conn = wc->qp->qp_context;
1458
	struct ib_device *ib_dev = isert_conn->device->ib_device;
1459 1460 1461 1462

	if (unlikely(wc->status != IB_WC_SUCCESS)) {
		isert_print_wc(wc, "login recv");
		return;
1463 1464
	}

1465 1466
	ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_req_dma,
			ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
1467

1468
	isert_conn->login_req_len = wc->byte_len - ISER_HEADERS_LEN;
1469

1470 1471
	if (isert_conn->conn) {
		struct iscsi_login *login = isert_conn->conn->conn_login;
1472

1473 1474
		if (login && !login->first_request)
			isert_rx_login_req(isert_conn);
1475
	}
1476

1477 1478 1479
	mutex_lock(&isert_conn->mutex);
	complete(&isert_conn->login_req_comp);
	mutex_unlock(&isert_conn->mutex);
1480

1481 1482
	ib_dma_sync_single_for_device(ib_dev, isert_conn->login_req_dma,
				ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
1483 1484 1485
}

static void
1486
isert_rdma_rw_ctx_destroy(struct isert_cmd *cmd, struct isert_conn *conn)
1487
{
1488 1489
	struct se_cmd *se_cmd = &cmd->iscsi_cmd->se_cmd;
	enum dma_data_direction dir = target_reverse_dma_direction(se_cmd);
1490

1491 1492
	if (!cmd->rw.nr_ops)
		return;
1493

1494 1495 1496 1497 1498 1499 1500 1501
	if (isert_prot_cmd(conn, se_cmd)) {
		rdma_rw_ctx_destroy_signature(&cmd->rw, conn->qp,
				conn->cm_id->port_num, se_cmd->t_data_sg,
				se_cmd->t_data_nents, se_cmd->t_prot_sg,
				se_cmd->t_prot_nents, dir);
	} else {
		rdma_rw_ctx_destroy(&cmd->rw, conn->qp, conn->cm_id->port_num,
				se_cmd->t_data_sg, se_cmd->t_data_nents, dir);
1502 1503
	}

1504
	cmd->rw.nr_ops = 0;
1505 1506
}

1507
static void
1508
isert_put_cmd(struct isert_cmd *isert_cmd, bool comp_err)
1509
{
1510
	struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;
1511
	struct isert_conn *isert_conn = isert_cmd->conn;
1512
	struct iscsi_conn *conn = isert_conn->conn;
1513
	struct iscsi_text_rsp *hdr;
1514

1515
	isert_dbg("Cmd %p\n", isert_cmd);
1516 1517 1518 1519 1520

	switch (cmd->iscsi_opcode) {
	case ISCSI_OP_SCSI_CMD:
		spin_lock_bh(&conn->cmd_lock);
		if (!list_empty(&cmd->i_conn_node))
1521
			list_del_init(&cmd->i_conn_node);
1522 1523
		spin_unlock_bh(&conn->cmd_lock);

1524
		if (cmd->data_direction == DMA_TO_DEVICE) {
1525
			iscsit_stop_dataout_timer(cmd);
1526 1527 1528 1529 1530 1531 1532 1533 1534 1535
			/*
			 * Check for special case during comp_err where
			 * WRITE_PENDING has been handed off from core,
			 * but requires an extra target_put_sess_cmd()
			 * before transport_generic_free_cmd() below.
			 */
			if (comp_err &&
			    cmd->se_cmd.t_state == TRANSPORT_WRITE_PENDING) {
				struct se_cmd *se_cmd = &cmd->se_cmd;

1536
				target_put_sess_cmd(se_cmd);
1537 1538
			}
		}
1539

1540
		isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn);
1541 1542
		transport_generic_free_cmd(&cmd->se_cmd, 0);
		break;
1543
	case ISCSI_OP_SCSI_TMFUNC:
1544 1545
		spin_lock_bh(&conn->cmd_lock);
		if (!list_empty(&cmd->i_conn_node))
1546
			list_del_init(&cmd->i_conn_node);
1547 1548
		spin_unlock_bh(&conn->cmd_lock);

1549 1550 1551 1552
		transport_generic_free_cmd(&cmd->se_cmd, 0);
		break;
	case ISCSI_OP_REJECT:
	case ISCSI_OP_NOOP_OUT:
1553
	case ISCSI_OP_TEXT:
1554 1555 1556 1557 1558
		hdr = (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header;
		/* If the continue bit is on, keep the command alive */
		if (hdr->flags & ISCSI_FLAG_TEXT_CONTINUE)
			break;

1559 1560
		spin_lock_bh(&conn->cmd_lock);
		if (!list_empty(&cmd->i_conn_node))
1561
			list_del_init(&cmd->i_conn_node);
1562 1563 1564 1565 1566 1567 1568 1569
		spin_unlock_bh(&conn->cmd_lock);

		/*
		 * Handle special case for REJECT when iscsi_add_reject*() has
		 * overwritten the original iscsi_opcode assignment, and the
		 * associated cmd->se_cmd needs to be released.
		 */
		if (cmd->se_cmd.se_tfo != NULL) {
1570
			isert_dbg("Calling transport_generic_free_cmd for 0x%02x\n",
1571
				 cmd->iscsi_opcode);
1572 1573 1574
			transport_generic_free_cmd(&cmd->se_cmd, 0);
			break;
		}
1575
		/* fall through */
1576
	default:
1577
		iscsit_release_cmd(cmd);
1578 1579 1580 1581 1582 1583 1584 1585
		break;
	}
}

static void
isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev)
{
	if (tx_desc->dma_addr != 0) {
1586
		isert_dbg("unmap single for tx_desc->dma_addr\n");
1587 1588 1589 1590 1591 1592 1593 1594
		ib_dma_unmap_single(ib_dev, tx_desc->dma_addr,
				    ISER_HEADERS_LEN, DMA_TO_DEVICE);
		tx_desc->dma_addr = 0;
	}
}

static void
isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd,
1595
		     struct ib_device *ib_dev, bool comp_err)
1596
{
1597
	if (isert_cmd->pdu_buf_dma != 0) {
1598
		isert_dbg("unmap single for isert_cmd->pdu_buf_dma\n");
1599 1600 1601
		ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma,
				    isert_cmd->pdu_buf_len, DMA_TO_DEVICE);
		isert_cmd->pdu_buf_dma = 0;
1602 1603 1604
	}

	isert_unmap_tx_desc(tx_desc, ib_dev);
1605
	isert_put_cmd(isert_cmd, comp_err);
1606 1607
}

1608 1609 1610 1611 1612 1613 1614 1615
static int
isert_check_pi_status(struct se_cmd *se_cmd, struct ib_mr *sig_mr)
{
	struct ib_mr_status mr_status;
	int ret;

	ret = ib_check_mr_status(sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status);
	if (ret) {
1616
		isert_err("ib_check_mr_status failed, ret %d\n", ret);
1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638
		goto fail_mr_status;
	}

	if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) {
		u64 sec_offset_err;
		u32 block_size = se_cmd->se_dev->dev_attrib.block_size + 8;

		switch (mr_status.sig_err.err_type) {
		case IB_SIG_BAD_GUARD:
			se_cmd->pi_err = TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED;
			break;
		case IB_SIG_BAD_REFTAG:
			se_cmd->pi_err = TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED;
			break;
		case IB_SIG_BAD_APPTAG:
			se_cmd->pi_err = TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED;
			break;
		}
		sec_offset_err = mr_status.sig_err.sig_err_offset;
		do_div(sec_offset_err, block_size);
		se_cmd->bad_sector = sec_offset_err + se_cmd->t_task_lba;

1639 1640 1641 1642 1643 1644
		isert_err("PI error found type %d at sector 0x%llx "
			  "expected 0x%x vs actual 0x%x\n",
			  mr_status.sig_err.err_type,
			  (unsigned long long)se_cmd->bad_sector,
			  mr_status.sig_err.expected,
			  mr_status.sig_err.actual);
1645 1646 1647 1648 1649 1650 1651
		ret = 1;
	}

fail_mr_status:
	return ret;
}

1652
static void
1653
isert_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc)
1654
{
1655
	struct isert_conn *isert_conn = wc->qp->qp_context;
1656
	struct isert_device *device = isert_conn->device;
1657
	struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe);
1658
	struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc);
1659
	struct se_cmd *cmd = &isert_cmd->iscsi_cmd->se_cmd;
1660 1661
	int ret = 0;

1662 1663 1664 1665 1666 1667 1668 1669 1670 1671
	if (unlikely(wc->status != IB_WC_SUCCESS)) {
		isert_print_wc(wc, "rdma write");
		if (wc->status != IB_WC_WR_FLUSH_ERR)
			iscsit_cause_connection_reinstatement(isert_conn->conn, 0);
		isert_completion_put(desc, isert_cmd, device->ib_device, true);
		return;
	}

	isert_dbg("Cmd %p\n", isert_cmd);

1672
	ret = isert_check_pi_status(cmd, isert_cmd->rw.reg->mr);
1673
	isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn);
1674

1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691
	if (ret) {
		/*
		 * transport_generic_request_failure() expects to have
		 * plus two references to handle queue-full, so re-add
		 * one here as target-core will have already dropped
		 * it after the first isert_put_datain() callback.
		 */
		kref_get(&cmd->cmd_kref);
		transport_generic_request_failure(cmd, cmd->pi_err);
	} else {
		/*
		 * XXX: isert_put_response() failure is not retried.
		 */
		ret = isert_put_response(isert_conn->conn, isert_cmd->iscsi_cmd);
		if (ret)
			pr_warn_ratelimited("isert_put_response() ret: %d\n", ret);
	}
1692 1693
}

1694
static void
1695
isert_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc)
1696
{
1697 1698 1699
	struct isert_conn *isert_conn = wc->qp->qp_context;
	struct isert_device *device = isert_conn->device;
	struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe);
1700
	struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc);
1701
	struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;
1702
	struct se_cmd *se_cmd = &cmd->se_cmd;
1703
	int ret = 0;
1704

1705 1706 1707 1708 1709 1710 1711 1712 1713 1714
	if (unlikely(wc->status != IB_WC_SUCCESS)) {
		isert_print_wc(wc, "rdma read");
		if (wc->status != IB_WC_WR_FLUSH_ERR)
			iscsit_cause_connection_reinstatement(isert_conn->conn, 0);
		isert_completion_put(desc, isert_cmd, device->ib_device, true);
		return;
	}

	isert_dbg("Cmd %p\n", isert_cmd);

1715
	iscsit_stop_dataout_timer(cmd);
1716 1717

	if (isert_prot_cmd(isert_conn, se_cmd))
1718
		ret = isert_check_pi_status(se_cmd, isert_cmd->rw.reg->mr);
1719 1720
	isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn);
	cmd->write_data_done = 0;
1721

1722
	isert_dbg("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd);
1723 1724 1725 1726 1727
	spin_lock_bh(&cmd->istate_lock);
	cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
	cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
	spin_unlock_bh(&cmd->istate_lock);

1728 1729 1730 1731 1732 1733 1734 1735
	/*
	 * transport_generic_request_failure() will drop the extra
	 * se_cmd->cmd_kref reference after T10-PI error, and handle
	 * any non-zero ->queue_status() callback error retries.
	 */
	if (ret)
		transport_generic_request_failure(se_cmd, se_cmd->pi_err);
	else
1736
		target_execute_cmd(se_cmd);
1737 1738 1739 1740 1741 1742 1743 1744
}

static void
isert_do_control_comp(struct work_struct *work)
{
	struct isert_cmd *isert_cmd = container_of(work,
			struct isert_cmd, comp_work);
	struct isert_conn *isert_conn = isert_cmd->conn;
1745
	struct ib_device *ib_dev = isert_conn->cm_id->device;
1746
	struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;
1747

1748 1749
	isert_dbg("Cmd %p i_state %d\n", isert_cmd, cmd->i_state);

1750 1751 1752
	switch (cmd->i_state) {
	case ISTATE_SEND_TASKMGTRSP:
		iscsit_tmr_post_handler(cmd, cmd->conn);
1753 1754 1755
		/* fall through */
	case ISTATE_SEND_REJECT:
	case ISTATE_SEND_TEXTRSP:
1756
		cmd->i_state = ISTATE_SENT_STATUS;
1757 1758
		isert_completion_put(&isert_cmd->tx_desc, isert_cmd,
				     ib_dev, false);
1759
		break;
1760 1761 1762 1763
	case ISTATE_SEND_LOGOUTRSP:
		iscsit_logout_post_handler(cmd, cmd->conn);
		break;
	default:
1764
		isert_err("Unknown i_state %d\n", cmd->i_state);
1765 1766 1767 1768 1769 1770
		dump_stack();
		break;
	}
}

static void
1771
isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc)
1772
{
1773 1774 1775
	struct isert_conn *isert_conn = wc->qp->qp_context;
	struct ib_device *ib_dev = isert_conn->cm_id->device;
	struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe);
1776

1777 1778 1779 1780
	if (unlikely(wc->status != IB_WC_SUCCESS)) {
		isert_print_wc(wc, "login send");
		if (wc->status != IB_WC_WR_FLUSH_ERR)
			iscsit_cause_connection_reinstatement(isert_conn->conn, 0);
1781
	}
1782

1783
	isert_unmap_tx_desc(tx_desc, ib_dev);
1784 1785 1786
}

static void
1787
isert_send_done(struct ib_cq *cq, struct ib_wc *wc)
1788
{
1789
	struct isert_conn *isert_conn = wc->qp->qp_context;
1790
	struct ib_device *ib_dev = isert_conn->cm_id->device;
1791
	struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe);
1792
	struct isert_cmd *isert_cmd = tx_desc_to_cmd(tx_desc);
1793

1794 1795 1796 1797 1798
	if (unlikely(wc->status != IB_WC_SUCCESS)) {
		isert_print_wc(wc, "send");
		if (wc->status != IB_WC_WR_FLUSH_ERR)
			iscsit_cause_connection_reinstatement(isert_conn->conn, 0);
		isert_completion_put(tx_desc, isert_cmd, ib_dev, true);
1799 1800 1801
		return;
	}

1802
	isert_dbg("Cmd %p\n", isert_cmd);
1803

1804 1805 1806 1807 1808 1809 1810 1811 1812 1813
	switch (isert_cmd->iscsi_cmd->i_state) {
	case ISTATE_SEND_TASKMGTRSP:
	case ISTATE_SEND_LOGOUTRSP:
	case ISTATE_SEND_REJECT:
	case ISTATE_SEND_TEXTRSP:
		isert_unmap_tx_desc(tx_desc, ib_dev);

		INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp);
		queue_work(isert_comp_wq, &isert_cmd->comp_work);
		return;
1814
	default:
1815 1816
		isert_cmd->iscsi_cmd->i_state = ISTATE_SENT_STATUS;
		isert_completion_put(tx_desc, isert_cmd, ib_dev, false);
1817 1818 1819 1820 1821 1822 1823 1824 1825
		break;
	}
}

static int
isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd)
{
	int ret;

1826 1827 1828 1829 1830 1831
	ret = isert_post_recv(isert_conn, isert_cmd->rx_desc);
	if (ret) {
		isert_err("ib_post_recv failed with %d\n", ret);
		return ret;
	}

1832
	ret = ib_post_send(isert_conn->qp, &isert_cmd->tx_desc.send_wr, NULL);
1833
	if (ret) {
1834
		isert_err("ib_post_send failed with %d\n", ret);
1835 1836 1837 1838 1839 1840 1841 1842
		return ret;
	}
	return ret;
}

static int
isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
{
1843
	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
1844
	struct isert_conn *isert_conn = conn->context;
1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857
	struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
	struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *)
				&isert_cmd->tx_desc.iscsi_header;

	isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
	iscsit_build_rsp_pdu(cmd, conn, true, hdr);
	isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
	/*
	 * Attach SENSE DATA payload to iSCSI Response PDU
	 */
	if (cmd->se_cmd.sense_buffer &&
	    ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ||
	    (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) {
1858
		struct isert_device *device = isert_conn->device;
1859
		struct ib_device *ib_dev = device->ib_device;
1860
		struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1];
1861
		u32 padding, pdu_len;
1862 1863 1864 1865 1866 1867 1868

		put_unaligned_be16(cmd->se_cmd.scsi_sense_length,
				   cmd->sense_buffer);
		cmd->se_cmd.scsi_sense_length += sizeof(__be16);

		padding = -(cmd->se_cmd.scsi_sense_length) & 3;
		hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length);
1869
		pdu_len = cmd->se_cmd.scsi_sense_length + padding;
1870

1871 1872
		isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev,
				(void *)cmd->sense_buffer, pdu_len,
1873
				DMA_TO_DEVICE);
1874 1875
		if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma))
			return -ENOMEM;
1876

1877 1878 1879
		isert_cmd->pdu_buf_len = pdu_len;
		tx_dsg->addr	= isert_cmd->pdu_buf_dma;
		tx_dsg->length	= pdu_len;
1880
		tx_dsg->lkey	= device->pd->local_dma_lkey;
1881 1882 1883
		isert_cmd->tx_desc.num_sge = 2;
	}

1884
	isert_init_send_wr(isert_conn, isert_cmd, send_wr);
1885

1886
	isert_dbg("Posting SCSI Response\n");
1887 1888 1889 1890

	return isert_post_response(isert_conn, isert_cmd);
}

1891 1892 1893 1894
static void
isert_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
{
	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
1895
	struct isert_conn *isert_conn = conn->context;
1896 1897 1898 1899 1900 1901 1902 1903

	spin_lock_bh(&conn->cmd_lock);
	if (!list_empty(&cmd->i_conn_node))
		list_del_init(&cmd->i_conn_node);
	spin_unlock_bh(&conn->cmd_lock);

	if (cmd->data_direction == DMA_TO_DEVICE)
		iscsit_stop_dataout_timer(cmd);
1904
	isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn);
1905 1906
}

1907 1908 1909
static enum target_prot_op
isert_get_sup_prot_ops(struct iscsi_conn *conn)
{
1910
	struct isert_conn *isert_conn = conn->context;
1911
	struct isert_device *device = isert_conn->device;
1912

1913 1914
	if (conn->tpg->tpg_attrib.t10_pi) {
		if (device->pi_capable) {
1915
			isert_info("conn %p PI offload enabled\n", isert_conn);
1916 1917 1918 1919 1920
			isert_conn->pi_support = true;
			return TARGET_PROT_ALL;
		}
	}

1921
	isert_info("conn %p PI offload disabled\n", isert_conn);
1922
	isert_conn->pi_support = false;
1923 1924 1925 1926

	return TARGET_PROT_NORMAL;
}

1927 1928 1929 1930
static int
isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
		bool nopout_response)
{
1931
	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
1932
	struct isert_conn *isert_conn = conn->context;
1933 1934 1935 1936 1937 1938 1939
	struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;

	isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
	iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *)
			       &isert_cmd->tx_desc.iscsi_header,
			       nopout_response);
	isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1940
	isert_init_send_wr(isert_conn, isert_cmd, send_wr);
1941

1942
	isert_dbg("conn %p Posting NOPIN Response\n", isert_conn);
1943 1944 1945 1946 1947 1948 1949

	return isert_post_response(isert_conn, isert_cmd);
}

static int
isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
{
1950
	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
1951
	struct isert_conn *isert_conn = conn->context;
1952 1953 1954 1955 1956 1957
	struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;

	isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
	iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *)
				&isert_cmd->tx_desc.iscsi_header);
	isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1958
	isert_init_send_wr(isert_conn, isert_cmd, send_wr);
1959

1960
	isert_dbg("conn %p Posting Logout Response\n", isert_conn);
1961 1962 1963 1964 1965 1966 1967

	return isert_post_response(isert_conn, isert_cmd);
}

static int
isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
{
1968
	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
1969
	struct isert_conn *isert_conn = conn->context;
1970 1971 1972 1973 1974 1975
	struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;

	isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
	iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *)
				  &isert_cmd->tx_desc.iscsi_header);
	isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1976
	isert_init_send_wr(isert_conn, isert_cmd, send_wr);
1977

1978
	isert_dbg("conn %p Posting Task Management Response\n", isert_conn);
1979 1980 1981 1982 1983 1984 1985

	return isert_post_response(isert_conn, isert_cmd);
}

static int
isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
{
1986
	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
1987
	struct isert_conn *isert_conn = conn->context;
1988
	struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
1989
	struct isert_device *device = isert_conn->device;
1990
	struct ib_device *ib_dev = device->ib_device;
1991 1992 1993
	struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1];
	struct iscsi_reject *hdr =
		(struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header;
1994 1995

	isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
1996
	iscsit_build_reject(cmd, conn, hdr);
1997
	isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1998 1999

	hton24(hdr->dlength, ISCSI_HDR_LEN);
2000
	isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev,
2001 2002
			(void *)cmd->buf_ptr, ISCSI_HDR_LEN,
			DMA_TO_DEVICE);
2003 2004
	if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma))
		return -ENOMEM;
2005 2006
	isert_cmd->pdu_buf_len = ISCSI_HDR_LEN;
	tx_dsg->addr	= isert_cmd->pdu_buf_dma;
2007
	tx_dsg->length	= ISCSI_HDR_LEN;
2008
	tx_dsg->lkey	= device->pd->local_dma_lkey;
2009 2010
	isert_cmd->tx_desc.num_sge = 2;

2011
	isert_init_send_wr(isert_conn, isert_cmd, send_wr);
2012

2013
	isert_dbg("conn %p Posting Reject\n", isert_conn);
2014 2015 2016 2017

	return isert_post_response(isert_conn, isert_cmd);
}

2018 2019 2020
static int
isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
{
2021
	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
2022
	struct isert_conn *isert_conn = conn->context;
2023 2024 2025 2026 2027 2028 2029
	struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
	struct iscsi_text_rsp *hdr =
		(struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header;
	u32 txt_rsp_len;
	int rc;

	isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
2030
	rc = iscsit_build_text_rsp(cmd, conn, hdr, ISCSI_INFINIBAND);
2031 2032 2033 2034 2035 2036 2037
	if (rc < 0)
		return rc;

	txt_rsp_len = rc;
	isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);

	if (txt_rsp_len) {
2038
		struct isert_device *device = isert_conn->device;
2039
		struct ib_device *ib_dev = device->ib_device;
2040 2041 2042 2043 2044
		struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1];
		void *txt_rsp_buf = cmd->buf_ptr;

		isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev,
				txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE);
2045 2046
		if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma))
			return -ENOMEM;
2047 2048 2049 2050

		isert_cmd->pdu_buf_len = txt_rsp_len;
		tx_dsg->addr	= isert_cmd->pdu_buf_dma;
		tx_dsg->length	= txt_rsp_len;
2051
		tx_dsg->lkey	= device->pd->local_dma_lkey;
2052 2053
		isert_cmd->tx_desc.num_sge = 2;
	}
2054
	isert_init_send_wr(isert_conn, isert_cmd, send_wr);
2055

2056
	isert_dbg("conn %p Text Response\n", isert_conn);
2057 2058 2059 2060

	return isert_post_response(isert_conn, isert_cmd);
}

2061
static inline void
2062
isert_set_dif_domain(struct se_cmd *se_cmd, struct ib_sig_domain *domain)
2063
{
2064
	domain->sig_type = IB_SIG_TYPE_T10_DIF;
2065 2066 2067
	domain->sig.dif.bg_type = IB_T10DIF_CRC;
	domain->sig.dif.pi_interval = se_cmd->se_dev->dev_attrib.block_size;
	domain->sig.dif.ref_tag = se_cmd->reftag_seed;
2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078
	/*
	 * At the moment we hard code those, but if in the future
	 * the target core would like to use it, we will take it
	 * from se_cmd.
	 */
	domain->sig.dif.apptag_check_mask = 0xffff;
	domain->sig.dif.app_escape = true;
	domain->sig.dif.ref_escape = true;
	if (se_cmd->prot_type == TARGET_DIF_TYPE1_PROT ||
	    se_cmd->prot_type == TARGET_DIF_TYPE2_PROT)
		domain->sig.dif.ref_remap = true;
2079 2080
};

2081 2082 2083
static int
isert_set_sig_attrs(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs)
{
2084 2085
	memset(sig_attrs, 0, sizeof(*sig_attrs));

2086 2087 2088
	switch (se_cmd->prot_op) {
	case TARGET_PROT_DIN_INSERT:
	case TARGET_PROT_DOUT_STRIP:
2089
		sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE;
2090
		isert_set_dif_domain(se_cmd, &sig_attrs->wire);
2091 2092 2093
		break;
	case TARGET_PROT_DOUT_INSERT:
	case TARGET_PROT_DIN_STRIP:
2094
		sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE;
2095
		isert_set_dif_domain(se_cmd, &sig_attrs->mem);
2096 2097 2098
		break;
	case TARGET_PROT_DIN_PASS:
	case TARGET_PROT_DOUT_PASS:
2099 2100
		isert_set_dif_domain(se_cmd, &sig_attrs->wire);
		isert_set_dif_domain(se_cmd, &sig_attrs->mem);
2101 2102
		break;
	default:
2103
		isert_err("Unsupported PI operation %d\n", se_cmd->prot_op);
2104 2105 2106
		return -EINVAL;
	}

2107 2108 2109 2110 2111 2112 2113
	if (se_cmd->prot_checks & TARGET_DIF_CHECK_GUARD)
		sig_attrs->check_mask |= IB_SIG_CHECK_GUARD;
	if (se_cmd->prot_checks & TARGET_DIF_CHECK_APPTAG)
		sig_attrs->check_mask |= IB_SIG_CHECK_APPTAG;
	if (se_cmd->prot_checks & TARGET_DIF_CHECK_REFTAG)
		sig_attrs->check_mask |= IB_SIG_CHECK_REFTAG;

2114 2115 2116 2117
	return 0;
}

static int
2118 2119 2120 2121 2122 2123 2124 2125
isert_rdma_rw_ctx_post(struct isert_cmd *cmd, struct isert_conn *conn,
		struct ib_cqe *cqe, struct ib_send_wr *chain_wr)
{
	struct se_cmd *se_cmd = &cmd->iscsi_cmd->se_cmd;
	enum dma_data_direction dir = target_reverse_dma_direction(se_cmd);
	u8 port_num = conn->cm_id->port_num;
	u64 addr;
	u32 rkey, offset;
2126 2127
	int ret;

2128 2129 2130
	if (cmd->ctx_init_done)
		goto rdma_ctx_post;

2131 2132 2133 2134
	if (dir == DMA_FROM_DEVICE) {
		addr = cmd->write_va;
		rkey = cmd->write_stag;
		offset = cmd->iscsi_cmd->write_data_done;
2135
	} else {
2136 2137 2138
		addr = cmd->read_va;
		rkey = cmd->read_stag;
		offset = 0;
2139
	}
2140

2141 2142
	if (isert_prot_cmd(conn, se_cmd)) {
		struct ib_sig_attrs sig_attrs;
2143

2144 2145 2146
		ret = isert_set_sig_attrs(se_cmd, &sig_attrs);
		if (ret)
			return ret;
2147

2148 2149 2150 2151 2152
		WARN_ON_ONCE(offset);
		ret = rdma_rw_ctx_signature_init(&cmd->rw, conn->qp, port_num,
				se_cmd->t_data_sg, se_cmd->t_data_nents,
				se_cmd->t_prot_sg, se_cmd->t_prot_nents,
				&sig_attrs, addr, rkey, dir);
2153
	} else {
2154 2155 2156
		ret = rdma_rw_ctx_init(&cmd->rw, conn->qp, port_num,
				se_cmd->t_data_sg, se_cmd->t_data_nents,
				offset, addr, rkey, dir);
2157
	}
2158

2159 2160 2161
	if (ret < 0) {
		isert_err("Cmd: %p failed to prepare RDMA res\n", cmd);
		return ret;
2162 2163
	}

2164 2165 2166
	cmd->ctx_init_done = true;

rdma_ctx_post:
2167 2168 2169
	ret = rdma_rw_ctx_post(&cmd->rw, conn->qp, port_num, cqe, chain_wr);
	if (ret < 0)
		isert_err("Cmd: %p failed to post RDMA res\n", cmd);
2170 2171 2172
	return ret;
}

2173 2174 2175 2176
static int
isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
{
	struct se_cmd *se_cmd = &cmd->se_cmd;
2177
	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
2178
	struct isert_conn *isert_conn = conn->context;
2179 2180
	struct ib_cqe *cqe = NULL;
	struct ib_send_wr *chain_wr = NULL;
2181 2182
	int rc;

2183
	isert_dbg("Cmd: %p RDMA_WRITE data_length: %u\n",
2184
		 isert_cmd, se_cmd->data_length);
2185

2186 2187 2188 2189
	if (isert_prot_cmd(isert_conn, se_cmd)) {
		isert_cmd->tx_desc.tx_cqe.done = isert_rdma_write_done;
		cqe = &isert_cmd->tx_desc.tx_cqe;
	} else {
2190 2191 2192 2193 2194 2195 2196 2197 2198
		/*
		 * Build isert_conn->tx_desc for iSCSI response PDU and attach
		 */
		isert_create_send_desc(isert_conn, isert_cmd,
				       &isert_cmd->tx_desc);
		iscsit_build_rsp_pdu(cmd, conn, true, (struct iscsi_scsi_rsp *)
				     &isert_cmd->tx_desc.iscsi_header);
		isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
		isert_init_send_wr(isert_conn, isert_cmd,
2199
				   &isert_cmd->tx_desc.send_wr);
2200 2201 2202 2203 2204 2205

		rc = isert_post_recv(isert_conn, isert_cmd->rx_desc);
		if (rc) {
			isert_err("ib_post_recv failed with %d\n", rc);
			return rc;
		}
2206

2207 2208
		chain_wr = &isert_cmd->tx_desc.send_wr;
	}
2209

2210 2211 2212 2213
	rc = isert_rdma_rw_ctx_post(isert_cmd, isert_conn, cqe, chain_wr);
	isert_dbg("Cmd: %p posted RDMA_WRITE for iSER Data READ rc: %d\n",
		  isert_cmd, rc);
	return rc;
2214 2215 2216 2217 2218
}

static int
isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery)
{
2219
	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
2220
	int ret;
2221

2222
	isert_dbg("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n",
2223
		 isert_cmd, cmd->se_cmd.data_length, cmd->write_data_done);
2224

2225
	isert_cmd->tx_desc.tx_cqe.done = isert_rdma_read_done;
2226 2227
	ret = isert_rdma_rw_ctx_post(isert_cmd, conn->context,
				     &isert_cmd->tx_desc.tx_cqe, NULL);
2228

2229 2230 2231
	isert_dbg("Cmd: %p posted RDMA_READ memory for ISER Data WRITE rc: %d\n",
		 isert_cmd, ret);
	return ret;
2232 2233 2234 2235 2236
}

static int
isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state)
{
2237 2238
	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
	int ret = 0;
2239 2240

	switch (state) {
2241 2242 2243 2244 2245 2246
	case ISTATE_REMOVE:
		spin_lock_bh(&conn->cmd_lock);
		list_del_init(&cmd->i_conn_node);
		spin_unlock_bh(&conn->cmd_lock);
		isert_put_cmd(isert_cmd, true);
		break;
2247 2248 2249 2250
	case ISTATE_SEND_NOPIN_WANT_RESPONSE:
		ret = isert_put_nopin(cmd, conn, false);
		break;
	default:
2251
		isert_err("Unknown immediate state: 0x%02x\n", state);
2252 2253 2254 2255 2256 2257 2258 2259 2260 2261
		ret = -EINVAL;
		break;
	}

	return ret;
}

static int
isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state)
{
2262
	struct isert_conn *isert_conn = conn->context;
2263 2264 2265 2266 2267
	int ret;

	switch (state) {
	case ISTATE_SEND_LOGOUTRSP:
		ret = isert_put_logout_rsp(cmd, conn);
2268 2269
		if (!ret)
			isert_conn->logout_posted = true;
2270 2271 2272 2273 2274 2275 2276 2277 2278 2279
		break;
	case ISTATE_SEND_NOPIN:
		ret = isert_put_nopin(cmd, conn, true);
		break;
	case ISTATE_SEND_TASKMGTRSP:
		ret = isert_put_tm_rsp(cmd, conn);
		break;
	case ISTATE_SEND_REJECT:
		ret = isert_put_reject(cmd, conn);
		break;
2280 2281 2282
	case ISTATE_SEND_TEXTRSP:
		ret = isert_put_text_rsp(cmd, conn);
		break;
2283 2284 2285 2286 2287 2288 2289 2290
	case ISTATE_SEND_STATUS:
		/*
		 * Special case for sending non GOOD SCSI status from TX thread
		 * context during pre se_cmd excecution failure.
		 */
		ret = isert_put_response(conn, cmd);
		break;
	default:
2291
		isert_err("Unknown response state: 0x%02x\n", state);
2292 2293 2294 2295 2296 2297 2298
		ret = -EINVAL;
		break;
	}

	return ret;
}

2299 2300 2301 2302 2303 2304 2305 2306 2307
struct rdma_cm_id *
isert_setup_id(struct isert_np *isert_np)
{
	struct iscsi_np *np = isert_np->np;
	struct rdma_cm_id *id;
	struct sockaddr *sa;
	int ret;

	sa = (struct sockaddr *)&np->np_sockaddr;
2308
	isert_dbg("ksockaddr: %p, sa: %p\n", &np->np_sockaddr, sa);
2309

2310
	id = rdma_create_id(&init_net, isert_cma_handler, isert_np,
2311 2312
			    RDMA_PS_TCP, IB_QPT_RC);
	if (IS_ERR(id)) {
2313
		isert_err("rdma_create_id() failed: %ld\n", PTR_ERR(id));
2314 2315 2316
		ret = PTR_ERR(id);
		goto out;
	}
2317
	isert_dbg("id %p context %p\n", id, id->context);
2318 2319 2320

	ret = rdma_bind_addr(id, sa);
	if (ret) {
2321
		isert_err("rdma_bind_addr() failed: %d\n", ret);
2322 2323 2324
		goto out_id;
	}

2325
	ret = rdma_listen(id, 0);
2326
	if (ret) {
2327
		isert_err("rdma_listen() failed: %d\n", ret);
2328 2329 2330 2331 2332 2333 2334 2335 2336 2337
		goto out_id;
	}

	return id;
out_id:
	rdma_destroy_id(id);
out:
	return ERR_PTR(ret);
}

2338 2339
static int
isert_setup_np(struct iscsi_np *np,
2340
	       struct sockaddr_storage *ksockaddr)
2341 2342 2343 2344 2345 2346
{
	struct isert_np *isert_np;
	struct rdma_cm_id *isert_lid;
	int ret;

	isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL);
2347
	if (!isert_np)
2348
		return -ENOMEM;
2349

2350 2351
	sema_init(&isert_np->sem, 0);
	mutex_init(&isert_np->mutex);
2352 2353
	INIT_LIST_HEAD(&isert_np->accepted);
	INIT_LIST_HEAD(&isert_np->pending);
2354
	isert_np->np = np;
2355 2356 2357 2358 2359 2360

	/*
	 * Setup the np->np_sockaddr from the passed sockaddr setup
	 * in iscsi_target_configfs.c code..
	 */
	memcpy(&np->np_sockaddr, ksockaddr,
2361
	       sizeof(struct sockaddr_storage));
2362

2363
	isert_lid = isert_setup_id(isert_np);
2364 2365 2366 2367 2368
	if (IS_ERR(isert_lid)) {
		ret = PTR_ERR(isert_lid);
		goto out;
	}

2369
	isert_np->cm_id = isert_lid;
2370 2371 2372 2373 2374 2375
	np->np_context = isert_np;

	return 0;

out:
	kfree(isert_np);
2376

2377 2378 2379 2380 2381 2382
	return ret;
}

static int
isert_rdma_accept(struct isert_conn *isert_conn)
{
2383
	struct rdma_cm_id *cm_id = isert_conn->cm_id;
2384 2385
	struct rdma_conn_param cp;
	int ret;
2386
	struct iser_cm_hdr rsp_hdr;
2387 2388 2389 2390 2391 2392

	memset(&cp, 0, sizeof(struct rdma_conn_param));
	cp.initiator_depth = isert_conn->initiator_depth;
	cp.retry_count = 7;
	cp.rnr_retry_count = 7;

2393
	memset(&rsp_hdr, 0, sizeof(rsp_hdr));
2394 2395 2396
	rsp_hdr.flags = ISERT_ZBVA_NOT_USED;
	if (!isert_conn->snd_w_inv)
		rsp_hdr.flags = rsp_hdr.flags | ISERT_SEND_W_INV_NOT_USED;
2397 2398 2399
	cp.private_data = (void *)&rsp_hdr;
	cp.private_data_len = sizeof(rsp_hdr);

2400 2401
	ret = rdma_accept(cm_id, &cp);
	if (ret) {
2402
		isert_err("rdma_accept() failed with: %d\n", ret);
2403 2404 2405 2406 2407 2408 2409 2410 2411
		return ret;
	}

	return 0;
}

static int
isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login)
{
2412
	struct isert_conn *isert_conn = conn->context;
2413 2414
	int ret;

2415
	isert_info("before login_req comp conn: %p\n", isert_conn);
2416 2417
	ret = wait_for_completion_interruptible(&isert_conn->login_req_comp);
	if (ret) {
2418
		isert_err("isert_conn %p interrupted before got login req\n",
2419 2420 2421 2422 2423
			  isert_conn);
		return ret;
	}
	reinit_completion(&isert_conn->login_req_comp);

2424 2425 2426 2427 2428 2429 2430 2431
	/*
	 * For login requests after the first PDU, isert_rx_login_req() will
	 * kick schedule_delayed_work(&conn->login_work) as the packet is
	 * received, which turns this callback from iscsi_target_do_login_rx()
	 * into a NOP.
	 */
	if (!login->first_request)
		return 0;
2432

2433 2434
	isert_rx_login_req(isert_conn);

2435 2436
	isert_info("before login_comp conn: %p\n", conn);
	ret = wait_for_completion_interruptible(&isert_conn->login_comp);
2437 2438 2439
	if (ret)
		return ret;

2440
	isert_info("processing login->req: %p\n", login->req);
2441

2442 2443 2444 2445 2446 2447 2448
	return 0;
}

static void
isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn,
		    struct isert_conn *isert_conn)
{
2449
	struct rdma_cm_id *cm_id = isert_conn->cm_id;
2450 2451 2452 2453
	struct rdma_route *cm_route = &cm_id->route;

	conn->login_family = np->np_sockaddr.ss_family;

2454 2455
	conn->login_sockaddr = cm_route->addr.dst_addr;
	conn->local_sockaddr = cm_route->addr.src_addr;
2456 2457 2458 2459 2460
}

static int
isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn)
{
2461
	struct isert_np *isert_np = np->np_context;
2462
	struct isert_conn *isert_conn;
2463
	int ret;
2464 2465

accept_wait:
2466
	ret = down_interruptible(&isert_np->sem);
2467
	if (ret)
2468 2469 2470
		return -ENODEV;

	spin_lock_bh(&np->np_thread_lock);
2471
	if (np->np_thread_state >= ISCSI_NP_THREAD_RESET) {
2472
		spin_unlock_bh(&np->np_thread_lock);
2473
		isert_dbg("np_thread_state %d\n",
2474 2475 2476 2477 2478
			 np->np_thread_state);
		/**
		 * No point in stalling here when np_thread
		 * is in state RESET/SHUTDOWN/EXIT - bail
		 **/
2479 2480 2481 2482
		return -ENODEV;
	}
	spin_unlock_bh(&np->np_thread_lock);

2483
	mutex_lock(&isert_np->mutex);
2484
	if (list_empty(&isert_np->pending)) {
2485
		mutex_unlock(&isert_np->mutex);
2486 2487
		goto accept_wait;
	}
2488 2489 2490
	isert_conn = list_first_entry(&isert_np->pending,
			struct isert_conn, node);
	list_del_init(&isert_conn->node);
2491
	mutex_unlock(&isert_np->mutex);
2492 2493 2494

	conn->context = isert_conn;
	isert_conn->conn = conn;
2495
	isert_conn->state = ISER_CONN_BOUND;
2496 2497 2498

	isert_set_conn_info(np, conn, isert_conn);

2499
	isert_dbg("Processing isert_conn: %p\n", isert_conn);
2500

2501 2502 2503 2504 2505 2506
	return 0;
}

static void
isert_free_np(struct iscsi_np *np)
{
2507
	struct isert_np *isert_np = np->np_context;
2508
	struct isert_conn *isert_conn, *n;
2509

2510 2511
	if (isert_np->cm_id)
		rdma_destroy_id(isert_np->cm_id);
2512

2513 2514 2515 2516 2517
	/*
	 * FIXME: At this point we don't have a good way to insure
	 * that at this point we don't have hanging connections that
	 * completed RDMA establishment but didn't start iscsi login
	 * process. So work-around this by cleaning up what ever piled
2518
	 * up in accepted and pending lists.
2519
	 */
2520
	mutex_lock(&isert_np->mutex);
2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533
	if (!list_empty(&isert_np->pending)) {
		isert_info("Still have isert pending connections\n");
		list_for_each_entry_safe(isert_conn, n,
					 &isert_np->pending,
					 node) {
			isert_info("cleaning isert_conn %p state (%d)\n",
				   isert_conn, isert_conn->state);
			isert_connect_release(isert_conn);
		}
	}

	if (!list_empty(&isert_np->accepted)) {
		isert_info("Still have isert accepted connections\n");
2534
		list_for_each_entry_safe(isert_conn, n,
2535 2536
					 &isert_np->accepted,
					 node) {
2537
			isert_info("cleaning isert_conn %p state (%d)\n",
2538 2539 2540 2541
				   isert_conn, isert_conn->state);
			isert_connect_release(isert_conn);
		}
	}
2542
	mutex_unlock(&isert_np->mutex);
2543

2544 2545 2546 2547
	np->np_context = NULL;
	kfree(isert_np);
}

2548 2549 2550 2551 2552 2553
static void isert_release_work(struct work_struct *work)
{
	struct isert_conn *isert_conn = container_of(work,
						     struct isert_conn,
						     release_work);

2554
	isert_info("Starting release conn %p\n", isert_conn);
2555

2556
	mutex_lock(&isert_conn->mutex);
2557
	isert_conn->state = ISER_CONN_DOWN;
2558
	mutex_unlock(&isert_conn->mutex);
2559

2560
	isert_info("Destroying conn %p\n", isert_conn);
2561 2562 2563
	isert_put_conn(isert_conn);
}

2564 2565 2566 2567 2568
static void
isert_wait4logout(struct isert_conn *isert_conn)
{
	struct iscsi_conn *conn = isert_conn->conn;

2569 2570
	isert_info("conn %p\n", isert_conn);

2571
	if (isert_conn->logout_posted) {
2572
		isert_info("conn %p wait for conn_logout_comp\n", isert_conn);
2573 2574 2575 2576 2577
		wait_for_completion_timeout(&conn->conn_logout_comp,
					    SECONDS_FOR_LOGOUT_COMP * HZ);
	}
}

2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612
/**
 * isert_put_unsol_pending_cmds() - Drop commands waiting for
 *     unsolicitate dataout
 * @conn:    iscsi connection
 *
 * We might still have commands that are waiting for unsolicited
 * dataouts messages. We must put the extra reference on those
 * before blocking on the target_wait_for_session_cmds
 */
static void
isert_put_unsol_pending_cmds(struct iscsi_conn *conn)
{
	struct iscsi_cmd *cmd, *tmp;
	static LIST_HEAD(drop_cmd_list);

	spin_lock_bh(&conn->cmd_lock);
	list_for_each_entry_safe(cmd, tmp, &conn->conn_cmd_list, i_conn_node) {
		if ((cmd->cmd_flags & ICF_NON_IMMEDIATE_UNSOLICITED_DATA) &&
		    (cmd->write_data_done < conn->sess->sess_ops->FirstBurstLength) &&
		    (cmd->write_data_done < cmd->se_cmd.data_length))
			list_move_tail(&cmd->i_conn_node, &drop_cmd_list);
	}
	spin_unlock_bh(&conn->cmd_lock);

	list_for_each_entry_safe(cmd, tmp, &drop_cmd_list, i_conn_node) {
		list_del_init(&cmd->i_conn_node);
		if (cmd->i_state != ISTATE_REMOVE) {
			struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);

			isert_info("conn %p dropping cmd %p\n", conn, cmd);
			isert_put_cmd(isert_cmd, true);
		}
	}
}

2613
static void isert_wait_conn(struct iscsi_conn *conn)
2614 2615 2616
{
	struct isert_conn *isert_conn = conn->context;

2617
	isert_info("Starting conn %p\n", isert_conn);
2618

2619
	mutex_lock(&isert_conn->mutex);
2620
	isert_conn_terminate(isert_conn);
2621
	mutex_unlock(&isert_conn->mutex);
2622

2623
	ib_drain_qp(isert_conn->qp);
2624
	isert_put_unsol_pending_cmds(conn);
2625
	isert_wait4logout(isert_conn);
2626

2627
	queue_work(isert_release_wq, &isert_conn->release_work);
2628 2629 2630 2631 2632
}

static void isert_free_conn(struct iscsi_conn *conn)
{
	struct isert_conn *isert_conn = conn->context;
2633

2634
	ib_drain_qp(isert_conn->qp);
2635 2636 2637
	isert_put_conn(isert_conn);
}

2638 2639 2640 2641 2642 2643 2644 2645 2646
static void isert_get_rx_pdu(struct iscsi_conn *conn)
{
	struct completion comp;

	init_completion(&comp);

	wait_for_completion_interruptible(&comp);
}

2647 2648 2649
static struct iscsit_transport iser_target_transport = {
	.name			= "IB/iSER",
	.transport_type		= ISCSI_INFINIBAND,
2650
	.rdma_shutdown		= true,
2651
	.priv_size		= sizeof(struct isert_cmd),
2652 2653 2654 2655
	.owner			= THIS_MODULE,
	.iscsit_setup_np	= isert_setup_np,
	.iscsit_accept_np	= isert_accept_np,
	.iscsit_free_np		= isert_free_np,
2656
	.iscsit_wait_conn	= isert_wait_conn,
2657 2658 2659 2660 2661 2662 2663 2664
	.iscsit_free_conn	= isert_free_conn,
	.iscsit_get_login_rx	= isert_get_login_rx,
	.iscsit_put_login_tx	= isert_put_login_tx,
	.iscsit_immediate_queue	= isert_immediate_queue,
	.iscsit_response_queue	= isert_response_queue,
	.iscsit_get_dataout	= isert_get_dataout,
	.iscsit_queue_data_in	= isert_put_datain,
	.iscsit_queue_status	= isert_put_response,
2665
	.iscsit_aborted_task	= isert_aborted_task,
2666
	.iscsit_get_rx_pdu	= isert_get_rx_pdu,
2667
	.iscsit_get_sup_prot_ops = isert_get_sup_prot_ops,
2668 2669 2670 2671 2672 2673
};

static int __init isert_init(void)
{
	int ret;

2674 2675
	isert_comp_wq = alloc_workqueue("isert_comp_wq",
					WQ_UNBOUND | WQ_HIGHPRI, 0);
2676
	if (!isert_comp_wq) {
2677
		isert_err("Unable to allocate isert_comp_wq\n");
2678
		return -ENOMEM;
2679 2680
	}

2681 2682 2683
	isert_release_wq = alloc_workqueue("isert_release_wq", WQ_UNBOUND,
					WQ_UNBOUND_MAX_ACTIVE);
	if (!isert_release_wq) {
2684
		isert_err("Unable to allocate isert_release_wq\n");
2685 2686 2687 2688
		ret = -ENOMEM;
		goto destroy_comp_wq;
	}

2689
	iscsit_register_transport(&iser_target_transport);
2690
	isert_info("iSER_TARGET[0] - Loaded iser_target_transport\n");
2691

2692 2693
	return 0;

2694 2695
destroy_comp_wq:
	destroy_workqueue(isert_comp_wq);
2696

2697 2698 2699 2700 2701
	return ret;
}

static void __exit isert_exit(void)
{
2702
	flush_scheduled_work();
2703
	destroy_workqueue(isert_release_wq);
2704 2705
	destroy_workqueue(isert_comp_wq);
	iscsit_unregister_transport(&iser_target_transport);
2706
	isert_info("iSER_TARGET[0] - Released iser_target_transport\n");
2707 2708 2709 2710 2711 2712 2713 2714
}

MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure");
MODULE_AUTHOR("nab@Linux-iSCSI.org");
MODULE_LICENSE("GPL");

module_init(isert_init);
module_exit(isert_exit);