pblk-init.c 32 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
/*
 * Copyright (C) 2015 IT University of Copenhagen (rrpc.c)
 * Copyright (C) 2016 CNEX Labs
 * Initial release: Javier Gonzalez <javier@cnexlabs.com>
 *                  Matias Bjorling <matias@cnexlabs.com>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License version
 * 2 as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * Implementation of a physical block-device target for Open-channel SSDs.
 *
 * pblk-init.c - pblk's initialization.
 */

#include "pblk.h"
23
#include "pblk-trace.h"
24

25
static unsigned int write_buffer_size;
26 27 28 29

module_param(write_buffer_size, uint, 0644);
MODULE_PARM_DESC(write_buffer_size, "number of entries in a write buffer");

30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47
struct pblk_global_caches {
	struct kmem_cache	*ws;
	struct kmem_cache	*rec;
	struct kmem_cache	*g_rq;
	struct kmem_cache	*w_rq;

	struct kref		kref;

	struct mutex		mutex; /* Ensures consistency between
					* caches and kref
					*/
};

static struct pblk_global_caches pblk_caches = {
	.mutex = __MUTEX_INITIALIZER(pblk_caches.mutex),
	.kref = KREF_INIT(0),
};

48
struct bio_set pblk_bio_set;
49 50 51 52 53 54 55 56 57 58

static int pblk_rw_io(struct request_queue *q, struct pblk *pblk,
			  struct bio *bio)
{
	int ret;

	/* Read requests must be <= 256kb due to NVMe's 64 bit completion bitmap
	 * constraint. Writes can be of arbitrary size.
	 */
	if (bio_data_dir(bio) == READ) {
59
		blk_queue_split(q, &bio);
60 61 62 63 64 65 66 67 68 69 70
		ret = pblk_submit_read(pblk, bio);
		if (ret == NVM_IO_DONE && bio_flagged(bio, BIO_CLONED))
			bio_put(bio);

		return ret;
	}

	/* Prevent deadlock in the case of a modest LUN configuration and large
	 * user I/Os. Unless stalled, the rate limiter leaves at least 256KB
	 * available for user I/O.
	 */
71
	if (pblk_get_secs(bio) > pblk_rl_max_io(&pblk->rl))
72
		blk_queue_split(q, &bio);
73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100

	return pblk_write_to_cache(pblk, bio, PBLK_IOTYPE_USER);
}

static blk_qc_t pblk_make_rq(struct request_queue *q, struct bio *bio)
{
	struct pblk *pblk = q->queuedata;

	if (bio_op(bio) == REQ_OP_DISCARD) {
		pblk_discard(pblk, bio);
		if (!(bio->bi_opf & REQ_PREFLUSH)) {
			bio_endio(bio);
			return BLK_QC_T_NONE;
		}
	}

	switch (pblk_rw_io(q, pblk, bio)) {
	case NVM_IO_ERR:
		bio_io_error(bio);
		break;
	case NVM_IO_DONE:
		bio_endio(bio);
		break;
	}

	return BLK_QC_T_NONE;
}

101 102 103 104
static size_t pblk_trans_map_size(struct pblk *pblk)
{
	int entry_size = 8;

105
	if (pblk->addrf_len < 32)
106 107 108 109 110
		entry_size = 4;

	return entry_size * pblk->rl.nr_secs;
}

111
#ifdef CONFIG_NVM_PBLK_DEBUG
112 113 114 115 116 117 118 119 120 121 122
static u32 pblk_l2p_crc(struct pblk *pblk)
{
	size_t map_size;
	u32 crc = ~(u32)0;

	map_size = pblk_trans_map_size(pblk);
	crc = crc32_le(crc, pblk->trans_map, map_size);
	return crc;
}
#endif

123 124 125 126 127
static void pblk_l2p_free(struct pblk *pblk)
{
	vfree(pblk->trans_map);
}

128 129 130 131 132 133 134 135 136
static int pblk_l2p_recover(struct pblk *pblk, bool factory_init)
{
	struct pblk_line *line = NULL;

	if (factory_init) {
		pblk_setup_uuid(pblk);
	} else {
		line = pblk_recov_l2p(pblk);
		if (IS_ERR(line)) {
137
			pblk_err(pblk, "could not recover l2p table\n");
138 139 140 141
			return -EFAULT;
		}
	}

142
#ifdef CONFIG_NVM_PBLK_DEBUG
143
	pblk_info(pblk, "init: L2P CRC: %x\n", pblk_l2p_crc(pblk));
144 145 146 147 148 149 150 151
#endif

	/* Free full lines directly as GC has not been started yet */
	pblk_gc_free_full_lines(pblk);

	if (!line) {
		/* Configure next line for user data */
		line = pblk_line_get_first_data(pblk);
152
		if (!line)
153 154 155 156 157 158 159
			return -EFAULT;
	}

	return 0;
}

static int pblk_l2p_init(struct pblk *pblk, bool factory_init)
160 161 162
{
	sector_t i;
	struct ppa_addr ppa;
163
	size_t map_size;
164
	int ret = 0;
165

166 167
	map_size = pblk_trans_map_size(pblk);
	pblk->trans_map = vmalloc(map_size);
168 169 170 171 172 173 174 175
	if (!pblk->trans_map)
		return -ENOMEM;

	pblk_ppa_set_empty(&ppa);

	for (i = 0; i < pblk->rl.nr_secs; i++)
		pblk_trans_map_set(pblk, i, ppa);

176 177 178 179 180
	ret = pblk_l2p_recover(pblk, factory_init);
	if (ret)
		vfree(pblk->trans_map);

	return ret;
181 182 183 184 185
}

static void pblk_rwb_free(struct pblk *pblk)
{
	if (pblk_rb_tear_down_check(&pblk->rwb))
186
		pblk_err(pblk, "write buffer error on tear down\n");
187

188
	pblk_rb_free(&pblk->rwb);
189 190 191 192 193 194
}

static int pblk_rwb_init(struct pblk *pblk)
{
	struct nvm_tgt_dev *dev = pblk->dev;
	struct nvm_geo *geo = &dev->geo;
195
	unsigned long buffer_size;
196
	int pgs_in_buffer, threshold;
197

198
	threshold = geo->mw_cunits * geo->all_luns;
199 200
	pgs_in_buffer = (max(geo->mw_cunits, geo->ws_opt) + geo->ws_opt)
								* geo->all_luns;
201 202

	if (write_buffer_size && (write_buffer_size > pgs_in_buffer))
203 204
		buffer_size = write_buffer_size;
	else
205
		buffer_size = pgs_in_buffer;
206

207
	return pblk_rb_init(&pblk->rwb, buffer_size, threshold, geo->csecs);
208 209
}

210 211
static int pblk_set_addrf_12(struct pblk *pblk, struct nvm_geo *geo,
			     struct nvm_addrf_12 *dst)
212
{
213 214
	struct nvm_addrf_12 *src = (struct nvm_addrf_12 *)&geo->addrf;
	int power_len;
215 216

	/* Re-calculate channel and lun format to adapt to configuration */
217 218
	power_len = get_count_order(geo->num_ch);
	if (1 << power_len != geo->num_ch) {
219
		pblk_err(pblk, "supports only power-of-two channel config.\n");
220 221
		return -EINVAL;
	}
222
	dst->ch_len = power_len;
223

224 225
	power_len = get_count_order(geo->num_lun);
	if (1 << power_len != geo->num_lun) {
226
		pblk_err(pblk, "supports only power-of-two LUN config.\n");
227 228
		return -EINVAL;
	}
229 230 231 232 233
	dst->lun_len = power_len;

	dst->blk_len = src->blk_len;
	dst->pg_len = src->pg_len;
	dst->pln_len = src->pln_len;
234
	dst->sec_len = src->sec_len;
235

236 237
	dst->sec_offset = 0;
	dst->pln_offset = dst->sec_len;
238 239 240 241 242
	dst->ch_offset = dst->pln_offset + dst->pln_len;
	dst->lun_offset = dst->ch_offset + dst->ch_len;
	dst->pg_offset = dst->lun_offset + dst->lun_len;
	dst->blk_offset = dst->pg_offset + dst->pg_len;

243
	dst->sec_mask = ((1ULL << dst->sec_len) - 1) << dst->sec_offset;
244 245 246 247 248 249 250 251 252
	dst->pln_mask = ((1ULL << dst->pln_len) - 1) << dst->pln_offset;
	dst->ch_mask = ((1ULL << dst->ch_len) - 1) << dst->ch_offset;
	dst->lun_mask = ((1ULL << dst->lun_len) - 1) << dst->lun_offset;
	dst->pg_mask = ((1ULL << dst->pg_len) - 1) << dst->pg_offset;
	dst->blk_mask = ((1ULL << dst->blk_len) - 1) << dst->blk_offset;

	return dst->blk_offset + src->blk_len;
}

253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282
static int pblk_set_addrf_20(struct nvm_geo *geo, struct nvm_addrf *adst,
			     struct pblk_addrf *udst)
{
	struct nvm_addrf *src = &geo->addrf;

	adst->ch_len = get_count_order(geo->num_ch);
	adst->lun_len = get_count_order(geo->num_lun);
	adst->chk_len = src->chk_len;
	adst->sec_len = src->sec_len;

	adst->sec_offset = 0;
	adst->ch_offset = adst->sec_len;
	adst->lun_offset = adst->ch_offset + adst->ch_len;
	adst->chk_offset = adst->lun_offset + adst->lun_len;

	adst->sec_mask = ((1ULL << adst->sec_len) - 1) << adst->sec_offset;
	adst->chk_mask = ((1ULL << adst->chk_len) - 1) << adst->chk_offset;
	adst->lun_mask = ((1ULL << adst->lun_len) - 1) << adst->lun_offset;
	adst->ch_mask = ((1ULL << adst->ch_len) - 1) << adst->ch_offset;

	udst->sec_stripe = geo->ws_opt;
	udst->ch_stripe = geo->num_ch;
	udst->lun_stripe = geo->num_lun;

	udst->sec_lun_stripe = udst->sec_stripe * udst->ch_stripe;
	udst->sec_ws_stripe = udst->sec_lun_stripe * udst->lun_stripe;

	return adst->chk_offset + adst->chk_len;
}

283
static int pblk_set_addrf(struct pblk *pblk)
284 285 286 287 288
{
	struct nvm_tgt_dev *dev = pblk->dev;
	struct nvm_geo *geo = &dev->geo;
	int mod;

289 290 291 292
	switch (geo->version) {
	case NVM_OCSSD_SPEC_12:
		div_u64_rem(geo->clba, pblk->min_write_pgs, &mod);
		if (mod) {
293
			pblk_err(pblk, "bad configuration of sectors/pages\n");
294 295 296
			return -EINVAL;
		}

297 298
		pblk->addrf_len = pblk_set_addrf_12(pblk, geo,
							(void *)&pblk->addrf);
299 300 301
		break;
	case NVM_OCSSD_SPEC_20:
		pblk->addrf_len = pblk_set_addrf_20(geo, (void *)&pblk->addrf,
302
							&pblk->uaddrf);
303 304
		break;
	default:
305
		pblk_err(pblk, "OCSSD revision not supported (%d)\n",
306
								geo->version);
307 308 309
		return -EINVAL;
	}

310 311 312
	return 0;
}

313
static int pblk_create_global_caches(void)
314
{
315 316

	pblk_caches.ws = kmem_cache_create("pblk_blk_ws",
317
				sizeof(struct pblk_line_ws), 0, 0, NULL);
318
	if (!pblk_caches.ws)
319 320
		return -ENOMEM;

321
	pblk_caches.rec = kmem_cache_create("pblk_rec",
322
				sizeof(struct pblk_rec_ctx), 0, 0, NULL);
323 324
	if (!pblk_caches.rec)
		goto fail_destroy_ws;
325

326
	pblk_caches.g_rq = kmem_cache_create("pblk_g_rq", pblk_g_rq_size,
327
				0, 0, NULL);
328 329
	if (!pblk_caches.g_rq)
		goto fail_destroy_rec;
330

331
	pblk_caches.w_rq = kmem_cache_create("pblk_w_rq", pblk_w_rq_size,
332
				0, 0, NULL);
333 334
	if (!pblk_caches.w_rq)
		goto fail_destroy_g_rq;
335 336

	return 0;
337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379

fail_destroy_g_rq:
	kmem_cache_destroy(pblk_caches.g_rq);
fail_destroy_rec:
	kmem_cache_destroy(pblk_caches.rec);
fail_destroy_ws:
	kmem_cache_destroy(pblk_caches.ws);

	return -ENOMEM;
}

static int pblk_get_global_caches(void)
{
	int ret;

	mutex_lock(&pblk_caches.mutex);

	if (kref_read(&pblk_caches.kref) > 0) {
		kref_get(&pblk_caches.kref);
		mutex_unlock(&pblk_caches.mutex);
		return 0;
	}

	ret = pblk_create_global_caches();

	if (!ret)
		kref_get(&pblk_caches.kref);

	mutex_unlock(&pblk_caches.mutex);

	return ret;
}

static void pblk_destroy_global_caches(struct kref *ref)
{
	struct pblk_global_caches *c;

	c = container_of(ref, struct pblk_global_caches, kref);

	kmem_cache_destroy(c->ws);
	kmem_cache_destroy(c->rec);
	kmem_cache_destroy(c->g_rq);
	kmem_cache_destroy(c->w_rq);
380 381
}

382
static void pblk_put_global_caches(void)
383
{
384 385 386
	mutex_lock(&pblk_caches.mutex);
	kref_put(&pblk_caches.kref, pblk_destroy_global_caches);
	mutex_unlock(&pblk_caches.mutex);
387 388
}

389 390 391 392
static int pblk_core_init(struct pblk *pblk)
{
	struct nvm_tgt_dev *dev = pblk->dev;
	struct nvm_geo *geo = &dev->geo;
393
	int ret, max_write_ppas;
394 395 396 397 398 399 400 401 402 403

	atomic64_set(&pblk->user_wa, 0);
	atomic64_set(&pblk->pad_wa, 0);
	atomic64_set(&pblk->gc_wa, 0);
	pblk->user_rst_wa = 0;
	pblk->pad_rst_wa = 0;
	pblk->gc_rst_wa = 0;

	atomic64_set(&pblk->nr_flush, 0);
	pblk->nr_flush_rst = 0;
404

405
	pblk->min_write_pgs = geo->ws_opt;
406 407
	max_write_ppas = pblk->min_write_pgs * geo->all_luns;
	pblk->max_write_pgs = min_t(int, max_write_ppas, NVM_MAX_VLBA);
408 409
	pblk->max_write_pgs = min_t(int, pblk->max_write_pgs,
		queue_max_hw_sectors(dev->q) / (geo->csecs >> SECTOR_SHIFT));
410 411
	pblk_set_sec_per_write(pblk, pblk->min_write_pgs);

412
	pblk->pad_dist = kcalloc(pblk->min_write_pgs - 1, sizeof(atomic64_t),
413 414
								GFP_KERNEL);
	if (!pblk->pad_dist)
415 416
		return -ENOMEM;

417
	if (pblk_get_global_caches())
418 419
		goto fail_free_pad_dist;

420
	/* Internal bios can be at most the sectors signaled by the device. */
421 422
	ret = mempool_init_page_pool(&pblk->page_bio_pool, NVM_MAX_VLBA, 0);
	if (ret)
423
		goto free_global_caches;
424

425
	ret = mempool_init_slab_pool(&pblk->gen_ws_pool, PBLK_GEN_WS_POOL_SIZE,
426
				     pblk_caches.ws);
427
	if (ret)
428
		goto free_page_bio_pool;
429

430
	ret = mempool_init_slab_pool(&pblk->rec_pool, geo->all_luns,
431
				     pblk_caches.rec);
432
	if (ret)
433
		goto free_gen_ws_pool;
434

435
	ret = mempool_init_slab_pool(&pblk->r_rq_pool, geo->all_luns,
436
				     pblk_caches.g_rq);
437
	if (ret)
438 439
		goto free_rec_pool;

440
	ret = mempool_init_slab_pool(&pblk->e_rq_pool, geo->all_luns,
441
				     pblk_caches.g_rq);
442
	if (ret)
443 444
		goto free_r_rq_pool;

445
	ret = mempool_init_slab_pool(&pblk->w_rq_pool, geo->all_luns,
446
				     pblk_caches.w_rq);
447
	if (ret)
448
		goto free_e_rq_pool;
449

450 451 452
	pblk->close_wq = alloc_workqueue("pblk-close-wq",
			WQ_MEM_RECLAIM | WQ_UNBOUND, PBLK_NR_CLOSE_JOBS);
	if (!pblk->close_wq)
453
		goto free_w_rq_pool;
454

455 456 457 458 459
	pblk->bb_wq = alloc_workqueue("pblk-bb-wq",
			WQ_MEM_RECLAIM | WQ_UNBOUND, 0);
	if (!pblk->bb_wq)
		goto free_close_wq;

460 461 462
	pblk->r_end_wq = alloc_workqueue("pblk-read-end-wq",
			WQ_MEM_RECLAIM | WQ_UNBOUND, 0);
	if (!pblk->r_end_wq)
463
		goto free_bb_wq;
464

465
	if (pblk_set_addrf(pblk))
466 467
		goto free_r_end_wq;

468
	INIT_LIST_HEAD(&pblk->compl_list);
469
	INIT_LIST_HEAD(&pblk->resubmit_list);
470

471 472
	return 0;

473 474
free_r_end_wq:
	destroy_workqueue(pblk->r_end_wq);
475 476 477 478
free_bb_wq:
	destroy_workqueue(pblk->bb_wq);
free_close_wq:
	destroy_workqueue(pblk->close_wq);
479
free_w_rq_pool:
480
	mempool_exit(&pblk->w_rq_pool);
481
free_e_rq_pool:
482
	mempool_exit(&pblk->e_rq_pool);
483
free_r_rq_pool:
484
	mempool_exit(&pblk->r_rq_pool);
485
free_rec_pool:
486
	mempool_exit(&pblk->rec_pool);
487
free_gen_ws_pool:
488
	mempool_exit(&pblk->gen_ws_pool);
489
free_page_bio_pool:
490
	mempool_exit(&pblk->page_bio_pool);
491
free_global_caches:
492
	pblk_put_global_caches();
493 494
fail_free_pad_dist:
	kfree(pblk->pad_dist);
495 496 497 498 499
	return -ENOMEM;
}

static void pblk_core_free(struct pblk *pblk)
{
500 501 502
	if (pblk->close_wq)
		destroy_workqueue(pblk->close_wq);

503 504 505
	if (pblk->r_end_wq)
		destroy_workqueue(pblk->r_end_wq);

506 507
	if (pblk->bb_wq)
		destroy_workqueue(pblk->bb_wq);
508

509 510 511 512 513 514
	mempool_exit(&pblk->page_bio_pool);
	mempool_exit(&pblk->gen_ws_pool);
	mempool_exit(&pblk->rec_pool);
	mempool_exit(&pblk->r_rq_pool);
	mempool_exit(&pblk->e_rq_pool);
	mempool_exit(&pblk->w_rq_pool);
515

516
	pblk_put_global_caches();
517
	kfree(pblk->pad_dist);
518 519
}

520 521 522 523 524 525 526 527 528 529 530 531 532 533
static void pblk_line_mg_free(struct pblk *pblk)
{
	struct pblk_line_mgmt *l_mg = &pblk->l_mg;
	int i;

	kfree(l_mg->bb_template);
	kfree(l_mg->bb_aux);
	kfree(l_mg->vsc_list);

	for (i = 0; i < PBLK_DATA_LINES; i++) {
		kfree(l_mg->sline_meta[i]);
		pblk_mfree(l_mg->eline_meta[i]->buf, l_mg->emeta_alloc_type);
		kfree(l_mg->eline_meta[i]);
	}
534 535 536

	mempool_destroy(l_mg->bitmap_pool);
	kmem_cache_destroy(l_mg->bitmap_cache);
537 538
}

539 540
static void pblk_line_meta_free(struct pblk_line_mgmt *l_mg,
				struct pblk_line *line)
541
{
542 543
	struct pblk_w_err_gc *w_err_gc = line->w_err_gc;

544 545
	kfree(line->blk_bitmap);
	kfree(line->erase_bitmap);
546
	kfree(line->chks);
547 548 549

	pblk_mfree(w_err_gc->lba_list, l_mg->emeta_alloc_type);
	kfree(w_err_gc);
550 551
}

552 553 554 555 556 557 558 559 560 561
static void pblk_lines_free(struct pblk *pblk)
{
	struct pblk_line_mgmt *l_mg = &pblk->l_mg;
	struct pblk_line *line;
	int i;

	spin_lock(&l_mg->free_lock);
	for (i = 0; i < l_mg->nr_lines; i++) {
		line = &pblk->lines[i];

562
		pblk_line_free(line);
563
		pblk_line_meta_free(l_mg, line);
564 565
	}
	spin_unlock(&l_mg->free_lock);
566 567 568 569 570

	pblk_line_mg_free(pblk);

	kfree(pblk->luns);
	kfree(pblk->lines);
571 572
}

573
static int pblk_luns_init(struct pblk *pblk)
574 575 576 577
{
	struct nvm_tgt_dev *dev = pblk->dev;
	struct nvm_geo *geo = &dev->geo;
	struct pblk_lun *rlun;
578
	int i;
579 580

	/* TODO: Implement unbalanced LUN support */
581
	if (geo->num_lun < 0) {
582
		pblk_err(pblk, "unbalanced LUN config.\n");
583 584 585
		return -EINVAL;
	}

586 587
	pblk->luns = kcalloc(geo->all_luns, sizeof(struct pblk_lun),
								GFP_KERNEL);
588 589 590
	if (!pblk->luns)
		return -ENOMEM;

591
	for (i = 0; i < geo->all_luns; i++) {
592
		/* Stripe across channels */
593 594 595
		int ch = i % geo->num_ch;
		int lun_raw = i / geo->num_ch;
		int lunid = lun_raw + ch * geo->num_lun;
596 597

		rlun = &pblk->luns[i];
598
		rlun->bppa = dev->luns[lunid];
599 600 601 602 603 604 605 606

		sema_init(&rlun->wr_sem, 1);
	}

	return 0;
}

/* See comment over struct line_emeta definition */
607
static unsigned int calc_emeta_len(struct pblk *pblk)
608
{
609 610 611 612 613 614 615
	struct pblk_line_meta *lm = &pblk->lm;
	struct pblk_line_mgmt *l_mg = &pblk->l_mg;
	struct nvm_tgt_dev *dev = pblk->dev;
	struct nvm_geo *geo = &dev->geo;

	/* Round to sector size so that lba_list starts on its own sector */
	lm->emeta_sec[1] = DIV_ROUND_UP(
616
			sizeof(struct line_emeta) + lm->blk_bitmap_len +
617 618
			sizeof(struct wa_counters), geo->csecs);
	lm->emeta_len[1] = lm->emeta_sec[1] * geo->csecs;
619 620 621 622

	/* Round to sector size so that vsc_list starts on its own sector */
	lm->dsec_per_line = lm->sec_per_line - lm->emeta_sec[0];
	lm->emeta_sec[2] = DIV_ROUND_UP(lm->dsec_per_line * sizeof(u64),
623 624
			geo->csecs);
	lm->emeta_len[2] = lm->emeta_sec[2] * geo->csecs;
625 626

	lm->emeta_sec[3] = DIV_ROUND_UP(l_mg->nr_lines * sizeof(u32),
627 628
			geo->csecs);
	lm->emeta_len[3] = lm->emeta_sec[3] * geo->csecs;
629 630 631 632

	lm->vsc_list_len = l_mg->nr_lines * sizeof(u32);

	return (lm->emeta_len[1] + lm->emeta_len[2] + lm->emeta_len[3]);
633 634
}

635
static int pblk_set_provision(struct pblk *pblk, int nr_free_chks)
636 637
{
	struct nvm_tgt_dev *dev = pblk->dev;
638 639
	struct pblk_line_mgmt *l_mg = &pblk->l_mg;
	struct pblk_line_meta *lm = &pblk->lm;
640 641
	struct nvm_geo *geo = &dev->geo;
	sector_t provisioned;
642
	int sec_meta, blk_meta;
643
	int minimum;
644

645 646 647 648
	if (geo->op == NVM_TARGET_DEFAULT_OP)
		pblk->op = PBLK_DEFAULT_OP;
	else
		pblk->op = geo->op;
649

650 651
	minimum = pblk_get_min_chks(pblk);
	provisioned = nr_free_chks;
652
	provisioned *= (100 - pblk->op);
653 654
	sector_div(provisioned, 100);

655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671
	if ((nr_free_chks - provisioned) < minimum) {
		if (geo->op != NVM_TARGET_DEFAULT_OP) {
			pblk_err(pblk, "OP too small to create a sane instance\n");
			return -EINTR;
		}

		/* If the user did not specify an OP value, and PBLK_DEFAULT_OP
		 * is not enough, calculate and set sane value
		 */

		provisioned = nr_free_chks - minimum;
		pblk->op =  (100 * minimum) / nr_free_chks;
		pblk_info(pblk, "Default OP insufficient, adjusting OP to %d\n",
				pblk->op);
	}

	pblk->op_blks = nr_free_chks - provisioned;
672

673 674 675
	/* Internally pblk manages all free blocks, but all calculations based
	 * on user capacity consider only provisioned blocks
	 */
676 677
	pblk->rl.total_blocks = nr_free_chks;
	pblk->rl.nr_secs = nr_free_chks * geo->clba;
678 679 680

	/* Consider sectors used for metadata */
	sec_meta = (lm->smeta_sec + lm->emeta_sec[0]) * l_mg->nr_free_lines;
681
	blk_meta = DIV_ROUND_UP(sec_meta, geo->clba);
682

683
	pblk->capacity = (provisioned - blk_meta) * geo->clba;
684

685 686 687 688
	atomic_set(&pblk->rl.free_blocks, nr_free_chks);
	atomic_set(&pblk->rl.free_user_blocks, nr_free_chks);

	return 0;
689 690
}

691
static int pblk_setup_line_meta_chk(struct pblk *pblk, struct pblk_line *line,
692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719
				   struct nvm_chk_meta *meta)
{
	struct nvm_tgt_dev *dev = pblk->dev;
	struct nvm_geo *geo = &dev->geo;
	struct pblk_line_meta *lm = &pblk->lm;
	int i, nr_bad_chks = 0;

	for (i = 0; i < lm->blk_per_line; i++) {
		struct pblk_lun *rlun = &pblk->luns[i];
		struct nvm_chk_meta *chunk;
		struct nvm_chk_meta *chunk_meta;
		struct ppa_addr ppa;
		int pos;

		ppa = rlun->bppa;
		pos = pblk_ppa_to_pos(geo, ppa);
		chunk = &line->chks[pos];

		ppa.m.chk = line->id;
		chunk_meta = pblk_chunk_get_off(pblk, meta, ppa);

		chunk->state = chunk_meta->state;
		chunk->type = chunk_meta->type;
		chunk->wi = chunk_meta->wi;
		chunk->slba = chunk_meta->slba;
		chunk->cnlb = chunk_meta->cnlb;
		chunk->wp = chunk_meta->wp;

720 721 722
		trace_pblk_chunk_state(pblk_disk_name(pblk), &ppa,
					chunk->state);

723 724 725 726 727
		if (chunk->type & NVM_CHK_TP_SZ_SPEC) {
			WARN_ONCE(1, "pblk: custom-sized chunks unsupported\n");
			continue;
		}

728 729 730
		if (!(chunk->state & NVM_CHK_ST_OFFLINE))
			continue;

731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752
		set_bit(pos, line->blk_bitmap);
		nr_bad_chks++;
	}

	return nr_bad_chks;
}

static long pblk_setup_line_meta(struct pblk *pblk, struct pblk_line *line,
				 void *chunk_meta, int line_id)
{
	struct pblk_line_mgmt *l_mg = &pblk->l_mg;
	struct pblk_line_meta *lm = &pblk->lm;
	long nr_bad_chks, chk_in_line;

	line->pblk = pblk;
	line->id = line_id;
	line->type = PBLK_LINETYPE_FREE;
	line->state = PBLK_LINESTATE_NEW;
	line->gc_group = PBLK_LINEGC_NONE;
	line->vsc = &l_mg->vsc_list[line_id];
	spin_lock_init(&line->lock);

753
	nr_bad_chks = pblk_setup_line_meta_chk(pblk, line, chunk_meta);
754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770

	chk_in_line = lm->blk_per_line - nr_bad_chks;
	if (nr_bad_chks < 0 || nr_bad_chks > lm->blk_per_line ||
					chk_in_line < lm->min_blk_line) {
		line->state = PBLK_LINESTATE_BAD;
		list_add_tail(&line->list, &l_mg->bad_list);
		return 0;
	}

	atomic_set(&line->blk_in_line, chk_in_line);
	list_add_tail(&line->list, &l_mg->free_list);
	l_mg->nr_free_lines++;

	return chk_in_line;
}

static int pblk_alloc_line_meta(struct pblk *pblk, struct pblk_line *line)
771 772 773 774 775 776 777 778
{
	struct pblk_line_meta *lm = &pblk->lm;

	line->blk_bitmap = kzalloc(lm->blk_bitmap_len, GFP_KERNEL);
	if (!line->blk_bitmap)
		return -ENOMEM;

	line->erase_bitmap = kzalloc(lm->blk_bitmap_len, GFP_KERNEL);
779 780 781
	if (!line->erase_bitmap)
		goto free_blk_bitmap;

782

783 784
	line->chks = kmalloc_array(lm->blk_per_line,
				   sizeof(struct nvm_chk_meta), GFP_KERNEL);
785 786 787 788 789 790
	if (!line->chks)
		goto free_erase_bitmap;

	line->w_err_gc = kzalloc(sizeof(struct pblk_w_err_gc), GFP_KERNEL);
	if (!line->w_err_gc)
		goto free_chks;
791 792

	return 0;
793 794 795 796 797 798 799 800

free_chks:
	kfree(line->chks);
free_erase_bitmap:
	kfree(line->erase_bitmap);
free_blk_bitmap:
	kfree(line->blk_bitmap);
	return -ENOMEM;
801 802 803
}

static int pblk_line_mg_init(struct pblk *pblk)
804
{
805 806
	struct nvm_tgt_dev *dev = pblk->dev;
	struct nvm_geo *geo = &dev->geo;
807 808
	struct pblk_line_mgmt *l_mg = &pblk->l_mg;
	struct pblk_line_meta *lm = &pblk->lm;
809 810
	int i, bb_distance;

811
	l_mg->nr_lines = geo->num_chk;
812 813 814 815 816 817 818 819 820 821 822 823 824
	l_mg->log_line = l_mg->data_line = NULL;
	l_mg->l_seq_nr = l_mg->d_seq_nr = 0;
	l_mg->nr_free_lines = 0;
	bitmap_zero(&l_mg->meta_bitmap, PBLK_DATA_LINES);

	INIT_LIST_HEAD(&l_mg->free_list);
	INIT_LIST_HEAD(&l_mg->corrupt_list);
	INIT_LIST_HEAD(&l_mg->bad_list);
	INIT_LIST_HEAD(&l_mg->gc_full_list);
	INIT_LIST_HEAD(&l_mg->gc_high_list);
	INIT_LIST_HEAD(&l_mg->gc_mid_list);
	INIT_LIST_HEAD(&l_mg->gc_low_list);
	INIT_LIST_HEAD(&l_mg->gc_empty_list);
825
	INIT_LIST_HEAD(&l_mg->gc_werr_list);
826 827 828

	INIT_LIST_HEAD(&l_mg->emeta_list);

829 830 831 832
	l_mg->gc_lists[0] = &l_mg->gc_werr_list;
	l_mg->gc_lists[1] = &l_mg->gc_high_list;
	l_mg->gc_lists[2] = &l_mg->gc_mid_list;
	l_mg->gc_lists[3] = &l_mg->gc_low_list;
833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848

	spin_lock_init(&l_mg->free_lock);
	spin_lock_init(&l_mg->close_lock);
	spin_lock_init(&l_mg->gc_lock);

	l_mg->vsc_list = kcalloc(l_mg->nr_lines, sizeof(__le32), GFP_KERNEL);
	if (!l_mg->vsc_list)
		goto fail;

	l_mg->bb_template = kzalloc(lm->sec_bitmap_len, GFP_KERNEL);
	if (!l_mg->bb_template)
		goto fail_free_vsc_list;

	l_mg->bb_aux = kzalloc(lm->sec_bitmap_len, GFP_KERNEL);
	if (!l_mg->bb_aux)
		goto fail_free_bb_template;
849 850 851 852 853 854 855 856 857 858

	/* smeta is always small enough to fit on a kmalloc memory allocation,
	 * emeta depends on the number of LUNs allocated to the pblk instance
	 */
	for (i = 0; i < PBLK_DATA_LINES; i++) {
		l_mg->sline_meta[i] = kmalloc(lm->smeta_len, GFP_KERNEL);
		if (!l_mg->sline_meta[i])
			goto fail_free_smeta;
	}

859 860 861 862 863 864 865 866 867 868 869
	l_mg->bitmap_cache = kmem_cache_create("pblk_lm_bitmap",
			lm->sec_bitmap_len, 0, 0, NULL);
	if (!l_mg->bitmap_cache)
		goto fail_free_smeta;

	/* the bitmap pool is used for both valid and map bitmaps */
	l_mg->bitmap_pool = mempool_create_slab_pool(PBLK_DATA_LINES * 2,
				l_mg->bitmap_cache);
	if (!l_mg->bitmap_pool)
		goto fail_destroy_bitmap_cache;

870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907
	/* emeta allocates three different buffers for managing metadata with
	 * in-memory and in-media layouts
	 */
	for (i = 0; i < PBLK_DATA_LINES; i++) {
		struct pblk_emeta *emeta;

		emeta = kmalloc(sizeof(struct pblk_emeta), GFP_KERNEL);
		if (!emeta)
			goto fail_free_emeta;

		if (lm->emeta_len[0] > KMALLOC_MAX_CACHE_SIZE) {
			l_mg->emeta_alloc_type = PBLK_VMALLOC_META;

			emeta->buf = vmalloc(lm->emeta_len[0]);
			if (!emeta->buf) {
				kfree(emeta);
				goto fail_free_emeta;
			}

			emeta->nr_entries = lm->emeta_sec[0];
			l_mg->eline_meta[i] = emeta;
		} else {
			l_mg->emeta_alloc_type = PBLK_KMALLOC_META;

			emeta->buf = kmalloc(lm->emeta_len[0], GFP_KERNEL);
			if (!emeta->buf) {
				kfree(emeta);
				goto fail_free_emeta;
			}

			emeta->nr_entries = lm->emeta_sec[0];
			l_mg->eline_meta[i] = emeta;
		}
	}

	for (i = 0; i < l_mg->nr_lines; i++)
		l_mg->vsc_list[i] = cpu_to_le32(EMPTY_ENTRY);

908 909 910 911
	bb_distance = (geo->all_luns) * geo->ws_opt;
	for (i = 0; i < lm->sec_per_line; i += bb_distance)
		bitmap_set(l_mg->bb_template, i, geo->ws_opt);

912 913 914 915
	return 0;

fail_free_emeta:
	while (--i >= 0) {
916 917 918 919
		if (l_mg->emeta_alloc_type == PBLK_VMALLOC_META)
			vfree(l_mg->eline_meta[i]->buf);
		else
			kfree(l_mg->eline_meta[i]->buf);
920
		kfree(l_mg->eline_meta[i]);
921
	}
922 923 924 925

	mempool_destroy(l_mg->bitmap_pool);
fail_destroy_bitmap_cache:
	kmem_cache_destroy(l_mg->bitmap_cache);
926 927
fail_free_smeta:
	for (i = 0; i < PBLK_DATA_LINES; i++)
928
		kfree(l_mg->sline_meta[i]);
929 930 931 932 933 934
	kfree(l_mg->bb_aux);
fail_free_bb_template:
	kfree(l_mg->bb_template);
fail_free_vsc_list:
	kfree(l_mg->vsc_list);
fail:
935 936 937
	return -ENOMEM;
}

938
static int pblk_line_meta_init(struct pblk *pblk)
939 940 941 942 943
{
	struct nvm_tgt_dev *dev = pblk->dev;
	struct nvm_geo *geo = &dev->geo;
	struct pblk_line_meta *lm = &pblk->lm;
	unsigned int smeta_len, emeta_len;
944
	int i;
945

946
	lm->sec_per_line = geo->clba * geo->all_luns;
947 948
	lm->blk_per_line = geo->all_luns;
	lm->blk_bitmap_len = BITS_TO_LONGS(geo->all_luns) * sizeof(long);
949
	lm->sec_bitmap_len = BITS_TO_LONGS(lm->sec_per_line) * sizeof(long);
950
	lm->lun_bitmap_len = BITS_TO_LONGS(geo->all_luns) * sizeof(long);
951 952
	lm->mid_thrs = lm->sec_per_line / 2;
	lm->high_thrs = lm->sec_per_line / 4;
953
	lm->meta_distance = (geo->all_luns / 2) * pblk->min_write_pgs;
954 955 956 957 958 959

	/* Calculate necessary pages for smeta. See comment over struct
	 * line_smeta definition
	 */
	i = 1;
add_smeta_page:
960 961
	lm->smeta_sec = i * geo->ws_opt;
	lm->smeta_len = lm->smeta_sec * geo->csecs;
962

963
	smeta_len = sizeof(struct line_smeta) + lm->lun_bitmap_len;
964 965 966 967 968 969 970 971 972 973
	if (smeta_len > lm->smeta_len) {
		i++;
		goto add_smeta_page;
	}

	/* Calculate necessary pages for emeta. See comment over struct
	 * line_emeta definition
	 */
	i = 1;
add_emeta_page:
974 975
	lm->emeta_sec[0] = i * geo->ws_opt;
	lm->emeta_len[0] = lm->emeta_sec[0] * geo->csecs;
976

977 978
	emeta_len = calc_emeta_len(pblk);
	if (emeta_len > lm->emeta_len[0]) {
979 980 981 982
		i++;
		goto add_emeta_page;
	}

983
	lm->emeta_bb = geo->all_luns > i ? geo->all_luns - i : 0;
984 985

	lm->min_blk_line = 1;
986
	if (geo->all_luns > 1)
987
		lm->min_blk_line += DIV_ROUND_UP(lm->smeta_sec +
988
					lm->emeta_sec[0], geo->clba);
989

990
	if (lm->min_blk_line > lm->blk_per_line) {
991
		pblk_err(pblk, "config. not supported. Min. LUN in line:%d\n",
992
							lm->blk_per_line);
993
		return -EINVAL;
994
	}
995

996 997 998 999 1000 1001 1002
	return 0;
}

static int pblk_lines_init(struct pblk *pblk)
{
	struct pblk_line_mgmt *l_mg = &pblk->l_mg;
	struct pblk_line *line;
1003
	void *chunk_meta;
1004
	int nr_free_chks = 0;
1005 1006 1007
	int i, ret;

	ret = pblk_line_meta_init(pblk);
1008
	if (ret)
1009
		return ret;
1010

1011 1012 1013 1014 1015 1016
	ret = pblk_line_mg_init(pblk);
	if (ret)
		return ret;

	ret = pblk_luns_init(pblk);
	if (ret)
1017 1018
		goto fail_free_meta;

1019
	chunk_meta = pblk_get_chunk_meta(pblk);
1020 1021
	if (IS_ERR(chunk_meta)) {
		ret = PTR_ERR(chunk_meta);
1022
		goto fail_free_luns;
1023
	}
1024 1025 1026

	pblk->lines = kcalloc(l_mg->nr_lines, sizeof(struct pblk_line),
								GFP_KERNEL);
1027 1028
	if (!pblk->lines) {
		ret = -ENOMEM;
1029
		goto fail_free_chunk_meta;
1030 1031
	}

1032 1033 1034
	for (i = 0; i < l_mg->nr_lines; i++) {
		line = &pblk->lines[i];

1035
		ret = pblk_alloc_line_meta(pblk, line);
1036
		if (ret)
1037
			goto fail_free_lines;
1038

1039
		nr_free_chks += pblk_setup_line_meta(pblk, line, chunk_meta, i);
1040 1041 1042

		trace_pblk_line_state(pblk_disk_name(pblk), line->id,
								line->state);
1043 1044
	}

1045
	if (!nr_free_chks) {
1046
		pblk_err(pblk, "too many bad blocks prevent for sane instance\n");
1047 1048
		ret = -EINTR;
		goto fail_free_lines;
1049 1050
	}

1051 1052 1053
	ret = pblk_set_provision(pblk, nr_free_chks);
	if (ret)
		goto fail_free_lines;
1054

1055
	vfree(chunk_meta);
1056
	return 0;
1057

1058
fail_free_lines:
1059
	while (--i >= 0)
1060
		pblk_line_meta_free(l_mg, &pblk->lines[i]);
1061
	kfree(pblk->lines);
1062 1063
fail_free_chunk_meta:
	kfree(chunk_meta);
1064 1065
fail_free_luns:
	kfree(pblk->luns);
1066
fail_free_meta:
1067
	pblk_line_mg_free(pblk);
1068 1069 1070 1071 1072 1073 1074 1075

	return ret;
}

static int pblk_writer_init(struct pblk *pblk)
{
	pblk->writer_ts = kthread_create(pblk_write_ts, pblk, "pblk-writer-t");
	if (IS_ERR(pblk->writer_ts)) {
1076 1077 1078
		int err = PTR_ERR(pblk->writer_ts);

		if (err != -EINTR)
1079
			pblk_err(pblk, "could not allocate writer kthread (%d)\n",
1080 1081
					err);
		return err;
1082 1083
	}

1084 1085 1086
	timer_setup(&pblk->wtimer, pblk_write_timer_fn, 0);
	mod_timer(&pblk->wtimer, jiffies + msecs_to_jiffies(100));

1087 1088 1089 1090 1091
	return 0;
}

static void pblk_writer_stop(struct pblk *pblk)
{
1092 1093 1094 1095 1096 1097 1098 1099 1100
	/* The pipeline must be stopped and the write buffer emptied before the
	 * write thread is stopped
	 */
	WARN(pblk_rb_read_count(&pblk->rwb),
			"Stopping not fully persisted write buffer\n");

	WARN(pblk_rb_sync_count(&pblk->rwb),
			"Stopping not fully synced write buffer\n");

1101
	del_timer_sync(&pblk->wtimer);
1102 1103 1104 1105 1106 1107 1108 1109
	if (pblk->writer_ts)
		kthread_stop(pblk->writer_ts);
}

static void pblk_free(struct pblk *pblk)
{
	pblk_lines_free(pblk);
	pblk_l2p_free(pblk);
1110 1111
	pblk_rwb_free(pblk);
	pblk_core_free(pblk);
1112 1113 1114 1115

	kfree(pblk);
}

1116
static void pblk_tear_down(struct pblk *pblk, bool graceful)
1117
{
1118 1119 1120
	if (graceful)
		__pblk_pipeline_flush(pblk);
	__pblk_pipeline_stop(pblk);
1121 1122 1123 1124
	pblk_writer_stop(pblk);
	pblk_rb_sync_l2p(&pblk->rwb);
	pblk_rl_free(&pblk->rl);

1125
	pblk_debug(pblk, "consistent tear down (graceful:%d)\n", graceful);
1126 1127
}

1128
static void pblk_exit(void *private, bool graceful)
1129 1130 1131
{
	struct pblk *pblk = private;

1132 1133
	pblk_gc_exit(pblk, graceful);
	pblk_tear_down(pblk, graceful);
1134

1135
#ifdef CONFIG_NVM_PBLK_DEBUG
1136
	pblk_info(pblk, "exit: L2P CRC: %x\n", pblk_l2p_crc(pblk));
1137 1138
#endif

1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157
	pblk_free(pblk);
}

static sector_t pblk_capacity(void *private)
{
	struct pblk *pblk = private;

	return pblk->capacity * NR_PHY_IN_LOG;
}

static void *pblk_init(struct nvm_tgt_dev *dev, struct gendisk *tdisk,
		       int flags)
{
	struct nvm_geo *geo = &dev->geo;
	struct request_queue *bqueue = dev->q;
	struct request_queue *tqueue = tdisk->queue;
	struct pblk *pblk;
	int ret;

1158 1159 1160 1161 1162 1163 1164
	pblk = kzalloc(sizeof(struct pblk), GFP_KERNEL);
	if (!pblk)
		return ERR_PTR(-ENOMEM);

	pblk->dev = dev;
	pblk->disk = tdisk;
	pblk->state = PBLK_STATE_RUNNING;
1165
	trace_pblk_state(pblk_disk_name(pblk), pblk->state);
1166 1167
	pblk->gc.gc_enabled = 0;

1168 1169
	if (!(geo->version == NVM_OCSSD_SPEC_12 ||
					geo->version == NVM_OCSSD_SPEC_20)) {
1170
		pblk_err(pblk, "OCSSD version not supported (%u)\n",
1171
							geo->version);
1172
		kfree(pblk);
1173 1174 1175
		return ERR_PTR(-EINVAL);
	}

1176
	spin_lock_init(&pblk->resubmit_lock);
1177 1178 1179
	spin_lock_init(&pblk->trans_lock);
	spin_lock_init(&pblk->lock);

1180
#ifdef CONFIG_NVM_PBLK_DEBUG
1181 1182 1183 1184 1185 1186 1187
	atomic_long_set(&pblk->inflight_writes, 0);
	atomic_long_set(&pblk->padded_writes, 0);
	atomic_long_set(&pblk->padded_wb, 0);
	atomic_long_set(&pblk->req_writes, 0);
	atomic_long_set(&pblk->sub_writes, 0);
	atomic_long_set(&pblk->sync_writes, 0);
	atomic_long_set(&pblk->inflight_reads, 0);
1188
	atomic_long_set(&pblk->cache_reads, 0);
1189 1190 1191 1192
	atomic_long_set(&pblk->sync_reads, 0);
	atomic_long_set(&pblk->recov_writes, 0);
	atomic_long_set(&pblk->recov_writes, 0);
	atomic_long_set(&pblk->recov_gc_writes, 0);
1193
	atomic_long_set(&pblk->recov_gc_reads, 0);
1194 1195 1196 1197 1198 1199 1200 1201 1202
#endif

	atomic_long_set(&pblk->read_failed, 0);
	atomic_long_set(&pblk->read_empty, 0);
	atomic_long_set(&pblk->read_high_ecc, 0);
	atomic_long_set(&pblk->read_failed_gc, 0);
	atomic_long_set(&pblk->write_failed, 0);
	atomic_long_set(&pblk->erase_failed, 0);

1203
	ret = pblk_core_init(pblk);
1204
	if (ret) {
1205
		pblk_err(pblk, "could not initialize core\n");
1206 1207 1208 1209 1210
		goto fail;
	}

	ret = pblk_lines_init(pblk);
	if (ret) {
1211
		pblk_err(pblk, "could not initialize lines\n");
1212
		goto fail_free_core;
1213 1214
	}

1215
	ret = pblk_rwb_init(pblk);
1216
	if (ret) {
1217
		pblk_err(pblk, "could not initialize write buffer\n");
1218
		goto fail_free_lines;
1219 1220
	}

1221
	ret = pblk_l2p_init(pblk, flags & NVM_TARGET_FACTORY);
1222
	if (ret) {
1223
		pblk_err(pblk, "could not initialize maps\n");
1224
		goto fail_free_rwb;
1225 1226 1227 1228
	}

	ret = pblk_writer_init(pblk);
	if (ret) {
1229
		if (ret != -EINTR)
1230
			pblk_err(pblk, "could not initialize write thread\n");
1231
		goto fail_free_l2p;
1232 1233 1234 1235
	}

	ret = pblk_gc_init(pblk);
	if (ret) {
1236
		pblk_err(pblk, "could not initialize gc\n");
1237 1238 1239 1240 1241 1242 1243 1244 1245
		goto fail_stop_writer;
	}

	/* inherit the size from the underlying device */
	blk_queue_logical_block_size(tqueue, queue_physical_block_size(bqueue));
	blk_queue_max_hw_sectors(tqueue, queue_max_hw_sectors(bqueue));

	blk_queue_write_cache(tqueue, true, false);

1246
	tqueue->limits.discard_granularity = geo->clba * geo->csecs;
1247 1248
	tqueue->limits.discard_alignment = 0;
	blk_queue_max_discard_sectors(tqueue, UINT_MAX >> 9);
1249
	blk_queue_flag_set(QUEUE_FLAG_DISCARD, tqueue);
1250

1251
	pblk_info(pblk, "luns:%u, lines:%d, secs:%llu, buf entries:%u\n",
1252
			geo->all_luns, pblk->l_mg.nr_lines,
1253 1254 1255 1256
			(unsigned long long)pblk->rl.nr_secs,
			pblk->rwb.nr_entries);

	wake_up_process(pblk->writer_ts);
1257 1258 1259 1260

	/* Check if we need to start GC */
	pblk_gc_should_kick(pblk);

1261 1262 1263 1264 1265 1266
	return pblk;

fail_stop_writer:
	pblk_writer_stop(pblk);
fail_free_l2p:
	pblk_l2p_free(pblk);
1267 1268 1269 1270
fail_free_rwb:
	pblk_rwb_free(pblk);
fail_free_lines:
	pblk_lines_free(pblk);
1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290
fail_free_core:
	pblk_core_free(pblk);
fail:
	kfree(pblk);
	return ERR_PTR(ret);
}

/* physical block device target */
static struct nvm_tgt_type tt_pblk = {
	.name		= "pblk",
	.version	= {1, 0, 0},

	.make_rq	= pblk_make_rq,
	.capacity	= pblk_capacity,

	.init		= pblk_init,
	.exit		= pblk_exit,

	.sysfs_init	= pblk_sysfs_init,
	.sysfs_exit	= pblk_sysfs_exit,
1291
	.owner		= THIS_MODULE,
1292 1293 1294 1295
};

static int __init pblk_module_init(void)
{
1296 1297
	int ret;

1298 1299 1300
	ret = bioset_init(&pblk_bio_set, BIO_POOL_SIZE, 0, 0);
	if (ret)
		return ret;
1301 1302
	ret = nvm_register_tgt_type(&tt_pblk);
	if (ret)
1303
		bioset_exit(&pblk_bio_set);
1304
	return ret;
1305 1306 1307 1308
}

static void pblk_module_exit(void)
{
1309
	bioset_exit(&pblk_bio_set);
1310 1311 1312 1313 1314 1315 1316 1317 1318
	nvm_unregister_tgt_type(&tt_pblk);
}

module_init(pblk_module_init);
module_exit(pblk_module_exit);
MODULE_AUTHOR("Javier Gonzalez <javier@cnexlabs.com>");
MODULE_AUTHOR("Matias Bjorling <matias@cnexlabs.com>");
MODULE_LICENSE("GPL v2");
MODULE_DESCRIPTION("Physical Block-Device for Open-Channel SSDs");