Commit 0a13daed authored by Linus Torvalds's avatar Linus Torvalds

Merge branch 'for-4.5/lightnvm' of git://git.kernel.dk/linux-block

Pull lightnvm fixes and updates from Jens Axboe:
 "This should have been part of the drivers branch, but it arrived a bit
  late and wasn't based on the official core block driver branch.  So
  they got a small scolding, but got a pass since it's still new.  Hence
  it's in a separate branch.

  This is mostly pure fixes, contained to lightnvm/, and minor feature
  additions"

* 'for-4.5/lightnvm' of git://git.kernel.dk/linux-block: (26 commits)
  lightnvm: ensure that nvm_dev_ops can be used without CONFIG_NVM
  lightnvm: introduce factory reset
  lightnvm: use system block for mm initialization
  lightnvm: introduce ioctl to initialize device
  lightnvm: core on-disk initialization
  lightnvm: introduce mlc lower page table mappings
  lightnvm: add mccap support
  lightnvm: manage open and closed blocks separately
  lightnvm: fix missing grown bad block type
  lightnvm: reference rrpc lun in rrpc block
  lightnvm: introduce nvm_submit_ppa
  lightnvm: move rq->error to nvm_rq->error
  lightnvm: support multiple ppas in nvm_erase_ppa
  lightnvm: move the pages per block check out of the loop
  lightnvm: sectors first in ppa list
  lightnvm: fix locking and mempool in rrpc_lun_gc
  lightnvm: put block back to gc list on its reclaim fail
  lightnvm: check bi_error in gc
  lightnvm: return the get_bb_tbl return value
  lightnvm: refactor end_io functions for sync
  ...
parents 64120354 a7fd9a4f
...@@ -436,9 +436,8 @@ static void null_del_dev(struct nullb *nullb) ...@@ -436,9 +436,8 @@ static void null_del_dev(struct nullb *nullb)
static void null_lnvm_end_io(struct request *rq, int error) static void null_lnvm_end_io(struct request *rq, int error)
{ {
struct nvm_rq *rqd = rq->end_io_data; struct nvm_rq *rqd = rq->end_io_data;
struct nvm_dev *dev = rqd->dev;
dev->mt->end_io(rqd, error); nvm_end_io(rqd, error);
blk_put_request(rq); blk_put_request(rq);
} }
......
...@@ -2,6 +2,6 @@ ...@@ -2,6 +2,6 @@
# Makefile for Open-Channel SSDs. # Makefile for Open-Channel SSDs.
# #
obj-$(CONFIG_NVM) := core.o obj-$(CONFIG_NVM) := core.o sysblk.o
obj-$(CONFIG_NVM_GENNVM) += gennvm.o obj-$(CONFIG_NVM_GENNVM) += gennvm.o
obj-$(CONFIG_NVM_RRPC) += rrpc.o obj-$(CONFIG_NVM_RRPC) += rrpc.o
This diff is collapsed.
...@@ -60,7 +60,8 @@ static int gennvm_luns_init(struct nvm_dev *dev, struct gen_nvm *gn) ...@@ -60,7 +60,8 @@ static int gennvm_luns_init(struct nvm_dev *dev, struct gen_nvm *gn)
lun->vlun.lun_id = i % dev->luns_per_chnl; lun->vlun.lun_id = i % dev->luns_per_chnl;
lun->vlun.chnl_id = i / dev->luns_per_chnl; lun->vlun.chnl_id = i / dev->luns_per_chnl;
lun->vlun.nr_free_blocks = dev->blks_per_lun; lun->vlun.nr_free_blocks = dev->blks_per_lun;
lun->vlun.nr_inuse_blocks = 0; lun->vlun.nr_open_blocks = 0;
lun->vlun.nr_closed_blocks = 0;
lun->vlun.nr_bad_blocks = 0; lun->vlun.nr_bad_blocks = 0;
} }
return 0; return 0;
...@@ -89,6 +90,7 @@ static int gennvm_block_bb(struct ppa_addr ppa, int nr_blocks, u8 *blks, ...@@ -89,6 +90,7 @@ static int gennvm_block_bb(struct ppa_addr ppa, int nr_blocks, u8 *blks,
list_move_tail(&blk->list, &lun->bb_list); list_move_tail(&blk->list, &lun->bb_list);
lun->vlun.nr_bad_blocks++; lun->vlun.nr_bad_blocks++;
lun->vlun.nr_free_blocks--;
} }
return 0; return 0;
...@@ -133,15 +135,15 @@ static int gennvm_block_map(u64 slba, u32 nlb, __le64 *entries, void *private) ...@@ -133,15 +135,15 @@ static int gennvm_block_map(u64 slba, u32 nlb, __le64 *entries, void *private)
pba = pba - (dev->sec_per_lun * lun_id); pba = pba - (dev->sec_per_lun * lun_id);
blk = &lun->vlun.blocks[div_u64(pba, dev->sec_per_blk)]; blk = &lun->vlun.blocks[div_u64(pba, dev->sec_per_blk)];
if (!blk->type) { if (!blk->state) {
/* at this point, we don't know anything about the /* at this point, we don't know anything about the
* block. It's up to the FTL on top to re-etablish the * block. It's up to the FTL on top to re-etablish the
* block state * block state. The block is assumed to be open.
*/ */
list_move_tail(&blk->list, &lun->used_list); list_move_tail(&blk->list, &lun->used_list);
blk->type = 1; blk->state = NVM_BLK_ST_OPEN;
lun->vlun.nr_free_blocks--; lun->vlun.nr_free_blocks--;
lun->vlun.nr_inuse_blocks++; lun->vlun.nr_open_blocks++;
} }
} }
...@@ -255,14 +257,14 @@ static void gennvm_unregister(struct nvm_dev *dev) ...@@ -255,14 +257,14 @@ static void gennvm_unregister(struct nvm_dev *dev)
module_put(THIS_MODULE); module_put(THIS_MODULE);
} }
static struct nvm_block *gennvm_get_blk(struct nvm_dev *dev, static struct nvm_block *gennvm_get_blk_unlocked(struct nvm_dev *dev,
struct nvm_lun *vlun, unsigned long flags) struct nvm_lun *vlun, unsigned long flags)
{ {
struct gen_lun *lun = container_of(vlun, struct gen_lun, vlun); struct gen_lun *lun = container_of(vlun, struct gen_lun, vlun);
struct nvm_block *blk = NULL; struct nvm_block *blk = NULL;
int is_gc = flags & NVM_IOTYPE_GC; int is_gc = flags & NVM_IOTYPE_GC;
spin_lock(&vlun->lock); assert_spin_locked(&vlun->lock);
if (list_empty(&lun->free_list)) { if (list_empty(&lun->free_list)) {
pr_err_ratelimited("gennvm: lun %u have no free pages available", pr_err_ratelimited("gennvm: lun %u have no free pages available",
...@@ -275,83 +277,64 @@ static struct nvm_block *gennvm_get_blk(struct nvm_dev *dev, ...@@ -275,83 +277,64 @@ static struct nvm_block *gennvm_get_blk(struct nvm_dev *dev,
blk = list_first_entry(&lun->free_list, struct nvm_block, list); blk = list_first_entry(&lun->free_list, struct nvm_block, list);
list_move_tail(&blk->list, &lun->used_list); list_move_tail(&blk->list, &lun->used_list);
blk->type = 1; blk->state = NVM_BLK_ST_OPEN;
lun->vlun.nr_free_blocks--; lun->vlun.nr_free_blocks--;
lun->vlun.nr_inuse_blocks++; lun->vlun.nr_open_blocks++;
out: out:
return blk;
}
static struct nvm_block *gennvm_get_blk(struct nvm_dev *dev,
struct nvm_lun *vlun, unsigned long flags)
{
struct nvm_block *blk;
spin_lock(&vlun->lock);
blk = gennvm_get_blk_unlocked(dev, vlun, flags);
spin_unlock(&vlun->lock); spin_unlock(&vlun->lock);
return blk; return blk;
} }
static void gennvm_put_blk(struct nvm_dev *dev, struct nvm_block *blk) static void gennvm_put_blk_unlocked(struct nvm_dev *dev, struct nvm_block *blk)
{ {
struct nvm_lun *vlun = blk->lun; struct nvm_lun *vlun = blk->lun;
struct gen_lun *lun = container_of(vlun, struct gen_lun, vlun); struct gen_lun *lun = container_of(vlun, struct gen_lun, vlun);
spin_lock(&vlun->lock); assert_spin_locked(&vlun->lock);
switch (blk->type) { if (blk->state & NVM_BLK_ST_OPEN) {
case 1:
list_move_tail(&blk->list, &lun->free_list); list_move_tail(&blk->list, &lun->free_list);
lun->vlun.nr_open_blocks--;
lun->vlun.nr_free_blocks++; lun->vlun.nr_free_blocks++;
lun->vlun.nr_inuse_blocks--; blk->state = NVM_BLK_ST_FREE;
blk->type = 0; } else if (blk->state & NVM_BLK_ST_CLOSED) {
break; list_move_tail(&blk->list, &lun->free_list);
case 2: lun->vlun.nr_closed_blocks--;
lun->vlun.nr_free_blocks++;
blk->state = NVM_BLK_ST_FREE;
} else if (blk->state & NVM_BLK_ST_BAD) {
list_move_tail(&blk->list, &lun->bb_list); list_move_tail(&blk->list, &lun->bb_list);
lun->vlun.nr_bad_blocks++; lun->vlun.nr_bad_blocks++;
lun->vlun.nr_inuse_blocks--; blk->state = NVM_BLK_ST_BAD;
break; } else {
default:
WARN_ON_ONCE(1); WARN_ON_ONCE(1);
pr_err("gennvm: erroneous block type (%lu -> %u)\n", pr_err("gennvm: erroneous block type (%lu -> %u)\n",
blk->id, blk->type); blk->id, blk->state);
list_move_tail(&blk->list, &lun->bb_list); list_move_tail(&blk->list, &lun->bb_list);
lun->vlun.nr_bad_blocks++; lun->vlun.nr_bad_blocks++;
lun->vlun.nr_inuse_blocks--; blk->state = NVM_BLK_ST_BAD;
}
spin_unlock(&vlun->lock);
}
static void gennvm_addr_to_generic_mode(struct nvm_dev *dev, struct nvm_rq *rqd)
{
int i;
if (rqd->nr_pages > 1) {
for (i = 0; i < rqd->nr_pages; i++)
rqd->ppa_list[i] = dev_to_generic_addr(dev,
rqd->ppa_list[i]);
} else {
rqd->ppa_addr = dev_to_generic_addr(dev, rqd->ppa_addr);
} }
} }
static void gennvm_generic_to_addr_mode(struct nvm_dev *dev, struct nvm_rq *rqd) static void gennvm_put_blk(struct nvm_dev *dev, struct nvm_block *blk)
{
int i;
if (rqd->nr_pages > 1) {
for (i = 0; i < rqd->nr_pages; i++)
rqd->ppa_list[i] = generic_to_dev_addr(dev,
rqd->ppa_list[i]);
} else {
rqd->ppa_addr = generic_to_dev_addr(dev, rqd->ppa_addr);
}
}
static int gennvm_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd)
{ {
if (!dev->ops->submit_io) struct nvm_lun *vlun = blk->lun;
return 0;
/* Convert address space */
gennvm_generic_to_addr_mode(dev, rqd);
rqd->dev = dev; spin_lock(&vlun->lock);
return dev->ops->submit_io(dev, rqd); gennvm_put_blk_unlocked(dev, blk);
spin_unlock(&vlun->lock);
} }
static void gennvm_blk_set_type(struct nvm_dev *dev, struct ppa_addr *ppa, static void gennvm_blk_set_type(struct nvm_dev *dev, struct ppa_addr *ppa,
...@@ -376,7 +359,7 @@ static void gennvm_blk_set_type(struct nvm_dev *dev, struct ppa_addr *ppa, ...@@ -376,7 +359,7 @@ static void gennvm_blk_set_type(struct nvm_dev *dev, struct ppa_addr *ppa,
blk = &lun->vlun.blocks[ppa->g.blk]; blk = &lun->vlun.blocks[ppa->g.blk];
/* will be moved to bb list on put_blk from target */ /* will be moved to bb list on put_blk from target */
blk->type = type; blk->state = type;
} }
/* mark block bad. It is expected the target recover from the error. */ /* mark block bad. It is expected the target recover from the error. */
...@@ -390,77 +373,51 @@ static void gennvm_mark_blk_bad(struct nvm_dev *dev, struct nvm_rq *rqd) ...@@ -390,77 +373,51 @@ static void gennvm_mark_blk_bad(struct nvm_dev *dev, struct nvm_rq *rqd)
if (dev->ops->set_bb_tbl(dev, rqd, 1)) if (dev->ops->set_bb_tbl(dev, rqd, 1))
return; return;
gennvm_addr_to_generic_mode(dev, rqd); nvm_addr_to_generic_mode(dev, rqd);
/* look up blocks and mark them as bad */ /* look up blocks and mark them as bad */
if (rqd->nr_pages > 1) if (rqd->nr_pages > 1)
for (i = 0; i < rqd->nr_pages; i++) for (i = 0; i < rqd->nr_pages; i++)
gennvm_blk_set_type(dev, &rqd->ppa_list[i], 2); gennvm_blk_set_type(dev, &rqd->ppa_list[i],
NVM_BLK_ST_BAD);
else else
gennvm_blk_set_type(dev, &rqd->ppa_addr, 2); gennvm_blk_set_type(dev, &rqd->ppa_addr, NVM_BLK_ST_BAD);
} }
static int gennvm_end_io(struct nvm_rq *rqd, int error) static void gennvm_end_io(struct nvm_rq *rqd)
{ {
struct nvm_tgt_instance *ins = rqd->ins; struct nvm_tgt_instance *ins = rqd->ins;
int ret = 0;
switch (error) { switch (rqd->error) {
case NVM_RSP_SUCCESS: case NVM_RSP_SUCCESS:
break;
case NVM_RSP_ERR_EMPTYPAGE: case NVM_RSP_ERR_EMPTYPAGE:
break; break;
case NVM_RSP_ERR_FAILWRITE: case NVM_RSP_ERR_FAILWRITE:
gennvm_mark_blk_bad(rqd->dev, rqd); gennvm_mark_blk_bad(rqd->dev, rqd);
default:
ret++;
} }
ret += ins->tt->end_io(rqd, error); ins->tt->end_io(rqd);
return ret;
} }
static int gennvm_erase_blk(struct nvm_dev *dev, struct nvm_block *blk, static int gennvm_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd)
unsigned long flags)
{ {
int plane_cnt = 0, pl_idx, ret; if (!dev->ops->submit_io)
struct ppa_addr addr; return -ENODEV;
struct nvm_rq rqd;
if (!dev->ops->erase_block)
return 0;
addr = block_to_ppa(dev, blk);
if (dev->plane_mode == NVM_PLANE_SINGLE) {
rqd.nr_pages = 1;
rqd.ppa_addr = addr;
} else {
plane_cnt = (1 << dev->plane_mode);
rqd.nr_pages = plane_cnt;
rqd.ppa_list = nvm_dev_dma_alloc(dev, GFP_KERNEL,
&rqd.dma_ppa_list);
if (!rqd.ppa_list) {
pr_err("gennvm: failed to allocate dma memory\n");
return -ENOMEM;
}
for (pl_idx = 0; pl_idx < plane_cnt; pl_idx++) {
addr.g.pl = pl_idx;
rqd.ppa_list[pl_idx] = addr;
}
}
gennvm_generic_to_addr_mode(dev, &rqd); /* Convert address space */
nvm_generic_to_addr_mode(dev, rqd);
ret = dev->ops->erase_block(dev, &rqd); rqd->dev = dev;
rqd->end_io = gennvm_end_io;
return dev->ops->submit_io(dev, rqd);
}
if (plane_cnt) static int gennvm_erase_blk(struct nvm_dev *dev, struct nvm_block *blk,
nvm_dev_dma_free(dev, rqd.ppa_list, rqd.dma_ppa_list); unsigned long flags)
{
struct ppa_addr addr = block_to_ppa(dev, blk);
return ret; return nvm_erase_ppa(dev, &addr, 1);
} }
static struct nvm_lun *gennvm_get_lun(struct nvm_dev *dev, int lunid) static struct nvm_lun *gennvm_get_lun(struct nvm_dev *dev, int lunid)
...@@ -480,10 +437,11 @@ static void gennvm_lun_info_print(struct nvm_dev *dev) ...@@ -480,10 +437,11 @@ static void gennvm_lun_info_print(struct nvm_dev *dev)
gennvm_for_each_lun(gn, lun, i) { gennvm_for_each_lun(gn, lun, i) {
spin_lock(&lun->vlun.lock); spin_lock(&lun->vlun.lock);
pr_info("%s: lun%8u\t%u\t%u\t%u\n", pr_info("%s: lun%8u\t%u\t%u\t%u\t%u\n",
dev->name, i, dev->name, i,
lun->vlun.nr_free_blocks, lun->vlun.nr_free_blocks,
lun->vlun.nr_inuse_blocks, lun->vlun.nr_open_blocks,
lun->vlun.nr_closed_blocks,
lun->vlun.nr_bad_blocks); lun->vlun.nr_bad_blocks);
spin_unlock(&lun->vlun.lock); spin_unlock(&lun->vlun.lock);
...@@ -491,21 +449,23 @@ static void gennvm_lun_info_print(struct nvm_dev *dev) ...@@ -491,21 +449,23 @@ static void gennvm_lun_info_print(struct nvm_dev *dev)
} }
static struct nvmm_type gennvm = { static struct nvmm_type gennvm = {
.name = "gennvm", .name = "gennvm",
.version = {0, 1, 0}, .version = {0, 1, 0},
.register_mgr = gennvm_register,
.unregister_mgr = gennvm_unregister,
.register_mgr = gennvm_register, .get_blk_unlocked = gennvm_get_blk_unlocked,
.unregister_mgr = gennvm_unregister, .put_blk_unlocked = gennvm_put_blk_unlocked,
.get_blk = gennvm_get_blk, .get_blk = gennvm_get_blk,
.put_blk = gennvm_put_blk, .put_blk = gennvm_put_blk,
.submit_io = gennvm_submit_io, .submit_io = gennvm_submit_io,
.end_io = gennvm_end_io, .erase_blk = gennvm_erase_blk,
.erase_blk = gennvm_erase_blk,
.get_lun = gennvm_get_lun, .get_lun = gennvm_get_lun,
.lun_info_print = gennvm_lun_info_print, .lun_info_print = gennvm_lun_info_print,
}; };
static int __init gennvm_module_init(void) static int __init gennvm_module_init(void)
......
...@@ -179,16 +179,23 @@ static void rrpc_set_lun_cur(struct rrpc_lun *rlun, struct rrpc_block *rblk) ...@@ -179,16 +179,23 @@ static void rrpc_set_lun_cur(struct rrpc_lun *rlun, struct rrpc_block *rblk)
static struct rrpc_block *rrpc_get_blk(struct rrpc *rrpc, struct rrpc_lun *rlun, static struct rrpc_block *rrpc_get_blk(struct rrpc *rrpc, struct rrpc_lun *rlun,
unsigned long flags) unsigned long flags)
{ {
struct nvm_lun *lun = rlun->parent;
struct nvm_block *blk; struct nvm_block *blk;
struct rrpc_block *rblk; struct rrpc_block *rblk;
blk = nvm_get_blk(rrpc->dev, rlun->parent, flags); spin_lock(&lun->lock);
if (!blk) blk = nvm_get_blk_unlocked(rrpc->dev, rlun->parent, flags);
if (!blk) {
pr_err("nvm: rrpc: cannot get new block from media manager\n");
spin_unlock(&lun->lock);
return NULL; return NULL;
}
rblk = &rlun->blocks[blk->id]; rblk = &rlun->blocks[blk->id];
blk->priv = rblk; list_add_tail(&rblk->list, &rlun->open_list);
spin_unlock(&lun->lock);
blk->priv = rblk;
bitmap_zero(rblk->invalid_pages, rrpc->dev->pgs_per_blk); bitmap_zero(rblk->invalid_pages, rrpc->dev->pgs_per_blk);
rblk->next_page = 0; rblk->next_page = 0;
rblk->nr_invalid_pages = 0; rblk->nr_invalid_pages = 0;
...@@ -199,7 +206,13 @@ static struct rrpc_block *rrpc_get_blk(struct rrpc *rrpc, struct rrpc_lun *rlun, ...@@ -199,7 +206,13 @@ static struct rrpc_block *rrpc_get_blk(struct rrpc *rrpc, struct rrpc_lun *rlun,
static void rrpc_put_blk(struct rrpc *rrpc, struct rrpc_block *rblk) static void rrpc_put_blk(struct rrpc *rrpc, struct rrpc_block *rblk)
{ {
nvm_put_blk(rrpc->dev, rblk->parent); struct rrpc_lun *rlun = rblk->rlun;
struct nvm_lun *lun = rlun->parent;
spin_lock(&lun->lock);
nvm_put_blk_unlocked(rrpc->dev, rblk->parent);
list_del(&rblk->list);
spin_unlock(&lun->lock);
} }
static void rrpc_put_blks(struct rrpc *rrpc) static void rrpc_put_blks(struct rrpc *rrpc)
...@@ -287,6 +300,8 @@ static int rrpc_move_valid_pages(struct rrpc *rrpc, struct rrpc_block *rblk) ...@@ -287,6 +300,8 @@ static int rrpc_move_valid_pages(struct rrpc *rrpc, struct rrpc_block *rblk)
} }
page = mempool_alloc(rrpc->page_pool, GFP_NOIO); page = mempool_alloc(rrpc->page_pool, GFP_NOIO);
if (!page)
return -ENOMEM;
while ((slot = find_first_zero_bit(rblk->invalid_pages, while ((slot = find_first_zero_bit(rblk->invalid_pages,
nr_pgs_per_blk)) < nr_pgs_per_blk) { nr_pgs_per_blk)) < nr_pgs_per_blk) {
...@@ -328,6 +343,10 @@ static int rrpc_move_valid_pages(struct rrpc *rrpc, struct rrpc_block *rblk) ...@@ -328,6 +343,10 @@ static int rrpc_move_valid_pages(struct rrpc *rrpc, struct rrpc_block *rblk)
goto finished; goto finished;
} }
wait_for_completion_io(&wait); wait_for_completion_io(&wait);
if (bio->bi_error) {
rrpc_inflight_laddr_release(rrpc, rqd);
goto finished;
}
bio_reset(bio); bio_reset(bio);
reinit_completion(&wait); reinit_completion(&wait);
...@@ -350,6 +369,8 @@ static int rrpc_move_valid_pages(struct rrpc *rrpc, struct rrpc_block *rblk) ...@@ -350,6 +369,8 @@ static int rrpc_move_valid_pages(struct rrpc *rrpc, struct rrpc_block *rblk)
wait_for_completion_io(&wait); wait_for_completion_io(&wait);
rrpc_inflight_laddr_release(rrpc, rqd); rrpc_inflight_laddr_release(rrpc, rqd);
if (bio->bi_error)
goto finished;
bio_reset(bio); bio_reset(bio);
} }
...@@ -373,16 +394,26 @@ static void rrpc_block_gc(struct work_struct *work) ...@@ -373,16 +394,26 @@ static void rrpc_block_gc(struct work_struct *work)
struct rrpc *rrpc = gcb->rrpc; struct rrpc *rrpc = gcb->rrpc;
struct rrpc_block *rblk = gcb->rblk; struct rrpc_block *rblk = gcb->rblk;
struct nvm_dev *dev = rrpc->dev; struct nvm_dev *dev = rrpc->dev;
struct nvm_lun *lun = rblk->parent->lun;
struct rrpc_lun *rlun = &rrpc->luns[lun->id - rrpc->lun_offset];
mempool_free(gcb, rrpc->gcb_pool);
pr_debug("nvm: block '%lu' being reclaimed\n", rblk->parent->id); pr_debug("nvm: block '%lu' being reclaimed\n", rblk->parent->id);
if (rrpc_move_valid_pages(rrpc, rblk)) if (rrpc_move_valid_pages(rrpc, rblk))
goto done; goto put_back;
if (nvm_erase_blk(dev, rblk->parent))
goto put_back;
nvm_erase_blk(dev, rblk->parent);
rrpc_put_blk(rrpc, rblk); rrpc_put_blk(rrpc, rblk);
done:
mempool_free(gcb, rrpc->gcb_pool); return;
put_back:
spin_lock(&rlun->lock);
list_add_tail(&rblk->prio, &rlun->prio_list);
spin_unlock(&rlun->lock);
} }
/* the block with highest number of invalid pages, will be in the beginning /* the block with highest number of invalid pages, will be in the beginning
...@@ -427,7 +458,7 @@ static void rrpc_lun_gc(struct work_struct *work) ...@@ -427,7 +458,7 @@ static void rrpc_lun_gc(struct work_struct *work)
if (nr_blocks_need < rrpc->nr_luns) if (nr_blocks_need < rrpc->nr_luns)
nr_blocks_need = rrpc->nr_luns; nr_blocks_need = rrpc->nr_luns;
spin_lock(&lun->lock); spin_lock(&rlun->lock);
while (nr_blocks_need > lun->nr_free_blocks && while (nr_blocks_need > lun->nr_free_blocks &&
!list_empty(&rlun->prio_list)) { !list_empty(&rlun->prio_list)) {
struct rrpc_block *rblock = block_prio_find_max(rlun); struct rrpc_block *rblock = block_prio_find_max(rlun);
...@@ -436,16 +467,16 @@ static void rrpc_lun_gc(struct work_struct *work) ...@@ -436,16 +467,16 @@ static void rrpc_lun_gc(struct work_struct *work)
if (!rblock->nr_invalid_pages) if (!rblock->nr_invalid_pages)
break; break;
gcb = mempool_alloc(rrpc->gcb_pool, GFP_ATOMIC);
if (!gcb)
break;
list_del_init(&rblock->prio); list_del_init(&rblock->prio);
BUG_ON(!block_is_full(rrpc, rblock)); BUG_ON(!block_is_full(rrpc, rblock));
pr_debug("rrpc: selected block '%lu' for GC\n", block->id); pr_debug("rrpc: selected block '%lu' for GC\n", block->id);
gcb = mempool_alloc(rrpc->gcb_pool, GFP_ATOMIC);
if (!gcb)
break;
gcb->rrpc = rrpc; gcb->rrpc = rrpc;
gcb->rblk = rblock; gcb->rblk = rblock;
INIT_WORK(&gcb->ws_gc, rrpc_block_gc); INIT_WORK(&gcb->ws_gc, rrpc_block_gc);
...@@ -454,7 +485,7 @@ static void rrpc_lun_gc(struct work_struct *work) ...@@ -454,7 +485,7 @@ static void rrpc_lun_gc(struct work_struct *work)
nr_blocks_need--; nr_blocks_need--;
} }
spin_unlock(&lun->lock); spin_unlock(&rlun->lock);
/* TODO: Hint that request queue can be started again */ /* TODO: Hint that request queue can be started again */
} }
...@@ -635,12 +666,24 @@ static void rrpc_end_io_write(struct rrpc *rrpc, struct rrpc_rq *rrqd, ...@@ -635,12 +666,24 @@ static void rrpc_end_io_write(struct rrpc *rrpc, struct rrpc_rq *rrqd,
lun = rblk->parent->lun; lun = rblk->parent->lun;
cmnt_size = atomic_inc_return(&rblk->data_cmnt_size); cmnt_size = atomic_inc_return(&rblk->data_cmnt_size);
if (unlikely(cmnt_size == rrpc->dev->pgs_per_blk)) if (unlikely(cmnt_size == rrpc->dev->pgs_per_blk)) {
struct nvm_block *blk = rblk->parent;
struct rrpc_lun *rlun = rblk->rlun;
spin_lock(&lun->lock);
lun->nr_open_blocks--;
lun->nr_closed_blocks++;
blk->state &= ~NVM_BLK_ST_OPEN;
blk->state |= NVM_BLK_ST_CLOSED;
list_move_tail(&rblk->list, &rlun->closed_list);
spin_unlock(&lun->lock);
rrpc_run_gc(rrpc, rblk); rrpc_run_gc(rrpc, rblk);
}
} }
} }
static int rrpc_end_io(struct nvm_rq *rqd, int error) static void rrpc_end_io(struct nvm_rq *rqd)
{ {
struct rrpc *rrpc = container_of(rqd->ins, struct rrpc, instance); struct rrpc *rrpc = container_of(rqd->ins, struct rrpc, instance);
struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd); struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd);
...@@ -650,11 +693,12 @@ static int rrpc_end_io(struct nvm_rq *rqd, int error) ...@@ -650,11 +693,12 @@ static int rrpc_end_io(struct nvm_rq *rqd, int error)
if (bio_data_dir(rqd->bio) == WRITE) if (bio_data_dir(rqd->bio) == WRITE)
rrpc_end_io_write(rrpc, rrqd, laddr, npages); rrpc_end_io_write(rrpc, rrqd, laddr, npages);
bio_put(rqd->bio);
if (rrqd->flags & NVM_IOTYPE_GC) if (rrqd->flags & NVM_IOTYPE_GC)
return 0; return;
rrpc_unlock_rq(rrpc, rqd); rrpc_unlock_rq(rrpc, rqd);
bio_put(rqd->bio);
if (npages > 1) if (npages > 1)
nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, rqd->dma_ppa_list); nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, rqd->dma_ppa_list);
...@@ -662,8 +706,6 @@ static int rrpc_end_io(struct nvm_rq *rqd, int error) ...@@ -662,8 +706,6 @@ static int rrpc_end_io(struct nvm_rq *rqd, int error)
nvm_dev_dma_free(rrpc->dev, rqd->metadata, rqd->dma_metadata); nvm_dev_dma_free(rrpc->dev, rqd->metadata, rqd->dma_metadata);
mempool_free(rqd, rrpc->rq_pool); mempool_free(rqd, rrpc->rq_pool);
return 0;
} }
static int rrpc_read_ppalist_rq(struct rrpc *rrpc, struct bio *bio, static int rrpc_read_ppalist_rq(struct rrpc *rrpc, struct bio *bio,
...@@ -841,6 +883,13 @@ static int rrpc_submit_io(struct rrpc *rrpc, struct bio *bio, ...@@ -841,6 +883,13 @@ static int rrpc_submit_io(struct rrpc *rrpc, struct bio *bio,
err = nvm_submit_io(rrpc->dev, rqd); err = nvm_submit_io(rrpc->dev, rqd);
if (err) { if (err) {
pr_err("rrpc: I/O submission failed: %d\n", err); pr_err("rrpc: I/O submission failed: %d\n", err);
bio_put(bio);
if (!(flags & NVM_IOTYPE_GC)) {
rrpc_unlock_rq(rrpc, rqd);
if (rqd->nr_pages > 1)
nvm_dev_dma_free(rrpc->dev,
rqd->ppa_list, rqd->dma_ppa_list);
}
return NVM_IO_ERR; return NVM_IO_ERR;
} }
...@@ -1090,6 +1139,11 @@ static int rrpc_luns_init(struct rrpc *rrpc, int lun_begin, int lun_end) ...@@ -1090,6 +1139,11 @@ static int rrpc_luns_init(struct rrpc *rrpc, int lun_begin, int lun_end)
struct rrpc_lun *rlun; struct rrpc_lun *rlun;
int i, j; int i, j;
if (dev->pgs_per_blk > MAX_INVALID_PAGES_STORAGE * BITS_PER_LONG) {
pr_err("rrpc: number of pages per block too high.");
return -EINVAL;
}
spin_lock_init(&rrpc->rev_lock); spin_lock_init(&rrpc->rev_lock);
rrpc->luns = kcalloc(rrpc->nr_luns, sizeof(struct rrpc_lun), rrpc->luns = kcalloc(rrpc->nr_luns, sizeof(struct rrpc_lun),
...@@ -1101,16 +1155,13 @@ static int rrpc_luns_init(struct rrpc *rrpc, int lun_begin, int lun_end) ...@@ -1101,16 +1155,13 @@ static int rrpc_luns_init(struct rrpc *rrpc, int lun_begin, int lun_end)
for (i = 0; i < rrpc->nr_luns; i++) { for (i = 0; i < rrpc->nr_luns; i++) {
struct nvm_lun *lun = dev->mt->get_lun(dev, lun_begin + i); struct nvm_lun *lun = dev->mt->get_lun(dev, lun_begin + i);
if (dev->pgs_per_blk >
MAX_INVALID_PAGES_STORAGE * BITS_PER_LONG) {
pr_err("rrpc: number of pages per block too high.");
goto err;
}
rlun = &rrpc->luns[i]; rlun = &rrpc->luns[i];
rlun->rrpc = rrpc; rlun->rrpc = rrpc;
rlun->parent = lun; rlun->parent = lun;
INIT_LIST_HEAD(&rlun->prio_list); INIT_LIST_HEAD(&rlun->prio_list);
INIT_LIST_HEAD(&rlun->open_list);
INIT_LIST_HEAD(&rlun->closed_list);
INIT_WORK(&rlun->ws_gc, rrpc_lun_gc); INIT_WORK(&rlun->ws_gc, rrpc_lun_gc);
spin_lock_init(&rlun->lock); spin_lock_init(&rlun->lock);
...@@ -1127,6 +1178,7 @@ static int rrpc_luns_init(struct rrpc *rrpc, int lun_begin, int lun_end) ...@@ -1127,6 +1178,7 @@ static int rrpc_luns_init(struct rrpc *rrpc, int lun_begin, int lun_end)
struct nvm_block *blk = &lun->blocks[j]; struct nvm_block *blk = &lun->blocks[j];
rblk->parent = blk; rblk->parent = blk;
rblk->rlun = rlun;
INIT_LIST_HEAD(&rblk->prio); INIT_LIST_HEAD(&rblk->prio);
spin_lock_init(&rblk->lock); spin_lock_init(&rblk->lock);
} }
......
...@@ -54,7 +54,9 @@ struct rrpc_rq { ...@@ -54,7 +54,9 @@ struct rrpc_rq {
struct rrpc_block { struct rrpc_block {
struct nvm_block *parent; struct nvm_block *parent;
struct rrpc_lun *rlun;
struct list_head prio; struct list_head prio;
struct list_head list;
#define MAX_INVALID_PAGES_STORAGE 8 #define MAX_INVALID_PAGES_STORAGE 8
/* Bitmap for invalid page intries */ /* Bitmap for invalid page intries */
...@@ -73,7 +75,16 @@ struct rrpc_lun { ...@@ -73,7 +75,16 @@ struct rrpc_lun {
struct nvm_lun *parent; struct nvm_lun *parent;
struct rrpc_block *cur, *gc_cur; struct rrpc_block *cur, *gc_cur;
struct rrpc_block *blocks; /* Reference to block allocation */ struct rrpc_block *blocks; /* Reference to block allocation */
struct list_head prio_list; /* Blocks that may be GC'ed */
struct list_head prio_list; /* Blocks that may be GC'ed */
struct list_head open_list; /* In-use open blocks. These are blocks
* that can be both written to and read
* from
*/
struct list_head closed_list; /* In-use closed blocks. These are
* blocks that can _only_ be read from
*/
struct work_struct ws_gc; struct work_struct ws_gc;
spinlock_t lock; spinlock_t lock;
......
This diff is collapsed.
...@@ -146,6 +146,16 @@ struct nvme_nvm_command { ...@@ -146,6 +146,16 @@ struct nvme_nvm_command {
}; };
}; };
struct nvme_nvm_lp_mlc {
__u16 num_pairs;
__u8 pairs[886];
};
struct nvme_nvm_lp_tbl {
__u8 id[8];
struct nvme_nvm_lp_mlc mlc;
};
struct nvme_nvm_id_group { struct nvme_nvm_id_group {
__u8 mtype; __u8 mtype;
__u8 fmtype; __u8 fmtype;
...@@ -169,7 +179,8 @@ struct nvme_nvm_id_group { ...@@ -169,7 +179,8 @@ struct nvme_nvm_id_group {
__le32 mpos; __le32 mpos;
__le32 mccap; __le32 mccap;
__le16 cpar; __le16 cpar;
__u8 reserved[906]; __u8 reserved[10];
struct nvme_nvm_lp_tbl lptbl;
} __packed; } __packed;
struct nvme_nvm_addr_format { struct nvme_nvm_addr_format {
...@@ -266,6 +277,15 @@ static int init_grps(struct nvm_id *nvm_id, struct nvme_nvm_id *nvme_nvm_id) ...@@ -266,6 +277,15 @@ static int init_grps(struct nvm_id *nvm_id, struct nvme_nvm_id *nvme_nvm_id)
dst->mccap = le32_to_cpu(src->mccap); dst->mccap = le32_to_cpu(src->mccap);
dst->cpar = le16_to_cpu(src->cpar); dst->cpar = le16_to_cpu(src->cpar);
if (dst->fmtype == NVM_ID_FMTYPE_MLC) {
memcpy(dst->lptbl.id, src->lptbl.id, 8);
dst->lptbl.mlc.num_pairs =
le16_to_cpu(src->lptbl.mlc.num_pairs);
/* 4 bits per pair */
memcpy(dst->lptbl.mlc.pairs, src->lptbl.mlc.pairs,
dst->lptbl.mlc.num_pairs >> 1);
}
} }
return 0; return 0;
...@@ -405,11 +425,6 @@ static int nvme_nvm_get_bb_tbl(struct nvm_dev *nvmdev, struct ppa_addr ppa, ...@@ -405,11 +425,6 @@ static int nvme_nvm_get_bb_tbl(struct nvm_dev *nvmdev, struct ppa_addr ppa,
ppa = dev_to_generic_addr(nvmdev, ppa); ppa = dev_to_generic_addr(nvmdev, ppa);
ret = update_bbtbl(ppa, nr_blocks, bb_tbl->blk, priv); ret = update_bbtbl(ppa, nr_blocks, bb_tbl->blk, priv);
if (ret) {
ret = -EINTR;
goto out;
}
out: out:
kfree(bb_tbl); kfree(bb_tbl);
return ret; return ret;
...@@ -453,11 +468,8 @@ static inline void nvme_nvm_rqtocmd(struct request *rq, struct nvm_rq *rqd, ...@@ -453,11 +468,8 @@ static inline void nvme_nvm_rqtocmd(struct request *rq, struct nvm_rq *rqd,
static void nvme_nvm_end_io(struct request *rq, int error) static void nvme_nvm_end_io(struct request *rq, int error)
{ {
struct nvm_rq *rqd = rq->end_io_data; struct nvm_rq *rqd = rq->end_io_data;
struct nvm_dev *dev = rqd->dev;
if (dev->mt && dev->mt->end_io(rqd, error)) nvm_end_io(rqd, error);
pr_err("nvme: err status: %x result: %lx\n",
rq->errors, (unsigned long)rq->special);
kfree(rq->cmd); kfree(rq->cmd);
blk_mq_free_request(rq); blk_mq_free_request(rq);
......
This diff is collapsed.
...@@ -33,6 +33,7 @@ ...@@ -33,6 +33,7 @@
#define NVM_TTYPE_NAME_MAX 48 #define NVM_TTYPE_NAME_MAX 48
#define NVM_TTYPE_MAX 63 #define NVM_TTYPE_MAX 63
#define NVM_MMTYPE_LEN 8
#define NVM_CTRL_FILE "/dev/lightnvm/control" #define NVM_CTRL_FILE "/dev/lightnvm/control"
...@@ -100,6 +101,26 @@ struct nvm_ioctl_remove { ...@@ -100,6 +101,26 @@ struct nvm_ioctl_remove {
__u32 flags; __u32 flags;
}; };
struct nvm_ioctl_dev_init {
char dev[DISK_NAME_LEN]; /* open-channel SSD device */
char mmtype[NVM_MMTYPE_LEN]; /* register to media manager */
__u32 flags;
};
enum {
NVM_FACTORY_ERASE_ONLY_USER = 1 << 0, /* erase only blocks used as
* host blks or grown blks */
NVM_FACTORY_RESET_HOST_BLKS = 1 << 1, /* remove host blk marks */
NVM_FACTORY_RESET_GRWN_BBLKS = 1 << 2, /* remove grown blk marks */
NVM_FACTORY_NR_BITS = 1 << 3, /* stops here */
};
struct nvm_ioctl_dev_factory {
char dev[DISK_NAME_LEN];
__u32 flags;
};
/* The ioctl type, 'L', 0x20 - 0x2F documented in ioctl-number.txt */ /* The ioctl type, 'L', 0x20 - 0x2F documented in ioctl-number.txt */
enum { enum {
...@@ -110,6 +131,12 @@ enum { ...@@ -110,6 +131,12 @@ enum {
/* device level cmds */ /* device level cmds */
NVM_DEV_CREATE_CMD, NVM_DEV_CREATE_CMD,
NVM_DEV_REMOVE_CMD, NVM_DEV_REMOVE_CMD,
/* Init a device to support LightNVM media managers */
NVM_DEV_INIT_CMD,
/* Factory reset device */
NVM_DEV_FACTORY_CMD,
}; };
#define NVM_IOCTL 'L' /* 0x4c */ #define NVM_IOCTL 'L' /* 0x4c */
...@@ -122,6 +149,10 @@ enum { ...@@ -122,6 +149,10 @@ enum {
struct nvm_ioctl_create) struct nvm_ioctl_create)
#define NVM_DEV_REMOVE _IOW(NVM_IOCTL, NVM_DEV_REMOVE_CMD, \ #define NVM_DEV_REMOVE _IOW(NVM_IOCTL, NVM_DEV_REMOVE_CMD, \
struct nvm_ioctl_remove) struct nvm_ioctl_remove)
#define NVM_DEV_INIT _IOW(NVM_IOCTL, NVM_DEV_INIT_CMD, \
struct nvm_ioctl_dev_init)
#define NVM_DEV_FACTORY _IOW(NVM_IOCTL, NVM_DEV_FACTORY_CMD, \
struct nvm_ioctl_dev_factory)
#define NVM_VERSION_MAJOR 1 #define NVM_VERSION_MAJOR 1
#define NVM_VERSION_MINOR 0 #define NVM_VERSION_MINOR 0
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment