Commit b84ae4a8 authored by Javier González's avatar Javier González Committed by Jens Axboe

lightnvm: pblk: simplify work_queue mempool

In pblk, we have a mempool to allocate a generic structure that we
pass along workqueues. This is heavily used in the GC path in order
to have enough inflight reads and fully utilize the GC bandwidth.

However, the current GC path copies data to the host memory and puts it
back into the write buffer. This requires a vmalloc allocation for the
data and a memory copy. Thus, guaranteeing the allocation by using a
mempool for the structure in itself does not give us much. Until we
implement support for vector copy to avoid moving data through the host,
just allocate the workqueue structure using kmalloc.

This allows us to have a much smaller mempool.
Reported-by: default avatarJens Axboe <axboe@kernel.dk>
Signed-off-by: default avatarJavier González <javier@cnexlabs.com>
Signed-off-by: default avatarMatias Bjørling <m@bjorling.me>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent bd432417
...@@ -33,7 +33,8 @@ static void pblk_mark_bb(struct pblk *pblk, struct pblk_line *line, ...@@ -33,7 +33,8 @@ static void pblk_mark_bb(struct pblk *pblk, struct pblk_line *line,
pr_err("pblk: attempted to erase bb: line:%d, pos:%d\n", pr_err("pblk: attempted to erase bb: line:%d, pos:%d\n",
line->id, pos); line->id, pos);
pblk_line_run_ws(pblk, NULL, ppa, pblk_line_mark_bb, pblk->bb_wq); pblk_gen_run_ws(pblk, NULL, ppa, pblk_line_mark_bb,
GFP_ATOMIC, pblk->bb_wq);
} }
static void __pblk_end_io_erase(struct pblk *pblk, struct nvm_rq *rqd) static void __pblk_end_io_erase(struct pblk *pblk, struct nvm_rq *rqd)
...@@ -1623,7 +1624,7 @@ void pblk_line_close_ws(struct work_struct *work) ...@@ -1623,7 +1624,7 @@ void pblk_line_close_ws(struct work_struct *work)
struct pblk_line *line = line_ws->line; struct pblk_line *line = line_ws->line;
pblk_line_close(pblk, line); pblk_line_close(pblk, line);
mempool_free(line_ws, pblk->line_ws_pool); mempool_free(line_ws, pblk->gen_ws_pool);
} }
void pblk_line_mark_bb(struct work_struct *work) void pblk_line_mark_bb(struct work_struct *work)
...@@ -1648,16 +1649,16 @@ void pblk_line_mark_bb(struct work_struct *work) ...@@ -1648,16 +1649,16 @@ void pblk_line_mark_bb(struct work_struct *work)
} }
kfree(ppa); kfree(ppa);
mempool_free(line_ws, pblk->line_ws_pool); mempool_free(line_ws, pblk->gen_ws_pool);
} }
void pblk_line_run_ws(struct pblk *pblk, struct pblk_line *line, void *priv, void pblk_gen_run_ws(struct pblk *pblk, struct pblk_line *line, void *priv,
void (*work)(struct work_struct *), void (*work)(struct work_struct *), gfp_t gfp_mask,
struct workqueue_struct *wq) struct workqueue_struct *wq)
{ {
struct pblk_line_ws *line_ws; struct pblk_line_ws *line_ws;
line_ws = mempool_alloc(pblk->line_ws_pool, GFP_ATOMIC); line_ws = mempool_alloc(pblk->gen_ws_pool, gfp_mask);
if (!line_ws) if (!line_ws)
return; return;
......
...@@ -136,12 +136,12 @@ static void pblk_put_line_back(struct pblk *pblk, struct pblk_line *line) ...@@ -136,12 +136,12 @@ static void pblk_put_line_back(struct pblk *pblk, struct pblk_line *line)
static void pblk_gc_line_ws(struct work_struct *work) static void pblk_gc_line_ws(struct work_struct *work)
{ {
struct pblk_line_ws *line_rq_ws = container_of(work, struct pblk_line_ws *gc_rq_ws = container_of(work,
struct pblk_line_ws, ws); struct pblk_line_ws, ws);
struct pblk *pblk = line_rq_ws->pblk; struct pblk *pblk = gc_rq_ws->pblk;
struct pblk_gc *gc = &pblk->gc; struct pblk_gc *gc = &pblk->gc;
struct pblk_line *line = line_rq_ws->line; struct pblk_line *line = gc_rq_ws->line;
struct pblk_gc_rq *gc_rq = line_rq_ws->priv; struct pblk_gc_rq *gc_rq = gc_rq_ws->priv;
up(&gc->gc_sem); up(&gc->gc_sem);
...@@ -151,7 +151,7 @@ static void pblk_gc_line_ws(struct work_struct *work) ...@@ -151,7 +151,7 @@ static void pblk_gc_line_ws(struct work_struct *work)
gc_rq->nr_secs); gc_rq->nr_secs);
} }
mempool_free(line_rq_ws, pblk->line_ws_pool); kfree(gc_rq_ws);
} }
static void pblk_gc_line_prepare_ws(struct work_struct *work) static void pblk_gc_line_prepare_ws(struct work_struct *work)
...@@ -164,7 +164,7 @@ static void pblk_gc_line_prepare_ws(struct work_struct *work) ...@@ -164,7 +164,7 @@ static void pblk_gc_line_prepare_ws(struct work_struct *work)
struct pblk_line_meta *lm = &pblk->lm; struct pblk_line_meta *lm = &pblk->lm;
struct pblk_gc *gc = &pblk->gc; struct pblk_gc *gc = &pblk->gc;
struct line_emeta *emeta_buf; struct line_emeta *emeta_buf;
struct pblk_line_ws *line_rq_ws; struct pblk_line_ws *gc_rq_ws;
struct pblk_gc_rq *gc_rq; struct pblk_gc_rq *gc_rq;
__le64 *lba_list; __le64 *lba_list;
int sec_left, nr_secs, bit; int sec_left, nr_secs, bit;
...@@ -223,19 +223,19 @@ static void pblk_gc_line_prepare_ws(struct work_struct *work) ...@@ -223,19 +223,19 @@ static void pblk_gc_line_prepare_ws(struct work_struct *work)
gc_rq->nr_secs = nr_secs; gc_rq->nr_secs = nr_secs;
gc_rq->line = line; gc_rq->line = line;
line_rq_ws = mempool_alloc(pblk->line_ws_pool, GFP_KERNEL); gc_rq_ws = kmalloc(sizeof(struct pblk_line_ws), GFP_KERNEL);
if (!line_rq_ws) if (!gc_rq_ws)
goto fail_free_gc_rq; goto fail_free_gc_rq;
line_rq_ws->pblk = pblk; gc_rq_ws->pblk = pblk;
line_rq_ws->line = line; gc_rq_ws->line = line;
line_rq_ws->priv = gc_rq; gc_rq_ws->priv = gc_rq;
down(&gc->gc_sem); down(&gc->gc_sem);
kref_get(&line->ref); kref_get(&line->ref);
INIT_WORK(&line_rq_ws->ws, pblk_gc_line_ws); INIT_WORK(&gc_rq_ws->ws, pblk_gc_line_ws);
queue_work(gc->gc_line_reader_wq, &line_rq_ws->ws); queue_work(gc->gc_line_reader_wq, &gc_rq_ws->ws);
sec_left -= nr_secs; sec_left -= nr_secs;
if (sec_left > 0) if (sec_left > 0)
...@@ -243,7 +243,7 @@ static void pblk_gc_line_prepare_ws(struct work_struct *work) ...@@ -243,7 +243,7 @@ static void pblk_gc_line_prepare_ws(struct work_struct *work)
out: out:
pblk_mfree(emeta_buf, l_mg->emeta_alloc_type); pblk_mfree(emeta_buf, l_mg->emeta_alloc_type);
mempool_free(line_ws, pblk->line_ws_pool); kfree(line_ws);
kref_put(&line->ref, pblk_line_put); kref_put(&line->ref, pblk_line_put);
atomic_dec(&gc->inflight_gc); atomic_dec(&gc->inflight_gc);
...@@ -256,7 +256,7 @@ static void pblk_gc_line_prepare_ws(struct work_struct *work) ...@@ -256,7 +256,7 @@ static void pblk_gc_line_prepare_ws(struct work_struct *work)
pblk_mfree(emeta_buf, l_mg->emeta_alloc_type); pblk_mfree(emeta_buf, l_mg->emeta_alloc_type);
pblk_put_line_back(pblk, line); pblk_put_line_back(pblk, line);
kref_put(&line->ref, pblk_line_put); kref_put(&line->ref, pblk_line_put);
mempool_free(line_ws, pblk->line_ws_pool); kfree(line_ws);
atomic_dec(&gc->inflight_gc); atomic_dec(&gc->inflight_gc);
pr_err("pblk: Failed to GC line %d\n", line->id); pr_err("pblk: Failed to GC line %d\n", line->id);
...@@ -269,7 +269,7 @@ static int pblk_gc_line(struct pblk *pblk, struct pblk_line *line) ...@@ -269,7 +269,7 @@ static int pblk_gc_line(struct pblk *pblk, struct pblk_line *line)
pr_debug("pblk: line '%d' being reclaimed for GC\n", line->id); pr_debug("pblk: line '%d' being reclaimed for GC\n", line->id);
line_ws = mempool_alloc(pblk->line_ws_pool, GFP_KERNEL); line_ws = kmalloc(sizeof(struct pblk_line_ws), GFP_KERNEL);
if (!line_ws) if (!line_ws)
return -ENOMEM; return -ENOMEM;
......
...@@ -20,7 +20,7 @@ ...@@ -20,7 +20,7 @@
#include "pblk.h" #include "pblk.h"
static struct kmem_cache *pblk_blk_ws_cache, *pblk_rec_cache, *pblk_g_rq_cache, static struct kmem_cache *pblk_ws_cache, *pblk_rec_cache, *pblk_g_rq_cache,
*pblk_w_rq_cache, *pblk_line_meta_cache; *pblk_w_rq_cache, *pblk_line_meta_cache;
static DECLARE_RWSEM(pblk_lock); static DECLARE_RWSEM(pblk_lock);
struct bio_set *pblk_bio_set; struct bio_set *pblk_bio_set;
...@@ -184,9 +184,9 @@ static int pblk_init_global_caches(struct pblk *pblk) ...@@ -184,9 +184,9 @@ static int pblk_init_global_caches(struct pblk *pblk)
char cache_name[PBLK_CACHE_NAME_LEN]; char cache_name[PBLK_CACHE_NAME_LEN];
down_write(&pblk_lock); down_write(&pblk_lock);
pblk_blk_ws_cache = kmem_cache_create("pblk_blk_ws", pblk_ws_cache = kmem_cache_create("pblk_blk_ws",
sizeof(struct pblk_line_ws), 0, 0, NULL); sizeof(struct pblk_line_ws), 0, 0, NULL);
if (!pblk_blk_ws_cache) { if (!pblk_ws_cache) {
up_write(&pblk_lock); up_write(&pblk_lock);
return -ENOMEM; return -ENOMEM;
} }
...@@ -194,7 +194,7 @@ static int pblk_init_global_caches(struct pblk *pblk) ...@@ -194,7 +194,7 @@ static int pblk_init_global_caches(struct pblk *pblk)
pblk_rec_cache = kmem_cache_create("pblk_rec", pblk_rec_cache = kmem_cache_create("pblk_rec",
sizeof(struct pblk_rec_ctx), 0, 0, NULL); sizeof(struct pblk_rec_ctx), 0, 0, NULL);
if (!pblk_rec_cache) { if (!pblk_rec_cache) {
kmem_cache_destroy(pblk_blk_ws_cache); kmem_cache_destroy(pblk_ws_cache);
up_write(&pblk_lock); up_write(&pblk_lock);
return -ENOMEM; return -ENOMEM;
} }
...@@ -202,7 +202,7 @@ static int pblk_init_global_caches(struct pblk *pblk) ...@@ -202,7 +202,7 @@ static int pblk_init_global_caches(struct pblk *pblk)
pblk_g_rq_cache = kmem_cache_create("pblk_g_rq", pblk_g_rq_size, pblk_g_rq_cache = kmem_cache_create("pblk_g_rq", pblk_g_rq_size,
0, 0, NULL); 0, 0, NULL);
if (!pblk_g_rq_cache) { if (!pblk_g_rq_cache) {
kmem_cache_destroy(pblk_blk_ws_cache); kmem_cache_destroy(pblk_ws_cache);
kmem_cache_destroy(pblk_rec_cache); kmem_cache_destroy(pblk_rec_cache);
up_write(&pblk_lock); up_write(&pblk_lock);
return -ENOMEM; return -ENOMEM;
...@@ -211,7 +211,7 @@ static int pblk_init_global_caches(struct pblk *pblk) ...@@ -211,7 +211,7 @@ static int pblk_init_global_caches(struct pblk *pblk)
pblk_w_rq_cache = kmem_cache_create("pblk_w_rq", pblk_w_rq_size, pblk_w_rq_cache = kmem_cache_create("pblk_w_rq", pblk_w_rq_size,
0, 0, NULL); 0, 0, NULL);
if (!pblk_w_rq_cache) { if (!pblk_w_rq_cache) {
kmem_cache_destroy(pblk_blk_ws_cache); kmem_cache_destroy(pblk_ws_cache);
kmem_cache_destroy(pblk_rec_cache); kmem_cache_destroy(pblk_rec_cache);
kmem_cache_destroy(pblk_g_rq_cache); kmem_cache_destroy(pblk_g_rq_cache);
up_write(&pblk_lock); up_write(&pblk_lock);
...@@ -223,7 +223,7 @@ static int pblk_init_global_caches(struct pblk *pblk) ...@@ -223,7 +223,7 @@ static int pblk_init_global_caches(struct pblk *pblk)
pblk_line_meta_cache = kmem_cache_create(cache_name, pblk_line_meta_cache = kmem_cache_create(cache_name,
pblk->lm.sec_bitmap_len, 0, 0, NULL); pblk->lm.sec_bitmap_len, 0, 0, NULL);
if (!pblk_line_meta_cache) { if (!pblk_line_meta_cache) {
kmem_cache_destroy(pblk_blk_ws_cache); kmem_cache_destroy(pblk_ws_cache);
kmem_cache_destroy(pblk_rec_cache); kmem_cache_destroy(pblk_rec_cache);
kmem_cache_destroy(pblk_g_rq_cache); kmem_cache_destroy(pblk_g_rq_cache);
kmem_cache_destroy(pblk_w_rq_cache); kmem_cache_destroy(pblk_w_rq_cache);
...@@ -246,20 +246,20 @@ static int pblk_core_init(struct pblk *pblk) ...@@ -246,20 +246,20 @@ static int pblk_core_init(struct pblk *pblk)
if (pblk_init_global_caches(pblk)) if (pblk_init_global_caches(pblk))
return -ENOMEM; return -ENOMEM;
/* internal bios can be at most the sectors signaled by the device. */ /* Internal bios can be at most the sectors signaled by the device. */
pblk->page_bio_pool = mempool_create_page_pool(nvm_max_phys_sects(dev), pblk->page_bio_pool = mempool_create_page_pool(nvm_max_phys_sects(dev),
0); 0);
if (!pblk->page_bio_pool) if (!pblk->page_bio_pool)
return -ENOMEM; return -ENOMEM;
pblk->line_ws_pool = mempool_create_slab_pool(PBLK_WS_POOL_SIZE, pblk->gen_ws_pool = mempool_create_slab_pool(PBLK_GEN_WS_POOL_SIZE,
pblk_blk_ws_cache); pblk_ws_cache);
if (!pblk->line_ws_pool) if (!pblk->gen_ws_pool)
goto free_page_bio_pool; goto free_page_bio_pool;
pblk->rec_pool = mempool_create_slab_pool(geo->nr_luns, pblk_rec_cache); pblk->rec_pool = mempool_create_slab_pool(geo->nr_luns, pblk_rec_cache);
if (!pblk->rec_pool) if (!pblk->rec_pool)
goto free_blk_ws_pool; goto free_gen_ws_pool;
pblk->g_rq_pool = mempool_create_slab_pool(PBLK_READ_REQ_POOL_SIZE, pblk->g_rq_pool = mempool_create_slab_pool(PBLK_READ_REQ_POOL_SIZE,
pblk_g_rq_cache); pblk_g_rq_cache);
...@@ -308,8 +308,8 @@ static int pblk_core_init(struct pblk *pblk) ...@@ -308,8 +308,8 @@ static int pblk_core_init(struct pblk *pblk)
mempool_destroy(pblk->g_rq_pool); mempool_destroy(pblk->g_rq_pool);
free_rec_pool: free_rec_pool:
mempool_destroy(pblk->rec_pool); mempool_destroy(pblk->rec_pool);
free_blk_ws_pool: free_gen_ws_pool:
mempool_destroy(pblk->line_ws_pool); mempool_destroy(pblk->gen_ws_pool);
free_page_bio_pool: free_page_bio_pool:
mempool_destroy(pblk->page_bio_pool); mempool_destroy(pblk->page_bio_pool);
return -ENOMEM; return -ENOMEM;
...@@ -324,13 +324,13 @@ static void pblk_core_free(struct pblk *pblk) ...@@ -324,13 +324,13 @@ static void pblk_core_free(struct pblk *pblk)
destroy_workqueue(pblk->bb_wq); destroy_workqueue(pblk->bb_wq);
mempool_destroy(pblk->page_bio_pool); mempool_destroy(pblk->page_bio_pool);
mempool_destroy(pblk->line_ws_pool); mempool_destroy(pblk->gen_ws_pool);
mempool_destroy(pblk->rec_pool); mempool_destroy(pblk->rec_pool);
mempool_destroy(pblk->g_rq_pool); mempool_destroy(pblk->g_rq_pool);
mempool_destroy(pblk->w_rq_pool); mempool_destroy(pblk->w_rq_pool);
mempool_destroy(pblk->line_meta_pool); mempool_destroy(pblk->line_meta_pool);
kmem_cache_destroy(pblk_blk_ws_cache); kmem_cache_destroy(pblk_ws_cache);
kmem_cache_destroy(pblk_rec_cache); kmem_cache_destroy(pblk_rec_cache);
kmem_cache_destroy(pblk_g_rq_cache); kmem_cache_destroy(pblk_g_rq_cache);
kmem_cache_destroy(pblk_w_rq_cache); kmem_cache_destroy(pblk_w_rq_cache);
......
...@@ -198,8 +198,8 @@ static void pblk_end_io_write_meta(struct nvm_rq *rqd) ...@@ -198,8 +198,8 @@ static void pblk_end_io_write_meta(struct nvm_rq *rqd)
sync = atomic_add_return(rqd->nr_ppas, &emeta->sync); sync = atomic_add_return(rqd->nr_ppas, &emeta->sync);
if (sync == emeta->nr_entries) if (sync == emeta->nr_entries)
pblk_line_run_ws(pblk, line, NULL, pblk_line_close_ws, pblk_gen_run_ws(pblk, line, NULL, pblk_line_close_ws,
pblk->close_wq); GFP_ATOMIC, pblk->close_wq);
bio_put(rqd->bio); bio_put(rqd->bio);
nvm_dev_dma_free(dev->parent, rqd->meta_list, rqd->dma_meta_list); nvm_dev_dma_free(dev->parent, rqd->meta_list, rqd->dma_meta_list);
......
...@@ -40,7 +40,6 @@ ...@@ -40,7 +40,6 @@
#define PBLK_MAX_REQ_ADDRS (64) #define PBLK_MAX_REQ_ADDRS (64)
#define PBLK_MAX_REQ_ADDRS_PW (6) #define PBLK_MAX_REQ_ADDRS_PW (6)
#define PBLK_WS_POOL_SIZE (128)
#define PBLK_META_POOL_SIZE (128) #define PBLK_META_POOL_SIZE (128)
#define PBLK_READ_REQ_POOL_SIZE (1024) #define PBLK_READ_REQ_POOL_SIZE (1024)
...@@ -61,6 +60,8 @@ ...@@ -61,6 +60,8 @@
#define ERASE 2 /* READ = 0, WRITE = 1 */ #define ERASE 2 /* READ = 0, WRITE = 1 */
#define PBLK_GEN_WS_POOL_SIZE (2)
enum { enum {
/* IO Types */ /* IO Types */
PBLK_IOTYPE_USER = 1 << 0, PBLK_IOTYPE_USER = 1 << 0,
...@@ -621,7 +622,7 @@ struct pblk { ...@@ -621,7 +622,7 @@ struct pblk {
struct list_head compl_list; struct list_head compl_list;
mempool_t *page_bio_pool; mempool_t *page_bio_pool;
mempool_t *line_ws_pool; mempool_t *gen_ws_pool;
mempool_t *rec_pool; mempool_t *rec_pool;
mempool_t *g_rq_pool; mempool_t *g_rq_pool;
mempool_t *w_rq_pool; mempool_t *w_rq_pool;
...@@ -725,8 +726,8 @@ void pblk_line_close_meta_sync(struct pblk *pblk); ...@@ -725,8 +726,8 @@ void pblk_line_close_meta_sync(struct pblk *pblk);
void pblk_line_close_ws(struct work_struct *work); void pblk_line_close_ws(struct work_struct *work);
void pblk_pipeline_stop(struct pblk *pblk); void pblk_pipeline_stop(struct pblk *pblk);
void pblk_line_mark_bb(struct work_struct *work); void pblk_line_mark_bb(struct work_struct *work);
void pblk_line_run_ws(struct pblk *pblk, struct pblk_line *line, void *priv, void pblk_gen_run_ws(struct pblk *pblk, struct pblk_line *line, void *priv,
void (*work)(struct work_struct *), void (*work)(struct work_struct *), gfp_t gfp_mask,
struct workqueue_struct *wq); struct workqueue_struct *wq);
u64 pblk_line_smeta_start(struct pblk *pblk, struct pblk_line *line); u64 pblk_line_smeta_start(struct pblk *pblk, struct pblk_line *line);
int pblk_line_read_smeta(struct pblk *pblk, struct pblk_line *line); int pblk_line_read_smeta(struct pblk *pblk, struct pblk_line *line);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment