Commit b74e58cd authored by Jens Axboe's avatar Jens Axboe

Merge branch 'nvme-5.6' of git://git.infradead.org/nvme into block-5.6

Pull NVMe fixes from Keith.

* 'nvme-5.6' of git://git.infradead.org/nvme:
  nvmet: update AEN list and array at one place
  nvmet: Fix controller use after free
  nvmet: Fix error print message at nvmet_install_queue function
  nvme-pci: remove nvmeq->tags
  nvmet: fix dsm failure when payload does not match sgl descriptor
  nvmet: Pass lockdep expression to RCU lists
parents c8ab4225 0f5be6a4
...@@ -167,7 +167,6 @@ struct nvme_queue { ...@@ -167,7 +167,6 @@ struct nvme_queue {
/* only used for poll queues: */ /* only used for poll queues: */
spinlock_t cq_poll_lock ____cacheline_aligned_in_smp; spinlock_t cq_poll_lock ____cacheline_aligned_in_smp;
volatile struct nvme_completion *cqes; volatile struct nvme_completion *cqes;
struct blk_mq_tags **tags;
dma_addr_t sq_dma_addr; dma_addr_t sq_dma_addr;
dma_addr_t cq_dma_addr; dma_addr_t cq_dma_addr;
u32 __iomem *q_db; u32 __iomem *q_db;
...@@ -376,29 +375,17 @@ static int nvme_admin_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, ...@@ -376,29 +375,17 @@ static int nvme_admin_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
WARN_ON(hctx_idx != 0); WARN_ON(hctx_idx != 0);
WARN_ON(dev->admin_tagset.tags[0] != hctx->tags); WARN_ON(dev->admin_tagset.tags[0] != hctx->tags);
WARN_ON(nvmeq->tags);
hctx->driver_data = nvmeq; hctx->driver_data = nvmeq;
nvmeq->tags = &dev->admin_tagset.tags[0];
return 0; return 0;
} }
static void nvme_admin_exit_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
{
struct nvme_queue *nvmeq = hctx->driver_data;
nvmeq->tags = NULL;
}
static int nvme_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, static int nvme_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
unsigned int hctx_idx) unsigned int hctx_idx)
{ {
struct nvme_dev *dev = data; struct nvme_dev *dev = data;
struct nvme_queue *nvmeq = &dev->queues[hctx_idx + 1]; struct nvme_queue *nvmeq = &dev->queues[hctx_idx + 1];
if (!nvmeq->tags)
nvmeq->tags = &dev->tagset.tags[hctx_idx];
WARN_ON(dev->tagset.tags[hctx_idx] != hctx->tags); WARN_ON(dev->tagset.tags[hctx_idx] != hctx->tags);
hctx->driver_data = nvmeq; hctx->driver_data = nvmeq;
return 0; return 0;
...@@ -948,6 +935,13 @@ static inline void nvme_ring_cq_doorbell(struct nvme_queue *nvmeq) ...@@ -948,6 +935,13 @@ static inline void nvme_ring_cq_doorbell(struct nvme_queue *nvmeq)
writel(head, nvmeq->q_db + nvmeq->dev->db_stride); writel(head, nvmeq->q_db + nvmeq->dev->db_stride);
} }
static inline struct blk_mq_tags *nvme_queue_tagset(struct nvme_queue *nvmeq)
{
if (!nvmeq->qid)
return nvmeq->dev->admin_tagset.tags[0];
return nvmeq->dev->tagset.tags[nvmeq->qid - 1];
}
static inline void nvme_handle_cqe(struct nvme_queue *nvmeq, u16 idx) static inline void nvme_handle_cqe(struct nvme_queue *nvmeq, u16 idx)
{ {
volatile struct nvme_completion *cqe = &nvmeq->cqes[idx]; volatile struct nvme_completion *cqe = &nvmeq->cqes[idx];
...@@ -972,7 +966,7 @@ static inline void nvme_handle_cqe(struct nvme_queue *nvmeq, u16 idx) ...@@ -972,7 +966,7 @@ static inline void nvme_handle_cqe(struct nvme_queue *nvmeq, u16 idx)
return; return;
} }
req = blk_mq_tag_to_rq(*nvmeq->tags, cqe->command_id); req = blk_mq_tag_to_rq(nvme_queue_tagset(nvmeq), cqe->command_id);
trace_nvme_sq(req, cqe->sq_head, nvmeq->sq_tail); trace_nvme_sq(req, cqe->sq_head, nvmeq->sq_tail);
nvme_end_request(req, cqe->status, cqe->result); nvme_end_request(req, cqe->status, cqe->result);
} }
...@@ -1572,7 +1566,6 @@ static const struct blk_mq_ops nvme_mq_admin_ops = { ...@@ -1572,7 +1566,6 @@ static const struct blk_mq_ops nvme_mq_admin_ops = {
.queue_rq = nvme_queue_rq, .queue_rq = nvme_queue_rq,
.complete = nvme_pci_complete_rq, .complete = nvme_pci_complete_rq,
.init_hctx = nvme_admin_init_hctx, .init_hctx = nvme_admin_init_hctx,
.exit_hctx = nvme_admin_exit_hctx,
.init_request = nvme_init_request, .init_request = nvme_init_request,
.timeout = nvme_timeout, .timeout = nvme_timeout,
}; };
......
...@@ -129,48 +129,52 @@ static u32 nvmet_async_event_result(struct nvmet_async_event *aen) ...@@ -129,48 +129,52 @@ static u32 nvmet_async_event_result(struct nvmet_async_event *aen)
return aen->event_type | (aen->event_info << 8) | (aen->log_page << 16); return aen->event_type | (aen->event_info << 8) | (aen->log_page << 16);
} }
static void nvmet_async_events_free(struct nvmet_ctrl *ctrl) static void nvmet_async_events_process(struct nvmet_ctrl *ctrl, u16 status)
{ {
struct nvmet_async_event *aen;
struct nvmet_req *req; struct nvmet_req *req;
while (1) { while (1) {
mutex_lock(&ctrl->lock); mutex_lock(&ctrl->lock);
if (!ctrl->nr_async_event_cmds) { aen = list_first_entry_or_null(&ctrl->async_events,
struct nvmet_async_event, entry);
if (!aen || !ctrl->nr_async_event_cmds) {
mutex_unlock(&ctrl->lock); mutex_unlock(&ctrl->lock);
return; break;
} }
req = ctrl->async_event_cmds[--ctrl->nr_async_event_cmds]; req = ctrl->async_event_cmds[--ctrl->nr_async_event_cmds];
if (status == 0)
nvmet_set_result(req, nvmet_async_event_result(aen));
list_del(&aen->entry);
kfree(aen);
mutex_unlock(&ctrl->lock); mutex_unlock(&ctrl->lock);
nvmet_req_complete(req, NVME_SC_INTERNAL | NVME_SC_DNR); nvmet_req_complete(req, status);
} }
} }
static void nvmet_async_event_work(struct work_struct *work) static void nvmet_async_events_free(struct nvmet_ctrl *ctrl)
{ {
struct nvmet_ctrl *ctrl =
container_of(work, struct nvmet_ctrl, async_event_work);
struct nvmet_async_event *aen;
struct nvmet_req *req; struct nvmet_req *req;
while (1) {
mutex_lock(&ctrl->lock); mutex_lock(&ctrl->lock);
aen = list_first_entry_or_null(&ctrl->async_events, while (ctrl->nr_async_event_cmds) {
struct nvmet_async_event, entry); req = ctrl->async_event_cmds[--ctrl->nr_async_event_cmds];
if (!aen || !ctrl->nr_async_event_cmds) {
mutex_unlock(&ctrl->lock); mutex_unlock(&ctrl->lock);
return; nvmet_req_complete(req, NVME_SC_INTERNAL | NVME_SC_DNR);
mutex_lock(&ctrl->lock);
} }
mutex_unlock(&ctrl->lock);
}
req = ctrl->async_event_cmds[--ctrl->nr_async_event_cmds]; static void nvmet_async_event_work(struct work_struct *work)
nvmet_set_result(req, nvmet_async_event_result(aen)); {
struct nvmet_ctrl *ctrl =
list_del(&aen->entry); container_of(work, struct nvmet_ctrl, async_event_work);
kfree(aen);
mutex_unlock(&ctrl->lock); nvmet_async_events_process(ctrl, 0);
nvmet_req_complete(req, 0);
}
} }
void nvmet_add_async_event(struct nvmet_ctrl *ctrl, u8 event_type, void nvmet_add_async_event(struct nvmet_ctrl *ctrl, u8 event_type,
...@@ -555,7 +559,8 @@ int nvmet_ns_enable(struct nvmet_ns *ns) ...@@ -555,7 +559,8 @@ int nvmet_ns_enable(struct nvmet_ns *ns)
} else { } else {
struct nvmet_ns *old; struct nvmet_ns *old;
list_for_each_entry_rcu(old, &subsys->namespaces, dev_link) { list_for_each_entry_rcu(old, &subsys->namespaces, dev_link,
lockdep_is_held(&subsys->lock)) {
BUG_ON(ns->nsid == old->nsid); BUG_ON(ns->nsid == old->nsid);
if (ns->nsid < old->nsid) if (ns->nsid < old->nsid)
break; break;
...@@ -752,19 +757,24 @@ static void nvmet_confirm_sq(struct percpu_ref *ref) ...@@ -752,19 +757,24 @@ static void nvmet_confirm_sq(struct percpu_ref *ref)
void nvmet_sq_destroy(struct nvmet_sq *sq) void nvmet_sq_destroy(struct nvmet_sq *sq)
{ {
u16 status = NVME_SC_INTERNAL | NVME_SC_DNR;
struct nvmet_ctrl *ctrl = sq->ctrl;
/* /*
* If this is the admin queue, complete all AERs so that our * If this is the admin queue, complete all AERs so that our
* queue doesn't have outstanding requests on it. * queue doesn't have outstanding requests on it.
*/ */
if (sq->ctrl && sq->ctrl->sqs && sq->ctrl->sqs[0] == sq) if (ctrl && ctrl->sqs && ctrl->sqs[0] == sq) {
nvmet_async_events_free(sq->ctrl); nvmet_async_events_process(ctrl, status);
nvmet_async_events_free(ctrl);
}
percpu_ref_kill_and_confirm(&sq->ref, nvmet_confirm_sq); percpu_ref_kill_and_confirm(&sq->ref, nvmet_confirm_sq);
wait_for_completion(&sq->confirm_done); wait_for_completion(&sq->confirm_done);
wait_for_completion(&sq->free_done); wait_for_completion(&sq->free_done);
percpu_ref_exit(&sq->ref); percpu_ref_exit(&sq->ref);
if (sq->ctrl) { if (ctrl) {
nvmet_ctrl_put(sq->ctrl); nvmet_ctrl_put(ctrl);
sq->ctrl = NULL; /* allows reusing the queue later */ sq->ctrl = NULL; /* allows reusing the queue later */
} }
} }
...@@ -938,6 +948,17 @@ bool nvmet_check_data_len(struct nvmet_req *req, size_t data_len) ...@@ -938,6 +948,17 @@ bool nvmet_check_data_len(struct nvmet_req *req, size_t data_len)
} }
EXPORT_SYMBOL_GPL(nvmet_check_data_len); EXPORT_SYMBOL_GPL(nvmet_check_data_len);
bool nvmet_check_data_len_lte(struct nvmet_req *req, size_t data_len)
{
if (unlikely(data_len > req->transfer_len)) {
req->error_loc = offsetof(struct nvme_common_command, dptr);
nvmet_req_complete(req, NVME_SC_SGL_INVALID_DATA | NVME_SC_DNR);
return false;
}
return true;
}
int nvmet_req_alloc_sgl(struct nvmet_req *req) int nvmet_req_alloc_sgl(struct nvmet_req *req)
{ {
struct pci_dev *p2p_dev = NULL; struct pci_dev *p2p_dev = NULL;
...@@ -1172,7 +1193,8 @@ static void nvmet_setup_p2p_ns_map(struct nvmet_ctrl *ctrl, ...@@ -1172,7 +1193,8 @@ static void nvmet_setup_p2p_ns_map(struct nvmet_ctrl *ctrl,
ctrl->p2p_client = get_device(req->p2p_client); ctrl->p2p_client = get_device(req->p2p_client);
list_for_each_entry_rcu(ns, &ctrl->subsys->namespaces, dev_link) list_for_each_entry_rcu(ns, &ctrl->subsys->namespaces, dev_link,
lockdep_is_held(&ctrl->subsys->lock))
nvmet_p2pmem_ns_add_p2p(ctrl, ns); nvmet_p2pmem_ns_add_p2p(ctrl, ns);
} }
......
...@@ -109,6 +109,7 @@ static u16 nvmet_install_queue(struct nvmet_ctrl *ctrl, struct nvmet_req *req) ...@@ -109,6 +109,7 @@ static u16 nvmet_install_queue(struct nvmet_ctrl *ctrl, struct nvmet_req *req)
u16 qid = le16_to_cpu(c->qid); u16 qid = le16_to_cpu(c->qid);
u16 sqsize = le16_to_cpu(c->sqsize); u16 sqsize = le16_to_cpu(c->sqsize);
struct nvmet_ctrl *old; struct nvmet_ctrl *old;
u16 ret;
old = cmpxchg(&req->sq->ctrl, NULL, ctrl); old = cmpxchg(&req->sq->ctrl, NULL, ctrl);
if (old) { if (old) {
...@@ -119,7 +120,8 @@ static u16 nvmet_install_queue(struct nvmet_ctrl *ctrl, struct nvmet_req *req) ...@@ -119,7 +120,8 @@ static u16 nvmet_install_queue(struct nvmet_ctrl *ctrl, struct nvmet_req *req)
if (!sqsize) { if (!sqsize) {
pr_warn("queue size zero!\n"); pr_warn("queue size zero!\n");
req->error_loc = offsetof(struct nvmf_connect_command, sqsize); req->error_loc = offsetof(struct nvmf_connect_command, sqsize);
return NVME_SC_CONNECT_INVALID_PARAM | NVME_SC_DNR; ret = NVME_SC_CONNECT_INVALID_PARAM | NVME_SC_DNR;
goto err;
} }
/* note: convert queue size from 0's-based value to 1's-based value */ /* note: convert queue size from 0's-based value to 1's-based value */
...@@ -132,16 +134,19 @@ static u16 nvmet_install_queue(struct nvmet_ctrl *ctrl, struct nvmet_req *req) ...@@ -132,16 +134,19 @@ static u16 nvmet_install_queue(struct nvmet_ctrl *ctrl, struct nvmet_req *req)
} }
if (ctrl->ops->install_queue) { if (ctrl->ops->install_queue) {
u16 ret = ctrl->ops->install_queue(req->sq); ret = ctrl->ops->install_queue(req->sq);
if (ret) { if (ret) {
pr_err("failed to install queue %d cntlid %d ret %x\n", pr_err("failed to install queue %d cntlid %d ret %x\n",
qid, ret, ctrl->cntlid); qid, ctrl->cntlid, ret);
return ret; goto err;
} }
} }
return 0; return 0;
err:
req->sq->ctrl = NULL;
return ret;
} }
static void nvmet_execute_admin_connect(struct nvmet_req *req) static void nvmet_execute_admin_connect(struct nvmet_req *req)
......
...@@ -280,7 +280,7 @@ static void nvmet_bdev_execute_discard(struct nvmet_req *req) ...@@ -280,7 +280,7 @@ static void nvmet_bdev_execute_discard(struct nvmet_req *req)
static void nvmet_bdev_execute_dsm(struct nvmet_req *req) static void nvmet_bdev_execute_dsm(struct nvmet_req *req)
{ {
if (!nvmet_check_data_len(req, nvmet_dsm_len(req))) if (!nvmet_check_data_len_lte(req, nvmet_dsm_len(req)))
return; return;
switch (le32_to_cpu(req->cmd->dsm.attributes)) { switch (le32_to_cpu(req->cmd->dsm.attributes)) {
......
...@@ -336,7 +336,7 @@ static void nvmet_file_dsm_work(struct work_struct *w) ...@@ -336,7 +336,7 @@ static void nvmet_file_dsm_work(struct work_struct *w)
static void nvmet_file_execute_dsm(struct nvmet_req *req) static void nvmet_file_execute_dsm(struct nvmet_req *req)
{ {
if (!nvmet_check_data_len(req, nvmet_dsm_len(req))) if (!nvmet_check_data_len_lte(req, nvmet_dsm_len(req)))
return; return;
INIT_WORK(&req->f.work, nvmet_file_dsm_work); INIT_WORK(&req->f.work, nvmet_file_dsm_work);
schedule_work(&req->f.work); schedule_work(&req->f.work);
......
...@@ -374,6 +374,7 @@ bool nvmet_req_init(struct nvmet_req *req, struct nvmet_cq *cq, ...@@ -374,6 +374,7 @@ bool nvmet_req_init(struct nvmet_req *req, struct nvmet_cq *cq,
struct nvmet_sq *sq, const struct nvmet_fabrics_ops *ops); struct nvmet_sq *sq, const struct nvmet_fabrics_ops *ops);
void nvmet_req_uninit(struct nvmet_req *req); void nvmet_req_uninit(struct nvmet_req *req);
bool nvmet_check_data_len(struct nvmet_req *req, size_t data_len); bool nvmet_check_data_len(struct nvmet_req *req, size_t data_len);
bool nvmet_check_data_len_lte(struct nvmet_req *req, size_t data_len);
void nvmet_req_complete(struct nvmet_req *req, u16 status); void nvmet_req_complete(struct nvmet_req *req, u16 status);
int nvmet_req_alloc_sgl(struct nvmet_req *req); int nvmet_req_alloc_sgl(struct nvmet_req *req);
void nvmet_req_free_sgl(struct nvmet_req *req); void nvmet_req_free_sgl(struct nvmet_req *req);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment