Commit 74943d45 authored by Keith Busch's avatar Keith Busch Committed by Jens Axboe

nvme-pci: remove volatile cqes

The completion queue entry is not volatile once the phase is confirmed.
Remove the volatile keywords and check the phase using the appropriate
READ_ONCE() accessor, allowing the compiler to optimize the remaining
completion path.
Signed-off-by: default avatarKeith Busch <kbusch@kernel.org>
Reviewed-by: default avatarSagi Grimberg <sagi@grimberg.me>
Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent b04df85d
...@@ -166,7 +166,7 @@ struct nvme_queue { ...@@ -166,7 +166,7 @@ struct nvme_queue {
void *sq_cmds; void *sq_cmds;
/* only used for poll queues: */ /* only used for poll queues: */
spinlock_t cq_poll_lock ____cacheline_aligned_in_smp; spinlock_t cq_poll_lock ____cacheline_aligned_in_smp;
volatile struct nvme_completion *cqes; struct nvme_completion *cqes;
dma_addr_t sq_dma_addr; dma_addr_t sq_dma_addr;
dma_addr_t cq_dma_addr; dma_addr_t cq_dma_addr;
u32 __iomem *q_db; u32 __iomem *q_db;
...@@ -922,8 +922,9 @@ static void nvme_pci_complete_rq(struct request *req) ...@@ -922,8 +922,9 @@ static void nvme_pci_complete_rq(struct request *req)
/* We read the CQE phase first to check if the rest of the entry is valid */ /* We read the CQE phase first to check if the rest of the entry is valid */
static inline bool nvme_cqe_pending(struct nvme_queue *nvmeq) static inline bool nvme_cqe_pending(struct nvme_queue *nvmeq)
{ {
return (le16_to_cpu(nvmeq->cqes[nvmeq->cq_head].status) & 1) == struct nvme_completion *hcqe = &nvmeq->cqes[nvmeq->cq_head];
nvmeq->cq_phase;
return (le16_to_cpu(READ_ONCE(hcqe->status)) & 1) == nvmeq->cq_phase;
} }
static inline void nvme_ring_cq_doorbell(struct nvme_queue *nvmeq) static inline void nvme_ring_cq_doorbell(struct nvme_queue *nvmeq)
...@@ -944,7 +945,7 @@ static inline struct blk_mq_tags *nvme_queue_tagset(struct nvme_queue *nvmeq) ...@@ -944,7 +945,7 @@ static inline struct blk_mq_tags *nvme_queue_tagset(struct nvme_queue *nvmeq)
static inline void nvme_handle_cqe(struct nvme_queue *nvmeq, u16 idx) static inline void nvme_handle_cqe(struct nvme_queue *nvmeq, u16 idx)
{ {
volatile struct nvme_completion *cqe = &nvmeq->cqes[idx]; struct nvme_completion *cqe = &nvmeq->cqes[idx];
struct request *req; struct request *req;
if (unlikely(cqe->command_id >= nvmeq->q_depth)) { if (unlikely(cqe->command_id >= nvmeq->q_depth)) {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment