Commit dd4645eb authored by Dave Jiang's avatar Dave Jiang Committed by Vinod Koul

dmaengine: IOATDMA: Allocate DMA descriptor ring in contig DMA memory

Future IOATDMA hardware will take advantage of descriptors residing in
contiguous memory. Setting the descriptor ring in max config DMA memory
of 2MB. Each channel will need 2 of these chunks. This should provide 64k
of 64B descriptors.
Signed-off-by: default avatarDave Jiang <dave.jiang@intel.com>
Signed-off-by: default avatarVinod Koul <vinod.koul@intel.com>
parent cd60cd96
...@@ -31,6 +31,7 @@ ...@@ -31,6 +31,7 @@
#include <linux/dma-mapping.h> #include <linux/dma-mapping.h>
#include <linux/workqueue.h> #include <linux/workqueue.h>
#include <linux/prefetch.h> #include <linux/prefetch.h>
#include <linux/sizes.h>
#include "dma.h" #include "dma.h"
#include "registers.h" #include "registers.h"
#include "hw.h" #include "hw.h"
...@@ -290,24 +291,30 @@ static dma_cookie_t ioat_tx_submit_unlock(struct dma_async_tx_descriptor *tx) ...@@ -290,24 +291,30 @@ static dma_cookie_t ioat_tx_submit_unlock(struct dma_async_tx_descriptor *tx)
} }
static struct ioat_ring_ent * static struct ioat_ring_ent *
ioat_alloc_ring_ent(struct dma_chan *chan, gfp_t flags) ioat_alloc_ring_ent(struct dma_chan *chan, int idx, gfp_t flags)
{ {
struct ioat_dma_descriptor *hw; struct ioat_dma_descriptor *hw;
struct ioat_ring_ent *desc; struct ioat_ring_ent *desc;
struct ioatdma_device *ioat_dma; struct ioatdma_device *ioat_dma;
struct ioatdma_chan *ioat_chan = to_ioat_chan(chan);
int chunk;
dma_addr_t phys; dma_addr_t phys;
u8 *pos;
off_t offs;
ioat_dma = to_ioatdma_device(chan->device); ioat_dma = to_ioatdma_device(chan->device);
hw = dma_pool_alloc(ioat_dma->dma_pool, flags, &phys);
if (!hw) chunk = idx / IOAT_DESCS_PER_2M;
return NULL; idx &= (IOAT_DESCS_PER_2M - 1);
offs = idx * IOAT_DESC_SZ;
pos = (u8 *)ioat_chan->descs[chunk].virt + offs;
phys = ioat_chan->descs[chunk].hw + offs;
hw = (struct ioat_dma_descriptor *)pos;
memset(hw, 0, sizeof(*hw)); memset(hw, 0, sizeof(*hw));
desc = kmem_cache_zalloc(ioat_cache, flags); desc = kmem_cache_zalloc(ioat_cache, flags);
if (!desc) { if (!desc)
dma_pool_free(ioat_dma->dma_pool, hw, phys);
return NULL; return NULL;
}
dma_async_tx_descriptor_init(&desc->txd, chan); dma_async_tx_descriptor_init(&desc->txd, chan);
desc->txd.tx_submit = ioat_tx_submit_unlock; desc->txd.tx_submit = ioat_tx_submit_unlock;
...@@ -318,29 +325,63 @@ ioat_alloc_ring_ent(struct dma_chan *chan, gfp_t flags) ...@@ -318,29 +325,63 @@ ioat_alloc_ring_ent(struct dma_chan *chan, gfp_t flags)
void ioat_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan) void ioat_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan)
{ {
struct ioatdma_device *ioat_dma;
ioat_dma = to_ioatdma_device(chan->device);
dma_pool_free(ioat_dma->dma_pool, desc->hw, desc->txd.phys);
kmem_cache_free(ioat_cache, desc); kmem_cache_free(ioat_cache, desc);
} }
struct ioat_ring_ent ** struct ioat_ring_ent **
ioat_alloc_ring(struct dma_chan *c, int order, gfp_t flags) ioat_alloc_ring(struct dma_chan *c, int order, gfp_t flags)
{ {
struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
struct ioat_ring_ent **ring; struct ioat_ring_ent **ring;
int descs = 1 << order; int total_descs = 1 << order;
int i; int i, chunks;
/* allocate the array to hold the software ring */ /* allocate the array to hold the software ring */
ring = kcalloc(descs, sizeof(*ring), flags); ring = kcalloc(total_descs, sizeof(*ring), flags);
if (!ring) if (!ring)
return NULL; return NULL;
for (i = 0; i < descs; i++) {
ring[i] = ioat_alloc_ring_ent(c, flags); ioat_chan->desc_chunks = chunks = (total_descs * IOAT_DESC_SZ) / SZ_2M;
for (i = 0; i < chunks; i++) {
struct ioat_descs *descs = &ioat_chan->descs[i];
descs->virt = dma_alloc_coherent(to_dev(ioat_chan),
SZ_2M, &descs->hw, flags);
if (!descs->virt && (i > 0)) {
int idx;
for (idx = 0; idx < i; idx++) {
dma_free_coherent(to_dev(ioat_chan), SZ_2M,
descs->virt, descs->hw);
descs->virt = NULL;
descs->hw = 0;
}
ioat_chan->desc_chunks = 0;
kfree(ring);
return NULL;
}
}
for (i = 0; i < total_descs; i++) {
ring[i] = ioat_alloc_ring_ent(c, i, flags);
if (!ring[i]) { if (!ring[i]) {
int idx;
while (i--) while (i--)
ioat_free_ring_ent(ring[i], c); ioat_free_ring_ent(ring[i], c);
for (idx = 0; idx < ioat_chan->desc_chunks; idx++) {
dma_free_coherent(to_dev(ioat_chan),
SZ_2M,
ioat_chan->descs[idx].virt,
ioat_chan->descs[idx].hw);
ioat_chan->descs[idx].virt = NULL;
ioat_chan->descs[idx].hw = 0;
}
ioat_chan->desc_chunks = 0;
kfree(ring); kfree(ring);
return NULL; return NULL;
} }
...@@ -348,7 +389,7 @@ ioat_alloc_ring(struct dma_chan *c, int order, gfp_t flags) ...@@ -348,7 +389,7 @@ ioat_alloc_ring(struct dma_chan *c, int order, gfp_t flags)
} }
/* link descs */ /* link descs */
for (i = 0; i < descs-1; i++) { for (i = 0; i < total_descs-1; i++) {
struct ioat_ring_ent *next = ring[i+1]; struct ioat_ring_ent *next = ring[i+1];
struct ioat_dma_descriptor *hw = ring[i]->hw; struct ioat_dma_descriptor *hw = ring[i]->hw;
......
...@@ -62,7 +62,6 @@ enum ioat_irq_mode { ...@@ -62,7 +62,6 @@ enum ioat_irq_mode {
* struct ioatdma_device - internal representation of a IOAT device * struct ioatdma_device - internal representation of a IOAT device
* @pdev: PCI-Express device * @pdev: PCI-Express device
* @reg_base: MMIO register space base address * @reg_base: MMIO register space base address
* @dma_pool: for allocating DMA descriptors
* @completion_pool: DMA buffers for completion ops * @completion_pool: DMA buffers for completion ops
* @sed_hw_pool: DMA super descriptor pools * @sed_hw_pool: DMA super descriptor pools
* @dma_dev: embedded struct dma_device * @dma_dev: embedded struct dma_device
...@@ -76,7 +75,6 @@ enum ioat_irq_mode { ...@@ -76,7 +75,6 @@ enum ioat_irq_mode {
struct ioatdma_device { struct ioatdma_device {
struct pci_dev *pdev; struct pci_dev *pdev;
void __iomem *reg_base; void __iomem *reg_base;
struct dma_pool *dma_pool;
struct dma_pool *completion_pool; struct dma_pool *completion_pool;
#define MAX_SED_POOLS 5 #define MAX_SED_POOLS 5
struct dma_pool *sed_hw_pool[MAX_SED_POOLS]; struct dma_pool *sed_hw_pool[MAX_SED_POOLS];
...@@ -90,6 +88,11 @@ struct ioatdma_device { ...@@ -90,6 +88,11 @@ struct ioatdma_device {
u32 cap; u32 cap;
}; };
struct ioat_descs {
void *virt;
dma_addr_t hw;
};
struct ioatdma_chan { struct ioatdma_chan {
struct dma_chan dma_chan; struct dma_chan dma_chan;
void __iomem *reg_base; void __iomem *reg_base;
...@@ -132,6 +135,8 @@ struct ioatdma_chan { ...@@ -132,6 +135,8 @@ struct ioatdma_chan {
u16 produce; u16 produce;
struct ioat_ring_ent **ring; struct ioat_ring_ent **ring;
spinlock_t prep_lock; spinlock_t prep_lock;
struct ioat_descs descs[2];
int desc_chunks;
}; };
struct ioat_sysfs_entry { struct ioat_sysfs_entry {
...@@ -301,6 +306,8 @@ static inline bool is_ioat_bug(unsigned long err) ...@@ -301,6 +306,8 @@ static inline bool is_ioat_bug(unsigned long err)
} }
#define IOAT_MAX_ORDER 16 #define IOAT_MAX_ORDER 16
#define IOAT_MAX_DESCS 65536
#define IOAT_DESCS_PER_2M 32768
static inline u32 ioat_ring_size(struct ioatdma_chan *ioat_chan) static inline u32 ioat_ring_size(struct ioatdma_chan *ioat_chan)
{ {
......
...@@ -73,6 +73,8 @@ ...@@ -73,6 +73,8 @@
int system_has_dca_enabled(struct pci_dev *pdev); int system_has_dca_enabled(struct pci_dev *pdev);
#define IOAT_DESC_SZ 64
struct ioat_dma_descriptor { struct ioat_dma_descriptor {
uint32_t size; uint32_t size;
union { union {
......
...@@ -28,6 +28,7 @@ ...@@ -28,6 +28,7 @@
#include <linux/prefetch.h> #include <linux/prefetch.h>
#include <linux/dca.h> #include <linux/dca.h>
#include <linux/aer.h> #include <linux/aer.h>
#include <linux/sizes.h>
#include "dma.h" #include "dma.h"
#include "registers.h" #include "registers.h"
#include "hw.h" #include "hw.h"
...@@ -496,15 +497,6 @@ static int ioat_probe(struct ioatdma_device *ioat_dma) ...@@ -496,15 +497,6 @@ static int ioat_probe(struct ioatdma_device *ioat_dma)
struct pci_dev *pdev = ioat_dma->pdev; struct pci_dev *pdev = ioat_dma->pdev;
struct device *dev = &pdev->dev; struct device *dev = &pdev->dev;
/* DMA coherent memory pool for DMA descriptor allocations */
ioat_dma->dma_pool = dma_pool_create("dma_desc_pool", dev,
sizeof(struct ioat_dma_descriptor),
64, 0);
if (!ioat_dma->dma_pool) {
err = -ENOMEM;
goto err_dma_pool;
}
ioat_dma->completion_pool = dma_pool_create("completion_pool", dev, ioat_dma->completion_pool = dma_pool_create("completion_pool", dev,
sizeof(u64), sizeof(u64),
SMP_CACHE_BYTES, SMP_CACHE_BYTES,
...@@ -512,7 +504,7 @@ static int ioat_probe(struct ioatdma_device *ioat_dma) ...@@ -512,7 +504,7 @@ static int ioat_probe(struct ioatdma_device *ioat_dma)
if (!ioat_dma->completion_pool) { if (!ioat_dma->completion_pool) {
err = -ENOMEM; err = -ENOMEM;
goto err_completion_pool; goto err_out;
} }
ioat_enumerate_channels(ioat_dma); ioat_enumerate_channels(ioat_dma);
...@@ -539,9 +531,7 @@ static int ioat_probe(struct ioatdma_device *ioat_dma) ...@@ -539,9 +531,7 @@ static int ioat_probe(struct ioatdma_device *ioat_dma)
ioat_disable_interrupts(ioat_dma); ioat_disable_interrupts(ioat_dma);
err_setup_interrupts: err_setup_interrupts:
dma_pool_destroy(ioat_dma->completion_pool); dma_pool_destroy(ioat_dma->completion_pool);
err_completion_pool: err_out:
dma_pool_destroy(ioat_dma->dma_pool);
err_dma_pool:
return err; return err;
} }
...@@ -552,7 +542,6 @@ static int ioat_register(struct ioatdma_device *ioat_dma) ...@@ -552,7 +542,6 @@ static int ioat_register(struct ioatdma_device *ioat_dma)
if (err) { if (err) {
ioat_disable_interrupts(ioat_dma); ioat_disable_interrupts(ioat_dma);
dma_pool_destroy(ioat_dma->completion_pool); dma_pool_destroy(ioat_dma->completion_pool);
dma_pool_destroy(ioat_dma->dma_pool);
} }
return err; return err;
...@@ -568,7 +557,6 @@ static void ioat_dma_remove(struct ioatdma_device *ioat_dma) ...@@ -568,7 +557,6 @@ static void ioat_dma_remove(struct ioatdma_device *ioat_dma)
dma_async_device_unregister(dma); dma_async_device_unregister(dma);
dma_pool_destroy(ioat_dma->dma_pool);
dma_pool_destroy(ioat_dma->completion_pool); dma_pool_destroy(ioat_dma->completion_pool);
INIT_LIST_HEAD(&dma->channels); INIT_LIST_HEAD(&dma->channels);
...@@ -658,6 +646,15 @@ static void ioat_free_chan_resources(struct dma_chan *c) ...@@ -658,6 +646,15 @@ static void ioat_free_chan_resources(struct dma_chan *c)
ioat_free_ring_ent(desc, c); ioat_free_ring_ent(desc, c);
} }
for (i = 0; i < ioat_chan->desc_chunks; i++) {
dma_free_coherent(to_dev(ioat_chan), SZ_2M,
ioat_chan->descs[i].virt,
ioat_chan->descs[i].hw);
ioat_chan->descs[i].virt = NULL;
ioat_chan->descs[i].hw = 0;
}
ioat_chan->desc_chunks = 0;
kfree(ioat_chan->ring); kfree(ioat_chan->ring);
ioat_chan->ring = NULL; ioat_chan->ring = NULL;
ioat_chan->alloc_order = 0; ioat_chan->alloc_order = 0;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment