Commit fafadcd1 authored by Christoph Hellwig's avatar Christoph Hellwig

swiotlb: don't dip into swiotlb pool for coherent allocations

All architectures that support swiotlb also have a zone that backs up
these less than full addressing allocations (usually ZONE_DMA32).

Because of that it is rather pointless to fall back to the global swiotlb
buffer if the normal dma direct allocation failed - the only thing this
will do is to eat up bounce buffers that would be more useful to serve
streaming mappings.
Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
Acked-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
Acked-by: default avatarKonrad Rzeszutek Wilk <konrad.wilk@oracle.com>
parent c4dae366
...@@ -112,7 +112,7 @@ static void *__dma_alloc(struct device *dev, size_t size, ...@@ -112,7 +112,7 @@ static void *__dma_alloc(struct device *dev, size_t size,
return addr; return addr;
} }
ptr = swiotlb_alloc(dev, size, dma_handle, flags, attrs); ptr = dma_direct_alloc_pages(dev, size, dma_handle, flags, attrs);
if (!ptr) if (!ptr)
goto no_mem; goto no_mem;
...@@ -133,7 +133,7 @@ static void *__dma_alloc(struct device *dev, size_t size, ...@@ -133,7 +133,7 @@ static void *__dma_alloc(struct device *dev, size_t size,
return coherent_ptr; return coherent_ptr;
no_map: no_map:
swiotlb_free(dev, size, ptr, *dma_handle, attrs); dma_direct_free_pages(dev, size, ptr, *dma_handle, attrs);
no_mem: no_mem:
return NULL; return NULL;
} }
...@@ -151,7 +151,7 @@ static void __dma_free(struct device *dev, size_t size, ...@@ -151,7 +151,7 @@ static void __dma_free(struct device *dev, size_t size,
return; return;
vunmap(vaddr); vunmap(vaddr);
} }
swiotlb_free(dev, size, swiotlb_addr, dma_handle, attrs); dma_direct_free_pages(dev, size, swiotlb_addr, dma_handle, attrs);
} }
static dma_addr_t __swiotlb_map_page(struct device *dev, struct page *page, static dma_addr_t __swiotlb_map_page(struct device *dev, struct page *page,
......
...@@ -67,11 +67,6 @@ extern void swiotlb_tbl_sync_single(struct device *hwdev, ...@@ -67,11 +67,6 @@ extern void swiotlb_tbl_sync_single(struct device *hwdev,
/* Accessory functions. */ /* Accessory functions. */
void *swiotlb_alloc(struct device *hwdev, size_t size, dma_addr_t *dma_handle,
gfp_t flags, unsigned long attrs);
void swiotlb_free(struct device *dev, size_t size, void *vaddr,
dma_addr_t dma_addr, unsigned long attrs);
extern dma_addr_t swiotlb_map_page(struct device *dev, struct page *page, extern dma_addr_t swiotlb_map_page(struct device *dev, struct page *page,
unsigned long offset, size_t size, unsigned long offset, size_t size,
enum dma_data_direction dir, enum dma_data_direction dir,
......
...@@ -622,78 +622,6 @@ void swiotlb_tbl_sync_single(struct device *hwdev, phys_addr_t tlb_addr, ...@@ -622,78 +622,6 @@ void swiotlb_tbl_sync_single(struct device *hwdev, phys_addr_t tlb_addr,
} }
} }
static inline bool dma_coherent_ok(struct device *dev, dma_addr_t addr,
size_t size)
{
u64 mask = DMA_BIT_MASK(32);
if (dev && dev->coherent_dma_mask)
mask = dev->coherent_dma_mask;
return addr + size - 1 <= mask;
}
static void *
swiotlb_alloc_buffer(struct device *dev, size_t size, dma_addr_t *dma_handle,
unsigned long attrs)
{
phys_addr_t phys_addr;
if (swiotlb_force == SWIOTLB_NO_FORCE)
goto out_warn;
phys_addr = swiotlb_tbl_map_single(dev,
__phys_to_dma(dev, io_tlb_start),
0, size, DMA_FROM_DEVICE, attrs);
if (phys_addr == SWIOTLB_MAP_ERROR)
goto out_warn;
*dma_handle = __phys_to_dma(dev, phys_addr);
if (!dma_coherent_ok(dev, *dma_handle, size))
goto out_unmap;
memset(phys_to_virt(phys_addr), 0, size);
return phys_to_virt(phys_addr);
out_unmap:
dev_warn(dev, "hwdev DMA mask = 0x%016Lx, dev_addr = 0x%016Lx\n",
(unsigned long long)dev->coherent_dma_mask,
(unsigned long long)*dma_handle);
/*
* DMA_TO_DEVICE to avoid memcpy in unmap_single.
* DMA_ATTR_SKIP_CPU_SYNC is optional.
*/
swiotlb_tbl_unmap_single(dev, phys_addr, size, DMA_TO_DEVICE,
DMA_ATTR_SKIP_CPU_SYNC);
out_warn:
if (!(attrs & DMA_ATTR_NO_WARN) && printk_ratelimit()) {
dev_warn(dev,
"swiotlb: coherent allocation failed, size=%zu\n",
size);
dump_stack();
}
return NULL;
}
static bool swiotlb_free_buffer(struct device *dev, size_t size,
dma_addr_t dma_addr)
{
phys_addr_t phys_addr = dma_to_phys(dev, dma_addr);
WARN_ON_ONCE(irqs_disabled());
if (!is_swiotlb_buffer(phys_addr))
return false;
/*
* DMA_TO_DEVICE to avoid memcpy in swiotlb_tbl_unmap_single.
* DMA_ATTR_SKIP_CPU_SYNC is optional.
*/
swiotlb_tbl_unmap_single(dev, phys_addr, size, DMA_TO_DEVICE,
DMA_ATTR_SKIP_CPU_SYNC);
return true;
}
static dma_addr_t swiotlb_bounce_page(struct device *dev, phys_addr_t *phys, static dma_addr_t swiotlb_bounce_page(struct device *dev, phys_addr_t *phys,
size_t size, enum dma_data_direction dir, unsigned long attrs) size_t size, enum dma_data_direction dir, unsigned long attrs)
{ {
...@@ -926,39 +854,10 @@ swiotlb_dma_supported(struct device *hwdev, u64 mask) ...@@ -926,39 +854,10 @@ swiotlb_dma_supported(struct device *hwdev, u64 mask)
return __phys_to_dma(hwdev, io_tlb_end - 1) <= mask; return __phys_to_dma(hwdev, io_tlb_end - 1) <= mask;
} }
void *swiotlb_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
gfp_t gfp, unsigned long attrs)
{
void *vaddr;
/* temporary workaround: */
if (gfp & __GFP_NOWARN)
attrs |= DMA_ATTR_NO_WARN;
/*
* Don't print a warning when the first allocation attempt fails.
* swiotlb_alloc_coherent() will print a warning when the DMA memory
* allocation ultimately failed.
*/
gfp |= __GFP_NOWARN;
vaddr = dma_direct_alloc(dev, size, dma_handle, gfp, attrs);
if (!vaddr)
vaddr = swiotlb_alloc_buffer(dev, size, dma_handle, attrs);
return vaddr;
}
void swiotlb_free(struct device *dev, size_t size, void *vaddr,
dma_addr_t dma_addr, unsigned long attrs)
{
if (!swiotlb_free_buffer(dev, size, dma_addr))
dma_direct_free(dev, size, vaddr, dma_addr, attrs);
}
const struct dma_map_ops swiotlb_dma_ops = { const struct dma_map_ops swiotlb_dma_ops = {
.mapping_error = dma_direct_mapping_error, .mapping_error = dma_direct_mapping_error,
.alloc = swiotlb_alloc, .alloc = dma_direct_alloc,
.free = swiotlb_free, .free = dma_direct_free,
.sync_single_for_cpu = swiotlb_sync_single_for_cpu, .sync_single_for_cpu = swiotlb_sync_single_for_cpu,
.sync_single_for_device = swiotlb_sync_single_for_device, .sync_single_for_device = swiotlb_sync_single_for_device,
.sync_sg_for_cpu = swiotlb_sync_sg_for_cpu, .sync_sg_for_cpu = swiotlb_sync_sg_for_cpu,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment