Commit ac60602a authored by Linus Torvalds's avatar Linus Torvalds

Merge tag 'dma-mapping-5.3-1' of git://git.infradead.org/users/hch/dma-mapping

Pull dma-mapping fixes from Christoph Hellwig:
 "Fix various regressions:

   - force unencrypted dma-coherent buffers if encryption bit can't fit
     into the dma coherent mask (Tom Lendacky)

   - avoid limiting request size if swiotlb is not used (me)

   - fix swiotlb handling in dma_direct_sync_sg_for_cpu/device (Fugang
     Duan)"

* tag 'dma-mapping-5.3-1' of git://git.infradead.org/users/hch/dma-mapping:
  dma-direct: correct the physical addr in dma_direct_sync_sg_for_cpu/device
  dma-direct: only limit the mapping size if swiotlb could be used
  dma-mapping: add a dma_addressing_limited helper
  dma-direct: Force unencrypted DMA under SME for certain DMA masks
parents c6dd78fc 449fa54d
...@@ -189,6 +189,7 @@ config S390 ...@@ -189,6 +189,7 @@ config S390
select VIRT_CPU_ACCOUNTING select VIRT_CPU_ACCOUNTING
select ARCH_HAS_SCALED_CPUTIME select ARCH_HAS_SCALED_CPUTIME
select HAVE_NMI select HAVE_NMI
select ARCH_HAS_FORCE_DMA_UNENCRYPTED
select SWIOTLB select SWIOTLB
select GENERIC_ALLOCATOR select GENERIC_ALLOCATOR
......
...@@ -30,7 +30,7 @@ ...@@ -30,7 +30,7 @@
#include <linux/export.h> #include <linux/export.h>
#include <linux/cma.h> #include <linux/cma.h>
#include <linux/gfp.h> #include <linux/gfp.h>
#include <linux/dma-mapping.h> #include <linux/dma-direct.h>
#include <asm/processor.h> #include <asm/processor.h>
#include <linux/uaccess.h> #include <linux/uaccess.h>
#include <asm/pgtable.h> #include <asm/pgtable.h>
...@@ -161,6 +161,11 @@ bool sev_active(void) ...@@ -161,6 +161,11 @@ bool sev_active(void)
return is_prot_virt_guest(); return is_prot_virt_guest();
} }
bool force_dma_unencrypted(struct device *dev)
{
return sev_active();
}
/* protected virtualization */ /* protected virtualization */
static void pv_init(void) static void pv_init(void)
{ {
......
...@@ -1526,6 +1526,7 @@ config AMD_MEM_ENCRYPT ...@@ -1526,6 +1526,7 @@ config AMD_MEM_ENCRYPT
depends on X86_64 && CPU_SUP_AMD depends on X86_64 && CPU_SUP_AMD
select DYNAMIC_PHYSICAL_MASK select DYNAMIC_PHYSICAL_MASK
select ARCH_USE_MEMREMAP_PROT select ARCH_USE_MEMREMAP_PROT
select ARCH_HAS_FORCE_DMA_UNENCRYPTED
---help--- ---help---
Say yes to enable support for the encryption of system memory. Say yes to enable support for the encryption of system memory.
This requires an AMD processor that supports Secure Memory This requires an AMD processor that supports Secure Memory
......
...@@ -15,6 +15,10 @@ ...@@ -15,6 +15,10 @@
#include <linux/dma-direct.h> #include <linux/dma-direct.h>
#include <linux/swiotlb.h> #include <linux/swiotlb.h>
#include <linux/mem_encrypt.h> #include <linux/mem_encrypt.h>
#include <linux/device.h>
#include <linux/kernel.h>
#include <linux/bitops.h>
#include <linux/dma-mapping.h>
#include <asm/tlbflush.h> #include <asm/tlbflush.h>
#include <asm/fixmap.h> #include <asm/fixmap.h>
...@@ -348,6 +352,32 @@ bool sev_active(void) ...@@ -348,6 +352,32 @@ bool sev_active(void)
} }
EXPORT_SYMBOL(sev_active); EXPORT_SYMBOL(sev_active);
/* Override for DMA direct allocation check - ARCH_HAS_FORCE_DMA_UNENCRYPTED */
bool force_dma_unencrypted(struct device *dev)
{
/*
* For SEV, all DMA must be to unencrypted addresses.
*/
if (sev_active())
return true;
/*
* For SME, all DMA must be to unencrypted addresses if the
* device does not support DMA to addresses that include the
* encryption mask.
*/
if (sme_active()) {
u64 dma_enc_mask = DMA_BIT_MASK(__ffs64(sme_me_mask));
u64 dma_dev_mask = min_not_zero(dev->coherent_dma_mask,
dev->bus_dma_mask);
if (dma_dev_mask <= dma_enc_mask)
return true;
}
return false;
}
/* Architecture __weak replacement functions */ /* Architecture __weak replacement functions */
void __init mem_encrypt_free_decrypted_mem(void) void __init mem_encrypt_free_decrypted_mem(void)
{ {
......
...@@ -32,6 +32,15 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size) ...@@ -32,6 +32,15 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
} }
#endif /* !CONFIG_ARCH_HAS_PHYS_TO_DMA */ #endif /* !CONFIG_ARCH_HAS_PHYS_TO_DMA */
#ifdef CONFIG_ARCH_HAS_FORCE_DMA_UNENCRYPTED
bool force_dma_unencrypted(struct device *dev);
#else
static inline bool force_dma_unencrypted(struct device *dev)
{
return false;
}
#endif /* CONFIG_ARCH_HAS_FORCE_DMA_UNENCRYPTED */
/* /*
* If memory encryption is supported, phys_to_dma will set the memory encryption * If memory encryption is supported, phys_to_dma will set the memory encryption
* bit in the DMA address, and dma_to_phys will clear it. The raw __phys_to_dma * bit in the DMA address, and dma_to_phys will clear it. The raw __phys_to_dma
......
...@@ -679,6 +679,20 @@ static inline int dma_coerce_mask_and_coherent(struct device *dev, u64 mask) ...@@ -679,6 +679,20 @@ static inline int dma_coerce_mask_and_coherent(struct device *dev, u64 mask)
return dma_set_mask_and_coherent(dev, mask); return dma_set_mask_and_coherent(dev, mask);
} }
/**
* dma_addressing_limited - return if the device is addressing limited
* @dev: device to check
*
* Return %true if the devices DMA mask is too small to address all memory in
* the system, else %false. Lack of addressing bits is the prime reason for
* bounce buffering, but might not be the only one.
*/
static inline bool dma_addressing_limited(struct device *dev)
{
return min_not_zero(*dev->dma_mask, dev->bus_dma_mask) <
dma_get_required_mask(dev);
}
#ifdef CONFIG_ARCH_HAS_SETUP_DMA_OPS #ifdef CONFIG_ARCH_HAS_SETUP_DMA_OPS
void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
const struct iommu_ops *iommu, bool coherent); const struct iommu_ops *iommu, bool coherent);
......
...@@ -48,6 +48,9 @@ config ARCH_HAS_DMA_COHERENT_TO_PFN ...@@ -48,6 +48,9 @@ config ARCH_HAS_DMA_COHERENT_TO_PFN
config ARCH_HAS_DMA_MMAP_PGPROT config ARCH_HAS_DMA_MMAP_PGPROT
bool bool
config ARCH_HAS_FORCE_DMA_UNENCRYPTED
bool
config DMA_NONCOHERENT_CACHE_SYNC config DMA_NONCOHERENT_CACHE_SYNC
bool bool
......
...@@ -23,14 +23,6 @@ ...@@ -23,14 +23,6 @@
#define ARCH_ZONE_DMA_BITS 24 #define ARCH_ZONE_DMA_BITS 24
#endif #endif
/*
* For AMD SEV all DMA must be to unencrypted addresses.
*/
static inline bool force_dma_unencrypted(void)
{
return sev_active();
}
static void report_addr(struct device *dev, dma_addr_t dma_addr, size_t size) static void report_addr(struct device *dev, dma_addr_t dma_addr, size_t size)
{ {
if (!dev->dma_mask) { if (!dev->dma_mask) {
...@@ -46,7 +38,7 @@ static void report_addr(struct device *dev, dma_addr_t dma_addr, size_t size) ...@@ -46,7 +38,7 @@ static void report_addr(struct device *dev, dma_addr_t dma_addr, size_t size)
static inline dma_addr_t phys_to_dma_direct(struct device *dev, static inline dma_addr_t phys_to_dma_direct(struct device *dev,
phys_addr_t phys) phys_addr_t phys)
{ {
if (force_dma_unencrypted()) if (force_dma_unencrypted(dev))
return __phys_to_dma(dev, phys); return __phys_to_dma(dev, phys);
return phys_to_dma(dev, phys); return phys_to_dma(dev, phys);
} }
...@@ -67,7 +59,7 @@ static gfp_t __dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask, ...@@ -67,7 +59,7 @@ static gfp_t __dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask,
if (dev->bus_dma_mask && dev->bus_dma_mask < dma_mask) if (dev->bus_dma_mask && dev->bus_dma_mask < dma_mask)
dma_mask = dev->bus_dma_mask; dma_mask = dev->bus_dma_mask;
if (force_dma_unencrypted()) if (force_dma_unencrypted(dev))
*phys_mask = __dma_to_phys(dev, dma_mask); *phys_mask = __dma_to_phys(dev, dma_mask);
else else
*phys_mask = dma_to_phys(dev, dma_mask); *phys_mask = dma_to_phys(dev, dma_mask);
...@@ -159,7 +151,7 @@ void *dma_direct_alloc_pages(struct device *dev, size_t size, ...@@ -159,7 +151,7 @@ void *dma_direct_alloc_pages(struct device *dev, size_t size,
} }
ret = page_address(page); ret = page_address(page);
if (force_dma_unencrypted()) { if (force_dma_unencrypted(dev)) {
set_memory_decrypted((unsigned long)ret, 1 << get_order(size)); set_memory_decrypted((unsigned long)ret, 1 << get_order(size));
*dma_handle = __phys_to_dma(dev, page_to_phys(page)); *dma_handle = __phys_to_dma(dev, page_to_phys(page));
} else { } else {
...@@ -192,7 +184,7 @@ void dma_direct_free_pages(struct device *dev, size_t size, void *cpu_addr, ...@@ -192,7 +184,7 @@ void dma_direct_free_pages(struct device *dev, size_t size, void *cpu_addr,
return; return;
} }
if (force_dma_unencrypted()) if (force_dma_unencrypted(dev))
set_memory_encrypted((unsigned long)cpu_addr, 1 << page_order); set_memory_encrypted((unsigned long)cpu_addr, 1 << page_order);
if (IS_ENABLED(CONFIG_ARCH_HAS_UNCACHED_SEGMENT) && if (IS_ENABLED(CONFIG_ARCH_HAS_UNCACHED_SEGMENT) &&
...@@ -242,12 +234,14 @@ void dma_direct_sync_sg_for_device(struct device *dev, ...@@ -242,12 +234,14 @@ void dma_direct_sync_sg_for_device(struct device *dev,
int i; int i;
for_each_sg(sgl, sg, nents, i) { for_each_sg(sgl, sg, nents, i) {
if (unlikely(is_swiotlb_buffer(sg_phys(sg)))) phys_addr_t paddr = dma_to_phys(dev, sg_dma_address(sg));
swiotlb_tbl_sync_single(dev, sg_phys(sg), sg->length,
if (unlikely(is_swiotlb_buffer(paddr)))
swiotlb_tbl_sync_single(dev, paddr, sg->length,
dir, SYNC_FOR_DEVICE); dir, SYNC_FOR_DEVICE);
if (!dev_is_dma_coherent(dev)) if (!dev_is_dma_coherent(dev))
arch_sync_dma_for_device(dev, sg_phys(sg), sg->length, arch_sync_dma_for_device(dev, paddr, sg->length,
dir); dir);
} }
} }
...@@ -279,11 +273,13 @@ void dma_direct_sync_sg_for_cpu(struct device *dev, ...@@ -279,11 +273,13 @@ void dma_direct_sync_sg_for_cpu(struct device *dev,
int i; int i;
for_each_sg(sgl, sg, nents, i) { for_each_sg(sgl, sg, nents, i) {
phys_addr_t paddr = dma_to_phys(dev, sg_dma_address(sg));
if (!dev_is_dma_coherent(dev)) if (!dev_is_dma_coherent(dev))
arch_sync_dma_for_cpu(dev, sg_phys(sg), sg->length, dir); arch_sync_dma_for_cpu(dev, paddr, sg->length, dir);
if (unlikely(is_swiotlb_buffer(sg_phys(sg)))) if (unlikely(is_swiotlb_buffer(paddr)))
swiotlb_tbl_sync_single(dev, sg_phys(sg), sg->length, dir, swiotlb_tbl_sync_single(dev, paddr, sg->length, dir,
SYNC_FOR_CPU); SYNC_FOR_CPU);
} }
...@@ -407,11 +403,9 @@ int dma_direct_supported(struct device *dev, u64 mask) ...@@ -407,11 +403,9 @@ int dma_direct_supported(struct device *dev, u64 mask)
size_t dma_direct_max_mapping_size(struct device *dev) size_t dma_direct_max_mapping_size(struct device *dev)
{ {
size_t size = SIZE_MAX;
/* If SWIOTLB is active, use its maximum mapping size */ /* If SWIOTLB is active, use its maximum mapping size */
if (is_swiotlb_active()) if (is_swiotlb_active() &&
size = swiotlb_max_mapping_size(dev); (dma_addressing_limited(dev) || swiotlb_force == SWIOTLB_FORCE))
return swiotlb_max_mapping_size(dev);
return size; return SIZE_MAX;
} }
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment