Commit d360f1ee authored by David S. Miller's avatar David S. Miller

[SPARC64]: Fix PCI IOMMU invalid iopte handling.

Instead of marking them as invalid, point them
at a dummy page.  This handles buggy third-party
bridges that erroneously prefetch sometimes.
Signed-off-by: default avatarDavid S. Miller <davem@redhat.com>
parent 14729dbe
...@@ -56,6 +56,39 @@ static void __iommu_flushall(struct pci_iommu *iommu) ...@@ -56,6 +56,39 @@ static void __iommu_flushall(struct pci_iommu *iommu)
} }
} }
#define IOPTE_CONSISTENT(CTX) \
(IOPTE_VALID | IOPTE_CACHE | \
(((CTX) << 47) & IOPTE_CONTEXT))
#define IOPTE_STREAMING(CTX) \
(IOPTE_CONSISTENT(CTX) | IOPTE_STBUF)
/* Existing mappings are never marked invalid, instead they
* are pointed to a dummy page.
*/
#define IOPTE_IS_DUMMY(iommu, iopte) \
((iopte_val(*iopte) & IOPTE_PAGE) == (iommu)->dummy_page_pa)
static void inline iopte_make_dummy(struct pci_iommu *iommu, iopte_t *iopte)
{
unsigned long val = iopte_val(*iopte);
val &= ~IOPTE_PAGE;
val |= iommu->dummy_page_pa;
iopte_val(*iopte) = val;
}
void pci_iommu_table_init(struct pci_iommu *iommu, int tsbsize)
{
int i;
tsbsize /= sizeof(iopte_t);
for (i = 0; i < tsbsize; i++)
iopte_make_dummy(iommu, &iommu->page_table[i]);
}
static iopte_t *alloc_streaming_cluster(struct pci_iommu *iommu, unsigned long npages) static iopte_t *alloc_streaming_cluster(struct pci_iommu *iommu, unsigned long npages)
{ {
iopte_t *iopte, *limit, *first; iopte_t *iopte, *limit, *first;
...@@ -79,7 +112,7 @@ static iopte_t *alloc_streaming_cluster(struct pci_iommu *iommu, unsigned long n ...@@ -79,7 +112,7 @@ static iopte_t *alloc_streaming_cluster(struct pci_iommu *iommu, unsigned long n
first = iopte; first = iopte;
for (;;) { for (;;) {
if (iopte_val(*iopte) == 0UL) { if (IOPTE_IS_DUMMY(iommu, iopte)) {
if ((iopte + (1 << cnum)) >= limit) if ((iopte + (1 << cnum)) >= limit)
ent = 0; ent = 0;
else else
...@@ -142,12 +175,12 @@ static iopte_t *alloc_consistent_cluster(struct pci_iommu *iommu, unsigned long ...@@ -142,12 +175,12 @@ static iopte_t *alloc_consistent_cluster(struct pci_iommu *iommu, unsigned long
iopte = iommu->page_table + (1 << (iommu->page_table_sz_bits - PBM_LOGCLUSTERS)); iopte = iommu->page_table + (1 << (iommu->page_table_sz_bits - PBM_LOGCLUSTERS));
while (iopte > iommu->page_table) { while (iopte > iommu->page_table) {
iopte--; iopte--;
if (!(iopte_val(*iopte) & IOPTE_VALID)) { if (IOPTE_IS_DUMMY(iommu, iopte)) {
unsigned long tmp = npages; unsigned long tmp = npages;
while (--tmp) { while (--tmp) {
iopte--; iopte--;
if (iopte_val(*iopte) & IOPTE_VALID) if (!IOPTE_IS_DUMMY(iommu, iopte))
break; break;
} }
if (tmp == 0) { if (tmp == 0) {
...@@ -162,15 +195,6 @@ static iopte_t *alloc_consistent_cluster(struct pci_iommu *iommu, unsigned long ...@@ -162,15 +195,6 @@ static iopte_t *alloc_consistent_cluster(struct pci_iommu *iommu, unsigned long
return NULL; return NULL;
} }
#define IOPTE_CONSISTENT(CTX) \
(IOPTE_VALID | IOPTE_CACHE | \
(((CTX) << 47) & IOPTE_CONTEXT))
#define IOPTE_STREAMING(CTX) \
(IOPTE_CONSISTENT(CTX) | IOPTE_STBUF)
#define IOPTE_INVALID 0UL
/* Allocate and map kernel buffer of size SIZE using consistent mode /* Allocate and map kernel buffer of size SIZE using consistent mode
* DMA for PCI device PDEV. Return non-NULL cpu-side address if * DMA for PCI device PDEV. Return non-NULL cpu-side address if
* successful and set *DMA_ADDRP to the PCI side dma address. * successful and set *DMA_ADDRP to the PCI side dma address.
...@@ -261,7 +285,7 @@ void pci_free_consistent(struct pci_dev *pdev, size_t size, void *cpu, dma_addr_ ...@@ -261,7 +285,7 @@ void pci_free_consistent(struct pci_dev *pdev, size_t size, void *cpu, dma_addr_
limit = (iommu->page_table + limit = (iommu->page_table +
(1 << (iommu->page_table_sz_bits - PBM_LOGCLUSTERS))); (1 << (iommu->page_table_sz_bits - PBM_LOGCLUSTERS)));
while (walk < limit) { while (walk < limit) {
if (iopte_val(*walk) != IOPTE_INVALID) if (!IOPTE_IS_DUMMY(iommu, walk))
break; break;
walk++; walk++;
} }
...@@ -280,7 +304,7 @@ void pci_free_consistent(struct pci_dev *pdev, size_t size, void *cpu, dma_addr_ ...@@ -280,7 +304,7 @@ void pci_free_consistent(struct pci_dev *pdev, size_t size, void *cpu, dma_addr_
ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL; ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL;
for (i = 0; i < npages; i++, iopte++) for (i = 0; i < npages; i++, iopte++)
iopte_val(*iopte) = IOPTE_INVALID; iopte_make_dummy(iommu, iopte);
if (iommu->iommu_ctxflush) { if (iommu->iommu_ctxflush) {
pci_iommu_write(iommu->iommu_ctxflush, ctx); pci_iommu_write(iommu->iommu_ctxflush, ctx);
...@@ -376,7 +400,7 @@ void pci_unmap_single(struct pci_dev *pdev, dma_addr_t bus_addr, size_t sz, int ...@@ -376,7 +400,7 @@ void pci_unmap_single(struct pci_dev *pdev, dma_addr_t bus_addr, size_t sz, int
base = iommu->page_table + base = iommu->page_table +
((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT); ((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT);
#ifdef DEBUG_PCI_IOMMU #ifdef DEBUG_PCI_IOMMU
if (iopte_val(*base) == IOPTE_INVALID) if (IOPTE_IS_DUMMY(iommu, base))
printk("pci_unmap_single called on non-mapped region %08x,%08x from %016lx\n", printk("pci_unmap_single called on non-mapped region %08x,%08x from %016lx\n",
bus_addr, sz, __builtin_return_address(0)); bus_addr, sz, __builtin_return_address(0));
#endif #endif
...@@ -415,7 +439,7 @@ void pci_unmap_single(struct pci_dev *pdev, dma_addr_t bus_addr, size_t sz, int ...@@ -415,7 +439,7 @@ void pci_unmap_single(struct pci_dev *pdev, dma_addr_t bus_addr, size_t sz, int
} }
/* Step 2: Clear out first TSB entry. */ /* Step 2: Clear out first TSB entry. */
iopte_val(*base) = IOPTE_INVALID; iopte_make_dummy(iommu, base);
free_streaming_cluster(iommu, bus_addr - iommu->page_table_map_base, free_streaming_cluster(iommu, bus_addr - iommu->page_table_map_base,
npages, ctx); npages, ctx);
...@@ -611,7 +635,7 @@ void pci_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, ...@@ -611,7 +635,7 @@ void pci_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems,
((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT); ((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT);
#ifdef DEBUG_PCI_IOMMU #ifdef DEBUG_PCI_IOMMU
if (iopte_val(*base) == IOPTE_INVALID) if (IOPTE_IS_DUMMY(iommu, base))
printk("pci_unmap_sg called on non-mapped region %016lx,%d from %016lx\n", sglist->dma_address, nelems, __builtin_return_address(0)); printk("pci_unmap_sg called on non-mapped region %016lx,%d from %016lx\n", sglist->dma_address, nelems, __builtin_return_address(0));
#endif #endif
...@@ -648,7 +672,7 @@ void pci_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, ...@@ -648,7 +672,7 @@ void pci_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems,
} }
/* Step 2: Clear out first TSB entry. */ /* Step 2: Clear out first TSB entry. */
iopte_val(*base) = IOPTE_INVALID; iopte_make_dummy(iommu, base);
free_streaming_cluster(iommu, bus_addr - iommu->page_table_map_base, free_streaming_cluster(iommu, bus_addr - iommu->page_table_map_base,
npages, ctx); npages, ctx);
......
...@@ -1241,6 +1241,14 @@ static void __init psycho_iommu_init(struct pci_controller_info *p) ...@@ -1241,6 +1241,14 @@ static void __init psycho_iommu_init(struct pci_controller_info *p)
* in pci_iommu.c * in pci_iommu.c
*/ */
iommu->dummy_page = __get_free_pages(GFP_KERNEL, 0);
if (!iommu->dummy_page) {
prom_printf("PSYCHO_IOMMU: Error, gfp(dummy_page) failed.\n");
prom_halt();
}
memset((void *)iommu->dummy_page, 0, PAGE_SIZE);
iommu->dummy_page_pa = (unsigned long) __pa(iommu->dummy_page);
/* Using assumed page size 8K with 128K entries we need 1MB iommu page /* Using assumed page size 8K with 128K entries we need 1MB iommu page
* table (128K ioptes * 8 bytes per iopte). This is * table (128K ioptes * 8 bytes per iopte). This is
* page order 7 on UltraSparc. * page order 7 on UltraSparc.
...@@ -1254,7 +1262,7 @@ static void __init psycho_iommu_init(struct pci_controller_info *p) ...@@ -1254,7 +1262,7 @@ static void __init psycho_iommu_init(struct pci_controller_info *p)
iommu->page_table_sz_bits = 17; iommu->page_table_sz_bits = 17;
iommu->page_table_map_base = 0xc0000000; iommu->page_table_map_base = 0xc0000000;
iommu->dma_addr_mask = 0xffffffff; iommu->dma_addr_mask = 0xffffffff;
memset((char *)tsbbase, 0, IO_TSB_SIZE); pci_iommu_table_init(iommu, IO_TSB_SIZE);
/* We start with no consistent mappings. */ /* We start with no consistent mappings. */
iommu->lowest_consistent_map = iommu->lowest_consistent_map =
......
...@@ -1293,6 +1293,14 @@ static void __init sabre_iommu_init(struct pci_controller_info *p, ...@@ -1293,6 +1293,14 @@ static void __init sabre_iommu_init(struct pci_controller_info *p,
* in pci_iommu.c * in pci_iommu.c
*/ */
iommu->dummy_page = __get_free_pages(GFP_KERNEL, 0);
if (!iommu->dummy_page) {
prom_printf("PSYCHO_IOMMU: Error, gfp(dummy_page) failed.\n");
prom_halt();
}
memset((void *)iommu->dummy_page, 0, PAGE_SIZE);
iommu->dummy_page_pa = (unsigned long) __pa(iommu->dummy_page);
tsbbase = __get_free_pages(GFP_KERNEL, order = get_order(tsbsize * 1024 * 8)); tsbbase = __get_free_pages(GFP_KERNEL, order = get_order(tsbsize * 1024 * 8));
if (!tsbbase) { if (!tsbbase) {
prom_printf("SABRE_IOMMU: Error, gfp(tsb) failed.\n"); prom_printf("SABRE_IOMMU: Error, gfp(tsb) failed.\n");
...@@ -1301,7 +1309,7 @@ static void __init sabre_iommu_init(struct pci_controller_info *p, ...@@ -1301,7 +1309,7 @@ static void __init sabre_iommu_init(struct pci_controller_info *p,
iommu->page_table = (iopte_t *)tsbbase; iommu->page_table = (iopte_t *)tsbbase;
iommu->page_table_map_base = dvma_offset; iommu->page_table_map_base = dvma_offset;
iommu->dma_addr_mask = dma_mask; iommu->dma_addr_mask = dma_mask;
memset((char *)tsbbase, 0, PAGE_SIZE << order); pci_iommu_table_init(iommu, PAGE_SIZE << order);
sabre_write(p->pbm_A.controller_regs + SABRE_IOMMU_TSBBASE, __pa(tsbbase)); sabre_write(p->pbm_A.controller_regs + SABRE_IOMMU_TSBBASE, __pa(tsbbase));
......
...@@ -1766,6 +1766,14 @@ static void schizo_pbm_iommu_init(struct pci_pbm_info *pbm) ...@@ -1766,6 +1766,14 @@ static void schizo_pbm_iommu_init(struct pci_pbm_info *pbm)
* in pci_iommu.c * in pci_iommu.c
*/ */
iommu->dummy_page = __get_free_pages(GFP_KERNEL, 0);
if (!iommu->dummy_page) {
prom_printf("PSYCHO_IOMMU: Error, gfp(dummy_page) failed.\n");
prom_halt();
}
memset((void *)iommu->dummy_page, 0, PAGE_SIZE);
iommu->dummy_page_pa = (unsigned long) __pa(iommu->dummy_page);
/* Using assumed page size 8K with 128K entries we need 1MB iommu page /* Using assumed page size 8K with 128K entries we need 1MB iommu page
* table (128K ioptes * 8 bytes per iopte). This is * table (128K ioptes * 8 bytes per iopte). This is
* page order 7 on UltraSparc. * page order 7 on UltraSparc.
...@@ -1780,7 +1788,7 @@ static void schizo_pbm_iommu_init(struct pci_pbm_info *pbm) ...@@ -1780,7 +1788,7 @@ static void schizo_pbm_iommu_init(struct pci_pbm_info *pbm)
iommu->page_table = (iopte_t *)tsbbase; iommu->page_table = (iopte_t *)tsbbase;
iommu->page_table_map_base = vdma[0]; iommu->page_table_map_base = vdma[0];
iommu->dma_addr_mask = dma_mask; iommu->dma_addr_mask = dma_mask;
memset((char *)tsbbase, 0, PAGE_SIZE << order); pci_iommu_table_init(iommu, PAGE_SIZE << order);
switch (tsbsize) { switch (tsbsize) {
case 64: case 64:
......
...@@ -70,6 +70,13 @@ struct pci_iommu { ...@@ -70,6 +70,13 @@ struct pci_iommu {
*/ */
u32 lowest_consistent_map; u32 lowest_consistent_map;
/* In order to deal with some buggy third-party PCI bridges that
* do wrong prefetching, we never mark valid mappings as invalid.
* Instead we point them at this dummy page.
*/
unsigned long dummy_page;
unsigned long dummy_page_pa;
/* If PBM_NCLUSTERS is ever decreased to 4 or lower, /* If PBM_NCLUSTERS is ever decreased to 4 or lower,
* or if largest supported page_table_sz * 8K goes above * or if largest supported page_table_sz * 8K goes above
* 2GB, you must increase the size of the type of * 2GB, you must increase the size of the type of
...@@ -93,6 +100,8 @@ struct pci_iommu { ...@@ -93,6 +100,8 @@ struct pci_iommu {
u32 dma_addr_mask; u32 dma_addr_mask;
}; };
extern void pci_iommu_table_init(struct pci_iommu *, int);
/* This describes a PCI bus module's streaming buffer. */ /* This describes a PCI bus module's streaming buffer. */
struct pci_strbuf { struct pci_strbuf {
int strbuf_enabled; /* Present and using it? */ int strbuf_enabled; /* Present and using it? */
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment