Commit 5637bc50 authored by Mike Rapoport's avatar Mike Rapoport Committed by David S. Miller

sparc64: add support for folded p4d page tables

Implement primitives necessary for the 4th level folding, add walks of p4d
level where appropriate and replace 5level-fixup.h with pgtable-nop4d.h.
Signed-off-by: default avatarMike Rapoport <rppt@linux.ibm.com>
Acked-by: default avatarDavid S. Miller <davem@davemloft.net>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 9167bd96
...@@ -16,12 +16,12 @@ ...@@ -16,12 +16,12 @@
extern struct kmem_cache *pgtable_cache; extern struct kmem_cache *pgtable_cache;
static inline void __pgd_populate(pgd_t *pgd, pud_t *pud) static inline void __p4d_populate(p4d_t *p4d, pud_t *pud)
{ {
pgd_set(pgd, pud); p4d_set(p4d, pud);
} }
#define pgd_populate(MM, PGD, PUD) __pgd_populate(PGD, PUD) #define p4d_populate(MM, P4D, PUD) __p4d_populate(P4D, PUD)
static inline pgd_t *pgd_alloc(struct mm_struct *mm) static inline pgd_t *pgd_alloc(struct mm_struct *mm)
{ {
......
...@@ -13,7 +13,7 @@ ...@@ -13,7 +13,7 @@
* the SpitFire page tables. * the SpitFire page tables.
*/ */
#include <asm-generic/5level-fixup.h> #include <asm-generic/pgtable-nop4d.h>
#include <linux/compiler.h> #include <linux/compiler.h>
#include <linux/const.h> #include <linux/const.h>
#include <asm/types.h> #include <asm/types.h>
...@@ -810,9 +810,9 @@ static inline int pmd_present(pmd_t pmd) ...@@ -810,9 +810,9 @@ static inline int pmd_present(pmd_t pmd)
#define pud_bad(pud) (pud_val(pud) & ~PAGE_MASK) #define pud_bad(pud) (pud_val(pud) & ~PAGE_MASK)
#define pgd_none(pgd) (!pgd_val(pgd)) #define p4d_none(p4d) (!p4d_val(p4d))
#define pgd_bad(pgd) (pgd_val(pgd) & ~PAGE_MASK) #define p4d_bad(p4d) (p4d_val(p4d) & ~PAGE_MASK)
#ifdef CONFIG_TRANSPARENT_HUGEPAGE #ifdef CONFIG_TRANSPARENT_HUGEPAGE
void set_pmd_at(struct mm_struct *mm, unsigned long addr, void set_pmd_at(struct mm_struct *mm, unsigned long addr,
...@@ -859,13 +859,13 @@ static inline unsigned long pud_page_vaddr(pud_t pud) ...@@ -859,13 +859,13 @@ static inline unsigned long pud_page_vaddr(pud_t pud)
#define pmd_clear(pmdp) (pmd_val(*(pmdp)) = 0UL) #define pmd_clear(pmdp) (pmd_val(*(pmdp)) = 0UL)
#define pud_present(pud) (pud_val(pud) != 0U) #define pud_present(pud) (pud_val(pud) != 0U)
#define pud_clear(pudp) (pud_val(*(pudp)) = 0UL) #define pud_clear(pudp) (pud_val(*(pudp)) = 0UL)
#define pgd_page_vaddr(pgd) \ #define p4d_page_vaddr(p4d) \
((unsigned long) __va(pgd_val(pgd))) ((unsigned long) __va(p4d_val(p4d)))
#define pgd_present(pgd) (pgd_val(pgd) != 0U) #define p4d_present(p4d) (p4d_val(p4d) != 0U)
#define pgd_clear(pgdp) (pgd_val(*(pgdp)) = 0UL) #define p4d_clear(p4dp) (p4d_val(*(p4dp)) = 0UL)
/* only used by the stubbed out hugetlb gup code, should never be called */ /* only used by the stubbed out hugetlb gup code, should never be called */
#define pgd_page(pgd) NULL #define p4d_page(p4d) NULL
static inline unsigned long pud_large(pud_t pud) static inline unsigned long pud_large(pud_t pud)
{ {
...@@ -884,8 +884,8 @@ static inline unsigned long pud_pfn(pud_t pud) ...@@ -884,8 +884,8 @@ static inline unsigned long pud_pfn(pud_t pud)
/* Same in both SUN4V and SUN4U. */ /* Same in both SUN4V and SUN4U. */
#define pte_none(pte) (!pte_val(pte)) #define pte_none(pte) (!pte_val(pte))
#define pgd_set(pgdp, pudp) \ #define p4d_set(p4dp, pudp) \
(pgd_val(*(pgdp)) = (__pa((unsigned long) (pudp)))) (p4d_val(*(p4dp)) = (__pa((unsigned long) (pudp))))
/* to find an entry in a page-table-directory. */ /* to find an entry in a page-table-directory. */
#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1)) #define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1))
...@@ -896,8 +896,8 @@ static inline unsigned long pud_pfn(pud_t pud) ...@@ -896,8 +896,8 @@ static inline unsigned long pud_pfn(pud_t pud)
/* Find an entry in the third-level page table.. */ /* Find an entry in the third-level page table.. */
#define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD - 1)) #define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))
#define pud_offset(pgdp, address) \ #define pud_offset(p4dp, address) \
((pud_t *) pgd_page_vaddr(*(pgdp)) + pud_index(address)) ((pud_t *) p4d_page_vaddr(*(p4dp)) + pud_index(address))
/* Find an entry in the second-level page table.. */ /* Find an entry in the second-level page table.. */
#define pmd_offset(pudp, address) \ #define pmd_offset(pudp, address) \
......
...@@ -299,6 +299,7 @@ static void flush_signal_insns(unsigned long address) ...@@ -299,6 +299,7 @@ static void flush_signal_insns(unsigned long address)
unsigned long pstate, paddr; unsigned long pstate, paddr;
pte_t *ptep, pte; pte_t *ptep, pte;
pgd_t *pgdp; pgd_t *pgdp;
p4d_t *p4dp;
pud_t *pudp; pud_t *pudp;
pmd_t *pmdp; pmd_t *pmdp;
...@@ -318,7 +319,10 @@ static void flush_signal_insns(unsigned long address) ...@@ -318,7 +319,10 @@ static void flush_signal_insns(unsigned long address)
pgdp = pgd_offset(current->mm, address); pgdp = pgd_offset(current->mm, address);
if (pgd_none(*pgdp)) if (pgd_none(*pgdp))
goto out_irqs_on; goto out_irqs_on;
pudp = pud_offset(pgdp, address); p4dp = p4d_offset(pgdp, address);
if (p4d_none(*p4dp))
goto out_irqs_on;
pudp = pud_offset(p4dp, address);
if (pud_none(*pudp)) if (pud_none(*pudp))
goto out_irqs_on; goto out_irqs_on;
pmdp = pmd_offset(pudp, address); pmdp = pmd_offset(pudp, address);
......
...@@ -1621,6 +1621,7 @@ static int __init pcpu_cpu_distance(unsigned int from, unsigned int to) ...@@ -1621,6 +1621,7 @@ static int __init pcpu_cpu_distance(unsigned int from, unsigned int to)
static void __init pcpu_populate_pte(unsigned long addr) static void __init pcpu_populate_pte(unsigned long addr)
{ {
pgd_t *pgd = pgd_offset_k(addr); pgd_t *pgd = pgd_offset_k(addr);
p4d_t *p4d;
pud_t *pud; pud_t *pud;
pmd_t *pmd; pmd_t *pmd;
...@@ -1633,7 +1634,17 @@ static void __init pcpu_populate_pte(unsigned long addr) ...@@ -1633,7 +1634,17 @@ static void __init pcpu_populate_pte(unsigned long addr)
pgd_populate(&init_mm, pgd, new); pgd_populate(&init_mm, pgd, new);
} }
pud = pud_offset(pgd, addr); p4d = p4d_offset(pgd, addr);
if (p4d_none(*p4d)) {
pud_t *new;
new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
if (!new)
goto err_alloc;
p4d_populate(&init_mm, p4d, new);
}
pud = pud_offset(p4d, addr);
if (pud_none(*pud)) { if (pud_none(*pud)) {
pmd_t *new; pmd_t *new;
......
...@@ -80,6 +80,7 @@ static void __kprobes bad_kernel_pc(struct pt_regs *regs, unsigned long vaddr) ...@@ -80,6 +80,7 @@ static void __kprobes bad_kernel_pc(struct pt_regs *regs, unsigned long vaddr)
static unsigned int get_user_insn(unsigned long tpc) static unsigned int get_user_insn(unsigned long tpc)
{ {
pgd_t *pgdp = pgd_offset(current->mm, tpc); pgd_t *pgdp = pgd_offset(current->mm, tpc);
p4d_t *p4dp;
pud_t *pudp; pud_t *pudp;
pmd_t *pmdp; pmd_t *pmdp;
pte_t *ptep, pte; pte_t *ptep, pte;
...@@ -88,7 +89,10 @@ static unsigned int get_user_insn(unsigned long tpc) ...@@ -88,7 +89,10 @@ static unsigned int get_user_insn(unsigned long tpc)
if (pgd_none(*pgdp) || unlikely(pgd_bad(*pgdp))) if (pgd_none(*pgdp) || unlikely(pgd_bad(*pgdp)))
goto out; goto out;
pudp = pud_offset(pgdp, tpc); p4dp = p4d_offset(pgdp, tpc);
if (p4d_none(*p4dp) || unlikely(p4d_bad(*p4dp)))
goto out;
pudp = pud_offset(p4dp, tpc);
if (pud_none(*pudp) || unlikely(pud_bad(*pudp))) if (pud_none(*pudp) || unlikely(pud_bad(*pudp)))
goto out; goto out;
......
...@@ -277,11 +277,13 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, ...@@ -277,11 +277,13 @@ pte_t *huge_pte_alloc(struct mm_struct *mm,
unsigned long addr, unsigned long sz) unsigned long addr, unsigned long sz)
{ {
pgd_t *pgd; pgd_t *pgd;
p4d_t *p4d;
pud_t *pud; pud_t *pud;
pmd_t *pmd; pmd_t *pmd;
pgd = pgd_offset(mm, addr); pgd = pgd_offset(mm, addr);
pud = pud_alloc(mm, pgd, addr); p4d = p4d_offset(pgd, addr);
pud = pud_alloc(mm, p4d, addr);
if (!pud) if (!pud)
return NULL; return NULL;
if (sz >= PUD_SIZE) if (sz >= PUD_SIZE)
...@@ -298,13 +300,17 @@ pte_t *huge_pte_offset(struct mm_struct *mm, ...@@ -298,13 +300,17 @@ pte_t *huge_pte_offset(struct mm_struct *mm,
unsigned long addr, unsigned long sz) unsigned long addr, unsigned long sz)
{ {
pgd_t *pgd; pgd_t *pgd;
p4d_t *p4d;
pud_t *pud; pud_t *pud;
pmd_t *pmd; pmd_t *pmd;
pgd = pgd_offset(mm, addr); pgd = pgd_offset(mm, addr);
if (pgd_none(*pgd)) if (pgd_none(*pgd))
return NULL; return NULL;
pud = pud_offset(pgd, addr); p4d = p4d_offset(pgd, addr);
if (p4d_none(*p4d))
return NULL;
pud = pud_offset(p4d, addr);
if (pud_none(*pud)) if (pud_none(*pud))
return NULL; return NULL;
if (is_hugetlb_pud(*pud)) if (is_hugetlb_pud(*pud))
...@@ -449,7 +455,7 @@ static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud, ...@@ -449,7 +455,7 @@ static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
mm_dec_nr_pmds(tlb->mm); mm_dec_nr_pmds(tlb->mm);
} }
static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd, static void hugetlb_free_pud_range(struct mmu_gather *tlb, p4d_t *p4d,
unsigned long addr, unsigned long end, unsigned long addr, unsigned long end,
unsigned long floor, unsigned long ceiling) unsigned long floor, unsigned long ceiling)
{ {
...@@ -458,7 +464,7 @@ static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd, ...@@ -458,7 +464,7 @@ static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
unsigned long start; unsigned long start;
start = addr; start = addr;
pud = pud_offset(pgd, addr); pud = pud_offset(p4d, addr);
do { do {
next = pud_addr_end(addr, end); next = pud_addr_end(addr, end);
if (pud_none_or_clear_bad(pud)) if (pud_none_or_clear_bad(pud))
...@@ -481,8 +487,8 @@ static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd, ...@@ -481,8 +487,8 @@ static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
if (end - 1 > ceiling - 1) if (end - 1 > ceiling - 1)
return; return;
pud = pud_offset(pgd, start); pud = pud_offset(p4d, start);
pgd_clear(pgd); p4d_clear(p4d);
pud_free_tlb(tlb, pud, start); pud_free_tlb(tlb, pud, start);
mm_dec_nr_puds(tlb->mm); mm_dec_nr_puds(tlb->mm);
} }
...@@ -492,6 +498,7 @@ void hugetlb_free_pgd_range(struct mmu_gather *tlb, ...@@ -492,6 +498,7 @@ void hugetlb_free_pgd_range(struct mmu_gather *tlb,
unsigned long floor, unsigned long ceiling) unsigned long floor, unsigned long ceiling)
{ {
pgd_t *pgd; pgd_t *pgd;
p4d_t *p4d;
unsigned long next; unsigned long next;
addr &= PMD_MASK; addr &= PMD_MASK;
...@@ -511,10 +518,11 @@ void hugetlb_free_pgd_range(struct mmu_gather *tlb, ...@@ -511,10 +518,11 @@ void hugetlb_free_pgd_range(struct mmu_gather *tlb,
return; return;
pgd = pgd_offset(tlb->mm, addr); pgd = pgd_offset(tlb->mm, addr);
p4d = p4d_offset(pgd, addr);
do { do {
next = pgd_addr_end(addr, end); next = p4d_addr_end(addr, end);
if (pgd_none_or_clear_bad(pgd)) if (p4d_none_or_clear_bad(p4d))
continue; continue;
hugetlb_free_pud_range(tlb, pgd, addr, next, floor, ceiling); hugetlb_free_pud_range(tlb, p4d, addr, next, floor, ceiling);
} while (pgd++, addr = next, addr != end); } while (p4d++, addr = next, addr != end);
} }
...@@ -530,7 +530,8 @@ void __kprobes flush_icache_range(unsigned long start, unsigned long end) ...@@ -530,7 +530,8 @@ void __kprobes flush_icache_range(unsigned long start, unsigned long end)
paddr = kaddr & mask; paddr = kaddr & mask;
else { else {
pgd_t *pgdp = pgd_offset_k(kaddr); pgd_t *pgdp = pgd_offset_k(kaddr);
pud_t *pudp = pud_offset(pgdp, kaddr); p4d_t *p4dp = p4d_offset(pgdp, kaddr);
pud_t *pudp = pud_offset(p4dp, kaddr);
pmd_t *pmdp = pmd_offset(pudp, kaddr); pmd_t *pmdp = pmd_offset(pudp, kaddr);
pte_t *ptep = pte_offset_kernel(pmdp, kaddr); pte_t *ptep = pte_offset_kernel(pmdp, kaddr);
...@@ -1653,6 +1654,7 @@ static unsigned long max_phys_bits = 40; ...@@ -1653,6 +1654,7 @@ static unsigned long max_phys_bits = 40;
bool kern_addr_valid(unsigned long addr) bool kern_addr_valid(unsigned long addr)
{ {
pgd_t *pgd; pgd_t *pgd;
p4d_t *p4d;
pud_t *pud; pud_t *pud;
pmd_t *pmd; pmd_t *pmd;
pte_t *pte; pte_t *pte;
...@@ -1674,7 +1676,11 @@ bool kern_addr_valid(unsigned long addr) ...@@ -1674,7 +1676,11 @@ bool kern_addr_valid(unsigned long addr)
if (pgd_none(*pgd)) if (pgd_none(*pgd))
return 0; return 0;
pud = pud_offset(pgd, addr); p4d = p4d_offset(pgd, addr);
if (p4d_none(*p4d))
return 0;
pud = pud_offset(p4d, addr);
if (pud_none(*pud)) if (pud_none(*pud))
return 0; return 0;
...@@ -1800,6 +1806,7 @@ static unsigned long __ref kernel_map_range(unsigned long pstart, ...@@ -1800,6 +1806,7 @@ static unsigned long __ref kernel_map_range(unsigned long pstart,
while (vstart < vend) { while (vstart < vend) {
unsigned long this_end, paddr = __pa(vstart); unsigned long this_end, paddr = __pa(vstart);
pgd_t *pgd = pgd_offset_k(vstart); pgd_t *pgd = pgd_offset_k(vstart);
p4d_t *p4d;
pud_t *pud; pud_t *pud;
pmd_t *pmd; pmd_t *pmd;
pte_t *pte; pte_t *pte;
...@@ -1814,7 +1821,20 @@ static unsigned long __ref kernel_map_range(unsigned long pstart, ...@@ -1814,7 +1821,20 @@ static unsigned long __ref kernel_map_range(unsigned long pstart,
alloc_bytes += PAGE_SIZE; alloc_bytes += PAGE_SIZE;
pgd_populate(&init_mm, pgd, new); pgd_populate(&init_mm, pgd, new);
} }
pud = pud_offset(pgd, vstart);
p4d = p4d_offset(pgd, vstart);
if (p4d_none(*p4d)) {
pud_t *new;
new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE,
PAGE_SIZE);
if (!new)
goto err_alloc;
alloc_bytes += PAGE_SIZE;
p4d_populate(&init_mm, p4d, new);
}
pud = pud_offset(p4d, vstart);
if (pud_none(*pud)) { if (pud_none(*pud)) {
pmd_t *new; pmd_t *new;
...@@ -2612,13 +2632,18 @@ int __meminit vmemmap_populate(unsigned long vstart, unsigned long vend, ...@@ -2612,13 +2632,18 @@ int __meminit vmemmap_populate(unsigned long vstart, unsigned long vend,
for (; vstart < vend; vstart += PMD_SIZE) { for (; vstart < vend; vstart += PMD_SIZE) {
pgd_t *pgd = vmemmap_pgd_populate(vstart, node); pgd_t *pgd = vmemmap_pgd_populate(vstart, node);
unsigned long pte; unsigned long pte;
p4d_t *p4d;
pud_t *pud; pud_t *pud;
pmd_t *pmd; pmd_t *pmd;
if (!pgd) if (!pgd)
return -ENOMEM; return -ENOMEM;
pud = vmemmap_pud_populate(pgd, vstart, node); p4d = vmemmap_p4d_populate(pgd, vstart, node);
if (!p4d)
return -ENOMEM;
pud = vmemmap_pud_populate(p4d, vstart, node);
if (!pud) if (!pud)
return -ENOMEM; return -ENOMEM;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment