Commit e19f97ed authored by Mike Rapoport's avatar Mike Rapoport Committed by Linus Torvalds

um: add support for folded p4d page tables

The UML port uses 4 and 5 level fixups to support higher level page
table directories in the generic VM code.

Implement primitives necessary for the 4th level folding, add walks of
p4d level where appropriate and drop usage of __ARCH_USE_5LEVEL_HACK.

Link: http://lkml.kernel.org/r/1572938135-31886-13-git-send-email-rppt@kernel.orgSigned-off-by: default avatarMike Rapoport <rppt@linux.ibm.com>
Cc: Anatoly Pugachev <matorola@gmail.com>
Cc: Anton Ivanov <anton.ivanov@cambridgegreys.com>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: "David S. Miller" <davem@davemloft.net>
Cc: Geert Uytterhoeven <geert@linux-m68k.org>
Cc: Greentime Hu <green.hu@gmail.com>
Cc: Greg Ungerer <gerg@linux-m68k.org>
Cc: Helge Deller <deller@gmx.de>
Cc: "James E.J. Bottomley" <James.Bottomley@HansenPartnership.com>
Cc: Jeff Dike <jdike@addtoit.com>
Cc: "Kirill A. Shutemov" <kirill@shutemov.name>
Cc: Mark Salter <msalter@redhat.com>
Cc: Matt Turner <mattst88@gmail.com>
Cc: Michal Simek <monstr@monstr.eu>
Cc: Peter Rosin <peda@axentia.se>
Cc: Richard Weinberger <richard@nod.at>
Cc: Rolf Eike Beer <eike-kernel@sf-tec.de>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Russell King <rmk+kernel@armlinux.org.uk>
Cc: Sam Creasey <sammy@sammy.net>
Cc: Vincent Chen <deanbo422@gmail.com>
Cc: Vineet Gupta <Vineet.Gupta1@synopsys.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 4e65e76f
...@@ -8,7 +8,6 @@ ...@@ -8,7 +8,6 @@
#ifndef __UM_PGTABLE_2LEVEL_H #ifndef __UM_PGTABLE_2LEVEL_H
#define __UM_PGTABLE_2LEVEL_H #define __UM_PGTABLE_2LEVEL_H
#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopmd.h> #include <asm-generic/pgtable-nopmd.h>
/* PGDIR_SHIFT determines what a third-level page table entry can map */ /* PGDIR_SHIFT determines what a third-level page table entry can map */
......
...@@ -7,7 +7,6 @@ ...@@ -7,7 +7,6 @@
#ifndef __UM_PGTABLE_3LEVEL_H #ifndef __UM_PGTABLE_3LEVEL_H
#define __UM_PGTABLE_3LEVEL_H #define __UM_PGTABLE_3LEVEL_H
#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopud.h> #include <asm-generic/pgtable-nopud.h>
/* PGDIR_SHIFT determines what a third-level page table entry can map */ /* PGDIR_SHIFT determines what a third-level page table entry can map */
......
...@@ -106,6 +106,9 @@ extern unsigned long end_iomem; ...@@ -106,6 +106,9 @@ extern unsigned long end_iomem;
#define pud_newpage(x) (pud_val(x) & _PAGE_NEWPAGE) #define pud_newpage(x) (pud_val(x) & _PAGE_NEWPAGE)
#define pud_mkuptodate(x) (pud_val(x) &= ~_PAGE_NEWPAGE) #define pud_mkuptodate(x) (pud_val(x) &= ~_PAGE_NEWPAGE)
#define p4d_newpage(x) (p4d_val(x) & _PAGE_NEWPAGE)
#define p4d_mkuptodate(x) (p4d_val(x) &= ~_PAGE_NEWPAGE)
#define pmd_page(pmd) phys_to_page(pmd_val(pmd) & PAGE_MASK) #define pmd_page(pmd) phys_to_page(pmd_val(pmd) & PAGE_MASK)
#define pte_page(x) pfn_to_page(pte_pfn(x)) #define pte_page(x) pfn_to_page(pte_pfn(x))
......
...@@ -96,6 +96,7 @@ static void __init fixrange_init(unsigned long start, unsigned long end, ...@@ -96,6 +96,7 @@ static void __init fixrange_init(unsigned long start, unsigned long end,
pgd_t *pgd_base) pgd_t *pgd_base)
{ {
pgd_t *pgd; pgd_t *pgd;
p4d_t *p4d;
pud_t *pud; pud_t *pud;
pmd_t *pmd; pmd_t *pmd;
int i, j; int i, j;
...@@ -107,7 +108,8 @@ static void __init fixrange_init(unsigned long start, unsigned long end, ...@@ -107,7 +108,8 @@ static void __init fixrange_init(unsigned long start, unsigned long end,
pgd = pgd_base + i; pgd = pgd_base + i;
for ( ; (i < PTRS_PER_PGD) && (vaddr < end); pgd++, i++) { for ( ; (i < PTRS_PER_PGD) && (vaddr < end); pgd++, i++) {
pud = pud_offset(pgd, vaddr); p4d = p4d_offset(pgd, vaddr);
pud = pud_offset(p4d, vaddr);
if (pud_none(*pud)) if (pud_none(*pud))
one_md_table_init(pud); one_md_table_init(pud);
pmd = pmd_offset(pud, vaddr); pmd = pmd_offset(pud, vaddr);
...@@ -124,6 +126,7 @@ static void __init fixaddr_user_init( void) ...@@ -124,6 +126,7 @@ static void __init fixaddr_user_init( void)
#ifdef CONFIG_ARCH_REUSE_HOST_VSYSCALL_AREA #ifdef CONFIG_ARCH_REUSE_HOST_VSYSCALL_AREA
long size = FIXADDR_USER_END - FIXADDR_USER_START; long size = FIXADDR_USER_END - FIXADDR_USER_START;
pgd_t *pgd; pgd_t *pgd;
p4d_t *p4d;
pud_t *pud; pud_t *pud;
pmd_t *pmd; pmd_t *pmd;
pte_t *pte; pte_t *pte;
...@@ -144,7 +147,8 @@ static void __init fixaddr_user_init( void) ...@@ -144,7 +147,8 @@ static void __init fixaddr_user_init( void)
for ( ; size > 0; size -= PAGE_SIZE, vaddr += PAGE_SIZE, for ( ; size > 0; size -= PAGE_SIZE, vaddr += PAGE_SIZE,
p += PAGE_SIZE) { p += PAGE_SIZE) {
pgd = swapper_pg_dir + pgd_index(vaddr); pgd = swapper_pg_dir + pgd_index(vaddr);
pud = pud_offset(pgd, vaddr); p4d = p4d_offset(pgd, vaddr);
pud = pud_offset(p4d, vaddr);
pmd = pmd_offset(pud, vaddr); pmd = pmd_offset(pud, vaddr);
pte = pte_offset_kernel(pmd, vaddr); pte = pte_offset_kernel(pmd, vaddr);
pte_set_val(*pte, p, PAGE_READONLY); pte_set_val(*pte, p, PAGE_READONLY);
......
...@@ -19,15 +19,21 @@ static int init_stub_pte(struct mm_struct *mm, unsigned long proc, ...@@ -19,15 +19,21 @@ static int init_stub_pte(struct mm_struct *mm, unsigned long proc,
unsigned long kernel) unsigned long kernel)
{ {
pgd_t *pgd; pgd_t *pgd;
p4d_t *p4d;
pud_t *pud; pud_t *pud;
pmd_t *pmd; pmd_t *pmd;
pte_t *pte; pte_t *pte;
pgd = pgd_offset(mm, proc); pgd = pgd_offset(mm, proc);
pud = pud_alloc(mm, pgd, proc);
if (!pud) p4d = p4d_alloc(mm, pgd, proc);
if (!p4d)
goto out; goto out;
pud = pud_alloc(mm, p4d, proc);
if (!pud)
goto out_pud;
pmd = pmd_alloc(mm, pud, proc); pmd = pmd_alloc(mm, pud, proc);
if (!pmd) if (!pmd)
goto out_pmd; goto out_pmd;
...@@ -44,6 +50,8 @@ static int init_stub_pte(struct mm_struct *mm, unsigned long proc, ...@@ -44,6 +50,8 @@ static int init_stub_pte(struct mm_struct *mm, unsigned long proc,
pmd_free(mm, pmd); pmd_free(mm, pmd);
out_pmd: out_pmd:
pud_free(mm, pud); pud_free(mm, pud);
out_pud:
p4d_free(mm, p4d);
out: out:
return -ENOMEM; return -ENOMEM;
} }
......
...@@ -17,6 +17,7 @@ ...@@ -17,6 +17,7 @@
pte_t *virt_to_pte(struct mm_struct *mm, unsigned long addr) pte_t *virt_to_pte(struct mm_struct *mm, unsigned long addr)
{ {
pgd_t *pgd; pgd_t *pgd;
p4d_t *p4d;
pud_t *pud; pud_t *pud;
pmd_t *pmd; pmd_t *pmd;
...@@ -27,7 +28,11 @@ pte_t *virt_to_pte(struct mm_struct *mm, unsigned long addr) ...@@ -27,7 +28,11 @@ pte_t *virt_to_pte(struct mm_struct *mm, unsigned long addr)
if (!pgd_present(*pgd)) if (!pgd_present(*pgd))
return NULL; return NULL;
pud = pud_offset(pgd, addr); p4d = p4d_offset(pgd, addr);
if (!p4d_present(*p4d))
return NULL;
pud = pud_offset(p4d, addr);
if (!pud_present(*pud)) if (!pud_present(*pud))
return NULL; return NULL;
......
...@@ -277,7 +277,7 @@ static inline int update_pmd_range(pud_t *pud, unsigned long addr, ...@@ -277,7 +277,7 @@ static inline int update_pmd_range(pud_t *pud, unsigned long addr,
return ret; return ret;
} }
static inline int update_pud_range(pgd_t *pgd, unsigned long addr, static inline int update_pud_range(p4d_t *p4d, unsigned long addr,
unsigned long end, unsigned long end,
struct host_vm_change *hvc) struct host_vm_change *hvc)
{ {
...@@ -285,7 +285,7 @@ static inline int update_pud_range(pgd_t *pgd, unsigned long addr, ...@@ -285,7 +285,7 @@ static inline int update_pud_range(pgd_t *pgd, unsigned long addr,
unsigned long next; unsigned long next;
int ret = 0; int ret = 0;
pud = pud_offset(pgd, addr); pud = pud_offset(p4d, addr);
do { do {
next = pud_addr_end(addr, end); next = pud_addr_end(addr, end);
if (!pud_present(*pud)) { if (!pud_present(*pud)) {
...@@ -299,6 +299,28 @@ static inline int update_pud_range(pgd_t *pgd, unsigned long addr, ...@@ -299,6 +299,28 @@ static inline int update_pud_range(pgd_t *pgd, unsigned long addr,
return ret; return ret;
} }
static inline int update_p4d_range(pgd_t *pgd, unsigned long addr,
unsigned long end,
struct host_vm_change *hvc)
{
p4d_t *p4d;
unsigned long next;
int ret = 0;
p4d = p4d_offset(pgd, addr);
do {
next = p4d_addr_end(addr, end);
if (!p4d_present(*p4d)) {
if (hvc->force || p4d_newpage(*p4d)) {
ret = add_munmap(addr, next - addr, hvc);
p4d_mkuptodate(*p4d);
}
} else
ret = update_pud_range(p4d, addr, next, hvc);
} while (p4d++, addr = next, ((addr < end) && !ret));
return ret;
}
void fix_range_common(struct mm_struct *mm, unsigned long start_addr, void fix_range_common(struct mm_struct *mm, unsigned long start_addr,
unsigned long end_addr, int force) unsigned long end_addr, int force)
{ {
...@@ -316,8 +338,8 @@ void fix_range_common(struct mm_struct *mm, unsigned long start_addr, ...@@ -316,8 +338,8 @@ void fix_range_common(struct mm_struct *mm, unsigned long start_addr,
ret = add_munmap(addr, next - addr, &hvc); ret = add_munmap(addr, next - addr, &hvc);
pgd_mkuptodate(*pgd); pgd_mkuptodate(*pgd);
} }
} } else
else ret = update_pud_range(pgd, addr, next, &hvc); ret = update_p4d_range(pgd, addr, next, &hvc);
} while (pgd++, addr = next, ((addr < end_addr) && !ret)); } while (pgd++, addr = next, ((addr < end_addr) && !ret));
if (!ret) if (!ret)
...@@ -338,6 +360,7 @@ static int flush_tlb_kernel_range_common(unsigned long start, unsigned long end) ...@@ -338,6 +360,7 @@ static int flush_tlb_kernel_range_common(unsigned long start, unsigned long end)
{ {
struct mm_struct *mm; struct mm_struct *mm;
pgd_t *pgd; pgd_t *pgd;
p4d_t *p4d;
pud_t *pud; pud_t *pud;
pmd_t *pmd; pmd_t *pmd;
pte_t *pte; pte_t *pte;
...@@ -364,7 +387,23 @@ static int flush_tlb_kernel_range_common(unsigned long start, unsigned long end) ...@@ -364,7 +387,23 @@ static int flush_tlb_kernel_range_common(unsigned long start, unsigned long end)
continue; continue;
} }
pud = pud_offset(pgd, addr); p4d = p4d_offset(pgd, addr);
if (!p4d_present(*p4d)) {
last = ADD_ROUND(addr, P4D_SIZE);
if (last > end)
last = end;
if (p4d_newpage(*p4d)) {
updated = 1;
err = add_munmap(addr, last - addr, &hvc);
if (err < 0)
panic("munmap failed, errno = %d\n",
-err);
}
addr = last;
continue;
}
pud = pud_offset(p4d, addr);
if (!pud_present(*pud)) { if (!pud_present(*pud)) {
last = ADD_ROUND(addr, PUD_SIZE); last = ADD_ROUND(addr, PUD_SIZE);
if (last > end) if (last > end)
...@@ -424,6 +463,7 @@ static int flush_tlb_kernel_range_common(unsigned long start, unsigned long end) ...@@ -424,6 +463,7 @@ static int flush_tlb_kernel_range_common(unsigned long start, unsigned long end)
void flush_tlb_page(struct vm_area_struct *vma, unsigned long address) void flush_tlb_page(struct vm_area_struct *vma, unsigned long address)
{ {
pgd_t *pgd; pgd_t *pgd;
p4d_t *p4d;
pud_t *pud; pud_t *pud;
pmd_t *pmd; pmd_t *pmd;
pte_t *pte; pte_t *pte;
...@@ -437,7 +477,11 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long address) ...@@ -437,7 +477,11 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long address)
if (!pgd_present(*pgd)) if (!pgd_present(*pgd))
goto kill; goto kill;
pud = pud_offset(pgd, address); p4d = p4d_offset(pgd, address);
if (!p4d_present(*p4d))
goto kill;
pud = pud_offset(p4d, address);
if (!pud_present(*pud)) if (!pud_present(*pud))
goto kill; goto kill;
......
...@@ -28,6 +28,7 @@ int handle_page_fault(unsigned long address, unsigned long ip, ...@@ -28,6 +28,7 @@ int handle_page_fault(unsigned long address, unsigned long ip,
struct mm_struct *mm = current->mm; struct mm_struct *mm = current->mm;
struct vm_area_struct *vma; struct vm_area_struct *vma;
pgd_t *pgd; pgd_t *pgd;
p4d_t *p4d;
pud_t *pud; pud_t *pud;
pmd_t *pmd; pmd_t *pmd;
pte_t *pte; pte_t *pte;
...@@ -104,7 +105,8 @@ int handle_page_fault(unsigned long address, unsigned long ip, ...@@ -104,7 +105,8 @@ int handle_page_fault(unsigned long address, unsigned long ip,
} }
pgd = pgd_offset(mm, address); pgd = pgd_offset(mm, address);
pud = pud_offset(pgd, address); p4d = p4d_offset(pgd, address);
pud = pud_offset(p4d, address);
pmd = pmd_offset(pud, address); pmd = pmd_offset(pud, address);
pte = pte_offset_kernel(pmd, address); pte = pte_offset_kernel(pmd, address);
} while (!pte_present(*pte)); } while (!pte_present(*pte));
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment