Commit 6eb82f99 authored by Joerg Roedel's avatar Joerg Roedel Committed by Ingo Molnar

x86/mm: Pre-allocate P4D/PUD pages for vmalloc area

Pre-allocate the page-table pages for the vmalloc area at the level
which needs synchronization on x86-64, which is P4D for 5-level and
PUD for 4-level paging.

Doing this at boot makes sure no synchronization of that area is
necessary at runtime. The synchronization takes the pgd_lock and
iterates over all page-tables in the system, so it can take quite long
and is better avoided.
Signed-off-by: default avatarJoerg Roedel <jroedel@suse.de>
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
Reviewed-by: default avatarMike Rapoport <rppt@linux.ibm.com>
Link: https://lore.kernel.org/r/20200721095953.6218-2-joro@8bytes.org
parent 92ed3019
...@@ -1238,6 +1238,56 @@ static void __init register_page_bootmem_info(void) ...@@ -1238,6 +1238,56 @@ static void __init register_page_bootmem_info(void)
#endif #endif
} }
/*
* Pre-allocates page-table pages for the vmalloc area in the kernel page-table.
* Only the level which needs to be synchronized between all page-tables is
* allocated because the synchronization can be expensive.
*/
static void __init preallocate_vmalloc_pages(void)
{
unsigned long addr;
const char *lvl;
for (addr = VMALLOC_START; addr <= VMALLOC_END; addr = ALIGN(addr + 1, PGDIR_SIZE)) {
pgd_t *pgd = pgd_offset_k(addr);
p4d_t *p4d;
pud_t *pud;
p4d = p4d_offset(pgd, addr);
if (p4d_none(*p4d)) {
/* Can only happen with 5-level paging */
p4d = p4d_alloc(&init_mm, pgd, addr);
if (!p4d) {
lvl = "p4d";
goto failed;
}
}
if (pgtable_l5_enabled())
continue;
pud = pud_offset(p4d, addr);
if (pud_none(*pud)) {
/* Ends up here only with 4-level paging */
pud = pud_alloc(&init_mm, p4d, addr);
if (!pud) {
lvl = "pud";
goto failed;
}
}
}
return;
failed:
/*
* The pages have to be there now or they will be missing in
* process page-tables later.
*/
panic("Failed to pre-allocate %s pages for vmalloc area\n", lvl);
}
void __init mem_init(void) void __init mem_init(void)
{ {
pci_iommu_alloc(); pci_iommu_alloc();
...@@ -1261,6 +1311,8 @@ void __init mem_init(void) ...@@ -1261,6 +1311,8 @@ void __init mem_init(void)
if (get_gate_vma(&init_mm)) if (get_gate_vma(&init_mm))
kclist_add(&kcore_vsyscall, (void *)VSYSCALL_ADDR, PAGE_SIZE, KCORE_USER); kclist_add(&kcore_vsyscall, (void *)VSYSCALL_ADDR, PAGE_SIZE, KCORE_USER);
preallocate_vmalloc_pages();
mem_init_print_info(NULL); mem_init_print_info(NULL);
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment