Commit 92a0f81d authored by Thomas Gleixner's avatar Thomas Gleixner Committed by Ingo Molnar

x86/cpu_entry_area: Move it out of the fixmap

Put the cpu_entry_area into a separate P4D entry. The fixmap gets too big
and 0-day already hit a case where the fixmap PTEs were cleared by
cleanup_highmap().

Aside of that the fixmap API is a pain as it's all backwards.
Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent ed1bbc40
...@@ -12,6 +12,7 @@ ffffea0000000000 - ffffeaffffffffff (=40 bits) virtual memory map (1TB) ...@@ -12,6 +12,7 @@ ffffea0000000000 - ffffeaffffffffff (=40 bits) virtual memory map (1TB)
... unused hole ... ... unused hole ...
ffffec0000000000 - fffffbffffffffff (=44 bits) kasan shadow memory (16TB) ffffec0000000000 - fffffbffffffffff (=44 bits) kasan shadow memory (16TB)
... unused hole ... ... unused hole ...
fffffe8000000000 - fffffeffffffffff (=39 bits) cpu_entry_area mapping
ffffff0000000000 - ffffff7fffffffff (=39 bits) %esp fixup stacks ffffff0000000000 - ffffff7fffffffff (=39 bits) %esp fixup stacks
... unused hole ... ... unused hole ...
ffffffef00000000 - fffffffeffffffff (=64 GB) EFI region mapping space ffffffef00000000 - fffffffeffffffff (=64 GB) EFI region mapping space
...@@ -35,6 +36,7 @@ ffd4000000000000 - ffd5ffffffffffff (=49 bits) virtual memory map (512TB) ...@@ -35,6 +36,7 @@ ffd4000000000000 - ffd5ffffffffffff (=49 bits) virtual memory map (512TB)
... unused hole ... ... unused hole ...
ffdf000000000000 - fffffc0000000000 (=53 bits) kasan shadow memory (8PB) ffdf000000000000 - fffffc0000000000 (=53 bits) kasan shadow memory (8PB)
... unused hole ... ... unused hole ...
fffffe8000000000 - fffffeffffffffff (=39 bits) cpu_entry_area mapping
ffffff0000000000 - ffffff7fffffffff (=39 bits) %esp fixup stacks ffffff0000000000 - ffffff7fffffffff (=39 bits) %esp fixup stacks
... unused hole ... ... unused hole ...
ffffffef00000000 - fffffffeffffffff (=64 GB) EFI region mapping space ffffffef00000000 - fffffffeffffffff (=64 GB) EFI region mapping space
......
...@@ -43,10 +43,26 @@ struct cpu_entry_area { ...@@ -43,10 +43,26 @@ struct cpu_entry_area {
}; };
#define CPU_ENTRY_AREA_SIZE (sizeof(struct cpu_entry_area)) #define CPU_ENTRY_AREA_SIZE (sizeof(struct cpu_entry_area))
#define CPU_ENTRY_AREA_PAGES (CPU_ENTRY_AREA_SIZE / PAGE_SIZE) #define CPU_ENTRY_AREA_TOT_SIZE (CPU_ENTRY_AREA_SIZE * NR_CPUS)
DECLARE_PER_CPU(struct cpu_entry_area *, cpu_entry_area); DECLARE_PER_CPU(struct cpu_entry_area *, cpu_entry_area);
extern void setup_cpu_entry_areas(void); extern void setup_cpu_entry_areas(void);
extern void cea_set_pte(void *cea_vaddr, phys_addr_t pa, pgprot_t flags);
#define CPU_ENTRY_AREA_RO_IDT CPU_ENTRY_AREA_BASE
#define CPU_ENTRY_AREA_PER_CPU (CPU_ENTRY_AREA_RO_IDT + PAGE_SIZE)
#define CPU_ENTRY_AREA_RO_IDT_VADDR ((void *)CPU_ENTRY_AREA_RO_IDT)
#define CPU_ENTRY_AREA_MAP_SIZE \
(CPU_ENTRY_AREA_PER_CPU + CPU_ENTRY_AREA_TOT_SIZE - CPU_ENTRY_AREA_BASE)
extern struct cpu_entry_area *get_cpu_entry_area(int cpu);
static inline struct entry_stack *cpu_entry_stack(int cpu)
{
return &get_cpu_entry_area(cpu)->entry_stack_page.stack;
}
#endif #endif
...@@ -7,6 +7,7 @@ ...@@ -7,6 +7,7 @@
#include <asm/mmu.h> #include <asm/mmu.h>
#include <asm/fixmap.h> #include <asm/fixmap.h>
#include <asm/irq_vectors.h> #include <asm/irq_vectors.h>
#include <asm/cpu_entry_area.h>
#include <linux/smp.h> #include <linux/smp.h>
#include <linux/percpu.h> #include <linux/percpu.h>
......
...@@ -25,7 +25,6 @@ ...@@ -25,7 +25,6 @@
#else #else
#include <uapi/asm/vsyscall.h> #include <uapi/asm/vsyscall.h>
#endif #endif
#include <asm/cpu_entry_area.h>
/* /*
* We can't declare FIXADDR_TOP as variable for x86_64 because vsyscall * We can't declare FIXADDR_TOP as variable for x86_64 because vsyscall
...@@ -84,7 +83,6 @@ enum fixed_addresses { ...@@ -84,7 +83,6 @@ enum fixed_addresses {
FIX_IO_APIC_BASE_0, FIX_IO_APIC_BASE_0,
FIX_IO_APIC_BASE_END = FIX_IO_APIC_BASE_0 + MAX_IO_APICS - 1, FIX_IO_APIC_BASE_END = FIX_IO_APIC_BASE_0 + MAX_IO_APICS - 1,
#endif #endif
FIX_RO_IDT, /* Virtual mapping for read-only IDT */
#ifdef CONFIG_X86_32 #ifdef CONFIG_X86_32
FIX_KMAP_BEGIN, /* reserved pte's for temporary kernel mappings */ FIX_KMAP_BEGIN, /* reserved pte's for temporary kernel mappings */
FIX_KMAP_END = FIX_KMAP_BEGIN+(KM_TYPE_NR*NR_CPUS)-1, FIX_KMAP_END = FIX_KMAP_BEGIN+(KM_TYPE_NR*NR_CPUS)-1,
...@@ -100,9 +98,6 @@ enum fixed_addresses { ...@@ -100,9 +98,6 @@ enum fixed_addresses {
#ifdef CONFIG_X86_INTEL_MID #ifdef CONFIG_X86_INTEL_MID
FIX_LNW_VRTC, FIX_LNW_VRTC,
#endif #endif
/* Fixmap entries to remap the GDTs, one per processor. */
FIX_CPU_ENTRY_AREA_TOP,
FIX_CPU_ENTRY_AREA_BOTTOM = FIX_CPU_ENTRY_AREA_TOP + (CPU_ENTRY_AREA_PAGES * NR_CPUS) - 1,
#ifdef CONFIG_ACPI_APEI_GHES #ifdef CONFIG_ACPI_APEI_GHES
/* Used for GHES mapping from assorted contexts */ /* Used for GHES mapping from assorted contexts */
...@@ -143,7 +138,7 @@ enum fixed_addresses { ...@@ -143,7 +138,7 @@ enum fixed_addresses {
extern void reserve_top_address(unsigned long reserve); extern void reserve_top_address(unsigned long reserve);
#define FIXADDR_SIZE (__end_of_permanent_fixed_addresses << PAGE_SHIFT) #define FIXADDR_SIZE (__end_of_permanent_fixed_addresses << PAGE_SHIFT)
#define FIXADDR_START (FIXADDR_TOP - FIXADDR_SIZE) #define FIXADDR_START (FIXADDR_TOP - FIXADDR_SIZE)
extern int fixmaps_set; extern int fixmaps_set;
...@@ -191,30 +186,5 @@ void __init *early_memremap_decrypted_wp(resource_size_t phys_addr, ...@@ -191,30 +186,5 @@ void __init *early_memremap_decrypted_wp(resource_size_t phys_addr,
void __early_set_fixmap(enum fixed_addresses idx, void __early_set_fixmap(enum fixed_addresses idx,
phys_addr_t phys, pgprot_t flags); phys_addr_t phys, pgprot_t flags);
static inline unsigned int __get_cpu_entry_area_page_index(int cpu, int page)
{
BUILD_BUG_ON(sizeof(struct cpu_entry_area) % PAGE_SIZE != 0);
return FIX_CPU_ENTRY_AREA_BOTTOM - cpu*CPU_ENTRY_AREA_PAGES - page;
}
#define __get_cpu_entry_area_offset_index(cpu, offset) ({ \
BUILD_BUG_ON(offset % PAGE_SIZE != 0); \
__get_cpu_entry_area_page_index(cpu, offset / PAGE_SIZE); \
})
#define get_cpu_entry_area_index(cpu, field) \
__get_cpu_entry_area_offset_index((cpu), offsetof(struct cpu_entry_area, field))
static inline struct cpu_entry_area *get_cpu_entry_area(int cpu)
{
return (struct cpu_entry_area *)__fix_to_virt(__get_cpu_entry_area_page_index(cpu, 0));
}
static inline struct entry_stack *cpu_entry_stack(int cpu)
{
return &get_cpu_entry_area(cpu)->entry_stack_page.stack;
}
#endif /* !__ASSEMBLY__ */ #endif /* !__ASSEMBLY__ */
#endif /* _ASM_X86_FIXMAP_H */ #endif /* _ASM_X86_FIXMAP_H */
...@@ -38,13 +38,22 @@ extern bool __vmalloc_start_set; /* set once high_memory is set */ ...@@ -38,13 +38,22 @@ extern bool __vmalloc_start_set; /* set once high_memory is set */
#define LAST_PKMAP 1024 #define LAST_PKMAP 1024
#endif #endif
#define PKMAP_BASE ((FIXADDR_START - PAGE_SIZE * (LAST_PKMAP + 1)) \ /*
& PMD_MASK) * Define this here and validate with BUILD_BUG_ON() in pgtable_32.c
* to avoid include recursion hell
*/
#define CPU_ENTRY_AREA_PAGES (NR_CPUS * 40)
#define CPU_ENTRY_AREA_BASE \
((FIXADDR_START - PAGE_SIZE * (CPU_ENTRY_AREA_PAGES + 1)) & PMD_MASK)
#define PKMAP_BASE \
((CPU_ENTRY_AREA_BASE - PAGE_SIZE) & PMD_MASK)
#ifdef CONFIG_HIGHMEM #ifdef CONFIG_HIGHMEM
# define VMALLOC_END (PKMAP_BASE - 2 * PAGE_SIZE) # define VMALLOC_END (PKMAP_BASE - 2 * PAGE_SIZE)
#else #else
# define VMALLOC_END (FIXADDR_START - 2 * PAGE_SIZE) # define VMALLOC_END (CPU_ENTRY_AREA_BASE - 2 * PAGE_SIZE)
#endif #endif
#define MODULES_VADDR VMALLOC_START #define MODULES_VADDR VMALLOC_START
......
...@@ -76,32 +76,41 @@ typedef struct { pteval_t pte; } pte_t; ...@@ -76,32 +76,41 @@ typedef struct { pteval_t pte; } pte_t;
#define PGDIR_MASK (~(PGDIR_SIZE - 1)) #define PGDIR_MASK (~(PGDIR_SIZE - 1))
/* See Documentation/x86/x86_64/mm.txt for a description of the memory map. */ /* See Documentation/x86/x86_64/mm.txt for a description of the memory map. */
#define MAXMEM _AC(__AC(1, UL) << MAX_PHYSMEM_BITS, UL) #define MAXMEM _AC(__AC(1, UL) << MAX_PHYSMEM_BITS, UL)
#ifdef CONFIG_X86_5LEVEL #ifdef CONFIG_X86_5LEVEL
#define VMALLOC_SIZE_TB _AC(16384, UL) # define VMALLOC_SIZE_TB _AC(16384, UL)
#define __VMALLOC_BASE _AC(0xff92000000000000, UL) # define __VMALLOC_BASE _AC(0xff92000000000000, UL)
#define __VMEMMAP_BASE _AC(0xffd4000000000000, UL) # define __VMEMMAP_BASE _AC(0xffd4000000000000, UL)
#else #else
#define VMALLOC_SIZE_TB _AC(32, UL) # define VMALLOC_SIZE_TB _AC(32, UL)
#define __VMALLOC_BASE _AC(0xffffc90000000000, UL) # define __VMALLOC_BASE _AC(0xffffc90000000000, UL)
#define __VMEMMAP_BASE _AC(0xffffea0000000000, UL) # define __VMEMMAP_BASE _AC(0xffffea0000000000, UL)
#endif #endif
#ifdef CONFIG_RANDOMIZE_MEMORY #ifdef CONFIG_RANDOMIZE_MEMORY
#define VMALLOC_START vmalloc_base # define VMALLOC_START vmalloc_base
#define VMEMMAP_START vmemmap_base # define VMEMMAP_START vmemmap_base
#else #else
#define VMALLOC_START __VMALLOC_BASE # define VMALLOC_START __VMALLOC_BASE
#define VMEMMAP_START __VMEMMAP_BASE # define VMEMMAP_START __VMEMMAP_BASE
#endif /* CONFIG_RANDOMIZE_MEMORY */ #endif /* CONFIG_RANDOMIZE_MEMORY */
#define VMALLOC_END (VMALLOC_START + _AC((VMALLOC_SIZE_TB << 40) - 1, UL))
#define MODULES_VADDR (__START_KERNEL_map + KERNEL_IMAGE_SIZE) #define VMALLOC_END (VMALLOC_START + _AC((VMALLOC_SIZE_TB << 40) - 1, UL))
#define MODULES_VADDR (__START_KERNEL_map + KERNEL_IMAGE_SIZE)
/* The module sections ends with the start of the fixmap */ /* The module sections ends with the start of the fixmap */
#define MODULES_END __fix_to_virt(__end_of_fixed_addresses + 1) #define MODULES_END __fix_to_virt(__end_of_fixed_addresses + 1)
#define MODULES_LEN (MODULES_END - MODULES_VADDR) #define MODULES_LEN (MODULES_END - MODULES_VADDR)
#define ESPFIX_PGD_ENTRY _AC(-2, UL)
#define ESPFIX_BASE_ADDR (ESPFIX_PGD_ENTRY << P4D_SHIFT) #define ESPFIX_PGD_ENTRY _AC(-2, UL)
#define EFI_VA_START ( -4 * (_AC(1, UL) << 30)) #define ESPFIX_BASE_ADDR (ESPFIX_PGD_ENTRY << P4D_SHIFT)
#define EFI_VA_END (-68 * (_AC(1, UL) << 30))
#define CPU_ENTRY_AREA_PGD _AC(-3, UL)
#define CPU_ENTRY_AREA_BASE (CPU_ENTRY_AREA_PGD << P4D_SHIFT)
#define EFI_VA_START ( -4 * (_AC(1, UL) << 30))
#define EFI_VA_END (-68 * (_AC(1, UL) << 30))
#define EARLY_DYNAMIC_PAGE_TABLES 64 #define EARLY_DYNAMIC_PAGE_TABLES 64
......
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
#include <linux/nmi.h> #include <linux/nmi.h>
#include <linux/sysfs.h> #include <linux/sysfs.h>
#include <asm/cpu_entry_area.h>
#include <asm/stacktrace.h> #include <asm/stacktrace.h>
#include <asm/unwind.h> #include <asm/unwind.h>
......
...@@ -951,8 +951,9 @@ void __init trap_init(void) ...@@ -951,8 +951,9 @@ void __init trap_init(void)
* "sidt" instruction will not leak the location of the kernel, and * "sidt" instruction will not leak the location of the kernel, and
* to defend the IDT against arbitrary memory write vulnerabilities. * to defend the IDT against arbitrary memory write vulnerabilities.
* It will be reloaded in cpu_init() */ * It will be reloaded in cpu_init() */
__set_fixmap(FIX_RO_IDT, __pa_symbol(idt_table), PAGE_KERNEL_RO); cea_set_pte(CPU_ENTRY_AREA_RO_IDT_VADDR, __pa_symbol(idt_table),
idt_descr.address = fix_to_virt(FIX_RO_IDT); PAGE_KERNEL_RO);
idt_descr.address = CPU_ENTRY_AREA_RO_IDT;
/* /*
* Should be a barrier for any external CPU state: * Should be a barrier for any external CPU state:
......
...@@ -15,11 +15,27 @@ static DEFINE_PER_CPU_PAGE_ALIGNED(char, exception_stacks ...@@ -15,11 +15,27 @@ static DEFINE_PER_CPU_PAGE_ALIGNED(char, exception_stacks
[(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ]); [(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ]);
#endif #endif
struct cpu_entry_area *get_cpu_entry_area(int cpu)
{
unsigned long va = CPU_ENTRY_AREA_PER_CPU + cpu * CPU_ENTRY_AREA_SIZE;
BUILD_BUG_ON(sizeof(struct cpu_entry_area) % PAGE_SIZE != 0);
return (struct cpu_entry_area *) va;
}
EXPORT_SYMBOL(get_cpu_entry_area);
void cea_set_pte(void *cea_vaddr, phys_addr_t pa, pgprot_t flags)
{
unsigned long va = (unsigned long) cea_vaddr;
set_pte_vaddr(va, pfn_pte(pa >> PAGE_SHIFT, flags));
}
static void __init static void __init
set_percpu_fixmap_pages(int idx, void *ptr, int pages, pgprot_t prot) cea_map_percpu_pages(void *cea_vaddr, void *ptr, int pages, pgprot_t prot)
{ {
for ( ; pages; pages--, idx--, ptr += PAGE_SIZE) for ( ; pages; pages--, cea_vaddr+= PAGE_SIZE, ptr += PAGE_SIZE)
__set_fixmap(idx, per_cpu_ptr_to_phys(ptr), prot); cea_set_pte(cea_vaddr, per_cpu_ptr_to_phys(ptr), prot);
} }
/* Setup the fixmap mappings only once per-processor */ /* Setup the fixmap mappings only once per-processor */
...@@ -47,10 +63,12 @@ static void __init setup_cpu_entry_area(int cpu) ...@@ -47,10 +63,12 @@ static void __init setup_cpu_entry_area(int cpu)
pgprot_t tss_prot = PAGE_KERNEL; pgprot_t tss_prot = PAGE_KERNEL;
#endif #endif
__set_fixmap(get_cpu_entry_area_index(cpu, gdt), get_cpu_gdt_paddr(cpu), gdt_prot); cea_set_pte(&get_cpu_entry_area(cpu)->gdt, get_cpu_gdt_paddr(cpu),
set_percpu_fixmap_pages(get_cpu_entry_area_index(cpu, entry_stack_page), gdt_prot);
per_cpu_ptr(&entry_stack_storage, cpu), 1,
PAGE_KERNEL); cea_map_percpu_pages(&get_cpu_entry_area(cpu)->entry_stack_page,
per_cpu_ptr(&entry_stack_storage, cpu), 1,
PAGE_KERNEL);
/* /*
* The Intel SDM says (Volume 3, 7.2.1): * The Intel SDM says (Volume 3, 7.2.1):
...@@ -72,10 +90,9 @@ static void __init setup_cpu_entry_area(int cpu) ...@@ -72,10 +90,9 @@ static void __init setup_cpu_entry_area(int cpu)
BUILD_BUG_ON((offsetof(struct tss_struct, x86_tss) ^ BUILD_BUG_ON((offsetof(struct tss_struct, x86_tss) ^
offsetofend(struct tss_struct, x86_tss)) & PAGE_MASK); offsetofend(struct tss_struct, x86_tss)) & PAGE_MASK);
BUILD_BUG_ON(sizeof(struct tss_struct) % PAGE_SIZE != 0); BUILD_BUG_ON(sizeof(struct tss_struct) % PAGE_SIZE != 0);
set_percpu_fixmap_pages(get_cpu_entry_area_index(cpu, tss), cea_map_percpu_pages(&get_cpu_entry_area(cpu)->tss,
&per_cpu(cpu_tss_rw, cpu), &per_cpu(cpu_tss_rw, cpu),
sizeof(struct tss_struct) / PAGE_SIZE, sizeof(struct tss_struct) / PAGE_SIZE, tss_prot);
tss_prot);
#ifdef CONFIG_X86_32 #ifdef CONFIG_X86_32
per_cpu(cpu_entry_area, cpu) = get_cpu_entry_area(cpu); per_cpu(cpu_entry_area, cpu) = get_cpu_entry_area(cpu);
...@@ -85,20 +102,37 @@ static void __init setup_cpu_entry_area(int cpu) ...@@ -85,20 +102,37 @@ static void __init setup_cpu_entry_area(int cpu)
BUILD_BUG_ON(sizeof(exception_stacks) % PAGE_SIZE != 0); BUILD_BUG_ON(sizeof(exception_stacks) % PAGE_SIZE != 0);
BUILD_BUG_ON(sizeof(exception_stacks) != BUILD_BUG_ON(sizeof(exception_stacks) !=
sizeof(((struct cpu_entry_area *)0)->exception_stacks)); sizeof(((struct cpu_entry_area *)0)->exception_stacks));
set_percpu_fixmap_pages(get_cpu_entry_area_index(cpu, exception_stacks), cea_map_percpu_pages(&get_cpu_entry_area(cpu)->exception_stacks,
&per_cpu(exception_stacks, cpu), &per_cpu(exception_stacks, cpu),
sizeof(exception_stacks) / PAGE_SIZE, sizeof(exception_stacks) / PAGE_SIZE, PAGE_KERNEL);
PAGE_KERNEL);
__set_fixmap(get_cpu_entry_area_index(cpu, entry_trampoline), cea_set_pte(&get_cpu_entry_area(cpu)->entry_trampoline,
__pa_symbol(_entry_trampoline), PAGE_KERNEL_RX); __pa_symbol(_entry_trampoline), PAGE_KERNEL_RX);
#endif #endif
} }
static __init void setup_cpu_entry_area_ptes(void)
{
#ifdef CONFIG_X86_32
unsigned long start, end;
BUILD_BUG_ON(CPU_ENTRY_AREA_PAGES * PAGE_SIZE < CPU_ENTRY_AREA_MAP_SIZE);
BUG_ON(CPU_ENTRY_AREA_BASE & ~PMD_MASK);
start = CPU_ENTRY_AREA_BASE;
end = start + CPU_ENTRY_AREA_MAP_SIZE;
for (; start < end; start += PMD_SIZE)
populate_extra_pte(start);
#endif
}
void __init setup_cpu_entry_areas(void) void __init setup_cpu_entry_areas(void)
{ {
unsigned int cpu; unsigned int cpu;
setup_cpu_entry_area_ptes();
for_each_possible_cpu(cpu) for_each_possible_cpu(cpu)
setup_cpu_entry_area(cpu); setup_cpu_entry_area(cpu);
} }
...@@ -58,6 +58,7 @@ enum address_markers_idx { ...@@ -58,6 +58,7 @@ enum address_markers_idx {
KASAN_SHADOW_START_NR, KASAN_SHADOW_START_NR,
KASAN_SHADOW_END_NR, KASAN_SHADOW_END_NR,
#endif #endif
CPU_ENTRY_AREA_NR,
#ifdef CONFIG_X86_ESPFIX64 #ifdef CONFIG_X86_ESPFIX64
ESPFIX_START_NR, ESPFIX_START_NR,
#endif #endif
...@@ -81,6 +82,7 @@ static struct addr_marker address_markers[] = { ...@@ -81,6 +82,7 @@ static struct addr_marker address_markers[] = {
[KASAN_SHADOW_START_NR] = { KASAN_SHADOW_START, "KASAN shadow" }, [KASAN_SHADOW_START_NR] = { KASAN_SHADOW_START, "KASAN shadow" },
[KASAN_SHADOW_END_NR] = { KASAN_SHADOW_END, "KASAN shadow end" }, [KASAN_SHADOW_END_NR] = { KASAN_SHADOW_END, "KASAN shadow end" },
#endif #endif
[CPU_ENTRY_AREA_NR] = { CPU_ENTRY_AREA_BASE,"CPU entry Area" },
#ifdef CONFIG_X86_ESPFIX64 #ifdef CONFIG_X86_ESPFIX64
[ESPFIX_START_NR] = { ESPFIX_BASE_ADDR, "ESPfix Area", 16 }, [ESPFIX_START_NR] = { ESPFIX_BASE_ADDR, "ESPfix Area", 16 },
#endif #endif
...@@ -104,6 +106,7 @@ enum address_markers_idx { ...@@ -104,6 +106,7 @@ enum address_markers_idx {
#ifdef CONFIG_HIGHMEM #ifdef CONFIG_HIGHMEM
PKMAP_BASE_NR, PKMAP_BASE_NR,
#endif #endif
CPU_ENTRY_AREA_NR,
FIXADDR_START_NR, FIXADDR_START_NR,
END_OF_SPACE_NR, END_OF_SPACE_NR,
}; };
...@@ -116,6 +119,7 @@ static struct addr_marker address_markers[] = { ...@@ -116,6 +119,7 @@ static struct addr_marker address_markers[] = {
#ifdef CONFIG_HIGHMEM #ifdef CONFIG_HIGHMEM
[PKMAP_BASE_NR] = { 0UL, "Persistent kmap() Area" }, [PKMAP_BASE_NR] = { 0UL, "Persistent kmap() Area" },
#endif #endif
[CPU_ENTRY_AREA_NR] = { 0UL, "CPU entry area" },
[FIXADDR_START_NR] = { 0UL, "Fixmap area" }, [FIXADDR_START_NR] = { 0UL, "Fixmap area" },
[END_OF_SPACE_NR] = { -1, NULL } [END_OF_SPACE_NR] = { -1, NULL }
}; };
...@@ -541,8 +545,8 @@ static int __init pt_dump_init(void) ...@@ -541,8 +545,8 @@ static int __init pt_dump_init(void)
address_markers[PKMAP_BASE_NR].start_address = PKMAP_BASE; address_markers[PKMAP_BASE_NR].start_address = PKMAP_BASE;
# endif # endif
address_markers[FIXADDR_START_NR].start_address = FIXADDR_START; address_markers[FIXADDR_START_NR].start_address = FIXADDR_START;
address_markers[CPU_ENTRY_AREA_NR].start_address = CPU_ENTRY_AREA_BASE;
#endif #endif
return 0; return 0;
} }
__initcall(pt_dump_init); __initcall(pt_dump_init);
...@@ -50,6 +50,7 @@ ...@@ -50,6 +50,7 @@
#include <asm/setup.h> #include <asm/setup.h>
#include <asm/set_memory.h> #include <asm/set_memory.h>
#include <asm/page_types.h> #include <asm/page_types.h>
#include <asm/cpu_entry_area.h>
#include <asm/init.h> #include <asm/init.h>
#include "mm_internal.h" #include "mm_internal.h"
...@@ -766,6 +767,7 @@ void __init mem_init(void) ...@@ -766,6 +767,7 @@ void __init mem_init(void)
mem_init_print_info(NULL); mem_init_print_info(NULL);
printk(KERN_INFO "virtual kernel memory layout:\n" printk(KERN_INFO "virtual kernel memory layout:\n"
" fixmap : 0x%08lx - 0x%08lx (%4ld kB)\n" " fixmap : 0x%08lx - 0x%08lx (%4ld kB)\n"
" cpu_entry : 0x%08lx - 0x%08lx (%4ld kB)\n"
#ifdef CONFIG_HIGHMEM #ifdef CONFIG_HIGHMEM
" pkmap : 0x%08lx - 0x%08lx (%4ld kB)\n" " pkmap : 0x%08lx - 0x%08lx (%4ld kB)\n"
#endif #endif
...@@ -777,6 +779,10 @@ void __init mem_init(void) ...@@ -777,6 +779,10 @@ void __init mem_init(void)
FIXADDR_START, FIXADDR_TOP, FIXADDR_START, FIXADDR_TOP,
(FIXADDR_TOP - FIXADDR_START) >> 10, (FIXADDR_TOP - FIXADDR_START) >> 10,
CPU_ENTRY_AREA_BASE,
CPU_ENTRY_AREA_BASE + CPU_ENTRY_AREA_MAP_SIZE,
CPU_ENTRY_AREA_MAP_SIZE >> 10,
#ifdef CONFIG_HIGHMEM #ifdef CONFIG_HIGHMEM
PKMAP_BASE, PKMAP_BASE+LAST_PKMAP*PAGE_SIZE, PKMAP_BASE, PKMAP_BASE+LAST_PKMAP*PAGE_SIZE,
(LAST_PKMAP*PAGE_SIZE) >> 10, (LAST_PKMAP*PAGE_SIZE) >> 10,
......
...@@ -15,6 +15,7 @@ ...@@ -15,6 +15,7 @@
#include <asm/tlbflush.h> #include <asm/tlbflush.h>
#include <asm/sections.h> #include <asm/sections.h>
#include <asm/pgtable.h> #include <asm/pgtable.h>
#include <asm/cpu_entry_area.h>
extern struct range pfn_mapped[E820_MAX_ENTRIES]; extern struct range pfn_mapped[E820_MAX_ENTRIES];
...@@ -322,31 +323,33 @@ void __init kasan_init(void) ...@@ -322,31 +323,33 @@ void __init kasan_init(void)
map_range(&pfn_mapped[i]); map_range(&pfn_mapped[i]);
} }
kasan_populate_zero_shadow( shadow_cpu_entry_begin = (void *)CPU_ENTRY_AREA_BASE;
kasan_mem_to_shadow((void *)PAGE_OFFSET + MAXMEM),
kasan_mem_to_shadow((void *)__START_KERNEL_map));
kasan_populate_shadow((unsigned long)kasan_mem_to_shadow(_stext),
(unsigned long)kasan_mem_to_shadow(_end),
early_pfn_to_nid(__pa(_stext)));
shadow_cpu_entry_begin = (void *)__fix_to_virt(FIX_CPU_ENTRY_AREA_BOTTOM);
shadow_cpu_entry_begin = kasan_mem_to_shadow(shadow_cpu_entry_begin); shadow_cpu_entry_begin = kasan_mem_to_shadow(shadow_cpu_entry_begin);
shadow_cpu_entry_begin = (void *)round_down((unsigned long)shadow_cpu_entry_begin, shadow_cpu_entry_begin = (void *)round_down((unsigned long)shadow_cpu_entry_begin,
PAGE_SIZE); PAGE_SIZE);
shadow_cpu_entry_end = (void *)(__fix_to_virt(FIX_CPU_ENTRY_AREA_TOP) + PAGE_SIZE); shadow_cpu_entry_end = (void *)(CPU_ENTRY_AREA_BASE +
CPU_ENTRY_AREA_MAP_SIZE);
shadow_cpu_entry_end = kasan_mem_to_shadow(shadow_cpu_entry_end); shadow_cpu_entry_end = kasan_mem_to_shadow(shadow_cpu_entry_end);
shadow_cpu_entry_end = (void *)round_up((unsigned long)shadow_cpu_entry_end, shadow_cpu_entry_end = (void *)round_up((unsigned long)shadow_cpu_entry_end,
PAGE_SIZE); PAGE_SIZE);
kasan_populate_zero_shadow(kasan_mem_to_shadow((void *)MODULES_END), kasan_populate_zero_shadow(
shadow_cpu_entry_begin); kasan_mem_to_shadow((void *)PAGE_OFFSET + MAXMEM),
shadow_cpu_entry_begin);
kasan_populate_shadow((unsigned long)shadow_cpu_entry_begin, kasan_populate_shadow((unsigned long)shadow_cpu_entry_begin,
(unsigned long)shadow_cpu_entry_end, 0); (unsigned long)shadow_cpu_entry_end, 0);
kasan_populate_zero_shadow(shadow_cpu_entry_end, (void *)KASAN_SHADOW_END); kasan_populate_zero_shadow(shadow_cpu_entry_end,
kasan_mem_to_shadow((void *)__START_KERNEL_map));
kasan_populate_shadow((unsigned long)kasan_mem_to_shadow(_stext),
(unsigned long)kasan_mem_to_shadow(_end),
early_pfn_to_nid(__pa(_stext)));
kasan_populate_zero_shadow(kasan_mem_to_shadow((void *)MODULES_END),
(void *)KASAN_SHADOW_END);
load_cr3(init_top_pgt); load_cr3(init_top_pgt);
__flush_tlb_all(); __flush_tlb_all();
......
...@@ -10,6 +10,7 @@ ...@@ -10,6 +10,7 @@
#include <linux/pagemap.h> #include <linux/pagemap.h>
#include <linux/spinlock.h> #include <linux/spinlock.h>
#include <asm/cpu_entry_area.h>
#include <asm/pgtable.h> #include <asm/pgtable.h>
#include <asm/pgalloc.h> #include <asm/pgalloc.h>
#include <asm/fixmap.h> #include <asm/fixmap.h>
......
...@@ -2261,7 +2261,6 @@ static void xen_set_fixmap(unsigned idx, phys_addr_t phys, pgprot_t prot) ...@@ -2261,7 +2261,6 @@ static void xen_set_fixmap(unsigned idx, phys_addr_t phys, pgprot_t prot)
switch (idx) { switch (idx) {
case FIX_BTMAP_END ... FIX_BTMAP_BEGIN: case FIX_BTMAP_END ... FIX_BTMAP_BEGIN:
case FIX_RO_IDT:
#ifdef CONFIG_X86_32 #ifdef CONFIG_X86_32
case FIX_WP_TEST: case FIX_WP_TEST:
# ifdef CONFIG_HIGHMEM # ifdef CONFIG_HIGHMEM
...@@ -2272,7 +2271,6 @@ static void xen_set_fixmap(unsigned idx, phys_addr_t phys, pgprot_t prot) ...@@ -2272,7 +2271,6 @@ static void xen_set_fixmap(unsigned idx, phys_addr_t phys, pgprot_t prot)
#endif #endif
case FIX_TEXT_POKE0: case FIX_TEXT_POKE0:
case FIX_TEXT_POKE1: case FIX_TEXT_POKE1:
case FIX_CPU_ENTRY_AREA_TOP ... FIX_CPU_ENTRY_AREA_BOTTOM:
/* All local page mappings */ /* All local page mappings */
pte = pfn_pte(phys, prot); pte = pfn_pte(phys, prot);
break; break;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment