Commit 0cc2dc49 authored by Mike Rapoport (IBM)'s avatar Mike Rapoport (IBM) Committed by Luis Chamberlain

arch: make execmem setup available regardless of CONFIG_MODULES

execmem does not depend on modules, on the contrary modules use
execmem.

To make execmem available when CONFIG_MODULES=n, for instance for
kprobes, split execmem_params initialization out from
arch/*/kernel/module.c and compile it when CONFIG_EXECMEM=y
Signed-off-by: default avatarMike Rapoport (IBM) <rppt@kernel.org>
Reviewed-by: default avatarPhilippe Mathieu-Daudé <philmd@linaro.org>
Signed-off-by: default avatarLuis Chamberlain <mcgrof@kernel.org>
parent 1b750c2f
...@@ -12,57 +12,14 @@ ...@@ -12,57 +12,14 @@
#include <linux/kernel.h> #include <linux/kernel.h>
#include <linux/mm.h> #include <linux/mm.h>
#include <linux/elf.h> #include <linux/elf.h>
#include <linux/vmalloc.h>
#include <linux/fs.h> #include <linux/fs.h>
#include <linux/string.h> #include <linux/string.h>
#include <linux/gfp.h>
#include <linux/execmem.h>
#include <asm/sections.h> #include <asm/sections.h>
#include <asm/smp_plat.h> #include <asm/smp_plat.h>
#include <asm/unwind.h> #include <asm/unwind.h>
#include <asm/opcodes.h> #include <asm/opcodes.h>
#ifdef CONFIG_XIP_KERNEL
/*
* The XIP kernel text is mapped in the module area for modules and
* some other stuff to work without any indirect relocations.
* MODULES_VADDR is redefined here and not in asm/memory.h to avoid
* recompiling the whole kernel when CONFIG_XIP_KERNEL is turned on/off.
*/
#undef MODULES_VADDR
#define MODULES_VADDR (((unsigned long)_exiprom + ~PMD_MASK) & PMD_MASK)
#endif
#ifdef CONFIG_MMU
static struct execmem_info execmem_info __ro_after_init;
struct execmem_info __init *execmem_arch_setup(void)
{
unsigned long fallback_start = 0, fallback_end = 0;
if (IS_ENABLED(CONFIG_ARM_MODULE_PLTS)) {
fallback_start = VMALLOC_START;
fallback_end = VMALLOC_END;
}
execmem_info = (struct execmem_info){
.ranges = {
[EXECMEM_DEFAULT] = {
.start = MODULES_VADDR,
.end = MODULES_END,
.pgprot = PAGE_KERNEL_EXEC,
.alignment = 1,
.fallback_start = fallback_start,
.fallback_end = fallback_end,
},
},
};
return &execmem_info;
}
#endif
bool module_init_section(const char *name) bool module_init_section(const char *name)
{ {
return strstarts(name, ".init") || return strstarts(name, ".init") ||
......
...@@ -22,6 +22,7 @@ ...@@ -22,6 +22,7 @@
#include <linux/sizes.h> #include <linux/sizes.h>
#include <linux/stop_machine.h> #include <linux/stop_machine.h>
#include <linux/swiotlb.h> #include <linux/swiotlb.h>
#include <linux/execmem.h>
#include <asm/cp15.h> #include <asm/cp15.h>
#include <asm/mach-types.h> #include <asm/mach-types.h>
...@@ -486,3 +487,47 @@ void free_initrd_mem(unsigned long start, unsigned long end) ...@@ -486,3 +487,47 @@ void free_initrd_mem(unsigned long start, unsigned long end)
free_reserved_area((void *)start, (void *)end, -1, "initrd"); free_reserved_area((void *)start, (void *)end, -1, "initrd");
} }
#endif #endif
#ifdef CONFIG_EXECMEM
#ifdef CONFIG_XIP_KERNEL
/*
* The XIP kernel text is mapped in the module area for modules and
* some other stuff to work without any indirect relocations.
* MODULES_VADDR is redefined here and not in asm/memory.h to avoid
* recompiling the whole kernel when CONFIG_XIP_KERNEL is turned on/off.
*/
#undef MODULES_VADDR
#define MODULES_VADDR (((unsigned long)_exiprom + ~PMD_MASK) & PMD_MASK)
#endif
#ifdef CONFIG_MMU
static struct execmem_info execmem_info __ro_after_init;
struct execmem_info __init *execmem_arch_setup(void)
{
unsigned long fallback_start = 0, fallback_end = 0;
if (IS_ENABLED(CONFIG_ARM_MODULE_PLTS)) {
fallback_start = VMALLOC_START;
fallback_end = VMALLOC_END;
}
execmem_info = (struct execmem_info){
.ranges = {
[EXECMEM_DEFAULT] = {
.start = MODULES_VADDR,
.end = MODULES_END,
.pgprot = PAGE_KERNEL_EXEC,
.alignment = 1,
.fallback_start = fallback_start,
.fallback_end = fallback_end,
},
},
};
return &execmem_info;
}
#endif /* CONFIG_MMU */
#endif /* CONFIG_EXECMEM */
...@@ -12,158 +12,18 @@ ...@@ -12,158 +12,18 @@
#include <linux/bitops.h> #include <linux/bitops.h>
#include <linux/elf.h> #include <linux/elf.h>
#include <linux/ftrace.h> #include <linux/ftrace.h>
#include <linux/gfp.h>
#include <linux/kasan.h> #include <linux/kasan.h>
#include <linux/kernel.h> #include <linux/kernel.h>
#include <linux/mm.h> #include <linux/mm.h>
#include <linux/moduleloader.h> #include <linux/moduleloader.h>
#include <linux/random.h> #include <linux/random.h>
#include <linux/scs.h> #include <linux/scs.h>
#include <linux/vmalloc.h>
#include <linux/execmem.h>
#include <asm/alternative.h> #include <asm/alternative.h>
#include <asm/insn.h> #include <asm/insn.h>
#include <asm/scs.h> #include <asm/scs.h>
#include <asm/sections.h> #include <asm/sections.h>
static u64 module_direct_base __ro_after_init = 0;
static u64 module_plt_base __ro_after_init = 0;
/*
* Choose a random page-aligned base address for a window of 'size' bytes which
* entirely contains the interval [start, end - 1].
*/
static u64 __init random_bounding_box(u64 size, u64 start, u64 end)
{
u64 max_pgoff, pgoff;
if ((end - start) >= size)
return 0;
max_pgoff = (size - (end - start)) / PAGE_SIZE;
pgoff = get_random_u32_inclusive(0, max_pgoff);
return start - pgoff * PAGE_SIZE;
}
/*
* Modules may directly reference data and text anywhere within the kernel
* image and other modules. References using PREL32 relocations have a +/-2G
* range, and so we need to ensure that the entire kernel image and all modules
* fall within a 2G window such that these are always within range.
*
* Modules may directly branch to functions and code within the kernel text,
* and to functions and code within other modules. These branches will use
* CALL26/JUMP26 relocations with a +/-128M range. Without PLTs, we must ensure
* that the entire kernel text and all module text falls within a 128M window
* such that these are always within range. With PLTs, we can expand this to a
* 2G window.
*
* We chose the 128M region to surround the entire kernel image (rather than
* just the text) as using the same bounds for the 128M and 2G regions ensures
* by construction that we never select a 128M region that is not a subset of
* the 2G region. For very large and unusual kernel configurations this means
* we may fall back to PLTs where they could have been avoided, but this keeps
* the logic significantly simpler.
*/
static int __init module_init_limits(void)
{
u64 kernel_end = (u64)_end;
u64 kernel_start = (u64)_text;
u64 kernel_size = kernel_end - kernel_start;
/*
* The default modules region is placed immediately below the kernel
* image, and is large enough to use the full 2G relocation range.
*/
BUILD_BUG_ON(KIMAGE_VADDR != MODULES_END);
BUILD_BUG_ON(MODULES_VSIZE < SZ_2G);
if (!kaslr_enabled()) {
if (kernel_size < SZ_128M)
module_direct_base = kernel_end - SZ_128M;
if (kernel_size < SZ_2G)
module_plt_base = kernel_end - SZ_2G;
} else {
u64 min = kernel_start;
u64 max = kernel_end;
if (IS_ENABLED(CONFIG_RANDOMIZE_MODULE_REGION_FULL)) {
pr_info("2G module region forced by RANDOMIZE_MODULE_REGION_FULL\n");
} else {
module_direct_base = random_bounding_box(SZ_128M, min, max);
if (module_direct_base) {
min = module_direct_base;
max = module_direct_base + SZ_128M;
}
}
module_plt_base = random_bounding_box(SZ_2G, min, max);
}
pr_info("%llu pages in range for non-PLT usage",
module_direct_base ? (SZ_128M - kernel_size) / PAGE_SIZE : 0);
pr_info("%llu pages in range for PLT usage",
module_plt_base ? (SZ_2G - kernel_size) / PAGE_SIZE : 0);
return 0;
}
static struct execmem_info execmem_info __ro_after_init;
struct execmem_info __init *execmem_arch_setup(void)
{
unsigned long fallback_start = 0, fallback_end = 0;
unsigned long start = 0, end = 0;
module_init_limits();
/*
* Where possible, prefer to allocate within direct branch range of the
* kernel such that no PLTs are necessary.
*/
if (module_direct_base) {
start = module_direct_base;
end = module_direct_base + SZ_128M;
if (module_plt_base) {
fallback_start = module_plt_base;
fallback_end = module_plt_base + SZ_2G;
}
} else if (module_plt_base) {
start = module_plt_base;
end = module_plt_base + SZ_2G;
}
execmem_info = (struct execmem_info){
.ranges = {
[EXECMEM_DEFAULT] = {
.start = start,
.end = end,
.pgprot = PAGE_KERNEL,
.alignment = 1,
.fallback_start = fallback_start,
.fallback_end = fallback_end,
},
[EXECMEM_KPROBES] = {
.start = VMALLOC_START,
.end = VMALLOC_END,
.pgprot = PAGE_KERNEL_ROX,
.alignment = 1,
},
[EXECMEM_BPF] = {
.start = VMALLOC_START,
.end = VMALLOC_END,
.pgprot = PAGE_KERNEL,
.alignment = 1,
},
},
};
return &execmem_info;
}
enum aarch64_reloc_op { enum aarch64_reloc_op {
RELOC_OP_NONE, RELOC_OP_NONE,
RELOC_OP_ABS, RELOC_OP_ABS,
......
...@@ -32,6 +32,7 @@ ...@@ -32,6 +32,7 @@
#include <linux/hugetlb.h> #include <linux/hugetlb.h>
#include <linux/acpi_iort.h> #include <linux/acpi_iort.h>
#include <linux/kmemleak.h> #include <linux/kmemleak.h>
#include <linux/execmem.h>
#include <asm/boot.h> #include <asm/boot.h>
#include <asm/fixmap.h> #include <asm/fixmap.h>
...@@ -432,3 +433,142 @@ void dump_mem_limit(void) ...@@ -432,3 +433,142 @@ void dump_mem_limit(void)
pr_emerg("Memory Limit: none\n"); pr_emerg("Memory Limit: none\n");
} }
} }
#ifdef CONFIG_EXECMEM
static u64 module_direct_base __ro_after_init = 0;
static u64 module_plt_base __ro_after_init = 0;
/*
* Choose a random page-aligned base address for a window of 'size' bytes which
* entirely contains the interval [start, end - 1].
*/
static u64 __init random_bounding_box(u64 size, u64 start, u64 end)
{
u64 max_pgoff, pgoff;
if ((end - start) >= size)
return 0;
max_pgoff = (size - (end - start)) / PAGE_SIZE;
pgoff = get_random_u32_inclusive(0, max_pgoff);
return start - pgoff * PAGE_SIZE;
}
/*
* Modules may directly reference data and text anywhere within the kernel
* image and other modules. References using PREL32 relocations have a +/-2G
* range, and so we need to ensure that the entire kernel image and all modules
* fall within a 2G window such that these are always within range.
*
* Modules may directly branch to functions and code within the kernel text,
* and to functions and code within other modules. These branches will use
* CALL26/JUMP26 relocations with a +/-128M range. Without PLTs, we must ensure
* that the entire kernel text and all module text falls within a 128M window
* such that these are always within range. With PLTs, we can expand this to a
* 2G window.
*
* We chose the 128M region to surround the entire kernel image (rather than
* just the text) as using the same bounds for the 128M and 2G regions ensures
* by construction that we never select a 128M region that is not a subset of
* the 2G region. For very large and unusual kernel configurations this means
* we may fall back to PLTs where they could have been avoided, but this keeps
* the logic significantly simpler.
*/
static int __init module_init_limits(void)
{
u64 kernel_end = (u64)_end;
u64 kernel_start = (u64)_text;
u64 kernel_size = kernel_end - kernel_start;
/*
* The default modules region is placed immediately below the kernel
* image, and is large enough to use the full 2G relocation range.
*/
BUILD_BUG_ON(KIMAGE_VADDR != MODULES_END);
BUILD_BUG_ON(MODULES_VSIZE < SZ_2G);
if (!kaslr_enabled()) {
if (kernel_size < SZ_128M)
module_direct_base = kernel_end - SZ_128M;
if (kernel_size < SZ_2G)
module_plt_base = kernel_end - SZ_2G;
} else {
u64 min = kernel_start;
u64 max = kernel_end;
if (IS_ENABLED(CONFIG_RANDOMIZE_MODULE_REGION_FULL)) {
pr_info("2G module region forced by RANDOMIZE_MODULE_REGION_FULL\n");
} else {
module_direct_base = random_bounding_box(SZ_128M, min, max);
if (module_direct_base) {
min = module_direct_base;
max = module_direct_base + SZ_128M;
}
}
module_plt_base = random_bounding_box(SZ_2G, min, max);
}
pr_info("%llu pages in range for non-PLT usage",
module_direct_base ? (SZ_128M - kernel_size) / PAGE_SIZE : 0);
pr_info("%llu pages in range for PLT usage",
module_plt_base ? (SZ_2G - kernel_size) / PAGE_SIZE : 0);
return 0;
}
static struct execmem_info execmem_info __ro_after_init;
struct execmem_info __init *execmem_arch_setup(void)
{
unsigned long fallback_start = 0, fallback_end = 0;
unsigned long start = 0, end = 0;
module_init_limits();
/*
* Where possible, prefer to allocate within direct branch range of the
* kernel such that no PLTs are necessary.
*/
if (module_direct_base) {
start = module_direct_base;
end = module_direct_base + SZ_128M;
if (module_plt_base) {
fallback_start = module_plt_base;
fallback_end = module_plt_base + SZ_2G;
}
} else if (module_plt_base) {
start = module_plt_base;
end = module_plt_base + SZ_2G;
}
execmem_info = (struct execmem_info){
.ranges = {
[EXECMEM_DEFAULT] = {
.start = start,
.end = end,
.pgprot = PAGE_KERNEL,
.alignment = 1,
.fallback_start = fallback_start,
.fallback_end = fallback_end,
},
[EXECMEM_KPROBES] = {
.start = VMALLOC_START,
.end = VMALLOC_END,
.pgprot = PAGE_KERNEL_ROX,
.alignment = 1,
},
[EXECMEM_BPF] = {
.start = VMALLOC_START,
.end = VMALLOC_END,
.pgprot = PAGE_KERNEL,
.alignment = 1,
},
},
};
return &execmem_info;
}
#endif /* CONFIG_EXECMEM */
...@@ -18,7 +18,6 @@ ...@@ -18,7 +18,6 @@
#include <linux/ftrace.h> #include <linux/ftrace.h>
#include <linux/string.h> #include <linux/string.h>
#include <linux/kernel.h> #include <linux/kernel.h>
#include <linux/execmem.h>
#include <asm/alternative.h> #include <asm/alternative.h>
#include <asm/inst.h> #include <asm/inst.h>
#include <asm/unwind.h> #include <asm/unwind.h>
...@@ -491,24 +490,6 @@ int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, ...@@ -491,24 +490,6 @@ int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab,
return 0; return 0;
} }
static struct execmem_info execmem_info __ro_after_init;
struct execmem_info __init *execmem_arch_setup(void)
{
execmem_info = (struct execmem_info){
.ranges = {
[EXECMEM_DEFAULT] = {
.start = MODULES_VADDR,
.end = MODULES_END,
.pgprot = PAGE_KERNEL,
.alignment = 1,
},
},
};
return &execmem_info;
}
static void module_init_ftrace_plt(const Elf_Ehdr *hdr, static void module_init_ftrace_plt(const Elf_Ehdr *hdr,
const Elf_Shdr *sechdrs, struct module *mod) const Elf_Shdr *sechdrs, struct module *mod)
{ {
......
...@@ -24,6 +24,7 @@ ...@@ -24,6 +24,7 @@
#include <linux/gfp.h> #include <linux/gfp.h>
#include <linux/hugetlb.h> #include <linux/hugetlb.h>
#include <linux/mmzone.h> #include <linux/mmzone.h>
#include <linux/execmem.h>
#include <asm/asm-offsets.h> #include <asm/asm-offsets.h>
#include <asm/bootinfo.h> #include <asm/bootinfo.h>
...@@ -248,3 +249,23 @@ EXPORT_SYMBOL(invalid_pmd_table); ...@@ -248,3 +249,23 @@ EXPORT_SYMBOL(invalid_pmd_table);
#endif #endif
pte_t invalid_pte_table[PTRS_PER_PTE] __page_aligned_bss; pte_t invalid_pte_table[PTRS_PER_PTE] __page_aligned_bss;
EXPORT_SYMBOL(invalid_pte_table); EXPORT_SYMBOL(invalid_pte_table);
#ifdef CONFIG_EXECMEM
static struct execmem_info execmem_info __ro_after_init;
struct execmem_info __init *execmem_arch_setup(void)
{
execmem_info = (struct execmem_info){
.ranges = {
[EXECMEM_DEFAULT] = {
.start = MODULES_VADDR,
.end = MODULES_END,
.pgprot = PAGE_KERNEL,
.alignment = 1,
},
},
};
return &execmem_info;
}
#endif /* CONFIG_EXECMEM */
...@@ -13,14 +13,12 @@ ...@@ -13,14 +13,12 @@
#include <linux/elf.h> #include <linux/elf.h>
#include <linux/mm.h> #include <linux/mm.h>
#include <linux/numa.h> #include <linux/numa.h>
#include <linux/vmalloc.h>
#include <linux/slab.h> #include <linux/slab.h>
#include <linux/fs.h> #include <linux/fs.h>
#include <linux/string.h> #include <linux/string.h>
#include <linux/kernel.h> #include <linux/kernel.h>
#include <linux/spinlock.h> #include <linux/spinlock.h>
#include <linux/jump_label.h> #include <linux/jump_label.h>
#include <linux/execmem.h>
#include <asm/jump_label.h> #include <asm/jump_label.h>
struct mips_hi16 { struct mips_hi16 {
...@@ -32,26 +30,6 @@ struct mips_hi16 { ...@@ -32,26 +30,6 @@ struct mips_hi16 {
static LIST_HEAD(dbe_list); static LIST_HEAD(dbe_list);
static DEFINE_SPINLOCK(dbe_lock); static DEFINE_SPINLOCK(dbe_lock);
#ifdef MODULES_VADDR
static struct execmem_info execmem_info __ro_after_init;
struct execmem_info __init *execmem_arch_setup(void)
{
execmem_info = (struct execmem_info){
.ranges = {
[EXECMEM_DEFAULT] = {
.start = MODULES_VADDR,
.end = MODULES_END,
.pgprot = PAGE_KERNEL,
.alignment = 1,
},
},
};
return &execmem_info;
}
#endif
static void apply_r_mips_32(u32 *location, u32 base, Elf_Addr v) static void apply_r_mips_32(u32 *location, u32 base, Elf_Addr v)
{ {
*location = base + v; *location = base + v;
......
...@@ -31,6 +31,7 @@ ...@@ -31,6 +31,7 @@
#include <linux/gfp.h> #include <linux/gfp.h>
#include <linux/kcore.h> #include <linux/kcore.h>
#include <linux/initrd.h> #include <linux/initrd.h>
#include <linux/execmem.h>
#include <asm/bootinfo.h> #include <asm/bootinfo.h>
#include <asm/cachectl.h> #include <asm/cachectl.h>
...@@ -576,3 +577,25 @@ EXPORT_SYMBOL_GPL(invalid_pmd_table); ...@@ -576,3 +577,25 @@ EXPORT_SYMBOL_GPL(invalid_pmd_table);
#endif #endif
pte_t invalid_pte_table[PTRS_PER_PTE] __page_aligned_bss; pte_t invalid_pte_table[PTRS_PER_PTE] __page_aligned_bss;
EXPORT_SYMBOL(invalid_pte_table); EXPORT_SYMBOL(invalid_pte_table);
#ifdef CONFIG_EXECMEM
#ifdef MODULES_VADDR
static struct execmem_info execmem_info __ro_after_init;
struct execmem_info __init *execmem_arch_setup(void)
{
execmem_info = (struct execmem_info){
.ranges = {
[EXECMEM_DEFAULT] = {
.start = MODULES_VADDR,
.end = MODULES_END,
.pgprot = PAGE_KERNEL,
.alignment = 1,
},
},
};
return &execmem_info;
}
#endif
#endif /* CONFIG_EXECMEM */
...@@ -13,33 +13,13 @@ ...@@ -13,33 +13,13 @@
#include <linux/moduleloader.h> #include <linux/moduleloader.h>
#include <linux/elf.h> #include <linux/elf.h>
#include <linux/mm.h> #include <linux/mm.h>
#include <linux/vmalloc.h>
#include <linux/slab.h> #include <linux/slab.h>
#include <linux/fs.h> #include <linux/fs.h>
#include <linux/string.h> #include <linux/string.h>
#include <linux/kernel.h> #include <linux/kernel.h>
#include <linux/execmem.h>
#include <asm/cacheflush.h> #include <asm/cacheflush.h>
static struct execmem_info execmem_info __ro_after_init;
struct execmem_info __init *execmem_arch_setup(void)
{
execmem_info = (struct execmem_info){
.ranges = {
[EXECMEM_DEFAULT] = {
.start = MODULES_VADDR,
.end = MODULES_END,
.pgprot = PAGE_KERNEL_EXEC,
.alignment = 1,
},
},
};
return &execmem_info;
}
int apply_relocate_add(Elf32_Shdr *sechdrs, const char *strtab, int apply_relocate_add(Elf32_Shdr *sechdrs, const char *strtab,
unsigned int symindex, unsigned int relsec, unsigned int symindex, unsigned int relsec,
struct module *mod) struct module *mod)
......
...@@ -26,6 +26,7 @@ ...@@ -26,6 +26,7 @@
#include <linux/memblock.h> #include <linux/memblock.h>
#include <linux/slab.h> #include <linux/slab.h>
#include <linux/binfmts.h> #include <linux/binfmts.h>
#include <linux/execmem.h>
#include <asm/setup.h> #include <asm/setup.h>
#include <asm/page.h> #include <asm/page.h>
...@@ -143,3 +144,23 @@ static const pgprot_t protection_map[16] = { ...@@ -143,3 +144,23 @@ static const pgprot_t protection_map[16] = {
[VM_SHARED | VM_EXEC | VM_WRITE | VM_READ] = MKP(1, 1, 1) [VM_SHARED | VM_EXEC | VM_WRITE | VM_READ] = MKP(1, 1, 1)
}; };
DECLARE_VM_GET_PAGE_PROT DECLARE_VM_GET_PAGE_PROT
#ifdef CONFIG_EXECMEM
static struct execmem_info execmem_info __ro_after_init;
struct execmem_info __init *execmem_arch_setup(void)
{
execmem_info = (struct execmem_info){
.ranges = {
[EXECMEM_DEFAULT] = {
.start = MODULES_VADDR,
.end = MODULES_END,
.pgprot = PAGE_KERNEL_EXEC,
.alignment = 1,
},
},
};
return &execmem_info;
}
#endif /* CONFIG_EXECMEM */
...@@ -41,7 +41,6 @@ ...@@ -41,7 +41,6 @@
#include <linux/moduleloader.h> #include <linux/moduleloader.h>
#include <linux/elf.h> #include <linux/elf.h>
#include <linux/vmalloc.h>
#include <linux/fs.h> #include <linux/fs.h>
#include <linux/ftrace.h> #include <linux/ftrace.h>
#include <linux/string.h> #include <linux/string.h>
...@@ -49,7 +48,6 @@ ...@@ -49,7 +48,6 @@
#include <linux/bug.h> #include <linux/bug.h>
#include <linux/mm.h> #include <linux/mm.h>
#include <linux/slab.h> #include <linux/slab.h>
#include <linux/execmem.h>
#include <asm/unwind.h> #include <asm/unwind.h>
#include <asm/sections.h> #include <asm/sections.h>
...@@ -174,24 +172,6 @@ static inline int reassemble_22(int as22) ...@@ -174,24 +172,6 @@ static inline int reassemble_22(int as22)
((as22 & 0x0003ff) << 3)); ((as22 & 0x0003ff) << 3));
} }
static struct execmem_info execmem_info __ro_after_init;
struct execmem_info __init *execmem_arch_setup(void)
{
execmem_info = (struct execmem_info){
.ranges = {
[EXECMEM_DEFAULT] = {
.start = VMALLOC_START,
.end = VMALLOC_END,
.pgprot = PAGE_KERNEL_RWX,
.alignment = 1,
},
},
};
return &execmem_info;
}
#ifndef CONFIG_64BIT #ifndef CONFIG_64BIT
static inline unsigned long count_gots(const Elf_Rela *rela, unsigned long n) static inline unsigned long count_gots(const Elf_Rela *rela, unsigned long n)
{ {
......
...@@ -24,6 +24,7 @@ ...@@ -24,6 +24,7 @@
#include <linux/nodemask.h> /* for node_online_map */ #include <linux/nodemask.h> /* for node_online_map */
#include <linux/pagemap.h> /* for release_pages */ #include <linux/pagemap.h> /* for release_pages */
#include <linux/compat.h> #include <linux/compat.h>
#include <linux/execmem.h>
#include <asm/pgalloc.h> #include <asm/pgalloc.h>
#include <asm/tlb.h> #include <asm/tlb.h>
...@@ -481,7 +482,7 @@ void free_initmem(void) ...@@ -481,7 +482,7 @@ void free_initmem(void)
/* finally dump all the instructions which were cached, since the /* finally dump all the instructions which were cached, since the
* pages are no-longer executable */ * pages are no-longer executable */
flush_icache_range(init_begin, init_end); flush_icache_range(init_begin, init_end);
free_initmem_default(POISON_FREE_INITMEM); free_initmem_default(POISON_FREE_INITMEM);
/* set up a new led state on systems shipped LED State panel */ /* set up a new led state on systems shipped LED State panel */
...@@ -992,3 +993,23 @@ static const pgprot_t protection_map[16] = { ...@@ -992,3 +993,23 @@ static const pgprot_t protection_map[16] = {
[VM_SHARED | VM_EXEC | VM_WRITE | VM_READ] = PAGE_RWX [VM_SHARED | VM_EXEC | VM_WRITE | VM_READ] = PAGE_RWX
}; };
DECLARE_VM_GET_PAGE_PROT DECLARE_VM_GET_PAGE_PROT
#ifdef CONFIG_EXECMEM
static struct execmem_info execmem_info __ro_after_init;
struct execmem_info __init *execmem_arch_setup(void)
{
execmem_info = (struct execmem_info){
.ranges = {
[EXECMEM_DEFAULT] = {
.start = VMALLOC_START,
.end = VMALLOC_END,
.pgprot = PAGE_KERNEL_RWX,
.alignment = 1,
},
},
};
return &execmem_info;
}
#endif /* CONFIG_EXECMEM */
...@@ -7,10 +7,8 @@ ...@@ -7,10 +7,8 @@
#include <linux/elf.h> #include <linux/elf.h>
#include <linux/moduleloader.h> #include <linux/moduleloader.h>
#include <linux/err.h> #include <linux/err.h>
#include <linux/vmalloc.h>
#include <linux/mm.h> #include <linux/mm.h>
#include <linux/bug.h> #include <linux/bug.h>
#include <linux/execmem.h>
#include <asm/module.h> #include <asm/module.h>
#include <linux/uaccess.h> #include <linux/uaccess.h>
#include <asm/firmware.h> #include <asm/firmware.h>
...@@ -89,64 +87,3 @@ int module_finalize(const Elf_Ehdr *hdr, ...@@ -89,64 +87,3 @@ int module_finalize(const Elf_Ehdr *hdr,
return 0; return 0;
} }
static struct execmem_info execmem_info __ro_after_init;
struct execmem_info __init *execmem_arch_setup(void)
{
pgprot_t kprobes_prot = strict_module_rwx_enabled() ? PAGE_KERNEL_ROX : PAGE_KERNEL_EXEC;
pgprot_t prot = strict_module_rwx_enabled() ? PAGE_KERNEL : PAGE_KERNEL_EXEC;
unsigned long fallback_start = 0, fallback_end = 0;
unsigned long start, end;
/*
* BOOK3S_32 and 8xx define MODULES_VADDR for text allocations and
* allow allocating data in the entire vmalloc space
*/
#ifdef MODULES_VADDR
unsigned long limit = (unsigned long)_etext - SZ_32M;
BUILD_BUG_ON(TASK_SIZE > MODULES_VADDR);
/* First try within 32M limit from _etext to avoid branch trampolines */
if (MODULES_VADDR < PAGE_OFFSET && MODULES_END > limit) {
start = limit;
fallback_start = MODULES_VADDR;
fallback_end = MODULES_END;
} else {
start = MODULES_VADDR;
}
end = MODULES_END;
#else
start = VMALLOC_START;
end = VMALLOC_END;
#endif
execmem_info = (struct execmem_info){
.ranges = {
[EXECMEM_DEFAULT] = {
.start = start,
.end = end,
.pgprot = prot,
.alignment = 1,
.fallback_start = fallback_start,
.fallback_end = fallback_end,
},
[EXECMEM_KPROBES] = {
.start = VMALLOC_START,
.end = VMALLOC_END,
.pgprot = kprobes_prot,
.alignment = 1,
},
[EXECMEM_MODULE_DATA] = {
.start = VMALLOC_START,
.end = VMALLOC_END,
.pgprot = PAGE_KERNEL,
.alignment = 1,
},
},
};
return &execmem_info;
}
...@@ -16,6 +16,7 @@ ...@@ -16,6 +16,7 @@
#include <linux/highmem.h> #include <linux/highmem.h>
#include <linux/suspend.h> #include <linux/suspend.h>
#include <linux/dma-direct.h> #include <linux/dma-direct.h>
#include <linux/execmem.h>
#include <asm/swiotlb.h> #include <asm/swiotlb.h>
#include <asm/machdep.h> #include <asm/machdep.h>
...@@ -406,3 +407,66 @@ int devmem_is_allowed(unsigned long pfn) ...@@ -406,3 +407,66 @@ int devmem_is_allowed(unsigned long pfn)
* the EHEA driver. Drop this when drivers/net/ethernet/ibm/ehea is removed. * the EHEA driver. Drop this when drivers/net/ethernet/ibm/ehea is removed.
*/ */
EXPORT_SYMBOL_GPL(walk_system_ram_range); EXPORT_SYMBOL_GPL(walk_system_ram_range);
#ifdef CONFIG_EXECMEM
static struct execmem_info execmem_info __ro_after_init;
struct execmem_info __init *execmem_arch_setup(void)
{
pgprot_t kprobes_prot = strict_module_rwx_enabled() ? PAGE_KERNEL_ROX : PAGE_KERNEL_EXEC;
pgprot_t prot = strict_module_rwx_enabled() ? PAGE_KERNEL : PAGE_KERNEL_EXEC;
unsigned long fallback_start = 0, fallback_end = 0;
unsigned long start, end;
/*
* BOOK3S_32 and 8xx define MODULES_VADDR for text allocations and
* allow allocating data in the entire vmalloc space
*/
#ifdef MODULES_VADDR
unsigned long limit = (unsigned long)_etext - SZ_32M;
BUILD_BUG_ON(TASK_SIZE > MODULES_VADDR);
/* First try within 32M limit from _etext to avoid branch trampolines */
if (MODULES_VADDR < PAGE_OFFSET && MODULES_END > limit) {
start = limit;
fallback_start = MODULES_VADDR;
fallback_end = MODULES_END;
} else {
start = MODULES_VADDR;
}
end = MODULES_END;
#else
start = VMALLOC_START;
end = VMALLOC_END;
#endif
execmem_info = (struct execmem_info){
.ranges = {
[EXECMEM_DEFAULT] = {
.start = start,
.end = end,
.pgprot = prot,
.alignment = 1,
.fallback_start = fallback_start,
.fallback_end = fallback_end,
},
[EXECMEM_KPROBES] = {
.start = VMALLOC_START,
.end = VMALLOC_END,
.pgprot = kprobes_prot,
.alignment = 1,
},
[EXECMEM_MODULE_DATA] = {
.start = VMALLOC_START,
.end = VMALLOC_END,
.pgprot = PAGE_KERNEL,
.alignment = 1,
},
},
};
return &execmem_info;
}
#endif /* CONFIG_EXECMEM */
...@@ -11,10 +11,8 @@ ...@@ -11,10 +11,8 @@
#include <linux/kernel.h> #include <linux/kernel.h>
#include <linux/log2.h> #include <linux/log2.h>
#include <linux/moduleloader.h> #include <linux/moduleloader.h>
#include <linux/vmalloc.h>
#include <linux/sizes.h> #include <linux/sizes.h>
#include <linux/pgtable.h> #include <linux/pgtable.h>
#include <linux/execmem.h>
#include <asm/alternative.h> #include <asm/alternative.h>
#include <asm/sections.h> #include <asm/sections.h>
...@@ -906,38 +904,6 @@ int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, ...@@ -906,38 +904,6 @@ int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab,
return 0; return 0;
} }
#ifdef CONFIG_MMU
static struct execmem_info execmem_info __ro_after_init;
struct execmem_info __init *execmem_arch_setup(void)
{
execmem_info = (struct execmem_info){
.ranges = {
[EXECMEM_DEFAULT] = {
.start = MODULES_VADDR,
.end = MODULES_END,
.pgprot = PAGE_KERNEL,
.alignment = 1,
},
[EXECMEM_KPROBES] = {
.start = VMALLOC_START,
.end = VMALLOC_END,
.pgprot = PAGE_KERNEL_READ_EXEC,
.alignment = 1,
},
[EXECMEM_BPF] = {
.start = BPF_JIT_REGION_START,
.end = BPF_JIT_REGION_END,
.pgprot = PAGE_KERNEL,
.alignment = PAGE_SIZE,
},
},
};
return &execmem_info;
}
#endif
int module_finalize(const Elf_Ehdr *hdr, int module_finalize(const Elf_Ehdr *hdr,
const Elf_Shdr *sechdrs, const Elf_Shdr *sechdrs,
struct module *me) struct module *me)
......
...@@ -24,6 +24,7 @@ ...@@ -24,6 +24,7 @@
#include <linux/elf.h> #include <linux/elf.h>
#endif #endif
#include <linux/kfence.h> #include <linux/kfence.h>
#include <linux/execmem.h>
#include <asm/fixmap.h> #include <asm/fixmap.h>
#include <asm/io.h> #include <asm/io.h>
...@@ -1481,3 +1482,37 @@ void __init pgtable_cache_init(void) ...@@ -1481,3 +1482,37 @@ void __init pgtable_cache_init(void)
preallocate_pgd_pages_range(MODULES_VADDR, MODULES_END, "bpf/modules"); preallocate_pgd_pages_range(MODULES_VADDR, MODULES_END, "bpf/modules");
} }
#endif #endif
#ifdef CONFIG_EXECMEM
#ifdef CONFIG_MMU
static struct execmem_info execmem_info __ro_after_init;
struct execmem_info __init *execmem_arch_setup(void)
{
execmem_info = (struct execmem_info){
.ranges = {
[EXECMEM_DEFAULT] = {
.start = MODULES_VADDR,
.end = MODULES_END,
.pgprot = PAGE_KERNEL,
.alignment = 1,
},
[EXECMEM_KPROBES] = {
.start = VMALLOC_START,
.end = VMALLOC_END,
.pgprot = PAGE_KERNEL_READ_EXEC,
.alignment = 1,
},
[EXECMEM_BPF] = {
.start = BPF_JIT_REGION_START,
.end = BPF_JIT_REGION_END,
.pgprot = PAGE_KERNEL,
.alignment = PAGE_SIZE,
},
},
};
return &execmem_info;
}
#endif /* CONFIG_MMU */
#endif /* CONFIG_EXECMEM */
...@@ -37,33 +37,6 @@ ...@@ -37,33 +37,6 @@
#define PLT_ENTRY_SIZE 22 #define PLT_ENTRY_SIZE 22
static struct execmem_info execmem_info __ro_after_init;
struct execmem_info __init *execmem_arch_setup(void)
{
unsigned long module_load_offset = 0;
unsigned long start;
if (kaslr_enabled())
module_load_offset = get_random_u32_inclusive(1, 1024) * PAGE_SIZE;
start = MODULES_VADDR + module_load_offset;
execmem_info = (struct execmem_info){
.ranges = {
[EXECMEM_DEFAULT] = {
.flags = EXECMEM_KASAN_SHADOW,
.start = start,
.end = MODULES_END,
.pgprot = PAGE_KERNEL,
.alignment = MODULE_ALIGN,
},
},
};
return &execmem_info;
}
#ifdef CONFIG_FUNCTION_TRACER #ifdef CONFIG_FUNCTION_TRACER
void module_arch_cleanup(struct module *mod) void module_arch_cleanup(struct module *mod)
{ {
......
...@@ -49,6 +49,7 @@ ...@@ -49,6 +49,7 @@
#include <asm/uv.h> #include <asm/uv.h>
#include <linux/virtio_anchor.h> #include <linux/virtio_anchor.h>
#include <linux/virtio_config.h> #include <linux/virtio_config.h>
#include <linux/execmem.h>
pgd_t swapper_pg_dir[PTRS_PER_PGD] __section(".bss..swapper_pg_dir"); pgd_t swapper_pg_dir[PTRS_PER_PGD] __section(".bss..swapper_pg_dir");
pgd_t invalid_pg_dir[PTRS_PER_PGD] __section(".bss..invalid_pg_dir"); pgd_t invalid_pg_dir[PTRS_PER_PGD] __section(".bss..invalid_pg_dir");
...@@ -302,3 +303,32 @@ void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap) ...@@ -302,3 +303,32 @@ void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
vmem_remove_mapping(start, size); vmem_remove_mapping(start, size);
} }
#endif /* CONFIG_MEMORY_HOTPLUG */ #endif /* CONFIG_MEMORY_HOTPLUG */
#ifdef CONFIG_EXECMEM
static struct execmem_info execmem_info __ro_after_init;
struct execmem_info __init *execmem_arch_setup(void)
{
unsigned long module_load_offset = 0;
unsigned long start;
if (kaslr_enabled())
module_load_offset = get_random_u32_inclusive(1, 1024) * PAGE_SIZE;
start = MODULES_VADDR + module_load_offset;
execmem_info = (struct execmem_info){
.ranges = {
[EXECMEM_DEFAULT] = {
.flags = EXECMEM_KASAN_SHADOW,
.start = start,
.end = MODULES_END,
.pgprot = PAGE_KERNEL,
.alignment = MODULE_ALIGN,
},
},
};
return &execmem_info;
}
#endif /* CONFIG_EXECMEM */
...@@ -14,7 +14,6 @@ ...@@ -14,7 +14,6 @@
#include <linux/string.h> #include <linux/string.h>
#include <linux/ctype.h> #include <linux/ctype.h>
#include <linux/mm.h> #include <linux/mm.h>
#include <linux/execmem.h>
#include <asm/processor.h> #include <asm/processor.h>
#include <asm/spitfire.h> #include <asm/spitfire.h>
...@@ -22,24 +21,6 @@ ...@@ -22,24 +21,6 @@
#include "entry.h" #include "entry.h"
static struct execmem_info execmem_info __ro_after_init;
struct execmem_info __init *execmem_arch_setup(void)
{
execmem_info = (struct execmem_info){
.ranges = {
[EXECMEM_DEFAULT] = {
.start = MODULES_VADDR,
.end = MODULES_END,
.pgprot = PAGE_KERNEL,
.alignment = 1,
},
},
};
return &execmem_info;
}
/* Make generic code ignore STT_REGISTER dummy undefined symbols. */ /* Make generic code ignore STT_REGISTER dummy undefined symbols. */
int module_frob_arch_sections(Elf_Ehdr *hdr, int module_frob_arch_sections(Elf_Ehdr *hdr,
Elf_Shdr *sechdrs, Elf_Shdr *sechdrs,
......
...@@ -14,3 +14,5 @@ obj-$(CONFIG_SPARC32) += leon_mm.o ...@@ -14,3 +14,5 @@ obj-$(CONFIG_SPARC32) += leon_mm.o
# Only used by sparc64 # Only used by sparc64
obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o
obj-$(CONFIG_EXECMEM) += execmem.o
// SPDX-License-Identifier: GPL-2.0
#include <linux/mm.h>
#include <linux/execmem.h>
static struct execmem_info execmem_info __ro_after_init;
struct execmem_info __init *execmem_arch_setup(void)
{
execmem_info = (struct execmem_info){
.ranges = {
[EXECMEM_DEFAULT] = {
.start = MODULES_VADDR,
.end = MODULES_END,
.pgprot = PAGE_KERNEL,
.alignment = 1,
},
},
};
return &execmem_info;
}
...@@ -19,7 +19,6 @@ ...@@ -19,7 +19,6 @@
#include <linux/jump_label.h> #include <linux/jump_label.h>
#include <linux/random.h> #include <linux/random.h>
#include <linux/memory.h> #include <linux/memory.h>
#include <linux/execmem.h>
#include <asm/text-patching.h> #include <asm/text-patching.h>
#include <asm/page.h> #include <asm/page.h>
...@@ -37,32 +36,6 @@ do { \ ...@@ -37,32 +36,6 @@ do { \
} while (0) } while (0)
#endif #endif
static struct execmem_info execmem_info __ro_after_init;
struct execmem_info __init *execmem_arch_setup(void)
{
unsigned long start, offset = 0;
if (kaslr_enabled())
offset = get_random_u32_inclusive(1, 1024) * PAGE_SIZE;
start = MODULES_VADDR + offset;
execmem_info = (struct execmem_info){
.ranges = {
[EXECMEM_DEFAULT] = {
.flags = EXECMEM_KASAN_SHADOW,
.start = start,
.end = MODULES_END,
.pgprot = PAGE_KERNEL,
.alignment = MODULE_ALIGN,
},
},
};
return &execmem_info;
}
#ifdef CONFIG_X86_32 #ifdef CONFIG_X86_32
int apply_relocate(Elf32_Shdr *sechdrs, int apply_relocate(Elf32_Shdr *sechdrs,
const char *strtab, const char *strtab,
......
...@@ -7,6 +7,7 @@ ...@@ -7,6 +7,7 @@
#include <linux/swapops.h> #include <linux/swapops.h>
#include <linux/kmemleak.h> #include <linux/kmemleak.h>
#include <linux/sched/task.h> #include <linux/sched/task.h>
#include <linux/execmem.h>
#include <asm/set_memory.h> #include <asm/set_memory.h>
#include <asm/cpu_device_id.h> #include <asm/cpu_device_id.h>
...@@ -1095,3 +1096,31 @@ unsigned long arch_max_swapfile_size(void) ...@@ -1095,3 +1096,31 @@ unsigned long arch_max_swapfile_size(void)
return pages; return pages;
} }
#endif #endif
#ifdef CONFIG_EXECMEM
static struct execmem_info execmem_info __ro_after_init;
struct execmem_info __init *execmem_arch_setup(void)
{
unsigned long start, offset = 0;
if (kaslr_enabled())
offset = get_random_u32_inclusive(1, 1024) * PAGE_SIZE;
start = MODULES_VADDR + offset;
execmem_info = (struct execmem_info){
.ranges = {
[EXECMEM_DEFAULT] = {
.flags = EXECMEM_KASAN_SHADOW,
.start = start,
.end = MODULES_END,
.pgprot = PAGE_KERNEL,
.alignment = MODULE_ALIGN,
},
},
};
return &execmem_info;
}
#endif /* CONFIG_EXECMEM */
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment