Commit 588ab3f9 authored by Linus Torvalds's avatar Linus Torvalds

Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux

Pull arm64 updates from Catalin Marinas:
 "Here are the main arm64 updates for 4.6.  There are some relatively
  intrusive changes to support KASLR, the reworking of the kernel
  virtual memory layout and initial page table creation.

  Summary:

   - Initial page table creation reworked to avoid breaking large block
     mappings (huge pages) into smaller ones.  The ARM architecture
     requires break-before-make in such cases to avoid TLB conflicts but
     that's not always possible on live page tables

   - Kernel virtual memory layout: the kernel image is no longer linked
     to the bottom of the linear mapping (PAGE_OFFSET) but at the bottom
     of the vmalloc space, allowing the kernel to be loaded (nearly)
     anywhere in physical RAM

   - Kernel ASLR: position independent kernel Image and modules being
     randomly mapped in the vmalloc space with the randomness is
     provided by UEFI (efi_get_random_bytes() patches merged via the
     arm64 tree, acked by Matt Fleming)

   - Implement relative exception tables for arm64, required by KASLR
     (initial code for ARCH_HAS_RELATIVE_EXTABLE added to lib/extable.c
     but actual x86 conversion to deferred to 4.7 because of the merge
     dependencies)

   - Support for the User Access Override feature of ARMv8.2: this
     allows uaccess functions (get_user etc.) to be implemented using
     LDTR/STTR instructions.  Such instructions, when run by the kernel,
     perform unprivileged accesses adding an extra level of protection.
     The set_fs() macro is used to "upgrade" such instruction to
     privileged accesses via the UAO bit

   - Half-precision floating point support (part of ARMv8.2)

   - Optimisations for CPUs with or without a hardware prefetcher (using
     run-time code patching)

   - copy_page performance improvement to deal with 128 bytes at a time

   - Sanity checks on the CPU capabilities (via CPUID) to prevent
     incompatible secondary CPUs from being brought up (e.g.  weird
     big.LITTLE configurations)

   - valid_user_regs() reworked for better sanity check of the
     sigcontext information (restored pstate information)

   - ACPI parking protocol implementation

   - CONFIG_DEBUG_RODATA enabled by default

   - VDSO code marked as read-only

   - DEBUG_PAGEALLOC support

   - ARCH_HAS_UBSAN_SANITIZE_ALL enabled

   - Erratum workaround Cavium ThunderX SoC

   - set_pte_at() fix for PROT_NONE mappings

   - Code clean-ups"

* tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: (99 commits)
  arm64: kasan: Fix zero shadow mapping overriding kernel image shadow
  arm64: kasan: Use actual memory node when populating the kernel image shadow
  arm64: Update PTE_RDONLY in set_pte_at() for PROT_NONE permission
  arm64: Fix misspellings in comments.
  arm64: efi: add missing frame pointer assignment
  arm64: make mrs_s prefixing implicit in read_cpuid
  arm64: enable CONFIG_DEBUG_RODATA by default
  arm64: Rework valid_user_regs
  arm64: mm: check at build time that PAGE_OFFSET divides the VA space evenly
  arm64: KVM: Move kvm_call_hyp back to its original localtion
  arm64: mm: treat memstart_addr as a signed quantity
  arm64: mm: list kernel sections in order
  arm64: lse: deal with clobbered IP registers after branch via PLT
  arm64: mm: dump: Use VA_START directly instead of private LOWEST_ADDR
  arm64: kconfig: add submenu for 8.2 architectural features
  arm64: kernel: acpi: fix ioremap in ACPI parking protocol cpu_postboot
  arm64: Add support for Half precision floating point
  arm64: Remove fixmap include fragility
  arm64: Add workaround for Cavium erratum 27456
  arm64: mm: Mark .rodata as RO
  ...
parents 3d15cfdb 2776e0e8
...@@ -109,7 +109,13 @@ Header notes: ...@@ -109,7 +109,13 @@ Header notes:
1 - 4K 1 - 4K
2 - 16K 2 - 16K
3 - 64K 3 - 64K
Bits 3-63: Reserved. Bit 3: Kernel physical placement
0 - 2MB aligned base should be as close as possible
to the base of DRAM, since memory below it is not
accessible via the linear mapping
1 - 2MB aligned base may be anywhere in physical
memory
Bits 4-63: Reserved.
- When image_size is zero, a bootloader should attempt to keep as much - When image_size is zero, a bootloader should attempt to keep as much
memory as possible free for use by the kernel immediately after the memory as possible free for use by the kernel immediately after the
...@@ -117,14 +123,14 @@ Header notes: ...@@ -117,14 +123,14 @@ Header notes:
depending on selected features, and is effectively unbound. depending on selected features, and is effectively unbound.
The Image must be placed text_offset bytes from a 2MB aligned base The Image must be placed text_offset bytes from a 2MB aligned base
address near the start of usable system RAM and called there. Memory address anywhere in usable system RAM and called there. The region
below that base address is currently unusable by Linux, and therefore it between the 2 MB aligned base address and the start of the image has no
is strongly recommended that this location is the start of system RAM. special significance to the kernel, and may be used for other purposes.
The region between the 2 MB aligned base address and the start of the
image has no special significance to the kernel, and may be used for
other purposes.
At least image_size bytes from the start of the image must be free for At least image_size bytes from the start of the image must be free for
use by the kernel. use by the kernel.
NOTE: versions prior to v4.6 cannot make use of memory below the
physical offset of the Image so it is recommended that the Image be
placed as close as possible to the start of system RAM.
Any memory described to the kernel (even that below the start of the Any memory described to the kernel (even that below the start of the
image) which is not marked as reserved from the kernel (e.g., with a image) which is not marked as reserved from the kernel (e.g., with a
......
...@@ -56,3 +56,4 @@ stable kernels. ...@@ -56,3 +56,4 @@ stable kernels.
| | | | | | | | | |
| Cavium | ThunderX ITS | #22375, #24313 | CAVIUM_ERRATUM_22375 | | Cavium | ThunderX ITS | #22375, #24313 | CAVIUM_ERRATUM_22375 |
| Cavium | ThunderX GICv3 | #23154 | CAVIUM_ERRATUM_23154 | | Cavium | ThunderX GICv3 | #23154 | CAVIUM_ERRATUM_23154 |
| Cavium | ThunderX Core | #27456 | CAVIUM_ERRATUM_27456 |
...@@ -9,7 +9,7 @@ ...@@ -9,7 +9,7 @@
| alpha: | TODO | | alpha: | TODO |
| arc: | TODO | | arc: | TODO |
| arm: | TODO | | arm: | TODO |
| arm64: | TODO | | arm64: | ok |
| avr32: | TODO | | avr32: | TODO |
| blackfin: | TODO | | blackfin: | TODO |
| c6x: | TODO | | c6x: | TODO |
......
...@@ -48,6 +48,8 @@ ...@@ -48,6 +48,8 @@
#define rr_lo_hi(a1, a2) a1, a2 #define rr_lo_hi(a1, a2) a1, a2
#endif #endif
#define kvm_ksym_ref(kva) (kva)
#ifndef __ASSEMBLY__ #ifndef __ASSEMBLY__
struct kvm; struct kvm;
struct kvm_vcpu; struct kvm_vcpu;
......
...@@ -1051,7 +1051,7 @@ static void cpu_init_hyp_mode(void *dummy) ...@@ -1051,7 +1051,7 @@ static void cpu_init_hyp_mode(void *dummy)
pgd_ptr = kvm_mmu_get_httbr(); pgd_ptr = kvm_mmu_get_httbr();
stack_page = __this_cpu_read(kvm_arm_hyp_stack_page); stack_page = __this_cpu_read(kvm_arm_hyp_stack_page);
hyp_stack_ptr = stack_page + PAGE_SIZE; hyp_stack_ptr = stack_page + PAGE_SIZE;
vector_ptr = (unsigned long)__kvm_hyp_vector; vector_ptr = (unsigned long)kvm_ksym_ref(__kvm_hyp_vector);
__cpu_init_hyp_mode(boot_pgd_ptr, pgd_ptr, hyp_stack_ptr, vector_ptr); __cpu_init_hyp_mode(boot_pgd_ptr, pgd_ptr, hyp_stack_ptr, vector_ptr);
__cpu_init_stage2(); __cpu_init_stage2();
...@@ -1220,13 +1220,15 @@ static int init_hyp_mode(void) ...@@ -1220,13 +1220,15 @@ static int init_hyp_mode(void)
/* /*
* Map the Hyp-code called directly from the host * Map the Hyp-code called directly from the host
*/ */
err = create_hyp_mappings(__hyp_text_start, __hyp_text_end); err = create_hyp_mappings(kvm_ksym_ref(__hyp_text_start),
kvm_ksym_ref(__hyp_text_end));
if (err) { if (err) {
kvm_err("Cannot map world-switch code\n"); kvm_err("Cannot map world-switch code\n");
goto out_err; goto out_err;
} }
err = create_hyp_mappings(__start_rodata, __end_rodata); err = create_hyp_mappings(kvm_ksym_ref(__start_rodata),
kvm_ksym_ref(__end_rodata));
if (err) { if (err) {
kvm_err("Cannot map rodata section\n"); kvm_err("Cannot map rodata section\n");
goto out_err; goto out_err;
......
...@@ -14,6 +14,7 @@ config ARM64 ...@@ -14,6 +14,7 @@ config ARM64
select ARCH_WANT_OPTIONAL_GPIOLIB select ARCH_WANT_OPTIONAL_GPIOLIB
select ARCH_WANT_COMPAT_IPC_PARSE_VERSION select ARCH_WANT_COMPAT_IPC_PARSE_VERSION
select ARCH_WANT_FRAME_POINTERS select ARCH_WANT_FRAME_POINTERS
select ARCH_HAS_UBSAN_SANITIZE_ALL
select ARM_AMBA select ARM_AMBA
select ARM_ARCH_TIMER select ARM_ARCH_TIMER
select ARM_GIC select ARM_GIC
...@@ -49,6 +50,7 @@ config ARM64 ...@@ -49,6 +50,7 @@ config ARM64
select HAVE_ALIGNED_STRUCT_PAGE if SLUB select HAVE_ALIGNED_STRUCT_PAGE if SLUB
select HAVE_ARCH_AUDITSYSCALL select HAVE_ARCH_AUDITSYSCALL
select HAVE_ARCH_BITREVERSE select HAVE_ARCH_BITREVERSE
select HAVE_ARCH_HUGE_VMAP
select HAVE_ARCH_JUMP_LABEL select HAVE_ARCH_JUMP_LABEL
select HAVE_ARCH_KASAN if SPARSEMEM_VMEMMAP && !(ARM64_16K_PAGES && ARM64_VA_BITS_48) select HAVE_ARCH_KASAN if SPARSEMEM_VMEMMAP && !(ARM64_16K_PAGES && ARM64_VA_BITS_48)
select HAVE_ARCH_KGDB select HAVE_ARCH_KGDB
...@@ -391,6 +393,7 @@ config ARM64_ERRATUM_843419 ...@@ -391,6 +393,7 @@ config ARM64_ERRATUM_843419
bool "Cortex-A53: 843419: A load or store might access an incorrect address" bool "Cortex-A53: 843419: A load or store might access an incorrect address"
depends on MODULES depends on MODULES
default y default y
select ARM64_MODULE_CMODEL_LARGE
help help
This option builds kernel modules using the large memory model in This option builds kernel modules using the large memory model in
order to avoid the use of the ADRP instruction, which can cause order to avoid the use of the ADRP instruction, which can cause
...@@ -430,6 +433,17 @@ config CAVIUM_ERRATUM_23154 ...@@ -430,6 +433,17 @@ config CAVIUM_ERRATUM_23154
If unsure, say Y. If unsure, say Y.
config CAVIUM_ERRATUM_27456
bool "Cavium erratum 27456: Broadcast TLBI instructions may cause icache corruption"
default y
help
On ThunderX T88 pass 1.x through 2.1 parts, broadcast TLBI
instructions may cause the icache to become corrupted if it
contains data for a non-current ASID. The fix is to
invalidate the icache when changing the mm context.
If unsure, say Y.
endmenu endmenu
...@@ -535,6 +549,9 @@ config HOTPLUG_CPU ...@@ -535,6 +549,9 @@ config HOTPLUG_CPU
source kernel/Kconfig.preempt source kernel/Kconfig.preempt
source kernel/Kconfig.hz source kernel/Kconfig.hz
config ARCH_SUPPORTS_DEBUG_PAGEALLOC
def_bool y
config ARCH_HAS_HOLES_MEMORYMODEL config ARCH_HAS_HOLES_MEMORYMODEL
def_bool y if SPARSEMEM def_bool y if SPARSEMEM
...@@ -763,10 +780,97 @@ config ARM64_VHE ...@@ -763,10 +780,97 @@ config ARM64_VHE
endmenu endmenu
menu "ARMv8.2 architectural features"
config ARM64_UAO
bool "Enable support for User Access Override (UAO)"
default y
help
User Access Override (UAO; part of the ARMv8.2 Extensions)
causes the 'unprivileged' variant of the load/store instructions to
be overriden to be privileged.
This option changes get_user() and friends to use the 'unprivileged'
variant of the load/store instructions. This ensures that user-space
really did have access to the supplied memory. When addr_limit is
set to kernel memory the UAO bit will be set, allowing privileged
access to kernel memory.
Choosing this option will cause copy_to_user() et al to use user-space
memory permissions.
The feature is detected at runtime, the kernel will use the
regular load/store instructions if the cpu does not implement the
feature.
endmenu
config ARM64_MODULE_CMODEL_LARGE
bool
config ARM64_MODULE_PLTS
bool
select ARM64_MODULE_CMODEL_LARGE
select HAVE_MOD_ARCH_SPECIFIC
config RELOCATABLE
bool
help
This builds the kernel as a Position Independent Executable (PIE),
which retains all relocation metadata required to relocate the
kernel binary at runtime to a different virtual address than the
address it was linked at.
Since AArch64 uses the RELA relocation format, this requires a
relocation pass at runtime even if the kernel is loaded at the
same address it was linked at.
config RANDOMIZE_BASE
bool "Randomize the address of the kernel image"
select ARM64_MODULE_PLTS
select RELOCATABLE
help
Randomizes the virtual address at which the kernel image is
loaded, as a security feature that deters exploit attempts
relying on knowledge of the location of kernel internals.
It is the bootloader's job to provide entropy, by passing a
random u64 value in /chosen/kaslr-seed at kernel entry.
When booting via the UEFI stub, it will invoke the firmware's
EFI_RNG_PROTOCOL implementation (if available) to supply entropy
to the kernel proper. In addition, it will randomise the physical
location of the kernel Image as well.
If unsure, say N.
config RANDOMIZE_MODULE_REGION_FULL
bool "Randomize the module region independently from the core kernel"
depends on RANDOMIZE_BASE
default y
help
Randomizes the location of the module region without considering the
location of the core kernel. This way, it is impossible for modules
to leak information about the location of core kernel data structures
but it does imply that function calls between modules and the core
kernel will need to be resolved via veneers in the module PLT.
When this option is not set, the module region will be randomized over
a limited range that contains the [_stext, _etext] interval of the
core kernel, so branch relocations are always in range.
endmenu endmenu
menu "Boot options" menu "Boot options"
config ARM64_ACPI_PARKING_PROTOCOL
bool "Enable support for the ARM64 ACPI parking protocol"
depends on ACPI
help
Enable support for the ARM64 ACPI parking protocol. If disabled
the kernel will not allow booting through the ARM64 ACPI parking
protocol even if the corresponding data is present in the ACPI
MADT table.
config CMDLINE config CMDLINE
string "Default kernel command string" string "Default kernel command string"
default "" default ""
......
...@@ -50,13 +50,13 @@ config DEBUG_SET_MODULE_RONX ...@@ -50,13 +50,13 @@ config DEBUG_SET_MODULE_RONX
config DEBUG_RODATA config DEBUG_RODATA
bool "Make kernel text and rodata read-only" bool "Make kernel text and rodata read-only"
default y
help help
If this is set, kernel text and rodata will be made read-only. This If this is set, kernel text and rodata will be made read-only. This
is to help catch accidental or malicious attempts to change the is to help catch accidental or malicious attempts to change the
kernel's executable code. Additionally splits rodata from kernel kernel's executable code.
text so it can be made explicitly non-executable.
If in doubt, say Y If in doubt, say Y
config DEBUG_ALIGN_RODATA config DEBUG_ALIGN_RODATA
depends on DEBUG_RODATA && ARM64_4K_PAGES depends on DEBUG_RODATA && ARM64_4K_PAGES
......
...@@ -15,6 +15,10 @@ CPPFLAGS_vmlinux.lds = -DTEXT_OFFSET=$(TEXT_OFFSET) ...@@ -15,6 +15,10 @@ CPPFLAGS_vmlinux.lds = -DTEXT_OFFSET=$(TEXT_OFFSET)
OBJCOPYFLAGS :=-O binary -R .note -R .note.gnu.build-id -R .comment -S OBJCOPYFLAGS :=-O binary -R .note -R .note.gnu.build-id -R .comment -S
GZFLAGS :=-9 GZFLAGS :=-9
ifneq ($(CONFIG_RELOCATABLE),)
LDFLAGS_vmlinux += -pie
endif
KBUILD_DEFCONFIG := defconfig KBUILD_DEFCONFIG := defconfig
# Check for binutils support for specific extensions # Check for binutils support for specific extensions
...@@ -43,10 +47,14 @@ endif ...@@ -43,10 +47,14 @@ endif
CHECKFLAGS += -D__aarch64__ CHECKFLAGS += -D__aarch64__
ifeq ($(CONFIG_ARM64_ERRATUM_843419), y) ifeq ($(CONFIG_ARM64_MODULE_CMODEL_LARGE), y)
KBUILD_CFLAGS_MODULE += -mcmodel=large KBUILD_CFLAGS_MODULE += -mcmodel=large
endif endif
ifeq ($(CONFIG_ARM64_MODULE_PLTS),y)
KBUILD_LDFLAGS_MODULE += -T $(srctree)/arch/arm64/kernel/module.lds
endif
# Default value # Default value
head-y := arch/arm64/kernel/head.o head-y := arch/arm64/kernel/head.o
......
...@@ -313,7 +313,7 @@ pinmux: pinmux@0,70000868 { ...@@ -313,7 +313,7 @@ pinmux: pinmux@0,70000868 {
/* /*
* There are two serial driver i.e. 8250 based simple serial * There are two serial driver i.e. 8250 based simple serial
* driver and APB DMA based serial driver for higher baudrate * driver and APB DMA based serial driver for higher baudrate
* and performace. To enable the 8250 based driver, the compatible * and performance. To enable the 8250 based driver, the compatible
* is "nvidia,tegra124-uart", "nvidia,tegra20-uart" and to enable * is "nvidia,tegra124-uart", "nvidia,tegra20-uart" and to enable
* the APB DMA based serial driver, the comptible is * the APB DMA based serial driver, the comptible is
* "nvidia,tegra124-hsuart", "nvidia,tegra30-hsuart". * "nvidia,tegra124-hsuart", "nvidia,tegra30-hsuart".
......
...@@ -345,7 +345,7 @@ pinmux: pinmux@0,700008d4 { ...@@ -345,7 +345,7 @@ pinmux: pinmux@0,700008d4 {
/* /*
* There are two serial driver i.e. 8250 based simple serial * There are two serial driver i.e. 8250 based simple serial
* driver and APB DMA based serial driver for higher baudrate * driver and APB DMA based serial driver for higher baudrate
* and performace. To enable the 8250 based driver, the compatible * and performance. To enable the 8250 based driver, the compatible
* is "nvidia,tegra124-uart", "nvidia,tegra20-uart" and to enable * is "nvidia,tegra124-uart", "nvidia,tegra20-uart" and to enable
* the APB DMA based serial driver, the comptible is * the APB DMA based serial driver, the comptible is
* "nvidia,tegra124-hsuart", "nvidia,tegra30-hsuart". * "nvidia,tegra124-hsuart", "nvidia,tegra30-hsuart".
......
...@@ -87,9 +87,26 @@ void __init acpi_init_cpus(void); ...@@ -87,9 +87,26 @@ void __init acpi_init_cpus(void);
static inline void acpi_init_cpus(void) { } static inline void acpi_init_cpus(void) { }
#endif /* CONFIG_ACPI */ #endif /* CONFIG_ACPI */
#ifdef CONFIG_ARM64_ACPI_PARKING_PROTOCOL
bool acpi_parking_protocol_valid(int cpu);
void __init
acpi_set_mailbox_entry(int cpu, struct acpi_madt_generic_interrupt *processor);
#else
static inline bool acpi_parking_protocol_valid(int cpu) { return false; }
static inline void
acpi_set_mailbox_entry(int cpu, struct acpi_madt_generic_interrupt *processor)
{}
#endif
static inline const char *acpi_get_enable_method(int cpu) static inline const char *acpi_get_enable_method(int cpu)
{ {
return acpi_psci_present() ? "psci" : NULL; if (acpi_psci_present())
return "psci";
if (acpi_parking_protocol_valid(cpu))
return "parking-protocol";
return NULL;
} }
#ifdef CONFIG_ACPI_APEI #ifdef CONFIG_ACPI_APEI
......
#ifndef __ASM_ALTERNATIVE_H #ifndef __ASM_ALTERNATIVE_H
#define __ASM_ALTERNATIVE_H #define __ASM_ALTERNATIVE_H
#include <asm/cpufeature.h>
#ifndef __ASSEMBLY__ #ifndef __ASSEMBLY__
#include <linux/init.h> #include <linux/init.h>
...@@ -63,6 +65,8 @@ void apply_alternatives(void *start, size_t length); ...@@ -63,6 +65,8 @@ void apply_alternatives(void *start, size_t length);
#else #else
#include <asm/assembler.h>
.macro altinstruction_entry orig_offset alt_offset feature orig_len alt_len .macro altinstruction_entry orig_offset alt_offset feature orig_len alt_len
.word \orig_offset - . .word \orig_offset - .
.word \alt_offset - . .word \alt_offset - .
...@@ -136,6 +140,65 @@ void apply_alternatives(void *start, size_t length); ...@@ -136,6 +140,65 @@ void apply_alternatives(void *start, size_t length);
alternative_insn insn1, insn2, cap, IS_ENABLED(cfg) alternative_insn insn1, insn2, cap, IS_ENABLED(cfg)
/*
* Generate the assembly for UAO alternatives with exception table entries.
* This is complicated as there is no post-increment or pair versions of the
* unprivileged instructions, and USER() only works for single instructions.
*/
#ifdef CONFIG_ARM64_UAO
.macro uao_ldp l, reg1, reg2, addr, post_inc
alternative_if_not ARM64_HAS_UAO
8888: ldp \reg1, \reg2, [\addr], \post_inc;
8889: nop;
nop;
alternative_else
ldtr \reg1, [\addr];
ldtr \reg2, [\addr, #8];
add \addr, \addr, \post_inc;
alternative_endif
_asm_extable 8888b,\l;
_asm_extable 8889b,\l;
.endm
.macro uao_stp l, reg1, reg2, addr, post_inc
alternative_if_not ARM64_HAS_UAO
8888: stp \reg1, \reg2, [\addr], \post_inc;
8889: nop;
nop;
alternative_else
sttr \reg1, [\addr];
sttr \reg2, [\addr, #8];
add \addr, \addr, \post_inc;
alternative_endif
_asm_extable 8888b,\l;
_asm_extable 8889b,\l;
.endm
.macro uao_user_alternative l, inst, alt_inst, reg, addr, post_inc
alternative_if_not ARM64_HAS_UAO
8888: \inst \reg, [\addr], \post_inc;
nop;
alternative_else
\alt_inst \reg, [\addr];
add \addr, \addr, \post_inc;
alternative_endif
_asm_extable 8888b,\l;
.endm
#else
.macro uao_ldp l, reg1, reg2, addr, post_inc
USER(\l, ldp \reg1, \reg2, [\addr], \post_inc)
.endm
.macro uao_stp l, reg1, reg2, addr, post_inc
USER(\l, stp \reg1, \reg2, [\addr], \post_inc)
.endm
.macro uao_user_alternative l, inst, alt_inst, reg, addr, post_inc
USER(\l, \inst \reg, [\addr], \post_inc)
.endm
#endif
#endif /* __ASSEMBLY__ */ #endif /* __ASSEMBLY__ */
/* /*
......
...@@ -94,12 +94,19 @@ ...@@ -94,12 +94,19 @@
dmb \opt dmb \opt
.endm .endm
/*
* Emit an entry into the exception table
*/
.macro _asm_extable, from, to
.pushsection __ex_table, "a"
.align 3
.long (\from - .), (\to - .)
.popsection
.endm
#define USER(l, x...) \ #define USER(l, x...) \
9999: x; \ 9999: x; \
.section __ex_table,"a"; \ _asm_extable 9999b, l
.align 3; \
.quad 9999b,l; \
.previous
/* /*
* Register aliases. * Register aliases.
...@@ -215,4 +222,15 @@ lr .req x30 // link register ...@@ -215,4 +222,15 @@ lr .req x30 // link register
.size __pi_##x, . - x; \ .size __pi_##x, . - x; \
ENDPROC(x) ENDPROC(x)
/*
* Emit a 64-bit absolute little endian symbol reference in a way that
* ensures that it will be resolved at build time, even when building a
* PIE binary. This requires cooperation from the linker script, which
* must emit the lo32/hi32 halves individually.
*/
.macro le64sym, sym
.long \sym\()_lo32
.long \sym\()_hi32
.endm
#endif /* __ASM_ASSEMBLER_H */ #endif /* __ASM_ASSEMBLER_H */
...@@ -36,7 +36,7 @@ static inline void atomic_andnot(int i, atomic_t *v) ...@@ -36,7 +36,7 @@ static inline void atomic_andnot(int i, atomic_t *v)
" stclr %w[i], %[v]\n") " stclr %w[i], %[v]\n")
: [i] "+r" (w0), [v] "+Q" (v->counter) : [i] "+r" (w0), [v] "+Q" (v->counter)
: "r" (x1) : "r" (x1)
: "x30"); : __LL_SC_CLOBBERS);
} }
static inline void atomic_or(int i, atomic_t *v) static inline void atomic_or(int i, atomic_t *v)
...@@ -48,7 +48,7 @@ static inline void atomic_or(int i, atomic_t *v) ...@@ -48,7 +48,7 @@ static inline void atomic_or(int i, atomic_t *v)
" stset %w[i], %[v]\n") " stset %w[i], %[v]\n")
: [i] "+r" (w0), [v] "+Q" (v->counter) : [i] "+r" (w0), [v] "+Q" (v->counter)
: "r" (x1) : "r" (x1)
: "x30"); : __LL_SC_CLOBBERS);
} }
static inline void atomic_xor(int i, atomic_t *v) static inline void atomic_xor(int i, atomic_t *v)
...@@ -60,7 +60,7 @@ static inline void atomic_xor(int i, atomic_t *v) ...@@ -60,7 +60,7 @@ static inline void atomic_xor(int i, atomic_t *v)
" steor %w[i], %[v]\n") " steor %w[i], %[v]\n")
: [i] "+r" (w0), [v] "+Q" (v->counter) : [i] "+r" (w0), [v] "+Q" (v->counter)
: "r" (x1) : "r" (x1)
: "x30"); : __LL_SC_CLOBBERS);
} }
static inline void atomic_add(int i, atomic_t *v) static inline void atomic_add(int i, atomic_t *v)
...@@ -72,7 +72,7 @@ static inline void atomic_add(int i, atomic_t *v) ...@@ -72,7 +72,7 @@ static inline void atomic_add(int i, atomic_t *v)
" stadd %w[i], %[v]\n") " stadd %w[i], %[v]\n")
: [i] "+r" (w0), [v] "+Q" (v->counter) : [i] "+r" (w0), [v] "+Q" (v->counter)
: "r" (x1) : "r" (x1)
: "x30"); : __LL_SC_CLOBBERS);
} }
#define ATOMIC_OP_ADD_RETURN(name, mb, cl...) \ #define ATOMIC_OP_ADD_RETURN(name, mb, cl...) \
...@@ -90,7 +90,7 @@ static inline int atomic_add_return##name(int i, atomic_t *v) \ ...@@ -90,7 +90,7 @@ static inline int atomic_add_return##name(int i, atomic_t *v) \
" add %w[i], %w[i], w30") \ " add %w[i], %w[i], w30") \
: [i] "+r" (w0), [v] "+Q" (v->counter) \ : [i] "+r" (w0), [v] "+Q" (v->counter) \
: "r" (x1) \ : "r" (x1) \
: "x30" , ##cl); \ : __LL_SC_CLOBBERS, ##cl); \
\ \
return w0; \ return w0; \
} }
...@@ -116,7 +116,7 @@ static inline void atomic_and(int i, atomic_t *v) ...@@ -116,7 +116,7 @@ static inline void atomic_and(int i, atomic_t *v)
" stclr %w[i], %[v]") " stclr %w[i], %[v]")
: [i] "+r" (w0), [v] "+Q" (v->counter) : [i] "+r" (w0), [v] "+Q" (v->counter)
: "r" (x1) : "r" (x1)
: "x30"); : __LL_SC_CLOBBERS);
} }
static inline void atomic_sub(int i, atomic_t *v) static inline void atomic_sub(int i, atomic_t *v)
...@@ -133,7 +133,7 @@ static inline void atomic_sub(int i, atomic_t *v) ...@@ -133,7 +133,7 @@ static inline void atomic_sub(int i, atomic_t *v)
" stadd %w[i], %[v]") " stadd %w[i], %[v]")
: [i] "+r" (w0), [v] "+Q" (v->counter) : [i] "+r" (w0), [v] "+Q" (v->counter)
: "r" (x1) : "r" (x1)
: "x30"); : __LL_SC_CLOBBERS);
} }
#define ATOMIC_OP_SUB_RETURN(name, mb, cl...) \ #define ATOMIC_OP_SUB_RETURN(name, mb, cl...) \
...@@ -153,7 +153,7 @@ static inline int atomic_sub_return##name(int i, atomic_t *v) \ ...@@ -153,7 +153,7 @@ static inline int atomic_sub_return##name(int i, atomic_t *v) \
" add %w[i], %w[i], w30") \ " add %w[i], %w[i], w30") \
: [i] "+r" (w0), [v] "+Q" (v->counter) \ : [i] "+r" (w0), [v] "+Q" (v->counter) \
: "r" (x1) \ : "r" (x1) \
: "x30" , ##cl); \ : __LL_SC_CLOBBERS , ##cl); \
\ \
return w0; \ return w0; \
} }
...@@ -177,7 +177,7 @@ static inline void atomic64_andnot(long i, atomic64_t *v) ...@@ -177,7 +177,7 @@ static inline void atomic64_andnot(long i, atomic64_t *v)
" stclr %[i], %[v]\n") " stclr %[i], %[v]\n")
: [i] "+r" (x0), [v] "+Q" (v->counter) : [i] "+r" (x0), [v] "+Q" (v->counter)
: "r" (x1) : "r" (x1)
: "x30"); : __LL_SC_CLOBBERS);
} }
static inline void atomic64_or(long i, atomic64_t *v) static inline void atomic64_or(long i, atomic64_t *v)
...@@ -189,7 +189,7 @@ static inline void atomic64_or(long i, atomic64_t *v) ...@@ -189,7 +189,7 @@ static inline void atomic64_or(long i, atomic64_t *v)
" stset %[i], %[v]\n") " stset %[i], %[v]\n")
: [i] "+r" (x0), [v] "+Q" (v->counter) : [i] "+r" (x0), [v] "+Q" (v->counter)
: "r" (x1) : "r" (x1)
: "x30"); : __LL_SC_CLOBBERS);
} }
static inline void atomic64_xor(long i, atomic64_t *v) static inline void atomic64_xor(long i, atomic64_t *v)
...@@ -201,7 +201,7 @@ static inline void atomic64_xor(long i, atomic64_t *v) ...@@ -201,7 +201,7 @@ static inline void atomic64_xor(long i, atomic64_t *v)
" steor %[i], %[v]\n") " steor %[i], %[v]\n")
: [i] "+r" (x0), [v] "+Q" (v->counter) : [i] "+r" (x0), [v] "+Q" (v->counter)
: "r" (x1) : "r" (x1)
: "x30"); : __LL_SC_CLOBBERS);
} }
static inline void atomic64_add(long i, atomic64_t *v) static inline void atomic64_add(long i, atomic64_t *v)
...@@ -213,7 +213,7 @@ static inline void atomic64_add(long i, atomic64_t *v) ...@@ -213,7 +213,7 @@ static inline void atomic64_add(long i, atomic64_t *v)
" stadd %[i], %[v]\n") " stadd %[i], %[v]\n")
: [i] "+r" (x0), [v] "+Q" (v->counter) : [i] "+r" (x0), [v] "+Q" (v->counter)
: "r" (x1) : "r" (x1)
: "x30"); : __LL_SC_CLOBBERS);
} }
#define ATOMIC64_OP_ADD_RETURN(name, mb, cl...) \ #define ATOMIC64_OP_ADD_RETURN(name, mb, cl...) \
...@@ -231,7 +231,7 @@ static inline long atomic64_add_return##name(long i, atomic64_t *v) \ ...@@ -231,7 +231,7 @@ static inline long atomic64_add_return##name(long i, atomic64_t *v) \
" add %[i], %[i], x30") \ " add %[i], %[i], x30") \
: [i] "+r" (x0), [v] "+Q" (v->counter) \ : [i] "+r" (x0), [v] "+Q" (v->counter) \
: "r" (x1) \ : "r" (x1) \
: "x30" , ##cl); \ : __LL_SC_CLOBBERS, ##cl); \
\ \
return x0; \ return x0; \
} }
...@@ -257,7 +257,7 @@ static inline void atomic64_and(long i, atomic64_t *v) ...@@ -257,7 +257,7 @@ static inline void atomic64_and(long i, atomic64_t *v)
" stclr %[i], %[v]") " stclr %[i], %[v]")
: [i] "+r" (x0), [v] "+Q" (v->counter) : [i] "+r" (x0), [v] "+Q" (v->counter)
: "r" (x1) : "r" (x1)
: "x30"); : __LL_SC_CLOBBERS);
} }
static inline void atomic64_sub(long i, atomic64_t *v) static inline void atomic64_sub(long i, atomic64_t *v)
...@@ -274,7 +274,7 @@ static inline void atomic64_sub(long i, atomic64_t *v) ...@@ -274,7 +274,7 @@ static inline void atomic64_sub(long i, atomic64_t *v)
" stadd %[i], %[v]") " stadd %[i], %[v]")
: [i] "+r" (x0), [v] "+Q" (v->counter) : [i] "+r" (x0), [v] "+Q" (v->counter)
: "r" (x1) : "r" (x1)
: "x30"); : __LL_SC_CLOBBERS);
} }
#define ATOMIC64_OP_SUB_RETURN(name, mb, cl...) \ #define ATOMIC64_OP_SUB_RETURN(name, mb, cl...) \
...@@ -294,7 +294,7 @@ static inline long atomic64_sub_return##name(long i, atomic64_t *v) \ ...@@ -294,7 +294,7 @@ static inline long atomic64_sub_return##name(long i, atomic64_t *v) \
" add %[i], %[i], x30") \ " add %[i], %[i], x30") \
: [i] "+r" (x0), [v] "+Q" (v->counter) \ : [i] "+r" (x0), [v] "+Q" (v->counter) \
: "r" (x1) \ : "r" (x1) \
: "x30" , ##cl); \ : __LL_SC_CLOBBERS, ##cl); \
\ \
return x0; \ return x0; \
} }
...@@ -330,7 +330,7 @@ static inline long atomic64_dec_if_positive(atomic64_t *v) ...@@ -330,7 +330,7 @@ static inline long atomic64_dec_if_positive(atomic64_t *v)
"2:") "2:")
: [ret] "+&r" (x0), [v] "+Q" (v->counter) : [ret] "+&r" (x0), [v] "+Q" (v->counter)
: :
: "x30", "cc", "memory"); : __LL_SC_CLOBBERS, "cc", "memory");
return x0; return x0;
} }
...@@ -359,7 +359,7 @@ static inline unsigned long __cmpxchg_case_##name(volatile void *ptr, \ ...@@ -359,7 +359,7 @@ static inline unsigned long __cmpxchg_case_##name(volatile void *ptr, \
" mov %" #w "[ret], " #w "30") \ " mov %" #w "[ret], " #w "30") \
: [ret] "+r" (x0), [v] "+Q" (*(unsigned long *)ptr) \ : [ret] "+r" (x0), [v] "+Q" (*(unsigned long *)ptr) \
: [old] "r" (x1), [new] "r" (x2) \ : [old] "r" (x1), [new] "r" (x2) \
: "x30" , ##cl); \ : __LL_SC_CLOBBERS, ##cl); \
\ \
return x0; \ return x0; \
} }
...@@ -416,7 +416,7 @@ static inline long __cmpxchg_double##name(unsigned long old1, \ ...@@ -416,7 +416,7 @@ static inline long __cmpxchg_double##name(unsigned long old1, \
[v] "+Q" (*(unsigned long *)ptr) \ [v] "+Q" (*(unsigned long *)ptr) \
: [new1] "r" (x2), [new2] "r" (x3), [ptr] "r" (x4), \ : [new1] "r" (x2), [new2] "r" (x3), [ptr] "r" (x4), \
[oldval1] "r" (oldval1), [oldval2] "r" (oldval2) \ [oldval1] "r" (oldval1), [oldval2] "r" (oldval2) \
: "x30" , ##cl); \ : __LL_SC_CLOBBERS, ##cl); \
\ \
return x0; \ return x0; \
} }
......
...@@ -11,4 +11,10 @@ ...@@ -11,4 +11,10 @@
#define MIN_FDT_ALIGN 8 #define MIN_FDT_ALIGN 8
#define MAX_FDT_SIZE SZ_2M #define MAX_FDT_SIZE SZ_2M
/*
* arm64 requires the kernel image to placed
* TEXT_OFFSET bytes beyond a 2 MB aligned base
*/
#define MIN_KIMG_ALIGN SZ_2M
#endif #endif
/*
* Copyright (C) 2012 ARM Ltd.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#ifndef __ASM_BRK_IMM_H
#define __ASM_BRK_IMM_H
/*
* #imm16 values used for BRK instruction generation
* Allowed values for kgdb are 0x400 - 0x7ff
* 0x100: for triggering a fault on purpose (reserved)
* 0x400: for dynamic BRK instruction
* 0x401: for compile time BRK instruction
* 0x800: kernel-mode BUG() and WARN() traps
*/
#define FAULT_BRK_IMM 0x100
#define KGDB_DYN_DBG_BRK_IMM 0x400
#define KGDB_COMPILED_DBG_BRK_IMM 0x401
#define BUG_BRK_IMM 0x800
#endif
...@@ -18,7 +18,7 @@ ...@@ -18,7 +18,7 @@
#ifndef _ARCH_ARM64_ASM_BUG_H #ifndef _ARCH_ARM64_ASM_BUG_H
#define _ARCH_ARM64_ASM_BUG_H #define _ARCH_ARM64_ASM_BUG_H
#include <asm/debug-monitors.h> #include <asm/brk-imm.h>
#ifdef CONFIG_GENERIC_BUG #ifdef CONFIG_GENERIC_BUG
#define HAVE_ARCH_BUG #define HAVE_ARCH_BUG
......
...@@ -36,6 +36,7 @@ struct cpuinfo_arm64 { ...@@ -36,6 +36,7 @@ struct cpuinfo_arm64 {
u64 reg_id_aa64isar1; u64 reg_id_aa64isar1;
u64 reg_id_aa64mmfr0; u64 reg_id_aa64mmfr0;
u64 reg_id_aa64mmfr1; u64 reg_id_aa64mmfr1;
u64 reg_id_aa64mmfr2;
u64 reg_id_aa64pfr0; u64 reg_id_aa64pfr0;
u64 reg_id_aa64pfr1; u64 reg_id_aa64pfr1;
......
...@@ -30,12 +30,13 @@ ...@@ -30,12 +30,13 @@
#define ARM64_HAS_LSE_ATOMICS 5 #define ARM64_HAS_LSE_ATOMICS 5
#define ARM64_WORKAROUND_CAVIUM_23154 6 #define ARM64_WORKAROUND_CAVIUM_23154 6
#define ARM64_WORKAROUND_834220 7 #define ARM64_WORKAROUND_834220 7
/* #define ARM64_HAS_NO_HW_PREFETCH 8 */ #define ARM64_HAS_NO_HW_PREFETCH 8
/* #define ARM64_HAS_UAO 9 */ #define ARM64_HAS_UAO 9
/* #define ARM64_ALT_PAN_NOT_UAO 10 */ #define ARM64_ALT_PAN_NOT_UAO 10
#define ARM64_HAS_VIRT_HOST_EXTN 11 #define ARM64_HAS_VIRT_HOST_EXTN 11
#define ARM64_WORKAROUND_CAVIUM_27456 12
#define ARM64_NCAPS 12 #define ARM64_NCAPS 13
#ifndef __ASSEMBLY__ #ifndef __ASSEMBLY__
...@@ -89,9 +90,10 @@ struct arm64_cpu_capabilities { ...@@ -89,9 +90,10 @@ struct arm64_cpu_capabilities {
struct { /* Feature register checking */ struct { /* Feature register checking */
u32 sys_reg; u32 sys_reg;
int field_pos; u8 field_pos;
int min_field_value; u8 min_field_value;
int hwcap_type; u8 hwcap_type;
bool sign;
unsigned long hwcap; unsigned long hwcap;
}; };
}; };
...@@ -121,15 +123,15 @@ static inline void cpus_set_cap(unsigned int num) ...@@ -121,15 +123,15 @@ static inline void cpus_set_cap(unsigned int num)
} }
static inline int __attribute_const__ static inline int __attribute_const__
cpuid_feature_extract_field_width(u64 features, int field, int width) cpuid_feature_extract_signed_field_width(u64 features, int field, int width)
{ {
return (s64)(features << (64 - width - field)) >> (64 - width); return (s64)(features << (64 - width - field)) >> (64 - width);
} }
static inline int __attribute_const__ static inline int __attribute_const__
cpuid_feature_extract_field(u64 features, int field) cpuid_feature_extract_signed_field(u64 features, int field)
{ {
return cpuid_feature_extract_field_width(features, field, 4); return cpuid_feature_extract_signed_field_width(features, field, 4);
} }
static inline unsigned int __attribute_const__ static inline unsigned int __attribute_const__
...@@ -149,17 +151,23 @@ static inline u64 arm64_ftr_mask(struct arm64_ftr_bits *ftrp) ...@@ -149,17 +151,23 @@ static inline u64 arm64_ftr_mask(struct arm64_ftr_bits *ftrp)
return (u64)GENMASK(ftrp->shift + ftrp->width - 1, ftrp->shift); return (u64)GENMASK(ftrp->shift + ftrp->width - 1, ftrp->shift);
} }
static inline int __attribute_const__
cpuid_feature_extract_field(u64 features, int field, bool sign)
{
return (sign) ?
cpuid_feature_extract_signed_field(features, field) :
cpuid_feature_extract_unsigned_field(features, field);
}
static inline s64 arm64_ftr_value(struct arm64_ftr_bits *ftrp, u64 val) static inline s64 arm64_ftr_value(struct arm64_ftr_bits *ftrp, u64 val)
{ {
return ftrp->sign ? return (s64)cpuid_feature_extract_field(val, ftrp->shift, ftrp->sign);
cpuid_feature_extract_field_width(val, ftrp->shift, ftrp->width) :
cpuid_feature_extract_unsigned_field_width(val, ftrp->shift, ftrp->width);
} }
static inline bool id_aa64mmfr0_mixed_endian_el0(u64 mmfr0) static inline bool id_aa64mmfr0_mixed_endian_el0(u64 mmfr0)
{ {
return cpuid_feature_extract_field(mmfr0, ID_AA64MMFR0_BIGENDEL_SHIFT) == 0x1 || return cpuid_feature_extract_unsigned_field(mmfr0, ID_AA64MMFR0_BIGENDEL_SHIFT) == 0x1 ||
cpuid_feature_extract_field(mmfr0, ID_AA64MMFR0_BIGENDEL0_SHIFT) == 0x1; cpuid_feature_extract_unsigned_field(mmfr0, ID_AA64MMFR0_BIGENDEL0_SHIFT) == 0x1;
} }
void __init setup_cpu_features(void); void __init setup_cpu_features(void);
...@@ -168,13 +176,7 @@ void update_cpu_capabilities(const struct arm64_cpu_capabilities *caps, ...@@ -168,13 +176,7 @@ void update_cpu_capabilities(const struct arm64_cpu_capabilities *caps,
const char *info); const char *info);
void check_local_cpu_errata(void); void check_local_cpu_errata(void);
#ifdef CONFIG_HOTPLUG_CPU
void verify_local_cpu_capabilities(void); void verify_local_cpu_capabilities(void);
#else
static inline void verify_local_cpu_capabilities(void)
{
}
#endif
u64 read_system_reg(u32 id); u64 read_system_reg(u32 id);
......
...@@ -32,12 +32,6 @@ ...@@ -32,12 +32,6 @@
#define MPIDR_AFFINITY_LEVEL(mpidr, level) \ #define MPIDR_AFFINITY_LEVEL(mpidr, level) \
((mpidr >> MPIDR_LEVEL_SHIFT(level)) & MPIDR_LEVEL_MASK) ((mpidr >> MPIDR_LEVEL_SHIFT(level)) & MPIDR_LEVEL_MASK)
#define read_cpuid(reg) ({ \
u64 __val; \
asm("mrs %0, " #reg : "=r" (__val)); \
__val; \
})
#define MIDR_REVISION_MASK 0xf #define MIDR_REVISION_MASK 0xf
#define MIDR_REVISION(midr) ((midr) & MIDR_REVISION_MASK) #define MIDR_REVISION(midr) ((midr) & MIDR_REVISION_MASK)
#define MIDR_PARTNUM_SHIFT 4 #define MIDR_PARTNUM_SHIFT 4
...@@ -57,11 +51,22 @@ ...@@ -57,11 +51,22 @@
#define MIDR_IMPLEMENTOR(midr) \ #define MIDR_IMPLEMENTOR(midr) \
(((midr) & MIDR_IMPLEMENTOR_MASK) >> MIDR_IMPLEMENTOR_SHIFT) (((midr) & MIDR_IMPLEMENTOR_MASK) >> MIDR_IMPLEMENTOR_SHIFT)
#define MIDR_CPU_PART(imp, partnum) \ #define MIDR_CPU_MODEL(imp, partnum) \
(((imp) << MIDR_IMPLEMENTOR_SHIFT) | \ (((imp) << MIDR_IMPLEMENTOR_SHIFT) | \
(0xf << MIDR_ARCHITECTURE_SHIFT) | \ (0xf << MIDR_ARCHITECTURE_SHIFT) | \
((partnum) << MIDR_PARTNUM_SHIFT)) ((partnum) << MIDR_PARTNUM_SHIFT))
#define MIDR_CPU_MODEL_MASK (MIDR_IMPLEMENTOR_MASK | MIDR_PARTNUM_MASK | \
MIDR_ARCHITECTURE_MASK)
#define MIDR_IS_CPU_MODEL_RANGE(midr, model, rv_min, rv_max) \
({ \
u32 _model = (midr) & MIDR_CPU_MODEL_MASK; \
u32 rv = (midr) & (MIDR_REVISION_MASK | MIDR_VARIANT_MASK); \
\
_model == (model) && rv >= (rv_min) && rv <= (rv_max); \
})
#define ARM_CPU_IMP_ARM 0x41 #define ARM_CPU_IMP_ARM 0x41
#define ARM_CPU_IMP_APM 0x50 #define ARM_CPU_IMP_APM 0x50
#define ARM_CPU_IMP_CAVIUM 0x43 #define ARM_CPU_IMP_CAVIUM 0x43
...@@ -75,8 +80,20 @@ ...@@ -75,8 +80,20 @@
#define CAVIUM_CPU_PART_THUNDERX 0x0A1 #define CAVIUM_CPU_PART_THUNDERX 0x0A1
#define MIDR_CORTEX_A53 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A53)
#define MIDR_CORTEX_A57 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A57)
#define MIDR_THUNDERX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX)
#ifndef __ASSEMBLY__ #ifndef __ASSEMBLY__
#include <asm/sysreg.h>
#define read_cpuid(reg) ({ \
u64 __val; \
asm("mrs_s %0, " __stringify(SYS_ ## reg) : "=r" (__val)); \
__val; \
})
/* /*
* The CPU ID never changes at run time, so we might as well tell the * The CPU ID never changes at run time, so we might as well tell the
* compiler that it's constant. Use this function to read the CPU ID * compiler that it's constant. Use this function to read the CPU ID
......
...@@ -20,6 +20,7 @@ ...@@ -20,6 +20,7 @@
#include <linux/errno.h> #include <linux/errno.h>
#include <linux/types.h> #include <linux/types.h>
#include <asm/brk-imm.h>
#include <asm/esr.h> #include <asm/esr.h>
#include <asm/insn.h> #include <asm/insn.h>
#include <asm/ptrace.h> #include <asm/ptrace.h>
...@@ -46,19 +47,6 @@ ...@@ -46,19 +47,6 @@
*/ */
#define BREAK_INSTR_SIZE AARCH64_INSN_SIZE #define BREAK_INSTR_SIZE AARCH64_INSN_SIZE
/*
* #imm16 values used for BRK instruction generation
* Allowed values for kgbd are 0x400 - 0x7ff
* 0x100: for triggering a fault on purpose (reserved)
* 0x400: for dynamic BRK instruction
* 0x401: for compile time BRK instruction
* 0x800: kernel-mode BUG() and WARN() traps
*/
#define FAULT_BRK_IMM 0x100
#define KGDB_DYN_DBG_BRK_IMM 0x400
#define KGDB_COMPILED_DBG_BRK_IMM 0x401
#define BUG_BRK_IMM 0x800
/* /*
* BRK instruction encoding * BRK instruction encoding
* The #imm16 value should be placed at bits[20:5] within BRK ins * The #imm16 value should be placed at bits[20:5] within BRK ins
......
...@@ -24,15 +24,6 @@ ...@@ -24,15 +24,6 @@
#include <asm/ptrace.h> #include <asm/ptrace.h>
#include <asm/user.h> #include <asm/user.h>
typedef unsigned long elf_greg_t;
#define ELF_NGREG (sizeof(struct user_pt_regs) / sizeof(elf_greg_t))
#define ELF_CORE_COPY_REGS(dest, regs) \
*(struct user_pt_regs *)&(dest) = (regs)->user_regs;
typedef elf_greg_t elf_gregset_t[ELF_NGREG];
typedef struct user_fpsimd_state elf_fpregset_t;
/* /*
* AArch64 static relocation types. * AArch64 static relocation types.
*/ */
...@@ -86,6 +77,8 @@ typedef struct user_fpsimd_state elf_fpregset_t; ...@@ -86,6 +77,8 @@ typedef struct user_fpsimd_state elf_fpregset_t;
#define R_AARCH64_MOVW_PREL_G2_NC 292 #define R_AARCH64_MOVW_PREL_G2_NC 292
#define R_AARCH64_MOVW_PREL_G3 293 #define R_AARCH64_MOVW_PREL_G3 293
#define R_AARCH64_RELATIVE 1027
/* /*
* These are used to set parameters in the core dumps. * These are used to set parameters in the core dumps.
*/ */
...@@ -127,6 +120,17 @@ typedef struct user_fpsimd_state elf_fpregset_t; ...@@ -127,6 +120,17 @@ typedef struct user_fpsimd_state elf_fpregset_t;
*/ */
#define ELF_ET_DYN_BASE (2 * TASK_SIZE_64 / 3) #define ELF_ET_DYN_BASE (2 * TASK_SIZE_64 / 3)
#ifndef __ASSEMBLY__
typedef unsigned long elf_greg_t;
#define ELF_NGREG (sizeof(struct user_pt_regs) / sizeof(elf_greg_t))
#define ELF_CORE_COPY_REGS(dest, regs) \
*(struct user_pt_regs *)&(dest) = (regs)->user_regs;
typedef elf_greg_t elf_gregset_t[ELF_NGREG];
typedef struct user_fpsimd_state elf_fpregset_t;
/* /*
* When the program starts, a1 contains a pointer to a function to be * When the program starts, a1 contains a pointer to a function to be
* registered with atexit, as per the SVR4 ABI. A value of 0 means we have no * registered with atexit, as per the SVR4 ABI. A value of 0 means we have no
...@@ -186,4 +190,6 @@ extern int aarch32_setup_vectors_page(struct linux_binprm *bprm, ...@@ -186,4 +190,6 @@ extern int aarch32_setup_vectors_page(struct linux_binprm *bprm,
#endif /* CONFIG_COMPAT */ #endif /* CONFIG_COMPAT */
#endif /* !__ASSEMBLY__ */
#endif #endif
...@@ -20,6 +20,7 @@ ...@@ -20,6 +20,7 @@
#include <linux/sizes.h> #include <linux/sizes.h>
#include <asm/boot.h> #include <asm/boot.h>
#include <asm/page.h> #include <asm/page.h>
#include <asm/pgtable-prot.h>
/* /*
* Here we define all the compile-time 'special' virtual * Here we define all the compile-time 'special' virtual
...@@ -62,6 +63,16 @@ enum fixed_addresses { ...@@ -62,6 +63,16 @@ enum fixed_addresses {
FIX_BTMAP_END = __end_of_permanent_fixed_addresses, FIX_BTMAP_END = __end_of_permanent_fixed_addresses,
FIX_BTMAP_BEGIN = FIX_BTMAP_END + TOTAL_FIX_BTMAPS - 1, FIX_BTMAP_BEGIN = FIX_BTMAP_END + TOTAL_FIX_BTMAPS - 1,
/*
* Used for kernel page table creation, so unmapped memory may be used
* for tables.
*/
FIX_PTE,
FIX_PMD,
FIX_PUD,
FIX_PGD,
__end_of_fixed_addresses __end_of_fixed_addresses
}; };
......
...@@ -48,7 +48,7 @@ static inline unsigned long ftrace_call_adjust(unsigned long addr) ...@@ -48,7 +48,7 @@ static inline unsigned long ftrace_call_adjust(unsigned long addr)
* See kernel/trace/trace_syscalls.c * See kernel/trace/trace_syscalls.c
* *
* x86 code says: * x86 code says:
* If the user realy wants these, then they should use the * If the user really wants these, then they should use the
* raw syscall tracepoints with filtering. * raw syscall tracepoints with filtering.
*/ */
#define ARCH_TRACE_IGNORE_COMPAT_SYSCALLS #define ARCH_TRACE_IGNORE_COMPAT_SYSCALLS
......
...@@ -42,10 +42,8 @@ ...@@ -42,10 +42,8 @@
"4: mov %w0, %w5\n" \ "4: mov %w0, %w5\n" \
" b 3b\n" \ " b 3b\n" \
" .popsection\n" \ " .popsection\n" \
" .pushsection __ex_table,\"a\"\n" \ _ASM_EXTABLE(1b, 4b) \
" .align 3\n" \ _ASM_EXTABLE(2b, 4b) \
" .quad 1b, 4b, 2b, 4b\n" \
" .popsection\n" \
ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, \ ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, \
CONFIG_ARM64_PAN) \ CONFIG_ARM64_PAN) \
: "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp) \ : "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp) \
...@@ -134,10 +132,8 @@ ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, CONFIG_ARM64_PAN) ...@@ -134,10 +132,8 @@ ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, CONFIG_ARM64_PAN)
"4: mov %w0, %w6\n" "4: mov %w0, %w6\n"
" b 3b\n" " b 3b\n"
" .popsection\n" " .popsection\n"
" .pushsection __ex_table,\"a\"\n" _ASM_EXTABLE(1b, 4b)
" .align 3\n" _ASM_EXTABLE(2b, 4b)
" .quad 1b, 4b, 2b, 4b\n"
" .popsection\n"
ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, CONFIG_ARM64_PAN) ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, CONFIG_ARM64_PAN)
: "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp) : "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp)
: "r" (oldval), "r" (newval), "Ir" (-EFAULT) : "r" (oldval), "r" (newval), "Ir" (-EFAULT)
......
...@@ -20,7 +20,7 @@ ...@@ -20,7 +20,7 @@
#include <linux/threads.h> #include <linux/threads.h>
#include <asm/irq.h> #include <asm/irq.h>
#define NR_IPI 5 #define NR_IPI 6
typedef struct { typedef struct {
unsigned int __softirq_pending; unsigned int __softirq_pending;
......
...@@ -7,13 +7,14 @@ ...@@ -7,13 +7,14 @@
#include <linux/linkage.h> #include <linux/linkage.h>
#include <asm/memory.h> #include <asm/memory.h>
#include <asm/pgtable-types.h>
/* /*
* KASAN_SHADOW_START: beginning of the kernel virtual addresses. * KASAN_SHADOW_START: beginning of the kernel virtual addresses.
* KASAN_SHADOW_END: KASAN_SHADOW_START + 1/8 of kernel virtual addresses. * KASAN_SHADOW_END: KASAN_SHADOW_START + 1/8 of kernel virtual addresses.
*/ */
#define KASAN_SHADOW_START (VA_START) #define KASAN_SHADOW_START (VA_START)
#define KASAN_SHADOW_END (KASAN_SHADOW_START + (1UL << (VA_BITS - 3))) #define KASAN_SHADOW_END (KASAN_SHADOW_START + KASAN_SHADOW_SIZE)
/* /*
* This value is used to map an address to the corresponding shadow * This value is used to map an address to the corresponding shadow
...@@ -28,10 +29,12 @@ ...@@ -28,10 +29,12 @@
#define KASAN_SHADOW_OFFSET (KASAN_SHADOW_END - (1ULL << (64 - 3))) #define KASAN_SHADOW_OFFSET (KASAN_SHADOW_END - (1ULL << (64 - 3)))
void kasan_init(void); void kasan_init(void);
void kasan_copy_shadow(pgd_t *pgdir);
asmlinkage void kasan_early_init(void); asmlinkage void kasan_early_init(void);
#else #else
static inline void kasan_init(void) { } static inline void kasan_init(void) { }
static inline void kasan_copy_shadow(pgd_t *pgdir) { }
#endif #endif
#endif #endif
......
...@@ -79,5 +79,17 @@ ...@@ -79,5 +79,17 @@
#define SWAPPER_MM_MMUFLAGS (PTE_ATTRINDX(MT_NORMAL) | SWAPPER_PTE_FLAGS) #define SWAPPER_MM_MMUFLAGS (PTE_ATTRINDX(MT_NORMAL) | SWAPPER_PTE_FLAGS)
#endif #endif
/*
* To make optimal use of block mappings when laying out the linear
* mapping, round down the base of physical memory to a size that can
* be mapped efficiently, i.e., either PUD_SIZE (4k granule) or PMD_SIZE
* (64k granule), or a multiple that can be mapped using contiguous bits
* in the page tables: 32 * PMD_SIZE (16k granule)
*/
#ifdef CONFIG_ARM64_64K_PAGES
#define ARM64_MEMSTART_ALIGN SZ_512M
#else
#define ARM64_MEMSTART_ALIGN SZ_1G
#endif
#endif /* __ASM_KERNEL_PGTABLE_H */ #endif /* __ASM_KERNEL_PGTABLE_H */
...@@ -62,7 +62,7 @@ ...@@ -62,7 +62,7 @@
/* /*
* The bits we set in HCR: * The bits we set in HCR:
* RW: 64bit by default, can be overriden for 32bit VMs * RW: 64bit by default, can be overridden for 32bit VMs
* TAC: Trap ACTLR * TAC: Trap ACTLR
* TSC: Trap SMC * TSC: Trap SMC
* TVM: Trap VM ops (until M+C set in SCTLR_EL1) * TVM: Trap VM ops (until M+C set in SCTLR_EL1)
......
...@@ -26,6 +26,8 @@ ...@@ -26,6 +26,8 @@
#define KVM_ARM64_DEBUG_DIRTY_SHIFT 0 #define KVM_ARM64_DEBUG_DIRTY_SHIFT 0
#define KVM_ARM64_DEBUG_DIRTY (1 << KVM_ARM64_DEBUG_DIRTY_SHIFT) #define KVM_ARM64_DEBUG_DIRTY (1 << KVM_ARM64_DEBUG_DIRTY_SHIFT)
#define kvm_ksym_ref(sym) phys_to_virt((u64)&sym - kimage_voffset)
#ifndef __ASSEMBLY__ #ifndef __ASSEMBLY__
struct kvm; struct kvm;
struct kvm_vcpu; struct kvm_vcpu;
......
...@@ -102,8 +102,8 @@ enum vcpu_sysreg { ...@@ -102,8 +102,8 @@ enum vcpu_sysreg {
TTBR1_EL1, /* Translation Table Base Register 1 */ TTBR1_EL1, /* Translation Table Base Register 1 */
TCR_EL1, /* Translation Control Register */ TCR_EL1, /* Translation Control Register */
ESR_EL1, /* Exception Syndrome Register */ ESR_EL1, /* Exception Syndrome Register */
AFSR0_EL1, /* Auxilary Fault Status Register 0 */ AFSR0_EL1, /* Auxiliary Fault Status Register 0 */
AFSR1_EL1, /* Auxilary Fault Status Register 1 */ AFSR1_EL1, /* Auxiliary Fault Status Register 1 */
FAR_EL1, /* Fault Address Register */ FAR_EL1, /* Fault Address Register */
MAIR_EL1, /* Memory Attribute Indirection Register */ MAIR_EL1, /* Memory Attribute Indirection Register */
VBAR_EL1, /* Vector Base Address Register */ VBAR_EL1, /* Vector Base Address Register */
...@@ -326,7 +326,9 @@ static inline void kvm_arch_mmu_notifier_invalidate_page(struct kvm *kvm, ...@@ -326,7 +326,9 @@ static inline void kvm_arch_mmu_notifier_invalidate_page(struct kvm *kvm,
struct kvm_vcpu *kvm_arm_get_running_vcpu(void); struct kvm_vcpu *kvm_arm_get_running_vcpu(void);
struct kvm_vcpu * __percpu *kvm_get_running_vcpus(void); struct kvm_vcpu * __percpu *kvm_get_running_vcpus(void);
u64 kvm_call_hyp(void *hypfn, ...); u64 __kvm_call_hyp(void *hypfn, ...);
#define kvm_call_hyp(f, ...) __kvm_call_hyp(kvm_ksym_ref(f), ##__VA_ARGS__)
void force_vm_exit(const cpumask_t *mask); void force_vm_exit(const cpumask_t *mask);
void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot); void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot);
...@@ -347,8 +349,8 @@ static inline void __cpu_init_hyp_mode(phys_addr_t boot_pgd_ptr, ...@@ -347,8 +349,8 @@ static inline void __cpu_init_hyp_mode(phys_addr_t boot_pgd_ptr,
* Call initialization code, and switch to the full blown * Call initialization code, and switch to the full blown
* HYP code. * HYP code.
*/ */
kvm_call_hyp((void *)boot_pgd_ptr, pgd_ptr, __kvm_call_hyp((void *)boot_pgd_ptr, pgd_ptr,
hyp_stack_ptr, vector_ptr); hyp_stack_ptr, vector_ptr);
} }
static inline void kvm_arch_hardware_disable(void) {} static inline void kvm_arch_hardware_disable(void) {}
......
...@@ -317,7 +317,7 @@ static inline unsigned int kvm_get_vmid_bits(void) ...@@ -317,7 +317,7 @@ static inline unsigned int kvm_get_vmid_bits(void)
{ {
int reg = read_system_reg(SYS_ID_AA64MMFR1_EL1); int reg = read_system_reg(SYS_ID_AA64MMFR1_EL1);
return (cpuid_feature_extract_field(reg, ID_AA64MMFR1_VMIDBITS_SHIFT) == 2) ? 16 : 8; return (cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR1_VMIDBITS_SHIFT) == 2) ? 16 : 8;
} }
#endif /* __ASSEMBLY__ */ #endif /* __ASSEMBLY__ */
......
...@@ -26,6 +26,7 @@ __asm__(".arch_extension lse"); ...@@ -26,6 +26,7 @@ __asm__(".arch_extension lse");
/* Macro for constructing calls to out-of-line ll/sc atomics */ /* Macro for constructing calls to out-of-line ll/sc atomics */
#define __LL_SC_CALL(op) "bl\t" __stringify(__LL_SC_PREFIX(op)) "\n" #define __LL_SC_CALL(op) "bl\t" __stringify(__LL_SC_PREFIX(op)) "\n"
#define __LL_SC_CLOBBERS "x16", "x17", "x30"
/* In-line patching at runtime */ /* In-line patching at runtime */
#define ARM64_LSE_ATOMIC_INSN(llsc, lse) \ #define ARM64_LSE_ATOMIC_INSN(llsc, lse) \
......
...@@ -24,6 +24,7 @@ ...@@ -24,6 +24,7 @@
#include <linux/compiler.h> #include <linux/compiler.h>
#include <linux/const.h> #include <linux/const.h>
#include <linux/types.h> #include <linux/types.h>
#include <asm/bug.h>
#include <asm/sizes.h> #include <asm/sizes.h>
/* /*
...@@ -45,15 +46,15 @@ ...@@ -45,15 +46,15 @@
* VA_START - the first kernel virtual address. * VA_START - the first kernel virtual address.
* TASK_SIZE - the maximum size of a user space task. * TASK_SIZE - the maximum size of a user space task.
* TASK_UNMAPPED_BASE - the lower boundary of the mmap VM area. * TASK_UNMAPPED_BASE - the lower boundary of the mmap VM area.
* The module space lives between the addresses given by TASK_SIZE
* and PAGE_OFFSET - it must be within 128MB of the kernel text.
*/ */
#define VA_BITS (CONFIG_ARM64_VA_BITS) #define VA_BITS (CONFIG_ARM64_VA_BITS)
#define VA_START (UL(0xffffffffffffffff) << VA_BITS) #define VA_START (UL(0xffffffffffffffff) << VA_BITS)
#define PAGE_OFFSET (UL(0xffffffffffffffff) << (VA_BITS - 1)) #define PAGE_OFFSET (UL(0xffffffffffffffff) << (VA_BITS - 1))
#define MODULES_END (PAGE_OFFSET) #define KIMAGE_VADDR (MODULES_END)
#define MODULES_VADDR (MODULES_END - SZ_64M) #define MODULES_END (MODULES_VADDR + MODULES_VSIZE)
#define PCI_IO_END (MODULES_VADDR - SZ_2M) #define MODULES_VADDR (VA_START + KASAN_SHADOW_SIZE)
#define MODULES_VSIZE (SZ_128M)
#define PCI_IO_END (PAGE_OFFSET - SZ_2M)
#define PCI_IO_START (PCI_IO_END - PCI_IO_SIZE) #define PCI_IO_START (PCI_IO_END - PCI_IO_SIZE)
#define FIXADDR_TOP (PCI_IO_START - SZ_2M) #define FIXADDR_TOP (PCI_IO_START - SZ_2M)
#define TASK_SIZE_64 (UL(1) << VA_BITS) #define TASK_SIZE_64 (UL(1) << VA_BITS)
...@@ -70,13 +71,28 @@ ...@@ -70,13 +71,28 @@
#define TASK_UNMAPPED_BASE (PAGE_ALIGN(TASK_SIZE / 4)) #define TASK_UNMAPPED_BASE (PAGE_ALIGN(TASK_SIZE / 4))
/*
* The size of the KASAN shadow region. This should be 1/8th of the
* size of the entire kernel virtual address space.
*/
#ifdef CONFIG_KASAN
#define KASAN_SHADOW_SIZE (UL(1) << (VA_BITS - 3))
#else
#define KASAN_SHADOW_SIZE (0)
#endif
/* /*
* Physical vs virtual RAM address space conversion. These are * Physical vs virtual RAM address space conversion. These are
* private definitions which should NOT be used outside memory.h * private definitions which should NOT be used outside memory.h
* files. Use virt_to_phys/phys_to_virt/__pa/__va instead. * files. Use virt_to_phys/phys_to_virt/__pa/__va instead.
*/ */
#define __virt_to_phys(x) (((phys_addr_t)(x) - PAGE_OFFSET + PHYS_OFFSET)) #define __virt_to_phys(x) ({ \
#define __phys_to_virt(x) ((unsigned long)((x) - PHYS_OFFSET + PAGE_OFFSET)) phys_addr_t __x = (phys_addr_t)(x); \
__x & BIT(VA_BITS - 1) ? (__x & ~PAGE_OFFSET) + PHYS_OFFSET : \
(__x - kimage_voffset); })
#define __phys_to_virt(x) ((unsigned long)((x) - PHYS_OFFSET) | PAGE_OFFSET)
#define __phys_to_kimg(x) ((unsigned long)((x) + kimage_voffset))
/* /*
* Convert a page to/from a physical address * Convert a page to/from a physical address
...@@ -100,19 +116,40 @@ ...@@ -100,19 +116,40 @@
#define MT_S2_NORMAL 0xf #define MT_S2_NORMAL 0xf
#define MT_S2_DEVICE_nGnRE 0x1 #define MT_S2_DEVICE_nGnRE 0x1
#ifdef CONFIG_ARM64_4K_PAGES
#define IOREMAP_MAX_ORDER (PUD_SHIFT)
#else
#define IOREMAP_MAX_ORDER (PMD_SHIFT)
#endif
#ifdef CONFIG_BLK_DEV_INITRD
#define __early_init_dt_declare_initrd(__start, __end) \
do { \
initrd_start = (__start); \
initrd_end = (__end); \
} while (0)
#endif
#ifndef __ASSEMBLY__ #ifndef __ASSEMBLY__
extern phys_addr_t memstart_addr; #include <linux/bitops.h>
#include <linux/mmdebug.h>
extern s64 memstart_addr;
/* PHYS_OFFSET - the physical address of the start of memory. */ /* PHYS_OFFSET - the physical address of the start of memory. */
#define PHYS_OFFSET ({ memstart_addr; }) #define PHYS_OFFSET ({ VM_BUG_ON(memstart_addr & 1); memstart_addr; })
/* the virtual base of the kernel image (minus TEXT_OFFSET) */
extern u64 kimage_vaddr;
/* the offset between the kernel virtual and physical mappings */
extern u64 kimage_voffset;
/* /*
* The maximum physical address that the linear direct mapping * Allow all memory at the discovery stage. We will clip it later.
* of system RAM can cover. (PAGE_OFFSET can be interpreted as
* a 2's complement signed quantity and negated to derive the
* maximum size of the linear mapping.)
*/ */
#define MAX_MEMBLOCK_ADDR ({ memstart_addr - PAGE_OFFSET - 1; }) #define MIN_MEMBLOCK_ADDR 0
#define MAX_MEMBLOCK_ADDR U64_MAX
/* /*
* PFNs are used to describe any physical page; this means * PFNs are used to describe any physical page; this means
......
...@@ -27,6 +27,7 @@ ...@@ -27,6 +27,7 @@
#include <asm-generic/mm_hooks.h> #include <asm-generic/mm_hooks.h>
#include <asm/cputype.h> #include <asm/cputype.h>
#include <asm/pgtable.h> #include <asm/pgtable.h>
#include <asm/tlbflush.h>
#ifdef CONFIG_PID_IN_CONTEXTIDR #ifdef CONFIG_PID_IN_CONTEXTIDR
static inline void contextidr_thread_switch(struct task_struct *next) static inline void contextidr_thread_switch(struct task_struct *next)
...@@ -48,7 +49,7 @@ static inline void contextidr_thread_switch(struct task_struct *next) ...@@ -48,7 +49,7 @@ static inline void contextidr_thread_switch(struct task_struct *next)
*/ */
static inline void cpu_set_reserved_ttbr0(void) static inline void cpu_set_reserved_ttbr0(void)
{ {
unsigned long ttbr = page_to_phys(empty_zero_page); unsigned long ttbr = virt_to_phys(empty_zero_page);
asm( asm(
" msr ttbr0_el1, %0 // set TTBR0\n" " msr ttbr0_el1, %0 // set TTBR0\n"
...@@ -73,7 +74,7 @@ static inline bool __cpu_uses_extended_idmap(void) ...@@ -73,7 +74,7 @@ static inline bool __cpu_uses_extended_idmap(void)
/* /*
* Set TCR.T0SZ to its default value (based on VA_BITS) * Set TCR.T0SZ to its default value (based on VA_BITS)
*/ */
static inline void cpu_set_default_tcr_t0sz(void) static inline void __cpu_set_tcr_t0sz(unsigned long t0sz)
{ {
unsigned long tcr; unsigned long tcr;
...@@ -86,7 +87,62 @@ static inline void cpu_set_default_tcr_t0sz(void) ...@@ -86,7 +87,62 @@ static inline void cpu_set_default_tcr_t0sz(void)
" msr tcr_el1, %0 ;" " msr tcr_el1, %0 ;"
" isb" " isb"
: "=&r" (tcr) : "=&r" (tcr)
: "r"(TCR_T0SZ(VA_BITS)), "I"(TCR_T0SZ_OFFSET), "I"(TCR_TxSZ_WIDTH)); : "r"(t0sz), "I"(TCR_T0SZ_OFFSET), "I"(TCR_TxSZ_WIDTH));
}
#define cpu_set_default_tcr_t0sz() __cpu_set_tcr_t0sz(TCR_T0SZ(VA_BITS))
#define cpu_set_idmap_tcr_t0sz() __cpu_set_tcr_t0sz(idmap_t0sz)
/*
* Remove the idmap from TTBR0_EL1 and install the pgd of the active mm.
*
* The idmap lives in the same VA range as userspace, but uses global entries
* and may use a different TCR_EL1.T0SZ. To avoid issues resulting from
* speculative TLB fetches, we must temporarily install the reserved page
* tables while we invalidate the TLBs and set up the correct TCR_EL1.T0SZ.
*
* If current is a not a user task, the mm covers the TTBR1_EL1 page tables,
* which should not be installed in TTBR0_EL1. In this case we can leave the
* reserved page tables in place.
*/
static inline void cpu_uninstall_idmap(void)
{
struct mm_struct *mm = current->active_mm;
cpu_set_reserved_ttbr0();
local_flush_tlb_all();
cpu_set_default_tcr_t0sz();
if (mm != &init_mm)
cpu_switch_mm(mm->pgd, mm);
}
static inline void cpu_install_idmap(void)
{
cpu_set_reserved_ttbr0();
local_flush_tlb_all();
cpu_set_idmap_tcr_t0sz();
cpu_switch_mm(idmap_pg_dir, &init_mm);
}
/*
* Atomically replaces the active TTBR1_EL1 PGD with a new VA-compatible PGD,
* avoiding the possibility of conflicting TLB entries being allocated.
*/
static inline void cpu_replace_ttbr1(pgd_t *pgd)
{
typedef void (ttbr_replace_func)(phys_addr_t);
extern ttbr_replace_func idmap_cpu_replace_ttbr1;
ttbr_replace_func *replace_phys;
phys_addr_t pgd_phys = virt_to_phys(pgd);
replace_phys = (void *)virt_to_phys(idmap_cpu_replace_ttbr1);
cpu_install_idmap();
replace_phys(pgd_phys);
cpu_uninstall_idmap();
} }
/* /*
...@@ -147,4 +203,6 @@ switch_mm(struct mm_struct *prev, struct mm_struct *next, ...@@ -147,4 +203,6 @@ switch_mm(struct mm_struct *prev, struct mm_struct *next,
#define deactivate_mm(tsk,mm) do { } while (0) #define deactivate_mm(tsk,mm) do { } while (0)
#define activate_mm(prev,next) switch_mm(prev, next, NULL) #define activate_mm(prev,next) switch_mm(prev, next, NULL)
void verify_cpu_asid_bits(void);
#endif #endif
...@@ -20,4 +20,21 @@ ...@@ -20,4 +20,21 @@
#define MODULE_ARCH_VERMAGIC "aarch64" #define MODULE_ARCH_VERMAGIC "aarch64"
#ifdef CONFIG_ARM64_MODULE_PLTS
struct mod_arch_specific {
struct elf64_shdr *plt;
int plt_num_entries;
int plt_max_entries;
};
#endif
u64 module_emit_plt_entry(struct module *mod, const Elf64_Rela *rela,
Elf64_Sym *sym);
#ifdef CONFIG_RANDOMIZE_BASE
extern u64 module_alloc_base;
#else
#define module_alloc_base ((u64)_etext - MODULES_VSIZE)
#endif
#endif /* __ASM_MODULE_H */ #endif /* __ASM_MODULE_H */
...@@ -42,11 +42,20 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) ...@@ -42,11 +42,20 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
free_page((unsigned long)pmd); free_page((unsigned long)pmd);
} }
static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) static inline void __pud_populate(pud_t *pud, phys_addr_t pmd, pudval_t prot)
{ {
set_pud(pud, __pud(__pa(pmd) | PMD_TYPE_TABLE)); set_pud(pud, __pud(pmd | prot));
} }
static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
{
__pud_populate(pud, __pa(pmd), PMD_TYPE_TABLE);
}
#else
static inline void __pud_populate(pud_t *pud, phys_addr_t pmd, pudval_t prot)
{
BUILD_BUG();
}
#endif /* CONFIG_PGTABLE_LEVELS > 2 */ #endif /* CONFIG_PGTABLE_LEVELS > 2 */
#if CONFIG_PGTABLE_LEVELS > 3 #if CONFIG_PGTABLE_LEVELS > 3
...@@ -62,11 +71,20 @@ static inline void pud_free(struct mm_struct *mm, pud_t *pud) ...@@ -62,11 +71,20 @@ static inline void pud_free(struct mm_struct *mm, pud_t *pud)
free_page((unsigned long)pud); free_page((unsigned long)pud);
} }
static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pud_t *pud) static inline void __pgd_populate(pgd_t *pgdp, phys_addr_t pud, pgdval_t prot)
{ {
set_pgd(pgd, __pgd(__pa(pud) | PUD_TYPE_TABLE)); set_pgd(pgdp, __pgd(pud | prot));
} }
static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pud_t *pud)
{
__pgd_populate(pgd, __pa(pud), PUD_TYPE_TABLE);
}
#else
static inline void __pgd_populate(pgd_t *pgdp, phys_addr_t pud, pgdval_t prot)
{
BUILD_BUG();
}
#endif /* CONFIG_PGTABLE_LEVELS > 3 */ #endif /* CONFIG_PGTABLE_LEVELS > 3 */
extern pgd_t *pgd_alloc(struct mm_struct *mm); extern pgd_t *pgd_alloc(struct mm_struct *mm);
......
/*
* Copyright (C) 2016 ARM Ltd.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef __ASM_PGTABLE_PROT_H
#define __ASM_PGTABLE_PROT_H
#include <asm/memory.h>
#include <asm/pgtable-hwdef.h>
#include <linux/const.h>
/*
* Software defined PTE bits definition.
*/
#define PTE_VALID (_AT(pteval_t, 1) << 0)
#define PTE_WRITE (PTE_DBM) /* same as DBM (51) */
#define PTE_DIRTY (_AT(pteval_t, 1) << 55)
#define PTE_SPECIAL (_AT(pteval_t, 1) << 56)
#define PTE_PROT_NONE (_AT(pteval_t, 1) << 58) /* only when !PTE_VALID */
#ifndef __ASSEMBLY__
#include <asm/pgtable-types.h>
#define PROT_DEFAULT (PTE_TYPE_PAGE | PTE_AF | PTE_SHARED)
#define PROT_SECT_DEFAULT (PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S)
#define PROT_DEVICE_nGnRnE (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_WRITE | PTE_ATTRINDX(MT_DEVICE_nGnRnE))
#define PROT_DEVICE_nGnRE (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_WRITE | PTE_ATTRINDX(MT_DEVICE_nGnRE))
#define PROT_NORMAL_NC (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_WRITE | PTE_ATTRINDX(MT_NORMAL_NC))
#define PROT_NORMAL_WT (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_WRITE | PTE_ATTRINDX(MT_NORMAL_WT))
#define PROT_NORMAL (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_WRITE | PTE_ATTRINDX(MT_NORMAL))
#define PROT_SECT_DEVICE_nGnRE (PROT_SECT_DEFAULT | PMD_SECT_PXN | PMD_SECT_UXN | PMD_ATTRINDX(MT_DEVICE_nGnRE))
#define PROT_SECT_NORMAL (PROT_SECT_DEFAULT | PMD_SECT_PXN | PMD_SECT_UXN | PMD_ATTRINDX(MT_NORMAL))
#define PROT_SECT_NORMAL_EXEC (PROT_SECT_DEFAULT | PMD_SECT_UXN | PMD_ATTRINDX(MT_NORMAL))
#define _PAGE_DEFAULT (PROT_DEFAULT | PTE_ATTRINDX(MT_NORMAL))
#define PAGE_KERNEL __pgprot(_PAGE_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_WRITE)
#define PAGE_KERNEL_RO __pgprot(_PAGE_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_RDONLY)
#define PAGE_KERNEL_ROX __pgprot(_PAGE_DEFAULT | PTE_UXN | PTE_DIRTY | PTE_RDONLY)
#define PAGE_KERNEL_EXEC __pgprot(_PAGE_DEFAULT | PTE_UXN | PTE_DIRTY | PTE_WRITE)
#define PAGE_KERNEL_EXEC_CONT __pgprot(_PAGE_DEFAULT | PTE_UXN | PTE_DIRTY | PTE_WRITE | PTE_CONT)
#define PAGE_HYP __pgprot(_PAGE_DEFAULT | PTE_HYP)
#define PAGE_HYP_DEVICE __pgprot(PROT_DEVICE_nGnRE | PTE_HYP)
#define PAGE_S2 __pgprot(PROT_DEFAULT | PTE_S2_MEMATTR(MT_S2_NORMAL) | PTE_S2_RDONLY)
#define PAGE_S2_DEVICE __pgprot(PROT_DEFAULT | PTE_S2_MEMATTR(MT_S2_DEVICE_nGnRE) | PTE_S2_RDONLY | PTE_UXN)
#define PAGE_NONE __pgprot(((_PAGE_DEFAULT) & ~PTE_VALID) | PTE_PROT_NONE | PTE_PXN | PTE_UXN)
#define PAGE_SHARED __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_UXN | PTE_WRITE)
#define PAGE_SHARED_EXEC __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_WRITE)
#define PAGE_COPY __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_UXN)
#define PAGE_COPY_EXEC __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN)
#define PAGE_READONLY __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_UXN)
#define PAGE_READONLY_EXEC __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN)
#define __P000 PAGE_NONE
#define __P001 PAGE_READONLY
#define __P010 PAGE_COPY
#define __P011 PAGE_COPY
#define __P100 PAGE_READONLY_EXEC
#define __P101 PAGE_READONLY_EXEC
#define __P110 PAGE_COPY_EXEC
#define __P111 PAGE_COPY_EXEC
#define __S000 PAGE_NONE
#define __S001 PAGE_READONLY
#define __S010 PAGE_SHARED
#define __S011 PAGE_SHARED
#define __S100 PAGE_READONLY_EXEC
#define __S101 PAGE_READONLY_EXEC
#define __S110 PAGE_SHARED_EXEC
#define __S111 PAGE_SHARED_EXEC
#endif /* __ASSEMBLY__ */
#endif /* __ASM_PGTABLE_PROT_H */
This diff is collapsed.
...@@ -29,8 +29,10 @@ ...@@ -29,8 +29,10 @@
#include <linux/string.h> #include <linux/string.h>
#include <asm/alternative.h>
#include <asm/fpsimd.h> #include <asm/fpsimd.h>
#include <asm/hw_breakpoint.h> #include <asm/hw_breakpoint.h>
#include <asm/lse.h>
#include <asm/pgtable-hwdef.h> #include <asm/pgtable-hwdef.h>
#include <asm/ptrace.h> #include <asm/ptrace.h>
#include <asm/types.h> #include <asm/types.h>
...@@ -177,9 +179,11 @@ static inline void prefetchw(const void *ptr) ...@@ -177,9 +179,11 @@ static inline void prefetchw(const void *ptr)
} }
#define ARCH_HAS_SPINLOCK_PREFETCH #define ARCH_HAS_SPINLOCK_PREFETCH
static inline void spin_lock_prefetch(const void *x) static inline void spin_lock_prefetch(const void *ptr)
{ {
prefetchw(x); asm volatile(ARM64_LSE_ATOMIC_INSN(
"prfm pstl1strm, %a0",
"nop") : : "p" (ptr));
} }
#define HAVE_ARCH_PICK_MMAP_LAYOUT #define HAVE_ARCH_PICK_MMAP_LAYOUT
...@@ -187,5 +191,6 @@ static inline void spin_lock_prefetch(const void *x) ...@@ -187,5 +191,6 @@ static inline void spin_lock_prefetch(const void *x)
#endif #endif
void cpu_enable_pan(void *__unused); void cpu_enable_pan(void *__unused);
void cpu_enable_uao(void *__unused);
#endif /* __ASM_PROCESSOR_H */ #endif /* __ASM_PROCESSOR_H */
...@@ -58,6 +58,7 @@ ...@@ -58,6 +58,7 @@
#define COMPAT_PSR_Z_BIT 0x40000000 #define COMPAT_PSR_Z_BIT 0x40000000
#define COMPAT_PSR_N_BIT 0x80000000 #define COMPAT_PSR_N_BIT 0x80000000
#define COMPAT_PSR_IT_MASK 0x0600fc00 /* If-Then execution state mask */ #define COMPAT_PSR_IT_MASK 0x0600fc00 /* If-Then execution state mask */
#define COMPAT_PSR_GE_MASK 0x000f0000
#ifdef CONFIG_CPU_BIG_ENDIAN #ifdef CONFIG_CPU_BIG_ENDIAN
#define COMPAT_PSR_ENDSTATE COMPAT_PSR_E_BIT #define COMPAT_PSR_ENDSTATE COMPAT_PSR_E_BIT
...@@ -151,35 +152,9 @@ static inline unsigned long regs_return_value(struct pt_regs *regs) ...@@ -151,35 +152,9 @@ static inline unsigned long regs_return_value(struct pt_regs *regs)
return regs->regs[0]; return regs->regs[0];
} }
/* /* We must avoid circular header include via sched.h */
* Are the current registers suitable for user mode? (used to maintain struct task_struct;
* security in signal handlers) int valid_user_regs(struct user_pt_regs *regs, struct task_struct *task);
*/
static inline int valid_user_regs(struct user_pt_regs *regs)
{
if (user_mode(regs) && (regs->pstate & PSR_I_BIT) == 0) {
regs->pstate &= ~(PSR_F_BIT | PSR_A_BIT);
/* The T bit is reserved for AArch64 */
if (!(regs->pstate & PSR_MODE32_BIT))
regs->pstate &= ~COMPAT_PSR_T_BIT;
return 1;
}
/*
* Force PSR to something logical...
*/
regs->pstate &= PSR_f | PSR_s | (PSR_x & ~PSR_A_BIT) | \
COMPAT_PSR_T_BIT | PSR_MODE32_BIT;
if (!(regs->pstate & PSR_MODE32_BIT)) {
regs->pstate &= ~COMPAT_PSR_T_BIT;
regs->pstate |= PSR_MODE_EL0t;
}
return 0;
}
#define instruction_pointer(regs) ((unsigned long)(regs)->pc) #define instruction_pointer(regs) ((unsigned long)(regs)->pc)
......
...@@ -16,6 +16,19 @@ ...@@ -16,6 +16,19 @@
#ifndef __ASM_SMP_H #ifndef __ASM_SMP_H
#define __ASM_SMP_H #define __ASM_SMP_H
/* Values for secondary_data.status */
#define CPU_MMU_OFF (-1)
#define CPU_BOOT_SUCCESS (0)
/* The cpu invoked ops->cpu_die, synchronise it with cpu_kill */
#define CPU_KILL_ME (1)
/* The cpu couldn't die gracefully and is looping in the kernel */
#define CPU_STUCK_IN_KERNEL (2)
/* Fatal system error detected by secondary CPU, crash the system */
#define CPU_PANIC_KERNEL (3)
#ifndef __ASSEMBLY__
#include <linux/threads.h> #include <linux/threads.h>
#include <linux/cpumask.h> #include <linux/cpumask.h>
#include <linux/thread_info.h> #include <linux/thread_info.h>
...@@ -54,19 +67,52 @@ asmlinkage void secondary_start_kernel(void); ...@@ -54,19 +67,52 @@ asmlinkage void secondary_start_kernel(void);
/* /*
* Initial data for bringing up a secondary CPU. * Initial data for bringing up a secondary CPU.
* @stack - sp for the secondary CPU
* @status - Result passed back from the secondary CPU to
* indicate failure.
*/ */
struct secondary_data { struct secondary_data {
void *stack; void *stack;
long status;
}; };
extern struct secondary_data secondary_data; extern struct secondary_data secondary_data;
extern long __early_cpu_boot_status;
extern void secondary_entry(void); extern void secondary_entry(void);
extern void arch_send_call_function_single_ipi(int cpu); extern void arch_send_call_function_single_ipi(int cpu);
extern void arch_send_call_function_ipi_mask(const struct cpumask *mask); extern void arch_send_call_function_ipi_mask(const struct cpumask *mask);
#ifdef CONFIG_ARM64_ACPI_PARKING_PROTOCOL
extern void arch_send_wakeup_ipi_mask(const struct cpumask *mask);
#else
static inline void arch_send_wakeup_ipi_mask(const struct cpumask *mask)
{
BUILD_BUG();
}
#endif
extern int __cpu_disable(void); extern int __cpu_disable(void);
extern void __cpu_die(unsigned int cpu); extern void __cpu_die(unsigned int cpu);
extern void cpu_die(void); extern void cpu_die(void);
extern void cpu_die_early(void);
static inline void cpu_park_loop(void)
{
for (;;) {
wfe();
wfi();
}
}
static inline void update_cpu_boot_status(int val)
{
WRITE_ONCE(secondary_data.status, val);
/* Ensure the visibility of the status update */
dsb(ishst);
}
#endif /* ifndef __ASSEMBLY__ */
#endif /* ifndef __ASM_SMP_H */ #endif /* ifndef __ASM_SMP_H */
...@@ -72,15 +72,19 @@ ...@@ -72,15 +72,19 @@
#define SYS_ID_AA64MMFR0_EL1 sys_reg(3, 0, 0, 7, 0) #define SYS_ID_AA64MMFR0_EL1 sys_reg(3, 0, 0, 7, 0)
#define SYS_ID_AA64MMFR1_EL1 sys_reg(3, 0, 0, 7, 1) #define SYS_ID_AA64MMFR1_EL1 sys_reg(3, 0, 0, 7, 1)
#define SYS_ID_AA64MMFR2_EL1 sys_reg(3, 0, 0, 7, 2)
#define SYS_CNTFRQ_EL0 sys_reg(3, 3, 14, 0, 0) #define SYS_CNTFRQ_EL0 sys_reg(3, 3, 14, 0, 0)
#define SYS_CTR_EL0 sys_reg(3, 3, 0, 0, 1) #define SYS_CTR_EL0 sys_reg(3, 3, 0, 0, 1)
#define SYS_DCZID_EL0 sys_reg(3, 3, 0, 0, 7) #define SYS_DCZID_EL0 sys_reg(3, 3, 0, 0, 7)
#define REG_PSTATE_PAN_IMM sys_reg(0, 0, 4, 0, 4) #define REG_PSTATE_PAN_IMM sys_reg(0, 0, 4, 0, 4)
#define REG_PSTATE_UAO_IMM sys_reg(0, 0, 4, 0, 3)
#define SET_PSTATE_PAN(x) __inst_arm(0xd5000000 | REG_PSTATE_PAN_IMM |\ #define SET_PSTATE_PAN(x) __inst_arm(0xd5000000 | REG_PSTATE_PAN_IMM |\
(!!x)<<8 | 0x1f) (!!x)<<8 | 0x1f)
#define SET_PSTATE_UAO(x) __inst_arm(0xd5000000 | REG_PSTATE_UAO_IMM |\
(!!x)<<8 | 0x1f)
/* SCTLR_EL1 */ /* SCTLR_EL1 */
#define SCTLR_EL1_CP15BEN (0x1 << 5) #define SCTLR_EL1_CP15BEN (0x1 << 5)
...@@ -137,6 +141,9 @@ ...@@ -137,6 +141,9 @@
#define ID_AA64MMFR1_VMIDBITS_SHIFT 4 #define ID_AA64MMFR1_VMIDBITS_SHIFT 4
#define ID_AA64MMFR1_HADBS_SHIFT 0 #define ID_AA64MMFR1_HADBS_SHIFT 0
/* id_aa64mmfr2 */
#define ID_AA64MMFR2_UAO_SHIFT 4
/* id_aa64dfr0 */ /* id_aa64dfr0 */
#define ID_AA64DFR0_CTX_CMPS_SHIFT 28 #define ID_AA64DFR0_CTX_CMPS_SHIFT 28
#define ID_AA64DFR0_WRPS_SHIFT 20 #define ID_AA64DFR0_WRPS_SHIFT 20
...@@ -196,16 +203,16 @@ ...@@ -196,16 +203,16 @@
#ifdef __ASSEMBLY__ #ifdef __ASSEMBLY__
.irp num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30 .irp num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30
.equ __reg_num_x\num, \num .equ .L__reg_num_x\num, \num
.endr .endr
.equ __reg_num_xzr, 31 .equ .L__reg_num_xzr, 31
.macro mrs_s, rt, sreg .macro mrs_s, rt, sreg
.inst 0xd5200000|(\sreg)|(__reg_num_\rt) .inst 0xd5200000|(\sreg)|(.L__reg_num_\rt)
.endm .endm
.macro msr_s, sreg, rt .macro msr_s, sreg, rt
.inst 0xd5000000|(\sreg)|(__reg_num_\rt) .inst 0xd5000000|(\sreg)|(.L__reg_num_\rt)
.endm .endm
#else #else
...@@ -214,16 +221,16 @@ ...@@ -214,16 +221,16 @@
asm( asm(
" .irp num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30\n" " .irp num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30\n"
" .equ __reg_num_x\\num, \\num\n" " .equ .L__reg_num_x\\num, \\num\n"
" .endr\n" " .endr\n"
" .equ __reg_num_xzr, 31\n" " .equ .L__reg_num_xzr, 31\n"
"\n" "\n"
" .macro mrs_s, rt, sreg\n" " .macro mrs_s, rt, sreg\n"
" .inst 0xd5200000|(\\sreg)|(__reg_num_\\rt)\n" " .inst 0xd5200000|(\\sreg)|(.L__reg_num_\\rt)\n"
" .endm\n" " .endm\n"
"\n" "\n"
" .macro msr_s, sreg, rt\n" " .macro msr_s, sreg, rt\n"
" .inst 0xd5000000|(\\sreg)|(__reg_num_\\rt)\n" " .inst 0xd5000000|(\\sreg)|(.L__reg_num_\\rt)\n"
" .endm\n" " .endm\n"
); );
......
...@@ -36,11 +36,11 @@ ...@@ -36,11 +36,11 @@
#define VERIFY_WRITE 1 #define VERIFY_WRITE 1
/* /*
* The exception table consists of pairs of addresses: the first is the * The exception table consists of pairs of relative offsets: the first
* address of an instruction that is allowed to fault, and the second is * is the relative offset to an instruction that is allowed to fault,
* the address at which the program should continue. No registers are * and the second is the relative offset at which the program should
* modified, so it is entirely up to the continuation code to figure out * continue. No registers are modified, so it is entirely up to the
* what to do. * continuation code to figure out what to do.
* *
* All the routines below use bits of fixup code that are out of line * All the routines below use bits of fixup code that are out of line
* with the main instruction path. This means when everything is well, * with the main instruction path. This means when everything is well,
...@@ -50,9 +50,11 @@ ...@@ -50,9 +50,11 @@
struct exception_table_entry struct exception_table_entry
{ {
unsigned long insn, fixup; int insn, fixup;
}; };
#define ARCH_HAS_RELATIVE_EXTABLE
extern int fixup_exception(struct pt_regs *regs); extern int fixup_exception(struct pt_regs *regs);
#define KERNEL_DS (-1UL) #define KERNEL_DS (-1UL)
...@@ -64,6 +66,16 @@ extern int fixup_exception(struct pt_regs *regs); ...@@ -64,6 +66,16 @@ extern int fixup_exception(struct pt_regs *regs);
static inline void set_fs(mm_segment_t fs) static inline void set_fs(mm_segment_t fs)
{ {
current_thread_info()->addr_limit = fs; current_thread_info()->addr_limit = fs;
/*
* Enable/disable UAO so that copy_to_user() etc can access
* kernel memory with the unprivileged instructions.
*/
if (IS_ENABLED(CONFIG_ARM64_UAO) && fs == KERNEL_DS)
asm(ALTERNATIVE("nop", SET_PSTATE_UAO(1), ARM64_HAS_UAO));
else
asm(ALTERNATIVE("nop", SET_PSTATE_UAO(0), ARM64_HAS_UAO,
CONFIG_ARM64_UAO));
} }
#define segment_eq(a, b) ((a) == (b)) #define segment_eq(a, b) ((a) == (b))
...@@ -105,6 +117,12 @@ static inline void set_fs(mm_segment_t fs) ...@@ -105,6 +117,12 @@ static inline void set_fs(mm_segment_t fs)
#define access_ok(type, addr, size) __range_ok(addr, size) #define access_ok(type, addr, size) __range_ok(addr, size)
#define user_addr_max get_fs #define user_addr_max get_fs
#define _ASM_EXTABLE(from, to) \
" .pushsection __ex_table, \"a\"\n" \
" .align 3\n" \
" .long (" #from " - .), (" #to " - .)\n" \
" .popsection\n"
/* /*
* The "__xxx" versions of the user access functions do not verify the address * The "__xxx" versions of the user access functions do not verify the address
* space - it must have been done previously with a separate "access_ok()" * space - it must have been done previously with a separate "access_ok()"
...@@ -113,9 +131,10 @@ static inline void set_fs(mm_segment_t fs) ...@@ -113,9 +131,10 @@ static inline void set_fs(mm_segment_t fs)
* The "__xxx_error" versions set the third argument to -EFAULT if an error * The "__xxx_error" versions set the third argument to -EFAULT if an error
* occurs, and leave it unchanged on success. * occurs, and leave it unchanged on success.
*/ */
#define __get_user_asm(instr, reg, x, addr, err) \ #define __get_user_asm(instr, alt_instr, reg, x, addr, err, feature) \
asm volatile( \ asm volatile( \
"1: " instr " " reg "1, [%2]\n" \ "1:"ALTERNATIVE(instr " " reg "1, [%2]\n", \
alt_instr " " reg "1, [%2]\n", feature) \
"2:\n" \ "2:\n" \
" .section .fixup, \"ax\"\n" \ " .section .fixup, \"ax\"\n" \
" .align 2\n" \ " .align 2\n" \
...@@ -123,10 +142,7 @@ static inline void set_fs(mm_segment_t fs) ...@@ -123,10 +142,7 @@ static inline void set_fs(mm_segment_t fs)
" mov %1, #0\n" \ " mov %1, #0\n" \
" b 2b\n" \ " b 2b\n" \
" .previous\n" \ " .previous\n" \
" .section __ex_table,\"a\"\n" \ _ASM_EXTABLE(1b, 3b) \
" .align 3\n" \
" .quad 1b, 3b\n" \
" .previous" \
: "+r" (err), "=&r" (x) \ : "+r" (err), "=&r" (x) \
: "r" (addr), "i" (-EFAULT)) : "r" (addr), "i" (-EFAULT))
...@@ -134,26 +150,30 @@ static inline void set_fs(mm_segment_t fs) ...@@ -134,26 +150,30 @@ static inline void set_fs(mm_segment_t fs)
do { \ do { \
unsigned long __gu_val; \ unsigned long __gu_val; \
__chk_user_ptr(ptr); \ __chk_user_ptr(ptr); \
asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, \ asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_ALT_PAN_NOT_UAO,\
CONFIG_ARM64_PAN)); \ CONFIG_ARM64_PAN)); \
switch (sizeof(*(ptr))) { \ switch (sizeof(*(ptr))) { \
case 1: \ case 1: \
__get_user_asm("ldrb", "%w", __gu_val, (ptr), (err)); \ __get_user_asm("ldrb", "ldtrb", "%w", __gu_val, (ptr), \
(err), ARM64_HAS_UAO); \
break; \ break; \
case 2: \ case 2: \
__get_user_asm("ldrh", "%w", __gu_val, (ptr), (err)); \ __get_user_asm("ldrh", "ldtrh", "%w", __gu_val, (ptr), \
(err), ARM64_HAS_UAO); \
break; \ break; \
case 4: \ case 4: \
__get_user_asm("ldr", "%w", __gu_val, (ptr), (err)); \ __get_user_asm("ldr", "ldtr", "%w", __gu_val, (ptr), \
(err), ARM64_HAS_UAO); \
break; \ break; \
case 8: \ case 8: \
__get_user_asm("ldr", "%", __gu_val, (ptr), (err)); \ __get_user_asm("ldr", "ldtr", "%", __gu_val, (ptr), \
(err), ARM64_HAS_UAO); \
break; \ break; \
default: \ default: \
BUILD_BUG(); \ BUILD_BUG(); \
} \ } \
(x) = (__force __typeof__(*(ptr)))__gu_val; \ (x) = (__force __typeof__(*(ptr)))__gu_val; \
asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, \ asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_ALT_PAN_NOT_UAO,\
CONFIG_ARM64_PAN)); \ CONFIG_ARM64_PAN)); \
} while (0) } while (0)
...@@ -181,19 +201,17 @@ do { \ ...@@ -181,19 +201,17 @@ do { \
((x) = 0, -EFAULT); \ ((x) = 0, -EFAULT); \
}) })
#define __put_user_asm(instr, reg, x, addr, err) \ #define __put_user_asm(instr, alt_instr, reg, x, addr, err, feature) \
asm volatile( \ asm volatile( \
"1: " instr " " reg "1, [%2]\n" \ "1:"ALTERNATIVE(instr " " reg "1, [%2]\n", \
alt_instr " " reg "1, [%2]\n", feature) \
"2:\n" \ "2:\n" \
" .section .fixup,\"ax\"\n" \ " .section .fixup,\"ax\"\n" \
" .align 2\n" \ " .align 2\n" \
"3: mov %w0, %3\n" \ "3: mov %w0, %3\n" \
" b 2b\n" \ " b 2b\n" \
" .previous\n" \ " .previous\n" \
" .section __ex_table,\"a\"\n" \ _ASM_EXTABLE(1b, 3b) \
" .align 3\n" \
" .quad 1b, 3b\n" \
" .previous" \
: "+r" (err) \ : "+r" (err) \
: "r" (x), "r" (addr), "i" (-EFAULT)) : "r" (x), "r" (addr), "i" (-EFAULT))
...@@ -201,25 +219,29 @@ do { \ ...@@ -201,25 +219,29 @@ do { \
do { \ do { \
__typeof__(*(ptr)) __pu_val = (x); \ __typeof__(*(ptr)) __pu_val = (x); \
__chk_user_ptr(ptr); \ __chk_user_ptr(ptr); \
asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, \ asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_ALT_PAN_NOT_UAO,\
CONFIG_ARM64_PAN)); \ CONFIG_ARM64_PAN)); \
switch (sizeof(*(ptr))) { \ switch (sizeof(*(ptr))) { \
case 1: \ case 1: \
__put_user_asm("strb", "%w", __pu_val, (ptr), (err)); \ __put_user_asm("strb", "sttrb", "%w", __pu_val, (ptr), \
(err), ARM64_HAS_UAO); \
break; \ break; \
case 2: \ case 2: \
__put_user_asm("strh", "%w", __pu_val, (ptr), (err)); \ __put_user_asm("strh", "sttrh", "%w", __pu_val, (ptr), \
(err), ARM64_HAS_UAO); \
break; \ break; \
case 4: \ case 4: \
__put_user_asm("str", "%w", __pu_val, (ptr), (err)); \ __put_user_asm("str", "sttr", "%w", __pu_val, (ptr), \
(err), ARM64_HAS_UAO); \
break; \ break; \
case 8: \ case 8: \
__put_user_asm("str", "%", __pu_val, (ptr), (err)); \ __put_user_asm("str", "sttr", "%", __pu_val, (ptr), \
(err), ARM64_HAS_UAO); \
break; \ break; \
default: \ default: \
BUILD_BUG(); \ BUILD_BUG(); \
} \ } \
asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, \ asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_ALT_PAN_NOT_UAO,\
CONFIG_ARM64_PAN)); \ CONFIG_ARM64_PAN)); \
} while (0) } while (0)
......
...@@ -16,6 +16,8 @@ ...@@ -16,6 +16,8 @@
#ifndef __ASM_WORD_AT_A_TIME_H #ifndef __ASM_WORD_AT_A_TIME_H
#define __ASM_WORD_AT_A_TIME_H #define __ASM_WORD_AT_A_TIME_H
#include <asm/uaccess.h>
#ifndef __AARCH64EB__ #ifndef __AARCH64EB__
#include <linux/kernel.h> #include <linux/kernel.h>
...@@ -81,10 +83,7 @@ static inline unsigned long load_unaligned_zeropad(const void *addr) ...@@ -81,10 +83,7 @@ static inline unsigned long load_unaligned_zeropad(const void *addr)
#endif #endif
" b 2b\n" " b 2b\n"
" .popsection\n" " .popsection\n"
" .pushsection __ex_table,\"a\"\n" _ASM_EXTABLE(1b, 3b)
" .align 3\n"
" .quad 1b, 3b\n"
" .popsection"
: "=&r" (ret), "=&r" (offset) : "=&r" (ret), "=&r" (offset)
: "r" (addr), "Q" (*(unsigned long *)addr)); : "r" (addr), "Q" (*(unsigned long *)addr));
......
...@@ -28,5 +28,7 @@ ...@@ -28,5 +28,7 @@
#define HWCAP_SHA2 (1 << 6) #define HWCAP_SHA2 (1 << 6)
#define HWCAP_CRC32 (1 << 7) #define HWCAP_CRC32 (1 << 7)
#define HWCAP_ATOMICS (1 << 8) #define HWCAP_ATOMICS (1 << 8)
#define HWCAP_FPHP (1 << 9)
#define HWCAP_ASIMDHP (1 << 10)
#endif /* _UAPI__ASM_HWCAP_H */ #endif /* _UAPI__ASM_HWCAP_H */
...@@ -45,6 +45,7 @@ ...@@ -45,6 +45,7 @@
#define PSR_A_BIT 0x00000100 #define PSR_A_BIT 0x00000100
#define PSR_D_BIT 0x00000200 #define PSR_D_BIT 0x00000200
#define PSR_PAN_BIT 0x00400000 #define PSR_PAN_BIT 0x00400000
#define PSR_UAO_BIT 0x00800000
#define PSR_Q_BIT 0x08000000 #define PSR_Q_BIT 0x08000000
#define PSR_V_BIT 0x10000000 #define PSR_V_BIT 0x10000000
#define PSR_C_BIT 0x20000000 #define PSR_C_BIT 0x20000000
......
...@@ -30,6 +30,7 @@ arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \ ...@@ -30,6 +30,7 @@ arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \
../../arm/kernel/opcodes.o ../../arm/kernel/opcodes.o
arm64-obj-$(CONFIG_FUNCTION_TRACER) += ftrace.o entry-ftrace.o arm64-obj-$(CONFIG_FUNCTION_TRACER) += ftrace.o entry-ftrace.o
arm64-obj-$(CONFIG_MODULES) += arm64ksyms.o module.o arm64-obj-$(CONFIG_MODULES) += arm64ksyms.o module.o
arm64-obj-$(CONFIG_ARM64_MODULE_PLTS) += module-plts.o
arm64-obj-$(CONFIG_PERF_EVENTS) += perf_regs.o perf_callchain.o arm64-obj-$(CONFIG_PERF_EVENTS) += perf_regs.o perf_callchain.o
arm64-obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o arm64-obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o
arm64-obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o arm64-obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o
...@@ -41,7 +42,9 @@ arm64-obj-$(CONFIG_EFI) += efi.o efi-entry.stub.o ...@@ -41,7 +42,9 @@ arm64-obj-$(CONFIG_EFI) += efi.o efi-entry.stub.o
arm64-obj-$(CONFIG_PCI) += pci.o arm64-obj-$(CONFIG_PCI) += pci.o
arm64-obj-$(CONFIG_ARMV8_DEPRECATED) += armv8_deprecated.o arm64-obj-$(CONFIG_ARMV8_DEPRECATED) += armv8_deprecated.o
arm64-obj-$(CONFIG_ACPI) += acpi.o arm64-obj-$(CONFIG_ACPI) += acpi.o
arm64-obj-$(CONFIG_ARM64_ACPI_PARKING_PROTOCOL) += acpi_parking_protocol.o
arm64-obj-$(CONFIG_PARAVIRT) += paravirt.o arm64-obj-$(CONFIG_PARAVIRT) += paravirt.o
arm64-obj-$(CONFIG_RANDOMIZE_BASE) += kaslr.o
obj-y += $(arm64-obj-y) vdso/ obj-y += $(arm64-obj-y) vdso/
obj-m += $(arm64-obj-m) obj-m += $(arm64-obj-m)
......
/*
* ARM64 ACPI Parking Protocol implementation
*
* Authors: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>
* Mark Salter <msalter@redhat.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <linux/acpi.h>
#include <linux/types.h>
#include <asm/cpu_ops.h>
struct parking_protocol_mailbox {
__le32 cpu_id;
__le32 reserved;
__le64 entry_point;
};
struct cpu_mailbox_entry {
struct parking_protocol_mailbox __iomem *mailbox;
phys_addr_t mailbox_addr;
u8 version;
u8 gic_cpu_id;
};
static struct cpu_mailbox_entry cpu_mailbox_entries[NR_CPUS];
void __init acpi_set_mailbox_entry(int cpu,
struct acpi_madt_generic_interrupt *p)
{
struct cpu_mailbox_entry *cpu_entry = &cpu_mailbox_entries[cpu];
cpu_entry->mailbox_addr = p->parked_address;
cpu_entry->version = p->parking_version;
cpu_entry->gic_cpu_id = p->cpu_interface_number;
}
bool acpi_parking_protocol_valid(int cpu)
{
struct cpu_mailbox_entry *cpu_entry = &cpu_mailbox_entries[cpu];
return cpu_entry->mailbox_addr && cpu_entry->version;
}
static int acpi_parking_protocol_cpu_init(unsigned int cpu)
{
pr_debug("%s: ACPI parked addr=%llx\n", __func__,
cpu_mailbox_entries[cpu].mailbox_addr);
return 0;
}
static int acpi_parking_protocol_cpu_prepare(unsigned int cpu)
{
return 0;
}
static int acpi_parking_protocol_cpu_boot(unsigned int cpu)
{
struct cpu_mailbox_entry *cpu_entry = &cpu_mailbox_entries[cpu];
struct parking_protocol_mailbox __iomem *mailbox;
__le32 cpu_id;
/*
* Map mailbox memory with attribute device nGnRE (ie ioremap -
* this deviates from the parking protocol specifications since
* the mailboxes are required to be mapped nGnRnE; the attribute
* discrepancy is harmless insofar as the protocol specification
* is concerned).
* If the mailbox is mistakenly allocated in the linear mapping
* by FW ioremap will fail since the mapping will be prevented
* by the kernel (it clashes with the linear mapping attributes
* specifications).
*/
mailbox = ioremap(cpu_entry->mailbox_addr, sizeof(*mailbox));
if (!mailbox)
return -EIO;
cpu_id = readl_relaxed(&mailbox->cpu_id);
/*
* Check if firmware has set-up the mailbox entry properly
* before kickstarting the respective cpu.
*/
if (cpu_id != ~0U) {
iounmap(mailbox);
return -ENXIO;
}
/*
* stash the mailbox address mapping to use it for further FW
* checks in the postboot method
*/
cpu_entry->mailbox = mailbox;
/*
* We write the entry point and cpu id as LE regardless of the
* native endianness of the kernel. Therefore, any boot-loaders
* that read this address need to convert this address to the
* Boot-Loader's endianness before jumping.
*/
writeq_relaxed(__pa(secondary_entry), &mailbox->entry_point);
writel_relaxed(cpu_entry->gic_cpu_id, &mailbox->cpu_id);
arch_send_wakeup_ipi_mask(cpumask_of(cpu));
return 0;
}
static void acpi_parking_protocol_cpu_postboot(void)
{
int cpu = smp_processor_id();
struct cpu_mailbox_entry *cpu_entry = &cpu_mailbox_entries[cpu];
struct parking_protocol_mailbox __iomem *mailbox = cpu_entry->mailbox;
__le64 entry_point;
entry_point = readl_relaxed(&mailbox->entry_point);
/*
* Check if firmware has cleared the entry_point as expected
* by the protocol specification.
*/
WARN_ON(entry_point);
}
const struct cpu_operations acpi_parking_protocol_ops = {
.name = "parking-protocol",
.cpu_init = acpi_parking_protocol_cpu_init,
.cpu_prepare = acpi_parking_protocol_cpu_prepare,
.cpu_boot = acpi_parking_protocol_cpu_boot,
.cpu_postboot = acpi_parking_protocol_cpu_postboot
};
...@@ -297,11 +297,8 @@ static void __init register_insn_emulation_sysctl(struct ctl_table *table) ...@@ -297,11 +297,8 @@ static void __init register_insn_emulation_sysctl(struct ctl_table *table)
"4: mov %w0, %w5\n" \ "4: mov %w0, %w5\n" \
" b 3b\n" \ " b 3b\n" \
" .popsection" \ " .popsection" \
" .pushsection __ex_table,\"a\"\n" \ _ASM_EXTABLE(0b, 4b) \
" .align 3\n" \ _ASM_EXTABLE(1b, 4b) \
" .quad 0b, 4b\n" \
" .quad 1b, 4b\n" \
" .popsection\n" \
ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, \ ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, \
CONFIG_ARM64_PAN) \ CONFIG_ARM64_PAN) \
: "=&r" (res), "+r" (data), "=&r" (temp) \ : "=&r" (res), "+r" (data), "=&r" (temp) \
......
...@@ -104,6 +104,8 @@ int main(void) ...@@ -104,6 +104,8 @@ int main(void)
DEFINE(TZ_MINWEST, offsetof(struct timezone, tz_minuteswest)); DEFINE(TZ_MINWEST, offsetof(struct timezone, tz_minuteswest));
DEFINE(TZ_DSTTIME, offsetof(struct timezone, tz_dsttime)); DEFINE(TZ_DSTTIME, offsetof(struct timezone, tz_dsttime));
BLANK(); BLANK();
DEFINE(CPU_BOOT_STACK, offsetof(struct secondary_data, stack));
BLANK();
#ifdef CONFIG_KVM_ARM_HOST #ifdef CONFIG_KVM_ARM_HOST
DEFINE(VCPU_CONTEXT, offsetof(struct kvm_vcpu, arch.ctxt)); DEFINE(VCPU_CONTEXT, offsetof(struct kvm_vcpu, arch.ctxt));
DEFINE(CPU_GP_REGS, offsetof(struct kvm_cpu_context, gp_regs)); DEFINE(CPU_GP_REGS, offsetof(struct kvm_cpu_context, gp_regs));
......
...@@ -21,24 +21,12 @@ ...@@ -21,24 +21,12 @@
#include <asm/cputype.h> #include <asm/cputype.h>
#include <asm/cpufeature.h> #include <asm/cpufeature.h>
#define MIDR_CORTEX_A53 MIDR_CPU_PART(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A53)
#define MIDR_CORTEX_A57 MIDR_CPU_PART(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A57)
#define MIDR_THUNDERX MIDR_CPU_PART(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX)
#define CPU_MODEL_MASK (MIDR_IMPLEMENTOR_MASK | MIDR_PARTNUM_MASK | \
MIDR_ARCHITECTURE_MASK)
static bool __maybe_unused static bool __maybe_unused
is_affected_midr_range(const struct arm64_cpu_capabilities *entry) is_affected_midr_range(const struct arm64_cpu_capabilities *entry)
{ {
u32 midr = read_cpuid_id(); return MIDR_IS_CPU_MODEL_RANGE(read_cpuid_id(), entry->midr_model,
entry->midr_range_min,
if ((midr & CPU_MODEL_MASK) != entry->midr_model) entry->midr_range_max);
return false;
midr &= MIDR_REVISION_MASK | MIDR_VARIANT_MASK;
return (midr >= entry->midr_range_min && midr <= entry->midr_range_max);
} }
#define MIDR_RANGE(model, min, max) \ #define MIDR_RANGE(model, min, max) \
...@@ -99,6 +87,15 @@ const struct arm64_cpu_capabilities arm64_errata[] = { ...@@ -99,6 +87,15 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
.capability = ARM64_WORKAROUND_CAVIUM_23154, .capability = ARM64_WORKAROUND_CAVIUM_23154,
MIDR_RANGE(MIDR_THUNDERX, 0x00, 0x01), MIDR_RANGE(MIDR_THUNDERX, 0x00, 0x01),
}, },
#endif
#ifdef CONFIG_CAVIUM_ERRATUM_27456
{
/* Cavium ThunderX, T88 pass 1.x - 2.1 */
.desc = "Cavium erratum 27456",
.capability = ARM64_WORKAROUND_CAVIUM_27456,
MIDR_RANGE(MIDR_THUNDERX, 0x00,
(1 << MIDR_VARIANT_SHIFT) | 1),
},
#endif #endif
{ {
} }
......
...@@ -25,19 +25,30 @@ ...@@ -25,19 +25,30 @@
#include <asm/smp_plat.h> #include <asm/smp_plat.h>
extern const struct cpu_operations smp_spin_table_ops; extern const struct cpu_operations smp_spin_table_ops;
extern const struct cpu_operations acpi_parking_protocol_ops;
extern const struct cpu_operations cpu_psci_ops; extern const struct cpu_operations cpu_psci_ops;
const struct cpu_operations *cpu_ops[NR_CPUS]; const struct cpu_operations *cpu_ops[NR_CPUS];
static const struct cpu_operations *supported_cpu_ops[] __initconst = { static const struct cpu_operations *dt_supported_cpu_ops[] __initconst = {
&smp_spin_table_ops, &smp_spin_table_ops,
&cpu_psci_ops, &cpu_psci_ops,
NULL, NULL,
}; };
static const struct cpu_operations *acpi_supported_cpu_ops[] __initconst = {
#ifdef CONFIG_ARM64_ACPI_PARKING_PROTOCOL
&acpi_parking_protocol_ops,
#endif
&cpu_psci_ops,
NULL,
};
static const struct cpu_operations * __init cpu_get_ops(const char *name) static const struct cpu_operations * __init cpu_get_ops(const char *name)
{ {
const struct cpu_operations **ops = supported_cpu_ops; const struct cpu_operations **ops;
ops = acpi_disabled ? dt_supported_cpu_ops : acpi_supported_cpu_ops;
while (*ops) { while (*ops) {
if (!strcmp(name, (*ops)->name)) if (!strcmp(name, (*ops)->name))
...@@ -75,8 +86,16 @@ static const char *__init cpu_read_enable_method(int cpu) ...@@ -75,8 +86,16 @@ static const char *__init cpu_read_enable_method(int cpu)
} }
} else { } else {
enable_method = acpi_get_enable_method(cpu); enable_method = acpi_get_enable_method(cpu);
if (!enable_method) if (!enable_method) {
pr_err("Unsupported ACPI enable-method\n"); /*
* In ACPI systems the boot CPU does not require
* checking the enable method since for some
* boot protocol (ie parking protocol) it need not
* be initialized. Don't warn spuriously.
*/
if (cpu != 0)
pr_err("Unsupported ACPI enable-method\n");
}
} }
return enable_method; return enable_method;
......
This diff is collapsed.
...@@ -59,6 +59,8 @@ static const char *const hwcap_str[] = { ...@@ -59,6 +59,8 @@ static const char *const hwcap_str[] = {
"sha2", "sha2",
"crc32", "crc32",
"atomics", "atomics",
"fphp",
"asimdhp",
NULL NULL
}; };
...@@ -210,6 +212,7 @@ static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info) ...@@ -210,6 +212,7 @@ static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info)
info->reg_id_aa64isar1 = read_cpuid(ID_AA64ISAR1_EL1); info->reg_id_aa64isar1 = read_cpuid(ID_AA64ISAR1_EL1);
info->reg_id_aa64mmfr0 = read_cpuid(ID_AA64MMFR0_EL1); info->reg_id_aa64mmfr0 = read_cpuid(ID_AA64MMFR0_EL1);
info->reg_id_aa64mmfr1 = read_cpuid(ID_AA64MMFR1_EL1); info->reg_id_aa64mmfr1 = read_cpuid(ID_AA64MMFR1_EL1);
info->reg_id_aa64mmfr2 = read_cpuid(ID_AA64MMFR2_EL1);
info->reg_id_aa64pfr0 = read_cpuid(ID_AA64PFR0_EL1); info->reg_id_aa64pfr0 = read_cpuid(ID_AA64PFR0_EL1);
info->reg_id_aa64pfr1 = read_cpuid(ID_AA64PFR1_EL1); info->reg_id_aa64pfr1 = read_cpuid(ID_AA64PFR1_EL1);
......
...@@ -34,7 +34,7 @@ ...@@ -34,7 +34,7 @@
/* Determine debug architecture. */ /* Determine debug architecture. */
u8 debug_monitors_arch(void) u8 debug_monitors_arch(void)
{ {
return cpuid_feature_extract_field(read_system_reg(SYS_ID_AA64DFR0_EL1), return cpuid_feature_extract_unsigned_field(read_system_reg(SYS_ID_AA64DFR0_EL1),
ID_AA64DFR0_DEBUGVER_SHIFT); ID_AA64DFR0_DEBUGVER_SHIFT);
} }
...@@ -186,20 +186,21 @@ static void clear_regs_spsr_ss(struct pt_regs *regs) ...@@ -186,20 +186,21 @@ static void clear_regs_spsr_ss(struct pt_regs *regs)
/* EL1 Single Step Handler hooks */ /* EL1 Single Step Handler hooks */
static LIST_HEAD(step_hook); static LIST_HEAD(step_hook);
static DEFINE_RWLOCK(step_hook_lock); static DEFINE_SPINLOCK(step_hook_lock);
void register_step_hook(struct step_hook *hook) void register_step_hook(struct step_hook *hook)
{ {
write_lock(&step_hook_lock); spin_lock(&step_hook_lock);
list_add(&hook->node, &step_hook); list_add_rcu(&hook->node, &step_hook);
write_unlock(&step_hook_lock); spin_unlock(&step_hook_lock);
} }
void unregister_step_hook(struct step_hook *hook) void unregister_step_hook(struct step_hook *hook)
{ {
write_lock(&step_hook_lock); spin_lock(&step_hook_lock);
list_del(&hook->node); list_del_rcu(&hook->node);
write_unlock(&step_hook_lock); spin_unlock(&step_hook_lock);
synchronize_rcu();
} }
/* /*
...@@ -213,15 +214,15 @@ static int call_step_hook(struct pt_regs *regs, unsigned int esr) ...@@ -213,15 +214,15 @@ static int call_step_hook(struct pt_regs *regs, unsigned int esr)
struct step_hook *hook; struct step_hook *hook;
int retval = DBG_HOOK_ERROR; int retval = DBG_HOOK_ERROR;
read_lock(&step_hook_lock); rcu_read_lock();
list_for_each_entry(hook, &step_hook, node) { list_for_each_entry_rcu(hook, &step_hook, node) {
retval = hook->fn(regs, esr); retval = hook->fn(regs, esr);
if (retval == DBG_HOOK_HANDLED) if (retval == DBG_HOOK_HANDLED)
break; break;
} }
read_unlock(&step_hook_lock); rcu_read_unlock();
return retval; return retval;
} }
......
...@@ -35,6 +35,7 @@ ENTRY(entry) ...@@ -35,6 +35,7 @@ ENTRY(entry)
* for image_addr variable passed to efi_entry(). * for image_addr variable passed to efi_entry().
*/ */
stp x29, x30, [sp, #-32]! stp x29, x30, [sp, #-32]!
mov x29, sp
/* /*
* Call efi_entry to do the real work. * Call efi_entry to do the real work.
...@@ -61,7 +62,7 @@ ENTRY(entry) ...@@ -61,7 +62,7 @@ ENTRY(entry)
*/ */
mov x20, x0 // DTB address mov x20, x0 // DTB address
ldr x0, [sp, #16] // relocated _text address ldr x0, [sp, #16] // relocated _text address
ldr x21, =stext_offset movz x21, #:abs_g0:stext_offset
add x21, x0, x21 add x21, x0, x21
/* /*
......
...@@ -45,7 +45,7 @@ ...@@ -45,7 +45,7 @@
* been used to perform kernel mode NEON in the meantime. * been used to perform kernel mode NEON in the meantime.
* *
* For (a), we add a 'cpu' field to struct fpsimd_state, which gets updated to * For (a), we add a 'cpu' field to struct fpsimd_state, which gets updated to
* the id of the current CPU everytime the state is loaded onto a CPU. For (b), * the id of the current CPU every time the state is loaded onto a CPU. For (b),
* we add the per-cpu variable 'fpsimd_last_state' (below), which contains the * we add the per-cpu variable 'fpsimd_last_state' (below), which contains the
* address of the userland FPSIMD state of the task that was loaded onto the CPU * address of the userland FPSIMD state of the task that was loaded onto the CPU
* the most recently, or NULL if kernel mode NEON has been performed after that. * the most recently, or NULL if kernel mode NEON has been performed after that.
......
...@@ -29,12 +29,14 @@ ...@@ -29,12 +29,14 @@
#include <asm/asm-offsets.h> #include <asm/asm-offsets.h>
#include <asm/cache.h> #include <asm/cache.h>
#include <asm/cputype.h> #include <asm/cputype.h>
#include <asm/elf.h>
#include <asm/kernel-pgtable.h> #include <asm/kernel-pgtable.h>
#include <asm/kvm_arm.h> #include <asm/kvm_arm.h>
#include <asm/memory.h> #include <asm/memory.h>
#include <asm/pgtable-hwdef.h> #include <asm/pgtable-hwdef.h>
#include <asm/pgtable.h> #include <asm/pgtable.h>
#include <asm/page.h> #include <asm/page.h>
#include <asm/smp.h>
#include <asm/sysreg.h> #include <asm/sysreg.h>
#include <asm/thread_info.h> #include <asm/thread_info.h>
#include <asm/virt.h> #include <asm/virt.h>
...@@ -68,12 +70,11 @@ ...@@ -68,12 +70,11 @@
* in the entry routines. * in the entry routines.
*/ */
__HEAD __HEAD
_head:
/* /*
* DO NOT MODIFY. Image header expected by Linux boot-loaders. * DO NOT MODIFY. Image header expected by Linux boot-loaders.
*/ */
#ifdef CONFIG_EFI #ifdef CONFIG_EFI
efi_head:
/* /*
* This add instruction has no meaningful effect except that * This add instruction has no meaningful effect except that
* its opcode forms the magic "MZ" signature required by UEFI. * its opcode forms the magic "MZ" signature required by UEFI.
...@@ -84,9 +85,9 @@ efi_head: ...@@ -84,9 +85,9 @@ efi_head:
b stext // branch to kernel start, magic b stext // branch to kernel start, magic
.long 0 // reserved .long 0 // reserved
#endif #endif
.quad _kernel_offset_le // Image load offset from start of RAM, little-endian le64sym _kernel_offset_le // Image load offset from start of RAM, little-endian
.quad _kernel_size_le // Effective size of kernel image, little-endian le64sym _kernel_size_le // Effective size of kernel image, little-endian
.quad _kernel_flags_le // Informative flags, little-endian le64sym _kernel_flags_le // Informative flags, little-endian
.quad 0 // reserved .quad 0 // reserved
.quad 0 // reserved .quad 0 // reserved
.quad 0 // reserved .quad 0 // reserved
...@@ -95,14 +96,14 @@ efi_head: ...@@ -95,14 +96,14 @@ efi_head:
.byte 0x4d .byte 0x4d
.byte 0x64 .byte 0x64
#ifdef CONFIG_EFI #ifdef CONFIG_EFI
.long pe_header - efi_head // Offset to the PE header. .long pe_header - _head // Offset to the PE header.
#else #else
.word 0 // reserved .word 0 // reserved
#endif #endif
#ifdef CONFIG_EFI #ifdef CONFIG_EFI
.globl __efistub_stext_offset .globl __efistub_stext_offset
.set __efistub_stext_offset, stext - efi_head .set __efistub_stext_offset, stext - _head
.align 3 .align 3
pe_header: pe_header:
.ascii "PE" .ascii "PE"
...@@ -125,7 +126,7 @@ optional_header: ...@@ -125,7 +126,7 @@ optional_header:
.long _end - stext // SizeOfCode .long _end - stext // SizeOfCode
.long 0 // SizeOfInitializedData .long 0 // SizeOfInitializedData
.long 0 // SizeOfUninitializedData .long 0 // SizeOfUninitializedData
.long __efistub_entry - efi_head // AddressOfEntryPoint .long __efistub_entry - _head // AddressOfEntryPoint
.long __efistub_stext_offset // BaseOfCode .long __efistub_stext_offset // BaseOfCode
extra_header_fields: extra_header_fields:
...@@ -140,7 +141,7 @@ extra_header_fields: ...@@ -140,7 +141,7 @@ extra_header_fields:
.short 0 // MinorSubsystemVersion .short 0 // MinorSubsystemVersion
.long 0 // Win32VersionValue .long 0 // Win32VersionValue
.long _end - efi_head // SizeOfImage .long _end - _head // SizeOfImage
// Everything before the kernel image is considered part of the header // Everything before the kernel image is considered part of the header
.long __efistub_stext_offset // SizeOfHeaders .long __efistub_stext_offset // SizeOfHeaders
...@@ -211,6 +212,7 @@ section_table: ...@@ -211,6 +212,7 @@ section_table:
ENTRY(stext) ENTRY(stext)
bl preserve_boot_args bl preserve_boot_args
bl el2_setup // Drop to EL1, w20=cpu_boot_mode bl el2_setup // Drop to EL1, w20=cpu_boot_mode
mov x23, xzr // KASLR offset, defaults to 0
adrp x24, __PHYS_OFFSET adrp x24, __PHYS_OFFSET
bl set_cpu_boot_mode_flag bl set_cpu_boot_mode_flag
bl __create_page_tables // x25=TTBR0, x26=TTBR1 bl __create_page_tables // x25=TTBR0, x26=TTBR1
...@@ -220,11 +222,13 @@ ENTRY(stext) ...@@ -220,11 +222,13 @@ ENTRY(stext)
* On return, the CPU will be ready for the MMU to be turned on and * On return, the CPU will be ready for the MMU to be turned on and
* the TCR will have been set. * the TCR will have been set.
*/ */
ldr x27, =__mmap_switched // address to jump to after ldr x27, 0f // address to jump to after
// MMU has been enabled // MMU has been enabled
adr_l lr, __enable_mmu // return (PIC) address adr_l lr, __enable_mmu // return (PIC) address
b __cpu_setup // initialise processor b __cpu_setup // initialise processor
ENDPROC(stext) ENDPROC(stext)
.align 3
0: .quad __mmap_switched - (_head - TEXT_OFFSET) + KIMAGE_VADDR
/* /*
* Preserve the arguments passed by the bootloader in x0 .. x3 * Preserve the arguments passed by the bootloader in x0 .. x3
...@@ -312,7 +316,7 @@ ENDPROC(preserve_boot_args) ...@@ -312,7 +316,7 @@ ENDPROC(preserve_boot_args)
__create_page_tables: __create_page_tables:
adrp x25, idmap_pg_dir adrp x25, idmap_pg_dir
adrp x26, swapper_pg_dir adrp x26, swapper_pg_dir
mov x27, lr mov x28, lr
/* /*
* Invalidate the idmap and swapper page tables to avoid potential * Invalidate the idmap and swapper page tables to avoid potential
...@@ -390,9 +394,11 @@ __create_page_tables: ...@@ -390,9 +394,11 @@ __create_page_tables:
* Map the kernel image (starting with PHYS_OFFSET). * Map the kernel image (starting with PHYS_OFFSET).
*/ */
mov x0, x26 // swapper_pg_dir mov x0, x26 // swapper_pg_dir
mov x5, #PAGE_OFFSET ldr x5, =KIMAGE_VADDR
add x5, x5, x23 // add KASLR displacement
create_pgd_entry x0, x5, x3, x6 create_pgd_entry x0, x5, x3, x6
ldr x6, =KERNEL_END // __va(KERNEL_END) ldr w6, kernel_img_size
add x6, x6, x5
mov x3, x24 // phys offset mov x3, x24 // phys offset
create_block_map x0, x7, x3, x5, x6 create_block_map x0, x7, x3, x5, x6
...@@ -406,9 +412,11 @@ __create_page_tables: ...@@ -406,9 +412,11 @@ __create_page_tables:
dmb sy dmb sy
bl __inval_cache_range bl __inval_cache_range
mov lr, x27 ret x28
ret
ENDPROC(__create_page_tables) ENDPROC(__create_page_tables)
kernel_img_size:
.long _end - (_head - TEXT_OFFSET)
.ltorg .ltorg
/* /*
...@@ -416,22 +424,80 @@ ENDPROC(__create_page_tables) ...@@ -416,22 +424,80 @@ ENDPROC(__create_page_tables)
*/ */
.set initial_sp, init_thread_union + THREAD_START_SP .set initial_sp, init_thread_union + THREAD_START_SP
__mmap_switched: __mmap_switched:
mov x28, lr // preserve LR
adr_l x8, vectors // load VBAR_EL1 with virtual
msr vbar_el1, x8 // vector table address
isb
// Clear BSS // Clear BSS
adr_l x0, __bss_start adr_l x0, __bss_start
mov x1, xzr mov x1, xzr
adr_l x2, __bss_stop adr_l x2, __bss_stop
sub x2, x2, x0 sub x2, x2, x0
bl __pi_memset bl __pi_memset
dsb ishst // Make zero page visible to PTW
#ifdef CONFIG_RELOCATABLE
/*
* Iterate over each entry in the relocation table, and apply the
* relocations in place.
*/
adr_l x8, __dynsym_start // start of symbol table
adr_l x9, __reloc_start // start of reloc table
adr_l x10, __reloc_end // end of reloc table
0: cmp x9, x10
b.hs 2f
ldp x11, x12, [x9], #24
ldr x13, [x9, #-8]
cmp w12, #R_AARCH64_RELATIVE
b.ne 1f
add x13, x13, x23 // relocate
str x13, [x11, x23]
b 0b
1: cmp w12, #R_AARCH64_ABS64
b.ne 0b
add x12, x12, x12, lsl #1 // symtab offset: 24x top word
add x12, x8, x12, lsr #(32 - 3) // ... shifted into bottom word
ldrsh w14, [x12, #6] // Elf64_Sym::st_shndx
ldr x15, [x12, #8] // Elf64_Sym::st_value
cmp w14, #-0xf // SHN_ABS (0xfff1) ?
add x14, x15, x23 // relocate
csel x15, x14, x15, ne
add x15, x13, x15
str x15, [x11, x23]
b 0b
2: adr_l x8, kimage_vaddr // make relocated kimage_vaddr
dc cvac, x8 // value visible to secondaries
dsb sy // with MMU off
#endif
adr_l sp, initial_sp, x4 adr_l sp, initial_sp, x4
mov x4, sp mov x4, sp
and x4, x4, #~(THREAD_SIZE - 1) and x4, x4, #~(THREAD_SIZE - 1)
msr sp_el0, x4 // Save thread_info msr sp_el0, x4 // Save thread_info
str_l x21, __fdt_pointer, x5 // Save FDT pointer str_l x21, __fdt_pointer, x5 // Save FDT pointer
str_l x24, memstart_addr, x6 // Save PHYS_OFFSET
ldr_l x4, kimage_vaddr // Save the offset between
sub x4, x4, x24 // the kernel virtual and
str_l x4, kimage_voffset, x5 // physical mappings
mov x29, #0 mov x29, #0
#ifdef CONFIG_KASAN #ifdef CONFIG_KASAN
bl kasan_early_init bl kasan_early_init
#endif
#ifdef CONFIG_RANDOMIZE_BASE
cbnz x23, 0f // already running randomized?
mov x0, x21 // pass FDT address in x0
bl kaslr_early_init // parse FDT for KASLR options
cbz x0, 0f // KASLR disabled? just proceed
mov x23, x0 // record KASLR offset
ret x28 // we must enable KASLR, return
// to __enable_mmu()
0:
#endif #endif
b start_kernel b start_kernel
ENDPROC(__mmap_switched) ENDPROC(__mmap_switched)
...@@ -441,6 +507,10 @@ ENDPROC(__mmap_switched) ...@@ -441,6 +507,10 @@ ENDPROC(__mmap_switched)
* hotplug and needs to have the same protections as the text region * hotplug and needs to have the same protections as the text region
*/ */
.section ".text","ax" .section ".text","ax"
ENTRY(kimage_vaddr)
.quad _text - TEXT_OFFSET
/* /*
* If we're fortunate enough to boot at EL2, ensure that the world is * If we're fortunate enough to boot at EL2, ensure that the world is
* sane before dropping to EL1. * sane before dropping to EL1.
...@@ -631,13 +701,20 @@ ENTRY(secondary_startup) ...@@ -631,13 +701,20 @@ ENTRY(secondary_startup)
adrp x26, swapper_pg_dir adrp x26, swapper_pg_dir
bl __cpu_setup // initialise processor bl __cpu_setup // initialise processor
ldr x21, =secondary_data ldr x8, kimage_vaddr
ldr x27, =__secondary_switched // address to jump to after enabling the MMU ldr w9, 0f
sub x27, x8, w9, sxtw // address to jump to after enabling the MMU
b __enable_mmu b __enable_mmu
ENDPROC(secondary_startup) ENDPROC(secondary_startup)
0: .long (_text - TEXT_OFFSET) - __secondary_switched
ENTRY(__secondary_switched) ENTRY(__secondary_switched)
ldr x0, [x21] // get secondary_data.stack adr_l x5, vectors
msr vbar_el1, x5
isb
adr_l x0, secondary_data
ldr x0, [x0, #CPU_BOOT_STACK] // get secondary_data.stack
mov sp, x0 mov sp, x0
and x0, x0, #~(THREAD_SIZE - 1) and x0, x0, #~(THREAD_SIZE - 1)
msr sp_el0, x0 // save thread_info msr sp_el0, x0 // save thread_info
...@@ -645,6 +722,29 @@ ENTRY(__secondary_switched) ...@@ -645,6 +722,29 @@ ENTRY(__secondary_switched)
b secondary_start_kernel b secondary_start_kernel
ENDPROC(__secondary_switched) ENDPROC(__secondary_switched)
/*
* The booting CPU updates the failed status @__early_cpu_boot_status,
* with MMU turned off.
*
* update_early_cpu_boot_status tmp, status
* - Corrupts tmp1, tmp2
* - Writes 'status' to __early_cpu_boot_status and makes sure
* it is committed to memory.
*/
.macro update_early_cpu_boot_status status, tmp1, tmp2
mov \tmp2, #\status
str_l \tmp2, __early_cpu_boot_status, \tmp1
dmb sy
dc ivac, \tmp1 // Invalidate potentially stale cache line
.endm
.pushsection .data..cacheline_aligned
.align L1_CACHE_SHIFT
ENTRY(__early_cpu_boot_status)
.long 0
.popsection
/* /*
* Enable the MMU. * Enable the MMU.
* *
...@@ -658,12 +758,12 @@ ENDPROC(__secondary_switched) ...@@ -658,12 +758,12 @@ ENDPROC(__secondary_switched)
*/ */
.section ".idmap.text", "ax" .section ".idmap.text", "ax"
__enable_mmu: __enable_mmu:
mrs x18, sctlr_el1 // preserve old SCTLR_EL1 value
mrs x1, ID_AA64MMFR0_EL1 mrs x1, ID_AA64MMFR0_EL1
ubfx x2, x1, #ID_AA64MMFR0_TGRAN_SHIFT, 4 ubfx x2, x1, #ID_AA64MMFR0_TGRAN_SHIFT, 4
cmp x2, #ID_AA64MMFR0_TGRAN_SUPPORTED cmp x2, #ID_AA64MMFR0_TGRAN_SUPPORTED
b.ne __no_granule_support b.ne __no_granule_support
ldr x5, =vectors update_early_cpu_boot_status 0, x1, x2
msr vbar_el1, x5
msr ttbr0_el1, x25 // load TTBR0 msr ttbr0_el1, x25 // load TTBR0
msr ttbr1_el1, x26 // load TTBR1 msr ttbr1_el1, x26 // load TTBR1
isb isb
...@@ -677,10 +777,33 @@ __enable_mmu: ...@@ -677,10 +777,33 @@ __enable_mmu:
ic iallu ic iallu
dsb nsh dsb nsh
isb isb
#ifdef CONFIG_RANDOMIZE_BASE
mov x19, x0 // preserve new SCTLR_EL1 value
blr x27
/*
* If we return here, we have a KASLR displacement in x23 which we need
* to take into account by discarding the current kernel mapping and
* creating a new one.
*/
msr sctlr_el1, x18 // disable the MMU
isb
bl __create_page_tables // recreate kernel mapping
msr sctlr_el1, x19 // re-enable the MMU
isb
ic ialluis // flush instructions fetched
isb // via old mapping
add x27, x27, x23 // relocated __mmap_switched
#endif
br x27 br x27
ENDPROC(__enable_mmu) ENDPROC(__enable_mmu)
__no_granule_support: __no_granule_support:
/* Indicate that this CPU can't boot and is stuck in the kernel */
update_early_cpu_boot_status CPU_STUCK_IN_KERNEL, x1, x2
1:
wfe wfe
b __no_granule_support wfi
b 1b
ENDPROC(__no_granule_support) ENDPROC(__no_granule_support)
...@@ -26,31 +26,40 @@ ...@@ -26,31 +26,40 @@
* There aren't any ELF relocations we can use to endian-swap values known only * There aren't any ELF relocations we can use to endian-swap values known only
* at link time (e.g. the subtraction of two symbol addresses), so we must get * at link time (e.g. the subtraction of two symbol addresses), so we must get
* the linker to endian-swap certain values before emitting them. * the linker to endian-swap certain values before emitting them.
*
* Note that, in order for this to work when building the ELF64 PIE executable
* (for KASLR), these values should not be referenced via R_AARCH64_ABS64
* relocations, since these are fixed up at runtime rather than at build time
* when PIE is in effect. So we need to split them up in 32-bit high and low
* words.
*/ */
#ifdef CONFIG_CPU_BIG_ENDIAN #ifdef CONFIG_CPU_BIG_ENDIAN
#define DATA_LE64(data) \ #define DATA_LE32(data) \
((((data) & 0x00000000000000ff) << 56) | \ ((((data) & 0x000000ff) << 24) | \
(((data) & 0x000000000000ff00) << 40) | \ (((data) & 0x0000ff00) << 8) | \
(((data) & 0x0000000000ff0000) << 24) | \ (((data) & 0x00ff0000) >> 8) | \
(((data) & 0x00000000ff000000) << 8) | \ (((data) & 0xff000000) >> 24))
(((data) & 0x000000ff00000000) >> 8) | \
(((data) & 0x0000ff0000000000) >> 24) | \
(((data) & 0x00ff000000000000) >> 40) | \
(((data) & 0xff00000000000000) >> 56))
#else #else
#define DATA_LE64(data) ((data) & 0xffffffffffffffff) #define DATA_LE32(data) ((data) & 0xffffffff)
#endif #endif
#define DEFINE_IMAGE_LE64(sym, data) \
sym##_lo32 = DATA_LE32((data) & 0xffffffff); \
sym##_hi32 = DATA_LE32((data) >> 32)
#ifdef CONFIG_CPU_BIG_ENDIAN #ifdef CONFIG_CPU_BIG_ENDIAN
#define __HEAD_FLAG_BE 1 #define __HEAD_FLAG_BE 1
#else #else
#define __HEAD_FLAG_BE 0 #define __HEAD_FLAG_BE 0
#endif #endif
#define __HEAD_FLAG_PAGE_SIZE ((PAGE_SHIFT - 10) / 2) #define __HEAD_FLAG_PAGE_SIZE ((PAGE_SHIFT - 10) / 2)
#define __HEAD_FLAG_PHYS_BASE 1
#define __HEAD_FLAGS ((__HEAD_FLAG_BE << 0) | \ #define __HEAD_FLAGS ((__HEAD_FLAG_BE << 0) | \
(__HEAD_FLAG_PAGE_SIZE << 1)) (__HEAD_FLAG_PAGE_SIZE << 1) | \
(__HEAD_FLAG_PHYS_BASE << 3))
/* /*
* These will output as part of the Image header, which should be little-endian * These will output as part of the Image header, which should be little-endian
...@@ -58,9 +67,9 @@ ...@@ -58,9 +67,9 @@
* endian swapped in head.S, all are done here for consistency. * endian swapped in head.S, all are done here for consistency.
*/ */
#define HEAD_SYMBOLS \ #define HEAD_SYMBOLS \
_kernel_size_le = DATA_LE64(_end - _text); \ DEFINE_IMAGE_LE64(_kernel_size_le, _end - _text); \
_kernel_offset_le = DATA_LE64(TEXT_OFFSET); \ DEFINE_IMAGE_LE64(_kernel_offset_le, TEXT_OFFSET); \
_kernel_flags_le = DATA_LE64(__HEAD_FLAGS); DEFINE_IMAGE_LE64(_kernel_flags_le, __HEAD_FLAGS);
#ifdef CONFIG_EFI #ifdef CONFIG_EFI
......
/*
* Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#include <linux/crc32.h>
#include <linux/init.h>
#include <linux/libfdt.h>
#include <linux/mm_types.h>
#include <linux/sched.h>
#include <linux/types.h>
#include <asm/fixmap.h>
#include <asm/kernel-pgtable.h>
#include <asm/memory.h>
#include <asm/mmu.h>
#include <asm/pgtable.h>
#include <asm/sections.h>
u64 __read_mostly module_alloc_base;
u16 __initdata memstart_offset_seed;
static __init u64 get_kaslr_seed(void *fdt)
{
int node, len;
u64 *prop;
u64 ret;
node = fdt_path_offset(fdt, "/chosen");
if (node < 0)
return 0;
prop = fdt_getprop_w(fdt, node, "kaslr-seed", &len);
if (!prop || len != sizeof(u64))
return 0;
ret = fdt64_to_cpu(*prop);
*prop = 0;
return ret;
}
static __init const u8 *get_cmdline(void *fdt)
{
static __initconst const u8 default_cmdline[] = CONFIG_CMDLINE;
if (!IS_ENABLED(CONFIG_CMDLINE_FORCE)) {
int node;
const u8 *prop;
node = fdt_path_offset(fdt, "/chosen");
if (node < 0)
goto out;
prop = fdt_getprop(fdt, node, "bootargs", NULL);
if (!prop)
goto out;
return prop;
}
out:
return default_cmdline;
}
extern void *__init __fixmap_remap_fdt(phys_addr_t dt_phys, int *size,
pgprot_t prot);
/*
* This routine will be executed with the kernel mapped at its default virtual
* address, and if it returns successfully, the kernel will be remapped, and
* start_kernel() will be executed from a randomized virtual offset. The
* relocation will result in all absolute references (e.g., static variables
* containing function pointers) to be reinitialized, and zero-initialized
* .bss variables will be reset to 0.
*/
u64 __init kaslr_early_init(u64 dt_phys)
{
void *fdt;
u64 seed, offset, mask, module_range;
const u8 *cmdline, *str;
int size;
/*
* Set a reasonable default for module_alloc_base in case
* we end up running with module randomization disabled.
*/
module_alloc_base = (u64)_etext - MODULES_VSIZE;
/*
* Try to map the FDT early. If this fails, we simply bail,
* and proceed with KASLR disabled. We will make another
* attempt at mapping the FDT in setup_machine()
*/
early_fixmap_init();
fdt = __fixmap_remap_fdt(dt_phys, &size, PAGE_KERNEL);
if (!fdt)
return 0;
/*
* Retrieve (and wipe) the seed from the FDT
*/
seed = get_kaslr_seed(fdt);
if (!seed)
return 0;
/*
* Check if 'nokaslr' appears on the command line, and
* return 0 if that is the case.
*/
cmdline = get_cmdline(fdt);
str = strstr(cmdline, "nokaslr");
if (str == cmdline || (str > cmdline && *(str - 1) == ' '))
return 0;
/*
* OK, so we are proceeding with KASLR enabled. Calculate a suitable
* kernel image offset from the seed. Let's place the kernel in the
* lower half of the VMALLOC area (VA_BITS - 2).
* Even if we could randomize at page granularity for 16k and 64k pages,
* let's always round to 2 MB so we don't interfere with the ability to
* map using contiguous PTEs
*/
mask = ((1UL << (VA_BITS - 2)) - 1) & ~(SZ_2M - 1);
offset = seed & mask;
/* use the top 16 bits to randomize the linear region */
memstart_offset_seed = seed >> 48;
/*
* The kernel Image should not extend across a 1GB/32MB/512MB alignment
* boundary (for 4KB/16KB/64KB granule kernels, respectively). If this
* happens, increase the KASLR offset by the size of the kernel image.
*/
if ((((u64)_text + offset) >> SWAPPER_TABLE_SHIFT) !=
(((u64)_end + offset) >> SWAPPER_TABLE_SHIFT))
offset = (offset + (u64)(_end - _text)) & mask;
if (IS_ENABLED(CONFIG_KASAN))
/*
* KASAN does not expect the module region to intersect the
* vmalloc region, since shadow memory is allocated for each
* module at load time, whereas the vmalloc region is shadowed
* by KASAN zero pages. So keep modules out of the vmalloc
* region if KASAN is enabled.
*/
return offset;
if (IS_ENABLED(CONFIG_RANDOMIZE_MODULE_REGION_FULL)) {
/*
* Randomize the module region independently from the core
* kernel. This prevents modules from leaking any information
* about the address of the kernel itself, but results in
* branches between modules and the core kernel that are
* resolved via PLTs. (Branches between modules will be
* resolved normally.)
*/
module_range = VMALLOC_END - VMALLOC_START - MODULES_VSIZE;
module_alloc_base = VMALLOC_START;
} else {
/*
* Randomize the module region by setting module_alloc_base to
* a PAGE_SIZE multiple in the range [_etext - MODULES_VSIZE,
* _stext) . This guarantees that the resulting region still
* covers [_stext, _etext], and that all relative branches can
* be resolved without veneers.
*/
module_range = MODULES_VSIZE - (u64)(_etext - _stext);
module_alloc_base = (u64)_etext + offset - MODULES_VSIZE;
}
/* use the lower 21 bits to randomize the base of the module region */
module_alloc_base += (module_range * (seed & ((1 << 21) - 1))) >> 21;
module_alloc_base &= PAGE_MASK;
return offset;
}
...@@ -292,8 +292,8 @@ static struct notifier_block kgdb_notifier = { ...@@ -292,8 +292,8 @@ static struct notifier_block kgdb_notifier = {
}; };
/* /*
* kgdb_arch_init - Perform any architecture specific initalization. * kgdb_arch_init - Perform any architecture specific initialization.
* This function will handle the initalization of any architecture * This function will handle the initialization of any architecture
* specific callbacks. * specific callbacks.
*/ */
int kgdb_arch_init(void) int kgdb_arch_init(void)
......
/*
* Copyright (C) 2014-2016 Linaro Ltd. <ard.biesheuvel@linaro.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#include <linux/elf.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/sort.h>
struct plt_entry {
/*
* A program that conforms to the AArch64 Procedure Call Standard
* (AAPCS64) must assume that a veneer that alters IP0 (x16) and/or
* IP1 (x17) may be inserted at any branch instruction that is
* exposed to a relocation that supports long branches. Since that
* is exactly what we are dealing with here, we are free to use x16
* as a scratch register in the PLT veneers.
*/
__le32 mov0; /* movn x16, #0x.... */
__le32 mov1; /* movk x16, #0x...., lsl #16 */
__le32 mov2; /* movk x16, #0x...., lsl #32 */
__le32 br; /* br x16 */
};
u64 module_emit_plt_entry(struct module *mod, const Elf64_Rela *rela,
Elf64_Sym *sym)
{
struct plt_entry *plt = (struct plt_entry *)mod->arch.plt->sh_addr;
int i = mod->arch.plt_num_entries;
u64 val = sym->st_value + rela->r_addend;
/*
* We only emit PLT entries against undefined (SHN_UNDEF) symbols,
* which are listed in the ELF symtab section, but without a type
* or a size.
* So, similar to how the module loader uses the Elf64_Sym::st_value
* field to store the resolved addresses of undefined symbols, let's
* borrow the Elf64_Sym::st_size field (whose value is never used by
* the module loader, even for symbols that are defined) to record
* the address of a symbol's associated PLT entry as we emit it for a
* zero addend relocation (which is the only kind we have to deal with
* in practice). This allows us to find duplicates without having to
* go through the table every time.
*/
if (rela->r_addend == 0 && sym->st_size != 0) {
BUG_ON(sym->st_size < (u64)plt || sym->st_size >= (u64)&plt[i]);
return sym->st_size;
}
mod->arch.plt_num_entries++;
BUG_ON(mod->arch.plt_num_entries > mod->arch.plt_max_entries);
/*
* MOVK/MOVN/MOVZ opcode:
* +--------+------------+--------+-----------+-------------+---------+
* | sf[31] | opc[30:29] | 100101 | hw[22:21] | imm16[20:5] | Rd[4:0] |
* +--------+------------+--------+-----------+-------------+---------+
*
* Rd := 0x10 (x16)
* hw := 0b00 (no shift), 0b01 (lsl #16), 0b10 (lsl #32)
* opc := 0b11 (MOVK), 0b00 (MOVN), 0b10 (MOVZ)
* sf := 1 (64-bit variant)
*/
plt[i] = (struct plt_entry){
cpu_to_le32(0x92800010 | (((~val ) & 0xffff)) << 5),
cpu_to_le32(0xf2a00010 | ((( val >> 16) & 0xffff)) << 5),
cpu_to_le32(0xf2c00010 | ((( val >> 32) & 0xffff)) << 5),
cpu_to_le32(0xd61f0200)
};
if (rela->r_addend == 0)
sym->st_size = (u64)&plt[i];
return (u64)&plt[i];
}
#define cmp_3way(a,b) ((a) < (b) ? -1 : (a) > (b))
static int cmp_rela(const void *a, const void *b)
{
const Elf64_Rela *x = a, *y = b;
int i;
/* sort by type, symbol index and addend */
i = cmp_3way(ELF64_R_TYPE(x->r_info), ELF64_R_TYPE(y->r_info));
if (i == 0)
i = cmp_3way(ELF64_R_SYM(x->r_info), ELF64_R_SYM(y->r_info));
if (i == 0)
i = cmp_3way(x->r_addend, y->r_addend);
return i;
}
static bool duplicate_rel(const Elf64_Rela *rela, int num)
{
/*
* Entries are sorted by type, symbol index and addend. That means
* that, if a duplicate entry exists, it must be in the preceding
* slot.
*/
return num > 0 && cmp_rela(rela + num, rela + num - 1) == 0;
}
static unsigned int count_plts(Elf64_Sym *syms, Elf64_Rela *rela, int num)
{
unsigned int ret = 0;
Elf64_Sym *s;
int i;
for (i = 0; i < num; i++) {
switch (ELF64_R_TYPE(rela[i].r_info)) {
case R_AARCH64_JUMP26:
case R_AARCH64_CALL26:
/*
* We only have to consider branch targets that resolve
* to undefined symbols. This is not simply a heuristic,
* it is a fundamental limitation, since the PLT itself
* is part of the module, and needs to be within 128 MB
* as well, so modules can never grow beyond that limit.
*/
s = syms + ELF64_R_SYM(rela[i].r_info);
if (s->st_shndx != SHN_UNDEF)
break;
/*
* Jump relocations with non-zero addends against
* undefined symbols are supported by the ELF spec, but
* do not occur in practice (e.g., 'jump n bytes past
* the entry point of undefined function symbol f').
* So we need to support them, but there is no need to
* take them into consideration when trying to optimize
* this code. So let's only check for duplicates when
* the addend is zero: this allows us to record the PLT
* entry address in the symbol table itself, rather than
* having to search the list for duplicates each time we
* emit one.
*/
if (rela[i].r_addend != 0 || !duplicate_rel(rela, i))
ret++;
break;
}
}
return ret;
}
int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs,
char *secstrings, struct module *mod)
{
unsigned long plt_max_entries = 0;
Elf64_Sym *syms = NULL;
int i;
/*
* Find the empty .plt section so we can expand it to store the PLT
* entries. Record the symtab address as well.
*/
for (i = 0; i < ehdr->e_shnum; i++) {
if (strcmp(".plt", secstrings + sechdrs[i].sh_name) == 0)
mod->arch.plt = sechdrs + i;
else if (sechdrs[i].sh_type == SHT_SYMTAB)
syms = (Elf64_Sym *)sechdrs[i].sh_addr;
}
if (!mod->arch.plt) {
pr_err("%s: module PLT section missing\n", mod->name);
return -ENOEXEC;
}
if (!syms) {
pr_err("%s: module symtab section missing\n", mod->name);
return -ENOEXEC;
}
for (i = 0; i < ehdr->e_shnum; i++) {
Elf64_Rela *rels = (void *)ehdr + sechdrs[i].sh_offset;
int numrels = sechdrs[i].sh_size / sizeof(Elf64_Rela);
Elf64_Shdr *dstsec = sechdrs + sechdrs[i].sh_info;
if (sechdrs[i].sh_type != SHT_RELA)
continue;
/* ignore relocations that operate on non-exec sections */
if (!(dstsec->sh_flags & SHF_EXECINSTR))
continue;
/* sort by type, symbol index and addend */
sort(rels, numrels, sizeof(Elf64_Rela), cmp_rela, NULL);
plt_max_entries += count_plts(syms, rels, numrels);
}
mod->arch.plt->sh_type = SHT_NOBITS;
mod->arch.plt->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
mod->arch.plt->sh_addralign = L1_CACHE_BYTES;
mod->arch.plt->sh_size = plt_max_entries * sizeof(struct plt_entry);
mod->arch.plt_num_entries = 0;
mod->arch.plt_max_entries = plt_max_entries;
return 0;
}
...@@ -34,10 +34,26 @@ void *module_alloc(unsigned long size) ...@@ -34,10 +34,26 @@ void *module_alloc(unsigned long size)
{ {
void *p; void *p;
p = __vmalloc_node_range(size, MODULE_ALIGN, MODULES_VADDR, MODULES_END, p = __vmalloc_node_range(size, MODULE_ALIGN, module_alloc_base,
module_alloc_base + MODULES_VSIZE,
GFP_KERNEL, PAGE_KERNEL_EXEC, 0, GFP_KERNEL, PAGE_KERNEL_EXEC, 0,
NUMA_NO_NODE, __builtin_return_address(0)); NUMA_NO_NODE, __builtin_return_address(0));
if (!p && IS_ENABLED(CONFIG_ARM64_MODULE_PLTS) &&
!IS_ENABLED(CONFIG_KASAN))
/*
* KASAN can only deal with module allocations being served
* from the reserved module region, since the remainder of
* the vmalloc region is already backed by zero shadow pages,
* and punching holes into it is non-trivial. Since the module
* region is not randomized when KASAN is enabled, it is even
* less likely that the module region gets exhausted, so we
* can simply omit this fallback in that case.
*/
p = __vmalloc_node_range(size, MODULE_ALIGN, VMALLOC_START,
VMALLOC_END, GFP_KERNEL, PAGE_KERNEL_EXEC, 0,
NUMA_NO_NODE, __builtin_return_address(0));
if (p && (kasan_module_alloc(p, size) < 0)) { if (p && (kasan_module_alloc(p, size) < 0)) {
vfree(p); vfree(p);
return NULL; return NULL;
...@@ -361,6 +377,13 @@ int apply_relocate_add(Elf64_Shdr *sechdrs, ...@@ -361,6 +377,13 @@ int apply_relocate_add(Elf64_Shdr *sechdrs,
case R_AARCH64_CALL26: case R_AARCH64_CALL26:
ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 26, ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 26,
AARCH64_INSN_IMM_26); AARCH64_INSN_IMM_26);
if (IS_ENABLED(CONFIG_ARM64_MODULE_PLTS) &&
ovf == -ERANGE) {
val = module_emit_plt_entry(me, &rel[i], sym);
ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2,
26, AARCH64_INSN_IMM_26);
}
break; break;
default: default:
......
SECTIONS {
.plt (NOLOAD) : { BYTE(0) }
}
...@@ -46,6 +46,7 @@ ...@@ -46,6 +46,7 @@
#include <linux/notifier.h> #include <linux/notifier.h>
#include <trace/events/power.h> #include <trace/events/power.h>
#include <asm/alternative.h>
#include <asm/compat.h> #include <asm/compat.h>
#include <asm/cacheflush.h> #include <asm/cacheflush.h>
#include <asm/fpsimd.h> #include <asm/fpsimd.h>
...@@ -280,6 +281,9 @@ int copy_thread(unsigned long clone_flags, unsigned long stack_start, ...@@ -280,6 +281,9 @@ int copy_thread(unsigned long clone_flags, unsigned long stack_start,
} else { } else {
memset(childregs, 0, sizeof(struct pt_regs)); memset(childregs, 0, sizeof(struct pt_regs));
childregs->pstate = PSR_MODE_EL1h; childregs->pstate = PSR_MODE_EL1h;
if (IS_ENABLED(CONFIG_ARM64_UAO) &&
cpus_have_cap(ARM64_HAS_UAO))
childregs->pstate |= PSR_UAO_BIT;
p->thread.cpu_context.x19 = stack_start; p->thread.cpu_context.x19 = stack_start;
p->thread.cpu_context.x20 = stk_sz; p->thread.cpu_context.x20 = stk_sz;
} }
...@@ -308,6 +312,17 @@ static void tls_thread_switch(struct task_struct *next) ...@@ -308,6 +312,17 @@ static void tls_thread_switch(struct task_struct *next)
: : "r" (tpidr), "r" (tpidrro)); : : "r" (tpidr), "r" (tpidrro));
} }
/* Restore the UAO state depending on next's addr_limit */
static void uao_thread_switch(struct task_struct *next)
{
if (IS_ENABLED(CONFIG_ARM64_UAO)) {
if (task_thread_info(next)->addr_limit == KERNEL_DS)
asm(ALTERNATIVE("nop", SET_PSTATE_UAO(1), ARM64_HAS_UAO));
else
asm(ALTERNATIVE("nop", SET_PSTATE_UAO(0), ARM64_HAS_UAO));
}
}
/* /*
* Thread switching. * Thread switching.
*/ */
...@@ -320,6 +335,7 @@ struct task_struct *__switch_to(struct task_struct *prev, ...@@ -320,6 +335,7 @@ struct task_struct *__switch_to(struct task_struct *prev,
tls_thread_switch(next); tls_thread_switch(next);
hw_breakpoint_thread_switch(next); hw_breakpoint_thread_switch(next);
contextidr_thread_switch(next); contextidr_thread_switch(next);
uao_thread_switch(next);
/* /*
* Complete any pending TLB or cache maintenance on this CPU in case * Complete any pending TLB or cache maintenance on this CPU in case
......
...@@ -500,7 +500,7 @@ static int gpr_set(struct task_struct *target, const struct user_regset *regset, ...@@ -500,7 +500,7 @@ static int gpr_set(struct task_struct *target, const struct user_regset *regset,
if (ret) if (ret)
return ret; return ret;
if (!valid_user_regs(&newregs)) if (!valid_user_regs(&newregs, target))
return -EINVAL; return -EINVAL;
task_pt_regs(target)->user_regs = newregs; task_pt_regs(target)->user_regs = newregs;
...@@ -770,7 +770,7 @@ static int compat_gpr_set(struct task_struct *target, ...@@ -770,7 +770,7 @@ static int compat_gpr_set(struct task_struct *target,
} }
if (valid_user_regs(&newregs.user_regs)) if (valid_user_regs(&newregs.user_regs, target))
*task_pt_regs(target) = newregs; *task_pt_regs(target) = newregs;
else else
ret = -EINVAL; ret = -EINVAL;
...@@ -1272,3 +1272,79 @@ asmlinkage void syscall_trace_exit(struct pt_regs *regs) ...@@ -1272,3 +1272,79 @@ asmlinkage void syscall_trace_exit(struct pt_regs *regs)
if (test_thread_flag(TIF_SYSCALL_TRACE)) if (test_thread_flag(TIF_SYSCALL_TRACE))
tracehook_report_syscall(regs, PTRACE_SYSCALL_EXIT); tracehook_report_syscall(regs, PTRACE_SYSCALL_EXIT);
} }
/*
* Bits which are always architecturally RES0 per ARM DDI 0487A.h
* Userspace cannot use these until they have an architectural meaning.
* We also reserve IL for the kernel; SS is handled dynamically.
*/
#define SPSR_EL1_AARCH64_RES0_BITS \
(GENMASK_ULL(63,32) | GENMASK_ULL(27, 22) | GENMASK_ULL(20, 10) | \
GENMASK_ULL(5, 5))
#define SPSR_EL1_AARCH32_RES0_BITS \
(GENMASK_ULL(63,32) | GENMASK_ULL(24, 22) | GENMASK_ULL(20,20))
static int valid_compat_regs(struct user_pt_regs *regs)
{
regs->pstate &= ~SPSR_EL1_AARCH32_RES0_BITS;
if (!system_supports_mixed_endian_el0()) {
if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
regs->pstate |= COMPAT_PSR_E_BIT;
else
regs->pstate &= ~COMPAT_PSR_E_BIT;
}
if (user_mode(regs) && (regs->pstate & PSR_MODE32_BIT) &&
(regs->pstate & COMPAT_PSR_A_BIT) == 0 &&
(regs->pstate & COMPAT_PSR_I_BIT) == 0 &&
(regs->pstate & COMPAT_PSR_F_BIT) == 0) {
return 1;
}
/*
* Force PSR to a valid 32-bit EL0t, preserving the same bits as
* arch/arm.
*/
regs->pstate &= COMPAT_PSR_N_BIT | COMPAT_PSR_Z_BIT |
COMPAT_PSR_C_BIT | COMPAT_PSR_V_BIT |
COMPAT_PSR_Q_BIT | COMPAT_PSR_IT_MASK |
COMPAT_PSR_GE_MASK | COMPAT_PSR_E_BIT |
COMPAT_PSR_T_BIT;
regs->pstate |= PSR_MODE32_BIT;
return 0;
}
static int valid_native_regs(struct user_pt_regs *regs)
{
regs->pstate &= ~SPSR_EL1_AARCH64_RES0_BITS;
if (user_mode(regs) && !(regs->pstate & PSR_MODE32_BIT) &&
(regs->pstate & PSR_D_BIT) == 0 &&
(regs->pstate & PSR_A_BIT) == 0 &&
(regs->pstate & PSR_I_BIT) == 0 &&
(regs->pstate & PSR_F_BIT) == 0) {
return 1;
}
/* Force PSR to a valid 64-bit EL0t */
regs->pstate &= PSR_N_BIT | PSR_Z_BIT | PSR_C_BIT | PSR_V_BIT;
return 0;
}
/*
* Are the current registers suitable for user mode? (used to maintain
* security in signal handlers)
*/
int valid_user_regs(struct user_pt_regs *regs, struct task_struct *task)
{
if (!test_tsk_thread_flag(task, TIF_SINGLESTEP))
regs->pstate &= ~DBG_SPSR_SS;
if (is_compat_thread(task_thread_info(task)))
return valid_compat_regs(regs);
else
return valid_native_regs(regs);
}
...@@ -62,6 +62,7 @@ ...@@ -62,6 +62,7 @@
#include <asm/memblock.h> #include <asm/memblock.h>
#include <asm/efi.h> #include <asm/efi.h>
#include <asm/xen/hypervisor.h> #include <asm/xen/hypervisor.h>
#include <asm/mmu_context.h>
phys_addr_t __fdt_pointer __initdata; phys_addr_t __fdt_pointer __initdata;
...@@ -313,6 +314,12 @@ void __init setup_arch(char **cmdline_p) ...@@ -313,6 +314,12 @@ void __init setup_arch(char **cmdline_p)
*/ */
local_async_enable(); local_async_enable();
/*
* TTBR0 is only used for the identity mapping at this stage. Make it
* point to zero page to avoid speculatively fetching new entries.
*/
cpu_uninstall_idmap();
efi_init(); efi_init();
arm64_memblock_init(); arm64_memblock_init();
...@@ -381,3 +388,32 @@ static int __init topology_init(void) ...@@ -381,3 +388,32 @@ static int __init topology_init(void)
return 0; return 0;
} }
subsys_initcall(topology_init); subsys_initcall(topology_init);
/*
* Dump out kernel offset information on panic.
*/
static int dump_kernel_offset(struct notifier_block *self, unsigned long v,
void *p)
{
u64 const kaslr_offset = kimage_vaddr - KIMAGE_VADDR;
if (IS_ENABLED(CONFIG_RANDOMIZE_BASE) && kaslr_offset > 0) {
pr_emerg("Kernel Offset: 0x%llx from 0x%lx\n",
kaslr_offset, KIMAGE_VADDR);
} else {
pr_emerg("Kernel Offset: disabled\n");
}
return 0;
}
static struct notifier_block kernel_offset_notifier = {
.notifier_call = dump_kernel_offset
};
static int __init register_kernel_offset_dumper(void)
{
atomic_notifier_chain_register(&panic_notifier_list,
&kernel_offset_notifier);
return 0;
}
__initcall(register_kernel_offset_dumper);
...@@ -115,7 +115,7 @@ static int restore_sigframe(struct pt_regs *regs, ...@@ -115,7 +115,7 @@ static int restore_sigframe(struct pt_regs *regs,
*/ */
regs->syscallno = ~0UL; regs->syscallno = ~0UL;
err |= !valid_user_regs(&regs->user_regs); err |= !valid_user_regs(&regs->user_regs, current);
if (err == 0) { if (err == 0) {
struct fpsimd_context *fpsimd_ctx = struct fpsimd_context *fpsimd_ctx =
...@@ -307,7 +307,7 @@ static void handle_signal(struct ksignal *ksig, struct pt_regs *regs) ...@@ -307,7 +307,7 @@ static void handle_signal(struct ksignal *ksig, struct pt_regs *regs)
/* /*
* Check that the resulting registers are actually sane. * Check that the resulting registers are actually sane.
*/ */
ret |= !valid_user_regs(&regs->user_regs); ret |= !valid_user_regs(&regs->user_regs, current);
/* /*
* Fast forward the stepping logic so we step into the signal * Fast forward the stepping logic so we step into the signal
......
...@@ -166,7 +166,7 @@ int copy_siginfo_to_user32(compat_siginfo_t __user *to, const siginfo_t *from) ...@@ -166,7 +166,7 @@ int copy_siginfo_to_user32(compat_siginfo_t __user *to, const siginfo_t *from)
#ifdef BUS_MCEERR_AO #ifdef BUS_MCEERR_AO
/* /*
* Other callers might not initialize the si_lsb field, * Other callers might not initialize the si_lsb field,
* so check explicitely for the right codes here. * so check explicitly for the right codes here.
*/ */
if (from->si_signo == SIGBUS && if (from->si_signo == SIGBUS &&
(from->si_code == BUS_MCEERR_AR || from->si_code == BUS_MCEERR_AO)) (from->si_code == BUS_MCEERR_AR || from->si_code == BUS_MCEERR_AO))
...@@ -356,7 +356,7 @@ static int compat_restore_sigframe(struct pt_regs *regs, ...@@ -356,7 +356,7 @@ static int compat_restore_sigframe(struct pt_regs *regs,
*/ */
regs->syscallno = ~0UL; regs->syscallno = ~0UL;
err |= !valid_user_regs(&regs->user_regs); err |= !valid_user_regs(&regs->user_regs, current);
aux = (struct compat_aux_sigframe __user *) sf->uc.uc_regspace; aux = (struct compat_aux_sigframe __user *) sf->uc.uc_regspace;
if (err == 0) if (err == 0)
......
...@@ -63,6 +63,8 @@ ...@@ -63,6 +63,8 @@
* where to place its SVC stack * where to place its SVC stack
*/ */
struct secondary_data secondary_data; struct secondary_data secondary_data;
/* Number of CPUs which aren't online, but looping in kernel text. */
int cpus_stuck_in_kernel;
enum ipi_msg_type { enum ipi_msg_type {
IPI_RESCHEDULE, IPI_RESCHEDULE,
...@@ -70,8 +72,19 @@ enum ipi_msg_type { ...@@ -70,8 +72,19 @@ enum ipi_msg_type {
IPI_CPU_STOP, IPI_CPU_STOP,
IPI_TIMER, IPI_TIMER,
IPI_IRQ_WORK, IPI_IRQ_WORK,
IPI_WAKEUP
}; };
#ifdef CONFIG_HOTPLUG_CPU
static int op_cpu_kill(unsigned int cpu);
#else
static inline int op_cpu_kill(unsigned int cpu)
{
return -ENOSYS;
}
#endif
/* /*
* Boot a secondary CPU, and assign it the specified idle task. * Boot a secondary CPU, and assign it the specified idle task.
* This also gives us the initial stack to use for this CPU. * This also gives us the initial stack to use for this CPU.
...@@ -89,12 +102,14 @@ static DECLARE_COMPLETION(cpu_running); ...@@ -89,12 +102,14 @@ static DECLARE_COMPLETION(cpu_running);
int __cpu_up(unsigned int cpu, struct task_struct *idle) int __cpu_up(unsigned int cpu, struct task_struct *idle)
{ {
int ret; int ret;
long status;
/* /*
* We need to tell the secondary core where to find its stack and the * We need to tell the secondary core where to find its stack and the
* page tables. * page tables.
*/ */
secondary_data.stack = task_stack_page(idle) + THREAD_START_SP; secondary_data.stack = task_stack_page(idle) + THREAD_START_SP;
update_cpu_boot_status(CPU_MMU_OFF);
__flush_dcache_area(&secondary_data, sizeof(secondary_data)); __flush_dcache_area(&secondary_data, sizeof(secondary_data));
/* /*
...@@ -118,6 +133,32 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle) ...@@ -118,6 +133,32 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle)
} }
secondary_data.stack = NULL; secondary_data.stack = NULL;
status = READ_ONCE(secondary_data.status);
if (ret && status) {
if (status == CPU_MMU_OFF)
status = READ_ONCE(__early_cpu_boot_status);
switch (status) {
default:
pr_err("CPU%u: failed in unknown state : 0x%lx\n",
cpu, status);
break;
case CPU_KILL_ME:
if (!op_cpu_kill(cpu)) {
pr_crit("CPU%u: died during early boot\n", cpu);
break;
}
/* Fall through */
pr_crit("CPU%u: may not have shut down cleanly\n", cpu);
case CPU_STUCK_IN_KERNEL:
pr_crit("CPU%u: is stuck in kernel\n", cpu);
cpus_stuck_in_kernel++;
break;
case CPU_PANIC_KERNEL:
panic("CPU%u detected unsupported configuration\n", cpu);
}
}
return ret; return ret;
} }
...@@ -149,9 +190,7 @@ asmlinkage void secondary_start_kernel(void) ...@@ -149,9 +190,7 @@ asmlinkage void secondary_start_kernel(void)
* TTBR0 is only used for the identity mapping at this stage. Make it * TTBR0 is only used for the identity mapping at this stage. Make it
* point to zero page to avoid speculatively fetching new entries. * point to zero page to avoid speculatively fetching new entries.
*/ */
cpu_set_reserved_ttbr0(); cpu_uninstall_idmap();
local_flush_tlb_all();
cpu_set_default_tcr_t0sz();
preempt_disable(); preempt_disable();
trace_hardirqs_off(); trace_hardirqs_off();
...@@ -185,6 +224,9 @@ asmlinkage void secondary_start_kernel(void) ...@@ -185,6 +224,9 @@ asmlinkage void secondary_start_kernel(void)
*/ */
pr_info("CPU%u: Booted secondary processor [%08x]\n", pr_info("CPU%u: Booted secondary processor [%08x]\n",
cpu, read_cpuid_id()); cpu, read_cpuid_id());
update_cpu_boot_status(CPU_BOOT_SUCCESS);
/* Make sure the status update is visible before we complete */
smp_wmb();
set_cpu_online(cpu, true); set_cpu_online(cpu, true);
complete(&cpu_running); complete(&cpu_running);
...@@ -313,6 +355,30 @@ void cpu_die(void) ...@@ -313,6 +355,30 @@ void cpu_die(void)
} }
#endif #endif
/*
* Kill the calling secondary CPU, early in bringup before it is turned
* online.
*/
void cpu_die_early(void)
{
int cpu = smp_processor_id();
pr_crit("CPU%d: will not boot\n", cpu);
/* Mark this CPU absent */
set_cpu_present(cpu, 0);
#ifdef CONFIG_HOTPLUG_CPU
update_cpu_boot_status(CPU_KILL_ME);
/* Check if we can park ourselves */
if (cpu_ops[cpu] && cpu_ops[cpu]->cpu_die)
cpu_ops[cpu]->cpu_die(cpu);
#endif
update_cpu_boot_status(CPU_STUCK_IN_KERNEL);
cpu_park_loop();
}
static void __init hyp_mode_check(void) static void __init hyp_mode_check(void)
{ {
if (is_hyp_mode_available()) if (is_hyp_mode_available())
...@@ -445,6 +511,17 @@ acpi_map_gic_cpu_interface(struct acpi_madt_generic_interrupt *processor) ...@@ -445,6 +511,17 @@ acpi_map_gic_cpu_interface(struct acpi_madt_generic_interrupt *processor)
/* map the logical cpu id to cpu MPIDR */ /* map the logical cpu id to cpu MPIDR */
cpu_logical_map(cpu_count) = hwid; cpu_logical_map(cpu_count) = hwid;
/*
* Set-up the ACPI parking protocol cpu entries
* while initializing the cpu_logical_map to
* avoid parsing MADT entries multiple times for
* nothing (ie a valid cpu_logical_map entry should
* contain a valid parking protocol data set to
* initialize the cpu if the parking protocol is
* the only available enable method).
*/
acpi_set_mailbox_entry(cpu_count, processor);
cpu_count++; cpu_count++;
} }
...@@ -627,6 +704,7 @@ static const char *ipi_types[NR_IPI] __tracepoint_string = { ...@@ -627,6 +704,7 @@ static const char *ipi_types[NR_IPI] __tracepoint_string = {
S(IPI_CPU_STOP, "CPU stop interrupts"), S(IPI_CPU_STOP, "CPU stop interrupts"),
S(IPI_TIMER, "Timer broadcast interrupts"), S(IPI_TIMER, "Timer broadcast interrupts"),
S(IPI_IRQ_WORK, "IRQ work interrupts"), S(IPI_IRQ_WORK, "IRQ work interrupts"),
S(IPI_WAKEUP, "CPU wake-up interrupts"),
}; };
static void smp_cross_call(const struct cpumask *target, unsigned int ipinr) static void smp_cross_call(const struct cpumask *target, unsigned int ipinr)
...@@ -670,6 +748,13 @@ void arch_send_call_function_single_ipi(int cpu) ...@@ -670,6 +748,13 @@ void arch_send_call_function_single_ipi(int cpu)
smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC); smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC);
} }
#ifdef CONFIG_ARM64_ACPI_PARKING_PROTOCOL
void arch_send_wakeup_ipi_mask(const struct cpumask *mask)
{
smp_cross_call(mask, IPI_WAKEUP);
}
#endif
#ifdef CONFIG_IRQ_WORK #ifdef CONFIG_IRQ_WORK
void arch_irq_work_raise(void) void arch_irq_work_raise(void)
{ {
...@@ -747,6 +832,14 @@ void handle_IPI(int ipinr, struct pt_regs *regs) ...@@ -747,6 +832,14 @@ void handle_IPI(int ipinr, struct pt_regs *regs)
break; break;
#endif #endif
#ifdef CONFIG_ARM64_ACPI_PARKING_PROTOCOL
case IPI_WAKEUP:
WARN_ONCE(!acpi_parking_protocol_valid(cpu),
"CPU%u: Wake-up IPI outside the ACPI parking protocol\n",
cpu);
break;
#endif
default: default:
pr_crit("CPU%u: Unknown IPI message 0x%x\n", cpu, ipinr); pr_crit("CPU%u: Unknown IPI message 0x%x\n", cpu, ipinr);
break; break;
......
...@@ -60,7 +60,6 @@ void __init cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *)) ...@@ -60,7 +60,6 @@ void __init cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *))
*/ */
int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) int cpu_suspend(unsigned long arg, int (*fn)(unsigned long))
{ {
struct mm_struct *mm = current->active_mm;
int ret; int ret;
unsigned long flags; unsigned long flags;
...@@ -87,22 +86,11 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) ...@@ -87,22 +86,11 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long))
ret = __cpu_suspend_enter(arg, fn); ret = __cpu_suspend_enter(arg, fn);
if (ret == 0) { if (ret == 0) {
/* /*
* We are resuming from reset with TTBR0_EL1 set to the * We are resuming from reset with the idmap active in TTBR0_EL1.
* idmap to enable the MMU; set the TTBR0 to the reserved * We must uninstall the idmap and restore the expected MMU
* page tables to prevent speculative TLB allocations, flush * state before we can possibly return to userspace.
* the local tlb and set the default tcr_el1.t0sz so that
* the TTBR0 address space set-up is properly restored.
* If the current active_mm != &init_mm we entered cpu_suspend
* with mappings in TTBR0 that must be restored, so we switch
* them back to complete the address space configuration
* restoration before returning.
*/ */
cpu_set_reserved_ttbr0(); cpu_uninstall_idmap();
local_flush_tlb_all();
cpu_set_default_tcr_t0sz();
if (mm != &init_mm)
cpu_switch_mm(mm->pgd, mm);
/* /*
* Restore per-cpu offset before any kernel * Restore per-cpu offset before any kernel
......
...@@ -21,9 +21,8 @@ ...@@ -21,9 +21,8 @@
#include <linux/const.h> #include <linux/const.h>
#include <asm/page.h> #include <asm/page.h>
__PAGE_ALIGNED_DATA
.globl vdso_start, vdso_end .globl vdso_start, vdso_end
.section .rodata
.balign PAGE_SIZE .balign PAGE_SIZE
vdso_start: vdso_start:
.incbin "arch/arm64/kernel/vdso/vdso.so" .incbin "arch/arm64/kernel/vdso/vdso.so"
......
...@@ -87,15 +87,16 @@ SECTIONS ...@@ -87,15 +87,16 @@ SECTIONS
EXIT_CALL EXIT_CALL
*(.discard) *(.discard)
*(.discard.*) *(.discard.*)
*(.interp .dynamic)
} }
. = PAGE_OFFSET + TEXT_OFFSET; . = KIMAGE_VADDR + TEXT_OFFSET;
.head.text : { .head.text : {
_text = .; _text = .;
HEAD_TEXT HEAD_TEXT
} }
ALIGN_DEBUG_RO ALIGN_DEBUG_RO_MIN(PAGE_SIZE)
.text : { /* Real text segment */ .text : { /* Real text segment */
_stext = .; /* Text and read-only data */ _stext = .; /* Text and read-only data */
__exception_text_start = .; __exception_text_start = .;
...@@ -113,13 +114,13 @@ SECTIONS ...@@ -113,13 +114,13 @@ SECTIONS
*(.got) /* Global offset table */ *(.got) /* Global offset table */
} }
RO_DATA(PAGE_SIZE) ALIGN_DEBUG_RO_MIN(PAGE_SIZE)
EXCEPTION_TABLE(8) RO_DATA(PAGE_SIZE) /* everything from this point to */
EXCEPTION_TABLE(8) /* _etext will be marked RO NX */
NOTES NOTES
ALIGN_DEBUG_RO
_etext = .; /* End of text and rodata section */
ALIGN_DEBUG_RO_MIN(PAGE_SIZE) ALIGN_DEBUG_RO_MIN(PAGE_SIZE)
_etext = .; /* End of text and rodata section */
__init_begin = .; __init_begin = .;
INIT_TEXT_SECTION(8) INIT_TEXT_SECTION(8)
...@@ -150,6 +151,21 @@ SECTIONS ...@@ -150,6 +151,21 @@ SECTIONS
.altinstr_replacement : { .altinstr_replacement : {
*(.altinstr_replacement) *(.altinstr_replacement)
} }
.rela : ALIGN(8) {
__reloc_start = .;
*(.rela .rela*)
__reloc_end = .;
}
.dynsym : ALIGN(8) {
__dynsym_start = .;
*(.dynsym)
}
.dynstr : {
*(.dynstr)
}
.hash : {
*(.hash)
}
. = ALIGN(PAGE_SIZE); . = ALIGN(PAGE_SIZE);
__init_end = .; __init_end = .;
...@@ -187,4 +203,4 @@ ASSERT(__idmap_text_end - (__idmap_text_start & ~(SZ_4K - 1)) <= SZ_4K, ...@@ -187,4 +203,4 @@ ASSERT(__idmap_text_end - (__idmap_text_start & ~(SZ_4K - 1)) <= SZ_4K,
/* /*
* If padding is applied before .head.text, virt<->phys conversions will fail. * If padding is applied before .head.text, virt<->phys conversions will fail.
*/ */
ASSERT(_text == (PAGE_OFFSET + TEXT_OFFSET), "HEAD is misaligned") ASSERT(_text == (KIMAGE_VADDR + TEXT_OFFSET), "HEAD is misaligned")
...@@ -22,7 +22,7 @@ ...@@ -22,7 +22,7 @@
#include <asm/cpufeature.h> #include <asm/cpufeature.h>
/* /*
* u64 kvm_call_hyp(void *hypfn, ...); * u64 __kvm_call_hyp(void *hypfn, ...);
* *
* This is not really a variadic function in the classic C-way and care must * This is not really a variadic function in the classic C-way and care must
* be taken when calling this to ensure parameters are passed in registers * be taken when calling this to ensure parameters are passed in registers
...@@ -39,7 +39,7 @@ ...@@ -39,7 +39,7 @@
* used to implement __hyp_get_vectors in the same way as in * used to implement __hyp_get_vectors in the same way as in
* arch/arm64/kernel/hyp_stub.S. * arch/arm64/kernel/hyp_stub.S.
*/ */
ENTRY(kvm_call_hyp) ENTRY(__kvm_call_hyp)
alternative_if_not ARM64_HAS_VIRT_HOST_EXTN alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
hvc #0 hvc #0
ret ret
...@@ -47,4 +47,4 @@ alternative_else ...@@ -47,4 +47,4 @@ alternative_else
b __vhe_hyp_call b __vhe_hyp_call
nop nop
alternative_endif alternative_endif
ENDPROC(kvm_call_hyp) ENDPROC(__kvm_call_hyp)
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
#include <linux/compiler.h> #include <linux/compiler.h>
#include <linux/kvm_host.h> #include <linux/kvm_host.h>
#include <asm/debug-monitors.h>
#include <asm/kvm_asm.h> #include <asm/kvm_asm.h>
#include <asm/kvm_hyp.h> #include <asm/kvm_hyp.h>
......
...@@ -1116,7 +1116,7 @@ static bool trap_dbgidr(struct kvm_vcpu *vcpu, ...@@ -1116,7 +1116,7 @@ static bool trap_dbgidr(struct kvm_vcpu *vcpu,
} else { } else {
u64 dfr = read_system_reg(SYS_ID_AA64DFR0_EL1); u64 dfr = read_system_reg(SYS_ID_AA64DFR0_EL1);
u64 pfr = read_system_reg(SYS_ID_AA64PFR0_EL1); u64 pfr = read_system_reg(SYS_ID_AA64PFR0_EL1);
u32 el3 = !!cpuid_feature_extract_field(pfr, ID_AA64PFR0_EL3_SHIFT); u32 el3 = !!cpuid_feature_extract_unsigned_field(pfr, ID_AA64PFR0_EL3_SHIFT);
p->regval = ((((dfr >> ID_AA64DFR0_WRPS_SHIFT) & 0xf) << 28) | p->regval = ((((dfr >> ID_AA64DFR0_WRPS_SHIFT) & 0xf) << 28) |
(((dfr >> ID_AA64DFR0_BRPS_SHIFT) & 0xf) << 24) | (((dfr >> ID_AA64DFR0_BRPS_SHIFT) & 0xf) << 24) |
......
...@@ -4,15 +4,16 @@ lib-y := bitops.o clear_user.o delay.o copy_from_user.o \ ...@@ -4,15 +4,16 @@ lib-y := bitops.o clear_user.o delay.o copy_from_user.o \
memcmp.o strcmp.o strncmp.o strlen.o strnlen.o \ memcmp.o strcmp.o strncmp.o strlen.o strnlen.o \
strchr.o strrchr.o strchr.o strrchr.o
# Tell the compiler to treat all general purpose registers as # Tell the compiler to treat all general purpose registers (with the
# callee-saved, which allows for efficient runtime patching of the bl # exception of the IP registers, which are already handled by the caller
# instruction in the caller with an atomic instruction when supported by # in case of a PLT) as callee-saved, which allows for efficient runtime
# the CPU. Result and argument registers are handled correctly, based on # patching of the bl instruction in the caller with an atomic instruction
# the function prototype. # when supported by the CPU. Result and argument registers are handled
# correctly, based on the function prototype.
lib-$(CONFIG_ARM64_LSE_ATOMICS) += atomic_ll_sc.o lib-$(CONFIG_ARM64_LSE_ATOMICS) += atomic_ll_sc.o
CFLAGS_atomic_ll_sc.o := -fcall-used-x0 -ffixed-x1 -ffixed-x2 \ CFLAGS_atomic_ll_sc.o := -fcall-used-x0 -ffixed-x1 -ffixed-x2 \
-ffixed-x3 -ffixed-x4 -ffixed-x5 -ffixed-x6 \ -ffixed-x3 -ffixed-x4 -ffixed-x5 -ffixed-x6 \
-ffixed-x7 -fcall-saved-x8 -fcall-saved-x9 \ -ffixed-x7 -fcall-saved-x8 -fcall-saved-x9 \
-fcall-saved-x10 -fcall-saved-x11 -fcall-saved-x12 \ -fcall-saved-x10 -fcall-saved-x11 -fcall-saved-x12 \
-fcall-saved-x13 -fcall-saved-x14 -fcall-saved-x15 \ -fcall-saved-x13 -fcall-saved-x14 -fcall-saved-x15 \
-fcall-saved-x16 -fcall-saved-x17 -fcall-saved-x18 -fcall-saved-x18
...@@ -33,28 +33,28 @@ ...@@ -33,28 +33,28 @@
* Alignment fixed up by hardware. * Alignment fixed up by hardware.
*/ */
ENTRY(__clear_user) ENTRY(__clear_user)
ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_HAS_PAN, \ ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_ALT_PAN_NOT_UAO, \
CONFIG_ARM64_PAN) CONFIG_ARM64_PAN)
mov x2, x1 // save the size for fixup return mov x2, x1 // save the size for fixup return
subs x1, x1, #8 subs x1, x1, #8
b.mi 2f b.mi 2f
1: 1:
USER(9f, str xzr, [x0], #8 ) uao_user_alternative 9f, str, sttr, xzr, x0, 8
subs x1, x1, #8 subs x1, x1, #8
b.pl 1b b.pl 1b
2: adds x1, x1, #4 2: adds x1, x1, #4
b.mi 3f b.mi 3f
USER(9f, str wzr, [x0], #4 ) uao_user_alternative 9f, str, sttr, wzr, x0, 4
sub x1, x1, #4 sub x1, x1, #4
3: adds x1, x1, #2 3: adds x1, x1, #2
b.mi 4f b.mi 4f
USER(9f, strh wzr, [x0], #2 ) uao_user_alternative 9f, strh, sttrh, wzr, x0, 2
sub x1, x1, #2 sub x1, x1, #2
4: adds x1, x1, #1 4: adds x1, x1, #1
b.mi 5f b.mi 5f
USER(9f, strb wzr, [x0] ) uao_user_alternative 9f, strb, sttrb, wzr, x0, 0
5: mov x0, #0 5: mov x0, #0
ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_HAS_PAN, \ ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_ALT_PAN_NOT_UAO, \
CONFIG_ARM64_PAN) CONFIG_ARM64_PAN)
ret ret
ENDPROC(__clear_user) ENDPROC(__clear_user)
......
...@@ -34,7 +34,7 @@ ...@@ -34,7 +34,7 @@
*/ */
.macro ldrb1 ptr, regB, val .macro ldrb1 ptr, regB, val
USER(9998f, ldrb \ptr, [\regB], \val) uao_user_alternative 9998f, ldrb, ldtrb, \ptr, \regB, \val
.endm .endm
.macro strb1 ptr, regB, val .macro strb1 ptr, regB, val
...@@ -42,7 +42,7 @@ ...@@ -42,7 +42,7 @@
.endm .endm
.macro ldrh1 ptr, regB, val .macro ldrh1 ptr, regB, val
USER(9998f, ldrh \ptr, [\regB], \val) uao_user_alternative 9998f, ldrh, ldtrh, \ptr, \regB, \val
.endm .endm
.macro strh1 ptr, regB, val .macro strh1 ptr, regB, val
...@@ -50,7 +50,7 @@ ...@@ -50,7 +50,7 @@
.endm .endm
.macro ldr1 ptr, regB, val .macro ldr1 ptr, regB, val
USER(9998f, ldr \ptr, [\regB], \val) uao_user_alternative 9998f, ldr, ldtr, \ptr, \regB, \val
.endm .endm
.macro str1 ptr, regB, val .macro str1 ptr, regB, val
...@@ -58,7 +58,7 @@ ...@@ -58,7 +58,7 @@
.endm .endm
.macro ldp1 ptr, regB, regC, val .macro ldp1 ptr, regB, regC, val
USER(9998f, ldp \ptr, \regB, [\regC], \val) uao_ldp 9998f, \ptr, \regB, \regC, \val
.endm .endm
.macro stp1 ptr, regB, regC, val .macro stp1 ptr, regB, regC, val
...@@ -67,11 +67,11 @@ ...@@ -67,11 +67,11 @@
end .req x5 end .req x5
ENTRY(__copy_from_user) ENTRY(__copy_from_user)
ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_HAS_PAN, \ ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_ALT_PAN_NOT_UAO, \
CONFIG_ARM64_PAN) CONFIG_ARM64_PAN)
add end, x0, x2 add end, x0, x2
#include "copy_template.S" #include "copy_template.S"
ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_HAS_PAN, \ ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_ALT_PAN_NOT_UAO, \
CONFIG_ARM64_PAN) CONFIG_ARM64_PAN)
mov x0, #0 // Nothing to copy mov x0, #0 // Nothing to copy
ret ret
......
...@@ -35,44 +35,44 @@ ...@@ -35,44 +35,44 @@
* x0 - bytes not copied * x0 - bytes not copied
*/ */
.macro ldrb1 ptr, regB, val .macro ldrb1 ptr, regB, val
USER(9998f, ldrb \ptr, [\regB], \val) uao_user_alternative 9998f, ldrb, ldtrb, \ptr, \regB, \val
.endm .endm
.macro strb1 ptr, regB, val .macro strb1 ptr, regB, val
USER(9998f, strb \ptr, [\regB], \val) uao_user_alternative 9998f, strb, sttrb, \ptr, \regB, \val
.endm .endm
.macro ldrh1 ptr, regB, val .macro ldrh1 ptr, regB, val
USER(9998f, ldrh \ptr, [\regB], \val) uao_user_alternative 9998f, ldrh, ldtrh, \ptr, \regB, \val
.endm .endm
.macro strh1 ptr, regB, val .macro strh1 ptr, regB, val
USER(9998f, strh \ptr, [\regB], \val) uao_user_alternative 9998f, strh, sttrh, \ptr, \regB, \val
.endm .endm
.macro ldr1 ptr, regB, val .macro ldr1 ptr, regB, val
USER(9998f, ldr \ptr, [\regB], \val) uao_user_alternative 9998f, ldr, ldtr, \ptr, \regB, \val
.endm .endm
.macro str1 ptr, regB, val .macro str1 ptr, regB, val
USER(9998f, str \ptr, [\regB], \val) uao_user_alternative 9998f, str, sttr, \ptr, \regB, \val
.endm .endm
.macro ldp1 ptr, regB, regC, val .macro ldp1 ptr, regB, regC, val
USER(9998f, ldp \ptr, \regB, [\regC], \val) uao_ldp 9998f, \ptr, \regB, \regC, \val
.endm .endm
.macro stp1 ptr, regB, regC, val .macro stp1 ptr, regB, regC, val
USER(9998f, stp \ptr, \regB, [\regC], \val) uao_stp 9998f, \ptr, \regB, \regC, \val
.endm .endm
end .req x5 end .req x5
ENTRY(__copy_in_user) ENTRY(__copy_in_user)
ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_HAS_PAN, \ ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_ALT_PAN_NOT_UAO, \
CONFIG_ARM64_PAN) CONFIG_ARM64_PAN)
add end, x0, x2 add end, x0, x2
#include "copy_template.S" #include "copy_template.S"
ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_HAS_PAN, \ ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_ALT_PAN_NOT_UAO, \
CONFIG_ARM64_PAN) CONFIG_ARM64_PAN)
mov x0, #0 mov x0, #0
ret ret
......
...@@ -18,6 +18,8 @@ ...@@ -18,6 +18,8 @@
#include <linux/const.h> #include <linux/const.h>
#include <asm/assembler.h> #include <asm/assembler.h>
#include <asm/page.h> #include <asm/page.h>
#include <asm/cpufeature.h>
#include <asm/alternative.h>
/* /*
* Copy a page from src to dest (both are page aligned) * Copy a page from src to dest (both are page aligned)
...@@ -27,20 +29,65 @@ ...@@ -27,20 +29,65 @@
* x1 - src * x1 - src
*/ */
ENTRY(copy_page) ENTRY(copy_page)
/* Assume cache line size is 64 bytes. */ alternative_if_not ARM64_HAS_NO_HW_PREFETCH
prfm pldl1strm, [x1, #64] nop
1: ldp x2, x3, [x1] nop
alternative_else
# Prefetch two cache lines ahead.
prfm pldl1strm, [x1, #128]
prfm pldl1strm, [x1, #256]
alternative_endif
ldp x2, x3, [x1]
ldp x4, x5, [x1, #16] ldp x4, x5, [x1, #16]
ldp x6, x7, [x1, #32] ldp x6, x7, [x1, #32]
ldp x8, x9, [x1, #48] ldp x8, x9, [x1, #48]
add x1, x1, #64 ldp x10, x11, [x1, #64]
prfm pldl1strm, [x1, #64] ldp x12, x13, [x1, #80]
ldp x14, x15, [x1, #96]
ldp x16, x17, [x1, #112]
mov x18, #(PAGE_SIZE - 128)
add x1, x1, #128
1:
subs x18, x18, #128
alternative_if_not ARM64_HAS_NO_HW_PREFETCH
nop
alternative_else
prfm pldl1strm, [x1, #384]
alternative_endif
stnp x2, x3, [x0] stnp x2, x3, [x0]
ldp x2, x3, [x1]
stnp x4, x5, [x0, #16] stnp x4, x5, [x0, #16]
ldp x4, x5, [x1, #16]
stnp x6, x7, [x0, #32] stnp x6, x7, [x0, #32]
ldp x6, x7, [x1, #32]
stnp x8, x9, [x0, #48] stnp x8, x9, [x0, #48]
add x0, x0, #64 ldp x8, x9, [x1, #48]
tst x1, #(PAGE_SIZE - 1) stnp x10, x11, [x0, #64]
b.ne 1b ldp x10, x11, [x1, #64]
stnp x12, x13, [x0, #80]
ldp x12, x13, [x1, #80]
stnp x14, x15, [x0, #96]
ldp x14, x15, [x1, #96]
stnp x16, x17, [x0, #112]
ldp x16, x17, [x1, #112]
add x0, x0, #128
add x1, x1, #128
b.gt 1b
stnp x2, x3, [x0]
stnp x4, x5, [x0, #16]
stnp x6, x7, [x0, #32]
stnp x8, x9, [x0, #48]
stnp x10, x11, [x0, #64]
stnp x12, x13, [x0, #80]
stnp x14, x15, [x0, #96]
stnp x16, x17, [x0, #112]
ret ret
ENDPROC(copy_page) ENDPROC(copy_page)
...@@ -37,7 +37,7 @@ ...@@ -37,7 +37,7 @@
.endm .endm
.macro strb1 ptr, regB, val .macro strb1 ptr, regB, val
USER(9998f, strb \ptr, [\regB], \val) uao_user_alternative 9998f, strb, sttrb, \ptr, \regB, \val
.endm .endm
.macro ldrh1 ptr, regB, val .macro ldrh1 ptr, regB, val
...@@ -45,7 +45,7 @@ ...@@ -45,7 +45,7 @@
.endm .endm
.macro strh1 ptr, regB, val .macro strh1 ptr, regB, val
USER(9998f, strh \ptr, [\regB], \val) uao_user_alternative 9998f, strh, sttrh, \ptr, \regB, \val
.endm .endm
.macro ldr1 ptr, regB, val .macro ldr1 ptr, regB, val
...@@ -53,7 +53,7 @@ ...@@ -53,7 +53,7 @@
.endm .endm
.macro str1 ptr, regB, val .macro str1 ptr, regB, val
USER(9998f, str \ptr, [\regB], \val) uao_user_alternative 9998f, str, sttr, \ptr, \regB, \val
.endm .endm
.macro ldp1 ptr, regB, regC, val .macro ldp1 ptr, regB, regC, val
...@@ -61,16 +61,16 @@ ...@@ -61,16 +61,16 @@
.endm .endm
.macro stp1 ptr, regB, regC, val .macro stp1 ptr, regB, regC, val
USER(9998f, stp \ptr, \regB, [\regC], \val) uao_stp 9998f, \ptr, \regB, \regC, \val
.endm .endm
end .req x5 end .req x5
ENTRY(__copy_to_user) ENTRY(__copy_to_user)
ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_HAS_PAN, \ ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_ALT_PAN_NOT_UAO, \
CONFIG_ARM64_PAN) CONFIG_ARM64_PAN)
add end, x0, x2 add end, x0, x2
#include "copy_template.S" #include "copy_template.S"
ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_HAS_PAN, \ ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_ALT_PAN_NOT_UAO, \
CONFIG_ARM64_PAN) CONFIG_ARM64_PAN)
mov x0, #0 mov x0, #0
ret ret
......
...@@ -211,7 +211,7 @@ CPU_LE( lsr tmp2, tmp2, tmp1 ) ...@@ -211,7 +211,7 @@ CPU_LE( lsr tmp2, tmp2, tmp1 )
.Lunequal_proc: .Lunequal_proc:
cbz diff, .Lremain8 cbz diff, .Lremain8
/*There is differnence occured in the latest comparison.*/ /* There is difference occurred in the latest comparison. */
.Lnot_limit: .Lnot_limit:
/* /*
* For little endian,reverse the low significant equal bits into MSB,then * For little endian,reverse the low significant equal bits into MSB,then
......
...@@ -24,6 +24,7 @@ ...@@ -24,6 +24,7 @@
#include <asm/cpufeature.h> #include <asm/cpufeature.h>
#include <asm/mmu_context.h> #include <asm/mmu_context.h>
#include <asm/smp.h>
#include <asm/tlbflush.h> #include <asm/tlbflush.h>
static u32 asid_bits; static u32 asid_bits;
...@@ -40,6 +41,45 @@ static cpumask_t tlb_flush_pending; ...@@ -40,6 +41,45 @@ static cpumask_t tlb_flush_pending;
#define ASID_FIRST_VERSION (1UL << asid_bits) #define ASID_FIRST_VERSION (1UL << asid_bits)
#define NUM_USER_ASIDS ASID_FIRST_VERSION #define NUM_USER_ASIDS ASID_FIRST_VERSION
/* Get the ASIDBits supported by the current CPU */
static u32 get_cpu_asid_bits(void)
{
u32 asid;
int fld = cpuid_feature_extract_unsigned_field(read_cpuid(ID_AA64MMFR0_EL1),
ID_AA64MMFR0_ASID_SHIFT);
switch (fld) {
default:
pr_warn("CPU%d: Unknown ASID size (%d); assuming 8-bit\n",
smp_processor_id(), fld);
/* Fallthrough */
case 0:
asid = 8;
break;
case 2:
asid = 16;
}
return asid;
}
/* Check if the current cpu's ASIDBits is compatible with asid_bits */
void verify_cpu_asid_bits(void)
{
u32 asid = get_cpu_asid_bits();
if (asid < asid_bits) {
/*
* We cannot decrease the ASID size at runtime, so panic if we support
* fewer ASID bits than the boot CPU.
*/
pr_crit("CPU%d: smaller ASID size(%u) than boot CPU (%u)\n",
smp_processor_id(), asid, asid_bits);
update_cpu_boot_status(CPU_PANIC_KERNEL);
cpu_park_loop();
}
}
static void flush_context(unsigned int cpu) static void flush_context(unsigned int cpu)
{ {
int i; int i;
...@@ -187,19 +227,7 @@ void check_and_switch_context(struct mm_struct *mm, unsigned int cpu) ...@@ -187,19 +227,7 @@ void check_and_switch_context(struct mm_struct *mm, unsigned int cpu)
static int asids_init(void) static int asids_init(void)
{ {
int fld = cpuid_feature_extract_field(read_cpuid(ID_AA64MMFR0_EL1), 4); asid_bits = get_cpu_asid_bits();
switch (fld) {
default:
pr_warn("Unknown ASID size (%d); assuming 8-bit\n", fld);
/* Fallthrough */
case 0:
asid_bits = 8;
break;
case 2:
asid_bits = 16;
}
/* If we end up with more CPUs than ASIDs, expect things to crash */ /* If we end up with more CPUs than ASIDs, expect things to crash */
WARN_ON(NUM_USER_ASIDS < num_possible_cpus()); WARN_ON(NUM_USER_ASIDS < num_possible_cpus());
atomic64_set(&asid_generation, ASID_FIRST_VERSION); atomic64_set(&asid_generation, ASID_FIRST_VERSION);
......
...@@ -27,15 +27,15 @@ ...@@ -27,15 +27,15 @@
#include <asm/pgtable.h> #include <asm/pgtable.h>
#include <asm/pgtable-hwdef.h> #include <asm/pgtable-hwdef.h>
#define LOWEST_ADDR (UL(0xffffffffffffffff) << VA_BITS)
struct addr_marker { struct addr_marker {
unsigned long start_address; unsigned long start_address;
const char *name; const char *name;
}; };
enum address_markers_idx { enum address_markers_idx {
VMALLOC_START_NR = 0, MODULES_START_NR = 0,
MODULES_END_NR,
VMALLOC_START_NR,
VMALLOC_END_NR, VMALLOC_END_NR,
#ifdef CONFIG_SPARSEMEM_VMEMMAP #ifdef CONFIG_SPARSEMEM_VMEMMAP
VMEMMAP_START_NR, VMEMMAP_START_NR,
...@@ -45,12 +45,12 @@ enum address_markers_idx { ...@@ -45,12 +45,12 @@ enum address_markers_idx {
FIXADDR_END_NR, FIXADDR_END_NR,
PCI_START_NR, PCI_START_NR,
PCI_END_NR, PCI_END_NR,
MODULES_START_NR,
MODULES_END_NR,
KERNEL_SPACE_NR, KERNEL_SPACE_NR,
}; };
static struct addr_marker address_markers[] = { static struct addr_marker address_markers[] = {
{ MODULES_VADDR, "Modules start" },
{ MODULES_END, "Modules end" },
{ VMALLOC_START, "vmalloc() Area" }, { VMALLOC_START, "vmalloc() Area" },
{ VMALLOC_END, "vmalloc() End" }, { VMALLOC_END, "vmalloc() End" },
#ifdef CONFIG_SPARSEMEM_VMEMMAP #ifdef CONFIG_SPARSEMEM_VMEMMAP
...@@ -61,9 +61,7 @@ static struct addr_marker address_markers[] = { ...@@ -61,9 +61,7 @@ static struct addr_marker address_markers[] = {
{ FIXADDR_TOP, "Fixmap end" }, { FIXADDR_TOP, "Fixmap end" },
{ PCI_IO_START, "PCI I/O start" }, { PCI_IO_START, "PCI I/O start" },
{ PCI_IO_END, "PCI I/O end" }, { PCI_IO_END, "PCI I/O end" },
{ MODULES_VADDR, "Modules start" }, { PAGE_OFFSET, "Linear Mapping" },
{ MODULES_END, "Modules end" },
{ PAGE_OFFSET, "Kernel Mapping" },
{ -1, NULL }, { -1, NULL },
}; };
...@@ -90,6 +88,11 @@ struct prot_bits { ...@@ -90,6 +88,11 @@ struct prot_bits {
static const struct prot_bits pte_bits[] = { static const struct prot_bits pte_bits[] = {
{ {
.mask = PTE_VALID,
.val = PTE_VALID,
.set = " ",
.clear = "F",
}, {
.mask = PTE_USER, .mask = PTE_USER,
.val = PTE_USER, .val = PTE_USER,
.set = "USR", .set = "USR",
...@@ -316,7 +319,7 @@ static int ptdump_show(struct seq_file *m, void *v) ...@@ -316,7 +319,7 @@ static int ptdump_show(struct seq_file *m, void *v)
.marker = address_markers, .marker = address_markers,
}; };
walk_pgd(&st, &init_mm, LOWEST_ADDR); walk_pgd(&st, &init_mm, VA_START);
note_page(&st, 0, 0, 0); note_page(&st, 0, 0, 0);
return 0; return 0;
......
...@@ -11,7 +11,7 @@ int fixup_exception(struct pt_regs *regs) ...@@ -11,7 +11,7 @@ int fixup_exception(struct pt_regs *regs)
fixup = search_exception_tables(instruction_pointer(regs)); fixup = search_exception_tables(instruction_pointer(regs));
if (fixup) if (fixup)
regs->pc = fixup->fixup; regs->pc = (unsigned long)&fixup->fixup + fixup->fixup;
return fixup != NULL; return fixup != NULL;
} }
...@@ -192,6 +192,14 @@ static int __do_page_fault(struct mm_struct *mm, unsigned long addr, ...@@ -192,6 +192,14 @@ static int __do_page_fault(struct mm_struct *mm, unsigned long addr,
return fault; return fault;
} }
static inline int permission_fault(unsigned int esr)
{
unsigned int ec = (esr & ESR_ELx_EC_MASK) >> ESR_ELx_EC_SHIFT;
unsigned int fsc_type = esr & ESR_ELx_FSC_TYPE;
return (ec == ESR_ELx_EC_DABT_CUR && fsc_type == ESR_ELx_FSC_PERM);
}
static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
struct pt_regs *regs) struct pt_regs *regs)
{ {
...@@ -225,12 +233,13 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, ...@@ -225,12 +233,13 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
mm_flags |= FAULT_FLAG_WRITE; mm_flags |= FAULT_FLAG_WRITE;
} }
/* if (permission_fault(esr) && (addr < USER_DS)) {
* PAN bit set implies the fault happened in kernel space, but not if (get_fs() == KERNEL_DS)
* in the arch's user access functions. die("Accessing user space memory with fs=KERNEL_DS", regs, esr);
*/
if (IS_ENABLED(CONFIG_ARM64_PAN) && (regs->pstate & PSR_PAN_BIT)) if (!search_exception_tables(regs->pc))
goto no_context; die("Accessing user space memory outside uaccess.h routines", regs, esr);
}
/* /*
* As per x86, we may deadlock here. However, since the kernel only * As per x86, we may deadlock here. However, since the kernel only
...@@ -568,3 +577,16 @@ void cpu_enable_pan(void *__unused) ...@@ -568,3 +577,16 @@ void cpu_enable_pan(void *__unused)
config_sctlr_el1(SCTLR_EL1_SPAN, 0); config_sctlr_el1(SCTLR_EL1_SPAN, 0);
} }
#endif /* CONFIG_ARM64_PAN */ #endif /* CONFIG_ARM64_PAN */
#ifdef CONFIG_ARM64_UAO
/*
* Kernel threads have fs=KERNEL_DS by default, and don't need to call
* set_fs(), devtmpfs in particular relies on this behaviour.
* We need to enable the feature at runtime (instead of adding it to
* PSR_MODE_EL1h) as the feature may not be implemented by the cpu.
*/
void cpu_enable_uao(void *__unused)
{
asm(SET_PSTATE_UAO(1));
}
#endif /* CONFIG_ARM64_UAO */
...@@ -35,7 +35,10 @@ ...@@ -35,7 +35,10 @@
#include <linux/efi.h> #include <linux/efi.h>
#include <linux/swiotlb.h> #include <linux/swiotlb.h>
#include <asm/boot.h>
#include <asm/fixmap.h> #include <asm/fixmap.h>
#include <asm/kasan.h>
#include <asm/kernel-pgtable.h>
#include <asm/memory.h> #include <asm/memory.h>
#include <asm/sections.h> #include <asm/sections.h>
#include <asm/setup.h> #include <asm/setup.h>
...@@ -45,7 +48,13 @@ ...@@ -45,7 +48,13 @@
#include "mm.h" #include "mm.h"
phys_addr_t memstart_addr __read_mostly = 0; /*
* We need to be able to catch inadvertent references to memstart_addr
* that occur (potentially in generic code) before arm64_memblock_init()
* executes, which assigns it its actual value. So use a default value
* that cannot be mistaken for a real physical address.
*/
s64 memstart_addr __read_mostly = -1;
phys_addr_t arm64_dma_phys_limit __read_mostly; phys_addr_t arm64_dma_phys_limit __read_mostly;
#ifdef CONFIG_BLK_DEV_INITRD #ifdef CONFIG_BLK_DEV_INITRD
...@@ -58,8 +67,8 @@ static int __init early_initrd(char *p) ...@@ -58,8 +67,8 @@ static int __init early_initrd(char *p)
if (*endp == ',') { if (*endp == ',') {
size = memparse(endp + 1, NULL); size = memparse(endp + 1, NULL);
initrd_start = (unsigned long)__va(start); initrd_start = start;
initrd_end = (unsigned long)__va(start + size); initrd_end = start + size;
} }
return 0; return 0;
} }
...@@ -159,7 +168,57 @@ early_param("mem", early_mem); ...@@ -159,7 +168,57 @@ early_param("mem", early_mem);
void __init arm64_memblock_init(void) void __init arm64_memblock_init(void)
{ {
memblock_enforce_memory_limit(memory_limit); const s64 linear_region_size = -(s64)PAGE_OFFSET;
/*
* Ensure that the linear region takes up exactly half of the kernel
* virtual address space. This way, we can distinguish a linear address
* from a kernel/module/vmalloc address by testing a single bit.
*/
BUILD_BUG_ON(linear_region_size != BIT(VA_BITS - 1));
/*
* Select a suitable value for the base of physical memory.
*/
memstart_addr = round_down(memblock_start_of_DRAM(),
ARM64_MEMSTART_ALIGN);
/*
* Remove the memory that we will not be able to cover with the
* linear mapping. Take care not to clip the kernel which may be
* high in memory.
*/
memblock_remove(max_t(u64, memstart_addr + linear_region_size, __pa(_end)),
ULLONG_MAX);
if (memblock_end_of_DRAM() > linear_region_size)
memblock_remove(0, memblock_end_of_DRAM() - linear_region_size);
/*
* Apply the memory limit if it was set. Since the kernel may be loaded
* high up in memory, add back the kernel region that must be accessible
* via the linear mapping.
*/
if (memory_limit != (phys_addr_t)ULLONG_MAX) {
memblock_enforce_memory_limit(memory_limit);
memblock_add(__pa(_text), (u64)(_end - _text));
}
if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) {
extern u16 memstart_offset_seed;
u64 range = linear_region_size -
(memblock_end_of_DRAM() - memblock_start_of_DRAM());
/*
* If the size of the linear region exceeds, by a sufficient
* margin, the size of the region that the available physical
* memory spans, randomize the linear region as well.
*/
if (memstart_offset_seed > 0 && range >= ARM64_MEMSTART_ALIGN) {
range = range / ARM64_MEMSTART_ALIGN + 1;
memstart_addr -= ARM64_MEMSTART_ALIGN *
((range * memstart_offset_seed) >> 16);
}
}
/* /*
* Register the kernel text, kernel data, initrd, and initial * Register the kernel text, kernel data, initrd, and initial
...@@ -167,8 +226,13 @@ void __init arm64_memblock_init(void) ...@@ -167,8 +226,13 @@ void __init arm64_memblock_init(void)
*/ */
memblock_reserve(__pa(_text), _end - _text); memblock_reserve(__pa(_text), _end - _text);
#ifdef CONFIG_BLK_DEV_INITRD #ifdef CONFIG_BLK_DEV_INITRD
if (initrd_start) if (initrd_start) {
memblock_reserve(__virt_to_phys(initrd_start), initrd_end - initrd_start); memblock_reserve(initrd_start, initrd_end - initrd_start);
/* the generic initrd code expects virtual addresses */
initrd_start = __phys_to_virt(initrd_start);
initrd_end = __phys_to_virt(initrd_end);
}
#endif #endif
early_init_fdt_scan_reserved_mem(); early_init_fdt_scan_reserved_mem();
...@@ -302,35 +366,38 @@ void __init mem_init(void) ...@@ -302,35 +366,38 @@ void __init mem_init(void)
#ifdef CONFIG_KASAN #ifdef CONFIG_KASAN
" kasan : 0x%16lx - 0x%16lx (%6ld GB)\n" " kasan : 0x%16lx - 0x%16lx (%6ld GB)\n"
#endif #endif
" modules : 0x%16lx - 0x%16lx (%6ld MB)\n"
" vmalloc : 0x%16lx - 0x%16lx (%6ld GB)\n" " vmalloc : 0x%16lx - 0x%16lx (%6ld GB)\n"
" .text : 0x%p" " - 0x%p" " (%6ld KB)\n"
" .rodata : 0x%p" " - 0x%p" " (%6ld KB)\n"
" .init : 0x%p" " - 0x%p" " (%6ld KB)\n"
" .data : 0x%p" " - 0x%p" " (%6ld KB)\n"
#ifdef CONFIG_SPARSEMEM_VMEMMAP #ifdef CONFIG_SPARSEMEM_VMEMMAP
" vmemmap : 0x%16lx - 0x%16lx (%6ld GB maximum)\n" " vmemmap : 0x%16lx - 0x%16lx (%6ld GB maximum)\n"
" 0x%16lx - 0x%16lx (%6ld MB actual)\n" " 0x%16lx - 0x%16lx (%6ld MB actual)\n"
#endif #endif
" fixed : 0x%16lx - 0x%16lx (%6ld KB)\n" " fixed : 0x%16lx - 0x%16lx (%6ld KB)\n"
" PCI I/O : 0x%16lx - 0x%16lx (%6ld MB)\n" " PCI I/O : 0x%16lx - 0x%16lx (%6ld MB)\n"
" modules : 0x%16lx - 0x%16lx (%6ld MB)\n" " memory : 0x%16lx - 0x%16lx (%6ld MB)\n",
" memory : 0x%16lx - 0x%16lx (%6ld MB)\n"
" .init : 0x%p" " - 0x%p" " (%6ld KB)\n"
" .text : 0x%p" " - 0x%p" " (%6ld KB)\n"
" .data : 0x%p" " - 0x%p" " (%6ld KB)\n",
#ifdef CONFIG_KASAN #ifdef CONFIG_KASAN
MLG(KASAN_SHADOW_START, KASAN_SHADOW_END), MLG(KASAN_SHADOW_START, KASAN_SHADOW_END),
#endif #endif
MLM(MODULES_VADDR, MODULES_END),
MLG(VMALLOC_START, VMALLOC_END), MLG(VMALLOC_START, VMALLOC_END),
MLK_ROUNDUP(_text, __start_rodata),
MLK_ROUNDUP(__start_rodata, _etext),
MLK_ROUNDUP(__init_begin, __init_end),
MLK_ROUNDUP(_sdata, _edata),
#ifdef CONFIG_SPARSEMEM_VMEMMAP #ifdef CONFIG_SPARSEMEM_VMEMMAP
MLG(VMEMMAP_START, MLG(VMEMMAP_START,
VMEMMAP_START + VMEMMAP_SIZE), VMEMMAP_START + VMEMMAP_SIZE),
MLM((unsigned long)virt_to_page(PAGE_OFFSET), MLM((unsigned long)phys_to_page(memblock_start_of_DRAM()),
(unsigned long)virt_to_page(high_memory)), (unsigned long)virt_to_page(high_memory)),
#endif #endif
MLK(FIXADDR_START, FIXADDR_TOP), MLK(FIXADDR_START, FIXADDR_TOP),
MLM(PCI_IO_START, PCI_IO_END), MLM(PCI_IO_START, PCI_IO_END),
MLM(MODULES_VADDR, MODULES_END), MLM(__phys_to_virt(memblock_start_of_DRAM()),
MLM(PAGE_OFFSET, (unsigned long)high_memory), (unsigned long)high_memory));
MLK_ROUNDUP(__init_begin, __init_end),
MLK_ROUNDUP(_text, _etext),
MLK_ROUNDUP(_sdata, _edata));
#undef MLK #undef MLK
#undef MLM #undef MLM
...@@ -343,8 +410,6 @@ void __init mem_init(void) ...@@ -343,8 +410,6 @@ void __init mem_init(void)
#ifdef CONFIG_COMPAT #ifdef CONFIG_COMPAT
BUILD_BUG_ON(TASK_SIZE_32 > TASK_SIZE_64); BUILD_BUG_ON(TASK_SIZE_32 > TASK_SIZE_64);
#endif #endif
BUILD_BUG_ON(TASK_SIZE_64 > MODULES_VADDR);
BUG_ON(TASK_SIZE_64 > MODULES_VADDR);
if (PAGE_SIZE >= 16384 && get_num_physpages() <= 128) { if (PAGE_SIZE >= 16384 && get_num_physpages() <= 128) {
extern int sysctl_overcommit_memory; extern int sysctl_overcommit_memory;
...@@ -358,8 +423,8 @@ void __init mem_init(void) ...@@ -358,8 +423,8 @@ void __init mem_init(void)
void free_initmem(void) void free_initmem(void)
{ {
fixup_init();
free_initmem_default(0); free_initmem_default(0);
fixup_init();
} }
#ifdef CONFIG_BLK_DEV_INITRD #ifdef CONFIG_BLK_DEV_INITRD
...@@ -380,3 +445,28 @@ static int __init keepinitrd_setup(char *__unused) ...@@ -380,3 +445,28 @@ static int __init keepinitrd_setup(char *__unused)
__setup("keepinitrd", keepinitrd_setup); __setup("keepinitrd", keepinitrd_setup);
#endif #endif
/*
* Dump out memory limit information on panic.
*/
static int dump_mem_limit(struct notifier_block *self, unsigned long v, void *p)
{
if (memory_limit != (phys_addr_t)ULLONG_MAX) {
pr_emerg("Memory Limit: %llu MB\n", memory_limit >> 20);
} else {
pr_emerg("Memory Limit: none\n");
}
return 0;
}
static struct notifier_block mem_limit_notifier = {
.notifier_call = dump_mem_limit,
};
static int __init register_mem_limit_dumper(void)
{
atomic_notifier_chain_register(&panic_notifier_list,
&mem_limit_notifier);
return 0;
}
__initcall(register_mem_limit_dumper);
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
...@@ -25,6 +25,8 @@ ...@@ -25,6 +25,8 @@
#include <asm/hwcap.h> #include <asm/hwcap.h>
#include <asm/pgtable-hwdef.h> #include <asm/pgtable-hwdef.h>
#include <asm/pgtable.h> #include <asm/pgtable.h>
#include <asm/cpufeature.h>
#include <asm/alternative.h>
#include "proc-macros.S" #include "proc-macros.S"
...@@ -137,9 +139,47 @@ ENTRY(cpu_do_switch_mm) ...@@ -137,9 +139,47 @@ ENTRY(cpu_do_switch_mm)
bfi x0, x1, #48, #16 // set the ASID bfi x0, x1, #48, #16 // set the ASID
msr ttbr0_el1, x0 // set TTBR0 msr ttbr0_el1, x0 // set TTBR0
isb isb
alternative_if_not ARM64_WORKAROUND_CAVIUM_27456
ret ret
nop
nop
nop
alternative_else
ic iallu
dsb nsh
isb
ret
alternative_endif
ENDPROC(cpu_do_switch_mm) ENDPROC(cpu_do_switch_mm)
.pushsection ".idmap.text", "ax"
/*
* void idmap_cpu_replace_ttbr1(phys_addr_t new_pgd)
*
* This is the low-level counterpart to cpu_replace_ttbr1, and should not be
* called by anything else. It can only be executed from a TTBR0 mapping.
*/
ENTRY(idmap_cpu_replace_ttbr1)
mrs x2, daif
msr daifset, #0xf
adrp x1, empty_zero_page
msr ttbr1_el1, x1
isb
tlbi vmalle1
dsb nsh
isb
msr ttbr1_el1, x0
isb
msr daif, x2
ret
ENDPROC(idmap_cpu_replace_ttbr1)
.popsection
/* /*
* __cpu_setup * __cpu_setup
* *
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment