Commit 0aaa6853 authored by Ard Biesheuvel's avatar Ard Biesheuvel Committed by Will Deacon

arm64: mm: fix booting with 52-bit address space

Joey reports that booting 52-bit VA capable builds on 52-bit VA capable
CPUs is broken since commit 0d9b1ffe ("arm64: mm: make vabits_actual
a build time constant if possible"). This is due to the fact that the
primary CPU reads the vabits_actual variable before it has been
assigned.

The reason for deferring the assignment of vabits_actual was that we try
to perform as few stores to memory as we can with the MMU and caches
off, due to the cache coherency issues it creates.

Since __cpu_setup() [which is where the read of vabits_actual occurs] is
also called on the secondary boot path, we cannot just read the CPU ID
registers directly, given that the size of the VA space is decided by
the capabilities of the primary CPU. So let's read vabits_actual only on
the secondary boot path, and read the CPU ID registers directly on the
primary boot path, by making it a function parameter of __cpu_setup().

To ensure that all users of vabits_actual (including kasan_early_init())
observe the correct value, move the assignment of vabits_actual back
into asm code, but still defer it to after the MMU and caches have been
enabled.

Cc: Will Deacon <will@kernel.org>
Cc: Anshuman Khandual <anshuman.khandual@arm.com>
Cc: Mark Rutland <mark.rutland@arm.com>
Fixes: 0d9b1ffe ("arm64: mm: make vabits_actual a build time constant if possible")
Reported-by: default avatarJoey Gouly <joey.gouly@arm.com>
Co-developed-by: default avatarJoey Gouly <joey.gouly@arm.com>
Signed-off-by: default avatarJoey Gouly <joey.gouly@arm.com>
Signed-off-by: default avatarArd Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20220701111045.2944309-1-ardb@kernel.orgSigned-off-by: default avatarWill Deacon <will@kernel.org>
parent bdbcd22d
...@@ -82,6 +82,7 @@ ...@@ -82,6 +82,7 @@
* x22 create_idmap() .. start_kernel() ID map VA of the DT blob * x22 create_idmap() .. start_kernel() ID map VA of the DT blob
* x23 primary_entry() .. start_kernel() physical misalignment/KASLR offset * x23 primary_entry() .. start_kernel() physical misalignment/KASLR offset
* x24 __primary_switch() linear map KASLR seed * x24 __primary_switch() linear map KASLR seed
* x25 primary_entry() .. start_kernel() supported VA size
* x28 create_idmap() callee preserved temp register * x28 create_idmap() callee preserved temp register
*/ */
SYM_CODE_START(primary_entry) SYM_CODE_START(primary_entry)
...@@ -96,6 +97,14 @@ SYM_CODE_START(primary_entry) ...@@ -96,6 +97,14 @@ SYM_CODE_START(primary_entry)
* On return, the CPU will be ready for the MMU to be turned on and * On return, the CPU will be ready for the MMU to be turned on and
* the TCR will have been set. * the TCR will have been set.
*/ */
#if VA_BITS > 48
mrs_s x0, SYS_ID_AA64MMFR2_EL1
tst x0, #0xf << ID_AA64MMFR2_LVA_SHIFT
mov x0, #VA_BITS
mov x25, #VA_BITS_MIN
csel x25, x25, x0, eq
mov x0, x25
#endif
bl __cpu_setup // initialise processor bl __cpu_setup // initialise processor
b __primary_switch b __primary_switch
SYM_CODE_END(primary_entry) SYM_CODE_END(primary_entry)
...@@ -434,6 +443,12 @@ SYM_FUNC_START_LOCAL(__primary_switched) ...@@ -434,6 +443,12 @@ SYM_FUNC_START_LOCAL(__primary_switched)
bl __pi_memset bl __pi_memset
dsb ishst // Make zero page visible to PTW dsb ishst // Make zero page visible to PTW
#if VA_BITS > 48
adr_l x8, vabits_actual // Set this early so KASAN early init
str x25, [x8] // ... observes the correct value
dc civac, x8 // Make visible to booting secondaries
#endif
#ifdef CONFIG_RANDOMIZE_BASE #ifdef CONFIG_RANDOMIZE_BASE
adrp x5, memstart_offset_seed // Save KASLR linear map seed adrp x5, memstart_offset_seed // Save KASLR linear map seed
strh w24, [x5, :lo12:memstart_offset_seed] strh w24, [x5, :lo12:memstart_offset_seed]
...@@ -579,6 +594,9 @@ SYM_FUNC_START_LOCAL(secondary_startup) ...@@ -579,6 +594,9 @@ SYM_FUNC_START_LOCAL(secondary_startup)
mov x20, x0 // preserve boot mode mov x20, x0 // preserve boot mode
bl switch_to_vhe bl switch_to_vhe
bl __cpu_secondary_check52bitva bl __cpu_secondary_check52bitva
#if VA_BITS > 48
ldr_l x0, vabits_actual
#endif
bl __cpu_setup // initialise processor bl __cpu_setup // initialise processor
adrp x1, swapper_pg_dir adrp x1, swapper_pg_dir
adrp x2, idmap_pg_dir adrp x2, idmap_pg_dir
......
...@@ -265,20 +265,7 @@ early_param("mem", early_mem); ...@@ -265,20 +265,7 @@ early_param("mem", early_mem);
void __init arm64_memblock_init(void) void __init arm64_memblock_init(void)
{ {
s64 linear_region_size; s64 linear_region_size = PAGE_END - _PAGE_OFFSET(vabits_actual);
#if VA_BITS > 48
if (cpuid_feature_extract_unsigned_field(
read_sysreg_s(SYS_ID_AA64MMFR2_EL1),
ID_AA64MMFR2_LVA_SHIFT))
vabits_actual = VA_BITS;
/* make the variable visible to secondaries with the MMU off */
dcache_clean_inval_poc((u64)&vabits_actual,
(u64)&vabits_actual + sizeof(vabits_actual));
#endif
linear_region_size = PAGE_END - _PAGE_OFFSET(vabits_actual);
/* /*
* Corner case: 52-bit VA capable systems running KVM in nVHE mode may * Corner case: 52-bit VA capable systems running KVM in nVHE mode may
......
...@@ -397,6 +397,8 @@ SYM_FUNC_END(idmap_kpti_install_ng_mappings) ...@@ -397,6 +397,8 @@ SYM_FUNC_END(idmap_kpti_install_ng_mappings)
* *
* Initialise the processor for turning the MMU on. * Initialise the processor for turning the MMU on.
* *
* Input:
* x0 - actual number of VA bits (ignored unless VA_BITS > 48)
* Output: * Output:
* Return in x0 the value of the SCTLR_EL1 register. * Return in x0 the value of the SCTLR_EL1 register.
*/ */
...@@ -466,8 +468,7 @@ SYM_FUNC_START(__cpu_setup) ...@@ -466,8 +468,7 @@ SYM_FUNC_START(__cpu_setup)
tcr_clear_errata_bits tcr, x9, x5 tcr_clear_errata_bits tcr, x9, x5
#ifdef CONFIG_ARM64_VA_BITS_52 #ifdef CONFIG_ARM64_VA_BITS_52
ldr_l x9, vabits_actual sub x9, xzr, x0
sub x9, xzr, x9
add x9, x9, #64 add x9, x9, #64
tcr_set_t1sz tcr, x9 tcr_set_t1sz tcr, x9
#else #else
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment