Commit f4ae17aa authored by Jayachandran C's avatar Jayachandran C Committed by Ralf Baechle

MIPS: mm: Use scratch for PGD when !CONFIG_MIPS_PGD_C0_CONTEXT

Allow usage of scratch register for current pgd even when
MIPS_PGD_C0_CONTEXT is not configured. MIPS_PGD_C0_CONTEXT is set
for 64r2 platforms to indicate availability of Xcontext for saving
cpuid, thus freeing Context to be used for saving PGD. This option
was also tied to using a scratch register for storing PGD.

This commit will allow usage of scratch register to store the current
pgd if one can be allocated for the platform, even when
MIPS_PGD_C0_CONTEXT is not set. The cpuid will be kept in the CP0
Context register in this case.

The code to store the current pgd for the TLB miss handler is now
generated in all cases. When scratch register is available, the PGD
is also stored in the scratch register.
Signed-off-by: default avatarJayachandran C <jchandra@broadcom.com>
Cc: linux-mips@linux-mips.org
Cc: Hauke Mehrtens <hauke@hauke-m.de>
Patchwork: https://patchwork.linux-mips.org/patch/5906/Signed-off-by: default avatarRalf Baechle <ralf@linux-mips.org>
parent 7f177a52
...@@ -24,14 +24,13 @@ ...@@ -24,14 +24,13 @@
#endif /* SMTC */ #endif /* SMTC */
#include <asm-generic/mm_hooks.h> #include <asm-generic/mm_hooks.h>
#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
#define TLBMISS_HANDLER_SETUP_PGD(pgd) \ #define TLBMISS_HANDLER_SETUP_PGD(pgd) \
do { \ do { \
extern void tlbmiss_handler_setup_pgd(unsigned long); \ extern void tlbmiss_handler_setup_pgd(unsigned long); \
tlbmiss_handler_setup_pgd((unsigned long)(pgd)); \ tlbmiss_handler_setup_pgd((unsigned long)(pgd)); \
} while (0) } while (0)
#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
#define TLBMISS_HANDLER_SETUP() \ #define TLBMISS_HANDLER_SETUP() \
do { \ do { \
TLBMISS_HANDLER_SETUP_PGD(swapper_pg_dir); \ TLBMISS_HANDLER_SETUP_PGD(swapper_pg_dir); \
...@@ -48,9 +47,6 @@ do { \ ...@@ -48,9 +47,6 @@ do { \
*/ */
extern unsigned long pgd_current[]; extern unsigned long pgd_current[];
#define TLBMISS_HANDLER_SETUP_PGD(pgd) \
pgd_current[smp_processor_id()] = (unsigned long)(pgd)
#define TLBMISS_HANDLER_SETUP() \ #define TLBMISS_HANDLER_SETUP() \
write_c0_context((unsigned long) smp_processor_id() << \ write_c0_context((unsigned long) smp_processor_id() << \
SMP_CPUID_REGSHIFT); \ SMP_CPUID_REGSHIFT); \
......
...@@ -16,12 +16,10 @@ ...@@ -16,12 +16,10 @@
#define FASTPATH_SIZE 128 #define FASTPATH_SIZE 128
#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
LEAF(tlbmiss_handler_setup_pgd) LEAF(tlbmiss_handler_setup_pgd)
.space 16 * 4 .space 16 * 4
END(tlbmiss_handler_setup_pgd) END(tlbmiss_handler_setup_pgd)
EXPORT(tlbmiss_handler_setup_pgd_end) EXPORT(tlbmiss_handler_setup_pgd_end)
#endif
LEAF(handle_tlbm) LEAF(handle_tlbm)
.space FASTPATH_SIZE * 4 .space FASTPATH_SIZE * 4
......
...@@ -799,11 +799,11 @@ build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, ...@@ -799,11 +799,11 @@ build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
} }
/* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */ /* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */
#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
if (pgd_reg != -1) { if (pgd_reg != -1) {
/* pgd is in pgd_reg */ /* pgd is in pgd_reg */
UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg); UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg);
} else { } else {
#if defined(CONFIG_MIPS_PGD_C0_CONTEXT)
/* /*
* &pgd << 11 stored in CONTEXT [23..63]. * &pgd << 11 stored in CONTEXT [23..63].
*/ */
...@@ -815,7 +815,6 @@ build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, ...@@ -815,7 +815,6 @@ build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
/* 1 0 1 0 1 << 6 xkphys cached */ /* 1 0 1 0 1 << 6 xkphys cached */
uasm_i_ori(p, ptr, ptr, 0x540); uasm_i_ori(p, ptr, ptr, 0x540);
uasm_i_drotr(p, ptr, ptr, 11); uasm_i_drotr(p, ptr, ptr, 11);
}
#elif defined(CONFIG_SMP) #elif defined(CONFIG_SMP)
UASM_i_CPUID_MFC0(p, ptr, SMP_CPUID_REG); UASM_i_CPUID_MFC0(p, ptr, SMP_CPUID_REG);
uasm_i_dsrl_safe(p, ptr, ptr, SMP_CPUID_PTRSHIFT); uasm_i_dsrl_safe(p, ptr, ptr, SMP_CPUID_PTRSHIFT);
...@@ -827,6 +826,7 @@ build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, ...@@ -827,6 +826,7 @@ build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
UASM_i_LA_mostly(p, ptr, pgdc); UASM_i_LA_mostly(p, ptr, pgdc);
uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr); uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr);
#endif #endif
}
uasm_l_vmalloc_done(l, *p); uasm_l_vmalloc_done(l, *p);
...@@ -921,6 +921,11 @@ build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, ...@@ -921,6 +921,11 @@ build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
static void __maybe_unused static void __maybe_unused
build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr) build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr)
{ {
if (pgd_reg != -1) {
/* pgd is in pgd_reg */
uasm_i_mfc0(p, ptr, c0_kscratch(), pgd_reg);
uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */
} else {
long pgdc = (long)pgd_current; long pgdc = (long)pgd_current;
/* 32 bit SMP has smp_processor_id() stored in CONTEXT. */ /* 32 bit SMP has smp_processor_id() stored in CONTEXT. */
...@@ -934,6 +939,7 @@ build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr) ...@@ -934,6 +939,7 @@ build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr)
#endif #endif
uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */ uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */
uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr); uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr);
}
uasm_i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */ uasm_i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */
uasm_i_sll(p, tmp, tmp, PGD_T_LOG2); uasm_i_sll(p, tmp, tmp, PGD_T_LOG2);
uasm_i_addu(p, ptr, ptr, tmp); /* add in pgd offset */ uasm_i_addu(p, ptr, ptr, tmp); /* add in pgd offset */
...@@ -1407,28 +1413,30 @@ static void build_r4000_tlb_refill_handler(void) ...@@ -1407,28 +1413,30 @@ static void build_r4000_tlb_refill_handler(void)
extern u32 handle_tlbl[], handle_tlbl_end[]; extern u32 handle_tlbl[], handle_tlbl_end[];
extern u32 handle_tlbs[], handle_tlbs_end[]; extern u32 handle_tlbs[], handle_tlbs_end[];
extern u32 handle_tlbm[], handle_tlbm_end[]; extern u32 handle_tlbm[], handle_tlbm_end[];
#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
extern u32 tlbmiss_handler_setup_pgd[], tlbmiss_handler_setup_pgd_end[]; extern u32 tlbmiss_handler_setup_pgd[], tlbmiss_handler_setup_pgd_end[];
static void build_r4000_setup_pgd(void) static void build_setup_pgd(void)
{ {
const int a0 = 4; const int a0 = 4;
const int a1 = 5; const int __maybe_unused a1 = 5;
const int __maybe_unused a2 = 6;
u32 *p = tlbmiss_handler_setup_pgd; u32 *p = tlbmiss_handler_setup_pgd;
const int tlbmiss_handler_setup_pgd_size = const int tlbmiss_handler_setup_pgd_size =
tlbmiss_handler_setup_pgd_end - tlbmiss_handler_setup_pgd; tlbmiss_handler_setup_pgd_end - tlbmiss_handler_setup_pgd;
struct uasm_label *l = labels; #ifndef CONFIG_MIPS_PGD_C0_CONTEXT
struct uasm_reloc *r = relocs; long pgdc = (long)pgd_current;
#endif
memset(tlbmiss_handler_setup_pgd, 0, tlbmiss_handler_setup_pgd_size * memset(tlbmiss_handler_setup_pgd, 0, tlbmiss_handler_setup_pgd_size *
sizeof(tlbmiss_handler_setup_pgd[0])); sizeof(tlbmiss_handler_setup_pgd[0]));
memset(labels, 0, sizeof(labels)); memset(labels, 0, sizeof(labels));
memset(relocs, 0, sizeof(relocs)); memset(relocs, 0, sizeof(relocs));
pgd_reg = allocate_kscratch(); pgd_reg = allocate_kscratch();
#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
if (pgd_reg == -1) { if (pgd_reg == -1) {
struct uasm_label *l = labels;
struct uasm_reloc *r = relocs;
/* PGD << 11 in c0_Context */ /* PGD << 11 in c0_Context */
/* /*
* If it is a ckseg0 address, convert to a physical * If it is a ckseg0 address, convert to a physical
...@@ -1450,6 +1458,26 @@ static void build_r4000_setup_pgd(void) ...@@ -1450,6 +1458,26 @@ static void build_r4000_setup_pgd(void)
uasm_i_jr(&p, 31); uasm_i_jr(&p, 31);
UASM_i_MTC0(&p, a0, c0_kscratch(), pgd_reg); UASM_i_MTC0(&p, a0, c0_kscratch(), pgd_reg);
} }
#else
#ifdef CONFIG_SMP
/* Save PGD to pgd_current[smp_processor_id()] */
UASM_i_CPUID_MFC0(&p, a1, SMP_CPUID_REG);
UASM_i_SRL_SAFE(&p, a1, a1, SMP_CPUID_PTRSHIFT);
UASM_i_LA_mostly(&p, a2, pgdc);
UASM_i_ADDU(&p, a2, a2, a1);
UASM_i_SW(&p, a0, uasm_rel_lo(pgdc), a2);
#else
UASM_i_LA_mostly(&p, a2, pgdc);
UASM_i_SW(&p, a0, uasm_rel_lo(pgdc), a2);
#endif /* SMP */
uasm_i_jr(&p, 31);
/* if pgd_reg is allocated, save PGD also to scratch register */
if (pgd_reg != -1)
UASM_i_MTC0(&p, a0, c0_kscratch(), pgd_reg);
else
uasm_i_nop(&p);
#endif
if (p >= tlbmiss_handler_setup_pgd_end) if (p >= tlbmiss_handler_setup_pgd_end)
panic("tlbmiss_handler_setup_pgd space exceeded"); panic("tlbmiss_handler_setup_pgd space exceeded");
...@@ -1460,7 +1488,6 @@ static void build_r4000_setup_pgd(void) ...@@ -1460,7 +1488,6 @@ static void build_r4000_setup_pgd(void)
dump_handler("tlbmiss_handler", tlbmiss_handler_setup_pgd, dump_handler("tlbmiss_handler", tlbmiss_handler_setup_pgd,
tlbmiss_handler_setup_pgd_size); tlbmiss_handler_setup_pgd_size);
} }
#endif
static void static void
iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr) iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr)
...@@ -2153,10 +2180,8 @@ static void flush_tlb_handlers(void) ...@@ -2153,10 +2180,8 @@ static void flush_tlb_handlers(void)
(unsigned long)handle_tlbs_end); (unsigned long)handle_tlbs_end);
local_flush_icache_range((unsigned long)handle_tlbm, local_flush_icache_range((unsigned long)handle_tlbm,
(unsigned long)handle_tlbm_end); (unsigned long)handle_tlbm_end);
#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
local_flush_icache_range((unsigned long)tlbmiss_handler_setup_pgd, local_flush_icache_range((unsigned long)tlbmiss_handler_setup_pgd,
(unsigned long)tlbmiss_handler_setup_pgd_end); (unsigned long)tlbmiss_handler_setup_pgd_end);
#endif
} }
void build_tlb_refill_handler(void) void build_tlb_refill_handler(void)
...@@ -2188,6 +2213,7 @@ void build_tlb_refill_handler(void) ...@@ -2188,6 +2213,7 @@ void build_tlb_refill_handler(void)
if (!run_once) { if (!run_once) {
if (!cpu_has_local_ebase) if (!cpu_has_local_ebase)
build_r3000_tlb_refill_handler(); build_r3000_tlb_refill_handler();
build_setup_pgd();
build_r3000_tlb_load_handler(); build_r3000_tlb_load_handler();
build_r3000_tlb_store_handler(); build_r3000_tlb_store_handler();
build_r3000_tlb_modify_handler(); build_r3000_tlb_modify_handler();
...@@ -2211,9 +2237,7 @@ void build_tlb_refill_handler(void) ...@@ -2211,9 +2237,7 @@ void build_tlb_refill_handler(void)
default: default:
if (!run_once) { if (!run_once) {
scratch_reg = allocate_kscratch(); scratch_reg = allocate_kscratch();
#ifdef CONFIG_MIPS_PGD_C0_CONTEXT build_setup_pgd();
build_r4000_setup_pgd();
#endif
build_r4000_tlb_load_handler(); build_r4000_tlb_load_handler();
build_r4000_tlb_store_handler(); build_r4000_tlb_store_handler();
build_r4000_tlb_modify_handler(); build_r4000_tlb_modify_handler();
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment