Skip to content

L
linux
Commit 01158487 authored by Huacai Chen's avatar Huacai Chen
Browse files
Options

LoongArch: Introduce hardware page table walker 

Loongson-3A6000 and newer processors have hardware page table walker
(PTW) support. PTW can handle all fastpaths of TLBI/TLBL/TLBS/TLBM
exceptions by hardware, software only need to handle slowpaths (page
faults).

BTW, PTW doesn't append _PAGE_MODIFIED for page table entries, so we
change pmd_dirty() and pte_dirty() to also check _PAGE_DIRTY for the
"dirty" attribute.
Signed-off-by: default avatarLiang Gao <gaoliang@loongson.cn>
Signed-off-by: default avatarJun Yi <yijun@loongson.cn>
Signed-off-by: default avatarHuacai Chen <chenhuacai@loongson.cn>
parent e031a5f3
Hide whitespace changes
Inline Side-by-side
Showing with 56 additions and 7 deletions
arch/loongarch/include/asm/cpu-features.h
View file @ 01158487
...@@ -64,6 +64,6 @@ ...@@ -64,6 +64,6 @@
#define cpu_has_eiodecode cpu_opt(LOONGARCH_CPU_EIODECODE) #define cpu_has_eiodecode cpu_opt(LOONGARCH_CPU_EIODECODE)
#define cpu_has_guestid cpu_opt(LOONGARCH_CPU_GUESTID) #define cpu_has_guestid cpu_opt(LOONGARCH_CPU_GUESTID)
#define cpu_has_hypervisor cpu_opt(LOONGARCH_CPU_HYPERVISOR) #define cpu_has_hypervisor cpu_opt(LOONGARCH_CPU_HYPERVISOR)
#define cpu_has_ptw cpu_opt(LOONGARCH_CPU_PTW)
#endif /* __ASM_CPU_FEATURES_H */ #endif /* __ASM_CPU_FEATURES_H */
arch/loongarch/include/asm/cpu.h
View file @ 01158487
...@@ -98,6 +98,7 @@ enum cpu_type_enum { ...@@ -98,6 +98,7 @@ enum cpu_type_enum {
#define CPU_FEATURE_EIODECODE 23 /* CPU has EXTIOI interrupt pin decode mode */ #define CPU_FEATURE_EIODECODE 23 /* CPU has EXTIOI interrupt pin decode mode */
#define CPU_FEATURE_GUESTID 24 /* CPU has GuestID feature */ #define CPU_FEATURE_GUESTID 24 /* CPU has GuestID feature */
#define CPU_FEATURE_HYPERVISOR 25 /* CPU has hypervisor (running in VM) */ #define CPU_FEATURE_HYPERVISOR 25 /* CPU has hypervisor (running in VM) */
#define CPU_FEATURE_PTW 26 /* CPU has hardware page table walker */
#define LOONGARCH_CPU_CPUCFG BIT_ULL(CPU_FEATURE_CPUCFG) #define LOONGARCH_CPU_CPUCFG BIT_ULL(CPU_FEATURE_CPUCFG)
#define LOONGARCH_CPU_LAM BIT_ULL(CPU_FEATURE_LAM) #define LOONGARCH_CPU_LAM BIT_ULL(CPU_FEATURE_LAM)
...@@ -125,5 +126,6 @@ enum cpu_type_enum { ...@@ -125,5 +126,6 @@ enum cpu_type_enum {
#define LOONGARCH_CPU_EIODECODE BIT_ULL(CPU_FEATURE_EIODECODE) #define LOONGARCH_CPU_EIODECODE BIT_ULL(CPU_FEATURE_EIODECODE)
#define LOONGARCH_CPU_GUESTID BIT_ULL(CPU_FEATURE_GUESTID) #define LOONGARCH_CPU_GUESTID BIT_ULL(CPU_FEATURE_GUESTID)
#define LOONGARCH_CPU_HYPERVISOR BIT_ULL(CPU_FEATURE_HYPERVISOR) #define LOONGARCH_CPU_HYPERVISOR BIT_ULL(CPU_FEATURE_HYPERVISOR)
#define LOONGARCH_CPU_PTW BIT_ULL(CPU_FEATURE_PTW)
#endif /* _ASM_CPU_H */ #endif /* _ASM_CPU_H */
arch/loongarch/include/asm/loongarch.h
View file @ 01158487
...@@ -135,6 +135,7 @@ __asm__(".macro parse_r var r\n\t" ...@@ -135,6 +135,7 @@ __asm__(".macro parse_r var r\n\t"
#define CPUCFG2_MIPSBT BIT(20) #define CPUCFG2_MIPSBT BIT(20)
#define CPUCFG2_LSPW BIT(21) #define CPUCFG2_LSPW BIT(21)
#define CPUCFG2_LAM BIT(22) #define CPUCFG2_LAM BIT(22)
#define CPUCFG2_PTW BIT(24)
#define LOONGARCH_CPUCFG3 0x3 #define LOONGARCH_CPUCFG3 0x3
#define CPUCFG3_CCDMA BIT(0) #define CPUCFG3_CCDMA BIT(0)
...@@ -412,6 +413,9 @@ __asm__(".macro parse_r var r\n\t" ...@@ -412,6 +413,9 @@ __asm__(".macro parse_r var r\n\t"
#define CSR_PWCTL0_PTBASE (_ULCAST_(0x1f) << CSR_PWCTL0_PTBASE_SHIFT) #define CSR_PWCTL0_PTBASE (_ULCAST_(0x1f) << CSR_PWCTL0_PTBASE_SHIFT)
#define LOONGARCH_CSR_PWCTL1 0x1d /* PWCtl1 */ #define LOONGARCH_CSR_PWCTL1 0x1d /* PWCtl1 */
#define CSR_PWCTL1_PTW_SHIFT 24
#define CSR_PWCTL1_PTW_WIDTH 1
#define CSR_PWCTL1_PTW (_ULCAST_(0x1) << CSR_PWCTL1_PTW_SHIFT)
#define CSR_PWCTL1_DIR3WIDTH_SHIFT 18 #define CSR_PWCTL1_DIR3WIDTH_SHIFT 18
#define CSR_PWCTL1_DIR3WIDTH_WIDTH 5 #define CSR_PWCTL1_DIR3WIDTH_WIDTH 5
#define CSR_PWCTL1_DIR3WIDTH (_ULCAST_(0x1f) << CSR_PWCTL1_DIR3WIDTH_SHIFT) #define CSR_PWCTL1_DIR3WIDTH (_ULCAST_(0x1f) << CSR_PWCTL1_DIR3WIDTH_SHIFT)
......
arch/loongarch/include/asm/pgtable.h
View file @ 01158487
...@@ -362,7 +362,7 @@ extern pgd_t invalid_pg_dir[]; ...@@ -362,7 +362,7 @@ extern pgd_t invalid_pg_dir[];
*/ */
static inline int pte_write(pte_t pte) { return pte_val(pte) & _PAGE_WRITE; } static inline int pte_write(pte_t pte) { return pte_val(pte) & _PAGE_WRITE; }
static inline int pte_young(pte_t pte) { return pte_val(pte) & _PAGE_ACCESSED; } static inline int pte_young(pte_t pte) { return pte_val(pte) & _PAGE_ACCESSED; }
static inline int pte_dirty(pte_t pte) { return pte_val(pte) & _PAGE_MODIFIED; } static inline int pte_dirty(pte_t pte) { return pte_val(pte) & (_PAGE_DIRTY | _PAGE_MODIFIED); }
static inline pte_t pte_mkold(pte_t pte) static inline pte_t pte_mkold(pte_t pte)
{ {
...@@ -506,7 +506,7 @@ static inline pmd_t pmd_wrprotect(pmd_t pmd) ...@@ -506,7 +506,7 @@ static inline pmd_t pmd_wrprotect(pmd_t pmd)
static inline int pmd_dirty(pmd_t pmd) static inline int pmd_dirty(pmd_t pmd)
{ {
return !!(pmd_val(pmd) & _PAGE_MODIFIED); return !!(pmd_val(pmd) & (_PAGE_DIRTY | _PAGE_MODIFIED));
} }
static inline pmd_t pmd_mkclean(pmd_t pmd) static inline pmd_t pmd_mkclean(pmd_t pmd)
......
arch/loongarch/include/asm/tlb.h
View file @ 01158487
...@@ -158,6 +158,9 @@ extern void handle_tlb_store(void); ...@@ -158,6 +158,9 @@ extern void handle_tlb_store(void);
extern void handle_tlb_modify(void); extern void handle_tlb_modify(void);
extern void handle_tlb_refill(void); extern void handle_tlb_refill(void);
extern void handle_tlb_protect(void); extern void handle_tlb_protect(void);
extern void handle_tlb_load_ptw(void);
extern void handle_tlb_store_ptw(void);
extern void handle_tlb_modify_ptw(void);
extern void dump_tlb_all(void); extern void dump_tlb_all(void);
extern void dump_tlb_regs(void); extern void dump_tlb_regs(void);
......
arch/loongarch/include/uapi/asm/hwcap.h
View file @ 01158487
...@@ -16,5 +16,6 @@ ...@@ -16,5 +16,6 @@
#define HWCAP_LOONGARCH_LBT_X86 (1 << 10) #define HWCAP_LOONGARCH_LBT_X86 (1 << 10)
#define HWCAP_LOONGARCH_LBT_ARM (1 << 11) #define HWCAP_LOONGARCH_LBT_ARM (1 << 11)
#define HWCAP_LOONGARCH_LBT_MIPS (1 << 12) #define HWCAP_LOONGARCH_LBT_MIPS (1 << 12)
#define HWCAP_LOONGARCH_PTW (1 << 13)
#endif /* _UAPI_ASM_HWCAP_H */ #endif /* _UAPI_ASM_HWCAP_H */
arch/loongarch/kernel/cpu-probe.c
View file @ 01158487
...@@ -136,6 +136,10 @@ static void cpu_probe_common(struct cpuinfo_loongarch *c) ...@@ -136,6 +136,10 @@ static void cpu_probe_common(struct cpuinfo_loongarch *c)
c->options |= LOONGARCH_CPU_CRYPTO; c->options |= LOONGARCH_CPU_CRYPTO;
elf_hwcap |= HWCAP_LOONGARCH_CRYPTO; elf_hwcap |= HWCAP_LOONGARCH_CRYPTO;
} }
if (config & CPUCFG2_PTW) {
c->options |= LOONGARCH_CPU_PTW;
elf_hwcap |= HWCAP_LOONGARCH_PTW;
}
if (config & CPUCFG2_LVZP) { if (config & CPUCFG2_LVZP) {
c->options |= LOONGARCH_CPU_LVZ; c->options |= LOONGARCH_CPU_LVZ;
elf_hwcap |= HWCAP_LOONGARCH_LVZ; elf_hwcap |= HWCAP_LOONGARCH_LVZ;
......
arch/loongarch/kernel/proc.c
View file @ 01158487
...@@ -80,6 +80,7 @@ static int show_cpuinfo(struct seq_file *m, void *v) ...@@ -80,6 +80,7 @@ static int show_cpuinfo(struct seq_file *m, void *v)
if (cpu_has_crc32) seq_printf(m, " crc32"); if (cpu_has_crc32) seq_printf(m, " crc32");
if (cpu_has_complex) seq_printf(m, " complex"); if (cpu_has_complex) seq_printf(m, " complex");
if (cpu_has_crypto) seq_printf(m, " crypto"); if (cpu_has_crypto) seq_printf(m, " crypto");
if (cpu_has_ptw) seq_printf(m, " ptw");
if (cpu_has_lvz) seq_printf(m, " lvz"); if (cpu_has_lvz) seq_printf(m, " lvz");
if (cpu_has_lbt_x86) seq_printf(m, " lbt_x86"); if (cpu_has_lbt_x86) seq_printf(m, " lbt_x86");
if (cpu_has_lbt_arm) seq_printf(m, " lbt_arm"); if (cpu_has_lbt_arm) seq_printf(m, " lbt_arm");
......
arch/loongarch/mm/tlb.c
View file @ 01158487
...@@ -167,6 +167,9 @@ void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t *ptep ...@@ -167,6 +167,9 @@ void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t *ptep
int idx; int idx;
unsigned long flags; unsigned long flags;
if (cpu_has_ptw)
return;
/* /*
* Handle debugger faulting in for debugee. * Handle debugger faulting in for debugee.
*/ */
...@@ -222,6 +225,9 @@ static void setup_ptwalker(void) ...@@ -222,6 +225,9 @@ static void setup_ptwalker(void)
pwctl0 = pte_i | pte_w << 5 | pmd_i << 10 | pmd_w << 15 | pud_i << 20 | pud_w << 25; pwctl0 = pte_i | pte_w << 5 | pmd_i << 10 | pmd_w << 15 | pud_i << 20 | pud_w << 25;
pwctl1 = pgd_i | pgd_w << 6; pwctl1 = pgd_i | pgd_w << 6;
if (cpu_has_ptw)
pwctl1 |= CSR_PWCTL1_PTW;
csr_write64(pwctl0, LOONGARCH_CSR_PWCTL0); csr_write64(pwctl0, LOONGARCH_CSR_PWCTL0);
csr_write64(pwctl1, LOONGARCH_CSR_PWCTL1); csr_write64(pwctl1, LOONGARCH_CSR_PWCTL1);
csr_write64((long)swapper_pg_dir, LOONGARCH_CSR_PGDH); csr_write64((long)swapper_pg_dir, LOONGARCH_CSR_PGDH);
...@@ -264,10 +270,17 @@ void setup_tlb_handler(int cpu) ...@@ -264,10 +270,17 @@ void setup_tlb_handler(int cpu)
if (cpu == 0) { if (cpu == 0) {
memcpy((void *)tlbrentry, handle_tlb_refill, 0x80); memcpy((void *)tlbrentry, handle_tlb_refill, 0x80);
local_flush_icache_range(tlbrentry, tlbrentry + 0x80); local_flush_icache_range(tlbrentry, tlbrentry + 0x80);
set_handler(EXCCODE_TLBI * VECSIZE, handle_tlb_load, VECSIZE); if (!cpu_has_ptw) {
set_handler(EXCCODE_TLBL * VECSIZE, handle_tlb_load, VECSIZE); set_handler(EXCCODE_TLBI * VECSIZE, handle_tlb_load, VECSIZE);
set_handler(EXCCODE_TLBS * VECSIZE, handle_tlb_store, VECSIZE); set_handler(EXCCODE_TLBL * VECSIZE, handle_tlb_load, VECSIZE);
set_handler(EXCCODE_TLBM * VECSIZE, handle_tlb_modify, VECSIZE); set_handler(EXCCODE_TLBS * VECSIZE, handle_tlb_store, VECSIZE);
set_handler(EXCCODE_TLBM * VECSIZE, handle_tlb_modify, VECSIZE);
} else {
set_handler(EXCCODE_TLBI * VECSIZE, handle_tlb_load_ptw, VECSIZE);
set_handler(EXCCODE_TLBL * VECSIZE, handle_tlb_load_ptw, VECSIZE);
set_handler(EXCCODE_TLBS * VECSIZE, handle_tlb_store_ptw, VECSIZE);
set_handler(EXCCODE_TLBM * VECSIZE, handle_tlb_modify_ptw, VECSIZE);
}
set_handler(EXCCODE_TLBNR * VECSIZE, handle_tlb_protect, VECSIZE); set_handler(EXCCODE_TLBNR * VECSIZE, handle_tlb_protect, VECSIZE);
set_handler(EXCCODE_TLBNX * VECSIZE, handle_tlb_protect, VECSIZE); set_handler(EXCCODE_TLBNX * VECSIZE, handle_tlb_protect, VECSIZE);
set_handler(EXCCODE_TLBPE * VECSIZE, handle_tlb_protect, VECSIZE); set_handler(EXCCODE_TLBPE * VECSIZE, handle_tlb_protect, VECSIZE);
......
arch/loongarch/mm/tlbex.S
View file @ 01158487
...@@ -190,6 +190,13 @@ nopage_tlb_load: ...@@ -190,6 +190,13 @@ nopage_tlb_load:
jr t0 jr t0
SYM_FUNC_END(handle_tlb_load) SYM_FUNC_END(handle_tlb_load)
SYM_FUNC_START(handle_tlb_load_ptw)
csrwr t0, LOONGARCH_CSR_KS0
csrwr t1, LOONGARCH_CSR_KS1
la_abs t0, tlb_do_page_fault_0
jr t0
SYM_FUNC_END(handle_tlb_load_ptw)
SYM_FUNC_START(handle_tlb_store) SYM_FUNC_START(handle_tlb_store)
csrwr t0, EXCEPTION_KS0 csrwr t0, EXCEPTION_KS0
csrwr t1, EXCEPTION_KS1 csrwr t1, EXCEPTION_KS1
...@@ -339,6 +346,13 @@ nopage_tlb_store: ...@@ -339,6 +346,13 @@ nopage_tlb_store:
jr t0 jr t0
SYM_FUNC_END(handle_tlb_store) SYM_FUNC_END(handle_tlb_store)
SYM_FUNC_START(handle_tlb_store_ptw)
csrwr t0, LOONGARCH_CSR_KS0
csrwr t1, LOONGARCH_CSR_KS1
la_abs t0, tlb_do_page_fault_1
jr t0
SYM_FUNC_END(handle_tlb_store_ptw)
SYM_FUNC_START(handle_tlb_modify) SYM_FUNC_START(handle_tlb_modify)
csrwr t0, EXCEPTION_KS0 csrwr t0, EXCEPTION_KS0
csrwr t1, EXCEPTION_KS1 csrwr t1, EXCEPTION_KS1
...@@ -486,6 +500,13 @@ nopage_tlb_modify: ...@@ -486,6 +500,13 @@ nopage_tlb_modify:
jr t0 jr t0
SYM_FUNC_END(handle_tlb_modify) SYM_FUNC_END(handle_tlb_modify)
SYM_FUNC_START(handle_tlb_modify_ptw)
csrwr t0, LOONGARCH_CSR_KS0
csrwr t1, LOONGARCH_CSR_KS1
la_abs t0, tlb_do_page_fault_1
jr t0
SYM_FUNC_END(handle_tlb_modify_ptw)
SYM_FUNC_START(handle_tlb_refill) SYM_FUNC_START(handle_tlb_refill)
csrwr t0, LOONGARCH_CSR_TLBRSAVE csrwr t0, LOONGARCH_CSR_TLBRSAVE
csrrd t0, LOONGARCH_CSR_PGD csrrd t0, LOONGARCH_CSR_PGD
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7