Commit cc33ae43 authored by David Daney's avatar David Daney Committed by Ralf Baechle

MIPS: Use BBIT instructions in TLB handlers

If the CPU supports BBIT0 and BBIT1, use them in TLB handlers as they
are more efficient than an AND followed by an branch and then
restoring the clobbered register.
Signed-off-by: default avatarDavid Daney <ddaney@caviumnetworks.com>
To: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/1873/Signed-off-by: default avatarRalf Baechle <ralf@linux-mips.org>
parent afc7c986
...@@ -65,6 +65,18 @@ static inline int __maybe_unused r10000_llsc_war(void) ...@@ -65,6 +65,18 @@ static inline int __maybe_unused r10000_llsc_war(void)
return R10000_LLSC_WAR; return R10000_LLSC_WAR;
} }
static int use_bbit_insns(void)
{
switch (current_cpu_type()) {
case CPU_CAVIUM_OCTEON:
case CPU_CAVIUM_OCTEON_PLUS:
case CPU_CAVIUM_OCTEON2:
return 1;
default:
return 0;
}
}
/* /*
* Found by experiment: At least some revisions of the 4kc throw under * Found by experiment: At least some revisions of the 4kc throw under
* some circumstances a machine check exception, triggered by invalid * some circumstances a machine check exception, triggered by invalid
...@@ -511,8 +523,12 @@ build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp, ...@@ -511,8 +523,12 @@ build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp,
unsigned int pmd, int lid) unsigned int pmd, int lid)
{ {
UASM_i_LW(p, tmp, 0, pmd); UASM_i_LW(p, tmp, 0, pmd);
uasm_i_andi(p, tmp, tmp, _PAGE_HUGE); if (use_bbit_insns()) {
uasm_il_bnez(p, r, tmp, lid); uasm_il_bbit1(p, r, tmp, ilog2(_PAGE_HUGE), lid);
} else {
uasm_i_andi(p, tmp, tmp, _PAGE_HUGE);
uasm_il_bnez(p, r, tmp, lid);
}
} }
static __cpuinit void build_huge_update_entries(u32 **p, static __cpuinit void build_huge_update_entries(u32 **p,
...@@ -1187,14 +1203,20 @@ build_pte_present(u32 **p, struct uasm_reloc **r, ...@@ -1187,14 +1203,20 @@ build_pte_present(u32 **p, struct uasm_reloc **r,
unsigned int pte, unsigned int ptr, enum label_id lid) unsigned int pte, unsigned int ptr, enum label_id lid)
{ {
if (kernel_uses_smartmips_rixi) { if (kernel_uses_smartmips_rixi) {
uasm_i_andi(p, pte, pte, _PAGE_PRESENT); if (use_bbit_insns()) {
uasm_il_beqz(p, r, pte, lid); uasm_il_bbit0(p, r, pte, ilog2(_PAGE_PRESENT), lid);
uasm_i_nop(p);
} else {
uasm_i_andi(p, pte, pte, _PAGE_PRESENT);
uasm_il_beqz(p, r, pte, lid);
iPTE_LW(p, pte, ptr);
}
} else { } else {
uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_READ); uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_READ); uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
uasm_il_bnez(p, r, pte, lid); uasm_il_bnez(p, r, pte, lid);
iPTE_LW(p, pte, ptr);
} }
iPTE_LW(p, pte, ptr);
} }
/* Make PTE valid, store result in PTR. */ /* Make PTE valid, store result in PTR. */
...@@ -1215,10 +1237,17 @@ static void __cpuinit ...@@ -1215,10 +1237,17 @@ static void __cpuinit
build_pte_writable(u32 **p, struct uasm_reloc **r, build_pte_writable(u32 **p, struct uasm_reloc **r,
unsigned int pte, unsigned int ptr, enum label_id lid) unsigned int pte, unsigned int ptr, enum label_id lid)
{ {
uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE); if (use_bbit_insns()) {
uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE); uasm_il_bbit0(p, r, pte, ilog2(_PAGE_PRESENT), lid);
uasm_il_bnez(p, r, pte, lid); uasm_i_nop(p);
iPTE_LW(p, pte, ptr); uasm_il_bbit0(p, r, pte, ilog2(_PAGE_WRITE), lid);
uasm_i_nop(p);
} else {
uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE);
uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE);
uasm_il_bnez(p, r, pte, lid);
iPTE_LW(p, pte, ptr);
}
} }
/* Make PTE writable, update software status bits as well, then store /* Make PTE writable, update software status bits as well, then store
...@@ -1242,9 +1271,14 @@ static void __cpuinit ...@@ -1242,9 +1271,14 @@ static void __cpuinit
build_pte_modifiable(u32 **p, struct uasm_reloc **r, build_pte_modifiable(u32 **p, struct uasm_reloc **r,
unsigned int pte, unsigned int ptr, enum label_id lid) unsigned int pte, unsigned int ptr, enum label_id lid)
{ {
uasm_i_andi(p, pte, pte, _PAGE_WRITE); if (use_bbit_insns()) {
uasm_il_beqz(p, r, pte, lid); uasm_il_bbit0(p, r, pte, ilog2(_PAGE_WRITE), lid);
iPTE_LW(p, pte, ptr); uasm_i_nop(p);
} else {
uasm_i_andi(p, pte, pte, _PAGE_WRITE);
uasm_il_beqz(p, r, pte, lid);
iPTE_LW(p, pte, ptr);
}
} }
#ifndef CONFIG_MIPS_PGD_C0_CONTEXT #ifndef CONFIG_MIPS_PGD_C0_CONTEXT
...@@ -1491,14 +1525,23 @@ static void __cpuinit build_r4000_tlb_load_handler(void) ...@@ -1491,14 +1525,23 @@ static void __cpuinit build_r4000_tlb_load_handler(void)
* If the page is not _PAGE_VALID, RI or XI could not * If the page is not _PAGE_VALID, RI or XI could not
* have triggered it. Skip the expensive test.. * have triggered it. Skip the expensive test..
*/ */
uasm_i_andi(&p, K0, K0, _PAGE_VALID); if (use_bbit_insns()) {
uasm_il_beqz(&p, &r, K0, label_tlbl_goaround1); uasm_il_bbit0(&p, &r, K0, ilog2(_PAGE_VALID),
label_tlbl_goaround1);
} else {
uasm_i_andi(&p, K0, K0, _PAGE_VALID);
uasm_il_beqz(&p, &r, K0, label_tlbl_goaround1);
}
uasm_i_nop(&p); uasm_i_nop(&p);
uasm_i_tlbr(&p); uasm_i_tlbr(&p);
/* Examine entrylo 0 or 1 based on ptr. */ /* Examine entrylo 0 or 1 based on ptr. */
uasm_i_andi(&p, K0, K1, sizeof(pte_t)); if (use_bbit_insns()) {
uasm_i_beqz(&p, K0, 8); uasm_i_bbit0(&p, K1, ilog2(sizeof(pte_t)), 8);
} else {
uasm_i_andi(&p, K0, K1, sizeof(pte_t));
uasm_i_beqz(&p, K0, 8);
}
UASM_i_MFC0(&p, K0, C0_ENTRYLO0); /* load it in the delay slot*/ UASM_i_MFC0(&p, K0, C0_ENTRYLO0); /* load it in the delay slot*/
UASM_i_MFC0(&p, K0, C0_ENTRYLO1); /* load it if ptr is odd */ UASM_i_MFC0(&p, K0, C0_ENTRYLO1); /* load it if ptr is odd */
...@@ -1506,12 +1549,18 @@ static void __cpuinit build_r4000_tlb_load_handler(void) ...@@ -1506,12 +1549,18 @@ static void __cpuinit build_r4000_tlb_load_handler(void)
* If the entryLo (now in K0) is valid (bit 1), RI or * If the entryLo (now in K0) is valid (bit 1), RI or
* XI must have triggered it. * XI must have triggered it.
*/ */
uasm_i_andi(&p, K0, K0, 2); if (use_bbit_insns()) {
uasm_il_bnez(&p, &r, K0, label_nopage_tlbl); uasm_il_bbit1(&p, &r, K0, 1, label_nopage_tlbl);
/* Reload the PTE value */
uasm_l_tlbl_goaround1(&l, p); iPTE_LW(&p, K0, K1);
/* Reload the PTE value */ uasm_l_tlbl_goaround1(&l, p);
iPTE_LW(&p, K0, K1); } else {
uasm_i_andi(&p, K0, K0, 2);
uasm_il_bnez(&p, &r, K0, label_nopage_tlbl);
uasm_l_tlbl_goaround1(&l, p);
/* Reload the PTE value */
iPTE_LW(&p, K0, K1);
}
} }
build_make_valid(&p, &r, K0, K1); build_make_valid(&p, &r, K0, K1);
build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1); build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);
...@@ -1531,23 +1580,35 @@ static void __cpuinit build_r4000_tlb_load_handler(void) ...@@ -1531,23 +1580,35 @@ static void __cpuinit build_r4000_tlb_load_handler(void)
* If the page is not _PAGE_VALID, RI or XI could not * If the page is not _PAGE_VALID, RI or XI could not
* have triggered it. Skip the expensive test.. * have triggered it. Skip the expensive test..
*/ */
uasm_i_andi(&p, K0, K0, _PAGE_VALID); if (use_bbit_insns()) {
uasm_il_beqz(&p, &r, K0, label_tlbl_goaround2); uasm_il_bbit0(&p, &r, K0, ilog2(_PAGE_VALID),
label_tlbl_goaround2);
} else {
uasm_i_andi(&p, K0, K0, _PAGE_VALID);
uasm_il_beqz(&p, &r, K0, label_tlbl_goaround2);
}
uasm_i_nop(&p); uasm_i_nop(&p);
uasm_i_tlbr(&p); uasm_i_tlbr(&p);
/* Examine entrylo 0 or 1 based on ptr. */ /* Examine entrylo 0 or 1 based on ptr. */
uasm_i_andi(&p, K0, K1, sizeof(pte_t)); if (use_bbit_insns()) {
uasm_i_beqz(&p, K0, 8); uasm_i_bbit0(&p, K1, ilog2(sizeof(pte_t)), 8);
} else {
uasm_i_andi(&p, K0, K1, sizeof(pte_t));
uasm_i_beqz(&p, K0, 8);
}
UASM_i_MFC0(&p, K0, C0_ENTRYLO0); /* load it in the delay slot*/ UASM_i_MFC0(&p, K0, C0_ENTRYLO0); /* load it in the delay slot*/
UASM_i_MFC0(&p, K0, C0_ENTRYLO1); /* load it if ptr is odd */ UASM_i_MFC0(&p, K0, C0_ENTRYLO1); /* load it if ptr is odd */
/* /*
* If the entryLo (now in K0) is valid (bit 1), RI or * If the entryLo (now in K0) is valid (bit 1), RI or
* XI must have triggered it. * XI must have triggered it.
*/ */
uasm_i_andi(&p, K0, K0, 2); if (use_bbit_insns()) {
uasm_il_beqz(&p, &r, K0, label_tlbl_goaround2); uasm_il_bbit0(&p, &r, K0, 1, label_tlbl_goaround2);
} else {
uasm_i_andi(&p, K0, K0, 2);
uasm_il_beqz(&p, &r, K0, label_tlbl_goaround2);
}
/* Reload the PTE value */ /* Reload the PTE value */
iPTE_LW(&p, K0, K1); iPTE_LW(&p, K0, K1);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment