Commit 112e7e21 authored by Linus Torvalds's avatar Linus Torvalds

Merge tag 'loongarch-6.5' of...

Merge tag 'loongarch-6.5' of git://git.kernel.org/pub/scm/linux/kernel/git/chenhuacai/linux-loongson

Pull LoongArch updates from Huacai Chen:

 - preliminary ClangBuiltLinux enablement

 - add support to clone a time namespace

 - add vector extensions support

 - add SMT (Simultaneous Multi-Threading) support

 - support dbar with different hints

 - introduce hardware page table walker

 - add jump-label implementation

 - add rethook and uprobes support

 - some bug fixes and other small changes

* tag 'loongarch-6.5' of git://git.kernel.org/pub/scm/linux/kernel/git/chenhuacai/linux-loongson: (28 commits)
  LoongArch: Remove five DIE_* definitions in kdebug.h
  LoongArch: Add uprobes support
  LoongArch: Use larch_insn_gen_break() for kprobes
  LoongArch: Add larch_insn_gen_break() to generate break insns
  LoongArch: Check for AMO instructions in insns_not_supported()
  LoongArch: Move three functions from kprobes.c to inst.c
  LoongArch: Replace kretprobe with rethook
  LoongArch: Add jump-label implementation
  LoongArch: Select HAVE_DEBUG_KMEMLEAK to support kmemleak
  LoongArch: Export some arch-specific pm interfaces
  LoongArch: Introduce hardware page table walker
  LoongArch: Support dbar with different hints
  LoongArch: Add SMT (Simultaneous Multi-Threading) support
  LoongArch: Add vector extensions support
  LoongArch: Add support to clone a time namespace
  Makefile: Add loongarch target flag for Clang compilation
  LoongArch: Mark Clang LTO as working
  LoongArch: Include KBUILD_CPPFLAGS in CHECKFLAGS invocation
  LoongArch: vDSO: Use CLANG_FLAGS instead of filtering out '--target='
  LoongArch: Tweak CFLAGS for Clang compatibility
  ...
parents e55e5df1 5ee35c76
......@@ -13,7 +13,7 @@
| csky: | ok |
| hexagon: | TODO |
| ia64: | TODO |
| loongarch: | TODO |
| loongarch: | ok |
| m68k: | TODO |
| microblaze: | TODO |
| mips: | ok |
......
......@@ -13,7 +13,7 @@
| csky: | ok |
| hexagon: | TODO |
| ia64: | TODO |
| loongarch: | TODO |
| loongarch: | ok |
| m68k: | TODO |
| microblaze: | ok |
| mips: | ok |
......
......@@ -5,6 +5,7 @@ config LOONGARCH
select ACPI
select ACPI_GENERIC_GSI if ACPI
select ACPI_MCFG if ACPI
select ACPI_PPTT if ACPI
select ACPI_SYSTEM_POWER_STATES_SUPPORT if ACPI
select ARCH_BINFMT_ELF_STATE
select ARCH_ENABLE_MEMORY_HOTPLUG
......@@ -49,6 +50,8 @@ config LOONGARCH
select ARCH_SUPPORTS_ACPI
select ARCH_SUPPORTS_ATOMIC_RMW
select ARCH_SUPPORTS_HUGETLBFS
select ARCH_SUPPORTS_LTO_CLANG
select ARCH_SUPPORTS_LTO_CLANG_THIN
select ARCH_SUPPORTS_NUMA_BALANCING
select ARCH_USE_BUILTIN_BSWAP
select ARCH_USE_CMPXCHG_LOCKREF
......@@ -81,9 +84,12 @@ config LOONGARCH
select GENERIC_SCHED_CLOCK
select GENERIC_SMP_IDLE_THREAD
select GENERIC_TIME_VSYSCALL
select GENERIC_VDSO_TIME_NS
select GPIOLIB
select HAS_IOPORT
select HAVE_ARCH_AUDITSYSCALL
select HAVE_ARCH_JUMP_LABEL
select HAVE_ARCH_JUMP_LABEL_RELATIVE
select HAVE_ARCH_MMAP_RND_BITS if MMU
select HAVE_ARCH_SECCOMP_FILTER
select HAVE_ARCH_TRACEHOOK
......@@ -91,6 +97,7 @@ config LOONGARCH
select HAVE_ASM_MODVERSIONS
select HAVE_CONTEXT_TRACKING_USER
select HAVE_C_RECORDMCOUNT
select HAVE_DEBUG_KMEMLEAK
select HAVE_DEBUG_STACKOVERFLOW
select HAVE_DMA_CONTIGUOUS
select HAVE_DYNAMIC_FTRACE
......@@ -121,6 +128,7 @@ config LOONGARCH
select HAVE_PERF_REGS
select HAVE_PERF_USER_STACK_DUMP
select HAVE_REGS_AND_STACK_ACCESS_API
select HAVE_RETHOOK
select HAVE_RSEQ
select HAVE_SAMPLE_FTRACE_DIRECT
select HAVE_SAMPLE_FTRACE_DIRECT_MULTI
......@@ -163,14 +171,6 @@ config 32BIT
config 64BIT
def_bool y
config CPU_HAS_FPU
bool
default y
config CPU_HAS_PREFETCH
bool
default y
config GENERIC_BUG
def_bool y
depends on BUG
......@@ -243,6 +243,15 @@ config SCHED_OMIT_FRAME_POINTER
config AS_HAS_EXPLICIT_RELOCS
def_bool $(as-instr,x:pcalau12i \$t0$(comma)%pc_hi20(x))
config AS_HAS_FCSR_CLASS
def_bool $(as-instr,movfcsr2gr \$t0$(comma)\$fcsr0)
config AS_HAS_LSX_EXTENSION
def_bool $(as-instr,vld \$vr0$(comma)\$a0$(comma)0)
config AS_HAS_LASX_EXTENSION
def_bool $(as-instr,xvld \$xr0$(comma)\$a0$(comma)0)
menu "Kernel type and options"
source "kernel/Kconfig.hz"
......@@ -374,6 +383,13 @@ config EFI_STUB
This kernel feature allows the kernel to be loaded directly by
EFI firmware without the use of a bootloader.
config SCHED_SMT
bool "SMT scheduler support"
default y
help
Improves scheduler's performance when there are multiple
threads in one physical core.
config SMP
bool "Multi-Processing support"
help
......@@ -483,6 +499,43 @@ config ARCH_STRICT_ALIGN
to run kernel only on systems with h/w unaligned access support in
order to optimise for performance.
config CPU_HAS_FPU
bool
default y
config CPU_HAS_LSX
bool "Support for the Loongson SIMD Extension"
depends on AS_HAS_LSX_EXTENSION
help
Loongson SIMD Extension (LSX) introduces 128 bit wide vector registers
and a set of SIMD instructions to operate on them. When this option
is enabled the kernel will support allocating & switching LSX
vector register contexts. If you know that your kernel will only be
running on CPUs which do not support LSX or that your userland will
not be making use of it then you may wish to say N here to reduce
the size & complexity of your kernel.
If unsure, say Y.
config CPU_HAS_LASX
bool "Support for the Loongson Advanced SIMD Extension"
depends on CPU_HAS_LSX
depends on AS_HAS_LASX_EXTENSION
help
Loongson Advanced SIMD Extension (LASX) introduces 256 bit wide vector
registers and a set of SIMD instructions to operate on them. When this
option is enabled the kernel will support allocating & switching LASX
vector register contexts. If you know that your kernel will only be
running on CPUs which do not support LASX or that your userland will
not be making use of it then you may wish to say N here to reduce
the size & complexity of your kernel.
If unsure, say Y.
config CPU_HAS_PREFETCH
bool
default y
config KEXEC
bool "Kexec system call"
select KEXEC_CORE
......@@ -592,6 +645,9 @@ config ARCH_MMAP_RND_BITS_MIN
config ARCH_MMAP_RND_BITS_MAX
default 18
config ARCH_SUPPORTS_UPROBES
def_bool y
menu "Power management options"
config ARCH_SUSPEND_POSSIBLE
......
......@@ -46,8 +46,8 @@ ld-emul = $(64bit-emul)
cflags-y += -mabi=lp64s
endif
cflags-y += -G0 -pipe -msoft-float
LDFLAGS_vmlinux += -G0 -static -n -nostdlib
cflags-y += -pipe -msoft-float
LDFLAGS_vmlinux += -static -n -nostdlib
# When the assembler supports explicit relocation hint, we must use it.
# GCC may have -mexplicit-relocs off by default if it was built with an old
......@@ -56,13 +56,18 @@ LDFLAGS_vmlinux += -G0 -static -n -nostdlib
# When the assembler does not supports explicit relocation hint, we can't use
# it. Disable it if the compiler supports it.
#
# If you've seen "unknown reloc hint" message building the kernel and you are
# now wondering why "-mexplicit-relocs" is not wrapped with cc-option: the
# combination of a "new" assembler and "old" compiler is not supported. Either
# upgrade the compiler or downgrade the assembler.
# The combination of a "new" assembler and "old" GCC is not supported, given
# the rarity of this combo and the extra complexity needed to make it work.
# Either upgrade the compiler or downgrade the assembler; the build will error
# out if it is the case (by probing for the model attribute; all supported
# compilers in this case would have support).
#
# Also, -mdirect-extern-access is useful in case of building with explicit
# relocs, for avoiding unnecessary GOT accesses. It is harmless to not have
# support though.
ifdef CONFIG_AS_HAS_EXPLICIT_RELOCS
cflags-y += -mexplicit-relocs
KBUILD_CFLAGS_KERNEL += -mdirect-extern-access
cflags-y += $(call cc-option,-mexplicit-relocs)
KBUILD_CFLAGS_KERNEL += $(call cc-option,-mdirect-extern-access)
else
cflags-y += $(call cc-option,-mno-explicit-relocs)
KBUILD_AFLAGS_KERNEL += -Wa,-mla-global-with-pcrel
......@@ -107,7 +112,7 @@ KBUILD_CFLAGS += -isystem $(shell $(CC) -print-file-name=include)
KBUILD_LDFLAGS += -m $(ld-emul)
ifdef CONFIG_LOONGARCH
CHECKFLAGS += $(shell $(CC) $(KBUILD_CFLAGS) -dM -E -x c /dev/null | \
CHECKFLAGS += $(shell $(CC) $(KBUILD_CPPFLAGS) $(KBUILD_CFLAGS) -dM -E -x c /dev/null | \
grep -E -vw '__GNUC_(MINOR_|PATCHLEVEL_)?_' | \
sed -e "s/^\#define /-D'/" -e "s/ /'='/" -e "s/$$/'/" -e 's/\$$/&&/g')
endif
......
......@@ -5,7 +5,6 @@ generic-y += mcs_spinlock.h
generic-y += parport.h
generic-y += early_ioremap.h
generic-y += qrwlock.h
generic-y += qspinlock.h
generic-y += rwsem.h
generic-y += segment.h
generic-y += user.h
......
......@@ -8,11 +8,14 @@
#ifndef _ASM_LOONGARCH_ACPI_H
#define _ASM_LOONGARCH_ACPI_H
#include <asm/suspend.h>
#ifdef CONFIG_ACPI
extern int acpi_strict;
extern int acpi_disabled;
extern int acpi_pci_disabled;
extern int acpi_noirq;
extern int pptt_enabled;
#define acpi_os_ioremap acpi_os_ioremap
void __iomem *acpi_os_ioremap(acpi_physical_address phys, acpi_size size);
......@@ -30,6 +33,14 @@ static inline bool acpi_has_cpu_in_madt(void)
}
extern struct list_head acpi_wakeup_device_list;
extern struct acpi_madt_core_pic acpi_core_pic[NR_CPUS];
extern int __init parse_acpi_topology(void);
static inline u32 get_acpi_id_for_cpu(unsigned int cpu)
{
return acpi_core_pic[cpu_logical_map(cpu)].processor_id;
}
#endif /* !CONFIG_ACPI */
......@@ -37,12 +48,10 @@ extern struct list_head acpi_wakeup_device_list;
extern int loongarch_acpi_suspend(void);
extern int (*acpi_suspend_lowlevel)(void);
extern void loongarch_suspend_enter(void);
static inline unsigned long acpi_get_wakeup_address(void)
{
#ifdef CONFIG_SUSPEND
extern void loongarch_wakeup_start(void);
return (unsigned long)loongarch_wakeup_start;
#endif
return 0UL;
......
This diff is collapsed.
......@@ -5,27 +5,56 @@
#ifndef __ASM_BARRIER_H
#define __ASM_BARRIER_H
#define __sync() __asm__ __volatile__("dbar 0" : : : "memory")
/*
* Hint encoding:
*
* Bit4: ordering or completion (0: completion, 1: ordering)
* Bit3: barrier for previous read (0: true, 1: false)
* Bit2: barrier for previous write (0: true, 1: false)
* Bit1: barrier for succeeding read (0: true, 1: false)
* Bit0: barrier for succeeding write (0: true, 1: false)
*
* Hint 0x700: barrier for "read after read" from the same address
*/
#define DBAR(hint) __asm__ __volatile__("dbar %0 " : : "I"(hint) : "memory")
#define crwrw 0b00000
#define cr_r_ 0b00101
#define c_w_w 0b01010
#define fast_wmb() __sync()
#define fast_rmb() __sync()
#define fast_mb() __sync()
#define fast_iob() __sync()
#define wbflush() __sync()
#define orwrw 0b10000
#define or_r_ 0b10101
#define o_w_w 0b11010
#define wmb() fast_wmb()
#define rmb() fast_rmb()
#define mb() fast_mb()
#define iob() fast_iob()
#define orw_w 0b10010
#define or_rw 0b10100
#define __smp_mb() __asm__ __volatile__("dbar 0" : : : "memory")
#define __smp_rmb() __asm__ __volatile__("dbar 0" : : : "memory")
#define __smp_wmb() __asm__ __volatile__("dbar 0" : : : "memory")
#define c_sync() DBAR(crwrw)
#define c_rsync() DBAR(cr_r_)
#define c_wsync() DBAR(c_w_w)
#define o_sync() DBAR(orwrw)
#define o_rsync() DBAR(or_r_)
#define o_wsync() DBAR(o_w_w)
#define ldacq_mb() DBAR(or_rw)
#define strel_mb() DBAR(orw_w)
#define mb() c_sync()
#define rmb() c_rsync()
#define wmb() c_wsync()
#define iob() c_sync()
#define wbflush() c_sync()
#define __smp_mb() o_sync()
#define __smp_rmb() o_rsync()
#define __smp_wmb() o_wsync()
#ifdef CONFIG_SMP
#define __WEAK_LLSC_MB " dbar 0 \n"
#define __WEAK_LLSC_MB " dbar 0x700 \n"
#else
#define __WEAK_LLSC_MB " \n"
#define __WEAK_LLSC_MB " \n"
#endif
#define __smp_mb__before_atomic() barrier()
......@@ -59,68 +88,19 @@ static inline unsigned long array_index_mask_nospec(unsigned long index,
return mask;
}
#define __smp_load_acquire(p) \
({ \
union { typeof(*p) __val; char __c[1]; } __u; \
unsigned long __tmp = 0; \
compiletime_assert_atomic_type(*p); \
switch (sizeof(*p)) { \
case 1: \
*(__u8 *)__u.__c = *(volatile __u8 *)p; \
__smp_mb(); \
break; \
case 2: \
*(__u16 *)__u.__c = *(volatile __u16 *)p; \
__smp_mb(); \
break; \
case 4: \
__asm__ __volatile__( \
"amor_db.w %[val], %[tmp], %[mem] \n" \
: [val] "=&r" (*(__u32 *)__u.__c) \
: [mem] "ZB" (*(u32 *) p), [tmp] "r" (__tmp) \
: "memory"); \
break; \
case 8: \
__asm__ __volatile__( \
"amor_db.d %[val], %[tmp], %[mem] \n" \
: [val] "=&r" (*(__u64 *)__u.__c) \
: [mem] "ZB" (*(u64 *) p), [tmp] "r" (__tmp) \
: "memory"); \
break; \
} \
(typeof(*p))__u.__val; \
#define __smp_load_acquire(p) \
({ \
typeof(*p) ___p1 = READ_ONCE(*p); \
compiletime_assert_atomic_type(*p); \
ldacq_mb(); \
___p1; \
})
#define __smp_store_release(p, v) \
do { \
union { typeof(*p) __val; char __c[1]; } __u = \
{ .__val = (__force typeof(*p)) (v) }; \
unsigned long __tmp; \
compiletime_assert_atomic_type(*p); \
switch (sizeof(*p)) { \
case 1: \
__smp_mb(); \
*(volatile __u8 *)p = *(__u8 *)__u.__c; \
break; \
case 2: \
__smp_mb(); \
*(volatile __u16 *)p = *(__u16 *)__u.__c; \
break; \
case 4: \
__asm__ __volatile__( \
"amswap_db.w %[tmp], %[val], %[mem] \n" \
: [mem] "+ZB" (*(u32 *)p), [tmp] "=&r" (__tmp) \
: [val] "r" (*(__u32 *)__u.__c) \
: ); \
break; \
case 8: \
__asm__ __volatile__( \
"amswap_db.d %[tmp], %[val], %[mem] \n" \
: [mem] "+ZB" (*(u64 *)p), [tmp] "=&r" (__tmp) \
: [val] "r" (*(__u64 *)__u.__c) \
: ); \
break; \
} \
#define __smp_store_release(p, v) \
do { \
compiletime_assert_atomic_type(*p); \
strel_mb(); \
WRITE_ONCE(*p, v); \
} while (0)
#define __smp_store_mb(p, v) \
......
......@@ -64,6 +64,6 @@
#define cpu_has_eiodecode cpu_opt(LOONGARCH_CPU_EIODECODE)
#define cpu_has_guestid cpu_opt(LOONGARCH_CPU_GUESTID)
#define cpu_has_hypervisor cpu_opt(LOONGARCH_CPU_HYPERVISOR)
#define cpu_has_ptw cpu_opt(LOONGARCH_CPU_PTW)
#endif /* __ASM_CPU_FEATURES_H */
......@@ -54,6 +54,7 @@ struct cpuinfo_loongarch {
struct cache_desc cache_leaves[CACHE_LEAVES_MAX];
int core; /* physical core number in package */
int package;/* physical package number */
int global_id; /* physical global thread number */
int vabits; /* Virtual Address size in bits */
int pabits; /* Physical Address size in bits */
unsigned int ksave_mask; /* Usable KSave mask. */
......
......@@ -98,6 +98,7 @@ enum cpu_type_enum {
#define CPU_FEATURE_EIODECODE 23 /* CPU has EXTIOI interrupt pin decode mode */
#define CPU_FEATURE_GUESTID 24 /* CPU has GuestID feature */
#define CPU_FEATURE_HYPERVISOR 25 /* CPU has hypervisor (running in VM) */
#define CPU_FEATURE_PTW 26 /* CPU has hardware page table walker */
#define LOONGARCH_CPU_CPUCFG BIT_ULL(CPU_FEATURE_CPUCFG)
#define LOONGARCH_CPU_LAM BIT_ULL(CPU_FEATURE_LAM)
......@@ -125,5 +126,6 @@ enum cpu_type_enum {
#define LOONGARCH_CPU_EIODECODE BIT_ULL(CPU_FEATURE_EIODECODE)
#define LOONGARCH_CPU_GUESTID BIT_ULL(CPU_FEATURE_GUESTID)
#define LOONGARCH_CPU_HYPERVISOR BIT_ULL(CPU_FEATURE_HYPERVISOR)
#define LOONGARCH_CPU_PTW BIT_ULL(CPU_FEATURE_PTW)
#endif /* _ASM_CPU_H */
......@@ -40,6 +40,7 @@
#define fs6 $f30
#define fs7 $f31
#ifndef CONFIG_AS_HAS_FCSR_CLASS
/*
* Current binutils expects *GPRs* at FCSR position for the FCSR
* operation instructions, so define aliases for those used.
......@@ -48,5 +49,11 @@
#define fcsr1 $r1
#define fcsr2 $r2
#define fcsr3 $r3
#else
#define fcsr0 $fcsr0
#define fcsr1 $fcsr1
#define fcsr2 $fcsr2
#define fcsr3 $fcsr3
#endif
#endif /* _ASM_FPREGDEF_H */
......@@ -28,6 +28,26 @@ extern void _init_fpu(unsigned int);
extern void _save_fp(struct loongarch_fpu *);
extern void _restore_fp(struct loongarch_fpu *);
extern void _save_lsx(struct loongarch_fpu *fpu);
extern void _restore_lsx(struct loongarch_fpu *fpu);
extern void _init_lsx_upper(void);
extern void _restore_lsx_upper(struct loongarch_fpu *fpu);
extern void _save_lasx(struct loongarch_fpu *fpu);
extern void _restore_lasx(struct loongarch_fpu *fpu);
extern void _init_lasx_upper(void);
extern void _restore_lasx_upper(struct loongarch_fpu *fpu);
static inline void enable_lsx(void);
static inline void disable_lsx(void);
static inline void save_lsx(struct task_struct *t);
static inline void restore_lsx(struct task_struct *t);
static inline void enable_lasx(void);
static inline void disable_lasx(void);
static inline void save_lasx(struct task_struct *t);
static inline void restore_lasx(struct task_struct *t);
/*
* Mask the FCSR Cause bits according to the Enable bits, observing
* that Unimplemented is always enabled.
......@@ -44,6 +64,29 @@ static inline int is_fp_enabled(void)
1 : 0;
}
static inline int is_lsx_enabled(void)
{
if (!cpu_has_lsx)
return 0;
return (csr_read32(LOONGARCH_CSR_EUEN) & CSR_EUEN_LSXEN) ?
1 : 0;
}
static inline int is_lasx_enabled(void)
{
if (!cpu_has_lasx)
return 0;
return (csr_read32(LOONGARCH_CSR_EUEN) & CSR_EUEN_LASXEN) ?
1 : 0;
}
static inline int is_simd_enabled(void)
{
return is_lsx_enabled() | is_lasx_enabled();
}
#define enable_fpu() set_csr_euen(CSR_EUEN_FPEN)
#define disable_fpu() clear_csr_euen(CSR_EUEN_FPEN)
......@@ -81,9 +124,22 @@ static inline void own_fpu(int restore)
static inline void lose_fpu_inatomic(int save, struct task_struct *tsk)
{
if (is_fpu_owner()) {
if (save)
_save_fp(&tsk->thread.fpu);
disable_fpu();
if (!is_simd_enabled()) {
if (save)
_save_fp(&tsk->thread.fpu);
disable_fpu();
} else {
if (save) {
if (!is_lasx_enabled())
save_lsx(tsk);
else
save_lasx(tsk);
}
disable_fpu();
disable_lsx();
disable_lasx();
clear_tsk_thread_flag(tsk, TIF_USEDSIMD);
}
clear_tsk_thread_flag(tsk, TIF_USEDFPU);
}
KSTK_EUEN(tsk) &= ~(CSR_EUEN_FPEN | CSR_EUEN_LSXEN | CSR_EUEN_LASXEN);
......@@ -129,4 +185,127 @@ static inline union fpureg *get_fpu_regs(struct task_struct *tsk)
return tsk->thread.fpu.fpr;
}
static inline int is_simd_owner(void)
{
return test_thread_flag(TIF_USEDSIMD);
}
#ifdef CONFIG_CPU_HAS_LSX
static inline void enable_lsx(void)
{
if (cpu_has_lsx)
csr_xchg32(CSR_EUEN_LSXEN, CSR_EUEN_LSXEN, LOONGARCH_CSR_EUEN);
}
static inline void disable_lsx(void)
{
if (cpu_has_lsx)
csr_xchg32(0, CSR_EUEN_LSXEN, LOONGARCH_CSR_EUEN);
}
static inline void save_lsx(struct task_struct *t)
{
if (cpu_has_lsx)
_save_lsx(&t->thread.fpu);
}
static inline void restore_lsx(struct task_struct *t)
{
if (cpu_has_lsx)
_restore_lsx(&t->thread.fpu);
}
static inline void init_lsx_upper(void)
{
/*
* Check cpu_has_lsx only if it's a constant. This will allow the
* compiler to optimise out code for CPUs without LSX without adding
* an extra redundant check for CPUs with LSX.
*/
if (__builtin_constant_p(cpu_has_lsx) && !cpu_has_lsx)
return;
_init_lsx_upper();
}
static inline void restore_lsx_upper(struct task_struct *t)
{
if (cpu_has_lsx)
_restore_lsx_upper(&t->thread.fpu);
}
#else
static inline void enable_lsx(void) {}
static inline void disable_lsx(void) {}
static inline void save_lsx(struct task_struct *t) {}
static inline void restore_lsx(struct task_struct *t) {}
static inline void init_lsx_upper(void) {}
static inline void restore_lsx_upper(struct task_struct *t) {}
#endif
#ifdef CONFIG_CPU_HAS_LASX
static inline void enable_lasx(void)
{
if (cpu_has_lasx)
csr_xchg32(CSR_EUEN_LASXEN, CSR_EUEN_LASXEN, LOONGARCH_CSR_EUEN);
}
static inline void disable_lasx(void)
{
if (cpu_has_lasx)
csr_xchg32(0, CSR_EUEN_LASXEN, LOONGARCH_CSR_EUEN);
}
static inline void save_lasx(struct task_struct *t)
{
if (cpu_has_lasx)
_save_lasx(&t->thread.fpu);
}
static inline void restore_lasx(struct task_struct *t)
{
if (cpu_has_lasx)
_restore_lasx(&t->thread.fpu);
}
static inline void init_lasx_upper(void)
{
if (cpu_has_lasx)
_init_lasx_upper();
}
static inline void restore_lasx_upper(struct task_struct *t)
{
if (cpu_has_lasx)
_restore_lasx_upper(&t->thread.fpu);
}
#else
static inline void enable_lasx(void) {}
static inline void disable_lasx(void) {}
static inline void save_lasx(struct task_struct *t) {}
static inline void restore_lasx(struct task_struct *t) {}
static inline void init_lasx_upper(void) {}
static inline void restore_lasx_upper(struct task_struct *t) {}
#endif
static inline int thread_lsx_context_live(void)
{
if (__builtin_constant_p(cpu_has_lsx) && !cpu_has_lsx)
return 0;
return test_thread_flag(TIF_LSX_CTX_LIVE);
}
static inline int thread_lasx_context_live(void)
{
if (__builtin_constant_p(cpu_has_lasx) && !cpu_has_lasx)
return 0;
return test_thread_flag(TIF_LASX_CTX_LIVE);
}
#endif /* _ASM_FPU_H */
......@@ -9,6 +9,22 @@
.equ .L__gpr_num_$r\num, \num
.endr
/* ABI names of registers */
.equ .L__gpr_num_$ra, 1
.equ .L__gpr_num_$tp, 2
.equ .L__gpr_num_$sp, 3
.irp num,0,1,2,3,4,5,6,7
.equ .L__gpr_num_$a\num, 4 + \num
.endr
.irp num,0,1,2,3,4,5,6,7,8
.equ .L__gpr_num_$t\num, 12 + \num
.endr
.equ .L__gpr_num_$s9, 22
.equ .L__gpr_num_$fp, 22
.irp num,0,1,2,3,4,5,6,7,8
.equ .L__gpr_num_$s\num, 23 + \num
.endr
#else /* __ASSEMBLY__ */
#define __DEFINE_ASM_GPR_NUMS \
......@@ -16,6 +32,20 @@
" .irp num,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31\n" \
" .equ .L__gpr_num_$r\\num, \\num\n" \
" .endr\n" \
" .equ .L__gpr_num_$ra, 1\n" \
" .equ .L__gpr_num_$tp, 2\n" \
" .equ .L__gpr_num_$sp, 3\n" \
" .irp num,0,1,2,3,4,5,6,7\n" \
" .equ .L__gpr_num_$a\\num, 4 + \\num\n" \
" .endr\n" \
" .irp num,0,1,2,3,4,5,6,7,8\n" \
" .equ .L__gpr_num_$t\\num, 12 + \\num\n" \
" .endr\n" \
" .equ .L__gpr_num_$s9, 22\n" \
" .equ .L__gpr_num_$fp, 22\n" \
" .irp num,0,1,2,3,4,5,6,7,8\n" \
" .equ .L__gpr_num_$s\\num, 23 + \\num\n" \
" .endr\n" \
#endif /* __ASSEMBLY__ */
......
......@@ -5,6 +5,7 @@
#ifndef _ASM_INST_H
#define _ASM_INST_H
#include <linux/bitops.h>
#include <linux/types.h>
#include <asm/asm.h>
#include <asm/ptrace.h>
......@@ -15,14 +16,22 @@
#define ADDR_IMMMASK_LU52ID 0xFFF0000000000000
#define ADDR_IMMMASK_LU32ID 0x000FFFFF00000000
#define ADDR_IMMMASK_LU12IW 0x00000000FFFFF000
#define ADDR_IMMMASK_ORI 0x0000000000000FFF
#define ADDR_IMMMASK_ADDU16ID 0x00000000FFFF0000
#define ADDR_IMMSHIFT_LU52ID 52
#define ADDR_IMMSBIDX_LU52ID 11
#define ADDR_IMMSHIFT_LU32ID 32
#define ADDR_IMMSBIDX_LU32ID 19
#define ADDR_IMMSHIFT_LU12IW 12
#define ADDR_IMMSBIDX_LU12IW 19
#define ADDR_IMMSHIFT_ORI 0
#define ADDR_IMMSBIDX_ORI 63
#define ADDR_IMMSHIFT_ADDU16ID 16
#define ADDR_IMMSBIDX_ADDU16ID 15
#define ADDR_IMM(addr, INSN) ((addr & ADDR_IMMMASK_##INSN) >> ADDR_IMMSHIFT_##INSN)
#define ADDR_IMM(addr, INSN) \
(sign_extend64(((addr & ADDR_IMMMASK_##INSN) >> ADDR_IMMSHIFT_##INSN), ADDR_IMMSBIDX_##INSN))
enum reg0i15_op {
break_op = 0x54,
......@@ -178,6 +187,32 @@ enum reg3_op {
amord_op = 0x70c7,
amxorw_op = 0x70c8,
amxord_op = 0x70c9,
ammaxw_op = 0x70ca,
ammaxd_op = 0x70cb,
amminw_op = 0x70cc,
ammind_op = 0x70cd,
ammaxwu_op = 0x70ce,
ammaxdu_op = 0x70cf,
amminwu_op = 0x70d0,
ammindu_op = 0x70d1,
amswapdbw_op = 0x70d2,
amswapdbd_op = 0x70d3,
amadddbw_op = 0x70d4,
amadddbd_op = 0x70d5,
amanddbw_op = 0x70d6,
amanddbd_op = 0x70d7,
amordbw_op = 0x70d8,
amordbd_op = 0x70d9,
amxordbw_op = 0x70da,
amxordbd_op = 0x70db,
ammaxdbw_op = 0x70dc,
ammaxdbd_op = 0x70dd,
ammindbw_op = 0x70de,
ammindbd_op = 0x70df,
ammaxdbwu_op = 0x70e0,
ammaxdbdu_op = 0x70e1,
ammindbwu_op = 0x70e2,
ammindbdu_op = 0x70e3,
fldgts_op = 0x70e8,
fldgtd_op = 0x70e9,
fldles_op = 0x70ea,
......@@ -435,6 +470,10 @@ static inline bool is_self_loop_ins(union loongarch_instruction *ip, struct pt_r
void simu_pc(struct pt_regs *regs, union loongarch_instruction insn);
void simu_branch(struct pt_regs *regs, union loongarch_instruction insn);
bool insns_not_supported(union loongarch_instruction insn);
bool insns_need_simulation(union loongarch_instruction insn);
void arch_simulate_insn(union loongarch_instruction insn, struct pt_regs *regs);
int larch_insn_read(void *addr, u32 *insnp);
int larch_insn_write(void *addr, u32 insn);
int larch_insn_patch_text(void *addr, u32 insn);
......@@ -443,13 +482,15 @@ u32 larch_insn_gen_nop(void);
u32 larch_insn_gen_b(unsigned long pc, unsigned long dest);
u32 larch_insn_gen_bl(unsigned long pc, unsigned long dest);
u32 larch_insn_gen_break(int imm);
u32 larch_insn_gen_or(enum loongarch_gpr rd, enum loongarch_gpr rj, enum loongarch_gpr rk);
u32 larch_insn_gen_move(enum loongarch_gpr rd, enum loongarch_gpr rj);
u32 larch_insn_gen_lu12iw(enum loongarch_gpr rd, int imm);
u32 larch_insn_gen_lu32id(enum loongarch_gpr rd, int imm);
u32 larch_insn_gen_lu52id(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm);
u32 larch_insn_gen_jirl(enum loongarch_gpr rd, enum loongarch_gpr rj, unsigned long pc, unsigned long dest);
u32 larch_insn_gen_jirl(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm);
static inline bool signed_imm_check(long val, unsigned int bit)
{
......@@ -461,6 +502,16 @@ static inline bool unsigned_imm_check(unsigned long val, unsigned int bit)
return val < (1UL << bit);
}
#define DEF_EMIT_REG0I15_FORMAT(NAME, OP) \
static inline void emit_##NAME(union loongarch_instruction *insn, \
int imm) \
{ \
insn->reg0i15_format.opcode = OP; \
insn->reg0i15_format.immediate = imm; \
}
DEF_EMIT_REG0I15_FORMAT(break, break_op)
#define DEF_EMIT_REG0I26_FORMAT(NAME, OP) \
static inline void emit_##NAME(union loongarch_instruction *insn, \
int offset) \
......
......@@ -62,7 +62,7 @@ extern pgprot_t pgprot_wc;
#define ioremap_cache(offset, size) \
ioremap_prot((offset), (size), pgprot_val(PAGE_KERNEL))
#define mmiowb() asm volatile ("dbar 0" ::: "memory")
#define mmiowb() wmb()
/*
* String version of I/O memory access operations.
......
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2023 Loongson Technology Corporation Limited
*
* Based on arch/arm64/include/asm/jump_label.h
*/
#ifndef __ASM_JUMP_LABEL_H
#define __ASM_JUMP_LABEL_H
#ifndef __ASSEMBLY__
#include <linux/types.h>
#define JUMP_LABEL_NOP_SIZE 4
#define JUMP_TABLE_ENTRY \
".pushsection __jump_table, \"aw\" \n\t" \
".align 3 \n\t" \
".long 1b - ., %l[l_yes] - . \n\t" \
".quad %0 - . \n\t" \
".popsection \n\t"
static __always_inline bool arch_static_branch(struct static_key * const key, const bool branch)
{
asm_volatile_goto(
"1: nop \n\t"
JUMP_TABLE_ENTRY
: : "i"(&((char *)key)[branch]) : : l_yes);
return false;
l_yes:
return true;
}
static __always_inline bool arch_static_branch_jump(struct static_key * const key, const bool branch)
{
asm_volatile_goto(
"1: b %l[l_yes] \n\t"
JUMP_TABLE_ENTRY
: : "i"(&((char *)key)[branch]) : : l_yes);
return false;
l_yes:
return true;
}
#endif /* __ASSEMBLY__ */
#endif /* __ASM_JUMP_LABEL_H */
......@@ -13,11 +13,6 @@ enum die_val {
DIE_FP,
DIE_SIMD,
DIE_TRAP,
DIE_PAGE_FAULT,
DIE_BREAK,
DIE_SSTEPBP,
DIE_UPROBE,
DIE_UPROBE_XOL,
};
#endif /* _ASM_LOONGARCH_KDEBUG_H */
......@@ -22,7 +22,7 @@ do { \
#define kretprobe_blacklist_size 0
typedef union loongarch_instruction kprobe_opcode_t;
typedef u32 kprobe_opcode_t;
/* Architecture specific copy of original instruction */
struct arch_specific_insn {
......@@ -49,9 +49,6 @@ bool kprobe_fault_handler(struct pt_regs *regs, int trapnr);
bool kprobe_breakpoint_handler(struct pt_regs *regs);
bool kprobe_singlestep_handler(struct pt_regs *regs);
void __kretprobe_trampoline(void);
void *trampoline_probe_handler(struct pt_regs *regs);
#else /* !CONFIG_KPROBES */
static inline bool kprobe_breakpoint_handler(struct pt_regs *regs) { return false; }
......
......@@ -56,10 +56,7 @@ __asm__(".macro parse_r var r\n\t"
#undef _IFC_REG
/* CPUCFG */
static inline u32 read_cpucfg(u32 reg)
{
return __cpucfg(reg);
}
#define read_cpucfg(reg) __cpucfg(reg)
#endif /* !__ASSEMBLY__ */
......@@ -138,6 +135,7 @@ static inline u32 read_cpucfg(u32 reg)
#define CPUCFG2_MIPSBT BIT(20)
#define CPUCFG2_LSPW BIT(21)
#define CPUCFG2_LAM BIT(22)
#define CPUCFG2_PTW BIT(24)
#define LOONGARCH_CPUCFG3 0x3
#define CPUCFG3_CCDMA BIT(0)
......@@ -206,56 +204,18 @@ static inline u32 read_cpucfg(u32 reg)
#ifndef __ASSEMBLY__
/* CSR */
static __always_inline u32 csr_read32(u32 reg)
{
return __csrrd_w(reg);
}
static __always_inline u64 csr_read64(u32 reg)
{
return __csrrd_d(reg);
}
static __always_inline void csr_write32(u32 val, u32 reg)
{
__csrwr_w(val, reg);
}
static __always_inline void csr_write64(u64 val, u32 reg)
{
__csrwr_d(val, reg);
}
static __always_inline u32 csr_xchg32(u32 val, u32 mask, u32 reg)
{
return __csrxchg_w(val, mask, reg);
}
static __always_inline u64 csr_xchg64(u64 val, u64 mask, u32 reg)
{
return __csrxchg_d(val, mask, reg);
}
#define csr_read32(reg) __csrrd_w(reg)
#define csr_read64(reg) __csrrd_d(reg)
#define csr_write32(val, reg) __csrwr_w(val, reg)
#define csr_write64(val, reg) __csrwr_d(val, reg)
#define csr_xchg32(val, mask, reg) __csrxchg_w(val, mask, reg)
#define csr_xchg64(val, mask, reg) __csrxchg_d(val, mask, reg)
/* IOCSR */
static __always_inline u32 iocsr_read32(u32 reg)
{
return __iocsrrd_w(reg);
}
static __always_inline u64 iocsr_read64(u32 reg)
{
return __iocsrrd_d(reg);
}
static __always_inline void iocsr_write32(u32 val, u32 reg)
{
__iocsrwr_w(val, reg);
}
static __always_inline void iocsr_write64(u64 val, u32 reg)
{
__iocsrwr_d(val, reg);
}
#define iocsr_read32(reg) __iocsrrd_w(reg)
#define iocsr_read64(reg) __iocsrrd_d(reg)
#define iocsr_write32(val, reg) __iocsrwr_w(val, reg)
#define iocsr_write64(val, reg) __iocsrwr_d(val, reg)
#endif /* !__ASSEMBLY__ */
......@@ -453,6 +413,9 @@ static __always_inline void iocsr_write64(u64 val, u32 reg)
#define CSR_PWCTL0_PTBASE (_ULCAST_(0x1f) << CSR_PWCTL0_PTBASE_SHIFT)
#define LOONGARCH_CSR_PWCTL1 0x1d /* PWCtl1 */
#define CSR_PWCTL1_PTW_SHIFT 24
#define CSR_PWCTL1_PTW_WIDTH 1
#define CSR_PWCTL1_PTW (_ULCAST_(0x1) << CSR_PWCTL1_PTW_SHIFT)
#define CSR_PWCTL1_DIR3WIDTH_SHIFT 18
#define CSR_PWCTL1_DIR3WIDTH_WIDTH 5
#define CSR_PWCTL1_DIR3WIDTH (_ULCAST_(0x1f) << CSR_PWCTL1_DIR3WIDTH_SHIFT)
......@@ -1441,11 +1404,18 @@ __BUILD_CSR_OP(tlbidx)
#define EXCCODE_INT_START 64
#define EXCCODE_INT_END (EXCCODE_INT_START + EXCCODE_INT_NUM - 1)
/* FPU register names */
/* FPU Status Register Names */
#ifndef CONFIG_AS_HAS_FCSR_CLASS
#define LOONGARCH_FCSR0 $r0
#define LOONGARCH_FCSR1 $r1
#define LOONGARCH_FCSR2 $r2
#define LOONGARCH_FCSR3 $r3
#else
#define LOONGARCH_FCSR0 $fcsr0
#define LOONGARCH_FCSR1 $fcsr1
#define LOONGARCH_FCSR2 $fcsr2
#define LOONGARCH_FCSR3 $fcsr3
#endif
/* FPU Status Register Values */
#define FPU_CSR_RSVD 0xe0e0fce0
......
......@@ -55,7 +55,7 @@ static inline struct plt_entry emit_plt_entry(unsigned long val)
lu12iw = larch_insn_gen_lu12iw(LOONGARCH_GPR_T1, ADDR_IMM(val, LU12IW));
lu32id = larch_insn_gen_lu32id(LOONGARCH_GPR_T1, ADDR_IMM(val, LU32ID));
lu52id = larch_insn_gen_lu52id(LOONGARCH_GPR_T1, LOONGARCH_GPR_T1, ADDR_IMM(val, LU52ID));
jirl = larch_insn_gen_jirl(0, LOONGARCH_GPR_T1, 0, (val & 0xfff));
jirl = larch_insn_gen_jirl(0, LOONGARCH_GPR_T1, ADDR_IMM(val, ORI));
return (struct plt_entry) { lu12iw, lu32id, lu52id, jirl };
}
......
......@@ -81,6 +81,7 @@ typedef struct { unsigned long pgprot; } pgprot_t;
#define __va(x) ((void *)((unsigned long)(x) + PAGE_OFFSET - PHYS_OFFSET))
#define pfn_to_kaddr(pfn) __va((pfn) << PAGE_SHIFT)
#define sym_to_pfn(x) __phys_to_pfn(__pa_symbol(x))
#define virt_to_pfn(kaddr) PFN_DOWN(PHYSADDR(kaddr))
#define virt_to_page(kaddr) pfn_to_page(virt_to_pfn(kaddr))
......
......@@ -14,7 +14,11 @@
* loaded. Tell the compiler this fact when using explicit relocs.
*/
#if defined(MODULE) && defined(CONFIG_AS_HAS_EXPLICIT_RELOCS)
#define PER_CPU_ATTRIBUTES __attribute__((model("extreme")))
# if __has_attribute(model)
# define PER_CPU_ATTRIBUTES __attribute__((model("extreme")))
# else
# error compiler support for the model attribute is necessary when a recent assembler is used
# endif
#endif
/* Use r21 for fast access */
......
......@@ -362,7 +362,7 @@ extern pgd_t invalid_pg_dir[];
*/
static inline int pte_write(pte_t pte) { return pte_val(pte) & _PAGE_WRITE; }
static inline int pte_young(pte_t pte) { return pte_val(pte) & _PAGE_ACCESSED; }
static inline int pte_dirty(pte_t pte) { return pte_val(pte) & _PAGE_MODIFIED; }
static inline int pte_dirty(pte_t pte) { return pte_val(pte) & (_PAGE_DIRTY | _PAGE_MODIFIED); }
static inline pte_t pte_mkold(pte_t pte)
{
......@@ -506,7 +506,7 @@ static inline pmd_t pmd_wrprotect(pmd_t pmd)
static inline int pmd_dirty(pmd_t pmd)
{
return !!(pmd_val(pmd) & _PAGE_MODIFIED);
return !!(pmd_val(pmd) & (_PAGE_DIRTY | _PAGE_MODIFIED));
}
static inline pmd_t pmd_mkclean(pmd_t pmd)
......
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_QSPINLOCK_H
#define _ASM_QSPINLOCK_H
#include <asm-generic/qspinlock_types.h>
#define queued_spin_unlock queued_spin_unlock
static inline void queued_spin_unlock(struct qspinlock *lock)
{
compiletime_assert_atomic_type(lock->locked);
c_sync();
WRITE_ONCE(lock->locked, 0);
}
#include <asm-generic/qspinlock.h>
#endif /* _ASM_QSPINLOCK_H */
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __ASM_SUSPEND_H
#define __ASM_SUSPEND_H
void loongarch_common_suspend(void);
void loongarch_common_resume(void);
void loongarch_suspend_enter(void);
void loongarch_wakeup_start(void);
#endif
......@@ -88,52 +88,47 @@ enum invtlb_ops {
INVTLB_GID_ADDR = 0x16,
};
/*
* invtlb op info addr
* (0x1 << 26) | (0x24 << 20) | (0x13 << 15) |
* (addr << 10) | (info << 5) | op
*/
static inline void invtlb(u32 op, u32 info, u64 addr)
static __always_inline void invtlb(u32 op, u32 info, u64 addr)
{
__asm__ __volatile__(
"parse_r addr,%0\n\t"
"parse_r info,%1\n\t"
".word ((0x6498000) | (addr << 10) | (info << 5) | %2)\n\t"
:
: "r"(addr), "r"(info), "i"(op)
"invtlb %0, %1, %2\n\t"
:
: "i"(op), "r"(info), "r"(addr)
: "memory"
);
}
static inline void invtlb_addr(u32 op, u32 info, u64 addr)
static __always_inline void invtlb_addr(u32 op, u32 info, u64 addr)
{
BUILD_BUG_ON(!__builtin_constant_p(info) || info != 0);
__asm__ __volatile__(
"parse_r addr,%0\n\t"
".word ((0x6498000) | (addr << 10) | (0 << 5) | %1)\n\t"
:
: "r"(addr), "i"(op)
"invtlb %0, $zero, %1\n\t"
:
: "i"(op), "r"(addr)
: "memory"
);
}
static inline void invtlb_info(u32 op, u32 info, u64 addr)
static __always_inline void invtlb_info(u32 op, u32 info, u64 addr)
{
BUILD_BUG_ON(!__builtin_constant_p(addr) || addr != 0);
__asm__ __volatile__(
"parse_r info,%0\n\t"
".word ((0x6498000) | (0 << 10) | (info << 5) | %1)\n\t"
:
: "r"(info), "i"(op)
"invtlb %0, %1, $zero\n\t"
:
: "i"(op), "r"(info)
: "memory"
);
}
static inline void invtlb_all(u32 op, u32 info, u64 addr)
static __always_inline void invtlb_all(u32 op, u32 info, u64 addr)
{
BUILD_BUG_ON(!__builtin_constant_p(info) || info != 0);
BUILD_BUG_ON(!__builtin_constant_p(addr) || addr != 0);
__asm__ __volatile__(
".word ((0x6498000) | (0 << 10) | (0 << 5) | %0)\n\t"
"invtlb %0, $zero, $zero\n\t"
:
: "i"(op)
:
: "memory"
);
}
......@@ -163,6 +158,9 @@ extern void handle_tlb_store(void);
extern void handle_tlb_modify(void);
extern void handle_tlb_refill(void);
extern void handle_tlb_protect(void);
extern void handle_tlb_load_ptw(void);
extern void handle_tlb_store_ptw(void);
extern void handle_tlb_modify_ptw(void);
extern void dump_tlb_all(void);
extern void dump_tlb_regs(void);
......
/* SPDX-License-Identifier: GPL-2.0-only */
#ifndef __ASM_LOONGARCH_UPROBES_H
#define __ASM_LOONGARCH_UPROBES_H
#include <asm/inst.h>
typedef u32 uprobe_opcode_t;
#define MAX_UINSN_BYTES 8
#define UPROBE_XOL_SLOT_BYTES MAX_UINSN_BYTES
#define UPROBE_SWBP_INSN larch_insn_gen_break(BRK_UPROBE_BP)
#define UPROBE_SWBP_INSN_SIZE LOONGARCH_INSN_SIZE
#define UPROBE_XOLBP_INSN larch_insn_gen_break(BRK_UPROBE_XOLBP)
struct arch_uprobe {
unsigned long resume_era;
u32 insn[2];
u32 ixol[2];
bool simulate;
};
struct arch_uprobe_task {
unsigned long saved_trap_nr;
};
#ifdef CONFIG_UPROBES
bool uprobe_breakpoint_handler(struct pt_regs *regs);
bool uprobe_singlestep_handler(struct pt_regs *regs);
#else /* !CONFIG_UPROBES */
static inline bool uprobe_breakpoint_handler(struct pt_regs *regs) { return false; }
static inline bool uprobe_singlestep_handler(struct pt_regs *regs) { return false; }
#endif /* CONFIG_UPROBES */
#endif /* __ASM_LOONGARCH_UPROBES_H */
......@@ -91,9 +91,16 @@ static inline bool loongarch_vdso_hres_capable(void)
static __always_inline const struct vdso_data *__arch_get_vdso_data(void)
{
return get_vdso_data();
return (const struct vdso_data *)get_vdso_data();
}
#ifdef CONFIG_TIME_NS
static __always_inline
const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd)
{
return (const struct vdso_data *)(get_vdso_data() + VVAR_TIMENS_PAGE_OFFSET * PAGE_SIZE);
}
#endif
#endif /* !__ASSEMBLY__ */
#endif /* __ASM_VDSO_GETTIMEOFDAY_H */
......@@ -16,10 +16,33 @@ struct vdso_pcpu_data {
struct loongarch_vdso_data {
struct vdso_pcpu_data pdata[NR_CPUS];
struct vdso_data data[CS_BASES]; /* Arch-independent data */
};
#define VDSO_DATA_SIZE PAGE_ALIGN(sizeof(struct loongarch_vdso_data))
/*
* The layout of vvar:
*
* high
* +---------------------+--------------------------+
* | loongarch vdso data | LOONGARCH_VDSO_DATA_SIZE |
* +---------------------+--------------------------+
* | time-ns vdso data | PAGE_SIZE |
* +---------------------+--------------------------+
* | generic vdso data | PAGE_SIZE |
* +---------------------+--------------------------+
* low
*/
#define LOONGARCH_VDSO_DATA_SIZE PAGE_ALIGN(sizeof(struct loongarch_vdso_data))
#define LOONGARCH_VDSO_DATA_PAGES (LOONGARCH_VDSO_DATA_SIZE >> PAGE_SHIFT)
enum vvar_pages {
VVAR_GENERIC_PAGE_OFFSET,
VVAR_TIMENS_PAGE_OFFSET,
VVAR_LOONGARCH_PAGES_START,
VVAR_LOONGARCH_PAGES_END = VVAR_LOONGARCH_PAGES_START + LOONGARCH_VDSO_DATA_PAGES - 1,
VVAR_NR_PAGES,
};
#define VVAR_SIZE (VVAR_NR_PAGES << PAGE_SHIFT)
static inline unsigned long get_vdso_base(void)
{
......@@ -34,10 +57,9 @@ static inline unsigned long get_vdso_base(void)
return addr;
}
static inline const struct vdso_data *get_vdso_data(void)
static inline unsigned long get_vdso_data(void)
{
return (const struct vdso_data *)(get_vdso_base()
- VDSO_DATA_SIZE + SMP_CACHE_BYTES * NR_CPUS);
return get_vdso_base() - VVAR_SIZE;
}
#endif /* __ASSEMBLY__ */
......@@ -16,5 +16,6 @@
#define HWCAP_LOONGARCH_LBT_X86 (1 << 10)
#define HWCAP_LOONGARCH_LBT_ARM (1 << 11)
#define HWCAP_LOONGARCH_LBT_MIPS (1 << 12)
#define HWCAP_LOONGARCH_PTW (1 << 13)
#endif /* _UAPI_ASM_HWCAP_H */
......@@ -41,9 +41,19 @@ struct user_pt_regs {
} __attribute__((aligned(8)));
struct user_fp_state {
uint64_t fpr[32];
uint64_t fcc;
uint32_t fcsr;
uint64_t fpr[32];
uint64_t fcc;
uint32_t fcsr;
};
struct user_lsx_state {
/* 32 registers, 128 bits width per register. */
uint64_t vregs[32*2];
};
struct user_lasx_state {
/* 32 registers, 256 bits width per register. */
uint64_t vregs[32*4];
};
struct user_watch_state {
......
......@@ -41,4 +41,22 @@ struct fpu_context {
__u32 fcsr;
};
/* LSX context */
#define LSX_CTX_MAGIC 0x53580001
#define LSX_CTX_ALIGN 16
struct lsx_context {
__u64 regs[2*32];
__u64 fcc;
__u32 fcsr;
};
/* LASX context */
#define LASX_CTX_MAGIC 0x41535801
#define LASX_CTX_ALIGN 32
struct lasx_context {
__u64 regs[4*32];
__u64 fcc;
__u32 fcsr;
};
#endif /* _UAPI_ASM_SIGCONTEXT_H */
......@@ -28,6 +28,8 @@ ifdef CONFIG_FUNCTION_TRACER
CFLAGS_REMOVE_inst.o = $(CC_FLAGS_FTRACE)
CFLAGS_REMOVE_time.o = $(CC_FLAGS_FTRACE)
CFLAGS_REMOVE_perf_event.o = $(CC_FLAGS_FTRACE)
CFLAGS_REMOVE_rethook.o = $(CC_FLAGS_FTRACE)
CFLAGS_REMOVE_rethook_trampoline.o = $(CC_FLAGS_FTRACE)
endif
obj-$(CONFIG_MODULES) += module.o module-sections.o
......@@ -52,6 +54,10 @@ obj-$(CONFIG_UNWINDER_PROLOGUE) += unwind_prologue.o
obj-$(CONFIG_PERF_EVENTS) += perf_event.o perf_regs.o
obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o
obj-$(CONFIG_KPROBES) += kprobes.o kprobes_trampoline.o
obj-$(CONFIG_KPROBES) += kprobes.o
obj-$(CONFIG_RETHOOK) += rethook.o rethook_trampoline.o
obj-$(CONFIG_UPROBES) += uprobes.o
obj-$(CONFIG_JUMP_LABEL) += jump_label.o
CPPFLAGS_vmlinux.lds := $(KBUILD_CFLAGS)
......@@ -33,6 +33,8 @@ u64 acpi_saved_sp;
#define PREFIX "ACPI: "
struct acpi_madt_core_pic acpi_core_pic[NR_CPUS];
void __init __iomem * __acpi_map_table(unsigned long phys, unsigned long size)
{
......@@ -99,6 +101,7 @@ acpi_parse_processor(union acpi_subtable_headers *header, const unsigned long en
acpi_table_print_madt_entry(&header->common);
#ifdef CONFIG_SMP
acpi_core_pic[processor->core_id] = *processor;
set_processor_mask(processor->core_id, processor->flags);
#endif
......@@ -140,6 +143,35 @@ static void __init acpi_process_madt(void)
loongson_sysconf.nr_cpus = num_processors;
}
int pptt_enabled;
int __init parse_acpi_topology(void)
{
int cpu, topology_id;
for_each_possible_cpu(cpu) {
topology_id = find_acpi_cpu_topology(cpu, 0);
if (topology_id < 0) {
pr_warn("Invalid BIOS PPTT\n");
return -ENOENT;
}
if (acpi_pptt_cpu_is_thread(cpu) <= 0)
cpu_data[cpu].core = topology_id;
else {
topology_id = find_acpi_cpu_topology(cpu, 1);
if (topology_id < 0)
return -ENOENT;
cpu_data[cpu].core = topology_id;
}
}
pptt_enabled = 1;
return 0;
}
#ifndef CONFIG_SUSPEND
int (*acpi_suspend_lowlevel)(void);
#else
......
......@@ -116,6 +116,18 @@ static void cpu_probe_common(struct cpuinfo_loongarch *c)
c->options |= LOONGARCH_CPU_FPU;
elf_hwcap |= HWCAP_LOONGARCH_FPU;
}
#ifdef CONFIG_CPU_HAS_LSX
if (config & CPUCFG2_LSX) {
c->options |= LOONGARCH_CPU_LSX;
elf_hwcap |= HWCAP_LOONGARCH_LSX;
}
#endif
#ifdef CONFIG_CPU_HAS_LASX
if (config & CPUCFG2_LASX) {
c->options |= LOONGARCH_CPU_LASX;
elf_hwcap |= HWCAP_LOONGARCH_LASX;
}
#endif
if (config & CPUCFG2_COMPLEX) {
c->options |= LOONGARCH_CPU_COMPLEX;
elf_hwcap |= HWCAP_LOONGARCH_COMPLEX;
......@@ -124,6 +136,10 @@ static void cpu_probe_common(struct cpuinfo_loongarch *c)
c->options |= LOONGARCH_CPU_CRYPTO;
elf_hwcap |= HWCAP_LOONGARCH_CRYPTO;
}
if (config & CPUCFG2_PTW) {
c->options |= LOONGARCH_CPU_PTW;
elf_hwcap |= HWCAP_LOONGARCH_PTW;
}
if (config & CPUCFG2_LVZP) {
c->options |= LOONGARCH_CPU_LVZ;
elf_hwcap |= HWCAP_LOONGARCH_LVZ;
......
......@@ -24,7 +24,7 @@
.byte 0x02 /* MajorLinkerVersion */
.byte 0x14 /* MinorLinkerVersion */
.long __inittext_end - .Lefi_header_end /* SizeOfCode */
.long _end - __initdata_begin /* SizeOfInitializedData */
.long _kernel_vsize /* SizeOfInitializedData */
.long 0 /* SizeOfUninitializedData */
.long __efistub_efi_pe_entry - _head /* AddressOfEntryPoint */
.long .Lefi_header_end - _head /* BaseOfCode */
......@@ -79,9 +79,9 @@
IMAGE_SCN_MEM_EXECUTE /* Characteristics */
.ascii ".data\0\0\0"
.long _end - __initdata_begin /* VirtualSize */
.long _kernel_vsize /* VirtualSize */
.long __initdata_begin - _head /* VirtualAddress */
.long _edata - __initdata_begin /* SizeOfRawData */
.long _kernel_rsize /* SizeOfRawData */
.long __initdata_begin - _head /* PointerToRawData */
.long 0 /* PointerToRelocations */
......
......@@ -145,6 +145,154 @@
movgr2fcsr fcsr0, \tmp0
.endm
.macro sc_save_lsx base
#ifdef CONFIG_CPU_HAS_LSX
EX vst $vr0, \base, (0 * LSX_REG_WIDTH)
EX vst $vr1, \base, (1 * LSX_REG_WIDTH)
EX vst $vr2, \base, (2 * LSX_REG_WIDTH)
EX vst $vr3, \base, (3 * LSX_REG_WIDTH)
EX vst $vr4, \base, (4 * LSX_REG_WIDTH)
EX vst $vr5, \base, (5 * LSX_REG_WIDTH)
EX vst $vr6, \base, (6 * LSX_REG_WIDTH)
EX vst $vr7, \base, (7 * LSX_REG_WIDTH)
EX vst $vr8, \base, (8 * LSX_REG_WIDTH)
EX vst $vr9, \base, (9 * LSX_REG_WIDTH)
EX vst $vr10, \base, (10 * LSX_REG_WIDTH)
EX vst $vr11, \base, (11 * LSX_REG_WIDTH)
EX vst $vr12, \base, (12 * LSX_REG_WIDTH)
EX vst $vr13, \base, (13 * LSX_REG_WIDTH)
EX vst $vr14, \base, (14 * LSX_REG_WIDTH)
EX vst $vr15, \base, (15 * LSX_REG_WIDTH)
EX vst $vr16, \base, (16 * LSX_REG_WIDTH)
EX vst $vr17, \base, (17 * LSX_REG_WIDTH)
EX vst $vr18, \base, (18 * LSX_REG_WIDTH)
EX vst $vr19, \base, (19 * LSX_REG_WIDTH)
EX vst $vr20, \base, (20 * LSX_REG_WIDTH)
EX vst $vr21, \base, (21 * LSX_REG_WIDTH)
EX vst $vr22, \base, (22 * LSX_REG_WIDTH)
EX vst $vr23, \base, (23 * LSX_REG_WIDTH)
EX vst $vr24, \base, (24 * LSX_REG_WIDTH)
EX vst $vr25, \base, (25 * LSX_REG_WIDTH)
EX vst $vr26, \base, (26 * LSX_REG_WIDTH)
EX vst $vr27, \base, (27 * LSX_REG_WIDTH)
EX vst $vr28, \base, (28 * LSX_REG_WIDTH)
EX vst $vr29, \base, (29 * LSX_REG_WIDTH)
EX vst $vr30, \base, (30 * LSX_REG_WIDTH)
EX vst $vr31, \base, (31 * LSX_REG_WIDTH)
#endif
.endm
.macro sc_restore_lsx base
#ifdef CONFIG_CPU_HAS_LSX
EX vld $vr0, \base, (0 * LSX_REG_WIDTH)
EX vld $vr1, \base, (1 * LSX_REG_WIDTH)
EX vld $vr2, \base, (2 * LSX_REG_WIDTH)
EX vld $vr3, \base, (3 * LSX_REG_WIDTH)
EX vld $vr4, \base, (4 * LSX_REG_WIDTH)
EX vld $vr5, \base, (5 * LSX_REG_WIDTH)
EX vld $vr6, \base, (6 * LSX_REG_WIDTH)
EX vld $vr7, \base, (7 * LSX_REG_WIDTH)
EX vld $vr8, \base, (8 * LSX_REG_WIDTH)
EX vld $vr9, \base, (9 * LSX_REG_WIDTH)
EX vld $vr10, \base, (10 * LSX_REG_WIDTH)
EX vld $vr11, \base, (11 * LSX_REG_WIDTH)
EX vld $vr12, \base, (12 * LSX_REG_WIDTH)
EX vld $vr13, \base, (13 * LSX_REG_WIDTH)
EX vld $vr14, \base, (14 * LSX_REG_WIDTH)
EX vld $vr15, \base, (15 * LSX_REG_WIDTH)
EX vld $vr16, \base, (16 * LSX_REG_WIDTH)
EX vld $vr17, \base, (17 * LSX_REG_WIDTH)
EX vld $vr18, \base, (18 * LSX_REG_WIDTH)
EX vld $vr19, \base, (19 * LSX_REG_WIDTH)
EX vld $vr20, \base, (20 * LSX_REG_WIDTH)
EX vld $vr21, \base, (21 * LSX_REG_WIDTH)
EX vld $vr22, \base, (22 * LSX_REG_WIDTH)
EX vld $vr23, \base, (23 * LSX_REG_WIDTH)
EX vld $vr24, \base, (24 * LSX_REG_WIDTH)
EX vld $vr25, \base, (25 * LSX_REG_WIDTH)
EX vld $vr26, \base, (26 * LSX_REG_WIDTH)
EX vld $vr27, \base, (27 * LSX_REG_WIDTH)
EX vld $vr28, \base, (28 * LSX_REG_WIDTH)
EX vld $vr29, \base, (29 * LSX_REG_WIDTH)
EX vld $vr30, \base, (30 * LSX_REG_WIDTH)
EX vld $vr31, \base, (31 * LSX_REG_WIDTH)
#endif
.endm
.macro sc_save_lasx base
#ifdef CONFIG_CPU_HAS_LASX
EX xvst $xr0, \base, (0 * LASX_REG_WIDTH)
EX xvst $xr1, \base, (1 * LASX_REG_WIDTH)
EX xvst $xr2, \base, (2 * LASX_REG_WIDTH)
EX xvst $xr3, \base, (3 * LASX_REG_WIDTH)
EX xvst $xr4, \base, (4 * LASX_REG_WIDTH)
EX xvst $xr5, \base, (5 * LASX_REG_WIDTH)
EX xvst $xr6, \base, (6 * LASX_REG_WIDTH)
EX xvst $xr7, \base, (7 * LASX_REG_WIDTH)
EX xvst $xr8, \base, (8 * LASX_REG_WIDTH)
EX xvst $xr9, \base, (9 * LASX_REG_WIDTH)
EX xvst $xr10, \base, (10 * LASX_REG_WIDTH)
EX xvst $xr11, \base, (11 * LASX_REG_WIDTH)
EX xvst $xr12, \base, (12 * LASX_REG_WIDTH)
EX xvst $xr13, \base, (13 * LASX_REG_WIDTH)
EX xvst $xr14, \base, (14 * LASX_REG_WIDTH)
EX xvst $xr15, \base, (15 * LASX_REG_WIDTH)
EX xvst $xr16, \base, (16 * LASX_REG_WIDTH)
EX xvst $xr17, \base, (17 * LASX_REG_WIDTH)
EX xvst $xr18, \base, (18 * LASX_REG_WIDTH)
EX xvst $xr19, \base, (19 * LASX_REG_WIDTH)
EX xvst $xr20, \base, (20 * LASX_REG_WIDTH)
EX xvst $xr21, \base, (21 * LASX_REG_WIDTH)
EX xvst $xr22, \base, (22 * LASX_REG_WIDTH)
EX xvst $xr23, \base, (23 * LASX_REG_WIDTH)
EX xvst $xr24, \base, (24 * LASX_REG_WIDTH)
EX xvst $xr25, \base, (25 * LASX_REG_WIDTH)
EX xvst $xr26, \base, (26 * LASX_REG_WIDTH)
EX xvst $xr27, \base, (27 * LASX_REG_WIDTH)
EX xvst $xr28, \base, (28 * LASX_REG_WIDTH)
EX xvst $xr29, \base, (29 * LASX_REG_WIDTH)
EX xvst $xr30, \base, (30 * LASX_REG_WIDTH)
EX xvst $xr31, \base, (31 * LASX_REG_WIDTH)
#endif
.endm
.macro sc_restore_lasx base
#ifdef CONFIG_CPU_HAS_LASX
EX xvld $xr0, \base, (0 * LASX_REG_WIDTH)
EX xvld $xr1, \base, (1 * LASX_REG_WIDTH)
EX xvld $xr2, \base, (2 * LASX_REG_WIDTH)
EX xvld $xr3, \base, (3 * LASX_REG_WIDTH)
EX xvld $xr4, \base, (4 * LASX_REG_WIDTH)
EX xvld $xr5, \base, (5 * LASX_REG_WIDTH)
EX xvld $xr6, \base, (6 * LASX_REG_WIDTH)
EX xvld $xr7, \base, (7 * LASX_REG_WIDTH)
EX xvld $xr8, \base, (8 * LASX_REG_WIDTH)
EX xvld $xr9, \base, (9 * LASX_REG_WIDTH)
EX xvld $xr10, \base, (10 * LASX_REG_WIDTH)
EX xvld $xr11, \base, (11 * LASX_REG_WIDTH)
EX xvld $xr12, \base, (12 * LASX_REG_WIDTH)
EX xvld $xr13, \base, (13 * LASX_REG_WIDTH)
EX xvld $xr14, \base, (14 * LASX_REG_WIDTH)
EX xvld $xr15, \base, (15 * LASX_REG_WIDTH)
EX xvld $xr16, \base, (16 * LASX_REG_WIDTH)
EX xvld $xr17, \base, (17 * LASX_REG_WIDTH)
EX xvld $xr18, \base, (18 * LASX_REG_WIDTH)
EX xvld $xr19, \base, (19 * LASX_REG_WIDTH)
EX xvld $xr20, \base, (20 * LASX_REG_WIDTH)
EX xvld $xr21, \base, (21 * LASX_REG_WIDTH)
EX xvld $xr22, \base, (22 * LASX_REG_WIDTH)
EX xvld $xr23, \base, (23 * LASX_REG_WIDTH)
EX xvld $xr24, \base, (24 * LASX_REG_WIDTH)
EX xvld $xr25, \base, (25 * LASX_REG_WIDTH)
EX xvld $xr26, \base, (26 * LASX_REG_WIDTH)
EX xvld $xr27, \base, (27 * LASX_REG_WIDTH)
EX xvld $xr28, \base, (28 * LASX_REG_WIDTH)
EX xvld $xr29, \base, (29 * LASX_REG_WIDTH)
EX xvld $xr30, \base, (30 * LASX_REG_WIDTH)
EX xvld $xr31, \base, (31 * LASX_REG_WIDTH)
#endif
.endm
/*
* Save a thread's fp context.
*/
......@@ -166,6 +314,76 @@ SYM_FUNC_START(_restore_fp)
jr ra
SYM_FUNC_END(_restore_fp)
#ifdef CONFIG_CPU_HAS_LSX
/*
* Save a thread's LSX vector context.
*/
SYM_FUNC_START(_save_lsx)
lsx_save_all a0 t1 t2
jr ra
SYM_FUNC_END(_save_lsx)
EXPORT_SYMBOL(_save_lsx)
/*
* Restore a thread's LSX vector context.
*/
SYM_FUNC_START(_restore_lsx)
lsx_restore_all a0 t1 t2
jr ra
SYM_FUNC_END(_restore_lsx)
SYM_FUNC_START(_save_lsx_upper)
lsx_save_all_upper a0 t0 t1
jr ra
SYM_FUNC_END(_save_lsx_upper)
SYM_FUNC_START(_restore_lsx_upper)
lsx_restore_all_upper a0 t0 t1
jr ra
SYM_FUNC_END(_restore_lsx_upper)
SYM_FUNC_START(_init_lsx_upper)
lsx_init_all_upper t1
jr ra
SYM_FUNC_END(_init_lsx_upper)
#endif
#ifdef CONFIG_CPU_HAS_LASX
/*
* Save a thread's LASX vector context.
*/
SYM_FUNC_START(_save_lasx)
lasx_save_all a0 t1 t2
jr ra
SYM_FUNC_END(_save_lasx)
EXPORT_SYMBOL(_save_lasx)
/*
* Restore a thread's LASX vector context.
*/
SYM_FUNC_START(_restore_lasx)
lasx_restore_all a0 t1 t2
jr ra
SYM_FUNC_END(_restore_lasx)
SYM_FUNC_START(_save_lasx_upper)
lasx_save_all_upper a0 t0 t1
jr ra
SYM_FUNC_END(_save_lasx_upper)
SYM_FUNC_START(_restore_lasx_upper)
lasx_restore_all_upper a0 t0 t1
jr ra
SYM_FUNC_END(_restore_lasx_upper)
SYM_FUNC_START(_init_lasx_upper)
lasx_init_all_upper t1
jr ra
SYM_FUNC_END(_init_lasx_upper)
#endif
/*
* Load the FPU with signalling NANS. This bit pattern we're using has
* the property that no matter whether considered as single or as double
......@@ -244,6 +462,58 @@ SYM_FUNC_START(_restore_fp_context)
jr ra
SYM_FUNC_END(_restore_fp_context)
/*
* a0: fpregs
* a1: fcc
* a2: fcsr
*/
SYM_FUNC_START(_save_lsx_context)
sc_save_fcc a1, t0, t1
sc_save_fcsr a2, t0
sc_save_lsx a0
li.w a0, 0 # success
jr ra
SYM_FUNC_END(_save_lsx_context)
/*
* a0: fpregs
* a1: fcc
* a2: fcsr
*/
SYM_FUNC_START(_restore_lsx_context)
sc_restore_lsx a0
sc_restore_fcc a1, t1, t2
sc_restore_fcsr a2, t1
li.w a0, 0 # success
jr ra
SYM_FUNC_END(_restore_lsx_context)
/*
* a0: fpregs
* a1: fcc
* a2: fcsr
*/
SYM_FUNC_START(_save_lasx_context)
sc_save_fcc a1, t0, t1
sc_save_fcsr a2, t0
sc_save_lasx a0
li.w a0, 0 # success
jr ra
SYM_FUNC_END(_save_lasx_context)
/*
* a0: fpregs
* a1: fcc
* a2: fcsr
*/
SYM_FUNC_START(_restore_lasx_context)
sc_restore_lasx a0
sc_restore_fcc a1, t1, t2
sc_restore_fcsr a2, t1
li.w a0, 0 # success
jr ra
SYM_FUNC_END(_restore_lasx_context)
SYM_FUNC_START(fault)
li.w a0, -EFAULT # failure
jr ra
......
......@@ -23,7 +23,7 @@ _head:
.word MZ_MAGIC /* "MZ", MS-DOS header */
.org 0x8
.dword kernel_entry /* Kernel entry point */
.dword _end - _text /* Kernel image effective size */
.dword _kernel_asize /* Kernel image effective size */
.quad PHYS_LINK_KADDR /* Kernel image load offset from start of RAM */
.org 0x38 /* 0x20 ~ 0x37 reserved */
.long LINUX_PE_MAGIC
......@@ -32,9 +32,9 @@ _head:
pe_header:
__EFI_PE_HEADER
SYM_DATA(kernel_asize, .long _end - _text);
SYM_DATA(kernel_fsize, .long _edata - _text);
SYM_DATA(kernel_offset, .long kernel_offset - _text);
SYM_DATA(kernel_asize, .long _kernel_asize);
SYM_DATA(kernel_fsize, .long _kernel_fsize);
SYM_DATA(kernel_offset, .long _kernel_offset);
#endif
......
......@@ -133,6 +133,51 @@ void simu_branch(struct pt_regs *regs, union loongarch_instruction insn)
}
}
bool insns_not_supported(union loongarch_instruction insn)
{
switch (insn.reg3_format.opcode) {
case amswapw_op ... ammindbdu_op:
pr_notice("atomic memory access instructions are not supported\n");
return true;
}
switch (insn.reg2i14_format.opcode) {
case llw_op:
case lld_op:
case scw_op:
case scd_op:
pr_notice("ll and sc instructions are not supported\n");
return true;
}
switch (insn.reg1i21_format.opcode) {
case bceqz_op:
pr_notice("bceqz and bcnez instructions are not supported\n");
return true;
}
return false;
}
bool insns_need_simulation(union loongarch_instruction insn)
{
if (is_pc_ins(&insn))
return true;
if (is_branch_ins(&insn))
return true;
return false;
}
void arch_simulate_insn(union loongarch_instruction insn, struct pt_regs *regs)
{
if (is_pc_ins(&insn))
simu_pc(regs, insn);
else if (is_branch_ins(&insn))
simu_branch(regs, insn);
}
int larch_insn_read(void *addr, u32 *insnp)
{
int ret;
......@@ -208,6 +253,20 @@ u32 larch_insn_gen_bl(unsigned long pc, unsigned long dest)
return insn.word;
}
u32 larch_insn_gen_break(int imm)
{
union loongarch_instruction insn;
if (imm < 0 || imm >= SZ_32K) {
pr_warn("The generated break instruction is out of range.\n");
return INSN_BREAK;
}
emit_break(&insn, imm);
return insn.word;
}
u32 larch_insn_gen_or(enum loongarch_gpr rd, enum loongarch_gpr rj, enum loongarch_gpr rk)
{
union loongarch_instruction insn;
......@@ -226,6 +285,11 @@ u32 larch_insn_gen_lu12iw(enum loongarch_gpr rd, int imm)
{
union loongarch_instruction insn;
if (imm < -SZ_512K || imm >= SZ_512K) {
pr_warn("The generated lu12i.w instruction is out of range.\n");
return INSN_BREAK;
}
emit_lu12iw(&insn, rd, imm);
return insn.word;
......@@ -235,6 +299,11 @@ u32 larch_insn_gen_lu32id(enum loongarch_gpr rd, int imm)
{
union loongarch_instruction insn;
if (imm < -SZ_512K || imm >= SZ_512K) {
pr_warn("The generated lu32i.d instruction is out of range.\n");
return INSN_BREAK;
}
emit_lu32id(&insn, rd, imm);
return insn.word;
......@@ -244,16 +313,26 @@ u32 larch_insn_gen_lu52id(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)
{
union loongarch_instruction insn;
if (imm < -SZ_2K || imm >= SZ_2K) {
pr_warn("The generated lu52i.d instruction is out of range.\n");
return INSN_BREAK;
}
emit_lu52id(&insn, rd, rj, imm);
return insn.word;
}
u32 larch_insn_gen_jirl(enum loongarch_gpr rd, enum loongarch_gpr rj, unsigned long pc, unsigned long dest)
u32 larch_insn_gen_jirl(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)
{
union loongarch_instruction insn;
emit_jirl(&insn, rj, rd, (dest - pc) >> 2);
if ((imm & 3) || imm < -SZ_128K || imm >= SZ_128K) {
pr_warn("The generated jirl instruction is out of range.\n");
return INSN_BREAK;
}
emit_jirl(&insn, rj, rd, imm >> 2);
return insn.word;
}
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2023 Loongson Technology Corporation Limited
*
* Based on arch/arm64/kernel/jump_label.c
*/
#include <linux/kernel.h>
#include <linux/jump_label.h>
#include <asm/inst.h>
void arch_jump_label_transform(struct jump_entry *entry, enum jump_label_type type)
{
u32 insn;
void *addr = (void *)jump_entry_code(entry);
if (type == JUMP_LABEL_JMP)
insn = larch_insn_gen_b(jump_entry_code(entry), jump_entry_target(entry));
else
insn = larch_insn_gen_nop();
larch_insn_patch_text(addr, insn);
}
......@@ -4,69 +4,16 @@
#include <linux/preempt.h>
#include <asm/break.h>
static const union loongarch_instruction breakpoint_insn = {
.reg0i15_format = {
.opcode = break_op,
.immediate = BRK_KPROBE_BP,
}
};
static const union loongarch_instruction singlestep_insn = {
.reg0i15_format = {
.opcode = break_op,
.immediate = BRK_KPROBE_SSTEPBP,
}
};
#define KPROBE_BP_INSN larch_insn_gen_break(BRK_KPROBE_BP)
#define KPROBE_SSTEPBP_INSN larch_insn_gen_break(BRK_KPROBE_SSTEPBP)
DEFINE_PER_CPU(struct kprobe *, current_kprobe);
DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
static bool insns_not_supported(union loongarch_instruction insn)
{
switch (insn.reg2i14_format.opcode) {
case llw_op:
case lld_op:
case scw_op:
case scd_op:
pr_notice("kprobe: ll and sc instructions are not supported\n");
return true;
}
switch (insn.reg1i21_format.opcode) {
case bceqz_op:
pr_notice("kprobe: bceqz and bcnez instructions are not supported\n");
return true;
}
return false;
}
NOKPROBE_SYMBOL(insns_not_supported);
static bool insns_need_simulation(struct kprobe *p)
{
if (is_pc_ins(&p->opcode))
return true;
if (is_branch_ins(&p->opcode))
return true;
return false;
}
NOKPROBE_SYMBOL(insns_need_simulation);
static void arch_simulate_insn(struct kprobe *p, struct pt_regs *regs)
{
if (is_pc_ins(&p->opcode))
simu_pc(regs, p->opcode);
else if (is_branch_ins(&p->opcode))
simu_branch(regs, p->opcode);
}
NOKPROBE_SYMBOL(arch_simulate_insn);
static void arch_prepare_ss_slot(struct kprobe *p)
{
p->ainsn.insn[0] = *p->addr;
p->ainsn.insn[1] = singlestep_insn;
p->ainsn.insn[1] = KPROBE_SSTEPBP_INSN;
p->ainsn.restore = (unsigned long)p->addr + LOONGARCH_INSN_SIZE;
}
NOKPROBE_SYMBOL(arch_prepare_ss_slot);
......@@ -79,17 +26,20 @@ NOKPROBE_SYMBOL(arch_prepare_simulate);
int arch_prepare_kprobe(struct kprobe *p)
{
union loongarch_instruction insn;
if ((unsigned long)p->addr & 0x3)
return -EILSEQ;
/* copy instruction */
p->opcode = *p->addr;
insn.word = p->opcode;
/* decode instruction */
if (insns_not_supported(p->opcode))
if (insns_not_supported(insn))
return -EINVAL;
if (insns_need_simulation(p)) {
if (insns_need_simulation(insn)) {
p->ainsn.insn = NULL;
} else {
p->ainsn.insn = get_insn_slot();
......@@ -110,7 +60,7 @@ NOKPROBE_SYMBOL(arch_prepare_kprobe);
/* Install breakpoint in text */
void arch_arm_kprobe(struct kprobe *p)
{
*p->addr = breakpoint_insn;
*p->addr = KPROBE_BP_INSN;
flush_insn_slot(p);
}
NOKPROBE_SYMBOL(arch_arm_kprobe);
......@@ -205,6 +155,8 @@ NOKPROBE_SYMBOL(post_kprobe_handler);
static void setup_singlestep(struct kprobe *p, struct pt_regs *regs,
struct kprobe_ctlblk *kcb, int reenter)
{
union loongarch_instruction insn;
if (reenter) {
save_previous_kprobe(kcb);
set_current_kprobe(p);
......@@ -220,7 +172,8 @@ static void setup_singlestep(struct kprobe *p, struct pt_regs *regs,
regs->csr_era = (unsigned long)p->ainsn.insn;
} else {
/* simulate single steping */
arch_simulate_insn(p, regs);
insn.word = p->opcode;
arch_simulate_insn(insn, regs);
/* now go for post processing */
post_kprobe_handler(p, kcb, regs);
}
......@@ -295,7 +248,7 @@ bool kprobe_breakpoint_handler(struct pt_regs *regs)
}
}
if (addr->word != breakpoint_insn.word) {
if (*addr != KPROBE_BP_INSN) {
/*
* The breakpoint instruction was removed right
* after we hit it. Another cpu has removed
......@@ -378,27 +331,6 @@ int __init arch_init_kprobes(void)
return 0;
}
/* ASM function that handles the kretprobes must not be probed */
NOKPROBE_SYMBOL(__kretprobe_trampoline);
/* Called from __kretprobe_trampoline */
void __used *trampoline_probe_handler(struct pt_regs *regs)
{
return (void *)kretprobe_trampoline_handler(regs, NULL);
}
NOKPROBE_SYMBOL(trampoline_probe_handler);
void arch_prepare_kretprobe(struct kretprobe_instance *ri,
struct pt_regs *regs)
{
ri->ret_addr = (kprobe_opcode_t *)regs->regs[1];
ri->fp = NULL;
/* Replace the return addr with trampoline addr */
regs->regs[1] = (unsigned long)&__kretprobe_trampoline;
}
NOKPROBE_SYMBOL(arch_prepare_kretprobe);
int arch_trampoline_kprobe(struct kprobe *p)
{
return 0;
......
......@@ -49,6 +49,7 @@ static int show_cpuinfo(struct seq_file *m, void *v)
seq_printf(m, "processor\t\t: %ld\n", n);
seq_printf(m, "package\t\t\t: %d\n", cpu_data[n].package);
seq_printf(m, "core\t\t\t: %d\n", cpu_data[n].core);
seq_printf(m, "global_id\t\t: %d\n", cpu_data[n].global_id);
seq_printf(m, "CPU Family\t\t: %s\n", __cpu_family[n]);
seq_printf(m, "Model Name\t\t: %s\n", __cpu_full_name[n]);
seq_printf(m, "CPU Revision\t\t: 0x%02x\n", version);
......@@ -79,6 +80,7 @@ static int show_cpuinfo(struct seq_file *m, void *v)
if (cpu_has_crc32) seq_printf(m, " crc32");
if (cpu_has_complex) seq_printf(m, " complex");
if (cpu_has_crypto) seq_printf(m, " crypto");
if (cpu_has_ptw) seq_printf(m, " ptw");
if (cpu_has_lvz) seq_printf(m, " lvz");
if (cpu_has_lbt_x86) seq_printf(m, " lbt_x86");
if (cpu_has_lbt_arm) seq_printf(m, " lbt_arm");
......
......@@ -117,8 +117,14 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
*/
preempt_disable();
if (is_fpu_owner())
save_fp(current);
if (is_fpu_owner()) {
if (is_lasx_enabled())
save_lasx(current);
else if (is_lsx_enabled())
save_lsx(current);
else
save_fp(current);
}
preempt_enable();
......@@ -285,7 +291,7 @@ unsigned long stack_top(void)
/* Space for the VDSO & data page */
top -= PAGE_ALIGN(current->thread.vdso->size);
top -= PAGE_SIZE;
top -= VVAR_SIZE;
/* Space to randomize the VDSO base */
if (current->flags & PF_RANDOMIZE)
......
......@@ -250,6 +250,90 @@ static int cfg_set(struct task_struct *target,
return 0;
}
#ifdef CONFIG_CPU_HAS_LSX
static void copy_pad_fprs(struct task_struct *target,
const struct user_regset *regset,
struct membuf *to, unsigned int live_sz)
{
int i, j;
unsigned long long fill = ~0ull;
unsigned int cp_sz, pad_sz;
cp_sz = min(regset->size, live_sz);
pad_sz = regset->size - cp_sz;
WARN_ON(pad_sz % sizeof(fill));
for (i = 0; i < NUM_FPU_REGS; i++) {
membuf_write(to, &target->thread.fpu.fpr[i], cp_sz);
for (j = 0; j < (pad_sz / sizeof(fill)); j++) {
membuf_store(to, fill);
}
}
}
static int simd_get(struct task_struct *target,
const struct user_regset *regset,
struct membuf to)
{
const unsigned int wr_size = NUM_FPU_REGS * regset->size;
if (!tsk_used_math(target)) {
/* The task hasn't used FP or LSX, fill with 0xff */
copy_pad_fprs(target, regset, &to, 0);
} else if (!test_tsk_thread_flag(target, TIF_LSX_CTX_LIVE)) {
/* Copy scalar FP context, fill the rest with 0xff */
copy_pad_fprs(target, regset, &to, 8);
#ifdef CONFIG_CPU_HAS_LASX
} else if (!test_tsk_thread_flag(target, TIF_LASX_CTX_LIVE)) {
/* Copy LSX 128 Bit context, fill the rest with 0xff */
copy_pad_fprs(target, regset, &to, 16);
#endif
} else if (sizeof(target->thread.fpu.fpr[0]) == regset->size) {
/* Trivially copy the vector registers */
membuf_write(&to, &target->thread.fpu.fpr, wr_size);
} else {
/* Copy as much context as possible, fill the rest with 0xff */
copy_pad_fprs(target, regset, &to, sizeof(target->thread.fpu.fpr[0]));
}
return 0;
}
static int simd_set(struct task_struct *target,
const struct user_regset *regset,
unsigned int pos, unsigned int count,
const void *kbuf, const void __user *ubuf)
{
const unsigned int wr_size = NUM_FPU_REGS * regset->size;
unsigned int cp_sz;
int i, err, start;
init_fp_ctx(target);
if (sizeof(target->thread.fpu.fpr[0]) == regset->size) {
/* Trivially copy the vector registers */
err = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
&target->thread.fpu.fpr,
0, wr_size);
} else {
/* Copy as much context as possible */
cp_sz = min_t(unsigned int, regset->size,
sizeof(target->thread.fpu.fpr[0]));
i = start = err = 0;
for (; i < NUM_FPU_REGS; i++, start += regset->size) {
err |= user_regset_copyin(&pos, &count, &kbuf, &ubuf,
&target->thread.fpu.fpr[i],
start, start + cp_sz);
}
}
return err;
}
#endif /* CONFIG_CPU_HAS_LSX */
#ifdef CONFIG_HAVE_HW_BREAKPOINT
/*
......@@ -708,6 +792,12 @@ enum loongarch_regset {
REGSET_GPR,
REGSET_FPR,
REGSET_CPUCFG,
#ifdef CONFIG_CPU_HAS_LSX
REGSET_LSX,
#endif
#ifdef CONFIG_CPU_HAS_LASX
REGSET_LASX,
#endif
#ifdef CONFIG_HAVE_HW_BREAKPOINT
REGSET_HW_BREAK,
REGSET_HW_WATCH,
......@@ -739,6 +829,26 @@ static const struct user_regset loongarch64_regsets[] = {
.regset_get = cfg_get,
.set = cfg_set,
},
#ifdef CONFIG_CPU_HAS_LSX
[REGSET_LSX] = {
.core_note_type = NT_LOONGARCH_LSX,
.n = NUM_FPU_REGS,
.size = 16,
.align = 16,
.regset_get = simd_get,
.set = simd_set,
},
#endif
#ifdef CONFIG_CPU_HAS_LASX
[REGSET_LASX] = {
.core_note_type = NT_LOONGARCH_LASX,
.n = NUM_FPU_REGS,
.size = 32,
.align = 32,
.regset_get = simd_get,
.set = simd_set,
},
#endif
#ifdef CONFIG_HAVE_HW_BREAKPOINT
[REGSET_HW_BREAK] = {
.core_note_type = NT_LOONGARCH_HW_BREAK,
......
// SPDX-License-Identifier: GPL-2.0
/*
* Generic return hook for LoongArch.
*/
#include <linux/kprobes.h>
#include <linux/rethook.h>
#include "rethook.h"
/* This is called from arch_rethook_trampoline() */
unsigned long __used arch_rethook_trampoline_callback(struct pt_regs *regs)
{
return rethook_trampoline_handler(regs, 0);
}
NOKPROBE_SYMBOL(arch_rethook_trampoline_callback);
void arch_rethook_prepare(struct rethook_node *rhn, struct pt_regs *regs, bool mcount)
{
rhn->frame = 0;
rhn->ret_addr = regs->regs[1];
/* replace return addr with trampoline */
regs->regs[1] = (unsigned long)arch_rethook_trampoline;
}
NOKPROBE_SYMBOL(arch_rethook_prepare);
/* ASM function that handles the rethook must not be probed itself */
NOKPROBE_SYMBOL(arch_rethook_trampoline);
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __LOONGARCH_RETHOOK_H
#define __LOONGARCH_RETHOOK_H
unsigned long arch_rethook_trampoline_callback(struct pt_regs *regs);
void arch_rethook_prepare(struct rethook_node *rhn, struct pt_regs *regs, bool mcount);
#endif
......@@ -75,7 +75,7 @@
csrxchg t0, t1, LOONGARCH_CSR_CRMD
.endm
SYM_CODE_START(__kretprobe_trampoline)
SYM_CODE_START(arch_rethook_trampoline)
addi.d sp, sp, -PT_SIZE
save_all_base_regs
......@@ -84,7 +84,7 @@ SYM_CODE_START(__kretprobe_trampoline)
move a0, sp /* pt_regs */
bl trampoline_probe_handler
bl arch_rethook_trampoline_callback
/* use the result as the return-address */
move ra, a0
......@@ -93,4 +93,4 @@ SYM_CODE_START(__kretprobe_trampoline)
addi.d sp, sp, PT_SIZE
jr ra
SYM_CODE_END(__kretprobe_trampoline)
SYM_CODE_END(arch_rethook_trampoline)
This diff is collapsed.
......@@ -8,6 +8,7 @@
* Copyright (C) 2000, 2001 Silicon Graphics, Inc.
* Copyright (C) 2000, 2001, 2003 Broadcom Corporation
*/
#include <linux/acpi.h>
#include <linux/cpu.h>
#include <linux/cpumask.h>
#include <linux/init.h>
......@@ -37,10 +38,6 @@ EXPORT_SYMBOL(__cpu_number_map);
int __cpu_logical_map[NR_CPUS]; /* Map logical to physical */
EXPORT_SYMBOL(__cpu_logical_map);
/* Number of threads (siblings) per CPU core */
int smp_num_siblings = 1;
EXPORT_SYMBOL(smp_num_siblings);
/* Representing the threads (siblings) of each logical CPU */
cpumask_t cpu_sibling_map[NR_CPUS] __read_mostly;
EXPORT_SYMBOL(cpu_sibling_map);
......@@ -118,7 +115,7 @@ static u32 ipi_read_clear(int cpu)
action = iocsr_read32(LOONGARCH_IOCSR_IPI_STATUS);
/* Clear the ipi register to clear the interrupt */
iocsr_write32(action, LOONGARCH_IOCSR_IPI_CLEAR);
smp_mb();
wbflush();
return action;
}
......@@ -210,6 +207,7 @@ static void __init fdt_smp_setup(void)
}
loongson_sysconf.nr_cpus = num_processors;
set_bit(0, &(loongson_sysconf.cores_io_master));
#endif
}
......@@ -228,9 +226,12 @@ void __init loongson_prepare_cpus(unsigned int max_cpus)
{
int i = 0;
parse_acpi_topology();
for (i = 0; i < loongson_sysconf.nr_cpus; i++) {
set_cpu_present(i, true);
csr_mail_send(0, __cpu_logical_map[i], 0);
cpu_data[i].global_id = __cpu_logical_map[i];
}
per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
......@@ -271,10 +272,10 @@ void loongson_init_secondary(void)
numa_add_cpu(cpu);
#endif
per_cpu(cpu_state, cpu) = CPU_ONLINE;
cpu_data[cpu].core =
cpu_logical_map(cpu) % loongson_sysconf.cores_per_package;
cpu_data[cpu].package =
cpu_logical_map(cpu) / loongson_sysconf.cores_per_package;
cpu_data[cpu].core = pptt_enabled ? cpu_data[cpu].core :
cpu_logical_map(cpu) % loongson_sysconf.cores_per_package;
}
void loongson_smp_finish(void)
......@@ -380,14 +381,10 @@ static inline void set_cpu_sibling_map(int cpu)
cpumask_set_cpu(cpu, &cpu_sibling_setup_map);
if (smp_num_siblings <= 1)
cpumask_set_cpu(cpu, &cpu_sibling_map[cpu]);
else {
for_each_cpu(i, &cpu_sibling_setup_map) {
if (cpus_are_siblings(cpu, i)) {
cpumask_set_cpu(i, &cpu_sibling_map[cpu]);
cpumask_set_cpu(cpu, &cpu_sibling_map[i]);
}
for_each_cpu(i, &cpu_sibling_setup_map) {
if (cpus_are_siblings(cpu, i)) {
cpumask_set_cpu(i, &cpu_sibling_map[cpu]);
cpumask_set_cpu(cpu, &cpu_sibling_map[i]);
}
}
}
......
......@@ -47,6 +47,7 @@
#include <asm/tlb.h>
#include <asm/types.h>
#include <asm/unwind.h>
#include <asm/uprobes.h>
#include "access-helper.h"
......@@ -689,7 +690,6 @@ asmlinkage void noinstr do_bp(struct pt_regs *regs)
if (regs->csr_prmd & CSR_PRMD_PIE)
local_irq_enable();
current->thread.trap_nr = read_csr_excode();
if (__get_inst(&opcode, (u32 *)era, user))
goto out_sigsegv;
......@@ -711,18 +711,17 @@ asmlinkage void noinstr do_bp(struct pt_regs *regs)
else
break;
case BRK_UPROBE_BP:
if (notify_die(DIE_UPROBE, "Uprobe", regs, bcode,
current->thread.trap_nr, SIGTRAP) == NOTIFY_STOP)
if (uprobe_breakpoint_handler(regs))
goto out;
else
break;
case BRK_UPROBE_XOLBP:
if (notify_die(DIE_UPROBE_XOL, "Uprobe_XOL", regs, bcode,
current->thread.trap_nr, SIGTRAP) == NOTIFY_STOP)
if (uprobe_singlestep_handler(regs))
goto out;
else
break;
default:
current->thread.trap_nr = read_csr_excode();
if (notify_die(DIE_TRAP, "Break", regs, bcode,
current->thread.trap_nr, SIGTRAP) == NOTIFY_STOP)
goto out;
......@@ -852,12 +851,67 @@ static void init_restore_fp(void)
BUG_ON(!is_fp_enabled());
}
static void init_restore_lsx(void)
{
enable_lsx();
if (!thread_lsx_context_live()) {
/* First time LSX context user */
init_restore_fp();
init_lsx_upper();
set_thread_flag(TIF_LSX_CTX_LIVE);
} else {
if (!is_simd_owner()) {
if (is_fpu_owner()) {
restore_lsx_upper(current);
} else {
__own_fpu();
restore_lsx(current);
}
}
}
set_thread_flag(TIF_USEDSIMD);
BUG_ON(!is_fp_enabled());
BUG_ON(!is_lsx_enabled());
}
static void init_restore_lasx(void)
{
enable_lasx();
if (!thread_lasx_context_live()) {
/* First time LASX context user */
init_restore_lsx();
init_lasx_upper();
set_thread_flag(TIF_LASX_CTX_LIVE);
} else {
if (is_fpu_owner() || is_simd_owner()) {
init_restore_lsx();
restore_lasx_upper(current);
} else {
__own_fpu();
enable_lsx();
restore_lasx(current);
}
}
set_thread_flag(TIF_USEDSIMD);
BUG_ON(!is_fp_enabled());
BUG_ON(!is_lsx_enabled());
BUG_ON(!is_lasx_enabled());
}
asmlinkage void noinstr do_fpu(struct pt_regs *regs)
{
irqentry_state_t state = irqentry_enter(regs);
local_irq_enable();
die_if_kernel("do_fpu invoked from kernel context!", regs);
BUG_ON(is_lsx_enabled());
BUG_ON(is_lasx_enabled());
preempt_disable();
init_restore_fp();
......@@ -872,9 +926,20 @@ asmlinkage void noinstr do_lsx(struct pt_regs *regs)
irqentry_state_t state = irqentry_enter(regs);
local_irq_enable();
force_sig(SIGILL);
local_irq_disable();
if (!cpu_has_lsx) {
force_sig(SIGILL);
goto out;
}
die_if_kernel("do_lsx invoked from kernel context!", regs);
BUG_ON(is_lasx_enabled());
preempt_disable();
init_restore_lsx();
preempt_enable();
out:
local_irq_disable();
irqentry_exit(regs, state);
}
......@@ -883,9 +948,19 @@ asmlinkage void noinstr do_lasx(struct pt_regs *regs)
irqentry_state_t state = irqentry_enter(regs);
local_irq_enable();
force_sig(SIGILL);
local_irq_disable();
if (!cpu_has_lasx) {
force_sig(SIGILL);
goto out;
}
die_if_kernel("do_lasx invoked from kernel context!", regs);
preempt_disable();
init_restore_lasx();
preempt_enable();
out:
local_irq_disable();
irqentry_exit(regs, state);
}
......@@ -924,7 +999,7 @@ asmlinkage void cache_parity_error(void)
/* For the moment, report the problem and hang. */
pr_err("Cache error exception:\n");
pr_err("csr_merrctl == %08x\n", csr_read32(LOONGARCH_CSR_MERRCTL));
pr_err("csr_merrera == %016llx\n", csr_read64(LOONGARCH_CSR_MERRERA));
pr_err("csr_merrera == %016lx\n", csr_read64(LOONGARCH_CSR_MERRERA));
panic("Can't handle the cache error!");
}
......
......@@ -485,8 +485,6 @@ static int __init debugfs_unaligned(void)
struct dentry *d;
d = debugfs_create_dir("loongarch", NULL);
if (IS_ERR_OR_NULL(d))
return -ENOMEM;
debugfs_create_u32("unaligned_instructions_user",
S_IRUGO, d, &unaligned_instructions_user);
......
// SPDX-License-Identifier: GPL-2.0-only
#include <linux/highmem.h>
#include <linux/ptrace.h>
#include <linux/sched.h>
#include <linux/uprobes.h>
#include <asm/cacheflush.h>
#define UPROBE_TRAP_NR UINT_MAX
int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe,
struct mm_struct *mm, unsigned long addr)
{
int idx;
union loongarch_instruction insn;
if (addr & 0x3)
return -EILSEQ;
for (idx = ARRAY_SIZE(auprobe->insn) - 1; idx >= 0; idx--) {
insn.word = auprobe->insn[idx];
if (insns_not_supported(insn))
return -EINVAL;
}
if (insns_need_simulation(insn)) {
auprobe->ixol[0] = larch_insn_gen_nop();
auprobe->simulate = true;
} else {
auprobe->ixol[0] = auprobe->insn[0];
auprobe->simulate = false;
}
auprobe->ixol[1] = UPROBE_XOLBP_INSN;
return 0;
}
int arch_uprobe_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
{
struct uprobe_task *utask = current->utask;
utask->autask.saved_trap_nr = current->thread.trap_nr;
current->thread.trap_nr = UPROBE_TRAP_NR;
instruction_pointer_set(regs, utask->xol_vaddr);
user_enable_single_step(current);
return 0;
}
int arch_uprobe_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
{
struct uprobe_task *utask = current->utask;
WARN_ON_ONCE(current->thread.trap_nr != UPROBE_TRAP_NR);
current->thread.trap_nr = utask->autask.saved_trap_nr;
if (auprobe->simulate)
instruction_pointer_set(regs, auprobe->resume_era);
else
instruction_pointer_set(regs, utask->vaddr + LOONGARCH_INSN_SIZE);
user_disable_single_step(current);
return 0;
}
void arch_uprobe_abort_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
{
struct uprobe_task *utask = current->utask;
current->thread.trap_nr = utask->autask.saved_trap_nr;
instruction_pointer_set(regs, utask->vaddr);
user_disable_single_step(current);
}
bool arch_uprobe_xol_was_trapped(struct task_struct *t)
{
if (t->thread.trap_nr != UPROBE_TRAP_NR)
return true;
return false;
}
bool arch_uprobe_skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs)
{
union loongarch_instruction insn;
if (!auprobe->simulate)
return false;
insn.word = auprobe->insn[0];
arch_simulate_insn(insn, regs);
auprobe->resume_era = regs->csr_era;
return true;
}
unsigned long arch_uretprobe_hijack_return_addr(unsigned long trampoline_vaddr,
struct pt_regs *regs)
{
unsigned long ra = regs->regs[1];
regs->regs[1] = trampoline_vaddr;
return ra;
}
bool arch_uretprobe_is_alive(struct return_instance *ret,
enum rp_check ctx, struct pt_regs *regs)
{
if (ctx == RP_CHECK_CHAIN_CALL)
return regs->regs[3] <= ret->stack;
else
return regs->regs[3] < ret->stack;
}
int arch_uprobe_exception_notify(struct notifier_block *self,
unsigned long val, void *data)
{
return NOTIFY_DONE;
}
bool uprobe_breakpoint_handler(struct pt_regs *regs)
{
if (uprobe_pre_sstep_notifier(regs))
return true;
return false;
}
bool uprobe_singlestep_handler(struct pt_regs *regs)
{
if (uprobe_post_sstep_notifier(regs))
return true;
return false;
}
unsigned long uprobe_get_swbp_addr(struct pt_regs *regs)
{
return instruction_pointer(regs);
}
void arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr,
void *src, unsigned long len)
{
void *kaddr = kmap_local_page(page);
void *dst = kaddr + (vaddr & ~PAGE_MASK);
memcpy(dst, src, len);
flush_icache_range((unsigned long)dst, (unsigned long)dst + len);
kunmap_local(kaddr);
}
......@@ -14,6 +14,7 @@
#include <linux/random.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/time_namespace.h>
#include <linux/timekeeper_internal.h>
#include <asm/page.h>
......@@ -26,12 +27,17 @@ extern char vdso_start[], vdso_end[];
/* Kernel-provided data used by the VDSO. */
static union {
u8 page[VDSO_DATA_SIZE];
u8 page[PAGE_SIZE];
struct vdso_data data[CS_BASES];
} generic_vdso_data __page_aligned_data;
static union {
u8 page[LOONGARCH_VDSO_DATA_SIZE];
struct loongarch_vdso_data vdata;
} loongarch_vdso_data __page_aligned_data;
static struct page *vdso_pages[] = { NULL };
struct vdso_data *vdso_data = loongarch_vdso_data.vdata.data;
struct vdso_data *vdso_data = generic_vdso_data.data;
struct vdso_pcpu_data *vdso_pdata = loongarch_vdso_data.vdata.pdata;
static int vdso_mremap(const struct vm_special_mapping *sm, struct vm_area_struct *new_vma)
......@@ -41,6 +47,43 @@ static int vdso_mremap(const struct vm_special_mapping *sm, struct vm_area_struc
return 0;
}
static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
struct vm_area_struct *vma, struct vm_fault *vmf)
{
unsigned long pfn;
struct page *timens_page = find_timens_vvar_page(vma);
switch (vmf->pgoff) {
case VVAR_GENERIC_PAGE_OFFSET:
if (!timens_page)
pfn = sym_to_pfn(vdso_data);
else
pfn = page_to_pfn(timens_page);
break;
#ifdef CONFIG_TIME_NS
case VVAR_TIMENS_PAGE_OFFSET:
/*
* If a task belongs to a time namespace then a namespace specific
* VVAR is mapped with the VVAR_GENERIC_PAGE_OFFSET and the real
* VVAR page is mapped with the VVAR_TIMENS_PAGE_OFFSET offset.
* See also the comment near timens_setup_vdso_data().
*/
if (!timens_page)
return VM_FAULT_SIGBUS;
else
pfn = sym_to_pfn(vdso_data);
break;
#endif /* CONFIG_TIME_NS */
case VVAR_LOONGARCH_PAGES_START ... VVAR_LOONGARCH_PAGES_END:
pfn = sym_to_pfn(&loongarch_vdso_data) + vmf->pgoff - VVAR_LOONGARCH_PAGES_START;
break;
default:
return VM_FAULT_SIGBUS;
}
return vmf_insert_pfn(vma, vmf->address, pfn);
}
struct loongarch_vdso_info vdso_info = {
.vdso = vdso_start,
.size = PAGE_SIZE,
......@@ -51,6 +94,7 @@ struct loongarch_vdso_info vdso_info = {
},
.data_mapping = {
.name = "[vvar]",
.fault = vvar_fault,
},
.offset_sigreturn = vdso_offset_sigreturn,
};
......@@ -73,6 +117,37 @@ static int __init init_vdso(void)
}
subsys_initcall(init_vdso);
#ifdef CONFIG_TIME_NS
struct vdso_data *arch_get_vdso_data(void *vvar_page)
{
return (struct vdso_data *)(vvar_page);
}
/*
* The vvar mapping contains data for a specific time namespace, so when a
* task changes namespace we must unmap its vvar data for the old namespace.
* Subsequent faults will map in data for the new namespace.
*
* For more details see timens_setup_vdso_data().
*/
int vdso_join_timens(struct task_struct *task, struct time_namespace *ns)
{
struct mm_struct *mm = task->mm;
struct vm_area_struct *vma;
VMA_ITERATOR(vmi, mm, 0);
mmap_read_lock(mm);
for_each_vma(vmi, vma) {
if (vma_is_special_mapping(vma, &vdso_info.data_mapping))
zap_vma_pages(vma);
}
mmap_read_unlock(mm);
return 0;
}
#endif
static unsigned long vdso_base(void)
{
unsigned long base = STACK_TOP;
......@@ -88,7 +163,7 @@ static unsigned long vdso_base(void)
int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
{
int ret;
unsigned long vvar_size, size, data_addr, vdso_addr;
unsigned long size, data_addr, vdso_addr;
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma;
struct loongarch_vdso_info *info = current->thread.vdso;
......@@ -100,32 +175,23 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
* Determine total area size. This includes the VDSO data itself
* and the data pages.
*/
vvar_size = VDSO_DATA_SIZE;
size = vvar_size + info->size;
size = VVAR_SIZE + info->size;
data_addr = get_unmapped_area(NULL, vdso_base(), size, 0, 0);
if (IS_ERR_VALUE(data_addr)) {
ret = data_addr;
goto out;
}
vdso_addr = data_addr + VDSO_DATA_SIZE;
vma = _install_special_mapping(mm, data_addr, vvar_size,
VM_READ | VM_MAYREAD,
vma = _install_special_mapping(mm, data_addr, VVAR_SIZE,
VM_READ | VM_MAYREAD | VM_PFNMAP,
&info->data_mapping);
if (IS_ERR(vma)) {
ret = PTR_ERR(vma);
goto out;
}
/* Map VDSO data page. */
ret = remap_pfn_range(vma, data_addr,
virt_to_phys(&loongarch_vdso_data) >> PAGE_SHIFT,
vvar_size, PAGE_READONLY);
if (ret)
goto out;
/* Map VDSO code page. */
vdso_addr = data_addr + VVAR_SIZE;
vma = _install_special_mapping(mm, vdso_addr, info->size,
VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC,
&info->code_mapping);
......
......@@ -136,6 +136,15 @@ SECTIONS
DWARF_DEBUG
ELF_DETAILS
#ifdef CONFIG_EFI_STUB
/* header symbols */
_kernel_asize = _end - _text;
_kernel_fsize = _edata - _text;
_kernel_vsize = _end - __initdata_begin;
_kernel_rsize = _edata - __initdata_begin;
_kernel_offset = kernel_offset - _text;
#endif
.gptab.sdata : {
*(.gptab.data)
*(.gptab.sdata)
......
......@@ -20,9 +20,9 @@ void dump_tlb_regs(void)
pr_info("Index : 0x%0x\n", read_csr_tlbidx());
pr_info("PageSize : 0x%0x\n", read_csr_pagesize());
pr_info("EntryHi : 0x%0*llx\n", field, read_csr_entryhi());
pr_info("EntryLo0 : 0x%0*llx\n", field, read_csr_entrylo0());
pr_info("EntryLo1 : 0x%0*llx\n", field, read_csr_entrylo1());
pr_info("EntryHi : 0x%0*lx\n", field, read_csr_entryhi());
pr_info("EntryLo0 : 0x%0*lx\n", field, read_csr_entrylo0());
pr_info("EntryLo1 : 0x%0*lx\n", field, read_csr_entrylo1());
}
static void dump_tlb(int first, int last)
......
......@@ -167,6 +167,9 @@ void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t *ptep
int idx;
unsigned long flags;
if (cpu_has_ptw)
return;
/*
* Handle debugger faulting in for debugee.
*/
......@@ -222,6 +225,9 @@ static void setup_ptwalker(void)
pwctl0 = pte_i | pte_w << 5 | pmd_i << 10 | pmd_w << 15 | pud_i << 20 | pud_w << 25;
pwctl1 = pgd_i | pgd_w << 6;
if (cpu_has_ptw)
pwctl1 |= CSR_PWCTL1_PTW;
csr_write64(pwctl0, LOONGARCH_CSR_PWCTL0);
csr_write64(pwctl1, LOONGARCH_CSR_PWCTL1);
csr_write64((long)swapper_pg_dir, LOONGARCH_CSR_PGDH);
......@@ -264,10 +270,17 @@ void setup_tlb_handler(int cpu)
if (cpu == 0) {
memcpy((void *)tlbrentry, handle_tlb_refill, 0x80);
local_flush_icache_range(tlbrentry, tlbrentry + 0x80);
set_handler(EXCCODE_TLBI * VECSIZE, handle_tlb_load, VECSIZE);
set_handler(EXCCODE_TLBL * VECSIZE, handle_tlb_load, VECSIZE);
set_handler(EXCCODE_TLBS * VECSIZE, handle_tlb_store, VECSIZE);
set_handler(EXCCODE_TLBM * VECSIZE, handle_tlb_modify, VECSIZE);
if (!cpu_has_ptw) {
set_handler(EXCCODE_TLBI * VECSIZE, handle_tlb_load, VECSIZE);
set_handler(EXCCODE_TLBL * VECSIZE, handle_tlb_load, VECSIZE);
set_handler(EXCCODE_TLBS * VECSIZE, handle_tlb_store, VECSIZE);
set_handler(EXCCODE_TLBM * VECSIZE, handle_tlb_modify, VECSIZE);
} else {
set_handler(EXCCODE_TLBI * VECSIZE, handle_tlb_load_ptw, VECSIZE);
set_handler(EXCCODE_TLBL * VECSIZE, handle_tlb_load_ptw, VECSIZE);
set_handler(EXCCODE_TLBS * VECSIZE, handle_tlb_store_ptw, VECSIZE);
set_handler(EXCCODE_TLBM * VECSIZE, handle_tlb_modify_ptw, VECSIZE);
}
set_handler(EXCCODE_TLBNR * VECSIZE, handle_tlb_protect, VECSIZE);
set_handler(EXCCODE_TLBNX * VECSIZE, handle_tlb_protect, VECSIZE);
set_handler(EXCCODE_TLBPE * VECSIZE, handle_tlb_protect, VECSIZE);
......
......@@ -184,12 +184,19 @@ tlb_huge_update_load:
ertn
nopage_tlb_load:
dbar 0
dbar 0x700
csrrd ra, EXCEPTION_KS2
la_abs t0, tlb_do_page_fault_0
jr t0
SYM_FUNC_END(handle_tlb_load)
SYM_FUNC_START(handle_tlb_load_ptw)
csrwr t0, LOONGARCH_CSR_KS0
csrwr t1, LOONGARCH_CSR_KS1
la_abs t0, tlb_do_page_fault_0
jr t0
SYM_FUNC_END(handle_tlb_load_ptw)
SYM_FUNC_START(handle_tlb_store)
csrwr t0, EXCEPTION_KS0
csrwr t1, EXCEPTION_KS1
......@@ -333,12 +340,19 @@ tlb_huge_update_store:
ertn
nopage_tlb_store:
dbar 0
dbar 0x700
csrrd ra, EXCEPTION_KS2
la_abs t0, tlb_do_page_fault_1
jr t0
SYM_FUNC_END(handle_tlb_store)
SYM_FUNC_START(handle_tlb_store_ptw)
csrwr t0, LOONGARCH_CSR_KS0
csrwr t1, LOONGARCH_CSR_KS1
la_abs t0, tlb_do_page_fault_1
jr t0
SYM_FUNC_END(handle_tlb_store_ptw)
SYM_FUNC_START(handle_tlb_modify)
csrwr t0, EXCEPTION_KS0
csrwr t1, EXCEPTION_KS1
......@@ -480,12 +494,19 @@ tlb_huge_update_modify:
ertn
nopage_tlb_modify:
dbar 0
dbar 0x700
csrrd ra, EXCEPTION_KS2
la_abs t0, tlb_do_page_fault_1
jr t0
SYM_FUNC_END(handle_tlb_modify)
SYM_FUNC_START(handle_tlb_modify_ptw)
csrwr t0, LOONGARCH_CSR_KS0
csrwr t1, LOONGARCH_CSR_KS1
la_abs t0, tlb_do_page_fault_1
jr t0
SYM_FUNC_END(handle_tlb_modify_ptw)
SYM_FUNC_START(handle_tlb_refill)
csrwr t0, LOONGARCH_CSR_TLBRSAVE
csrrd t0, LOONGARCH_CSR_PGD
......
......@@ -27,7 +27,7 @@ struct saved_registers {
};
static struct saved_registers saved_regs;
static void arch_common_suspend(void)
void loongarch_common_suspend(void)
{
save_counter();
saved_regs.pgd = csr_read64(LOONGARCH_CSR_PGDL);
......@@ -40,7 +40,7 @@ static void arch_common_suspend(void)
loongarch_suspend_addr = loongson_sysconf.suspend_addr;
}
static void arch_common_resume(void)
void loongarch_common_resume(void)
{
sync_counter();
local_flush_tlb_all();
......@@ -62,12 +62,12 @@ int loongarch_acpi_suspend(void)
enable_gpe_wakeup();
enable_pci_wakeup();
arch_common_suspend();
loongarch_common_suspend();
/* processor specific suspend */
loongarch_suspend_enter();
arch_common_resume();
loongarch_common_resume();
return 0;
}
......@@ -12,16 +12,13 @@ ccflags-vdso := \
$(filter -E%,$(KBUILD_CFLAGS)) \
$(filter -march=%,$(KBUILD_CFLAGS)) \
$(filter -m%-float,$(KBUILD_CFLAGS)) \
$(CLANG_FLAGS) \
-D__VDSO__
ifeq ($(cc-name),clang)
ccflags-vdso += $(filter --target=%,$(KBUILD_CFLAGS))
endif
cflags-vdso := $(ccflags-vdso) \
-isystem $(shell $(CC) -print-file-name=include) \
$(filter -W%,$(filter-out -Wa$(comma)%,$(KBUILD_CFLAGS))) \
-O2 -g -fno-strict-aliasing -fno-common -fno-builtin -G0 \
-O2 -g -fno-strict-aliasing -fno-common -fno-builtin \
-fno-stack-protector -fno-jump-tables -DDISABLE_BRANCH_PROFILING \
$(call cc-option, -fno-asynchronous-unwind-tables) \
$(call cc-option, -fno-stack-protector)
......
......@@ -21,7 +21,7 @@ static __always_inline int read_cpu_id(void)
static __always_inline const struct vdso_pcpu_data *get_pcpu_data(void)
{
return (struct vdso_pcpu_data *)(get_vdso_base() - VDSO_DATA_SIZE);
return (struct vdso_pcpu_data *)(get_vdso_data() + VVAR_LOONGARCH_PAGES_START * PAGE_SIZE);
}
extern
......
......@@ -542,10 +542,10 @@ config ACPI_PFRUT
if ARM64
source "drivers/acpi/arm64/Kconfig"
endif
config ACPI_PPTT
bool
endif
config ACPI_PCC
bool "ACPI PCC Address Space"
......
......@@ -4,6 +4,7 @@
CLANG_TARGET_FLAGS_arm := arm-linux-gnueabi
CLANG_TARGET_FLAGS_arm64 := aarch64-linux-gnu
CLANG_TARGET_FLAGS_hexagon := hexagon-linux-musl
CLANG_TARGET_FLAGS_loongarch := loongarch64-linux-gnusf
CLANG_TARGET_FLAGS_m68k := m68k-linux-gnu
CLANG_TARGET_FLAGS_mips := mipsel-linux-gnu
CLANG_TARGET_FLAGS_powerpc := powerpc64le-linux-gnu
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment