entry.S 18 KB
Newer Older
Catalin Marinas's avatar
Catalin Marinas committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
/*
 * Low-level exception handling code
 *
 * Copyright (C) 2012 ARM Ltd.
 * Authors:	Catalin Marinas <catalin.marinas@arm.com>
 *		Will Deacon <will.deacon@arm.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <linux/init.h>
#include <linux/linkage.h>

24
#include <asm/alternative.h>
Catalin Marinas's avatar
Catalin Marinas committed
25 26
#include <asm/assembler.h>
#include <asm/asm-offsets.h>
27
#include <asm/cpufeature.h>
Catalin Marinas's avatar
Catalin Marinas committed
28
#include <asm/errno.h>
29
#include <asm/esr.h>
30
#include <asm/irq.h>
31
#include <asm/memory.h>
Catalin Marinas's avatar
Catalin Marinas committed
32 33 34
#include <asm/thread_info.h>
#include <asm/unistd.h>

35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60
/*
 * Context tracking subsystem.  Used to instrument transitions
 * between user and kernel mode.
 */
	.macro ct_user_exit, syscall = 0
#ifdef CONFIG_CONTEXT_TRACKING
	bl	context_tracking_user_exit
	.if \syscall == 1
	/*
	 * Save/restore needed during syscalls.  Restore syscall arguments from
	 * the values already saved on stack during kernel_entry.
	 */
	ldp	x0, x1, [sp]
	ldp	x2, x3, [sp, #S_X2]
	ldp	x4, x5, [sp, #S_X4]
	ldp	x6, x7, [sp, #S_X6]
	.endif
#endif
	.endm

	.macro ct_user_enter
#ifdef CONFIG_CONTEXT_TRACKING
	bl	context_tracking_user_enter
#endif
	.endm

Catalin Marinas's avatar
Catalin Marinas committed
61 62 63 64 65 66 67 68 69 70
/*
 * Bad Abort numbers
 *-----------------
 */
#define BAD_SYNC	0
#define BAD_IRQ		1
#define BAD_FIQ		2
#define BAD_ERROR	3

	.macro	kernel_entry, el, regsize = 64
71
	sub	sp, sp, #S_FRAME_SIZE
Catalin Marinas's avatar
Catalin Marinas committed
72 73 74
	.if	\regsize == 32
	mov	w0, w0				// zero upper 32 bits of x0
	.endif
75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90
	stp	x0, x1, [sp, #16 * 0]
	stp	x2, x3, [sp, #16 * 1]
	stp	x4, x5, [sp, #16 * 2]
	stp	x6, x7, [sp, #16 * 3]
	stp	x8, x9, [sp, #16 * 4]
	stp	x10, x11, [sp, #16 * 5]
	stp	x12, x13, [sp, #16 * 6]
	stp	x14, x15, [sp, #16 * 7]
	stp	x16, x17, [sp, #16 * 8]
	stp	x18, x19, [sp, #16 * 9]
	stp	x20, x21, [sp, #16 * 10]
	stp	x22, x23, [sp, #16 * 11]
	stp	x24, x25, [sp, #16 * 12]
	stp	x26, x27, [sp, #16 * 13]
	stp	x28, x29, [sp, #16 * 14]

Catalin Marinas's avatar
Catalin Marinas committed
91 92
	.if	\el == 0
	mrs	x21, sp_el0
93 94
	mov	tsk, sp
	and	tsk, tsk, #~(THREAD_SIZE - 1)	// Ensure MDSCR_EL1.SS is clear,
95 96
	ldr	x19, [tsk, #TI_FLAGS]		// since we can unmask debug
	disable_step_tsk x19, x20		// exceptions when scheduling.
97 98

	mov	x29, xzr			// fp pointed to user-space
Catalin Marinas's avatar
Catalin Marinas committed
99 100
	.else
	add	x21, sp, #S_FRAME_SIZE
101 102 103 104 105 106
	get_thread_info tsk
	/* Save the task's original addr_limit and set USER_DS (TASK_SIZE_64) */
	ldr	x20, [tsk, #TI_ADDR_LIMIT]
	str	x20, [sp, #S_ORIG_ADDR_LIMIT]
	mov	x20, #TASK_SIZE_64
	str	x20, [tsk, #TI_ADDR_LIMIT]
107
	/* No need to reset PSTATE.UAO, hardware's already set it to 0 for us */
108
	.endif /* \el == 0 */
Catalin Marinas's avatar
Catalin Marinas committed
109 110 111 112 113 114 115 116 117 118 119 120 121
	mrs	x22, elr_el1
	mrs	x23, spsr_el1
	stp	lr, x21, [sp, #S_LR]
	stp	x22, x23, [sp, #S_PC]

	/*
	 * Set syscallno to -1 by default (overridden later if real syscall).
	 */
	.if	\el == 0
	mvn	x21, xzr
	str	x21, [sp, #S_SYSCALLNO]
	.endif

122 123 124 125 126 127 128
	/*
	 * Set sp_el0 to current thread_info.
	 */
	.if	\el == 0
	msr	sp_el0, tsk
	.endif

Catalin Marinas's avatar
Catalin Marinas committed
129 130 131 132 133 134 135 136 137
	/*
	 * Registers that may be useful after this macro is invoked:
	 *
	 * x21 - aborted SP
	 * x22 - aborted PC
	 * x23 - aborted PSTATE
	*/
	.endm

138
	.macro	kernel_exit, el
139 140 141 142 143 144 145 146
	.if	\el != 0
	/* Restore the task's original addr_limit. */
	ldr	x20, [sp, #S_ORIG_ADDR_LIMIT]
	str	x20, [tsk, #TI_ADDR_LIMIT]

	/* No need to restore UAO, it will be restored from SPSR_EL1 */
	.endif

Catalin Marinas's avatar
Catalin Marinas committed
147 148
	ldp	x21, x22, [sp, #S_PC]		// load ELR, SPSR
	.if	\el == 0
149
	ct_user_enter
Catalin Marinas's avatar
Catalin Marinas committed
150
	ldr	x23, [sp, #S_SP]		// load return stack pointer
151
	msr	sp_el0, x23
152
#ifdef CONFIG_ARM64_ERRATUM_845719
153
alternative_if ARM64_WORKAROUND_845719
154 155 156 157
	tbz	x22, #4, 1f
#ifdef CONFIG_PID_IN_CONTEXTIDR
	mrs	x29, contextidr_el1
	msr	contextidr_el1, x29
158
#else
159
	msr contextidr_el1, xzr
160
#endif
161
1:
162
alternative_else_nop_endif
163
#endif
Catalin Marinas's avatar
Catalin Marinas committed
164
	.endif
165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183
	msr	elr_el1, x21			// set up the return data
	msr	spsr_el1, x22
	ldp	x0, x1, [sp, #16 * 0]
	ldp	x2, x3, [sp, #16 * 1]
	ldp	x4, x5, [sp, #16 * 2]
	ldp	x6, x7, [sp, #16 * 3]
	ldp	x8, x9, [sp, #16 * 4]
	ldp	x10, x11, [sp, #16 * 5]
	ldp	x12, x13, [sp, #16 * 6]
	ldp	x14, x15, [sp, #16 * 7]
	ldp	x16, x17, [sp, #16 * 8]
	ldp	x18, x19, [sp, #16 * 9]
	ldp	x20, x21, [sp, #16 * 10]
	ldp	x22, x23, [sp, #16 * 11]
	ldp	x24, x25, [sp, #16 * 12]
	ldp	x26, x27, [sp, #16 * 13]
	ldp	x28, x29, [sp, #16 * 14]
	ldr	lr, [sp, #S_LR]
	add	sp, sp, #S_FRAME_SIZE		// restore sp
Catalin Marinas's avatar
Catalin Marinas committed
184 185 186 187
	eret					// return to kernel
	.endm

	.macro	get_thread_info, rd
188
	mrs	\rd, sp_el0
Catalin Marinas's avatar
Catalin Marinas committed
189 190
	.endm

191
	.macro	irq_stack_entry
192 193 194
	mov	x19, sp			// preserve the original sp

	/*
195 196 197
	 * Compare sp with the current thread_info, if the top
	 * ~(THREAD_SIZE - 1) bits match, we are on a task stack, and
	 * should switch to the irq stack.
198
	 */
199 200 201
	and	x25, x19, #~(THREAD_SIZE - 1)
	cmp	x25, tsk
	b.ne	9998f
202

203
	this_cpu_ptr irq_stack, x25, x26
204 205
	mov	x26, #IRQ_STACK_START_SP
	add	x26, x25, x26
206 207

	/* switch to the irq stack */
208 209
	mov	sp, x26

210 211 212 213 214
	/*
	 * Add a dummy stack frame, this non-standard format is fixed up
	 * by unwind_frame()
	 */
	stp     x29, x19, [sp, #-16]!
215 216 217 218 219 220 221 222 223 224 225 226 227
	mov	x29, sp

9998:
	.endm

	/*
	 * x19 should be preserved between irq_stack_entry and
	 * irq_stack_exit.
	 */
	.macro	irq_stack_exit
	mov	sp, x19
	.endm

Catalin Marinas's avatar
Catalin Marinas committed
228 229 230 231 232 233 234 235 236 237 238 239 240 241 242
/*
 * These are the registers used in the syscall handler, and allow us to
 * have in theory up to 7 arguments to a function - x0 to x6.
 *
 * x7 is reserved for the system call number in 32-bit mode.
 */
sc_nr	.req	x25		// number of system calls
scno	.req	x26		// syscall number
stbl	.req	x27		// syscall table pointer
tsk	.req	x28		// current thread_info

/*
 * Interrupt handling.
 */
	.macro	irq_handler
243
	ldr_l	x1, handle_arch_irq
Catalin Marinas's avatar
Catalin Marinas committed
244
	mov	x0, sp
245
	irq_stack_entry
Catalin Marinas's avatar
Catalin Marinas committed
246
	blr	x1
247
	irq_stack_exit
Catalin Marinas's avatar
Catalin Marinas committed
248 249 250 251 252 253 254
	.endm

	.text

/*
 * Exception vectors.
 */
255
	.pushsection ".entry.text", "ax"
Catalin Marinas's avatar
Catalin Marinas committed
256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290

	.align	11
ENTRY(vectors)
	ventry	el1_sync_invalid		// Synchronous EL1t
	ventry	el1_irq_invalid			// IRQ EL1t
	ventry	el1_fiq_invalid			// FIQ EL1t
	ventry	el1_error_invalid		// Error EL1t

	ventry	el1_sync			// Synchronous EL1h
	ventry	el1_irq				// IRQ EL1h
	ventry	el1_fiq_invalid			// FIQ EL1h
	ventry	el1_error_invalid		// Error EL1h

	ventry	el0_sync			// Synchronous 64-bit EL0
	ventry	el0_irq				// IRQ 64-bit EL0
	ventry	el0_fiq_invalid			// FIQ 64-bit EL0
	ventry	el0_error_invalid		// Error 64-bit EL0

#ifdef CONFIG_COMPAT
	ventry	el0_sync_compat			// Synchronous 32-bit EL0
	ventry	el0_irq_compat			// IRQ 32-bit EL0
	ventry	el0_fiq_invalid_compat		// FIQ 32-bit EL0
	ventry	el0_error_invalid_compat	// Error 32-bit EL0
#else
	ventry	el0_sync_invalid		// Synchronous 32-bit EL0
	ventry	el0_irq_invalid			// IRQ 32-bit EL0
	ventry	el0_fiq_invalid			// FIQ 32-bit EL0
	ventry	el0_error_invalid		// Error 32-bit EL0
#endif
END(vectors)

/*
 * Invalid mode handlers
 */
	.macro	inv_entry, el, reason, regsize = 64
291
	kernel_entry \el, \regsize
Catalin Marinas's avatar
Catalin Marinas committed
292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346
	mov	x0, sp
	mov	x1, #\reason
	mrs	x2, esr_el1
	b	bad_mode
	.endm

el0_sync_invalid:
	inv_entry 0, BAD_SYNC
ENDPROC(el0_sync_invalid)

el0_irq_invalid:
	inv_entry 0, BAD_IRQ
ENDPROC(el0_irq_invalid)

el0_fiq_invalid:
	inv_entry 0, BAD_FIQ
ENDPROC(el0_fiq_invalid)

el0_error_invalid:
	inv_entry 0, BAD_ERROR
ENDPROC(el0_error_invalid)

#ifdef CONFIG_COMPAT
el0_fiq_invalid_compat:
	inv_entry 0, BAD_FIQ, 32
ENDPROC(el0_fiq_invalid_compat)

el0_error_invalid_compat:
	inv_entry 0, BAD_ERROR, 32
ENDPROC(el0_error_invalid_compat)
#endif

el1_sync_invalid:
	inv_entry 1, BAD_SYNC
ENDPROC(el1_sync_invalid)

el1_irq_invalid:
	inv_entry 1, BAD_IRQ
ENDPROC(el1_irq_invalid)

el1_fiq_invalid:
	inv_entry 1, BAD_FIQ
ENDPROC(el1_fiq_invalid)

el1_error_invalid:
	inv_entry 1, BAD_ERROR
ENDPROC(el1_error_invalid)

/*
 * EL1 mode handlers.
 */
	.align	6
el1_sync:
	kernel_entry 1
	mrs	x1, esr_el1			// read the syndrome register
347 348
	lsr	x24, x1, #ESR_ELx_EC_SHIFT	// exception class
	cmp	x24, #ESR_ELx_EC_DABT_CUR	// data abort in EL1
Catalin Marinas's avatar
Catalin Marinas committed
349
	b.eq	el1_da
350 351
	cmp	x24, #ESR_ELx_EC_IABT_CUR	// instruction abort in EL1
	b.eq	el1_ia
352
	cmp	x24, #ESR_ELx_EC_SYS64		// configurable trap
Catalin Marinas's avatar
Catalin Marinas committed
353
	b.eq	el1_undef
354
	cmp	x24, #ESR_ELx_EC_SP_ALIGN	// stack alignment exception
Catalin Marinas's avatar
Catalin Marinas committed
355
	b.eq	el1_sp_pc
356
	cmp	x24, #ESR_ELx_EC_PC_ALIGN	// pc alignment exception
Catalin Marinas's avatar
Catalin Marinas committed
357
	b.eq	el1_sp_pc
358
	cmp	x24, #ESR_ELx_EC_UNKNOWN	// unknown exception in EL1
Catalin Marinas's avatar
Catalin Marinas committed
359
	b.eq	el1_undef
360
	cmp	x24, #ESR_ELx_EC_BREAKPT_CUR	// debug exception in EL1
Catalin Marinas's avatar
Catalin Marinas committed
361 362
	b.ge	el1_dbg
	b	el1_inv
363 364 365 366 367

el1_ia:
	/*
	 * Fall through to the Data abort case
	 */
Catalin Marinas's avatar
Catalin Marinas committed
368 369 370 371 372
el1_da:
	/*
	 * Data abort handling
	 */
	mrs	x0, far_el1
373
	enable_dbg
Catalin Marinas's avatar
Catalin Marinas committed
374 375 376 377 378 379 380 381 382 383 384 385 386 387 388
	// re-enable interrupts if they were enabled in the aborted context
	tbnz	x23, #7, 1f			// PSR_I_BIT
	enable_irq
1:
	mov	x2, sp				// struct pt_regs
	bl	do_mem_abort

	// disable interrupts before pulling preserved data off the stack
	disable_irq
	kernel_exit 1
el1_sp_pc:
	/*
	 * Stack or PC alignment exception handling
	 */
	mrs	x0, far_el1
389
	enable_dbg
Catalin Marinas's avatar
Catalin Marinas committed
390 391 392 393 394 395
	mov	x2, sp
	b	do_sp_pc_abort
el1_undef:
	/*
	 * Undefined instruction
	 */
396
	enable_dbg
Catalin Marinas's avatar
Catalin Marinas committed
397 398 399 400 401 402
	mov	x0, sp
	b	do_undefinstr
el1_dbg:
	/*
	 * Debug exception handling
	 */
403
	cmp	x24, #ESR_ELx_EC_BRK64		// if BRK64
404
	cinc	x24, x24, eq			// set bit '0'
Catalin Marinas's avatar
Catalin Marinas committed
405 406 407 408 409 410 411
	tbz	x24, #0, el1_inv		// EL1 only
	mrs	x0, far_el1
	mov	x2, sp				// struct pt_regs
	bl	do_debug_exception
	kernel_exit 1
el1_inv:
	// TODO: add support for undefined instructions in kernel mode
412
	enable_dbg
Catalin Marinas's avatar
Catalin Marinas committed
413
	mov	x0, sp
414
	mov	x2, x1
Catalin Marinas's avatar
Catalin Marinas committed
415 416 417 418 419 420 421
	mov	x1, #BAD_SYNC
	b	bad_mode
ENDPROC(el1_sync)

	.align	6
el1_irq:
	kernel_entry 1
422
	enable_dbg
Catalin Marinas's avatar
Catalin Marinas committed
423 424 425
#ifdef CONFIG_TRACE_IRQFLAGS
	bl	trace_hardirqs_off
#endif
426

Catalin Marinas's avatar
Catalin Marinas committed
427
	irq_handler
428

Catalin Marinas's avatar
Catalin Marinas committed
429
#ifdef CONFIG_PREEMPT
Neil Zhang's avatar
Neil Zhang committed
430
	ldr	w24, [tsk, #TI_PREEMPT]		// get preempt count
431
	cbnz	w24, 1f				// preempt count != 0
Catalin Marinas's avatar
Catalin Marinas committed
432 433 434 435 436 437 438 439 440 441 442 443 444 445
	ldr	x0, [tsk, #TI_FLAGS]		// get flags
	tbz	x0, #TIF_NEED_RESCHED, 1f	// needs rescheduling?
	bl	el1_preempt
1:
#endif
#ifdef CONFIG_TRACE_IRQFLAGS
	bl	trace_hardirqs_on
#endif
	kernel_exit 1
ENDPROC(el1_irq)

#ifdef CONFIG_PREEMPT
el1_preempt:
	mov	x24, lr
446
1:	bl	preempt_schedule_irq		// irq en/disable is done inside
Catalin Marinas's avatar
Catalin Marinas committed
447 448 449 450 451 452 453 454 455 456 457 458
	ldr	x0, [tsk, #TI_FLAGS]		// get new tasks TI_FLAGS
	tbnz	x0, #TIF_NEED_RESCHED, 1b	// needs rescheduling?
	ret	x24
#endif

/*
 * EL0 mode handlers.
 */
	.align	6
el0_sync:
	kernel_entry 0
	mrs	x25, esr_el1			// read the syndrome register
459 460
	lsr	x24, x25, #ESR_ELx_EC_SHIFT	// exception class
	cmp	x24, #ESR_ELx_EC_SVC64		// SVC in 64-bit state
Catalin Marinas's avatar
Catalin Marinas committed
461
	b.eq	el0_svc
462
	cmp	x24, #ESR_ELx_EC_DABT_LOW	// data abort in EL0
Catalin Marinas's avatar
Catalin Marinas committed
463
	b.eq	el0_da
464
	cmp	x24, #ESR_ELx_EC_IABT_LOW	// instruction abort in EL0
Catalin Marinas's avatar
Catalin Marinas committed
465
	b.eq	el0_ia
466
	cmp	x24, #ESR_ELx_EC_FP_ASIMD	// FP/ASIMD access
Catalin Marinas's avatar
Catalin Marinas committed
467
	b.eq	el0_fpsimd_acc
468
	cmp	x24, #ESR_ELx_EC_FP_EXC64	// FP/ASIMD exception
Catalin Marinas's avatar
Catalin Marinas committed
469
	b.eq	el0_fpsimd_exc
470
	cmp	x24, #ESR_ELx_EC_SYS64		// configurable trap
471
	b.eq	el0_sys
472
	cmp	x24, #ESR_ELx_EC_SP_ALIGN	// stack alignment exception
Catalin Marinas's avatar
Catalin Marinas committed
473
	b.eq	el0_sp_pc
474
	cmp	x24, #ESR_ELx_EC_PC_ALIGN	// pc alignment exception
Catalin Marinas's avatar
Catalin Marinas committed
475
	b.eq	el0_sp_pc
476
	cmp	x24, #ESR_ELx_EC_UNKNOWN	// unknown exception in EL0
Catalin Marinas's avatar
Catalin Marinas committed
477
	b.eq	el0_undef
478
	cmp	x24, #ESR_ELx_EC_BREAKPT_LOW	// debug exception in EL0
Catalin Marinas's avatar
Catalin Marinas committed
479 480 481 482 483 484 485 486
	b.ge	el0_dbg
	b	el0_inv

#ifdef CONFIG_COMPAT
	.align	6
el0_sync_compat:
	kernel_entry 0, 32
	mrs	x25, esr_el1			// read the syndrome register
487 488
	lsr	x24, x25, #ESR_ELx_EC_SHIFT	// exception class
	cmp	x24, #ESR_ELx_EC_SVC32		// SVC in 32-bit state
Catalin Marinas's avatar
Catalin Marinas committed
489
	b.eq	el0_svc_compat
490
	cmp	x24, #ESR_ELx_EC_DABT_LOW	// data abort in EL0
Catalin Marinas's avatar
Catalin Marinas committed
491
	b.eq	el0_da
492
	cmp	x24, #ESR_ELx_EC_IABT_LOW	// instruction abort in EL0
Catalin Marinas's avatar
Catalin Marinas committed
493
	b.eq	el0_ia
494
	cmp	x24, #ESR_ELx_EC_FP_ASIMD	// FP/ASIMD access
Catalin Marinas's avatar
Catalin Marinas committed
495
	b.eq	el0_fpsimd_acc
496
	cmp	x24, #ESR_ELx_EC_FP_EXC32	// FP/ASIMD exception
Catalin Marinas's avatar
Catalin Marinas committed
497
	b.eq	el0_fpsimd_exc
498 499
	cmp	x24, #ESR_ELx_EC_PC_ALIGN	// pc alignment exception
	b.eq	el0_sp_pc
500
	cmp	x24, #ESR_ELx_EC_UNKNOWN	// unknown exception in EL0
Catalin Marinas's avatar
Catalin Marinas committed
501
	b.eq	el0_undef
502
	cmp	x24, #ESR_ELx_EC_CP15_32	// CP15 MRC/MCR trap
503
	b.eq	el0_undef
504
	cmp	x24, #ESR_ELx_EC_CP15_64	// CP15 MRRC/MCRR trap
505
	b.eq	el0_undef
506
	cmp	x24, #ESR_ELx_EC_CP14_MR	// CP14 MRC/MCR trap
507
	b.eq	el0_undef
508
	cmp	x24, #ESR_ELx_EC_CP14_LS	// CP14 LDC/STC trap
509
	b.eq	el0_undef
510
	cmp	x24, #ESR_ELx_EC_CP14_64	// CP14 MRRC/MCRR trap
511
	b.eq	el0_undef
512
	cmp	x24, #ESR_ELx_EC_BREAKPT_LOW	// debug exception in EL0
Catalin Marinas's avatar
Catalin Marinas committed
513 514 515 516 517 518
	b.ge	el0_dbg
	b	el0_inv
el0_svc_compat:
	/*
	 * AArch32 syscall handling
	 */
519
	adrp	stbl, compat_sys_call_table	// load compat syscall table pointer
Catalin Marinas's avatar
Catalin Marinas committed
520 521 522 523 524 525 526 527 528 529 530 531 532 533
	uxtw	scno, w7			// syscall number in w7 (r7)
	mov     sc_nr, #__NR_compat_syscalls
	b	el0_svc_naked

	.align	6
el0_irq_compat:
	kernel_entry 0, 32
	b	el0_irq_naked
#endif

el0_da:
	/*
	 * Data abort handling
	 */
534
	mrs	x26, far_el1
Catalin Marinas's avatar
Catalin Marinas committed
535
	// enable interrupts before calling the main handler
536
	enable_dbg_and_irq
537
	ct_user_exit
538
	bic	x0, x26, #(0xff << 56)
Catalin Marinas's avatar
Catalin Marinas committed
539 540
	mov	x1, x25
	mov	x2, sp
541 542
	bl	do_mem_abort
	b	ret_to_user
Catalin Marinas's avatar
Catalin Marinas committed
543 544 545 546
el0_ia:
	/*
	 * Instruction abort handling
	 */
547
	mrs	x26, far_el1
Catalin Marinas's avatar
Catalin Marinas committed
548
	// enable interrupts before calling the main handler
549
	enable_dbg_and_irq
550
	ct_user_exit
551
	mov	x0, x26
Mark Rutland's avatar
Mark Rutland committed
552
	mov	x1, x25
Catalin Marinas's avatar
Catalin Marinas committed
553
	mov	x2, sp
554 555
	bl	do_mem_abort
	b	ret_to_user
Catalin Marinas's avatar
Catalin Marinas committed
556 557 558 559
el0_fpsimd_acc:
	/*
	 * Floating Point or Advanced SIMD access
	 */
560
	enable_dbg
561
	ct_user_exit
Catalin Marinas's avatar
Catalin Marinas committed
562 563
	mov	x0, x25
	mov	x1, sp
564 565
	bl	do_fpsimd_acc
	b	ret_to_user
Catalin Marinas's avatar
Catalin Marinas committed
566 567 568 569
el0_fpsimd_exc:
	/*
	 * Floating Point or Advanced SIMD exception
	 */
570
	enable_dbg
571
	ct_user_exit
Catalin Marinas's avatar
Catalin Marinas committed
572 573
	mov	x0, x25
	mov	x1, sp
574 575
	bl	do_fpsimd_exc
	b	ret_to_user
Catalin Marinas's avatar
Catalin Marinas committed
576 577 578 579
el0_sp_pc:
	/*
	 * Stack or PC alignment exception handling
	 */
580
	mrs	x26, far_el1
Catalin Marinas's avatar
Catalin Marinas committed
581
	// enable interrupts before calling the main handler
582
	enable_dbg_and_irq
583
	ct_user_exit
584
	mov	x0, x26
Catalin Marinas's avatar
Catalin Marinas committed
585 586
	mov	x1, x25
	mov	x2, sp
587 588
	bl	do_sp_pc_abort
	b	ret_to_user
Catalin Marinas's avatar
Catalin Marinas committed
589 590 591 592
el0_undef:
	/*
	 * Undefined instruction
	 */
593
	// enable interrupts before calling the main handler
594
	enable_dbg_and_irq
595
	ct_user_exit
596
	mov	x0, sp
597 598
	bl	do_undefinstr
	b	ret_to_user
599 600 601 602 603 604 605 606 607 608
el0_sys:
	/*
	 * System instructions, for trapped cache maintenance instructions
	 */
	enable_dbg_and_irq
	ct_user_exit
	mov	x0, x25
	mov	x1, sp
	bl	do_sysinstr
	b	ret_to_user
Catalin Marinas's avatar
Catalin Marinas committed
609 610 611 612 613 614 615 616
el0_dbg:
	/*
	 * Debug exception handling
	 */
	tbnz	x24, #0, el0_inv		// EL0 only
	mrs	x0, far_el1
	mov	x1, x25
	mov	x2, sp
617 618
	bl	do_debug_exception
	enable_dbg
619
	ct_user_exit
620
	b	ret_to_user
Catalin Marinas's avatar
Catalin Marinas committed
621
el0_inv:
622
	enable_dbg
623
	ct_user_exit
Catalin Marinas's avatar
Catalin Marinas committed
624 625
	mov	x0, sp
	mov	x1, #BAD_SYNC
626
	mov	x2, x25
627 628
	bl	bad_mode
	b	ret_to_user
Catalin Marinas's avatar
Catalin Marinas committed
629 630 631 632 633 634 635 636 637 638
ENDPROC(el0_sync)

	.align	6
el0_irq:
	kernel_entry 0
el0_irq_naked:
	enable_dbg
#ifdef CONFIG_TRACE_IRQFLAGS
	bl	trace_hardirqs_off
#endif
639

640
	ct_user_exit
Catalin Marinas's avatar
Catalin Marinas committed
641
	irq_handler
642

Catalin Marinas's avatar
Catalin Marinas committed
643 644 645 646 647 648 649 650 651 652 653 654 655 656 657
#ifdef CONFIG_TRACE_IRQFLAGS
	bl	trace_hardirqs_on
#endif
	b	ret_to_user
ENDPROC(el0_irq)

/*
 * Register switch for AArch64. The callee-saved registers need to be saved
 * and restored. On entry:
 *   x0 = previous task_struct (must be preserved across the switch)
 *   x1 = next task_struct
 * Previous and next are guaranteed not to be the same.
 *
 */
ENTRY(cpu_switch_to)
658 659
	mov	x10, #THREAD_CPU_CONTEXT
	add	x8, x0, x10
Catalin Marinas's avatar
Catalin Marinas committed
660 661 662 663 664 665 666 667
	mov	x9, sp
	stp	x19, x20, [x8], #16		// store callee-saved registers
	stp	x21, x22, [x8], #16
	stp	x23, x24, [x8], #16
	stp	x25, x26, [x8], #16
	stp	x27, x28, [x8], #16
	stp	x29, x9, [x8], #16
	str	lr, [x8]
668
	add	x8, x1, x10
Catalin Marinas's avatar
Catalin Marinas committed
669 670 671 672 673 674 675 676
	ldp	x19, x20, [x8], #16		// restore callee-saved registers
	ldp	x21, x22, [x8], #16
	ldp	x23, x24, [x8], #16
	ldp	x25, x26, [x8], #16
	ldp	x27, x28, [x8], #16
	ldp	x29, x9, [x8], #16
	ldr	lr, [x8]
	mov	sp, x9
677 678
	and	x9, x9, #~(THREAD_SIZE - 1)
	msr	sp_el0, x9
Catalin Marinas's avatar
Catalin Marinas committed
679 680 681 682 683 684 685 686 687
	ret
ENDPROC(cpu_switch_to)

/*
 * This is the fast syscall return path.  We do as little as possible here,
 * and this includes saving x0 back into the kernel stack.
 */
ret_fast_syscall:
	disable_irq				// disable interrupts
688
	str	x0, [sp, #S_X0]			// returned x0
689 690 691
	ldr	x1, [tsk, #TI_FLAGS]		// re-check for syscall tracing
	and	x2, x1, #_TIF_SYSCALL_WORK
	cbnz	x2, ret_fast_syscall_trace
Catalin Marinas's avatar
Catalin Marinas committed
692
	and	x2, x1, #_TIF_WORK_MASK
693
	cbnz	x2, work_pending
694
	enable_step_tsk x1, x2
695
	kernel_exit 0
696 697
ret_fast_syscall_trace:
	enable_irq				// enable interrupts
698
	b	__sys_trace_return_skipped	// we already saved x0
Catalin Marinas's avatar
Catalin Marinas committed
699 700 701 702 703 704 705

/*
 * Ok, we need to do extra processing, enter the slow path.
 */
work_pending:
	mov	x0, sp				// 'regs'
	bl	do_notify_resume
706
#ifdef CONFIG_TRACE_IRQFLAGS
707
	bl	trace_hardirqs_on		// enabled while in userspace
708
#endif
709 710
	ldr	x1, [tsk, #TI_FLAGS]		// re-check for single-step
	b	finish_ret_to_user
Catalin Marinas's avatar
Catalin Marinas committed
711 712 713
/*
 * "slow" syscall return path.
 */
714
ret_to_user:
Catalin Marinas's avatar
Catalin Marinas committed
715 716 717 718
	disable_irq				// disable interrupts
	ldr	x1, [tsk, #TI_FLAGS]
	and	x2, x1, #_TIF_WORK_MASK
	cbnz	x2, work_pending
719
finish_ret_to_user:
720
	enable_step_tsk x1, x2
721
	kernel_exit 0
Catalin Marinas's avatar
Catalin Marinas committed
722 723 724 725 726 727 728
ENDPROC(ret_to_user)

/*
 * This is how we return from a fork.
 */
ENTRY(ret_from_fork)
	bl	schedule_tail
729 730 731 732
	cbz	x19, 1f				// not a kernel thread
	mov	x0, x20
	blr	x19
1:	get_thread_info tsk
Catalin Marinas's avatar
Catalin Marinas committed
733 734 735 736 737 738 739 740 741 742 743 744 745
	b	ret_to_user
ENDPROC(ret_from_fork)

/*
 * SVC handler.
 */
	.align	6
el0_svc:
	adrp	stbl, sys_call_table		// load syscall table pointer
	uxtw	scno, w8			// syscall number in w8
	mov	sc_nr, #__NR_syscalls
el0_svc_naked:					// compat entry point
	stp	x0, scno, [sp, #S_ORIG_X0]	// save the original x0 and syscall number
746
	enable_dbg_and_irq
747
	ct_user_exit 1
Catalin Marinas's avatar
Catalin Marinas committed
748

749 750 751
	ldr	x16, [tsk, #TI_FLAGS]		// check for syscall hooks
	tst	x16, #_TIF_SYSCALL_WORK
	b.ne	__sys_trace
Catalin Marinas's avatar
Catalin Marinas committed
752 753 754
	cmp     scno, sc_nr                     // check upper syscall limit
	b.hs	ni_sys
	ldr	x16, [stbl, scno, lsl #3]	// address in the syscall table
755 756
	blr	x16				// call sys_* routine
	b	ret_fast_syscall
Catalin Marinas's avatar
Catalin Marinas committed
757 758
ni_sys:
	mov	x0, sp
759 760
	bl	do_ni_syscall
	b	ret_fast_syscall
Catalin Marinas's avatar
Catalin Marinas committed
761 762 763 764 765 766 767
ENDPROC(el0_svc)

	/*
	 * This is the really slow path.  We're going to be doing context
	 * switches, and waiting for our parent to respond.
	 */
__sys_trace:
768 769 770 771 772 773
	mov	w0, #-1				// set default errno for
	cmp     scno, x0			// user-issued syscall(-1)
	b.ne	1f
	mov	x0, #-ENOSYS
	str	x0, [sp, #S_X0]
1:	mov	x0, sp
774
	bl	syscall_trace_enter
775 776
	cmp	w0, #-1				// skip the syscall?
	b.eq	__sys_trace_return_skipped
Catalin Marinas's avatar
Catalin Marinas committed
777 778 779
	uxtw	scno, w0			// syscall number (possibly new)
	mov	x1, sp				// pointer to regs
	cmp	scno, sc_nr			// check upper syscall limit
780
	b.hs	__ni_sys_trace
Catalin Marinas's avatar
Catalin Marinas committed
781 782 783 784 785
	ldp	x0, x1, [sp]			// restore the syscall args
	ldp	x2, x3, [sp, #S_X2]
	ldp	x4, x5, [sp, #S_X4]
	ldp	x6, x7, [sp, #S_X6]
	ldr	x16, [stbl, scno, lsl #3]	// address in the syscall table
786
	blr	x16				// call sys_* routine
Catalin Marinas's avatar
Catalin Marinas committed
787 788

__sys_trace_return:
789 790
	str	x0, [sp, #S_X0]			// save returned x0
__sys_trace_return_skipped:
791 792
	mov	x0, sp
	bl	syscall_trace_exit
Catalin Marinas's avatar
Catalin Marinas committed
793 794
	b	ret_to_user

795 796 797 798 799
__ni_sys_trace:
	mov	x0, sp
	bl	do_ni_syscall
	b	__sys_trace_return

800 801
	.popsection				// .entry.text

Catalin Marinas's avatar
Catalin Marinas committed
802 803 804 805 806 807 808
/*
 * Special system call wrappers.
 */
ENTRY(sys_rt_sigreturn_wrapper)
	mov	x0, sp
	b	sys_rt_sigreturn
ENDPROC(sys_rt_sigreturn_wrapper)