entry.S 42.2 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4 5
/*
 * ia64/kernel/entry.S
 *
 * Kernel entry points.
 *
6
 * Copyright (C) 1998-2003 Hewlett-Packard Co
Linus Torvalds's avatar
Linus Torvalds committed
7
 *	David Mosberger-Tang <davidm@hpl.hp.com>
8 9 10 11 12
 * Copyright (C) 1999, 2002-2003
 *	Asit Mallick <Asit.K.Mallick@intel.com>
 * 	Don Dugger <Don.Dugger@intel.com>
 *	Suresh Siddha <suresh.b.siddha@intel.com>
 *	Fenghua Yu <fenghua.yu@intel.com>
Linus Torvalds's avatar
Linus Torvalds committed
13 14 15 16 17 18 19 20
 * Copyright (C) 1999 VA Linux Systems
 * Copyright (C) 1999 Walt Drummond <drummond@valinux.com>
 */
/*
 * ia64_switch_to now places correct virtual mapping in in TR2 for
 * kernel stack. This allows us to handle interrupts without changing
 * to physical mode.
 *
Linus Torvalds's avatar
Linus Torvalds committed
21
 * Jonathan Nicklin	<nicklin@missioncriticallinux.com>
Linus Torvalds's avatar
Linus Torvalds committed
22 23
 * Patrick O'Rourke	<orourke@missioncriticallinux.com>
 * 11/07/2000
24
 */
Linus Torvalds's avatar
Linus Torvalds committed
25 26 27
/*
 * Global (preserved) predicate usage on syscall entry/exit path:
 *
28 29
 *	pKStk:		See entry.h.
 *	pUStk:		See entry.h.
Linus Torvalds's avatar
Linus Torvalds committed
30 31 32 33 34 35
 *	pSys:		See entry.h.
 *	pNonSys:	!pSys
 */

#include <linux/config.h>

David Mosberger's avatar
David Mosberger committed
36
#include <asm/asmmacro.h>
Linus Torvalds's avatar
Linus Torvalds committed
37 38
#include <asm/cache.h>
#include <asm/errno.h>
Linus Torvalds's avatar
Linus Torvalds committed
39
#include <asm/kregs.h>
Linus Torvalds's avatar
Linus Torvalds committed
40
#include <asm/offsets.h>
David Mosberger's avatar
David Mosberger committed
41
#include <asm/pgtable.h>
42
#include <asm/percpu.h>
Linus Torvalds's avatar
Linus Torvalds committed
43
#include <asm/processor.h>
David Mosberger's avatar
David Mosberger committed
44
#include <asm/thread_info.h>
Linus Torvalds's avatar
Linus Torvalds committed
45 46
#include <asm/unistd.h>

Linus Torvalds's avatar
Linus Torvalds committed
47
#include "minstate.h"
Linus Torvalds's avatar
Linus Torvalds committed
48 49 50 51 52 53

	/*
	 * execve() is special because in case of success, we need to
	 * setup a null register window frame.
	 */
ENTRY(ia64_execve)
Linus Torvalds's avatar
Linus Torvalds committed
54
	.prologue ASM_UNW_PRLG_RP|ASM_UNW_PRLG_PFS, ASM_UNW_PRLG_GRSAVE(3)
Linus Torvalds's avatar
Linus Torvalds committed
55 56
	alloc loc1=ar.pfs,3,2,4,0
	mov loc0=rp
Linus Torvalds's avatar
Linus Torvalds committed
57
	.body
Linus Torvalds's avatar
Linus Torvalds committed
58 59 60 61 62
	mov out0=in0			// filename
	;;				// stop bit between alloc and call
	mov out1=in1			// argv
	mov out2=in2			// envp
	add out3=16,sp			// regs
Linus Torvalds's avatar
Linus Torvalds committed
63
	br.call.sptk.many rp=sys_execve
64 65 66 67 68 69 70 71 72 73 74
.ret0:
#ifdef CONFIG_IA32_SUPPORT
	/*
	 * Check if we're returning to ia32 mode. If so, we need to restore ia32 registers
	 * from pt_regs.
	 */
	adds r16=PT(CR_IPSR)+16,sp
	;;
	ld8 r16=[r16]
#endif
	cmp4.ge p6,p7=r8,r0
Linus Torvalds's avatar
Linus Torvalds committed
75 76
	mov ar.pfs=loc1			// restore ar.pfs
	sxt4 r8=r8			// return 64-bit result
Linus Torvalds's avatar
Linus Torvalds committed
77 78
	;;
	stf.spill [sp]=f0
79
(p6)	cmp.ne pKStk,pUStk=r0,r0	// a successful execve() lands us in user-mode...
Linus Torvalds's avatar
Linus Torvalds committed
80
	mov rp=loc0
Linus Torvalds's avatar
Linus Torvalds committed
81
(p6)	mov ar.pfs=r0			// clear ar.pfs on success
Linus Torvalds's avatar
Linus Torvalds committed
82
(p7)	br.ret.sptk.many rp
Linus Torvalds's avatar
Linus Torvalds committed
83

Linus Torvalds's avatar
Linus Torvalds committed
84 85
	/*
	 * In theory, we'd have to zap this state only to prevent leaking of
Linus Torvalds's avatar
Linus Torvalds committed
86
	 * security sensitive state (e.g., if current->mm->dumpable is zero).  However,
Linus Torvalds's avatar
Linus Torvalds committed
87 88 89
	 * this executes in less than 20 cycles even on Itanium, so it's not worth
	 * optimizing for...).
	 */
90
	mov ar.unat=0; 		mov ar.lc=0
Linus Torvalds's avatar
Linus Torvalds committed
91 92 93 94
	mov r4=0;		mov f2=f0;		mov b1=r0
	mov r5=0;		mov f3=f0;		mov b2=r0
	mov r6=0;		mov f4=f0;		mov b3=r0
	mov r7=0;		mov f5=f0;		mov b4=r0
95
	ldf.fill f12=[sp];	mov f13=f0;		mov b5=r0
Linus Torvalds's avatar
Linus Torvalds committed
96 97 98 99 100 101
	ldf.fill f14=[sp];	ldf.fill f15=[sp];	mov f16=f0
	ldf.fill f17=[sp];	ldf.fill f18=[sp];	mov f19=f0
	ldf.fill f20=[sp];	ldf.fill f21=[sp];	mov f22=f0
	ldf.fill f23=[sp];	ldf.fill f24=[sp];	mov f25=f0
	ldf.fill f26=[sp];	ldf.fill f27=[sp];	mov f28=f0
	ldf.fill f29=[sp];	ldf.fill f30=[sp];	mov f31=f0
102 103 104 105 106 107
#ifdef CONFIG_IA32_SUPPORT
	tbit.nz p6,p0=r16, IA64_PSR_IS_BIT
	movl loc0=ia64_ret_from_ia32_execve
	;;
(p6)	mov rp=loc0
#endif
Linus Torvalds's avatar
Linus Torvalds committed
108
	br.ret.sptk.many rp
Linus Torvalds's avatar
Linus Torvalds committed
109 110
END(ia64_execve)

111
/*
112
 * sys_clone2(u64 flags, u64 ustack_base, u64 ustack_size, u64 parent_tidptr, u64 child_tidptr,
113
 *	      u64 tls)
114
 */
Linus Torvalds's avatar
Linus Torvalds committed
115
GLOBAL_ENTRY(sys_clone2)
David Mosberger's avatar
David Mosberger committed
116
	.prologue ASM_UNW_PRLG_RP|ASM_UNW_PRLG_PFS, ASM_UNW_PRLG_GRSAVE(6)
117
	alloc r16=ar.pfs,6,2,6,0
Linus Torvalds's avatar
Linus Torvalds committed
118
	DO_SAVE_SWITCH_STACK
119
	adds r2=PT(R16)+IA64_SWITCH_STACK_SIZE+16,sp
Linus Torvalds's avatar
Linus Torvalds committed
120 121
	mov loc0=rp
	mov loc1=r16				// save ar.pfs across do_fork
Linus Torvalds's avatar
Linus Torvalds committed
122
	.body
Linus Torvalds's avatar
Linus Torvalds committed
123 124
	mov out1=in1
	mov out3=in2
125
	tbit.nz p6,p0=in0,CLONE_SETTLS_BIT
126
	mov out4=in3	// parent_tidptr: valid only w/CLONE_PARENT_SETTID
127
	;;
128
(p6)	st8 [r2]=in5				// store TLS in r16 for copy_thread()
129
	mov out5=in4	// child_tidptr:  valid only w/CLONE_CHILD_SETTID or CLONE_CHILD_CLEARTID
Linus Torvalds's avatar
Linus Torvalds committed
130
	adds out2=IA64_SWITCH_STACK_SIZE+16,sp	// out2 = &regs
131
	mov out0=in0				// out0 = clone_flags
132
	br.call.sptk.many rp=do_fork
Linus Torvalds's avatar
Linus Torvalds committed
133
.ret1:	.restore sp
Linus Torvalds's avatar
Linus Torvalds committed
134 135 136 137 138 139
	adds sp=IA64_SWITCH_STACK_SIZE,sp	// pop the switch stack
	mov ar.pfs=loc1
	mov rp=loc0
	br.ret.sptk.many rp
END(sys_clone2)

140
/*
141
 * sys_clone(u64 flags, u64 ustack_base, u64 parent_tidptr, u64 child_tidptr, u64 tls)
142 143
 *	Deprecated.  Use sys_clone2() instead.
 */
Linus Torvalds's avatar
Linus Torvalds committed
144
GLOBAL_ENTRY(sys_clone)
David Mosberger's avatar
David Mosberger committed
145
	.prologue ASM_UNW_PRLG_RP|ASM_UNW_PRLG_PFS, ASM_UNW_PRLG_GRSAVE(5)
146
	alloc r16=ar.pfs,5,2,6,0
Linus Torvalds's avatar
Linus Torvalds committed
147
	DO_SAVE_SWITCH_STACK
148
	adds r2=PT(R16)+IA64_SWITCH_STACK_SIZE+16,sp
Linus Torvalds's avatar
Linus Torvalds committed
149 150
	mov loc0=rp
	mov loc1=r16				// save ar.pfs across do_fork
Linus Torvalds's avatar
Linus Torvalds committed
151
	.body
Linus Torvalds's avatar
Linus Torvalds committed
152
	mov out1=in1
David Mosberger's avatar
David Mosberger committed
153
	mov out3=16				// stacksize (compensates for 16-byte scratch area)
154
	tbit.nz p6,p0=in0,CLONE_SETTLS_BIT
155
	mov out4=in2	// parent_tidptr: valid only w/CLONE_PARENT_SETTID
156
	;;
157 158
(p6)	st8 [r2]=in4				// store TLS in r13 (tp)
	mov out5=in3	// child_tidptr:  valid only w/CLONE_CHILD_SETTID or CLONE_CHILD_CLEARTID
Linus Torvalds's avatar
Linus Torvalds committed
159
	adds out2=IA64_SWITCH_STACK_SIZE+16,sp	// out2 = &regs
160
	mov out0=in0				// out0 = clone_flags
161
	br.call.sptk.many rp=do_fork
Linus Torvalds's avatar
Linus Torvalds committed
162
.ret2:	.restore sp
Linus Torvalds's avatar
Linus Torvalds committed
163 164 165 166 167 168 169 170
	adds sp=IA64_SWITCH_STACK_SIZE,sp	// pop the switch stack
	mov ar.pfs=loc1
	mov rp=loc0
	br.ret.sptk.many rp
END(sys_clone)

/*
 * prev_task <- ia64_switch_to(struct task_struct *next)
David Mosberger's avatar
David Mosberger committed
171 172 173
 *	With Ingo's new scheduler, interrupts are disabled when this routine gets
 *	called.  The code starting at .map relies on this.  The rest of the code
 *	doesn't care about the interrupt masking status.
Linus Torvalds's avatar
Linus Torvalds committed
174 175
 */
GLOBAL_ENTRY(ia64_switch_to)
Linus Torvalds's avatar
Linus Torvalds committed
176
	.prologue
Linus Torvalds's avatar
Linus Torvalds committed
177 178
	alloc r16=ar.pfs,1,0,0,0
	DO_SAVE_SWITCH_STACK
Linus Torvalds's avatar
Linus Torvalds committed
179
	.body
Linus Torvalds's avatar
Linus Torvalds committed
180 181

	adds r22=IA64_TASK_THREAD_KSP_OFFSET,r13
182
	movl r25=init_task
Linus Torvalds's avatar
Linus Torvalds committed
183
	mov r27=IA64_KR(CURRENT_STACK)
184
	adds r21=IA64_TASK_THREAD_KSP_OFFSET,in0
185
	dep r20=0,in0,61,3		// physical address of "next"
Linus Torvalds's avatar
Linus Torvalds committed
186 187
	;;
	st8 [r22]=sp			// save kernel stack pointer of old task
Linus Torvalds's avatar
Linus Torvalds committed
188
	shr.u r26=r20,IA64_GRANULE_SHIFT
189
	cmp.eq p7,p6=r25,in0
Linus Torvalds's avatar
Linus Torvalds committed
190 191
	;;
	/*
Linus Torvalds's avatar
Linus Torvalds committed
192
	 * If we've already mapped this task's page, we can skip doing it again.
Linus Torvalds's avatar
Linus Torvalds committed
193
	 */
194
(p6)	cmp.eq p7,p6=r26,r27
Linus Torvalds's avatar
Linus Torvalds committed
195
(p6)	br.cond.dpnt .map
Linus Torvalds's avatar
Linus Torvalds committed
196
	;;
Linus Torvalds's avatar
Linus Torvalds committed
197
.done:
198
(p6)	ssm psr.ic			// if we had to map, reenable the psr.ic bit FIRST!!!
Linus Torvalds's avatar
Linus Torvalds committed
199 200
	;;
(p6)	srlz.d
Linus Torvalds's avatar
Linus Torvalds committed
201
	ld8 sp=[r21]			// load kernel stack pointer of new task
202
	mov IA64_KR(CURRENT)=in0	// update "current" application register
203
	mov r8=r13			// return pointer to previously running task
Linus Torvalds's avatar
Linus Torvalds committed
204 205
	mov r13=in0			// set "current" pointer
	;;
Linus Torvalds's avatar
Linus Torvalds committed
206
	DO_LOAD_SWITCH_STACK
Linus Torvalds's avatar
Linus Torvalds committed
207 208 209

#ifdef CONFIG_SMP
	sync.i				// ensure "fc"s done by this CPU are visible on other CPUs
Linus Torvalds's avatar
Linus Torvalds committed
210
#endif
Linus Torvalds's avatar
Linus Torvalds committed
211
	br.ret.sptk.many rp		// boogie on out in new context
Linus Torvalds's avatar
Linus Torvalds committed
212 213

.map:
David Mosberger's avatar
David Mosberger committed
214
	rsm psr.ic			// interrupts (psr.i) are already disabled here
Linus Torvalds's avatar
Linus Torvalds committed
215
	movl r25=PAGE_KERNEL
Linus Torvalds's avatar
Linus Torvalds committed
216 217 218
	;;
	srlz.d
	or r23=r25,r20			// construct PA | page properties
Linus Torvalds's avatar
Linus Torvalds committed
219
	mov r25=IA64_GRANULE_SHIFT<<2
Linus Torvalds's avatar
Linus Torvalds committed
220 221 222 223
	;;
	mov cr.itir=r25
	mov cr.ifa=in0			// VA of next task...
	;;
Linus Torvalds's avatar
Linus Torvalds committed
224 225
	mov r25=IA64_TR_CURRENT_STACK
	mov IA64_KR(CURRENT_STACK)=r26	// remember last page we mapped...
Linus Torvalds's avatar
Linus Torvalds committed
226 227
	;;
	itr.d dtr[r25]=r23		// wire in new mapping...
Linus Torvalds's avatar
Linus Torvalds committed
228
	br.cond.sptk .done
Linus Torvalds's avatar
Linus Torvalds committed
229 230 231
END(ia64_switch_to)

/*
232 233 234 235 236 237
 * Note that interrupts are enabled during save_switch_stack and load_switch_stack.  This
 * means that we may get an interrupt with "sp" pointing to the new kernel stack while
 * ar.bspstore is still pointing to the old kernel backing store area.  Since ar.rsc,
 * ar.rnat, ar.bsp, and ar.bspstore are all preserved by interrupts, this is not a
 * problem.  Also, we don't need to specify unwind information for preserved registers
 * that are not modified in save_switch_stack as the right unwind information is already
Linus Torvalds's avatar
Linus Torvalds committed
238 239 240 241 242 243 244 245 246 247
 * specified at the call-site of save_switch_stack.
 */

/*
 * save_switch_stack:
 *	- r16 holds ar.pfs
 *	- b7 holds address to return to
 *	- rp (b0) holds return address to save
 */
GLOBAL_ENTRY(save_switch_stack)
Linus Torvalds's avatar
Linus Torvalds committed
248 249
	.prologue
	.altrp b7
Linus Torvalds's avatar
Linus Torvalds committed
250
	flushrs			// flush dirty regs to backing store (must be first in insn group)
Linus Torvalds's avatar
Linus Torvalds committed
251
	.save @priunat,r17
Linus Torvalds's avatar
Linus Torvalds committed
252
	mov r17=ar.unat		// preserve caller's
Linus Torvalds's avatar
Linus Torvalds committed
253
	.body
254 255 256 257
#ifdef CONFIG_ITANIUM
	adds r2=16+128,sp
	adds r3=16+64,sp
	adds r14=SW(R4)+16,sp
Linus Torvalds's avatar
Linus Torvalds committed
258
	;;
259
	st8.spill [r14]=r4,16		// spill r4
Linus Torvalds's avatar
Linus Torvalds committed
260
	lfetch.fault.excl.nt1 [r3],128
Linus Torvalds's avatar
Linus Torvalds committed
261
	;;
Linus Torvalds's avatar
Linus Torvalds committed
262 263 264 265 266 267
	lfetch.fault.excl.nt1 [r2],128
	lfetch.fault.excl.nt1 [r3],128
	;;
	lfetch.fault.excl [r2]
	lfetch.fault.excl [r3]
	adds r15=SW(R5)+16,sp
268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286
#else
	add r2=16+3*128,sp
	add r3=16,sp
	add r14=SW(R4)+16,sp
	;;
	st8.spill [r14]=r4,SW(R6)-SW(R4)	// spill r4 and prefetch offset 0x1c0
	lfetch.fault.excl.nt1 [r3],128	//		prefetch offset 0x010
	;;
	lfetch.fault.excl.nt1 [r3],128	//		prefetch offset 0x090
	lfetch.fault.excl.nt1 [r2],128	//		prefetch offset 0x190
	;;
	lfetch.fault.excl.nt1 [r3]	//		prefetch offset 0x110
	lfetch.fault.excl.nt1 [r2]	//		prefetch offset 0x210
	adds r15=SW(R5)+16,sp
#endif
	;;
	st8.spill [r15]=r5,SW(R7)-SW(R5)	// spill r5
	mov.m ar.rsc=0			// put RSE in mode: enforced lazy, little endian, pl 0
	add r2=SW(F2)+16,sp		// r2 = &sw->f2
Linus Torvalds's avatar
Linus Torvalds committed
287
	;;
288 289 290
	st8.spill [r14]=r6,SW(B0)-SW(R6)	// spill r6
	mov.m r18=ar.fpsr		// preserve fpsr
	add r3=SW(F3)+16,sp		// r3 = &sw->f3
Linus Torvalds's avatar
Linus Torvalds committed
291 292
	;;
	stf.spill [r2]=f2,32
293
	mov.m r19=ar.rnat
Linus Torvalds's avatar
Linus Torvalds committed
294
	mov r21=b0
295 296 297

	stf.spill [r3]=f3,32
	st8.spill [r15]=r7,SW(B2)-SW(R7)	// spill r7
Linus Torvalds's avatar
Linus Torvalds committed
298
	mov r22=b1
Linus Torvalds's avatar
Linus Torvalds committed
299
	;;
Linus Torvalds's avatar
Linus Torvalds committed
300
	// since we're done with the spills, read and save ar.unat:
301 302
	mov.m r29=ar.unat
	mov.m r20=ar.bspstore
Linus Torvalds's avatar
Linus Torvalds committed
303
	mov r23=b2
Linus Torvalds's avatar
Linus Torvalds committed
304 305
	stf.spill [r2]=f4,32
	stf.spill [r3]=f5,32
Linus Torvalds's avatar
Linus Torvalds committed
306
	mov r24=b3
Linus Torvalds's avatar
Linus Torvalds committed
307
	;;
308 309
	st8 [r14]=r21,SW(B1)-SW(B0)		// save b0
	st8 [r15]=r23,SW(B3)-SW(B2)		// save b2
Linus Torvalds's avatar
Linus Torvalds committed
310 311
	mov r25=b4
	mov r26=b5
Linus Torvalds's avatar
Linus Torvalds committed
312
	;;
313 314
	st8 [r14]=r22,SW(B4)-SW(B1)		// save b1
	st8 [r15]=r24,SW(AR_PFS)-SW(B3)		// save b3
Linus Torvalds's avatar
Linus Torvalds committed
315
	mov r21=ar.lc		// I-unit
Linus Torvalds's avatar
Linus Torvalds committed
316 317 318
	stf.spill [r2]=f12,32
	stf.spill [r3]=f13,32
	;;
319 320
	st8 [r14]=r25,SW(B5)-SW(B4)		// save b4
	st8 [r15]=r16,SW(AR_LC)-SW(AR_PFS)	// save ar.pfs
Linus Torvalds's avatar
Linus Torvalds committed
321 322 323
	stf.spill [r2]=f14,32
	stf.spill [r3]=f15,32
	;;
324 325
	st8 [r14]=r26				// save b5
	st8 [r15]=r21				// save ar.lc
Linus Torvalds's avatar
Linus Torvalds committed
326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346
	stf.spill [r2]=f16,32
	stf.spill [r3]=f17,32
	;;
	stf.spill [r2]=f18,32
	stf.spill [r3]=f19,32
	;;
	stf.spill [r2]=f20,32
	stf.spill [r3]=f21,32
	;;
	stf.spill [r2]=f22,32
	stf.spill [r3]=f23,32
	;;
	stf.spill [r2]=f24,32
	stf.spill [r3]=f25,32
	;;
	stf.spill [r2]=f26,32
	stf.spill [r3]=f27,32
	;;
	stf.spill [r2]=f28,32
	stf.spill [r3]=f29,32
	;;
347 348 349 350 351 352 353
	stf.spill [r2]=f30,SW(AR_UNAT)-SW(F30)
	stf.spill [r3]=f31,SW(PR)-SW(F31)
	add r14=SW(CALLER_UNAT)+16,sp
	;;
	st8 [r2]=r29,SW(AR_RNAT)-SW(AR_UNAT)	// save ar.unat
	st8 [r14]=r17,SW(AR_FPSR)-SW(CALLER_UNAT) // save caller_unat
	mov r21=pr
Linus Torvalds's avatar
Linus Torvalds committed
354
	;;
355 356
	st8 [r2]=r19,SW(AR_BSPSTORE)-SW(AR_RNAT) // save ar.rnat
	st8 [r3]=r21				// save predicate registers
Linus Torvalds's avatar
Linus Torvalds committed
357
	;;
358 359
	st8 [r2]=r20				// save ar.bspstore
	st8 [r14]=r18				// save fpsr
Linus Torvalds's avatar
Linus Torvalds committed
360
	mov ar.rsc=3		// put RSE back into eager mode, pl 0
Linus Torvalds's avatar
Linus Torvalds committed
361
	br.cond.sptk.many b7
Linus Torvalds's avatar
Linus Torvalds committed
362 363 364 365
END(save_switch_stack)

/*
 * load_switch_stack:
Linus Torvalds's avatar
Linus Torvalds committed
366
 *	- "invala" MUST be done at call site (normally in DO_LOAD_SWITCH_STACK)
Linus Torvalds's avatar
Linus Torvalds committed
367
 *	- b7 holds address to return to
Linus Torvalds's avatar
Linus Torvalds committed
368
 *	- must not touch r8-r11
Linus Torvalds's avatar
Linus Torvalds committed
369 370
 */
ENTRY(load_switch_stack)
Linus Torvalds's avatar
Linus Torvalds committed
371 372 373
	.prologue
	.altrp b7

Linus Torvalds's avatar
Linus Torvalds committed
374
	.body
Linus Torvalds's avatar
Linus Torvalds committed
375 376 377 378 379 380 381 382 383
	lfetch.fault.nt1 [sp]
	adds r2=SW(AR_BSPSTORE)+16,sp
	adds r3=SW(AR_UNAT)+16,sp
	mov ar.rsc=0						// put RSE into enforced lazy mode
	adds r14=SW(CALLER_UNAT)+16,sp
	adds r15=SW(AR_FPSR)+16,sp
	;;
	ld8 r27=[r2],(SW(B0)-SW(AR_BSPSTORE))	// bspstore
	ld8 r29=[r3],(SW(B1)-SW(AR_UNAT))	// unat
Linus Torvalds's avatar
Linus Torvalds committed
384 385 386 387 388 389 390 391 392 393
	;;
	ld8 r21=[r2],16		// restore b0
	ld8 r22=[r3],16		// restore b1
	;;
	ld8 r23=[r2],16		// restore b2
	ld8 r24=[r3],16		// restore b3
	;;
	ld8 r25=[r2],16		// restore b4
	ld8 r26=[r3],16		// restore b5
	;;
Linus Torvalds's avatar
Linus Torvalds committed
394 395
	ld8 r16=[r2],(SW(PR)-SW(AR_PFS))	// ar.pfs
	ld8 r17=[r3],(SW(AR_RNAT)-SW(AR_LC))	// ar.lc
Linus Torvalds's avatar
Linus Torvalds committed
396
	;;
Linus Torvalds's avatar
Linus Torvalds committed
397 398
	ld8 r28=[r2]		// restore pr
	ld8 r30=[r3]		// restore rnat
Linus Torvalds's avatar
Linus Torvalds committed
399
	;;
Linus Torvalds's avatar
Linus Torvalds committed
400 401
	ld8 r18=[r14],16	// restore caller's unat
	ld8 r19=[r15],24	// restore fpsr
Linus Torvalds's avatar
Linus Torvalds committed
402
	;;
Linus Torvalds's avatar
Linus Torvalds committed
403 404
	ldf.fill f2=[r14],32
	ldf.fill f3=[r15],32
Linus Torvalds's avatar
Linus Torvalds committed
405
	;;
Linus Torvalds's avatar
Linus Torvalds committed
406 407
	ldf.fill f4=[r14],32
	ldf.fill f5=[r15],32
Linus Torvalds's avatar
Linus Torvalds committed
408
	;;
Linus Torvalds's avatar
Linus Torvalds committed
409 410 411 412 413
	ldf.fill f12=[r14],32
	ldf.fill f13=[r15],32
	;;
	ldf.fill f14=[r14],32
	ldf.fill f15=[r15],32
Linus Torvalds's avatar
Linus Torvalds committed
414
	;;
Linus Torvalds's avatar
Linus Torvalds committed
415 416
	ldf.fill f16=[r14],32
	ldf.fill f17=[r15],32
Linus Torvalds's avatar
Linus Torvalds committed
417
	;;
Linus Torvalds's avatar
Linus Torvalds committed
418 419 420
	ldf.fill f18=[r14],32
	ldf.fill f19=[r15],32
	mov b0=r21
Linus Torvalds's avatar
Linus Torvalds committed
421
	;;
Linus Torvalds's avatar
Linus Torvalds committed
422 423
	ldf.fill f20=[r14],32
	ldf.fill f21=[r15],32
Linus Torvalds's avatar
Linus Torvalds committed
424 425
	mov b1=r22
	;;
Linus Torvalds's avatar
Linus Torvalds committed
426 427
	ldf.fill f22=[r14],32
	ldf.fill f23=[r15],32
Linus Torvalds's avatar
Linus Torvalds committed
428 429
	mov b2=r23
	;;
Linus Torvalds's avatar
Linus Torvalds committed
430 431
	mov ar.bspstore=r27
	mov ar.unat=r29		// establish unat holding the NaT bits for r4-r7
Linus Torvalds's avatar
Linus Torvalds committed
432 433
	mov b3=r24
	;;
Linus Torvalds's avatar
Linus Torvalds committed
434 435
	ldf.fill f24=[r14],32
	ldf.fill f25=[r15],32
Linus Torvalds's avatar
Linus Torvalds committed
436 437
	mov b4=r25
	;;
Linus Torvalds's avatar
Linus Torvalds committed
438 439
	ldf.fill f26=[r14],32
	ldf.fill f27=[r15],32
Linus Torvalds's avatar
Linus Torvalds committed
440 441
	mov b5=r26
	;;
Linus Torvalds's avatar
Linus Torvalds committed
442 443 444
	ldf.fill f28=[r14],32
	ldf.fill f29=[r15],32
	mov ar.pfs=r16
Linus Torvalds's avatar
Linus Torvalds committed
445
	;;
Linus Torvalds's avatar
Linus Torvalds committed
446 447 448
	ldf.fill f30=[r14],32
	ldf.fill f31=[r15],24
	mov ar.lc=r17
Linus Torvalds's avatar
Linus Torvalds committed
449
	;;
Linus Torvalds's avatar
Linus Torvalds committed
450 451 452
	ld8.fill r4=[r14],16
	ld8.fill r5=[r15],16
	mov pr=r28,-1
Linus Torvalds's avatar
Linus Torvalds committed
453
	;;
Linus Torvalds's avatar
Linus Torvalds committed
454 455 456
	ld8.fill r6=[r14],16
	ld8.fill r7=[r15],16

Linus Torvalds's avatar
Linus Torvalds committed
457
	mov ar.unat=r18				// restore caller's unat
Linus Torvalds's avatar
Linus Torvalds committed
458
	mov ar.rnat=r30				// must restore after bspstore but before rsc!
Linus Torvalds's avatar
Linus Torvalds committed
459 460
	mov ar.fpsr=r19				// restore fpsr
	mov ar.rsc=3				// put RSE back into eager mode, pl 0
Linus Torvalds's avatar
Linus Torvalds committed
461
	br.cond.sptk.many b7
Linus Torvalds's avatar
Linus Torvalds committed
462 463 464 465 466 467 468 469 470 471 472
END(load_switch_stack)

GLOBAL_ENTRY(__ia64_syscall)
	.regstk 6,0,0,0
	mov r15=in5				// put syscall number in place
	break __BREAK_SYSCALL
	movl r2=errno
	cmp.eq p6,p7=-1,r10
	;;
(p6)	st4 [r2]=r8
(p6)	mov r8=-1
Linus Torvalds's avatar
Linus Torvalds committed
473
	br.ret.sptk.many rp
Linus Torvalds's avatar
Linus Torvalds committed
474 475
END(__ia64_syscall)

476 477 478 479 480 481 482 483 484 485 486 487
GLOBAL_ENTRY(execve)
	mov r15=__NR_execve			// put syscall number in place
	break __BREAK_SYSCALL
	br.ret.sptk.many rp
END(execve)

GLOBAL_ENTRY(clone)
	mov r15=__NR_clone			// put syscall number in place
	break __BREAK_SYSCALL
	br.ret.sptk.many rp
END(clone)

Linus Torvalds's avatar
Linus Torvalds committed
488 489 490 491 492 493
	/*
	 * Invoke a system call, but do some tracing before and after the call.
	 * We MUST preserve the current register frame throughout this routine
	 * because some system calls (such as ia64_execve) directly
	 * manipulate ar.pfs.
	 */
Linus Torvalds's avatar
Linus Torvalds committed
494 495
GLOBAL_ENTRY(ia64_trace_syscall)
	PT_REGS_UNWIND_INFO(0)
496 497 498
	/*
	 * We need to preserve the scratch registers f6-f11 in case the system
	 * call is sigreturn.
499
	 */
500 501 502 503 504 505 506 507 508 509 510
	adds r16=PT(F6)+16,sp
	adds r17=PT(F7)+16,sp
	;;
 	stf.spill [r16]=f6,32
 	stf.spill [r17]=f7,32
	;;
 	stf.spill [r16]=f8,32
 	stf.spill [r17]=f9,32
	;;
 	stf.spill [r16]=f10
 	stf.spill [r17]=f11
511
	br.call.sptk.many rp=syscall_trace_enter // give parent a chance to catch syscall args
512 513 514 515 516 517 518 519 520 521 522
	adds r16=PT(F6)+16,sp
	adds r17=PT(F7)+16,sp
	;;
	ldf.fill f6=[r16],32
	ldf.fill f7=[r17],32
	;;
	ldf.fill f8=[r16],32
	ldf.fill f9=[r17],32
	;;
	ldf.fill f10=[r16]
	ldf.fill f11=[r17]
523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541
	// the syscall number may have changed, so re-load it and re-calculate the
	// syscall entry-point:
	adds r15=PT(R15)+16,sp			// r15 = &pt_regs.r15 (syscall #)
	;;
	ld8 r15=[r15]
	mov r3=NR_syscalls - 1
	;;
	adds r15=-1024,r15
	movl r16=sys_call_table
	;;
	shladd r20=r15,3,r16			// r20 = sys_call_table + 8*(syscall-1024)
	cmp.leu p6,p7=r15,r3
	;;
(p6)	ld8 r20=[r20]				// load address of syscall entry point
(p7)	movl r20=sys_ni_syscall
	;;
	mov b6=r20
	br.call.sptk.many rp=b6			// do the syscall
.strace_check_retval:
Linus Torvalds's avatar
Linus Torvalds committed
542
	cmp.lt p6,p0=r8,r0			// syscall failed?
Linus Torvalds's avatar
Linus Torvalds committed
543 544
	adds r2=PT(R8)+16,sp			// r2 = &pt_regs.r8
	adds r3=PT(R10)+16,sp			// r3 = &pt_regs.r10
Linus Torvalds's avatar
Linus Torvalds committed
545
	mov r10=0
Linus Torvalds's avatar
Linus Torvalds committed
546
(p6)	br.cond.sptk strace_error		// syscall failed ->
Linus Torvalds's avatar
Linus Torvalds committed
547
	;;					// avoid RAW on r10
548
.strace_save_retval:
549 550
.mem.offset 0,0; st8.spill [r2]=r8		// store return value in slot for r8
.mem.offset 8,0; st8.spill [r3]=r10		// clear error indication in slot for r10
551
	br.call.sptk.many rp=syscall_trace_leave // give parent a chance to catch return value
552
.ret3:	br.cond.sptk ia64_leave_syscall
Linus Torvalds's avatar
Linus Torvalds committed
553 554 555 556 557 558 559 560 561 562

strace_error:
	ld8 r3=[r2]				// load pt_regs.r8
	sub r9=0,r8				// negate return value to get errno value
	;;
	cmp.ne p6,p0=r3,r0			// is pt_regs.r8!=0?
	adds r3=16,r2				// r3=&pt_regs.r10
	;;
(p6)	mov r10=-1
(p6)	mov r8=r9
563
	br.cond.sptk .strace_save_retval
Linus Torvalds's avatar
Linus Torvalds committed
564 565
END(ia64_trace_syscall)

566 567 568 569 570 571 572 573 574 575 576 577
	/*
	 * When traced and returning from sigreturn, we invoke syscall_trace but then
	 * go straight to ia64_leave_kernel rather than ia64_leave_syscall.
	 */
GLOBAL_ENTRY(ia64_strace_leave_kernel)
	PT_REGS_UNWIND_INFO(0)
{	/*
	 * Some versions of gas generate bad unwind info if the first instruction of a
	 * procedure doesn't go into the first slot of a bundle.  This is a workaround.
	 */
	nop.m 0
	nop.i 0
578
	br.call.sptk.many rp=syscall_trace_leave // give parent a chance to catch return value
579 580 581 582
}
.ret4:	br.cond.sptk ia64_leave_kernel
END(ia64_strace_leave_kernel)

Linus Torvalds's avatar
Linus Torvalds committed
583 584
GLOBAL_ENTRY(ia64_ret_from_clone)
	PT_REGS_UNWIND_INFO(0)
585 586 587 588 589 590
{	/*
	 * Some versions of gas generate bad unwind info if the first instruction of a
	 * procedure doesn't go into the first slot of a bundle.  This is a workaround.
	 */
	nop.m 0
	nop.i 0
Linus Torvalds's avatar
Linus Torvalds committed
591
	/*
Linus Torvalds's avatar
Linus Torvalds committed
592
	 * We need to call schedule_tail() to complete the scheduling process.
593
	 * Called by ia64_switch_to() after do_fork()->copy_thread().  r8 contains the
Linus Torvalds's avatar
Linus Torvalds committed
594 595
	 * address of the previously executing task.
	 */
Linus Torvalds's avatar
Linus Torvalds committed
596
	br.call.sptk.many rp=ia64_invoke_schedule_tail
597
}
Linus Torvalds's avatar
Linus Torvalds committed
598
.ret8:
David Mosberger's avatar
David Mosberger committed
599
	adds r2=TI_FLAGS+IA64_TASK_SIZE,r13
Linus Torvalds's avatar
Linus Torvalds committed
600
	;;
David Mosberger's avatar
David Mosberger committed
601
	ld4 r2=[r2]
Linus Torvalds's avatar
Linus Torvalds committed
602 603
	;;
	mov r8=0
604 605 606
	and r2=_TIF_SYSCALL_TRACEAUDIT,r2
	;;
	cmp.ne p6,p0=r2,r0
607
(p6)	br.cond.spnt .strace_check_retval
Linus Torvalds's avatar
Linus Torvalds committed
608 609 610 611 612 613
	;;					// added stop bits to prevent r8 dependency
END(ia64_ret_from_clone)
	// fall through
GLOBAL_ENTRY(ia64_ret_from_syscall)
	PT_REGS_UNWIND_INFO(0)
	cmp.ge p6,p7=r8,r0			// syscall executed successfully?
Linus Torvalds's avatar
Linus Torvalds committed
614 615
	adds r2=PT(R8)+16,sp			// r2 = &pt_regs.r8
	adds r3=PT(R10)+16,sp			// r3 = &pt_regs.r10
Linus Torvalds's avatar
Linus Torvalds committed
616
	;;
617 618
.mem.offset 0,0; (p6) st8.spill [r2]=r8	// store return value in slot for r8 and set unat bit
.mem.offset 8,0; (p6) st8.spill [r3]=r0	// clear error indication in slot for r10 and set unat bit
Linus Torvalds's avatar
Linus Torvalds committed
619
(p7)	br.cond.spnt handle_syscall_error	// handle potential syscall failure
Linus Torvalds's avatar
Linus Torvalds committed
620 621
END(ia64_ret_from_syscall)
	// fall through
622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638
/*
 * ia64_leave_syscall(): Same as ia64_leave_kernel, except that it doesn't
 *	need to switch to bank 0 and doesn't restore the scratch registers.
 *	To avoid leaking kernel bits, the scratch registers are set to
 *	the following known-to-be-safe values:
 *
 *		  r1: restored (global pointer)
 *		  r2: cleared
 *		  r3: 1 (when returning to user-level)
 *	      r8-r11: restored (syscall return value(s))
 *		 r12: restored (user-level stack pointer)
 *		 r13: restored (user-level thread pointer)
 *		 r14: cleared
 *		 r15: restored (syscall #)
 *	     r16-r19: cleared
 *		 r20: user-level ar.fpsr
 *		 r21: user-level b0
639
 *		 r22: user-level b6
640 641 642 643 644 645 646 647 648 649 650 651
 *		 r23: user-level ar.bspstore
 *		 r24: user-level ar.rnat
 *		 r25: user-level ar.unat
 *		 r26: user-level ar.pfs
 *		 r27: user-level ar.rsc
 *		 r28: user-level ip
 *		 r29: user-level psr
 *		 r30: user-level cfm
 *		 r31: user-level pr
 *	      f6-f11: cleared
 *		  pr: restored (user-level pr)
 *		  b0: restored (user-level rp)
652 653
 *	          b6: restored
 *		  b7: cleared
654 655 656 657 658 659 660 661 662 663 664
 *	     ar.unat: restored (user-level ar.unat)
 *	      ar.pfs: restored (user-level ar.pfs)
 *	      ar.rsc: restored (user-level ar.rsc)
 *	     ar.rnat: restored (user-level ar.rnat)
 *	 ar.bspstore: restored (user-level ar.bspstore)
 *	     ar.fpsr: restored (user-level ar.fpsr)
 *	      ar.ccv: cleared
 *	      ar.csd: cleared
 *	      ar.ssd: cleared
 */
GLOBAL_ENTRY(ia64_leave_syscall)
Linus Torvalds's avatar
Linus Torvalds committed
665
	PT_REGS_UNWIND_INFO(0)
666 667
	/*
	 * work.need_resched etc. mustn't get changed by this CPU before it returns to
668 669 670 671 672 673 674 675
	 * user- or fsys-mode, hence we disable interrupts early on.
	 *
	 * p6 controls whether current_thread_info()->flags needs to be check for
	 * extra work.  We always check for extra work when returning to user-level.
	 * With CONFIG_PREEMPT, we also check for extra work when the preempt_count
	 * is 0.  After extra work processing has been completed, execution
	 * resumes at .work_processed_syscall with p6 set to 1 if the extra-work-check
	 * needs to be redone.
676
	 */
677 678
#ifdef CONFIG_PREEMPT
	rsm psr.i				// disable interrupts
679
	cmp.eq pLvSys,p0=r0,r0			// pLvSys=1: leave from syscall
680 681
(pKStk) adds r20=TI_PRE_COUNT+IA64_TASK_SIZE,r13
	;;
682 683 684
	.pred.rel.mutex pUStk,pKStk
(pKStk) ld4 r21=[r20]			// r21 <- preempt_count
(pUStk)	mov r21=0			// r21 <- 0
Linus Torvalds's avatar
Linus Torvalds committed
685
	;;
686 687 688 689 690 691 692
	cmp.eq p6,p0=r21,r0		// p6 <- pUStk || (preempt_count == 0)
#else /* !CONFIG_PREEMPT */
(pUStk)	rsm psr.i
	cmp.eq pLvSys,p0=r0,r0		// pLvSys=1: leave from syscall
(pUStk)	cmp.eq.unc p6,p0=r0,r0		// p6 <- pUStk
#endif
.work_processed_syscall:
693
	adds r16=PT(LOADRS)+16,r12
694 695 696 697 698 699
	adds r17=PT(AR_BSPSTORE)+16,r12
	adds r18=TI_FLAGS+IA64_TASK_SIZE,r13
	;;
(p6)	ld4 r31=[r18]				// load current_thread_info()->flags
	ld8 r19=[r16],PT(B6)-PT(LOADRS)		// load ar.rsc value for "loadrs"
	nop.i 0
700
	;;
701 702
	ld8 r23=[r17],PT(R9)-PT(AR_BSPSTORE)	// load ar.bspstore (may be garbage)
	ld8 r22=[r16],PT(R8)-PT(B6)		// load b6
703
(p6)	and r15=TIF_WORK_MASK,r31		// any work other than TIF_SYSCALL_TRACE?
704
	;;
705 706

	mov.m ar.ccv=r0		// clear ar.ccv
707
(p6)	cmp4.ne.unc p6,p0=r15, r0		// any special work pending?
708
(p6)	br.cond.spnt .work_pending
Linus Torvalds's avatar
Linus Torvalds committed
709 710
	;;
	// start restoring the state saved on the kernel stack (struct pt_regs):
711 712
	ld8.fill r8=[r16],16
	ld8.fill r9=[r17],16
713
	mov f6=f0		// clear f6
714 715 716
	;;
	ld8.fill r10=[r16],16
	ld8.fill r11=[r17],16
717
	mov f7=f0		// clear f7
718
	;;
719 720
	ld8 r29=[r16],16	// load cr.ipsr
	ld8 r28=[r17],16	// load cr.iip
721
	mov f8=f0		// clear f8
722
	;;
723 724
	ld8 r30=[r16],16	// load cr.ifs
	ld8 r25=[r17],16	// load ar.unat
725 726 727 728
	cmp.eq p9,p0=r0,r0	// set p9 to indicate that we should restore cr.ifs
	;;
	rsm psr.i | psr.ic	// initiate turning off of interrupt and interruption collection
	invala			// invalidate ALAT
729
	mov f9=f0		// clear f9
730 731 732 733

	mov.m ar.ssd=r0		// clear ar.ssd
	mov.m ar.csd=r0		// clear ar.csd
	mov f10=f0		// clear f10
734
	;;
735 736
	ld8 r26=[r16],16	// load ar.pfs
	ld8 r27=[r17],PT(PR)-PT(AR_RSC)	// load ar.rsc
737
	mov f11=f0		// clear f11
738
	;;
739 740
	ld8 r24=[r16],PT(B0)-PT(AR_RNAT)	// load ar.rnat (may be garbage)
	ld8 r31=[r17],PT(R1)-PT(PR)		// load predicates
741
(pUStk) add r14=IA64_TASK_THREAD_ON_USTACK_OFFSET,r13
742
	;;
743
	ld8 r21=[r16],PT(R12)-PT(B0) // load b0
744
	ld8.fill r1=[r17],16	// load r1
745
(pUStk) mov r3=1
746 747 748 749 750
	;;
	ld8.fill r12=[r16],16
	ld8.fill r13=[r17],16
	mov r2=r0		// clear r2
	;;
751
	ld8 r20=[r16]		// load ar.fpsr
752 753 754
	ld8.fill r15=[r17]	// load r15
	mov b7=r0		// clear b7
	;;
755
(pUStk) st1 [r14]=r3
756
	addl r17=THIS_CPU(ia64_phys_stacked_size_p8),r0
757
	;;
758 759
	mov r16=ar.bsp		// get existing backing store pointer
	srlz.i			// ensure interruption collection is off
760
	mov r14=r0		// clear r14
761 762 763
	;;
	ld4 r17=[r17]		// r17 = cpu_data->phys_stacked_size_p8
	mov b6=r22				// restore b6
764 765
	shr.u r18=r19,16	// get byte size of existing "dirty" partition
(pKStk) br.cond.dpnt.many skip_rbs_switch
766
(pNonSys) br.cond.dpnt.many dont_preserve_current_frame
767 768
	br.cond.sptk.many rbs_switch
END(ia64_leave_syscall)
769

770 771 772 773 774 775 776 777 778 779 780 781 782
#ifdef CONFIG_IA32_SUPPORT
GLOBAL_ENTRY(ia64_ret_from_ia32_execve)
	PT_REGS_UNWIND_INFO(0)
	adds r2=PT(R8)+16,sp			// r2 = &pt_regs.r8
	adds r3=PT(R10)+16,sp			// r3 = &pt_regs.r10
	;;
	.mem.offset 0,0
	st8.spill [r2]=r8	// store return value in slot for r8 and set unat bit
	.mem.offset 8,0
	st8.spill [r3]=r0	// clear error indication in slot for r10 and set unat bit
END(ia64_ret_from_ia32_execve_syscall)
	// fall through
#endif /* CONFIG_IA32_SUPPORT */
783 784
GLOBAL_ENTRY(ia64_leave_kernel)
	PT_REGS_UNWIND_INFO(0)
785 786
	/*
	 * work.need_resched etc. mustn't get changed by this CPU before it returns to
787 788 789 790 791 792 793 794
	 * user- or fsys-mode, hence we disable interrupts early on.
	 *
	 * p6 controls whether current_thread_info()->flags needs to be check for
	 * extra work.  We always check for extra work when returning to user-level.
	 * With CONFIG_PREEMPT, we also check for extra work when the preempt_count
	 * is 0.  After extra work processing has been completed, execution
	 * resumes at .work_processed_syscall with p6 set to 1 if the extra-work-check
	 * needs to be redone.
795
	 */
796 797
#ifdef CONFIG_PREEMPT
	rsm psr.i				// disable interrupts
798
	cmp.eq p0,pLvSys=r0,r0			// pLvSys=0: leave from kernel
799
(pKStk)	adds r20=TI_PRE_COUNT+IA64_TASK_SIZE,r13
Linus Torvalds's avatar
Linus Torvalds committed
800
	;;
801 802 803 804
	.pred.rel.mutex pUStk,pKStk
(pKStk)	ld4 r21=[r20]			// r21 <- preempt_count
(pUStk)	mov r21=0			// r21 <- 0
	;;
805 806 807 808 809 810 811
	cmp.eq p6,p0=r21,r0		// p6 <- pUStk || (preempt_count == 0)
#else
(pUStk)	rsm psr.i
	cmp.eq p0,pLvSys=r0,r0		// pLvSys=0: leave from kernel
(pUStk)	cmp.eq.unc p6,p0=r0,r0		// p6 <- pUStk
#endif
.work_processed_kernel:
812
	adds r17=TI_FLAGS+IA64_TASK_SIZE,r13
Linus Torvalds's avatar
Linus Torvalds committed
813
	;;
814 815
(p6)	ld4 r31=[r17]				// load current_thread_info()->flags
	adds r21=PT(PR)+16,r12
Linus Torvalds's avatar
Linus Torvalds committed
816
	;;
817 818

	lfetch [r21],PT(CR_IPSR)-PT(PR)
819 820 821
	adds r2=PT(B6)+16,r12
	adds r3=PT(R16)+16,r12
	;;
822 823
	lfetch [r21]
	ld8 r28=[r2],8		// load b6
824
	adds r29=PT(R24)+16,r12
825 826

	ld8.fill r16=[r3],PT(AR_CSD)-PT(R16)
827 828 829 830
	adds r30=PT(AR_CCV)+16,r12
(p6)	and r19=TIF_WORK_MASK,r31		// any work other than TIF_SYSCALL_TRACE?
	;;
	ld8.fill r24=[r29]
831
	ld8 r15=[r30]		// load ar.ccv
David Mosberger's avatar
David Mosberger committed
832
(p6)	cmp4.ne.unc p6,p0=r19, r0		// any special work pending?
Linus Torvalds's avatar
Linus Torvalds committed
833
	;;
834 835
	ld8 r29=[r2],16		// load b7
	ld8 r30=[r3],16		// load ar.csd
David Mosberger's avatar
David Mosberger committed
836
(p6)	br.cond.spnt .work_pending
837
	;;
838
	ld8 r31=[r2],16		// load ar.ssd
839 840 841 842 843 844 845
	ld8.fill r8=[r3],16
	;;
	ld8.fill r9=[r2],16
	ld8.fill r10=[r3],PT(R17)-PT(R10)
	;;
	ld8.fill r11=[r2],PT(R18)-PT(R11)
	ld8.fill r17=[r3],16
Linus Torvalds's avatar
Linus Torvalds committed
846 847 848 849 850 851
	;;
	ld8.fill r18=[r2],16
	ld8.fill r19=[r3],16
	;;
	ld8.fill r20=[r2],16
	ld8.fill r21=[r3],16
852 853
	mov ar.csd=r30
	mov ar.ssd=r31
Linus Torvalds's avatar
Linus Torvalds committed
854
	;;
855 856
	rsm psr.i | psr.ic	// initiate turning off of interrupt and interruption collection
	invala			// invalidate ALAT
Linus Torvalds's avatar
Linus Torvalds committed
857
	;;
858 859 860
	ld8.fill r22=[r2],24
	ld8.fill r23=[r3],24
	mov b6=r28
Linus Torvalds's avatar
Linus Torvalds committed
861
	;;
862 863 864
	ld8.fill r25=[r2],16
	ld8.fill r26=[r3],16
	mov b7=r29
Linus Torvalds's avatar
Linus Torvalds committed
865
	;;
866 867
	ld8.fill r27=[r2],16
	ld8.fill r28=[r3],16
Linus Torvalds's avatar
Linus Torvalds committed
868
	;;
869 870
	ld8.fill r29=[r2],16
	ld8.fill r30=[r3],24
Linus Torvalds's avatar
Linus Torvalds committed
871
	;;
872 873
	ld8.fill r31=[r2],PT(F9)-PT(R31)
	adds r3=PT(F10)-PT(F6),r3
Linus Torvalds's avatar
Linus Torvalds committed
874
	;;
875 876
	ldf.fill f9=[r2],PT(F6)-PT(F9)
	ldf.fill f10=[r3],PT(F8)-PT(F10)
Linus Torvalds's avatar
Linus Torvalds committed
877
	;;
878
	ldf.fill f6=[r2],PT(F7)-PT(F6)
Linus Torvalds's avatar
Linus Torvalds committed
879
	;;
880 881
	ldf.fill f7=[r2],PT(F11)-PT(F7)
	ldf.fill f8=[r3],32
Linus Torvalds's avatar
Linus Torvalds committed
882
	;;
David Mosberger's avatar
David Mosberger committed
883
	srlz.i			// ensure interruption collection is off
884
	mov ar.ccv=r15
Linus Torvalds's avatar
Linus Torvalds committed
885
	;;
886
	bsw.0			// switch back to bank 0 (no stop bit required beforehand...)
Linus Torvalds's avatar
Linus Torvalds committed
887
	;;
888
	ldf.fill f11=[r2]
889
(pUStk)	mov r18=IA64_KR(CURRENT)	// Itanium 2: 12 cycle read latency
890 891
	adds r16=PT(CR_IPSR)+16,r12
	adds r17=PT(CR_IIP)+16,r12
Linus Torvalds's avatar
Linus Torvalds committed
892
	;;
893 894
	ld8 r29=[r16],16	// load cr.ipsr
	ld8 r28=[r17],16	// load cr.iip
Linus Torvalds's avatar
Linus Torvalds committed
895
	;;
896 897
	ld8 r30=[r16],16	// load cr.ifs
	ld8 r25=[r17],16	// load ar.unat
Linus Torvalds's avatar
Linus Torvalds committed
898
	;;
899 900
	ld8 r26=[r16],16	// load ar.pfs
	ld8 r27=[r17],16	// load ar.rsc
901
	cmp.eq p9,p0=r0,r0	// set p9 to indicate that we should restore cr.ifs
Linus Torvalds's avatar
Linus Torvalds committed
902
	;;
903 904
	ld8 r24=[r16],16	// load ar.rnat (may be garbage)
	ld8 r23=[r17],16// load ar.bspstore (may be garbage)
Linus Torvalds's avatar
Linus Torvalds committed
905
	;;
906 907
	ld8 r31=[r16],16	// load predicates
	ld8 r21=[r17],16	// load b0
Linus Torvalds's avatar
Linus Torvalds committed
908
	;;
Linus Torvalds's avatar
Linus Torvalds committed
909
	ld8 r19=[r16],16	// load ar.rsc value for "loadrs"
Linus Torvalds's avatar
Linus Torvalds committed
910 911 912 913
	ld8.fill r1=[r17],16	// load r1
	;;
	ld8.fill r12=[r16],16
	ld8.fill r13=[r17],16
914
(pUStk)	adds r18=IA64_TASK_THREAD_ON_USTACK_OFFSET,r18
Linus Torvalds's avatar
Linus Torvalds committed
915
	;;
916
	ld8 r20=[r16],16	// ar.fpsr
917 918 919 920
	ld8.fill r15=[r17],16
	;;
	ld8.fill r14=[r16],16
	ld8.fill r2=[r17]
921 922
(pUStk)	mov r17=1
	;;
923
	ld8.fill r3=[r16]
924
(pUStk)	st1 [r18]=r17		// restore current->thread.on_ustack
Linus Torvalds's avatar
Linus Torvalds committed
925
	shr.u r18=r19,16	// get byte size of existing "dirty" partition
Linus Torvalds's avatar
Linus Torvalds committed
926
	;;
Linus Torvalds's avatar
Linus Torvalds committed
927
	mov r16=ar.bsp		// get existing backing store pointer
928
	addl r17=THIS_CPU(ia64_phys_stacked_size_p8),r0
Linus Torvalds's avatar
Linus Torvalds committed
929 930
	;;
	ld4 r17=[r17]		// r17 = cpu_data->phys_stacked_size_p8
931 932
(pKStk)	br.cond.dpnt skip_rbs_switch

Linus Torvalds's avatar
Linus Torvalds committed
933 934 935 936 937
	/*
	 * Restore user backing store.
	 *
	 * NOTE: alloc, loadrs, and cover can't be predicated.
	 */
Linus Torvalds's avatar
Linus Torvalds committed
938
(pNonSys) br.cond.dpnt dont_preserve_current_frame
939 940

rbs_switch:
David Mosberger's avatar
David Mosberger committed
941
	cover				// add current frame into dirty partition and set cr.ifs
Linus Torvalds's avatar
Linus Torvalds committed
942
	;;
Linus Torvalds's avatar
Linus Torvalds committed
943 944 945
	mov r19=ar.bsp			// get new backing store pointer
	sub r16=r16,r18			// krbs = old bsp - size of dirty partition
	cmp.ne p9,p0=r0,r0		// clear p9 to skip restore of cr.ifs
Linus Torvalds's avatar
Linus Torvalds committed
946
	;;
Linus Torvalds's avatar
Linus Torvalds committed
947 948
	sub r19=r19,r16			// calculate total byte size of dirty partition
	add r18=64,r18			// don't force in0-in7 into memory...
Linus Torvalds's avatar
Linus Torvalds committed
949
	;;
Linus Torvalds's avatar
Linus Torvalds committed
950
	shl r19=r19,16			// shift size of dirty partition into loadrs position
Linus Torvalds's avatar
Linus Torvalds committed
951 952
	;;
dont_preserve_current_frame:
Linus Torvalds's avatar
Linus Torvalds committed
953 954 955
	/*
	 * To prevent leaking bits between the kernel and user-space,
	 * we must clear the stacked registers in the "invalid" partition here.
956 957
	 * Not pretty, but at least it's fast (3.34 registers/cycle on Itanium,
	 * 5 registers/cycle on McKinley).
Linus Torvalds's avatar
Linus Torvalds committed
958 959 960
	 */
#	define pRecurse	p6
#	define pReturn	p7
961
#ifdef CONFIG_ITANIUM
Linus Torvalds's avatar
Linus Torvalds committed
962
#	define Nregs	10
963 964 965
#else
#	define Nregs	14
#endif
Linus Torvalds's avatar
Linus Torvalds committed
966
	alloc loc0=ar.pfs,2,Nregs-2,2,0
David Mosberger's avatar
David Mosberger committed
967
	shr.u loc1=r18,9		// RNaTslots <= floor(dirtySize / (64*8))
Linus Torvalds's avatar
Linus Torvalds committed
968 969 970 971 972 973
	sub r17=r17,r18			// r17 = (physStackedSize + 8) - dirtySize
	;;
	mov ar.rsc=r19			// load ar.rsc to be used for "loadrs"
	shladd in0=loc1,3,r17
	mov in1=0
	;;
974
	.align 32
Linus Torvalds's avatar
Linus Torvalds committed
975
rse_clear_invalid:
976
#ifdef CONFIG_ITANIUM
Linus Torvalds's avatar
Linus Torvalds committed
977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993
	// cycle 0
 { .mii
	alloc loc0=ar.pfs,2,Nregs-2,2,0
	cmp.lt pRecurse,p0=Nregs*8,in0	// if more than Nregs regs left to clear, (re)curse
	add out0=-Nregs*8,in0
}{ .mfb
	add out1=1,in1			// increment recursion count
	nop.f 0
	nop.b 0				// can't do br.call here because of alloc (WAW on CFM)
	;;
}{ .mfi	// cycle 1
	mov loc1=0
	nop.f 0
	mov loc2=0
}{ .mib
	mov loc3=0
	mov loc4=0
994
(pRecurse) br.call.sptk.many b0=rse_clear_invalid
Linus Torvalds's avatar
Linus Torvalds committed
995 996 997 998 999 1000 1001 1002

}{ .mfi	// cycle 2
	mov loc5=0
	nop.f 0
	cmp.ne pReturn,p0=r0,in1	// if recursion count != 0, we need to do a br.ret
}{ .mib
	mov loc6=0
	mov loc7=0
1003
(pReturn) br.ret.sptk.many b0
Linus Torvalds's avatar
Linus Torvalds committed
1004
}
1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016
#else /* !CONFIG_ITANIUM */
	alloc loc0=ar.pfs,2,Nregs-2,2,0
	cmp.lt pRecurse,p0=Nregs*8,in0	// if more than Nregs regs left to clear, (re)curse
	add out0=-Nregs*8,in0
	add out1=1,in1			// increment recursion count
	mov loc1=0
	mov loc2=0
	;;
	mov loc3=0
	mov loc4=0
	mov loc5=0
	mov loc6=0
David Mosberger's avatar
David Mosberger committed
1017
	mov loc7=0
1018
(pRecurse) br.call.sptk.few b0=rse_clear_invalid
1019 1020
	;;
	mov loc8=0
David Mosberger's avatar
David Mosberger committed
1021
	mov loc9=0
1022 1023 1024
	cmp.ne pReturn,p0=r0,in1	// if recursion count != 0, we need to do a br.ret
	mov loc10=0
	mov loc11=0
1025
(pReturn) br.ret.sptk.many b0
1026
#endif /* !CONFIG_ITANIUM */
Linus Torvalds's avatar
Linus Torvalds committed
1027 1028
#	undef pRecurse
#	undef pReturn
1029
	;;
Linus Torvalds's avatar
Linus Torvalds committed
1030
	alloc r17=ar.pfs,0,0,0,0	// drop current register frame
Linus Torvalds's avatar
Linus Torvalds committed
1031 1032 1033 1034
	;;
	loadrs
	;;
skip_rbs_switch:
1035
(pLvSys)	mov r19=r0		// clear r19 for leave_syscall, no-op otherwise
1036 1037 1038 1039
	mov b0=r21
	mov ar.pfs=r26
(pUStk)	mov ar.bspstore=r23
(p9)	mov cr.ifs=r30
1040
(pLvSys)mov r16=r0		// clear r16 for leave_syscall, no-op otherwise
1041 1042
	mov cr.ipsr=r29
	mov ar.fpsr=r20
1043
(pLvSys)mov r17=r0		// clear r17 for leave_syscall, no-op otherwise
1044 1045 1046
	mov cr.iip=r28
	;;
(pUStk)	mov ar.rnat=r24		// must happen with RSE in lazy mode
1047
(pLvSys)mov r18=r0		// clear r18 for leave_syscall, no-op otherwise
1048 1049 1050
	mov ar.rsc=r27
	mov ar.unat=r25
	mov pr=r31,-1
Linus Torvalds's avatar
Linus Torvalds committed
1051
	rfi
David Mosberger's avatar
David Mosberger committed
1052

1053 1054 1055 1056 1057 1058 1059
	/*
	 * On entry:
	 *	r20 = &current->thread_info->pre_count (if CONFIG_PREEMPT)
	 *	r31 = current->thread_info->flags
	 * On exit:
	 *	p6 = TRUE if work-pending-check needs to be redone
	 */
David Mosberger's avatar
David Mosberger committed
1060
.work_pending:
1061 1062 1063
	tbit.nz p6,p0=r31,TIF_SIGDELAYED		// signal delayed from  MCA/INIT/NMI/PMI context?
(p6)	br.cond.sptk.few .sigdelayed
	;;
1064
	tbit.z p6,p0=r31,TIF_NEED_RESCHED		// current_thread_info()->need_resched==0?
David Mosberger's avatar
David Mosberger committed
1065
(p6)	br.cond.sptk.few .notify
1066
#ifdef CONFIG_PREEMPT
1067
(pKStk) dep r21=-1,r0,PREEMPT_ACTIVE_BIT,1
1068 1069
	;;
(pKStk) st4 [r20]=r21
1070
	ssm psr.i		// enable interrupts
1071
#endif
David Mosberger's avatar
David Mosberger committed
1072 1073
	br.call.spnt.many rp=schedule
.ret9:	cmp.eq p6,p0=r0,r0				// p6 <- 1
1074
	rsm psr.i		// disable interrupts
David Mosberger's avatar
David Mosberger committed
1075
	;;
1076
#ifdef CONFIG_PREEMPT
1077 1078
(pKStk)	adds r20=TI_PRE_COUNT+IA64_TASK_SIZE,r13
	;;
1079
(pKStk)	st4 [r20]=r0		// preempt_count() <- 0
1080
#endif
1081 1082
(pLvSys)br.cond.sptk.many .work_processed_syscall	// re-check
	br.cond.sptk.many .work_processed_kernel	// re-check
David Mosberger's avatar
David Mosberger committed
1083 1084

.notify:
1085
(pUStk)	br.call.spnt.many rp=notify_resume_user
David Mosberger's avatar
David Mosberger committed
1086
.ret10:	cmp.ne p6,p0=r0,r0				// p6 <- 0
1087 1088
(pLvSys)br.cond.sptk.many .work_processed_syscall	// don't re-check
	br.cond.sptk.many .work_processed_kernel	// don't re-check
1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100

// There is a delayed signal that was detected in MCA/INIT/NMI/PMI context where
// it could not be delivered.  Deliver it now.  The signal might be for us and
// may set TIF_SIGPENDING, so redrive ia64_leave_* after processing the delayed
// signal.

.sigdelayed:
	br.call.sptk.many rp=do_sigdelayed
	cmp.eq p6,p0=r0,r0				// p6 <- 1, always re-check
(pLvSys)br.cond.sptk.many .work_processed_syscall	// re-check
	br.cond.sptk.many .work_processed_kernel	// re-check

Linus Torvalds's avatar
Linus Torvalds committed
1101 1102 1103 1104
END(ia64_leave_kernel)

ENTRY(handle_syscall_error)
	/*
1105 1106 1107 1108
	 * Some system calls (e.g., ptrace, mmap) can return arbitrary values which could
	 * lead us to mistake a negative return value as a failed syscall.  Those syscall
	 * must deposit a non-zero value in pt_regs.r8 to indicate an error.  If
	 * pt_regs.r8 is zero, we assume that the call completed successfully.
Linus Torvalds's avatar
Linus Torvalds committed
1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122
	 */
	PT_REGS_UNWIND_INFO(0)
	ld8 r3=[r2]		// load pt_regs.r8
	sub r9=0,r8		// negate return value to get errno
	;;
	mov r10=-1		// return -1 in pt_regs.r10 to indicate error
	cmp.eq p6,p7=r3,r0	// is pt_regs.r8==0?
	adds r3=16,r2		// r3=&pt_regs.r10
	;;
(p6)	mov r9=r8
(p6)	mov r10=0
	;;
.mem.offset 0,0; st8.spill [r2]=r9	// store errno in pt_regs.r8 and set unat bit
.mem.offset 8,0; st8.spill [r3]=r10	// store error indication in pt_regs.r10 and set unat bit
1123
	br.cond.sptk ia64_leave_syscall
Linus Torvalds's avatar
Linus Torvalds committed
1124 1125 1126 1127 1128 1129
END(handle_syscall_error)

	/*
	 * Invoke schedule_tail(task) while preserving in0-in7, which may be needed
	 * in case a system call gets restarted.
	 */
Linus Torvalds's avatar
Linus Torvalds committed
1130
GLOBAL_ENTRY(ia64_invoke_schedule_tail)
Linus Torvalds's avatar
Linus Torvalds committed
1131
	.prologue ASM_UNW_PRLG_RP|ASM_UNW_PRLG_PFS, ASM_UNW_PRLG_GRSAVE(8)
Linus Torvalds's avatar
Linus Torvalds committed
1132 1133 1134 1135
	alloc loc1=ar.pfs,8,2,1,0
	mov loc0=rp
	mov out0=r8				// Address of previous task
	;;
Linus Torvalds's avatar
Linus Torvalds committed
1136
	br.call.sptk.many rp=schedule_tail
Linus Torvalds's avatar
Linus Torvalds committed
1137 1138 1139
.ret11:	mov ar.pfs=loc1
	mov rp=loc0
	br.ret.sptk.many rp
Linus Torvalds's avatar
Linus Torvalds committed
1140
END(ia64_invoke_schedule_tail)
Linus Torvalds's avatar
Linus Torvalds committed
1141

Linus Torvalds's avatar
Linus Torvalds committed
1142
	/*
1143
	 * Setup stack and call do_notify_resume_user().  Note that pSys and pNonSys need to
Linus Torvalds's avatar
Linus Torvalds committed
1144 1145 1146
	 * be set up by the caller.  We declare 8 input registers so the system call
	 * args get preserved, in case we need to restart a system call.
	 */
David Mosberger's avatar
David Mosberger committed
1147
ENTRY(notify_resume_user)
Linus Torvalds's avatar
Linus Torvalds committed
1148 1149 1150 1151 1152
	.prologue ASM_UNW_PRLG_RP|ASM_UNW_PRLG_PFS, ASM_UNW_PRLG_GRSAVE(8)
	alloc loc1=ar.pfs,8,2,3,0 // preserve all eight input regs in case of syscall restart!
	mov r9=ar.unat
	mov loc0=rp				// save return address
	mov out0=0				// there is no "oldset"
1153
	adds out1=8,sp				// out1=&sigscratch->ar_pfs
Linus Torvalds's avatar
Linus Torvalds committed
1154 1155 1156 1157 1158 1159
(pSys)	mov out2=1				// out2==1 => we're in a syscall
	;;
(pNonSys) mov out2=0				// out2==0 => not a syscall
	.fframe 16
	.spillpsp ar.unat, 16			// (note that offset is relative to psp+0x10!)
	st8 [sp]=r9,-16				// allocate space for ar.unat and save it
1160
	st8 [out1]=loc1,-8			// save ar.pfs, out1=&sigscratch
Linus Torvalds's avatar
Linus Torvalds committed
1161
	.body
David Mosberger's avatar
David Mosberger committed
1162
	br.call.sptk.many rp=do_notify_resume_user
Linus Torvalds's avatar
Linus Torvalds committed
1163 1164 1165
.ret15:	.restore sp
	adds sp=16,sp				// pop scratch stack space
	;;
David Mosberger's avatar
David Mosberger committed
1166
	ld8 r9=[sp]				// load new unat from sigscratch->scratch_unat
Linus Torvalds's avatar
Linus Torvalds committed
1167 1168 1169 1170 1171
	mov rp=loc0
	;;
	mov ar.unat=r9
	mov ar.pfs=loc1
	br.ret.sptk.many rp
1172
END(notify_resume_user)
Linus Torvalds's avatar
Linus Torvalds committed
1173 1174 1175 1176 1177 1178 1179 1180

GLOBAL_ENTRY(sys_rt_sigsuspend)
	.prologue ASM_UNW_PRLG_RP|ASM_UNW_PRLG_PFS, ASM_UNW_PRLG_GRSAVE(8)
	alloc loc1=ar.pfs,8,2,3,0 // preserve all eight input regs in case of syscall restart!
	mov r9=ar.unat
	mov loc0=rp				// save return address
	mov out0=in0				// mask
	mov out1=in1				// sigsetsize
1181
	adds out2=8,sp				// out2=&sigscratch->ar_pfs
Linus Torvalds's avatar
Linus Torvalds committed
1182 1183 1184 1185
	;;
	.fframe 16
	.spillpsp ar.unat, 16			// (note that offset is relative to psp+0x10!)
	st8 [sp]=r9,-16				// allocate space for ar.unat and save it
1186
	st8 [out2]=loc1,-8			// save ar.pfs, out2=&sigscratch
Linus Torvalds's avatar
Linus Torvalds committed
1187
	.body
Linus Torvalds's avatar
Linus Torvalds committed
1188
	br.call.sptk.many rp=ia64_rt_sigsuspend
Linus Torvalds's avatar
Linus Torvalds committed
1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201
.ret17:	.restore sp
	adds sp=16,sp				// pop scratch stack space
	;;
	ld8 r9=[sp]				// load new unat from sw->caller_unat
	mov rp=loc0
	;;
	mov ar.unat=r9
	mov ar.pfs=loc1
	br.ret.sptk.many rp
END(sys_rt_sigsuspend)

ENTRY(sys_rt_sigreturn)
	PT_REGS_UNWIND_INFO(0)
Linus Torvalds's avatar
Linus Torvalds committed
1202
	alloc r2=ar.pfs,0,0,1,0
Linus Torvalds's avatar
Linus Torvalds committed
1203 1204 1205 1206
	.prologue
	PT_REGS_SAVES(16)
	adds sp=-16,sp
	.body
Linus Torvalds's avatar
Linus Torvalds committed
1207
	cmp.eq pNonSys,pSys=r0,r0		// sigreturn isn't a normal syscall...
Linus Torvalds's avatar
Linus Torvalds committed
1208
	;;
1209 1210 1211 1212 1213
	/*
	 * leave_kernel() restores f6-f11 from pt_regs, but since the streamlined
	 * syscall-entry path does not save them we save them here instead.  Note: we
	 * don't need to save any other registers that are not saved by the stream-lined
	 * syscall path, because restore_sigcontext() restores them.
1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225
	 */
	adds r16=PT(F6)+32,sp
	adds r17=PT(F7)+32,sp
	;;
 	stf.spill [r16]=f6,32
 	stf.spill [r17]=f7,32
	;;
 	stf.spill [r16]=f8,32
 	stf.spill [r17]=f9,32
	;;
 	stf.spill [r16]=f10
 	stf.spill [r17]=f11
Linus Torvalds's avatar
Linus Torvalds committed
1226
	adds out0=16,sp				// out0 = &sigscratch
Linus Torvalds's avatar
Linus Torvalds committed
1227
	br.call.sptk.many rp=ia64_rt_sigreturn
Linus Torvalds's avatar
Linus Torvalds committed
1228 1229
.ret19:	.restore sp 0
	adds sp=16,sp
Linus Torvalds's avatar
Linus Torvalds committed
1230 1231
	;;
	ld8 r9=[sp]				// load new ar.unat
Linus Torvalds's avatar
Linus Torvalds committed
1232
	mov.sptk b7=r8,ia64_leave_kernel
Linus Torvalds's avatar
Linus Torvalds committed
1233 1234
	;;
	mov ar.unat=r9
Linus Torvalds's avatar
Linus Torvalds committed
1235
	br.many b7
Linus Torvalds's avatar
Linus Torvalds committed
1236 1237 1238
END(sys_rt_sigreturn)

GLOBAL_ENTRY(ia64_prepare_handle_unaligned)
Linus Torvalds's avatar
Linus Torvalds committed
1239
	.prologue
1240 1241 1242 1243
	/*
	 * r16 = fake ar.pfs, we simply need to make sure privilege is still 0
	 */
	mov r16=r0
Linus Torvalds's avatar
Linus Torvalds committed
1244
	DO_SAVE_SWITCH_STACK
1245
	br.call.sptk.many rp=ia64_handle_unaligned	// stack frame setup in ivt
Linus Torvalds's avatar
Linus Torvalds committed
1246
.ret21:	.body
Linus Torvalds's avatar
Linus Torvalds committed
1247
	DO_LOAD_SWITCH_STACK
1248
	br.cond.sptk.many rp				// goes to ia64_leave_kernel
Linus Torvalds's avatar
Linus Torvalds committed
1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274
END(ia64_prepare_handle_unaligned)

	//
	// unw_init_running(void (*callback)(info, arg), void *arg)
	//
#	define EXTRA_FRAME_SIZE	((UNW_FRAME_INFO_SIZE+15)&~15)

GLOBAL_ENTRY(unw_init_running)
	.prologue ASM_UNW_PRLG_RP|ASM_UNW_PRLG_PFS, ASM_UNW_PRLG_GRSAVE(2)
	alloc loc1=ar.pfs,2,3,3,0
	;;
	ld8 loc2=[in0],8
	mov loc0=rp
	mov r16=loc1
	DO_SAVE_SWITCH_STACK
	.body

	.prologue ASM_UNW_PRLG_RP|ASM_UNW_PRLG_PFS, ASM_UNW_PRLG_GRSAVE(2)
	.fframe IA64_SWITCH_STACK_SIZE+EXTRA_FRAME_SIZE
	SWITCH_STACK_SAVES(EXTRA_FRAME_SIZE)
	adds sp=-EXTRA_FRAME_SIZE,sp
	.body
	;;
	adds out0=16,sp				// &info
	mov out1=r13				// current
	adds out2=16+EXTRA_FRAME_SIZE,sp	// &switch_stack
Linus Torvalds's avatar
Linus Torvalds committed
1275
	br.call.sptk.many rp=unw_init_frame_info
Linus Torvalds's avatar
Linus Torvalds committed
1276 1277 1278 1279 1280 1281
1:	adds out0=16,sp				// &info
	mov b6=loc2
	mov loc2=gp				// save gp across indirect function call
	;;
	ld8 gp=[in0]
	mov out1=in1				// arg
Linus Torvalds's avatar
Linus Torvalds committed
1282
	br.call.sptk.many rp=b6			// invoke the callback function
Linus Torvalds's avatar
Linus Torvalds committed
1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380
1:	mov gp=loc2				// restore gp

	// For now, we don't allow changing registers from within
	// unw_init_running; if we ever want to allow that, we'd
	// have to do a load_switch_stack here:
	.restore sp
	adds sp=IA64_SWITCH_STACK_SIZE+EXTRA_FRAME_SIZE,sp

	mov ar.pfs=loc1
	mov rp=loc0
	br.ret.sptk.many rp
END(unw_init_running)

	.rodata
	.align 8
	.globl sys_call_table
sys_call_table:
	data8 sys_ni_syscall		//  This must be sys_ni_syscall!  See ivt.S.
	data8 sys_exit				// 1025
	data8 sys_read
	data8 sys_write
	data8 sys_open
	data8 sys_close
	data8 sys_creat				// 1030
	data8 sys_link
	data8 sys_unlink
	data8 ia64_execve
	data8 sys_chdir
	data8 sys_fchdir			// 1035
	data8 sys_utimes
	data8 sys_mknod
	data8 sys_chmod
	data8 sys_chown
	data8 sys_lseek				// 1040
	data8 sys_getpid
	data8 sys_getppid
	data8 sys_mount
	data8 sys_umount
	data8 sys_setuid			// 1045
	data8 sys_getuid
	data8 sys_geteuid
	data8 sys_ptrace
	data8 sys_access
	data8 sys_sync				// 1050
	data8 sys_fsync
	data8 sys_fdatasync
	data8 sys_kill
	data8 sys_rename
	data8 sys_mkdir				// 1055
	data8 sys_rmdir
	data8 sys_dup
	data8 sys_pipe
	data8 sys_times
	data8 ia64_brk				// 1060
	data8 sys_setgid
	data8 sys_getgid
	data8 sys_getegid
	data8 sys_acct
	data8 sys_ioctl				// 1065
	data8 sys_fcntl
	data8 sys_umask
	data8 sys_chroot
	data8 sys_ustat
	data8 sys_dup2				// 1070
	data8 sys_setreuid
	data8 sys_setregid
	data8 sys_getresuid
	data8 sys_setresuid
	data8 sys_getresgid			// 1075
	data8 sys_setresgid
	data8 sys_getgroups
	data8 sys_setgroups
	data8 sys_getpgid
	data8 sys_setpgid			// 1080
	data8 sys_setsid
	data8 sys_getsid
	data8 sys_sethostname
	data8 sys_setrlimit
	data8 sys_getrlimit			// 1085
	data8 sys_getrusage
	data8 sys_gettimeofday
	data8 sys_settimeofday
	data8 sys_select
	data8 sys_poll				// 1090
	data8 sys_symlink
	data8 sys_readlink
	data8 sys_uselib
	data8 sys_swapon
	data8 sys_swapoff			// 1095
	data8 sys_reboot
	data8 sys_truncate
	data8 sys_ftruncate
	data8 sys_fchmod
	data8 sys_fchown			// 1100
	data8 ia64_getpriority
	data8 sys_setpriority
	data8 sys_statfs
	data8 sys_fstatfs
Linus Torvalds's avatar
Linus Torvalds committed
1381
	data8 sys_gettid			// 1105
Linus Torvalds's avatar
Linus Torvalds committed
1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395
	data8 sys_semget
	data8 sys_semop
	data8 sys_semctl
	data8 sys_msgget
	data8 sys_msgsnd			// 1110
	data8 sys_msgrcv
	data8 sys_msgctl
	data8 sys_shmget
	data8 ia64_shmat
	data8 sys_shmdt				// 1115
	data8 sys_shmctl
	data8 sys_syslog
	data8 sys_setitimer
	data8 sys_getitimer
1396 1397 1398
	data8 sys_ni_syscall			// 1120		/* was: ia64_oldstat */
	data8 sys_ni_syscall					/* was: ia64_oldlstat */
	data8 sys_ni_syscall					/* was: ia64_oldfstat */
Linus Torvalds's avatar
Linus Torvalds committed
1399 1400
	data8 sys_vhangup
	data8 sys_lchown
1401
	data8 sys_remap_file_pages		// 1125
Linus Torvalds's avatar
Linus Torvalds committed
1402 1403 1404 1405 1406 1407
	data8 sys_wait4
	data8 sys_sysinfo
	data8 sys_clone
	data8 sys_setdomainname
	data8 sys_newuname			// 1130
	data8 sys_adjtimex
1408
	data8 sys_ni_syscall					/* was: ia64_create_module */
Linus Torvalds's avatar
Linus Torvalds committed
1409 1410
	data8 sys_init_module
	data8 sys_delete_module
1411 1412
	data8 sys_ni_syscall			// 1135		/* was: sys_get_kernel_syms */
	data8 sys_ni_syscall					/* was: sys_query_module */
Linus Torvalds's avatar
Linus Torvalds committed
1413 1414 1415 1416
	data8 sys_quotactl
	data8 sys_bdflush
	data8 sys_sysfs
	data8 sys_personality			// 1140
1417
	data8 sys_ni_syscall		// sys_afs_syscall
Linus Torvalds's avatar
Linus Torvalds committed
1418 1419 1420 1421 1422 1423
	data8 sys_setfsuid
	data8 sys_setfsgid
	data8 sys_getdents
	data8 sys_flock				// 1145
	data8 sys_readv
	data8 sys_writev
1424 1425
	data8 sys_pread64
	data8 sys_pwrite64
Linus Torvalds's avatar
Linus Torvalds committed
1426 1427 1428 1429 1430 1431
	data8 sys_sysctl			// 1150
	data8 sys_mmap
	data8 sys_munmap
	data8 sys_mlock
	data8 sys_mlockall
	data8 sys_mprotect			// 1155
1432
	data8 ia64_mremap
Linus Torvalds's avatar
Linus Torvalds committed
1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462
	data8 sys_msync
	data8 sys_munlock
	data8 sys_munlockall
	data8 sys_sched_getparam		// 1160
	data8 sys_sched_setparam
	data8 sys_sched_getscheduler
	data8 sys_sched_setscheduler
	data8 sys_sched_yield
	data8 sys_sched_get_priority_max	// 1165
	data8 sys_sched_get_priority_min
	data8 sys_sched_rr_get_interval
	data8 sys_nanosleep
	data8 sys_nfsservctl
	data8 sys_prctl				// 1170
	data8 sys_getpagesize
	data8 sys_mmap2
	data8 sys_pciconfig_read
	data8 sys_pciconfig_write
	data8 sys_perfmonctl			// 1175
	data8 sys_sigaltstack
	data8 sys_rt_sigaction
	data8 sys_rt_sigpending
	data8 sys_rt_sigprocmask
	data8 sys_rt_sigqueueinfo		// 1180
	data8 sys_rt_sigreturn
	data8 sys_rt_sigsuspend
	data8 sys_rt_sigtimedwait
	data8 sys_getcwd
	data8 sys_capget			// 1185
	data8 sys_capset
1463
	data8 sys_sendfile64
Linus Torvalds's avatar
Linus Torvalds committed
1464 1465 1466 1467 1468 1469 1470 1471 1472
	data8 sys_ni_syscall		// sys_getpmsg (STREAMS)
	data8 sys_ni_syscall		// sys_putpmsg (STREAMS)
	data8 sys_socket			// 1190
	data8 sys_bind
	data8 sys_connect
	data8 sys_listen
	data8 sys_accept
	data8 sys_getsockname			// 1195
	data8 sys_getpeername
Linus Torvalds's avatar
Linus Torvalds committed
1473
	data8 sys_socketpair
Linus Torvalds's avatar
Linus Torvalds committed
1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490
	data8 sys_send
	data8 sys_sendto
	data8 sys_recv				// 1200
	data8 sys_recvfrom
	data8 sys_shutdown
	data8 sys_setsockopt
	data8 sys_getsockopt
	data8 sys_sendmsg			// 1205
	data8 sys_recvmsg
	data8 sys_pivot_root
	data8 sys_mincore
	data8 sys_madvise
	data8 sys_newstat			// 1210
	data8 sys_newlstat
	data8 sys_newfstat
	data8 sys_clone2
	data8 sys_getdents64
Linus Torvalds's avatar
Linus Torvalds committed
1491
	data8 sys_getunwind			// 1215
Linus Torvalds's avatar
Linus Torvalds committed
1492
	data8 sys_readahead
David Mosberger's avatar
David Mosberger committed
1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504
	data8 sys_setxattr
	data8 sys_lsetxattr
	data8 sys_fsetxattr
	data8 sys_getxattr			// 1220
	data8 sys_lgetxattr
	data8 sys_fgetxattr
	data8 sys_listxattr
	data8 sys_llistxattr
	data8 sys_flistxattr			// 1225
	data8 sys_removexattr
	data8 sys_lremovexattr
	data8 sys_fremovexattr
1505
	data8 sys_tkill
1506
	data8 sys_futex				// 1230
1507 1508
	data8 sys_sched_setaffinity
	data8 sys_sched_getaffinity
1509
	data8 sys_set_tid_address
1510
	data8 sys_fadvise64_64
1511
	data8 sys_tgkill 			// 1235
1512
	data8 sys_exit_group
1513
	data8 sys_lookup_dcookie
1514 1515 1516 1517 1518
	data8 sys_io_setup
	data8 sys_io_destroy
	data8 sys_io_getevents			// 1240
	data8 sys_io_submit
	data8 sys_io_cancel
David Mosberger's avatar
David Mosberger committed
1519 1520 1521
	data8 sys_epoll_create
	data8 sys_epoll_ctl
	data8 sys_epoll_wait			// 1245
David Mosberger's avatar
David Mosberger committed
1522
	data8 sys_restart_syscall
1523
	data8 sys_semtimedop
1524 1525 1526 1527 1528 1529 1530 1531 1532
	data8 sys_timer_create
	data8 sys_timer_settime
	data8 sys_timer_gettime			// 1250
	data8 sys_timer_getoverrun
	data8 sys_timer_delete
	data8 sys_clock_settime
	data8 sys_clock_gettime
	data8 sys_clock_getres			// 1255
	data8 sys_clock_nanosleep
1533 1534
	data8 sys_fstatfs64
	data8 sys_statfs64
1535 1536 1537
	data8 sys_mbind
	data8 sys_get_mempolicy			// 1260
	data8 sys_set_mempolicy
1538 1539 1540 1541 1542 1543
	data8 sys_mq_open
	data8 sys_mq_unlink
	data8 sys_mq_timedsend
	data8 sys_mq_timedreceive		// 1265
	data8 sys_mq_notify
	data8 sys_mq_getsetattr
1544
	data8 sys_ni_syscall			// reserved for kexec_load
1545 1546
	data8 sys_ni_syscall			// reserved for vserver
	data8 sys_waitid			// 1270
1547 1548 1549
	data8 sys_add_key
	data8 sys_request_key
	data8 sys_keyctl
1550 1551 1552 1553 1554 1555
	data8 sys_ni_syscall
	data8 sys_ni_syscall			// 1275
	data8 sys_ni_syscall
	data8 sys_ni_syscall
	data8 sys_ni_syscall
	data8 sys_ni_syscall
1556 1557

	.org sys_call_table + 8*NR_syscalls	// guard against failures to increase NR_syscalls