[PATCH] Move __preempt_*lock into kernel_spinlock, clean up.

 - create in_lock_functions() to match in_sched_functions(). Export it
   for use in oprofile.
 - use char __lock_text_start[] instead of long __lock_text_start when
   declaring linker symbols. Rusty fixed a number of these a while ago
   based on advice from rth.
 - Move __preempt_*_lock into kernel/spinlock.c and make it inline. This
   means locks are only one deep.
 - Make in_sched_functions() check in_lock_functions()
Signed-off-by: Anton Blanchard <anton@samba.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

arch/arm/kernel/time.c

@@ -57,8 +57,7 @@ unsigned long profile_pc(struct pt_regs *regs)
 {
 	unsigned long fp, pc = instruction_pointer(regs);
 
-	if (pc >= (unsigned long)&__lock_text_start &&
-	    pc <= (unsigned long)&__lock_text_end) {
+	if (in_lock_functions(pc)) {
 		fp = thread_saved_fp(current);
 		pc = pc_pointer(((unsigned long *)fp)[-1]);
 	}

arch/i386/kernel/time.c

@@ -205,8 +205,7 @@ unsigned long profile_pc(struct pt_regs *regs)
 {
 	unsigned long pc = instruction_pointer(regs);
 
-	if (pc >= (unsigned long)&__lock_text_start &&
-	    pc <= (unsigned long)&__lock_text_end)
+	if (in_lock_functions(pc))
 		return *(unsigned long *)(regs->ebp + 4);
 
 	return pc;

arch/ppc/kernel/time.c

@@ -113,8 +113,7 @@ unsigned long profile_pc(struct pt_regs *regs)
 {
 	unsigned long pc = instruction_pointer(regs);
 
-	if (pc >= (unsigned long)&__lock_text_start &&
-	    pc <= (unsigned long)&__lock_text_end)
+	if (in_lock_functions(pc))
 		return regs->link;
 
 	return pc;

arch/ppc64/kernel/time.c

@@ -163,8 +163,7 @@ unsigned long profile_pc(struct pt_regs *regs)
 {
 	unsigned long pc = instruction_pointer(regs);
 
-	if (pc >= (unsigned long)&__lock_text_start &&
-	    pc <= (unsigned long)&__lock_text_end)
+	if (in_lock_functions(pc))
 		return regs->link;
 
 	return pc;

arch/sparc/kernel/time.c

@@ -81,20 +81,22 @@ struct intersil *intersil_clock;
 
 unsigned long profile_pc(struct pt_regs *regs)
 {
-	extern int __copy_user_begin, __copy_user_end;
-	extern int __atomic_begin, __atomic_end;
-	extern int __bzero_begin, __bzero_end;
-	extern int __bitops_begin, __bitops_end;
+	extern char __copy_user_begin[], __copy_user_end[];
+	extern char __atomic_begin[], __atomic_end[];
+	extern char __bzero_begin[], __bzero_end[];
+	extern char __bitops_begin[], __bitops_end[];
+
 	unsigned long pc = regs->pc;
 
-	if ((pc >= (unsigned long) &__copy_user_begin &&
-	     pc < (unsigned long) &__copy_user_end) ||
-	    (pc >= (unsigned long) &__atomic_begin &&
-	     pc < (unsigned long) &__atomic_end) ||
-	    (pc >= (unsigned long) &__bzero_begin &&
-	     pc < (unsigned long) &__bzero_end) ||
-	    (pc >= (unsigned long) &__bitops_begin &&
-	     pc < (unsigned long) &__bitops_end))
+	if (in_lock_functions(pc) ||
+	    (pc >= (unsigned long) __copy_user_begin &&
+	     pc < (unsigned long) __copy_user_end) ||
+	    (pc >= (unsigned long) __atomic_begin &&
+	     pc < (unsigned long) __atomic_end) ||
+	    (pc >= (unsigned long) __bzero_begin &&
+	     pc < (unsigned long) __bzero_end) ||
+	    (pc >= (unsigned long) __bitops_begin &&
+	     pc < (unsigned long) __bitops_end))
 		pc = regs->u_regs[UREG_RETPC];
 	return pc;
 }

arch/sparc64/kernel/time.c

@@ -73,8 +73,7 @@ unsigned long profile_pc(struct pt_regs *regs)
 {
 	unsigned long pc = instruction_pointer(regs);
 
-	if (pc >= (unsigned long)&__lock_text_start &&
-	    pc <= (unsigned long)&__lock_text_end)
+	if (in_lock_functions(pc))
 		return regs->u_regs[UREG_RETPC];
 	return pc;
 }

arch/x86_64/kernel/time.c

@@ -184,8 +184,7 @@ unsigned long profile_pc(struct pt_regs *regs)
 {
 	unsigned long pc = instruction_pointer(regs);
 
-	if (pc >= (unsigned long)&__lock_text_start &&
-	    pc <= (unsigned long)&__lock_text_end)
+	if (in_lock_functions(pc))
 		return *(unsigned long *)regs->rbp;
 	return pc;
 }

include/linux/spinlock.h

@@ -68,11 +68,11 @@ void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags);
 void __lockfunc _write_unlock_irq(rwlock_t *lock);
 void __lockfunc _write_unlock_bh(rwlock_t *lock);
 int __lockfunc _spin_trylock_bh(spinlock_t *lock);
-
-extern unsigned long __lock_text_start;
-extern unsigned long __lock_text_end;
+int in_lock_functions(unsigned long addr);
 #else
 
+#define in_lock_functions(ADDR) 0
+
 #if !defined(CONFIG_PREEMPT) && !defined(CONFIG_DEBUG_SPINLOCK)
 # define atomic_dec_and_lock(atomic,lock) atomic_dec_and_test(atomic)
 # define ATOMIC_DEC_AND_LOCK
@@ -405,11 +405,6 @@ do { \
 
 /* Where's read_trylock? */
 
-#if defined(CONFIG_SMP) && defined(CONFIG_PREEMPT)
-void __preempt_spin_lock(spinlock_t *lock);
-void __preempt_write_lock(rwlock_t *lock);
-#endif
-
 #define spin_lock(lock)		_spin_lock(lock)
 #define write_lock(lock)	_write_lock(lock)
 #define read_lock(lock)		_read_lock(lock)

kernel/sched.c

@@ -4654,8 +4654,9 @@ int in_sched_functions(unsigned long addr)
 {
 	/* Linker adds these: start and end of __sched functions */
 	extern char __sched_text_start[], __sched_text_end[];
-	return addr >= (unsigned long)__sched_text_start
-		&& addr < (unsigned long)__sched_text_end;
+	return in_lock_functions(addr) ||
+		(addr >= (unsigned long)__sched_text_start
+		&& addr < (unsigned long)__sched_text_end);
 }
 
 void __init sched_init(void)
@@ -4747,49 +4748,3 @@ void __might_sleep(char *file, int line)
 }
 EXPORT_SYMBOL(__might_sleep);
 #endif
-
-
-#if defined(CONFIG_SMP) && defined(CONFIG_PREEMPT)
-/*
- * This could be a long-held lock.  If another CPU holds it for a long time,
- * and that CPU is not asked to reschedule then *this* CPU will spin on the
- * lock for a long time, even if *this* CPU is asked to reschedule.
- *
- * So what we do here, in the slow (contended) path is to spin on the lock by
- * hand while permitting preemption.
- *
- * Called inside preempt_disable().
- */
-void __sched __preempt_spin_lock(spinlock_t *lock)
-{
-	if (preempt_count() > 1) {
-		_raw_spin_lock(lock);
-		return;
-	}
-	do {
-		preempt_enable();
-		while (spin_is_locked(lock))
-			cpu_relax();
-		preempt_disable();
-	} while (!_raw_spin_trylock(lock));
-}
-
-EXPORT_SYMBOL(__preempt_spin_lock);
-
-void __sched __preempt_write_lock(rwlock_t *lock)
-{
-	if (preempt_count() > 1) {
-		_raw_write_lock(lock);
-		return;
-	}
-
-	do {
-		preempt_enable();
-		while (rwlock_is_locked(lock))
-			cpu_relax();
-		preempt_disable();
-	} while (!_raw_write_trylock(lock));
-}
-
-EXPORT_SYMBOL(__preempt_write_lock);
-#endif /* defined(CONFIG_SMP) && defined(CONFIG_PREEMPT) */

kernel/spinlock.c

@@ -33,7 +33,32 @@ int __lockfunc _write_trylock(rwlock_t *lock)
 }
 EXPORT_SYMBOL(_write_trylock);
 
-#if defined(CONFIG_SMP) && defined(CONFIG_PREEMPT)
+#ifdef CONFIG_PREEMPT
+/*
+ * This could be a long-held lock.  If another CPU holds it for a long time,
+ * and that CPU is not asked to reschedule then *this* CPU will spin on the
+ * lock for a long time, even if *this* CPU is asked to reschedule.
+ *
+ * So what we do here, in the slow (contended) path is to spin on the lock by
+ * hand while permitting preemption.
+ *
+ * Called inside preempt_disable().
+ */
+static inline void __preempt_spin_lock(spinlock_t *lock)
+{
+	if (preempt_count() > 1) {
+		_raw_spin_lock(lock);
+		return;
+	}
+
+	do {
+		preempt_enable();
+		while (spin_is_locked(lock))
+			cpu_relax();
+		preempt_disable();
+	} while (!_raw_spin_trylock(lock));
+}
+
 void __lockfunc _spin_lock(spinlock_t *lock)
 {
 	preempt_disable();
@@ -41,6 +66,21 @@ void __lockfunc _spin_lock(spinlock_t *lock)
 		__preempt_spin_lock(lock);
 }
 
+static inline void __preempt_write_lock(rwlock_t *lock)
+{
+	if (preempt_count() > 1) {
+		_raw_write_lock(lock);
+		return;
+	}
+
+	do {
+		preempt_enable();
+		while (rwlock_is_locked(lock))
+			cpu_relax();
+		preempt_disable();
+	} while (!_raw_write_trylock(lock));
+}
+
 void __lockfunc _write_lock(rwlock_t *lock)
 {
 	preempt_disable();
@@ -256,3 +296,13 @@ int __lockfunc _spin_trylock_bh(spinlock_t *lock)
 	return 0;
 }
 EXPORT_SYMBOL(_spin_trylock_bh);
+
+int in_lock_functions(unsigned long addr)
+{
+	/* Linker adds these: start and end of __lockfunc functions */
+	extern char __lock_text_start[], __lock_text_end[];
+
+	return addr >= (unsigned long)__lock_text_start
+	&& addr < (unsigned long)__lock_text_end;
+}
+EXPORT_SYMBOL(in_lock_functions);