[PATCH] ppc64: rework secondary SMT thread setup at boot

Our (ab)use of cpu_possible_map in setup_system to start secondary SMT
threads bothers me.  Mark such threads in cpu_possible_map during early
boot; let RTAS tell us which present cpus are still offline later so we can
start them.

I'm not totally sure about this one, it might be better to set up
cpu_sibling_map in prom_hold_cpus and use that in setup_system.
Signed-off-by: Nathan Lynch <nathanl@austin.ibm.com>
Signed-off-by: Paul Mackerras <paulus@samba.org>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

arch/ppc64/kernel/prom.c

@@ -1076,6 +1076,8 @@ static void __init prom_hold_cpus(unsigned long mem)
 				cpu_set(cpuid, RELOC(cpu_available_map));
 				cpu_set(cpuid, RELOC(cpu_present_at_boot));
 				cpu_set(cpuid, RELOC(cpu_present_map));
+				cpu_set(cpuid, RELOC(cpu_possible_map));
+				_systemcfg->processorCount++;
 				prom_printf("available\n");
 			} else {
 				prom_printf("not available\n");

arch/ppc64/kernel/setup.c

@@ -232,16 +232,17 @@ void setup_system(unsigned long r3, unsigned long r4, unsigned long r5,
 		chrp_init(r3, r4, r5, r6, r7);
 
 #ifdef CONFIG_SMP
-		/* Start secondary threads on SMT systems */
-		for (i = 0; i < NR_CPUS; i++) {
-			if (cpu_available(i) && !cpu_possible(i)) {
+		/* Start secondary threads on SMT systems; primary threads
+		 * are already in the running state.
+		 */
+		for_each_present_cpu(i) {
+			if (query_cpu_stopped
+			    (get_hard_smp_processor_id(i)) == 0) {
 				printk("%16.16x : starting thread\n", i);
 				rtas_call(rtas_token("start-cpu"), 3, 1, &ret,
 					  get_hard_smp_processor_id(i), 
 					  (u32)*((unsigned long *)pseries_secondary_smp_init),
 					  i);
-				cpu_set(i, cpu_possible_map);
-				systemcfg->processorCount++;
 			}
 		}
 #endif /* CONFIG_SMP */

arch/ppc64/kernel/smp.c

@@ -225,7 +225,6 @@ static void __devinit smp_openpic_setup_cpu(int cpu)
 	do_openpic_setup_cpu();
 }
 
-#ifdef CONFIG_HOTPLUG_CPU
 /* Get state of physical CPU.
  * Return codes:
  *	0	- The processor is in the RTAS stopped state
@@ -234,13 +233,14 @@ static void __devinit smp_openpic_setup_cpu(int cpu)
  *	-1	- Hardware Error
  *	-2	- Hardware Busy, Try again later.
  */
-static int query_cpu_stopped(unsigned int pcpu)
+int query_cpu_stopped(unsigned int pcpu)
 {
 	int cpu_status;
 	int status, qcss_tok;
 
 	qcss_tok = rtas_token("query-cpu-stopped-state");
-	BUG_ON(qcss_tok == RTAS_UNKNOWN_SERVICE);
+	if (qcss_tok == RTAS_UNKNOWN_SERVICE)
+		return -1;
 	status = rtas_call(qcss_tok, 1, 2, &cpu_status, pcpu);
 	if (status != 0) {
 		printk(KERN_ERR
@@ -251,6 +251,8 @@ static int query_cpu_stopped(unsigned int pcpu)
 	return cpu_status;
 }
 
+#ifdef CONFIG_HOTPLUG_CPU
+
 int __cpu_disable(void)
 {
 	/* FIXME: go put this in a header somewhere */

include/asm-ppc64/smp.h

@@ -73,6 +73,7 @@ void smp_init_pSeries(void);
 extern int __cpu_disable(void);
 extern void __cpu_die(unsigned int cpu);
 extern void cpu_die(void) __attribute__((noreturn));
+extern int query_cpu_stopped(unsigned int pcpu);
 #endif /* !(CONFIG_SMP) */
 
 #define get_hard_smp_processor_id(CPU) (paca[(CPU)].hw_cpu_id)