smp.c 25.9 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
/*
Martin Schwidefsky's avatar
Martin Schwidefsky committed
2
 *  SMP related functions
Linus Torvalds's avatar
Linus Torvalds committed
3
 *
Martin Schwidefsky's avatar
Martin Schwidefsky committed
4 5 6 7
 *    Copyright IBM Corp. 1999,2012
 *    Author(s): Denis Joseph Barrow,
 *		 Martin Schwidefsky <schwidefsky@de.ibm.com>,
 *		 Heiko Carstens <heiko.carstens@de.ibm.com>,
Linus Torvalds's avatar
Linus Torvalds committed
8
 *
9
 *  based on other smp stuff by
Linus Torvalds's avatar
Linus Torvalds committed
10 11 12
 *    (c) 1995 Alan Cox, CymruNET Ltd  <alan@cymru.net>
 *    (c) 1998 Ingo Molnar
 *
Martin Schwidefsky's avatar
Martin Schwidefsky committed
13 14 15
 * The code outside of smp.c uses logical cpu numbers, only smp.c does
 * the translation of logical to physical cpu ids. All new code that
 * operates on physical cpu numbers needs to go into smp.c.
Linus Torvalds's avatar
Linus Torvalds committed
16 17
 */

18 19 20
#define KMSG_COMPONENT "cpu"
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt

21
#include <linux/workqueue.h>
Linus Torvalds's avatar
Linus Torvalds committed
22 23 24
#include <linux/module.h>
#include <linux/init.h>
#include <linux/mm.h>
Alexey Dobriyan's avatar
Alexey Dobriyan committed
25
#include <linux/err.h>
Linus Torvalds's avatar
Linus Torvalds committed
26 27 28 29
#include <linux/spinlock.h>
#include <linux/kernel_stat.h>
#include <linux/delay.h>
#include <linux/interrupt.h>
30
#include <linux/irqflags.h>
Linus Torvalds's avatar
Linus Torvalds committed
31
#include <linux/cpu.h>
32
#include <linux/slab.h>
Michael Holzheu's avatar
Michael Holzheu committed
33
#include <linux/crash_dump.h>
34
#include <asm/asm-offsets.h>
35 36
#include <asm/switch_to.h>
#include <asm/facility.h>
37
#include <asm/ipl.h>
38
#include <asm/setup.h>
Linus Torvalds's avatar
Linus Torvalds committed
39 40
#include <asm/irq.h>
#include <asm/tlbflush.h>
41
#include <asm/timer.h>
Michael Holzheu's avatar
Michael Holzheu committed
42
#include <asm/lowcore.h>
43
#include <asm/sclp.h>
44
#include <asm/vdso.h>
45
#include <asm/debug.h>
46
#include <asm/os_info.h>
47
#include <asm/sigp.h>
48
#include "entry.h"
Linus Torvalds's avatar
Linus Torvalds committed
49

Martin Schwidefsky's avatar
Martin Schwidefsky committed
50 51 52 53 54 55
enum {
	ec_schedule = 0,
	ec_call_function,
	ec_call_function_single,
	ec_stop_cpu,
};
56

Martin Schwidefsky's avatar
Martin Schwidefsky committed
57
enum {
58 59 60 61
	CPU_STATE_STANDBY,
	CPU_STATE_CONFIGURED,
};

Martin Schwidefsky's avatar
Martin Schwidefsky committed
62 63 64 65 66 67 68 69 70 71 72 73 74 75 76
struct pcpu {
	struct cpu cpu;
	struct _lowcore *lowcore;	/* lowcore page(s) for the cpu */
	unsigned long async_stack;	/* async stack for the cpu */
	unsigned long panic_stack;	/* panic stack for the cpu */
	unsigned long ec_mask;		/* bit mask for ec_xxx functions */
	int state;			/* physical cpu state */
	u32 status;			/* last status received via sigp */
	u16 address;			/* physical cpu address */
};

static u8 boot_cpu_type;
static u16 boot_cpu_address;
static struct pcpu pcpu_devices[NR_CPUS];

77
DEFINE_MUTEX(smp_cpu_state_mutex);
78

Martin Schwidefsky's avatar
Martin Schwidefsky committed
79 80 81 82 83 84 85
/*
 * Signal processor helper functions.
 */
static inline int __pcpu_sigp(u16 addr, u8 order, u32 parm, u32 *status)
{
	register unsigned int reg1 asm ("1") = parm;
	int cc;
86

Martin Schwidefsky's avatar
Martin Schwidefsky committed
87 88 89 90 91 92 93 94 95
	asm volatile(
		"	sigp	%1,%2,0(%3)\n"
		"	ipm	%0\n"
		"	srl	%0,28\n"
		: "=d" (cc), "+d" (reg1) : "d" (addr), "a" (order) : "cc");
	if (status && cc == 1)
		*status = reg1;
	return cc;
}
Linus Torvalds's avatar
Linus Torvalds committed
96

Martin Schwidefsky's avatar
Martin Schwidefsky committed
97
static inline int __pcpu_sigp_relax(u16 addr, u8 order, u32 parm, u32 *status)
98
{
Martin Schwidefsky's avatar
Martin Schwidefsky committed
99
	int cc;
100

Martin Schwidefsky's avatar
Martin Schwidefsky committed
101 102
	while (1) {
		cc = __pcpu_sigp(addr, order, parm, status);
103
		if (cc != SIGP_CC_BUSY)
Martin Schwidefsky's avatar
Martin Schwidefsky committed
104 105
			return cc;
		cpu_relax();
106 107 108
	}
}

Martin Schwidefsky's avatar
Martin Schwidefsky committed
109
static int pcpu_sigp_retry(struct pcpu *pcpu, u8 order, u32 parm)
110
{
Martin Schwidefsky's avatar
Martin Schwidefsky committed
111 112 113 114
	int cc, retry;

	for (retry = 0; ; retry++) {
		cc = __pcpu_sigp(pcpu->address, order, parm, &pcpu->status);
115
		if (cc != SIGP_CC_BUSY)
Martin Schwidefsky's avatar
Martin Schwidefsky committed
116 117 118 119 120 121 122 123 124
			break;
		if (retry >= 3)
			udelay(10);
	}
	return cc;
}

static inline int pcpu_stopped(struct pcpu *pcpu)
{
125 126
	if (__pcpu_sigp(pcpu->address, SIGP_SENSE,
			0, &pcpu->status) != SIGP_CC_STATUS_STORED)
Martin Schwidefsky's avatar
Martin Schwidefsky committed
127
		return 0;
128
	return !!(pcpu->status & (SIGP_STATUS_CHECK_STOP|SIGP_STATUS_STOPPED));
Martin Schwidefsky's avatar
Martin Schwidefsky committed
129 130 131
}

static inline int pcpu_running(struct pcpu *pcpu)
132
{
133 134
	if (__pcpu_sigp(pcpu->address, SIGP_SENSE_RUNNING,
			0, &pcpu->status) != SIGP_CC_STATUS_STORED)
Martin Schwidefsky's avatar
Martin Schwidefsky committed
135
		return 1;
136 137
	/* Status stored condition code is equivalent to cpu not running. */
	return 0;
138 139
}

140
/*
Martin Schwidefsky's avatar
Martin Schwidefsky committed
141
 * Find struct pcpu by cpu address.
142
 */
Martin Schwidefsky's avatar
Martin Schwidefsky committed
143
static struct pcpu *pcpu_find_address(const struct cpumask *mask, int address)
144 145 146
{
	int cpu;

Martin Schwidefsky's avatar
Martin Schwidefsky committed
147 148 149 150 151 152 153 154 155 156 157 158
	for_each_cpu(cpu, mask)
		if (pcpu_devices[cpu].address == address)
			return pcpu_devices + cpu;
	return NULL;
}

static void pcpu_ec_call(struct pcpu *pcpu, int ec_bit)
{
	int order;

	set_bit(ec_bit, &pcpu->ec_mask);
	order = pcpu_running(pcpu) ?
159
		SIGP_EXTERNAL_CALL : SIGP_EMERGENCY_SIGNAL;
Martin Schwidefsky's avatar
Martin Schwidefsky committed
160 161 162 163 164 165 166 167 168 169 170 171 172 173
	pcpu_sigp_retry(pcpu, order, 0);
}

static int __cpuinit pcpu_alloc_lowcore(struct pcpu *pcpu, int cpu)
{
	struct _lowcore *lc;

	if (pcpu != &pcpu_devices[0]) {
		pcpu->lowcore =	(struct _lowcore *)
			__get_free_pages(GFP_KERNEL | GFP_DMA, LC_ORDER);
		pcpu->async_stack = __get_free_pages(GFP_KERNEL, ASYNC_ORDER);
		pcpu->panic_stack = __get_free_page(GFP_KERNEL);
		if (!pcpu->lowcore || !pcpu->panic_stack || !pcpu->async_stack)
			goto out;
174
	}
Martin Schwidefsky's avatar
Martin Schwidefsky committed
175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191
	lc = pcpu->lowcore;
	memcpy(lc, &S390_lowcore, 512);
	memset((char *) lc + 512, 0, sizeof(*lc) - 512);
	lc->async_stack = pcpu->async_stack + ASYNC_SIZE;
	lc->panic_stack = pcpu->panic_stack + PAGE_SIZE;
	lc->cpu_nr = cpu;
#ifndef CONFIG_64BIT
	if (MACHINE_HAS_IEEE) {
		lc->extended_save_area_addr = get_zeroed_page(GFP_KERNEL);
		if (!lc->extended_save_area_addr)
			goto out;
	}
#else
	if (vdso_alloc_per_cpu(lc))
		goto out;
#endif
	lowcore_ptr[cpu] = lc;
192
	pcpu_sigp_retry(pcpu, SIGP_SET_PREFIX, (u32)(unsigned long) lc);
Martin Schwidefsky's avatar
Martin Schwidefsky committed
193 194 195 196 197 198 199 200
	return 0;
out:
	if (pcpu != &pcpu_devices[0]) {
		free_page(pcpu->panic_stack);
		free_pages(pcpu->async_stack, ASYNC_ORDER);
		free_pages((unsigned long) pcpu->lowcore, LC_ORDER);
	}
	return -ENOMEM;
201 202
}

203 204
#ifdef CONFIG_HOTPLUG_CPU

Martin Schwidefsky's avatar
Martin Schwidefsky committed
205
static void pcpu_free_lowcore(struct pcpu *pcpu)
206
{
207
	pcpu_sigp_retry(pcpu, SIGP_SET_PREFIX, 0);
Martin Schwidefsky's avatar
Martin Schwidefsky committed
208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225
	lowcore_ptr[pcpu - pcpu_devices] = NULL;
#ifndef CONFIG_64BIT
	if (MACHINE_HAS_IEEE) {
		struct _lowcore *lc = pcpu->lowcore;

		free_page((unsigned long) lc->extended_save_area_addr);
		lc->extended_save_area_addr = 0;
	}
#else
	vdso_free_per_cpu(pcpu->lowcore);
#endif
	if (pcpu != &pcpu_devices[0]) {
		free_page(pcpu->panic_stack);
		free_pages(pcpu->async_stack, ASYNC_ORDER);
		free_pages((unsigned long) pcpu->lowcore, LC_ORDER);
	}
}

226 227
#endif /* CONFIG_HOTPLUG_CPU */

Martin Schwidefsky's avatar
Martin Schwidefsky committed
228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265
static void pcpu_prepare_secondary(struct pcpu *pcpu, int cpu)
{
	struct _lowcore *lc = pcpu->lowcore;

	atomic_inc(&init_mm.context.attach_count);
	lc->cpu_nr = cpu;
	lc->percpu_offset = __per_cpu_offset[cpu];
	lc->kernel_asce = S390_lowcore.kernel_asce;
	lc->machine_flags = S390_lowcore.machine_flags;
	lc->ftrace_func = S390_lowcore.ftrace_func;
	lc->user_timer = lc->system_timer = lc->steal_timer = 0;
	__ctl_store(lc->cregs_save_area, 0, 15);
	save_access_regs((unsigned int *) lc->access_regs_save_area);
	memcpy(lc->stfle_fac_list, S390_lowcore.stfle_fac_list,
	       MAX_FACILITY_BIT/8);
}

static void pcpu_attach_task(struct pcpu *pcpu, struct task_struct *tsk)
{
	struct _lowcore *lc = pcpu->lowcore;
	struct thread_info *ti = task_thread_info(tsk);

	lc->kernel_stack = (unsigned long) task_stack_page(tsk) + THREAD_SIZE;
	lc->thread_info = (unsigned long) task_thread_info(tsk);
	lc->current_task = (unsigned long) tsk;
	lc->user_timer = ti->user_timer;
	lc->system_timer = ti->system_timer;
	lc->steal_timer = 0;
}

static void pcpu_start_fn(struct pcpu *pcpu, void (*func)(void *), void *data)
{
	struct _lowcore *lc = pcpu->lowcore;

	lc->restart_stack = lc->kernel_stack;
	lc->restart_fn = (unsigned long) func;
	lc->restart_data = (unsigned long) data;
	lc->restart_source = -1UL;
266
	pcpu_sigp_retry(pcpu, SIGP_RESTART, 0);
Martin Schwidefsky's avatar
Martin Schwidefsky committed
267 268 269 270 271 272 273 274
}

/*
 * Call function via PSW restart on pcpu and stop the current cpu.
 */
static void pcpu_delegate(struct pcpu *pcpu, void (*func)(void *),
			  void *data, unsigned long stack)
{
275 276 277 278 279 280 281
	struct _lowcore *lc = lowcore_ptr[pcpu - pcpu_devices];
	struct {
		unsigned long	stack;
		void		*func;
		void		*data;
		unsigned long	source;
	} restart = { stack, func, data, stap() };
Martin Schwidefsky's avatar
Martin Schwidefsky committed
282 283

	__load_psw_mask(psw_kernel_bits);
284
	if (pcpu->address == restart.source)
Martin Schwidefsky's avatar
Martin Schwidefsky committed
285 286
		func(data);	/* should not return */
	/* Stop target cpu (if func returns this stops the current cpu). */
287
	pcpu_sigp_retry(pcpu, SIGP_STOP, 0);
Martin Schwidefsky's avatar
Martin Schwidefsky committed
288
	/* Restart func on the target cpu and stop the current cpu. */
289
	memcpy_absolute(&lc->restart_stack, &restart, sizeof(restart));
Martin Schwidefsky's avatar
Martin Schwidefsky committed
290
	asm volatile(
291
		"0:	sigp	0,%0,%2	# sigp restart to target cpu\n"
Martin Schwidefsky's avatar
Martin Schwidefsky committed
292
		"	brc	2,0b	# busy, try again\n"
293
		"1:	sigp	0,%1,%3	# sigp stop to current cpu\n"
Martin Schwidefsky's avatar
Martin Schwidefsky committed
294
		"	brc	2,1b	# busy, try again\n"
295 296 297
		: : "d" (pcpu->address), "d" (restart.source),
		    "K" (SIGP_RESTART), "K" (SIGP_STOP)
		: "0", "1", "cc");
Martin Schwidefsky's avatar
Martin Schwidefsky committed
298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320
	for (;;) ;
}

/*
 * Call function on an online CPU.
 */
void smp_call_online_cpu(void (*func)(void *), void *data)
{
	struct pcpu *pcpu;

	/* Use the current cpu if it is online. */
	pcpu = pcpu_find_address(cpu_online_mask, stap());
	if (!pcpu)
		/* Use the first online cpu. */
		pcpu = pcpu_devices + cpumask_first(cpu_online_mask);
	pcpu_delegate(pcpu, func, data, (unsigned long) restart_stack);
}

/*
 * Call function on the ipl CPU.
 */
void smp_call_ipl_cpu(void (*func)(void *), void *data)
{
321 322
	pcpu_delegate(&pcpu_devices[0], func, data,
		      pcpu_devices->panic_stack + PAGE_SIZE);
Martin Schwidefsky's avatar
Martin Schwidefsky committed
323 324 325 326 327 328 329 330 331 332
}

int smp_find_processor_id(u16 address)
{
	int cpu;

	for_each_present_cpu(cpu)
		if (pcpu_devices[cpu].address == address)
			return cpu;
	return -1;
333 334
}

Martin Schwidefsky's avatar
Martin Schwidefsky committed
335
int smp_vcpu_scheduled(int cpu)
336
{
Martin Schwidefsky's avatar
Martin Schwidefsky committed
337 338 339 340 341 342 343
	return pcpu_running(pcpu_devices + cpu);
}

void smp_yield(void)
{
	if (MACHINE_HAS_DIAG44)
		asm volatile("diag 0,0,0x44");
344 345
}

Martin Schwidefsky's avatar
Martin Schwidefsky committed
346
void smp_yield_cpu(int cpu)
347
{
Martin Schwidefsky's avatar
Martin Schwidefsky committed
348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367
	if (MACHINE_HAS_DIAG9C)
		asm volatile("diag %0,0,0x9c"
			     : : "d" (pcpu_devices[cpu].address));
	else if (MACHINE_HAS_DIAG44)
		asm volatile("diag 0,0,0x44");
}

/*
 * Send cpus emergency shutdown signal. This gives the cpus the
 * opportunity to complete outstanding interrupts.
 */
void smp_emergency_stop(cpumask_t *cpumask)
{
	u64 end;
	int cpu;

	end = get_clock() + (1000000UL << 12);
	for_each_cpu(cpu, cpumask) {
		struct pcpu *pcpu = pcpu_devices + cpu;
		set_bit(ec_stop_cpu, &pcpu->ec_mask);
368 369
		while (__pcpu_sigp(pcpu->address, SIGP_EMERGENCY_SIGNAL,
				   0, NULL) == SIGP_CC_BUSY &&
Martin Schwidefsky's avatar
Martin Schwidefsky committed
370 371 372 373 374 375 376 377 378
		       get_clock() < end)
			cpu_relax();
	}
	while (get_clock() < end) {
		for_each_cpu(cpu, cpumask)
			if (pcpu_stopped(pcpu_devices + cpu))
				cpumask_clear_cpu(cpu, cpumask);
		if (cpumask_empty(cpumask))
			break;
379
		cpu_relax();
Martin Schwidefsky's avatar
Martin Schwidefsky committed
380
	}
381 382
}

Martin Schwidefsky's avatar
Martin Schwidefsky committed
383 384 385
/*
 * Stop all cpus but the current one.
 */
386
void smp_send_stop(void)
Linus Torvalds's avatar
Linus Torvalds committed
387
{
388 389
	cpumask_t cpumask;
	int cpu;
Linus Torvalds's avatar
Linus Torvalds committed
390

391
	/* Disable all interrupts/machine checks */
392
	__load_psw_mask(psw_kernel_bits | PSW_MASK_DAT);
393
	trace_hardirqs_off();
Linus Torvalds's avatar
Linus Torvalds committed
394

395
	debug_set_critical();
396 397 398
	cpumask_copy(&cpumask, cpu_online_mask);
	cpumask_clear_cpu(smp_processor_id(), &cpumask);

Martin Schwidefsky's avatar
Martin Schwidefsky committed
399 400
	if (oops_in_progress)
		smp_emergency_stop(&cpumask);
Linus Torvalds's avatar
Linus Torvalds committed
401

402 403
	/* stop all processors */
	for_each_cpu(cpu, &cpumask) {
Martin Schwidefsky's avatar
Martin Schwidefsky committed
404
		struct pcpu *pcpu = pcpu_devices + cpu;
405
		pcpu_sigp_retry(pcpu, SIGP_STOP, 0);
Martin Schwidefsky's avatar
Martin Schwidefsky committed
406
		while (!pcpu_stopped(pcpu))
Heiko Carstens's avatar
Heiko Carstens committed
407 408 409 410
			cpu_relax();
	}
}

Martin Schwidefsky's avatar
Martin Schwidefsky committed
411 412 413 414 415
/*
 * Stop the current cpu.
 */
void smp_stop_cpu(void)
{
416
	pcpu_sigp_retry(pcpu_devices + smp_processor_id(), SIGP_STOP, 0);
Martin Schwidefsky's avatar
Martin Schwidefsky committed
417 418 419
	for (;;) ;
}

Linus Torvalds's avatar
Linus Torvalds committed
420 421 422 423
/*
 * This is the main routine where commands issued by other
 * cpus are handled.
 */
424
static void do_ext_call_interrupt(struct ext_code ext_code,
425
				  unsigned int param32, unsigned long param64)
Linus Torvalds's avatar
Linus Torvalds committed
426
{
427
	unsigned long bits;
Martin Schwidefsky's avatar
Martin Schwidefsky committed
428
	int cpu;
Linus Torvalds's avatar
Linus Torvalds committed
429

Martin Schwidefsky's avatar
Martin Schwidefsky committed
430
	cpu = smp_processor_id();
431
	if (ext_code.code == 0x1202)
Martin Schwidefsky's avatar
Martin Schwidefsky committed
432
		kstat_cpu(cpu).irqs[EXTINT_EXC]++;
433
	else
Martin Schwidefsky's avatar
Martin Schwidefsky committed
434
		kstat_cpu(cpu).irqs[EXTINT_EMS]++;
435 436 437
	/*
	 * handle bit signal external calls
	 */
Martin Schwidefsky's avatar
Martin Schwidefsky committed
438
	bits = xchg(&pcpu_devices[cpu].ec_mask, 0);
Linus Torvalds's avatar
Linus Torvalds committed
439

440 441 442
	if (test_bit(ec_stop_cpu, &bits))
		smp_stop_cpu();

443 444 445
	if (test_bit(ec_schedule, &bits))
		scheduler_ipi();

446
	if (test_bit(ec_call_function, &bits))
447 448 449 450
		generic_smp_call_function_interrupt();

	if (test_bit(ec_call_function_single, &bits))
		generic_smp_call_function_single_interrupt();
451

Linus Torvalds's avatar
Linus Torvalds committed
452 453
}

454
void arch_send_call_function_ipi_mask(const struct cpumask *mask)
455 456 457
{
	int cpu;

458
	for_each_cpu(cpu, mask)
Martin Schwidefsky's avatar
Martin Schwidefsky committed
459
		pcpu_ec_call(pcpu_devices + cpu, ec_call_function);
460 461 462 463
}

void arch_send_call_function_single_ipi(int cpu)
{
Martin Schwidefsky's avatar
Martin Schwidefsky committed
464
	pcpu_ec_call(pcpu_devices + cpu, ec_call_function_single);
465 466
}

467
#ifndef CONFIG_64BIT
Linus Torvalds's avatar
Linus Torvalds committed
468 469 470
/*
 * this function sends a 'purge tlb' signal to another CPU.
 */
471
static void smp_ptlb_callback(void *info)
Linus Torvalds's avatar
Linus Torvalds committed
472
{
Martin Schwidefsky's avatar
Martin Schwidefsky committed
473
	__tlb_flush_local();
Linus Torvalds's avatar
Linus Torvalds committed
474 475 476 477
}

void smp_ptlb_all(void)
{
478
	on_each_cpu(smp_ptlb_callback, NULL, 1);
Linus Torvalds's avatar
Linus Torvalds committed
479 480
}
EXPORT_SYMBOL(smp_ptlb_all);
481
#endif /* ! CONFIG_64BIT */
Linus Torvalds's avatar
Linus Torvalds committed
482 483 484 485 486 487 488 489

/*
 * this function sends a 'reschedule' IPI to another CPU.
 * it goes straight through and wastes no time serializing
 * anything. Worst case is that we lose a reschedule ...
 */
void smp_send_reschedule(int cpu)
{
Martin Schwidefsky's avatar
Martin Schwidefsky committed
490
	pcpu_ec_call(pcpu_devices + cpu, ec_schedule);
Linus Torvalds's avatar
Linus Torvalds committed
491 492 493 494 495
}

/*
 * parameter area for the set/clear control bit callbacks
 */
496
struct ec_creg_mask_parms {
Martin Schwidefsky's avatar
Martin Schwidefsky committed
497 498 499
	unsigned long orval;
	unsigned long andval;
	int cr;
500
};
Linus Torvalds's avatar
Linus Torvalds committed
501 502 503 504

/*
 * callback for setting/clearing control bits
 */
505 506
static void smp_ctl_bit_callback(void *info)
{
507
	struct ec_creg_mask_parms *pp = info;
Linus Torvalds's avatar
Linus Torvalds committed
508
	unsigned long cregs[16];
509

510
	__ctl_store(cregs, 0, 15);
Martin Schwidefsky's avatar
Martin Schwidefsky committed
511
	cregs[pp->cr] = (cregs[pp->cr] & pp->andval) | pp->orval;
512
	__ctl_load(cregs, 0, 15);
Linus Torvalds's avatar
Linus Torvalds committed
513 514 515 516 517
}

/*
 * Set a bit in a control register of all cpus
 */
518 519
void smp_ctl_set_bit(int cr, int bit)
{
Martin Schwidefsky's avatar
Martin Schwidefsky committed
520
	struct ec_creg_mask_parms parms = { 1UL << bit, -1UL, cr };
Linus Torvalds's avatar
Linus Torvalds committed
521

522
	on_each_cpu(smp_ctl_bit_callback, &parms, 1);
Linus Torvalds's avatar
Linus Torvalds committed
523
}
524
EXPORT_SYMBOL(smp_ctl_set_bit);
Linus Torvalds's avatar
Linus Torvalds committed
525 526 527 528

/*
 * Clear a bit in a control register of all cpus
 */
529 530
void smp_ctl_clear_bit(int cr, int bit)
{
Martin Schwidefsky's avatar
Martin Schwidefsky committed
531
	struct ec_creg_mask_parms parms = { 0, ~(1UL << bit), cr };
Linus Torvalds's avatar
Linus Torvalds committed
532

533
	on_each_cpu(smp_ctl_bit_callback, &parms, 1);
Linus Torvalds's avatar
Linus Torvalds committed
534
}
535
EXPORT_SYMBOL(smp_ctl_clear_bit);
Linus Torvalds's avatar
Linus Torvalds committed
536

Michael Holzheu's avatar
Michael Holzheu committed
537
#if defined(CONFIG_ZFCPDUMP) || defined(CONFIG_CRASH_DUMP)
Michael Holzheu's avatar
Michael Holzheu committed
538

Martin Schwidefsky's avatar
Martin Schwidefsky committed
539 540 541 542
struct save_area *zfcpdump_save_areas[NR_CPUS + 1];
EXPORT_SYMBOL_GPL(zfcpdump_save_areas);

static void __init smp_get_save_area(int cpu, u16 address)
Michael Holzheu's avatar
Michael Holzheu committed
543
{
Martin Schwidefsky's avatar
Martin Schwidefsky committed
544 545 546
	void *lc = pcpu_devices[0].lowcore;
	struct save_area *save_area;

Michael Holzheu's avatar
Michael Holzheu committed
547
	if (is_kdump_kernel())
Michael Holzheu's avatar
Michael Holzheu committed
548
		return;
Martin Schwidefsky's avatar
Martin Schwidefsky committed
549 550 551
	if (!OLDMEM_BASE && (address == boot_cpu_address ||
			     ipl_info.type != IPL_TYPE_FCP_DUMP))
		return;
552
	if (cpu >= NR_CPUS) {
Martin Schwidefsky's avatar
Martin Schwidefsky committed
553 554
		pr_warning("CPU %i exceeds the maximum %i and is excluded "
			   "from the dump\n", cpu, NR_CPUS - 1);
555
		return;
Michael Holzheu's avatar
Michael Holzheu committed
556
	}
Martin Schwidefsky's avatar
Martin Schwidefsky committed
557 558 559 560 561 562 563 564 565 566 567 568 569
	save_area = kmalloc(sizeof(struct save_area), GFP_KERNEL);
	if (!save_area)
		panic("could not allocate memory for save area\n");
	zfcpdump_save_areas[cpu] = save_area;
#ifdef CONFIG_CRASH_DUMP
	if (address == boot_cpu_address) {
		/* Copy the registers of the boot cpu. */
		copy_oldmem_page(1, (void *) save_area, sizeof(*save_area),
				 SAVE_AREA_BASE - PAGE_SIZE, 0);
		return;
	}
#endif
	/* Get the registers of a non-boot cpu. */
570
	__pcpu_sigp_relax(address, SIGP_STOP_AND_STORE_STATUS, 0, NULL);
Martin Schwidefsky's avatar
Martin Schwidefsky committed
571
	memcpy_real(save_area, lc + SAVE_AREA_BASE, sizeof(*save_area));
Michael Holzheu's avatar
Michael Holzheu committed
572 573
}

Martin Schwidefsky's avatar
Martin Schwidefsky committed
574
int smp_store_status(int cpu)
575
{
Martin Schwidefsky's avatar
Martin Schwidefsky committed
576
	struct pcpu *pcpu;
577

Martin Schwidefsky's avatar
Martin Schwidefsky committed
578
	pcpu = pcpu_devices + cpu;
579 580
	if (__pcpu_sigp_relax(pcpu->address, SIGP_STOP_AND_STORE_STATUS,
			      0, NULL) != SIGP_CC_ORDER_CODE_ACCEPTED)
Martin Schwidefsky's avatar
Martin Schwidefsky committed
581
		return -EIO;
582 583 584
	return 0;
}

Martin Schwidefsky's avatar
Martin Schwidefsky committed
585
#else /* CONFIG_ZFCPDUMP || CONFIG_CRASH_DUMP */
586

Martin Schwidefsky's avatar
Martin Schwidefsky committed
587
static inline void smp_get_save_area(int cpu, u16 address) { }
588

Martin Schwidefsky's avatar
Martin Schwidefsky committed
589
#endif /* CONFIG_ZFCPDUMP || CONFIG_CRASH_DUMP */
590

Martin Schwidefsky's avatar
Martin Schwidefsky committed
591
static struct sclp_cpu_info *smp_get_cpu_info(void)
592
{
Martin Schwidefsky's avatar
Martin Schwidefsky committed
593
	static int use_sigp_detection;
594
	struct sclp_cpu_info *info;
Martin Schwidefsky's avatar
Martin Schwidefsky committed
595 596 597 598 599 600
	int address;

	info = kzalloc(sizeof(*info), GFP_KERNEL);
	if (info && (use_sigp_detection || sclp_get_cpu_info(info))) {
		use_sigp_detection = 1;
		for (address = 0; address <= MAX_CPU_ADDRESS; address++) {
601 602
			if (__pcpu_sigp_relax(address, SIGP_SENSE, 0, NULL) ==
			    SIGP_CC_NOT_OPERATIONAL)
Martin Schwidefsky's avatar
Martin Schwidefsky committed
603 604 605 606 607
				continue;
			info->cpu[info->configured].address = address;
			info->configured++;
		}
		info->combined = info->configured;
608
	}
Martin Schwidefsky's avatar
Martin Schwidefsky committed
609
	return info;
610 611
}

Martin Schwidefsky's avatar
Martin Schwidefsky committed
612 613 614 615
static int __devinit smp_add_present_cpu(int cpu);

static int __devinit __smp_rescan_cpus(struct sclp_cpu_info *info,
				       int sysfs_add)
616
{
Martin Schwidefsky's avatar
Martin Schwidefsky committed
617
	struct pcpu *pcpu;
618
	cpumask_t avail;
Martin Schwidefsky's avatar
Martin Schwidefsky committed
619
	int cpu, nr, i;
620

Martin Schwidefsky's avatar
Martin Schwidefsky committed
621
	nr = 0;
622
	cpumask_xor(&avail, cpu_possible_mask, cpu_present_mask);
Martin Schwidefsky's avatar
Martin Schwidefsky committed
623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641
	cpu = cpumask_first(&avail);
	for (i = 0; (i < info->combined) && (cpu < nr_cpu_ids); i++) {
		if (info->has_cpu_type && info->cpu[i].type != boot_cpu_type)
			continue;
		if (pcpu_find_address(cpu_present_mask, info->cpu[i].address))
			continue;
		pcpu = pcpu_devices + cpu;
		pcpu->address = info->cpu[i].address;
		pcpu->state = (cpu >= info->configured) ?
			CPU_STATE_STANDBY : CPU_STATE_CONFIGURED;
		cpu_set_polarization(cpu, POLARIZATION_UNKNOWN);
		set_cpu_present(cpu, true);
		if (sysfs_add && smp_add_present_cpu(cpu) != 0)
			set_cpu_present(cpu, false);
		else
			nr++;
		cpu = cpumask_next(cpu, &avail);
	}
	return nr;
Linus Torvalds's avatar
Linus Torvalds committed
642 643
}

644 645 646 647 648
static void __init smp_detect_cpus(void)
{
	unsigned int cpu, c_cpus, s_cpus;
	struct sclp_cpu_info *info;

Martin Schwidefsky's avatar
Martin Schwidefsky committed
649
	info = smp_get_cpu_info();
650 651 652 653
	if (!info)
		panic("smp_detect_cpus failed to allocate memory\n");
	if (info->has_cpu_type) {
		for (cpu = 0; cpu < info->combined; cpu++) {
Martin Schwidefsky's avatar
Martin Schwidefsky committed
654 655 656 657 658
			if (info->cpu[cpu].address != boot_cpu_address)
				continue;
			/* The boot cpu dictates the cpu type. */
			boot_cpu_type = info->cpu[cpu].type;
			break;
659 660
		}
	}
Martin Schwidefsky's avatar
Martin Schwidefsky committed
661
	c_cpus = s_cpus = 0;
662
	for (cpu = 0; cpu < info->combined; cpu++) {
Martin Schwidefsky's avatar
Martin Schwidefsky committed
663
		if (info->has_cpu_type && info->cpu[cpu].type != boot_cpu_type)
664
			continue;
Martin Schwidefsky's avatar
Martin Schwidefsky committed
665 666 667 668
		if (cpu < info->configured) {
			smp_get_save_area(c_cpus, info->cpu[cpu].address);
			c_cpus++;
		} else
669 670
			s_cpus++;
	}
671
	pr_info("%d configured CPUs, %d standby CPUs\n", c_cpus, s_cpus);
672
	get_online_cpus();
Martin Schwidefsky's avatar
Martin Schwidefsky committed
673
	__smp_rescan_cpus(info, 0);
674
	put_online_cpus();
Martin Schwidefsky's avatar
Martin Schwidefsky committed
675
	kfree(info);
676 677
}

Linus Torvalds's avatar
Linus Torvalds committed
678
/*
679
 *	Activate a secondary processor.
Linus Torvalds's avatar
Linus Torvalds committed
680
 */
Martin Schwidefsky's avatar
Martin Schwidefsky committed
681
static void __cpuinit smp_start_secondary(void *cpuvoid)
Linus Torvalds's avatar
Linus Torvalds committed
682
{
Martin Schwidefsky's avatar
Martin Schwidefsky committed
683 684 685 686 687 688 689 690
	S390_lowcore.last_update_clock = get_clock();
	S390_lowcore.restart_stack = (unsigned long) restart_stack;
	S390_lowcore.restart_fn = (unsigned long) do_restart;
	S390_lowcore.restart_data = 0;
	S390_lowcore.restart_source = -1UL;
	restore_access_regs(S390_lowcore.access_regs_save_area);
	__ctl_load(S390_lowcore.cregs_save_area, 0, 15);
	__load_psw_mask(psw_kernel_bits | PSW_MASK_DAT);
691
	cpu_init();
692
	preempt_disable();
693 694
	init_cpu_timer();
	init_cpu_vtimer();
Heiko Carstens's avatar
Heiko Carstens committed
695
	pfault_init();
696
	notify_cpu_starting(smp_processor_id());
697
	ipi_call_lock();
698
	set_cpu_online(smp_processor_id(), true);
699
	ipi_call_unlock();
Linus Torvalds's avatar
Linus Torvalds committed
700
	local_irq_enable();
701 702
	/* cpu_idle will call schedule for us */
	cpu_idle();
Linus Torvalds's avatar
Linus Torvalds committed
703 704 705
}

/* Upping and downing of CPUs */
706
int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *tidle)
Linus Torvalds's avatar
Linus Torvalds committed
707
{
Martin Schwidefsky's avatar
Martin Schwidefsky committed
708 709
	struct pcpu *pcpu;
	int rc;
Linus Torvalds's avatar
Linus Torvalds committed
710

Martin Schwidefsky's avatar
Martin Schwidefsky committed
711 712
	pcpu = pcpu_devices + cpu;
	if (pcpu->state != CPU_STATE_CONFIGURED)
713
		return -EIO;
714 715
	if (pcpu_sigp_retry(pcpu, SIGP_INITIAL_CPU_RESET, 0) !=
	    SIGP_CC_ORDER_CODE_ACCEPTED)
716
		return -EIO;
717

Martin Schwidefsky's avatar
Martin Schwidefsky committed
718 719 720 721
	rc = pcpu_alloc_lowcore(pcpu, cpu);
	if (rc)
		return rc;
	pcpu_prepare_secondary(pcpu, cpu);
722
	pcpu_attach_task(pcpu, tidle);
Martin Schwidefsky's avatar
Martin Schwidefsky committed
723
	pcpu_start_fn(pcpu, smp_start_secondary, NULL);
Linus Torvalds's avatar
Linus Torvalds committed
724 725 726 727 728
	while (!cpu_online(cpu))
		cpu_relax();
	return 0;
}

729
static int __init setup_possible_cpus(char *s)
730
{
Martin Schwidefsky's avatar
Martin Schwidefsky committed
731
	int max, cpu;
732

Martin Schwidefsky's avatar
Martin Schwidefsky committed
733 734
	if (kstrtoint(s, 0, &max) < 0)
		return 0;
735
	init_cpu_possible(cpumask_of(0));
Martin Schwidefsky's avatar
Martin Schwidefsky committed
736
	for (cpu = 1; cpu < max && cpu < nr_cpu_ids; cpu++)
737
		set_cpu_possible(cpu, true);
738 739 740 741
	return 0;
}
early_param("possible_cpus", setup_possible_cpus);

742 743
#ifdef CONFIG_HOTPLUG_CPU

744
int __cpu_disable(void)
Linus Torvalds's avatar
Linus Torvalds committed
745
{
Martin Schwidefsky's avatar
Martin Schwidefsky committed
746
	unsigned long cregs[16];
Linus Torvalds's avatar
Linus Torvalds committed
747

Martin Schwidefsky's avatar
Martin Schwidefsky committed
748 749
	set_cpu_online(smp_processor_id(), false);
	/* Disable pseudo page faults on this cpu. */
Heiko Carstens's avatar
Heiko Carstens committed
750
	pfault_fini();
Martin Schwidefsky's avatar
Martin Schwidefsky committed
751 752 753 754 755 756
	/* Disable interrupt sources via control register. */
	__ctl_store(cregs, 0, 15);
	cregs[0]  &= ~0x0000ee70UL;	/* disable all external interrupts */
	cregs[6]  &= ~0xff000000UL;	/* disable all I/O interrupts */
	cregs[14] &= ~0x1f000000UL;	/* disable most machine checks */
	__ctl_load(cregs, 0, 15);
Linus Torvalds's avatar
Linus Torvalds committed
757 758 759
	return 0;
}

760
void __cpu_die(unsigned int cpu)
Linus Torvalds's avatar
Linus Torvalds committed
761
{
Martin Schwidefsky's avatar
Martin Schwidefsky committed
762 763
	struct pcpu *pcpu;

Linus Torvalds's avatar
Linus Torvalds committed
764
	/* Wait until target cpu is down */
Martin Schwidefsky's avatar
Martin Schwidefsky committed
765 766
	pcpu = pcpu_devices + cpu;
	while (!pcpu_stopped(pcpu))
Linus Torvalds's avatar
Linus Torvalds committed
767
		cpu_relax();
Martin Schwidefsky's avatar
Martin Schwidefsky committed
768
	pcpu_free_lowcore(pcpu);
769
	atomic_dec(&init_mm.context.attach_count);
Linus Torvalds's avatar
Linus Torvalds committed
770 771
}

772
void __noreturn cpu_die(void)
Linus Torvalds's avatar
Linus Torvalds committed
773 774
{
	idle_task_exit();
775
	pcpu_sigp_retry(pcpu_devices + smp_processor_id(), SIGP_STOP, 0);
Martin Schwidefsky's avatar
Martin Schwidefsky committed
776
	for (;;) ;
Linus Torvalds's avatar
Linus Torvalds committed
777 778
}

779 780
#endif /* CONFIG_HOTPLUG_CPU */

Linus Torvalds's avatar
Linus Torvalds committed
781 782
void __init smp_prepare_cpus(unsigned int max_cpus)
{
783 784 785
	/* request the 0x1201 emergency signal external interrupt */
	if (register_external_interrupt(0x1201, do_ext_call_interrupt) != 0)
		panic("Couldn't request external interrupt 0x1201");
786 787 788
	/* request the 0x1202 external call external interrupt */
	if (register_external_interrupt(0x1202, do_ext_call_interrupt) != 0)
		panic("Couldn't request external interrupt 0x1202");
Martin Schwidefsky's avatar
Martin Schwidefsky committed
789
	smp_detect_cpus();
Linus Torvalds's avatar
Linus Torvalds committed
790 791
}

792
void __init smp_prepare_boot_cpu(void)
Linus Torvalds's avatar
Linus Torvalds committed
793
{
Martin Schwidefsky's avatar
Martin Schwidefsky committed
794 795 796 797 798 799 800 801
	struct pcpu *pcpu = pcpu_devices;

	boot_cpu_address = stap();
	pcpu->state = CPU_STATE_CONFIGURED;
	pcpu->address = boot_cpu_address;
	pcpu->lowcore = (struct _lowcore *)(unsigned long) store_prefix();
	pcpu->async_stack = S390_lowcore.async_stack - ASYNC_SIZE;
	pcpu->panic_stack = S390_lowcore.panic_stack - PAGE_SIZE;
Linus Torvalds's avatar
Linus Torvalds committed
802
	S390_lowcore.percpu_offset = __per_cpu_offset[0];
803
	cpu_set_polarization(0, POLARIZATION_UNKNOWN);
Martin Schwidefsky's avatar
Martin Schwidefsky committed
804 805
	set_cpu_present(0, true);
	set_cpu_online(0, true);
Linus Torvalds's avatar
Linus Torvalds committed
806 807
}

808
void __init smp_cpus_done(unsigned int max_cpus)
Linus Torvalds's avatar
Linus Torvalds committed
809 810 811
{
}

812 813 814 815 816
void __init smp_setup_processor_id(void)
{
	S390_lowcore.cpu_nr = 0;
}

Linus Torvalds's avatar
Linus Torvalds committed
817 818 819 820 821 822 823 824
/*
 * the frequency of the profiling timer can be changed
 * by writing a multiplier value into /proc/profile.
 *
 * usually you want to run this on all CPUs ;)
 */
int setup_profiling_timer(unsigned int multiplier)
{
825
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
826 827
}

828
#ifdef CONFIG_HOTPLUG_CPU
829
static ssize_t cpu_configure_show(struct device *dev,
Martin Schwidefsky's avatar
Martin Schwidefsky committed
830
				  struct device_attribute *attr, char *buf)
831 832 833 834
{
	ssize_t count;

	mutex_lock(&smp_cpu_state_mutex);
Martin Schwidefsky's avatar
Martin Schwidefsky committed
835
	count = sprintf(buf, "%d\n", pcpu_devices[dev->id].state);
836 837 838 839
	mutex_unlock(&smp_cpu_state_mutex);
	return count;
}

840
static ssize_t cpu_configure_store(struct device *dev,
Martin Schwidefsky's avatar
Martin Schwidefsky committed
841 842
				   struct device_attribute *attr,
				   const char *buf, size_t count)
843
{
Martin Schwidefsky's avatar
Martin Schwidefsky committed
844 845
	struct pcpu *pcpu;
	int cpu, val, rc;
846 847 848 849 850 851
	char delim;

	if (sscanf(buf, "%d %c", &val, &delim) != 1)
		return -EINVAL;
	if (val != 0 && val != 1)
		return -EINVAL;
852
	get_online_cpus();
853
	mutex_lock(&smp_cpu_state_mutex);
854
	rc = -EBUSY;
855
	/* disallow configuration changes of online cpus and cpu 0 */
Martin Schwidefsky's avatar
Martin Schwidefsky committed
856
	cpu = dev->id;
857
	if (cpu_online(cpu) || cpu == 0)
858
		goto out;
Martin Schwidefsky's avatar
Martin Schwidefsky committed
859
	pcpu = pcpu_devices + cpu;
860 861 862
	rc = 0;
	switch (val) {
	case 0:
Martin Schwidefsky's avatar
Martin Schwidefsky committed
863 864 865 866 867 868 869 870
		if (pcpu->state != CPU_STATE_CONFIGURED)
			break;
		rc = sclp_cpu_deconfigure(pcpu->address);
		if (rc)
			break;
		pcpu->state = CPU_STATE_STANDBY;
		cpu_set_polarization(cpu, POLARIZATION_UNKNOWN);
		topology_expect_change();
871 872
		break;
	case 1:
Martin Schwidefsky's avatar
Martin Schwidefsky committed
873 874 875 876 877 878 879 880
		if (pcpu->state != CPU_STATE_STANDBY)
			break;
		rc = sclp_cpu_configure(pcpu->address);
		if (rc)
			break;
		pcpu->state = CPU_STATE_CONFIGURED;
		cpu_set_polarization(cpu, POLARIZATION_UNKNOWN);
		topology_expect_change();
881 882 883 884 885 886
		break;
	default:
		break;
	}
out:
	mutex_unlock(&smp_cpu_state_mutex);
887
	put_online_cpus();
888 889
	return rc ? rc : count;
}
890
static DEVICE_ATTR(configure, 0644, cpu_configure_show, cpu_configure_store);
891 892
#endif /* CONFIG_HOTPLUG_CPU */

893 894
static ssize_t show_cpu_address(struct device *dev,
				struct device_attribute *attr, char *buf)
895
{
Martin Schwidefsky's avatar
Martin Schwidefsky committed
896
	return sprintf(buf, "%d\n", pcpu_devices[dev->id].address);
897
}
898
static DEVICE_ATTR(address, 0444, show_cpu_address, NULL);
899 900 901

static struct attribute *cpu_common_attrs[] = {
#ifdef CONFIG_HOTPLUG_CPU
902
	&dev_attr_configure.attr,
903
#endif
904
	&dev_attr_address.attr,
905 906 907 908 909 910
	NULL,
};

static struct attribute_group cpu_common_attr_group = {
	.attrs = cpu_common_attrs,
};
Linus Torvalds's avatar
Linus Torvalds committed
911

912 913
static ssize_t show_idle_count(struct device *dev,
				struct device_attribute *attr, char *buf)
914
{
Martin Schwidefsky's avatar
Martin Schwidefsky committed
915
	struct s390_idle_data *idle = &per_cpu(s390_idle, dev->id);
916
	unsigned long long idle_count;
917
	unsigned int sequence;
918

Martin Schwidefsky's avatar
Martin Schwidefsky committed
919 920 921 922 923 924
	do {
		sequence = ACCESS_ONCE(idle->sequence);
		idle_count = ACCESS_ONCE(idle->idle_count);
		if (ACCESS_ONCE(idle->idle_enter))
			idle_count++;
	} while ((sequence & 1) || (idle->sequence != sequence));
925 926
	return sprintf(buf, "%llu\n", idle_count);
}
927
static DEVICE_ATTR(idle_count, 0444, show_idle_count, NULL);
928

929 930
static ssize_t show_idle_time(struct device *dev,
				struct device_attribute *attr, char *buf)
931
{
Martin Schwidefsky's avatar
Martin Schwidefsky committed
932 933
	struct s390_idle_data *idle = &per_cpu(s390_idle, dev->id);
	unsigned long long now, idle_time, idle_enter, idle_exit;
934
	unsigned int sequence;
935

Martin Schwidefsky's avatar
Martin Schwidefsky committed
936 937 938 939 940 941 942 943
	do {
		now = get_clock();
		sequence = ACCESS_ONCE(idle->sequence);
		idle_time = ACCESS_ONCE(idle->idle_time);
		idle_enter = ACCESS_ONCE(idle->idle_enter);
		idle_exit = ACCESS_ONCE(idle->idle_exit);
	} while ((sequence & 1) || (idle->sequence != sequence));
	idle_time += idle_enter ? ((idle_exit ? : now) - idle_enter) : 0;
944
	return sprintf(buf, "%llu\n", idle_time >> 12);
945
}
946
static DEVICE_ATTR(idle_time_us, 0444, show_idle_time, NULL);
947

948
static struct attribute *cpu_online_attrs[] = {
949 950
	&dev_attr_idle_count.attr,
	&dev_attr_idle_time_us.attr,
951 952 953
	NULL,
};

954 955
static struct attribute_group cpu_online_attr_group = {
	.attrs = cpu_online_attrs,
956 957
};

958 959 960 961
static int __cpuinit smp_cpu_notify(struct notifier_block *self,
				    unsigned long action, void *hcpu)
{
	unsigned int cpu = (unsigned int)(long)hcpu;
Martin Schwidefsky's avatar
Martin Schwidefsky committed
962
	struct cpu *c = &pcpu_devices[cpu].cpu;
963
	struct device *s = &c->dev;
964
	struct s390_idle_data *idle;
965
	int err = 0;
966 967 968

	switch (action) {
	case CPU_ONLINE:
969
	case CPU_ONLINE_FROZEN:
970
		idle = &per_cpu(s390_idle, cpu);
971
		memset(idle, 0, sizeof(struct s390_idle_data));
972
		err = sysfs_create_group(&s->kobj, &cpu_online_attr_group);
973 974
		break;
	case CPU_DEAD:
975
	case CPU_DEAD_FROZEN:
976
		sysfs_remove_group(&s->kobj, &cpu_online_attr_group);
977 978
		break;
	}
979
	return notifier_from_errno(err);
980 981 982
}

static struct notifier_block __cpuinitdata smp_cpu_nb = {
983
	.notifier_call = smp_cpu_notify,
984 985
};

986
static int __devinit smp_add_present_cpu(int cpu)
987
{
Martin Schwidefsky's avatar
Martin Schwidefsky committed
988
	struct cpu *c = &pcpu_devices[cpu].cpu;
989
	struct device *s = &c->dev;
990 991 992 993 994 995 996 997 998
	int rc;

	c->hotpluggable = 1;
	rc = register_cpu(c, cpu);
	if (rc)
		goto out;
	rc = sysfs_create_group(&s->kobj, &cpu_common_attr_group);
	if (rc)
		goto out_cpu;
999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012
	if (cpu_online(cpu)) {
		rc = sysfs_create_group(&s->kobj, &cpu_online_attr_group);
		if (rc)
			goto out_online;
	}
	rc = topology_cpu_init(c);
	if (rc)
		goto out_topology;
	return 0;

out_topology:
	if (cpu_online(cpu))
		sysfs_remove_group(&s->kobj, &cpu_online_attr_group);
out_online:
1013 1014 1015 1016 1017 1018 1019 1020 1021 1022
	sysfs_remove_group(&s->kobj, &cpu_common_attr_group);
out_cpu:
#ifdef CONFIG_HOTPLUG_CPU
	unregister_cpu(c);
#endif
out:
	return rc;
}

#ifdef CONFIG_HOTPLUG_CPU
1023

1024
int __ref smp_rescan_cpus(void)
1025
{
Martin Schwidefsky's avatar
Martin Schwidefsky committed
1026 1027
	struct sclp_cpu_info *info;
	int nr;
1028

Martin Schwidefsky's avatar
Martin Schwidefsky committed
1029 1030 1031
	info = smp_get_cpu_info();
	if (!info)
		return -ENOMEM;
1032
	get_online_cpus();
1033
	mutex_lock(&smp_cpu_state_mutex);
Martin Schwidefsky's avatar
Martin Schwidefsky committed
1034
	nr = __smp_rescan_cpus(info, 1);
1035
	mutex_unlock(&smp_cpu_state_mutex);
1036
	put_online_cpus();
Martin Schwidefsky's avatar
Martin Schwidefsky committed
1037 1038
	kfree(info);
	if (nr)
1039
		topology_schedule_update();
Martin Schwidefsky's avatar
Martin Schwidefsky committed
1040
	return 0;
1041 1042
}

1043 1044
static ssize_t __ref rescan_store(struct device *dev,
				  struct device_attribute *attr,
1045
				  const char *buf,
1046 1047 1048 1049 1050
				  size_t count)
{
	int rc;

	rc = smp_rescan_cpus();
1051 1052
	return rc ? rc : count;
}
1053
static DEVICE_ATTR(rescan, 0200, NULL, rescan_store);
1054 1055
#endif /* CONFIG_HOTPLUG_CPU */

1056
static int __init s390_smp_init(void)
Linus Torvalds's avatar
Linus Torvalds committed
1057
{
1058
	int cpu, rc;
1059 1060

	register_cpu_notifier(&smp_cpu_nb);
1061
#ifdef CONFIG_HOTPLUG_CPU
1062
	rc = device_create_file(cpu_subsys.dev_root, &dev_attr_rescan);
1063 1064 1065 1066 1067
	if (rc)
		return rc;
#endif
	for_each_present_cpu(cpu) {
		rc = smp_add_present_cpu(cpu);
1068 1069
		if (rc)
			return rc;
Linus Torvalds's avatar
Linus Torvalds committed
1070 1071 1072
	}
	return 0;
}
1073
subsys_initcall(s390_smp_init);