Skip to content

L
linux
Commit 1ef8b835 authored by Martin Schwidefsky's avatar Martin Schwidefsky Committed by Linus Torvalds
Browse files
Options

[PATCH] s390: cpu-idle notifier 

This patch introduces a notifier chain for cpu idle.  There are two events
CPU_IDLE and CPU_NOT_IDLE that are called just before the cpu goes to sleep
and right after the cpu woke up again.  The notifier is used to simplify the
no-hz-timer-in-idle feature and the virtual cpu timers.  In addition the
virtual cpu timer functions have been moved to arch/s390/kernel/vtime.c, which
gives a nice separation between the timer functions related to real time and
the timer functions related to virtual cpu time.
Signed-off-by: default avatarMartin Schwidefsky <schwidefsky@de.ibm.com>
Signed-off-by: default avatarAndrew Morton <akpm@osdl.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@osdl.org>
parent 6ac48313
Show whitespace changes
Inline Side-by-side
Showing with 578 additions and 538 deletions
arch/s390/kernel/Makefile
View file @ 1ef8b835
......@@ -23,6 +23,8 @@ obj-$(CONFIG_BINFMT_ELF32) += binfmt_elf32.o
obj-$(CONFIG_ARCH_S390_31) += entry.o reipl.o
obj-$(CONFIG_ARCH_S390X) += entry64.o reipl64.o
obj-$(CONFIG_VIRT_TIMER) += vtime.o
#
# This is just to get the dependencies...
#
......
arch/s390/kernel/process.c
View file @ 1ef8b835
......@@ -34,6 +34,8 @@
#include <linux/delay.h>
#include <linux/reboot.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/notifier.h>
#include <asm/uaccess.h>
#include <asm/pgtable.h>
......@@ -41,9 +43,7 @@
#include <asm/io.h>
#include <asm/processor.h>
#include <asm/irq.h>
#if defined(CONFIG_VIRT_TIMER) || defined (CONFIG_NO_IDLE_HZ)
#include <asm/timer.h>
#endif
asmlinkage void ret_from_fork(void) __asm__("ret_from_fork");
......@@ -68,13 +68,39 @@ unsigned long thread_saved_pc(struct task_struct *tsk)
}
/*
* The idle loop on a S390...
* Need to know about CPUs going idle?
*/
static struct notifier_block *idle_chain;
int register_idle_notifier(struct notifier_block *nb)
{
return notifier_chain_register(&idle_chain, nb);
}
EXPORT_SYMBOL(register_idle_notifier);
int unregister_idle_notifier(struct notifier_block *nb)
{
return notifier_chain_unregister(&idle_chain, nb);
}
EXPORT_SYMBOL(unregister_idle_notifier);
void do_monitor_call(struct pt_regs *regs, long interruption_code)
{
/* disable monitor call class 0 */
__ctl_clear_bit(8, 15);
notifier_call_chain(&idle_chain, CPU_NOT_IDLE,
(void *)(long) smp_processor_id());
}
/*
* The idle loop on a S390...
*/
void default_idle(void)
{
psw_t wait_psw;
unsigned long reg;
int cpu, rc;
local_irq_disable();
if (need_resched()) {
......@@ -83,15 +109,18 @@ void default_idle(void)
return;
}
#if defined(CONFIG_VIRT_TIMER) || defined (CONFIG_NO_IDLE_HZ)
/*
* hook to stop timers that should not tick while CPU is idle
*/
if (stop_timers()) {
/* CPU is going idle. */
cpu = smp_processor_id();
rc = notifier_call_chain(&idle_chain, CPU_IDLE, (void *)(long) cpu);
if (rc != NOTIFY_OK && rc != NOTIFY_DONE)
BUG();
if (rc != NOTIFY_OK) {
local_irq_enable();
return;
}
#endif
/* enable monitor call class 0 */
__ctl_set_bit(8, 15);
/*
* Wait for external, I/O or machine check interrupt and
......
arch/s390/kernel/s390_ksyms.c
View file @ 1ef8b835
......@@ -19,9 +19,6 @@
#ifdef CONFIG_IP_MULTICAST
#include <net/arp.h>
#endif
#ifdef CONFIG_VIRT_TIMER
#include <asm/timer.h>
#endif
/*
* memory management
......
arch/s390/kernel/smp.c
View file @ 1ef8b835
......@@ -509,6 +509,7 @@ void __init smp_check_cpus(unsigned int max_cpus)
* Activate a secondary processor.
*/
extern void init_cpu_timer(void);
extern void init_cpu_vtimer(void);
extern int pfault_init(void);
extern int pfault_token(void);
......@@ -518,6 +519,9 @@ int __devinit start_secondary(void *cpuvoid)
cpu_init();
/* init per CPU timer */
init_cpu_timer();
#ifdef CONFIG_VIRT_TIMER
init_cpu_vtimer();
#endif
#ifdef CONFIG_PFAULT
/* Enable pfault pseudo page faults on this cpu. */
pfault_init();
......
arch/s390/kernel/time.c
View file @ 1ef8b835
/*
* arch/s390/kernel/time.c
* Time of day based timer functions.
*
* S390 version
* Copyright (C) 1999 IBM Deutschland Entwicklung GmbH, IBM Corporation
......@@ -11,6 +12,7 @@
* Copyright (C) 1991, 1992, 1995 Linus Torvalds
*/
#include <linux/config.h>
#include <linux/errno.h>
#include <linux/module.h>
#include <linux/sched.h>
......@@ -26,16 +28,14 @@
#include <linux/types.h>
#include <linux/profile.h>
#include <linux/timex.h>
#include <linux/config.h>
#include <linux/notifier.h>
#include <asm/uaccess.h>
#include <asm/delay.h>
#include <asm/s390_ext.h>
#include <asm/div64.h>
#include <asm/irq.h>
#ifdef CONFIG_VIRT_TIMER
#include <asm/timer.h>
#endif
/* change this if you have some constant time drift */
#define USECS_PER_JIFFY ((unsigned long) 1000000/HZ)
......@@ -60,12 +60,6 @@ static u64 xtime_cc;
extern unsigned long wall_jiffies;
#ifdef CONFIG_VIRT_TIMER
#define VTIMER_MAGIC (0x4b87ad6e + 1)
static ext_int_info_t ext_int_info_timer;
DEFINE_PER_CPU(struct vtimer_queue, virt_cpu_timer);
#endif
/*
* Scheduler clock - returns current time in nanosec units.
*/
......@@ -236,6 +230,8 @@ void account_ticks(struct pt_regs *regs)
__u32 ticks;
/* Calculate how many ticks have passed. */
if (S390_lowcore.int_clock < S390_lowcore.jiffy_timer)
return;
tmp = S390_lowcore.int_clock - S390_lowcore.jiffy_timer;
if (tmp >= 2*CLK_TICKS_PER_JIFFY) { /* more than two ticks ? */
ticks = __calculate_ticks(tmp) + 1;
......@@ -283,174 +279,6 @@ void account_ticks(struct pt_regs *regs)
s390_do_profile(regs);
}
#ifdef CONFIG_VIRT_TIMER
void start_cpu_timer(void)
{
struct vtimer_queue *vt_list;
vt_list = &per_cpu(virt_cpu_timer, smp_processor_id());
set_vtimer(vt_list->idle);
}
int stop_cpu_timer(void)
{
__u64 done;
struct vtimer_queue *vt_list;
vt_list = &per_cpu(virt_cpu_timer, smp_processor_id());
/* nothing to do */
if (list_empty(&vt_list->list)) {
vt_list->idle = VTIMER_MAX_SLICE;
goto fire;
}
/* store progress */
asm volatile ("STPT %0" : "=m" (done));
/*
* If done is negative we do not stop the CPU timer
* because we will get instantly an interrupt that
* will start the CPU timer again.
*/
if (done & 1LL<<63)
return 1;
else
vt_list->offset += vt_list->to_expire - done;
/* save the actual expire value */
vt_list->idle = done;
/*
* We cannot halt the CPU timer, we just write a value that
* nearly never expires (only after 71 years) and re-write
* the stored expire value if we continue the timer
*/
fire:
set_vtimer(VTIMER_MAX_SLICE);
return 0;
}
void set_vtimer(__u64 expires)
{
asm volatile ("SPT %0" : : "m" (expires));
/* store expire time for this CPU timer */
per_cpu(virt_cpu_timer, smp_processor_id()).to_expire = expires;
}
/*
* Sorted add to a list. List is linear searched until first bigger
* element is found.
*/
void list_add_sorted(struct vtimer_list *timer, struct list_head *head)
{
struct vtimer_list *event;
list_for_each_entry(event, head, entry) {
if (event->expires > timer->expires) {
list_add_tail(&timer->entry, &event->entry);
return;
}
}
list_add_tail(&timer->entry, head);
}
/*
* Do the callback functions of expired vtimer events.
* Called from within the interrupt handler.
*/
static void do_callbacks(struct list_head *cb_list, struct pt_regs *regs)
{
struct vtimer_queue *vt_list;
struct vtimer_list *event, *tmp;
void (*fn)(unsigned long, struct pt_regs*);
unsigned long data;
if (list_empty(cb_list))
return;
vt_list = &per_cpu(virt_cpu_timer, smp_processor_id());
list_for_each_entry_safe(event, tmp, cb_list, entry) {
fn = event->function;
data = event->data;
fn(data, regs);
if (!event->interval)
/* delete one shot timer */
list_del_init(&event->entry);
else {
/* move interval timer back to list */
spin_lock(&vt_list->lock);
list_del_init(&event->entry);
list_add_sorted(event, &vt_list->list);
spin_unlock(&vt_list->lock);
}
}
}
/*
* Handler for the virtual CPU timer.
*/
static void do_cpu_timer_interrupt(struct pt_regs *regs, __u16 error_code)
{
int cpu;
__u64 next, delta;
struct vtimer_queue *vt_list;
struct vtimer_list *event, *tmp;
struct list_head *ptr;
/* the callback queue */
struct list_head cb_list;
INIT_LIST_HEAD(&cb_list);
cpu = smp_processor_id();
vt_list = &per_cpu(virt_cpu_timer, cpu);
/* walk timer list, fire all expired events */
spin_lock(&vt_list->lock);
if (vt_list->to_expire < VTIMER_MAX_SLICE)
vt_list->offset += vt_list->to_expire;
list_for_each_entry_safe(event, tmp, &vt_list->list, entry) {
if (event->expires > vt_list->offset)
/* found first unexpired event, leave */
break;
/* re-charge interval timer, we have to add the offset */
if (event->interval)
event->expires = event->interval + vt_list->offset;
/* move expired timer to the callback queue */
list_move_tail(&event->entry, &cb_list);
}
spin_unlock(&vt_list->lock);
do_callbacks(&cb_list, regs);
/* next event is first in list */
spin_lock(&vt_list->lock);
if (!list_empty(&vt_list->list)) {
ptr = vt_list->list.next;
event = list_entry(ptr, struct vtimer_list, entry);
next = event->expires - vt_list->offset;
/* add the expired time from this interrupt handler
* and the callback functions
*/
asm volatile ("STPT %0" : "=m" (delta));
delta = 0xffffffffffffffffLL - delta + 1;
vt_list->offset += delta;
next -= delta;
} else {
vt_list->offset = 0;
next = VTIMER_MAX_SLICE;
}
spin_unlock(&vt_list->lock);
set_vtimer(next);
}
#endif
#ifdef CONFIG_NO_IDLE_HZ
#ifdef CONFIG_NO_IDLE_HZ_INIT
......@@ -459,66 +287,11 @@ int sysctl_hz_timer = 0;
int sysctl_hz_timer = 1;
#endif
/*
* Start the HZ tick on the current CPU.
* Only cpu_idle may call this function.
*/
void start_hz_timer(struct pt_regs *regs)
{
__u64 tmp;
__u32 ticks;
if (!cpu_isset(smp_processor_id(), nohz_cpu_mask))
return;
/* Calculate how many ticks have passed */
asm volatile ("STCK 0(%0)" : : "a" (&tmp) : "memory", "cc");
tmp = tmp + CLK_TICKS_PER_JIFFY - S390_lowcore.jiffy_timer;
ticks = __calculate_ticks(tmp);
S390_lowcore.jiffy_timer += CLK_TICKS_PER_JIFFY * (__u64) ticks;
/* Set the clock comparator to the next tick. */
tmp = S390_lowcore.jiffy_timer + CPU_DEVIATION;
asm volatile ("SCKC %0" : : "m" (tmp));
/* Charge the ticks. */
if (ticks > 0) {
#ifdef CONFIG_SMP
/*
* Do not rely on the boot cpu to do the calls to do_timer.
* Spread it over all cpus instead.
*/
write_seqlock(&xtime_lock);
if (S390_lowcore.jiffy_timer > xtime_cc) {
__u32 xticks;
tmp = S390_lowcore.jiffy_timer - xtime_cc;
if (tmp >= 2*CLK_TICKS_PER_JIFFY) {
xticks = __calculate_ticks(tmp);
xtime_cc += (__u64) xticks*CLK_TICKS_PER_JIFFY;
} else {
xticks = 1;
xtime_cc += CLK_TICKS_PER_JIFFY;
}
while (xticks--)
do_timer(regs);
}
write_sequnlock(&xtime_lock);
while (ticks--)
update_process_times(user_mode(regs));
#else
while (ticks--)
do_timer(regs);
#endif
}
cpu_clear(smp_processor_id(), nohz_cpu_mask);
}
/*
* Stop the HZ tick on the current CPU.
* Only cpu_idle may call this function.
*/
void stop_hz_timer(void)
static inline void stop_hz_timer(void)
{
__u64 timer;
......@@ -541,57 +314,52 @@ void stop_hz_timer(void)
timer = jiffies_timer_cc + timer * CLK_TICKS_PER_JIFFY;
asm volatile ("SCKC %0" : : "m" (timer));
}
#endif
#if defined(CONFIG_VIRT_TIMER) || defined(CONFIG_NO_IDLE_HZ)
void do_monitor_call(struct pt_regs *regs, long interruption_code)
{
/* disable monitor call class 0 */
__ctl_clear_bit(8, 15);
#ifdef CONFIG_VIRT_TIMER
start_cpu_timer();
#endif
#ifdef CONFIG_NO_IDLE_HZ
start_hz_timer(regs);
#endif
}
/*
* called from cpu_idle to stop any timers
* returns 1 if CPU should not be stopped
* Start the HZ tick on the current CPU.
* Only cpu_idle may call this function.
*/
int stop_timers(void)
static inline void start_hz_timer(void)
{
#ifdef CONFIG_VIRT_TIMER
if (stop_cpu_timer())
return 1;
#endif
if (!cpu_isset(smp_processor_id(), nohz_cpu_mask))
return;
account_ticks(__KSTK_PTREGS(current));
cpu_clear(smp_processor_id(), nohz_cpu_mask);
}
#ifdef CONFIG_NO_IDLE_HZ
static int nohz_idle_notify(struct notifier_block *self,
unsigned long action, void *hcpu)
{
switch (action) {
case CPU_IDLE:
stop_hz_timer();
#endif
break;
case CPU_NOT_IDLE:
start_hz_timer();
break;
}
return NOTIFY_OK;
}
/* enable monitor call class 0 */
__ctl_set_bit(8, 15);
static struct notifier_block nohz_idle_nb = {
.notifier_call = nohz_idle_notify,
};
return 0;
void __init nohz_init(void)
{
if (register_idle_notifier(&nohz_idle_nb))
panic("Couldn't register idle notifier");
}
#endif
/*
* Start the clock comparator and the virtual CPU timer
* on the current CPU.
* Start the clock comparator on the current CPU.
*/
void init_cpu_timer(void)
{
unsigned long cr0;
__u64 timer;
#ifdef CONFIG_VIRT_TIMER
struct vtimer_queue *vt_list;
#endif
timer = jiffies_timer_cc + jiffies_64 * CLK_TICKS_PER_JIFFY;
S390_lowcore.jiffy_timer = timer + CLK_TICKS_PER_JIFFY;
......@@ -601,24 +369,10 @@ void init_cpu_timer(void)
__ctl_store(cr0, 0, 0);
cr0 |= 0x800;
__ctl_load(cr0, 0, 0);
#ifdef CONFIG_VIRT_TIMER
/* kick the virtual timer */
timer = VTIMER_MAX_SLICE;
asm volatile ("SPT %0" : : "m" (timer));
__ctl_store(cr0, 0, 0);
cr0 |= 0x400;
__ctl_load(cr0, 0, 0);
vt_list = &per_cpu(virt_cpu_timer, smp_processor_id());
INIT_LIST_HEAD(&vt_list->list);
spin_lock_init(&vt_list->lock);
vt_list->to_expire = 0;
vt_list->offset = 0;
vt_list->idle = 0;
#endif
}
extern void vtime_init(void);
/*
* Initialize the TOD clock and the CPU timer of
* the boot cpu.
......@@ -661,247 +415,14 @@ void __init time_init(void)
&ext_int_info_cc) != 0)
panic("Couldn't request external interrupt 0x1004");
#ifdef CONFIG_VIRT_TIMER
/* request the cpu timer external interrupt */
if (register_early_external_interrupt(0x1005, do_cpu_timer_interrupt,
&ext_int_info_timer) != 0)
panic("Couldn't request external interrupt 0x1005");
#endif
init_cpu_timer();
}
#ifdef CONFIG_VIRT_TIMER
void init_virt_timer(struct vtimer_list *timer)
{
timer->magic = VTIMER_MAGIC;
timer->function = NULL;
INIT_LIST_HEAD(&timer->entry);
spin_lock_init(&timer->lock);
}
static inline int check_vtimer(struct vtimer_list *timer)
{
if (timer->magic != VTIMER_MAGIC)
return -EINVAL;
return 0;
}
static inline int vtimer_pending(struct vtimer_list *timer)
{
return (!list_empty(&timer->entry));
}
/*
* this function should only run on the specified CPU
*/
static void internal_add_vtimer(struct vtimer_list *timer)
{
unsigned long flags;
__u64 done;
struct vtimer_list *event;
struct vtimer_queue *vt_list;
vt_list = &per_cpu(virt_cpu_timer, timer->cpu);
spin_lock_irqsave(&vt_list->lock, flags);
if (timer->cpu != smp_processor_id())
printk("internal_add_vtimer: BUG, running on wrong CPU");
/* if list is empty we only have to set the timer */
if (list_empty(&vt_list->list)) {
/* reset the offset, this may happen if the last timer was
* just deleted by mod_virt_timer and the interrupt
* didn't happen until here
*/
vt_list->offset = 0;
goto fire;
}
/* save progress */
asm volatile ("STPT %0" : "=m" (done));
/* calculate completed work */
done = vt_list->to_expire - done + vt_list->offset;
vt_list->offset = 0;
list_for_each_entry(event, &vt_list->list, entry)
event->expires -= done;
fire:
list_add_sorted(timer, &vt_list->list);
/* get first element, which is the next vtimer slice */
event = list_entry(vt_list->list.next, struct vtimer_list, entry);
set_vtimer(event->expires);
spin_unlock_irqrestore(&vt_list->lock, flags);
/* release CPU aquired in prepare_vtimer or mod_virt_timer() */
put_cpu();
}
static inline int prepare_vtimer(struct vtimer_list *timer)
{
if (check_vtimer(timer) || !timer->function) {
printk("add_virt_timer: uninitialized timer\n");
return -EINVAL;
}
if (!timer->expires || timer->expires > VTIMER_MAX_SLICE) {
printk("add_virt_timer: invalid timer expire value!\n");
return -EINVAL;
}
if (vtimer_pending(timer)) {
printk("add_virt_timer: timer pending\n");
return -EBUSY;
}
timer->cpu = get_cpu();
return 0;
}
/*
* add_virt_timer - add an oneshot virtual CPU timer
*/
void add_virt_timer(void *new)
{
struct vtimer_list *timer;
timer = (struct vtimer_list *)new;
if (prepare_vtimer(timer) < 0)
return;
timer->interval = 0;
internal_add_vtimer(timer);
}
/*
* add_virt_timer_int - add an interval virtual CPU timer
*/
void add_virt_timer_periodic(void *new)
{
struct vtimer_list *timer;
timer = (struct vtimer_list *)new;
if (prepare_vtimer(timer) < 0)
return;
timer->interval = timer->expires;
internal_add_vtimer(timer);
}
/*
* If we change a pending timer the function must be called on the CPU
* where the timer is running on, e.g. by smp_call_function_on()
*
* The original mod_timer adds the timer if it is not pending. For compatibility
* we do the same. The timer will be added on the current CPU as a oneshot timer.
*
* returns whether it has modified a pending timer (1) or not (0)
*/
int mod_virt_timer(struct vtimer_list *timer, __u64 expires)
{
struct vtimer_queue *vt_list;
unsigned long flags;
int cpu;
if (check_vtimer(timer) || !timer->function) {
printk("mod_virt_timer: uninitialized timer\n");
return -EINVAL;
}
if (!expires || expires > VTIMER_MAX_SLICE) {
printk("mod_virt_timer: invalid expire range\n");
return -EINVAL;
}
/*
* This is a common optimization triggered by the
* networking code - if the timer is re-modified
* to be the same thing then just return:
*/
if (timer->expires == expires && vtimer_pending(timer))
return 1;
cpu = get_cpu();
vt_list = &per_cpu(virt_cpu_timer, cpu);
/* disable interrupts before test if timer is pending */
spin_lock_irqsave(&vt_list->lock, flags);
/* if timer isn't pending add it on the current CPU */
if (!vtimer_pending(timer)) {
spin_unlock_irqrestore(&vt_list->lock, flags);
/* we do not activate an interval timer with mod_virt_timer */
timer->interval = 0;
timer->expires = expires;
timer->cpu = cpu;
internal_add_vtimer(timer);
return 0;
}
/* check if we run on the right CPU */
if (timer->cpu != cpu) {
printk("mod_virt_timer: running on wrong CPU, check your code\n");
spin_unlock_irqrestore(&vt_list->lock, flags);
put_cpu();
return -EINVAL;
}
list_del_init(&timer->entry);
timer->expires = expires;
/* also change the interval if we have an interval timer */
if (timer->interval)
timer->interval = expires;
/* the timer can't expire anymore so we can release the lock */
spin_unlock_irqrestore(&vt_list->lock, flags);
internal_add_vtimer(timer);
return 1;
}
/*
* delete a virtual timer
*
* returns whether the deleted timer was pending (1) or not (0)
*/
int del_virt_timer(struct vtimer_list *timer)
{
unsigned long flags;
struct vtimer_queue *vt_list;
if (check_vtimer(timer)) {
printk("del_virt_timer: timer not initialized\n");
return -EINVAL;
}
/* check if timer is pending */
if (!vtimer_pending(timer))
return 0;
if (!cpu_online(timer->cpu)) {
printk("del_virt_timer: CPU not present!\n");
return -1;
}
vt_list = &per_cpu(virt_cpu_timer, timer->cpu);
spin_lock_irqsave(&vt_list->lock, flags);
/* we don't interrupt a running timer, just let it expire! */
list_del_init(&timer->entry);
/* last timer removed */
if (list_empty(&vt_list->list)) {
vt_list->to_expire = 0;
vt_list->offset = 0;
}
#ifdef CONFIG_NO_IDLE_HZ
nohz_init();
#endif
spin_unlock_irqrestore(&vt_list->lock, flags);
return 1;
}
#ifdef CONFIG_VIRT_TIMER
vtime_init();
#endif
}
arch/s390/kernel/traps.c
View file @ 1ef8b835
......@@ -64,9 +64,7 @@ extern void pfault_fini(void);
extern void pfault_interrupt(struct pt_regs *regs, __u16 error_code);
static ext_int_info_t ext_int_pfault;
#endif
#if defined(CONFIG_NO_IDLE_HZ) || defined(CONFIG_VIRT_TIMER)
extern pgm_check_handler_t do_monitor_call;
#endif
#define stack_pointer ({ void **sp; asm("la %0,0(15)" : "=&d" (sp)); sp; })
......@@ -620,9 +618,8 @@ void __init trap_init(void)
#endif /* CONFIG_ARCH_S390X */
pgm_check_table[0x15] = &operand_exception;
pgm_check_table[0x1C] = &privileged_op;
#if defined(CONFIG_VIRT_TIMER) || defined(CONFIG_NO_IDLE_HZ)
pgm_check_table[0x40] = &do_monitor_call;
#endif
if (MACHINE_IS_VM) {
/*
* First try to get pfault pseudo page faults going.
......
arch/s390/kernel/vtime.c 0 → 100644
View file @ 1ef8b835
/*
* arch/s390/kernel/vtime.c
* Virtual cpu timer based timer functions.
*
* S390 version
* Copyright (C) 2004 IBM Deutschland Entwicklung GmbH, IBM Corporation
* Author(s): Jan Glauber <jan.glauber@de.ibm.com>
*/
#include <linux/config.h>
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/time.h>
#include <linux/delay.h>
#include <linux/init.h>
#include <linux/smp.h>
#include <linux/types.h>
#include <linux/timex.h>
#include <linux/notifier.h>
#include <asm/s390_ext.h>
#include <asm/timer.h>
#define VTIMER_MAGIC (0x4b87ad6e + 1)
static ext_int_info_t ext_int_info_timer;
DEFINE_PER_CPU(struct vtimer_queue, virt_cpu_timer);
void start_cpu_timer(void)
{
struct vtimer_queue *vt_list;
vt_list = &per_cpu(virt_cpu_timer, smp_processor_id());
set_vtimer(vt_list->idle);
}
void stop_cpu_timer(void)
{
__u64 done;
struct vtimer_queue *vt_list;
vt_list = &per_cpu(virt_cpu_timer, smp_processor_id());
/* nothing to do */
if (list_empty(&vt_list->list)) {
vt_list->idle = VTIMER_MAX_SLICE;
goto fire;
}
/* store progress */
asm volatile ("STPT %0" : "=m" (done));
/*
* If done is negative we do not stop the CPU timer
* because we will get instantly an interrupt that
* will start the CPU timer again.
*/
if (done & 1LL<<63)
return;
else
vt_list->offset += vt_list->to_expire - done;
/* save the actual expire value */
vt_list->idle = done;
/*
* We cannot halt the CPU timer, we just write a value that
* nearly never expires (only after 71 years) and re-write
* the stored expire value if we continue the timer
*/
fire:
set_vtimer(VTIMER_MAX_SLICE);
}
void set_vtimer(__u64 expires)
{
asm volatile ("SPT %0" : : "m" (expires));
/* store expire time for this CPU timer */
per_cpu(virt_cpu_timer, smp_processor_id()).to_expire = expires;
}
/*
* Sorted add to a list. List is linear searched until first bigger
* element is found.
*/
void list_add_sorted(struct vtimer_list *timer, struct list_head *head)
{
struct vtimer_list *event;
list_for_each_entry(event, head, entry) {
if (event->expires > timer->expires) {
list_add_tail(&timer->entry, &event->entry);
return;
}
}
list_add_tail(&timer->entry, head);
}
/*
* Do the callback functions of expired vtimer events.
* Called from within the interrupt handler.
*/
static void do_callbacks(struct list_head *cb_list, struct pt_regs *regs)
{
struct vtimer_queue *vt_list;
struct vtimer_list *event, *tmp;
void (*fn)(unsigned long, struct pt_regs*);
unsigned long data;
if (list_empty(cb_list))
return;
vt_list = &per_cpu(virt_cpu_timer, smp_processor_id());
list_for_each_entry_safe(event, tmp, cb_list, entry) {
fn = event->function;
data = event->data;
fn(data, regs);
if (!event->interval)
/* delete one shot timer */
list_del_init(&event->entry);
else {
/* move interval timer back to list */
spin_lock(&vt_list->lock);
list_del_init(&event->entry);
list_add_sorted(event, &vt_list->list);
spin_unlock(&vt_list->lock);
}
}
}
/*
* Handler for the virtual CPU timer.
*/
static void do_cpu_timer_interrupt(struct pt_regs *regs, __u16 error_code)
{
int cpu;
__u64 next, delta;
struct vtimer_queue *vt_list;
struct vtimer_list *event, *tmp;
struct list_head *ptr;
/* the callback queue */
struct list_head cb_list;
INIT_LIST_HEAD(&cb_list);
cpu = smp_processor_id();
vt_list = &per_cpu(virt_cpu_timer, cpu);
/* walk timer list, fire all expired events */
spin_lock(&vt_list->lock);
if (vt_list->to_expire < VTIMER_MAX_SLICE)
vt_list->offset += vt_list->to_expire;
list_for_each_entry_safe(event, tmp, &vt_list->list, entry) {
if (event->expires > vt_list->offset)
/* found first unexpired event, leave */
break;
/* re-charge interval timer, we have to add the offset */
if (event->interval)
event->expires = event->interval + vt_list->offset;
/* move expired timer to the callback queue */
list_move_tail(&event->entry, &cb_list);
}
spin_unlock(&vt_list->lock);
do_callbacks(&cb_list, regs);
/* next event is first in list */
spin_lock(&vt_list->lock);
if (!list_empty(&vt_list->list)) {
ptr = vt_list->list.next;
event = list_entry(ptr, struct vtimer_list, entry);
next = event->expires - vt_list->offset;
/* add the expired time from this interrupt handler
* and the callback functions
*/
asm volatile ("STPT %0" : "=m" (delta));
delta = 0xffffffffffffffffLL - delta + 1;
vt_list->offset += delta;
next -= delta;
} else {
vt_list->offset = 0;
next = VTIMER_MAX_SLICE;
}
spin_unlock(&vt_list->lock);
set_vtimer(next);
}
void init_virt_timer(struct vtimer_list *timer)
{
timer->magic = VTIMER_MAGIC;
timer->function = NULL;
INIT_LIST_HEAD(&timer->entry);
spin_lock_init(&timer->lock);
}
EXPORT_SYMBOL(init_virt_timer);
static inline int check_vtimer(struct vtimer_list *timer)
{
if (timer->magic != VTIMER_MAGIC)
return -EINVAL;
return 0;
}
static inline int vtimer_pending(struct vtimer_list *timer)
{
return (!list_empty(&timer->entry));
}
/*
* this function should only run on the specified CPU
*/
static void internal_add_vtimer(struct vtimer_list *timer)
{
unsigned long flags;
__u64 done;
struct vtimer_list *event;
struct vtimer_queue *vt_list;
vt_list = &per_cpu(virt_cpu_timer, timer->cpu);
spin_lock_irqsave(&vt_list->lock, flags);
if (timer->cpu != smp_processor_id())
printk("internal_add_vtimer: BUG, running on wrong CPU");
/* if list is empty we only have to set the timer */
if (list_empty(&vt_list->list)) {
/* reset the offset, this may happen if the last timer was
* just deleted by mod_virt_timer and the interrupt
* didn't happen until here
*/
vt_list->offset = 0;
goto fire;
}
/* save progress */
asm volatile ("STPT %0" : "=m" (done));
/* calculate completed work */
done = vt_list->to_expire - done + vt_list->offset;
vt_list->offset = 0;
list_for_each_entry(event, &vt_list->list, entry)
event->expires -= done;
fire:
list_add_sorted(timer, &vt_list->list);
/* get first element, which is the next vtimer slice */
event = list_entry(vt_list->list.next, struct vtimer_list, entry);
set_vtimer(event->expires);
spin_unlock_irqrestore(&vt_list->lock, flags);
/* release CPU aquired in prepare_vtimer or mod_virt_timer() */
put_cpu();
}
static inline int prepare_vtimer(struct vtimer_list *timer)
{
if (check_vtimer(timer) || !timer->function) {
printk("add_virt_timer: uninitialized timer\n");
return -EINVAL;
}
if (!timer->expires || timer->expires > VTIMER_MAX_SLICE) {
printk("add_virt_timer: invalid timer expire value!\n");
return -EINVAL;
}
if (vtimer_pending(timer)) {
printk("add_virt_timer: timer pending\n");
return -EBUSY;
}
timer->cpu = get_cpu();
return 0;
}
/*
* add_virt_timer - add an oneshot virtual CPU timer
*/
void add_virt_timer(void *new)
{
struct vtimer_list *timer;
timer = (struct vtimer_list *)new;
if (prepare_vtimer(timer) < 0)
return;
timer->interval = 0;
internal_add_vtimer(timer);
}
EXPORT_SYMBOL(add_virt_timer);
/*
* add_virt_timer_int - add an interval virtual CPU timer
*/
void add_virt_timer_periodic(void *new)
{
struct vtimer_list *timer;
timer = (struct vtimer_list *)new;
if (prepare_vtimer(timer) < 0)
return;
timer->interval = timer->expires;
internal_add_vtimer(timer);
}
EXPORT_SYMBOL(add_virt_timer_periodic);
/*
* If we change a pending timer the function must be called on the CPU
* where the timer is running on, e.g. by smp_call_function_on()
*
* The original mod_timer adds the timer if it is not pending. For compatibility
* we do the same. The timer will be added on the current CPU as a oneshot timer.
*
* returns whether it has modified a pending timer (1) or not (0)
*/
int mod_virt_timer(struct vtimer_list *timer, __u64 expires)
{
struct vtimer_queue *vt_list;
unsigned long flags;
int cpu;
if (check_vtimer(timer) || !timer->function) {
printk("mod_virt_timer: uninitialized timer\n");
return -EINVAL;
}
if (!expires || expires > VTIMER_MAX_SLICE) {
printk("mod_virt_timer: invalid expire range\n");
return -EINVAL;
}
/*
* This is a common optimization triggered by the
* networking code - if the timer is re-modified
* to be the same thing then just return:
*/
if (timer->expires == expires && vtimer_pending(timer))
return 1;
cpu = get_cpu();
vt_list = &per_cpu(virt_cpu_timer, cpu);
/* disable interrupts before test if timer is pending */
spin_lock_irqsave(&vt_list->lock, flags);
/* if timer isn't pending add it on the current CPU */
if (!vtimer_pending(timer)) {
spin_unlock_irqrestore(&vt_list->lock, flags);
/* we do not activate an interval timer with mod_virt_timer */
timer->interval = 0;
timer->expires = expires;
timer->cpu = cpu;
internal_add_vtimer(timer);
return 0;
}
/* check if we run on the right CPU */
if (timer->cpu != cpu) {
printk("mod_virt_timer: running on wrong CPU, check your code\n");
spin_unlock_irqrestore(&vt_list->lock, flags);
put_cpu();
return -EINVAL;
}
list_del_init(&timer->entry);
timer->expires = expires;
/* also change the interval if we have an interval timer */
if (timer->interval)
timer->interval = expires;
/* the timer can't expire anymore so we can release the lock */
spin_unlock_irqrestore(&vt_list->lock, flags);
internal_add_vtimer(timer);
return 1;
}
EXPORT_SYMBOL(mod_virt_timer);
/*
* delete a virtual timer
*
* returns whether the deleted timer was pending (1) or not (0)
*/
int del_virt_timer(struct vtimer_list *timer)
{
unsigned long flags;
struct vtimer_queue *vt_list;
if (check_vtimer(timer)) {
printk("del_virt_timer: timer not initialized\n");
return -EINVAL;
}
/* check if timer is pending */
if (!vtimer_pending(timer))
return 0;
vt_list = &per_cpu(virt_cpu_timer, timer->cpu);
spin_lock_irqsave(&vt_list->lock, flags);
/* we don't interrupt a running timer, just let it expire! */
list_del_init(&timer->entry);
/* last timer removed */
if (list_empty(&vt_list->list)) {
vt_list->to_expire = 0;
vt_list->offset = 0;
}
spin_unlock_irqrestore(&vt_list->lock, flags);
return 1;
}
EXPORT_SYMBOL(del_virt_timer);
/*
* Start the virtual CPU timer on the current CPU.
*/
void init_cpu_vtimer(void)
{
struct vtimer_queue *vt_list;
unsigned long cr0;
__u64 timer;
/* kick the virtual timer */
timer = VTIMER_MAX_SLICE;
asm volatile ("SPT %0" : : "m" (timer));
__ctl_store(cr0, 0, 0);
cr0 |= 0x400;
__ctl_load(cr0, 0, 0);
vt_list = &per_cpu(virt_cpu_timer, smp_processor_id());
INIT_LIST_HEAD(&vt_list->list);
spin_lock_init(&vt_list->lock);
vt_list->to_expire = 0;
vt_list->offset = 0;
vt_list->idle = 0;
}
static int vtimer_idle_notify(struct notifier_block *self,
unsigned long action, void *hcpu)
{
switch (action) {
case CPU_IDLE:
stop_cpu_timer();
break;
case CPU_NOT_IDLE:
start_cpu_timer();
break;
}
return NOTIFY_OK;
}
static struct notifier_block vtimer_idle_nb = {
.notifier_call = vtimer_idle_notify,
};
void __init vtime_init(void)
{
/* request the cpu timer external interrupt */
if (register_early_external_interrupt(0x1005, do_cpu_timer_interrupt,
&ext_int_info_timer) != 0)
panic("Couldn't request external interrupt 0x1005");
if (register_idle_notifier(&vtimer_idle_nb))
panic("Couldn't register idle notifier");
init_cpu_vtimer();
}
include/asm-s390/processor.h
View file @ 1ef8b835
......@@ -319,6 +319,16 @@ static inline void disabled_wait(unsigned long code)
#endif /* __s390x__ */
}
/*
* CPU idle notifier chain.
*/
#define CPU_IDLE 0
#define CPU_NOT_IDLE 1
struct notifier_block;
int register_idle_notifier(struct notifier_block *nb);
int unregister_idle_notifier(struct notifier_block *nb);
#endif
#endif /* __ASM_S390_PROCESSOR_H */
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7