Commit 0855040e authored by Andi Kleen's avatar Andi Kleen Committed by Linus Torvalds

[PATCH] x86-64: copy ioperm bitmaps more efficiently at context switch

Copy ioperm bitmaps more efficiently at context switch time.  Only copy
upto the highest used port.

Originally from Ingo Molnar
Signed-off-by: default avatarAndi Kleen <ak@muc.de>
Signed-off-by: default avatarAndrew Morton <akpm@osdl.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@osdl.org>
parent ed5bb129
...@@ -28,12 +28,12 @@ static void set_bitmap(unsigned long *bitmap, unsigned int base, unsigned int ex ...@@ -28,12 +28,12 @@ static void set_bitmap(unsigned long *bitmap, unsigned int base, unsigned int ex
clear_bit(i, bitmap); clear_bit(i, bitmap);
} }
/* /*
* this changes the io permissions bitmap in the current task. * this changes the io permissions bitmap in the current task.
*/ */
asmlinkage long sys_ioperm(unsigned long from, unsigned long num, int turn_on) asmlinkage long sys_ioperm(unsigned long from, unsigned long num, int turn_on)
{ {
unsigned int i, max_long, bytes, bytes_updated;
struct thread_struct * t = &current->thread; struct thread_struct * t = &current->thread;
struct tss_struct * tss; struct tss_struct * tss;
unsigned long *bitmap; unsigned long *bitmap;
...@@ -59,16 +59,34 @@ asmlinkage long sys_ioperm(unsigned long from, unsigned long num, int turn_on) ...@@ -59,16 +59,34 @@ asmlinkage long sys_ioperm(unsigned long from, unsigned long num, int turn_on)
/* /*
* do it in the per-thread copy and in the TSS ... * do it in the per-thread copy and in the TSS ...
*
* Disable preemption via get_cpu() - we must not switch away
* because the ->io_bitmap_max value must match the bitmap
* contents:
*/ */
set_bitmap(t->io_bitmap_ptr, from, num, !turn_on);
tss = &per_cpu(init_tss, get_cpu()); tss = &per_cpu(init_tss, get_cpu());
if (tss->io_bitmap_base == IO_BITMAP_OFFSET) { /* already active? */
set_bitmap(tss->io_bitmap, from, num, !turn_on); set_bitmap(t->io_bitmap_ptr, from, num, !turn_on);
} else {
memcpy(tss->io_bitmap, t->io_bitmap_ptr, IO_BITMAP_BYTES); /*
tss->io_bitmap_base = IO_BITMAP_OFFSET; /* Activate it in the TSS */ * Search for a (possibly new) maximum. This is simple and stupid,
} * to keep it obviously correct:
*/
max_long = 0;
for (i = 0; i < IO_BITMAP_LONGS; i++)
if (t->io_bitmap_ptr[i] != ~0UL)
max_long = i;
bytes = (max_long + 1) * sizeof(long);
bytes_updated = max(bytes, t->io_bitmap_max);
t->io_bitmap_max = bytes;
/* Update the TSS: */
memcpy(tss->io_bitmap, t->io_bitmap_ptr, bytes_updated);
put_cpu(); put_cpu();
return 0; return 0;
} }
......
...@@ -250,11 +250,17 @@ void show_regs(struct pt_regs *regs) ...@@ -250,11 +250,17 @@ void show_regs(struct pt_regs *regs)
void exit_thread(void) void exit_thread(void)
{ {
struct task_struct *me = current; struct task_struct *me = current;
struct thread_struct *t = &me->thread;
if (me->thread.io_bitmap_ptr) { if (me->thread.io_bitmap_ptr) {
struct tss_struct *tss = &per_cpu(init_tss,get_cpu()); struct tss_struct *tss = &per_cpu(init_tss, get_cpu());
kfree(me->thread.io_bitmap_ptr);
me->thread.io_bitmap_ptr = NULL; kfree(t->io_bitmap_ptr);
tss->io_bitmap_base = INVALID_IO_BITMAP_OFFSET; t->io_bitmap_ptr = NULL;
/*
* Careful, clear this in the TSS too:
*/
memset(tss->io_bitmap, 0xff, t->io_bitmap_max);
t->io_bitmap_max = 0;
put_cpu(); put_cpu();
} }
} }
...@@ -362,8 +368,10 @@ int copy_thread(int nr, unsigned long clone_flags, unsigned long rsp, ...@@ -362,8 +368,10 @@ int copy_thread(int nr, unsigned long clone_flags, unsigned long rsp,
if (unlikely(me->thread.io_bitmap_ptr != NULL)) { if (unlikely(me->thread.io_bitmap_ptr != NULL)) {
p->thread.io_bitmap_ptr = kmalloc(IO_BITMAP_BYTES, GFP_KERNEL); p->thread.io_bitmap_ptr = kmalloc(IO_BITMAP_BYTES, GFP_KERNEL);
if (!p->thread.io_bitmap_ptr) if (!p->thread.io_bitmap_ptr) {
p->thread.io_bitmap_max = 0;
return -ENOMEM; return -ENOMEM;
}
memcpy(p->thread.io_bitmap_ptr, me->thread.io_bitmap_ptr, IO_BITMAP_BYTES); memcpy(p->thread.io_bitmap_ptr, me->thread.io_bitmap_ptr, IO_BITMAP_BYTES);
} }
...@@ -382,8 +390,10 @@ int copy_thread(int nr, unsigned long clone_flags, unsigned long rsp, ...@@ -382,8 +390,10 @@ int copy_thread(int nr, unsigned long clone_flags, unsigned long rsp,
} }
err = 0; err = 0;
out: out:
if (err && p->thread.io_bitmap_ptr) if (err && p->thread.io_bitmap_ptr) {
kfree(p->thread.io_bitmap_ptr); kfree(p->thread.io_bitmap_ptr);
p->thread.io_bitmap_max = 0;
}
return err; return err;
} }
...@@ -490,22 +500,18 @@ struct task_struct *__switch_to(struct task_struct *prev_p, struct task_struct * ...@@ -490,22 +500,18 @@ struct task_struct *__switch_to(struct task_struct *prev_p, struct task_struct *
* Handle the IO bitmap * Handle the IO bitmap
*/ */
if (unlikely(prev->io_bitmap_ptr || next->io_bitmap_ptr)) { if (unlikely(prev->io_bitmap_ptr || next->io_bitmap_ptr)) {
if (next->io_bitmap_ptr) { if (next->io_bitmap_ptr)
/* /*
* 2 cachelines copy ... not good, but not that * Copy the relevant range of the IO bitmap.
* bad either. Anyone got something better? * Normally this is 128 bytes or less:
* This only affects processes which use ioperm(). */
*/ memcpy(tss->io_bitmap, next->io_bitmap_ptr,
memcpy(tss->io_bitmap, next->io_bitmap_ptr, IO_BITMAP_BYTES); max(prev->io_bitmap_max, next->io_bitmap_max));
tss->io_bitmap_base = IO_BITMAP_OFFSET; else {
} else {
/* /*
* a bitmap offset pointing outside of the TSS limit * Clear any possible leftover bits:
* causes a nicely controllable SIGSEGV if a process
* tries to use a port IO instruction. The first
* sys_ioperm() call sets up the bitmap properly.
*/ */
tss->io_bitmap_base = INVALID_IO_BITMAP_OFFSET; memset(tss->io_bitmap, 0xff, prev->io_bitmap_max);
} }
} }
......
...@@ -241,6 +241,7 @@ void __init cpu_init (void) ...@@ -241,6 +241,7 @@ void __init cpu_init (void)
unsigned long v; unsigned long v;
char *estacks = NULL; char *estacks = NULL;
struct task_struct *me; struct task_struct *me;
int i;
/* CPU 0 is initialised in head64.c */ /* CPU 0 is initialised in head64.c */
if (cpu != 0) { if (cpu != 0) {
...@@ -304,12 +305,13 @@ void __init cpu_init (void) ...@@ -304,12 +305,13 @@ void __init cpu_init (void)
t->ist[v] = (unsigned long)estacks; t->ist[v] = (unsigned long)estacks;
} }
t->io_bitmap_base = INVALID_IO_BITMAP_OFFSET; t->io_bitmap_base = offsetof(struct tss_struct, io_bitmap);
/* /*
* This is required because the CPU will access up to * <= is required because the CPU will access up to
* 8 bits beyond the end of the IO permission bitmap. * 8 bits beyond the end of the IO permission bitmap.
*/ */
t->io_bitmap[IO_BITMAP_LONGS] = ~0UL; for (i = 0; i <= IO_BITMAP_LONGS; i++)
t->io_bitmap[i] = ~0UL;
atomic_inc(&init_mm.mm_count); atomic_inc(&init_mm.mm_count);
me->active_mm = &init_mm; me->active_mm = &init_mm;
......
...@@ -254,6 +254,7 @@ struct thread_struct { ...@@ -254,6 +254,7 @@ struct thread_struct {
switch faster for a limited number of ioperm using tasks. -AK */ switch faster for a limited number of ioperm using tasks. -AK */
int ioperm; int ioperm;
unsigned long *io_bitmap_ptr; unsigned long *io_bitmap_ptr;
unsigned io_bitmap_max;
/* cached TLS descriptors. */ /* cached TLS descriptors. */
u64 tls_array[GDT_ENTRY_TLS_ENTRIES]; u64 tls_array[GDT_ENTRY_TLS_ENTRIES];
} __attribute__((aligned(16))); } __attribute__((aligned(16)));
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment