Commit ee3c72a1 authored by Christoph Lameter's avatar Christoph Lameter Committed by Linus Torvalds

SLUB: Avoid touching page struct when freeing to per cpu slab

Set c->node to -1 if we allocate from a debug slab instead for SlabDebug
which requires access the page struct cacheline.
Signed-off-by: default avatarChristoph Lameter <clameter@sgi.com>
Tested-by: default avatarAlexey Dobriyan <adobriyan@sw.ru>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent b3fba8da
...@@ -1537,6 +1537,7 @@ static void *__slab_alloc(struct kmem_cache *s, ...@@ -1537,6 +1537,7 @@ static void *__slab_alloc(struct kmem_cache *s,
c->page->inuse++; c->page->inuse++;
c->page->freelist = object[c->offset]; c->page->freelist = object[c->offset];
c->node = -1;
slab_unlock(c->page); slab_unlock(c->page);
return object; return object;
} }
...@@ -1560,8 +1561,7 @@ static void __always_inline *slab_alloc(struct kmem_cache *s, ...@@ -1560,8 +1561,7 @@ static void __always_inline *slab_alloc(struct kmem_cache *s,
local_irq_save(flags); local_irq_save(flags);
c = get_cpu_slab(s, smp_processor_id()); c = get_cpu_slab(s, smp_processor_id());
if (unlikely(!c->page || !c->freelist || if (unlikely(!c->freelist || !node_match(c, node)))
!node_match(c, node)))
object = __slab_alloc(s, gfpflags, node, addr, c); object = __slab_alloc(s, gfpflags, node, addr, c);
...@@ -1670,7 +1670,7 @@ static void __always_inline slab_free(struct kmem_cache *s, ...@@ -1670,7 +1670,7 @@ static void __always_inline slab_free(struct kmem_cache *s,
local_irq_save(flags); local_irq_save(flags);
debug_check_no_locks_freed(object, s->objsize); debug_check_no_locks_freed(object, s->objsize);
c = get_cpu_slab(s, smp_processor_id()); c = get_cpu_slab(s, smp_processor_id());
if (likely(page == c->page && !SlabDebug(page))) { if (likely(page == c->page && c->node >= 0)) {
object[c->offset] = c->freelist; object[c->offset] = c->freelist;
c->freelist = object; c->freelist = object;
} else } else
...@@ -3250,12 +3250,16 @@ static unsigned long slab_objects(struct kmem_cache *s, ...@@ -3250,12 +3250,16 @@ static unsigned long slab_objects(struct kmem_cache *s,
for_each_possible_cpu(cpu) { for_each_possible_cpu(cpu) {
struct page *page; struct page *page;
int node;
struct kmem_cache_cpu *c = get_cpu_slab(s, cpu); struct kmem_cache_cpu *c = get_cpu_slab(s, cpu);
if (!c) if (!c)
continue; continue;
page = c->page; page = c->page;
node = c->node;
if (node < 0)
continue;
if (page) { if (page) {
if (flags & SO_CPU) { if (flags & SO_CPU) {
int x = 0; int x = 0;
...@@ -3265,9 +3269,9 @@ static unsigned long slab_objects(struct kmem_cache *s, ...@@ -3265,9 +3269,9 @@ static unsigned long slab_objects(struct kmem_cache *s,
else else
x = 1; x = 1;
total += x; total += x;
nodes[c->node] += x; nodes[node] += x;
} }
per_cpu[c->node]++; per_cpu[node]++;
} }
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment