Commit c11abbba authored by Linus Torvalds's avatar Linus Torvalds

Merge branch 'slub/lockless' of git://git.kernel.org/pub/scm/linux/kernel/git/penberg/slab-2.6

* 'slub/lockless' of git://git.kernel.org/pub/scm/linux/kernel/git/penberg/slab-2.6: (21 commits)
  slub: When allocating a new slab also prep the first object
  slub: disable interrupts in cmpxchg_double_slab when falling back to pagelock
  Avoid duplicate _count variables in page_struct
  Revert "SLUB: Fix build breakage in linux/mm_types.h"
  SLUB: Fix build breakage in linux/mm_types.h
  slub: slabinfo update for cmpxchg handling
  slub: Not necessary to check for empty slab on load_freelist
  slub: fast release on full slab
  slub: Add statistics for the case that the current slab does not match the node
  slub: Get rid of the another_slab label
  slub: Avoid disabling interrupts in free slowpath
  slub: Disable interrupts in free_debug processing
  slub: Invert locking and avoid slab lock
  slub: Rework allocator fastpaths
  slub: Pass kmem_cache struct to lock and freeze slab
  slub: explicit list_lock taking
  slub: Add cmpxchg_double_slab()
  mm: Rearrange struct page
  slub: Move page->frozen handling near where the page->freelist handling occurs
  slub: Do not use frozen page flag but a bit in the page counters
  ...
parents 1d3fe4a7 9e577e8b
...@@ -30,23 +30,61 @@ struct address_space; ...@@ -30,23 +30,61 @@ struct address_space;
* moment. Note that we have no way to track which tasks are using * moment. Note that we have no way to track which tasks are using
* a page, though if it is a pagecache page, rmap structures can tell us * a page, though if it is a pagecache page, rmap structures can tell us
* who is mapping it. * who is mapping it.
*
* The objects in struct page are organized in double word blocks in
* order to allows us to use atomic double word operations on portions
* of struct page. That is currently only used by slub but the arrangement
* allows the use of atomic double word operations on the flags/mapping
* and lru list pointers also.
*/ */
struct page { struct page {
/* First double word block */
unsigned long flags; /* Atomic flags, some possibly unsigned long flags; /* Atomic flags, some possibly
* updated asynchronously */ * updated asynchronously */
atomic_t _count; /* Usage count, see below. */ struct address_space *mapping; /* If low bit clear, points to
* inode address_space, or NULL.
* If page mapped as anonymous
* memory, low bit is set, and
* it points to anon_vma object:
* see PAGE_MAPPING_ANON below.
*/
/* Second double word */
struct {
union {
pgoff_t index; /* Our offset within mapping. */
void *freelist; /* slub first free object */
};
union {
/* Used for cmpxchg_double in slub */
unsigned long counters;
struct {
union { union {
atomic_t _mapcount; /* Count of ptes mapped in mms, atomic_t _mapcount; /* Count of ptes mapped in mms,
* to show when page is mapped * to show when page is mapped
* & limit reverse map searches. * & limit reverse map searches.
*/ */
struct { /* SLUB */
u16 inuse; struct {
u16 objects; unsigned inuse:16;
unsigned objects:15;
unsigned frozen:1;
};
};
atomic_t _count; /* Usage count, see below. */
};
}; };
}; };
/* Third double word block */
struct list_head lru; /* Pageout list, eg. active_list
* protected by zone->lru_lock !
*/
/* Remainder is not double word aligned */
union { union {
struct {
unsigned long private; /* Mapping-private opaque data: unsigned long private; /* Mapping-private opaque data:
* usually used for buffer_heads * usually used for buffer_heads
* if PagePrivate set; used for * if PagePrivate set; used for
...@@ -54,27 +92,13 @@ struct page { ...@@ -54,27 +92,13 @@ struct page {
* indicates order in the buddy * indicates order in the buddy
* system if PG_buddy is set. * system if PG_buddy is set.
*/ */
struct address_space *mapping; /* If low bit clear, points to
* inode address_space, or NULL.
* If page mapped as anonymous
* memory, low bit is set, and
* it points to anon_vma object:
* see PAGE_MAPPING_ANON below.
*/
};
#if USE_SPLIT_PTLOCKS #if USE_SPLIT_PTLOCKS
spinlock_t ptl; spinlock_t ptl;
#endif #endif
struct kmem_cache *slab; /* SLUB: Pointer to slab */ struct kmem_cache *slab; /* SLUB: Pointer to slab */
struct page *first_page; /* Compound tail pages */ struct page *first_page; /* Compound tail pages */
}; };
union {
pgoff_t index; /* Our offset within mapping. */
void *freelist; /* SLUB: freelist req. slab lock */
};
struct list_head lru; /* Pageout list, eg. active_list
* protected by zone->lru_lock !
*/
/* /*
* On machines where all RAM is mapped into kernel address space, * On machines where all RAM is mapped into kernel address space,
* we can simply calculate the virtual address. On machines with * we can simply calculate the virtual address. On machines with
...@@ -100,7 +124,16 @@ struct page { ...@@ -100,7 +124,16 @@ struct page {
*/ */
void *shadow; void *shadow;
#endif #endif
}; }
/*
* If another subsystem starts using the double word pairing for atomic
* operations on struct page then it must change the #if to ensure
* proper alignment of the page struct.
*/
#if defined(CONFIG_SLUB) && defined(CONFIG_CMPXCHG_LOCAL)
__attribute__((__aligned__(2*sizeof(unsigned long))))
#endif
;
typedef unsigned long __nocast vm_flags_t; typedef unsigned long __nocast vm_flags_t;
......
...@@ -124,9 +124,6 @@ enum pageflags { ...@@ -124,9 +124,6 @@ enum pageflags {
/* SLOB */ /* SLOB */
PG_slob_free = PG_private, PG_slob_free = PG_private,
/* SLUB */
PG_slub_frozen = PG_active,
}; };
#ifndef __GENERATING_BOUNDS_H #ifndef __GENERATING_BOUNDS_H
...@@ -212,8 +209,6 @@ PAGEFLAG(SwapBacked, swapbacked) __CLEARPAGEFLAG(SwapBacked, swapbacked) ...@@ -212,8 +209,6 @@ PAGEFLAG(SwapBacked, swapbacked) __CLEARPAGEFLAG(SwapBacked, swapbacked)
__PAGEFLAG(SlobFree, slob_free) __PAGEFLAG(SlobFree, slob_free)
__PAGEFLAG(SlubFrozen, slub_frozen)
/* /*
* Private page markings that may be used by the filesystem that owns the page * Private page markings that may be used by the filesystem that owns the page
* for its own purposes. * for its own purposes.
......
...@@ -24,6 +24,7 @@ enum stat_item { ...@@ -24,6 +24,7 @@ enum stat_item {
ALLOC_FROM_PARTIAL, /* Cpu slab acquired from partial list */ ALLOC_FROM_PARTIAL, /* Cpu slab acquired from partial list */
ALLOC_SLAB, /* Cpu slab acquired from page allocator */ ALLOC_SLAB, /* Cpu slab acquired from page allocator */
ALLOC_REFILL, /* Refill cpu slab from slab freelist */ ALLOC_REFILL, /* Refill cpu slab from slab freelist */
ALLOC_NODE_MISMATCH, /* Switching cpu slab */
FREE_SLAB, /* Slab freed to the page allocator */ FREE_SLAB, /* Slab freed to the page allocator */
CPUSLAB_FLUSH, /* Abandoning of the cpu slab */ CPUSLAB_FLUSH, /* Abandoning of the cpu slab */
DEACTIVATE_FULL, /* Cpu slab was full when deactivated */ DEACTIVATE_FULL, /* Cpu slab was full when deactivated */
...@@ -31,8 +32,10 @@ enum stat_item { ...@@ -31,8 +32,10 @@ enum stat_item {
DEACTIVATE_TO_HEAD, /* Cpu slab was moved to the head of partials */ DEACTIVATE_TO_HEAD, /* Cpu slab was moved to the head of partials */
DEACTIVATE_TO_TAIL, /* Cpu slab was moved to the tail of partials */ DEACTIVATE_TO_TAIL, /* Cpu slab was moved to the tail of partials */
DEACTIVATE_REMOTE_FREES,/* Slab contained remotely freed objects */ DEACTIVATE_REMOTE_FREES,/* Slab contained remotely freed objects */
DEACTIVATE_BYPASS, /* Implicit deactivation */
ORDER_FALLBACK, /* Number of times fallback was necessary */ ORDER_FALLBACK, /* Number of times fallback was necessary */
CMPXCHG_DOUBLE_CPU_FAIL,/* Failure of this_cpu_cmpxchg_double */ CMPXCHG_DOUBLE_CPU_FAIL,/* Failure of this_cpu_cmpxchg_double */
CMPXCHG_DOUBLE_FAIL, /* Number of times that cmpxchg double did not match */
NR_SLUB_STAT_ITEMS }; NR_SLUB_STAT_ITEMS };
struct kmem_cache_cpu { struct kmem_cache_cpu {
......
...@@ -2,10 +2,11 @@ ...@@ -2,10 +2,11 @@
* SLUB: A slab allocator that limits cache line use instead of queuing * SLUB: A slab allocator that limits cache line use instead of queuing
* objects in per cpu and per node lists. * objects in per cpu and per node lists.
* *
* The allocator synchronizes using per slab locks and only * The allocator synchronizes using per slab locks or atomic operatios
* uses a centralized lock to manage a pool of partial slabs. * and only uses a centralized lock to manage a pool of partial slabs.
* *
* (C) 2007 SGI, Christoph Lameter * (C) 2007 SGI, Christoph Lameter
* (C) 2011 Linux Foundation, Christoph Lameter
*/ */
#include <linux/mm.h> #include <linux/mm.h>
...@@ -33,15 +34,27 @@ ...@@ -33,15 +34,27 @@
/* /*
* Lock order: * Lock order:
* 1. slab_lock(page) * 1. slub_lock (Global Semaphore)
* 2. slab->list_lock * 2. node->list_lock
* 3. slab_lock(page) (Only on some arches and for debugging)
* *
* The slab_lock protects operations on the object of a particular * slub_lock
* slab and its metadata in the page struct. If the slab lock *
* has been taken then no allocations nor frees can be performed * The role of the slub_lock is to protect the list of all the slabs
* on the objects in the slab nor can the slab be added or removed * and to synchronize major metadata changes to slab cache structures.
* from the partial or full lists since this would mean modifying *
* the page_struct of the slab. * The slab_lock is only used for debugging and on arches that do not
* have the ability to do a cmpxchg_double. It only protects the second
* double word in the page struct. Meaning
* A. page->freelist -> List of object free in a page
* B. page->counters -> Counters of objects
* C. page->frozen -> frozen state
*
* If a slab is frozen then it is exempt from list management. It is not
* on any list. The processor that froze the slab is the one who can
* perform list operations on the page. Other processors may put objects
* onto the freelist but the processor that froze the slab is the only
* one that can retrieve the objects from the page's freelist.
* *
* The list_lock protects the partial and full list on each node and * The list_lock protects the partial and full list on each node and
* the partial slab counter. If taken then no new slabs may be added or * the partial slab counter. If taken then no new slabs may be added or
...@@ -54,20 +67,6 @@ ...@@ -54,20 +67,6 @@
* slabs, operations can continue without any centralized lock. F.e. * slabs, operations can continue without any centralized lock. F.e.
* allocating a long series of objects that fill up slabs does not require * allocating a long series of objects that fill up slabs does not require
* the list lock. * the list lock.
*
* The lock order is sometimes inverted when we are trying to get a slab
* off a list. We take the list_lock and then look for a page on the list
* to use. While we do that objects in the slabs may be freed. We can
* only operate on the slab if we have also taken the slab_lock. So we use
* a slab_trylock() on the slab. If trylock was successful then no frees
* can occur anymore and we can use the slab for allocations etc. If the
* slab_trylock() does not succeed then frees are in progress in the slab and
* we must stay away from it for a while since we may cause a bouncing
* cacheline if we try to acquire the lock. So go onto the next slab.
* If all pages are busy then we may allocate a new slab instead of reusing
* a partial slab. A new slab has no one operating on it and thus there is
* no danger of cacheline contention.
*
* Interrupts are disabled during allocation and deallocation in order to * Interrupts are disabled during allocation and deallocation in order to
* make the slab allocator safe to use in the context of an irq. In addition * make the slab allocator safe to use in the context of an irq. In addition
* interrupts are disabled to ensure that the processor does not change * interrupts are disabled to ensure that the processor does not change
...@@ -132,6 +131,9 @@ static inline int kmem_cache_debug(struct kmem_cache *s) ...@@ -132,6 +131,9 @@ static inline int kmem_cache_debug(struct kmem_cache *s)
/* Enable to test recovery from slab corruption on boot */ /* Enable to test recovery from slab corruption on boot */
#undef SLUB_RESILIENCY_TEST #undef SLUB_RESILIENCY_TEST
/* Enable to log cmpxchg failures */
#undef SLUB_DEBUG_CMPXCHG
/* /*
* Mininum number of partial slabs. These will be left on the partial * Mininum number of partial slabs. These will be left on the partial
* lists even if they are empty. kmem_cache_shrink may reclaim them. * lists even if they are empty. kmem_cache_shrink may reclaim them.
...@@ -167,10 +169,11 @@ static inline int kmem_cache_debug(struct kmem_cache *s) ...@@ -167,10 +169,11 @@ static inline int kmem_cache_debug(struct kmem_cache *s)
#define OO_SHIFT 16 #define OO_SHIFT 16
#define OO_MASK ((1 << OO_SHIFT) - 1) #define OO_MASK ((1 << OO_SHIFT) - 1)
#define MAX_OBJS_PER_PAGE 65535 /* since page.objects is u16 */ #define MAX_OBJS_PER_PAGE 32767 /* since page.objects is u15 */
/* Internal SLUB flags */ /* Internal SLUB flags */
#define __OBJECT_POISON 0x80000000UL /* Poison object */ #define __OBJECT_POISON 0x80000000UL /* Poison object */
#define __CMPXCHG_DOUBLE 0x40000000UL /* Use cmpxchg_double */
static int kmem_size = sizeof(struct kmem_cache); static int kmem_size = sizeof(struct kmem_cache);
...@@ -343,11 +346,99 @@ static inline int oo_objects(struct kmem_cache_order_objects x) ...@@ -343,11 +346,99 @@ static inline int oo_objects(struct kmem_cache_order_objects x)
return x.x & OO_MASK; return x.x & OO_MASK;
} }
/*
* Per slab locking using the pagelock
*/
static __always_inline void slab_lock(struct page *page)
{
bit_spin_lock(PG_locked, &page->flags);
}
static __always_inline void slab_unlock(struct page *page)
{
__bit_spin_unlock(PG_locked, &page->flags);
}
/* Interrupts must be disabled (for the fallback code to work right) */
static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct page *page,
void *freelist_old, unsigned long counters_old,
void *freelist_new, unsigned long counters_new,
const char *n)
{
VM_BUG_ON(!irqs_disabled());
#ifdef CONFIG_CMPXCHG_DOUBLE
if (s->flags & __CMPXCHG_DOUBLE) {
if (cmpxchg_double(&page->freelist,
freelist_old, counters_old,
freelist_new, counters_new))
return 1;
} else
#endif
{
slab_lock(page);
if (page->freelist == freelist_old && page->counters == counters_old) {
page->freelist = freelist_new;
page->counters = counters_new;
slab_unlock(page);
return 1;
}
slab_unlock(page);
}
cpu_relax();
stat(s, CMPXCHG_DOUBLE_FAIL);
#ifdef SLUB_DEBUG_CMPXCHG
printk(KERN_INFO "%s %s: cmpxchg double redo ", n, s->name);
#endif
return 0;
}
static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct page *page,
void *freelist_old, unsigned long counters_old,
void *freelist_new, unsigned long counters_new,
const char *n)
{
#ifdef CONFIG_CMPXCHG_DOUBLE
if (s->flags & __CMPXCHG_DOUBLE) {
if (cmpxchg_double(&page->freelist,
freelist_old, counters_old,
freelist_new, counters_new))
return 1;
} else
#endif
{
unsigned long flags;
local_irq_save(flags);
slab_lock(page);
if (page->freelist == freelist_old && page->counters == counters_old) {
page->freelist = freelist_new;
page->counters = counters_new;
slab_unlock(page);
local_irq_restore(flags);
return 1;
}
slab_unlock(page);
local_irq_restore(flags);
}
cpu_relax();
stat(s, CMPXCHG_DOUBLE_FAIL);
#ifdef SLUB_DEBUG_CMPXCHG
printk(KERN_INFO "%s %s: cmpxchg double redo ", n, s->name);
#endif
return 0;
}
#ifdef CONFIG_SLUB_DEBUG #ifdef CONFIG_SLUB_DEBUG
/* /*
* Determine a map of object in use on a page. * Determine a map of object in use on a page.
* *
* Slab lock or node listlock must be held to guarantee that the page does * Node listlock must be held to guarantee that the page does
* not vanish from under us. * not vanish from under us.
*/ */
static void get_map(struct kmem_cache *s, struct page *page, unsigned long *map) static void get_map(struct kmem_cache *s, struct page *page, unsigned long *map)
...@@ -838,10 +929,11 @@ static int check_slab(struct kmem_cache *s, struct page *page) ...@@ -838,10 +929,11 @@ static int check_slab(struct kmem_cache *s, struct page *page)
static int on_freelist(struct kmem_cache *s, struct page *page, void *search) static int on_freelist(struct kmem_cache *s, struct page *page, void *search)
{ {
int nr = 0; int nr = 0;
void *fp = page->freelist; void *fp;
void *object = NULL; void *object = NULL;
unsigned long max_objects; unsigned long max_objects;
fp = page->freelist;
while (fp && nr <= page->objects) { while (fp && nr <= page->objects) {
if (fp == search) if (fp == search)
return 1; return 1;
...@@ -946,26 +1038,27 @@ static inline void slab_free_hook(struct kmem_cache *s, void *x) ...@@ -946,26 +1038,27 @@ static inline void slab_free_hook(struct kmem_cache *s, void *x)
/* /*
* Tracking of fully allocated slabs for debugging purposes. * Tracking of fully allocated slabs for debugging purposes.
*
* list_lock must be held.
*/ */
static void add_full(struct kmem_cache_node *n, struct page *page) static void add_full(struct kmem_cache *s,
struct kmem_cache_node *n, struct page *page)
{ {
spin_lock(&n->list_lock); if (!(s->flags & SLAB_STORE_USER))
return;
list_add(&page->lru, &n->full); list_add(&page->lru, &n->full);
spin_unlock(&n->list_lock);
} }
/*
* list_lock must be held.
*/
static void remove_full(struct kmem_cache *s, struct page *page) static void remove_full(struct kmem_cache *s, struct page *page)
{ {
struct kmem_cache_node *n;
if (!(s->flags & SLAB_STORE_USER)) if (!(s->flags & SLAB_STORE_USER))
return; return;
n = get_node(s, page_to_nid(page));
spin_lock(&n->list_lock);
list_del(&page->lru); list_del(&page->lru);
spin_unlock(&n->list_lock);
} }
/* Tracking of the number of slabs for debugging purposes */ /* Tracking of the number of slabs for debugging purposes */
...@@ -1021,11 +1114,6 @@ static noinline int alloc_debug_processing(struct kmem_cache *s, struct page *pa ...@@ -1021,11 +1114,6 @@ static noinline int alloc_debug_processing(struct kmem_cache *s, struct page *pa
if (!check_slab(s, page)) if (!check_slab(s, page))
goto bad; goto bad;
if (!on_freelist(s, page, object)) {
object_err(s, page, object, "Object already allocated");
goto bad;
}
if (!check_valid_pointer(s, page, object)) { if (!check_valid_pointer(s, page, object)) {
object_err(s, page, object, "Freelist Pointer check fails"); object_err(s, page, object, "Freelist Pointer check fails");
goto bad; goto bad;
...@@ -1058,6 +1146,12 @@ static noinline int alloc_debug_processing(struct kmem_cache *s, struct page *pa ...@@ -1058,6 +1146,12 @@ static noinline int alloc_debug_processing(struct kmem_cache *s, struct page *pa
static noinline int free_debug_processing(struct kmem_cache *s, static noinline int free_debug_processing(struct kmem_cache *s,
struct page *page, void *object, unsigned long addr) struct page *page, void *object, unsigned long addr)
{ {
unsigned long flags;
int rc = 0;
local_irq_save(flags);
slab_lock(page);
if (!check_slab(s, page)) if (!check_slab(s, page))
goto fail; goto fail;
...@@ -1072,7 +1166,7 @@ static noinline int free_debug_processing(struct kmem_cache *s, ...@@ -1072,7 +1166,7 @@ static noinline int free_debug_processing(struct kmem_cache *s,
} }
if (!check_object(s, page, object, SLUB_RED_ACTIVE)) if (!check_object(s, page, object, SLUB_RED_ACTIVE))
return 0; goto out;
if (unlikely(s != page->slab)) { if (unlikely(s != page->slab)) {
if (!PageSlab(page)) { if (!PageSlab(page)) {
...@@ -1089,18 +1183,19 @@ static noinline int free_debug_processing(struct kmem_cache *s, ...@@ -1089,18 +1183,19 @@ static noinline int free_debug_processing(struct kmem_cache *s,
goto fail; goto fail;
} }
/* Special debug activities for freeing objects */
if (!PageSlubFrozen(page) && !page->freelist)
remove_full(s, page);
if (s->flags & SLAB_STORE_USER) if (s->flags & SLAB_STORE_USER)
set_track(s, object, TRACK_FREE, addr); set_track(s, object, TRACK_FREE, addr);
trace(s, page, object, 0); trace(s, page, object, 0);
init_object(s, object, SLUB_RED_INACTIVE); init_object(s, object, SLUB_RED_INACTIVE);
return 1; rc = 1;
out:
slab_unlock(page);
local_irq_restore(flags);
return rc;
fail: fail:
slab_fix(s, "Object at 0x%p not freed", object); slab_fix(s, "Object at 0x%p not freed", object);
return 0; goto out;
} }
static int __init setup_slub_debug(char *str) static int __init setup_slub_debug(char *str)
...@@ -1200,7 +1295,9 @@ static inline int slab_pad_check(struct kmem_cache *s, struct page *page) ...@@ -1200,7 +1295,9 @@ static inline int slab_pad_check(struct kmem_cache *s, struct page *page)
{ return 1; } { return 1; }
static inline int check_object(struct kmem_cache *s, struct page *page, static inline int check_object(struct kmem_cache *s, struct page *page,
void *object, u8 val) { return 1; } void *object, u8 val) { return 1; }
static inline void add_full(struct kmem_cache_node *n, struct page *page) {} static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n,
struct page *page) {}
static inline void remove_full(struct kmem_cache *s, struct page *page) {}
static inline unsigned long kmem_cache_flags(unsigned long objsize, static inline unsigned long kmem_cache_flags(unsigned long objsize,
unsigned long flags, const char *name, unsigned long flags, const char *name,
void (*ctor)(void *)) void (*ctor)(void *))
...@@ -1252,6 +1349,11 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) ...@@ -1252,6 +1349,11 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
struct kmem_cache_order_objects oo = s->oo; struct kmem_cache_order_objects oo = s->oo;
gfp_t alloc_gfp; gfp_t alloc_gfp;
flags &= gfp_allowed_mask;
if (flags & __GFP_WAIT)
local_irq_enable();
flags |= s->allocflags; flags |= s->allocflags;
/* /*
...@@ -1268,12 +1370,17 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) ...@@ -1268,12 +1370,17 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
* Try a lower order alloc if possible * Try a lower order alloc if possible
*/ */
page = alloc_slab_page(flags, node, oo); page = alloc_slab_page(flags, node, oo);
if (!page)
return NULL;
if (page)
stat(s, ORDER_FALLBACK); stat(s, ORDER_FALLBACK);
} }
if (flags & __GFP_WAIT)
local_irq_disable();
if (!page)
return NULL;
if (kmemcheck_enabled if (kmemcheck_enabled
&& !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) { && !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) {
int pages = 1 << oo_order(oo); int pages = 1 << oo_order(oo);
...@@ -1341,6 +1448,7 @@ static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node) ...@@ -1341,6 +1448,7 @@ static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
page->freelist = start; page->freelist = start;
page->inuse = 0; page->inuse = 0;
page->frozen = 1;
out: out:
return page; return page;
} }
...@@ -1418,77 +1526,87 @@ static void discard_slab(struct kmem_cache *s, struct page *page) ...@@ -1418,77 +1526,87 @@ static void discard_slab(struct kmem_cache *s, struct page *page)
} }
/* /*
* Per slab locking using the pagelock * Management of partially allocated slabs.
*/ *
static __always_inline void slab_lock(struct page *page) * list_lock must be held.
{
bit_spin_lock(PG_locked, &page->flags);
}
static __always_inline void slab_unlock(struct page *page)
{
__bit_spin_unlock(PG_locked, &page->flags);
}
static __always_inline int slab_trylock(struct page *page)
{
int rc = 1;
rc = bit_spin_trylock(PG_locked, &page->flags);
return rc;
}
/*
* Management of partially allocated slabs
*/ */
static void add_partial(struct kmem_cache_node *n, static inline void add_partial(struct kmem_cache_node *n,
struct page *page, int tail) struct page *page, int tail)
{ {
spin_lock(&n->list_lock);
n->nr_partial++; n->nr_partial++;
if (tail) if (tail)
list_add_tail(&page->lru, &n->partial); list_add_tail(&page->lru, &n->partial);
else else
list_add(&page->lru, &n->partial); list_add(&page->lru, &n->partial);
spin_unlock(&n->list_lock);
} }
static inline void __remove_partial(struct kmem_cache_node *n, /*
* list_lock must be held.
*/
static inline void remove_partial(struct kmem_cache_node *n,
struct page *page) struct page *page)
{ {
list_del(&page->lru); list_del(&page->lru);
n->nr_partial--; n->nr_partial--;
} }
static void remove_partial(struct kmem_cache *s, struct page *page)
{
struct kmem_cache_node *n = get_node(s, page_to_nid(page));
spin_lock(&n->list_lock);
__remove_partial(n, page);
spin_unlock(&n->list_lock);
}
/* /*
* Lock slab and remove from the partial list. * Lock slab, remove from the partial list and put the object into the
* per cpu freelist.
* *
* Must hold list_lock. * Must hold list_lock.
*/ */
static inline int lock_and_freeze_slab(struct kmem_cache_node *n, static inline int acquire_slab(struct kmem_cache *s,
struct page *page) struct kmem_cache_node *n, struct page *page)
{ {
if (slab_trylock(page)) { void *freelist;
__remove_partial(n, page); unsigned long counters;
__SetPageSlubFrozen(page); struct page new;
/*
* Zap the freelist and set the frozen bit.
* The old freelist is the list of objects for the
* per cpu allocation list.
*/
do {
freelist = page->freelist;
counters = page->counters;
new.counters = counters;
new.inuse = page->objects;
VM_BUG_ON(new.frozen);
new.frozen = 1;
} while (!__cmpxchg_double_slab(s, page,
freelist, counters,
NULL, new.counters,
"lock and freeze"));
remove_partial(n, page);
if (freelist) {
/* Populate the per cpu freelist */
this_cpu_write(s->cpu_slab->freelist, freelist);
this_cpu_write(s->cpu_slab->page, page);
this_cpu_write(s->cpu_slab->node, page_to_nid(page));
return 1; return 1;
} } else {
/*
* Slab page came from the wrong list. No object to allocate
* from. Put it onto the correct list and continue partial
* scan.
*/
printk(KERN_ERR "SLUB: %s : Page without available objects on"
" partial list\n", s->name);
return 0; return 0;
}
} }
/* /*
* Try to allocate a partial slab from a specific node. * Try to allocate a partial slab from a specific node.
*/ */
static struct page *get_partial_node(struct kmem_cache_node *n) static struct page *get_partial_node(struct kmem_cache *s,
struct kmem_cache_node *n)
{ {
struct page *page; struct page *page;
...@@ -1503,7 +1621,7 @@ static struct page *get_partial_node(struct kmem_cache_node *n) ...@@ -1503,7 +1621,7 @@ static struct page *get_partial_node(struct kmem_cache_node *n)
spin_lock(&n->list_lock); spin_lock(&n->list_lock);
list_for_each_entry(page, &n->partial, lru) list_for_each_entry(page, &n->partial, lru)
if (lock_and_freeze_slab(n, page)) if (acquire_slab(s, n, page))
goto out; goto out;
page = NULL; page = NULL;
out: out:
...@@ -1554,7 +1672,7 @@ static struct page *get_any_partial(struct kmem_cache *s, gfp_t flags) ...@@ -1554,7 +1672,7 @@ static struct page *get_any_partial(struct kmem_cache *s, gfp_t flags)
if (n && cpuset_zone_allowed_hardwall(zone, flags) && if (n && cpuset_zone_allowed_hardwall(zone, flags) &&
n->nr_partial > s->min_partial) { n->nr_partial > s->min_partial) {
page = get_partial_node(n); page = get_partial_node(s, n);
if (page) { if (page) {
put_mems_allowed(); put_mems_allowed();
return page; return page;
...@@ -1574,60 +1692,13 @@ static struct page *get_partial(struct kmem_cache *s, gfp_t flags, int node) ...@@ -1574,60 +1692,13 @@ static struct page *get_partial(struct kmem_cache *s, gfp_t flags, int node)
struct page *page; struct page *page;
int searchnode = (node == NUMA_NO_NODE) ? numa_node_id() : node; int searchnode = (node == NUMA_NO_NODE) ? numa_node_id() : node;
page = get_partial_node(get_node(s, searchnode)); page = get_partial_node(s, get_node(s, searchnode));
if (page || node != NUMA_NO_NODE) if (page || node != NUMA_NO_NODE)
return page; return page;
return get_any_partial(s, flags); return get_any_partial(s, flags);
} }
/*
* Move a page back to the lists.
*
* Must be called with the slab lock held.
*
* On exit the slab lock will have been dropped.
*/
static void unfreeze_slab(struct kmem_cache *s, struct page *page, int tail)
__releases(bitlock)
{
struct kmem_cache_node *n = get_node(s, page_to_nid(page));
__ClearPageSlubFrozen(page);
if (page->inuse) {
if (page->freelist) {
add_partial(n, page, tail);
stat(s, tail ? DEACTIVATE_TO_TAIL : DEACTIVATE_TO_HEAD);
} else {
stat(s, DEACTIVATE_FULL);
if (kmem_cache_debug(s) && (s->flags & SLAB_STORE_USER))
add_full(n, page);
}
slab_unlock(page);
} else {
stat(s, DEACTIVATE_EMPTY);
if (n->nr_partial < s->min_partial) {
/*
* Adding an empty slab to the partial slabs in order
* to avoid page allocator overhead. This slab needs
* to come after the other slabs with objects in
* so that the others get filled first. That way the
* size of the partial list stays small.
*
* kmem_cache_shrink can reclaim any empty slabs from
* the partial list.
*/
add_partial(n, page, 1);
slab_unlock(page);
} else {
slab_unlock(page);
stat(s, FREE_SLAB);
discard_slab(s, page);
}
}
}
#ifdef CONFIG_PREEMPT #ifdef CONFIG_PREEMPT
/* /*
* Calculate the next globally unique transaction for disambiguiation * Calculate the next globally unique transaction for disambiguiation
...@@ -1694,45 +1765,164 @@ void init_kmem_cache_cpus(struct kmem_cache *s) ...@@ -1694,45 +1765,164 @@ void init_kmem_cache_cpus(struct kmem_cache *s)
for_each_possible_cpu(cpu) for_each_possible_cpu(cpu)
per_cpu_ptr(s->cpu_slab, cpu)->tid = init_tid(cpu); per_cpu_ptr(s->cpu_slab, cpu)->tid = init_tid(cpu);
} }
/*
* Remove the cpu slab
*/
/* /*
* Remove the cpu slab * Remove the cpu slab
*/ */
static void deactivate_slab(struct kmem_cache *s, struct kmem_cache_cpu *c) static void deactivate_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
__releases(bitlock)
{ {
enum slab_modes { M_NONE, M_PARTIAL, M_FULL, M_FREE };
struct page *page = c->page; struct page *page = c->page;
int tail = 1; struct kmem_cache_node *n = get_node(s, page_to_nid(page));
int lock = 0;
enum slab_modes l = M_NONE, m = M_NONE;
void *freelist;
void *nextfree;
int tail = 0;
struct page new;
struct page old;
if (page->freelist) if (page->freelist) {
stat(s, DEACTIVATE_REMOTE_FREES); stat(s, DEACTIVATE_REMOTE_FREES);
tail = 1;
}
c->tid = next_tid(c->tid);
c->page = NULL;
freelist = c->freelist;
c->freelist = NULL;
/* /*
* Merge cpu freelist into slab freelist. Typically we get here * Stage one: Free all available per cpu objects back
* because both freelists are empty. So this is unlikely * to the page freelist while it is still frozen. Leave the
* to occur. * last one.
*
* There is no need to take the list->lock because the page
* is still frozen.
*/ */
while (unlikely(c->freelist)) { while (freelist && (nextfree = get_freepointer(s, freelist))) {
void **object; void *prior;
unsigned long counters;
tail = 0; /* Hot objects. Put the slab first */ do {
prior = page->freelist;
counters = page->counters;
set_freepointer(s, freelist, prior);
new.counters = counters;
new.inuse--;
VM_BUG_ON(!new.frozen);
/* Retrieve object from cpu_freelist */ } while (!__cmpxchg_double_slab(s, page,
object = c->freelist; prior, counters,
c->freelist = get_freepointer(s, c->freelist); freelist, new.counters,
"drain percpu freelist"));
/* And put onto the regular freelist */ freelist = nextfree;
set_freepointer(s, object, page->freelist); }
page->freelist = object;
page->inuse--; /*
* Stage two: Ensure that the page is unfrozen while the
* list presence reflects the actual number of objects
* during unfreeze.
*
* We setup the list membership and then perform a cmpxchg
* with the count. If there is a mismatch then the page
* is not unfrozen but the page is on the wrong list.
*
* Then we restart the process which may have to remove
* the page from the list that we just put it on again
* because the number of objects in the slab may have
* changed.
*/
redo:
old.freelist = page->freelist;
old.counters = page->counters;
VM_BUG_ON(!old.frozen);
/* Determine target state of the slab */
new.counters = old.counters;
if (freelist) {
new.inuse--;
set_freepointer(s, freelist, old.freelist);
new.freelist = freelist;
} else
new.freelist = old.freelist;
new.frozen = 0;
if (!new.inuse && n->nr_partial < s->min_partial)
m = M_FREE;
else if (new.freelist) {
m = M_PARTIAL;
if (!lock) {
lock = 1;
/*
* Taking the spinlock removes the possiblity
* that acquire_slab() will see a slab page that
* is frozen
*/
spin_lock(&n->list_lock);
}
} else {
m = M_FULL;
if (kmem_cache_debug(s) && !lock) {
lock = 1;
/*
* This also ensures that the scanning of full
* slabs from diagnostic functions will not see
* any frozen slabs.
*/
spin_lock(&n->list_lock);
}
}
if (l != m) {
if (l == M_PARTIAL)
remove_partial(n, page);
else if (l == M_FULL)
remove_full(s, page);
if (m == M_PARTIAL) {
add_partial(n, page, tail);
stat(s, tail ? DEACTIVATE_TO_TAIL : DEACTIVATE_TO_HEAD);
} else if (m == M_FULL) {
stat(s, DEACTIVATE_FULL);
add_full(s, n, page);
}
}
l = m;
if (!__cmpxchg_double_slab(s, page,
old.freelist, old.counters,
new.freelist, new.counters,
"unfreezing slab"))
goto redo;
if (lock)
spin_unlock(&n->list_lock);
if (m == M_FREE) {
stat(s, DEACTIVATE_EMPTY);
discard_slab(s, page);
stat(s, FREE_SLAB);
} }
c->page = NULL;
c->tid = next_tid(c->tid);
unfreeze_slab(s, page, tail);
} }
static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c) static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
{ {
stat(s, CPUSLAB_FLUSH); stat(s, CPUSLAB_FLUSH);
slab_lock(c->page);
deactivate_slab(s, c); deactivate_slab(s, c);
} }
...@@ -1861,6 +2051,8 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, ...@@ -1861,6 +2051,8 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
void **object; void **object;
struct page *page; struct page *page;
unsigned long flags; unsigned long flags;
struct page new;
unsigned long counters;
local_irq_save(flags); local_irq_save(flags);
#ifdef CONFIG_PREEMPT #ifdef CONFIG_PREEMPT
...@@ -1879,72 +2071,97 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, ...@@ -1879,72 +2071,97 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
if (!page) if (!page)
goto new_slab; goto new_slab;
slab_lock(page); if (unlikely(!node_match(c, node))) {
if (unlikely(!node_match(c, node))) stat(s, ALLOC_NODE_MISMATCH);
goto another_slab; deactivate_slab(s, c);
goto new_slab;
}
stat(s, ALLOC_REFILL); stat(s, ALLOC_SLOWPATH);
load_freelist: do {
object = page->freelist; object = page->freelist;
if (unlikely(!object)) counters = page->counters;
goto another_slab; new.counters = counters;
if (kmem_cache_debug(s)) VM_BUG_ON(!new.frozen);
goto debug;
c->freelist = get_freepointer(s, object); /*
page->inuse = page->objects; * If there is no object left then we use this loop to
page->freelist = NULL; * deactivate the slab which is simple since no objects
* are left in the slab and therefore we do not need to
* put the page back onto the partial list.
*
* If there are objects left then we retrieve them
* and use them to refill the per cpu queue.
*/
slab_unlock(page); new.inuse = page->objects;
new.frozen = object != NULL;
} while (!__cmpxchg_double_slab(s, page,
object, counters,
NULL, new.counters,
"__slab_alloc"));
if (unlikely(!object)) {
c->page = NULL;
stat(s, DEACTIVATE_BYPASS);
goto new_slab;
}
stat(s, ALLOC_REFILL);
load_freelist:
VM_BUG_ON(!page->frozen);
c->freelist = get_freepointer(s, object);
c->tid = next_tid(c->tid); c->tid = next_tid(c->tid);
local_irq_restore(flags); local_irq_restore(flags);
stat(s, ALLOC_SLOWPATH);
return object; return object;
another_slab:
deactivate_slab(s, c);
new_slab: new_slab:
page = get_partial(s, gfpflags, node); page = get_partial(s, gfpflags, node);
if (page) { if (page) {
stat(s, ALLOC_FROM_PARTIAL); stat(s, ALLOC_FROM_PARTIAL);
c->node = page_to_nid(page); object = c->freelist;
c->page = page;
if (kmem_cache_debug(s))
goto debug;
goto load_freelist; goto load_freelist;
} }
gfpflags &= gfp_allowed_mask;
if (gfpflags & __GFP_WAIT)
local_irq_enable();
page = new_slab(s, gfpflags, node); page = new_slab(s, gfpflags, node);
if (gfpflags & __GFP_WAIT)
local_irq_disable();
if (page) { if (page) {
c = __this_cpu_ptr(s->cpu_slab); c = __this_cpu_ptr(s->cpu_slab);
stat(s, ALLOC_SLAB);
if (c->page) if (c->page)
flush_slab(s, c); flush_slab(s, c);
slab_lock(page); /*
__SetPageSlubFrozen(page); * No other reference to the page yet so we can
* muck around with it freely without cmpxchg
*/
object = page->freelist;
page->freelist = NULL;
page->inuse = page->objects;
stat(s, ALLOC_SLAB);
c->node = page_to_nid(page); c->node = page_to_nid(page);
c->page = page; c->page = page;
if (kmem_cache_debug(s))
goto debug;
goto load_freelist; goto load_freelist;
} }
if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit()) if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit())
slab_out_of_memory(s, gfpflags, node); slab_out_of_memory(s, gfpflags, node);
local_irq_restore(flags); local_irq_restore(flags);
return NULL; return NULL;
debug: debug:
if (!alloc_debug_processing(s, page, object, addr)) if (!object || !alloc_debug_processing(s, page, object, addr))
goto another_slab; goto new_slab;
page->inuse++; c->freelist = get_freepointer(s, object);
page->freelist = get_freepointer(s, object);
deactivate_slab(s, c); deactivate_slab(s, c);
c->page = NULL; c->page = NULL;
c->node = NUMA_NO_NODE; c->node = NUMA_NO_NODE;
...@@ -2096,26 +2313,62 @@ static void __slab_free(struct kmem_cache *s, struct page *page, ...@@ -2096,26 +2313,62 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
{ {
void *prior; void *prior;
void **object = (void *)x; void **object = (void *)x;
unsigned long flags; int was_frozen;
int inuse;
struct page new;
unsigned long counters;
struct kmem_cache_node *n = NULL;
unsigned long uninitialized_var(flags);
local_irq_save(flags);
slab_lock(page);
stat(s, FREE_SLOWPATH); stat(s, FREE_SLOWPATH);
if (kmem_cache_debug(s) && !free_debug_processing(s, page, x, addr)) if (kmem_cache_debug(s) && !free_debug_processing(s, page, x, addr))
goto out_unlock; return;
do {
prior = page->freelist; prior = page->freelist;
counters = page->counters;
set_freepointer(s, object, prior); set_freepointer(s, object, prior);
page->freelist = object; new.counters = counters;
page->inuse--; was_frozen = new.frozen;
new.inuse--;
if ((!new.inuse || !prior) && !was_frozen && !n) {
n = get_node(s, page_to_nid(page));
/*
* Speculatively acquire the list_lock.
* If the cmpxchg does not succeed then we may
* drop the list_lock without any processing.
*
* Otherwise the list_lock will synchronize with
* other processors updating the list of slabs.
*/
spin_lock_irqsave(&n->list_lock, flags);
}
inuse = new.inuse;
if (unlikely(PageSlubFrozen(page))) { } while (!cmpxchg_double_slab(s, page,
prior, counters,
object, new.counters,
"__slab_free"));
if (likely(!n)) {
/*
* The list lock was not taken therefore no list
* activity can be necessary.
*/
if (was_frozen)
stat(s, FREE_FROZEN); stat(s, FREE_FROZEN);
goto out_unlock; return;
} }
if (unlikely(!page->inuse)) /*
* was_frozen may have been set after we acquired the list_lock in
* an earlier loop. So we need to check it here again.
*/
if (was_frozen)
stat(s, FREE_FROZEN);
else {
if (unlikely(!inuse && n->nr_partial > s->min_partial))
goto slab_empty; goto slab_empty;
/* /*
...@@ -2123,13 +2376,12 @@ static void __slab_free(struct kmem_cache *s, struct page *page, ...@@ -2123,13 +2376,12 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
* then add it. * then add it.
*/ */
if (unlikely(!prior)) { if (unlikely(!prior)) {
add_partial(get_node(s, page_to_nid(page)), page, 1); remove_full(s, page);
add_partial(n, page, 0);
stat(s, FREE_ADD_PARTIAL); stat(s, FREE_ADD_PARTIAL);
} }
}
out_unlock: spin_unlock_irqrestore(&n->list_lock, flags);
slab_unlock(page);
local_irq_restore(flags);
return; return;
slab_empty: slab_empty:
...@@ -2137,11 +2389,11 @@ static void __slab_free(struct kmem_cache *s, struct page *page, ...@@ -2137,11 +2389,11 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
/* /*
* Slab still on the partial list. * Slab still on the partial list.
*/ */
remove_partial(s, page); remove_partial(n, page);
stat(s, FREE_REMOVE_PARTIAL); stat(s, FREE_REMOVE_PARTIAL);
} }
slab_unlock(page);
local_irq_restore(flags); spin_unlock_irqrestore(&n->list_lock, flags);
stat(s, FREE_SLAB); stat(s, FREE_SLAB);
discard_slab(s, page); discard_slab(s, page);
} }
...@@ -2415,7 +2667,6 @@ static void early_kmem_cache_node_alloc(int node) ...@@ -2415,7 +2667,6 @@ static void early_kmem_cache_node_alloc(int node)
{ {
struct page *page; struct page *page;
struct kmem_cache_node *n; struct kmem_cache_node *n;
unsigned long flags;
BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node)); BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node));
...@@ -2433,6 +2684,7 @@ static void early_kmem_cache_node_alloc(int node) ...@@ -2433,6 +2684,7 @@ static void early_kmem_cache_node_alloc(int node)
BUG_ON(!n); BUG_ON(!n);
page->freelist = get_freepointer(kmem_cache_node, n); page->freelist = get_freepointer(kmem_cache_node, n);
page->inuse++; page->inuse++;
page->frozen = 0;
kmem_cache_node->node[node] = n; kmem_cache_node->node[node] = n;
#ifdef CONFIG_SLUB_DEBUG #ifdef CONFIG_SLUB_DEBUG
init_object(kmem_cache_node, n, SLUB_RED_ACTIVE); init_object(kmem_cache_node, n, SLUB_RED_ACTIVE);
...@@ -2441,14 +2693,7 @@ static void early_kmem_cache_node_alloc(int node) ...@@ -2441,14 +2693,7 @@ static void early_kmem_cache_node_alloc(int node)
init_kmem_cache_node(n, kmem_cache_node); init_kmem_cache_node(n, kmem_cache_node);
inc_slabs_node(kmem_cache_node, node, page->objects); inc_slabs_node(kmem_cache_node, node, page->objects);
/*
* lockdep requires consistent irq usage for each lock
* so even though there cannot be a race this early in
* the boot sequence, we still disable irqs.
*/
local_irq_save(flags);
add_partial(n, page, 0); add_partial(n, page, 0);
local_irq_restore(flags);
} }
static void free_kmem_cache_nodes(struct kmem_cache *s) static void free_kmem_cache_nodes(struct kmem_cache *s)
...@@ -2654,6 +2899,12 @@ static int kmem_cache_open(struct kmem_cache *s, ...@@ -2654,6 +2899,12 @@ static int kmem_cache_open(struct kmem_cache *s,
} }
} }
#ifdef CONFIG_CMPXCHG_DOUBLE
if (system_has_cmpxchg_double() && (s->flags & SLAB_DEBUG_FLAGS) == 0)
/* Enable fast mode */
s->flags |= __CMPXCHG_DOUBLE;
#endif
/* /*
* The larger the object size is, the more pages we want on the partial * The larger the object size is, the more pages we want on the partial
* list to avoid pounding the page allocator excessively. * list to avoid pounding the page allocator excessively.
...@@ -2726,7 +2977,7 @@ static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n) ...@@ -2726,7 +2977,7 @@ static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n)
spin_lock_irqsave(&n->list_lock, flags); spin_lock_irqsave(&n->list_lock, flags);
list_for_each_entry_safe(page, h, &n->partial, lru) { list_for_each_entry_safe(page, h, &n->partial, lru) {
if (!page->inuse) { if (!page->inuse) {
__remove_partial(n, page); remove_partial(n, page);
discard_slab(s, page); discard_slab(s, page);
} else { } else {
list_slab_objects(s, page, list_slab_objects(s, page,
...@@ -3094,14 +3345,8 @@ int kmem_cache_shrink(struct kmem_cache *s) ...@@ -3094,14 +3345,8 @@ int kmem_cache_shrink(struct kmem_cache *s)
* list_lock. page->inuse here is the upper limit. * list_lock. page->inuse here is the upper limit.
*/ */
list_for_each_entry_safe(page, t, &n->partial, lru) { list_for_each_entry_safe(page, t, &n->partial, lru) {
if (!page->inuse && slab_trylock(page)) { if (!page->inuse) {
/* remove_partial(n, page);
* Must hold slab lock here because slab_free
* may have freed the last object and be
* waiting to release the slab.
*/
__remove_partial(n, page);
slab_unlock(page);
discard_slab(s, page); discard_slab(s, page);
} else { } else {
list_move(&page->lru, list_move(&page->lru,
...@@ -3689,12 +3934,9 @@ static int validate_slab(struct kmem_cache *s, struct page *page, ...@@ -3689,12 +3934,9 @@ static int validate_slab(struct kmem_cache *s, struct page *page,
static void validate_slab_slab(struct kmem_cache *s, struct page *page, static void validate_slab_slab(struct kmem_cache *s, struct page *page,
unsigned long *map) unsigned long *map)
{ {
if (slab_trylock(page)) { slab_lock(page);
validate_slab(s, page, map); validate_slab(s, page, map);
slab_unlock(page); slab_unlock(page);
} else
printk(KERN_INFO "SLUB %s: Skipped busy slab 0x%p\n",
s->name, page);
} }
static int validate_slab_node(struct kmem_cache *s, static int validate_slab_node(struct kmem_cache *s,
...@@ -4342,8 +4584,10 @@ static ssize_t sanity_checks_store(struct kmem_cache *s, ...@@ -4342,8 +4584,10 @@ static ssize_t sanity_checks_store(struct kmem_cache *s,
const char *buf, size_t length) const char *buf, size_t length)
{ {
s->flags &= ~SLAB_DEBUG_FREE; s->flags &= ~SLAB_DEBUG_FREE;
if (buf[0] == '1') if (buf[0] == '1') {
s->flags &= ~__CMPXCHG_DOUBLE;
s->flags |= SLAB_DEBUG_FREE; s->flags |= SLAB_DEBUG_FREE;
}
return length; return length;
} }
SLAB_ATTR(sanity_checks); SLAB_ATTR(sanity_checks);
...@@ -4357,8 +4601,10 @@ static ssize_t trace_store(struct kmem_cache *s, const char *buf, ...@@ -4357,8 +4601,10 @@ static ssize_t trace_store(struct kmem_cache *s, const char *buf,
size_t length) size_t length)
{ {
s->flags &= ~SLAB_TRACE; s->flags &= ~SLAB_TRACE;
if (buf[0] == '1') if (buf[0] == '1') {
s->flags &= ~__CMPXCHG_DOUBLE;
s->flags |= SLAB_TRACE; s->flags |= SLAB_TRACE;
}
return length; return length;
} }
SLAB_ATTR(trace); SLAB_ATTR(trace);
...@@ -4375,8 +4621,10 @@ static ssize_t red_zone_store(struct kmem_cache *s, ...@@ -4375,8 +4621,10 @@ static ssize_t red_zone_store(struct kmem_cache *s,
return -EBUSY; return -EBUSY;
s->flags &= ~SLAB_RED_ZONE; s->flags &= ~SLAB_RED_ZONE;
if (buf[0] == '1') if (buf[0] == '1') {
s->flags &= ~__CMPXCHG_DOUBLE;
s->flags |= SLAB_RED_ZONE; s->flags |= SLAB_RED_ZONE;
}
calculate_sizes(s, -1); calculate_sizes(s, -1);
return length; return length;
} }
...@@ -4394,8 +4642,10 @@ static ssize_t poison_store(struct kmem_cache *s, ...@@ -4394,8 +4642,10 @@ static ssize_t poison_store(struct kmem_cache *s,
return -EBUSY; return -EBUSY;
s->flags &= ~SLAB_POISON; s->flags &= ~SLAB_POISON;
if (buf[0] == '1') if (buf[0] == '1') {
s->flags &= ~__CMPXCHG_DOUBLE;
s->flags |= SLAB_POISON; s->flags |= SLAB_POISON;
}
calculate_sizes(s, -1); calculate_sizes(s, -1);
return length; return length;
} }
...@@ -4413,8 +4663,10 @@ static ssize_t store_user_store(struct kmem_cache *s, ...@@ -4413,8 +4663,10 @@ static ssize_t store_user_store(struct kmem_cache *s,
return -EBUSY; return -EBUSY;
s->flags &= ~SLAB_STORE_USER; s->flags &= ~SLAB_STORE_USER;
if (buf[0] == '1') if (buf[0] == '1') {
s->flags &= ~__CMPXCHG_DOUBLE;
s->flags |= SLAB_STORE_USER; s->flags |= SLAB_STORE_USER;
}
calculate_sizes(s, -1); calculate_sizes(s, -1);
return length; return length;
} }
...@@ -4579,6 +4831,7 @@ STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial); ...@@ -4579,6 +4831,7 @@ STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial);
STAT_ATTR(ALLOC_FROM_PARTIAL, alloc_from_partial); STAT_ATTR(ALLOC_FROM_PARTIAL, alloc_from_partial);
STAT_ATTR(ALLOC_SLAB, alloc_slab); STAT_ATTR(ALLOC_SLAB, alloc_slab);
STAT_ATTR(ALLOC_REFILL, alloc_refill); STAT_ATTR(ALLOC_REFILL, alloc_refill);
STAT_ATTR(ALLOC_NODE_MISMATCH, alloc_node_mismatch);
STAT_ATTR(FREE_SLAB, free_slab); STAT_ATTR(FREE_SLAB, free_slab);
STAT_ATTR(CPUSLAB_FLUSH, cpuslab_flush); STAT_ATTR(CPUSLAB_FLUSH, cpuslab_flush);
STAT_ATTR(DEACTIVATE_FULL, deactivate_full); STAT_ATTR(DEACTIVATE_FULL, deactivate_full);
...@@ -4586,7 +4839,10 @@ STAT_ATTR(DEACTIVATE_EMPTY, deactivate_empty); ...@@ -4586,7 +4839,10 @@ STAT_ATTR(DEACTIVATE_EMPTY, deactivate_empty);
STAT_ATTR(DEACTIVATE_TO_HEAD, deactivate_to_head); STAT_ATTR(DEACTIVATE_TO_HEAD, deactivate_to_head);
STAT_ATTR(DEACTIVATE_TO_TAIL, deactivate_to_tail); STAT_ATTR(DEACTIVATE_TO_TAIL, deactivate_to_tail);
STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees); STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees);
STAT_ATTR(DEACTIVATE_BYPASS, deactivate_bypass);
STAT_ATTR(ORDER_FALLBACK, order_fallback); STAT_ATTR(ORDER_FALLBACK, order_fallback);
STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail);
STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail);
#endif #endif
static struct attribute *slab_attrs[] = { static struct attribute *slab_attrs[] = {
...@@ -4636,6 +4892,7 @@ static struct attribute *slab_attrs[] = { ...@@ -4636,6 +4892,7 @@ static struct attribute *slab_attrs[] = {
&alloc_from_partial_attr.attr, &alloc_from_partial_attr.attr,
&alloc_slab_attr.attr, &alloc_slab_attr.attr,
&alloc_refill_attr.attr, &alloc_refill_attr.attr,
&alloc_node_mismatch_attr.attr,
&free_slab_attr.attr, &free_slab_attr.attr,
&cpuslab_flush_attr.attr, &cpuslab_flush_attr.attr,
&deactivate_full_attr.attr, &deactivate_full_attr.attr,
...@@ -4643,7 +4900,10 @@ static struct attribute *slab_attrs[] = { ...@@ -4643,7 +4900,10 @@ static struct attribute *slab_attrs[] = {
&deactivate_to_head_attr.attr, &deactivate_to_head_attr.attr,
&deactivate_to_tail_attr.attr, &deactivate_to_tail_attr.attr,
&deactivate_remote_frees_attr.attr, &deactivate_remote_frees_attr.attr,
&deactivate_bypass_attr.attr,
&order_fallback_attr.attr, &order_fallback_attr.attr,
&cmpxchg_double_fail_attr.attr,
&cmpxchg_double_cpu_fail_attr.attr,
#endif #endif
#ifdef CONFIG_FAILSLAB #ifdef CONFIG_FAILSLAB
&failslab_attr.attr, &failslab_attr.attr,
......
...@@ -2,8 +2,9 @@ ...@@ -2,8 +2,9 @@
* Slabinfo: Tool to get reports about slabs * Slabinfo: Tool to get reports about slabs
* *
* (C) 2007 sgi, Christoph Lameter * (C) 2007 sgi, Christoph Lameter
* (C) 2011 Linux Foundation, Christoph Lameter
* *
* Compile by: * Compile with:
* *
* gcc -o slabinfo slabinfo.c * gcc -o slabinfo slabinfo.c
*/ */
...@@ -39,6 +40,8 @@ struct slabinfo { ...@@ -39,6 +40,8 @@ struct slabinfo {
unsigned long cpuslab_flush, deactivate_full, deactivate_empty; unsigned long cpuslab_flush, deactivate_full, deactivate_empty;
unsigned long deactivate_to_head, deactivate_to_tail; unsigned long deactivate_to_head, deactivate_to_tail;
unsigned long deactivate_remote_frees, order_fallback; unsigned long deactivate_remote_frees, order_fallback;
unsigned long cmpxchg_double_cpu_fail, cmpxchg_double_fail;
unsigned long alloc_node_mismatch, deactivate_bypass;
int numa[MAX_NODES]; int numa[MAX_NODES];
int numa_partial[MAX_NODES]; int numa_partial[MAX_NODES];
} slabinfo[MAX_SLABS]; } slabinfo[MAX_SLABS];
...@@ -99,7 +102,7 @@ static void fatal(const char *x, ...) ...@@ -99,7 +102,7 @@ static void fatal(const char *x, ...)
static void usage(void) static void usage(void)
{ {
printf("slabinfo 5/7/2007. (c) 2007 sgi.\n\n" printf("slabinfo 4/15/2011. (c) 2007 sgi/(c) 2011 Linux Foundation.\n\n"
"slabinfo [-ahnpvtsz] [-d debugopts] [slab-regexp]\n" "slabinfo [-ahnpvtsz] [-d debugopts] [slab-regexp]\n"
"-a|--aliases Show aliases\n" "-a|--aliases Show aliases\n"
"-A|--activity Most active slabs first\n" "-A|--activity Most active slabs first\n"
...@@ -293,7 +296,7 @@ int line = 0; ...@@ -293,7 +296,7 @@ int line = 0;
static void first_line(void) static void first_line(void)
{ {
if (show_activity) if (show_activity)
printf("Name Objects Alloc Free %%Fast Fallb O\n"); printf("Name Objects Alloc Free %%Fast Fallb O CmpX UL\n");
else else
printf("Name Objects Objsize Space " printf("Name Objects Objsize Space "
"Slabs/Part/Cpu O/S O %%Fr %%Ef Flg\n"); "Slabs/Part/Cpu O/S O %%Fr %%Ef Flg\n");
...@@ -379,14 +382,14 @@ static void show_tracking(struct slabinfo *s) ...@@ -379,14 +382,14 @@ static void show_tracking(struct slabinfo *s)
printf("\n%s: Kernel object allocation\n", s->name); printf("\n%s: Kernel object allocation\n", s->name);
printf("-----------------------------------------------------------------------\n"); printf("-----------------------------------------------------------------------\n");
if (read_slab_obj(s, "alloc_calls")) if (read_slab_obj(s, "alloc_calls"))
printf(buffer); printf("%s", buffer);
else else
printf("No Data\n"); printf("No Data\n");
printf("\n%s: Kernel object freeing\n", s->name); printf("\n%s: Kernel object freeing\n", s->name);
printf("------------------------------------------------------------------------\n"); printf("------------------------------------------------------------------------\n");
if (read_slab_obj(s, "free_calls")) if (read_slab_obj(s, "free_calls"))
printf(buffer); printf("%s", buffer);
else else
printf("No Data\n"); printf("No Data\n");
...@@ -400,7 +403,7 @@ static void ops(struct slabinfo *s) ...@@ -400,7 +403,7 @@ static void ops(struct slabinfo *s)
if (read_slab_obj(s, "ops")) { if (read_slab_obj(s, "ops")) {
printf("\n%s: kmem_cache operations\n", s->name); printf("\n%s: kmem_cache operations\n", s->name);
printf("--------------------------------------------\n"); printf("--------------------------------------------\n");
printf(buffer); printf("%s", buffer);
} else } else
printf("\n%s has no kmem_cache operations\n", s->name); printf("\n%s has no kmem_cache operations\n", s->name);
} }
...@@ -462,19 +465,32 @@ static void slab_stats(struct slabinfo *s) ...@@ -462,19 +465,32 @@ static void slab_stats(struct slabinfo *s)
if (s->cpuslab_flush) if (s->cpuslab_flush)
printf("Flushes %8lu\n", s->cpuslab_flush); printf("Flushes %8lu\n", s->cpuslab_flush);
if (s->alloc_refill)
printf("Refill %8lu\n", s->alloc_refill);
total = s->deactivate_full + s->deactivate_empty + total = s->deactivate_full + s->deactivate_empty +
s->deactivate_to_head + s->deactivate_to_tail; s->deactivate_to_head + s->deactivate_to_tail + s->deactivate_bypass;
if (total) if (total) {
printf("Deactivate Full=%lu(%lu%%) Empty=%lu(%lu%%) " printf("\nSlab Deactivation Ocurrences %%\n");
"ToHead=%lu(%lu%%) ToTail=%lu(%lu%%)\n", printf("-------------------------------------------------\n");
s->deactivate_full, (s->deactivate_full * 100) / total, printf("Slab full %7lu %3lu%%\n",
s->deactivate_empty, (s->deactivate_empty * 100) / total, s->deactivate_full, (s->deactivate_full * 100) / total);
s->deactivate_to_head, (s->deactivate_to_head * 100) / total, printf("Slab empty %7lu %3lu%%\n",
s->deactivate_empty, (s->deactivate_empty * 100) / total);
printf("Moved to head of partial list %7lu %3lu%%\n",
s->deactivate_to_head, (s->deactivate_to_head * 100) / total);
printf("Moved to tail of partial list %7lu %3lu%%\n",
s->deactivate_to_tail, (s->deactivate_to_tail * 100) / total); s->deactivate_to_tail, (s->deactivate_to_tail * 100) / total);
printf("Deactivation bypass %7lu %3lu%%\n",
s->deactivate_bypass, (s->deactivate_bypass * 100) / total);
printf("Refilled from foreign frees %7lu %3lu%%\n",
s->alloc_refill, (s->alloc_refill * 100) / total);
printf("Node mismatch %7lu %3lu%%\n",
s->alloc_node_mismatch, (s->alloc_node_mismatch * 100) / total);
}
if (s->cmpxchg_double_fail || s->cmpxchg_double_cpu_fail)
printf("\nCmpxchg_double Looping\n------------------------\n");
printf("Locked Cmpxchg Double redos %lu\nUnlocked Cmpxchg Double redos %lu\n",
s->cmpxchg_double_fail, s->cmpxchg_double_cpu_fail);
} }
static void report(struct slabinfo *s) static void report(struct slabinfo *s)
...@@ -573,12 +589,13 @@ static void slabcache(struct slabinfo *s) ...@@ -573,12 +589,13 @@ static void slabcache(struct slabinfo *s)
total_alloc = s->alloc_fastpath + s->alloc_slowpath; total_alloc = s->alloc_fastpath + s->alloc_slowpath;
total_free = s->free_fastpath + s->free_slowpath; total_free = s->free_fastpath + s->free_slowpath;
printf("%-21s %8ld %10ld %10ld %3ld %3ld %5ld %1d\n", printf("%-21s %8ld %10ld %10ld %3ld %3ld %5ld %1d %4ld %4ld\n",
s->name, s->objects, s->name, s->objects,
total_alloc, total_free, total_alloc, total_free,
total_alloc ? (s->alloc_fastpath * 100 / total_alloc) : 0, total_alloc ? (s->alloc_fastpath * 100 / total_alloc) : 0,
total_free ? (s->free_fastpath * 100 / total_free) : 0, total_free ? (s->free_fastpath * 100 / total_free) : 0,
s->order_fallback, s->order); s->order_fallback, s->order, s->cmpxchg_double_fail,
s->cmpxchg_double_cpu_fail);
} }
else else
printf("%-21s %8ld %7d %8s %14s %4d %1d %3ld %3ld %s\n", printf("%-21s %8ld %7d %8s %14s %4d %1d %3ld %3ld %s\n",
...@@ -1190,6 +1207,10 @@ static void read_slab_dir(void) ...@@ -1190,6 +1207,10 @@ static void read_slab_dir(void)
slab->deactivate_to_tail = get_obj("deactivate_to_tail"); slab->deactivate_to_tail = get_obj("deactivate_to_tail");
slab->deactivate_remote_frees = get_obj("deactivate_remote_frees"); slab->deactivate_remote_frees = get_obj("deactivate_remote_frees");
slab->order_fallback = get_obj("order_fallback"); slab->order_fallback = get_obj("order_fallback");
slab->cmpxchg_double_cpu_fail = get_obj("cmpxchg_double_cpu_fail");
slab->cmpxchg_double_fail = get_obj("cmpxchg_double_fail");
slab->alloc_node_mismatch = get_obj("alloc_node_mismatch");
slab->deactivate_bypass = get_obj("deactivate_bypass");
chdir(".."); chdir("..");
if (slab->name[0] == ':') if (slab->name[0] == ':')
alias_targets++; alias_targets++;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment