slub.c 183 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
Christoph Lameter's avatar
Christoph Lameter committed
2 3 4 5
/*
 * SLUB: A slab allocator that limits cache line use instead of queuing
 * objects in per cpu and per node lists.
 *
6
 * The allocator synchronizes using per slab locks or atomic operations
7
 * and only uses a centralized lock to manage a pool of partial slabs.
Christoph Lameter's avatar
Christoph Lameter committed
8
 *
Christoph Lameter's avatar
Christoph Lameter committed
9
 * (C) 2007 SGI, Christoph Lameter
10
 * (C) 2011 Linux Foundation, Christoph Lameter
Christoph Lameter's avatar
Christoph Lameter committed
11 12 13
 */

#include <linux/mm.h>
14
#include <linux/swap.h> /* mm_account_reclaimed_pages() */
Christoph Lameter's avatar
Christoph Lameter committed
15 16 17
#include <linux/module.h>
#include <linux/bit_spinlock.h>
#include <linux/interrupt.h>
Andrew Morton's avatar
Andrew Morton committed
18
#include <linux/swab.h>
Christoph Lameter's avatar
Christoph Lameter committed
19 20
#include <linux/bitops.h>
#include <linux/slab.h>
21
#include "slab.h"
22
#include <linux/proc_fs.h>
Christoph Lameter's avatar
Christoph Lameter committed
23
#include <linux/seq_file.h>
24
#include <linux/kasan.h>
25
#include <linux/kmsan.h>
Christoph Lameter's avatar
Christoph Lameter committed
26 27 28 29
#include <linux/cpu.h>
#include <linux/cpuset.h>
#include <linux/mempolicy.h>
#include <linux/ctype.h>
30
#include <linux/stackdepot.h>
31
#include <linux/debugobjects.h>
Christoph Lameter's avatar
Christoph Lameter committed
32
#include <linux/kallsyms.h>
33
#include <linux/kfence.h>
34
#include <linux/memory.h>
Roman Zippel's avatar
Roman Zippel committed
35
#include <linux/math64.h>
Akinobu Mita's avatar
Akinobu Mita committed
36
#include <linux/fault-inject.h>
37
#include <linux/kmemleak.h>
38
#include <linux/stacktrace.h>
39
#include <linux/prefetch.h>
40
#include <linux/memcontrol.h>
41
#include <linux/random.h>
42
#include <kunit/test.h>
43
#include <kunit/test-bug.h>
44
#include <linux/sort.h>
Christoph Lameter's avatar
Christoph Lameter committed
45

46
#include <linux/debugfs.h>
47 48
#include <trace/events/kmem.h>

49 50
#include "internal.h"

Christoph Lameter's avatar
Christoph Lameter committed
51 52
/*
 * Lock order:
53
 *   1. slab_mutex (Global Mutex)
54 55
 *   2. node->list_lock (Spinlock)
 *   3. kmem_cache->cpu_slab->lock (Local lock)
56
 *   4. slab_lock(slab) (Only on some arches)
57
 *   5. object_map_lock (Only for debugging)
Christoph Lameter's avatar
Christoph Lameter committed
58
 *
59
 *   slab_mutex
60
 *
61
 *   The role of the slab_mutex is to protect the list of all the slabs
62
 *   and to synchronize major metadata changes to slab cache structures.
63 64 65 66 67 68
 *   Also synchronizes memory hotplug callbacks.
 *
 *   slab_lock
 *
 *   The slab_lock is a wrapper around the page lock, thus it is a bit
 *   spinlock.
69
 *
70 71 72
 *   The slab_lock is only used on arches that do not have the ability
 *   to do a cmpxchg_double. It only protects:
 *
73 74 75 76
 *	A. slab->freelist	-> List of free objects in a slab
 *	B. slab->inuse		-> Number of objects in use
 *	C. slab->objects	-> Number of objects in slab
 *	D. slab->frozen		-> frozen state
77
 *
78 79
 *   Frozen slabs
 *
80 81 82
 *   If a slab is frozen then it is exempt from list management. It is
 *   the cpu slab which is actively allocated from by the processor that
 *   froze it and it is not on any list. The processor that froze the
83
 *   slab is the one who can perform list operations on the slab. Other
84 85
 *   processors may put objects onto the freelist but the processor that
 *   froze the slab is the only one that can retrieve the objects from the
86
 *   slab's freelist.
Christoph Lameter's avatar
Christoph Lameter committed
87
 *
88 89 90 91 92 93 94 95 96 97 98 99 100 101
 *   CPU partial slabs
 *
 *   The partially empty slabs cached on the CPU partial list are used
 *   for performance reasons, which speeds up the allocation process.
 *   These slabs are not frozen, but are also exempt from list management,
 *   by clearing the PG_workingset flag when moving out of the node
 *   partial list. Please see __slab_free() for more details.
 *
 *   To sum up, the current scheme is:
 *   - node partial slab: PG_Workingset && !frozen
 *   - cpu partial slab: !PG_Workingset && !frozen
 *   - cpu slab: !PG_Workingset && frozen
 *   - full slab: !PG_Workingset && !frozen
 *
102 103
 *   list_lock
 *
Christoph Lameter's avatar
Christoph Lameter committed
104 105 106 107 108 109 110 111 112 113 114
 *   The list_lock protects the partial and full list on each node and
 *   the partial slab counter. If taken then no new slabs may be added or
 *   removed from the lists nor make the number of partial slabs be modified.
 *   (Note that the total number of slabs is an atomic value that may be
 *   modified without taking the list lock).
 *
 *   The list_lock is a centralized lock and thus we avoid taking it as
 *   much as possible. As long as SLUB does not have to handle partial
 *   slabs, operations can continue without any centralized lock. F.e.
 *   allocating a long series of objects that fill up slabs does not require
 *   the list lock.
115
 *
116 117 118
 *   For debug caches, all allocations are forced to go through a list_lock
 *   protected region to serialize against concurrent validation.
 *
119 120 121 122 123 124
 *   cpu_slab->lock local lock
 *
 *   This locks protect slowpath manipulation of all kmem_cache_cpu fields
 *   except the stat counters. This is a percpu structure manipulated only by
 *   the local cpu, so the lock protects against being preempted or interrupted
 *   by an irq. Fast path operations rely on lockless operations instead.
125 126 127 128 129
 *
 *   On PREEMPT_RT, the local lock neither disables interrupts nor preemption
 *   which means the lockless fastpath cannot be used as it might interfere with
 *   an in-progress slow path operations. In this case the local lock is always
 *   taken but it still utilizes the freelist for the common operations.
130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149
 *
 *   lockless fastpaths
 *
 *   The fast path allocation (slab_alloc_node()) and freeing (do_slab_free())
 *   are fully lockless when satisfied from the percpu slab (and when
 *   cmpxchg_double is possible to use, otherwise slab_lock is taken).
 *   They also don't disable preemption or migration or irqs. They rely on
 *   the transaction id (tid) field to detect being preempted or moved to
 *   another cpu.
 *
 *   irq, preemption, migration considerations
 *
 *   Interrupts are disabled as part of list_lock or local_lock operations, or
 *   around the slab_lock operation, in order to make the slab allocator safe
 *   to use in the context of an irq.
 *
 *   In addition, preemption (or migration on PREEMPT_RT) is disabled in the
 *   allocation slowpath, bulk allocation, and put_cpu_partial(), so that the
 *   local cpu doesn't change in the process and e.g. the kmem_cache_cpu pointer
 *   doesn't have to be revalidated in each section protected by the local lock.
Christoph Lameter's avatar
Christoph Lameter committed
150 151 152 153
 *
 * SLUB assigns one slab for allocation to each processor.
 * Allocations only occur from these slabs called cpu slabs.
 *
Christoph Lameter's avatar
Christoph Lameter committed
154 155
 * Slabs with free elements are kept on a partial list and during regular
 * operations no list for full slabs is used. If an object in a full slab is
Christoph Lameter's avatar
Christoph Lameter committed
156
 * freed then the slab will show up again on the partial lists.
Christoph Lameter's avatar
Christoph Lameter committed
157 158
 * We track full slabs for debugging purposes though because otherwise we
 * cannot scan all objects.
Christoph Lameter's avatar
Christoph Lameter committed
159 160 161 162 163
 *
 * Slabs are freed when they become empty. Teardown and setup is
 * minimal so we rely on the page allocators per cpu caches for
 * fast frees and allocs.
 *
164
 * slab->frozen		The slab is frozen and exempt from list processing.
165 166 167 168 169 170 171 172 173 174 175
 * 			This means that the slab is dedicated to a purpose
 * 			such as satisfying allocations for a specific
 * 			processor. Objects may be freed in the slab while
 * 			it is frozen but slab_free will then skip the usual
 * 			list operations. It is up to the processor holding
 * 			the slab to integrate the slab into the slab lists
 * 			when the slab is no longer needed.
 *
 * 			One use of this flag is to mark slabs that are
 * 			used for allocations. Then such a slab becomes a cpu
 * 			slab. The cpu slab may be equipped with an additional
176
 * 			freelist that allows lockless access to
177 178
 * 			free objects in addition to the regular freelist
 * 			that requires the slab lock.
Christoph Lameter's avatar
Christoph Lameter committed
179
 *
Yu Zhao's avatar
Yu Zhao committed
180
 * SLAB_DEBUG_FLAGS	Slab requires special handling due to debug
Christoph Lameter's avatar
Christoph Lameter committed
181
 * 			options set. This moves	slab handling out of
182
 * 			the fast path and disables lockless freelists.
Christoph Lameter's avatar
Christoph Lameter committed
183 184
 */

185 186 187 188 189
/*
 * We could simply use migrate_disable()/enable() but as long as it's a
 * function call even on !PREEMPT_RT, use inline preempt_disable() there.
 */
#ifndef CONFIG_PREEMPT_RT
190 191 192
#define slub_get_cpu_ptr(var)		get_cpu_ptr(var)
#define slub_put_cpu_ptr(var)		put_cpu_ptr(var)
#define USE_LOCKLESS_FAST_PATH()	(true)
193 194 195 196 197 198 199 200 201 202 203
#else
#define slub_get_cpu_ptr(var)		\
({					\
	migrate_disable();		\
	this_cpu_ptr(var);		\
})
#define slub_put_cpu_ptr(var)		\
do {					\
	(void)(var);			\
	migrate_enable();		\
} while (0)
204
#define USE_LOCKLESS_FAST_PATH()	(false)
205 206
#endif

207 208 209 210 211 212
#ifndef CONFIG_SLUB_TINY
#define __fastpath_inline __always_inline
#else
#define __fastpath_inline
#endif

213 214 215 216 217 218
#ifdef CONFIG_SLUB_DEBUG
#ifdef CONFIG_SLUB_DEBUG_ON
DEFINE_STATIC_KEY_TRUE(slub_debug_enabled);
#else
DEFINE_STATIC_KEY_FALSE(slub_debug_enabled);
#endif
219
#endif		/* CONFIG_SLUB_DEBUG */
220

221 222 223 224
/* Structure holding parameters for get_partial() call chain */
struct partial_context {
	gfp_t flags;
	unsigned int orig_size;
225
	void *object;
226 227
};

228 229 230
static inline bool kmem_cache_debug(struct kmem_cache *s)
{
	return kmem_cache_debug_flags(s, SLAB_DEBUG_FLAGS);
231
}
232

233 234 235 236 237 238
static inline bool slub_debug_orig_size(struct kmem_cache *s)
{
	return (kmem_cache_debug_flags(s, SLAB_STORE_USER) &&
			(s->flags & SLAB_KMALLOC));
}

239
void *fixup_red_left(struct kmem_cache *s, void *p)
240
{
241
	if (kmem_cache_debug_flags(s, SLAB_RED_ZONE))
242 243 244 245 246
		p += s->red_left_pad;

	return p;
}

247 248 249 250 251 252 253 254 255
static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s)
{
#ifdef CONFIG_SLUB_CPU_PARTIAL
	return !kmem_cache_debug(s);
#else
	return false;
#endif
}

Christoph Lameter's avatar
Christoph Lameter committed
256 257 258 259 260 261 262 263
/*
 * Issues still to be resolved:
 *
 * - Support PAGE_ALLOC_DEBUG. Should be easy to do.
 *
 * - Variable sizing of the per node arrays
 */

264 265 266
/* Enable to log cmpxchg failures */
#undef SLUB_DEBUG_CMPXCHG

267
#ifndef CONFIG_SLUB_TINY
268
/*
269
 * Minimum number of partial slabs. These will be left on the partial
270 271
 * lists even if they are empty. kmem_cache_shrink may reclaim them.
 */
272
#define MIN_PARTIAL 5
Christoph Lameter's avatar
Christoph Lameter committed
273

274 275 276
/*
 * Maximum number of desirable partial slabs.
 * The existence of more partial slabs makes kmem_cache_shrink
277
 * sort the partial list by the number of objects in use.
278 279
 */
#define MAX_PARTIAL 10
280 281 282 283
#else
#define MIN_PARTIAL 0
#define MAX_PARTIAL 0
#endif
284

285
#define DEBUG_DEFAULT_FLAGS (SLAB_CONSISTENCY_CHECKS | SLAB_RED_ZONE | \
Christoph Lameter's avatar
Christoph Lameter committed
286
				SLAB_POISON | SLAB_STORE_USER)
Christoph Lameter's avatar
Christoph Lameter committed
287

288 289 290 291 292 293 294 295
/*
 * These debug flags cannot use CMPXCHG because there might be consistency
 * issues when checking or reading debug information
 */
#define SLAB_NO_CMPXCHG (SLAB_CONSISTENCY_CHECKS | SLAB_STORE_USER | \
				SLAB_TRACE)


296
/*
297
 * Debugging flags that require metadata to be stored in the slab.  These get
298
 * disabled when slab_debug=O is used and a cache's min order increases with
299
 * metadata.
300
 */
301
#define DEBUG_METADATA_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)
302

303 304
#define OO_SHIFT	16
#define OO_MASK		((1 << OO_SHIFT) - 1)
305
#define MAX_OBJS_PER_PAGE	32767 /* since slab.objects is u15 */
306

Christoph Lameter's avatar
Christoph Lameter committed
307
/* Internal SLUB flags */
308
/* Poison object */
309
#define __OBJECT_POISON		__SLAB_FLAG_BIT(_SLAB_OBJECT_POISON)
310
/* Use cmpxchg_double */
311 312

#ifdef system_has_freelist_aba
313
#define __CMPXCHG_DOUBLE	__SLAB_FLAG_BIT(_SLAB_CMPXCHG_DOUBLE)
314
#else
315
#define __CMPXCHG_DOUBLE	__SLAB_FLAG_UNUSED
316
#endif
Christoph Lameter's avatar
Christoph Lameter committed
317

318 319 320
/*
 * Tracking user of a slab.
 */
321
#define TRACK_ADDRS_COUNT 16
322
struct track {
323
	unsigned long addr;	/* Called from address */
324 325
#ifdef CONFIG_STACKDEPOT
	depot_stack_handle_t handle;
326
#endif
327 328 329 330 331 332 333
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
};

enum track_item { TRACK_ALLOC, TRACK_FREE };

334
#ifdef SLAB_SUPPORTS_SYSFS
Christoph Lameter's avatar
Christoph Lameter committed
335 336 337
static int sysfs_slab_add(struct kmem_cache *);
static int sysfs_slab_alias(struct kmem_cache *, const char *);
#else
338 339 340
static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; }
static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p)
							{ return 0; }
Christoph Lameter's avatar
Christoph Lameter committed
341 342
#endif

343 344 345 346 347 348
#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_SLUB_DEBUG)
static void debugfs_slab_add(struct kmem_cache *);
#else
static inline void debugfs_slab_add(struct kmem_cache *s) { }
#endif

349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393
enum stat_item {
	ALLOC_FASTPATH,		/* Allocation from cpu slab */
	ALLOC_SLOWPATH,		/* Allocation by getting a new cpu slab */
	FREE_FASTPATH,		/* Free to cpu slab */
	FREE_SLOWPATH,		/* Freeing not to cpu slab */
	FREE_FROZEN,		/* Freeing to frozen slab */
	FREE_ADD_PARTIAL,	/* Freeing moves slab to partial list */
	FREE_REMOVE_PARTIAL,	/* Freeing removes last object */
	ALLOC_FROM_PARTIAL,	/* Cpu slab acquired from node partial list */
	ALLOC_SLAB,		/* Cpu slab acquired from page allocator */
	ALLOC_REFILL,		/* Refill cpu slab from slab freelist */
	ALLOC_NODE_MISMATCH,	/* Switching cpu slab */
	FREE_SLAB,		/* Slab freed to the page allocator */
	CPUSLAB_FLUSH,		/* Abandoning of the cpu slab */
	DEACTIVATE_FULL,	/* Cpu slab was full when deactivated */
	DEACTIVATE_EMPTY,	/* Cpu slab was empty when deactivated */
	DEACTIVATE_TO_HEAD,	/* Cpu slab was moved to the head of partials */
	DEACTIVATE_TO_TAIL,	/* Cpu slab was moved to the tail of partials */
	DEACTIVATE_REMOTE_FREES,/* Slab contained remotely freed objects */
	DEACTIVATE_BYPASS,	/* Implicit deactivation */
	ORDER_FALLBACK,		/* Number of times fallback was necessary */
	CMPXCHG_DOUBLE_CPU_FAIL,/* Failures of this_cpu_cmpxchg_double */
	CMPXCHG_DOUBLE_FAIL,	/* Failures of slab freelist update */
	CPU_PARTIAL_ALLOC,	/* Used cpu partial on alloc */
	CPU_PARTIAL_FREE,	/* Refill cpu partial on free */
	CPU_PARTIAL_NODE,	/* Refill cpu partial from node partial */
	CPU_PARTIAL_DRAIN,	/* Drain cpu partial to node partial */
	NR_SLUB_STAT_ITEMS
};

#ifndef CONFIG_SLUB_TINY
/*
 * When changing the layout, make sure freelist and tid are still compatible
 * with this_cpu_cmpxchg_double() alignment requirements.
 */
struct kmem_cache_cpu {
	union {
		struct {
			void **freelist;	/* Pointer to next available object */
			unsigned long tid;	/* Globally unique transaction id */
		};
		freelist_aba_t freelist_tid;
	};
	struct slab *slab;	/* The slab from which we are allocating */
#ifdef CONFIG_SLUB_CPU_PARTIAL
394
	struct slab *partial;	/* Partially allocated slabs */
395 396 397 398 399 400 401 402
#endif
	local_lock_t lock;	/* Protects the fields above */
#ifdef CONFIG_SLUB_STATS
	unsigned int stat[NR_SLUB_STAT_ITEMS];
#endif
};
#endif /* CONFIG_SLUB_TINY */

403
static inline void stat(const struct kmem_cache *s, enum stat_item si)
404 405
{
#ifdef CONFIG_SLUB_STATS
406 407 408 409 410
	/*
	 * The rmw is racy on a preemptible kernel but this is acceptable, so
	 * avoid this_cpu_add()'s irq-disable overhead.
	 */
	raw_cpu_inc(s->cpu_slab->stat[si]);
411 412 413
#endif
}

414 415 416 417 418 419 420 421
static inline
void stat_add(const struct kmem_cache *s, enum stat_item si, int v)
{
#ifdef CONFIG_SLUB_STATS
	raw_cpu_add(s->cpu_slab->stat[si], v);
#endif
}

422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448
/*
 * The slab lists for all objects.
 */
struct kmem_cache_node {
	spinlock_t list_lock;
	unsigned long nr_partial;
	struct list_head partial;
#ifdef CONFIG_SLUB_DEBUG
	atomic_long_t nr_slabs;
	atomic_long_t total_objects;
	struct list_head full;
#endif
};

static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node)
{
	return s->node[node];
}

/*
 * Iterator over all nodes. The body will be executed for each node that has
 * a kmem_cache_node structure allocated (which is true for all online nodes)
 */
#define for_each_kmem_cache_node(__s, __node, __n) \
	for (__node = 0; __node < nr_node_ids; __node++) \
		 if ((__n = get_node(__s, __node)))

449 450 451 452 453 454 455 456
/*
 * Tracks for which NUMA nodes we have kmem_cache_nodes allocated.
 * Corresponds to node_state[N_NORMAL_MEMORY], but can temporarily
 * differ during memory hotplug/hotremove operations.
 * Protected by slab_mutex.
 */
static nodemask_t slab_nodes;

457
#ifndef CONFIG_SLUB_TINY
458 459 460 461
/*
 * Workqueue used for flush_cpu_slab().
 */
static struct workqueue_struct *flushwq;
462
#endif
463

Christoph Lameter's avatar
Christoph Lameter committed
464 465 466 467
/********************************************************************
 * 			Core slab cache functions
 *******************************************************************/

468 469 470 471 472 473
/*
 * freeptr_t represents a SLUB freelist pointer, which might be encoded
 * and not dereferenceable if CONFIG_SLAB_FREELIST_HARDENED is enabled.
 */
typedef struct { unsigned long v; } freeptr_t;

474 475 476 477 478
/*
 * Returns freelist pointer (ptr). With hardening, this is obfuscated
 * with an XOR of the address where the pointer is held and a per-cache
 * random number.
 */
479 480
static inline freeptr_t freelist_ptr_encode(const struct kmem_cache *s,
					    void *ptr, unsigned long ptr_addr)
481
{
482 483
	unsigned long encoded;

484
#ifdef CONFIG_SLAB_FREELIST_HARDENED
485
	encoded = (unsigned long)ptr ^ s->random ^ swab(ptr_addr);
486
#else
487
	encoded = (unsigned long)ptr;
488
#endif
489
	return (freeptr_t){.v = encoded};
490 491 492 493 494 495 496 497
}

static inline void *freelist_ptr_decode(const struct kmem_cache *s,
					freeptr_t ptr, unsigned long ptr_addr)
{
	void *decoded;

#ifdef CONFIG_SLAB_FREELIST_HARDENED
498
	decoded = (void *)(ptr.v ^ s->random ^ swab(ptr_addr));
499
#else
500
	decoded = (void *)ptr.v;
501
#endif
502
	return decoded;
503 504
}

505 506
static inline void *get_freepointer(struct kmem_cache *s, void *object)
{
507 508 509
	unsigned long ptr_addr;
	freeptr_t p;

510
	object = kasan_reset_tag(object);
511 512 513
	ptr_addr = (unsigned long)object + s->offset;
	p = *(freeptr_t *)(ptr_addr);
	return freelist_ptr_decode(s, p, ptr_addr);
514 515
}

516
#ifndef CONFIG_SLUB_TINY
517 518
static void prefetch_freepointer(const struct kmem_cache *s, void *object)
{
519
	prefetchw(object + s->offset);
520
}
521
#endif
522

523 524 525 526 527 528 529 530 531 532 533
/*
 * When running under KMSAN, get_freepointer_safe() may return an uninitialized
 * pointer value in the case the current thread loses the race for the next
 * memory chunk in the freelist. In that case this_cpu_cmpxchg_double() in
 * slab_alloc_node() will fail, so the uninitialized value won't be used, but
 * KMSAN will still check all arguments of cmpxchg because of imperfect
 * handling of inline assembly.
 * To work around this problem, we apply __no_kmsan_checks to ensure that
 * get_freepointer_safe() returns initialized memory.
 */
__no_kmsan_checks
534 535
static inline void *get_freepointer_safe(struct kmem_cache *s, void *object)
{
536
	unsigned long freepointer_addr;
537
	freeptr_t p;
538

539
	if (!debug_pagealloc_enabled_static())
540 541
		return get_freepointer(s, object);

542
	object = kasan_reset_tag(object);
543
	freepointer_addr = (unsigned long)object + s->offset;
544 545
	copy_from_kernel_nofault(&p, (freeptr_t *)freepointer_addr, sizeof(p));
	return freelist_ptr_decode(s, p, freepointer_addr);
546 547
}

548 549
static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
{
550 551
	unsigned long freeptr_addr = (unsigned long)object + s->offset;

552 553 554 555
#ifdef CONFIG_SLAB_FREELIST_HARDENED
	BUG_ON(object == fp); /* naive detection of double free or corruption */
#endif

556
	freeptr_addr = (unsigned long)kasan_reset_tag((void *)freeptr_addr);
557
	*(freeptr_t *)freeptr_addr = freelist_ptr_encode(s, fp, freeptr_addr);
558 559 560
}

/* Loop over all objects in a slab */
561
#define for_each_object(__p, __s, __addr, __objects) \
562 563 564
	for (__p = fixup_red_left(__s, __addr); \
		__p < (__addr) + (__objects) * (__s)->size; \
		__p += (__s)->size)
565

566
static inline unsigned int order_objects(unsigned int order, unsigned int size)
567
{
568
	return ((unsigned int)PAGE_SIZE << order) / size;
569 570
}

571
static inline struct kmem_cache_order_objects oo_make(unsigned int order,
572
		unsigned int size)
573 574
{
	struct kmem_cache_order_objects x = {
575
		(order << OO_SHIFT) + order_objects(order, size)
576 577 578 579 580
	};

	return x;
}

581
static inline unsigned int oo_order(struct kmem_cache_order_objects x)
582
{
583
	return x.x >> OO_SHIFT;
584 585
}

586
static inline unsigned int oo_objects(struct kmem_cache_order_objects x)
587
{
588
	return x.x & OO_MASK;
589 590
}

591 592 593
#ifdef CONFIG_SLUB_CPU_PARTIAL
static void slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects)
{
594
	unsigned int nr_slabs;
595 596 597 598 599

	s->cpu_partial = nr_objects;

	/*
	 * We take the number of objects but actually limit the number of
600 601
	 * slabs on the per cpu partial list, in order to limit excessive
	 * growth of the list. For simplicity we assume that the slabs will
602 603
	 * be half-full.
	 */
604 605
	nr_slabs = DIV_ROUND_UP(nr_objects * 2, oo_objects(s->oo));
	s->cpu_partial_slabs = nr_slabs;
606 607 608 609 610 611 612 613
}
#else
static inline void
slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects)
{
}
#endif /* CONFIG_SLUB_CPU_PARTIAL */

614 615 616
/*
 * Per slab locking using the pagelock
 */
617
static __always_inline void slab_lock(struct slab *slab)
618
{
619 620
	struct page *page = slab_page(slab);

621
	VM_BUG_ON_PAGE(PageTail(page), page);
622 623 624
	bit_spin_lock(PG_locked, &page->flags);
}

625
static __always_inline void slab_unlock(struct slab *slab)
626
{
627 628
	struct page *page = slab_page(slab);

629
	VM_BUG_ON_PAGE(PageTail(page), page);
630
	bit_spin_unlock(PG_locked, &page->flags);
631 632
}

633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666
static inline bool
__update_freelist_fast(struct slab *slab,
		      void *freelist_old, unsigned long counters_old,
		      void *freelist_new, unsigned long counters_new)
{
#ifdef system_has_freelist_aba
	freelist_aba_t old = { .freelist = freelist_old, .counter = counters_old };
	freelist_aba_t new = { .freelist = freelist_new, .counter = counters_new };

	return try_cmpxchg_freelist(&slab->freelist_counter.full, &old.full, new.full);
#else
	return false;
#endif
}

static inline bool
__update_freelist_slow(struct slab *slab,
		      void *freelist_old, unsigned long counters_old,
		      void *freelist_new, unsigned long counters_new)
{
	bool ret = false;

	slab_lock(slab);
	if (slab->freelist == freelist_old &&
	    slab->counters == counters_old) {
		slab->freelist = freelist_new;
		slab->counters = counters_new;
		ret = true;
	}
	slab_unlock(slab);

	return ret;
}

667 668
/*
 * Interrupts must be disabled (for the fallback code to work right), typically
669 670 671 672
 * by an _irqsave() lock variant. On PREEMPT_RT the preempt_disable(), which is
 * part of bit_spin_lock(), is sufficient because the policy is not to allow any
 * allocation/ free operation in hardirq context. Therefore nothing can
 * interrupt the operation.
673
 */
674
static inline bool __slab_update_freelist(struct kmem_cache *s, struct slab *slab,
675 676 677 678
		void *freelist_old, unsigned long counters_old,
		void *freelist_new, unsigned long counters_new,
		const char *n)
{
679 680
	bool ret;

681
	if (USE_LOCKLESS_FAST_PATH())
682
		lockdep_assert_irqs_disabled();
683

684
	if (s->flags & __CMPXCHG_DOUBLE) {
685 686 687 688 689
		ret = __update_freelist_fast(slab, freelist_old, counters_old,
				            freelist_new, counters_new);
	} else {
		ret = __update_freelist_slow(slab, freelist_old, counters_old,
				            freelist_new, counters_new);
690
	}
691 692
	if (likely(ret))
		return true;
693 694 695 696 697

	cpu_relax();
	stat(s, CMPXCHG_DOUBLE_FAIL);

#ifdef SLUB_DEBUG_CMPXCHG
698
	pr_info("%s %s: cmpxchg double redo ", n, s->name);
699 700
#endif

701
	return false;
702 703
}

704
static inline bool slab_update_freelist(struct kmem_cache *s, struct slab *slab,
705 706 707 708
		void *freelist_old, unsigned long counters_old,
		void *freelist_new, unsigned long counters_new,
		const char *n)
{
709 710
	bool ret;

711
	if (s->flags & __CMPXCHG_DOUBLE) {
712 713 714
		ret = __update_freelist_fast(slab, freelist_old, counters_old,
				            freelist_new, counters_new);
	} else {
715 716 717
		unsigned long flags;

		local_irq_save(flags);
718 719
		ret = __update_freelist_slow(slab, freelist_old, counters_old,
				            freelist_new, counters_new);
720
		local_irq_restore(flags);
721
	}
722 723
	if (likely(ret))
		return true;
724 725 726 727 728

	cpu_relax();
	stat(s, CMPXCHG_DOUBLE_FAIL);

#ifdef SLUB_DEBUG_CMPXCHG
729
	pr_info("%s %s: cmpxchg double redo ", n, s->name);
730 731
#endif

732
	return false;
733 734
}

735
#ifdef CONFIG_SLUB_DEBUG
736
static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];
737
static DEFINE_SPINLOCK(object_map_lock);
738

739
static void __fill_map(unsigned long *obj_map, struct kmem_cache *s,
740
		       struct slab *slab)
741
{
742
	void *addr = slab_address(slab);
743 744
	void *p;

745
	bitmap_zero(obj_map, slab->objects);
746

747
	for (p = slab->freelist; p; p = get_freepointer(s, p))
748 749 750
		set_bit(__obj_to_index(s, addr, p), obj_map);
}

751 752 753 754 755
#if IS_ENABLED(CONFIG_KUNIT)
static bool slab_add_kunit_errors(void)
{
	struct kunit_resource *resource;

756
	if (!kunit_get_current_test())
757 758 759 760 761 762 763 764 765 766 767 768 769 770
		return false;

	resource = kunit_find_named_resource(current->kunit_test, "slab_errors");
	if (!resource)
		return false;

	(*(int *)resource->data)++;
	kunit_put_resource(resource);
	return true;
}
#else
static inline bool slab_add_kunit_errors(void) { return false; }
#endif

771
static inline unsigned int size_from_object(struct kmem_cache *s)
772 773 774 775 776 777 778 779 780 781 782 783 784 785 786
{
	if (s->flags & SLAB_RED_ZONE)
		return s->size - s->red_left_pad;

	return s->size;
}

static inline void *restore_red_left(struct kmem_cache *s, void *p)
{
	if (s->flags & SLAB_RED_ZONE)
		p -= s->red_left_pad;

	return p;
}

787 788 789
/*
 * Debug settings:
 */
790
#if defined(CONFIG_SLUB_DEBUG_ON)
791
static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;
792
#else
793
static slab_flags_t slub_debug;
794
#endif
795

796
static char *slub_debug_string;
797
static int disable_higher_order_debug;
798

799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814
/*
 * slub is about to manipulate internal object metadata.  This memory lies
 * outside the range of the allocated object, so accessing it would normally
 * be reported by kasan as a bounds error.  metadata_access_enable() is used
 * to tell kasan that these accesses are OK.
 */
static inline void metadata_access_enable(void)
{
	kasan_disable_current();
}

static inline void metadata_access_disable(void)
{
	kasan_enable_current();
}

Christoph Lameter's avatar
Christoph Lameter committed
815 816 817
/*
 * Object debugging
 */
818 819 820

/* Verify that a pointer has an address that is valid within a slab page */
static inline int check_valid_pointer(struct kmem_cache *s,
821
				struct slab *slab, void *object)
822 823 824 825 826 827
{
	void *base;

	if (!object)
		return 1;

828
	base = slab_address(slab);
829
	object = kasan_reset_tag(object);
830
	object = restore_red_left(s, object);
831
	if (object < base || object >= base + slab->objects * s->size ||
832 833 834 835 836 837 838
		(object - base) % s->size) {
		return 0;
	}

	return 1;
}

839 840
static void print_section(char *level, char *text, u8 *addr,
			  unsigned int length)
Christoph Lameter's avatar
Christoph Lameter committed
841
{
842
	metadata_access_enable();
843 844
	print_hex_dump(level, text, DUMP_PREFIX_ADDRESS,
			16, 1, kasan_reset_tag((void *)addr), length, 1);
845
	metadata_access_disable();
Christoph Lameter's avatar
Christoph Lameter committed
846 847
}

848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867
/*
 * See comment in calculate_sizes().
 */
static inline bool freeptr_outside_object(struct kmem_cache *s)
{
	return s->offset >= s->inuse;
}

/*
 * Return offset of the end of info block which is inuse + free pointer if
 * not overlapping with object.
 */
static inline unsigned int get_info_end(struct kmem_cache *s)
{
	if (freeptr_outside_object(s))
		return s->inuse + sizeof(void *);
	else
		return s->inuse;
}

Christoph Lameter's avatar
Christoph Lameter committed
868 869 870 871 872
static struct track *get_track(struct kmem_cache *s, void *object,
	enum track_item alloc)
{
	struct track *p;

873
	p = object + get_info_end(s);
Christoph Lameter's avatar
Christoph Lameter committed
874

875
	return kasan_reset_tag(p + alloc);
Christoph Lameter's avatar
Christoph Lameter committed
876 877
}

878
#ifdef CONFIG_STACKDEPOT
879 880 881
static noinline depot_stack_handle_t set_track_prepare(void)
{
	depot_stack_handle_t handle;
882
	unsigned long entries[TRACK_ADDRS_COUNT];
883
	unsigned int nr_entries;
884

885
	nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 3);
886 887 888 889 890 891 892 893 894
	handle = stack_depot_save(entries, nr_entries, GFP_NOWAIT);

	return handle;
}
#else
static inline depot_stack_handle_t set_track_prepare(void)
{
	return 0;
}
895
#endif
896

897 898 899 900 901 902 903 904 905
static void set_track_update(struct kmem_cache *s, void *object,
			     enum track_item alloc, unsigned long addr,
			     depot_stack_handle_t handle)
{
	struct track *p = get_track(s, object, alloc);

#ifdef CONFIG_STACKDEPOT
	p->handle = handle;
#endif
906 907 908 909
	p->addr = addr;
	p->cpu = smp_processor_id();
	p->pid = current->pid;
	p->when = jiffies;
Christoph Lameter's avatar
Christoph Lameter committed
910 911
}

912 913 914 915 916 917 918 919
static __always_inline void set_track(struct kmem_cache *s, void *object,
				      enum track_item alloc, unsigned long addr)
{
	depot_stack_handle_t handle = set_track_prepare();

	set_track_update(s, object, alloc, addr, handle);
}

Christoph Lameter's avatar
Christoph Lameter committed
920 921
static void init_tracking(struct kmem_cache *s, void *object)
{
922 923
	struct track *p;

924 925 926
	if (!(s->flags & SLAB_STORE_USER))
		return;

927 928
	p = get_track(s, object, TRACK_ALLOC);
	memset(p, 0, 2*sizeof(struct track));
Christoph Lameter's avatar
Christoph Lameter committed
929 930
}

931
static void print_track(const char *s, struct track *t, unsigned long pr_time)
Christoph Lameter's avatar
Christoph Lameter committed
932
{
933 934
	depot_stack_handle_t handle __maybe_unused;

Christoph Lameter's avatar
Christoph Lameter committed
935 936 937
	if (!t->addr)
		return;

938
	pr_err("%s in %pS age=%lu cpu=%u pid=%d\n",
939
	       s, (void *)t->addr, pr_time - t->when, t->cpu, t->pid);
940 941 942 943 944 945
#ifdef CONFIG_STACKDEPOT
	handle = READ_ONCE(t->handle);
	if (handle)
		stack_depot_print(handle);
	else
		pr_err("object allocation/free stack trace missing\n");
946
#endif
947 948
}

949
void print_tracking(struct kmem_cache *s, void *object)
950
{
951
	unsigned long pr_time = jiffies;
952 953 954
	if (!(s->flags & SLAB_STORE_USER))
		return;

955 956
	print_track("Allocated", get_track(s, object, TRACK_ALLOC), pr_time);
	print_track("Freed", get_track(s, object, TRACK_FREE), pr_time);
957 958
}

959
static void print_slab_info(const struct slab *slab)
960
{
961
	struct folio *folio = (struct folio *)slab_folio(slab);
962

963 964 965
	pr_err("Slab 0x%p objects=%u used=%u fp=0x%p flags=%pGp\n",
	       slab, slab->objects, slab->inuse, slab->freelist,
	       folio_flags(folio, 0));
966 967
}

968 969 970 971 972 973 974 975 976 977
/*
 * kmalloc caches has fixed sizes (mostly power of 2), and kmalloc() API
 * family will round up the real request size to these fixed ones, so
 * there could be an extra area than what is requested. Save the original
 * request size in the meta data area, for better debug and sanity check.
 */
static inline void set_orig_size(struct kmem_cache *s,
				void *object, unsigned int orig_size)
{
	void *p = kasan_reset_tag(object);
978
	unsigned int kasan_meta_size;
979 980 981 982

	if (!slub_debug_orig_size(s))
		return;

983
	/*
984 985 986 987
	 * KASAN can save its free meta data inside of the object at offset 0.
	 * If this meta data size is larger than 'orig_size', it will overlap
	 * the data redzone in [orig_size+1, object_size]. Thus, we adjust
	 * 'orig_size' to be as at least as big as KASAN's meta data.
988
	 */
989 990 991
	kasan_meta_size = kasan_metadata_size(s, true);
	if (kasan_meta_size > orig_size)
		orig_size = kasan_meta_size;
992

993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011
	p += get_info_end(s);
	p += sizeof(struct track) * 2;

	*(unsigned int *)p = orig_size;
}

static inline unsigned int get_orig_size(struct kmem_cache *s, void *object)
{
	void *p = kasan_reset_tag(object);

	if (!slub_debug_orig_size(s))
		return s->object_size;

	p += get_info_end(s);
	p += sizeof(struct track) * 2;

	return *(unsigned int *)p;
}

1012 1013 1014 1015 1016
void skip_orig_size_check(struct kmem_cache *s, const void *object)
{
	set_orig_size(s, (void *)object, s->object_size);
}

1017 1018
static void slab_bug(struct kmem_cache *s, char *fmt, ...)
{
1019
	struct va_format vaf;
1020 1021 1022
	va_list args;

	va_start(args, fmt);
1023 1024
	vaf.fmt = fmt;
	vaf.va = &args;
1025
	pr_err("=============================================================================\n");
1026
	pr_err("BUG %s (%s): %pV\n", s->name, print_tainted(), &vaf);
1027
	pr_err("-----------------------------------------------------------------------------\n\n");
1028
	va_end(args);
Christoph Lameter's avatar
Christoph Lameter committed
1029 1030
}

1031
__printf(2, 3)
1032 1033
static void slab_fix(struct kmem_cache *s, char *fmt, ...)
{
1034
	struct va_format vaf;
1035 1036
	va_list args;

1037 1038 1039
	if (slab_add_kunit_errors())
		return;

1040
	va_start(args, fmt);
1041 1042 1043
	vaf.fmt = fmt;
	vaf.va = &args;
	pr_err("FIX %s: %pV\n", s->name, &vaf);
1044 1045 1046
	va_end(args);
}

1047
static void print_trailer(struct kmem_cache *s, struct slab *slab, u8 *p)
Christoph Lameter's avatar
Christoph Lameter committed
1048 1049
{
	unsigned int off;	/* Offset of last byte */
1050
	u8 *addr = slab_address(slab);
1051 1052 1053

	print_tracking(s, p);

1054
	print_slab_info(slab);
1055

1056
	pr_err("Object 0x%p @offset=%tu fp=0x%p\n\n",
1057
	       p, p - addr, get_freepointer(s, p));
1058

1059
	if (s->flags & SLAB_RED_ZONE)
1060
		print_section(KERN_ERR, "Redzone  ", p - s->red_left_pad,
1061
			      s->red_left_pad);
1062
	else if (p > addr + 16)
1063
		print_section(KERN_ERR, "Bytes b4 ", p - 16, 16);
Christoph Lameter's avatar
Christoph Lameter committed
1064

1065
	print_section(KERN_ERR,         "Object   ", p,
1066
		      min_t(unsigned int, s->object_size, PAGE_SIZE));
Christoph Lameter's avatar
Christoph Lameter committed
1067
	if (s->flags & SLAB_RED_ZONE)
1068
		print_section(KERN_ERR, "Redzone  ", p + s->object_size,
1069
			s->inuse - s->object_size);
Christoph Lameter's avatar
Christoph Lameter committed
1070

1071
	off = get_info_end(s);
Christoph Lameter's avatar
Christoph Lameter committed
1072

1073
	if (s->flags & SLAB_STORE_USER)
Christoph Lameter's avatar
Christoph Lameter committed
1074 1075
		off += 2 * sizeof(struct track);

1076 1077 1078
	if (slub_debug_orig_size(s))
		off += sizeof(unsigned int);

1079
	off += kasan_metadata_size(s, false);
1080

1081
	if (off != size_from_object(s))
Christoph Lameter's avatar
Christoph Lameter committed
1082
		/* Beginning of the filler is the free pointer */
1083
		print_section(KERN_ERR, "Padding  ", p + off,
1084
			      size_from_object(s) - off);
1085 1086

	dump_stack();
Christoph Lameter's avatar
Christoph Lameter committed
1087 1088
}

1089
static void object_err(struct kmem_cache *s, struct slab *slab,
Christoph Lameter's avatar
Christoph Lameter committed
1090 1091
			u8 *object, char *reason)
{
1092 1093 1094
	if (slab_add_kunit_errors())
		return;

1095
	slab_bug(s, "%s", reason);
1096
	print_trailer(s, slab, object);
1097
	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
Christoph Lameter's avatar
Christoph Lameter committed
1098 1099
}

1100
static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab,
1101 1102 1103
			       void **freelist, void *nextfree)
{
	if ((s->flags & SLAB_CONSISTENCY_CHECKS) &&
1104 1105
	    !check_valid_pointer(s, slab, nextfree) && freelist) {
		object_err(s, slab, *freelist, "Freechain corrupt");
1106 1107 1108 1109 1110 1111 1112 1113
		*freelist = NULL;
		slab_fix(s, "Isolate corrupted freechain");
		return true;
	}

	return false;
}

1114
static __printf(3, 4) void slab_err(struct kmem_cache *s, struct slab *slab,
1115
			const char *fmt, ...)
Christoph Lameter's avatar
Christoph Lameter committed
1116 1117 1118 1119
{
	va_list args;
	char buf[100];

1120 1121 1122
	if (slab_add_kunit_errors())
		return;

1123 1124
	va_start(args, fmt);
	vsnprintf(buf, sizeof(buf), fmt, args);
Christoph Lameter's avatar
Christoph Lameter committed
1125
	va_end(args);
1126
	slab_bug(s, "%s", buf);
1127
	print_slab_info(slab);
Christoph Lameter's avatar
Christoph Lameter committed
1128
	dump_stack();
1129
	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
Christoph Lameter's avatar
Christoph Lameter committed
1130 1131
}

1132
static void init_object(struct kmem_cache *s, void *object, u8 val)
Christoph Lameter's avatar
Christoph Lameter committed
1133
{
1134
	u8 *p = kasan_reset_tag(object);
1135
	unsigned int poison_size = s->object_size;
Christoph Lameter's avatar
Christoph Lameter committed
1136

1137
	if (s->flags & SLAB_RED_ZONE) {
1138 1139
		memset(p - s->red_left_pad, val, s->red_left_pad);

1140 1141 1142 1143 1144 1145 1146 1147 1148 1149
		if (slub_debug_orig_size(s) && val == SLUB_RED_ACTIVE) {
			/*
			 * Redzone the extra allocated space by kmalloc than
			 * requested, and the poison size will be limited to
			 * the original request size accordingly.
			 */
			poison_size = get_orig_size(s, object);
		}
	}

Christoph Lameter's avatar
Christoph Lameter committed
1150
	if (s->flags & __OBJECT_POISON) {
1151 1152
		memset(p, POISON_FREE, poison_size - 1);
		p[poison_size - 1] = POISON_END;
Christoph Lameter's avatar
Christoph Lameter committed
1153 1154 1155
	}

	if (s->flags & SLAB_RED_ZONE)
1156
		memset(p + poison_size, val, s->inuse - poison_size);
Christoph Lameter's avatar
Christoph Lameter committed
1157 1158
}

1159 1160 1161
static void restore_bytes(struct kmem_cache *s, char *message, u8 data,
						void *from, void *to)
{
1162
	slab_fix(s, "Restoring %s 0x%p-0x%p=0x%x", message, from, to - 1, data);
1163 1164 1165
	memset(from, data, to - from);
}

1166
static int check_bytes_and_report(struct kmem_cache *s, struct slab *slab,
1167
			u8 *object, char *what,
1168
			u8 *start, unsigned int value, unsigned int bytes)
1169 1170 1171
{
	u8 *fault;
	u8 *end;
1172
	u8 *addr = slab_address(slab);
1173

1174
	metadata_access_enable();
1175
	fault = memchr_inv(kasan_reset_tag(start), value, bytes);
1176
	metadata_access_disable();
1177 1178 1179 1180 1181 1182 1183
	if (!fault)
		return 1;

	end = start + bytes;
	while (end > fault && end[-1] == value)
		end--;

1184 1185 1186
	if (slab_add_kunit_errors())
		goto skip_bug_print;

1187
	slab_bug(s, "%s overwritten", what);
1188
	pr_err("0x%p-0x%p @offset=%tu. First byte 0x%x instead of 0x%x\n",
1189 1190
					fault, end - 1, fault - addr,
					fault[0], value);
1191
	print_trailer(s, slab, object);
1192
	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
1193

1194
skip_bug_print:
1195 1196
	restore_bytes(s, what, value, fault, end);
	return 0;
Christoph Lameter's avatar
Christoph Lameter committed
1197 1198 1199 1200 1201 1202 1203 1204
}

/*
 * Object layout:
 *
 * object address
 * 	Bytes of the object to be managed.
 * 	If the freepointer may overlay the object then the free
1205
 *	pointer is at the middle of the object.
Christoph Lameter's avatar
Christoph Lameter committed
1206
 *
Christoph Lameter's avatar
Christoph Lameter committed
1207 1208 1209
 * 	Poisoning uses 0x6b (POISON_FREE) and the last byte is
 * 	0xa5 (POISON_END)
 *
1210
 * object + s->object_size
Christoph Lameter's avatar
Christoph Lameter committed
1211
 * 	Padding to reach word boundary. This is also used for Redzoning.
Christoph Lameter's avatar
Christoph Lameter committed
1212
 * 	Padding is extended by another word if Redzoning is enabled and
1213
 * 	object_size == inuse.
Christoph Lameter's avatar
Christoph Lameter committed
1214
 *
Christoph Lameter's avatar
Christoph Lameter committed
1215 1216 1217 1218
 * 	We fill with 0xbb (RED_INACTIVE) for inactive objects and with
 * 	0xcc (RED_ACTIVE) for objects in use.
 *
 * object + s->inuse
Christoph Lameter's avatar
Christoph Lameter committed
1219 1220
 * 	Meta data starts here.
 *
Christoph Lameter's avatar
Christoph Lameter committed
1221 1222
 * 	A. Free pointer (if we cannot overwrite object on free)
 * 	B. Tracking data for SLAB_STORE_USER
1223 1224
 *	C. Original request size for kmalloc object (SLAB_STORE_USER enabled)
 *	D. Padding to reach required alignment boundary or at minimum
Christoph Lameter's avatar
Christoph Lameter committed
1225
 * 		one word if debugging is on to be able to detect writes
Christoph Lameter's avatar
Christoph Lameter committed
1226 1227 1228
 * 		before the word boundary.
 *
 *	Padding is done using 0x5a (POISON_INUSE)
Christoph Lameter's avatar
Christoph Lameter committed
1229 1230
 *
 * object + s->size
Christoph Lameter's avatar
Christoph Lameter committed
1231
 * 	Nothing is used beyond s->size.
Christoph Lameter's avatar
Christoph Lameter committed
1232
 *
1233
 * If slabcaches are merged then the object_size and inuse boundaries are mostly
Christoph Lameter's avatar
Christoph Lameter committed
1234
 * ignored. And therefore no slab options that rely on these boundaries
Christoph Lameter's avatar
Christoph Lameter committed
1235 1236 1237
 * may be used with merged slabcaches.
 */

1238
static int check_pad_bytes(struct kmem_cache *s, struct slab *slab, u8 *p)
Christoph Lameter's avatar
Christoph Lameter committed
1239
{
1240
	unsigned long off = get_info_end(s);	/* The end of info */
Christoph Lameter's avatar
Christoph Lameter committed
1241

1242
	if (s->flags & SLAB_STORE_USER) {
Christoph Lameter's avatar
Christoph Lameter committed
1243 1244 1245
		/* We also have user information there */
		off += 2 * sizeof(struct track);

1246 1247 1248 1249
		if (s->flags & SLAB_KMALLOC)
			off += sizeof(unsigned int);
	}

1250
	off += kasan_metadata_size(s, false);
1251

1252
	if (size_from_object(s) == off)
Christoph Lameter's avatar
Christoph Lameter committed
1253 1254
		return 1;

1255
	return check_bytes_and_report(s, slab, p, "Object padding",
1256
			p + off, POISON_INUSE, size_from_object(s) - off);
Christoph Lameter's avatar
Christoph Lameter committed
1257 1258
}

1259
/* Check the pad bytes at the end of a slab page */
1260
static void slab_pad_check(struct kmem_cache *s, struct slab *slab)
Christoph Lameter's avatar
Christoph Lameter committed
1261
{
1262 1263 1264
	u8 *start;
	u8 *fault;
	u8 *end;
1265
	u8 *pad;
1266 1267
	int length;
	int remainder;
Christoph Lameter's avatar
Christoph Lameter committed
1268 1269

	if (!(s->flags & SLAB_POISON))
1270
		return;
Christoph Lameter's avatar
Christoph Lameter committed
1271

1272 1273
	start = slab_address(slab);
	length = slab_size(slab);
1274 1275
	end = start + length;
	remainder = length % s->size;
Christoph Lameter's avatar
Christoph Lameter committed
1276
	if (!remainder)
1277
		return;
Christoph Lameter's avatar
Christoph Lameter committed
1278

1279
	pad = end - remainder;
1280
	metadata_access_enable();
1281
	fault = memchr_inv(kasan_reset_tag(pad), POISON_INUSE, remainder);
1282
	metadata_access_disable();
1283
	if (!fault)
1284
		return;
1285 1286 1287
	while (end > fault && end[-1] == POISON_INUSE)
		end--;

1288
	slab_err(s, slab, "Padding overwritten. 0x%p-0x%p @offset=%tu",
1289
			fault, end - 1, fault - start);
1290
	print_section(KERN_ERR, "Padding ", pad, remainder);
1291

1292
	restore_bytes(s, "slab padding", POISON_INUSE, fault, end);
Christoph Lameter's avatar
Christoph Lameter committed
1293 1294
}

1295
static int check_object(struct kmem_cache *s, struct slab *slab,
1296
					void *object, u8 val)
Christoph Lameter's avatar
Christoph Lameter committed
1297 1298
{
	u8 *p = object;
1299
	u8 *endobject = object + s->object_size;
1300
	unsigned int orig_size, kasan_meta_size;
Christoph Lameter's avatar
Christoph Lameter committed
1301 1302

	if (s->flags & SLAB_RED_ZONE) {
1303
		if (!check_bytes_and_report(s, slab, object, "Left Redzone",
1304 1305 1306
			object - s->red_left_pad, val, s->red_left_pad))
			return 0;

1307
		if (!check_bytes_and_report(s, slab, object, "Right Redzone",
1308
			endobject, val, s->inuse - s->object_size))
Christoph Lameter's avatar
Christoph Lameter committed
1309
			return 0;
1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320

		if (slub_debug_orig_size(s) && val == SLUB_RED_ACTIVE) {
			orig_size = get_orig_size(s, object);

			if (s->object_size > orig_size  &&
				!check_bytes_and_report(s, slab, object,
					"kmalloc Redzone", p + orig_size,
					val, s->object_size - orig_size)) {
				return 0;
			}
		}
Christoph Lameter's avatar
Christoph Lameter committed
1321
	} else {
1322
		if ((s->flags & SLAB_POISON) && s->object_size < s->inuse) {
1323
			check_bytes_and_report(s, slab, p, "Alignment padding",
1324 1325
				endobject, POISON_INUSE,
				s->inuse - s->object_size);
Ingo Molnar's avatar
Ingo Molnar committed
1326
		}
Christoph Lameter's avatar
Christoph Lameter committed
1327 1328 1329
	}

	if (s->flags & SLAB_POISON) {
1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346
		if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON)) {
			/*
			 * KASAN can save its free meta data inside of the
			 * object at offset 0. Thus, skip checking the part of
			 * the redzone that overlaps with the meta data.
			 */
			kasan_meta_size = kasan_metadata_size(s, true);
			if (kasan_meta_size < s->object_size - 1 &&
			    !check_bytes_and_report(s, slab, p, "Poison",
					p + kasan_meta_size, POISON_FREE,
					s->object_size - kasan_meta_size - 1))
				return 0;
			if (kasan_meta_size < s->object_size &&
			    !check_bytes_and_report(s, slab, p, "End Poison",
					p + s->object_size - 1, POISON_END, 1))
				return 0;
		}
Christoph Lameter's avatar
Christoph Lameter committed
1347 1348 1349
		/*
		 * check_pad_bytes cleans up on its own.
		 */
1350
		check_pad_bytes(s, slab, p);
Christoph Lameter's avatar
Christoph Lameter committed
1351 1352
	}

1353
	if (!freeptr_outside_object(s) && val == SLUB_RED_ACTIVE)
Christoph Lameter's avatar
Christoph Lameter committed
1354 1355 1356 1357 1358 1359 1360
		/*
		 * Object and freepointer overlap. Cannot check
		 * freepointer while object is allocated.
		 */
		return 1;

	/* Check free pointer validity */
1361 1362
	if (!check_valid_pointer(s, slab, get_freepointer(s, p))) {
		object_err(s, slab, p, "Freepointer corrupt");
Christoph Lameter's avatar
Christoph Lameter committed
1363
		/*
1364
		 * No choice but to zap it and thus lose the remainder
Christoph Lameter's avatar
Christoph Lameter committed
1365
		 * of the free objects in this slab. May cause
Christoph Lameter's avatar
Christoph Lameter committed
1366
		 * another error because the object count is now wrong.
Christoph Lameter's avatar
Christoph Lameter committed
1367
		 */
1368
		set_freepointer(s, p, NULL);
Christoph Lameter's avatar
Christoph Lameter committed
1369 1370 1371 1372 1373
		return 0;
	}
	return 1;
}

1374
static int check_slab(struct kmem_cache *s, struct slab *slab)
Christoph Lameter's avatar
Christoph Lameter committed
1375
{
1376 1377
	int maxobj;

1378 1379
	if (!folio_test_slab(slab_folio(slab))) {
		slab_err(s, slab, "Not a valid slab page");
Christoph Lameter's avatar
Christoph Lameter committed
1380 1381
		return 0;
	}
1382

1383 1384 1385 1386
	maxobj = order_objects(slab_order(slab), s->size);
	if (slab->objects > maxobj) {
		slab_err(s, slab, "objects %u > max %u",
			slab->objects, maxobj);
1387 1388
		return 0;
	}
1389 1390 1391
	if (slab->inuse > slab->objects) {
		slab_err(s, slab, "inuse %u > max %u",
			slab->inuse, slab->objects);
Christoph Lameter's avatar
Christoph Lameter committed
1392 1393 1394
		return 0;
	}
	/* Slab_pad_check fixes things up after itself */
1395
	slab_pad_check(s, slab);
Christoph Lameter's avatar
Christoph Lameter committed
1396 1397 1398 1399
	return 1;
}

/*
1400
 * Determine if a certain object in a slab is on the freelist. Must hold the
Christoph Lameter's avatar
Christoph Lameter committed
1401
 * slab lock to guarantee that the chains are in a consistent state.
Christoph Lameter's avatar
Christoph Lameter committed
1402
 */
1403
static int on_freelist(struct kmem_cache *s, struct slab *slab, void *search)
Christoph Lameter's avatar
Christoph Lameter committed
1404 1405
{
	int nr = 0;
1406
	void *fp;
Christoph Lameter's avatar
Christoph Lameter committed
1407
	void *object = NULL;
1408
	int max_objects;
Christoph Lameter's avatar
Christoph Lameter committed
1409

1410 1411
	fp = slab->freelist;
	while (fp && nr <= slab->objects) {
Christoph Lameter's avatar
Christoph Lameter committed
1412 1413
		if (fp == search)
			return 1;
1414
		if (!check_valid_pointer(s, slab, fp)) {
Christoph Lameter's avatar
Christoph Lameter committed
1415
			if (object) {
1416
				object_err(s, slab, object,
Christoph Lameter's avatar
Christoph Lameter committed
1417
					"Freechain corrupt");
1418
				set_freepointer(s, object, NULL);
Christoph Lameter's avatar
Christoph Lameter committed
1419
			} else {
1420 1421 1422
				slab_err(s, slab, "Freepointer corrupt");
				slab->freelist = NULL;
				slab->inuse = slab->objects;
1423
				slab_fix(s, "Freelist cleared");
Christoph Lameter's avatar
Christoph Lameter committed
1424 1425 1426 1427 1428 1429 1430 1431 1432
				return 0;
			}
			break;
		}
		object = fp;
		fp = get_freepointer(s, object);
		nr++;
	}

1433
	max_objects = order_objects(slab_order(slab), s->size);
1434 1435
	if (max_objects > MAX_OBJS_PER_PAGE)
		max_objects = MAX_OBJS_PER_PAGE;
1436

1437 1438 1439 1440
	if (slab->objects != max_objects) {
		slab_err(s, slab, "Wrong number of objects. Found %d but should be %d",
			 slab->objects, max_objects);
		slab->objects = max_objects;
1441
		slab_fix(s, "Number of objects adjusted");
1442
	}
1443 1444 1445 1446
	if (slab->inuse != slab->objects - nr) {
		slab_err(s, slab, "Wrong object count. Counter is %d but counted were %d",
			 slab->inuse, slab->objects - nr);
		slab->inuse = slab->objects - nr;
1447
		slab_fix(s, "Object count adjusted");
Christoph Lameter's avatar
Christoph Lameter committed
1448 1449 1450 1451
	}
	return search == NULL;
}

1452
static void trace(struct kmem_cache *s, struct slab *slab, void *object,
1453
								int alloc)
1454 1455
{
	if (s->flags & SLAB_TRACE) {
1456
		pr_info("TRACE %s %s 0x%p inuse=%d fp=0x%p\n",
1457 1458
			s->name,
			alloc ? "alloc" : "free",
1459 1460
			object, slab->inuse,
			slab->freelist);
1461 1462

		if (!alloc)
1463
			print_section(KERN_INFO, "Object ", (void *)object,
1464
					s->object_size);
1465 1466 1467 1468 1469

		dump_stack();
	}
}

1470
/*
Christoph Lameter's avatar
Christoph Lameter committed
1471
 * Tracking of fully allocated slabs for debugging purposes.
1472
 */
1473
static void add_full(struct kmem_cache *s,
1474
	struct kmem_cache_node *n, struct slab *slab)
1475
{
1476 1477 1478
	if (!(s->flags & SLAB_STORE_USER))
		return;

1479
	lockdep_assert_held(&n->list_lock);
1480
	list_add(&slab->slab_list, &n->full);
1481 1482
}

1483
static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct slab *slab)
1484 1485 1486 1487
{
	if (!(s->flags & SLAB_STORE_USER))
		return;

1488
	lockdep_assert_held(&n->list_lock);
1489
	list_del(&slab->slab_list);
1490 1491
}

1492 1493 1494 1495 1496
static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
{
	return atomic_long_read(&n->nr_slabs);
}

1497
static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects)
1498 1499 1500
{
	struct kmem_cache_node *n = get_node(s, node);

1501 1502
	atomic_long_inc(&n->nr_slabs);
	atomic_long_add(objects, &n->total_objects);
1503
}
1504
static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects)
1505 1506 1507 1508
{
	struct kmem_cache_node *n = get_node(s, node);

	atomic_long_dec(&n->nr_slabs);
1509
	atomic_long_sub(objects, &n->total_objects);
1510 1511 1512
}

/* Object debug checks for alloc/free paths */
1513
static void setup_object_debug(struct kmem_cache *s, void *object)
1514
{
1515
	if (!kmem_cache_debug_flags(s, SLAB_STORE_USER|SLAB_RED_ZONE|__OBJECT_POISON))
1516 1517
		return;

1518
	init_object(s, object, SLUB_RED_INACTIVE);
1519 1520 1521
	init_tracking(s, object);
}

1522
static
1523
void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr)
1524
{
1525
	if (!kmem_cache_debug_flags(s, SLAB_POISON))
1526 1527 1528
		return;

	metadata_access_enable();
1529
	memset(kasan_reset_tag(addr), POISON_INUSE, slab_size(slab));
1530 1531 1532
	metadata_access_disable();
}

1533
static inline int alloc_consistency_checks(struct kmem_cache *s,
1534
					struct slab *slab, void *object)
Christoph Lameter's avatar
Christoph Lameter committed
1535
{
1536
	if (!check_slab(s, slab))
1537
		return 0;
Christoph Lameter's avatar
Christoph Lameter committed
1538

1539 1540
	if (!check_valid_pointer(s, slab, object)) {
		object_err(s, slab, object, "Freelist Pointer check fails");
1541
		return 0;
Christoph Lameter's avatar
Christoph Lameter committed
1542 1543
	}

1544
	if (!check_object(s, slab, object, SLUB_RED_INACTIVE))
1545 1546 1547 1548 1549
		return 0;

	return 1;
}

1550
static noinline bool alloc_debug_processing(struct kmem_cache *s,
1551
			struct slab *slab, void *object, int orig_size)
1552 1553
{
	if (s->flags & SLAB_CONSISTENCY_CHECKS) {
1554
		if (!alloc_consistency_checks(s, slab, object))
1555 1556
			goto bad;
	}
Christoph Lameter's avatar
Christoph Lameter committed
1557

1558
	/* Success. Perform special debug activities for allocs */
1559
	trace(s, slab, object, 1);
1560
	set_orig_size(s, object, orig_size);
1561
	init_object(s, object, SLUB_RED_ACTIVE);
1562
	return true;
1563

Christoph Lameter's avatar
Christoph Lameter committed
1564
bad:
1565
	if (folio_test_slab(slab_folio(slab))) {
Christoph Lameter's avatar
Christoph Lameter committed
1566 1567 1568
		/*
		 * If this is a slab page then lets do the best we can
		 * to avoid issues in the future. Marking all objects
Christoph Lameter's avatar
Christoph Lameter committed
1569
		 * as used avoids touching the remaining objects.
Christoph Lameter's avatar
Christoph Lameter committed
1570
		 */
1571
		slab_fix(s, "Marking all objects used");
1572 1573
		slab->inuse = slab->objects;
		slab->freelist = NULL;
Christoph Lameter's avatar
Christoph Lameter committed
1574
	}
1575
	return false;
Christoph Lameter's avatar
Christoph Lameter committed
1576 1577
}

1578
static inline int free_consistency_checks(struct kmem_cache *s,
1579
		struct slab *slab, void *object, unsigned long addr)
Christoph Lameter's avatar
Christoph Lameter committed
1580
{
1581 1582
	if (!check_valid_pointer(s, slab, object)) {
		slab_err(s, slab, "Invalid object pointer 0x%p", object);
1583
		return 0;
Christoph Lameter's avatar
Christoph Lameter committed
1584 1585
	}

1586 1587
	if (on_freelist(s, slab, object)) {
		object_err(s, slab, object, "Object already free");
1588
		return 0;
Christoph Lameter's avatar
Christoph Lameter committed
1589 1590
	}

1591
	if (!check_object(s, slab, object, SLUB_RED_ACTIVE))
1592
		return 0;
Christoph Lameter's avatar
Christoph Lameter committed
1593

1594 1595 1596
	if (unlikely(s != slab->slab_cache)) {
		if (!folio_test_slab(slab_folio(slab))) {
			slab_err(s, slab, "Attempt to free object(0x%p) outside of slab",
Joe Perches's avatar
Joe Perches committed
1597
				 object);
1598
		} else if (!slab->slab_cache) {
1599 1600
			pr_err("SLUB <none>: no slab for object 0x%p.\n",
			       object);
1601
			dump_stack();
1602
		} else
1603
			object_err(s, slab, object,
1604
					"page slab pointer corrupt.");
1605 1606 1607 1608 1609
		return 0;
	}
	return 1;
}

1610
/*
1611
 * Parse a block of slab_debug options. Blocks are delimited by ';'
1612 1613 1614 1615 1616 1617 1618 1619 1620 1621
 *
 * @str:    start of block
 * @flags:  returns parsed flags, or DEBUG_DEFAULT_FLAGS if none specified
 * @slabs:  return start of list of slabs, or NULL when there's no list
 * @init:   assume this is initial parsing and not per-kmem-create parsing
 *
 * returns the start of next block if there's any, or NULL
 */
static char *
parse_slub_debug_flags(char *str, slab_flags_t *flags, char **slabs, bool init)
1622
{
1623
	bool higher_order_disable = false;
1624

1625 1626 1627 1628 1629
	/* Skip any completely empty blocks */
	while (*str && *str == ';')
		str++;

	if (*str == ',') {
1630 1631 1632 1633
		/*
		 * No options but restriction on slabs. This means full
		 * debugging for slabs matching a pattern.
		 */
1634
		*flags = DEBUG_DEFAULT_FLAGS;
1635
		goto check_slabs;
1636 1637
	}
	*flags = 0;
1638

1639 1640
	/* Determine which debug features should be switched on */
	for (; *str && *str != ',' && *str != ';'; str++) {
1641
		switch (tolower(*str)) {
1642 1643 1644
		case '-':
			*flags = 0;
			break;
1645
		case 'f':
1646
			*flags |= SLAB_CONSISTENCY_CHECKS;
1647 1648
			break;
		case 'z':
1649
			*flags |= SLAB_RED_ZONE;
1650 1651
			break;
		case 'p':
1652
			*flags |= SLAB_POISON;
1653 1654
			break;
		case 'u':
1655
			*flags |= SLAB_STORE_USER;
1656 1657
			break;
		case 't':
1658
			*flags |= SLAB_TRACE;
1659
			break;
1660
		case 'a':
1661
			*flags |= SLAB_FAILSLAB;
1662
			break;
1663 1664 1665 1666 1667
		case 'o':
			/*
			 * Avoid enabling debugging on caches if its minimum
			 * order would increase as a result.
			 */
1668
			higher_order_disable = true;
1669
			break;
1670
		default:
1671
			if (init)
1672
				pr_err("slab_debug option '%c' unknown. skipped\n", *str);
1673
		}
1674
	}
1675
check_slabs:
1676
	if (*str == ',')
1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700
		*slabs = ++str;
	else
		*slabs = NULL;

	/* Skip over the slab list */
	while (*str && *str != ';')
		str++;

	/* Skip any completely empty blocks */
	while (*str && *str == ';')
		str++;

	if (init && higher_order_disable)
		disable_higher_order_debug = 1;

	if (*str)
		return str;
	else
		return NULL;
}

static int __init setup_slub_debug(char *str)
{
	slab_flags_t flags;
1701
	slab_flags_t global_flags;
1702 1703 1704 1705 1706
	char *saved_str;
	char *slab_list;
	bool global_slub_debug_changed = false;
	bool slab_list_specified = false;

1707
	global_flags = DEBUG_DEFAULT_FLAGS;
1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718
	if (*str++ != '=' || !*str)
		/*
		 * No options specified. Switch on full debugging.
		 */
		goto out;

	saved_str = str;
	while (str) {
		str = parse_slub_debug_flags(str, &flags, &slab_list, true);

		if (!slab_list) {
1719
			global_flags = flags;
1720 1721 1722
			global_slub_debug_changed = true;
		} else {
			slab_list_specified = true;
1723
			if (flags & SLAB_STORE_USER)
1724
				stack_depot_request_early_init();
1725 1726 1727 1728 1729
		}
	}

	/*
	 * For backwards compatibility, a single list of flags with list of
1730
	 * slabs means debugging is only changed for those slabs, so the global
1731
	 * slab_debug should be unchanged (0 or DEBUG_DEFAULT_FLAGS, depending
1732
	 * on CONFIG_SLUB_DEBUG_ON). We can extended that to multiple lists as
1733 1734 1735 1736
	 * long as there is no option specifying flags without a slab list.
	 */
	if (slab_list_specified) {
		if (!global_slub_debug_changed)
1737
			global_flags = slub_debug;
1738 1739
		slub_debug_string = saved_str;
	}
1740
out:
1741
	slub_debug = global_flags;
1742
	if (slub_debug & SLAB_STORE_USER)
1743
		stack_depot_request_early_init();
1744 1745
	if (slub_debug != 0 || slub_debug_string)
		static_branch_enable(&slub_debug_enabled);
1746 1747
	else
		static_branch_disable(&slub_debug_enabled);
1748 1749 1750 1751
	if ((static_branch_unlikely(&init_on_alloc) ||
	     static_branch_unlikely(&init_on_free)) &&
	    (slub_debug & SLAB_POISON))
		pr_info("mem auto-init: SLAB_POISON will take precedence over init_on_alloc/init_on_free\n");
1752 1753 1754
	return 1;
}

1755 1756
__setup("slab_debug", setup_slub_debug);
__setup_param("slub_debug", slub_debug, setup_slub_debug, 0);
1757

1758 1759 1760 1761 1762 1763 1764
/*
 * kmem_cache_flags - apply debugging options to the cache
 * @flags:		flags to set
 * @name:		name of the cache
 *
 * Debug option(s) are applied to @flags. In addition to the debug
 * option(s), if a slab name (or multiple) is specified i.e.
1765
 * slab_debug=<Debug-Options>,<slab name1>,<slab name2> ...
1766 1767
 * then only the select slabs will receive the debug option(s).
 */
1768
slab_flags_t kmem_cache_flags(slab_flags_t flags, const char *name)
1769
{
1770 1771
	char *iter;
	size_t len;
1772 1773
	char *next_block;
	slab_flags_t block_flags;
1774 1775
	slab_flags_t slub_debug_local = slub_debug;

1776 1777 1778
	if (flags & SLAB_NO_USER_FLAGS)
		return flags;

1779 1780 1781 1782 1783 1784 1785
	/*
	 * If the slab cache is for debugging (e.g. kmemleak) then
	 * don't store user (stack trace) information by default,
	 * but let the user enable it via the command line below.
	 */
	if (flags & SLAB_NOLEAKTRACE)
		slub_debug_local &= ~SLAB_STORE_USER;
1786 1787

	len = strlen(name);
1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807
	next_block = slub_debug_string;
	/* Go through all blocks of debug options, see if any matches our slab's name */
	while (next_block) {
		next_block = parse_slub_debug_flags(next_block, &block_flags, &iter, false);
		if (!iter)
			continue;
		/* Found a block that has a slab list, search it */
		while (*iter) {
			char *end, *glob;
			size_t cmplen;

			end = strchrnul(iter, ',');
			if (next_block && next_block < end)
				end = next_block - 1;

			glob = strnchr(iter, end - iter, '*');
			if (glob)
				cmplen = glob - iter;
			else
				cmplen = max_t(size_t, len, (end - iter));
1808

1809 1810 1811 1812
			if (!strncmp(name, iter, cmplen)) {
				flags |= block_flags;
				return flags;
			}
1813

1814 1815 1816
			if (!*end || *end == ';')
				break;
			iter = end + 1;
1817 1818
		}
	}
1819

1820
	return flags | slub_debug_local;
1821
}
1822
#else /* !CONFIG_SLUB_DEBUG */
1823
static inline void setup_object_debug(struct kmem_cache *s, void *object) {}
1824
static inline
1825
void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr) {}
1826

1827 1828
static inline bool alloc_debug_processing(struct kmem_cache *s,
	struct slab *slab, void *object, int orig_size) { return true; }
1829

1830 1831 1832
static inline bool free_debug_processing(struct kmem_cache *s,
	struct slab *slab, void *head, void *tail, int *bulk_cnt,
	unsigned long addr, depot_stack_handle_t handle) { return true; }
1833

1834
static inline void slab_pad_check(struct kmem_cache *s, struct slab *slab) {}
1835
static inline int check_object(struct kmem_cache *s, struct slab *slab,
1836
			void *object, u8 val) { return 1; }
1837
static inline depot_stack_handle_t set_track_prepare(void) { return 0; }
1838 1839
static inline void set_track(struct kmem_cache *s, void *object,
			     enum track_item alloc, unsigned long addr) {}
1840
static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n,
1841
					struct slab *slab) {}
1842
static inline void remove_full(struct kmem_cache *s, struct kmem_cache_node *n,
1843
					struct slab *slab) {}
1844
slab_flags_t kmem_cache_flags(slab_flags_t flags, const char *name)
1845 1846 1847
{
	return flags;
}
1848
#define slub_debug 0
1849

1850 1851
#define disable_higher_order_debug 0

1852 1853
static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
							{ return 0; }
1854 1855 1856 1857
static inline void inc_slabs_node(struct kmem_cache *s, int node,
							int objects) {}
static inline void dec_slabs_node(struct kmem_cache *s, int node,
							int objects) {}
1858

1859
#ifndef CONFIG_SLUB_TINY
1860
static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab,
1861
			       void **freelist, void *nextfree)
1862 1863 1864
{
	return false;
}
1865
#endif
1866 1867
#endif /* CONFIG_SLUB_DEBUG */

1868 1869 1870 1871 1872 1873
static inline enum node_stat_item cache_vmstat_idx(struct kmem_cache *s)
{
	return (s->flags & SLAB_RECLAIM_ACCOUNT) ?
		NR_SLAB_RECLAIMABLE_B : NR_SLAB_UNRECLAIMABLE_B;
}

1874 1875
#ifdef CONFIG_SLAB_OBJ_EXT

1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893
#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG

static inline void mark_objexts_empty(struct slabobj_ext *obj_exts)
{
	struct slabobj_ext *slab_exts;
	struct slab *obj_exts_slab;

	obj_exts_slab = virt_to_slab(obj_exts);
	slab_exts = slab_obj_exts(obj_exts_slab);
	if (slab_exts) {
		unsigned int offs = obj_to_index(obj_exts_slab->slab_cache,
						 obj_exts_slab, obj_exts);
		/* codetag should be NULL */
		WARN_ON(slab_exts[offs].ref.ct);
		set_codetag_empty(&slab_exts[offs].ref);
	}
}

1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914
static inline void mark_failed_objexts_alloc(struct slab *slab)
{
	slab->obj_exts = OBJEXTS_ALLOC_FAIL;
}

static inline void handle_failed_objexts_alloc(unsigned long obj_exts,
			struct slabobj_ext *vec, unsigned int objects)
{
	/*
	 * If vector previously failed to allocate then we have live
	 * objects with no tag reference. Mark all references in this
	 * vector as empty to avoid warnings later on.
	 */
	if (obj_exts & OBJEXTS_ALLOC_FAIL) {
		unsigned int i;

		for (i = 0; i < objects; i++)
			set_codetag_empty(&vec[i].ref);
	}
}

1915 1916 1917
#else /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */

static inline void mark_objexts_empty(struct slabobj_ext *obj_exts) {}
1918 1919 1920
static inline void mark_failed_objexts_alloc(struct slab *slab) {}
static inline void handle_failed_objexts_alloc(unsigned long obj_exts,
			struct slabobj_ext *vec, unsigned int objects) {}
1921 1922 1923

#endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */

1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935
/*
 * The allocated objcg pointers array is not accounted directly.
 * Moreover, it should not come from DMA buffer and is not readily
 * reclaimable. So those GFP bits should be masked off.
 */
#define OBJCGS_CLEAR_MASK	(__GFP_DMA | __GFP_RECLAIMABLE | \
				__GFP_ACCOUNT | __GFP_NOFAIL)

static int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,
			       gfp_t gfp, bool new_slab)
{
	unsigned int objects = objs_per_slab(s, slab);
1936 1937 1938
	unsigned long new_exts;
	unsigned long old_exts;
	struct slabobj_ext *vec;
1939 1940

	gfp &= ~OBJCGS_CLEAR_MASK;
1941 1942
	/* Prevent recursive extension vector allocation */
	gfp |= __GFP_NO_OBJ_EXT;
1943 1944
	vec = kcalloc_node(objects, sizeof(struct slabobj_ext), gfp,
			   slab_nid(slab));
1945 1946 1947 1948 1949
	if (!vec) {
		/* Mark vectors which failed to allocate */
		if (new_slab)
			mark_failed_objexts_alloc(slab);

1950
		return -ENOMEM;
1951
	}
1952

1953
	new_exts = (unsigned long)vec;
1954
#ifdef CONFIG_MEMCG
1955
	new_exts |= MEMCG_DATA_OBJEXTS;
1956
#endif
1957 1958
	old_exts = slab->obj_exts;
	handle_failed_objexts_alloc(old_exts, vec, objects);
1959 1960 1961 1962 1963 1964
	if (new_slab) {
		/*
		 * If the slab is brand new and nobody can yet access its
		 * obj_exts, no synchronization is required and obj_exts can
		 * be simply assigned.
		 */
1965 1966
		slab->obj_exts = new_exts;
	} else if (cmpxchg(&slab->obj_exts, old_exts, new_exts) != old_exts) {
1967 1968 1969 1970 1971
		/*
		 * If the slab is already in use, somebody can allocate and
		 * assign slabobj_exts in parallel. In this case the existing
		 * objcg vector should be reused.
		 */
1972
		mark_objexts_empty(vec);
1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988
		kfree(vec);
		return 0;
	}

	kmemleak_not_leak(vec);
	return 0;
}

static inline void free_slab_obj_exts(struct slab *slab)
{
	struct slabobj_ext *obj_exts;

	obj_exts = slab_obj_exts(slab);
	if (!obj_exts)
		return;

1989 1990 1991 1992 1993 1994 1995 1996
	/*
	 * obj_exts was created with __GFP_NO_OBJ_EXT flag, therefore its
	 * corresponding extension will be NULL. alloc_tag_sub() will throw a
	 * warning if slab has extensions but the extension of an object is
	 * NULL, therefore replace NULL with CODETAG_EMPTY to indicate that
	 * the extension for obj_exts is expected to be NULL.
	 */
	mark_objexts_empty(obj_exts);
1997 1998 1999
	kfree(obj_exts);
	slab->obj_exts = 0;
}
2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020

static inline bool need_slab_obj_ext(void)
{
	if (mem_alloc_profiling_enabled())
		return true;

	/*
	 * CONFIG_MEMCG_KMEM creates vector of obj_cgroup objects conditionally
	 * inside memcg_slab_post_alloc_hook. No other users for now.
	 */
	return false;
}

static inline struct slabobj_ext *
prepare_slab_obj_exts_hook(struct kmem_cache *s, gfp_t flags, void *p)
{
	struct slab *slab;

	if (!p)
		return NULL;

2021
	if (s->flags & (SLAB_NO_OBJ_EXT | SLAB_NOLEAKTRACE))
2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059
		return NULL;

	if (flags & __GFP_NO_OBJ_EXT)
		return NULL;

	slab = virt_to_slab(p);
	if (!slab_obj_exts(slab) &&
	    WARN(alloc_slab_obj_exts(slab, s, flags, false),
		 "%s, %s: Failed to create slab extension vector!\n",
		 __func__, s->name))
		return NULL;

	return slab_obj_exts(slab) + obj_to_index(s, slab, p);
}

static inline void
alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p,
			     int objects)
{
#ifdef CONFIG_MEM_ALLOC_PROFILING
	struct slabobj_ext *obj_exts;
	int i;

	if (!mem_alloc_profiling_enabled())
		return;

	obj_exts = slab_obj_exts(slab);
	if (!obj_exts)
		return;

	for (i = 0; i < objects; i++) {
		unsigned int off = obj_to_index(s, slab, p[i]);

		alloc_tag_sub(&obj_exts[off].ref, s->size);
	}
#endif
}

2060
#else /* CONFIG_SLAB_OBJ_EXT */
2061

2062 2063 2064 2065 2066 2067 2068
static int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,
			       gfp_t gfp, bool new_slab)
{
	return 0;
}

static inline void free_slab_obj_exts(struct slab *slab)
2069 2070
{
}
2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088

static inline bool need_slab_obj_ext(void)
{
	return false;
}

static inline struct slabobj_ext *
prepare_slab_obj_exts_hook(struct kmem_cache *s, gfp_t flags, void *p)
{
	return NULL;
}

static inline void
alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p,
			     int objects)
{
}

2089
#endif /* CONFIG_SLAB_OBJ_EXT */
2090

2091
#ifdef CONFIG_MEMCG_KMEM
2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103
static inline size_t obj_full_size(struct kmem_cache *s)
{
	/*
	 * For each accounted object there is an extra space which is used
	 * to store obj_cgroup membership. Charge it too.
	 */
	return s->size + sizeof(struct obj_cgroup *);
}

/*
 * Returns false if the allocation should fail.
 */
2104 2105 2106 2107
static bool __memcg_slab_pre_alloc_hook(struct kmem_cache *s,
					struct list_lru *lru,
					struct obj_cgroup **objcgp,
					size_t objects, gfp_t flags)
2108 2109 2110 2111 2112 2113
{
	/*
	 * The obtained objcg pointer is safe to use within the current scope,
	 * defined by current task or set_active_memcg() pair.
	 * obj_cgroup_get() is used to get a permanent reference.
	 */
2114
	struct obj_cgroup *objcg = current_obj_cgroup();
2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136
	if (!objcg)
		return true;

	if (lru) {
		int ret;
		struct mem_cgroup *memcg;

		memcg = get_mem_cgroup_from_objcg(objcg);
		ret = memcg_list_lru_alloc(memcg, lru, flags);
		css_put(&memcg->css);

		if (ret)
			return false;
	}

	if (obj_cgroup_charge(objcg, flags, objects * obj_full_size(s)))
		return false;

	*objcgp = objcg;
	return true;
}

2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158
/*
 * Returns false if the allocation should fail.
 */
static __fastpath_inline
bool memcg_slab_pre_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
			       struct obj_cgroup **objcgp, size_t objects,
			       gfp_t flags)
{
	if (!memcg_kmem_online())
		return true;

	if (likely(!(flags & __GFP_ACCOUNT) && !(s->flags & SLAB_ACCOUNT)))
		return true;

	return likely(__memcg_slab_pre_alloc_hook(s, lru, objcgp, objects,
						  flags));
}

static void __memcg_slab_post_alloc_hook(struct kmem_cache *s,
					 struct obj_cgroup *objcg,
					 gfp_t flags, size_t size,
					 void **p)
2159 2160 2161 2162 2163
{
	struct slab *slab;
	unsigned long off;
	size_t i;

2164
	flags &= gfp_allowed_mask;
2165 2166 2167 2168 2169

	for (i = 0; i < size; i++) {
		if (likely(p[i])) {
			slab = virt_to_slab(p[i]);

2170 2171
			if (!slab_obj_exts(slab) &&
			    alloc_slab_obj_exts(slab, s, flags, false)) {
2172 2173 2174 2175 2176 2177
				obj_cgroup_uncharge(objcg, obj_full_size(s));
				continue;
			}

			off = obj_to_index(s, slab, p[i]);
			obj_cgroup_get(objcg);
2178
			slab_obj_exts(slab)[off].objcg = objcg;
2179 2180 2181 2182 2183 2184 2185 2186
			mod_objcg_state(objcg, slab_pgdat(slab),
					cache_vmstat_idx(s), obj_full_size(s));
		} else {
			obj_cgroup_uncharge(objcg, obj_full_size(s));
		}
	}
}

2187 2188 2189 2190 2191 2192 2193 2194 2195 2196
static __fastpath_inline
void memcg_slab_post_alloc_hook(struct kmem_cache *s, struct obj_cgroup *objcg,
				gfp_t flags, size_t size, void **p)
{
	if (likely(!memcg_kmem_online() || !objcg))
		return;

	return __memcg_slab_post_alloc_hook(s, objcg, flags, size, p);
}

2197 2198
static void __memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab,
				   void **p, int objects,
2199
				   struct slabobj_ext *obj_exts)
2200
{
2201
	for (int i = 0; i < objects; i++) {
2202 2203 2204 2205
		struct obj_cgroup *objcg;
		unsigned int off;

		off = obj_to_index(s, slab, p[i]);
2206
		objcg = obj_exts[off].objcg;
2207 2208 2209
		if (!objcg)
			continue;

2210
		obj_exts[off].objcg = NULL;
2211 2212 2213 2214 2215 2216
		obj_cgroup_uncharge(objcg, obj_full_size(s));
		mod_objcg_state(objcg, slab_pgdat(slab), cache_vmstat_idx(s),
				-obj_full_size(s));
		obj_cgroup_put(objcg);
	}
}
2217 2218 2219 2220 2221

static __fastpath_inline
void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p,
			  int objects)
{
2222
	struct slabobj_ext *obj_exts;
2223 2224 2225 2226

	if (!memcg_kmem_online())
		return;

2227 2228
	obj_exts = slab_obj_exts(slab);
	if (likely(!obj_exts))
2229 2230
		return;

2231
	__memcg_slab_free_hook(s, slab, p, objects, obj_exts);
2232
}
2233 2234 2235 2236 2237 2238 2239 2240

static inline
void memcg_slab_alloc_error_hook(struct kmem_cache *s, int objects,
			   struct obj_cgroup *objcg)
{
	if (objcg)
		obj_cgroup_uncharge(objcg, objects * obj_full_size(s));
}
2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260
#else /* CONFIG_MEMCG_KMEM */
static inline bool memcg_slab_pre_alloc_hook(struct kmem_cache *s,
					     struct list_lru *lru,
					     struct obj_cgroup **objcgp,
					     size_t objects, gfp_t flags)
{
	return true;
}

static inline void memcg_slab_post_alloc_hook(struct kmem_cache *s,
					      struct obj_cgroup *objcg,
					      gfp_t flags, size_t size,
					      void **p)
{
}

static inline void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab,
					void **p, int objects)
{
}
2261 2262 2263 2264 2265 2266

static inline
void memcg_slab_alloc_error_hook(struct kmem_cache *s, int objects,
				 struct obj_cgroup *objcg)
{
}
2267 2268
#endif /* CONFIG_MEMCG_KMEM */

2269 2270 2271
/*
 * Hooks for other subsystems that check memory allocations. In a typical
 * production configuration these hooks all should produce no code at all.
2272 2273
 *
 * Returns true if freeing of the object can proceed, false if its reuse
2274
 * was delayed by KASAN quarantine, or it was returned to KFENCE.
2275
 */
2276 2277
static __always_inline
bool slab_free_hook(struct kmem_cache *s, void *x, bool init)
2278 2279
{
	kmemleak_free_recursive(x, s->flags);
2280
	kmsan_slab_free(s, x);
2281

2282
	debug_check_no_locks_freed(x, s->object_size);
2283 2284 2285

	if (!(s->flags & SLAB_DEBUG_OBJECTS))
		debug_check_no_obj_freed(x, s->object_size);
2286

2287 2288 2289 2290 2291
	/* Use KCSAN to help debug racy use-after-free. */
	if (!(s->flags & SLAB_TYPESAFE_BY_RCU))
		__kcsan_check_access(x, s->object_size,
				     KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ASSERT);

2292 2293 2294
	if (kfence_free(x))
		return false;

2295 2296 2297 2298 2299 2300 2301 2302
	/*
	 * As memory initialization might be integrated into KASAN,
	 * kasan_slab_free and initialization memset's must be
	 * kept together to avoid discrepancies in behavior.
	 *
	 * The initialization memset's clear the object and the metadata,
	 * but don't touch the SLAB redzone.
	 */
2303
	if (unlikely(init)) {
2304 2305 2306 2307 2308 2309 2310 2311 2312
		int rsize;

		if (!kasan_has_integrated_init())
			memset(kasan_reset_tag(x), 0, s->object_size);
		rsize = (s->flags & SLAB_RED_ZONE) ? s->red_left_pad : 0;
		memset((char *)kasan_reset_tag(x) + s->inuse, 0,
		       s->size - s->inuse - rsize);
	}
	/* KASAN might put x into memory quarantine, delaying its reuse. */
2313
	return !kasan_slab_free(s, x, init);
2314
}
2315

2316 2317 2318
static __fastpath_inline
bool slab_free_freelist_hook(struct kmem_cache *s, void **head, void **tail,
			     int *cnt)
2319
{
2320 2321 2322

	void *object;
	void *next = *head;
2323
	void *old_tail = *tail;
2324
	bool init;
2325

2326
	if (is_kfence_address(next)) {
2327
		slab_free_hook(s, next, false);
2328
		return false;
2329 2330
	}

2331 2332 2333
	/* Head and tail of the reconstructed freelist */
	*head = NULL;
	*tail = NULL;
2334

2335 2336
	init = slab_want_init_on_free(s);

2337 2338 2339 2340
	do {
		object = next;
		next = get_freepointer(s, object);

2341
		/* If object's reuse doesn't have to be delayed */
2342
		if (likely(slab_free_hook(s, object, init))) {
2343 2344 2345 2346 2347
			/* Move object to the new freelist */
			set_freepointer(s, object, *head);
			*head = object;
			if (!*tail)
				*tail = object;
2348 2349 2350 2351 2352 2353
		} else {
			/*
			 * Adjust the reconstructed freelist depth
			 * accordingly if object's reuse is delayed.
			 */
			--(*cnt);
2354 2355 2356 2357
		}
	} while (object != old_tail);

	return *head != NULL;
2358 2359
}

2360
static void *setup_object(struct kmem_cache *s, void *object)
2361
{
2362
	setup_object_debug(s, object);
2363
	object = kasan_init_slab_obj(s, object);
2364
	if (unlikely(s->ctor)) {
2365
		kasan_unpoison_new_object(s, object);
2366
		s->ctor(object);
2367
		kasan_poison_new_object(s, object);
2368
	}
2369
	return object;
2370 2371
}

Christoph Lameter's avatar
Christoph Lameter committed
2372 2373 2374
/*
 * Slab allocation and freeing
 */
2375 2376
static inline struct slab *alloc_slab_page(gfp_t flags, int node,
		struct kmem_cache_order_objects oo)
2377
{
2378 2379
	struct folio *folio;
	struct slab *slab;
2380
	unsigned int order = oo_order(oo);
2381

2382
	folio = (struct folio *)alloc_pages_node(node, flags, order);
2383 2384 2385 2386 2387
	if (!folio)
		return NULL;

	slab = folio_slab(folio);
	__folio_set_slab(folio);
2388 2389
	/* Make the flag visible before any changes to folio->mapping */
	smp_wmb();
2390
	if (folio_is_pfmemalloc(folio))
2391 2392 2393
		slab_set_pfmemalloc(slab);

	return slab;
2394 2395
}

2396 2397 2398 2399
#ifdef CONFIG_SLAB_FREELIST_RANDOM
/* Pre-initialize the random sequence cache */
static int init_cache_random_seq(struct kmem_cache *s)
{
2400
	unsigned int count = oo_objects(s->oo);
2401 2402
	int err;

2403 2404 2405 2406
	/* Bailout if already initialised */
	if (s->random_seq)
		return 0;

2407 2408 2409 2410 2411 2412 2413 2414 2415
	err = cache_random_seq_create(s, count, GFP_KERNEL);
	if (err) {
		pr_err("SLUB: Unable to initialize free list for %s\n",
			s->name);
		return err;
	}

	/* Transform to an offset on the set of pages */
	if (s->random_seq) {
2416 2417
		unsigned int i;

2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437
		for (i = 0; i < count; i++)
			s->random_seq[i] *= s->size;
	}
	return 0;
}

/* Initialize each random sequence freelist per cache */
static void __init init_freelist_randomization(void)
{
	struct kmem_cache *s;

	mutex_lock(&slab_mutex);

	list_for_each_entry(s, &slab_caches, list)
		init_cache_random_seq(s);

	mutex_unlock(&slab_mutex);
}

/* Get the next entry on the pre-computed freelist randomized */
2438
static void *next_freelist_entry(struct kmem_cache *s,
2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459
				unsigned long *pos, void *start,
				unsigned long page_limit,
				unsigned long freelist_count)
{
	unsigned int idx;

	/*
	 * If the target page allocation failed, the number of objects on the
	 * page might be smaller than the usual size defined by the cache.
	 */
	do {
		idx = s->random_seq[*pos];
		*pos += 1;
		if (*pos >= freelist_count)
			*pos = 0;
	} while (unlikely(idx >= page_limit));

	return (char *)start + idx;
}

/* Shuffle the single linked freelist based on a random pre-computed sequence */
2460
static bool shuffle_freelist(struct kmem_cache *s, struct slab *slab)
2461 2462 2463 2464 2465 2466
{
	void *start;
	void *cur;
	void *next;
	unsigned long idx, pos, page_limit, freelist_count;

2467
	if (slab->objects < 2 || !s->random_seq)
2468 2469 2470
		return false;

	freelist_count = oo_objects(s->oo);
2471
	pos = get_random_u32_below(freelist_count);
2472

2473 2474
	page_limit = slab->objects * s->size;
	start = fixup_red_left(s, slab_address(slab));
2475 2476

	/* First entry is used as the base of the freelist */
2477
	cur = next_freelist_entry(s, &pos, start, page_limit, freelist_count);
2478
	cur = setup_object(s, cur);
2479
	slab->freelist = cur;
2480

2481
	for (idx = 1; idx < slab->objects; idx++) {
2482
		next = next_freelist_entry(s, &pos, start, page_limit,
2483
			freelist_count);
2484
		next = setup_object(s, next);
2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497
		set_freepointer(s, cur, next);
		cur = next;
	}
	set_freepointer(s, cur, NULL);

	return true;
}
#else
static inline int init_cache_random_seq(struct kmem_cache *s)
{
	return 0;
}
static inline void init_freelist_randomization(void) { }
2498
static inline bool shuffle_freelist(struct kmem_cache *s, struct slab *slab)
2499 2500 2501 2502 2503
{
	return false;
}
#endif /* CONFIG_SLAB_FREELIST_RANDOM */

2504 2505 2506 2507
static __always_inline void account_slab(struct slab *slab, int order,
					 struct kmem_cache *s, gfp_t gfp)
{
	if (memcg_kmem_online() && (s->flags & SLAB_ACCOUNT))
2508
		alloc_slab_obj_exts(slab, s, gfp, true);
2509 2510 2511 2512 2513 2514 2515 2516

	mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s),
			    PAGE_SIZE << order);
}

static __always_inline void unaccount_slab(struct slab *slab, int order,
					   struct kmem_cache *s)
{
2517
	if (memcg_kmem_online() || need_slab_obj_ext())
2518
		free_slab_obj_exts(slab);
2519 2520 2521 2522 2523

	mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s),
			    -(PAGE_SIZE << order));
}

2524
static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
Christoph Lameter's avatar
Christoph Lameter committed
2525
{
2526
	struct slab *slab;
2527
	struct kmem_cache_order_objects oo = s->oo;
2528
	gfp_t alloc_gfp;
2529
	void *start, *p, *next;
2530
	int idx;
2531
	bool shuffle;
Christoph Lameter's avatar
Christoph Lameter committed
2532

2533 2534
	flags &= gfp_allowed_mask;

2535
	flags |= s->allocflags;
2536

2537 2538 2539 2540 2541
	/*
	 * Let the initial higher-order allocation fail under memory pressure
	 * so we fall-back to the minimum order allocation.
	 */
	alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL;
2542
	if ((alloc_gfp & __GFP_DIRECT_RECLAIM) && oo_order(oo) > oo_order(s->min))
2543
		alloc_gfp = (alloc_gfp | __GFP_NOMEMALLOC) & ~__GFP_RECLAIM;
2544

2545
	slab = alloc_slab_page(alloc_gfp, node, oo);
2546
	if (unlikely(!slab)) {
2547
		oo = s->min;
2548
		alloc_gfp = flags;
2549 2550 2551 2552
		/*
		 * Allocation may have failed due to fragmentation.
		 * Try a lower order alloc if possible
		 */
2553
		slab = alloc_slab_page(alloc_gfp, node, oo);
2554
		if (unlikely(!slab))
2555
			return NULL;
2556
		stat(s, ORDER_FALLBACK);
2557
	}
2558

2559
	slab->objects = oo_objects(oo);
2560 2561
	slab->inuse = 0;
	slab->frozen = 0;
Christoph Lameter's avatar
Christoph Lameter committed
2562

2563
	account_slab(slab, oo_order(oo), s, flags);
2564

2565
	slab->slab_cache = s;
Christoph Lameter's avatar
Christoph Lameter committed
2566

2567
	kasan_poison_slab(slab);
Christoph Lameter's avatar
Christoph Lameter committed
2568

2569
	start = slab_address(slab);
Christoph Lameter's avatar
Christoph Lameter committed
2570

2571
	setup_slab_debug(s, slab, start);
2572

2573
	shuffle = shuffle_freelist(s, slab);
2574 2575

	if (!shuffle) {
2576
		start = fixup_red_left(s, start);
2577
		start = setup_object(s, start);
2578 2579
		slab->freelist = start;
		for (idx = 0, p = start; idx < slab->objects - 1; idx++) {
2580
			next = p + s->size;
2581
			next = setup_object(s, next);
2582 2583 2584 2585
			set_freepointer(s, p, next);
			p = next;
		}
		set_freepointer(s, p, NULL);
Christoph Lameter's avatar
Christoph Lameter committed
2586 2587
	}

2588
	return slab;
Christoph Lameter's avatar
Christoph Lameter committed
2589 2590
}

2591
static struct slab *new_slab(struct kmem_cache *s, gfp_t flags, int node)
2592
{
2593 2594
	if (unlikely(flags & GFP_SLAB_BUG_MASK))
		flags = kmalloc_fix_flags(flags);
2595

2596 2597
	WARN_ON_ONCE(s->ctor && (flags & __GFP_ZERO));

2598 2599 2600 2601
	return allocate_slab(s,
		flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node);
}

2602
static void __free_slab(struct kmem_cache *s, struct slab *slab)
Christoph Lameter's avatar
Christoph Lameter committed
2603
{
2604 2605
	struct folio *folio = slab_folio(slab);
	int order = folio_order(folio);
2606
	int pages = 1 << order;
Christoph Lameter's avatar
Christoph Lameter committed
2607

2608 2609
	__slab_clear_pfmemalloc(slab);
	folio->mapping = NULL;
2610 2611 2612
	/* Make the mapping reset visible before clearing the flag */
	smp_wmb();
	__folio_clear_slab(folio);
2613
	mm_account_reclaimed_pages(pages);
2614
	unaccount_slab(slab, order, s);
2615
	__free_pages(&folio->page, order);
Christoph Lameter's avatar
Christoph Lameter committed
2616 2617 2618 2619
}

static void rcu_free_slab(struct rcu_head *h)
{
2620
	struct slab *slab = container_of(h, struct slab, rcu_head);
2621

2622
	__free_slab(slab->slab_cache, slab);
Christoph Lameter's avatar
Christoph Lameter committed
2623 2624
}

2625
static void free_slab(struct kmem_cache *s, struct slab *slab)
Christoph Lameter's avatar
Christoph Lameter committed
2626
{
2627 2628 2629 2630 2631 2632 2633 2634 2635
	if (kmem_cache_debug_flags(s, SLAB_CONSISTENCY_CHECKS)) {
		void *p;

		slab_pad_check(s, slab);
		for_each_object(p, s, slab_address(slab), slab->objects)
			check_object(s, slab, p, SLUB_RED_INACTIVE);
	}

	if (unlikely(s->flags & SLAB_TYPESAFE_BY_RCU))
2636
		call_rcu(&slab->rcu_head, rcu_free_slab);
2637
	else
2638
		__free_slab(s, slab);
Christoph Lameter's avatar
Christoph Lameter committed
2639 2640
}

2641
static void discard_slab(struct kmem_cache *s, struct slab *slab)
Christoph Lameter's avatar
Christoph Lameter committed
2642
{
2643 2644
	dec_slabs_node(s, slab_nid(slab), slab->objects);
	free_slab(s, slab);
Christoph Lameter's avatar
Christoph Lameter committed
2645 2646
}

2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665
/*
 * SLUB reuses PG_workingset bit to keep track of whether it's on
 * the per-node partial list.
 */
static inline bool slab_test_node_partial(const struct slab *slab)
{
	return folio_test_workingset((struct folio *)slab_folio(slab));
}

static inline void slab_set_node_partial(struct slab *slab)
{
	set_bit(PG_workingset, folio_flags(slab_folio(slab), 0));
}

static inline void slab_clear_node_partial(struct slab *slab)
{
	clear_bit(PG_workingset, folio_flags(slab_folio(slab), 0));
}

Christoph Lameter's avatar
Christoph Lameter committed
2666
/*
2667
 * Management of partially allocated slabs.
Christoph Lameter's avatar
Christoph Lameter committed
2668
 */
2669
static inline void
2670
__add_partial(struct kmem_cache_node *n, struct slab *slab, int tail)
Christoph Lameter's avatar
Christoph Lameter committed
2671
{
Christoph Lameter's avatar
Christoph Lameter committed
2672
	n->nr_partial++;
2673
	if (tail == DEACTIVATE_TO_TAIL)
2674
		list_add_tail(&slab->slab_list, &n->partial);
2675
	else
2676
		list_add(&slab->slab_list, &n->partial);
2677
	slab_set_node_partial(slab);
Christoph Lameter's avatar
Christoph Lameter committed
2678 2679
}

2680
static inline void add_partial(struct kmem_cache_node *n,
2681
				struct slab *slab, int tail)
2682
{
2683
	lockdep_assert_held(&n->list_lock);
2684
	__add_partial(n, slab, tail);
2685
}
2686

2687
static inline void remove_partial(struct kmem_cache_node *n,
2688
					struct slab *slab)
2689 2690
{
	lockdep_assert_held(&n->list_lock);
2691
	list_del(&slab->slab_list);
2692
	slab_clear_node_partial(slab);
2693
	n->nr_partial--;
2694 2695
}

2696
/*
2697
 * Called only for kmem_cache_debug() caches instead of remove_partial(), with a
2698 2699 2700 2701 2702
 * slab from the n->partial list. Remove only a single object from the slab, do
 * the alloc_debug_processing() checks and leave the slab on the list, or move
 * it to full list if it was the last free object.
 */
static void *alloc_single_from_partial(struct kmem_cache *s,
2703
		struct kmem_cache_node *n, struct slab *slab, int orig_size)
2704 2705 2706 2707 2708 2709 2710 2711 2712
{
	void *object;

	lockdep_assert_held(&n->list_lock);

	object = slab->freelist;
	slab->freelist = get_freepointer(s, object);
	slab->inuse++;

2713
	if (!alloc_debug_processing(s, slab, object, orig_size)) {
2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731
		remove_partial(n, slab);
		return NULL;
	}

	if (slab->inuse == slab->objects) {
		remove_partial(n, slab);
		add_full(s, n, slab);
	}

	return object;
}

/*
 * Called only for kmem_cache_debug() caches to allocate from a freshly
 * allocated slab. Allocate a single object instead of whole freelist
 * and put the slab to the partial (or full) list.
 */
static void *alloc_single_from_new_slab(struct kmem_cache *s,
2732
					struct slab *slab, int orig_size)
2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743
{
	int nid = slab_nid(slab);
	struct kmem_cache_node *n = get_node(s, nid);
	unsigned long flags;
	void *object;


	object = slab->freelist;
	slab->freelist = get_freepointer(s, object);
	slab->inuse = 1;

2744
	if (!alloc_debug_processing(s, slab, object, orig_size))
2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764
		/*
		 * It's not really expected that this would fail on a
		 * freshly allocated slab, but a concurrent memory
		 * corruption in theory could cause that.
		 */
		return NULL;

	spin_lock_irqsave(&n->list_lock, flags);

	if (slab->inuse == slab->objects)
		add_full(s, n, slab);
	else
		add_partial(n, slab, DEACTIVATE_TO_HEAD);

	inc_slabs_node(s, nid, slab->objects);
	spin_unlock_irqrestore(&n->list_lock, flags);

	return object;
}

2765
#ifdef CONFIG_SLUB_CPU_PARTIAL
2766
static void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain);
2767
#else
2768
static inline void put_cpu_partial(struct kmem_cache *s, struct slab *slab,
2769 2770
				   int drain) { }
#endif
2771
static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags);
2772

Christoph Lameter's avatar
Christoph Lameter committed
2773
/*
Christoph Lameter's avatar
Christoph Lameter committed
2774
 * Try to allocate a partial slab from a specific node.
Christoph Lameter's avatar
Christoph Lameter committed
2775
 */
2776 2777 2778
static struct slab *get_partial_node(struct kmem_cache *s,
				     struct kmem_cache_node *n,
				     struct partial_context *pc)
Christoph Lameter's avatar
Christoph Lameter committed
2779
{
2780
	struct slab *slab, *slab2, *partial = NULL;
2781
	unsigned long flags;
2782
	unsigned int partial_slabs = 0;
Christoph Lameter's avatar
Christoph Lameter committed
2783 2784 2785 2786

	/*
	 * Racy check. If we mistakenly see no partial slabs then we
	 * just allocate an empty slab. If we mistakenly try to get a
2787
	 * partial slab and there is none available then get_partial()
Christoph Lameter's avatar
Christoph Lameter committed
2788
	 * will return NULL.
Christoph Lameter's avatar
Christoph Lameter committed
2789 2790 2791 2792
	 */
	if (!n || !n->nr_partial)
		return NULL;

2793
	spin_lock_irqsave(&n->list_lock, flags);
2794
	list_for_each_entry_safe(slab, slab2, &n->partial, slab_list) {
2795
		if (!pfmemalloc_match(slab, pc->flags))
2796 2797
			continue;

2798
		if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) {
2799
			void *object = alloc_single_from_partial(s, n, slab,
2800
							pc->orig_size);
2801 2802 2803
			if (object) {
				partial = slab;
				pc->object = object;
2804
				break;
2805
			}
2806 2807 2808
			continue;
		}

2809
		remove_partial(n, slab);
2810

2811 2812
		if (!partial) {
			partial = slab;
2813 2814
			stat(s, ALLOC_FROM_PARTIAL);
		} else {
2815
			put_cpu_partial(s, slab, 0);
2816
			stat(s, CPU_PARTIAL_NODE);
2817
			partial_slabs++;
2818
		}
2819
#ifdef CONFIG_SLUB_CPU_PARTIAL
2820
		if (!kmem_cache_has_cpu_partial(s)
2821
			|| partial_slabs > s->cpu_partial_slabs / 2)
2822
			break;
2823 2824 2825
#else
		break;
#endif
2826

2827
	}
2828
	spin_unlock_irqrestore(&n->list_lock, flags);
2829
	return partial;
Christoph Lameter's avatar
Christoph Lameter committed
2830 2831 2832
}

/*
2833
 * Get a slab from somewhere. Search in increasing NUMA distances.
Christoph Lameter's avatar
Christoph Lameter committed
2834
 */
2835 2836
static struct slab *get_any_partial(struct kmem_cache *s,
				    struct partial_context *pc)
Christoph Lameter's avatar
Christoph Lameter committed
2837 2838 2839
{
#ifdef CONFIG_NUMA
	struct zonelist *zonelist;
2840
	struct zoneref *z;
2841
	struct zone *zone;
2842
	enum zone_type highest_zoneidx = gfp_zone(pc->flags);
2843
	struct slab *slab;
2844
	unsigned int cpuset_mems_cookie;
Christoph Lameter's avatar
Christoph Lameter committed
2845 2846

	/*
Christoph Lameter's avatar
Christoph Lameter committed
2847 2848 2849 2850
	 * The defrag ratio allows a configuration of the tradeoffs between
	 * inter node defragmentation and node local allocations. A lower
	 * defrag_ratio increases the tendency to do local allocations
	 * instead of attempting to obtain partial slabs from other nodes.
Christoph Lameter's avatar
Christoph Lameter committed
2851
	 *
Christoph Lameter's avatar
Christoph Lameter committed
2852 2853 2854 2855
	 * If the defrag_ratio is set to 0 then kmalloc() always
	 * returns node local objects. If the ratio is higher then kmalloc()
	 * may return off node objects because partial slabs are obtained
	 * from other nodes and filled up.
Christoph Lameter's avatar
Christoph Lameter committed
2856
	 *
2857 2858 2859 2860 2861
	 * If /sys/kernel/slab/xx/remote_node_defrag_ratio is set to 100
	 * (which makes defrag_ratio = 1000) then every (well almost)
	 * allocation will first attempt to defrag slab caches on other nodes.
	 * This means scanning over all nodes to look for partial slabs which
	 * may be expensive if we do it every time we are trying to find a slab
Christoph Lameter's avatar
Christoph Lameter committed
2862
	 * with available objects.
Christoph Lameter's avatar
Christoph Lameter committed
2863
	 */
2864 2865
	if (!s->remote_node_defrag_ratio ||
			get_cycles() % 1024 > s->remote_node_defrag_ratio)
Christoph Lameter's avatar
Christoph Lameter committed
2866 2867
		return NULL;

2868
	do {
2869
		cpuset_mems_cookie = read_mems_allowed_begin();
2870
		zonelist = node_zonelist(mempolicy_slab_node(), pc->flags);
2871
		for_each_zone_zonelist(zone, z, zonelist, highest_zoneidx) {
2872 2873 2874 2875
			struct kmem_cache_node *n;

			n = get_node(s, zone_to_nid(zone));

2876
			if (n && cpuset_zone_allowed(zone, pc->flags) &&
2877
					n->nr_partial > s->min_partial) {
2878 2879
				slab = get_partial_node(s, n, pc);
				if (slab) {
2880
					/*
2881 2882 2883 2884 2885
					 * Don't check read_mems_allowed_retry()
					 * here - if mems_allowed was updated in
					 * parallel, that was a harmless race
					 * between allocation and the cpuset
					 * update
2886
					 */
2887
					return slab;
2888
				}
2889
			}
Christoph Lameter's avatar
Christoph Lameter committed
2890
		}
2891
	} while (read_mems_allowed_retry(cpuset_mems_cookie));
2892
#endif	/* CONFIG_NUMA */
Christoph Lameter's avatar
Christoph Lameter committed
2893 2894 2895 2896
	return NULL;
}

/*
2897
 * Get a partial slab, lock it and return it.
Christoph Lameter's avatar
Christoph Lameter committed
2898
 */
2899 2900
static struct slab *get_partial(struct kmem_cache *s, int node,
				struct partial_context *pc)
Christoph Lameter's avatar
Christoph Lameter committed
2901
{
2902
	struct slab *slab;
2903 2904 2905 2906
	int searchnode = node;

	if (node == NUMA_NO_NODE)
		searchnode = numa_mem_id();
Christoph Lameter's avatar
Christoph Lameter committed
2907

2908 2909 2910
	slab = get_partial_node(s, get_node(s, searchnode), pc);
	if (slab || node != NUMA_NO_NODE)
		return slab;
Christoph Lameter's avatar
Christoph Lameter committed
2911

2912
	return get_any_partial(s, pc);
Christoph Lameter's avatar
Christoph Lameter committed
2913 2914
}

2915 2916
#ifndef CONFIG_SLUB_TINY

2917
#ifdef CONFIG_PREEMPTION
2918
/*
2919
 * Calculate the next globally unique transaction for disambiguation
2920 2921 2922 2923 2924 2925 2926 2927 2928 2929
 * during cmpxchg. The transactions start with the cpu number and are then
 * incremented by CONFIG_NR_CPUS.
 */
#define TID_STEP  roundup_pow_of_two(CONFIG_NR_CPUS)
#else
/*
 * No preemption supported therefore also no need to check for
 * different cpus.
 */
#define TID_STEP 1
2930
#endif /* CONFIG_PREEMPTION */
2931 2932 2933 2934 2935 2936

static inline unsigned long next_tid(unsigned long tid)
{
	return tid + TID_STEP;
}

2937
#ifdef SLUB_DEBUG_CMPXCHG
2938 2939 2940 2941 2942 2943 2944 2945 2946
static inline unsigned int tid_to_cpu(unsigned long tid)
{
	return tid % TID_STEP;
}

static inline unsigned long tid_to_event(unsigned long tid)
{
	return tid / TID_STEP;
}
2947
#endif
2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959

static inline unsigned int init_tid(int cpu)
{
	return cpu;
}

static inline void note_cmpxchg_failure(const char *n,
		const struct kmem_cache *s, unsigned long tid)
{
#ifdef SLUB_DEBUG_CMPXCHG
	unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid);

2960
	pr_info("%s %s: cmpxchg redo ", n, s->name);
2961

2962
#ifdef CONFIG_PREEMPTION
2963
	if (tid_to_cpu(tid) != tid_to_cpu(actual_tid))
2964
		pr_warn("due to cpu change %d -> %d\n",
2965 2966 2967 2968
			tid_to_cpu(tid), tid_to_cpu(actual_tid));
	else
#endif
	if (tid_to_event(tid) != tid_to_event(actual_tid))
2969
		pr_warn("due to cpu running other code. Event %ld->%ld\n",
2970 2971
			tid_to_event(tid), tid_to_event(actual_tid));
	else
2972
		pr_warn("for unknown reason: actual=%lx was=%lx target=%lx\n",
2973 2974
			actual_tid, tid, next_tid(tid));
#endif
2975
	stat(s, CMPXCHG_DOUBLE_CPU_FAIL);
2976 2977
}

2978
static void init_kmem_cache_cpus(struct kmem_cache *s)
2979 2980
{
	int cpu;
2981
	struct kmem_cache_cpu *c;
2982

2983 2984 2985 2986 2987
	for_each_possible_cpu(cpu) {
		c = per_cpu_ptr(s->cpu_slab, cpu);
		local_lock_init(&c->lock);
		c->tid = init_tid(cpu);
	}
2988
}
2989

Christoph Lameter's avatar
Christoph Lameter committed
2990
/*
2991
 * Finishes removing the cpu slab. Merges cpu's freelist with slab's freelist,
2992 2993 2994
 * unfreezes the slabs and puts it on the proper list.
 * Assumes the slab has been already safely taken away from kmem_cache_cpu
 * by the caller.
Christoph Lameter's avatar
Christoph Lameter committed
2995
 */
2996
static void deactivate_slab(struct kmem_cache *s, struct slab *slab,
2997
			    void *freelist)
Christoph Lameter's avatar
Christoph Lameter committed
2998
{
2999
	struct kmem_cache_node *n = get_node(s, slab_nid(slab));
3000
	int free_delta = 0;
3001
	void *nextfree, *freelist_iter, *freelist_tail;
3002
	int tail = DEACTIVATE_TO_HEAD;
3003
	unsigned long flags = 0;
3004 3005
	struct slab new;
	struct slab old;
3006

3007
	if (slab->freelist) {
3008
		stat(s, DEACTIVATE_REMOTE_FREES);
3009
		tail = DEACTIVATE_TO_TAIL;
3010 3011
	}

3012
	/*
3013 3014
	 * Stage one: Count the objects on cpu's freelist as free_delta and
	 * remember the last object in freelist_tail for later splicing.
3015
	 */
3016 3017 3018 3019
	freelist_tail = NULL;
	freelist_iter = freelist;
	while (freelist_iter) {
		nextfree = get_freepointer(s, freelist_iter);
3020

3021 3022
		/*
		 * If 'nextfree' is invalid, it is possible that the object at
3023 3024
		 * 'freelist_iter' is already corrupted.  So isolate all objects
		 * starting at 'freelist_iter' by skipping them.
3025
		 */
3026
		if (freelist_corrupted(s, slab, &freelist_iter, nextfree))
3027 3028
			break;

3029 3030
		freelist_tail = freelist_iter;
		free_delta++;
3031

3032
		freelist_iter = nextfree;
3033 3034
	}

3035
	/*
3036 3037
	 * Stage two: Unfreeze the slab while splicing the per-cpu
	 * freelist to the head of slab's freelist.
3038
	 */
3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057
	do {
		old.freelist = READ_ONCE(slab->freelist);
		old.counters = READ_ONCE(slab->counters);
		VM_BUG_ON(!old.frozen);

		/* Determine target state of the slab */
		new.counters = old.counters;
		new.frozen = 0;
		if (freelist_tail) {
			new.inuse -= free_delta;
			set_freepointer(s, freelist_tail, old.freelist);
			new.freelist = freelist;
		} else {
			new.freelist = old.freelist;
		}
	} while (!slab_update_freelist(s, slab,
		old.freelist, old.counters,
		new.freelist, new.counters,
		"unfreezing slab"));
3058

3059 3060 3061
	/*
	 * Stage three: Manipulate the slab list based on the updated state.
	 */
3062
	if (!new.inuse && n->nr_partial >= s->min_partial) {
3063 3064 3065
		stat(s, DEACTIVATE_EMPTY);
		discard_slab(s, slab);
		stat(s, FREE_SLAB);
3066 3067 3068 3069
	} else if (new.freelist) {
		spin_lock_irqsave(&n->list_lock, flags);
		add_partial(n, slab, tail);
		spin_unlock_irqrestore(&n->list_lock, flags);
3070
		stat(s, tail);
3071
	} else {
3072
		stat(s, DEACTIVATE_FULL);
3073
	}
Christoph Lameter's avatar
Christoph Lameter committed
3074 3075
}

3076
#ifdef CONFIG_SLUB_CPU_PARTIAL
3077
static void __put_partials(struct kmem_cache *s, struct slab *partial_slab)
3078
{
3079
	struct kmem_cache_node *n = NULL, *n2 = NULL;
3080
	struct slab *slab, *slab_to_discard = NULL;
3081
	unsigned long flags = 0;
3082

3083 3084 3085
	while (partial_slab) {
		slab = partial_slab;
		partial_slab = slab->next;
3086

3087
		n2 = get_node(s, slab_nid(slab));
3088 3089
		if (n != n2) {
			if (n)
3090
				spin_unlock_irqrestore(&n->list_lock, flags);
3091 3092

			n = n2;
3093
			spin_lock_irqsave(&n->list_lock, flags);
3094
		}
3095

3096
		if (unlikely(!slab->inuse && n->nr_partial >= s->min_partial)) {
3097 3098
			slab->next = slab_to_discard;
			slab_to_discard = slab;
3099
		} else {
3100
			add_partial(n, slab, DEACTIVATE_TO_TAIL);
3101
			stat(s, FREE_ADD_PARTIAL);
3102 3103 3104 3105
		}
	}

	if (n)
3106
		spin_unlock_irqrestore(&n->list_lock, flags);
3107

3108 3109 3110
	while (slab_to_discard) {
		slab = slab_to_discard;
		slab_to_discard = slab_to_discard->next;
3111 3112

		stat(s, DEACTIVATE_EMPTY);
3113
		discard_slab(s, slab);
3114 3115
		stat(s, FREE_SLAB);
	}
3116
}
3117

3118
/*
3119
 * Put all the cpu partial slabs to the node partial list.
3120
 */
3121
static void put_partials(struct kmem_cache *s)
3122
{
3123
	struct slab *partial_slab;
3124 3125
	unsigned long flags;

3126
	local_lock_irqsave(&s->cpu_slab->lock, flags);
3127
	partial_slab = this_cpu_read(s->cpu_slab->partial);
3128
	this_cpu_write(s->cpu_slab->partial, NULL);
3129
	local_unlock_irqrestore(&s->cpu_slab->lock, flags);
3130

3131
	if (partial_slab)
3132
		__put_partials(s, partial_slab);
3133 3134
}

3135 3136
static void put_partials_cpu(struct kmem_cache *s,
			     struct kmem_cache_cpu *c)
3137
{
3138
	struct slab *partial_slab;
3139

3140
	partial_slab = slub_percpu_partial(c);
3141 3142
	c->partial = NULL;

3143
	if (partial_slab)
3144
		__put_partials(s, partial_slab);
3145 3146 3147
}

/*
3148
 * Put a slab into a partial slab slot if available.
3149 3150 3151 3152
 *
 * If we did not find a slot then simply move all the partials to the
 * per node partial list.
 */
3153
static void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain)
3154
{
3155
	struct slab *oldslab;
3156
	struct slab *slab_to_put = NULL;
3157
	unsigned long flags;
3158
	int slabs = 0;
3159

3160
	local_lock_irqsave(&s->cpu_slab->lock, flags);
3161

3162
	oldslab = this_cpu_read(s->cpu_slab->partial);
3163

3164 3165
	if (oldslab) {
		if (drain && oldslab->slabs >= s->cpu_partial_slabs) {
3166 3167 3168 3169 3170
			/*
			 * Partial array is full. Move the existing set to the
			 * per node partial list. Postpone the actual unfreezing
			 * outside of the critical section.
			 */
3171
			slab_to_put = oldslab;
3172
			oldslab = NULL;
3173
		} else {
3174
			slabs = oldslab->slabs;
3175
		}
3176
	}
3177

3178
	slabs++;
3179

3180 3181
	slab->slabs = slabs;
	slab->next = oldslab;
3182

3183
	this_cpu_write(s->cpu_slab->partial, slab);
3184

3185
	local_unlock_irqrestore(&s->cpu_slab->lock, flags);
3186

3187 3188
	if (slab_to_put) {
		__put_partials(s, slab_to_put);
3189 3190
		stat(s, CPU_PARTIAL_DRAIN);
	}
3191 3192
}

3193 3194
#else	/* CONFIG_SLUB_CPU_PARTIAL */

3195 3196 3197
static inline void put_partials(struct kmem_cache *s) { }
static inline void put_partials_cpu(struct kmem_cache *s,
				    struct kmem_cache_cpu *c) { }
3198 3199 3200

#endif	/* CONFIG_SLUB_CPU_PARTIAL */

3201
static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
Christoph Lameter's avatar
Christoph Lameter committed
3202
{
3203
	unsigned long flags;
3204
	struct slab *slab;
3205 3206
	void *freelist;

3207
	local_lock_irqsave(&s->cpu_slab->lock, flags);
3208

3209
	slab = c->slab;
3210
	freelist = c->freelist;
3211

3212
	c->slab = NULL;
3213
	c->freelist = NULL;
3214
	c->tid = next_tid(c->tid);
3215

3216
	local_unlock_irqrestore(&s->cpu_slab->lock, flags);
3217

3218 3219
	if (slab) {
		deactivate_slab(s, slab, freelist);
3220 3221
		stat(s, CPUSLAB_FLUSH);
	}
Christoph Lameter's avatar
Christoph Lameter committed
3222 3223
}

3224
static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu)
Christoph Lameter's avatar
Christoph Lameter committed
3225
{
3226
	struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
3227
	void *freelist = c->freelist;
3228
	struct slab *slab = c->slab;
Christoph Lameter's avatar
Christoph Lameter committed
3229

3230
	c->slab = NULL;
3231 3232 3233
	c->freelist = NULL;
	c->tid = next_tid(c->tid);

3234 3235
	if (slab) {
		deactivate_slab(s, slab, freelist);
3236 3237
		stat(s, CPUSLAB_FLUSH);
	}
3238

3239
	put_partials_cpu(s, c);
Christoph Lameter's avatar
Christoph Lameter committed
3240 3241
}

3242 3243 3244 3245 3246 3247
struct slub_flush_work {
	struct work_struct work;
	struct kmem_cache *s;
	bool skip;
};

3248 3249 3250
/*
 * Flush cpu slab.
 *
3251
 * Called from CPU work handler with migration disabled.
3252
 */
3253
static void flush_cpu_slab(struct work_struct *w)
Christoph Lameter's avatar
Christoph Lameter committed
3254
{
3255 3256 3257 3258 3259 3260 3261 3262
	struct kmem_cache *s;
	struct kmem_cache_cpu *c;
	struct slub_flush_work *sfw;

	sfw = container_of(w, struct slub_flush_work, work);

	s = sfw->s;
	c = this_cpu_ptr(s->cpu_slab);
3263

3264
	if (c->slab)
3265
		flush_slab(s, c);
Christoph Lameter's avatar
Christoph Lameter committed
3266

3267
	put_partials(s);
Christoph Lameter's avatar
Christoph Lameter committed
3268 3269
}

3270
static bool has_cpu_slab(int cpu, struct kmem_cache *s)
3271 3272 3273
{
	struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);

3274
	return c->slab || slub_percpu_partial(c);
3275 3276
}

3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296
static DEFINE_MUTEX(flush_lock);
static DEFINE_PER_CPU(struct slub_flush_work, slub_flush);

static void flush_all_cpus_locked(struct kmem_cache *s)
{
	struct slub_flush_work *sfw;
	unsigned int cpu;

	lockdep_assert_cpus_held();
	mutex_lock(&flush_lock);

	for_each_online_cpu(cpu) {
		sfw = &per_cpu(slub_flush, cpu);
		if (!has_cpu_slab(cpu, s)) {
			sfw->skip = true;
			continue;
		}
		INIT_WORK(&sfw->work, flush_cpu_slab);
		sfw->skip = false;
		sfw->s = s;
3297
		queue_work_on(cpu, flushwq, &sfw->work);
3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309
	}

	for_each_online_cpu(cpu) {
		sfw = &per_cpu(slub_flush, cpu);
		if (sfw->skip)
			continue;
		flush_work(&sfw->work);
	}

	mutex_unlock(&flush_lock);
}

Christoph Lameter's avatar
Christoph Lameter committed
3310 3311
static void flush_all(struct kmem_cache *s)
{
3312 3313 3314
	cpus_read_lock();
	flush_all_cpus_locked(s);
	cpus_read_unlock();
Christoph Lameter's avatar
Christoph Lameter committed
3315 3316
}

3317 3318 3319 3320 3321 3322 3323 3324 3325
/*
 * Use the cpu notifier to insure that the cpu slabs are flushed when
 * necessary.
 */
static int slub_cpu_dead(unsigned int cpu)
{
	struct kmem_cache *s;

	mutex_lock(&slab_mutex);
3326
	list_for_each_entry(s, &slab_caches, list)
3327 3328 3329 3330 3331
		__flush_cpu_slab(s, cpu);
	mutex_unlock(&slab_mutex);
	return 0;
}

3332 3333 3334 3335 3336 3337 3338
#else /* CONFIG_SLUB_TINY */
static inline void flush_all_cpus_locked(struct kmem_cache *s) { }
static inline void flush_all(struct kmem_cache *s) { }
static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu) { }
static inline int slub_cpu_dead(unsigned int cpu) { return 0; }
#endif /* CONFIG_SLUB_TINY */

3339 3340 3341 3342
/*
 * Check if the objects in a per cpu structure fit numa
 * locality expectations.
 */
3343
static inline int node_match(struct slab *slab, int node)
3344 3345
{
#ifdef CONFIG_NUMA
3346
	if (node != NUMA_NO_NODE && slab_nid(slab) != node)
3347 3348 3349 3350 3351
		return 0;
#endif
	return 1;
}

3352
#ifdef CONFIG_SLUB_DEBUG
3353
static int count_free(struct slab *slab)
3354
{
3355
	return slab->objects - slab->inuse;
3356 3357
}

3358 3359 3360 3361
static inline unsigned long node_nr_objs(struct kmem_cache_node *n)
{
	return atomic_long_read(&n->total_objects);
}
3362 3363

/* Supports checking bulk free of a constructed freelist */
3364 3365 3366
static inline bool free_debug_processing(struct kmem_cache *s,
	struct slab *slab, void *head, void *tail, int *bulk_cnt,
	unsigned long addr, depot_stack_handle_t handle)
3367
{
3368
	bool checks_ok = false;
3369 3370 3371 3372 3373 3374 3375 3376
	void *object = head;
	int cnt = 0;

	if (s->flags & SLAB_CONSISTENCY_CHECKS) {
		if (!check_slab(s, slab))
			goto out;
	}

3377
	if (slab->inuse < *bulk_cnt) {
3378
		slab_err(s, slab, "Slab has %d allocated objects but %d are to be freed\n",
3379
			 slab->inuse, *bulk_cnt);
3380 3381 3382
		goto out;
	}

3383
next_object:
3384

3385
	if (++cnt > *bulk_cnt)
3386
		goto out_cnt;
3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403

	if (s->flags & SLAB_CONSISTENCY_CHECKS) {
		if (!free_consistency_checks(s, slab, object, addr))
			goto out;
	}

	if (s->flags & SLAB_STORE_USER)
		set_track_update(s, object, TRACK_FREE, addr, handle);
	trace(s, slab, object, 0);
	/* Freepointer not overwritten by init_object(), SLAB_POISON moved it */
	init_object(s, object, SLUB_RED_INACTIVE);

	/* Reached end of constructed freelist yet? */
	if (object != tail) {
		object = get_freepointer(s, object);
		goto next_object;
	}
3404
	checks_ok = true;
3405

3406
out_cnt:
3407
	if (cnt != *bulk_cnt) {
3408
		slab_err(s, slab, "Bulk free expected %d objects but found %d\n",
3409 3410
			 *bulk_cnt, cnt);
		*bulk_cnt = cnt;
3411 3412
	}

3413
out:
3414 3415

	if (!checks_ok)
3416
		slab_fix(s, "Object at 0x%p not freed", object);
3417

3418
	return checks_ok;
3419
}
3420 3421
#endif /* CONFIG_SLUB_DEBUG */

3422
#if defined(CONFIG_SLUB_DEBUG) || defined(SLAB_SUPPORTS_SYSFS)
3423
static unsigned long count_partial(struct kmem_cache_node *n,
3424
					int (*get_count)(struct slab *))
3425 3426 3427
{
	unsigned long flags;
	unsigned long x = 0;
3428
	struct slab *slab;
3429 3430

	spin_lock_irqsave(&n->list_lock, flags);
3431 3432
	list_for_each_entry(slab, &n->partial, slab_list)
		x += get_count(slab);
3433 3434 3435
	spin_unlock_irqrestore(&n->list_lock, flags);
	return x;
}
3436
#endif /* CONFIG_SLUB_DEBUG || SLAB_SUPPORTS_SYSFS */
3437

3438
#ifdef CONFIG_SLUB_DEBUG
3439 3440 3441
static noinline void
slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid)
{
3442 3443
	static DEFINE_RATELIMIT_STATE(slub_oom_rs, DEFAULT_RATELIMIT_INTERVAL,
				      DEFAULT_RATELIMIT_BURST);
3444
	int node;
3445
	struct kmem_cache_node *n;
3446

3447 3448 3449
	if ((gfpflags & __GFP_NOWARN) || !__ratelimit(&slub_oom_rs))
		return;

3450 3451
	pr_warn("SLUB: Unable to allocate memory on node %d, gfp=%#x(%pGg)\n",
		nid, gfpflags, &gfpflags);
3452
	pr_warn("  cache: %s, object size: %u, buffer size: %u, default order: %u, min order: %u\n",
3453 3454
		s->name, s->object_size, s->size, oo_order(s->oo),
		oo_order(s->min));
3455

3456
	if (oo_order(s->min) > get_order(s->object_size))
3457
		pr_warn("  %s debugging increased min order, use slab_debug=O to disable.\n",
3458
			s->name);
3459

3460
	for_each_kmem_cache_node(s, node, n) {
3461 3462 3463 3464
		unsigned long nr_slabs;
		unsigned long nr_objs;
		unsigned long nr_free;

3465 3466 3467
		nr_free  = count_partial(n, count_free);
		nr_slabs = node_nr_slabs(n);
		nr_objs  = node_nr_objs(n);
3468

3469
		pr_warn("  node %d: slabs: %ld, objs: %ld, free: %ld\n",
3470 3471 3472
			node, nr_slabs, nr_objs, nr_free);
	}
}
3473 3474 3475 3476
#else /* CONFIG_SLUB_DEBUG */
static inline void
slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid) { }
#endif
3477

3478
static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags)
3479
{
3480
	if (unlikely(slab_test_pfmemalloc(slab)))
3481 3482 3483 3484 3485
		return gfp_pfmemalloc_allowed(gfpflags);

	return true;
}

3486
#ifndef CONFIG_SLUB_TINY
3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498
static inline bool
__update_cpu_freelist_fast(struct kmem_cache *s,
			   void *freelist_old, void *freelist_new,
			   unsigned long tid)
{
	freelist_aba_t old = { .freelist = freelist_old, .counter = tid };
	freelist_aba_t new = { .freelist = freelist_new, .counter = next_tid(tid) };

	return this_cpu_try_cmpxchg_freelist(s->cpu_slab->freelist_tid.full,
					     &old.full, new.full);
}

3499
/*
3500 3501
 * Check the slab->freelist and either transfer the freelist to the
 * per cpu freelist or deactivate the slab.
3502
 *
3503
 * The slab is still frozen if the return value is not NULL.
3504
 *
3505
 * If this function returns NULL then the slab has been unfrozen.
3506
 */
3507
static inline void *get_freelist(struct kmem_cache *s, struct slab *slab)
3508
{
3509
	struct slab new;
3510 3511 3512
	unsigned long counters;
	void *freelist;

3513 3514
	lockdep_assert_held(this_cpu_ptr(&s->cpu_slab->lock));

3515
	do {
3516 3517
		freelist = slab->freelist;
		counters = slab->counters;
3518

3519 3520
		new.counters = counters;

3521
		new.inuse = slab->objects;
3522 3523
		new.frozen = freelist != NULL;

3524
	} while (!__slab_update_freelist(s, slab,
3525 3526 3527 3528 3529 3530 3531
		freelist, counters,
		NULL, new.counters,
		"get_freelist"));

	return freelist;
}

3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558
/*
 * Freeze the partial slab and return the pointer to the freelist.
 */
static inline void *freeze_slab(struct kmem_cache *s, struct slab *slab)
{
	struct slab new;
	unsigned long counters;
	void *freelist;

	do {
		freelist = slab->freelist;
		counters = slab->counters;

		new.counters = counters;
		VM_BUG_ON(new.frozen);

		new.inuse = slab->objects;
		new.frozen = 1;

	} while (!slab_update_freelist(s, slab,
		freelist, counters,
		NULL, new.counters,
		"freeze_slab"));

	return freelist;
}

Christoph Lameter's avatar
Christoph Lameter committed
3559
/*
3560 3561 3562 3563 3564 3565
 * Slow path. The lockless freelist is empty or we need to perform
 * debugging duties.
 *
 * Processing is still very fast if new objects have been freed to the
 * regular freelist. In that case we simply take over the regular freelist
 * as the lockless freelist and zap the regular freelist.
Christoph Lameter's avatar
Christoph Lameter committed
3566
 *
3567 3568 3569
 * If that is not working then we fall back to the partial lists. We take the
 * first element of the freelist as the object to allocate now and move the
 * rest of the freelist to the lockless freelist.
Christoph Lameter's avatar
Christoph Lameter committed
3570
 *
3571
 * And if we were unable to get a new slab from the partial slab lists then
Christoph Lameter's avatar
Christoph Lameter committed
3572 3573
 * we need to allocate a new slab. This is the slowest path since it involves
 * a call to the page allocator and the setup of a new slab.
3574
 *
3575
 * Version of __slab_alloc to use when we know that preemption is
3576
 * already disabled (which is the case for bulk allocation).
Christoph Lameter's avatar
Christoph Lameter committed
3577
 */
3578
static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
3579
			  unsigned long addr, struct kmem_cache_cpu *c, unsigned int orig_size)
Christoph Lameter's avatar
Christoph Lameter committed
3580
{
3581
	void *freelist;
3582
	struct slab *slab;
3583
	unsigned long flags;
3584
	struct partial_context pc;
Christoph Lameter's avatar
Christoph Lameter committed
3585

3586 3587
	stat(s, ALLOC_SLOWPATH);

3588
reread_slab:
3589

3590 3591
	slab = READ_ONCE(c->slab);
	if (!slab) {
3592 3593 3594 3595 3596
		/*
		 * if the node is not online or has no normal memory, just
		 * ignore the node constraint
		 */
		if (unlikely(node != NUMA_NO_NODE &&
3597
			     !node_isset(node, slab_nodes)))
3598
			node = NUMA_NO_NODE;
Christoph Lameter's avatar
Christoph Lameter committed
3599
		goto new_slab;
3600
	}
3601

3602
	if (unlikely(!node_match(slab, node))) {
3603 3604 3605 3606
		/*
		 * same as above but node_match() being false already
		 * implies node != NUMA_NO_NODE
		 */
3607
		if (!node_isset(node, slab_nodes)) {
3608 3609
			node = NUMA_NO_NODE;
		} else {
3610
			stat(s, ALLOC_NODE_MISMATCH);
3611
			goto deactivate_slab;
3612
		}
3613
	}
Christoph Lameter's avatar
Christoph Lameter committed
3614

3615 3616 3617 3618 3619
	/*
	 * By rights, we should be searching for a slab page that was
	 * PFMEMALLOC but right now, we are losing the pfmemalloc
	 * information when the page leaves the per-cpu allocator
	 */
3620
	if (unlikely(!pfmemalloc_match(slab, gfpflags)))
3621
		goto deactivate_slab;
3622

3623
	/* must check again c->slab in case we got preempted and it changed */
3624
	local_lock_irqsave(&s->cpu_slab->lock, flags);
3625
	if (unlikely(slab != c->slab)) {
3626
		local_unlock_irqrestore(&s->cpu_slab->lock, flags);
3627
		goto reread_slab;
3628
	}
3629 3630
	freelist = c->freelist;
	if (freelist)
3631
		goto load_freelist;
3632

3633
	freelist = get_freelist(s, slab);
Christoph Lameter's avatar
Christoph Lameter committed
3634

3635
	if (!freelist) {
3636
		c->slab = NULL;
3637
		c->tid = next_tid(c->tid);
3638
		local_unlock_irqrestore(&s->cpu_slab->lock, flags);
3639
		stat(s, DEACTIVATE_BYPASS);
3640
		goto new_slab;
3641
	}
Christoph Lameter's avatar
Christoph Lameter committed
3642

3643
	stat(s, ALLOC_REFILL);
Christoph Lameter's avatar
Christoph Lameter committed
3644

3645
load_freelist:
3646

3647
	lockdep_assert_held(this_cpu_ptr(&s->cpu_slab->lock));
3648

3649 3650
	/*
	 * freelist is pointing to the list of objects to be used.
3651 3652
	 * slab is pointing to the slab from which the objects are obtained.
	 * That slab must be frozen for per cpu allocations to work.
3653
	 */
3654
	VM_BUG_ON(!c->slab->frozen);
3655
	c->freelist = get_freepointer(s, freelist);
3656
	c->tid = next_tid(c->tid);
3657
	local_unlock_irqrestore(&s->cpu_slab->lock, flags);
3658
	return freelist;
Christoph Lameter's avatar
Christoph Lameter committed
3659

3660 3661
deactivate_slab:

3662
	local_lock_irqsave(&s->cpu_slab->lock, flags);
3663
	if (slab != c->slab) {
3664
		local_unlock_irqrestore(&s->cpu_slab->lock, flags);
3665
		goto reread_slab;
3666
	}
3667
	freelist = c->freelist;
3668
	c->slab = NULL;
3669
	c->freelist = NULL;
3670
	c->tid = next_tid(c->tid);
3671
	local_unlock_irqrestore(&s->cpu_slab->lock, flags);
3672
	deactivate_slab(s, slab, freelist);
3673

Christoph Lameter's avatar
Christoph Lameter committed
3674
new_slab:
3675

3676 3677
#ifdef CONFIG_SLUB_CPU_PARTIAL
	while (slub_percpu_partial(c)) {
3678
		local_lock_irqsave(&s->cpu_slab->lock, flags);
3679
		if (unlikely(c->slab)) {
3680
			local_unlock_irqrestore(&s->cpu_slab->lock, flags);
3681
			goto reread_slab;
3682
		}
3683
		if (unlikely(!slub_percpu_partial(c))) {
3684
			local_unlock_irqrestore(&s->cpu_slab->lock, flags);
3685 3686
			/* we were preempted and partial list got empty */
			goto new_objects;
3687
		}
3688

3689
		slab = slub_percpu_partial(c);
3690
		slub_set_percpu_partial(c, slab);
3691

3692 3693 3694 3695 3696 3697 3698
		if (likely(node_match(slab, node) &&
			   pfmemalloc_match(slab, gfpflags))) {
			c->slab = slab;
			freelist = get_freelist(s, slab);
			VM_BUG_ON(!freelist);
			stat(s, CPU_PARTIAL_ALLOC);
			goto load_freelist;
3699 3700
		}

3701 3702 3703 3704
		local_unlock_irqrestore(&s->cpu_slab->lock, flags);

		slab->next = NULL;
		__put_partials(s, slab);
Christoph Lameter's avatar
Christoph Lameter committed
3705
	}
3706
#endif
Christoph Lameter's avatar
Christoph Lameter committed
3707

3708 3709
new_objects:

3710 3711
	pc.flags = gfpflags;
	pc.orig_size = orig_size;
3712 3713
	slab = get_partial(s, node, &pc);
	if (slab) {
3714
		if (kmem_cache_debug(s)) {
3715
			freelist = pc.object;
3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726
			/*
			 * For debug caches here we had to go through
			 * alloc_single_from_partial() so just store the
			 * tracking info and return the object.
			 */
			if (s->flags & SLAB_STORE_USER)
				set_track(s, freelist, TRACK_ALLOC, addr);

			return freelist;
		}

3727
		freelist = freeze_slab(s, slab);
3728 3729
		goto retry_load_slab;
	}
3730

3731
	slub_put_cpu_ptr(s->cpu_slab);
3732
	slab = new_slab(s, gfpflags, node);
3733
	c = slub_get_cpu_ptr(s->cpu_slab);
3734

3735
	if (unlikely(!slab)) {
3736
		slab_out_of_memory(s, gfpflags, node);
3737
		return NULL;
Christoph Lameter's avatar
Christoph Lameter committed
3738
	}
3739

3740 3741 3742
	stat(s, ALLOC_SLAB);

	if (kmem_cache_debug(s)) {
3743
		freelist = alloc_single_from_new_slab(s, slab, orig_size);
3744 3745 3746 3747 3748 3749 3750 3751 3752 3753

		if (unlikely(!freelist))
			goto new_objects;

		if (s->flags & SLAB_STORE_USER)
			set_track(s, freelist, TRACK_ALLOC, addr);

		return freelist;
	}

3754
	/*
3755
	 * No other reference to the slab yet so we can
3756 3757
	 * muck around with it freely without cmpxchg
	 */
3758 3759
	freelist = slab->freelist;
	slab->freelist = NULL;
3760 3761
	slab->inuse = slab->objects;
	slab->frozen = 1;
3762

3763
	inc_slabs_node(s, slab_nid(slab), slab->objects);
3764

3765
	if (unlikely(!pfmemalloc_match(slab, gfpflags))) {
3766 3767 3768 3769
		/*
		 * For !pfmemalloc_match() case we don't load freelist so that
		 * we don't make further mismatched allocations easier.
		 */
3770 3771 3772
		deactivate_slab(s, slab, get_freepointer(s, freelist));
		return freelist;
	}
3773

3774
retry_load_slab:
3775

3776
	local_lock_irqsave(&s->cpu_slab->lock, flags);
3777
	if (unlikely(c->slab)) {
3778
		void *flush_freelist = c->freelist;
3779
		struct slab *flush_slab = c->slab;
3780

3781
		c->slab = NULL;
3782 3783 3784
		c->freelist = NULL;
		c->tid = next_tid(c->tid);

3785
		local_unlock_irqrestore(&s->cpu_slab->lock, flags);
3786

3787
		deactivate_slab(s, flush_slab, flush_freelist);
3788 3789 3790

		stat(s, CPUSLAB_FLUSH);

3791
		goto retry_load_slab;
3792
	}
3793
	c->slab = slab;
3794

3795
	goto load_freelist;
3796 3797
}

3798
/*
3799 3800 3801
 * A wrapper for ___slab_alloc() for contexts where preemption is not yet
 * disabled. Compensates for possible cpu changes by refetching the per cpu area
 * pointer.
3802 3803
 */
static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
3804
			  unsigned long addr, struct kmem_cache_cpu *c, unsigned int orig_size)
3805 3806 3807
{
	void *p;

3808
#ifdef CONFIG_PREEMPT_COUNT
3809 3810
	/*
	 * We may have been preempted and rescheduled on a different
3811
	 * cpu before disabling preemption. Need to reload cpu area
3812 3813
	 * pointer.
	 */
3814
	c = slub_get_cpu_ptr(s->cpu_slab);
3815 3816
#endif

3817
	p = ___slab_alloc(s, gfpflags, node, addr, c, orig_size);
3818
#ifdef CONFIG_PREEMPT_COUNT
3819
	slub_put_cpu_ptr(s->cpu_slab);
3820
#endif
3821 3822 3823
	return p;
}

3824
static __always_inline void *__slab_alloc_node(struct kmem_cache *s,
3825
		gfp_t gfpflags, int node, unsigned long addr, size_t orig_size)
3826
{
3827
	struct kmem_cache_cpu *c;
3828
	struct slab *slab;
3829
	unsigned long tid;
3830
	void *object;
3831

3832 3833 3834 3835 3836 3837
redo:
	/*
	 * Must read kmem_cache cpu data via this cpu ptr. Preemption is
	 * enabled. We may switch back and forth between cpus while
	 * reading from one cpu area. That does not matter as long
	 * as we end up on the original cpu again when doing the cmpxchg.
3838
	 *
3839 3840 3841 3842 3843
	 * We must guarantee that tid and kmem_cache_cpu are retrieved on the
	 * same cpu. We read first the kmem_cache_cpu pointer and use it to read
	 * the tid. If we are preempted and switched to another cpu between the
	 * two reads, it's OK as the two are still associated with the same cpu
	 * and cmpxchg later will validate the cpu.
3844
	 */
3845 3846
	c = raw_cpu_ptr(s->cpu_slab);
	tid = READ_ONCE(c->tid);
3847 3848 3849 3850

	/*
	 * Irqless object alloc/free algorithm used here depends on sequence
	 * of fetching cpu_slab's data. tid should be fetched before anything
3851
	 * on c to guarantee that object and slab associated with previous tid
3852
	 * won't be used with current tid. If we fetch tid first, object and
3853
	 * slab could be one associated with next tid and our alloc/free
3854 3855 3856
	 * request will be failed. In this case, we will retry. So, no problem.
	 */
	barrier();
3857 3858 3859 3860 3861 3862 3863 3864

	/*
	 * The transaction ids are globally unique per cpu and per operation on
	 * a per cpu queue. Thus they can be guarantee that the cmpxchg_double
	 * occurs on the right processor and that there was no operation on the
	 * linked list in between.
	 */

3865
	object = c->freelist;
3866
	slab = c->slab;
3867 3868

	if (!USE_LOCKLESS_FAST_PATH() ||
3869
	    unlikely(!object || !slab || !node_match(slab, node))) {
3870
		object = __slab_alloc(s, gfpflags, node, addr, c, orig_size);
3871
	} else {
3872 3873
		void *next_object = get_freepointer_safe(s, object);

3874
		/*
Lucas De Marchi's avatar
Lucas De Marchi committed
3875
		 * The cmpxchg will only match if there was no additional
3876 3877
		 * operation and if we are on the right processor.
		 *
3878 3879
		 * The cmpxchg does the following atomically (without lock
		 * semantics!)
3880 3881 3882 3883
		 * 1. Relocate first pointer to the current per cpu area.
		 * 2. Verify that tid and freelist have not been changed
		 * 3. If they were not changed replace tid and freelist
		 *
3884 3885 3886
		 * Since this is without lock semantics the protection is only
		 * against code executing on this cpu *not* from access by
		 * other cpus.
3887
		 */
3888
		if (unlikely(!__update_cpu_freelist_fast(s, object, next_object, tid))) {
3889 3890 3891
			note_cmpxchg_failure("slab_alloc", s, tid);
			goto redo;
		}
3892
		prefetch_freepointer(s, next_object);
3893
		stat(s, ALLOC_FASTPATH);
3894
	}
3895

3896 3897
	return object;
}
3898 3899 3900 3901 3902 3903 3904 3905 3906 3907
#else /* CONFIG_SLUB_TINY */
static void *__slab_alloc_node(struct kmem_cache *s,
		gfp_t gfpflags, int node, unsigned long addr, size_t orig_size)
{
	struct partial_context pc;
	struct slab *slab;
	void *object;

	pc.flags = gfpflags;
	pc.orig_size = orig_size;
3908
	slab = get_partial(s, node, &pc);
3909

3910 3911
	if (slab)
		return pc.object;
3912 3913 3914 3915 3916 3917 3918 3919 3920 3921 3922 3923

	slab = new_slab(s, gfpflags, node);
	if (unlikely(!slab)) {
		slab_out_of_memory(s, gfpflags, node);
		return NULL;
	}

	object = alloc_single_from_new_slab(s, slab, orig_size);

	return object;
}
#endif /* CONFIG_SLUB_TINY */
3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936

/*
 * If the object has been wiped upon free, make sure it's fully initialized by
 * zeroing out freelist pointer.
 */
static __always_inline void maybe_wipe_obj_freeptr(struct kmem_cache *s,
						   void *obj)
{
	if (unlikely(slab_want_init_on_free(s)) && obj)
		memset((void *)((char *)kasan_reset_tag(obj) + s->offset),
			0, sizeof(void *));
}

3937 3938 3939 3940 3941 3942 3943 3944
noinline int should_failslab(struct kmem_cache *s, gfp_t gfpflags)
{
	if (__should_failslab(s, gfpflags))
		return -ENOMEM;
	return 0;
}
ALLOW_ERROR_INJECTION(should_failslab, ERRNO);

3945 3946 3947 3948 3949
static __fastpath_inline
struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s,
				       struct list_lru *lru,
				       struct obj_cgroup **objcgp,
				       size_t size, gfp_t flags)
3950 3951 3952 3953 3954
{
	flags &= gfp_allowed_mask;

	might_alloc(flags);

3955
	if (unlikely(should_failslab(s, flags)))
3956 3957
		return NULL;

3958
	if (unlikely(!memcg_slab_pre_alloc_hook(s, lru, objcgp, size, flags)))
3959 3960 3961 3962 3963
		return NULL;

	return s;
}

3964 3965 3966 3967
static __fastpath_inline
void slab_post_alloc_hook(struct kmem_cache *s,	struct obj_cgroup *objcg,
			  gfp_t flags, size_t size, void **p, bool init,
			  unsigned int orig_size)
3968 3969
{
	unsigned int zero_size = s->object_size;
3970
	struct slabobj_ext *obj_exts;
3971 3972
	bool kasan_init = init;
	size_t i;
3973
	gfp_t init_flags = flags & gfp_allowed_mask;
3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987

	/*
	 * For kmalloc object, the allocated memory size(object_size) is likely
	 * larger than the requested size(orig_size). If redzone check is
	 * enabled for the extra space, don't zero it, as it will be redzoned
	 * soon. The redzone operation for this extra space could be seen as a
	 * replacement of current poisoning under certain debug option, and
	 * won't break other sanity checks.
	 */
	if (kmem_cache_debug_flags(s, SLAB_STORE_USER | SLAB_RED_ZONE) &&
	    (s->flags & SLAB_KMALLOC))
		zero_size = orig_size;

	/*
3988
	 * When slab_debug is enabled, avoid memory initialization integrated
3989 3990 3991
	 * into KASAN and instead zero out the memory via the memset below with
	 * the proper size. Otherwise, KASAN might overwrite SLUB redzones and
	 * cause false-positive reports. This does not lead to a performance
3992
	 * penalty on production builds, as slab_debug is not intended to be
3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005
	 * enabled there.
	 */
	if (__slub_debug_enabled())
		kasan_init = false;

	/*
	 * As memory initialization might be integrated into KASAN,
	 * kasan_slab_alloc and initialization memset must be
	 * kept together to avoid discrepancies in behavior.
	 *
	 * As p[i] might get tagged, memset and kmemleak hook come after KASAN.
	 */
	for (i = 0; i < size; i++) {
4006
		p[i] = kasan_slab_alloc(s, p[i], init_flags, kasan_init);
4007 4008 4009 4010
		if (p[i] && init && (!kasan_init ||
				     !kasan_has_integrated_init()))
			memset(p[i], 0, zero_size);
		kmemleak_alloc_recursive(p[i], s->object_size, 1,
4011 4012
					 s->flags, init_flags);
		kmsan_slab_alloc(s, p[i], init_flags);
4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024
		if (need_slab_obj_ext()) {
			obj_exts = prepare_slab_obj_exts_hook(s, flags, p[i]);
#ifdef CONFIG_MEM_ALLOC_PROFILING
			/*
			 * Currently obj_exts is used only for allocation profiling.
			 * If other users appear then mem_alloc_profiling_enabled()
			 * check should be added before alloc_tag_add().
			 */
			if (likely(obj_exts))
				alloc_tag_add(&obj_exts->ref, current->alloc_tag, s->size);
#endif
		}
4025 4026 4027 4028 4029
	}

	memcg_slab_post_alloc_hook(s, objcg, flags, size, p);
}

4030 4031 4032 4033 4034 4035 4036 4037 4038 4039
/*
 * Inlined fastpath so that allocation functions (kmalloc, kmem_cache_alloc)
 * have the fastpath folded into their functions. So no function call
 * overhead for requests that can be satisfied on the fastpath.
 *
 * The fastpath works by first checking if the lockless freelist can be used.
 * If not then __slab_alloc is called for slow processing.
 *
 * Otherwise we can simply pick the next object from the lockless free list.
 */
4040
static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s, struct list_lru *lru,
4041 4042 4043 4044 4045 4046 4047
		gfp_t gfpflags, int node, unsigned long addr, size_t orig_size)
{
	void *object;
	struct obj_cgroup *objcg = NULL;
	bool init = false;

	s = slab_pre_alloc_hook(s, lru, &objcg, 1, gfpflags);
4048
	if (unlikely(!s))
4049 4050 4051 4052 4053 4054 4055 4056
		return NULL;

	object = kfence_alloc(s, orig_size, gfpflags);
	if (unlikely(object))
		goto out;

	object = __slab_alloc_node(s, gfpflags, node, addr, orig_size);

4057
	maybe_wipe_obj_freeptr(s, object);
4058
	init = slab_want_init_on_alloc(gfpflags, s);
4059

4060
out:
4061 4062 4063 4064 4065
	/*
	 * When init equals 'true', like for kzalloc() family, only
	 * @orig_size bytes might be zeroed instead of s->object_size
	 */
	slab_post_alloc_hook(s, objcg, gfpflags, 1, &object, init, orig_size);
4066

4067
	return object;
Christoph Lameter's avatar
Christoph Lameter committed
4068 4069
}

4070
void *kmem_cache_alloc_noprof(struct kmem_cache *s, gfp_t gfpflags)
4071
{
4072 4073
	void *ret = slab_alloc_node(s, NULL, gfpflags, NUMA_NO_NODE, _RET_IP_,
				    s->object_size);
4074

4075
	trace_kmem_cache_alloc(_RET_IP_, ret, s, gfpflags, NUMA_NO_NODE);
4076 4077

	return ret;
4078
}
4079
EXPORT_SYMBOL(kmem_cache_alloc_noprof);
4080

4081
void *kmem_cache_alloc_lru_noprof(struct kmem_cache *s, struct list_lru *lru,
4082
			   gfp_t gfpflags)
Christoph Lameter's avatar
Christoph Lameter committed
4083
{
4084 4085
	void *ret = slab_alloc_node(s, lru, gfpflags, NUMA_NO_NODE, _RET_IP_,
				    s->object_size);
4086

4087
	trace_kmem_cache_alloc(_RET_IP_, ret, s, gfpflags, NUMA_NO_NODE);
4088 4089

	return ret;
Christoph Lameter's avatar
Christoph Lameter committed
4090
}
4091
EXPORT_SYMBOL(kmem_cache_alloc_lru_noprof);
4092

4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105
/**
 * kmem_cache_alloc_node - Allocate an object on the specified node
 * @s: The cache to allocate from.
 * @gfpflags: See kmalloc().
 * @node: node number of the target node.
 *
 * Identical to kmem_cache_alloc but it will allocate memory on the given
 * node, which can improve the performance for cpu bound structures.
 *
 * Fallback to other node is possible if __GFP_THISNODE is not set.
 *
 * Return: pointer to the new object or %NULL in case of error
 */
4106
void *kmem_cache_alloc_node_noprof(struct kmem_cache *s, gfp_t gfpflags, int node)
Christoph Lameter's avatar
Christoph Lameter committed
4107
{
4108
	void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, s->object_size);
4109

4110
	trace_kmem_cache_alloc(_RET_IP_, ret, s, gfpflags, node);
4111 4112

	return ret;
Christoph Lameter's avatar
Christoph Lameter committed
4113
}
4114
EXPORT_SYMBOL(kmem_cache_alloc_node_noprof);
Christoph Lameter's avatar
Christoph Lameter committed
4115

4116 4117 4118 4119 4120 4121
/*
 * To avoid unnecessary overhead, we pass through large allocation requests
 * directly to the page allocator. We use __GFP_COMP, because we will need to
 * know the allocation order to free the pages properly in kfree.
 */
static void *__kmalloc_large_node(size_t size, gfp_t flags, int node)
4122
{
4123
	struct folio *folio;
4124 4125 4126 4127 4128 4129 4130
	void *ptr = NULL;
	unsigned int order = get_order(size);

	if (unlikely(flags & GFP_SLAB_BUG_MASK))
		flags = kmalloc_fix_flags(flags);

	flags |= __GFP_COMP;
4131
	folio = (struct folio *)alloc_pages_node_noprof(node, flags, order);
4132 4133 4134
	if (folio) {
		ptr = folio_address(folio);
		lruvec_stat_mod_folio(folio, NR_SLAB_UNRECLAIMABLE_B,
4135 4136 4137 4138 4139 4140 4141 4142 4143
				      PAGE_SIZE << order);
	}

	ptr = kasan_kmalloc_large(ptr, size, flags);
	/* As ptr might get tagged, call kmemleak hook after KASAN. */
	kmemleak_alloc(ptr, size, 1, flags);
	kmsan_kmalloc_large(ptr, size, flags);

	return ptr;
4144
}
Christoph Lameter's avatar
Christoph Lameter committed
4145

4146
void *kmalloc_large_noprof(size_t size, gfp_t flags)
4147
{
4148 4149 4150 4151 4152
	void *ret = __kmalloc_large_node(size, flags, NUMA_NO_NODE);

	trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << get_order(size),
		      flags, NUMA_NO_NODE);
	return ret;
4153
}
4154
EXPORT_SYMBOL(kmalloc_large_noprof);
4155

4156
void *kmalloc_large_node_noprof(size_t size, gfp_t flags, int node)
4157
{
4158 4159 4160 4161 4162
	void *ret = __kmalloc_large_node(size, flags, node);

	trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << get_order(size),
		      flags, node);
	return ret;
4163
}
4164
EXPORT_SYMBOL(kmalloc_large_node_noprof);
4165

4166 4167 4168
static __always_inline
void *__do_kmalloc_node(size_t size, gfp_t flags, int node,
			unsigned long caller)
Christoph Lameter's avatar
Christoph Lameter committed
4169
{
4170 4171
	struct kmem_cache *s;
	void *ret;
4172

4173 4174 4175 4176 4177 4178
	if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
		ret = __kmalloc_large_node(size, flags, node);
		trace_kmalloc(caller, ret, size,
			      PAGE_SIZE << get_order(size), flags, node);
		return ret;
	}
4179

4180 4181 4182 4183 4184 4185 4186 4187
	if (unlikely(!size))
		return ZERO_SIZE_PTR;

	s = kmalloc_slab(size, flags, caller);

	ret = slab_alloc_node(s, NULL, flags, node, caller, size);
	ret = kasan_kmalloc(s, ret, size, flags);
	trace_kmalloc(caller, ret, size, s->size, flags, node);
4188
	return ret;
Christoph Lameter's avatar
Christoph Lameter committed
4189
}
4190

4191
void *__kmalloc_node_noprof(size_t size, gfp_t flags, int node)
4192 4193 4194
{
	return __do_kmalloc_node(size, flags, node, _RET_IP_);
}
4195
EXPORT_SYMBOL(__kmalloc_node_noprof);
4196

4197
void *__kmalloc_noprof(size_t size, gfp_t flags)
4198 4199 4200
{
	return __do_kmalloc_node(size, flags, NUMA_NO_NODE, _RET_IP_);
}
4201
EXPORT_SYMBOL(__kmalloc_noprof);
4202

4203 4204
void *kmalloc_node_track_caller_noprof(size_t size, gfp_t flags,
				       int node, unsigned long caller)
4205 4206 4207
{
	return __do_kmalloc_node(size, flags, node, caller);
}
4208
EXPORT_SYMBOL(kmalloc_node_track_caller_noprof);
4209

4210
void *kmalloc_trace_noprof(struct kmem_cache *s, gfp_t gfpflags, size_t size)
4211 4212 4213 4214 4215 4216 4217 4218 4219
{
	void *ret = slab_alloc_node(s, NULL, gfpflags, NUMA_NO_NODE,
					    _RET_IP_, size);

	trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags, NUMA_NO_NODE);

	ret = kasan_kmalloc(s, ret, size, gfpflags);
	return ret;
}
4220
EXPORT_SYMBOL(kmalloc_trace_noprof);
4221

4222
void *kmalloc_node_trace_noprof(struct kmem_cache *s, gfp_t gfpflags,
4223 4224 4225 4226 4227 4228 4229 4230 4231
			 int node, size_t size)
{
	void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, size);

	trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags, node);

	ret = kasan_kmalloc(s, ret, size, gfpflags);
	return ret;
}
4232
EXPORT_SYMBOL(kmalloc_node_trace_noprof);
Christoph Lameter's avatar
Christoph Lameter committed
4233

4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294
static noinline void free_to_partial_list(
	struct kmem_cache *s, struct slab *slab,
	void *head, void *tail, int bulk_cnt,
	unsigned long addr)
{
	struct kmem_cache_node *n = get_node(s, slab_nid(slab));
	struct slab *slab_free = NULL;
	int cnt = bulk_cnt;
	unsigned long flags;
	depot_stack_handle_t handle = 0;

	if (s->flags & SLAB_STORE_USER)
		handle = set_track_prepare();

	spin_lock_irqsave(&n->list_lock, flags);

	if (free_debug_processing(s, slab, head, tail, &cnt, addr, handle)) {
		void *prior = slab->freelist;

		/* Perform the actual freeing while we still hold the locks */
		slab->inuse -= cnt;
		set_freepointer(s, tail, prior);
		slab->freelist = head;

		/*
		 * If the slab is empty, and node's partial list is full,
		 * it should be discarded anyway no matter it's on full or
		 * partial list.
		 */
		if (slab->inuse == 0 && n->nr_partial >= s->min_partial)
			slab_free = slab;

		if (!prior) {
			/* was on full list */
			remove_full(s, n, slab);
			if (!slab_free) {
				add_partial(n, slab, DEACTIVATE_TO_TAIL);
				stat(s, FREE_ADD_PARTIAL);
			}
		} else if (slab_free) {
			remove_partial(n, slab);
			stat(s, FREE_REMOVE_PARTIAL);
		}
	}

	if (slab_free) {
		/*
		 * Update the counters while still holding n->list_lock to
		 * prevent spurious validation warnings
		 */
		dec_slabs_node(s, slab_nid(slab_free), slab_free->objects);
	}

	spin_unlock_irqrestore(&n->list_lock, flags);

	if (slab_free) {
		stat(s, FREE_SLAB);
		free_slab(s, slab_free);
	}
}

Christoph Lameter's avatar
Christoph Lameter committed
4295
/*
4296
 * Slow path handling. This may still be called frequently since objects
4297
 * have a longer lifetime than the cpu slabs in most processing loads.
Christoph Lameter's avatar
Christoph Lameter committed
4298
 *
4299
 * So we still attempt to reduce cache line usage. Just take the slab
4300
 * lock and free the item. If there is no additional partial slab
4301
 * handling required then we can return immediately.
Christoph Lameter's avatar
Christoph Lameter committed
4302
 */
4303
static void __slab_free(struct kmem_cache *s, struct slab *slab,
4304 4305 4306
			void *head, void *tail, int cnt,
			unsigned long addr)

Christoph Lameter's avatar
Christoph Lameter committed
4307 4308
{
	void *prior;
4309
	int was_frozen;
4310
	struct slab new;
4311 4312
	unsigned long counters;
	struct kmem_cache_node *n = NULL;
4313
	unsigned long flags;
4314
	bool on_node_partial;
Christoph Lameter's avatar
Christoph Lameter committed
4315

4316
	stat(s, FREE_SLOWPATH);
Christoph Lameter's avatar
Christoph Lameter committed
4317

4318
	if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) {
4319
		free_to_partial_list(s, slab, head, tail, cnt, addr);
4320
		return;
4321
	}
Christoph Lameter's avatar
Christoph Lameter committed
4322

4323
	do {
4324 4325 4326 4327
		if (unlikely(n)) {
			spin_unlock_irqrestore(&n->list_lock, flags);
			n = NULL;
		}
4328 4329
		prior = slab->freelist;
		counters = slab->counters;
4330
		set_freepointer(s, tail, prior);
4331 4332
		new.counters = counters;
		was_frozen = new.frozen;
4333
		new.inuse -= cnt;
4334
		if ((!new.inuse || !prior) && !was_frozen) {
4335 4336
			/* Needs to be taken off a list */
			if (!kmem_cache_has_cpu_partial(s) || prior) {
4337

4338
				n = get_node(s, slab_nid(slab));
4339 4340 4341 4342 4343 4344 4345 4346 4347 4348
				/*
				 * Speculatively acquire the list_lock.
				 * If the cmpxchg does not succeed then we may
				 * drop the list_lock without any processing.
				 *
				 * Otherwise the list_lock will synchronize with
				 * other processors updating the list of slabs.
				 */
				spin_lock_irqsave(&n->list_lock, flags);

4349
				on_node_partial = slab_test_node_partial(slab);
4350
			}
4351
		}
Christoph Lameter's avatar
Christoph Lameter committed
4352

4353
	} while (!slab_update_freelist(s, slab,
4354
		prior, counters,
4355
		head, new.counters,
4356
		"__slab_free"));
Christoph Lameter's avatar
Christoph Lameter committed
4357

4358
	if (likely(!n)) {
4359

4360 4361 4362 4363 4364 4365
		if (likely(was_frozen)) {
			/*
			 * The list lock was not taken therefore no list
			 * activity can be necessary.
			 */
			stat(s, FREE_FROZEN);
4366
		} else if (kmem_cache_has_cpu_partial(s) && !prior) {
4367
			/*
4368
			 * If we started with a full slab then put it onto the
4369 4370
			 * per cpu partial list.
			 */
4371
			put_cpu_partial(s, slab, 1);
4372 4373
			stat(s, CPU_PARTIAL_FREE);
		}
4374

4375 4376
		return;
	}
Christoph Lameter's avatar
Christoph Lameter committed
4377

4378 4379 4380 4381 4382 4383 4384 4385 4386
	/*
	 * This slab was partially empty but not on the per-node partial list,
	 * in which case we shouldn't manipulate its list, just return.
	 */
	if (prior && !on_node_partial) {
		spin_unlock_irqrestore(&n->list_lock, flags);
		return;
	}

4387
	if (unlikely(!new.inuse && n->nr_partial >= s->min_partial))
4388 4389
		goto slab_empty;

Christoph Lameter's avatar
Christoph Lameter committed
4390
	/*
4391 4392
	 * Objects left in the slab. If it was not on the partial list before
	 * then add it.
Christoph Lameter's avatar
Christoph Lameter committed
4393
	 */
4394
	if (!kmem_cache_has_cpu_partial(s) && unlikely(!prior)) {
4395
		add_partial(n, slab, DEACTIVATE_TO_TAIL);
4396
		stat(s, FREE_ADD_PARTIAL);
4397
	}
4398
	spin_unlock_irqrestore(&n->list_lock, flags);
Christoph Lameter's avatar
Christoph Lameter committed
4399 4400 4401
	return;

slab_empty:
4402
	if (prior) {
Christoph Lameter's avatar
Christoph Lameter committed
4403
		/*
4404
		 * Slab on the partial list.
Christoph Lameter's avatar
Christoph Lameter committed
4405
		 */
4406
		remove_partial(n, slab);
4407
		stat(s, FREE_REMOVE_PARTIAL);
4408
	}
4409

4410
	spin_unlock_irqrestore(&n->list_lock, flags);
4411
	stat(s, FREE_SLAB);
4412
	discard_slab(s, slab);
Christoph Lameter's avatar
Christoph Lameter committed
4413 4414
}

4415
#ifndef CONFIG_SLUB_TINY
4416 4417 4418 4419 4420 4421 4422 4423 4424 4425
/*
 * Fastpath with forced inlining to produce a kfree and kmem_cache_free that
 * can perform fastpath freeing without additional function calls.
 *
 * The fastpath is only possible if we are freeing to the current cpu slab
 * of this processor. This typically the case if we have just allocated
 * the item before.
 *
 * If fastpath is not possible then fall back to __slab_free where we deal
 * with all sorts of special processing.
4426 4427
 *
 * Bulk free of a freelist with several objects (all pointing to the
4428
 * same slab) possible by specifying head and tail ptr, plus objects
4429
 * count (cnt). Bulk free indicated by tail pointer being set.
4430
 */
4431
static __always_inline void do_slab_free(struct kmem_cache *s,
4432
				struct slab *slab, void *head, void *tail,
4433
				int cnt, unsigned long addr)
4434
{
4435
	struct kmem_cache_cpu *c;
4436
	unsigned long tid;
4437
	void **freelist;
4438

4439 4440 4441 4442 4443
redo:
	/*
	 * Determine the currently cpus per cpu slab.
	 * The cpu may change afterward. However that does not matter since
	 * data is retrieved via this pointer. If we are on the same cpu
4444
	 * during the cmpxchg then the free will succeed.
4445
	 */
4446 4447
	c = raw_cpu_ptr(s->cpu_slab);
	tid = READ_ONCE(c->tid);
4448

4449 4450
	/* Same with comment on barrier() in slab_alloc_node() */
	barrier();
4451

4452
	if (unlikely(slab != c->slab)) {
4453
		__slab_free(s, slab, head, tail, cnt, addr);
4454 4455 4456 4457 4458
		return;
	}

	if (USE_LOCKLESS_FAST_PATH()) {
		freelist = READ_ONCE(c->freelist);
4459

4460
		set_freepointer(s, tail, freelist);
4461

4462
		if (unlikely(!__update_cpu_freelist_fast(s, freelist, head, tid))) {
4463 4464 4465
			note_cmpxchg_failure("slab_free", s, tid);
			goto redo;
		}
4466 4467
	} else {
		/* Update the free list under the local lock */
4468 4469
		local_lock(&s->cpu_slab->lock);
		c = this_cpu_ptr(s->cpu_slab);
4470
		if (unlikely(slab != c->slab)) {
4471 4472 4473 4474 4475 4476
			local_unlock(&s->cpu_slab->lock);
			goto redo;
		}
		tid = c->tid;
		freelist = c->freelist;

4477
		set_freepointer(s, tail, freelist);
4478 4479 4480 4481
		c->freelist = head;
		c->tid = next_tid(tid);

		local_unlock(&s->cpu_slab->lock);
4482
	}
4483
	stat_add(s, FREE_FASTPATH, cnt);
4484
}
4485 4486 4487 4488 4489
#else /* CONFIG_SLUB_TINY */
static void do_slab_free(struct kmem_cache *s,
				struct slab *slab, void *head, void *tail,
				int cnt, unsigned long addr)
{
4490
	__slab_free(s, slab, head, tail, cnt, addr);
4491 4492
}
#endif /* CONFIG_SLUB_TINY */
4493

4494 4495 4496 4497 4498
static __fastpath_inline
void slab_free(struct kmem_cache *s, struct slab *slab, void *object,
	       unsigned long addr)
{
	memcg_slab_free_hook(s, slab, &object, 1);
4499
	alloc_tagging_slab_free_hook(s, slab, &object, 1);
4500

4501
	if (likely(slab_free_hook(s, object, slab_want_init_on_free(s))))
4502 4503 4504 4505 4506 4507
		do_slab_free(s, slab, object, object, 1, addr);
}

static __fastpath_inline
void slab_free_bulk(struct kmem_cache *s, struct slab *slab, void *head,
		    void *tail, void **p, int cnt, unsigned long addr)
4508
{
4509
	memcg_slab_free_hook(s, slab, p, cnt);
4510
	alloc_tagging_slab_free_hook(s, slab, p, cnt);
4511
	/*
4512 4513
	 * With KASAN enabled slab_free_freelist_hook modifies the freelist
	 * to remove objects, whose reuse must be delayed.
4514
	 */
4515
	if (likely(slab_free_freelist_hook(s, &head, &tail, &cnt)))
4516
		do_slab_free(s, slab, head, tail, cnt, addr);
4517 4518
}

4519
#ifdef CONFIG_KASAN_GENERIC
4520 4521
void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr)
{
4522
	do_slab_free(cache, virt_to_slab(x), x, x, 1, addr);
4523 4524 4525
}
#endif

4526
static inline struct kmem_cache *virt_to_cache(const void *obj)
4527
{
4528 4529 4530 4531 4532 4533
	struct slab *slab;

	slab = virt_to_slab(obj);
	if (WARN_ONCE(!slab, "%s: Object is not a Slab page!\n", __func__))
		return NULL;
	return slab->slab_cache;
4534 4535
}

4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547 4548 4549 4550 4551
static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x)
{
	struct kmem_cache *cachep;

	if (!IS_ENABLED(CONFIG_SLAB_FREELIST_HARDENED) &&
	    !kmem_cache_debug_flags(s, SLAB_CONSISTENCY_CHECKS))
		return s;

	cachep = virt_to_cache(x);
	if (WARN(cachep && cachep != s,
		 "%s: Wrong slab cache. %s but object is from %s\n",
		 __func__, s->name, cachep->name))
		print_tracking(cachep, x);
	return cachep;
}

4552 4553 4554 4555 4556 4557 4558 4559
/**
 * kmem_cache_free - Deallocate an object
 * @s: The cache the allocation was from.
 * @x: The previously allocated object.
 *
 * Free an object which was previously allocated from this
 * cache.
 */
Christoph Lameter's avatar
Christoph Lameter committed
4560 4561
void kmem_cache_free(struct kmem_cache *s, void *x)
{
4562 4563
	s = cache_from_obj(s, x);
	if (!s)
4564
		return;
4565
	trace_kmem_cache_free(_RET_IP_, x, s);
4566
	slab_free(s, virt_to_slab(x), x, _RET_IP_);
Christoph Lameter's avatar
Christoph Lameter committed
4567 4568 4569
}
EXPORT_SYMBOL(kmem_cache_free);

4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580
static void free_large_kmalloc(struct folio *folio, void *object)
{
	unsigned int order = folio_order(folio);

	if (WARN_ON_ONCE(order == 0))
		pr_warn_once("object pointer: 0x%p\n", object);

	kmemleak_free(object);
	kasan_kfree_large(object);
	kmsan_kfree_large(object);

4581
	lruvec_stat_mod_folio(folio, NR_SLAB_UNRECLAIMABLE_B,
4582
			      -(PAGE_SIZE << order));
4583
	folio_put(folio);
4584 4585 4586 4587 4588 4589 4590 4591 4592 4593 4594 4595 4596 4597 4598 4599 4600 4601 4602 4603 4604 4605 4606 4607 4608 4609 4610 4611
}

/**
 * kfree - free previously allocated memory
 * @object: pointer returned by kmalloc() or kmem_cache_alloc()
 *
 * If @object is NULL, no operation is performed.
 */
void kfree(const void *object)
{
	struct folio *folio;
	struct slab *slab;
	struct kmem_cache *s;
	void *x = (void *)object;

	trace_kfree(_RET_IP_, object);

	if (unlikely(ZERO_OR_NULL_PTR(object)))
		return;

	folio = virt_to_folio(object);
	if (unlikely(!folio_test_slab(folio))) {
		free_large_kmalloc(folio, (void *)object);
		return;
	}

	slab = folio_slab(folio);
	s = slab->slab_cache;
4612
	slab_free(s, slab, x, _RET_IP_);
4613 4614 4615
}
EXPORT_SYMBOL(kfree);

4616
struct detached_freelist {
4617
	struct slab *slab;
4618 4619 4620
	void *tail;
	void *freelist;
	int cnt;
4621
	struct kmem_cache *s;
4622
};
4623

4624 4625 4626
/*
 * This function progressively scans the array with free objects (with
 * a limited look ahead) and extract objects belonging to the same
4627 4628
 * slab.  It builds a detached freelist directly within the given
 * slab/objects.  This can happen without any need for
4629 4630 4631 4632 4633 4634 4635
 * synchronization, because the objects are owned by running process.
 * The freelist is build up as a single linked list in the objects.
 * The idea is, that this detached freelist can then be bulk
 * transferred to the real freelist(s), but only requiring a single
 * synchronization primitive.  Look ahead in the array is limited due
 * to performance reasons.
 */
4636 4637 4638
static inline
int build_detached_freelist(struct kmem_cache *s, size_t size,
			    void **p, struct detached_freelist *df)
4639 4640 4641
{
	int lookahead = 3;
	void *object;
4642
	struct folio *folio;
4643
	size_t same;
4644

4645
	object = p[--size];
4646
	folio = virt_to_folio(object);
4647 4648
	if (!s) {
		/* Handle kalloc'ed objects */
4649
		if (unlikely(!folio_test_slab(folio))) {
4650
			free_large_kmalloc(folio, object);
4651
			df->slab = NULL;
4652 4653 4654
			return size;
		}
		/* Derive kmem_cache from object */
4655 4656
		df->slab = folio_slab(folio);
		df->s = df->slab->slab_cache;
4657
	} else {
4658
		df->slab = folio_slab(folio);
4659 4660
		df->s = cache_from_obj(s, object); /* Support for memcg */
	}
4661

4662 4663 4664 4665 4666
	/* Start new detached freelist */
	df->tail = object;
	df->freelist = object;
	df->cnt = 1;

4667 4668 4669 4670 4671 4672
	if (is_kfence_address(object))
		return size;

	set_freepointer(df->s, object, NULL);

	same = size;
4673 4674
	while (size) {
		object = p[--size];
4675 4676
		/* df->slab is always set at this point */
		if (df->slab == virt_to_slab(object)) {
4677
			/* Opportunity build freelist */
4678
			set_freepointer(df->s, object, df->freelist);
4679 4680
			df->freelist = object;
			df->cnt++;
4681 4682 4683
			same--;
			if (size != same)
				swap(p[size], p[same]);
4684
			continue;
4685
		}
4686 4687 4688 4689

		/* Limit look ahead search */
		if (!--lookahead)
			break;
4690
	}
4691

4692
	return same;
4693 4694
}

4695 4696 4697 4698 4699 4700 4701 4702 4703 4704 4705 4706 4707 4708 4709 4710 4711 4712 4713 4714 4715
/*
 * Internal bulk free of objects that were not initialised by the post alloc
 * hooks and thus should not be processed by the free hooks
 */
static void __kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
{
	if (!size)
		return;

	do {
		struct detached_freelist df;

		size = build_detached_freelist(s, size, p, &df);
		if (!df.slab)
			continue;

		do_slab_free(df.s, df.slab, df.freelist, df.tail, df.cnt,
			     _RET_IP_);
	} while (likely(size));
}

4716
/* Note that interrupts must be enabled when calling this function. */
4717
void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
4718
{
4719
	if (!size)
4720 4721 4722 4723 4724 4725
		return;

	do {
		struct detached_freelist df;

		size = build_detached_freelist(s, size, p, &df);
4726
		if (!df.slab)
4727 4728
			continue;

4729 4730
		slab_free_bulk(df.s, df.slab, df.freelist, df.tail, &p[size],
			       df.cnt, _RET_IP_);
4731
	} while (likely(size));
4732 4733 4734
}
EXPORT_SYMBOL(kmem_cache_free_bulk);

4735
#ifndef CONFIG_SLUB_TINY
4736 4737 4738
static inline
int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
			    void **p)
4739
{
4740
	struct kmem_cache_cpu *c;
4741
	unsigned long irqflags;
4742 4743 4744 4745 4746 4747 4748
	int i;

	/*
	 * Drain objects in the per cpu slab, while disabling local
	 * IRQs, which protects against PREEMPT and interrupts
	 * handlers invoking normal fastpath.
	 */
4749
	c = slub_get_cpu_ptr(s->cpu_slab);
4750
	local_lock_irqsave(&s->cpu_slab->lock, irqflags);
4751 4752

	for (i = 0; i < size; i++) {
4753
		void *object = kfence_alloc(s, s->object_size, flags);
4754

4755 4756 4757 4758 4759 4760
		if (unlikely(object)) {
			p[i] = object;
			continue;
		}

		object = c->freelist;
4761
		if (unlikely(!object)) {
4762 4763 4764 4765 4766 4767 4768 4769 4770
			/*
			 * We may have removed an object from c->freelist using
			 * the fastpath in the previous iteration; in that case,
			 * c->tid has not been bumped yet.
			 * Since ___slab_alloc() may reenable interrupts while
			 * allocating memory, we should bump c->tid now.
			 */
			c->tid = next_tid(c->tid);

4771
			local_unlock_irqrestore(&s->cpu_slab->lock, irqflags);
4772

4773 4774 4775 4776
			/*
			 * Invoking slow path likely have side-effect
			 * of re-populating per CPU c->freelist
			 */
4777
			p[i] = ___slab_alloc(s, flags, NUMA_NO_NODE,
4778
					    _RET_IP_, c, s->object_size);
4779 4780 4781
			if (unlikely(!p[i]))
				goto error;

4782
			c = this_cpu_ptr(s->cpu_slab);
4783 4784
			maybe_wipe_obj_freeptr(s, p[i]);

4785
			local_lock_irqsave(&s->cpu_slab->lock, irqflags);
4786

4787 4788
			continue; /* goto for-loop */
		}
4789 4790
		c->freelist = get_freepointer(s, object);
		p[i] = object;
4791
		maybe_wipe_obj_freeptr(s, p[i]);
4792
		stat(s, ALLOC_FASTPATH);
4793 4794
	}
	c->tid = next_tid(c->tid);
4795
	local_unlock_irqrestore(&s->cpu_slab->lock, irqflags);
4796
	slub_put_cpu_ptr(s->cpu_slab);
4797

4798
	return i;
4799

4800
error:
4801
	slub_put_cpu_ptr(s->cpu_slab);
4802
	__kmem_cache_free_bulk(s, i, p);
4803
	return 0;
4804 4805

}
4806 4807
#else /* CONFIG_SLUB_TINY */
static int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags,
4808
				   size_t size, void **p)
4809 4810 4811 4812 4813 4814 4815 4816 4817 4818 4819 4820 4821 4822 4823 4824 4825 4826 4827 4828 4829 4830
{
	int i;

	for (i = 0; i < size; i++) {
		void *object = kfence_alloc(s, s->object_size, flags);

		if (unlikely(object)) {
			p[i] = object;
			continue;
		}

		p[i] = __slab_alloc_node(s, flags, NUMA_NO_NODE,
					 _RET_IP_, s->object_size);
		if (unlikely(!p[i]))
			goto error;

		maybe_wipe_obj_freeptr(s, p[i]);
	}

	return i;

error:
4831
	__kmem_cache_free_bulk(s, i, p);
4832 4833 4834
	return 0;
}
#endif /* CONFIG_SLUB_TINY */
4835 4836

/* Note that interrupts must be enabled when calling this function. */
4837 4838
int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size,
				 void **p)
4839 4840 4841 4842 4843 4844 4845 4846 4847 4848 4849 4850
{
	int i;
	struct obj_cgroup *objcg = NULL;

	if (!size)
		return 0;

	/* memcg and kmem_cache debug support */
	s = slab_pre_alloc_hook(s, NULL, &objcg, size, flags);
	if (unlikely(!s))
		return 0;

4851
	i = __kmem_cache_alloc_bulk(s, flags, size, p);
4852 4853 4854 4855 4856

	/*
	 * memcg and kmem_cache debug support and memory initialization.
	 * Done outside of the IRQ disabled fastpath loop.
	 */
4857
	if (likely(i != 0)) {
4858
		slab_post_alloc_hook(s, objcg, flags, size, p,
4859
			slab_want_init_on_alloc(flags, s), s->object_size);
4860 4861 4862 4863
	} else {
		memcg_slab_alloc_error_hook(s, size, objcg);
	}

4864
	return i;
4865
}
4866
EXPORT_SYMBOL(kmem_cache_alloc_bulk_noprof);
4867 4868


Christoph Lameter's avatar
Christoph Lameter committed
4869
/*
Christoph Lameter's avatar
Christoph Lameter committed
4870 4871 4872 4873
 * Object placement in a slab is made very easy because we always start at
 * offset 0. If we tune the size of the object to the alignment then we can
 * get the required alignment by putting one properly sized object after
 * another.
Christoph Lameter's avatar
Christoph Lameter committed
4874 4875 4876 4877
 *
 * Notice that the allocation order determines the sizes of the per cpu
 * caches. Each processor has always one slab available for allocations.
 * Increasing the allocation order reduces the number of times that slabs
Christoph Lameter's avatar
Christoph Lameter committed
4878
 * must be moved on and off the partial lists and is therefore a factor in
Christoph Lameter's avatar
Christoph Lameter committed
4879 4880 4881 4882
 * locking overhead.
 */

/*
Ingo Molnar's avatar
Ingo Molnar committed
4883
 * Minimum / Maximum order of slab pages. This influences locking overhead
Christoph Lameter's avatar
Christoph Lameter committed
4884 4885 4886 4887
 * and slab fragmentation. A higher order reduces the number of partial slabs
 * and increases the number of allocations possible without having to
 * take the list_lock.
 */
4888
static unsigned int slub_min_order;
4889 4890
static unsigned int slub_max_order =
	IS_ENABLED(CONFIG_SLUB_TINY) ? 1 : PAGE_ALLOC_COSTLY_ORDER;
4891
static unsigned int slub_min_objects;
Christoph Lameter's avatar
Christoph Lameter committed
4892 4893 4894 4895

/*
 * Calculate the order of allocation given an slab object size.
 *
Christoph Lameter's avatar
Christoph Lameter committed
4896 4897 4898 4899
 * The order of allocation has significant impact on performance and other
 * system components. Generally order 0 allocations should be preferred since
 * order 0 does not cause fragmentation in the page allocator. Larger objects
 * be problematic to put into order 0 slabs because there may be too much
4900
 * unused space left. We go to a higher order if more than 1/16th of the slab
Christoph Lameter's avatar
Christoph Lameter committed
4901 4902 4903 4904 4905 4906
 * would be wasted.
 *
 * In order to reach satisfactory performance we must ensure that a minimum
 * number of objects is in one slab. Otherwise we may generate too much
 * activity on the partial lists which requires taking the list_lock. This is
 * less a concern for large slabs though which are rarely used.
Christoph Lameter's avatar
Christoph Lameter committed
4907
 *
4908 4909
 * slab_max_order specifies the order where we begin to stop considering the
 * number of objects in a slab as critical. If we reach slab_max_order then
Christoph Lameter's avatar
Christoph Lameter committed
4910 4911
 * we try to keep the page order as low as possible. So we accept more waste
 * of space in favor of a small page order.
Christoph Lameter's avatar
Christoph Lameter committed
4912
 *
Christoph Lameter's avatar
Christoph Lameter committed
4913 4914
 * Higher order allocations also allow the placement of more objects in a
 * slab and thereby reduce object handling overhead. If the user has
4915
 * requested a higher minimum order then we start with that one instead of
Christoph Lameter's avatar
Christoph Lameter committed
4916
 * the smallest order which will fit the object.
Christoph Lameter's avatar
Christoph Lameter committed
4917
 */
4918
static inline unsigned int calc_slab_order(unsigned int size,
4919
		unsigned int min_order, unsigned int max_order,
4920
		unsigned int fract_leftover)
Christoph Lameter's avatar
Christoph Lameter committed
4921
{
4922
	unsigned int order;
Christoph Lameter's avatar
Christoph Lameter committed
4923

4924
	for (order = min_order; order <= max_order; order++) {
Christoph Lameter's avatar
Christoph Lameter committed
4925

4926 4927
		unsigned int slab_size = (unsigned int)PAGE_SIZE << order;
		unsigned int rem;
Christoph Lameter's avatar
Christoph Lameter committed
4928

4929
		rem = slab_size % size;
Christoph Lameter's avatar
Christoph Lameter committed
4930

4931
		if (rem <= slab_size / fract_leftover)
Christoph Lameter's avatar
Christoph Lameter committed
4932 4933
			break;
	}
Christoph Lameter's avatar
Christoph Lameter committed
4934

Christoph Lameter's avatar
Christoph Lameter committed
4935 4936 4937
	return order;
}

4938
static inline int calculate_order(unsigned int size)
4939
{
4940 4941 4942
	unsigned int order;
	unsigned int min_objects;
	unsigned int max_objects;
4943
	unsigned int min_order;
4944 4945

	min_objects = slub_min_objects;
4946 4947 4948 4949 4950 4951 4952 4953 4954 4955
	if (!min_objects) {
		/*
		 * Some architectures will only update present cpus when
		 * onlining them, so don't trust the number if it's just 1. But
		 * we also don't want to use nr_cpu_ids always, as on some other
		 * architectures, there can be many possible cpus, but never
		 * onlined. Here we compromise between trying to avoid too high
		 * order on systems that appear larger than they are, and too
		 * low order on systems that appear smaller than they are.
		 */
4956
		unsigned int nr_cpus = num_present_cpus();
4957 4958 4959 4960
		if (nr_cpus <= 1)
			nr_cpus = nr_cpu_ids;
		min_objects = 4 * (fls(nr_cpus) + 1);
	}
4961 4962
	/* min_objects can't be 0 because get_order(0) is undefined */
	max_objects = max(order_objects(slub_max_order, size), 1U);
4963 4964
	min_objects = min(min_objects, max_objects);

4965 4966 4967 4968 4969
	min_order = max_t(unsigned int, slub_min_order,
			  get_order(min_objects * size));
	if (order_objects(min_order, size) > MAX_OBJS_PER_PAGE)
		return get_order(size * MAX_OBJS_PER_PAGE) - 1;

4970 4971 4972 4973 4974 4975
	/*
	 * Attempt to find best configuration for a slab. This works by first
	 * attempting to generate a layout with the best possible configuration
	 * and backing off gradually.
	 *
	 * We start with accepting at most 1/16 waste and try to find the
4976 4977
	 * smallest order from min_objects-derived/slab_min_order up to
	 * slab_max_order that will satisfy the constraint. Note that increasing
4978 4979 4980
	 * the order can only result in same or less fractional waste, not more.
	 *
	 * If that fails, we increase the acceptable fraction of waste and try
4981 4982
	 * again. The last iteration with fraction of 1/2 would effectively
	 * accept any waste and give us the order determined by min_objects, as
4983
	 * long as at least single object fits within slab_max_order.
4984
	 */
4985
	for (unsigned int fraction = 16; fraction > 1; fraction /= 2) {
4986
		order = calc_slab_order(size, min_order, slub_max_order,
4987 4988 4989
					fraction);
		if (order <= slub_max_order)
			return order;
4990 4991 4992
	}

	/*
4993
	 * Doh this slab cannot be placed using slab_max_order.
4994
	 */
4995
	order = get_order(size);
4996
	if (order <= MAX_PAGE_ORDER)
4997 4998 4999 5000
		return order;
	return -ENOSYS;
}

5001
static void
5002
init_kmem_cache_node(struct kmem_cache_node *n)
Christoph Lameter's avatar
Christoph Lameter committed
5003 5004 5005 5006
{
	n->nr_partial = 0;
	spin_lock_init(&n->list_lock);
	INIT_LIST_HEAD(&n->partial);
5007
#ifdef CONFIG_SLUB_DEBUG
5008
	atomic_long_set(&n->nr_slabs, 0);
5009
	atomic_long_set(&n->total_objects, 0);
5010
	INIT_LIST_HEAD(&n->full);
5011
#endif
Christoph Lameter's avatar
Christoph Lameter committed
5012 5013
}

5014
#ifndef CONFIG_SLUB_TINY
5015
static inline int alloc_kmem_cache_cpus(struct kmem_cache *s)
5016
{
5017
	BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE <
5018 5019
			NR_KMALLOC_TYPES * KMALLOC_SHIFT_HIGH *
			sizeof(struct kmem_cache_cpu));
5020

5021
	/*
5022 5023
	 * Must align to double word boundary for the double cmpxchg
	 * instructions to work; see __pcpu_double_call_return_bool().
5024
	 */
5025 5026
	s->cpu_slab = __alloc_percpu(sizeof(struct kmem_cache_cpu),
				     2 * sizeof(void *));
5027 5028 5029 5030 5031

	if (!s->cpu_slab)
		return 0;

	init_kmem_cache_cpus(s);
5032

5033
	return 1;
5034
}
5035 5036 5037 5038 5039 5040
#else
static inline int alloc_kmem_cache_cpus(struct kmem_cache *s)
{
	return 1;
}
#endif /* CONFIG_SLUB_TINY */
5041

5042 5043
static struct kmem_cache *kmem_cache_node;

Christoph Lameter's avatar
Christoph Lameter committed
5044 5045 5046 5047 5048
/*
 * No kmalloc_node yet so do it by hand. We know that this is the first
 * slab on the node for this slabcache. There are no concurrent accesses
 * possible.
 *
5049 5050
 * Note that this function only works on the kmem_cache_node
 * when allocating for the kmem_cache_node. This is used for bootstrapping
5051
 * memory on a fresh node that has no slab structures yet.
Christoph Lameter's avatar
Christoph Lameter committed
5052
 */
5053
static void early_kmem_cache_node_alloc(int node)
Christoph Lameter's avatar
Christoph Lameter committed
5054
{
5055
	struct slab *slab;
Christoph Lameter's avatar
Christoph Lameter committed
5056 5057
	struct kmem_cache_node *n;

5058
	BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node));
Christoph Lameter's avatar
Christoph Lameter committed
5059

5060
	slab = new_slab(kmem_cache_node, GFP_NOWAIT, node);
Christoph Lameter's avatar
Christoph Lameter committed
5061

5062 5063
	BUG_ON(!slab);
	if (slab_nid(slab) != node) {
5064 5065
		pr_err("SLUB: Unable to allocate memory from node %d\n", node);
		pr_err("SLUB: Allocating a useless per node structure in order to be able to continue\n");
5066 5067
	}

5068
	n = slab->freelist;
Christoph Lameter's avatar
Christoph Lameter committed
5069
	BUG_ON(!n);
5070
#ifdef CONFIG_SLUB_DEBUG
5071
	init_object(kmem_cache_node, n, SLUB_RED_ACTIVE);
5072
	init_tracking(kmem_cache_node, n);
5073
#endif
5074
	n = kasan_slab_alloc(kmem_cache_node, n, GFP_KERNEL, false);
5075 5076
	slab->freelist = get_freepointer(kmem_cache_node, n);
	slab->inuse = 1;
5077
	kmem_cache_node->node[node] = n;
5078
	init_kmem_cache_node(n);
5079
	inc_slabs_node(kmem_cache_node, node, slab->objects);
Christoph Lameter's avatar
Christoph Lameter committed
5080

5081
	/*
5082 5083
	 * No locks need to be taken here as it has just been
	 * initialized and there is no concurrent access.
5084
	 */
5085
	__add_partial(n, slab, DEACTIVATE_TO_HEAD);
Christoph Lameter's avatar
Christoph Lameter committed
5086 5087 5088 5089 5090
}

static void free_kmem_cache_nodes(struct kmem_cache *s)
{
	int node;
5091
	struct kmem_cache_node *n;
Christoph Lameter's avatar
Christoph Lameter committed
5092

5093
	for_each_kmem_cache_node(s, node, n) {
Christoph Lameter's avatar
Christoph Lameter committed
5094
		s->node[node] = NULL;
5095
		kmem_cache_free(kmem_cache_node, n);
Christoph Lameter's avatar
Christoph Lameter committed
5096 5097 5098
	}
}

5099 5100
void __kmem_cache_release(struct kmem_cache *s)
{
5101
	cache_random_seq_destroy(s);
5102
#ifndef CONFIG_SLUB_TINY
5103
	free_percpu(s->cpu_slab);
5104
#endif
5105 5106 5107
	free_kmem_cache_nodes(s);
}

5108
static int init_kmem_cache_nodes(struct kmem_cache *s)
Christoph Lameter's avatar
Christoph Lameter committed
5109 5110 5111
{
	int node;

5112
	for_each_node_mask(node, slab_nodes) {
Christoph Lameter's avatar
Christoph Lameter committed
5113 5114
		struct kmem_cache_node *n;

5115
		if (slab_state == DOWN) {
5116
			early_kmem_cache_node_alloc(node);
5117 5118
			continue;
		}
5119
		n = kmem_cache_alloc_node(kmem_cache_node,
5120
						GFP_KERNEL, node);
Christoph Lameter's avatar
Christoph Lameter committed
5121

5122 5123 5124
		if (!n) {
			free_kmem_cache_nodes(s);
			return 0;
Christoph Lameter's avatar
Christoph Lameter committed
5125
		}
5126

5127
		init_kmem_cache_node(n);
5128
		s->node[node] = n;
Christoph Lameter's avatar
Christoph Lameter committed
5129 5130 5131 5132
	}
	return 1;
}

5133 5134 5135
static void set_cpu_partial(struct kmem_cache *s)
{
#ifdef CONFIG_SLUB_CPU_PARTIAL
5136 5137
	unsigned int nr_objects;

5138 5139 5140 5141 5142 5143 5144 5145 5146
	/*
	 * cpu_partial determined the maximum number of objects kept in the
	 * per cpu partial lists of a processor.
	 *
	 * Per cpu partial lists mainly contain slabs that just have one
	 * object freed. If they are used for allocation then they can be
	 * filled up again with minimal effort. The slab will never hit the
	 * per node partial lists and therefore no locking will be required.
	 *
5147 5148 5149
	 * For backwards compatibility reasons, this is determined as number
	 * of objects, even though we now limit maximum number of pages, see
	 * slub_set_cpu_partial()
5150 5151
	 */
	if (!kmem_cache_has_cpu_partial(s))
5152
		nr_objects = 0;
5153
	else if (s->size >= PAGE_SIZE)
5154
		nr_objects = 6;
5155
	else if (s->size >= 1024)
5156
		nr_objects = 24;
5157
	else if (s->size >= 256)
5158
		nr_objects = 52;
5159
	else
5160
		nr_objects = 120;
5161 5162

	slub_set_cpu_partial(s, nr_objects);
5163 5164 5165
#endif
}

Christoph Lameter's avatar
Christoph Lameter committed
5166 5167 5168 5169
/*
 * calculate_sizes() determines the order and the distribution of data within
 * a slab object.
 */
5170
static int calculate_sizes(struct kmem_cache *s)
Christoph Lameter's avatar
Christoph Lameter committed
5171
{
5172
	slab_flags_t flags = s->flags;
5173
	unsigned int size = s->object_size;
5174
	unsigned int order;
Christoph Lameter's avatar
Christoph Lameter committed
5175

5176 5177 5178 5179 5180 5181 5182 5183
	/*
	 * Round up object size to the next word boundary. We can only
	 * place the free pointer at word boundaries and this determines
	 * the possible location of the free pointer.
	 */
	size = ALIGN(size, sizeof(void *));

#ifdef CONFIG_SLUB_DEBUG
Christoph Lameter's avatar
Christoph Lameter committed
5184 5185 5186 5187 5188
	/*
	 * Determine if we can poison the object itself. If the user of
	 * the slab may touch the object after free or before allocation
	 * then we should never poison the object itself.
	 */
5189
	if ((flags & SLAB_POISON) && !(flags & SLAB_TYPESAFE_BY_RCU) &&
5190
			!s->ctor)
Christoph Lameter's avatar
Christoph Lameter committed
5191 5192 5193 5194 5195 5196
		s->flags |= __OBJECT_POISON;
	else
		s->flags &= ~__OBJECT_POISON;


	/*
Christoph Lameter's avatar
Christoph Lameter committed
5197
	 * If we are Redzoning then check if there is some space between the
Christoph Lameter's avatar
Christoph Lameter committed
5198
	 * end of the object and the free pointer. If not then add an
Christoph Lameter's avatar
Christoph Lameter committed
5199
	 * additional word to have some bytes to store Redzone information.
Christoph Lameter's avatar
Christoph Lameter committed
5200
	 */
5201
	if ((flags & SLAB_RED_ZONE) && size == s->object_size)
Christoph Lameter's avatar
Christoph Lameter committed
5202
		size += sizeof(void *);
5203
#endif
Christoph Lameter's avatar
Christoph Lameter committed
5204 5205

	/*
Christoph Lameter's avatar
Christoph Lameter committed
5206
	 * With that we have determined the number of bytes in actual use
5207
	 * by the object and redzoning.
Christoph Lameter's avatar
Christoph Lameter committed
5208 5209 5210
	 */
	s->inuse = size;

5211 5212
	if (slub_debug_orig_size(s) ||
	    (flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON)) ||
5213 5214
	    ((flags & SLAB_RED_ZONE) && s->object_size < sizeof(void *)) ||
	    s->ctor) {
Christoph Lameter's avatar
Christoph Lameter committed
5215 5216 5217 5218 5219 5220
		/*
		 * Relocate free pointer after the object if it is not
		 * permitted to overwrite the first word of the object on
		 * kmem_cache_free.
		 *
		 * This is the case if we do RCU, have a constructor or
5221 5222
		 * destructor, are poisoning the objects, or are
		 * redzoning an object smaller than sizeof(void *).
5223 5224 5225 5226 5227
		 *
		 * The assumption that s->offset >= s->inuse means free
		 * pointer is outside of the object is used in the
		 * freeptr_outside_object() function. If that is no
		 * longer true, the function needs to be modified.
Christoph Lameter's avatar
Christoph Lameter committed
5228 5229 5230
		 */
		s->offset = size;
		size += sizeof(void *);
5231
	} else {
5232 5233 5234 5235 5236
		/*
		 * Store freelist pointer near middle of object to keep
		 * it away from the edges of the object to avoid small
		 * sized over/underflows from neighboring allocations.
		 */
5237
		s->offset = ALIGN_DOWN(s->object_size / 2, sizeof(void *));
Christoph Lameter's avatar
Christoph Lameter committed
5238 5239
	}

5240
#ifdef CONFIG_SLUB_DEBUG
5241
	if (flags & SLAB_STORE_USER) {
Christoph Lameter's avatar
Christoph Lameter committed
5242 5243 5244 5245 5246
		/*
		 * Need to store information about allocs and frees after
		 * the object.
		 */
		size += 2 * sizeof(struct track);
5247 5248 5249 5250 5251

		/* Save the original kmalloc request size */
		if (flags & SLAB_KMALLOC)
			size += sizeof(unsigned int);
	}
5252
#endif
Christoph Lameter's avatar
Christoph Lameter committed
5253

5254 5255
	kasan_cache_create(s, &size, &s->flags);
#ifdef CONFIG_SLUB_DEBUG
5256
	if (flags & SLAB_RED_ZONE) {
Christoph Lameter's avatar
Christoph Lameter committed
5257 5258 5259 5260
		/*
		 * Add some empty padding so that we can catch
		 * overwrites from earlier objects rather than let
		 * tracking information or the free pointer be
5261
		 * corrupted if a user writes before the start
Christoph Lameter's avatar
Christoph Lameter committed
5262 5263 5264
		 * of the object.
		 */
		size += sizeof(void *);
5265 5266 5267 5268 5269

		s->red_left_pad = sizeof(void *);
		s->red_left_pad = ALIGN(s->red_left_pad, s->align);
		size += s->red_left_pad;
	}
5270
#endif
Christoph Lameter's avatar
Christoph Lameter committed
5271

Christoph Lameter's avatar
Christoph Lameter committed
5272 5273 5274 5275 5276
	/*
	 * SLUB stores one object immediately after another beginning from
	 * offset 0. In order to align the objects we have to simply size
	 * each object to conform to the alignment.
	 */
5277
	size = ALIGN(size, s->align);
Christoph Lameter's avatar
Christoph Lameter committed
5278
	s->size = size;
5279
	s->reciprocal_size = reciprocal_value(size);
5280
	order = calculate_order(size);
Christoph Lameter's avatar
Christoph Lameter committed
5281

5282
	if ((int)order < 0)
Christoph Lameter's avatar
Christoph Lameter committed
5283 5284
		return 0;

5285
	s->allocflags = 0;
5286
	if (order)
5287 5288 5289
		s->allocflags |= __GFP_COMP;

	if (s->flags & SLAB_CACHE_DMA)
5290
		s->allocflags |= GFP_DMA;
5291

5292 5293 5294
	if (s->flags & SLAB_CACHE_DMA32)
		s->allocflags |= GFP_DMA32;

5295 5296 5297
	if (s->flags & SLAB_RECLAIM_ACCOUNT)
		s->allocflags |= __GFP_RECLAIMABLE;

Christoph Lameter's avatar
Christoph Lameter committed
5298 5299 5300
	/*
	 * Determine the number of objects per slab
	 */
5301 5302
	s->oo = oo_make(order, size);
	s->min = oo_make(get_order(size), size);
Christoph Lameter's avatar
Christoph Lameter committed
5303

5304
	return !!oo_objects(s->oo);
Christoph Lameter's avatar
Christoph Lameter committed
5305 5306
}

5307
static int kmem_cache_open(struct kmem_cache *s, slab_flags_t flags)
Christoph Lameter's avatar
Christoph Lameter committed
5308
{
5309
	s->flags = kmem_cache_flags(flags, s->name);
5310 5311 5312
#ifdef CONFIG_SLAB_FREELIST_HARDENED
	s->random = get_random_long();
#endif
Christoph Lameter's avatar
Christoph Lameter committed
5313

5314
	if (!calculate_sizes(s))
Christoph Lameter's avatar
Christoph Lameter committed
5315
		goto error;
5316 5317 5318 5319 5320
	if (disable_higher_order_debug) {
		/*
		 * Disable debugging flags that store metadata if the min slab
		 * order increased.
		 */
5321
		if (get_order(s->size) > get_order(s->object_size)) {
5322 5323
			s->flags &= ~DEBUG_METADATA_FLAGS;
			s->offset = 0;
5324
			if (!calculate_sizes(s))
5325 5326 5327
				goto error;
		}
	}
Christoph Lameter's avatar
Christoph Lameter committed
5328

5329 5330
#ifdef system_has_freelist_aba
	if (system_has_freelist_aba() && !(s->flags & SLAB_NO_CMPXCHG)) {
5331 5332
		/* Enable fast mode */
		s->flags |= __CMPXCHG_DOUBLE;
5333
	}
5334 5335
#endif

5336
	/*
5337
	 * The larger the object size is, the more slabs we want on the partial
5338 5339
	 * list to avoid pounding the page allocator excessively.
	 */
5340 5341
	s->min_partial = min_t(unsigned long, MAX_PARTIAL, ilog2(s->size) / 2);
	s->min_partial = max_t(unsigned long, MIN_PARTIAL, s->min_partial);
5342

5343
	set_cpu_partial(s);
5344

Christoph Lameter's avatar
Christoph Lameter committed
5345
#ifdef CONFIG_NUMA
5346
	s->remote_node_defrag_ratio = 1000;
Christoph Lameter's avatar
Christoph Lameter committed
5347
#endif
5348 5349 5350 5351 5352 5353 5354

	/* Initialize the pre-computed randomized freelist if slab is up */
	if (slab_state >= UP) {
		if (init_cache_random_seq(s))
			goto error;
	}

5355
	if (!init_kmem_cache_nodes(s))
5356
		goto error;
Christoph Lameter's avatar
Christoph Lameter committed
5357

5358
	if (alloc_kmem_cache_cpus(s))
5359
		return 0;
5360

Christoph Lameter's avatar
Christoph Lameter committed
5361
error:
5362
	__kmem_cache_release(s);
5363
	return -EINVAL;
Christoph Lameter's avatar
Christoph Lameter committed
5364 5365
}

5366
static void list_slab_objects(struct kmem_cache *s, struct slab *slab,
5367
			      const char *text)
5368 5369
{
#ifdef CONFIG_SLUB_DEBUG
5370
	void *addr = slab_address(slab);
5371
	void *p;
5372

5373
	slab_err(s, slab, text, s->name);
5374

5375 5376 5377
	spin_lock(&object_map_lock);
	__fill_map(object_map, s, slab);

5378
	for_each_object(p, s, addr, slab->objects) {
5379

5380
		if (!test_bit(__obj_to_index(s, addr, p), object_map)) {
5381
			pr_err("Object 0x%p @offset=%tu\n", p, p - addr);
5382 5383 5384
			print_tracking(s, p);
		}
	}
5385
	spin_unlock(&object_map_lock);
5386 5387 5388
#endif
}

Christoph Lameter's avatar
Christoph Lameter committed
5389
/*
5390
 * Attempt to free all partial slabs on a node.
5391 5392
 * This is called from __kmem_cache_shutdown(). We must take list_lock
 * because sysfs file might still access partial list after the shutdowning.
Christoph Lameter's avatar
Christoph Lameter committed
5393
 */
5394
static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n)
Christoph Lameter's avatar
Christoph Lameter committed
5395
{
5396
	LIST_HEAD(discard);
5397
	struct slab *slab, *h;
Christoph Lameter's avatar
Christoph Lameter committed
5398

5399 5400
	BUG_ON(irqs_disabled());
	spin_lock_irq(&n->list_lock);
5401 5402 5403 5404
	list_for_each_entry_safe(slab, h, &n->partial, slab_list) {
		if (!slab->inuse) {
			remove_partial(n, slab);
			list_add(&slab->slab_list, &discard);
5405
		} else {
5406
			list_slab_objects(s, slab,
5407
			  "Objects remaining in %s on __kmem_cache_shutdown()");
5408
		}
5409
	}
5410
	spin_unlock_irq(&n->list_lock);
5411

5412 5413
	list_for_each_entry_safe(slab, h, &discard, slab_list)
		discard_slab(s, slab);
Christoph Lameter's avatar
Christoph Lameter committed
5414 5415
}

5416 5417 5418 5419 5420 5421
bool __kmem_cache_empty(struct kmem_cache *s)
{
	int node;
	struct kmem_cache_node *n;

	for_each_kmem_cache_node(s, node, n)
5422
		if (n->nr_partial || node_nr_slabs(n))
5423 5424 5425 5426
			return false;
	return true;
}

Christoph Lameter's avatar
Christoph Lameter committed
5427
/*
Christoph Lameter's avatar
Christoph Lameter committed
5428
 * Release all resources used by a slab cache.
Christoph Lameter's avatar
Christoph Lameter committed
5429
 */
5430
int __kmem_cache_shutdown(struct kmem_cache *s)
Christoph Lameter's avatar
Christoph Lameter committed
5431 5432
{
	int node;
5433
	struct kmem_cache_node *n;
Christoph Lameter's avatar
Christoph Lameter committed
5434

5435
	flush_all_cpus_locked(s);
Christoph Lameter's avatar
Christoph Lameter committed
5436
	/* Attempt to free all objects */
5437
	for_each_kmem_cache_node(s, node, n) {
5438
		free_partial(s, n);
5439
		if (n->nr_partial || node_nr_slabs(n))
Christoph Lameter's avatar
Christoph Lameter committed
5440 5441 5442 5443 5444
			return 1;
	}
	return 0;
}

5445
#ifdef CONFIG_PRINTK
5446
void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab)
5447 5448 5449 5450 5451 5452
{
	void *base;
	int __maybe_unused i;
	unsigned int objnr;
	void *objp;
	void *objp0;
5453
	struct kmem_cache *s = slab->slab_cache;
5454 5455 5456
	struct track __maybe_unused *trackp;

	kpp->kp_ptr = object;
5457
	kpp->kp_slab = slab;
5458
	kpp->kp_slab_cache = s;
5459
	base = slab_address(slab);
5460 5461 5462 5463 5464 5465
	objp0 = kasan_reset_tag(object);
#ifdef CONFIG_SLUB_DEBUG
	objp = restore_red_left(s, objp0);
#else
	objp = objp0;
#endif
5466
	objnr = obj_to_index(s, slab, objp);
5467 5468 5469
	kpp->kp_data_offset = (unsigned long)((char *)objp0 - (char *)objp);
	objp = base + s->size * objnr;
	kpp->kp_objp = objp;
5470 5471
	if (WARN_ON_ONCE(objp < base || objp >= base + slab->objects * s->size
			 || (objp - base) % s->size) ||
5472 5473 5474
	    !(s->flags & SLAB_STORE_USER))
		return;
#ifdef CONFIG_SLUB_DEBUG
5475
	objp = fixup_red_left(s, objp);
5476 5477
	trackp = get_track(s, objp, TRACK_ALLOC);
	kpp->kp_ret = (void *)trackp->addr;
5478 5479 5480 5481 5482
#ifdef CONFIG_STACKDEPOT
	{
		depot_stack_handle_t handle;
		unsigned long *entries;
		unsigned int nr_entries;
5483

5484 5485 5486 5487 5488 5489
		handle = READ_ONCE(trackp->handle);
		if (handle) {
			nr_entries = stack_depot_fetch(handle, &entries);
			for (i = 0; i < KS_ADDRS_COUNT && i < nr_entries; i++)
				kpp->kp_stack[i] = (void *)entries[i];
		}
5490

5491 5492 5493 5494 5495 5496 5497
		trackp = get_track(s, objp, TRACK_FREE);
		handle = READ_ONCE(trackp->handle);
		if (handle) {
			nr_entries = stack_depot_fetch(handle, &entries);
			for (i = 0; i < KS_ADDRS_COUNT && i < nr_entries; i++)
				kpp->kp_free_stack[i] = (void *)entries[i];
		}
5498
	}
5499 5500 5501
#endif
#endif
}
5502
#endif
5503

Christoph Lameter's avatar
Christoph Lameter committed
5504 5505 5506 5507 5508 5509
/********************************************************************
 *		Kmalloc subsystem
 *******************************************************************/

static int __init setup_slub_min_order(char *str)
{
5510
	get_option(&str, (int *)&slub_min_order);
Christoph Lameter's avatar
Christoph Lameter committed
5511

5512 5513 5514
	if (slub_min_order > slub_max_order)
		slub_max_order = slub_min_order;

Christoph Lameter's avatar
Christoph Lameter committed
5515 5516 5517
	return 1;
}

5518 5519 5520
__setup("slab_min_order=", setup_slub_min_order);
__setup_param("slub_min_order=", slub_min_order, setup_slub_min_order, 0);

Christoph Lameter's avatar
Christoph Lameter committed
5521 5522 5523

static int __init setup_slub_max_order(char *str)
{
5524
	get_option(&str, (int *)&slub_max_order);
5525
	slub_max_order = min_t(unsigned int, slub_max_order, MAX_PAGE_ORDER);
Christoph Lameter's avatar
Christoph Lameter committed
5526

5527 5528 5529
	if (slub_min_order > slub_max_order)
		slub_min_order = slub_max_order;

Christoph Lameter's avatar
Christoph Lameter committed
5530 5531 5532
	return 1;
}

5533 5534
__setup("slab_max_order=", setup_slub_max_order);
__setup_param("slub_max_order=", slub_max_order, setup_slub_max_order, 0);
Christoph Lameter's avatar
Christoph Lameter committed
5535 5536 5537

static int __init setup_slub_min_objects(char *str)
{
5538
	get_option(&str, (int *)&slub_min_objects);
Christoph Lameter's avatar
Christoph Lameter committed
5539 5540 5541 5542

	return 1;
}

5543 5544
__setup("slab_min_objects=", setup_slub_min_objects);
__setup_param("slub_min_objects=", slub_min_objects, setup_slub_min_objects, 0);
Christoph Lameter's avatar
Christoph Lameter committed
5545

5546 5547
#ifdef CONFIG_HARDENED_USERCOPY
/*
5548 5549 5550
 * Rejects incorrectly sized objects and objects that are to be copied
 * to/from userspace but do not fall entirely within the containing slab
 * cache's usercopy region.
5551 5552 5553 5554
 *
 * Returns NULL if check passes, otherwise const char * to name of cache
 * to indicate an error.
 */
5555 5556
void __check_heap_object(const void *ptr, unsigned long n,
			 const struct slab *slab, bool to_user)
5557 5558
{
	struct kmem_cache *s;
5559
	unsigned int offset;
5560
	bool is_kfence = is_kfence_address(ptr);
5561

5562 5563
	ptr = kasan_reset_tag(ptr);

5564
	/* Find object and usable object size. */
5565
	s = slab->slab_cache;
5566 5567

	/* Reject impossible pointers. */
5568
	if (ptr < slab_address(slab))
5569 5570
		usercopy_abort("SLUB object not in SLUB page?!", NULL,
			       to_user, 0, n);
5571 5572

	/* Find offset within object. */
5573 5574 5575
	if (is_kfence)
		offset = ptr - kfence_object_start(ptr);
	else
5576
		offset = (ptr - slab_address(slab)) % s->size;
5577 5578

	/* Adjust for redzone and reject if within the redzone. */
5579
	if (!is_kfence && kmem_cache_debug_flags(s, SLAB_RED_ZONE)) {
5580
		if (offset < s->red_left_pad)
5581 5582
			usercopy_abort("SLUB object in left red zone",
				       s->name, to_user, offset, n);
5583 5584 5585
		offset -= s->red_left_pad;
	}

5586 5587 5588 5589
	/* Allow address range falling entirely within usercopy region. */
	if (offset >= s->useroffset &&
	    offset - s->useroffset <= s->usersize &&
	    n <= s->useroffset - offset + s->usersize)
5590
		return;
5591

5592
	usercopy_abort("SLUB object", s->name, to_user, offset, n);
5593 5594 5595
}
#endif /* CONFIG_HARDENED_USERCOPY */

5596 5597
#define SHRINK_PROMOTE_MAX 32

5598
/*
5599 5600 5601
 * kmem_cache_shrink discards empty slabs and promotes the slabs filled
 * up most to the head of the partial lists. New allocations will then
 * fill those up and thus they can be removed from the partial lists.
Christoph Lameter's avatar
Christoph Lameter committed
5602 5603 5604 5605
 *
 * The slabs with the least items are placed last. This results in them
 * being allocated from last increasing the chance that the last objects
 * are freed in them.
5606
 */
5607
static int __kmem_cache_do_shrink(struct kmem_cache *s)
5608 5609 5610 5611
{
	int node;
	int i;
	struct kmem_cache_node *n;
5612 5613
	struct slab *slab;
	struct slab *t;
5614 5615
	struct list_head discard;
	struct list_head promote[SHRINK_PROMOTE_MAX];
5616
	unsigned long flags;
5617
	int ret = 0;
5618

5619
	for_each_kmem_cache_node(s, node, n) {
5620 5621 5622
		INIT_LIST_HEAD(&discard);
		for (i = 0; i < SHRINK_PROMOTE_MAX; i++)
			INIT_LIST_HEAD(promote + i);
5623 5624 5625 5626

		spin_lock_irqsave(&n->list_lock, flags);

		/*
5627
		 * Build lists of slabs to discard or promote.
5628
		 *
Christoph Lameter's avatar
Christoph Lameter committed
5629
		 * Note that concurrent frees may occur while we hold the
5630
		 * list_lock. slab->inuse here is the upper limit.
5631
		 */
5632 5633
		list_for_each_entry_safe(slab, t, &n->partial, slab_list) {
			int free = slab->objects - slab->inuse;
5634

5635
			/* Do not reread slab->inuse */
5636 5637 5638 5639 5640
			barrier();

			/* We do not keep full slabs on the list */
			BUG_ON(free <= 0);

5641 5642
			if (free == slab->objects) {
				list_move(&slab->slab_list, &discard);
5643
				slab_clear_node_partial(slab);
5644
				n->nr_partial--;
5645
				dec_slabs_node(s, node, slab->objects);
5646
			} else if (free <= SHRINK_PROMOTE_MAX)
5647
				list_move(&slab->slab_list, promote + free - 1);
5648 5649 5650
		}

		/*
5651 5652
		 * Promote the slabs filled up most to the head of the
		 * partial list.
5653
		 */
5654 5655
		for (i = SHRINK_PROMOTE_MAX - 1; i >= 0; i--)
			list_splice(promote + i, &n->partial);
5656 5657

		spin_unlock_irqrestore(&n->list_lock, flags);
5658 5659

		/* Release empty slabs */
5660
		list_for_each_entry_safe(slab, t, &discard, slab_list)
5661
			free_slab(s, slab);
5662

5663
		if (node_nr_slabs(n))
5664
			ret = 1;
5665 5666
	}

5667
	return ret;
5668 5669
}

5670 5671 5672 5673 5674 5675
int __kmem_cache_shrink(struct kmem_cache *s)
{
	flush_all(s);
	return __kmem_cache_do_shrink(s);
}

5676 5677 5678 5679
static int slab_mem_going_offline_callback(void *arg)
{
	struct kmem_cache *s;

5680
	mutex_lock(&slab_mutex);
5681 5682 5683 5684
	list_for_each_entry(s, &slab_caches, list) {
		flush_all_cpus_locked(s);
		__kmem_cache_do_shrink(s);
	}
5685
	mutex_unlock(&slab_mutex);
5686 5687 5688 5689 5690 5691 5692 5693 5694

	return 0;
}

static void slab_mem_offline_callback(void *arg)
{
	struct memory_notify *marg = arg;
	int offline_node;

5695
	offline_node = marg->status_change_nid_normal;
5696 5697 5698 5699 5700 5701 5702 5703

	/*
	 * If the node still has available memory. we need kmem_cache_node
	 * for it yet.
	 */
	if (offline_node < 0)
		return;

5704
	mutex_lock(&slab_mutex);
5705
	node_clear(offline_node, slab_nodes);
5706 5707 5708 5709 5710
	/*
	 * We no longer free kmem_cache_node structures here, as it would be
	 * racy with all get_node() users, and infeasible to protect them with
	 * slab_mutex.
	 */
5711
	mutex_unlock(&slab_mutex);
5712 5713 5714 5715 5716 5717 5718
}

static int slab_mem_going_online_callback(void *arg)
{
	struct kmem_cache_node *n;
	struct kmem_cache *s;
	struct memory_notify *marg = arg;
5719
	int nid = marg->status_change_nid_normal;
5720 5721 5722 5723 5724 5725 5726 5727 5728 5729
	int ret = 0;

	/*
	 * If the node's memory is already available, then kmem_cache_node is
	 * already created. Nothing to do.
	 */
	if (nid < 0)
		return 0;

	/*
5730
	 * We are bringing a node online. No memory is available yet. We must
5731 5732 5733
	 * allocate a kmem_cache_node structure in order to bring the node
	 * online.
	 */
5734
	mutex_lock(&slab_mutex);
5735
	list_for_each_entry(s, &slab_caches, list) {
5736 5737 5738 5739 5740 5741
		/*
		 * The structure may already exist if the node was previously
		 * onlined and offlined.
		 */
		if (get_node(s, nid))
			continue;
5742 5743 5744 5745 5746
		/*
		 * XXX: kmem_cache_alloc_node will fallback to other nodes
		 *      since memory is not yet available from the node that
		 *      is brought up.
		 */
5747
		n = kmem_cache_alloc(kmem_cache_node, GFP_KERNEL);
5748 5749 5750 5751
		if (!n) {
			ret = -ENOMEM;
			goto out;
		}
5752
		init_kmem_cache_node(n);
5753 5754
		s->node[nid] = n;
	}
5755 5756 5757 5758 5759
	/*
	 * Any cache created after this point will also have kmem_cache_node
	 * initialized for the new node.
	 */
	node_set(nid, slab_nodes);
5760
out:
5761
	mutex_unlock(&slab_mutex);
5762 5763 5764 5765 5766 5767 5768 5769 5770 5771 5772 5773 5774 5775 5776 5777 5778 5779 5780 5781 5782 5783 5784
	return ret;
}

static int slab_memory_callback(struct notifier_block *self,
				unsigned long action, void *arg)
{
	int ret = 0;

	switch (action) {
	case MEM_GOING_ONLINE:
		ret = slab_mem_going_online_callback(arg);
		break;
	case MEM_GOING_OFFLINE:
		ret = slab_mem_going_offline_callback(arg);
		break;
	case MEM_OFFLINE:
	case MEM_CANCEL_ONLINE:
		slab_mem_offline_callback(arg);
		break;
	case MEM_ONLINE:
	case MEM_CANCEL_OFFLINE:
		break;
	}
5785 5786 5787 5788
	if (ret)
		ret = notifier_from_errno(ret);
	else
		ret = NOTIFY_OK;
5789 5790 5791
	return ret;
}

Christoph Lameter's avatar
Christoph Lameter committed
5792 5793 5794 5795
/********************************************************************
 *			Basic setup of slabs
 *******************************************************************/

5796 5797
/*
 * Used for early kmem_cache structures that were allocated using
5798 5799
 * the page allocator. Allocate them properly then fix up the pointers
 * that may be pointing to the wrong kmem_cache structure.
5800 5801
 */

5802
static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache)
5803 5804
{
	int node;
5805
	struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
5806
	struct kmem_cache_node *n;
5807

5808
	memcpy(s, static_cache, kmem_cache->object_size);
5809

5810 5811 5812 5813 5814 5815
	/*
	 * This runs very early, and only the boot processor is supposed to be
	 * up.  Even if it weren't true, IRQs are not up so we couldn't fire
	 * IPIs around.
	 */
	__flush_cpu_slab(s, smp_processor_id());
5816
	for_each_kmem_cache_node(s, node, n) {
5817
		struct slab *p;
5818

5819
		list_for_each_entry(p, &n->partial, slab_list)
5820
			p->slab_cache = s;
5821

Li Zefan's avatar
Li Zefan committed
5822
#ifdef CONFIG_SLUB_DEBUG
5823
		list_for_each_entry(p, &n->full, slab_list)
5824
			p->slab_cache = s;
5825 5826
#endif
	}
5827 5828
	list_add(&s->list, &slab_caches);
	return s;
5829 5830
}

Christoph Lameter's avatar
Christoph Lameter committed
5831 5832
void __init kmem_cache_init(void)
{
5833 5834
	static __initdata struct kmem_cache boot_kmem_cache,
		boot_kmem_cache_node;
5835
	int node;
5836

5837 5838 5839
	if (debug_guardpage_minorder())
		slub_max_order = 0;

5840 5841 5842 5843
	/* Print slub debugging pointers without hashing */
	if (__slub_debug_enabled())
		no_hash_pointers_enable(NULL);

5844 5845
	kmem_cache_node = &boot_kmem_cache_node;
	kmem_cache = &boot_kmem_cache;
5846

5847 5848 5849 5850 5851 5852 5853
	/*
	 * Initialize the nodemask for which we will allocate per node
	 * structures. Here we don't need taking slab_mutex yet.
	 */
	for_each_node_state(node, N_NORMAL_MEMORY)
		node_set(node, slab_nodes);

5854
	create_boot_cache(kmem_cache_node, "kmem_cache_node",
5855 5856
			sizeof(struct kmem_cache_node),
			SLAB_HWCACHE_ALIGN | SLAB_NO_OBJ_EXT, 0, 0);
5857

5858
	hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
Christoph Lameter's avatar
Christoph Lameter committed
5859 5860 5861 5862

	/* Able to allocate the per node structures */
	slab_state = PARTIAL;

5863 5864 5865
	create_boot_cache(kmem_cache, "kmem_cache",
			offsetof(struct kmem_cache, node) +
				nr_node_ids * sizeof(struct kmem_cache_node *),
5866
			SLAB_HWCACHE_ALIGN | SLAB_NO_OBJ_EXT, 0, 0);
5867

5868 5869
	kmem_cache = bootstrap(&boot_kmem_cache);
	kmem_cache_node = bootstrap(&boot_kmem_cache_node);
5870 5871

	/* Now we can use the kmem_cache to allocate kmalloc slabs */
5872
	setup_kmalloc_cache_index_table();
5873
	create_kmalloc_caches();
Christoph Lameter's avatar
Christoph Lameter committed
5874

5875 5876 5877
	/* Setup random freelists for each cache */
	init_freelist_randomization();

5878 5879
	cpuhp_setup_state_nocalls(CPUHP_SLUB_DEAD, "slub:dead", NULL,
				  slub_cpu_dead);
Christoph Lameter's avatar
Christoph Lameter committed
5880

5881
	pr_info("SLUB: HWalign=%d, Order=%u-%u, MinObjects=%u, CPUs=%u, Nodes=%u\n",
5882
		cache_line_size(),
Christoph Lameter's avatar
Christoph Lameter committed
5883 5884 5885 5886
		slub_min_order, slub_max_order, slub_min_objects,
		nr_cpu_ids, nr_node_ids);
}

5887 5888
void __init kmem_cache_init_late(void)
{
5889
#ifndef CONFIG_SLUB_TINY
5890 5891
	flushwq = alloc_workqueue("slub_flushwq", WQ_MEM_RECLAIM, 0);
	WARN_ON(!flushwq);
5892
#endif
5893 5894
}

5895
struct kmem_cache *
5896
__kmem_cache_alias(const char *name, unsigned int size, unsigned int align,
5897
		   slab_flags_t flags, void (*ctor)(void *))
Christoph Lameter's avatar
Christoph Lameter committed
5898
{
5899
	struct kmem_cache *s;
Christoph Lameter's avatar
Christoph Lameter committed
5900

5901
	s = find_mergeable(size, align, flags, name, ctor);
Christoph Lameter's avatar
Christoph Lameter committed
5902
	if (s) {
5903 5904 5905
		if (sysfs_slab_alias(s, name))
			return NULL;

Christoph Lameter's avatar
Christoph Lameter committed
5906
		s->refcount++;
5907

Christoph Lameter's avatar
Christoph Lameter committed
5908 5909 5910 5911
		/*
		 * Adjust the object sizes so that we clear
		 * the complete object on kzalloc.
		 */
5912
		s->object_size = max(s->object_size, size);
5913
		s->inuse = max(s->inuse, ALIGN(size, sizeof(void *)));
5914
	}
Christoph Lameter's avatar
Christoph Lameter committed
5915

5916 5917
	return s;
}
5918

5919
int __kmem_cache_create(struct kmem_cache *s, slab_flags_t flags)
5920
{
5921 5922 5923 5924 5925
	int err;

	err = kmem_cache_open(s, flags);
	if (err)
		return err;
5926

5927 5928 5929 5930
	/* Mutex is not taken during early boot */
	if (slab_state <= UP)
		return 0;

5931
	err = sysfs_slab_add(s);
5932
	if (err) {
5933
		__kmem_cache_release(s);
5934 5935
		return err;
	}
5936

5937 5938 5939
	if (s->flags & SLAB_STORE_USER)
		debugfs_slab_add(s);

5940
	return 0;
Christoph Lameter's avatar
Christoph Lameter committed
5941 5942
}

5943
#ifdef SLAB_SUPPORTS_SYSFS
5944
static int count_inuse(struct slab *slab)
5945
{
5946
	return slab->inuse;
5947 5948
}

5949
static int count_total(struct slab *slab)
5950
{
5951
	return slab->objects;
5952
}
5953
#endif
5954

5955
#ifdef CONFIG_SLUB_DEBUG
5956
static void validate_slab(struct kmem_cache *s, struct slab *slab,
5957
			  unsigned long *obj_map)
5958 5959
{
	void *p;
5960
	void *addr = slab_address(slab);
5961

5962
	if (!check_slab(s, slab) || !on_freelist(s, slab, NULL))
5963
		return;
5964 5965

	/* Now we know that a valid freelist exists */
5966 5967
	__fill_map(obj_map, s, slab);
	for_each_object(p, s, addr, slab->objects) {
5968
		u8 val = test_bit(__obj_to_index(s, addr, p), obj_map) ?
5969
			 SLUB_RED_INACTIVE : SLUB_RED_ACTIVE;
5970

5971
		if (!check_object(s, slab, p, val))
5972 5973
			break;
	}
5974 5975
}

5976
static int validate_slab_node(struct kmem_cache *s,
5977
		struct kmem_cache_node *n, unsigned long *obj_map)
5978 5979
{
	unsigned long count = 0;
5980
	struct slab *slab;
5981 5982 5983 5984
	unsigned long flags;

	spin_lock_irqsave(&n->list_lock, flags);

5985 5986
	list_for_each_entry(slab, &n->partial, slab_list) {
		validate_slab(s, slab, obj_map);
5987 5988
		count++;
	}
5989
	if (count != n->nr_partial) {
5990 5991
		pr_err("SLUB %s: %ld partial slabs counted but counter=%ld\n",
		       s->name, count, n->nr_partial);
5992 5993
		slab_add_kunit_errors();
	}
5994 5995 5996 5997

	if (!(s->flags & SLAB_STORE_USER))
		goto out;

5998 5999
	list_for_each_entry(slab, &n->full, slab_list) {
		validate_slab(s, slab, obj_map);
6000 6001
		count++;
	}
6002
	if (count != node_nr_slabs(n)) {
6003
		pr_err("SLUB: %s %ld slabs counted but counter=%ld\n",
6004
		       s->name, count, node_nr_slabs(n));
6005 6006
		slab_add_kunit_errors();
	}
6007 6008 6009 6010 6011 6012

out:
	spin_unlock_irqrestore(&n->list_lock, flags);
	return count;
}

6013
long validate_slab_cache(struct kmem_cache *s)
6014 6015 6016
{
	int node;
	unsigned long count = 0;
6017
	struct kmem_cache_node *n;
6018 6019 6020 6021 6022
	unsigned long *obj_map;

	obj_map = bitmap_alloc(oo_objects(s->oo), GFP_KERNEL);
	if (!obj_map)
		return -ENOMEM;
6023 6024

	flush_all(s);
6025
	for_each_kmem_cache_node(s, node, n)
6026 6027 6028
		count += validate_slab_node(s, n, obj_map);

	bitmap_free(obj_map);
6029

6030 6031
	return count;
}
6032 6033
EXPORT_SYMBOL(validate_slab_cache);

6034
#ifdef CONFIG_DEBUG_FS
6035
/*
Christoph Lameter's avatar
Christoph Lameter committed
6036
 * Generate lists of code addresses where slabcache objects are allocated
6037 6038 6039 6040
 * and freed.
 */

struct location {
6041
	depot_stack_handle_t handle;
6042
	unsigned long count;
6043
	unsigned long addr;
6044
	unsigned long waste;
6045 6046 6047 6048 6049
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
Rusty Russell's avatar
Rusty Russell committed
6050
	DECLARE_BITMAP(cpus, NR_CPUS);
6051
	nodemask_t nodes;
6052 6053 6054 6055 6056 6057
};

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
6058
	loff_t idx;
6059 6060
};

6061 6062
static struct dentry *slab_debugfs_root;

6063 6064 6065 6066 6067 6068 6069
static void free_loc_track(struct loc_track *t)
{
	if (t->max)
		free_pages((unsigned long)t->loc,
			get_order(sizeof(struct location) * t->max));
}

6070
static int alloc_loc_track(struct loc_track *t, unsigned long max, gfp_t flags)
6071 6072 6073 6074 6075 6076
{
	struct location *l;
	int order;

	order = get_order(sizeof(struct location) * max);

6077
	l = (void *)__get_free_pages(flags, order);
6078 6079 6080 6081 6082 6083 6084 6085 6086 6087 6088 6089 6090
	if (!l)
		return 0;

	if (t->count) {
		memcpy(l, t->loc, sizeof(struct location) * t->count);
		free_loc_track(t);
	}
	t->max = max;
	t->loc = l;
	return 1;
}

static int add_location(struct loc_track *t, struct kmem_cache *s,
6091 6092
				const struct track *track,
				unsigned int orig_size)
6093 6094 6095
{
	long start, end, pos;
	struct location *l;
6096
	unsigned long caddr, chandle, cwaste;
6097
	unsigned long age = jiffies - track->when;
6098
	depot_stack_handle_t handle = 0;
6099
	unsigned int waste = s->object_size - orig_size;
6100

6101 6102 6103
#ifdef CONFIG_STACKDEPOT
	handle = READ_ONCE(track->handle);
#endif
6104 6105 6106 6107 6108 6109 6110 6111 6112 6113 6114 6115 6116
	start = -1;
	end = t->count;

	for ( ; ; ) {
		pos = start + (end - start + 1) / 2;

		/*
		 * There is nothing at "end". If we end up there
		 * we need to add something to before end.
		 */
		if (pos == end)
			break;

6117 6118 6119 6120 6121 6122
		l = &t->loc[pos];
		caddr = l->addr;
		chandle = l->handle;
		cwaste = l->waste;
		if ((track->addr == caddr) && (handle == chandle) &&
			(waste == cwaste)) {
6123 6124 6125 6126 6127 6128 6129 6130 6131 6132 6133 6134 6135 6136

			l->count++;
			if (track->when) {
				l->sum_time += age;
				if (age < l->min_time)
					l->min_time = age;
				if (age > l->max_time)
					l->max_time = age;

				if (track->pid < l->min_pid)
					l->min_pid = track->pid;
				if (track->pid > l->max_pid)
					l->max_pid = track->pid;

Rusty Russell's avatar
Rusty Russell committed
6137 6138
				cpumask_set_cpu(track->cpu,
						to_cpumask(l->cpus));
6139 6140
			}
			node_set(page_to_nid(virt_to_page(track)), l->nodes);
6141 6142 6143
			return 1;
		}

6144
		if (track->addr < caddr)
6145
			end = pos;
6146 6147
		else if (track->addr == caddr && handle < chandle)
			end = pos;
6148 6149 6150
		else if (track->addr == caddr && handle == chandle &&
				waste < cwaste)
			end = pos;
6151 6152 6153 6154 6155
		else
			start = pos;
	}

	/*
Christoph Lameter's avatar
Christoph Lameter committed
6156
	 * Not found. Insert new tracking element.
6157
	 */
6158
	if (t->count >= t->max && !alloc_loc_track(t, 2 * t->max, GFP_ATOMIC))
6159 6160 6161 6162 6163 6164 6165 6166
		return 0;

	l = t->loc + pos;
	if (pos < t->count)
		memmove(l + 1, l,
			(t->count - pos) * sizeof(struct location));
	t->count++;
	l->count = 1;
6167 6168 6169 6170 6171 6172
	l->addr = track->addr;
	l->sum_time = age;
	l->min_time = age;
	l->max_time = age;
	l->min_pid = track->pid;
	l->max_pid = track->pid;
6173
	l->handle = handle;
6174
	l->waste = waste;
Rusty Russell's avatar
Rusty Russell committed
6175 6176
	cpumask_clear(to_cpumask(l->cpus));
	cpumask_set_cpu(track->cpu, to_cpumask(l->cpus));
6177 6178
	nodes_clear(l->nodes);
	node_set(page_to_nid(virt_to_page(track)), l->nodes);
6179 6180 6181 6182
	return 1;
}

static void process_slab(struct loc_track *t, struct kmem_cache *s,
6183
		struct slab *slab, enum track_item alloc,
6184
		unsigned long *obj_map)
6185
{
6186
	void *addr = slab_address(slab);
6187
	bool is_alloc = (alloc == TRACK_ALLOC);
6188 6189
	void *p;

6190
	__fill_map(obj_map, s, slab);
6191

6192
	for_each_object(p, s, addr, slab->objects)
6193
		if (!test_bit(__obj_to_index(s, addr, p), obj_map))
6194 6195 6196
			add_location(t, s, get_track(s, p, alloc),
				     is_alloc ? get_orig_size(s, p) :
						s->object_size);
6197
}
6198
#endif  /* CONFIG_DEBUG_FS   */
6199
#endif	/* CONFIG_SLUB_DEBUG */
6200

6201
#ifdef SLAB_SUPPORTS_SYSFS
Christoph Lameter's avatar
Christoph Lameter committed
6202
enum slab_stat_type {
6203 6204 6205 6206 6207
	SL_ALL,			/* All slabs */
	SL_PARTIAL,		/* Only partially allocated slabs */
	SL_CPU,			/* Only slabs used for cpu caches */
	SL_OBJECTS,		/* Determine allocated objects not slabs */
	SL_TOTAL		/* Determine object capacity not slabs */
Christoph Lameter's avatar
Christoph Lameter committed
6208 6209
};

6210
#define SO_ALL		(1 << SL_ALL)
Christoph Lameter's avatar
Christoph Lameter committed
6211 6212 6213
#define SO_PARTIAL	(1 << SL_PARTIAL)
#define SO_CPU		(1 << SL_CPU)
#define SO_OBJECTS	(1 << SL_OBJECTS)
6214
#define SO_TOTAL	(1 << SL_TOTAL)
Christoph Lameter's avatar
Christoph Lameter committed
6215

6216
static ssize_t show_slab_objects(struct kmem_cache *s,
6217
				 char *buf, unsigned long flags)
Christoph Lameter's avatar
Christoph Lameter committed
6218 6219 6220 6221 6222
{
	unsigned long total = 0;
	int node;
	int x;
	unsigned long *nodes;
6223
	int len = 0;
Christoph Lameter's avatar
Christoph Lameter committed
6224

6225
	nodes = kcalloc(nr_node_ids, sizeof(unsigned long), GFP_KERNEL);
6226 6227
	if (!nodes)
		return -ENOMEM;
Christoph Lameter's avatar
Christoph Lameter committed
6228

6229 6230
	if (flags & SO_CPU) {
		int cpu;
Christoph Lameter's avatar
Christoph Lameter committed
6231

6232
		for_each_possible_cpu(cpu) {
6233 6234
			struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab,
							       cpu);
6235
			int node;
6236
			struct slab *slab;
6237

6238 6239
			slab = READ_ONCE(c->slab);
			if (!slab)
6240
				continue;
6241

6242
			node = slab_nid(slab);
6243
			if (flags & SO_TOTAL)
6244
				x = slab->objects;
6245
			else if (flags & SO_OBJECTS)
6246
				x = slab->inuse;
6247 6248
			else
				x = 1;
6249

6250 6251 6252
			total += x;
			nodes[node] += x;

6253
#ifdef CONFIG_SLUB_CPU_PARTIAL
6254 6255 6256
			slab = slub_percpu_partial_read_once(c);
			if (slab) {
				node = slab_nid(slab);
6257 6258 6259 6260 6261
				if (flags & SO_TOTAL)
					WARN_ON_ONCE(1);
				else if (flags & SO_OBJECTS)
					WARN_ON_ONCE(1);
				else
6262
					x = slab->slabs;
6263 6264
				total += x;
				nodes[node] += x;
6265
			}
6266
#endif
Christoph Lameter's avatar
Christoph Lameter committed
6267 6268 6269
		}
	}

6270 6271 6272 6273 6274 6275 6276 6277 6278 6279 6280
	/*
	 * It is impossible to take "mem_hotplug_lock" here with "kernfs_mutex"
	 * already held which will conflict with an existing lock order:
	 *
	 * mem_hotplug_lock->slab_mutex->kernfs_mutex
	 *
	 * We don't really need mem_hotplug_lock (to hold off
	 * slab_mem_going_offline_callback) here because slab's memory hot
	 * unplug code doesn't destroy the kmem_cache->node[] data.
	 */

6281
#ifdef CONFIG_SLUB_DEBUG
6282
	if (flags & SO_ALL) {
6283 6284 6285
		struct kmem_cache_node *n;

		for_each_kmem_cache_node(s, node, n) {
6286

6287
			if (flags & SO_TOTAL)
6288
				x = node_nr_objs(n);
6289
			else if (flags & SO_OBJECTS)
6290
				x = node_nr_objs(n) - count_partial(n, count_free);
Christoph Lameter's avatar
Christoph Lameter committed
6291
			else
6292
				x = node_nr_slabs(n);
Christoph Lameter's avatar
Christoph Lameter committed
6293 6294 6295 6296
			total += x;
			nodes[node] += x;
		}

6297 6298 6299
	} else
#endif
	if (flags & SO_PARTIAL) {
6300
		struct kmem_cache_node *n;
Christoph Lameter's avatar
Christoph Lameter committed
6301

6302
		for_each_kmem_cache_node(s, node, n) {
6303 6304 6305 6306
			if (flags & SO_TOTAL)
				x = count_partial(n, count_total);
			else if (flags & SO_OBJECTS)
				x = count_partial(n, count_inuse);
Christoph Lameter's avatar
Christoph Lameter committed
6307
			else
6308
				x = n->nr_partial;
Christoph Lameter's avatar
Christoph Lameter committed
6309 6310 6311 6312
			total += x;
			nodes[node] += x;
		}
	}
6313 6314

	len += sysfs_emit_at(buf, len, "%lu", total);
Christoph Lameter's avatar
Christoph Lameter committed
6315
#ifdef CONFIG_NUMA
6316
	for (node = 0; node < nr_node_ids; node++) {
Christoph Lameter's avatar
Christoph Lameter committed
6317
		if (nodes[node])
6318 6319 6320
			len += sysfs_emit_at(buf, len, " N%d=%lu",
					     node, nodes[node]);
	}
Christoph Lameter's avatar
Christoph Lameter committed
6321
#endif
6322
	len += sysfs_emit_at(buf, len, "\n");
Christoph Lameter's avatar
Christoph Lameter committed
6323
	kfree(nodes);
6324 6325

	return len;
Christoph Lameter's avatar
Christoph Lameter committed
6326 6327 6328
}

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)
6329
#define to_slab(n) container_of(n, struct kmem_cache, kobj)
Christoph Lameter's avatar
Christoph Lameter committed
6330 6331 6332 6333 6334 6335 6336 6337

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
};

#define SLAB_ATTR_RO(_name) \
6338
	static struct slab_attribute _name##_attr = __ATTR_RO_MODE(_name, 0400)
Christoph Lameter's avatar
Christoph Lameter committed
6339 6340

#define SLAB_ATTR(_name) \
6341
	static struct slab_attribute _name##_attr = __ATTR_RW_MODE(_name, 0600)
Christoph Lameter's avatar
Christoph Lameter committed
6342 6343 6344

static ssize_t slab_size_show(struct kmem_cache *s, char *buf)
{
6345
	return sysfs_emit(buf, "%u\n", s->size);
Christoph Lameter's avatar
Christoph Lameter committed
6346 6347 6348 6349 6350
}
SLAB_ATTR_RO(slab_size);

static ssize_t align_show(struct kmem_cache *s, char *buf)
{
6351
	return sysfs_emit(buf, "%u\n", s->align);
Christoph Lameter's avatar
Christoph Lameter committed
6352 6353 6354 6355 6356
}
SLAB_ATTR_RO(align);

static ssize_t object_size_show(struct kmem_cache *s, char *buf)
{
6357
	return sysfs_emit(buf, "%u\n", s->object_size);
Christoph Lameter's avatar
Christoph Lameter committed
6358 6359 6360 6361 6362
}
SLAB_ATTR_RO(object_size);

static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf)
{
6363
	return sysfs_emit(buf, "%u\n", oo_objects(s->oo));
Christoph Lameter's avatar
Christoph Lameter committed
6364 6365 6366 6367 6368
}
SLAB_ATTR_RO(objs_per_slab);

static ssize_t order_show(struct kmem_cache *s, char *buf)
{
6369
	return sysfs_emit(buf, "%u\n", oo_order(s->oo));
Christoph Lameter's avatar
Christoph Lameter committed
6370
}
6371
SLAB_ATTR_RO(order);
Christoph Lameter's avatar
Christoph Lameter committed
6372

6373 6374
static ssize_t min_partial_show(struct kmem_cache *s, char *buf)
{
6375
	return sysfs_emit(buf, "%lu\n", s->min_partial);
6376 6377 6378 6379 6380 6381 6382 6383
}

static ssize_t min_partial_store(struct kmem_cache *s, const char *buf,
				 size_t length)
{
	unsigned long min;
	int err;

6384
	err = kstrtoul(buf, 10, &min);
6385 6386 6387
	if (err)
		return err;

6388
	s->min_partial = min;
6389 6390 6391 6392
	return length;
}
SLAB_ATTR(min_partial);

6393 6394
static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf)
{
6395 6396 6397 6398 6399 6400
	unsigned int nr_partial = 0;
#ifdef CONFIG_SLUB_CPU_PARTIAL
	nr_partial = s->cpu_partial;
#endif

	return sysfs_emit(buf, "%u\n", nr_partial);
6401 6402 6403 6404 6405
}

static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf,
				 size_t length)
{
6406
	unsigned int objects;
6407 6408
	int err;

6409
	err = kstrtouint(buf, 10, &objects);
6410 6411
	if (err)
		return err;
6412
	if (objects && !kmem_cache_has_cpu_partial(s))
6413
		return -EINVAL;
6414

6415
	slub_set_cpu_partial(s, objects);
6416 6417 6418 6419 6420
	flush_all(s);
	return length;
}
SLAB_ATTR(cpu_partial);

Christoph Lameter's avatar
Christoph Lameter committed
6421 6422
static ssize_t ctor_show(struct kmem_cache *s, char *buf)
{
6423 6424
	if (!s->ctor)
		return 0;
6425
	return sysfs_emit(buf, "%pS\n", s->ctor);
Christoph Lameter's avatar
Christoph Lameter committed
6426 6427 6428 6429 6430
}
SLAB_ATTR_RO(ctor);

static ssize_t aliases_show(struct kmem_cache *s, char *buf)
{
6431
	return sysfs_emit(buf, "%d\n", s->refcount < 0 ? 0 : s->refcount - 1);
Christoph Lameter's avatar
Christoph Lameter committed
6432 6433 6434 6435 6436
}
SLAB_ATTR_RO(aliases);

static ssize_t partial_show(struct kmem_cache *s, char *buf)
{
6437
	return show_slab_objects(s, buf, SO_PARTIAL);
Christoph Lameter's avatar
Christoph Lameter committed
6438 6439 6440 6441 6442
}
SLAB_ATTR_RO(partial);

static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf)
{
6443
	return show_slab_objects(s, buf, SO_CPU);
Christoph Lameter's avatar
Christoph Lameter committed
6444 6445 6446
}
SLAB_ATTR_RO(cpu_slabs);

6447 6448 6449 6450 6451 6452
static ssize_t objects_partial_show(struct kmem_cache *s, char *buf)
{
	return show_slab_objects(s, buf, SO_PARTIAL|SO_OBJECTS);
}
SLAB_ATTR_RO(objects_partial);

6453 6454 6455
static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf)
{
	int objects = 0;
6456
	int slabs = 0;
6457
	int cpu __maybe_unused;
6458
	int len = 0;
6459

6460
#ifdef CONFIG_SLUB_CPU_PARTIAL
6461
	for_each_online_cpu(cpu) {
6462
		struct slab *slab;
6463

6464
		slab = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu));
6465

6466 6467
		if (slab)
			slabs += slab->slabs;
6468
	}
6469
#endif
6470

6471
	/* Approximate half-full slabs, see slub_set_cpu_partial() */
6472 6473
	objects = (slabs * oo_objects(s->oo)) / 2;
	len += sysfs_emit_at(buf, len, "%d(%d)", objects, slabs);
6474

6475
#ifdef CONFIG_SLUB_CPU_PARTIAL
6476
	for_each_online_cpu(cpu) {
6477
		struct slab *slab;
6478

6479 6480 6481 6482
		slab = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu));
		if (slab) {
			slabs = READ_ONCE(slab->slabs);
			objects = (slabs * oo_objects(s->oo)) / 2;
6483
			len += sysfs_emit_at(buf, len, " C%d=%d(%d)",
6484
					     cpu, objects, slabs);
6485
		}
6486 6487
	}
#endif
6488 6489 6490
	len += sysfs_emit_at(buf, len, "\n");

	return len;
6491 6492 6493
}
SLAB_ATTR_RO(slabs_cpu_partial);

6494 6495
static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf)
{
6496
	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT));
6497
}
6498
SLAB_ATTR_RO(reclaim_account);
6499 6500 6501

static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf)
{
6502
	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN));
6503 6504 6505 6506 6507 6508
}
SLAB_ATTR_RO(hwcache_align);

#ifdef CONFIG_ZONE_DMA
static ssize_t cache_dma_show(struct kmem_cache *s, char *buf)
{
6509
	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA));
6510 6511 6512 6513
}
SLAB_ATTR_RO(cache_dma);
#endif

6514
#ifdef CONFIG_HARDENED_USERCOPY
6515 6516
static ssize_t usersize_show(struct kmem_cache *s, char *buf)
{
6517
	return sysfs_emit(buf, "%u\n", s->usersize);
6518 6519
}
SLAB_ATTR_RO(usersize);
6520
#endif
6521

6522 6523
static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf)
{
6524
	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_TYPESAFE_BY_RCU));
6525 6526 6527
}
SLAB_ATTR_RO(destroy_by_rcu);

6528
#ifdef CONFIG_SLUB_DEBUG
6529 6530 6531 6532 6533 6534
static ssize_t slabs_show(struct kmem_cache *s, char *buf)
{
	return show_slab_objects(s, buf, SO_ALL);
}
SLAB_ATTR_RO(slabs);

6535 6536 6537 6538 6539 6540
static ssize_t total_objects_show(struct kmem_cache *s, char *buf)
{
	return show_slab_objects(s, buf, SO_ALL|SO_TOTAL);
}
SLAB_ATTR_RO(total_objects);

6541 6542 6543 6544 6545 6546
static ssize_t objects_show(struct kmem_cache *s, char *buf)
{
	return show_slab_objects(s, buf, SO_ALL|SO_OBJECTS);
}
SLAB_ATTR_RO(objects);

Christoph Lameter's avatar
Christoph Lameter committed
6547 6548
static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf)
{
6549
	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_CONSISTENCY_CHECKS));
Christoph Lameter's avatar
Christoph Lameter committed
6550
}
6551
SLAB_ATTR_RO(sanity_checks);
Christoph Lameter's avatar
Christoph Lameter committed
6552 6553 6554

static ssize_t trace_show(struct kmem_cache *s, char *buf)
{
6555
	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_TRACE));
Christoph Lameter's avatar
Christoph Lameter committed
6556
}
6557
SLAB_ATTR_RO(trace);
Christoph Lameter's avatar
Christoph Lameter committed
6558 6559 6560

static ssize_t red_zone_show(struct kmem_cache *s, char *buf)
{
6561
	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE));
Christoph Lameter's avatar
Christoph Lameter committed
6562 6563
}

6564
SLAB_ATTR_RO(red_zone);
Christoph Lameter's avatar
Christoph Lameter committed
6565 6566 6567

static ssize_t poison_show(struct kmem_cache *s, char *buf)
{
6568
	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_POISON));
Christoph Lameter's avatar
Christoph Lameter committed
6569 6570
}

6571
SLAB_ATTR_RO(poison);
Christoph Lameter's avatar
Christoph Lameter committed
6572 6573 6574

static ssize_t store_user_show(struct kmem_cache *s, char *buf)
{
6575
	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_STORE_USER));
Christoph Lameter's avatar
Christoph Lameter committed
6576 6577
}

6578
SLAB_ATTR_RO(store_user);
Christoph Lameter's avatar
Christoph Lameter committed
6579

6580 6581 6582 6583 6584 6585 6586 6587
static ssize_t validate_show(struct kmem_cache *s, char *buf)
{
	return 0;
}

static ssize_t validate_store(struct kmem_cache *s,
			const char *buf, size_t length)
{
6588 6589
	int ret = -EINVAL;

6590
	if (buf[0] == '1' && kmem_cache_debug(s)) {
6591 6592 6593 6594 6595
		ret = validate_slab_cache(s);
		if (ret >= 0)
			ret = length;
	}
	return ret;
6596 6597
}
SLAB_ATTR(validate);
6598 6599 6600 6601 6602 6603

#endif /* CONFIG_SLUB_DEBUG */

#ifdef CONFIG_FAILSLAB
static ssize_t failslab_show(struct kmem_cache *s, char *buf)
{
6604
	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_FAILSLAB));
6605
}
6606 6607 6608 6609 6610 6611 6612 6613 6614 6615 6616 6617 6618 6619 6620

static ssize_t failslab_store(struct kmem_cache *s, const char *buf,
				size_t length)
{
	if (s->refcount > 1)
		return -EINVAL;

	if (buf[0] == '1')
		WRITE_ONCE(s->flags, s->flags | SLAB_FAILSLAB);
	else
		WRITE_ONCE(s->flags, s->flags & ~SLAB_FAILSLAB);

	return length;
}
SLAB_ATTR(failslab);
6621
#endif
6622

6623 6624 6625 6626 6627 6628 6629 6630
static ssize_t shrink_show(struct kmem_cache *s, char *buf)
{
	return 0;
}

static ssize_t shrink_store(struct kmem_cache *s,
			const char *buf, size_t length)
{
6631
	if (buf[0] == '1')
6632
		kmem_cache_shrink(s);
6633
	else
6634 6635 6636 6637 6638
		return -EINVAL;
	return length;
}
SLAB_ATTR(shrink);

Christoph Lameter's avatar
Christoph Lameter committed
6639
#ifdef CONFIG_NUMA
6640
static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf)
Christoph Lameter's avatar
Christoph Lameter committed
6641
{
6642
	return sysfs_emit(buf, "%u\n", s->remote_node_defrag_ratio / 10);
Christoph Lameter's avatar
Christoph Lameter committed
6643 6644
}

6645
static ssize_t remote_node_defrag_ratio_store(struct kmem_cache *s,
Christoph Lameter's avatar
Christoph Lameter committed
6646 6647
				const char *buf, size_t length)
{
6648
	unsigned int ratio;
6649 6650
	int err;

6651
	err = kstrtouint(buf, 10, &ratio);
6652 6653
	if (err)
		return err;
6654 6655
	if (ratio > 100)
		return -ERANGE;
6656

6657
	s->remote_node_defrag_ratio = ratio * 10;
Christoph Lameter's avatar
Christoph Lameter committed
6658 6659 6660

	return length;
}
6661
SLAB_ATTR(remote_node_defrag_ratio);
Christoph Lameter's avatar
Christoph Lameter committed
6662 6663
#endif

6664 6665 6666 6667 6668
#ifdef CONFIG_SLUB_STATS
static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si)
{
	unsigned long sum  = 0;
	int cpu;
6669
	int len = 0;
6670
	int *data = kmalloc_array(nr_cpu_ids, sizeof(int), GFP_KERNEL);
6671 6672 6673 6674 6675

	if (!data)
		return -ENOMEM;

	for_each_online_cpu(cpu) {
6676
		unsigned x = per_cpu_ptr(s->cpu_slab, cpu)->stat[si];
6677 6678 6679 6680 6681

		data[cpu] = x;
		sum += x;
	}

6682
	len += sysfs_emit_at(buf, len, "%lu", sum);
6683

6684
#ifdef CONFIG_SMP
6685
	for_each_online_cpu(cpu) {
6686 6687 6688
		if (data[cpu])
			len += sysfs_emit_at(buf, len, " C%d=%u",
					     cpu, data[cpu]);
6689
	}
6690
#endif
6691
	kfree(data);
6692 6693 6694
	len += sysfs_emit_at(buf, len, "\n");

	return len;
6695 6696
}

6697 6698 6699 6700 6701
static void clear_stat(struct kmem_cache *s, enum stat_item si)
{
	int cpu;

	for_each_online_cpu(cpu)
6702
		per_cpu_ptr(s->cpu_slab, cpu)->stat[si] = 0;
6703 6704
}

6705 6706 6707 6708 6709
#define STAT_ATTR(si, text) 					\
static ssize_t text##_show(struct kmem_cache *s, char *buf)	\
{								\
	return show_stat(s, buf, si);				\
}								\
6710 6711 6712 6713 6714 6715 6716 6717 6718
static ssize_t text##_store(struct kmem_cache *s,		\
				const char *buf, size_t length)	\
{								\
	if (buf[0] != '0')					\
		return -EINVAL;					\
	clear_stat(s, si);					\
	return length;						\
}								\
SLAB_ATTR(text);						\
6719 6720 6721 6722 6723 6724 6725 6726 6727 6728 6729

STAT_ATTR(ALLOC_FASTPATH, alloc_fastpath);
STAT_ATTR(ALLOC_SLOWPATH, alloc_slowpath);
STAT_ATTR(FREE_FASTPATH, free_fastpath);
STAT_ATTR(FREE_SLOWPATH, free_slowpath);
STAT_ATTR(FREE_FROZEN, free_frozen);
STAT_ATTR(FREE_ADD_PARTIAL, free_add_partial);
STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial);
STAT_ATTR(ALLOC_FROM_PARTIAL, alloc_from_partial);
STAT_ATTR(ALLOC_SLAB, alloc_slab);
STAT_ATTR(ALLOC_REFILL, alloc_refill);
6730
STAT_ATTR(ALLOC_NODE_MISMATCH, alloc_node_mismatch);
6731 6732 6733 6734 6735 6736 6737
STAT_ATTR(FREE_SLAB, free_slab);
STAT_ATTR(CPUSLAB_FLUSH, cpuslab_flush);
STAT_ATTR(DEACTIVATE_FULL, deactivate_full);
STAT_ATTR(DEACTIVATE_EMPTY, deactivate_empty);
STAT_ATTR(DEACTIVATE_TO_HEAD, deactivate_to_head);
STAT_ATTR(DEACTIVATE_TO_TAIL, deactivate_to_tail);
STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees);
6738
STAT_ATTR(DEACTIVATE_BYPASS, deactivate_bypass);
6739
STAT_ATTR(ORDER_FALLBACK, order_fallback);
6740 6741
STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail);
STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail);
6742 6743
STAT_ATTR(CPU_PARTIAL_ALLOC, cpu_partial_alloc);
STAT_ATTR(CPU_PARTIAL_FREE, cpu_partial_free);
6744 6745
STAT_ATTR(CPU_PARTIAL_NODE, cpu_partial_node);
STAT_ATTR(CPU_PARTIAL_DRAIN, cpu_partial_drain);
6746
#endif	/* CONFIG_SLUB_STATS */
6747

6748 6749 6750 6751 6752 6753 6754 6755 6756 6757 6758 6759 6760 6761 6762 6763 6764 6765 6766 6767 6768 6769 6770
#ifdef CONFIG_KFENCE
static ssize_t skip_kfence_show(struct kmem_cache *s, char *buf)
{
	return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_SKIP_KFENCE));
}

static ssize_t skip_kfence_store(struct kmem_cache *s,
			const char *buf, size_t length)
{
	int ret = length;

	if (buf[0] == '0')
		s->flags &= ~SLAB_SKIP_KFENCE;
	else if (buf[0] == '1')
		s->flags |= SLAB_SKIP_KFENCE;
	else
		ret = -EINVAL;

	return ret;
}
SLAB_ATTR(skip_kfence);
#endif

6771
static struct attribute *slab_attrs[] = {
Christoph Lameter's avatar
Christoph Lameter committed
6772 6773 6774 6775
	&slab_size_attr.attr,
	&object_size_attr.attr,
	&objs_per_slab_attr.attr,
	&order_attr.attr,
6776
	&min_partial_attr.attr,
6777
	&cpu_partial_attr.attr,
6778
	&objects_partial_attr.attr,
Christoph Lameter's avatar
Christoph Lameter committed
6779 6780 6781 6782 6783 6784 6785 6786
	&partial_attr.attr,
	&cpu_slabs_attr.attr,
	&ctor_attr.attr,
	&aliases_attr.attr,
	&align_attr.attr,
	&hwcache_align_attr.attr,
	&reclaim_account_attr.attr,
	&destroy_by_rcu_attr.attr,
6787
	&shrink_attr.attr,
6788
	&slabs_cpu_partial_attr.attr,
6789
#ifdef CONFIG_SLUB_DEBUG
6790
	&total_objects_attr.attr,
6791
	&objects_attr.attr,
6792 6793 6794
	&slabs_attr.attr,
	&sanity_checks_attr.attr,
	&trace_attr.attr,
Christoph Lameter's avatar
Christoph Lameter committed
6795 6796 6797
	&red_zone_attr.attr,
	&poison_attr.attr,
	&store_user_attr.attr,
6798
	&validate_attr.attr,
6799
#endif
Christoph Lameter's avatar
Christoph Lameter committed
6800 6801 6802 6803
#ifdef CONFIG_ZONE_DMA
	&cache_dma_attr.attr,
#endif
#ifdef CONFIG_NUMA
6804
	&remote_node_defrag_ratio_attr.attr,
6805 6806 6807 6808 6809 6810 6811 6812 6813 6814 6815 6816
#endif
#ifdef CONFIG_SLUB_STATS
	&alloc_fastpath_attr.attr,
	&alloc_slowpath_attr.attr,
	&free_fastpath_attr.attr,
	&free_slowpath_attr.attr,
	&free_frozen_attr.attr,
	&free_add_partial_attr.attr,
	&free_remove_partial_attr.attr,
	&alloc_from_partial_attr.attr,
	&alloc_slab_attr.attr,
	&alloc_refill_attr.attr,
6817
	&alloc_node_mismatch_attr.attr,
6818 6819 6820 6821 6822 6823 6824
	&free_slab_attr.attr,
	&cpuslab_flush_attr.attr,
	&deactivate_full_attr.attr,
	&deactivate_empty_attr.attr,
	&deactivate_to_head_attr.attr,
	&deactivate_to_tail_attr.attr,
	&deactivate_remote_frees_attr.attr,
6825
	&deactivate_bypass_attr.attr,
6826
	&order_fallback_attr.attr,
6827 6828
	&cmpxchg_double_fail_attr.attr,
	&cmpxchg_double_cpu_fail_attr.attr,
6829 6830
	&cpu_partial_alloc_attr.attr,
	&cpu_partial_free_attr.attr,
6831 6832
	&cpu_partial_node_attr.attr,
	&cpu_partial_drain_attr.attr,
Christoph Lameter's avatar
Christoph Lameter committed
6833
#endif
6834 6835 6836
#ifdef CONFIG_FAILSLAB
	&failslab_attr.attr,
#endif
6837
#ifdef CONFIG_HARDENED_USERCOPY
6838
	&usersize_attr.attr,
6839
#endif
6840 6841 6842
#ifdef CONFIG_KFENCE
	&skip_kfence_attr.attr,
#endif
6843

Christoph Lameter's avatar
Christoph Lameter committed
6844 6845 6846
	NULL
};

6847
static const struct attribute_group slab_attr_group = {
Christoph Lameter's avatar
Christoph Lameter committed
6848 6849 6850 6851 6852 6853 6854 6855 6856 6857 6858 6859 6860 6861 6862 6863
	.attrs = slab_attrs,
};

static ssize_t slab_attr_show(struct kobject *kobj,
				struct attribute *attr,
				char *buf)
{
	struct slab_attribute *attribute;
	struct kmem_cache *s;

	attribute = to_slab_attr(attr);
	s = to_slab(kobj);

	if (!attribute->show)
		return -EIO;

6864
	return attribute->show(s, buf);
Christoph Lameter's avatar
Christoph Lameter committed
6865 6866 6867 6868 6869 6870 6871 6872 6873 6874 6875 6876 6877 6878 6879
}

static ssize_t slab_attr_store(struct kobject *kobj,
				struct attribute *attr,
				const char *buf, size_t len)
{
	struct slab_attribute *attribute;
	struct kmem_cache *s;

	attribute = to_slab_attr(attr);
	s = to_slab(kobj);

	if (!attribute->store)
		return -EIO;

6880
	return attribute->store(s, buf, len);
Christoph Lameter's avatar
Christoph Lameter committed
6881 6882
}

6883 6884 6885 6886 6887
static void kmem_cache_release(struct kobject *k)
{
	slab_kmem_cache_release(to_slab(k));
}

6888
static const struct sysfs_ops slab_sysfs_ops = {
Christoph Lameter's avatar
Christoph Lameter committed
6889 6890 6891 6892
	.show = slab_attr_show,
	.store = slab_attr_store,
};

6893
static const struct kobj_type slab_ktype = {
Christoph Lameter's avatar
Christoph Lameter committed
6894
	.sysfs_ops = &slab_sysfs_ops,
6895
	.release = kmem_cache_release,
Christoph Lameter's avatar
Christoph Lameter committed
6896 6897
};

6898
static struct kset *slab_kset;
Christoph Lameter's avatar
Christoph Lameter committed
6899

6900 6901 6902 6903 6904
static inline struct kset *cache_kset(struct kmem_cache *s)
{
	return slab_kset;
}

6905
#define ID_STR_LENGTH 32
Christoph Lameter's avatar
Christoph Lameter committed
6906 6907

/* Create a unique string id for a slab cache:
Christoph Lameter's avatar
Christoph Lameter committed
6908 6909
 *
 * Format	:[flags-]size
Christoph Lameter's avatar
Christoph Lameter committed
6910 6911 6912 6913 6914 6915
 */
static char *create_unique_id(struct kmem_cache *s)
{
	char *name = kmalloc(ID_STR_LENGTH, GFP_KERNEL);
	char *p = name;

6916 6917
	if (!name)
		return ERR_PTR(-ENOMEM);
Christoph Lameter's avatar
Christoph Lameter committed
6918 6919 6920 6921 6922 6923 6924 6925 6926 6927 6928

	*p++ = ':';
	/*
	 * First flags affecting slabcache operations. We will only
	 * get here for aliasable slabs so we do not need to support
	 * too many flags. The flags here must cover all flags that
	 * are matched during merging to guarantee that the id is
	 * unique.
	 */
	if (s->flags & SLAB_CACHE_DMA)
		*p++ = 'd';
6929 6930
	if (s->flags & SLAB_CACHE_DMA32)
		*p++ = 'D';
Christoph Lameter's avatar
Christoph Lameter committed
6931 6932
	if (s->flags & SLAB_RECLAIM_ACCOUNT)
		*p++ = 'a';
6933
	if (s->flags & SLAB_CONSISTENCY_CHECKS)
Christoph Lameter's avatar
Christoph Lameter committed
6934
		*p++ = 'F';
6935 6936
	if (s->flags & SLAB_ACCOUNT)
		*p++ = 'A';
Christoph Lameter's avatar
Christoph Lameter committed
6937 6938
	if (p != name + 1)
		*p++ = '-';
6939
	p += snprintf(p, ID_STR_LENGTH - (p - name), "%07u", s->size);
6940

6941 6942 6943 6944
	if (WARN_ON(p > name + ID_STR_LENGTH - 1)) {
		kfree(name);
		return ERR_PTR(-EINVAL);
	}
6945
	kmsan_unpoison_memory(name, p - name);
Christoph Lameter's avatar
Christoph Lameter committed
6946 6947 6948 6949 6950 6951 6952
	return name;
}

static int sysfs_slab_add(struct kmem_cache *s)
{
	int err;
	const char *name;
6953
	struct kset *kset = cache_kset(s);
6954
	int unmergeable = slab_unmergeable(s);
Christoph Lameter's avatar
Christoph Lameter committed
6955

6956 6957 6958 6959
	if (!unmergeable && disable_higher_order_debug &&
			(slub_debug & DEBUG_METADATA_FLAGS))
		unmergeable = 1;

Christoph Lameter's avatar
Christoph Lameter committed
6960 6961 6962 6963 6964 6965
	if (unmergeable) {
		/*
		 * Slabcache can never be merged so we can use the name proper.
		 * This is typically the case for debug situations. In that
		 * case we can catch duplicate names easily.
		 */
6966
		sysfs_remove_link(&slab_kset->kobj, s->name);
Christoph Lameter's avatar
Christoph Lameter committed
6967 6968 6969 6970 6971 6972 6973
		name = s->name;
	} else {
		/*
		 * Create a unique name for the slab as a target
		 * for the symlinks.
		 */
		name = create_unique_id(s);
6974 6975
		if (IS_ERR(name))
			return PTR_ERR(name);
Christoph Lameter's avatar
Christoph Lameter committed
6976 6977
	}

6978
	s->kobj.kset = kset;
6979
	err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name);
6980
	if (err)
6981
		goto out;
Christoph Lameter's avatar
Christoph Lameter committed
6982 6983

	err = sysfs_create_group(&s->kobj, &slab_attr_group);
6984 6985
	if (err)
		goto out_del_kobj;
6986

Christoph Lameter's avatar
Christoph Lameter committed
6987 6988 6989 6990
	if (!unmergeable) {
		/* Setup first alias */
		sysfs_slab_alias(s, s->name);
	}
6991 6992 6993 6994 6995 6996 6997
out:
	if (!unmergeable)
		kfree(name);
	return err;
out_del_kobj:
	kobject_del(&s->kobj);
	goto out;
Christoph Lameter's avatar
Christoph Lameter committed
6998 6999
}

7000 7001
void sysfs_slab_unlink(struct kmem_cache *s)
{
7002
	kobject_del(&s->kobj);
7003 7004
}

7005 7006
void sysfs_slab_release(struct kmem_cache *s)
{
7007
	kobject_put(&s->kobj);
Christoph Lameter's avatar
Christoph Lameter committed
7008 7009 7010 7011
}

/*
 * Need to buffer aliases during bootup until sysfs becomes
7012
 * available lest we lose that information.
Christoph Lameter's avatar
Christoph Lameter committed
7013 7014 7015 7016 7017 7018 7019
 */
struct saved_alias {
	struct kmem_cache *s;
	const char *name;
	struct saved_alias *next;
};

Adrian Bunk's avatar
Adrian Bunk committed
7020
static struct saved_alias *alias_list;
Christoph Lameter's avatar
Christoph Lameter committed
7021 7022 7023 7024 7025

static int sysfs_slab_alias(struct kmem_cache *s, const char *name)
{
	struct saved_alias *al;

7026
	if (slab_state == FULL) {
Christoph Lameter's avatar
Christoph Lameter committed
7027 7028 7029
		/*
		 * If we have a leftover link then remove it.
		 */
7030 7031
		sysfs_remove_link(&slab_kset->kobj, name);
		return sysfs_create_link(&slab_kset->kobj, &s->kobj, name);
Christoph Lameter's avatar
Christoph Lameter committed
7032 7033 7034 7035 7036 7037 7038 7039 7040 7041
	}

	al = kmalloc(sizeof(struct saved_alias), GFP_KERNEL);
	if (!al)
		return -ENOMEM;

	al->s = s;
	al->name = name;
	al->next = alias_list;
	alias_list = al;
7042
	kmsan_unpoison_memory(al, sizeof(*al));
Christoph Lameter's avatar
Christoph Lameter committed
7043 7044 7045 7046 7047
	return 0;
}

static int __init slab_sysfs_init(void)
{
7048
	struct kmem_cache *s;
Christoph Lameter's avatar
Christoph Lameter committed
7049 7050
	int err;

7051
	mutex_lock(&slab_mutex);
7052

7053
	slab_kset = kset_create_and_add("slab", NULL, kernel_kobj);
7054
	if (!slab_kset) {
7055
		mutex_unlock(&slab_mutex);
7056
		pr_err("Cannot register slab subsystem.\n");
7057
		return -ENOMEM;
Christoph Lameter's avatar
Christoph Lameter committed
7058 7059
	}

7060
	slab_state = FULL;
7061

7062
	list_for_each_entry(s, &slab_caches, list) {
7063
		err = sysfs_slab_add(s);
7064
		if (err)
7065 7066
			pr_err("SLUB: Unable to add boot slab %s to sysfs\n",
			       s->name);
7067
	}
Christoph Lameter's avatar
Christoph Lameter committed
7068 7069 7070 7071 7072 7073

	while (alias_list) {
		struct saved_alias *al = alias_list;

		alias_list = alias_list->next;
		err = sysfs_slab_alias(al->s, al->name);
7074
		if (err)
7075 7076
			pr_err("SLUB: Unable to add boot slab alias %s to sysfs\n",
			       al->name);
Christoph Lameter's avatar
Christoph Lameter committed
7077 7078 7079
		kfree(al);
	}

7080
	mutex_unlock(&slab_mutex);
Christoph Lameter's avatar
Christoph Lameter committed
7081 7082
	return 0;
}
7083
late_initcall(slab_sysfs_init);
7084
#endif /* SLAB_SUPPORTS_SYSFS */
7085

7086 7087 7088 7089
#if defined(CONFIG_SLUB_DEBUG) && defined(CONFIG_DEBUG_FS)
static int slab_debugfs_show(struct seq_file *seq, void *v)
{
	struct loc_track *t = seq->private;
7090 7091
	struct location *l;
	unsigned long idx;
7092

7093
	idx = (unsigned long) t->idx;
7094 7095 7096 7097 7098 7099 7100 7101 7102 7103
	if (idx < t->count) {
		l = &t->loc[idx];

		seq_printf(seq, "%7ld ", l->count);

		if (l->addr)
			seq_printf(seq, "%pS", (void *)l->addr);
		else
			seq_puts(seq, "<not-available>");

7104 7105 7106 7107
		if (l->waste)
			seq_printf(seq, " waste=%lu/%lu",
				l->count * l->waste, l->waste);

7108 7109 7110 7111 7112 7113 7114 7115 7116 7117 7118 7119 7120 7121 7122 7123 7124 7125 7126 7127 7128
		if (l->sum_time != l->min_time) {
			seq_printf(seq, " age=%ld/%llu/%ld",
				l->min_time, div_u64(l->sum_time, l->count),
				l->max_time);
		} else
			seq_printf(seq, " age=%ld", l->min_time);

		if (l->min_pid != l->max_pid)
			seq_printf(seq, " pid=%ld-%ld", l->min_pid, l->max_pid);
		else
			seq_printf(seq, " pid=%ld",
				l->min_pid);

		if (num_online_cpus() > 1 && !cpumask_empty(to_cpumask(l->cpus)))
			seq_printf(seq, " cpus=%*pbl",
				 cpumask_pr_args(to_cpumask(l->cpus)));

		if (nr_online_nodes > 1 && !nodes_empty(l->nodes))
			seq_printf(seq, " nodes=%*pbl",
				 nodemask_pr_args(&l->nodes));

7129 7130 7131 7132 7133 7134 7135 7136 7137 7138 7139 7140 7141 7142 7143
#ifdef CONFIG_STACKDEPOT
		{
			depot_stack_handle_t handle;
			unsigned long *entries;
			unsigned int nr_entries, j;

			handle = READ_ONCE(l->handle);
			if (handle) {
				nr_entries = stack_depot_fetch(handle, &entries);
				seq_puts(seq, "\n");
				for (j = 0; j < nr_entries; j++)
					seq_printf(seq, "        %pS\n", (void *)entries[j]);
			}
		}
#endif
7144 7145 7146 7147 7148 7149 7150 7151 7152 7153 7154 7155 7156 7157 7158 7159 7160
		seq_puts(seq, "\n");
	}

	if (!idx && !t->count)
		seq_puts(seq, "No data\n");

	return 0;
}

static void slab_debugfs_stop(struct seq_file *seq, void *v)
{
}

static void *slab_debugfs_next(struct seq_file *seq, void *v, loff_t *ppos)
{
	struct loc_track *t = seq->private;

7161
	t->idx = ++(*ppos);
7162
	if (*ppos <= t->count)
7163
		return ppos;
7164 7165 7166 7167

	return NULL;
}

7168 7169 7170 7171 7172 7173 7174 7175 7176 7177 7178
static int cmp_loc_by_count(const void *a, const void *b, const void *data)
{
	struct location *loc1 = (struct location *)a;
	struct location *loc2 = (struct location *)b;

	if (loc1->count > loc2->count)
		return -1;
	else
		return 1;
}

7179 7180
static void *slab_debugfs_start(struct seq_file *seq, loff_t *ppos)
{
7181 7182 7183
	struct loc_track *t = seq->private;

	t->idx = *ppos;
7184 7185 7186 7187 7188 7189 7190 7191 7192 7193 7194 7195 7196 7197 7198 7199 7200 7201 7202
	return ppos;
}

static const struct seq_operations slab_debugfs_sops = {
	.start  = slab_debugfs_start,
	.next   = slab_debugfs_next,
	.stop   = slab_debugfs_stop,
	.show   = slab_debugfs_show,
};

static int slab_debug_trace_open(struct inode *inode, struct file *filep)
{

	struct kmem_cache_node *n;
	enum track_item alloc;
	int node;
	struct loc_track *t = __seq_open_private(filep, &slab_debugfs_sops,
						sizeof(struct loc_track));
	struct kmem_cache *s = file_inode(filep)->i_private;
7203 7204
	unsigned long *obj_map;

7205 7206 7207
	if (!t)
		return -ENOMEM;

7208
	obj_map = bitmap_alloc(oo_objects(s->oo), GFP_KERNEL);
7209 7210
	if (!obj_map) {
		seq_release_private(inode, filep);
7211
		return -ENOMEM;
7212
	}
7213 7214 7215 7216 7217 7218

	if (strcmp(filep->f_path.dentry->d_name.name, "alloc_traces") == 0)
		alloc = TRACK_ALLOC;
	else
		alloc = TRACK_FREE;

7219 7220
	if (!alloc_loc_track(t, PAGE_SIZE / sizeof(struct location), GFP_KERNEL)) {
		bitmap_free(obj_map);
7221
		seq_release_private(inode, filep);
7222
		return -ENOMEM;
7223
	}
7224 7225 7226

	for_each_kmem_cache_node(s, node, n) {
		unsigned long flags;
7227
		struct slab *slab;
7228

7229
		if (!node_nr_slabs(n))
7230 7231 7232
			continue;

		spin_lock_irqsave(&n->list_lock, flags);
7233 7234 7235 7236
		list_for_each_entry(slab, &n->partial, slab_list)
			process_slab(t, s, slab, alloc, obj_map);
		list_for_each_entry(slab, &n->full, slab_list)
			process_slab(t, s, slab, alloc, obj_map);
7237 7238 7239
		spin_unlock_irqrestore(&n->list_lock, flags);
	}

7240 7241 7242 7243
	/* Sort locations by count */
	sort_r(t->loc, t->count, sizeof(struct location),
		cmp_loc_by_count, NULL, NULL);

7244
	bitmap_free(obj_map);
7245 7246 7247 7248 7249 7250 7251 7252 7253 7254 7255 7256 7257 7258 7259 7260 7261 7262 7263 7264 7265 7266 7267 7268 7269 7270 7271 7272 7273 7274 7275 7276 7277 7278 7279 7280 7281
	return 0;
}

static int slab_debug_trace_release(struct inode *inode, struct file *file)
{
	struct seq_file *seq = file->private_data;
	struct loc_track *t = seq->private;

	free_loc_track(t);
	return seq_release_private(inode, file);
}

static const struct file_operations slab_debugfs_fops = {
	.open    = slab_debug_trace_open,
	.read    = seq_read,
	.llseek  = seq_lseek,
	.release = slab_debug_trace_release,
};

static void debugfs_slab_add(struct kmem_cache *s)
{
	struct dentry *slab_cache_dir;

	if (unlikely(!slab_debugfs_root))
		return;

	slab_cache_dir = debugfs_create_dir(s->name, slab_debugfs_root);

	debugfs_create_file("alloc_traces", 0400,
		slab_cache_dir, s, &slab_debugfs_fops);

	debugfs_create_file("free_traces", 0400,
		slab_cache_dir, s, &slab_debugfs_fops);
}

void debugfs_slab_release(struct kmem_cache *s)
{
7282
	debugfs_lookup_and_remove(s->name, slab_debugfs_root);
7283 7284 7285 7286 7287 7288 7289 7290 7291 7292 7293 7294 7295 7296 7297 7298 7299
}

static int __init slab_debugfs_init(void)
{
	struct kmem_cache *s;

	slab_debugfs_root = debugfs_create_dir("slab", NULL);

	list_for_each_entry(s, &slab_caches, list)
		if (s->flags & SLAB_STORE_USER)
			debugfs_slab_add(s);

	return 0;

}
__initcall(slab_debugfs_init);
#endif
7300 7301 7302
/*
 * The /proc/slabinfo ABI
 */
7303
#ifdef CONFIG_SLUB_DEBUG
7304
void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo)
7305 7306
{
	unsigned long nr_slabs = 0;
7307 7308
	unsigned long nr_objs = 0;
	unsigned long nr_free = 0;
7309
	int node;
7310
	struct kmem_cache_node *n;
7311

7312
	for_each_kmem_cache_node(s, node, n) {
7313 7314
		nr_slabs += node_nr_slabs(n);
		nr_objs += node_nr_objs(n);
7315
		nr_free += count_partial(n, count_free);
7316 7317
	}

7318 7319 7320 7321 7322 7323
	sinfo->active_objs = nr_objs - nr_free;
	sinfo->num_objs = nr_objs;
	sinfo->active_slabs = nr_slabs;
	sinfo->num_slabs = nr_slabs;
	sinfo->objects_per_slab = oo_objects(s->oo);
	sinfo->cache_order = oo_order(s->oo);
7324 7325
}

7326
void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s)
7327 7328 7329
{
}

7330 7331
ssize_t slabinfo_write(struct file *file, const char __user *buffer,
		       size_t count, loff_t *ppos)
7332
{
7333
	return -EIO;
7334
}
7335
#endif /* CONFIG_SLUB_DEBUG */