hugetlb.c 218 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0-only
Linus Torvalds's avatar
Linus Torvalds committed
2 3
/*
 * Generic hugetlb support.
4
 * (C) Nadia Yvette Chambers, April 2004
Linus Torvalds's avatar
Linus Torvalds committed
5 6 7 8
 */
#include <linux/list.h>
#include <linux/init.h>
#include <linux/mm.h>
9
#include <linux/seq_file.h>
Linus Torvalds's avatar
Linus Torvalds committed
10 11
#include <linux/sysctl.h>
#include <linux/highmem.h>
Andrea Arcangeli's avatar
Andrea Arcangeli committed
12
#include <linux/mmu_notifier.h>
Linus Torvalds's avatar
Linus Torvalds committed
13
#include <linux/nodemask.h>
14
#include <linux/pagemap.h>
15
#include <linux/mempolicy.h>
16
#include <linux/compiler.h>
17
#include <linux/cpuset.h>
18
#include <linux/mutex.h>
19
#include <linux/memblock.h>
20
#include <linux/sysfs.h>
21
#include <linux/slab.h>
22
#include <linux/sched/mm.h>
23
#include <linux/mmdebug.h>
24
#include <linux/sched/signal.h>
25
#include <linux/rmap.h>
26
#include <linux/string_helpers.h>
27 28
#include <linux/swap.h>
#include <linux/swapops.h>
29
#include <linux/jhash.h>
30
#include <linux/numa.h>
31
#include <linux/llist.h>
32
#include <linux/cma.h>
33
#include <linux/migrate.h>
34
#include <linux/nospec.h>
35
#include <linux/delayacct.h>
36
#include <linux/memory.h>
37
#include <linux/mm_inline.h>
38
#include <linux/padata.h>
39

40
#include <asm/page.h>
41
#include <asm/pgalloc.h>
42
#include <asm/tlb.h>
43

44
#include <linux/io.h>
45
#include <linux/hugetlb.h>
46
#include <linux/hugetlb_cgroup.h>
47
#include <linux/node.h>
48
#include <linux/page_owner.h>
49
#include "internal.h"
50
#include "hugetlb_vmemmap.h"
Linus Torvalds's avatar
Linus Torvalds committed
51

52
int hugetlb_max_hstate __read_mostly;
53 54
unsigned int default_hstate_idx;
struct hstate hstates[HUGE_MAX_HSTATE];
55

56
#ifdef CONFIG_CMA
57
static struct cma *hugetlb_cma[MAX_NUMNODES];
58
static unsigned long hugetlb_cma_size_in_node[MAX_NUMNODES] __initdata;
59
static bool hugetlb_cma_folio(struct folio *folio, unsigned int order)
60
{
61
	return cma_pages_valid(hugetlb_cma[folio_nid(folio)], &folio->page,
62 63 64
				1 << order);
}
#else
65
static bool hugetlb_cma_folio(struct folio *folio, unsigned int order)
66 67 68
{
	return false;
}
69 70
#endif
static unsigned long hugetlb_cma_size __initdata;
71

72
__initdata struct list_head huge_boot_pages[MAX_NUMNODES];
73

74 75 76
/* for command line parsing */
static struct hstate * __initdata parsed_hstate;
static unsigned long __initdata default_hstate_max_huge_pages;
77
static bool __initdata parsed_valid_hugepagesz = true;
78
static bool __initdata parsed_default_hugepagesz;
79
static unsigned int default_hugepages_in_node[MAX_NUMNODES] __initdata;
80

81
/*
82 83
 * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages,
 * free_huge_pages, and surplus_huge_pages.
84
 */
85
DEFINE_SPINLOCK(hugetlb_lock);
86

87 88 89 90 91
/*
 * Serializes faults on the same logical page.  This is used to
 * prevent spurious OOMs when the hugepage pool is fully utilized.
 */
static int num_fault_mutexes;
92
struct mutex *hugetlb_fault_mutex_table ____cacheline_aligned_in_smp;
93

94 95
/* Forward declaration */
static int hugetlb_acct_memory(struct hstate *h, long delta);
96 97
static void hugetlb_vma_lock_free(struct vm_area_struct *vma);
static void hugetlb_vma_lock_alloc(struct vm_area_struct *vma);
98
static void __hugetlb_vma_unlock_write_free(struct vm_area_struct *vma);
99 100
static void hugetlb_unshare_pmds(struct vm_area_struct *vma,
		unsigned long start, unsigned long end);
101
static struct resv_map *vma_resv_map(struct vm_area_struct *vma);
102

103
static inline bool subpool_is_free(struct hugepage_subpool *spool)
104
{
105 106 107 108 109 110 111 112 113
	if (spool->count)
		return false;
	if (spool->max_hpages != -1)
		return spool->used_hpages == 0;
	if (spool->min_hpages != -1)
		return spool->rsv_hpages == spool->min_hpages;

	return true;
}
114

115 116
static inline void unlock_or_release_subpool(struct hugepage_subpool *spool,
						unsigned long irq_flags)
117
{
118
	spin_unlock_irqrestore(&spool->lock, irq_flags);
119 120

	/* If no pages are used, and no other handles to the subpool
121
	 * remain, give up any reservations based on minimum size and
122
	 * free the subpool */
123
	if (subpool_is_free(spool)) {
124 125 126
		if (spool->min_hpages != -1)
			hugetlb_acct_memory(spool->hstate,
						-spool->min_hpages);
127
		kfree(spool);
128
	}
129 130
}

131 132
struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages,
						long min_hpages)
133 134 135
{
	struct hugepage_subpool *spool;

136
	spool = kzalloc(sizeof(*spool), GFP_KERNEL);
137 138 139 140 141
	if (!spool)
		return NULL;

	spin_lock_init(&spool->lock);
	spool->count = 1;
142 143 144 145 146 147 148 149 150
	spool->max_hpages = max_hpages;
	spool->hstate = h;
	spool->min_hpages = min_hpages;

	if (min_hpages != -1 && hugetlb_acct_memory(h, min_hpages)) {
		kfree(spool);
		return NULL;
	}
	spool->rsv_hpages = min_hpages;
151 152 153 154 155 156

	return spool;
}

void hugepage_put_subpool(struct hugepage_subpool *spool)
{
157 158 159
	unsigned long flags;

	spin_lock_irqsave(&spool->lock, flags);
160 161
	BUG_ON(!spool->count);
	spool->count--;
162
	unlock_or_release_subpool(spool, flags);
163 164
}

165 166 167
/*
 * Subpool accounting for allocating and reserving pages.
 * Return -ENOMEM if there are not enough resources to satisfy the
168
 * request.  Otherwise, return the number of pages by which the
169 170
 * global pools must be adjusted (upward).  The returned value may
 * only be different than the passed value (delta) in the case where
171
 * a subpool minimum size must be maintained.
172 173
 */
static long hugepage_subpool_get_pages(struct hugepage_subpool *spool,
174 175
				      long delta)
{
176
	long ret = delta;
177 178

	if (!spool)
179
		return ret;
180

181
	spin_lock_irq(&spool->lock);
182 183 184 185 186 187 188 189

	if (spool->max_hpages != -1) {		/* maximum size accounting */
		if ((spool->used_hpages + delta) <= spool->max_hpages)
			spool->used_hpages += delta;
		else {
			ret = -ENOMEM;
			goto unlock_ret;
		}
190 191
	}

192 193
	/* minimum size accounting */
	if (spool->min_hpages != -1 && spool->rsv_hpages) {
194 195 196 197 198 199 200 201 202 203 204 205 206 207
		if (delta > spool->rsv_hpages) {
			/*
			 * Asking for more reserves than those already taken on
			 * behalf of subpool.  Return difference.
			 */
			ret = delta - spool->rsv_hpages;
			spool->rsv_hpages = 0;
		} else {
			ret = 0;	/* reserves already accounted for */
			spool->rsv_hpages -= delta;
		}
	}

unlock_ret:
208
	spin_unlock_irq(&spool->lock);
209 210 211
	return ret;
}

212 213 214 215 216 217 218
/*
 * Subpool accounting for freeing and unreserving pages.
 * Return the number of global page reservations that must be dropped.
 * The return value may only be different than the passed value (delta)
 * in the case where a subpool minimum size must be maintained.
 */
static long hugepage_subpool_put_pages(struct hugepage_subpool *spool,
219 220
				       long delta)
{
221
	long ret = delta;
222
	unsigned long flags;
223

224
	if (!spool)
225
		return delta;
226

227
	spin_lock_irqsave(&spool->lock, flags);
228 229 230 231

	if (spool->max_hpages != -1)		/* maximum size accounting */
		spool->used_hpages -= delta;

232 233
	 /* minimum size accounting */
	if (spool->min_hpages != -1 && spool->used_hpages < spool->min_hpages) {
234 235 236 237 238 239 240 241 242 243 244 245 246 247
		if (spool->rsv_hpages + delta <= spool->min_hpages)
			ret = 0;
		else
			ret = spool->rsv_hpages + delta - spool->min_hpages;

		spool->rsv_hpages += delta;
		if (spool->rsv_hpages > spool->min_hpages)
			spool->rsv_hpages = spool->min_hpages;
	}

	/*
	 * If hugetlbfs_put_super couldn't free spool due to an outstanding
	 * quota reference, free it now.
	 */
248
	unlock_or_release_subpool(spool, flags);
249 250

	return ret;
251 252 253 254 255 256 257 258 259
}

static inline struct hugepage_subpool *subpool_inode(struct inode *inode)
{
	return HUGETLBFS_SB(inode->i_sb)->spool;
}

static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma)
{
Al Viro's avatar
Al Viro committed
260
	return subpool_inode(file_inode(vma->vm_file));
261 262
}

263 264 265 266 267 268 269 270 271
/*
 * hugetlb vma_lock helper routines
 */
void hugetlb_vma_lock_read(struct vm_area_struct *vma)
{
	if (__vma_shareable_lock(vma)) {
		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;

		down_read(&vma_lock->rw_sema);
272 273 274 275
	} else if (__vma_private_lock(vma)) {
		struct resv_map *resv_map = vma_resv_map(vma);

		down_read(&resv_map->rw_sema);
276 277 278 279 280 281 282 283 284
	}
}

void hugetlb_vma_unlock_read(struct vm_area_struct *vma)
{
	if (__vma_shareable_lock(vma)) {
		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;

		up_read(&vma_lock->rw_sema);
285 286 287 288
	} else if (__vma_private_lock(vma)) {
		struct resv_map *resv_map = vma_resv_map(vma);

		up_read(&resv_map->rw_sema);
289 290 291 292 293 294 295 296 297
	}
}

void hugetlb_vma_lock_write(struct vm_area_struct *vma)
{
	if (__vma_shareable_lock(vma)) {
		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;

		down_write(&vma_lock->rw_sema);
298 299 300 301
	} else if (__vma_private_lock(vma)) {
		struct resv_map *resv_map = vma_resv_map(vma);

		down_write(&resv_map->rw_sema);
302 303 304 305 306 307 308 309 310
	}
}

void hugetlb_vma_unlock_write(struct vm_area_struct *vma)
{
	if (__vma_shareable_lock(vma)) {
		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;

		up_write(&vma_lock->rw_sema);
311 312 313 314
	} else if (__vma_private_lock(vma)) {
		struct resv_map *resv_map = vma_resv_map(vma);

		up_write(&resv_map->rw_sema);
315 316 317 318 319 320
	}
}

int hugetlb_vma_trylock_write(struct vm_area_struct *vma)
{

321 322
	if (__vma_shareable_lock(vma)) {
		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
323

324 325 326 327 328 329 330 331
		return down_write_trylock(&vma_lock->rw_sema);
	} else if (__vma_private_lock(vma)) {
		struct resv_map *resv_map = vma_resv_map(vma);

		return down_write_trylock(&resv_map->rw_sema);
	}

	return 1;
332 333 334 335 336 337 338 339
}

void hugetlb_vma_assert_locked(struct vm_area_struct *vma)
{
	if (__vma_shareable_lock(vma)) {
		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;

		lockdep_assert_held(&vma_lock->rw_sema);
340 341 342 343
	} else if (__vma_private_lock(vma)) {
		struct resv_map *resv_map = vma_resv_map(vma);

		lockdep_assert_held(&resv_map->rw_sema);
344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375
	}
}

void hugetlb_vma_lock_release(struct kref *kref)
{
	struct hugetlb_vma_lock *vma_lock = container_of(kref,
			struct hugetlb_vma_lock, refs);

	kfree(vma_lock);
}

static void __hugetlb_vma_unlock_write_put(struct hugetlb_vma_lock *vma_lock)
{
	struct vm_area_struct *vma = vma_lock->vma;

	/*
	 * vma_lock structure may or not be released as a result of put,
	 * it certainly will no longer be attached to vma so clear pointer.
	 * Semaphore synchronizes access to vma_lock->vma field.
	 */
	vma_lock->vma = NULL;
	vma->vm_private_data = NULL;
	up_write(&vma_lock->rw_sema);
	kref_put(&vma_lock->refs, hugetlb_vma_lock_release);
}

static void __hugetlb_vma_unlock_write_free(struct vm_area_struct *vma)
{
	if (__vma_shareable_lock(vma)) {
		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;

		__hugetlb_vma_unlock_write_put(vma_lock);
376 377 378 379 380
	} else if (__vma_private_lock(vma)) {
		struct resv_map *resv_map = vma_resv_map(vma);

		/* no free for anon vmas, but still need to unlock */
		up_write(&resv_map->rw_sema);
381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433
	}
}

static void hugetlb_vma_lock_free(struct vm_area_struct *vma)
{
	/*
	 * Only present in sharable vmas.
	 */
	if (!vma || !__vma_shareable_lock(vma))
		return;

	if (vma->vm_private_data) {
		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;

		down_write(&vma_lock->rw_sema);
		__hugetlb_vma_unlock_write_put(vma_lock);
	}
}

static void hugetlb_vma_lock_alloc(struct vm_area_struct *vma)
{
	struct hugetlb_vma_lock *vma_lock;

	/* Only establish in (flags) sharable vmas */
	if (!vma || !(vma->vm_flags & VM_MAYSHARE))
		return;

	/* Should never get here with non-NULL vm_private_data */
	if (vma->vm_private_data)
		return;

	vma_lock = kmalloc(sizeof(*vma_lock), GFP_KERNEL);
	if (!vma_lock) {
		/*
		 * If we can not allocate structure, then vma can not
		 * participate in pmd sharing.  This is only a possible
		 * performance enhancement and memory saving issue.
		 * However, the lock is also used to synchronize page
		 * faults with truncation.  If the lock is not present,
		 * unlikely races could leave pages in a file past i_size
		 * until the file is removed.  Warn in the unlikely case of
		 * allocation failure.
		 */
		pr_warn_once("HugeTLB: unable to allocate vma specific lock\n");
		return;
	}

	kref_init(&vma_lock->refs);
	init_rwsem(&vma_lock->rw_sema);
	vma_lock->vma = vma;
	vma->vm_private_data = vma_lock;
}

434 435 436 437 438 439
/* Helper that removes a struct file_region from the resv_map cache and returns
 * it for use.
 */
static struct file_region *
get_file_region_entry_from_cache(struct resv_map *resv, long from, long to)
{
440
	struct file_region *nrg;
441 442 443 444 445 446 447 448 449 450 451 452 453

	VM_BUG_ON(resv->region_cache_count <= 0);

	resv->region_cache_count--;
	nrg = list_first_entry(&resv->region_cache, struct file_region, link);
	list_del(&nrg->link);

	nrg->from = from;
	nrg->to = to;

	return nrg;
}

454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475
static void copy_hugetlb_cgroup_uncharge_info(struct file_region *nrg,
					      struct file_region *rg)
{
#ifdef CONFIG_CGROUP_HUGETLB
	nrg->reservation_counter = rg->reservation_counter;
	nrg->css = rg->css;
	if (rg->css)
		css_get(rg->css);
#endif
}

/* Helper that records hugetlb_cgroup uncharge info. */
static void record_hugetlb_cgroup_uncharge_info(struct hugetlb_cgroup *h_cg,
						struct hstate *h,
						struct resv_map *resv,
						struct file_region *nrg)
{
#ifdef CONFIG_CGROUP_HUGETLB
	if (h_cg) {
		nrg->reservation_counter =
			&h_cg->rsvd_hugepage[hstate_index(h)];
		nrg->css = &h_cg->css;
476 477 478 479 480 481 482 483 484 485 486
		/*
		 * The caller will hold exactly one h_cg->css reference for the
		 * whole contiguous reservation region. But this area might be
		 * scattered when there are already some file_regions reside in
		 * it. As a result, many file_regions may share only one css
		 * reference. In order to ensure that one file_region must hold
		 * exactly one h_cg->css reference, we should do css_get for
		 * each file_region and leave the reference held by caller
		 * untouched.
		 */
		css_get(&h_cg->css);
487 488 489 490 491 492 493 494 495 496 497 498 499
		if (!resv->pages_per_hpage)
			resv->pages_per_hpage = pages_per_huge_page(h);
		/* pages_per_hpage should be the same for all entries in
		 * a resv_map.
		 */
		VM_BUG_ON(resv->pages_per_hpage != pages_per_huge_page(h));
	} else {
		nrg->reservation_counter = NULL;
		nrg->css = NULL;
	}
#endif
}

500 501 502 503 504 505 506 507
static void put_uncharge_info(struct file_region *rg)
{
#ifdef CONFIG_CGROUP_HUGETLB
	if (rg->css)
		css_put(rg->css);
#endif
}

508 509 510 511
static bool has_same_uncharge_info(struct file_region *rg,
				   struct file_region *org)
{
#ifdef CONFIG_CGROUP_HUGETLB
512
	return rg->reservation_counter == org->reservation_counter &&
513 514 515 516 517 518 519 520 521
	       rg->css == org->css;

#else
	return true;
#endif
}

static void coalesce_file_region(struct resv_map *resv, struct file_region *rg)
{
522
	struct file_region *nrg, *prg;
523 524 525 526 527 528 529

	prg = list_prev_entry(rg, link);
	if (&prg->link != &resv->regions && prg->to == rg->from &&
	    has_same_uncharge_info(prg, rg)) {
		prg->to = rg->to;

		list_del(&rg->link);
530
		put_uncharge_info(rg);
531 532
		kfree(rg);

533
		rg = prg;
534 535 536 537 538 539 540 541
	}

	nrg = list_next_entry(rg, link);
	if (&nrg->link != &resv->regions && nrg->from == rg->to &&
	    has_same_uncharge_info(nrg, rg)) {
		nrg->from = rg->from;

		list_del(&rg->link);
542
		put_uncharge_info(rg);
543 544 545 546
		kfree(rg);
	}
}

547
static inline long
548
hugetlb_resv_map_add(struct resv_map *map, struct list_head *rg, long from,
549 550 551 552 553 554 555 556
		     long to, struct hstate *h, struct hugetlb_cgroup *cg,
		     long *regions_needed)
{
	struct file_region *nrg;

	if (!regions_needed) {
		nrg = get_file_region_entry_from_cache(map, from, to);
		record_hugetlb_cgroup_uncharge_info(cg, h, map, nrg);
557
		list_add(&nrg->link, rg);
558 559 560 561 562 563 564
		coalesce_file_region(map, nrg);
	} else
		*regions_needed += 1;

	return to - from;
}

565 566 567 568 569 570 571
/*
 * Must be called with resv->lock held.
 *
 * Calling this with regions_needed != NULL will count the number of pages
 * to be added but will not modify the linked list. And regions_needed will
 * indicate the number of file_regions needed in the cache to carry out to add
 * the regions for this range.
572 573
 */
static long add_reservation_in_range(struct resv_map *resv, long f, long t,
574
				     struct hugetlb_cgroup *h_cg,
575
				     struct hstate *h, long *regions_needed)
576
{
577
	long add = 0;
578
	struct list_head *head = &resv->regions;
579
	long last_accounted_offset = f;
580 581
	struct file_region *iter, *trg = NULL;
	struct list_head *rg = NULL;
582

583 584
	if (regions_needed)
		*regions_needed = 0;
585

586
	/* In this loop, we essentially handle an entry for the range
587
	 * [last_accounted_offset, iter->from), at every iteration, with some
588 589
	 * bounds checking.
	 */
590
	list_for_each_entry_safe(iter, trg, head, link) {
591
		/* Skip irrelevant regions that start before our range. */
592
		if (iter->from < f) {
593 594 595
			/* If this region ends after the last accounted offset,
			 * then we need to update last_accounted_offset.
			 */
596 597
			if (iter->to > last_accounted_offset)
				last_accounted_offset = iter->to;
598 599
			continue;
		}
600

601 602 603
		/* When we find a region that starts beyond our range, we've
		 * finished.
		 */
604 605
		if (iter->from >= t) {
			rg = iter->link.prev;
606
			break;
607
		}
608

609
		/* Add an entry for last_accounted_offset -> iter->from, and
610 611
		 * update last_accounted_offset.
		 */
612 613
		if (iter->from > last_accounted_offset)
			add += hugetlb_resv_map_add(resv, iter->link.prev,
614
						    last_accounted_offset,
615
						    iter->from, h, h_cg,
616
						    regions_needed);
617

618
		last_accounted_offset = iter->to;
619 620 621 622 623
	}

	/* Handle the case where our range extends beyond
	 * last_accounted_offset.
	 */
624 625
	if (!rg)
		rg = head->prev;
626 627 628
	if (last_accounted_offset < t)
		add += hugetlb_resv_map_add(resv, rg, last_accounted_offset,
					    t, h, h_cg, regions_needed);
629 630 631 632 633 634 635 636 637 638

	return add;
}

/* Must be called with resv->lock acquired. Will drop lock to allocate entries.
 */
static int allocate_file_region_entries(struct resv_map *resv,
					int regions_needed)
	__must_hold(&resv->lock)
{
639
	LIST_HEAD(allocated_regions);
640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659
	int to_allocate = 0, i = 0;
	struct file_region *trg = NULL, *rg = NULL;

	VM_BUG_ON(regions_needed < 0);

	/*
	 * Check for sufficient descriptors in the cache to accommodate
	 * the number of in progress add operations plus regions_needed.
	 *
	 * This is a while loop because when we drop the lock, some other call
	 * to region_add or region_del may have consumed some region_entries,
	 * so we keep looping here until we finally have enough entries for
	 * (adds_in_progress + regions_needed).
	 */
	while (resv->region_cache_count <
	       (resv->adds_in_progress + regions_needed)) {
		to_allocate = resv->adds_in_progress + regions_needed -
			      resv->region_cache_count;

		/* At this point, we should have enough entries in the cache
Ingo Molnar's avatar
Ingo Molnar committed
660
		 * for all the existing adds_in_progress. We should only be
661
		 * needing to allocate for regions_needed.
662
		 */
663 664 665 666 667 668 669 670
		VM_BUG_ON(resv->region_cache_count < resv->adds_in_progress);

		spin_unlock(&resv->lock);
		for (i = 0; i < to_allocate; i++) {
			trg = kmalloc(sizeof(*trg), GFP_KERNEL);
			if (!trg)
				goto out_of_memory;
			list_add(&trg->link, &allocated_regions);
671 672
		}

673 674
		spin_lock(&resv->lock);

675 676
		list_splice(&allocated_regions, &resv->region_cache);
		resv->region_cache_count += to_allocate;
677 678
	}

679
	return 0;
680

681 682 683 684 685 686
out_of_memory:
	list_for_each_entry_safe(rg, trg, &allocated_regions, link) {
		list_del(&rg->link);
		kfree(rg);
	}
	return -ENOMEM;
687 688
}

689 690
/*
 * Add the huge page range represented by [f, t) to the reserve
691 692 693 694 695
 * map.  Regions will be taken from the cache to fill in this range.
 * Sufficient regions should exist in the cache due to the previous
 * call to region_chg with the same range, but in some cases the cache will not
 * have sufficient entries due to races with other code doing region_add or
 * region_del.  The extra needed entries will be allocated.
696
 *
697 698 699 700
 * regions_needed is the out value provided by a previous call to region_chg.
 *
 * Return the number of new huge pages added to the map.  This number is greater
 * than or equal to zero.  If file_region entries needed to be allocated for
701
 * this operation and we were not able to allocate, it returns -ENOMEM.
702 703 704
 * region_add of regions of length 1 never allocate file_regions and cannot
 * fail; region_chg will always allocate at least 1 entry and a region_add for
 * 1 page will only require at most 1 entry.
705
 */
706
static long region_add(struct resv_map *resv, long f, long t,
707 708
		       long in_regions_needed, struct hstate *h,
		       struct hugetlb_cgroup *h_cg)
709
{
710
	long add = 0, actual_regions_needed = 0;
711

712
	spin_lock(&resv->lock);
713 714 715
retry:

	/* Count how many regions are actually needed to execute this add. */
716 717
	add_reservation_in_range(resv, f, t, NULL, NULL,
				 &actual_regions_needed);
718

719
	/*
720 721 722 723 724 725 726
	 * Check for sufficient descriptors in the cache to accommodate
	 * this add operation. Note that actual_regions_needed may be greater
	 * than in_regions_needed, as the resv_map may have been modified since
	 * the region_chg call. In this case, we need to make sure that we
	 * allocate extra entries, such that we have enough for all the
	 * existing adds_in_progress, plus the excess needed for this
	 * operation.
727
	 */
728 729 730 731 732 733 734 735
	if (actual_regions_needed > in_regions_needed &&
	    resv->region_cache_count <
		    resv->adds_in_progress +
			    (actual_regions_needed - in_regions_needed)) {
		/* region_add operation of range 1 should never need to
		 * allocate file_region entries.
		 */
		VM_BUG_ON(t - f <= 1);
736

737 738 739 740
		if (allocate_file_region_entries(
			    resv, actual_regions_needed - in_regions_needed)) {
			return -ENOMEM;
		}
741

742
		goto retry;
743 744
	}

745
	add = add_reservation_in_range(resv, f, t, h_cg, h, NULL);
746 747

	resv->adds_in_progress -= in_regions_needed;
748

749
	spin_unlock(&resv->lock);
750
	return add;
751 752
}

753 754 755 756 757 758 759
/*
 * Examine the existing reserve map and determine how many
 * huge pages in the specified range [f, t) are NOT currently
 * represented.  This routine is called before a subsequent
 * call to region_add that will actually modify the reserve
 * map to add the specified range [f, t).  region_chg does
 * not change the number of huge pages represented by the
760 761 762 763 764 765 766
 * map.  A number of new file_region structures is added to the cache as a
 * placeholder, for the subsequent region_add call to use. At least 1
 * file_region structure is added.
 *
 * out_regions_needed is the number of regions added to the
 * resv->adds_in_progress.  This value needs to be provided to a follow up call
 * to region_add or region_abort for proper accounting.
767 768 769 770 771
 *
 * Returns the number of huge pages that need to be added to the existing
 * reservation map for the range [f, t).  This number is greater or equal to
 * zero.  -ENOMEM is returned if a new file_region structure or cache entry
 * is needed and can not be allocated.
772
 */
773 774
static long region_chg(struct resv_map *resv, long f, long t,
		       long *out_regions_needed)
775 776 777
{
	long chg = 0;

778
	spin_lock(&resv->lock);
779

780
	/* Count how many hugepages in this range are NOT represented. */
781
	chg = add_reservation_in_range(resv, f, t, NULL, NULL,
782
				       out_regions_needed);
783

784 785
	if (*out_regions_needed == 0)
		*out_regions_needed = 1;
786

787 788
	if (allocate_file_region_entries(resv, *out_regions_needed))
		return -ENOMEM;
789

790
	resv->adds_in_progress += *out_regions_needed;
791 792

	spin_unlock(&resv->lock);
793 794 795
	return chg;
}

796 797 798 799 800
/*
 * Abort the in progress add operation.  The adds_in_progress field
 * of the resv_map keeps track of the operations in progress between
 * calls to region_chg and region_add.  Operations are sometimes
 * aborted after the call to region_chg.  In such cases, region_abort
801 802 803
 * is called to decrement the adds_in_progress counter. regions_needed
 * is the value returned by the region_chg call, it is used to decrement
 * the adds_in_progress counter.
804 805 806 807 808
 *
 * NOTE: The range arguments [f, t) are not needed or used in this
 * routine.  They are kept to make reading the calling code easier as
 * arguments will match the associated region_chg call.
 */
809 810
static void region_abort(struct resv_map *resv, long f, long t,
			 long regions_needed)
811 812 813
{
	spin_lock(&resv->lock);
	VM_BUG_ON(!resv->region_cache_count);
814
	resv->adds_in_progress -= regions_needed;
815 816 817
	spin_unlock(&resv->lock);
}

818
/*
819 820 821 822 823 824 825 826 827 828 829 830
 * Delete the specified range [f, t) from the reserve map.  If the
 * t parameter is LONG_MAX, this indicates that ALL regions after f
 * should be deleted.  Locate the regions which intersect [f, t)
 * and either trim, delete or split the existing regions.
 *
 * Returns the number of huge pages deleted from the reserve map.
 * In the normal case, the return value is zero or more.  In the
 * case where a region must be split, a new region descriptor must
 * be allocated.  If the allocation fails, -ENOMEM will be returned.
 * NOTE: If the parameter t == LONG_MAX, then we will never split
 * a region and possibly return -ENOMEM.  Callers specifying
 * t == LONG_MAX do not need to check for -ENOMEM error.
831
 */
832
static long region_del(struct resv_map *resv, long f, long t)
833
{
834
	struct list_head *head = &resv->regions;
835
	struct file_region *rg, *trg;
836 837
	struct file_region *nrg = NULL;
	long del = 0;
838

839
retry:
840
	spin_lock(&resv->lock);
841
	list_for_each_entry_safe(rg, trg, head, link) {
842 843 844 845 846 847 848 849
		/*
		 * Skip regions before the range to be deleted.  file_region
		 * ranges are normally of the form [from, to).  However, there
		 * may be a "placeholder" entry in the map which is of the form
		 * (from, to) with from == to.  Check for placeholder entries
		 * at the beginning of the range to be deleted.
		 */
		if (rg->to <= f && (rg->to != rg->from || rg->to != f))
850
			continue;
851

852
		if (rg->from >= t)
853 854
			break;

855 856 857 858 859 860 861 862 863 864 865 866 867
		if (f > rg->from && t < rg->to) { /* Must split region */
			/*
			 * Check for an entry in the cache before dropping
			 * lock and attempting allocation.
			 */
			if (!nrg &&
			    resv->region_cache_count > resv->adds_in_progress) {
				nrg = list_first_entry(&resv->region_cache,
							struct file_region,
							link);
				list_del(&nrg->link);
				resv->region_cache_count--;
			}
868

869 870 871 872 873 874 875 876 877
			if (!nrg) {
				spin_unlock(&resv->lock);
				nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
				if (!nrg)
					return -ENOMEM;
				goto retry;
			}

			del += t - f;
878
			hugetlb_cgroup_uncharge_file_region(
879
				resv, rg, t - f, false);
880 881 882 883

			/* New entry for end of split region */
			nrg->from = t;
			nrg->to = rg->to;
884 885 886

			copy_hugetlb_cgroup_uncharge_info(nrg, rg);

887 888 889 890 891 892 893
			INIT_LIST_HEAD(&nrg->link);

			/* Original entry is trimmed */
			rg->to = f;

			list_add(&nrg->link, &rg->link);
			nrg = NULL;
894
			break;
895 896 897 898
		}

		if (f <= rg->from && t >= rg->to) { /* Remove entire region */
			del += rg->to - rg->from;
899
			hugetlb_cgroup_uncharge_file_region(resv, rg,
900
							    rg->to - rg->from, true);
901 902 903 904 905 906
			list_del(&rg->link);
			kfree(rg);
			continue;
		}

		if (f <= rg->from) {	/* Trim beginning of region */
907
			hugetlb_cgroup_uncharge_file_region(resv, rg,
908
							    t - rg->from, false);
909

910 911 912
			del += t - rg->from;
			rg->from = t;
		} else {		/* Trim end of region */
913
			hugetlb_cgroup_uncharge_file_region(resv, rg,
914
							    rg->to - f, false);
915 916 917

			del += rg->to - f;
			rg->to = f;
918
		}
919
	}
920 921

	spin_unlock(&resv->lock);
922 923
	kfree(nrg);
	return del;
924 925
}

926 927 928 929 930 931 932 933 934
/*
 * A rare out of memory error was encountered which prevented removal of
 * the reserve map region for a page.  The huge page itself was free'ed
 * and removed from the page cache.  This routine will adjust the subpool
 * usage count, and the global reserve count if needed.  By incrementing
 * these counts, the reserve map entry which could not be deleted will
 * appear as a "reserved" entry instead of simply dangling with incorrect
 * counts.
 */
935
void hugetlb_fix_reserve_counts(struct inode *inode)
936 937 938
{
	struct hugepage_subpool *spool = subpool_inode(inode);
	long rsv_adjust;
939
	bool reserved = false;
940 941

	rsv_adjust = hugepage_subpool_get_pages(spool, 1);
942
	if (rsv_adjust > 0) {
943 944
		struct hstate *h = hstate_inode(inode);

945 946 947 948
		if (!hugetlb_acct_memory(h, 1))
			reserved = true;
	} else if (!rsv_adjust) {
		reserved = true;
949
	}
950 951 952

	if (!reserved)
		pr_warn("hugetlb: Huge Page Reserved count may go negative.\n");
953 954
}

955 956 957 958
/*
 * Count and return the number of huge pages in the reserve map
 * that intersect with the range [f, t).
 */
959
static long region_count(struct resv_map *resv, long f, long t)
960
{
961
	struct list_head *head = &resv->regions;
962 963 964
	struct file_region *rg;
	long chg = 0;

965
	spin_lock(&resv->lock);
966 967
	/* Locate each segment we overlap with, and count that overlap. */
	list_for_each_entry(rg, head, link) {
968 969
		long seg_from;
		long seg_to;
970 971 972 973 974 975 976 977 978 979 980

		if (rg->to <= f)
			continue;
		if (rg->from >= t)
			break;

		seg_from = max(rg->from, f);
		seg_to = min(rg->to, t);

		chg += seg_to - seg_from;
	}
981
	spin_unlock(&resv->lock);
982 983 984 985

	return chg;
}

986 987
/*
 * Convert the address within this vma to the page offset within
988
 * the mapping, huge page units here.
989
 */
990 991
static pgoff_t vma_hugecache_offset(struct hstate *h,
			struct vm_area_struct *vma, unsigned long address)
992
{
993 994
	return ((address - vma->vm_start) >> huge_page_shift(h)) +
			(vma->vm_pgoff >> huge_page_order(h));
995 996
}

997 998 999 1000 1001 1002 1003 1004
/**
 * vma_kernel_pagesize - Page size granularity for this VMA.
 * @vma: The user mapping.
 *
 * Folios in this VMA will be aligned to, and at least the size of the
 * number of bytes returned by this function.
 *
 * Return: The default size of the folios allocated when backing a VMA.
1005 1006 1007
 */
unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
{
1008 1009 1010
	if (vma->vm_ops && vma->vm_ops->pagesize)
		return vma->vm_ops->pagesize(vma);
	return PAGE_SIZE;
1011
}
1012
EXPORT_SYMBOL_GPL(vma_kernel_pagesize);
1013

1014 1015 1016
/*
 * Return the page size being used by the MMU to back a VMA. In the majority
 * of cases, the page size used by the kernel matches the MMU size. On
1017 1018
 * architectures where it differs, an architecture-specific 'strong'
 * version of this symbol is required.
1019
 */
1020
__weak unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
1021 1022 1023 1024
{
	return vma_kernel_pagesize(vma);
}

1025 1026 1027 1028 1029 1030 1031
/*
 * Flags for MAP_PRIVATE reservations.  These are stored in the bottom
 * bits of the reservation map pointer, which are always clear due to
 * alignment.
 */
#define HPAGE_RESV_OWNER    (1UL << 0)
#define HPAGE_RESV_UNMAPPED (1UL << 1)
1032
#define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)
1033

1034 1035 1036 1037 1038
/*
 * These helpers are used to track how many pages are reserved for
 * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
 * is guaranteed to have their future faults succeed.
 *
1039
 * With the exception of hugetlb_dup_vma_private() which is called at fork(),
1040 1041 1042
 * the reserve counters are updated with the hugetlb_lock held. It is safe
 * to reset the VMA at fork() time as it is not in use yet and there is no
 * chance of the global counters getting corrupted as a result of the values.
1043 1044 1045 1046 1047 1048 1049 1050 1051
 *
 * The private mapping reservation is represented in a subtly different
 * manner to a shared mapping.  A shared mapping has a region map associated
 * with the underlying file, this region map represents the backing file
 * pages which have ever had a reservation assigned which this persists even
 * after the page is instantiated.  A private mapping has a region map
 * associated with the original mmap which is attached to all VMAs which
 * reference it, this region map represents those offsets which have consumed
 * reservation ie. where pages have been instantiated.
1052
 */
1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063
static unsigned long get_vma_private_data(struct vm_area_struct *vma)
{
	return (unsigned long)vma->vm_private_data;
}

static void set_vma_private_data(struct vm_area_struct *vma,
							unsigned long value)
{
	vma->vm_private_data = (void *)value;
}

1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082
static void
resv_map_set_hugetlb_cgroup_uncharge_info(struct resv_map *resv_map,
					  struct hugetlb_cgroup *h_cg,
					  struct hstate *h)
{
#ifdef CONFIG_CGROUP_HUGETLB
	if (!h_cg || !h) {
		resv_map->reservation_counter = NULL;
		resv_map->pages_per_hpage = 0;
		resv_map->css = NULL;
	} else {
		resv_map->reservation_counter =
			&h_cg->rsvd_hugepage[hstate_index(h)];
		resv_map->pages_per_hpage = pages_per_huge_page(h);
		resv_map->css = &h_cg->css;
	}
#endif
}

1083
struct resv_map *resv_map_alloc(void)
1084 1085
{
	struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL);
1086 1087 1088 1089 1090
	struct file_region *rg = kmalloc(sizeof(*rg), GFP_KERNEL);

	if (!resv_map || !rg) {
		kfree(resv_map);
		kfree(rg);
1091
		return NULL;
1092
	}
1093 1094

	kref_init(&resv_map->refs);
1095
	spin_lock_init(&resv_map->lock);
1096
	INIT_LIST_HEAD(&resv_map->regions);
1097
	init_rwsem(&resv_map->rw_sema);
1098

1099
	resv_map->adds_in_progress = 0;
1100 1101 1102 1103 1104 1105 1106
	/*
	 * Initialize these to 0. On shared mappings, 0's here indicate these
	 * fields don't do cgroup accounting. On private mappings, these will be
	 * re-initialized to the proper values, to indicate that hugetlb cgroup
	 * reservations are to be un-charged from here.
	 */
	resv_map_set_hugetlb_cgroup_uncharge_info(resv_map, NULL, NULL);
1107 1108 1109 1110 1111

	INIT_LIST_HEAD(&resv_map->region_cache);
	list_add(&rg->link, &resv_map->region_cache);
	resv_map->region_cache_count = 1;

1112 1113 1114
	return resv_map;
}

1115
void resv_map_release(struct kref *ref)
1116 1117
{
	struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
1118 1119
	struct list_head *head = &resv_map->region_cache;
	struct file_region *rg, *trg;
1120 1121

	/* Clear out any active regions before we release the map. */
1122
	region_del(resv_map, 0, LONG_MAX);
1123 1124 1125 1126 1127 1128 1129 1130 1131

	/* ... and any entries left in the cache */
	list_for_each_entry_safe(rg, trg, head, link) {
		list_del(&rg->link);
		kfree(rg);
	}

	VM_BUG_ON(resv_map->adds_in_progress);

1132 1133 1134
	kfree(resv_map);
}

1135 1136
static inline struct resv_map *inode_resv_map(struct inode *inode)
{
1137 1138 1139 1140 1141 1142 1143 1144
	/*
	 * At inode evict time, i_mapping may not point to the original
	 * address space within the inode.  This original address space
	 * contains the pointer to the resv_map.  So, always use the
	 * address space embedded within the inode.
	 * The VERY common case is inode->mapping == &inode->i_data but,
	 * this may not be true for device special inodes.
	 */
1145
	return (struct resv_map *)(&inode->i_data)->i_private_data;
1146 1147
}

1148
static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
1149
{
1150
	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
1151 1152 1153 1154 1155 1156 1157
	if (vma->vm_flags & VM_MAYSHARE) {
		struct address_space *mapping = vma->vm_file->f_mapping;
		struct inode *inode = mapping->host;

		return inode_resv_map(inode);

	} else {
1158 1159
		return (struct resv_map *)(get_vma_private_data(vma) &
							~HPAGE_RESV_MASK);
1160
	}
1161 1162
}

1163
static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
1164
{
1165 1166
	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
	VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
1167

1168
	set_vma_private_data(vma, (unsigned long)map);
1169 1170 1171 1172
}

static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
{
1173 1174
	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
	VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
1175 1176

	set_vma_private_data(vma, get_vma_private_data(vma) | flags);
1177 1178 1179 1180
}

static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
{
1181
	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
1182 1183

	return (get_vma_private_data(vma) & flag) != 0;
1184 1185
}

1186 1187 1188 1189 1190 1191 1192
bool __vma_private_lock(struct vm_area_struct *vma)
{
	return !(vma->vm_flags & VM_MAYSHARE) &&
		get_vma_private_data(vma) & ~HPAGE_RESV_MASK &&
		is_vma_resv_set(vma, HPAGE_RESV_OWNER);
}

1193
void hugetlb_dup_vma_private(struct vm_area_struct *vma)
1194
{
1195
	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
1196 1197
	/*
	 * Clear vm_private_data
1198 1199 1200 1201 1202 1203
	 * - For shared mappings this is a per-vma semaphore that may be
	 *   allocated in a subsequent call to hugetlb_vm_op_open.
	 *   Before clearing, make sure pointer is not associated with vma
	 *   as this will leak the structure.  This is the case when called
	 *   via clear_vma_resv_huge_pages() and hugetlb_vm_op_open has already
	 *   been called to allocate a new structure.
1204 1205 1206 1207
	 * - For MAP_PRIVATE mappings, this is the reserve map which does
	 *   not apply to children.  Faults generated by the children are
	 *   not guaranteed to succeed, even if read-only.
	 */
1208 1209 1210 1211 1212 1213 1214
	if (vma->vm_flags & VM_MAYSHARE) {
		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;

		if (vma_lock && vma_lock->vma != vma)
			vma->vm_private_data = NULL;
	} else
		vma->vm_private_data = NULL;
1215 1216
}

1217 1218
/*
 * Reset and decrement one ref on hugepage private reservation.
1219
 * Called with mm->mmap_lock writer semaphore held.
1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239
 * This function should be only used by move_vma() and operate on
 * same sized vma. It should never come here with last ref on the
 * reservation.
 */
void clear_vma_resv_huge_pages(struct vm_area_struct *vma)
{
	/*
	 * Clear the old hugetlb private page reservation.
	 * It has already been transferred to new_vma.
	 *
	 * During a mremap() operation of a hugetlb vma we call move_vma()
	 * which copies vma into new_vma and unmaps vma. After the copy
	 * operation both new_vma and vma share a reference to the resv_map
	 * struct, and at that point vma is about to be unmapped. We don't
	 * want to return the reservation to the pool at unmap of vma because
	 * the reservation still lives on in new_vma, so simply decrement the
	 * ref here and remove the resv_map reference from this vma.
	 */
	struct resv_map *reservations = vma_resv_map(vma);

1240 1241
	if (reservations && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
		resv_map_put_hugetlb_cgroup_uncharge_info(reservations);
1242
		kref_put(&reservations->refs, resv_map_release);
1243
	}
1244

1245
	hugetlb_dup_vma_private(vma);
1246 1247
}

1248
/* Returns true if the VMA has associated reserve pages */
1249
static bool vma_has_reserves(struct vm_area_struct *vma, long chg)
1250
{
1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261
	if (vma->vm_flags & VM_NORESERVE) {
		/*
		 * This address is already reserved by other process(chg == 0),
		 * so, we should decrement reserved count. Without decrementing,
		 * reserve count remains after releasing inode, because this
		 * allocated page will go into page cache and is regarded as
		 * coming from reserved pool in releasing step.  Currently, we
		 * don't have any other solution to deal with this situation
		 * properly, so add work-around here.
		 */
		if (vma->vm_flags & VM_MAYSHARE && chg == 0)
1262
			return true;
1263
		else
1264
			return false;
1265
	}
1266 1267

	/* Shared mappings always use reserves */
1268 1269 1270 1271 1272
	if (vma->vm_flags & VM_MAYSHARE) {
		/*
		 * We know VM_NORESERVE is not set.  Therefore, there SHOULD
		 * be a region map for all pages.  The only situation where
		 * there is no region map is if a hole was punched via
1273
		 * fallocate.  In this case, there really are no reserves to
1274 1275 1276 1277 1278 1279 1280
		 * use.  This situation is indicated if chg != 0.
		 */
		if (chg)
			return false;
		else
			return true;
	}
1281 1282 1283 1284 1285

	/*
	 * Only the process that called mmap() has reserves for
	 * private mappings.
	 */
1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306
	if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
		/*
		 * Like the shared case above, a hole punch or truncate
		 * could have been performed on the private mapping.
		 * Examine the value of chg to determine if reserves
		 * actually exist or were previously consumed.
		 * Very Subtle - The value of chg comes from a previous
		 * call to vma_needs_reserves().  The reserve map for
		 * private mappings has different (opposite) semantics
		 * than that of shared mappings.  vma_needs_reserves()
		 * has already taken this difference in semantics into
		 * account.  Therefore, the meaning of chg is the same
		 * as in the shared case above.  Code could easily be
		 * combined, but keeping it separate draws attention to
		 * subtle differences.
		 */
		if (chg)
			return false;
		else
			return true;
	}
1307

1308
	return false;
1309 1310
}

1311
static void enqueue_hugetlb_folio(struct hstate *h, struct folio *folio)
Linus Torvalds's avatar
Linus Torvalds committed
1312
{
1313
	int nid = folio_nid(folio);
1314 1315

	lockdep_assert_held(&hugetlb_lock);
1316
	VM_BUG_ON_FOLIO(folio_ref_count(folio), folio);
1317

1318
	list_move(&folio->lru, &h->hugepage_freelists[nid]);
1319 1320
	h->free_huge_pages++;
	h->free_huge_pages_node[nid]++;
1321
	folio_set_hugetlb_freed(folio);
Linus Torvalds's avatar
Linus Torvalds committed
1322 1323
}

1324 1325
static struct folio *dequeue_hugetlb_folio_node_exact(struct hstate *h,
								int nid)
1326
{
1327
	struct folio *folio;
1328
	bool pin = !!(current->flags & PF_MEMALLOC_PIN);
1329

1330
	lockdep_assert_held(&hugetlb_lock);
1331 1332
	list_for_each_entry(folio, &h->hugepage_freelists[nid], lru) {
		if (pin && !folio_is_longterm_pinnable(folio))
1333
			continue;
1334

1335
		if (folio_test_hwpoison(folio))
1336 1337
			continue;

1338 1339 1340
		list_move(&folio->lru, &h->hugepage_activelist);
		folio_ref_unfreeze(folio, 1);
		folio_clear_hugetlb_freed(folio);
1341 1342
		h->free_huge_pages--;
		h->free_huge_pages_node[nid]--;
1343
		return folio;
1344 1345
	}

1346
	return NULL;
1347 1348
}

1349 1350
static struct folio *dequeue_hugetlb_folio_nodemask(struct hstate *h, gfp_t gfp_mask,
							int nid, nodemask_t *nmask)
1351
{
1352 1353 1354 1355
	unsigned int cpuset_mems_cookie;
	struct zonelist *zonelist;
	struct zone *zone;
	struct zoneref *z;
1356
	int node = NUMA_NO_NODE;
1357

1358 1359 1360 1361 1362
	zonelist = node_zonelist(nid, gfp_mask);

retry_cpuset:
	cpuset_mems_cookie = read_mems_allowed_begin();
	for_each_zone_zonelist_nodemask(zone, z, zonelist, gfp_zone(gfp_mask), nmask) {
1363
		struct folio *folio;
1364 1365 1366 1367 1368 1369 1370 1371 1372 1373

		if (!cpuset_zone_allowed(zone, gfp_mask))
			continue;
		/*
		 * no need to ask again on the same node. Pool is node rather than
		 * zone aware
		 */
		if (zone_to_nid(zone) == node)
			continue;
		node = zone_to_nid(zone);
1374

1375 1376 1377
		folio = dequeue_hugetlb_folio_node_exact(h, node);
		if (folio)
			return folio;
1378
	}
1379 1380 1381
	if (unlikely(read_mems_allowed_retry(cpuset_mems_cookie)))
		goto retry_cpuset;

1382 1383 1384
	return NULL;
}

1385 1386 1387 1388 1389
static unsigned long available_huge_pages(struct hstate *h)
{
	return h->free_huge_pages - h->resv_huge_pages;
}

1390
static struct folio *dequeue_hugetlb_folio_vma(struct hstate *h,
1391
				struct vm_area_struct *vma,
1392 1393
				unsigned long address, int avoid_reserve,
				long chg)
Linus Torvalds's avatar
Linus Torvalds committed
1394
{
1395
	struct folio *folio = NULL;
1396
	struct mempolicy *mpol;
1397
	gfp_t gfp_mask;
1398
	nodemask_t *nodemask;
1399
	int nid;
Linus Torvalds's avatar
Linus Torvalds committed
1400

1401 1402 1403 1404 1405
	/*
	 * A child process with MAP_PRIVATE mappings created by their parent
	 * have no page reserves. This check ensures that reservations are
	 * not "stolen". The child may still get SIGKILLed
	 */
1406
	if (!vma_has_reserves(vma, chg) && !available_huge_pages(h))
1407
		goto err;
1408

1409
	/* If reserves cannot be used, ensure enough pages are in the pool */
1410
	if (avoid_reserve && !available_huge_pages(h))
1411
		goto err;
1412

1413 1414
	gfp_mask = htlb_alloc_mask(h);
	nid = huge_node(vma, address, gfp_mask, &mpol, &nodemask);
1415 1416

	if (mpol_is_preferred_many(mpol)) {
1417 1418
		folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask,
							nid, nodemask);
1419 1420 1421 1422 1423

		/* Fallback to all nodes if page==NULL */
		nodemask = NULL;
	}

1424 1425 1426
	if (!folio)
		folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask,
							nid, nodemask);
1427

1428 1429
	if (folio && !avoid_reserve && vma_has_reserves(vma, chg)) {
		folio_set_hugetlb_restore_reserve(folio);
1430
		h->resv_huge_pages--;
Linus Torvalds's avatar
Linus Torvalds committed
1431
	}
1432

1433
	mpol_cond_put(mpol);
1434
	return folio;
1435 1436 1437

err:
	return NULL;
Linus Torvalds's avatar
Linus Torvalds committed
1438 1439
}

1440 1441 1442 1443 1444 1445 1446 1447 1448
/*
 * common helper functions for hstate_next_node_to_{alloc|free}.
 * We may have allocated or freed a huge page based on a different
 * nodes_allowed previously, so h->next_node_to_{alloc|free} might
 * be outside of *nodes_allowed.  Ensure that we use an allowed
 * node for alloc or free.
 */
static int next_node_allowed(int nid, nodemask_t *nodes_allowed)
{
1449
	nid = next_node_in(nid, *nodes_allowed);
1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467
	VM_BUG_ON(nid >= MAX_NUMNODES);

	return nid;
}

static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)
{
	if (!node_isset(nid, *nodes_allowed))
		nid = next_node_allowed(nid, nodes_allowed);
	return nid;
}

/*
 * returns the previously saved node ["this node"] from which to
 * allocate a persistent huge page for the pool and advance the
 * next node from which to allocate, handling wrap at end of node
 * mask.
 */
1468
static int hstate_next_node_to_alloc(int *next_node,
1469 1470 1471 1472 1473 1474
					nodemask_t *nodes_allowed)
{
	int nid;

	VM_BUG_ON(!nodes_allowed);

1475 1476
	nid = get_valid_node_allowed(*next_node, nodes_allowed);
	*next_node = next_node_allowed(nid, nodes_allowed);
1477 1478 1479 1480 1481

	return nid;
}

/*
1482
 * helper for remove_pool_hugetlb_folio() - return the previously saved
1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498
 * node ["this node"] from which to free a huge page.  Advance the
 * next node id whether or not we find a free huge page to free so
 * that the next attempt to free addresses the next node.
 */
static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed)
{
	int nid;

	VM_BUG_ON(!nodes_allowed);

	nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);
	h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);

	return nid;
}

1499
#define for_each_node_mask_to_alloc(next_node, nr_nodes, node, mask)		\
1500 1501
	for (nr_nodes = nodes_weight(*mask);				\
		nr_nodes > 0 &&						\
1502
		((node = hstate_next_node_to_alloc(next_node, mask)) || 1);	\
1503 1504 1505 1506 1507 1508 1509 1510
		nr_nodes--)

#define for_each_node_mask_to_free(hs, nr_nodes, node, mask)		\
	for (nr_nodes = nodes_weight(*mask);				\
		nr_nodes > 0 &&						\
		((node = hstate_next_node_to_free(hs, mask)) || 1);	\
		nr_nodes--)

1511
/* used to demote non-gigantic_huge pages as well */
1512
static void __destroy_compound_gigantic_folio(struct folio *folio,
1513
					unsigned int order, bool demote)
1514 1515 1516
{
	int i;
	int nr_pages = 1 << order;
1517
	struct page *p;
1518

1519
	atomic_set(&folio->_entire_mapcount, 0);
1520
	atomic_set(&folio->_nr_pages_mapped, 0);
1521
	atomic_set(&folio->_pincount, 0);
1522

1523
	for (i = 1; i < nr_pages; i++) {
1524
		p = folio_page(folio, i);
1525
		p->flags &= ~PAGE_FLAGS_CHECK_AT_FREE;
1526
		p->mapping = NULL;
1527
		clear_compound_head(p);
1528 1529
		if (!demote)
			set_page_refcounted(p);
1530 1531
	}

1532
	__folio_clear_head(folio);
1533 1534
}

1535
static void destroy_compound_hugetlb_folio_for_demote(struct folio *folio,
1536 1537
					unsigned int order)
{
1538
	__destroy_compound_gigantic_folio(folio, order, true);
1539 1540 1541
}

#ifdef CONFIG_ARCH_HAS_GIGANTIC_PAGE
1542
static void destroy_compound_gigantic_folio(struct folio *folio,
1543 1544
					unsigned int order)
{
1545
	__destroy_compound_gigantic_folio(folio, order, false);
1546 1547
}

1548
static void free_gigantic_folio(struct folio *folio, unsigned int order)
1549
{
1550 1551 1552 1553
	/*
	 * If the page isn't allocated using the cma allocator,
	 * cma_release() returns false.
	 */
1554
#ifdef CONFIG_CMA
1555 1556 1557
	int nid = folio_nid(folio);

	if (cma_release(hugetlb_cma[nid], &folio->page, 1 << order))
1558
		return;
1559
#endif
1560

1561
	free_contig_range(folio_pfn(folio), 1 << order);
1562 1563
}

1564
#ifdef CONFIG_CONTIG_ALLOC
1565
static struct folio *alloc_gigantic_folio(struct hstate *h, gfp_t gfp_mask,
1566
		int nid, nodemask_t *nodemask)
1567
{
1568
	struct page *page;
1569
	unsigned long nr_pages = pages_per_huge_page(h);
1570 1571
	if (nid == NUMA_NO_NODE)
		nid = numa_mem_id();
1572

1573 1574
#ifdef CONFIG_CMA
	{
1575 1576
		int node;

1577 1578 1579
		if (hugetlb_cma[nid]) {
			page = cma_alloc(hugetlb_cma[nid], nr_pages,
					huge_page_order(h), true);
1580
			if (page)
1581
				return page_folio(page);
1582
		}
1583 1584 1585 1586 1587 1588 1589 1590 1591

		if (!(gfp_mask & __GFP_THISNODE)) {
			for_each_node_mask(node, *nodemask) {
				if (node == nid || !hugetlb_cma[node])
					continue;

				page = cma_alloc(hugetlb_cma[node], nr_pages,
						huge_page_order(h), true);
				if (page)
1592
					return page_folio(page);
1593 1594
			}
		}
1595
	}
1596
#endif
1597

1598 1599
	page = alloc_contig_pages(nr_pages, gfp_mask, nid, nodemask);
	return page ? page_folio(page) : NULL;
1600 1601
}

1602
#else /* !CONFIG_CONTIG_ALLOC */
1603
static struct folio *alloc_gigantic_folio(struct hstate *h, gfp_t gfp_mask,
1604 1605 1606 1607 1608
					int nid, nodemask_t *nodemask)
{
	return NULL;
}
#endif /* CONFIG_CONTIG_ALLOC */
1609

1610
#else /* !CONFIG_ARCH_HAS_GIGANTIC_PAGE */
1611
static struct folio *alloc_gigantic_folio(struct hstate *h, gfp_t gfp_mask,
1612 1613 1614 1615
					int nid, nodemask_t *nodemask)
{
	return NULL;
}
1616 1617
static inline void free_gigantic_folio(struct folio *folio,
						unsigned int order) { }
1618
static inline void destroy_compound_gigantic_folio(struct folio *folio,
1619
						unsigned int order) { }
1620 1621
#endif

1622
/*
1623
 * Remove hugetlb folio from lists.
1624 1625 1626
 * If vmemmap exists for the folio, clear the hugetlb flag so that the
 * folio appears as just a compound page.  Otherwise, wait until after
 * allocating vmemmap to clear the flag.
1627
 *
1628
 * A reference is held on the folio, except in the case of demote.
1629 1630 1631
 *
 * Must be called with hugetlb lock held.
 */
1632
static void __remove_hugetlb_folio(struct hstate *h, struct folio *folio,
1633 1634
							bool adjust_surplus,
							bool demote)
1635
{
1636
	int nid = folio_nid(folio);
1637

1638 1639
	VM_BUG_ON_FOLIO(hugetlb_cgroup_from_folio(folio), folio);
	VM_BUG_ON_FOLIO(hugetlb_cgroup_from_folio_rsvd(folio), folio);
1640

1641
	lockdep_assert_held(&hugetlb_lock);
1642 1643 1644
	if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
		return;

1645
	list_del(&folio->lru);
1646

1647
	if (folio_test_hugetlb_freed(folio)) {
1648 1649 1650 1651 1652 1653 1654 1655
		h->free_huge_pages--;
		h->free_huge_pages_node[nid]--;
	}
	if (adjust_surplus) {
		h->surplus_huge_pages--;
		h->surplus_huge_pages_node[nid]--;
	}

1656
	/*
1657
	 * We can only clear the hugetlb flag after allocating vmemmap
1658 1659 1660 1661
	 * pages.  Otherwise, someone (memory error handling) may try to write
	 * to tail struct pages.
	 */
	if (!folio_test_hugetlb_vmemmap_optimized(folio))
1662
		__folio_clear_hugetlb(folio);
1663 1664 1665 1666

	 /*
	  * In the case of demote we do not ref count the page as it will soon
	  * be turned into a page of smaller size.
1667
	 */
1668
	if (!demote)
1669
		folio_ref_unfreeze(folio, 1);
1670 1671 1672 1673 1674

	h->nr_huge_pages--;
	h->nr_huge_pages_node[nid]--;
}

1675
static void remove_hugetlb_folio(struct hstate *h, struct folio *folio,
1676 1677
							bool adjust_surplus)
{
1678
	__remove_hugetlb_folio(h, folio, adjust_surplus, false);
1679 1680
}

1681
static void remove_hugetlb_folio_for_demote(struct hstate *h, struct folio *folio,
1682 1683
							bool adjust_surplus)
{
1684
	__remove_hugetlb_folio(h, folio, adjust_surplus, true);
1685 1686
}

1687
static void add_hugetlb_folio(struct hstate *h, struct folio *folio,
1688 1689 1690
			     bool adjust_surplus)
{
	int zeroed;
1691
	int nid = folio_nid(folio);
1692

1693
	VM_BUG_ON_FOLIO(!folio_test_hugetlb_vmemmap_optimized(folio), folio);
1694 1695 1696

	lockdep_assert_held(&hugetlb_lock);

1697
	INIT_LIST_HEAD(&folio->lru);
1698 1699 1700 1701 1702 1703 1704 1705
	h->nr_huge_pages++;
	h->nr_huge_pages_node[nid]++;

	if (adjust_surplus) {
		h->surplus_huge_pages++;
		h->surplus_huge_pages_node[nid]++;
	}

1706
	__folio_set_hugetlb(folio);
1707
	folio_change_private(folio, NULL);
1708
	/*
1709 1710
	 * We have to set hugetlb_vmemmap_optimized again as above
	 * folio_change_private(folio, NULL) cleared it.
1711
	 */
1712
	folio_set_hugetlb_vmemmap_optimized(folio);
1713 1714

	/*
1715
	 * This folio is about to be managed by the hugetlb allocator and
1716 1717
	 * should have no users.  Drop our reference, and check for others
	 * just in case.
1718
	 */
1719 1720
	zeroed = folio_put_testzero(folio);
	if (unlikely(!zeroed))
1721
		/*
1722 1723 1724 1725
		 * It is VERY unlikely soneone else has taken a ref
		 * on the folio.  In this case, we simply return as
		 * free_huge_folio() will be called when this other ref
		 * is dropped.
1726 1727 1728
		 */
		return;

1729
	arch_clear_hugepage_flags(&folio->page);
1730
	enqueue_hugetlb_folio(h, folio);
1731 1732
}

1733 1734
static void __update_and_free_hugetlb_folio(struct hstate *h,
						struct folio *folio)
1735
{
1736
	bool clear_flag = folio_test_hugetlb_vmemmap_optimized(folio);
1737

1738
	if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
1739
		return;
1740

1741 1742 1743 1744
	/*
	 * If we don't know which subpages are hwpoisoned, we can't free
	 * the hugepage, so it's leaked intentionally.
	 */
1745
	if (folio_test_hugetlb_raw_hwp_unreliable(folio))
1746 1747
		return;

1748
	/*
1749
	 * If folio is not vmemmap optimized (!clear_flag), then the folio
1750
	 * is no longer identified as a hugetlb page.  hugetlb_vmemmap_restore_folio
1751 1752
	 * can only be passed hugetlb pages and will BUG otherwise.
	 */
1753
	if (clear_flag && hugetlb_vmemmap_restore_folio(h, folio)) {
1754 1755 1756 1757 1758 1759
		spin_lock_irq(&hugetlb_lock);
		/*
		 * If we cannot allocate vmemmap pages, just refuse to free the
		 * page and put the page back on the hugetlb free list and treat
		 * as a surplus page.
		 */
1760
		add_hugetlb_folio(h, folio, true);
1761 1762 1763 1764
		spin_unlock_irq(&hugetlb_lock);
		return;
	}

1765 1766 1767 1768
	/*
	 * Move PageHWPoison flag from head page to the raw error pages,
	 * which makes any healthy subpages reusable.
	 */
1769
	if (unlikely(folio_test_hwpoison(folio)))
1770
		folio_clear_hugetlb_hwpoison(folio);
1771

1772 1773
	/*
	 * If vmemmap pages were allocated above, then we need to clear the
1774
	 * hugetlb flag under the hugetlb lock.
1775
	 */
1776
	if (folio_test_hugetlb(folio)) {
1777
		spin_lock_irq(&hugetlb_lock);
1778
		__folio_clear_hugetlb(folio);
1779 1780 1781
		spin_unlock_irq(&hugetlb_lock);
	}

1782 1783
	/*
	 * Non-gigantic pages demoted from CMA allocated gigantic pages
1784
	 * need to be given back to CMA in free_gigantic_folio.
1785 1786
	 */
	if (hstate_is_gigantic(h) ||
1787
	    hugetlb_cma_folio(folio, huge_page_order(h))) {
1788
		destroy_compound_gigantic_folio(folio, huge_page_order(h));
1789
		free_gigantic_folio(folio, huge_page_order(h));
1790
	} else {
1791 1792
		INIT_LIST_HEAD(&folio->_deferred_list);
		folio_put(folio);
1793
	}
1794 1795
}

1796
/*
1797
 * As update_and_free_hugetlb_folio() can be called under any context, so we cannot
1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815
 * use GFP_KERNEL to allocate vmemmap pages. However, we can defer the
 * actual freeing in a workqueue to prevent from using GFP_ATOMIC to allocate
 * the vmemmap pages.
 *
 * free_hpage_workfn() locklessly retrieves the linked list of pages to be
 * freed and frees them one-by-one. As the page->mapping pointer is going
 * to be cleared in free_hpage_workfn() anyway, it is reused as the llist_node
 * structure of a lockless linked list of huge pages to be freed.
 */
static LLIST_HEAD(hpage_freelist);

static void free_hpage_workfn(struct work_struct *work)
{
	struct llist_node *node;

	node = llist_del_all(&hpage_freelist);

	while (node) {
1816
		struct folio *folio;
1817 1818
		struct hstate *h;

1819 1820
		folio = container_of((struct address_space **)node,
				     struct folio, mapping);
1821
		node = node->next;
1822
		folio->mapping = NULL;
1823
		/*
1824 1825
		 * The VM_BUG_ON_FOLIO(!folio_test_hugetlb(folio), folio) in
		 * folio_hstate() is going to trigger because a previous call to
1826 1827
		 * remove_hugetlb_folio() will clear the hugetlb bit, so do
		 * not use folio_hstate() directly.
1828
		 */
1829
		h = size_to_hstate(folio_size(folio));
1830

1831
		__update_and_free_hugetlb_folio(h, folio);
1832 1833 1834 1835 1836 1837 1838 1839

		cond_resched();
	}
}
static DECLARE_WORK(free_hpage_work, free_hpage_workfn);

static inline void flush_free_hpage_work(struct hstate *h)
{
1840
	if (hugetlb_vmemmap_optimizable(h))
1841 1842 1843
		flush_work(&free_hpage_work);
}

1844
static void update_and_free_hugetlb_folio(struct hstate *h, struct folio *folio,
1845 1846
				 bool atomic)
{
1847
	if (!folio_test_hugetlb_vmemmap_optimized(folio) || !atomic) {
1848
		__update_and_free_hugetlb_folio(h, folio);
1849 1850 1851 1852 1853 1854 1855 1856 1857 1858
		return;
	}

	/*
	 * Defer freeing to avoid using GFP_ATOMIC to allocate vmemmap pages.
	 *
	 * Only call schedule_work() if hpage_freelist is previously
	 * empty. Otherwise, schedule_work() had been called but the workfn
	 * hasn't retrieved the list yet.
	 */
1859
	if (llist_add((struct llist_node *)&folio->mapping, &hpage_freelist))
1860 1861 1862
		schedule_work(&free_hpage_work);
}

1863 1864 1865
static void bulk_vmemmap_restore_error(struct hstate *h,
					struct list_head *folio_list,
					struct list_head *non_hvo_folios)
1866
{
1867
	struct folio *folio, *t_folio;
1868

1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879
	if (!list_empty(non_hvo_folios)) {
		/*
		 * Free any restored hugetlb pages so that restore of the
		 * entire list can be retried.
		 * The idea is that in the common case of ENOMEM errors freeing
		 * hugetlb pages with vmemmap we will free up memory so that we
		 * can allocate vmemmap for more hugetlb pages.
		 */
		list_for_each_entry_safe(folio, t_folio, non_hvo_folios, lru) {
			list_del(&folio->lru);
			spin_lock_irq(&hugetlb_lock);
1880
			__folio_clear_hugetlb(folio);
1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896
			spin_unlock_irq(&hugetlb_lock);
			update_and_free_hugetlb_folio(h, folio, false);
			cond_resched();
		}
	} else {
		/*
		 * In the case where there are no folios which can be
		 * immediately freed, we loop through the list trying to restore
		 * vmemmap individually in the hope that someone elsewhere may
		 * have done something to cause success (such as freeing some
		 * memory).  If unable to restore a hugetlb page, the hugetlb
		 * page is made a surplus page and removed from the list.
		 * If are able to restore vmemmap and free one hugetlb page, we
		 * quit processing the list to retry the bulk operation.
		 */
		list_for_each_entry_safe(folio, t_folio, folio_list, lru)
1897
			if (hugetlb_vmemmap_restore_folio(h, folio)) {
1898
				list_del(&folio->lru);
1899 1900 1901
				spin_lock_irq(&hugetlb_lock);
				add_hugetlb_folio(h, folio, true);
				spin_unlock_irq(&hugetlb_lock);
1902 1903 1904
			} else {
				list_del(&folio->lru);
				spin_lock_irq(&hugetlb_lock);
1905
				__folio_clear_hugetlb(folio);
1906 1907 1908 1909 1910
				spin_unlock_irq(&hugetlb_lock);
				update_and_free_hugetlb_folio(h, folio, false);
				cond_resched();
				break;
			}
1911
	}
1912 1913 1914 1915 1916 1917 1918 1919
}

static void update_and_free_pages_bulk(struct hstate *h,
						struct list_head *folio_list)
{
	long ret;
	struct folio *folio, *t_folio;
	LIST_HEAD(non_hvo_folios);
1920 1921

	/*
1922 1923 1924
	 * First allocate required vmemmmap (if necessary) for all folios.
	 * Carefully handle errors and free up any available hugetlb pages
	 * in an effort to make forward progress.
1925
	 */
1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937
retry:
	ret = hugetlb_vmemmap_restore_folios(h, folio_list, &non_hvo_folios);
	if (ret < 0) {
		bulk_vmemmap_restore_error(h, folio_list, &non_hvo_folios);
		goto retry;
	}

	/*
	 * At this point, list should be empty, ret should be >= 0 and there
	 * should only be pages on the non_hvo_folios list.
	 * Do note that the non_hvo_folios list could be empty.
	 * Without HVO enabled, ret will be 0 and there is no need to call
1938
	 * __folio_clear_hugetlb as this was done previously.
1939 1940 1941 1942
	 */
	VM_WARN_ON(!list_empty(folio_list));
	VM_WARN_ON(ret < 0);
	if (!list_empty(&non_hvo_folios) && ret) {
1943
		spin_lock_irq(&hugetlb_lock);
1944
		list_for_each_entry(folio, &non_hvo_folios, lru)
1945
			__folio_clear_hugetlb(folio);
1946 1947 1948
		spin_unlock_irq(&hugetlb_lock);
	}

1949
	list_for_each_entry_safe(folio, t_folio, &non_hvo_folios, lru) {
1950
		update_and_free_hugetlb_folio(h, folio, false);
1951 1952 1953 1954
		cond_resched();
	}
}

1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965
struct hstate *size_to_hstate(unsigned long size)
{
	struct hstate *h;

	for_each_hstate(h) {
		if (huge_page_size(h) == size)
			return h;
	}
	return NULL;
}

1966
void free_huge_folio(struct folio *folio)
1967
{
1968 1969
	/*
	 * Can't pass hstate in here because it is called from the
1970
	 * generic mm code.
1971
	 */
1972 1973 1974
	struct hstate *h = folio_hstate(folio);
	int nid = folio_nid(folio);
	struct hugepage_subpool *spool = hugetlb_folio_subpool(folio);
1975
	bool restore_reserve;
1976
	unsigned long flags;
1977

1978 1979
	VM_BUG_ON_FOLIO(folio_ref_count(folio), folio);
	VM_BUG_ON_FOLIO(folio_mapcount(folio), folio);
1980

1981 1982 1983 1984 1985 1986
	hugetlb_set_folio_subpool(folio, NULL);
	if (folio_test_anon(folio))
		__ClearPageAnonExclusive(&folio->page);
	folio->mapping = NULL;
	restore_reserve = folio_test_hugetlb_restore_reserve(folio);
	folio_clear_hugetlb_restore_reserve(folio);
1987

1988
	/*
1989
	 * If HPageRestoreReserve was set on page, page allocation consumed a
1990 1991 1992
	 * reservation.  If the page was associated with a subpool, there
	 * would have been a page reserved in the subpool before allocation
	 * via hugepage_subpool_get_pages().  Since we are 'restoring' the
1993
	 * reservation, do not call hugepage_subpool_put_pages() as this will
1994
	 * remove the reserved page from the subpool.
1995
	 */
1996 1997 1998 1999 2000 2001 2002 2003 2004 2005
	if (!restore_reserve) {
		/*
		 * A return code of zero implies that the subpool will be
		 * under its minimum size if the reservation is not restored
		 * after page is free.  Therefore, force restore_reserve
		 * operation.
		 */
		if (hugepage_subpool_put_pages(spool, 1) == 0)
			restore_reserve = true;
	}
2006

2007
	spin_lock_irqsave(&hugetlb_lock, flags);
2008
	folio_clear_hugetlb_migratable(folio);
2009 2010 2011 2012
	hugetlb_cgroup_uncharge_folio(hstate_index(h),
				     pages_per_huge_page(h), folio);
	hugetlb_cgroup_uncharge_folio_rsvd(hstate_index(h),
					  pages_per_huge_page(h), folio);
2013
	mem_cgroup_uncharge(folio);
2014 2015 2016
	if (restore_reserve)
		h->resv_huge_pages++;

2017
	if (folio_test_hugetlb_temporary(folio)) {
2018
		remove_hugetlb_folio(h, folio, false);
2019
		spin_unlock_irqrestore(&hugetlb_lock, flags);
2020
		update_and_free_hugetlb_folio(h, folio, true);
2021
	} else if (h->surplus_huge_pages_node[nid]) {
2022
		/* remove the page from active list */
2023
		remove_hugetlb_folio(h, folio, true);
2024
		spin_unlock_irqrestore(&hugetlb_lock, flags);
2025
		update_and_free_hugetlb_folio(h, folio, true);
2026
	} else {
2027
		arch_clear_hugepage_flags(&folio->page);
2028
		enqueue_hugetlb_folio(h, folio);
2029
		spin_unlock_irqrestore(&hugetlb_lock, flags);
2030 2031 2032
	}
}

2033 2034 2035 2036 2037 2038 2039 2040 2041 2042
/*
 * Must be called with the hugetlb lock held
 */
static void __prep_account_new_huge_page(struct hstate *h, int nid)
{
	lockdep_assert_held(&hugetlb_lock);
	h->nr_huge_pages++;
	h->nr_huge_pages_node[nid]++;
}

2043
static void init_new_hugetlb_folio(struct hstate *h, struct folio *folio)
2044
{
2045
	__folio_set_hugetlb(folio);
2046 2047 2048 2049
	INIT_LIST_HEAD(&folio->lru);
	hugetlb_set_folio_subpool(folio, NULL);
	set_hugetlb_cgroup(folio, NULL);
	set_hugetlb_cgroup_rsvd(folio, NULL);
2050 2051
}

2052 2053 2054
static void __prep_new_hugetlb_folio(struct hstate *h, struct folio *folio)
{
	init_new_hugetlb_folio(h, folio);
2055
	hugetlb_vmemmap_optimize_folio(h, folio);
2056 2057
}

2058
static void prep_new_hugetlb_folio(struct hstate *h, struct folio *folio, int nid)
2059
{
2060
	__prep_new_hugetlb_folio(h, folio);
2061
	spin_lock_irq(&hugetlb_lock);
2062
	__prep_account_new_huge_page(h, nid);
2063
	spin_unlock_irq(&hugetlb_lock);
2064 2065
}

2066 2067
static bool __prep_compound_gigantic_folio(struct folio *folio,
					unsigned int order, bool demote)
2068
{
2069
	int i, j;
2070
	int nr_pages = 1 << order;
2071
	struct page *p;
2072

2073
	__folio_clear_reserved(folio);
2074
	for (i = 0; i < nr_pages; i++) {
2075
		p = folio_page(folio, i);
2076

2077 2078 2079 2080
		/*
		 * For gigantic hugepages allocated through bootmem at
		 * boot, it's safer to be consistent with the not-gigantic
		 * hugepages and clear the PG_reserved bit from all tail pages
2081
		 * too.  Otherwise drivers using get_user_pages() to access tail
2082 2083 2084 2085 2086 2087 2088
		 * pages may get the reference counting wrong if they see
		 * PG_reserved set on a tail page (despite the head page not
		 * having PG_reserved set).  Enforcing this consistency between
		 * head and tail pages allows drivers to optimize away a check
		 * on the head page when they need know if put_page() is needed
		 * after get_user_pages().
		 */
2089 2090
		if (i != 0)	/* head page cleared above */
			__ClearPageReserved(p);
2091 2092 2093 2094 2095 2096 2097 2098 2099 2100
		/*
		 * Subtle and very unlikely
		 *
		 * Gigantic 'page allocators' such as memblock or cma will
		 * return a set of pages with each page ref counted.  We need
		 * to turn this set of pages into a compound page with tail
		 * page ref counts set to zero.  Code such as speculative page
		 * cache adding could take a ref on a 'to be' tail page.
		 * We need to respect any increased ref count, and only set
		 * the ref count to zero if count is currently 1.  If count
2101 2102 2103 2104
		 * is not 1, we return an error.  An error return indicates
		 * the set of pages can not be converted to a gigantic page.
		 * The caller who allocated the pages should then discard the
		 * pages using the appropriate free interface.
2105 2106
		 *
		 * In the case of demote, the ref count will be zero.
2107
		 */
2108 2109 2110 2111 2112 2113 2114
		if (!demote) {
			if (!page_ref_freeze(p, 1)) {
				pr_warn("HugeTLB page can not be used due to unexpected inflated ref count\n");
				goto out_error;
			}
		} else {
			VM_BUG_ON_PAGE(page_count(p), p);
2115
		}
2116
		if (i != 0)
2117
			set_compound_head(p, &folio->page);
2118
	}
2119
	__folio_set_head(folio);
2120
	/* we rely on prep_new_hugetlb_folio to set the hugetlb flag */
2121
	folio_set_order(folio, order);
2122
	atomic_set(&folio->_entire_mapcount, -1);
2123
	atomic_set(&folio->_nr_pages_mapped, 0);
2124
	atomic_set(&folio->_pincount, 0);
2125 2126 2127
	return true;

out_error:
2128 2129
	/* undo page modifications made above */
	for (j = 0; j < i; j++) {
2130
		p = folio_page(folio, j);
2131 2132
		if (j != 0)
			clear_compound_head(p);
2133 2134 2135
		set_page_refcounted(p);
	}
	/* need to clear PG_reserved on remaining tail pages  */
2136
	for (; j < nr_pages; j++) {
2137
		p = folio_page(folio, j);
2138
		__ClearPageReserved(p);
2139
	}
2140
	return false;
2141 2142
}

2143 2144
static bool prep_compound_gigantic_folio(struct folio *folio,
							unsigned int order)
2145
{
2146
	return __prep_compound_gigantic_folio(folio, order, false);
2147 2148
}

2149
static bool prep_compound_gigantic_folio_for_demote(struct folio *folio,
2150 2151
							unsigned int order)
{
2152
	return __prep_compound_gigantic_folio(folio, order, true);
2153 2154
}

2155 2156 2157
/*
 * Find and lock address space (mapping) in write mode.
 *
2158 2159 2160
 * Upon entry, the page is locked which means that page_mapping() is
 * stable.  Due to locking order, we can only trylock_write.  If we can
 * not get the lock, simply return NULL to caller.
2161 2162 2163
 */
struct address_space *hugetlb_page_mapping_lock_write(struct page *hpage)
{
2164
	struct address_space *mapping = page_mapping(hpage);
2165 2166 2167 2168 2169 2170 2171

	if (!mapping)
		return mapping;

	if (i_mmap_trylock_write(mapping))
		return mapping;

2172
	return NULL;
2173 2174
}

2175
static struct folio *alloc_buddy_hugetlb_folio(struct hstate *h,
2176 2177
		gfp_t gfp_mask, int nid, nodemask_t *nmask,
		nodemask_t *node_alloc_noretry)
Linus Torvalds's avatar
Linus Torvalds committed
2178
{
2179
	int order = huge_page_order(h);
Linus Torvalds's avatar
Linus Torvalds committed
2180
	struct page *page;
2181
	bool alloc_try_hard = true;
2182
	bool retry = true;
2183

2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195
	/*
	 * By default we always try hard to allocate the page with
	 * __GFP_RETRY_MAYFAIL flag.  However, if we are allocating pages in
	 * a loop (to adjust global huge page counts) and previous allocation
	 * failed, do not continue to try hard on the same node.  Use the
	 * node_alloc_noretry bitmap to manage this state information.
	 */
	if (node_alloc_noretry && node_isset(nid, *node_alloc_noretry))
		alloc_try_hard = false;
	gfp_mask |= __GFP_COMP|__GFP_NOWARN;
	if (alloc_try_hard)
		gfp_mask |= __GFP_RETRY_MAYFAIL;
2196 2197
	if (nid == NUMA_NO_NODE)
		nid = numa_mem_id();
2198
retry:
2199
	page = __alloc_pages(gfp_mask, order, nid, nmask);
2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212

	/* Freeze head page */
	if (page && !page_ref_freeze(page, 1)) {
		__free_pages(page, order);
		if (retry) {	/* retry once */
			retry = false;
			goto retry;
		}
		/* WOW!  twice in a row. */
		pr_warn("HugeTLB head page unexpected inflated ref count\n");
		page = NULL;
	}

2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228
	/*
	 * If we did not specify __GFP_RETRY_MAYFAIL, but still got a page this
	 * indicates an overall state change.  Clear bit so that we resume
	 * normal 'try hard' allocations.
	 */
	if (node_alloc_noretry && page && !alloc_try_hard)
		node_clear(nid, *node_alloc_noretry);

	/*
	 * If we tried hard to get a page but failed, set bit so that
	 * subsequent attempts will not try as hard until there is an
	 * overall state change.
	 */
	if (node_alloc_noretry && !page && alloc_try_hard)
		node_set(nid, *node_alloc_noretry);

2229 2230 2231 2232 2233 2234 2235
	if (!page) {
		__count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
		return NULL;
	}

	__count_vm_event(HTLB_BUDDY_PGALLOC);
	return page_folio(page);
2236 2237
}

2238 2239 2240
static struct folio *__alloc_fresh_hugetlb_folio(struct hstate *h,
				gfp_t gfp_mask, int nid, nodemask_t *nmask,
				nodemask_t *node_alloc_noretry)
2241
{
2242
	struct folio *folio;
2243
	bool retry = false;
2244

2245
retry:
2246
	if (hstate_is_gigantic(h))
2247
		folio = alloc_gigantic_folio(h, gfp_mask, nid, nmask);
2248
	else
2249
		folio = alloc_buddy_hugetlb_folio(h, gfp_mask,
2250
				nid, nmask, node_alloc_noretry);
2251
	if (!folio)
2252
		return NULL;
2253

2254
	if (hstate_is_gigantic(h)) {
2255
		if (!prep_compound_gigantic_folio(folio, huge_page_order(h))) {
2256 2257 2258 2259
			/*
			 * Rare failure to convert pages to compound page.
			 * Free pages and try again - ONCE!
			 */
2260
			free_gigantic_folio(folio, huge_page_order(h));
2261 2262 2263 2264 2265 2266 2267
			if (!retry) {
				retry = true;
				goto retry;
			}
			return NULL;
		}
	}
2268

2269
	return folio;
2270 2271
}

2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284
static struct folio *only_alloc_fresh_hugetlb_folio(struct hstate *h,
		gfp_t gfp_mask, int nid, nodemask_t *nmask,
		nodemask_t *node_alloc_noretry)
{
	struct folio *folio;

	folio = __alloc_fresh_hugetlb_folio(h, gfp_mask, nid, nmask,
						node_alloc_noretry);
	if (folio)
		init_new_hugetlb_folio(h, folio);
	return folio;
}

2285
/*
2286 2287 2288 2289 2290
 * Common helper to allocate a fresh hugetlb page. All specific allocators
 * should use this function to get new hugetlb pages
 *
 * Note that returned page is 'frozen':  ref count of head page and all tail
 * pages is zero.
2291
 */
2292 2293 2294
static struct folio *alloc_fresh_hugetlb_folio(struct hstate *h,
		gfp_t gfp_mask, int nid, nodemask_t *nmask,
		nodemask_t *node_alloc_noretry)
2295
{
2296
	struct folio *folio;
2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312

	folio = __alloc_fresh_hugetlb_folio(h, gfp_mask, nid, nmask,
						node_alloc_noretry);
	if (!folio)
		return NULL;

	prep_new_hugetlb_folio(h, folio, folio_nid(folio));
	return folio;
}

static void prep_and_add_allocated_folios(struct hstate *h,
					struct list_head *folio_list)
{
	unsigned long flags;
	struct folio *folio, *tmp_f;

2313 2314 2315
	/* Send list for bulk vmemmap optimization processing */
	hugetlb_vmemmap_optimize_folios(h, folio_list);

2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330
	/* Add all new pool pages to free lists in one lock cycle */
	spin_lock_irqsave(&hugetlb_lock, flags);
	list_for_each_entry_safe(folio, tmp_f, folio_list, lru) {
		__prep_account_new_huge_page(h, folio_nid(folio));
		enqueue_hugetlb_folio(h, folio);
	}
	spin_unlock_irqrestore(&hugetlb_lock, flags);
}

/*
 * Allocates a fresh hugetlb page in a node interleaved manner.  The page
 * will later be added to the appropriate hugetlb pool.
 */
static struct folio *alloc_pool_huge_folio(struct hstate *h,
					nodemask_t *nodes_allowed,
2331 2332
					nodemask_t *node_alloc_noretry,
					int *next_node)
2333
{
2334
	gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;
2335
	int nr_nodes, node;
2336

2337
	for_each_node_mask_to_alloc(next_node, nr_nodes, node, nodes_allowed) {
2338 2339 2340
		struct folio *folio;

		folio = only_alloc_fresh_hugetlb_folio(h, gfp_mask, node,
2341
					nodes_allowed, node_alloc_noretry);
2342 2343
		if (folio)
			return folio;
2344 2345
	}

2346
	return NULL;
2347 2348
}

2349
/*
2350 2351 2352 2353
 * Remove huge page from pool from next node to free.  Attempt to keep
 * persistent huge pages more or less balanced over allowed nodes.
 * This routine only 'removes' the hugetlb page.  The caller must make
 * an additional call to free the page to low level allocators.
2354 2355
 * Called with hugetlb_lock locked.
 */
2356 2357
static struct folio *remove_pool_hugetlb_folio(struct hstate *h,
		nodemask_t *nodes_allowed, bool acct_surplus)
2358
{
2359
	int nr_nodes, node;
2360
	struct folio *folio = NULL;
2361

2362
	lockdep_assert_held(&hugetlb_lock);
2363
	for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
2364 2365 2366 2367
		/*
		 * If we're returning unused surplus pages, only examine
		 * nodes with surplus pages.
		 */
2368 2369
		if ((!acct_surplus || h->surplus_huge_pages_node[node]) &&
		    !list_empty(&h->hugepage_freelists[node])) {
2370 2371
			folio = list_entry(h->hugepage_freelists[node].next,
					  struct folio, lru);
2372
			remove_hugetlb_folio(h, folio, acct_surplus);
2373
			break;
2374
		}
2375
	}
2376

2377
	return folio;
2378 2379
}

2380 2381
/*
 * Dissolve a given free hugepage into free buddy pages. This function does
2382 2383 2384
 * nothing for in-use hugepages and non-hugepages.
 * This function returns values like below:
 *
2385 2386 2387 2388 2389 2390 2391 2392
 *  -ENOMEM: failed to allocate vmemmap pages to free the freed hugepages
 *           when the system is under memory pressure and the feature of
 *           freeing unused vmemmap pages associated with each hugetlb page
 *           is enabled.
 *  -EBUSY:  failed to dissolved free hugepages or the hugepage is in-use
 *           (allocated or reserved.)
 *       0:  successfully dissolved free hugepages or the page is not a
 *           hugepage (considered as already dissolved)
2393
 */
2394
int dissolve_free_huge_page(struct page *page)
2395
{
2396
	int rc = -EBUSY;
2397
	struct folio *folio = page_folio(page);
2398

2399
retry:
2400
	/* Not to disrupt normal path by vainly holding hugetlb_lock */
2401
	if (!folio_test_hugetlb(folio))
2402 2403
		return 0;

2404
	spin_lock_irq(&hugetlb_lock);
2405
	if (!folio_test_hugetlb(folio)) {
2406 2407 2408 2409
		rc = 0;
		goto out;
	}

2410 2411
	if (!folio_ref_count(folio)) {
		struct hstate *h = folio_hstate(folio);
2412
		if (!available_huge_pages(h))
2413
			goto out;
2414 2415 2416 2417 2418

		/*
		 * We should make sure that the page is already on the free list
		 * when it is dissolved.
		 */
2419
		if (unlikely(!folio_test_hugetlb_freed(folio))) {
2420
			spin_unlock_irq(&hugetlb_lock);
2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433
			cond_resched();

			/*
			 * Theoretically, we should return -EBUSY when we
			 * encounter this race. In fact, we have a chance
			 * to successfully dissolve the page if we do a
			 * retry. Because the race window is quite small.
			 * If we seize this opportunity, it is an optimization
			 * for increasing the success rate of dissolving page.
			 */
			goto retry;
		}

2434
		remove_hugetlb_folio(h, folio, false);
2435
		h->max_huge_pages--;
2436
		spin_unlock_irq(&hugetlb_lock);
2437 2438

		/*
2439 2440
		 * Normally update_and_free_hugtlb_folio will allocate required vmemmmap
		 * before freeing the page.  update_and_free_hugtlb_folio will fail to
2441 2442 2443 2444
		 * free the page if it can not allocate required vmemmap.  We
		 * need to adjust max_huge_pages if the page is not freed.
		 * Attempt to allocate vmemmmap here so that we can take
		 * appropriate action on failure.
2445 2446 2447 2448
		 *
		 * The folio_test_hugetlb check here is because
		 * remove_hugetlb_folio will clear hugetlb folio flag for
		 * non-vmemmap optimized hugetlb folios.
2449
		 */
2450
		if (folio_test_hugetlb(folio)) {
2451
			rc = hugetlb_vmemmap_restore_folio(h, folio);
2452 2453 2454 2455 2456 2457 2458 2459
			if (rc) {
				spin_lock_irq(&hugetlb_lock);
				add_hugetlb_folio(h, folio, false);
				h->max_huge_pages++;
				goto out;
			}
		} else
			rc = 0;
2460

2461
		update_and_free_hugetlb_folio(h, folio, false);
2462
		return rc;
2463
	}
2464
out:
2465
	spin_unlock_irq(&hugetlb_lock);
2466
	return rc;
2467 2468 2469 2470 2471
}

/*
 * Dissolve free hugepages in a given pfn range. Used by memory hotplug to
 * make specified memory blocks removable from the system.
2472 2473
 * Note that this will dissolve a free gigantic hugepage completely, if any
 * part of it lies within the given range.
2474 2475
 * Also note that if dissolve_free_huge_page() returns with an error, all
 * free hugepages that were dissolved before that error are lost.
2476
 */
2477
int dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn)
2478 2479
{
	unsigned long pfn;
2480
	struct page *page;
2481
	int rc = 0;
2482 2483
	unsigned int order;
	struct hstate *h;
2484

2485
	if (!hugepages_supported())
2486
		return rc;
2487

2488 2489 2490 2491 2492
	order = huge_page_order(&default_hstate);
	for_each_hstate(h)
		order = min(order, huge_page_order(h));

	for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << order) {
2493
		page = pfn_to_page(pfn);
2494 2495 2496
		rc = dissolve_free_huge_page(page);
		if (rc)
			break;
2497
	}
2498 2499

	return rc;
2500 2501
}

2502 2503 2504
/*
 * Allocates a fresh surplus page from the page allocator.
 */
2505 2506
static struct folio *alloc_surplus_hugetlb_folio(struct hstate *h,
				gfp_t gfp_mask,	int nid, nodemask_t *nmask)
2507
{
2508
	struct folio *folio = NULL;
2509

2510
	if (hstate_is_gigantic(h))
2511 2512
		return NULL;

2513
	spin_lock_irq(&hugetlb_lock);
2514 2515
	if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages)
		goto out_unlock;
2516
	spin_unlock_irq(&hugetlb_lock);
2517

2518 2519
	folio = alloc_fresh_hugetlb_folio(h, gfp_mask, nid, nmask, NULL);
	if (!folio)
2520
		return NULL;
2521

2522
	spin_lock_irq(&hugetlb_lock);
2523 2524 2525 2526
	/*
	 * We could have raced with the pool size change.
	 * Double check that and simply deallocate the new page
	 * if we would end up overcommiting the surpluses. Abuse
2527
	 * temporary page to workaround the nasty free_huge_folio
2528 2529 2530
	 * codeflow
	 */
	if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
2531
		folio_set_hugetlb_temporary(folio);
2532
		spin_unlock_irq(&hugetlb_lock);
2533
		free_huge_folio(folio);
2534
		return NULL;
2535
	}
2536

2537
	h->surplus_huge_pages++;
2538
	h->surplus_huge_pages_node[folio_nid(folio)]++;
2539

2540
out_unlock:
2541
	spin_unlock_irq(&hugetlb_lock);
2542

2543
	return folio;
2544 2545
}

2546
static struct folio *alloc_migrate_hugetlb_folio(struct hstate *h, gfp_t gfp_mask,
2547
				     int nid, nodemask_t *nmask)
2548
{
2549
	struct folio *folio;
2550 2551 2552 2553

	if (hstate_is_gigantic(h))
		return NULL;

2554 2555
	folio = alloc_fresh_hugetlb_folio(h, gfp_mask, nid, nmask, NULL);
	if (!folio)
2556 2557
		return NULL;

2558
	/* fresh huge pages are frozen */
2559
	folio_ref_unfreeze(folio, 1);
2560 2561 2562 2563
	/*
	 * We do not account these pages as surplus because they are only
	 * temporary and will be released properly on the last reference
	 */
2564
	folio_set_hugetlb_temporary(folio);
2565

2566
	return folio;
2567 2568
}

2569 2570 2571
/*
 * Use the VMA's mpolicy to allocate a huge page from the buddy.
 */
2572
static
2573
struct folio *alloc_buddy_hugetlb_folio_with_mpol(struct hstate *h,
2574 2575
		struct vm_area_struct *vma, unsigned long addr)
{
2576
	struct folio *folio = NULL;
2577 2578 2579 2580 2581 2582
	struct mempolicy *mpol;
	gfp_t gfp_mask = htlb_alloc_mask(h);
	int nid;
	nodemask_t *nodemask;

	nid = huge_node(vma, addr, gfp_mask, &mpol, &nodemask);
2583 2584 2585 2586
	if (mpol_is_preferred_many(mpol)) {
		gfp_t gfp = gfp_mask | __GFP_NOWARN;

		gfp &=  ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL);
2587
		folio = alloc_surplus_hugetlb_folio(h, gfp, nid, nodemask);
2588

2589 2590 2591 2592
		/* Fallback to all nodes if page==NULL */
		nodemask = NULL;
	}

2593 2594
	if (!folio)
		folio = alloc_surplus_hugetlb_folio(h, gfp_mask, nid, nodemask);
2595
	mpol_cond_put(mpol);
2596
	return folio;
2597 2598
}

2599 2600
/* folio migration callback function */
struct folio *alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid,
2601
		nodemask_t *nmask, gfp_t gfp_mask)
2602
{
2603
	spin_lock_irq(&hugetlb_lock);
2604
	if (available_huge_pages(h)) {
2605
		struct folio *folio;
2606

2607 2608 2609
		folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask,
						preferred_nid, nmask);
		if (folio) {
2610
			spin_unlock_irq(&hugetlb_lock);
2611
			return folio;
2612 2613
		}
	}
2614
	spin_unlock_irq(&hugetlb_lock);
2615

2616
	return alloc_migrate_hugetlb_folio(h, gfp_mask, preferred_nid, nmask);
2617 2618
}

2619
/*
Lucas De Marchi's avatar
Lucas De Marchi committed
2620
 * Increase the hugetlb pool such that it can accommodate a reservation
2621 2622
 * of size 'delta'.
 */
2623
static int gather_surplus_pages(struct hstate *h, long delta)
2624
	__must_hold(&hugetlb_lock)
2625
{
2626
	LIST_HEAD(surplus_list);
2627
	struct folio *folio, *tmp;
2628 2629 2630
	int ret;
	long i;
	long needed, allocated;
2631
	bool alloc_ok = true;
2632

2633
	lockdep_assert_held(&hugetlb_lock);
2634
	needed = (h->resv_huge_pages + delta) - h->free_huge_pages;
2635
	if (needed <= 0) {
2636
		h->resv_huge_pages += delta;
2637
		return 0;
2638
	}
2639 2640 2641 2642 2643

	allocated = 0;

	ret = -ENOMEM;
retry:
2644
	spin_unlock_irq(&hugetlb_lock);
2645
	for (i = 0; i < needed; i++) {
2646
		folio = alloc_surplus_hugetlb_folio(h, htlb_alloc_mask(h),
2647
				NUMA_NO_NODE, NULL);
2648
		if (!folio) {
2649 2650 2651
			alloc_ok = false;
			break;
		}
2652
		list_add(&folio->lru, &surplus_list);
2653
		cond_resched();
2654
	}
2655
	allocated += i;
2656 2657 2658 2659 2660

	/*
	 * After retaking hugetlb_lock, we need to recalculate 'needed'
	 * because either resv_huge_pages or free_huge_pages may have changed.
	 */
2661
	spin_lock_irq(&hugetlb_lock);
2662 2663
	needed = (h->resv_huge_pages + delta) -
			(h->free_huge_pages + allocated);
2664 2665 2666 2667 2668 2669 2670 2671 2672 2673
	if (needed > 0) {
		if (alloc_ok)
			goto retry;
		/*
		 * We were not able to allocate enough pages to
		 * satisfy the entire reservation so we free what
		 * we've allocated so far.
		 */
		goto free;
	}
2674 2675
	/*
	 * The surplus_list now contains _at_least_ the number of extra pages
Lucas De Marchi's avatar
Lucas De Marchi committed
2676
	 * needed to accommodate the reservation.  Add the appropriate number
2677
	 * of pages to the hugetlb pool and free the extras back to the buddy
2678 2679 2680
	 * allocator.  Commit the entire reservation here to prevent another
	 * process from stealing the pages as they are added to the pool but
	 * before they are reserved.
2681 2682
	 */
	needed += allocated;
2683
	h->resv_huge_pages += delta;
2684
	ret = 0;
2685

2686
	/* Free the needed pages to the hugetlb pool */
2687
	list_for_each_entry_safe(folio, tmp, &surplus_list, lru) {
2688 2689
		if ((--needed) < 0)
			break;
2690
		/* Add the page to the hugetlb allocator */
2691
		enqueue_hugetlb_folio(h, folio);
2692
	}
2693
free:
2694
	spin_unlock_irq(&hugetlb_lock);
2695

2696 2697
	/*
	 * Free unnecessary surplus pages to the buddy allocator.
2698
	 * Pages have no ref count, call free_huge_folio directly.
2699
	 */
2700 2701
	list_for_each_entry_safe(folio, tmp, &surplus_list, lru)
		free_huge_folio(folio);
2702
	spin_lock_irq(&hugetlb_lock);
2703 2704 2705 2706 2707

	return ret;
}

/*
2708 2709 2710 2711 2712 2713
 * This routine has two main purposes:
 * 1) Decrement the reservation count (resv_huge_pages) by the value passed
 *    in unused_resv_pages.  This corresponds to the prior adjustments made
 *    to the associated reservation map.
 * 2) Free any unused surplus pages that may have been allocated to satisfy
 *    the reservation.  As many as unused_resv_pages may be freed.
2714
 */
2715 2716
static void return_unused_surplus_pages(struct hstate *h,
					unsigned long unused_resv_pages)
2717 2718
{
	unsigned long nr_pages;
2719 2720
	LIST_HEAD(page_list);

2721
	lockdep_assert_held(&hugetlb_lock);
2722 2723
	/* Uncommit the reservation */
	h->resv_huge_pages -= unused_resv_pages;
2724

2725
	if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
2726
		goto out;
2727

2728 2729 2730 2731
	/*
	 * Part (or even all) of the reservation could have been backed
	 * by pre-allocated pages. Only free surplus pages.
	 */
2732
	nr_pages = min(unused_resv_pages, h->surplus_huge_pages);
2733

2734 2735
	/*
	 * We want to release as many surplus pages as possible, spread
2736 2737 2738
	 * evenly across all nodes with memory. Iterate across these nodes
	 * until we can no longer free unreserved surplus pages. This occurs
	 * when the nodes with surplus pages have no free pages.
2739
	 * remove_pool_hugetlb_folio() will balance the freed pages across the
2740
	 * on-line nodes with memory and will handle the hstate accounting.
2741 2742
	 */
	while (nr_pages--) {
2743 2744 2745 2746
		struct folio *folio;

		folio = remove_pool_hugetlb_folio(h, &node_states[N_MEMORY], 1);
		if (!folio)
2747
			goto out;
2748

2749
		list_add(&folio->lru, &page_list);
2750
	}
2751 2752

out:
2753
	spin_unlock_irq(&hugetlb_lock);
2754
	update_and_free_pages_bulk(h, &page_list);
2755
	spin_lock_irq(&hugetlb_lock);
2756 2757
}

2758

2759
/*
2760
 * vma_needs_reservation, vma_commit_reservation and vma_end_reservation
2761
 * are used by the huge page allocation routines to manage reservations.
2762 2763 2764 2765 2766 2767
 *
 * vma_needs_reservation is called to determine if the huge page at addr
 * within the vma has an associated reservation.  If a reservation is
 * needed, the value 1 is returned.  The caller is then responsible for
 * managing the global reservation and subpool usage counts.  After
 * the huge page has been allocated, vma_commit_reservation is called
2768 2769 2770
 * to add the page to the reservation map.  If the page allocation fails,
 * the reservation must be ended instead of committed.  vma_end_reservation
 * is called in such cases.
2771 2772 2773 2774 2775 2776
 *
 * In the normal case, vma_commit_reservation returns the same value
 * as the preceding vma_needs_reservation call.  The only time this
 * is not the case is if a reserve map was changed between calls.  It
 * is the responsibility of the caller to notice the difference and
 * take appropriate action.
2777 2778 2779 2780 2781
 *
 * vma_add_reservation is used in error paths where a reservation must
 * be restored when a newly allocated huge page must be freed.  It is
 * to be called after calling vma_needs_reservation to determine if a
 * reservation exists.
2782 2783 2784 2785 2786
 *
 * vma_del_reservation is used in error paths where an entry in the reserve
 * map was created during huge page allocation and must be removed.  It is to
 * be called after calling vma_needs_reservation to determine if a reservation
 * exists.
2787
 */
2788 2789 2790
enum vma_resv_mode {
	VMA_NEEDS_RESV,
	VMA_COMMIT_RESV,
2791
	VMA_END_RESV,
2792
	VMA_ADD_RESV,
2793
	VMA_DEL_RESV,
2794
};
2795 2796
static long __vma_reservation_common(struct hstate *h,
				struct vm_area_struct *vma, unsigned long addr,
2797
				enum vma_resv_mode mode)
2798
{
2799 2800
	struct resv_map *resv;
	pgoff_t idx;
2801
	long ret;
2802
	long dummy_out_regions_needed;
2803

2804 2805
	resv = vma_resv_map(vma);
	if (!resv)
2806
		return 1;
2807

2808
	idx = vma_hugecache_offset(h, vma, addr);
2809 2810
	switch (mode) {
	case VMA_NEEDS_RESV:
2811 2812 2813 2814 2815 2816
		ret = region_chg(resv, idx, idx + 1, &dummy_out_regions_needed);
		/* We assume that vma_reservation_* routines always operate on
		 * 1 page, and that adding to resv map a 1 page entry can only
		 * ever require 1 region.
		 */
		VM_BUG_ON(dummy_out_regions_needed != 1);
2817 2818
		break;
	case VMA_COMMIT_RESV:
2819
		ret = region_add(resv, idx, idx + 1, 1, NULL, NULL);
2820 2821
		/* region_add calls of range 1 should never fail. */
		VM_BUG_ON(ret < 0);
2822
		break;
2823
	case VMA_END_RESV:
2824
		region_abort(resv, idx, idx + 1, 1);
2825 2826
		ret = 0;
		break;
2827
	case VMA_ADD_RESV:
2828
		if (vma->vm_flags & VM_MAYSHARE) {
2829
			ret = region_add(resv, idx, idx + 1, 1, NULL, NULL);
2830 2831 2832 2833
			/* region_add calls of range 1 should never fail. */
			VM_BUG_ON(ret < 0);
		} else {
			region_abort(resv, idx, idx + 1, 1);
2834 2835 2836
			ret = region_del(resv, idx, idx + 1);
		}
		break;
2837 2838 2839 2840 2841 2842 2843 2844 2845 2846
	case VMA_DEL_RESV:
		if (vma->vm_flags & VM_MAYSHARE) {
			region_abort(resv, idx, idx + 1, 1);
			ret = region_del(resv, idx, idx + 1);
		} else {
			ret = region_add(resv, idx, idx + 1, 1, NULL, NULL);
			/* region_add calls of range 1 should never fail. */
			VM_BUG_ON(ret < 0);
		}
		break;
2847 2848 2849
	default:
		BUG();
	}
2850

2851
	if (vma->vm_flags & VM_MAYSHARE || mode == VMA_DEL_RESV)
2852
		return ret;
2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872
	/*
	 * We know private mapping must have HPAGE_RESV_OWNER set.
	 *
	 * In most cases, reserves always exist for private mappings.
	 * However, a file associated with mapping could have been
	 * hole punched or truncated after reserves were consumed.
	 * As subsequent fault on such a range will not use reserves.
	 * Subtle - The reserve map for private mappings has the
	 * opposite meaning than that of shared mappings.  If NO
	 * entry is in the reserve map, it means a reservation exists.
	 * If an entry exists in the reserve map, it means the
	 * reservation has already been consumed.  As a result, the
	 * return value of this routine is the opposite of the
	 * value returned from reserve map manipulation routines above.
	 */
	if (ret > 0)
		return 0;
	if (ret == 0)
		return 1;
	return ret;
2873
}
2874 2875

static long vma_needs_reservation(struct hstate *h,
2876
			struct vm_area_struct *vma, unsigned long addr)
2877
{
2878
	return __vma_reservation_common(h, vma, addr, VMA_NEEDS_RESV);
2879
}
2880

2881 2882 2883
static long vma_commit_reservation(struct hstate *h,
			struct vm_area_struct *vma, unsigned long addr)
{
2884 2885 2886
	return __vma_reservation_common(h, vma, addr, VMA_COMMIT_RESV);
}

2887
static void vma_end_reservation(struct hstate *h,
2888 2889
			struct vm_area_struct *vma, unsigned long addr)
{
2890
	(void)__vma_reservation_common(h, vma, addr, VMA_END_RESV);
2891 2892
}

2893 2894 2895 2896 2897 2898
static long vma_add_reservation(struct hstate *h,
			struct vm_area_struct *vma, unsigned long addr)
{
	return __vma_reservation_common(h, vma, addr, VMA_ADD_RESV);
}

2899 2900 2901 2902 2903 2904
static long vma_del_reservation(struct hstate *h,
			struct vm_area_struct *vma, unsigned long addr)
{
	return __vma_reservation_common(h, vma, addr, VMA_DEL_RESV);
}

2905
/*
2906
 * This routine is called to restore reservation information on error paths.
2907 2908
 * It should ONLY be called for folios allocated via alloc_hugetlb_folio(),
 * and the hugetlb mutex should remain held when calling this routine.
2909 2910
 *
 * It handles two specific cases:
2911 2912 2913
 * 1) A reservation was in place and the folio consumed the reservation.
 *    hugetlb_restore_reserve is set in the folio.
 * 2) No reservation was in place for the page, so hugetlb_restore_reserve is
2914
 *    not set.  However, alloc_hugetlb_folio always updates the reserve map.
2915
 *
2916 2917
 * In case 1, free_huge_folio later in the error path will increment the
 * global reserve count.  But, free_huge_folio does not have enough context
2918 2919
 * to adjust the reservation map.  This case deals primarily with private
 * mappings.  Adjust the reserve map here to be consistent with global
2920
 * reserve count adjustments to be made by free_huge_folio.  Make sure the
2921 2922
 * reserve map indicates there is a reservation present.
 *
2923
 * In case 2, simply undo reserve map modifications done by alloc_hugetlb_folio.
2924
 */
2925
void restore_reserve_on_error(struct hstate *h, struct vm_area_struct *vma,
2926
			unsigned long address, struct folio *folio)
2927
{
2928
	long rc = vma_needs_reservation(h, vma, address);
2929

2930
	if (folio_test_hugetlb_restore_reserve(folio)) {
2931
		if (unlikely(rc < 0))
2932 2933
			/*
			 * Rare out of memory condition in reserve map
2934 2935
			 * manipulation.  Clear hugetlb_restore_reserve so
			 * that global reserve count will not be incremented
2936
			 * by free_huge_folio.  This will make it appear
2937
			 * as though the reservation for this folio was
2938
			 * consumed.  This may prevent the task from
2939
			 * faulting in the folio at a later time.  This
2940 2941 2942
			 * is better than inconsistent global huge page
			 * accounting of reserve counts.
			 */
2943
			folio_clear_hugetlb_restore_reserve(folio);
2944 2945 2946 2947 2948 2949 2950 2951
		else if (rc)
			(void)vma_add_reservation(h, vma, address);
		else
			vma_end_reservation(h, vma, address);
	} else {
		if (!rc) {
			/*
			 * This indicates there is an entry in the reserve map
2952 2953
			 * not added by alloc_hugetlb_folio.  We know it was added
			 * before the alloc_hugetlb_folio call, otherwise
2954
			 * hugetlb_restore_reserve would be set on the folio.
2955 2956 2957 2958 2959
			 * Remove the entry so that a subsequent allocation
			 * does not consume a reservation.
			 */
			rc = vma_del_reservation(h, vma, address);
			if (rc < 0)
2960
				/*
2961 2962
				 * VERY rare out of memory condition.  Since
				 * we can not delete the entry, set
2963 2964
				 * hugetlb_restore_reserve so that the reserve
				 * count will be incremented when the folio
2965 2966
				 * is freed.  This reserve will be consumed
				 * on a subsequent allocation.
2967
				 */
2968
				folio_set_hugetlb_restore_reserve(folio);
2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983
		} else if (rc < 0) {
			/*
			 * Rare out of memory condition from
			 * vma_needs_reservation call.  Memory allocation is
			 * only attempted if a new entry is needed.  Therefore,
			 * this implies there is not an entry in the
			 * reserve map.
			 *
			 * For shared mappings, no entry in the map indicates
			 * no reservation.  We are done.
			 */
			if (!(vma->vm_flags & VM_MAYSHARE))
				/*
				 * For private mappings, no entry indicates
				 * a reservation is present.  Since we can
2984 2985
				 * not add an entry, set hugetlb_restore_reserve
				 * on the folio so reserve count will be
2986 2987 2988
				 * incremented when freed.  This reserve will
				 * be consumed on a subsequent allocation.
				 */
2989
				folio_set_hugetlb_restore_reserve(folio);
2990
		} else
2991 2992 2993 2994
			/*
			 * No reservation present, do nothing
			 */
			 vma_end_reservation(h, vma, address);
2995 2996 2997
	}
}

2998
/*
2999 3000
 * alloc_and_dissolve_hugetlb_folio - Allocate a new folio and dissolve
 * the old one
3001
 * @h: struct hstate old page belongs to
3002
 * @old_folio: Old folio to dissolve
3003
 * @list: List to isolate the page in case we need to
3004 3005
 * Returns 0 on success, otherwise negated error.
 */
3006 3007
static int alloc_and_dissolve_hugetlb_folio(struct hstate *h,
			struct folio *old_folio, struct list_head *list)
3008 3009
{
	gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;
3010
	int nid = folio_nid(old_folio);
3011
	struct folio *new_folio = NULL;
3012 3013 3014 3015
	int ret = 0;

retry:
	spin_lock_irq(&hugetlb_lock);
3016
	if (!folio_test_hugetlb(old_folio)) {
3017
		/*
3018
		 * Freed from under us. Drop new_folio too.
3019 3020
		 */
		goto free_new;
3021
	} else if (folio_ref_count(old_folio)) {
3022 3023
		bool isolated;

3024
		/*
3025
		 * Someone has grabbed the folio, try to isolate it here.
3026
		 * Fail with -EBUSY if not possible.
3027
		 */
3028
		spin_unlock_irq(&hugetlb_lock);
3029 3030
		isolated = isolate_hugetlb(old_folio, list);
		ret = isolated ? 0 : -EBUSY;
3031
		spin_lock_irq(&hugetlb_lock);
3032
		goto free_new;
3033
	} else if (!folio_test_hugetlb_freed(old_folio)) {
3034
		/*
3035
		 * Folio's refcount is 0 but it has not been enqueued in the
3036 3037 3038 3039 3040 3041 3042
		 * freelist yet. Race window is small, so we can succeed here if
		 * we retry.
		 */
		spin_unlock_irq(&hugetlb_lock);
		cond_resched();
		goto retry;
	} else {
3043 3044 3045 3046 3047 3048 3049 3050 3051 3052
		if (!new_folio) {
			spin_unlock_irq(&hugetlb_lock);
			new_folio = alloc_buddy_hugetlb_folio(h, gfp_mask, nid,
							      NULL, NULL);
			if (!new_folio)
				return -ENOMEM;
			__prep_new_hugetlb_folio(h, new_folio);
			goto retry;
		}

3053
		/*
3054
		 * Ok, old_folio is still a genuine free hugepage. Remove it from
3055 3056
		 * the freelist and decrease the counters. These will be
		 * incremented again when calling __prep_account_new_huge_page()
3057 3058
		 * and enqueue_hugetlb_folio() for new_folio. The counters will
		 * remain stable since this happens under the lock.
3059
		 */
3060
		remove_hugetlb_folio(h, old_folio, false);
3061 3062

		/*
3063
		 * Ref count on new_folio is already zero as it was dropped
3064
		 * earlier.  It can be directly added to the pool free list.
3065 3066
		 */
		__prep_account_new_huge_page(h, nid);
3067
		enqueue_hugetlb_folio(h, new_folio);
3068 3069

		/*
3070
		 * Folio has been replaced, we can safely free the old one.
3071 3072
		 */
		spin_unlock_irq(&hugetlb_lock);
3073
		update_and_free_hugetlb_folio(h, old_folio, false);
3074 3075 3076 3077 3078 3079
	}

	return ret;

free_new:
	spin_unlock_irq(&hugetlb_lock);
3080 3081 3082 3083 3084
	if (new_folio) {
		/* Folio has a zero ref count, but needs a ref to be freed */
		folio_ref_unfreeze(new_folio, 1);
		update_and_free_hugetlb_folio(h, new_folio, false);
	}
3085 3086 3087 3088

	return ret;
}

3089
int isolate_or_dissolve_huge_page(struct page *page, struct list_head *list)
3090 3091
{
	struct hstate *h;
3092
	struct folio *folio = page_folio(page);
3093
	int ret = -EBUSY;
3094 3095 3096 3097 3098 3099 3100

	/*
	 * The page might have been dissolved from under our feet, so make sure
	 * to carefully check the state under the lock.
	 * Return success when racing as if we dissolved the page ourselves.
	 */
	spin_lock_irq(&hugetlb_lock);
3101 3102
	if (folio_test_hugetlb(folio)) {
		h = folio_hstate(folio);
3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116
	} else {
		spin_unlock_irq(&hugetlb_lock);
		return 0;
	}
	spin_unlock_irq(&hugetlb_lock);

	/*
	 * Fence off gigantic pages as there is a cyclic dependency between
	 * alloc_contig_range and them. Return -ENOMEM as this has the effect
	 * of bailing out right away without further retrying.
	 */
	if (hstate_is_gigantic(h))
		return -ENOMEM;

3117
	if (folio_ref_count(folio) && isolate_hugetlb(folio, list))
3118
		ret = 0;
3119
	else if (!folio_ref_count(folio))
3120
		ret = alloc_and_dissolve_hugetlb_folio(h, folio, list);
3121 3122

	return ret;
3123 3124
}

3125
struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma,
3126
				    unsigned long addr, int avoid_reserve)
Linus Torvalds's avatar
Linus Torvalds committed
3127
{
3128
	struct hugepage_subpool *spool = subpool_vma(vma);
3129
	struct hstate *h = hstate_vma(vma);
3130
	struct folio *folio;
3131
	long map_chg, map_commit, nr_pages = pages_per_huge_page(h);
3132
	long gbl_chg;
3133
	int memcg_charge_ret, ret, idx;
3134
	struct hugetlb_cgroup *h_cg = NULL;
3135
	struct mem_cgroup *memcg;
3136
	bool deferred_reserve;
3137 3138 3139 3140 3141 3142 3143 3144
	gfp_t gfp = htlb_alloc_mask(h) | __GFP_RETRY_MAYFAIL;

	memcg = get_mem_cgroup_from_current();
	memcg_charge_ret = mem_cgroup_hugetlb_try_charge(memcg, gfp, nr_pages);
	if (memcg_charge_ret == -ENOMEM) {
		mem_cgroup_put(memcg);
		return ERR_PTR(-ENOMEM);
	}
3145

3146
	idx = hstate_index(h);
3147
	/*
3148 3149 3150
	 * Examine the region/reserve map to determine if the process
	 * has a reservation for the page to be allocated.  A return
	 * code of zero indicates a reservation exists (no change).
3151
	 */
3152
	map_chg = gbl_chg = vma_needs_reservation(h, vma, addr);
3153 3154 3155 3156
	if (map_chg < 0) {
		if (!memcg_charge_ret)
			mem_cgroup_cancel_charge(memcg, nr_pages);
		mem_cgroup_put(memcg);
3157
		return ERR_PTR(-ENOMEM);
3158
	}
3159 3160 3161 3162 3163 3164 3165 3166 3167 3168

	/*
	 * Processes that did not create the mapping will have no
	 * reserves as indicated by the region/reserve map. Check
	 * that the allocation will not exceed the subpool limit.
	 * Allocations for MAP_NORESERVE mappings also need to be
	 * checked against any subpool limit.
	 */
	if (map_chg || avoid_reserve) {
		gbl_chg = hugepage_subpool_get_pages(spool, 1);
3169 3170
		if (gbl_chg < 0)
			goto out_end_reservation;
Linus Torvalds's avatar
Linus Torvalds committed
3171

3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183
		/*
		 * Even though there was no reservation in the region/reserve
		 * map, there could be reservations associated with the
		 * subpool that can be used.  This would be indicated if the
		 * return value of hugepage_subpool_get_pages() is zero.
		 * However, if avoid_reserve is specified we still avoid even
		 * the subpool reservations.
		 */
		if (avoid_reserve)
			gbl_chg = 1;
	}

3184 3185
	/* If this allocation is not consuming a reservation, charge it now.
	 */
3186
	deferred_reserve = map_chg || avoid_reserve;
3187 3188 3189 3190 3191 3192 3193
	if (deferred_reserve) {
		ret = hugetlb_cgroup_charge_cgroup_rsvd(
			idx, pages_per_huge_page(h), &h_cg);
		if (ret)
			goto out_subpool_put;
	}

3194
	ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg);
3195
	if (ret)
3196
		goto out_uncharge_cgroup_reservation;
3197

3198
	spin_lock_irq(&hugetlb_lock);
3199 3200 3201 3202 3203
	/*
	 * glb_chg is passed to indicate whether or not a page must be taken
	 * from the global free pool (global change).  gbl_chg == 0 indicates
	 * a reservation exists for the allocation.
	 */
3204 3205
	folio = dequeue_hugetlb_folio_vma(h, vma, addr, avoid_reserve, gbl_chg);
	if (!folio) {
3206
		spin_unlock_irq(&hugetlb_lock);
3207 3208
		folio = alloc_buddy_hugetlb_folio_with_mpol(h, vma, addr);
		if (!folio)
3209
			goto out_uncharge_cgroup;
3210
		spin_lock_irq(&hugetlb_lock);
3211
		if (!avoid_reserve && vma_has_reserves(vma, gbl_chg)) {
3212
			folio_set_hugetlb_restore_reserve(folio);
3213 3214
			h->resv_huge_pages--;
		}
3215 3216
		list_add(&folio->lru, &h->hugepage_activelist);
		folio_ref_unfreeze(folio, 1);
3217
		/* Fall through */
Ken Chen's avatar
Ken Chen committed
3218
	}
3219 3220

	hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, folio);
3221 3222 3223 3224 3225
	/* If allocation is not consuming a reservation, also store the
	 * hugetlb_cgroup pointer on the page.
	 */
	if (deferred_reserve) {
		hugetlb_cgroup_commit_charge_rsvd(idx, pages_per_huge_page(h),
3226
						  h_cg, folio);
3227 3228
	}

3229
	spin_unlock_irq(&hugetlb_lock);
3230

3231
	hugetlb_set_folio_subpool(folio, spool);
3232

3233 3234
	map_commit = vma_commit_reservation(h, vma, addr);
	if (unlikely(map_chg > map_commit)) {
3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247
		/*
		 * The page was added to the reservation map between
		 * vma_needs_reservation and vma_commit_reservation.
		 * This indicates a race with hugetlb_reserve_pages.
		 * Adjust for the subpool count incremented above AND
		 * in hugetlb_reserve_pages for the same page.  Also,
		 * the reservation count added in hugetlb_reserve_pages
		 * no longer applies.
		 */
		long rsv_adjust;

		rsv_adjust = hugepage_subpool_put_pages(spool, 1);
		hugetlb_acct_memory(h, -rsv_adjust);
3248 3249
		if (deferred_reserve) {
			spin_lock_irq(&hugetlb_lock);
3250 3251
			hugetlb_cgroup_uncharge_folio_rsvd(hstate_index(h),
					pages_per_huge_page(h), folio);
3252 3253
			spin_unlock_irq(&hugetlb_lock);
		}
3254
	}
3255 3256 3257 3258 3259

	if (!memcg_charge_ret)
		mem_cgroup_commit_charge(folio, memcg);
	mem_cgroup_put(memcg);

3260
	return folio;
3261 3262 3263

out_uncharge_cgroup:
	hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg);
3264 3265 3266 3267
out_uncharge_cgroup_reservation:
	if (deferred_reserve)
		hugetlb_cgroup_uncharge_cgroup_rsvd(idx, pages_per_huge_page(h),
						    h_cg);
3268
out_subpool_put:
3269
	if (map_chg || avoid_reserve)
3270
		hugepage_subpool_put_pages(spool, 1);
3271
out_end_reservation:
3272
	vma_end_reservation(h, vma, addr);
3273 3274 3275
	if (!memcg_charge_ret)
		mem_cgroup_cancel_charge(memcg, nr_pages);
	mem_cgroup_put(memcg);
3276
	return ERR_PTR(-ENOSPC);
3277 3278
}

3279
int alloc_bootmem_huge_page(struct hstate *h, int nid)
3280
	__attribute__ ((weak, alias("__alloc_bootmem_huge_page")));
3281
int __alloc_bootmem_huge_page(struct hstate *h, int nid)
3282
{
3283
	struct huge_bootmem_page *m = NULL; /* initialize for clang */
3284
	int nr_nodes, node = nid;
3285

3286 3287 3288 3289 3290 3291 3292 3293 3294
	/* do node specific alloc */
	if (nid != NUMA_NO_NODE) {
		m = memblock_alloc_try_nid_raw(huge_page_size(h), huge_page_size(h),
				0, MEMBLOCK_ALLOC_ACCESSIBLE, nid);
		if (!m)
			return 0;
		goto found;
	}
	/* allocate from next node when distributing huge pages */
3295
	for_each_node_mask_to_alloc(&h->next_nid_to_alloc, nr_nodes, node, &node_states[N_MEMORY]) {
3296
		m = memblock_alloc_try_nid_raw(
3297
				huge_page_size(h), huge_page_size(h),
3298
				0, MEMBLOCK_ALLOC_ACCESSIBLE, node);
3299 3300 3301 3302 3303 3304 3305 3306
		/*
		 * Use the beginning of the huge page to store the
		 * huge_bootmem_page struct (until gather_bootmem
		 * puts them into the mem_map).
		 */
		if (!m)
			return 0;
		goto found;
3307 3308 3309
	}

found:
3310 3311 3312 3313 3314 3315 3316 3317 3318 3319

	/*
	 * Only initialize the head struct page in memmap_init_reserved_pages,
	 * rest of the struct pages will be initialized by the HugeTLB
	 * subsystem itself.
	 * The head struct page is used to get folio information by the HugeTLB
	 * subsystem like zone id and node id.
	 */
	memblock_reserved_mark_noinit(virt_to_phys((void *)m + PAGE_SIZE),
		huge_page_size(h) - PAGE_SIZE);
3320
	/* Put them into a private list first because mem_map is not up yet */
3321
	INIT_LIST_HEAD(&m->list);
3322
	list_add(&m->list, &huge_boot_pages[node]);
3323 3324 3325 3326
	m->hstate = h;
	return 1;
}

3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356
/* Initialize [start_page:end_page_number] tail struct pages of a hugepage */
static void __init hugetlb_folio_init_tail_vmemmap(struct folio *folio,
					unsigned long start_page_number,
					unsigned long end_page_number)
{
	enum zone_type zone = zone_idx(folio_zone(folio));
	int nid = folio_nid(folio);
	unsigned long head_pfn = folio_pfn(folio);
	unsigned long pfn, end_pfn = head_pfn + end_page_number;
	int ret;

	for (pfn = head_pfn + start_page_number; pfn < end_pfn; pfn++) {
		struct page *page = pfn_to_page(pfn);

		__init_single_page(page, pfn, zone, nid);
		prep_compound_tail((struct page *)folio, pfn - head_pfn);
		ret = page_ref_freeze(page, 1);
		VM_BUG_ON(!ret);
	}
}

static void __init hugetlb_folio_init_vmemmap(struct folio *folio,
					      struct hstate *h,
					      unsigned long nr_pages)
{
	int ret;

	/* Prepare folio head */
	__folio_clear_reserved(folio);
	__folio_set_head(folio);
3357
	ret = folio_ref_freeze(folio, 1);
3358 3359 3360 3361 3362 3363
	VM_BUG_ON(!ret);
	/* Initialize the necessary tail struct pages */
	hugetlb_folio_init_tail_vmemmap(folio, 1, nr_pages);
	prep_compound_head((struct page *)folio, huge_page_order(h));
}

3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384
static void __init prep_and_add_bootmem_folios(struct hstate *h,
					struct list_head *folio_list)
{
	unsigned long flags;
	struct folio *folio, *tmp_f;

	/* Send list for bulk vmemmap optimization processing */
	hugetlb_vmemmap_optimize_folios(h, folio_list);

	list_for_each_entry_safe(folio, tmp_f, folio_list, lru) {
		if (!folio_test_hugetlb_vmemmap_optimized(folio)) {
			/*
			 * If HVO fails, initialize all tail struct pages
			 * We do not worry about potential long lock hold
			 * time as this is early in boot and there should
			 * be no contention.
			 */
			hugetlb_folio_init_tail_vmemmap(folio,
					HUGETLB_VMEMMAP_RESERVE_PAGES,
					pages_per_huge_page(h));
		}
3385 3386
		/* Subdivide locks to achieve better parallel performance */
		spin_lock_irqsave(&hugetlb_lock, flags);
3387 3388
		__prep_account_new_huge_page(h, folio_nid(folio));
		enqueue_hugetlb_folio(h, folio);
3389
		spin_unlock_irqrestore(&hugetlb_lock, flags);
3390 3391 3392
	}
}

3393 3394
/*
 * Put bootmem huge pages into the standard lists after mem_map is up.
3395
 * Note: This only applies to gigantic (order > MAX_PAGE_ORDER) pages.
3396
 */
3397
static void __init gather_bootmem_prealloc_node(unsigned long nid)
3398
{
3399
	LIST_HEAD(folio_list);
3400
	struct huge_bootmem_page *m;
3401
	struct hstate *h = NULL, *prev_h = NULL;
3402

3403
	list_for_each_entry(m, &huge_boot_pages[nid], list) {
3404
		struct page *page = virt_to_page(m);
3405
		struct folio *folio = (void *)page;
3406 3407 3408 3409 3410 3411 3412

		h = m->hstate;
		/*
		 * It is possible to have multiple huge page sizes (hstates)
		 * in this list.  If so, process each size separately.
		 */
		if (h != prev_h && prev_h != NULL)
3413
			prep_and_add_bootmem_folios(prev_h, &folio_list);
3414
		prev_h = h;
3415

3416
		VM_BUG_ON(!hstate_is_gigantic(h));
3417
		WARN_ON(folio_ref_count(folio) != 1);
3418 3419 3420

		hugetlb_folio_init_vmemmap(folio, h,
					   HUGETLB_VMEMMAP_RESERVE_PAGES);
3421
		init_new_hugetlb_folio(h, folio);
3422
		list_add(&folio->lru, &folio_list);
3423

3424
		/*
3425 3426 3427
		 * We need to restore the 'stolen' pages to totalram_pages
		 * in order to fix confusing memory reports from free(1) and
		 * other side-effects, like CommitLimit going negative.
3428
		 */
3429
		adjust_managed_page_count(page, pages_per_huge_page(h));
3430
		cond_resched();
3431
	}
3432

3433
	prep_and_add_bootmem_folios(h, &folio_list);
3434
}
3435

3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460
static void __init gather_bootmem_prealloc_parallel(unsigned long start,
						    unsigned long end, void *arg)
{
	int nid;

	for (nid = start; nid < end; nid++)
		gather_bootmem_prealloc_node(nid);
}

static void __init gather_bootmem_prealloc(void)
{
	struct padata_mt_job job = {
		.thread_fn	= gather_bootmem_prealloc_parallel,
		.fn_arg		= NULL,
		.start		= 0,
		.size		= num_node_state(N_MEMORY),
		.align		= 1,
		.min_chunk	= 1,
		.max_threads	= num_node_state(N_MEMORY),
		.numa_aware	= true,
	};

	padata_do_multithreaded(&job);
}

3461 3462 3463 3464 3465 3466 3467 3468 3469 3470
static void __init hugetlb_hstate_alloc_pages_onenode(struct hstate *h, int nid)
{
	unsigned long i;
	char buf[32];

	for (i = 0; i < h->max_huge_pages_node[nid]; ++i) {
		if (hstate_is_gigantic(h)) {
			if (!alloc_bootmem_huge_page(h, nid))
				break;
		} else {
3471
			struct folio *folio;
3472 3473
			gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;

3474
			folio = alloc_fresh_hugetlb_folio(h, gfp_mask, nid,
3475
					&node_states[N_MEMORY], NULL);
3476
			if (!folio)
3477
				break;
3478
			free_huge_folio(folio); /* free it into the hugepage allocator */
3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490
		}
		cond_resched();
	}
	if (i == h->max_huge_pages_node[nid])
		return;

	string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32);
	pr_warn("HugeTLB: allocating %u of page size %s failed node%d.  Only allocated %lu hugepages.\n",
		h->max_huge_pages_node[nid], buf, nid, i);
	h->max_huge_pages -= (h->max_huge_pages_node[nid] - i);
	h->max_huge_pages_node[nid] = i;
}
3491

3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518
static bool __init hugetlb_hstate_alloc_pages_specific_nodes(struct hstate *h)
{
	int i;
	bool node_specific_alloc = false;

	for_each_online_node(i) {
		if (h->max_huge_pages_node[i] > 0) {
			hugetlb_hstate_alloc_pages_onenode(h, i);
			node_specific_alloc = true;
		}
	}

	return node_specific_alloc;
}

static void __init hugetlb_hstate_alloc_pages_errcheck(unsigned long allocated, struct hstate *h)
{
	if (allocated < h->max_huge_pages) {
		char buf[32];

		string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32);
		pr_warn("HugeTLB: allocating %lu of page size %s failed.  Only allocated %lu hugepages.\n",
			h->max_huge_pages, buf, allocated);
		h->max_huge_pages = allocated;
	}
}

3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542
static void __init hugetlb_pages_alloc_boot_node(unsigned long start, unsigned long end, void *arg)
{
	struct hstate *h = (struct hstate *)arg;
	int i, num = end - start;
	nodemask_t node_alloc_noretry;
	LIST_HEAD(folio_list);
	int next_node = first_online_node;

	/* Bit mask controlling how hard we retry per-node allocations.*/
	nodes_clear(node_alloc_noretry);

	for (i = 0; i < num; ++i) {
		struct folio *folio = alloc_pool_huge_folio(h, &node_states[N_MEMORY],
						&node_alloc_noretry, &next_node);
		if (!folio)
			break;

		list_move(&folio->lru, &folio_list);
		cond_resched();
	}

	prep_and_add_allocated_folios(h, &folio_list);
}

3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557
static unsigned long __init hugetlb_gigantic_pages_alloc_boot(struct hstate *h)
{
	unsigned long i;

	for (i = 0; i < h->max_huge_pages; ++i) {
		if (!alloc_bootmem_huge_page(h, NUMA_NO_NODE))
			break;
		cond_resched();
	}

	return i;
}

static unsigned long __init hugetlb_pages_alloc_boot(struct hstate *h)
{
3558 3559 3560 3561 3562
	struct padata_mt_job job = {
		.fn_arg		= h,
		.align		= 1,
		.numa_aware	= true
	};
3563

3564 3565 3566
	job.thread_fn	= hugetlb_pages_alloc_boot_node;
	job.start	= 0;
	job.size	= h->max_huge_pages;
3567

3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589
	/*
	 * job.max_threads is twice the num_node_state(N_MEMORY),
	 *
	 * Tests below indicate that a multiplier of 2 significantly improves
	 * performance, and although larger values also provide improvements,
	 * the gains are marginal.
	 *
	 * Therefore, choosing 2 as the multiplier strikes a good balance between
	 * enhancing parallel processing capabilities and maintaining efficient
	 * resource management.
	 *
	 * +------------+-------+-------+-------+-------+-------+
	 * | multiplier |   1   |   2   |   3   |   4   |   5   |
	 * +------------+-------+-------+-------+-------+-------+
	 * | 256G 2node | 358ms | 215ms | 157ms | 134ms | 126ms |
	 * | 2T   4node | 979ms | 679ms | 543ms | 489ms | 481ms |
	 * | 50G  2node | 71ms  | 44ms  | 37ms  | 30ms  | 31ms  |
	 * +------------+-------+-------+-------+-------+-------+
	 */
	job.max_threads	= num_node_state(N_MEMORY) * 2;
	job.min_chunk	= h->max_huge_pages / num_node_state(N_MEMORY) / 2;
	padata_do_multithreaded(&job);
3590

3591
	return h->nr_huge_pages;
3592 3593
}

3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604
/*
 * NOTE: this routine is called in different contexts for gigantic and
 * non-gigantic pages.
 * - For gigantic pages, this is called early in the boot process and
 *   pages are allocated from memblock allocated or something similar.
 *   Gigantic pages are actually added to pools later with the routine
 *   gather_bootmem_prealloc.
 * - For non-gigantic pages, this is called later in the boot process after
 *   all of mm is up and functional.  Pages are allocated from buddy and
 *   then added to hugetlb pools.
 */
3605
static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
Linus Torvalds's avatar
Linus Torvalds committed
3606
{
3607
	unsigned long allocated;
3608
	static bool initialized __initdata;
3609 3610 3611 3612 3613 3614 3615

	/* skip gigantic hugepages allocation if hugetlb_cma enabled */
	if (hstate_is_gigantic(h) && hugetlb_cma_size) {
		pr_warn_once("HugeTLB: hugetlb_cma is enabled, skip boot time allocation\n");
		return;
	}

3616 3617 3618 3619 3620 3621 3622 3623 3624
	/* hugetlb_hstate_alloc_pages will be called many times, initialize huge_boot_pages once */
	if (!initialized) {
		int i = 0;

		for (i = 0; i < MAX_NUMNODES; i++)
			INIT_LIST_HEAD(&huge_boot_pages[i]);
		initialized = true;
	}

3625
	/* do node specific alloc */
3626
	if (hugetlb_hstate_alloc_pages_specific_nodes(h))
3627 3628 3629
		return;

	/* below will do all node balanced alloc */
3630 3631 3632 3633
	if (hstate_is_gigantic(h))
		allocated = hugetlb_gigantic_pages_alloc_boot(h);
	else
		allocated = hugetlb_pages_alloc_boot(h);
3634

3635
	hugetlb_hstate_alloc_pages_errcheck(allocated, h);
3636 3637 3638 3639
}

static void __init hugetlb_init_hstates(void)
{
3640
	struct hstate *h, *h2;
3641 3642

	for_each_hstate(h) {
3643
		/* oversize hugepages were init'ed in early boot */
3644
		if (!hstate_is_gigantic(h))
3645
			hugetlb_hstate_alloc_pages(h);
3646 3647 3648 3649 3650 3651

		/*
		 * Set demote order for each hstate.  Note that
		 * h->demote_order is initially 0.
		 * - We can not demote gigantic pages if runtime freeing
		 *   is not supported, so skip this.
3652 3653
		 * - If CMA allocation is possible, we can not demote
		 *   HUGETLB_PAGE_ORDER or smaller size pages.
3654 3655 3656
		 */
		if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
			continue;
3657 3658
		if (hugetlb_cma_size && h->order <= HUGETLB_PAGE_ORDER)
			continue;
3659 3660 3661 3662 3663 3664 3665
		for_each_hstate(h2) {
			if (h2 == h)
				continue;
			if (h2->order < h->order &&
			    h2->order > h->demote_order)
				h->demote_order = h2->order;
		}
3666 3667 3668 3669 3670 3671 3672 3673
	}
}

static void __init report_hugepages(void)
{
	struct hstate *h;

	for_each_hstate(h) {
Andi Kleen's avatar
Andi Kleen committed
3674
		char buf[32];
3675 3676

		string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32);
3677
		pr_info("HugeTLB: registered %s page size, pre-allocated %ld pages\n",
3678
			buf, h->free_huge_pages);
3679 3680
		pr_info("HugeTLB: %d KiB vmemmap can be freed for a %s page\n",
			hugetlb_vmemmap_optimizable_size(h) / SZ_1K, buf);
3681 3682 3683
	}
}

Linus Torvalds's avatar
Linus Torvalds committed
3684
#ifdef CONFIG_HIGHMEM
3685 3686
static void try_to_free_low(struct hstate *h, unsigned long count,
						nodemask_t *nodes_allowed)
Linus Torvalds's avatar
Linus Torvalds committed
3687
{
3688
	int i;
3689
	LIST_HEAD(page_list);
3690

3691
	lockdep_assert_held(&hugetlb_lock);
3692
	if (hstate_is_gigantic(h))
3693 3694
		return;

3695 3696 3697
	/*
	 * Collect pages to be freed on a list, and free after dropping lock
	 */
3698
	for_each_node_mask(i, *nodes_allowed) {
3699
		struct folio *folio, *next;
3700
		struct list_head *freel = &h->hugepage_freelists[i];
3701
		list_for_each_entry_safe(folio, next, freel, lru) {
3702
			if (count >= h->nr_huge_pages)
3703
				goto out;
3704
			if (folio_test_highmem(folio))
Linus Torvalds's avatar
Linus Torvalds committed
3705
				continue;
3706 3707
			remove_hugetlb_folio(h, folio, false);
			list_add(&folio->lru, &page_list);
Linus Torvalds's avatar
Linus Torvalds committed
3708 3709
		}
	}
3710 3711

out:
3712
	spin_unlock_irq(&hugetlb_lock);
3713
	update_and_free_pages_bulk(h, &page_list);
3714
	spin_lock_irq(&hugetlb_lock);
Linus Torvalds's avatar
Linus Torvalds committed
3715 3716
}
#else
3717 3718
static inline void try_to_free_low(struct hstate *h, unsigned long count,
						nodemask_t *nodes_allowed)
Linus Torvalds's avatar
Linus Torvalds committed
3719 3720 3721 3722
{
}
#endif

3723 3724 3725 3726 3727
/*
 * Increment or decrement surplus_huge_pages.  Keep node-specific counters
 * balanced by operating on them in a round-robin fashion.
 * Returns 1 if an adjustment was made.
 */
3728 3729
static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed,
				int delta)
3730
{
3731
	int nr_nodes, node;
3732

3733
	lockdep_assert_held(&hugetlb_lock);
3734 3735
	VM_BUG_ON(delta != -1 && delta != 1);

3736
	if (delta < 0) {
3737
		for_each_node_mask_to_alloc(&h->next_nid_to_alloc, nr_nodes, node, nodes_allowed) {
3738 3739
			if (h->surplus_huge_pages_node[node])
				goto found;
3740
		}
3741 3742 3743 3744 3745
	} else {
		for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
			if (h->surplus_huge_pages_node[node] <
					h->nr_huge_pages_node[node])
				goto found;
3746
		}
3747 3748
	}
	return 0;
3749

3750 3751 3752 3753
found:
	h->surplus_huge_pages += delta;
	h->surplus_huge_pages_node[node] += delta;
	return 1;
3754 3755
}

3756
#define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
3757
static int set_max_huge_pages(struct hstate *h, unsigned long count, int nid,
3758
			      nodemask_t *nodes_allowed)
Linus Torvalds's avatar
Linus Torvalds committed
3759
{
3760 3761 3762
	unsigned long min_count;
	unsigned long allocated;
	struct folio *folio;
3763
	LIST_HEAD(page_list);
3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774
	NODEMASK_ALLOC(nodemask_t, node_alloc_noretry, GFP_KERNEL);

	/*
	 * Bit mask controlling how hard we retry per-node allocations.
	 * If we can not allocate the bit mask, do not attempt to allocate
	 * the requested huge pages.
	 */
	if (node_alloc_noretry)
		nodes_clear(*node_alloc_noretry);
	else
		return -ENOMEM;
Linus Torvalds's avatar
Linus Torvalds committed
3775

3776 3777 3778 3779 3780
	/*
	 * resize_lock mutex prevents concurrent adjustments to number of
	 * pages in hstate via the proc/sysfs interfaces.
	 */
	mutex_lock(&h->resize_lock);
3781
	flush_free_hpage_work(h);
3782
	spin_lock_irq(&hugetlb_lock);
3783

3784 3785 3786 3787 3788 3789 3790 3791 3792
	/*
	 * Check for a node specific request.
	 * Changing node specific huge page count may require a corresponding
	 * change to the global count.  In any case, the passed node mask
	 * (nodes_allowed) will restrict alloc/free to the specified node.
	 */
	if (nid != NUMA_NO_NODE) {
		unsigned long old_count = count;

3793 3794 3795
		count += persistent_huge_pages(h) -
			 (h->nr_huge_pages_node[nid] -
			  h->surplus_huge_pages_node[nid]);
3796 3797 3798 3799 3800 3801 3802 3803 3804 3805
		/*
		 * User may have specified a large count value which caused the
		 * above calculation to overflow.  In this case, they wanted
		 * to allocate as many huge pages as possible.  Set count to
		 * largest possible value to align with their intention.
		 */
		if (count < old_count)
			count = ULONG_MAX;
	}

3806 3807 3808 3809 3810 3811 3812 3813 3814
	/*
	 * Gigantic pages runtime allocation depend on the capability for large
	 * page range allocation.
	 * If the system does not provide this feature, return an error when
	 * the user tries to allocate gigantic pages but let the user free the
	 * boottime allocated gigantic pages.
	 */
	if (hstate_is_gigantic(h) && !IS_ENABLED(CONFIG_CONTIG_ALLOC)) {
		if (count > persistent_huge_pages(h)) {
3815
			spin_unlock_irq(&hugetlb_lock);
3816
			mutex_unlock(&h->resize_lock);
3817
			NODEMASK_FREE(node_alloc_noretry);
3818 3819 3820 3821
			return -EINVAL;
		}
		/* Fall through to decrease pool */
	}
3822

3823 3824 3825 3826
	/*
	 * Increase the pool size
	 * First take pages out of surplus state.  Then make up the
	 * remaining difference by allocating fresh huge pages.
3827
	 *
3828
	 * We might race with alloc_surplus_hugetlb_folio() here and be unable
3829 3830 3831 3832
	 * to convert a surplus huge page to a normal huge page. That is
	 * not critical, though, it just means the overall size of the
	 * pool might be one hugepage larger than it needs to be, but
	 * within all the constraints specified by the sysctls.
3833
	 */
3834
	while (h->surplus_huge_pages && count > persistent_huge_pages(h)) {
3835
		if (!adjust_pool_surplus(h, nodes_allowed, -1))
3836 3837 3838
			break;
	}

3839 3840
	allocated = 0;
	while (count > (persistent_huge_pages(h) + allocated)) {
3841 3842
		/*
		 * If this allocation races such that we no longer need the
3843
		 * page, free_huge_folio will handle it by freeing the page
3844 3845
		 * and reducing the surplus.
		 */
3846
		spin_unlock_irq(&hugetlb_lock);
3847 3848 3849 3850

		/* yield cpu to avoid soft lockup */
		cond_resched();

3851
		folio = alloc_pool_huge_folio(h, nodes_allowed,
3852 3853
						node_alloc_noretry,
						&h->next_nid_to_alloc);
3854 3855 3856
		if (!folio) {
			prep_and_add_allocated_folios(h, &page_list);
			spin_lock_irq(&hugetlb_lock);
3857
			goto out;
3858 3859 3860 3861
		}

		list_add(&folio->lru, &page_list);
		allocated++;
3862

3863
		/* Bail for signals. Probably ctrl-c from user */
3864 3865 3866
		if (signal_pending(current)) {
			prep_and_add_allocated_folios(h, &page_list);
			spin_lock_irq(&hugetlb_lock);
3867
			goto out;
3868 3869 3870 3871 3872 3873 3874 3875 3876 3877
		}

		spin_lock_irq(&hugetlb_lock);
	}

	/* Add allocated pages to the pool */
	if (!list_empty(&page_list)) {
		spin_unlock_irq(&hugetlb_lock);
		prep_and_add_allocated_folios(h, &page_list);
		spin_lock_irq(&hugetlb_lock);
3878 3879 3880 3881 3882 3883 3884 3885
	}

	/*
	 * Decrease the pool size
	 * First return free pages to the buddy allocator (being careful
	 * to keep enough around to satisfy reservations).  Then place
	 * pages into surplus state as needed so the pool will shrink
	 * to the desired size as pages become free.
3886 3887 3888 3889
	 *
	 * By placing pages into the surplus state independent of the
	 * overcommit value, we are allowing the surplus pool size to
	 * exceed overcommit. There are few sane options here. Since
3890
	 * alloc_surplus_hugetlb_folio() is checking the global counter,
3891 3892 3893
	 * though, we'll note that we're not allowed to exceed surplus
	 * and won't grow the pool anywhere else. Not until one of the
	 * sysctls are changed, or the surplus pages go out of use.
3894
	 */
3895
	min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages;
3896
	min_count = max(count, min_count);
3897
	try_to_free_low(h, min_count, nodes_allowed);
3898 3899 3900 3901

	/*
	 * Collect pages to be removed on list without dropping lock
	 */
3902
	while (min_count < persistent_huge_pages(h)) {
3903 3904
		folio = remove_pool_hugetlb_folio(h, nodes_allowed, 0);
		if (!folio)
Linus Torvalds's avatar
Linus Torvalds committed
3905
			break;
3906

3907
		list_add(&folio->lru, &page_list);
Linus Torvalds's avatar
Linus Torvalds committed
3908
	}
3909
	/* free the pages after dropping lock */
3910
	spin_unlock_irq(&hugetlb_lock);
3911
	update_and_free_pages_bulk(h, &page_list);
3912
	flush_free_hpage_work(h);
3913
	spin_lock_irq(&hugetlb_lock);
3914

3915
	while (count < persistent_huge_pages(h)) {
3916
		if (!adjust_pool_surplus(h, nodes_allowed, 1))
3917 3918 3919
			break;
	}
out:
3920
	h->max_huge_pages = persistent_huge_pages(h);
3921
	spin_unlock_irq(&hugetlb_lock);
3922
	mutex_unlock(&h->resize_lock);
3923

3924 3925
	NODEMASK_FREE(node_alloc_noretry);

3926
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
3927 3928
}

3929
static int demote_free_hugetlb_folio(struct hstate *h, struct folio *folio)
3930
{
3931
	int i, nid = folio_nid(folio);
3932
	struct hstate *target_hstate;
3933
	struct page *subpage;
3934
	struct folio *inner_folio;
3935 3936 3937 3938
	int rc = 0;

	target_hstate = size_to_hstate(PAGE_SIZE << h->demote_order);

3939
	remove_hugetlb_folio_for_demote(h, folio, false);
3940 3941
	spin_unlock_irq(&hugetlb_lock);

3942 3943 3944
	/*
	 * If vmemmap already existed for folio, the remove routine above would
	 * have cleared the hugetlb folio flag.  Hence the folio is technically
3945
	 * no longer a hugetlb folio.  hugetlb_vmemmap_restore_folio can only be
3946 3947 3948
	 * passed hugetlb folios and will BUG otherwise.
	 */
	if (folio_test_hugetlb(folio)) {
3949
		rc = hugetlb_vmemmap_restore_folio(h, folio);
3950 3951 3952 3953 3954 3955 3956
		if (rc) {
			/* Allocation of vmemmmap failed, we can not demote folio */
			spin_lock_irq(&hugetlb_lock);
			folio_ref_unfreeze(folio, 1);
			add_hugetlb_folio(h, folio, false);
			return rc;
		}
3957 3958 3959
	}

	/*
3960
	 * Use destroy_compound_hugetlb_folio_for_demote for all huge page
3961
	 * sizes as it will not ref count folios.
3962
	 */
3963
	destroy_compound_hugetlb_folio_for_demote(folio, huge_page_order(h));
3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975

	/*
	 * Taking target hstate mutex synchronizes with set_max_huge_pages.
	 * Without the mutex, pages added to target hstate could be marked
	 * as surplus.
	 *
	 * Note that we already hold h->resize_lock.  To prevent deadlock,
	 * use the convention of always taking larger size hstate mutex first.
	 */
	mutex_lock(&target_hstate->resize_lock);
	for (i = 0; i < pages_per_huge_page(h);
				i += pages_per_huge_page(target_hstate)) {
3976 3977
		subpage = folio_page(folio, i);
		inner_folio = page_folio(subpage);
3978
		if (hstate_is_gigantic(target_hstate))
3979
			prep_compound_gigantic_folio_for_demote(inner_folio,
3980 3981
							target_hstate->order);
		else
3982
			prep_compound_page(subpage, target_hstate->order);
3983 3984
		folio_change_private(inner_folio, NULL);
		prep_new_hugetlb_folio(target_hstate, inner_folio, nid);
3985
		free_huge_folio(inner_folio);
3986 3987 3988 3989 3990 3991 3992 3993 3994 3995
	}
	mutex_unlock(&target_hstate->resize_lock);

	spin_lock_irq(&hugetlb_lock);

	/*
	 * Not absolutely necessary, but for consistency update max_huge_pages
	 * based on pool changes for the demoted page.
	 */
	h->max_huge_pages--;
3996 3997
	target_hstate->max_huge_pages +=
		pages_per_huge_page(h) / pages_per_huge_page(target_hstate);
3998 3999 4000 4001

	return rc;
}

4002 4003 4004
static int demote_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed)
	__must_hold(&hugetlb_lock)
{
4005
	int nr_nodes, node;
4006
	struct folio *folio;
4007 4008 4009 4010 4011 4012 4013 4014 4015

	lockdep_assert_held(&hugetlb_lock);

	/* We should never get here if no demote order */
	if (!h->demote_order) {
		pr_warn("HugeTLB: NULL demote order passed to demote_pool_huge_page.\n");
		return -EINVAL;		/* internal error */
	}

4016
	for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
4017 4018
		list_for_each_entry(folio, &h->hugepage_freelists[node], lru) {
			if (folio_test_hwpoison(folio))
4019
				continue;
4020
			return demote_free_hugetlb_folio(h, folio);
4021 4022 4023
		}
	}

4024 4025 4026 4027 4028
	/*
	 * Only way to get here is if all pages on free lists are poisoned.
	 * Return -EBUSY so that caller will not retry.
	 */
	return -EBUSY;
4029 4030
}

4031 4032 4033
#define HSTATE_ATTR_RO(_name) \
	static struct kobj_attribute _name##_attr = __ATTR_RO(_name)

4034 4035 4036
#define HSTATE_ATTR_WO(_name) \
	static struct kobj_attribute _name##_attr = __ATTR_WO(_name)

4037
#define HSTATE_ATTR(_name) \
4038
	static struct kobj_attribute _name##_attr = __ATTR_RW(_name)
4039 4040 4041 4042

static struct kobject *hugepages_kobj;
static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];

4043 4044 4045
static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);

static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp)
4046 4047
{
	int i;
4048

4049
	for (i = 0; i < HUGE_MAX_HSTATE; i++)
4050 4051 4052
		if (hstate_kobjs[i] == kobj) {
			if (nidp)
				*nidp = NUMA_NO_NODE;
4053
			return &hstates[i];
4054 4055 4056
		}

	return kobj_to_node_hstate(kobj, nidp);
4057 4058
}

4059
static ssize_t nr_hugepages_show_common(struct kobject *kobj,
4060 4061
					struct kobj_attribute *attr, char *buf)
{
4062 4063 4064 4065 4066 4067 4068 4069 4070 4071
	struct hstate *h;
	unsigned long nr_huge_pages;
	int nid;

	h = kobj_to_hstate(kobj, &nid);
	if (nid == NUMA_NO_NODE)
		nr_huge_pages = h->nr_huge_pages;
	else
		nr_huge_pages = h->nr_huge_pages_node[nid];

4072
	return sysfs_emit(buf, "%lu\n", nr_huge_pages);
4073
}
4074

4075 4076 4077
static ssize_t __nr_hugepages_store_common(bool obey_mempolicy,
					   struct hstate *h, int nid,
					   unsigned long count, size_t len)
4078 4079
{
	int err;
4080
	nodemask_t nodes_allowed, *n_mask;
4081

4082 4083
	if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
		return -EINVAL;
4084

4085 4086 4087 4088 4089
	if (nid == NUMA_NO_NODE) {
		/*
		 * global hstate attribute
		 */
		if (!(obey_mempolicy &&
4090 4091 4092 4093 4094
				init_nodemask_of_mempolicy(&nodes_allowed)))
			n_mask = &node_states[N_MEMORY];
		else
			n_mask = &nodes_allowed;
	} else {
4095
		/*
4096 4097
		 * Node specific request.  count adjustment happens in
		 * set_max_huge_pages() after acquiring hugetlb_lock.
4098
		 */
4099 4100
		init_nodemask_of_node(&nodes_allowed, nid);
		n_mask = &nodes_allowed;
4101
	}
4102

4103
	err = set_max_huge_pages(h, count, nid, n_mask);
4104

4105
	return err ? err : len;
4106 4107
}

4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 4122 4123 4124
static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
					 struct kobject *kobj, const char *buf,
					 size_t len)
{
	struct hstate *h;
	unsigned long count;
	int nid;
	int err;

	err = kstrtoul(buf, 10, &count);
	if (err)
		return err;

	h = kobj_to_hstate(kobj, &nid);
	return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len);
}

4125 4126 4127 4128 4129 4130 4131 4132 4133
static ssize_t nr_hugepages_show(struct kobject *kobj,
				       struct kobj_attribute *attr, char *buf)
{
	return nr_hugepages_show_common(kobj, attr, buf);
}

static ssize_t nr_hugepages_store(struct kobject *kobj,
	       struct kobj_attribute *attr, const char *buf, size_t len)
{
4134
	return nr_hugepages_store_common(false, kobj, buf, len);
4135 4136 4137
}
HSTATE_ATTR(nr_hugepages);

4138 4139 4140 4141 4142 4143 4144
#ifdef CONFIG_NUMA

/*
 * hstate attribute for optionally mempolicy-based constraint on persistent
 * huge page alloc/free.
 */
static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
4145 4146
					   struct kobj_attribute *attr,
					   char *buf)
4147 4148 4149 4150 4151 4152 4153
{
	return nr_hugepages_show_common(kobj, attr, buf);
}

static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
	       struct kobj_attribute *attr, const char *buf, size_t len)
{
4154
	return nr_hugepages_store_common(true, kobj, buf, len);
4155 4156 4157 4158 4159
}
HSTATE_ATTR(nr_hugepages_mempolicy);
#endif


4160 4161 4162
static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
					struct kobj_attribute *attr, char *buf)
{
4163
	struct hstate *h = kobj_to_hstate(kobj, NULL);
4164
	return sysfs_emit(buf, "%lu\n", h->nr_overcommit_huge_pages);
4165
}
4166

4167 4168 4169 4170 4171
static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
		struct kobj_attribute *attr, const char *buf, size_t count)
{
	int err;
	unsigned long input;
4172
	struct hstate *h = kobj_to_hstate(kobj, NULL);
4173

4174
	if (hstate_is_gigantic(h))
4175 4176
		return -EINVAL;

4177
	err = kstrtoul(buf, 10, &input);
4178
	if (err)
4179
		return err;
4180

4181
	spin_lock_irq(&hugetlb_lock);
4182
	h->nr_overcommit_huge_pages = input;
4183
	spin_unlock_irq(&hugetlb_lock);
4184 4185 4186 4187 4188 4189 4190 4191

	return count;
}
HSTATE_ATTR(nr_overcommit_hugepages);

static ssize_t free_hugepages_show(struct kobject *kobj,
					struct kobj_attribute *attr, char *buf)
{
4192 4193 4194 4195 4196 4197 4198 4199 4200 4201
	struct hstate *h;
	unsigned long free_huge_pages;
	int nid;

	h = kobj_to_hstate(kobj, &nid);
	if (nid == NUMA_NO_NODE)
		free_huge_pages = h->free_huge_pages;
	else
		free_huge_pages = h->free_huge_pages_node[nid];

4202
	return sysfs_emit(buf, "%lu\n", free_huge_pages);
4203 4204 4205 4206 4207 4208
}
HSTATE_ATTR_RO(free_hugepages);

static ssize_t resv_hugepages_show(struct kobject *kobj,
					struct kobj_attribute *attr, char *buf)
{
4209
	struct hstate *h = kobj_to_hstate(kobj, NULL);
4210
	return sysfs_emit(buf, "%lu\n", h->resv_huge_pages);
4211 4212 4213 4214 4215 4216
}
HSTATE_ATTR_RO(resv_hugepages);

static ssize_t surplus_hugepages_show(struct kobject *kobj,
					struct kobj_attribute *attr, char *buf)
{
4217 4218 4219 4220 4221 4222 4223 4224 4225 4226
	struct hstate *h;
	unsigned long surplus_huge_pages;
	int nid;

	h = kobj_to_hstate(kobj, &nid);
	if (nid == NUMA_NO_NODE)
		surplus_huge_pages = h->surplus_huge_pages;
	else
		surplus_huge_pages = h->surplus_huge_pages_node[nid];

4227
	return sysfs_emit(buf, "%lu\n", surplus_huge_pages);
4228 4229 4230
}
HSTATE_ATTR_RO(surplus_hugepages);

4231 4232 4233 4234 4235 4236 4237
static ssize_t demote_store(struct kobject *kobj,
	       struct kobj_attribute *attr, const char *buf, size_t len)
{
	unsigned long nr_demote;
	unsigned long nr_available;
	nodemask_t nodes_allowed, *n_mask;
	struct hstate *h;
4238
	int err;
4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288
	int nid;

	err = kstrtoul(buf, 10, &nr_demote);
	if (err)
		return err;
	h = kobj_to_hstate(kobj, &nid);

	if (nid != NUMA_NO_NODE) {
		init_nodemask_of_node(&nodes_allowed, nid);
		n_mask = &nodes_allowed;
	} else {
		n_mask = &node_states[N_MEMORY];
	}

	/* Synchronize with other sysfs operations modifying huge pages */
	mutex_lock(&h->resize_lock);
	spin_lock_irq(&hugetlb_lock);

	while (nr_demote) {
		/*
		 * Check for available pages to demote each time thorough the
		 * loop as demote_pool_huge_page will drop hugetlb_lock.
		 */
		if (nid != NUMA_NO_NODE)
			nr_available = h->free_huge_pages_node[nid];
		else
			nr_available = h->free_huge_pages;
		nr_available -= h->resv_huge_pages;
		if (!nr_available)
			break;

		err = demote_pool_huge_page(h, n_mask);
		if (err)
			break;

		nr_demote--;
	}

	spin_unlock_irq(&hugetlb_lock);
	mutex_unlock(&h->resize_lock);

	if (err)
		return err;
	return len;
}
HSTATE_ATTR_WO(demote);

static ssize_t demote_size_show(struct kobject *kobj,
					struct kobj_attribute *attr, char *buf)
{
4289
	struct hstate *h = kobj_to_hstate(kobj, NULL);
4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308
	unsigned long demote_size = (PAGE_SIZE << h->demote_order) / SZ_1K;

	return sysfs_emit(buf, "%lukB\n", demote_size);
}

static ssize_t demote_size_store(struct kobject *kobj,
					struct kobj_attribute *attr,
					const char *buf, size_t count)
{
	struct hstate *h, *demote_hstate;
	unsigned long demote_size;
	unsigned int demote_order;

	demote_size = (unsigned long)memparse(buf, NULL);

	demote_hstate = size_to_hstate(demote_size);
	if (!demote_hstate)
		return -EINVAL;
	demote_order = demote_hstate->order;
4309 4310
	if (demote_order < HUGETLB_PAGE_ORDER)
		return -EINVAL;
4311 4312

	/* demote order must be smaller than hstate order */
4313
	h = kobj_to_hstate(kobj, NULL);
4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325
	if (demote_order >= h->order)
		return -EINVAL;

	/* resize_lock synchronizes access to demote size and writes */
	mutex_lock(&h->resize_lock);
	h->demote_order = demote_order;
	mutex_unlock(&h->resize_lock);

	return count;
}
HSTATE_ATTR(demote_size);

4326 4327 4328 4329 4330 4331
static struct attribute *hstate_attrs[] = {
	&nr_hugepages_attr.attr,
	&nr_overcommit_hugepages_attr.attr,
	&free_hugepages_attr.attr,
	&resv_hugepages_attr.attr,
	&surplus_hugepages_attr.attr,
4332 4333 4334
#ifdef CONFIG_NUMA
	&nr_hugepages_mempolicy_attr.attr,
#endif
4335 4336 4337
	NULL,
};

4338
static const struct attribute_group hstate_attr_group = {
4339 4340 4341
	.attrs = hstate_attrs,
};

4342 4343 4344 4345 4346 4347 4348 4349 4350 4351
static struct attribute *hstate_demote_attrs[] = {
	&demote_size_attr.attr,
	&demote_attr.attr,
	NULL,
};

static const struct attribute_group hstate_demote_attr_group = {
	.attrs = hstate_demote_attrs,
};

4352 4353
static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent,
				    struct kobject **hstate_kobjs,
4354
				    const struct attribute_group *hstate_attr_group)
4355 4356
{
	int retval;
4357
	int hi = hstate_index(h);
4358

4359 4360
	hstate_kobjs[hi] = kobject_create_and_add(h->name, parent);
	if (!hstate_kobjs[hi])
4361 4362
		return -ENOMEM;

4363
	retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group);
4364
	if (retval) {
4365
		kobject_put(hstate_kobjs[hi]);
4366
		hstate_kobjs[hi] = NULL;
4367
		return retval;
4368
	}
4369

4370
	if (h->demote_order) {
4371 4372 4373
		retval = sysfs_create_group(hstate_kobjs[hi],
					    &hstate_demote_attr_group);
		if (retval) {
4374
			pr_warn("HugeTLB unable to create demote interfaces for %s\n", h->name);
4375 4376 4377 4378 4379
			sysfs_remove_group(hstate_kobjs[hi], hstate_attr_group);
			kobject_put(hstate_kobjs[hi]);
			hstate_kobjs[hi] = NULL;
			return retval;
		}
4380 4381
	}

4382
	return 0;
4383 4384
}

4385
#ifdef CONFIG_NUMA
4386
static bool hugetlb_sysfs_initialized __ro_after_init;
4387 4388 4389

/*
 * node_hstate/s - associate per node hstate attributes, via their kobjects,
4390 4391 4392
 * with node devices in node_devices[] using a parallel array.  The array
 * index of a node device or _hstate == node id.
 * This is here to avoid any static dependency of the node device driver, in
4393 4394 4395 4396 4397 4398
 * the base kernel, on the hugetlb module.
 */
struct node_hstate {
	struct kobject		*hugepages_kobj;
	struct kobject		*hstate_kobjs[HUGE_MAX_HSTATE];
};
4399
static struct node_hstate node_hstates[MAX_NUMNODES];
4400 4401

/*
4402
 * A subset of global hstate attributes for node devices
4403 4404 4405 4406 4407 4408 4409 4410
 */
static struct attribute *per_node_hstate_attrs[] = {
	&nr_hugepages_attr.attr,
	&free_hugepages_attr.attr,
	&surplus_hugepages_attr.attr,
	NULL,
};

4411
static const struct attribute_group per_node_hstate_attr_group = {
4412 4413 4414 4415
	.attrs = per_node_hstate_attrs,
};

/*
4416
 * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438
 * Returns node id via non-NULL nidp.
 */
static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
{
	int nid;

	for (nid = 0; nid < nr_node_ids; nid++) {
		struct node_hstate *nhs = &node_hstates[nid];
		int i;
		for (i = 0; i < HUGE_MAX_HSTATE; i++)
			if (nhs->hstate_kobjs[i] == kobj) {
				if (nidp)
					*nidp = nid;
				return &hstates[i];
			}
	}

	BUG();
	return NULL;
}

/*
4439
 * Unregister hstate attributes from a single node device.
4440 4441
 * No-op if no hstate attributes attached.
 */
4442
void hugetlb_unregister_node(struct node *node)
4443 4444
{
	struct hstate *h;
4445
	struct node_hstate *nhs = &node_hstates[node->dev.id];
4446 4447

	if (!nhs->hugepages_kobj)
4448
		return;		/* no hstate attributes */
4449

4450 4451
	for_each_hstate(h) {
		int idx = hstate_index(h);
4452 4453 4454 4455 4456 4457 4458 4459 4460
		struct kobject *hstate_kobj = nhs->hstate_kobjs[idx];

		if (!hstate_kobj)
			continue;
		if (h->demote_order)
			sysfs_remove_group(hstate_kobj, &hstate_demote_attr_group);
		sysfs_remove_group(hstate_kobj, &per_node_hstate_attr_group);
		kobject_put(hstate_kobj);
		nhs->hstate_kobjs[idx] = NULL;
4461
	}
4462 4463 4464 4465 4466 4467 4468

	kobject_put(nhs->hugepages_kobj);
	nhs->hugepages_kobj = NULL;
}


/*
4469
 * Register hstate attributes for a single node device.
4470 4471
 * No-op if attributes already registered.
 */
4472
void hugetlb_register_node(struct node *node)
4473 4474
{
	struct hstate *h;
4475
	struct node_hstate *nhs = &node_hstates[node->dev.id];
4476 4477
	int err;

4478 4479 4480
	if (!hugetlb_sysfs_initialized)
		return;

4481 4482 4483 4484
	if (nhs->hugepages_kobj)
		return;		/* already allocated */

	nhs->hugepages_kobj = kobject_create_and_add("hugepages",
4485
							&node->dev.kobj);
4486 4487 4488 4489 4490 4491 4492 4493
	if (!nhs->hugepages_kobj)
		return;

	for_each_hstate(h) {
		err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj,
						nhs->hstate_kobjs,
						&per_node_hstate_attr_group);
		if (err) {
4494
			pr_err("HugeTLB: Unable to add hstate %s for node %d\n",
4495
				h->name, node->dev.id);
4496 4497 4498 4499 4500 4501 4502
			hugetlb_unregister_node(node);
			break;
		}
	}
}

/*
4503
 * hugetlb init time:  register hstate attributes for all registered node
4504 4505
 * devices of nodes that have memory.  All on-line nodes should have
 * registered their associated device by this time.
4506
 */
4507
static void __init hugetlb_register_all_nodes(void)
4508 4509 4510
{
	int nid;

4511
	for_each_online_node(nid)
4512
		hugetlb_register_node(node_devices[nid]);
4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524 4525 4526 4527
}
#else	/* !CONFIG_NUMA */

static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
{
	BUG();
	if (nidp)
		*nidp = -1;
	return NULL;
}

static void hugetlb_register_all_nodes(void) { }

#endif

4528 4529 4530 4531 4532 4533 4534 4535
#ifdef CONFIG_CMA
static void __init hugetlb_cma_check(void);
#else
static inline __init void hugetlb_cma_check(void)
{
}
#endif

4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547 4548 4549 4550 4551 4552 4553 4554 4555 4556 4557
static void __init hugetlb_sysfs_init(void)
{
	struct hstate *h;
	int err;

	hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
	if (!hugepages_kobj)
		return;

	for_each_hstate(h) {
		err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
					 hstate_kobjs, &hstate_attr_group);
		if (err)
			pr_err("HugeTLB: Unable to add hstate %s", h->name);
	}

#ifdef CONFIG_NUMA
	hugetlb_sysfs_initialized = true;
#endif
	hugetlb_register_all_nodes();
}

4558 4559 4560 4561 4562 4563
#ifdef CONFIG_SYSCTL
static void hugetlb_sysctl_init(void);
#else
static inline void hugetlb_sysctl_init(void) { }
#endif

4564 4565
static int __init hugetlb_init(void)
{
4566 4567
	int i;

4568 4569 4570
	BUILD_BUG_ON(sizeof_field(struct page, private) * BITS_PER_BYTE <
			__NR_HPAGEFLAGS);

4571 4572 4573
	if (!hugepages_supported()) {
		if (hugetlb_max_hstate || default_hstate_max_huge_pages)
			pr_warn("HugeTLB: huge pages not supported, ignoring associated command-line parameters\n");
4574
		return 0;
4575
	}
4576

4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588 4589 4590 4591 4592 4593 4594 4595 4596 4597 4598 4599 4600 4601 4602 4603 4604
	/*
	 * Make sure HPAGE_SIZE (HUGETLB_PAGE_ORDER) hstate exists.  Some
	 * architectures depend on setup being done here.
	 */
	hugetlb_add_hstate(HUGETLB_PAGE_ORDER);
	if (!parsed_default_hugepagesz) {
		/*
		 * If we did not parse a default huge page size, set
		 * default_hstate_idx to HPAGE_SIZE hstate. And, if the
		 * number of huge pages for this default size was implicitly
		 * specified, set that here as well.
		 * Note that the implicit setting will overwrite an explicit
		 * setting.  A warning will be printed in this case.
		 */
		default_hstate_idx = hstate_index(size_to_hstate(HPAGE_SIZE));
		if (default_hstate_max_huge_pages) {
			if (default_hstate.max_huge_pages) {
				char buf[32];

				string_get_size(huge_page_size(&default_hstate),
					1, STRING_UNITS_2, buf, 32);
				pr_warn("HugeTLB: Ignoring hugepages=%lu associated with %s page size\n",
					default_hstate.max_huge_pages, buf);
				pr_warn("HugeTLB: Using hugepages=%lu for number of default huge pages\n",
					default_hstate_max_huge_pages);
			}
			default_hstate.max_huge_pages =
				default_hstate_max_huge_pages;
4605

4606
			for_each_online_node(i)
4607 4608
				default_hstate.max_huge_pages_node[i] =
					default_hugepages_in_node[i];
4609
		}
4610
	}
4611

4612
	hugetlb_cma_check();
4613
	hugetlb_init_hstates();
4614
	gather_bootmem_prealloc();
4615 4616 4617
	report_hugepages();

	hugetlb_sysfs_init();
4618
	hugetlb_cgroup_file_init();
4619
	hugetlb_sysctl_init();
4620

4621 4622 4623 4624 4625
#ifdef CONFIG_SMP
	num_fault_mutexes = roundup_pow_of_two(8 * num_possible_cpus());
#else
	num_fault_mutexes = 1;
#endif
4626
	hugetlb_fault_mutex_table =
4627 4628
		kmalloc_array(num_fault_mutexes, sizeof(struct mutex),
			      GFP_KERNEL);
4629
	BUG_ON(!hugetlb_fault_mutex_table);
4630 4631

	for (i = 0; i < num_fault_mutexes; i++)
4632
		mutex_init(&hugetlb_fault_mutex_table[i]);
4633 4634
	return 0;
}
4635
subsys_initcall(hugetlb_init);
4636

4637 4638
/* Overwritten by architectures with more huge page sizes */
bool __init __attribute((weak)) arch_hugetlb_valid_size(unsigned long size)
4639
{
4640
	return size == HPAGE_SIZE;
4641 4642
}

4643
void __init hugetlb_add_hstate(unsigned int order)
4644 4645
{
	struct hstate *h;
4646 4647
	unsigned long i;

4648 4649 4650
	if (size_to_hstate(PAGE_SIZE << order)) {
		return;
	}
4651
	BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE);
4652
	BUG_ON(order < order_base_2(__NR_USED_SUBPAGE));
4653
	h = &hstates[hugetlb_max_hstate++];
4654
	mutex_init(&h->resize_lock);
4655
	h->order = order;
4656
	h->mask = ~(huge_page_size(h) - 1);
4657 4658
	for (i = 0; i < MAX_NUMNODES; ++i)
		INIT_LIST_HEAD(&h->hugepage_freelists[i]);
4659
	INIT_LIST_HEAD(&h->hugepage_activelist);
4660 4661
	h->next_nid_to_alloc = first_memory_node;
	h->next_nid_to_free = first_memory_node;
4662
	snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
4663
					huge_page_size(h)/SZ_1K);
4664

4665 4666 4667
	parsed_hstate = h;
}

4668 4669 4670 4671
bool __init __weak hugetlb_node_alloc_supported(void)
{
	return true;
}
4672 4673 4674 4675 4676 4677

static void __init hugepages_clear_pages_in_node(void)
{
	if (!hugetlb_max_hstate) {
		default_hstate_max_huge_pages = 0;
		memset(default_hugepages_in_node, 0,
4678
			sizeof(default_hugepages_in_node));
4679 4680 4681
	} else {
		parsed_hstate->max_huge_pages = 0;
		memset(parsed_hstate->max_huge_pages_node, 0,
4682
			sizeof(parsed_hstate->max_huge_pages_node));
4683 4684 4685
	}
}

4686 4687 4688 4689 4690 4691 4692 4693
/*
 * hugepages command line processing
 * hugepages normally follows a valid hugepagsz or default_hugepagsz
 * specification.  If not, ignore the hugepages value.  hugepages can also
 * be the first huge page command line  option in which case it implicitly
 * specifies the number of huge pages for the default size.
 */
static int __init hugepages_setup(char *s)
4694 4695
{
	unsigned long *mhp;
4696
	static unsigned long *last_mhp;
4697 4698 4699 4700
	int node = NUMA_NO_NODE;
	int count;
	unsigned long tmp;
	char *p = s;
4701

4702
	if (!parsed_valid_hugepagesz) {
4703
		pr_warn("HugeTLB: hugepages=%s does not follow a valid hugepagesz, ignoring\n", s);
4704
		parsed_valid_hugepagesz = true;
4705
		return 1;
4706
	}
4707

4708
	/*
4709 4710 4711 4712
	 * !hugetlb_max_hstate means we haven't parsed a hugepagesz= parameter
	 * yet, so this hugepages= parameter goes to the "default hstate".
	 * Otherwise, it goes with the previously parsed hugepagesz or
	 * default_hugepagesz.
4713
	 */
4714
	else if (!hugetlb_max_hstate)
4715 4716 4717 4718
		mhp = &default_hstate_max_huge_pages;
	else
		mhp = &parsed_hstate->max_huge_pages;

4719
	if (mhp == last_mhp) {
4720
		pr_warn("HugeTLB: hugepages= specified twice without interleaving hugepagesz=, ignoring hugepages=%s\n", s);
4721
		return 1;
4722 4723
	}

4724 4725 4726 4727 4728 4729 4730 4731
	while (*p) {
		count = 0;
		if (sscanf(p, "%lu%n", &tmp, &count) != 1)
			goto invalid;
		/* Parameter is node format */
		if (p[count] == ':') {
			if (!hugetlb_node_alloc_supported()) {
				pr_warn("HugeTLB: architecture can't support node specific alloc, ignoring!\n");
4732
				return 1;
4733
			}
4734
			if (tmp >= MAX_NUMNODES || !node_online(tmp))
4735
				goto invalid;
4736
			node = array_index_nospec(tmp, MAX_NUMNODES);
4737 4738 4739 4740 4741 4742 4743 4744 4745 4746 4747 4748 4749 4750 4751 4752 4753 4754 4755 4756 4757
			p += count + 1;
			/* Parse hugepages */
			if (sscanf(p, "%lu%n", &tmp, &count) != 1)
				goto invalid;
			if (!hugetlb_max_hstate)
				default_hugepages_in_node[node] = tmp;
			else
				parsed_hstate->max_huge_pages_node[node] = tmp;
			*mhp += tmp;
			/* Go to parse next node*/
			if (p[count] == ',')
				p += count + 1;
			else
				break;
		} else {
			if (p != s)
				goto invalid;
			*mhp = tmp;
			break;
		}
	}
4758

4759 4760
	/*
	 * Global state is always initialized later in hugetlb_init.
4761
	 * But we need to allocate gigantic hstates here early to still
4762 4763
	 * use the bootmem allocator.
	 */
4764
	if (hugetlb_max_hstate && hstate_is_gigantic(parsed_hstate))
4765 4766 4767 4768
		hugetlb_hstate_alloc_pages(parsed_hstate);

	last_mhp = mhp;

4769
	return 1;
4770 4771 4772

invalid:
	pr_warn("HugeTLB: Invalid hugepages parameter %s\n", p);
4773
	hugepages_clear_pages_in_node();
4774
	return 1;
4775
}
4776
__setup("hugepages=", hugepages_setup);
4777

4778 4779 4780 4781 4782 4783 4784
/*
 * hugepagesz command line processing
 * A specific huge page size can only be specified once with hugepagesz.
 * hugepagesz is followed by hugepages on the command line.  The global
 * variable 'parsed_valid_hugepagesz' is used to determine if prior
 * hugepagesz argument was valid.
 */
4785
static int __init hugepagesz_setup(char *s)
4786
{
4787
	unsigned long size;
4788 4789 4790
	struct hstate *h;

	parsed_valid_hugepagesz = false;
4791 4792 4793
	size = (unsigned long)memparse(s, NULL);

	if (!arch_hugetlb_valid_size(size)) {
4794
		pr_err("HugeTLB: unsupported hugepagesz=%s\n", s);
4795
		return 1;
4796 4797
	}

4798 4799 4800 4801 4802 4803 4804 4805 4806 4807 4808 4809
	h = size_to_hstate(size);
	if (h) {
		/*
		 * hstate for this size already exists.  This is normally
		 * an error, but is allowed if the existing hstate is the
		 * default hstate.  More specifically, it is only allowed if
		 * the number of huge pages for the default hstate was not
		 * previously specified.
		 */
		if (!parsed_default_hugepagesz ||  h != &default_hstate ||
		    default_hstate.max_huge_pages) {
			pr_warn("HugeTLB: hugepagesz=%s specified twice, ignoring\n", s);
4810
			return 1;
4811 4812 4813 4814 4815 4816 4817 4818 4819 4820
		}

		/*
		 * No need to call hugetlb_add_hstate() as hstate already
		 * exists.  But, do set parsed_hstate so that a following
		 * hugepages= parameter will be applied to this hstate.
		 */
		parsed_hstate = h;
		parsed_valid_hugepagesz = true;
		return 1;
4821 4822
	}

4823
	hugetlb_add_hstate(ilog2(size) - PAGE_SHIFT);
4824
	parsed_valid_hugepagesz = true;
4825 4826
	return 1;
}
4827 4828
__setup("hugepagesz=", hugepagesz_setup);

4829 4830 4831 4832
/*
 * default_hugepagesz command line input
 * Only one instance of default_hugepagesz allowed on command line.
 */
4833
static int __init default_hugepagesz_setup(char *s)
4834
{
4835
	unsigned long size;
4836
	int i;
4837

4838 4839 4840
	parsed_valid_hugepagesz = false;
	if (parsed_default_hugepagesz) {
		pr_err("HugeTLB: default_hugepagesz previously specified, ignoring %s\n", s);
4841
		return 1;
4842 4843
	}

4844 4845 4846
	size = (unsigned long)memparse(s, NULL);

	if (!arch_hugetlb_valid_size(size)) {
4847
		pr_err("HugeTLB: unsupported default_hugepagesz=%s\n", s);
4848
		return 1;
4849 4850
	}

4851 4852 4853 4854 4855 4856 4857 4858 4859
	hugetlb_add_hstate(ilog2(size) - PAGE_SHIFT);
	parsed_valid_hugepagesz = true;
	parsed_default_hugepagesz = true;
	default_hstate_idx = hstate_index(size_to_hstate(size));

	/*
	 * The number of default huge pages (for this size) could have been
	 * specified as the first hugetlb parameter: hugepages=X.  If so,
	 * then default_hstate_max_huge_pages is set.  If the default huge
4860
	 * page size is gigantic (> MAX_PAGE_ORDER), then the pages must be
4861 4862 4863 4864
	 * allocated here from bootmem allocator.
	 */
	if (default_hstate_max_huge_pages) {
		default_hstate.max_huge_pages = default_hstate_max_huge_pages;
4865
		for_each_online_node(i)
4866 4867
			default_hstate.max_huge_pages_node[i] =
				default_hugepages_in_node[i];
4868 4869 4870 4871 4872
		if (hstate_is_gigantic(&default_hstate))
			hugetlb_hstate_alloc_pages(&default_hstate);
		default_hstate_max_huge_pages = 0;
	}

4873 4874
	return 1;
}
4875
__setup("default_hugepagesz=", default_hugepagesz_setup);
4876

4877 4878 4879 4880 4881 4882 4883 4884 4885 4886 4887 4888 4889 4890 4891 4892 4893
static nodemask_t *policy_mbind_nodemask(gfp_t gfp)
{
#ifdef CONFIG_NUMA
	struct mempolicy *mpol = get_task_policy(current);

	/*
	 * Only enforce MPOL_BIND policy which overlaps with cpuset policy
	 * (from policy_nodemask) specifically for hugetlb case
	 */
	if (mpol->mode == MPOL_BIND &&
		(apply_policy_zone(mpol, gfp_zone(gfp)) &&
		 cpuset_nodemask_valid_mems_allowed(&mpol->nodes)))
		return &mpol->nodes;
#endif
	return NULL;
}

4894
static unsigned int allowed_mems_nr(struct hstate *h)
4895 4896 4897
{
	int node;
	unsigned int nr = 0;
4898
	nodemask_t *mbind_nodemask;
4899 4900 4901
	unsigned int *array = h->free_huge_pages_node;
	gfp_t gfp_mask = htlb_alloc_mask(h);

4902
	mbind_nodemask = policy_mbind_nodemask(gfp_mask);
4903
	for_each_node_mask(node, cpuset_current_mems_allowed) {
4904
		if (!mbind_nodemask || node_isset(node, *mbind_nodemask))
4905 4906
			nr += array[node];
	}
4907 4908 4909 4910 4911

	return nr;
}

#ifdef CONFIG_SYSCTL
4912 4913 4914 4915 4916 4917 4918 4919 4920 4921 4922 4923 4924 4925 4926 4927
static int proc_hugetlb_doulongvec_minmax(struct ctl_table *table, int write,
					  void *buffer, size_t *length,
					  loff_t *ppos, unsigned long *out)
{
	struct ctl_table dup_table;

	/*
	 * In order to avoid races with __do_proc_doulongvec_minmax(), we
	 * can duplicate the @table and alter the duplicate of it.
	 */
	dup_table = *table;
	dup_table.data = out;

	return proc_doulongvec_minmax(&dup_table, write, buffer, length, ppos);
}

4928 4929
static int hugetlb_sysctl_handler_common(bool obey_mempolicy,
			 struct ctl_table *table, int write,
4930
			 void *buffer, size_t *length, loff_t *ppos)
Linus Torvalds's avatar
Linus Torvalds committed
4931
{
4932
	struct hstate *h = &default_hstate;
4933
	unsigned long tmp = h->max_huge_pages;
4934
	int ret;
4935

4936
	if (!hugepages_supported())
4937
		return -EOPNOTSUPP;
4938

4939 4940
	ret = proc_hugetlb_doulongvec_minmax(table, write, buffer, length, ppos,
					     &tmp);
4941 4942
	if (ret)
		goto out;
4943

4944 4945 4946
	if (write)
		ret = __nr_hugepages_store_common(obey_mempolicy, h,
						  NUMA_NO_NODE, tmp, *length);
4947 4948
out:
	return ret;
Linus Torvalds's avatar
Linus Torvalds committed
4949
}
4950

4951
static int hugetlb_sysctl_handler(struct ctl_table *table, int write,
4952
			  void *buffer, size_t *length, loff_t *ppos)
4953 4954 4955 4956 4957 4958 4959
{

	return hugetlb_sysctl_handler_common(false, table, write,
							buffer, length, ppos);
}

#ifdef CONFIG_NUMA
4960
static int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write,
4961
			  void *buffer, size_t *length, loff_t *ppos)
4962 4963 4964 4965 4966 4967
{
	return hugetlb_sysctl_handler_common(true, table, write,
							buffer, length, ppos);
}
#endif /* CONFIG_NUMA */

4968
static int hugetlb_overcommit_handler(struct ctl_table *table, int write,
4969
		void *buffer, size_t *length, loff_t *ppos)
4970
{
4971
	struct hstate *h = &default_hstate;
4972
	unsigned long tmp;
4973
	int ret;
4974

4975
	if (!hugepages_supported())
4976
		return -EOPNOTSUPP;
4977

4978
	tmp = h->nr_overcommit_huge_pages;
4979

4980
	if (write && hstate_is_gigantic(h))
4981 4982
		return -EINVAL;

4983 4984
	ret = proc_hugetlb_doulongvec_minmax(table, write, buffer, length, ppos,
					     &tmp);
4985 4986
	if (ret)
		goto out;
4987 4988

	if (write) {
4989
		spin_lock_irq(&hugetlb_lock);
4990
		h->nr_overcommit_huge_pages = tmp;
4991
		spin_unlock_irq(&hugetlb_lock);
4992
	}
4993 4994
out:
	return ret;
4995 4996
}

4997 4998 4999 5000 5001 5002 5003 5004 5005 5006 5007 5008 5009 5010 5011 5012 5013 5014 5015 5016 5017 5018 5019 5020 5021 5022 5023 5024 5025 5026 5027 5028 5029 5030 5031 5032 5033 5034
static struct ctl_table hugetlb_table[] = {
	{
		.procname	= "nr_hugepages",
		.data		= NULL,
		.maxlen		= sizeof(unsigned long),
		.mode		= 0644,
		.proc_handler	= hugetlb_sysctl_handler,
	},
#ifdef CONFIG_NUMA
	{
		.procname       = "nr_hugepages_mempolicy",
		.data           = NULL,
		.maxlen         = sizeof(unsigned long),
		.mode           = 0644,
		.proc_handler   = &hugetlb_mempolicy_sysctl_handler,
	},
#endif
	{
		.procname	= "hugetlb_shm_group",
		.data		= &sysctl_hugetlb_shm_group,
		.maxlen		= sizeof(gid_t),
		.mode		= 0644,
		.proc_handler	= proc_dointvec,
	},
	{
		.procname	= "nr_overcommit_hugepages",
		.data		= NULL,
		.maxlen		= sizeof(unsigned long),
		.mode		= 0644,
		.proc_handler	= hugetlb_overcommit_handler,
	},
	{ }
};

static void hugetlb_sysctl_init(void)
{
	register_sysctl_init("vm", hugetlb_table);
}
Linus Torvalds's avatar
Linus Torvalds committed
5035 5036
#endif /* CONFIG_SYSCTL */

5037
void hugetlb_report_meminfo(struct seq_file *m)
Linus Torvalds's avatar
Linus Torvalds committed
5038
{
5039 5040 5041
	struct hstate *h;
	unsigned long total = 0;

5042 5043
	if (!hugepages_supported())
		return;
5044 5045 5046 5047

	for_each_hstate(h) {
		unsigned long count = h->nr_huge_pages;

5048
		total += huge_page_size(h) * count;
5049 5050 5051 5052 5053 5054 5055 5056 5057 5058 5059 5060

		if (h == &default_hstate)
			seq_printf(m,
				   "HugePages_Total:   %5lu\n"
				   "HugePages_Free:    %5lu\n"
				   "HugePages_Rsvd:    %5lu\n"
				   "HugePages_Surp:    %5lu\n"
				   "Hugepagesize:   %8lu kB\n",
				   count,
				   h->free_huge_pages,
				   h->resv_huge_pages,
				   h->surplus_huge_pages,
5061
				   huge_page_size(h) / SZ_1K);
5062 5063
	}

5064
	seq_printf(m, "Hugetlb:        %8lu kB\n", total / SZ_1K);
Linus Torvalds's avatar
Linus Torvalds committed
5065 5066
}

5067
int hugetlb_report_node_meminfo(char *buf, int len, int nid)
Linus Torvalds's avatar
Linus Torvalds committed
5068
{
5069
	struct hstate *h = &default_hstate;
5070

5071 5072
	if (!hugepages_supported())
		return 0;
5073 5074 5075 5076 5077 5078 5079 5080

	return sysfs_emit_at(buf, len,
			     "Node %d HugePages_Total: %5u\n"
			     "Node %d HugePages_Free:  %5u\n"
			     "Node %d HugePages_Surp:  %5u\n",
			     nid, h->nr_huge_pages_node[nid],
			     nid, h->free_huge_pages_node[nid],
			     nid, h->surplus_huge_pages_node[nid]);
Linus Torvalds's avatar
Linus Torvalds committed
5081 5082
}

5083
void hugetlb_show_meminfo_node(int nid)
5084 5085 5086
{
	struct hstate *h;

5087 5088 5089
	if (!hugepages_supported())
		return;

5090 5091 5092 5093 5094 5095 5096
	for_each_hstate(h)
		printk("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n",
			nid,
			h->nr_huge_pages_node[nid],
			h->free_huge_pages_node[nid],
			h->surplus_huge_pages_node[nid],
			huge_page_size(h) / SZ_1K);
5097 5098
}

5099 5100 5101
void hugetlb_report_usage(struct seq_file *m, struct mm_struct *mm)
{
	seq_printf(m, "HugetlbPages:\t%8lu kB\n",
5102
		   K(atomic_long_read(&mm->hugetlb_usage)));
5103 5104
}

Linus Torvalds's avatar
Linus Torvalds committed
5105 5106 5107
/* Return the number pages of memory we physically have, in PAGE_SIZE units. */
unsigned long hugetlb_total_pages(void)
{
5108 5109 5110 5111 5112 5113
	struct hstate *h;
	unsigned long nr_total_pages = 0;

	for_each_hstate(h)
		nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h);
	return nr_total_pages;
Linus Torvalds's avatar
Linus Torvalds committed
5114 5115
}

5116
static int hugetlb_acct_memory(struct hstate *h, long delta)
5117 5118 5119
{
	int ret = -ENOMEM;

5120 5121 5122
	if (!delta)
		return 0;

5123
	spin_lock_irq(&hugetlb_lock);
5124 5125 5126 5127 5128 5129 5130 5131 5132 5133 5134 5135 5136 5137 5138 5139
	/*
	 * When cpuset is configured, it breaks the strict hugetlb page
	 * reservation as the accounting is done on a global variable. Such
	 * reservation is completely rubbish in the presence of cpuset because
	 * the reservation is not checked against page availability for the
	 * current cpuset. Application can still potentially OOM'ed by kernel
	 * with lack of free htlb page in cpuset that the task is in.
	 * Attempt to enforce strict accounting with cpuset is almost
	 * impossible (or too ugly) because cpuset is too fluid that
	 * task or memory node can be dynamically moved between cpusets.
	 *
	 * The change of semantics for shared hugetlb mapping with cpuset is
	 * undesirable. However, in order to preserve some of the semantics,
	 * we fall back to check against current free page availability as
	 * a best attempt and hopefully to minimize the impact of changing
	 * semantics that cpuset has.
5140 5141 5142 5143 5144 5145
	 *
	 * Apart from cpuset, we also have memory policy mechanism that
	 * also determines from which node the kernel will allocate memory
	 * in a NUMA system. So similar to cpuset, we also should consider
	 * the memory policy of the current task. Similar to the description
	 * above.
5146 5147
	 */
	if (delta > 0) {
5148
		if (gather_surplus_pages(h, delta) < 0)
5149 5150
			goto out;

5151
		if (delta > allowed_mems_nr(h)) {
5152
			return_unused_surplus_pages(h, delta);
5153 5154 5155 5156 5157 5158
			goto out;
		}
	}

	ret = 0;
	if (delta < 0)
5159
		return_unused_surplus_pages(h, (unsigned long) -delta);
5160 5161

out:
5162
	spin_unlock_irq(&hugetlb_lock);
5163 5164 5165
	return ret;
}

5166 5167
static void hugetlb_vm_op_open(struct vm_area_struct *vma)
{
5168
	struct resv_map *resv = vma_resv_map(vma);
5169 5170

	/*
5171
	 * HPAGE_RESV_OWNER indicates a private mapping.
5172 5173 5174
	 * This new VMA should share its siblings reservation map if present.
	 * The VMA will only ever have a valid reservation map pointer where
	 * it is being copied for another still existing VMA.  As that VMA
Lucas De Marchi's avatar
Lucas De Marchi committed
5175
	 * has a reference to the reservation map it cannot disappear until
5176 5177 5178
	 * after this open call completes.  It is therefore safe to take a
	 * new reference here without additional locking.
	 */
5179 5180
	if (resv && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
		resv_map_dup_hugetlb_cgroup_uncharge_info(resv);
5181
		kref_get(&resv->refs);
5182
	}
5183

5184 5185
	/*
	 * vma_lock structure for sharable mappings is vma specific.
5186 5187 5188
	 * Clear old pointer (if copied via vm_area_dup) and allocate
	 * new structure.  Before clearing, make sure vma_lock is not
	 * for this vma.
5189 5190
	 */
	if (vma->vm_flags & VM_MAYSHARE) {
5191 5192 5193 5194 5195 5196 5197 5198 5199 5200
		struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;

		if (vma_lock) {
			if (vma_lock->vma != vma) {
				vma->vm_private_data = NULL;
				hugetlb_vma_lock_alloc(vma);
			} else
				pr_warn("HugeTLB: vma_lock already exists in %s.\n", __func__);
		} else
			hugetlb_vma_lock_alloc(vma);
5201
	}
5202 5203
}

5204 5205
static void hugetlb_vm_op_close(struct vm_area_struct *vma)
{
5206
	struct hstate *h = hstate_vma(vma);
5207
	struct resv_map *resv;
5208
	struct hugepage_subpool *spool = subpool_vma(vma);
5209
	unsigned long reserve, start, end;
5210
	long gbl_reserve;
5211

5212 5213 5214
	hugetlb_vma_lock_free(vma);

	resv = vma_resv_map(vma);
5215 5216
	if (!resv || !is_vma_resv_set(vma, HPAGE_RESV_OWNER))
		return;
5217

5218 5219
	start = vma_hugecache_offset(h, vma, vma->vm_start);
	end = vma_hugecache_offset(h, vma, vma->vm_end);
5220

5221
	reserve = (end - start) - region_count(resv, start, end);
5222
	hugetlb_cgroup_uncharge_counter(resv, start, end);
5223
	if (reserve) {
5224 5225 5226 5227 5228 5229
		/*
		 * Decrement reserve counts.  The global reserve count may be
		 * adjusted if the subpool has a minimum size.
		 */
		gbl_reserve = hugepage_subpool_put_pages(spool, reserve);
		hugetlb_acct_memory(h, -gbl_reserve);
5230
	}
5231 5232

	kref_put(&resv->refs, resv_map_release);
5233 5234
}

5235 5236 5237 5238
static int hugetlb_vm_op_split(struct vm_area_struct *vma, unsigned long addr)
{
	if (addr & ~(huge_page_mask(hstate_vma(vma))))
		return -EINVAL;
5239 5240 5241 5242 5243 5244 5245 5246 5247 5248 5249 5250 5251 5252 5253 5254 5255 5256 5257

	/*
	 * PMD sharing is only possible for PUD_SIZE-aligned address ranges
	 * in HugeTLB VMAs. If we will lose PUD_SIZE alignment due to this
	 * split, unshare PMDs in the PUD_SIZE interval surrounding addr now.
	 */
	if (addr & ~PUD_MASK) {
		/*
		 * hugetlb_vm_op_split is called right before we attempt to
		 * split the VMA. We will need to unshare PMDs in the old and
		 * new VMAs, so let's unshare before we split.
		 */
		unsigned long floor = addr & PUD_MASK;
		unsigned long ceil = floor + PUD_SIZE;

		if (floor >= vma->vm_start && ceil <= vma->vm_end)
			hugetlb_unshare_pmds(vma, floor, ceil);
	}

5258 5259 5260
	return 0;
}

5261 5262
static unsigned long hugetlb_vm_op_pagesize(struct vm_area_struct *vma)
{
5263
	return huge_page_size(hstate_vma(vma));
5264 5265
}

Linus Torvalds's avatar
Linus Torvalds committed
5266 5267 5268
/*
 * We cannot handle pagefaults against hugetlb pages at all.  They cause
 * handle_mm_fault() to try to instantiate regular-sized pages in the
5269
 * hugepage VMA.  do_page_fault() is supposed to trap this, so BUG is we get
Linus Torvalds's avatar
Linus Torvalds committed
5270 5271
 * this far.
 */
5272
static vm_fault_t hugetlb_vm_op_fault(struct vm_fault *vmf)
Linus Torvalds's avatar
Linus Torvalds committed
5273 5274
{
	BUG();
Nick Piggin's avatar
Nick Piggin committed
5275
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
5276 5277
}

5278 5279 5280 5281 5282 5283 5284
/*
 * When a new function is introduced to vm_operations_struct and added
 * to hugetlb_vm_ops, please consider adding the function to shm_vm_ops.
 * This is because under System V memory model, mappings created via
 * shmget/shmat with "huge page" specified are backed by hugetlbfs files,
 * their original vm_ops are overwritten with shm_vm_ops.
 */
5285
const struct vm_operations_struct hugetlb_vm_ops = {
Nick Piggin's avatar
Nick Piggin committed
5286
	.fault = hugetlb_vm_op_fault,
5287
	.open = hugetlb_vm_op_open,
5288
	.close = hugetlb_vm_op_close,
5289
	.may_split = hugetlb_vm_op_split,
5290
	.pagesize = hugetlb_vm_op_pagesize,
Linus Torvalds's avatar
Linus Torvalds committed
5291 5292
};

5293 5294
static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
				int writable)
5295 5296
{
	pte_t entry;
5297
	unsigned int shift = huge_page_shift(hstate_vma(vma));
5298

5299
	if (writable) {
5300 5301
		entry = huge_pte_mkwrite(huge_pte_mkdirty(mk_huge_pte(page,
					 vma->vm_page_prot)));
5302
	} else {
5303 5304
		entry = huge_pte_wrprotect(mk_huge_pte(page,
					   vma->vm_page_prot));
5305 5306
	}
	entry = pte_mkyoung(entry);
5307
	entry = arch_make_huge_pte(entry, shift, vma->vm_flags);
5308 5309 5310 5311

	return entry;
}

5312 5313 5314 5315 5316
static void set_huge_ptep_writable(struct vm_area_struct *vma,
				   unsigned long address, pte_t *ptep)
{
	pte_t entry;

5317
	entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(ptep)));
5318
	if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1))
5319
		update_mmu_cache(vma, address, ptep);
5320 5321
}

5322
bool is_hugetlb_entry_migration(pte_t pte)
5323 5324 5325 5326
{
	swp_entry_t swp;

	if (huge_pte_none(pte) || pte_present(pte))
5327
		return false;
5328
	swp = pte_to_swp_entry(pte);
5329
	if (is_migration_entry(swp))
5330
		return true;
5331
	else
5332
		return false;
5333 5334
}

5335
bool is_hugetlb_entry_hwpoisoned(pte_t pte)
5336 5337 5338 5339
{
	swp_entry_t swp;

	if (huge_pte_none(pte) || pte_present(pte))
5340
		return false;
5341
	swp = pte_to_swp_entry(pte);
5342
	if (is_hwpoison_entry(swp))
5343
		return true;
5344
	else
5345
		return false;
5346
}
5347

5348
static void
5349
hugetlb_install_folio(struct vm_area_struct *vma, pte_t *ptep, unsigned long addr,
5350
		      struct folio *new_folio, pte_t old, unsigned long sz)
5351
{
5352 5353
	pte_t newpte = make_huge_pte(vma, &new_folio->page, 1);

5354
	__folio_mark_uptodate(new_folio);
5355
	hugetlb_add_new_anon_rmap(new_folio, vma, addr);
5356 5357
	if (userfaultfd_wp(vma) && huge_pte_uffd_wp(old))
		newpte = huge_pte_mkuffd_wp(newpte);
5358
	set_huge_pte_at(vma->vm_mm, addr, ptep, newpte, sz);
5359
	hugetlb_count_add(pages_per_huge_page(hstate_vma(vma)), vma->vm_mm);
5360
	folio_set_hugetlb_migratable(new_folio);
5361 5362
}

5363
int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
5364 5365
			    struct vm_area_struct *dst_vma,
			    struct vm_area_struct *src_vma)
5366
{
5367
	pte_t *src_pte, *dst_pte, entry;
5368
	struct folio *pte_folio;
5369
	unsigned long addr;
5370 5371
	bool cow = is_cow_mapping(src_vma->vm_flags);
	struct hstate *h = hstate_vma(src_vma);
5372
	unsigned long sz = huge_page_size(h);
5373
	unsigned long npages = pages_per_huge_page(h);
5374
	struct mmu_notifier_range range;
5375
	unsigned long last_addr_mask;
5376
	int ret = 0;
5377

5378
	if (cow) {
5379
		mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, src,
5380 5381
					src_vma->vm_start,
					src_vma->vm_end);
5382
		mmu_notifier_invalidate_range_start(&range);
5383
		vma_assert_write_locked(src_vma);
5384
		raw_write_seqcount_begin(&src->write_protect_seq);
5385 5386 5387
	} else {
		/*
		 * For shared mappings the vma lock must be held before
5388
		 * calling hugetlb_walk() in the src vma. Otherwise, the
5389 5390 5391 5392
		 * returned ptep could go away if part of a shared pmd and
		 * another thread calls huge_pmd_unshare.
		 */
		hugetlb_vma_lock_read(src_vma);
5393
	}
5394

5395
	last_addr_mask = hugetlb_mask_last_page(h);
5396
	for (addr = src_vma->vm_start; addr < src_vma->vm_end; addr += sz) {
5397
		spinlock_t *src_ptl, *dst_ptl;
5398
		src_pte = hugetlb_walk(src_vma, addr, sz);
5399 5400
		if (!src_pte) {
			addr |= last_addr_mask;
5401
			continue;
5402
		}
5403
		dst_pte = huge_pte_alloc(dst, dst_vma, addr, sz);
5404 5405 5406 5407
		if (!dst_pte) {
			ret = -ENOMEM;
			break;
		}
5408

5409 5410 5411
		/*
		 * If the pagetables are shared don't copy or take references.
		 *
5412
		 * dst_pte == src_pte is the common case of src/dest sharing.
5413
		 * However, src could have 'unshared' and dst shares with
5414 5415
		 * another vma. So page_count of ptep page is checked instead
		 * to reliably determine whether pte is shared.
5416
		 */
5417
		if (page_count(virt_to_page(dst_pte)) > 1) {
5418
			addr |= last_addr_mask;
5419
			continue;
5420
		}
5421

5422 5423 5424
		dst_ptl = huge_pte_lock(h, dst, dst_pte);
		src_ptl = huge_pte_lockptr(h, src, src_pte);
		spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
5425
		entry = huge_ptep_get(src_pte);
5426
again:
5427
		if (huge_pte_none(entry)) {
5428
			/*
5429
			 * Skip if src entry none.
5430
			 */
5431
			;
5432
		} else if (unlikely(is_hugetlb_entry_hwpoisoned(entry))) {
5433
			if (!userfaultfd_wp(dst_vma))
5434
				entry = huge_pte_clear_uffd_wp(entry);
5435
			set_huge_pte_at(dst, addr, dst_pte, entry, sz);
5436
		} else if (unlikely(is_hugetlb_entry_migration(entry))) {
5437
			swp_entry_t swp_entry = pte_to_swp_entry(entry);
5438
			bool uffd_wp = pte_swp_uffd_wp(entry);
5439

5440
			if (!is_readable_migration_entry(swp_entry) && cow) {
5441 5442 5443 5444
				/*
				 * COW mappings require pages in both
				 * parent and child to be set to read.
				 */
5445 5446
				swp_entry = make_readable_migration_entry(
							swp_offset(swp_entry));
5447
				entry = swp_entry_to_pte(swp_entry);
5448
				if (userfaultfd_wp(src_vma) && uffd_wp)
5449
					entry = pte_swp_mkuffd_wp(entry);
5450
				set_huge_pte_at(src, addr, src_pte, entry, sz);
5451
			}
5452
			if (!userfaultfd_wp(dst_vma))
5453
				entry = huge_pte_clear_uffd_wp(entry);
5454
			set_huge_pte_at(dst, addr, dst_pte, entry, sz);
5455
		} else if (unlikely(is_pte_marker(entry))) {
5456 5457 5458 5459 5460
			pte_marker marker = copy_pte_marker(
				pte_to_swp_entry(entry), dst_vma);

			if (marker)
				set_huge_pte_at(dst, addr, dst_pte,
5461
						make_pte_marker(marker), sz);
5462
		} else {
5463
			entry = huge_ptep_get(src_pte);
5464 5465
			pte_folio = page_folio(pte_page(entry));
			folio_get(pte_folio);
5466 5467

			/*
5468 5469 5470 5471
			 * Failing to duplicate the anon rmap is a rare case
			 * where we see pinned hugetlb pages while they're
			 * prone to COW. We need to do the COW earlier during
			 * fork.
5472 5473 5474 5475 5476
			 *
			 * When pre-allocating the page or copying data, we
			 * need to be without the pgtable locks since we could
			 * sleep during the process.
			 */
5477
			if (!folio_test_anon(pte_folio)) {
5478
				hugetlb_add_file_rmap(pte_folio);
5479
			} else if (hugetlb_try_dup_anon_rmap(pte_folio, src_vma)) {
5480
				pte_t src_pte_old = entry;
5481
				struct folio *new_folio;
5482 5483 5484 5485

				spin_unlock(src_ptl);
				spin_unlock(dst_ptl);
				/* Do not use reserve as it's private owned */
5486 5487
				new_folio = alloc_hugetlb_folio(dst_vma, addr, 1);
				if (IS_ERR(new_folio)) {
5488
					folio_put(pte_folio);
5489
					ret = PTR_ERR(new_folio);
5490 5491
					break;
				}
5492
				ret = copy_user_large_folio(new_folio,
5493 5494 5495
							    pte_folio,
							    addr, dst_vma);
				folio_put(pte_folio);
5496 5497 5498 5499
				if (ret) {
					folio_put(new_folio);
					break;
				}
5500

5501
				/* Install the new hugetlb folio if src pte stable */
5502 5503 5504 5505 5506
				dst_ptl = huge_pte_lock(h, dst, dst_pte);
				src_ptl = huge_pte_lockptr(h, src, src_pte);
				spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
				entry = huge_ptep_get(src_pte);
				if (!pte_same(src_pte_old, entry)) {
5507
					restore_reserve_on_error(h, dst_vma, addr,
5508
								new_folio);
5509
					folio_put(new_folio);
5510
					/* huge_ptep of dst_pte won't change as in child */
5511 5512
					goto again;
				}
5513
				hugetlb_install_folio(dst_vma, dst_pte, addr,
5514
						      new_folio, src_pte_old, sz);
5515 5516 5517 5518 5519
				spin_unlock(src_ptl);
				spin_unlock(dst_ptl);
				continue;
			}

5520
			if (cow) {
5521 5522 5523 5524 5525
				/*
				 * No need to notify as we are downgrading page
				 * table protection not changing it to point
				 * to a new page.
				 *
5526
				 * See Documentation/mm/mmu_notifier.rst
5527
				 */
5528
				huge_ptep_set_wrprotect(src, addr, src_pte);
5529
				entry = huge_pte_wrprotect(entry);
5530
			}
5531

5532 5533 5534
			if (!userfaultfd_wp(dst_vma))
				entry = huge_pte_clear_uffd_wp(entry);

5535
			set_huge_pte_at(dst, addr, dst_pte, entry, sz);
5536
			hugetlb_count_add(npages, dst);
5537
		}
5538 5539
		spin_unlock(src_ptl);
		spin_unlock(dst_ptl);
5540 5541
	}

5542 5543
	if (cow) {
		raw_write_seqcount_end(&src->write_protect_seq);
5544
		mmu_notifier_invalidate_range_end(&range);
5545 5546
	} else {
		hugetlb_vma_unlock_read(src_vma);
5547
	}
5548 5549

	return ret;
5550 5551
}

5552
static void move_huge_pte(struct vm_area_struct *vma, unsigned long old_addr,
5553 5554
			  unsigned long new_addr, pte_t *src_pte, pte_t *dst_pte,
			  unsigned long sz)
5555 5556 5557 5558
{
	struct hstate *h = hstate_vma(vma);
	struct mm_struct *mm = vma->vm_mm;
	spinlock_t *src_ptl, *dst_ptl;
5559
	pte_t pte;
5560 5561 5562 5563 5564 5565

	dst_ptl = huge_pte_lock(h, mm, dst_pte);
	src_ptl = huge_pte_lockptr(h, mm, src_pte);

	/*
	 * We don't have to worry about the ordering of src and dst ptlocks
5566
	 * because exclusive mmap_lock (or the i_mmap_lock) prevents deadlock.
5567 5568 5569 5570 5571
	 */
	if (src_ptl != dst_ptl)
		spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);

	pte = huge_ptep_get_and_clear(mm, old_addr, src_pte);
5572
	set_huge_pte_at(mm, new_addr, dst_pte, pte, sz);
5573 5574 5575 5576 5577 5578 5579 5580 5581 5582 5583 5584 5585 5586 5587 5588

	if (src_ptl != dst_ptl)
		spin_unlock(src_ptl);
	spin_unlock(dst_ptl);
}

int move_hugetlb_page_tables(struct vm_area_struct *vma,
			     struct vm_area_struct *new_vma,
			     unsigned long old_addr, unsigned long new_addr,
			     unsigned long len)
{
	struct hstate *h = hstate_vma(vma);
	struct address_space *mapping = vma->vm_file->f_mapping;
	unsigned long sz = huge_page_size(h);
	struct mm_struct *mm = vma->vm_mm;
	unsigned long old_end = old_addr + len;
5589
	unsigned long last_addr_mask;
5590 5591
	pte_t *src_pte, *dst_pte;
	struct mmu_notifier_range range;
5592
	bool shared_pmd = false;
5593

5594
	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, old_addr,
5595 5596
				old_end);
	adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
5597 5598 5599 5600 5601 5602
	/*
	 * In case of shared PMDs, we should cover the maximum possible
	 * range.
	 */
	flush_cache_range(vma, range.start, range.end);

5603
	mmu_notifier_invalidate_range_start(&range);
5604
	last_addr_mask = hugetlb_mask_last_page(h);
5605
	/* Prevent race with file truncation */
5606
	hugetlb_vma_lock_write(vma);
5607 5608
	i_mmap_lock_write(mapping);
	for (; old_addr < old_end; old_addr += sz, new_addr += sz) {
5609
		src_pte = hugetlb_walk(vma, old_addr, sz);
5610 5611 5612
		if (!src_pte) {
			old_addr |= last_addr_mask;
			new_addr |= last_addr_mask;
5613
			continue;
5614
		}
5615 5616 5617
		if (huge_pte_none(huge_ptep_get(src_pte)))
			continue;

5618
		if (huge_pmd_unshare(mm, vma, old_addr, src_pte)) {
5619
			shared_pmd = true;
5620 5621
			old_addr |= last_addr_mask;
			new_addr |= last_addr_mask;
5622
			continue;
5623
		}
5624 5625 5626 5627 5628

		dst_pte = huge_pte_alloc(mm, new_vma, new_addr, sz);
		if (!dst_pte)
			break;

5629
		move_huge_pte(vma, old_addr, new_addr, src_pte, dst_pte, sz);
5630
	}
5631 5632

	if (shared_pmd)
5633
		flush_hugetlb_tlb_range(vma, range.start, range.end);
5634
	else
5635
		flush_hugetlb_tlb_range(vma, old_end - len, old_end);
5636
	mmu_notifier_invalidate_range_end(&range);
5637
	i_mmap_unlock_write(mapping);
5638
	hugetlb_vma_unlock_write(vma);
5639 5640 5641 5642

	return len + old_addr - old_end;
}

5643 5644 5645
void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
			    unsigned long start, unsigned long end,
			    struct page *ref_page, zap_flags_t zap_flags)
5646 5647 5648
{
	struct mm_struct *mm = vma->vm_mm;
	unsigned long address;
5649
	pte_t *ptep;
5650
	pte_t pte;
5651
	spinlock_t *ptl;
5652
	struct page *page;
5653 5654
	struct hstate *h = hstate_vma(vma);
	unsigned long sz = huge_page_size(h);
5655
	bool adjust_reservation = false;
5656
	unsigned long last_addr_mask;
5657
	bool force_flush = false;
5658

5659
	WARN_ON(!is_vm_hugetlb_page(vma));
5660 5661
	BUG_ON(start & ~huge_page_mask(h));
	BUG_ON(end & ~huge_page_mask(h));
5662

5663 5664 5665 5666
	/*
	 * This is a hugetlb vma, all the pte entries should point
	 * to huge page.
	 */
5667
	tlb_change_page_size(tlb, sz);
5668
	tlb_start_vma(tlb, vma);
5669

5670
	last_addr_mask = hugetlb_mask_last_page(h);
5671 5672
	address = start;
	for (; address < end; address += sz) {
5673
		ptep = hugetlb_walk(vma, address, sz);
5674 5675
		if (!ptep) {
			address |= last_addr_mask;
5676
			continue;
5677
		}
5678

5679
		ptl = huge_pte_lock(h, mm, ptep);
5680
		if (huge_pmd_unshare(mm, vma, address, ptep)) {
5681
			spin_unlock(ptl);
5682 5683
			tlb_flush_pmd_range(tlb, address & PUD_MASK, PUD_SIZE);
			force_flush = true;
5684
			address |= last_addr_mask;
5685 5686
			continue;
		}
5687

5688
		pte = huge_ptep_get(ptep);
5689 5690 5691 5692
		if (huge_pte_none(pte)) {
			spin_unlock(ptl);
			continue;
		}
5693 5694

		/*
5695 5696
		 * Migrating hugepage or HWPoisoned hugepage is already
		 * unmapped and its refcount is dropped, so just clear pte here.
5697
		 */
5698
		if (unlikely(!pte_present(pte))) {
5699 5700 5701 5702 5703 5704 5705 5706 5707
			/*
			 * If the pte was wr-protected by uffd-wp in any of the
			 * swap forms, meanwhile the caller does not want to
			 * drop the uffd-wp bit in this zap, then replace the
			 * pte with a marker.
			 */
			if (pte_swp_uffd_wp_any(pte) &&
			    !(zap_flags & ZAP_FLAG_DROP_MARKER))
				set_huge_pte_at(mm, address, ptep,
5708 5709
						make_pte_marker(PTE_MARKER_UFFD_WP),
						sz);
5710 5711
			else
				huge_pte_clear(mm, address, ptep, sz);
5712 5713
			spin_unlock(ptl);
			continue;
5714
		}
5715 5716

		page = pte_page(pte);
5717 5718 5719 5720 5721 5722
		/*
		 * If a reference page is supplied, it is because a specific
		 * page is being unmapped, not a range. Ensure the page we
		 * are about to unmap is the actual page of interest.
		 */
		if (ref_page) {
5723 5724 5725 5726
			if (page != ref_page) {
				spin_unlock(ptl);
				continue;
			}
5727 5728 5729 5730 5731 5732 5733 5734
			/*
			 * Mark the VMA as having unmapped its page so that
			 * future faults in this VMA will fail rather than
			 * looking like data was lost
			 */
			set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
		}

5735
		pte = huge_ptep_get_and_clear(mm, address, ptep);
5736
		tlb_remove_huge_tlb_entry(h, tlb, ptep, address);
5737
		if (huge_pte_dirty(pte))
5738
			set_page_dirty(page);
5739 5740 5741 5742
		/* Leave a uffd-wp pte marker if needed */
		if (huge_pte_uffd_wp(pte) &&
		    !(zap_flags & ZAP_FLAG_DROP_MARKER))
			set_huge_pte_at(mm, address, ptep,
5743 5744
					make_pte_marker(PTE_MARKER_UFFD_WP),
					sz);
5745
		hugetlb_count_sub(pages_per_huge_page(h), mm);
5746
		hugetlb_remove_rmap(page_folio(page));
5747

5748 5749 5750 5751 5752 5753 5754 5755 5756 5757 5758 5759 5760
		/*
		 * Restore the reservation for anonymous page, otherwise the
		 * backing page could be stolen by someone.
		 * If there we are freeing a surplus, do not set the restore
		 * reservation bit.
		 */
		if (!h->surplus_huge_pages && __vma_private_lock(vma) &&
		    folio_test_anon(page_folio(page))) {
			folio_set_hugetlb_restore_reserve(page_folio(page));
			/* Reservation to be adjusted after the spin lock */
			adjust_reservation = true;
		}

5761
		spin_unlock(ptl);
5762 5763 5764 5765 5766 5767 5768 5769 5770 5771 5772

		/*
		 * Adjust the reservation for the region that will have the
		 * reserve restored. Keep in mind that vma_needs_reservation() changes
		 * resv->adds_in_progress if it succeeds. If this is not done,
		 * do_exit() will not see it, and will keep the reservation
		 * forever.
		 */
		if (adjust_reservation && vma_needs_reservation(h, vma, address))
			vma_add_reservation(h, vma, address);

5773
		tlb_remove_page_size(tlb, page, huge_page_size(h));
5774 5775 5776 5777 5778
		/*
		 * Bail out after unmapping reference page if supplied
		 */
		if (ref_page)
			break;
5779
	}
5780
	tlb_end_vma(tlb, vma);
5781 5782 5783 5784 5785 5786 5787 5788 5789 5790 5791 5792 5793 5794 5795 5796

	/*
	 * If we unshared PMDs, the TLB flush was not recorded in mmu_gather. We
	 * could defer the flush until now, since by holding i_mmap_rwsem we
	 * guaranteed that the last refernece would not be dropped. But we must
	 * do the flushing before we return, as otherwise i_mmap_rwsem will be
	 * dropped and the last reference to the shared PMDs page might be
	 * dropped as well.
	 *
	 * In theory we could defer the freeing of the PMD pages as well, but
	 * huge_pmd_unshare() relies on the exact page_count for the PMD page to
	 * detect sharing, so we cannot defer the release of the page either.
	 * Instead, do flush now.
	 */
	if (force_flush)
		tlb_flush_mmu_tlbonly(tlb);
Linus Torvalds's avatar
Linus Torvalds committed
5797
}
5798

5799 5800
void __hugetlb_zap_begin(struct vm_area_struct *vma,
			 unsigned long *start, unsigned long *end)
5801
{
5802 5803 5804 5805
	if (!vma->vm_file)	/* hugetlbfs_file_mmap error */
		return;

	adjust_range_if_pmd_sharing_possible(vma, start, end);
5806
	hugetlb_vma_lock_write(vma);
5807 5808 5809
	if (vma->vm_file)
		i_mmap_lock_write(vma->vm_file->f_mapping);
}
5810

5811 5812 5813 5814
void __hugetlb_zap_end(struct vm_area_struct *vma,
		       struct zap_details *details)
{
	zap_flags_t zap_flags = details ? details->zap_flags : 0;
5815

5816 5817
	if (!vma->vm_file)	/* hugetlbfs_file_mmap error */
		return;
5818

5819 5820 5821 5822 5823 5824 5825 5826 5827 5828 5829 5830 5831 5832
	if (zap_flags & ZAP_FLAG_UNMAP) {	/* final unmap */
		/*
		 * Unlock and free the vma lock before releasing i_mmap_rwsem.
		 * When the vma_lock is freed, this makes the vma ineligible
		 * for pmd sharing.  And, i_mmap_rwsem is required to set up
		 * pmd sharing.  This is important as page tables for this
		 * unmapped range will be asynchrously deleted.  If the page
		 * tables are shared, there will be issues when accessed by
		 * someone else.
		 */
		__hugetlb_vma_unlock_write_free(vma);
	} else {
		hugetlb_vma_unlock_write(vma);
	}
5833 5834 5835

	if (vma->vm_file)
		i_mmap_unlock_write(vma->vm_file->f_mapping);
5836 5837
}

5838
void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
5839 5840
			  unsigned long end, struct page *ref_page,
			  zap_flags_t zap_flags)
5841
{
5842
	struct mmu_notifier_range range;
5843
	struct mmu_gather tlb;
5844

5845
	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
5846 5847 5848
				start, end);
	adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
	mmu_notifier_invalidate_range_start(&range);
5849
	tlb_gather_mmu(&tlb, vma->vm_mm);
5850

5851
	__unmap_hugepage_range(&tlb, vma, start, end, ref_page, zap_flags);
5852 5853

	mmu_notifier_invalidate_range_end(&range);
5854
	tlb_finish_mmu(&tlb);
5855 5856
}

5857 5858
/*
 * This is called when the original mapper is failing to COW a MAP_PRIVATE
5859
 * mapping it owns the reserve page for. The intention is to unmap the page
5860 5861 5862
 * from other VMAs and let the children be SIGKILLed if they are faulting the
 * same region.
 */
5863 5864
static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
			      struct page *page, unsigned long address)
5865
{
5866
	struct hstate *h = hstate_vma(vma);
5867 5868 5869 5870 5871 5872 5873 5874
	struct vm_area_struct *iter_vma;
	struct address_space *mapping;
	pgoff_t pgoff;

	/*
	 * vm_pgoff is in PAGE_SIZE units, hence the different calculation
	 * from page cache lookup which is in HPAGE_SIZE units.
	 */
5875
	address = address & huge_page_mask(h);
5876 5877
	pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) +
			vma->vm_pgoff;
5878
	mapping = vma->vm_file->f_mapping;
5879

5880 5881 5882 5883 5884
	/*
	 * Take the mapping lock for the duration of the table walk. As
	 * this mapping should be shared between all the VMAs,
	 * __unmap_hugepage_range() is called as the lock is already held
	 */
5885
	i_mmap_lock_write(mapping);
5886
	vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) {
5887 5888 5889 5890
		/* Do not unmap the current VMA */
		if (iter_vma == vma)
			continue;

5891 5892 5893 5894 5895 5896 5897 5898
		/*
		 * Shared VMAs have their own reserves and do not affect
		 * MAP_PRIVATE accounting but it is possible that a shared
		 * VMA is using the same page so check and skip such VMAs.
		 */
		if (iter_vma->vm_flags & VM_MAYSHARE)
			continue;

5899 5900 5901 5902 5903 5904 5905 5906
		/*
		 * Unmap the page from other VMAs without their own reserves.
		 * They get marked to be SIGKILLed if they fault in these
		 * areas. This is because a future no-page fault on this VMA
		 * could insert a zeroed page instead of the data existing
		 * from the time of fork. This would look like data corruption
		 */
		if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
5907
			unmap_hugepage_range(iter_vma, address,
5908
					     address + huge_page_size(h), page, 0);
5909
	}
5910
	i_mmap_unlock_write(mapping);
5911 5912
}

5913
/*
5914
 * hugetlb_wp() should be called with page lock of the original hugepage held.
5915
 * Called with hugetlb_fault_mutex_table held and pte_page locked so we
5916 5917
 * cannot race with other handlers or page migration.
 * Keep the pte_same checks anyway to make transition from the mutex easier.
5918
 */
5919 5920
static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma,
		       unsigned long address, pte_t *ptep, unsigned int flags,
5921 5922
		       struct folio *pagecache_folio, spinlock_t *ptl,
		       struct vm_fault *vmf)
5923
{
5924
	const bool unshare = flags & FAULT_FLAG_UNSHARE;
5925
	pte_t pte = huge_ptep_get(ptep);
5926
	struct hstate *h = hstate_vma(vma);
5927
	struct folio *old_folio;
5928
	struct folio *new_folio;
5929 5930
	int outside_reserve = 0;
	vm_fault_t ret = 0;
5931
	unsigned long haddr = address & huge_page_mask(h);
5932
	struct mmu_notifier_range range;
5933

5934 5935 5936 5937 5938 5939 5940 5941 5942 5943 5944
	/*
	 * Never handle CoW for uffd-wp protected pages.  It should be only
	 * handled when the uffd-wp protection is removed.
	 *
	 * Note that only the CoW optimization path (in hugetlb_no_page())
	 * can trigger this, because hugetlb_fault() will always resolve
	 * uffd-wp bit first.
	 */
	if (!unshare && huge_pte_uffd_wp(pte))
		return 0;

5945 5946 5947 5948 5949 5950 5951 5952 5953 5954 5955 5956 5957
	/*
	 * hugetlb does not support FOLL_FORCE-style write faults that keep the
	 * PTE mapped R/O such as maybe_mkwrite() would do.
	 */
	if (WARN_ON_ONCE(!unshare && !(vma->vm_flags & VM_WRITE)))
		return VM_FAULT_SIGSEGV;

	/* Let's take out MAP_SHARED mappings first. */
	if (vma->vm_flags & VM_MAYSHARE) {
		set_huge_ptep_writable(vma, haddr, ptep);
		return 0;
	}

5958
	old_folio = page_folio(pte_page(pte));
5959

5960 5961
	delayacct_wpcopy_start();

5962
retry_avoidcopy:
5963 5964 5965 5966
	/*
	 * If no-one else is actually using this page, we're the exclusive
	 * owner and can reuse this page.
	 */
5967
	if (folio_mapcount(old_folio) == 1 && folio_test_anon(old_folio)) {
5968
		if (!PageAnonExclusive(&old_folio->page)) {
5969
			folio_move_anon_rmap(old_folio, vma);
5970 5971
			SetPageAnonExclusive(&old_folio->page);
		}
5972 5973
		if (likely(!unshare))
			set_huge_ptep_writable(vma, haddr, ptep);
5974 5975

		delayacct_wpcopy_end();
Nick Piggin's avatar
Nick Piggin committed
5976
		return 0;
5977
	}
5978 5979
	VM_BUG_ON_PAGE(folio_test_anon(old_folio) &&
		       PageAnonExclusive(&old_folio->page), &old_folio->page);
5980

5981 5982 5983 5984 5985 5986 5987 5988 5989
	/*
	 * If the process that created a MAP_PRIVATE mapping is about to
	 * perform a COW due to a shared page count, attempt to satisfy
	 * the allocation without using the existing reserves. The pagecache
	 * page is used to determine if the reserve at this address was
	 * consumed or not. If reserves were used, a partial faulted mapping
	 * at the time of fork() could consume its reserves on COW instead
	 * of the full address range.
	 */
5990
	if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
5991
			old_folio != pagecache_folio)
5992 5993
		outside_reserve = 1;

5994
	folio_get(old_folio);
5995

5996 5997 5998 5999
	/*
	 * Drop page table lock as buddy allocator may be called. It will
	 * be acquired again before returning to the caller, as expected.
	 */
6000
	spin_unlock(ptl);
6001
	new_folio = alloc_hugetlb_folio(vma, haddr, outside_reserve);
6002

6003
	if (IS_ERR(new_folio)) {
6004 6005 6006 6007 6008 6009 6010 6011
		/*
		 * If a process owning a MAP_PRIVATE mapping fails to COW,
		 * it is due to references held by a child and an insufficient
		 * huge page pool. To guarantee the original mappers
		 * reliability, unmap the page from child processes. The child
		 * may get SIGKILLed if it later faults.
		 */
		if (outside_reserve) {
6012 6013 6014 6015
			struct address_space *mapping = vma->vm_file->f_mapping;
			pgoff_t idx;
			u32 hash;

6016
			folio_put(old_folio);
6017 6018 6019 6020 6021 6022 6023 6024 6025 6026 6027 6028 6029 6030
			/*
			 * Drop hugetlb_fault_mutex and vma_lock before
			 * unmapping.  unmapping needs to hold vma_lock
			 * in write mode.  Dropping vma_lock in read mode
			 * here is OK as COW mappings do not interact with
			 * PMD sharing.
			 *
			 * Reacquire both after unmap operation.
			 */
			idx = vma_hugecache_offset(h, vma, haddr);
			hash = hugetlb_fault_mutex_hash(mapping, idx);
			hugetlb_vma_unlock_read(vma);
			mutex_unlock(&hugetlb_fault_mutex_table[hash]);

6031
			unmap_ref_private(mm, vma, &old_folio->page, haddr);
6032 6033 6034

			mutex_lock(&hugetlb_fault_mutex_table[hash]);
			hugetlb_vma_lock_read(vma);
6035
			spin_lock(ptl);
6036
			ptep = hugetlb_walk(vma, haddr, huge_page_size(h));
6037 6038 6039 6040 6041 6042 6043
			if (likely(ptep &&
				   pte_same(huge_ptep_get(ptep), pte)))
				goto retry_avoidcopy;
			/*
			 * race occurs while re-acquiring page table
			 * lock, and our job is done.
			 */
6044
			delayacct_wpcopy_end();
6045
			return 0;
6046 6047
		}

6048
		ret = vmf_error(PTR_ERR(new_folio));
6049
		goto out_release_old;
6050 6051
	}

6052 6053 6054 6055
	/*
	 * When the original hugepage is shared one, it does not have
	 * anon_vma prepared.
	 */
6056 6057
	ret = vmf_anon_prepare(vmf);
	if (unlikely(ret))
6058
		goto out_release_all;
6059

6060
	if (copy_user_large_folio(new_folio, old_folio, address, vma)) {
6061 6062 6063
		ret = VM_FAULT_HWPOISON_LARGE;
		goto out_release_all;
	}
6064
	__folio_mark_uptodate(new_folio);
6065

6066
	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, haddr,
6067
				haddr + huge_page_size(h));
6068
	mmu_notifier_invalidate_range_start(&range);
6069

6070
	/*
6071
	 * Retake the page table lock to check for racing updates
6072 6073
	 * before the page tables are altered
	 */
6074
	spin_lock(ptl);
6075
	ptep = hugetlb_walk(vma, haddr, huge_page_size(h));
6076
	if (likely(ptep && pte_same(huge_ptep_get(ptep), pte))) {
6077 6078
		pte_t newpte = make_huge_pte(vma, &new_folio->page, !unshare);

6079
		/* Break COW or unshare */
6080
		huge_ptep_clear_flush(vma, haddr, ptep);
6081
		hugetlb_remove_rmap(old_folio);
6082
		hugetlb_add_new_anon_rmap(new_folio, vma, haddr);
6083 6084
		if (huge_pte_uffd_wp(pte))
			newpte = huge_pte_mkuffd_wp(newpte);
6085
		set_huge_pte_at(mm, haddr, ptep, newpte, huge_page_size(h));
6086
		folio_set_hugetlb_migratable(new_folio);
6087
		/* Make the old page be freed below */
6088
		new_folio = old_folio;
6089
	}
6090
	spin_unlock(ptl);
6091
	mmu_notifier_invalidate_range_end(&range);
6092
out_release_all:
6093 6094 6095 6096
	/*
	 * No restore in case of successful pagetable update (Break COW or
	 * unshare)
	 */
6097
	if (new_folio != old_folio)
6098
		restore_reserve_on_error(h, vma, haddr, new_folio);
6099
	folio_put(new_folio);
6100
out_release_old:
6101
	folio_put(old_folio);
6102

6103
	spin_lock(ptl); /* Caller expects lock to be held */
6104 6105

	delayacct_wpcopy_end();
6106
	return ret;
6107 6108
}

6109 6110 6111 6112
/*
 * Return whether there is a pagecache page to back given address within VMA.
 */
static bool hugetlbfs_pagecache_present(struct hstate *h,
Hugh Dickins's avatar
Hugh Dickins committed
6113 6114
			struct vm_area_struct *vma, unsigned long address)
{
6115
	struct address_space *mapping = vma->vm_file->f_mapping;
6116
	pgoff_t idx = linear_page_index(vma, address);
6117
	struct folio *folio;
Hugh Dickins's avatar
Hugh Dickins committed
6118

6119 6120 6121 6122 6123
	folio = filemap_get_folio(mapping, idx);
	if (IS_ERR(folio))
		return false;
	folio_put(folio);
	return true;
Hugh Dickins's avatar
Hugh Dickins committed
6124 6125
}

6126
int hugetlb_add_to_page_cache(struct folio *folio, struct address_space *mapping,
6127 6128 6129 6130
			   pgoff_t idx)
{
	struct inode *inode = mapping->host;
	struct hstate *h = hstate_inode(inode);
6131
	int err;
6132

6133
	idx <<= huge_page_order(h);
6134 6135 6136 6137 6138
	__folio_set_locked(folio);
	err = __filemap_add_folio(mapping, folio, idx, GFP_KERNEL, NULL);

	if (unlikely(err)) {
		__folio_clear_locked(folio);
6139
		return err;
6140
	}
6141
	folio_clear_hugetlb_restore_reserve(folio);
6142

6143
	/*
6144
	 * mark folio dirty so that it will not be removed from cache/file
6145 6146
	 * by non-hugetlbfs specific code paths.
	 */
6147
	folio_mark_dirty(folio);
6148

6149 6150 6151 6152 6153 6154
	spin_lock(&inode->i_lock);
	inode->i_blocks += blocks_per_huge_page(h);
	spin_unlock(&inode->i_lock);
	return 0;
}

6155
static inline vm_fault_t hugetlb_handle_userfault(struct vm_fault *vmf,
6156 6157 6158 6159 6160 6161
						  struct address_space *mapping,
						  unsigned long reason)
{
	u32 hash;

	/*
6162 6163 6164
	 * vma_lock and hugetlb_fault_mutex must be dropped before handling
	 * userfault. Also mmap_lock could be dropped due to handling
	 * userfault, any vma operation should be careful from here.
6165
	 */
6166 6167
	hugetlb_vma_unlock_read(vmf->vma);
	hash = hugetlb_fault_mutex_hash(mapping, vmf->pgoff);
6168
	mutex_unlock(&hugetlb_fault_mutex_table[hash]);
6169
	return handle_userfault(vmf, reason);
6170 6171
}

6172 6173 6174 6175 6176 6177 6178 6179 6180 6181 6182 6183 6184 6185 6186 6187 6188
/*
 * Recheck pte with pgtable lock.  Returns true if pte didn't change, or
 * false if pte changed or is changing.
 */
static bool hugetlb_pte_stable(struct hstate *h, struct mm_struct *mm,
			       pte_t *ptep, pte_t old_pte)
{
	spinlock_t *ptl;
	bool same;

	ptl = huge_pte_lock(h, mm, ptep);
	same = pte_same(huge_ptep_get(ptep), old_pte);
	spin_unlock(ptl);

	return same;
}

6189 6190 6191
static vm_fault_t hugetlb_no_page(struct mm_struct *mm,
			struct vm_area_struct *vma,
			struct address_space *mapping, pgoff_t idx,
6192
			unsigned long address, pte_t *ptep,
6193 6194
			pte_t old_pte, unsigned int flags,
			struct vm_fault *vmf)
6195
{
6196
	struct hstate *h = hstate_vma(vma);
6197
	vm_fault_t ret = VM_FAULT_SIGBUS;
6198
	int anon_rmap = 0;
6199
	unsigned long size;
6200
	struct folio *folio;
6201
	pte_t new_pte;
6202
	spinlock_t *ptl;
6203
	unsigned long haddr = address & huge_page_mask(h);
6204
	bool new_folio, new_pagecache_folio = false;
6205
	u32 hash = hugetlb_fault_mutex_hash(mapping, idx);
6206

6207 6208 6209
	/*
	 * Currently, we are forced to kill the process in the event the
	 * original mapper has unmapped pages from the child due to a failed
6210 6211
	 * COW/unsharing. Warn that such a situation has occurred as it may not
	 * be obvious.
6212 6213
	 */
	if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) {
6214
		pr_warn_ratelimited("PID %d killed due to inadequate hugepage pool\n",
6215
			   current->pid);
6216
		goto out;
6217 6218
	}

6219
	/*
6220 6221
	 * Use page lock to guard against racing truncation
	 * before we get page_table_lock.
6222
	 */
6223
	new_folio = false;
6224
	folio = filemap_lock_hugetlb_folio(h, mapping, idx);
6225
	if (IS_ERR(folio)) {
6226 6227 6228
		size = i_size_read(mapping->host) >> huge_page_shift(h);
		if (idx >= size)
			goto out;
6229
		/* Check for page in userfault range */
6230 6231 6232 6233 6234 6235 6236 6237 6238 6239 6240 6241 6242 6243 6244 6245 6246 6247 6248 6249 6250 6251 6252
		if (userfaultfd_missing(vma)) {
			/*
			 * Since hugetlb_no_page() was examining pte
			 * without pgtable lock, we need to re-test under
			 * lock because the pte may not be stable and could
			 * have changed from under us.  Try to detect
			 * either changed or during-changing ptes and retry
			 * properly when needed.
			 *
			 * Note that userfaultfd is actually fine with
			 * false positives (e.g. caused by pte changed),
			 * but not wrong logical events (e.g. caused by
			 * reading a pte during changing).  The latter can
			 * confuse the userspace, so the strictness is very
			 * much preferred.  E.g., MISSING event should
			 * never happen on the page after UFFDIO_COPY has
			 * correctly installed the page and returned.
			 */
			if (!hugetlb_pte_stable(h, mm, ptep, old_pte)) {
				ret = 0;
				goto out;
			}

6253
			return hugetlb_handle_userfault(vmf, mapping,
6254 6255
							VM_UFFD_MISSING);
		}
6256

6257 6258 6259 6260 6261 6262
		if (!(vma->vm_flags & VM_MAYSHARE)) {
			ret = vmf_anon_prepare(vmf);
			if (unlikely(ret))
				goto out;
		}

6263 6264
		folio = alloc_hugetlb_folio(vma, haddr, 0);
		if (IS_ERR(folio)) {
6265 6266 6267 6268 6269 6270 6271 6272 6273 6274 6275 6276
			/*
			 * Returning error will result in faulting task being
			 * sent SIGBUS.  The hugetlb fault mutex prevents two
			 * tasks from racing to fault in the same page which
			 * could result in false unable to allocate errors.
			 * Page migration does not take the fault mutex, but
			 * does a clear then write of pte's under page table
			 * lock.  Page fault code could race with migration,
			 * notice the clear pte and try to allocate a page
			 * here.  Before returning error, get ptl and make
			 * sure there really is no pte entry.
			 */
6277
			if (hugetlb_pte_stable(h, mm, ptep, old_pte))
6278
				ret = vmf_error(PTR_ERR(folio));
6279 6280
			else
				ret = 0;
6281 6282
			goto out;
		}
6283 6284 6285
		clear_huge_page(&folio->page, address, pages_per_huge_page(h));
		__folio_mark_uptodate(folio);
		new_folio = true;
6286

6287
		if (vma->vm_flags & VM_MAYSHARE) {
6288
			int err = hugetlb_add_to_page_cache(folio, mapping, idx);
6289
			if (err) {
6290 6291 6292 6293 6294 6295 6296
				/*
				 * err can't be -EEXIST which implies someone
				 * else consumed the reservation since hugetlb
				 * fault mutex is held when add a hugetlb page
				 * to the page cache. So it's safe to call
				 * restore_reserve_on_error() here.
				 */
6297
				restore_reserve_on_error(h, vma, haddr, folio);
6298
				folio_put(folio);
6299
				ret = VM_FAULT_SIGBUS;
6300 6301
				goto out;
			}
6302
			new_pagecache_folio = true;
6303
		} else {
6304
			folio_lock(folio);
6305
			anon_rmap = 1;
6306
		}
6307
	} else {
6308 6309 6310 6311 6312
		/*
		 * If memory error occurs between mmap() and fault, some process
		 * don't have hwpoisoned swap entry for errored virtual address.
		 * So we need to block hugepage fault by PG_hwpoison bit check.
		 */
6313
		if (unlikely(folio_test_hwpoison(folio))) {
6314
			ret = VM_FAULT_HWPOISON_LARGE |
6315
				VM_FAULT_SET_HINDEX(hstate_index(h));
6316 6317
			goto backout_unlocked;
		}
6318 6319 6320

		/* Check for page in userfault range. */
		if (userfaultfd_minor(vma)) {
6321 6322
			folio_unlock(folio);
			folio_put(folio);
6323 6324 6325 6326 6327
			/* See comment in userfaultfd_missing() block above */
			if (!hugetlb_pte_stable(h, mm, ptep, old_pte)) {
				ret = 0;
				goto out;
			}
6328
			return hugetlb_handle_userfault(vmf, mapping,
6329
							VM_UFFD_MINOR);
6330
		}
6331
	}
6332

6333 6334 6335 6336 6337 6338
	/*
	 * If we are going to COW a private mapping later, we examine the
	 * pending reservations for this page now. This will ensure that
	 * any allocations necessary to record that reservation occur outside
	 * the spinlock.
	 */
6339
	if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
6340
		if (vma_needs_reservation(h, vma, haddr) < 0) {
6341 6342 6343
			ret = VM_FAULT_OOM;
			goto backout_unlocked;
		}
6344
		/* Just decrements count, does not deallocate */
6345
		vma_end_reservation(h, vma, haddr);
6346
	}
6347

6348
	ptl = huge_pte_lock(h, mm, ptep);
Nick Piggin's avatar
Nick Piggin committed
6349
	ret = 0;
6350 6351
	/* If pte changed from under us, retry */
	if (!pte_same(huge_ptep_get(ptep), old_pte))
6352 6353
		goto backout;

6354
	if (anon_rmap)
6355
		hugetlb_add_new_anon_rmap(folio, vma, haddr);
6356
	else
6357
		hugetlb_add_file_rmap(folio);
6358
	new_pte = make_huge_pte(vma, &folio->page, ((vma->vm_flags & VM_WRITE)
6359
				&& (vma->vm_flags & VM_SHARED)));
6360 6361 6362 6363 6364
	/*
	 * If this pte was previously wr-protected, keep it wr-protected even
	 * if populated.
	 */
	if (unlikely(pte_marker_uffd_wp(old_pte)))
6365
		new_pte = huge_pte_mkuffd_wp(new_pte);
6366
	set_huge_pte_at(mm, haddr, ptep, new_pte, huge_page_size(h));
6367

6368
	hugetlb_count_add(pages_per_huge_page(h), mm);
6369
	if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
6370
		/* Optimization, do the COW without a second fault */
6371
		ret = hugetlb_wp(mm, vma, address, ptep, flags, folio, ptl, vmf);
6372 6373
	}

6374
	spin_unlock(ptl);
6375 6376

	/*
6377 6378
	 * Only set hugetlb_migratable in newly allocated pages.  Existing pages
	 * found in the pagecache may not have hugetlb_migratable if they have
6379
	 * been isolated for migration.
6380
	 */
6381 6382
	if (new_folio)
		folio_set_hugetlb_migratable(folio);
6383

6384
	folio_unlock(folio);
6385
out:
6386 6387
	hugetlb_vma_unlock_read(vma);
	mutex_unlock(&hugetlb_fault_mutex_table[hash]);
6388
	return ret;
6389 6390

backout:
6391
	spin_unlock(ptl);
6392
backout_unlocked:
6393
	if (new_folio && !new_pagecache_folio)
6394
		restore_reserve_on_error(h, vma, haddr, folio);
6395

6396 6397
	folio_unlock(folio);
	folio_put(folio);
6398
	goto out;
6399 6400
}

6401
#ifdef CONFIG_SMP
6402
u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx)
6403 6404 6405 6406
{
	unsigned long key[2];
	u32 hash;

6407 6408
	key[0] = (unsigned long) mapping;
	key[1] = idx;
6409

6410
	hash = jhash2((u32 *)&key, sizeof(key)/(sizeof(u32)), 0);
6411 6412 6413 6414 6415

	return hash & (num_fault_mutexes - 1);
}
#else
/*
6416
 * For uniprocessor systems we always use a single mutex, so just
6417 6418
 * return 0 and avoid the hashing overhead.
 */
6419
u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx)
6420 6421 6422 6423 6424
{
	return 0;
}
#endif

6425
vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
6426
			unsigned long address, unsigned int flags)
6427
{
6428
	pte_t *ptep, entry;
6429
	spinlock_t *ptl;
6430
	vm_fault_t ret;
6431
	u32 hash;
6432
	struct folio *folio = NULL;
6433
	struct folio *pagecache_folio = NULL;
6434
	struct hstate *h = hstate_vma(vma);
6435
	struct address_space *mapping;
6436
	int need_wait_lock = 0;
6437
	unsigned long haddr = address & huge_page_mask(h);
6438 6439 6440 6441 6442 6443 6444 6445 6446 6447 6448 6449 6450
	struct vm_fault vmf = {
		.vma = vma,
		.address = haddr,
		.real_address = address,
		.flags = flags,
		.pgoff = vma_hugecache_offset(h, vma, haddr),
		/* TODO: Track hugetlb faults using vm_fault */

		/*
		 * Some fields may not be initialized, be careful as it may
		 * be hard to debug if called functions make assumptions
		 */
	};
6451

6452 6453 6454 6455 6456
	/*
	 * Serialize hugepage allocation and instantiation, so that we don't
	 * get spurious allocation failures if two CPUs race to instantiate
	 * the same page in the page cache.
	 */
6457
	mapping = vma->vm_file->f_mapping;
6458
	hash = hugetlb_fault_mutex_hash(mapping, vmf.pgoff);
6459
	mutex_lock(&hugetlb_fault_mutex_table[hash]);
6460

6461 6462 6463 6464 6465 6466 6467 6468 6469 6470 6471 6472 6473
	/*
	 * Acquire vma lock before calling huge_pte_alloc and hold
	 * until finished with ptep.  This prevents huge_pmd_unshare from
	 * being called elsewhere and making the ptep no longer valid.
	 */
	hugetlb_vma_lock_read(vma);
	ptep = huge_pte_alloc(mm, vma, haddr, huge_page_size(h));
	if (!ptep) {
		hugetlb_vma_unlock_read(vma);
		mutex_unlock(&hugetlb_fault_mutex_table[hash]);
		return VM_FAULT_OOM;
	}

6474
	entry = huge_ptep_get(ptep);
6475 6476 6477 6478 6479 6480 6481 6482 6483 6484 6485
	if (huge_pte_none_mostly(entry)) {
		if (is_pte_marker(entry)) {
			pte_marker marker =
				pte_marker_get(pte_to_swp_entry(entry));

			if (marker & PTE_MARKER_POISONED) {
				ret = VM_FAULT_HWPOISON_LARGE;
				goto out_mutex;
			}
		}

6486
		/*
6487 6488
		 * Other PTE markers should be handled the same way as none PTE.
		 *
6489 6490 6491
		 * hugetlb_no_page will drop vma lock and hugetlb fault
		 * mutex internally, which make us return immediately.
		 */
6492
		return hugetlb_no_page(mm, vma, mapping, vmf.pgoff, address,
6493
					ptep, entry, flags, &vmf);
6494
	}
6495

Nick Piggin's avatar
Nick Piggin committed
6496
	ret = 0;
6497

6498 6499 6500
	/*
	 * entry could be a migration/hwpoison entry at this point, so this
	 * check prevents the kernel from going below assuming that we have
6501 6502 6503
	 * an active hugepage in pagecache. This goto expects the 2nd page
	 * fault, and is_hugetlb_entry_(migration|hwpoisoned) check will
	 * properly handle it.
6504
	 */
6505 6506 6507 6508 6509 6510 6511 6512 6513 6514 6515 6516 6517 6518 6519
	if (!pte_present(entry)) {
		if (unlikely(is_hugetlb_entry_migration(entry))) {
			/*
			 * Release the hugetlb fault lock now, but retain
			 * the vma lock, because it is needed to guard the
			 * huge_pte_lockptr() later in
			 * migration_entry_wait_huge(). The vma lock will
			 * be released there.
			 */
			mutex_unlock(&hugetlb_fault_mutex_table[hash]);
			migration_entry_wait_huge(vma, ptep);
			return 0;
		} else if (unlikely(is_hugetlb_entry_hwpoisoned(entry)))
			ret = VM_FAULT_HWPOISON_LARGE |
			    VM_FAULT_SET_HINDEX(hstate_index(h));
6520
		goto out_mutex;
6521
	}
6522

6523
	/*
6524 6525
	 * If we are going to COW/unshare the mapping later, we examine the
	 * pending reservations for this page now. This will ensure that any
6526
	 * allocations necessary to record that reservation occur outside the
6527 6528
	 * spinlock. Also lookup the pagecache page now as it is used to
	 * determine if a reservation has been consumed.
6529
	 */
6530
	if ((flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) &&
6531
	    !(vma->vm_flags & VM_MAYSHARE) && !huge_pte_write(entry)) {
6532
		if (vma_needs_reservation(h, vma, haddr) < 0) {
6533
			ret = VM_FAULT_OOM;
6534
			goto out_mutex;
6535
		}
6536
		/* Just decrements count, does not deallocate */
6537
		vma_end_reservation(h, vma, haddr);
6538

6539 6540
		pagecache_folio = filemap_lock_hugetlb_folio(h, mapping,
							     vmf.pgoff);
6541 6542
		if (IS_ERR(pagecache_folio))
			pagecache_folio = NULL;
6543 6544
	}

6545 6546
	ptl = huge_pte_lock(h, mm, ptep);

6547
	/* Check for a racing update before calling hugetlb_wp() */
6548 6549 6550
	if (unlikely(!pte_same(entry, huge_ptep_get(ptep))))
		goto out_ptl;

6551 6552 6553
	/* Handle userfault-wp first, before trying to lock more pages */
	if (userfaultfd_wp(vma) && huge_pte_uffd_wp(huge_ptep_get(ptep)) &&
	    (flags & FAULT_FLAG_WRITE) && !huge_pte_write(entry)) {
6554 6555 6556 6557 6558 6559 6560 6561 6562
		if (!userfaultfd_wp_async(vma)) {
			spin_unlock(ptl);
			if (pagecache_folio) {
				folio_unlock(pagecache_folio);
				folio_put(pagecache_folio);
			}
			hugetlb_vma_unlock_read(vma);
			mutex_unlock(&hugetlb_fault_mutex_table[hash]);
			return handle_userfault(&vmf, VM_UFFD_WP);
6563
		}
6564 6565

		entry = huge_pte_clear_uffd_wp(entry);
6566 6567
		set_huge_pte_at(mm, haddr, ptep, entry,
				huge_page_size(hstate_vma(vma)));
6568
		/* Fallthrough to CoW */
6569 6570
	}

6571
	/*
6572
	 * hugetlb_wp() requires page locks of pte_page(entry) and
6573
	 * pagecache_folio, so here we need take the former one
6574
	 * when folio != pagecache_folio or !pagecache_folio.
6575
	 */
6576 6577 6578
	folio = page_folio(pte_page(entry));
	if (folio != pagecache_folio)
		if (!folio_trylock(folio)) {
6579 6580 6581
			need_wait_lock = 1;
			goto out_ptl;
		}
6582

6583
	folio_get(folio);
6584

6585
	if (flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) {
6586
		if (!huge_pte_write(entry)) {
6587
			ret = hugetlb_wp(mm, vma, address, ptep, flags,
6588
					 pagecache_folio, ptl, &vmf);
6589
			goto out_put_page;
6590 6591
		} else if (likely(flags & FAULT_FLAG_WRITE)) {
			entry = huge_pte_mkdirty(entry);
6592 6593 6594
		}
	}
	entry = pte_mkyoung(entry);
6595
	if (huge_ptep_set_access_flags(vma, haddr, ptep, entry,
6596
						flags & FAULT_FLAG_WRITE))
6597
		update_mmu_cache(vma, haddr, ptep);
6598
out_put_page:
6599 6600 6601
	if (folio != pagecache_folio)
		folio_unlock(folio);
	folio_put(folio);
6602 6603
out_ptl:
	spin_unlock(ptl);
6604

6605 6606 6607
	if (pagecache_folio) {
		folio_unlock(pagecache_folio);
		folio_put(pagecache_folio);
6608
	}
6609
out_mutex:
6610
	hugetlb_vma_unlock_read(vma);
6611
	mutex_unlock(&hugetlb_fault_mutex_table[hash]);
6612 6613 6614 6615 6616 6617 6618 6619
	/*
	 * Generally it's safe to hold refcount during waiting page lock. But
	 * here we just wait to defer the next page fault to avoid busy loop and
	 * the page is not used after unlocked before returning from the current
	 * page fault. So we are safe from accessing freed page, even if we wait
	 * here without taking refcount.
	 */
	if (need_wait_lock)
6620
		folio_wait_locked(folio);
6621
	return ret;
6622 6623
}

6624
#ifdef CONFIG_USERFAULTFD
6625 6626 6627 6628 6629 6630 6631 6632 6633 6634 6635 6636 6637 6638 6639 6640 6641 6642 6643 6644
/*
 * Can probably be eliminated, but still used by hugetlb_mfill_atomic_pte().
 */
static struct folio *alloc_hugetlb_folio_vma(struct hstate *h,
		struct vm_area_struct *vma, unsigned long address)
{
	struct mempolicy *mpol;
	nodemask_t *nodemask;
	struct folio *folio;
	gfp_t gfp_mask;
	int node;

	gfp_mask = htlb_alloc_mask(h);
	node = huge_node(vma, address, gfp_mask, &mpol, &nodemask);
	folio = alloc_hugetlb_folio_nodemask(h, node, nodemask, gfp_mask);
	mpol_cond_put(mpol);

	return folio;
}

6645
/*
6646 6647
 * Used by userfaultfd UFFDIO_* ioctls. Based on userfaultfd's mfill_atomic_pte
 * with modifications for hugetlb pages.
6648
 */
6649
int hugetlb_mfill_atomic_pte(pte_t *dst_pte,
6650 6651 6652
			     struct vm_area_struct *dst_vma,
			     unsigned long dst_addr,
			     unsigned long src_addr,
6653
			     uffd_flags_t flags,
6654
			     struct folio **foliop)
6655
{
6656
	struct mm_struct *dst_mm = dst_vma->vm_mm;
6657 6658
	bool is_continue = uffd_flags_mode_is(flags, MFILL_ATOMIC_CONTINUE);
	bool wp_enabled = (flags & MFILL_ATOMIC_WP);
6659 6660 6661
	struct hstate *h = hstate_vma(dst_vma);
	struct address_space *mapping = dst_vma->vm_file->f_mapping;
	pgoff_t idx = vma_hugecache_offset(h, dst_vma, dst_addr);
6662
	unsigned long size;
6663
	int vm_shared = dst_vma->vm_flags & VM_SHARED;
6664 6665
	pte_t _dst_pte;
	spinlock_t *ptl;
6666
	int ret = -ENOMEM;
6667
	struct folio *folio;
6668
	int writable;
6669
	bool folio_in_pagecache = false;
6670

6671 6672 6673 6674 6675 6676 6677 6678 6679 6680
	if (uffd_flags_mode_is(flags, MFILL_ATOMIC_POISON)) {
		ptl = huge_pte_lock(h, dst_mm, dst_pte);

		/* Don't overwrite any existing PTEs (even markers) */
		if (!huge_pte_none(huge_ptep_get(dst_pte))) {
			spin_unlock(ptl);
			return -EEXIST;
		}

		_dst_pte = make_pte_marker(PTE_MARKER_POISONED);
6681 6682
		set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte,
				huge_page_size(h));
6683 6684 6685 6686 6687 6688 6689 6690

		/* No need to invalidate - it was non-present before */
		update_mmu_cache(dst_vma, dst_addr, dst_pte);

		spin_unlock(ptl);
		return 0;
	}

6691 6692
	if (is_continue) {
		ret = -EFAULT;
6693
		folio = filemap_lock_hugetlb_folio(h, mapping, idx);
6694
		if (IS_ERR(folio))
6695
			goto out;
6696
		folio_in_pagecache = true;
6697 6698
	} else if (!*foliop) {
		/* If a folio already exists, then it's UFFDIO_COPY for
6699 6700 6701 6702 6703 6704 6705 6706
		 * a non-missing case. Return -EEXIST.
		 */
		if (vm_shared &&
		    hugetlbfs_pagecache_present(h, dst_vma, dst_addr)) {
			ret = -EEXIST;
			goto out;
		}

6707 6708
		folio = alloc_hugetlb_folio(dst_vma, dst_addr, 0);
		if (IS_ERR(folio)) {
6709
			ret = -ENOMEM;
6710
			goto out;
6711
		}
6712

6713 6714
		ret = copy_folio_from_user(folio, (const void __user *) src_addr,
					   false);
6715

6716
		/* fallback to copy_from_user outside mmap_lock */
6717
		if (unlikely(ret)) {
6718
			ret = -ENOENT;
6719
			/* Free the allocated folio which may have
6720 6721
			 * consumed a reservation.
			 */
6722
			restore_reserve_on_error(h, dst_vma, dst_addr, folio);
6723
			folio_put(folio);
6724

6725
			/* Allocate a temporary folio to hold the copied
6726 6727
			 * contents.
			 */
6728 6729
			folio = alloc_hugetlb_folio_vma(h, dst_vma, dst_addr);
			if (!folio) {
6730 6731 6732
				ret = -ENOMEM;
				goto out;
			}
6733 6734
			*foliop = folio;
			/* Set the outparam foliop and return to the caller to
6735
			 * copy the contents outside the lock. Don't free the
6736
			 * folio.
6737
			 */
6738 6739 6740
			goto out;
		}
	} else {
6741 6742
		if (vm_shared &&
		    hugetlbfs_pagecache_present(h, dst_vma, dst_addr)) {
6743
			folio_put(*foliop);
6744
			ret = -EEXIST;
6745
			*foliop = NULL;
6746 6747 6748
			goto out;
		}

6749 6750
		folio = alloc_hugetlb_folio(dst_vma, dst_addr, 0);
		if (IS_ERR(folio)) {
6751
			folio_put(*foliop);
6752
			ret = -ENOMEM;
6753
			*foliop = NULL;
6754 6755
			goto out;
		}
6756
		ret = copy_user_large_folio(folio, *foliop, dst_addr, dst_vma);
6757 6758
		folio_put(*foliop);
		*foliop = NULL;
6759 6760
		if (ret) {
			folio_put(folio);
6761 6762
			goto out;
		}
6763 6764 6765
	}

	/*
6766 6767 6768 6769 6770 6771 6772 6773 6774
	 * If we just allocated a new page, we need a memory barrier to ensure
	 * that preceding stores to the page become visible before the
	 * set_pte_at() write. The memory barrier inside __folio_mark_uptodate
	 * is what we need.
	 *
	 * In the case where we have not allocated a new page (is_continue),
	 * the page must already be uptodate. UFFDIO_CONTINUE already includes
	 * an earlier smp_wmb() to ensure that prior stores will be visible
	 * before the set_pte_at() write.
6775
	 */
6776 6777 6778 6779
	if (!is_continue)
		__folio_mark_uptodate(folio);
	else
		WARN_ON_ONCE(!folio_test_uptodate(folio));
6780

6781 6782
	/* Add shared, newly allocated pages to the page cache. */
	if (vm_shared && !is_continue) {
6783 6784 6785 6786
		size = i_size_read(mapping->host) >> huge_page_shift(h);
		ret = -EFAULT;
		if (idx >= size)
			goto out_release_nounlock;
6787

6788 6789
		/*
		 * Serialization between remove_inode_hugepages() and
6790
		 * hugetlb_add_to_page_cache() below happens through the
6791 6792 6793
		 * hugetlb_fault_mutex_table that here must be hold by
		 * the caller.
		 */
6794
		ret = hugetlb_add_to_page_cache(folio, mapping, idx);
6795 6796
		if (ret)
			goto out_release_nounlock;
6797
		folio_in_pagecache = true;
6798 6799
	}

6800
	ptl = huge_pte_lock(h, dst_mm, dst_pte);
6801

6802
	ret = -EIO;
6803
	if (folio_test_hwpoison(folio))
6804 6805
		goto out_release_unlock;

6806 6807 6808 6809 6810
	/*
	 * We allow to overwrite a pte marker: consider when both MISSING|WP
	 * registered, we firstly wr-protect a none pte which has no page cache
	 * page backing it, then access the page.
	 */
6811
	ret = -EEXIST;
6812
	if (!huge_pte_none_mostly(huge_ptep_get(dst_pte)))
6813 6814
		goto out_release_unlock;

6815
	if (folio_in_pagecache)
6816
		hugetlb_add_file_rmap(folio);
6817
	else
6818
		hugetlb_add_new_anon_rmap(folio, dst_vma, dst_addr);
6819

6820 6821 6822 6823
	/*
	 * For either: (1) CONTINUE on a non-shared VMA, or (2) UFFDIO_COPY
	 * with wp flag set, don't set pte write bit.
	 */
6824
	if (wp_enabled || (is_continue && !vm_shared))
6825 6826 6827 6828
		writable = 0;
	else
		writable = dst_vma->vm_flags & VM_WRITE;

6829
	_dst_pte = make_huge_pte(dst_vma, &folio->page, writable);
6830 6831 6832 6833 6834 6835 6836
	/*
	 * Always mark UFFDIO_COPY page dirty; note that this may not be
	 * extremely important for hugetlbfs for now since swapping is not
	 * supported, but we should still be clear in that this page cannot be
	 * thrown away at will, even if write bit not set.
	 */
	_dst_pte = huge_pte_mkdirty(_dst_pte);
6837 6838
	_dst_pte = pte_mkyoung(_dst_pte);

6839
	if (wp_enabled)
6840 6841
		_dst_pte = huge_pte_mkuffd_wp(_dst_pte);

6842
	set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte, huge_page_size(h));
6843 6844 6845 6846 6847 6848 6849

	hugetlb_count_add(pages_per_huge_page(h), dst_mm);

	/* No need to invalidate - it was non-present before */
	update_mmu_cache(dst_vma, dst_addr, dst_pte);

	spin_unlock(ptl);
6850
	if (!is_continue)
6851
		folio_set_hugetlb_migratable(folio);
6852
	if (vm_shared || is_continue)
6853
		folio_unlock(folio);
6854 6855 6856 6857 6858
	ret = 0;
out:
	return ret;
out_release_unlock:
	spin_unlock(ptl);
6859
	if (vm_shared || is_continue)
6860
		folio_unlock(folio);
6861
out_release_nounlock:
6862
	if (!folio_in_pagecache)
6863
		restore_reserve_on_error(h, dst_vma, dst_addr, folio);
6864
	folio_put(folio);
6865 6866
	goto out;
}
6867
#endif /* CONFIG_USERFAULTFD */
6868

6869
struct page *hugetlb_follow_page_mask(struct vm_area_struct *vma,
6870 6871
				      unsigned long address, unsigned int flags,
				      unsigned int *page_mask)
6872 6873 6874 6875 6876 6877 6878
{
	struct hstate *h = hstate_vma(vma);
	struct mm_struct *mm = vma->vm_mm;
	unsigned long haddr = address & huge_page_mask(h);
	struct page *page = NULL;
	spinlock_t *ptl;
	pte_t *pte, entry;
6879
	int ret;
6880

6881
	hugetlb_vma_lock_read(vma);
6882
	pte = hugetlb_walk(vma, haddr, huge_page_size(h));
6883
	if (!pte)
6884
		goto out_unlock;
6885 6886 6887 6888

	ptl = huge_pte_lock(h, mm, pte);
	entry = huge_ptep_get(pte);
	if (pte_present(entry)) {
6889 6890 6891 6892 6893 6894 6895 6896 6897 6898 6899 6900 6901 6902 6903
		page = pte_page(entry);

		if (!huge_pte_write(entry)) {
			if (flags & FOLL_WRITE) {
				page = NULL;
				goto out;
			}

			if (gup_must_unshare(vma, flags, page)) {
				/* Tell the caller to do unsharing */
				page = ERR_PTR(-EMLINK);
				goto out;
			}
		}

6904
		page = nth_page(page, ((address & ~huge_page_mask(h)) >> PAGE_SHIFT));
6905

6906 6907 6908 6909 6910 6911
		/*
		 * Note that page may be a sub-page, and with vmemmap
		 * optimizations the page struct may be read only.
		 * try_grab_page() will increase the ref count on the
		 * head page, so this will be OK.
		 *
6912 6913
		 * try_grab_page() should always be able to get the page here,
		 * because we hold the ptl lock and have verified pte_present().
6914
		 */
6915 6916 6917 6918
		ret = try_grab_page(page, flags);

		if (WARN_ON_ONCE(ret)) {
			page = ERR_PTR(ret);
6919 6920
			goto out;
		}
6921 6922

		*page_mask = (1U << huge_page_order(h)) - 1;
6923 6924 6925
	}
out:
	spin_unlock(ptl);
6926 6927
out_unlock:
	hugetlb_vma_unlock_read(vma);
6928 6929 6930 6931 6932 6933 6934 6935 6936

	/*
	 * Fixup retval for dump requests: if pagecache doesn't exist,
	 * don't try to allocate a new page but just skip it.
	 */
	if (!page && (flags & FOLL_DUMP) &&
	    !hugetlbfs_pagecache_present(h, vma, address))
		page = ERR_PTR(-EFAULT);

6937 6938 6939
	return page;
}

6940
long hugetlb_change_protection(struct vm_area_struct *vma,
6941 6942
		unsigned long address, unsigned long end,
		pgprot_t newprot, unsigned long cp_flags)
6943 6944 6945 6946 6947
{
	struct mm_struct *mm = vma->vm_mm;
	unsigned long start = address;
	pte_t *ptep;
	pte_t pte;
6948
	struct hstate *h = hstate_vma(vma);
6949
	long pages = 0, psize = huge_page_size(h);
6950
	bool shared_pmd = false;
6951
	struct mmu_notifier_range range;
6952
	unsigned long last_addr_mask;
6953 6954
	bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
	bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
6955 6956 6957

	/*
	 * In the case of shared PMDs, the area to flush could be beyond
6958
	 * start/end.  Set range.start/range.end to cover the maximum possible
6959 6960
	 * range if PMD sharing is possible.
	 */
6961
	mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_VMA,
6962
				0, mm, start, end);
6963
	adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
6964 6965

	BUG_ON(address >= end);
6966
	flush_cache_range(vma, range.start, range.end);
6967

6968
	mmu_notifier_invalidate_range_start(&range);
6969
	hugetlb_vma_lock_write(vma);
6970
	i_mmap_lock_write(vma->vm_file->f_mapping);
6971
	last_addr_mask = hugetlb_mask_last_page(h);
6972
	for (; address < end; address += psize) {
6973
		spinlock_t *ptl;
6974
		ptep = hugetlb_walk(vma, address, psize);
6975
		if (!ptep) {
6976 6977 6978 6979 6980 6981 6982 6983 6984
			if (!uffd_wp) {
				address |= last_addr_mask;
				continue;
			}
			/*
			 * Userfaultfd wr-protect requires pgtable
			 * pre-allocations to install pte markers.
			 */
			ptep = huge_pte_alloc(mm, vma, address, psize);
6985 6986
			if (!ptep) {
				pages = -ENOMEM;
6987
				break;
6988
			}
6989
		}
6990
		ptl = huge_pte_lock(h, mm, ptep);
6991
		if (huge_pmd_unshare(mm, vma, address, ptep)) {
6992 6993 6994 6995 6996 6997
			/*
			 * When uffd-wp is enabled on the vma, unshare
			 * shouldn't happen at all.  Warn about it if it
			 * happened due to some reason.
			 */
			WARN_ON_ONCE(uffd_wp || uffd_wp_resolve);
6998
			pages++;
6999
			spin_unlock(ptl);
7000
			shared_pmd = true;
7001
			address |= last_addr_mask;
7002
			continue;
7003
		}
7004 7005
		pte = huge_ptep_get(ptep);
		if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) {
7006 7007
			/* Nothing to do. */
		} else if (unlikely(is_hugetlb_entry_migration(pte))) {
7008
			swp_entry_t entry = pte_to_swp_entry(pte);
7009
			struct page *page = pfn_swap_entry_to_page(entry);
7010
			pte_t newpte = pte;
7011

7012
			if (is_writable_migration_entry(entry)) {
7013 7014 7015 7016 7017 7018
				if (PageAnon(page))
					entry = make_readable_exclusive_migration_entry(
								swp_offset(entry));
				else
					entry = make_readable_migration_entry(
								swp_offset(entry));
7019 7020 7021
				newpte = swp_entry_to_pte(entry);
				pages++;
			}
7022 7023 7024 7025 7026 7027

			if (uffd_wp)
				newpte = pte_swp_mkuffd_wp(newpte);
			else if (uffd_wp_resolve)
				newpte = pte_swp_clear_uffd_wp(newpte);
			if (!pte_same(pte, newpte))
7028
				set_huge_pte_at(mm, address, ptep, newpte, psize);
7029
		} else if (unlikely(is_pte_marker(pte))) {
7030 7031 7032 7033 7034 7035 7036
			/*
			 * Do nothing on a poison marker; page is
			 * corrupted, permissons do not apply.  Here
			 * pte_marker_uffd_wp()==true implies !poison
			 * because they're mutual exclusive.
			 */
			if (pte_marker_uffd_wp(pte) && uffd_wp_resolve)
7037
				/* Safe to modify directly (non-present->none). */
7038
				huge_pte_clear(mm, address, ptep, psize);
7039
		} else if (!huge_pte_none(pte)) {
7040
			pte_t old_pte;
7041
			unsigned int shift = huge_page_shift(hstate_vma(vma));
7042 7043

			old_pte = huge_ptep_modify_prot_start(vma, address, ptep);
7044
			pte = huge_pte_modify(old_pte, newprot);
7045
			pte = arch_make_huge_pte(pte, shift, vma->vm_flags);
7046
			if (uffd_wp)
7047
				pte = huge_pte_mkuffd_wp(pte);
7048 7049
			else if (uffd_wp_resolve)
				pte = huge_pte_clear_uffd_wp(pte);
7050
			huge_ptep_modify_prot_commit(vma, address, ptep, old_pte, pte);
7051
			pages++;
7052 7053 7054 7055 7056
		} else {
			/* None pte */
			if (unlikely(uffd_wp))
				/* Safe to modify directly (none->non-present). */
				set_huge_pte_at(mm, address, ptep,
7057 7058
						make_pte_marker(PTE_MARKER_UFFD_WP),
						psize);
7059
		}
7060
		spin_unlock(ptl);
7061
	}
7062
	/*
7063
	 * Must flush TLB before releasing i_mmap_rwsem: x86's huge_pmd_unshare
7064
	 * may have cleared our pud entry and done put_page on the page table:
7065
	 * once we release i_mmap_rwsem, another task can do the final put_page
7066 7067
	 * and that page table be reused and filled with junk.  If we actually
	 * did unshare a page of pmds, flush the range corresponding to the pud.
7068
	 */
7069
	if (shared_pmd)
7070
		flush_hugetlb_tlb_range(vma, range.start, range.end);
7071 7072
	else
		flush_hugetlb_tlb_range(vma, start, end);
7073
	/*
7074 7075 7076
	 * No need to call mmu_notifier_arch_invalidate_secondary_tlbs() we are
	 * downgrading page table protection not changing it to point to a new
	 * page.
7077
	 *
7078
	 * See Documentation/mm/mmu_notifier.rst
7079
	 */
7080
	i_mmap_unlock_write(vma->vm_file->f_mapping);
7081
	hugetlb_vma_unlock_write(vma);
7082
	mmu_notifier_invalidate_range_end(&range);
7083

7084
	return pages > 0 ? (pages << h->order) : pages;
7085 7086
}

7087 7088
/* Return true if reservation was successful, false otherwise.  */
bool hugetlb_reserve_pages(struct inode *inode,
7089
					long from, long to,
7090
					struct vm_area_struct *vma,
7091
					vm_flags_t vm_flags)
7092
{
7093
	long chg = -1, add = -1;
7094
	struct hstate *h = hstate_inode(inode);
7095
	struct hugepage_subpool *spool = subpool_inode(inode);
7096
	struct resv_map *resv_map;
7097
	struct hugetlb_cgroup *h_cg = NULL;
7098
	long gbl_reserve, regions_needed = 0;
7099

7100 7101 7102
	/* This should never happen */
	if (from > to) {
		VM_WARN(1, "%s called with a negative range\n", __func__);
7103
		return false;
7104 7105
	}

7106
	/*
7107 7108
	 * vma specific semaphore used for pmd sharing and fault/truncation
	 * synchronization
7109 7110 7111
	 */
	hugetlb_vma_lock_alloc(vma);

7112 7113 7114
	/*
	 * Only apply hugepage reservation if asked. At fault time, an
	 * attempt will be made for VM_NORESERVE to allocate a page
7115
	 * without using reserves
7116
	 */
7117
	if (vm_flags & VM_NORESERVE)
7118
		return true;
7119

7120 7121 7122 7123 7124 7125
	/*
	 * Shared mappings base their reservation on the number of pages that
	 * are already allocated on behalf of the file. Private mappings need
	 * to reserve the full area even if read-only as mprotect() may be
	 * called to make the mapping read-write. Assume !vma is a shm mapping
	 */
7126
	if (!vma || vma->vm_flags & VM_MAYSHARE) {
7127 7128 7129 7130 7131
		/*
		 * resv_map can not be NULL as hugetlb_reserve_pages is only
		 * called for inodes for which resv_maps were created (see
		 * hugetlbfs_get_inode).
		 */
7132
		resv_map = inode_resv_map(inode);
7133

7134
		chg = region_chg(resv_map, from, to, &regions_needed);
7135
	} else {
7136
		/* Private mapping. */
7137
		resv_map = resv_map_alloc();
7138
		if (!resv_map)
7139
			goto out_err;
7140

7141
		chg = to - from;
7142

7143 7144 7145 7146
		set_vma_resv_map(vma, resv_map);
		set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
	}

7147
	if (chg < 0)
7148
		goto out_err;
7149

7150 7151
	if (hugetlb_cgroup_charge_cgroup_rsvd(hstate_index(h),
				chg * pages_per_huge_page(h), &h_cg) < 0)
7152 7153 7154 7155 7156 7157 7158 7159 7160
		goto out_err;

	if (vma && !(vma->vm_flags & VM_MAYSHARE) && h_cg) {
		/* For private mappings, the hugetlb_cgroup uncharge info hangs
		 * of the resv_map.
		 */
		resv_map_set_hugetlb_cgroup_uncharge_info(resv_map, h_cg, h);
	}

7161 7162 7163 7164 7165 7166
	/*
	 * There must be enough pages in the subpool for the mapping. If
	 * the subpool has a minimum size, there may be some global
	 * reservations already in place (gbl_reserve).
	 */
	gbl_reserve = hugepage_subpool_get_pages(spool, chg);
7167
	if (gbl_reserve < 0)
7168
		goto out_uncharge_cgroup;
7169 7170

	/*
7171
	 * Check enough hugepages are available for the reservation.
7172
	 * Hand the pages back to the subpool if there are not
7173
	 */
7174
	if (hugetlb_acct_memory(h, gbl_reserve) < 0)
7175
		goto out_put_pages;
7176 7177 7178 7179 7180 7181 7182 7183 7184 7185 7186 7187

	/*
	 * Account for the reservations made. Shared mappings record regions
	 * that have reservations as they are shared by multiple VMAs.
	 * When the last VMA disappears, the region map says how much
	 * the reservation was and the page cache tells how much of
	 * the reservation was consumed. Private mappings are per-VMA and
	 * only the consumed reservations are tracked. When the VMA
	 * disappears, the original reservation is the VMA size and the
	 * consumed reservations are stored in the map. Hence, nothing
	 * else has to be done for private mappings here
	 */
7188
	if (!vma || vma->vm_flags & VM_MAYSHARE) {
7189
		add = region_add(resv_map, from, to, regions_needed, h, h_cg);
7190 7191 7192

		if (unlikely(add < 0)) {
			hugetlb_acct_memory(h, -gbl_reserve);
7193
			goto out_put_pages;
7194
		} else if (unlikely(chg > add)) {
7195 7196 7197
			/*
			 * pages in this range were added to the reserve
			 * map between region_chg and region_add.  This
7198
			 * indicates a race with alloc_hugetlb_folio.  Adjust
7199 7200 7201 7202 7203
			 * the subpool and reserve counts modified above
			 * based on the difference.
			 */
			long rsv_adjust;

7204 7205 7206 7207
			/*
			 * hugetlb_cgroup_uncharge_cgroup_rsvd() will put the
			 * reference to h_cg->css. See comment below for detail.
			 */
7208 7209 7210 7211
			hugetlb_cgroup_uncharge_cgroup_rsvd(
				hstate_index(h),
				(chg - add) * pages_per_huge_page(h), h_cg);

7212 7213 7214
			rsv_adjust = hugepage_subpool_put_pages(spool,
								chg - add);
			hugetlb_acct_memory(h, -rsv_adjust);
7215 7216 7217 7218 7219 7220 7221 7222
		} else if (h_cg) {
			/*
			 * The file_regions will hold their own reference to
			 * h_cg->css. So we should release the reference held
			 * via hugetlb_cgroup_charge_cgroup_rsvd() when we are
			 * done.
			 */
			hugetlb_cgroup_put_rsvd_cgroup(h_cg);
7223 7224
		}
	}
7225 7226
	return true;

7227 7228 7229 7230 7231 7232
out_put_pages:
	/* put back original number of pages, chg */
	(void)hugepage_subpool_put_pages(spool, chg);
out_uncharge_cgroup:
	hugetlb_cgroup_uncharge_cgroup_rsvd(hstate_index(h),
					    chg * pages_per_huge_page(h), h_cg);
7233
out_err:
7234
	hugetlb_vma_lock_free(vma);
7235
	if (!vma || vma->vm_flags & VM_MAYSHARE)
7236 7237 7238 7239 7240
		/* Only call region_abort if the region_chg succeeded but the
		 * region_add failed or didn't run.
		 */
		if (chg >= 0 && add < 0)
			region_abort(resv_map, from, to, regions_needed);
7241
	if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
7242
		kref_put(&resv_map->refs, resv_map_release);
7243 7244
		set_vma_resv_map(vma, NULL);
	}
7245
	return false;
7246 7247
}

7248 7249
long hugetlb_unreserve_pages(struct inode *inode, long start, long end,
								long freed)
7250
{
7251
	struct hstate *h = hstate_inode(inode);
7252
	struct resv_map *resv_map = inode_resv_map(inode);
7253
	long chg = 0;
7254
	struct hugepage_subpool *spool = subpool_inode(inode);
7255
	long gbl_reserve;
Ken Chen's avatar
Ken Chen committed
7256

7257 7258 7259 7260
	/*
	 * Since this routine can be called in the evict inode path for all
	 * hugetlbfs inodes, resv_map could be NULL.
	 */
7261 7262 7263 7264 7265 7266 7267 7268 7269 7270 7271
	if (resv_map) {
		chg = region_del(resv_map, start, end);
		/*
		 * region_del() can fail in the rare case where a region
		 * must be split and another region descriptor can not be
		 * allocated.  If end == LONG_MAX, it will not fail.
		 */
		if (chg < 0)
			return chg;
	}

Ken Chen's avatar
Ken Chen committed
7272
	spin_lock(&inode->i_lock);
7273
	inode->i_blocks -= (blocks_per_huge_page(h) * freed);
Ken Chen's avatar
Ken Chen committed
7274 7275
	spin_unlock(&inode->i_lock);

7276 7277 7278
	/*
	 * If the subpool has a minimum size, the number of global
	 * reservations to be released may be adjusted.
7279 7280 7281
	 *
	 * Note that !resv_map implies freed == 0. So (chg - freed)
	 * won't go negative.
7282 7283 7284
	 */
	gbl_reserve = hugepage_subpool_put_pages(spool, (chg - freed));
	hugetlb_acct_memory(h, -gbl_reserve);
7285 7286

	return 0;
7287
}
7288

7289 7290 7291 7292 7293 7294 7295 7296 7297 7298 7299
#ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
static unsigned long page_table_shareable(struct vm_area_struct *svma,
				struct vm_area_struct *vma,
				unsigned long addr, pgoff_t idx)
{
	unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) +
				svma->vm_start;
	unsigned long sbase = saddr & PUD_MASK;
	unsigned long s_end = sbase + PUD_SIZE;

	/* Allow segments to share if only one is marked locked */
7300 7301
	unsigned long vm_flags = vma->vm_flags & ~VM_LOCKED_MASK;
	unsigned long svm_flags = svma->vm_flags & ~VM_LOCKED_MASK;
7302 7303 7304 7305

	/*
	 * match the virtual addresses, permission and the alignment of the
	 * page table page.
7306 7307
	 *
	 * Also, vma_lock (vm_private_data) is required for sharing.
7308 7309 7310
	 */
	if (pmd_index(addr) != pmd_index(saddr) ||
	    vm_flags != svm_flags ||
7311 7312
	    !range_in_vma(svma, sbase, s_end) ||
	    !svma->vm_private_data)
7313 7314 7315 7316 7317
		return 0;

	return saddr;
}

7318
bool want_pmd_share(struct vm_area_struct *vma, unsigned long addr)
7319
{
7320 7321 7322
	unsigned long start = addr & PUD_MASK;
	unsigned long end = start + PUD_SIZE;

7323 7324 7325 7326
#ifdef CONFIG_USERFAULTFD
	if (uffd_disable_huge_pmd_share(vma))
		return false;
#endif
7327 7328 7329
	/*
	 * check on proper vm_flags and page table alignment
	 */
7330 7331
	if (!(vma->vm_flags & VM_MAYSHARE))
		return false;
7332
	if (!vma->vm_private_data)	/* vma lock required for sharing */
7333 7334 7335 7336 7337 7338
		return false;
	if (!range_in_vma(vma, start, end))
		return false;
	return true;
}

7339 7340 7341 7342 7343 7344 7345 7346
/*
 * Determine if start,end range within vma could be mapped by shared pmd.
 * If yes, adjust start and end to cover range associated with possible
 * shared pmd mappings.
 */
void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
				unsigned long *start, unsigned long *end)
{
7347 7348
	unsigned long v_start = ALIGN(vma->vm_start, PUD_SIZE),
		v_end = ALIGN_DOWN(vma->vm_end, PUD_SIZE);
7349

7350
	/*
Ingo Molnar's avatar
Ingo Molnar committed
7351 7352
	 * vma needs to span at least one aligned PUD size, and the range
	 * must be at least partially within in.
7353 7354 7355
	 */
	if (!(vma->vm_flags & VM_MAYSHARE) || !(v_end > v_start) ||
		(*end <= v_start) || (*start >= v_end))
7356 7357
		return;

7358
	/* Extend the range to be PUD aligned for a worst case scenario */
7359 7360
	if (*start > v_start)
		*start = ALIGN_DOWN(*start, PUD_SIZE);
7361

7362 7363
	if (*end < v_end)
		*end = ALIGN(*end, PUD_SIZE);
7364 7365
}

7366 7367 7368 7369
/*
 * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc()
 * and returns the corresponding pte. While this is not necessary for the
 * !shared pmd case because we can allocate the pmd later as well, it makes the
7370 7371 7372 7373
 * code much cleaner. pmd allocation is essential for the shared case because
 * pud has to be populated inside the same i_mmap_rwsem section - otherwise
 * racing tasks could either miss the sharing (see huge_pte_offset) or select a
 * bad pmd for sharing.
7374
 */
7375 7376
pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,
		      unsigned long addr, pud_t *pud)
7377 7378 7379 7380 7381 7382 7383 7384 7385
{
	struct address_space *mapping = vma->vm_file->f_mapping;
	pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) +
			vma->vm_pgoff;
	struct vm_area_struct *svma;
	unsigned long saddr;
	pte_t *spte = NULL;
	pte_t *pte;

7386
	i_mmap_lock_read(mapping);
7387 7388 7389 7390 7391 7392
	vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) {
		if (svma == vma)
			continue;

		saddr = page_table_shareable(svma, vma, addr, idx);
		if (saddr) {
7393 7394
			spte = hugetlb_walk(svma, saddr,
					    vma_mmu_pagesize(svma));
7395 7396 7397 7398 7399 7400 7401 7402 7403 7404
			if (spte) {
				get_page(virt_to_page(spte));
				break;
			}
		}
	}

	if (!spte)
		goto out;

7405
	spin_lock(&mm->page_table_lock);
7406
	if (pud_none(*pud)) {
7407 7408
		pud_populate(mm, pud,
				(pmd_t *)((unsigned long)spte & PAGE_MASK));
7409
		mm_inc_nr_pmds(mm);
7410
	} else {
7411
		put_page(virt_to_page(spte));
7412
	}
7413
	spin_unlock(&mm->page_table_lock);
7414 7415
out:
	pte = (pte_t *)pmd_alloc(mm, pud, addr);
7416
	i_mmap_unlock_read(mapping);
7417 7418 7419 7420 7421 7422 7423 7424 7425 7426
	return pte;
}

/*
 * unmap huge page backed by shared pte.
 *
 * Hugetlb pte page is ref counted at the time of mapping.  If pte is shared
 * indicated by page_count > 1, unmap is achieved by clearing pud and
 * decrementing the ref count. If count == 1, the pte page is not shared.
 *
7427
 * Called with page table lock held.
7428 7429 7430 7431
 *
 * returns: 1 successfully unmapped a shared pte page
 *	    0 the underlying pte page is not shared, or it is the last user
 */
7432
int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma,
7433
					unsigned long addr, pte_t *ptep)
7434
{
7435 7436 7437
	pgd_t *pgd = pgd_offset(mm, addr);
	p4d_t *p4d = p4d_offset(pgd, addr);
	pud_t *pud = pud_offset(p4d, addr);
7438

7439
	i_mmap_assert_write_locked(vma->vm_file->f_mapping);
7440
	hugetlb_vma_assert_locked(vma);
7441 7442 7443 7444 7445 7446
	BUG_ON(page_count(virt_to_page(ptep)) == 0);
	if (page_count(virt_to_page(ptep)) == 1)
		return 0;

	pud_clear(pud);
	put_page(virt_to_page(ptep));
7447
	mm_dec_nr_pmds(mm);
7448 7449
	return 1;
}
7450

7451
#else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
7452

7453 7454
pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,
		      unsigned long addr, pud_t *pud)
7455 7456 7457
{
	return NULL;
}
7458

7459
int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma,
7460
				unsigned long addr, pte_t *ptep)
7461 7462 7463
{
	return 0;
}
7464 7465 7466 7467 7468

void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
				unsigned long *start, unsigned long *end)
{
}
7469 7470 7471 7472 7473

bool want_pmd_share(struct vm_area_struct *vma, unsigned long addr)
{
	return false;
}
7474 7475
#endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */

7476
#ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB
7477
pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
7478 7479 7480
			unsigned long addr, unsigned long sz)
{
	pgd_t *pgd;
7481
	p4d_t *p4d;
7482 7483 7484 7485
	pud_t *pud;
	pte_t *pte = NULL;

	pgd = pgd_offset(mm, addr);
7486 7487 7488
	p4d = p4d_alloc(mm, pgd, addr);
	if (!p4d)
		return NULL;
7489
	pud = pud_alloc(mm, p4d, addr);
7490 7491 7492 7493 7494
	if (pud) {
		if (sz == PUD_SIZE) {
			pte = (pte_t *)pud;
		} else {
			BUG_ON(sz != PMD_SIZE);
7495
			if (want_pmd_share(vma, addr) && pud_none(*pud))
7496
				pte = huge_pmd_share(mm, vma, addr, pud);
7497 7498 7499 7500
			else
				pte = (pte_t *)pmd_alloc(mm, pud, addr);
		}
	}
7501 7502 7503 7504 7505 7506

	if (pte) {
		pte_t pteval = ptep_get_lockless(pte);

		BUG_ON(pte_present(pteval) && !pte_huge(pteval));
	}
7507 7508 7509 7510

	return pte;
}

7511 7512 7513 7514
/*
 * huge_pte_offset() - Walk the page table to resolve the hugepage
 * entry at address @addr
 *
7515 7516
 * Return: Pointer to page table entry (PUD or PMD) for
 * address @addr, or NULL if a !p*d_present() entry is encountered and the
7517 7518 7519
 * size @sz doesn't match the hugepage size at this level of the page
 * table.
 */
7520 7521
pte_t *huge_pte_offset(struct mm_struct *mm,
		       unsigned long addr, unsigned long sz)
7522 7523
{
	pgd_t *pgd;
7524
	p4d_t *p4d;
7525 7526
	pud_t *pud;
	pmd_t *pmd;
7527 7528

	pgd = pgd_offset(mm, addr);
7529 7530 7531 7532 7533
	if (!pgd_present(*pgd))
		return NULL;
	p4d = p4d_offset(pgd, addr);
	if (!p4d_present(*p4d))
		return NULL;
7534

7535
	pud = pud_offset(p4d, addr);
7536 7537
	if (sz == PUD_SIZE)
		/* must be pud huge, non-present or none */
7538
		return (pte_t *)pud;
7539
	if (!pud_present(*pud))
7540
		return NULL;
7541
	/* must have a valid entry and size to go further */
7542

7543 7544 7545
	pmd = pmd_offset(pud, addr);
	/* must be pmd huge, non-present or none */
	return (pte_t *)pmd;
7546 7547
}

7548 7549 7550 7551 7552 7553 7554 7555 7556 7557 7558 7559 7560 7561 7562 7563 7564 7565 7566 7567 7568 7569 7570 7571
/*
 * Return a mask that can be used to update an address to the last huge
 * page in a page table page mapping size.  Used to skip non-present
 * page table entries when linearly scanning address ranges.  Architectures
 * with unique huge page to page table relationships can define their own
 * version of this routine.
 */
unsigned long hugetlb_mask_last_page(struct hstate *h)
{
	unsigned long hp_size = huge_page_size(h);

	if (hp_size == PUD_SIZE)
		return P4D_SIZE - PUD_SIZE;
	else if (hp_size == PMD_SIZE)
		return PUD_SIZE - PMD_SIZE;
	else
		return 0UL;
}

#else

/* See description above.  Architectures can provide their own version. */
__weak unsigned long hugetlb_mask_last_page(struct hstate *h)
{
7572 7573 7574 7575
#ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
	if (huge_page_size(h) == PMD_SIZE)
		return PUD_SIZE - PMD_SIZE;
#endif
7576 7577 7578
	return 0UL;
}

7579 7580 7581 7582 7583 7584
#endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */

/*
 * These functions are overwritable if your architecture needs its own
 * behavior.
 */
7585
bool isolate_hugetlb(struct folio *folio, struct list_head *list)
7586
{
7587
	bool ret = true;
7588

7589
	spin_lock_irq(&hugetlb_lock);
7590 7591 7592
	if (!folio_test_hugetlb(folio) ||
	    !folio_test_hugetlb_migratable(folio) ||
	    !folio_try_get(folio)) {
7593
		ret = false;
7594 7595
		goto unlock;
	}
7596 7597
	folio_clear_hugetlb_migratable(folio);
	list_move_tail(&folio->lru, list);
7598
unlock:
7599
	spin_unlock_irq(&hugetlb_lock);
7600
	return ret;
7601 7602
}

7603
int get_hwpoison_hugetlb_folio(struct folio *folio, bool *hugetlb, bool unpoison)
7604 7605 7606 7607 7608
{
	int ret = 0;

	*hugetlb = false;
	spin_lock_irq(&hugetlb_lock);
7609
	if (folio_test_hugetlb(folio)) {
7610
		*hugetlb = true;
7611
		if (folio_test_hugetlb_freed(folio))
7612
			ret = 0;
7613 7614
		else if (folio_test_hugetlb_migratable(folio) || unpoison)
			ret = folio_try_get(folio);
7615 7616
		else
			ret = -EBUSY;
7617 7618 7619 7620 7621
	}
	spin_unlock_irq(&hugetlb_lock);
	return ret;
}

7622 7623
int get_huge_page_for_hwpoison(unsigned long pfn, int flags,
				bool *migratable_cleared)
7624 7625 7626 7627
{
	int ret;

	spin_lock_irq(&hugetlb_lock);
7628
	ret = __get_huge_page_for_hwpoison(pfn, flags, migratable_cleared);
7629 7630 7631 7632
	spin_unlock_irq(&hugetlb_lock);
	return ret;
}

7633
void folio_putback_active_hugetlb(struct folio *folio)
7634
{
7635
	spin_lock_irq(&hugetlb_lock);
7636 7637
	folio_set_hugetlb_migratable(folio);
	list_move_tail(&folio->lru, &(folio_hstate(folio))->hugepage_activelist);
7638
	spin_unlock_irq(&hugetlb_lock);
7639
	folio_put(folio);
7640
}
7641

7642
void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, int reason)
7643
{
7644
	struct hstate *h = folio_hstate(old_folio);
7645

7646 7647
	hugetlb_cgroup_migrate(old_folio, new_folio);
	set_page_owner_migrate_reason(&new_folio->page, reason);
7648 7649

	/*
7650
	 * transfer temporary state of the new hugetlb folio. This is
7651 7652 7653 7654 7655 7656 7657 7658
	 * reverse to other transitions because the newpage is going to
	 * be final while the old one will be freed so it takes over
	 * the temporary status.
	 *
	 * Also note that we have to transfer the per-node surplus state
	 * here as well otherwise the global surplus count will not match
	 * the per-node's.
	 */
7659 7660 7661 7662 7663 7664
	if (folio_test_hugetlb_temporary(new_folio)) {
		int old_nid = folio_nid(old_folio);
		int new_nid = folio_nid(new_folio);

		folio_set_hugetlb_temporary(old_folio);
		folio_clear_hugetlb_temporary(new_folio);
7665 7666


7667 7668 7669 7670 7671 7672
		/*
		 * There is no need to transfer the per-node surplus state
		 * when we do not cross the node.
		 */
		if (new_nid == old_nid)
			return;
7673
		spin_lock_irq(&hugetlb_lock);
7674 7675 7676 7677
		if (h->surplus_huge_pages_node[old_nid]) {
			h->surplus_huge_pages_node[old_nid]--;
			h->surplus_huge_pages_node[new_nid]++;
		}
7678
		spin_unlock_irq(&hugetlb_lock);
7679 7680
	}
}
7681

7682 7683 7684
static void hugetlb_unshare_pmds(struct vm_area_struct *vma,
				   unsigned long start,
				   unsigned long end)
7685 7686 7687 7688 7689
{
	struct hstate *h = hstate_vma(vma);
	unsigned long sz = huge_page_size(h);
	struct mm_struct *mm = vma->vm_mm;
	struct mmu_notifier_range range;
7690
	unsigned long address;
7691 7692 7693 7694 7695 7696 7697 7698 7699
	spinlock_t *ptl;
	pte_t *ptep;

	if (!(vma->vm_flags & VM_MAYSHARE))
		return;

	if (start >= end)
		return;

7700
	flush_cache_range(vma, start, end);
7701 7702 7703 7704
	/*
	 * No need to call adjust_range_if_pmd_sharing_possible(), because
	 * we have already done the PUD_SIZE alignment.
	 */
7705
	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm,
7706 7707
				start, end);
	mmu_notifier_invalidate_range_start(&range);
7708
	hugetlb_vma_lock_write(vma);
7709 7710
	i_mmap_lock_write(vma->vm_file->f_mapping);
	for (address = start; address < end; address += PUD_SIZE) {
7711
		ptep = hugetlb_walk(vma, address, sz);
7712 7713 7714
		if (!ptep)
			continue;
		ptl = huge_pte_lock(h, mm, ptep);
7715
		huge_pmd_unshare(mm, vma, address, ptep);
7716 7717 7718 7719
		spin_unlock(ptl);
	}
	flush_hugetlb_tlb_range(vma, start, end);
	i_mmap_unlock_write(vma->vm_file->f_mapping);
7720
	hugetlb_vma_unlock_write(vma);
7721
	/*
7722
	 * No need to call mmu_notifier_arch_invalidate_secondary_tlbs(), see
7723
	 * Documentation/mm/mmu_notifier.rst.
7724 7725 7726 7727
	 */
	mmu_notifier_invalidate_range_end(&range);
}

7728 7729 7730 7731 7732 7733 7734 7735 7736 7737
/*
 * This function will unconditionally remove all the shared pmd pgtable entries
 * within the specific vma for a hugetlbfs memory range.
 */
void hugetlb_unshare_all_pmds(struct vm_area_struct *vma)
{
	hugetlb_unshare_pmds(vma, ALIGN(vma->vm_start, PUD_SIZE),
			ALIGN_DOWN(vma->vm_end, PUD_SIZE));
}

7738 7739 7740 7741 7742
#ifdef CONFIG_CMA
static bool cma_reserve_called __initdata;

static int __init cmdline_parse_hugetlb_cma(char *p)
{
7743 7744 7745 7746 7747 7748 7749 7750 7751
	int nid, count = 0;
	unsigned long tmp;
	char *s = p;

	while (*s) {
		if (sscanf(s, "%lu%n", &tmp, &count) != 1)
			break;

		if (s[count] == ':') {
7752
			if (tmp >= MAX_NUMNODES)
7753
				break;
7754
			nid = array_index_nospec(tmp, MAX_NUMNODES);
7755 7756 7757 7758 7759 7760 7761 7762 7763 7764 7765 7766 7767 7768 7769 7770 7771 7772 7773 7774

			s += count + 1;
			tmp = memparse(s, &s);
			hugetlb_cma_size_in_node[nid] = tmp;
			hugetlb_cma_size += tmp;

			/*
			 * Skip the separator if have one, otherwise
			 * break the parsing.
			 */
			if (*s == ',')
				s++;
			else
				break;
		} else {
			hugetlb_cma_size = memparse(p, &p);
			break;
		}
	}

7775 7776 7777 7778 7779 7780 7781 7782
	return 0;
}

early_param("hugetlb_cma", cmdline_parse_hugetlb_cma);

void __init hugetlb_cma_reserve(int order)
{
	unsigned long size, reserved, per_node;
7783
	bool node_specific_cma_alloc = false;
7784 7785
	int nid;

7786 7787 7788 7789 7790 7791 7792
	/*
	 * HugeTLB CMA reservation is required for gigantic
	 * huge pages which could not be allocated via the
	 * page allocator. Just warn if there is any change
	 * breaking this assumption.
	 */
	VM_WARN_ON(order <= MAX_PAGE_ORDER);
7793 7794
	cma_reserve_called = true;

7795 7796 7797 7798 7799 7800 7801
	if (!hugetlb_cma_size)
		return;

	for (nid = 0; nid < MAX_NUMNODES; nid++) {
		if (hugetlb_cma_size_in_node[nid] == 0)
			continue;

7802
		if (!node_online(nid)) {
7803 7804 7805 7806 7807 7808 7809 7810 7811 7812 7813 7814 7815 7816 7817 7818 7819
			pr_warn("hugetlb_cma: invalid node %d specified\n", nid);
			hugetlb_cma_size -= hugetlb_cma_size_in_node[nid];
			hugetlb_cma_size_in_node[nid] = 0;
			continue;
		}

		if (hugetlb_cma_size_in_node[nid] < (PAGE_SIZE << order)) {
			pr_warn("hugetlb_cma: cma area of node %d should be at least %lu MiB\n",
				nid, (PAGE_SIZE << order) / SZ_1M);
			hugetlb_cma_size -= hugetlb_cma_size_in_node[nid];
			hugetlb_cma_size_in_node[nid] = 0;
		} else {
			node_specific_cma_alloc = true;
		}
	}

	/* Validate the CMA size again in case some invalid nodes specified. */
7820 7821 7822 7823 7824 7825
	if (!hugetlb_cma_size)
		return;

	if (hugetlb_cma_size < (PAGE_SIZE << order)) {
		pr_warn("hugetlb_cma: cma area should be at least %lu MiB\n",
			(PAGE_SIZE << order) / SZ_1M);
7826
		hugetlb_cma_size = 0;
7827 7828 7829
		return;
	}

7830 7831 7832 7833 7834 7835 7836 7837 7838
	if (!node_specific_cma_alloc) {
		/*
		 * If 3 GB area is requested on a machine with 4 numa nodes,
		 * let's allocate 1 GB on first three nodes and ignore the last one.
		 */
		per_node = DIV_ROUND_UP(hugetlb_cma_size, nr_online_nodes);
		pr_info("hugetlb_cma: reserve %lu MiB, up to %lu MiB per node\n",
			hugetlb_cma_size / SZ_1M, per_node / SZ_1M);
	}
7839 7840

	reserved = 0;
7841
	for_each_online_node(nid) {
7842
		int res;
7843
		char name[CMA_MAX_NAME];
7844

7845 7846 7847 7848 7849 7850 7851 7852 7853
		if (node_specific_cma_alloc) {
			if (hugetlb_cma_size_in_node[nid] == 0)
				continue;

			size = hugetlb_cma_size_in_node[nid];
		} else {
			size = min(per_node, hugetlb_cma_size - reserved);
		}

7854 7855
		size = round_up(size, PAGE_SIZE << order);

7856
		snprintf(name, sizeof(name), "hugetlb%d", nid);
7857 7858 7859 7860 7861 7862 7863
		/*
		 * Note that 'order per bit' is based on smallest size that
		 * may be returned to CMA allocator in the case of
		 * huge page demotion.
		 */
		res = cma_declare_contiguous_nid(0, size, 0,
						PAGE_SIZE << HUGETLB_PAGE_ORDER,
7864
						 0, false, name,
7865 7866 7867 7868 7869 7870 7871 7872 7873 7874 7875 7876 7877 7878
						 &hugetlb_cma[nid], nid);
		if (res) {
			pr_warn("hugetlb_cma: reservation failed: err %d, node %d",
				res, nid);
			continue;
		}

		reserved += size;
		pr_info("hugetlb_cma: reserved %lu MiB on node %d\n",
			size / SZ_1M, nid);

		if (reserved >= hugetlb_cma_size)
			break;
	}
7879 7880 7881 7882 7883 7884 7885

	if (!reserved)
		/*
		 * hugetlb_cma_size is used to determine if allocations from
		 * cma are possible.  Set to zero if no cma regions are set up.
		 */
		hugetlb_cma_size = 0;
7886 7887
}

7888
static void __init hugetlb_cma_check(void)
7889 7890 7891 7892 7893 7894 7895 7896
{
	if (!hugetlb_cma_size || cma_reserve_called)
		return;

	pr_warn("hugetlb_cma: the option isn't supported by current arch\n");
}

#endif /* CONFIG_CMA */